def _update_settings_of_user(
path: str,
tar_file: TarFile,
user_tars: List[TarInfo],
user: str,
local_users: Dict[str, Optional[str]],
) -> None:
"""Update files within user directory
A user can be split in two tiers.
Customer-Users belong to a customer when working on the CME. They only
work on the GUI of their corresponding remote site. They are allowed to
customize their bookmarks, views, dashboards, reports, etc. These user
local configurations are retained when receiving files from master as
changes are activated.
This means all "user_*" files are retained during sync.
Non-customer-users (e.g. GLOBAL users) normally work on the central
site and thus they should be able to use their customizations when they
log into remote sites. Thus all files are synced in their case.
No backup of the remote site dir happens during sync, data is removed,
added, skipped in place to avoid collisions."""
is_customer_user = local_users.get(user) is not None
_cleanup_user_dir(path, is_customer_user)
if is_customer_user:
user_tars = [m for m in user_tars if not is_user_file(m.name)]
tar_file.extractall(os.path.dirname(path), members=user_tars)
def install_repo(self, repo):
if repo in KNOWN_PUBLIC_REPOS:
repo = KNOWN_PUBLIC_REPOS[repo]['path'] # replace it by the url
git_path = which('git')
if not git_path:
return ('git command not found: You need to have git installed on '
'your system to be able to install git based plugins.', )
# TODO: Update download path of plugin.
if repo.endswith('tar.gz'):
tar = TarFile(fileobj=urlopen(repo))
tar.extractall(path=self.plugin_dir)
s = repo.split(':')[-1].split('/')[-2:]
human_name = '/'.join(s).rstrip('.tar.gz')
else:
human_name = human_name_for_git_url(repo)
p = subprocess.Popen([git_path, 'clone', repo, human_name], cwd=self.plugin_dir, stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
feedback = p.stdout.read().decode('utf-8')
error_feedback = p.stderr.read().decode('utf-8')
if p.wait():
return "Could not load this plugin: \n\n%s\n\n---\n\n%s" % (feedback, error_feedback),
self.add_plugin_repo(human_name, repo)
return self.update_dynamic_plugins()
def install(self, mess, args):
""" install a plugin repository from the given source or a known public repo (see !repos to find those).
for example from a known repo : !install err-codebot
for example a git url : [email protected]:gbin/plugin.git
or an url towards a tar.gz archive : http://www.gootz.net/plugin-latest.tar.gz
"""
if not args.strip():
return "You should have an urls/git repo argument"
if args in KNOWN_PUBLIC_REPOS:
args = KNOWN_PUBLIC_REPOS[args][0] # replace it by the url
git_path = which('git')
if not git_path:
return 'git command not found: You need to have git installed on your system to by able to install git based plugins.'
if args.endswith('tar.gz'):
tar = TarFile(fileobj=urlopen(args))
tar.extractall(path= PLUGIN_DIR)
human_name = args.split('/')[-1][:-7]
else:
human_name = human_name_for_git_url(args)
p = subprocess.Popen([git_path, 'clone', args, human_name], cwd = PLUGIN_DIR, stdout = subprocess.PIPE, stderr = subprocess.PIPE)
feedback = p.stdout.read()
error_feedback = p.stderr.read()
if p.wait():
return "Could not load this plugin : \n%s\n---\n%s" % (feedback, error_feedback)
self.add_plugin_repo(human_name, args)
errors = self.update_dynamic_plugins()
if errors:
self.send(mess.getFrom(), 'Some plugins are generating errors:\n' + '\n'.join(errors) , message_type=mess.getType())
else:
self.send(mess.getFrom(), "A new plugin repository named %s has been installed correctly from %s. Refreshing the plugins commands..." % (human_name, args), message_type=mess.getType())
self.activate_non_started_plugins()
return "Plugin reload done."
def repos_install(self, mess, args):
""" install a plugin repository from the given source or a known public repo (see !repos to find those).
for example from a known repo : !install err-codebot
for example a git url : [email protected]:gbin/plugin.git
or an url towards a tar.gz archive : http://www.gootz.net/plugin-latest.tar.gz
"""
if not args.strip():
return "You should have an urls/git repo argument"
if args in KNOWN_PUBLIC_REPOS:
args = KNOWN_PUBLIC_REPOS[args][0] # replace it by the url
git_path = which('git')
if not git_path:
return 'git command not found: You need to have git installed on your system to by able to install git based plugins.'
if args.endswith('tar.gz'):
tar = TarFile(fileobj=urlopen(args))
tar.extractall(path=self.plugin_dir)
human_name = args.split('/')[-1][:-7]
else:
human_name = human_name_for_git_url(args)
p = subprocess.Popen([git_path, 'clone', args, human_name], cwd=self.plugin_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
feedback = p.stdout.read().decode('utf-8')
error_feedback = p.stderr.read().decode('utf-8')
if p.wait():
return "Could not load this plugin : \n%s\n---\n%s" % (feedback, error_feedback)
self.add_plugin_repo(human_name, args)
errors = self.update_dynamic_plugins()
if errors:
self.send(mess.getFrom(), 'Some plugins are generating errors:\n' + '\n'.join(errors), message_type=mess.getType())
else:
self.send(mess.getFrom(), "A new plugin repository named %s has been installed correctly from %s. Refreshing the plugins commands..." % (human_name, args), message_type=mess.getType())
self.activate_non_started_plugins()
return "Plugin reload done."
def getDataFromTarfile(tarfile):
tf = TarFile(tarfile)
members = [m.name for m in tf.getmembers()]
if list(set([os.path.exists(x) for x in members])) != [True]:
tf.extractall()
tf.close
return members
def _get_checkpoint_dir():
from appdirs import AppDirs
dirs = AppDirs(appname="nimare", appauthor="neurostuff", version="1.0")
checkpoint_dir = os.path.join(dirs.user_data_dir, "ohbm2018_model")
if not os.path.exists(checkpoint_dir):
LGR.info('Downloading the model (this is a one-off operation)...')
url = "https://zenodo.org/record/1257721/files/ohbm2018_model.tar.xz?download=1"
# Streaming, so we can iterate over the response.
r = requests.get(url, stream=True)
f = BytesIO()
# Total size in bytes.
total_size = int(r.headers.get('content-length', 0))
block_size = 1024 * 1024
wrote = 0
for data in tqdm(r.iter_content(block_size), total=math.ceil(total_size // block_size),
unit='MB', unit_scale=True):
wrote = wrote + len(data)
f.write(data)
if total_size != 0 and wrote != total_size:
raise Exception("Download interrupted")
f.seek(0)
LGR.info('Uncompressing the model to %s...'.format(checkpoint_dir))
tarfile = TarFile(fileobj=LZMAFile(f), mode="r")
tarfile.extractall(dirs.user_data_dir)
return checkpoint_dir
def install_repo(self, repo):
if repo in KNOWN_PUBLIC_REPOS:
repo = KNOWN_PUBLIC_REPOS[repo]['path'] # replace it by the url
git_path = which('git')
if not git_path:
return ('git command not found: You need to have git installed on '
'your system to be able to install git based plugins.', )
# TODO: Update download path of plugin.
if repo.endswith('tar.gz'):
tar = TarFile(fileobj=urlopen(repo))
tar.extractall(path=self.plugin_dir)
s = repo.split(':')[-1].split('/')[-2:]
human_name = '/'.join(s).rstrip('.tar.gz')
else:
human_name = human_name_for_git_url(repo)
p = subprocess.Popen([git_path, 'clone', repo, human_name],
cwd=self.plugin_dir,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
feedback = p.stdout.read().decode('utf-8')
error_feedback = p.stderr.read().decode('utf-8')
if p.wait():
return "Could not load this plugin: \n\n%s\n\n---\n\n%s" % (
feedback, error_feedback),
self.add_plugin_repo(human_name, repo)
return self.update_dynamic_plugins()
def analyze():
try:
fn = 'temp/{}.tar'.format(
md5(request.remote_addr.encode()).hexdigest())
if request.method == 'POST':
fp = request.files['file']
fp.save(fn)
if not is_tarfile(fn):
return '<script>alert("Uploaded file is not \'tar\' file.");history.back(-1);</script>'
tf = TarFile(fn)
tf.extractall(fn.split('.')[0])
bd1 = fn.split('/')[1].split('.')[0]
bd2 = fn.split('/')[1]
return render_template('analyze',
path=bd1,
fn=bd1,
files=tf.getnames())
except Exception as e:
return response('Error', 500)
finally:
try:
os.remove(fn)
except:
return response('Error', 500)
def _prepare_dataset(cls):
tar = TarFile(cls.dataset_tar_local)
tar.extractall(cls.dataset_dir)
# Get rid of non-image files
for f in os.listdir(cls.dataset_dir):
if f[-3:] != 'jpg':
os.remove(os.path.join(cls.dataset_dir, f))
images = {
'up': [],
'down': os.listdir(cls.dataset_dir),
'left': [],
'right': []
}
for rot_deg, orientation in zip((90, 180, 270),
('left', 'up', 'right')):
for img_file in images['down']:
img = cv2.imread(os.path.join(cls.dataset_dir, img_file))
new_img_file = orientation + '_' + img_file
cv2.imwrite(os.path.join(cls.dataset_dir, new_img_file),
rotate_image(img, rot_deg))
images[orientation].append(new_img_file)
return images
def _extract_and_upload(archive: tarfile.TarFile, dry_run: bool) -> None:
archive.extractall(members=strip_member_components(archive, 1))
# TODO(cyrille): Also drop Showcase demos.
subprocess.run('rm -r *Demo', shell=True, check=True)
all_files = glob.glob('*', recursive=True)
if dry_run:
logging.info('Client assets:\n%s', '\n'.join(all_files))
else:
subprocess.run(
('swift', 'upload', _OPEN_STACK_CONTAINER, '--skip-identical', *all_files), check=True)
subprocess.run(
('aws', 's3', 'cp', os.getcwd(), f's3://{_S3_BUCKET}/', '--recursive'), check=True)
def uncompress_groupby_to_df(self,
result_tar,
groupby_col_list,
agg_list,
where_terms_list,
aggregate=False):
# uncompress result returned by the groupby and convert it to a Pandas DataFrame
tmp_dir = None
try:
try:
tar_file = TarFile(fileobj=StringIO(result_tar))
tmp_dir = tempfile.mkdtemp(prefix='tar_dir_')
tar_file.extractall(tmp_dir)
except TarError:
self.logger.exception("Could not create/extract tar.")
raise ValueError(result_tar)
del result_tar
del tar_file
ct = None
# now untar and aggregate the individual shard results
for i, sub_tar in enumerate(glob.glob(os.path.join(tmp_dir, '*'))):
new_dir = os.path.join(tmp_dir, 'bcolz_' + str(i))
rm_file_or_dir(new_dir)
with tarfile.open(sub_tar, mode='r') as tar_file:
tar_file.extractall(new_dir)
# rm_file_or_dir(sub_tar)
ctable_dir = glob.glob(os.path.join(new_dir, '*'))[0]
new_ct = ctable(rootdir=ctable_dir, mode='a')
if i == 0:
ct = new_ct
else:
ct.append(new_ct)
# aggregate by groupby parameters
if ct is None:
result_df = pd.DataFrame()
elif aggregate:
new_dir = os.path.join(tmp_dir, 'end_result')
rm_file_or_dir(new_dir)
# we can only sum now
new_agg_list = [[x[2], 'sum', x[2]] for x in agg_list]
result_ctable = ct.groupby(groupby_col_list,
new_agg_list,
rootdir=new_dir)
result_df = result_ctable.todataframe()
else:
result_df = ct.todataframe()
finally:
rm_file_or_dir(tmp_dir)
return result_df
def download(self, src, dest, extract_here=False):
client = connect()
with SpooledTemporaryFile() as file:
file.write(client.copy(self.container_id, src).read())
file.seek(0)
tfile = TarFile(fileobj=file)
if extract_here:
base = len(os.path.basename(src)) + 1
for member in tfile.getmembers():
member.name = member.name[base:]
tfile.extractall(path=dest)
def install_repo(self, repo):
"""
Install the repository from repo
:param repo:
The url, git url or path on disk of a repository. It can point to either a git repo or
a .tar.gz of a plugin
:returns:
The path on disk where the repo has been installed on.
:raises: :class:`~RepoException` if an error occured.
"""
self.check_for_index_update()
# try to find if we have something with that name in our index
if repo in self[REPO_INDEX]:
human_name = repo
repo_url = next(iter(self[REPO_INDEX][repo].values()))['repo']
else:
# This is a repo url, make up a plugin definition for it
# TODO read the definition if possible.
human_name = human_name_for_git_url(repo)
repo_url = repo
git_path = which('git')
if not git_path:
raise RepoException(
'git command not found: You need to have git installed on '
'your system to be able to install git based plugins.', )
# TODO: Update download path of plugin.
if repo_url.endswith('tar.gz'):
tar = TarFile(fileobj=urllib.urlopen(repo_url))
tar.extractall(path=self.plugin_dir)
s = repo_url.split(':')[-1].split('/')[-2:]
human_name = human_name or '/'.join(s).rstrip('.tar.gz')
else:
human_name = human_name or human_name_for_git_url(repo_url)
p = subprocess.Popen([git_path, 'clone', repo_url, human_name],
cwd=self.plugin_dir,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
feedback = p.stdout.read().decode('utf-8')
error_feedback = p.stderr.read().decode('utf-8')
if p.wait():
raise RepoException(
"Could not load this plugin: \n\n%s\n\n---\n\n%s" %
(feedback, error_feedback))
self.add_plugin_repo(human_name, repo_url)
return os.path.join(self.plugin_dir, human_name)
def ensure_template(self,
layer_stack,
connection=None): # connection needs to be here
"""Create the base image for a container"""
# find how many of the intermediate snapshots we already have
last, success_count = self.find_furthest(layer_stack)
logging.debug("Delta tree creating with head start: " +
str(success_count))
# layer stack is a list of deltas in the order they were written in
layers_so_far = '/'.join(layer_stack[:success_count])
for layer in layer_stack[success_count:]:
logging.debug("Creating layer: " + layer)
# the / delimited list of sha256 deltas used to create this layer
layers_so_far += ('/' if len(layers_so_far) != 0 else '') + layer
# the next layer is a inherited (cloned) from a snapshot
dest_fs = 'tf/layer-' + shortuuid.uuid(
) # need a uuid for namespace collisions
call([
'zfs', 'clone', '-o', 'recordsize=8k', '-o', 'compression=on',
last.snapshot, dest_fs
],
stdout=DEVNULL)
# fetch via http then untar a pretend file
layer_http = requests.get('http://%s:1025/%s' %
(self.location_ip, layer))
with io.BytesIO(layer_http.content) as f:
tar = TarFile(fileobj=f)
tar.extractall('/' + dest_fs)
# create the snapshot and mark it so we know what it represents
call(['zfs', 'snapshot', '%s@final' % dest_fs], stdout=DEVNULL)
call([
'zfs', 'set', ':layer_stack=' + layers_so_far,
'%s@final' % dest_fs
])
# clean the mess up
call(['zfs', 'unmount', dest_fs], stdout=DEVNULL)
call(['rmdir', '/' + dest_fs])
# let the delta tree know
new_node = DtNode(dest_fs + "@final")
last.deltas[layer] = new_node
last = new_node
return last.snapshot
def writer(self):
"""Expect written bytes to be a tarball."""
result = BytesIO()
yield result
result.seek(0, 0)
try:
tarball = TarFile(fileobj=result, mode="r")
if self.path.exists():
self.path.remove()
self.path.createDirectory()
tarball.extractall(self.path.path)
except:
# This should really be dealt with, e.g. logged:
# https://clusterhq.atlassian.net/browse/FLOC-122
pass
def writer(self):
"""Expect written bytes to be a tarball."""
result = BytesIO()
yield result
result.seek(0, 0)
try:
tarball = TarFile(fileobj=result, mode="r")
if self.path.exists():
self.path.remove()
self.path.createDirectory()
tarball.extractall(self.path.path)
except:
# This should really be dealt with, e.g. logged:
# https://github.com/ClusterHQ/flocker/issues/122
pass
def writer(self):
"""Expect written bytes to be a tarball."""
result = BytesIO()
yield result
result.seek(0, 0)
try:
tarball = TarFile(fileobj=result, mode="r")
if self.path.exists():
self.path.remove()
self.path.createDirectory()
tarball.extractall(self.path.path)
except:
# This should really be dealt with, e.g. logged:
# https://clusterhq.atlassian.net/browse/FLOC-122
pass
def writer(self):
"""Expect written bytes to be a tarball."""
result = BytesIO()
yield result
result.seek(0, 0)
try:
tarball = TarFile(fileobj=result, mode="r")
if self.path.exists():
self.path.remove()
self.path.createDirectory()
tarball.extractall(self.path.path)
except:
# This should really be dealt with, e.g. logged:
# https://github.com/ClusterHQ/flocker/issues/122
pass
def __handle_file(self, plugin_file):
# Uncompress the file.
temp_dir = tempfile.gettempdir()
if is_zipfile(plugin_file):
compress_fd = zipfile.ZipFile(plugin_file, allowZip64=True)
compress_fd.extractall(path=temp_dir)
elif is_bz2file(plugin_file):
#first check if we can handle as tar.bz2 (big chances)
try:
compress_fd = TarFile(name=plugin_file, mode="r:bz2")
compress_fd.extractall(path=temp_dir)
except CompressionError:
print "Upz!, fail in compressed file handling, Retrying"
try:
compress_fd = bz2.BZ2File(plugin_file)
tmp_fd = tempfile.TemporaryFile(suffix=".tar",
prefix="ncmprs")
tmp_fd.file.write(compress_fd.read())
tmp_fd.file.flush()
tar_fd = TarFile.taropen(name=None, fileobj=tmp_fd)
tar_fd.extractall(path=temp_dir)
tar_fd.close()
tmp_fd.close()
except:
print "Upz!, fail in compressed file handling, Again! :("
return None
elif is_gzipfile(plugin_file):
#first check if we can handle as tar.gz (big chances)
try:
compress_fd = TarFile(name=plugin_file, mode="r:gz")
compress_fd.extractall(path=temp_dir)
except CompressionError:
print "Upz!, fail in compressed file handling, Retrying"
try:
compress_fd = gzip.GzipFile(plugin_file)
tmp_fd = tempfile.TemporaryFile(suffix=".tar",
prefix="ncmprs")
tmp_fd.file.write(compress_fd.read())
tmp_fd.file.flush()
tar_fd = TarFile.taropen(name=None, fileobj=tmp_fd)
tar_fd.extractall(path=temp_dir)
tar_fd.close()
tmp_fd.close()
except:
print "Upz!, fail in compressed file handling, Again! :("
return None
return self.__handle_dir(temp_dir)
def __handle_file(self, plugin_file):
# Uncompress the file.
temp_dir = tempfile.gettempdir()
if is_zipfile(plugin_file):
compress_fd = zipfile.ZipFile(plugin_file, allowZip64=True)
compress_fd.extractall(path=temp_dir)
elif is_bz2file(plugin_file):
#first check if we can handle as tar.bz2 (big chances)
try:
compress_fd = TarFile(name=plugin_file, mode="r:bz2")
compress_fd.extractall(path=temp_dir)
except CompressionError:
print "Upz!, fail in compressed file handling, Retrying"
try:
compress_fd = bz2.BZ2File(plugin_file)
tmp_fd = tempfile.TemporaryFile(suffix=".tar", prefix="ncmprs")
tmp_fd.file.write(compress_fd.read())
tmp_fd.file.flush()
tar_fd = TarFile.taropen(name=None, fileobj=tmp_fd)
tar_fd.extractall(path=temp_dir)
tar_fd.close()
tmp_fd.close()
except:
print "Upz!, fail in compressed file handling, Again! :("
return None
elif is_gzipfile(plugin_file):
#first check if we can handle as tar.gz (big chances)
try:
compress_fd = TarFile(name=plugin_file, mode="r:gz")
compress_fd.extractall(path=temp_dir)
except CompressionError:
print "Upz!, fail in compressed file handling, Retrying"
try:
compress_fd = gzip.GzipFile(plugin_file)
tmp_fd = tempfile.TemporaryFile(suffix=".tar", prefix="ncmprs")
tmp_fd.file.write(compress_fd.read())
tmp_fd.file.flush()
tar_fd = TarFile.taropen(name=None, fileobj=tmp_fd)
tar_fd.extractall(path=temp_dir)
tar_fd.close()
tmp_fd.close()
except:
print "Upz!, fail in compressed file handling, Again! :("
return None
return self.__handle_dir(temp_dir)
def install_repo(self, repo):
"""
Install the repository from repo
:param repo:
The url, git url or path on disk of a repository. It can point to either a git repo or
a .tar.gz of a plugin
:returns:
The path on disk where the repo has been installed on.
:raises: :class:`~RepoException` if an error occured.
"""
self.check_for_index_update()
# try to find if we have something with that name in our index
if repo in self[REPO_INDEX]:
human_name = repo
repo_url = next(iter(self[REPO_INDEX][repo].values()))['repo']
else:
# This is a repo url, make up a plugin definition for it
# TODO read the definition if possible.
human_name = human_name_for_git_url(repo)
repo_url = repo
git_path = which('git')
if not git_path:
raise RepoException('git command not found: You need to have git installed on '
'your system to be able to install git based plugins.', )
# TODO: Update download path of plugin.
if repo_url.endswith('tar.gz'):
tar = TarFile(fileobj=urllib.urlopen(repo_url))
tar.extractall(path=self.plugin_dir)
s = repo_url.split(':')[-1].split('/')[-2:]
human_name = human_name or '/'.join(s).rstrip('.tar.gz')
else:
human_name = human_name or human_name_for_git_url(repo_url)
p = subprocess.Popen([git_path, 'clone', repo_url, human_name], cwd=self.plugin_dir, stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
feedback = p.stdout.read().decode('utf-8')
error_feedback = p.stderr.read().decode('utf-8')
if p.wait():
raise RepoException("Could not load this plugin: \n\n%s\n\n---\n\n%s" % (feedback, error_feedback))
self.add_plugin_repo(human_name, repo_url)
return os.path.join(self.plugin_dir, human_name)
def set_up(self):
if self._repo_path.is_dir():
shutil.rmtree(self._repo_path)
archiver = TarFile(self._repo_path.with_suffix('.tar'))
archiver.extractall(self._repo_path.parent)
if self.test_parameters.work_directory.is_dir():
shutil.rmtree(self.test_parameters.work_directory)
if self._scheme.endswith('+git'):
# TODO: We would start the daemon here
raise NotImplementedError("Git protocol not supported")
elif self._scheme.endswith('+svn'):
command: List[str] = [
'svnserve', '--root',
str(self._repo_path), '-X', '--foreground'
]
self._server = subprocess.Popen(command)
def update(self, name, version, target):
shutil.rmtree(target)
os.mkdir(target)
try:
updateServer=rpyc.connect(updateServerName,18862)
validNames = config.readConfig("login.cfg")
password = validNames['admin']['password']
tar = updateServer.root.get(password, name, version)
s = StringIO(decompress(tar))
f = TarFile(mode='r', fileobj=s)
f.extractall(target)
print "updated",name
return True
except:
traceback.print_exc()
print "Failed to connet to the update server :("
return False
def extract_filesystem_bundle(docker_driver, container_id=None, image_name=None):
temporary_dir = tempfile.mkdtemp()
# Get and save filesystem bundle
if container_id is not None:
data = docker_driver.get_docker_client().export(container=container_id).data
name = container_id
else:
data = docker_driver.get_docker_client().get_image(image=image_name).data
name = image_name.replace('/', '_').replace(':', '_')
with open(temporary_dir + "/" + name + ".tar", "wb") as file:
file.write(data)
# Untar filesystem bundle
tarfile = TarFile(temporary_dir + "/" + name + ".tar")
tarfile.extractall(temporary_dir)
os.remove(temporary_dir + "/" + name + ".tar")
if image_name is not None:
layers = _get_layers_from_manifest(temporary_dir)
_untar_layers(temporary_dir, layers)
# Return
return temporary_dir
def train_x(self):
'''Iterates over training data.
Yields:
data: A numpy array for each training video.
'''
for name in self.train_names():
datafile = self.open(f'data/{name}.tar', 'rb')
extraction_dir = self.cache_dir / 'unpacked'
tar = TarFile(fileobj=datafile)
tar.extractall(extraction_dir)
data = sorted(x.name for x in tar)
data = (f'{extraction_dir}/{name}' for name in data)
data = (cv2.imread(name, cv2.IMREAD_GRAYSCALE) for name in data)
data = np.stack(data)
yield data
tar.close()
datafile.close()
def test_something(self):
tarfile = TarFile("ressources/20200414-090526_archive.tar")
tarfile.extractall(path="ressources")
tarfile.close()
sql = Path()
sql.joinpath("ressources")
for i in sql.rglob("logicaldoc.sql"):
self.assertTrue((str(i)).__contains__("ressources/home/cibo/src/backup"))
for i in sql.rglob("repository/index/"):
self.assertTrue((str(i)).__contains__("community/repository/index"))
for i in sql.rglob("repository/docs/"):
self.assertTrue((str(i)).__contains__("community/repository/docs"))
for i in sql.rglob("conf/context.properties"):
self.assertTrue((str(i)).__contains__("conf/context"))
ret = Path(i)
self.assertTrue((str(ret.parent)).endswith("conf"))
def optional_extract(self, output, tarname):
"""Extracts test repository data if needed
Checks whether directory exists or is older than archive.
"""
tarname = get_test_file(tarname)
if (not os.path.exists(output) or
os.path.getmtime(output) < os.path.getmtime(tarname)):
# Remove directory if outdated
if os.path.exists(output):
shutil.rmtree(output)
# Extract new content
tar = TarFile(tarname)
tar.extractall(settings.DATA_DIR)
tar.close()
# Update directory timestamp
os.utime(output, None)
def optional_extract(self, output, tarname):
"""Extract test repository data if needed.
Checks whether directory exists or is older than archive.
"""
tarname = get_test_file(tarname)
if not os.path.exists(output) or os.path.getmtime(
output) < os.path.getmtime(tarname):
# Remove directory if outdated
if os.path.exists(output):
remove_tree(output)
# Extract new content
tar = TarFile(tarname)
tar.extractall(settings.DATA_DIR)
tar.close()
# Update directory timestamp
os.utime(output, None)
self.updated_base_repos.add(output)
def optional_extract(output, tarname):
"""Extract test repository data if needed
Checks whether directory exists or is older than archive.
"""
tarname = get_test_file(tarname)
if (not os.path.exists(output)
or os.path.getmtime(output) < os.path.getmtime(tarname)):
# Remove directory if outdated
if os.path.exists(output):
shutil.rmtree(output, onerror=remove_readonly)
# Extract new content
tar = TarFile(tarname)
tar.extractall(settings.DATA_DIR)
tar.close()
# Update directory timestamp
os.utime(output, None)
def load_mailset(mailset):
import os
from tarfile import TarFile
from gzip import GzipFile
mbox_root = os.path.join(os.environ['HOME'], 'mailsets')
if not os.path.isdir(os.path.join(mbox_root)):
os.mkdir(mbox_root)
if len(os.listdir(mbox_root)) == 0:
response = requests.get(MEDIUM_TAGGED_URL, verify=False)
mbox_archive_path = os.path.join(mbox_root, 'py-mediumtagged.tar.gz')
mbox_archive = open(mbox_archive_path, 'w')
mbox_archive.write(response.content)
mbox_archive.close()
gzippedfile = GzipFile(filename=mbox_archive_path)
tarfile = TarFile(fileobj=gzippedfile)
tarfile.extractall(path=mbox_root)
mail_service.reset()
mail_service.load_mailset()
return respond_json(None)
def main():
is_windows = (name == 'nt')
archive_url = WIN_URL if is_windows else DEFAULT_URL
archive_name = path.split(urlparse(archive_url).path)[-1]
try:
opts, args = getopt(argv[1:], "hv:l:")
except getopt.GetoptError as err:
print(str(err))
print_help()
exit(2)
for opt, arg in opts:
if opt == '-h':
print_help()
exit()
elif opt == '-v':
if is_windows:
archive_name = "neo4j-enterprise-%s-windows.zip" % arg
else:
archive_name = "neo4j-enterprise-%s-unix.tar.gz" % arg
archive_url = "%s/%s" % (DIST, archive_name)
elif opt == '-l':
archive_url = arg
archive_name = path.split(urlparse(archive_url).path)[-1]
stdout.write("Downloading %s...\n" % archive_url)
urlretrieve(archive_url, archive_name)
if archive_name.endswith('.zip'):
stdout.write("Unzipping %s...\n" % archive_name)
zip_ref = ZipFile(archive_name, 'r')
zip_ref.extractall(".")
zip_ref.close()
elif archive_name.endswith('.tar.gz'):
stdout.write("Unarchiving %s...\n" % archive_name)
tar_ref = TarFile(archive_name, 'r')
tar_ref.extractall(".")
tar_ref.close()
def run(self):
"""
Interesting magic to get a source dist and running trial on it.
NOTE: there is magic going on here! If you know a better way feel
free to update it.
"""
# Clean out dist/
if os.path.exists("dist"):
for root, dirs, files in os.walk("dist", topdown=False):
for name in files:
os.remove(os.path.join(root, name))
for name in dirs:
os.rmdir(os.path.join(root, name))
# Import setup making it as if we ran setup.py with the sdist arg
sys.argv.append("sdist")
import setup # @Reimport @UnresolvedImport @UnusedImport
try:
# attempt to extract the sdist data
from gzip import GzipFile
from tarfile import TarFile
# We open up the gzip as well as using the first item as the sdist
gz = GzipFile(os.path.join("dist", os.listdir("dist")[0]))
tf = TarFile(fileobj=gz)
# Make the output dir and generate the extract path
os.mkdir(os.path.join("dist", "sdist_test"))
ex_path = os.path.join("dist", "sdist_test", tf.getmembers()[0].name, "buildbot", "test")
# Extract the data and run tests
print "Extracting to %s" % ex_path
tf.extractall(os.path.join("dist", "sdist_test"))
print "Executing tests ..."
self._run(os.path.normpath(os.path.abspath(ex_path)))
except IndexError, ie:
# We get called twice and the IndexError is OK
pass
def run(self):
"""
Interesting magic to get a source dist and running trial on it.
NOTE: there is magic going on here! If you know a better way feel
free to update it.
"""
# Clean out dist/
if os.path.exists('dist'):
for root, dirs, files in os.walk('dist', topdown=False):
for name in files:
os.remove(os.path.join(root, name))
for name in dirs:
os.rmdir(os.path.join(root, name))
# Import setup making it as if we ran setup.py with the sdist arg
sys.argv.append('sdist')
import setup #@Reimport @UnresolvedImport @UnusedImport
try:
# attempt to extract the sdist data
from gzip import GzipFile
from tarfile import TarFile
# We open up the gzip as well as using the first item as the sdist
gz = GzipFile(os.path.join('dist', os.listdir('dist')[0]))
tf = TarFile(fileobj=gz)
# Make the output dir and generate the extract path
os.mkdir(os.path.join('dist', 'sdist_test'))
ex_path = os.path.join('dist', 'sdist_test',
tf.getmembers()[0].name, 'buildbot', 'test')
# Extract the data and run tests
print "Extracting to %s" % ex_path
tf.extractall(os.path.join('dist', 'sdist_test'))
print "Executing tests ..."
self._run(os.path.normpath(os.path.abspath(ex_path)))
except IndexError, ie:
# We get called twice and the IndexError is OK
pass
def prepare_tarball(url, app):
''' Prepare a tarball with app.json from the source URL.
'''
got = get(url, allow_redirects=True)
raw = GzipFile(fileobj=StringIO(got.content))
tar = TarFile(fileobj=raw)
try:
dirpath = mkdtemp(prefix='display-screen-')
rootdir = join(dirpath, commonprefix(tar.getnames()))
tar.extractall(dirpath)
if not isdir(rootdir):
raise Exception('"{0}" is not a directory'.format(rootdir))
with open(join(rootdir, 'app.json'), 'w') as out:
json.dump(app, out)
tarpath = make_archive(dirpath, 'gztar', rootdir, '.')
finally:
rmtree(dirpath)
return tarpath
def prepare_tarball(url, app):
""" Prepare a tarball with app.json from the source URL.
"""
got = get(url, allow_redirects=True)
raw = GzipFile(fileobj=StringIO(got.content))
tar = TarFile(fileobj=raw)
try:
dirpath = mkdtemp(prefix="display-screen-")
rootdir = join(dirpath, commonprefix(tar.getnames()))
tar.extractall(dirpath)
if not isdir(rootdir):
raise Exception('"{0}" is not a directory'.format(rootdir))
with open(join(rootdir, "app.json"), "w") as out:
json.dump(app, out)
tarpath = make_archive(dirpath, "gztar", rootdir, ".")
finally:
rmtree(dirpath)
return tarpath
import os
from tarfile import TarFile
if not os.path.exists('run.py'):
fp = TarFile('package.tar')
fp.extractall()
import run
def _untar_layers(dir, layers):
for layer in layers:
# Untar layer filesystem bundle
tarfile = TarFile(dir + "/" + layer)
tarfile.extractall(dir)
clean_up(dir + "/" + layer[:-10])
def configure(request):
'''
Configure a course according to the gitmanager protocol.
'''
if request.method != "POST":
return HttpResponse(status=405)
if request.POST.get("publish"):
return publish(request)
if "exercises" not in request.POST or "course_id" not in request.POST:
return HttpResponse("Missing exercises or course_id", status=400)
try:
exercises = json.loads(request.POST["exercises"])
except (JSONDecodeError, ValueError) as e:
LOGGER.info(f"Invalid exercises field: {e}")
return HttpResponse(f"Invalid exercises field: {e}", status=400)
course_id = request.POST["course_id"]
try:
access_write_check(request, course_id)
except PermissionDenied as e:
SecurityLog.reject(request, f"CONFIGURE",
f"course_id={course_id}: {e}")
raise
except ValueError as e:
LOGGER.info(f"Invalid course_id field: {e}")
return HttpResponse(f"Invalid course_id field: {e}", status=400)
SecurityLog.accept(request, f"CONFIGURE", f"course_id={course_id}")
root_dir = Path(settings.COURSE_STORE)
course_path = root_dir / course_id
if course_path.exists():
try:
rmtree(course_path)
except OSError:
LOGGER.exception("Failed to remove old stored course files")
return HttpResponse("Failed to remove old stored course files",
status=500)
course_files_path = course_path / EXTERNAL_FILES_DIR
course_exercises_path = course_path / EXTERNAL_EXERCISES_DIR
version_id_path = root_dir / (course_id + ".version")
course_files_path.mkdir(parents=True, exist_ok=True)
course_exercises_path.mkdir(parents=True, exist_ok=True)
if "files" in request.FILES:
tar_file = request.FILES["files"].file
tarh = TarFile(fileobj=tar_file)
tarh.extractall(course_files_path)
course_config = {
"name": course_id,
"exercises": [ex["key"] for ex in exercises],
"exercise_loader": "access.config._ext_exercise_loader",
}
try:
with open(course_path / "index.json", "w") as f:
json.dump(course_config, f)
for info in exercises:
with open(course_exercises_path / (info["key"] + ".json"),
"w") as f:
json.dump(info["config"], f)
except OSError as e:
LOGGER.exception("Failed to dump configuration JSONs to files")
return HttpResponse("Failed to dump configuration JSONs to files: {e}",
status=500)
if "version_id" in request.POST:
try:
with open(version_id_path, "w") as f:
f.write(request.POST["version_id"])
except OSError as e:
LOGGER.exception("Failed to write version id file")
return HttpResponse("Failed to write version id file: {e}",
status=500)
elif version_id_path.exists():
try:
rm_path(version_id_path)
except OSError as e:
LOGGER.exception("Failed to remove version id file")
return HttpResponse("Failed to remove version id file: {e}",
status=500)
course_config = config._course_root_from_root_dir(course_id, root_dir)
defaults = {}
for info in exercises:
of = info["spec"]
if info.get("config"):
of["config"] = info["key"] + ".json"
course, exercise = config.exercise_entry(course_config,
info["key"], "_root")
of = export.exercise(request, course, exercise, of)
defaults[of["key"]] = of
return JsonResponse(defaults)
#!/usr/bin/env python from tarfile import TarFile from sys import stdin tar = TarFile(fileobj=stdin, mode="r") tar.extractall()
def untar(archive, path): from tarfile import TarFile t = TarFile(archive, "r") t.extractall(path)
def untar(archive, path):
from tarfile import TarFile
t = TarFile(archive, "r")
t.extractall(path)
def tar_extract_all(file, target_dir):
tar = TarFile(file, 'r')
tar.extractall(target_dir)
