def tree_in_zip(path, stream=None):
print('Trying {}'.format(path))
if stream is not None and ZIP_FILE.search(path) is not None:
zf = ZipFile(stream)
#Check if the file is not the first in the loop and wheter it is a .zip.
elif stream is None and ZIP_FILE.search(path) is not None:
zf = ZipFile(path)
#Check if the first file is .zip.
else:
if stream is not None and TAR_FILE.search(path) is not None :
try:
zf=TarFile.open(fileobj=stream, encoding="utf-8",mode='r:*')
except ReadError:
pass
#If the file is not the first in the loop and it is a .tar(.xx) tries to read it as an archive.
elif stream is None and TAR_FILE.search(path) is not None:
try:
zf=TarFile.open(path, mode='r:*')
except ReadError:
pass
#If the first file is a .tar(.xx) tries to read it as an archive.
try:
if type(zf)==ZipFile:
try:
for name in zf.namelist():
count_extensions(name, EXTENSIONS, NR_EXTENSIONS)
#Count extensions of interest in the archive.
manifesto_maker(zf, name, MANIFESTO)
#fills in the dictionary
if ZIP_FILE.search(name) is not None:
print('-Found {}'.format(name))
yield from tree_in_zip(path+'/'+name, BytesIO(zf.read(name)))
elif TAR_FILE.search(name) is not None:
print('-Found {}'.format(name))
yield from tree_in_zip(path+'/'+name, BytesIO(zf.read(name)))
else:
yield path+'/'+name
except BadZipFile:
pass
#Search for further archives (.zip/.tar). Exception needed to not to stop at a corrupted archive.
elif type(zf)==TarFile:
#No need for try checked the file at the begining.
for name in zf.getnames():
count_extensions(name, EXTENSIONS, NR_EXTENSIONS)
manifesto_maker(zf, name, MANIFESTO)
if ZIP_FILE.search(name) is not None:
print('-Found {}'.format(name))
yield from tree_in_zip(path+'/'+name, BytesIO(zf.read(name)))
elif TAR_FILE.search(name) is not None:
print('- Found {}'.format(name))
yield from tree_in_zip(path+'/'+name, BytesIO(zf.read(name)))
else:
yield path+'/'+name
else:
pass
#if file is not .tar/.zip skip it
except UnboundLocalError:
pass
def rebuild(filename, tag=None, format="gz"):
import tempfile, shutil
tmpdir = tempfile.mkdtemp()
zonedir = os.path.join(tmpdir, "zoneinfo")
moduledir = os.path.dirname(__file__)
if tag: tag = "-" + tag
targetname = "zoneinfo%s.tar.%s" % (tag, format)
try:
tf = TarFile.open(filename)
# The "backwards" zone file contains links to other files, so must be
# processed as last
for name in sorted(tf.getnames(),
key=lambda k: k != "backward" and k or "z"):
if not (name.endswith(".sh") or name.endswith(".tab")
or name == "leapseconds"):
tf.extract(name, tmpdir)
filepath = os.path.join(tmpdir, name)
os.system("zic -d %s %s" % (zonedir, filepath))
tf.close()
target = os.path.join(moduledir, targetname)
for entry in os.listdir(moduledir):
if entry.startswith("zoneinfo") and ".tar." in entry:
os.unlink(os.path.join(moduledir, entry))
tf = TarFile.open(target, "w:%s" % format)
for entry in os.listdir(zonedir):
entrypath = os.path.join(zonedir, entry)
tf.add(entrypath, entry)
tf.close()
finally:
shutil.rmtree(tmpdir)
def rebuild(filename, tag=None, format="gz", zonegroups=[], metadata=None):
"""Rebuild the internal timezone info in dateutil/zoneinfo/zoneinfo*tar*
filename is the timezone tarball from ``ftp.iana.org/tz``.
"""
tmpdir = tempfile.mkdtemp()
zonedir = os.path.join(tmpdir, "zoneinfo")
moduledir = os.path.dirname(__file__)
try:
with TarFile.open(filename) as tf:
for name in zonegroups:
tf.extract(name, tmpdir)
filepaths = [os.path.join(tmpdir, n) for n in zonegroups]
try:
check_call(["zic", "-d", zonedir] + filepaths)
except OSError as e:
_print_on_nosuchfile(e)
raise
# write metadata file
with open(os.path.join(zonedir, METADATA_FN), 'w') as f:
json.dump(metadata, f, indent=4, sort_keys=True)
target = os.path.join(moduledir, ZONEFILENAME)
with TarFile.open(target, "w:%s" % format) as tf:
for entry in os.listdir(zonedir):
entrypath = os.path.join(zonedir, entry)
tf.add(entrypath, entry)
finally:
shutil.rmtree(tmpdir)
def rebuild(filename, tag=None, format="gz"):
import tempfile, shutil
tmpdir = tempfile.mkdtemp()
zonedir = os.path.join(tmpdir, "zoneinfo")
moduledir = os.path.dirname(__file__)
if tag: tag = "-"+tag
targetname = "zoneinfo%s.tar.%s" % (tag, format)
try:
tf = TarFile.open(filename)
# The "backwards" zone file contains links to other files, so must be
# processed as last
for name in sorted(tf.getnames(),
key=lambda k: k != "backward" and k or "z"):
if not (name.endswith(".sh") or
name.endswith(".tab") or
name == "leapseconds"):
tf.extract(name, tmpdir)
filepath = os.path.join(tmpdir, name)
os.system("zic -d %s %s" % (zonedir, filepath))
tf.close()
target = os.path.join(moduledir, targetname)
for entry in os.listdir(moduledir):
if entry.startswith("zoneinfo") and ".tar." in entry:
os.unlink(os.path.join(moduledir, entry))
tf = TarFile.open(target, "w:%s" % format)
for entry in os.listdir(zonedir):
entrypath = os.path.join(zonedir, entry)
tf.add(entrypath, entry)
tf.close()
finally:
shutil.rmtree(tmpdir)
def rebuild(filename, tag=None, format="gz", zonegroups=[], metadata=None):
"""Rebuild the internal timezone info in dateutil/zoneinfo/zoneinfo*tar*
filename is the timezone tarball from ftp.iana.org/tz.
"""
tmpdir = tempfile.mkdtemp()
zonedir = os.path.join(tmpdir, "zoneinfo")
moduledir = os.path.dirname(__file__)
try:
with TarFile.open(filename) as tf:
for name in zonegroups:
tf.extract(name, tmpdir)
filepaths = [os.path.join(tmpdir, n) for n in zonegroups]
try:
check_call(["zic", "-d", zonedir] + filepaths)
except OSError as e:
_print_on_nosuchfile(e)
raise
# write metadata file
with open(os.path.join(zonedir, METADATA_FN), 'w') as f:
json.dump(metadata, f, indent=4, sort_keys=True)
target = os.path.join(moduledir, ZONEFILENAME)
with TarFile.open(target, "w:%s" % format) as tf:
for entry in os.listdir(zonedir):
entrypath = os.path.join(zonedir, entry)
tf.add(entrypath, entry)
finally:
shutil.rmtree(tmpdir)
def install_packages(self):
self._update_package_with_install_path()
installed_packages = []
for p in self._packages:
if 'install_from' not in p:
print("[dem] Could not find package: {}, version: {}".format(p['name'], p['version']))
else:
if not self._cache.is_package_installed(p['name'], p['version']):
print('[dem] installing {}-{}'.format(p['name'], p['version']))
if p['install_from_ext'] == 'zip':
with ZipFile(p['install_from'], 'r') as archive:
locations = self._extract(archive, p)
elif p['install_from_ext'] == 'tar.gz':
with TarFile.open(p['install_from'], 'r:gz') as archive:
locations = self._extract(archive, p)
elif p['install_from_ext'] == 'tar.bz2':
with TarFile.open(p['install_from'], 'r:bz2') as archive:
locations = self._extract(archive, p)
elif p['install_from_ext'] == 'gz':
with gzip.open(p['install_from'], 'r') as archive:
locations = self._extract(archive, p)
if 'pkg-config' in p:
PkgConfigProcessor.replace_prefix(locations, p['pkg-config'])
else:
print('[dem] {}-{} already installed'.format(p['name'], p['version']))
locations = self._cache.install_locations(p['name'])
package = dict()
package[p['name']] = {'version': p['version'], 'type': 'local', 'install_locations': locations}
installed_packages.append(package)
return installed_packages
def rebuild(filename, tag=None, format="gz"):
import tempfile, shutil
tmpdir = tempfile.mkdtemp()
zonedir = os.path.join(tmpdir, "zoneinfo")
moduledir = os.path.dirname(__file__)
if tag: tag = "-"+tag
targetname = "zoneinfo%s.tar.%s" % (tag, format)
try:
tf = TarFile.open(filename)
for name in tf.getnames():
if not (name.endswith(".sh") or
name.endswith(".tab") or
name == "leapseconds"):
tf.extract(name, tmpdir)
filepath = os.path.join(tmpdir, name)
os.system("zic -d %s %s" % (zonedir, filepath))
tf.close()
target = os.path.join(moduledir, targetname)
for entry in os.listdir(moduledir):
if entry.startswith("zoneinfo") and ".tar." in entry:
os.unlink(os.path.join(moduledir, entry))
tf = TarFile.open(target, "w:%s" % format)
for entry in os.listdir(zonedir):
entrypath = os.path.join(zonedir, entry)
tf.add(entrypath, entry)
tf.close()
finally:
shutil.rmtree(tmpdir)
def _populate_metadata(self):
with TemporaryDirectory() as tempdir:
meta_src = os.path.join(tempdir, self.META_TEMPLATE_TARGET)
meta_dest = os.path.join(tempdir, self.META_TARGET)
tarball_dest = os.path.join(tempdir,
os.path.basename(self.filename))
if self.remote_file:
download(self.filename, tarball_dest)
print('Extracting {0}'.format(tarball_dest))
try:
with TarFile.open(tarball_dest, 'r:*') as tarball:
tarball.extract(self.META_TEMPLATE_TARGET, tempdir)
except KeyError:
try:
# Older version of conda-build: Use META_TARGET instead
self.old_behavior = True
with TarFile.open(tarball_dest, 'r:*') as tarball:
tarball.extract(self.META_TARGET, tempdir)
except KeyError:
print('Proprietary package lacks required data!')
print()
return None
if not self.old_behavior:
shutil.move(meta_src, meta_dest)
print()
return MetaData(meta_dest)
def archiveFromGit(versionedPackageName, committish):
prefix = '%s/' % versionedPackageName
distBase = 'derived/dist/'
umask = (stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP
| stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH)
def exclude(info):
'''Returns True iff the given tar entry should be excluded.
'''
return any((
info.name.endswith('/.gitignore'),
info.name.startswith(prefix + 'doc/internal'),
))
proc = Popen(
('git', 'archive', '--prefix=' + prefix, '--format=tar', committish),
bufsize=-1,
stdin=None,
stdout=PIPE,
stderr=sys.stderr,
)
try:
outTarPath = distBase + versionedPackageName + '.tar.gz'
print('archive:', outTarPath)
if not isdir(distBase):
makedirs(distBase)
try:
with TarFile.open(outTarPath, 'w:gz') as outTar:
# Copy entries from "git archive" into output tarball,
# except for excluded entries.
numIncluded = numExcluded = 0
with TarFile.open(mode='r|', fileobj=proc.stdout) as inTar:
for info in inTar:
if exclude(info):
if verbose:
print('EX', info.name)
numExcluded += 1
else:
if verbose:
print('IN', info.name)
numIncluded += 1
info.uid = info.gid = 1000
info.uname = info.gname = 'openmsx'
info.mode = info.mode & umask
outTar.addfile(info, inTar.extractfile(info))
print('entries: %d included, %d excluded' %
(numIncluded, numExcluded))
except:
# Clean up partial output file.
remove(outTarPath)
raise
except:
proc.terminate()
raise
else:
data, _ = proc.communicate()
if len(data) != 0:
print('WARNING: %d more bytes of data from "git archive" after '
'tar stream ended' % len(data),
file=sys.stderr)
def rebuild(filename, tag=None, format="gz"):
import tempfile, shutil
tmpdir = tempfile.mkdtemp()
zonedir = os.path.join(tmpdir, "zoneinfo")
moduledir = os.path.dirname(__file__)
if tag: tag = "-"+tag
targetname = "zoneinfo%s.tar.%s" % (tag, format)
try:
tf = TarFile.open(filename)
for name in tf.getnames():
if not (name.endswith(".sh") or
name.endswith(".tab") or
name == "leapseconds"):
tf.extract(name, tmpdir)
filepath = os.path.join(tmpdir, name)
os.system("zic -d %s %s" % (zonedir, filepath))
tf.close()
target = os.path.join(moduledir, targetname)
for entry in os.listdir(moduledir):
if entry.startswith("zoneinfo") and ".tar." in entry:
os.unlink(os.path.join(moduledir, entry))
tf = TarFile.open(target, "w:%s" % format)
for entry in os.listdir(zonedir):
entrypath = os.path.join(zonedir, entry)
tf.add(entrypath, entry)
tf.close()
finally:
shutil.rmtree(tmpdir)
def test_getclasstgz(self):
api = WimsAPI(WIMS_URL, "myself", "toto")
status, response = api.getclasstgz(999999, "myclass")
self.assertTrue(status)
try:
TarFile.open(fileobj=BytesIO(response), mode="r:gz")
except TarError as e:
self.fail("Response was not a valid tgz :\n" + str(e))
def createPluginStructure():
global BUILD_PATH
if os.path.exists(BUILD_PATH):
shutil.rmtree(BUILD_PATH)
# prevent win 8 io error
time.sleep(1)
if not os.path.exists(BUILD_PATH):
os.makedirs(BUILD_PATH)
os.chdir(BUILD_PATH)
BUILD_PATH = os.getcwd()
control_path = path_join(BUILD_PATH, "CONTROL")
os.mkdir(control_path)
shutil.move(PLUGIN_HASH_FILE, path_join(BUILD_PATH, "CONTROL"))
createControl(control_path)
#createConfFiles(control_path)
#createPreInst(control_path)
#createPostInst(control_path)
#createPreRM(control_path)
createPostRM(control_path)
createDebianBinary()
native_tar = False
if not native_tar:
tar = TarFile.open("control.tar.gz", "w:gz")
tar.add("CONTROL", ".", filter=tar_filter)
tar.close()
tar = TarFile.open("data.tar.gz", "w:gz")
tar.add(PLUGIN)
tar.add(COMPONENTS_PATH)
tar.close()
# tar_name = path_join(BUILD_PATH, "control.tar.gz")
# os.chdir(path_join(BUILD_PATH, "CONTROL"))
# os.system("chmod 755 *")
# makeTarGz(".", tar_name)
# os.chdir(BUILD_PATH)
# tar_name = path_join(BUILD_PATH, "data.tar.gz")
# makeTarGz(PLUGIN, tar_name)
if native_tar:
os.system("chmod 755 CONTROL/*")
cmd = "tar -C CONTROL -czf control.tar.gz ."
os.system("chmod 755 CONTROL/*")
os.system(cmd)
cmd = "tar -C . -czf data.tar.gz %s" % (PLUGIN)
os.system(cmd)
def __enter__(self):
if self.tf is not None:
raise ValueError('Cannot re-enter')
if '://' in self.web_archive:
info('Downloading from {0}'.format(self.web_archive))
dl = requests.get(self.web_archive)
self.tf = TarFile.open(path(self.web_archive).basename(),
'r:*', fileobj=io.BytesIO(dl.content))
else:
self.tf = TarFile.open(self.web_archive)
return self.tf
def extract_package(package_name, path, logger):
try:
if ".tar" in package_name:
TarFile.open(package_name).extractall(path)
elif ".zip" in package_name:
ZipFile(package_name).extractall(path)
else:
raise FileTypeError("It's not a TAR or ZIP archive.")
except Exception as err:
register_exception(alert_admin=True,
prefix="Elsevier error extracting package.")
logger.error("Error extraction package file: %s %s"
% (path, err))
def extract_package(package_name, path, logger):
try:
if ".tar" in package_name:
TarFile.open(package_name).extractall(path)
elif ".zip" in package_name:
ZipFile(package_name).extractall(path)
else:
raise FileTypeError("It's not a TAR or ZIP archive.")
except Exception as err:
register_exception(alert_admin=True,
prefix="Elsevier error extracting package.")
logger.error("Error extraction package file: %s %s"
% (path, err))
def setUp(self):
"""Set up each test."""
super().setUp()
self.fake_bucket_name = tempfile.mkdtemp()
mytar = TarFile.open('./koku/masu/test/data/test_local_bucket.tar.gz')
mytar.extractall(path=self.fake_bucket_name)
os.makedirs(DATA_DIR, exist_ok=True)
self.mock_task = Mock(request=Mock(id=str(self.fake.uuid4()),
return_value={}))
self.report_downloader = ReportDownloader(
task=self.mock_task,
customer_name=self.fake_customer_name,
access_credential=self.fake_auth_credential,
report_source=self.fake_bucket_name,
provider_type='AWS-local',
provider_uuid=self.aws_provider_uuid,
)
self.aws_local_report_downloader = AWSLocalReportDownloader(
**{
'task': self.mock_task,
'customer_name': self.fake_customer_name,
'auth_credential': self.fake_auth_credential,
'bucket': self.fake_bucket_name,
'provider_uuid': self.aws_provider_uuid,
}
)
def _extract_file(self, fname, extract_dir):
"""
This method receives an argument for the archive to extract and the
destination path.
"""
with TarFile.open(fname, "r") as tar_file:
if self.members is None:
get_logger().info(
"Untarring contents of '%s' to '%s'", fname, extract_dir
)
# Unpack all files from the archive into our new folder
tar_file.extractall(path=extract_dir)
else:
for member in self.members:
get_logger().info(
"Extracting '%s' from '%s' to '%s'", member, fname, extract_dir
)
# make sure the target folder exists for nested members
parts = Path(member).parts
if len(parts) > 1:
full_dir_path = os.path.join(extract_dir, *parts[:-1])
os.makedirs(full_dir_path, exist_ok=True)
# Extract the data file from within the archive
# Python 2.7: extractfile doesn't return a context manager
data_file = tar_file.extractfile(member)
try:
# Save it to our desired file name
with open(os.path.join(extract_dir, member), "wb") as output:
output.write(data_file.read())
finally:
data_file.close()
def _import_gpg_keyring(self, abs_temp_dir, abs_gpg_home_dir, package_filename, package_yyyymmdd):
rel_archlinux_gpg_path = 'archlinux-keyring-%s/archlinux.gpg' % package_yyyymmdd
with TarFile.open(package_filename) as tf:
tf.extract(rel_archlinux_gpg_path, path=abs_temp_dir)
abs_archlinux_gpg_path = os.path.join(abs_temp_dir, rel_archlinux_gpg_path)
self._import_gpg_key_file(abs_gpg_home_dir, abs_archlinux_gpg_path)
def _get_bin_for_release(self, release_file_path: Path):
if not release_file_path.exists():
raise ValueError(f'Release file {release_file_path} not found')
if release_file_path.suffix == '.gz':
opener = TarFile.open(release_file_path, 'r:*')
else:
opener = ZipFile(release_file_path, 'r')
with opener as archive:
if release_file_path.suffix == '.gz':
contents = archive.getnames()
else:
contents = archive.namelist()
if len(contents) != 1:
raise ValueError(
f'Release archive has unexpected content. '
f'Expected 1 file, found {len(contents)}: {", ".join(contents)}',
)
bin_file_path = self._bin_path.joinpath(contents[0])
if not bin_file_path.exists():
log.debug(
'Extracting Raiden binary',
release_file_name=release_file_path.name,
bin_file_name=bin_file_path.name,
)
archive.extract(contents[0], self._bin_path)
bin_file_path.chmod(0o770)
return bin_file_path
def open_archive(path):
from tarfile import TarFile
archive = TarFile.open(name=path, mode="w:gz", bufsize=16*1024)
try:
yield archive
finally:
archive.close()
def download(archive_url, archive_name, extract_to_path='.', require_basic_auth=False):
# download the file to extract_to_path
if not path.exists(extract_to_path):
makedirs(extract_to_path)
archive_path = path.join(extract_to_path, archive_name)
stdout.write("Downloading '%s' to '%s'...\n" % (archive_url, archive_path))
source = urlopen(archive_url) if not require_basic_auth else teamcityurlopen(archive_url)
with open(archive_path, "wb") as destination:
more = True
while more:
data = source.read(8192)
if data:
destination.write(data)
else:
more = False
if archive_name.endswith('.zip'):
stdout.write("Unzipping '%s' to '%s'...\n" % (archive_path, extract_to_path))
zip_ref = ZipFile(archive_path, 'r')
zip_ref.extractall(extract_to_path)
unzip_folder = zip_ref.namelist()[0]
zip_ref.close()
return unzip_folder
elif archive_name.endswith('.tar.gz'):
stdout.write("Unarchiving '%s' to '%s'...\n" % (archive_path, extract_to_path))
tar_ref = TarFile.open(archive_path)
tar_ref.extractall(extract_to_path)
untar_folder=tar_ref.getnames()[0]
tar_ref.close()
return untar_folder
def push(self, local, remote):
if os.path.isdir(local):
if not os.path.exists(self.cache): os.mkdir(self.cache)
cached_local = local.replace('/', '_')[1:] + '.tar.gz'
cached_file = os.path.join(self.cache, cached_local)
if not os.path.exists(cached_file):
log.debug('Creating cache file %s', cached_file)
tar = TarFile.open(cached_file, 'w:gz')
tar.add(os.path.realpath(local), arcname='.')
log.debug('Done...')
tar.close()
with open(cached_file,'r') as f:
self.run('mkdir -p {0}'.format(remote))
return self.run(
'tar zxf - -C {0}'.format(remote),
in_=ProcessFile(f)
)
else:
with open(local) as f:
return self.run(
'cat > {0}'.format(remote),
in_=ProcessFile(f)
)
def upload(self, token, metadata, reader):
name = metadata["name"]
version = metadata["version"]
rep = self.repos.get(name)
if rep is None:
where = os.path.join(self.root, name)
need_init = True
if os.path.exists(where):
self.warning("%s exists - cleared", where)
shutil.rmtree(where)
os.mkdir(where)
else:
where = dirname(rep.path)
need_init = False
with TarFile.open(mode="r|gz", fileobj=reader) as tar:
tar.extractall(where)
if not need_init:
self.add_version(rep, version)
else:
self.repos[name] = rep = pygit2.init_repository(where)
try:
self.add_version(rep, version)
except Exception as e:
shutil.rmtree(where)
del self.repos[name]
self.error("Failed to initialize %s", name)
raise from_none(e)
rep.config["forge.tokens"] = self.scramble(token)
self._generate_images(metadata, rep)
def retrieveCoreInfo():
DIR_PREFIX = 'libretro-super-master/dist/info/'
res = urllib2.urlopen('https://github.com/libretro/libretro-super/archive/master.tar.gz')
archive = StringIO(res.read())
tf = TarFile.open(mode='r:gz', fileobj=archive)
# filter files and sort them
files = [f for f in tf.getmembers() if f.name.startswith(DIR_PREFIX)]
files = sorted(files, key=lambda m: m.name.split(DIR_PREFIX)[1])
def infoToDict( files=[]):
output = {'cores': OrderedDict()}
for f in files:
info = OrderedDict()
for l in tf.extractfile(f).readlines():
s = shlex.shlex(l)
s.quotes = '"'
op = []
try:
while True:
t = s.get_token()
if t == s.eof:
break
op.append(t)
# 'var' '=' 'value'
if len(op) == 3:
assert op[1] in ('=', ':')
# For: 'some:var = value' syntax
# Merge 'some:var' into 'somevar' (?)
if op[1] == ':':
op = [''.join(op)]
continue
# Strip quotes
if op[2][0] == '"' and op[2][-1] == '"':
op[2] = op[2][1:-1]
# Convert string to boolean
if op[2] in ('true', 'True'):
op[2] = True
elif op[2] in ('false', 'False'):
op[2] = False
# Decode utf-8 into unicode
if isinstance(op[2], str):
op[2] = op[2].decode('utf-8')
info[op[0]] = op[2]
break
except (ValueError, AssertionError) as e:
print("Could not process file %s (%s) " % (f.name, e), file=sys.stderr)
continue
output['cores'][f.name.split(DIR_PREFIX)[1].split('.info')[0]] = info
return output
return infoToDict(files)
def handle():
try:
if request.method == 'GET': return _as_text( app.config[ 'TAR_DIGEST' ] )
try:
signature = request.form[ 'signature' ]
except KeyError:
uid = None
else:
uid = extract_uid( signature )
if not uid:
EVENTS_LOG.info( 'Unauthorized: {0}@{1}'.format( uid, request.remote_addr ) )
return _as_text( '# {0}\n'.format( _( 'Invalid or absent signature!' ) ), 401 )
if 'tar' in request.form: # this is an upload
data = decodestring( request.form[ 'tar' ] )
dest = join( app.config[ 'UPLOAD_DIR' ], uid, str( int( time() * 1000 ) ) + '.tar' )
with open( dest, 'w' ) as f: f.write( data )
tf = TarFile.open( dest, mode = 'r' )
names = tf.getnames()
tf.close()
EVENTS_LOG.info( 'Upload: {0}@{1}'.format( uid, request.remote_addr ) )
return _as_text( '\n'.join( sorted( names ) ) )
else: # this is a download
EVENTS_LOG.info( 'Download: {0}@{1}'.format( uid, request.remote_addr ) )
return _as_text( app.config[ 'TAR_DATA' ] )
except:
if app.debug:
raise
else:
app.logger.exception( '' )
return _as_text( '# {0}\n'.format( _( 'An unexpected server error occurred!' ) ), 500 )
def _get_bin_for_release(self, release_file_path: Path):
if not release_file_path.exists():
raise ValueError(f'Release file {release_file_path} not found')
if release_file_path.suffix == '.gz':
opener = TarFile.open(release_file_path, 'r:*')
else:
opener = ZipFile(release_file_path, 'r')
with opener as archive:
if release_file_path.suffix == '.gz':
contents = archive.getnames()
else:
contents = archive.namelist()
if len(contents) != 1:
raise ValueError(
f'Release archive has unexpected content. '
f'Expected 1 file, found {len(contents)}: {", ".join(contents)}',
)
bin_file_path = self._bin_path.joinpath(contents[0])
if not bin_file_path.exists():
log.debug(
'Extracting Raiden binary',
release_file_name=release_file_path.name,
bin_file_name=bin_file_path.name,
)
archive.extract(contents[0], self._bin_path)
bin_file_path.chmod(0o770)
return bin_file_path
def _verifyTarballEntry(self, fileish, entry_name, data):
fileish.seek(0)
tarfile = TarFile.open('foo.tar.gz', fileobj=fileish, mode='r:gz')
extract = tarfile.extractfile(entry_name)
found = extract.read()
self.assertEqual(found, data)
def fetch(self):
"""
Download the specififc workflow bundle.
"""
self.path = self.path or ""
if not self.path or (os.path.exists(self.path) and
os.path.samefile(self.path, os.getcwd())):
self.path = os.path.join(self.path, "%s@%s" % (self.name,
self.version))
if os.path.exists(self.path):
if self.force:
shutil.rmtree(self.path)
else:
raise ValueError("Directory %s already exists and -f/--force "
"option was not specified" % self.path)
os.mkdir(self.path)
url = "%s%s?%s" % (
self.base, root.common.forge.fetch_name,
urlencode((("name", self.name), ("version", self.version))))
self.debug("Requesting %s", url)
fn = wget.download(url, self.path)
print("")
self.debug("Downloaded %s", fn)
with TarFile.open(fn) as tar:
tar.extractall(self.path)
os.remove(fn)
self.info("Put the following files into %s:\n %s", self.path,
"\n ".join(sorted(os.listdir(self.path))))
metadata = self._parse_metadata(self.path)
self._check_deps(metadata)
return self.path, metadata
def test_windows_with_mocks(self):
from infi.logs_collector.items import windows
result, archive_path = logs_collector.run("test", windows(), None, None)
self.assertTrue(path.exists(archive_path))
self.assertTrue(archive_path.endswith(".tar.gz"))
archive = TarFile.open(archive_path, "r:gz")
def extract_blahp(self, distro):
"""
Extract the BLAHP shared libs and bins for the target platform and dump
them to a temporary directory
"""
tempdir = tempfile.mkdtemp()
self.log.debug("Temporary working directory: %s" % tempdir)
tarfile = os.path.join(self.cachedir,self.version,"bosco-1.2-x86_64_" + distro + ".tar.gz")
cdir = 'condor-8.6.6-x86_64_' + distro + '-stripped/'
blahp_files = [
'lib/libclassad.so.8.6.6',
'lib/libclassad.so.8',
'lib/libclassad.so',
'lib/libcondor_utils_8_6_6.so',
'sbin/condor_ft-gahp' ]
blahp_dirs = [
'lib/condor/',
'libexec/glite/bin',
'libexec/glite/etc' ]
# open the tarball, extract blahp_files and blahp_dirs to tmp
with TarFile.open(tarfile) as t:
members = []
for f in blahp_files:
members.append(t.getmember(os.path.join(cdir,f)))
for d in blahp_dirs:
match = os.path.join(cdir, d)
files = [t.getmember(s) for s in t.getnames() if re.match(match, s)]
members.extend(files)
#self.log.debug("Extracting %s to %s" % (members, tempdir))
t.extractall(tempdir,members)
# once things are in tmp, we need to need to move things around and
# make some directories
dirs = [ 'bosco/glite/log', 'bosco/sandbox' ]
self.log.debug("Creating BOSCO directories...")
for d in dirs:
os.makedirs(os.path.join(tempdir, d))
# list of files and directories that need to move from the extracted tarball to the bosco dir
to_move = (
['lib','bosco/glite/lib'],
['libexec/glite/bin', 'bosco/glite/bin'],
['libexec/glite/etc', 'bosco/glite/etc'],
['sbin/condor_ft-gahp', 'bosco/glite/bin/condor_ft-gahp'] )
for t in to_move:
src = os.path.join(tempdir,cdir,t[0])
dst = os.path.join(tempdir,t[1])
self.log.debug("Moving %s to %s" % (src,dst))
shutil.move(src,dst)
self.log.debug("Deleting old directory: %s " % cdir)
shutil.rmtree(os.path.join(tempdir,cdir))
return tempdir
def get(self, filename="", file_hash=""):
""" Retrieve file information of hashed tree """
if filename:
# File accesibility
if not os.path.exists(filename):
raise IOError('Unaccesible file')
file_hash = self._get_hash(filename)
if not file_hash:
raise ValueError('Hash of file is mandatory')
path, tarfile, hashed_filename = self._get_path(file_hash)
# Tarfile accesibility
if not os.path.exists(os.path.join(path, tarfile)):
raise IOError('Unaccesible tarfile %s', os.path.join(path, tarfile))
tar = TarFile.open(name=os.path.join(path, tarfile), mode='r:%s' % self.external_compressor)
data = tar.extractfile(hashed_filename)
if not data:
raise ValueError('Member does not exists')
if self.internal_compressor:
return self.decoder.decode( self.internal_compressor.decompress( data.read() ) )
else:
return self.decoder.decode( data.read() )
def setUp(self):
"""Set up each test."""
super().setUp()
self.fake_bucket_name = tempfile.mkdtemp()
mytar = TarFile.open("./koku/masu/test/data/test_local_bucket.tar.gz")
mytar.extractall(path=self.fake_bucket_name)
os.makedirs(DATA_DIR, exist_ok=True)
self.credentials = {"role_arn": self.fake_auth_credential}
self.data_source = {"bucket": self.fake_bucket_name}
self.report_downloader = ReportDownloader(
customer_name=self.fake_customer_name,
credentials=self.credentials,
data_source=self.data_source,
provider_type=Provider.PROVIDER_AWS_LOCAL,
provider_uuid=self.aws_provider_uuid,
)
self.aws_local_report_downloader = AWSLocalReportDownloader(
**{
"customer_name": self.fake_customer_name,
"credentials": self.credentials,
"data_source": self.data_source,
"provider_uuid": self.aws_provider_uuid,
})
def put(self, content, filename="", file_hash=""):
""" Store file information in hashed tree """
if not filename and not file_hash:
raise ValueError('Filename or FileHash is mandatory')
if filename:
# File accesibility
if not os.path.exists(filename):
raise IOError('Unaccesible file %s', filename)
# Calc hash
file_hash = self._get_hash(filename)
if not file_hash:
raise ValueError('Hash of file is mandatory')
# Get file path for hash
path, tarfile, hashed_filename = self._get_path(file_hash)
# Create file path
try:
os.makedirs(path)
except WindowsError:
pass
except OSError:
pass
# Open tarfile
if self.external_compressor:
# External compressor is not suited for adding files.
raise ValueError('You cannot use external compressor for write files')
with TarFile.open(name=os.path.join(path, tarfile), mode='a') as tar:
with FileLock(os.path.join(path, tarfile)) as lock:
# Test if file already exists into tarfile
try:
tar.getmember(hashed_filename)
raise ValueError('Member already exists')
except KeyError:
pass
except:
raise
data = self.encoder.encode(content)
if self.internal_compressor:
data = self.internal_compressor.compress(data)
data_file = StringIO(data)
mtime = time.time()
ti = TarInfo(hashed_filename)
ti.size = data_file.len
ti.mtime = mtime
tar.addfile(tarinfo=ti, fileobj=data_file)
tar.close()
return file_hash
def test_download_bucket_with_prefix(self):
"""Test to verify that basic report downloading works."""
fake_bucket = tempfile.mkdtemp()
mytar = TarFile.open(
"./koku/masu/test/data/test_local_bucket_prefix.tar.gz")
mytar.extractall(fake_bucket)
test_report_date = datetime(year=2018, month=8, day=7)
fake_data_source = {"bucket": fake_bucket}
with patch.object(DateAccessor, "today",
return_value=test_report_date):
report_downloader = ReportDownloader(
self.fake_customer_name,
self.credentials,
fake_data_source,
Provider.PROVIDER_AWS_LOCAL,
self.aws_provider_uuid,
)
# Names from test report .gz file
report_context = {
"date": test_report_date.date(),
"manifest_id": 1,
"comporession": "GZIP",
"current_file":
"./koku/masu/test/data/test_local_bucket.tar.gz",
}
report_downloader.download_report(report_context)
expected_path = "{}/{}/{}".format(DATA_DIR, self.fake_customer_name,
"aws-local")
self.assertTrue(os.path.isdir(expected_path))
shutil.rmtree(fake_bucket)
def run(self):
self.ensure_directories_writable()
self._messenger.info('Searching for latest release...')
version_str = self._determine_latest_version()
version_tuple = self._parse_version(version_str)
self._messenger.info('Found {} to be latest.'.format(version_str))
tarball_download_url = self._create_tarball_download_url(
version_tuple, self._architecture)
signatur_download_url = '{}.asc'.format(tarball_download_url)
# Signature first, so we fail earlier if we do
abs_filename_signature = self._download_file(signatur_download_url)
abs_filename_tarball = self._download_file(tarball_download_url)
abs_temp_dir = os.path.abspath(tempfile.mkdtemp())
try:
abs_gpg_home_dir = self._initialize_gpg_home(abs_temp_dir)
release_pubring_gpg = resource_filename(resources.__name__,
'ncopa.asc')
self._import_gpg_key_file(abs_gpg_home_dir, release_pubring_gpg)
self._verify_file_gpg(abs_filename_tarball, abs_filename_signature,
abs_gpg_home_dir)
self._messenger.info('Extracting to "{}"...'.format(
self._abs_target_dir))
with TarFile.open(abs_filename_tarball) as tf:
tf.extractall(path=self._abs_target_dir)
finally:
self._messenger.info('Cleaning up "{}"...'.format(abs_temp_dir))
shutil.rmtree(abs_temp_dir)
def gettz(name):
for cachedname, tzinfo in CACHE:
if cachedname == name:
return tzinfo
name_parts = name.lstrip('/').split('/')
for part in name_parts:
if part == os.path.pardir or os.path.sep in part:
raise ValueError('Bad path segment: %r' % part)
filename = os.path.join(ZONEINFODIR, *name_parts)
try:
zonefile = open(filename, "rb")
except:
tzinfo = None
else:
tzinfo = tzfile(zonefile)
zonefile.close()
if tzinfo is None and ZONEINFOFILE:
tf = TarFile.open(ZONEINFOFILE)
try:
zonefile = tf.extractfile(name)
except KeyError:
tzinfo = None
else:
tzinfo = tzfile(zonefile)
tf.close()
if tzinfo is not None:
CACHE.insert(0, (name, tzinfo))
del CACHE[CACHESIZE:]
return tzinfo
def _get_attachments_from_stream(self, data_file):
"""Retrive attachment from tar file.
Return a dict containing all attachment ready to be saved
in Odoo.
The key is the name of file without extention
The value the PNG content encoded in base64
:param data_file: raw statement file sent to openerp (not in b64)
:type data_file: basestring subclass
:return: Return a dict containing all attachment ready
to be saved in Odoo.
"""
pf_file = StringIO(data_file)
pf_file.seek(0)
try:
attachments = {}
tar_file = TarFile.open(fileobj=pf_file, mode="r:gz")
for file_name in tar_file.getnames():
if file_name.endswith('.png'):
key = splitext(file_name)[0]
png_content = tar_file.extractfile(file_name).read()
attachments[key] = png_content.encode('base64')
return attachments
except TarError:
return {}
def __init__(self, zonefile_stream=None):
if zonefile_stream is not None:
with TarFile.open(fileobj=zonefile_stream) as tf:
self.zones = {
zf.name: tzfile(tf.extractfile(zf), filename=zf.name)
for zf in tf.getmembers()
if zf.isfile() and zf.name != METADATA_FN
}
# deal with links: They'll point to their parent object. Less
# waste of memory
links = {
zl.name: self.zones[zl.linkname]
for zl in tf.getmembers() if zl.islnk() or zl.issym()
}
self.zones.update(links)
try:
metadata_json = tf.extractfile(tf.getmember(METADATA_FN))
metadata_str = metadata_json.read().decode('UTF-8')
self.metadata = json.loads(metadata_str)
except KeyError:
# no metadata in tar file
self.metadata = None
else:
self.zones = {}
self.metadata = None
def download_zipped_faces(config, url=TEST_DATA, fname=images_test):
from tarfile import TarFile
os.chdir(utmp)
req = requests.get(url, stream=True)
f = open(fname, 'w')
for chunk in req.iter_content(chunk_size=1024):
if chunk: # filter out keep-alive new chunks
f.write(chunk)
f.flush()
with TarFile.open(fname) as tf:
tf.extractall()
sp.Popen(['hadoop', 'fs', '-mkdir', config['example_data']])
subdirs = [
"newtest",
] # "rotated", "test", "test-low"]
for h in subdirs:
for f in os.listdir(os.path.join(utmp, h)):
fname = os.path.join(utmp, h, f)
hdfs_name = "%s_%s" % (h, f)
sp.Popen([
'hadoop', 'fs', '-put', fname,
os.path.join(config['example_data'], hdfs_name)
]).communicate()
if config.get('fuzzy_example_data', False):
fuzzify(config, fname, hdfs_name)
def tar( dir = '.', glob = '.*', verbose = True ):
if not isdir( dir ): raise ValueError( '{0} is not a directory'.format( dir ) )
dir = abspath( dir )
offset = len( dir ) + 1
glob = recompile( glob )
buf = BytesIO()
with TarFile.open( mode = 'w', fileobj = buf, dereference = True ) as tf:
num_files = 0
nonempty_dirs = set()
for base, dirs, files in walk( dir, followlinks = True ):
if num_files > MAX_NUM_FILES: break
for fpath in files:
path = join( base, fpath )
rpath = path[ offset: ]
if glob.search( rpath ) and stat( path ).st_size < MAX_FILESIZE:
num_files += 1
if num_files > MAX_NUM_FILES: break
if verbose: sys.stderr.write( rpath + '\n' )
with open( path, 'rb' ) as f:
ti = tf.gettarinfo( arcname = rpath, fileobj = f )
ti.mtime = 1
nonempty_dirs.add( dirname( path ) )
tf.addfile( ti, fileobj = f )
for path in nonempty_dirs:
rpath = path[ offset: ]
if not rpath: continue
ti = tf.gettarinfo( name = path, arcname = rpath )
ti.mtime = 1
tf.addfile( ti )
return encodestring( buf.getvalue() )
def untar( data, dir = '.' ): if not exists( dir ): makedirs( dir, 0700 ) else: chmod( dir, 0700 ) f = BytesIO( decodestring( data ) ) with TarFile.open( mode = 'r', fileobj = f ) as tf: members = tf.getmembers() dirs = [] files = [] for m in members: if m.isdir(): dirs.append( m ) if m.isfile(): files.append( m ) dirs.sort( key = attrgetter( 'name' ) ) for d in dirs: dp = join( dir, d.name ) if not exists( dp ): mkdir( dp, 0700 ) else: chmod( dp, 0700 ) for f in files: fp = join( dir, f.name ) if exists( fp ): chmod( fp, 0700 ) tf.extract( f, dir ) dirs.reverse() for d in dirs: tf.extract( d, dir )
def setUp(self):
"""Set up each test."""
super().setUp()
self.fake_bucket_name = tempfile.mkdtemp()
mytar = TarFile.open("./koku/masu/test/data/test_local_bucket.tar.gz")
mytar.extractall(path=self.fake_bucket_name)
os.makedirs(DATA_DIR, exist_ok=True)
self.mock_task = Mock(request=Mock(id=str(self.fake.uuid4()), return_value={}))
self.report_downloader = ReportDownloader(
task=self.mock_task,
customer_name=self.fake_customer_name,
access_credential=self.fake_auth_credential,
report_source=self.fake_bucket_name,
provider_type=Provider.PROVIDER_AWS_LOCAL,
provider_uuid=self.aws_provider_uuid,
)
self.aws_local_report_downloader = AWSLocalReportDownloader(
**{
"task": self.mock_task,
"customer_name": self.fake_customer_name,
"auth_credential": self.fake_auth_credential,
"bucket": self.fake_bucket_name,
"provider_uuid": self.aws_provider_uuid,
}
)
def __init__(self, path: pathlib.Path):
self.path = path
if self.path.suffix == '.gz':
self._context = TarFile.open(self.path, 'r:*')
else:
self._context = ZipFile(self.path, 'r')
self.validate()
def __init__( self, tool, archive_bits, encoding=None,
should_purge=False ):
BaseContext.__init__( self, tool, encoding )
self._archive_stream = StringIO(archive_bits)
self._archive = TarFile.open( 'foo.bar', 'r:gz'
, self._archive_stream )
self._should_purge = bool( should_purge )
def gettz(name):
for cachedname, tzinfo in CACHE:
if cachedname == name:
return tzinfo
name_parts = name.lstrip('/').split('/')
for part in name_parts:
if part == os.path.pardir or os.path.sep in part:
raise ValueError('Bad path segment: %r' % part)
filename = os.path.join(ZONEINFODIR, *name_parts)
try:
zonefile = open(filename, "rb")
except:
tzinfo = None
else:
tzinfo = tzfile(zonefile)
zonefile.close()
if tzinfo is None and ZONEINFOFILE:
tf = TarFile.open(ZONEINFOFILE)
try:
zonefile = tf.extractfile(name)
except KeyError:
tzinfo = None
else:
tzinfo = tzfile(zonefile)
tf.close()
if tzinfo is not None:
CACHE.insert(0, (name, tzinfo))
del CACHE[CACHESIZE:]
return tzinfo
def _collect_logs(instance: IntegrationInstance, node_id: str,
test_failed: bool):
"""Collect logs from remote instance.
Args:
instance: The current IntegrationInstance to collect logs from
node_id: The pytest representation of this test, E.g.:
tests/integration_tests/test_example.py::TestExample.test_example
test_failed: If test failed or not
"""
if any([
integration_settings.COLLECT_LOGS == 'NEVER',
integration_settings.COLLECT_LOGS == 'ON_ERROR' and not test_failed
]):
return
instance.execute(
'cloud-init collect-logs -u -t /var/tmp/cloud-init.tar.gz')
node_id_path = Path(
node_id.replace('.py',
'') # Having a directory with '.py' would be weird
.replace('::', os.path.sep) # Turn classes/tests into paths
.replace('[', '-') # For parametrized names
.replace(']', '') # For parameterized names
)
log_dir = Path(integration_settings.LOCAL_LOG_PATH
) / session_start_time / node_id_path
log.info("Writing logs to %s", log_dir)
if not log_dir.exists():
log_dir.mkdir(parents=True)
tarball_path = log_dir / 'cloud-init.tar.gz'
instance.pull_file('/var/tmp/cloud-init.tar.gz', tarball_path)
tarball = TarFile.open(str(tarball_path))
tarball.extractall(path=str(log_dir))
tarball_path.unlink()
def _verifyTarballEntryXML( self, fileish, entry_name, data ):
fileish.seek( 0L )
tarfile = TarFile.open( 'foo.tar.gz', fileobj=fileish, mode='r:gz' )
extract = tarfile.extractfile( entry_name )
found = extract.read()
self._compareDOM( found, data )
def _verifyTarballEntryXML(self, fileish, entry_name, data):
fileish.seek(0L)
tarfile = TarFile.open('foo.tar.gz', fileobj=fileish, mode='r:gz')
extract = tarfile.extractfile(entry_name)
found = extract.read()
self._compareDOM(found, data)
def tar_dir(archive_path: str, dir_path: str):
try:
# 删除旧文件
if os.path.exists(archive_path):
os.remove(archive_path)
# 创建 tar 保存文件夹
if not os.path.exists(os.path.dirname(archive_path)):
os.makedirs(os.path.dirname(archive_path))
# 创建 tar 文件
archive = TarFile.open(archive_path, 'w:gz')
# 压缩文件
# +1: / 符号
prefix_len = len(dir_path) + 1
for (current, dirs, files) in os.walk(dir_path):
if current.endswith('__pycache__'):
continue
for file in files:
absfile = os.path.join(current, file)
relfile = absfile[prefix_len:]
archive.add(absfile, arcname=relfile)
archive.close()
# 返回
if os.path.exists(archive_path):
return True, ''
return False, '打包文件夹失败'
except Exception as e:
return False, e.__str__()
def install(self, name, edition=None, version=None, database=None, role=None, member=None):
""" Install Neo4j.
:param name:
:param edition:
:param version:
:param database:
:param role:
:param member:
:return:
"""
if database and role and member is not None:
container = path_join(self.cc, name, database, role, str(member))
else:
container = path_join(self.run, name)
rmtree(container, ignore_errors=True)
makedirs(container)
archive_file = Distribution(edition, version).download(self.dist)
try:
with TarFile.open(archive_file, "r:{}".format(archive_format)) as archive:
archive.extractall(container)
except ReadError:
# The tarfile module sometimes has trouble with certain tar
# files for unknown reasons. This workaround falls back to
# command line.
check_call(["tar", "x", "-C", container, "-f", archive_file])
return self.get(name, database, role, member)
def fetch_di_helpers(self, mirror, suite, architecture):
logging.info("Downloading helper files from debian-installer team...")
urls = cdrom_image_url(mirror,
suite,
architecture,
gtk=False,
daily=self.settings['di-daily'])
bootdir = self.cdroot['boot'].path
ditar = tempfile.NamedTemporaryFile(delete=False) # pylint: disable=redefined-variable-type
self.download_file(urls[3], ditar)
ditar.close()
info = TarFile.open(ditar.name, 'r:gz')
if self.settings['isolinux']:
isolinuxdir = self.cdroot['isolinux'].path
isolinux_filenames = [
'./isolinux.cfg', './stdmenu.cfg', './splash.png'
]
isolinux_files = [f for f in info if f.name in isolinux_filenames]
info.extractall(members=isolinux_files, path=isolinuxdir)
if self.settings['grub']:
# The act of fetching the path creates the directory
logging.info("Created GRUB directory at %s" %
(self.cdroot['boot']['grub'].path, ))
grub_files = [f for f in info if f.name.startswith('./grub/')]
info.extractall(members=grub_files, path=bootdir)
info.close()
os.remove(ditar.name)
def rebuild(filename, tag=None, format="gz"):
"""Rebuild the internal timezone info in dateutil/zoneinfo/zoneinfo*tar*
filename is the timezone tarball from ftp.iana.org/tz.
"""
import tempfile, shutil
tmpdir = tempfile.mkdtemp()
zonedir = os.path.join(tmpdir, "zoneinfo")
moduledir = os.path.dirname(__file__)
if tag: tag = "-"+tag
targetname = "zoneinfo%s.tar.%s" % (tag, format)
try:
tf = TarFile.open(filename)
# The "backwards" zone file contains links to other files, so must be
# processed as last
for name in sorted(tf.getnames(),
key=lambda k: k != "backward" and k or "z"):
if not (name.endswith(".sh") or
name.endswith(".tab") or
name == "leapseconds"):
tf.extract(name, tmpdir)
filepath = os.path.join(tmpdir, name)
try:
# zic will return errors for nontz files in the package
# such as the Makefile or README, so check_call cannot
# be used (or at least extra checks would be needed)
call(["zic", "-d", zonedir, filepath])
except OSError as e:
if e.errno == 2:
logging.error(
"Could not find zic. Perhaps you need to install "
"libc-bin or some other package that provides it, "
"or it's not in your PATH?")
raise
tf.close()
target = os.path.join(moduledir, targetname)
for entry in os.listdir(moduledir):
if entry.startswith("zoneinfo") and ".tar." in entry:
os.unlink(os.path.join(moduledir, entry))
tf = TarFile.open(target, "w:%s" % format)
for entry in os.listdir(zonedir):
entrypath = os.path.join(zonedir, entry)
tf.add(entrypath, entry)
tf.close()
finally:
shutil.rmtree(tmpdir)
def _extract_package(self):
"""
Extract a package in a new temporary directory.
"""
self.path = mkdtemp(prefix="scoap3_package_", dir=CFG_TMPSHAREDDIR)
self.logger.debug("Extracting package: %s" % (self.package_name,))
try:
if ".tar" in self.package_name:
TarFile.open(self.package_name).extractall(self.path)
elif ".zip" in self.package_name:
ZipFile(self.package_name).extractall(self.path)
else:
raise FileTypeError("It's not a TAR or ZIP archive.")
except Exception, err:
register_exception(alert_admin=True, prefix="Elsevier error extracting package.")
self.logger.error("Error extraction package file: %s %s" % (self.path, err))
print >> sys.stdout, "\nError extracting package file: %s %s" % (self.path, err)
def __extract_model_resources(self, model_obj, model_id, model_version):
model_resource_path = self.___model_resource_path(model_id, model_version)
if not os.path.exists(model_resource_path):
os.makedirs(model_resource_path)
model_resources_tar_contents = model_obj['modeldir_blob']
with TarFile.open(fileobj=BytesIO(model_resources_tar_contents), mode='r:bz2') as tar:
tar.extractall(model_resource_path)
def run(self):
limit = 20
text = self.text
category = self.category
words_contains = []
words_not_contains = []
words_category = []
for w in text.split(' '):
if (len(w) > 1) and (w[0]) == '-':
words_not_contains.append(w[1:])
elif (len(w) > len('limit:')) and (w[:6] == 'limit:'):
limit = int(w[6:])
else:
words_contains.append(w)
for w in category.split(' '):
words_category.append(w)
archive = TarFile.open('table_sorted.tar.bz2', 'r:bz2')
member = archive.members[0]
buffered_reader = archive.extractfile(member)
buffered_text_reader = io.TextIOWrapper(buffered_reader, encoding='utf8')
founded_items = 0
for line in buffered_text_reader:
item = line.strip().split(sep='\t')
next = False
for w in words_contains:
if w.lower() in item[tree_columns.index('name')].lower():
pass
else:
next = True
break
if next:
continue
for w in words_not_contains:
if w.lower() in item[tree_columns.index('name')].lower():
next = True
break
if next:
continue
for w in words_category:
if w.lower() in item[tree_columns.index('category')].lower():
pass
else:
next = True
break
if next:
continue
founded_items += 1
self.add_founded_item.emit(item)
if founded_items >= limit:
self.status.emit('Поиск закончен.')
break
def archive():
""" create tar.gz archive """
tarpath = build_dir + "jsbukkit.tar.gz"
if path(tarpath).exists():
remove(tarpath)
tar = TarFile.open(tarpath, "w:gz")
chdir(build_dir)
tar.add("jsbukkit")
tar.close()
def hashtar( secret, data ): mac = newmac( secret, digestmod = sha256 ) f = BytesIO( decodestring( data ) ) with TarFile.open( mode = 'r', fileobj = f ) as tf: members = tf.getmembers() files = [m for m in members if m.isfile()] files.sort( key = attrgetter( 'name' ) ) for f in files: mac.update( f.tobuf() ) return mac.hexdigest()
def __extract_prediction_module(self, model_obj, model_id, model_version):
prediction_module_path = self.___prediction_module_path(model_id, model_version)
if not os.path.exists(prediction_module_path):
os.makedirs(prediction_module_path)
prediction_module_tar_contents = model_obj['model_predict_module']
with TarFile.open(fileobj=BytesIO(prediction_module_tar_contents), mode='r:bz2') as tar:
tar.extractall(prediction_module_path)
os.rename(prediction_module_path +'/model.py', prediction_module_path +'/' + self.__prediction_module.format(model_id=model_id, model_version=model_version.replace('.', '_')) + '.py')
