def tree_in_zip(path, stream=None): print('Trying {}'.format(path)) if stream is not None and ZIP_FILE.search(path) is not None: zf = ZipFile(stream) #Check if the file is not the first in the loop and wheter it is a .zip. elif stream is None and ZIP_FILE.search(path) is not None: zf = ZipFile(path) #Check if the first file is .zip. else: if stream is not None and TAR_FILE.search(path) is not None : try: zf=TarFile.open(fileobj=stream, encoding="utf-8",mode='r:*') except ReadError: pass #If the file is not the first in the loop and it is a .tar(.xx) tries to read it as an archive. elif stream is None and TAR_FILE.search(path) is not None: try: zf=TarFile.open(path, mode='r:*') except ReadError: pass #If the first file is a .tar(.xx) tries to read it as an archive. try: if type(zf)==ZipFile: try: for name in zf.namelist(): count_extensions(name, EXTENSIONS, NR_EXTENSIONS) #Count extensions of interest in the archive. manifesto_maker(zf, name, MANIFESTO) #fills in the dictionary if ZIP_FILE.search(name) is not None: print('-Found {}'.format(name)) yield from tree_in_zip(path+'/'+name, BytesIO(zf.read(name))) elif TAR_FILE.search(name) is not None: print('-Found {}'.format(name)) yield from tree_in_zip(path+'/'+name, BytesIO(zf.read(name))) else: yield path+'/'+name except BadZipFile: pass #Search for further archives (.zip/.tar). Exception needed to not to stop at a corrupted archive. elif type(zf)==TarFile: #No need for try checked the file at the begining. for name in zf.getnames(): count_extensions(name, EXTENSIONS, NR_EXTENSIONS) manifesto_maker(zf, name, MANIFESTO) if ZIP_FILE.search(name) is not None: print('-Found {}'.format(name)) yield from tree_in_zip(path+'/'+name, BytesIO(zf.read(name))) elif TAR_FILE.search(name) is not None: print('- Found {}'.format(name)) yield from tree_in_zip(path+'/'+name, BytesIO(zf.read(name))) else: yield path+'/'+name else: pass #if file is not .tar/.zip skip it except UnboundLocalError: pass
def rebuild(filename, tag=None, format="gz"): import tempfile, shutil tmpdir = tempfile.mkdtemp() zonedir = os.path.join(tmpdir, "zoneinfo") moduledir = os.path.dirname(__file__) if tag: tag = "-" + tag targetname = "zoneinfo%s.tar.%s" % (tag, format) try: tf = TarFile.open(filename) # The "backwards" zone file contains links to other files, so must be # processed as last for name in sorted(tf.getnames(), key=lambda k: k != "backward" and k or "z"): if not (name.endswith(".sh") or name.endswith(".tab") or name == "leapseconds"): tf.extract(name, tmpdir) filepath = os.path.join(tmpdir, name) os.system("zic -d %s %s" % (zonedir, filepath)) tf.close() target = os.path.join(moduledir, targetname) for entry in os.listdir(moduledir): if entry.startswith("zoneinfo") and ".tar." in entry: os.unlink(os.path.join(moduledir, entry)) tf = TarFile.open(target, "w:%s" % format) for entry in os.listdir(zonedir): entrypath = os.path.join(zonedir, entry) tf.add(entrypath, entry) tf.close() finally: shutil.rmtree(tmpdir)
def rebuild(filename, tag=None, format="gz", zonegroups=[], metadata=None): """Rebuild the internal timezone info in dateutil/zoneinfo/zoneinfo*tar* filename is the timezone tarball from ``ftp.iana.org/tz``. """ tmpdir = tempfile.mkdtemp() zonedir = os.path.join(tmpdir, "zoneinfo") moduledir = os.path.dirname(__file__) try: with TarFile.open(filename) as tf: for name in zonegroups: tf.extract(name, tmpdir) filepaths = [os.path.join(tmpdir, n) for n in zonegroups] try: check_call(["zic", "-d", zonedir] + filepaths) except OSError as e: _print_on_nosuchfile(e) raise # write metadata file with open(os.path.join(zonedir, METADATA_FN), 'w') as f: json.dump(metadata, f, indent=4, sort_keys=True) target = os.path.join(moduledir, ZONEFILENAME) with TarFile.open(target, "w:%s" % format) as tf: for entry in os.listdir(zonedir): entrypath = os.path.join(zonedir, entry) tf.add(entrypath, entry) finally: shutil.rmtree(tmpdir)
def rebuild(filename, tag=None, format="gz"): import tempfile, shutil tmpdir = tempfile.mkdtemp() zonedir = os.path.join(tmpdir, "zoneinfo") moduledir = os.path.dirname(__file__) if tag: tag = "-"+tag targetname = "zoneinfo%s.tar.%s" % (tag, format) try: tf = TarFile.open(filename) # The "backwards" zone file contains links to other files, so must be # processed as last for name in sorted(tf.getnames(), key=lambda k: k != "backward" and k or "z"): if not (name.endswith(".sh") or name.endswith(".tab") or name == "leapseconds"): tf.extract(name, tmpdir) filepath = os.path.join(tmpdir, name) os.system("zic -d %s %s" % (zonedir, filepath)) tf.close() target = os.path.join(moduledir, targetname) for entry in os.listdir(moduledir): if entry.startswith("zoneinfo") and ".tar." in entry: os.unlink(os.path.join(moduledir, entry)) tf = TarFile.open(target, "w:%s" % format) for entry in os.listdir(zonedir): entrypath = os.path.join(zonedir, entry) tf.add(entrypath, entry) tf.close() finally: shutil.rmtree(tmpdir)
def rebuild(filename, tag=None, format="gz", zonegroups=[], metadata=None): """Rebuild the internal timezone info in dateutil/zoneinfo/zoneinfo*tar* filename is the timezone tarball from ftp.iana.org/tz. """ tmpdir = tempfile.mkdtemp() zonedir = os.path.join(tmpdir, "zoneinfo") moduledir = os.path.dirname(__file__) try: with TarFile.open(filename) as tf: for name in zonegroups: tf.extract(name, tmpdir) filepaths = [os.path.join(tmpdir, n) for n in zonegroups] try: check_call(["zic", "-d", zonedir] + filepaths) except OSError as e: _print_on_nosuchfile(e) raise # write metadata file with open(os.path.join(zonedir, METADATA_FN), 'w') as f: json.dump(metadata, f, indent=4, sort_keys=True) target = os.path.join(moduledir, ZONEFILENAME) with TarFile.open(target, "w:%s" % format) as tf: for entry in os.listdir(zonedir): entrypath = os.path.join(zonedir, entry) tf.add(entrypath, entry) finally: shutil.rmtree(tmpdir)
def install_packages(self): self._update_package_with_install_path() installed_packages = [] for p in self._packages: if 'install_from' not in p: print("[dem] Could not find package: {}, version: {}".format(p['name'], p['version'])) else: if not self._cache.is_package_installed(p['name'], p['version']): print('[dem] installing {}-{}'.format(p['name'], p['version'])) if p['install_from_ext'] == 'zip': with ZipFile(p['install_from'], 'r') as archive: locations = self._extract(archive, p) elif p['install_from_ext'] == 'tar.gz': with TarFile.open(p['install_from'], 'r:gz') as archive: locations = self._extract(archive, p) elif p['install_from_ext'] == 'tar.bz2': with TarFile.open(p['install_from'], 'r:bz2') as archive: locations = self._extract(archive, p) elif p['install_from_ext'] == 'gz': with gzip.open(p['install_from'], 'r') as archive: locations = self._extract(archive, p) if 'pkg-config' in p: PkgConfigProcessor.replace_prefix(locations, p['pkg-config']) else: print('[dem] {}-{} already installed'.format(p['name'], p['version'])) locations = self._cache.install_locations(p['name']) package = dict() package[p['name']] = {'version': p['version'], 'type': 'local', 'install_locations': locations} installed_packages.append(package) return installed_packages
def rebuild(filename, tag=None, format="gz"): import tempfile, shutil tmpdir = tempfile.mkdtemp() zonedir = os.path.join(tmpdir, "zoneinfo") moduledir = os.path.dirname(__file__) if tag: tag = "-"+tag targetname = "zoneinfo%s.tar.%s" % (tag, format) try: tf = TarFile.open(filename) for name in tf.getnames(): if not (name.endswith(".sh") or name.endswith(".tab") or name == "leapseconds"): tf.extract(name, tmpdir) filepath = os.path.join(tmpdir, name) os.system("zic -d %s %s" % (zonedir, filepath)) tf.close() target = os.path.join(moduledir, targetname) for entry in os.listdir(moduledir): if entry.startswith("zoneinfo") and ".tar." in entry: os.unlink(os.path.join(moduledir, entry)) tf = TarFile.open(target, "w:%s" % format) for entry in os.listdir(zonedir): entrypath = os.path.join(zonedir, entry) tf.add(entrypath, entry) tf.close() finally: shutil.rmtree(tmpdir)
def _populate_metadata(self): with TemporaryDirectory() as tempdir: meta_src = os.path.join(tempdir, self.META_TEMPLATE_TARGET) meta_dest = os.path.join(tempdir, self.META_TARGET) tarball_dest = os.path.join(tempdir, os.path.basename(self.filename)) if self.remote_file: download(self.filename, tarball_dest) print('Extracting {0}'.format(tarball_dest)) try: with TarFile.open(tarball_dest, 'r:*') as tarball: tarball.extract(self.META_TEMPLATE_TARGET, tempdir) except KeyError: try: # Older version of conda-build: Use META_TARGET instead self.old_behavior = True with TarFile.open(tarball_dest, 'r:*') as tarball: tarball.extract(self.META_TARGET, tempdir) except KeyError: print('Proprietary package lacks required data!') print() return None if not self.old_behavior: shutil.move(meta_src, meta_dest) print() return MetaData(meta_dest)
def archiveFromGit(versionedPackageName, committish): prefix = '%s/' % versionedPackageName distBase = 'derived/dist/' umask = (stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH) def exclude(info): '''Returns True iff the given tar entry should be excluded. ''' return any(( info.name.endswith('/.gitignore'), info.name.startswith(prefix + 'doc/internal'), )) proc = Popen( ('git', 'archive', '--prefix=' + prefix, '--format=tar', committish), bufsize=-1, stdin=None, stdout=PIPE, stderr=sys.stderr, ) try: outTarPath = distBase + versionedPackageName + '.tar.gz' print('archive:', outTarPath) if not isdir(distBase): makedirs(distBase) try: with TarFile.open(outTarPath, 'w:gz') as outTar: # Copy entries from "git archive" into output tarball, # except for excluded entries. numIncluded = numExcluded = 0 with TarFile.open(mode='r|', fileobj=proc.stdout) as inTar: for info in inTar: if exclude(info): if verbose: print('EX', info.name) numExcluded += 1 else: if verbose: print('IN', info.name) numIncluded += 1 info.uid = info.gid = 1000 info.uname = info.gname = 'openmsx' info.mode = info.mode & umask outTar.addfile(info, inTar.extractfile(info)) print('entries: %d included, %d excluded' % (numIncluded, numExcluded)) except: # Clean up partial output file. remove(outTarPath) raise except: proc.terminate() raise else: data, _ = proc.communicate() if len(data) != 0: print('WARNING: %d more bytes of data from "git archive" after ' 'tar stream ended' % len(data), file=sys.stderr)
def test_getclasstgz(self): api = WimsAPI(WIMS_URL, "myself", "toto") status, response = api.getclasstgz(999999, "myclass") self.assertTrue(status) try: TarFile.open(fileobj=BytesIO(response), mode="r:gz") except TarError as e: self.fail("Response was not a valid tgz :\n" + str(e))
def createPluginStructure(): global BUILD_PATH if os.path.exists(BUILD_PATH): shutil.rmtree(BUILD_PATH) # prevent win 8 io error time.sleep(1) if not os.path.exists(BUILD_PATH): os.makedirs(BUILD_PATH) os.chdir(BUILD_PATH) BUILD_PATH = os.getcwd() control_path = path_join(BUILD_PATH, "CONTROL") os.mkdir(control_path) shutil.move(PLUGIN_HASH_FILE, path_join(BUILD_PATH, "CONTROL")) createControl(control_path) #createConfFiles(control_path) #createPreInst(control_path) #createPostInst(control_path) #createPreRM(control_path) createPostRM(control_path) createDebianBinary() native_tar = False if not native_tar: tar = TarFile.open("control.tar.gz", "w:gz") tar.add("CONTROL", ".", filter=tar_filter) tar.close() tar = TarFile.open("data.tar.gz", "w:gz") tar.add(PLUGIN) tar.add(COMPONENTS_PATH) tar.close() # tar_name = path_join(BUILD_PATH, "control.tar.gz") # os.chdir(path_join(BUILD_PATH, "CONTROL")) # os.system("chmod 755 *") # makeTarGz(".", tar_name) # os.chdir(BUILD_PATH) # tar_name = path_join(BUILD_PATH, "data.tar.gz") # makeTarGz(PLUGIN, tar_name) if native_tar: os.system("chmod 755 CONTROL/*") cmd = "tar -C CONTROL -czf control.tar.gz ." os.system("chmod 755 CONTROL/*") os.system(cmd) cmd = "tar -C . -czf data.tar.gz %s" % (PLUGIN) os.system(cmd)
def __enter__(self): if self.tf is not None: raise ValueError('Cannot re-enter') if '://' in self.web_archive: info('Downloading from {0}'.format(self.web_archive)) dl = requests.get(self.web_archive) self.tf = TarFile.open(path(self.web_archive).basename(), 'r:*', fileobj=io.BytesIO(dl.content)) else: self.tf = TarFile.open(self.web_archive) return self.tf
def extract_package(package_name, path, logger): try: if ".tar" in package_name: TarFile.open(package_name).extractall(path) elif ".zip" in package_name: ZipFile(package_name).extractall(path) else: raise FileTypeError("It's not a TAR or ZIP archive.") except Exception as err: register_exception(alert_admin=True, prefix="Elsevier error extracting package.") logger.error("Error extraction package file: %s %s" % (path, err))
def setUp(self): """Set up each test.""" super().setUp() self.fake_bucket_name = tempfile.mkdtemp() mytar = TarFile.open('./koku/masu/test/data/test_local_bucket.tar.gz') mytar.extractall(path=self.fake_bucket_name) os.makedirs(DATA_DIR, exist_ok=True) self.mock_task = Mock(request=Mock(id=str(self.fake.uuid4()), return_value={})) self.report_downloader = ReportDownloader( task=self.mock_task, customer_name=self.fake_customer_name, access_credential=self.fake_auth_credential, report_source=self.fake_bucket_name, provider_type='AWS-local', provider_uuid=self.aws_provider_uuid, ) self.aws_local_report_downloader = AWSLocalReportDownloader( **{ 'task': self.mock_task, 'customer_name': self.fake_customer_name, 'auth_credential': self.fake_auth_credential, 'bucket': self.fake_bucket_name, 'provider_uuid': self.aws_provider_uuid, } )
def _extract_file(self, fname, extract_dir): """ This method receives an argument for the archive to extract and the destination path. """ with TarFile.open(fname, "r") as tar_file: if self.members is None: get_logger().info( "Untarring contents of '%s' to '%s'", fname, extract_dir ) # Unpack all files from the archive into our new folder tar_file.extractall(path=extract_dir) else: for member in self.members: get_logger().info( "Extracting '%s' from '%s' to '%s'", member, fname, extract_dir ) # make sure the target folder exists for nested members parts = Path(member).parts if len(parts) > 1: full_dir_path = os.path.join(extract_dir, *parts[:-1]) os.makedirs(full_dir_path, exist_ok=True) # Extract the data file from within the archive # Python 2.7: extractfile doesn't return a context manager data_file = tar_file.extractfile(member) try: # Save it to our desired file name with open(os.path.join(extract_dir, member), "wb") as output: output.write(data_file.read()) finally: data_file.close()
def _import_gpg_keyring(self, abs_temp_dir, abs_gpg_home_dir, package_filename, package_yyyymmdd): rel_archlinux_gpg_path = 'archlinux-keyring-%s/archlinux.gpg' % package_yyyymmdd with TarFile.open(package_filename) as tf: tf.extract(rel_archlinux_gpg_path, path=abs_temp_dir) abs_archlinux_gpg_path = os.path.join(abs_temp_dir, rel_archlinux_gpg_path) self._import_gpg_key_file(abs_gpg_home_dir, abs_archlinux_gpg_path)
def _get_bin_for_release(self, release_file_path: Path): if not release_file_path.exists(): raise ValueError(f'Release file {release_file_path} not found') if release_file_path.suffix == '.gz': opener = TarFile.open(release_file_path, 'r:*') else: opener = ZipFile(release_file_path, 'r') with opener as archive: if release_file_path.suffix == '.gz': contents = archive.getnames() else: contents = archive.namelist() if len(contents) != 1: raise ValueError( f'Release archive has unexpected content. ' f'Expected 1 file, found {len(contents)}: {", ".join(contents)}', ) bin_file_path = self._bin_path.joinpath(contents[0]) if not bin_file_path.exists(): log.debug( 'Extracting Raiden binary', release_file_name=release_file_path.name, bin_file_name=bin_file_path.name, ) archive.extract(contents[0], self._bin_path) bin_file_path.chmod(0o770) return bin_file_path
def open_archive(path): from tarfile import TarFile archive = TarFile.open(name=path, mode="w:gz", bufsize=16*1024) try: yield archive finally: archive.close()
def download(archive_url, archive_name, extract_to_path='.', require_basic_auth=False): # download the file to extract_to_path if not path.exists(extract_to_path): makedirs(extract_to_path) archive_path = path.join(extract_to_path, archive_name) stdout.write("Downloading '%s' to '%s'...\n" % (archive_url, archive_path)) source = urlopen(archive_url) if not require_basic_auth else teamcityurlopen(archive_url) with open(archive_path, "wb") as destination: more = True while more: data = source.read(8192) if data: destination.write(data) else: more = False if archive_name.endswith('.zip'): stdout.write("Unzipping '%s' to '%s'...\n" % (archive_path, extract_to_path)) zip_ref = ZipFile(archive_path, 'r') zip_ref.extractall(extract_to_path) unzip_folder = zip_ref.namelist()[0] zip_ref.close() return unzip_folder elif archive_name.endswith('.tar.gz'): stdout.write("Unarchiving '%s' to '%s'...\n" % (archive_path, extract_to_path)) tar_ref = TarFile.open(archive_path) tar_ref.extractall(extract_to_path) untar_folder=tar_ref.getnames()[0] tar_ref.close() return untar_folder
def push(self, local, remote): if os.path.isdir(local): if not os.path.exists(self.cache): os.mkdir(self.cache) cached_local = local.replace('/', '_')[1:] + '.tar.gz' cached_file = os.path.join(self.cache, cached_local) if not os.path.exists(cached_file): log.debug('Creating cache file %s', cached_file) tar = TarFile.open(cached_file, 'w:gz') tar.add(os.path.realpath(local), arcname='.') log.debug('Done...') tar.close() with open(cached_file,'r') as f: self.run('mkdir -p {0}'.format(remote)) return self.run( 'tar zxf - -C {0}'.format(remote), in_=ProcessFile(f) ) else: with open(local) as f: return self.run( 'cat > {0}'.format(remote), in_=ProcessFile(f) )
def upload(self, token, metadata, reader): name = metadata["name"] version = metadata["version"] rep = self.repos.get(name) if rep is None: where = os.path.join(self.root, name) need_init = True if os.path.exists(where): self.warning("%s exists - cleared", where) shutil.rmtree(where) os.mkdir(where) else: where = dirname(rep.path) need_init = False with TarFile.open(mode="r|gz", fileobj=reader) as tar: tar.extractall(where) if not need_init: self.add_version(rep, version) else: self.repos[name] = rep = pygit2.init_repository(where) try: self.add_version(rep, version) except Exception as e: shutil.rmtree(where) del self.repos[name] self.error("Failed to initialize %s", name) raise from_none(e) rep.config["forge.tokens"] = self.scramble(token) self._generate_images(metadata, rep)
def retrieveCoreInfo(): DIR_PREFIX = 'libretro-super-master/dist/info/' res = urllib2.urlopen('https://github.com/libretro/libretro-super/archive/master.tar.gz') archive = StringIO(res.read()) tf = TarFile.open(mode='r:gz', fileobj=archive) # filter files and sort them files = [f for f in tf.getmembers() if f.name.startswith(DIR_PREFIX)] files = sorted(files, key=lambda m: m.name.split(DIR_PREFIX)[1]) def infoToDict( files=[]): output = {'cores': OrderedDict()} for f in files: info = OrderedDict() for l in tf.extractfile(f).readlines(): s = shlex.shlex(l) s.quotes = '"' op = [] try: while True: t = s.get_token() if t == s.eof: break op.append(t) # 'var' '=' 'value' if len(op) == 3: assert op[1] in ('=', ':') # For: 'some:var = value' syntax # Merge 'some:var' into 'somevar' (?) if op[1] == ':': op = [''.join(op)] continue # Strip quotes if op[2][0] == '"' and op[2][-1] == '"': op[2] = op[2][1:-1] # Convert string to boolean if op[2] in ('true', 'True'): op[2] = True elif op[2] in ('false', 'False'): op[2] = False # Decode utf-8 into unicode if isinstance(op[2], str): op[2] = op[2].decode('utf-8') info[op[0]] = op[2] break except (ValueError, AssertionError) as e: print("Could not process file %s (%s) " % (f.name, e), file=sys.stderr) continue output['cores'][f.name.split(DIR_PREFIX)[1].split('.info')[0]] = info return output return infoToDict(files)
def handle(): try: if request.method == 'GET': return _as_text( app.config[ 'TAR_DIGEST' ] ) try: signature = request.form[ 'signature' ] except KeyError: uid = None else: uid = extract_uid( signature ) if not uid: EVENTS_LOG.info( 'Unauthorized: {0}@{1}'.format( uid, request.remote_addr ) ) return _as_text( '# {0}\n'.format( _( 'Invalid or absent signature!' ) ), 401 ) if 'tar' in request.form: # this is an upload data = decodestring( request.form[ 'tar' ] ) dest = join( app.config[ 'UPLOAD_DIR' ], uid, str( int( time() * 1000 ) ) + '.tar' ) with open( dest, 'w' ) as f: f.write( data ) tf = TarFile.open( dest, mode = 'r' ) names = tf.getnames() tf.close() EVENTS_LOG.info( 'Upload: {0}@{1}'.format( uid, request.remote_addr ) ) return _as_text( '\n'.join( sorted( names ) ) ) else: # this is a download EVENTS_LOG.info( 'Download: {0}@{1}'.format( uid, request.remote_addr ) ) return _as_text( app.config[ 'TAR_DATA' ] ) except: if app.debug: raise else: app.logger.exception( '' ) return _as_text( '# {0}\n'.format( _( 'An unexpected server error occurred!' ) ), 500 )
def _verifyTarballEntry(self, fileish, entry_name, data): fileish.seek(0) tarfile = TarFile.open('foo.tar.gz', fileobj=fileish, mode='r:gz') extract = tarfile.extractfile(entry_name) found = extract.read() self.assertEqual(found, data)
def fetch(self): """ Download the specififc workflow bundle. """ self.path = self.path or "" if not self.path or (os.path.exists(self.path) and os.path.samefile(self.path, os.getcwd())): self.path = os.path.join(self.path, "%s@%s" % (self.name, self.version)) if os.path.exists(self.path): if self.force: shutil.rmtree(self.path) else: raise ValueError("Directory %s already exists and -f/--force " "option was not specified" % self.path) os.mkdir(self.path) url = "%s%s?%s" % ( self.base, root.common.forge.fetch_name, urlencode((("name", self.name), ("version", self.version)))) self.debug("Requesting %s", url) fn = wget.download(url, self.path) print("") self.debug("Downloaded %s", fn) with TarFile.open(fn) as tar: tar.extractall(self.path) os.remove(fn) self.info("Put the following files into %s:\n %s", self.path, "\n ".join(sorted(os.listdir(self.path)))) metadata = self._parse_metadata(self.path) self._check_deps(metadata) return self.path, metadata
def test_windows_with_mocks(self): from infi.logs_collector.items import windows result, archive_path = logs_collector.run("test", windows(), None, None) self.assertTrue(path.exists(archive_path)) self.assertTrue(archive_path.endswith(".tar.gz")) archive = TarFile.open(archive_path, "r:gz")
def extract_blahp(self, distro): """ Extract the BLAHP shared libs and bins for the target platform and dump them to a temporary directory """ tempdir = tempfile.mkdtemp() self.log.debug("Temporary working directory: %s" % tempdir) tarfile = os.path.join(self.cachedir,self.version,"bosco-1.2-x86_64_" + distro + ".tar.gz") cdir = 'condor-8.6.6-x86_64_' + distro + '-stripped/' blahp_files = [ 'lib/libclassad.so.8.6.6', 'lib/libclassad.so.8', 'lib/libclassad.so', 'lib/libcondor_utils_8_6_6.so', 'sbin/condor_ft-gahp' ] blahp_dirs = [ 'lib/condor/', 'libexec/glite/bin', 'libexec/glite/etc' ] # open the tarball, extract blahp_files and blahp_dirs to tmp with TarFile.open(tarfile) as t: members = [] for f in blahp_files: members.append(t.getmember(os.path.join(cdir,f))) for d in blahp_dirs: match = os.path.join(cdir, d) files = [t.getmember(s) for s in t.getnames() if re.match(match, s)] members.extend(files) #self.log.debug("Extracting %s to %s" % (members, tempdir)) t.extractall(tempdir,members) # once things are in tmp, we need to need to move things around and # make some directories dirs = [ 'bosco/glite/log', 'bosco/sandbox' ] self.log.debug("Creating BOSCO directories...") for d in dirs: os.makedirs(os.path.join(tempdir, d)) # list of files and directories that need to move from the extracted tarball to the bosco dir to_move = ( ['lib','bosco/glite/lib'], ['libexec/glite/bin', 'bosco/glite/bin'], ['libexec/glite/etc', 'bosco/glite/etc'], ['sbin/condor_ft-gahp', 'bosco/glite/bin/condor_ft-gahp'] ) for t in to_move: src = os.path.join(tempdir,cdir,t[0]) dst = os.path.join(tempdir,t[1]) self.log.debug("Moving %s to %s" % (src,dst)) shutil.move(src,dst) self.log.debug("Deleting old directory: %s " % cdir) shutil.rmtree(os.path.join(tempdir,cdir)) return tempdir
def get(self, filename="", file_hash=""): """ Retrieve file information of hashed tree """ if filename: # File accesibility if not os.path.exists(filename): raise IOError('Unaccesible file') file_hash = self._get_hash(filename) if not file_hash: raise ValueError('Hash of file is mandatory') path, tarfile, hashed_filename = self._get_path(file_hash) # Tarfile accesibility if not os.path.exists(os.path.join(path, tarfile)): raise IOError('Unaccesible tarfile %s', os.path.join(path, tarfile)) tar = TarFile.open(name=os.path.join(path, tarfile), mode='r:%s' % self.external_compressor) data = tar.extractfile(hashed_filename) if not data: raise ValueError('Member does not exists') if self.internal_compressor: return self.decoder.decode( self.internal_compressor.decompress( data.read() ) ) else: return self.decoder.decode( data.read() )
def setUp(self): """Set up each test.""" super().setUp() self.fake_bucket_name = tempfile.mkdtemp() mytar = TarFile.open("./koku/masu/test/data/test_local_bucket.tar.gz") mytar.extractall(path=self.fake_bucket_name) os.makedirs(DATA_DIR, exist_ok=True) self.credentials = {"role_arn": self.fake_auth_credential} self.data_source = {"bucket": self.fake_bucket_name} self.report_downloader = ReportDownloader( customer_name=self.fake_customer_name, credentials=self.credentials, data_source=self.data_source, provider_type=Provider.PROVIDER_AWS_LOCAL, provider_uuid=self.aws_provider_uuid, ) self.aws_local_report_downloader = AWSLocalReportDownloader( **{ "customer_name": self.fake_customer_name, "credentials": self.credentials, "data_source": self.data_source, "provider_uuid": self.aws_provider_uuid, })
def put(self, content, filename="", file_hash=""): """ Store file information in hashed tree """ if not filename and not file_hash: raise ValueError('Filename or FileHash is mandatory') if filename: # File accesibility if not os.path.exists(filename): raise IOError('Unaccesible file %s', filename) # Calc hash file_hash = self._get_hash(filename) if not file_hash: raise ValueError('Hash of file is mandatory') # Get file path for hash path, tarfile, hashed_filename = self._get_path(file_hash) # Create file path try: os.makedirs(path) except WindowsError: pass except OSError: pass # Open tarfile if self.external_compressor: # External compressor is not suited for adding files. raise ValueError('You cannot use external compressor for write files') with TarFile.open(name=os.path.join(path, tarfile), mode='a') as tar: with FileLock(os.path.join(path, tarfile)) as lock: # Test if file already exists into tarfile try: tar.getmember(hashed_filename) raise ValueError('Member already exists') except KeyError: pass except: raise data = self.encoder.encode(content) if self.internal_compressor: data = self.internal_compressor.compress(data) data_file = StringIO(data) mtime = time.time() ti = TarInfo(hashed_filename) ti.size = data_file.len ti.mtime = mtime tar.addfile(tarinfo=ti, fileobj=data_file) tar.close() return file_hash
def test_download_bucket_with_prefix(self): """Test to verify that basic report downloading works.""" fake_bucket = tempfile.mkdtemp() mytar = TarFile.open( "./koku/masu/test/data/test_local_bucket_prefix.tar.gz") mytar.extractall(fake_bucket) test_report_date = datetime(year=2018, month=8, day=7) fake_data_source = {"bucket": fake_bucket} with patch.object(DateAccessor, "today", return_value=test_report_date): report_downloader = ReportDownloader( self.fake_customer_name, self.credentials, fake_data_source, Provider.PROVIDER_AWS_LOCAL, self.aws_provider_uuid, ) # Names from test report .gz file report_context = { "date": test_report_date.date(), "manifest_id": 1, "comporession": "GZIP", "current_file": "./koku/masu/test/data/test_local_bucket.tar.gz", } report_downloader.download_report(report_context) expected_path = "{}/{}/{}".format(DATA_DIR, self.fake_customer_name, "aws-local") self.assertTrue(os.path.isdir(expected_path)) shutil.rmtree(fake_bucket)
def run(self): self.ensure_directories_writable() self._messenger.info('Searching for latest release...') version_str = self._determine_latest_version() version_tuple = self._parse_version(version_str) self._messenger.info('Found {} to be latest.'.format(version_str)) tarball_download_url = self._create_tarball_download_url( version_tuple, self._architecture) signatur_download_url = '{}.asc'.format(tarball_download_url) # Signature first, so we fail earlier if we do abs_filename_signature = self._download_file(signatur_download_url) abs_filename_tarball = self._download_file(tarball_download_url) abs_temp_dir = os.path.abspath(tempfile.mkdtemp()) try: abs_gpg_home_dir = self._initialize_gpg_home(abs_temp_dir) release_pubring_gpg = resource_filename(resources.__name__, 'ncopa.asc') self._import_gpg_key_file(abs_gpg_home_dir, release_pubring_gpg) self._verify_file_gpg(abs_filename_tarball, abs_filename_signature, abs_gpg_home_dir) self._messenger.info('Extracting to "{}"...'.format( self._abs_target_dir)) with TarFile.open(abs_filename_tarball) as tf: tf.extractall(path=self._abs_target_dir) finally: self._messenger.info('Cleaning up "{}"...'.format(abs_temp_dir)) shutil.rmtree(abs_temp_dir)
def gettz(name): for cachedname, tzinfo in CACHE: if cachedname == name: return tzinfo name_parts = name.lstrip('/').split('/') for part in name_parts: if part == os.path.pardir or os.path.sep in part: raise ValueError('Bad path segment: %r' % part) filename = os.path.join(ZONEINFODIR, *name_parts) try: zonefile = open(filename, "rb") except: tzinfo = None else: tzinfo = tzfile(zonefile) zonefile.close() if tzinfo is None and ZONEINFOFILE: tf = TarFile.open(ZONEINFOFILE) try: zonefile = tf.extractfile(name) except KeyError: tzinfo = None else: tzinfo = tzfile(zonefile) tf.close() if tzinfo is not None: CACHE.insert(0, (name, tzinfo)) del CACHE[CACHESIZE:] return tzinfo
def _get_attachments_from_stream(self, data_file): """Retrive attachment from tar file. Return a dict containing all attachment ready to be saved in Odoo. The key is the name of file without extention The value the PNG content encoded in base64 :param data_file: raw statement file sent to openerp (not in b64) :type data_file: basestring subclass :return: Return a dict containing all attachment ready to be saved in Odoo. """ pf_file = StringIO(data_file) pf_file.seek(0) try: attachments = {} tar_file = TarFile.open(fileobj=pf_file, mode="r:gz") for file_name in tar_file.getnames(): if file_name.endswith('.png'): key = splitext(file_name)[0] png_content = tar_file.extractfile(file_name).read() attachments[key] = png_content.encode('base64') return attachments except TarError: return {}
def __init__(self, zonefile_stream=None): if zonefile_stream is not None: with TarFile.open(fileobj=zonefile_stream) as tf: self.zones = { zf.name: tzfile(tf.extractfile(zf), filename=zf.name) for zf in tf.getmembers() if zf.isfile() and zf.name != METADATA_FN } # deal with links: They'll point to their parent object. Less # waste of memory links = { zl.name: self.zones[zl.linkname] for zl in tf.getmembers() if zl.islnk() or zl.issym() } self.zones.update(links) try: metadata_json = tf.extractfile(tf.getmember(METADATA_FN)) metadata_str = metadata_json.read().decode('UTF-8') self.metadata = json.loads(metadata_str) except KeyError: # no metadata in tar file self.metadata = None else: self.zones = {} self.metadata = None
def download_zipped_faces(config, url=TEST_DATA, fname=images_test): from tarfile import TarFile os.chdir(utmp) req = requests.get(url, stream=True) f = open(fname, 'w') for chunk in req.iter_content(chunk_size=1024): if chunk: # filter out keep-alive new chunks f.write(chunk) f.flush() with TarFile.open(fname) as tf: tf.extractall() sp.Popen(['hadoop', 'fs', '-mkdir', config['example_data']]) subdirs = [ "newtest", ] # "rotated", "test", "test-low"] for h in subdirs: for f in os.listdir(os.path.join(utmp, h)): fname = os.path.join(utmp, h, f) hdfs_name = "%s_%s" % (h, f) sp.Popen([ 'hadoop', 'fs', '-put', fname, os.path.join(config['example_data'], hdfs_name) ]).communicate() if config.get('fuzzy_example_data', False): fuzzify(config, fname, hdfs_name)
def tar( dir = '.', glob = '.*', verbose = True ): if not isdir( dir ): raise ValueError( '{0} is not a directory'.format( dir ) ) dir = abspath( dir ) offset = len( dir ) + 1 glob = recompile( glob ) buf = BytesIO() with TarFile.open( mode = 'w', fileobj = buf, dereference = True ) as tf: num_files = 0 nonempty_dirs = set() for base, dirs, files in walk( dir, followlinks = True ): if num_files > MAX_NUM_FILES: break for fpath in files: path = join( base, fpath ) rpath = path[ offset: ] if glob.search( rpath ) and stat( path ).st_size < MAX_FILESIZE: num_files += 1 if num_files > MAX_NUM_FILES: break if verbose: sys.stderr.write( rpath + '\n' ) with open( path, 'rb' ) as f: ti = tf.gettarinfo( arcname = rpath, fileobj = f ) ti.mtime = 1 nonempty_dirs.add( dirname( path ) ) tf.addfile( ti, fileobj = f ) for path in nonempty_dirs: rpath = path[ offset: ] if not rpath: continue ti = tf.gettarinfo( name = path, arcname = rpath ) ti.mtime = 1 tf.addfile( ti ) return encodestring( buf.getvalue() )
def untar( data, dir = '.' ): if not exists( dir ): makedirs( dir, 0700 ) else: chmod( dir, 0700 ) f = BytesIO( decodestring( data ) ) with TarFile.open( mode = 'r', fileobj = f ) as tf: members = tf.getmembers() dirs = [] files = [] for m in members: if m.isdir(): dirs.append( m ) if m.isfile(): files.append( m ) dirs.sort( key = attrgetter( 'name' ) ) for d in dirs: dp = join( dir, d.name ) if not exists( dp ): mkdir( dp, 0700 ) else: chmod( dp, 0700 ) for f in files: fp = join( dir, f.name ) if exists( fp ): chmod( fp, 0700 ) tf.extract( f, dir ) dirs.reverse() for d in dirs: tf.extract( d, dir )
def setUp(self): """Set up each test.""" super().setUp() self.fake_bucket_name = tempfile.mkdtemp() mytar = TarFile.open("./koku/masu/test/data/test_local_bucket.tar.gz") mytar.extractall(path=self.fake_bucket_name) os.makedirs(DATA_DIR, exist_ok=True) self.mock_task = Mock(request=Mock(id=str(self.fake.uuid4()), return_value={})) self.report_downloader = ReportDownloader( task=self.mock_task, customer_name=self.fake_customer_name, access_credential=self.fake_auth_credential, report_source=self.fake_bucket_name, provider_type=Provider.PROVIDER_AWS_LOCAL, provider_uuid=self.aws_provider_uuid, ) self.aws_local_report_downloader = AWSLocalReportDownloader( **{ "task": self.mock_task, "customer_name": self.fake_customer_name, "auth_credential": self.fake_auth_credential, "bucket": self.fake_bucket_name, "provider_uuid": self.aws_provider_uuid, } )
def __init__(self, path: pathlib.Path): self.path = path if self.path.suffix == '.gz': self._context = TarFile.open(self.path, 'r:*') else: self._context = ZipFile(self.path, 'r') self.validate()
def __init__( self, tool, archive_bits, encoding=None, should_purge=False ): BaseContext.__init__( self, tool, encoding ) self._archive_stream = StringIO(archive_bits) self._archive = TarFile.open( 'foo.bar', 'r:gz' , self._archive_stream ) self._should_purge = bool( should_purge )
def _collect_logs(instance: IntegrationInstance, node_id: str, test_failed: bool): """Collect logs from remote instance. Args: instance: The current IntegrationInstance to collect logs from node_id: The pytest representation of this test, E.g.: tests/integration_tests/test_example.py::TestExample.test_example test_failed: If test failed or not """ if any([ integration_settings.COLLECT_LOGS == 'NEVER', integration_settings.COLLECT_LOGS == 'ON_ERROR' and not test_failed ]): return instance.execute( 'cloud-init collect-logs -u -t /var/tmp/cloud-init.tar.gz') node_id_path = Path( node_id.replace('.py', '') # Having a directory with '.py' would be weird .replace('::', os.path.sep) # Turn classes/tests into paths .replace('[', '-') # For parametrized names .replace(']', '') # For parameterized names ) log_dir = Path(integration_settings.LOCAL_LOG_PATH ) / session_start_time / node_id_path log.info("Writing logs to %s", log_dir) if not log_dir.exists(): log_dir.mkdir(parents=True) tarball_path = log_dir / 'cloud-init.tar.gz' instance.pull_file('/var/tmp/cloud-init.tar.gz', tarball_path) tarball = TarFile.open(str(tarball_path)) tarball.extractall(path=str(log_dir)) tarball_path.unlink()
def _verifyTarballEntryXML( self, fileish, entry_name, data ): fileish.seek( 0L ) tarfile = TarFile.open( 'foo.tar.gz', fileobj=fileish, mode='r:gz' ) extract = tarfile.extractfile( entry_name ) found = extract.read() self._compareDOM( found, data )
def _verifyTarballEntryXML(self, fileish, entry_name, data): fileish.seek(0L) tarfile = TarFile.open('foo.tar.gz', fileobj=fileish, mode='r:gz') extract = tarfile.extractfile(entry_name) found = extract.read() self._compareDOM(found, data)
def tar_dir(archive_path: str, dir_path: str): try: # 删除旧文件 if os.path.exists(archive_path): os.remove(archive_path) # 创建 tar 保存文件夹 if not os.path.exists(os.path.dirname(archive_path)): os.makedirs(os.path.dirname(archive_path)) # 创建 tar 文件 archive = TarFile.open(archive_path, 'w:gz') # 压缩文件 # +1: / 符号 prefix_len = len(dir_path) + 1 for (current, dirs, files) in os.walk(dir_path): if current.endswith('__pycache__'): continue for file in files: absfile = os.path.join(current, file) relfile = absfile[prefix_len:] archive.add(absfile, arcname=relfile) archive.close() # 返回 if os.path.exists(archive_path): return True, '' return False, '打包文件夹失败' except Exception as e: return False, e.__str__()
def install(self, name, edition=None, version=None, database=None, role=None, member=None): """ Install Neo4j. :param name: :param edition: :param version: :param database: :param role: :param member: :return: """ if database and role and member is not None: container = path_join(self.cc, name, database, role, str(member)) else: container = path_join(self.run, name) rmtree(container, ignore_errors=True) makedirs(container) archive_file = Distribution(edition, version).download(self.dist) try: with TarFile.open(archive_file, "r:{}".format(archive_format)) as archive: archive.extractall(container) except ReadError: # The tarfile module sometimes has trouble with certain tar # files for unknown reasons. This workaround falls back to # command line. check_call(["tar", "x", "-C", container, "-f", archive_file]) return self.get(name, database, role, member)
def fetch_di_helpers(self, mirror, suite, architecture): logging.info("Downloading helper files from debian-installer team...") urls = cdrom_image_url(mirror, suite, architecture, gtk=False, daily=self.settings['di-daily']) bootdir = self.cdroot['boot'].path ditar = tempfile.NamedTemporaryFile(delete=False) # pylint: disable=redefined-variable-type self.download_file(urls[3], ditar) ditar.close() info = TarFile.open(ditar.name, 'r:gz') if self.settings['isolinux']: isolinuxdir = self.cdroot['isolinux'].path isolinux_filenames = [ './isolinux.cfg', './stdmenu.cfg', './splash.png' ] isolinux_files = [f for f in info if f.name in isolinux_filenames] info.extractall(members=isolinux_files, path=isolinuxdir) if self.settings['grub']: # The act of fetching the path creates the directory logging.info("Created GRUB directory at %s" % (self.cdroot['boot']['grub'].path, )) grub_files = [f for f in info if f.name.startswith('./grub/')] info.extractall(members=grub_files, path=bootdir) info.close() os.remove(ditar.name)
def rebuild(filename, tag=None, format="gz"): """Rebuild the internal timezone info in dateutil/zoneinfo/zoneinfo*tar* filename is the timezone tarball from ftp.iana.org/tz. """ import tempfile, shutil tmpdir = tempfile.mkdtemp() zonedir = os.path.join(tmpdir, "zoneinfo") moduledir = os.path.dirname(__file__) if tag: tag = "-"+tag targetname = "zoneinfo%s.tar.%s" % (tag, format) try: tf = TarFile.open(filename) # The "backwards" zone file contains links to other files, so must be # processed as last for name in sorted(tf.getnames(), key=lambda k: k != "backward" and k or "z"): if not (name.endswith(".sh") or name.endswith(".tab") or name == "leapseconds"): tf.extract(name, tmpdir) filepath = os.path.join(tmpdir, name) try: # zic will return errors for nontz files in the package # such as the Makefile or README, so check_call cannot # be used (or at least extra checks would be needed) call(["zic", "-d", zonedir, filepath]) except OSError as e: if e.errno == 2: logging.error( "Could not find zic. Perhaps you need to install " "libc-bin or some other package that provides it, " "or it's not in your PATH?") raise tf.close() target = os.path.join(moduledir, targetname) for entry in os.listdir(moduledir): if entry.startswith("zoneinfo") and ".tar." in entry: os.unlink(os.path.join(moduledir, entry)) tf = TarFile.open(target, "w:%s" % format) for entry in os.listdir(zonedir): entrypath = os.path.join(zonedir, entry) tf.add(entrypath, entry) tf.close() finally: shutil.rmtree(tmpdir)
def _extract_package(self): """ Extract a package in a new temporary directory. """ self.path = mkdtemp(prefix="scoap3_package_", dir=CFG_TMPSHAREDDIR) self.logger.debug("Extracting package: %s" % (self.package_name,)) try: if ".tar" in self.package_name: TarFile.open(self.package_name).extractall(self.path) elif ".zip" in self.package_name: ZipFile(self.package_name).extractall(self.path) else: raise FileTypeError("It's not a TAR or ZIP archive.") except Exception, err: register_exception(alert_admin=True, prefix="Elsevier error extracting package.") self.logger.error("Error extraction package file: %s %s" % (self.path, err)) print >> sys.stdout, "\nError extracting package file: %s %s" % (self.path, err)
def __extract_model_resources(self, model_obj, model_id, model_version): model_resource_path = self.___model_resource_path(model_id, model_version) if not os.path.exists(model_resource_path): os.makedirs(model_resource_path) model_resources_tar_contents = model_obj['modeldir_blob'] with TarFile.open(fileobj=BytesIO(model_resources_tar_contents), mode='r:bz2') as tar: tar.extractall(model_resource_path)
def run(self): limit = 20 text = self.text category = self.category words_contains = [] words_not_contains = [] words_category = [] for w in text.split(' '): if (len(w) > 1) and (w[0]) == '-': words_not_contains.append(w[1:]) elif (len(w) > len('limit:')) and (w[:6] == 'limit:'): limit = int(w[6:]) else: words_contains.append(w) for w in category.split(' '): words_category.append(w) archive = TarFile.open('table_sorted.tar.bz2', 'r:bz2') member = archive.members[0] buffered_reader = archive.extractfile(member) buffered_text_reader = io.TextIOWrapper(buffered_reader, encoding='utf8') founded_items = 0 for line in buffered_text_reader: item = line.strip().split(sep='\t') next = False for w in words_contains: if w.lower() in item[tree_columns.index('name')].lower(): pass else: next = True break if next: continue for w in words_not_contains: if w.lower() in item[tree_columns.index('name')].lower(): next = True break if next: continue for w in words_category: if w.lower() in item[tree_columns.index('category')].lower(): pass else: next = True break if next: continue founded_items += 1 self.add_founded_item.emit(item) if founded_items >= limit: self.status.emit('Поиск закончен.') break
def archive(): """ create tar.gz archive """ tarpath = build_dir + "jsbukkit.tar.gz" if path(tarpath).exists(): remove(tarpath) tar = TarFile.open(tarpath, "w:gz") chdir(build_dir) tar.add("jsbukkit") tar.close()
def hashtar( secret, data ): mac = newmac( secret, digestmod = sha256 ) f = BytesIO( decodestring( data ) ) with TarFile.open( mode = 'r', fileobj = f ) as tf: members = tf.getmembers() files = [m for m in members if m.isfile()] files.sort( key = attrgetter( 'name' ) ) for f in files: mac.update( f.tobuf() ) return mac.hexdigest()
def __extract_prediction_module(self, model_obj, model_id, model_version): prediction_module_path = self.___prediction_module_path(model_id, model_version) if not os.path.exists(prediction_module_path): os.makedirs(prediction_module_path) prediction_module_tar_contents = model_obj['model_predict_module'] with TarFile.open(fileobj=BytesIO(prediction_module_tar_contents), mode='r:bz2') as tar: tar.extractall(prediction_module_path) os.rename(prediction_module_path +'/model.py', prediction_module_path +'/' + self.__prediction_module.format(model_id=model_id, model_version=model_version.replace('.', '_')) + '.py')