def create_archive(self): (handle, path) = mkstemp(dir=self.temp_dir) os.close(handle) archive = TarFile(path, mode="w") archive.add(os.path.join(_common.RSRC, "full.mp3"), "full.mp3") archive.close() return path
class FileSeekerTar(FileSeekerBase): def __init__(self, tar_file_path, temp_folder): FileSeekerBase.__init__(self) self.tar_file = TarFile(tar_file_path) self.temp_folder = temp_folder def search(self, filepattern): pathlist = [] for member in self.tar_file.getmembers(): if fnmatch.fnmatch(member.name, filepattern): try: clean_name = sanitize_file_path(member.name) full_path = os.path.join(self.temp_folder, Path(clean_name)) if member.isdir(): os.makedirs(full_path) else: parent_dir = os.path.dirname(full_path) if not os.path.exists(parent_dir): os.makedirs(parent_dir) with open(full_path, "wb") as fout: fout.write( ExFileObject(self.tar_file, member).read()) fout.close() os.utime(full_path, (member.mtime, member.mtime)) pathlist.append(full_path) except Exception as ex: logfunc( f'Could not write file to filesystem, path was {member.name}' + str(ex)) return pathlist def cleanup(self): self.tar_file.close()
def detect( cls, target_file, magic_type ): filename = os.path.basename( target_file.name ) if not filename.endswith( '.tar.gz'): return None if not magic_type.startswith( 'gzip compressed data' ): return None ( filename, _, _ ) = filename.rsplit( '.', 2 ) try: ( package, version ) = filename.rsplit( '-', 1 ) # ie: cinp-0.9.2.tar.gz except ValueError: return None gzfile = GzipFile( fileobj=target_file.file, mode='r' ) tarfile = TarFile( fileobj=gzfile, mode='r' ) try: info = tarfile.extractfile( '{0}/PKG-INFO'.format( filename ) ) except KeyError: return None tarfile.close() gzfile.close() if info is None: return None return cls( filename, package, 'all', version, 'python' )
def write(self, file_name): if not self.data or not os.path.isdir(self.data): raise Exception('Must set data before building') gzfile = GzipFile(file_name, 'w') tar = TarFile(fileobj=gzfile, mode='w') buff = BytesIO(json.dumps(self.control).encode()) info = TarInfo(name='./CONTROL') info.size = buff.getbuffer().nbytes tar.addfile(tarinfo=info, fileobj=buff) if self.init is not None: buff = BytesIO(self.init.encode()) info = TarInfo(name='./INIT') info.size = buff.getbuffer().nbytes tar.addfile(tarinfo=info, fileobj=buff) data = BytesIO() datatar = TarFile(fileobj=data, mode='w') datatar.add(self.data, '/') datatar.close() data.seek(0) info = TarInfo(name='./DATA') info.size = data.getbuffer().nbytes tar.addfile(tarinfo=info, fileobj=data) tar.close() gzfile.close()
def pack_archive(request_staging_dir, archive, pr): """Create a tar file containing the files that are in the MigrationArchive object""" # if the file exists then delete it! try: os.unlink(tar_file_path) except: pass # create the tar file tar_file = TarFile(tar_file_path, mode='w') logging.debug(("Created TarFile archive file: {}").format(tar_file_path)) # get the MigrationFiles belonging to this archive migration_files = archive.migrationfile_set.all() # loop over the MigrationFiles in the MigrationArchive for mp in migration_paths: # don't add if it's a directory - files under the directory will # be added if not (os.path.isdir(mp[0])): tar_file.add(mp[0], arcname=mp[1]) logging.debug( (" Adding file to TarFile archive: {}").format(mp[0])) tar_file.close() ### end of parallelisation return tar_file_path
def pack_archives(archive_list, q): """Pack the files in the archive_list into tar files""" for archive_info in archive_list: # first element is tarfile path / archive location tar_file_path = archive_info[0] try: os.unlink(tar_file_path) except: pass # create the tar file tar_file = TarFile(tar_file_path, mode='w') logging.debug( ("Created TarFile archive file: {}").format(tar_file_path)) # second element contains the MigrationFiles for this archive migration_paths = archive_info[1] # loop over the MigrationFiles in the MigrationArchive for mp in migration_paths: # don't add if it's a directory - files under the directory will # be added if not (os.path.isdir(mp[0])): tar_file.add(mp[0], arcname=mp[1]) logging.debug( (" Adding file to TarFile archive: {}").format(mp[0])) tar_file.close() # calculate digest (element 2), digest format (element 3) # and size (element 4) and add to archive archive_info[2] = calculate_digest_adler32(tar_file_path) archive_info[3] = "ADLER32" archive_info[4] = os.stat(tar_file_path).st_size q.put(archive_list)
def detect( cls, target_file, magic_type ): filename = os.path.basename( target_file.name ) if not filename.endswith( '.tar.gz'): return None if not magic_type.startswith( 'gzip compressed data' ): return None ( filename, _, _ ) = filename.rsplit( '.', 2 ) gzfile = GzipFile( fileobj=target_file.file, mode='r' ) tarfile = TarFile( fileobj=gzfile, mode='r' ) try: manifest = json.loads( tarfile.extractfile( 'MANIFEST.json' ).read() ) except ( KeyError, TypeError, json.JSONDecodeError ): return None tarfile.close() gzfile.close() if 'collection_info' not in manifest: return None try: ( namespace, name, version ) = filename.split( '-' ) except ValueError: raise ValueError( 'Unrecognized Galaxy file name Format' ) return cls( filename, '{0}-{1}'.format( namespace, name ), 'all', version, 'galaxy' )
def create_archive(self): (handle, path) = mkstemp(dir=self.temp_dir) os.close(handle) archive = TarFile(path, mode='w') archive.add(os.path.join(_common.RSRC, 'full.mp3'), 'full.mp3') archive.close() return path
def copy_to_container(container: "Container", source_path: str, target_path: str) -> None: """ Copy a file into a Docker container :param container: Container object :param source_path: Source file path :param target_path: Target file path (in the container) :return: """ # https://github.com/docker/docker-py/issues/1771 with open(source_path, "rb") as f: data = f.read() tarinfo = TarInfo(name=os.path.basename(target_path)) tarinfo.size = len(data) tarinfo.mtime = int(time.time()) stream = BytesIO() tar = TarFile(fileobj=stream, mode="w") tar.addfile(tarinfo, BytesIO(data)) tar.close() stream.seek(0) container.put_archive(path=os.path.dirname(target_path), data=stream.read())
def _createScriptExtensionTarArchive(self, sourceDirectory, scriptExtensionName): """ Creates a TAR archive for the given script extension. """ tarFileName = scriptExtensionName + ".tar" tarFilePath = os.path.join(self.__buildConfiguration.distDirectory, tarFileName) tarFile = TarFile(tarFilePath, "w") for inputDirectory in ["lib", "src"]: baseDirectory = os.path.join(sourceDirectory, inputDirectory) if os.path.exists(baseDirectory): for packageDirName in os.listdir(baseDirectory): pythonModulesToAddList = list() packageDirectory = os.path.join(baseDirectory, packageDirName) if os.path.exists(packageDirectory): for walkTuple in os.walk(packageDirectory): directoryPath = walkTuple[0] fileNameList = walkTuple[2] for fileName in fileNameList: if fileName.endswith(".py") or fileName == "SCRIPTS": filePath = os.path.join(directoryPath, fileName) pythonModulesToAddList.append(filePath) for pythonModule in pythonModulesToAddList: startPosition = pythonModule.find(baseDirectory) + len(baseDirectory) + 1 archiveName = pythonModule[startPosition:] tarFile.add(pythonModule, archiveName) tarFile.close() if self.verbose: print("Created tar archive '%s'." % tarFilePath)
def _createScriptExtensionTarArchive(self, sourceDirectory, scriptExtensionName): """ Creates a TAR archive for the given script extension. """ tarFileName = scriptExtensionName + ".tar" tarFilePath = os.path.join(self.__buildConfiguration.distDirectory, tarFileName) tarFile = TarFile(tarFilePath, "w") for inputDirectory in ["lib", "src"]: baseDirectory = os.path.join(sourceDirectory, inputDirectory) if os.path.exists(baseDirectory): for packageDirName in os.listdir(baseDirectory): pythonModulesToAddList = list() packageDirectory = os.path.join(baseDirectory, packageDirName) if os.path.exists(packageDirectory): for walkTuple in os.walk(packageDirectory): directoryPath = walkTuple[0] fileNameList = walkTuple[2] for fileName in fileNameList: if fileName.endswith( ".py") or fileName == "SCRIPTS": filePath = os.path.join( directoryPath, fileName) pythonModulesToAddList.append(filePath) for pythonModule in pythonModulesToAddList: startPosition = pythonModule.find(baseDirectory) + len( baseDirectory) + 1 archiveName = pythonModule[startPosition:] tarFile.add(pythonModule, archiveName) tarFile.close() if self.verbose: print("Created tar archive '%s'." % tarFilePath)
def move_certs(self, paths): self.log.info("Staging internal ssl certs for %s", self._log_name) yield self.pull_image(self.move_certs_image) # create the volume volume_name = self.format_volume_name(self.certs_volume_name, self) # create volume passes even if it already exists self.log.info("Creating ssl volume %s for %s", volume_name, self._log_name) yield self.docker('create_volume', volume_name) # create a tar archive of the internal cert files # docker.put_archive takes a tarfile and a running container # and unpacks the archive into the container nb_paths = {} tar_buf = BytesIO() archive = TarFile(fileobj=tar_buf, mode='w') for key, hub_path in paths.items(): fname = os.path.basename(hub_path) nb_paths[key] = '/certs/' + fname with open(hub_path, 'rb') as f: content = f.read() tarinfo = TarInfo(name=fname) tarinfo.size = len(content) tarinfo.mtime = os.stat(hub_path).st_mtime tarinfo.mode = 0o644 archive.addfile(tarinfo, BytesIO(content)) archive.close() tar_buf.seek(0) # run a container to stage the certs, # mounting the volume at /certs/ host_config = self.client.create_host_config( binds={ volume_name: {"bind": "/certs", "mode": "rw"}, }, ) container = yield self.docker('create_container', self.move_certs_image, volumes=["/certs"], host_config=host_config, ) container_id = container['Id'] self.log.debug( "Container %s is creating ssl certs for %s", container_id[:12], self._log_name, ) # start the container yield self.docker('start', container_id) # stage the archive to the container try: yield self.docker( 'put_archive', container=container_id, path='/certs', data=tar_buf, ) finally: yield self.docker('remove_container', container_id) return nb_paths
def _readManifest(self): raw = TarFile(self.filename) manifest = raw.extractfile('manifest.json').read().decode() raw.close() self._manifest = json.loads(manifest)[0]
def download(self): """ Ein Download wird ausgeführt """ self.init2() # Basisklasse einrichten simulation = self.request.POST.get("simulation", False) self._setup_path() if simulation: self.request.echo("<h1>Download Simulation!</h1><pre>") self.request.echo("request path: %s\n" % self.request_path) log_typ = "download simulation start" else: log_typ = "download start" self.db.log(log_typ, self.context['request_path']) artist = self.request.POST.get("artist", "") album = self.request.POST.get("album", "") files, _ = self._read_dir() args = {"prefix": "PyDown_%s_" % self.request.environ["REMOTE_USER"]} if self.request.cfg["temp"]: args["dir"] = self.request.cfg["temp"] temp = NamedTemporaryFile(**args) tar = TarFile(mode="w", fileobj=temp) if simulation: self.request.write("-" * 80) self.request.write("\n") for file_info in files: filename = file_info[0] abs_path = posixpath.join(self.request_path, filename) arcname = posixpath.join(artist, album, filename) if simulation: #~ self.request.write("absolute path..: %s\n" % abs_path) self.request.write("<strong>%s</strong>\n" % arcname) try: tar.add(abs_path, arcname) except IOError, e: self.request.write( "<h1>Error</h1><h2>Can't create archive: %s</h2>" % e) try: tar.close() except: pass try: temp.close() except: pass return
def reader(self): """Package up filesystem contents as a tarball.""" result = BytesIO() tarball = TarFile(fileobj=result, mode="w") for child in self.path.children(): tarball.add(child.path, arcname=child.basename(), recursive=True) tarball.close() result.seek(0, 0) yield result
def download(self): """ Ein Download wird ausgeführt """ self.init2() # Basisklasse einrichten simulation = self.request.POST.get("simulation", False) self._setup_path() if simulation: self.request.echo("<h1>Download Simulation!</h1><pre>") self.request.echo("request path: %s\n" % self.request_path) log_typ = "download simulation start" else: log_typ = "download start" self.db.log(log_typ, self.context['request_path']) artist = self.request.POST.get("artist", "") album = self.request.POST.get("album", "") files, _ = self._read_dir() args = {"prefix": "PyDown_%s_" % self.request.environ["REMOTE_USER"]} if self.request.cfg["temp"]: args["dir"] = self.request.cfg["temp"] temp = NamedTemporaryFile(**args) tar = TarFile(mode="w", fileobj=temp) if simulation: self.request.write("-"*80) self.request.write("\n") for file_info in files: filename = file_info[0] abs_path = posixpath.join(self.request_path, filename) arcname = posixpath.join(artist, album, filename) if simulation: #~ self.request.write("absolute path..: %s\n" % abs_path) self.request.write("<strong>%s</strong>\n" % arcname) try: tar.add(abs_path, arcname) except IOError, e: self.request.write("<h1>Error</h1><h2>Can't create archive: %s</h2>" % e) try: tar.close() except: pass try: temp.close() except: pass return
def make_tarball(archive_name, *args): tf = TarFile(archive_name, mode='w') for filename in args: tf.add(filename) print(f'{filename} : added') tf.close() # extract from the archive
def layerInfo(self, layer): try: return self._layerInfo[layer] except KeyError: pass raw = TarFile(self.filename) config = raw.extractfile('{0}/json'.format(layer)).read().decode() self._layerInfo[layer] = json.loads(config) raw.close() return self._layerInfo[layer]
def write_tar(filename): from tarfile import TarFile try: tf = TarFile(filename, 'w') logger.debug('Writing tar archive to %s' % filename) _write_files_to_archive(tf.add, files) tf.close() logger.debug('Completed tar archive size is %i' % os.stat(filename).st_size) except IOError as ex: logger.warn("I/O error({0}) while writing tar archive: {1}".format(e.errno, e.strerror)) os.unlink(filename) finally: tf.close()
def config(self): if self._config is not None: return self._config if self._manifest is None: self._readManifest() raw = TarFile(self.filename) config = raw.extractfile(self._manifest['Config']).read().decode() self._config = json.loads(config) raw.close() return self._config
def collect_into_archive(): session = connect.session(connect.cchdo()) # let's go and get all the directories. ftypes = ['Bottle', 'Exchange Bottle', 'Exchange Bottle (Zipped)'] bot_query = or_(*[Document.FileType == x for x in ftypes]) doc_types = ['Documentation', 'PDF Documentation'] bot_doc_query = or_(*[Document.FileType == x for x in ftypes + doc_types]) expocodes_with_bottle = session.query(distinct( Document.ExpoCode)).filter(bot_query).all() expocodes_with_bottle = [x[0] for x in expocodes_with_bottle] expocodes_with_bottle.remove(None) expocodes_with_bottle.remove('NULL') tempdir = mkdtemp() log.debug(tempdir) # Get all required files for the cruises. for expocode in expocodes_with_bottle: docs = session.query(Document).filter( Document.ExpoCode == expocode).filter(bot_doc_query).all() cruise_dir = os.path.join(tempdir, _clean_for_filename(expocode)) os.makedirs(cruise_dir) #log.debug(expocode) for doc in docs: datapath = doc.FileName tmppath = os.path.join(cruise_dir, os.path.basename(datapath)) try: shutil.copy(datapath, tmppath) except IOError: log.warn(u'missing file: {}'.format(datapath)) session.close() #for root, dirs, files in os.walk(path): # for momo in dirs: # os.chown(os.path.join(root, momo), 502, 20) # for momo in files: #os.chown(os.path.join(root, momo), 502, 20) cwd = os.getcwd() os.chdir(tempdir) tarball = TarFile(mode='w', fileobj=sys.stdout) tarball.add('.') tarball.close() os.chdir(cwd) shutil.rmtree(tempdir)
def write_tar(filename): from tarfile import TarFile try: tf = TarFile(filename, 'w') logger.debug('Writing tar archive to %s' % filename) _write_files_to_archive(tf.add, files) tf.close() logger.debug('Completed tar archive size is %i' % os.stat(filename).st_size) except IOError as ex: logger.warn("I/O error({0}) while writing tar archive: {1}".format( e.errno, e.strerror)) os.unlink(filename) finally: tf.close()
def pull(hostname: str = None): home_dir = Path.home().__str__() io_stream = io.BytesIO() tar = TarFile(fileobj=io_stream, mode="w") for file_path in CONFIG.files + CONFIG.host_files.get(hostname, []): if isinstance(file_path, str): tar.add(name=f"{home_dir}/{file_path}", arcname=file_path) elif isinstance(file_path, dict): tar.add(name=f"{home_dir}/{file_path['src']}", arcname=file_path['dst']) tar.close() io_stream.seek(0) return Response(io_stream.read1(), mimetype='application/x-tar')
def write_file(archive: tarfile.TarFile, file: IO, name: str, dataset_dir: Union[Path, str]): if isinstance(dataset_dir, str): dataset_dir = Path(dataset_dir) assert dataset_dir.is_dir() inter_dir, short_file_name = get_stored_file_name(name) file_path = dataset_dir / inter_dir / short_file_name with file_path.open("wb") as out_file: out_file.write(file.read()) file.close() archive.close()
def load_from_file(self, f): tar = TarFile(f, "r") # load info file f = tar.extractfile("info.py") self.agedesc, self.generation = eval(f.read(-1), {"__builtins__": None}) f.close() # load agents for info in tar.getmembers(): if (splitext(info.name)[1]==".agt" and info.isfile()): f = tar.extractfile(info) self.add(Agent(self.agedesc, file = f)) f.close() tar.close()
def _check_tar_file(self, content, rootdir, datafiles, simpleNames=False, noTxt=False): with NamedTemporaryFile('w') as tempfile: tempfile.write(content) tempfile.flush() if getsize(tempfile.name) > 0: expect(is_tarfile(tempfile.name)).to_be_truthy() try: tf = TarFile(tempfile.name, 'r') self._check_names(datafiles, tf.getnames(), rootdir, simpleNames, noTxt) finally: tf.close() else: self._check_names(datafiles, [], rootdir, simpleNames, noTxt)
def create_archive(filepaths): tarstream = BytesIO() tarfile = TarFile(fileobj=tarstream, mode='w') for filepath in filepaths: file = open(filepath, 'r') file_data = file.read() tarinfo = TarInfo(name=basename(file.name)) tarinfo.size = len(file_data) tarinfo.mtime = time() tarfile.addfile(tarinfo, BytesIO(file_data)) tarfile.close() tarstream.seek(0) return tarstream
def unpack_archive(archive_staging_dir, archive, external_id, target_path, filelist=None): """Unpack a tar file containing the files that are in the MigrationArchive object""" # create the name of the archive archive_path = archive.get_archive_name(archive_staging_dir) # create the target directory if it doesn't exist try: os.makedirs(target_path) except: pass try: tar_file = TarFile(archive_path, 'r') # check that the tar_file digest matches the digest in the database digest = calculate_digest(archive_path) if digest != archive.digest: error_string = ( "Digest does not match for archive: {}").format(archive_path) raise Exception(error_string) except: error_string = ("Could not find archive path: {}").format(archive_path) raise Exception(error_string) # untar each file for tar_info in tar_file.getmembers(): try: # if filelist only extract those in the filelist if filelist: if tar_info.name in filelist: tar_file.extract(tar_info, path=target_path) else: tar_file.extract(tar_info, path=target_path) logging.debug( (" Extracting file: {} from archive: {} to directory: {}" ).format(tar_info.name, archive.get_id(), target_path)) except Exception as e: error_string = ( "Could not extract file: {} from archive {} to path: {}, exception: {}" ).format(tar_info.name, archive.get_id(), target_path, str(e)) logging.error(error_string) raise Exception(error_string) tar_file.close()
def _check_tar_file(self, content, rootdir, datafiles, simpleNames=False, noTxt=False): with NamedTemporaryFile('w') as tempfile: for c in content: tempfile.write(c) tempfile.flush() if getsize(tempfile.name) > 0: self.assertTrue(is_tarfile(tempfile.name)) try: tf = TarFile(tempfile.name, 'r') self._check_names(datafiles, tf.getnames(), rootdir, simpleNames, noTxt) finally: tf.close() else: self._check_names(datafiles, [], rootdir, simpleNames, noTxt)
def load_from_file(self, f): tar = TarFile(f, "r") # load info file f = tar.extractfile("info.py") self.agedesc, self.generation = eval(f.read(-1), {"__builtins__": None}) f.close() # load agents for info in tar.getmembers(): if (splitext(info.name)[1] == ".agt" and info.isfile()): f = tar.extractfile(info) self.add(Agent(self.agedesc, file=f)) f.close() tar.close()
def run(self, args, argv): # Create a temporary tarball with our whole build context and # dockerfile for the update tmp = tempfile.NamedTemporaryFile(suffix="dckr.tar.gz") tmp_tar = TarFile(fileobj=tmp, mode='w') # Add the executable to the tarball, using the current # configured binfmt_misc path. If we don't get a path then we # only need the support libraries copied ff, enabled = _check_binfmt_misc(args.executable) if not enabled: print("binfmt_misc not enabled, update disabled") return 1 if ff: tmp_tar.add(args.executable, arcname=ff) # Add any associated libraries libs = _get_so_libs(args.executable) if libs: for l in libs: tmp_tar.add(os.path.realpath(l), arcname=l) # Create a Docker buildfile df = StringIO() df.write(u"FROM %s\n" % args.tag) df.write(u"ADD . /\n") df_bytes = BytesIO(bytes(df.getvalue(), "UTF-8")) df_tar = TarInfo(name="Dockerfile") df_tar.size = df_bytes.getbuffer().nbytes tmp_tar.addfile(df_tar, fileobj=df_bytes) tmp_tar.close() # reset the file pointers tmp.flush() tmp.seek(0) # Run the build with our tarball context dkr = Docker() dkr.update_image(args.tag, tmp, quiet=args.quiet) return 0
def run(self, args, argv): # Create a temporary tarball with our whole build context and # dockerfile for the update tmp = tempfile.NamedTemporaryFile(suffix="dckr.tar.gz") tmp_tar = TarFile(fileobj=tmp, mode='w') # Add the executable to the tarball, using the current # configured binfmt_misc path. If we don't get a path then we # only need the support libraries copied ff, enabled = _check_binfmt_misc(args.executable) if not enabled: print("binfmt_misc not enabled, update disabled") return 1 if ff: tmp_tar.add(args.executable, arcname=ff) # Add any associated libraries libs = _get_so_libs(args.executable) if libs: for l in libs: tmp_tar.add(os.path.realpath(l), arcname=l) # Create a Docker buildfile df = StringIO() df.write("FROM %s\n" % args.tag) df.write("ADD . /\n") df.seek(0) df_tar = TarInfo(name="Dockerfile") df_tar.size = len(df.buf) tmp_tar.addfile(df_tar, fileobj=df) tmp_tar.close() # reset the file pointers tmp.flush() tmp.seek(0) # Run the build with our tarball context dkr = Docker() dkr.update_image(args.tag, tmp, quiet=args.quiet) return 0
def save_to_file(self, f): tar = TarFile(f, "w") # save info file f = StringIO(repr((self.agedesc, self.generation))) info = tar.gettarinfo(None, "info.py", f) tar.addfile(info, f) f.close() # save agents for i in range(len(self.agents)): f = StringIO() self.agents[i].save_to_file(f) info = tar.gettarinfo(None, str(i) + ".agt", f) tar.addfile(info, f) f.close() tar.close()
def save_to_file(self, f): tar = TarFile(f, "w") # save info file f = StringIO(repr((self.agedesc, self.generation))) info = tar.gettarinfo(None, "info.py", f) tar.addfile(info, f) f.close() # save agents for i in range(len(self.agents)): f = StringIO() self.agents[i].save_to_file(f) info = tar.gettarinfo(None, str(i)+".agt", f) tar.addfile(info, f) f.close() tar.close()
def saveLayer(self, layer, dir_path): target_writer = tempfile.NamedTemporaryFile(delete=False) raw = TarFile(self.filename) layer_reader = raw.extractfile('{0}/layer.tar'.format(layer)) hasher = hashlib.sha256() buff = layer_reader.read(CHUNK_SIZE) while buff: hasher.update(buff) target_writer.write(buff) buff = layer_reader.read(CHUNK_SIZE) layer_reader.close() target_writer.close() raw.close() shutil.move(target_writer.name, os.path.join(dir_path, 'sha256:' + hasher.hexdigest()))
def upload(ssh_client, source_path, dest_path): remote_command = "mkdir -p {0}; cd {0}; tar xf - >/dev/null 2>&1" transport = ssh_client.get_transport() channel = transport.open_channel("session") #channel.exec_command("cat > test.tar") channel.exec_command(remote_command.format(dest_path)) #channel.exec_command("cat | ./get_tar.py") stream = channel.makefile("wb") from tarfile import TarFile tar_stream = TarFile(fileobj=stream, mode="w") tar_stream.add(source_path) tar_stream.close() channel.close() ssh_client.close()
class FileSeekerTar(FileSeekerBase): def __init__(self, tar_file_path, temp_folder): FileSeekerBase.__init__(self) self.tar_file = TarFile(tar_file_path) self.temp_folder = temp_folder def search(self, filepattern): pathlist = [] for member in self.tar_file.getmembers(): if fnmatch.fnmatch(member.name, filepattern): try: self.tar_file.extract(member.name, path=self.temp_folder) pathlist.append( os.path.join(self.temp_folder, Path(member.name))) except: logfunc('Could not write file to filesystem') return pathlist def cleanup(self): self.tar_file.close()
def _check_tar_file(self, content, rootdir, datafiles, simpleNames=False, noTxt=False): with NamedTemporaryFile('w') as tempfile: for c in content: tempfile.write(c) tempfile.flush() if getsize(tempfile.name) > 0: expect(is_tarfile(tempfile.name)).to_be_truthy() try: tf = TarFile(tempfile.name, 'r') self._check_names(datafiles, tf.getnames(), rootdir, simpleNames, noTxt) finally: tf.close() else: self._check_names(datafiles, [], rootdir, simpleNames, noTxt)
def test_something(self): tarfile = TarFile("ressources/20200414-090526_archive.tar") tarfile.extractall(path="ressources") tarfile.close() sql = Path() sql.joinpath("ressources") for i in sql.rglob("logicaldoc.sql"): self.assertTrue((str(i)).__contains__("ressources/home/cibo/src/backup")) for i in sql.rglob("repository/index/"): self.assertTrue((str(i)).__contains__("community/repository/index")) for i in sql.rglob("repository/docs/"): self.assertTrue((str(i)).__contains__("community/repository/docs")) for i in sql.rglob("conf/context.properties"): self.assertTrue((str(i)).__contains__("conf/context")) ret = Path(i) self.assertTrue((str(ret.parent)).endswith("conf"))
def optional_extract(self, output, tarname): """Extracts test repository data if needed Checks whether directory exists or is older than archive. """ tarname = get_test_file(tarname) if (not os.path.exists(output) or os.path.getmtime(output) < os.path.getmtime(tarname)): # Remove directory if outdated if os.path.exists(output): shutil.rmtree(output) # Extract new content tar = TarFile(tarname) tar.extractall(settings.DATA_DIR) tar.close() # Update directory timestamp os.utime(output, None)
def run(self, args, argv): # Create a temporary tarball with our whole build context and # dockerfile for the update tmp = tempfile.NamedTemporaryFile(suffix="dckr.tar.gz") tmp_tar = TarFile(fileobj=tmp, mode='w') # Add the executable to the tarball bn = os.path.basename(args.executable) ff = "/usr/bin/%s" % bn tmp_tar.add(args.executable, arcname=ff) # Add any associated libraries libs = _get_so_libs(args.executable) if libs: for l in libs: tmp_tar.add(os.path.realpath(l), arcname=l) # Create a Docker buildfile df = StringIO() df.write("FROM %s\n" % args.tag) df.write("ADD . /\n") df.seek(0) df_tar = TarInfo(name="Dockerfile") df_tar.size = len(df.buf) tmp_tar.addfile(df_tar, fileobj=df) tmp_tar.close() # reset the file pointers tmp.flush() tmp.seek(0) # Run the build with our tarball context dkr = Docker() dkr.update_image(args.tag, tmp, quiet=args.quiet) return 0
def reader(self, remote_snapshots=None): """ Package up filesystem contents as a tarball. """ result = BytesIO() tarball = TarFile(fileobj=result, mode="w") for child in self.path.children(): tarball.add(child.path, arcname=child.basename(), recursive=True) tarball.close() # You can append anything to the end of a tar stream without corrupting # it. Smuggle some data about the snapshots through here. This lets # tests verify that an incremental stream is really being produced # without forcing us to implement actual incremental streams on top of # dumb directories. if remote_snapshots: result.write( u"\nincremental stream based on\n{}".format( u"\n".join(snapshot.name for snapshot in remote_snapshots) ).encode("ascii") ) result.seek(0, 0) yield result
def test_can_put_extracted_file_from_tar(self): tempdir = self.make_tempdir() tarname = os.path.join(tempdir, "mytar.tar") filename = os.path.join(tempdir, "foo") # Set up a file to add the tarfile. with open(filename, "w") as f: f.write("bar") # Setup the tar file by adding the file to it. # Note there is no context handler for TarFile in python 2.6 try: tar = TarFile(tarname, "w") tar.add(filename, "foo") finally: tar.close() # See if an extracted file can be uploaded to s3. try: tar = TarFile(tarname, "r") with closing(tar.extractfile("foo")) as f: self.assert_can_put_object(body=f) finally: tar.close()
def write_tar(filename): from tarfile import TarFile tf = TarFile(filename, 'w') logger.debug('Writing tar archive to %s' % filename) _write_files_to_archive(tf.add, files) tf.close()
def plot_predictions(self): epoch, batch, data = self.get_next_batch(train=False) # get a test batch num_classes = self.test_data_provider.get_num_classes() NUM_ROWS = 2 NUM_COLS = 4 NUM_IMGS = NUM_ROWS * NUM_COLS if not self.save_preds else data[0].shape[1] NUM_TOP_CLASSES = min(num_classes, 5) # show this many top labels NUM_OUTPUTS = self.model_state["layers"][self.softmax_name]["outputs"] PRED_IDX = 1 label_names = [lab.split(",")[0] for lab in self.test_data_provider.batch_meta["label_names"]] if self.only_errors: preds = n.zeros((data[0].shape[1], NUM_OUTPUTS), dtype=n.single) else: preds = n.zeros((NUM_IMGS, NUM_OUTPUTS), dtype=n.single) # rand_idx = nr.permutation(n.r_[n.arange(1), n.where(data[1] == 552)[1], n.where(data[1] == 795)[1], n.where(data[1] == 449)[1], n.where(data[1] == 274)[1]])[:NUM_IMGS] rand_idx = nr.randint(0, data[0].shape[1], NUM_IMGS) if NUM_IMGS < data[0].shape[1]: data = [n.require(d[:, rand_idx], requirements="C") for d in data] # data += [preds] # Run the model print [d.shape for d in data], preds.shape self.libmodel.startFeatureWriter(data, [preds], [self.softmax_name]) IGPUModel.finish_batch(self) print preds data[0] = self.test_data_provider.get_plottable_data(data[0]) if self.save_preds: if not gfile.Exists(self.save_preds): gfile.MakeDirs(self.save_preds) preds_thresh = preds > 0.5 # Binarize predictions data[0] = data[0] * 255.0 data[0][data[0] < 0] = 0 data[0][data[0] > 255] = 255 data[0] = n.require(data[0], dtype=n.uint8) dir_name = "%s_predictions_batch_%d" % (os.path.basename(self.save_file), batch) tar_name = os.path.join(self.save_preds, "%s.tar" % dir_name) tfo = gfile.GFile(tar_name, "w") tf = TarFile(fileobj=tfo, mode="w") for img_idx in xrange(NUM_IMGS): img = data[0][img_idx, :, :, :] imsave = Image.fromarray(img) prefix = ( "CORRECT" if data[1][0, img_idx] == preds_thresh[img_idx, PRED_IDX] else "FALSE_POS" if preds_thresh[img_idx, PRED_IDX] == 1 else "FALSE_NEG" ) file_name = "%s_%.2f_%d_%05d_%d.png" % ( prefix, preds[img_idx, PRED_IDX], batch, img_idx, data[1][0, img_idx], ) # gf = gfile.GFile(file_name, "w") file_string = StringIO() imsave.save(file_string, "PNG") tarinf = TarInfo(os.path.join(dir_name, file_name)) tarinf.size = file_string.tell() file_string.seek(0) tf.addfile(tarinf, file_string) tf.close() tfo.close() # gf.close() print "Wrote %d prediction PNGs to %s" % (preds.shape[0], tar_name) else: fig = pl.figure(3, figsize=(12, 9)) fig.text(0.4, 0.95, "%s test samples" % ("Mistaken" if self.only_errors else "Random")) if self.only_errors: # what the net got wrong if NUM_OUTPUTS > 1: err_idx = [i for i, p in enumerate(preds.argmax(axis=1)) if p not in n.where(data[2][:, i] > 0)[0]] else: err_idx = n.where(data[1][0, :] != preds[:, 0].T)[0] print err_idx err_idx = r.sample(err_idx, min(len(err_idx), NUM_IMGS)) data[0], data[1], preds = data[0][:, err_idx], data[1][:, err_idx], preds[err_idx, :] import matplotlib.gridspec as gridspec import matplotlib.colors as colors cconv = colors.ColorConverter() gs = gridspec.GridSpec(NUM_ROWS * 2, NUM_COLS, width_ratios=[1] * NUM_COLS, height_ratios=[2, 1] * NUM_ROWS) # print data[1] for row in xrange(NUM_ROWS): for col in xrange(NUM_COLS): img_idx = row * NUM_COLS + col if data[0].shape[0] <= img_idx: break pl.subplot(gs[(row * 2) * NUM_COLS + col]) # pl.subplot(NUM_ROWS*2, NUM_COLS, row * 2 * NUM_COLS + col + 1) pl.xticks([]) pl.yticks([]) img = data[0][img_idx, :, :, :] img = img.squeeze() if len(img.shape) > 2: # more than 2 dimensions if img.shape[2] is 2: # if two channels # copy 2nd to 3rd channel for visualization a1 = img a2 = img[:, :, 1] a2 = a2[:, :, n.newaxis] img = n.concatenate((a1, a2), axis=2) pl.imshow(img, interpolation="lanczos") else: pl.imshow(img, interpolation="lanczos", cmap=pl.gray()) show_title = data[1].shape[0] == 1 true_label = [int(data[1][0, img_idx])] if show_title else n.where(data[1][:, img_idx] == 1)[0] # print true_label # print preds[img_idx,:].shape # print preds[img_idx,:].max() true_label_names = [label_names[i] for i in true_label] img_labels = sorted(zip(preds[img_idx, :], label_names), key=lambda x: x[0])[-NUM_TOP_CLASSES:] # print img_labels axes = pl.subplot(gs[(row * 2 + 1) * NUM_COLS + col]) height = 0.5 ylocs = n.array(range(NUM_TOP_CLASSES)) * height pl.barh( ylocs, [l[0] for l in img_labels], height=height, color=["#ffaaaa" if l[1] in true_label_names else "#aaaaff" for l in img_labels], ) # pl.title(", ".join(true_labels)) if show_title: pl.title(", ".join(true_label_names), fontsize=15, fontweight="bold") else: print true_label_names pl.yticks( ylocs + height / 2, [l[1] for l in img_labels], x=1, backgroundcolor=cconv.to_rgba("0.65", alpha=0.5), weight="bold", ) for line in enumerate(axes.get_yticklines()): line[1].set_visible(False) # pl.xticks([width], ['']) # pl.yticks([]) pl.xticks([]) pl.ylim(0, ylocs[-1] + height) pl.xlim(0, 1)