def create_archive(self):
(handle, path) = mkstemp(dir=self.temp_dir)
os.close(handle)
archive = TarFile(path, mode="w")
archive.add(os.path.join(_common.RSRC, "full.mp3"), "full.mp3")
archive.close()
return path
class FileSeekerTar(FileSeekerBase):
def __init__(self, tar_file_path, temp_folder):
FileSeekerBase.__init__(self)
self.tar_file = TarFile(tar_file_path)
self.temp_folder = temp_folder
def search(self, filepattern):
pathlist = []
for member in self.tar_file.getmembers():
if fnmatch.fnmatch(member.name, filepattern):
try:
clean_name = sanitize_file_path(member.name)
full_path = os.path.join(self.temp_folder,
Path(clean_name))
if member.isdir():
os.makedirs(full_path)
else:
parent_dir = os.path.dirname(full_path)
if not os.path.exists(parent_dir):
os.makedirs(parent_dir)
with open(full_path, "wb") as fout:
fout.write(
ExFileObject(self.tar_file, member).read())
fout.close()
os.utime(full_path, (member.mtime, member.mtime))
pathlist.append(full_path)
except Exception as ex:
logfunc(
f'Could not write file to filesystem, path was {member.name}'
+ str(ex))
return pathlist
def cleanup(self):
self.tar_file.close()
def detect( cls, target_file, magic_type ):
filename = os.path.basename( target_file.name )
if not filename.endswith( '.tar.gz'):
return None
if not magic_type.startswith( 'gzip compressed data' ):
return None
( filename, _, _ ) = filename.rsplit( '.', 2 )
try:
( package, version ) = filename.rsplit( '-', 1 ) # ie: cinp-0.9.2.tar.gz
except ValueError:
return None
gzfile = GzipFile( fileobj=target_file.file, mode='r' )
tarfile = TarFile( fileobj=gzfile, mode='r' )
try:
info = tarfile.extractfile( '{0}/PKG-INFO'.format( filename ) )
except KeyError:
return None
tarfile.close()
gzfile.close()
if info is None:
return None
return cls( filename, package, 'all', version, 'python' )
def write(self, file_name):
if not self.data or not os.path.isdir(self.data):
raise Exception('Must set data before building')
gzfile = GzipFile(file_name, 'w')
tar = TarFile(fileobj=gzfile, mode='w')
buff = BytesIO(json.dumps(self.control).encode())
info = TarInfo(name='./CONTROL')
info.size = buff.getbuffer().nbytes
tar.addfile(tarinfo=info, fileobj=buff)
if self.init is not None:
buff = BytesIO(self.init.encode())
info = TarInfo(name='./INIT')
info.size = buff.getbuffer().nbytes
tar.addfile(tarinfo=info, fileobj=buff)
data = BytesIO()
datatar = TarFile(fileobj=data, mode='w')
datatar.add(self.data, '/')
datatar.close()
data.seek(0)
info = TarInfo(name='./DATA')
info.size = data.getbuffer().nbytes
tar.addfile(tarinfo=info, fileobj=data)
tar.close()
gzfile.close()
def pack_archive(request_staging_dir, archive, pr):
"""Create a tar file containing the files that are in the
MigrationArchive object"""
# if the file exists then delete it!
try:
os.unlink(tar_file_path)
except:
pass
# create the tar file
tar_file = TarFile(tar_file_path, mode='w')
logging.debug(("Created TarFile archive file: {}").format(tar_file_path))
# get the MigrationFiles belonging to this archive
migration_files = archive.migrationfile_set.all()
# loop over the MigrationFiles in the MigrationArchive
for mp in migration_paths:
# don't add if it's a directory - files under the directory will
# be added
if not (os.path.isdir(mp[0])):
tar_file.add(mp[0], arcname=mp[1])
logging.debug(
(" Adding file to TarFile archive: {}").format(mp[0]))
tar_file.close()
### end of parallelisation
return tar_file_path
def pack_archives(archive_list, q):
"""Pack the files in the archive_list into tar files"""
for archive_info in archive_list:
# first element is tarfile path / archive location
tar_file_path = archive_info[0]
try:
os.unlink(tar_file_path)
except:
pass
# create the tar file
tar_file = TarFile(tar_file_path, mode='w')
logging.debug(
("Created TarFile archive file: {}").format(tar_file_path))
# second element contains the MigrationFiles for this archive
migration_paths = archive_info[1]
# loop over the MigrationFiles in the MigrationArchive
for mp in migration_paths:
# don't add if it's a directory - files under the directory will
# be added
if not (os.path.isdir(mp[0])):
tar_file.add(mp[0], arcname=mp[1])
logging.debug(
(" Adding file to TarFile archive: {}").format(mp[0]))
tar_file.close()
# calculate digest (element 2), digest format (element 3)
# and size (element 4) and add to archive
archive_info[2] = calculate_digest_adler32(tar_file_path)
archive_info[3] = "ADLER32"
archive_info[4] = os.stat(tar_file_path).st_size
q.put(archive_list)
def detect( cls, target_file, magic_type ):
filename = os.path.basename( target_file.name )
if not filename.endswith( '.tar.gz'):
return None
if not magic_type.startswith( 'gzip compressed data' ):
return None
( filename, _, _ ) = filename.rsplit( '.', 2 )
gzfile = GzipFile( fileobj=target_file.file, mode='r' )
tarfile = TarFile( fileobj=gzfile, mode='r' )
try:
manifest = json.loads( tarfile.extractfile( 'MANIFEST.json' ).read() )
except ( KeyError, TypeError, json.JSONDecodeError ):
return None
tarfile.close()
gzfile.close()
if 'collection_info' not in manifest:
return None
try:
( namespace, name, version ) = filename.split( '-' )
except ValueError:
raise ValueError( 'Unrecognized Galaxy file name Format' )
return cls( filename, '{0}-{1}'.format( namespace, name ), 'all', version, 'galaxy' )
def create_archive(self):
(handle, path) = mkstemp(dir=self.temp_dir)
os.close(handle)
archive = TarFile(path, mode='w')
archive.add(os.path.join(_common.RSRC, 'full.mp3'), 'full.mp3')
archive.close()
return path
def copy_to_container(container: "Container", source_path: str,
target_path: str) -> None:
"""
Copy a file into a Docker container
:param container: Container object
:param source_path: Source file path
:param target_path: Target file path (in the container)
:return:
"""
# https://github.com/docker/docker-py/issues/1771
with open(source_path, "rb") as f:
data = f.read()
tarinfo = TarInfo(name=os.path.basename(target_path))
tarinfo.size = len(data)
tarinfo.mtime = int(time.time())
stream = BytesIO()
tar = TarFile(fileobj=stream, mode="w")
tar.addfile(tarinfo, BytesIO(data))
tar.close()
stream.seek(0)
container.put_archive(path=os.path.dirname(target_path),
data=stream.read())
def _createScriptExtensionTarArchive(self, sourceDirectory, scriptExtensionName):
""" Creates a TAR archive for the given script extension. """
tarFileName = scriptExtensionName + ".tar"
tarFilePath = os.path.join(self.__buildConfiguration.distDirectory, tarFileName)
tarFile = TarFile(tarFilePath, "w")
for inputDirectory in ["lib", "src"]:
baseDirectory = os.path.join(sourceDirectory, inputDirectory)
if os.path.exists(baseDirectory):
for packageDirName in os.listdir(baseDirectory):
pythonModulesToAddList = list()
packageDirectory = os.path.join(baseDirectory, packageDirName)
if os.path.exists(packageDirectory):
for walkTuple in os.walk(packageDirectory):
directoryPath = walkTuple[0]
fileNameList = walkTuple[2]
for fileName in fileNameList:
if fileName.endswith(".py") or fileName == "SCRIPTS":
filePath = os.path.join(directoryPath, fileName)
pythonModulesToAddList.append(filePath)
for pythonModule in pythonModulesToAddList:
startPosition = pythonModule.find(baseDirectory) + len(baseDirectory) + 1
archiveName = pythonModule[startPosition:]
tarFile.add(pythonModule, archiveName)
tarFile.close()
if self.verbose:
print("Created tar archive '%s'." % tarFilePath)
def _createScriptExtensionTarArchive(self, sourceDirectory,
scriptExtensionName):
""" Creates a TAR archive for the given script extension. """
tarFileName = scriptExtensionName + ".tar"
tarFilePath = os.path.join(self.__buildConfiguration.distDirectory,
tarFileName)
tarFile = TarFile(tarFilePath, "w")
for inputDirectory in ["lib", "src"]:
baseDirectory = os.path.join(sourceDirectory, inputDirectory)
if os.path.exists(baseDirectory):
for packageDirName in os.listdir(baseDirectory):
pythonModulesToAddList = list()
packageDirectory = os.path.join(baseDirectory,
packageDirName)
if os.path.exists(packageDirectory):
for walkTuple in os.walk(packageDirectory):
directoryPath = walkTuple[0]
fileNameList = walkTuple[2]
for fileName in fileNameList:
if fileName.endswith(
".py") or fileName == "SCRIPTS":
filePath = os.path.join(
directoryPath, fileName)
pythonModulesToAddList.append(filePath)
for pythonModule in pythonModulesToAddList:
startPosition = pythonModule.find(baseDirectory) + len(
baseDirectory) + 1
archiveName = pythonModule[startPosition:]
tarFile.add(pythonModule, archiveName)
tarFile.close()
if self.verbose:
print("Created tar archive '%s'." % tarFilePath)
def move_certs(self, paths):
self.log.info("Staging internal ssl certs for %s", self._log_name)
yield self.pull_image(self.move_certs_image)
# create the volume
volume_name = self.format_volume_name(self.certs_volume_name, self)
# create volume passes even if it already exists
self.log.info("Creating ssl volume %s for %s", volume_name, self._log_name)
yield self.docker('create_volume', volume_name)
# create a tar archive of the internal cert files
# docker.put_archive takes a tarfile and a running container
# and unpacks the archive into the container
nb_paths = {}
tar_buf = BytesIO()
archive = TarFile(fileobj=tar_buf, mode='w')
for key, hub_path in paths.items():
fname = os.path.basename(hub_path)
nb_paths[key] = '/certs/' + fname
with open(hub_path, 'rb') as f:
content = f.read()
tarinfo = TarInfo(name=fname)
tarinfo.size = len(content)
tarinfo.mtime = os.stat(hub_path).st_mtime
tarinfo.mode = 0o644
archive.addfile(tarinfo, BytesIO(content))
archive.close()
tar_buf.seek(0)
# run a container to stage the certs,
# mounting the volume at /certs/
host_config = self.client.create_host_config(
binds={
volume_name: {"bind": "/certs", "mode": "rw"},
},
)
container = yield self.docker('create_container',
self.move_certs_image,
volumes=["/certs"],
host_config=host_config,
)
container_id = container['Id']
self.log.debug(
"Container %s is creating ssl certs for %s",
container_id[:12], self._log_name,
)
# start the container
yield self.docker('start', container_id)
# stage the archive to the container
try:
yield self.docker(
'put_archive',
container=container_id,
path='/certs',
data=tar_buf,
)
finally:
yield self.docker('remove_container', container_id)
return nb_paths
def move_certs(self, paths):
self.log.info("Staging internal ssl certs for %s", self._log_name)
yield self.pull_image(self.move_certs_image)
# create the volume
volume_name = self.format_volume_name(self.certs_volume_name, self)
# create volume passes even if it already exists
self.log.info("Creating ssl volume %s for %s", volume_name, self._log_name)
yield self.docker('create_volume', volume_name)
# create a tar archive of the internal cert files
# docker.put_archive takes a tarfile and a running container
# and unpacks the archive into the container
nb_paths = {}
tar_buf = BytesIO()
archive = TarFile(fileobj=tar_buf, mode='w')
for key, hub_path in paths.items():
fname = os.path.basename(hub_path)
nb_paths[key] = '/certs/' + fname
with open(hub_path, 'rb') as f:
content = f.read()
tarinfo = TarInfo(name=fname)
tarinfo.size = len(content)
tarinfo.mtime = os.stat(hub_path).st_mtime
tarinfo.mode = 0o644
archive.addfile(tarinfo, BytesIO(content))
archive.close()
tar_buf.seek(0)
# run a container to stage the certs,
# mounting the volume at /certs/
host_config = self.client.create_host_config(
binds={
volume_name: {"bind": "/certs", "mode": "rw"},
},
)
container = yield self.docker('create_container',
self.move_certs_image,
volumes=["/certs"],
host_config=host_config,
)
container_id = container['Id']
self.log.debug(
"Container %s is creating ssl certs for %s",
container_id[:12], self._log_name,
)
# start the container
yield self.docker('start', container_id)
# stage the archive to the container
try:
yield self.docker(
'put_archive',
container=container_id,
path='/certs',
data=tar_buf,
)
finally:
yield self.docker('remove_container', container_id)
return nb_paths
def _readManifest(self):
raw = TarFile(self.filename)
manifest = raw.extractfile('manifest.json').read().decode()
raw.close()
self._manifest = json.loads(manifest)[0]
def download(self):
"""
Ein Download wird ausgeführt
"""
self.init2() # Basisklasse einrichten
simulation = self.request.POST.get("simulation", False)
self._setup_path()
if simulation:
self.request.echo("<h1>Download Simulation!</h1><pre>")
self.request.echo("request path: %s\n" % self.request_path)
log_typ = "download simulation start"
else:
log_typ = "download start"
self.db.log(log_typ, self.context['request_path'])
artist = self.request.POST.get("artist", "")
album = self.request.POST.get("album", "")
files, _ = self._read_dir()
args = {"prefix": "PyDown_%s_" % self.request.environ["REMOTE_USER"]}
if self.request.cfg["temp"]:
args["dir"] = self.request.cfg["temp"]
temp = NamedTemporaryFile(**args)
tar = TarFile(mode="w", fileobj=temp)
if simulation:
self.request.write("-" * 80)
self.request.write("\n")
for file_info in files:
filename = file_info[0]
abs_path = posixpath.join(self.request_path, filename)
arcname = posixpath.join(artist, album, filename)
if simulation:
#~ self.request.write("absolute path..: %s\n" % abs_path)
self.request.write("<strong>%s</strong>\n" % arcname)
try:
tar.add(abs_path, arcname)
except IOError, e:
self.request.write(
"<h1>Error</h1><h2>Can't create archive: %s</h2>" % e)
try:
tar.close()
except:
pass
try:
temp.close()
except:
pass
return
def reader(self):
"""Package up filesystem contents as a tarball."""
result = BytesIO()
tarball = TarFile(fileobj=result, mode="w")
for child in self.path.children():
tarball.add(child.path, arcname=child.basename(), recursive=True)
tarball.close()
result.seek(0, 0)
yield result
def reader(self):
"""Package up filesystem contents as a tarball."""
result = BytesIO()
tarball = TarFile(fileobj=result, mode="w")
for child in self.path.children():
tarball.add(child.path, arcname=child.basename(), recursive=True)
tarball.close()
result.seek(0, 0)
yield result
def download(self):
"""
Ein Download wird ausgeführt
"""
self.init2() # Basisklasse einrichten
simulation = self.request.POST.get("simulation", False)
self._setup_path()
if simulation:
self.request.echo("<h1>Download Simulation!</h1><pre>")
self.request.echo("request path: %s\n" % self.request_path)
log_typ = "download simulation start"
else:
log_typ = "download start"
self.db.log(log_typ, self.context['request_path'])
artist = self.request.POST.get("artist", "")
album = self.request.POST.get("album", "")
files, _ = self._read_dir()
args = {"prefix": "PyDown_%s_" % self.request.environ["REMOTE_USER"]}
if self.request.cfg["temp"]:
args["dir"] = self.request.cfg["temp"]
temp = NamedTemporaryFile(**args)
tar = TarFile(mode="w", fileobj=temp)
if simulation:
self.request.write("-"*80)
self.request.write("\n")
for file_info in files:
filename = file_info[0]
abs_path = posixpath.join(self.request_path, filename)
arcname = posixpath.join(artist, album, filename)
if simulation:
#~ self.request.write("absolute path..: %s\n" % abs_path)
self.request.write("<strong>%s</strong>\n" % arcname)
try:
tar.add(abs_path, arcname)
except IOError, e:
self.request.write("<h1>Error</h1><h2>Can't create archive: %s</h2>" % e)
try:
tar.close()
except:
pass
try:
temp.close()
except:
pass
return
def make_tarball(archive_name, *args):
tf = TarFile(archive_name, mode='w')
for filename in args:
tf.add(filename)
print(f'{filename} : added')
tf.close()
# extract from the archive
def layerInfo(self, layer):
try:
return self._layerInfo[layer]
except KeyError:
pass
raw = TarFile(self.filename)
config = raw.extractfile('{0}/json'.format(layer)).read().decode()
self._layerInfo[layer] = json.loads(config)
raw.close()
return self._layerInfo[layer]
def write_tar(filename):
from tarfile import TarFile
try:
tf = TarFile(filename, 'w')
logger.debug('Writing tar archive to %s' % filename)
_write_files_to_archive(tf.add, files)
tf.close()
logger.debug('Completed tar archive size is %i' % os.stat(filename).st_size)
except IOError as ex:
logger.warn("I/O error({0}) while writing tar archive: {1}".format(e.errno, e.strerror))
os.unlink(filename)
finally:
tf.close()
def config(self):
if self._config is not None:
return self._config
if self._manifest is None:
self._readManifest()
raw = TarFile(self.filename)
config = raw.extractfile(self._manifest['Config']).read().decode()
self._config = json.loads(config)
raw.close()
return self._config
def collect_into_archive():
session = connect.session(connect.cchdo())
# let's go and get all the directories.
ftypes = ['Bottle', 'Exchange Bottle', 'Exchange Bottle (Zipped)']
bot_query = or_(*[Document.FileType == x for x in ftypes])
doc_types = ['Documentation', 'PDF Documentation']
bot_doc_query = or_(*[Document.FileType == x for x in ftypes + doc_types])
expocodes_with_bottle = session.query(distinct(
Document.ExpoCode)).filter(bot_query).all()
expocodes_with_bottle = [x[0] for x in expocodes_with_bottle]
expocodes_with_bottle.remove(None)
expocodes_with_bottle.remove('NULL')
tempdir = mkdtemp()
log.debug(tempdir)
# Get all required files for the cruises.
for expocode in expocodes_with_bottle:
docs = session.query(Document).filter(
Document.ExpoCode == expocode).filter(bot_doc_query).all()
cruise_dir = os.path.join(tempdir, _clean_for_filename(expocode))
os.makedirs(cruise_dir)
#log.debug(expocode)
for doc in docs:
datapath = doc.FileName
tmppath = os.path.join(cruise_dir, os.path.basename(datapath))
try:
shutil.copy(datapath, tmppath)
except IOError:
log.warn(u'missing file: {}'.format(datapath))
session.close()
#for root, dirs, files in os.walk(path):
# for momo in dirs:
# os.chown(os.path.join(root, momo), 502, 20)
# for momo in files:
#os.chown(os.path.join(root, momo), 502, 20)
cwd = os.getcwd()
os.chdir(tempdir)
tarball = TarFile(mode='w', fileobj=sys.stdout)
tarball.add('.')
tarball.close()
os.chdir(cwd)
shutil.rmtree(tempdir)
def write_tar(filename):
from tarfile import TarFile
try:
tf = TarFile(filename, 'w')
logger.debug('Writing tar archive to %s' % filename)
_write_files_to_archive(tf.add, files)
tf.close()
logger.debug('Completed tar archive size is %i' %
os.stat(filename).st_size)
except IOError as ex:
logger.warn("I/O error({0}) while writing tar archive: {1}".format(
e.errno, e.strerror))
os.unlink(filename)
finally:
tf.close()
def pull(hostname: str = None):
home_dir = Path.home().__str__()
io_stream = io.BytesIO()
tar = TarFile(fileobj=io_stream, mode="w")
for file_path in CONFIG.files + CONFIG.host_files.get(hostname, []):
if isinstance(file_path, str):
tar.add(name=f"{home_dir}/{file_path}", arcname=file_path)
elif isinstance(file_path, dict):
tar.add(name=f"{home_dir}/{file_path['src']}",
arcname=file_path['dst'])
tar.close()
io_stream.seek(0)
return Response(io_stream.read1(), mimetype='application/x-tar')
def write_file(archive: tarfile.TarFile, file: IO, name: str,
dataset_dir: Union[Path, str]):
if isinstance(dataset_dir, str):
dataset_dir = Path(dataset_dir)
assert dataset_dir.is_dir()
inter_dir, short_file_name = get_stored_file_name(name)
file_path = dataset_dir / inter_dir / short_file_name
with file_path.open("wb") as out_file:
out_file.write(file.read())
file.close()
archive.close()
def load_from_file(self, f):
tar = TarFile(f, "r")
# load info file
f = tar.extractfile("info.py")
self.agedesc, self.generation = eval(f.read(-1), {"__builtins__": None})
f.close()
# load agents
for info in tar.getmembers():
if (splitext(info.name)[1]==".agt" and info.isfile()):
f = tar.extractfile(info)
self.add(Agent(self.agedesc, file = f))
f.close()
tar.close()
def _check_tar_file(self, content, rootdir, datafiles,
simpleNames=False, noTxt=False):
with NamedTemporaryFile('w') as tempfile:
tempfile.write(content)
tempfile.flush()
if getsize(tempfile.name) > 0:
expect(is_tarfile(tempfile.name)).to_be_truthy()
try:
tf = TarFile(tempfile.name, 'r')
self._check_names(datafiles, tf.getnames(),
rootdir, simpleNames, noTxt)
finally:
tf.close()
else:
self._check_names(datafiles, [],
rootdir, simpleNames, noTxt)
def create_archive(filepaths):
tarstream = BytesIO()
tarfile = TarFile(fileobj=tarstream, mode='w')
for filepath in filepaths:
file = open(filepath, 'r')
file_data = file.read()
tarinfo = TarInfo(name=basename(file.name))
tarinfo.size = len(file_data)
tarinfo.mtime = time()
tarfile.addfile(tarinfo, BytesIO(file_data))
tarfile.close()
tarstream.seek(0)
return tarstream
def unpack_archive(archive_staging_dir,
archive,
external_id,
target_path,
filelist=None):
"""Unpack a tar file containing the files that are in the
MigrationArchive object"""
# create the name of the archive
archive_path = archive.get_archive_name(archive_staging_dir)
# create the target directory if it doesn't exist
try:
os.makedirs(target_path)
except:
pass
try:
tar_file = TarFile(archive_path, 'r')
# check that the tar_file digest matches the digest in the database
digest = calculate_digest(archive_path)
if digest != archive.digest:
error_string = (
"Digest does not match for archive: {}").format(archive_path)
raise Exception(error_string)
except:
error_string = ("Could not find archive path: {}").format(archive_path)
raise Exception(error_string)
# untar each file
for tar_info in tar_file.getmembers():
try:
# if filelist only extract those in the filelist
if filelist:
if tar_info.name in filelist:
tar_file.extract(tar_info, path=target_path)
else:
tar_file.extract(tar_info, path=target_path)
logging.debug(
(" Extracting file: {} from archive: {} to directory: {}"
).format(tar_info.name, archive.get_id(), target_path))
except Exception as e:
error_string = (
"Could not extract file: {} from archive {} to path: {}, exception: {}"
).format(tar_info.name, archive.get_id(), target_path, str(e))
logging.error(error_string)
raise Exception(error_string)
tar_file.close()
def _check_tar_file(self, content, rootdir, datafiles,
simpleNames=False, noTxt=False):
with NamedTemporaryFile('w') as tempfile:
for c in content:
tempfile.write(c)
tempfile.flush()
if getsize(tempfile.name) > 0:
self.assertTrue(is_tarfile(tempfile.name))
try:
tf = TarFile(tempfile.name, 'r')
self._check_names(datafiles, tf.getnames(),
rootdir, simpleNames, noTxt)
finally:
tf.close()
else:
self._check_names(datafiles, [],
rootdir, simpleNames, noTxt)
def load_from_file(self, f):
tar = TarFile(f, "r")
# load info file
f = tar.extractfile("info.py")
self.agedesc, self.generation = eval(f.read(-1),
{"__builtins__": None})
f.close()
# load agents
for info in tar.getmembers():
if (splitext(info.name)[1] == ".agt" and info.isfile()):
f = tar.extractfile(info)
self.add(Agent(self.agedesc, file=f))
f.close()
tar.close()
def run(self, args, argv):
# Create a temporary tarball with our whole build context and
# dockerfile for the update
tmp = tempfile.NamedTemporaryFile(suffix="dckr.tar.gz")
tmp_tar = TarFile(fileobj=tmp, mode='w')
# Add the executable to the tarball, using the current
# configured binfmt_misc path. If we don't get a path then we
# only need the support libraries copied
ff, enabled = _check_binfmt_misc(args.executable)
if not enabled:
print("binfmt_misc not enabled, update disabled")
return 1
if ff:
tmp_tar.add(args.executable, arcname=ff)
# Add any associated libraries
libs = _get_so_libs(args.executable)
if libs:
for l in libs:
tmp_tar.add(os.path.realpath(l), arcname=l)
# Create a Docker buildfile
df = StringIO()
df.write(u"FROM %s\n" % args.tag)
df.write(u"ADD . /\n")
df_bytes = BytesIO(bytes(df.getvalue(), "UTF-8"))
df_tar = TarInfo(name="Dockerfile")
df_tar.size = df_bytes.getbuffer().nbytes
tmp_tar.addfile(df_tar, fileobj=df_bytes)
tmp_tar.close()
# reset the file pointers
tmp.flush()
tmp.seek(0)
# Run the build with our tarball context
dkr = Docker()
dkr.update_image(args.tag, tmp, quiet=args.quiet)
return 0
def run(self, args, argv):
# Create a temporary tarball with our whole build context and
# dockerfile for the update
tmp = tempfile.NamedTemporaryFile(suffix="dckr.tar.gz")
tmp_tar = TarFile(fileobj=tmp, mode='w')
# Add the executable to the tarball, using the current
# configured binfmt_misc path. If we don't get a path then we
# only need the support libraries copied
ff, enabled = _check_binfmt_misc(args.executable)
if not enabled:
print("binfmt_misc not enabled, update disabled")
return 1
if ff:
tmp_tar.add(args.executable, arcname=ff)
# Add any associated libraries
libs = _get_so_libs(args.executable)
if libs:
for l in libs:
tmp_tar.add(os.path.realpath(l), arcname=l)
# Create a Docker buildfile
df = StringIO()
df.write("FROM %s\n" % args.tag)
df.write("ADD . /\n")
df.seek(0)
df_tar = TarInfo(name="Dockerfile")
df_tar.size = len(df.buf)
tmp_tar.addfile(df_tar, fileobj=df)
tmp_tar.close()
# reset the file pointers
tmp.flush()
tmp.seek(0)
# Run the build with our tarball context
dkr = Docker()
dkr.update_image(args.tag, tmp, quiet=args.quiet)
return 0
def save_to_file(self, f):
tar = TarFile(f, "w")
# save info file
f = StringIO(repr((self.agedesc, self.generation)))
info = tar.gettarinfo(None, "info.py", f)
tar.addfile(info, f)
f.close()
# save agents
for i in range(len(self.agents)):
f = StringIO()
self.agents[i].save_to_file(f)
info = tar.gettarinfo(None, str(i) + ".agt", f)
tar.addfile(info, f)
f.close()
tar.close()
def save_to_file(self, f):
tar = TarFile(f, "w")
# save info file
f = StringIO(repr((self.agedesc, self.generation)))
info = tar.gettarinfo(None, "info.py", f)
tar.addfile(info, f)
f.close()
# save agents
for i in range(len(self.agents)):
f = StringIO()
self.agents[i].save_to_file(f)
info = tar.gettarinfo(None, str(i)+".agt", f)
tar.addfile(info, f)
f.close()
tar.close()
def saveLayer(self, layer, dir_path):
target_writer = tempfile.NamedTemporaryFile(delete=False)
raw = TarFile(self.filename)
layer_reader = raw.extractfile('{0}/layer.tar'.format(layer))
hasher = hashlib.sha256()
buff = layer_reader.read(CHUNK_SIZE)
while buff:
hasher.update(buff)
target_writer.write(buff)
buff = layer_reader.read(CHUNK_SIZE)
layer_reader.close()
target_writer.close()
raw.close()
shutil.move(target_writer.name,
os.path.join(dir_path, 'sha256:' + hasher.hexdigest()))
def upload(ssh_client, source_path, dest_path):
remote_command = "mkdir -p {0}; cd {0}; tar xf - >/dev/null 2>&1"
transport = ssh_client.get_transport()
channel = transport.open_channel("session")
#channel.exec_command("cat > test.tar")
channel.exec_command(remote_command.format(dest_path))
#channel.exec_command("cat | ./get_tar.py")
stream = channel.makefile("wb")
from tarfile import TarFile
tar_stream = TarFile(fileobj=stream, mode="w")
tar_stream.add(source_path)
tar_stream.close()
channel.close()
ssh_client.close()
class FileSeekerTar(FileSeekerBase):
def __init__(self, tar_file_path, temp_folder):
FileSeekerBase.__init__(self)
self.tar_file = TarFile(tar_file_path)
self.temp_folder = temp_folder
def search(self, filepattern):
pathlist = []
for member in self.tar_file.getmembers():
if fnmatch.fnmatch(member.name, filepattern):
try:
self.tar_file.extract(member.name, path=self.temp_folder)
pathlist.append(
os.path.join(self.temp_folder, Path(member.name)))
except:
logfunc('Could not write file to filesystem')
return pathlist
def cleanup(self):
self.tar_file.close()
def _check_tar_file(self,
content,
rootdir,
datafiles,
simpleNames=False,
noTxt=False):
with NamedTemporaryFile('w') as tempfile:
for c in content:
tempfile.write(c)
tempfile.flush()
if getsize(tempfile.name) > 0:
expect(is_tarfile(tempfile.name)).to_be_truthy()
try:
tf = TarFile(tempfile.name, 'r')
self._check_names(datafiles, tf.getnames(), rootdir,
simpleNames, noTxt)
finally:
tf.close()
else:
self._check_names(datafiles, [], rootdir, simpleNames, noTxt)
def test_something(self):
tarfile = TarFile("ressources/20200414-090526_archive.tar")
tarfile.extractall(path="ressources")
tarfile.close()
sql = Path()
sql.joinpath("ressources")
for i in sql.rglob("logicaldoc.sql"):
self.assertTrue((str(i)).__contains__("ressources/home/cibo/src/backup"))
for i in sql.rglob("repository/index/"):
self.assertTrue((str(i)).__contains__("community/repository/index"))
for i in sql.rglob("repository/docs/"):
self.assertTrue((str(i)).__contains__("community/repository/docs"))
for i in sql.rglob("conf/context.properties"):
self.assertTrue((str(i)).__contains__("conf/context"))
ret = Path(i)
self.assertTrue((str(ret.parent)).endswith("conf"))
def optional_extract(self, output, tarname):
"""Extracts test repository data if needed
Checks whether directory exists or is older than archive.
"""
tarname = get_test_file(tarname)
if (not os.path.exists(output) or
os.path.getmtime(output) < os.path.getmtime(tarname)):
# Remove directory if outdated
if os.path.exists(output):
shutil.rmtree(output)
# Extract new content
tar = TarFile(tarname)
tar.extractall(settings.DATA_DIR)
tar.close()
# Update directory timestamp
os.utime(output, None)
def run(self, args, argv):
# Create a temporary tarball with our whole build context and
# dockerfile for the update
tmp = tempfile.NamedTemporaryFile(suffix="dckr.tar.gz")
tmp_tar = TarFile(fileobj=tmp, mode='w')
# Add the executable to the tarball
bn = os.path.basename(args.executable)
ff = "/usr/bin/%s" % bn
tmp_tar.add(args.executable, arcname=ff)
# Add any associated libraries
libs = _get_so_libs(args.executable)
if libs:
for l in libs:
tmp_tar.add(os.path.realpath(l), arcname=l)
# Create a Docker buildfile
df = StringIO()
df.write("FROM %s\n" % args.tag)
df.write("ADD . /\n")
df.seek(0)
df_tar = TarInfo(name="Dockerfile")
df_tar.size = len(df.buf)
tmp_tar.addfile(df_tar, fileobj=df)
tmp_tar.close()
# reset the file pointers
tmp.flush()
tmp.seek(0)
# Run the build with our tarball context
dkr = Docker()
dkr.update_image(args.tag, tmp, quiet=args.quiet)
return 0
def reader(self, remote_snapshots=None):
"""
Package up filesystem contents as a tarball.
"""
result = BytesIO()
tarball = TarFile(fileobj=result, mode="w")
for child in self.path.children():
tarball.add(child.path, arcname=child.basename(), recursive=True)
tarball.close()
# You can append anything to the end of a tar stream without corrupting
# it. Smuggle some data about the snapshots through here. This lets
# tests verify that an incremental stream is really being produced
# without forcing us to implement actual incremental streams on top of
# dumb directories.
if remote_snapshots:
result.write(
u"\nincremental stream based on\n{}".format(
u"\n".join(snapshot.name for snapshot in remote_snapshots)
).encode("ascii")
)
result.seek(0, 0)
yield result
def test_can_put_extracted_file_from_tar(self):
tempdir = self.make_tempdir()
tarname = os.path.join(tempdir, "mytar.tar")
filename = os.path.join(tempdir, "foo")
# Set up a file to add the tarfile.
with open(filename, "w") as f:
f.write("bar")
# Setup the tar file by adding the file to it.
# Note there is no context handler for TarFile in python 2.6
try:
tar = TarFile(tarname, "w")
tar.add(filename, "foo")
finally:
tar.close()
# See if an extracted file can be uploaded to s3.
try:
tar = TarFile(tarname, "r")
with closing(tar.extractfile("foo")) as f:
self.assert_can_put_object(body=f)
finally:
tar.close()
def write_tar(filename):
from tarfile import TarFile
tf = TarFile(filename, 'w')
logger.debug('Writing tar archive to %s' % filename)
_write_files_to_archive(tf.add, files)
tf.close()
def plot_predictions(self):
epoch, batch, data = self.get_next_batch(train=False) # get a test batch
num_classes = self.test_data_provider.get_num_classes()
NUM_ROWS = 2
NUM_COLS = 4
NUM_IMGS = NUM_ROWS * NUM_COLS if not self.save_preds else data[0].shape[1]
NUM_TOP_CLASSES = min(num_classes, 5) # show this many top labels
NUM_OUTPUTS = self.model_state["layers"][self.softmax_name]["outputs"]
PRED_IDX = 1
label_names = [lab.split(",")[0] for lab in self.test_data_provider.batch_meta["label_names"]]
if self.only_errors:
preds = n.zeros((data[0].shape[1], NUM_OUTPUTS), dtype=n.single)
else:
preds = n.zeros((NUM_IMGS, NUM_OUTPUTS), dtype=n.single)
# rand_idx = nr.permutation(n.r_[n.arange(1), n.where(data[1] == 552)[1], n.where(data[1] == 795)[1], n.where(data[1] == 449)[1], n.where(data[1] == 274)[1]])[:NUM_IMGS]
rand_idx = nr.randint(0, data[0].shape[1], NUM_IMGS)
if NUM_IMGS < data[0].shape[1]:
data = [n.require(d[:, rand_idx], requirements="C") for d in data]
# data += [preds]
# Run the model
print [d.shape for d in data], preds.shape
self.libmodel.startFeatureWriter(data, [preds], [self.softmax_name])
IGPUModel.finish_batch(self)
print preds
data[0] = self.test_data_provider.get_plottable_data(data[0])
if self.save_preds:
if not gfile.Exists(self.save_preds):
gfile.MakeDirs(self.save_preds)
preds_thresh = preds > 0.5 # Binarize predictions
data[0] = data[0] * 255.0
data[0][data[0] < 0] = 0
data[0][data[0] > 255] = 255
data[0] = n.require(data[0], dtype=n.uint8)
dir_name = "%s_predictions_batch_%d" % (os.path.basename(self.save_file), batch)
tar_name = os.path.join(self.save_preds, "%s.tar" % dir_name)
tfo = gfile.GFile(tar_name, "w")
tf = TarFile(fileobj=tfo, mode="w")
for img_idx in xrange(NUM_IMGS):
img = data[0][img_idx, :, :, :]
imsave = Image.fromarray(img)
prefix = (
"CORRECT"
if data[1][0, img_idx] == preds_thresh[img_idx, PRED_IDX]
else "FALSE_POS"
if preds_thresh[img_idx, PRED_IDX] == 1
else "FALSE_NEG"
)
file_name = "%s_%.2f_%d_%05d_%d.png" % (
prefix,
preds[img_idx, PRED_IDX],
batch,
img_idx,
data[1][0, img_idx],
)
# gf = gfile.GFile(file_name, "w")
file_string = StringIO()
imsave.save(file_string, "PNG")
tarinf = TarInfo(os.path.join(dir_name, file_name))
tarinf.size = file_string.tell()
file_string.seek(0)
tf.addfile(tarinf, file_string)
tf.close()
tfo.close()
# gf.close()
print "Wrote %d prediction PNGs to %s" % (preds.shape[0], tar_name)
else:
fig = pl.figure(3, figsize=(12, 9))
fig.text(0.4, 0.95, "%s test samples" % ("Mistaken" if self.only_errors else "Random"))
if self.only_errors:
# what the net got wrong
if NUM_OUTPUTS > 1:
err_idx = [i for i, p in enumerate(preds.argmax(axis=1)) if p not in n.where(data[2][:, i] > 0)[0]]
else:
err_idx = n.where(data[1][0, :] != preds[:, 0].T)[0]
print err_idx
err_idx = r.sample(err_idx, min(len(err_idx), NUM_IMGS))
data[0], data[1], preds = data[0][:, err_idx], data[1][:, err_idx], preds[err_idx, :]
import matplotlib.gridspec as gridspec
import matplotlib.colors as colors
cconv = colors.ColorConverter()
gs = gridspec.GridSpec(NUM_ROWS * 2, NUM_COLS, width_ratios=[1] * NUM_COLS, height_ratios=[2, 1] * NUM_ROWS)
# print data[1]
for row in xrange(NUM_ROWS):
for col in xrange(NUM_COLS):
img_idx = row * NUM_COLS + col
if data[0].shape[0] <= img_idx:
break
pl.subplot(gs[(row * 2) * NUM_COLS + col])
# pl.subplot(NUM_ROWS*2, NUM_COLS, row * 2 * NUM_COLS + col + 1)
pl.xticks([])
pl.yticks([])
img = data[0][img_idx, :, :, :]
img = img.squeeze()
if len(img.shape) > 2: # more than 2 dimensions
if img.shape[2] is 2: # if two channels
# copy 2nd to 3rd channel for visualization
a1 = img
a2 = img[:, :, 1]
a2 = a2[:, :, n.newaxis]
img = n.concatenate((a1, a2), axis=2)
pl.imshow(img, interpolation="lanczos")
else:
pl.imshow(img, interpolation="lanczos", cmap=pl.gray())
show_title = data[1].shape[0] == 1
true_label = [int(data[1][0, img_idx])] if show_title else n.where(data[1][:, img_idx] == 1)[0]
# print true_label
# print preds[img_idx,:].shape
# print preds[img_idx,:].max()
true_label_names = [label_names[i] for i in true_label]
img_labels = sorted(zip(preds[img_idx, :], label_names), key=lambda x: x[0])[-NUM_TOP_CLASSES:]
# print img_labels
axes = pl.subplot(gs[(row * 2 + 1) * NUM_COLS + col])
height = 0.5
ylocs = n.array(range(NUM_TOP_CLASSES)) * height
pl.barh(
ylocs,
[l[0] for l in img_labels],
height=height,
color=["#ffaaaa" if l[1] in true_label_names else "#aaaaff" for l in img_labels],
)
# pl.title(", ".join(true_labels))
if show_title:
pl.title(", ".join(true_label_names), fontsize=15, fontweight="bold")
else:
print true_label_names
pl.yticks(
ylocs + height / 2,
[l[1] for l in img_labels],
x=1,
backgroundcolor=cconv.to_rgba("0.65", alpha=0.5),
weight="bold",
)
for line in enumerate(axes.get_yticklines()):
line[1].set_visible(False)
# pl.xticks([width], [''])
# pl.yticks([])
pl.xticks([])
pl.ylim(0, ylocs[-1] + height)
pl.xlim(0, 1)
