def find_all_links(filename, include_target=False):
"""Dereferences symlinks from a path.
If include_target is True, this also returns the real path of the final
target.
Example:
/
a -> b
b
g -> c
c -> ../a/d
d
e -> /f
f
>>> find_all_links('/a/g/e', True)
['/a', '/b/c', '/b/g', '/b/d/e', '/f']
"""
files = set()
filename = Path(filename)
assert filename.absolute()
path = find_all_links_recursive(filename, files)
files = list(files)
if include_target:
files.append(path)
return files
def search_for_files(self, files):
# Make a set of all the requested files
requested = dict((f.path, f) for f in self.filter_files(files))
found = {} # {path: pkgname}
# Request a few files at a time so we don't hit the command-line size
# limit
iter_batch = iter(requested)
while True:
batch = list(itertools.islice(iter_batch, MAX_ARGV))
if not batch:
break
proc = subprocess.Popen(['dpkg-query', '-S'] +
[path.path for path in batch],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
out, err = proc.communicate()
for l in out.splitlines():
pkgname, path = l.split(b': ', 1)
path = Path(path.strip())
# 8-bit safe encoding, because this might be a localized error
# message (that we don't care about)
pkgname = pkgname.decode('iso-8859-1')
if ', ' in pkgname: # Multiple packages
found[path] = None
continue
pkgname = pkgname.split(':', 1)[0] # Remove :arch
if path in requested:
if ' ' not in pkgname:
# If we had assigned it to a package already, undo
if path in found:
found[path] = None
# Else assign to the package
else:
found[path] = pkgname
# Remaining files are not from packages
self.unknown_files.update(
f for f in files
if f.path in requested and found.get(f.path) is None)
nb_pkg_files = 0
for path, pkgname in iteritems(found):
if pkgname is None:
continue
if pkgname in self.packages:
package = self.packages[pkgname]
else:
package = self._create_package(pkgname)
self.packages[pkgname] = package
package.add_file(requested.pop(path))
nb_pkg_files += 1
logger.info("%d packages with %d files, and %d other files",
len(self.packages),
nb_pkg_files,
len(self.unknown_files))
def showfiles(args):
"""Writes out the input and output files.
Works both for a pack file and for an extracted directory.
"""
pack = Path(args.pack[0])
if not pack.exists():
logging.critical("Pack or directory %s does not exist", pack)
sys.exit(1)
if pack.is_dir():
# Reads info from an unpacked directory
runs, packages, other_files = load_config_file(pack / 'config.yml',
canonical=True)
# The '.reprounzip' file is a pickled dictionary, it contains the name
# of the files that replaced each input file (if upload was used)
with pack.open('rb', '.reprounzip') as fp:
unpacked_info = pickle.load(fp)
input_files = unpacked_info.get('input_files', {})
print("Input files:")
for i, run in enumerate(runs):
if len(runs) > 1:
print(" Run %d:" % i)
for input_name, path in iteritems(run['input_files']):
print(" %s (%s)" % (input_name, path))
if input_files.get(input_name) is not None:
assigned = PosixPath(input_files[input_name])
else:
assigned = "(original)"
print(" %s" % assigned)
print("Output files:")
for i, run in enumerate(runs):
if len(runs) > 1:
print(" Run %d:" % i)
for output_name, path in iteritems(run['output_files']):
print(" %s (%s)" % (output_name, path))
else: # pack.is_file()
# Reads info from a pack file
runs, packages, other_files = load_config(pack)
print("Input files:")
for i, run in enumerate(runs):
if len(runs) > 1:
print(" Run %d:" % i)
for input_name, path in iteritems(run['input_files']):
print(" %s (%s)" % (input_name, path))
print("Output files:")
for i, run in enumerate(runs):
if len(runs) > 1:
print(" Run %d:" % i)
for output_name, path in iteritems(run['output_files']):
print(" %s (%s)" % (output_name, path))
def vagrant_destroy_dir(args):
"""Destroys the directory.
"""
target = Path(args.target[0])
read_dict(target)
signals.pre_destroy(target=target)
target.rmtree()
signals.post_destroy(target=target)
def docker_destroy_dir(args):
"""Destroys the directory.
"""
target = Path(args.target[0])
read_dict(target / '.reprounzip')
logging.info("Removing directory %s...", target)
signals.pre_destroy(target=target)
target.rmtree()
signals.post_destroy(target=target)
def showfiles(args):
"""Writes out the input and output files.
Works both for a pack file and for an extracted directory.
"""
pack = Path(args.pack[0])
if not pack.exists():
logging.critical("Pack or directory %s does not exist", pack)
sys.exit(1)
if pack.is_dir():
# Reads info from an unpacked directory
config = load_config_file(pack / 'config.yml',
canonical=True)
# The '.reprounzip' file is a pickled dictionary, it contains the name
# of the files that replaced each input file (if upload was used)
with pack.open('rb', '.reprounzip') as fp:
unpacked_info = pickle.load(fp)
assigned_input_files = unpacked_info.get('input_files', {})
print("Input files:")
for input_name, f in iteritems(config.inputs_outputs):
if not f.read_runs:
continue
print(" %s (%s)" % (input_name, f.path))
if assigned_input_files.get(input_name) is not None:
assigned = assigned_input_files[input_name]
else:
assigned = "(original)"
print(" %s" % assigned)
print("Output files:")
for output_name, f in iteritems(config.inputs_outputs):
if f.write_runs:
print(" %s (%s)" % (output_name, f.path))
else: # pack.is_file()
# Reads info from a pack file
config = load_config(pack)
print("Input files:")
for input_name, f in iteritems(config.inputs_outputs):
if f.read_runs:
print(" %s (%s)" % (input_name, f.path))
print("Output files:")
for output_name, f in iteritems(config.inputs_outputs):
if f.write_runs:
print(" %s (%s)" % (output_name, f.path))
def load_config(pack):
"""Utility method loading the YAML configuration from inside a pack file.
Decompresses the config.yml file from the tarball to a temporary file then
loads it. Note that decompressing a single file is inefficient, thus
calling this method can be slow.
"""
tmp = Path.tempdir(prefix='reprozip_')
try:
# Loads info from package
tar = tarfile.open(str(pack), 'r:*')
f = tar.extractfile('METADATA/version')
version = f.read()
f.close()
if version != b'REPROZIP VERSION 1\n':
logging.critical("Unknown pack format")
sys.exit(1)
tar.extract('METADATA/config.yml', path=str(tmp))
tar.close()
configfile = tmp / 'METADATA/config.yml'
ret = reprounzip.common.load_config(configfile, canonical=True)
finally:
tmp.rmtree()
return ret
def _extract_file(self, member, target):
member = copy.copy(member)
member.name = str(target.components[-1])
self.tar.extract(member,
path=str(Path.cwd() / target.parent))
target.chmod(0o644)
assert target.is_file()
def graph(args):
"""graph subcommand.
Reads in the trace sqlite3 database and writes out a graph in GraphViz DOT
format.
"""
if args.pack is not None:
tmp = Path.tempdir(prefix='reprounzip_')
try:
tar = tarfile.open(args.pack, 'r:*')
f = tar.extractfile('METADATA/version')
version = f.read()
f.close()
if version != b'REPROZIP VERSION 1\n':
logging.critical("Unknown pack format")
sys.exit(1)
try:
tar.extract('METADATA/config.yml', path=str(tmp))
tar.extract('METADATA/trace.sqlite3', path=str(tmp))
except KeyError as e:
logging.critical("Error extracting from pack: %s", e.args[0])
generate(Path(args.target[0]),
tmp / 'METADATA',
args.all_forks)
finally:
tmp.rmtree()
else:
generate(Path(args.target[0]), Path(args.dir), args.all_forks)
def trace(binary, argv, directory, append, verbosity=1):
"""Main function for the trace subcommand.
"""
cwd = Path.cwd()
if (any(cwd.lies_under(c) for c in magic_dirs + system_dirs) and
not cwd.lies_under('/usr/local')):
logging.warning(
"You are running this experiment from a system directory! "
"Autodetection of non-system files will probably not work as "
"intended")
# Trace directory
if not append:
if directory.exists():
logging.info("Removing existing directory %s", directory)
directory.rmtree()
directory.mkdir(parents=True)
else:
if not directory.exists():
logging.warning("--continue was specified but %s does not exist "
"-- creating", directory)
directory.mkdir(parents=True)
# Runs the trace
database = directory / 'trace.sqlite3'
logging.info("Running program")
# Might raise _pytracer.Error
c = _pytracer.execute(binary, argv, database.path, verbosity)
if c != 0:
if c & 0x0100:
logging.warning("Program appears to have been terminated by "
"signal %d", c & 0xFF)
else:
logging.warning("Program exited with non-zero code %d", c)
logging.info("Program completed")
def test_cwd(self):
"""Tests cwd, in_dir."""
cwd = os.getcwd()
if os.name == 'nt' and isinstance(cwd, bytes):
cwd = cwd.decode('mbcs')
elif os.name != 'nt' and isinstance(cwd, unicode):
cwd = cwd.encode(sys.getfilesystemencoding())
self.assertEqual(Path.cwd().path, cwd)
tmp = Path.tempdir()
with tmp.in_dir():
self.assertEqual(Path.cwd(), tmp)
self.assertNotEqual(Path.cwd(), tmp)
self.assertTrue(tmp.exists())
tmp.rmdir()
self.assertFalse(tmp.exists())
def setup_usage_report(name, version):
"""Sets up the usagestats module.
"""
global _usage_report
# Unpack CA certificate
fd, certificate_file = Path.tempfile(prefix='rpz_stats_ca_', suffix='.pem')
with certificate_file.open('wb') as fp:
fp.write(usage_report_ca)
os.close(fd)
atexit.register(os.remove, certificate_file.path)
_usage_report = usagestats.Stats(
'~/.reprozip/usage_stats',
usagestats.Prompt(enable='%s usage_report --enable' % name,
disable='%s usage_report --disable' % name),
os.environ.get('REPROZIP_USAGE_URL',
'https://reprozip-stats.poly.edu/'),
version='%s %s' % (name, version),
unique_user_id=True,
env_var='REPROZIP_USAGE_STATS',
ssl_verify=certificate_file.path)
try:
os.getcwd().encode('ascii')
except (UnicodeEncodeError, UnicodeDecodeError):
record_usage(cwd_ascii=False)
else:
record_usage(cwd_ascii=True)
def wrapper(*args, **kwargs):
tmp = Path.tempdir(prefix='reprozip_tests_')
try:
with tmp.in_dir():
return f(*args, **kwargs)
finally:
tmp.rmtree(ignore_errors=True)
def __init__(self, pack):
self.pack = Path(pack)
self.tar = tarfile.open(str(self.pack), 'r:*')
f = self.tar.extractfile('METADATA/version')
version = f.read()
f.close()
if version.startswith(b'REPROZIP VERSION '):
try:
version = int(version[17:].rstrip())
except ValueError:
version = None
if version in (1, 2):
self.version = version
self.data_prefix = PosixPath(b'DATA')
else:
raise ValueError(
"Unknown format version %r (maybe you should upgrade "
"reprounzip? I only know versions 1 and 2" % version)
else:
raise ValueError("File doesn't appear to be a RPZ pack")
if self.version == 1:
self.data = self.tar
elif version == 2:
self.data = tarfile.open(
fileobj=self.tar.extractfile('DATA.tar.gz'),
mode='r:*')
else:
assert False
def testrun(args):
"""testrun subcommand.
Runs the command with the tracer using a temporary sqlite3 database, then
reads it and dumps it out.
Not really useful, except for debugging.
"""
fd, database = Path.tempfile(prefix='reprozip_', suffix='.sqlite3')
os.close(fd)
try:
if args.arg0 is not None:
argv = [args.arg0] + args.cmdline[1:]
else:
argv = args.cmdline
logging.debug("Starting tracer, binary=%r, argv=%r",
args.cmdline[0], argv)
c = _pytracer.execute(args.cmdline[0], argv, database.path,
args.verbosity)
print("\n\n-----------------------------------------------------------"
"--------------------")
print_db(database)
if c != 0:
if c & 0x0100:
print("\nWarning: program appears to have been terminated by "
"signal %d" % (c & 0xFF))
else:
print("\nWarning: program exited with non-zero code %d" % c)
finally:
database.remove()
def prepare_upload(self, files):
if 'current_image' not in self.unpacked_info:
sys.stderr.write("Image doesn't exist yet, have you run "
"setup/build?\n")
sys.exit(1)
self.build_directory = Path.tempdir(prefix='reprozip_build_')
self.docker_copy = []
def _get_package_for_file(self, filename):
p = subprocess.Popen(['dpkg', '-S', filename.path],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
out, err = p.communicate()
for l in out.splitlines():
pkgname, f = l.split(b': ', 1)
f = Path(f.strip())
# 8-bit safe encoding, because this might be a localized error
# message (that we don't care about)
pkgname = (pkgname.decode('iso-8859-1')
.split(':', 1)[0]) # Removes :arch
self.package_files[f] = pkgname
if f == filename:
if ' ' not in pkgname:
return pkgname
return None
def with_trace(self):
"""Context manager that extracts the trace database to a temporary file.
"""
fd, tmp = Path.tempfile(prefix='reprounzip_')
os.close(fd)
self.extract_trace(tmp)
yield tmp
tmp.remove()
def setup_logging(tag, verbosity):
"""Sets up the logging module.
"""
levels = [logging.CRITICAL, logging.WARNING, logging.INFO, logging.DEBUG]
console_level = levels[min(verbosity, 3)]
file_level = logging.INFO
min_level = min(console_level, file_level)
# Create formatter, with same format as C extension
fmt = "[%s] %%(asctime)s %%(levelname)s: %%(message)s" % tag
formatter = LoggingDateFormatter(fmt)
# Console logger
handler = logging.StreamHandler()
handler.setLevel(console_level)
handler.setFormatter(formatter)
# Set up logger
logger = logging.root
logger.setLevel(min_level)
logger.addHandler(handler)
# File logger
dotrpz = Path('~/.reprozip').expand_user()
try:
if not dotrpz.is_dir():
dotrpz.mkdir()
filehandler = logging.handlers.RotatingFileHandler(str(dotrpz / 'log'),
mode='a',
delay=False,
maxBytes=400000,
backupCount=5)
except (IOError, OSError):
logging.warning("Couldn't create log file %s", dotrpz / 'log')
else:
filehandler.setFormatter(formatter)
filehandler.setLevel(file_level)
logger.addHandler(filehandler)
filehandler.emit(logging.root.makeRecord(
__name__.split('.', 1)[0],
logging.INFO,
"(log start)", 0,
"Log opened %s %s",
(datetime.now().strftime("%Y-%m-%d"), sys.argv),
None))
def get_reprozip_ca_certificate():
"""Gets the ReproZip CA certificate filename.
"""
fd, certificate_file = Path.tempfile(prefix='rpz_stats_ca_', suffix='.pem')
with certificate_file.open('wb') as fp:
fp.write(usage_report_ca)
os.close(fd)
atexit.register(os.remove, certificate_file.path)
return certificate_file
def download_and_print(self, remote_path):
# Download to temporary file
fd, temp = Path.tempfile(prefix='reprozip_output_')
os.close(fd)
self.download(remote_path, temp)
# Output to stdout
with temp.open('rb') as fp:
copyfile(fp, stdout_bytes)
temp.remove()
def directory_destroy(args):
"""Destroys the directory.
"""
target = Path(args.target[0])
metadata_read(target, 'directory')
logger.info("Removing directory %s...", target)
signals.pre_destroy(target=target)
rmtree_fixed(target)
signals.post_destroy(target=target)
def directory_destroy(args):
"""Destroys the directory.
"""
target = Path(args.target[0])
read_dict(target / '.reprounzip', 'directory')
logging.info("Removing directory %s...", target)
signals.pre_destroy(target=target)
rmtree_fixed(target)
signals.post_destroy(target=target)
def vagrant_destroy_vm(args):
"""Destroys the VM through Vagrant.
"""
target = Path(args.target[0])
read_dict(target)
retcode = subprocess.call(['vagrant', 'destroy', '-f'], cwd=target.path)
if retcode != 0:
logging.critical("vagrant destroy failed with code %d, ignoring...",
retcode)
def extract_config(self, target):
"""Extracts the config to the specified path.
It is up to the caller to remove that file once done.
"""
member = copy.copy(self.tar.getmember('METADATA/config.yml'))
member.name = str(target.components[-1])
self.tar.extract(member, path=str(Path.cwd() / target.parent))
target.chmod(0o644)
assert target.is_file()
def extract_config(self, target):
"""Extracts the config to the specified path.
It is up to the caller to remove that file once done.
"""
member = copy.copy(self.tar.getmember('METADATA/config.yml'))
member.name = str(target.components[-1])
self.tar.extract(member,
path=str(Path.cwd() / target.parent))
assert target.is_file()
def reset(args):
"""reset subcommand.
Just regenerates the configuration (config.yml) from the trace
(trace.sqlite3).
"""
reprozip.tracer.trace.write_configuration(Path(args.dir),
args.identify_packages,
args.find_inputs_outputs,
overwrite=True)
def trace(binary, argv, directory, append, verbosity=1):
"""Main function for the trace subcommand.
"""
cwd = Path.cwd()
if (any(cwd.lies_under(c) for c in magic_dirs + system_dirs)
and not cwd.lies_under('/usr/local')):
logging.warning(
"You are running this experiment from a system directory! "
"Autodetection of non-system files will probably not work as "
"intended")
# Trace directory
if directory.exists():
if append is None:
r = tty_prompt(
"Trace directory %s exists\n"
"(a)ppend run to the trace, (d)elete it or (s)top? [a/d/s] " %
directory, 'aAdDsS')
if r is None:
logging.critical(
"Trace directory %s exists\n"
"Please use either --continue or --overwrite\n", directory)
sys.exit(125)
elif r in 'sS':
sys.exit(125)
elif r in 'dD':
directory.rmtree()
directory.mkdir()
logging.warning(
"You can use --overwrite to replace the existing trace "
"(or --continue to append\nwithout prompt)")
elif append is False:
logging.info("Removing existing trace directory %s", directory)
directory.rmtree()
directory.mkdir(parents=True)
else:
if append is True:
logging.warning("--continue was set but trace doesn't exist yet")
directory.mkdir()
# Runs the trace
database = directory / 'trace.sqlite3'
logging.info("Running program")
# Might raise _pytracer.Error
c = _pytracer.execute(binary, argv, database.path, verbosity)
if c != 0:
if c & 0x0100:
logging.warning(
"Program appears to have been terminated by "
"signal %d", c & 0xFF)
else:
logging.warning("Program exited with non-zero code %d", c)
logging.info("Program completed")
return c
def main():
parser = argparse.ArgumentParser(
description="Adds __future__ imports to Python files")
parser.add_argument('-v',
'--verbose',
action='count',
dest='verbosity',
default=1)
parser.add_argument('-e',
'--enable',
action='append',
help="Future import to enable")
parser.add_argument('file',
nargs=argparse.ONE_OR_MORE,
help="File or directory in which to replace")
args = parser.parse_args()
levels = [logging.CRITICAL, logging.WARNING, logging.INFO, logging.DEBUG]
logging.basicConfig(level=levels[args.verbosity])
if not args.enable:
logging.critical("Nothing to do")
sys.exit(1)
enable = set(to_bytes(feature) for feature in args.enable)
unrecognized = enable - FUTURES
if unrecognized:
logging.critical("Error: unknown futures %s" % ', '.join(unrecognized))
sys.exit(1)
for target in args.file:
target = Path(target)
if target.is_file():
if not target.name.endswith('.py'):
logging.warning("File %s doesn't end with .py, processing "
"anyway..." % target)
process_file(target, enable)
elif target.is_dir():
logging.info("Processing %s recursively..." % target)
for filename in target.recursedir('*.py'):
process_file(filename, enable)
else:
logging.warning("Skipping %s..." % target)
def add_data(self, filename):
if filename in self.seen:
return
path = Path('/')
for c in filename.components[1:]:
path = path / c
if path in self.seen:
continue
logging.debug("%s -> %s", path, data_path(path))
self.tar.add(str(path), str(data_path(path)), recursive=False)
self.seen.add(path)
def graph(args):
"""graph subcommand.
Reads in the trace sqlite3 database and writes out a graph in GraphViz DOT
format or JSON.
"""
def call_generate(args, config, trace):
generate(Path(args.target[0]), config, trace, args.all_forks,
args.format, args.packages, args.processes, args.otherfiles,
args.regex_filter, args.regex_replace, args.aggregate)
if args.pack is not None:
rpz_pack = RPZPack(args.pack)
with rpz_pack.with_config() as config:
with rpz_pack.with_trace() as trace:
call_generate(args, config, trace)
else:
call_generate(args,
Path(args.dir) / 'config.yml',
Path(args.dir) / 'trace.sqlite3')
def docker_setup(args):
"""Does both create and build.
Removes the directory if building fails.
"""
docker_setup_create(args)
try:
docker_setup_build(args)
except:
Path(args.target[0]).rmtree(ignore_errors=True)
raise
def make_paths(cls, obj):
if isinstance(obj, set):
return set(cls.make_paths(e) for e in obj)
elif isinstance(obj, list):
return [cls.make_paths(e) for e in obj]
elif isinstance(obj, AbstractPath):
return obj
elif isinstance(obj, (bytes, unicode_)):
return Path(obj)
else:
assert False
def test_tempfile(self):
"""Tests tempfile."""
fd, f = Path.tempfile()
os.close(fd)
try:
self.assertTrue(f.exists())
self.assertTrue(f.is_file())
self.assertTrue(f.is_absolute)
finally:
f.remove()
self.assertFalse(f.exists())
def test_tempfile(self):
"""Tests tempfile."""
fd, f = Path.tempfile()
os.close(fd)
try:
self.assertTrue(f.exists())
self.assertTrue(f.is_file())
self.assertTrue(f.is_absolute)
finally:
f.remove()
self.assertFalse(f.exists())
def download(self, remote_path, local_path):
# Docker copies to a file in the specified directory, cannot just take
# a file name (#4272)
tmpdir = Path.tempdir(prefix='reprozip_docker_output_')
try:
subprocess.check_call(['docker', 'cp',
self.container + b':' + remote_path.path,
tmpdir.path])
(tmpdir / remote_path.name).copyfile(local_path)
finally:
tmpdir.rmtree()
def extract_trace(self, target):
"""Extracts the trace database to the specified path.
It is up to the caller to remove that file once done.
"""
target = Path(target)
if self.version == 1:
member = self.tar.getmember('METADATA/trace.sqlite3')
elif self.version == 2:
try:
member = self.tar.getmember('METADATA/trace.sqlite3.gz')
except KeyError:
member = self.tar.getmember('METADATA/trace.sqlite3')
else:
assert False
member = copy.copy(member)
member.name = str(target.components[-1])
self.tar.extract(member,
path=str(Path.cwd() / target.parent))
assert target.is_file()
def print_info(args):
"""Writes out some information about a pack file.
"""
pack = Path(args.pack[0])
info = get_package_info(pack, read_data=args.json or args.verbosity >= 2)
if args.json:
json.dump(info, sys.stdout, indent=2)
sys.stdout.write('\n')
else:
_print_package_info(pack, info, args.verbosity)
def upload_file(self, local_path, input_path):
stem, ext = local_path.stem, local_path.ext
name = local_path.name
nb = 0
while (self.build_directory / name).exists():
nb += 1
name = stem + ('_%d' % nb).encode('ascii') + ext
name = Path(name)
local_path.copyfile(self.build_directory / name)
logging.info("Copied file %s to %s", local_path, name)
self.docker_copy.append((name, input_path))
def chroot_destroy(args):
"""Destroys the directory, unmounting first if necessary.
"""
target = Path(args.target[0])
chroot_unmount(target)
logger.info("Removing directory %s...", target)
signals.pre_destroy(target=target)
rmtree_fixed(target)
signals.post_destroy(target=target)
def do_vistrails(target):
"""Create a VisTrails workflow that runs the experiment.
This is called from signals after an experiment has been setup by any
unpacker.
"""
record_usage(do_vistrails=True)
unpacker = signals.unpacker
dot_vistrails = Path('~/.vistrails').expand_user()
runs, packages, other_files = load_config(target / 'config.yml',
canonical=True)
for i, run in enumerate(runs):
module_name = write_cltools_module(run, dot_vistrails)
# Writes VisTrails workflow
bundle = target / 'vistrails.vt'
logging.info("Writing VisTrails workflow %s...", bundle)
vtdir = Path.tempdir(prefix='reprounzip_vistrails_')
try:
with vtdir.open('w', 'vistrail', encoding='utf-8',
newline='\n') as fp:
vistrail = VISTRAILS_TEMPLATE
cmdline = ' '.join(shell_escape(arg) for arg in run['argv'])
vistrail = vistrail.format(date='2014-11-12 15:31:18',
unpacker=unpacker,
directory=escape_xml(
str(target.absolute())),
cmdline=escape_xml(cmdline),
module_name=module_name,
run=i)
fp.write(vistrail)
with bundle.open('wb') as fp:
z = zipfile.ZipFile(fp, 'w')
with vtdir.in_dir():
for path in Path('.').recursedir():
z.write(str(path))
z.close()
finally:
vtdir.rmtree()
def chroot_run(args):
"""Runs the command in the chroot.
"""
target = Path(args.target[0])
read_dict(target / '.reprounzip', 'chroot')
cmdline = args.cmdline
# Loads config
runs, packages, other_files = load_config_file(target / 'config.yml', True)
selected_runs = get_runs(runs, args.run, cmdline)
root = target / 'root'
# X11 handler
x11 = X11Handler(args.x11, ('local', socket.gethostname()),
args.x11_display)
cmds = []
for run_number in selected_runs:
run = runs[run_number]
cmd = 'cd %s && ' % shell_escape(run['workingdir'])
cmd += '/usr/bin/env -i '
environ = x11.fix_env(run['environ'])
cmd += ' '.join('%s=%s' % (k, shell_escape(v))
for k, v in iteritems(environ))
cmd += ' '
# FIXME : Use exec -a or something if binary != argv[0]
if cmdline is None:
argv = [run['binary']] + run['argv'][1:]
else:
argv = cmdline
cmd += ' '.join(shell_escape(a) for a in argv)
userspec = '%s:%s' % (run.get('uid', 1000), run.get('gid', 1000))
cmd = 'chroot --userspec=%s %s /bin/sh -c %s' % (
userspec, shell_escape(unicode_(root)), shell_escape(cmd))
cmds.append(cmd)
cmds = [
'chroot %s /bin/sh -c %s' %
(shell_escape(unicode_(root)), shell_escape(c)) for c in x11.init_cmds
] + cmds
cmds = ' && '.join(cmds)
# Starts forwarding
forwarders = []
for portnum, connector in x11.port_forward:
fwd = LocalForwarder(connector, portnum)
forwarders.append(fwd)
signals.pre_run(target=target)
retcode = interruptible_call(cmds, shell=True)
stderr.write("\n*** Command finished, status: %d\n" % retcode)
signals.post_run(target=target, retcode=retcode)
def docker_upload(args):
"""Replaces an input file in the container.
"""
target = Path(args.target[0])
files = args.file
unpacked_info = read_dict(target)
input_files = unpacked_info.setdefault('input_files', {})
try:
ContainerUploader(target, input_files, files, unpacked_info)
finally:
write_dict(target, unpacked_info)
def download_and_print(self, remote_path):
# Download to temporary file
fd, temp = Path.tempfile(prefix='reprozip_output_')
os.close(fd)
download_status = self.download(remote_path, temp)
if download_status is not None and not download_status:
return False
# Output to stdout
with temp.open('rb') as fp:
copyfile(fp, stdout_bytes)
temp.remove()
return True
def download_and_print(self, remote_path):
# Download to temporary file
fd, temp = Path.tempfile(prefix='reprozip_output_')
os.close(fd)
download_status = self.download(remote_path, temp)
if download_status is not None and not download_status:
return False
# Output to stdout
with temp.open('rb') as fp:
copyfile(fp, stdout_bytes)
temp.remove()
return True
def setUpClass(cls):
"""Builds a test hierarchy."""
cls.tmp = Path.tempdir()
cls.tmp.open('w', 'file').close()
cls.tmp.open('w', u'r\xE9mi\'s file').close()
d = cls.tmp.mkdir(u'r\xE9pertoire')
d.open('w', 'file').close()
d.mkdir('nested')
if issubclass(Path, PosixPath):
(d / 'last').symlink('..')
else:
d.open('w', 'last').close()
def setUpClass(cls):
"""Builds a test hierarchy."""
cls.tmp = Path.tempdir()
cls.tmp.open('w', 'file').close()
cls.tmp.open('w', u'r\xE9mi\'s file').close()
d = cls.tmp.mkdir(u'r\xE9pertoire')
d.open('w', 'file').close()
d.mkdir('nested')
if issubclass(Path, PosixPath):
(d / 'last').symlink('..')
else:
d.open('w', 'last').close()
def download(self, job_id, files, **kwargs):
"""Downloads files from server.
"""
check_jobid(job_id)
if not files:
return
if isinstance(files, string_types):
files = [files]
directory = False
recursive = kwargs.pop('recursive', True)
if 'destination' in kwargs and 'directory' in kwargs:
raise TypeError("Only use one of 'destination' or 'directory'")
elif 'destination' in kwargs:
destination = Path(kwargs.pop('destination'))
if len(files) != 1:
raise ValueError("'destination' specified but multiple files "
"given; did you mean to use 'directory'?")
elif 'directory' in kwargs:
destination = Path(kwargs.pop('directory'))
directory = True
if kwargs:
raise TypeError("Got unexpected keyword arguments")
# Might raise JobNotFound
status, target, result = self.status(job_id)
scp_client = self.get_scp_client()
for filename in files:
logger.info("Downloading %s", target / filename)
if directory:
scp_client.get(str(target / filename),
str(destination / filename),
recursive=recursive)
else:
scp_client.get(str(target / filename),
str(destination),
recursive=recursive)
def upload(args):
"""Replaces an input file in the directory.
"""
target = Path(args.target[0])
files = args.file
unpacked_info = read_dict(target / '.reprounzip', args.type)
input_files = unpacked_info.setdefault('input_files', {})
try:
LocalUploader(target, input_files, files, args.type,
args.type == 'chroot' and args.restore_owner)
finally:
write_dict(target / '.reprounzip', unpacked_info, args.type)
def find_all_links_recursive(filename, files):
path = Path('/')
for c in filename.components[1:]:
# At this point, path is a canonical path, and all links in it have
# been resolved
# We add the next path component
path = path / c
# That component is possibly a link
if path.is_link():
# Adds the link itself
files.add(path)
target = path.read_link(absolute=True)
# Here, target might contain a number of symlinks
if target not in files:
# Recurse on this new path
find_all_links_recursive(target, files)
# Restores the invariant; realpath might resolve several links here
path = path.resolve()
return path
def post(self):
self._notebook_file = Path(self.get_body_argument('file'))
name = self._notebook_file.unicodename
if name.endswith('.ipynb'):
name = name[:-6]
name = u'%s_%s.rpz' % (name, datetime.now().strftime('%Y%m%d-%H%M%S'))
self._pack_file = self._notebook_file.parent / name
self.nbapp.log.info("reprozip: tracing request from client: file=%r",
self._notebook_file)
self._tempdir = Path.tempdir()
self.nbapp.log.info("reprozip: created temp directory %r",
self._tempdir)
proc = Subprocess([
sys.executable, '-c',
'from reprozip_jupyter.main import main; main()', 'trace',
'--dont-save-notebook', '-d', self._tempdir.path,
self._notebook_file.path
],
stdin=subprocess.PIPE)
proc.stdin.close()
proc.set_exit_callback(self._trace_done)
self.nbapp.log.info("reprozip: started tracing...")
def upload(args):
"""Replaces an input file in the directory.
"""
target = Path(args.target[0])
files = args.file
unpacked_info = metadata_read(target, args.type)
input_files = unpacked_info.setdefault('input_files', {})
try:
LocalUploader(target, input_files, files,
args.type, args.type == 'chroot' and args.restore_owner)
finally:
metadata_write(target, unpacked_info, args.type)
def vagrant_upload(args):
"""Replaces an input file in the VM.
"""
target = Path(args.target[0])
files = args.file
unpacked_info = read_dict(target / '.reprounzip')
input_files = unpacked_info.setdefault('input_files', {})
use_chroot = unpacked_info['use_chroot']
try:
SSHUploader(target, input_files, files, use_chroot)
finally:
write_dict(target / '.reprounzip', unpacked_info)
def find_all_links_recursive(filename, files):
path = Path('/')
for c in filename.components[1:]:
# At this point, path is a canonical path, and all links in it have
# been resolved
# We add the next path component
path = path / c
# That component is possibly a link
if path.is_link():
# Adds the link itself
files.add(path)
target = path.read_link(absolute=True)
# Here, target might contain a number of symlinks
if target not in files:
# Recurse on this new path
find_all_links_recursive(target, files)
# Restores the invariant; realpath might resolve several links here
path = path.resolve()
return path
def setup_logging(tag, verbosity):
"""Sets up the logging module.
"""
levels = [logging.CRITICAL, logging.WARNING, logging.INFO, logging.DEBUG]
console_level = levels[min(verbosity, 3)]
file_level = logging.INFO
min_level = min(console_level, file_level)
# Create formatter, with same format as C extension
fmt = "[%s] %%(asctime)s %%(levelname)s: %%(message)s" % tag
formatter = LoggingDateFormatter(fmt)
# Console logger
handler = logging.StreamHandler()
handler.setLevel(console_level)
handler.setFormatter(formatter)
# Set up logger
logger = logging.root
logger.setLevel(min_level)
logger.addHandler(handler)
# File logger
dotrpz = Path('~/.reprozip').expand_user()
try:
if not dotrpz.is_dir():
dotrpz.mkdir()
filehandler = logging.handlers.RotatingFileHandler(str(dotrpz / 'log'),
mode='a',
delay=False,
maxBytes=400000,
backupCount=5)
except (IOError, OSError):
logging.warning("Couldn't create log file %s", dotrpz / 'log')
else:
filehandler.setFormatter(formatter)
filehandler.setLevel(file_level)
logger.addHandler(filehandler)
def __init__(self, path):
path = Path(path)
size = None
if path.exists():
if path.is_link():
self.comment = "Link to %s" % path.read_link(absolute=True)
elif path.is_dir():
self.comment = "Directory"
else:
size = path.size()
self.comment = hsize(size)
File.__init__(self, path, size)
def combine(args):
"""combine subcommand.
Reads in multiple trace databases and combines them into one.
The runs from the original traces are appended ('run_id' field gets
translated to avoid conflicts).
"""
traces = []
for tracepath in args.traces:
if tracepath == '-':
tracepath = Path(args.dir) / 'trace.sqlite3'
else:
tracepath = Path(tracepath)
if tracepath.is_dir():
tracepath = tracepath / 'trace.sqlite3'
traces.append(tracepath)
reprozip.traceutils.combine_traces(traces, Path(args.dir))
reprozip.tracer.trace.write_configuration(Path(args.dir),
args.identify_packages,
args.find_inputs_outputs,
overwrite=True)
def download(self, remote_path, local_path):
# Docker copies to a file in the specified directory, cannot just take
# a file name (#4272)
tmpdir = Path.tempdir(prefix='reprozip_docker_output_')
try:
ret = subprocess.call(['docker', 'cp',
self.container + b':' + remote_path.path,
tmpdir.path])
if ret != 0:
logging.critical("Can't get output file: %s", remote_path)
sys.exit(1)
(tmpdir / remote_path.name).copyfile(local_path)
finally:
tmpdir.rmtree()