def chroot_unmount(target):
"""Unmount magic directories, if they are mounted.
"""
unpacked_info = metadata_read(target, 'chroot')
mounted = unpacked_info.get('mounted', False)
if not mounted:
return False
target = target.resolve()
for m in ('/dev', '/proc'):
d = join_root(target / 'root', Path(m))
if d.exists():
logger.info("Unmounting %s...", d)
# Unmounts recursively
subprocess.check_call(
'grep %s /proc/mounts | '
'cut -f2 -d" " | '
'sort -r | '
'xargs umount' % d,
shell=True)
unpacked_info['mounted'] = False
metadata_write(target, unpacked_info, 'chroot')
return True
def chroot_unmount(target):
"""Unmount magic directories, if they are mounted.
"""
unpacked_info = metadata_read(target, 'chroot')
mounted = unpacked_info.get('mounted', False)
if not mounted:
return False
target = target.resolve()
for m in ('/dev', '/proc'):
d = join_root(target / 'root', Path(m))
if d.exists():
logging.info("Unmounting %s...", d)
# Unmounts recursively
subprocess.check_call(
'grep %s /proc/mounts | '
'cut -f2 -d" " | '
'sort -r | '
'xargs umount' % d,
shell=True)
unpacked_info['mounted'] = False
metadata_write(target, unpacked_info, 'chroot')
return True
def chroot_run(args):
"""Runs the command in the chroot.
"""
target = Path(args.target[0])
unpacked_info = metadata_read(target, 'chroot')
cmdline = args.cmdline
# Loads config
config = load_config_file(target / 'config.yml', True)
runs = config.runs
selected_runs = get_runs(runs, args.run, cmdline)
root = target / 'root'
# X11 handler
x11 = X11Handler(args.x11, ('local', socket.gethostname()),
args.x11_display)
cmds = []
for run_number in selected_runs:
run = runs[run_number]
cmd = 'cd %s && ' % shell_escape(run['workingdir'])
cmd += '/usr/bin/env -i '
environ = x11.fix_env(run['environ'])
environ = fixup_environment(environ, args)
cmd += ' '.join('%s=%s' % (shell_escape(k), shell_escape(v))
for k, v in iteritems(environ))
cmd += ' '
# FIXME : Use exec -a or something if binary != argv[0]
if cmdline is None:
argv = [run['binary']] + run['argv'][1:]
else:
argv = cmdline
cmd += ' '.join(shell_escape(a) for a in argv)
userspec = '%s:%s' % (run.get('uid', 1000), run.get('gid', 1000))
cmd = 'chroot --userspec=%s %s /bin/sh -c %s' % (
userspec, shell_escape(unicode_(root)), shell_escape(cmd))
cmds.append(cmd)
cmds = [
'chroot %s /bin/sh -c %s' %
(shell_escape(unicode_(root)), shell_escape(c)) for c in x11.init_cmds
] + cmds
cmds = ' && '.join(cmds)
# Starts forwarding
forwarders = []
for portnum, connector in x11.port_forward:
fwd = LocalForwarder(connector, portnum)
forwarders.append(fwd)
signals.pre_run(target=target)
retcode = interruptible_call(cmds, shell=True)
stderr.write("\n*** Command finished, status: %d\n" % retcode)
signals.post_run(target=target, retcode=retcode)
# Update input file status
metadata_update_run(config, unpacked_info, selected_runs)
metadata_write(target, unpacked_info, 'chroot')
def upload(args):
"""Replaces an input file in the directory.
"""
target = Path(args.target[0])
files = args.file
unpacked_info = metadata_read(target, args.type)
input_files = unpacked_info.setdefault('input_files', {})
try:
LocalUploader(target, input_files, files,
args.type, args.type == 'chroot' and args.restore_owner)
finally:
metadata_write(target, unpacked_info, args.type)
def upload(args):
"""Replaces an input file in the directory.
"""
target = Path(args.target[0])
files = args.file
unpacked_info = metadata_read(target, args.type)
input_files = unpacked_info.setdefault('input_files', {})
try:
LocalUploader(target, input_files, files,
args.type, args.type == 'chroot' and args.restore_owner)
finally:
metadata_write(target, unpacked_info, args.type)
def chroot_mount(args):
"""Mounts /dev and /proc inside the chroot directory.
"""
target = Path(args.target[0])
unpacked_info = metadata_read(target, 'chroot')
for m in ('/dev', '/dev/pts', '/proc'):
d = join_root(target / 'root', Path(m))
d.mkdir(parents=True)
logging.info("Mounting %s on %s...", m, d)
subprocess.check_call(['mount', '-o', 'bind', m, str(d)])
unpacked_info['mounted'] = True
metadata_write(target, unpacked_info, 'chroot')
logging.warning("The host's /dev and /proc have been mounted into the "
"chroot. Do NOT remove the unpacked directory with "
"rm -rf, it WILL WIPE the host's /dev directory.")
def chroot_mount(args):
"""Mounts /dev and /proc inside the chroot directory.
"""
target = Path(args.target[0])
unpacked_info = metadata_read(target, 'chroot')
for m in ('/dev', '/dev/pts', '/proc'):
d = join_root(target / 'root', Path(m))
d.mkdir(parents=True)
logging.info("Mounting %s on %s...", m, d)
subprocess.check_call(['mount', '-o', 'bind', m, str(d)])
unpacked_info['mounted'] = True
metadata_write(target, unpacked_info, 'chroot')
logging.warning("The host's /dev and /proc have been mounted into the "
"chroot. Do NOT remove the unpacked directory with "
"rm -rf, it WILL WIPE the host's /dev directory.")
def chroot_create(args):
"""Unpacks the experiment in a folder so it can be run with chroot.
All the files in the pack are unpacked; system files are copied only if
they were not packed, and busybox is installed if /bin/sh wasn't packed.
In addition, input files are put in a tar.gz (so they can be put back after
an upload) and the configuration file is extracted.
"""
if not args.pack:
logger.critical("setup/create needs the pack filename")
sys.exit(1)
pack = Path(args.pack[0])
target = Path(args.target[0])
if target.exists():
logger.critical("Target directory exists")
sys.exit(1)
if not issubclass(DefaultAbstractPath, PosixPath):
logger.critical("Not unpacking on POSIX system")
sys.exit(1)
signals.pre_setup(target=target, pack=pack)
# We can only restore owner/group of files if running as root
restore_owner = should_restore_owner(args.restore_owner)
# Unpacks configuration file
rpz_pack = RPZPack(pack)
rpz_pack.extract_config(target / 'config.yml')
# Loads config
config = load_config_file(target / 'config.yml', True)
packages = config.packages
target.mkdir()
root = (target / 'root').absolute()
root.mkdir()
try:
# Checks that everything was packed
packages_not_packed = [pkg for pkg in packages if not pkg.packfiles]
if packages_not_packed:
record_usage(chroot_missing_pkgs=True)
logger.warning("According to configuration, some files were left "
"out because they belong to the following "
"packages:%s\nWill copy files from HOST SYSTEM",
''.join('\n %s' % pkg
for pkg in packages_not_packed))
missing_files = False
for pkg in packages_not_packed:
for f in pkg.files:
path = Path(f.path)
if not path.exists():
logger.error(
"Missing file %s (from package %s) on host, "
"experiment will probably miss it",
path, pkg.name)
missing_files = True
continue
dest = join_root(root, path)
dest.parent.mkdir(parents=True)
if path.is_link():
dest.symlink(path.read_link())
else:
path.copy(dest)
if restore_owner:
stat = path.stat()
dest.chown(stat.st_uid, stat.st_gid)
if missing_files:
record_usage(chroot_mising_files=True)
# Unpacks files
members = rpz_pack.list_data()
for m in members:
# Remove 'DATA/' prefix
m.name = str(rpz_pack.remove_data_prefix(m.name))
if not restore_owner:
uid = os.getuid()
gid = os.getgid()
for m in members:
m.uid = uid
m.gid = gid
logger.info("Extracting files...")
rpz_pack.extract_data(root, members)
rpz_pack.close()
resolvconf_src = Path('/etc/resolv.conf')
if resolvconf_src.exists():
try:
resolvconf_src.copy(root / 'etc/resolv.conf')
except IOError:
pass
# Sets up /bin/sh and /usr/bin/env, downloading busybox if necessary
sh_path = join_root(root, Path('/bin/sh'))
env_path = join_root(root, Path('/usr/bin/env'))
if not sh_path.lexists() or not env_path.lexists():
logger.info("Setting up busybox...")
busybox_path = join_root(root, Path('/bin/busybox'))
busybox_path.parent.mkdir(parents=True)
with make_dir_writable(join_root(root, Path('/bin'))):
download_file(busybox_url(config.runs[0]['architecture']),
busybox_path,
'busybox-%s' % config.runs[0]['architecture'])
busybox_path.chmod(0o755)
if not sh_path.lexists():
sh_path.parent.mkdir(parents=True)
sh_path.symlink('/bin/busybox')
if not env_path.lexists():
env_path.parent.mkdir(parents=True)
env_path.symlink('/bin/busybox')
# Original input files, so upload can restore them
input_files = [f.path for f in config.inputs_outputs.values()
if f.read_runs]
if input_files:
logger.info("Packing up original input files...")
inputtar = tarfile.open(str(target / 'inputs.tar.gz'), 'w:gz')
for ifile in input_files:
filename = join_root(root, ifile)
if filename.exists():
inputtar.add(str(filename), str(ifile))
inputtar.close()
# Meta-data for reprounzip
metadata_write(target, metadata_initial_iofiles(config), 'chroot')
signals.post_setup(target=target, pack=pack)
except Exception:
rmtree_fixed(root)
raise
def directory_run(args):
"""Runs the command in the directory.
"""
target = Path(args.target[0])
unpacked_info = metadata_read(target, 'directory')
cmdline = args.cmdline
# Loads config
config = load_config_file(target / 'config.yml', True)
runs = config.runs
selected_runs = get_runs(runs, args.run, cmdline)
root = (target / 'root').absolute()
# Gets library paths
lib_dirs = []
logger.debug("Running: %s", "/sbin/ldconfig -v -N")
p = subprocess.Popen(['/sbin/ldconfig', '-v', '-N'],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, _ = p.communicate()
try:
for line in stdout.splitlines():
if len(line) < 2 or line[0] in (b' ', b'\t'):
continue
if line.endswith(b':'):
lib_dirs.append(Path(line[:-1]))
finally:
if p.returncode != 0:
raise subprocess.CalledProcessError(p.returncode,
['/sbin/ldconfig', '-v', '-N'])
lib_dirs = ('export LD_LIBRARY_PATH=%s' % ':'.join(
shell_escape(str(join_root(root, d)))
for d in lib_dirs))
cmds = [lib_dirs]
for run_number in selected_runs:
run = runs[run_number]
cmd = 'cd %s && ' % shell_escape(
str(join_root(root, Path(run['workingdir']))))
cmd += '/usr/bin/env -i '
environ = run['environ']
environ = fixup_environment(environ, args)
if args.x11:
if 'DISPLAY' in os.environ:
environ['DISPLAY'] = os.environ['DISPLAY']
if 'XAUTHORITY' in os.environ:
environ['XAUTHORITY'] = os.environ['XAUTHORITY']
cmd += ' '.join('%s=%s' % (shell_escape(k), shell_escape(v))
for k, v in environ.items()
if k != 'PATH')
cmd += ' '
# PATH
# Get the original PATH components
path = [PosixPath(d)
for d in run['environ'].get('PATH', '').split(':')]
# The same paths but in the directory
dir_path = [join_root(root, d)
for d in path
if d.root == '/']
# Rebuild string
path = ':'.join(str(d) for d in dir_path + path)
cmd += 'PATH=%s ' % shell_escape(path)
# FIXME : Use exec -a or something if binary != argv[0]
if cmdline is None:
argv = run['argv']
# Rewrites command-line arguments that are absolute filenames
rewritten = False
for i in range(len(argv)):
try:
p = Path(argv[i])
except UnicodeEncodeError:
continue
if p.is_absolute:
rp = join_root(root, p)
if (rp.exists() or
(len(rp.components) > 3 and rp.parent.exists())):
argv[i] = str(rp)
rewritten = True
if rewritten:
logger.warning("Rewrote command-line as: %s",
' '.join(shell_escape(a) for a in argv))
else:
argv = cmdline
cmd += ' '.join(shell_escape(a) for a in argv)
cmds.append(cmd)
cmds = ' && '.join(cmds)
signals.pre_run(target=target)
logger.debug("Running: %s", cmds)
retcode = interruptible_call(cmds, shell=True)
print("\n*** Command finished, status: %d\n" % retcode, file=sys.stderr)
signals.post_run(target=target, retcode=retcode)
# Update input file status
metadata_update_run(config, unpacked_info, selected_runs)
metadata_write(target, unpacked_info, 'directory')
def write_dict(path, dct):
metadata_write(path, dct, 'docker')
def write_dict(path, dct):
metadata_write(path, dct, 'vagrant')
def write_dict(path, dct):
metadata_write(path, dct, 'docker')
def chroot_run(args):
"""Runs the command in the chroot.
"""
target = Path(args.target[0])
unpacked_info = metadata_read(target, 'chroot')
cmdline = args.cmdline
# Loads config
config = load_config_file(target / 'config.yml', True)
runs = config.runs
selected_runs = get_runs(runs, args.run, cmdline)
root = target / 'root'
# X11 handler
x11 = X11Handler(args.x11, ('local', socket.gethostname()),
args.x11_display)
cmds = []
for run_number in selected_runs:
run = runs[run_number]
cmd = 'cd %s && ' % shell_escape(run['workingdir'])
cmd += '/usr/bin/env -i '
environ = x11.fix_env(run['environ'])
environ = fixup_environment(environ, args)
cmd += ' '.join('%s=%s' % (shell_escape(k), shell_escape(v))
for k, v in iteritems(environ))
cmd += ' '
# FIXME : Use exec -a or something if binary != argv[0]
if cmdline is None:
argv = [run['binary']] + run['argv'][1:]
else:
argv = cmdline
cmd += ' '.join(shell_escape(a) for a in argv)
userspec = '%s:%s' % (run.get('uid', 1000),
run.get('gid', 1000))
cmd = 'chroot --userspec=%s %s /bin/sh -c %s' % (
userspec,
shell_escape(unicode_(root)),
shell_escape(cmd))
cmds.append(cmd)
cmds = ['chroot %s /bin/sh -c %s' % (shell_escape(unicode_(root)),
shell_escape(c))
for c in x11.init_cmds] + cmds
cmds = ' && '.join(cmds)
# Starts forwarding
forwarders = []
for portnum, connector in x11.port_forward:
fwd = LocalForwarder(connector, portnum)
forwarders.append(fwd)
signals.pre_run(target=target)
retcode = interruptible_call(cmds, shell=True)
stderr.write("\n*** Command finished, status: %d\n" % retcode)
signals.post_run(target=target, retcode=retcode)
# Update input file status
metadata_update_run(config, unpacked_info, selected_runs)
metadata_write(target, unpacked_info, 'chroot')
def chroot_create(args):
"""Unpacks the experiment in a folder so it can be run with chroot.
All the files in the pack are unpacked; system files are copied only if
they were not packed, and busybox is installed if /bin/sh wasn't packed.
In addition, input files are put in a tar.gz (so they can be put back after
an upload) and the configuration file is extracted.
"""
if not args.pack:
logging.critical("setup/create needs the pack filename")
sys.exit(1)
pack = Path(args.pack[0])
target = Path(args.target[0])
if target.exists():
logging.critical("Target directory exists")
sys.exit(1)
if DefaultAbstractPath is not PosixPath:
logging.critical("Not unpacking on POSIX system")
sys.exit(1)
signals.pre_setup(target=target, pack=pack)
# We can only restore owner/group of files if running as root
restore_owner = should_restore_owner(args.restore_owner)
# Unpacks configuration file
rpz_pack = RPZPack(pack)
rpz_pack.extract_config(target / 'config.yml')
# Loads config
config = load_config_file(target / 'config.yml', True)
packages = config.packages
target.mkdir()
root = (target / 'root').absolute()
root.mkdir()
try:
# Checks that everything was packed
packages_not_packed = [pkg for pkg in packages if not pkg.packfiles]
if packages_not_packed:
record_usage(chroot_missing_pkgs=True)
logging.warning("According to configuration, some files were left "
"out because they belong to the following "
"packages:%s\nWill copy files from HOST SYSTEM",
''.join('\n %s' % pkg
for pkg in packages_not_packed))
missing_files = False
for pkg in packages_not_packed:
for f in pkg.files:
path = Path(f.path)
if not path.exists():
logging.error(
"Missing file %s (from package %s) on host, "
"experiment will probably miss it",
path, pkg.name)
missing_files = True
continue
dest = join_root(root, path)
dest.parent.mkdir(parents=True)
if path.is_link():
dest.symlink(path.read_link())
else:
path.copy(dest)
if restore_owner:
stat = path.stat()
dest.chown(stat.st_uid, stat.st_gid)
if missing_files:
record_usage(chroot_mising_files=True)
# Unpacks files
members = rpz_pack.list_data()
for m in members:
# Remove 'DATA/' prefix
m.name = str(rpz_pack.remove_data_prefix(m.name))
if not restore_owner:
uid = os.getuid()
gid = os.getgid()
for m in members:
m.uid = uid
m.gid = gid
logging.info("Extracting files...")
rpz_pack.extract_data(root, members)
rpz_pack.close()
# Sets up /bin/sh and /usr/bin/env, downloading busybox if necessary
sh_path = join_root(root, Path('/bin/sh'))
env_path = join_root(root, Path('/usr/bin/env'))
if not sh_path.lexists() or not env_path.lexists():
logging.info("Setting up busybox...")
busybox_path = join_root(root, Path('/bin/busybox'))
busybox_path.parent.mkdir(parents=True)
with make_dir_writable(join_root(root, Path('/bin'))):
download_file(busybox_url(config.runs[0]['architecture']),
busybox_path,
'busybox-%s' % config.runs[0]['architecture'])
busybox_path.chmod(0o755)
if not sh_path.lexists():
sh_path.parent.mkdir(parents=True)
sh_path.symlink('/bin/busybox')
if not env_path.lexists():
env_path.parent.mkdir(parents=True)
env_path.symlink('/bin/busybox')
# Original input files, so upload can restore them
input_files = [f.path for f in itervalues(config.inputs_outputs)
if f.read_runs]
if input_files:
logging.info("Packing up original input files...")
inputtar = tarfile.open(str(target / 'inputs.tar.gz'), 'w:gz')
for ifile in input_files:
filename = join_root(root, ifile)
if filename.exists():
inputtar.add(str(filename), str(ifile))
inputtar.close()
# Meta-data for reprounzip
metadata_write(target, metadata_initial_iofiles(config), 'chroot')
signals.post_setup(target=target, pack=pack)
except Exception:
rmtree_fixed(root)
raise
def directory_run(args):
"""Runs the command in the directory.
"""
target = Path(args.target[0])
unpacked_info = metadata_read(target, 'directory')
cmdline = args.cmdline
# Loads config
config = load_config_file(target / 'config.yml', True)
runs = config.runs
selected_runs = get_runs(runs, args.run, cmdline)
root = (target / 'root').absolute()
# Gets library paths
lib_dirs = []
p = subprocess.Popen(['/sbin/ldconfig', '-v', '-N'],
stdout=subprocess.PIPE)
try:
for l in p.stdout:
if len(l) < 3 or l[0] in (b' ', b'\t'):
continue
if l.endswith(b':\n'):
lib_dirs.append(Path(l[:-2]))
finally:
p.communicate()
lib_dirs = ('export LD_LIBRARY_PATH=%s' % ':'.join(
shell_escape(unicode_(join_root(root, d)))
for d in lib_dirs))
cmds = [lib_dirs]
for run_number in selected_runs:
run = runs[run_number]
cmd = 'cd %s && ' % shell_escape(
unicode_(join_root(root,
Path(run['workingdir']))))
cmd += '/usr/bin/env -i '
environ = run['environ']
environ = fixup_environment(environ, args)
if args.x11:
if 'DISPLAY' in os.environ:
environ['DISPLAY'] = os.environ['DISPLAY']
if 'XAUTHORITY' in os.environ:
environ['XAUTHORITY'] = os.environ['XAUTHORITY']
cmd += ' '.join('%s=%s' % (shell_escape(k), shell_escape(v))
for k, v in iteritems(environ)
if k != 'PATH')
cmd += ' '
# PATH
# Get the original PATH components
path = [PosixPath(d)
for d in run['environ'].get('PATH', '').split(':')]
# The same paths but in the directory
dir_path = [join_root(root, d)
for d in path
if d.root == '/']
# Rebuild string
path = ':'.join(unicode_(d) for d in dir_path + path)
cmd += 'PATH=%s ' % shell_escape(path)
# FIXME : Use exec -a or something if binary != argv[0]
if cmdline is None:
argv = run['argv']
# Rewrites command-line arguments that are absolute filenames
rewritten = False
for i in irange(len(argv)):
try:
p = Path(argv[i])
except UnicodeEncodeError:
continue
if p.is_absolute:
rp = join_root(root, p)
if (rp.exists() or
(len(rp.components) > 3 and rp.parent.exists())):
argv[i] = str(rp)
rewritten = True
if rewritten:
logging.warning("Rewrote command-line as: %s",
' '.join(shell_escape(a) for a in argv))
else:
argv = cmdline
cmd += ' '.join(shell_escape(a) for a in argv)
cmds.append(cmd)
cmds = ' && '.join(cmds)
signals.pre_run(target=target)
retcode = interruptible_call(cmds, shell=True)
stderr.write("\n*** Command finished, status: %d\n" % retcode)
signals.post_run(target=target, retcode=retcode)
# Update input file status
metadata_update_run(config, unpacked_info, selected_runs)
metadata_write(target, unpacked_info, 'directory')
def directory_create(args):
"""Unpacks the experiment in a folder.
Only the files that are not part of a package are copied (unless they are
missing from the system and were packed).
In addition, input files are put in a tar.gz (so they can be put back after
an upload) and the configuration file is extracted.
"""
if not args.pack:
logging.critical("setup needs the pack filename")
sys.exit(1)
pack = Path(args.pack[0])
target = Path(args.target[0])
if target.exists():
logging.critical("Target directory exists")
sys.exit(1)
if not issubclass(DefaultAbstractPath, PosixPath):
logging.critical("Not unpacking on POSIX system")
sys.exit(1)
signals.pre_setup(target=target, pack=pack)
# Unpacks configuration file
rpz_pack = RPZPack(pack)
rpz_pack.extract_config(target / 'config.yml')
# Loads config
config = load_config_file(target / 'config.yml', True)
packages = config.packages
target.mkdir()
root = (target / 'root').absolute()
# Checks packages
missing_files = False
for pkg in packages:
if pkg.packfiles:
continue
for f in pkg.files:
if not Path(f.path).exists():
logging.error(
"Missing file %s (from package %s that wasn't packed) "
"on host, experiment will probably miss it.",
f, pkg.name)
missing_files = True
if missing_files:
record_usage(directory_missing_pkgs=True)
logging.error("Some packages are missing, you should probably install "
"them.\nUse 'reprounzip installpkgs -h' for help")
root.mkdir()
try:
# Unpacks files
members = rpz_pack.list_data()
for m in members:
# Remove 'DATA/' prefix
m.name = str(rpz_pack.remove_data_prefix(m.name))
# Makes symlink targets relative
if m.issym():
linkname = PosixPath(m.linkname)
if linkname.is_absolute:
m.linkname = join_root(root, PosixPath(m.linkname)).path
logging.info("Extracting files...")
rpz_pack.extract_data(root, members)
rpz_pack.close()
# Original input files, so upload can restore them
input_files = [f.path for f in itervalues(config.inputs_outputs)
if f.read_runs]
if input_files:
logging.info("Packing up original input files...")
inputtar = tarfile.open(str(target / 'inputs.tar.gz'), 'w:gz')
for ifile in input_files:
filename = join_root(root, ifile)
if filename.exists():
inputtar.add(str(filename), str(ifile))
inputtar.close()
# Meta-data for reprounzip
metadata_write(target, metadata_initial_iofiles(config), 'directory')
signals.post_setup(target=target, pack=pack)
except Exception:
rmtree_fixed(root)
raise
def directory_create(args):
"""Unpacks the experiment in a folder.
Only the files that are not part of a package are copied (unless they are
missing from the system and were packed).
In addition, input files are put in a tar.gz (so they can be put back after
an upload) and the configuration file is extracted.
"""
if not args.pack:
logger.critical("setup needs the pack filename")
sys.exit(1)
pack = Path(args.pack[0])
target = Path(args.target[0])
if target.exists():
logger.critical("Target directory exists")
sys.exit(1)
if not issubclass(DefaultAbstractPath, PosixPath):
logger.critical("Not unpacking on POSIX system")
sys.exit(1)
signals.pre_setup(target=target, pack=pack)
# Unpacks configuration file
rpz_pack = RPZPack(pack)
rpz_pack.extract_config(target / 'config.yml')
# Loads config
config = load_config_file(target / 'config.yml', True)
packages = config.packages
target.mkdir()
root = (target / 'root').absolute()
# Checks packages
missing_files = False
for pkg in packages:
if pkg.packfiles:
continue
for f in pkg.files:
if not Path(f.path).exists():
logger.error(
"Missing file %s (from package %s that wasn't packed) "
"on host, experiment will probably miss it.",
f, pkg.name)
missing_files = True
if missing_files:
record_usage(directory_missing_pkgs=True)
logger.error("Some packages are missing, you should probably install "
"them.\nUse 'reprounzip installpkgs -h' for help")
root.mkdir()
try:
# Unpacks files
members = rpz_pack.list_data()
for m in members:
# Remove 'DATA/' prefix
m.name = str(rpz_pack.remove_data_prefix(m.name))
# Makes symlink targets relative
if m.issym():
linkname = PosixPath(m.linkname)
if linkname.is_absolute:
m.linkname = join_root(root, PosixPath(m.linkname)).path
logger.info("Extracting files...")
rpz_pack.extract_data(root, members)
rpz_pack.close()
# Original input files, so upload can restore them
input_files = [f.path for f in config.inputs_outputs.values()
if f.read_runs]
if input_files:
logger.info("Packing up original input files...")
inputtar = tarfile.open(str(target / 'inputs.tar.gz'), 'w:gz')
for ifile in input_files:
filename = join_root(root, ifile)
if filename.exists():
inputtar.add(str(filename), str(ifile))
inputtar.close()
# Meta-data for reprounzip
metadata_write(target, metadata_initial_iofiles(config), 'directory')
signals.post_setup(target=target, pack=pack)
except Exception:
rmtree_fixed(root)
raise
def write_dict(path, dct):
metadata_write(path, dct, 'vagrant')
