def unique_names():
"""Generates unique sequences of bytes.
"""
characters = (b"abcdefghijklmnopqrstuvwxyz" b"0123456789")
characters = [characters[i:i + 1] for i in irange(len(characters))]
rng = random.Random()
while True:
letters = [rng.choice(characters) for i in irange(10)]
yield b''.join(letters)
def get_runs(runs, selected_runs, cmdline):
"""Selects which run(s) to execute based on parts of the command-line.
Will return an iterable of run numbers. Might also fail loudly or exit
after printing the original command-line.
"""
name_map = dict((r['id'], i) for i, r in enumerate(runs) if 'id' in r)
run_list = []
def parse_run(s):
try:
r = int(s)
except ValueError:
logging.critical("Error: Unknown run %s", s)
raise UsageError
if r < 0 or r >= len(runs):
logging.critical("Error: Expected 0 <= run <= %d, got %d",
len(runs) - 1, r)
sys.exit(1)
return r
if selected_runs is None:
run_list = list(irange(len(runs)))
else:
for run_item in selected_runs.split(','):
run_item = run_item.strip()
if run_item in name_map:
run_list.append(name_map[run_item])
continue
sep = run_item.find('-')
if sep == -1:
run_list.append(parse_run(run_item))
else:
if sep > 0:
first = parse_run(run_item[:sep])
else:
first = 0
if sep + 1 < len(run_item):
last = parse_run(run_item[sep + 1:])
else:
last = len(runs) - 1
if last < first:
logging.critical("Error: Last run number should be "
"greater than the first")
sys.exit(1)
run_list.extend(irange(first, last + 1))
# --cmdline without arguments: display the original command-line
if cmdline == []:
print("Original command-lines:")
for run in run_list:
print(' '.join(shell_escape(arg)
for arg in runs[run]['argv']))
sys.exit(0)
return run_list
def get_runs(runs, selected_runs, cmdline):
"""Selects which run(s) to execute based on parts of the command-line.
Will return an iterable of run numbers. Might also fail loudly or exit
after printing the original command-line.
"""
name_map = dict((r['id'], i) for i, r in enumerate(runs) if 'id' in r)
run_list = []
def parse_run(s):
try:
r = int(s)
except ValueError:
logging.critical("Error: Unknown run %s", s)
raise UsageError
if r < 0 or r >= len(runs):
logging.critical("Error: Expected 0 <= run <= %d, got %d",
len(runs) - 1, r)
sys.exit(1)
return r
if selected_runs is None:
run_list = list(irange(len(runs)))
else:
for run_item in selected_runs.split(','):
run_item = run_item.strip()
if run_item in name_map:
run_list.append(name_map[run_item])
continue
sep = run_item.find('-')
if sep == -1:
run_list.append(parse_run(run_item))
else:
if sep > 0:
first = parse_run(run_item[:sep])
else:
first = 0
if sep + 1 < len(run_item):
last = parse_run(run_item[sep + 1:])
else:
last = len(runs) - 1
if last < first:
logging.critical("Error: Last run number should be "
"greater than the first")
sys.exit(1)
run_list.extend(irange(first, last + 1))
# --cmdline without arguments: display the original command-line
if cmdline == []:
print("Original command-lines:")
for run in run_list:
print(' '.join(shell_escape(arg)
for arg in runs[run]['argv']))
sys.exit(0)
return run_list
def unique_names():
"""Generates unique sequences of bytes.
"""
characters = (b"abcdefghijklmnopqrstuvwxyz"
b"0123456789")
characters = [characters[i:i + 1] for i in irange(len(characters))]
rng = random.Random()
while True:
letters = [rng.choice(characters) for i in irange(10)]
yield b''.join(letters)
def test_make_unique_name(self):
"""Tests the make_unique_name() function."""
names = [make_unique_name(b'/some/prefix_') for i in irange(3)]
for n in names:
self.assertTrue(n and isinstance(n, bytes)
and n[:13] == b'/some/prefix_')
self.assertEqual(len(set(names)), len(names))
def get_runs(runs, selected_run, cmdline):
"""Selects which run(s) to execute based on parts of the command-line.
Will return an iterable of run numbers. Might also fail loudly or exit
after printing the original command-line.
"""
if selected_run is None and len(runs) == 1:
selected_run = 0
# --cmdline without arguments: display the original command-line
if cmdline == []:
if selected_run is None:
logging.critical("There are several runs in this pack -- you have "
"to choose which one to use with --cmdline")
sys.exit(1)
print("Original command-line:")
print(' '.join(shell_escape(arg)
for arg in runs[selected_run]['argv']))
sys.exit(0)
if selected_run is None:
selected_run = irange(len(runs))
else:
selected_run = (int(selected_run),)
return selected_run
def test_make_unique_name(self):
"""Tests the make_unique_name() function."""
names = [make_unique_name(b'/some/prefix_') for i in irange(3)]
for n in names:
self.assertTrue(n and isinstance(n, bytes) and
n[:13] == b'/some/prefix_')
self.assertEqual(len(set(names)), len(names))
def get_runs(runs, selected_runs, cmdline):
"""Selects which run(s) to execute based on parts of the command-line.
Will return an iterable of run numbers. Might also fail loudly or exit
after printing the original command-line.
"""
if selected_runs is None:
if len(runs) == 1:
selected_runs = '0'
else:
logging.critical("There are several runs in this pack -- you have "
"to choose which one to use")
sys.exit(1)
def parse_run(s):
try:
r = int(s)
except ValueError:
logging.critical("Error: Run is not a number")
raise UsageError
if r < 0 or r >= len(runs):
logging.critical("Error: Expected 0 <= run <= %d, got %d",
len(runs) - 1, r)
sys.exit(1)
return r
sep = selected_runs.find('-')
if sep == -1:
selected_runs = parse_run(selected_runs),
else:
if sep > 0:
first = parse_run(selected_runs[:sep])
else:
first = 0
if sep + 1 < len(selected_runs):
last = parse_run(selected_runs[sep + 1:])
else:
last = len(runs) - 1
if last <= first:
logging.critical("Error: Last run number should be greater than "
"the first")
sys.exit(1)
selected_runs = irange(first, last + 1)
# --cmdline without arguments: display the original command-line
if cmdline == []:
print("Original command-lines:")
for run in selected_runs:
print(' '.join(shell_escape(arg)
for arg in runs[run]['argv']))
sys.exit(0)
return selected_runs
def find_ssh_executable(name='ssh'):
exts = os.environ.get('PATHEXT', '').split(os.pathsep)
dirs = list(os.environ.get('PATH', '').split(os.pathsep))
par, join = os.path.dirname, os.path.join
# executable might be bin/python or ReproUnzip\python
# or ReproUnzip\Python27\python or ReproUnzip\Python27\Scripts\something
loc = par(sys.executable)
local_dirs = []
for i in irange(3):
local_dirs.extend([loc, join(loc, 'ssh')])
loc = par(loc)
for pathdir in local_dirs + dirs:
for ext in exts:
fullpath = os.path.join(pathdir, name + ext)
if os.path.isfile(fullpath):
return fullpath
return None
def find_ssh_executable(name='ssh'):
exts = os.environ.get('PATHEXT', '').split(os.pathsep)
dirs = list(os.environ.get('PATH', '').split(os.pathsep))
par, join = os.path.dirname, os.path.join
# executable might be bin/python or ReproUnzip\python
# or ReproUnzip\Python27\python or ReproUnzip\Python27\Scripts\something
loc = par(sys.executable)
local_dirs = []
for i in irange(3):
local_dirs.extend([loc, join(loc, 'ssh')])
loc = par(loc)
for pathdir in local_dirs + dirs:
for ext in exts:
fullpath = os.path.join(pathdir, name + ext)
if os.path.isfile(fullpath):
return fullpath
return None
def init_cmds(self):
"""Gets the commands to setup X on the server before the experiment.
"""
if not self.enabled or self.xauth_record is None:
return []
if self.target[0] == 'local':
xauth_record = Xauth(Xauth.FAMILY_LOCAL, self.target[1],
self.display, self.xauth_record.name,
self.xauth_record.data)
elif self.target[0] == 'internet':
xauth_record = Xauth(Xauth.FAMILY_INTERNET,
socket.inet_aton(self.target[1]),
self.display, self.xauth_record.name,
self.xauth_record.data)
else:
raise RuntimeError("Invalid target display type")
buf = xauth_record.as_bytes()
xauth = ''.join(
('\\x%02x' % ord(buf[i:i + 1])) for i in irange(len(buf)))
return ['echo -ne "%s" > %s' % (xauth, self.xauth)]
def init_cmds(self):
"""Gets the commands to setup X on the server before the experiment.
"""
if not self.enabled or self.xauth_record is None:
return []
if self.target[0] == 'local':
xauth_record = Xauth(Xauth.FAMILY_LOCAL,
self.target[1],
self.display,
self.xauth_record.name,
self.xauth_record.data)
elif self.target[0] == 'internet':
xauth_record = Xauth(Xauth.FAMILY_INTERNET,
socket.inet_aton(self.target[1]),
self.display,
self.xauth_record.name,
self.xauth_record.data)
else:
raise RuntimeError("Invalid target display type")
buf = xauth_record.as_bytes()
xauth = ''.join(('\\x%02x' % ord(buf[i:i + 1]))
for i in irange(len(buf)))
return ['echo -ne "%s" > %s' % (xauth, self.xauth)]
def docker_setup_create(args):
"""Sets up the experiment to be run in a Docker-built container.
"""
pack = Path(args.pack[0])
target = Path(args.target[0])
if target.exists():
logging.critical("Target directory exists")
sys.exit(1)
signals.pre_setup(target=target, pack=pack)
target.mkdir()
try:
# Unpacks configuration file
rpz_pack = RPZPack(pack)
rpz_pack.extract_config(target / 'config.yml')
# Loads config
runs, packages, other_files = config = load_config(
target / 'config.yml', True)
if args.base_image:
record_usage(docker_explicit_base=True)
base_image = args.base_image[0]
if args.distribution:
target_distribution = args.distribution[0]
else:
target_distribution = None
else:
target_distribution, base_image = select_image(runs)
logging.info("Using base image %s", base_image)
logging.debug("Distribution: %s", target_distribution or "unknown")
rpz_pack.copy_data_tar(target / 'data.tgz')
arch = runs[0]['architecture']
# Writes Dockerfile
logging.info("Writing %s...", target / 'Dockerfile')
with (target / 'Dockerfile').open('w', encoding='utf-8',
newline='\n') as fp:
fp.write('FROM %s\n\n' % base_image)
# Installs busybox
download_file(busybox_url(arch),
target / 'busybox',
'busybox-%s' % arch)
fp.write('COPY busybox /busybox\n')
# Installs rpzsudo
download_file(sudo_url(arch),
target / 'rpzsudo',
'rpzsudo-%s' % arch)
fp.write('COPY rpzsudo /rpzsudo\n\n')
fp.write('COPY data.tgz /reprozip_data.tgz\n\n')
fp.write('COPY rpz-files.list /rpz-files.list\n')
fp.write('RUN \\\n'
' chmod +x /busybox /rpzsudo && \\\n')
if args.install_pkgs:
# Install every package through package manager
missing_packages = []
else:
# Only install packages that were not packed
missing_packages = [pkg for pkg in packages if pkg.packfiles]
packages = [pkg for pkg in packages if not pkg.packfiles]
if packages:
record_usage(docker_install_pkgs=True)
try:
installer = select_installer(pack, runs,
target_distribution)
except CantFindInstaller as e:
logging.error("Need to install %d packages but couldn't "
"select a package installer: %s",
len(packages), e)
sys.exit(1)
# Updates package sources
update_script = installer.update_script()
if update_script:
fp.write(' %s && \\\n' % update_script)
# Installs necessary packages
fp.write(' %s && \\\n' % installer.install_script(packages))
logging.info("Dockerfile will install the %d software "
"packages that were not packed", len(packages))
else:
record_usage(docker_install_pkgs=False)
# Untar
paths = set()
pathlist = []
# Add intermediate directories, and check for existence in the tar
logging.info("Generating file list...")
missing_files = chain.from_iterable(pkg.files
for pkg in missing_packages)
data_files = rpz_pack.data_filenames()
listoffiles = list(chain(other_files, missing_files))
for f in listoffiles:
if f.path.name == 'resolv.conf' and (
f.path.lies_under('/etc') or
f.path.lies_under('/run') or
f.path.lies_under('/var')):
continue
path = PosixPath('/')
for c in rpz_pack.remove_data_prefix(f.path).components:
path = path / c
if path in paths:
continue
paths.add(path)
if path in data_files:
pathlist.append(path)
else:
logging.info("Missing file %s", path)
rpz_pack.close()
# FIXME : for some reason we need reversed() here, I'm not sure why
# Need to read more of tar's docs.
# TAR bug: --no-overwrite-dir removes --keep-old-files
with (target / 'rpz-files.list').open('wb') as lfp:
for p in reversed(pathlist):
lfp.write(join_root(rpz_pack.data_prefix, p).path)
lfp.write(b'\0')
fp.write(' cd / && '
'(tar zpxf /reprozip_data.tgz -U --recursive-unlink '
'--numeric-owner --strip=1 --null -T /rpz-files.list || '
'/busybox echo "TAR reports errors, this might or might '
'not prevent the execution to run")\n')
# Setup entry point
fp.write('COPY rpz_entrypoint.sh /rpz_entrypoint.sh\n'
'ENTRYPOINT ["/busybox", "sh", "/rpz_entrypoint.sh"]\n')
# Write entry point script
logging.info("Writing %s...", target / 'rpz_entrypoint.sh')
with (target / 'rpz_entrypoint.sh').open('w', encoding='utf-8',
newline='\n') as fp:
# The entrypoint gets some arguments from the run command
# By default, it just does all the runs
# "run N" executes the run with that number
# "cmd STR" sets a replacement command-line for the next run
# "do STR" executes a command as-is
fp.write(
'#!/bin/sh\n'
'\n'
'COMMAND=\n'
'ENVVARS=\n'
'\n'
'if [ $# = 0 ]; then\n'
' exec /busybox sh /rpz_entrypoint.sh')
for nb in irange(len(runs)):
fp.write(' run %d' % nb)
fp.write(
'\n'
'fi\n'
'\n'
'while [ $# != 0 ]; do\n'
' case "$1" in\n'
' help)\n'
' echo "Image built from reprounzip-docker" >&2\n'
' echo "Usage: docker run <image> [cmd word [word '
'...]] [run <R>]" >&2\n'
' echo " \\`cmd ...\\` changes the command for '
'the next \\`run\\` option" >&2\n'
' echo " \\`run <name|number>\\` runs the '
'specified run" >&2\n'
' echo "By default, all the runs are executed." '
'>&2\n'
' echo "The runs in this image are:" >&2\n')
for run in runs:
fp.write(
' echo " {name}: {cmdline}" >&2\n'.format(
name=run['id'],
cmdline=' '.join(shell_escape(a)
for a in run['argv'])))
fp.write(
' exit 0\n'
' ;;\n'
' do)\n'
' shift\n'
' $1\n'
' ;;\n'
' env)\n'
' shift\n'
' ENVVARS="$1"\n'
' ;;\n'
' cmd)\n'
' shift\n'
' COMMAND="$1"\n'
' ;;\n'
' run)\n'
' shift\n'
' case "$1" in\n')
for i, run in enumerate(runs):
cmdline = ' '.join([run['binary']] + run['argv'][1:])
fp.write(
' {name})\n'
' RUNCOMMAND={cmd}\n'
' RUNWD={wd}\n'
' RUNENV={env}\n'
' RUNUID={uid}\n'
' RUNGID={gid}\n'
' ;;\n'.format(
name='%s|%d' % (run['id'], i),
cmd=shell_escape(cmdline),
wd=shell_escape(run['workingdir']),
env=shell_escape(' '.join(
'%s=%s' % (shell_escape(k), shell_escape(v))
for k, v in iteritems(run['environ']))),
uid=run.get('uid', 1000),
gid=run.get('gid', 1000)))
fp.write(
' *)\n'
' echo "RPZ: Unknown run $1" >&2\n'
' exit 1\n'
' ;;\n'
' esac\n'
' if [ -n "$COMMAND" ]; then\n'
' RUNCOMMAND="$COMMAND"\n'
' COMMAND=\n'
' fi\n'
' export RUNWD; export RUNENV; export ENVVARS; '
'export RUNCOMMAND\n'
' /rpzsudo "#$RUNUID" "#$RUNGID" /busybox sh -c '
'"cd \\"\\$RUNWD\\" && /busybox env -i $RUNENV $ENVVARS '
'$RUNCOMMAND"\n'
' ENVVARS=\n'
' ;;\n'
' *)\n'
' echo "RPZ: Unknown option $1" >&2\n'
' exit 1\n'
' ;;\n'
' esac\n'
' shift\n'
'done\n')
# Meta-data for reprounzip
write_dict(target, metadata_initial_iofiles(config))
signals.post_setup(target=target, pack=pack)
except Exception:
target.rmtree(ignore_errors=True)
raise
def directory_run(args):
"""Runs the command in the directory.
"""
target = Path(args.target[0])
unpacked_info = metadata_read(target, 'directory')
cmdline = args.cmdline
# Loads config
config = load_config_file(target / 'config.yml', True)
runs = config.runs
selected_runs = get_runs(runs, args.run, cmdline)
root = (target / 'root').absolute()
# Gets library paths
lib_dirs = []
p = subprocess.Popen(['/sbin/ldconfig', '-v', '-N'],
stdout=subprocess.PIPE)
try:
for l in p.stdout:
if len(l) < 3 or l[0] in (b' ', b'\t'):
continue
if l.endswith(b':\n'):
lib_dirs.append(Path(l[:-2]))
finally:
p.communicate()
lib_dirs = ('export LD_LIBRARY_PATH=%s' % ':'.join(
shell_escape(unicode_(join_root(root, d)))
for d in lib_dirs))
cmds = [lib_dirs]
for run_number in selected_runs:
run = runs[run_number]
cmd = 'cd %s && ' % shell_escape(
unicode_(join_root(root,
Path(run['workingdir']))))
cmd += '/usr/bin/env -i '
environ = run['environ']
environ = fixup_environment(environ, args)
if args.x11:
if 'DISPLAY' in os.environ:
environ['DISPLAY'] = os.environ['DISPLAY']
if 'XAUTHORITY' in os.environ:
environ['XAUTHORITY'] = os.environ['XAUTHORITY']
cmd += ' '.join('%s=%s' % (shell_escape(k), shell_escape(v))
for k, v in iteritems(environ)
if k != 'PATH')
cmd += ' '
# PATH
# Get the original PATH components
path = [PosixPath(d)
for d in run['environ'].get('PATH', '').split(':')]
# The same paths but in the directory
dir_path = [join_root(root, d)
for d in path
if d.root == '/']
# Rebuild string
path = ':'.join(unicode_(d) for d in dir_path + path)
cmd += 'PATH=%s ' % shell_escape(path)
# FIXME : Use exec -a or something if binary != argv[0]
if cmdline is None:
argv = run['argv']
# Rewrites command-line arguments that are absolute filenames
rewritten = False
for i in irange(len(argv)):
try:
p = Path(argv[i])
except UnicodeEncodeError:
continue
if p.is_absolute:
rp = join_root(root, p)
if (rp.exists() or
(len(rp.components) > 3 and rp.parent.exists())):
argv[i] = str(rp)
rewritten = True
if rewritten:
logging.warning("Rewrote command-line as: %s",
' '.join(shell_escape(a) for a in argv))
else:
argv = cmdline
cmd += ' '.join(shell_escape(a) for a in argv)
cmds.append(cmd)
cmds = ' && '.join(cmds)
signals.pre_run(target=target)
retcode = interruptible_call(cmds, shell=True)
stderr.write("\n*** Command finished, status: %d\n" % retcode)
signals.post_run(target=target, retcode=retcode)
# Update input file status
metadata_update_run(config, unpacked_info, selected_runs)
metadata_write(target, unpacked_info, 'directory')
def test_unique_names(self):
"""Tests the unique_names generator."""
names = [next(unique_names) for i in irange(3)]
for n in names:
self.assertTrue(n and isinstance(n, bytes))
self.assertEqual(len(set(names)), len(names))
def directory_run(args):
"""Runs the command in the directory.
"""
target = Path(args.target[0])
unpacked_info = metadata_read(target, 'directory')
cmdline = args.cmdline
# Loads config
config = load_config_file(target / 'config.yml', True)
runs = config.runs
selected_runs = get_runs(runs, args.run, cmdline)
root = (target / 'root').absolute()
# Gets library paths
lib_dirs = []
p = subprocess.Popen(['/sbin/ldconfig', '-v', '-N'],
stdout=subprocess.PIPE)
try:
for l in p.stdout:
if len(l) < 3 or l[0] in (b' ', b'\t'):
continue
if l.endswith(b':\n'):
lib_dirs.append(Path(l[:-2]))
finally:
p.wait()
lib_dirs = (
'export LD_LIBRARY_PATH=%s' %
':'.join(shell_escape(unicode_(join_root(root, d))) for d in lib_dirs))
cmds = [lib_dirs]
for run_number in selected_runs:
run = runs[run_number]
cmd = 'cd %s && ' % shell_escape(
unicode_(join_root(root, Path(run['workingdir']))))
cmd += '/usr/bin/env -i '
environ = run['environ']
environ = fixup_environment(environ, args)
if args.x11:
if 'DISPLAY' in os.environ:
environ['DISPLAY'] = os.environ['DISPLAY']
if 'XAUTHORITY' in os.environ:
environ['XAUTHORITY'] = os.environ['XAUTHORITY']
cmd += ' '.join('%s=%s' % (shell_escape(k), shell_escape(v))
for k, v in iteritems(environ) if k != 'PATH')
cmd += ' '
# PATH
# Get the original PATH components
path = [
PosixPath(d) for d in run['environ'].get('PATH', '').split(':')
]
# The same paths but in the directory
dir_path = [join_root(root, d) for d in path if d.root == '/']
# Rebuild string
path = ':'.join(unicode_(d) for d in dir_path + path)
cmd += 'PATH=%s ' % shell_escape(path)
# FIXME : Use exec -a or something if binary != argv[0]
if cmdline is None:
argv = run['argv']
# Rewrites command-line arguments that are absolute filenames
rewritten = False
for i in irange(len(argv)):
try:
p = Path(argv[i])
except UnicodeEncodeError:
continue
if p.is_absolute:
rp = join_root(root, p)
if (rp.exists() or
(len(rp.components) > 3 and rp.parent.exists())):
argv[i] = str(rp)
rewritten = True
if rewritten:
logger.warning("Rewrote command-line as: %s",
' '.join(shell_escape(a) for a in argv))
else:
argv = cmdline
cmd += ' '.join(shell_escape(a) for a in argv)
cmds.append(cmd)
cmds = ' && '.join(cmds)
signals.pre_run(target=target)
retcode = interruptible_call(cmds, shell=True)
stderr.write("\n*** Command finished, status: %d\n" % retcode)
signals.post_run(target=target, retcode=retcode)
# Update input file status
metadata_update_run(config, unpacked_info, selected_runs)
metadata_write(target, unpacked_info, 'directory')
def docker_setup_create(args):
"""Sets up the experiment to be run in a Docker-built container.
"""
pack = Path(args.pack[0])
target = Path(args.target[0])
if target.exists():
logger.critical("Target directory exists")
sys.exit(1)
signals.pre_setup(target=target, pack=pack)
target.mkdir()
try:
# Unpacks configuration file
rpz_pack = RPZPack(pack)
rpz_pack.extract_config(target / 'config.yml')
# Loads config
runs, packages, other_files = config = load_config(
target / 'config.yml', True)
if args.base_image:
record_usage(docker_explicit_base=True)
base_image = args.base_image[0]
if args.distribution:
target_distribution = args.distribution[0]
else:
target_distribution = None
else:
target_distribution, base_image = select_image(runs)
logger.info("Using base image %s", base_image)
logger.debug("Distribution: %s", target_distribution or "unknown")
rpz_pack.copy_data_tar(target / 'data.tgz')
arch = runs[0]['architecture']
# Writes Dockerfile
logger.info("Writing %s...", target / 'Dockerfile')
with (target / 'Dockerfile').open('w', encoding='utf-8',
newline='\n') as fp:
fp.write('FROM %s\n\n' % base_image)
# Installs busybox
download_file(busybox_url(arch),
target / 'busybox',
'busybox-%s' % arch)
fp.write('COPY busybox /busybox\n')
# Installs rpzsudo
download_file(sudo_url(arch),
target / 'rpzsudo',
'rpzsudo-%s' % arch)
fp.write('COPY rpzsudo /rpzsudo\n\n')
fp.write('COPY data.tgz /reprozip_data.tgz\n\n')
fp.write('COPY rpz-files.list /rpz-files.list\n')
fp.write('RUN \\\n'
' chmod +x /busybox /rpzsudo && \\\n')
if args.install_pkgs:
# Install every package through package manager
missing_packages = []
else:
# Only install packages that were not packed
missing_packages = [pkg for pkg in packages if pkg.packfiles]
packages = [pkg for pkg in packages if not pkg.packfiles]
if packages:
record_usage(docker_install_pkgs=True)
try:
installer = select_installer(pack, runs,
target_distribution)
except CantFindInstaller as e:
logger.error("Need to install %d packages but couldn't "
"select a package installer: %s",
len(packages), e)
sys.exit(1)
# Updates package sources
update_script = installer.update_script()
if update_script:
fp.write(' %s && \\\n' % update_script)
# Installs necessary packages
fp.write(' %s && \\\n' % installer.install_script(packages))
logger.info("Dockerfile will install the %d software "
"packages that were not packed", len(packages))
else:
record_usage(docker_install_pkgs=False)
# Untar
paths = set()
pathlist = []
# Add intermediate directories, and check for existence in the tar
logger.info("Generating file list...")
missing_files = chain.from_iterable(pkg.files
for pkg in missing_packages)
data_files = rpz_pack.data_filenames()
listoffiles = list(chain(other_files, missing_files))
for f in listoffiles:
if f.path.name == 'resolv.conf' and (
f.path.lies_under('/etc') or
f.path.lies_under('/run') or
f.path.lies_under('/var')):
continue
path = PosixPath('/')
for c in rpz_pack.remove_data_prefix(f.path).components:
path = path / c
if path in paths:
continue
paths.add(path)
if path in data_files:
pathlist.append(path)
else:
logger.info("Missing file %s", path)
rpz_pack.close()
# FIXME : for some reason we need reversed() here, I'm not sure why
# Need to read more of tar's docs.
# TAR bug: --no-overwrite-dir removes --keep-old-files
with (target / 'rpz-files.list').open('wb') as lfp:
for p in reversed(pathlist):
lfp.write(join_root(rpz_pack.data_prefix, p).path)
lfp.write(b'\0')
fp.write(' cd / && '
'(tar zpxf /reprozip_data.tgz -U --recursive-unlink '
'--numeric-owner --strip=1 --null -T /rpz-files.list || '
'/busybox echo "TAR reports errors, this might or might '
'not prevent the execution to run")\n')
# Setup entry point
fp.write('COPY rpz_entrypoint.sh /rpz_entrypoint.sh\n'
'ENTRYPOINT ["/busybox", "sh", "/rpz_entrypoint.sh"]\n')
# Write entry point script
logger.info("Writing %s...", target / 'rpz_entrypoint.sh')
with (target / 'rpz_entrypoint.sh').open('w', encoding='utf-8',
newline='\n') as fp:
# The entrypoint gets some arguments from the run command
# By default, it just does all the runs
# "run N" executes the run with that number
# "cmd STR" sets a replacement command-line for the next run
# "do STR" executes a command as-is
fp.write(
'#!/bin/sh\n'
'\n'
'COMMAND=\n'
'ENVVARS=\n'
'\n'
'if [ $# = 0 ]; then\n'
' exec /busybox sh /rpz_entrypoint.sh')
for nb in irange(len(runs)):
fp.write(' run %d' % nb)
fp.write(
'\n'
'fi\n'
'\n'
'while [ $# != 0 ]; do\n'
' case "$1" in\n'
' help)\n'
' echo "Image built from reprounzip-docker" >&2\n'
' echo "Usage: docker run <image> [cmd "word [word '
'...]"] [run <R>]" >&2\n'
' echo " \\`cmd ...\\` changes the command for '
'the next \\`run\\` option" >&2\n'
' echo " \\`run <name|number>\\` runs the '
'specified run" >&2\n'
' echo "By default, all the runs are executed." '
'>&2\n'
' echo "The runs in this image are:" >&2\n')
for run in runs:
fp.write(
' echo " {name}: {cmdline}" >&2\n'.format(
name=run['id'],
cmdline=' '.join(shell_escape(a)
for a in run['argv'])))
fp.write(
' exit 0\n'
' ;;\n'
' do)\n'
' shift\n'
' $1\n'
' ;;\n'
' env)\n'
' shift\n'
' ENVVARS="$1"\n'
' ;;\n'
' cmd)\n'
' shift\n'
' COMMAND="$1"\n'
' ;;\n'
' run)\n'
' shift\n'
' case "$1" in\n')
for i, run in enumerate(runs):
cmdline = ' '.join([run['binary']] + run['argv'][1:])
fp.write(
' {name})\n'
' RUNCOMMAND={cmd}\n'
' RUNWD={wd}\n'
' RUNENV={env}\n'
' RUNUID={uid}\n'
' RUNGID={gid}\n'
' ;;\n'.format(
name='%s|%d' % (run['id'], i),
cmd=shell_escape(cmdline),
wd=shell_escape(run['workingdir']),
env=shell_escape(' '.join(
'%s=%s' % (shell_escape(k), shell_escape(v))
for k, v in iteritems(run['environ']))),
uid=run.get('uid', 1000),
gid=run.get('gid', 1000)))
fp.write(
' *)\n'
' echo "RPZ: Unknown run $1" >&2\n'
' exit 1\n'
' ;;\n'
' esac\n'
' if [ -n "$COMMAND" ]; then\n'
' RUNCOMMAND="$COMMAND"\n'
' COMMAND=\n'
' fi\n'
' export RUNWD; export RUNENV; export ENVVARS; '
'export RUNCOMMAND\n'
' /rpzsudo "#$RUNUID" "#$RUNGID" /busybox sh -c '
'"cd \\"\\$RUNWD\\" && /busybox env -i $RUNENV $ENVVARS '
'$RUNCOMMAND; echo \\"*** Command finished, status: \\$?\\""\n'
' ENVVARS=\n'
' ;;\n'
' *)\n'
' echo "RPZ: Unknown option $1" >&2\n'
' exit 1\n'
' ;;\n'
' esac\n'
' shift\n'
'done\n')
# Meta-data for reprounzip
write_dict(target, metadata_initial_iofiles(config))
signals.post_setup(target=target, pack=pack)
except Exception:
target.rmtree(ignore_errors=True)
raise
class Signal(object):
"""A signal, with its set of arguments.
This holds the expected parameters that the signal expects, in several
categories:
* `expected_args` are the arguments of the signals that must be set. Trying
to emit the signal without these will show a warning and won't touch the
listeners. Listeners can rely on these being set.
* `new_args` are new arguments that listeners cannot yet rely on but that
emitters should try to pass in. Missing arguments doesn't show a warning
yet but might in the future.
* `old_args` are arguments that you might still pass in but that you should
move away from; they will show a warning stating their deprecation.
Listeners can subscribe to a signal, and may be any callable hashable
object.
"""
REQUIRED, OPTIONAL, DEPRECATED = irange(3)
def __init__(self, expected_args=[], new_args=[], old_args=[]):
self._args = {}
self._args.update((arg, Signal.REQUIRED) for arg in expected_args)
self._args.update((arg, Signal.OPTIONAL) for arg in new_args)
self._args.update((arg, Signal.DEPRECATED) for arg in old_args)
if (len(expected_args) + len(new_args) + len(old_args) != len(
self._args)):
raise ValueError("Repeated argument names")
self._listeners = set()
def __call__(self, **kwargs):
info = {}
for arg, argtype in iteritems(self._args):
if argtype == Signal.REQUIRED:
try:
info[arg] = kwargs.pop(arg)
except KeyError:
warnings.warn("signal: Missing required argument %s; "
"signal ignored" % arg,
category=SignalWarning,
stacklevel=2)
return
else:
if arg in kwargs:
info[arg] = kwargs.pop(arg)
if argtype == Signal.DEPRECATED:
warnings.warn("signal: Argument %s is deprecated" %
arg,
category=SignalWarning,
stacklevel=2)
if kwargs:
arg = next(iter(kwargs))
warnings.warn("signal: Unexpected argument %s; signal ignored" %
arg,
category=SignalWarning,
stacklevel=2)
return
for listener in self._listeners:
try:
listener(**info)
except Exception:
traceback.print_exc()
warnings.warn("signal: Got an exception calling a signal",
category=SignalWarning)
def subscribe(self, func):
"""Adds the given callable to the listeners.
It must be callable and hashable (it will be put in a set).
It will be called with the signals' arguments as keywords. Because new
parameters might be introduced, it should accept these by using::
def my_listener(param1, param2, **kwargs_):
"""
if not callable(func):
raise TypeError("%r object is not callable" % type(func))
self._listeners.add(func)
def unsubscribe(self, func):
"""Removes the given callable from the listeners.
If the listener wasn't subscribed, does nothing.
"""
self._listeners.discard(func)
def test_unique_names(self):
"""Tests the unique_names generator."""
names = [next(unique_names) for i in irange(3)]
for n in names:
self.assertTrue(n and isinstance(n, bytes))
self.assertEqual(len(set(names)), len(names))
