def docker_run(args):
"""Runs the experiment in the container.
"""
target = Path(args.target[0])
unpacked_info = read_dict(target)
cmdline = args.cmdline
# Sanity check
if args.detach and args.x11:
logging.critical("Error: Can't use X11 forwarding if you're detaching")
raise UsageError
# Loads config
config = load_config(target / 'config.yml', True)
runs = config.runs
selected_runs = get_runs(runs, args.run, cmdline)
# Get current image name
if 'current_image' in unpacked_info:
image = unpacked_info['current_image']
logging.debug("Running from image %s", image.decode('ascii'))
else:
logging.critical("Image doesn't exist yet, have you run setup/build?")
sys.exit(1)
# Name of new container
if args.detach:
container = make_unique_name(b'reprounzip_detached_')
else:
container = make_unique_name(b'reprounzip_run_')
hostname = runs[selected_runs[0]].get('hostname', 'reprounzip')
# Port forwarding
port_options = []
for port_host, port_container, proto in parse_ports(args.expose_port):
port_options.extend(['-p',
'%s:%s%s' % (port_host, port_container, proto)])
# X11 handler
if args.x11:
local_ip = get_local_addr()
docker_host = local_ip
if os.environ.get('DOCKER_HOST'):
m = _dockerhost_re.match(os.environ['DOCKER_HOST'])
if m is not None:
docker_host = m.group(1)
if args.tunneled_x11:
x11 = X11Handler(True, ('internet', docker_host), args.x11_display)
else:
x11 = X11Handler(True, ('internet', local_ip), args.x11_display)
if (docker_host != local_ip and docker_host != 'localhost' and
not docker_host.startswith('127.') and
not docker_host.startswith('192.168.99.')):
ssh_cmdline = ' '.join(
'-R*:%(p)d:127.0.0.1:%(p)d' % {'p': port}
for port, connector in x11.port_forward)
logging.warning(
"You requested X11 forwarding but the Docker container "
"appears to be running remotely. It is probable that it "
"won't be able to connect to the local display. Creating "
"a remote SSH tunnel and running with --tunneled-x11 "
"might help (%s).",
ssh_cmdline)
else:
x11 = X11Handler(False, ('local', hostname), args.x11_display)
cmd = []
for run_number in selected_runs:
run = runs[run_number]
env_set, env_unset = x11.env_fixes(run['environ'])
a_env_set, a_env_unset = parse_environment_args(args)
env_set.update(a_env_set)
env_unset.extend(a_env_unset)
if env_set or env_unset:
cmd.append('env')
env = []
for k in env_unset:
env.append('-u')
env.append(shell_escape(k))
for k, v in iteritems(env_set):
env.append('%s=%s' % (shell_escape(k), shell_escape(v)))
cmd.append(' '.join(env))
# FIXME : Use exec -a or something if binary != argv[0]
if cmdline is not None:
cmd.append('cmd')
cmd.append(' '.join(shell_escape(a) for a in cmdline))
cmd.append('run')
cmd.append('%d' % run_number)
cmd = list(chain.from_iterable([['do', shell_escape(c)]
for c in x11.init_cmds] +
[cmd]))
if logging.getLogger().isEnabledFor(logging.DEBUG):
logging.debug("Passing arguments to Docker image:")
for c in cmd:
logging.debug(c)
signals.pre_run(target=target)
# Creates forwarders
forwarders = []
for port, connector in x11.port_forward:
forwarders.append(LocalForwarder(connector, port))
if args.detach:
logging.info("Start container %s (detached)",
container.decode('ascii'))
retcode = interruptible_call(args.docker_cmd.split() +
['run', b'--name=' + container,
'-h', hostname,
'-d', '-t'] +
port_options +
args.docker_option +
[image] + cmd)
if retcode != 0:
logging.critical("docker run failed with code %d", retcode)
subprocess.call(['docker', 'rm', '-f', container])
sys.exit(1)
return
# Run command in container
logging.info("Starting container %s", container.decode('ascii'))
retcode = interruptible_call(args.docker_cmd.split() +
['run', b'--name=' + container,
'-h', hostname,
'-i', '-t'] +
port_options +
args.docker_option +
[image] + cmd)
if retcode != 0:
logging.critical("docker run failed with code %d", retcode)
subprocess.call(['docker', 'rm', '-f', container])
sys.exit(1)
# Get exit status from "docker inspect"
out = subprocess.check_output(args.docker_cmd.split() +
['inspect', container])
outjson = json.loads(out.decode('ascii'))
if (outjson[0]["State"]["Running"] is not False or
outjson[0]["State"]["Paused"] is not False):
logging.error("Invalid container state after execution:\n%s",
json.dumps(outjson[0]["State"]))
retcode = outjson[0]["State"]["ExitCode"]
stderr.write("\n*** Command finished, status: %d\n" % retcode)
# Commit to create new image
new_image = make_unique_name(b'reprounzip_image_')
logging.info("Committing container %s to image %s",
container.decode('ascii'), new_image.decode('ascii'))
subprocess.check_call(args.docker_cmd.split() +
['commit', container, new_image])
# Update image name
unpacked_info['current_image'] = new_image
write_dict(target, unpacked_info)
# Remove the container
logging.info("Destroying container %s", container.decode('ascii'))
retcode = subprocess.call(args.docker_cmd.split() + ['rm', container])
if retcode != 0:
logging.error("Error deleting container %s", container.decode('ascii'))
# Untag previous image, unless it is the initial_image
if image != unpacked_info['initial_image']:
logging.info("Untagging previous image %s", image.decode('ascii'))
subprocess.check_call(args.docker_cmd.split() + ['rmi', image])
# Update input file status
metadata_update_run(config, unpacked_info, selected_runs)
write_dict(target, unpacked_info)
signals.post_run(target=target, retcode=retcode)
def docker_run(args):
"""Runs the experiment in the container.
"""
target = Path(args.target[0])
unpacked_info = read_dict(target)
cmdline = args.cmdline
# Sanity check
if args.detach and args.x11:
logger.critical("Error: Can't use X11 forwarding if you're detaching")
raise UsageError
# Loads config
config = load_config(target / 'config.yml', True)
runs = config.runs
selected_runs = get_runs(runs, args.run, cmdline)
# Get current image name
if 'current_image' in unpacked_info:
image = unpacked_info['current_image']
logger.debug("Running from image %s", image.decode('ascii'))
else:
logger.critical("Image doesn't exist yet, have you run setup/build?")
sys.exit(1)
# Name of new container
if args.detach:
container = make_unique_name(b'reprounzip_detached_')
else:
container = make_unique_name(b'reprounzip_run_')
hostname = runs[selected_runs[0]].get('hostname', 'reprounzip')
# Port forwarding
port_options = []
for port_host, port_container, proto in parse_ports(args.expose_port):
port_options.extend(['-p',
'%s:%s/%s' % (port_host, port_container, proto)])
# X11 handler
if args.x11:
local_ip = get_local_addr()
docker_host = local_ip
if os.environ.get('DOCKER_HOST'):
m = _dockerhost_re.match(os.environ['DOCKER_HOST'])
if m is not None:
docker_host = m.group(1)
if args.tunneled_x11:
x11 = X11Handler(True, ('internet', docker_host), args.x11_display)
else:
x11 = X11Handler(True, ('internet', local_ip), args.x11_display)
if (docker_host != local_ip and docker_host != 'localhost' and
not docker_host.startswith('127.') and
not docker_host.startswith('192.168.99.')):
ssh_cmdline = ' '.join(
'-R*:%(p)d:127.0.0.1:%(p)d' % {'p': port}
for port, connector in x11.port_forward)
logger.warning(
"You requested X11 forwarding but the Docker container "
"appears to be running remotely. It is probable that it "
"won't be able to connect to the local display. Creating "
"a remote SSH tunnel and running with --tunneled-x11 "
"might help (%s).",
ssh_cmdline)
else:
x11 = X11Handler(False, ('local', hostname), args.x11_display)
cmd = []
for run_number in selected_runs:
run = runs[run_number]
env_set, env_unset = x11.env_fixes(run['environ'])
a_env_set, a_env_unset = parse_environment_args(args)
env_set.update(a_env_set)
env_unset.extend(a_env_unset)
if env_set or env_unset:
cmd.append('env')
env = []
for k in env_unset:
env.append('-u')
env.append(shell_escape(k))
for k, v in iteritems(env_set):
env.append('%s=%s' % (shell_escape(k), shell_escape(v)))
cmd.append(' '.join(env))
# FIXME : Use exec -a or something if binary != argv[0]
if cmdline is not None:
cmd.append('cmd')
cmd.append(' '.join(shell_escape(a) for a in cmdline))
cmd.append('run')
cmd.append('%d' % run_number)
cmd = list(chain.from_iterable([['do', shell_escape(c)]
for c in x11.init_cmds] +
[cmd]))
if logger.isEnabledFor(logging.DEBUG):
logger.debug("Passing arguments to Docker image:")
for c in cmd:
logger.debug(c)
signals.pre_run(target=target)
# Creates forwarders
forwarders = []
for port, connector in x11.port_forward:
forwarders.append(LocalForwarder(connector, port))
if args.detach:
logger.info("Start container %s (detached)",
container.decode('ascii'))
retcode = interruptible_call(args.docker_cmd.split() +
['run', b'--name=' + container,
'-h', hostname,
'-d', '-t'] +
port_options +
args.docker_option +
[image] + cmd)
if retcode != 0:
logger.critical("docker run failed with code %d", retcode)
subprocess.call(['docker', 'rm', '-f', container])
sys.exit(1)
return
# Run command in container
logger.info("Starting container %s", container.decode('ascii'))
retcode = interruptible_call(args.docker_cmd.split() +
['run', b'--name=' + container,
'-h', hostname,
'-i', '-t'] +
port_options +
args.docker_option +
[image] + cmd,
request_tty=True)
# The image prints out the exit status(es) itself
if retcode != 0:
logger.critical("docker run failed with code %d", retcode)
subprocess.call(['docker', 'rm', '-f', container])
sys.exit(1)
# Commit to create new image
new_image = make_unique_name(b'reprounzip_image_')
logger.info("Committing container %s to image %s",
container.decode('ascii'), new_image.decode('ascii'))
subprocess.check_call(args.docker_cmd.split() +
['commit', container, new_image])
# Update image name
unpacked_info['current_image'] = new_image
write_dict(target, unpacked_info)
# Remove the container
logger.info("Destroying container %s", container.decode('ascii'))
retcode = subprocess.call(args.docker_cmd.split() + ['rm', container])
if retcode != 0:
logger.error("Error deleting container %s", container.decode('ascii'))
# Untag previous image, unless it is the initial_image
if image != unpacked_info['initial_image']:
logger.info("Untagging previous image %s", image.decode('ascii'))
subprocess.check_call(args.docker_cmd.split() + ['rmi', image])
# Update input file status
metadata_update_run(config, unpacked_info, selected_runs)
write_dict(target, unpacked_info)
signals.post_run(target=target, retcode=retcode)
def vagrant_run(args):
"""Runs the experiment in the virtual machine.
"""
target = Path(args.target[0])
unpacked_info = read_dict(target)
use_chroot = unpacked_info['use_chroot']
cmdline = args.cmdline
check_vagrant_version()
# Loads config
config = load_config(target / 'config.yml', True)
runs = config.runs
selected_runs = get_runs(runs, args.run, cmdline)
hostname = runs[selected_runs[0]].get('hostname', 'reprounzip')
# Port forwarding
ports = parse_ports(args.expose_port)
# If the requested ports are not a subset of the ones already set on the
# VM, we have to update the Vagrantfile and issue `vagrant reload`, which
# will reboot the machine
req_ports = set(ports)
set_ports = set(unpacked_info.get('ports', []))
if not req_ports.issubset(set_ports):
# Build new set of forwarded ports: the ones already set + the one just
# requested
# The ones we request now override the previous config
all_ports = dict(
(host, (guest, proto)) for host, guest, proto in set_ports)
for host, guest, proto in req_ports:
all_ports[host] = guest, proto
unpacked_info['ports'] = sorted(
(host, guest, proto)
for host, (guest, proto) in iteritems(all_ports))
write_vagrantfile(target, unpacked_info)
logger.info("Some requested ports are not yet forwarded, running "
"'vagrant reload'")
retcode = subprocess.call(['vagrant', 'reload', '--no-provision'],
cwd=target.path)
if retcode != 0:
logger.critical("vagrant reload failed with code %d, aborting",
retcode)
sys.exit(1)
write_dict(target, unpacked_info)
# X11 handler
if unpacked_info['gui']:
x11 = LocalX11Handler()
else:
x11 = X11Handler(args.x11, ('local', hostname), args.x11_display)
cmds = []
for run_number in selected_runs:
run = runs[run_number]
cmd = 'cd %s && ' % shell_escape(run['workingdir'])
if use_chroot:
cmd += '/busybox env -i '
else:
cmd += '/usr/bin/env -i '
environ = x11.fix_env(run['environ'])
environ = fixup_environment(environ, args)
cmd += ' '.join('%s=%s' % (shell_escape(k), shell_escape(v))
for k, v in iteritems(environ))
cmd += ' '
# FIXME : Use exec -a or something if binary != argv[0]
if cmdline is None:
argv = [run['binary']] + run['argv'][1:]
else:
argv = cmdline
cmd += ' '.join(shell_escape(a) for a in argv)
uid = run.get('uid', 1000)
gid = run.get('gid', 1000)
if use_chroot:
userspec = '%s:%s' % (uid, gid)
cmd = ('chroot --userspec=%s /experimentroot '
'/bin/sh -c %s' % (userspec, shell_escape(cmd)))
else:
cmd = 'sudo -u \'#%d\' sh -c %s' % (uid, shell_escape(cmd))
cmds.append(cmd)
if use_chroot:
cmds = [
'chroot /experimentroot /bin/sh -c %s' % shell_escape(c)
for c in x11.init_cmds
] + cmds
else:
cmds = x11.init_cmds + cmds
cmds = ' && '.join(cmds)
# Sets the hostname to the original experiment's machine's
# FIXME: not reentrant: this restores the Vagrant machine's hostname after
# the run, which might cause issues if several "reprounzip vagrant run" are
# running at once
cmds = ('OLD_HOSTNAME=$(/bin/hostname); /bin/hostname %s; ' % hostname +
cmds + '; RES=$?; /bin/hostname "$OLD_HOSTNAME"; exit $RES')
cmds = '/usr/bin/sudo /bin/sh -c %s' % shell_escape(cmds)
# Gets vagrant SSH parameters
info = machine_setup(target)
signals.pre_run(target=target)
interactive = not (args.no_stdin
or os.environ.get('REPROUNZIP_NON_INTERACTIVE'))
retcode = run_interactive(info, interactive, cmds, not args.no_pty,
x11.port_forward)
stderr.write("\r\n*** Command finished, status: %d\r\n" % retcode)
# Update input file status
metadata_update_run(config, unpacked_info, selected_runs)
write_dict(target, unpacked_info)
signals.post_run(target=target, retcode=retcode)
def vagrant_setup_create(args):
"""Sets up the experiment to be run in a Vagrant-built virtual machine.
This can either build a chroot or not.
If building a chroot, we do just like without Vagrant: we copy all the
files and only get what's missing from the host. But we do install
automatically the packages whose files are required.
If not building a chroot, we install all the packages, and only unpack
files that don't come from packages.
In short: files from packages with packfiles=True will only be used if
building a chroot.
"""
if not args.pack:
logger.critical("setup/create needs the pack filename")
sys.exit(1)
pack = Path(args.pack[0])
target = Path(args.target[0])
if target.exists():
logger.critical("Target directory exists")
sys.exit(1)
use_chroot = args.use_chroot
mount_bind = args.bind_magic_dirs
record_usage(use_chroot=use_chroot, mount_bind=mount_bind)
signals.pre_setup(target=target, pack=pack)
# Unpacks configuration file
rpz_pack = RPZPack(pack)
rpz_pack.extract_config(target / 'config.yml')
# Loads config
runs, packages, other_files = config = load_config(target / 'config.yml',
True)
if not args.memory:
memory = None
else:
try:
memory = int(args.memory[-1])
except ValueError:
logger.critical("Invalid value for memory size: %r", args.memory)
sys.exit(1)
ports = parse_ports(args.expose_port)
if args.base_image and args.base_image[0]:
record_usage(vagrant_explicit_image=True)
box = args.base_image[0]
if args.distribution:
target_distribution = args.distribution[0]
else:
target_distribution = None
else:
target_distribution, box = select_box(runs, gui=args.gui)
logger.info("Using box %s", box)
logger.debug("Distribution: %s", target_distribution or "unknown")
# If using chroot, we might still need to install packages to get missing
# (not packed) files
if use_chroot:
packages = [pkg for pkg in packages if not pkg.packfiles]
if packages:
record_usage(vagrant_install_pkgs=True)
logger.info(
"Some packages were not packed, so we'll install and "
"copy their files\n"
"Packages that are missing:\n%s",
' '.join(pkg.name for pkg in packages))
if packages:
try:
installer = select_installer(pack, runs, target_distribution)
except CantFindInstaller as e:
logger.error(
"Need to install %d packages but couldn't select a "
"package installer: %s", len(packages), e)
target.mkdir(parents=True)
try:
# Writes setup script
logger.info("Writing setup script %s...", target / 'setup.sh')
with (target / 'setup.sh').open('w', encoding='utf-8',
newline='\n') as fp:
fp.write('#!/bin/sh\n\nset -e\n\n')
if packages:
# Updates package sources
update_script = installer.update_script()
if update_script:
fp.write(update_script)
fp.write('\n')
# Installs necessary packages
fp.write(installer.install_script(packages))
fp.write('\n')
# TODO : Compare package versions (painful because of sh)
# Untar
if use_chroot:
fp.write('\n' 'mkdir /experimentroot; cd /experimentroot\n')
fp.write('tar zpxf /vagrant/data.tgz --numeric-owner '
'--strip=1 %s\n' % rpz_pack.data_prefix)
if mount_bind:
fp.write('\n'
'mkdir -p /experimentroot/dev\n'
'mkdir -p /experimentroot/proc\n')
for pkg in packages:
fp.write('\n# Copies files from package %s\n' % pkg.name)
for f in pkg.files:
f = f.path
dest = join_root(PosixPath('/experimentroot'), f)
fp.write('mkdir -p %s\n' %
shell_escape(unicode_(f.parent)))
fp.write('cp -L %s %s\n' % (shell_escape(
unicode_(f)), shell_escape(unicode_(dest))))
fp.write(
'\n'
'cp /etc/resolv.conf /experimentroot/etc/resolv.conf\n')
else:
fp.write('\ncd /\n')
paths = set()
pathlist = []
# Adds intermediate directories, and checks for existence in
# the tar
logger.info("Generating file list...")
data_files = rpz_pack.data_filenames()
for f in other_files:
if f.path.name == 'resolv.conf' and (
f.path.lies_under('/etc')
or f.path.lies_under('/run')
or f.path.lies_under('/var')):
continue
path = PosixPath('/')
for c in rpz_pack.remove_data_prefix(f.path).components:
path = path / c
if path in paths:
continue
paths.add(path)
if path in data_files:
pathlist.append(path)
else:
logger.info("Missing file %s", path)
# FIXME : for some reason we need reversed() here, I'm not sure
# why. Need to read more of tar's docs.
# TAR bug: --no-overwrite-dir removes --keep-old-files
# TAR bug: there is no way to make --keep-old-files not report
# an error if an existing file is encountered. --skip-old-files
# was introduced too recently. Instead, we just ignore the exit
# status
with (target / 'rpz-files.list').open('wb') as lfp:
for p in reversed(pathlist):
lfp.write(join_root(rpz_pack.data_prefix, p).path)
lfp.write(b'\0')
fp.write('tar zpxf /vagrant/data.tgz --keep-old-files '
'--numeric-owner --strip=1 '
'--null -T /vagrant/rpz-files.list || /bin/true\n')
# Copies busybox
if use_chroot:
arch = runs[0]['architecture']
download_file(busybox_url(arch), target / 'busybox',
'busybox-%s' % arch)
fp.write(r'''
cp /vagrant/busybox /experimentroot/busybox
chmod +x /experimentroot/busybox
mkdir -p /experimentroot/bin
[ -e /experimentroot/bin/sh ] || \
ln -s /busybox /experimentroot/bin/sh
''')
# Copies pack
logger.info("Copying pack file...")
rpz_pack.copy_data_tar(target / 'data.tgz')
rpz_pack.close()
# Meta-data for reprounzip
metadata = metadata_initial_iofiles(
config, {
'use_chroot': use_chroot,
'gui': args.gui,
'ports': ports,
'box': box,
'memory': memory
})
write_dict(target, metadata)
# Writes Vagrant file
write_vagrantfile(target, metadata)
signals.post_setup(target=target, pack=pack)
except Exception:
target.rmtree(ignore_errors=True)
raise
def vagrant_run(args):
"""Runs the experiment in the virtual machine.
"""
target = Path(args.target[0])
unpacked_info = read_dict(target)
use_chroot = unpacked_info['use_chroot']
cmdline = args.cmdline
check_vagrant_version()
# Loads config
config = load_config(target / 'config.yml', True)
runs = config.runs
selected_runs = get_runs(runs, args.run, cmdline)
hostname = runs[selected_runs[0]].get('hostname', 'reprounzip')
# Port forwarding
ports = parse_ports(args.expose_port)
# If the requested ports are not a subset of the ones already set on the
# VM, we have to update the Vagrantfile and issue `vagrant reload`, which
# will reboot the machine
req_ports = set(ports)
set_ports = set(unpacked_info.get('ports', []))
if not req_ports.issubset(set_ports):
# Build new set of forwarded ports: the ones already set + the one just
# requested
# The ones we request now override the previous config
all_ports = dict((host, (guest, proto))
for host, guest, proto in set_ports)
for host, guest, proto in req_ports:
all_ports[host] = guest, proto
unpacked_info['ports'] = sorted(
(host, guest, proto)
for host, (guest, proto) in iteritems(all_ports))
write_vagrantfile(target, unpacked_info)
logger.info("Some requested ports are not yet forwarded, running "
"'vagrant reload'")
retcode = subprocess.call(['vagrant', 'reload', '--no-provision'],
cwd=target.path)
if retcode != 0:
logger.critical("vagrant reload failed with code %d, aborting",
retcode)
sys.exit(1)
write_dict(target, unpacked_info)
# X11 handler
if unpacked_info['gui']:
x11 = LocalX11Handler()
else:
x11 = X11Handler(args.x11, ('local', hostname), args.x11_display)
cmds = []
for run_number in selected_runs:
run = runs[run_number]
cmd = 'cd %s && ' % shell_escape(run['workingdir'])
if use_chroot:
cmd += '/busybox env -i '
else:
cmd += '/usr/bin/env -i '
environ = x11.fix_env(run['environ'])
environ = fixup_environment(environ, args)
cmd += ' '.join('%s=%s' % (shell_escape(k), shell_escape(v))
for k, v in iteritems(environ))
cmd += ' '
# FIXME : Use exec -a or something if binary != argv[0]
if cmdline is None:
argv = [run['binary']] + run['argv'][1:]
else:
argv = cmdline
cmd += ' '.join(shell_escape(a) for a in argv)
uid = run.get('uid', 1000)
gid = run.get('gid', 1000)
if use_chroot:
userspec = '%s:%s' % (uid, gid)
cmd = ('chroot --userspec=%s /experimentroot '
'/bin/sh -c %s' % (
userspec,
shell_escape(cmd)))
else:
cmd = 'sudo -u \'#%d\' sh -c %s' % (uid, shell_escape(cmd))
cmds.append(cmd)
if use_chroot:
cmds = ['chroot /experimentroot /bin/sh -c %s' % shell_escape(c)
for c in x11.init_cmds] + cmds
else:
cmds = x11.init_cmds + cmds
cmds = ' && '.join(cmds)
# Sets the hostname to the original experiment's machine's
# FIXME: not reentrant: this restores the Vagrant machine's hostname after
# the run, which might cause issues if several "reprounzip vagrant run" are
# running at once
cmds = ('OLD_HOSTNAME=$(/bin/hostname); /bin/hostname %s; ' % hostname +
cmds +
'; RES=$?; /bin/hostname "$OLD_HOSTNAME"; exit $RES')
cmds = '/usr/bin/sudo /bin/sh -c %s' % shell_escape(cmds)
# Gets vagrant SSH parameters
info = machine_setup(target)
signals.pre_run(target=target)
interactive = not (args.no_stdin or
os.environ.get('REPROUNZIP_NON_INTERACTIVE'))
retcode = run_interactive(info, interactive,
cmds,
not args.no_pty,
x11.port_forward)
stderr.write("\r\n*** Command finished, status: %d\r\n" % retcode)
# Update input file status
metadata_update_run(config, unpacked_info, selected_runs)
write_dict(target, unpacked_info)
signals.post_run(target=target, retcode=retcode)
def vagrant_setup_create(args):
"""Sets up the experiment to be run in a Vagrant-built virtual machine.
This can either build a chroot or not.
If building a chroot, we do just like without Vagrant: we copy all the
files and only get what's missing from the host. But we do install
automatically the packages whose files are required.
If not building a chroot, we install all the packages, and only unpack
files that don't come from packages.
In short: files from packages with packfiles=True will only be used if
building a chroot.
"""
if not args.pack:
logger.critical("setup/create needs the pack filename")
sys.exit(1)
pack = Path(args.pack[0])
target = Path(args.target[0])
if target.exists():
logger.critical("Target directory exists")
sys.exit(1)
use_chroot = args.use_chroot
mount_bind = args.bind_magic_dirs
record_usage(use_chroot=use_chroot,
mount_bind=mount_bind)
signals.pre_setup(target=target, pack=pack)
# Unpacks configuration file
rpz_pack = RPZPack(pack)
rpz_pack.extract_config(target / 'config.yml')
# Loads config
runs, packages, other_files = config = load_config(target / 'config.yml',
True)
if not args.memory:
memory = None
else:
try:
memory = int(args.memory[-1])
except ValueError:
logger.critical("Invalid value for memory size: %r", args.memory)
sys.exit(1)
ports = parse_ports(args.expose_port)
if args.base_image and args.base_image[0]:
record_usage(vagrant_explicit_image=True)
box = args.base_image[0]
if args.distribution:
target_distribution = args.distribution[0]
else:
target_distribution = None
else:
target_distribution, box = select_box(runs, gui=args.gui)
logger.info("Using box %s", box)
logger.debug("Distribution: %s", target_distribution or "unknown")
# If using chroot, we might still need to install packages to get missing
# (not packed) files
if use_chroot:
packages = [pkg for pkg in packages if not pkg.packfiles]
if packages:
record_usage(vagrant_install_pkgs=True)
logger.info("Some packages were not packed, so we'll install and "
"copy their files\n"
"Packages that are missing:\n%s",
' '.join(pkg.name for pkg in packages))
if packages:
try:
installer = select_installer(pack, runs, target_distribution)
except CantFindInstaller as e:
logger.error("Need to install %d packages but couldn't select a "
"package installer: %s",
len(packages), e)
target.mkdir(parents=True)
try:
# Writes setup script
logger.info("Writing setup script %s...", target / 'setup.sh')
with (target / 'setup.sh').open('w', encoding='utf-8',
newline='\n') as fp:
fp.write('#!/bin/sh\n\nset -e\n\n')
if packages:
# Updates package sources
update_script = installer.update_script()
if update_script:
fp.write(update_script)
fp.write('\n')
# Installs necessary packages
fp.write(installer.install_script(packages))
fp.write('\n')
# TODO : Compare package versions (painful because of sh)
# Untar
if use_chroot:
fp.write('\n'
'mkdir /experimentroot; cd /experimentroot\n')
fp.write('tar zpxf /vagrant/data.tgz --numeric-owner '
'--strip=1 %s\n' % rpz_pack.data_prefix)
if mount_bind:
fp.write('\n'
'mkdir -p /experimentroot/dev\n'
'mkdir -p /experimentroot/proc\n')
for pkg in packages:
fp.write('\n# Copies files from package %s\n' % pkg.name)
for f in pkg.files:
f = f.path
dest = join_root(PosixPath('/experimentroot'), f)
fp.write('mkdir -p %s\n' %
shell_escape(unicode_(f.parent)))
fp.write('cp -L %s %s\n' % (
shell_escape(unicode_(f)),
shell_escape(unicode_(dest))))
fp.write(
'\n'
'cp /etc/resolv.conf /experimentroot/etc/resolv.conf\n')
else:
fp.write('\ncd /\n')
paths = set()
pathlist = []
# Adds intermediate directories, and checks for existence in
# the tar
logger.info("Generating file list...")
data_files = rpz_pack.data_filenames()
for f in other_files:
if f.path.name == 'resolv.conf' and (
f.path.lies_under('/etc') or
f.path.lies_under('/run') or
f.path.lies_under('/var')):
continue
path = PosixPath('/')
for c in rpz_pack.remove_data_prefix(f.path).components:
path = path / c
if path in paths:
continue
paths.add(path)
if path in data_files:
pathlist.append(path)
else:
logger.info("Missing file %s", path)
# FIXME : for some reason we need reversed() here, I'm not sure
# why. Need to read more of tar's docs.
# TAR bug: --no-overwrite-dir removes --keep-old-files
# TAR bug: there is no way to make --keep-old-files not report
# an error if an existing file is encountered. --skip-old-files
# was introduced too recently. Instead, we just ignore the exit
# status
with (target / 'rpz-files.list').open('wb') as lfp:
for p in reversed(pathlist):
lfp.write(join_root(rpz_pack.data_prefix, p).path)
lfp.write(b'\0')
fp.write('tar zpxf /vagrant/data.tgz --keep-old-files '
'--numeric-owner --strip=1 '
'--null -T /vagrant/rpz-files.list || /bin/true\n')
# Copies busybox
if use_chroot:
arch = runs[0]['architecture']
download_file(busybox_url(arch),
target / 'busybox',
'busybox-%s' % arch)
fp.write(r'''
cp /vagrant/busybox /experimentroot/busybox
chmod +x /experimentroot/busybox
mkdir -p /experimentroot/bin
[ -e /experimentroot/bin/sh ] || \
ln -s /busybox /experimentroot/bin/sh
''')
# Copies pack
logger.info("Copying pack file...")
rpz_pack.copy_data_tar(target / 'data.tgz')
rpz_pack.close()
# Meta-data for reprounzip
metadata = metadata_initial_iofiles(config,
{'use_chroot': use_chroot,
'gui': args.gui,
'ports': ports,
'box': box,
'memory': memory})
write_dict(target, metadata)
# Writes Vagrant file
write_vagrantfile(target, metadata)
signals.post_setup(target=target, pack=pack)
except Exception:
target.rmtree(ignore_errors=True)
raise