def chroot_run(args):
"""Runs the command in the chroot.
"""
target = Path(args.target[0])
unpacked_info = metadata_read(target, 'chroot')
cmdline = args.cmdline
# Loads config
config = load_config_file(target / 'config.yml', True)
runs = config.runs
selected_runs = get_runs(runs, args.run, cmdline)
root = target / 'root'
# X11 handler
x11 = X11Handler(args.x11, ('local', socket.gethostname()),
args.x11_display)
cmds = []
for run_number in selected_runs:
run = runs[run_number]
cmd = 'cd %s && ' % shell_escape(run['workingdir'])
cmd += '/usr/bin/env -i '
environ = x11.fix_env(run['environ'])
environ = fixup_environment(environ, args)
cmd += ' '.join('%s=%s' % (shell_escape(k), shell_escape(v))
for k, v in iteritems(environ))
cmd += ' '
# FIXME : Use exec -a or something if binary != argv[0]
if cmdline is None:
argv = [run['binary']] + run['argv'][1:]
else:
argv = cmdline
cmd += ' '.join(shell_escape(a) for a in argv)
userspec = '%s:%s' % (run.get('uid', 1000), run.get('gid', 1000))
cmd = 'chroot --userspec=%s %s /bin/sh -c %s' % (
userspec, shell_escape(unicode_(root)), shell_escape(cmd))
cmds.append(cmd)
cmds = [
'chroot %s /bin/sh -c %s' %
(shell_escape(unicode_(root)), shell_escape(c)) for c in x11.init_cmds
] + cmds
cmds = ' && '.join(cmds)
# Starts forwarding
forwarders = []
for portnum, connector in x11.port_forward:
fwd = LocalForwarder(connector, portnum)
forwarders.append(fwd)
signals.pre_run(target=target)
retcode = interruptible_call(cmds, shell=True)
stderr.write("\n*** Command finished, status: %d\n" % retcode)
signals.post_run(target=target, retcode=retcode)
# Update input file status
metadata_update_run(config, unpacked_info, selected_runs)
metadata_write(target, unpacked_info, 'chroot')
def directory_run(args):
"""Runs the command in the directory.
"""
target = Path(args.target[0])
unpacked_info = metadata_read(target, 'directory')
cmdline = args.cmdline
# Loads config
config = load_config_file(target / 'config.yml', True)
runs = config.runs
selected_runs = get_runs(runs, args.run, cmdline)
root = (target / 'root').absolute()
# Gets library paths
lib_dirs = []
logger.debug("Running: %s", "/sbin/ldconfig -v -N")
p = subprocess.Popen(['/sbin/ldconfig', '-v', '-N'],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, _ = p.communicate()
try:
for line in stdout.splitlines():
if len(line) < 2 or line[0] in (b' ', b'\t'):
continue
if line.endswith(b':'):
lib_dirs.append(Path(line[:-1]))
finally:
if p.returncode != 0:
raise subprocess.CalledProcessError(p.returncode,
['/sbin/ldconfig', '-v', '-N'])
lib_dirs = ('export LD_LIBRARY_PATH=%s' % ':'.join(
shell_escape(str(join_root(root, d)))
for d in lib_dirs))
cmds = [lib_dirs]
for run_number in selected_runs:
run = runs[run_number]
cmd = 'cd %s && ' % shell_escape(
str(join_root(root, Path(run['workingdir']))))
cmd += '/usr/bin/env -i '
environ = run['environ']
environ = fixup_environment(environ, args)
if args.x11:
if 'DISPLAY' in os.environ:
environ['DISPLAY'] = os.environ['DISPLAY']
if 'XAUTHORITY' in os.environ:
environ['XAUTHORITY'] = os.environ['XAUTHORITY']
cmd += ' '.join('%s=%s' % (shell_escape(k), shell_escape(v))
for k, v in environ.items()
if k != 'PATH')
cmd += ' '
# PATH
# Get the original PATH components
path = [PosixPath(d)
for d in run['environ'].get('PATH', '').split(':')]
# The same paths but in the directory
dir_path = [join_root(root, d)
for d in path
if d.root == '/']
# Rebuild string
path = ':'.join(str(d) for d in dir_path + path)
cmd += 'PATH=%s ' % shell_escape(path)
# FIXME : Use exec -a or something if binary != argv[0]
if cmdline is None:
argv = run['argv']
# Rewrites command-line arguments that are absolute filenames
rewritten = False
for i in range(len(argv)):
try:
p = Path(argv[i])
except UnicodeEncodeError:
continue
if p.is_absolute:
rp = join_root(root, p)
if (rp.exists() or
(len(rp.components) > 3 and rp.parent.exists())):
argv[i] = str(rp)
rewritten = True
if rewritten:
logger.warning("Rewrote command-line as: %s",
' '.join(shell_escape(a) for a in argv))
else:
argv = cmdline
cmd += ' '.join(shell_escape(a) for a in argv)
cmds.append(cmd)
cmds = ' && '.join(cmds)
signals.pre_run(target=target)
logger.debug("Running: %s", cmds)
retcode = interruptible_call(cmds, shell=True)
print("\n*** Command finished, status: %d\n" % retcode, file=sys.stderr)
signals.post_run(target=target, retcode=retcode)
# Update input file status
metadata_update_run(config, unpacked_info, selected_runs)
metadata_write(target, unpacked_info, 'directory')
def docker_run(args):
"""Runs the experiment in the container.
"""
target = Path(args.target[0])
unpacked_info = read_dict(target)
cmdline = args.cmdline
# Sanity check
if args.detach and args.x11:
logging.critical("Error: Can't use X11 forwarding if you're detaching")
raise UsageError
# Loads config
config = load_config(target / 'config.yml', True)
runs = config.runs
selected_runs = get_runs(runs, args.run, cmdline)
# Get current image name
if 'current_image' in unpacked_info:
image = unpacked_info['current_image']
logging.debug("Running from image %s", image.decode('ascii'))
else:
logging.critical("Image doesn't exist yet, have you run setup/build?")
sys.exit(1)
# Name of new container
if args.detach:
container = make_unique_name(b'reprounzip_detached_')
else:
container = make_unique_name(b'reprounzip_run_')
hostname = runs[selected_runs[0]].get('hostname', 'reprounzip')
# X11 handler
if args.x11:
local_ip = get_local_addr()
docker_host = local_ip
if os.environ.get('DOCKER_HOST'):
m = _dockerhost_re.match(os.environ['DOCKER_HOST'])
if m is not None:
docker_host = m.group(1)
if args.tunneled_x11:
x11 = X11Handler(True, ('internet', docker_host), args.x11_display)
else:
x11 = X11Handler(True, ('internet', local_ip), args.x11_display)
if (docker_host != local_ip and docker_host != 'localhost'
and not docker_host.startswith('127.')
and not docker_host.startswith('192.168.99.')):
ssh_cmdline = ' '.join('-R*:%(p)d:127.0.0.1:%(p)d' %
{'p': port}
for port, connector in x11.port_forward)
logging.warning(
"You requested X11 forwarding but the Docker container "
"appears to be running remotely. It is probable that it "
"won't be able to connect to the local display. Creating "
"a remote SSH tunnel and running with --tunneled-x11 "
"might help (%s).", ssh_cmdline)
else:
x11 = X11Handler(False, ('local', hostname), args.x11_display)
cmds = []
for run_number in selected_runs:
run = runs[run_number]
cmd = 'cd %s && ' % shell_escape(run['workingdir'])
cmd += '/busybox env -i '
environ = x11.fix_env(run['environ'])
environ = fixup_environment(environ, args)
cmd += ' '.join('%s=%s' % (shell_escape(k), shell_escape(v))
for k, v in iteritems(environ))
cmd += ' '
# FIXME : Use exec -a or something if binary != argv[0]
if cmdline is None:
argv = [run['binary']] + run['argv'][1:]
else:
argv = cmdline
cmd += ' '.join(shell_escape(a) for a in argv)
uid = run.get('uid', 1000)
gid = run.get('gid', 1000)
cmd = '/rpzsudo \'#%d\' \'#%d\' /busybox sh -c %s' % (
uid, gid, shell_escape(cmd))
cmds.append(cmd)
cmds = x11.init_cmds + cmds
cmds = ' && '.join(cmds)
signals.pre_run(target=target)
# Creates forwarders
forwarders = []
for port, connector in x11.port_forward:
forwarders.append(LocalForwarder(connector, port))
if args.detach:
logging.info("Start container %s (detached)",
container.decode('ascii'))
retcode = interruptible_call([
'docker', 'run', b'--name=' + container, '-h', hostname, '-d', '-t'
] + args.docker_option + [image, '/busybox', 'sh', '-c', cmds])
if retcode != 0:
logging.critical("docker run failed with code %d", retcode)
subprocess.call(['docker', 'rm', '-f', container])
sys.exit(1)
return
# Run command in container
logging.info("Starting container %s", container.decode('ascii'))
retcode = interruptible_call(
['docker', 'run', b'--name=' + container, '-h', hostname, '-i', '-t'] +
args.docker_option + [image, '/busybox', 'sh', '-c', cmds])
if retcode != 0:
logging.critical("docker run failed with code %d", retcode)
subprocess.call(['docker', 'rm', '-f', container])
sys.exit(1)
# Get exit status from "docker inspect"
out = subprocess.check_output(['docker', 'inspect', container])
outjson = json.loads(out.decode('ascii'))
if (outjson[0]["State"]["Running"] is not False
or outjson[0]["State"]["Paused"] is not False):
logging.error("Invalid container state after execution:\n%s",
json.dumps(outjson[0]["State"]))
retcode = outjson[0]["State"]["ExitCode"]
stderr.write("\n*** Command finished, status: %d\n" % retcode)
# Commit to create new image
new_image = make_unique_name(b'reprounzip_image_')
logging.info("Committing container %s to image %s",
container.decode('ascii'), new_image.decode('ascii'))
subprocess.check_call(['docker', 'commit', container, new_image])
# Update image name
unpacked_info['current_image'] = new_image
write_dict(target, unpacked_info)
# Remove the container
logging.info("Destroying container %s", container.decode('ascii'))
retcode = subprocess.call(['docker', 'rm', container])
if retcode != 0:
logging.error("Error deleting container %s", container.decode('ascii'))
# Untag previous image, unless it is the initial_image
if image != unpacked_info['initial_image']:
logging.info("Untagging previous image %s", image.decode('ascii'))
subprocess.check_call(['docker', 'rmi', image])
# Update input file status
metadata_update_run(config, unpacked_info, selected_runs)
write_dict(target, unpacked_info)
signals.post_run(target=target, retcode=retcode)
def docker_run(args):
"""Runs the experiment in the container.
"""
target = Path(args.target[0])
unpacked_info = read_dict(target)
cmdline = args.cmdline
# Sanity check
if args.detach and args.x11:
logging.critical("Error: Can't use X11 forwarding if you're detaching")
raise UsageError
# Loads config
config = load_config(target / 'config.yml', True)
runs = config.runs
selected_runs = get_runs(runs, args.run, cmdline)
# Get current image name
if 'current_image' in unpacked_info:
image = unpacked_info['current_image']
logging.debug("Running from image %s", image.decode('ascii'))
else:
logging.critical("Image doesn't exist yet, have you run setup/build?")
sys.exit(1)
# Name of new container
if args.detach:
container = make_unique_name(b'reprounzip_detached_')
else:
container = make_unique_name(b'reprounzip_run_')
hostname = runs[selected_runs[0]].get('hostname', 'reprounzip')
# Port forwarding
port_options = []
for port_host, port_container, proto in parse_ports(args.expose_port):
port_options.extend(['-p',
'%s:%s%s' % (port_host, port_container, proto)])
# X11 handler
if args.x11:
local_ip = get_local_addr()
docker_host = local_ip
if os.environ.get('DOCKER_HOST'):
m = _dockerhost_re.match(os.environ['DOCKER_HOST'])
if m is not None:
docker_host = m.group(1)
if args.tunneled_x11:
x11 = X11Handler(True, ('internet', docker_host), args.x11_display)
else:
x11 = X11Handler(True, ('internet', local_ip), args.x11_display)
if (docker_host != local_ip and docker_host != 'localhost' and
not docker_host.startswith('127.') and
not docker_host.startswith('192.168.99.')):
ssh_cmdline = ' '.join(
'-R*:%(p)d:127.0.0.1:%(p)d' % {'p': port}
for port, connector in x11.port_forward)
logging.warning(
"You requested X11 forwarding but the Docker container "
"appears to be running remotely. It is probable that it "
"won't be able to connect to the local display. Creating "
"a remote SSH tunnel and running with --tunneled-x11 "
"might help (%s).",
ssh_cmdline)
else:
x11 = X11Handler(False, ('local', hostname), args.x11_display)
cmd = []
for run_number in selected_runs:
run = runs[run_number]
env_set, env_unset = x11.env_fixes(run['environ'])
a_env_set, a_env_unset = parse_environment_args(args)
env_set.update(a_env_set)
env_unset.extend(a_env_unset)
if env_set or env_unset:
cmd.append('env')
env = []
for k in env_unset:
env.append('-u')
env.append(shell_escape(k))
for k, v in iteritems(env_set):
env.append('%s=%s' % (shell_escape(k), shell_escape(v)))
cmd.append(' '.join(env))
# FIXME : Use exec -a or something if binary != argv[0]
if cmdline is not None:
cmd.append('cmd')
cmd.append(' '.join(shell_escape(a) for a in cmdline))
cmd.append('run')
cmd.append('%d' % run_number)
cmd = list(chain.from_iterable([['do', shell_escape(c)]
for c in x11.init_cmds] +
[cmd]))
if logging.getLogger().isEnabledFor(logging.DEBUG):
logging.debug("Passing arguments to Docker image:")
for c in cmd:
logging.debug(c)
signals.pre_run(target=target)
# Creates forwarders
forwarders = []
for port, connector in x11.port_forward:
forwarders.append(LocalForwarder(connector, port))
if args.detach:
logging.info("Start container %s (detached)",
container.decode('ascii'))
retcode = interruptible_call(args.docker_cmd.split() +
['run', b'--name=' + container,
'-h', hostname,
'-d', '-t'] +
port_options +
args.docker_option +
[image] + cmd)
if retcode != 0:
logging.critical("docker run failed with code %d", retcode)
subprocess.call(['docker', 'rm', '-f', container])
sys.exit(1)
return
# Run command in container
logging.info("Starting container %s", container.decode('ascii'))
retcode = interruptible_call(args.docker_cmd.split() +
['run', b'--name=' + container,
'-h', hostname,
'-i', '-t'] +
port_options +
args.docker_option +
[image] + cmd)
if retcode != 0:
logging.critical("docker run failed with code %d", retcode)
subprocess.call(['docker', 'rm', '-f', container])
sys.exit(1)
# Get exit status from "docker inspect"
out = subprocess.check_output(args.docker_cmd.split() +
['inspect', container])
outjson = json.loads(out.decode('ascii'))
if (outjson[0]["State"]["Running"] is not False or
outjson[0]["State"]["Paused"] is not False):
logging.error("Invalid container state after execution:\n%s",
json.dumps(outjson[0]["State"]))
retcode = outjson[0]["State"]["ExitCode"]
stderr.write("\n*** Command finished, status: %d\n" % retcode)
# Commit to create new image
new_image = make_unique_name(b'reprounzip_image_')
logging.info("Committing container %s to image %s",
container.decode('ascii'), new_image.decode('ascii'))
subprocess.check_call(args.docker_cmd.split() +
['commit', container, new_image])
# Update image name
unpacked_info['current_image'] = new_image
write_dict(target, unpacked_info)
# Remove the container
logging.info("Destroying container %s", container.decode('ascii'))
retcode = subprocess.call(args.docker_cmd.split() + ['rm', container])
if retcode != 0:
logging.error("Error deleting container %s", container.decode('ascii'))
# Untag previous image, unless it is the initial_image
if image != unpacked_info['initial_image']:
logging.info("Untagging previous image %s", image.decode('ascii'))
subprocess.check_call(args.docker_cmd.split() + ['rmi', image])
# Update input file status
metadata_update_run(config, unpacked_info, selected_runs)
write_dict(target, unpacked_info)
signals.post_run(target=target, retcode=retcode)
def vagrant_run(args):
"""Runs the experiment in the virtual machine.
"""
target = Path(args.target[0])
unpacked_info = read_dict(target)
use_chroot = unpacked_info.get('use_chroot', True)
cmdline = args.cmdline
check_vagrant_version()
# Loads config
config = load_config(target / 'config.yml', True)
runs = config.runs
selected_runs = get_runs(runs, args.run, cmdline)
hostname = runs[selected_runs[0]].get('hostname', 'reprounzip')
# X11 handler
x11 = X11Handler(args.x11, ('local', hostname), args.x11_display)
cmds = []
for run_number in selected_runs:
run = runs[run_number]
cmd = 'cd %s && ' % shell_escape(run['workingdir'])
if use_chroot:
cmd += '/busybox env -i '
else:
cmd += '/usr/bin/env -i '
environ = x11.fix_env(run['environ'])
environ = fixup_environment(environ, args)
cmd += ' '.join('%s=%s' % (k, shell_escape(v))
for k, v in iteritems(environ))
cmd += ' '
# FIXME : Use exec -a or something if binary != argv[0]
if cmdline is None:
argv = [run['binary']] + run['argv'][1:]
else:
argv = cmdline
cmd += ' '.join(shell_escape(a) for a in argv)
uid = run.get('uid', 1000)
gid = run.get('gid', 1000)
if use_chroot:
userspec = '%s:%s' % (uid, gid)
cmd = ('chroot --userspec=%s /experimentroot '
'/bin/sh -c %s' % (
userspec,
shell_escape(cmd)))
else:
cmd = 'sudo -u \'#%d\' sh -c %s' % (uid, shell_escape(cmd))
cmds.append(cmd)
if use_chroot:
cmds = ['chroot /experimentroot /bin/sh -c %s' % shell_escape(c)
for c in x11.init_cmds] + cmds
else:
cmds = x11.init_cmds + cmds
cmds = ' && '.join(cmds)
# Sets the hostname to the original experiment's machine's
# FIXME: not reentrant: this restores the Vagrant machine's hostname after
# the run, which might cause issues if several "reprounzip vagrant run" are
# running at once
cmds = ('OLD_HOSTNAME=$(/bin/hostname); /bin/hostname %s; ' % hostname +
cmds +
'; RES=$?; /bin/hostname "$OLD_HOSTNAME"; exit $RES')
cmds = '/usr/bin/sudo /bin/sh -c %s' % shell_escape(cmds)
# Gets vagrant SSH parameters
info = machine_setup(target, unpacked_info['use_chroot'])
signals.pre_run(target=target)
interactive = not (args.no_stdin or
os.environ.get('REPROUNZIP_NON_INTERACTIVE'))
retcode = run_interactive(info, interactive,
cmds,
not args.no_pty,
x11.port_forward)
stderr.write("\r\n*** Command finished, status: %d\r\n" % retcode)
# Update input file status
metadata_update_run(config, unpacked_info, selected_runs)
write_dict(target, unpacked_info)
signals.post_run(target=target, retcode=retcode)
def directory_run(args):
"""Runs the command in the directory.
"""
target = Path(args.target[0])
unpacked_info = metadata_read(target, 'directory')
cmdline = args.cmdline
# Loads config
config = load_config_file(target / 'config.yml', True)
runs = config.runs
selected_runs = get_runs(runs, args.run, cmdline)
root = (target / 'root').absolute()
# Gets library paths
lib_dirs = []
p = subprocess.Popen(['/sbin/ldconfig', '-v', '-N'],
stdout=subprocess.PIPE)
try:
for l in p.stdout:
if len(l) < 3 or l[0] in (b' ', b'\t'):
continue
if l.endswith(b':\n'):
lib_dirs.append(Path(l[:-2]))
finally:
p.communicate()
lib_dirs = ('export LD_LIBRARY_PATH=%s' % ':'.join(
shell_escape(unicode_(join_root(root, d)))
for d in lib_dirs))
cmds = [lib_dirs]
for run_number in selected_runs:
run = runs[run_number]
cmd = 'cd %s && ' % shell_escape(
unicode_(join_root(root,
Path(run['workingdir']))))
cmd += '/usr/bin/env -i '
environ = run['environ']
environ = fixup_environment(environ, args)
if args.x11:
if 'DISPLAY' in os.environ:
environ['DISPLAY'] = os.environ['DISPLAY']
if 'XAUTHORITY' in os.environ:
environ['XAUTHORITY'] = os.environ['XAUTHORITY']
cmd += ' '.join('%s=%s' % (shell_escape(k), shell_escape(v))
for k, v in iteritems(environ)
if k != 'PATH')
cmd += ' '
# PATH
# Get the original PATH components
path = [PosixPath(d)
for d in run['environ'].get('PATH', '').split(':')]
# The same paths but in the directory
dir_path = [join_root(root, d)
for d in path
if d.root == '/']
# Rebuild string
path = ':'.join(unicode_(d) for d in dir_path + path)
cmd += 'PATH=%s ' % shell_escape(path)
# FIXME : Use exec -a or something if binary != argv[0]
if cmdline is None:
argv = run['argv']
# Rewrites command-line arguments that are absolute filenames
rewritten = False
for i in irange(len(argv)):
try:
p = Path(argv[i])
except UnicodeEncodeError:
continue
if p.is_absolute:
rp = join_root(root, p)
if (rp.exists() or
(len(rp.components) > 3 and rp.parent.exists())):
argv[i] = str(rp)
rewritten = True
if rewritten:
logging.warning("Rewrote command-line as: %s",
' '.join(shell_escape(a) for a in argv))
else:
argv = cmdline
cmd += ' '.join(shell_escape(a) for a in argv)
cmds.append(cmd)
cmds = ' && '.join(cmds)
signals.pre_run(target=target)
retcode = interruptible_call(cmds, shell=True)
stderr.write("\n*** Command finished, status: %d\n" % retcode)
signals.post_run(target=target, retcode=retcode)
# Update input file status
metadata_update_run(config, unpacked_info, selected_runs)
metadata_write(target, unpacked_info, 'directory')
def chroot_run(args):
"""Runs the command in the chroot.
"""
target = Path(args.target[0])
unpacked_info = metadata_read(target, 'chroot')
cmdline = args.cmdline
# Loads config
config = load_config_file(target / 'config.yml', True)
runs = config.runs
selected_runs = get_runs(runs, args.run, cmdline)
root = target / 'root'
# X11 handler
x11 = X11Handler(args.x11, ('local', socket.gethostname()),
args.x11_display)
cmds = []
for run_number in selected_runs:
run = runs[run_number]
cmd = 'cd %s && ' % shell_escape(run['workingdir'])
cmd += '/usr/bin/env -i '
environ = x11.fix_env(run['environ'])
environ = fixup_environment(environ, args)
cmd += ' '.join('%s=%s' % (shell_escape(k), shell_escape(v))
for k, v in iteritems(environ))
cmd += ' '
# FIXME : Use exec -a or something if binary != argv[0]
if cmdline is None:
argv = [run['binary']] + run['argv'][1:]
else:
argv = cmdline
cmd += ' '.join(shell_escape(a) for a in argv)
userspec = '%s:%s' % (run.get('uid', 1000),
run.get('gid', 1000))
cmd = 'chroot --userspec=%s %s /bin/sh -c %s' % (
userspec,
shell_escape(unicode_(root)),
shell_escape(cmd))
cmds.append(cmd)
cmds = ['chroot %s /bin/sh -c %s' % (shell_escape(unicode_(root)),
shell_escape(c))
for c in x11.init_cmds] + cmds
cmds = ' && '.join(cmds)
# Starts forwarding
forwarders = []
for portnum, connector in x11.port_forward:
fwd = LocalForwarder(connector, portnum)
forwarders.append(fwd)
signals.pre_run(target=target)
retcode = interruptible_call(cmds, shell=True)
stderr.write("\n*** Command finished, status: %d\n" % retcode)
signals.post_run(target=target, retcode=retcode)
# Update input file status
metadata_update_run(config, unpacked_info, selected_runs)
metadata_write(target, unpacked_info, 'chroot')
def docker_run(args):
"""Runs the experiment in the container.
"""
target = Path(args.target[0])
unpacked_info = read_dict(target)
cmdline = args.cmdline
# Sanity check
if args.detach and args.x11:
logger.critical("Error: Can't use X11 forwarding if you're detaching")
raise UsageError
# Loads config
config = load_config(target / 'config.yml', True)
runs = config.runs
selected_runs = get_runs(runs, args.run, cmdline)
# Get current image name
if 'current_image' in unpacked_info:
image = unpacked_info['current_image']
logger.debug("Running from image %s", image.decode('ascii'))
else:
logger.critical("Image doesn't exist yet, have you run setup/build?")
sys.exit(1)
# Name of new container
if args.detach:
container = make_unique_name(b'reprounzip_detached_')
else:
container = make_unique_name(b'reprounzip_run_')
hostname = runs[selected_runs[0]].get('hostname', 'reprounzip')
# Port forwarding
port_options = []
for port_host, port_container, proto in parse_ports(args.expose_port):
port_options.extend(['-p',
'%s:%s/%s' % (port_host, port_container, proto)])
# X11 handler
if args.x11:
local_ip = get_local_addr()
docker_host = local_ip
if os.environ.get('DOCKER_HOST'):
m = _dockerhost_re.match(os.environ['DOCKER_HOST'])
if m is not None:
docker_host = m.group(1)
if args.tunneled_x11:
x11 = X11Handler(True, ('internet', docker_host), args.x11_display)
else:
x11 = X11Handler(True, ('internet', local_ip), args.x11_display)
if (docker_host != local_ip and docker_host != 'localhost' and
not docker_host.startswith('127.') and
not docker_host.startswith('192.168.99.')):
ssh_cmdline = ' '.join(
'-R*:%(p)d:127.0.0.1:%(p)d' % {'p': port}
for port, connector in x11.port_forward)
logger.warning(
"You requested X11 forwarding but the Docker container "
"appears to be running remotely. It is probable that it "
"won't be able to connect to the local display. Creating "
"a remote SSH tunnel and running with --tunneled-x11 "
"might help (%s).",
ssh_cmdline)
else:
x11 = X11Handler(False, ('local', hostname), args.x11_display)
cmd = []
for run_number in selected_runs:
run = runs[run_number]
env_set, env_unset = x11.env_fixes(run['environ'])
a_env_set, a_env_unset = parse_environment_args(args)
env_set.update(a_env_set)
env_unset.extend(a_env_unset)
if env_set or env_unset:
cmd.append('env')
env = []
for k in env_unset:
env.append('-u')
env.append(shell_escape(k))
for k, v in iteritems(env_set):
env.append('%s=%s' % (shell_escape(k), shell_escape(v)))
cmd.append(' '.join(env))
# FIXME : Use exec -a or something if binary != argv[0]
if cmdline is not None:
cmd.append('cmd')
cmd.append(' '.join(shell_escape(a) for a in cmdline))
cmd.append('run')
cmd.append('%d' % run_number)
cmd = list(chain.from_iterable([['do', shell_escape(c)]
for c in x11.init_cmds] +
[cmd]))
if logger.isEnabledFor(logging.DEBUG):
logger.debug("Passing arguments to Docker image:")
for c in cmd:
logger.debug(c)
signals.pre_run(target=target)
# Creates forwarders
forwarders = []
for port, connector in x11.port_forward:
forwarders.append(LocalForwarder(connector, port))
if args.detach:
logger.info("Start container %s (detached)",
container.decode('ascii'))
retcode = interruptible_call(args.docker_cmd.split() +
['run', b'--name=' + container,
'-h', hostname,
'-d', '-t'] +
port_options +
args.docker_option +
[image] + cmd)
if retcode != 0:
logger.critical("docker run failed with code %d", retcode)
subprocess.call(['docker', 'rm', '-f', container])
sys.exit(1)
return
# Run command in container
logger.info("Starting container %s", container.decode('ascii'))
retcode = interruptible_call(args.docker_cmd.split() +
['run', b'--name=' + container,
'-h', hostname,
'-i', '-t'] +
port_options +
args.docker_option +
[image] + cmd,
request_tty=True)
# The image prints out the exit status(es) itself
if retcode != 0:
logger.critical("docker run failed with code %d", retcode)
subprocess.call(['docker', 'rm', '-f', container])
sys.exit(1)
# Commit to create new image
new_image = make_unique_name(b'reprounzip_image_')
logger.info("Committing container %s to image %s",
container.decode('ascii'), new_image.decode('ascii'))
subprocess.check_call(args.docker_cmd.split() +
['commit', container, new_image])
# Update image name
unpacked_info['current_image'] = new_image
write_dict(target, unpacked_info)
# Remove the container
logger.info("Destroying container %s", container.decode('ascii'))
retcode = subprocess.call(args.docker_cmd.split() + ['rm', container])
if retcode != 0:
logger.error("Error deleting container %s", container.decode('ascii'))
# Untag previous image, unless it is the initial_image
if image != unpacked_info['initial_image']:
logger.info("Untagging previous image %s", image.decode('ascii'))
subprocess.check_call(args.docker_cmd.split() + ['rmi', image])
# Update input file status
metadata_update_run(config, unpacked_info, selected_runs)
write_dict(target, unpacked_info)
signals.post_run(target=target, retcode=retcode)
def vagrant_run(args):
"""Runs the experiment in the virtual machine.
"""
target = Path(args.target[0])
unpacked_info = read_dict(target)
use_chroot = unpacked_info['use_chroot']
cmdline = args.cmdline
check_vagrant_version()
# Loads config
config = load_config(target / 'config.yml', True)
runs = config.runs
selected_runs = get_runs(runs, args.run, cmdline)
hostname = runs[selected_runs[0]].get('hostname', 'reprounzip')
# X11 handler
if unpacked_info['gui']:
x11 = LocalX11Handler()
else:
x11 = X11Handler(args.x11, ('local', hostname), args.x11_display)
cmds = []
for run_number in selected_runs:
run = runs[run_number]
cmd = 'cd %s && ' % shell_escape(run['workingdir'])
if use_chroot:
cmd += '/busybox env -i '
else:
cmd += '/usr/bin/env -i '
environ = x11.fix_env(run['environ'])
environ = fixup_environment(environ, args)
cmd += ' '.join('%s=%s' % (shell_escape(k), shell_escape(v))
for k, v in iteritems(environ))
cmd += ' '
# FIXME : Use exec -a or something if binary != argv[0]
if cmdline is None:
argv = [run['binary']] + run['argv'][1:]
else:
argv = cmdline
cmd += ' '.join(shell_escape(a) for a in argv)
uid = run.get('uid', 1000)
gid = run.get('gid', 1000)
if use_chroot:
userspec = '%s:%s' % (uid, gid)
cmd = ('chroot --userspec=%s /experimentroot '
'/bin/sh -c %s' % (
userspec,
shell_escape(cmd)))
else:
cmd = 'sudo -u \'#%d\' sh -c %s' % (uid, shell_escape(cmd))
cmds.append(cmd)
if use_chroot:
cmds = ['chroot /experimentroot /bin/sh -c %s' % shell_escape(c)
for c in x11.init_cmds] + cmds
else:
cmds = x11.init_cmds + cmds
cmds = ' && '.join(cmds)
# Sets the hostname to the original experiment's machine's
# FIXME: not reentrant: this restores the Vagrant machine's hostname after
# the run, which might cause issues if several "reprounzip vagrant run" are
# running at once
cmds = ('OLD_HOSTNAME=$(/bin/hostname); /bin/hostname %s; ' % hostname +
cmds +
'; RES=$?; /bin/hostname "$OLD_HOSTNAME"; exit $RES')
cmds = '/usr/bin/sudo /bin/sh -c %s' % shell_escape(cmds)
# Gets vagrant SSH parameters
info = machine_setup(target)
signals.pre_run(target=target)
interactive = not (args.no_stdin or
os.environ.get('REPROUNZIP_NON_INTERACTIVE'))
retcode = run_interactive(info, interactive,
cmds,
not args.no_pty,
x11.port_forward)
stderr.write("\r\n*** Command finished, status: %d\r\n" % retcode)
# Update input file status
metadata_update_run(config, unpacked_info, selected_runs)
write_dict(target, unpacked_info)
signals.post_run(target=target, retcode=retcode)
def vagrant_run(args):
"""Runs the experiment in the virtual machine.
"""
target = Path(args.target[0])
unpacked_info = read_dict(target)
use_chroot = unpacked_info['use_chroot']
cmdline = args.cmdline
check_vagrant_version()
# Loads config
config = load_config(target / 'config.yml', True)
runs = config.runs
selected_runs = get_runs(runs, args.run, cmdline)
hostname = runs[selected_runs[0]].get('hostname', 'reprounzip')
# Port forwarding
ports = parse_ports(args.expose_port)
# If the requested ports are not a subset of the ones already set on the
# VM, we have to update the Vagrantfile and issue `vagrant reload`, which
# will reboot the machine
req_ports = set(ports)
set_ports = set(unpacked_info.get('ports', []))
if not req_ports.issubset(set_ports):
# Build new set of forwarded ports: the ones already set + the one just
# requested
# The ones we request now override the previous config
all_ports = dict(
(host, (guest, proto)) for host, guest, proto in set_ports)
for host, guest, proto in req_ports:
all_ports[host] = guest, proto
unpacked_info['ports'] = sorted(
(host, guest, proto)
for host, (guest, proto) in iteritems(all_ports))
write_vagrantfile(target, unpacked_info)
logger.info("Some requested ports are not yet forwarded, running "
"'vagrant reload'")
retcode = subprocess.call(['vagrant', 'reload', '--no-provision'],
cwd=target.path)
if retcode != 0:
logger.critical("vagrant reload failed with code %d, aborting",
retcode)
sys.exit(1)
write_dict(target, unpacked_info)
# X11 handler
if unpacked_info['gui']:
x11 = LocalX11Handler()
else:
x11 = X11Handler(args.x11, ('local', hostname), args.x11_display)
cmds = []
for run_number in selected_runs:
run = runs[run_number]
cmd = 'cd %s && ' % shell_escape(run['workingdir'])
if use_chroot:
cmd += '/busybox env -i '
else:
cmd += '/usr/bin/env -i '
environ = x11.fix_env(run['environ'])
environ = fixup_environment(environ, args)
cmd += ' '.join('%s=%s' % (shell_escape(k), shell_escape(v))
for k, v in iteritems(environ))
cmd += ' '
# FIXME : Use exec -a or something if binary != argv[0]
if cmdline is None:
argv = [run['binary']] + run['argv'][1:]
else:
argv = cmdline
cmd += ' '.join(shell_escape(a) for a in argv)
uid = run.get('uid', 1000)
gid = run.get('gid', 1000)
if use_chroot:
userspec = '%s:%s' % (uid, gid)
cmd = ('chroot --userspec=%s /experimentroot '
'/bin/sh -c %s' % (userspec, shell_escape(cmd)))
else:
cmd = 'sudo -u \'#%d\' sh -c %s' % (uid, shell_escape(cmd))
cmds.append(cmd)
if use_chroot:
cmds = [
'chroot /experimentroot /bin/sh -c %s' % shell_escape(c)
for c in x11.init_cmds
] + cmds
else:
cmds = x11.init_cmds + cmds
cmds = ' && '.join(cmds)
# Sets the hostname to the original experiment's machine's
# FIXME: not reentrant: this restores the Vagrant machine's hostname after
# the run, which might cause issues if several "reprounzip vagrant run" are
# running at once
cmds = ('OLD_HOSTNAME=$(/bin/hostname); /bin/hostname %s; ' % hostname +
cmds + '; RES=$?; /bin/hostname "$OLD_HOSTNAME"; exit $RES')
cmds = '/usr/bin/sudo /bin/sh -c %s' % shell_escape(cmds)
# Gets vagrant SSH parameters
info = machine_setup(target)
signals.pre_run(target=target)
interactive = not (args.no_stdin
or os.environ.get('REPROUNZIP_NON_INTERACTIVE'))
retcode = run_interactive(info, interactive, cmds, not args.no_pty,
x11.port_forward)
stderr.write("\r\n*** Command finished, status: %d\r\n" % retcode)
# Update input file status
metadata_update_run(config, unpacked_info, selected_runs)
write_dict(target, unpacked_info)
signals.post_run(target=target, retcode=retcode)
def docker_run(args):
"""Runs the experiment in the container.
"""
target = Path(args.target[0])
unpacked_info = read_dict(target)
cmdline = args.cmdline
# Sanity check
if args.detach and args.x11:
logging.critical("Error: Can't use X11 forwarding if you're detaching")
raise UsageError
# Loads config
config = load_config(target / 'config.yml', True)
runs = config.runs
selected_runs = get_runs(runs, args.run, cmdline)
# Get current image name
if 'current_image' in unpacked_info:
image = unpacked_info['current_image']
logging.debug("Running from image %s", image.decode('ascii'))
else:
logging.critical("Image doesn't exist yet, have you run setup/build?")
sys.exit(1)
# Name of new container
if args.detach:
container = make_unique_name(b'reprounzip_detached_')
else:
container = make_unique_name(b'reprounzip_run_')
hostname = runs[selected_runs[0]].get('hostname', 'reprounzip')
# X11 handler
if args.x11:
local_ip = get_local_addr()
docker_host = local_ip
if os.environ.get('DOCKER_HOST'):
m = _dockerhost_re.match(os.environ['DOCKER_HOST'])
if m is not None:
docker_host = m.group(1)
if args.tunneled_x11:
x11 = X11Handler(True, ('internet', docker_host), args.x11_display)
else:
x11 = X11Handler(True, ('internet', local_ip), args.x11_display)
if (docker_host != local_ip and docker_host != 'localhost' and
not docker_host.startswith('127.') and
not docker_host.startswith('192.168.99.')):
ssh_cmdline = ' '.join(
'-R*:%(p)d:127.0.0.1:%(p)d' % {'p': port}
for port, connector in x11.port_forward)
logging.warning(
"You requested X11 forwarding but the Docker container "
"appears to be running remotely. It is probable that it "
"won't be able to connect to the local display. Creating "
"a remote SSH tunnel and running with --tunneled-x11 "
"might help (%s).",
ssh_cmdline)
else:
x11 = X11Handler(False, ('local', hostname), args.x11_display)
cmds = []
for run_number in selected_runs:
run = runs[run_number]
cmd = 'cd %s && ' % shell_escape(run['workingdir'])
cmd += '/busybox env -i '
environ = x11.fix_env(run['environ'])
environ = fixup_environment(environ, args)
cmd += ' '.join('%s=%s' % (shell_escape(k), shell_escape(v))
for k, v in iteritems(environ))
cmd += ' '
# FIXME : Use exec -a or something if binary != argv[0]
if cmdline is None:
argv = [run['binary']] + run['argv'][1:]
else:
argv = cmdline
cmd += ' '.join(shell_escape(a) for a in argv)
uid = run.get('uid', 1000)
gid = run.get('gid', 1000)
cmd = '/rpzsudo \'#%d\' \'#%d\' /busybox sh -c %s' % (
uid, gid,
shell_escape(cmd))
cmds.append(cmd)
cmds = x11.init_cmds + cmds
cmds = ' && '.join(cmds)
signals.pre_run(target=target)
# Creates forwarders
forwarders = []
for port, connector in x11.port_forward:
forwarders.append(LocalForwarder(connector, port))
if args.detach:
logging.info("Start container %s (detached)",
container.decode('ascii'))
retcode = interruptible_call(['docker', 'run', b'--name=' + container,
'-h', hostname,
'-d', '-t'] +
args.docker_option +
[image, '/busybox', 'sh', '-c', cmds])
if retcode != 0:
logging.critical("docker run failed with code %d", retcode)
subprocess.call(['docker', 'rm', '-f', container])
sys.exit(1)
return
# Run command in container
logging.info("Starting container %s", container.decode('ascii'))
retcode = interruptible_call(['docker', 'run', b'--name=' + container,
'-h', hostname,
'-i', '-t'] +
args.docker_option +
[image, '/busybox', 'sh', '-c', cmds])
if retcode != 0:
logging.critical("docker run failed with code %d", retcode)
subprocess.call(['docker', 'rm', '-f', container])
sys.exit(1)
# Get exit status from "docker inspect"
out = subprocess.check_output(['docker', 'inspect', container])
outjson = json.loads(out.decode('ascii'))
if (outjson[0]["State"]["Running"] is not False or
outjson[0]["State"]["Paused"] is not False):
logging.error("Invalid container state after execution:\n%s",
json.dumps(outjson[0]["State"]))
retcode = outjson[0]["State"]["ExitCode"]
stderr.write("\n*** Command finished, status: %d\n" % retcode)
# Commit to create new image
new_image = make_unique_name(b'reprounzip_image_')
logging.info("Committing container %s to image %s",
container.decode('ascii'), new_image.decode('ascii'))
subprocess.check_call(['docker', 'commit', container, new_image])
# Update image name
unpacked_info['current_image'] = new_image
write_dict(target, unpacked_info)
# Remove the container
logging.info("Destroying container %s", container.decode('ascii'))
retcode = subprocess.call(['docker', 'rm', container])
if retcode != 0:
logging.error("Error deleting container %s", container.decode('ascii'))
# Untag previous image, unless it is the initial_image
if image != unpacked_info['initial_image']:
logging.info("Untagging previous image %s", image.decode('ascii'))
subprocess.check_call(['docker', 'rmi', image])
# Update input file status
metadata_update_run(config, unpacked_info, selected_runs)
write_dict(target, unpacked_info)
signals.post_run(target=target, retcode=retcode)
def vagrant_run(args):
"""Runs the experiment in the virtual machine.
"""
target = Path(args.target[0])
unpacked_info = read_dict(target)
use_chroot = unpacked_info['use_chroot']
cmdline = args.cmdline
check_vagrant_version()
# Loads config
config = load_config(target / 'config.yml', True)
runs = config.runs
selected_runs = get_runs(runs, args.run, cmdline)
hostname = runs[selected_runs[0]].get('hostname', 'reprounzip')
# Port forwarding
ports = parse_ports(args.expose_port)
# If the requested ports are not a subset of the ones already set on the
# VM, we have to update the Vagrantfile and issue `vagrant reload`, which
# will reboot the machine
req_ports = set(ports)
set_ports = set(unpacked_info.get('ports', []))
if not req_ports.issubset(set_ports):
# Build new set of forwarded ports: the ones already set + the one just
# requested
# The ones we request now override the previous config
all_ports = dict((host, (guest, proto))
for host, guest, proto in set_ports)
for host, guest, proto in req_ports:
all_ports[host] = guest, proto
unpacked_info['ports'] = sorted(
(host, guest, proto)
for host, (guest, proto) in iteritems(all_ports))
write_vagrantfile(target, unpacked_info)
logger.info("Some requested ports are not yet forwarded, running "
"'vagrant reload'")
retcode = subprocess.call(['vagrant', 'reload', '--no-provision'],
cwd=target.path)
if retcode != 0:
logger.critical("vagrant reload failed with code %d, aborting",
retcode)
sys.exit(1)
write_dict(target, unpacked_info)
# X11 handler
if unpacked_info['gui']:
x11 = LocalX11Handler()
else:
x11 = X11Handler(args.x11, ('local', hostname), args.x11_display)
cmds = []
for run_number in selected_runs:
run = runs[run_number]
cmd = 'cd %s && ' % shell_escape(run['workingdir'])
if use_chroot:
cmd += '/busybox env -i '
else:
cmd += '/usr/bin/env -i '
environ = x11.fix_env(run['environ'])
environ = fixup_environment(environ, args)
cmd += ' '.join('%s=%s' % (shell_escape(k), shell_escape(v))
for k, v in iteritems(environ))
cmd += ' '
# FIXME : Use exec -a or something if binary != argv[0]
if cmdline is None:
argv = [run['binary']] + run['argv'][1:]
else:
argv = cmdline
cmd += ' '.join(shell_escape(a) for a in argv)
uid = run.get('uid', 1000)
gid = run.get('gid', 1000)
if use_chroot:
userspec = '%s:%s' % (uid, gid)
cmd = ('chroot --userspec=%s /experimentroot '
'/bin/sh -c %s' % (
userspec,
shell_escape(cmd)))
else:
cmd = 'sudo -u \'#%d\' sh -c %s' % (uid, shell_escape(cmd))
cmds.append(cmd)
if use_chroot:
cmds = ['chroot /experimentroot /bin/sh -c %s' % shell_escape(c)
for c in x11.init_cmds] + cmds
else:
cmds = x11.init_cmds + cmds
cmds = ' && '.join(cmds)
# Sets the hostname to the original experiment's machine's
# FIXME: not reentrant: this restores the Vagrant machine's hostname after
# the run, which might cause issues if several "reprounzip vagrant run" are
# running at once
cmds = ('OLD_HOSTNAME=$(/bin/hostname); /bin/hostname %s; ' % hostname +
cmds +
'; RES=$?; /bin/hostname "$OLD_HOSTNAME"; exit $RES')
cmds = '/usr/bin/sudo /bin/sh -c %s' % shell_escape(cmds)
# Gets vagrant SSH parameters
info = machine_setup(target)
signals.pre_run(target=target)
interactive = not (args.no_stdin or
os.environ.get('REPROUNZIP_NON_INTERACTIVE'))
retcode = run_interactive(info, interactive,
cmds,
not args.no_pty,
x11.port_forward)
stderr.write("\r\n*** Command finished, status: %d\r\n" % retcode)
# Update input file status
metadata_update_run(config, unpacked_info, selected_runs)
write_dict(target, unpacked_info)
signals.post_run(target=target, retcode=retcode)