def get_ssh_parameters(target):
"""Reads the SSH parameters from ``vagrant ssh`` command output.
"""
try:
stdout = check_output(['vagrant', 'ssh-config'],
cwd=target.path,
stderr=subprocess.PIPE)
except subprocess.CalledProcessError:
# Makes sure the VM is running
subprocess.check_call(['vagrant', 'up'], cwd=target.path)
# Try again
stdout = check_output(['vagrant', 'ssh-config'], cwd=target.path)
info = {}
for line in stdout.split(b'\n'):
line = line.strip().split(b' ', 1)
if len(line) != 2:
continue
info[line[0].decode('utf-8').lower()] = line[1].decode('utf-8')
if 'identityfile' in info:
key_file = info['identityfile']
else:
key_file = Path('~/.vagrant.d/insecure_private_key').expand_user()
ret = dict(hostname=info.get('hostname', '127.0.0.1'),
port=int(info.get('port', 2222)),
username=info.get('user', 'vagrant'),
key_filename=key_file)
logging.debug("SSH parameters from Vagrant: %s@%s:%s, key=%s",
ret['username'], ret['hostname'], ret['port'],
ret['key_filename'])
return ret
def get_ssh_parameters(target):
"""Reads the SSH parameters from ``vagrant ssh`` command output.
"""
try:
stdout = check_output(['vagrant', 'ssh-config'],
cwd=target.path,
stderr=subprocess.PIPE)
except subprocess.CalledProcessError:
# Makes sure the VM is running
subprocess.check_call(['vagrant', 'up'],
cwd=target.path)
# Try again
stdout = check_output(['vagrant', 'ssh-config'],
cwd=target.path)
info = {}
for line in stdout.split(b'\n'):
line = line.strip().split(b' ', 1)
if len(line) != 2:
continue
info[line[0].decode('utf-8').lower()] = line[1].decode('utf-8')
if 'identityfile' in info:
key_file = info['identityfile']
else:
key_file = Path('~/.vagrant.d/insecure_private_key').expand_user()
ret = dict(hostname=info.get('hostname', '127.0.0.1'),
port=int(info.get('port', 2222)),
username=info.get('user', 'vagrant'),
key_filename=key_file)
logging.debug("SSH parameters from Vagrant: %s@%s:%s, key=%s",
ret['username'], ret['hostname'], ret['port'],
ret['key_filename'])
return ret
def get_local_addr(iface='docker0'):
"""Gets the local IP address of the local machine.
Returns an IPv4 address as a unicode object, in digits-and-dots format.
>>> get_local_addr('lo')
'127.0.0.1'
>>> get_local_addr()
'172.17.42.1'
"""
try:
try:
out = check_output(['/bin/ip', 'addr', 'show', iface])
except subprocess.CalledProcessError:
logging.info(
"No interface '%s', querying any interface for an IP...",
iface)
out = check_output(['/bin/ip', 'addr', 'show'])
except (OSError, subprocess.CalledProcessError):
# This is probably going to return '127.0.0.1', and that is bad
return socket.gethostbyname(socket.gethostname())
else:
for line in reversed(out.splitlines()):
m = _addr_re.search(line)
if m is not None:
return m.group(1).decode('ascii')
def check_vagrant_version():
try:
out = check_output(['vagrant', '--version'])
except (subprocess.CalledProcessError, OSError):
logging.error("Couldn't run vagrant")
sys.exit(1)
out = out.decode('ascii').strip().lower().split()
if out[0] == 'vagrant':
if LooseVersion(out[1]) < LooseVersion('1.1'):
logging.error("Vagrant >=1.1 is required; detected version: %s",
out[1])
sys.exit(1)
else:
logging.error("Vagrant >=1.1 is required")
sys.exit(1)
def check_vagrant_version():
try:
stdout = check_output(['vagrant', '--version'])
except subprocess.CalledProcessError:
logging.error("Couldn't run vagrant")
sys.exit(1)
stdout = stdout.decode('ascii').strip().lower().split()
if stdout[0] == 'vagrant':
if LooseVersion(stdout[1]) < LooseVersion('1.1'):
logging.error("Vagrant >=1.1 is required; detected version: %s",
stdout[1])
sys.exit(1)
else:
logging.error("Vagrant >=1.1 is required")
sys.exit(1)
def test_has_vagrant(pack, **kwargs):
"""Compatibility test: has vagrant (ok) or not (maybe).
"""
if not _executable_in_path('vagrant'):
return COMPAT_MAYBE, "vagrant not found in PATH"
try:
out = check_output(['vagrant', '--version'])
except subprocess.CalledProcessError:
return COMPAT_NO, ("vagrant was found in PATH but doesn't seem to "
"work properly")
out = out.decode('ascii').strip().lower().split()
if out[0] == 'vagrant':
if LooseVersion(out[1]) >= LooseVersion('1.1'):
return COMPAT_OK
else:
return COMPAT_NO, ("Vagrant >=1.1 is required; detected version: "
"%s" % out[1])
else:
return COMPAT_NO, "Vagrant >=1.1 is required"
def test_has_vagrant(pack, **kwargs):
"""Compatibility test: has vagrant (ok) or not (maybe).
"""
if not _executable_in_path('vagrant'):
return COMPAT_MAYBE, "vagrant not found in PATH"
try:
stdout = check_output(['vagrant', '--version'])
except subprocess.CalledProcessError:
return COMPAT_NO, ("vagrant was found in PATH but doesn't seem to "
"work properly")
stdout = stdout.decode('ascii').strip().lower().split()
if stdout[0] == 'vagrant':
if LooseVersion(stdout[1]) >= LooseVersion('1.1'):
return COMPAT_OK
else:
return COMPAT_NO, ("Vagrant >=1.1 is required; detected version: "
"%s" % stdout[1])
else:
return COMPAT_NO, "Vagrant >=1.1 is required"
def machine_setup(target, use_chroot):
"""Prepare the machine and get SSH parameters from ``vagrant ssh``.
"""
try:
out = check_output(['vagrant', 'ssh-config'],
cwd=target.path,
stderr=subprocess.PIPE)
except subprocess.CalledProcessError:
# Makes sure the VM is running
logging.info("Calling 'vagrant up'...")
try:
retcode = subprocess.check_call(['vagrant', 'up'], cwd=target.path)
except OSError:
logging.critical("vagrant executable not found")
sys.exit(1)
else:
if retcode != 0:
logging.critical("vagrant up failed with code %d", retcode)
sys.exit(1)
# Try again
out = check_output(['vagrant', 'ssh-config'],
cwd=target.path)
vagrant_info = {}
for line in out.split(b'\n'):
line = line.strip().split(b' ', 1)
if len(line) != 2:
continue
value = line[1].decode('utf-8')
if len(value) >= 2 and value[0] == '"' and value[-1] == '"':
# Vagrant should really be escaping special characters here, but
# it's not -- https://github.com/mitchellh/vagrant/issues/6428
value = value[1:-1]
vagrant_info[line[0].decode('utf-8').lower()] = value
if 'identityfile' in vagrant_info:
key_file = vagrant_info['identityfile']
else:
key_file = Path('~/.vagrant.d/insecure_private_key').expand_user()
info = dict(hostname=vagrant_info.get('hostname', '127.0.0.1'),
port=int(vagrant_info.get('port', 2222)),
username=vagrant_info.get('user', 'vagrant'),
key_filename=key_file)
logging.debug("SSH parameters from Vagrant: %s@%s:%s, key=%s",
info['username'], info['hostname'], info['port'],
info['key_filename'])
if use_chroot:
# Mount directories
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(IgnoreMissingKey())
ssh.connect(**info)
chan = ssh.get_transport().open_session()
chan.exec_command(
'/usr/bin/sudo /bin/sh -c %s' % shell_escape(
'for i in dev proc; do '
'if ! grep "^/experimentroot/$i$" /proc/mounts; then '
'mount -o rbind /$i /experimentroot/$i; '
'fi; '
'done'))
ssh.close()
return info
def docker_run(args):
"""Runs the experiment in the container.
"""
target = Path(args.target[0])
unpacked_info = read_dict(target / '.reprounzip')
cmdline = args.cmdline
# Loads config
runs, packages, other_files = load_config(target / 'config.yml', True)
selected_runs = get_runs(runs, args.run, cmdline)
# Get current image name
if 'current_image' in unpacked_info:
image = unpacked_info['current_image']
logging.debug("Running from image %s", image.decode('ascii'))
else:
logging.critical("Image doesn't exist yet, have you run setup/build?")
sys.exit(1)
# Name of new container
container = make_unique_name(b'reprounzip_run_')
hostname = runs[selected_runs[0]].get('hostname', 'reprounzip')
# Get the local bridge IP
ip_str = get_iface_addr('docker0')
# X11 handler
x11 = X11Handler(args.x11, ('internet', ip_str), args.x11_display)
cmds = []
for run_number in selected_runs:
run = runs[run_number]
cmd = 'cd %s && ' % shell_escape(run['workingdir'])
cmd += '/busybox env -i '
environ = x11.fix_env(run['environ'])
cmd += ' '.join('%s=%s' % (k, shell_escape(v))
for k, v in iteritems(environ))
cmd += ' '
# FIXME : Use exec -a or something if binary != argv[0]
if cmdline is None:
argv = [run['binary']] + run['argv'][1:]
else:
argv = cmdline
cmd += ' '.join(shell_escape(a) for a in argv)
uid = run.get('uid', 1000)
gid = run.get('gid', 1000)
cmd = '/rpzsudo \'#%d\' \'#%d\' /busybox sh -c %s\n' % (
uid, gid,
shell_escape(cmd))
cmds.append(cmd)
cmds = x11.init_cmds + cmds
cmds = ' && '.join(cmds)
signals.pre_run(target=target)
# Creates forwarders
forwarders = []
for port, connector in x11.port_forward:
forwarders.append(LocalForwarder(connector, port))
# Run command in container
logging.info("Starting container %s", container.decode('ascii'))
retcode = interruptible_call(['docker', 'run', b'--name=' + container,
'-h', hostname,
'-i', '-t', image,
'/busybox', 'sh', '-c', cmds])
if retcode != 0:
logging.critical("docker run failed with code %d", retcode)
subprocess.call(['docker', 'rm', '-f', container])
sys.exit(1)
# Get exit status from "docker inspect"
out = check_output(['docker', 'inspect', container])
outjson = json.loads(out.decode('ascii'))
if (outjson[0]["State"]["Running"] is not False or
outjson[0]["State"]["Paused"] is not False):
logging.error("Invalid container state after execution:\n%s",
json.dumps(outjson[0]["State"]))
retcode = outjson[0]["State"]["ExitCode"]
stderr.write("\n*** Command finished, status: %d\n" % retcode)
# Commit to create new image
new_image = make_unique_name(b'reprounzip_image_')
logging.info("Committing container %s to image %s",
container.decode('ascii'), new_image.decode('ascii'))
subprocess.check_call(['docker', 'commit', container, new_image])
# Update image name
unpacked_info['current_image'] = new_image
write_dict(target / '.reprounzip', unpacked_info)
# Remove the container
logging.info("Destroying container %s", container.decode('ascii'))
retcode = subprocess.call(['docker', 'rm', container])
if retcode != 0:
logging.error("Error deleting container %s", container.decode('ascii'))
# Untag previous image, unless it is the initial_image
if image != unpacked_info['initial_image']:
logging.info("Untagging previous image %s", image.decode('ascii'))
subprocess.check_call(['docker', 'rmi', image])
signals.post_run(target=target, retcode=retcode)
def docker_run(args):
"""Runs the experiment in the container.
"""
target = Path(args.target[0])
unpacked_info = read_dict(target)
cmdline = args.cmdline
# Loads config
config = load_config(target / 'config.yml', True)
runs = config.runs
selected_runs = get_runs(runs, args.run, cmdline)
# Get current image name
if 'current_image' in unpacked_info:
image = unpacked_info['current_image']
logging.debug("Running from image %s", image.decode('ascii'))
else:
logging.critical("Image doesn't exist yet, have you run setup/build?")
sys.exit(1)
# Name of new container
container = make_unique_name(b'reprounzip_run_')
hostname = runs[selected_runs[0]].get('hostname', 'reprounzip')
# X11 handler
if args.x11:
local_ip = get_local_addr()
docker_host = local_ip
if os.environ.get('DOCKER_HOST'):
m = _dockerhost_re.match(os.environ['DOCKER_HOST'])
if m is not None:
docker_host = m.group(1)
if args.tunneled_x11:
x11 = X11Handler(True, ('internet', docker_host), args.x11_display)
else:
x11 = X11Handler(True, ('internet', local_ip), args.x11_display)
if (docker_host != local_ip and docker_host != 'localhost' and
not docker_host.startswith('127.') and
not docker_host.startswith('192.168.99.')):
ssh_cmdline = ' '.join(
'-R*:%(p)d:127.0.0.1:%(p)d' % {'p': port}
for port, connector in x11.port_forward)
logging.warning(
"You requested X11 forwarding but the Docker container "
"appears to be running remotely. It is probable that it "
"won't be able to connect to the local display. Creating "
"a remote SSH tunnel and running with --tunneled-x11 "
"might help (%s).",
ssh_cmdline)
else:
x11 = X11Handler(False, ('local', hostname), args.x11_display)
cmds = []
for run_number in selected_runs:
run = runs[run_number]
cmd = 'cd %s && ' % shell_escape(run['workingdir'])
cmd += '/busybox env -i '
environ = x11.fix_env(run['environ'])
environ = fixup_environment(environ, args)
cmd += ' '.join('%s=%s' % (k, shell_escape(v))
for k, v in iteritems(environ))
cmd += ' '
# FIXME : Use exec -a or something if binary != argv[0]
if cmdline is None:
argv = [run['binary']] + run['argv'][1:]
else:
argv = cmdline
cmd += ' '.join(shell_escape(a) for a in argv)
uid = run.get('uid', 1000)
gid = run.get('gid', 1000)
cmd = '/rpzsudo \'#%d\' \'#%d\' /busybox sh -c %s' % (
uid, gid,
shell_escape(cmd))
cmds.append(cmd)
cmds = x11.init_cmds + cmds
cmds = ' && '.join(cmds)
signals.pre_run(target=target)
# Creates forwarders
forwarders = []
for port, connector in x11.port_forward:
forwarders.append(LocalForwarder(connector, port))
# Run command in container
logging.info("Starting container %s", container.decode('ascii'))
retcode = interruptible_call(['docker', 'run', b'--name=' + container,
'-h', hostname,
'-i', '-t', image,
'/busybox', 'sh', '-c', cmds])
if retcode != 0:
logging.critical("docker run failed with code %d", retcode)
subprocess.call(['docker', 'rm', '-f', container])
sys.exit(1)
# Get exit status from "docker inspect"
out = check_output(['docker', 'inspect', container])
outjson = json.loads(out.decode('ascii'))
if (outjson[0]["State"]["Running"] is not False or
outjson[0]["State"]["Paused"] is not False):
logging.error("Invalid container state after execution:\n%s",
json.dumps(outjson[0]["State"]))
retcode = outjson[0]["State"]["ExitCode"]
stderr.write("\n*** Command finished, status: %d\n" % retcode)
# Commit to create new image
new_image = make_unique_name(b'reprounzip_image_')
logging.info("Committing container %s to image %s",
container.decode('ascii'), new_image.decode('ascii'))
subprocess.check_call(['docker', 'commit', container, new_image])
# Update image name
unpacked_info['current_image'] = new_image
write_dict(target, unpacked_info)
# Remove the container
logging.info("Destroying container %s", container.decode('ascii'))
retcode = subprocess.call(['docker', 'rm', container])
if retcode != 0:
logging.error("Error deleting container %s", container.decode('ascii'))
# Untag previous image, unless it is the initial_image
if image != unpacked_info['initial_image']:
logging.info("Untagging previous image %s", image.decode('ascii'))
subprocess.check_call(['docker', 'rmi', image])
# Update input file status
metadata_update_run(config, unpacked_info, selected_runs)
write_dict(target, unpacked_info)
signals.post_run(target=target, retcode=retcode)
