def download(self, remote_path, local_path):
if self.use_chroot:
remote_path = join_root(PosixPath('/experimentroot'), remote_path)
temp = make_unique_name(b'reprozip_output_')
rtemp = PosixPath('/vagrant') / temp
ltemp = self.target / temp
# Copy file to shared folder
logging.info("Copying file to shared folder...")
chan = self.ssh.get_transport().open_session()
cp_cmd = '/bin/cp %s %s' % (
shell_escape(remote_path.path),
shell_escape(rtemp.path))
chown_cmd = '/bin/chown vagrant %s' % shell_escape(rtemp.path)
chmod_cmd = '/bin/chmod 644 %s' % shell_escape(rtemp.path)
chan.exec_command('/usr/bin/sudo /bin/sh -c %s' % shell_escape(
' && '.join((cp_cmd, chown_cmd, chmod_cmd))))
if chan.recv_exit_status() != 0:
logging.critical("Couldn't copy file in virtual machine")
try:
ltemp.remove()
except OSError:
pass
return False
# Move file to final destination
try:
ltemp.rename(local_path)
except OSError as e:
logging.critical("Couldn't download output file: %s\n%s",
remote_path, str(e))
ltemp.remove()
return False
return True
def upload_file(self, local_path, input_path):
if self.use_chroot:
remote_path = join_root(PosixPath('/experimentroot'),
input_path)
else:
remote_path = input_path
# Upload to a temporary file first
logging.info("Uploading file via SCP...")
rtemp = PosixPath(make_unique_name(b'/tmp/reprozip_input_'))
self.client_scp.put(local_path.path, rtemp.path, recursive=False)
# Move it
logging.info("Moving file into place...")
chan = self.ssh.get_transport().open_session()
chown_cmd = '/bin/chown --reference=%s %s' % (
shell_escape(remote_path.path),
shell_escape(rtemp.path))
chmod_cmd = '/bin/chmod --reference=%s %s' % (
shell_escape(remote_path.path),
shell_escape(rtemp.path))
mv_cmd = '/bin/mv %s %s' % (
shell_escape(rtemp.path),
shell_escape(remote_path.path))
chan.exec_command('/usr/bin/sudo /bin/sh -c %s' % shell_escape(
';'.join((chown_cmd, chmod_cmd, mv_cmd))))
if chan.recv_exit_status() != 0:
logging.critical("Couldn't move file in virtual machine")
sys.exit(1)
chan.close()
def prepare_download(self, files):
# Create a container from the image
self.container = make_unique_name(b'reprounzip_dl_')
logging.info("Creating container %s", self.container.decode('ascii'))
subprocess.check_call(['docker', 'create',
b'--name=' + self.container,
self.image])
def test_make_unique_name(self):
"""Tests the make_unique_name() function."""
names = [make_unique_name(b'/some/prefix_') for i in irange(3)]
for n in names:
self.assertTrue(n and isinstance(n, bytes)
and n[:13] == b'/some/prefix_')
self.assertEqual(len(set(names)), len(names))
def docker_setup_build(args):
"""Builds the container from the Dockerfile
"""
target = Path(args.target[0])
unpacked_info = read_dict(target / '.reprounzip')
if 'initial_image' in unpacked_info:
logging.critical("Image already built")
sys.exit(1)
image = make_unique_name(b'reprounzip_image_')
logging.info("Calling 'docker build'...")
try:
retcode = subprocess.call(['docker', 'build', '-t', image, '.'],
cwd=target.path)
except OSError:
logging.critical("docker executable not found")
sys.exit(1)
else:
if retcode != 0:
logging.critical("docker build failed with code %d", retcode)
sys.exit(1)
logging.info("Initial image created: %s", image.decode('ascii'))
unpacked_info['initial_image'] = image
unpacked_info['current_image'] = image
write_dict(target / '.reprounzip', unpacked_info)
def docker_setup_build(args):
"""Builds the container from the Dockerfile
"""
target = Path(args.target[0])
unpacked_info = read_dict(target)
if 'initial_image' in unpacked_info:
logging.critical("Image already built")
sys.exit(1)
image = make_unique_name(b'reprounzip_image_')
logging.info("Calling 'docker build'...")
try:
retcode = subprocess.call(['docker', 'build', '-t'] +
args.docker_option + [image, '.'],
cwd=target.path)
except OSError:
logging.critical("docker executable not found")
sys.exit(1)
else:
if retcode != 0:
logging.critical("docker build failed with code %d", retcode)
sys.exit(1)
logging.info("Initial image created: %s", image.decode('ascii'))
unpacked_info['initial_image'] = image
unpacked_info['current_image'] = image
write_dict(target, unpacked_info)
def upload_file(self, local_path, input_path):
if self.use_chroot:
remote_path = join_root(PosixPath('/experimentroot'),
PosixPath(input_path))
else:
remote_path = input_path
# Upload to a temporary file first
logging.info("Uploading file via SCP...")
rtemp = PosixPath(make_unique_name(b'/tmp/reprozip_input_'))
self.client_scp.put(local_path.path, rtemp.path, recursive=False)
# Move it
logging.info("Moving file into place...")
chan = self.ssh.get_transport().open_session()
chown_cmd = '/bin/chown --reference=%s %s' % (shell_escape(
remote_path.path), shell_escape(rtemp.path))
chmod_cmd = '/bin/chmod --reference=%s %s' % (shell_escape(
remote_path.path), shell_escape(rtemp.path))
mv_cmd = '/bin/mv %s %s' % (shell_escape(
rtemp.path), shell_escape(remote_path.path))
chan.exec_command('/usr/bin/sudo /bin/sh -c %s' %
shell_escape(';'.join(
(chown_cmd, chmod_cmd, mv_cmd))))
if chan.recv_exit_status() != 0:
logging.critical("Couldn't move file in virtual machine")
sys.exit(1)
chan.close()
def download(self, remote_path, local_path):
if self.use_chroot:
remote_path = join_root(PosixPath('/experimentroot'), remote_path)
temp = make_unique_name(b'reprozip_output_')
rtemp = PosixPath('/vagrant') / temp
ltemp = self.target / temp
# Copy file to shared folder
logging.info("Copying file to shared folder...")
chan = self.ssh.get_transport().open_session()
cp_cmd = '/bin/cp %s %s' % (
shell_escape(remote_path.path),
shell_escape(rtemp.path))
chown_cmd = '/bin/chown vagrant %s' % shell_escape(rtemp.path)
chmod_cmd = '/bin/chmod 644 %s' % shell_escape(rtemp.path)
chan.exec_command('/usr/bin/sudo /bin/sh -c %s' % shell_escape(
' && '.join((cp_cmd, chown_cmd, chmod_cmd))))
if chan.recv_exit_status() != 0:
logging.critical("Couldn't copy file in virtual machine")
try:
ltemp.remove()
except OSError:
pass
return False
# Move file to final destination
try:
ltemp.rename(local_path)
except OSError as e:
logging.critical("Couldn't download output file: %s\n%s",
remote_path, str(e))
ltemp.remove()
return False
return True
def upload_file(self, local_path, input_path):
if self.use_chroot:
remote_path = join_root(PosixPath('/experimentroot'), input_path)
else:
remote_path = input_path
temp = make_unique_name(b'reprozip_input_')
ltemp = self.target / temp
rtemp = PosixPath('/vagrant') / temp
# Copy file to shared folder
logging.info("Copying file to shared folder...")
local_path.copyfile(ltemp)
# Move it
logging.info("Moving file into place...")
chan = self.ssh.get_transport().open_session()
chown_cmd = '/bin/chown --reference=%s %s' % (shell_escape(
remote_path.path), shell_escape(rtemp.path))
chmod_cmd = '/bin/chmod --reference=%s %s' % (shell_escape(
remote_path.path), shell_escape(rtemp.path))
mv_cmd = '/bin/mv %s %s' % (shell_escape(
rtemp.path), shell_escape(remote_path.path))
chan.exec_command('/usr/bin/sudo /bin/sh -c %s' %
shell_escape(' && '.join(
(chown_cmd, chmod_cmd, mv_cmd))))
if chan.recv_exit_status() != 0:
logging.critical("Couldn't move file in virtual machine")
try:
ltemp.remove()
except OSError:
pass
sys.exit(1)
chan.close()
def test_make_unique_name(self):
"""Tests the make_unique_name() function."""
names = [make_unique_name(b'/some/prefix_') for i in irange(3)]
for n in names:
self.assertTrue(n and isinstance(n, bytes) and
n[:13] == b'/some/prefix_')
self.assertEqual(len(set(names)), len(names))
def finalize(self):
if not self.docker_copy:
self.build_directory.rmtree()
return
from_image = self.unpacked_info['current_image']
with self.build_directory.open('w',
'Dockerfile',
encoding='utf-8',
newline='\n') as dockerfile:
dockerfile.write('FROM %s\n\n' % from_image.decode('ascii'))
for src, target in self.docker_copy:
# FIXME : spaces in filenames will probably break Docker
dockerfile.write(
'COPY \\\n %s \\\n %s\n' % (shell_escape(
unicode_(src)), shell_escape(unicode_(target))))
if self.docker_copy:
dockerfile.write('RUN /busybox chown 1000:1000 \\\n'
' %s\n' % ' \\\n '.join(
shell_escape(unicode_(target))
for src, target in self.docker_copy))
# TODO : restore permissions?
image = make_unique_name(b'reprounzip_image_')
retcode = subprocess.call(self.docker_cmd +
['build', '-t', image, '.'],
cwd=self.build_directory.path)
if retcode != 0:
logging.critical("docker build failed with code %d", retcode)
sys.exit(1)
else:
logging.info("New image created: %s", image.decode('ascii'))
if from_image != self.unpacked_info['initial_image']:
logging.info("Untagging previous image %s",
from_image.decode('ascii'))
retcode = subprocess.call(self.docker_cmd +
['rmi', from_image])
if retcode != 0:
logging.warning(
"Can't remove previous image, docker "
"returned %d", retcode)
self.unpacked_info['current_image'] = image
write_dict(self.target, self.unpacked_info)
self.build_directory.rmtree()
def docker_setup_build(args):
"""Builds the container from the Dockerfile
"""
target = Path(args.target[0])
unpacked_info = read_dict(target)
if 'initial_image' in unpacked_info:
logger.critical("Image already built")
sys.exit(1)
if args.image_name:
image = args.image_name[0]
if not isinstance(image, bytes):
image = image.encode('ascii')
else:
image = make_unique_name(b'reprounzip_image_')
logger.info("Calling 'docker build'...")
try:
retcode = subprocess.call(args.docker_cmd.split() + ['build', '-t'] +
args.docker_option + [image, '.'],
cwd=target.path)
except OSError:
logger.critical("docker executable not found")
sys.exit(1)
else:
if retcode != 0:
logger.critical("docker build failed with code %d", retcode)
sys.exit(1)
logger.info("Initial image created: %s", image.decode('ascii'))
unpacked_info['initial_image'] = image
unpacked_info['current_image'] = image
if 'DOCKER_MACHINE_NAME' in os.environ:
unpacked_info['docker_host'] = {
'type': 'docker-machine',
'name': os.environ['DOCKER_MACHINE_NAME']
}
elif 'DOCKER_HOST' in os.environ:
unpacked_info['docker_host'] = {
'type':
'custom',
'env':
dict((k, v) for k, v in iteritems(os.environ)
if k.startswith('DOCKER_'))
}
write_dict(target, unpacked_info)
def finalize(self):
if not self.docker_copy:
self.build_directory.rmtree()
return
from_image = self.unpacked_info['current_image']
with self.build_directory.open('w', 'Dockerfile',
encoding='utf-8',
newline='\n') as dockerfile:
dockerfile.write('FROM %s\n\n' % from_image.decode('ascii'))
for src, target in self.docker_copy:
# FIXME : spaces in filenames will probably break Docker
dockerfile.write(
'COPY \\\n %s \\\n %s\n' % (
shell_escape(unicode_(src)),
shell_escape(unicode_(target))))
if self.docker_copy:
dockerfile.write('RUN /busybox chown 1000:1000 \\\n'
' %s\n' % ' \\\n '.join(
shell_escape(unicode_(target))
for src, target in self.docker_copy))
# TODO : restore permissions?
image = make_unique_name(b'reprounzip_image_')
retcode = subprocess.call(self.docker_cmd +
['build', '-t', image, '.'],
cwd=self.build_directory.path)
if retcode != 0:
logger.critical("docker build failed with code %d", retcode)
sys.exit(1)
else:
logger.info("New image created: %s", image.decode('ascii'))
if from_image != self.unpacked_info['initial_image']:
logger.info("Untagging previous image %s",
from_image.decode('ascii'))
retcode = subprocess.call(self.docker_cmd +
['rmi', from_image])
if retcode != 0:
logger.warning("Can't remove previous image, docker "
"returned %d", retcode)
self.unpacked_info['current_image'] = image
write_dict(self.target, self.unpacked_info)
self.build_directory.rmtree()
def docker_setup_build(args):
"""Builds the container from the Dockerfile
"""
target = Path(args.target[0])
unpacked_info = read_dict(target)
if 'initial_image' in unpacked_info:
logger.critical("Image already built")
sys.exit(1)
if args.image_name:
image = args.image_name[0]
if not isinstance(image, bytes):
image = image.encode('ascii')
else:
image = make_unique_name(b'reprounzip_image_')
logger.info("Calling 'docker build'...")
try:
retcode = subprocess.call(args.docker_cmd.split() + ['build', '-t'] +
args.docker_option + [image, '.'],
cwd=target.path)
except OSError:
logger.critical("docker executable not found")
sys.exit(1)
else:
if retcode != 0:
logger.critical("docker build failed with code %d", retcode)
sys.exit(1)
logger.info("Initial image created: %s", image.decode('ascii'))
unpacked_info['initial_image'] = image
unpacked_info['current_image'] = image
if 'DOCKER_MACHINE_NAME' in os.environ:
unpacked_info['docker_host'] = {
'type': 'docker-machine',
'name': os.environ['DOCKER_MACHINE_NAME']}
elif 'DOCKER_HOST' in os.environ:
unpacked_info['docker_host'] = {
'type': 'custom',
'env': dict((k, v)
for k, v in iteritems(os.environ)
if k.startswith('DOCKER_'))}
write_dict(target, unpacked_info)
def upload_file(self, local_path, input_path):
if self.use_chroot:
remote_path = join_root(PosixPath('/experimentroot'),
input_path)
else:
remote_path = input_path
temp = make_unique_name(b'reprozip_input_')
ltemp = self.target / temp
rtemp = PosixPath('/vagrant') / temp
# Copy file to shared folder
logging.info("Copying file to shared folder...")
local_path.copyfile(ltemp)
# Move it
logging.info("Moving file into place...")
chan = self.ssh.get_transport().open_session()
chown_cmd = '/bin/chown --reference=%s %s' % (
shell_escape(remote_path.path),
shell_escape(rtemp.path))
chmod_cmd = '/bin/chmod --reference=%s %s' % (
shell_escape(remote_path.path),
shell_escape(rtemp.path))
mv_cmd = '/bin/mv %s %s' % (
shell_escape(rtemp.path),
shell_escape(remote_path.path))
chan.exec_command('/usr/bin/sudo /bin/sh -c %s' % shell_escape(
' && '.join((chown_cmd, chmod_cmd, mv_cmd))))
if chan.recv_exit_status() != 0:
logging.critical("Couldn't move file in virtual machine")
try:
ltemp.remove()
except OSError:
pass
sys.exit(1)
chan.close()
def docker_run(args):
"""Runs the experiment in the container.
"""
target = Path(args.target[0])
unpacked_info = read_dict(target)
cmdline = args.cmdline
# Sanity check
if args.detach and args.x11:
logging.critical("Error: Can't use X11 forwarding if you're detaching")
raise UsageError
# Loads config
config = load_config(target / 'config.yml', True)
runs = config.runs
selected_runs = get_runs(runs, args.run, cmdline)
# Get current image name
if 'current_image' in unpacked_info:
image = unpacked_info['current_image']
logging.debug("Running from image %s", image.decode('ascii'))
else:
logging.critical("Image doesn't exist yet, have you run setup/build?")
sys.exit(1)
# Name of new container
if args.detach:
container = make_unique_name(b'reprounzip_detached_')
else:
container = make_unique_name(b'reprounzip_run_')
hostname = runs[selected_runs[0]].get('hostname', 'reprounzip')
# X11 handler
if args.x11:
local_ip = get_local_addr()
docker_host = local_ip
if os.environ.get('DOCKER_HOST'):
m = _dockerhost_re.match(os.environ['DOCKER_HOST'])
if m is not None:
docker_host = m.group(1)
if args.tunneled_x11:
x11 = X11Handler(True, ('internet', docker_host), args.x11_display)
else:
x11 = X11Handler(True, ('internet', local_ip), args.x11_display)
if (docker_host != local_ip and docker_host != 'localhost' and
not docker_host.startswith('127.') and
not docker_host.startswith('192.168.99.')):
ssh_cmdline = ' '.join(
'-R*:%(p)d:127.0.0.1:%(p)d' % {'p': port}
for port, connector in x11.port_forward)
logging.warning(
"You requested X11 forwarding but the Docker container "
"appears to be running remotely. It is probable that it "
"won't be able to connect to the local display. Creating "
"a remote SSH tunnel and running with --tunneled-x11 "
"might help (%s).",
ssh_cmdline)
else:
x11 = X11Handler(False, ('local', hostname), args.x11_display)
cmds = []
for run_number in selected_runs:
run = runs[run_number]
cmd = 'cd %s && ' % shell_escape(run['workingdir'])
cmd += '/busybox env -i '
environ = x11.fix_env(run['environ'])
environ = fixup_environment(environ, args)
cmd += ' '.join('%s=%s' % (shell_escape(k), shell_escape(v))
for k, v in iteritems(environ))
cmd += ' '
# FIXME : Use exec -a or something if binary != argv[0]
if cmdline is None:
argv = [run['binary']] + run['argv'][1:]
else:
argv = cmdline
cmd += ' '.join(shell_escape(a) for a in argv)
uid = run.get('uid', 1000)
gid = run.get('gid', 1000)
cmd = '/rpzsudo \'#%d\' \'#%d\' /busybox sh -c %s' % (
uid, gid,
shell_escape(cmd))
cmds.append(cmd)
cmds = x11.init_cmds + cmds
cmds = ' && '.join(cmds)
signals.pre_run(target=target)
# Creates forwarders
forwarders = []
for port, connector in x11.port_forward:
forwarders.append(LocalForwarder(connector, port))
if args.detach:
logging.info("Start container %s (detached)",
container.decode('ascii'))
retcode = interruptible_call(['docker', 'run', b'--name=' + container,
'-h', hostname,
'-d', '-t'] +
args.docker_option +
[image, '/busybox', 'sh', '-c', cmds])
if retcode != 0:
logging.critical("docker run failed with code %d", retcode)
subprocess.call(['docker', 'rm', '-f', container])
sys.exit(1)
return
# Run command in container
logging.info("Starting container %s", container.decode('ascii'))
retcode = interruptible_call(['docker', 'run', b'--name=' + container,
'-h', hostname,
'-i', '-t'] +
args.docker_option +
[image, '/busybox', 'sh', '-c', cmds])
if retcode != 0:
logging.critical("docker run failed with code %d", retcode)
subprocess.call(['docker', 'rm', '-f', container])
sys.exit(1)
# Get exit status from "docker inspect"
out = subprocess.check_output(['docker', 'inspect', container])
outjson = json.loads(out.decode('ascii'))
if (outjson[0]["State"]["Running"] is not False or
outjson[0]["State"]["Paused"] is not False):
logging.error("Invalid container state after execution:\n%s",
json.dumps(outjson[0]["State"]))
retcode = outjson[0]["State"]["ExitCode"]
stderr.write("\n*** Command finished, status: %d\n" % retcode)
# Commit to create new image
new_image = make_unique_name(b'reprounzip_image_')
logging.info("Committing container %s to image %s",
container.decode('ascii'), new_image.decode('ascii'))
subprocess.check_call(['docker', 'commit', container, new_image])
# Update image name
unpacked_info['current_image'] = new_image
write_dict(target, unpacked_info)
# Remove the container
logging.info("Destroying container %s", container.decode('ascii'))
retcode = subprocess.call(['docker', 'rm', container])
if retcode != 0:
logging.error("Error deleting container %s", container.decode('ascii'))
# Untag previous image, unless it is the initial_image
if image != unpacked_info['initial_image']:
logging.info("Untagging previous image %s", image.decode('ascii'))
subprocess.check_call(['docker', 'rmi', image])
# Update input file status
metadata_update_run(config, unpacked_info, selected_runs)
write_dict(target, unpacked_info)
signals.post_run(target=target, retcode=retcode)
def docker_run(args):
"""Runs the experiment in the container.
"""
target = Path(args.target[0])
unpacked_info = read_dict(target)
cmdline = args.cmdline
# Sanity check
if args.detach and args.x11:
logging.critical("Error: Can't use X11 forwarding if you're detaching")
raise UsageError
# Loads config
config = load_config(target / 'config.yml', True)
runs = config.runs
selected_runs = get_runs(runs, args.run, cmdline)
# Get current image name
if 'current_image' in unpacked_info:
image = unpacked_info['current_image']
logging.debug("Running from image %s", image.decode('ascii'))
else:
logging.critical("Image doesn't exist yet, have you run setup/build?")
sys.exit(1)
# Name of new container
if args.detach:
container = make_unique_name(b'reprounzip_detached_')
else:
container = make_unique_name(b'reprounzip_run_')
hostname = runs[selected_runs[0]].get('hostname', 'reprounzip')
# Port forwarding
port_options = []
for port_host, port_container, proto in parse_ports(args.expose_port):
port_options.extend(['-p',
'%s:%s%s' % (port_host, port_container, proto)])
# X11 handler
if args.x11:
local_ip = get_local_addr()
docker_host = local_ip
if os.environ.get('DOCKER_HOST'):
m = _dockerhost_re.match(os.environ['DOCKER_HOST'])
if m is not None:
docker_host = m.group(1)
if args.tunneled_x11:
x11 = X11Handler(True, ('internet', docker_host), args.x11_display)
else:
x11 = X11Handler(True, ('internet', local_ip), args.x11_display)
if (docker_host != local_ip and docker_host != 'localhost' and
not docker_host.startswith('127.') and
not docker_host.startswith('192.168.99.')):
ssh_cmdline = ' '.join(
'-R*:%(p)d:127.0.0.1:%(p)d' % {'p': port}
for port, connector in x11.port_forward)
logging.warning(
"You requested X11 forwarding but the Docker container "
"appears to be running remotely. It is probable that it "
"won't be able to connect to the local display. Creating "
"a remote SSH tunnel and running with --tunneled-x11 "
"might help (%s).",
ssh_cmdline)
else:
x11 = X11Handler(False, ('local', hostname), args.x11_display)
cmd = []
for run_number in selected_runs:
run = runs[run_number]
env_set, env_unset = x11.env_fixes(run['environ'])
a_env_set, a_env_unset = parse_environment_args(args)
env_set.update(a_env_set)
env_unset.extend(a_env_unset)
if env_set or env_unset:
cmd.append('env')
env = []
for k in env_unset:
env.append('-u')
env.append(shell_escape(k))
for k, v in iteritems(env_set):
env.append('%s=%s' % (shell_escape(k), shell_escape(v)))
cmd.append(' '.join(env))
# FIXME : Use exec -a or something if binary != argv[0]
if cmdline is not None:
cmd.append('cmd')
cmd.append(' '.join(shell_escape(a) for a in cmdline))
cmd.append('run')
cmd.append('%d' % run_number)
cmd = list(chain.from_iterable([['do', shell_escape(c)]
for c in x11.init_cmds] +
[cmd]))
if logging.getLogger().isEnabledFor(logging.DEBUG):
logging.debug("Passing arguments to Docker image:")
for c in cmd:
logging.debug(c)
signals.pre_run(target=target)
# Creates forwarders
forwarders = []
for port, connector in x11.port_forward:
forwarders.append(LocalForwarder(connector, port))
if args.detach:
logging.info("Start container %s (detached)",
container.decode('ascii'))
retcode = interruptible_call(args.docker_cmd.split() +
['run', b'--name=' + container,
'-h', hostname,
'-d', '-t'] +
port_options +
args.docker_option +
[image] + cmd)
if retcode != 0:
logging.critical("docker run failed with code %d", retcode)
subprocess.call(['docker', 'rm', '-f', container])
sys.exit(1)
return
# Run command in container
logging.info("Starting container %s", container.decode('ascii'))
retcode = interruptible_call(args.docker_cmd.split() +
['run', b'--name=' + container,
'-h', hostname,
'-i', '-t'] +
port_options +
args.docker_option +
[image] + cmd)
if retcode != 0:
logging.critical("docker run failed with code %d", retcode)
subprocess.call(['docker', 'rm', '-f', container])
sys.exit(1)
# Get exit status from "docker inspect"
out = subprocess.check_output(args.docker_cmd.split() +
['inspect', container])
outjson = json.loads(out.decode('ascii'))
if (outjson[0]["State"]["Running"] is not False or
outjson[0]["State"]["Paused"] is not False):
logging.error("Invalid container state after execution:\n%s",
json.dumps(outjson[0]["State"]))
retcode = outjson[0]["State"]["ExitCode"]
stderr.write("\n*** Command finished, status: %d\n" % retcode)
# Commit to create new image
new_image = make_unique_name(b'reprounzip_image_')
logging.info("Committing container %s to image %s",
container.decode('ascii'), new_image.decode('ascii'))
subprocess.check_call(args.docker_cmd.split() +
['commit', container, new_image])
# Update image name
unpacked_info['current_image'] = new_image
write_dict(target, unpacked_info)
# Remove the container
logging.info("Destroying container %s", container.decode('ascii'))
retcode = subprocess.call(args.docker_cmd.split() + ['rm', container])
if retcode != 0:
logging.error("Error deleting container %s", container.decode('ascii'))
# Untag previous image, unless it is the initial_image
if image != unpacked_info['initial_image']:
logging.info("Untagging previous image %s", image.decode('ascii'))
subprocess.check_call(args.docker_cmd.split() + ['rmi', image])
# Update input file status
metadata_update_run(config, unpacked_info, selected_runs)
write_dict(target, unpacked_info)
signals.post_run(target=target, retcode=retcode)
def docker_run(args):
"""Runs the experiment in the container.
"""
target = Path(args.target[0])
unpacked_info = read_dict(target / '.reprounzip')
cmdline = args.cmdline
# Loads config
runs, packages, other_files = load_config(target / 'config.yml', True)
selected_runs = get_runs(runs, args.run, cmdline)
# Destroy previous container
if 'ran_container' in unpacked_info:
container = unpacked_info.pop('ran_container')
logging.info("Destroying previous container %s",
container.decode('ascii'))
retcode = subprocess.call(['docker', 'rm', '-f', container])
if retcode != 0:
logging.error("Error deleting previous container %s",
container.decode('ascii'))
write_dict(target / '.reprounzip', unpacked_info)
# Use the initial image directly
if 'current_image' in unpacked_info:
image = unpacked_info['current_image']
logging.debug("Running from image %s", image.decode('ascii'))
else:
logging.critical("Image doesn't exist yet, have you run setup/build?")
sys.exit(1)
# Name of new container
container = make_unique_name(b'reprounzip_run_')
hostname = runs[selected_runs[0]].get('hostname', 'reprounzip')
# Get the local bridge IP
ip_str = get_iface_addr('docker0')
# X11 handler
x11 = X11Handler(args.x11, ('internet', ip_str), args.x11_display)
cmds = []
for run_number in selected_runs:
run = runs[run_number]
cmd = 'cd %s && ' % shell_escape(run['workingdir'])
cmd += '/usr/bin/env -i '
environ = x11.fix_env(run['environ'])
cmd += ' '.join('%s=%s' % (k, shell_escape(v))
for k, v in iteritems(environ))
cmd += ' '
# FIXME : Use exec -a or something if binary != argv[0]
if cmdline is None:
argv = [run['binary']] + run['argv'][1:]
else:
argv = cmdline
cmd += ' '.join(shell_escape(a) for a in argv)
uid = run.get('uid', 1000)
cmd = 'sudo -u \'#%d\' /bin/busybox sh -c %s\n' % (uid,
shell_escape(cmd))
cmds.append(cmd)
cmds = x11.init_cmds + cmds
cmds = ' && '.join(cmds)
signals.pre_run(target=target)
# Creates forwarders
forwarders = []
for port, connector in x11.port_forward:
forwarders.append(
LocalForwarder(connector, port))
# Run command in container
logging.info("Starting container %s", container.decode('ascii'))
retcode = interruptible_call(['docker', 'run', b'--name=' + container,
'-h', hostname,
'-i', '-t', image,
'/bin/busybox', 'sh', '-c', cmds])
if retcode != 0:
logging.critical("docker run failed with code %d", retcode)
sys.exit(1)
# Get exit status from "docker inspect"
out = subprocess.check_output(['docker', 'inspect', container])
outjson = json.loads(out.decode('ascii'))
if (outjson[0]["State"]["Running"] is not False or
outjson[0]["State"]["Paused"] is not False):
logging.error("Invalid container state after execution:\n%s",
json.dumps(outjson[0]["State"]))
retcode = outjson[0]["State"]["ExitCode"]
sys.stderr.write("\n*** Command finished, status: %d\n" % retcode)
# Store container name (so we can download output files)
unpacked_info['ran_container'] = container
write_dict(target / '.reprounzip', unpacked_info)
signals.post_run(target=target, retcode=retcode)
def finalize(self):
if not self.docker_copy:
self.build_directory.rmtree()
return
from_image = self.unpacked_info['current_image']
with self.build_directory.open('w',
'Dockerfile',
encoding='utf-8',
newline='\n') as dockerfile:
dockerfile.write('FROM %s\n\n' % from_image.decode('ascii'))
for src, target in self.docker_copy:
# FIXME : spaces in filenames will probably break Docker
dockerfile.write(
'COPY \\\n %s \\\n %s\n' %
(shell_escape(str(src)), shell_escape(str(target))))
for src, target in self.docker_copy:
uid = gid = None
# Keep permissions if the file is already in there
tar = tarfile.open(str(self.target / 'data.tgz'), 'r:*')
try:
info = tar.getmember(
str(join_root(PosixPath(b'DATA'), target)))
uid, gid = info.uid, info.gid
except KeyError:
pass
# Otherwise default on the first run's UID/GID
if uid is None:
uid, gid = self.default_ownership
# Lastly, use 1000
if uid is None:
uid = gid = 1000
dockerfile.write('RUN ["/busybox", "chown", "%d:%d", %s]\n' % (
uid,
gid,
json.dumps(str(target)),
))
image = make_unique_name(b'reprounzip_image_')
retcode = subprocess.call(self.docker_cmd +
['build', '-t', image, '.'],
cwd=self.build_directory.path)
if retcode != 0:
logger.critical("docker build failed with code %d", retcode)
sys.exit(1)
else:
logger.info("New image created: %s", image.decode('ascii'))
if from_image != self.unpacked_info['initial_image']:
logger.info("Untagging previous image %s",
from_image.decode('ascii'))
retcode = subprocess.call(self.docker_cmd +
['rmi', from_image])
if retcode != 0:
logger.warning(
"Can't remove previous image, docker "
"returned %d", retcode)
self.unpacked_info['current_image'] = image
write_dict(self.target, self.unpacked_info)
self.build_directory.rmtree()
def docker_run(args):
"""Runs the experiment in the container.
"""
target = Path(args.target[0])
unpacked_info = read_dict(target)
cmdline = args.cmdline
# Sanity check
if args.detach and args.x11:
logger.critical("Error: Can't use X11 forwarding if you're detaching")
raise UsageError
# Loads config
config = load_config(target / 'config.yml', True)
runs = config.runs
selected_runs = get_runs(runs, args.run, cmdline)
# Get current image name
if 'current_image' in unpacked_info:
image = unpacked_info['current_image']
logger.debug("Running from image %s", image.decode('ascii'))
else:
logger.critical("Image doesn't exist yet, have you run setup/build?")
sys.exit(1)
# Name of new container
if args.detach:
container = make_unique_name(b'reprounzip_detached_')
else:
container = make_unique_name(b'reprounzip_run_')
hostname = runs[selected_runs[0]].get('hostname', 'reprounzip')
# Port forwarding
port_options = []
for port_host, port_container, proto in parse_ports(args.expose_port):
port_options.extend(['-p',
'%s:%s/%s' % (port_host, port_container, proto)])
# X11 handler
if args.x11:
local_ip = get_local_addr()
docker_host = local_ip
if os.environ.get('DOCKER_HOST'):
m = _dockerhost_re.match(os.environ['DOCKER_HOST'])
if m is not None:
docker_host = m.group(1)
if args.tunneled_x11:
x11 = X11Handler(True, ('internet', docker_host), args.x11_display)
else:
x11 = X11Handler(True, ('internet', local_ip), args.x11_display)
if (docker_host != local_ip and docker_host != 'localhost' and
not docker_host.startswith('127.') and
not docker_host.startswith('192.168.99.')):
ssh_cmdline = ' '.join(
'-R*:%(p)d:127.0.0.1:%(p)d' % {'p': port}
for port, connector in x11.port_forward)
logger.warning(
"You requested X11 forwarding but the Docker container "
"appears to be running remotely. It is probable that it "
"won't be able to connect to the local display. Creating "
"a remote SSH tunnel and running with --tunneled-x11 "
"might help (%s).",
ssh_cmdline)
else:
x11 = X11Handler(False, ('local', hostname), args.x11_display)
cmd = []
for run_number in selected_runs:
run = runs[run_number]
env_set, env_unset = x11.env_fixes(run['environ'])
a_env_set, a_env_unset = parse_environment_args(args)
env_set.update(a_env_set)
env_unset.extend(a_env_unset)
if env_set or env_unset:
cmd.append('env')
env = []
for k in env_unset:
env.append('-u')
env.append(shell_escape(k))
for k, v in iteritems(env_set):
env.append('%s=%s' % (shell_escape(k), shell_escape(v)))
cmd.append(' '.join(env))
# FIXME : Use exec -a or something if binary != argv[0]
if cmdline is not None:
cmd.append('cmd')
cmd.append(' '.join(shell_escape(a) for a in cmdline))
cmd.append('run')
cmd.append('%d' % run_number)
cmd = list(chain.from_iterable([['do', shell_escape(c)]
for c in x11.init_cmds] +
[cmd]))
if logger.isEnabledFor(logging.DEBUG):
logger.debug("Passing arguments to Docker image:")
for c in cmd:
logger.debug(c)
signals.pre_run(target=target)
# Creates forwarders
forwarders = []
for port, connector in x11.port_forward:
forwarders.append(LocalForwarder(connector, port))
if args.detach:
logger.info("Start container %s (detached)",
container.decode('ascii'))
retcode = interruptible_call(args.docker_cmd.split() +
['run', b'--name=' + container,
'-h', hostname,
'-d', '-t'] +
port_options +
args.docker_option +
[image] + cmd)
if retcode != 0:
logger.critical("docker run failed with code %d", retcode)
subprocess.call(['docker', 'rm', '-f', container])
sys.exit(1)
return
# Run command in container
logger.info("Starting container %s", container.decode('ascii'))
retcode = interruptible_call(args.docker_cmd.split() +
['run', b'--name=' + container,
'-h', hostname,
'-i', '-t'] +
port_options +
args.docker_option +
[image] + cmd,
request_tty=True)
# The image prints out the exit status(es) itself
if retcode != 0:
logger.critical("docker run failed with code %d", retcode)
subprocess.call(['docker', 'rm', '-f', container])
sys.exit(1)
# Commit to create new image
new_image = make_unique_name(b'reprounzip_image_')
logger.info("Committing container %s to image %s",
container.decode('ascii'), new_image.decode('ascii'))
subprocess.check_call(args.docker_cmd.split() +
['commit', container, new_image])
# Update image name
unpacked_info['current_image'] = new_image
write_dict(target, unpacked_info)
# Remove the container
logger.info("Destroying container %s", container.decode('ascii'))
retcode = subprocess.call(args.docker_cmd.split() + ['rm', container])
if retcode != 0:
logger.error("Error deleting container %s", container.decode('ascii'))
# Untag previous image, unless it is the initial_image
if image != unpacked_info['initial_image']:
logger.info("Untagging previous image %s", image.decode('ascii'))
subprocess.check_call(args.docker_cmd.split() + ['rmi', image])
# Update input file status
metadata_update_run(config, unpacked_info, selected_runs)
write_dict(target, unpacked_info)
signals.post_run(target=target, retcode=retcode)
def docker_run(args):
"""Runs the experiment in the container.
"""
target = Path(args.target[0])
unpacked_info = read_dict(target)
cmdline = args.cmdline
# Sanity check
if args.detach and args.x11:
logging.critical("Error: Can't use X11 forwarding if you're detaching")
raise UsageError
# Loads config
config = load_config(target / 'config.yml', True)
runs = config.runs
selected_runs = get_runs(runs, args.run, cmdline)
# Get current image name
if 'current_image' in unpacked_info:
image = unpacked_info['current_image']
logging.debug("Running from image %s", image.decode('ascii'))
else:
logging.critical("Image doesn't exist yet, have you run setup/build?")
sys.exit(1)
# Name of new container
if args.detach:
container = make_unique_name(b'reprounzip_detached_')
else:
container = make_unique_name(b'reprounzip_run_')
hostname = runs[selected_runs[0]].get('hostname', 'reprounzip')
# X11 handler
if args.x11:
local_ip = get_local_addr()
docker_host = local_ip
if os.environ.get('DOCKER_HOST'):
m = _dockerhost_re.match(os.environ['DOCKER_HOST'])
if m is not None:
docker_host = m.group(1)
if args.tunneled_x11:
x11 = X11Handler(True, ('internet', docker_host), args.x11_display)
else:
x11 = X11Handler(True, ('internet', local_ip), args.x11_display)
if (docker_host != local_ip and docker_host != 'localhost'
and not docker_host.startswith('127.')
and not docker_host.startswith('192.168.99.')):
ssh_cmdline = ' '.join('-R*:%(p)d:127.0.0.1:%(p)d' %
{'p': port}
for port, connector in x11.port_forward)
logging.warning(
"You requested X11 forwarding but the Docker container "
"appears to be running remotely. It is probable that it "
"won't be able to connect to the local display. Creating "
"a remote SSH tunnel and running with --tunneled-x11 "
"might help (%s).", ssh_cmdline)
else:
x11 = X11Handler(False, ('local', hostname), args.x11_display)
cmds = []
for run_number in selected_runs:
run = runs[run_number]
cmd = 'cd %s && ' % shell_escape(run['workingdir'])
cmd += '/busybox env -i '
environ = x11.fix_env(run['environ'])
environ = fixup_environment(environ, args)
cmd += ' '.join('%s=%s' % (shell_escape(k), shell_escape(v))
for k, v in iteritems(environ))
cmd += ' '
# FIXME : Use exec -a or something if binary != argv[0]
if cmdline is None:
argv = [run['binary']] + run['argv'][1:]
else:
argv = cmdline
cmd += ' '.join(shell_escape(a) for a in argv)
uid = run.get('uid', 1000)
gid = run.get('gid', 1000)
cmd = '/rpzsudo \'#%d\' \'#%d\' /busybox sh -c %s' % (
uid, gid, shell_escape(cmd))
cmds.append(cmd)
cmds = x11.init_cmds + cmds
cmds = ' && '.join(cmds)
signals.pre_run(target=target)
# Creates forwarders
forwarders = []
for port, connector in x11.port_forward:
forwarders.append(LocalForwarder(connector, port))
if args.detach:
logging.info("Start container %s (detached)",
container.decode('ascii'))
retcode = interruptible_call([
'docker', 'run', b'--name=' + container, '-h', hostname, '-d', '-t'
] + args.docker_option + [image, '/busybox', 'sh', '-c', cmds])
if retcode != 0:
logging.critical("docker run failed with code %d", retcode)
subprocess.call(['docker', 'rm', '-f', container])
sys.exit(1)
return
# Run command in container
logging.info("Starting container %s", container.decode('ascii'))
retcode = interruptible_call(
['docker', 'run', b'--name=' + container, '-h', hostname, '-i', '-t'] +
args.docker_option + [image, '/busybox', 'sh', '-c', cmds])
if retcode != 0:
logging.critical("docker run failed with code %d", retcode)
subprocess.call(['docker', 'rm', '-f', container])
sys.exit(1)
# Get exit status from "docker inspect"
out = subprocess.check_output(['docker', 'inspect', container])
outjson = json.loads(out.decode('ascii'))
if (outjson[0]["State"]["Running"] is not False
or outjson[0]["State"]["Paused"] is not False):
logging.error("Invalid container state after execution:\n%s",
json.dumps(outjson[0]["State"]))
retcode = outjson[0]["State"]["ExitCode"]
stderr.write("\n*** Command finished, status: %d\n" % retcode)
# Commit to create new image
new_image = make_unique_name(b'reprounzip_image_')
logging.info("Committing container %s to image %s",
container.decode('ascii'), new_image.decode('ascii'))
subprocess.check_call(['docker', 'commit', container, new_image])
# Update image name
unpacked_info['current_image'] = new_image
write_dict(target, unpacked_info)
# Remove the container
logging.info("Destroying container %s", container.decode('ascii'))
retcode = subprocess.call(['docker', 'rm', container])
if retcode != 0:
logging.error("Error deleting container %s", container.decode('ascii'))
# Untag previous image, unless it is the initial_image
if image != unpacked_info['initial_image']:
logging.info("Untagging previous image %s", image.decode('ascii'))
subprocess.check_call(['docker', 'rmi', image])
# Update input file status
metadata_update_run(config, unpacked_info, selected_runs)
write_dict(target, unpacked_info)
signals.post_run(target=target, retcode=retcode)
def docker_run(args):
"""Runs the experiment in the container.
"""
target = Path(args.target[0])
unpacked_info = read_dict(target / '.reprounzip')
cmdline = args.cmdline
# Loads config
runs, packages, other_files = load_config(target / 'config.yml', True)
selected_runs = get_runs(runs, args.run, cmdline)
# Destroy previous container
if 'ran_container' in unpacked_info:
container = unpacked_info.pop('ran_container')
logging.info("Destroying previous container %s",
container.decode('ascii'))
retcode = subprocess.call(['docker', 'rm', '-f', container])
if retcode != 0:
logging.error("Error deleting previous container %s",
container.decode('ascii'))
write_dict(target / '.reprounzip', unpacked_info)
# Use the initial image directly
if 'current_image' in unpacked_info:
image = unpacked_info['current_image']
logging.debug("Running from image %s", image.decode('ascii'))
else:
logging.critical("Image doesn't exist yet, have you run setup/build?")
sys.exit(1)
# Name of new container
container = make_unique_name(b'reprounzip_run_')
cmds = []
for run_number in selected_runs:
run = runs[run_number]
cmd = 'cd %s && ' % shell_escape(run['workingdir'])
cmd += '/usr/bin/env -i '
cmd += ' '.join('%s=%s' % (k, shell_escape(v))
for k, v in iteritems(run['environ']))
cmd += ' '
# FIXME : Use exec -a or something if binary != argv[0]
if cmdline is None:
argv = [run['binary']] + run['argv'][1:]
else:
argv = cmdline
cmd += ' '.join(shell_escape(a) for a in argv)
uid = run.get('uid', 1000)
cmd = 'sudo -u \'#%d\' sh -c %s\n' % (uid, shell_escape(cmd))
cmds.append(cmd)
cmds = ' && '.join(cmds)
signals.pre_run(target=target)
# Run command in container
logging.info("Starting container %s", container.decode('ascii'))
retcode = subprocess.call(['docker', 'run', b'--name=' + container,
'-i', '-t', image,
'/bin/sh', '-c', cmds])
sys.stderr.write("\n*** Command finished, status: %d\n" % retcode)
# Store container name (so we can download output files)
unpacked_info['ran_container'] = container
write_dict(target / '.reprounzip', unpacked_info)
signals.post_run(target=target, retcode=retcode)
