def chroot_run(args): """Runs the command in the chroot. """ target = Path(args.target[0]) unpacked_info = metadata_read(target, 'chroot') cmdline = args.cmdline # Loads config config = load_config_file(target / 'config.yml', True) runs = config.runs selected_runs = get_runs(runs, args.run, cmdline) root = target / 'root' # X11 handler x11 = X11Handler(args.x11, ('local', socket.gethostname()), args.x11_display) cmds = [] for run_number in selected_runs: run = runs[run_number] cmd = 'cd %s && ' % shell_escape(run['workingdir']) cmd += '/usr/bin/env -i ' environ = x11.fix_env(run['environ']) environ = fixup_environment(environ, args) cmd += ' '.join('%s=%s' % (shell_escape(k), shell_escape(v)) for k, v in iteritems(environ)) cmd += ' ' # FIXME : Use exec -a or something if binary != argv[0] if cmdline is None: argv = [run['binary']] + run['argv'][1:] else: argv = cmdline cmd += ' '.join(shell_escape(a) for a in argv) userspec = '%s:%s' % (run.get('uid', 1000), run.get('gid', 1000)) cmd = 'chroot --userspec=%s %s /bin/sh -c %s' % ( userspec, shell_escape(unicode_(root)), shell_escape(cmd)) cmds.append(cmd) cmds = [ 'chroot %s /bin/sh -c %s' % (shell_escape(unicode_(root)), shell_escape(c)) for c in x11.init_cmds ] + cmds cmds = ' && '.join(cmds) # Starts forwarding forwarders = [] for portnum, connector in x11.port_forward: fwd = LocalForwarder(connector, portnum) forwarders.append(fwd) signals.pre_run(target=target) retcode = interruptible_call(cmds, shell=True) stderr.write("\n*** Command finished, status: %d\n" % retcode) signals.post_run(target=target, retcode=retcode) # Update input file status metadata_update_run(config, unpacked_info, selected_runs) metadata_write(target, unpacked_info, 'chroot')
def chroot_run(args): """Runs the command in the chroot. """ target = Path(args.target[0]) read_dict(target / '.reprounzip', 'chroot') cmdline = args.cmdline # Loads config runs, packages, other_files = load_config_file(target / 'config.yml', True) selected_runs = get_runs(runs, args.run, cmdline) root = target / 'root' # X11 handler x11 = X11Handler(args.x11, ('local', socket.gethostname()), args.x11_display) cmds = [] for run_number in selected_runs: run = runs[run_number] cmd = 'cd %s && ' % shell_escape(run['workingdir']) cmd += '/usr/bin/env -i ' environ = x11.fix_env(run['environ']) cmd += ' '.join('%s=%s' % (k, shell_escape(v)) for k, v in iteritems(environ)) cmd += ' ' # FIXME : Use exec -a or something if binary != argv[0] if cmdline is None: argv = [run['binary']] + run['argv'][1:] else: argv = cmdline cmd += ' '.join(shell_escape(a) for a in argv) userspec = '%s:%s' % (run.get('uid', 1000), run.get('gid', 1000)) cmd = 'chroot --userspec=%s %s /bin/sh -c %s' % ( userspec, shell_escape(unicode_(root)), shell_escape(cmd)) cmds.append(cmd) cmds = ['chroot %s /bin/sh -c %s' % (shell_escape(unicode_(root)), shell_escape(c)) for c in x11.init_cmds] + cmds cmds = ' && '.join(cmds) # Starts forwarding forwarders = [] for portnum, connector in x11.port_forward: fwd = LocalForwarder(connector, portnum) forwarders.append(fwd) signals.pre_run(target=target) retcode = interruptible_call(cmds, shell=True) sys.stderr.write("\n*** Command finished, status: %d\n" % retcode) signals.post_run(target=target, retcode=retcode)
def directory_run(args): """Runs the command in the directory. """ target = Path(args.target[0]) unpacked_info = metadata_read(target, 'directory') cmdline = args.cmdline # Loads config config = load_config_file(target / 'config.yml', True) runs = config.runs selected_runs = get_runs(runs, args.run, cmdline) root = (target / 'root').absolute() # Gets library paths lib_dirs = [] logger.debug("Running: %s", "/sbin/ldconfig -v -N") p = subprocess.Popen(['/sbin/ldconfig', '-v', '-N'], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, _ = p.communicate() try: for line in stdout.splitlines(): if len(line) < 2 or line[0] in (b' ', b'\t'): continue if line.endswith(b':'): lib_dirs.append(Path(line[:-1])) finally: if p.returncode != 0: raise subprocess.CalledProcessError(p.returncode, ['/sbin/ldconfig', '-v', '-N']) lib_dirs = ('export LD_LIBRARY_PATH=%s' % ':'.join( shell_escape(str(join_root(root, d))) for d in lib_dirs)) cmds = [lib_dirs] for run_number in selected_runs: run = runs[run_number] cmd = 'cd %s && ' % shell_escape( str(join_root(root, Path(run['workingdir'])))) cmd += '/usr/bin/env -i ' environ = run['environ'] environ = fixup_environment(environ, args) if args.x11: if 'DISPLAY' in os.environ: environ['DISPLAY'] = os.environ['DISPLAY'] if 'XAUTHORITY' in os.environ: environ['XAUTHORITY'] = os.environ['XAUTHORITY'] cmd += ' '.join('%s=%s' % (shell_escape(k), shell_escape(v)) for k, v in environ.items() if k != 'PATH') cmd += ' ' # PATH # Get the original PATH components path = [PosixPath(d) for d in run['environ'].get('PATH', '').split(':')] # The same paths but in the directory dir_path = [join_root(root, d) for d in path if d.root == '/'] # Rebuild string path = ':'.join(str(d) for d in dir_path + path) cmd += 'PATH=%s ' % shell_escape(path) # FIXME : Use exec -a or something if binary != argv[0] if cmdline is None: argv = run['argv'] # Rewrites command-line arguments that are absolute filenames rewritten = False for i in range(len(argv)): try: p = Path(argv[i]) except UnicodeEncodeError: continue if p.is_absolute: rp = join_root(root, p) if (rp.exists() or (len(rp.components) > 3 and rp.parent.exists())): argv[i] = str(rp) rewritten = True if rewritten: logger.warning("Rewrote command-line as: %s", ' '.join(shell_escape(a) for a in argv)) else: argv = cmdline cmd += ' '.join(shell_escape(a) for a in argv) cmds.append(cmd) cmds = ' && '.join(cmds) signals.pre_run(target=target) logger.debug("Running: %s", cmds) retcode = interruptible_call(cmds, shell=True) print("\n*** Command finished, status: %d\n" % retcode, file=sys.stderr) signals.post_run(target=target, retcode=retcode) # Update input file status metadata_update_run(config, unpacked_info, selected_runs) metadata_write(target, unpacked_info, 'directory')
def run_interactive(ssh_info, interactive, cmd, request_pty, forwarded_ports): """Runs a command on an SSH server. If `interactive` is True, we'll try to find an ``ssh`` executable, falling back to paramiko if it's not found. The terminal handling code is a bit wonky, so using ``ssh`` is definitely a good idea, especially on Windows. Non-interactive commands should run fine. :param ssh_info: dict with `hostname`, `port`, `username`, `key_filename`, passed directly to paramiko :type ssh_info: dict :param interactive: whether to connect local input to the remote process :type interactive: bool :param cmd: command-line to run on the server :type cmd: basestring :param request_pty: whether to request a PTY from the SSH server :type request_pty: bool :param forwarded_ports: ports to forward back to us; iterable of pairs ``(port_number, connector)`` where `port_number` is the remote port number and `connector` is the connector object used to build the connected socket to forward to on this side """ if interactive: ssh_exe = find_ssh_executable() else: ssh_exe = None if interactive and ssh_exe: record_usage(vagrant_ssh='ssh') args = [ ssh_exe, '-t' if request_pty else '-T', # Force allocation of PTY '-o', 'StrictHostKeyChecking=no', # Silently accept host keys '-o', 'UserKnownHostsFile=/dev/null', # Don't store host keys '-i', ssh_info['key_filename'], '-p', '%d' % ssh_info['port'] ] for remote_port, connector in forwarded_ports: # Remote port will connect to a local port fwd = LocalForwarder(connector) args.append('-R%d:127.0.0.1:%d' % (remote_port, fwd.local_port)) args.append('%s@%s' % (ssh_info['username'], ssh_info['hostname'])) args.append(cmd) return interruptible_call(args) else: record_usage(vagrant_ssh='interactive' if interactive else 'simple') # Connects to the machine ssh = paramiko.SSHClient() ssh.set_missing_host_key_policy(IgnoreMissingKey()) ssh.connect(**ssh_info) # Starts forwarding forwarders = [] for remote_port, connector in forwarded_ports: forwarders.append( SSHForwarder(ssh.get_transport(), remote_port, connector)) chan = ssh.get_transport().open_session() if request_pty: chan.get_pty() # Execute command logging.info("Connected via SSH, running command...") chan.exec_command(cmd) # Get output if interactive: interactive_shell(chan) else: chan.shutdown_write() while True: data = chan.recv(1024) if len(data) == 0: break sys.stdout.buffer.write(data) sys.stdout.flush() retcode = chan.recv_exit_status() ssh.close() return retcode
def docker_run(args): """Runs the experiment in the container. """ target = Path(args.target[0]) unpacked_info = read_dict(target) cmdline = args.cmdline # Sanity check if args.detach and args.x11: logging.critical("Error: Can't use X11 forwarding if you're detaching") raise UsageError # Loads config config = load_config(target / 'config.yml', True) runs = config.runs selected_runs = get_runs(runs, args.run, cmdline) # Get current image name if 'current_image' in unpacked_info: image = unpacked_info['current_image'] logging.debug("Running from image %s", image.decode('ascii')) else: logging.critical("Image doesn't exist yet, have you run setup/build?") sys.exit(1) # Name of new container if args.detach: container = make_unique_name(b'reprounzip_detached_') else: container = make_unique_name(b'reprounzip_run_') hostname = runs[selected_runs[0]].get('hostname', 'reprounzip') # X11 handler if args.x11: local_ip = get_local_addr() docker_host = local_ip if os.environ.get('DOCKER_HOST'): m = _dockerhost_re.match(os.environ['DOCKER_HOST']) if m is not None: docker_host = m.group(1) if args.tunneled_x11: x11 = X11Handler(True, ('internet', docker_host), args.x11_display) else: x11 = X11Handler(True, ('internet', local_ip), args.x11_display) if (docker_host != local_ip and docker_host != 'localhost' and not docker_host.startswith('127.') and not docker_host.startswith('192.168.99.')): ssh_cmdline = ' '.join('-R*:%(p)d:127.0.0.1:%(p)d' % {'p': port} for port, connector in x11.port_forward) logging.warning( "You requested X11 forwarding but the Docker container " "appears to be running remotely. It is probable that it " "won't be able to connect to the local display. Creating " "a remote SSH tunnel and running with --tunneled-x11 " "might help (%s).", ssh_cmdline) else: x11 = X11Handler(False, ('local', hostname), args.x11_display) cmds = [] for run_number in selected_runs: run = runs[run_number] cmd = 'cd %s && ' % shell_escape(run['workingdir']) cmd += '/busybox env -i ' environ = x11.fix_env(run['environ']) environ = fixup_environment(environ, args) cmd += ' '.join('%s=%s' % (shell_escape(k), shell_escape(v)) for k, v in iteritems(environ)) cmd += ' ' # FIXME : Use exec -a or something if binary != argv[0] if cmdline is None: argv = [run['binary']] + run['argv'][1:] else: argv = cmdline cmd += ' '.join(shell_escape(a) for a in argv) uid = run.get('uid', 1000) gid = run.get('gid', 1000) cmd = '/rpzsudo \'#%d\' \'#%d\' /busybox sh -c %s' % ( uid, gid, shell_escape(cmd)) cmds.append(cmd) cmds = x11.init_cmds + cmds cmds = ' && '.join(cmds) signals.pre_run(target=target) # Creates forwarders forwarders = [] for port, connector in x11.port_forward: forwarders.append(LocalForwarder(connector, port)) if args.detach: logging.info("Start container %s (detached)", container.decode('ascii')) retcode = interruptible_call([ 'docker', 'run', b'--name=' + container, '-h', hostname, '-d', '-t' ] + args.docker_option + [image, '/busybox', 'sh', '-c', cmds]) if retcode != 0: logging.critical("docker run failed with code %d", retcode) subprocess.call(['docker', 'rm', '-f', container]) sys.exit(1) return # Run command in container logging.info("Starting container %s", container.decode('ascii')) retcode = interruptible_call( ['docker', 'run', b'--name=' + container, '-h', hostname, '-i', '-t'] + args.docker_option + [image, '/busybox', 'sh', '-c', cmds]) if retcode != 0: logging.critical("docker run failed with code %d", retcode) subprocess.call(['docker', 'rm', '-f', container]) sys.exit(1) # Get exit status from "docker inspect" out = subprocess.check_output(['docker', 'inspect', container]) outjson = json.loads(out.decode('ascii')) if (outjson[0]["State"]["Running"] is not False or outjson[0]["State"]["Paused"] is not False): logging.error("Invalid container state after execution:\n%s", json.dumps(outjson[0]["State"])) retcode = outjson[0]["State"]["ExitCode"] stderr.write("\n*** Command finished, status: %d\n" % retcode) # Commit to create new image new_image = make_unique_name(b'reprounzip_image_') logging.info("Committing container %s to image %s", container.decode('ascii'), new_image.decode('ascii')) subprocess.check_call(['docker', 'commit', container, new_image]) # Update image name unpacked_info['current_image'] = new_image write_dict(target, unpacked_info) # Remove the container logging.info("Destroying container %s", container.decode('ascii')) retcode = subprocess.call(['docker', 'rm', container]) if retcode != 0: logging.error("Error deleting container %s", container.decode('ascii')) # Untag previous image, unless it is the initial_image if image != unpacked_info['initial_image']: logging.info("Untagging previous image %s", image.decode('ascii')) subprocess.check_call(['docker', 'rmi', image]) # Update input file status metadata_update_run(config, unpacked_info, selected_runs) write_dict(target, unpacked_info) signals.post_run(target=target, retcode=retcode)
def docker_run(args): """Runs the experiment in the container. """ target = Path(args.target[0]) unpacked_info = read_dict(target) cmdline = args.cmdline # Sanity check if args.detach and args.x11: logging.critical("Error: Can't use X11 forwarding if you're detaching") raise UsageError # Loads config config = load_config(target / 'config.yml', True) runs = config.runs selected_runs = get_runs(runs, args.run, cmdline) # Get current image name if 'current_image' in unpacked_info: image = unpacked_info['current_image'] logging.debug("Running from image %s", image.decode('ascii')) else: logging.critical("Image doesn't exist yet, have you run setup/build?") sys.exit(1) # Name of new container if args.detach: container = make_unique_name(b'reprounzip_detached_') else: container = make_unique_name(b'reprounzip_run_') hostname = runs[selected_runs[0]].get('hostname', 'reprounzip') # Port forwarding port_options = [] for port_host, port_container, proto in parse_ports(args.expose_port): port_options.extend(['-p', '%s:%s%s' % (port_host, port_container, proto)]) # X11 handler if args.x11: local_ip = get_local_addr() docker_host = local_ip if os.environ.get('DOCKER_HOST'): m = _dockerhost_re.match(os.environ['DOCKER_HOST']) if m is not None: docker_host = m.group(1) if args.tunneled_x11: x11 = X11Handler(True, ('internet', docker_host), args.x11_display) else: x11 = X11Handler(True, ('internet', local_ip), args.x11_display) if (docker_host != local_ip and docker_host != 'localhost' and not docker_host.startswith('127.') and not docker_host.startswith('192.168.99.')): ssh_cmdline = ' '.join( '-R*:%(p)d:127.0.0.1:%(p)d' % {'p': port} for port, connector in x11.port_forward) logging.warning( "You requested X11 forwarding but the Docker container " "appears to be running remotely. It is probable that it " "won't be able to connect to the local display. Creating " "a remote SSH tunnel and running with --tunneled-x11 " "might help (%s).", ssh_cmdline) else: x11 = X11Handler(False, ('local', hostname), args.x11_display) cmd = [] for run_number in selected_runs: run = runs[run_number] env_set, env_unset = x11.env_fixes(run['environ']) a_env_set, a_env_unset = parse_environment_args(args) env_set.update(a_env_set) env_unset.extend(a_env_unset) if env_set or env_unset: cmd.append('env') env = [] for k in env_unset: env.append('-u') env.append(shell_escape(k)) for k, v in iteritems(env_set): env.append('%s=%s' % (shell_escape(k), shell_escape(v))) cmd.append(' '.join(env)) # FIXME : Use exec -a or something if binary != argv[0] if cmdline is not None: cmd.append('cmd') cmd.append(' '.join(shell_escape(a) for a in cmdline)) cmd.append('run') cmd.append('%d' % run_number) cmd = list(chain.from_iterable([['do', shell_escape(c)] for c in x11.init_cmds] + [cmd])) if logging.getLogger().isEnabledFor(logging.DEBUG): logging.debug("Passing arguments to Docker image:") for c in cmd: logging.debug(c) signals.pre_run(target=target) # Creates forwarders forwarders = [] for port, connector in x11.port_forward: forwarders.append(LocalForwarder(connector, port)) if args.detach: logging.info("Start container %s (detached)", container.decode('ascii')) retcode = interruptible_call(args.docker_cmd.split() + ['run', b'--name=' + container, '-h', hostname, '-d', '-t'] + port_options + args.docker_option + [image] + cmd) if retcode != 0: logging.critical("docker run failed with code %d", retcode) subprocess.call(['docker', 'rm', '-f', container]) sys.exit(1) return # Run command in container logging.info("Starting container %s", container.decode('ascii')) retcode = interruptible_call(args.docker_cmd.split() + ['run', b'--name=' + container, '-h', hostname, '-i', '-t'] + port_options + args.docker_option + [image] + cmd) if retcode != 0: logging.critical("docker run failed with code %d", retcode) subprocess.call(['docker', 'rm', '-f', container]) sys.exit(1) # Get exit status from "docker inspect" out = subprocess.check_output(args.docker_cmd.split() + ['inspect', container]) outjson = json.loads(out.decode('ascii')) if (outjson[0]["State"]["Running"] is not False or outjson[0]["State"]["Paused"] is not False): logging.error("Invalid container state after execution:\n%s", json.dumps(outjson[0]["State"])) retcode = outjson[0]["State"]["ExitCode"] stderr.write("\n*** Command finished, status: %d\n" % retcode) # Commit to create new image new_image = make_unique_name(b'reprounzip_image_') logging.info("Committing container %s to image %s", container.decode('ascii'), new_image.decode('ascii')) subprocess.check_call(args.docker_cmd.split() + ['commit', container, new_image]) # Update image name unpacked_info['current_image'] = new_image write_dict(target, unpacked_info) # Remove the container logging.info("Destroying container %s", container.decode('ascii')) retcode = subprocess.call(args.docker_cmd.split() + ['rm', container]) if retcode != 0: logging.error("Error deleting container %s", container.decode('ascii')) # Untag previous image, unless it is the initial_image if image != unpacked_info['initial_image']: logging.info("Untagging previous image %s", image.decode('ascii')) subprocess.check_call(args.docker_cmd.split() + ['rmi', image]) # Update input file status metadata_update_run(config, unpacked_info, selected_runs) write_dict(target, unpacked_info) signals.post_run(target=target, retcode=retcode)
def docker_run(args): """Runs the experiment in the container. """ target = Path(args.target[0]) unpacked_info = read_dict(target / '.reprounzip') cmdline = args.cmdline # Loads config runs, packages, other_files = load_config(target / 'config.yml', True) selected_runs = get_runs(runs, args.run, cmdline) # Destroy previous container if 'ran_container' in unpacked_info: container = unpacked_info.pop('ran_container') logging.info("Destroying previous container %s", container.decode('ascii')) retcode = subprocess.call(['docker', 'rm', '-f', container]) if retcode != 0: logging.error("Error deleting previous container %s", container.decode('ascii')) write_dict(target / '.reprounzip', unpacked_info) # Use the initial image directly if 'current_image' in unpacked_info: image = unpacked_info['current_image'] logging.debug("Running from image %s", image.decode('ascii')) else: logging.critical("Image doesn't exist yet, have you run setup/build?") sys.exit(1) # Name of new container container = make_unique_name(b'reprounzip_run_') hostname = runs[selected_runs[0]].get('hostname', 'reprounzip') # Get the local bridge IP ip_str = get_iface_addr('docker0') # X11 handler x11 = X11Handler(args.x11, ('internet', ip_str), args.x11_display) cmds = [] for run_number in selected_runs: run = runs[run_number] cmd = 'cd %s && ' % shell_escape(run['workingdir']) cmd += '/usr/bin/env -i ' environ = x11.fix_env(run['environ']) cmd += ' '.join('%s=%s' % (k, shell_escape(v)) for k, v in iteritems(environ)) cmd += ' ' # FIXME : Use exec -a or something if binary != argv[0] if cmdline is None: argv = [run['binary']] + run['argv'][1:] else: argv = cmdline cmd += ' '.join(shell_escape(a) for a in argv) uid = run.get('uid', 1000) cmd = 'sudo -u \'#%d\' /bin/busybox sh -c %s\n' % (uid, shell_escape(cmd)) cmds.append(cmd) cmds = x11.init_cmds + cmds cmds = ' && '.join(cmds) signals.pre_run(target=target) # Creates forwarders forwarders = [] for port, connector in x11.port_forward: forwarders.append( LocalForwarder(connector, port)) # Run command in container logging.info("Starting container %s", container.decode('ascii')) retcode = interruptible_call(['docker', 'run', b'--name=' + container, '-h', hostname, '-i', '-t', image, '/bin/busybox', 'sh', '-c', cmds]) if retcode != 0: logging.critical("docker run failed with code %d", retcode) sys.exit(1) # Get exit status from "docker inspect" out = subprocess.check_output(['docker', 'inspect', container]) outjson = json.loads(out.decode('ascii')) if (outjson[0]["State"]["Running"] is not False or outjson[0]["State"]["Paused"] is not False): logging.error("Invalid container state after execution:\n%s", json.dumps(outjson[0]["State"])) retcode = outjson[0]["State"]["ExitCode"] sys.stderr.write("\n*** Command finished, status: %d\n" % retcode) # Store container name (so we can download output files) unpacked_info['ran_container'] = container write_dict(target / '.reprounzip', unpacked_info) signals.post_run(target=target, retcode=retcode)
def directory_run(args): """Runs the command in the directory. """ target = Path(args.target[0]) unpacked_info = metadata_read(target, 'directory') cmdline = args.cmdline # Loads config config = load_config_file(target / 'config.yml', True) runs = config.runs selected_runs = get_runs(runs, args.run, cmdline) root = (target / 'root').absolute() # Gets library paths lib_dirs = [] p = subprocess.Popen(['/sbin/ldconfig', '-v', '-N'], stdout=subprocess.PIPE) try: for l in p.stdout: if len(l) < 3 or l[0] in (b' ', b'\t'): continue if l.endswith(b':\n'): lib_dirs.append(Path(l[:-2])) finally: p.communicate() lib_dirs = ('export LD_LIBRARY_PATH=%s' % ':'.join( shell_escape(unicode_(join_root(root, d))) for d in lib_dirs)) cmds = [lib_dirs] for run_number in selected_runs: run = runs[run_number] cmd = 'cd %s && ' % shell_escape( unicode_(join_root(root, Path(run['workingdir'])))) cmd += '/usr/bin/env -i ' environ = run['environ'] environ = fixup_environment(environ, args) if args.x11: if 'DISPLAY' in os.environ: environ['DISPLAY'] = os.environ['DISPLAY'] if 'XAUTHORITY' in os.environ: environ['XAUTHORITY'] = os.environ['XAUTHORITY'] cmd += ' '.join('%s=%s' % (shell_escape(k), shell_escape(v)) for k, v in iteritems(environ) if k != 'PATH') cmd += ' ' # PATH # Get the original PATH components path = [PosixPath(d) for d in run['environ'].get('PATH', '').split(':')] # The same paths but in the directory dir_path = [join_root(root, d) for d in path if d.root == '/'] # Rebuild string path = ':'.join(unicode_(d) for d in dir_path + path) cmd += 'PATH=%s ' % shell_escape(path) # FIXME : Use exec -a or something if binary != argv[0] if cmdline is None: argv = run['argv'] # Rewrites command-line arguments that are absolute filenames rewritten = False for i in irange(len(argv)): try: p = Path(argv[i]) except UnicodeEncodeError: continue if p.is_absolute: rp = join_root(root, p) if (rp.exists() or (len(rp.components) > 3 and rp.parent.exists())): argv[i] = str(rp) rewritten = True if rewritten: logging.warning("Rewrote command-line as: %s", ' '.join(shell_escape(a) for a in argv)) else: argv = cmdline cmd += ' '.join(shell_escape(a) for a in argv) cmds.append(cmd) cmds = ' && '.join(cmds) signals.pre_run(target=target) retcode = interruptible_call(cmds, shell=True) stderr.write("\n*** Command finished, status: %d\n" % retcode) signals.post_run(target=target, retcode=retcode) # Update input file status metadata_update_run(config, unpacked_info, selected_runs) metadata_write(target, unpacked_info, 'directory')
def docker_run(args): """Runs the experiment in the container. """ target = Path(args.target[0]) unpacked_info = read_dict(target) cmdline = args.cmdline # Sanity check if args.detach and args.x11: logger.critical("Error: Can't use X11 forwarding if you're detaching") raise UsageError # Loads config config = load_config(target / 'config.yml', True) runs = config.runs selected_runs = get_runs(runs, args.run, cmdline) # Get current image name if 'current_image' in unpacked_info: image = unpacked_info['current_image'] logger.debug("Running from image %s", image.decode('ascii')) else: logger.critical("Image doesn't exist yet, have you run setup/build?") sys.exit(1) # Name of new container if args.detach: container = make_unique_name(b'reprounzip_detached_') else: container = make_unique_name(b'reprounzip_run_') hostname = runs[selected_runs[0]].get('hostname', 'reprounzip') # Port forwarding port_options = [] for port_host, port_container, proto in parse_ports(args.expose_port): port_options.extend(['-p', '%s:%s/%s' % (port_host, port_container, proto)]) # X11 handler if args.x11: local_ip = get_local_addr() docker_host = local_ip if os.environ.get('DOCKER_HOST'): m = _dockerhost_re.match(os.environ['DOCKER_HOST']) if m is not None: docker_host = m.group(1) if args.tunneled_x11: x11 = X11Handler(True, ('internet', docker_host), args.x11_display) else: x11 = X11Handler(True, ('internet', local_ip), args.x11_display) if (docker_host != local_ip and docker_host != 'localhost' and not docker_host.startswith('127.') and not docker_host.startswith('192.168.99.')): ssh_cmdline = ' '.join( '-R*:%(p)d:127.0.0.1:%(p)d' % {'p': port} for port, connector in x11.port_forward) logger.warning( "You requested X11 forwarding but the Docker container " "appears to be running remotely. It is probable that it " "won't be able to connect to the local display. Creating " "a remote SSH tunnel and running with --tunneled-x11 " "might help (%s).", ssh_cmdline) else: x11 = X11Handler(False, ('local', hostname), args.x11_display) cmd = [] for run_number in selected_runs: run = runs[run_number] env_set, env_unset = x11.env_fixes(run['environ']) a_env_set, a_env_unset = parse_environment_args(args) env_set.update(a_env_set) env_unset.extend(a_env_unset) if env_set or env_unset: cmd.append('env') env = [] for k in env_unset: env.append('-u') env.append(shell_escape(k)) for k, v in iteritems(env_set): env.append('%s=%s' % (shell_escape(k), shell_escape(v))) cmd.append(' '.join(env)) # FIXME : Use exec -a or something if binary != argv[0] if cmdline is not None: cmd.append('cmd') cmd.append(' '.join(shell_escape(a) for a in cmdline)) cmd.append('run') cmd.append('%d' % run_number) cmd = list(chain.from_iterable([['do', shell_escape(c)] for c in x11.init_cmds] + [cmd])) if logger.isEnabledFor(logging.DEBUG): logger.debug("Passing arguments to Docker image:") for c in cmd: logger.debug(c) signals.pre_run(target=target) # Creates forwarders forwarders = [] for port, connector in x11.port_forward: forwarders.append(LocalForwarder(connector, port)) if args.detach: logger.info("Start container %s (detached)", container.decode('ascii')) retcode = interruptible_call(args.docker_cmd.split() + ['run', b'--name=' + container, '-h', hostname, '-d', '-t'] + port_options + args.docker_option + [image] + cmd) if retcode != 0: logger.critical("docker run failed with code %d", retcode) subprocess.call(['docker', 'rm', '-f', container]) sys.exit(1) return # Run command in container logger.info("Starting container %s", container.decode('ascii')) retcode = interruptible_call(args.docker_cmd.split() + ['run', b'--name=' + container, '-h', hostname, '-i', '-t'] + port_options + args.docker_option + [image] + cmd, request_tty=True) # The image prints out the exit status(es) itself if retcode != 0: logger.critical("docker run failed with code %d", retcode) subprocess.call(['docker', 'rm', '-f', container]) sys.exit(1) # Commit to create new image new_image = make_unique_name(b'reprounzip_image_') logger.info("Committing container %s to image %s", container.decode('ascii'), new_image.decode('ascii')) subprocess.check_call(args.docker_cmd.split() + ['commit', container, new_image]) # Update image name unpacked_info['current_image'] = new_image write_dict(target, unpacked_info) # Remove the container logger.info("Destroying container %s", container.decode('ascii')) retcode = subprocess.call(args.docker_cmd.split() + ['rm', container]) if retcode != 0: logger.error("Error deleting container %s", container.decode('ascii')) # Untag previous image, unless it is the initial_image if image != unpacked_info['initial_image']: logger.info("Untagging previous image %s", image.decode('ascii')) subprocess.check_call(args.docker_cmd.split() + ['rmi', image]) # Update input file status metadata_update_run(config, unpacked_info, selected_runs) write_dict(target, unpacked_info) signals.post_run(target=target, retcode=retcode)
def docker_run(args): """Runs the experiment in the container. """ target = Path(args.target[0]) unpacked_info = read_dict(target) cmdline = args.cmdline # Sanity check if args.detach and args.x11: logging.critical("Error: Can't use X11 forwarding if you're detaching") raise UsageError # Loads config config = load_config(target / 'config.yml', True) runs = config.runs selected_runs = get_runs(runs, args.run, cmdline) # Get current image name if 'current_image' in unpacked_info: image = unpacked_info['current_image'] logging.debug("Running from image %s", image.decode('ascii')) else: logging.critical("Image doesn't exist yet, have you run setup/build?") sys.exit(1) # Name of new container if args.detach: container = make_unique_name(b'reprounzip_detached_') else: container = make_unique_name(b'reprounzip_run_') hostname = runs[selected_runs[0]].get('hostname', 'reprounzip') # X11 handler if args.x11: local_ip = get_local_addr() docker_host = local_ip if os.environ.get('DOCKER_HOST'): m = _dockerhost_re.match(os.environ['DOCKER_HOST']) if m is not None: docker_host = m.group(1) if args.tunneled_x11: x11 = X11Handler(True, ('internet', docker_host), args.x11_display) else: x11 = X11Handler(True, ('internet', local_ip), args.x11_display) if (docker_host != local_ip and docker_host != 'localhost' and not docker_host.startswith('127.') and not docker_host.startswith('192.168.99.')): ssh_cmdline = ' '.join( '-R*:%(p)d:127.0.0.1:%(p)d' % {'p': port} for port, connector in x11.port_forward) logging.warning( "You requested X11 forwarding but the Docker container " "appears to be running remotely. It is probable that it " "won't be able to connect to the local display. Creating " "a remote SSH tunnel and running with --tunneled-x11 " "might help (%s).", ssh_cmdline) else: x11 = X11Handler(False, ('local', hostname), args.x11_display) cmds = [] for run_number in selected_runs: run = runs[run_number] cmd = 'cd %s && ' % shell_escape(run['workingdir']) cmd += '/busybox env -i ' environ = x11.fix_env(run['environ']) environ = fixup_environment(environ, args) cmd += ' '.join('%s=%s' % (shell_escape(k), shell_escape(v)) for k, v in iteritems(environ)) cmd += ' ' # FIXME : Use exec -a or something if binary != argv[0] if cmdline is None: argv = [run['binary']] + run['argv'][1:] else: argv = cmdline cmd += ' '.join(shell_escape(a) for a in argv) uid = run.get('uid', 1000) gid = run.get('gid', 1000) cmd = '/rpzsudo \'#%d\' \'#%d\' /busybox sh -c %s' % ( uid, gid, shell_escape(cmd)) cmds.append(cmd) cmds = x11.init_cmds + cmds cmds = ' && '.join(cmds) signals.pre_run(target=target) # Creates forwarders forwarders = [] for port, connector in x11.port_forward: forwarders.append(LocalForwarder(connector, port)) if args.detach: logging.info("Start container %s (detached)", container.decode('ascii')) retcode = interruptible_call(['docker', 'run', b'--name=' + container, '-h', hostname, '-d', '-t'] + args.docker_option + [image, '/busybox', 'sh', '-c', cmds]) if retcode != 0: logging.critical("docker run failed with code %d", retcode) subprocess.call(['docker', 'rm', '-f', container]) sys.exit(1) return # Run command in container logging.info("Starting container %s", container.decode('ascii')) retcode = interruptible_call(['docker', 'run', b'--name=' + container, '-h', hostname, '-i', '-t'] + args.docker_option + [image, '/busybox', 'sh', '-c', cmds]) if retcode != 0: logging.critical("docker run failed with code %d", retcode) subprocess.call(['docker', 'rm', '-f', container]) sys.exit(1) # Get exit status from "docker inspect" out = subprocess.check_output(['docker', 'inspect', container]) outjson = json.loads(out.decode('ascii')) if (outjson[0]["State"]["Running"] is not False or outjson[0]["State"]["Paused"] is not False): logging.error("Invalid container state after execution:\n%s", json.dumps(outjson[0]["State"])) retcode = outjson[0]["State"]["ExitCode"] stderr.write("\n*** Command finished, status: %d\n" % retcode) # Commit to create new image new_image = make_unique_name(b'reprounzip_image_') logging.info("Committing container %s to image %s", container.decode('ascii'), new_image.decode('ascii')) subprocess.check_call(['docker', 'commit', container, new_image]) # Update image name unpacked_info['current_image'] = new_image write_dict(target, unpacked_info) # Remove the container logging.info("Destroying container %s", container.decode('ascii')) retcode = subprocess.call(['docker', 'rm', container]) if retcode != 0: logging.error("Error deleting container %s", container.decode('ascii')) # Untag previous image, unless it is the initial_image if image != unpacked_info['initial_image']: logging.info("Untagging previous image %s", image.decode('ascii')) subprocess.check_call(['docker', 'rmi', image]) # Update input file status metadata_update_run(config, unpacked_info, selected_runs) write_dict(target, unpacked_info) signals.post_run(target=target, retcode=retcode)
def directory_run(args): """Runs the command in the directory. """ target = Path(args.target[0]) read_dict(target / '.reprounzip', 'directory') cmdline = args.cmdline # Loads config runs, packages, other_files = load_config_file(target / 'config.yml', True) selected_runs = get_runs(runs, args.run, cmdline) root = (target / 'root').absolute() # Gets library paths lib_dirs = [] p = subprocess.Popen(['/sbin/ldconfig', '-v', '-N'], stdout=subprocess.PIPE) try: for l in p.stdout: if len(l) < 3 or l[0] in (b' ', b'\t'): continue if l.endswith(b':\n'): lib_dirs.append(Path(l[:-2])) finally: p.wait() lib_dirs = ( 'export LD_LIBRARY_PATH=%s' % ':'.join(shell_escape(unicode_(join_root(root, d))) for d in lib_dirs)) cmds = [lib_dirs] for run_number in selected_runs: run = runs[run_number] cmd = 'cd %s && ' % shell_escape( unicode_(join_root(root, Path(run['workingdir'])))) cmd += '/usr/bin/env -i ' environ = run['environ'] if args.x11: if 'DISPLAY' in os.environ: environ['DISPLAY'] = os.environ['DISPLAY'] if 'XAUTHORITY' in os.environ: environ['XAUTHORITY'] = os.environ['XAUTHORITY'] cmd += ' '.join('%s=%s' % (k, shell_escape(v)) for k, v in iteritems(environ) if k != 'PATH') cmd += ' ' # PATH # Get the original PATH components path = [ PosixPath(d) for d in run['environ'].get('PATH', '').split(':') ] # The same paths but in the directory dir_path = [join_root(root, d) for d in path if d.root == '/'] # Rebuild string path = ':'.join(unicode_(d) for d in dir_path + path) cmd += 'PATH=%s ' % shell_escape(path) # FIXME : Use exec -a or something if binary != argv[0] if cmdline is None: argv = run['argv'] # Rewrites command-line arguments that are absolute filenames rewritten = False for i in irange(len(argv)): try: p = Path(argv[i]) except UnicodeEncodeError: continue if p.is_absolute: rp = join_root(root, p) if (rp.exists() or (len(rp.components) > 3 and rp.parent.exists())): argv[i] = str(rp) rewritten = True if rewritten: logging.warning("Rewrote command-line as: %s", ' '.join(shell_escape(a) for a in argv)) else: argv = cmdline cmd += ' '.join(shell_escape(a) for a in argv) cmds.append(cmd) cmds = ' && '.join(cmds) signals.pre_run(target=target) retcode = interruptible_call(cmds, shell=True) stderr.write("\n*** Command finished, status: %d\n" % retcode) signals.post_run(target=target, retcode=retcode)
def run_interactive(ssh_info, interactive, cmd, request_pty, forwarded_ports): """Runs a command on an SSH server. If `interactive` is True, we'll try to find an ``ssh`` executable, falling back to paramiko if it's not found. The terminal handling code is a bit wonky, so using ``ssh`` is definitely a good idea, especially on Windows. Non-interactive commands should run fine. :param ssh_info: dict with `hostname`, `port`, `username`, `key_filename`, passed directly to paramiko :type ssh_info: dict :param interactive: whether to connect local input to the remote process :type interactive: bool :param cmd: command-line to run on the server :type cmd: str :param request_pty: whether to request a PTY from the SSH server :type request_pty: bool :param forwarded_ports: ports to forward back to us; iterable of pairs ``(port_number, connector)`` where `port_number` is the remote port number and `connector` is the connector object used to build the connected socket to forward to on this side :type forwarded_ports: collections.Iterable[(int, object)] """ if interactive: ssh_exe = find_ssh_executable() else: ssh_exe = None if interactive and ssh_exe: record_usage(vagrant_ssh='ssh') args = [ssh_exe, '-t' if request_pty else '-T', # Force allocation of PTY '-o', 'StrictHostKeyChecking=no', # Silently accept host keys '-o', 'UserKnownHostsFile=/dev/null', # Don't store host keys '-i', ssh_info['key_filename'], '-p', '%d' % ssh_info['port']] for remote_port, connector in forwarded_ports: # Remote port will connect to a local port fwd = LocalForwarder(connector) args.append('-R%d:127.0.0.1:%d' % (remote_port, fwd.local_port)) args.append('%s@%s' % (ssh_info['username'], ssh_info['hostname'])) args.append(cmd) return interruptible_call(args) else: record_usage(vagrant_ssh='interactive' if interactive else 'simple') # Connects to the machine ssh = paramiko.SSHClient() ssh.set_missing_host_key_policy(IgnoreMissingKey()) ssh.connect(**ssh_info) # Starts forwarding forwarders = [] for remote_port, connector in forwarded_ports: forwarders.append( SSHForwarder(ssh.get_transport(), remote_port, connector)) chan = ssh.get_transport().open_session() if request_pty: chan.get_pty() # Execute command logging.info("Connected via SSH, running command...") chan.exec_command(cmd) # Get output if interactive: interactive_shell(chan) else: chan.shutdown_write() while True: data = chan.recv(1024) if len(data) == 0: break stdout_bytes.write(data) stdout_bytes.flush() retcode = chan.recv_exit_status() ssh.close() return retcode