def docker_run(args): """Runs the experiment in the container. """ target = Path(args.target[0]) unpacked_info = read_dict(target) cmdline = args.cmdline # Sanity check if args.detach and args.x11: logging.critical("Error: Can't use X11 forwarding if you're detaching") raise UsageError # Loads config config = load_config(target / 'config.yml', True) runs = config.runs selected_runs = get_runs(runs, args.run, cmdline) # Get current image name if 'current_image' in unpacked_info: image = unpacked_info['current_image'] logging.debug("Running from image %s", image.decode('ascii')) else: logging.critical("Image doesn't exist yet, have you run setup/build?") sys.exit(1) # Name of new container if args.detach: container = make_unique_name(b'reprounzip_detached_') else: container = make_unique_name(b'reprounzip_run_') hostname = runs[selected_runs[0]].get('hostname', 'reprounzip') # Port forwarding port_options = [] for port_host, port_container, proto in parse_ports(args.expose_port): port_options.extend(['-p', '%s:%s%s' % (port_host, port_container, proto)]) # X11 handler if args.x11: local_ip = get_local_addr() docker_host = local_ip if os.environ.get('DOCKER_HOST'): m = _dockerhost_re.match(os.environ['DOCKER_HOST']) if m is not None: docker_host = m.group(1) if args.tunneled_x11: x11 = X11Handler(True, ('internet', docker_host), args.x11_display) else: x11 = X11Handler(True, ('internet', local_ip), args.x11_display) if (docker_host != local_ip and docker_host != 'localhost' and not docker_host.startswith('127.') and not docker_host.startswith('192.168.99.')): ssh_cmdline = ' '.join( '-R*:%(p)d:127.0.0.1:%(p)d' % {'p': port} for port, connector in x11.port_forward) logging.warning( "You requested X11 forwarding but the Docker container " "appears to be running remotely. It is probable that it " "won't be able to connect to the local display. Creating " "a remote SSH tunnel and running with --tunneled-x11 " "might help (%s).", ssh_cmdline) else: x11 = X11Handler(False, ('local', hostname), args.x11_display) cmd = [] for run_number in selected_runs: run = runs[run_number] env_set, env_unset = x11.env_fixes(run['environ']) a_env_set, a_env_unset = parse_environment_args(args) env_set.update(a_env_set) env_unset.extend(a_env_unset) if env_set or env_unset: cmd.append('env') env = [] for k in env_unset: env.append('-u') env.append(shell_escape(k)) for k, v in iteritems(env_set): env.append('%s=%s' % (shell_escape(k), shell_escape(v))) cmd.append(' '.join(env)) # FIXME : Use exec -a or something if binary != argv[0] if cmdline is not None: cmd.append('cmd') cmd.append(' '.join(shell_escape(a) for a in cmdline)) cmd.append('run') cmd.append('%d' % run_number) cmd = list(chain.from_iterable([['do', shell_escape(c)] for c in x11.init_cmds] + [cmd])) if logging.getLogger().isEnabledFor(logging.DEBUG): logging.debug("Passing arguments to Docker image:") for c in cmd: logging.debug(c) signals.pre_run(target=target) # Creates forwarders forwarders = [] for port, connector in x11.port_forward: forwarders.append(LocalForwarder(connector, port)) if args.detach: logging.info("Start container %s (detached)", container.decode('ascii')) retcode = interruptible_call(args.docker_cmd.split() + ['run', b'--name=' + container, '-h', hostname, '-d', '-t'] + port_options + args.docker_option + [image] + cmd) if retcode != 0: logging.critical("docker run failed with code %d", retcode) subprocess.call(['docker', 'rm', '-f', container]) sys.exit(1) return # Run command in container logging.info("Starting container %s", container.decode('ascii')) retcode = interruptible_call(args.docker_cmd.split() + ['run', b'--name=' + container, '-h', hostname, '-i', '-t'] + port_options + args.docker_option + [image] + cmd) if retcode != 0: logging.critical("docker run failed with code %d", retcode) subprocess.call(['docker', 'rm', '-f', container]) sys.exit(1) # Get exit status from "docker inspect" out = subprocess.check_output(args.docker_cmd.split() + ['inspect', container]) outjson = json.loads(out.decode('ascii')) if (outjson[0]["State"]["Running"] is not False or outjson[0]["State"]["Paused"] is not False): logging.error("Invalid container state after execution:\n%s", json.dumps(outjson[0]["State"])) retcode = outjson[0]["State"]["ExitCode"] stderr.write("\n*** Command finished, status: %d\n" % retcode) # Commit to create new image new_image = make_unique_name(b'reprounzip_image_') logging.info("Committing container %s to image %s", container.decode('ascii'), new_image.decode('ascii')) subprocess.check_call(args.docker_cmd.split() + ['commit', container, new_image]) # Update image name unpacked_info['current_image'] = new_image write_dict(target, unpacked_info) # Remove the container logging.info("Destroying container %s", container.decode('ascii')) retcode = subprocess.call(args.docker_cmd.split() + ['rm', container]) if retcode != 0: logging.error("Error deleting container %s", container.decode('ascii')) # Untag previous image, unless it is the initial_image if image != unpacked_info['initial_image']: logging.info("Untagging previous image %s", image.decode('ascii')) subprocess.check_call(args.docker_cmd.split() + ['rmi', image]) # Update input file status metadata_update_run(config, unpacked_info, selected_runs) write_dict(target, unpacked_info) signals.post_run(target=target, retcode=retcode)
def docker_run(args): """Runs the experiment in the container. """ target = Path(args.target[0]) unpacked_info = read_dict(target) cmdline = args.cmdline # Sanity check if args.detach and args.x11: logger.critical("Error: Can't use X11 forwarding if you're detaching") raise UsageError # Loads config config = load_config(target / 'config.yml', True) runs = config.runs selected_runs = get_runs(runs, args.run, cmdline) # Get current image name if 'current_image' in unpacked_info: image = unpacked_info['current_image'] logger.debug("Running from image %s", image.decode('ascii')) else: logger.critical("Image doesn't exist yet, have you run setup/build?") sys.exit(1) # Name of new container if args.detach: container = make_unique_name(b'reprounzip_detached_') else: container = make_unique_name(b'reprounzip_run_') hostname = runs[selected_runs[0]].get('hostname', 'reprounzip') # Port forwarding port_options = [] for port_host, port_container, proto in parse_ports(args.expose_port): port_options.extend(['-p', '%s:%s/%s' % (port_host, port_container, proto)]) # X11 handler if args.x11: local_ip = get_local_addr() docker_host = local_ip if os.environ.get('DOCKER_HOST'): m = _dockerhost_re.match(os.environ['DOCKER_HOST']) if m is not None: docker_host = m.group(1) if args.tunneled_x11: x11 = X11Handler(True, ('internet', docker_host), args.x11_display) else: x11 = X11Handler(True, ('internet', local_ip), args.x11_display) if (docker_host != local_ip and docker_host != 'localhost' and not docker_host.startswith('127.') and not docker_host.startswith('192.168.99.')): ssh_cmdline = ' '.join( '-R*:%(p)d:127.0.0.1:%(p)d' % {'p': port} for port, connector in x11.port_forward) logger.warning( "You requested X11 forwarding but the Docker container " "appears to be running remotely. It is probable that it " "won't be able to connect to the local display. Creating " "a remote SSH tunnel and running with --tunneled-x11 " "might help (%s).", ssh_cmdline) else: x11 = X11Handler(False, ('local', hostname), args.x11_display) cmd = [] for run_number in selected_runs: run = runs[run_number] env_set, env_unset = x11.env_fixes(run['environ']) a_env_set, a_env_unset = parse_environment_args(args) env_set.update(a_env_set) env_unset.extend(a_env_unset) if env_set or env_unset: cmd.append('env') env = [] for k in env_unset: env.append('-u') env.append(shell_escape(k)) for k, v in iteritems(env_set): env.append('%s=%s' % (shell_escape(k), shell_escape(v))) cmd.append(' '.join(env)) # FIXME : Use exec -a or something if binary != argv[0] if cmdline is not None: cmd.append('cmd') cmd.append(' '.join(shell_escape(a) for a in cmdline)) cmd.append('run') cmd.append('%d' % run_number) cmd = list(chain.from_iterable([['do', shell_escape(c)] for c in x11.init_cmds] + [cmd])) if logger.isEnabledFor(logging.DEBUG): logger.debug("Passing arguments to Docker image:") for c in cmd: logger.debug(c) signals.pre_run(target=target) # Creates forwarders forwarders = [] for port, connector in x11.port_forward: forwarders.append(LocalForwarder(connector, port)) if args.detach: logger.info("Start container %s (detached)", container.decode('ascii')) retcode = interruptible_call(args.docker_cmd.split() + ['run', b'--name=' + container, '-h', hostname, '-d', '-t'] + port_options + args.docker_option + [image] + cmd) if retcode != 0: logger.critical("docker run failed with code %d", retcode) subprocess.call(['docker', 'rm', '-f', container]) sys.exit(1) return # Run command in container logger.info("Starting container %s", container.decode('ascii')) retcode = interruptible_call(args.docker_cmd.split() + ['run', b'--name=' + container, '-h', hostname, '-i', '-t'] + port_options + args.docker_option + [image] + cmd, request_tty=True) # The image prints out the exit status(es) itself if retcode != 0: logger.critical("docker run failed with code %d", retcode) subprocess.call(['docker', 'rm', '-f', container]) sys.exit(1) # Commit to create new image new_image = make_unique_name(b'reprounzip_image_') logger.info("Committing container %s to image %s", container.decode('ascii'), new_image.decode('ascii')) subprocess.check_call(args.docker_cmd.split() + ['commit', container, new_image]) # Update image name unpacked_info['current_image'] = new_image write_dict(target, unpacked_info) # Remove the container logger.info("Destroying container %s", container.decode('ascii')) retcode = subprocess.call(args.docker_cmd.split() + ['rm', container]) if retcode != 0: logger.error("Error deleting container %s", container.decode('ascii')) # Untag previous image, unless it is the initial_image if image != unpacked_info['initial_image']: logger.info("Untagging previous image %s", image.decode('ascii')) subprocess.check_call(args.docker_cmd.split() + ['rmi', image]) # Update input file status metadata_update_run(config, unpacked_info, selected_runs) write_dict(target, unpacked_info) signals.post_run(target=target, retcode=retcode)
def vagrant_run(args): """Runs the experiment in the virtual machine. """ target = Path(args.target[0]) unpacked_info = read_dict(target) use_chroot = unpacked_info['use_chroot'] cmdline = args.cmdline check_vagrant_version() # Loads config config = load_config(target / 'config.yml', True) runs = config.runs selected_runs = get_runs(runs, args.run, cmdline) hostname = runs[selected_runs[0]].get('hostname', 'reprounzip') # Port forwarding ports = parse_ports(args.expose_port) # If the requested ports are not a subset of the ones already set on the # VM, we have to update the Vagrantfile and issue `vagrant reload`, which # will reboot the machine req_ports = set(ports) set_ports = set(unpacked_info.get('ports', [])) if not req_ports.issubset(set_ports): # Build new set of forwarded ports: the ones already set + the one just # requested # The ones we request now override the previous config all_ports = dict( (host, (guest, proto)) for host, guest, proto in set_ports) for host, guest, proto in req_ports: all_ports[host] = guest, proto unpacked_info['ports'] = sorted( (host, guest, proto) for host, (guest, proto) in iteritems(all_ports)) write_vagrantfile(target, unpacked_info) logger.info("Some requested ports are not yet forwarded, running " "'vagrant reload'") retcode = subprocess.call(['vagrant', 'reload', '--no-provision'], cwd=target.path) if retcode != 0: logger.critical("vagrant reload failed with code %d, aborting", retcode) sys.exit(1) write_dict(target, unpacked_info) # X11 handler if unpacked_info['gui']: x11 = LocalX11Handler() else: x11 = X11Handler(args.x11, ('local', hostname), args.x11_display) cmds = [] for run_number in selected_runs: run = runs[run_number] cmd = 'cd %s && ' % shell_escape(run['workingdir']) if use_chroot: cmd += '/busybox env -i ' else: cmd += '/usr/bin/env -i ' environ = x11.fix_env(run['environ']) environ = fixup_environment(environ, args) cmd += ' '.join('%s=%s' % (shell_escape(k), shell_escape(v)) for k, v in iteritems(environ)) cmd += ' ' # FIXME : Use exec -a or something if binary != argv[0] if cmdline is None: argv = [run['binary']] + run['argv'][1:] else: argv = cmdline cmd += ' '.join(shell_escape(a) for a in argv) uid = run.get('uid', 1000) gid = run.get('gid', 1000) if use_chroot: userspec = '%s:%s' % (uid, gid) cmd = ('chroot --userspec=%s /experimentroot ' '/bin/sh -c %s' % (userspec, shell_escape(cmd))) else: cmd = 'sudo -u \'#%d\' sh -c %s' % (uid, shell_escape(cmd)) cmds.append(cmd) if use_chroot: cmds = [ 'chroot /experimentroot /bin/sh -c %s' % shell_escape(c) for c in x11.init_cmds ] + cmds else: cmds = x11.init_cmds + cmds cmds = ' && '.join(cmds) # Sets the hostname to the original experiment's machine's # FIXME: not reentrant: this restores the Vagrant machine's hostname after # the run, which might cause issues if several "reprounzip vagrant run" are # running at once cmds = ('OLD_HOSTNAME=$(/bin/hostname); /bin/hostname %s; ' % hostname + cmds + '; RES=$?; /bin/hostname "$OLD_HOSTNAME"; exit $RES') cmds = '/usr/bin/sudo /bin/sh -c %s' % shell_escape(cmds) # Gets vagrant SSH parameters info = machine_setup(target) signals.pre_run(target=target) interactive = not (args.no_stdin or os.environ.get('REPROUNZIP_NON_INTERACTIVE')) retcode = run_interactive(info, interactive, cmds, not args.no_pty, x11.port_forward) stderr.write("\r\n*** Command finished, status: %d\r\n" % retcode) # Update input file status metadata_update_run(config, unpacked_info, selected_runs) write_dict(target, unpacked_info) signals.post_run(target=target, retcode=retcode)
def vagrant_setup_create(args): """Sets up the experiment to be run in a Vagrant-built virtual machine. This can either build a chroot or not. If building a chroot, we do just like without Vagrant: we copy all the files and only get what's missing from the host. But we do install automatically the packages whose files are required. If not building a chroot, we install all the packages, and only unpack files that don't come from packages. In short: files from packages with packfiles=True will only be used if building a chroot. """ if not args.pack: logger.critical("setup/create needs the pack filename") sys.exit(1) pack = Path(args.pack[0]) target = Path(args.target[0]) if target.exists(): logger.critical("Target directory exists") sys.exit(1) use_chroot = args.use_chroot mount_bind = args.bind_magic_dirs record_usage(use_chroot=use_chroot, mount_bind=mount_bind) signals.pre_setup(target=target, pack=pack) # Unpacks configuration file rpz_pack = RPZPack(pack) rpz_pack.extract_config(target / 'config.yml') # Loads config runs, packages, other_files = config = load_config(target / 'config.yml', True) if not args.memory: memory = None else: try: memory = int(args.memory[-1]) except ValueError: logger.critical("Invalid value for memory size: %r", args.memory) sys.exit(1) ports = parse_ports(args.expose_port) if args.base_image and args.base_image[0]: record_usage(vagrant_explicit_image=True) box = args.base_image[0] if args.distribution: target_distribution = args.distribution[0] else: target_distribution = None else: target_distribution, box = select_box(runs, gui=args.gui) logger.info("Using box %s", box) logger.debug("Distribution: %s", target_distribution or "unknown") # If using chroot, we might still need to install packages to get missing # (not packed) files if use_chroot: packages = [pkg for pkg in packages if not pkg.packfiles] if packages: record_usage(vagrant_install_pkgs=True) logger.info( "Some packages were not packed, so we'll install and " "copy their files\n" "Packages that are missing:\n%s", ' '.join(pkg.name for pkg in packages)) if packages: try: installer = select_installer(pack, runs, target_distribution) except CantFindInstaller as e: logger.error( "Need to install %d packages but couldn't select a " "package installer: %s", len(packages), e) target.mkdir(parents=True) try: # Writes setup script logger.info("Writing setup script %s...", target / 'setup.sh') with (target / 'setup.sh').open('w', encoding='utf-8', newline='\n') as fp: fp.write('#!/bin/sh\n\nset -e\n\n') if packages: # Updates package sources update_script = installer.update_script() if update_script: fp.write(update_script) fp.write('\n') # Installs necessary packages fp.write(installer.install_script(packages)) fp.write('\n') # TODO : Compare package versions (painful because of sh) # Untar if use_chroot: fp.write('\n' 'mkdir /experimentroot; cd /experimentroot\n') fp.write('tar zpxf /vagrant/data.tgz --numeric-owner ' '--strip=1 %s\n' % rpz_pack.data_prefix) if mount_bind: fp.write('\n' 'mkdir -p /experimentroot/dev\n' 'mkdir -p /experimentroot/proc\n') for pkg in packages: fp.write('\n# Copies files from package %s\n' % pkg.name) for f in pkg.files: f = f.path dest = join_root(PosixPath('/experimentroot'), f) fp.write('mkdir -p %s\n' % shell_escape(unicode_(f.parent))) fp.write('cp -L %s %s\n' % (shell_escape( unicode_(f)), shell_escape(unicode_(dest)))) fp.write( '\n' 'cp /etc/resolv.conf /experimentroot/etc/resolv.conf\n') else: fp.write('\ncd /\n') paths = set() pathlist = [] # Adds intermediate directories, and checks for existence in # the tar logger.info("Generating file list...") data_files = rpz_pack.data_filenames() for f in other_files: if f.path.name == 'resolv.conf' and ( f.path.lies_under('/etc') or f.path.lies_under('/run') or f.path.lies_under('/var')): continue path = PosixPath('/') for c in rpz_pack.remove_data_prefix(f.path).components: path = path / c if path in paths: continue paths.add(path) if path in data_files: pathlist.append(path) else: logger.info("Missing file %s", path) # FIXME : for some reason we need reversed() here, I'm not sure # why. Need to read more of tar's docs. # TAR bug: --no-overwrite-dir removes --keep-old-files # TAR bug: there is no way to make --keep-old-files not report # an error if an existing file is encountered. --skip-old-files # was introduced too recently. Instead, we just ignore the exit # status with (target / 'rpz-files.list').open('wb') as lfp: for p in reversed(pathlist): lfp.write(join_root(rpz_pack.data_prefix, p).path) lfp.write(b'\0') fp.write('tar zpxf /vagrant/data.tgz --keep-old-files ' '--numeric-owner --strip=1 ' '--null -T /vagrant/rpz-files.list || /bin/true\n') # Copies busybox if use_chroot: arch = runs[0]['architecture'] download_file(busybox_url(arch), target / 'busybox', 'busybox-%s' % arch) fp.write(r''' cp /vagrant/busybox /experimentroot/busybox chmod +x /experimentroot/busybox mkdir -p /experimentroot/bin [ -e /experimentroot/bin/sh ] || \ ln -s /busybox /experimentroot/bin/sh ''') # Copies pack logger.info("Copying pack file...") rpz_pack.copy_data_tar(target / 'data.tgz') rpz_pack.close() # Meta-data for reprounzip metadata = metadata_initial_iofiles( config, { 'use_chroot': use_chroot, 'gui': args.gui, 'ports': ports, 'box': box, 'memory': memory }) write_dict(target, metadata) # Writes Vagrant file write_vagrantfile(target, metadata) signals.post_setup(target=target, pack=pack) except Exception: target.rmtree(ignore_errors=True) raise
def vagrant_run(args): """Runs the experiment in the virtual machine. """ target = Path(args.target[0]) unpacked_info = read_dict(target) use_chroot = unpacked_info['use_chroot'] cmdline = args.cmdline check_vagrant_version() # Loads config config = load_config(target / 'config.yml', True) runs = config.runs selected_runs = get_runs(runs, args.run, cmdline) hostname = runs[selected_runs[0]].get('hostname', 'reprounzip') # Port forwarding ports = parse_ports(args.expose_port) # If the requested ports are not a subset of the ones already set on the # VM, we have to update the Vagrantfile and issue `vagrant reload`, which # will reboot the machine req_ports = set(ports) set_ports = set(unpacked_info.get('ports', [])) if not req_ports.issubset(set_ports): # Build new set of forwarded ports: the ones already set + the one just # requested # The ones we request now override the previous config all_ports = dict((host, (guest, proto)) for host, guest, proto in set_ports) for host, guest, proto in req_ports: all_ports[host] = guest, proto unpacked_info['ports'] = sorted( (host, guest, proto) for host, (guest, proto) in iteritems(all_ports)) write_vagrantfile(target, unpacked_info) logger.info("Some requested ports are not yet forwarded, running " "'vagrant reload'") retcode = subprocess.call(['vagrant', 'reload', '--no-provision'], cwd=target.path) if retcode != 0: logger.critical("vagrant reload failed with code %d, aborting", retcode) sys.exit(1) write_dict(target, unpacked_info) # X11 handler if unpacked_info['gui']: x11 = LocalX11Handler() else: x11 = X11Handler(args.x11, ('local', hostname), args.x11_display) cmds = [] for run_number in selected_runs: run = runs[run_number] cmd = 'cd %s && ' % shell_escape(run['workingdir']) if use_chroot: cmd += '/busybox env -i ' else: cmd += '/usr/bin/env -i ' environ = x11.fix_env(run['environ']) environ = fixup_environment(environ, args) cmd += ' '.join('%s=%s' % (shell_escape(k), shell_escape(v)) for k, v in iteritems(environ)) cmd += ' ' # FIXME : Use exec -a or something if binary != argv[0] if cmdline is None: argv = [run['binary']] + run['argv'][1:] else: argv = cmdline cmd += ' '.join(shell_escape(a) for a in argv) uid = run.get('uid', 1000) gid = run.get('gid', 1000) if use_chroot: userspec = '%s:%s' % (uid, gid) cmd = ('chroot --userspec=%s /experimentroot ' '/bin/sh -c %s' % ( userspec, shell_escape(cmd))) else: cmd = 'sudo -u \'#%d\' sh -c %s' % (uid, shell_escape(cmd)) cmds.append(cmd) if use_chroot: cmds = ['chroot /experimentroot /bin/sh -c %s' % shell_escape(c) for c in x11.init_cmds] + cmds else: cmds = x11.init_cmds + cmds cmds = ' && '.join(cmds) # Sets the hostname to the original experiment's machine's # FIXME: not reentrant: this restores the Vagrant machine's hostname after # the run, which might cause issues if several "reprounzip vagrant run" are # running at once cmds = ('OLD_HOSTNAME=$(/bin/hostname); /bin/hostname %s; ' % hostname + cmds + '; RES=$?; /bin/hostname "$OLD_HOSTNAME"; exit $RES') cmds = '/usr/bin/sudo /bin/sh -c %s' % shell_escape(cmds) # Gets vagrant SSH parameters info = machine_setup(target) signals.pre_run(target=target) interactive = not (args.no_stdin or os.environ.get('REPROUNZIP_NON_INTERACTIVE')) retcode = run_interactive(info, interactive, cmds, not args.no_pty, x11.port_forward) stderr.write("\r\n*** Command finished, status: %d\r\n" % retcode) # Update input file status metadata_update_run(config, unpacked_info, selected_runs) write_dict(target, unpacked_info) signals.post_run(target=target, retcode=retcode)
def vagrant_setup_create(args): """Sets up the experiment to be run in a Vagrant-built virtual machine. This can either build a chroot or not. If building a chroot, we do just like without Vagrant: we copy all the files and only get what's missing from the host. But we do install automatically the packages whose files are required. If not building a chroot, we install all the packages, and only unpack files that don't come from packages. In short: files from packages with packfiles=True will only be used if building a chroot. """ if not args.pack: logger.critical("setup/create needs the pack filename") sys.exit(1) pack = Path(args.pack[0]) target = Path(args.target[0]) if target.exists(): logger.critical("Target directory exists") sys.exit(1) use_chroot = args.use_chroot mount_bind = args.bind_magic_dirs record_usage(use_chroot=use_chroot, mount_bind=mount_bind) signals.pre_setup(target=target, pack=pack) # Unpacks configuration file rpz_pack = RPZPack(pack) rpz_pack.extract_config(target / 'config.yml') # Loads config runs, packages, other_files = config = load_config(target / 'config.yml', True) if not args.memory: memory = None else: try: memory = int(args.memory[-1]) except ValueError: logger.critical("Invalid value for memory size: %r", args.memory) sys.exit(1) ports = parse_ports(args.expose_port) if args.base_image and args.base_image[0]: record_usage(vagrant_explicit_image=True) box = args.base_image[0] if args.distribution: target_distribution = args.distribution[0] else: target_distribution = None else: target_distribution, box = select_box(runs, gui=args.gui) logger.info("Using box %s", box) logger.debug("Distribution: %s", target_distribution or "unknown") # If using chroot, we might still need to install packages to get missing # (not packed) files if use_chroot: packages = [pkg for pkg in packages if not pkg.packfiles] if packages: record_usage(vagrant_install_pkgs=True) logger.info("Some packages were not packed, so we'll install and " "copy their files\n" "Packages that are missing:\n%s", ' '.join(pkg.name for pkg in packages)) if packages: try: installer = select_installer(pack, runs, target_distribution) except CantFindInstaller as e: logger.error("Need to install %d packages but couldn't select a " "package installer: %s", len(packages), e) target.mkdir(parents=True) try: # Writes setup script logger.info("Writing setup script %s...", target / 'setup.sh') with (target / 'setup.sh').open('w', encoding='utf-8', newline='\n') as fp: fp.write('#!/bin/sh\n\nset -e\n\n') if packages: # Updates package sources update_script = installer.update_script() if update_script: fp.write(update_script) fp.write('\n') # Installs necessary packages fp.write(installer.install_script(packages)) fp.write('\n') # TODO : Compare package versions (painful because of sh) # Untar if use_chroot: fp.write('\n' 'mkdir /experimentroot; cd /experimentroot\n') fp.write('tar zpxf /vagrant/data.tgz --numeric-owner ' '--strip=1 %s\n' % rpz_pack.data_prefix) if mount_bind: fp.write('\n' 'mkdir -p /experimentroot/dev\n' 'mkdir -p /experimentroot/proc\n') for pkg in packages: fp.write('\n# Copies files from package %s\n' % pkg.name) for f in pkg.files: f = f.path dest = join_root(PosixPath('/experimentroot'), f) fp.write('mkdir -p %s\n' % shell_escape(unicode_(f.parent))) fp.write('cp -L %s %s\n' % ( shell_escape(unicode_(f)), shell_escape(unicode_(dest)))) fp.write( '\n' 'cp /etc/resolv.conf /experimentroot/etc/resolv.conf\n') else: fp.write('\ncd /\n') paths = set() pathlist = [] # Adds intermediate directories, and checks for existence in # the tar logger.info("Generating file list...") data_files = rpz_pack.data_filenames() for f in other_files: if f.path.name == 'resolv.conf' and ( f.path.lies_under('/etc') or f.path.lies_under('/run') or f.path.lies_under('/var')): continue path = PosixPath('/') for c in rpz_pack.remove_data_prefix(f.path).components: path = path / c if path in paths: continue paths.add(path) if path in data_files: pathlist.append(path) else: logger.info("Missing file %s", path) # FIXME : for some reason we need reversed() here, I'm not sure # why. Need to read more of tar's docs. # TAR bug: --no-overwrite-dir removes --keep-old-files # TAR bug: there is no way to make --keep-old-files not report # an error if an existing file is encountered. --skip-old-files # was introduced too recently. Instead, we just ignore the exit # status with (target / 'rpz-files.list').open('wb') as lfp: for p in reversed(pathlist): lfp.write(join_root(rpz_pack.data_prefix, p).path) lfp.write(b'\0') fp.write('tar zpxf /vagrant/data.tgz --keep-old-files ' '--numeric-owner --strip=1 ' '--null -T /vagrant/rpz-files.list || /bin/true\n') # Copies busybox if use_chroot: arch = runs[0]['architecture'] download_file(busybox_url(arch), target / 'busybox', 'busybox-%s' % arch) fp.write(r''' cp /vagrant/busybox /experimentroot/busybox chmod +x /experimentroot/busybox mkdir -p /experimentroot/bin [ -e /experimentroot/bin/sh ] || \ ln -s /busybox /experimentroot/bin/sh ''') # Copies pack logger.info("Copying pack file...") rpz_pack.copy_data_tar(target / 'data.tgz') rpz_pack.close() # Meta-data for reprounzip metadata = metadata_initial_iofiles(config, {'use_chroot': use_chroot, 'gui': args.gui, 'ports': ports, 'box': box, 'memory': memory}) write_dict(target, metadata) # Writes Vagrant file write_vagrantfile(target, metadata) signals.post_setup(target=target, pack=pack) except Exception: target.rmtree(ignore_errors=True) raise