def should_restore_owner(param): """Computes whether to restore original files' owners. """ if os.getuid() != 0: if param is True: # Restoring the owner was explicitely requested logging.critical("Not running as root, cannot restore files' " "owner/group as requested") sys.exit(1) elif param is None: # Nothing was requested logging.warning("Not running as root, won't restore files' " "owner/group") ret = False else: # If False: skip warning ret = False else: if param is None: # Nothing was requested logging.info("Running as root, we will restore files' " "owner/group") ret = True elif param is True: ret = True else: # If False: skip warning ret = False record_usage(restore_owner=ret) return ret
def should_restore_owner(param): """Computes whether to restore original files' owners. """ if os.getuid() != 0: if param is True: # Restoring the owner was explicitely requested logger.critical("Not running as root, cannot restore files' " "owner/group as requested") sys.exit(1) elif param is None: # Nothing was requested logger.warning("Not running as root, won't restore files' " "owner/group") ret = False else: # If False: skip warning ret = False else: if param is None: # Nothing was requested logger.info("Running as root, we will restore files' " "owner/group") ret = True elif param is True: ret = True else: # If False: skip warning ret = False record_usage(restore_owner=ret) return ret
def should_mount_magic_dirs(param): """Computes whether to mount directories inside the chroot. """ if os.getuid() != 0: if param is True: # Restoring the owner was explicitely requested logger.critical("Not running as root, cannot mount /dev and " "/proc") sys.exit(1) elif param is None: # Nothing was requested logger.warning("Not running as root, won't mount /dev and /proc") ret = False else: # If False: skip warning ret = False else: if param is None: # Nothing was requested logger.info("Running as root, will mount /dev and /proc") ret = True elif param is True: ret = True else: # If False: skip warning ret = False record_usage(mount_magic_dirs=ret) return ret
def should_mount_magic_dirs(param): """Computes whether to mount directories inside the chroot. """ if os.getuid() != 0: if param is True: # Restoring the owner was explicitely requested logging.critical("Not running as root, cannot mount /dev and " "/proc") sys.exit(1) elif param is None: # Nothing was requested logging.warning("Not running as root, won't mount /dev and /proc") ret = False else: # If False: skip warning ret = False else: if param is None: # Nothing was requested logging.info("Running as root, will mount /dev and /proc") ret = True elif param is True: ret = True else: # If False: skip warning ret = False record_usage(mount_magic_dirs=ret) return ret
def select_box(runs): """Selects a box for the experiment, with the correct distribution. """ distribution, version = runs[0]['distribution'] distribution = distribution.lower() architecture = runs[0]['architecture'] record_usage(vagrant_select_box='%s;%s;%s' % (distribution, version, architecture)) if architecture not in ('i686', 'x86_64'): logging.critical("Error: unsupported architecture %s", architecture) sys.exit(1) # Ubuntu if distribution == 'ubuntu': if version == '12.04': if architecture == 'i686': return 'ubuntu', 'hashicorp/precise32' else: # architecture == 'x86_64' return 'ubuntu', 'hashicorp/precise64' if version == '14.04': if architecture == 'i686': return 'ubuntu', 'ubuntu/trusty32' else: # architecture == 'x86_64' return 'ubuntu', 'ubuntu/trusty64' if version != '15.04': logging.warning("using Ubuntu 15.04 'Vivid' instead of '%s'", version) if architecture == 'i686': return 'ubuntu', 'ubuntu/vivid32' else: # architecture == 'x86_64': return 'ubuntu', 'ubuntu/vivid64' # Debian else: if distribution != 'debian': logging.warning("unsupported distribution %s, using Debian", distribution) version = '8' if (version == '7' or version.startswith('7.') or version.startswith('wheezy')): if architecture == 'i686': return 'debian', 'remram/debian-7-i386' else: # architecture == 'x86_64' return 'debian', 'remram/debian-7-amd64' if (version != '8' and not version.startswith('8.') and not version.startswith('jessie')): logging.warning("using Debian 8 'Jessie' instead of '%s'", version) if architecture == 'i686': return 'debian', 'remram/debian-8-i386' else: # architecture == 'x86_64': return 'debian', 'remram/debian-8-amd64'
def select_box(runs): """Selects a box for the experiment, with the correct distribution. """ distribution, version = runs[0]['distribution'] distribution = distribution.lower() architecture = runs[0]['architecture'] record_usage(vagrant_select_box='%s;%s;%s' % (distribution, version, architecture)) if architecture not in ('i686', 'x86_64'): logging.critical("Error: unsupported architecture %s", architecture) sys.exit(1) # Ubuntu if distribution == 'ubuntu': if version == '12.04': if architecture == 'i686': return 'ubuntu', 'hashicorp/precise32' else: # architecture == 'x86_64' return 'ubuntu', 'hashicorp/precise64' if version == '14.04': if architecture == 'i686': return 'ubuntu', 'ubuntu/trusty32' else: # architecture == 'x86_64' return 'ubuntu', 'ubuntu/trusty64' if version != '15.04': logging.warning("using Ubuntu 15.04 'Vivid' instead of '%s'", version) if architecture == 'i686': return 'ubuntu', 'ubuntu/vivid32' else: # architecture == 'x86_64': return 'ubuntu', 'ubuntu/vivid64' # Debian else: if distribution != 'debian': logging.warning("unsupported distribution %s, using Debian", distribution) version = '8' if (version == '7' or version.startswith('7.') or version.startswith('wheezy')): if architecture == 'i686': return 'debian', 'remram/debian-7-i386' else: # architecture == 'x86_64' return 'debian', 'remram/debian-7-amd64' if (version != '8' and not version.startswith('8.') and not version.startswith('jessie')): logging.warning("using Debian 8 'Jessie' instead of '%s'", version) if architecture == 'i686': return 'debian', 'remram/debian-8-i386' else: # architecture == 'x86_64': return 'debian', 'remram/debian-8-amd64'
def select_box(runs, gui=False): """Selects a box for the experiment, with the correct distribution. """ distribution, version = runs[0]['distribution'] distribution = distribution.lower() architecture = runs[0]['architecture'] record_usage(vagrant_select_box='%s;%s;%s;gui=%s' % (distribution, version, architecture, gui)) if architecture not in ('i686', 'x86_64'): logging.critical("Error: unsupported architecture %s", architecture) sys.exit(1) def find_distribution(parameter, distribution, version, architecture): boxes = parameter['boxes'] for distrib in boxes: if re.match(distrib['name'], distribution) is not None: result = find_version(distrib, version, architecture) if result is not None: return result default = parameter['default'] logging.warning("Unsupported distribution '%s', using %s", distribution, default['name']) result = default['architectures'].get(architecture) if result: return default['distribution'], result def find_version(distrib, version, architecture): if version is not None: for box in distrib['versions']: if re.match(box['version'], version) is not None: result = box['architectures'].get(architecture) if result is not None: return box['distribution'], result box = distrib['default'] if version is not None: logging.warning("Using %s instead of '%s'", box['name'], version) result = box['architectures'].get(architecture) if result is not None: return box['distribution'], result result = find_distribution( get_parameter('vagrant_boxes_x' if gui else 'vagrant_boxes'), distribution, version, architecture) if result is None: logging.critical("Error: couldn't find a base box for required " "architecture") sys.exit(1) return result
def select_box(runs, gui=False): """Selects a box for the experiment, with the correct distribution. """ distribution, version = runs[0]['distribution'] distribution = distribution.lower() architecture = runs[0]['architecture'] record_usage(vagrant_select_box='%s;%s;%s;gui=%s' % (distribution, version, architecture, gui)) if architecture not in ('i686', 'x86_64'): logger.critical("Error: unsupported architecture %s", architecture) sys.exit(1) def find_distribution(parameter, distribution, version, architecture): boxes = parameter['boxes'] for distrib in boxes: if re.match(distrib['name'], distribution) is not None: result = find_version(distrib, version, architecture) if result is not None: return result default = parameter['default'] logger.warning("Unsupported distribution '%s', using %s", distribution, default['name']) result = default['architectures'].get(architecture) if result: return default['distribution'], result def find_version(distrib, version, architecture): if version is not None: for box in distrib['versions']: if re.match(box['version'], version) is not None: result = box['architectures'].get(architecture) if result is not None: return box['distribution'], result box = distrib['default'] if version is not None: logger.warning("Using %s instead of '%s'", box['name'], version) result = box['architectures'].get(architecture) if result is not None: return box['distribution'], result result = find_distribution( get_parameter('vagrant_boxes_x' if gui else 'vagrant_boxes'), distribution, version, architecture) if result is None: logger.critical("Error: couldn't find a base box for required " "architecture") sys.exit(1) return result
def select_image(runs): """Selects a base image for the experiment, with the correct distribution. """ distribution, version = runs[0]['distribution'] distribution = distribution.lower() architecture = runs[0]['architecture'] record_usage(docker_select_box='%s;%s;%s' % (distribution, version, architecture)) if architecture == 'i686': logging.info("wanted architecture was i686, but we'll use x86_64 with " "Docker") elif architecture != 'x86_64': logging.error("Error: unsupported architecture %s", architecture) sys.exit(1) # Ubuntu if distribution == 'ubuntu': if version == '12.04': return 'ubuntu', 'ubuntu:12.04' elif version == '14.04': return 'ubuntu', 'ubuntu:14.04' elif version == '14.10': return 'ubuntu', 'ubuntu:14.10' else: if version != '15.04': logging.warning("using Ubuntu 15.04 'Vivid' instead of '%s'", version) return 'ubuntu', 'ubuntu:15.04' # Debian else: if distribution != 'debian': logging.warning("unsupported distribution %s, using Debian", distribution) version = '8' if (version == '6' or version.startswith('6.') or version.startswith('squeeze')): return 'debian', 'debian:squeeze' elif (version == '7' or version.startswith('7.') or version.startswith('wheezy')): return 'debian', 'debian:wheezy' else: if (version != '8' and not version.startswith('8.') and not version.startswith('jessie')): logging.warning("using Debian 8 'Jessie' instead of '%s'", version) return 'debian', 'debian:jessie'
def installpkgs(args): """Installs the necessary packages on the current machine. """ if not THIS_DISTRIBUTION: logger.critical("Not running on Linux") sys.exit(1) pack = args.pack[0] missing = args.missing # Loads config runs, packages, other_files = load_config(pack) try: installer = select_installer(pack, runs) except CantFindInstaller as e: logger.error("Couldn't select a package installer: %s", e) if args.summary: # Print out a list of packages with their status if missing: print("Packages not present in pack:") packages = [pkg for pkg in packages if not pkg.packfiles] else: print("All packages:") pkgs = installer.get_packages_info(packages) for pkg in packages: print(" %s (required version: %s, status: %s)" % (pkg.name, pkg.version, pkgs[pkg.name][1])) else: if missing: # With --missing, ignore packages whose files were packed packages = [pkg for pkg in packages if not pkg.packfiles] # Installs packages record_usage(installpkgs_installing=len(packages)) r, pkgs = installer.install(packages, assume_yes=args.assume_yes) for pkg in packages: req = pkg.version real = pkgs[pkg.name][1] if real == PKG_NOT_INSTALLED: logger.warning("package %s was not installed", pkg.name) else: logger.warning( "version %s of %s was installed, instead of " "%s", real, pkg.name, req) if r != 0: logger.critical("Installer exited with %d", r) sys.exit(r)
def installpkgs(args): """Installs the necessary packages on the current machine. """ if not THIS_DISTRIBUTION: logging.critical("Not running on Linux") sys.exit(1) pack = args.pack[0] missing = args.missing # Loads config runs, packages, other_files = load_config(pack) try: installer = select_installer(pack, runs) except CantFindInstaller as e: logging.error("Couldn't select a package installer: %s", e) if args.summary: # Print out a list of packages with their status if missing: print("Packages not present in pack:") packages = [pkg for pkg in packages if not pkg.packfiles] else: print("All packages:") pkgs = installer.get_packages_info(packages) for pkg in packages: print(" %s (required version: %s, status: %s)" % ( pkg.name, pkg.version, pkgs[pkg.name][1])) else: if missing: # With --missing, ignore packages whose files were packed packages = [pkg for pkg in packages if not pkg.packfiles] # Installs packages record_usage(installpkgs_installing=len(packages)) r, pkgs = installer.install(packages, assume_yes=args.assume_yes) for pkg in packages: req = pkg.version real = pkgs[pkg.name][1] if real == PKG_NOT_INSTALLED: logging.warning("package %s was not installed", pkg.name) else: logging.warning("version %s of %s was installed, instead of " "%s", real, pkg.name, req) if r != 0: logging.critical("Installer exited with %d", r) sys.exit(r)
def select_image(runs): """Selects a base image for the experiment, with the correct distribution. """ distribution, version = runs[0]['distribution'] distribution = distribution.lower() architecture = runs[0]['architecture'] record_usage(docker_select_box='%s;%s;%s' % (distribution, version, architecture)) if architecture == 'i686': logging.info("Wanted architecture was i686, but we'll use x86_64 with " "Docker") elif architecture != 'x86_64': logging.error("Error: unsupported architecture %s", architecture) sys.exit(1) def find_distribution(parameter, distribution, version): images = parameter['images'] for distrib in images: if re.match(distrib['name'], distribution) is not None: result = find_version(distrib, version) if result is not None: return result default = parameter['default'] logging.warning("Unsupported distribution '%s', using %s", distribution, default['name']) return default['distribution'], default['image'] def find_version(distrib, version): if version is not None: for image in distrib['versions']: if re.match(image['version'], version) is not None: return image['distribution'], image['image'] image = distrib['default'] if version is not None: logging.warning("Using %s instead of '%s'", image['name'], version) return image['distribution'], image['image'] return find_distribution(get_parameter('docker_images'), distribution, version)
def select_image(runs): """Selects a base image for the experiment, with the correct distribution. """ distribution, version = runs[0]['distribution'] distribution = distribution.lower() architecture = runs[0]['architecture'] record_usage(docker_select_box='%s;%s;%s' % (distribution, version, architecture)) if architecture == 'i686': logger.info("Wanted architecture was i686, but we'll use x86_64 with " "Docker") elif architecture != 'x86_64': logger.error("Error: unsupported architecture %s", architecture) sys.exit(1) def find_distribution(parameter, distribution, version): images = parameter['images'] for distrib in images: if re.match(distrib['name'], distribution) is not None: result = find_version(distrib, version) if result is not None: return result default = parameter['default'] logger.warning("Unsupported distribution '%s', using %s", distribution, default['name']) return default['distribution'], default['image'] def find_version(distrib, version): if version is not None: for image in distrib['versions']: if re.match(image['version'], version) is not None: return image['distribution'], image['image'] image = distrib['default'] if version is not None: logger.warning("Using %s instead of '%s'", image['name'], version) return image['distribution'], image['image'] return find_distribution(get_parameter('docker_images'), distribution, version)
def do_vistrails(target): """Create a VisTrails workflow that runs the experiment. This is called from signals after an experiment has been setup by any unpacker. """ record_usage(do_vistrails=True) unpacker = signals.unpacker dot_vistrails = Path('~/.vistrails').expand_user() runs, packages, other_files = load_config(target / 'config.yml', canonical=True) for i, run in enumerate(runs): module_name = write_cltools_module(run, dot_vistrails) # Writes VisTrails workflow bundle = target / 'vistrails.vt' logging.info("Writing VisTrails workflow %s...", bundle) vtdir = Path.tempdir(prefix='reprounzip_vistrails_') try: with vtdir.open('w', 'vistrail', encoding='utf-8', newline='\n') as fp: vistrail = VISTRAILS_TEMPLATE cmdline = ' '.join(shell_escape(arg) for arg in run['argv']) vistrail = vistrail.format( date='2014-11-12 15:31:18', unpacker=unpacker, directory=escape_xml(str(target.absolute())), cmdline=escape_xml(cmdline), module_name=module_name, run=i) fp.write(vistrail) with bundle.open('wb') as fp: z = zipfile.ZipFile(fp, 'w') with vtdir.in_dir(): for path in Path('.').recursedir(): z.write(str(path)) z.close() finally: vtdir.rmtree()
def do_vistrails(target): """Create a VisTrails workflow that runs the experiment. This is called from signals after an experiment has been setup by any unpacker. """ record_usage(do_vistrails=True) unpacker = signals.unpacker dot_vistrails = Path('~/.vistrails').expand_user() runs, packages, other_files = load_config(target / 'config.yml', canonical=True) for i, run in enumerate(runs): module_name = write_cltools_module(run, dot_vistrails) # Writes VisTrails workflow bundle = target / 'vistrails.vt' logging.info("Writing VisTrails workflow %s...", bundle) vtdir = Path.tempdir(prefix='reprounzip_vistrails_') try: with vtdir.open('w', 'vistrail', encoding='utf-8', newline='\n') as fp: vistrail = VISTRAILS_TEMPLATE cmdline = ' '.join(shell_escape(arg) for arg in run['argv']) vistrail = vistrail.format(date='2014-11-12 15:31:18', unpacker=unpacker, directory=escape_xml( str(target.absolute())), cmdline=escape_xml(cmdline), module_name=module_name, run=i) fp.write(vistrail) with bundle.open('wb') as fp: z = zipfile.ZipFile(fp, 'w') with vtdir.in_dir(): for path in Path('.').recursedir(): z.write(str(path)) z.close() finally: vtdir.rmtree()
def vagrant_setup_create(args): """Sets up the experiment to be run in a Vagrant-built virtual machine. This can either build a chroot or not. If building a chroot, we do just like without Vagrant: we copy all the files and only get what's missing from the host. But we do install automatically the packages whose files are required. If not building a chroot, we install all the packages, and only unpack files that don't come from packages. In short: files from packages with packfiles=True will only be used if building a chroot. """ if not args.pack: logging.critical("setup/create needs the pack filename") sys.exit(1) pack = Path(args.pack[0]) target = Path(args.target[0]) if target.exists(): logging.critical("Target directory exists") sys.exit(1) use_chroot = args.use_chroot mount_bind = args.bind_magic_dirs record_usage(use_chroot=use_chroot, mount_bind=mount_bind) signals.pre_setup(target=target, pack=pack) # Unpacks configuration file rpz_pack = RPZPack(pack) rpz_pack.extract_config(target / 'config.yml') # Loads config runs, packages, other_files = config = load_config(target / 'config.yml', True) if not args.memory: memory = None else: try: memory = int(args.memory[-1]) except ValueError: logging.critical("Invalid value for memory size: %r", args.memory) sys.exit(1) if args.base_image and args.base_image[0]: record_usage(vagrant_explicit_image=True) box = args.base_image[0] if args.distribution: target_distribution = args.distribution[0] else: target_distribution = None else: target_distribution, box = select_box(runs) logging.info("Using box %s", box) logging.debug("Distribution: %s", target_distribution or "unknown") # If using chroot, we might still need to install packages to get missing # (not packed) files if use_chroot: packages = [pkg for pkg in packages if not pkg.packfiles] if packages: record_usage(vagrant_install_pkgs=True) logging.info("Some packages were not packed, so we'll install and " "copy their files\n" "Packages that are missing:\n%s", ' '.join(pkg.name for pkg in packages)) if packages: try: installer = select_installer(pack, runs, target_distribution) except CantFindInstaller as e: logging.error("Need to install %d packages but couldn't select a " "package installer: %s", len(packages), e) target.mkdir(parents=True) try: # Writes setup script logging.info("Writing setup script %s...", target / 'setup.sh') with (target / 'setup.sh').open('w', encoding='utf-8', newline='\n') as fp: fp.write('#!/bin/sh\n\nset -e\n\n') if packages: # Updates package sources fp.write(installer.update_script()) fp.write('\n') # Installs necessary packages fp.write(installer.install_script(packages)) fp.write('\n') # TODO : Compare package versions (painful because of sh) # Untar if use_chroot: fp.write('\n' 'mkdir /experimentroot; cd /experimentroot\n') fp.write('tar zpxf /vagrant/data.tgz --numeric-owner ' '--strip=1 %s\n' % rpz_pack.data_prefix) if mount_bind: fp.write('\n' 'mkdir -p /experimentroot/dev\n' 'mkdir -p /experimentroot/proc\n') for pkg in packages: fp.write('\n# Copies files from package %s\n' % pkg.name) for f in pkg.files: f = f.path dest = join_root(PosixPath('/experimentroot'), f) fp.write('mkdir -p %s\n' % shell_escape(unicode_(f.parent))) fp.write('cp -L %s %s\n' % ( shell_escape(unicode_(f)), shell_escape(unicode_(dest)))) else: fp.write('\ncd /\n') paths = set() pathlist = [] # Adds intermediate directories, and checks for existence in # the tar for f in other_files: path = PosixPath('/') for c in rpz_pack.remove_data_prefix(f.path).components: path = path / c if path in paths: continue paths.add(path) try: rpz_pack.get_data(path) except KeyError: logging.info("Missing file %s", path) else: pathlist.append(path) # FIXME : for some reason we need reversed() here, I'm not sure # why. Need to read more of tar's docs. # TAR bug: --no-overwrite-dir removes --keep-old-files # TAR bug: there is no way to make --keep-old-files not report # an error if an existing file is encountered. --skip-old-files # was introduced too recently. Instead, we just ignore the exit # status with (target / 'rpz-files.list').open('wb') as lfp: for p in reversed(pathlist): lfp.write(join_root(rpz_pack.data_prefix, p).path) lfp.write(b'\0') fp.write('tar zpxf /vagrant/data.tgz --keep-old-files ' '--numeric-owner --strip=1 ' '--null -T /vagrant/rpz-files.list || /bin/true\n') # Copies busybox if use_chroot: arch = runs[0]['architecture'] download_file(busybox_url(arch), target / 'busybox', 'busybox-%s' % arch) fp.write(r''' cp /vagrant/busybox /experimentroot/busybox chmod +x /experimentroot/busybox mkdir -p /experimentroot/bin [ -e /experimentroot/bin/sh ] || \ ln -s /busybox /experimentroot/bin/sh ''') # Copies pack logging.info("Copying pack file...") rpz_pack.copy_data_tar(target / 'data.tgz') rpz_pack.close() # Writes Vagrant file logging.info("Writing %s...", target / 'Vagrantfile') with (target / 'Vagrantfile').open('w', encoding='utf-8', newline='\n') as fp: # Vagrant header and version fp.write( '# -*- mode: ruby -*-\n' '# vi: set ft=ruby\n\n' 'VAGRANTFILE_API_VERSION = "2"\n\n' 'Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|\n') # Selects which box to install fp.write(' config.vm.box = "%s"\n' % box) # Run the setup script on the virtual machine fp.write(' config.vm.provision "shell", path: "setup.sh"\n') # Memory size if memory is not None: fp.write(' config.vm.provider "virtualbox" do |v|\n' ' v.memory = %d\n' ' end\n' % memory) fp.write('end\n') # Meta-data for reprounzip write_dict(target, metadata_initial_iofiles(config, {'use_chroot': use_chroot})) signals.post_setup(target=target, pack=pack) except Exception: target.rmtree(ignore_errors=True) raise
def do_vistrails(target, pack=None, **kwargs): """Create a VisTrails workflow that runs the experiment. This is called from signals after an experiment has been setup by any unpacker. """ record_usage(do_vistrails=True) config = load_config(target / 'config.yml', canonical=True) # Writes VisTrails workflow bundle = target / 'vistrails.vt' logging.info("Writing VisTrails workflow %s...", bundle) vtdir = Path.tempdir(prefix='reprounzip_vistrails_') ids = IdScope() try: with vtdir.open('w', 'vistrail', encoding='utf-8', newline='\n') as fp: wf = Workflow(fp, ids) # Directory module, refering to this directory d = wf.add_module('%s:Directory' % rpz_id, rpz_version) wf.add_function(d, 'directory', [(directory_sig, str(target.resolve()))]) connect_from = d for i, run in enumerate(config.runs): inputs = sorted(n for n, f in iteritems(config.inputs_outputs) if i in f.read_runs) outputs = sorted(n for n, f in iteritems(config.inputs_outputs) if i in f.write_runs) ports = itertools.chain((('input', p) for p in inputs), (('output', p) for p in outputs)) # Run module r = wf.add_module('%s:Run' % rpz_id, rpz_version) wf.add_function(r, 'cmdline', [ (string_sig, ' '.join(shell_escape(arg) for arg in run['argv']))]) wf.add_function(r, 'run_number', [(integer_sig, i)]) # Port specs for input/output files for type_, name in ports: wf.add_port_spec(r, name, type_, [file_pkg_mod]) # Draw connection wf.connect(connect_from, experiment_sig, 'experiment', r, experiment_sig, 'experiment') connect_from = r wf.close() with bundle.open('wb') as fp: z = zipfile.ZipFile(fp, 'w') with vtdir.in_dir(): for path in Path('.').recursedir(): z.write(str(path)) z.close() finally: vtdir.rmtree()
def docker_setup_create(args): """Sets up the experiment to be run in a Docker-built container. """ pack = Path(args.pack[0]) target = Path(args.target[0]) if target.exists(): logging.critical("Target directory exists") sys.exit(1) signals.pre_setup(target=target, pack=pack) # Unpacks configuration file rpz_pack = RPZPack(pack) rpz_pack.extract_config(target / 'config.yml') # Loads config runs, packages, other_files = load_config(target / 'config.yml', True) if args.base_image: record_usage(docker_explicit_base=True) base_image = args.base_image[0] if args.distribution: target_distribution = args.distribution[0] else: target_distribution = None else: target_distribution, base_image = select_image(runs) logging.info("Using base image %s", base_image) logging.debug("Distribution: %s", target_distribution or "unknown") target.mkdir(parents=True) rpz_pack.copy_data_tar(target / 'data.tgz') arch = runs[0]['architecture'] # Writes Dockerfile logging.info("Writing %s...", target / 'Dockerfile') with (target / 'Dockerfile').open('w', encoding='utf-8', newline='\n') as fp: fp.write('FROM %s\n\n' % base_image) # Installs busybox download_file(busybox_url(arch), target / 'busybox', 'busybox-%s' % arch) fp.write('COPY busybox /busybox\n') # Installs rpzsudo download_file(sudo_url(arch), target / 'rpzsudo', 'rpzsudo-%s' % arch) fp.write('COPY rpzsudo /rpzsudo\n\n') fp.write('COPY data.tgz /reprozip_data.tgz\n\n') fp.write('COPY rpz-files.list /rpz-files.list\n') fp.write('RUN \\\n' ' chmod +x /busybox /rpzsudo && \\\n') if args.install_pkgs: # Install every package through package manager missing_packages = [] else: # Only install packages that were not packed missing_packages = [pkg for pkg in packages if pkg.packfiles] packages = [pkg for pkg in packages if not pkg.packfiles] if packages: record_usage(docker_install_pkgs=True) try: installer = select_installer(pack, runs, target_distribution) except CantFindInstaller as e: logging.error("Need to install %d packages but couldn't " "select a package installer: %s", len(packages), e) sys.exit(1) # Updates package sources fp.write(' %s && \\\n' % installer.update_script()) # Installs necessary packages fp.write(' %s && \\\n' % installer.install_script(packages)) logging.info("Dockerfile will install the %d software packages " "that were not packed", len(packages)) else: record_usage(docker_install_pkgs=False) # Untar paths = set() pathlist = [] # Adds intermediate directories, and checks for existence in the tar missing_files = chain.from_iterable(pkg.files for pkg in missing_packages) for f in chain(other_files, missing_files): path = PosixPath('/') for c in rpz_pack.remove_data_prefix(f.path).components: path = path / c if path in paths: continue paths.add(path) try: rpz_pack.get_data(path) except KeyError: logging.info("Missing file %s", path) else: pathlist.append(path) rpz_pack.close() # FIXME : for some reason we need reversed() here, I'm not sure why. # Need to read more of tar's docs. # TAR bug: --no-overwrite-dir removes --keep-old-files with (target / 'rpz-files.list').open('wb') as lfp: for p in reversed(pathlist): lfp.write(join_root(rpz_pack.data_prefix, p).path) lfp.write(b'\0') fp.write(' cd / && ' '(tar zpxf /reprozip_data.tgz -U --recursive-unlink ' '--numeric-owner --strip=1 --null -T /rpz-files.list || ' '/busybox echo "TAR reports errors, this might or might ' 'not prevent the execution to run")\n') # Meta-data for reprounzip write_dict(target / '.reprounzip', {}) signals.post_setup(target=target, pack=pack)
def run_interactive(ssh_info, interactive, cmd, request_pty, forwarded_ports): """Runs a command on an SSH server. If `interactive` is True, we'll try to find an ``ssh`` executable, falling back to paramiko if it's not found. The terminal handling code is a bit wonky, so using ``ssh`` is definitely a good idea, especially on Windows. Non-interactive commands should run fine. :param ssh_info: dict with `hostname`, `port`, `username`, `key_filename`, passed directly to paramiko :type ssh_info: dict :param interactive: whether to connect local input to the remote process :type interactive: bool :param cmd: command-line to run on the server :type cmd: str :param request_pty: whether to request a PTY from the SSH server :type request_pty: bool :param forwarded_ports: ports to forward back to us; iterable of pairs ``(port_number, connector)`` where `port_number` is the remote port number and `connector` is the connector object used to build the connected socket to forward to on this side :type forwarded_ports: collections.Iterable[(int, object)] """ if interactive: ssh_exe = find_ssh_executable() else: ssh_exe = None if interactive and ssh_exe: record_usage(vagrant_ssh='ssh') args = [ssh_exe, '-t' if request_pty else '-T', # Force allocation of PTY '-o', 'StrictHostKeyChecking=no', # Silently accept host keys '-o', 'UserKnownHostsFile=/dev/null', # Don't store host keys '-i', ssh_info['key_filename'], '-p', '%d' % ssh_info['port']] for remote_port, connector in forwarded_ports: # Remote port will connect to a local port fwd = LocalForwarder(connector) args.append('-R%d:127.0.0.1:%d' % (remote_port, fwd.local_port)) args.append('%s@%s' % (ssh_info['username'], ssh_info['hostname'])) args.append(cmd) return interruptible_call(args) else: record_usage(vagrant_ssh='interactive' if interactive else 'simple') # Connects to the machine ssh = paramiko.SSHClient() ssh.set_missing_host_key_policy(IgnoreMissingKey()) ssh.connect(**ssh_info) # Starts forwarding forwarders = [] for remote_port, connector in forwarded_ports: forwarders.append( SSHForwarder(ssh.get_transport(), remote_port, connector)) chan = ssh.get_transport().open_session() if request_pty: chan.get_pty() # Execute command logging.info("Connected via SSH, running command...") chan.exec_command(cmd) # Get output if interactive: interactive_shell(chan) else: chan.shutdown_write() while True: data = chan.recv(1024) if len(data) == 0: break stdout_bytes.write(data) stdout_bytes.flush() retcode = chan.recv_exit_status() ssh.close() return retcode
def chroot_create(args): """Unpacks the experiment in a folder so it can be run with chroot. All the files in the pack are unpacked; system files are copied only if they were not packed, and busybox is installed if /bin/sh wasn't packed. In addition, input files are put in a tar.gz (so they can be put back after an upload) and the configuration file is extracted. """ if not args.pack: logger.critical("setup/create needs the pack filename") sys.exit(1) pack = Path(args.pack[0]) target = Path(args.target[0]) if target.exists(): logger.critical("Target directory exists") sys.exit(1) if not issubclass(DefaultAbstractPath, PosixPath): logger.critical("Not unpacking on POSIX system") sys.exit(1) signals.pre_setup(target=target, pack=pack) # We can only restore owner/group of files if running as root restore_owner = should_restore_owner(args.restore_owner) # Unpacks configuration file rpz_pack = RPZPack(pack) rpz_pack.extract_config(target / 'config.yml') # Loads config config = load_config_file(target / 'config.yml', True) packages = config.packages target.mkdir() root = (target / 'root').absolute() root.mkdir() try: # Checks that everything was packed packages_not_packed = [pkg for pkg in packages if not pkg.packfiles] if packages_not_packed: record_usage(chroot_missing_pkgs=True) logger.warning("According to configuration, some files were left " "out because they belong to the following " "packages:%s\nWill copy files from HOST SYSTEM", ''.join('\n %s' % pkg for pkg in packages_not_packed)) missing_files = False for pkg in packages_not_packed: for f in pkg.files: path = Path(f.path) if not path.exists(): logger.error( "Missing file %s (from package %s) on host, " "experiment will probably miss it", path, pkg.name) missing_files = True continue dest = join_root(root, path) dest.parent.mkdir(parents=True) if path.is_link(): dest.symlink(path.read_link()) else: path.copy(dest) if restore_owner: stat = path.stat() dest.chown(stat.st_uid, stat.st_gid) if missing_files: record_usage(chroot_mising_files=True) # Unpacks files members = rpz_pack.list_data() for m in members: # Remove 'DATA/' prefix m.name = str(rpz_pack.remove_data_prefix(m.name)) if not restore_owner: uid = os.getuid() gid = os.getgid() for m in members: m.uid = uid m.gid = gid logger.info("Extracting files...") rpz_pack.extract_data(root, members) rpz_pack.close() resolvconf_src = Path('/etc/resolv.conf') if resolvconf_src.exists(): try: resolvconf_src.copy(root / 'etc/resolv.conf') except IOError: pass # Sets up /bin/sh and /usr/bin/env, downloading busybox if necessary sh_path = join_root(root, Path('/bin/sh')) env_path = join_root(root, Path('/usr/bin/env')) if not sh_path.lexists() or not env_path.lexists(): logger.info("Setting up busybox...") busybox_path = join_root(root, Path('/bin/busybox')) busybox_path.parent.mkdir(parents=True) with make_dir_writable(join_root(root, Path('/bin'))): download_file(busybox_url(config.runs[0]['architecture']), busybox_path, 'busybox-%s' % config.runs[0]['architecture']) busybox_path.chmod(0o755) if not sh_path.lexists(): sh_path.parent.mkdir(parents=True) sh_path.symlink('/bin/busybox') if not env_path.lexists(): env_path.parent.mkdir(parents=True) env_path.symlink('/bin/busybox') # Original input files, so upload can restore them input_files = [f.path for f in config.inputs_outputs.values() if f.read_runs] if input_files: logger.info("Packing up original input files...") inputtar = tarfile.open(str(target / 'inputs.tar.gz'), 'w:gz') for ifile in input_files: filename = join_root(root, ifile) if filename.exists(): inputtar.add(str(filename), str(ifile)) inputtar.close() # Meta-data for reprounzip metadata_write(target, metadata_initial_iofiles(config), 'chroot') signals.post_setup(target=target, pack=pack) except Exception: rmtree_fixed(root) raise
def directory_create(args): """Unpacks the experiment in a folder. Only the files that are not part of a package are copied (unless they are missing from the system and were packed). In addition, input files are put in a tar.gz (so they can be put back after an upload) and the configuration file is extracted. """ if not args.pack: logging.critical("setup needs the pack filename") sys.exit(1) pack = Path(args.pack[0]) target = Path(args.target[0]) if target.exists(): logging.critical("Target directory exists") sys.exit(1) if not issubclass(DefaultAbstractPath, PosixPath): logging.critical("Not unpacking on POSIX system") sys.exit(1) signals.pre_setup(target=target, pack=pack) # Unpacks configuration file rpz_pack = RPZPack(pack) rpz_pack.extract_config(target / 'config.yml') # Loads config config = load_config_file(target / 'config.yml', True) packages = config.packages target.mkdir() root = (target / 'root').absolute() # Checks packages missing_files = False for pkg in packages: if pkg.packfiles: continue for f in pkg.files: if not Path(f.path).exists(): logging.error( "Missing file %s (from package %s that wasn't packed) " "on host, experiment will probably miss it.", f, pkg.name) missing_files = True if missing_files: record_usage(directory_missing_pkgs=True) logging.error("Some packages are missing, you should probably install " "them.\nUse 'reprounzip installpkgs -h' for help") root.mkdir() try: # Unpacks files members = rpz_pack.list_data() for m in members: # Remove 'DATA/' prefix m.name = str(rpz_pack.remove_data_prefix(m.name)) # Makes symlink targets relative if m.issym(): linkname = PosixPath(m.linkname) if linkname.is_absolute: m.linkname = join_root(root, PosixPath(m.linkname)).path logging.info("Extracting files...") rpz_pack.extract_data(root, members) rpz_pack.close() # Original input files, so upload can restore them input_files = [f.path for f in itervalues(config.inputs_outputs) if f.read_runs] if input_files: logging.info("Packing up original input files...") inputtar = tarfile.open(str(target / 'inputs.tar.gz'), 'w:gz') for ifile in input_files: filename = join_root(root, ifile) if filename.exists(): inputtar.add(str(filename), str(ifile)) inputtar.close() # Meta-data for reprounzip metadata_write(target, metadata_initial_iofiles(config), 'directory') signals.post_setup(target=target, pack=pack) except Exception: rmtree_fixed(root) raise
def main(): """Entry point when called on the command-line. """ # Locale locale.setlocale(locale.LC_ALL, '') # Parses command-line # General options def add_options(opts): opts.add_argument('--version', action='version', version="reprounzip version %s" % __version__) # Loads plugins for name, func, descr, descr_1 in get_plugins('reprounzip.plugins'): func() parser = RPUZArgumentParser( description="reprounzip is the ReproZip component responsible for " "unpacking and reproducing an experiment previously " "packed with reprozip", epilog="Please report issues to [email protected]") add_options(parser) parser.add_argument('-v', '--verbose', action='count', default=1, dest='verbosity', help="augments verbosity level") subparsers = parser.add_subparsers(title="subcommands", metavar='') # usage_report subcommand parser_stats = subparsers.add_parser( 'usage_report', help="Enables or disables anonymous usage reports") add_options(parser_stats) parser_stats.add_argument('--enable', action='store_true') parser_stats.add_argument('--disable', action='store_true') parser_stats.set_defaults(func=usage_report) # Loads unpackers for name, func, descr, descr_1 in get_plugins('reprounzip.unpackers'): plugin_parser = subparsers.add_parser( name, help=descr_1, description=descr, formatter_class=argparse.RawDescriptionHelpFormatter) add_options(plugin_parser) info = func(plugin_parser) plugin_parser.set_defaults(selected_unpacker=name) if info is None: info = {} unpackers[name] = info signals.pre_parse_args(parser=parser, subparsers=subparsers) args = parser.parse_args() signals.post_parse_args(args=args) if getattr(args, 'func', None) is None: parser.print_help(sys.stderr) sys.exit(2) signals.unpacker = getattr(args, 'selected_unpacker', None) setup_logging('REPROUNZIP', args.verbosity) setup_usage_report('reprounzip', __version__) if hasattr(args, 'selected_unpacker'): record_usage(unpacker=args.selected_unpacker) signals.pre_setup.subscribe(lambda **kw: record_usage(setup=True)) signals.pre_run.subscribe(lambda **kw: record_usage(run=True)) try: try: args.func(args) except UsageError: raise except Exception as e: signals.application_finishing(reason=e) submit_usage_report(result=type(e).__name__) raise else: signals.application_finishing(reason=None) except UsageError: parser.print_help(sys.stderr) sys.exit(2) submit_usage_report(result='success') sys.exit(0)
def chroot_create(args): """Unpacks the experiment in a folder so it can be run with chroot. All the files in the pack are unpacked; system files are copied only if they were not packed, and busybox is installed if /bin/sh wasn't packed. In addition, input files are put in a tar.gz (so they can be put back after an upload) and the configuration file is extracted. """ if not args.pack: logging.critical("setup/create needs the pack filename") sys.exit(1) pack = Path(args.pack[0]) target = Path(args.target[0]) if target.exists(): logging.critical("Target directory exists") sys.exit(1) if DefaultAbstractPath is not PosixPath: logging.critical("Not unpacking on POSIX system") sys.exit(1) signals.pre_setup(target=target, pack=pack) # We can only restore owner/group of files if running as root restore_owner = should_restore_owner(args.restore_owner) # Unpacks configuration file rpz_pack = RPZPack(pack) rpz_pack.extract_config(target / 'config.yml') # Loads config config = load_config_file(target / 'config.yml', True) packages = config.packages target.mkdir() root = (target / 'root').absolute() root.mkdir() try: # Checks that everything was packed packages_not_packed = [pkg for pkg in packages if not pkg.packfiles] if packages_not_packed: record_usage(chroot_missing_pkgs=True) logging.warning("According to configuration, some files were left " "out because they belong to the following " "packages:%s\nWill copy files from HOST SYSTEM", ''.join('\n %s' % pkg for pkg in packages_not_packed)) missing_files = False for pkg in packages_not_packed: for f in pkg.files: path = Path(f.path) if not path.exists(): logging.error( "Missing file %s (from package %s) on host, " "experiment will probably miss it", path, pkg.name) missing_files = True continue dest = join_root(root, path) dest.parent.mkdir(parents=True) if path.is_link(): dest.symlink(path.read_link()) else: path.copy(dest) if restore_owner: stat = path.stat() dest.chown(stat.st_uid, stat.st_gid) if missing_files: record_usage(chroot_mising_files=True) # Unpacks files members = rpz_pack.list_data() for m in members: # Remove 'DATA/' prefix m.name = str(rpz_pack.remove_data_prefix(m.name)) if not restore_owner: uid = os.getuid() gid = os.getgid() for m in members: m.uid = uid m.gid = gid logging.info("Extracting files...") rpz_pack.extract_data(root, members) rpz_pack.close() # Sets up /bin/sh and /usr/bin/env, downloading busybox if necessary sh_path = join_root(root, Path('/bin/sh')) env_path = join_root(root, Path('/usr/bin/env')) if not sh_path.lexists() or not env_path.lexists(): logging.info("Setting up busybox...") busybox_path = join_root(root, Path('/bin/busybox')) busybox_path.parent.mkdir(parents=True) with make_dir_writable(join_root(root, Path('/bin'))): download_file(busybox_url(config.runs[0]['architecture']), busybox_path, 'busybox-%s' % config.runs[0]['architecture']) busybox_path.chmod(0o755) if not sh_path.lexists(): sh_path.parent.mkdir(parents=True) sh_path.symlink('/bin/busybox') if not env_path.lexists(): env_path.parent.mkdir(parents=True) env_path.symlink('/bin/busybox') # Original input files, so upload can restore them input_files = [f.path for f in itervalues(config.inputs_outputs) if f.read_runs] if input_files: logging.info("Packing up original input files...") inputtar = tarfile.open(str(target / 'inputs.tar.gz'), 'w:gz') for ifile in input_files: filename = join_root(root, ifile) if filename.exists(): inputtar.add(str(filename), str(ifile)) inputtar.close() # Meta-data for reprounzip metadata_write(target, metadata_initial_iofiles(config), 'chroot') signals.post_setup(target=target, pack=pack) except Exception: rmtree_fixed(root) raise
def vagrant_setup_create(args): """Sets up the experiment to be run in a Vagrant-built virtual machine. This can either build a chroot or not. If building a chroot, we do just like without Vagrant: we copy all the files and only get what's missing from the host. But we do install automatically the packages whose files are required. If not building a chroot, we install all the packages, and only unpack files that don't come from packages. In short: files from packages with packfiles=True will only be used if building a chroot. """ if not args.pack: logging.critical("setup/create needs the pack filename") sys.exit(1) pack = Path(args.pack[0]) target = Path(args.target[0]) if target.exists(): logging.critical("Target directory exists") sys.exit(1) use_chroot = args.use_chroot mount_bind = args.bind_magic_dirs record_usage(use_chroot=use_chroot, mount_bind=mount_bind) signals.pre_setup(target=target, pack=pack) # Unpacks configuration file rpz_pack = RPZPack(pack) rpz_pack.extract_config(target / 'config.yml') # Loads config runs, packages, other_files = config = load_config(target / 'config.yml', True) if not args.memory: memory = None else: try: memory = int(args.memory[-1]) except ValueError: logging.critical("Invalid value for memory size: %r", args.memory) sys.exit(1) if args.base_image and args.base_image[0]: record_usage(vagrant_explicit_image=True) box = args.base_image[0] if args.distribution: target_distribution = args.distribution[0] else: target_distribution = None else: target_distribution, box = select_box(runs, gui=args.gui) logging.info("Using box %s", box) logging.debug("Distribution: %s", target_distribution or "unknown") # If using chroot, we might still need to install packages to get missing # (not packed) files if use_chroot: packages = [pkg for pkg in packages if not pkg.packfiles] if packages: record_usage(vagrant_install_pkgs=True) logging.info("Some packages were not packed, so we'll install and " "copy their files\n" "Packages that are missing:\n%s", ' '.join(pkg.name for pkg in packages)) if packages: try: installer = select_installer(pack, runs, target_distribution) except CantFindInstaller as e: logging.error("Need to install %d packages but couldn't select a " "package installer: %s", len(packages), e) target.mkdir(parents=True) try: # Writes setup script logging.info("Writing setup script %s...", target / 'setup.sh') with (target / 'setup.sh').open('w', encoding='utf-8', newline='\n') as fp: fp.write('#!/bin/sh\n\nset -e\n\n') if packages: # Updates package sources update_script = installer.update_script() if update_script: fp.write(update_script) fp.write('\n') # Installs necessary packages fp.write(installer.install_script(packages)) fp.write('\n') # TODO : Compare package versions (painful because of sh) # Untar if use_chroot: fp.write('\n' 'mkdir /experimentroot; cd /experimentroot\n') fp.write('tar zpxf /vagrant/data.tgz --numeric-owner ' '--strip=1 %s\n' % rpz_pack.data_prefix) if mount_bind: fp.write('\n' 'mkdir -p /experimentroot/dev\n' 'mkdir -p /experimentroot/proc\n') for pkg in packages: fp.write('\n# Copies files from package %s\n' % pkg.name) for f in pkg.files: f = f.path dest = join_root(PosixPath('/experimentroot'), f) fp.write('mkdir -p %s\n' % shell_escape(unicode_(f.parent))) fp.write('cp -L %s %s\n' % ( shell_escape(unicode_(f)), shell_escape(unicode_(dest)))) fp.write( '\n' 'cp /etc/resolv.conf /experimentroot/etc/resolv.conf\n') else: fp.write('\ncd /\n') paths = set() pathlist = [] # Adds intermediate directories, and checks for existence in # the tar logging.info("Generating file list...") data_files = rpz_pack.data_filenames() for f in other_files: if f.path.name == 'resolv.conf' and ( f.path.lies_under('/etc') or f.path.lies_under('/run') or f.path.lies_under('/var')): continue path = PosixPath('/') for c in rpz_pack.remove_data_prefix(f.path).components: path = path / c if path in paths: continue paths.add(path) if path in data_files: pathlist.append(path) else: logging.info("Missing file %s", path) # FIXME : for some reason we need reversed() here, I'm not sure # why. Need to read more of tar's docs. # TAR bug: --no-overwrite-dir removes --keep-old-files # TAR bug: there is no way to make --keep-old-files not report # an error if an existing file is encountered. --skip-old-files # was introduced too recently. Instead, we just ignore the exit # status with (target / 'rpz-files.list').open('wb') as lfp: for p in reversed(pathlist): lfp.write(join_root(rpz_pack.data_prefix, p).path) lfp.write(b'\0') fp.write('tar zpxf /vagrant/data.tgz --keep-old-files ' '--numeric-owner --strip=1 ' '--null -T /vagrant/rpz-files.list || /bin/true\n') # Copies busybox if use_chroot: arch = runs[0]['architecture'] download_file(busybox_url(arch), target / 'busybox', 'busybox-%s' % arch) fp.write(r''' cp /vagrant/busybox /experimentroot/busybox chmod +x /experimentroot/busybox mkdir -p /experimentroot/bin [ -e /experimentroot/bin/sh ] || \ ln -s /busybox /experimentroot/bin/sh ''') # Copies pack logging.info("Copying pack file...") rpz_pack.copy_data_tar(target / 'data.tgz') rpz_pack.close() # Writes Vagrant file logging.info("Writing %s...", target / 'Vagrantfile') with (target / 'Vagrantfile').open('w', encoding='utf-8', newline='\n') as fp: # Vagrant header and version fp.write( '# -*- mode: ruby -*-\n' '# vi: set ft=ruby\n\n' 'VAGRANTFILE_API_VERSION = "2"\n\n' 'Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|\n') # Selects which box to install fp.write(' config.vm.box = "%s"\n' % box) # Run the setup script on the virtual machine fp.write(' config.vm.provision "shell", path: "setup.sh"\n') # Memory size if memory is not None or args.gui: fp.write(' config.vm.provider "virtualbox" do |v|\n') if memory is not None: fp.write(' v.memory = %d\n' % memory) if args.gui: fp.write(' v.gui = true\n') fp.write(' end\n') fp.write('end\n') # Meta-data for reprounzip write_dict(target, metadata_initial_iofiles(config, {'use_chroot': use_chroot, 'gui': args.gui})) signals.post_setup(target=target, pack=pack) except Exception: target.rmtree(ignore_errors=True) raise
def directory_create(args): """Unpacks the experiment in a folder. Only the files that are not part of a package are copied (unless they are missing from the system and were packed). In addition, input files are put in a tar.gz (so they can be put back after an upload) and the configuration file is extracted. """ if not args.pack: logging.critical("setup needs the pack filename") sys.exit(1) pack = Path(args.pack[0]) target = Path(args.target[0]) if target.exists(): logging.critical("Target directory exists") sys.exit(1) if not issubclass(DefaultAbstractPath, PosixPath): logging.critical("Not unpacking on POSIX system") sys.exit(1) signals.pre_setup(target=target, pack=pack) # Unpacks configuration file tar = tarfile.open(str(pack), 'r:*') member = tar.getmember('METADATA/config.yml') member.name = 'config.yml' tar.extract(member, str(target)) # Loads config runs, packages, other_files = load_config_file(target / 'config.yml', True) target.mkdir() root = (target / 'root').absolute() root.mkdir() # Checks packages missing_files = False for pkg in packages: if pkg.packfiles: continue for f in pkg.files: f = Path(f.path) if not f.exists(): logging.error( "Missing file %s (from package %s that wasn't packed) " "on host, experiment will probably miss it.", f, pkg.name) missing_files = True if missing_files: record_usage(directory_missing_pkgs=True) logging.error("Some packages are missing, you should probably install " "them.\nUse 'reprounzip installpkgs -h' for help") # Unpacks files if any('..' in m.name or m.name.startswith('/') for m in tar.getmembers()): logging.critical("Tar archive contains invalid pathnames") sys.exit(1) members = [m for m in tar.getmembers() if m.name.startswith('DATA/')] for m in members: m.name = m.name[5:] # Makes symlink targets relative for m in members: if not m.issym(): continue linkname = PosixPath(m.linkname) if linkname.is_absolute: m.linkname = join_root(root, PosixPath(m.linkname)).path logging.info("Extracting files...") tar.extractall(str(root), members) tar.close() # Gets library paths lib_dirs = [] p = subprocess.Popen(['/sbin/ldconfig', '-v', '-N'], stdout=subprocess.PIPE) try: for l in p.stdout: if len(l) < 3 or l[0] in (b' ', b'\t'): continue if l.endswith(b':\n'): lib_dirs.append(Path(l[:-2])) finally: p.wait() # Original input files, so upload can restore them if any(run['input_files'] for run in runs): logging.info("Packing up original input files...") inputtar = tarfile.open(str(target / 'inputs.tar.gz'), 'w:gz') for run in runs: for ifile in itervalues(run['input_files']): inputtar.add(str(join_root(root, PosixPath(ifile))), str(PosixPath(ifile))) inputtar.close() # Meta-data for reprounzip write_dict(target / '.reprounzip', {}, 'directory') signals.post_setup(target=target)
def vagrant_setup_create(args): """Sets up the experiment to be run in a Vagrant-built virtual machine. This can either build a chroot or not. If building a chroot, we do just like without Vagrant: we copy all the files and only get what's missing from the host. But we do install automatically the packages whose files are required. If not building a chroot, we install all the packages, and only unpack files that don't come from packages. In short: files from packages with packfiles=True will only be used if building a chroot. """ if not args.pack: logging.critical("setup/create needs the pack filename") sys.exit(1) pack = Path(args.pack[0]) target = Path(args.target[0]) if target.exists(): logging.critical("Target directory exists") sys.exit(1) use_chroot = args.use_chroot mount_bind = args.bind_magic_dirs record_usage(use_chroot=use_chroot, mount_bind=mount_bind) signals.pre_setup(target=target, pack=pack) # Unpacks configuration file tar = tarfile.open(str(pack), 'r:*') member = tar.getmember('METADATA/config.yml') member.name = 'config.yml' tar.extract(member, str(target)) tar.close() # Loads config runs, packages, other_files = load_config(target / 'config.yml', True) if args.base_image and args.base_image[0]: record_usage(vagrant_explicit_image=True) box = args.base_image[0] if args.distribution: target_distribution = args.distribution[0] else: target_distribution = None else: target_distribution, box = select_box(runs) logging.info("Using box %s", box) logging.debug("Distribution: %s", target_distribution or "unknown") # If using chroot, we might still need to install packages to get missing # (not packed) files if use_chroot: packages = [pkg for pkg in packages if not pkg.packfiles] if packages: record_usage(vagrant_install_pkgs=True) logging.info( "Some packages were not packed, so we'll install and " "copy their files\n" "Packages that are missing:\n%s", ' '.join(pkg.name for pkg in packages)) if packages: try: installer = select_installer(pack, runs, target_distribution) except CantFindInstaller as e: logging.error( "Need to install %d packages but couldn't select a " "package installer: %s", len(packages), e) target.mkdir(parents=True) # Writes setup script logging.info("Writing setup script %s...", target / 'setup.sh') with (target / 'setup.sh').open('w', encoding='utf-8', newline='\n') as fp: fp.write('#!/bin/sh\n\nset -e\n\n') if packages: # Updates package sources fp.write(installer.update_script()) fp.write('\n') # Installs necessary packages fp.write(installer.install_script(packages)) fp.write('\n') # TODO : Compare package versions (painful because of sh) # Untar if use_chroot: fp.write('\n' 'mkdir /experimentroot; cd /experimentroot\n') fp.write('tar zpxf /vagrant/experiment.rpz ' '--numeric-owner --strip=1 DATA\n') if mount_bind: fp.write('\n' 'mkdir -p /experimentroot/dev\n' 'mount -o rbind /dev /experimentroot/dev\n' 'mkdir -p /experimentroot/proc\n' 'mount -o rbind /proc /experimentroot/proc\n') for pkg in packages: fp.write('\n# Copies files from package %s\n' % pkg.name) for f in pkg.files: f = f.path dest = join_root(PosixPath('/experimentroot'), f) fp.write('mkdir -p %s\n' % shell_escape(unicode_(f.parent))) fp.write('cp -L %s %s\n' % (shell_escape( unicode_(f)), shell_escape(unicode_(dest)))) else: fp.write('\ncd /\n') paths = set() pathlist = [] dataroot = PosixPath('DATA') # Adds intermediate directories, and checks for existence in the # tar tar = tarfile.open(str(pack), 'r:*') for f in other_files: path = PosixPath('/') for c in f.path.components[1:]: path = path / c if path in paths: continue paths.add(path) datapath = join_root(dataroot, path) try: tar.getmember(str(datapath)) except KeyError: logging.info("Missing file %s", datapath) else: pathlist.append(unicode_(datapath)) tar.close() # FIXME : for some reason we need reversed() here, I'm not sure # why. Need to read more of tar's docs. # TAR bug: --no-overwrite-dir removes --keep-old-files # TAR bug: there is no way to make --keep-old-files not report an # error if an existing file is encountered. --skip-old-files was # introduced too recently. Instead, we just ignore the exit status fp.write('tar zpxf /vagrant/experiment.rpz --keep-old-files ' '--numeric-owner --strip=1 %s || /bin/true\n' % ' '.join(shell_escape(p) for p in reversed(pathlist))) # Copies /bin/sh + dependencies if use_chroot: url = busybox_url(runs[0]['architecture']) fp.write(r''' mkdir -p /experimentroot/bin mkdir -p /experimentroot/usr/bin if [ ! -e /experimentroot/bin/sh -o ! -e /experimentroot/usr/bin/env ]; then wget --quiet -O /experimentroot/bin/busybox {url} chmod +x /experimentroot/bin/busybox fi [ -e /experimentroot/bin/sh ] || \ ln -s /bin/busybox /experimentroot/bin/sh [ -e /experimentroot/usr/bin/env ] || \ ln -s /bin/busybox /experimentroot/usr/bin/env '''.format(url=url)) # Copies pack logging.info("Copying pack file...") pack.copyfile(target / 'experiment.rpz') # Writes Vagrant file logging.info("Writing %s...", target / 'Vagrantfile') with (target / 'Vagrantfile').open('w', encoding='utf-8', newline='\n') as fp: # Vagrant header and version fp.write('# -*- mode: ruby -*-\n' '# vi: set ft=ruby\n\n' 'VAGRANTFILE_API_VERSION = "2"\n\n' 'Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|\n') # Selects which box to install fp.write(' config.vm.box = "%s"\n' % box) # Run the setup script on the virtual machine fp.write(' config.vm.provision "shell", path: "setup.sh"\n') fp.write('end\n') # Meta-data for reprounzip write_dict(target / '.reprounzip', {'use_chroot': use_chroot}) signals.post_setup(target=target)
def main(): """Entry point when called on the command-line. """ # Locale locale.setlocale(locale.LC_ALL, '') # Parses command-line # General options def add_options(opts): opts.add_argument('--version', action='version', version="reprounzip version %s" % __version__) # Loads plugins for name, func, descr, descr_1 in get_plugins('reprounzip.plugins'): func() parser = RPUZArgumentParser( description="reprounzip is the ReproZip component responsible for " "unpacking and reproducing an experiment previously " "packed with reprozip", epilog="Please report issues to [email protected]") add_options(parser) parser.add_argument('-v', '--verbose', action='count', default=1, dest='verbosity', help="augments verbosity level") subparsers = parser.add_subparsers(title="subcommands", metavar='') # usage_report subcommand parser_stats = subparsers.add_parser( 'usage_report', help="Enables or disables anonymous usage reports") add_options(parser_stats) parser_stats.add_argument('--enable', action='store_true') parser_stats.add_argument('--disable', action='store_true') parser_stats.set_defaults(func=usage_report) # Loads unpackers for name, func, descr, descr_1 in get_plugins('reprounzip.unpackers'): plugin_parser = subparsers.add_parser( name, help=descr_1, description=descr, formatter_class=argparse.RawDescriptionHelpFormatter) add_options(plugin_parser) info = func(plugin_parser) plugin_parser.set_defaults(selected_unpacker=name) if info is None: info = {} unpackers[name] = info signals.pre_parse_args(parser=parser, subparsers=subparsers) args = parser.parse_args() signals.post_parse_args(args=args) if getattr(args, 'func', None) is None: parser.print_help(sys.stderr) sys.exit(2) signals.unpacker = getattr(args, 'selected_unpacker', None) setup_logging('REPROUNZIP', args.verbosity) setup_usage_report('reprounzip', __version__) if hasattr(args, 'selected_unpacker'): record_usage(unpacker=args.selected_unpacker) signals.pre_setup.subscribe(lambda **kw: record_usage(setup=True)) signals.pre_run.subscribe(lambda **kw: record_usage(run=True)) try: try: args.func(args) except UsageError: raise except Exception as e: signals.application_finishing(reason=e) submit_usage_report(result=type(e).__name__) raise else: signals.application_finishing(reason=None) except UsageError: parser.print_help(sys.stderr) sys.exit(2) submit_usage_report(result='success') sys.exit(0)
def docker_setup_create(args): """Sets up the experiment to be run in a Docker-built container. """ pack = Path(args.pack[0]) target = Path(args.target[0]) if target.exists(): logging.critical("Target directory exists") sys.exit(1) signals.pre_setup(target=target, pack=pack) target.mkdir() try: # Unpacks configuration file rpz_pack = RPZPack(pack) rpz_pack.extract_config(target / 'config.yml') # Loads config runs, packages, other_files = config = load_config( target / 'config.yml', True) if args.base_image: record_usage(docker_explicit_base=True) base_image = args.base_image[0] if args.distribution: target_distribution = args.distribution[0] else: target_distribution = None else: target_distribution, base_image = select_image(runs) logging.info("Using base image %s", base_image) logging.debug("Distribution: %s", target_distribution or "unknown") rpz_pack.copy_data_tar(target / 'data.tgz') arch = runs[0]['architecture'] # Writes Dockerfile logging.info("Writing %s...", target / 'Dockerfile') with (target / 'Dockerfile').open('w', encoding='utf-8', newline='\n') as fp: fp.write('FROM %s\n\n' % base_image) # Installs busybox download_file(busybox_url(arch), target / 'busybox', 'busybox-%s' % arch) fp.write('COPY busybox /busybox\n') # Installs rpzsudo download_file(sudo_url(arch), target / 'rpzsudo', 'rpzsudo-%s' % arch) fp.write('COPY rpzsudo /rpzsudo\n\n') fp.write('COPY data.tgz /reprozip_data.tgz\n\n') fp.write('COPY rpz-files.list /rpz-files.list\n') fp.write('RUN \\\n' ' chmod +x /busybox /rpzsudo && \\\n') if args.install_pkgs: # Install every package through package manager missing_packages = [] else: # Only install packages that were not packed missing_packages = [pkg for pkg in packages if pkg.packfiles] packages = [pkg for pkg in packages if not pkg.packfiles] if packages: record_usage(docker_install_pkgs=True) try: installer = select_installer(pack, runs, target_distribution) except CantFindInstaller as e: logging.error( "Need to install %d packages but couldn't " "select a package installer: %s", len(packages), e) sys.exit(1) # Updates package sources update_script = installer.update_script() if update_script: fp.write(' %s && \\\n' % update_script) # Installs necessary packages fp.write(' %s && \\\n' % installer.install_script(packages)) logging.info( "Dockerfile will install the %d software " "packages that were not packed", len(packages)) else: record_usage(docker_install_pkgs=False) # Untar paths = set() pathlist = [] # Add intermediate directories, and check for existence in the tar missing_files = chain.from_iterable(pkg.files for pkg in missing_packages) data_files = rpz_pack.data_filenames() listoffiles = list(chain(other_files, missing_files)) for f in listoffiles: path = PosixPath('/') for c in rpz_pack.remove_data_prefix(f.path).components: path = path / c if path in paths: continue paths.add(path) if path in data_files: pathlist.append(path) else: logging.info("Missing file %s", path) rpz_pack.close() # FIXME : for some reason we need reversed() here, I'm not sure why # Need to read more of tar's docs. # TAR bug: --no-overwrite-dir removes --keep-old-files with (target / 'rpz-files.list').open('wb') as lfp: for p in reversed(pathlist): lfp.write(join_root(rpz_pack.data_prefix, p).path) lfp.write(b'\0') fp.write(' cd / && ' '(tar zpxf /reprozip_data.tgz -U --recursive-unlink ' '--numeric-owner --strip=1 --null -T /rpz-files.list || ' '/busybox echo "TAR reports errors, this might or might ' 'not prevent the execution to run")\n') # Meta-data for reprounzip write_dict(target, metadata_initial_iofiles(config)) signals.post_setup(target=target, pack=pack) except Exception: target.rmtree(ignore_errors=True) raise
def docker_setup_create(args): """Sets up the experiment to be run in a Docker-built container. """ pack = Path(args.pack[0]) target = Path(args.target[0]) if target.exists(): logging.critical("Target directory exists") sys.exit(1) signals.pre_setup(target=target, pack=pack) # Unpacks configuration file tar = tarfile.open(str(pack), 'r:*') member = tar.getmember('METADATA/config.yml') member.name = 'config.yml' tar.extract(member, str(target)) tar.close() # Loads config runs, packages, other_files = load_config(target / 'config.yml', True) if args.base_image: record_usage(docker_explicit_base=True) base_image = args.base_image[0] if args.distribution: target_distribution = args.distribution[0] else: target_distribution = None else: target_distribution, base_image = select_image(runs) logging.info("Using base image %s", base_image) logging.debug("Distribution: %s", target_distribution or "unknown") target.mkdir(parents=True) pack.copyfile(target / 'experiment.rpz') # Writes Dockerfile logging.info("Writing %s...", target / 'Dockerfile') with (target / 'Dockerfile').open('w', encoding='utf-8', newline='\n') as fp: fp.write('FROM %s\n\n' % base_image) # Installs busybox download_file(busybox_url(runs[0]['architecture']), target / 'busybox') fp.write('COPY busybox /bin/busybox\n') fp.write('COPY experiment.rpz /reprozip_experiment.rpz\n\n') fp.write('RUN \\\n' ' chmod +x /bin/busybox && \\\n') if args.install_pkgs: # Install every package through package manager missing_packages = [] else: # Only install packages that were not packed missing_packages = [pkg for pkg in packages if pkg.packfiles] packages = [pkg for pkg in packages if not pkg.packfiles] # FIXME : Right now, we need 'sudo' to be available (and it's not # necessarily in the base image) if packages: record_usage(docker_install_pkgs=True) else: record_usage(docker_install_pkgs="sudo") packages += [Package('sudo', None, packfiles=False)] if packages: try: installer = select_installer(pack, runs, target_distribution) except CantFindInstaller as e: logging.error("Need to install %d packages but couldn't " "select a package installer: %s", len(packages), e) sys.exit(1) # Updates package sources fp.write(' %s && \\\n' % installer.update_script()) # Installs necessary packages fp.write(' %s && \\\n' % installer.install_script(packages)) logging.info("Dockerfile will install the %d software packages that " "were not packed", len(packages)) # Untar paths = set() pathlist = [] dataroot = PosixPath('DATA') # Adds intermediate directories, and checks for existence in the tar tar = tarfile.open(str(pack), 'r:*') missing_files = chain.from_iterable(pkg.files for pkg in missing_packages) for f in chain(other_files, missing_files): path = PosixPath('/') for c in f.path.components[1:]: path = path / c if path in paths: continue paths.add(path) datapath = join_root(dataroot, path) try: tar.getmember(str(datapath)) except KeyError: logging.info("Missing file %s", datapath) else: pathlist.append(unicode_(datapath)) tar.close() # FIXME : for some reason we need reversed() here, I'm not sure why. # Need to read more of tar's docs. # TAR bug: --no-overwrite-dir removes --keep-old-files fp.write(' cd / && tar zpxf /reprozip_experiment.rpz ' '--numeric-owner --strip=1 %s\n' % ' '.join(shell_escape(p) for p in reversed(pathlist))) # Meta-data for reprounzip write_dict(target / '.reprounzip', {}) signals.post_setup(target=target)
def directory_create(args): """Unpacks the experiment in a folder. Only the files that are not part of a package are copied (unless they are missing from the system and were packed). In addition, input files are put in a tar.gz (so they can be put back after an upload) and the configuration file is extracted. """ if not args.pack: logging.critical("setup needs the pack filename") sys.exit(1) pack = Path(args.pack[0]) target = Path(args.target[0]) if target.exists(): logging.critical("Target directory exists") sys.exit(1) if not issubclass(DefaultAbstractPath, PosixPath): logging.critical("Not unpacking on POSIX system") sys.exit(1) signals.pre_setup(target=target, pack=pack) # Unpacks configuration file tar = tarfile.open(str(pack), 'r:*') member = tar.getmember('METADATA/config.yml') member.name = 'config.yml' tar.extract(member, str(target)) # Loads config runs, packages, other_files = load_config_file(target / 'config.yml', True) target.mkdir() root = (target / 'root').absolute() root.mkdir() # Checks packages missing_files = False for pkg in packages: if pkg.packfiles: continue for f in pkg.files: f = Path(f.path) if not f.exists(): logging.error( "Missing file %s (from package %s that wasn't packed) " "on host, experiment will probably miss it.", f, pkg.name) missing_files = True if missing_files: record_usage(directory_missing_pkgs=True) logging.error( "Some packages are missing, you should probably install " "them.\nUse 'reprounzip installpkgs -h' for help") # Unpacks files if any('..' in m.name or m.name.startswith('/') for m in tar.getmembers()): logging.critical("Tar archive contains invalid pathnames") sys.exit(1) members = [m for m in tar.getmembers() if m.name.startswith('DATA/')] for m in members: m.name = m.name[5:] # Makes symlink targets relative for m in members: if not m.issym(): continue linkname = PosixPath(m.linkname) if linkname.is_absolute: m.linkname = join_root(root, PosixPath(m.linkname)).path logging.info("Extracting files...") tar.extractall(str(root), members) tar.close() # Gets library paths lib_dirs = [] p = subprocess.Popen(['/sbin/ldconfig', '-v', '-N'], stdout=subprocess.PIPE) try: for l in p.stdout: if len(l) < 3 or l[0] in (b' ', b'\t'): continue if l.endswith(b':\n'): lib_dirs.append(Path(l[:-2])) finally: p.wait() # Original input files, so upload can restore them if any(run['input_files'] for run in runs): logging.info("Packing up original input files...") inputtar = tarfile.open(str(target / 'inputs.tar.gz'), 'w:gz') for run in runs: for ifile in itervalues(run['input_files']): inputtar.add(str(join_root(root, PosixPath(ifile))), str(PosixPath(ifile))) inputtar.close() # Meta-data for reprounzip write_dict(target / '.reprounzip', {}, 'directory') signals.post_setup(target=target)
def vagrant_setup_create(args): """Sets up the experiment to be run in a Vagrant-built virtual machine. This can either build a chroot or not. If building a chroot, we do just like without Vagrant: we copy all the files and only get what's missing from the host. But we do install automatically the packages whose files are required. If not building a chroot, we install all the packages, and only unpack files that don't come from packages. In short: files from packages with packfiles=True will only be used if building a chroot. """ if not args.pack: logging.critical("setup/create needs the pack filename") sys.exit(1) pack = Path(args.pack[0]) target = Path(args.target[0]) if target.exists(): logging.critical("Target directory exists") sys.exit(1) use_chroot = args.use_chroot mount_bind = args.bind_magic_dirs record_usage(use_chroot=use_chroot, mount_bind=mount_bind) signals.pre_setup(target=target, pack=pack) # Unpacks configuration file tar = tarfile.open(str(pack), 'r:*') member = tar.getmember('METADATA/config.yml') member.name = 'config.yml' tar.extract(member, str(target)) tar.close() # Loads config runs, packages, other_files = load_config(target / 'config.yml', True) if args.base_image and args.base_image[0]: record_usage(vagrant_explicit_image=True) box = args.base_image[0] if args.distribution: target_distribution = args.distribution[0] else: target_distribution = None else: target_distribution, box = select_box(runs) logging.info("Using box %s", box) logging.debug("Distribution: %s", target_distribution or "unknown") # If using chroot, we might still need to install packages to get missing # (not packed) files if use_chroot: packages = [pkg for pkg in packages if not pkg.packfiles] if packages: record_usage(vagrant_install_pkgs=True) logging.info("Some packages were not packed, so we'll install and " "copy their files\n" "Packages that are missing:\n%s", ' '.join(pkg.name for pkg in packages)) if packages: try: installer = select_installer(pack, runs, target_distribution) except CantFindInstaller as e: logging.error("Need to install %d packages but couldn't select a " "package installer: %s", len(packages), e) target.mkdir(parents=True) # Writes setup script logging.info("Writing setup script %s...", target / 'setup.sh') with (target / 'setup.sh').open('w', encoding='utf-8', newline='\n') as fp: fp.write('#!/bin/sh\n\nset -e\n\n') if packages: # Updates package sources fp.write(installer.update_script()) fp.write('\n') # Installs necessary packages fp.write(installer.install_script(packages)) fp.write('\n') # TODO : Compare package versions (painful because of sh) # Untar if use_chroot: fp.write('\n' 'mkdir /experimentroot; cd /experimentroot\n') fp.write('tar zpxf /vagrant/experiment.rpz ' '--numeric-owner --strip=1 DATA\n') if mount_bind: fp.write('\n' 'mkdir -p /experimentroot/dev\n' 'mount -o rbind /dev /experimentroot/dev\n' 'mkdir -p /experimentroot/proc\n' 'mount -o rbind /proc /experimentroot/proc\n') for pkg in packages: fp.write('\n# Copies files from package %s\n' % pkg.name) for f in pkg.files: f = f.path dest = join_root(PosixPath('/experimentroot'), f) fp.write('mkdir -p %s\n' % shell_escape(unicode_(f.parent))) fp.write('cp -L %s %s\n' % ( shell_escape(unicode_(f)), shell_escape(unicode_(dest)))) else: fp.write('\ncd /\n') paths = set() pathlist = [] dataroot = PosixPath('DATA') # Adds intermediate directories, and checks for existence in the # tar tar = tarfile.open(str(pack), 'r:*') for f in other_files: path = PosixPath('/') for c in f.path.components[1:]: path = path / c if path in paths: continue paths.add(path) datapath = join_root(dataroot, path) try: tar.getmember(str(datapath)) except KeyError: logging.info("Missing file %s", datapath) else: pathlist.append(unicode_(datapath)) tar.close() # FIXME : for some reason we need reversed() here, I'm not sure # why. Need to read more of tar's docs. # TAR bug: --no-overwrite-dir removes --keep-old-files # TAR bug: there is no way to make --keep-old-files not report an # error if an existing file is encountered. --skip-old-files was # introduced too recently. Instead, we just ignore the exit status fp.write('tar zpxf /vagrant/experiment.rpz --keep-old-files ' '--numeric-owner --strip=1 %s || /bin/true\n' % ' '.join(shell_escape(p) for p in reversed(pathlist))) # Copies /bin/sh + dependencies if use_chroot: url = busybox_url(runs[0]['architecture']) fp.write(r''' mkdir -p /experimentroot/bin mkdir -p /experimentroot/usr/bin if [ ! -e /experimentroot/bin/sh -o ! -e /experimentroot/usr/bin/env ]; then wget --quiet -O /experimentroot/bin/busybox {url} chmod +x /experimentroot/bin/busybox fi [ -e /experimentroot/bin/sh ] || \ ln -s /bin/busybox /experimentroot/bin/sh [ -e /experimentroot/usr/bin/env ] || \ ln -s /bin/busybox /experimentroot/usr/bin/env '''.format(url=url)) # Copies pack logging.info("Copying pack file...") pack.copyfile(target / 'experiment.rpz') # Writes Vagrant file logging.info("Writing %s...", target / 'Vagrantfile') with (target / 'Vagrantfile').open('w', encoding='utf-8', newline='\n') as fp: # Vagrant header and version fp.write('# -*- mode: ruby -*-\n' '# vi: set ft=ruby\n\n' 'VAGRANTFILE_API_VERSION = "2"\n\n' 'Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|\n') # Selects which box to install fp.write(' config.vm.box = "%s"\n' % box) # Run the setup script on the virtual machine fp.write(' config.vm.provision "shell", path: "setup.sh"\n') fp.write('end\n') # Meta-data for reprounzip write_dict(target / '.reprounzip', {'use_chroot': use_chroot}) signals.post_setup(target=target)
def run_interactive(ssh_info, interactive, cmd, request_pty, forwarded_ports): """Runs a command on an SSH server. If `interactive` is True, we'll try to find an ``ssh`` executable, falling back to paramiko if it's not found. The terminal handling code is a bit wonky, so using ``ssh`` is definitely a good idea, especially on Windows. Non-interactive commands should run fine. :param ssh_info: dict with `hostname`, `port`, `username`, `key_filename`, passed directly to paramiko :type ssh_info: dict :param interactive: whether to connect local input to the remote process :type interactive: bool :param cmd: command-line to run on the server :type cmd: basestring :param request_pty: whether to request a PTY from the SSH server :type request_pty: bool :param forwarded_ports: ports to forward back to us; iterable of pairs ``(port_number, connector)`` where `port_number` is the remote port number and `connector` is the connector object used to build the connected socket to forward to on this side """ if interactive: ssh_exe = find_ssh_executable() else: ssh_exe = None if interactive and ssh_exe: record_usage(vagrant_ssh='ssh') args = [ ssh_exe, '-t' if request_pty else '-T', # Force allocation of PTY '-o', 'StrictHostKeyChecking=no', # Silently accept host keys '-o', 'UserKnownHostsFile=/dev/null', # Don't store host keys '-i', ssh_info['key_filename'], '-p', '%d' % ssh_info['port'] ] for remote_port, connector in forwarded_ports: # Remote port will connect to a local port fwd = LocalForwarder(connector) args.append('-R%d:127.0.0.1:%d' % (remote_port, fwd.local_port)) args.append('%s@%s' % (ssh_info['username'], ssh_info['hostname'])) args.append(cmd) return interruptible_call(args) else: record_usage(vagrant_ssh='interactive' if interactive else 'simple') # Connects to the machine ssh = paramiko.SSHClient() ssh.set_missing_host_key_policy(IgnoreMissingKey()) ssh.connect(**ssh_info) # Starts forwarding forwarders = [] for remote_port, connector in forwarded_ports: forwarders.append( SSHForwarder(ssh.get_transport(), remote_port, connector)) chan = ssh.get_transport().open_session() if request_pty: chan.get_pty() # Execute command logging.info("Connected via SSH, running command...") chan.exec_command(cmd) # Get output if interactive: interactive_shell(chan) else: chan.shutdown_write() while True: data = chan.recv(1024) if len(data) == 0: break sys.stdout.buffer.write(data) sys.stdout.flush() retcode = chan.recv_exit_status() ssh.close() return retcode
def docker_setup_create(args): """Sets up the experiment to be run in a Docker-built container. """ pack = Path(args.pack[0]) target = Path(args.target[0]) if target.exists(): logging.critical("Target directory exists") sys.exit(1) signals.pre_setup(target=target, pack=pack) target.mkdir() try: # Unpacks configuration file rpz_pack = RPZPack(pack) rpz_pack.extract_config(target / 'config.yml') # Loads config runs, packages, other_files = config = load_config( target / 'config.yml', True) if args.base_image: record_usage(docker_explicit_base=True) base_image = args.base_image[0] if args.distribution: target_distribution = args.distribution[0] else: target_distribution = None else: target_distribution, base_image = select_image(runs) logging.info("Using base image %s", base_image) logging.debug("Distribution: %s", target_distribution or "unknown") rpz_pack.copy_data_tar(target / 'data.tgz') arch = runs[0]['architecture'] # Writes Dockerfile logging.info("Writing %s...", target / 'Dockerfile') with (target / 'Dockerfile').open('w', encoding='utf-8', newline='\n') as fp: fp.write('FROM %s\n\n' % base_image) # Installs busybox download_file(busybox_url(arch), target / 'busybox', 'busybox-%s' % arch) fp.write('COPY busybox /busybox\n') # Installs rpzsudo download_file(sudo_url(arch), target / 'rpzsudo', 'rpzsudo-%s' % arch) fp.write('COPY rpzsudo /rpzsudo\n\n') fp.write('COPY data.tgz /reprozip_data.tgz\n\n') fp.write('COPY rpz-files.list /rpz-files.list\n') fp.write('RUN \\\n' ' chmod +x /busybox /rpzsudo && \\\n') if args.install_pkgs: # Install every package through package manager missing_packages = [] else: # Only install packages that were not packed missing_packages = [pkg for pkg in packages if pkg.packfiles] packages = [pkg for pkg in packages if not pkg.packfiles] if packages: record_usage(docker_install_pkgs=True) try: installer = select_installer(pack, runs, target_distribution) except CantFindInstaller as e: logging.error("Need to install %d packages but couldn't " "select a package installer: %s", len(packages), e) sys.exit(1) # Updates package sources update_script = installer.update_script() if update_script: fp.write(' %s && \\\n' % update_script) # Installs necessary packages fp.write(' %s && \\\n' % installer.install_script(packages)) logging.info("Dockerfile will install the %d software " "packages that were not packed", len(packages)) else: record_usage(docker_install_pkgs=False) # Untar paths = set() pathlist = [] # Add intermediate directories, and check for existence in the tar logging.info("Generating file list...") missing_files = chain.from_iterable(pkg.files for pkg in missing_packages) data_files = rpz_pack.data_filenames() listoffiles = list(chain(other_files, missing_files)) for f in listoffiles: if f.path.name == 'resolv.conf' and ( f.path.lies_under('/etc') or f.path.lies_under('/run') or f.path.lies_under('/var')): continue path = PosixPath('/') for c in rpz_pack.remove_data_prefix(f.path).components: path = path / c if path in paths: continue paths.add(path) if path in data_files: pathlist.append(path) else: logging.info("Missing file %s", path) rpz_pack.close() # FIXME : for some reason we need reversed() here, I'm not sure why # Need to read more of tar's docs. # TAR bug: --no-overwrite-dir removes --keep-old-files with (target / 'rpz-files.list').open('wb') as lfp: for p in reversed(pathlist): lfp.write(join_root(rpz_pack.data_prefix, p).path) lfp.write(b'\0') fp.write(' cd / && ' '(tar zpxf /reprozip_data.tgz -U --recursive-unlink ' '--numeric-owner --strip=1 --null -T /rpz-files.list || ' '/busybox echo "TAR reports errors, this might or might ' 'not prevent the execution to run")\n') # Setup entry point fp.write('COPY rpz_entrypoint.sh /rpz_entrypoint.sh\n' 'ENTRYPOINT ["/busybox", "sh", "/rpz_entrypoint.sh"]\n') # Write entry point script logging.info("Writing %s...", target / 'rpz_entrypoint.sh') with (target / 'rpz_entrypoint.sh').open('w', encoding='utf-8', newline='\n') as fp: # The entrypoint gets some arguments from the run command # By default, it just does all the runs # "run N" executes the run with that number # "cmd STR" sets a replacement command-line for the next run # "do STR" executes a command as-is fp.write( '#!/bin/sh\n' '\n' 'COMMAND=\n' 'ENVVARS=\n' '\n' 'if [ $# = 0 ]; then\n' ' exec /busybox sh /rpz_entrypoint.sh') for nb in irange(len(runs)): fp.write(' run %d' % nb) fp.write( '\n' 'fi\n' '\n' 'while [ $# != 0 ]; do\n' ' case "$1" in\n' ' help)\n' ' echo "Image built from reprounzip-docker" >&2\n' ' echo "Usage: docker run <image> [cmd word [word ' '...]] [run <R>]" >&2\n' ' echo " \\`cmd ...\\` changes the command for ' 'the next \\`run\\` option" >&2\n' ' echo " \\`run <name|number>\\` runs the ' 'specified run" >&2\n' ' echo "By default, all the runs are executed." ' '>&2\n' ' echo "The runs in this image are:" >&2\n') for run in runs: fp.write( ' echo " {name}: {cmdline}" >&2\n'.format( name=run['id'], cmdline=' '.join(shell_escape(a) for a in run['argv']))) fp.write( ' exit 0\n' ' ;;\n' ' do)\n' ' shift\n' ' $1\n' ' ;;\n' ' env)\n' ' shift\n' ' ENVVARS="$1"\n' ' ;;\n' ' cmd)\n' ' shift\n' ' COMMAND="$1"\n' ' ;;\n' ' run)\n' ' shift\n' ' case "$1" in\n') for i, run in enumerate(runs): cmdline = ' '.join([run['binary']] + run['argv'][1:]) fp.write( ' {name})\n' ' RUNCOMMAND={cmd}\n' ' RUNWD={wd}\n' ' RUNENV={env}\n' ' RUNUID={uid}\n' ' RUNGID={gid}\n' ' ;;\n'.format( name='%s|%d' % (run['id'], i), cmd=shell_escape(cmdline), wd=shell_escape(run['workingdir']), env=shell_escape(' '.join( '%s=%s' % (shell_escape(k), shell_escape(v)) for k, v in iteritems(run['environ']))), uid=run.get('uid', 1000), gid=run.get('gid', 1000))) fp.write( ' *)\n' ' echo "RPZ: Unknown run $1" >&2\n' ' exit 1\n' ' ;;\n' ' esac\n' ' if [ -n "$COMMAND" ]; then\n' ' RUNCOMMAND="$COMMAND"\n' ' COMMAND=\n' ' fi\n' ' export RUNWD; export RUNENV; export ENVVARS; ' 'export RUNCOMMAND\n' ' /rpzsudo "#$RUNUID" "#$RUNGID" /busybox sh -c ' '"cd \\"\\$RUNWD\\" && /busybox env -i $RUNENV $ENVVARS ' '$RUNCOMMAND"\n' ' ENVVARS=\n' ' ;;\n' ' *)\n' ' echo "RPZ: Unknown option $1" >&2\n' ' exit 1\n' ' ;;\n' ' esac\n' ' shift\n' 'done\n') # Meta-data for reprounzip write_dict(target, metadata_initial_iofiles(config)) signals.post_setup(target=target, pack=pack) except Exception: target.rmtree(ignore_errors=True) raise
def chroot_create(args): """Unpacks the experiment in a folder so it can be run with chroot. All the files in the pack are unpacked; system files are copied only if they were not packed, and busybox is installed if /bin/sh wasn't packed. In addition, input files are put in a tar.gz (so they can be put back after an upload) and the configuration file is extracted. """ if not args.pack: logging.critical("setup/create needs the pack filename") sys.exit(1) pack = Path(args.pack[0]) target = Path(args.target[0]) if target.exists(): logging.critical("Target directory exists") sys.exit(1) if DefaultAbstractPath is not PosixPath: logging.critical("Not unpacking on POSIX system") sys.exit(1) signals.pre_setup(target=target, pack=pack) # We can only restore owner/group of files if running as root restore_owner = should_restore_owner(args.restore_owner) # Unpacks configuration file tar = tarfile.open(str(pack), 'r:*') member = tar.getmember('METADATA/config.yml') member.name = 'config.yml' tar.extract(member, str(target)) # Loads config runs, packages, other_files = load_config_file(target / 'config.yml', True) target.mkdir() root = (target / 'root').absolute() root.mkdir() # Checks that everything was packed packages_not_packed = [pkg for pkg in packages if not pkg.packfiles] if packages_not_packed: record_usage(chroot_missing_pkgs=True) logging.warning( "According to configuration, some files were left out " "because they belong to the following packages:%s" "\nWill copy files from HOST SYSTEM", ''.join('\n %s' % pkg for pkg in packages_not_packed)) missing_files = False for pkg in packages_not_packed: for f in pkg.files: f = Path(f.path) if not f.exists(): logging.error( "Missing file %s (from package %s) on host, " "experiment will probably miss it", f, pkg.name) missing_files = True continue dest = join_root(root, f) dest.parent.mkdir(parents=True) if f.is_link(): dest.symlink(f.read_link()) else: f.copy(dest) if restore_owner: stat = f.stat() dest.chown(stat.st_uid, stat.st_gid) if missing_files: record_usage(chroot_mising_files=True) # Unpacks files if any('..' in m.name or m.name.startswith('/') for m in tar.getmembers()): logging.critical("Tar archive contains invalid pathnames") sys.exit(1) members = [m for m in tar.getmembers() if m.name.startswith('DATA/')] for m in members: m.name = m.name[5:] if not restore_owner: uid = os.getuid() gid = os.getgid() for m in members: m.uid = uid m.gid = gid logging.info("Extracting files...") tar.extractall(str(root), members) tar.close() # Sets up /bin/sh and /usr/bin/env, downloading busybox if necessary sh_path = join_root(root, Path('/bin/sh')) env_path = join_root(root, Path('/usr/bin/env')) if not sh_path.lexists() or not env_path.lexists(): logging.info("Setting up busybox...") busybox_path = join_root(root, Path('/bin/busybox')) busybox_path.parent.mkdir(parents=True) with make_dir_writable(join_root(root, Path('/bin'))): download_file(busybox_url(runs[0]['architecture']), busybox_path) busybox_path.chmod(0o755) if not sh_path.lexists(): sh_path.parent.mkdir(parents=True) sh_path.symlink('/bin/busybox') if not env_path.lexists(): env_path.parent.mkdir(parents=True) env_path.symlink('/bin/busybox') # Original input files, so upload can restore them if any(run['input_files'] for run in runs): logging.info("Packing up original input files...") inputtar = tarfile.open(str(target / 'inputs.tar.gz'), 'w:gz') for run in runs: for ifile in itervalues(run['input_files']): inputtar.add(str(join_root(root, PosixPath(ifile))), str(PosixPath(ifile))) inputtar.close() # Meta-data for reprounzip write_dict(target / '.reprounzip', {}, 'chroot') signals.post_setup(target=target)
def chroot_create(args): """Unpacks the experiment in a folder so it can be run with chroot. All the files in the pack are unpacked; system files are copied only if they were not packed, and busybox is installed if /bin/sh wasn't packed. In addition, input files are put in a tar.gz (so they can be put back after an upload) and the configuration file is extracted. """ if not args.pack: logging.critical("setup/create needs the pack filename") sys.exit(1) pack = Path(args.pack[0]) target = Path(args.target[0]) if target.exists(): logging.critical("Target directory exists") sys.exit(1) if DefaultAbstractPath is not PosixPath: logging.critical("Not unpacking on POSIX system") sys.exit(1) signals.pre_setup(target=target, pack=pack) # We can only restore owner/group of files if running as root restore_owner = should_restore_owner(args.restore_owner) # Unpacks configuration file tar = tarfile.open(str(pack), 'r:*') member = tar.getmember('METADATA/config.yml') member.name = 'config.yml' tar.extract(member, str(target)) # Loads config runs, packages, other_files = load_config_file(target / 'config.yml', True) target.mkdir() root = (target / 'root').absolute() root.mkdir() # Checks that everything was packed packages_not_packed = [pkg for pkg in packages if not pkg.packfiles] if packages_not_packed: record_usage(chroot_missing_pkgs=True) logging.warning("According to configuration, some files were left out " "because they belong to the following packages:%s" "\nWill copy files from HOST SYSTEM", ''.join('\n %s' % pkg for pkg in packages_not_packed)) missing_files = False for pkg in packages_not_packed: for f in pkg.files: f = Path(f.path) if not f.exists(): logging.error( "Missing file %s (from package %s) on host, " "experiment will probably miss it", f, pkg.name) missing_files = True continue dest = join_root(root, f) dest.parent.mkdir(parents=True) if f.is_link(): dest.symlink(f.read_link()) else: f.copy(dest) if restore_owner: stat = f.stat() dest.chown(stat.st_uid, stat.st_gid) if missing_files: record_usage(chroot_mising_files=True) # Unpacks files if any('..' in m.name or m.name.startswith('/') for m in tar.getmembers()): logging.critical("Tar archive contains invalid pathnames") sys.exit(1) members = [m for m in tar.getmembers() if m.name.startswith('DATA/')] for m in members: m.name = m.name[5:] if not restore_owner: uid = os.getuid() gid = os.getgid() for m in members: m.uid = uid m.gid = gid logging.info("Extracting files...") tar.extractall(str(root), members) tar.close() # Sets up /bin/sh and /usr/bin/env, downloading busybox if necessary sh_path = join_root(root, Path('/bin/sh')) env_path = join_root(root, Path('/usr/bin/env')) if not sh_path.lexists() or not env_path.lexists(): logging.info("Setting up busybox...") busybox_path = join_root(root, Path('/bin/busybox')) busybox_path.parent.mkdir(parents=True) with make_dir_writable(join_root(root, Path('/bin'))): download_file(busybox_url(runs[0]['architecture']), busybox_path) busybox_path.chmod(0o755) if not sh_path.lexists(): sh_path.parent.mkdir(parents=True) sh_path.symlink('/bin/busybox') if not env_path.lexists(): env_path.parent.mkdir(parents=True) env_path.symlink('/bin/busybox') # Original input files, so upload can restore them if any(run['input_files'] for run in runs): logging.info("Packing up original input files...") inputtar = tarfile.open(str(target / 'inputs.tar.gz'), 'w:gz') for run in runs: for ifile in itervalues(run['input_files']): inputtar.add(str(join_root(root, PosixPath(ifile))), str(PosixPath(ifile))) inputtar.close() # Meta-data for reprounzip write_dict(target / '.reprounzip', {}, 'chroot') signals.post_setup(target=target)
def directory_create(args): """Unpacks the experiment in a folder. Only the files that are not part of a package are copied (unless they are missing from the system and were packed). In addition, input files are put in a tar.gz (so they can be put back after an upload) and the configuration file is extracted. """ if not args.pack: logger.critical("setup needs the pack filename") sys.exit(1) pack = Path(args.pack[0]) target = Path(args.target[0]) if target.exists(): logger.critical("Target directory exists") sys.exit(1) if not issubclass(DefaultAbstractPath, PosixPath): logger.critical("Not unpacking on POSIX system") sys.exit(1) signals.pre_setup(target=target, pack=pack) # Unpacks configuration file rpz_pack = RPZPack(pack) rpz_pack.extract_config(target / 'config.yml') # Loads config config = load_config_file(target / 'config.yml', True) packages = config.packages target.mkdir() root = (target / 'root').absolute() # Checks packages missing_files = False for pkg in packages: if pkg.packfiles: continue for f in pkg.files: if not Path(f.path).exists(): logger.error( "Missing file %s (from package %s that wasn't packed) " "on host, experiment will probably miss it.", f, pkg.name) missing_files = True if missing_files: record_usage(directory_missing_pkgs=True) logger.error("Some packages are missing, you should probably install " "them.\nUse 'reprounzip installpkgs -h' for help") root.mkdir() try: # Unpacks files members = rpz_pack.list_data() for m in members: # Remove 'DATA/' prefix m.name = str(rpz_pack.remove_data_prefix(m.name)) # Makes symlink targets relative if m.issym(): linkname = PosixPath(m.linkname) if linkname.is_absolute: m.linkname = join_root(root, PosixPath(m.linkname)).path logger.info("Extracting files...") rpz_pack.extract_data(root, members) rpz_pack.close() # Original input files, so upload can restore them input_files = [f.path for f in config.inputs_outputs.values() if f.read_runs] if input_files: logger.info("Packing up original input files...") inputtar = tarfile.open(str(target / 'inputs.tar.gz'), 'w:gz') for ifile in input_files: filename = join_root(root, ifile) if filename.exists(): inputtar.add(str(filename), str(ifile)) inputtar.close() # Meta-data for reprounzip metadata_write(target, metadata_initial_iofiles(config), 'directory') signals.post_setup(target=target, pack=pack) except Exception: rmtree_fixed(root) raise
def docker_setup_create(args): """Sets up the experiment to be run in a Docker-built container. """ pack = Path(args.pack[0]) target = Path(args.target[0]) if target.exists(): logger.critical("Target directory exists") sys.exit(1) signals.pre_setup(target=target, pack=pack) target.mkdir() try: # Unpacks configuration file rpz_pack = RPZPack(pack) rpz_pack.extract_config(target / 'config.yml') # Loads config runs, packages, other_files = config = load_config( target / 'config.yml', True) if args.base_image: record_usage(docker_explicit_base=True) base_image = args.base_image[0] if args.distribution: target_distribution = args.distribution[0] else: target_distribution = None else: target_distribution, base_image = select_image(runs) logger.info("Using base image %s", base_image) logger.debug("Distribution: %s", target_distribution or "unknown") rpz_pack.copy_data_tar(target / 'data.tgz') arch = runs[0]['architecture'] # Writes Dockerfile logger.info("Writing %s...", target / 'Dockerfile') with (target / 'Dockerfile').open('w', encoding='utf-8', newline='\n') as fp: fp.write('FROM %s\n\n' % base_image) # Installs busybox download_file(busybox_url(arch), target / 'busybox', 'busybox-%s' % arch) fp.write('COPY busybox /busybox\n') # Installs rpzsudo download_file(sudo_url(arch), target / 'rpzsudo', 'rpzsudo-%s' % arch) fp.write('COPY rpzsudo /rpzsudo\n\n') fp.write('COPY data.tgz /reprozip_data.tgz\n\n') fp.write('COPY rpz-files.list /rpz-files.list\n') fp.write('RUN \\\n' ' chmod +x /busybox /rpzsudo && \\\n') if args.install_pkgs: # Install every package through package manager missing_packages = [] else: # Only install packages that were not packed missing_packages = [pkg for pkg in packages if pkg.packfiles] packages = [pkg for pkg in packages if not pkg.packfiles] if packages: record_usage(docker_install_pkgs=True) try: installer = select_installer(pack, runs, target_distribution) except CantFindInstaller as e: logger.error("Need to install %d packages but couldn't " "select a package installer: %s", len(packages), e) sys.exit(1) # Updates package sources update_script = installer.update_script() if update_script: fp.write(' %s && \\\n' % update_script) # Installs necessary packages fp.write(' %s && \\\n' % installer.install_script(packages)) logger.info("Dockerfile will install the %d software " "packages that were not packed", len(packages)) else: record_usage(docker_install_pkgs=False) # Untar paths = set() pathlist = [] # Add intermediate directories, and check for existence in the tar logger.info("Generating file list...") missing_files = chain.from_iterable(pkg.files for pkg in missing_packages) data_files = rpz_pack.data_filenames() listoffiles = list(chain(other_files, missing_files)) for f in listoffiles: if f.path.name == 'resolv.conf' and ( f.path.lies_under('/etc') or f.path.lies_under('/run') or f.path.lies_under('/var')): continue path = PosixPath('/') for c in rpz_pack.remove_data_prefix(f.path).components: path = path / c if path in paths: continue paths.add(path) if path in data_files: pathlist.append(path) else: logger.info("Missing file %s", path) rpz_pack.close() # FIXME : for some reason we need reversed() here, I'm not sure why # Need to read more of tar's docs. # TAR bug: --no-overwrite-dir removes --keep-old-files with (target / 'rpz-files.list').open('wb') as lfp: for p in reversed(pathlist): lfp.write(join_root(rpz_pack.data_prefix, p).path) lfp.write(b'\0') fp.write(' cd / && ' '(tar zpxf /reprozip_data.tgz -U --recursive-unlink ' '--numeric-owner --strip=1 --null -T /rpz-files.list || ' '/busybox echo "TAR reports errors, this might or might ' 'not prevent the execution to run")\n') # Setup entry point fp.write('COPY rpz_entrypoint.sh /rpz_entrypoint.sh\n' 'ENTRYPOINT ["/busybox", "sh", "/rpz_entrypoint.sh"]\n') # Write entry point script logger.info("Writing %s...", target / 'rpz_entrypoint.sh') with (target / 'rpz_entrypoint.sh').open('w', encoding='utf-8', newline='\n') as fp: # The entrypoint gets some arguments from the run command # By default, it just does all the runs # "run N" executes the run with that number # "cmd STR" sets a replacement command-line for the next run # "do STR" executes a command as-is fp.write( '#!/bin/sh\n' '\n' 'COMMAND=\n' 'ENVVARS=\n' '\n' 'if [ $# = 0 ]; then\n' ' exec /busybox sh /rpz_entrypoint.sh') for nb in irange(len(runs)): fp.write(' run %d' % nb) fp.write( '\n' 'fi\n' '\n' 'while [ $# != 0 ]; do\n' ' case "$1" in\n' ' help)\n' ' echo "Image built from reprounzip-docker" >&2\n' ' echo "Usage: docker run <image> [cmd "word [word ' '...]"] [run <R>]" >&2\n' ' echo " \\`cmd ...\\` changes the command for ' 'the next \\`run\\` option" >&2\n' ' echo " \\`run <name|number>\\` runs the ' 'specified run" >&2\n' ' echo "By default, all the runs are executed." ' '>&2\n' ' echo "The runs in this image are:" >&2\n') for run in runs: fp.write( ' echo " {name}: {cmdline}" >&2\n'.format( name=run['id'], cmdline=' '.join(shell_escape(a) for a in run['argv']))) fp.write( ' exit 0\n' ' ;;\n' ' do)\n' ' shift\n' ' $1\n' ' ;;\n' ' env)\n' ' shift\n' ' ENVVARS="$1"\n' ' ;;\n' ' cmd)\n' ' shift\n' ' COMMAND="$1"\n' ' ;;\n' ' run)\n' ' shift\n' ' case "$1" in\n') for i, run in enumerate(runs): cmdline = ' '.join([run['binary']] + run['argv'][1:]) fp.write( ' {name})\n' ' RUNCOMMAND={cmd}\n' ' RUNWD={wd}\n' ' RUNENV={env}\n' ' RUNUID={uid}\n' ' RUNGID={gid}\n' ' ;;\n'.format( name='%s|%d' % (run['id'], i), cmd=shell_escape(cmdline), wd=shell_escape(run['workingdir']), env=shell_escape(' '.join( '%s=%s' % (shell_escape(k), shell_escape(v)) for k, v in iteritems(run['environ']))), uid=run.get('uid', 1000), gid=run.get('gid', 1000))) fp.write( ' *)\n' ' echo "RPZ: Unknown run $1" >&2\n' ' exit 1\n' ' ;;\n' ' esac\n' ' if [ -n "$COMMAND" ]; then\n' ' RUNCOMMAND="$COMMAND"\n' ' COMMAND=\n' ' fi\n' ' export RUNWD; export RUNENV; export ENVVARS; ' 'export RUNCOMMAND\n' ' /rpzsudo "#$RUNUID" "#$RUNGID" /busybox sh -c ' '"cd \\"\\$RUNWD\\" && /busybox env -i $RUNENV $ENVVARS ' '$RUNCOMMAND; echo \\"*** Command finished, status: \\$?\\""\n' ' ENVVARS=\n' ' ;;\n' ' *)\n' ' echo "RPZ: Unknown option $1" >&2\n' ' exit 1\n' ' ;;\n' ' esac\n' ' shift\n' 'done\n') # Meta-data for reprounzip write_dict(target, metadata_initial_iofiles(config)) signals.post_setup(target=target, pack=pack) except Exception: target.rmtree(ignore_errors=True) raise