def chroot_run(args): """Runs the command in the chroot. """ target = Path(args.target[0]) unpacked_info = metadata_read(target, 'chroot') cmdline = args.cmdline # Loads config config = load_config_file(target / 'config.yml', True) runs = config.runs selected_runs = get_runs(runs, args.run, cmdline) root = target / 'root' # X11 handler x11 = X11Handler(args.x11, ('local', socket.gethostname()), args.x11_display) cmds = [] for run_number in selected_runs: run = runs[run_number] cmd = 'cd %s && ' % shell_escape(run['workingdir']) cmd += '/usr/bin/env -i ' environ = x11.fix_env(run['environ']) environ = fixup_environment(environ, args) cmd += ' '.join('%s=%s' % (shell_escape(k), shell_escape(v)) for k, v in iteritems(environ)) cmd += ' ' # FIXME : Use exec -a or something if binary != argv[0] if cmdline is None: argv = [run['binary']] + run['argv'][1:] else: argv = cmdline cmd += ' '.join(shell_escape(a) for a in argv) userspec = '%s:%s' % (run.get('uid', 1000), run.get('gid', 1000)) cmd = 'chroot --userspec=%s %s /bin/sh -c %s' % ( userspec, shell_escape(unicode_(root)), shell_escape(cmd)) cmds.append(cmd) cmds = [ 'chroot %s /bin/sh -c %s' % (shell_escape(unicode_(root)), shell_escape(c)) for c in x11.init_cmds ] + cmds cmds = ' && '.join(cmds) # Starts forwarding forwarders = [] for portnum, connector in x11.port_forward: fwd = LocalForwarder(connector, portnum) forwarders.append(fwd) signals.pre_run(target=target) retcode = interruptible_call(cmds, shell=True) stderr.write("\n*** Command finished, status: %d\n" % retcode) signals.post_run(target=target, retcode=retcode) # Update input file status metadata_update_run(config, unpacked_info, selected_runs) metadata_write(target, unpacked_info, 'chroot')
def showfiles(args): """Writes out the input and output files. Works both for a pack file and for an extracted directory. """ pack = Path(args.pack[0]) if not pack.exists(): logging.critical("Pack or directory %s does not exist", pack) sys.exit(1) if pack.is_dir(): # Reads info from an unpacked directory runs, packages, other_files = load_config_file(pack / 'config.yml', canonical=True) # The '.reprounzip' file is a pickled dictionary, it contains the name # of the files that replaced each input file (if upload was used) with pack.open('rb', '.reprounzip') as fp: unpacked_info = pickle.load(fp) input_files = unpacked_info.get('input_files', {}) print("Input files:") for i, run in enumerate(runs): if len(runs) > 1: print(" Run %d:" % i) for input_name, path in iteritems(run['input_files']): print(" %s (%s)" % (input_name, path)) if input_files.get(input_name) is not None: assigned = PosixPath(input_files[input_name]) else: assigned = "(original)" print(" %s" % assigned) print("Output files:") for i, run in enumerate(runs): if len(runs) > 1: print(" Run %d:" % i) for output_name, path in iteritems(run['output_files']): print(" %s (%s)" % (output_name, path)) else: # pack.is_file() # Reads info from a pack file runs, packages, other_files = load_config(pack) print("Input files:") for i, run in enumerate(runs): if len(runs) > 1: print(" Run %d:" % i) for input_name, path in iteritems(run['input_files']): print(" %s (%s)" % (input_name, path)) print("Output files:") for i, run in enumerate(runs): if len(runs) > 1: print(" Run %d:" % i) for output_name, path in iteritems(run['output_files']): print(" %s (%s)" % (output_name, path))
def chroot_run(args): """Runs the command in the chroot. """ target = Path(args.target[0]) read_dict(target / '.reprounzip', 'chroot') cmdline = args.cmdline # Loads config runs, packages, other_files = load_config_file(target / 'config.yml', True) selected_runs = get_runs(runs, args.run, cmdline) root = target / 'root' # X11 handler x11 = X11Handler(args.x11, ('local', socket.gethostname()), args.x11_display) cmds = [] for run_number in selected_runs: run = runs[run_number] cmd = 'cd %s && ' % shell_escape(run['workingdir']) cmd += '/usr/bin/env -i ' environ = x11.fix_env(run['environ']) cmd += ' '.join('%s=%s' % (k, shell_escape(v)) for k, v in iteritems(environ)) cmd += ' ' # FIXME : Use exec -a or something if binary != argv[0] if cmdline is None: argv = [run['binary']] + run['argv'][1:] else: argv = cmdline cmd += ' '.join(shell_escape(a) for a in argv) userspec = '%s:%s' % (run.get('uid', 1000), run.get('gid', 1000)) cmd = 'chroot --userspec=%s %s /bin/sh -c %s' % ( userspec, shell_escape(unicode_(root)), shell_escape(cmd)) cmds.append(cmd) cmds = ['chroot %s /bin/sh -c %s' % (shell_escape(unicode_(root)), shell_escape(c)) for c in x11.init_cmds] + cmds cmds = ' && '.join(cmds) # Starts forwarding forwarders = [] for portnum, connector in x11.port_forward: fwd = LocalForwarder(connector, portnum) forwarders.append(fwd) signals.pre_run(target=target) retcode = interruptible_call(cmds, shell=True) sys.stderr.write("\n*** Command finished, status: %d\n" % retcode) signals.post_run(target=target, retcode=retcode)
def showfiles(args): """Writes out the input and output files. Works both for a pack file and for an extracted directory. """ pack = Path(args.pack[0]) if not pack.exists(): logging.critical("Pack or directory %s does not exist", pack) sys.exit(1) if pack.is_dir(): # Reads info from an unpacked directory config = load_config_file(pack / 'config.yml', canonical=True) # The '.reprounzip' file is a pickled dictionary, it contains the name # of the files that replaced each input file (if upload was used) with pack.open('rb', '.reprounzip') as fp: unpacked_info = pickle.load(fp) assigned_input_files = unpacked_info.get('input_files', {}) print("Input files:") for input_name, f in iteritems(config.inputs_outputs): if not f.read_runs: continue print(" %s (%s)" % (input_name, f.path)) if assigned_input_files.get(input_name) is not None: assigned = assigned_input_files[input_name] else: assigned = "(original)" print(" %s" % assigned) print("Output files:") for output_name, f in iteritems(config.inputs_outputs): if f.write_runs: print(" %s (%s)" % (output_name, f.path)) else: # pack.is_file() # Reads info from a pack file config = load_config(pack) print("Input files:") for input_name, f in iteritems(config.inputs_outputs): if f.read_runs: print(" %s (%s)" % (input_name, f.path)) print("Output files:") for output_name, f in iteritems(config.inputs_outputs): if f.write_runs: print(" %s (%s)" % (output_name, f.path))
def chroot_run(args): """Runs the command in the chroot. """ target = Path(args.target[0]) read_dict(target / '.reprounzip', 'chroot') cmdline = args.cmdline # Loads config runs, packages, other_files = load_config_file(target / 'config.yml', True) selected_runs = get_runs(runs, args.run, cmdline) root = target / 'root' cmds = [] for run_number in selected_runs: run = runs[run_number] cmd = 'cd %s && ' % shell_escape(run['workingdir']) cmd += '/usr/bin/env -i ' cmd += ' '.join('%s=%s' % (k, shell_escape(v)) for k, v in iteritems(run['environ'])) cmd += ' ' # FIXME : Use exec -a or something if binary != argv[0] if cmdline is None: argv = [run['binary']] + run['argv'][1:] else: argv = cmdline cmd += ' '.join(shell_escape(a) for a in argv) userspec = '%s:%s' % (run.get('uid', 1000), run.get('gid', 1000)) cmd = 'chroot --userspec=%s %s /bin/sh -c %s' % ( userspec, shell_escape(unicode_(root)), shell_escape(cmd)) cmds.append(cmd) cmds = ' && '.join(cmds) signals.pre_run(target=target) retcode = subprocess.call(cmds, shell=True) sys.stderr.write("\n*** Command finished, status: %d\n" % retcode) signals.post_run(target=target, retcode=retcode)
def directory_run(args): """Runs the command in the directory. """ target = Path(args.target[0]) unpacked_info = metadata_read(target, 'directory') cmdline = args.cmdline # Loads config config = load_config_file(target / 'config.yml', True) runs = config.runs selected_runs = get_runs(runs, args.run, cmdline) root = (target / 'root').absolute() # Gets library paths lib_dirs = [] logger.debug("Running: %s", "/sbin/ldconfig -v -N") p = subprocess.Popen(['/sbin/ldconfig', '-v', '-N'], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, _ = p.communicate() try: for line in stdout.splitlines(): if len(line) < 2 or line[0] in (b' ', b'\t'): continue if line.endswith(b':'): lib_dirs.append(Path(line[:-1])) finally: if p.returncode != 0: raise subprocess.CalledProcessError(p.returncode, ['/sbin/ldconfig', '-v', '-N']) lib_dirs = ('export LD_LIBRARY_PATH=%s' % ':'.join( shell_escape(str(join_root(root, d))) for d in lib_dirs)) cmds = [lib_dirs] for run_number in selected_runs: run = runs[run_number] cmd = 'cd %s && ' % shell_escape( str(join_root(root, Path(run['workingdir'])))) cmd += '/usr/bin/env -i ' environ = run['environ'] environ = fixup_environment(environ, args) if args.x11: if 'DISPLAY' in os.environ: environ['DISPLAY'] = os.environ['DISPLAY'] if 'XAUTHORITY' in os.environ: environ['XAUTHORITY'] = os.environ['XAUTHORITY'] cmd += ' '.join('%s=%s' % (shell_escape(k), shell_escape(v)) for k, v in environ.items() if k != 'PATH') cmd += ' ' # PATH # Get the original PATH components path = [PosixPath(d) for d in run['environ'].get('PATH', '').split(':')] # The same paths but in the directory dir_path = [join_root(root, d) for d in path if d.root == '/'] # Rebuild string path = ':'.join(str(d) for d in dir_path + path) cmd += 'PATH=%s ' % shell_escape(path) # FIXME : Use exec -a or something if binary != argv[0] if cmdline is None: argv = run['argv'] # Rewrites command-line arguments that are absolute filenames rewritten = False for i in range(len(argv)): try: p = Path(argv[i]) except UnicodeEncodeError: continue if p.is_absolute: rp = join_root(root, p) if (rp.exists() or (len(rp.components) > 3 and rp.parent.exists())): argv[i] = str(rp) rewritten = True if rewritten: logger.warning("Rewrote command-line as: %s", ' '.join(shell_escape(a) for a in argv)) else: argv = cmdline cmd += ' '.join(shell_escape(a) for a in argv) cmds.append(cmd) cmds = ' && '.join(cmds) signals.pre_run(target=target) logger.debug("Running: %s", cmds) retcode = interruptible_call(cmds, shell=True) print("\n*** Command finished, status: %d\n" % retcode, file=sys.stderr) signals.post_run(target=target, retcode=retcode) # Update input file status metadata_update_run(config, unpacked_info, selected_runs) metadata_write(target, unpacked_info, 'directory')
def chroot_create(args): """Unpacks the experiment in a folder so it can be run with chroot. All the files in the pack are unpacked; system files are copied only if they were not packed, and busybox is installed if /bin/sh wasn't packed. In addition, input files are put in a tar.gz (so they can be put back after an upload) and the configuration file is extracted. """ if not args.pack: logging.critical("setup/create needs the pack filename") sys.exit(1) pack = Path(args.pack[0]) target = Path(args.target[0]) if target.exists(): logging.critical("Target directory exists") sys.exit(1) if DefaultAbstractPath is not PosixPath: logging.critical("Not unpacking on POSIX system") sys.exit(1) signals.pre_setup(target=target, pack=pack) # We can only restore owner/group of files if running as root restore_owner = should_restore_owner(args.restore_owner) # Unpacks configuration file tar = tarfile.open(str(pack), 'r:*') member = tar.getmember('METADATA/config.yml') member.name = 'config.yml' tar.extract(member, str(target)) # Loads config runs, packages, other_files = load_config_file(target / 'config.yml', True) target.mkdir() root = (target / 'root').absolute() root.mkdir() # Checks that everything was packed packages_not_packed = [pkg for pkg in packages if not pkg.packfiles] if packages_not_packed: logging.warning( "According to configuration, some files were left out " "because they belong to the following packages:%s" "\nWill copy files from HOST SYSTEM", ''.join('\n %s' % pkg for pkg in packages_not_packed)) for pkg in packages_not_packed: for f in pkg.files: f = Path(f.path) if not f.exists(): logging.error( "Missing file %s (from package %s) on host, " "experiment will probably miss it", f, pkg.name) dest = join_root(root, f) dest.parent.mkdir(parents=True) if f.is_link(): dest.symlink(f.read_link()) else: f.copy(dest) if restore_owner: stat = f.stat() dest.chown(stat.st_uid, stat.st_gid) # Unpacks files if any('..' in m.name or m.name.startswith('/') for m in tar.getmembers()): logging.critical("Tar archive contains invalid pathnames") sys.exit(1) members = [m for m in tar.getmembers() if m.name.startswith('DATA/')] for m in members: m.name = m.name[5:] if not restore_owner: uid = os.getuid() gid = os.getgid() for m in members: m.uid = uid m.gid = gid logging.info("Extracting files...") tar.extractall(str(root), members) tar.close() # Sets up /bin/sh and /usr/bin/env, downloading busybox if necessary sh_path = join_root(root, Path('/bin/sh')) env_path = join_root(root, Path('/usr/bin/env')) if not sh_path.lexists() or not env_path.lexists(): logging.info("Setting up busybox...") busybox_path = join_root(root, Path('/bin/busybox')) busybox_path.parent.mkdir(parents=True) with make_dir_writable(join_root(root, Path('/bin'))): download_file(busybox_url(runs[0]['architecture']), busybox_path) busybox_path.chmod(0o755) if not sh_path.lexists(): sh_path.parent.mkdir(parents=True) sh_path.symlink('/bin/busybox') if not env_path.lexists(): env_path.parent.mkdir(parents=True) env_path.symlink('/bin/busybox') # Original input files, so upload can restore them if any(run['input_files'] for run in runs): logging.info("Packing up original input files...") inputtar = tarfile.open(str(target / 'inputs.tar.gz'), 'w:gz') for run in runs: for ifile in itervalues(run['input_files']): inputtar.add(str(join_root(root, PosixPath(ifile))), str(PosixPath(ifile))) inputtar.close() # Meta-data for reprounzip write_dict(target / '.reprounzip', {}, 'chroot') signals.post_setup(target=target)
def showfiles(args): """Writes out the input and output files. Works both for a pack file and for an extracted directory. """ def parse_run(runs, s): for i, run in enumerate(runs): if run['id'] == s: return i try: r = int(s) except ValueError: logger.critical("Error: Unknown run %s", s) raise UsageError if r < 0 or r >= len(runs): logger.critical("Error: Expected 0 <= run <= %d, got %d", len(runs) - 1, r) sys.exit(1) return r show_inputs = args.input or not args.output show_outputs = args.output or not args.input def file_filter(fio): if file_filter.run is None: return ((show_inputs and fio.read_runs) or (show_outputs and fio.write_runs)) else: return ((show_inputs and file_filter.run in fio.read_runs) or (show_outputs and file_filter.run in fio.write_runs)) file_filter.run = None pack = Path(args.pack[0]) if not pack.exists(): logger.critical("Pack or directory %s does not exist", pack) sys.exit(1) if pack.is_dir(): # Reads info from an unpacked directory config = load_config_file(pack / 'config.yml', canonical=True) # Filter files by run if args.run is not None: file_filter.run = parse_run(config.runs, args.run) # The '.reprounzip' file is a pickled dictionary, it contains the name # of the files that replaced each input file (if upload was used) unpacked_info = metadata_read(pack, None) assigned_input_files = unpacked_info.get('input_files', {}) if show_inputs: shown = False for input_name, f in sorted(config.inputs_outputs.items()): if f.read_runs and file_filter(f): if not shown: print("Input files:") shown = True if args.verbosity >= 2: print(" %s (%s)" % (input_name, f.path)) else: print(" %s" % input_name) assigned = assigned_input_files.get(input_name) if assigned is None: assigned = "(original)" elif assigned is False: assigned = "(not created)" elif assigned is True: assigned = "(generated)" else: assert isinstance(assigned, (bytes, str)) print(" %s" % assigned) if not shown: print("Input files: none") if show_outputs: shown = False for output_name, f in sorted(config.inputs_outputs.items()): if f.write_runs and file_filter(f): if not shown: print("Output files:") shown = True if args.verbosity >= 2: print(" %s (%s)" % (output_name, f.path)) else: print(" %s" % output_name) if not shown: print("Output files: none") else: # pack.is_file() # Reads info from a pack file config = load_config(pack) # Filter files by run if args.run is not None: file_filter.run = parse_run(config.runs, args.run) if any(f.read_runs for f in config.inputs_outputs.values()): print("Input files:") for input_name, f in sorted(config.inputs_outputs.items()): if f.read_runs and file_filter(f): if args.verbosity >= 2: print(" %s (%s)" % (input_name, f.path)) else: print(" %s" % input_name) else: print("Input files: none") if any(f.write_runs for f in config.inputs_outputs.values()): print("Output files:") for output_name, f in sorted(config.inputs_outputs.items()): if f.write_runs and file_filter(f): if args.verbosity >= 2: print(" %s (%s)" % (output_name, f.path)) else: print(" %s" % output_name) else: print("Output files: none")
def directory_create(args): """Unpacks the experiment in a folder. Only the files that are not part of a package are copied (unless they are missing from the system and were packed). In addition, input files are put in a tar.gz (so they can be put back after an upload) and the configuration file is extracted. """ if not args.pack: logging.critical("setup needs the pack filename") sys.exit(1) pack = Path(args.pack[0]) target = Path(args.target[0]) if target.exists(): logging.critical("Target directory exists") sys.exit(1) if not issubclass(DefaultAbstractPath, PosixPath): logging.critical("Not unpacking on POSIX system") sys.exit(1) signals.pre_setup(target=target, pack=pack) # Unpacks configuration file rpz_pack = RPZPack(pack) rpz_pack.extract_config(target / 'config.yml') # Loads config config = load_config_file(target / 'config.yml', True) packages = config.packages target.mkdir() root = (target / 'root').absolute() # Checks packages missing_files = False for pkg in packages: if pkg.packfiles: continue for f in pkg.files: if not Path(f.path).exists(): logging.error( "Missing file %s (from package %s that wasn't packed) " "on host, experiment will probably miss it.", f, pkg.name) missing_files = True if missing_files: record_usage(directory_missing_pkgs=True) logging.error("Some packages are missing, you should probably install " "them.\nUse 'reprounzip installpkgs -h' for help") root.mkdir() try: # Unpacks files members = rpz_pack.list_data() for m in members: # Remove 'DATA/' prefix m.name = str(rpz_pack.remove_data_prefix(m.name)) # Makes symlink targets relative if m.issym(): linkname = PosixPath(m.linkname) if linkname.is_absolute: m.linkname = join_root(root, PosixPath(m.linkname)).path logging.info("Extracting files...") rpz_pack.extract_data(root, members) rpz_pack.close() # Original input files, so upload can restore them input_files = [f.path for f in itervalues(config.inputs_outputs) if f.read_runs] if input_files: logging.info("Packing up original input files...") inputtar = tarfile.open(str(target / 'inputs.tar.gz'), 'w:gz') for ifile in input_files: filename = join_root(root, ifile) if filename.exists(): inputtar.add(str(filename), str(ifile)) inputtar.close() # Meta-data for reprounzip metadata_write(target, metadata_initial_iofiles(config), 'directory') signals.post_setup(target=target, pack=pack) except Exception: rmtree_fixed(root) raise
def chroot_create(args): """Unpacks the experiment in a folder so it can be run with chroot. All the files in the pack are unpacked; system files are copied only if they were not packed, and busybox is installed if /bin/sh wasn't packed. In addition, input files are put in a tar.gz (so they can be put back after an upload) and the configuration file is extracted. """ if not args.pack: logging.critical("setup/create needs the pack filename") sys.exit(1) pack = Path(args.pack[0]) target = Path(args.target[0]) if target.exists(): logging.critical("Target directory exists") sys.exit(1) if DefaultAbstractPath is not PosixPath: logging.critical("Not unpacking on POSIX system") sys.exit(1) signals.pre_setup(target=target, pack=pack) # We can only restore owner/group of files if running as root restore_owner = should_restore_owner(args.restore_owner) # Unpacks configuration file rpz_pack = RPZPack(pack) rpz_pack.extract_config(target / 'config.yml') # Loads config config = load_config_file(target / 'config.yml', True) packages = config.packages target.mkdir() root = (target / 'root').absolute() root.mkdir() try: # Checks that everything was packed packages_not_packed = [pkg for pkg in packages if not pkg.packfiles] if packages_not_packed: record_usage(chroot_missing_pkgs=True) logging.warning("According to configuration, some files were left " "out because they belong to the following " "packages:%s\nWill copy files from HOST SYSTEM", ''.join('\n %s' % pkg for pkg in packages_not_packed)) missing_files = False for pkg in packages_not_packed: for f in pkg.files: path = Path(f.path) if not path.exists(): logging.error( "Missing file %s (from package %s) on host, " "experiment will probably miss it", path, pkg.name) missing_files = True continue dest = join_root(root, path) dest.parent.mkdir(parents=True) if path.is_link(): dest.symlink(path.read_link()) else: path.copy(dest) if restore_owner: stat = path.stat() dest.chown(stat.st_uid, stat.st_gid) if missing_files: record_usage(chroot_mising_files=True) # Unpacks files members = rpz_pack.list_data() for m in members: # Remove 'DATA/' prefix m.name = str(rpz_pack.remove_data_prefix(m.name)) if not restore_owner: uid = os.getuid() gid = os.getgid() for m in members: m.uid = uid m.gid = gid logging.info("Extracting files...") rpz_pack.extract_data(root, members) rpz_pack.close() # Sets up /bin/sh and /usr/bin/env, downloading busybox if necessary sh_path = join_root(root, Path('/bin/sh')) env_path = join_root(root, Path('/usr/bin/env')) if not sh_path.lexists() or not env_path.lexists(): logging.info("Setting up busybox...") busybox_path = join_root(root, Path('/bin/busybox')) busybox_path.parent.mkdir(parents=True) with make_dir_writable(join_root(root, Path('/bin'))): download_file(busybox_url(config.runs[0]['architecture']), busybox_path, 'busybox-%s' % config.runs[0]['architecture']) busybox_path.chmod(0o755) if not sh_path.lexists(): sh_path.parent.mkdir(parents=True) sh_path.symlink('/bin/busybox') if not env_path.lexists(): env_path.parent.mkdir(parents=True) env_path.symlink('/bin/busybox') # Original input files, so upload can restore them input_files = [f.path for f in itervalues(config.inputs_outputs) if f.read_runs] if input_files: logging.info("Packing up original input files...") inputtar = tarfile.open(str(target / 'inputs.tar.gz'), 'w:gz') for ifile in input_files: filename = join_root(root, ifile) if filename.exists(): inputtar.add(str(filename), str(ifile)) inputtar.close() # Meta-data for reprounzip metadata_write(target, metadata_initial_iofiles(config), 'chroot') signals.post_setup(target=target, pack=pack) except Exception: rmtree_fixed(root) raise
def directory_run(args): """Runs the command in the directory. """ target = Path(args.target[0]) unpacked_info = metadata_read(target, 'directory') cmdline = args.cmdline # Loads config config = load_config_file(target / 'config.yml', True) runs = config.runs selected_runs = get_runs(runs, args.run, cmdline) root = (target / 'root').absolute() # Gets library paths lib_dirs = [] p = subprocess.Popen(['/sbin/ldconfig', '-v', '-N'], stdout=subprocess.PIPE) try: for l in p.stdout: if len(l) < 3 or l[0] in (b' ', b'\t'): continue if l.endswith(b':\n'): lib_dirs.append(Path(l[:-2])) finally: p.communicate() lib_dirs = ('export LD_LIBRARY_PATH=%s' % ':'.join( shell_escape(unicode_(join_root(root, d))) for d in lib_dirs)) cmds = [lib_dirs] for run_number in selected_runs: run = runs[run_number] cmd = 'cd %s && ' % shell_escape( unicode_(join_root(root, Path(run['workingdir'])))) cmd += '/usr/bin/env -i ' environ = run['environ'] environ = fixup_environment(environ, args) if args.x11: if 'DISPLAY' in os.environ: environ['DISPLAY'] = os.environ['DISPLAY'] if 'XAUTHORITY' in os.environ: environ['XAUTHORITY'] = os.environ['XAUTHORITY'] cmd += ' '.join('%s=%s' % (shell_escape(k), shell_escape(v)) for k, v in iteritems(environ) if k != 'PATH') cmd += ' ' # PATH # Get the original PATH components path = [PosixPath(d) for d in run['environ'].get('PATH', '').split(':')] # The same paths but in the directory dir_path = [join_root(root, d) for d in path if d.root == '/'] # Rebuild string path = ':'.join(unicode_(d) for d in dir_path + path) cmd += 'PATH=%s ' % shell_escape(path) # FIXME : Use exec -a or something if binary != argv[0] if cmdline is None: argv = run['argv'] # Rewrites command-line arguments that are absolute filenames rewritten = False for i in irange(len(argv)): try: p = Path(argv[i]) except UnicodeEncodeError: continue if p.is_absolute: rp = join_root(root, p) if (rp.exists() or (len(rp.components) > 3 and rp.parent.exists())): argv[i] = str(rp) rewritten = True if rewritten: logging.warning("Rewrote command-line as: %s", ' '.join(shell_escape(a) for a in argv)) else: argv = cmdline cmd += ' '.join(shell_escape(a) for a in argv) cmds.append(cmd) cmds = ' && '.join(cmds) signals.pre_run(target=target) retcode = interruptible_call(cmds, shell=True) stderr.write("\n*** Command finished, status: %d\n" % retcode) signals.post_run(target=target, retcode=retcode) # Update input file status metadata_update_run(config, unpacked_info, selected_runs) metadata_write(target, unpacked_info, 'directory')
def directory_run(args): """Runs the command in the directory. """ target = Path(args.target[0]) read_dict(target / '.reprounzip', 'directory') cmdline = args.cmdline # Loads config runs, packages, other_files = load_config_file(target / 'config.yml', True) selected_runs = get_runs(runs, args.run, cmdline) root = target / 'root' # Gets library paths lib_dirs = [] p = subprocess.Popen(['/sbin/ldconfig', '-v', '-N'], stdout=subprocess.PIPE) try: for l in p.stdout: if len(l) < 3 or l[0] in (b' ', b'\t'): continue if l.endswith(b':\n'): lib_dirs.append(Path(l[:-2])) finally: p.wait() lib_dirs = ( 'export LD_LIBRARY_PATH=%s' % ':'.join(shell_escape(unicode_(join_root(root, d))) for d in lib_dirs)) cmds = [lib_dirs] for run_number in selected_runs: run = runs[run_number] cmd = 'cd %s && ' % shell_escape( unicode_(join_root(root, Path(run['workingdir'])))) cmd += '/usr/bin/env -i ' cmd += ' '.join('%s=%s' % (k, shell_escape(v)) for k, v in iteritems(run['environ']) if k != 'PATH') cmd += ' ' # PATH # Get the original PATH components path = [ PosixPath(d) for d in run['environ'].get('PATH', '').split(':') ] # The same paths but in the directory dir_path = [join_root(root, d) for d in path if d.root == '/'] # Rebuild string path = ':'.join(unicode_(d) for d in dir_path + path) cmd += 'PATH=%s ' % shell_escape(path) # FIXME : Use exec -a or something if binary != argv[0] if cmdline is None: argv = run['argv'] else: argv = cmdline cmd += ' '.join(shell_escape(a) for a in argv) cmds.append(cmd) cmds = ' && '.join(cmds) signals.pre_run(target=target) retcode = subprocess.call(cmds, shell=True) sys.stderr.write("\n*** Command finished, status: %d\n" % retcode) signals.post_run(target=target, retcode=retcode)
def chroot_create(args): """Unpacks the experiment in a folder so it can be run with chroot. All the files in the pack are unpacked; system files are copied only if they were not packed, and busybox is installed if /bin/sh wasn't packed. In addition, input files are put in a tar.gz (so they can be put back after an upload) and the configuration file is extracted. """ if not args.pack: logger.critical("setup/create needs the pack filename") sys.exit(1) pack = Path(args.pack[0]) target = Path(args.target[0]) if target.exists(): logger.critical("Target directory exists") sys.exit(1) if not issubclass(DefaultAbstractPath, PosixPath): logger.critical("Not unpacking on POSIX system") sys.exit(1) signals.pre_setup(target=target, pack=pack) # We can only restore owner/group of files if running as root restore_owner = should_restore_owner(args.restore_owner) # Unpacks configuration file rpz_pack = RPZPack(pack) rpz_pack.extract_config(target / 'config.yml') # Loads config config = load_config_file(target / 'config.yml', True) packages = config.packages target.mkdir() root = (target / 'root').absolute() root.mkdir() try: # Checks that everything was packed packages_not_packed = [pkg for pkg in packages if not pkg.packfiles] if packages_not_packed: record_usage(chroot_missing_pkgs=True) logger.warning("According to configuration, some files were left " "out because they belong to the following " "packages:%s\nWill copy files from HOST SYSTEM", ''.join('\n %s' % pkg for pkg in packages_not_packed)) missing_files = False for pkg in packages_not_packed: for f in pkg.files: path = Path(f.path) if not path.exists(): logger.error( "Missing file %s (from package %s) on host, " "experiment will probably miss it", path, pkg.name) missing_files = True continue dest = join_root(root, path) dest.parent.mkdir(parents=True) if path.is_link(): dest.symlink(path.read_link()) else: path.copy(dest) if restore_owner: stat = path.stat() dest.chown(stat.st_uid, stat.st_gid) if missing_files: record_usage(chroot_mising_files=True) # Unpacks files members = rpz_pack.list_data() for m in members: # Remove 'DATA/' prefix m.name = str(rpz_pack.remove_data_prefix(m.name)) if not restore_owner: uid = os.getuid() gid = os.getgid() for m in members: m.uid = uid m.gid = gid logger.info("Extracting files...") rpz_pack.extract_data(root, members) rpz_pack.close() resolvconf_src = Path('/etc/resolv.conf') if resolvconf_src.exists(): try: resolvconf_src.copy(root / 'etc/resolv.conf') except IOError: pass # Sets up /bin/sh and /usr/bin/env, downloading busybox if necessary sh_path = join_root(root, Path('/bin/sh')) env_path = join_root(root, Path('/usr/bin/env')) if not sh_path.lexists() or not env_path.lexists(): logger.info("Setting up busybox...") busybox_path = join_root(root, Path('/bin/busybox')) busybox_path.parent.mkdir(parents=True) with make_dir_writable(join_root(root, Path('/bin'))): download_file(busybox_url(config.runs[0]['architecture']), busybox_path, 'busybox-%s' % config.runs[0]['architecture']) busybox_path.chmod(0o755) if not sh_path.lexists(): sh_path.parent.mkdir(parents=True) sh_path.symlink('/bin/busybox') if not env_path.lexists(): env_path.parent.mkdir(parents=True) env_path.symlink('/bin/busybox') # Original input files, so upload can restore them input_files = [f.path for f in config.inputs_outputs.values() if f.read_runs] if input_files: logger.info("Packing up original input files...") inputtar = tarfile.open(str(target / 'inputs.tar.gz'), 'w:gz') for ifile in input_files: filename = join_root(root, ifile) if filename.exists(): inputtar.add(str(filename), str(ifile)) inputtar.close() # Meta-data for reprounzip metadata_write(target, metadata_initial_iofiles(config), 'chroot') signals.post_setup(target=target, pack=pack) except Exception: rmtree_fixed(root) raise
def chroot_create(args): """Unpacks the experiment in a folder so it can be run with chroot. All the files in the pack are unpacked; system files are copied only if they were not packed, and busybox is installed if /bin/sh wasn't packed. In addition, input files are put in a tar.gz (so they can be put back after an upload) and the configuration file is extracted. """ if not args.pack: logging.critical("setup/create needs the pack filename") sys.exit(1) pack = Path(args.pack[0]) target = Path(args.target[0]) if target.exists(): logging.critical("Target directory exists") sys.exit(1) if DefaultAbstractPath is not PosixPath: logging.critical("Not unpacking on POSIX system") sys.exit(1) signals.pre_setup(target=target, pack=pack) # We can only restore owner/group of files if running as root restore_owner = should_restore_owner(args.restore_owner) # Unpacks configuration file tar = tarfile.open(str(pack), 'r:*') member = tar.getmember('METADATA/config.yml') member.name = 'config.yml' tar.extract(member, str(target)) # Loads config runs, packages, other_files = load_config_file(target / 'config.yml', True) target.mkdir() root = (target / 'root').absolute() root.mkdir() # Checks that everything was packed packages_not_packed = [pkg for pkg in packages if not pkg.packfiles] if packages_not_packed: record_usage(chroot_missing_pkgs=True) logging.warning("According to configuration, some files were left out " "because they belong to the following packages:%s" "\nWill copy files from HOST SYSTEM", ''.join('\n %s' % pkg for pkg in packages_not_packed)) missing_files = False for pkg in packages_not_packed: for f in pkg.files: f = Path(f.path) if not f.exists(): logging.error( "Missing file %s (from package %s) on host, " "experiment will probably miss it", f, pkg.name) missing_files = True continue dest = join_root(root, f) dest.parent.mkdir(parents=True) if f.is_link(): dest.symlink(f.read_link()) else: f.copy(dest) if restore_owner: stat = f.stat() dest.chown(stat.st_uid, stat.st_gid) if missing_files: record_usage(chroot_mising_files=True) # Unpacks files if any('..' in m.name or m.name.startswith('/') for m in tar.getmembers()): logging.critical("Tar archive contains invalid pathnames") sys.exit(1) members = [m for m in tar.getmembers() if m.name.startswith('DATA/')] for m in members: m.name = m.name[5:] if not restore_owner: uid = os.getuid() gid = os.getgid() for m in members: m.uid = uid m.gid = gid logging.info("Extracting files...") tar.extractall(str(root), members) tar.close() # Sets up /bin/sh and /usr/bin/env, downloading busybox if necessary sh_path = join_root(root, Path('/bin/sh')) env_path = join_root(root, Path('/usr/bin/env')) if not sh_path.lexists() or not env_path.lexists(): logging.info("Setting up busybox...") busybox_path = join_root(root, Path('/bin/busybox')) busybox_path.parent.mkdir(parents=True) with make_dir_writable(join_root(root, Path('/bin'))): download_file(busybox_url(runs[0]['architecture']), busybox_path) busybox_path.chmod(0o755) if not sh_path.lexists(): sh_path.parent.mkdir(parents=True) sh_path.symlink('/bin/busybox') if not env_path.lexists(): env_path.parent.mkdir(parents=True) env_path.symlink('/bin/busybox') # Original input files, so upload can restore them if any(run['input_files'] for run in runs): logging.info("Packing up original input files...") inputtar = tarfile.open(str(target / 'inputs.tar.gz'), 'w:gz') for run in runs: for ifile in itervalues(run['input_files']): inputtar.add(str(join_root(root, PosixPath(ifile))), str(PosixPath(ifile))) inputtar.close() # Meta-data for reprounzip write_dict(target / '.reprounzip', {}, 'chroot') signals.post_setup(target=target)
def directory_create(args): """Unpacks the experiment in a folder. Only the files that are not part of a package are copied (unless they are missing from the system and were packed). In addition, input files are put in a tar.gz (so they can be put back after an upload) and the configuration file is extracted. """ if not args.pack: logging.critical("setup needs the pack filename") sys.exit(1) pack = Path(args.pack[0]) target = Path(args.target[0]) if target.exists(): logging.critical("Target directory exists") sys.exit(1) if not issubclass(DefaultAbstractPath, PosixPath): logging.critical("Not unpacking on POSIX system") sys.exit(1) signals.pre_setup(target=target, pack=pack) # Unpacks configuration file tar = tarfile.open(str(pack), 'r:*') member = tar.getmember('METADATA/config.yml') member.name = 'config.yml' tar.extract(member, str(target)) # Loads config runs, packages, other_files = load_config_file(target / 'config.yml', True) target.mkdir() root = (target / 'root').absolute() root.mkdir() # Checks packages missing_files = False for pkg in packages: if pkg.packfiles: continue for f in pkg.files: f = Path(f.path) if not f.exists(): logging.error( "Missing file %s (from package %s that wasn't packed) " "on host, experiment will probably miss it.", f, pkg.name) missing_files = True if missing_files: record_usage(directory_missing_pkgs=True) logging.error( "Some packages are missing, you should probably install " "them.\nUse 'reprounzip installpkgs -h' for help") # Unpacks files if any('..' in m.name or m.name.startswith('/') for m in tar.getmembers()): logging.critical("Tar archive contains invalid pathnames") sys.exit(1) members = [m for m in tar.getmembers() if m.name.startswith('DATA/')] for m in members: m.name = m.name[5:] # Makes symlink targets relative for m in members: if not m.issym(): continue linkname = PosixPath(m.linkname) if linkname.is_absolute: m.linkname = join_root(root, PosixPath(m.linkname)).path logging.info("Extracting files...") tar.extractall(str(root), members) tar.close() # Gets library paths lib_dirs = [] p = subprocess.Popen(['/sbin/ldconfig', '-v', '-N'], stdout=subprocess.PIPE) try: for l in p.stdout: if len(l) < 3 or l[0] in (b' ', b'\t'): continue if l.endswith(b':\n'): lib_dirs.append(Path(l[:-2])) finally: p.wait() # Original input files, so upload can restore them if any(run['input_files'] for run in runs): logging.info("Packing up original input files...") inputtar = tarfile.open(str(target / 'inputs.tar.gz'), 'w:gz') for run in runs: for ifile in itervalues(run['input_files']): inputtar.add(str(join_root(root, PosixPath(ifile))), str(PosixPath(ifile))) inputtar.close() # Meta-data for reprounzip write_dict(target / '.reprounzip', {}, 'directory') signals.post_setup(target=target)
def directory_run(args): """Runs the command in the directory. """ target = Path(args.target[0]) read_dict(target / '.reprounzip', 'directory') cmdline = args.cmdline # Loads config runs, packages, other_files = load_config_file(target / 'config.yml', True) selected_runs = get_runs(runs, args.run, cmdline) root = (target / 'root').absolute() # Gets library paths lib_dirs = [] p = subprocess.Popen(['/sbin/ldconfig', '-v', '-N'], stdout=subprocess.PIPE) try: for l in p.stdout: if len(l) < 3 or l[0] in (b' ', b'\t'): continue if l.endswith(b':\n'): lib_dirs.append(Path(l[:-2])) finally: p.wait() lib_dirs = ( 'export LD_LIBRARY_PATH=%s' % ':'.join(shell_escape(unicode_(join_root(root, d))) for d in lib_dirs)) cmds = [lib_dirs] for run_number in selected_runs: run = runs[run_number] cmd = 'cd %s && ' % shell_escape( unicode_(join_root(root, Path(run['workingdir'])))) cmd += '/usr/bin/env -i ' environ = run['environ'] if args.x11: if 'DISPLAY' in os.environ: environ['DISPLAY'] = os.environ['DISPLAY'] if 'XAUTHORITY' in os.environ: environ['XAUTHORITY'] = os.environ['XAUTHORITY'] cmd += ' '.join('%s=%s' % (k, shell_escape(v)) for k, v in iteritems(environ) if k != 'PATH') cmd += ' ' # PATH # Get the original PATH components path = [ PosixPath(d) for d in run['environ'].get('PATH', '').split(':') ] # The same paths but in the directory dir_path = [join_root(root, d) for d in path if d.root == '/'] # Rebuild string path = ':'.join(unicode_(d) for d in dir_path + path) cmd += 'PATH=%s ' % shell_escape(path) # FIXME : Use exec -a or something if binary != argv[0] if cmdline is None: argv = run['argv'] # Rewrites command-line arguments that are absolute filenames rewritten = False for i in irange(len(argv)): try: p = Path(argv[i]) except UnicodeEncodeError: continue if p.is_absolute: rp = join_root(root, p) if (rp.exists() or (len(rp.components) > 3 and rp.parent.exists())): argv[i] = str(rp) rewritten = True if rewritten: logging.warning("Rewrote command-line as: %s", ' '.join(shell_escape(a) for a in argv)) else: argv = cmdline cmd += ' '.join(shell_escape(a) for a in argv) cmds.append(cmd) cmds = ' && '.join(cmds) signals.pre_run(target=target) retcode = interruptible_call(cmds, shell=True) stderr.write("\n*** Command finished, status: %d\n" % retcode) signals.post_run(target=target, retcode=retcode)
def directory_create(args): """Unpacks the experiment in a folder. Only the files that are not part of a package are copied (unless they are missing from the system and were packed). In addition, input files are put in a tar.gz (so they can be put back after an upload) and the configuration file is extracted. """ if not args.pack: logging.critical("setup needs the pack filename") sys.exit(1) pack = Path(args.pack[0]) target = Path(args.target[0]) if target.exists(): logging.critical("Target directory exists") sys.exit(1) if DefaultAbstractPath is not PosixPath: logging.critical("Not unpacking on POSIX system") sys.exit(1) signals.pre_setup(target=target, pack=pack) # Unpacks configuration file tar = tarfile.open(str(pack), 'r:*') member = tar.getmember('METADATA/config.yml') member.name = 'config.yml' tar.extract(member, str(target)) # Loads config runs, packages, other_files = load_config_file(target / 'config.yml', True) target.mkdir() root = (target / 'root').absolute() root.mkdir() # Checks packages missing_files = False for pkg in packages: if pkg.packfiles: continue for f in pkg.files: f = Path(f.path) if not f.exists(): logging.error( "Missing file %s (from package %s that wasn't packed) " "on host, experiment will probably miss it.", f, pkg.name) missing_files = True if missing_files: logging.error("Some packages are missing, you should probably install " "them.\nUse 'reprounzip installpkgs -h' for help") # Unpacks files if any('..' in m.name or m.name.startswith('/') for m in tar.getmembers()): logging.critical("Tar archive contains invalid pathnames") sys.exit(1) members = [m for m in tar.getmembers() if m.name.startswith('DATA/')] for m in members: m.name = m.name[5:] # Makes symlink targets relative for m in members: if not m.issym(): continue linkname = PosixPath(m.linkname) if linkname.is_absolute: m.linkname = join_root(root, PosixPath(m.linkname)).path logging.info("Extracting files...") tar.extractall(str(root), members) tar.close() # Gets library paths lib_dirs = [] p = subprocess.Popen(['/sbin/ldconfig', '-v', '-N'], stdout=subprocess.PIPE) try: for l in p.stdout: if len(l) < 3 or l[0] in (b' ', b'\t'): continue if l.endswith(b':\n'): lib_dirs.append(Path(l[:-2])) finally: p.wait() # Original input files, so upload can restore them if any(run['input_files'] for run in runs): logging.info("Packing up original input files...") inputtar = tarfile.open(str(target / 'inputs.tar.gz'), 'w:gz') for run in runs: for ifile in itervalues(run['input_files']): inputtar.add(str(join_root(root, PosixPath(ifile))), str(PosixPath(ifile))) inputtar.close() # Meta-data for reprounzip write_dict(target / '.reprounzip', {}, 'directory') signals.post_setup(target=target)
def directory_create(args): """Unpacks the experiment in a folder. Only the files that are not part of a package are copied (unless they are missing from the system and were packed). In addition, input files are put in a tar.gz (so they can be put back after an upload) and the configuration file is extracted. """ if not args.pack: logger.critical("setup needs the pack filename") sys.exit(1) pack = Path(args.pack[0]) target = Path(args.target[0]) if target.exists(): logger.critical("Target directory exists") sys.exit(1) if not issubclass(DefaultAbstractPath, PosixPath): logger.critical("Not unpacking on POSIX system") sys.exit(1) signals.pre_setup(target=target, pack=pack) # Unpacks configuration file rpz_pack = RPZPack(pack) rpz_pack.extract_config(target / 'config.yml') # Loads config config = load_config_file(target / 'config.yml', True) packages = config.packages target.mkdir() root = (target / 'root').absolute() # Checks packages missing_files = False for pkg in packages: if pkg.packfiles: continue for f in pkg.files: if not Path(f.path).exists(): logger.error( "Missing file %s (from package %s that wasn't packed) " "on host, experiment will probably miss it.", f, pkg.name) missing_files = True if missing_files: record_usage(directory_missing_pkgs=True) logger.error("Some packages are missing, you should probably install " "them.\nUse 'reprounzip installpkgs -h' for help") root.mkdir() try: # Unpacks files members = rpz_pack.list_data() for m in members: # Remove 'DATA/' prefix m.name = str(rpz_pack.remove_data_prefix(m.name)) # Makes symlink targets relative if m.issym(): linkname = PosixPath(m.linkname) if linkname.is_absolute: m.linkname = join_root(root, PosixPath(m.linkname)).path logger.info("Extracting files...") rpz_pack.extract_data(root, members) rpz_pack.close() # Original input files, so upload can restore them input_files = [f.path for f in config.inputs_outputs.values() if f.read_runs] if input_files: logger.info("Packing up original input files...") inputtar = tarfile.open(str(target / 'inputs.tar.gz'), 'w:gz') for ifile in input_files: filename = join_root(root, ifile) if filename.exists(): inputtar.add(str(filename), str(ifile)) inputtar.close() # Meta-data for reprounzip metadata_write(target, metadata_initial_iofiles(config), 'directory') signals.post_setup(target=target, pack=pack) except Exception: rmtree_fixed(root) raise
def showfiles(args): """Writes out the input and output files. Works both for a pack file and for an extracted directory. """ def parse_run(runs, s): for i, run in enumerate(runs): if run['id'] == s: return i try: r = int(s) except ValueError: logging.critical("Error: Unknown run %s", s) raise UsageError if r < 0 or r >= len(runs): logging.critical("Error: Expected 0 <= run <= %d, got %d", len(runs) - 1, r) sys.exit(1) return r show_inputs = args.input or not args.output show_outputs = args.output or not args.input def file_filter(fio): if file_filter.run is None: return ((show_inputs and fio.read_runs) or (show_outputs and fio.write_runs)) else: return ((show_inputs and file_filter.run in fio.read_runs) or (show_outputs and file_filter.run in fio.write_runs)) file_filter.run = None pack = Path(args.pack[0]) if not pack.exists(): logging.critical("Pack or directory %s does not exist", pack) sys.exit(1) if pack.is_dir(): # Reads info from an unpacked directory config = load_config_file(pack / 'config.yml', canonical=True) # Filter files by run if args.run is not None: file_filter.run = parse_run(config.runs, args.run) # The '.reprounzip' file is a pickled dictionary, it contains the name # of the files that replaced each input file (if upload was used) unpacked_info = metadata_read(pack, None) assigned_input_files = unpacked_info.get('input_files', {}) if show_inputs: shown = False for input_name, f in sorted(iteritems(config.inputs_outputs)): if f.read_runs and file_filter(f): if not shown: print("Input files:") shown = True if args.verbosity >= 2: print(" %s (%s)" % (input_name, f.path)) else: print(" %s" % input_name) assigned = assigned_input_files.get(input_name) if assigned is None: assigned = "(original)" elif assigned is False: assigned = "(not created)" elif assigned is True: assigned = "(generated)" else: assert isinstance(assigned, (bytes, unicode_)) print(" %s" % assigned) if not shown: print("Input files: none") if show_outputs: shown = False for output_name, f in sorted(iteritems(config.inputs_outputs)): if f.write_runs and file_filter(f): if not shown: print("Output files:") shown = True if args.verbosity >= 2: print(" %s (%s)" % (output_name, f.path)) else: print(" %s" % output_name) if not shown: print("Output files: none") else: # pack.is_file() # Reads info from a pack file config = load_config(pack) # Filter files by run if args.run is not None: file_filter.run = parse_run(config.runs, args.run) if any(f.read_runs for f in itervalues(config.inputs_outputs)): print("Input files:") for input_name, f in sorted(iteritems(config.inputs_outputs)): if f.read_runs and file_filter(f): if args.verbosity >= 2: print(" %s (%s)" % (input_name, f.path)) else: print(" %s" % input_name) else: print("Input files: none") if any(f.write_runs for f in itervalues(config.inputs_outputs)): print("Output files:") for output_name, f in sorted(iteritems(config.inputs_outputs)): if f.write_runs and file_filter(f): if args.verbosity >= 2: print(" %s (%s)" % (output_name, f.path)) else: print(" %s" % output_name) else: print("Output files: none")