def download(self, remote_path, local_path): if self.use_chroot: remote_path = join_root(PosixPath('/experimentroot'), remote_path) temp = make_unique_name(b'reprozip_output_') rtemp = PosixPath('/vagrant') / temp ltemp = self.target / temp # Copy file to shared folder logging.info("Copying file to shared folder...") chan = self.ssh.get_transport().open_session() cp_cmd = '/bin/cp %s %s' % ( shell_escape(remote_path.path), shell_escape(rtemp.path)) chown_cmd = '/bin/chown vagrant %s' % shell_escape(rtemp.path) chmod_cmd = '/bin/chmod 644 %s' % shell_escape(rtemp.path) chan.exec_command('/usr/bin/sudo /bin/sh -c %s' % shell_escape( ' && '.join((cp_cmd, chown_cmd, chmod_cmd)))) if chan.recv_exit_status() != 0: logging.critical("Couldn't copy file in virtual machine") try: ltemp.remove() except OSError: pass return False # Move file to final destination try: ltemp.rename(local_path) except OSError as e: logging.critical("Couldn't download output file: %s\n%s", remote_path, str(e)) ltemp.remove() return False return True
def upload_file(self, local_path, input_path): if self.use_chroot: remote_path = join_root(PosixPath('/experimentroot'), PosixPath(input_path)) else: remote_path = input_path # Upload to a temporary file first logging.info("Uploading file via SCP...") rtemp = PosixPath(make_unique_name(b'/tmp/reprozip_input_')) self.client_scp.put(local_path.path, rtemp.path, recursive=False) # Move it logging.info("Moving file into place...") chan = self.ssh.get_transport().open_session() chown_cmd = '/bin/chown --reference=%s %s' % (shell_escape( remote_path.path), shell_escape(rtemp.path)) chmod_cmd = '/bin/chmod --reference=%s %s' % (shell_escape( remote_path.path), shell_escape(rtemp.path)) mv_cmd = '/bin/mv %s %s' % (shell_escape( rtemp.path), shell_escape(remote_path.path)) chan.exec_command('/usr/bin/sudo /bin/sh -c %s' % shell_escape(';'.join( (chown_cmd, chmod_cmd, mv_cmd)))) if chan.recv_exit_status() != 0: logging.critical("Couldn't move file in virtual machine") sys.exit(1) chan.close()
def extract_original_input(self, input_name, input_path, temp): tar = tarfile.open(str(self.target / 'inputs.tar.gz'), 'r:*') member = tar.getmember(str(join_root(PosixPath(''), input_path))) member.name = str(temp.name) tar.extract(member, str(temp.parent)) tar.close() return temp
def chroot_unmount(target): """Unmount magic directories, if they are mounted. """ unpacked_info = metadata_read(target, 'chroot') mounted = unpacked_info.get('mounted', False) if not mounted: return False target = target.resolve() for m in ('/dev', '/proc'): d = join_root(target / 'root', Path(m)) if d.exists(): logging.info("Unmounting %s...", d) # Unmounts recursively subprocess.check_call( 'grep %s /proc/mounts | ' 'cut -f2 -d" " | ' 'sort -r | ' 'xargs umount' % d, shell=True) unpacked_info['mounted'] = False metadata_write(target, unpacked_info, 'chroot') return True
def upload_file(self, local_path, input_path): if self.use_chroot: remote_path = join_root(PosixPath('/experimentroot'), input_path) else: remote_path = input_path # Upload to a temporary file first logging.info("Uploading file via SCP...") rtemp = PosixPath(make_unique_name(b'/tmp/reprozip_input_')) self.client_scp.put(local_path.path, rtemp.path, recursive=False) # Move it logging.info("Moving file into place...") chan = self.ssh.get_transport().open_session() chown_cmd = '/bin/chown --reference=%s %s' % ( shell_escape(remote_path.path), shell_escape(rtemp.path)) chmod_cmd = '/bin/chmod --reference=%s %s' % ( shell_escape(remote_path.path), shell_escape(rtemp.path)) mv_cmd = '/bin/mv %s %s' % ( shell_escape(rtemp.path), shell_escape(remote_path.path)) chan.exec_command('/usr/bin/sudo /bin/sh -c %s' % shell_escape( ';'.join((chown_cmd, chmod_cmd, mv_cmd)))) if chan.recv_exit_status() != 0: logging.critical("Couldn't move file in virtual machine") sys.exit(1) chan.close()
def chroot_unmount(target): """Unmount magic directories, if they are mounted. """ unpacked_info = metadata_read(target, 'chroot') mounted = unpacked_info.get('mounted', False) if not mounted: return False target = target.resolve() for m in ('/dev', '/proc'): d = join_root(target / 'root', Path(m)) if d.exists(): logger.info("Unmounting %s...", d) # Unmounts recursively subprocess.check_call( 'grep %s /proc/mounts | ' 'cut -f2 -d" " | ' 'sort -r | ' 'xargs umount' % d, shell=True) unpacked_info['mounted'] = False metadata_write(target, unpacked_info, 'chroot') return True
def upload_file(self, local_path, input_path): if self.use_chroot: remote_path = join_root(PosixPath('/experimentroot'), input_path) else: remote_path = input_path temp = make_unique_name(b'reprozip_input_') ltemp = self.target / temp rtemp = PosixPath('/vagrant') / temp # Copy file to shared folder logging.info("Copying file to shared folder...") local_path.copyfile(ltemp) # Move it logging.info("Moving file into place...") chan = self.ssh.get_transport().open_session() chown_cmd = '/bin/chown --reference=%s %s' % (shell_escape( remote_path.path), shell_escape(rtemp.path)) chmod_cmd = '/bin/chmod --reference=%s %s' % (shell_escape( remote_path.path), shell_escape(rtemp.path)) mv_cmd = '/bin/mv %s %s' % (shell_escape( rtemp.path), shell_escape(remote_path.path)) chan.exec_command('/usr/bin/sudo /bin/sh -c %s' % shell_escape(' && '.join( (chown_cmd, chmod_cmd, mv_cmd)))) if chan.recv_exit_status() != 0: logging.critical("Couldn't move file in virtual machine") try: ltemp.remove() except OSError: pass sys.exit(1) chan.close()
def download_and_print(self, remote_path): remote_path = join_root(self.root, remote_path) # Output to stdout if not remote_path.exists(): logging.critical("Can't get output file (doesn't exist): %s", remote_path) sys.exit(1) with remote_path.open('rb') as fp: copyfile(fp, stdout_bytes)
def download(self, remote_path, local_path): remote_path = join_root(self.root, remote_path) # Copy if not remote_path.exists(): logging.critical("Can't get output file (doesn't exist): %s", remote_path) sys.exit(1) remote_path.copyfile(local_path) remote_path.copymode(local_path)
def upload_file(self, local_path, input_path): remote_path = join_root(self.root, input_path) # Copy orig_stat = remote_path.stat() with make_dir_writable(remote_path.parent): local_path.copyfile(remote_path) remote_path.chmod(orig_stat.st_mode & 0o7777) if self.restore_owner: remote_path.chown(orig_stat.st_uid, orig_stat.st_gid)
def download(self, remote_path, local_path): if self.use_chroot: remote_path = join_root(PosixPath('/experimentroot'), remote_path) try: self.client_scp.get(remote_path.path, local_path.path, recursive=False) except scp.SCPException as e: logging.critical("Couldn't download output file: %s\n%s", remote_path, str(e)) sys.exit(1)
def download_and_print(self, remote_path): remote_path = join_root(self.root, remote_path) # Output to stdout if not remote_path.exists(): logger.critical("Can't get output file (doesn't exist): %s", remote_path) return False with remote_path.open('rb') as fp: copyfile(fp, sys.stdout.buffer) return True
def extract_original_input(self, input_name, input_path, temp): tar = tarfile.open(str(self.target / 'inputs.tar.gz'), 'r:*') try: member = tar.getmember(str(join_root(PosixPath(''), input_path))) except KeyError: return None member = copy.copy(member) member.name = str(temp.components[-1]) tar.extract(member, str(temp.parent)) tar.close() return temp
def download_and_print(self, remote_path): remote_path = join_root(self.root, remote_path) # Output to stdout with remote_path.open('rb') as fp: chunk = fp.read(1024) if chunk: sys.stdout.buffer.write(chunk) while len(chunk) == 1024: chunk = fp.read(1024) if chunk: sys.stdout.buffer.write(chunk)
def download_and_print(self, remote_path): remote_path = join_root(self.root, remote_path) # Output to stdout if not remote_path.exists(): logging.critical("Can't get output file (doesn't exist): %s", remote_path) sys.exit(1) with remote_path.open('rb') as fp: chunk = fp.read(1024) if chunk: sys.stdout.buffer.write(chunk) while len(chunk) == 1024: chunk = fp.read(1024) if chunk: sys.stdout.buffer.write(chunk)
def chroot_mount(args): """Mounts /dev and /proc inside the chroot directory. """ target = Path(args.target[0]) read_dict(target / '.reprounzip', 'chroot') for m in ('/dev', '/dev/pts', '/proc'): d = join_root(target / 'root', Path(m)) d.mkdir(parents=True) logging.info("Mounting %s on %s...", m, d) subprocess.check_call(['mount', '-o', 'bind', m, str(d)]) write_dict(target / '.reprounzip', {'mounted': True}, 'chroot') logging.warning("The host's /dev and /proc have been mounted into the " "chroot. Do NOT remove the unpacked directory with " "rm -rf, it WILL WIPE the host's /dev directory.")
def chroot_destroy(args): """Destroys the directory, unmounting first if necessary. """ target = Path(args.target[0]) mounted = read_dict(target / '.reprounzip', 'chroot').get('mounted', False) if mounted: for m in ('/dev', '/proc'): d = join_root(target / 'root', Path(m)) if d.exists(): logging.info("Unmounting %s...", d) subprocess.check_call(['umount', str(d)]) logging.info("Removing directory %s...", target) signals.pre_destroy(target=target) rmtree_fixed(target) signals.post_destroy(target=target)
def chroot_destroy_unmount(args): """Unmounts the bound magic dirs. """ target = Path(args.target[0]) unpacked_info = read_dict(target / '.reprounzip', 'chroot') mounted = unpacked_info.get('mounted', False) if not mounted: logging.critical("Magic directories were not mounted") sys.exit(1) unpacked_info['mounted'] = False write_dict(target / '.reprounzip', unpacked_info, 'chroot') for m in ('/dev', '/proc'): d = join_root(target / 'root', Path(m)) if d.exists(): logging.info("Unmounting %s...", d) subprocess.check_call(['umount', str(d)])
def upload_file(self, local_path, input_path): if self.use_chroot: remote_path = join_root(PosixPath('/experimentroot'), input_path) else: remote_path = input_path temp = make_unique_name(b'reprozip_input_') ltemp = self.target / temp rtemp = PosixPath('/vagrant') / temp # Copy file to shared folder logging.info("Copying file to shared folder...") local_path.copyfile(ltemp) # Move it logging.info("Moving file into place...") chan = self.ssh.get_transport().open_session() chown_cmd = '/bin/chown --reference=%s %s' % ( shell_escape(remote_path.path), shell_escape(rtemp.path)) chmod_cmd = '/bin/chmod --reference=%s %s' % ( shell_escape(remote_path.path), shell_escape(rtemp.path)) mv_cmd = '/bin/mv %s %s' % ( shell_escape(rtemp.path), shell_escape(remote_path.path)) chan.exec_command('/usr/bin/sudo /bin/sh -c %s' % shell_escape( ' && '.join((chown_cmd, chmod_cmd, mv_cmd)))) if chan.recv_exit_status() != 0: logging.critical("Couldn't move file in virtual machine") try: ltemp.remove() except OSError: pass sys.exit(1) chan.close()
def chroot_mount(args): """Mounts /dev and /proc inside the chroot directory. """ target = Path(args.target[0]) unpacked_info = metadata_read(target, 'chroot') # Create proc mount d = target / 'root/proc' d.mkdir(parents=True) subprocess.check_call(['mount', '-t', 'proc', 'none', str(d)]) # Bind /dev from host for m in ('/dev', '/dev/pts'): d = join_root(target / 'root', Path(m)) d.mkdir(parents=True) logger.info("Mounting %s on %s...", m, d) subprocess.check_call(['mount', '-o', 'bind', m, str(d)]) unpacked_info['mounted'] = True metadata_write(target, unpacked_info, 'chroot') logger.warning("The host's /dev and /proc have been mounted into the " "chroot. Do NOT remove the unpacked directory with " "rm -rf, it WILL WIPE the host's /dev directory.")
def directory_run(args): """Runs the command in the directory. """ target = Path(args.target[0]) unpacked_info = metadata_read(target, 'directory') cmdline = args.cmdline # Loads config config = load_config_file(target / 'config.yml', True) runs = config.runs selected_runs = get_runs(runs, args.run, cmdline) root = (target / 'root').absolute() # Gets library paths lib_dirs = [] logger.debug("Running: %s", "/sbin/ldconfig -v -N") p = subprocess.Popen(['/sbin/ldconfig', '-v', '-N'], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, _ = p.communicate() try: for line in stdout.splitlines(): if len(line) < 2 or line[0] in (b' ', b'\t'): continue if line.endswith(b':'): lib_dirs.append(Path(line[:-1])) finally: if p.returncode != 0: raise subprocess.CalledProcessError(p.returncode, ['/sbin/ldconfig', '-v', '-N']) lib_dirs = ('export LD_LIBRARY_PATH=%s' % ':'.join( shell_escape(str(join_root(root, d))) for d in lib_dirs)) cmds = [lib_dirs] for run_number in selected_runs: run = runs[run_number] cmd = 'cd %s && ' % shell_escape( str(join_root(root, Path(run['workingdir'])))) cmd += '/usr/bin/env -i ' environ = run['environ'] environ = fixup_environment(environ, args) if args.x11: if 'DISPLAY' in os.environ: environ['DISPLAY'] = os.environ['DISPLAY'] if 'XAUTHORITY' in os.environ: environ['XAUTHORITY'] = os.environ['XAUTHORITY'] cmd += ' '.join('%s=%s' % (shell_escape(k), shell_escape(v)) for k, v in environ.items() if k != 'PATH') cmd += ' ' # PATH # Get the original PATH components path = [PosixPath(d) for d in run['environ'].get('PATH', '').split(':')] # The same paths but in the directory dir_path = [join_root(root, d) for d in path if d.root == '/'] # Rebuild string path = ':'.join(str(d) for d in dir_path + path) cmd += 'PATH=%s ' % shell_escape(path) # FIXME : Use exec -a or something if binary != argv[0] if cmdline is None: argv = run['argv'] # Rewrites command-line arguments that are absolute filenames rewritten = False for i in range(len(argv)): try: p = Path(argv[i]) except UnicodeEncodeError: continue if p.is_absolute: rp = join_root(root, p) if (rp.exists() or (len(rp.components) > 3 and rp.parent.exists())): argv[i] = str(rp) rewritten = True if rewritten: logger.warning("Rewrote command-line as: %s", ' '.join(shell_escape(a) for a in argv)) else: argv = cmdline cmd += ' '.join(shell_escape(a) for a in argv) cmds.append(cmd) cmds = ' && '.join(cmds) signals.pre_run(target=target) logger.debug("Running: %s", cmds) retcode = interruptible_call(cmds, shell=True) print("\n*** Command finished, status: %d\n" % retcode, file=sys.stderr) signals.post_run(target=target, retcode=retcode) # Update input file status metadata_update_run(config, unpacked_info, selected_runs) metadata_write(target, unpacked_info, 'directory')
def directory_create(args): """Unpacks the experiment in a folder. Only the files that are not part of a package are copied (unless they are missing from the system and were packed). In addition, input files are put in a tar.gz (so they can be put back after an upload) and the configuration file is extracted. """ if not args.pack: logging.critical("setup needs the pack filename") sys.exit(1) pack = Path(args.pack[0]) target = Path(args.target[0]) if target.exists(): logging.critical("Target directory exists") sys.exit(1) if not issubclass(DefaultAbstractPath, PosixPath): logging.critical("Not unpacking on POSIX system") sys.exit(1) signals.pre_setup(target=target, pack=pack) # Unpacks configuration file tar = tarfile.open(str(pack), 'r:*') member = tar.getmember('METADATA/config.yml') member.name = 'config.yml' tar.extract(member, str(target)) # Loads config runs, packages, other_files = load_config_file(target / 'config.yml', True) target.mkdir() root = (target / 'root').absolute() root.mkdir() # Checks packages missing_files = False for pkg in packages: if pkg.packfiles: continue for f in pkg.files: f = Path(f.path) if not f.exists(): logging.error( "Missing file %s (from package %s that wasn't packed) " "on host, experiment will probably miss it.", f, pkg.name) missing_files = True if missing_files: record_usage(directory_missing_pkgs=True) logging.error( "Some packages are missing, you should probably install " "them.\nUse 'reprounzip installpkgs -h' for help") # Unpacks files if any('..' in m.name or m.name.startswith('/') for m in tar.getmembers()): logging.critical("Tar archive contains invalid pathnames") sys.exit(1) members = [m for m in tar.getmembers() if m.name.startswith('DATA/')] for m in members: m.name = m.name[5:] # Makes symlink targets relative for m in members: if not m.issym(): continue linkname = PosixPath(m.linkname) if linkname.is_absolute: m.linkname = join_root(root, PosixPath(m.linkname)).path logging.info("Extracting files...") tar.extractall(str(root), members) tar.close() # Gets library paths lib_dirs = [] p = subprocess.Popen(['/sbin/ldconfig', '-v', '-N'], stdout=subprocess.PIPE) try: for l in p.stdout: if len(l) < 3 or l[0] in (b' ', b'\t'): continue if l.endswith(b':\n'): lib_dirs.append(Path(l[:-2])) finally: p.wait() # Original input files, so upload can restore them if any(run['input_files'] for run in runs): logging.info("Packing up original input files...") inputtar = tarfile.open(str(target / 'inputs.tar.gz'), 'w:gz') for run in runs: for ifile in itervalues(run['input_files']): inputtar.add(str(join_root(root, PosixPath(ifile))), str(PosixPath(ifile))) inputtar.close() # Meta-data for reprounzip write_dict(target / '.reprounzip', {}, 'directory') signals.post_setup(target=target)
def vagrant_setup_create(args): """Sets up the experiment to be run in a Vagrant-built virtual machine. This can either build a chroot or not. If building a chroot, we do just like without Vagrant: we copy all the files and only get what's missing from the host. But we do install automatically the packages whose files are required. If not building a chroot, we install all the packages, and only unpack files that don't come from packages. In short: files from packages with packfiles=True will only be used if building a chroot. """ if not args.pack: logging.critical("setup/create needs the pack filename") sys.exit(1) pack = Path(args.pack[0]) target = Path(args.target[0]) if target.exists(): logging.critical("Target directory exists") sys.exit(1) use_chroot = args.use_chroot mount_bind = args.bind_magic_dirs record_usage(use_chroot=use_chroot, mount_bind=mount_bind) signals.pre_setup(target=target, pack=pack) # Unpacks configuration file tar = tarfile.open(str(pack), 'r:*') member = tar.getmember('METADATA/config.yml') member.name = 'config.yml' tar.extract(member, str(target)) tar.close() # Loads config runs, packages, other_files = load_config(target / 'config.yml', True) if args.base_image and args.base_image[0]: record_usage(vagrant_explicit_image=True) box = args.base_image[0] if args.distribution: target_distribution = args.distribution[0] else: target_distribution = None else: target_distribution, box = select_box(runs) logging.info("Using box %s", box) logging.debug("Distribution: %s", target_distribution or "unknown") # If using chroot, we might still need to install packages to get missing # (not packed) files if use_chroot: packages = [pkg for pkg in packages if not pkg.packfiles] if packages: record_usage(vagrant_install_pkgs=True) logging.info("Some packages were not packed, so we'll install and " "copy their files\n" "Packages that are missing:\n%s", ' '.join(pkg.name for pkg in packages)) if packages: try: installer = select_installer(pack, runs, target_distribution) except CantFindInstaller as e: logging.error("Need to install %d packages but couldn't select a " "package installer: %s", len(packages), e) target.mkdir(parents=True) # Writes setup script logging.info("Writing setup script %s...", target / 'setup.sh') with (target / 'setup.sh').open('w', encoding='utf-8', newline='\n') as fp: fp.write('#!/bin/sh\n\nset -e\n\n') if packages: # Updates package sources fp.write(installer.update_script()) fp.write('\n') # Installs necessary packages fp.write(installer.install_script(packages)) fp.write('\n') # TODO : Compare package versions (painful because of sh) # Untar if use_chroot: fp.write('\n' 'mkdir /experimentroot; cd /experimentroot\n') fp.write('tar zpxf /vagrant/experiment.rpz ' '--numeric-owner --strip=1 DATA\n') if mount_bind: fp.write('\n' 'mkdir -p /experimentroot/dev\n' 'mount -o rbind /dev /experimentroot/dev\n' 'mkdir -p /experimentroot/proc\n' 'mount -o rbind /proc /experimentroot/proc\n') for pkg in packages: fp.write('\n# Copies files from package %s\n' % pkg.name) for f in pkg.files: f = f.path dest = join_root(PosixPath('/experimentroot'), f) fp.write('mkdir -p %s\n' % shell_escape(unicode_(f.parent))) fp.write('cp -L %s %s\n' % ( shell_escape(unicode_(f)), shell_escape(unicode_(dest)))) else: fp.write('\ncd /\n') paths = set() pathlist = [] dataroot = PosixPath('DATA') # Adds intermediate directories, and checks for existence in the # tar tar = tarfile.open(str(pack), 'r:*') for f in other_files: path = PosixPath('/') for c in f.path.components[1:]: path = path / c if path in paths: continue paths.add(path) datapath = join_root(dataroot, path) try: tar.getmember(str(datapath)) except KeyError: logging.info("Missing file %s", datapath) else: pathlist.append(unicode_(datapath)) tar.close() # FIXME : for some reason we need reversed() here, I'm not sure # why. Need to read more of tar's docs. # TAR bug: --no-overwrite-dir removes --keep-old-files # TAR bug: there is no way to make --keep-old-files not report an # error if an existing file is encountered. --skip-old-files was # introduced too recently. Instead, we just ignore the exit status fp.write('tar zpxf /vagrant/experiment.rpz --keep-old-files ' '--numeric-owner --strip=1 %s || /bin/true\n' % ' '.join(shell_escape(p) for p in reversed(pathlist))) # Copies /bin/sh + dependencies if use_chroot: url = busybox_url(runs[0]['architecture']) fp.write(r''' mkdir -p /experimentroot/bin mkdir -p /experimentroot/usr/bin if [ ! -e /experimentroot/bin/sh -o ! -e /experimentroot/usr/bin/env ]; then wget --quiet -O /experimentroot/bin/busybox {url} chmod +x /experimentroot/bin/busybox fi [ -e /experimentroot/bin/sh ] || \ ln -s /bin/busybox /experimentroot/bin/sh [ -e /experimentroot/usr/bin/env ] || \ ln -s /bin/busybox /experimentroot/usr/bin/env '''.format(url=url)) # Copies pack logging.info("Copying pack file...") pack.copyfile(target / 'experiment.rpz') # Writes Vagrant file logging.info("Writing %s...", target / 'Vagrantfile') with (target / 'Vagrantfile').open('w', encoding='utf-8', newline='\n') as fp: # Vagrant header and version fp.write('# -*- mode: ruby -*-\n' '# vi: set ft=ruby\n\n' 'VAGRANTFILE_API_VERSION = "2"\n\n' 'Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|\n') # Selects which box to install fp.write(' config.vm.box = "%s"\n' % box) # Run the setup script on the virtual machine fp.write(' config.vm.provision "shell", path: "setup.sh"\n') fp.write('end\n') # Meta-data for reprounzip write_dict(target / '.reprounzip', {'use_chroot': use_chroot}) signals.post_setup(target=target)
def vagrant_setup_create(args): """Sets up the experiment to be run in a Vagrant-built virtual machine. This can either build a chroot or not. If building a chroot, we do just like without Vagrant: we copy all the files and only get what's missing from the host. But we do install automatically the packages whose files are required. If not building a chroot, we install all the packages, and only unpack files that don't come from packages. In short: files from packages with packfiles=True will only be used if building a chroot. """ if not args.pack: logging.critical("setup/create needs the pack filename") sys.exit(1) pack = Path(args.pack[0]) target = Path(args.target[0]) if target.exists(): logging.critical("Target directory exists") sys.exit(1) use_chroot = args.use_chroot mount_bind = args.bind_magic_dirs record_usage(use_chroot=use_chroot, mount_bind=mount_bind) signals.pre_setup(target=target, pack=pack) # Unpacks configuration file rpz_pack = RPZPack(pack) rpz_pack.extract_config(target / 'config.yml') # Loads config runs, packages, other_files = config = load_config(target / 'config.yml', True) if not args.memory: memory = None else: try: memory = int(args.memory[-1]) except ValueError: logging.critical("Invalid value for memory size: %r", args.memory) sys.exit(1) if args.base_image and args.base_image[0]: record_usage(vagrant_explicit_image=True) box = args.base_image[0] if args.distribution: target_distribution = args.distribution[0] else: target_distribution = None else: target_distribution, box = select_box(runs) logging.info("Using box %s", box) logging.debug("Distribution: %s", target_distribution or "unknown") # If using chroot, we might still need to install packages to get missing # (not packed) files if use_chroot: packages = [pkg for pkg in packages if not pkg.packfiles] if packages: record_usage(vagrant_install_pkgs=True) logging.info("Some packages were not packed, so we'll install and " "copy their files\n" "Packages that are missing:\n%s", ' '.join(pkg.name for pkg in packages)) if packages: try: installer = select_installer(pack, runs, target_distribution) except CantFindInstaller as e: logging.error("Need to install %d packages but couldn't select a " "package installer: %s", len(packages), e) target.mkdir(parents=True) try: # Writes setup script logging.info("Writing setup script %s...", target / 'setup.sh') with (target / 'setup.sh').open('w', encoding='utf-8', newline='\n') as fp: fp.write('#!/bin/sh\n\nset -e\n\n') if packages: # Updates package sources fp.write(installer.update_script()) fp.write('\n') # Installs necessary packages fp.write(installer.install_script(packages)) fp.write('\n') # TODO : Compare package versions (painful because of sh) # Untar if use_chroot: fp.write('\n' 'mkdir /experimentroot; cd /experimentroot\n') fp.write('tar zpxf /vagrant/data.tgz --numeric-owner ' '--strip=1 %s\n' % rpz_pack.data_prefix) if mount_bind: fp.write('\n' 'mkdir -p /experimentroot/dev\n' 'mkdir -p /experimentroot/proc\n') for pkg in packages: fp.write('\n# Copies files from package %s\n' % pkg.name) for f in pkg.files: f = f.path dest = join_root(PosixPath('/experimentroot'), f) fp.write('mkdir -p %s\n' % shell_escape(unicode_(f.parent))) fp.write('cp -L %s %s\n' % ( shell_escape(unicode_(f)), shell_escape(unicode_(dest)))) else: fp.write('\ncd /\n') paths = set() pathlist = [] # Adds intermediate directories, and checks for existence in # the tar for f in other_files: path = PosixPath('/') for c in rpz_pack.remove_data_prefix(f.path).components: path = path / c if path in paths: continue paths.add(path) try: rpz_pack.get_data(path) except KeyError: logging.info("Missing file %s", path) else: pathlist.append(path) # FIXME : for some reason we need reversed() here, I'm not sure # why. Need to read more of tar's docs. # TAR bug: --no-overwrite-dir removes --keep-old-files # TAR bug: there is no way to make --keep-old-files not report # an error if an existing file is encountered. --skip-old-files # was introduced too recently. Instead, we just ignore the exit # status with (target / 'rpz-files.list').open('wb') as lfp: for p in reversed(pathlist): lfp.write(join_root(rpz_pack.data_prefix, p).path) lfp.write(b'\0') fp.write('tar zpxf /vagrant/data.tgz --keep-old-files ' '--numeric-owner --strip=1 ' '--null -T /vagrant/rpz-files.list || /bin/true\n') # Copies busybox if use_chroot: arch = runs[0]['architecture'] download_file(busybox_url(arch), target / 'busybox', 'busybox-%s' % arch) fp.write(r''' cp /vagrant/busybox /experimentroot/busybox chmod +x /experimentroot/busybox mkdir -p /experimentroot/bin [ -e /experimentroot/bin/sh ] || \ ln -s /busybox /experimentroot/bin/sh ''') # Copies pack logging.info("Copying pack file...") rpz_pack.copy_data_tar(target / 'data.tgz') rpz_pack.close() # Writes Vagrant file logging.info("Writing %s...", target / 'Vagrantfile') with (target / 'Vagrantfile').open('w', encoding='utf-8', newline='\n') as fp: # Vagrant header and version fp.write( '# -*- mode: ruby -*-\n' '# vi: set ft=ruby\n\n' 'VAGRANTFILE_API_VERSION = "2"\n\n' 'Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|\n') # Selects which box to install fp.write(' config.vm.box = "%s"\n' % box) # Run the setup script on the virtual machine fp.write(' config.vm.provision "shell", path: "setup.sh"\n') # Memory size if memory is not None: fp.write(' config.vm.provider "virtualbox" do |v|\n' ' v.memory = %d\n' ' end\n' % memory) fp.write('end\n') # Meta-data for reprounzip write_dict(target, metadata_initial_iofiles(config, {'use_chroot': use_chroot})) signals.post_setup(target=target, pack=pack) except Exception: target.rmtree(ignore_errors=True) raise
def docker_setup_create(args): """Sets up the experiment to be run in a Docker-built container. """ pack = Path(args.pack[0]) target = Path(args.target[0]) if target.exists(): logging.critical("Target directory exists") sys.exit(1) signals.pre_setup(target=target, pack=pack) # Unpacks configuration file tar = tarfile.open(str(pack), 'r:*') member = tar.getmember('METADATA/config.yml') member.name = 'config.yml' tar.extract(member, str(target)) tar.close() # Loads config runs, packages, other_files = load_config(target / 'config.yml', True) if args.base_image: record_usage(docker_explicit_base=True) base_image = args.base_image[0] if args.distribution: target_distribution = args.distribution[0] else: target_distribution = None else: target_distribution, base_image = select_image(runs) logging.info("Using base image %s", base_image) logging.debug("Distribution: %s", target_distribution or "unknown") target.mkdir(parents=True) pack.copyfile(target / 'experiment.rpz') # Writes Dockerfile logging.info("Writing %s...", target / 'Dockerfile') with (target / 'Dockerfile').open('w', encoding='utf-8', newline='\n') as fp: fp.write('FROM %s\n\n' % base_image) # Installs busybox download_file(busybox_url(runs[0]['architecture']), target / 'busybox') fp.write('COPY busybox /bin/busybox\n') fp.write('COPY experiment.rpz /reprozip_experiment.rpz\n\n') fp.write('RUN \\\n' ' chmod +x /bin/busybox && \\\n') if args.install_pkgs: # Install every package through package manager missing_packages = [] else: # Only install packages that were not packed missing_packages = [pkg for pkg in packages if pkg.packfiles] packages = [pkg for pkg in packages if not pkg.packfiles] # FIXME : Right now, we need 'sudo' to be available (and it's not # necessarily in the base image) if packages: record_usage(docker_install_pkgs=True) else: record_usage(docker_install_pkgs="sudo") packages += [Package('sudo', None, packfiles=False)] if packages: try: installer = select_installer(pack, runs, target_distribution) except CantFindInstaller as e: logging.error("Need to install %d packages but couldn't " "select a package installer: %s", len(packages), e) sys.exit(1) # Updates package sources fp.write(' %s && \\\n' % installer.update_script()) # Installs necessary packages fp.write(' %s && \\\n' % installer.install_script(packages)) logging.info("Dockerfile will install the %d software packages that " "were not packed", len(packages)) # Untar paths = set() pathlist = [] dataroot = PosixPath('DATA') # Adds intermediate directories, and checks for existence in the tar tar = tarfile.open(str(pack), 'r:*') missing_files = chain.from_iterable(pkg.files for pkg in missing_packages) for f in chain(other_files, missing_files): path = PosixPath('/') for c in f.path.components[1:]: path = path / c if path in paths: continue paths.add(path) datapath = join_root(dataroot, path) try: tar.getmember(str(datapath)) except KeyError: logging.info("Missing file %s", datapath) else: pathlist.append(unicode_(datapath)) tar.close() # FIXME : for some reason we need reversed() here, I'm not sure why. # Need to read more of tar's docs. # TAR bug: --no-overwrite-dir removes --keep-old-files fp.write(' cd / && tar zpxf /reprozip_experiment.rpz ' '--numeric-owner --strip=1 %s\n' % ' '.join(shell_escape(p) for p in reversed(pathlist))) # Meta-data for reprounzip write_dict(target / '.reprounzip', {}) signals.post_setup(target=target)
def directory_create(args): """Unpacks the experiment in a folder. Only the files that are not part of a package are copied (unless they are missing from the system and were packed). In addition, input files are put in a tar.gz (so they can be put back after an upload) and the configuration file is extracted. """ if not args.pack: logging.critical("setup needs the pack filename") sys.exit(1) pack = Path(args.pack[0]) target = Path(args.target[0]) if target.exists(): logging.critical("Target directory exists") sys.exit(1) if not issubclass(DefaultAbstractPath, PosixPath): logging.critical("Not unpacking on POSIX system") sys.exit(1) signals.pre_setup(target=target, pack=pack) # Unpacks configuration file rpz_pack = RPZPack(pack) rpz_pack.extract_config(target / 'config.yml') # Loads config config = load_config_file(target / 'config.yml', True) packages = config.packages target.mkdir() root = (target / 'root').absolute() # Checks packages missing_files = False for pkg in packages: if pkg.packfiles: continue for f in pkg.files: if not Path(f.path).exists(): logging.error( "Missing file %s (from package %s that wasn't packed) " "on host, experiment will probably miss it.", f, pkg.name) missing_files = True if missing_files: record_usage(directory_missing_pkgs=True) logging.error("Some packages are missing, you should probably install " "them.\nUse 'reprounzip installpkgs -h' for help") root.mkdir() try: # Unpacks files members = rpz_pack.list_data() for m in members: # Remove 'DATA/' prefix m.name = str(rpz_pack.remove_data_prefix(m.name)) # Makes symlink targets relative if m.issym(): linkname = PosixPath(m.linkname) if linkname.is_absolute: m.linkname = join_root(root, PosixPath(m.linkname)).path logging.info("Extracting files...") rpz_pack.extract_data(root, members) rpz_pack.close() # Original input files, so upload can restore them input_files = [f.path for f in itervalues(config.inputs_outputs) if f.read_runs] if input_files: logging.info("Packing up original input files...") inputtar = tarfile.open(str(target / 'inputs.tar.gz'), 'w:gz') for ifile in input_files: filename = join_root(root, ifile) if filename.exists(): inputtar.add(str(filename), str(ifile)) inputtar.close() # Meta-data for reprounzip metadata_write(target, metadata_initial_iofiles(config), 'directory') signals.post_setup(target=target, pack=pack) except Exception: rmtree_fixed(root) raise
def functional_tests(raise_warnings, interactive, run_vagrant, run_docker): # Tests on Python < 2.7.3: need to use separate reprozip Python (with known # working version of Python) if sys.version_info < (2, 7, 3): bug13676 = True if 'REPROZIP_PYTHON' not in os.environ: sys.stderr.write("Error: using reprozip with Python %s!\n" % sys.version.split(' ', 1)[0]) sys.exit(1) else: bug13676 = False rpz = [os.environ.get('REPROZIP_PYTHON', sys.executable)] rpuz = [os.environ.get('REPROUNZIP_PYTHON', sys.executable)] # Can't match on the SignalWarning category here because of a Python bug # http://bugs.python.org/issue22543 if raise_warnings: rpz.extend(['-W', 'error:signal']) rpuz.extend(['-W', 'error:signal']) if 'COVER' in os.environ: rpz.extend(['-m'] + os.environ['COVER'].split(' ')) rpuz.extend(['-m'] + os.environ['COVER'].split(' ')) reprozip_main = tests.parent / 'reprozip/reprozip/main.py' reprounzip_main = tests.parent / 'reprounzip/reprounzip/main.py' verbose = ['-v'] * 3 rpz.extend([reprozip_main.absolute().path] + verbose) rpuz.extend([reprounzip_main.absolute().path] + verbose) print("Command lines are:\n%r\n%r" % (rpz, rpuz)) # ######################################## # testrun /bin/echo # output = check_output(rpz + ['testrun', '/bin/echo', 'outputhere']) assert any(b' 1 | /bin/echo outputhere ' in l for l in output.splitlines()) output = check_output(rpz + ['testrun', '-a', '/fake/path/echo', '/bin/echo', 'outputhere']) assert any(b' 1 | (/bin/echo) /fake/path/echo outputhere ' in l for l in output.splitlines()) # ######################################## # testrun multiple commands # check_call(rpz + ['testrun', 'bash', '-c', 'cat ../../../../../etc/passwd;' 'cd /var/lib;' 'cat ../../etc/group']) check_call(rpz + ['trace', 'bash', '-c', 'cat /etc/passwd;echo']) check_call(rpz + ['trace', '--continue', 'sh', '-c', 'cat /etc/group;/usr/bin/id']) check_call(rpz + ['pack']) if not bug13676: check_call(rpuz + ['graph', 'graph.dot']) check_call(rpuz + ['graph', 'graph2.dot', 'experiment.rpz']) sudo = ['sudo', '-E'] # -E to keep REPROZIP_USAGE_STATS # ######################################## # 'simple' program: trace, pack, info, unpack # def check_simple(args, stream, infile=1): output = check_output(args, stream).splitlines() try: first = output.index(b"Read 6 bytes") except ValueError: stderr.write("output = %r\n" % output) raise if infile == 1: assert output[first + 1] == b"a = 29, b = 13" assert output[first + 2] == b"result = 42" else: # infile == 2 assert output[first + 1] == b"a = 25, b = 11" assert output[first + 2] == b"result = 36" # Build build('simple', ['simple.c']) # Trace check_call(rpz + ['trace', '-d', 'rpz-simple', './simple', (tests / 'simple_input.txt').path, 'simple_output.txt']) orig_output_location = Path('simple_output.txt').absolute() assert orig_output_location.is_file() with orig_output_location.open(encoding='utf-8') as fp: assert fp.read().strip() == '42' orig_output_location.remove() # Read config with Path('rpz-simple/config.yml').open(encoding='utf-8') as fp: conf = yaml.safe_load(fp) other_files = set(Path(f).absolute() for f in conf['other_files']) expected = [Path('simple'), (tests / 'simple_input.txt')] assert other_files.issuperset([f.resolve() for f in expected]) # Check input and output files inputs_outputs = conf['inputs_outputs'] # Exactly one input: "arg1", "...simple_input.txt" # Output: 'arg2', "...simple_output.txt" # There might be more output files: the C coverage files found = 0 for fdict in inputs_outputs: if Path(fdict['path']).name == b'simple_input.txt': assert fdict['name'] == 'arg1' assert fdict['read_by_runs'] == [0] assert not fdict.get('written_by_runs') found |= 0x01 elif Path(fdict['path']).name == b'simple_output.txt': assert fdict['name'] == 'arg2' assert not fdict.get('read_by_runs') assert fdict['written_by_runs'] == [0] found |= 0x02 else: # No other inputs assert not fdict.get('read_by_runs') assert found == 0x03 # Pack check_call(rpz + ['pack', '-d', 'rpz-simple', 'simple.rpz']) Path('simple').remove() # Info check_call(rpuz + ['info', 'simple.rpz']) # Show files check_call(rpuz + ['showfiles', 'simple.rpz']) # Lists packages check_call(rpuz + ['installpkgs', '--summary', 'simple.rpz']) # Unpack directory check_call(rpuz + ['directory', 'setup', 'simple.rpz', 'simpledir']) # Run directory check_simple(rpuz + ['directory', 'run', 'simpledir'], 'err') output_in_dir = join_root(Path('simpledir/root'), orig_output_location) with output_in_dir.open(encoding='utf-8') as fp: assert fp.read().strip() == '42' # Delete with wrong command (should fail) p = subprocess.Popen(rpuz + ['chroot', 'destroy', 'simpledir'], stderr=subprocess.PIPE) out, err = p.communicate() assert p.poll() != 0 err = err.splitlines() assert b"Wrong unpacker used" in err[0] assert err[1].startswith(b"usage: ") # Delete directory check_call(rpuz + ['directory', 'destroy', 'simpledir']) # Unpack chroot check_call(sudo + rpuz + ['chroot', 'setup', '--bind-magic-dirs', 'simple.rpz', 'simplechroot']) try: output_in_chroot = join_root(Path('simplechroot/root'), orig_output_location) # Run chroot check_simple(sudo + rpuz + ['chroot', 'run', 'simplechroot'], 'err') with output_in_chroot.open(encoding='utf-8') as fp: assert fp.read().strip() == '42' # Get output file check_call(sudo + rpuz + ['chroot', 'download', 'simplechroot', 'arg2:output1.txt']) with Path('output1.txt').open(encoding='utf-8') as fp: assert fp.read().strip() == '42' # Replace input file check_call(sudo + rpuz + ['chroot', 'upload', 'simplechroot', '%s:arg1' % (tests / 'simple_input2.txt')]) check_call(sudo + rpuz + ['chroot', 'upload', 'simplechroot']) # Run again check_simple(sudo + rpuz + ['chroot', 'run', 'simplechroot'], 'err', 2) with output_in_chroot.open(encoding='utf-8') as fp: assert fp.read().strip() == '36' # Reset input file check_call(sudo + rpuz + ['chroot', 'upload', 'simplechroot', ':arg1']) # Run again check_simple(sudo + rpuz + ['chroot', 'run', 'simplechroot'], 'err') with output_in_chroot.open(encoding='utf-8') as fp: assert fp.read().strip() == '42' # Delete with wrong command (should fail) p = subprocess.Popen(rpuz + ['directory', 'destroy', 'simplechroot'], stderr=subprocess.PIPE) out, err = p.communicate() assert p.poll() != 0 err = err.splitlines() assert b"Wrong unpacker used" in err[0] assert err[1].startswith(b"usage:") finally: # Delete chroot check_call(sudo + rpuz + ['chroot', 'destroy', 'simplechroot']) if not (tests / 'vagrant').exists(): check_call(['sudo', 'sh', '-c', 'mkdir %(d)s; chmod 777 %(d)s' % {'d': tests / 'vagrant'}]) # Unpack Vagrant-chroot check_call(rpuz + ['vagrant', 'setup/create', '--memory', '512', '--use-chroot', 'simple.rpz', (tests / 'vagrant/simplevagrantchroot').path]) print("\nVagrant project set up in simplevagrantchroot") try: if run_vagrant: check_simple(rpuz + ['vagrant', 'run', '--no-stdin', (tests / 'vagrant/simplevagrantchroot').path], 'out') # Get output file check_call(rpuz + ['vagrant', 'download', (tests / 'vagrant/simplevagrantchroot').path, 'arg2:voutput1.txt']) with Path('voutput1.txt').open(encoding='utf-8') as fp: assert fp.read().strip() == '42' # Replace input file check_call(rpuz + ['vagrant', 'upload', (tests / 'vagrant/simplevagrantchroot').path, '%s:arg1' % (tests / 'simple_input2.txt')]) check_call(rpuz + ['vagrant', 'upload', (tests / 'vagrant/simplevagrantchroot').path]) # Run again check_simple(rpuz + ['vagrant', 'run', '--no-stdin', (tests / 'vagrant/simplevagrantchroot').path], 'out', 2) # Get output file check_call(rpuz + ['vagrant', 'download', (tests / 'vagrant/simplevagrantchroot').path, 'arg2:voutput2.txt']) with Path('voutput2.txt').open(encoding='utf-8') as fp: assert fp.read().strip() == '36' # Reset input file check_call(rpuz + ['vagrant', 'upload', (tests / 'vagrant/simplevagrantchroot').path, ':arg1']) # Run again check_simple(rpuz + ['vagrant', 'run', '--no-stdin', (tests / 'vagrant/simplevagrantchroot').path], 'out') # Get output file check_call(rpuz + ['vagrant', 'download', (tests / 'vagrant/simplevagrantchroot').path, 'arg2:voutput1.txt']) with Path('voutput1.txt').open(encoding='utf-8') as fp: assert fp.read().strip() == '42' # Destroy check_call(rpuz + ['vagrant', 'destroy', (tests / 'vagrant/simplevagrantchroot').path]) elif interactive: print("Test and press enter") sys.stdin.readline() finally: if (tests / 'vagrant/simplevagrantchroot').exists(): (tests / 'vagrant/simplevagrantchroot').rmtree() # Unpack Vagrant without chroot check_call(rpuz + ['vagrant', 'setup/create', '--dont-use-chroot', 'simple.rpz', (tests / 'vagrant/simplevagrant').path]) print("\nVagrant project set up in simplevagrant") try: if run_vagrant: check_simple(rpuz + ['vagrant', 'run', '--no-stdin', (tests / 'vagrant/simplevagrant').path], 'out') # Get output file check_call(rpuz + ['vagrant', 'download', (tests / 'vagrant/simplevagrant').path, 'arg2:woutput1.txt']) with Path('woutput1.txt').open(encoding='utf-8') as fp: assert fp.read().strip() == '42' # Replace input file check_call(rpuz + ['vagrant', 'upload', (tests / 'vagrant/simplevagrant').path, '%s:arg1' % (tests / 'simple_input2.txt')]) check_call(rpuz + ['vagrant', 'upload', (tests / 'vagrant/simplevagrant').path]) # Run again check_simple(rpuz + ['vagrant', 'run', '--no-stdin', (tests / 'vagrant/simplevagrant').path], 'out', 2) # Get output file check_call(rpuz + ['vagrant', 'download', (tests / 'vagrant/simplevagrant').path, 'arg2:woutput2.txt']) with Path('woutput2.txt').open(encoding='utf-8') as fp: assert fp.read().strip() == '36' # Reset input file check_call(rpuz + ['vagrant', 'upload', (tests / 'vagrant/simplevagrant').path, ':arg1']) # Run again check_simple(rpuz + ['vagrant', 'run', '--no-stdin', (tests / 'vagrant/simplevagrant').path], 'out') # Get output file check_call(rpuz + ['vagrant', 'download', (tests / 'vagrant/simplevagrant').path, 'arg2:voutput1.txt']) with Path('voutput1.txt').open(encoding='utf-8') as fp: assert fp.read().strip() == '42' # Destroy check_call(rpuz + ['vagrant', 'destroy', (tests / 'vagrant/simplevagrant').path]) elif interactive: print("Test and press enter") sys.stdin.readline() finally: if (tests / 'vagrant/simplevagrant').exists(): (tests / 'vagrant/simplevagrant').rmtree() # Unpack Docker check_call(rpuz + ['docker', 'setup/create', 'simple.rpz', 'simpledocker']) print("\nDocker project set up in simpledocker") try: if run_docker: check_call(rpuz + ['docker', 'setup/build', 'simpledocker']) check_simple(rpuz + ['docker', 'run', 'simpledocker'], 'out') # Get output file check_call(rpuz + ['docker', 'download', 'simpledocker', 'arg2:doutput1.txt']) with Path('doutput1.txt').open(encoding='utf-8') as fp: assert fp.read().strip() == '42' # Replace input file check_call(rpuz + ['docker', 'upload', 'simpledocker', '%s:arg1' % (tests / 'simple_input2.txt')]) check_call(rpuz + ['docker', 'upload', 'simpledocker']) check_call(rpuz + ['showfiles', 'simpledocker']) # Run again check_simple(rpuz + ['docker', 'run', 'simpledocker'], 'out', 2) # Get output file check_call(rpuz + ['docker', 'download', 'simpledocker', 'arg2:doutput2.txt']) with Path('doutput2.txt').open(encoding='utf-8') as fp: assert fp.read().strip() == '36' # Reset input file check_call(rpuz + ['docker', 'upload', 'simpledocker', ':arg1']) # Run again check_simple(rpuz + ['docker', 'run', 'simpledocker'], 'out') # Get output file check_call(rpuz + ['docker', 'download', 'simpledocker', 'arg2:doutput1.txt']) with Path('doutput1.txt').open(encoding='utf-8') as fp: assert fp.read().strip() == '42' # Destroy check_call(rpuz + ['docker', 'destroy', 'simpledocker']) elif interactive: print("Test and press enter") sys.stdin.readline() finally: if Path('simpledocker').exists(): Path('simpledocker').rmtree() # ######################################## # 'threads' program: testrun # # Build build('threads', ['threads.c'], ['-lpthread']) # Trace output = check_output(rpz + ['testrun', './threads'], 'err') assert any(b'successfully exec\'d /bin/./echo' in l for l in output.splitlines()) # ######################################## # 'threads2' program: testrun # # Build build('threads2', ['threads2.c'], ['-lpthread']) # Trace output = check_output(rpz + ['testrun', './threads2'], 'err') assert any(b'successfully exec\'d /bin/echo' in l for l in output.splitlines()) # ######################################## # 'segv' program: testrun # # Build build('segv', ['segv.c']) # Trace check_call(rpz + ['testrun', './segv']) # ######################################## # 'exec_echo' program: trace, pack, run --cmdline # # Build build('exec_echo', ['exec_echo.c']) # Trace check_call(rpz + ['trace', './exec_echo', 'originalexecechooutput']) # Pack check_call(rpz + ['pack', 'exec_echo.rpz']) # Unpack chroot check_call(sudo + rpuz + ['chroot', 'setup', 'exec_echo.rpz', 'echochroot']) try: # Run original command-line output = check_output(sudo + rpuz + ['chroot', 'run', 'echochroot']) assert output == b'originalexecechooutput\n' # Prints out command-line output = check_output(sudo + rpuz + ['chroot', 'run', 'echochroot', '--cmdline']) assert any(b'./exec_echo originalexecechooutput' == s.strip() for s in output.split(b'\n')) # Run with different command-line output = check_output(sudo + rpuz + [ 'chroot', 'run', 'echochroot', '--cmdline', './exec_echo', 'changedexecechooutput']) assert output == b'changedexecechooutput\n' finally: check_call(sudo + rpuz + ['chroot', 'destroy', 'echochroot']) # ######################################## # 'exec_echo' program: testrun # This is built with -m32 so that we transition: # python (x64) -> exec_echo (i386) -> echo (x64) # if sys.maxsize > 2 ** 32: # Build build('exec_echo32', ['exec_echo.c'], ['-m32']) # Trace check_call(rpz + ['testrun', './exec_echo32 42']) else: print("Can't try exec_echo transitions: not running on 64bits") # ######################################## # Tracing non-existing program # check_call(rpz + ['testrun', './doesntexist']) # ######################################## # 'connect' program: testrun # # Build build('connect', ['connect.c']) # Trace err = check_output(rpz + ['testrun', './connect'], 'err') err = err.split(b'\n') assert not any(b'program exited with non-zero code' in l for l in err) assert any(re.search(br'process connected to [0-9.]+:80', l) for l in err) # ######################################## # 'vfork' program: testrun # # Build build('vfork', ['vfork.c']) # Trace err = check_output(rpz + ['testrun', './vfork'], 'err') err = err.split(b'\n') assert not any(b'program exited with non-zero code' in l for l in err) # ######################################## # 'rename' program: trace # # Build build('rename', ['rename.c']) # Trace check_call(rpz + ['trace', '-d', 'rename-trace', './rename']) with Path('rename-trace/config.yml').open(encoding='utf-8') as fp: config = yaml.safe_load(fp) # Check that written files were logged database = Path.cwd() / 'rename-trace/trace.sqlite3' if PY3: # On PY3, connect() only accepts unicode conn = sqlite3.connect(str(database)) else: conn = sqlite3.connect(database.path) conn.row_factory = sqlite3.Row rows = conn.execute( ''' SELECT name FROM opened_files ''') files = set(Path(r[0]) for r in rows) for n in ('dir1/file', 'dir2/file', 'dir2/brokensymlink', 'dir2/symlink'): if (Path.cwd() / n) not in files: raise AssertionError("Missing file: %s" % (Path.cwd() / n)) conn.close() # Check that created files won't be packed for f in config.get('other_files'): if 'dir2' in Path(f).parent.components: raise AssertionError("Created file shouldn't be packed: %s" % Path(f)) # ######################################## # Test shebang corner-cases # Path('a').symlink('b') with Path('b').open('w') as fp: fp.write('#!%s 0\nsome content\n' % (Path.cwd() / 'c')) Path('b').chmod(0o744) Path('c').symlink('d') with Path('d').open('w') as fp: fp.write('#!e') Path('d').chmod(0o744) with Path('e').open('w') as fp: fp.write('#!/bin/echo') Path('e').chmod(0o744) # Trace out = check_output(rpz + ['trace', '--dont-identify-packages', '-d', 'shebang-trace', './a', '1', '2']) out = out.splitlines()[0] assert out == ('e %s 0 ./a 1 2' % (Path.cwd() / 'c')).encode('ascii') # Check config with Path('shebang-trace/config.yml').open(encoding='utf-8') as fp: config = yaml.safe_load(fp) other_files = set(Path(f) for f in config['other_files'] if f.startswith('%s/' % Path.cwd())) # Check database database = Path.cwd() / 'shebang-trace/trace.sqlite3' if PY3: # On PY3, connect() only accepts unicode conn = sqlite3.connect(str(database)) else: conn = sqlite3.connect(database.path) conn.row_factory = sqlite3.Row rows = conn.execute( ''' SELECT name FROM opened_files ''') opened = [Path(r[0]) for r in rows if r[0].startswith('%s/' % Path.cwd())] rows = conn.execute( ''' SELECT name, argv FROM executed_files ''') executed = [(Path(r[0]), r[1]) for r in rows if Path(r[0]).lies_under(Path.cwd())] print("other_files: %r" % sorted(other_files)) print("opened: %r" % opened) print("executed: %r" % executed) assert other_files == set(Path.cwd() / p for p in ('a', 'b', 'c', 'd', 'e')) if not bug13676: assert opened == [Path.cwd() / 'c', Path.cwd() / 'e'] assert executed == [(Path.cwd() / 'a', './a\x001\x002\x00')] # ######################################## # Test old packages # old_packages = [ ('simple-0.4.0.rpz', 'https://drive.google.com/uc?export=download&id=0B3ucPz7GSthBVG4xZW1V' 'eDhXNTQ'), ('simple-0.6.0.rpz', 'https://drive.google.com/uc?export=download&id=0B3ucPz7GSthBbl9SUjhr' 'cUdtbGs'), ('simple-0.7.1.rpz', 'https://drive.google.com/uc?export=download&id=0B3ucPz7GSthBRGp2Vm5V' 'QVpWOGs'), ] for name, url in old_packages: print("Testing old package %s" % name) f = Path(name) if not f.exists(): download_file(url, f) # Info check_call(rpuz + ['info', name]) # Show files check_call(rpuz + ['showfiles', name]) # Lists packages check_call(rpuz + ['installpkgs', '--summary', name]) # Unpack directory check_call(rpuz + ['directory', 'setup', name, 'simpledir']) # Run directory check_simple(rpuz + ['directory', 'run', 'simpledir'], 'err') output_in_dir = Path('simpledir/root/tmp') output_in_dir = output_in_dir.listdir('reprozip_*')[0] output_in_dir = output_in_dir / 'simple_output.txt' with output_in_dir.open(encoding='utf-8') as fp: assert fp.read().strip() == '42' # Delete with wrong command (should fail) p = subprocess.Popen(rpuz + ['chroot', 'destroy', 'simpledir'], stderr=subprocess.PIPE) out, err = p.communicate() assert p.poll() != 0 err = err.splitlines() assert b"Wrong unpacker used" in err[0] assert err[1].startswith(b"usage: ") # Delete directory check_call(rpuz + ['directory', 'destroy', 'simpledir']) # ######################################## # Copies back coverage report # coverage = Path('.coverage') if coverage.exists(): coverage.copyfile(tests.parent / '.coverage.runpy')
def functional_tests(raise_warnings, interactive, run_vagrant, run_docker): # Tests on Python < 2.7.3: need to use separate reprozip Python (with known # working version of Python) if sys.version_info < (2, 7, 3): bug13676 = True if 'REPROZIP_PYTHON' not in os.environ: sys.stderr.write("Error: using reprozip with Python %s!\n" % sys.version.split(' ', 1)[0]) sys.exit(1) else: bug13676 = False rpz = [os.environ.get('REPROZIP_PYTHON', sys.executable)] rpuz = [os.environ.get('REPROUNZIP_PYTHON', sys.executable)] # Can't match on the SignalWarning category here because of a Python bug # http://bugs.python.org/issue22543 if raise_warnings: rpz.extend(['-W', 'error:signal']) rpuz.extend(['-W', 'error:signal']) if 'COVER' in os.environ: rpz.extend(['-m'] + os.environ['COVER'].split(' ')) rpuz.extend(['-m'] + os.environ['COVER'].split(' ')) reprozip_main = tests.parent / 'reprozip/reprozip/main.py' reprounzip_main = tests.parent / 'reprounzip/reprounzip/main.py' verbose = ['-v'] * 3 rpz.extend([reprozip_main.absolute().path] + verbose) rpuz.extend([reprounzip_main.absolute().path] + verbose) print("Command lines are:\n%r\n%r" % (rpz, rpuz)) # ######################################## # testrun /bin/echo # output = check_output(rpz + ['testrun', '/bin/echo', 'outputhere']) assert any(b' 1 | /bin/echo outputhere ' in l for l in output.splitlines()) output = check_output( rpz + ['testrun', '-a', '/fake/path/echo', '/bin/echo', 'outputhere']) assert any(b' 1 | (/bin/echo) /fake/path/echo outputhere ' in l for l in output.splitlines()) # ######################################## # testrun multiple commands # check_call(rpz + [ 'testrun', 'bash', '-c', 'cat ../../../../../etc/passwd;' 'cd /var/lib;' 'cat ../../etc/group' ]) check_call(rpz + ['trace', 'bash', '-c', 'cat /etc/passwd;echo']) check_call( rpz + ['trace', '--continue', 'sh', '-c', 'cat /etc/group;/usr/bin/id']) check_call(rpz + ['pack']) if not bug13676: check_call(rpuz + ['graph', 'graph.dot']) check_call(rpuz + ['graph', 'graph2.dot', 'experiment.rpz']) sudo = ['sudo', '-E'] # -E to keep REPROZIP_USAGE_STATS # ######################################## # 'simple' program: trace, pack, info, unpack # # Build build('simple', ['simple.c']) # Trace check_call(rpz + [ 'trace', '-d', 'rpz-simple', './simple', (tests / 'simple_input.txt').path, 'simple_output.txt' ]) orig_output_location = Path('simple_output.txt').absolute() assert orig_output_location.is_file() with orig_output_location.open(encoding='utf-8') as fp: assert fp.read().strip() == '42' orig_output_location.remove() # Read config with Path('rpz-simple/config.yml').open(encoding='utf-8') as fp: conf = yaml.safe_load(fp) other_files = set(Path(f).absolute() for f in conf['other_files']) expected = [Path('simple'), (tests / 'simple_input.txt')] assert other_files.issuperset([f.resolve() for f in expected]) # Check input and output files input_files = conf['runs'][0]['input_files'] assert (dict((k, Path(f).name) for k, f in iteritems(input_files)) == { 'arg': b'simple_input.txt' }) output_files = conf['runs'][0]['output_files'] print(dict((k, Path(f).name) for k, f in iteritems(output_files))) # Here we don't test for dict equality, since we might have C coverage # files in the mix assert Path(output_files['arg']).name == b'simple_output.txt' # Pack check_call(rpz + ['pack', '-d', 'rpz-simple', 'simple.rpz']) Path('simple').remove() # Info check_call(rpuz + ['info', 'simple.rpz']) # Show files check_call(rpuz + ['showfiles', 'simple.rpz']) # Lists packages check_call(rpuz + ['installpkgs', '--summary', 'simple.rpz']) # Unpack directory check_call(rpuz + ['directory', 'setup', 'simple.rpz', 'simpledir']) # Run directory check_call(rpuz + ['directory', 'run', 'simpledir']) output_in_dir = join_root(Path('simpledir/root'), orig_output_location) with output_in_dir.open(encoding='utf-8') as fp: assert fp.read().strip() == '42' # Delete with wrong command (should fail) p = subprocess.Popen(rpuz + ['chroot', 'destroy', 'simpledir'], stderr=subprocess.PIPE) stdout, stderr = p.communicate() assert p.poll() != 0 stderr = stderr.splitlines() assert b"Wrong unpacker used" in stderr[0] assert stderr[1].startswith(b"usage: ") # Delete directory check_call(rpuz + ['directory', 'destroy', 'simpledir']) # Unpack chroot check_call( sudo + rpuz + ['chroot', 'setup', '--bind-magic-dirs', 'simple.rpz', 'simplechroot']) try: # Run chroot check_call(sudo + rpuz + ['chroot', 'run', 'simplechroot']) output_in_chroot = join_root(Path('simplechroot/root'), orig_output_location) with output_in_chroot.open(encoding='utf-8') as fp: assert fp.read().strip() == '42' # Get output file check_call(sudo + rpuz + ['chroot', 'download', 'simplechroot', 'arg:output1.txt']) with Path('output1.txt').open(encoding='utf-8') as fp: assert fp.read().strip() == '42' # Replace input file check_call(sudo + rpuz + [ 'chroot', 'upload', 'simplechroot', '%s:arg' % (tests / 'simple_input2.txt') ]) check_call(sudo + rpuz + ['chroot', 'upload', 'simplechroot']) # Run again check_call(sudo + rpuz + ['chroot', 'run', 'simplechroot']) output_in_chroot = join_root(Path('simplechroot/root'), orig_output_location) with output_in_chroot.open(encoding='utf-8') as fp: assert fp.read().strip() == '36' # Delete with wrong command (should fail) p = subprocess.Popen(rpuz + ['directory', 'destroy', 'simplechroot'], stderr=subprocess.PIPE) stdout, stderr = p.communicate() assert p.poll() != 0 stderr = stderr.splitlines() assert b"Wrong unpacker used" in stderr[0] assert stderr[1].startswith(b"usage:") finally: # Delete chroot check_call(sudo + rpuz + ['chroot', 'destroy', 'simplechroot']) if not (tests / 'vagrant').exists(): check_call([ 'sudo', 'sh', '-c', 'mkdir %(d)s; chmod 777 %(d)s' % { 'd': tests / 'vagrant' } ]) # Unpack Vagrant-chroot check_call(rpuz + [ 'vagrant', 'setup/create', '--use-chroot', 'simple.rpz', (tests / 'vagrant/simplevagrantchroot').path ]) print("\nVagrant project set up in simplevagrantchroot") try: if run_vagrant: check_call(rpuz + [ 'vagrant', 'run', '--no-stdin', (tests / 'vagrant/simplevagrantchroot').path ]) # Destroy check_call(rpuz + [ 'vagrant', 'destroy', (tests / 'vagrant/simplevagrantchroot').path ]) elif interactive: print("Test and press enter") sys.stdin.readline() finally: if (tests / 'vagrant/simplevagrantchroot').exists(): (tests / 'vagrant/simplevagrantchroot').rmtree() # Unpack Vagrant without chroot check_call(rpuz + [ 'vagrant', 'setup/create', '--dont-use-chroot', 'simple.rpz', (tests / 'vagrant/simplevagrant').path ]) print("\nVagrant project set up in simplevagrant") try: if run_vagrant: check_call(rpuz + [ 'vagrant', 'run', '--no-stdin', (tests / 'vagrant/simplevagrant').path ]) # Destroy check_call( rpuz + ['vagrant', 'destroy', (tests / 'vagrant/simplevagrant').path]) elif interactive: print("Test and press enter") sys.stdin.readline() finally: if (tests / 'vagrant/simplevagrant').exists(): (tests / 'vagrant/simplevagrant').rmtree() # Unpack Docker check_call(rpuz + ['docker', 'setup/create', 'simple.rpz', 'simpledocker']) print("\nDocker project set up in simpledocker") try: if run_docker: check_call(rpuz + ['docker', 'setup/build', 'simpledocker']) check_call(rpuz + ['docker', 'run', 'simpledocker']) # Get output file check_call( rpuz + ['docker', 'download', 'simpledocker', 'arg:doutput1.txt']) with Path('doutput1.txt').open(encoding='utf-8') as fp: assert fp.read().strip() == '42' # Replace input file check_call(rpuz + [ 'docker', 'upload', 'simpledocker', '%s:arg' % (tests / 'simple_input2.txt') ]) check_call(rpuz + ['docker', 'upload', 'simpledocker']) check_call(rpuz + ['showfiles', 'simpledocker']) # Run again check_call(rpuz + ['docker', 'run', 'simpledocker']) # Get output file check_call( rpuz + ['docker', 'download', 'simpledocker', 'arg:doutput2.txt']) with Path('doutput2.txt').open(encoding='utf-8') as fp: assert fp.read().strip() == '36' # Destroy check_call(rpuz + ['docker', 'destroy', 'simpledocker']) elif interactive: print("Test and press enter") sys.stdin.readline() finally: if Path('simpledocker').exists(): Path('simpledocker').rmtree() # ######################################## # 'threads' program: testrun # # Build build('threads', ['threads.c'], ['-lpthread']) # Trace check_call(rpz + ['testrun', './threads']) # ######################################## # 'segv' program: testrun # # Build build('segv', ['segv.c']) # Trace check_call(rpz + ['testrun', './segv']) # ######################################## # 'exec_echo' program: trace, pack, run --cmdline # # Build build('exec_echo', ['exec_echo.c']) # Trace check_call(rpz + ['trace', './exec_echo', 'originalexecechooutput']) # Pack check_call(rpz + ['pack', 'exec_echo.rpz']) # Unpack chroot check_call(sudo + rpuz + ['chroot', 'setup', 'exec_echo.rpz', 'echochroot']) try: # Run original command-line output = check_output(sudo + rpuz + ['chroot', 'run', 'echochroot']) assert output == b'originalexecechooutput\n' # Prints out command-line output = check_output(sudo + rpuz + ['chroot', 'run', 'echochroot', '--cmdline']) assert any(b'./exec_echo originalexecechooutput' == s.strip() for s in output.split(b'\n')) # Run with different command-line output = check_output(sudo + rpuz + [ 'chroot', 'run', 'echochroot', '--cmdline', './exec_echo', 'changedexecechooutput' ]) assert output == b'changedexecechooutput\n' finally: check_call(sudo + rpuz + ['chroot', 'destroy', 'echochroot']) # ######################################## # 'exec_echo' program: testrun # This is built with -m32 so that we transition: # python (x64) -> exec_echo (i386) -> echo (x64) # if sys.maxsize > 2**32: # Build build('exec_echo32', ['exec_echo.c'], ['-m32']) # Trace check_call(rpz + ['testrun', './exec_echo32 42']) else: print("Can't try exec_echo transitions: not running on 64bits") # ######################################## # Tracing non-existing program # check_call(rpz + ['testrun', './doesntexist']) # ######################################## # 'connect' program: testrun # # Build build('connect', ['connect.c']) # Trace stderr = check_errout(rpz + ['testrun', './connect']) stderr = stderr.split(b'\n') assert not any(b'program exited with non-zero code' in l for l in stderr) assert any( re.search(br'process connected to [0-9.]+:80', l) for l in stderr) # ######################################## # Copies back coverage report # coverage = Path('.coverage') if coverage.exists(): coverage.copyfile(tests.parent / '.coverage.runpy')
def directory_create(args): """Unpacks the experiment in a folder. Only the files that are not part of a package are copied (unless they are missing from the system and were packed). In addition, input files are put in a tar.gz (so they can be put back after an upload) and the configuration file is extracted. """ if not args.pack: logging.critical("setup needs the pack filename") sys.exit(1) pack = Path(args.pack[0]) target = Path(args.target[0]) if target.exists(): logging.critical("Target directory exists") sys.exit(1) if DefaultAbstractPath is not PosixPath: logging.critical("Not unpacking on POSIX system") sys.exit(1) signals.pre_setup(target=target, pack=pack) # Unpacks configuration file tar = tarfile.open(str(pack), 'r:*') member = tar.getmember('METADATA/config.yml') member.name = 'config.yml' tar.extract(member, str(target)) # Loads config runs, packages, other_files = load_config_file(target / 'config.yml', True) target.mkdir() root = (target / 'root').absolute() root.mkdir() # Checks packages missing_files = False for pkg in packages: if pkg.packfiles: continue for f in pkg.files: f = Path(f.path) if not f.exists(): logging.error( "Missing file %s (from package %s that wasn't packed) " "on host, experiment will probably miss it.", f, pkg.name) missing_files = True if missing_files: logging.error("Some packages are missing, you should probably install " "them.\nUse 'reprounzip installpkgs -h' for help") # Unpacks files if any('..' in m.name or m.name.startswith('/') for m in tar.getmembers()): logging.critical("Tar archive contains invalid pathnames") sys.exit(1) members = [m for m in tar.getmembers() if m.name.startswith('DATA/')] for m in members: m.name = m.name[5:] # Makes symlink targets relative for m in members: if not m.issym(): continue linkname = PosixPath(m.linkname) if linkname.is_absolute: m.linkname = join_root(root, PosixPath(m.linkname)).path logging.info("Extracting files...") tar.extractall(str(root), members) tar.close() # Gets library paths lib_dirs = [] p = subprocess.Popen(['/sbin/ldconfig', '-v', '-N'], stdout=subprocess.PIPE) try: for l in p.stdout: if len(l) < 3 or l[0] in (b' ', b'\t'): continue if l.endswith(b':\n'): lib_dirs.append(Path(l[:-2])) finally: p.wait() # Original input files, so upload can restore them if any(run['input_files'] for run in runs): logging.info("Packing up original input files...") inputtar = tarfile.open(str(target / 'inputs.tar.gz'), 'w:gz') for run in runs: for ifile in itervalues(run['input_files']): inputtar.add(str(join_root(root, PosixPath(ifile))), str(PosixPath(ifile))) inputtar.close() # Meta-data for reprounzip write_dict(target / '.reprounzip', {}, 'directory') signals.post_setup(target=target)
def chroot_create(args): """Unpacks the experiment in a folder so it can be run with chroot. All the files in the pack are unpacked; system files are copied only if they were not packed, and busybox is installed if /bin/sh wasn't packed. In addition, input files are put in a tar.gz (so they can be put back after an upload) and the configuration file is extracted. """ if not args.pack: logging.critical("setup/create needs the pack filename") sys.exit(1) pack = Path(args.pack[0]) target = Path(args.target[0]) if target.exists(): logging.critical("Target directory exists") sys.exit(1) if DefaultAbstractPath is not PosixPath: logging.critical("Not unpacking on POSIX system") sys.exit(1) signals.pre_setup(target=target, pack=pack) # We can only restore owner/group of files if running as root restore_owner = should_restore_owner(args.restore_owner) # Unpacks configuration file rpz_pack = RPZPack(pack) rpz_pack.extract_config(target / 'config.yml') # Loads config config = load_config_file(target / 'config.yml', True) packages = config.packages target.mkdir() root = (target / 'root').absolute() root.mkdir() try: # Checks that everything was packed packages_not_packed = [pkg for pkg in packages if not pkg.packfiles] if packages_not_packed: record_usage(chroot_missing_pkgs=True) logging.warning("According to configuration, some files were left " "out because they belong to the following " "packages:%s\nWill copy files from HOST SYSTEM", ''.join('\n %s' % pkg for pkg in packages_not_packed)) missing_files = False for pkg in packages_not_packed: for f in pkg.files: path = Path(f.path) if not path.exists(): logging.error( "Missing file %s (from package %s) on host, " "experiment will probably miss it", path, pkg.name) missing_files = True continue dest = join_root(root, path) dest.parent.mkdir(parents=True) if path.is_link(): dest.symlink(path.read_link()) else: path.copy(dest) if restore_owner: stat = path.stat() dest.chown(stat.st_uid, stat.st_gid) if missing_files: record_usage(chroot_mising_files=True) # Unpacks files members = rpz_pack.list_data() for m in members: # Remove 'DATA/' prefix m.name = str(rpz_pack.remove_data_prefix(m.name)) if not restore_owner: uid = os.getuid() gid = os.getgid() for m in members: m.uid = uid m.gid = gid logging.info("Extracting files...") rpz_pack.extract_data(root, members) rpz_pack.close() # Sets up /bin/sh and /usr/bin/env, downloading busybox if necessary sh_path = join_root(root, Path('/bin/sh')) env_path = join_root(root, Path('/usr/bin/env')) if not sh_path.lexists() or not env_path.lexists(): logging.info("Setting up busybox...") busybox_path = join_root(root, Path('/bin/busybox')) busybox_path.parent.mkdir(parents=True) with make_dir_writable(join_root(root, Path('/bin'))): download_file(busybox_url(config.runs[0]['architecture']), busybox_path, 'busybox-%s' % config.runs[0]['architecture']) busybox_path.chmod(0o755) if not sh_path.lexists(): sh_path.parent.mkdir(parents=True) sh_path.symlink('/bin/busybox') if not env_path.lexists(): env_path.parent.mkdir(parents=True) env_path.symlink('/bin/busybox') # Original input files, so upload can restore them input_files = [f.path for f in itervalues(config.inputs_outputs) if f.read_runs] if input_files: logging.info("Packing up original input files...") inputtar = tarfile.open(str(target / 'inputs.tar.gz'), 'w:gz') for ifile in input_files: filename = join_root(root, ifile) if filename.exists(): inputtar.add(str(filename), str(ifile)) inputtar.close() # Meta-data for reprounzip metadata_write(target, metadata_initial_iofiles(config), 'chroot') signals.post_setup(target=target, pack=pack) except Exception: rmtree_fixed(root) raise
def chroot_create(args): """Unpacks the experiment in a folder so it can be run with chroot. All the files in the pack are unpacked; system files are copied only if they were not packed, and busybox is installed if /bin/sh wasn't packed. In addition, input files are put in a tar.gz (so they can be put back after an upload) and the configuration file is extracted. """ if not args.pack: logging.critical("setup/create needs the pack filename") sys.exit(1) pack = Path(args.pack[0]) target = Path(args.target[0]) if target.exists(): logging.critical("Target directory exists") sys.exit(1) if DefaultAbstractPath is not PosixPath: logging.critical("Not unpacking on POSIX system") sys.exit(1) signals.pre_setup(target=target, pack=pack) # We can only restore owner/group of files if running as root restore_owner = should_restore_owner(args.restore_owner) # Unpacks configuration file tar = tarfile.open(str(pack), 'r:*') member = tar.getmember('METADATA/config.yml') member.name = 'config.yml' tar.extract(member, str(target)) # Loads config runs, packages, other_files = load_config_file(target / 'config.yml', True) target.mkdir() root = (target / 'root').absolute() root.mkdir() # Checks that everything was packed packages_not_packed = [pkg for pkg in packages if not pkg.packfiles] if packages_not_packed: logging.warning( "According to configuration, some files were left out " "because they belong to the following packages:%s" "\nWill copy files from HOST SYSTEM", ''.join('\n %s' % pkg for pkg in packages_not_packed)) for pkg in packages_not_packed: for f in pkg.files: f = Path(f.path) if not f.exists(): logging.error( "Missing file %s (from package %s) on host, " "experiment will probably miss it", f, pkg.name) dest = join_root(root, f) dest.parent.mkdir(parents=True) if f.is_link(): dest.symlink(f.read_link()) else: f.copy(dest) if restore_owner: stat = f.stat() dest.chown(stat.st_uid, stat.st_gid) # Unpacks files if any('..' in m.name or m.name.startswith('/') for m in tar.getmembers()): logging.critical("Tar archive contains invalid pathnames") sys.exit(1) members = [m for m in tar.getmembers() if m.name.startswith('DATA/')] for m in members: m.name = m.name[5:] if not restore_owner: uid = os.getuid() gid = os.getgid() for m in members: m.uid = uid m.gid = gid logging.info("Extracting files...") tar.extractall(str(root), members) tar.close() # Sets up /bin/sh and /usr/bin/env, downloading busybox if necessary sh_path = join_root(root, Path('/bin/sh')) env_path = join_root(root, Path('/usr/bin/env')) if not sh_path.lexists() or not env_path.lexists(): logging.info("Setting up busybox...") busybox_path = join_root(root, Path('/bin/busybox')) busybox_path.parent.mkdir(parents=True) with make_dir_writable(join_root(root, Path('/bin'))): download_file(busybox_url(runs[0]['architecture']), busybox_path) busybox_path.chmod(0o755) if not sh_path.lexists(): sh_path.parent.mkdir(parents=True) sh_path.symlink('/bin/busybox') if not env_path.lexists(): env_path.parent.mkdir(parents=True) env_path.symlink('/bin/busybox') # Original input files, so upload can restore them if any(run['input_files'] for run in runs): logging.info("Packing up original input files...") inputtar = tarfile.open(str(target / 'inputs.tar.gz'), 'w:gz') for run in runs: for ifile in itervalues(run['input_files']): inputtar.add(str(join_root(root, PosixPath(ifile))), str(PosixPath(ifile))) inputtar.close() # Meta-data for reprounzip write_dict(target / '.reprounzip', {}, 'chroot') signals.post_setup(target=target)
def directory_create(args): """Unpacks the experiment in a folder. Only the files that are not part of a package are copied (unless they are missing from the system and were packed). In addition, input files are put in a tar.gz (so they can be put back after an upload) and the configuration file is extracted. """ if not args.pack: logger.critical("setup needs the pack filename") sys.exit(1) pack = Path(args.pack[0]) target = Path(args.target[0]) if target.exists(): logger.critical("Target directory exists") sys.exit(1) if not issubclass(DefaultAbstractPath, PosixPath): logger.critical("Not unpacking on POSIX system") sys.exit(1) signals.pre_setup(target=target, pack=pack) # Unpacks configuration file rpz_pack = RPZPack(pack) rpz_pack.extract_config(target / 'config.yml') # Loads config config = load_config_file(target / 'config.yml', True) packages = config.packages target.mkdir() root = (target / 'root').absolute() # Checks packages missing_files = False for pkg in packages: if pkg.packfiles: continue for f in pkg.files: if not Path(f.path).exists(): logger.error( "Missing file %s (from package %s that wasn't packed) " "on host, experiment will probably miss it.", f, pkg.name) missing_files = True if missing_files: record_usage(directory_missing_pkgs=True) logger.error("Some packages are missing, you should probably install " "them.\nUse 'reprounzip installpkgs -h' for help") root.mkdir() try: # Unpacks files members = rpz_pack.list_data() for m in members: # Remove 'DATA/' prefix m.name = str(rpz_pack.remove_data_prefix(m.name)) # Makes symlink targets relative if m.issym(): linkname = PosixPath(m.linkname) if linkname.is_absolute: m.linkname = join_root(root, PosixPath(m.linkname)).path logger.info("Extracting files...") rpz_pack.extract_data(root, members) rpz_pack.close() # Original input files, so upload can restore them input_files = [f.path for f in config.inputs_outputs.values() if f.read_runs] if input_files: logger.info("Packing up original input files...") inputtar = tarfile.open(str(target / 'inputs.tar.gz'), 'w:gz') for ifile in input_files: filename = join_root(root, ifile) if filename.exists(): inputtar.add(str(filename), str(ifile)) inputtar.close() # Meta-data for reprounzip metadata_write(target, metadata_initial_iofiles(config), 'directory') signals.post_setup(target=target, pack=pack) except Exception: rmtree_fixed(root) raise
def directory_run(args): """Runs the command in the directory. """ target = Path(args.target[0]) read_dict(target / '.reprounzip', 'directory') cmdline = args.cmdline # Loads config runs, packages, other_files = load_config_file(target / 'config.yml', True) selected_runs = get_runs(runs, args.run, cmdline) root = (target / 'root').absolute() # Gets library paths lib_dirs = [] p = subprocess.Popen(['/sbin/ldconfig', '-v', '-N'], stdout=subprocess.PIPE) try: for l in p.stdout: if len(l) < 3 or l[0] in (b' ', b'\t'): continue if l.endswith(b':\n'): lib_dirs.append(Path(l[:-2])) finally: p.wait() lib_dirs = ( 'export LD_LIBRARY_PATH=%s' % ':'.join(shell_escape(unicode_(join_root(root, d))) for d in lib_dirs)) cmds = [lib_dirs] for run_number in selected_runs: run = runs[run_number] cmd = 'cd %s && ' % shell_escape( unicode_(join_root(root, Path(run['workingdir'])))) cmd += '/usr/bin/env -i ' environ = run['environ'] if args.x11: if 'DISPLAY' in os.environ: environ['DISPLAY'] = os.environ['DISPLAY'] if 'XAUTHORITY' in os.environ: environ['XAUTHORITY'] = os.environ['XAUTHORITY'] cmd += ' '.join('%s=%s' % (k, shell_escape(v)) for k, v in iteritems(environ) if k != 'PATH') cmd += ' ' # PATH # Get the original PATH components path = [ PosixPath(d) for d in run['environ'].get('PATH', '').split(':') ] # The same paths but in the directory dir_path = [join_root(root, d) for d in path if d.root == '/'] # Rebuild string path = ':'.join(unicode_(d) for d in dir_path + path) cmd += 'PATH=%s ' % shell_escape(path) # FIXME : Use exec -a or something if binary != argv[0] if cmdline is None: argv = run['argv'] # Rewrites command-line arguments that are absolute filenames rewritten = False for i in irange(len(argv)): try: p = Path(argv[i]) except UnicodeEncodeError: continue if p.is_absolute: rp = join_root(root, p) if (rp.exists() or (len(rp.components) > 3 and rp.parent.exists())): argv[i] = str(rp) rewritten = True if rewritten: logging.warning("Rewrote command-line as: %s", ' '.join(shell_escape(a) for a in argv)) else: argv = cmdline cmd += ' '.join(shell_escape(a) for a in argv) cmds.append(cmd) cmds = ' && '.join(cmds) signals.pre_run(target=target) retcode = interruptible_call(cmds, shell=True) stderr.write("\n*** Command finished, status: %d\n" % retcode) signals.post_run(target=target, retcode=retcode)
def chroot_create(args): """Unpacks the experiment in a folder so it can be run with chroot. All the files in the pack are unpacked; system files are copied only if they were not packed, and busybox is installed if /bin/sh wasn't packed. In addition, input files are put in a tar.gz (so they can be put back after an upload) and the configuration file is extracted. """ if not args.pack: logger.critical("setup/create needs the pack filename") sys.exit(1) pack = Path(args.pack[0]) target = Path(args.target[0]) if target.exists(): logger.critical("Target directory exists") sys.exit(1) if not issubclass(DefaultAbstractPath, PosixPath): logger.critical("Not unpacking on POSIX system") sys.exit(1) signals.pre_setup(target=target, pack=pack) # We can only restore owner/group of files if running as root restore_owner = should_restore_owner(args.restore_owner) # Unpacks configuration file rpz_pack = RPZPack(pack) rpz_pack.extract_config(target / 'config.yml') # Loads config config = load_config_file(target / 'config.yml', True) packages = config.packages target.mkdir() root = (target / 'root').absolute() root.mkdir() try: # Checks that everything was packed packages_not_packed = [pkg for pkg in packages if not pkg.packfiles] if packages_not_packed: record_usage(chroot_missing_pkgs=True) logger.warning("According to configuration, some files were left " "out because they belong to the following " "packages:%s\nWill copy files from HOST SYSTEM", ''.join('\n %s' % pkg for pkg in packages_not_packed)) missing_files = False for pkg in packages_not_packed: for f in pkg.files: path = Path(f.path) if not path.exists(): logger.error( "Missing file %s (from package %s) on host, " "experiment will probably miss it", path, pkg.name) missing_files = True continue dest = join_root(root, path) dest.parent.mkdir(parents=True) if path.is_link(): dest.symlink(path.read_link()) else: path.copy(dest) if restore_owner: stat = path.stat() dest.chown(stat.st_uid, stat.st_gid) if missing_files: record_usage(chroot_mising_files=True) # Unpacks files members = rpz_pack.list_data() for m in members: # Remove 'DATA/' prefix m.name = str(rpz_pack.remove_data_prefix(m.name)) if not restore_owner: uid = os.getuid() gid = os.getgid() for m in members: m.uid = uid m.gid = gid logger.info("Extracting files...") rpz_pack.extract_data(root, members) rpz_pack.close() resolvconf_src = Path('/etc/resolv.conf') if resolvconf_src.exists(): try: resolvconf_src.copy(root / 'etc/resolv.conf') except IOError: pass # Sets up /bin/sh and /usr/bin/env, downloading busybox if necessary sh_path = join_root(root, Path('/bin/sh')) env_path = join_root(root, Path('/usr/bin/env')) if not sh_path.lexists() or not env_path.lexists(): logger.info("Setting up busybox...") busybox_path = join_root(root, Path('/bin/busybox')) busybox_path.parent.mkdir(parents=True) with make_dir_writable(join_root(root, Path('/bin'))): download_file(busybox_url(config.runs[0]['architecture']), busybox_path, 'busybox-%s' % config.runs[0]['architecture']) busybox_path.chmod(0o755) if not sh_path.lexists(): sh_path.parent.mkdir(parents=True) sh_path.symlink('/bin/busybox') if not env_path.lexists(): env_path.parent.mkdir(parents=True) env_path.symlink('/bin/busybox') # Original input files, so upload can restore them input_files = [f.path for f in config.inputs_outputs.values() if f.read_runs] if input_files: logger.info("Packing up original input files...") inputtar = tarfile.open(str(target / 'inputs.tar.gz'), 'w:gz') for ifile in input_files: filename = join_root(root, ifile) if filename.exists(): inputtar.add(str(filename), str(ifile)) inputtar.close() # Meta-data for reprounzip metadata_write(target, metadata_initial_iofiles(config), 'chroot') signals.post_setup(target=target, pack=pack) except Exception: rmtree_fixed(root) raise
def download(self, remote_path, local_path): remote_path = join_root(self.root, remote_path) # Copy remote_path.copyfile(local_path) remote_path.copymode(local_path)
def vagrant_setup_create(args): """Sets up the experiment to be run in a Vagrant-built virtual machine. This can either build a chroot or not. If building a chroot, we do just like without Vagrant: we copy all the files and only get what's missing from the host. But we do install automatically the packages whose files are required. If not building a chroot, we install all the packages, and only unpack files that don't come from packages. In short: files from packages with packfiles=True will only be used if building a chroot. """ if not args.pack: logging.critical("setup/create needs the pack filename") sys.exit(1) pack = Path(args.pack[0]) target = Path(args.target[0]) if target.exists(): logging.critical("Target directory exists") sys.exit(1) use_chroot = args.use_chroot mount_bind = args.bind_magic_dirs record_usage(use_chroot=use_chroot, mount_bind=mount_bind) signals.pre_setup(target=target, pack=pack) # Unpacks configuration file tar = tarfile.open(str(pack), 'r:*') member = tar.getmember('METADATA/config.yml') member.name = 'config.yml' tar.extract(member, str(target)) tar.close() # Loads config runs, packages, other_files = load_config(target / 'config.yml', True) if args.base_image and args.base_image[0]: record_usage(vagrant_explicit_image=True) box = args.base_image[0] if args.distribution: target_distribution = args.distribution[0] else: target_distribution = None else: target_distribution, box = select_box(runs) logging.info("Using box %s", box) logging.debug("Distribution: %s", target_distribution or "unknown") # If using chroot, we might still need to install packages to get missing # (not packed) files if use_chroot: packages = [pkg for pkg in packages if not pkg.packfiles] if packages: record_usage(vagrant_install_pkgs=True) logging.info( "Some packages were not packed, so we'll install and " "copy their files\n" "Packages that are missing:\n%s", ' '.join(pkg.name for pkg in packages)) if packages: try: installer = select_installer(pack, runs, target_distribution) except CantFindInstaller as e: logging.error( "Need to install %d packages but couldn't select a " "package installer: %s", len(packages), e) target.mkdir(parents=True) # Writes setup script logging.info("Writing setup script %s...", target / 'setup.sh') with (target / 'setup.sh').open('w', encoding='utf-8', newline='\n') as fp: fp.write('#!/bin/sh\n\nset -e\n\n') if packages: # Updates package sources fp.write(installer.update_script()) fp.write('\n') # Installs necessary packages fp.write(installer.install_script(packages)) fp.write('\n') # TODO : Compare package versions (painful because of sh) # Untar if use_chroot: fp.write('\n' 'mkdir /experimentroot; cd /experimentroot\n') fp.write('tar zpxf /vagrant/experiment.rpz ' '--numeric-owner --strip=1 DATA\n') if mount_bind: fp.write('\n' 'mkdir -p /experimentroot/dev\n' 'mount -o rbind /dev /experimentroot/dev\n' 'mkdir -p /experimentroot/proc\n' 'mount -o rbind /proc /experimentroot/proc\n') for pkg in packages: fp.write('\n# Copies files from package %s\n' % pkg.name) for f in pkg.files: f = f.path dest = join_root(PosixPath('/experimentroot'), f) fp.write('mkdir -p %s\n' % shell_escape(unicode_(f.parent))) fp.write('cp -L %s %s\n' % (shell_escape( unicode_(f)), shell_escape(unicode_(dest)))) else: fp.write('\ncd /\n') paths = set() pathlist = [] dataroot = PosixPath('DATA') # Adds intermediate directories, and checks for existence in the # tar tar = tarfile.open(str(pack), 'r:*') for f in other_files: path = PosixPath('/') for c in f.path.components[1:]: path = path / c if path in paths: continue paths.add(path) datapath = join_root(dataroot, path) try: tar.getmember(str(datapath)) except KeyError: logging.info("Missing file %s", datapath) else: pathlist.append(unicode_(datapath)) tar.close() # FIXME : for some reason we need reversed() here, I'm not sure # why. Need to read more of tar's docs. # TAR bug: --no-overwrite-dir removes --keep-old-files # TAR bug: there is no way to make --keep-old-files not report an # error if an existing file is encountered. --skip-old-files was # introduced too recently. Instead, we just ignore the exit status fp.write('tar zpxf /vagrant/experiment.rpz --keep-old-files ' '--numeric-owner --strip=1 %s || /bin/true\n' % ' '.join(shell_escape(p) for p in reversed(pathlist))) # Copies /bin/sh + dependencies if use_chroot: url = busybox_url(runs[0]['architecture']) fp.write(r''' mkdir -p /experimentroot/bin mkdir -p /experimentroot/usr/bin if [ ! -e /experimentroot/bin/sh -o ! -e /experimentroot/usr/bin/env ]; then wget --quiet -O /experimentroot/bin/busybox {url} chmod +x /experimentroot/bin/busybox fi [ -e /experimentroot/bin/sh ] || \ ln -s /bin/busybox /experimentroot/bin/sh [ -e /experimentroot/usr/bin/env ] || \ ln -s /bin/busybox /experimentroot/usr/bin/env '''.format(url=url)) # Copies pack logging.info("Copying pack file...") pack.copyfile(target / 'experiment.rpz') # Writes Vagrant file logging.info("Writing %s...", target / 'Vagrantfile') with (target / 'Vagrantfile').open('w', encoding='utf-8', newline='\n') as fp: # Vagrant header and version fp.write('# -*- mode: ruby -*-\n' '# vi: set ft=ruby\n\n' 'VAGRANTFILE_API_VERSION = "2"\n\n' 'Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|\n') # Selects which box to install fp.write(' config.vm.box = "%s"\n' % box) # Run the setup script on the virtual machine fp.write(' config.vm.provision "shell", path: "setup.sh"\n') fp.write('end\n') # Meta-data for reprounzip write_dict(target / '.reprounzip', {'use_chroot': use_chroot}) signals.post_setup(target=target)
def functional_tests(raise_warnings, interactive, run_vagrant, run_docker): python = [sys.executable] # Can't match on the SignalWarning category here because of a Python bug # http://bugs.python.org/issue22543 python.extend(['-W', 'error:signal']) if 'COVER' in os.environ: python.extend(['-m'] + os.environ['COVER'].split(' ')) reprozip_main = tests.parent / 'reprozip/reprozip/main.py' reprounzip_main = tests.parent / 'reprounzip/reprounzip/main.py' verbose = ['-v'] * 3 rpz = python + [reprozip_main.absolute().path] + verbose rpuz = python + [reprounzip_main.absolute().path] + verbose # ######################################## # 'simple' program: trace, pack, info, unpack # # Build build('simple', ['simple.c']) # Trace check_call(rpz + [ 'trace', '-d', 'rpz-simple', './simple', (tests / 'simple_input.txt').path, 'simple_output.txt' ]) orig_output_location = Path('simple_output.txt').absolute() assert orig_output_location.is_file() with orig_output_location.open(encoding='utf-8') as fp: assert fp.read().strip() == '42' orig_output_location.remove() # Read config with Path('rpz-simple/config.yml').open(encoding='utf-8') as fp: conf = yaml.safe_load(fp) other_files = set(Path(f).absolute() for f in conf['other_files']) expected = [Path('simple'), (tests / 'simple_input.txt')] assert other_files.issuperset([f.resolve() for f in expected]) # Check input and output files input_files = conf['runs'][0]['input_files'] assert (dict((k, Path(f).name) for k, f in iteritems(input_files)) == { 'arg': b'simple_input.txt' }) output_files = conf['runs'][0]['output_files'] print(dict((k, Path(f).name) for k, f in iteritems(output_files))) # Here we don't test for dict equality, since we might have C coverage # files in the mix assert Path(output_files['arg']).name == b'simple_output.txt' # Pack check_call(rpz + ['pack', '-d', 'rpz-simple', 'simple.rpz']) Path('simple').remove() # Info check_call(rpuz + ['info', 'simple.rpz']) # Show files check_call(rpuz + ['showfiles', 'simple.rpz']) # Lists packages check_call(rpuz + ['installpkgs', '--summary', 'simple.rpz']) # Unpack directory check_call(rpuz + ['directory', 'setup', 'simple.rpz', 'simpledir']) # Run directory check_call(rpuz + ['directory', 'run', 'simpledir']) output_in_dir = join_root(Path('simpledir/root'), orig_output_location) with output_in_dir.open(encoding='utf-8') as fp: assert fp.read().strip() == '42' # Delete with wrong command (should fail) assert call(rpuz + ['chroot', 'destroy', 'simpledir']) != 0 # Delete directory check_call(rpuz + ['directory', 'destroy', 'simpledir']) # Unpack chroot check_call( ['sudo'] + rpuz + ['chroot', 'setup', '--bind-magic-dirs', 'simple.rpz', 'simplechroot']) # Run chroot check_call(['sudo'] + rpuz + ['chroot', 'run', 'simplechroot']) output_in_chroot = join_root(Path('simplechroot/root'), orig_output_location) with output_in_chroot.open(encoding='utf-8') as fp: assert fp.read().strip() == '42' # Get output file check_call(['sudo'] + rpuz + ['chroot', 'download', 'simplechroot', 'arg:output1.txt']) with Path('output1.txt').open(encoding='utf-8') as fp: assert fp.read().strip() == '42' # Replace input file check_call(['sudo'] + rpuz + [ 'chroot', 'upload', 'simplechroot', '%s:arg' % (tests / 'simple_input2.txt') ]) check_call(['sudo'] + rpuz + ['chroot', 'upload', 'simplechroot']) # Run again check_call(['sudo'] + rpuz + ['chroot', 'run', 'simplechroot']) output_in_chroot = join_root(Path('simplechroot/root'), orig_output_location) with output_in_chroot.open(encoding='utf-8') as fp: assert fp.read().strip() == '36' # Delete with wrong command (should fail) assert call(rpuz + ['directory', 'destroy', 'simplechroot']) != 0 # Delete chroot check_call(['sudo'] + rpuz + ['chroot', 'destroy', 'simplechroot']) if not Path('/vagrant').exists(): check_call(['sudo', 'sh', '-c', 'mkdir /vagrant; chmod 777 /vagrant']) # Unpack Vagrant-chroot check_call(rpuz + [ 'vagrant', 'setup/create', '--use-chroot', 'simple.rpz', '/vagrant/simplevagrantchroot' ]) print("\nVagrant project set up in simplevagrantchroot") try: if run_vagrant: check_call(rpuz + [ 'vagrant', 'run', '--no-stdin', '/vagrant/simplevagrantchroot' ]) # Destroy check_call(rpuz + ['vagrant', 'destroy', '/vagrant/simplevagrantchroot']) elif interactive: print("Test and press enter") sys.stdin.readline() finally: if Path('/vagrant/simplevagrantchroot').exists(): Path('/vagrant/simplevagrantchroot').rmtree() # Unpack Vagrant without chroot check_call(rpuz + [ 'vagrant', 'setup/create', '--dont-use-chroot', 'simple.rpz', '/vagrant/simplevagrant' ]) print("\nVagrant project set up in simplevagrant") try: if run_vagrant: check_call( rpuz + ['vagrant', 'run', '--no-stdin', '/vagrant/simplevagrant']) # Destroy check_call(rpuz + ['vagrant', 'destroy', '/vagrant/simplevagrant']) elif interactive: print("Test and press enter") sys.stdin.readline() finally: if Path('/vagrant/simplevagrant').exists(): Path('/vagrant/simplevagrant').rmtree() # Unpack Docker check_call(rpuz + ['docker', 'setup/create', 'simple.rpz', 'simpledocker']) print("\nDocker project set up in simpledocker") try: if run_docker: check_call(rpuz + ['docker', 'setup/build', 'simpledocker']) check_call(rpuz + ['docker', 'run', 'simpledocker']) # Get output file check_call( rpuz + ['docker', 'download', 'simpledocker', 'arg:doutput1.txt']) with Path('doutput1.txt').open(encoding='utf-8') as fp: assert fp.read().strip() == '42' # Replace input file check_call(rpuz + [ 'docker', 'upload', 'simpledocker', '%s:arg' % (tests / 'simple_input2.txt') ]) check_call(rpuz + ['docker', 'upload', 'simpledocker']) check_call(rpuz + ['showfiles', 'simpledocker']) # Run again check_call(rpuz + ['docker', 'run', 'simpledocker']) # Get output file check_call( rpuz + ['docker', 'download', 'simpledocker', 'arg:doutput2.txt']) with Path('doutput2.txt').open(encoding='utf-8') as fp: assert fp.read().strip() == '36' # Destroy check_call(rpuz + ['docker', 'destroy', 'simpledocker']) elif interactive: print("Test and press enter") sys.stdin.readline() finally: if Path('simpledocker').exists(): Path('simpledocker').rmtree() # ######################################## # 'threads' program: testrun # # Build build('threads', ['threads.c'], ['-lpthread']) # Trace check_call(rpz + ['testrun', './threads']) # ######################################## # 'segv' program: testrun # # Build build('segv', ['segv.c']) # Trace check_call(rpz + ['testrun', './segv']) # ######################################## # 'exec_echo' program: trace, pack, run --cmdline # # Build build('exec_echo', ['exec_echo.c']) # Trace check_call(rpz + ['trace', './exec_echo', 'originalexecechooutput']) # Pack check_call(rpz + ['pack', 'exec_echo.rpz']) # Unpack chroot check_call(['sudo'] + rpuz + ['chroot', 'setup', 'exec_echo.rpz', 'echochroot']) try: # Run original command-line output = check_output(['sudo'] + rpuz + ['chroot', 'run', 'echochroot']) assert output == b'originalexecechooutput\n' # Prints out command-line output = check_output(['sudo'] + rpuz + ['chroot', 'run', 'echochroot', '--cmdline']) assert any(b'./exec_echo originalexecechooutput' == s.strip() for s in output.split(b'\n')) # Run with different command-line output = check_output(['sudo'] + rpuz + [ 'chroot', 'run', 'echochroot', '--cmdline', './exec_echo', 'changedexecechooutput' ]) assert output == b'changedexecechooutput\n' finally: check_call(['sudo'] + rpuz + ['chroot', 'destroy', 'echochroot']) # ######################################## # 'exec_echo' program: testrun # This is built with -m32 so that we transition: # python (x64) -> exec_echo (i386) -> echo (x64) # if sys.maxsize > 2**32: # Build build('exec_echo32', ['exec_echo.c'], ['-m32']) # Trace check_call(rpz + ['testrun', './exec_echo32 42']) else: print("Can't try exec_echo transitions: not running on 64bits") # ######################################## # Tracing non-existing program # check_call(rpz + ['testrun', './doesntexist']) # ######################################## # 'connect' program: testrun # # Build build('connect', ['connect.c']) # Trace stderr = check_errout(rpz + ['testrun', './connect']) stderr = stderr.split(b'\n') assert not any(b'program exited with non-zero code' in l for l in stderr) assert any( re.search(br'process connected to [0-9.]+:80', l) for l in stderr) # ######################################## # Copies back coverage report # coverage = Path('.coverage') if coverage.exists(): coverage.copyfile(tests.parent / '.coverage.runpy')
def directory_run(args): """Runs the command in the directory. """ target = Path(args.target[0]) unpacked_info = metadata_read(target, 'directory') cmdline = args.cmdline # Loads config config = load_config_file(target / 'config.yml', True) runs = config.runs selected_runs = get_runs(runs, args.run, cmdline) root = (target / 'root').absolute() # Gets library paths lib_dirs = [] p = subprocess.Popen(['/sbin/ldconfig', '-v', '-N'], stdout=subprocess.PIPE) try: for l in p.stdout: if len(l) < 3 or l[0] in (b' ', b'\t'): continue if l.endswith(b':\n'): lib_dirs.append(Path(l[:-2])) finally: p.communicate() lib_dirs = ('export LD_LIBRARY_PATH=%s' % ':'.join( shell_escape(unicode_(join_root(root, d))) for d in lib_dirs)) cmds = [lib_dirs] for run_number in selected_runs: run = runs[run_number] cmd = 'cd %s && ' % shell_escape( unicode_(join_root(root, Path(run['workingdir'])))) cmd += '/usr/bin/env -i ' environ = run['environ'] environ = fixup_environment(environ, args) if args.x11: if 'DISPLAY' in os.environ: environ['DISPLAY'] = os.environ['DISPLAY'] if 'XAUTHORITY' in os.environ: environ['XAUTHORITY'] = os.environ['XAUTHORITY'] cmd += ' '.join('%s=%s' % (shell_escape(k), shell_escape(v)) for k, v in iteritems(environ) if k != 'PATH') cmd += ' ' # PATH # Get the original PATH components path = [PosixPath(d) for d in run['environ'].get('PATH', '').split(':')] # The same paths but in the directory dir_path = [join_root(root, d) for d in path if d.root == '/'] # Rebuild string path = ':'.join(unicode_(d) for d in dir_path + path) cmd += 'PATH=%s ' % shell_escape(path) # FIXME : Use exec -a or something if binary != argv[0] if cmdline is None: argv = run['argv'] # Rewrites command-line arguments that are absolute filenames rewritten = False for i in irange(len(argv)): try: p = Path(argv[i]) except UnicodeEncodeError: continue if p.is_absolute: rp = join_root(root, p) if (rp.exists() or (len(rp.components) > 3 and rp.parent.exists())): argv[i] = str(rp) rewritten = True if rewritten: logging.warning("Rewrote command-line as: %s", ' '.join(shell_escape(a) for a in argv)) else: argv = cmdline cmd += ' '.join(shell_escape(a) for a in argv) cmds.append(cmd) cmds = ' && '.join(cmds) signals.pre_run(target=target) retcode = interruptible_call(cmds, shell=True) stderr.write("\n*** Command finished, status: %d\n" % retcode) signals.post_run(target=target, retcode=retcode) # Update input file status metadata_update_run(config, unpacked_info, selected_runs) metadata_write(target, unpacked_info, 'directory')
def vagrant_setup_create(args): """Sets up the experiment to be run in a Vagrant-built virtual machine. This can either build a chroot or not. If building a chroot, we do just like without Vagrant: we copy all the files and only get what's missing from the host. But we do install automatically the packages whose files are required. If not building a chroot, we install all the packages, and only unpack files that don't come from packages. In short: files from packages with packfiles=True will only be used if building a chroot. """ if not args.pack: logging.critical("setup/create needs the pack filename") sys.exit(1) pack = Path(args.pack[0]) target = Path(args.target[0]) if target.exists(): logging.critical("Target directory exists") sys.exit(1) use_chroot = args.use_chroot mount_bind = args.bind_magic_dirs record_usage(use_chroot=use_chroot, mount_bind=mount_bind) signals.pre_setup(target=target, pack=pack) # Unpacks configuration file rpz_pack = RPZPack(pack) rpz_pack.extract_config(target / 'config.yml') # Loads config runs, packages, other_files = config = load_config(target / 'config.yml', True) if not args.memory: memory = None else: try: memory = int(args.memory[-1]) except ValueError: logging.critical("Invalid value for memory size: %r", args.memory) sys.exit(1) if args.base_image and args.base_image[0]: record_usage(vagrant_explicit_image=True) box = args.base_image[0] if args.distribution: target_distribution = args.distribution[0] else: target_distribution = None else: target_distribution, box = select_box(runs, gui=args.gui) logging.info("Using box %s", box) logging.debug("Distribution: %s", target_distribution or "unknown") # If using chroot, we might still need to install packages to get missing # (not packed) files if use_chroot: packages = [pkg for pkg in packages if not pkg.packfiles] if packages: record_usage(vagrant_install_pkgs=True) logging.info("Some packages were not packed, so we'll install and " "copy their files\n" "Packages that are missing:\n%s", ' '.join(pkg.name for pkg in packages)) if packages: try: installer = select_installer(pack, runs, target_distribution) except CantFindInstaller as e: logging.error("Need to install %d packages but couldn't select a " "package installer: %s", len(packages), e) target.mkdir(parents=True) try: # Writes setup script logging.info("Writing setup script %s...", target / 'setup.sh') with (target / 'setup.sh').open('w', encoding='utf-8', newline='\n') as fp: fp.write('#!/bin/sh\n\nset -e\n\n') if packages: # Updates package sources update_script = installer.update_script() if update_script: fp.write(update_script) fp.write('\n') # Installs necessary packages fp.write(installer.install_script(packages)) fp.write('\n') # TODO : Compare package versions (painful because of sh) # Untar if use_chroot: fp.write('\n' 'mkdir /experimentroot; cd /experimentroot\n') fp.write('tar zpxf /vagrant/data.tgz --numeric-owner ' '--strip=1 %s\n' % rpz_pack.data_prefix) if mount_bind: fp.write('\n' 'mkdir -p /experimentroot/dev\n' 'mkdir -p /experimentroot/proc\n') for pkg in packages: fp.write('\n# Copies files from package %s\n' % pkg.name) for f in pkg.files: f = f.path dest = join_root(PosixPath('/experimentroot'), f) fp.write('mkdir -p %s\n' % shell_escape(unicode_(f.parent))) fp.write('cp -L %s %s\n' % ( shell_escape(unicode_(f)), shell_escape(unicode_(dest)))) fp.write( '\n' 'cp /etc/resolv.conf /experimentroot/etc/resolv.conf\n') else: fp.write('\ncd /\n') paths = set() pathlist = [] # Adds intermediate directories, and checks for existence in # the tar logging.info("Generating file list...") data_files = rpz_pack.data_filenames() for f in other_files: if f.path.name == 'resolv.conf' and ( f.path.lies_under('/etc') or f.path.lies_under('/run') or f.path.lies_under('/var')): continue path = PosixPath('/') for c in rpz_pack.remove_data_prefix(f.path).components: path = path / c if path in paths: continue paths.add(path) if path in data_files: pathlist.append(path) else: logging.info("Missing file %s", path) # FIXME : for some reason we need reversed() here, I'm not sure # why. Need to read more of tar's docs. # TAR bug: --no-overwrite-dir removes --keep-old-files # TAR bug: there is no way to make --keep-old-files not report # an error if an existing file is encountered. --skip-old-files # was introduced too recently. Instead, we just ignore the exit # status with (target / 'rpz-files.list').open('wb') as lfp: for p in reversed(pathlist): lfp.write(join_root(rpz_pack.data_prefix, p).path) lfp.write(b'\0') fp.write('tar zpxf /vagrant/data.tgz --keep-old-files ' '--numeric-owner --strip=1 ' '--null -T /vagrant/rpz-files.list || /bin/true\n') # Copies busybox if use_chroot: arch = runs[0]['architecture'] download_file(busybox_url(arch), target / 'busybox', 'busybox-%s' % arch) fp.write(r''' cp /vagrant/busybox /experimentroot/busybox chmod +x /experimentroot/busybox mkdir -p /experimentroot/bin [ -e /experimentroot/bin/sh ] || \ ln -s /busybox /experimentroot/bin/sh ''') # Copies pack logging.info("Copying pack file...") rpz_pack.copy_data_tar(target / 'data.tgz') rpz_pack.close() # Writes Vagrant file logging.info("Writing %s...", target / 'Vagrantfile') with (target / 'Vagrantfile').open('w', encoding='utf-8', newline='\n') as fp: # Vagrant header and version fp.write( '# -*- mode: ruby -*-\n' '# vi: set ft=ruby\n\n' 'VAGRANTFILE_API_VERSION = "2"\n\n' 'Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|\n') # Selects which box to install fp.write(' config.vm.box = "%s"\n' % box) # Run the setup script on the virtual machine fp.write(' config.vm.provision "shell", path: "setup.sh"\n') # Memory size if memory is not None or args.gui: fp.write(' config.vm.provider "virtualbox" do |v|\n') if memory is not None: fp.write(' v.memory = %d\n' % memory) if args.gui: fp.write(' v.gui = true\n') fp.write(' end\n') fp.write('end\n') # Meta-data for reprounzip write_dict(target, metadata_initial_iofiles(config, {'use_chroot': use_chroot, 'gui': args.gui})) signals.post_setup(target=target, pack=pack) except Exception: target.rmtree(ignore_errors=True) raise
def chroot_create(args): """Unpacks the experiment in a folder so it can be run with chroot. All the files in the pack are unpacked; system files are copied only if they were not packed, and busybox is installed if /bin/sh wasn't packed. In addition, input files are put in a tar.gz (so they can be put back after an upload) and the configuration file is extracted. """ if not args.pack: logging.critical("setup/create needs the pack filename") sys.exit(1) pack = Path(args.pack[0]) target = Path(args.target[0]) if target.exists(): logging.critical("Target directory exists") sys.exit(1) if DefaultAbstractPath is not PosixPath: logging.critical("Not unpacking on POSIX system") sys.exit(1) signals.pre_setup(target=target, pack=pack) # We can only restore owner/group of files if running as root restore_owner = should_restore_owner(args.restore_owner) # Unpacks configuration file tar = tarfile.open(str(pack), 'r:*') member = tar.getmember('METADATA/config.yml') member.name = 'config.yml' tar.extract(member, str(target)) # Loads config runs, packages, other_files = load_config_file(target / 'config.yml', True) target.mkdir() root = (target / 'root').absolute() root.mkdir() # Checks that everything was packed packages_not_packed = [pkg for pkg in packages if not pkg.packfiles] if packages_not_packed: record_usage(chroot_missing_pkgs=True) logging.warning("According to configuration, some files were left out " "because they belong to the following packages:%s" "\nWill copy files from HOST SYSTEM", ''.join('\n %s' % pkg for pkg in packages_not_packed)) missing_files = False for pkg in packages_not_packed: for f in pkg.files: f = Path(f.path) if not f.exists(): logging.error( "Missing file %s (from package %s) on host, " "experiment will probably miss it", f, pkg.name) missing_files = True continue dest = join_root(root, f) dest.parent.mkdir(parents=True) if f.is_link(): dest.symlink(f.read_link()) else: f.copy(dest) if restore_owner: stat = f.stat() dest.chown(stat.st_uid, stat.st_gid) if missing_files: record_usage(chroot_mising_files=True) # Unpacks files if any('..' in m.name or m.name.startswith('/') for m in tar.getmembers()): logging.critical("Tar archive contains invalid pathnames") sys.exit(1) members = [m for m in tar.getmembers() if m.name.startswith('DATA/')] for m in members: m.name = m.name[5:] if not restore_owner: uid = os.getuid() gid = os.getgid() for m in members: m.uid = uid m.gid = gid logging.info("Extracting files...") tar.extractall(str(root), members) tar.close() # Sets up /bin/sh and /usr/bin/env, downloading busybox if necessary sh_path = join_root(root, Path('/bin/sh')) env_path = join_root(root, Path('/usr/bin/env')) if not sh_path.lexists() or not env_path.lexists(): logging.info("Setting up busybox...") busybox_path = join_root(root, Path('/bin/busybox')) busybox_path.parent.mkdir(parents=True) with make_dir_writable(join_root(root, Path('/bin'))): download_file(busybox_url(runs[0]['architecture']), busybox_path) busybox_path.chmod(0o755) if not sh_path.lexists(): sh_path.parent.mkdir(parents=True) sh_path.symlink('/bin/busybox') if not env_path.lexists(): env_path.parent.mkdir(parents=True) env_path.symlink('/bin/busybox') # Original input files, so upload can restore them if any(run['input_files'] for run in runs): logging.info("Packing up original input files...") inputtar = tarfile.open(str(target / 'inputs.tar.gz'), 'w:gz') for run in runs: for ifile in itervalues(run['input_files']): inputtar.add(str(join_root(root, PosixPath(ifile))), str(PosixPath(ifile))) inputtar.close() # Meta-data for reprounzip write_dict(target / '.reprounzip', {}, 'chroot') signals.post_setup(target=target)
def functional_tests(raise_warnings, interactive, run_vagrant, run_docker): rpz_python = [os.environ.get('REPROZIP_PYTHON', sys.executable)] rpuz_python = [os.environ.get('REPROUNZIP_PYTHON', sys.executable)] # Can't match on the SignalWarning category here because of a Python bug # http://bugs.python.org/issue22543 if raise_warnings: rpz_python.extend(['-W', 'error:signal']) rpuz_python.extend(['-W', 'error:signal']) if 'COVER' in os.environ: rpz_python.extend(['-m'] + os.environ['COVER'].split(' ')) rpuz_python.extend(['-m'] + os.environ['COVER'].split(' ')) reprozip_main = tests.parent / 'reprozip/reprozip/main.py' reprounzip_main = tests.parent / 'reprounzip/reprounzip/main.py' verbose = ['-v'] * 3 rpz = rpz_python + [reprozip_main.absolute().path] + verbose rpuz = rpuz_python + [reprounzip_main.absolute().path] + verbose print("Command lines are:\n%r\n%r" % (rpz, rpuz)) # ######################################## # testrun /bin/echo # output = check_output(rpz + ['testrun', '/bin/echo', 'outputhere']) assert any(b' 1 | /bin/echo outputhere ' in l for l in output.splitlines()) output = check_output( rpz + ['testrun', '-a', '/fake/path/echo', '/bin/echo', 'outputhere']) assert any(b' 1 | (/bin/echo) /fake/path/echo outputhere ' in l for l in output.splitlines()) # ######################################## # testrun multiple commands # check_call(rpz + [ 'testrun', 'bash', '-c', 'cat ../../../../../etc/passwd;' 'cd /var/lib;' 'cat ../../etc/group' ]) check_call(rpz + ['trace', '--overwrite', 'bash', '-c', 'cat /etc/passwd;echo']) check_call( rpz + ['trace', '--continue', 'sh', '-c', 'cat /etc/group;/usr/bin/id']) check_call(rpz + ['pack']) check_call(rpuz + ['graph', 'graph.dot']) check_call(rpuz + ['graph', 'graph2.dot', 'experiment.rpz']) sudo = ['sudo', '-E'] # -E to keep REPROZIP_USAGE_STATS # ######################################## # 'simple' program: trace, pack, info, unpack # def check_simple(args, stream, infile=1): output = check_output(args, stream).splitlines() try: first = output.index(b"Read 6 bytes") except ValueError: stderr.write("output = %r\n" % output) raise if infile == 1: assert output[first + 1] == b"a = 29, b = 13" assert output[first + 2] == b"result = 42" else: # infile == 2 assert output[first + 1] == b"a = 25, b = 11" assert output[first + 2] == b"result = 36" # Build build('simple', ['simple.c']) # Trace check_call(rpz + [ 'trace', '--overwrite', '-d', 'rpz-simple', './simple', (tests / 'simple_input.txt').path, 'simple_output.txt' ]) orig_output_location = Path('simple_output.txt').absolute() assert orig_output_location.is_file() with orig_output_location.open(encoding='utf-8') as fp: assert fp.read().strip() == '42' orig_output_location.remove() # Read config with Path('rpz-simple/config.yml').open(encoding='utf-8') as fp: conf = yaml.safe_load(fp) other_files = set(Path(f).absolute() for f in conf['other_files']) expected = [Path('simple'), (tests / 'simple_input.txt')] assert other_files.issuperset([f.resolve() for f in expected]) # Check input and output files inputs_outputs = conf['inputs_outputs'] # Exactly one input: "arg1", "...simple_input.txt" # Output: 'arg2', "...simple_output.txt" # There might be more output files: the C coverage files found = 0 for fdict in inputs_outputs: if Path(fdict['path']).name == b'simple_input.txt': assert fdict['name'] == 'arg1' assert fdict['read_by_runs'] == [0] assert not fdict.get('written_by_runs') found |= 0x01 elif Path(fdict['path']).name == b'simple_output.txt': assert fdict['name'] == 'arg2' assert not fdict.get('read_by_runs') assert fdict['written_by_runs'] == [0] found |= 0x02 else: # No other inputs assert not fdict.get('read_by_runs') assert found == 0x03 # Pack check_call(rpz + ['pack', '-d', 'rpz-simple', 'simple.rpz']) Path('simple').rename('simple.orig') # Info check_call(rpuz + ['info', 'simple.rpz']) # Show files check_call(rpuz + ['showfiles', 'simple.rpz']) # Lists packages check_call(rpuz + ['installpkgs', '--summary', 'simple.rpz']) # Unpack directory check_call(rpuz + ['directory', 'setup', 'simple.rpz', 'simpledir']) # Run directory check_simple(rpuz + ['directory', 'run', 'simpledir'], 'err') output_in_dir = join_root(Path('simpledir/root'), orig_output_location) with output_in_dir.open(encoding='utf-8') as fp: assert fp.read().strip() == '42' # Delete with wrong command (should fail) p = subprocess.Popen(rpuz + ['chroot', 'destroy', 'simpledir'], stderr=subprocess.PIPE) out, err = p.communicate() assert p.poll() != 0 err = err.splitlines() assert b"Wrong unpacker used" in err[0] assert err[1].startswith(b"usage: ") # Delete directory check_call(rpuz + ['directory', 'destroy', 'simpledir']) # Unpack chroot check_call( sudo + rpuz + ['chroot', 'setup', '--bind-magic-dirs', 'simple.rpz', 'simplechroot']) try: output_in_chroot = join_root(Path('simplechroot/root'), orig_output_location) # Run chroot check_simple(sudo + rpuz + ['chroot', 'run', 'simplechroot'], 'err') with output_in_chroot.open(encoding='utf-8') as fp: assert fp.read().strip() == '42' # Get output file check_call(sudo + rpuz + ['chroot', 'download', 'simplechroot', 'arg2:output1.txt']) with Path('output1.txt').open(encoding='utf-8') as fp: assert fp.read().strip() == '42' # Get random file check_call(sudo + rpuz + [ 'chroot', 'download', 'simplechroot', '%s:binc.bin' % (Path.cwd() / 'simple') ]) assert same_files('simple.orig', 'binc.bin') # Replace input file check_call(sudo + rpuz + [ 'chroot', 'upload', 'simplechroot', '%s:arg1' % (tests / 'simple_input2.txt') ]) check_call(sudo + rpuz + ['chroot', 'upload', 'simplechroot']) # Run again check_simple(sudo + rpuz + ['chroot', 'run', 'simplechroot'], 'err', 2) with output_in_chroot.open(encoding='utf-8') as fp: assert fp.read().strip() == '36' # Reset input file check_call(sudo + rpuz + ['chroot', 'upload', 'simplechroot', ':arg1']) # Run again check_simple(sudo + rpuz + ['chroot', 'run', 'simplechroot'], 'err') with output_in_chroot.open(encoding='utf-8') as fp: assert fp.read().strip() == '42' # Replace input file via path check_call(sudo + rpuz + [ 'chroot', 'upload', 'simplechroot', '%s:%s' % (tests / 'simple_input2.txt', tests / 'simple_input.txt') ]) check_call(sudo + rpuz + ['chroot', 'upload', 'simplechroot']) # Run again check_simple(sudo + rpuz + ['chroot', 'run', 'simplechroot'], 'err', 2) # Delete with wrong command (should fail) p = subprocess.Popen(rpuz + ['directory', 'destroy', 'simplechroot'], stderr=subprocess.PIPE) out, err = p.communicate() assert p.poll() != 0 err = err.splitlines() assert b"Wrong unpacker used" in err[0] assert err[1].startswith(b"usage:") finally: # Delete chroot check_call(sudo + rpuz + ['chroot', 'destroy', 'simplechroot']) # Use reprounzip-vistrails with chroot check_call(sudo + rpuz + [ 'chroot', 'setup', '--bind-magic-dirs', 'simple.rpz', 'simplechroot_vt' ]) try: output_in_chroot = join_root(Path('simplechroot_vt/root'), orig_output_location) # Run using reprounzip-vistrails check_simple( sudo + rpuz_python + [ '-m', 'reprounzip.plugins.vistrails', '1', 'chroot', 'simplechroot_vt', '0', '--input-file', 'arg1:%s' % (tests / 'simple_input2.txt'), '--output-file', 'arg2:output_vt.txt' ], 'err', 2) with output_in_chroot.open(encoding='utf-8') as fp: assert fp.read().strip() == '36' finally: # Delete chroot check_call(sudo + rpuz + ['chroot', 'destroy', 'simplechroot_vt']) if not (tests / 'vagrant').exists(): check_call([ 'sudo', 'sh', '-c', 'mkdir %(d)s; chmod 777 %(d)s' % { 'd': tests / 'vagrant' } ]) # Unpack Vagrant-chroot check_call(rpuz + [ 'vagrant', 'setup/create', '--memory', '512', '--use-chroot', 'simple.rpz', (tests / 'vagrant/simplevagrantchroot').path ]) print("\nVagrant project set up in simplevagrantchroot") try: if run_vagrant: check_simple( rpuz + [ 'vagrant', 'run', '--no-stdin', (tests / 'vagrant/simplevagrantchroot').path ], 'out') # Get output file check_call(rpuz + [ 'vagrant', 'download', (tests / 'vagrant/simplevagrantchroot' ).path, 'arg2:voutput1.txt' ]) with Path('voutput1.txt').open(encoding='utf-8') as fp: assert fp.read().strip() == '42' # Get random file check_call(rpuz + [ 'vagrant', 'download', (tests / 'vagrant/simplevagrantchroot').path, '%s:binvc.bin' % (Path.cwd() / 'simple') ]) assert same_files('simple.orig', 'binvc.bin') # Replace input file check_call(rpuz + [ 'vagrant', 'upload', (tests / 'vagrant/simplevagrantchroot').path, '%s:arg1' % (tests / 'simple_input2.txt') ]) check_call(rpuz + [ 'vagrant', 'upload', (tests / 'vagrant/simplevagrantchroot').path ]) # Run again check_simple( rpuz + [ 'vagrant', 'run', '--no-stdin', (tests / 'vagrant/simplevagrantchroot').path ], 'out', 2) # Get output file check_call(rpuz + [ 'vagrant', 'download', (tests / 'vagrant/simplevagrantchroot' ).path, 'arg2:voutput2.txt' ]) with Path('voutput2.txt').open(encoding='utf-8') as fp: assert fp.read().strip() == '36' # Reset input file check_call(rpuz + [ 'vagrant', 'upload', (tests / 'vagrant/simplevagrantchroot').path, ':arg1' ]) # Run again check_simple( rpuz + [ 'vagrant', 'run', '--no-stdin', (tests / 'vagrant/simplevagrantchroot').path ], 'out') # Get output file check_call(rpuz + [ 'vagrant', 'download', (tests / 'vagrant/simplevagrantchroot' ).path, 'arg2:voutput1.txt' ]) with Path('voutput1.txt').open(encoding='utf-8') as fp: assert fp.read().strip() == '42' # Replace input file via path check_call(rpuz + [ 'vagrant', 'upload', (tests / 'vagrant/simplevagrantchroot').path, '%s:%s' % (tests / 'simple_input2.txt', tests / 'simple_input.txt') ]) # Run again check_simple( rpuz + [ 'vagrant', 'run', '--no-stdin', (tests / 'vagrant/simplevagrantchroot').path ], 'out', 2) # Destroy check_call(rpuz + [ 'vagrant', 'destroy', (tests / 'vagrant/simplevagrantchroot').path ]) elif interactive: print("Test and press enter") sys.stdin.readline() finally: if (tests / 'vagrant/simplevagrantchroot').exists(): (tests / 'vagrant/simplevagrantchroot').rmtree() # Unpack Vagrant without chroot check_call(rpuz + [ 'vagrant', 'setup/create', '--dont-use-chroot', 'simple.rpz', (tests / 'vagrant/simplevagrant').path ]) print("\nVagrant project set up in simplevagrant") try: if run_vagrant: check_simple( rpuz + [ 'vagrant', 'run', '--no-stdin', (tests / 'vagrant/simplevagrant').path ], 'out') # Get output file check_call(rpuz + [ 'vagrant', 'download', (tests / 'vagrant/simplevagrant').path, 'arg2:woutput1.txt' ]) with Path('woutput1.txt').open(encoding='utf-8') as fp: assert fp.read().strip() == '42' # Get random file check_call(rpuz + [ 'vagrant', 'download', (tests / 'vagrant/simplevagrant').path, '%s:binvs.bin' % (Path.cwd() / 'simple') ]) assert same_files('simple.orig', 'binvs.bin') # Replace input file check_call(rpuz + [ 'vagrant', 'upload', (tests / 'vagrant/simplevagrant').path, '%s:arg1' % (tests / 'simple_input2.txt') ]) check_call( rpuz + ['vagrant', 'upload', (tests / 'vagrant/simplevagrant').path]) # Run again check_simple( rpuz + [ 'vagrant', 'run', '--no-stdin', (tests / 'vagrant/simplevagrant').path ], 'out', 2) # Get output file check_call(rpuz + [ 'vagrant', 'download', (tests / 'vagrant/simplevagrant').path, 'arg2:woutput2.txt' ]) with Path('woutput2.txt').open(encoding='utf-8') as fp: assert fp.read().strip() == '36' # Reset input file check_call(rpuz + [ 'vagrant', 'upload', (tests / 'vagrant/simplevagrant').path, ':arg1' ]) # Run again check_simple( rpuz + [ 'vagrant', 'run', '--no-stdin', (tests / 'vagrant/simplevagrant').path ], 'out') # Get output file check_call(rpuz + [ 'vagrant', 'download', (tests / 'vagrant/simplevagrant').path, 'arg2:voutput1.txt' ]) with Path('voutput1.txt').open(encoding='utf-8') as fp: assert fp.read().strip() == '42' # Destroy check_call( rpuz + ['vagrant', 'destroy', (tests / 'vagrant/simplevagrant').path]) elif interactive: print("Test and press enter") sys.stdin.readline() finally: if (tests / 'vagrant/simplevagrant').exists(): (tests / 'vagrant/simplevagrant').rmtree() # Unpack Docker check_call(rpuz + ['docker', 'setup/create', 'simple.rpz', 'simpledocker']) print("\nDocker project set up in simpledocker") try: if run_docker: check_call(rpuz + ['docker', 'setup/build', 'simpledocker']) check_simple(rpuz + ['docker', 'run', 'simpledocker'], 'out') # Get output file check_call( rpuz + ['docker', 'download', 'simpledocker', 'arg2:doutput1.txt']) with Path('doutput1.txt').open(encoding='utf-8') as fp: assert fp.read().strip() == '42' # Get random file check_call(rpuz + [ 'docker', 'download', 'simpledocker', '%s:bind.bin' % (Path.cwd() / 'simple') ]) assert same_files('simple.orig', 'bind.bin') # Replace input file check_call(rpuz + [ 'docker', 'upload', 'simpledocker', '%s:arg1' % (tests / 'simple_input2.txt') ]) check_call(rpuz + ['docker', 'upload', 'simpledocker']) check_call(rpuz + ['showfiles', 'simpledocker']) # Run again check_simple(rpuz + ['docker', 'run', 'simpledocker'], 'out', 2) # Get output file check_call( rpuz + ['docker', 'download', 'simpledocker', 'arg2:doutput2.txt']) with Path('doutput2.txt').open(encoding='utf-8') as fp: assert fp.read().strip() == '36' # Reset input file check_call(rpuz + ['docker', 'upload', 'simpledocker', ':arg1']) # Run again check_simple(rpuz + ['docker', 'run', 'simpledocker'], 'out') # Get output file check_call( rpuz + ['docker', 'download', 'simpledocker', 'arg2:doutput1.txt']) with Path('doutput1.txt').open(encoding='utf-8') as fp: assert fp.read().strip() == '42' # Replace input file via path check_call(rpuz + [ 'docker', 'upload', 'simpledocker', '%s:%s' % (tests / 'simple_input2.txt', tests / 'simple_input.txt') ]) # Run again check_simple(rpuz + ['docker', 'run', 'simpledocker'], 'out', 2) # Destroy check_call(rpuz + ['docker', 'destroy', 'simpledocker']) elif interactive: print("Test and press enter") sys.stdin.readline() finally: if Path('simpledocker').exists(): Path('simpledocker').rmtree() # ######################################## # 'threads' program: testrun # # Build build('threads', ['threads.c'], ['-lpthread']) # Trace output = check_output(rpz + ['testrun', './threads'], 'err') assert any(b'successfully exec\'d /bin/./echo' in l for l in output.splitlines()) # ######################################## # 'threads2' program: testrun # # Build build('threads2', ['threads2.c'], ['-lpthread']) # Trace output = check_output(rpz + ['testrun', './threads2'], 'err') assert any(b'successfully exec\'d /bin/echo' in l for l in output.splitlines()) # ######################################## # 'segv' program: testrun # # Build build('segv', ['segv.c']) # Trace check_call(rpz + ['testrun', './segv']) # ######################################## # 'exec_echo' program: trace, pack, run --cmdline # # Build build('exec_echo', ['exec_echo.c']) # Trace check_call( rpz + ['trace', '--overwrite', './exec_echo', 'originalexecechooutput']) # Pack check_call(rpz + ['pack', 'exec_echo.rpz']) # Unpack chroot check_call(sudo + rpuz + ['chroot', 'setup', 'exec_echo.rpz', 'echochroot']) try: # Run original command-line output = check_output(sudo + rpuz + ['chroot', 'run', 'echochroot']) assert output == b'originalexecechooutput\n' # Prints out command-line output = check_output(sudo + rpuz + ['chroot', 'run', 'echochroot', '--cmdline']) assert any(b'./exec_echo originalexecechooutput' == s.strip() for s in output.split(b'\n')) # Run with different command-line output = check_output(sudo + rpuz + [ 'chroot', 'run', 'echochroot', '--cmdline', './exec_echo', 'changedexecechooutput' ]) assert output == b'changedexecechooutput\n' finally: check_call(sudo + rpuz + ['chroot', 'destroy', 'echochroot']) # ######################################## # 'exec_echo' program: testrun # This is built with -m32 so that we transition: # python (x64) -> exec_echo (i386) -> echo (x64) # if sys.maxsize > 2**32: # Build build('exec_echo32', ['exec_echo.c'], ['-m32']) # Trace check_call(rpz + ['testrun', './exec_echo32', '42']) else: print("Can't try exec_echo transitions: not running on 64bits") # ######################################## # Tracing non-existing program # ret = call(rpz + ['testrun', './doesntexist']) assert ret == 127 # ######################################## # 'connect' program: testrun # # Build build('connect', ['connect.c']) # Trace err = check_output(rpz + ['testrun', './connect'], 'err') err = err.split(b'\n') assert not any(b'program exited with non-zero code' in l for l in err) assert any(re.search(br'process connected to [0-9.]+:80', l) for l in err) # ######################################## # 'vfork' program: testrun # # Build build('vfork', ['vfork.c']) # Trace err = check_output(rpz + ['testrun', './vfork'], 'err') err = err.split(b'\n') assert not any(b'program exited with non-zero code' in l for l in err) # ######################################## # 'rename' program: trace # # Build build('rename', ['rename.c']) # Trace check_call(rpz + ['trace', '--overwrite', '-d', 'rename-trace', './rename']) with Path('rename-trace/config.yml').open(encoding='utf-8') as fp: config = yaml.safe_load(fp) # Check that written files were logged database = Path.cwd() / 'rename-trace/trace.sqlite3' if PY3: # On PY3, connect() only accepts unicode conn = sqlite3.connect(str(database)) else: conn = sqlite3.connect(database.path) conn.row_factory = sqlite3.Row rows = conn.execute(''' SELECT name FROM opened_files ''') files = set(Path(r[0]) for r in rows) for n in ('dir1/file', 'dir2/file', 'dir2/brokensymlink', 'dir2/symlink'): if (Path.cwd() / n) not in files: raise AssertionError("Missing file: %s" % (Path.cwd() / n)) conn.close() # Check that created files won't be packed for f in config.get('other_files'): if 'dir2' in Path(f).parent.components: raise AssertionError("Created file shouldn't be packed: %s" % Path(f)) # ######################################## # 'readwrite' program: trace # Build build('readwrite', ['readwrite.c']) # Create test folder Path('readwrite_test').mkdir() with Path('readwrite_test/existing').open('w'): pass # Trace existing one check_call(rpz + [ 'trace', '--overwrite', '-d', 'readwrite-E-trace', './readwrite', 'readwrite_test/existing' ]) # Check that file was logged as read and written database = Path.cwd() / 'readwrite-E-trace/trace.sqlite3' if PY3: # On PY3, connect() only accepts unicode conn = sqlite3.connect(str(database)) else: conn = sqlite3.connect(database.path) conn.row_factory = sqlite3.Row rows = list( conn.execute( ''' SELECT mode FROM opened_files WHERE name = ? ''', (str(Path('readwrite_test/existing').absolute()), ))) conn.close() assert rows assert rows[0][0] == FILE_READ | FILE_WRITE # Trace non-existing one check_call(rpz + [ 'trace', '--overwrite', '-d', 'readwrite-N-trace', './readwrite', 'readwrite_test/nonexisting' ]) # Check that file was logged as written only database = Path.cwd() / 'readwrite-N-trace/trace.sqlite3' if PY3: # On PY3, connect() only accepts unicode conn = sqlite3.connect(str(database)) else: conn = sqlite3.connect(database.path) conn.row_factory = sqlite3.Row rows = list( conn.execute( ''' SELECT mode FROM opened_files WHERE name = ? ''', (str(Path('readwrite_test/nonexisting').absolute()), ))) conn.close() assert rows assert rows[0][0] == FILE_WRITE # Trace a failure: inaccessible file ret = call(rpz + [ 'trace', '--overwrite', '-d', 'readwrite-F-trace', './readwrite', 'readwrite_test/non/existing/file' ]) assert ret == 1 # ######################################## # Test shebang corner-cases # Path('a').symlink('b') with Path('b').open('w') as fp: fp.write('#!%s 0\nsome content\n' % (Path.cwd() / 'c')) Path('b').chmod(0o744) Path('c').symlink('d') with Path('d').open('w') as fp: fp.write('#!e') Path('d').chmod(0o744) with Path('e').open('w') as fp: fp.write('#!/bin/echo') Path('e').chmod(0o744) # Trace out = check_output(rpz + [ 'trace', '--overwrite', '-d', 'shebang-trace', '--dont-identify-packages', './a', '1', '2' ]) out = out.splitlines()[0] assert out == ('e %s 0 ./a 1 2' % (Path.cwd() / 'c')).encode('ascii') # Check config with Path('shebang-trace/config.yml').open(encoding='utf-8') as fp: config = yaml.safe_load(fp) other_files = set( Path(f) for f in config['other_files'] if f.startswith('%s/' % Path.cwd())) # Check database database = Path.cwd() / 'shebang-trace/trace.sqlite3' if PY3: # On PY3, connect() only accepts unicode conn = sqlite3.connect(str(database)) else: conn = sqlite3.connect(database.path) conn.row_factory = sqlite3.Row rows = conn.execute(''' SELECT name FROM opened_files ''') opened = [Path(r[0]) for r in rows if r[0].startswith('%s/' % Path.cwd())] rows = conn.execute(''' SELECT name, argv FROM executed_files ''') executed = [(Path(r[0]), r[1]) for r in rows if Path(r[0]).lies_under(Path.cwd())] print("other_files: %r" % sorted(other_files)) print("opened: %r" % opened) print("executed: %r" % executed) assert other_files == set(Path.cwd() / p for p in ('a', 'b', 'c', 'd', 'e')) assert opened == [Path.cwd() / 'c', Path.cwd() / 'e'] assert executed == [(Path.cwd() / 'a', './a\x001\x002\x00')] # ######################################## # Test old packages # old_packages = [ ('simple-0.4.0.rpz', 'https://drive.google.com/uc?export=download&id=0B3ucPz7GSthBVG4xZW1V' 'eDhXNTQ'), ('simple-0.6.0.rpz', 'https://drive.google.com/uc?export=download&id=0B3ucPz7GSthBbl9SUjhr' 'cUdtbGs'), ('simple-0.7.1.rpz', 'https://drive.google.com/uc?export=download&id=0B3ucPz7GSthBRGp2Vm5V' 'QVpWOGs'), ] for name, url in old_packages: print("Testing old package %s" % name) f = Path(name) if not f.exists(): download_file_retry(url, f) # Info check_call(rpuz + ['info', name]) # Show files check_call(rpuz + ['showfiles', name]) # Lists packages check_call(rpuz + ['installpkgs', '--summary', name]) # Unpack directory check_call(rpuz + ['directory', 'setup', name, 'simpledir']) # Run directory check_simple(rpuz + ['directory', 'run', 'simpledir'], 'err') output_in_dir = Path('simpledir/root/tmp') output_in_dir = output_in_dir.listdir('reprozip_*')[0] output_in_dir = output_in_dir / 'simple_output.txt' with output_in_dir.open(encoding='utf-8') as fp: assert fp.read().strip() == '42' # Delete with wrong command (should fail) p = subprocess.Popen(rpuz + ['chroot', 'destroy', 'simpledir'], stderr=subprocess.PIPE) out, err = p.communicate() assert p.poll() != 0 err = err.splitlines() assert b"Wrong unpacker used" in err[0] assert err[1].startswith(b"usage: ") # Delete directory check_call(rpuz + ['directory', 'destroy', 'simpledir']) # ######################################## # Copies back coverage report # coverage = Path('.coverage') if coverage.exists(): coverage.copyfile(tests.parent / '.coverage.runpy')
def functional_tests(raise_warnings, interactive, run_vagrant, run_docker): python = [sys.executable] # Can't match on the SignalWarning category here because of a Python bug # http://bugs.python.org/issue22543 python.extend(['-W', 'error:signal']) if 'COVER' in os.environ: python.extend(['-m'] + os.environ['COVER'].split(' ')) reprozip_main = tests.parent / 'reprozip/reprozip/main.py' reprounzip_main = tests.parent / 'reprounzip/reprounzip/main.py' verbose = ['-v'] * 3 rpz = python + [reprozip_main.absolute().path] + verbose rpuz = python + [reprounzip_main.absolute().path] + verbose # ######################################## # 'simple' program: trace, pack, info, unpack # # Build build('simple', ['simple.c']) # Trace check_call(rpz + ['trace', '-d', 'rpz-simple', './simple', (tests / 'simple_input.txt').path, 'simple_output.txt']) orig_output_location = Path('simple_output.txt').absolute() assert orig_output_location.is_file() with orig_output_location.open(encoding='utf-8') as fp: assert fp.read().strip() == '42' orig_output_location.remove() # Read config with Path('rpz-simple/config.yml').open(encoding='utf-8') as fp: conf = yaml.safe_load(fp) other_files = set(Path(f).absolute() for f in conf['other_files']) expected = [Path('simple'), (tests / 'simple_input.txt')] assert other_files.issuperset([f.resolve() for f in expected]) # Check input and output files input_files = conf['runs'][0]['input_files'] assert (dict((k, Path(f).name) for k, f in iteritems(input_files)) == {'arg': b'simple_input.txt'}) output_files = conf['runs'][0]['output_files'] print(dict((k, Path(f).name) for k, f in iteritems(output_files))) # Here we don't test for dict equality, since we might have C coverage # files in the mix assert Path(output_files['arg']).name == b'simple_output.txt' # Pack check_call(rpz + ['pack', '-d', 'rpz-simple', 'simple.rpz']) Path('simple').remove() # Info check_call(rpuz + ['info', 'simple.rpz']) # Show files check_call(rpuz + ['showfiles', 'simple.rpz']) # Lists packages check_call(rpuz + ['installpkgs', '--summary', 'simple.rpz']) # Unpack directory check_call(rpuz + ['directory', 'setup', 'simple.rpz', 'simpledir']) # Run directory check_call(rpuz + ['directory', 'run', 'simpledir']) output_in_dir = join_root(Path('simpledir/root'), orig_output_location) with output_in_dir.open(encoding='utf-8') as fp: assert fp.read().strip() == '42' # Delete with wrong command (should fail) assert call(rpuz + ['chroot', 'destroy', 'simpledir']) != 0 # Delete directory check_call(rpuz + ['directory', 'destroy', 'simpledir']) # Unpack chroot check_call(['sudo'] + rpuz + ['chroot', 'setup', '--bind-magic-dirs', 'simple.rpz', 'simplechroot']) # Run chroot check_call(['sudo'] + rpuz + ['chroot', 'run', 'simplechroot']) output_in_chroot = join_root(Path('simplechroot/root'), orig_output_location) with output_in_chroot.open(encoding='utf-8') as fp: assert fp.read().strip() == '42' # Get output file check_call(['sudo'] + rpuz + ['chroot', 'download', 'simplechroot', 'arg:output1.txt']) with Path('output1.txt').open(encoding='utf-8') as fp: assert fp.read().strip() == '42' # Replace input file check_call(['sudo'] + rpuz + ['chroot', 'upload', 'simplechroot', '%s:arg' % (tests / 'simple_input2.txt')]) check_call(['sudo'] + rpuz + ['chroot', 'upload', 'simplechroot']) # Run again check_call(['sudo'] + rpuz + ['chroot', 'run', 'simplechroot']) output_in_chroot = join_root(Path('simplechroot/root'), orig_output_location) with output_in_chroot.open(encoding='utf-8') as fp: assert fp.read().strip() == '36' # Delete with wrong command (should fail) assert call(rpuz + ['directory', 'destroy', 'simplechroot']) != 0 # Delete chroot check_call(['sudo'] + rpuz + ['chroot', 'destroy', 'simplechroot']) if not Path('/vagrant').exists(): check_call(['sudo', 'sh', '-c', 'mkdir /vagrant; chmod 777 /vagrant']) # Unpack Vagrant-chroot check_call(rpuz + ['vagrant', 'setup/create', '--use-chroot', 'simple.rpz', '/vagrant/simplevagrantchroot']) print("\nVagrant project set up in simplevagrantchroot") try: if run_vagrant: check_call(rpuz + ['vagrant', 'run', '--no-stdin', '/vagrant/simplevagrantchroot']) # Destroy check_call(rpuz + ['vagrant', 'destroy', '/vagrant/simplevagrantchroot']) elif interactive: print("Test and press enter") sys.stdin.readline() finally: if Path('/vagrant/simplevagrantchroot').exists(): Path('/vagrant/simplevagrantchroot').rmtree() # Unpack Vagrant without chroot check_call(rpuz + ['vagrant', 'setup/create', '--dont-use-chroot', 'simple.rpz', '/vagrant/simplevagrant']) print("\nVagrant project set up in simplevagrant") try: if run_vagrant: check_call(rpuz + ['vagrant', 'run', '--no-stdin', '/vagrant/simplevagrant']) # Destroy check_call(rpuz + ['vagrant', 'destroy', '/vagrant/simplevagrant']) elif interactive: print("Test and press enter") sys.stdin.readline() finally: if Path('/vagrant/simplevagrant').exists(): Path('/vagrant/simplevagrant').rmtree() # Unpack Docker check_call(rpuz + ['docker', 'setup/create', 'simple.rpz', 'simpledocker']) print("\nDocker project set up in simpledocker") try: if run_docker: check_call(rpuz + ['docker', 'setup/build', 'simpledocker']) check_call(rpuz + ['docker', 'run', 'simpledocker']) # Get output file check_call(rpuz + ['docker', 'download', 'simpledocker', 'arg:doutput1.txt']) with Path('doutput1.txt').open(encoding='utf-8') as fp: assert fp.read().strip() == '42' # Replace input file check_call(rpuz + ['docker', 'upload', 'simpledocker', '%s:arg' % (tests / 'simple_input2.txt')]) check_call(rpuz + ['docker', 'upload', 'simpledocker']) check_call(rpuz + ['showfiles', 'simpledocker']) # Run again check_call(rpuz + ['docker', 'run', 'simpledocker']) # Get output file check_call(rpuz + ['docker', 'download', 'simpledocker', 'arg:doutput2.txt']) with Path('doutput2.txt').open(encoding='utf-8') as fp: assert fp.read().strip() == '36' # Destroy check_call(rpuz + ['docker', 'destroy', 'simpledocker']) elif interactive: print("Test and press enter") sys.stdin.readline() finally: if Path('simpledocker').exists(): Path('simpledocker').rmtree() # ######################################## # 'threads' program: testrun # # Build build('threads', ['threads.c'], ['-lpthread']) # Trace check_call(rpz + ['testrun', './threads']) # ######################################## # 'segv' program: testrun # # Build build('segv', ['segv.c']) # Trace check_call(rpz + ['testrun', './segv']) # ######################################## # 'exec_echo' program: trace, pack, run --cmdline # # Build build('exec_echo', ['exec_echo.c']) # Trace check_call(rpz + ['trace', './exec_echo', 'originalexecechooutput']) # Pack check_call(rpz + ['pack', 'exec_echo.rpz']) # Unpack chroot check_call(['sudo'] + rpuz + ['chroot', 'setup', 'exec_echo.rpz', 'echochroot']) try: # Run original command-line output = check_output(['sudo'] + rpuz + ['chroot', 'run', 'echochroot']) assert output == b'originalexecechooutput\n' # Prints out command-line output = check_output(['sudo'] + rpuz + ['chroot', 'run', 'echochroot', '--cmdline']) assert any(b'./exec_echo originalexecechooutput' == s.strip() for s in output.split(b'\n')) # Run with different command-line output = check_output(['sudo'] + rpuz + [ 'chroot', 'run', 'echochroot', '--cmdline', './exec_echo', 'changedexecechooutput']) assert output == b'changedexecechooutput\n' finally: check_call(['sudo'] + rpuz + ['chroot', 'destroy', 'echochroot']) # ######################################## # 'exec_echo' program: testrun # This is built with -m32 so that we transition: # python (x64) -> exec_echo (i386) -> echo (x64) # if sys.maxsize > 2 ** 32: # Build build('exec_echo32', ['exec_echo.c'], ['-m32']) # Trace check_call(rpz + ['testrun', './exec_echo32 42']) else: print("Can't try exec_echo transitions: not running on 64bits") # ######################################## # Tracing non-existing program # check_call(rpz + ['testrun', './doesntexist']) # ######################################## # 'connect' program: testrun # # Build build('connect', ['connect.c']) # Trace stderr = check_errout(rpz + ['testrun', './connect']) stderr = stderr.split(b'\n') assert not any(b'program exited with non-zero code' in l for l in stderr) assert any(re.search(br'process connected to [0-9.]+:80', l) for l in stderr) # ######################################## # Copies back coverage report # coverage = Path('.coverage') if coverage.exists(): coverage.copyfile(tests.parent / '.coverage.runpy')