def download_file(url, dest, cachename=None):
"""Downloads a file using a local cache.
If the file cannot be downloaded or if it wasn't modified, the cached
version will be used instead.
The cache lives in ``~/.cache/reprozip/``.
"""
if cachename is None:
cachename = dest.name
request = Request(url)
if 'XDG_CACHE_HOME' in os.environ:
cache = Path(os.environ['XDG_CACHE_HOME'])
else:
cache = Path('~/.cache').expand_user()
cache = cache / 'reprozip' / cachename
if cache.exists():
mtime = email.utils.formatdate(cache.mtime(), usegmt=True)
request.add_header('If-Modified-Since', mtime)
cache.parent.mkdir(parents=True)
try:
response = urlopen(request)
except URLError as e:
if cache.exists():
if isinstance(e, HTTPError) and e.code == 304:
logging.info("Cached file %s is up to date", cachename)
else:
logging.warning("Couldn't download %s: %s", url, e)
cache.copy(dest)
return
else:
raise
if response is None:
logging.info("Cached file %s is up to date", cachename)
cache.copy(dest)
return
logging.info("Downloading %s", url)
try:
CHUNK_SIZE = 4096
with cache.open('wb') as f:
while True:
chunk = response.read(CHUNK_SIZE)
if not chunk:
break
f.write(chunk)
response.close()
except Exception as e: # pragma: no cover
try:
cache.remove()
except OSError:
pass
raise e
cache.copy(dest)
def download_file(url, dest, cachename=None):
"""Downloads a file using a local cache.
If the file cannot be downloaded or if it wasn't modified, the cached
version will be used instead.
The cache lives in ``~/.cache/reprozip/``.
"""
if cachename is None:
cachename = dest.name
request = Request(url)
if 'XDG_CACHE_HOME' in os.environ:
cache = Path(os.environ['XDG_CACHE_HOME'])
else:
cache = Path('~/.cache').expand_user()
cache = cache / 'reprozip' / cachename
if cache.exists():
mtime = email.utils.formatdate(cache.mtime(), usegmt=True)
request.add_header('If-Modified-Since', mtime)
cache.parent.mkdir(parents=True)
try:
response = urlopen(request)
except URLError as e:
if cache.exists():
if isinstance(e, HTTPError) and e.code == 304:
logging.info("Cached file %s is up to date", cachename)
else:
logging.warning("Couldn't download %s: %s", url, e)
cache.copy(dest)
return
else:
raise
if response is None:
logging.info("Cached file %s is up to date", cachename)
cache.copy(dest)
return
logging.info("Downloading %s", url)
try:
CHUNK_SIZE = 4096
with cache.open('wb') as f:
while True:
chunk = response.read(CHUNK_SIZE)
if not chunk:
break
f.write(chunk)
response.close()
except Exception as e: # pragma: no cover
try:
cache.remove()
except OSError:
pass
raise e
cache.copy(dest)
def download_file(url, dest, cachename=None):
"""Downloads a file using a local cache.
If the file cannot be downloaded or if it wasn't modified, the cached
version will be used instead.
The cache lives in ``~/.cache/reprozip/``.
"""
if cachename is None:
cachename = dest.components[-1]
request = Request(url)
if 'XDG_CACHE_HOME' in os.environ:
cache = Path(os.environ['XDG_CACHE_HOME'])
else:
cache = Path('~/.cache').expand_user()
cache = cache / 'reprozip' / cachename
if cache.exists():
mtime = email.utils.formatdate(cache.mtime(), usegmt=True)
request.add_header('If-Modified-Since', mtime)
cache.parent.mkdir(parents=True)
try:
response = urlopen(request, timeout=2 if cache.exists() else 10)
except URLError as e:
if cache.exists():
if isinstance(e, HTTPError) and e.code == 304:
logging.info("Download %s: cache is up to date", cachename)
else:
logging.warning("Download %s: error downloading %s: %s",
cachename, url, e)
cache.copy(dest)
return
else:
raise
if response is None:
logging.info("Download %s: cache is up to date", cachename)
cache.copy(dest)
return
logging.info("Download %s: downloading %s", cachename, url)
try:
with cache.open('wb') as f:
copyfile(response, f)
response.close()
except Exception as e: # pragma: no cover
try:
cache.remove()
except OSError:
pass
raise e
logging.info("Downloaded %s successfully", cachename)
cache.copy(dest)
def showfiles(args):
"""Writes out the input and output files.
Works both for a pack file and for an extracted directory.
"""
pack = Path(args.pack[0])
if not pack.exists():
logging.critical("Pack or directory %s does not exist", pack)
sys.exit(1)
if pack.is_dir():
# Reads info from an unpacked directory
runs, packages, other_files = load_config_file(pack / 'config.yml',
canonical=True)
# The '.reprounzip' file is a pickled dictionary, it contains the name
# of the files that replaced each input file (if upload was used)
with pack.open('rb', '.reprounzip') as fp:
unpacked_info = pickle.load(fp)
input_files = unpacked_info.get('input_files', {})
print("Input files:")
for i, run in enumerate(runs):
if len(runs) > 1:
print(" Run %d:" % i)
for input_name, path in iteritems(run['input_files']):
print(" %s (%s)" % (input_name, path))
if input_files.get(input_name) is not None:
assigned = PosixPath(input_files[input_name])
else:
assigned = "(original)"
print(" %s" % assigned)
print("Output files:")
for i, run in enumerate(runs):
if len(runs) > 1:
print(" Run %d:" % i)
for output_name, path in iteritems(run['output_files']):
print(" %s (%s)" % (output_name, path))
else: # pack.is_file()
# Reads info from a pack file
runs, packages, other_files = load_config(pack)
print("Input files:")
for i, run in enumerate(runs):
if len(runs) > 1:
print(" Run %d:" % i)
for input_name, path in iteritems(run['input_files']):
print(" %s (%s)" % (input_name, path))
print("Output files:")
for i, run in enumerate(runs):
if len(runs) > 1:
print(" Run %d:" % i)
for output_name, path in iteritems(run['output_files']):
print(" %s (%s)" % (output_name, path))
def __init__(self, path):
path = Path(path)
size = None
if path.exists():
if path.is_link():
self.comment = "Link to %s" % path.read_link(absolute=True)
elif path.is_dir():
self.comment = "Directory"
else:
size = path.size()
self.comment = hsize(size)
File.__init__(self, path, size)
def __init__(self, path):
path = Path(path)
size = None
if path.exists():
if path.is_link():
self.comment = "Link to %s" % path.read_link(absolute=True)
elif path.is_dir():
self.comment = "Directory"
else:
size = path.size()
self.comment = hsize(size)
File.__init__(self, path, size)
def showfiles(args):
"""Writes out the input and output files.
Works both for a pack file and for an extracted directory.
"""
pack = Path(args.pack[0])
if not pack.exists():
logging.critical("Pack or directory %s does not exist", pack)
sys.exit(1)
if pack.is_dir():
# Reads info from an unpacked directory
config = load_config_file(pack / 'config.yml',
canonical=True)
# The '.reprounzip' file is a pickled dictionary, it contains the name
# of the files that replaced each input file (if upload was used)
with pack.open('rb', '.reprounzip') as fp:
unpacked_info = pickle.load(fp)
assigned_input_files = unpacked_info.get('input_files', {})
print("Input files:")
for input_name, f in iteritems(config.inputs_outputs):
if not f.read_runs:
continue
print(" %s (%s)" % (input_name, f.path))
if assigned_input_files.get(input_name) is not None:
assigned = assigned_input_files[input_name]
else:
assigned = "(original)"
print(" %s" % assigned)
print("Output files:")
for output_name, f in iteritems(config.inputs_outputs):
if f.write_runs:
print(" %s (%s)" % (output_name, f.path))
else: # pack.is_file()
# Reads info from a pack file
config = load_config(pack)
print("Input files:")
for input_name, f in iteritems(config.inputs_outputs):
if f.read_runs:
print(" %s (%s)" % (input_name, f.path))
print("Output files:")
for output_name, f in iteritems(config.inputs_outputs):
if f.write_runs:
print(" %s (%s)" % (output_name, f.path))
def vagrant_setup_create(args):
"""Sets up the experiment to be run in a Vagrant-built virtual machine.
This can either build a chroot or not.
If building a chroot, we do just like without Vagrant: we copy all the
files and only get what's missing from the host. But we do install
automatically the packages whose files are required.
If not building a chroot, we install all the packages, and only unpack
files that don't come from packages.
In short: files from packages with packfiles=True will only be used if
building a chroot.
"""
if not args.pack:
logging.critical("setup/create needs the pack filename")
sys.exit(1)
pack = Path(args.pack[0])
target = Path(args.target[0])
if target.exists():
logging.critical("Target directory exists")
sys.exit(1)
use_chroot = args.use_chroot
mount_bind = args.bind_magic_dirs
record_usage(use_chroot=use_chroot,
mount_bind=mount_bind)
signals.pre_setup(target=target, pack=pack)
# Unpacks configuration file
rpz_pack = RPZPack(pack)
rpz_pack.extract_config(target / 'config.yml')
# Loads config
runs, packages, other_files = config = load_config(target / 'config.yml',
True)
if not args.memory:
memory = None
else:
try:
memory = int(args.memory[-1])
except ValueError:
logging.critical("Invalid value for memory size: %r", args.memory)
sys.exit(1)
if args.base_image and args.base_image[0]:
record_usage(vagrant_explicit_image=True)
box = args.base_image[0]
if args.distribution:
target_distribution = args.distribution[0]
else:
target_distribution = None
else:
target_distribution, box = select_box(runs)
logging.info("Using box %s", box)
logging.debug("Distribution: %s", target_distribution or "unknown")
# If using chroot, we might still need to install packages to get missing
# (not packed) files
if use_chroot:
packages = [pkg for pkg in packages if not pkg.packfiles]
if packages:
record_usage(vagrant_install_pkgs=True)
logging.info("Some packages were not packed, so we'll install and "
"copy their files\n"
"Packages that are missing:\n%s",
' '.join(pkg.name for pkg in packages))
if packages:
try:
installer = select_installer(pack, runs, target_distribution)
except CantFindInstaller as e:
logging.error("Need to install %d packages but couldn't select a "
"package installer: %s",
len(packages), e)
target.mkdir(parents=True)
try:
# Writes setup script
logging.info("Writing setup script %s...", target / 'setup.sh')
with (target / 'setup.sh').open('w', encoding='utf-8',
newline='\n') as fp:
fp.write('#!/bin/sh\n\nset -e\n\n')
if packages:
# Updates package sources
fp.write(installer.update_script())
fp.write('\n')
# Installs necessary packages
fp.write(installer.install_script(packages))
fp.write('\n')
# TODO : Compare package versions (painful because of sh)
# Untar
if use_chroot:
fp.write('\n'
'mkdir /experimentroot; cd /experimentroot\n')
fp.write('tar zpxf /vagrant/data.tgz --numeric-owner '
'--strip=1 %s\n' % rpz_pack.data_prefix)
if mount_bind:
fp.write('\n'
'mkdir -p /experimentroot/dev\n'
'mkdir -p /experimentroot/proc\n')
for pkg in packages:
fp.write('\n# Copies files from package %s\n' % pkg.name)
for f in pkg.files:
f = f.path
dest = join_root(PosixPath('/experimentroot'), f)
fp.write('mkdir -p %s\n' %
shell_escape(unicode_(f.parent)))
fp.write('cp -L %s %s\n' % (
shell_escape(unicode_(f)),
shell_escape(unicode_(dest))))
else:
fp.write('\ncd /\n')
paths = set()
pathlist = []
# Adds intermediate directories, and checks for existence in
# the tar
for f in other_files:
path = PosixPath('/')
for c in rpz_pack.remove_data_prefix(f.path).components:
path = path / c
if path in paths:
continue
paths.add(path)
try:
rpz_pack.get_data(path)
except KeyError:
logging.info("Missing file %s", path)
else:
pathlist.append(path)
# FIXME : for some reason we need reversed() here, I'm not sure
# why. Need to read more of tar's docs.
# TAR bug: --no-overwrite-dir removes --keep-old-files
# TAR bug: there is no way to make --keep-old-files not report
# an error if an existing file is encountered. --skip-old-files
# was introduced too recently. Instead, we just ignore the exit
# status
with (target / 'rpz-files.list').open('wb') as lfp:
for p in reversed(pathlist):
lfp.write(join_root(rpz_pack.data_prefix, p).path)
lfp.write(b'\0')
fp.write('tar zpxf /vagrant/data.tgz --keep-old-files '
'--numeric-owner --strip=1 '
'--null -T /vagrant/rpz-files.list || /bin/true\n')
# Copies busybox
if use_chroot:
arch = runs[0]['architecture']
download_file(busybox_url(arch),
target / 'busybox',
'busybox-%s' % arch)
fp.write(r'''
cp /vagrant/busybox /experimentroot/busybox
chmod +x /experimentroot/busybox
mkdir -p /experimentroot/bin
[ -e /experimentroot/bin/sh ] || \
ln -s /busybox /experimentroot/bin/sh
''')
# Copies pack
logging.info("Copying pack file...")
rpz_pack.copy_data_tar(target / 'data.tgz')
rpz_pack.close()
# Writes Vagrant file
logging.info("Writing %s...", target / 'Vagrantfile')
with (target / 'Vagrantfile').open('w', encoding='utf-8',
newline='\n') as fp:
# Vagrant header and version
fp.write(
'# -*- mode: ruby -*-\n'
'# vi: set ft=ruby\n\n'
'VAGRANTFILE_API_VERSION = "2"\n\n'
'Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|\n')
# Selects which box to install
fp.write(' config.vm.box = "%s"\n' % box)
# Run the setup script on the virtual machine
fp.write(' config.vm.provision "shell", path: "setup.sh"\n')
# Memory size
if memory is not None:
fp.write(' config.vm.provider "virtualbox" do |v|\n'
' v.memory = %d\n'
' end\n' % memory)
fp.write('end\n')
# Meta-data for reprounzip
write_dict(target,
metadata_initial_iofiles(config,
{'use_chroot': use_chroot}))
signals.post_setup(target=target, pack=pack)
except Exception:
target.rmtree(ignore_errors=True)
raise
def download_file(url, dest, cachename=None, ssl_verify=None):
"""Downloads a file using a local cache.
If the file cannot be downloaded or if it wasn't modified, the cached
version will be used instead.
The cache lives in ``~/.cache/reprozip/``.
"""
if cachename is None:
if dest is None:
raise ValueError("One of 'dest' or 'cachename' must be specified")
cachename = dest.components[-1]
headers = {}
if 'XDG_CACHE_HOME' in os.environ:
cache = Path(os.environ['XDG_CACHE_HOME'])
else:
cache = Path('~/.cache').expand_user()
cache = cache / 'reprozip' / cachename
if cache.exists():
mtime = email.utils.formatdate(cache.mtime(), usegmt=True)
headers['If-Modified-Since'] = mtime
cache.parent.mkdir(parents=True)
try:
response = requests.get(url,
headers=headers,
timeout=2 if cache.exists() else 10,
stream=True,
verify=ssl_verify)
response.raise_for_status()
if response.status_code == 304:
raise requests.HTTPError(
'304 File is up to date, no data returned', response=response)
except requests.RequestException as e:
if cache.exists():
if e.response and e.response.status_code == 304:
logging.info("Download %s: cache is up to date", cachename)
else:
logging.warning("Download %s: error downloading %s: %s",
cachename, url, e)
if dest is not None:
cache.copy(dest)
return dest
else:
return cache
else:
raise
logging.info("Download %s: downloading %s", cachename, url)
try:
with cache.open('wb') as f:
for chunk in response.iter_content(4096):
f.write(chunk)
response.close()
except Exception as e: # pragma: no cover
try:
cache.remove()
except OSError:
pass
raise e
logging.info("Downloaded %s successfully", cachename)
if dest is not None:
cache.copy(dest)
return dest
else:
return cache
def docker_setup_create(args):
"""Sets up the experiment to be run in a Docker-built container.
"""
pack = Path(args.pack[0])
target = Path(args.target[0])
if target.exists():
logging.critical("Target directory exists")
sys.exit(1)
signals.pre_setup(target=target, pack=pack)
# Unpacks configuration file
tar = tarfile.open(str(pack), 'r:*')
member = tar.getmember('METADATA/config.yml')
member.name = 'config.yml'
tar.extract(member, str(target))
tar.close()
# Loads config
runs, packages, other_files = load_config(target / 'config.yml', True)
if args.base_image:
record_usage(docker_explicit_base=True)
base_image = args.base_image[0]
if args.distribution:
target_distribution = args.distribution[0]
else:
target_distribution = None
else:
target_distribution, base_image = select_image(runs)
logging.info("Using base image %s", base_image)
logging.debug("Distribution: %s", target_distribution or "unknown")
target.mkdir(parents=True)
pack.copyfile(target / 'experiment.rpz')
# Writes Dockerfile
logging.info("Writing %s...", target / 'Dockerfile')
with (target / 'Dockerfile').open('w',
encoding='utf-8', newline='\n') as fp:
fp.write('FROM %s\n\n' % base_image)
# Installs busybox
download_file(busybox_url(runs[0]['architecture']),
target / 'busybox')
fp.write('COPY busybox /bin/busybox\n')
fp.write('COPY experiment.rpz /reprozip_experiment.rpz\n\n')
fp.write('RUN \\\n'
' chmod +x /bin/busybox && \\\n')
if args.install_pkgs:
# Install every package through package manager
missing_packages = []
else:
# Only install packages that were not packed
missing_packages = [pkg for pkg in packages if pkg.packfiles]
packages = [pkg for pkg in packages if not pkg.packfiles]
# FIXME : Right now, we need 'sudo' to be available (and it's not
# necessarily in the base image)
if packages:
record_usage(docker_install_pkgs=True)
else:
record_usage(docker_install_pkgs="sudo")
packages += [Package('sudo', None, packfiles=False)]
if packages:
try:
installer = select_installer(pack, runs, target_distribution)
except CantFindInstaller as e:
logging.error("Need to install %d packages but couldn't "
"select a package installer: %s",
len(packages), e)
sys.exit(1)
# Updates package sources
fp.write(' %s && \\\n' % installer.update_script())
# Installs necessary packages
fp.write(' %s && \\\n' % installer.install_script(packages))
logging.info("Dockerfile will install the %d software packages that "
"were not packed", len(packages))
# Untar
paths = set()
pathlist = []
dataroot = PosixPath('DATA')
# Adds intermediate directories, and checks for existence in the tar
tar = tarfile.open(str(pack), 'r:*')
missing_files = chain.from_iterable(pkg.files
for pkg in missing_packages)
for f in chain(other_files, missing_files):
path = PosixPath('/')
for c in f.path.components[1:]:
path = path / c
if path in paths:
continue
paths.add(path)
datapath = join_root(dataroot, path)
try:
tar.getmember(str(datapath))
except KeyError:
logging.info("Missing file %s", datapath)
else:
pathlist.append(unicode_(datapath))
tar.close()
# FIXME : for some reason we need reversed() here, I'm not sure why.
# Need to read more of tar's docs.
# TAR bug: --no-overwrite-dir removes --keep-old-files
fp.write(' cd / && tar zpxf /reprozip_experiment.rpz '
'--numeric-owner --strip=1 %s\n' %
' '.join(shell_escape(p) for p in reversed(pathlist)))
# Meta-data for reprounzip
write_dict(target / '.reprounzip', {})
signals.post_setup(target=target)
def docker_setup_create(args):
"""Sets up the experiment to be run in a Docker-built container.
"""
pack = Path(args.pack[0])
target = Path(args.target[0])
if target.exists():
logging.critical("Target directory exists")
sys.exit(1)
signals.pre_setup(target=target, pack=pack)
target.mkdir()
try:
# Unpacks configuration file
rpz_pack = RPZPack(pack)
rpz_pack.extract_config(target / 'config.yml')
# Loads config
runs, packages, other_files = config = load_config(
target / 'config.yml', True)
if args.base_image:
record_usage(docker_explicit_base=True)
base_image = args.base_image[0]
if args.distribution:
target_distribution = args.distribution[0]
else:
target_distribution = None
else:
target_distribution, base_image = select_image(runs)
logging.info("Using base image %s", base_image)
logging.debug("Distribution: %s", target_distribution or "unknown")
rpz_pack.copy_data_tar(target / 'data.tgz')
arch = runs[0]['architecture']
# Writes Dockerfile
logging.info("Writing %s...", target / 'Dockerfile')
with (target / 'Dockerfile').open('w', encoding='utf-8',
newline='\n') as fp:
fp.write('FROM %s\n\n' % base_image)
# Installs busybox
download_file(busybox_url(arch),
target / 'busybox',
'busybox-%s' % arch)
fp.write('COPY busybox /busybox\n')
# Installs rpzsudo
download_file(sudo_url(arch),
target / 'rpzsudo',
'rpzsudo-%s' % arch)
fp.write('COPY rpzsudo /rpzsudo\n\n')
fp.write('COPY data.tgz /reprozip_data.tgz\n\n')
fp.write('COPY rpz-files.list /rpz-files.list\n')
fp.write('RUN \\\n'
' chmod +x /busybox /rpzsudo && \\\n')
if args.install_pkgs:
# Install every package through package manager
missing_packages = []
else:
# Only install packages that were not packed
missing_packages = [pkg for pkg in packages if pkg.packfiles]
packages = [pkg for pkg in packages if not pkg.packfiles]
if packages:
record_usage(docker_install_pkgs=True)
try:
installer = select_installer(pack, runs,
target_distribution)
except CantFindInstaller as e:
logging.error("Need to install %d packages but couldn't "
"select a package installer: %s",
len(packages), e)
sys.exit(1)
# Updates package sources
update_script = installer.update_script()
if update_script:
fp.write(' %s && \\\n' % update_script)
# Installs necessary packages
fp.write(' %s && \\\n' % installer.install_script(packages))
logging.info("Dockerfile will install the %d software "
"packages that were not packed", len(packages))
else:
record_usage(docker_install_pkgs=False)
# Untar
paths = set()
pathlist = []
# Add intermediate directories, and check for existence in the tar
logging.info("Generating file list...")
missing_files = chain.from_iterable(pkg.files
for pkg in missing_packages)
data_files = rpz_pack.data_filenames()
listoffiles = list(chain(other_files, missing_files))
for f in listoffiles:
if f.path.name == 'resolv.conf' and (
f.path.lies_under('/etc') or
f.path.lies_under('/run') or
f.path.lies_under('/var')):
continue
path = PosixPath('/')
for c in rpz_pack.remove_data_prefix(f.path).components:
path = path / c
if path in paths:
continue
paths.add(path)
if path in data_files:
pathlist.append(path)
else:
logging.info("Missing file %s", path)
rpz_pack.close()
# FIXME : for some reason we need reversed() here, I'm not sure why
# Need to read more of tar's docs.
# TAR bug: --no-overwrite-dir removes --keep-old-files
with (target / 'rpz-files.list').open('wb') as lfp:
for p in reversed(pathlist):
lfp.write(join_root(rpz_pack.data_prefix, p).path)
lfp.write(b'\0')
fp.write(' cd / && '
'(tar zpxf /reprozip_data.tgz -U --recursive-unlink '
'--numeric-owner --strip=1 --null -T /rpz-files.list || '
'/busybox echo "TAR reports errors, this might or might '
'not prevent the execution to run")\n')
# Setup entry point
fp.write('COPY rpz_entrypoint.sh /rpz_entrypoint.sh\n'
'ENTRYPOINT ["/busybox", "sh", "/rpz_entrypoint.sh"]\n')
# Write entry point script
logging.info("Writing %s...", target / 'rpz_entrypoint.sh')
with (target / 'rpz_entrypoint.sh').open('w', encoding='utf-8',
newline='\n') as fp:
# The entrypoint gets some arguments from the run command
# By default, it just does all the runs
# "run N" executes the run with that number
# "cmd STR" sets a replacement command-line for the next run
# "do STR" executes a command as-is
fp.write(
'#!/bin/sh\n'
'\n'
'COMMAND=\n'
'ENVVARS=\n'
'\n'
'if [ $# = 0 ]; then\n'
' exec /busybox sh /rpz_entrypoint.sh')
for nb in irange(len(runs)):
fp.write(' run %d' % nb)
fp.write(
'\n'
'fi\n'
'\n'
'while [ $# != 0 ]; do\n'
' case "$1" in\n'
' help)\n'
' echo "Image built from reprounzip-docker" >&2\n'
' echo "Usage: docker run <image> [cmd word [word '
'...]] [run <R>]" >&2\n'
' echo " \\`cmd ...\\` changes the command for '
'the next \\`run\\` option" >&2\n'
' echo " \\`run <name|number>\\` runs the '
'specified run" >&2\n'
' echo "By default, all the runs are executed." '
'>&2\n'
' echo "The runs in this image are:" >&2\n')
for run in runs:
fp.write(
' echo " {name}: {cmdline}" >&2\n'.format(
name=run['id'],
cmdline=' '.join(shell_escape(a)
for a in run['argv'])))
fp.write(
' exit 0\n'
' ;;\n'
' do)\n'
' shift\n'
' $1\n'
' ;;\n'
' env)\n'
' shift\n'
' ENVVARS="$1"\n'
' ;;\n'
' cmd)\n'
' shift\n'
' COMMAND="$1"\n'
' ;;\n'
' run)\n'
' shift\n'
' case "$1" in\n')
for i, run in enumerate(runs):
cmdline = ' '.join([run['binary']] + run['argv'][1:])
fp.write(
' {name})\n'
' RUNCOMMAND={cmd}\n'
' RUNWD={wd}\n'
' RUNENV={env}\n'
' RUNUID={uid}\n'
' RUNGID={gid}\n'
' ;;\n'.format(
name='%s|%d' % (run['id'], i),
cmd=shell_escape(cmdline),
wd=shell_escape(run['workingdir']),
env=shell_escape(' '.join(
'%s=%s' % (shell_escape(k), shell_escape(v))
for k, v in iteritems(run['environ']))),
uid=run.get('uid', 1000),
gid=run.get('gid', 1000)))
fp.write(
' *)\n'
' echo "RPZ: Unknown run $1" >&2\n'
' exit 1\n'
' ;;\n'
' esac\n'
' if [ -n "$COMMAND" ]; then\n'
' RUNCOMMAND="$COMMAND"\n'
' COMMAND=\n'
' fi\n'
' export RUNWD; export RUNENV; export ENVVARS; '
'export RUNCOMMAND\n'
' /rpzsudo "#$RUNUID" "#$RUNGID" /busybox sh -c '
'"cd \\"\\$RUNWD\\" && /busybox env -i $RUNENV $ENVVARS '
'$RUNCOMMAND"\n'
' ENVVARS=\n'
' ;;\n'
' *)\n'
' echo "RPZ: Unknown option $1" >&2\n'
' exit 1\n'
' ;;\n'
' esac\n'
' shift\n'
'done\n')
# Meta-data for reprounzip
write_dict(target, metadata_initial_iofiles(config))
signals.post_setup(target=target, pack=pack)
except Exception:
target.rmtree(ignore_errors=True)
raise
def docker_setup_create(args):
"""Sets up the experiment to be run in a Docker-built container.
"""
pack = Path(args.pack[0])
target = Path(args.target[0])
if target.exists():
logging.critical("Target directory exists")
sys.exit(1)
signals.pre_setup(target=target, pack=pack)
# Unpacks configuration file
rpz_pack = RPZPack(pack)
rpz_pack.extract_config(target / 'config.yml')
# Loads config
runs, packages, other_files = load_config(target / 'config.yml', True)
if args.base_image:
record_usage(docker_explicit_base=True)
base_image = args.base_image[0]
if args.distribution:
target_distribution = args.distribution[0]
else:
target_distribution = None
else:
target_distribution, base_image = select_image(runs)
logging.info("Using base image %s", base_image)
logging.debug("Distribution: %s", target_distribution or "unknown")
target.mkdir(parents=True)
rpz_pack.copy_data_tar(target / 'data.tgz')
arch = runs[0]['architecture']
# Writes Dockerfile
logging.info("Writing %s...", target / 'Dockerfile')
with (target / 'Dockerfile').open('w',
encoding='utf-8', newline='\n') as fp:
fp.write('FROM %s\n\n' % base_image)
# Installs busybox
download_file(busybox_url(arch),
target / 'busybox',
'busybox-%s' % arch)
fp.write('COPY busybox /busybox\n')
# Installs rpzsudo
download_file(sudo_url(arch),
target / 'rpzsudo',
'rpzsudo-%s' % arch)
fp.write('COPY rpzsudo /rpzsudo\n\n')
fp.write('COPY data.tgz /reprozip_data.tgz\n\n')
fp.write('COPY rpz-files.list /rpz-files.list\n')
fp.write('RUN \\\n'
' chmod +x /busybox /rpzsudo && \\\n')
if args.install_pkgs:
# Install every package through package manager
missing_packages = []
else:
# Only install packages that were not packed
missing_packages = [pkg for pkg in packages if pkg.packfiles]
packages = [pkg for pkg in packages if not pkg.packfiles]
if packages:
record_usage(docker_install_pkgs=True)
try:
installer = select_installer(pack, runs, target_distribution)
except CantFindInstaller as e:
logging.error("Need to install %d packages but couldn't "
"select a package installer: %s",
len(packages), e)
sys.exit(1)
# Updates package sources
fp.write(' %s && \\\n' % installer.update_script())
# Installs necessary packages
fp.write(' %s && \\\n' % installer.install_script(packages))
logging.info("Dockerfile will install the %d software packages "
"that were not packed", len(packages))
else:
record_usage(docker_install_pkgs=False)
# Untar
paths = set()
pathlist = []
# Adds intermediate directories, and checks for existence in the tar
missing_files = chain.from_iterable(pkg.files
for pkg in missing_packages)
for f in chain(other_files, missing_files):
path = PosixPath('/')
for c in rpz_pack.remove_data_prefix(f.path).components:
path = path / c
if path in paths:
continue
paths.add(path)
try:
rpz_pack.get_data(path)
except KeyError:
logging.info("Missing file %s", path)
else:
pathlist.append(path)
rpz_pack.close()
# FIXME : for some reason we need reversed() here, I'm not sure why.
# Need to read more of tar's docs.
# TAR bug: --no-overwrite-dir removes --keep-old-files
with (target / 'rpz-files.list').open('wb') as lfp:
for p in reversed(pathlist):
lfp.write(join_root(rpz_pack.data_prefix, p).path)
lfp.write(b'\0')
fp.write(' cd / && '
'(tar zpxf /reprozip_data.tgz -U --recursive-unlink '
'--numeric-owner --strip=1 --null -T /rpz-files.list || '
'/busybox echo "TAR reports errors, this might or might '
'not prevent the execution to run")\n')
# Meta-data for reprounzip
write_dict(target / '.reprounzip', {})
signals.post_setup(target=target, pack=pack)
def directory_create(args):
"""Unpacks the experiment in a folder.
Only the files that are not part of a package are copied (unless they are
missing from the system and were packed).
In addition, input files are put in a tar.gz (so they can be put back after
an upload) and the configuration file is extracted.
"""
if not args.pack:
logging.critical("setup needs the pack filename")
sys.exit(1)
pack = Path(args.pack[0])
target = Path(args.target[0])
if target.exists():
logging.critical("Target directory exists")
sys.exit(1)
if not issubclass(DefaultAbstractPath, PosixPath):
logging.critical("Not unpacking on POSIX system")
sys.exit(1)
signals.pre_setup(target=target, pack=pack)
# Unpacks configuration file
rpz_pack = RPZPack(pack)
rpz_pack.extract_config(target / 'config.yml')
# Loads config
config = load_config_file(target / 'config.yml', True)
packages = config.packages
target.mkdir()
root = (target / 'root').absolute()
# Checks packages
missing_files = False
for pkg in packages:
if pkg.packfiles:
continue
for f in pkg.files:
if not Path(f.path).exists():
logging.error(
"Missing file %s (from package %s that wasn't packed) "
"on host, experiment will probably miss it.",
f, pkg.name)
missing_files = True
if missing_files:
record_usage(directory_missing_pkgs=True)
logging.error("Some packages are missing, you should probably install "
"them.\nUse 'reprounzip installpkgs -h' for help")
root.mkdir()
try:
# Unpacks files
members = rpz_pack.list_data()
for m in members:
# Remove 'DATA/' prefix
m.name = str(rpz_pack.remove_data_prefix(m.name))
# Makes symlink targets relative
if m.issym():
linkname = PosixPath(m.linkname)
if linkname.is_absolute:
m.linkname = join_root(root, PosixPath(m.linkname)).path
logging.info("Extracting files...")
rpz_pack.extract_data(root, members)
rpz_pack.close()
# Original input files, so upload can restore them
input_files = [f.path for f in itervalues(config.inputs_outputs)
if f.read_runs]
if input_files:
logging.info("Packing up original input files...")
inputtar = tarfile.open(str(target / 'inputs.tar.gz'), 'w:gz')
for ifile in input_files:
filename = join_root(root, ifile)
if filename.exists():
inputtar.add(str(filename), str(ifile))
inputtar.close()
# Meta-data for reprounzip
metadata_write(target, metadata_initial_iofiles(config), 'directory')
signals.post_setup(target=target, pack=pack)
except Exception:
rmtree_fixed(root)
raise
def chroot_create(args):
"""Unpacks the experiment in a folder so it can be run with chroot.
All the files in the pack are unpacked; system files are copied only if
they were not packed, and busybox is installed if /bin/sh wasn't packed.
In addition, input files are put in a tar.gz (so they can be put back after
an upload) and the configuration file is extracted.
"""
if not args.pack:
logging.critical("setup/create needs the pack filename")
sys.exit(1)
pack = Path(args.pack[0])
target = Path(args.target[0])
if target.exists():
logging.critical("Target directory exists")
sys.exit(1)
if DefaultAbstractPath is not PosixPath:
logging.critical("Not unpacking on POSIX system")
sys.exit(1)
signals.pre_setup(target=target, pack=pack)
# We can only restore owner/group of files if running as root
restore_owner = should_restore_owner(args.restore_owner)
# Unpacks configuration file
tar = tarfile.open(str(pack), 'r:*')
member = tar.getmember('METADATA/config.yml')
member.name = 'config.yml'
tar.extract(member, str(target))
# Loads config
runs, packages, other_files = load_config_file(target / 'config.yml', True)
target.mkdir()
root = (target / 'root').absolute()
root.mkdir()
# Checks that everything was packed
packages_not_packed = [pkg for pkg in packages if not pkg.packfiles]
if packages_not_packed:
logging.warning(
"According to configuration, some files were left out "
"because they belong to the following packages:%s"
"\nWill copy files from HOST SYSTEM",
''.join('\n %s' % pkg for pkg in packages_not_packed))
for pkg in packages_not_packed:
for f in pkg.files:
f = Path(f.path)
if not f.exists():
logging.error(
"Missing file %s (from package %s) on host, "
"experiment will probably miss it", f, pkg.name)
dest = join_root(root, f)
dest.parent.mkdir(parents=True)
if f.is_link():
dest.symlink(f.read_link())
else:
f.copy(dest)
if restore_owner:
stat = f.stat()
dest.chown(stat.st_uid, stat.st_gid)
# Unpacks files
if any('..' in m.name or m.name.startswith('/') for m in tar.getmembers()):
logging.critical("Tar archive contains invalid pathnames")
sys.exit(1)
members = [m for m in tar.getmembers() if m.name.startswith('DATA/')]
for m in members:
m.name = m.name[5:]
if not restore_owner:
uid = os.getuid()
gid = os.getgid()
for m in members:
m.uid = uid
m.gid = gid
logging.info("Extracting files...")
tar.extractall(str(root), members)
tar.close()
# Sets up /bin/sh and /usr/bin/env, downloading busybox if necessary
sh_path = join_root(root, Path('/bin/sh'))
env_path = join_root(root, Path('/usr/bin/env'))
if not sh_path.lexists() or not env_path.lexists():
logging.info("Setting up busybox...")
busybox_path = join_root(root, Path('/bin/busybox'))
busybox_path.parent.mkdir(parents=True)
with make_dir_writable(join_root(root, Path('/bin'))):
download_file(busybox_url(runs[0]['architecture']), busybox_path)
busybox_path.chmod(0o755)
if not sh_path.lexists():
sh_path.parent.mkdir(parents=True)
sh_path.symlink('/bin/busybox')
if not env_path.lexists():
env_path.parent.mkdir(parents=True)
env_path.symlink('/bin/busybox')
# Original input files, so upload can restore them
if any(run['input_files'] for run in runs):
logging.info("Packing up original input files...")
inputtar = tarfile.open(str(target / 'inputs.tar.gz'), 'w:gz')
for run in runs:
for ifile in itervalues(run['input_files']):
inputtar.add(str(join_root(root, PosixPath(ifile))),
str(PosixPath(ifile)))
inputtar.close()
# Meta-data for reprounzip
write_dict(target / '.reprounzip', {}, 'chroot')
signals.post_setup(target=target)
def download_file(url, dest, cachename=None, ssl_verify=None):
"""Downloads a file using a local cache.
If the file cannot be downloaded or if it wasn't modified, the cached
version will be used instead.
The cache lives in ``~/.cache/reprozip/``.
"""
if cachename is None:
if dest is None:
raise ValueError("One of 'dest' or 'cachename' must be specified")
cachename = dest.components[-1]
headers = {}
if 'XDG_CACHE_HOME' in os.environ:
cache = Path(os.environ['XDG_CACHE_HOME'])
else:
cache = Path('~/.cache').expand_user()
cache = cache / 'reprozip' / cachename
if cache.exists():
mtime = email.utils.formatdate(cache.mtime(), usegmt=True)
headers['If-Modified-Since'] = mtime
cache.parent.mkdir(parents=True)
try:
response = requests.get(url, headers=headers,
timeout=2 if cache.exists() else 10,
stream=True, verify=ssl_verify)
response.raise_for_status()
if response.status_code == 304:
raise requests.HTTPError(
'304 File is up to date, no data returned',
response=response)
except requests.RequestException as e:
if cache.exists():
if e.response and e.response.status_code == 304:
logging.info("Download %s: cache is up to date", cachename)
else:
logging.warning("Download %s: error downloading %s: %s",
cachename, url, e)
if dest is not None:
cache.copy(dest)
return dest
else:
return cache
else:
raise
logging.info("Download %s: downloading %s", cachename, url)
try:
with cache.open('wb') as f:
for chunk in response.iter_content(4096):
f.write(chunk)
response.close()
except Exception as e: # pragma: no cover
try:
cache.remove()
except OSError:
pass
raise e
logging.info("Downloaded %s successfully", cachename)
if dest is not None:
cache.copy(dest)
return dest
else:
return cache
def chroot_create(args):
"""Unpacks the experiment in a folder so it can be run with chroot.
All the files in the pack are unpacked; system files are copied only if
they were not packed, and busybox is installed if /bin/sh wasn't packed.
In addition, input files are put in a tar.gz (so they can be put back after
an upload) and the configuration file is extracted.
"""
if not args.pack:
logging.critical("setup/create needs the pack filename")
sys.exit(1)
pack = Path(args.pack[0])
target = Path(args.target[0])
if target.exists():
logging.critical("Target directory exists")
sys.exit(1)
if DefaultAbstractPath is not PosixPath:
logging.critical("Not unpacking on POSIX system")
sys.exit(1)
signals.pre_setup(target=target, pack=pack)
# We can only restore owner/group of files if running as root
restore_owner = should_restore_owner(args.restore_owner)
# Unpacks configuration file
tar = tarfile.open(str(pack), 'r:*')
member = tar.getmember('METADATA/config.yml')
member.name = 'config.yml'
tar.extract(member, str(target))
# Loads config
runs, packages, other_files = load_config_file(target / 'config.yml', True)
target.mkdir()
root = (target / 'root').absolute()
root.mkdir()
# Checks that everything was packed
packages_not_packed = [pkg for pkg in packages if not pkg.packfiles]
if packages_not_packed:
record_usage(chroot_missing_pkgs=True)
logging.warning("According to configuration, some files were left out "
"because they belong to the following packages:%s"
"\nWill copy files from HOST SYSTEM",
''.join('\n %s' % pkg
for pkg in packages_not_packed))
missing_files = False
for pkg in packages_not_packed:
for f in pkg.files:
f = Path(f.path)
if not f.exists():
logging.error(
"Missing file %s (from package %s) on host, "
"experiment will probably miss it",
f, pkg.name)
missing_files = True
continue
dest = join_root(root, f)
dest.parent.mkdir(parents=True)
if f.is_link():
dest.symlink(f.read_link())
else:
f.copy(dest)
if restore_owner:
stat = f.stat()
dest.chown(stat.st_uid, stat.st_gid)
if missing_files:
record_usage(chroot_mising_files=True)
# Unpacks files
if any('..' in m.name or m.name.startswith('/') for m in tar.getmembers()):
logging.critical("Tar archive contains invalid pathnames")
sys.exit(1)
members = [m for m in tar.getmembers() if m.name.startswith('DATA/')]
for m in members:
m.name = m.name[5:]
if not restore_owner:
uid = os.getuid()
gid = os.getgid()
for m in members:
m.uid = uid
m.gid = gid
logging.info("Extracting files...")
tar.extractall(str(root), members)
tar.close()
# Sets up /bin/sh and /usr/bin/env, downloading busybox if necessary
sh_path = join_root(root, Path('/bin/sh'))
env_path = join_root(root, Path('/usr/bin/env'))
if not sh_path.lexists() or not env_path.lexists():
logging.info("Setting up busybox...")
busybox_path = join_root(root, Path('/bin/busybox'))
busybox_path.parent.mkdir(parents=True)
with make_dir_writable(join_root(root, Path('/bin'))):
download_file(busybox_url(runs[0]['architecture']),
busybox_path)
busybox_path.chmod(0o755)
if not sh_path.lexists():
sh_path.parent.mkdir(parents=True)
sh_path.symlink('/bin/busybox')
if not env_path.lexists():
env_path.parent.mkdir(parents=True)
env_path.symlink('/bin/busybox')
# Original input files, so upload can restore them
if any(run['input_files'] for run in runs):
logging.info("Packing up original input files...")
inputtar = tarfile.open(str(target / 'inputs.tar.gz'), 'w:gz')
for run in runs:
for ifile in itervalues(run['input_files']):
inputtar.add(str(join_root(root, PosixPath(ifile))),
str(PosixPath(ifile)))
inputtar.close()
# Meta-data for reprounzip
write_dict(target / '.reprounzip', {}, 'chroot')
signals.post_setup(target=target)
def vagrant_setup_create(args):
"""Sets up the experiment to be run in a Vagrant-built virtual machine.
This can either build a chroot or not.
If building a chroot, we do just like without Vagrant: we copy all the
files and only get what's missing from the host. But we do install
automatically the packages whose files are required.
If not building a chroot, we install all the packages, and only unpack
files that don't come from packages.
In short: files from packages with packfiles=True will only be used if
building a chroot.
"""
if not args.pack:
logging.critical("setup/create needs the pack filename")
sys.exit(1)
pack = Path(args.pack[0])
target = Path(args.target[0])
if target.exists():
logging.critical("Target directory exists")
sys.exit(1)
use_chroot = args.use_chroot
mount_bind = args.bind_magic_dirs
record_usage(use_chroot=use_chroot,
mount_bind=mount_bind)
signals.pre_setup(target=target, pack=pack)
# Unpacks configuration file
tar = tarfile.open(str(pack), 'r:*')
member = tar.getmember('METADATA/config.yml')
member.name = 'config.yml'
tar.extract(member, str(target))
tar.close()
# Loads config
runs, packages, other_files = load_config(target / 'config.yml', True)
if args.base_image and args.base_image[0]:
record_usage(vagrant_explicit_image=True)
box = args.base_image[0]
if args.distribution:
target_distribution = args.distribution[0]
else:
target_distribution = None
else:
target_distribution, box = select_box(runs)
logging.info("Using box %s", box)
logging.debug("Distribution: %s", target_distribution or "unknown")
# If using chroot, we might still need to install packages to get missing
# (not packed) files
if use_chroot:
packages = [pkg for pkg in packages if not pkg.packfiles]
if packages:
record_usage(vagrant_install_pkgs=True)
logging.info("Some packages were not packed, so we'll install and "
"copy their files\n"
"Packages that are missing:\n%s",
' '.join(pkg.name for pkg in packages))
if packages:
try:
installer = select_installer(pack, runs, target_distribution)
except CantFindInstaller as e:
logging.error("Need to install %d packages but couldn't select a "
"package installer: %s",
len(packages), e)
target.mkdir(parents=True)
# Writes setup script
logging.info("Writing setup script %s...", target / 'setup.sh')
with (target / 'setup.sh').open('w', encoding='utf-8', newline='\n') as fp:
fp.write('#!/bin/sh\n\nset -e\n\n')
if packages:
# Updates package sources
fp.write(installer.update_script())
fp.write('\n')
# Installs necessary packages
fp.write(installer.install_script(packages))
fp.write('\n')
# TODO : Compare package versions (painful because of sh)
# Untar
if use_chroot:
fp.write('\n'
'mkdir /experimentroot; cd /experimentroot\n')
fp.write('tar zpxf /vagrant/experiment.rpz '
'--numeric-owner --strip=1 DATA\n')
if mount_bind:
fp.write('\n'
'mkdir -p /experimentroot/dev\n'
'mount -o rbind /dev /experimentroot/dev\n'
'mkdir -p /experimentroot/proc\n'
'mount -o rbind /proc /experimentroot/proc\n')
for pkg in packages:
fp.write('\n# Copies files from package %s\n' % pkg.name)
for f in pkg.files:
f = f.path
dest = join_root(PosixPath('/experimentroot'), f)
fp.write('mkdir -p %s\n' %
shell_escape(unicode_(f.parent)))
fp.write('cp -L %s %s\n' % (
shell_escape(unicode_(f)),
shell_escape(unicode_(dest))))
else:
fp.write('\ncd /\n')
paths = set()
pathlist = []
dataroot = PosixPath('DATA')
# Adds intermediate directories, and checks for existence in the
# tar
tar = tarfile.open(str(pack), 'r:*')
for f in other_files:
path = PosixPath('/')
for c in f.path.components[1:]:
path = path / c
if path in paths:
continue
paths.add(path)
datapath = join_root(dataroot, path)
try:
tar.getmember(str(datapath))
except KeyError:
logging.info("Missing file %s", datapath)
else:
pathlist.append(unicode_(datapath))
tar.close()
# FIXME : for some reason we need reversed() here, I'm not sure
# why. Need to read more of tar's docs.
# TAR bug: --no-overwrite-dir removes --keep-old-files
# TAR bug: there is no way to make --keep-old-files not report an
# error if an existing file is encountered. --skip-old-files was
# introduced too recently. Instead, we just ignore the exit status
fp.write('tar zpxf /vagrant/experiment.rpz --keep-old-files '
'--numeric-owner --strip=1 %s || /bin/true\n' %
' '.join(shell_escape(p) for p in reversed(pathlist)))
# Copies /bin/sh + dependencies
if use_chroot:
url = busybox_url(runs[0]['architecture'])
fp.write(r'''
mkdir -p /experimentroot/bin
mkdir -p /experimentroot/usr/bin
if [ ! -e /experimentroot/bin/sh -o ! -e /experimentroot/usr/bin/env ]; then
wget --quiet -O /experimentroot/bin/busybox {url}
chmod +x /experimentroot/bin/busybox
fi
[ -e /experimentroot/bin/sh ] || \
ln -s /bin/busybox /experimentroot/bin/sh
[ -e /experimentroot/usr/bin/env ] || \
ln -s /bin/busybox /experimentroot/usr/bin/env
'''.format(url=url))
# Copies pack
logging.info("Copying pack file...")
pack.copyfile(target / 'experiment.rpz')
# Writes Vagrant file
logging.info("Writing %s...", target / 'Vagrantfile')
with (target / 'Vagrantfile').open('w', encoding='utf-8',
newline='\n') as fp:
# Vagrant header and version
fp.write('# -*- mode: ruby -*-\n'
'# vi: set ft=ruby\n\n'
'VAGRANTFILE_API_VERSION = "2"\n\n'
'Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|\n')
# Selects which box to install
fp.write(' config.vm.box = "%s"\n' % box)
# Run the setup script on the virtual machine
fp.write(' config.vm.provision "shell", path: "setup.sh"\n')
fp.write('end\n')
# Meta-data for reprounzip
write_dict(target / '.reprounzip', {'use_chroot': use_chroot})
signals.post_setup(target=target)
def directory_create(args):
"""Unpacks the experiment in a folder.
Only the files that are not part of a package are copied (unless they are
missing from the system and were packed).
In addition, input files are put in a tar.gz (so they can be put back after
an upload) and the configuration file is extracted.
"""
if not args.pack:
logging.critical("setup needs the pack filename")
sys.exit(1)
pack = Path(args.pack[0])
target = Path(args.target[0])
if target.exists():
logging.critical("Target directory exists")
sys.exit(1)
if not issubclass(DefaultAbstractPath, PosixPath):
logging.critical("Not unpacking on POSIX system")
sys.exit(1)
signals.pre_setup(target=target, pack=pack)
# Unpacks configuration file
tar = tarfile.open(str(pack), 'r:*')
member = tar.getmember('METADATA/config.yml')
member.name = 'config.yml'
tar.extract(member, str(target))
# Loads config
runs, packages, other_files = load_config_file(target / 'config.yml', True)
target.mkdir()
root = (target / 'root').absolute()
root.mkdir()
# Checks packages
missing_files = False
for pkg in packages:
if pkg.packfiles:
continue
for f in pkg.files:
f = Path(f.path)
if not f.exists():
logging.error(
"Missing file %s (from package %s that wasn't packed) "
"on host, experiment will probably miss it.",
f, pkg.name)
missing_files = True
if missing_files:
record_usage(directory_missing_pkgs=True)
logging.error(
"Some packages are missing, you should probably install "
"them.\nUse 'reprounzip installpkgs -h' for help")
# Unpacks files
if any('..' in m.name or m.name.startswith('/') for m in tar.getmembers()):
logging.critical("Tar archive contains invalid pathnames")
sys.exit(1)
members = [m for m in tar.getmembers() if m.name.startswith('DATA/')]
for m in members:
m.name = m.name[5:]
# Makes symlink targets relative
for m in members:
if not m.issym():
continue
linkname = PosixPath(m.linkname)
if linkname.is_absolute:
m.linkname = join_root(root, PosixPath(m.linkname)).path
logging.info("Extracting files...")
tar.extractall(str(root), members)
tar.close()
# Gets library paths
lib_dirs = []
p = subprocess.Popen(['/sbin/ldconfig', '-v', '-N'],
stdout=subprocess.PIPE)
try:
for l in p.stdout:
if len(l) < 3 or l[0] in (b' ', b'\t'):
continue
if l.endswith(b':\n'):
lib_dirs.append(Path(l[:-2]))
finally:
p.wait()
# Original input files, so upload can restore them
if any(run['input_files'] for run in runs):
logging.info("Packing up original input files...")
inputtar = tarfile.open(str(target / 'inputs.tar.gz'), 'w:gz')
for run in runs:
for ifile in itervalues(run['input_files']):
inputtar.add(str(join_root(root, PosixPath(ifile))),
str(PosixPath(ifile)))
inputtar.close()
# Meta-data for reprounzip
write_dict(target / '.reprounzip', {}, 'directory')
signals.post_setup(target=target)
def functional_tests(raise_warnings, interactive, run_vagrant, run_docker):
# Tests on Python < 2.7.3: need to use separate reprozip Python (with known
# working version of Python)
if sys.version_info < (2, 7, 3):
bug13676 = True
if 'REPROZIP_PYTHON' not in os.environ:
sys.stderr.write("Error: using reprozip with Python %s!\n" %
sys.version.split(' ', 1)[0])
sys.exit(1)
else:
bug13676 = False
rpz = [os.environ.get('REPROZIP_PYTHON', sys.executable)]
rpuz = [os.environ.get('REPROUNZIP_PYTHON', sys.executable)]
# Can't match on the SignalWarning category here because of a Python bug
# http://bugs.python.org/issue22543
if raise_warnings:
rpz.extend(['-W', 'error:signal'])
rpuz.extend(['-W', 'error:signal'])
if 'COVER' in os.environ:
rpz.extend(['-m'] + os.environ['COVER'].split(' '))
rpuz.extend(['-m'] + os.environ['COVER'].split(' '))
reprozip_main = tests.parent / 'reprozip/reprozip/main.py'
reprounzip_main = tests.parent / 'reprounzip/reprounzip/main.py'
verbose = ['-v'] * 3
rpz.extend([reprozip_main.absolute().path] + verbose)
rpuz.extend([reprounzip_main.absolute().path] + verbose)
print("Command lines are:\n%r\n%r" % (rpz, rpuz))
# ########################################
# testrun /bin/echo
#
output = check_output(rpz + ['testrun', '/bin/echo', 'outputhere'])
assert any(b' 1 | /bin/echo outputhere ' in l for l in output.splitlines())
output = check_output(
rpz + ['testrun', '-a', '/fake/path/echo', '/bin/echo', 'outputhere'])
assert any(b' 1 | (/bin/echo) /fake/path/echo outputhere ' in l
for l in output.splitlines())
# ########################################
# testrun multiple commands
#
check_call(rpz + [
'testrun', 'bash', '-c', 'cat ../../../../../etc/passwd;'
'cd /var/lib;'
'cat ../../etc/group'
])
check_call(rpz + ['trace', 'bash', '-c', 'cat /etc/passwd;echo'])
check_call(
rpz +
['trace', '--continue', 'sh', '-c', 'cat /etc/group;/usr/bin/id'])
check_call(rpz + ['pack'])
if not bug13676:
check_call(rpuz + ['graph', 'graph.dot'])
check_call(rpuz + ['graph', 'graph2.dot', 'experiment.rpz'])
sudo = ['sudo', '-E'] # -E to keep REPROZIP_USAGE_STATS
# ########################################
# 'simple' program: trace, pack, info, unpack
#
# Build
build('simple', ['simple.c'])
# Trace
check_call(rpz + [
'trace', '-d', 'rpz-simple', './simple',
(tests / 'simple_input.txt').path, 'simple_output.txt'
])
orig_output_location = Path('simple_output.txt').absolute()
assert orig_output_location.is_file()
with orig_output_location.open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
orig_output_location.remove()
# Read config
with Path('rpz-simple/config.yml').open(encoding='utf-8') as fp:
conf = yaml.safe_load(fp)
other_files = set(Path(f).absolute() for f in conf['other_files'])
expected = [Path('simple'), (tests / 'simple_input.txt')]
assert other_files.issuperset([f.resolve() for f in expected])
# Check input and output files
input_files = conf['runs'][0]['input_files']
assert (dict((k, Path(f).name) for k, f in iteritems(input_files)) == {
'arg': b'simple_input.txt'
})
output_files = conf['runs'][0]['output_files']
print(dict((k, Path(f).name) for k, f in iteritems(output_files)))
# Here we don't test for dict equality, since we might have C coverage
# files in the mix
assert Path(output_files['arg']).name == b'simple_output.txt'
# Pack
check_call(rpz + ['pack', '-d', 'rpz-simple', 'simple.rpz'])
Path('simple').remove()
# Info
check_call(rpuz + ['info', 'simple.rpz'])
# Show files
check_call(rpuz + ['showfiles', 'simple.rpz'])
# Lists packages
check_call(rpuz + ['installpkgs', '--summary', 'simple.rpz'])
# Unpack directory
check_call(rpuz + ['directory', 'setup', 'simple.rpz', 'simpledir'])
# Run directory
check_call(rpuz + ['directory', 'run', 'simpledir'])
output_in_dir = join_root(Path('simpledir/root'), orig_output_location)
with output_in_dir.open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Delete with wrong command (should fail)
p = subprocess.Popen(rpuz + ['chroot', 'destroy', 'simpledir'],
stderr=subprocess.PIPE)
stdout, stderr = p.communicate()
assert p.poll() != 0
stderr = stderr.splitlines()
assert b"Wrong unpacker used" in stderr[0]
assert stderr[1].startswith(b"usage: ")
# Delete directory
check_call(rpuz + ['directory', 'destroy', 'simpledir'])
# Unpack chroot
check_call(
sudo + rpuz +
['chroot', 'setup', '--bind-magic-dirs', 'simple.rpz', 'simplechroot'])
try:
# Run chroot
check_call(sudo + rpuz + ['chroot', 'run', 'simplechroot'])
output_in_chroot = join_root(Path('simplechroot/root'),
orig_output_location)
with output_in_chroot.open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Get output file
check_call(sudo + rpuz +
['chroot', 'download', 'simplechroot', 'arg:output1.txt'])
with Path('output1.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Replace input file
check_call(sudo + rpuz + [
'chroot', 'upload', 'simplechroot',
'%s:arg' % (tests / 'simple_input2.txt')
])
check_call(sudo + rpuz + ['chroot', 'upload', 'simplechroot'])
# Run again
check_call(sudo + rpuz + ['chroot', 'run', 'simplechroot'])
output_in_chroot = join_root(Path('simplechroot/root'),
orig_output_location)
with output_in_chroot.open(encoding='utf-8') as fp:
assert fp.read().strip() == '36'
# Delete with wrong command (should fail)
p = subprocess.Popen(rpuz + ['directory', 'destroy', 'simplechroot'],
stderr=subprocess.PIPE)
stdout, stderr = p.communicate()
assert p.poll() != 0
stderr = stderr.splitlines()
assert b"Wrong unpacker used" in stderr[0]
assert stderr[1].startswith(b"usage:")
finally:
# Delete chroot
check_call(sudo + rpuz + ['chroot', 'destroy', 'simplechroot'])
if not (tests / 'vagrant').exists():
check_call([
'sudo', 'sh', '-c',
'mkdir %(d)s; chmod 777 %(d)s' % {
'd': tests / 'vagrant'
}
])
# Unpack Vagrant-chroot
check_call(rpuz + [
'vagrant', 'setup/create', '--use-chroot', 'simple.rpz',
(tests / 'vagrant/simplevagrantchroot').path
])
print("\nVagrant project set up in simplevagrantchroot")
try:
if run_vagrant:
check_call(rpuz + [
'vagrant', 'run', '--no-stdin',
(tests / 'vagrant/simplevagrantchroot').path
])
# Destroy
check_call(rpuz + [
'vagrant', 'destroy', (tests /
'vagrant/simplevagrantchroot').path
])
elif interactive:
print("Test and press enter")
sys.stdin.readline()
finally:
if (tests / 'vagrant/simplevagrantchroot').exists():
(tests / 'vagrant/simplevagrantchroot').rmtree()
# Unpack Vagrant without chroot
check_call(rpuz + [
'vagrant', 'setup/create', '--dont-use-chroot', 'simple.rpz',
(tests / 'vagrant/simplevagrant').path
])
print("\nVagrant project set up in simplevagrant")
try:
if run_vagrant:
check_call(rpuz + [
'vagrant', 'run', '--no-stdin',
(tests / 'vagrant/simplevagrant').path
])
# Destroy
check_call(
rpuz +
['vagrant', 'destroy', (tests / 'vagrant/simplevagrant').path])
elif interactive:
print("Test and press enter")
sys.stdin.readline()
finally:
if (tests / 'vagrant/simplevagrant').exists():
(tests / 'vagrant/simplevagrant').rmtree()
# Unpack Docker
check_call(rpuz + ['docker', 'setup/create', 'simple.rpz', 'simpledocker'])
print("\nDocker project set up in simpledocker")
try:
if run_docker:
check_call(rpuz + ['docker', 'setup/build', 'simpledocker'])
check_call(rpuz + ['docker', 'run', 'simpledocker'])
# Get output file
check_call(
rpuz +
['docker', 'download', 'simpledocker', 'arg:doutput1.txt'])
with Path('doutput1.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Replace input file
check_call(rpuz + [
'docker', 'upload', 'simpledocker',
'%s:arg' % (tests / 'simple_input2.txt')
])
check_call(rpuz + ['docker', 'upload', 'simpledocker'])
check_call(rpuz + ['showfiles', 'simpledocker'])
# Run again
check_call(rpuz + ['docker', 'run', 'simpledocker'])
# Get output file
check_call(
rpuz +
['docker', 'download', 'simpledocker', 'arg:doutput2.txt'])
with Path('doutput2.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '36'
# Destroy
check_call(rpuz + ['docker', 'destroy', 'simpledocker'])
elif interactive:
print("Test and press enter")
sys.stdin.readline()
finally:
if Path('simpledocker').exists():
Path('simpledocker').rmtree()
# ########################################
# 'threads' program: testrun
#
# Build
build('threads', ['threads.c'], ['-lpthread'])
# Trace
check_call(rpz + ['testrun', './threads'])
# ########################################
# 'segv' program: testrun
#
# Build
build('segv', ['segv.c'])
# Trace
check_call(rpz + ['testrun', './segv'])
# ########################################
# 'exec_echo' program: trace, pack, run --cmdline
#
# Build
build('exec_echo', ['exec_echo.c'])
# Trace
check_call(rpz + ['trace', './exec_echo', 'originalexecechooutput'])
# Pack
check_call(rpz + ['pack', 'exec_echo.rpz'])
# Unpack chroot
check_call(sudo + rpuz +
['chroot', 'setup', 'exec_echo.rpz', 'echochroot'])
try:
# Run original command-line
output = check_output(sudo + rpuz + ['chroot', 'run', 'echochroot'])
assert output == b'originalexecechooutput\n'
# Prints out command-line
output = check_output(sudo + rpuz +
['chroot', 'run', 'echochroot', '--cmdline'])
assert any(b'./exec_echo originalexecechooutput' == s.strip()
for s in output.split(b'\n'))
# Run with different command-line
output = check_output(sudo + rpuz + [
'chroot', 'run', 'echochroot', '--cmdline', './exec_echo',
'changedexecechooutput'
])
assert output == b'changedexecechooutput\n'
finally:
check_call(sudo + rpuz + ['chroot', 'destroy', 'echochroot'])
# ########################################
# 'exec_echo' program: testrun
# This is built with -m32 so that we transition:
# python (x64) -> exec_echo (i386) -> echo (x64)
#
if sys.maxsize > 2**32:
# Build
build('exec_echo32', ['exec_echo.c'], ['-m32'])
# Trace
check_call(rpz + ['testrun', './exec_echo32 42'])
else:
print("Can't try exec_echo transitions: not running on 64bits")
# ########################################
# Tracing non-existing program
#
check_call(rpz + ['testrun', './doesntexist'])
# ########################################
# 'connect' program: testrun
#
# Build
build('connect', ['connect.c'])
# Trace
stderr = check_errout(rpz + ['testrun', './connect'])
stderr = stderr.split(b'\n')
assert not any(b'program exited with non-zero code' in l for l in stderr)
assert any(
re.search(br'process connected to [0-9.]+:80', l) for l in stderr)
# ########################################
# Copies back coverage report
#
coverage = Path('.coverage')
if coverage.exists():
coverage.copyfile(tests.parent / '.coverage.runpy')
def functional_tests(raise_warnings, interactive, run_vagrant, run_docker):
# Tests on Python < 2.7.3: need to use separate reprozip Python (with known
# working version of Python)
if sys.version_info < (2, 7, 3):
bug13676 = True
if 'REPROZIP_PYTHON' not in os.environ:
sys.stderr.write("Error: using reprozip with Python %s!\n" %
sys.version.split(' ', 1)[0])
sys.exit(1)
else:
bug13676 = False
rpz = [os.environ.get('REPROZIP_PYTHON', sys.executable)]
rpuz = [os.environ.get('REPROUNZIP_PYTHON', sys.executable)]
# Can't match on the SignalWarning category here because of a Python bug
# http://bugs.python.org/issue22543
if raise_warnings:
rpz.extend(['-W', 'error:signal'])
rpuz.extend(['-W', 'error:signal'])
if 'COVER' in os.environ:
rpz.extend(['-m'] + os.environ['COVER'].split(' '))
rpuz.extend(['-m'] + os.environ['COVER'].split(' '))
reprozip_main = tests.parent / 'reprozip/reprozip/main.py'
reprounzip_main = tests.parent / 'reprounzip/reprounzip/main.py'
verbose = ['-v'] * 3
rpz.extend([reprozip_main.absolute().path] + verbose)
rpuz.extend([reprounzip_main.absolute().path] + verbose)
print("Command lines are:\n%r\n%r" % (rpz, rpuz))
# ########################################
# testrun /bin/echo
#
output = check_output(rpz + ['testrun', '/bin/echo', 'outputhere'])
assert any(b' 1 | /bin/echo outputhere ' in l
for l in output.splitlines())
output = check_output(rpz + ['testrun', '-a', '/fake/path/echo',
'/bin/echo', 'outputhere'])
assert any(b' 1 | (/bin/echo) /fake/path/echo outputhere ' in l
for l in output.splitlines())
# ########################################
# testrun multiple commands
#
check_call(rpz + ['testrun', 'bash', '-c',
'cat ../../../../../etc/passwd;'
'cd /var/lib;'
'cat ../../etc/group'])
check_call(rpz + ['trace',
'bash', '-c', 'cat /etc/passwd;echo'])
check_call(rpz + ['trace', '--continue',
'sh', '-c', 'cat /etc/group;/usr/bin/id'])
check_call(rpz + ['pack'])
if not bug13676:
check_call(rpuz + ['graph', 'graph.dot'])
check_call(rpuz + ['graph', 'graph2.dot', 'experiment.rpz'])
sudo = ['sudo', '-E'] # -E to keep REPROZIP_USAGE_STATS
# ########################################
# 'simple' program: trace, pack, info, unpack
#
def check_simple(args, stream, infile=1):
output = check_output(args, stream).splitlines()
try:
first = output.index(b"Read 6 bytes")
except ValueError:
stderr.write("output = %r\n" % output)
raise
if infile == 1:
assert output[first + 1] == b"a = 29, b = 13"
assert output[first + 2] == b"result = 42"
else: # infile == 2
assert output[first + 1] == b"a = 25, b = 11"
assert output[first + 2] == b"result = 36"
# Build
build('simple', ['simple.c'])
# Trace
check_call(rpz + ['trace', '-d', 'rpz-simple',
'./simple',
(tests / 'simple_input.txt').path,
'simple_output.txt'])
orig_output_location = Path('simple_output.txt').absolute()
assert orig_output_location.is_file()
with orig_output_location.open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
orig_output_location.remove()
# Read config
with Path('rpz-simple/config.yml').open(encoding='utf-8') as fp:
conf = yaml.safe_load(fp)
other_files = set(Path(f).absolute() for f in conf['other_files'])
expected = [Path('simple'), (tests / 'simple_input.txt')]
assert other_files.issuperset([f.resolve() for f in expected])
# Check input and output files
inputs_outputs = conf['inputs_outputs']
# Exactly one input: "arg1", "...simple_input.txt"
# Output: 'arg2', "...simple_output.txt"
# There might be more output files: the C coverage files
found = 0
for fdict in inputs_outputs:
if Path(fdict['path']).name == b'simple_input.txt':
assert fdict['name'] == 'arg1'
assert fdict['read_by_runs'] == [0]
assert not fdict.get('written_by_runs')
found |= 0x01
elif Path(fdict['path']).name == b'simple_output.txt':
assert fdict['name'] == 'arg2'
assert not fdict.get('read_by_runs')
assert fdict['written_by_runs'] == [0]
found |= 0x02
else:
# No other inputs
assert not fdict.get('read_by_runs')
assert found == 0x03
# Pack
check_call(rpz + ['pack', '-d', 'rpz-simple', 'simple.rpz'])
Path('simple').remove()
# Info
check_call(rpuz + ['info', 'simple.rpz'])
# Show files
check_call(rpuz + ['showfiles', 'simple.rpz'])
# Lists packages
check_call(rpuz + ['installpkgs', '--summary', 'simple.rpz'])
# Unpack directory
check_call(rpuz + ['directory', 'setup', 'simple.rpz', 'simpledir'])
# Run directory
check_simple(rpuz + ['directory', 'run', 'simpledir'], 'err')
output_in_dir = join_root(Path('simpledir/root'), orig_output_location)
with output_in_dir.open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Delete with wrong command (should fail)
p = subprocess.Popen(rpuz + ['chroot', 'destroy', 'simpledir'],
stderr=subprocess.PIPE)
out, err = p.communicate()
assert p.poll() != 0
err = err.splitlines()
assert b"Wrong unpacker used" in err[0]
assert err[1].startswith(b"usage: ")
# Delete directory
check_call(rpuz + ['directory', 'destroy', 'simpledir'])
# Unpack chroot
check_call(sudo + rpuz + ['chroot', 'setup', '--bind-magic-dirs',
'simple.rpz', 'simplechroot'])
try:
output_in_chroot = join_root(Path('simplechroot/root'),
orig_output_location)
# Run chroot
check_simple(sudo + rpuz + ['chroot', 'run', 'simplechroot'], 'err')
with output_in_chroot.open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Get output file
check_call(sudo + rpuz + ['chroot', 'download', 'simplechroot',
'arg2:output1.txt'])
with Path('output1.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Replace input file
check_call(sudo + rpuz + ['chroot', 'upload', 'simplechroot',
'%s:arg1' % (tests / 'simple_input2.txt')])
check_call(sudo + rpuz + ['chroot', 'upload', 'simplechroot'])
# Run again
check_simple(sudo + rpuz + ['chroot', 'run', 'simplechroot'], 'err', 2)
with output_in_chroot.open(encoding='utf-8') as fp:
assert fp.read().strip() == '36'
# Reset input file
check_call(sudo + rpuz + ['chroot', 'upload', 'simplechroot', ':arg1'])
# Run again
check_simple(sudo + rpuz + ['chroot', 'run', 'simplechroot'], 'err')
with output_in_chroot.open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Delete with wrong command (should fail)
p = subprocess.Popen(rpuz + ['directory', 'destroy', 'simplechroot'],
stderr=subprocess.PIPE)
out, err = p.communicate()
assert p.poll() != 0
err = err.splitlines()
assert b"Wrong unpacker used" in err[0]
assert err[1].startswith(b"usage:")
finally:
# Delete chroot
check_call(sudo + rpuz + ['chroot', 'destroy', 'simplechroot'])
if not (tests / 'vagrant').exists():
check_call(['sudo', 'sh', '-c',
'mkdir %(d)s; chmod 777 %(d)s' % {'d': tests / 'vagrant'}])
# Unpack Vagrant-chroot
check_call(rpuz + ['vagrant', 'setup/create', '--memory', '512',
'--use-chroot', 'simple.rpz',
(tests / 'vagrant/simplevagrantchroot').path])
print("\nVagrant project set up in simplevagrantchroot")
try:
if run_vagrant:
check_simple(rpuz + ['vagrant', 'run', '--no-stdin',
(tests / 'vagrant/simplevagrantchroot').path],
'out')
# Get output file
check_call(rpuz + ['vagrant', 'download',
(tests / 'vagrant/simplevagrantchroot').path,
'arg2:voutput1.txt'])
with Path('voutput1.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Replace input file
check_call(rpuz + ['vagrant', 'upload',
(tests / 'vagrant/simplevagrantchroot').path,
'%s:arg1' % (tests / 'simple_input2.txt')])
check_call(rpuz + ['vagrant', 'upload',
(tests / 'vagrant/simplevagrantchroot').path])
# Run again
check_simple(rpuz + ['vagrant', 'run', '--no-stdin',
(tests / 'vagrant/simplevagrantchroot').path],
'out', 2)
# Get output file
check_call(rpuz + ['vagrant', 'download',
(tests / 'vagrant/simplevagrantchroot').path,
'arg2:voutput2.txt'])
with Path('voutput2.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '36'
# Reset input file
check_call(rpuz + ['vagrant', 'upload',
(tests / 'vagrant/simplevagrantchroot').path,
':arg1'])
# Run again
check_simple(rpuz + ['vagrant', 'run', '--no-stdin',
(tests / 'vagrant/simplevagrantchroot').path],
'out')
# Get output file
check_call(rpuz + ['vagrant', 'download',
(tests / 'vagrant/simplevagrantchroot').path,
'arg2:voutput1.txt'])
with Path('voutput1.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Destroy
check_call(rpuz + ['vagrant', 'destroy',
(tests / 'vagrant/simplevagrantchroot').path])
elif interactive:
print("Test and press enter")
sys.stdin.readline()
finally:
if (tests / 'vagrant/simplevagrantchroot').exists():
(tests / 'vagrant/simplevagrantchroot').rmtree()
# Unpack Vagrant without chroot
check_call(rpuz + ['vagrant', 'setup/create', '--dont-use-chroot',
'simple.rpz',
(tests / 'vagrant/simplevagrant').path])
print("\nVagrant project set up in simplevagrant")
try:
if run_vagrant:
check_simple(rpuz + ['vagrant', 'run', '--no-stdin',
(tests / 'vagrant/simplevagrant').path],
'out')
# Get output file
check_call(rpuz + ['vagrant', 'download',
(tests / 'vagrant/simplevagrant').path,
'arg2:woutput1.txt'])
with Path('woutput1.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Replace input file
check_call(rpuz + ['vagrant', 'upload',
(tests / 'vagrant/simplevagrant').path,
'%s:arg1' % (tests / 'simple_input2.txt')])
check_call(rpuz + ['vagrant', 'upload',
(tests / 'vagrant/simplevagrant').path])
# Run again
check_simple(rpuz + ['vagrant', 'run', '--no-stdin',
(tests / 'vagrant/simplevagrant').path],
'out', 2)
# Get output file
check_call(rpuz + ['vagrant', 'download',
(tests / 'vagrant/simplevagrant').path,
'arg2:woutput2.txt'])
with Path('woutput2.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '36'
# Reset input file
check_call(rpuz + ['vagrant', 'upload',
(tests / 'vagrant/simplevagrant').path,
':arg1'])
# Run again
check_simple(rpuz + ['vagrant', 'run', '--no-stdin',
(tests / 'vagrant/simplevagrant').path],
'out')
# Get output file
check_call(rpuz + ['vagrant', 'download',
(tests / 'vagrant/simplevagrant').path,
'arg2:voutput1.txt'])
with Path('voutput1.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Destroy
check_call(rpuz + ['vagrant', 'destroy',
(tests / 'vagrant/simplevagrant').path])
elif interactive:
print("Test and press enter")
sys.stdin.readline()
finally:
if (tests / 'vagrant/simplevagrant').exists():
(tests / 'vagrant/simplevagrant').rmtree()
# Unpack Docker
check_call(rpuz + ['docker', 'setup/create', 'simple.rpz', 'simpledocker'])
print("\nDocker project set up in simpledocker")
try:
if run_docker:
check_call(rpuz + ['docker', 'setup/build', 'simpledocker'])
check_simple(rpuz + ['docker', 'run', 'simpledocker'], 'out')
# Get output file
check_call(rpuz + ['docker', 'download', 'simpledocker',
'arg2:doutput1.txt'])
with Path('doutput1.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Replace input file
check_call(rpuz + ['docker', 'upload', 'simpledocker',
'%s:arg1' % (tests / 'simple_input2.txt')])
check_call(rpuz + ['docker', 'upload', 'simpledocker'])
check_call(rpuz + ['showfiles', 'simpledocker'])
# Run again
check_simple(rpuz + ['docker', 'run', 'simpledocker'], 'out', 2)
# Get output file
check_call(rpuz + ['docker', 'download', 'simpledocker',
'arg2:doutput2.txt'])
with Path('doutput2.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '36'
# Reset input file
check_call(rpuz + ['docker', 'upload', 'simpledocker',
':arg1'])
# Run again
check_simple(rpuz + ['docker', 'run', 'simpledocker'], 'out')
# Get output file
check_call(rpuz + ['docker', 'download', 'simpledocker',
'arg2:doutput1.txt'])
with Path('doutput1.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Destroy
check_call(rpuz + ['docker', 'destroy', 'simpledocker'])
elif interactive:
print("Test and press enter")
sys.stdin.readline()
finally:
if Path('simpledocker').exists():
Path('simpledocker').rmtree()
# ########################################
# 'threads' program: testrun
#
# Build
build('threads', ['threads.c'], ['-lpthread'])
# Trace
output = check_output(rpz + ['testrun', './threads'], 'err')
assert any(b'successfully exec\'d /bin/./echo' in l
for l in output.splitlines())
# ########################################
# 'threads2' program: testrun
#
# Build
build('threads2', ['threads2.c'], ['-lpthread'])
# Trace
output = check_output(rpz + ['testrun', './threads2'], 'err')
assert any(b'successfully exec\'d /bin/echo' in l
for l in output.splitlines())
# ########################################
# 'segv' program: testrun
#
# Build
build('segv', ['segv.c'])
# Trace
check_call(rpz + ['testrun', './segv'])
# ########################################
# 'exec_echo' program: trace, pack, run --cmdline
#
# Build
build('exec_echo', ['exec_echo.c'])
# Trace
check_call(rpz + ['trace', './exec_echo', 'originalexecechooutput'])
# Pack
check_call(rpz + ['pack', 'exec_echo.rpz'])
# Unpack chroot
check_call(sudo + rpuz + ['chroot', 'setup',
'exec_echo.rpz', 'echochroot'])
try:
# Run original command-line
output = check_output(sudo + rpuz + ['chroot', 'run',
'echochroot'])
assert output == b'originalexecechooutput\n'
# Prints out command-line
output = check_output(sudo + rpuz + ['chroot', 'run',
'echochroot', '--cmdline'])
assert any(b'./exec_echo originalexecechooutput' == s.strip()
for s in output.split(b'\n'))
# Run with different command-line
output = check_output(sudo + rpuz + [
'chroot', 'run', 'echochroot',
'--cmdline', './exec_echo', 'changedexecechooutput'])
assert output == b'changedexecechooutput\n'
finally:
check_call(sudo + rpuz + ['chroot', 'destroy', 'echochroot'])
# ########################################
# 'exec_echo' program: testrun
# This is built with -m32 so that we transition:
# python (x64) -> exec_echo (i386) -> echo (x64)
#
if sys.maxsize > 2 ** 32:
# Build
build('exec_echo32', ['exec_echo.c'], ['-m32'])
# Trace
check_call(rpz + ['testrun', './exec_echo32 42'])
else:
print("Can't try exec_echo transitions: not running on 64bits")
# ########################################
# Tracing non-existing program
#
check_call(rpz + ['testrun', './doesntexist'])
# ########################################
# 'connect' program: testrun
#
# Build
build('connect', ['connect.c'])
# Trace
err = check_output(rpz + ['testrun', './connect'], 'err')
err = err.split(b'\n')
assert not any(b'program exited with non-zero code' in l for l in err)
assert any(re.search(br'process connected to [0-9.]+:80', l)
for l in err)
# ########################################
# 'vfork' program: testrun
#
# Build
build('vfork', ['vfork.c'])
# Trace
err = check_output(rpz + ['testrun', './vfork'], 'err')
err = err.split(b'\n')
assert not any(b'program exited with non-zero code' in l for l in err)
# ########################################
# 'rename' program: trace
#
# Build
build('rename', ['rename.c'])
# Trace
check_call(rpz + ['trace', '-d', 'rename-trace', './rename'])
with Path('rename-trace/config.yml').open(encoding='utf-8') as fp:
config = yaml.safe_load(fp)
# Check that written files were logged
database = Path.cwd() / 'rename-trace/trace.sqlite3'
if PY3:
# On PY3, connect() only accepts unicode
conn = sqlite3.connect(str(database))
else:
conn = sqlite3.connect(database.path)
conn.row_factory = sqlite3.Row
rows = conn.execute(
'''
SELECT name FROM opened_files
''')
files = set(Path(r[0]) for r in rows)
for n in ('dir1/file', 'dir2/file', 'dir2/brokensymlink', 'dir2/symlink'):
if (Path.cwd() / n) not in files:
raise AssertionError("Missing file: %s" % (Path.cwd() / n))
conn.close()
# Check that created files won't be packed
for f in config.get('other_files'):
if 'dir2' in Path(f).parent.components:
raise AssertionError("Created file shouldn't be packed: %s" %
Path(f))
# ########################################
# Test shebang corner-cases
#
Path('a').symlink('b')
with Path('b').open('w') as fp:
fp.write('#!%s 0\nsome content\n' % (Path.cwd() / 'c'))
Path('b').chmod(0o744)
Path('c').symlink('d')
with Path('d').open('w') as fp:
fp.write('#!e')
Path('d').chmod(0o744)
with Path('e').open('w') as fp:
fp.write('#!/bin/echo')
Path('e').chmod(0o744)
# Trace
out = check_output(rpz + ['trace', '--dont-identify-packages',
'-d', 'shebang-trace', './a', '1', '2'])
out = out.splitlines()[0]
assert out == ('e %s 0 ./a 1 2' % (Path.cwd() / 'c')).encode('ascii')
# Check config
with Path('shebang-trace/config.yml').open(encoding='utf-8') as fp:
config = yaml.safe_load(fp)
other_files = set(Path(f) for f in config['other_files']
if f.startswith('%s/' % Path.cwd()))
# Check database
database = Path.cwd() / 'shebang-trace/trace.sqlite3'
if PY3:
# On PY3, connect() only accepts unicode
conn = sqlite3.connect(str(database))
else:
conn = sqlite3.connect(database.path)
conn.row_factory = sqlite3.Row
rows = conn.execute(
'''
SELECT name FROM opened_files
''')
opened = [Path(r[0]) for r in rows
if r[0].startswith('%s/' % Path.cwd())]
rows = conn.execute(
'''
SELECT name, argv FROM executed_files
''')
executed = [(Path(r[0]), r[1]) for r in rows
if Path(r[0]).lies_under(Path.cwd())]
print("other_files: %r" % sorted(other_files))
print("opened: %r" % opened)
print("executed: %r" % executed)
assert other_files == set(Path.cwd() / p
for p in ('a', 'b', 'c', 'd', 'e'))
if not bug13676:
assert opened == [Path.cwd() / 'c', Path.cwd() / 'e']
assert executed == [(Path.cwd() / 'a', './a\x001\x002\x00')]
# ########################################
# Test old packages
#
old_packages = [
('simple-0.4.0.rpz',
'https://drive.google.com/uc?export=download&id=0B3ucPz7GSthBVG4xZW1V'
'eDhXNTQ'),
('simple-0.6.0.rpz',
'https://drive.google.com/uc?export=download&id=0B3ucPz7GSthBbl9SUjhr'
'cUdtbGs'),
('simple-0.7.1.rpz',
'https://drive.google.com/uc?export=download&id=0B3ucPz7GSthBRGp2Vm5V'
'QVpWOGs'),
]
for name, url in old_packages:
print("Testing old package %s" % name)
f = Path(name)
if not f.exists():
download_file(url, f)
# Info
check_call(rpuz + ['info', name])
# Show files
check_call(rpuz + ['showfiles', name])
# Lists packages
check_call(rpuz + ['installpkgs', '--summary', name])
# Unpack directory
check_call(rpuz + ['directory', 'setup', name, 'simpledir'])
# Run directory
check_simple(rpuz + ['directory', 'run', 'simpledir'], 'err')
output_in_dir = Path('simpledir/root/tmp')
output_in_dir = output_in_dir.listdir('reprozip_*')[0]
output_in_dir = output_in_dir / 'simple_output.txt'
with output_in_dir.open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Delete with wrong command (should fail)
p = subprocess.Popen(rpuz + ['chroot', 'destroy', 'simpledir'],
stderr=subprocess.PIPE)
out, err = p.communicate()
assert p.poll() != 0
err = err.splitlines()
assert b"Wrong unpacker used" in err[0]
assert err[1].startswith(b"usage: ")
# Delete directory
check_call(rpuz + ['directory', 'destroy', 'simpledir'])
# ########################################
# Copies back coverage report
#
coverage = Path('.coverage')
if coverage.exists():
coverage.copyfile(tests.parent / '.coverage.runpy')
def functional_tests(raise_warnings, interactive, run_vagrant, run_docker):
python = [sys.executable]
# Can't match on the SignalWarning category here because of a Python bug
# http://bugs.python.org/issue22543
python.extend(['-W', 'error:signal'])
if 'COVER' in os.environ:
python.extend(['-m'] + os.environ['COVER'].split(' '))
reprozip_main = tests.parent / 'reprozip/reprozip/main.py'
reprounzip_main = tests.parent / 'reprounzip/reprounzip/main.py'
verbose = ['-v'] * 3
rpz = python + [reprozip_main.absolute().path] + verbose
rpuz = python + [reprounzip_main.absolute().path] + verbose
# ########################################
# 'simple' program: trace, pack, info, unpack
#
# Build
build('simple', ['simple.c'])
# Trace
check_call(rpz + [
'trace', '-d', 'rpz-simple', './simple',
(tests / 'simple_input.txt').path, 'simple_output.txt'
])
orig_output_location = Path('simple_output.txt').absolute()
assert orig_output_location.is_file()
with orig_output_location.open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
orig_output_location.remove()
# Read config
with Path('rpz-simple/config.yml').open(encoding='utf-8') as fp:
conf = yaml.safe_load(fp)
other_files = set(Path(f).absolute() for f in conf['other_files'])
expected = [Path('simple'), (tests / 'simple_input.txt')]
assert other_files.issuperset([f.resolve() for f in expected])
# Check input and output files
input_files = conf['runs'][0]['input_files']
assert (dict((k, Path(f).name) for k, f in iteritems(input_files)) == {
'arg': b'simple_input.txt'
})
output_files = conf['runs'][0]['output_files']
print(dict((k, Path(f).name) for k, f in iteritems(output_files)))
# Here we don't test for dict equality, since we might have C coverage
# files in the mix
assert Path(output_files['arg']).name == b'simple_output.txt'
# Pack
check_call(rpz + ['pack', '-d', 'rpz-simple', 'simple.rpz'])
Path('simple').remove()
# Info
check_call(rpuz + ['info', 'simple.rpz'])
# Show files
check_call(rpuz + ['showfiles', 'simple.rpz'])
# Lists packages
check_call(rpuz + ['installpkgs', '--summary', 'simple.rpz'])
# Unpack directory
check_call(rpuz + ['directory', 'setup', 'simple.rpz', 'simpledir'])
# Run directory
check_call(rpuz + ['directory', 'run', 'simpledir'])
output_in_dir = join_root(Path('simpledir/root'), orig_output_location)
with output_in_dir.open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Delete with wrong command (should fail)
assert call(rpuz + ['chroot', 'destroy', 'simpledir']) != 0
# Delete directory
check_call(rpuz + ['directory', 'destroy', 'simpledir'])
# Unpack chroot
check_call(
['sudo'] + rpuz +
['chroot', 'setup', '--bind-magic-dirs', 'simple.rpz', 'simplechroot'])
# Run chroot
check_call(['sudo'] + rpuz + ['chroot', 'run', 'simplechroot'])
output_in_chroot = join_root(Path('simplechroot/root'),
orig_output_location)
with output_in_chroot.open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Get output file
check_call(['sudo'] + rpuz +
['chroot', 'download', 'simplechroot', 'arg:output1.txt'])
with Path('output1.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Replace input file
check_call(['sudo'] + rpuz + [
'chroot', 'upload', 'simplechroot',
'%s:arg' % (tests / 'simple_input2.txt')
])
check_call(['sudo'] + rpuz + ['chroot', 'upload', 'simplechroot'])
# Run again
check_call(['sudo'] + rpuz + ['chroot', 'run', 'simplechroot'])
output_in_chroot = join_root(Path('simplechroot/root'),
orig_output_location)
with output_in_chroot.open(encoding='utf-8') as fp:
assert fp.read().strip() == '36'
# Delete with wrong command (should fail)
assert call(rpuz + ['directory', 'destroy', 'simplechroot']) != 0
# Delete chroot
check_call(['sudo'] + rpuz + ['chroot', 'destroy', 'simplechroot'])
if not Path('/vagrant').exists():
check_call(['sudo', 'sh', '-c', 'mkdir /vagrant; chmod 777 /vagrant'])
# Unpack Vagrant-chroot
check_call(rpuz + [
'vagrant', 'setup/create', '--use-chroot', 'simple.rpz',
'/vagrant/simplevagrantchroot'
])
print("\nVagrant project set up in simplevagrantchroot")
try:
if run_vagrant:
check_call(rpuz + [
'vagrant', 'run', '--no-stdin', '/vagrant/simplevagrantchroot'
])
# Destroy
check_call(rpuz +
['vagrant', 'destroy', '/vagrant/simplevagrantchroot'])
elif interactive:
print("Test and press enter")
sys.stdin.readline()
finally:
if Path('/vagrant/simplevagrantchroot').exists():
Path('/vagrant/simplevagrantchroot').rmtree()
# Unpack Vagrant without chroot
check_call(rpuz + [
'vagrant', 'setup/create', '--dont-use-chroot', 'simple.rpz',
'/vagrant/simplevagrant'
])
print("\nVagrant project set up in simplevagrant")
try:
if run_vagrant:
check_call(
rpuz +
['vagrant', 'run', '--no-stdin', '/vagrant/simplevagrant'])
# Destroy
check_call(rpuz + ['vagrant', 'destroy', '/vagrant/simplevagrant'])
elif interactive:
print("Test and press enter")
sys.stdin.readline()
finally:
if Path('/vagrant/simplevagrant').exists():
Path('/vagrant/simplevagrant').rmtree()
# Unpack Docker
check_call(rpuz + ['docker', 'setup/create', 'simple.rpz', 'simpledocker'])
print("\nDocker project set up in simpledocker")
try:
if run_docker:
check_call(rpuz + ['docker', 'setup/build', 'simpledocker'])
check_call(rpuz + ['docker', 'run', 'simpledocker'])
# Get output file
check_call(
rpuz +
['docker', 'download', 'simpledocker', 'arg:doutput1.txt'])
with Path('doutput1.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Replace input file
check_call(rpuz + [
'docker', 'upload', 'simpledocker',
'%s:arg' % (tests / 'simple_input2.txt')
])
check_call(rpuz + ['docker', 'upload', 'simpledocker'])
check_call(rpuz + ['showfiles', 'simpledocker'])
# Run again
check_call(rpuz + ['docker', 'run', 'simpledocker'])
# Get output file
check_call(
rpuz +
['docker', 'download', 'simpledocker', 'arg:doutput2.txt'])
with Path('doutput2.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '36'
# Destroy
check_call(rpuz + ['docker', 'destroy', 'simpledocker'])
elif interactive:
print("Test and press enter")
sys.stdin.readline()
finally:
if Path('simpledocker').exists():
Path('simpledocker').rmtree()
# ########################################
# 'threads' program: testrun
#
# Build
build('threads', ['threads.c'], ['-lpthread'])
# Trace
check_call(rpz + ['testrun', './threads'])
# ########################################
# 'segv' program: testrun
#
# Build
build('segv', ['segv.c'])
# Trace
check_call(rpz + ['testrun', './segv'])
# ########################################
# 'exec_echo' program: trace, pack, run --cmdline
#
# Build
build('exec_echo', ['exec_echo.c'])
# Trace
check_call(rpz + ['trace', './exec_echo', 'originalexecechooutput'])
# Pack
check_call(rpz + ['pack', 'exec_echo.rpz'])
# Unpack chroot
check_call(['sudo'] + rpuz +
['chroot', 'setup', 'exec_echo.rpz', 'echochroot'])
try:
# Run original command-line
output = check_output(['sudo'] + rpuz +
['chroot', 'run', 'echochroot'])
assert output == b'originalexecechooutput\n'
# Prints out command-line
output = check_output(['sudo'] + rpuz +
['chroot', 'run', 'echochroot', '--cmdline'])
assert any(b'./exec_echo originalexecechooutput' == s.strip()
for s in output.split(b'\n'))
# Run with different command-line
output = check_output(['sudo'] + rpuz + [
'chroot', 'run', 'echochroot', '--cmdline', './exec_echo',
'changedexecechooutput'
])
assert output == b'changedexecechooutput\n'
finally:
check_call(['sudo'] + rpuz + ['chroot', 'destroy', 'echochroot'])
# ########################################
# 'exec_echo' program: testrun
# This is built with -m32 so that we transition:
# python (x64) -> exec_echo (i386) -> echo (x64)
#
if sys.maxsize > 2**32:
# Build
build('exec_echo32', ['exec_echo.c'], ['-m32'])
# Trace
check_call(rpz + ['testrun', './exec_echo32 42'])
else:
print("Can't try exec_echo transitions: not running on 64bits")
# ########################################
# Tracing non-existing program
#
check_call(rpz + ['testrun', './doesntexist'])
# ########################################
# 'connect' program: testrun
#
# Build
build('connect', ['connect.c'])
# Trace
stderr = check_errout(rpz + ['testrun', './connect'])
stderr = stderr.split(b'\n')
assert not any(b'program exited with non-zero code' in l for l in stderr)
assert any(
re.search(br'process connected to [0-9.]+:80', l) for l in stderr)
# ########################################
# Copies back coverage report
#
coverage = Path('.coverage')
if coverage.exists():
coverage.copyfile(tests.parent / '.coverage.runpy')
def trace(binary, argv, directory, append, verbosity='unset'):
"""Main function for the trace subcommand.
"""
if verbosity != 'unset':
warnings.warn("The 'verbosity' parameter for trace() is deprecated. "
"Please set a level on the 'reprozip' logger instead.",
DeprecationWarning)
if not isinstance(directory, Path):
directory = Path(directory)
if isinstance(binary, Path):
binary = binary.path
cwd = Path.cwd()
if (any(cwd.lies_under(c) for c in magic_dirs + system_dirs) and
not cwd.lies_under('/usr/local')):
logger.warning(
"You are running this experiment from a system directory! "
"Autodetection of non-system files will probably not work as "
"intended")
# Trace directory
if directory.exists():
if append is None:
r = tty_prompt(
"Trace directory %s exists\n"
"(a)ppend run to the trace, (d)elete it or (s)top? [a/d/s] " %
directory,
'aAdDsS')
if r is None:
logger.critical(
"Trace directory %s exists\n"
"Please use either --continue or --overwrite\n",
directory)
sys.exit(125)
elif r in 'sS':
sys.exit(125)
elif r in 'dD':
directory.rmtree()
directory.mkdir()
logger.warning(
"You can use --overwrite to replace the existing trace "
"(or --continue to append\nwithout prompt)")
elif append is False:
logger.info("Removing existing trace directory %s", directory)
directory.rmtree()
directory.mkdir(parents=True)
else:
if append is True:
logger.warning("--continue was set but trace doesn't exist yet")
directory.mkdir()
# Runs the trace
database = directory / 'trace.sqlite3'
logger.info("Running program")
# Might raise _pytracer.Error
c = _pytracer.execute(binary, argv, database.path)
if c != 0:
if c & 0x0100:
logger.warning("Program appears to have been terminated by "
"signal %d", c & 0xFF)
else:
logger.warning("Program exited with non-zero code %d", c)
logger.info("Program completed")
return c
def docker_setup_create(args):
"""Sets up the experiment to be run in a Docker-built container.
"""
pack = Path(args.pack[0])
target = Path(args.target[0])
if target.exists():
logger.critical("Target directory exists")
sys.exit(1)
signals.pre_setup(target=target, pack=pack)
target.mkdir()
try:
# Unpacks configuration file
rpz_pack = RPZPack(pack)
rpz_pack.extract_config(target / 'config.yml')
# Loads config
runs, packages, other_files = config = load_config(
target / 'config.yml', True)
if args.base_image:
record_usage(docker_explicit_base=True)
base_image = args.base_image[0]
if args.distribution:
target_distribution = args.distribution[0]
else:
target_distribution = None
else:
target_distribution, base_image = select_image(runs)
logger.info("Using base image %s", base_image)
logger.debug("Distribution: %s", target_distribution or "unknown")
rpz_pack.copy_data_tar(target / 'data.tgz')
arch = runs[0]['architecture']
# Writes Dockerfile
logger.info("Writing %s...", target / 'Dockerfile')
with (target / 'Dockerfile').open('w', encoding='utf-8',
newline='\n') as fp:
fp.write('FROM %s\n\n' % base_image)
# Installs busybox
download_file(busybox_url(arch),
target / 'busybox',
'busybox-%s' % arch)
fp.write('COPY busybox /busybox\n')
# Installs rpzsudo
download_file(sudo_url(arch),
target / 'rpzsudo',
'rpzsudo-%s' % arch)
fp.write('COPY rpzsudo /rpzsudo\n\n')
fp.write('COPY data.tgz /reprozip_data.tgz\n\n')
fp.write('COPY rpz-files.list /rpz-files.list\n')
fp.write('RUN \\\n'
' chmod +x /busybox /rpzsudo && \\\n')
if args.install_pkgs:
# Install every package through package manager
missing_packages = []
else:
# Only install packages that were not packed
missing_packages = [pkg for pkg in packages if pkg.packfiles]
packages = [pkg for pkg in packages if not pkg.packfiles]
if packages:
record_usage(docker_install_pkgs=True)
try:
installer = select_installer(pack, runs,
target_distribution)
except CantFindInstaller as e:
logger.error("Need to install %d packages but couldn't "
"select a package installer: %s",
len(packages), e)
sys.exit(1)
# Updates package sources
update_script = installer.update_script()
if update_script:
fp.write(' %s && \\\n' % update_script)
# Installs necessary packages
fp.write(' %s && \\\n' % installer.install_script(packages))
logger.info("Dockerfile will install the %d software "
"packages that were not packed", len(packages))
else:
record_usage(docker_install_pkgs=False)
# Untar
paths = set()
pathlist = []
# Add intermediate directories, and check for existence in the tar
logger.info("Generating file list...")
missing_files = chain.from_iterable(pkg.files
for pkg in missing_packages)
data_files = rpz_pack.data_filenames()
listoffiles = list(chain(other_files, missing_files))
for f in listoffiles:
if f.path.name == 'resolv.conf' and (
f.path.lies_under('/etc') or
f.path.lies_under('/run') or
f.path.lies_under('/var')):
continue
path = PosixPath('/')
for c in rpz_pack.remove_data_prefix(f.path).components:
path = path / c
if path in paths:
continue
paths.add(path)
if path in data_files:
pathlist.append(path)
else:
logger.info("Missing file %s", path)
rpz_pack.close()
# FIXME : for some reason we need reversed() here, I'm not sure why
# Need to read more of tar's docs.
# TAR bug: --no-overwrite-dir removes --keep-old-files
with (target / 'rpz-files.list').open('wb') as lfp:
for p in reversed(pathlist):
lfp.write(join_root(rpz_pack.data_prefix, p).path)
lfp.write(b'\0')
fp.write(' cd / && '
'(tar zpxf /reprozip_data.tgz -U --recursive-unlink '
'--numeric-owner --strip=1 --null -T /rpz-files.list || '
'/busybox echo "TAR reports errors, this might or might '
'not prevent the execution to run")\n')
# Setup entry point
fp.write('COPY rpz_entrypoint.sh /rpz_entrypoint.sh\n'
'ENTRYPOINT ["/busybox", "sh", "/rpz_entrypoint.sh"]\n')
# Write entry point script
logger.info("Writing %s...", target / 'rpz_entrypoint.sh')
with (target / 'rpz_entrypoint.sh').open('w', encoding='utf-8',
newline='\n') as fp:
# The entrypoint gets some arguments from the run command
# By default, it just does all the runs
# "run N" executes the run with that number
# "cmd STR" sets a replacement command-line for the next run
# "do STR" executes a command as-is
fp.write(
'#!/bin/sh\n'
'\n'
'COMMAND=\n'
'ENVVARS=\n'
'\n'
'if [ $# = 0 ]; then\n'
' exec /busybox sh /rpz_entrypoint.sh')
for nb in irange(len(runs)):
fp.write(' run %d' % nb)
fp.write(
'\n'
'fi\n'
'\n'
'while [ $# != 0 ]; do\n'
' case "$1" in\n'
' help)\n'
' echo "Image built from reprounzip-docker" >&2\n'
' echo "Usage: docker run <image> [cmd "word [word '
'...]"] [run <R>]" >&2\n'
' echo " \\`cmd ...\\` changes the command for '
'the next \\`run\\` option" >&2\n'
' echo " \\`run <name|number>\\` runs the '
'specified run" >&2\n'
' echo "By default, all the runs are executed." '
'>&2\n'
' echo "The runs in this image are:" >&2\n')
for run in runs:
fp.write(
' echo " {name}: {cmdline}" >&2\n'.format(
name=run['id'],
cmdline=' '.join(shell_escape(a)
for a in run['argv'])))
fp.write(
' exit 0\n'
' ;;\n'
' do)\n'
' shift\n'
' $1\n'
' ;;\n'
' env)\n'
' shift\n'
' ENVVARS="$1"\n'
' ;;\n'
' cmd)\n'
' shift\n'
' COMMAND="$1"\n'
' ;;\n'
' run)\n'
' shift\n'
' case "$1" in\n')
for i, run in enumerate(runs):
cmdline = ' '.join([run['binary']] + run['argv'][1:])
fp.write(
' {name})\n'
' RUNCOMMAND={cmd}\n'
' RUNWD={wd}\n'
' RUNENV={env}\n'
' RUNUID={uid}\n'
' RUNGID={gid}\n'
' ;;\n'.format(
name='%s|%d' % (run['id'], i),
cmd=shell_escape(cmdline),
wd=shell_escape(run['workingdir']),
env=shell_escape(' '.join(
'%s=%s' % (shell_escape(k), shell_escape(v))
for k, v in iteritems(run['environ']))),
uid=run.get('uid', 1000),
gid=run.get('gid', 1000)))
fp.write(
' *)\n'
' echo "RPZ: Unknown run $1" >&2\n'
' exit 1\n'
' ;;\n'
' esac\n'
' if [ -n "$COMMAND" ]; then\n'
' RUNCOMMAND="$COMMAND"\n'
' COMMAND=\n'
' fi\n'
' export RUNWD; export RUNENV; export ENVVARS; '
'export RUNCOMMAND\n'
' /rpzsudo "#$RUNUID" "#$RUNGID" /busybox sh -c '
'"cd \\"\\$RUNWD\\" && /busybox env -i $RUNENV $ENVVARS '
'$RUNCOMMAND; echo \\"*** Command finished, status: \\$?\\""\n'
' ENVVARS=\n'
' ;;\n'
' *)\n'
' echo "RPZ: Unknown option $1" >&2\n'
' exit 1\n'
' ;;\n'
' esac\n'
' shift\n'
'done\n')
# Meta-data for reprounzip
write_dict(target, metadata_initial_iofiles(config))
signals.post_setup(target=target, pack=pack)
except Exception:
target.rmtree(ignore_errors=True)
raise
def showfiles(args):
"""Writes out the input and output files.
Works both for a pack file and for an extracted directory.
"""
def parse_run(runs, s):
for i, run in enumerate(runs):
if run['id'] == s:
return i
try:
r = int(s)
except ValueError:
logging.critical("Error: Unknown run %s", s)
raise UsageError
if r < 0 or r >= len(runs):
logging.critical("Error: Expected 0 <= run <= %d, got %d",
len(runs) - 1, r)
sys.exit(1)
return r
show_inputs = args.input or not args.output
show_outputs = args.output or not args.input
def file_filter(fio):
if file_filter.run is None:
return ((show_inputs and fio.read_runs) or
(show_outputs and fio.write_runs))
else:
return ((show_inputs and file_filter.run in fio.read_runs) or
(show_outputs and file_filter.run in fio.write_runs))
file_filter.run = None
pack = Path(args.pack[0])
if not pack.exists():
logging.critical("Pack or directory %s does not exist", pack)
sys.exit(1)
if pack.is_dir():
# Reads info from an unpacked directory
config = load_config_file(pack / 'config.yml',
canonical=True)
# Filter files by run
if args.run is not None:
file_filter.run = parse_run(config.runs, args.run)
# The '.reprounzip' file is a pickled dictionary, it contains the name
# of the files that replaced each input file (if upload was used)
unpacked_info = metadata_read(pack, None)
assigned_input_files = unpacked_info.get('input_files', {})
if show_inputs:
shown = False
for input_name, f in sorted(iteritems(config.inputs_outputs)):
if f.read_runs and file_filter(f):
if not shown:
print("Input files:")
shown = True
if args.verbosity >= 2:
print(" %s (%s)" % (input_name, f.path))
else:
print(" %s" % input_name)
assigned = assigned_input_files.get(input_name)
if assigned is None:
assigned = "(original)"
elif assigned is False:
assigned = "(not created)"
elif assigned is True:
assigned = "(generated)"
else:
assert isinstance(assigned, (bytes, unicode_))
print(" %s" % assigned)
if not shown:
print("Input files: none")
if show_outputs:
shown = False
for output_name, f in sorted(iteritems(config.inputs_outputs)):
if f.write_runs and file_filter(f):
if not shown:
print("Output files:")
shown = True
if args.verbosity >= 2:
print(" %s (%s)" % (output_name, f.path))
else:
print(" %s" % output_name)
if not shown:
print("Output files: none")
else: # pack.is_file()
# Reads info from a pack file
config = load_config(pack)
# Filter files by run
if args.run is not None:
file_filter.run = parse_run(config.runs, args.run)
if any(f.read_runs for f in itervalues(config.inputs_outputs)):
print("Input files:")
for input_name, f in sorted(iteritems(config.inputs_outputs)):
if f.read_runs and file_filter(f):
if args.verbosity >= 2:
print(" %s (%s)" % (input_name, f.path))
else:
print(" %s" % input_name)
else:
print("Input files: none")
if any(f.write_runs for f in itervalues(config.inputs_outputs)):
print("Output files:")
for output_name, f in sorted(iteritems(config.inputs_outputs)):
if f.write_runs and file_filter(f):
if args.verbosity >= 2:
print(" %s (%s)" % (output_name, f.path))
else:
print(" %s" % output_name)
else:
print("Output files: none")
def run(self, files):
reprounzip.common.record_usage(upload_files=len(files))
runs = self.get_runs_from_config()
# No argument: list all the input files and exit
if not files:
print("Input files:")
for i, run in enumerate(runs):
if len(runs) > 1:
print(" Run %d:" % i)
for input_name in run['input_files']:
if self.input_files.get(input_name) is not None:
assigned = PosixPath(self.input_files[input_name])
else:
assigned = "(original)"
print(" %s: %s" % (input_name, assigned))
return
self.prepare_upload(files)
# Get the path of each input file
all_input_files = {}
for run in runs:
all_input_files.update(run['input_files'])
try:
# Upload files
for filespec in files:
filespec_split = filespec.rsplit(':', 1)
if len(filespec_split) != 2:
logging.critical("Invalid file specification: %r",
filespec)
sys.exit(1)
local_path, input_name = filespec_split
try:
input_path = PosixPath(all_input_files[input_name])
except KeyError:
logging.critical("Invalid input file: %r", input_name)
sys.exit(1)
temp = None
if not local_path:
# Restore original file from pack
logging.debug("Restoring input file %s", input_path)
fd, temp = Path.tempfile(prefix='reprozip_input_')
os.close(fd)
local_path = self.extract_original_input(
input_name, input_path, temp)
else:
local_path = Path(local_path)
logging.debug("Uploading file %s to %s", local_path,
input_path)
if not local_path.exists():
logging.critical("Local file %s doesn't exist",
local_path)
sys.exit(1)
self.upload_file(local_path, input_path)
if temp is not None:
temp.remove()
self.input_files.pop(input_name, None)
else:
self.input_files[input_name] = local_path.absolute().path
finally:
self.finalize()
def run(self, files):
reprounzip.common.record_usage(upload_files=len(files))
input_files = dict(
(n, f.path)
for n, f in iteritems(self.get_config().inputs_outputs)
if f.read_runs)
# No argument: list all the input files and exit
if not files:
print("Input files:")
for input_name in input_files:
if self.input_files.get(input_name) is not None:
assigned = self.input_files[input_name]
else:
assigned = "(original)"
print(" %s: %s" % (input_name, assigned))
return
self.prepare_upload(files)
try:
# Upload files
for filespec in files:
filespec_split = filespec.rsplit(':', 1)
if len(filespec_split) != 2:
logging.critical("Invalid file specification: %r",
filespec)
sys.exit(1)
local_path, input_name = filespec_split
try:
input_path = input_files[input_name]
except KeyError:
logging.critical("Invalid input file: %r", input_name)
sys.exit(1)
temp = None
if not local_path:
# Restore original file from pack
logging.debug("Restoring input file %s", input_path)
fd, temp = Path.tempfile(prefix='reprozip_input_')
os.close(fd)
local_path = self.extract_original_input(input_name,
input_path,
temp)
if local_path is None:
temp.remove()
logging.warning("No original packed, can't restore "
"input file %s", input_name)
continue
else:
local_path = Path(local_path)
logging.debug("Uploading file %s to %s",
local_path, input_path)
if not local_path.exists():
logging.critical("Local file %s doesn't exist",
local_path)
sys.exit(1)
self.upload_file(local_path, input_path)
if temp is not None:
temp.remove()
self.input_files.pop(input_name, None)
else:
self.input_files[input_name] = local_path.absolute().path
finally:
self.finalize()
def chroot_create(args):
"""Unpacks the experiment in a folder so it can be run with chroot.
All the files in the pack are unpacked; system files are copied only if
they were not packed, and busybox is installed if /bin/sh wasn't packed.
In addition, input files are put in a tar.gz (so they can be put back after
an upload) and the configuration file is extracted.
"""
if not args.pack:
logging.critical("setup/create needs the pack filename")
sys.exit(1)
pack = Path(args.pack[0])
target = Path(args.target[0])
if target.exists():
logging.critical("Target directory exists")
sys.exit(1)
if DefaultAbstractPath is not PosixPath:
logging.critical("Not unpacking on POSIX system")
sys.exit(1)
signals.pre_setup(target=target, pack=pack)
# We can only restore owner/group of files if running as root
restore_owner = should_restore_owner(args.restore_owner)
# Unpacks configuration file
rpz_pack = RPZPack(pack)
rpz_pack.extract_config(target / 'config.yml')
# Loads config
config = load_config_file(target / 'config.yml', True)
packages = config.packages
target.mkdir()
root = (target / 'root').absolute()
root.mkdir()
try:
# Checks that everything was packed
packages_not_packed = [pkg for pkg in packages if not pkg.packfiles]
if packages_not_packed:
record_usage(chroot_missing_pkgs=True)
logging.warning("According to configuration, some files were left "
"out because they belong to the following "
"packages:%s\nWill copy files from HOST SYSTEM",
''.join('\n %s' % pkg
for pkg in packages_not_packed))
missing_files = False
for pkg in packages_not_packed:
for f in pkg.files:
path = Path(f.path)
if not path.exists():
logging.error(
"Missing file %s (from package %s) on host, "
"experiment will probably miss it",
path, pkg.name)
missing_files = True
continue
dest = join_root(root, path)
dest.parent.mkdir(parents=True)
if path.is_link():
dest.symlink(path.read_link())
else:
path.copy(dest)
if restore_owner:
stat = path.stat()
dest.chown(stat.st_uid, stat.st_gid)
if missing_files:
record_usage(chroot_mising_files=True)
# Unpacks files
members = rpz_pack.list_data()
for m in members:
# Remove 'DATA/' prefix
m.name = str(rpz_pack.remove_data_prefix(m.name))
if not restore_owner:
uid = os.getuid()
gid = os.getgid()
for m in members:
m.uid = uid
m.gid = gid
logging.info("Extracting files...")
rpz_pack.extract_data(root, members)
rpz_pack.close()
# Sets up /bin/sh and /usr/bin/env, downloading busybox if necessary
sh_path = join_root(root, Path('/bin/sh'))
env_path = join_root(root, Path('/usr/bin/env'))
if not sh_path.lexists() or not env_path.lexists():
logging.info("Setting up busybox...")
busybox_path = join_root(root, Path('/bin/busybox'))
busybox_path.parent.mkdir(parents=True)
with make_dir_writable(join_root(root, Path('/bin'))):
download_file(busybox_url(config.runs[0]['architecture']),
busybox_path,
'busybox-%s' % config.runs[0]['architecture'])
busybox_path.chmod(0o755)
if not sh_path.lexists():
sh_path.parent.mkdir(parents=True)
sh_path.symlink('/bin/busybox')
if not env_path.lexists():
env_path.parent.mkdir(parents=True)
env_path.symlink('/bin/busybox')
# Original input files, so upload can restore them
input_files = [f.path for f in itervalues(config.inputs_outputs)
if f.read_runs]
if input_files:
logging.info("Packing up original input files...")
inputtar = tarfile.open(str(target / 'inputs.tar.gz'), 'w:gz')
for ifile in input_files:
filename = join_root(root, ifile)
if filename.exists():
inputtar.add(str(filename), str(ifile))
inputtar.close()
# Meta-data for reprounzip
metadata_write(target, metadata_initial_iofiles(config), 'chroot')
signals.post_setup(target=target, pack=pack)
except Exception:
rmtree_fixed(root)
raise
def run(self, files):
runs = self.get_runs_from_config()
# No argument: list all the input files and exit
if not files:
print("Input files:")
for i, run in enumerate(runs):
if len(runs) > 1:
print(" Run %d:" % i)
for input_name in run['input_files']:
if self.input_files.get(input_name) is not None:
assigned = PosixPath(self.input_files[input_name])
else:
assigned = "(original)"
print(" %s: %s" % (input_name, assigned))
return
self.prepare_upload(files)
# Get the path of each input file
all_input_files = {}
for run in runs:
all_input_files.update(run['input_files'])
try:
# Upload files
for filespec in files:
filespec_split = filespec.rsplit(':', 1)
if len(filespec_split) != 2:
logging.critical("Invalid file specification: %r",
filespec)
sys.exit(1)
local_path, input_name = filespec_split
try:
input_path = PosixPath(all_input_files[input_name])
except KeyError:
logging.critical("Invalid input file: %r", input_name)
sys.exit(1)
temp = None
if not local_path:
# Restore original file from pack
logging.debug("Restoring input file %s", input_path)
fd, temp = Path.tempfile(prefix='reprozip_input_')
os.close(fd)
local_path = self.extract_original_input(input_name,
input_path,
temp)
else:
local_path = Path(local_path)
logging.debug("Uploading file %s to %s",
local_path, input_path)
if not local_path.exists():
logging.critical("Local file %s doesn't exist",
local_path)
sys.exit(1)
self.upload_file(local_path, input_path)
if temp is not None:
temp.remove()
del self.input_files[input_name]
else:
self.input_files[input_name] = local_path.absolute().path
finally:
self.finalize()
def run(self, files):
reprounzip.common.record_usage(upload_files=len(files))
inputs_outputs = self.get_config().inputs_outputs
# No argument: list all the input files and exit
if not files:
print("Input files:")
for input_name in sorted(n for n, f in iteritems(inputs_outputs)
if f.read_runs):
assigned = self.input_files.get(input_name)
if assigned is None:
assigned = "(original)"
elif assigned is False:
assigned = "(not created)"
elif assigned is True:
assigned = "(generated)"
else:
assert isinstance(assigned, (bytes, unicode_))
print(" %s: %s" % (input_name, assigned))
return
self.prepare_upload(files)
try:
# Upload files
for filespec in files:
filespec_split = filespec.rsplit(':', 1)
if len(filespec_split) != 2:
logging.critical("Invalid file specification: %r",
filespec)
sys.exit(1)
local_path, input_name = filespec_split
if input_name.startswith('/'):
input_path = PosixPath(input_name)
else:
try:
input_path = inputs_outputs[input_name].path
except KeyError:
logging.critical("Invalid input file: %r", input_name)
sys.exit(1)
temp = None
if not local_path:
# Restore original file from pack
logging.debug("Restoring input file %s", input_path)
fd, temp = Path.tempfile(prefix='reprozip_input_')
os.close(fd)
local_path = self.extract_original_input(
input_name, input_path, temp)
if local_path is None:
temp.remove()
logging.warning(
"No original packed, can't restore "
"input file %s", input_name)
continue
else:
local_path = Path(local_path)
logging.debug("Uploading file %s to %s", local_path,
input_path)
if not local_path.exists():
logging.critical("Local file %s doesn't exist",
local_path)
sys.exit(1)
self.upload_file(local_path, input_path)
if temp is not None:
temp.remove()
self.input_files.pop(input_name, None)
else:
self.input_files[input_name] = local_path.absolute().path
finally:
self.finalize()
def directory_create(args):
"""Unpacks the experiment in a folder.
Only the files that are not part of a package are copied (unless they are
missing from the system and were packed).
In addition, input files are put in a tar.gz (so they can be put back after
an upload) and the configuration file is extracted.
"""
if not args.pack:
logger.critical("setup needs the pack filename")
sys.exit(1)
pack = Path(args.pack[0])
target = Path(args.target[0])
if target.exists():
logger.critical("Target directory exists")
sys.exit(1)
if not issubclass(DefaultAbstractPath, PosixPath):
logger.critical("Not unpacking on POSIX system")
sys.exit(1)
signals.pre_setup(target=target, pack=pack)
# Unpacks configuration file
rpz_pack = RPZPack(pack)
rpz_pack.extract_config(target / 'config.yml')
# Loads config
config = load_config_file(target / 'config.yml', True)
packages = config.packages
target.mkdir()
root = (target / 'root').absolute()
# Checks packages
missing_files = False
for pkg in packages:
if pkg.packfiles:
continue
for f in pkg.files:
if not Path(f.path).exists():
logger.error(
"Missing file %s (from package %s that wasn't packed) "
"on host, experiment will probably miss it.",
f, pkg.name)
missing_files = True
if missing_files:
record_usage(directory_missing_pkgs=True)
logger.error("Some packages are missing, you should probably install "
"them.\nUse 'reprounzip installpkgs -h' for help")
root.mkdir()
try:
# Unpacks files
members = rpz_pack.list_data()
for m in members:
# Remove 'DATA/' prefix
m.name = str(rpz_pack.remove_data_prefix(m.name))
# Makes symlink targets relative
if m.issym():
linkname = PosixPath(m.linkname)
if linkname.is_absolute:
m.linkname = join_root(root, PosixPath(m.linkname)).path
logger.info("Extracting files...")
rpz_pack.extract_data(root, members)
rpz_pack.close()
# Original input files, so upload can restore them
input_files = [f.path for f in config.inputs_outputs.values()
if f.read_runs]
if input_files:
logger.info("Packing up original input files...")
inputtar = tarfile.open(str(target / 'inputs.tar.gz'), 'w:gz')
for ifile in input_files:
filename = join_root(root, ifile)
if filename.exists():
inputtar.add(str(filename), str(ifile))
inputtar.close()
# Meta-data for reprounzip
metadata_write(target, metadata_initial_iofiles(config), 'directory')
signals.post_setup(target=target, pack=pack)
except Exception:
rmtree_fixed(root)
raise
def trace(binary, argv, directory, append, verbosity='unset'):
"""Main function for the trace subcommand.
"""
if verbosity != 'unset':
warnings.warn(
"The 'verbosity' parameter for trace() is deprecated. "
"Please set a level on the 'reprozip' logger instead.",
DeprecationWarning)
if not isinstance(directory, Path):
directory = Path(directory)
if isinstance(binary, Path):
binary = binary.path
cwd = Path.cwd()
if (any(cwd.lies_under(c) for c in magic_dirs + system_dirs)
and not cwd.lies_under('/usr/local')):
logger.warning(
"You are running this experiment from a system directory! "
"Autodetection of non-system files will probably not work as "
"intended")
# Trace directory
if directory.exists():
if append is None:
r = tty_prompt(
"Trace directory %s exists\n"
"(a)ppend run to the trace, (d)elete it or (s)top? [a/d/s] " %
directory, 'aAdDsS')
if r is None:
logger.critical(
"Trace directory %s exists\n"
"Please use either --continue or --overwrite\n", directory)
sys.exit(125)
elif r in 'sS':
sys.exit(125)
elif r in 'dD':
directory.rmtree()
directory.mkdir()
logger.warning(
"You can use --overwrite to replace the existing trace "
"(or --continue to append\nwithout prompt)")
elif append is False:
logger.info("Removing existing trace directory %s", directory)
directory.rmtree()
directory.mkdir(parents=True)
else:
if append is True:
logger.warning("--continue was set but trace doesn't exist yet")
directory.mkdir()
# Runs the trace
database = directory / 'trace.sqlite3'
logger.info("Running program")
# Might raise _pytracer.Error
c = _pytracer.execute(binary, argv, database.path)
if c != 0:
if c & 0x0100:
logger.warning(
"Program appears to have been terminated by "
"signal %d", c & 0xFF)
else:
logger.warning("Program exited with non-zero code %d", c)
logger.info("Program completed")
return c
def functional_tests(raise_warnings, interactive, run_vagrant, run_docker):
# Tests on Python < 2.7.3: need to use separate reprozip Python (with known
# working version of Python)
if sys.version_info < (2, 7, 3):
bug13676 = True
if 'REPROZIP_PYTHON' not in os.environ:
sys.stderr.write("Error: using reprozip with Python %s!\n" %
sys.version.split(' ', 1)[0])
sys.exit(1)
else:
bug13676 = False
rpz = [os.environ.get('REPROZIP_PYTHON', sys.executable)]
rpuz = [os.environ.get('REPROUNZIP_PYTHON', sys.executable)]
# Can't match on the SignalWarning category here because of a Python bug
# http://bugs.python.org/issue22543
if raise_warnings:
rpz.extend(['-W', 'error:signal'])
rpuz.extend(['-W', 'error:signal'])
if 'COVER' in os.environ:
rpz.extend(['-m'] + os.environ['COVER'].split(' '))
rpuz.extend(['-m'] + os.environ['COVER'].split(' '))
reprozip_main = tests.parent / 'reprozip/reprozip/main.py'
reprounzip_main = tests.parent / 'reprounzip/reprounzip/main.py'
verbose = ['-v'] * 3
rpz.extend([reprozip_main.absolute().path] + verbose)
rpuz.extend([reprounzip_main.absolute().path] + verbose)
print("Command lines are:\n%r\n%r" % (rpz, rpuz))
# ########################################
# testrun /bin/echo
#
output = check_output(rpz + ['testrun', '/bin/echo', 'outputhere'])
assert any(b' 1 | /bin/echo outputhere ' in l
for l in output.splitlines())
output = check_output(rpz + ['testrun', '-a', '/fake/path/echo',
'/bin/echo', 'outputhere'])
assert any(b' 1 | (/bin/echo) /fake/path/echo outputhere ' in l
for l in output.splitlines())
# ########################################
# testrun multiple commands
#
check_call(rpz + ['testrun', 'bash', '-c',
'cat ../../../../../etc/passwd;'
'cd /var/lib;'
'cat ../../etc/group'])
check_call(rpz + ['trace',
'bash', '-c', 'cat /etc/passwd;echo'])
check_call(rpz + ['trace', '--continue',
'sh', '-c', 'cat /etc/group;/usr/bin/id'])
check_call(rpz + ['pack'])
if not bug13676:
check_call(rpuz + ['graph', 'graph.dot'])
check_call(rpuz + ['graph', 'graph2.dot', 'experiment.rpz'])
sudo = ['sudo', '-E'] # -E to keep REPROZIP_USAGE_STATS
# ########################################
# 'simple' program: trace, pack, info, unpack
#
def check_simple(args, stream, infile=1):
output = check_output(args, stream).splitlines()
try:
first = output.index(b"Read 6 bytes")
except ValueError:
stderr.write("output = %r\n" % output)
raise
if infile == 1:
assert output[first + 1] == b"a = 29, b = 13"
assert output[first + 2] == b"result = 42"
else: # infile == 2
assert output[first + 1] == b"a = 25, b = 11"
assert output[first + 2] == b"result = 36"
# Build
build('simple', ['simple.c'])
# Trace
check_call(rpz + ['trace', '-d', 'rpz-simple',
'./simple',
(tests / 'simple_input.txt').path,
'simple_output.txt'])
orig_output_location = Path('simple_output.txt').absolute()
assert orig_output_location.is_file()
with orig_output_location.open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
orig_output_location.remove()
# Read config
with Path('rpz-simple/config.yml').open(encoding='utf-8') as fp:
conf = yaml.safe_load(fp)
other_files = set(Path(f).absolute() for f in conf['other_files'])
expected = [Path('simple'), (tests / 'simple_input.txt')]
assert other_files.issuperset([f.resolve() for f in expected])
# Check input and output files
inputs_outputs = conf['inputs_outputs']
# Exactly one input: "arg1", "...simple_input.txt"
# Output: 'arg2', "...simple_output.txt"
# There might be more output files: the C coverage files
found = 0
for fdict in inputs_outputs:
if Path(fdict['path']).name == b'simple_input.txt':
assert fdict['name'] == 'arg1'
assert fdict['read_by_runs'] == [0]
assert not fdict.get('written_by_runs')
found |= 0x01
elif Path(fdict['path']).name == b'simple_output.txt':
assert fdict['name'] == 'arg2'
assert not fdict.get('read_by_runs')
assert fdict['written_by_runs'] == [0]
found |= 0x02
else:
# No other inputs
assert not fdict.get('read_by_runs')
assert found == 0x03
# Pack
check_call(rpz + ['pack', '-d', 'rpz-simple', 'simple.rpz'])
Path('simple').remove()
# Info
check_call(rpuz + ['info', 'simple.rpz'])
# Show files
check_call(rpuz + ['showfiles', 'simple.rpz'])
# Lists packages
check_call(rpuz + ['installpkgs', '--summary', 'simple.rpz'])
# Unpack directory
check_call(rpuz + ['directory', 'setup', 'simple.rpz', 'simpledir'])
# Run directory
check_simple(rpuz + ['directory', 'run', 'simpledir'], 'err')
output_in_dir = join_root(Path('simpledir/root'), orig_output_location)
with output_in_dir.open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Delete with wrong command (should fail)
p = subprocess.Popen(rpuz + ['chroot', 'destroy', 'simpledir'],
stderr=subprocess.PIPE)
out, err = p.communicate()
assert p.poll() != 0
err = err.splitlines()
assert b"Wrong unpacker used" in err[0]
assert err[1].startswith(b"usage: ")
# Delete directory
check_call(rpuz + ['directory', 'destroy', 'simpledir'])
# Unpack chroot
check_call(sudo + rpuz + ['chroot', 'setup', '--bind-magic-dirs',
'simple.rpz', 'simplechroot'])
try:
output_in_chroot = join_root(Path('simplechroot/root'),
orig_output_location)
# Run chroot
check_simple(sudo + rpuz + ['chroot', 'run', 'simplechroot'], 'err')
with output_in_chroot.open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Get output file
check_call(sudo + rpuz + ['chroot', 'download', 'simplechroot',
'arg2:output1.txt'])
with Path('output1.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Replace input file
check_call(sudo + rpuz + ['chroot', 'upload', 'simplechroot',
'%s:arg1' % (tests / 'simple_input2.txt')])
check_call(sudo + rpuz + ['chroot', 'upload', 'simplechroot'])
# Run again
check_simple(sudo + rpuz + ['chroot', 'run', 'simplechroot'], 'err', 2)
with output_in_chroot.open(encoding='utf-8') as fp:
assert fp.read().strip() == '36'
# Reset input file
check_call(sudo + rpuz + ['chroot', 'upload', 'simplechroot', ':arg1'])
# Run again
check_simple(sudo + rpuz + ['chroot', 'run', 'simplechroot'], 'err')
with output_in_chroot.open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Delete with wrong command (should fail)
p = subprocess.Popen(rpuz + ['directory', 'destroy', 'simplechroot'],
stderr=subprocess.PIPE)
out, err = p.communicate()
assert p.poll() != 0
err = err.splitlines()
assert b"Wrong unpacker used" in err[0]
assert err[1].startswith(b"usage:")
finally:
# Delete chroot
check_call(sudo + rpuz + ['chroot', 'destroy', 'simplechroot'])
if not (tests / 'vagrant').exists():
check_call(['sudo', 'sh', '-c',
'mkdir %(d)s; chmod 777 %(d)s' % {'d': tests / 'vagrant'}])
# Unpack Vagrant-chroot
check_call(rpuz + ['vagrant', 'setup/create', '--use-chroot', 'simple.rpz',
(tests / 'vagrant/simplevagrantchroot').path])
print("\nVagrant project set up in simplevagrantchroot")
try:
if run_vagrant:
check_simple(rpuz + ['vagrant', 'run', '--no-stdin',
(tests / 'vagrant/simplevagrantchroot').path],
'out')
# Get output file
check_call(rpuz + ['vagrant', 'download',
(tests / 'vagrant/simplevagrantchroot').path,
'arg2:voutput1.txt'])
with Path('voutput1.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Replace input file
check_call(rpuz + ['vagrant', 'upload',
(tests / 'vagrant/simplevagrantchroot').path,
'%s:arg1' % (tests / 'simple_input2.txt')])
check_call(rpuz + ['vagrant', 'upload',
(tests / 'vagrant/simplevagrantchroot').path])
# Run again
check_simple(rpuz + ['vagrant', 'run', '--no-stdin',
(tests / 'vagrant/simplevagrantchroot').path],
'out', 2)
# Get output file
check_call(rpuz + ['vagrant', 'download',
(tests / 'vagrant/simplevagrantchroot').path,
'arg2:voutput2.txt'])
with Path('voutput2.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '36'
# Reset input file
check_call(rpuz + ['vagrant', 'upload',
(tests / 'vagrant/simplevagrantchroot').path,
':arg1'])
# Run again
check_simple(rpuz + ['vagrant', 'run', '--no-stdin',
(tests / 'vagrant/simplevagrantchroot').path],
'out')
# Get output file
check_call(rpuz + ['vagrant', 'download',
(tests / 'vagrant/simplevagrantchroot').path,
'arg2:voutput1.txt'])
with Path('voutput1.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Destroy
check_call(rpuz + ['vagrant', 'destroy',
(tests / 'vagrant/simplevagrantchroot').path])
elif interactive:
print("Test and press enter")
sys.stdin.readline()
finally:
if (tests / 'vagrant/simplevagrantchroot').exists():
(tests / 'vagrant/simplevagrantchroot').rmtree()
# Unpack Vagrant without chroot
check_call(rpuz + ['vagrant', 'setup/create', '--dont-use-chroot',
'simple.rpz',
(tests / 'vagrant/simplevagrant').path])
print("\nVagrant project set up in simplevagrant")
try:
if run_vagrant:
check_simple(rpuz + ['vagrant', 'run', '--no-stdin',
(tests / 'vagrant/simplevagrant').path],
'out')
# Get output file
check_call(rpuz + ['vagrant', 'download',
(tests / 'vagrant/simplevagrant').path,
'arg2:woutput1.txt'])
with Path('woutput1.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Replace input file
check_call(rpuz + ['vagrant', 'upload',
(tests / 'vagrant/simplevagrant').path,
'%s:arg1' % (tests / 'simple_input2.txt')])
check_call(rpuz + ['vagrant', 'upload',
(tests / 'vagrant/simplevagrant').path])
# Run again
check_simple(rpuz + ['vagrant', 'run', '--no-stdin',
(tests / 'vagrant/simplevagrant').path],
'out', 2)
# Get output file
check_call(rpuz + ['vagrant', 'download',
(tests / 'vagrant/simplevagrant').path,
'arg2:woutput2.txt'])
with Path('woutput2.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '36'
# Reset input file
check_call(rpuz + ['vagrant', 'upload',
(tests / 'vagrant/simplevagrant').path,
':arg1'])
# Run again
check_simple(rpuz + ['vagrant', 'run', '--no-stdin',
(tests / 'vagrant/simplevagrant').path],
'out')
# Get output file
check_call(rpuz + ['vagrant', 'download',
(tests / 'vagrant/simplevagrant').path,
'arg2:voutput1.txt'])
with Path('voutput1.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Destroy
check_call(rpuz + ['vagrant', 'destroy',
(tests / 'vagrant/simplevagrant').path])
elif interactive:
print("Test and press enter")
sys.stdin.readline()
finally:
if (tests / 'vagrant/simplevagrant').exists():
(tests / 'vagrant/simplevagrant').rmtree()
# Unpack Docker
check_call(rpuz + ['docker', 'setup/create', 'simple.rpz', 'simpledocker'])
print("\nDocker project set up in simpledocker")
try:
if run_docker:
check_call(rpuz + ['docker', 'setup/build', 'simpledocker'])
check_simple(rpuz + ['docker', 'run', 'simpledocker'], 'out')
# Get output file
check_call(rpuz + ['docker', 'download', 'simpledocker',
'arg2:doutput1.txt'])
with Path('doutput1.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Replace input file
check_call(rpuz + ['docker', 'upload', 'simpledocker',
'%s:arg1' % (tests / 'simple_input2.txt')])
check_call(rpuz + ['docker', 'upload', 'simpledocker'])
check_call(rpuz + ['showfiles', 'simpledocker'])
# Run again
check_simple(rpuz + ['docker', 'run', 'simpledocker'], 'out', 2)
# Get output file
check_call(rpuz + ['docker', 'download', 'simpledocker',
'arg2:doutput2.txt'])
with Path('doutput2.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '36'
# Reset input file
check_call(rpuz + ['docker', 'upload', 'simpledocker',
':arg1'])
# Run again
check_simple(rpuz + ['docker', 'run', 'simpledocker'], 'out')
# Get output file
check_call(rpuz + ['docker', 'download', 'simpledocker',
'arg2:doutput1.txt'])
with Path('doutput1.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Destroy
check_call(rpuz + ['docker', 'destroy', 'simpledocker'])
elif interactive:
print("Test and press enter")
sys.stdin.readline()
finally:
if Path('simpledocker').exists():
Path('simpledocker').rmtree()
# ########################################
# 'threads' program: testrun
#
# Build
build('threads', ['threads.c'], ['-lpthread'])
# Trace
output = check_output(rpz + ['testrun', './threads'], 'err')
assert any(b'successfully exec\'d /bin/./echo' in l
for l in output.splitlines())
# ########################################
# 'threads2' program: testrun
#
# Build
build('threads2', ['threads2.c'], ['-lpthread'])
# Trace
output = check_output(rpz + ['testrun', './threads2'], 'err')
assert any(b'successfully exec\'d /bin/echo' in l
for l in output.splitlines())
# ########################################
# 'segv' program: testrun
#
# Build
build('segv', ['segv.c'])
# Trace
check_call(rpz + ['testrun', './segv'])
# ########################################
# 'exec_echo' program: trace, pack, run --cmdline
#
# Build
build('exec_echo', ['exec_echo.c'])
# Trace
check_call(rpz + ['trace', './exec_echo', 'originalexecechooutput'])
# Pack
check_call(rpz + ['pack', 'exec_echo.rpz'])
# Unpack chroot
check_call(sudo + rpuz + ['chroot', 'setup',
'exec_echo.rpz', 'echochroot'])
try:
# Run original command-line
output = check_output(sudo + rpuz + ['chroot', 'run',
'echochroot'])
assert output == b'originalexecechooutput\n'
# Prints out command-line
output = check_output(sudo + rpuz + ['chroot', 'run',
'echochroot', '--cmdline'])
assert any(b'./exec_echo originalexecechooutput' == s.strip()
for s in output.split(b'\n'))
# Run with different command-line
output = check_output(sudo + rpuz + [
'chroot', 'run', 'echochroot',
'--cmdline', './exec_echo', 'changedexecechooutput'])
assert output == b'changedexecechooutput\n'
finally:
check_call(sudo + rpuz + ['chroot', 'destroy', 'echochroot'])
# ########################################
# 'exec_echo' program: testrun
# This is built with -m32 so that we transition:
# python (x64) -> exec_echo (i386) -> echo (x64)
#
if sys.maxsize > 2 ** 32:
# Build
build('exec_echo32', ['exec_echo.c'], ['-m32'])
# Trace
check_call(rpz + ['testrun', './exec_echo32 42'])
else:
print("Can't try exec_echo transitions: not running on 64bits")
# ########################################
# Tracing non-existing program
#
check_call(rpz + ['testrun', './doesntexist'])
# ########################################
# 'connect' program: testrun
#
# Build
build('connect', ['connect.c'])
# Trace
err = check_output(rpz + ['testrun', './connect'], 'err')
err = err.split(b'\n')
assert not any(b'program exited with non-zero code' in l for l in err)
assert any(re.search(br'process connected to [0-9.]+:80', l)
for l in err)
# ########################################
# 'vfork' program: testrun
#
# Build
build('vfork', ['vfork.c'])
# Trace
err = check_output(rpz + ['testrun', './vfork'], 'err')
err = err.split(b'\n')
assert not any(b'program exited with non-zero code' in l for l in err)
# ########################################
# 'rename' program: trace
#
# Build
build('rename', ['rename.c'])
# Trace
check_call(rpz + ['trace', '-d', 'rename-trace', './rename'])
with Path('rename-trace/config.yml').open(encoding='utf-8') as fp:
config = yaml.safe_load(fp)
# Check that written files were logged
database = Path.cwd() / 'rename-trace/trace.sqlite3'
if PY3:
# On PY3, connect() only accepts unicode
conn = sqlite3.connect(str(database))
else:
conn = sqlite3.connect(database.path)
conn.row_factory = sqlite3.Row
rows = conn.execute(
'''
SELECT name FROM opened_files
''')
files = set(Path(r[0]) for r in rows)
for n in ('dir1/file', 'dir2/file', 'dir2/brokensymlink', 'dir2/symlink'):
if (Path.cwd() / n) not in files:
raise AssertionError("Missing file: %s" % (Path.cwd() / n))
conn.close()
# Check that created files won't be packed
for f in config.get('other_files'):
if 'dir2' in Path(f).parent.components:
raise AssertionError("Created file shouldn't be packed: %s" %
Path(f))
# ########################################
# Copies back coverage report
#
coverage = Path('.coverage')
if coverage.exists():
coverage.copyfile(tests.parent / '.coverage.runpy')
def showfiles(args):
"""Writes out the input and output files.
Works both for a pack file and for an extracted directory.
"""
def parse_run(runs, s):
for i, run in enumerate(runs):
if run['id'] == s:
return i
try:
r = int(s)
except ValueError:
logger.critical("Error: Unknown run %s", s)
raise UsageError
if r < 0 or r >= len(runs):
logger.critical("Error: Expected 0 <= run <= %d, got %d",
len(runs) - 1, r)
sys.exit(1)
return r
show_inputs = args.input or not args.output
show_outputs = args.output or not args.input
def file_filter(fio):
if file_filter.run is None:
return ((show_inputs and fio.read_runs)
or (show_outputs and fio.write_runs))
else:
return ((show_inputs and file_filter.run in fio.read_runs)
or (show_outputs and file_filter.run in fio.write_runs))
file_filter.run = None
pack = Path(args.pack[0])
if not pack.exists():
logger.critical("Pack or directory %s does not exist", pack)
sys.exit(1)
if pack.is_dir():
# Reads info from an unpacked directory
config = load_config_file(pack / 'config.yml', canonical=True)
# Filter files by run
if args.run is not None:
file_filter.run = parse_run(config.runs, args.run)
# The '.reprounzip' file is a pickled dictionary, it contains the name
# of the files that replaced each input file (if upload was used)
unpacked_info = metadata_read(pack, None)
assigned_input_files = unpacked_info.get('input_files', {})
if show_inputs:
shown = False
for input_name, f in sorted(config.inputs_outputs.items()):
if f.read_runs and file_filter(f):
if not shown:
print("Input files:")
shown = True
if args.verbosity >= 2:
print(" %s (%s)" % (input_name, f.path))
else:
print(" %s" % input_name)
assigned = assigned_input_files.get(input_name)
if assigned is None:
assigned = "(original)"
elif assigned is False:
assigned = "(not created)"
elif assigned is True:
assigned = "(generated)"
else:
assert isinstance(assigned, (bytes, str))
print(" %s" % assigned)
if not shown:
print("Input files: none")
if show_outputs:
shown = False
for output_name, f in sorted(config.inputs_outputs.items()):
if f.write_runs and file_filter(f):
if not shown:
print("Output files:")
shown = True
if args.verbosity >= 2:
print(" %s (%s)" % (output_name, f.path))
else:
print(" %s" % output_name)
if not shown:
print("Output files: none")
else: # pack.is_file()
# Reads info from a pack file
config = load_config(pack)
# Filter files by run
if args.run is not None:
file_filter.run = parse_run(config.runs, args.run)
if any(f.read_runs for f in config.inputs_outputs.values()):
print("Input files:")
for input_name, f in sorted(config.inputs_outputs.items()):
if f.read_runs and file_filter(f):
if args.verbosity >= 2:
print(" %s (%s)" % (input_name, f.path))
else:
print(" %s" % input_name)
else:
print("Input files: none")
if any(f.write_runs for f in config.inputs_outputs.values()):
print("Output files:")
for output_name, f in sorted(config.inputs_outputs.items()):
if f.write_runs and file_filter(f):
if args.verbosity >= 2:
print(" %s (%s)" % (output_name, f.path))
else:
print(" %s" % output_name)
else:
print("Output files: none")
def vagrant_setup_create(args):
"""Sets up the experiment to be run in a Vagrant-built virtual machine.
This can either build a chroot or not.
If building a chroot, we do just like without Vagrant: we copy all the
files and only get what's missing from the host. But we do install
automatically the packages whose files are required.
If not building a chroot, we install all the packages, and only unpack
files that don't come from packages.
In short: files from packages with packfiles=True will only be used if
building a chroot.
"""
if not args.pack:
logging.critical("setup/create needs the pack filename")
sys.exit(1)
pack = Path(args.pack[0])
target = Path(args.target[0])
if target.exists():
logging.critical("Target directory exists")
sys.exit(1)
use_chroot = args.use_chroot
mount_bind = args.bind_magic_dirs
record_usage(use_chroot=use_chroot,
mount_bind=mount_bind)
signals.pre_setup(target=target, pack=pack)
# Unpacks configuration file
rpz_pack = RPZPack(pack)
rpz_pack.extract_config(target / 'config.yml')
# Loads config
runs, packages, other_files = config = load_config(target / 'config.yml',
True)
if not args.memory:
memory = None
else:
try:
memory = int(args.memory[-1])
except ValueError:
logging.critical("Invalid value for memory size: %r", args.memory)
sys.exit(1)
if args.base_image and args.base_image[0]:
record_usage(vagrant_explicit_image=True)
box = args.base_image[0]
if args.distribution:
target_distribution = args.distribution[0]
else:
target_distribution = None
else:
target_distribution, box = select_box(runs, gui=args.gui)
logging.info("Using box %s", box)
logging.debug("Distribution: %s", target_distribution or "unknown")
# If using chroot, we might still need to install packages to get missing
# (not packed) files
if use_chroot:
packages = [pkg for pkg in packages if not pkg.packfiles]
if packages:
record_usage(vagrant_install_pkgs=True)
logging.info("Some packages were not packed, so we'll install and "
"copy their files\n"
"Packages that are missing:\n%s",
' '.join(pkg.name for pkg in packages))
if packages:
try:
installer = select_installer(pack, runs, target_distribution)
except CantFindInstaller as e:
logging.error("Need to install %d packages but couldn't select a "
"package installer: %s",
len(packages), e)
target.mkdir(parents=True)
try:
# Writes setup script
logging.info("Writing setup script %s...", target / 'setup.sh')
with (target / 'setup.sh').open('w', encoding='utf-8',
newline='\n') as fp:
fp.write('#!/bin/sh\n\nset -e\n\n')
if packages:
# Updates package sources
update_script = installer.update_script()
if update_script:
fp.write(update_script)
fp.write('\n')
# Installs necessary packages
fp.write(installer.install_script(packages))
fp.write('\n')
# TODO : Compare package versions (painful because of sh)
# Untar
if use_chroot:
fp.write('\n'
'mkdir /experimentroot; cd /experimentroot\n')
fp.write('tar zpxf /vagrant/data.tgz --numeric-owner '
'--strip=1 %s\n' % rpz_pack.data_prefix)
if mount_bind:
fp.write('\n'
'mkdir -p /experimentroot/dev\n'
'mkdir -p /experimentroot/proc\n')
for pkg in packages:
fp.write('\n# Copies files from package %s\n' % pkg.name)
for f in pkg.files:
f = f.path
dest = join_root(PosixPath('/experimentroot'), f)
fp.write('mkdir -p %s\n' %
shell_escape(unicode_(f.parent)))
fp.write('cp -L %s %s\n' % (
shell_escape(unicode_(f)),
shell_escape(unicode_(dest))))
fp.write(
'\n'
'cp /etc/resolv.conf /experimentroot/etc/resolv.conf\n')
else:
fp.write('\ncd /\n')
paths = set()
pathlist = []
# Adds intermediate directories, and checks for existence in
# the tar
logging.info("Generating file list...")
data_files = rpz_pack.data_filenames()
for f in other_files:
if f.path.name == 'resolv.conf' and (
f.path.lies_under('/etc') or
f.path.lies_under('/run') or
f.path.lies_under('/var')):
continue
path = PosixPath('/')
for c in rpz_pack.remove_data_prefix(f.path).components:
path = path / c
if path in paths:
continue
paths.add(path)
if path in data_files:
pathlist.append(path)
else:
logging.info("Missing file %s", path)
# FIXME : for some reason we need reversed() here, I'm not sure
# why. Need to read more of tar's docs.
# TAR bug: --no-overwrite-dir removes --keep-old-files
# TAR bug: there is no way to make --keep-old-files not report
# an error if an existing file is encountered. --skip-old-files
# was introduced too recently. Instead, we just ignore the exit
# status
with (target / 'rpz-files.list').open('wb') as lfp:
for p in reversed(pathlist):
lfp.write(join_root(rpz_pack.data_prefix, p).path)
lfp.write(b'\0')
fp.write('tar zpxf /vagrant/data.tgz --keep-old-files '
'--numeric-owner --strip=1 '
'--null -T /vagrant/rpz-files.list || /bin/true\n')
# Copies busybox
if use_chroot:
arch = runs[0]['architecture']
download_file(busybox_url(arch),
target / 'busybox',
'busybox-%s' % arch)
fp.write(r'''
cp /vagrant/busybox /experimentroot/busybox
chmod +x /experimentroot/busybox
mkdir -p /experimentroot/bin
[ -e /experimentroot/bin/sh ] || \
ln -s /busybox /experimentroot/bin/sh
''')
# Copies pack
logging.info("Copying pack file...")
rpz_pack.copy_data_tar(target / 'data.tgz')
rpz_pack.close()
# Writes Vagrant file
logging.info("Writing %s...", target / 'Vagrantfile')
with (target / 'Vagrantfile').open('w', encoding='utf-8',
newline='\n') as fp:
# Vagrant header and version
fp.write(
'# -*- mode: ruby -*-\n'
'# vi: set ft=ruby\n\n'
'VAGRANTFILE_API_VERSION = "2"\n\n'
'Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|\n')
# Selects which box to install
fp.write(' config.vm.box = "%s"\n' % box)
# Run the setup script on the virtual machine
fp.write(' config.vm.provision "shell", path: "setup.sh"\n')
# Memory size
if memory is not None or args.gui:
fp.write(' config.vm.provider "virtualbox" do |v|\n')
if memory is not None:
fp.write(' v.memory = %d\n' % memory)
if args.gui:
fp.write(' v.gui = true\n')
fp.write(' end\n')
fp.write('end\n')
# Meta-data for reprounzip
write_dict(target,
metadata_initial_iofiles(config,
{'use_chroot': use_chroot,
'gui': args.gui}))
signals.post_setup(target=target, pack=pack)
except Exception:
target.rmtree(ignore_errors=True)
raise
def docker_setup_create(args):
"""Sets up the experiment to be run in a Docker-built container.
"""
pack = Path(args.pack[0])
target = Path(args.target[0])
if target.exists():
logging.critical("Target directory exists")
sys.exit(1)
signals.pre_setup(target=target, pack=pack)
target.mkdir()
try:
# Unpacks configuration file
rpz_pack = RPZPack(pack)
rpz_pack.extract_config(target / 'config.yml')
# Loads config
runs, packages, other_files = config = load_config(
target / 'config.yml', True)
if args.base_image:
record_usage(docker_explicit_base=True)
base_image = args.base_image[0]
if args.distribution:
target_distribution = args.distribution[0]
else:
target_distribution = None
else:
target_distribution, base_image = select_image(runs)
logging.info("Using base image %s", base_image)
logging.debug("Distribution: %s", target_distribution or "unknown")
rpz_pack.copy_data_tar(target / 'data.tgz')
arch = runs[0]['architecture']
# Writes Dockerfile
logging.info("Writing %s...", target / 'Dockerfile')
with (target / 'Dockerfile').open('w', encoding='utf-8',
newline='\n') as fp:
fp.write('FROM %s\n\n' % base_image)
# Installs busybox
download_file(busybox_url(arch), target / 'busybox',
'busybox-%s' % arch)
fp.write('COPY busybox /busybox\n')
# Installs rpzsudo
download_file(sudo_url(arch), target / 'rpzsudo',
'rpzsudo-%s' % arch)
fp.write('COPY rpzsudo /rpzsudo\n\n')
fp.write('COPY data.tgz /reprozip_data.tgz\n\n')
fp.write('COPY rpz-files.list /rpz-files.list\n')
fp.write('RUN \\\n' ' chmod +x /busybox /rpzsudo && \\\n')
if args.install_pkgs:
# Install every package through package manager
missing_packages = []
else:
# Only install packages that were not packed
missing_packages = [pkg for pkg in packages if pkg.packfiles]
packages = [pkg for pkg in packages if not pkg.packfiles]
if packages:
record_usage(docker_install_pkgs=True)
try:
installer = select_installer(pack, runs,
target_distribution)
except CantFindInstaller as e:
logging.error(
"Need to install %d packages but couldn't "
"select a package installer: %s", len(packages), e)
sys.exit(1)
# Updates package sources
update_script = installer.update_script()
if update_script:
fp.write(' %s && \\\n' % update_script)
# Installs necessary packages
fp.write(' %s && \\\n' % installer.install_script(packages))
logging.info(
"Dockerfile will install the %d software "
"packages that were not packed", len(packages))
else:
record_usage(docker_install_pkgs=False)
# Untar
paths = set()
pathlist = []
# Add intermediate directories, and check for existence in the tar
missing_files = chain.from_iterable(pkg.files
for pkg in missing_packages)
data_files = rpz_pack.data_filenames()
listoffiles = list(chain(other_files, missing_files))
for f in listoffiles:
path = PosixPath('/')
for c in rpz_pack.remove_data_prefix(f.path).components:
path = path / c
if path in paths:
continue
paths.add(path)
if path in data_files:
pathlist.append(path)
else:
logging.info("Missing file %s", path)
rpz_pack.close()
# FIXME : for some reason we need reversed() here, I'm not sure why
# Need to read more of tar's docs.
# TAR bug: --no-overwrite-dir removes --keep-old-files
with (target / 'rpz-files.list').open('wb') as lfp:
for p in reversed(pathlist):
lfp.write(join_root(rpz_pack.data_prefix, p).path)
lfp.write(b'\0')
fp.write(' cd / && '
'(tar zpxf /reprozip_data.tgz -U --recursive-unlink '
'--numeric-owner --strip=1 --null -T /rpz-files.list || '
'/busybox echo "TAR reports errors, this might or might '
'not prevent the execution to run")\n')
# Meta-data for reprounzip
write_dict(target, metadata_initial_iofiles(config))
signals.post_setup(target=target, pack=pack)
except Exception:
target.rmtree(ignore_errors=True)
raise
def vagrant_setup_create(args):
"""Sets up the experiment to be run in a Vagrant-built virtual machine.
This can either build a chroot or not.
If building a chroot, we do just like without Vagrant: we copy all the
files and only get what's missing from the host. But we do install
automatically the packages whose files are required.
If not building a chroot, we install all the packages, and only unpack
files that don't come from packages.
In short: files from packages with packfiles=True will only be used if
building a chroot.
"""
if not args.pack:
logging.critical("setup/create needs the pack filename")
sys.exit(1)
pack = Path(args.pack[0])
target = Path(args.target[0])
if target.exists():
logging.critical("Target directory exists")
sys.exit(1)
use_chroot = args.use_chroot
mount_bind = args.bind_magic_dirs
record_usage(use_chroot=use_chroot, mount_bind=mount_bind)
signals.pre_setup(target=target, pack=pack)
# Unpacks configuration file
tar = tarfile.open(str(pack), 'r:*')
member = tar.getmember('METADATA/config.yml')
member.name = 'config.yml'
tar.extract(member, str(target))
tar.close()
# Loads config
runs, packages, other_files = load_config(target / 'config.yml', True)
if args.base_image and args.base_image[0]:
record_usage(vagrant_explicit_image=True)
box = args.base_image[0]
if args.distribution:
target_distribution = args.distribution[0]
else:
target_distribution = None
else:
target_distribution, box = select_box(runs)
logging.info("Using box %s", box)
logging.debug("Distribution: %s", target_distribution or "unknown")
# If using chroot, we might still need to install packages to get missing
# (not packed) files
if use_chroot:
packages = [pkg for pkg in packages if not pkg.packfiles]
if packages:
record_usage(vagrant_install_pkgs=True)
logging.info(
"Some packages were not packed, so we'll install and "
"copy their files\n"
"Packages that are missing:\n%s",
' '.join(pkg.name for pkg in packages))
if packages:
try:
installer = select_installer(pack, runs, target_distribution)
except CantFindInstaller as e:
logging.error(
"Need to install %d packages but couldn't select a "
"package installer: %s", len(packages), e)
target.mkdir(parents=True)
# Writes setup script
logging.info("Writing setup script %s...", target / 'setup.sh')
with (target / 'setup.sh').open('w', encoding='utf-8', newline='\n') as fp:
fp.write('#!/bin/sh\n\nset -e\n\n')
if packages:
# Updates package sources
fp.write(installer.update_script())
fp.write('\n')
# Installs necessary packages
fp.write(installer.install_script(packages))
fp.write('\n')
# TODO : Compare package versions (painful because of sh)
# Untar
if use_chroot:
fp.write('\n' 'mkdir /experimentroot; cd /experimentroot\n')
fp.write('tar zpxf /vagrant/experiment.rpz '
'--numeric-owner --strip=1 DATA\n')
if mount_bind:
fp.write('\n'
'mkdir -p /experimentroot/dev\n'
'mount -o rbind /dev /experimentroot/dev\n'
'mkdir -p /experimentroot/proc\n'
'mount -o rbind /proc /experimentroot/proc\n')
for pkg in packages:
fp.write('\n# Copies files from package %s\n' % pkg.name)
for f in pkg.files:
f = f.path
dest = join_root(PosixPath('/experimentroot'), f)
fp.write('mkdir -p %s\n' %
shell_escape(unicode_(f.parent)))
fp.write('cp -L %s %s\n' % (shell_escape(
unicode_(f)), shell_escape(unicode_(dest))))
else:
fp.write('\ncd /\n')
paths = set()
pathlist = []
dataroot = PosixPath('DATA')
# Adds intermediate directories, and checks for existence in the
# tar
tar = tarfile.open(str(pack), 'r:*')
for f in other_files:
path = PosixPath('/')
for c in f.path.components[1:]:
path = path / c
if path in paths:
continue
paths.add(path)
datapath = join_root(dataroot, path)
try:
tar.getmember(str(datapath))
except KeyError:
logging.info("Missing file %s", datapath)
else:
pathlist.append(unicode_(datapath))
tar.close()
# FIXME : for some reason we need reversed() here, I'm not sure
# why. Need to read more of tar's docs.
# TAR bug: --no-overwrite-dir removes --keep-old-files
# TAR bug: there is no way to make --keep-old-files not report an
# error if an existing file is encountered. --skip-old-files was
# introduced too recently. Instead, we just ignore the exit status
fp.write('tar zpxf /vagrant/experiment.rpz --keep-old-files '
'--numeric-owner --strip=1 %s || /bin/true\n' %
' '.join(shell_escape(p) for p in reversed(pathlist)))
# Copies /bin/sh + dependencies
if use_chroot:
url = busybox_url(runs[0]['architecture'])
fp.write(r'''
mkdir -p /experimentroot/bin
mkdir -p /experimentroot/usr/bin
if [ ! -e /experimentroot/bin/sh -o ! -e /experimentroot/usr/bin/env ]; then
wget --quiet -O /experimentroot/bin/busybox {url}
chmod +x /experimentroot/bin/busybox
fi
[ -e /experimentroot/bin/sh ] || \
ln -s /bin/busybox /experimentroot/bin/sh
[ -e /experimentroot/usr/bin/env ] || \
ln -s /bin/busybox /experimentroot/usr/bin/env
'''.format(url=url))
# Copies pack
logging.info("Copying pack file...")
pack.copyfile(target / 'experiment.rpz')
# Writes Vagrant file
logging.info("Writing %s...", target / 'Vagrantfile')
with (target / 'Vagrantfile').open('w', encoding='utf-8',
newline='\n') as fp:
# Vagrant header and version
fp.write('# -*- mode: ruby -*-\n'
'# vi: set ft=ruby\n\n'
'VAGRANTFILE_API_VERSION = "2"\n\n'
'Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|\n')
# Selects which box to install
fp.write(' config.vm.box = "%s"\n' % box)
# Run the setup script on the virtual machine
fp.write(' config.vm.provision "shell", path: "setup.sh"\n')
fp.write('end\n')
# Meta-data for reprounzip
write_dict(target / '.reprounzip', {'use_chroot': use_chroot})
signals.post_setup(target=target)
def chroot_create(args):
"""Unpacks the experiment in a folder so it can be run with chroot.
All the files in the pack are unpacked; system files are copied only if
they were not packed, and busybox is installed if /bin/sh wasn't packed.
In addition, input files are put in a tar.gz (so they can be put back after
an upload) and the configuration file is extracted.
"""
if not args.pack:
logger.critical("setup/create needs the pack filename")
sys.exit(1)
pack = Path(args.pack[0])
target = Path(args.target[0])
if target.exists():
logger.critical("Target directory exists")
sys.exit(1)
if not issubclass(DefaultAbstractPath, PosixPath):
logger.critical("Not unpacking on POSIX system")
sys.exit(1)
signals.pre_setup(target=target, pack=pack)
# We can only restore owner/group of files if running as root
restore_owner = should_restore_owner(args.restore_owner)
# Unpacks configuration file
rpz_pack = RPZPack(pack)
rpz_pack.extract_config(target / 'config.yml')
# Loads config
config = load_config_file(target / 'config.yml', True)
packages = config.packages
target.mkdir()
root = (target / 'root').absolute()
root.mkdir()
try:
# Checks that everything was packed
packages_not_packed = [pkg for pkg in packages if not pkg.packfiles]
if packages_not_packed:
record_usage(chroot_missing_pkgs=True)
logger.warning("According to configuration, some files were left "
"out because they belong to the following "
"packages:%s\nWill copy files from HOST SYSTEM",
''.join('\n %s' % pkg
for pkg in packages_not_packed))
missing_files = False
for pkg in packages_not_packed:
for f in pkg.files:
path = Path(f.path)
if not path.exists():
logger.error(
"Missing file %s (from package %s) on host, "
"experiment will probably miss it",
path, pkg.name)
missing_files = True
continue
dest = join_root(root, path)
dest.parent.mkdir(parents=True)
if path.is_link():
dest.symlink(path.read_link())
else:
path.copy(dest)
if restore_owner:
stat = path.stat()
dest.chown(stat.st_uid, stat.st_gid)
if missing_files:
record_usage(chroot_mising_files=True)
# Unpacks files
members = rpz_pack.list_data()
for m in members:
# Remove 'DATA/' prefix
m.name = str(rpz_pack.remove_data_prefix(m.name))
if not restore_owner:
uid = os.getuid()
gid = os.getgid()
for m in members:
m.uid = uid
m.gid = gid
logger.info("Extracting files...")
rpz_pack.extract_data(root, members)
rpz_pack.close()
resolvconf_src = Path('/etc/resolv.conf')
if resolvconf_src.exists():
try:
resolvconf_src.copy(root / 'etc/resolv.conf')
except IOError:
pass
# Sets up /bin/sh and /usr/bin/env, downloading busybox if necessary
sh_path = join_root(root, Path('/bin/sh'))
env_path = join_root(root, Path('/usr/bin/env'))
if not sh_path.lexists() or not env_path.lexists():
logger.info("Setting up busybox...")
busybox_path = join_root(root, Path('/bin/busybox'))
busybox_path.parent.mkdir(parents=True)
with make_dir_writable(join_root(root, Path('/bin'))):
download_file(busybox_url(config.runs[0]['architecture']),
busybox_path,
'busybox-%s' % config.runs[0]['architecture'])
busybox_path.chmod(0o755)
if not sh_path.lexists():
sh_path.parent.mkdir(parents=True)
sh_path.symlink('/bin/busybox')
if not env_path.lexists():
env_path.parent.mkdir(parents=True)
env_path.symlink('/bin/busybox')
# Original input files, so upload can restore them
input_files = [f.path for f in config.inputs_outputs.values()
if f.read_runs]
if input_files:
logger.info("Packing up original input files...")
inputtar = tarfile.open(str(target / 'inputs.tar.gz'), 'w:gz')
for ifile in input_files:
filename = join_root(root, ifile)
if filename.exists():
inputtar.add(str(filename), str(ifile))
inputtar.close()
# Meta-data for reprounzip
metadata_write(target, metadata_initial_iofiles(config), 'chroot')
signals.post_setup(target=target, pack=pack)
except Exception:
rmtree_fixed(root)
raise
def directory_create(args):
"""Unpacks the experiment in a folder.
Only the files that are not part of a package are copied (unless they are
missing from the system and were packed).
In addition, input files are put in a tar.gz (so they can be put back after
an upload) and the configuration file is extracted.
"""
if not args.pack:
logging.critical("setup needs the pack filename")
sys.exit(1)
pack = Path(args.pack[0])
target = Path(args.target[0])
if target.exists():
logging.critical("Target directory exists")
sys.exit(1)
if DefaultAbstractPath is not PosixPath:
logging.critical("Not unpacking on POSIX system")
sys.exit(1)
signals.pre_setup(target=target, pack=pack)
# Unpacks configuration file
tar = tarfile.open(str(pack), 'r:*')
member = tar.getmember('METADATA/config.yml')
member.name = 'config.yml'
tar.extract(member, str(target))
# Loads config
runs, packages, other_files = load_config_file(target / 'config.yml', True)
target.mkdir()
root = (target / 'root').absolute()
root.mkdir()
# Checks packages
missing_files = False
for pkg in packages:
if pkg.packfiles:
continue
for f in pkg.files:
f = Path(f.path)
if not f.exists():
logging.error(
"Missing file %s (from package %s that wasn't packed) "
"on host, experiment will probably miss it.", f, pkg.name)
missing_files = True
if missing_files:
logging.error("Some packages are missing, you should probably install "
"them.\nUse 'reprounzip installpkgs -h' for help")
# Unpacks files
if any('..' in m.name or m.name.startswith('/') for m in tar.getmembers()):
logging.critical("Tar archive contains invalid pathnames")
sys.exit(1)
members = [m for m in tar.getmembers() if m.name.startswith('DATA/')]
for m in members:
m.name = m.name[5:]
# Makes symlink targets relative
for m in members:
if not m.issym():
continue
linkname = PosixPath(m.linkname)
if linkname.is_absolute:
m.linkname = join_root(root, PosixPath(m.linkname)).path
logging.info("Extracting files...")
tar.extractall(str(root), members)
tar.close()
# Gets library paths
lib_dirs = []
p = subprocess.Popen(['/sbin/ldconfig', '-v', '-N'],
stdout=subprocess.PIPE)
try:
for l in p.stdout:
if len(l) < 3 or l[0] in (b' ', b'\t'):
continue
if l.endswith(b':\n'):
lib_dirs.append(Path(l[:-2]))
finally:
p.wait()
# Original input files, so upload can restore them
if any(run['input_files'] for run in runs):
logging.info("Packing up original input files...")
inputtar = tarfile.open(str(target / 'inputs.tar.gz'), 'w:gz')
for run in runs:
for ifile in itervalues(run['input_files']):
inputtar.add(str(join_root(root, PosixPath(ifile))),
str(PosixPath(ifile)))
inputtar.close()
# Meta-data for reprounzip
write_dict(target / '.reprounzip', {}, 'directory')
signals.post_setup(target=target)
def functional_tests(raise_warnings, interactive, run_vagrant, run_docker):
rpz_python = [os.environ.get('REPROZIP_PYTHON', sys.executable)]
rpuz_python = [os.environ.get('REPROUNZIP_PYTHON', sys.executable)]
# Can't match on the SignalWarning category here because of a Python bug
# http://bugs.python.org/issue22543
if raise_warnings:
rpz_python.extend(['-W', 'error:signal'])
rpuz_python.extend(['-W', 'error:signal'])
if 'COVER' in os.environ:
rpz_python.extend(['-m'] + os.environ['COVER'].split(' '))
rpuz_python.extend(['-m'] + os.environ['COVER'].split(' '))
reprozip_main = tests.parent / 'reprozip/reprozip/main.py'
reprounzip_main = tests.parent / 'reprounzip/reprounzip/main.py'
verbose = ['-v'] * 3
rpz = rpz_python + [reprozip_main.absolute().path] + verbose
rpuz = rpuz_python + [reprounzip_main.absolute().path] + verbose
print("Command lines are:\n%r\n%r" % (rpz, rpuz))
# ########################################
# testrun /bin/echo
#
output = check_output(rpz + ['testrun', '/bin/echo', 'outputhere'])
assert any(b' 1 | /bin/echo outputhere ' in l for l in output.splitlines())
output = check_output(
rpz + ['testrun', '-a', '/fake/path/echo', '/bin/echo', 'outputhere'])
assert any(b' 1 | (/bin/echo) /fake/path/echo outputhere ' in l
for l in output.splitlines())
# ########################################
# testrun multiple commands
#
check_call(rpz + [
'testrun', 'bash', '-c', 'cat ../../../../../etc/passwd;'
'cd /var/lib;'
'cat ../../etc/group'
])
check_call(rpz +
['trace', '--overwrite', 'bash', '-c', 'cat /etc/passwd;echo'])
check_call(
rpz +
['trace', '--continue', 'sh', '-c', 'cat /etc/group;/usr/bin/id'])
check_call(rpz + ['pack'])
check_call(rpuz + ['graph', 'graph.dot'])
check_call(rpuz + ['graph', 'graph2.dot', 'experiment.rpz'])
sudo = ['sudo', '-E'] # -E to keep REPROZIP_USAGE_STATS
# ########################################
# 'simple' program: trace, pack, info, unpack
#
def check_simple(args, stream, infile=1):
output = check_output(args, stream).splitlines()
try:
first = output.index(b"Read 6 bytes")
except ValueError:
stderr.write("output = %r\n" % output)
raise
if infile == 1:
assert output[first + 1] == b"a = 29, b = 13"
assert output[first + 2] == b"result = 42"
else: # infile == 2
assert output[first + 1] == b"a = 25, b = 11"
assert output[first + 2] == b"result = 36"
# Build
build('simple', ['simple.c'])
# Trace
check_call(rpz + [
'trace', '--overwrite', '-d', 'rpz-simple', './simple',
(tests / 'simple_input.txt').path, 'simple_output.txt'
])
orig_output_location = Path('simple_output.txt').absolute()
assert orig_output_location.is_file()
with orig_output_location.open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
orig_output_location.remove()
# Read config
with Path('rpz-simple/config.yml').open(encoding='utf-8') as fp:
conf = yaml.safe_load(fp)
other_files = set(Path(f).absolute() for f in conf['other_files'])
expected = [Path('simple'), (tests / 'simple_input.txt')]
assert other_files.issuperset([f.resolve() for f in expected])
# Check input and output files
inputs_outputs = conf['inputs_outputs']
# Exactly one input: "arg1", "...simple_input.txt"
# Output: 'arg2', "...simple_output.txt"
# There might be more output files: the C coverage files
found = 0
for fdict in inputs_outputs:
if Path(fdict['path']).name == b'simple_input.txt':
assert fdict['name'] == 'arg1'
assert fdict['read_by_runs'] == [0]
assert not fdict.get('written_by_runs')
found |= 0x01
elif Path(fdict['path']).name == b'simple_output.txt':
assert fdict['name'] == 'arg2'
assert not fdict.get('read_by_runs')
assert fdict['written_by_runs'] == [0]
found |= 0x02
else:
# No other inputs
assert not fdict.get('read_by_runs')
assert found == 0x03
# Pack
check_call(rpz + ['pack', '-d', 'rpz-simple', 'simple.rpz'])
Path('simple').rename('simple.orig')
# Info
check_call(rpuz + ['info', 'simple.rpz'])
# Show files
check_call(rpuz + ['showfiles', 'simple.rpz'])
# Lists packages
check_call(rpuz + ['installpkgs', '--summary', 'simple.rpz'])
# Unpack directory
check_call(rpuz + ['directory', 'setup', 'simple.rpz', 'simpledir'])
# Run directory
check_simple(rpuz + ['directory', 'run', 'simpledir'], 'err')
output_in_dir = join_root(Path('simpledir/root'), orig_output_location)
with output_in_dir.open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Delete with wrong command (should fail)
p = subprocess.Popen(rpuz + ['chroot', 'destroy', 'simpledir'],
stderr=subprocess.PIPE)
out, err = p.communicate()
assert p.poll() != 0
err = err.splitlines()
assert b"Wrong unpacker used" in err[0]
assert err[1].startswith(b"usage: ")
# Delete directory
check_call(rpuz + ['directory', 'destroy', 'simpledir'])
# Unpack chroot
check_call(
sudo + rpuz +
['chroot', 'setup', '--bind-magic-dirs', 'simple.rpz', 'simplechroot'])
try:
output_in_chroot = join_root(Path('simplechroot/root'),
orig_output_location)
# Run chroot
check_simple(sudo + rpuz + ['chroot', 'run', 'simplechroot'], 'err')
with output_in_chroot.open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Get output file
check_call(sudo + rpuz +
['chroot', 'download', 'simplechroot', 'arg2:output1.txt'])
with Path('output1.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Get random file
check_call(sudo + rpuz + [
'chroot', 'download', 'simplechroot',
'%s:binc.bin' % (Path.cwd() / 'simple')
])
assert same_files('simple.orig', 'binc.bin')
# Replace input file
check_call(sudo + rpuz + [
'chroot', 'upload', 'simplechroot',
'%s:arg1' % (tests / 'simple_input2.txt')
])
check_call(sudo + rpuz + ['chroot', 'upload', 'simplechroot'])
# Run again
check_simple(sudo + rpuz + ['chroot', 'run', 'simplechroot'], 'err', 2)
with output_in_chroot.open(encoding='utf-8') as fp:
assert fp.read().strip() == '36'
# Reset input file
check_call(sudo + rpuz + ['chroot', 'upload', 'simplechroot', ':arg1'])
# Run again
check_simple(sudo + rpuz + ['chroot', 'run', 'simplechroot'], 'err')
with output_in_chroot.open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Replace input file via path
check_call(sudo + rpuz + [
'chroot', 'upload', 'simplechroot',
'%s:%s' % (tests / 'simple_input2.txt', tests / 'simple_input.txt')
])
check_call(sudo + rpuz + ['chroot', 'upload', 'simplechroot'])
# Run again
check_simple(sudo + rpuz + ['chroot', 'run', 'simplechroot'], 'err', 2)
# Delete with wrong command (should fail)
p = subprocess.Popen(rpuz + ['directory', 'destroy', 'simplechroot'],
stderr=subprocess.PIPE)
out, err = p.communicate()
assert p.poll() != 0
err = err.splitlines()
assert b"Wrong unpacker used" in err[0]
assert err[1].startswith(b"usage:")
finally:
# Delete chroot
check_call(sudo + rpuz + ['chroot', 'destroy', 'simplechroot'])
# Use reprounzip-vistrails with chroot
check_call(sudo + rpuz + [
'chroot', 'setup', '--bind-magic-dirs', 'simple.rpz', 'simplechroot_vt'
])
try:
output_in_chroot = join_root(Path('simplechroot_vt/root'),
orig_output_location)
# Run using reprounzip-vistrails
check_simple(
sudo + rpuz_python + [
'-m', 'reprounzip.plugins.vistrails', '1', 'chroot',
'simplechroot_vt', '0', '--input-file',
'arg1:%s' % (tests / 'simple_input2.txt'), '--output-file',
'arg2:output_vt.txt'
], 'err', 2)
with output_in_chroot.open(encoding='utf-8') as fp:
assert fp.read().strip() == '36'
finally:
# Delete chroot
check_call(sudo + rpuz + ['chroot', 'destroy', 'simplechroot_vt'])
if not (tests / 'vagrant').exists():
check_call([
'sudo', 'sh', '-c',
'mkdir %(d)s; chmod 777 %(d)s' % {
'd': tests / 'vagrant'
}
])
# Unpack Vagrant-chroot
check_call(rpuz + [
'vagrant', 'setup/create', '--memory', '512', '--use-chroot',
'simple.rpz', (tests / 'vagrant/simplevagrantchroot').path
])
print("\nVagrant project set up in simplevagrantchroot")
try:
if run_vagrant:
check_simple(
rpuz + [
'vagrant', 'run', '--no-stdin',
(tests / 'vagrant/simplevagrantchroot').path
], 'out')
# Get output file
check_call(rpuz + [
'vagrant', 'download', (tests / 'vagrant/simplevagrantchroot'
).path, 'arg2:voutput1.txt'
])
with Path('voutput1.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Get random file
check_call(rpuz + [
'vagrant', 'download', (tests /
'vagrant/simplevagrantchroot').path,
'%s:binvc.bin' % (Path.cwd() / 'simple')
])
assert same_files('simple.orig', 'binvc.bin')
# Replace input file
check_call(rpuz + [
'vagrant', 'upload', (tests /
'vagrant/simplevagrantchroot').path,
'%s:arg1' % (tests / 'simple_input2.txt')
])
check_call(rpuz + [
'vagrant', 'upload', (tests /
'vagrant/simplevagrantchroot').path
])
# Run again
check_simple(
rpuz + [
'vagrant', 'run', '--no-stdin',
(tests / 'vagrant/simplevagrantchroot').path
], 'out', 2)
# Get output file
check_call(rpuz + [
'vagrant', 'download', (tests / 'vagrant/simplevagrantchroot'
).path, 'arg2:voutput2.txt'
])
with Path('voutput2.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '36'
# Reset input file
check_call(rpuz + [
'vagrant', 'upload',
(tests / 'vagrant/simplevagrantchroot').path, ':arg1'
])
# Run again
check_simple(
rpuz + [
'vagrant', 'run', '--no-stdin',
(tests / 'vagrant/simplevagrantchroot').path
], 'out')
# Get output file
check_call(rpuz + [
'vagrant', 'download', (tests / 'vagrant/simplevagrantchroot'
).path, 'arg2:voutput1.txt'
])
with Path('voutput1.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Replace input file via path
check_call(rpuz + [
'vagrant', 'upload', (tests /
'vagrant/simplevagrantchroot').path,
'%s:%s' %
(tests / 'simple_input2.txt', tests / 'simple_input.txt')
])
# Run again
check_simple(
rpuz + [
'vagrant', 'run', '--no-stdin',
(tests / 'vagrant/simplevagrantchroot').path
], 'out', 2)
# Destroy
check_call(rpuz + [
'vagrant', 'destroy', (tests /
'vagrant/simplevagrantchroot').path
])
elif interactive:
print("Test and press enter")
sys.stdin.readline()
finally:
if (tests / 'vagrant/simplevagrantchroot').exists():
(tests / 'vagrant/simplevagrantchroot').rmtree()
# Unpack Vagrant without chroot
check_call(rpuz + [
'vagrant', 'setup/create', '--dont-use-chroot', 'simple.rpz',
(tests / 'vagrant/simplevagrant').path
])
print("\nVagrant project set up in simplevagrant")
try:
if run_vagrant:
check_simple(
rpuz + [
'vagrant', 'run', '--no-stdin',
(tests / 'vagrant/simplevagrant').path
], 'out')
# Get output file
check_call(rpuz + [
'vagrant', 'download',
(tests / 'vagrant/simplevagrant').path, 'arg2:woutput1.txt'
])
with Path('woutput1.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Get random file
check_call(rpuz + [
'vagrant', 'download', (tests / 'vagrant/simplevagrant').path,
'%s:binvs.bin' % (Path.cwd() / 'simple')
])
assert same_files('simple.orig', 'binvs.bin')
# Replace input file
check_call(rpuz + [
'vagrant', 'upload', (tests / 'vagrant/simplevagrant').path,
'%s:arg1' % (tests / 'simple_input2.txt')
])
check_call(
rpuz +
['vagrant', 'upload', (tests / 'vagrant/simplevagrant').path])
# Run again
check_simple(
rpuz + [
'vagrant', 'run', '--no-stdin',
(tests / 'vagrant/simplevagrant').path
], 'out', 2)
# Get output file
check_call(rpuz + [
'vagrant', 'download',
(tests / 'vagrant/simplevagrant').path, 'arg2:woutput2.txt'
])
with Path('woutput2.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '36'
# Reset input file
check_call(rpuz + [
'vagrant', 'upload', (tests /
'vagrant/simplevagrant').path, ':arg1'
])
# Run again
check_simple(
rpuz + [
'vagrant', 'run', '--no-stdin',
(tests / 'vagrant/simplevagrant').path
], 'out')
# Get output file
check_call(rpuz + [
'vagrant', 'download',
(tests / 'vagrant/simplevagrant').path, 'arg2:voutput1.txt'
])
with Path('voutput1.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Destroy
check_call(
rpuz +
['vagrant', 'destroy', (tests / 'vagrant/simplevagrant').path])
elif interactive:
print("Test and press enter")
sys.stdin.readline()
finally:
if (tests / 'vagrant/simplevagrant').exists():
(tests / 'vagrant/simplevagrant').rmtree()
# Unpack Docker
check_call(rpuz + ['docker', 'setup/create', 'simple.rpz', 'simpledocker'])
print("\nDocker project set up in simpledocker")
try:
if run_docker:
check_call(rpuz + ['docker', 'setup/build', 'simpledocker'])
check_simple(rpuz + ['docker', 'run', 'simpledocker'], 'out')
# Get output file
check_call(
rpuz +
['docker', 'download', 'simpledocker', 'arg2:doutput1.txt'])
with Path('doutput1.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Get random file
check_call(rpuz + [
'docker', 'download', 'simpledocker',
'%s:bind.bin' % (Path.cwd() / 'simple')
])
assert same_files('simple.orig', 'bind.bin')
# Replace input file
check_call(rpuz + [
'docker', 'upload', 'simpledocker',
'%s:arg1' % (tests / 'simple_input2.txt')
])
check_call(rpuz + ['docker', 'upload', 'simpledocker'])
check_call(rpuz + ['showfiles', 'simpledocker'])
# Run again
check_simple(rpuz + ['docker', 'run', 'simpledocker'], 'out', 2)
# Get output file
check_call(
rpuz +
['docker', 'download', 'simpledocker', 'arg2:doutput2.txt'])
with Path('doutput2.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '36'
# Reset input file
check_call(rpuz + ['docker', 'upload', 'simpledocker', ':arg1'])
# Run again
check_simple(rpuz + ['docker', 'run', 'simpledocker'], 'out')
# Get output file
check_call(
rpuz +
['docker', 'download', 'simpledocker', 'arg2:doutput1.txt'])
with Path('doutput1.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Replace input file via path
check_call(rpuz + [
'docker', 'upload', 'simpledocker',
'%s:%s' %
(tests / 'simple_input2.txt', tests / 'simple_input.txt')
])
# Run again
check_simple(rpuz + ['docker', 'run', 'simpledocker'], 'out', 2)
# Destroy
check_call(rpuz + ['docker', 'destroy', 'simpledocker'])
elif interactive:
print("Test and press enter")
sys.stdin.readline()
finally:
if Path('simpledocker').exists():
Path('simpledocker').rmtree()
# ########################################
# 'threads' program: testrun
#
# Build
build('threads', ['threads.c'], ['-lpthread'])
# Trace
output = check_output(rpz + ['testrun', './threads'], 'err')
assert any(b'successfully exec\'d /bin/./echo' in l
for l in output.splitlines())
# ########################################
# 'threads2' program: testrun
#
# Build
build('threads2', ['threads2.c'], ['-lpthread'])
# Trace
output = check_output(rpz + ['testrun', './threads2'], 'err')
assert any(b'successfully exec\'d /bin/echo' in l
for l in output.splitlines())
# ########################################
# 'segv' program: testrun
#
# Build
build('segv', ['segv.c'])
# Trace
check_call(rpz + ['testrun', './segv'])
# ########################################
# 'exec_echo' program: trace, pack, run --cmdline
#
# Build
build('exec_echo', ['exec_echo.c'])
# Trace
check_call(
rpz +
['trace', '--overwrite', './exec_echo', 'originalexecechooutput'])
# Pack
check_call(rpz + ['pack', 'exec_echo.rpz'])
# Unpack chroot
check_call(sudo + rpuz +
['chroot', 'setup', 'exec_echo.rpz', 'echochroot'])
try:
# Run original command-line
output = check_output(sudo + rpuz + ['chroot', 'run', 'echochroot'])
assert output == b'originalexecechooutput\n'
# Prints out command-line
output = check_output(sudo + rpuz +
['chroot', 'run', 'echochroot', '--cmdline'])
assert any(b'./exec_echo originalexecechooutput' == s.strip()
for s in output.split(b'\n'))
# Run with different command-line
output = check_output(sudo + rpuz + [
'chroot', 'run', 'echochroot', '--cmdline', './exec_echo',
'changedexecechooutput'
])
assert output == b'changedexecechooutput\n'
finally:
check_call(sudo + rpuz + ['chroot', 'destroy', 'echochroot'])
# ########################################
# 'exec_echo' program: testrun
# This is built with -m32 so that we transition:
# python (x64) -> exec_echo (i386) -> echo (x64)
#
if sys.maxsize > 2**32:
# Build
build('exec_echo32', ['exec_echo.c'], ['-m32'])
# Trace
check_call(rpz + ['testrun', './exec_echo32', '42'])
else:
print("Can't try exec_echo transitions: not running on 64bits")
# ########################################
# Tracing non-existing program
#
ret = call(rpz + ['testrun', './doesntexist'])
assert ret == 127
# ########################################
# 'connect' program: testrun
#
# Build
build('connect', ['connect.c'])
# Trace
err = check_output(rpz + ['testrun', './connect'], 'err')
err = err.split(b'\n')
assert not any(b'program exited with non-zero code' in l for l in err)
assert any(re.search(br'process connected to [0-9.]+:80', l) for l in err)
# ########################################
# 'vfork' program: testrun
#
# Build
build('vfork', ['vfork.c'])
# Trace
err = check_output(rpz + ['testrun', './vfork'], 'err')
err = err.split(b'\n')
assert not any(b'program exited with non-zero code' in l for l in err)
# ########################################
# 'rename' program: trace
#
# Build
build('rename', ['rename.c'])
# Trace
check_call(rpz +
['trace', '--overwrite', '-d', 'rename-trace', './rename'])
with Path('rename-trace/config.yml').open(encoding='utf-8') as fp:
config = yaml.safe_load(fp)
# Check that written files were logged
database = Path.cwd() / 'rename-trace/trace.sqlite3'
if PY3:
# On PY3, connect() only accepts unicode
conn = sqlite3.connect(str(database))
else:
conn = sqlite3.connect(database.path)
conn.row_factory = sqlite3.Row
rows = conn.execute('''
SELECT name FROM opened_files
''')
files = set(Path(r[0]) for r in rows)
for n in ('dir1/file', 'dir2/file', 'dir2/brokensymlink', 'dir2/symlink'):
if (Path.cwd() / n) not in files:
raise AssertionError("Missing file: %s" % (Path.cwd() / n))
conn.close()
# Check that created files won't be packed
for f in config.get('other_files'):
if 'dir2' in Path(f).parent.components:
raise AssertionError("Created file shouldn't be packed: %s" %
Path(f))
# ########################################
# 'readwrite' program: trace
# Build
build('readwrite', ['readwrite.c'])
# Create test folder
Path('readwrite_test').mkdir()
with Path('readwrite_test/existing').open('w'):
pass
# Trace existing one
check_call(rpz + [
'trace', '--overwrite', '-d', 'readwrite-E-trace', './readwrite',
'readwrite_test/existing'
])
# Check that file was logged as read and written
database = Path.cwd() / 'readwrite-E-trace/trace.sqlite3'
if PY3:
# On PY3, connect() only accepts unicode
conn = sqlite3.connect(str(database))
else:
conn = sqlite3.connect(database.path)
conn.row_factory = sqlite3.Row
rows = list(
conn.execute(
'''
SELECT mode FROM opened_files
WHERE name = ?
''', (str(Path('readwrite_test/existing').absolute()), )))
conn.close()
assert rows
assert rows[0][0] == FILE_READ | FILE_WRITE
# Trace non-existing one
check_call(rpz + [
'trace', '--overwrite', '-d', 'readwrite-N-trace', './readwrite',
'readwrite_test/nonexisting'
])
# Check that file was logged as written only
database = Path.cwd() / 'readwrite-N-trace/trace.sqlite3'
if PY3:
# On PY3, connect() only accepts unicode
conn = sqlite3.connect(str(database))
else:
conn = sqlite3.connect(database.path)
conn.row_factory = sqlite3.Row
rows = list(
conn.execute(
'''
SELECT mode FROM opened_files
WHERE name = ?
''', (str(Path('readwrite_test/nonexisting').absolute()), )))
conn.close()
assert rows
assert rows[0][0] == FILE_WRITE
# Trace a failure: inaccessible file
ret = call(rpz + [
'trace', '--overwrite', '-d', 'readwrite-F-trace', './readwrite',
'readwrite_test/non/existing/file'
])
assert ret == 1
# ########################################
# Test shebang corner-cases
#
Path('a').symlink('b')
with Path('b').open('w') as fp:
fp.write('#!%s 0\nsome content\n' % (Path.cwd() / 'c'))
Path('b').chmod(0o744)
Path('c').symlink('d')
with Path('d').open('w') as fp:
fp.write('#!e')
Path('d').chmod(0o744)
with Path('e').open('w') as fp:
fp.write('#!/bin/echo')
Path('e').chmod(0o744)
# Trace
out = check_output(rpz + [
'trace', '--overwrite', '-d', 'shebang-trace',
'--dont-identify-packages', './a', '1', '2'
])
out = out.splitlines()[0]
assert out == ('e %s 0 ./a 1 2' % (Path.cwd() / 'c')).encode('ascii')
# Check config
with Path('shebang-trace/config.yml').open(encoding='utf-8') as fp:
config = yaml.safe_load(fp)
other_files = set(
Path(f) for f in config['other_files']
if f.startswith('%s/' % Path.cwd()))
# Check database
database = Path.cwd() / 'shebang-trace/trace.sqlite3'
if PY3:
# On PY3, connect() only accepts unicode
conn = sqlite3.connect(str(database))
else:
conn = sqlite3.connect(database.path)
conn.row_factory = sqlite3.Row
rows = conn.execute('''
SELECT name FROM opened_files
''')
opened = [Path(r[0]) for r in rows if r[0].startswith('%s/' % Path.cwd())]
rows = conn.execute('''
SELECT name, argv FROM executed_files
''')
executed = [(Path(r[0]), r[1]) for r in rows
if Path(r[0]).lies_under(Path.cwd())]
print("other_files: %r" % sorted(other_files))
print("opened: %r" % opened)
print("executed: %r" % executed)
assert other_files == set(Path.cwd() / p
for p in ('a', 'b', 'c', 'd', 'e'))
assert opened == [Path.cwd() / 'c', Path.cwd() / 'e']
assert executed == [(Path.cwd() / 'a', './a\x001\x002\x00')]
# ########################################
# Test old packages
#
old_packages = [
('simple-0.4.0.rpz',
'https://drive.google.com/uc?export=download&id=0B3ucPz7GSthBVG4xZW1V'
'eDhXNTQ'),
('simple-0.6.0.rpz',
'https://drive.google.com/uc?export=download&id=0B3ucPz7GSthBbl9SUjhr'
'cUdtbGs'),
('simple-0.7.1.rpz',
'https://drive.google.com/uc?export=download&id=0B3ucPz7GSthBRGp2Vm5V'
'QVpWOGs'),
]
for name, url in old_packages:
print("Testing old package %s" % name)
f = Path(name)
if not f.exists():
download_file_retry(url, f)
# Info
check_call(rpuz + ['info', name])
# Show files
check_call(rpuz + ['showfiles', name])
# Lists packages
check_call(rpuz + ['installpkgs', '--summary', name])
# Unpack directory
check_call(rpuz + ['directory', 'setup', name, 'simpledir'])
# Run directory
check_simple(rpuz + ['directory', 'run', 'simpledir'], 'err')
output_in_dir = Path('simpledir/root/tmp')
output_in_dir = output_in_dir.listdir('reprozip_*')[0]
output_in_dir = output_in_dir / 'simple_output.txt'
with output_in_dir.open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Delete with wrong command (should fail)
p = subprocess.Popen(rpuz + ['chroot', 'destroy', 'simpledir'],
stderr=subprocess.PIPE)
out, err = p.communicate()
assert p.poll() != 0
err = err.splitlines()
assert b"Wrong unpacker used" in err[0]
assert err[1].startswith(b"usage: ")
# Delete directory
check_call(rpuz + ['directory', 'destroy', 'simpledir'])
# ########################################
# Copies back coverage report
#
coverage = Path('.coverage')
if coverage.exists():
coverage.copyfile(tests.parent / '.coverage.runpy')
def functional_tests(raise_warnings, interactive, run_vagrant, run_docker):
python = [sys.executable]
# Can't match on the SignalWarning category here because of a Python bug
# http://bugs.python.org/issue22543
python.extend(['-W', 'error:signal'])
if 'COVER' in os.environ:
python.extend(['-m'] + os.environ['COVER'].split(' '))
reprozip_main = tests.parent / 'reprozip/reprozip/main.py'
reprounzip_main = tests.parent / 'reprounzip/reprounzip/main.py'
verbose = ['-v'] * 3
rpz = python + [reprozip_main.absolute().path] + verbose
rpuz = python + [reprounzip_main.absolute().path] + verbose
# ########################################
# 'simple' program: trace, pack, info, unpack
#
# Build
build('simple', ['simple.c'])
# Trace
check_call(rpz + ['trace', '-d', 'rpz-simple',
'./simple',
(tests / 'simple_input.txt').path,
'simple_output.txt'])
orig_output_location = Path('simple_output.txt').absolute()
assert orig_output_location.is_file()
with orig_output_location.open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
orig_output_location.remove()
# Read config
with Path('rpz-simple/config.yml').open(encoding='utf-8') as fp:
conf = yaml.safe_load(fp)
other_files = set(Path(f).absolute() for f in conf['other_files'])
expected = [Path('simple'), (tests / 'simple_input.txt')]
assert other_files.issuperset([f.resolve() for f in expected])
# Check input and output files
input_files = conf['runs'][0]['input_files']
assert (dict((k, Path(f).name)
for k, f in iteritems(input_files)) ==
{'arg': b'simple_input.txt'})
output_files = conf['runs'][0]['output_files']
print(dict((k, Path(f).name) for k, f in iteritems(output_files)))
# Here we don't test for dict equality, since we might have C coverage
# files in the mix
assert Path(output_files['arg']).name == b'simple_output.txt'
# Pack
check_call(rpz + ['pack', '-d', 'rpz-simple', 'simple.rpz'])
Path('simple').remove()
# Info
check_call(rpuz + ['info', 'simple.rpz'])
# Show files
check_call(rpuz + ['showfiles', 'simple.rpz'])
# Lists packages
check_call(rpuz + ['installpkgs', '--summary', 'simple.rpz'])
# Unpack directory
check_call(rpuz + ['directory', 'setup', 'simple.rpz', 'simpledir'])
# Run directory
check_call(rpuz + ['directory', 'run', 'simpledir'])
output_in_dir = join_root(Path('simpledir/root'), orig_output_location)
with output_in_dir.open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Delete with wrong command (should fail)
assert call(rpuz + ['chroot', 'destroy', 'simpledir']) != 0
# Delete directory
check_call(rpuz + ['directory', 'destroy', 'simpledir'])
# Unpack chroot
check_call(['sudo'] + rpuz + ['chroot', 'setup', '--bind-magic-dirs',
'simple.rpz', 'simplechroot'])
# Run chroot
check_call(['sudo'] + rpuz + ['chroot', 'run', 'simplechroot'])
output_in_chroot = join_root(Path('simplechroot/root'),
orig_output_location)
with output_in_chroot.open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Get output file
check_call(['sudo'] + rpuz + ['chroot', 'download', 'simplechroot',
'arg:output1.txt'])
with Path('output1.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Replace input file
check_call(['sudo'] + rpuz + ['chroot', 'upload', 'simplechroot',
'%s:arg' % (tests / 'simple_input2.txt')])
check_call(['sudo'] + rpuz + ['chroot', 'upload', 'simplechroot'])
# Run again
check_call(['sudo'] + rpuz + ['chroot', 'run', 'simplechroot'])
output_in_chroot = join_root(Path('simplechroot/root'),
orig_output_location)
with output_in_chroot.open(encoding='utf-8') as fp:
assert fp.read().strip() == '36'
# Delete with wrong command (should fail)
assert call(rpuz + ['directory', 'destroy', 'simplechroot']) != 0
# Delete chroot
check_call(['sudo'] + rpuz + ['chroot', 'destroy', 'simplechroot'])
if not Path('/vagrant').exists():
check_call(['sudo', 'sh', '-c', 'mkdir /vagrant; chmod 777 /vagrant'])
# Unpack Vagrant-chroot
check_call(rpuz + ['vagrant', 'setup/create', '--use-chroot', 'simple.rpz',
'/vagrant/simplevagrantchroot'])
print("\nVagrant project set up in simplevagrantchroot")
try:
if run_vagrant:
check_call(rpuz + ['vagrant', 'run', '--no-stdin',
'/vagrant/simplevagrantchroot'])
# Destroy
check_call(rpuz + ['vagrant', 'destroy',
'/vagrant/simplevagrantchroot'])
elif interactive:
print("Test and press enter")
sys.stdin.readline()
finally:
if Path('/vagrant/simplevagrantchroot').exists():
Path('/vagrant/simplevagrantchroot').rmtree()
# Unpack Vagrant without chroot
check_call(rpuz + ['vagrant', 'setup/create', '--dont-use-chroot',
'simple.rpz',
'/vagrant/simplevagrant'])
print("\nVagrant project set up in simplevagrant")
try:
if run_vagrant:
check_call(rpuz + ['vagrant', 'run', '--no-stdin',
'/vagrant/simplevagrant'])
# Destroy
check_call(rpuz + ['vagrant', 'destroy', '/vagrant/simplevagrant'])
elif interactive:
print("Test and press enter")
sys.stdin.readline()
finally:
if Path('/vagrant/simplevagrant').exists():
Path('/vagrant/simplevagrant').rmtree()
# Unpack Docker
check_call(rpuz + ['docker', 'setup/create', 'simple.rpz', 'simpledocker'])
print("\nDocker project set up in simpledocker")
try:
if run_docker:
check_call(rpuz + ['docker', 'setup/build', 'simpledocker'])
check_call(rpuz + ['docker', 'run', 'simpledocker'])
# Get output file
check_call(rpuz + ['docker', 'download', 'simpledocker',
'arg:doutput1.txt'])
with Path('doutput1.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Replace input file
check_call(rpuz + ['docker', 'upload', 'simpledocker',
'%s:arg' % (tests / 'simple_input2.txt')])
check_call(rpuz + ['docker', 'upload', 'simpledocker'])
check_call(rpuz + ['showfiles', 'simpledocker'])
# Run again
check_call(rpuz + ['docker', 'run', 'simpledocker'])
# Get output file
check_call(rpuz + ['docker', 'download', 'simpledocker',
'arg:doutput2.txt'])
with Path('doutput2.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '36'
# Destroy
check_call(rpuz + ['docker', 'destroy', 'simpledocker'])
elif interactive:
print("Test and press enter")
sys.stdin.readline()
finally:
if Path('simpledocker').exists():
Path('simpledocker').rmtree()
# ########################################
# 'threads' program: testrun
#
# Build
build('threads', ['threads.c'], ['-lpthread'])
# Trace
check_call(rpz + ['testrun', './threads'])
# ########################################
# 'segv' program: testrun
#
# Build
build('segv', ['segv.c'])
# Trace
check_call(rpz + ['testrun', './segv'])
# ########################################
# 'exec_echo' program: trace, pack, run --cmdline
#
# Build
build('exec_echo', ['exec_echo.c'])
# Trace
check_call(rpz + ['trace', './exec_echo', 'originalexecechooutput'])
# Pack
check_call(rpz + ['pack', 'exec_echo.rpz'])
# Unpack chroot
check_call(['sudo'] + rpuz + ['chroot', 'setup',
'exec_echo.rpz', 'echochroot'])
try:
# Run original command-line
output = check_output(['sudo'] + rpuz + ['chroot', 'run',
'echochroot'])
assert output == b'originalexecechooutput\n'
# Prints out command-line
output = check_output(['sudo'] + rpuz + ['chroot', 'run',
'echochroot', '--cmdline'])
assert any(b'./exec_echo originalexecechooutput' == s.strip()
for s in output.split(b'\n'))
# Run with different command-line
output = check_output(['sudo'] + rpuz + [
'chroot', 'run', 'echochroot',
'--cmdline', './exec_echo', 'changedexecechooutput'])
assert output == b'changedexecechooutput\n'
finally:
check_call(['sudo'] + rpuz + ['chroot', 'destroy', 'echochroot'])
# ########################################
# 'exec_echo' program: testrun
# This is built with -m32 so that we transition:
# python (x64) -> exec_echo (i386) -> echo (x64)
#
if sys.maxsize > 2 ** 32:
# Build
build('exec_echo32', ['exec_echo.c'], ['-m32'])
# Trace
check_call(rpz + ['testrun', './exec_echo32 42'])
else:
print("Can't try exec_echo transitions: not running on 64bits")
# ########################################
# Tracing non-existing program
#
check_call(rpz + ['testrun', './doesntexist'])
# ########################################
# 'connect' program: testrun
#
# Build
build('connect', ['connect.c'])
# Trace
stderr = check_errout(rpz + ['testrun', './connect'])
stderr = stderr.split(b'\n')
assert not any(b'program exited with non-zero code' in l for l in stderr)
assert any(re.search(br'process connected to [0-9.]+:80', l)
for l in stderr)
# ########################################
# Copies back coverage report
#
coverage = Path('.coverage')
if coverage.exists():
coverage.copyfile(tests.parent / '.coverage.runpy')
