def main():
parser = argparse.ArgumentParser(
description="Adds __future__ imports to Python files")
parser.add_argument('-v',
'--verbose',
action='count',
dest='verbosity',
default=1)
parser.add_argument('-e',
'--enable',
action='append',
help="Future import to enable")
parser.add_argument('file',
nargs=argparse.ONE_OR_MORE,
help="File or directory in which to replace")
args = parser.parse_args()
levels = [logging.CRITICAL, logging.WARNING, logging.INFO, logging.DEBUG]
logging.basicConfig(level=levels[args.verbosity])
if not args.enable:
logging.critical("Nothing to do")
sys.exit(1)
enable = set(to_bytes(feature) for feature in args.enable)
unrecognized = enable - FUTURES
if unrecognized:
logging.critical("Error: unknown futures %s" % ', '.join(unrecognized))
sys.exit(1)
for target in args.file:
target = Path(target)
if target.is_file():
if not target.name.endswith('.py'):
logging.warning("File %s doesn't end with .py, processing "
"anyway..." % target)
process_file(target, enable)
elif target.is_dir():
logging.info("Processing %s recursively..." % target)
for filename in target.recursedir('*.py'):
process_file(filename, enable)
else:
logging.warning("Skipping %s..." % target)
def extract_trace(self, target):
"""Extracts the trace database to the specified path.
It is up to the caller to remove that file once done.
"""
target = Path(target)
if self.version == 1:
member = self.tar.getmember('METADATA/trace.sqlite3')
elif self.version == 2:
try:
member = self.tar.getmember('METADATA/trace.sqlite3.gz')
except KeyError:
member = self.tar.getmember('METADATA/trace.sqlite3')
else:
assert False
member = copy.copy(member)
member.name = str(target.components[-1])
self.tar.extract(member, path=str(Path.cwd() / target.parent))
target.chmod(0o644)
assert target.is_file()
def extract_trace(self, target):
"""Extracts the trace database to the specified path.
It is up to the caller to remove that file once done.
"""
target = Path(target)
if self.version == 1:
member = self.tar.getmember('METADATA/trace.sqlite3')
elif self.version == 2:
try:
member = self.tar.getmember('METADATA/trace.sqlite3.gz')
except KeyError:
member = self.tar.getmember('METADATA/trace.sqlite3')
else:
assert False
member = copy.copy(member)
member.name = str(target.components[-1])
self.tar.extract(member,
path=str(Path.cwd() / target.parent))
assert target.is_file()
def main():
parser = argparse.ArgumentParser(
description="Adds __future__ imports to Python files")
parser.add_argument('-v', '--verbose', action='count', dest='verbosity',
default=1)
parser.add_argument('-e', '--enable', action='append',
help="Future import to enable")
parser.add_argument('file',
nargs=argparse.ONE_OR_MORE,
help="File or directory in which to replace")
args = parser.parse_args()
levels = [logging.CRITICAL, logging.WARNING, logging.INFO, logging.DEBUG]
logging.basicConfig(level=levels[args.verbosity])
if not args.enable:
logging.critical("Nothing to do")
sys.exit(1)
enable = set(to_bytes(feature) for feature in args.enable)
unrecognized = enable - FUTURES
if unrecognized:
logging.critical("Error: unknown futures %s" % ', '.join(unrecognized))
sys.exit(1)
for target in args.file:
target = Path(target)
if target.is_file():
if not target.name.endswith('.py'):
logging.warning("File %s doesn't end with .py, processing "
"anyway..." % target)
process_file(target, enable)
elif target.is_dir():
logging.info("Processing %s recursively..." % target)
for filename in target.recursedir('*.py'):
process_file(filename, enable)
else:
logging.warning("Skipping %s..." % target)
def __init__(self, enabled, target, display=None):
self.enabled = enabled
if not self.enabled:
return
self.target = target
self.xauth = PosixPath('/.reprounzip_xauthority')
self.display = (int(display) if display is not None
else self.DISPLAY_NUMBER)
logger.debug("X11 support enabled; will create Xauthority file %s "
"for experiment. Display number is %d", self.xauth,
self.display)
# List of addresses that match the $DISPLAY variable
possible, local_display = self._locate_display()
tcp_portnum = ((6000 + local_display) if local_display is not None
else None)
if ('XAUTHORITY' in os.environ and
Path(os.environ['XAUTHORITY']).is_file()):
xauthority = Path(os.environ['XAUTHORITY'])
# Note: I'm assuming here that Xauthority has no XDG support
else:
xauthority = Path('~').expand_user() / '.Xauthority'
# Read Xauthority file
xauth_entries = {}
if xauthority.is_file():
with xauthority.open('rb') as fp:
fp.seek(0, os.SEEK_END)
size = fp.tell()
fp.seek(0, os.SEEK_SET)
while fp.tell() < size:
entry = Xauth.from_file(fp)
if (entry.name == 'MIT-MAGIC-COOKIE-1' and
entry.number == local_display):
if entry.family == Xauth.FAMILY_LOCAL:
xauth_entries[(entry.family, None)] = entry
elif (entry.family == Xauth.FAMILY_INTERNET or
entry.family == Xauth.FAMILY_INTERNET6):
xauth_entries[(entry.family,
entry.address)] = entry
# FIXME: this completely ignores addresses
logger.debug("Possible X endpoints: %s", (possible,))
# Select socket and authentication cookie
self.xauth_record = None
self.connection_info = None
for family, address in possible:
# Checks that we have a cookie
entry = family, (None if family is Xauth.FAMILY_LOCAL else address)
if entry not in xauth_entries:
continue
if family == Xauth.FAMILY_LOCAL and hasattr(socket, 'AF_UNIX'):
# Checks that the socket exists
if not Path(address).exists():
continue
self.connection_info = (socket.AF_UNIX, socket.SOCK_STREAM,
address)
self.xauth_record = xauth_entries[(family, None)]
logger.debug("Will connect to local X display via UNIX "
"socket %s", address)
break
else:
# Checks that we have a cookie
family = self.X2SOCK[family]
self.connection_info = (family, socket.SOCK_STREAM,
(address, tcp_portnum))
self.xauth_record = xauth_entries[(family, address)]
logger.debug("Will connect to X display %s:%d via %s/TCP",
address, tcp_portnum,
"IPv6" if family == socket.AF_INET6 else "IPv4")
break
# Didn't find an Xauthority record -- assume no authentication is
# needed, but still set self.connection_info
if self.connection_info is None:
for family, address in possible:
# Only try UNIX sockets, we'll use 127.0.0.1 otherwise
if family == Xauth.FAMILY_LOCAL:
if not hasattr(socket, 'AF_UNIX'):
continue
self.connection_info = (socket.AF_UNIX, socket.SOCK_STREAM,
address)
logger.debug("Will connect to X display via UNIX socket "
"%s, no authentication", address)
break
else:
self.connection_info = (socket.AF_INET, socket.SOCK_STREAM,
('127.0.0.1', tcp_portnum))
logger.debug("Will connect to X display 127.0.0.1:%d via "
"IPv4/TCP, no authentication",
tcp_portnum)
if self.connection_info is None:
raise RuntimeError("Couldn't determine how to connect to local X "
"server, DISPLAY is %s" % (
repr(os.environ['DISPLAY'])
if 'DISPLAY' in os.environ
else 'not set'))
def get_files(conn):
"""Find all the files used by the experiment by reading the trace.
"""
files = {}
access_files = [set()]
# Finds run timestamps, so we can sort input/output files by run
proc_cursor = conn.cursor()
executions = proc_cursor.execute('''
SELECT timestamp
FROM processes
WHERE parent ISNULL
ORDER BY id;
''')
run_timestamps = [r_timestamp for r_timestamp, in executions][1:]
proc_cursor.close()
# Adds dynamic linkers
for libdir in (Path('/lib'), Path('/lib64')):
if libdir.exists():
for linker in libdir.listdir('*ld-linux*'):
for filename in find_all_links(linker, True):
if filename not in files:
f = TracedFile(filename)
f.read(None)
files[f.path] = f
# Loops on executed files, and opened files, at the same time
cur = conn.cursor()
rows = cur.execute('''
SELECT 'exec' AS event_type, name, NULL AS mode, timestamp
FROM executed_files
UNION ALL
SELECT 'open' AS event_type, name, mode, timestamp
FROM opened_files
ORDER BY timestamp;
''')
executed = set()
run = 0
for event_type, r_name, r_mode, r_timestamp in rows:
if event_type == 'exec':
r_mode = FILE_READ
r_name = Path(normalize_path(r_name))
# Stays on the current run
while run_timestamps and r_timestamp > run_timestamps[0]:
del run_timestamps[0]
access_files.append(set())
run += 1
# Adds symbolic links as read files
for filename in find_all_links(
r_name.parent if r_mode & FILE_LINK else r_name, False):
if filename not in files:
f = TracedFile(filename)
f.read(run)
files[f.path] = f
# Go to final target
if not r_mode & FILE_LINK:
r_name = r_name.resolve()
if event_type == 'exec':
executed.add(r_name)
if r_name not in files:
f = TracedFile(r_name)
files[f.path] = f
else:
f = files[r_name]
if r_mode & FILE_READ:
f.read(run)
if r_mode & FILE_WRITE:
f.write(run)
# Mark the parent directory as read
if r_name.parent not in files:
fp = TracedFile(r_name.parent)
fp.read(run)
files[fp.path] = fp
# Identifies input files
if r_name.is_file() and r_name not in executed:
access_files[-1].add(f)
cur.close()
# Further filters input files
inputs = [
[
fi.path for fi in lst
# Input files are regular files,
if fi.path.is_file() and
# ONLY_READ,
fi.runs[r] == TracedFile.ONLY_READ and
# not executable,
# FIXME : currently disabled; only remove executed files
# not fi.path.stat().st_mode & 0b111 and
fi.path not in executed and
# not in a system directory
not any(fi.path.lies_under(m) for m in magic_dirs + system_dirs)
] for r, lst in enumerate(access_files)
]
# Identify output files
outputs = [
[
fi.path for fi in lst
# Output files are regular files,
if fi.path.is_file() and
# WRITTEN
fi.runs[r] == TracedFile.WRITTEN and
# not in a system directory
not any(fi.path.lies_under(m) for m in magic_dirs + system_dirs)
] for r, lst in enumerate(access_files)
]
# Run the list of files through the filter plugins
run_filter_plugins(files, inputs)
# Files removed from plugins should be removed from inputs as well
inputs = [[path for path in lst if path in files] for lst in inputs]
# Displays a warning for READ_THEN_WRITTEN files
read_then_written_files = [
fi for fi in files.values()
if fi.what == TracedFile.READ_THEN_WRITTEN and not any(
fi.path.lies_under(m) for m in magic_dirs)
]
if read_then_written_files:
logger.warning(
"Some files were read and then written. We will only pack the "
"final version of the file; reproducible experiments shouldn't "
"change their input files")
logger.info("Paths:\n%s",
", ".join(str(fi.path) for fi in read_then_written_files))
files = set(fi for fi in files.values()
if fi.what != TracedFile.WRITTEN and not any(
fi.path.lies_under(m) for m in magic_dirs))
return files, inputs, outputs
def get_files(conn):
"""Find all the files used by the experiment by reading the trace.
"""
files = {}
access_files = [set()]
# Finds run timestamps, so we can sort input/output files by run
proc_cursor = conn.cursor()
executions = proc_cursor.execute(
'''
SELECT timestamp
FROM processes
WHERE parent ISNULL
ORDER BY id;
''')
run_timestamps = [r_timestamp for r_timestamp, in executions][1:]
proc_cursor.close()
# Adds dynamic linkers
for libdir in (Path('/lib'), Path('/lib64')):
if libdir.exists():
for linker in libdir.listdir('*ld-linux*'):
for filename in find_all_links(linker, True):
if filename not in files:
f = TracedFile(filename)
f.read()
files[f.path] = f
# Loops on executed files, and opened files, at the same time
cur = conn.cursor()
rows = cur.execute(
'''
SELECT 'exec' AS event_type, name, NULL AS mode, timestamp
FROM executed_files
UNION ALL
SELECT 'open' AS event_type, name, mode, timestamp
FROM opened_files
ORDER BY timestamp;
''')
executed = set()
for event_type, r_name, r_mode, r_timestamp in rows:
if event_type == 'exec':
r_mode = FILE_READ
r_name = Path(r_name)
if event_type == 'exec':
executed.add(r_name)
# Stays on the current run
while run_timestamps and r_timestamp > run_timestamps[0]:
del run_timestamps[0]
access_files.append(set())
# Adds symbolic links as read files
for filename in find_all_links(r_name.parent if r_mode & FILE_LINK
else r_name, False):
if filename not in files:
f = TracedFile(filename)
f.read()
files[f.path] = f
# Go to final target
if not r_mode & FILE_LINK:
r_name = r_name.resolve()
if r_name not in files:
f = TracedFile(r_name)
files[f.path] = f
else:
f = files[r_name]
if r_mode & FILE_WRITE:
f.write()
# Mark the parent directory as read
if r_name.parent not in files:
fp = TracedFile(r_name.parent)
fp.read()
files[fp.path] = fp
elif r_mode & FILE_READ:
f.read()
# Identifies input files
if r_name.is_file() and r_name not in executed:
access_files[-1].add(f)
cur.close()
# Further filters input files
inputs = [[fi.path
for fi in lst
# Input files are regular files,
if fi.path.is_file() and
# ONLY_READ,
fi.what == TracedFile.ONLY_READ and
# not executable,
# FIXME : currently disabled; only remove executed files
# not fi.path.stat().st_mode & 0b111 and
fi.path not in executed and
# not in a system directory
not any(fi.path.lies_under(m)
for m in magic_dirs + system_dirs)]
for lst in access_files]
# Identify output files
outputs = [[fi.path
for fi in lst
# Output files are regular files,
if fi.path.is_file() and
# WRITTEN
fi.what == TracedFile.WRITTEN and
# not in a system directory
not any(fi.path.lies_under(m)
for m in magic_dirs + system_dirs)]
for lst in access_files]
# Displays a warning for READ_THEN_WRITTEN files
read_then_written_files = [
fi
for fi in itervalues(files)
if fi.what == TracedFile.READ_THEN_WRITTEN and
not any(fi.path.lies_under(m) for m in magic_dirs)]
if read_then_written_files:
logging.warning(
"Some files were read and then written. We will only pack the "
"final version of the file; reproducible experiments shouldn't "
"change their input files:\n%s",
", ".join(unicode_(fi.path) for fi in read_then_written_files))
files = set(
fi
for fi in itervalues(files)
if fi.what != TracedFile.WRITTEN and not any(fi.path.lies_under(m)
for m in magic_dirs))
return files, inputs, outputs
def write_configuration(directory, sort_packages, overwrite=False):
"""Writes the canonical YAML configuration file.
"""
database = directory / 'trace.sqlite3'
if PY3:
# On PY3, connect() only accepts unicode
conn = sqlite3.connect(str(database))
else:
conn = sqlite3.connect(database.path)
conn.row_factory = sqlite3.Row
# Reads info from database
files, inputs, outputs = get_files(conn)
# Identifies which file comes from which package
if sort_packages:
files, packages = identify_packages(files)
else:
packages = []
# Makes sure all the directories used as working directories are packed
# (they already do if files from them are used, but empty directories do
# not get packed inside a tar archive)
files.update(d for d in list_directories(conn) if d.path.is_dir())
# Writes configuration file
config = directory / 'config.yml'
distribution = platform.linux_distribution()[0:2]
oldconfig = not overwrite and config.exists()
cur = conn.cursor()
if oldconfig:
# Loads in previous config
runs, oldpkgs, oldfiles, patterns = load_config(config,
canonical=False,
File=TracedFile)
# Here, additional patterns are discarded
executions = cur.execute(
'''
SELECT e.name, e.argv, e.envp, e.workingdir, p.exitcode
FROM executed_files e
INNER JOIN processes p on p.id=e.id
WHERE p.parent ISNULL
ORDER BY p.id DESC
LIMIT 1;
''')
inputs = inputs[-1:]
files, packages = merge_files(files, packages,
oldfiles,
oldpkgs)
else:
runs = []
executions = cur.execute(
'''
SELECT e.name, e.argv, e.envp, e.workingdir, p.exitcode
FROM executed_files e
INNER JOIN processes p on p.id=e.id
WHERE p.parent ISNULL
ORDER BY p.id;
''')
for ((r_name, r_argv, r_envp, r_workingdir, r_exitcode),
input_files, output_files) in izip(executions, inputs, outputs):
# Decodes command-line
argv = r_argv.split('\0')
if not argv[-1]:
argv = argv[:-1]
# Decodes environment
envp = r_envp.split('\0')
if not envp[-1]:
envp = envp[:-1]
environ = dict(v.split('=', 1) for v in envp)
# Gets files from command-line
command_line_files = {}
for i, arg in enumerate(argv):
p = Path(r_workingdir, arg).resolve()
if p.is_file():
command_line_files[p] = i
input_files_on_cmdline = sum(1
for in_file in input_files
if in_file in command_line_files)
output_files_on_cmdline = sum(1
for out_file in input_files
if out_file in command_line_files)
# Labels input files
input_files_dict = {}
for in_file in input_files:
# If file is on the command-line
if in_file in command_line_files:
if input_files_on_cmdline > 1:
label = "arg_%d" % command_line_files[in_file]
else:
label = "arg"
# Else, use file's name
else:
label = in_file.unicodename
# Make labels unique
uniquelabel = label
i = 1
while uniquelabel in input_files_dict:
i += 1
uniquelabel = '%s_%d' % (label, i)
input_files_dict[uniquelabel] = str(in_file)
# TODO : Note that right now, we keep as input files the ones that
# don't appear on the command-line
# Labels output files
output_files_dict = {}
for out_file in output_files:
# If file is on the command-line
if out_file in command_line_files:
if output_files_on_cmdline > 1:
label = "arg_%d" % command_line_files[out_file]
else:
label = "arg"
# Else, use file's name
else:
label = out_file.unicodename
# Make labels unique
uniquelabel = label
i = 1
while uniquelabel in output_files_dict:
i += 1
uniquelabel = '%s_%d' % (label, i)
output_files_dict[uniquelabel] = str(out_file)
# TODO : Note that right now, we keep as output files the ones that
# don't appear on the command-line
runs.append({'binary': r_name, 'argv': argv,
'workingdir': Path(r_workingdir).path,
'architecture': platform.machine().lower(),
'distribution': distribution,
'hostname': platform.node(),
'system': [platform.system(), platform.release()],
'environ': environ,
'uid': os.getuid(),
'gid': os.getgid(),
'signal' if r_exitcode & 0x0100 else 'exitcode':
r_exitcode & 0xFF,
'input_files': input_files_dict,
'output_files': output_files_dict})
cur.close()
conn.close()
save_config(config, runs, packages, files, reprozip_version)
print("Configuration file written in {0!s}".format(config))
print("Edit that file then run the packer -- "
"use 'reprozip pack -h' for help")
def functional_tests(raise_warnings, interactive, run_vagrant, run_docker):
# Tests on Python < 2.7.3: need to use separate reprozip Python (with known
# working version of Python)
if sys.version_info < (2, 7, 3):
bug13676 = True
if 'REPROZIP_PYTHON' not in os.environ:
sys.stderr.write("Error: using reprozip with Python %s!\n" %
sys.version.split(' ', 1)[0])
sys.exit(1)
else:
bug13676 = False
rpz = [os.environ.get('REPROZIP_PYTHON', sys.executable)]
rpuz = [os.environ.get('REPROUNZIP_PYTHON', sys.executable)]
# Can't match on the SignalWarning category here because of a Python bug
# http://bugs.python.org/issue22543
if raise_warnings:
rpz.extend(['-W', 'error:signal'])
rpuz.extend(['-W', 'error:signal'])
if 'COVER' in os.environ:
rpz.extend(['-m'] + os.environ['COVER'].split(' '))
rpuz.extend(['-m'] + os.environ['COVER'].split(' '))
reprozip_main = tests.parent / 'reprozip/reprozip/main.py'
reprounzip_main = tests.parent / 'reprounzip/reprounzip/main.py'
verbose = ['-v'] * 3
rpz.extend([reprozip_main.absolute().path] + verbose)
rpuz.extend([reprounzip_main.absolute().path] + verbose)
print("Command lines are:\n%r\n%r" % (rpz, rpuz))
# ########################################
# testrun /bin/echo
#
output = check_output(rpz + ['testrun', '/bin/echo', 'outputhere'])
assert any(b' 1 | /bin/echo outputhere ' in l
for l in output.splitlines())
output = check_output(rpz + ['testrun', '-a', '/fake/path/echo',
'/bin/echo', 'outputhere'])
assert any(b' 1 | (/bin/echo) /fake/path/echo outputhere ' in l
for l in output.splitlines())
# ########################################
# testrun multiple commands
#
check_call(rpz + ['testrun', 'bash', '-c',
'cat ../../../../../etc/passwd;'
'cd /var/lib;'
'cat ../../etc/group'])
check_call(rpz + ['trace',
'bash', '-c', 'cat /etc/passwd;echo'])
check_call(rpz + ['trace', '--continue',
'sh', '-c', 'cat /etc/group;/usr/bin/id'])
check_call(rpz + ['pack'])
if not bug13676:
check_call(rpuz + ['graph', 'graph.dot'])
check_call(rpuz + ['graph', 'graph2.dot', 'experiment.rpz'])
sudo = ['sudo', '-E'] # -E to keep REPROZIP_USAGE_STATS
# ########################################
# 'simple' program: trace, pack, info, unpack
#
def check_simple(args, stream, infile=1):
output = check_output(args, stream).splitlines()
try:
first = output.index(b"Read 6 bytes")
except ValueError:
stderr.write("output = %r\n" % output)
raise
if infile == 1:
assert output[first + 1] == b"a = 29, b = 13"
assert output[first + 2] == b"result = 42"
else: # infile == 2
assert output[first + 1] == b"a = 25, b = 11"
assert output[first + 2] == b"result = 36"
# Build
build('simple', ['simple.c'])
# Trace
check_call(rpz + ['trace', '-d', 'rpz-simple',
'./simple',
(tests / 'simple_input.txt').path,
'simple_output.txt'])
orig_output_location = Path('simple_output.txt').absolute()
assert orig_output_location.is_file()
with orig_output_location.open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
orig_output_location.remove()
# Read config
with Path('rpz-simple/config.yml').open(encoding='utf-8') as fp:
conf = yaml.safe_load(fp)
other_files = set(Path(f).absolute() for f in conf['other_files'])
expected = [Path('simple'), (tests / 'simple_input.txt')]
assert other_files.issuperset([f.resolve() for f in expected])
# Check input and output files
inputs_outputs = conf['inputs_outputs']
# Exactly one input: "arg1", "...simple_input.txt"
# Output: 'arg2', "...simple_output.txt"
# There might be more output files: the C coverage files
found = 0
for fdict in inputs_outputs:
if Path(fdict['path']).name == b'simple_input.txt':
assert fdict['name'] == 'arg1'
assert fdict['read_by_runs'] == [0]
assert not fdict.get('written_by_runs')
found |= 0x01
elif Path(fdict['path']).name == b'simple_output.txt':
assert fdict['name'] == 'arg2'
assert not fdict.get('read_by_runs')
assert fdict['written_by_runs'] == [0]
found |= 0x02
else:
# No other inputs
assert not fdict.get('read_by_runs')
assert found == 0x03
# Pack
check_call(rpz + ['pack', '-d', 'rpz-simple', 'simple.rpz'])
Path('simple').remove()
# Info
check_call(rpuz + ['info', 'simple.rpz'])
# Show files
check_call(rpuz + ['showfiles', 'simple.rpz'])
# Lists packages
check_call(rpuz + ['installpkgs', '--summary', 'simple.rpz'])
# Unpack directory
check_call(rpuz + ['directory', 'setup', 'simple.rpz', 'simpledir'])
# Run directory
check_simple(rpuz + ['directory', 'run', 'simpledir'], 'err')
output_in_dir = join_root(Path('simpledir/root'), orig_output_location)
with output_in_dir.open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Delete with wrong command (should fail)
p = subprocess.Popen(rpuz + ['chroot', 'destroy', 'simpledir'],
stderr=subprocess.PIPE)
out, err = p.communicate()
assert p.poll() != 0
err = err.splitlines()
assert b"Wrong unpacker used" in err[0]
assert err[1].startswith(b"usage: ")
# Delete directory
check_call(rpuz + ['directory', 'destroy', 'simpledir'])
# Unpack chroot
check_call(sudo + rpuz + ['chroot', 'setup', '--bind-magic-dirs',
'simple.rpz', 'simplechroot'])
try:
output_in_chroot = join_root(Path('simplechroot/root'),
orig_output_location)
# Run chroot
check_simple(sudo + rpuz + ['chroot', 'run', 'simplechroot'], 'err')
with output_in_chroot.open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Get output file
check_call(sudo + rpuz + ['chroot', 'download', 'simplechroot',
'arg2:output1.txt'])
with Path('output1.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Replace input file
check_call(sudo + rpuz + ['chroot', 'upload', 'simplechroot',
'%s:arg1' % (tests / 'simple_input2.txt')])
check_call(sudo + rpuz + ['chroot', 'upload', 'simplechroot'])
# Run again
check_simple(sudo + rpuz + ['chroot', 'run', 'simplechroot'], 'err', 2)
with output_in_chroot.open(encoding='utf-8') as fp:
assert fp.read().strip() == '36'
# Reset input file
check_call(sudo + rpuz + ['chroot', 'upload', 'simplechroot', ':arg1'])
# Run again
check_simple(sudo + rpuz + ['chroot', 'run', 'simplechroot'], 'err')
with output_in_chroot.open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Delete with wrong command (should fail)
p = subprocess.Popen(rpuz + ['directory', 'destroy', 'simplechroot'],
stderr=subprocess.PIPE)
out, err = p.communicate()
assert p.poll() != 0
err = err.splitlines()
assert b"Wrong unpacker used" in err[0]
assert err[1].startswith(b"usage:")
finally:
# Delete chroot
check_call(sudo + rpuz + ['chroot', 'destroy', 'simplechroot'])
if not (tests / 'vagrant').exists():
check_call(['sudo', 'sh', '-c',
'mkdir %(d)s; chmod 777 %(d)s' % {'d': tests / 'vagrant'}])
# Unpack Vagrant-chroot
check_call(rpuz + ['vagrant', 'setup/create', '--use-chroot', 'simple.rpz',
(tests / 'vagrant/simplevagrantchroot').path])
print("\nVagrant project set up in simplevagrantchroot")
try:
if run_vagrant:
check_simple(rpuz + ['vagrant', 'run', '--no-stdin',
(tests / 'vagrant/simplevagrantchroot').path],
'out')
# Get output file
check_call(rpuz + ['vagrant', 'download',
(tests / 'vagrant/simplevagrantchroot').path,
'arg2:voutput1.txt'])
with Path('voutput1.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Replace input file
check_call(rpuz + ['vagrant', 'upload',
(tests / 'vagrant/simplevagrantchroot').path,
'%s:arg1' % (tests / 'simple_input2.txt')])
check_call(rpuz + ['vagrant', 'upload',
(tests / 'vagrant/simplevagrantchroot').path])
# Run again
check_simple(rpuz + ['vagrant', 'run', '--no-stdin',
(tests / 'vagrant/simplevagrantchroot').path],
'out', 2)
# Get output file
check_call(rpuz + ['vagrant', 'download',
(tests / 'vagrant/simplevagrantchroot').path,
'arg2:voutput2.txt'])
with Path('voutput2.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '36'
# Reset input file
check_call(rpuz + ['vagrant', 'upload',
(tests / 'vagrant/simplevagrantchroot').path,
':arg1'])
# Run again
check_simple(rpuz + ['vagrant', 'run', '--no-stdin',
(tests / 'vagrant/simplevagrantchroot').path],
'out')
# Get output file
check_call(rpuz + ['vagrant', 'download',
(tests / 'vagrant/simplevagrantchroot').path,
'arg2:voutput1.txt'])
with Path('voutput1.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Destroy
check_call(rpuz + ['vagrant', 'destroy',
(tests / 'vagrant/simplevagrantchroot').path])
elif interactive:
print("Test and press enter")
sys.stdin.readline()
finally:
if (tests / 'vagrant/simplevagrantchroot').exists():
(tests / 'vagrant/simplevagrantchroot').rmtree()
# Unpack Vagrant without chroot
check_call(rpuz + ['vagrant', 'setup/create', '--dont-use-chroot',
'simple.rpz',
(tests / 'vagrant/simplevagrant').path])
print("\nVagrant project set up in simplevagrant")
try:
if run_vagrant:
check_simple(rpuz + ['vagrant', 'run', '--no-stdin',
(tests / 'vagrant/simplevagrant').path],
'out')
# Get output file
check_call(rpuz + ['vagrant', 'download',
(tests / 'vagrant/simplevagrant').path,
'arg2:woutput1.txt'])
with Path('woutput1.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Replace input file
check_call(rpuz + ['vagrant', 'upload',
(tests / 'vagrant/simplevagrant').path,
'%s:arg1' % (tests / 'simple_input2.txt')])
check_call(rpuz + ['vagrant', 'upload',
(tests / 'vagrant/simplevagrant').path])
# Run again
check_simple(rpuz + ['vagrant', 'run', '--no-stdin',
(tests / 'vagrant/simplevagrant').path],
'out', 2)
# Get output file
check_call(rpuz + ['vagrant', 'download',
(tests / 'vagrant/simplevagrant').path,
'arg2:woutput2.txt'])
with Path('woutput2.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '36'
# Reset input file
check_call(rpuz + ['vagrant', 'upload',
(tests / 'vagrant/simplevagrant').path,
':arg1'])
# Run again
check_simple(rpuz + ['vagrant', 'run', '--no-stdin',
(tests / 'vagrant/simplevagrant').path],
'out')
# Get output file
check_call(rpuz + ['vagrant', 'download',
(tests / 'vagrant/simplevagrant').path,
'arg2:voutput1.txt'])
with Path('voutput1.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Destroy
check_call(rpuz + ['vagrant', 'destroy',
(tests / 'vagrant/simplevagrant').path])
elif interactive:
print("Test and press enter")
sys.stdin.readline()
finally:
if (tests / 'vagrant/simplevagrant').exists():
(tests / 'vagrant/simplevagrant').rmtree()
# Unpack Docker
check_call(rpuz + ['docker', 'setup/create', 'simple.rpz', 'simpledocker'])
print("\nDocker project set up in simpledocker")
try:
if run_docker:
check_call(rpuz + ['docker', 'setup/build', 'simpledocker'])
check_simple(rpuz + ['docker', 'run', 'simpledocker'], 'out')
# Get output file
check_call(rpuz + ['docker', 'download', 'simpledocker',
'arg2:doutput1.txt'])
with Path('doutput1.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Replace input file
check_call(rpuz + ['docker', 'upload', 'simpledocker',
'%s:arg1' % (tests / 'simple_input2.txt')])
check_call(rpuz + ['docker', 'upload', 'simpledocker'])
check_call(rpuz + ['showfiles', 'simpledocker'])
# Run again
check_simple(rpuz + ['docker', 'run', 'simpledocker'], 'out', 2)
# Get output file
check_call(rpuz + ['docker', 'download', 'simpledocker',
'arg2:doutput2.txt'])
with Path('doutput2.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '36'
# Reset input file
check_call(rpuz + ['docker', 'upload', 'simpledocker',
':arg1'])
# Run again
check_simple(rpuz + ['docker', 'run', 'simpledocker'], 'out')
# Get output file
check_call(rpuz + ['docker', 'download', 'simpledocker',
'arg2:doutput1.txt'])
with Path('doutput1.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Destroy
check_call(rpuz + ['docker', 'destroy', 'simpledocker'])
elif interactive:
print("Test and press enter")
sys.stdin.readline()
finally:
if Path('simpledocker').exists():
Path('simpledocker').rmtree()
# ########################################
# 'threads' program: testrun
#
# Build
build('threads', ['threads.c'], ['-lpthread'])
# Trace
output = check_output(rpz + ['testrun', './threads'], 'err')
assert any(b'successfully exec\'d /bin/./echo' in l
for l in output.splitlines())
# ########################################
# 'threads2' program: testrun
#
# Build
build('threads2', ['threads2.c'], ['-lpthread'])
# Trace
output = check_output(rpz + ['testrun', './threads2'], 'err')
assert any(b'successfully exec\'d /bin/echo' in l
for l in output.splitlines())
# ########################################
# 'segv' program: testrun
#
# Build
build('segv', ['segv.c'])
# Trace
check_call(rpz + ['testrun', './segv'])
# ########################################
# 'exec_echo' program: trace, pack, run --cmdline
#
# Build
build('exec_echo', ['exec_echo.c'])
# Trace
check_call(rpz + ['trace', './exec_echo', 'originalexecechooutput'])
# Pack
check_call(rpz + ['pack', 'exec_echo.rpz'])
# Unpack chroot
check_call(sudo + rpuz + ['chroot', 'setup',
'exec_echo.rpz', 'echochroot'])
try:
# Run original command-line
output = check_output(sudo + rpuz + ['chroot', 'run',
'echochroot'])
assert output == b'originalexecechooutput\n'
# Prints out command-line
output = check_output(sudo + rpuz + ['chroot', 'run',
'echochroot', '--cmdline'])
assert any(b'./exec_echo originalexecechooutput' == s.strip()
for s in output.split(b'\n'))
# Run with different command-line
output = check_output(sudo + rpuz + [
'chroot', 'run', 'echochroot',
'--cmdline', './exec_echo', 'changedexecechooutput'])
assert output == b'changedexecechooutput\n'
finally:
check_call(sudo + rpuz + ['chroot', 'destroy', 'echochroot'])
# ########################################
# 'exec_echo' program: testrun
# This is built with -m32 so that we transition:
# python (x64) -> exec_echo (i386) -> echo (x64)
#
if sys.maxsize > 2 ** 32:
# Build
build('exec_echo32', ['exec_echo.c'], ['-m32'])
# Trace
check_call(rpz + ['testrun', './exec_echo32 42'])
else:
print("Can't try exec_echo transitions: not running on 64bits")
# ########################################
# Tracing non-existing program
#
check_call(rpz + ['testrun', './doesntexist'])
# ########################################
# 'connect' program: testrun
#
# Build
build('connect', ['connect.c'])
# Trace
err = check_output(rpz + ['testrun', './connect'], 'err')
err = err.split(b'\n')
assert not any(b'program exited with non-zero code' in l for l in err)
assert any(re.search(br'process connected to [0-9.]+:80', l)
for l in err)
# ########################################
# 'vfork' program: testrun
#
# Build
build('vfork', ['vfork.c'])
# Trace
err = check_output(rpz + ['testrun', './vfork'], 'err')
err = err.split(b'\n')
assert not any(b'program exited with non-zero code' in l for l in err)
# ########################################
# 'rename' program: trace
#
# Build
build('rename', ['rename.c'])
# Trace
check_call(rpz + ['trace', '-d', 'rename-trace', './rename'])
with Path('rename-trace/config.yml').open(encoding='utf-8') as fp:
config = yaml.safe_load(fp)
# Check that written files were logged
database = Path.cwd() / 'rename-trace/trace.sqlite3'
if PY3:
# On PY3, connect() only accepts unicode
conn = sqlite3.connect(str(database))
else:
conn = sqlite3.connect(database.path)
conn.row_factory = sqlite3.Row
rows = conn.execute(
'''
SELECT name FROM opened_files
''')
files = set(Path(r[0]) for r in rows)
for n in ('dir1/file', 'dir2/file', 'dir2/brokensymlink', 'dir2/symlink'):
if (Path.cwd() / n) not in files:
raise AssertionError("Missing file: %s" % (Path.cwd() / n))
conn.close()
# Check that created files won't be packed
for f in config.get('other_files'):
if 'dir2' in Path(f).parent.components:
raise AssertionError("Created file shouldn't be packed: %s" %
Path(f))
# ########################################
# Copies back coverage report
#
coverage = Path('.coverage')
if coverage.exists():
coverage.copyfile(tests.parent / '.coverage.runpy')
def functional_tests(raise_warnings, interactive, run_vagrant, run_docker):
# Tests on Python < 2.7.3: need to use separate reprozip Python (with known
# working version of Python)
if sys.version_info < (2, 7, 3):
bug13676 = True
if 'REPROZIP_PYTHON' not in os.environ:
sys.stderr.write("Error: using reprozip with Python %s!\n" %
sys.version.split(' ', 1)[0])
sys.exit(1)
else:
bug13676 = False
rpz = [os.environ.get('REPROZIP_PYTHON', sys.executable)]
rpuz = [os.environ.get('REPROUNZIP_PYTHON', sys.executable)]
# Can't match on the SignalWarning category here because of a Python bug
# http://bugs.python.org/issue22543
if raise_warnings:
rpz.extend(['-W', 'error:signal'])
rpuz.extend(['-W', 'error:signal'])
if 'COVER' in os.environ:
rpz.extend(['-m'] + os.environ['COVER'].split(' '))
rpuz.extend(['-m'] + os.environ['COVER'].split(' '))
reprozip_main = tests.parent / 'reprozip/reprozip/main.py'
reprounzip_main = tests.parent / 'reprounzip/reprounzip/main.py'
verbose = ['-v'] * 3
rpz.extend([reprozip_main.absolute().path] + verbose)
rpuz.extend([reprounzip_main.absolute().path] + verbose)
print("Command lines are:\n%r\n%r" % (rpz, rpuz))
# ########################################
# testrun /bin/echo
#
output = check_output(rpz + ['testrun', '/bin/echo', 'outputhere'])
assert any(b' 1 | /bin/echo outputhere ' in l
for l in output.splitlines())
output = check_output(rpz + ['testrun', '-a', '/fake/path/echo',
'/bin/echo', 'outputhere'])
assert any(b' 1 | (/bin/echo) /fake/path/echo outputhere ' in l
for l in output.splitlines())
# ########################################
# testrun multiple commands
#
check_call(rpz + ['testrun', 'bash', '-c',
'cat ../../../../../etc/passwd;'
'cd /var/lib;'
'cat ../../etc/group'])
check_call(rpz + ['trace',
'bash', '-c', 'cat /etc/passwd;echo'])
check_call(rpz + ['trace', '--continue',
'sh', '-c', 'cat /etc/group;/usr/bin/id'])
check_call(rpz + ['pack'])
if not bug13676:
check_call(rpuz + ['graph', 'graph.dot'])
check_call(rpuz + ['graph', 'graph2.dot', 'experiment.rpz'])
sudo = ['sudo', '-E'] # -E to keep REPROZIP_USAGE_STATS
# ########################################
# 'simple' program: trace, pack, info, unpack
#
def check_simple(args, stream, infile=1):
output = check_output(args, stream).splitlines()
try:
first = output.index(b"Read 6 bytes")
except ValueError:
stderr.write("output = %r\n" % output)
raise
if infile == 1:
assert output[first + 1] == b"a = 29, b = 13"
assert output[first + 2] == b"result = 42"
else: # infile == 2
assert output[first + 1] == b"a = 25, b = 11"
assert output[first + 2] == b"result = 36"
# Build
build('simple', ['simple.c'])
# Trace
check_call(rpz + ['trace', '-d', 'rpz-simple',
'./simple',
(tests / 'simple_input.txt').path,
'simple_output.txt'])
orig_output_location = Path('simple_output.txt').absolute()
assert orig_output_location.is_file()
with orig_output_location.open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
orig_output_location.remove()
# Read config
with Path('rpz-simple/config.yml').open(encoding='utf-8') as fp:
conf = yaml.safe_load(fp)
other_files = set(Path(f).absolute() for f in conf['other_files'])
expected = [Path('simple'), (tests / 'simple_input.txt')]
assert other_files.issuperset([f.resolve() for f in expected])
# Check input and output files
inputs_outputs = conf['inputs_outputs']
# Exactly one input: "arg1", "...simple_input.txt"
# Output: 'arg2', "...simple_output.txt"
# There might be more output files: the C coverage files
found = 0
for fdict in inputs_outputs:
if Path(fdict['path']).name == b'simple_input.txt':
assert fdict['name'] == 'arg1'
assert fdict['read_by_runs'] == [0]
assert not fdict.get('written_by_runs')
found |= 0x01
elif Path(fdict['path']).name == b'simple_output.txt':
assert fdict['name'] == 'arg2'
assert not fdict.get('read_by_runs')
assert fdict['written_by_runs'] == [0]
found |= 0x02
else:
# No other inputs
assert not fdict.get('read_by_runs')
assert found == 0x03
# Pack
check_call(rpz + ['pack', '-d', 'rpz-simple', 'simple.rpz'])
Path('simple').remove()
# Info
check_call(rpuz + ['info', 'simple.rpz'])
# Show files
check_call(rpuz + ['showfiles', 'simple.rpz'])
# Lists packages
check_call(rpuz + ['installpkgs', '--summary', 'simple.rpz'])
# Unpack directory
check_call(rpuz + ['directory', 'setup', 'simple.rpz', 'simpledir'])
# Run directory
check_simple(rpuz + ['directory', 'run', 'simpledir'], 'err')
output_in_dir = join_root(Path('simpledir/root'), orig_output_location)
with output_in_dir.open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Delete with wrong command (should fail)
p = subprocess.Popen(rpuz + ['chroot', 'destroy', 'simpledir'],
stderr=subprocess.PIPE)
out, err = p.communicate()
assert p.poll() != 0
err = err.splitlines()
assert b"Wrong unpacker used" in err[0]
assert err[1].startswith(b"usage: ")
# Delete directory
check_call(rpuz + ['directory', 'destroy', 'simpledir'])
# Unpack chroot
check_call(sudo + rpuz + ['chroot', 'setup', '--bind-magic-dirs',
'simple.rpz', 'simplechroot'])
try:
output_in_chroot = join_root(Path('simplechroot/root'),
orig_output_location)
# Run chroot
check_simple(sudo + rpuz + ['chroot', 'run', 'simplechroot'], 'err')
with output_in_chroot.open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Get output file
check_call(sudo + rpuz + ['chroot', 'download', 'simplechroot',
'arg2:output1.txt'])
with Path('output1.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Replace input file
check_call(sudo + rpuz + ['chroot', 'upload', 'simplechroot',
'%s:arg1' % (tests / 'simple_input2.txt')])
check_call(sudo + rpuz + ['chroot', 'upload', 'simplechroot'])
# Run again
check_simple(sudo + rpuz + ['chroot', 'run', 'simplechroot'], 'err', 2)
with output_in_chroot.open(encoding='utf-8') as fp:
assert fp.read().strip() == '36'
# Reset input file
check_call(sudo + rpuz + ['chroot', 'upload', 'simplechroot', ':arg1'])
# Run again
check_simple(sudo + rpuz + ['chroot', 'run', 'simplechroot'], 'err')
with output_in_chroot.open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Delete with wrong command (should fail)
p = subprocess.Popen(rpuz + ['directory', 'destroy', 'simplechroot'],
stderr=subprocess.PIPE)
out, err = p.communicate()
assert p.poll() != 0
err = err.splitlines()
assert b"Wrong unpacker used" in err[0]
assert err[1].startswith(b"usage:")
finally:
# Delete chroot
check_call(sudo + rpuz + ['chroot', 'destroy', 'simplechroot'])
if not (tests / 'vagrant').exists():
check_call(['sudo', 'sh', '-c',
'mkdir %(d)s; chmod 777 %(d)s' % {'d': tests / 'vagrant'}])
# Unpack Vagrant-chroot
check_call(rpuz + ['vagrant', 'setup/create', '--memory', '512',
'--use-chroot', 'simple.rpz',
(tests / 'vagrant/simplevagrantchroot').path])
print("\nVagrant project set up in simplevagrantchroot")
try:
if run_vagrant:
check_simple(rpuz + ['vagrant', 'run', '--no-stdin',
(tests / 'vagrant/simplevagrantchroot').path],
'out')
# Get output file
check_call(rpuz + ['vagrant', 'download',
(tests / 'vagrant/simplevagrantchroot').path,
'arg2:voutput1.txt'])
with Path('voutput1.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Replace input file
check_call(rpuz + ['vagrant', 'upload',
(tests / 'vagrant/simplevagrantchroot').path,
'%s:arg1' % (tests / 'simple_input2.txt')])
check_call(rpuz + ['vagrant', 'upload',
(tests / 'vagrant/simplevagrantchroot').path])
# Run again
check_simple(rpuz + ['vagrant', 'run', '--no-stdin',
(tests / 'vagrant/simplevagrantchroot').path],
'out', 2)
# Get output file
check_call(rpuz + ['vagrant', 'download',
(tests / 'vagrant/simplevagrantchroot').path,
'arg2:voutput2.txt'])
with Path('voutput2.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '36'
# Reset input file
check_call(rpuz + ['vagrant', 'upload',
(tests / 'vagrant/simplevagrantchroot').path,
':arg1'])
# Run again
check_simple(rpuz + ['vagrant', 'run', '--no-stdin',
(tests / 'vagrant/simplevagrantchroot').path],
'out')
# Get output file
check_call(rpuz + ['vagrant', 'download',
(tests / 'vagrant/simplevagrantchroot').path,
'arg2:voutput1.txt'])
with Path('voutput1.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Destroy
check_call(rpuz + ['vagrant', 'destroy',
(tests / 'vagrant/simplevagrantchroot').path])
elif interactive:
print("Test and press enter")
sys.stdin.readline()
finally:
if (tests / 'vagrant/simplevagrantchroot').exists():
(tests / 'vagrant/simplevagrantchroot').rmtree()
# Unpack Vagrant without chroot
check_call(rpuz + ['vagrant', 'setup/create', '--dont-use-chroot',
'simple.rpz',
(tests / 'vagrant/simplevagrant').path])
print("\nVagrant project set up in simplevagrant")
try:
if run_vagrant:
check_simple(rpuz + ['vagrant', 'run', '--no-stdin',
(tests / 'vagrant/simplevagrant').path],
'out')
# Get output file
check_call(rpuz + ['vagrant', 'download',
(tests / 'vagrant/simplevagrant').path,
'arg2:woutput1.txt'])
with Path('woutput1.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Replace input file
check_call(rpuz + ['vagrant', 'upload',
(tests / 'vagrant/simplevagrant').path,
'%s:arg1' % (tests / 'simple_input2.txt')])
check_call(rpuz + ['vagrant', 'upload',
(tests / 'vagrant/simplevagrant').path])
# Run again
check_simple(rpuz + ['vagrant', 'run', '--no-stdin',
(tests / 'vagrant/simplevagrant').path],
'out', 2)
# Get output file
check_call(rpuz + ['vagrant', 'download',
(tests / 'vagrant/simplevagrant').path,
'arg2:woutput2.txt'])
with Path('woutput2.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '36'
# Reset input file
check_call(rpuz + ['vagrant', 'upload',
(tests / 'vagrant/simplevagrant').path,
':arg1'])
# Run again
check_simple(rpuz + ['vagrant', 'run', '--no-stdin',
(tests / 'vagrant/simplevagrant').path],
'out')
# Get output file
check_call(rpuz + ['vagrant', 'download',
(tests / 'vagrant/simplevagrant').path,
'arg2:voutput1.txt'])
with Path('voutput1.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Destroy
check_call(rpuz + ['vagrant', 'destroy',
(tests / 'vagrant/simplevagrant').path])
elif interactive:
print("Test and press enter")
sys.stdin.readline()
finally:
if (tests / 'vagrant/simplevagrant').exists():
(tests / 'vagrant/simplevagrant').rmtree()
# Unpack Docker
check_call(rpuz + ['docker', 'setup/create', 'simple.rpz', 'simpledocker'])
print("\nDocker project set up in simpledocker")
try:
if run_docker:
check_call(rpuz + ['docker', 'setup/build', 'simpledocker'])
check_simple(rpuz + ['docker', 'run', 'simpledocker'], 'out')
# Get output file
check_call(rpuz + ['docker', 'download', 'simpledocker',
'arg2:doutput1.txt'])
with Path('doutput1.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Replace input file
check_call(rpuz + ['docker', 'upload', 'simpledocker',
'%s:arg1' % (tests / 'simple_input2.txt')])
check_call(rpuz + ['docker', 'upload', 'simpledocker'])
check_call(rpuz + ['showfiles', 'simpledocker'])
# Run again
check_simple(rpuz + ['docker', 'run', 'simpledocker'], 'out', 2)
# Get output file
check_call(rpuz + ['docker', 'download', 'simpledocker',
'arg2:doutput2.txt'])
with Path('doutput2.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '36'
# Reset input file
check_call(rpuz + ['docker', 'upload', 'simpledocker',
':arg1'])
# Run again
check_simple(rpuz + ['docker', 'run', 'simpledocker'], 'out')
# Get output file
check_call(rpuz + ['docker', 'download', 'simpledocker',
'arg2:doutput1.txt'])
with Path('doutput1.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Destroy
check_call(rpuz + ['docker', 'destroy', 'simpledocker'])
elif interactive:
print("Test and press enter")
sys.stdin.readline()
finally:
if Path('simpledocker').exists():
Path('simpledocker').rmtree()
# ########################################
# 'threads' program: testrun
#
# Build
build('threads', ['threads.c'], ['-lpthread'])
# Trace
output = check_output(rpz + ['testrun', './threads'], 'err')
assert any(b'successfully exec\'d /bin/./echo' in l
for l in output.splitlines())
# ########################################
# 'threads2' program: testrun
#
# Build
build('threads2', ['threads2.c'], ['-lpthread'])
# Trace
output = check_output(rpz + ['testrun', './threads2'], 'err')
assert any(b'successfully exec\'d /bin/echo' in l
for l in output.splitlines())
# ########################################
# 'segv' program: testrun
#
# Build
build('segv', ['segv.c'])
# Trace
check_call(rpz + ['testrun', './segv'])
# ########################################
# 'exec_echo' program: trace, pack, run --cmdline
#
# Build
build('exec_echo', ['exec_echo.c'])
# Trace
check_call(rpz + ['trace', './exec_echo', 'originalexecechooutput'])
# Pack
check_call(rpz + ['pack', 'exec_echo.rpz'])
# Unpack chroot
check_call(sudo + rpuz + ['chroot', 'setup',
'exec_echo.rpz', 'echochroot'])
try:
# Run original command-line
output = check_output(sudo + rpuz + ['chroot', 'run',
'echochroot'])
assert output == b'originalexecechooutput\n'
# Prints out command-line
output = check_output(sudo + rpuz + ['chroot', 'run',
'echochroot', '--cmdline'])
assert any(b'./exec_echo originalexecechooutput' == s.strip()
for s in output.split(b'\n'))
# Run with different command-line
output = check_output(sudo + rpuz + [
'chroot', 'run', 'echochroot',
'--cmdline', './exec_echo', 'changedexecechooutput'])
assert output == b'changedexecechooutput\n'
finally:
check_call(sudo + rpuz + ['chroot', 'destroy', 'echochroot'])
# ########################################
# 'exec_echo' program: testrun
# This is built with -m32 so that we transition:
# python (x64) -> exec_echo (i386) -> echo (x64)
#
if sys.maxsize > 2 ** 32:
# Build
build('exec_echo32', ['exec_echo.c'], ['-m32'])
# Trace
check_call(rpz + ['testrun', './exec_echo32 42'])
else:
print("Can't try exec_echo transitions: not running on 64bits")
# ########################################
# Tracing non-existing program
#
check_call(rpz + ['testrun', './doesntexist'])
# ########################################
# 'connect' program: testrun
#
# Build
build('connect', ['connect.c'])
# Trace
err = check_output(rpz + ['testrun', './connect'], 'err')
err = err.split(b'\n')
assert not any(b'program exited with non-zero code' in l for l in err)
assert any(re.search(br'process connected to [0-9.]+:80', l)
for l in err)
# ########################################
# 'vfork' program: testrun
#
# Build
build('vfork', ['vfork.c'])
# Trace
err = check_output(rpz + ['testrun', './vfork'], 'err')
err = err.split(b'\n')
assert not any(b'program exited with non-zero code' in l for l in err)
# ########################################
# 'rename' program: trace
#
# Build
build('rename', ['rename.c'])
# Trace
check_call(rpz + ['trace', '-d', 'rename-trace', './rename'])
with Path('rename-trace/config.yml').open(encoding='utf-8') as fp:
config = yaml.safe_load(fp)
# Check that written files were logged
database = Path.cwd() / 'rename-trace/trace.sqlite3'
if PY3:
# On PY3, connect() only accepts unicode
conn = sqlite3.connect(str(database))
else:
conn = sqlite3.connect(database.path)
conn.row_factory = sqlite3.Row
rows = conn.execute(
'''
SELECT name FROM opened_files
''')
files = set(Path(r[0]) for r in rows)
for n in ('dir1/file', 'dir2/file', 'dir2/brokensymlink', 'dir2/symlink'):
if (Path.cwd() / n) not in files:
raise AssertionError("Missing file: %s" % (Path.cwd() / n))
conn.close()
# Check that created files won't be packed
for f in config.get('other_files'):
if 'dir2' in Path(f).parent.components:
raise AssertionError("Created file shouldn't be packed: %s" %
Path(f))
# ########################################
# Test shebang corner-cases
#
Path('a').symlink('b')
with Path('b').open('w') as fp:
fp.write('#!%s 0\nsome content\n' % (Path.cwd() / 'c'))
Path('b').chmod(0o744)
Path('c').symlink('d')
with Path('d').open('w') as fp:
fp.write('#!e')
Path('d').chmod(0o744)
with Path('e').open('w') as fp:
fp.write('#!/bin/echo')
Path('e').chmod(0o744)
# Trace
out = check_output(rpz + ['trace', '--dont-identify-packages',
'-d', 'shebang-trace', './a', '1', '2'])
out = out.splitlines()[0]
assert out == ('e %s 0 ./a 1 2' % (Path.cwd() / 'c')).encode('ascii')
# Check config
with Path('shebang-trace/config.yml').open(encoding='utf-8') as fp:
config = yaml.safe_load(fp)
other_files = set(Path(f) for f in config['other_files']
if f.startswith('%s/' % Path.cwd()))
# Check database
database = Path.cwd() / 'shebang-trace/trace.sqlite3'
if PY3:
# On PY3, connect() only accepts unicode
conn = sqlite3.connect(str(database))
else:
conn = sqlite3.connect(database.path)
conn.row_factory = sqlite3.Row
rows = conn.execute(
'''
SELECT name FROM opened_files
''')
opened = [Path(r[0]) for r in rows
if r[0].startswith('%s/' % Path.cwd())]
rows = conn.execute(
'''
SELECT name, argv FROM executed_files
''')
executed = [(Path(r[0]), r[1]) for r in rows
if Path(r[0]).lies_under(Path.cwd())]
print("other_files: %r" % sorted(other_files))
print("opened: %r" % opened)
print("executed: %r" % executed)
assert other_files == set(Path.cwd() / p
for p in ('a', 'b', 'c', 'd', 'e'))
if not bug13676:
assert opened == [Path.cwd() / 'c', Path.cwd() / 'e']
assert executed == [(Path.cwd() / 'a', './a\x001\x002\x00')]
# ########################################
# Test old packages
#
old_packages = [
('simple-0.4.0.rpz',
'https://drive.google.com/uc?export=download&id=0B3ucPz7GSthBVG4xZW1V'
'eDhXNTQ'),
('simple-0.6.0.rpz',
'https://drive.google.com/uc?export=download&id=0B3ucPz7GSthBbl9SUjhr'
'cUdtbGs'),
('simple-0.7.1.rpz',
'https://drive.google.com/uc?export=download&id=0B3ucPz7GSthBRGp2Vm5V'
'QVpWOGs'),
]
for name, url in old_packages:
print("Testing old package %s" % name)
f = Path(name)
if not f.exists():
download_file(url, f)
# Info
check_call(rpuz + ['info', name])
# Show files
check_call(rpuz + ['showfiles', name])
# Lists packages
check_call(rpuz + ['installpkgs', '--summary', name])
# Unpack directory
check_call(rpuz + ['directory', 'setup', name, 'simpledir'])
# Run directory
check_simple(rpuz + ['directory', 'run', 'simpledir'], 'err')
output_in_dir = Path('simpledir/root/tmp')
output_in_dir = output_in_dir.listdir('reprozip_*')[0]
output_in_dir = output_in_dir / 'simple_output.txt'
with output_in_dir.open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Delete with wrong command (should fail)
p = subprocess.Popen(rpuz + ['chroot', 'destroy', 'simpledir'],
stderr=subprocess.PIPE)
out, err = p.communicate()
assert p.poll() != 0
err = err.splitlines()
assert b"Wrong unpacker used" in err[0]
assert err[1].startswith(b"usage: ")
# Delete directory
check_call(rpuz + ['directory', 'destroy', 'simpledir'])
# ########################################
# Copies back coverage report
#
coverage = Path('.coverage')
if coverage.exists():
coverage.copyfile(tests.parent / '.coverage.runpy')
def functional_tests(raise_warnings, interactive, run_vagrant, run_docker):
python = [sys.executable]
# Can't match on the SignalWarning category here because of a Python bug
# http://bugs.python.org/issue22543
python.extend(['-W', 'error:signal'])
if 'COVER' in os.environ:
python.extend(['-m'] + os.environ['COVER'].split(' '))
reprozip_main = tests.parent / 'reprozip/reprozip/main.py'
reprounzip_main = tests.parent / 'reprounzip/reprounzip/main.py'
verbose = ['-v'] * 3
rpz = python + [reprozip_main.absolute().path] + verbose
rpuz = python + [reprounzip_main.absolute().path] + verbose
# ########################################
# 'simple' program: trace, pack, info, unpack
#
# Build
build('simple', ['simple.c'])
# Trace
check_call(rpz + [
'trace', '-d', 'rpz-simple', './simple',
(tests / 'simple_input.txt').path, 'simple_output.txt'
])
orig_output_location = Path('simple_output.txt').absolute()
assert orig_output_location.is_file()
with orig_output_location.open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
orig_output_location.remove()
# Read config
with Path('rpz-simple/config.yml').open(encoding='utf-8') as fp:
conf = yaml.safe_load(fp)
other_files = set(Path(f).absolute() for f in conf['other_files'])
expected = [Path('simple'), (tests / 'simple_input.txt')]
assert other_files.issuperset([f.resolve() for f in expected])
# Check input and output files
input_files = conf['runs'][0]['input_files']
assert (dict((k, Path(f).name) for k, f in iteritems(input_files)) == {
'arg': b'simple_input.txt'
})
output_files = conf['runs'][0]['output_files']
print(dict((k, Path(f).name) for k, f in iteritems(output_files)))
# Here we don't test for dict equality, since we might have C coverage
# files in the mix
assert Path(output_files['arg']).name == b'simple_output.txt'
# Pack
check_call(rpz + ['pack', '-d', 'rpz-simple', 'simple.rpz'])
Path('simple').remove()
# Info
check_call(rpuz + ['info', 'simple.rpz'])
# Show files
check_call(rpuz + ['showfiles', 'simple.rpz'])
# Lists packages
check_call(rpuz + ['installpkgs', '--summary', 'simple.rpz'])
# Unpack directory
check_call(rpuz + ['directory', 'setup', 'simple.rpz', 'simpledir'])
# Run directory
check_call(rpuz + ['directory', 'run', 'simpledir'])
output_in_dir = join_root(Path('simpledir/root'), orig_output_location)
with output_in_dir.open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Delete with wrong command (should fail)
assert call(rpuz + ['chroot', 'destroy', 'simpledir']) != 0
# Delete directory
check_call(rpuz + ['directory', 'destroy', 'simpledir'])
# Unpack chroot
check_call(
['sudo'] + rpuz +
['chroot', 'setup', '--bind-magic-dirs', 'simple.rpz', 'simplechroot'])
# Run chroot
check_call(['sudo'] + rpuz + ['chroot', 'run', 'simplechroot'])
output_in_chroot = join_root(Path('simplechroot/root'),
orig_output_location)
with output_in_chroot.open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Get output file
check_call(['sudo'] + rpuz +
['chroot', 'download', 'simplechroot', 'arg:output1.txt'])
with Path('output1.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Replace input file
check_call(['sudo'] + rpuz + [
'chroot', 'upload', 'simplechroot',
'%s:arg' % (tests / 'simple_input2.txt')
])
check_call(['sudo'] + rpuz + ['chroot', 'upload', 'simplechroot'])
# Run again
check_call(['sudo'] + rpuz + ['chroot', 'run', 'simplechroot'])
output_in_chroot = join_root(Path('simplechroot/root'),
orig_output_location)
with output_in_chroot.open(encoding='utf-8') as fp:
assert fp.read().strip() == '36'
# Delete with wrong command (should fail)
assert call(rpuz + ['directory', 'destroy', 'simplechroot']) != 0
# Delete chroot
check_call(['sudo'] + rpuz + ['chroot', 'destroy', 'simplechroot'])
if not Path('/vagrant').exists():
check_call(['sudo', 'sh', '-c', 'mkdir /vagrant; chmod 777 /vagrant'])
# Unpack Vagrant-chroot
check_call(rpuz + [
'vagrant', 'setup/create', '--use-chroot', 'simple.rpz',
'/vagrant/simplevagrantchroot'
])
print("\nVagrant project set up in simplevagrantchroot")
try:
if run_vagrant:
check_call(rpuz + [
'vagrant', 'run', '--no-stdin', '/vagrant/simplevagrantchroot'
])
# Destroy
check_call(rpuz +
['vagrant', 'destroy', '/vagrant/simplevagrantchroot'])
elif interactive:
print("Test and press enter")
sys.stdin.readline()
finally:
if Path('/vagrant/simplevagrantchroot').exists():
Path('/vagrant/simplevagrantchroot').rmtree()
# Unpack Vagrant without chroot
check_call(rpuz + [
'vagrant', 'setup/create', '--dont-use-chroot', 'simple.rpz',
'/vagrant/simplevagrant'
])
print("\nVagrant project set up in simplevagrant")
try:
if run_vagrant:
check_call(
rpuz +
['vagrant', 'run', '--no-stdin', '/vagrant/simplevagrant'])
# Destroy
check_call(rpuz + ['vagrant', 'destroy', '/vagrant/simplevagrant'])
elif interactive:
print("Test and press enter")
sys.stdin.readline()
finally:
if Path('/vagrant/simplevagrant').exists():
Path('/vagrant/simplevagrant').rmtree()
# Unpack Docker
check_call(rpuz + ['docker', 'setup/create', 'simple.rpz', 'simpledocker'])
print("\nDocker project set up in simpledocker")
try:
if run_docker:
check_call(rpuz + ['docker', 'setup/build', 'simpledocker'])
check_call(rpuz + ['docker', 'run', 'simpledocker'])
# Get output file
check_call(
rpuz +
['docker', 'download', 'simpledocker', 'arg:doutput1.txt'])
with Path('doutput1.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Replace input file
check_call(rpuz + [
'docker', 'upload', 'simpledocker',
'%s:arg' % (tests / 'simple_input2.txt')
])
check_call(rpuz + ['docker', 'upload', 'simpledocker'])
check_call(rpuz + ['showfiles', 'simpledocker'])
# Run again
check_call(rpuz + ['docker', 'run', 'simpledocker'])
# Get output file
check_call(
rpuz +
['docker', 'download', 'simpledocker', 'arg:doutput2.txt'])
with Path('doutput2.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '36'
# Destroy
check_call(rpuz + ['docker', 'destroy', 'simpledocker'])
elif interactive:
print("Test and press enter")
sys.stdin.readline()
finally:
if Path('simpledocker').exists():
Path('simpledocker').rmtree()
# ########################################
# 'threads' program: testrun
#
# Build
build('threads', ['threads.c'], ['-lpthread'])
# Trace
check_call(rpz + ['testrun', './threads'])
# ########################################
# 'segv' program: testrun
#
# Build
build('segv', ['segv.c'])
# Trace
check_call(rpz + ['testrun', './segv'])
# ########################################
# 'exec_echo' program: trace, pack, run --cmdline
#
# Build
build('exec_echo', ['exec_echo.c'])
# Trace
check_call(rpz + ['trace', './exec_echo', 'originalexecechooutput'])
# Pack
check_call(rpz + ['pack', 'exec_echo.rpz'])
# Unpack chroot
check_call(['sudo'] + rpuz +
['chroot', 'setup', 'exec_echo.rpz', 'echochroot'])
try:
# Run original command-line
output = check_output(['sudo'] + rpuz +
['chroot', 'run', 'echochroot'])
assert output == b'originalexecechooutput\n'
# Prints out command-line
output = check_output(['sudo'] + rpuz +
['chroot', 'run', 'echochroot', '--cmdline'])
assert any(b'./exec_echo originalexecechooutput' == s.strip()
for s in output.split(b'\n'))
# Run with different command-line
output = check_output(['sudo'] + rpuz + [
'chroot', 'run', 'echochroot', '--cmdline', './exec_echo',
'changedexecechooutput'
])
assert output == b'changedexecechooutput\n'
finally:
check_call(['sudo'] + rpuz + ['chroot', 'destroy', 'echochroot'])
# ########################################
# 'exec_echo' program: testrun
# This is built with -m32 so that we transition:
# python (x64) -> exec_echo (i386) -> echo (x64)
#
if sys.maxsize > 2**32:
# Build
build('exec_echo32', ['exec_echo.c'], ['-m32'])
# Trace
check_call(rpz + ['testrun', './exec_echo32 42'])
else:
print("Can't try exec_echo transitions: not running on 64bits")
# ########################################
# Tracing non-existing program
#
check_call(rpz + ['testrun', './doesntexist'])
# ########################################
# 'connect' program: testrun
#
# Build
build('connect', ['connect.c'])
# Trace
stderr = check_errout(rpz + ['testrun', './connect'])
stderr = stderr.split(b'\n')
assert not any(b'program exited with non-zero code' in l for l in stderr)
assert any(
re.search(br'process connected to [0-9.]+:80', l) for l in stderr)
# ########################################
# Copies back coverage report
#
coverage = Path('.coverage')
if coverage.exists():
coverage.copyfile(tests.parent / '.coverage.runpy')
def get_files(conn):
"""Find all the files used by the experiment by reading the trace.
"""
files = {}
access_files = [set()]
# Finds run timestamps, so we can sort input/output files by run
proc_cursor = conn.cursor()
executions = proc_cursor.execute(
'''
SELECT timestamp
FROM processes
WHERE parent ISNULL
ORDER BY id;
''')
run_timestamps = [r_timestamp for r_timestamp, in executions][1:]
proc_cursor.close()
# Adds dynamic linkers
for libdir in (Path('/lib'), Path('/lib64')):
if libdir.exists():
for linker in libdir.listdir('*ld-linux*'):
for filename in find_all_links(linker, True):
if filename not in files:
f = TracedFile(filename)
f.read()
files[f.path] = f
# Adds executed files
exec_cursor = conn.cursor()
executed_files = exec_cursor.execute(
'''
SELECT name, timestamp
FROM executed_files
ORDER BY timestamp;
''')
executed = set()
# ... and opened files
open_cursor = conn.cursor()
opened_files = open_cursor.execute(
'''
SELECT name, mode, timestamp
FROM opened_files
ORDER BY timestamp;
''')
# Loop on both lists at once
rows = heapq.merge(((r[1], 'exec', r) for r in executed_files),
((r[2], 'open', r) for r in opened_files))
for ts, event_type, data in rows:
if event_type == 'exec':
r_name, r_timestamp = data
r_mode = FILE_READ
else: # event_type == 'open'
r_name, r_mode, r_timestamp = data
r_name = Path(r_name)
if event_type == 'exec':
executed.add(r_name)
# Stays on the current run
while run_timestamps and r_timestamp > run_timestamps[0]:
del run_timestamps[0]
access_files.append(set())
# Adds symbolic links as read files
for filename in find_all_links(r_name, False):
if filename not in files:
f = TracedFile(filename)
f.read()
files[f.path] = f
# Adds final target
r_name = r_name.resolve()
if r_name not in files:
f = TracedFile(r_name)
files[f.path] = f
else:
f = files[r_name]
if r_mode & FILE_WRITE:
f.write()
elif r_mode & FILE_READ:
f.read()
# Identifies input files
if r_name.is_file() and r_name not in executed:
access_files[-1].add(f)
exec_cursor.close()
open_cursor.close()
# Further filters input files
inputs = [[fi.path
for fi in lst
# Input files are regular files,
if fi.path.is_file() and
# ONLY_READ,
fi.what == TracedFile.ONLY_READ and
# not executable,
# FIXME : currently disabled. Maybe only remove executed files?
# not fi.path.stat().st_mode & 0b111 and
# not in a system directory
not any(fi.path.lies_under(m)
for m in magic_dirs + system_dirs)]
for lst in access_files]
# Identify output files
outputs = [[fi.path
for fi in lst
# Output files are regular files,
if fi.path.is_file() and
# WRITTEN
fi.what == TracedFile.WRITTEN and
# not in a system directory
not any(fi.path.lies_under(m)
for m in magic_dirs + system_dirs)]
for lst in access_files]
# Displays a warning for READ_THEN_WRITTEN files
read_then_written_files = [
fi
for fi in itervalues(files)
if fi.what == TracedFile.READ_THEN_WRITTEN and
not any(fi.path.lies_under(m) for m in magic_dirs)]
if read_then_written_files:
logging.warning(
"Some files were read and then written. We will only pack the "
"final version of the file; reproducible experiments "
"shouldn't change their input files:\n%s",
", ".join(unicode_(fi.path) for fi in read_then_written_files))
files = set(
fi
for fi in itervalues(files)
if fi.what != TracedFile.WRITTEN and not any(fi.path.lies_under(m)
for m in magic_dirs))
return files, inputs, outputs
def write_configuration(directory, sort_packages, overwrite=False):
"""Writes the canonical YAML configuration file.
"""
database = directory / 'trace.sqlite3'
if PY3:
# On PY3, connect() only accepts unicode
conn = sqlite3.connect(str(database))
else:
conn = sqlite3.connect(database.path)
conn.row_factory = sqlite3.Row
# Reads info from database
files, inputs, outputs = get_files(conn)
# Identifies which file comes from which package
if sort_packages:
files, packages = identify_packages(files)
else:
packages = []
# Makes sure all the directories used as working directories are packed
# (they already do if files from them are used, but empty directories do
# not get packed inside a tar archive)
files.update(d for d in list_directories(conn) if d.path.is_dir())
# Writes configuration file
config = directory / 'config.yml'
distribution = platform.linux_distribution()[0:2]
oldconfig = not overwrite and config.exists()
cur = conn.cursor()
if not oldconfig:
runs = []
# This gets all the top-level processes (p.parent ISNULL) and the first
# executed file for that process (sorting by ids, which are
# chronological)
executions = cur.execute(
'''
SELECT e.name, e.argv, e.envp, e.workingdir, p.exitcode
FROM processes p
JOIN executed_files e ON e.id=(
SELECT id FROM executed_files e2
WHERE e2.process=p.id
ORDER BY e2.id
LIMIT 1
)
WHERE p.parent ISNULL;
''')
else:
# Loads in previous config
runs, oldpkgs, oldfiles, patterns = load_config(config,
canonical=False,
File=TracedFile)
# Here, additional patterns are discarded
# Same query as previous block but only gets last process
executions = cur.execute(
'''
SELECT e.name, e.argv, e.envp, e.workingdir, p.exitcode
FROM processes p
JOIN executed_files e ON e.id=(
SELECT id FROM executed_files e2
WHERE e2.process=p.id
ORDER BY e2.id
LIMIT 1
)
WHERE p.parent ISNULL
ORDER BY p.id DESC
LIMIT 1;
''')
inputs = inputs[-1:]
outputs = outputs[-1:]
files, packages = merge_files(files, packages,
oldfiles,
oldpkgs)
for ((r_name, r_argv, r_envp, r_workingdir, r_exitcode),
input_files, output_files) in izip(executions, inputs, outputs):
# Decodes command-line
argv = r_argv.split('\0')
if not argv[-1]:
argv = argv[:-1]
# Decodes environment
envp = r_envp.split('\0')
if not envp[-1]:
envp = envp[:-1]
environ = dict(v.split('=', 1) for v in envp)
# Gets files from command-line
command_line_files = {}
for i, arg in enumerate(argv):
p = Path(r_workingdir, arg).resolve()
if p.is_file():
command_line_files[p] = i
input_files_on_cmdline = sum(1
for in_file in input_files
if in_file in command_line_files)
output_files_on_cmdline = sum(1
for out_file in output_files
if out_file in command_line_files)
# Labels input files
input_files_dict = {}
for in_file in input_files:
# If file is on the command-line
if in_file in command_line_files:
if input_files_on_cmdline > 1:
label = "arg_%d" % command_line_files[in_file]
else:
label = "arg"
# Else, use file's name
else:
label = in_file.unicodename
# Make labels unique
uniquelabel = label
i = 1
while uniquelabel in input_files_dict:
i += 1
uniquelabel = '%s_%d' % (label, i)
input_files_dict[uniquelabel] = str(in_file)
# TODO : Note that right now, we keep as input files the ones that
# don't appear on the command-line
# Labels output files
output_files_dict = {}
for out_file in output_files:
# If file is on the command-line
if out_file in command_line_files:
if output_files_on_cmdline > 1:
label = "arg_%d" % command_line_files[out_file]
else:
label = "arg"
# Else, use file's name
else:
label = out_file.unicodename
# Make labels unique
uniquelabel = label
i = 1
while uniquelabel in output_files_dict:
i += 1
uniquelabel = '%s_%d' % (label, i)
output_files_dict[uniquelabel] = str(out_file)
# TODO : Note that right now, we keep as output files the ones that
# don't appear on the command-line
runs.append({'binary': r_name, 'argv': argv,
'workingdir': Path(r_workingdir).path,
'architecture': platform.machine().lower(),
'distribution': distribution,
'hostname': platform.node(),
'system': [platform.system(), platform.release()],
'environ': environ,
'uid': os.getuid(),
'gid': os.getgid(),
'signal' if r_exitcode & 0x0100 else 'exitcode':
r_exitcode & 0xFF,
'input_files': input_files_dict,
'output_files': output_files_dict})
cur.close()
conn.close()
save_config(config, runs, packages, files, reprozip_version)
print("Configuration file written in {0!s}".format(config))
print("Edit that file then run the packer -- "
"use 'reprozip pack -h' for help")
def functional_tests(raise_warnings, interactive, run_vagrant, run_docker):
rpz_python = [os.environ.get('REPROZIP_PYTHON', sys.executable)]
rpuz_python = [os.environ.get('REPROUNZIP_PYTHON', sys.executable)]
# Can't match on the SignalWarning category here because of a Python bug
# http://bugs.python.org/issue22543
if raise_warnings:
rpz_python.extend(['-W', 'error:signal'])
rpuz_python.extend(['-W', 'error:signal'])
if 'COVER' in os.environ:
rpz_python.extend(['-m'] + os.environ['COVER'].split(' '))
rpuz_python.extend(['-m'] + os.environ['COVER'].split(' '))
reprozip_main = tests.parent / 'reprozip/reprozip/main.py'
reprounzip_main = tests.parent / 'reprounzip/reprounzip/main.py'
verbose = ['-v'] * 3
rpz = rpz_python + [reprozip_main.absolute().path] + verbose
rpuz = rpuz_python + [reprounzip_main.absolute().path] + verbose
print("Command lines are:\n%r\n%r" % (rpz, rpuz))
# ########################################
# testrun /bin/echo
#
output = check_output(rpz + ['testrun', '/bin/echo', 'outputhere'])
assert any(b' 1 | /bin/echo outputhere ' in l for l in output.splitlines())
output = check_output(
rpz + ['testrun', '-a', '/fake/path/echo', '/bin/echo', 'outputhere'])
assert any(b' 1 | (/bin/echo) /fake/path/echo outputhere ' in l
for l in output.splitlines())
# ########################################
# testrun multiple commands
#
check_call(rpz + [
'testrun', 'bash', '-c', 'cat ../../../../../etc/passwd;'
'cd /var/lib;'
'cat ../../etc/group'
])
check_call(rpz +
['trace', '--overwrite', 'bash', '-c', 'cat /etc/passwd;echo'])
check_call(
rpz +
['trace', '--continue', 'sh', '-c', 'cat /etc/group;/usr/bin/id'])
check_call(rpz + ['pack'])
check_call(rpuz + ['graph', 'graph.dot'])
check_call(rpuz + ['graph', 'graph2.dot', 'experiment.rpz'])
sudo = ['sudo', '-E'] # -E to keep REPROZIP_USAGE_STATS
# ########################################
# 'simple' program: trace, pack, info, unpack
#
def check_simple(args, stream, infile=1):
output = check_output(args, stream).splitlines()
try:
first = output.index(b"Read 6 bytes")
except ValueError:
stderr.write("output = %r\n" % output)
raise
if infile == 1:
assert output[first + 1] == b"a = 29, b = 13"
assert output[first + 2] == b"result = 42"
else: # infile == 2
assert output[first + 1] == b"a = 25, b = 11"
assert output[first + 2] == b"result = 36"
# Build
build('simple', ['simple.c'])
# Trace
check_call(rpz + [
'trace', '--overwrite', '-d', 'rpz-simple', './simple',
(tests / 'simple_input.txt').path, 'simple_output.txt'
])
orig_output_location = Path('simple_output.txt').absolute()
assert orig_output_location.is_file()
with orig_output_location.open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
orig_output_location.remove()
# Read config
with Path('rpz-simple/config.yml').open(encoding='utf-8') as fp:
conf = yaml.safe_load(fp)
other_files = set(Path(f).absolute() for f in conf['other_files'])
expected = [Path('simple'), (tests / 'simple_input.txt')]
assert other_files.issuperset([f.resolve() for f in expected])
# Check input and output files
inputs_outputs = conf['inputs_outputs']
# Exactly one input: "arg1", "...simple_input.txt"
# Output: 'arg2', "...simple_output.txt"
# There might be more output files: the C coverage files
found = 0
for fdict in inputs_outputs:
if Path(fdict['path']).name == b'simple_input.txt':
assert fdict['name'] == 'arg1'
assert fdict['read_by_runs'] == [0]
assert not fdict.get('written_by_runs')
found |= 0x01
elif Path(fdict['path']).name == b'simple_output.txt':
assert fdict['name'] == 'arg2'
assert not fdict.get('read_by_runs')
assert fdict['written_by_runs'] == [0]
found |= 0x02
else:
# No other inputs
assert not fdict.get('read_by_runs')
assert found == 0x03
# Pack
check_call(rpz + ['pack', '-d', 'rpz-simple', 'simple.rpz'])
Path('simple').rename('simple.orig')
# Info
check_call(rpuz + ['info', 'simple.rpz'])
# Show files
check_call(rpuz + ['showfiles', 'simple.rpz'])
# Lists packages
check_call(rpuz + ['installpkgs', '--summary', 'simple.rpz'])
# Unpack directory
check_call(rpuz + ['directory', 'setup', 'simple.rpz', 'simpledir'])
# Run directory
check_simple(rpuz + ['directory', 'run', 'simpledir'], 'err')
output_in_dir = join_root(Path('simpledir/root'), orig_output_location)
with output_in_dir.open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Delete with wrong command (should fail)
p = subprocess.Popen(rpuz + ['chroot', 'destroy', 'simpledir'],
stderr=subprocess.PIPE)
out, err = p.communicate()
assert p.poll() != 0
err = err.splitlines()
assert b"Wrong unpacker used" in err[0]
assert err[1].startswith(b"usage: ")
# Delete directory
check_call(rpuz + ['directory', 'destroy', 'simpledir'])
# Unpack chroot
check_call(
sudo + rpuz +
['chroot', 'setup', '--bind-magic-dirs', 'simple.rpz', 'simplechroot'])
try:
output_in_chroot = join_root(Path('simplechroot/root'),
orig_output_location)
# Run chroot
check_simple(sudo + rpuz + ['chroot', 'run', 'simplechroot'], 'err')
with output_in_chroot.open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Get output file
check_call(sudo + rpuz +
['chroot', 'download', 'simplechroot', 'arg2:output1.txt'])
with Path('output1.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Get random file
check_call(sudo + rpuz + [
'chroot', 'download', 'simplechroot',
'%s:binc.bin' % (Path.cwd() / 'simple')
])
assert same_files('simple.orig', 'binc.bin')
# Replace input file
check_call(sudo + rpuz + [
'chroot', 'upload', 'simplechroot',
'%s:arg1' % (tests / 'simple_input2.txt')
])
check_call(sudo + rpuz + ['chroot', 'upload', 'simplechroot'])
# Run again
check_simple(sudo + rpuz + ['chroot', 'run', 'simplechroot'], 'err', 2)
with output_in_chroot.open(encoding='utf-8') as fp:
assert fp.read().strip() == '36'
# Reset input file
check_call(sudo + rpuz + ['chroot', 'upload', 'simplechroot', ':arg1'])
# Run again
check_simple(sudo + rpuz + ['chroot', 'run', 'simplechroot'], 'err')
with output_in_chroot.open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Replace input file via path
check_call(sudo + rpuz + [
'chroot', 'upload', 'simplechroot',
'%s:%s' % (tests / 'simple_input2.txt', tests / 'simple_input.txt')
])
check_call(sudo + rpuz + ['chroot', 'upload', 'simplechroot'])
# Run again
check_simple(sudo + rpuz + ['chroot', 'run', 'simplechroot'], 'err', 2)
# Delete with wrong command (should fail)
p = subprocess.Popen(rpuz + ['directory', 'destroy', 'simplechroot'],
stderr=subprocess.PIPE)
out, err = p.communicate()
assert p.poll() != 0
err = err.splitlines()
assert b"Wrong unpacker used" in err[0]
assert err[1].startswith(b"usage:")
finally:
# Delete chroot
check_call(sudo + rpuz + ['chroot', 'destroy', 'simplechroot'])
# Use reprounzip-vistrails with chroot
check_call(sudo + rpuz + [
'chroot', 'setup', '--bind-magic-dirs', 'simple.rpz', 'simplechroot_vt'
])
try:
output_in_chroot = join_root(Path('simplechroot_vt/root'),
orig_output_location)
# Run using reprounzip-vistrails
check_simple(
sudo + rpuz_python + [
'-m', 'reprounzip.plugins.vistrails', '1', 'chroot',
'simplechroot_vt', '0', '--input-file',
'arg1:%s' % (tests / 'simple_input2.txt'), '--output-file',
'arg2:output_vt.txt'
], 'err', 2)
with output_in_chroot.open(encoding='utf-8') as fp:
assert fp.read().strip() == '36'
finally:
# Delete chroot
check_call(sudo + rpuz + ['chroot', 'destroy', 'simplechroot_vt'])
if not (tests / 'vagrant').exists():
check_call([
'sudo', 'sh', '-c',
'mkdir %(d)s; chmod 777 %(d)s' % {
'd': tests / 'vagrant'
}
])
# Unpack Vagrant-chroot
check_call(rpuz + [
'vagrant', 'setup/create', '--memory', '512', '--use-chroot',
'simple.rpz', (tests / 'vagrant/simplevagrantchroot').path
])
print("\nVagrant project set up in simplevagrantchroot")
try:
if run_vagrant:
check_simple(
rpuz + [
'vagrant', 'run', '--no-stdin',
(tests / 'vagrant/simplevagrantchroot').path
], 'out')
# Get output file
check_call(rpuz + [
'vagrant', 'download', (tests / 'vagrant/simplevagrantchroot'
).path, 'arg2:voutput1.txt'
])
with Path('voutput1.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Get random file
check_call(rpuz + [
'vagrant', 'download', (tests /
'vagrant/simplevagrantchroot').path,
'%s:binvc.bin' % (Path.cwd() / 'simple')
])
assert same_files('simple.orig', 'binvc.bin')
# Replace input file
check_call(rpuz + [
'vagrant', 'upload', (tests /
'vagrant/simplevagrantchroot').path,
'%s:arg1' % (tests / 'simple_input2.txt')
])
check_call(rpuz + [
'vagrant', 'upload', (tests /
'vagrant/simplevagrantchroot').path
])
# Run again
check_simple(
rpuz + [
'vagrant', 'run', '--no-stdin',
(tests / 'vagrant/simplevagrantchroot').path
], 'out', 2)
# Get output file
check_call(rpuz + [
'vagrant', 'download', (tests / 'vagrant/simplevagrantchroot'
).path, 'arg2:voutput2.txt'
])
with Path('voutput2.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '36'
# Reset input file
check_call(rpuz + [
'vagrant', 'upload',
(tests / 'vagrant/simplevagrantchroot').path, ':arg1'
])
# Run again
check_simple(
rpuz + [
'vagrant', 'run', '--no-stdin',
(tests / 'vagrant/simplevagrantchroot').path
], 'out')
# Get output file
check_call(rpuz + [
'vagrant', 'download', (tests / 'vagrant/simplevagrantchroot'
).path, 'arg2:voutput1.txt'
])
with Path('voutput1.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Replace input file via path
check_call(rpuz + [
'vagrant', 'upload', (tests /
'vagrant/simplevagrantchroot').path,
'%s:%s' %
(tests / 'simple_input2.txt', tests / 'simple_input.txt')
])
# Run again
check_simple(
rpuz + [
'vagrant', 'run', '--no-stdin',
(tests / 'vagrant/simplevagrantchroot').path
], 'out', 2)
# Destroy
check_call(rpuz + [
'vagrant', 'destroy', (tests /
'vagrant/simplevagrantchroot').path
])
elif interactive:
print("Test and press enter")
sys.stdin.readline()
finally:
if (tests / 'vagrant/simplevagrantchroot').exists():
(tests / 'vagrant/simplevagrantchroot').rmtree()
# Unpack Vagrant without chroot
check_call(rpuz + [
'vagrant', 'setup/create', '--dont-use-chroot', 'simple.rpz',
(tests / 'vagrant/simplevagrant').path
])
print("\nVagrant project set up in simplevagrant")
try:
if run_vagrant:
check_simple(
rpuz + [
'vagrant', 'run', '--no-stdin',
(tests / 'vagrant/simplevagrant').path
], 'out')
# Get output file
check_call(rpuz + [
'vagrant', 'download',
(tests / 'vagrant/simplevagrant').path, 'arg2:woutput1.txt'
])
with Path('woutput1.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Get random file
check_call(rpuz + [
'vagrant', 'download', (tests / 'vagrant/simplevagrant').path,
'%s:binvs.bin' % (Path.cwd() / 'simple')
])
assert same_files('simple.orig', 'binvs.bin')
# Replace input file
check_call(rpuz + [
'vagrant', 'upload', (tests / 'vagrant/simplevagrant').path,
'%s:arg1' % (tests / 'simple_input2.txt')
])
check_call(
rpuz +
['vagrant', 'upload', (tests / 'vagrant/simplevagrant').path])
# Run again
check_simple(
rpuz + [
'vagrant', 'run', '--no-stdin',
(tests / 'vagrant/simplevagrant').path
], 'out', 2)
# Get output file
check_call(rpuz + [
'vagrant', 'download',
(tests / 'vagrant/simplevagrant').path, 'arg2:woutput2.txt'
])
with Path('woutput2.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '36'
# Reset input file
check_call(rpuz + [
'vagrant', 'upload', (tests /
'vagrant/simplevagrant').path, ':arg1'
])
# Run again
check_simple(
rpuz + [
'vagrant', 'run', '--no-stdin',
(tests / 'vagrant/simplevagrant').path
], 'out')
# Get output file
check_call(rpuz + [
'vagrant', 'download',
(tests / 'vagrant/simplevagrant').path, 'arg2:voutput1.txt'
])
with Path('voutput1.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Destroy
check_call(
rpuz +
['vagrant', 'destroy', (tests / 'vagrant/simplevagrant').path])
elif interactive:
print("Test and press enter")
sys.stdin.readline()
finally:
if (tests / 'vagrant/simplevagrant').exists():
(tests / 'vagrant/simplevagrant').rmtree()
# Unpack Docker
check_call(rpuz + ['docker', 'setup/create', 'simple.rpz', 'simpledocker'])
print("\nDocker project set up in simpledocker")
try:
if run_docker:
check_call(rpuz + ['docker', 'setup/build', 'simpledocker'])
check_simple(rpuz + ['docker', 'run', 'simpledocker'], 'out')
# Get output file
check_call(
rpuz +
['docker', 'download', 'simpledocker', 'arg2:doutput1.txt'])
with Path('doutput1.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Get random file
check_call(rpuz + [
'docker', 'download', 'simpledocker',
'%s:bind.bin' % (Path.cwd() / 'simple')
])
assert same_files('simple.orig', 'bind.bin')
# Replace input file
check_call(rpuz + [
'docker', 'upload', 'simpledocker',
'%s:arg1' % (tests / 'simple_input2.txt')
])
check_call(rpuz + ['docker', 'upload', 'simpledocker'])
check_call(rpuz + ['showfiles', 'simpledocker'])
# Run again
check_simple(rpuz + ['docker', 'run', 'simpledocker'], 'out', 2)
# Get output file
check_call(
rpuz +
['docker', 'download', 'simpledocker', 'arg2:doutput2.txt'])
with Path('doutput2.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '36'
# Reset input file
check_call(rpuz + ['docker', 'upload', 'simpledocker', ':arg1'])
# Run again
check_simple(rpuz + ['docker', 'run', 'simpledocker'], 'out')
# Get output file
check_call(
rpuz +
['docker', 'download', 'simpledocker', 'arg2:doutput1.txt'])
with Path('doutput1.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Replace input file via path
check_call(rpuz + [
'docker', 'upload', 'simpledocker',
'%s:%s' %
(tests / 'simple_input2.txt', tests / 'simple_input.txt')
])
# Run again
check_simple(rpuz + ['docker', 'run', 'simpledocker'], 'out', 2)
# Destroy
check_call(rpuz + ['docker', 'destroy', 'simpledocker'])
elif interactive:
print("Test and press enter")
sys.stdin.readline()
finally:
if Path('simpledocker').exists():
Path('simpledocker').rmtree()
# ########################################
# 'threads' program: testrun
#
# Build
build('threads', ['threads.c'], ['-lpthread'])
# Trace
output = check_output(rpz + ['testrun', './threads'], 'err')
assert any(b'successfully exec\'d /bin/./echo' in l
for l in output.splitlines())
# ########################################
# 'threads2' program: testrun
#
# Build
build('threads2', ['threads2.c'], ['-lpthread'])
# Trace
output = check_output(rpz + ['testrun', './threads2'], 'err')
assert any(b'successfully exec\'d /bin/echo' in l
for l in output.splitlines())
# ########################################
# 'segv' program: testrun
#
# Build
build('segv', ['segv.c'])
# Trace
check_call(rpz + ['testrun', './segv'])
# ########################################
# 'exec_echo' program: trace, pack, run --cmdline
#
# Build
build('exec_echo', ['exec_echo.c'])
# Trace
check_call(
rpz +
['trace', '--overwrite', './exec_echo', 'originalexecechooutput'])
# Pack
check_call(rpz + ['pack', 'exec_echo.rpz'])
# Unpack chroot
check_call(sudo + rpuz +
['chroot', 'setup', 'exec_echo.rpz', 'echochroot'])
try:
# Run original command-line
output = check_output(sudo + rpuz + ['chroot', 'run', 'echochroot'])
assert output == b'originalexecechooutput\n'
# Prints out command-line
output = check_output(sudo + rpuz +
['chroot', 'run', 'echochroot', '--cmdline'])
assert any(b'./exec_echo originalexecechooutput' == s.strip()
for s in output.split(b'\n'))
# Run with different command-line
output = check_output(sudo + rpuz + [
'chroot', 'run', 'echochroot', '--cmdline', './exec_echo',
'changedexecechooutput'
])
assert output == b'changedexecechooutput\n'
finally:
check_call(sudo + rpuz + ['chroot', 'destroy', 'echochroot'])
# ########################################
# 'exec_echo' program: testrun
# This is built with -m32 so that we transition:
# python (x64) -> exec_echo (i386) -> echo (x64)
#
if sys.maxsize > 2**32:
# Build
build('exec_echo32', ['exec_echo.c'], ['-m32'])
# Trace
check_call(rpz + ['testrun', './exec_echo32', '42'])
else:
print("Can't try exec_echo transitions: not running on 64bits")
# ########################################
# Tracing non-existing program
#
ret = call(rpz + ['testrun', './doesntexist'])
assert ret == 127
# ########################################
# 'connect' program: testrun
#
# Build
build('connect', ['connect.c'])
# Trace
err = check_output(rpz + ['testrun', './connect'], 'err')
err = err.split(b'\n')
assert not any(b'program exited with non-zero code' in l for l in err)
assert any(re.search(br'process connected to [0-9.]+:80', l) for l in err)
# ########################################
# 'vfork' program: testrun
#
# Build
build('vfork', ['vfork.c'])
# Trace
err = check_output(rpz + ['testrun', './vfork'], 'err')
err = err.split(b'\n')
assert not any(b'program exited with non-zero code' in l for l in err)
# ########################################
# 'rename' program: trace
#
# Build
build('rename', ['rename.c'])
# Trace
check_call(rpz +
['trace', '--overwrite', '-d', 'rename-trace', './rename'])
with Path('rename-trace/config.yml').open(encoding='utf-8') as fp:
config = yaml.safe_load(fp)
# Check that written files were logged
database = Path.cwd() / 'rename-trace/trace.sqlite3'
if PY3:
# On PY3, connect() only accepts unicode
conn = sqlite3.connect(str(database))
else:
conn = sqlite3.connect(database.path)
conn.row_factory = sqlite3.Row
rows = conn.execute('''
SELECT name FROM opened_files
''')
files = set(Path(r[0]) for r in rows)
for n in ('dir1/file', 'dir2/file', 'dir2/brokensymlink', 'dir2/symlink'):
if (Path.cwd() / n) not in files:
raise AssertionError("Missing file: %s" % (Path.cwd() / n))
conn.close()
# Check that created files won't be packed
for f in config.get('other_files'):
if 'dir2' in Path(f).parent.components:
raise AssertionError("Created file shouldn't be packed: %s" %
Path(f))
# ########################################
# 'readwrite' program: trace
# Build
build('readwrite', ['readwrite.c'])
# Create test folder
Path('readwrite_test').mkdir()
with Path('readwrite_test/existing').open('w'):
pass
# Trace existing one
check_call(rpz + [
'trace', '--overwrite', '-d', 'readwrite-E-trace', './readwrite',
'readwrite_test/existing'
])
# Check that file was logged as read and written
database = Path.cwd() / 'readwrite-E-trace/trace.sqlite3'
if PY3:
# On PY3, connect() only accepts unicode
conn = sqlite3.connect(str(database))
else:
conn = sqlite3.connect(database.path)
conn.row_factory = sqlite3.Row
rows = list(
conn.execute(
'''
SELECT mode FROM opened_files
WHERE name = ?
''', (str(Path('readwrite_test/existing').absolute()), )))
conn.close()
assert rows
assert rows[0][0] == FILE_READ | FILE_WRITE
# Trace non-existing one
check_call(rpz + [
'trace', '--overwrite', '-d', 'readwrite-N-trace', './readwrite',
'readwrite_test/nonexisting'
])
# Check that file was logged as written only
database = Path.cwd() / 'readwrite-N-trace/trace.sqlite3'
if PY3:
# On PY3, connect() only accepts unicode
conn = sqlite3.connect(str(database))
else:
conn = sqlite3.connect(database.path)
conn.row_factory = sqlite3.Row
rows = list(
conn.execute(
'''
SELECT mode FROM opened_files
WHERE name = ?
''', (str(Path('readwrite_test/nonexisting').absolute()), )))
conn.close()
assert rows
assert rows[0][0] == FILE_WRITE
# Trace a failure: inaccessible file
ret = call(rpz + [
'trace', '--overwrite', '-d', 'readwrite-F-trace', './readwrite',
'readwrite_test/non/existing/file'
])
assert ret == 1
# ########################################
# Test shebang corner-cases
#
Path('a').symlink('b')
with Path('b').open('w') as fp:
fp.write('#!%s 0\nsome content\n' % (Path.cwd() / 'c'))
Path('b').chmod(0o744)
Path('c').symlink('d')
with Path('d').open('w') as fp:
fp.write('#!e')
Path('d').chmod(0o744)
with Path('e').open('w') as fp:
fp.write('#!/bin/echo')
Path('e').chmod(0o744)
# Trace
out = check_output(rpz + [
'trace', '--overwrite', '-d', 'shebang-trace',
'--dont-identify-packages', './a', '1', '2'
])
out = out.splitlines()[0]
assert out == ('e %s 0 ./a 1 2' % (Path.cwd() / 'c')).encode('ascii')
# Check config
with Path('shebang-trace/config.yml').open(encoding='utf-8') as fp:
config = yaml.safe_load(fp)
other_files = set(
Path(f) for f in config['other_files']
if f.startswith('%s/' % Path.cwd()))
# Check database
database = Path.cwd() / 'shebang-trace/trace.sqlite3'
if PY3:
# On PY3, connect() only accepts unicode
conn = sqlite3.connect(str(database))
else:
conn = sqlite3.connect(database.path)
conn.row_factory = sqlite3.Row
rows = conn.execute('''
SELECT name FROM opened_files
''')
opened = [Path(r[0]) for r in rows if r[0].startswith('%s/' % Path.cwd())]
rows = conn.execute('''
SELECT name, argv FROM executed_files
''')
executed = [(Path(r[0]), r[1]) for r in rows
if Path(r[0]).lies_under(Path.cwd())]
print("other_files: %r" % sorted(other_files))
print("opened: %r" % opened)
print("executed: %r" % executed)
assert other_files == set(Path.cwd() / p
for p in ('a', 'b', 'c', 'd', 'e'))
assert opened == [Path.cwd() / 'c', Path.cwd() / 'e']
assert executed == [(Path.cwd() / 'a', './a\x001\x002\x00')]
# ########################################
# Test old packages
#
old_packages = [
('simple-0.4.0.rpz',
'https://drive.google.com/uc?export=download&id=0B3ucPz7GSthBVG4xZW1V'
'eDhXNTQ'),
('simple-0.6.0.rpz',
'https://drive.google.com/uc?export=download&id=0B3ucPz7GSthBbl9SUjhr'
'cUdtbGs'),
('simple-0.7.1.rpz',
'https://drive.google.com/uc?export=download&id=0B3ucPz7GSthBRGp2Vm5V'
'QVpWOGs'),
]
for name, url in old_packages:
print("Testing old package %s" % name)
f = Path(name)
if not f.exists():
download_file_retry(url, f)
# Info
check_call(rpuz + ['info', name])
# Show files
check_call(rpuz + ['showfiles', name])
# Lists packages
check_call(rpuz + ['installpkgs', '--summary', name])
# Unpack directory
check_call(rpuz + ['directory', 'setup', name, 'simpledir'])
# Run directory
check_simple(rpuz + ['directory', 'run', 'simpledir'], 'err')
output_in_dir = Path('simpledir/root/tmp')
output_in_dir = output_in_dir.listdir('reprozip_*')[0]
output_in_dir = output_in_dir / 'simple_output.txt'
with output_in_dir.open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Delete with wrong command (should fail)
p = subprocess.Popen(rpuz + ['chroot', 'destroy', 'simpledir'],
stderr=subprocess.PIPE)
out, err = p.communicate()
assert p.poll() != 0
err = err.splitlines()
assert b"Wrong unpacker used" in err[0]
assert err[1].startswith(b"usage: ")
# Delete directory
check_call(rpuz + ['directory', 'destroy', 'simpledir'])
# ########################################
# Copies back coverage report
#
coverage = Path('.coverage')
if coverage.exists():
coverage.copyfile(tests.parent / '.coverage.runpy')
def functional_tests(raise_warnings, interactive, run_vagrant, run_docker):
# Tests on Python < 2.7.3: need to use separate reprozip Python (with known
# working version of Python)
if sys.version_info < (2, 7, 3):
bug13676 = True
if 'REPROZIP_PYTHON' not in os.environ:
sys.stderr.write("Error: using reprozip with Python %s!\n" %
sys.version.split(' ', 1)[0])
sys.exit(1)
else:
bug13676 = False
rpz = [os.environ.get('REPROZIP_PYTHON', sys.executable)]
rpuz = [os.environ.get('REPROUNZIP_PYTHON', sys.executable)]
# Can't match on the SignalWarning category here because of a Python bug
# http://bugs.python.org/issue22543
if raise_warnings:
rpz.extend(['-W', 'error:signal'])
rpuz.extend(['-W', 'error:signal'])
if 'COVER' in os.environ:
rpz.extend(['-m'] + os.environ['COVER'].split(' '))
rpuz.extend(['-m'] + os.environ['COVER'].split(' '))
reprozip_main = tests.parent / 'reprozip/reprozip/main.py'
reprounzip_main = tests.parent / 'reprounzip/reprounzip/main.py'
verbose = ['-v'] * 3
rpz.extend([reprozip_main.absolute().path] + verbose)
rpuz.extend([reprounzip_main.absolute().path] + verbose)
print("Command lines are:\n%r\n%r" % (rpz, rpuz))
# ########################################
# testrun /bin/echo
#
output = check_output(rpz + ['testrun', '/bin/echo', 'outputhere'])
assert any(b' 1 | /bin/echo outputhere ' in l for l in output.splitlines())
output = check_output(
rpz + ['testrun', '-a', '/fake/path/echo', '/bin/echo', 'outputhere'])
assert any(b' 1 | (/bin/echo) /fake/path/echo outputhere ' in l
for l in output.splitlines())
# ########################################
# testrun multiple commands
#
check_call(rpz + [
'testrun', 'bash', '-c', 'cat ../../../../../etc/passwd;'
'cd /var/lib;'
'cat ../../etc/group'
])
check_call(rpz + ['trace', 'bash', '-c', 'cat /etc/passwd;echo'])
check_call(
rpz +
['trace', '--continue', 'sh', '-c', 'cat /etc/group;/usr/bin/id'])
check_call(rpz + ['pack'])
if not bug13676:
check_call(rpuz + ['graph', 'graph.dot'])
check_call(rpuz + ['graph', 'graph2.dot', 'experiment.rpz'])
sudo = ['sudo', '-E'] # -E to keep REPROZIP_USAGE_STATS
# ########################################
# 'simple' program: trace, pack, info, unpack
#
# Build
build('simple', ['simple.c'])
# Trace
check_call(rpz + [
'trace', '-d', 'rpz-simple', './simple',
(tests / 'simple_input.txt').path, 'simple_output.txt'
])
orig_output_location = Path('simple_output.txt').absolute()
assert orig_output_location.is_file()
with orig_output_location.open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
orig_output_location.remove()
# Read config
with Path('rpz-simple/config.yml').open(encoding='utf-8') as fp:
conf = yaml.safe_load(fp)
other_files = set(Path(f).absolute() for f in conf['other_files'])
expected = [Path('simple'), (tests / 'simple_input.txt')]
assert other_files.issuperset([f.resolve() for f in expected])
# Check input and output files
input_files = conf['runs'][0]['input_files']
assert (dict((k, Path(f).name) for k, f in iteritems(input_files)) == {
'arg': b'simple_input.txt'
})
output_files = conf['runs'][0]['output_files']
print(dict((k, Path(f).name) for k, f in iteritems(output_files)))
# Here we don't test for dict equality, since we might have C coverage
# files in the mix
assert Path(output_files['arg']).name == b'simple_output.txt'
# Pack
check_call(rpz + ['pack', '-d', 'rpz-simple', 'simple.rpz'])
Path('simple').remove()
# Info
check_call(rpuz + ['info', 'simple.rpz'])
# Show files
check_call(rpuz + ['showfiles', 'simple.rpz'])
# Lists packages
check_call(rpuz + ['installpkgs', '--summary', 'simple.rpz'])
# Unpack directory
check_call(rpuz + ['directory', 'setup', 'simple.rpz', 'simpledir'])
# Run directory
check_call(rpuz + ['directory', 'run', 'simpledir'])
output_in_dir = join_root(Path('simpledir/root'), orig_output_location)
with output_in_dir.open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Delete with wrong command (should fail)
p = subprocess.Popen(rpuz + ['chroot', 'destroy', 'simpledir'],
stderr=subprocess.PIPE)
stdout, stderr = p.communicate()
assert p.poll() != 0
stderr = stderr.splitlines()
assert b"Wrong unpacker used" in stderr[0]
assert stderr[1].startswith(b"usage: ")
# Delete directory
check_call(rpuz + ['directory', 'destroy', 'simpledir'])
# Unpack chroot
check_call(
sudo + rpuz +
['chroot', 'setup', '--bind-magic-dirs', 'simple.rpz', 'simplechroot'])
try:
# Run chroot
check_call(sudo + rpuz + ['chroot', 'run', 'simplechroot'])
output_in_chroot = join_root(Path('simplechroot/root'),
orig_output_location)
with output_in_chroot.open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Get output file
check_call(sudo + rpuz +
['chroot', 'download', 'simplechroot', 'arg:output1.txt'])
with Path('output1.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Replace input file
check_call(sudo + rpuz + [
'chroot', 'upload', 'simplechroot',
'%s:arg' % (tests / 'simple_input2.txt')
])
check_call(sudo + rpuz + ['chroot', 'upload', 'simplechroot'])
# Run again
check_call(sudo + rpuz + ['chroot', 'run', 'simplechroot'])
output_in_chroot = join_root(Path('simplechroot/root'),
orig_output_location)
with output_in_chroot.open(encoding='utf-8') as fp:
assert fp.read().strip() == '36'
# Delete with wrong command (should fail)
p = subprocess.Popen(rpuz + ['directory', 'destroy', 'simplechroot'],
stderr=subprocess.PIPE)
stdout, stderr = p.communicate()
assert p.poll() != 0
stderr = stderr.splitlines()
assert b"Wrong unpacker used" in stderr[0]
assert stderr[1].startswith(b"usage:")
finally:
# Delete chroot
check_call(sudo + rpuz + ['chroot', 'destroy', 'simplechroot'])
if not (tests / 'vagrant').exists():
check_call([
'sudo', 'sh', '-c',
'mkdir %(d)s; chmod 777 %(d)s' % {
'd': tests / 'vagrant'
}
])
# Unpack Vagrant-chroot
check_call(rpuz + [
'vagrant', 'setup/create', '--use-chroot', 'simple.rpz',
(tests / 'vagrant/simplevagrantchroot').path
])
print("\nVagrant project set up in simplevagrantchroot")
try:
if run_vagrant:
check_call(rpuz + [
'vagrant', 'run', '--no-stdin',
(tests / 'vagrant/simplevagrantchroot').path
])
# Destroy
check_call(rpuz + [
'vagrant', 'destroy', (tests /
'vagrant/simplevagrantchroot').path
])
elif interactive:
print("Test and press enter")
sys.stdin.readline()
finally:
if (tests / 'vagrant/simplevagrantchroot').exists():
(tests / 'vagrant/simplevagrantchroot').rmtree()
# Unpack Vagrant without chroot
check_call(rpuz + [
'vagrant', 'setup/create', '--dont-use-chroot', 'simple.rpz',
(tests / 'vagrant/simplevagrant').path
])
print("\nVagrant project set up in simplevagrant")
try:
if run_vagrant:
check_call(rpuz + [
'vagrant', 'run', '--no-stdin',
(tests / 'vagrant/simplevagrant').path
])
# Destroy
check_call(
rpuz +
['vagrant', 'destroy', (tests / 'vagrant/simplevagrant').path])
elif interactive:
print("Test and press enter")
sys.stdin.readline()
finally:
if (tests / 'vagrant/simplevagrant').exists():
(tests / 'vagrant/simplevagrant').rmtree()
# Unpack Docker
check_call(rpuz + ['docker', 'setup/create', 'simple.rpz', 'simpledocker'])
print("\nDocker project set up in simpledocker")
try:
if run_docker:
check_call(rpuz + ['docker', 'setup/build', 'simpledocker'])
check_call(rpuz + ['docker', 'run', 'simpledocker'])
# Get output file
check_call(
rpuz +
['docker', 'download', 'simpledocker', 'arg:doutput1.txt'])
with Path('doutput1.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Replace input file
check_call(rpuz + [
'docker', 'upload', 'simpledocker',
'%s:arg' % (tests / 'simple_input2.txt')
])
check_call(rpuz + ['docker', 'upload', 'simpledocker'])
check_call(rpuz + ['showfiles', 'simpledocker'])
# Run again
check_call(rpuz + ['docker', 'run', 'simpledocker'])
# Get output file
check_call(
rpuz +
['docker', 'download', 'simpledocker', 'arg:doutput2.txt'])
with Path('doutput2.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '36'
# Destroy
check_call(rpuz + ['docker', 'destroy', 'simpledocker'])
elif interactive:
print("Test and press enter")
sys.stdin.readline()
finally:
if Path('simpledocker').exists():
Path('simpledocker').rmtree()
# ########################################
# 'threads' program: testrun
#
# Build
build('threads', ['threads.c'], ['-lpthread'])
# Trace
check_call(rpz + ['testrun', './threads'])
# ########################################
# 'segv' program: testrun
#
# Build
build('segv', ['segv.c'])
# Trace
check_call(rpz + ['testrun', './segv'])
# ########################################
# 'exec_echo' program: trace, pack, run --cmdline
#
# Build
build('exec_echo', ['exec_echo.c'])
# Trace
check_call(rpz + ['trace', './exec_echo', 'originalexecechooutput'])
# Pack
check_call(rpz + ['pack', 'exec_echo.rpz'])
# Unpack chroot
check_call(sudo + rpuz +
['chroot', 'setup', 'exec_echo.rpz', 'echochroot'])
try:
# Run original command-line
output = check_output(sudo + rpuz + ['chroot', 'run', 'echochroot'])
assert output == b'originalexecechooutput\n'
# Prints out command-line
output = check_output(sudo + rpuz +
['chroot', 'run', 'echochroot', '--cmdline'])
assert any(b'./exec_echo originalexecechooutput' == s.strip()
for s in output.split(b'\n'))
# Run with different command-line
output = check_output(sudo + rpuz + [
'chroot', 'run', 'echochroot', '--cmdline', './exec_echo',
'changedexecechooutput'
])
assert output == b'changedexecechooutput\n'
finally:
check_call(sudo + rpuz + ['chroot', 'destroy', 'echochroot'])
# ########################################
# 'exec_echo' program: testrun
# This is built with -m32 so that we transition:
# python (x64) -> exec_echo (i386) -> echo (x64)
#
if sys.maxsize > 2**32:
# Build
build('exec_echo32', ['exec_echo.c'], ['-m32'])
# Trace
check_call(rpz + ['testrun', './exec_echo32 42'])
else:
print("Can't try exec_echo transitions: not running on 64bits")
# ########################################
# Tracing non-existing program
#
check_call(rpz + ['testrun', './doesntexist'])
# ########################################
# 'connect' program: testrun
#
# Build
build('connect', ['connect.c'])
# Trace
stderr = check_errout(rpz + ['testrun', './connect'])
stderr = stderr.split(b'\n')
assert not any(b'program exited with non-zero code' in l for l in stderr)
assert any(
re.search(br'process connected to [0-9.]+:80', l) for l in stderr)
# ########################################
# Copies back coverage report
#
coverage = Path('.coverage')
if coverage.exists():
coverage.copyfile(tests.parent / '.coverage.runpy')
def __init__(self, enabled, target, display=None):
self.enabled = enabled
if not self.enabled:
return
self.target = target
self.xauth = PosixPath('/.reprounzip_xauthority')
self.display = display if display is not None else self.DISPLAY_NUMBER
logging.debug(
"X11 support enabled; will create Xauthority file %s "
"for experiment. Display number is %d", self.xauth, self.display)
# List of addresses that match the $DISPLAY variable
possible, local_display = self._locate_display()
tcp_portnum = ((6000 +
local_display) if local_display is not None else None)
if ('XAUTHORITY' in os.environ
and Path(os.environ['XAUTHORITY']).is_file()):
xauthority = Path(os.environ['XAUTHORITY'])
# Note: I'm assuming here that Xauthority has no XDG support
else:
xauthority = Path('~').expand_user() / '.Xauthority'
# Read Xauthority file
xauth_entries = {}
if xauthority.is_file():
with xauthority.open('rb') as fp:
fp.seek(0, os.SEEK_END)
size = fp.tell()
fp.seek(0, os.SEEK_SET)
while fp.tell() < size:
entry = Xauth.from_file(fp)
if (entry.name == 'MIT-MAGIC-COOKIE-1'
and entry.number == local_display):
if entry.family == Xauth.FAMILY_LOCAL:
xauth_entries[(entry.family, None)] = entry
elif (entry.family == Xauth.FAMILY_INTERNET
or entry.family == Xauth.FAMILY_INTERNET6):
xauth_entries[(entry.family,
entry.address)] = entry
# FIXME: this completely ignores addresses
logging.debug("Possible X endpoints: %s", (possible, ))
# Select socket and authentication cookie
self.xauth_record = None
self.connection_info = None
for family, address in possible:
# Checks that we have a cookie
entry = family, (None if family is Xauth.FAMILY_LOCAL else address)
if entry not in xauth_entries:
continue
if family == Xauth.FAMILY_LOCAL and hasattr(socket, 'AF_UNIX'):
# Checks that the socket exists
if not Path(address).exists():
continue
self.connection_info = (socket.AF_UNIX, socket.SOCK_STREAM,
address)
self.xauth_record = xauth_entries[(family, None)]
logging.debug(
"Will connect to local X display via UNIX "
"socket %s", address)
break
else:
# Checks that we have a cookie
family = self.X2SOCK[family]
self.connection_info = (family, socket.SOCK_STREAM,
(address, tcp_portnum))
self.xauth_record = xauth_entries[(family, address)]
logging.debug("Will connect to X display %s:%d via %s/TCP",
address, tcp_portnum,
"IPv6" if family == socket.AF_INET6 else "IPv4")
break
# Didn't find an Xauthority record -- assume no authentication is
# needed, but still set self.connection_info
if self.connection_info is None:
for family, address in possible:
# Only try UNIX sockets, we'll use 127.0.0.1 otherwise
if family == Xauth.FAMILY_LOCAL:
if not hasattr(socket, 'AF_UNIX'):
continue
self.connection_info = (socket.AF_UNIX, socket.SOCK_STREAM,
address)
logging.debug(
"Will connect to X display via UNIX socket "
"%s, no authentication", address)
break
else:
self.connection_info = (socket.AF_INET, socket.SOCK_STREAM,
('127.0.0.1', tcp_portnum))
logging.debug(
"Will connect to X display 127.0.0.1:%d via IPv4/TCP, "
"no authentication", tcp_portnum)
if self.connection_info is None:
raise RuntimeError("Couldn't determine how to connect to local X "
"server, DISPLAY is %s" %
(repr(os.environ['DISPLAY'])
if 'DISPLAY' is os.environ else 'not set'))
def functional_tests(raise_warnings, interactive, run_vagrant, run_docker):
python = [sys.executable]
# Can't match on the SignalWarning category here because of a Python bug
# http://bugs.python.org/issue22543
python.extend(['-W', 'error:signal'])
if 'COVER' in os.environ:
python.extend(['-m'] + os.environ['COVER'].split(' '))
reprozip_main = tests.parent / 'reprozip/reprozip/main.py'
reprounzip_main = tests.parent / 'reprounzip/reprounzip/main.py'
verbose = ['-v'] * 3
rpz = python + [reprozip_main.absolute().path] + verbose
rpuz = python + [reprounzip_main.absolute().path] + verbose
# ########################################
# 'simple' program: trace, pack, info, unpack
#
# Build
build('simple', ['simple.c'])
# Trace
check_call(rpz + ['trace', '-d', 'rpz-simple',
'./simple',
(tests / 'simple_input.txt').path,
'simple_output.txt'])
orig_output_location = Path('simple_output.txt').absolute()
assert orig_output_location.is_file()
with orig_output_location.open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
orig_output_location.remove()
# Read config
with Path('rpz-simple/config.yml').open(encoding='utf-8') as fp:
conf = yaml.safe_load(fp)
other_files = set(Path(f).absolute() for f in conf['other_files'])
expected = [Path('simple'), (tests / 'simple_input.txt')]
assert other_files.issuperset([f.resolve() for f in expected])
# Check input and output files
input_files = conf['runs'][0]['input_files']
assert (dict((k, Path(f).name)
for k, f in iteritems(input_files)) ==
{'arg': b'simple_input.txt'})
output_files = conf['runs'][0]['output_files']
print(dict((k, Path(f).name) for k, f in iteritems(output_files)))
# Here we don't test for dict equality, since we might have C coverage
# files in the mix
assert Path(output_files['arg']).name == b'simple_output.txt'
# Pack
check_call(rpz + ['pack', '-d', 'rpz-simple', 'simple.rpz'])
Path('simple').remove()
# Info
check_call(rpuz + ['info', 'simple.rpz'])
# Show files
check_call(rpuz + ['showfiles', 'simple.rpz'])
# Lists packages
check_call(rpuz + ['installpkgs', '--summary', 'simple.rpz'])
# Unpack directory
check_call(rpuz + ['directory', 'setup', 'simple.rpz', 'simpledir'])
# Run directory
check_call(rpuz + ['directory', 'run', 'simpledir'])
output_in_dir = join_root(Path('simpledir/root'), orig_output_location)
with output_in_dir.open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Delete with wrong command (should fail)
assert call(rpuz + ['chroot', 'destroy', 'simpledir']) != 0
# Delete directory
check_call(rpuz + ['directory', 'destroy', 'simpledir'])
# Unpack chroot
check_call(['sudo'] + rpuz + ['chroot', 'setup', '--bind-magic-dirs',
'simple.rpz', 'simplechroot'])
# Run chroot
check_call(['sudo'] + rpuz + ['chroot', 'run', 'simplechroot'])
output_in_chroot = join_root(Path('simplechroot/root'),
orig_output_location)
with output_in_chroot.open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Get output file
check_call(['sudo'] + rpuz + ['chroot', 'download', 'simplechroot',
'arg:output1.txt'])
with Path('output1.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Replace input file
check_call(['sudo'] + rpuz + ['chroot', 'upload', 'simplechroot',
'%s:arg' % (tests / 'simple_input2.txt')])
check_call(['sudo'] + rpuz + ['chroot', 'upload', 'simplechroot'])
# Run again
check_call(['sudo'] + rpuz + ['chroot', 'run', 'simplechroot'])
output_in_chroot = join_root(Path('simplechroot/root'),
orig_output_location)
with output_in_chroot.open(encoding='utf-8') as fp:
assert fp.read().strip() == '36'
# Delete with wrong command (should fail)
assert call(rpuz + ['directory', 'destroy', 'simplechroot']) != 0
# Delete chroot
check_call(['sudo'] + rpuz + ['chroot', 'destroy', 'simplechroot'])
if not Path('/vagrant').exists():
check_call(['sudo', 'sh', '-c', 'mkdir /vagrant; chmod 777 /vagrant'])
# Unpack Vagrant-chroot
check_call(rpuz + ['vagrant', 'setup/create', '--use-chroot', 'simple.rpz',
'/vagrant/simplevagrantchroot'])
print("\nVagrant project set up in simplevagrantchroot")
try:
if run_vagrant:
check_call(rpuz + ['vagrant', 'run', '--no-stdin',
'/vagrant/simplevagrantchroot'])
# Destroy
check_call(rpuz + ['vagrant', 'destroy',
'/vagrant/simplevagrantchroot'])
elif interactive:
print("Test and press enter")
sys.stdin.readline()
finally:
if Path('/vagrant/simplevagrantchroot').exists():
Path('/vagrant/simplevagrantchroot').rmtree()
# Unpack Vagrant without chroot
check_call(rpuz + ['vagrant', 'setup/create', '--dont-use-chroot',
'simple.rpz',
'/vagrant/simplevagrant'])
print("\nVagrant project set up in simplevagrant")
try:
if run_vagrant:
check_call(rpuz + ['vagrant', 'run', '--no-stdin',
'/vagrant/simplevagrant'])
# Destroy
check_call(rpuz + ['vagrant', 'destroy', '/vagrant/simplevagrant'])
elif interactive:
print("Test and press enter")
sys.stdin.readline()
finally:
if Path('/vagrant/simplevagrant').exists():
Path('/vagrant/simplevagrant').rmtree()
# Unpack Docker
check_call(rpuz + ['docker', 'setup/create', 'simple.rpz', 'simpledocker'])
print("\nDocker project set up in simpledocker")
try:
if run_docker:
check_call(rpuz + ['docker', 'setup/build', 'simpledocker'])
check_call(rpuz + ['docker', 'run', 'simpledocker'])
# Get output file
check_call(rpuz + ['docker', 'download', 'simpledocker',
'arg:doutput1.txt'])
with Path('doutput1.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Replace input file
check_call(rpuz + ['docker', 'upload', 'simpledocker',
'%s:arg' % (tests / 'simple_input2.txt')])
check_call(rpuz + ['docker', 'upload', 'simpledocker'])
check_call(rpuz + ['showfiles', 'simpledocker'])
# Run again
check_call(rpuz + ['docker', 'run', 'simpledocker'])
# Get output file
check_call(rpuz + ['docker', 'download', 'simpledocker',
'arg:doutput2.txt'])
with Path('doutput2.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '36'
# Destroy
check_call(rpuz + ['docker', 'destroy', 'simpledocker'])
elif interactive:
print("Test and press enter")
sys.stdin.readline()
finally:
if Path('simpledocker').exists():
Path('simpledocker').rmtree()
# ########################################
# 'threads' program: testrun
#
# Build
build('threads', ['threads.c'], ['-lpthread'])
# Trace
check_call(rpz + ['testrun', './threads'])
# ########################################
# 'segv' program: testrun
#
# Build
build('segv', ['segv.c'])
# Trace
check_call(rpz + ['testrun', './segv'])
# ########################################
# 'exec_echo' program: trace, pack, run --cmdline
#
# Build
build('exec_echo', ['exec_echo.c'])
# Trace
check_call(rpz + ['trace', './exec_echo', 'originalexecechooutput'])
# Pack
check_call(rpz + ['pack', 'exec_echo.rpz'])
# Unpack chroot
check_call(['sudo'] + rpuz + ['chroot', 'setup',
'exec_echo.rpz', 'echochroot'])
try:
# Run original command-line
output = check_output(['sudo'] + rpuz + ['chroot', 'run',
'echochroot'])
assert output == b'originalexecechooutput\n'
# Prints out command-line
output = check_output(['sudo'] + rpuz + ['chroot', 'run',
'echochroot', '--cmdline'])
assert any(b'./exec_echo originalexecechooutput' == s.strip()
for s in output.split(b'\n'))
# Run with different command-line
output = check_output(['sudo'] + rpuz + [
'chroot', 'run', 'echochroot',
'--cmdline', './exec_echo', 'changedexecechooutput'])
assert output == b'changedexecechooutput\n'
finally:
check_call(['sudo'] + rpuz + ['chroot', 'destroy', 'echochroot'])
# ########################################
# 'exec_echo' program: testrun
# This is built with -m32 so that we transition:
# python (x64) -> exec_echo (i386) -> echo (x64)
#
if sys.maxsize > 2 ** 32:
# Build
build('exec_echo32', ['exec_echo.c'], ['-m32'])
# Trace
check_call(rpz + ['testrun', './exec_echo32 42'])
else:
print("Can't try exec_echo transitions: not running on 64bits")
# ########################################
# Tracing non-existing program
#
check_call(rpz + ['testrun', './doesntexist'])
# ########################################
# 'connect' program: testrun
#
# Build
build('connect', ['connect.c'])
# Trace
stderr = check_errout(rpz + ['testrun', './connect'])
stderr = stderr.split(b'\n')
assert not any(b'program exited with non-zero code' in l for l in stderr)
assert any(re.search(br'process connected to [0-9.]+:80', l)
for l in stderr)
# ########################################
# Copies back coverage report
#
coverage = Path('.coverage')
if coverage.exists():
coverage.copyfile(tests.parent / '.coverage.runpy')
