def test_trace_and_prune():
client = get_docker_client()
container = client.containers.run('python:3.8-slim',
detach=True,
tty=True,
security_opt=['seccomp:unconfined'])
commands = ["python --version", "python -c print()"]
try:
# Respond yes to delete things.
with mock.patch('builtins.input', return_value="y"):
trace_and_prune(container.name, commands, "/usr/local")
for cmd in commands:
ret, result = container.exec_run(cmd)
result = result.decode()
if ret:
assert False, "unexpected non-zero return code when running '{}'".format(
cmd)
# This should fail.
ret, result = container.exec_run("pip --help")
if not ret:
assert False, "unexpected zero return code when running '{}'".format(
cmd)
finally:
container.stop()
container.remove()
def test_trace_and_prune_with_mounted_volume(tmpdir):
client = get_docker_client()
# Create a file in a mounted directory.
with (tmpdir / "foobar.txt").open("w", encoding="utf-8") as f:
f.write("Foobar")
container = client.containers.run(
'python:3.8-slim',
detach=True,
tty=True,
security_opt=['seccomp:unconfined'],
volumes={str(tmpdir): {
'bind': '/work',
'mode': 'rw'
}})
commands = ["python --version", "python -c print()"]
try:
# Respond yes to delete things.
with mock.patch('builtins.input', return_value="y"):
with pytest.raises(ValueError):
trace_and_prune(container.name, commands,
["/usr/local", "/work"])
# This should work.
with mock.patch('builtins.input', return_value="y"):
trace_and_prune(container.name, commands, ["/usr/local"])
finally:
container.stop()
container.remove()
def test_docker_container_from_specs(specs, bash_test_file):
""""""
client = get_docker_client()
docker_is_running(client)
df = Dockerfile(specs).render()
refpath = bash_test_file[5:].split(".")[0]
refpath = os.path.join(DOCKER_CACHEDIR, "Dockerfile." + refpath)
if os.path.exists(refpath):
logger.info("loading cached reference dockerfile")
with open(refpath, "r") as fp:
reference = fp.read()
if _dockerfiles_equivalent(df, reference):
logger.info("test equal to reference dockerfile, passing")
return # do not build and test because nothing has changed
logger.info("building docker image")
image, build_logs = client.images.build(fileobj=io.BytesIO(df.encode()),
rm=True)
bash_test_file = posixpath.join("/testscripts", bash_test_file)
test_cmd = "bash " + bash_test_file
res = client.containers.run(image, test_cmd, **_container_run_kwds)
passed = res.decode().endswith("passed")
assert passed
if passed:
os.makedirs(os.path.dirname(refpath), exist_ok=True)
with open(refpath, "w") as fp:
fp.write(df)
def _get_mounts(container):
client = get_docker_client()
# [{'Type': 'bind',
# 'Source': '/path/to/source',
# 'Destination': '/destination',
# 'Mode': 'ro',
# 'RW': False,
# 'Propagation': 'rprivate'}]
return client.api.inspect_container(container)['Mounts']
def test_ReproZipMinimizer_no_ptrace():
client = get_docker_client()
container = client.containers.run("debian:stretch", detach=True, tty=True)
commands = ["du --help", "ls --help"]
tmpdir = tempfile.mkdtemp()
try:
minimizer = ReproZipMinimizer(container.id, commands, packfile_save_dir=tmpdir)
with pytest.raises(RuntimeError): # ptrace should fail
minimizer.run()
except Exception:
raise
finally:
container.stop()
container.remove()
def __init__(self, container, commands, packfile_save_dir='.', **kwargs):
try:
container.put_archive
self.container = container
except AttributeError:
client = get_docker_client()
self.container = client.containers.get(container)
if isinstance(commands, str):
commands = [commands]
self.commands = commands
self.packfile_save_dir = packfile_save_dir
self.shell_filepath = os.path.join(BASE_PATH, 'utils',
'reprozip_trace_runner.sh')
def _create_packfile(commands, dir):
"""Create packfile from list `commands` in debian:stretch container."""
client = get_docker_client()
image = "debian@sha256:427752aa7da803378f765f5a8efba421df5925cbde8ab011717f3642f406fb15"
container = client.containers.run(
image, detach=True, tty=True, security_opt=['seccomp:unconfined'])
try:
minimizer = ReproZipMinimizer(
container.id, commands, packfile_save_dir=dir)
packfile_path = minimizer.run()
except Exception:
raise
finally:
container.stop()
container.remove()
return packfile_path
def test_ReproZipMinimizer():
client = get_docker_client()
container = client.containers.run(
"debian:stretch", detach=True, tty=True, security_opt=["seccomp:unconfined"]
)
commands = ["du --help", "ls --help"]
tmpdir = tempfile.mkdtemp()
try:
minimizer = ReproZipMinimizer(container.id, commands, packfile_save_dir=tmpdir)
packfile_path = minimizer.run()
except Exception:
raise
finally:
container.stop()
container.remove()
assert os.path.isfile(packfile_path), "Pack file not saved."
def copy_file_from_container(container, src, dest='.'):
"""Copy file `filepath` from a running Docker `container`, and save it on
the host to `save_path` with the original filename.
Parameters
----------
container : str or container object
Container from which file is copied.
src : str
Filepath within container.
dest : str
Directory on the host in which to save file.
Returns
-------
local_filepath : str
Relative path to saved file on the host.
"""
import tarfile
import tempfile
client = get_docker_client()
try:
container.put_archive
container = container
except AttributeError:
container = client.containers.get(container)
tar_stream, tar_info = container.get_archive(src)
try:
with tempfile.NamedTemporaryFile() as tmp:
for data in tar_stream:
tmp.write(data)
tmp.flush()
with tarfile.TarFile(tmp.name) as tar:
tar.extractall(path=dest)
return os.path.join(dest, tar_info['name'])
except Exception as e:
raise
finally:
tar_stream.close()
def copy_file_to_container(container, src, dest):
"""Copy `local_filepath` into `container`:`container_path`.
Parameters
----------
container : str or container object
Container to which file is copied.
src : str
Filepath on the host.
dest : str
Directory inside container. Original filename is preserved.
Returns
-------
success : bool
True if copy was a success. False otherwise.
"""
# https://gist.github.com/zbyte64/6800eae10ce082bb78f0b7a2cca5cbc2
from io import BytesIO
import tarfile
client = get_docker_client()
try:
container.put_archive
container = container
except AttributeError:
container = client.containers.get(container)
with BytesIO() as tar_stream:
with tarfile.TarFile(fileobj=tar_stream, mode='w') as tar:
filename = os.path.split(src)[-1]
tar.add(src, arcname=filename, recursive=False)
tar_stream.seek(0)
return container.put_archive(dest, tar_stream)
def trace_and_prune(container, commands, directories_to_prune):
client = get_docker_client()
container = client.containers.get(container)
if isinstance(commands, str):
commands = [commands]
if isinstance(directories_to_prune, str):
directories_to_prune = [directories_to_prune]
cmds = ' '.join('"{}"'.format(c) for c in commands)
copy_file_to_container(container, TRACE_SCRIPT, '/tmp/')
trace_cmd = "bash /tmp/_trace.sh " + cmds
logger.info("running command within container {}: {}"
"".format(container.id, trace_cmd))
_, log_gen = container.exec_run(trace_cmd, stream=True)
for log in log_gen:
log = log.decode().strip()
logger.info(log)
if (("REPROZIP" in log and "couldn't use ptrace" in log)
or "neurodocker (in container): error" in log.lower()
or "_pytracer.Error" in log):
raise RuntimeError("Error: {}".format(log))
# Get files to prune.
copy_file_to_container(container, PRUNE_SCRIPT, '/tmp/')
ret, result = container.exec_run(
"/tmp/reprozip-miniconda/bin/python /tmp/_prune.py"
" --config-file /tmp/neurodocker-reprozip-trace/config.yml"
" --dirs-to-prune {}".format(" ".join(directories_to_prune)).split())
result = result.decode().strip()
if ret:
raise RuntimeError("Failed: {}".format(result))
ret, result = container.exec_run(
['cat', '/tmp/neurodocker-reprozip-trace/TO_DELETE.list'])
result = result.decode().strip()
if ret:
raise RuntimeError("Error: {}".format(result))
files_to_remove = result.splitlines()
if not len(files_to_remove):
print("No files to remove. Quitting.")
return
# Check if any files to be removed are in mounted directories.
mounts = _get_mounts(container.name)
if mounts:
for m in mounts:
for p in files_to_remove:
if Path(m['Destination']) in Path(p).parents:
raise ValueError(
"Attempting to remove files in a mounted directory."
" Directory in the container '{}' is host directory"
" '{}'. Remove this mounted directory from the"
" minification command and rerun.".format(
m['Destination'], m['Source']))
print("\nWARNING!!! THE FOLLOWING FILES WILL BE REMOVED:")
print('\n ', end='')
print('\n '.join(sorted(files_to_remove)))
print(
'\nWARNING: PROCEEDING MAY RESULT IN IRREVERSIBLE DATA LOSS, FOR EXAMPLE'
' IF ATTEMPTING TO REMOVE FILES IN DIRECTORIES MOUNTED FROM THE HOST.'
' PROCEED WITH EXTREME CAUTION! NEURODOCKER ASSUMES NO RESPONSIBILITY'
' FOR DATA LOSS. ALL OF THE FILES LISTED ABOVE WILL BE REMOVED.')
response = 'placeholder'
try:
while response.lower() not in {'y', 'n', ''}:
response = input('Proceed (y/N)? ')
except KeyboardInterrupt:
print("\nQuitting.")
return
if response.lower() in {'', 'n'}:
print("Quitting.")
return
print("Removing files ...")
ret, result = container.exec_run(
'xargs -d "\n" -a /tmp/neurodocker-reprozip-trace/TO_DELETE.list rm -f'
)
result = result.decode().split()
if ret:
raise RuntimeError("Error: {}".format(result))
ret, result = container.exec_run(
'rm -rf /tmp/neurodocker-reprozip-trace /tmp/reprozip-miniconda /tmp/_trace.sh /tmp/_prune.py'
)
result = result.decode().split()
if ret:
raise RuntimeError("Error: {}".format(result))
print("\n\nFinished removing files.")
print("Next, create a new Docker image with the minified container:")
print("\n docker export {} | docker import - imagename\n".format(
container.name))
