def test_trace_and_prune(): client = get_docker_client() container = client.containers.run('python:3.8-slim', detach=True, tty=True, security_opt=['seccomp:unconfined']) commands = ["python --version", "python -c print()"] try: # Respond yes to delete things. with mock.patch('builtins.input', return_value="y"): trace_and_prune(container.name, commands, "/usr/local") for cmd in commands: ret, result = container.exec_run(cmd) result = result.decode() if ret: assert False, "unexpected non-zero return code when running '{}'".format( cmd) # This should fail. ret, result = container.exec_run("pip --help") if not ret: assert False, "unexpected zero return code when running '{}'".format( cmd) finally: container.stop() container.remove()
def test_trace_and_prune_with_mounted_volume(tmpdir): client = get_docker_client() # Create a file in a mounted directory. with (tmpdir / "foobar.txt").open("w", encoding="utf-8") as f: f.write("Foobar") container = client.containers.run( 'python:3.8-slim', detach=True, tty=True, security_opt=['seccomp:unconfined'], volumes={str(tmpdir): { 'bind': '/work', 'mode': 'rw' }}) commands = ["python --version", "python -c print()"] try: # Respond yes to delete things. with mock.patch('builtins.input', return_value="y"): with pytest.raises(ValueError): trace_and_prune(container.name, commands, ["/usr/local", "/work"]) # This should work. with mock.patch('builtins.input', return_value="y"): trace_and_prune(container.name, commands, ["/usr/local"]) finally: container.stop() container.remove()
def test_docker_container_from_specs(specs, bash_test_file): """""" client = get_docker_client() docker_is_running(client) df = Dockerfile(specs).render() refpath = bash_test_file[5:].split(".")[0] refpath = os.path.join(DOCKER_CACHEDIR, "Dockerfile." + refpath) if os.path.exists(refpath): logger.info("loading cached reference dockerfile") with open(refpath, "r") as fp: reference = fp.read() if _dockerfiles_equivalent(df, reference): logger.info("test equal to reference dockerfile, passing") return # do not build and test because nothing has changed logger.info("building docker image") image, build_logs = client.images.build(fileobj=io.BytesIO(df.encode()), rm=True) bash_test_file = posixpath.join("/testscripts", bash_test_file) test_cmd = "bash " + bash_test_file res = client.containers.run(image, test_cmd, **_container_run_kwds) passed = res.decode().endswith("passed") assert passed if passed: os.makedirs(os.path.dirname(refpath), exist_ok=True) with open(refpath, "w") as fp: fp.write(df)
def _get_mounts(container): client = get_docker_client() # [{'Type': 'bind', # 'Source': '/path/to/source', # 'Destination': '/destination', # 'Mode': 'ro', # 'RW': False, # 'Propagation': 'rprivate'}] return client.api.inspect_container(container)['Mounts']
def test_ReproZipMinimizer_no_ptrace(): client = get_docker_client() container = client.containers.run("debian:stretch", detach=True, tty=True) commands = ["du --help", "ls --help"] tmpdir = tempfile.mkdtemp() try: minimizer = ReproZipMinimizer(container.id, commands, packfile_save_dir=tmpdir) with pytest.raises(RuntimeError): # ptrace should fail minimizer.run() except Exception: raise finally: container.stop() container.remove()
def __init__(self, container, commands, packfile_save_dir='.', **kwargs): try: container.put_archive self.container = container except AttributeError: client = get_docker_client() self.container = client.containers.get(container) if isinstance(commands, str): commands = [commands] self.commands = commands self.packfile_save_dir = packfile_save_dir self.shell_filepath = os.path.join(BASE_PATH, 'utils', 'reprozip_trace_runner.sh')
def _create_packfile(commands, dir): """Create packfile from list `commands` in debian:stretch container.""" client = get_docker_client() image = "debian@sha256:427752aa7da803378f765f5a8efba421df5925cbde8ab011717f3642f406fb15" container = client.containers.run( image, detach=True, tty=True, security_opt=['seccomp:unconfined']) try: minimizer = ReproZipMinimizer( container.id, commands, packfile_save_dir=dir) packfile_path = minimizer.run() except Exception: raise finally: container.stop() container.remove() return packfile_path
def test_ReproZipMinimizer(): client = get_docker_client() container = client.containers.run( "debian:stretch", detach=True, tty=True, security_opt=["seccomp:unconfined"] ) commands = ["du --help", "ls --help"] tmpdir = tempfile.mkdtemp() try: minimizer = ReproZipMinimizer(container.id, commands, packfile_save_dir=tmpdir) packfile_path = minimizer.run() except Exception: raise finally: container.stop() container.remove() assert os.path.isfile(packfile_path), "Pack file not saved."
def copy_file_from_container(container, src, dest='.'): """Copy file `filepath` from a running Docker `container`, and save it on the host to `save_path` with the original filename. Parameters ---------- container : str or container object Container from which file is copied. src : str Filepath within container. dest : str Directory on the host in which to save file. Returns ------- local_filepath : str Relative path to saved file on the host. """ import tarfile import tempfile client = get_docker_client() try: container.put_archive container = container except AttributeError: container = client.containers.get(container) tar_stream, tar_info = container.get_archive(src) try: with tempfile.NamedTemporaryFile() as tmp: for data in tar_stream: tmp.write(data) tmp.flush() with tarfile.TarFile(tmp.name) as tar: tar.extractall(path=dest) return os.path.join(dest, tar_info['name']) except Exception as e: raise finally: tar_stream.close()
def copy_file_to_container(container, src, dest): """Copy `local_filepath` into `container`:`container_path`. Parameters ---------- container : str or container object Container to which file is copied. src : str Filepath on the host. dest : str Directory inside container. Original filename is preserved. Returns ------- success : bool True if copy was a success. False otherwise. """ # https://gist.github.com/zbyte64/6800eae10ce082bb78f0b7a2cca5cbc2 from io import BytesIO import tarfile client = get_docker_client() try: container.put_archive container = container except AttributeError: container = client.containers.get(container) with BytesIO() as tar_stream: with tarfile.TarFile(fileobj=tar_stream, mode='w') as tar: filename = os.path.split(src)[-1] tar.add(src, arcname=filename, recursive=False) tar_stream.seek(0) return container.put_archive(dest, tar_stream)
def trace_and_prune(container, commands, directories_to_prune): client = get_docker_client() container = client.containers.get(container) if isinstance(commands, str): commands = [commands] if isinstance(directories_to_prune, str): directories_to_prune = [directories_to_prune] cmds = ' '.join('"{}"'.format(c) for c in commands) copy_file_to_container(container, TRACE_SCRIPT, '/tmp/') trace_cmd = "bash /tmp/_trace.sh " + cmds logger.info("running command within container {}: {}" "".format(container.id, trace_cmd)) _, log_gen = container.exec_run(trace_cmd, stream=True) for log in log_gen: log = log.decode().strip() logger.info(log) if (("REPROZIP" in log and "couldn't use ptrace" in log) or "neurodocker (in container): error" in log.lower() or "_pytracer.Error" in log): raise RuntimeError("Error: {}".format(log)) # Get files to prune. copy_file_to_container(container, PRUNE_SCRIPT, '/tmp/') ret, result = container.exec_run( "/tmp/reprozip-miniconda/bin/python /tmp/_prune.py" " --config-file /tmp/neurodocker-reprozip-trace/config.yml" " --dirs-to-prune {}".format(" ".join(directories_to_prune)).split()) result = result.decode().strip() if ret: raise RuntimeError("Failed: {}".format(result)) ret, result = container.exec_run( ['cat', '/tmp/neurodocker-reprozip-trace/TO_DELETE.list']) result = result.decode().strip() if ret: raise RuntimeError("Error: {}".format(result)) files_to_remove = result.splitlines() if not len(files_to_remove): print("No files to remove. Quitting.") return # Check if any files to be removed are in mounted directories. mounts = _get_mounts(container.name) if mounts: for m in mounts: for p in files_to_remove: if Path(m['Destination']) in Path(p).parents: raise ValueError( "Attempting to remove files in a mounted directory." " Directory in the container '{}' is host directory" " '{}'. Remove this mounted directory from the" " minification command and rerun.".format( m['Destination'], m['Source'])) print("\nWARNING!!! THE FOLLOWING FILES WILL BE REMOVED:") print('\n ', end='') print('\n '.join(sorted(files_to_remove))) print( '\nWARNING: PROCEEDING MAY RESULT IN IRREVERSIBLE DATA LOSS, FOR EXAMPLE' ' IF ATTEMPTING TO REMOVE FILES IN DIRECTORIES MOUNTED FROM THE HOST.' ' PROCEED WITH EXTREME CAUTION! NEURODOCKER ASSUMES NO RESPONSIBILITY' ' FOR DATA LOSS. ALL OF THE FILES LISTED ABOVE WILL BE REMOVED.') response = 'placeholder' try: while response.lower() not in {'y', 'n', ''}: response = input('Proceed (y/N)? ') except KeyboardInterrupt: print("\nQuitting.") return if response.lower() in {'', 'n'}: print("Quitting.") return print("Removing files ...") ret, result = container.exec_run( 'xargs -d "\n" -a /tmp/neurodocker-reprozip-trace/TO_DELETE.list rm -f' ) result = result.decode().split() if ret: raise RuntimeError("Error: {}".format(result)) ret, result = container.exec_run( 'rm -rf /tmp/neurodocker-reprozip-trace /tmp/reprozip-miniconda /tmp/_trace.sh /tmp/_prune.py' ) result = result.decode().split() if ret: raise RuntimeError("Error: {}".format(result)) print("\n\nFinished removing files.") print("Next, create a new Docker image with the minified container:") print("\n docker export {} | docker import - imagename\n".format( container.name))