def test_execute_xcom_behavior(self): self.client_mock.pull.return_value = [b'{"status":"pull log"}'] kwargs = { 'api_version': '1.19', 'command': 'env', 'environment': { 'UNIT': 'TEST' }, 'private_environment': { 'PRIVATE': 'MESSAGE' }, 'image': 'ubuntu:latest', 'network_mode': 'bridge', 'owner': 'unittest', 'task_id': 'unittest', 'mounts': [ Mount(source='/host/path', target='/container/path', type='bind') ], 'working_dir': '/container/path', 'shm_size': 1000, 'host_tmp_dir': '/host/airflow', 'container_name': 'test_container', 'tty': True, } xcom_push_operator = DockerOperator(**kwargs, do_xcom_push=True) no_xcom_push_operator = DockerOperator(**kwargs, do_xcom_push=False) xcom_push_result = xcom_push_operator.execute(None) no_xcom_push_result = no_xcom_push_operator.execute(None) assert xcom_push_result == b'container log' assert no_xcom_push_result is None
def create_bind_mount(target: str, bind) -> Mount: mode = 'rw' src = None if isinstance(bind, str): src = bind elif isinstance(bind, dict): src = bind.get('src') mode = bind.get('mode', 'rw') else: raise TypeError(f'Invalid bind volume definition {target}') return Mount( type='bind', target=target, source=src, read_only=mode != 'rw', )
def make_linux_binaries(repo_root: Path) -> Set[Path]: """ Create binaries for Linux in a Docker container. Args: repo_root: The path to the root of the repository. Returns: A set of paths to the built binaries. """ client = docker.from_env(version='auto') dist_dir = repo_root / 'dist' assert not dist_dir.exists() or not set(dist_dir.iterdir()) target_dir = '/e2e' code_mount = Mount( source=str(repo_root.absolute()), target=target_dir, type='bind', ) cmd_in_container = [ 'pip', 'install', '.[packaging]', '&&', 'python', 'admin/create_pyinstaller_binaries.py', ] command = 'bash -c "{cmd}"'.format(cmd=' '.join(cmd_in_container)) container = client.containers.run( image='python:3.7', mounts=[code_mount], command=command, working_dir=target_dir, remove=True, detach=True, ) for line in container.logs(stream=True): line = line.strip() LOGGER.info(line) status_code = container.wait()['StatusCode'] assert status_code == 0 return set(dist_dir.iterdir())
def bind_mount(source: Path, target: Path, **kwargs: Any) -> Mount: """Helper for Docker mount objects. Arguments: source: the host path to be mounted target: the container path the source should be mounted to Keyword arguments: Passed through to the underlying docker.services.Mount object initialization """ return Mount( source=str(source), target=str(target), type='bind', **kwargs )
def __init__(self, name='dockeranon_workstation', apps='', ip='192.168.5.1'): if WorkstationBuilder.CONTAINER_ID is not None: return with TemporaryDirectory() as dir_name: # Copy files for f in os.listdir('dockerfiles/workstation'): if f == 'Dockerfile': with open('dockerfiles/workstation/Dockerfile') as fd_in: with open('{0}/Dockerfile'.format(dir_name), 'w') as fd_out: for line in fd_in.readlines(): line = line.replace( '<WORKSTATION_IMAGE>', Config.get('WORKSTATION_IMAGE')) line = line.replace('<WORKSTATION_APPS>', self._apps_install(apps)) fd_out.write(line) else: shutil.copy('dockerfiles/workstation/{0}'.format(f), dir_name) # Build image client.images.build(pull=True, path=dir_name, tag=name) # Launch mounts = [] if Config.get('MOUNT_POINT_USE'): try: os.makedirs(Config.get('MOUNT_POINT')) except FileExistsError: pass mount = Mount(target='/home/user/Share', source=Config.get('MOUNT_POINT'), type='bind') mounts.append(mount) container = client.containers.run( name, cap_add=["NET_ADMIN"], detach=True, hostname=Config.get('WORKSTATION_HOSTNAME'), mounts=mounts) WorkstationBuilder.CONTAINER_ID = container.id
def run_testssl_against_target(self, target, results_path, report_name='testssl_result.json'): self.target = target self.results_path = results_path self.report_name = report_name self.environment = [ "TARGET={0}".format(self.target), "REPORT_NAME={0}".format(self.report_name) ] results_mount = Mount("/results", self.results_path, type="bind") self.client.containers.run(self.depcheck_docker, mounts=[results_mount], environment=self.environment) logger.info( "Successfully ran TestSSL against the Target. Please find *.json file in the results directory" )
def mounts(self): if len(self.volume_binds): driver = self.mount_driver_config # for host_loc, vol in self.volume_binds.items(): # self.log.info('---- {}'.format(host_loc)) # self.log.info('---- {}'.format(vol)) return [ Mount( target=vol["bind"], source=host_loc, type="volume", read_only=vol["mode"] == "ro", driver_config=driver, ) for host_loc, vol in self.volume_binds.items() ] else: return []
def convert_to_run_config(image_name, container_spec): # Convert every mount specification to a docker.types.Mount mounts = [] for ms in container_spec.get('mounts', []): mount = Mount(target=ms['target'], source=ms['source'], type='bind') mounts.append(mount) run_config = { 'image': image_name, 'command': container_spec.get('command'), 'name': container_spec['name'], 'detach': True, 'mounts': mounts, 'network': config.DOCKER_NETWORK, 'environment': container_spec.get('environment', {}), 'ports': container_spec.get('ports', {}), 'hostname': container_spec.get('hostname'), } return run_config
def __init__(self, path, statusService): self.statusService = statusService self.client = docker.from_env() self.image, logs = self.client.images.build(path=f'/projects/{path}', dockerfile='Dockerfile') mounts = [ Mount(target="/var/run/docker.sock", source="/var/run/docker.sock", type='bind') ] self.container = self.client.containers.run(image=self.image.id, tty=True, detach=True, mounts=mounts) self.statusService.add_image_ids(self.image.id, self.container.id, path, datetime.datetime.now()) for logLine in logs: print(logLine)
def convert_to_run_config(image_name, container_spec): # Convert every mount specification to a docker.types.Mount mounts = [] for ms in container_spec.get("mounts", []): mount = Mount(target=ms["target"], source=ms["source"], type="bind") mounts.append(mount) run_config = { "image": image_name, "command": container_spec.get("command"), "name": container_spec["name"], "detach": container_spec.get("detach", True), "mounts": mounts, "network": config.DOCKER_NETWORK, "environment": container_spec.get("environment", {}), "ports": container_spec.get("ports", {}), "hostname": container_spec.get("hostname"), "auto_remove": container_spec.get("auto_remove", False), } return run_config
def run(data): s = "from worker " + json.dumps(data) logging.info(s) client = docker.from_env() volumes = list( map( lambda l: Mount(l["target"], l["source"], type=l["type"], read_only=l["read_only"]), data["mounts"])) logging.info("volumes = {0}".format(volumes)) ret = client.containers.run(data["image"], command=data.get("command"), mounts=volumes, remove=True, stdout=True, stderr=True) logging.info("ret = {0}".format(ret)) return ret.decode("utf-8")
def _get_mounts(user_working_dir: str): """ Retrieve the list of folders to be mounted. It gets the Mount object from the given user working directory, and can include any other needed folder. :param user_working_dir: User working path. :returns: List of docker Mount objects. """ # mount target dir needs to be absolute target_dir = default_workdir user_dir = Mount(target=target_dir, source=user_working_dir, type='bind') # WARNING: mounting .COMPSs makes it fail # compss_dir = os.environ['HOME'] + '/.COMPSs' # os.makedirs(compss_dir, exist_ok=True) # # compss_log_dir = Mount(target='/root/.COMPSs', # source=compss_dir, # type='bind') mounts = [user_dir] # , compss_log_dir] return mounts
def test_linux_binaries() -> None: """ ``make_linux_binaries`` creates binaries which can be run on Linux. """ binary_paths = make_linux_binaries( repo_root=Path(__file__).parent.parent.parent, ) binary_path_names = set(path.name for path in binary_paths) assert binary_path_names == {'dcos-docker', 'dcos-aws', 'dcos-vagrant'} mounts = [] remote_binaries_dir = Path('/binaries') remote_paths = [] for path in binary_paths: remote_path = remote_binaries_dir / path.name mounts.append( Mount( source=str(path.absolute()), target=str(remote_path), type='bind', ), ) remote_paths.append(remote_path) client = docker.from_env(version='auto') for remote_path in remote_paths: cmd_in_container = [ 'chmod', '+x', str(remote_path), '&&', str(remote_path), '--help', ] cmd = 'bash -c "{cmd}"'.format(cmd=' '.join(cmd_in_container)) client.containers.run( image='python:3.6', mounts=mounts, command=cmd, remove=True, )
def getUserMounts(self): dockMounts = [] for mount in self.conf["mount"]: src, label, mode = splitEsc(mount, ":", 2) src = os.path.expanduser(src) if os.path.isfile(src): raise AliDockError("mount {src} is a file: only dirs allowed".format(src=src)) if not label: label = os.path.basename(src) elif "/" in label or label in [".", ".."]: raise AliDockError("mount label {label} is invalid: label cannot contain a slash" "and cannot be equal to \"..\" or \".\"".format(label=label)) mnt = os.path.join("/", "mnt", label) if not mode: mode = "rw" if mode not in ["rw", "ro"]: raise AliDockError("supported modes for mounts are \"rw\" and \"ro\", " "not {mode}".format(mode=mode)) dockMounts.append(Mount(mnt, src, type="bind", read_only=(mode == "ro"), consistency="cached")) return dockMounts
def _run_image(self) -> Optional[Union[List[str], str]]: """Run a Docker container with the provided image""" self.log.info('Starting docker container from image %s', self.image) if not self.cli: raise Exception("The 'cli' should be initialized before!") if self.mount_tmp_dir: with TemporaryDirectory(prefix='airflowtmp', dir=self.host_tmp_dir) as host_tmp_dir_generated: tmp_mount = Mount(self.tmp_dir, host_tmp_dir_generated, "bind") try: return self._run_image_with_mounts(self.mounts + [tmp_mount], add_tmp_variable=True) except APIError as e: if host_tmp_dir_generated in str(e): self.log.warning( "Using remote engine or docker-in-docker and mounting temporary " "volume from host is not supported. Falling back to " "`mount_tmp_dir=False` mode. You can set `mount_tmp_dir` parameter" " to False to disable mounting and remove the warning" ) return self._run_image_with_mounts(self.mounts, add_tmp_variable=False) raise else: return self._run_image_with_mounts(self.mounts, add_tmp_variable=False)
def get_mount_list(project_cfg): mount_list = [] mounts = get_value_from_dict('mounts', project_cfg) for mount in mounts: if mount: param_dict = {} for param in mount.split(','): key_word = param.split('=') if len(key_word) != 2: return None, 'Error mount {}.'.format(mount) param_dict[key_word[0]] = key_word[1] try: mount_list.append( Mount(target=param_dict['target'], source=param_dict['source'], type='bind')) except Exception as e: return None, e return mount_list, 'Successfully to get mount list.'
def get_default_args(self): return { # Shared configuration for all tasks: 'owner': 'airflow', 'retries': 3, # Shared configuration for all Docker tasks: 'extra_hosts': { 'h020nn': '192.168.1.103', 'h020jt': '192.168.1.104', # Note that H3 config uses proper domain names like h3rm.wa.bl.uk }, 'mounts': [ Mount( source=self.storage_path, target='/storage', type='bind' ) ], 'email_on_failure': True, 'email': [ Variable.get('alert_email_address') ], 'auto_remove': False, # True is a bit aggressive and stops Airflow grabbing container logs. 'do_xcom_push': False, # This is not currently working with DockerOperators so defaulting to off for now. 'mount_tmp_dir': False, # Not supported by docker-in-docker tasks }
def run_docker_container(image: str, timeout: int = 300, command: Optional[str] = None, reraise: bool = False, mount: Optional[Tuple[str, str]] = None, label: str = 'Docker', include_stderr: bool = True) -> str: container = None try: kwargs = {'mounts': [Mount(*mount, read_only=False, type='bind')]} if mount else {} client = docker.from_env() container = client.containers.run(image, command=command, network_disabled=True, detach=True, **kwargs) container.wait(timeout=timeout) return container.logs(stderr=include_stderr).decode() except ReadTimeout: logging.warning('[{}]: timeout while processing'.format(label)) if reraise: raise except (DockerException, IOError): logging.warning('[{}]: encountered process error while processing'.format(label)) if reraise: raise finally: if container: with suppress(DockerException): container.stop() container.remove()
def start_checker(docker_client, task, host_workdir, filename): name = get_container_name("checker", task["_id"]) image = get_or_pull_image(docker_client, CHECKER_IMAGE) # remove container should it exists (should not) try: remove_container(docker_client, name) prune_containers(docker_client, {"label": [f"filename={filename}"]}) except docker.errors.NotFound: pass # in container paths workdir = pathlib.Path("/data") filepath = workdir.joinpath(filename) mounts = [Mount(str(workdir), str(host_workdir), type="bind", read_only=True)] command = [ "zimcheck", task["upload"]["zim"]["zimcheck"] or "-A", str(filepath), ] return run_container( docker_client, image=image, command=command, detach=True, name=name, mounts=mounts, labels={ "zimfarm": "", "task_id": task["_id"], "tid": short_id(task["_id"]), "schedule_name": task["schedule_name"], "filename": filename, }, remove=False, )
def run_lambda(task, event): client = docker.from_env() container_name = task['runtime'].lower().replace(" ", "") mount = Mount(type="volume", source=task['task_id'], target="/var/task") response = client.containers.run(f"lambci/lambda:{container_name}", command=[f"{task['task_handler']}", dumps(event)], mounts=[mount], stderr=True, remove=True) # TODO: magic of 2 enters is very flaky, Need to think on how to workound, probably with specific logging log = response.decode("utf-8", errors='ignore') if container_name == "python3.7": results = re.search(r'({.+?)}', log).group(0) else: results = log.split("\n\n")[1] data = {"ts": int(mktime(datetime.utcnow().timetuple())), 'results': results, 'stderr': log} headers = { "Content-Type": "application/json", "Token": task['token'] } post(f'{APP_HOST}/task/{task["task_id"]}/results', headers=headers, data=dumps(data)) return results
def get_dynamic_mounts(param_env): mounts = [] try: response = requests.get("http://orchest-webserver/store/datasources") response.raise_for_status() datasources = response.json() for datasource in datasources: if datasource["source_type"] == "host-directory": mount = Mount(target="/data/%s" % datasource["name"], source=datasource["connection_details"] ["absolute_host_path"], type='bind') mounts.append(mount) except Exception as e: print(e) return mounts
def get_mount_for(self, source: Union[str, tarfile.TarFile], destination: str, mount_point: str) -> Tuple[Mount, str]: """Return a mount and the location of a tarfile to put in that mount. source should be the location of the source files destination should be the mount point inside the container mount_point should be the host mount point. """ tmpstore = os.path.join(root, 'tmp', 'quick_deployments', 'tmpstore') check_isdir(tmpstore) if isinstance(source, str): with tarfile.open( os.path.join(tmpstore, '%s.tar' % hash_of_str(mount_point)[:15]), 'w') as tf: for f in list_recursively(source): tf.add(f) else: os.makedirs(tmpstore) # Extract the received tarfile into a temporary storage. source.extractall(tmpstore) # then write the temporary storage to a new archive. with tarfile.open( os.path.join(root, 'tmp', 'quick_deployments', '%s.tar' % hash_of_str(mount_point)[:15]), 'w') as tf: for f in list_recursively(tmpstore): tf.add(f) for f in list_recursively(tmpstore): os.remove(f) os.removedirs(tmpstore) mnt = Mount(target=destination, source=mount_point, type='bind', read_only=True) return mnt, os.path.join(root, 'tmp', 'quick_deployments', '%s.tar' % hash_of_str(mount_point)[:15])
def mounts(self): if len(self.volume_binds): binds = self.volume_binds.items() if self.hub_volume_local_path: for volume_name, vol in binds: local_volume_dir = self.hub_volume_local_path_for_volume( volume_name) if not os.path.exists(local_volume_dir): self.log.info( "Creating local mount directory: {}".format( local_volume_dir)) os.mkdir(local_volume_dir) return [ Mount( target=vol["bind"], source=volume_name, type="volume", read_only=self.volume_mode_read_only(vol["mode"]), driver_config=self.mount_driver_config_for_volume( volume_name), ) for volume_name, vol in binds ] else: return []
def _validate_volume_line(volume_line): parts = volume_line.split(":") if len(parts) != 4: raise ValueError(f"Volume {volume_line!r} is malformed") if parts[0] != "bind": raise ValueError(f"Volume {volume_line!r} type must be 'bind:'") if parts[1] not in ("ro", "rw"): raise ValueError( f"Volume {volume_line!r} options must be 'ro' or 'rw'") volume_type, mode, outside, inside = parts if not os.path.exists(outside): raise ValueError(f"Volume source {outside!r} does not exist") if not os.path.isabs(outside): raise ValueError(f"Volume source {outside!r} must be an absolute path") if not os.path.isabs(inside): raise ValueError(f"Mount point {inside!r} must be an absolute path") return Mount( source=outside, target=inside, type=volume_type, read_only=bool(mode == "ro"), )
def __init__(self, config_line: str) -> None: parts = config_line.split(":") if len(parts) != 4: raise ValueError(f"Volume {config_line!r} is malformed") if parts[0] != "bind": raise ValueError(f"Volume {config_line!r} type must be 'bind:'") if parts[1] not in ("ro", "rw"): raise ValueError( f"Volume {config_line!r} options must be 'ro' or 'rw'") volume_type, mode, outside, inside = parts if not os.path.isabs(outside): raise ValueError( f"Volume source {outside!r} must be an absolute path") if not os.path.isabs(inside): raise ValueError( f"Mount point {inside!r} must be an absolute path") self.docker_mount = Mount( source=outside, target=inside, type=volume_type, read_only=bool(mode == "ro"), )
config_path = join(dirname(realpath(__file__)), "jupyterhub_configs", "dummy_auth_config.py") # image build jhub_image = {"path": docker_path, "tag": IMAGE, "rm": "True", "pull": "True"} target_config = "/etc/jupyterhub/jupyterhub_config.py" # container cmd jhub_cont = { "image": IMAGE, "name": IMAGE_NAME, "mounts": [ Mount(source=config_path, target=target_config, read_only=True, type="bind") ], "ports": { PORT: PORT }, "detach": "True", } @pytest.mark.parametrize("build_image", [jhub_image], indirect=["build_image"]) @pytest.mark.parametrize("container", [jhub_cont], indirect=["container"]) def test_dummy_auth(build_image, container): """ Test that the client is able to.
def test_parse_mount_bind_windows(self): with mock.patch('docker.types.services.IS_WINDOWS_PLATFORM', True): mount = Mount.parse_mount_string('C:/foo/bar:/baz') assert mount['Source'] == "C:/foo/bar" assert mount['Target'] == "/baz" assert mount['Type'] == 'bind'
def get_basic_docker_config( self, lean_config: Dict[str, Any], algorithm_file: Path, output_dir: Path, debugging_method: Optional[DebuggingMethod]) -> Dict[str, Any]: """Creates a basic Docker config to run the engine with. This method constructs the parts of the Docker config that is the same for both the engine and the optimizer. :param lean_config: the LEAN configuration to use :param algorithm_file: the path to the file containing the algorithm :param output_dir: the directory to save output data to :param debugging_method: the debugging method if debugging needs to be enabled, None if not :return: the Docker configuration containing basic configuration to run Lean """ project_dir = algorithm_file.parent # Create the output directory if it doesn't exist yet if not output_dir.exists(): output_dir.mkdir(parents=True) # Create the storage directory if it doesn't exist yet storage_dir = project_dir / "storage" if not storage_dir.exists(): storage_dir.mkdir(parents=True) lean_config["data-folder"] = "/Lean/Data" lean_config["results-destination-folder"] = "/Results" lean_config["object-store-root"] = "/Storage" config_path = self._temp_manager.create_temporary_directory( ) / "config.json" with config_path.open("w+", encoding="utf-8") as file: file.write(json.dumps(lean_config, indent=4)) # The dict containing all options passed to `docker run` # See all available options at https://docker-py.readthedocs.io/en/stable/containers.html run_options: Dict[str, Any] = { "commands": [], "environment": {}, "stop_signal": "SIGINT" if debugging_method is None else "SIGKILL", "mounts": [ Mount(target="/Lean/Launcher/bin/Debug/config.json", source=str(config_path), type="bind", read_only=True) ], "volumes": {}, "ports": {} } # Mount the data directory data_dir = self._lean_config_manager.get_data_directory() run_options["volumes"][str(data_dir)] = { "bind": "/Lean/Data", "mode": "rw" } # Mount the output directory run_options["volumes"][str(output_dir)] = { "bind": "/Results", "mode": "rw" } # Mount the local object store directory run_options["volumes"][str(storage_dir)] = { "bind": "/Storage", "mode": "rw" } # Make sure host.docker.internal resolves on Linux # See https://github.com/QuantConnect/Lean/pull/5092 if platform.system() == "Linux": run_options["extra_hosts"] = {"host.docker.internal": "172.17.0.1"} # Set up language-specific run options if algorithm_file.name.endswith(".py"): self.set_up_python_options(project_dir, "/LeanCLI", run_options) else: self.set_up_csharp_options(project_dir, run_options) return run_options
def test_parse_mount_named_volume(self): mount = Mount.parse_mount_string("foobar:/baz") assert mount['Source'] == 'foobar' assert mount['Target'] == '/baz' assert mount['Type'] == 'volume'
def test_parse_mount_bind(self): mount = Mount.parse_mount_string('/foo/bar:/baz') assert mount['Source'] == "/foo/bar" assert mount['Target'] == "/baz" assert mount['Type'] == 'bind'
def test_parse_mount_string_short_form(self): mount = Mount.parse_mount_string("/foo/bar:/baz") assert mount['Source'] == "/foo/bar" assert mount['Target'] == "/baz" assert not mount['ReadOnly']
def test_parse_mount_string_ro(self): mount = Mount.parse_mount_string("/foo/bar:/baz:ro") assert mount['Source'] == "/foo/bar" assert mount['Target'] == "/baz" assert mount['ReadOnly'] is True
def test_parse_mount_string_no_source(self): mount = Mount.parse_mount_string("foo/bar") assert mount['Source'] is None assert mount['Target'] == "foo/bar" assert not mount['ReadOnly']
def test_parse_mount_string_invalid(self): with pytest.raises(InvalidArgument): Mount.parse_mount_string("foo:bar:baz:rw")