def test_execute_xcom_behavior(self):
self.client_mock.pull.return_value = [b'{"status":"pull log"}']
kwargs = {
'api_version':
'1.19',
'command':
'env',
'environment': {
'UNIT': 'TEST'
},
'private_environment': {
'PRIVATE': 'MESSAGE'
},
'image':
'ubuntu:latest',
'network_mode':
'bridge',
'owner':
'unittest',
'task_id':
'unittest',
'mounts': [
Mount(source='/host/path',
target='/container/path',
type='bind')
],
'working_dir':
'/container/path',
'shm_size':
1000,
'host_tmp_dir':
'/host/airflow',
'container_name':
'test_container',
'tty':
True,
}
xcom_push_operator = DockerOperator(**kwargs, do_xcom_push=True)
no_xcom_push_operator = DockerOperator(**kwargs, do_xcom_push=False)
xcom_push_result = xcom_push_operator.execute(None)
no_xcom_push_result = no_xcom_push_operator.execute(None)
assert xcom_push_result == b'container log'
assert no_xcom_push_result is None
def create_bind_mount(target: str, bind) -> Mount:
mode = 'rw'
src = None
if isinstance(bind, str):
src = bind
elif isinstance(bind, dict):
src = bind.get('src')
mode = bind.get('mode', 'rw')
else:
raise TypeError(f'Invalid bind volume definition {target}')
return Mount(
type='bind',
target=target,
source=src,
read_only=mode != 'rw',
)
def make_linux_binaries(repo_root: Path) -> Set[Path]:
"""
Create binaries for Linux in a Docker container.
Args:
repo_root: The path to the root of the repository.
Returns:
A set of paths to the built binaries.
"""
client = docker.from_env(version='auto')
dist_dir = repo_root / 'dist'
assert not dist_dir.exists() or not set(dist_dir.iterdir())
target_dir = '/e2e'
code_mount = Mount(
source=str(repo_root.absolute()),
target=target_dir,
type='bind',
)
cmd_in_container = [
'pip',
'install',
'.[packaging]',
'&&',
'python',
'admin/create_pyinstaller_binaries.py',
]
command = 'bash -c "{cmd}"'.format(cmd=' '.join(cmd_in_container))
container = client.containers.run(
image='python:3.7',
mounts=[code_mount],
command=command,
working_dir=target_dir,
remove=True,
detach=True,
)
for line in container.logs(stream=True):
line = line.strip()
LOGGER.info(line)
status_code = container.wait()['StatusCode']
assert status_code == 0
return set(dist_dir.iterdir())
def bind_mount(source: Path, target: Path, **kwargs: Any) -> Mount:
"""Helper for Docker mount objects.
Arguments:
source: the host path to be mounted
target: the container path the source should be mounted to
Keyword arguments:
Passed through to the underlying docker.services.Mount object
initialization
"""
return Mount(
source=str(source),
target=str(target),
type='bind',
**kwargs
)
def __init__(self,
name='dockeranon_workstation',
apps='',
ip='192.168.5.1'):
if WorkstationBuilder.CONTAINER_ID is not None:
return
with TemporaryDirectory() as dir_name:
# Copy files
for f in os.listdir('dockerfiles/workstation'):
if f == 'Dockerfile':
with open('dockerfiles/workstation/Dockerfile') as fd_in:
with open('{0}/Dockerfile'.format(dir_name),
'w') as fd_out:
for line in fd_in.readlines():
line = line.replace(
'<WORKSTATION_IMAGE>',
Config.get('WORKSTATION_IMAGE'))
line = line.replace('<WORKSTATION_APPS>',
self._apps_install(apps))
fd_out.write(line)
else:
shutil.copy('dockerfiles/workstation/{0}'.format(f),
dir_name)
# Build image
client.images.build(pull=True, path=dir_name, tag=name)
# Launch
mounts = []
if Config.get('MOUNT_POINT_USE'):
try:
os.makedirs(Config.get('MOUNT_POINT'))
except FileExistsError:
pass
mount = Mount(target='/home/user/Share',
source=Config.get('MOUNT_POINT'),
type='bind')
mounts.append(mount)
container = client.containers.run(
name,
cap_add=["NET_ADMIN"],
detach=True,
hostname=Config.get('WORKSTATION_HOSTNAME'),
mounts=mounts)
WorkstationBuilder.CONTAINER_ID = container.id
def run_testssl_against_target(self,
target,
results_path,
report_name='testssl_result.json'):
self.target = target
self.results_path = results_path
self.report_name = report_name
self.environment = [
"TARGET={0}".format(self.target),
"REPORT_NAME={0}".format(self.report_name)
]
results_mount = Mount("/results", self.results_path, type="bind")
self.client.containers.run(self.depcheck_docker,
mounts=[results_mount],
environment=self.environment)
logger.info(
"Successfully ran TestSSL against the Target. Please find *.json file in the results directory"
)
def mounts(self):
if len(self.volume_binds):
driver = self.mount_driver_config
# for host_loc, vol in self.volume_binds.items():
# self.log.info('---- {}'.format(host_loc))
# self.log.info('---- {}'.format(vol))
return [
Mount(
target=vol["bind"],
source=host_loc,
type="volume",
read_only=vol["mode"] == "ro",
driver_config=driver,
)
for host_loc, vol in self.volume_binds.items()
]
else:
return []
def convert_to_run_config(image_name, container_spec):
# Convert every mount specification to a docker.types.Mount
mounts = []
for ms in container_spec.get('mounts', []):
mount = Mount(target=ms['target'], source=ms['source'], type='bind')
mounts.append(mount)
run_config = {
'image': image_name,
'command': container_spec.get('command'),
'name': container_spec['name'],
'detach': True,
'mounts': mounts,
'network': config.DOCKER_NETWORK,
'environment': container_spec.get('environment', {}),
'ports': container_spec.get('ports', {}),
'hostname': container_spec.get('hostname'),
}
return run_config
def __init__(self, path, statusService):
self.statusService = statusService
self.client = docker.from_env()
self.image, logs = self.client.images.build(path=f'/projects/{path}',
dockerfile='Dockerfile')
mounts = [
Mount(target="/var/run/docker.sock",
source="/var/run/docker.sock",
type='bind')
]
self.container = self.client.containers.run(image=self.image.id,
tty=True,
detach=True,
mounts=mounts)
self.statusService.add_image_ids(self.image.id, self.container.id,
path, datetime.datetime.now())
for logLine in logs:
print(logLine)
def convert_to_run_config(image_name, container_spec):
# Convert every mount specification to a docker.types.Mount
mounts = []
for ms in container_spec.get("mounts", []):
mount = Mount(target=ms["target"], source=ms["source"], type="bind")
mounts.append(mount)
run_config = {
"image": image_name,
"command": container_spec.get("command"),
"name": container_spec["name"],
"detach": container_spec.get("detach", True),
"mounts": mounts,
"network": config.DOCKER_NETWORK,
"environment": container_spec.get("environment", {}),
"ports": container_spec.get("ports", {}),
"hostname": container_spec.get("hostname"),
"auto_remove": container_spec.get("auto_remove", False),
}
return run_config
def run(data):
s = "from worker " + json.dumps(data)
logging.info(s)
client = docker.from_env()
volumes = list(
map(
lambda l: Mount(l["target"],
l["source"],
type=l["type"],
read_only=l["read_only"]), data["mounts"]))
logging.info("volumes = {0}".format(volumes))
ret = client.containers.run(data["image"],
command=data.get("command"),
mounts=volumes,
remove=True,
stdout=True,
stderr=True)
logging.info("ret = {0}".format(ret))
return ret.decode("utf-8")
def _get_mounts(user_working_dir: str):
"""
Retrieve the list of folders to be mounted. It gets the Mount object
from the given user working directory, and can include any other needed
folder.
:param user_working_dir: User working path.
:returns: List of docker Mount objects.
"""
# mount target dir needs to be absolute
target_dir = default_workdir
user_dir = Mount(target=target_dir, source=user_working_dir, type='bind')
# WARNING: mounting .COMPSs makes it fail
# compss_dir = os.environ['HOME'] + '/.COMPSs'
# os.makedirs(compss_dir, exist_ok=True)
#
# compss_log_dir = Mount(target='/root/.COMPSs',
# source=compss_dir,
# type='bind')
mounts = [user_dir] # , compss_log_dir]
return mounts
def test_linux_binaries() -> None:
"""
``make_linux_binaries`` creates binaries which can be run on Linux.
"""
binary_paths = make_linux_binaries(
repo_root=Path(__file__).parent.parent.parent, )
binary_path_names = set(path.name for path in binary_paths)
assert binary_path_names == {'dcos-docker', 'dcos-aws', 'dcos-vagrant'}
mounts = []
remote_binaries_dir = Path('/binaries')
remote_paths = []
for path in binary_paths:
remote_path = remote_binaries_dir / path.name
mounts.append(
Mount(
source=str(path.absolute()),
target=str(remote_path),
type='bind',
), )
remote_paths.append(remote_path)
client = docker.from_env(version='auto')
for remote_path in remote_paths:
cmd_in_container = [
'chmod',
'+x',
str(remote_path),
'&&',
str(remote_path),
'--help',
]
cmd = 'bash -c "{cmd}"'.format(cmd=' '.join(cmd_in_container))
client.containers.run(
image='python:3.6',
mounts=mounts,
command=cmd,
remove=True,
)
def getUserMounts(self):
dockMounts = []
for mount in self.conf["mount"]:
src, label, mode = splitEsc(mount, ":", 2)
src = os.path.expanduser(src)
if os.path.isfile(src):
raise AliDockError("mount {src} is a file: only dirs allowed".format(src=src))
if not label:
label = os.path.basename(src)
elif "/" in label or label in [".", ".."]:
raise AliDockError("mount label {label} is invalid: label cannot contain a slash"
"and cannot be equal to \"..\" or \".\"".format(label=label))
mnt = os.path.join("/", "mnt", label)
if not mode:
mode = "rw"
if mode not in ["rw", "ro"]:
raise AliDockError("supported modes for mounts are \"rw\" and \"ro\", "
"not {mode}".format(mode=mode))
dockMounts.append(Mount(mnt, src, type="bind", read_only=(mode == "ro"),
consistency="cached"))
return dockMounts
def _run_image(self) -> Optional[Union[List[str], str]]:
"""Run a Docker container with the provided image"""
self.log.info('Starting docker container from image %s', self.image)
if not self.cli:
raise Exception("The 'cli' should be initialized before!")
if self.mount_tmp_dir:
with TemporaryDirectory(prefix='airflowtmp', dir=self.host_tmp_dir) as host_tmp_dir_generated:
tmp_mount = Mount(self.tmp_dir, host_tmp_dir_generated, "bind")
try:
return self._run_image_with_mounts(self.mounts + [tmp_mount], add_tmp_variable=True)
except APIError as e:
if host_tmp_dir_generated in str(e):
self.log.warning(
"Using remote engine or docker-in-docker and mounting temporary "
"volume from host is not supported. Falling back to "
"`mount_tmp_dir=False` mode. You can set `mount_tmp_dir` parameter"
" to False to disable mounting and remove the warning"
)
return self._run_image_with_mounts(self.mounts, add_tmp_variable=False)
raise
else:
return self._run_image_with_mounts(self.mounts, add_tmp_variable=False)
def get_mount_list(project_cfg):
mount_list = []
mounts = get_value_from_dict('mounts', project_cfg)
for mount in mounts:
if mount:
param_dict = {}
for param in mount.split(','):
key_word = param.split('=')
if len(key_word) != 2:
return None, 'Error mount {}.'.format(mount)
param_dict[key_word[0]] = key_word[1]
try:
mount_list.append(
Mount(target=param_dict['target'],
source=param_dict['source'],
type='bind'))
except Exception as e:
return None, e
return mount_list, 'Successfully to get mount list.'
def get_default_args(self):
return {
# Shared configuration for all tasks:
'owner': 'airflow',
'retries': 3,
# Shared configuration for all Docker tasks:
'extra_hosts': {
'h020nn': '192.168.1.103',
'h020jt': '192.168.1.104',
# Note that H3 config uses proper domain names like h3rm.wa.bl.uk
},
'mounts': [
Mount( source=self.storage_path, target='/storage', type='bind' )
],
'email_on_failure': True,
'email': [
Variable.get('alert_email_address')
],
'auto_remove': False, # True is a bit aggressive and stops Airflow grabbing container logs.
'do_xcom_push': False, # This is not currently working with DockerOperators so defaulting to off for now.
'mount_tmp_dir': False, # Not supported by docker-in-docker tasks
}
def run_docker_container(image: str, timeout: int = 300, command: Optional[str] = None, reraise: bool = False,
mount: Optional[Tuple[str, str]] = None, label: str = 'Docker', include_stderr: bool = True) -> str:
container = None
try:
kwargs = {'mounts': [Mount(*mount, read_only=False, type='bind')]} if mount else {}
client = docker.from_env()
container = client.containers.run(image, command=command, network_disabled=True, detach=True, **kwargs)
container.wait(timeout=timeout)
return container.logs(stderr=include_stderr).decode()
except ReadTimeout:
logging.warning('[{}]: timeout while processing'.format(label))
if reraise:
raise
except (DockerException, IOError):
logging.warning('[{}]: encountered process error while processing'.format(label))
if reraise:
raise
finally:
if container:
with suppress(DockerException):
container.stop()
container.remove()
def start_checker(docker_client, task, host_workdir, filename):
name = get_container_name("checker", task["_id"])
image = get_or_pull_image(docker_client, CHECKER_IMAGE)
# remove container should it exists (should not)
try:
remove_container(docker_client, name)
prune_containers(docker_client, {"label": [f"filename={filename}"]})
except docker.errors.NotFound:
pass
# in container paths
workdir = pathlib.Path("/data")
filepath = workdir.joinpath(filename)
mounts = [Mount(str(workdir), str(host_workdir), type="bind", read_only=True)]
command = [
"zimcheck",
task["upload"]["zim"]["zimcheck"] or "-A",
str(filepath),
]
return run_container(
docker_client,
image=image,
command=command,
detach=True,
name=name,
mounts=mounts,
labels={
"zimfarm": "",
"task_id": task["_id"],
"tid": short_id(task["_id"]),
"schedule_name": task["schedule_name"],
"filename": filename,
},
remove=False,
)
def run_lambda(task, event):
client = docker.from_env()
container_name = task['runtime'].lower().replace(" ", "")
mount = Mount(type="volume", source=task['task_id'], target="/var/task")
response = client.containers.run(f"lambci/lambda:{container_name}",
command=[f"{task['task_handler']}", dumps(event)],
mounts=[mount], stderr=True, remove=True)
# TODO: magic of 2 enters is very flaky, Need to think on how to workound, probably with specific logging
log = response.decode("utf-8", errors='ignore')
if container_name == "python3.7":
results = re.search(r'({.+?)}', log).group(0)
else:
results = log.split("\n\n")[1]
data = {"ts": int(mktime(datetime.utcnow().timetuple())), 'results': results, 'stderr': log}
headers = {
"Content-Type": "application/json",
"Token": task['token']
}
post(f'{APP_HOST}/task/{task["task_id"]}/results', headers=headers, data=dumps(data))
return results
def get_dynamic_mounts(param_env):
mounts = []
try:
response = requests.get("http://orchest-webserver/store/datasources")
response.raise_for_status()
datasources = response.json()
for datasource in datasources:
if datasource["source_type"] == "host-directory":
mount = Mount(target="/data/%s" % datasource["name"],
source=datasource["connection_details"]
["absolute_host_path"],
type='bind')
mounts.append(mount)
except Exception as e:
print(e)
return mounts
def get_mount_for(self, source: Union[str, tarfile.TarFile],
destination: str, mount_point: str) -> Tuple[Mount, str]:
"""Return a mount and the location of a tarfile to put in that mount.
source should be the location of the source files
destination should be the mount point inside the container
mount_point should be the host mount point.
"""
tmpstore = os.path.join(root, 'tmp', 'quick_deployments', 'tmpstore')
check_isdir(tmpstore)
if isinstance(source, str):
with tarfile.open(
os.path.join(tmpstore,
'%s.tar' % hash_of_str(mount_point)[:15]),
'w') as tf:
for f in list_recursively(source):
tf.add(f)
else:
os.makedirs(tmpstore)
# Extract the received tarfile into a temporary storage.
source.extractall(tmpstore)
# then write the temporary storage to a new archive.
with tarfile.open(
os.path.join(root, 'tmp', 'quick_deployments',
'%s.tar' % hash_of_str(mount_point)[:15]),
'w') as tf:
for f in list_recursively(tmpstore):
tf.add(f)
for f in list_recursively(tmpstore):
os.remove(f)
os.removedirs(tmpstore)
mnt = Mount(target=destination,
source=mount_point,
type='bind',
read_only=True)
return mnt, os.path.join(root, 'tmp', 'quick_deployments',
'%s.tar' % hash_of_str(mount_point)[:15])
def mounts(self):
if len(self.volume_binds):
binds = self.volume_binds.items()
if self.hub_volume_local_path:
for volume_name, vol in binds:
local_volume_dir = self.hub_volume_local_path_for_volume(
volume_name)
if not os.path.exists(local_volume_dir):
self.log.info(
"Creating local mount directory: {}".format(
local_volume_dir))
os.mkdir(local_volume_dir)
return [
Mount(
target=vol["bind"],
source=volume_name,
type="volume",
read_only=self.volume_mode_read_only(vol["mode"]),
driver_config=self.mount_driver_config_for_volume(
volume_name),
) for volume_name, vol in binds
]
else:
return []
def _validate_volume_line(volume_line):
parts = volume_line.split(":")
if len(parts) != 4:
raise ValueError(f"Volume {volume_line!r} is malformed")
if parts[0] != "bind":
raise ValueError(f"Volume {volume_line!r} type must be 'bind:'")
if parts[1] not in ("ro", "rw"):
raise ValueError(
f"Volume {volume_line!r} options must be 'ro' or 'rw'")
volume_type, mode, outside, inside = parts
if not os.path.exists(outside):
raise ValueError(f"Volume source {outside!r} does not exist")
if not os.path.isabs(outside):
raise ValueError(f"Volume source {outside!r} must be an absolute path")
if not os.path.isabs(inside):
raise ValueError(f"Mount point {inside!r} must be an absolute path")
return Mount(
source=outside,
target=inside,
type=volume_type,
read_only=bool(mode == "ro"),
)
def __init__(self, config_line: str) -> None:
parts = config_line.split(":")
if len(parts) != 4:
raise ValueError(f"Volume {config_line!r} is malformed")
if parts[0] != "bind":
raise ValueError(f"Volume {config_line!r} type must be 'bind:'")
if parts[1] not in ("ro", "rw"):
raise ValueError(
f"Volume {config_line!r} options must be 'ro' or 'rw'")
volume_type, mode, outside, inside = parts
if not os.path.isabs(outside):
raise ValueError(
f"Volume source {outside!r} must be an absolute path")
if not os.path.isabs(inside):
raise ValueError(
f"Mount point {inside!r} must be an absolute path")
self.docker_mount = Mount(
source=outside,
target=inside,
type=volume_type,
read_only=bool(mode == "ro"),
)
config_path = join(dirname(realpath(__file__)), "jupyterhub_configs",
"dummy_auth_config.py")
# image build
jhub_image = {"path": docker_path, "tag": IMAGE, "rm": "True", "pull": "True"}
target_config = "/etc/jupyterhub/jupyterhub_config.py"
# container cmd
jhub_cont = {
"image":
IMAGE,
"name":
IMAGE_NAME,
"mounts": [
Mount(source=config_path,
target=target_config,
read_only=True,
type="bind")
],
"ports": {
PORT: PORT
},
"detach":
"True",
}
@pytest.mark.parametrize("build_image", [jhub_image], indirect=["build_image"])
@pytest.mark.parametrize("container", [jhub_cont], indirect=["container"])
def test_dummy_auth(build_image, container):
"""
Test that the client is able to.
def test_parse_mount_bind_windows(self):
with mock.patch('docker.types.services.IS_WINDOWS_PLATFORM', True):
mount = Mount.parse_mount_string('C:/foo/bar:/baz')
assert mount['Source'] == "C:/foo/bar"
assert mount['Target'] == "/baz"
assert mount['Type'] == 'bind'
def get_basic_docker_config(
self, lean_config: Dict[str, Any], algorithm_file: Path,
output_dir: Path,
debugging_method: Optional[DebuggingMethod]) -> Dict[str, Any]:
"""Creates a basic Docker config to run the engine with.
This method constructs the parts of the Docker config that is the same for both the engine and the optimizer.
:param lean_config: the LEAN configuration to use
:param algorithm_file: the path to the file containing the algorithm
:param output_dir: the directory to save output data to
:param debugging_method: the debugging method if debugging needs to be enabled, None if not
:return: the Docker configuration containing basic configuration to run Lean
"""
project_dir = algorithm_file.parent
# Create the output directory if it doesn't exist yet
if not output_dir.exists():
output_dir.mkdir(parents=True)
# Create the storage directory if it doesn't exist yet
storage_dir = project_dir / "storage"
if not storage_dir.exists():
storage_dir.mkdir(parents=True)
lean_config["data-folder"] = "/Lean/Data"
lean_config["results-destination-folder"] = "/Results"
lean_config["object-store-root"] = "/Storage"
config_path = self._temp_manager.create_temporary_directory(
) / "config.json"
with config_path.open("w+", encoding="utf-8") as file:
file.write(json.dumps(lean_config, indent=4))
# The dict containing all options passed to `docker run`
# See all available options at https://docker-py.readthedocs.io/en/stable/containers.html
run_options: Dict[str, Any] = {
"commands": [],
"environment": {},
"stop_signal":
"SIGINT" if debugging_method is None else "SIGKILL",
"mounts": [
Mount(target="/Lean/Launcher/bin/Debug/config.json",
source=str(config_path),
type="bind",
read_only=True)
],
"volumes": {},
"ports": {}
}
# Mount the data directory
data_dir = self._lean_config_manager.get_data_directory()
run_options["volumes"][str(data_dir)] = {
"bind": "/Lean/Data",
"mode": "rw"
}
# Mount the output directory
run_options["volumes"][str(output_dir)] = {
"bind": "/Results",
"mode": "rw"
}
# Mount the local object store directory
run_options["volumes"][str(storage_dir)] = {
"bind": "/Storage",
"mode": "rw"
}
# Make sure host.docker.internal resolves on Linux
# See https://github.com/QuantConnect/Lean/pull/5092
if platform.system() == "Linux":
run_options["extra_hosts"] = {"host.docker.internal": "172.17.0.1"}
# Set up language-specific run options
if algorithm_file.name.endswith(".py"):
self.set_up_python_options(project_dir, "/LeanCLI", run_options)
else:
self.set_up_csharp_options(project_dir, run_options)
return run_options
def test_parse_mount_named_volume(self):
mount = Mount.parse_mount_string("foobar:/baz")
assert mount['Source'] == 'foobar'
assert mount['Target'] == '/baz'
assert mount['Type'] == 'volume'
def test_parse_mount_bind(self):
mount = Mount.parse_mount_string('/foo/bar:/baz')
assert mount['Source'] == "/foo/bar"
assert mount['Target'] == "/baz"
assert mount['Type'] == 'bind'
def test_parse_mount_string_short_form(self):
mount = Mount.parse_mount_string("/foo/bar:/baz")
assert mount['Source'] == "/foo/bar"
assert mount['Target'] == "/baz"
assert not mount['ReadOnly']
def test_parse_mount_string_ro(self):
mount = Mount.parse_mount_string("/foo/bar:/baz:ro")
assert mount['Source'] == "/foo/bar"
assert mount['Target'] == "/baz"
assert mount['ReadOnly'] is True
def test_parse_mount_string_no_source(self):
mount = Mount.parse_mount_string("foo/bar")
assert mount['Source'] is None
assert mount['Target'] == "foo/bar"
assert not mount['ReadOnly']
def test_parse_mount_string_invalid(self):
with pytest.raises(InvalidArgument):
Mount.parse_mount_string("foo:bar:baz:rw")
def test_parse_mount_named_volume(self):
mount = Mount.parse_mount_string("foobar:/baz")
assert mount['Source'] == 'foobar'
assert mount['Target'] == '/baz'
assert mount['Type'] == 'volume'
def test_parse_mount_bind(self):
mount = Mount.parse_mount_string('/foo/bar:/baz')
assert mount['Source'] == "/foo/bar"
assert mount['Target'] == "/baz"
assert mount['Type'] == 'bind'
def test_parse_mount_bind_windows(self):
with mock.patch('docker.types.services.IS_WINDOWS_PLATFORM', True):
mount = Mount.parse_mount_string('C:/foo/bar:/baz')
assert mount['Source'] == "C:/foo/bar"
assert mount['Target'] == "/baz"
assert mount['Type'] == 'bind'
