def test_not_list(self, settings_mock, apps_mock):
apps_mock.get_app_configs.return_value = []
settings_mock.RESOLWE_CUSTOM_TOOLS_PATHS = "/custom_tools"
base_executor = BaseFlowExecutorPreparer()
with self.assertRaisesRegex(KeyError, "setting must be a list"):
base_executor.get_tools_paths()
def test_not_list(self, settings_mock, apps_mock):
apps_mock.get_app_configs.return_value = []
settings_mock.RESOLWE_CUSTOM_TOOLS_PATHS = '/custom_tools'
base_executor = BaseFlowExecutorPreparer()
with self.assertRaisesRegex(KeyError, 'setting must be a list'):
base_executor.get_tools_paths()
def test_get_tools_paths(self, settings_mock, path_mock, apps_mock):
def path_mock_side_effect(*args):
if args[0] == "/custom_tools":
return mock.MagicMock(path=args[0])
else:
return app_config_path_mock
path_mock.side_effect = path_mock_side_effect
app_config1_mock = mock.MagicMock()
app_config1_mock.name = "/resolwe/test_app1"
app_config2_mock = mock.MagicMock()
app_config2_mock.name = "/resolwe/test_app1"
apps_mock.get_app_configs.return_value = [app_config1_mock, app_config2_mock]
first_mock = mock.MagicMock(is_dir=lambda: False, path="/resolwe/test_app1")
second_mock = mock.MagicMock(is_dir=lambda: True, path="/resolwe/test_app2")
app_config_path_mock = mock.MagicMock()
app_config_path_mock.__truediv__.side_effect = [first_mock, second_mock]
settings_mock.RESOLWE_CUSTOM_TOOLS_PATHS = ["/custom_tools"]
base_executor = BaseFlowExecutorPreparer()
tools_list = [mock.path for mock in base_executor.get_tools_paths()]
self.assertEqual(len(tools_list), 2)
self.assertIn("/resolwe/test_app2", tools_list)
self.assertIn("/custom_tools", tools_list)
def _get_tools_configmaps(self, core_api):
"""Get and return configmaps for tools."""
def dict_from_directory(directory: Path) -> Dict[str, str]:
"""Get dictionary from given directory.
File names are keys and corresponding file contents are values.
"""
return {
entry.name: entry.read_text()
for entry in directory.glob("*")
if entry.is_file()
}
configmap_names = []
preparer = BaseFlowExecutorPreparer()
tools_paths = preparer.get_tools_paths()
for tool_path in tools_paths:
tool_path = Path(tool_path)
data = dict_from_directory(tool_path)
data_md5 = hashlib.md5(
json.dumps(data, sort_keys=True).encode()
).hexdigest()
configmap_name = self._sanitize_kubernetes_label(
f"tools-{tool_path.name}-{data_md5}"
)
self._create_configmap_if_needed(configmap_name, data, core_api)
configmap_names.append(configmap_name)
return configmap_names
def test_get_tools_paths(self, settings_mock, os_mock, apps_mock):
apps_mock.get_app_configs.return_value = [
mock.MagicMock(path='/resolwe/test_app1'),
mock.MagicMock(path='/resolwe/test_app2'),
]
os_mock.path.join = os.path.join
os_mock.path.isdir.side_effect = [False, True]
settings_mock.RESOLWE_CUSTOM_TOOLS_PATHS = ['/custom_tools']
base_executor = BaseFlowExecutorPreparer()
tools_list = base_executor.get_tools_paths()
self.assertEqual(len(tools_list), 2)
self.assertIn('/resolwe/test_app2/tools', tools_list)
self.assertIn('/custom_tools', tools_list)
def test_get_tools_paths(self, settings_mock, os_mock, apps_mock):
apps_mock.get_app_configs.return_value = [
mock.MagicMock(path='/resolwe/test_app1'),
mock.MagicMock(path='/resolwe/test_app2'),
]
os_mock.path.join = os.path.join
os_mock.path.isdir.side_effect = [False, True]
settings_mock.RESOLWE_CUSTOM_TOOLS_PATHS = ['/custom_tools']
base_executor = BaseFlowExecutorPreparer()
tools_list = base_executor.get_tools_paths()
self.assertEqual(len(tools_list), 2)
self.assertIn('/resolwe/test_app2/tools', tools_list)
self.assertIn('/custom_tools', tools_list)
def _processing_mountpoints(
self, location_subpath: Path, execution_engine_name: str
):
"""Mountpoints for processing container.
Processing and input volume (if defined) and all mountable connectors
are mounted inside container. All except processing volume are mounted
read-only.
"""
mount_points = [
{
"name": storage_settings.FLOW_VOLUMES["processing"]["config"]["name"],
"mountPath": os.fspath(constants.PROCESSING_VOLUME),
"subPath": os.fspath(location_subpath),
"readOnly": False,
},
]
if "input" in storage_settings.FLOW_VOLUMES:
mount_points.append(
{
"name": storage_settings.FLOW_VOLUMES["input"]["config"]["name"],
"mountPath": os.fspath(constants.INPUTS_VOLUME),
"readOnly": False,
}
)
mount_points += [
{
"name": connector.name,
"mountPath": f"/{storage_name}_{connector.name}",
"readOnly": storage_name != "upload",
}
for storage_name, connector in self._get_mountable_connectors()
]
mount_points += [
{
"name": "files-volume",
"mountPath": "/etc/passwd",
"subPath": "passwd",
},
{
"name": "files-volume",
"mountPath": "/etc/group",
"subPath": "group",
},
{
"name": "files-volume",
"mountPath": "/socket_utils.py",
"subPath": "socket-utils",
},
{
"name": "files-volume",
"mountPath": "/processing.py",
"subPath": "startup-script",
},
{
"name": "files-volume",
"mountPath": "/constants.py",
"subPath": "constants",
},
{
"name": "sockets-volume",
"mountPath": os.fspath(constants.SOCKETS_VOLUME),
},
{
"name": "secrets-volume",
"mountPath": os.fspath(constants.SECRETS_VOLUME),
"readOnly": True,
},
]
if "tools" in storage_settings.FLOW_VOLUMES:
# Create volumes for tools. In kubernetes all tools must be inside
# volume of type persistent_volume or host_path.
volume = storage_settings.FLOW_VOLUMES["tools"]
preparer = BaseFlowExecutorPreparer()
tools_paths = preparer.get_tools_paths()
base_tools_path = volume["config"]["path"]
subpath = volume["config"].get("subpath", "")
if subpath:
tools_paths = [f"{subpath}/{path}" for path in tools_paths]
mount_points += [
{
"name": volume["config"]["name"],
"mountPath": os.fspath(Path("/usr/local/bin/resolwe") / str(index)),
"subPath": os.fspath(Path(tool).relative_to(base_tools_path)),
}
for index, tool in enumerate(tools_paths)
]
from resolwe.flow.managers import manager
execution_engine = manager.get_execution_engine(execution_engine_name)
runtime_volumes = execution_engine.prepare_volumes()
subpath = storage_settings.FLOW_VOLUMES["runtime"]["config"].get("subpath", "")
if subpath:
runtime_volumes = {
f"{subpath}/{src}": dst for src, dst in runtime_volumes.items()
}
mount_points += [
{
"name": storage_settings.FLOW_VOLUMES["runtime"]["config"]["name"],
"mountPath": dst,
"subPath": src,
}
for src, dst in runtime_volumes.items()
]
return mount_points
def _processing_mountpoints(self, location_subpath: Path):
"""Mountpoints for processing container."""
mount_points = [
{
"name": "efs-root",
"mountPath": os.fspath(constants.DATA_ALL_VOLUME),
"subPath": os.fspath(self.efs_data_dir),
"readOnly": True,
},
{
"name": "efs-root",
"mountPath": os.fspath(constants.DATA_VOLUME),
"subPath": os.fspath(self.efs_data_dir / location_subpath),
"readOnly": False,
},
{
"name": "ebs-root",
"mountPath": os.fspath(constants.DATA_LOCAL_VOLUME),
"subPath": os.fspath(location_subpath),
"readOnly": False,
},
# {
# "name": "fsx-root",
# "mountPath": constants.DATA_VOLUME,
# "subPath": os.fspath(location_subpath),
# "readOnly": False,
# },
{
"name": "efs-root",
"mountPath": os.fspath(constants.UPLOAD_VOLUME),
"subPath": os.fspath(self.efs_upload_dir),
"readOnly": False,
},
{
"name": "secrets-volume",
"mountPath": os.fspath(constants.SECRETS_VOLUME),
"readOnly": True,
},
{
"name": "files-volume",
"mountPath": "/etc/passwd",
"subPath": "passwd",
},
{
"name": "files-volume",
"mountPath": "/etc/group",
"subPath": "group",
},
{
"name": "files-volume",
"mountPath": "/socket_utils.py",
"subPath": "socket-utils",
},
{
"name": "files-volume",
"mountPath": "/processing.py",
"subPath": "startup-script",
},
{
"name": "files-volume",
"mountPath": "/constants.py",
"subPath": "constants",
},
{
"name": "sockets-volume",
"mountPath": os.fspath(constants.SOCKETS_VOLUME),
},
]
# Create volumes for tools. Only consider tools located on efs.
preparer = BaseFlowExecutorPreparer()
tools_paths = preparer.get_tools_paths()
mount_points += [{
"name":
"efs-root",
"mountPath":
os.fspath(Path("/usr/local/bin/resolwe") / str(index)),
"subPath":
os.fspath(Path(tool).relative_to(self.efs_root)),
} for index, tool in enumerate(tools_paths)]
# Create volumes for runtime (all read-only).
django_settings_path: Path = (self.runtime_dir / location_subpath /
ExecutorFiles.SETTINGS_SUBDIR /
ExecutorFiles.DJANGO_SETTINGS)
django_settings = json.loads(django_settings_path.read_text())
mount_points += [{
"name":
"efs-root",
"mountPath":
dst,
"subPath":
os.fspath(self.efs_runtime_dir / location_subpath / src),
}
for src, dst in django_settings.get(
"RUNTIME_VOLUME_MAPS", {}).items()]
# Add any extra volumes verbatim.
mount_points += getattr(settings, "FLOW_DOCKER_EXTRA_VOLUMES", [])
logger.debug(f"Runtime volume mount points: {mount_points}.")
return mount_points
def _processing_mountpoints(
self, location_subpath: Path, execution_engine_name: str
):
"""Mountpoints for processing container.
Processing and input volume (if defined) and all mountable connectors
are mounted inside container. All except processing volume are mounted
read-only.
"""
mount_points = [
{
"name": constants.PROCESSING_VOLUME_NAME,
"mountPath": os.fspath(constants.PROCESSING_VOLUME),
"subPath": os.fspath(location_subpath),
"readOnly": False,
},
]
if constants.INPUTS_VOLUME_NAME in storage_settings.FLOW_VOLUMES:
mount_points.append(
{
"name": constants.INPUTS_VOLUME_NAME,
"mountPath": os.fspath(constants.INPUTS_VOLUME),
"readOnly": False,
}
)
mount_points += [
{
"name": connector.name,
"mountPath": f"/{storage_name}_{connector.name}",
"readOnly": storage_name != "upload",
}
for storage_name, connector in self._get_mountable_connectors()
]
mount_points += [
{
"name": "files-volume",
"mountPath": "/etc/passwd",
"subPath": "passwd",
},
{
"name": "files-volume",
"mountPath": "/etc/group",
"subPath": "group",
},
{
"name": "files-volume",
"mountPath": "/socket_utils.py",
"subPath": "socket-utils",
},
{
"name": "files-volume",
"mountPath": "/processing.py",
"subPath": "startup-script",
},
{
"name": "files-volume",
"mountPath": "/constants.py",
"subPath": "constants",
},
{
"name": "sockets-volume",
"mountPath": os.fspath(constants.SOCKETS_VOLUME),
},
{
"name": "secrets-volume",
"mountPath": os.fspath(constants.SECRETS_VOLUME),
"readOnly": True,
},
]
for tool_index in range(len(BaseFlowExecutorPreparer().get_tools_paths())):
mount_points.append(
{
"name": f"tools-{tool_index}",
"mountPath": f"/usr/local/bin/resolwe/{tool_index}",
}
)
from resolwe.flow.managers import manager
execution_engine = manager.get_execution_engine(execution_engine_name)
runtime_volumes = execution_engine.prepare_volumes()
subpath = storage_settings.FLOW_VOLUMES["runtime"]["config"].get("subpath", "")
if subpath:
runtime_volumes = {
f"{subpath}/{src}": dst for src, dst in runtime_volumes.items()
}
mount_points += [
{
"name": "runtime",
"mountPath": dst,
"subPath": src,
}
for src, dst in runtime_volumes.items()
]
return mount_points