def collect(self):
"""Get tools' locations and copy them to a single location."""
for app_name, tools_path in get_apps_tools().items():
self.stdout.write("Copying files from '{}'.".format(tools_path))
app_name = app_name.replace(".", "_")
app_destination_path = os.path.join(self.destination_path,
app_name)
if not os.path.isdir(app_destination_path):
os.mkdir(app_destination_path)
for root, dirs, files in os.walk(tools_path):
for dir_name in dirs:
dir_source_path = os.path.join(root, dir_name)
dir_destination_path = self.change_path_prefix(
dir_source_path, tools_path, self.destination_path,
app_name)
if not os.path.isdir(dir_destination_path):
os.mkdir(dir_destination_path)
for file_name in files:
file_source_path = os.path.join(root, file_name)
file_destination_path = self.change_path_prefix(
file_source_path, tools_path, self.destination_path,
app_name)
shutil.copy2(file_source_path, file_destination_path)
def collect(self):
"""Get tools' locations and copy them to a single location."""
for app_name, tools_path in get_apps_tools().items():
self.stdout.write("Copying files from '{}'.".format(tools_path))
app_name = app_name.replace('.', '_')
app_destination_path = os.path.join(self.destination_path, app_name)
if not os.path.isdir(app_destination_path):
os.mkdir(app_destination_path)
for root, dirs, files in os.walk(tools_path):
for dir_name in dirs:
dir_source_path = os.path.join(root, dir_name)
dir_destination_path = self.change_path_prefix(
dir_source_path, tools_path, self.destination_path, app_name
)
if not os.path.isdir(dir_destination_path):
os.mkdir(dir_destination_path)
for file_name in files:
file_source_path = os.path.join(root, file_name)
file_destination_path = self.change_path_prefix(
file_source_path, tools_path, self.destination_path, app_name
)
shutil.copy2(file_source_path, file_destination_path)
def update_tools_configmaps(self, core_api):
"""Create or update configmaps for tools."""
def dict_from_directory(directory: Path) -> Dict[str, str]:
"""Get dictionary from given directory.
File names are keys and corresponding file contents are values.
"""
return {
entry.name: entry.read_text()
for entry in directory.glob("*")
if entry.is_file()
}
configmaps = dict()
for app_name, tool_path in get_apps_tools().items():
logger.info(__("Processing '{}' from '{}'.", app_name, tool_path))
data = dict_from_directory(tool_path)
data_md5 = hashlib.md5(
json.dumps(data, sort_keys=True).encode()
).hexdigest()
configmap_name = sanitize_kubernetes_label(f"tools-{app_name}-{data_md5}")
logger.info(__("Assigned configmap name '{}'.", configmap_name))
self.update_configmap(configmap_name, data, core_api)
configmaps[sanitize_kubernetes_label(app_name)] = configmap_name
description_configmap_name = getattr(
settings, "KUBERNETES_TOOLS_CONFIGMAPS", "tools-configmaps"
)
logger.info(__("Updating main configmap '{}'", description_configmap_name))
self.update_configmap(description_configmap_name, configmaps, core_api)
def get_tools_paths(self, from_applications=False):
"""Get tools' paths."""
if settings.DEBUG or is_testing() or from_applications:
return list(get_apps_tools().values())
else:
tools_root = storage_settings.FLOW_VOLUMES["tools"]["config"]["path"]
subdirs = next(os.walk(tools_root))[1]
return [os.path.join(tools_root, sdir) for sdir in subdirs]
def get_tools_paths(self):
"""Get tools' paths."""
if settings.DEBUG or is_testing():
return list(get_apps_tools().values())
else:
tools_root = settings.FLOW_TOOLS_ROOT
subdirs = next(os.walk(tools_root))[1]
return [os.path.join(tools_root, sdir) for sdir in subdirs]
def get_tools_paths(self):
"""Get tools' paths."""
if settings.DEBUG or is_testing():
return list(get_apps_tools().values())
else:
tools_root = settings.FLOW_TOOLS_ROOT
subdirs = next(os.walk(tools_root))[1]
return [os.path.join(tools_root, sdir) for sdir in subdirs]
def _processing_mountpoints(self, location_subpath: Path,
execution_engine_name: str):
"""Mountpoints for processing container.
Processing and input volume (if defined) and all mountable connectors
are mounted inside container. All except processing volume are mounted
read-only.
"""
mount_points = [
{
"name": constants.PROCESSING_VOLUME_NAME,
"mountPath": os.fspath(constants.PROCESSING_VOLUME),
"subPath": os.fspath(location_subpath),
"readOnly": False,
},
]
if constants.INPUTS_VOLUME_NAME in storage_settings.FLOW_VOLUMES:
mount_points.append({
"name": constants.INPUTS_VOLUME_NAME,
"mountPath": os.fspath(constants.INPUTS_VOLUME),
"readOnly": False,
})
mount_points += [{
"name": connector.name,
"mountPath": f"/{storage_name}_{connector.name}",
"readOnly": storage_name != "upload",
} for storage_name, connector in get_mountable_connectors()]
mount_points += [
{
"name": "files-volume",
"mountPath": "/etc/passwd",
"subPath": "passwd",
},
{
"name": "files-volume",
"mountPath": "/etc/group",
"subPath": "group",
},
{
"name": "files-volume",
"mountPath": "/socket_utils.py",
"subPath": "socket-utils",
},
{
"name": "files-volume",
"mountPath": "/processing.py",
"subPath": "startup-script",
},
{
"name": "files-volume",
"mountPath": "/constants.py",
"subPath": "constants",
},
{
"name": "files-volume",
"mountPath": f"/{constants.BOOTSTRAP_PYTHON_RUNTIME}",
"subPath": "bootstrap-python-runtime",
},
{
"name": "files-volume",
"mountPath": "/communicator.py",
"subPath": "communicator",
},
{
"name": constants.SOCKETS_VOLUME_NAME,
"mountPath": os.fspath(constants.SOCKETS_VOLUME),
},
{
"name": constants.SECRETS_VOLUME_NAME,
"mountPath": os.fspath(constants.SECRETS_VOLUME),
"readOnly": True,
},
]
for tool_index in range(len(get_apps_tools())):
mount_points.append({
"name":
f"tools-{tool_index}",
"mountPath":
f"/usr/local/bin/resolwe/{tool_index}",
})
return mount_points