def collect(self): """Get tools' locations and copy them to a single location.""" for app_name, tools_path in get_apps_tools().items(): self.stdout.write("Copying files from '{}'.".format(tools_path)) app_name = app_name.replace(".", "_") app_destination_path = os.path.join(self.destination_path, app_name) if not os.path.isdir(app_destination_path): os.mkdir(app_destination_path) for root, dirs, files in os.walk(tools_path): for dir_name in dirs: dir_source_path = os.path.join(root, dir_name) dir_destination_path = self.change_path_prefix( dir_source_path, tools_path, self.destination_path, app_name) if not os.path.isdir(dir_destination_path): os.mkdir(dir_destination_path) for file_name in files: file_source_path = os.path.join(root, file_name) file_destination_path = self.change_path_prefix( file_source_path, tools_path, self.destination_path, app_name) shutil.copy2(file_source_path, file_destination_path)
def collect(self): """Get tools' locations and copy them to a single location.""" for app_name, tools_path in get_apps_tools().items(): self.stdout.write("Copying files from '{}'.".format(tools_path)) app_name = app_name.replace('.', '_') app_destination_path = os.path.join(self.destination_path, app_name) if not os.path.isdir(app_destination_path): os.mkdir(app_destination_path) for root, dirs, files in os.walk(tools_path): for dir_name in dirs: dir_source_path = os.path.join(root, dir_name) dir_destination_path = self.change_path_prefix( dir_source_path, tools_path, self.destination_path, app_name ) if not os.path.isdir(dir_destination_path): os.mkdir(dir_destination_path) for file_name in files: file_source_path = os.path.join(root, file_name) file_destination_path = self.change_path_prefix( file_source_path, tools_path, self.destination_path, app_name ) shutil.copy2(file_source_path, file_destination_path)
def update_tools_configmaps(self, core_api): """Create or update configmaps for tools.""" def dict_from_directory(directory: Path) -> Dict[str, str]: """Get dictionary from given directory. File names are keys and corresponding file contents are values. """ return { entry.name: entry.read_text() for entry in directory.glob("*") if entry.is_file() } configmaps = dict() for app_name, tool_path in get_apps_tools().items(): logger.info(__("Processing '{}' from '{}'.", app_name, tool_path)) data = dict_from_directory(tool_path) data_md5 = hashlib.md5( json.dumps(data, sort_keys=True).encode() ).hexdigest() configmap_name = sanitize_kubernetes_label(f"tools-{app_name}-{data_md5}") logger.info(__("Assigned configmap name '{}'.", configmap_name)) self.update_configmap(configmap_name, data, core_api) configmaps[sanitize_kubernetes_label(app_name)] = configmap_name description_configmap_name = getattr( settings, "KUBERNETES_TOOLS_CONFIGMAPS", "tools-configmaps" ) logger.info(__("Updating main configmap '{}'", description_configmap_name)) self.update_configmap(description_configmap_name, configmaps, core_api)
def get_tools_paths(self, from_applications=False): """Get tools' paths.""" if settings.DEBUG or is_testing() or from_applications: return list(get_apps_tools().values()) else: tools_root = storage_settings.FLOW_VOLUMES["tools"]["config"]["path"] subdirs = next(os.walk(tools_root))[1] return [os.path.join(tools_root, sdir) for sdir in subdirs]
def get_tools_paths(self): """Get tools' paths.""" if settings.DEBUG or is_testing(): return list(get_apps_tools().values()) else: tools_root = settings.FLOW_TOOLS_ROOT subdirs = next(os.walk(tools_root))[1] return [os.path.join(tools_root, sdir) for sdir in subdirs]
def _processing_mountpoints(self, location_subpath: Path, execution_engine_name: str): """Mountpoints for processing container. Processing and input volume (if defined) and all mountable connectors are mounted inside container. All except processing volume are mounted read-only. """ mount_points = [ { "name": constants.PROCESSING_VOLUME_NAME, "mountPath": os.fspath(constants.PROCESSING_VOLUME), "subPath": os.fspath(location_subpath), "readOnly": False, }, ] if constants.INPUTS_VOLUME_NAME in storage_settings.FLOW_VOLUMES: mount_points.append({ "name": constants.INPUTS_VOLUME_NAME, "mountPath": os.fspath(constants.INPUTS_VOLUME), "readOnly": False, }) mount_points += [{ "name": connector.name, "mountPath": f"/{storage_name}_{connector.name}", "readOnly": storage_name != "upload", } for storage_name, connector in get_mountable_connectors()] mount_points += [ { "name": "files-volume", "mountPath": "/etc/passwd", "subPath": "passwd", }, { "name": "files-volume", "mountPath": "/etc/group", "subPath": "group", }, { "name": "files-volume", "mountPath": "/socket_utils.py", "subPath": "socket-utils", }, { "name": "files-volume", "mountPath": "/processing.py", "subPath": "startup-script", }, { "name": "files-volume", "mountPath": "/constants.py", "subPath": "constants", }, { "name": "files-volume", "mountPath": f"/{constants.BOOTSTRAP_PYTHON_RUNTIME}", "subPath": "bootstrap-python-runtime", }, { "name": "files-volume", "mountPath": "/communicator.py", "subPath": "communicator", }, { "name": constants.SOCKETS_VOLUME_NAME, "mountPath": os.fspath(constants.SOCKETS_VOLUME), }, { "name": constants.SECRETS_VOLUME_NAME, "mountPath": os.fspath(constants.SECRETS_VOLUME), "readOnly": True, }, ] for tool_index in range(len(get_apps_tools())): mount_points.append({ "name": f"tools-{tool_index}", "mountPath": f"/usr/local/bin/resolwe/{tool_index}", }) return mount_points