def _get_metadata_configuration(self, schemaspace, name=None): """ Retrieve associated metadata configuration based on schemaspace provided and optional instance name :return: metadata in json format """ try: if not name: return MetadataManager(schemaspace=schemaspace).get_all() else: return MetadataManager(schemaspace=schemaspace).get(name) except BaseException as err: self.log.error(f"Error retrieving metadata configuration for {name}", exc_info=True) raise RuntimeError(f"Error retrieving metadata configuration for {name}", err) from err
async def put(self, schemaspace, resource): schemaspace = url_unescape(schemaspace) resource = url_unescape(resource) parent = self.settings.get("elyra") try: payload = self.get_json_body() # Get the current resource to ensure its pre-existence metadata_manager = MetadataManager(schemaspace=schemaspace, parent=parent) metadata_manager.get(resource) # Check if name is in the payload and varies from resource, if so, raise 400 if "name" in payload and payload["name"] != resource: raise NotImplementedError( f"The attempt to rename instance '{resource}' to '{payload['name']}' is not supported." ) instance = Metadata.from_dict(schemaspace, {**payload}) self.log.debug( f"MetadataHandler: Updating metadata instance '{resource}' in schemaspace '{schemaspace}'..." ) metadata = metadata_manager.update(resource, instance) except (ValidationError, ValueError, NotImplementedError) as err: raise web.HTTPError(400, str(err)) from err except MetadataNotFoundError as err: raise web.HTTPError(404, str(err)) from err except Exception as err: raise web.HTTPError(500, repr(err)) from err self.set_status(200) self.set_header("Content-Type", "application/json") self.finish(metadata.to_dict(trim=True))
async def post(self, schemaspace): schemaspace = url_unescape(schemaspace) parent = self.settings.get("elyra") try: instance = self._validate_body(schemaspace) self.log.debug( f"MetadataHandler: Creating metadata instance '{instance.name}' in schemaspace '{schemaspace}'..." ) metadata_manager = MetadataManager(schemaspace=schemaspace, parent=parent) metadata = metadata_manager.create(instance.name, instance) except (ValidationError, ValueError, SyntaxError) as err: raise web.HTTPError(400, str(err)) from err except (MetadataNotFoundError, SchemaNotFoundError) as err: raise web.HTTPError(404, str(err)) from err except MetadataExistsError as err: raise web.HTTPError(409, str(err)) from err except Exception as err: raise web.HTTPError(500, repr(err)) from err self.set_status(201) self.set_header("Content-Type", "application/json") location = url_path_join(self.base_url, "elyra", "metadata", schemaspace, metadata.name) self.set_header("Location", location) self.finish(metadata.to_dict(trim=True))
def on_load(self, **kwargs: Any) -> None: super().on_load(**kwargs) if self.metadata.get("auth_type") is None: # Inject auth_type property for metadata persisted using Elyra < 3.3: # - api_username and api_password present -> use DEX Legacy # - otherwise -> use no authentication type if (len(self.metadata.get("api_username", "").strip()) == 0 or len( self.metadata.get("api_password", "").strip()) == 0): self.metadata[ "auth_type"] = SupportedAuthProviders.NO_AUTHENTICATION.name else: self.metadata[ "auth_type"] = SupportedAuthProviders.DEX_LEGACY.name if self.metadata.get("cos_auth_type") is None: # Inject cos_auth_type property for metadata persisted using Elyra < 3.4: # - cos_username and cos_password must be present # - cos_secret may be present (above statement also applies in this case) if self.metadata.get("cos_username") and self.metadata.get( "cos_password"): if len(self.metadata.get("cos_secret", "").strip()) == 0: self.metadata["cos_auth_type"] = "USER_CREDENTIALS" else: self.metadata["cos_auth_type"] = "KUBERNETES_SECRET" # save changes MetadataManager(schemaspace="runtimes").update(self.name, self, for_migration=True) return None
def on_load(self, **kwargs: Any) -> None: super().on_load(**kwargs) update_required = False if self.metadata.get("git_type") is None: # Inject git_type property for metadata persisted using Elyra < 3.5: self.metadata["git_type"] = SupportedGitTypes.GITHUB.name update_required = True if self.metadata.get("cos_auth_type") is None: # Inject cos_auth_type property for metadata persisted using Elyra < 3.4: # - cos_username and cos_password must be present # - cos_secret may be present (above statement also applies in this case) if self.metadata.get("cos_username") and self.metadata.get( "cos_password"): if len(self.metadata.get("cos_secret", "")) == 0: self.metadata["cos_auth_type"] = "USER_CREDENTIALS" else: self.metadata["cos_auth_type"] = "KUBERNETES_SECRET" update_required = True if update_required: # save changes MetadataManager(schemaspace="runtimes").update(self.name, self, for_migration=True) return None
def on_modified(self, event): """Fires when the component manifest file is modified.""" self.log.debug( f"ManifestFileChangeHandler: file '{event.src_path}' has been modified." ) manifest = self.component_cache._load_manifest(filename=event.src_path) if manifest: # only update the manifest if there is work to do for catalog, action in manifest.items(): self.log.debug( f"ManifestFileChangeHandler: inserting ({catalog},{action}) into update queue..." ) if action == "delete": # The metadata instance has already been deleted, so we must # fabricate an instance that only consists of a catalog name catalog_instance = ComponentCatalogMetadata(name=catalog) else: # cache_action == 'modify': # Fetch the catalog instance associated with this action catalog_instance = MetadataManager( schemaspace=ComponentCatalogs. COMPONENT_CATALOGS_SCHEMASPACE_ID).get(name=catalog) self.component_cache.update(catalog=catalog_instance, action=action) self.component_cache.update_manifest( filename=event.src_path) # clear the manifest
def _determine_runtime(runtime_config: str) -> str: """Derives the runtime (processor) from the runtime_config.""" # If not present or 'local', treat as special case. if not runtime_config or runtime_config.upper() == RuntimeProcessorType.LOCAL.name: return RuntimeProcessorType.LOCAL.name.lower() runtime_metadata = MetadataManager(schemaspace=Runtimes.RUNTIMES_SCHEMASPACE_ID).get(runtime_config) return runtime_metadata.schema_name
def catalog_instance(component_cache, request): """Creates an instance of a component catalog and removes after test.""" instance_metadata = request.param instance_name = "component_cache" md_mgr = MetadataManager(schemaspace=ComponentCatalogs.COMPONENT_CATALOGS_SCHEMASPACE_ID) catalog = md_mgr.create(instance_name, Metadata(**instance_metadata)) component_cache.wait_for_all_cache_tasks() yield catalog md_mgr.remove(instance_name)
def _get_runtime_config(runtime_config_name: Optional[str]) -> Optional[RuntimesMetadata]: """Fetch runtime configuration for the specified name""" if not runtime_config_name or runtime_config_name == "local": # No runtime configuration was specified or it is local. # Cannot use metadata manager to determine the runtime type. return None try: metadata_manager = MetadataManager(schemaspace=Runtimes.RUNTIMES_SCHEMASPACE_NAME) return metadata_manager.get(runtime_config_name) except Exception as e: raise click.ClickException(f"Invalid runtime configuration: {runtime_config_name}\n {e}")
def __init__(self, **kwargs): super().__init__(**kwargs) self.complex_properties: List[str] = [] self.metadata_manager = MetadataManager(schemaspace=self.schemaspace) # First, process the schema_name option so we can then load the appropriate schema # file to build the schema-based options. If help is requested, give it to them. # As an added benefit, if the schemaspace has one schema, got ahead and default that value. # If multiple, add the list so proper messaging can be applied. As a result, we need to # to build the option here since this is where we have access to the schemas. schema_list = list(self.schemas.keys()) if len(schema_list) == 1: self.schema_name_option = CliOption( "--schema_name", name="schema_name", default_value=schema_list[0], description="The schema_name of the metadata instance to " f"install (defaults to '{schema_list[0]}')", required=True, ) else: enum = schema_list self.schema_name_option = CliOption( "--schema_name", name="schema_name", enum=enum, description= "The schema_name of the metadata instance to install. " f"Must be one of: {enum}", required=True, ) self.options.extend([self.schema_name_option, self.name_option]) # Since we need to know if the replace option is in use prior to normal option processing, # go ahead and check for its existence on the command-line and process if present. if self.replace_flag.cli_option in self.argv_mappings.keys(): self.process_cli_option(self.replace_flag) # Determine if --json, --file, or --replace are in use and relax required properties if so. bulk_metadata = self._process_json_based_options() relax_required = bulk_metadata or self.replace_flag.value # This needs to occur following json-based options since they may add it as an option self.process_cli_option(self.schema_name_option, check_help=True) # Schema appears to be a valid name, convert its properties to options and continue schema = self.schemas[self.schema_name_option.value] # Convert schema properties to options, gathering complex property names schema_options = self._schema_to_options(schema, relax_required) self.options.extend(schema_options)
def tests_manager(jp_environ, schemaspace_location, request): metadata_mgr = MetadataManager(schemaspace=METADATA_TEST_SCHEMASPACE, metadata_store_class=request.param) store_mgr = metadata_mgr.metadata_store create_instance(store_mgr, schemaspace_location, "valid", valid_metadata_json) create_instance(store_mgr, schemaspace_location, "another", another_metadata_json) create_instance(store_mgr, schemaspace_location, "invalid", invalid_metadata_json) create_instance(store_mgr, schemaspace_location, "bad", invalid_json) create_instance(store_mgr, schemaspace_location, "invalid_schema_name", invalid_schema_name_json) return metadata_mgr
def test_manager_add_invalid(tests_manager): with pytest.raises(ValueError): MetadataManager(schemaspace="invalid") # Attempt with non Metadata instance with pytest.raises(TypeError): tests_manager.create(valid_metadata_json) # and invalid parameters with pytest.raises(TypeError): tests_manager.create(None, invalid_no_display_name_json) with pytest.raises(ValueError): tests_manager.create("foo", None)
def airflow_runtime_instance(): """Creates an airflow RTC and removes it after test.""" instance_name = "valid_airflow_test_config" instance_config_file = Path(__file__).parent / "resources" / "runtime_configs" / f"{instance_name}.json" with open(instance_config_file, "r") as fd: instance_config = json.load(fd) md_mgr = MetadataManager(schemaspace=Runtimes.RUNTIMES_SCHEMASPACE_ID) # clean possible orphaned instance... try: md_mgr.remove(instance_name) except Exception: pass runtime_instance = md_mgr.create(instance_name, Metadata(**instance_config)) yield runtime_instance.name md_mgr.remove(runtime_instance.name)
def metadata_manager_with_teardown(jp_environ): """ This fixture provides a MetadataManager instance for certain tests that modify the component catalog. This ensures the catalog instance is removed even when the test fails """ metadata_manager = MetadataManager(schemaspace=ComponentCatalogs.COMPONENT_CATALOGS_SCHEMASPACE_ID) # Run test with provided metadata manager yield metadata_manager # Remove test catalog try: if metadata_manager.get(TEST_CATALOG_NAME): metadata_manager.remove(TEST_CATALOG_NAME) except Exception: pass
def refresh(self): """Triggers a refresh of all catalogs in the component cache. Raises RefreshInProgressError if a complete refresh is in progress. Note that we do not preclude non-server processes from performing a complete refresh. In such cases, each of the catalog entries will be written to the manifest, which will be placed into the update queue. As a result, non-server applications could by-pass the "refresh in progress" constraint, but we're assuming a CLI application won't be as likely to "pound" refresh like a UI application can. """ if self.is_server_process and self.cache_manager.is_refreshing(): raise RefreshInProgressError() catalogs = MetadataManager(schemaspace=ComponentCatalogs.COMPONENT_CATALOGS_SCHEMASPACE_ID).get_all() for catalog in catalogs: self._insert_request(self.refresh_queue, catalog, "modify")
async def get(self, schemaspace): schemaspace = url_unescape(schemaspace) parent = self.settings.get("elyra") try: metadata_manager = MetadataManager(schemaspace=schemaspace, parent=parent) metadata = metadata_manager.get_all() except (ValidationError, ValueError) as err: raise web.HTTPError(400, str(err)) from err except MetadataNotFoundError as err: raise web.HTTPError(404, str(err)) from err except Exception as err: raise web.HTTPError(500, repr(err)) from err metadata_model = {schemaspace: [r.to_dict(trim=True) for r in metadata]} self.set_header("Content-Type", "application/json") self.finish(metadata_model)
def _determine_runtime_type(runtime_config: str) -> str: """Derives the runtime type (platform) from the runtime_config.""" # Pull the runtime_type (platform) from the runtime_config # Need to special case 'local' runtime_config instances if runtime_config.lower() == "local": runtime_type = RuntimeProcessorType.LOCAL else: runtime_metadata = MetadataManager(schemaspace=Runtimes.RUNTIMES_SCHEMASPACE_ID).get(runtime_config) runtime_type_name = runtime_metadata.metadata.get("runtime_type") try: runtime_type = RuntimeProcessorType.get_instance_by_name(runtime_type_name) except (KeyError, TypeError): raise ValueError( f"Unsupported pipeline runtime: '{runtime_type_name}' " f"found in config '{runtime_config}'!" ) return runtime_type.name
async def delete(self, schemaspace, resource): schemaspace = url_unescape(schemaspace) resource = url_unescape(resource) parent = self.settings.get("elyra") try: self.log.debug( f"MetadataHandler: Deleting metadata instance '{resource}' in schemaspace '{schemaspace}'..." ) metadata_manager = MetadataManager(schemaspace=schemaspace, parent=parent) metadata_manager.remove(resource) except (ValidationError, ValueError) as err: raise web.HTTPError(400, str(err)) from err except PermissionError as err: raise web.HTTPError(403, str(err)) from err except MetadataNotFoundError as err: raise web.HTTPError(404, str(err)) from err except Exception as err: raise web.HTTPError(500, repr(err)) from err self.set_status(204) self.finish()
def on_load(self, **kwargs: Any) -> None: """Perform any necessary adjustments, migrations when instance is loaded.""" # If there's no runtime_type property in the instance metadata, infer from schema_name if "runtime_type" not in self.metadata: if self.schema_name == "kfp": self.metadata["runtime_type"] = "KUBEFLOW_PIPELINES" elif self.schema_name == "airflow": self.metadata["runtime_type"] = "APACHE_AIRFLOW" elif self.schema_name == "argo": self.metadata["runtime_type"] = "ARGO" else: raise ValueError( f"Unknown Runtimes schema name detected: '{self.schema_name}'! Skipping..." ) getLogger("ServerApp").info( f"Upgrading runtime {self.schema_name} instance '{self.name}' " f"to include runtime_type '{self.metadata['runtime_type']}'..." ) MetadataManager(schemaspace="runtimes").update(self.name, self, for_migration=True)
async def put(self, catalog): # Validate the body cache_refresh = self.get_json_body() if "action" not in cache_refresh or cache_refresh[ "action"] != "refresh": raise web.HTTPError( 400, reason="A body of {'action': 'refresh'} is required.") try: # Ensure given catalog name is a metadata instance catalog_instance = MetadataManager( schemaspace=ComponentCatalogs.COMPONENT_CATALOGS_SCHEMASPACE_ID ).get(name=catalog) except MetadataNotFoundError: raise web.HTTPError(404, f"Catalog '{catalog}' cannot be found.") self.log.debug( f"Refreshing component cache for catalog with name '{catalog}'...") ComponentCache.instance().update(catalog=catalog_instance, action="modify") self.set_status(204) await self.finish()
def __init__(self, **kwargs): super().__init__(**kwargs) self.metadata_manager = MetadataManager(schemaspace=self.schemaspace)
def tests_hierarchy_manager( setup_hierarchy): # Only uses FileMetadataStore for storage right now. return MetadataManager(schemaspace=METADATA_TEST_SCHEMASPACE)