def _get_metadata_configuration(self, schemaspace, name=None):
"""
Retrieve associated metadata configuration based on schemaspace provided and optional instance name
:return: metadata in json format
"""
try:
if not name:
return MetadataManager(schemaspace=schemaspace).get_all()
else:
return MetadataManager(schemaspace=schemaspace).get(name)
except BaseException as err:
self.log.error(f"Error retrieving metadata configuration for {name}", exc_info=True)
raise RuntimeError(f"Error retrieving metadata configuration for {name}", err) from err
async def put(self, schemaspace, resource):
schemaspace = url_unescape(schemaspace)
resource = url_unescape(resource)
parent = self.settings.get("elyra")
try:
payload = self.get_json_body()
# Get the current resource to ensure its pre-existence
metadata_manager = MetadataManager(schemaspace=schemaspace, parent=parent)
metadata_manager.get(resource)
# Check if name is in the payload and varies from resource, if so, raise 400
if "name" in payload and payload["name"] != resource:
raise NotImplementedError(
f"The attempt to rename instance '{resource}' to '{payload['name']}' is not supported."
)
instance = Metadata.from_dict(schemaspace, {**payload})
self.log.debug(
f"MetadataHandler: Updating metadata instance '{resource}' in schemaspace '{schemaspace}'..."
)
metadata = metadata_manager.update(resource, instance)
except (ValidationError, ValueError, NotImplementedError) as err:
raise web.HTTPError(400, str(err)) from err
except MetadataNotFoundError as err:
raise web.HTTPError(404, str(err)) from err
except Exception as err:
raise web.HTTPError(500, repr(err)) from err
self.set_status(200)
self.set_header("Content-Type", "application/json")
self.finish(metadata.to_dict(trim=True))
async def post(self, schemaspace):
schemaspace = url_unescape(schemaspace)
parent = self.settings.get("elyra")
try:
instance = self._validate_body(schemaspace)
self.log.debug(
f"MetadataHandler: Creating metadata instance '{instance.name}' in schemaspace '{schemaspace}'..."
)
metadata_manager = MetadataManager(schemaspace=schemaspace, parent=parent)
metadata = metadata_manager.create(instance.name, instance)
except (ValidationError, ValueError, SyntaxError) as err:
raise web.HTTPError(400, str(err)) from err
except (MetadataNotFoundError, SchemaNotFoundError) as err:
raise web.HTTPError(404, str(err)) from err
except MetadataExistsError as err:
raise web.HTTPError(409, str(err)) from err
except Exception as err:
raise web.HTTPError(500, repr(err)) from err
self.set_status(201)
self.set_header("Content-Type", "application/json")
location = url_path_join(self.base_url, "elyra", "metadata", schemaspace, metadata.name)
self.set_header("Location", location)
self.finish(metadata.to_dict(trim=True))
def on_load(self, **kwargs: Any) -> None:
super().on_load(**kwargs)
if self.metadata.get("auth_type") is None:
# Inject auth_type property for metadata persisted using Elyra < 3.3:
# - api_username and api_password present -> use DEX Legacy
# - otherwise -> use no authentication type
if (len(self.metadata.get("api_username", "").strip()) == 0 or len(
self.metadata.get("api_password", "").strip()) == 0):
self.metadata[
"auth_type"] = SupportedAuthProviders.NO_AUTHENTICATION.name
else:
self.metadata[
"auth_type"] = SupportedAuthProviders.DEX_LEGACY.name
if self.metadata.get("cos_auth_type") is None:
# Inject cos_auth_type property for metadata persisted using Elyra < 3.4:
# - cos_username and cos_password must be present
# - cos_secret may be present (above statement also applies in this case)
if self.metadata.get("cos_username") and self.metadata.get(
"cos_password"):
if len(self.metadata.get("cos_secret", "").strip()) == 0:
self.metadata["cos_auth_type"] = "USER_CREDENTIALS"
else:
self.metadata["cos_auth_type"] = "KUBERNETES_SECRET"
# save changes
MetadataManager(schemaspace="runtimes").update(self.name,
self,
for_migration=True)
return None
def on_load(self, **kwargs: Any) -> None:
super().on_load(**kwargs)
update_required = False
if self.metadata.get("git_type") is None:
# Inject git_type property for metadata persisted using Elyra < 3.5:
self.metadata["git_type"] = SupportedGitTypes.GITHUB.name
update_required = True
if self.metadata.get("cos_auth_type") is None:
# Inject cos_auth_type property for metadata persisted using Elyra < 3.4:
# - cos_username and cos_password must be present
# - cos_secret may be present (above statement also applies in this case)
if self.metadata.get("cos_username") and self.metadata.get(
"cos_password"):
if len(self.metadata.get("cos_secret", "")) == 0:
self.metadata["cos_auth_type"] = "USER_CREDENTIALS"
else:
self.metadata["cos_auth_type"] = "KUBERNETES_SECRET"
update_required = True
if update_required:
# save changes
MetadataManager(schemaspace="runtimes").update(self.name,
self,
for_migration=True)
return None
def on_modified(self, event):
"""Fires when the component manifest file is modified."""
self.log.debug(
f"ManifestFileChangeHandler: file '{event.src_path}' has been modified."
)
manifest = self.component_cache._load_manifest(filename=event.src_path)
if manifest: # only update the manifest if there is work to do
for catalog, action in manifest.items():
self.log.debug(
f"ManifestFileChangeHandler: inserting ({catalog},{action}) into update queue..."
)
if action == "delete":
# The metadata instance has already been deleted, so we must
# fabricate an instance that only consists of a catalog name
catalog_instance = ComponentCatalogMetadata(name=catalog)
else: # cache_action == 'modify':
# Fetch the catalog instance associated with this action
catalog_instance = MetadataManager(
schemaspace=ComponentCatalogs.
COMPONENT_CATALOGS_SCHEMASPACE_ID).get(name=catalog)
self.component_cache.update(catalog=catalog_instance,
action=action)
self.component_cache.update_manifest(
filename=event.src_path) # clear the manifest
def _determine_runtime(runtime_config: str) -> str:
"""Derives the runtime (processor) from the runtime_config."""
# If not present or 'local', treat as special case.
if not runtime_config or runtime_config.upper() == RuntimeProcessorType.LOCAL.name:
return RuntimeProcessorType.LOCAL.name.lower()
runtime_metadata = MetadataManager(schemaspace=Runtimes.RUNTIMES_SCHEMASPACE_ID).get(runtime_config)
return runtime_metadata.schema_name
def catalog_instance(component_cache, request):
"""Creates an instance of a component catalog and removes after test."""
instance_metadata = request.param
instance_name = "component_cache"
md_mgr = MetadataManager(schemaspace=ComponentCatalogs.COMPONENT_CATALOGS_SCHEMASPACE_ID)
catalog = md_mgr.create(instance_name, Metadata(**instance_metadata))
component_cache.wait_for_all_cache_tasks()
yield catalog
md_mgr.remove(instance_name)
def _get_runtime_config(runtime_config_name: Optional[str]) -> Optional[RuntimesMetadata]:
"""Fetch runtime configuration for the specified name"""
if not runtime_config_name or runtime_config_name == "local":
# No runtime configuration was specified or it is local.
# Cannot use metadata manager to determine the runtime type.
return None
try:
metadata_manager = MetadataManager(schemaspace=Runtimes.RUNTIMES_SCHEMASPACE_NAME)
return metadata_manager.get(runtime_config_name)
except Exception as e:
raise click.ClickException(f"Invalid runtime configuration: {runtime_config_name}\n {e}")
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.complex_properties: List[str] = []
self.metadata_manager = MetadataManager(schemaspace=self.schemaspace)
# First, process the schema_name option so we can then load the appropriate schema
# file to build the schema-based options. If help is requested, give it to them.
# As an added benefit, if the schemaspace has one schema, got ahead and default that value.
# If multiple, add the list so proper messaging can be applied. As a result, we need to
# to build the option here since this is where we have access to the schemas.
schema_list = list(self.schemas.keys())
if len(schema_list) == 1:
self.schema_name_option = CliOption(
"--schema_name",
name="schema_name",
default_value=schema_list[0],
description="The schema_name of the metadata instance to "
f"install (defaults to '{schema_list[0]}')",
required=True,
)
else:
enum = schema_list
self.schema_name_option = CliOption(
"--schema_name",
name="schema_name",
enum=enum,
description=
"The schema_name of the metadata instance to install. "
f"Must be one of: {enum}",
required=True,
)
self.options.extend([self.schema_name_option, self.name_option])
# Since we need to know if the replace option is in use prior to normal option processing,
# go ahead and check for its existence on the command-line and process if present.
if self.replace_flag.cli_option in self.argv_mappings.keys():
self.process_cli_option(self.replace_flag)
# Determine if --json, --file, or --replace are in use and relax required properties if so.
bulk_metadata = self._process_json_based_options()
relax_required = bulk_metadata or self.replace_flag.value
# This needs to occur following json-based options since they may add it as an option
self.process_cli_option(self.schema_name_option, check_help=True)
# Schema appears to be a valid name, convert its properties to options and continue
schema = self.schemas[self.schema_name_option.value]
# Convert schema properties to options, gathering complex property names
schema_options = self._schema_to_options(schema, relax_required)
self.options.extend(schema_options)
def tests_manager(jp_environ, schemaspace_location, request):
metadata_mgr = MetadataManager(schemaspace=METADATA_TEST_SCHEMASPACE,
metadata_store_class=request.param)
store_mgr = metadata_mgr.metadata_store
create_instance(store_mgr, schemaspace_location, "valid",
valid_metadata_json)
create_instance(store_mgr, schemaspace_location, "another",
another_metadata_json)
create_instance(store_mgr, schemaspace_location, "invalid",
invalid_metadata_json)
create_instance(store_mgr, schemaspace_location, "bad", invalid_json)
create_instance(store_mgr, schemaspace_location, "invalid_schema_name",
invalid_schema_name_json)
return metadata_mgr
def test_manager_add_invalid(tests_manager):
with pytest.raises(ValueError):
MetadataManager(schemaspace="invalid")
# Attempt with non Metadata instance
with pytest.raises(TypeError):
tests_manager.create(valid_metadata_json)
# and invalid parameters
with pytest.raises(TypeError):
tests_manager.create(None, invalid_no_display_name_json)
with pytest.raises(ValueError):
tests_manager.create("foo", None)
def airflow_runtime_instance():
"""Creates an airflow RTC and removes it after test."""
instance_name = "valid_airflow_test_config"
instance_config_file = Path(__file__).parent / "resources" / "runtime_configs" / f"{instance_name}.json"
with open(instance_config_file, "r") as fd:
instance_config = json.load(fd)
md_mgr = MetadataManager(schemaspace=Runtimes.RUNTIMES_SCHEMASPACE_ID)
# clean possible orphaned instance...
try:
md_mgr.remove(instance_name)
except Exception:
pass
runtime_instance = md_mgr.create(instance_name, Metadata(**instance_config))
yield runtime_instance.name
md_mgr.remove(runtime_instance.name)
def metadata_manager_with_teardown(jp_environ):
"""
This fixture provides a MetadataManager instance for certain tests that modify the component
catalog. This ensures the catalog instance is removed even when the test fails
"""
metadata_manager = MetadataManager(schemaspace=ComponentCatalogs.COMPONENT_CATALOGS_SCHEMASPACE_ID)
# Run test with provided metadata manager
yield metadata_manager
# Remove test catalog
try:
if metadata_manager.get(TEST_CATALOG_NAME):
metadata_manager.remove(TEST_CATALOG_NAME)
except Exception:
pass
def refresh(self):
"""Triggers a refresh of all catalogs in the component cache.
Raises RefreshInProgressError if a complete refresh is in progress.
Note that we do not preclude non-server processes from performing a
complete refresh. In such cases, each of the catalog entries will be
written to the manifest, which will be placed into the update queue.
As a result, non-server applications could by-pass the "refresh in progress"
constraint, but we're assuming a CLI application won't be as likely to
"pound" refresh like a UI application can.
"""
if self.is_server_process and self.cache_manager.is_refreshing():
raise RefreshInProgressError()
catalogs = MetadataManager(schemaspace=ComponentCatalogs.COMPONENT_CATALOGS_SCHEMASPACE_ID).get_all()
for catalog in catalogs:
self._insert_request(self.refresh_queue, catalog, "modify")
async def get(self, schemaspace):
schemaspace = url_unescape(schemaspace)
parent = self.settings.get("elyra")
try:
metadata_manager = MetadataManager(schemaspace=schemaspace, parent=parent)
metadata = metadata_manager.get_all()
except (ValidationError, ValueError) as err:
raise web.HTTPError(400, str(err)) from err
except MetadataNotFoundError as err:
raise web.HTTPError(404, str(err)) from err
except Exception as err:
raise web.HTTPError(500, repr(err)) from err
metadata_model = {schemaspace: [r.to_dict(trim=True) for r in metadata]}
self.set_header("Content-Type", "application/json")
self.finish(metadata_model)
def _determine_runtime_type(runtime_config: str) -> str:
"""Derives the runtime type (platform) from the runtime_config."""
# Pull the runtime_type (platform) from the runtime_config
# Need to special case 'local' runtime_config instances
if runtime_config.lower() == "local":
runtime_type = RuntimeProcessorType.LOCAL
else:
runtime_metadata = MetadataManager(schemaspace=Runtimes.RUNTIMES_SCHEMASPACE_ID).get(runtime_config)
runtime_type_name = runtime_metadata.metadata.get("runtime_type")
try:
runtime_type = RuntimeProcessorType.get_instance_by_name(runtime_type_name)
except (KeyError, TypeError):
raise ValueError(
f"Unsupported pipeline runtime: '{runtime_type_name}' " f"found in config '{runtime_config}'!"
)
return runtime_type.name
async def delete(self, schemaspace, resource):
schemaspace = url_unescape(schemaspace)
resource = url_unescape(resource)
parent = self.settings.get("elyra")
try:
self.log.debug(
f"MetadataHandler: Deleting metadata instance '{resource}' in schemaspace '{schemaspace}'..."
)
metadata_manager = MetadataManager(schemaspace=schemaspace, parent=parent)
metadata_manager.remove(resource)
except (ValidationError, ValueError) as err:
raise web.HTTPError(400, str(err)) from err
except PermissionError as err:
raise web.HTTPError(403, str(err)) from err
except MetadataNotFoundError as err:
raise web.HTTPError(404, str(err)) from err
except Exception as err:
raise web.HTTPError(500, repr(err)) from err
self.set_status(204)
self.finish()
def on_load(self, **kwargs: Any) -> None:
"""Perform any necessary adjustments, migrations when instance is loaded."""
# If there's no runtime_type property in the instance metadata, infer from schema_name
if "runtime_type" not in self.metadata:
if self.schema_name == "kfp":
self.metadata["runtime_type"] = "KUBEFLOW_PIPELINES"
elif self.schema_name == "airflow":
self.metadata["runtime_type"] = "APACHE_AIRFLOW"
elif self.schema_name == "argo":
self.metadata["runtime_type"] = "ARGO"
else:
raise ValueError(
f"Unknown Runtimes schema name detected: '{self.schema_name}'! Skipping..."
)
getLogger("ServerApp").info(
f"Upgrading runtime {self.schema_name} instance '{self.name}' "
f"to include runtime_type '{self.metadata['runtime_type']}'..."
)
MetadataManager(schemaspace="runtimes").update(self.name,
self,
for_migration=True)
async def put(self, catalog):
# Validate the body
cache_refresh = self.get_json_body()
if "action" not in cache_refresh or cache_refresh[
"action"] != "refresh":
raise web.HTTPError(
400, reason="A body of {'action': 'refresh'} is required.")
try:
# Ensure given catalog name is a metadata instance
catalog_instance = MetadataManager(
schemaspace=ComponentCatalogs.COMPONENT_CATALOGS_SCHEMASPACE_ID
).get(name=catalog)
except MetadataNotFoundError:
raise web.HTTPError(404, f"Catalog '{catalog}' cannot be found.")
self.log.debug(
f"Refreshing component cache for catalog with name '{catalog}'...")
ComponentCache.instance().update(catalog=catalog_instance,
action="modify")
self.set_status(204)
await self.finish()
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.metadata_manager = MetadataManager(schemaspace=self.schemaspace)
def tests_hierarchy_manager(
setup_hierarchy): # Only uses FileMetadataStore for storage right now.
return MetadataManager(schemaspace=METADATA_TEST_SCHEMASPACE)