async def put(self, schemaspace, resource):
schemaspace = url_unescape(schemaspace)
resource = url_unescape(resource)
parent = self.settings.get("elyra")
try:
payload = self.get_json_body()
# Get the current resource to ensure its pre-existence
metadata_manager = MetadataManager(schemaspace=schemaspace, parent=parent)
metadata_manager.get(resource)
# Check if name is in the payload and varies from resource, if so, raise 400
if "name" in payload and payload["name"] != resource:
raise NotImplementedError(
f"The attempt to rename instance '{resource}' to '{payload['name']}' is not supported."
)
instance = Metadata.from_dict(schemaspace, {**payload})
self.log.debug(
f"MetadataHandler: Updating metadata instance '{resource}' in schemaspace '{schemaspace}'..."
)
metadata = metadata_manager.update(resource, instance)
except (ValidationError, ValueError, NotImplementedError) as err:
raise web.HTTPError(400, str(err)) from err
except MetadataNotFoundError as err:
raise web.HTTPError(404, str(err)) from err
except Exception as err:
raise web.HTTPError(500, repr(err)) from err
self.set_status(200)
self.set_header("Content-Type", "application/json")
self.finish(metadata.to_dict(trim=True))
async def post(self, schemaspace):
schemaspace = url_unescape(schemaspace)
parent = self.settings.get("elyra")
try:
instance = self._validate_body(schemaspace)
self.log.debug(
f"MetadataHandler: Creating metadata instance '{instance.name}' in schemaspace '{schemaspace}'..."
)
metadata_manager = MetadataManager(schemaspace=schemaspace, parent=parent)
metadata = metadata_manager.create(instance.name, instance)
except (ValidationError, ValueError, SyntaxError) as err:
raise web.HTTPError(400, str(err)) from err
except (MetadataNotFoundError, SchemaNotFoundError) as err:
raise web.HTTPError(404, str(err)) from err
except MetadataExistsError as err:
raise web.HTTPError(409, str(err)) from err
except Exception as err:
raise web.HTTPError(500, repr(err)) from err
self.set_status(201)
self.set_header("Content-Type", "application/json")
location = url_path_join(self.base_url, "elyra", "metadata", schemaspace, metadata.name)
self.set_header("Location", location)
self.finish(metadata.to_dict(trim=True))
class SchemaspaceRemove(SchemaspaceBase):
"""Handles the 'remove' subcommand functionality for a specific schemaspace."""
name_option = CliOption(
"--name",
name="name",
description="The name of the metadata instance to remove",
required=True)
# 'Remove' options
options = [name_option]
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.metadata_manager = MetadataManager(schemaspace=self.schemaspace)
def start(self):
super().start() # process options
name = self.name_option.value
try:
self.metadata_manager.get(name)
except MetadataNotFoundError as mnfe:
self.log_and_exit(mnfe)
except ValidationError: # Probably deleting invalid instance
pass
self.metadata_manager.remove(name)
print(
f"Metadata instance '{name}' removed from schemaspace '{self.schemaspace}'."
)
def catalog_instance(component_cache, request):
"""Creates an instance of a component catalog and removes after test."""
instance_metadata = request.param
instance_name = "component_cache"
md_mgr = MetadataManager(schemaspace=ComponentCatalogs.COMPONENT_CATALOGS_SCHEMASPACE_ID)
catalog = md_mgr.create(instance_name, Metadata(**instance_metadata))
component_cache.wait_for_all_cache_tasks()
yield catalog
md_mgr.remove(instance_name)
def _get_runtime_config(runtime_config_name: Optional[str]) -> Optional[RuntimesMetadata]:
"""Fetch runtime configuration for the specified name"""
if not runtime_config_name or runtime_config_name == "local":
# No runtime configuration was specified or it is local.
# Cannot use metadata manager to determine the runtime type.
return None
try:
metadata_manager = MetadataManager(schemaspace=Runtimes.RUNTIMES_SCHEMASPACE_NAME)
return metadata_manager.get(runtime_config_name)
except Exception as e:
raise click.ClickException(f"Invalid runtime configuration: {runtime_config_name}\n {e}")
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.complex_properties: List[str] = []
self.metadata_manager = MetadataManager(schemaspace=self.schemaspace)
# First, process the schema_name option so we can then load the appropriate schema
# file to build the schema-based options. If help is requested, give it to them.
# As an added benefit, if the schemaspace has one schema, got ahead and default that value.
# If multiple, add the list so proper messaging can be applied. As a result, we need to
# to build the option here since this is where we have access to the schemas.
schema_list = list(self.schemas.keys())
if len(schema_list) == 1:
self.schema_name_option = CliOption(
"--schema_name",
name="schema_name",
default_value=schema_list[0],
description="The schema_name of the metadata instance to "
f"install (defaults to '{schema_list[0]}')",
required=True,
)
else:
enum = schema_list
self.schema_name_option = CliOption(
"--schema_name",
name="schema_name",
enum=enum,
description=
"The schema_name of the metadata instance to install. "
f"Must be one of: {enum}",
required=True,
)
self.options.extend([self.schema_name_option, self.name_option])
# Since we need to know if the replace option is in use prior to normal option processing,
# go ahead and check for its existence on the command-line and process if present.
if self.replace_flag.cli_option in self.argv_mappings.keys():
self.process_cli_option(self.replace_flag)
# Determine if --json, --file, or --replace are in use and relax required properties if so.
bulk_metadata = self._process_json_based_options()
relax_required = bulk_metadata or self.replace_flag.value
# This needs to occur following json-based options since they may add it as an option
self.process_cli_option(self.schema_name_option, check_help=True)
# Schema appears to be a valid name, convert its properties to options and continue
schema = self.schemas[self.schema_name_option.value]
# Convert schema properties to options, gathering complex property names
schema_options = self._schema_to_options(schema, relax_required)
self.options.extend(schema_options)
def _get_metadata_configuration(self, schemaspace, name=None):
"""
Retrieve associated metadata configuration based on schemaspace provided and optional instance name
:return: metadata in json format
"""
try:
if not name:
return MetadataManager(schemaspace=schemaspace).get_all()
else:
return MetadataManager(schemaspace=schemaspace).get(name)
except BaseException as err:
self.log.error(f"Error retrieving metadata configuration for {name}", exc_info=True)
raise RuntimeError(f"Error retrieving metadata configuration for {name}", err) from err
def metadata_manager_with_teardown(jp_environ):
"""
This fixture provides a MetadataManager instance for certain tests that modify the component
catalog. This ensures the catalog instance is removed even when the test fails
"""
metadata_manager = MetadataManager(schemaspace=ComponentCatalogs.COMPONENT_CATALOGS_SCHEMASPACE_ID)
# Run test with provided metadata manager
yield metadata_manager
# Remove test catalog
try:
if metadata_manager.get(TEST_CATALOG_NAME):
metadata_manager.remove(TEST_CATALOG_NAME)
except Exception:
pass
def name_arg(self):
# Name can be derived from display_name using normalization method.
if self._name_arg is None:
if self.value is not None and self.display_name_arg is not None:
self._name_arg = MetadataManager.get_normalized_name(
self.display_name_arg)
return self._name_arg
def on_modified(self, event):
"""Fires when the component manifest file is modified."""
self.log.debug(
f"ManifestFileChangeHandler: file '{event.src_path}' has been modified."
)
manifest = self.component_cache._load_manifest(filename=event.src_path)
if manifest: # only update the manifest if there is work to do
for catalog, action in manifest.items():
self.log.debug(
f"ManifestFileChangeHandler: inserting ({catalog},{action}) into update queue..."
)
if action == "delete":
# The metadata instance has already been deleted, so we must
# fabricate an instance that only consists of a catalog name
catalog_instance = ComponentCatalogMetadata(name=catalog)
else: # cache_action == 'modify':
# Fetch the catalog instance associated with this action
catalog_instance = MetadataManager(
schemaspace=ComponentCatalogs.
COMPONENT_CATALOGS_SCHEMASPACE_ID).get(name=catalog)
self.component_cache.update(catalog=catalog_instance,
action=action)
self.component_cache.update_manifest(
filename=event.src_path) # clear the manifest
def on_load(self, **kwargs: Any) -> None:
super().on_load(**kwargs)
if self.metadata.get("auth_type") is None:
# Inject auth_type property for metadata persisted using Elyra < 3.3:
# - api_username and api_password present -> use DEX Legacy
# - otherwise -> use no authentication type
if (len(self.metadata.get("api_username", "").strip()) == 0 or len(
self.metadata.get("api_password", "").strip()) == 0):
self.metadata[
"auth_type"] = SupportedAuthProviders.NO_AUTHENTICATION.name
else:
self.metadata[
"auth_type"] = SupportedAuthProviders.DEX_LEGACY.name
if self.metadata.get("cos_auth_type") is None:
# Inject cos_auth_type property for metadata persisted using Elyra < 3.4:
# - cos_username and cos_password must be present
# - cos_secret may be present (above statement also applies in this case)
if self.metadata.get("cos_username") and self.metadata.get(
"cos_password"):
if len(self.metadata.get("cos_secret", "").strip()) == 0:
self.metadata["cos_auth_type"] = "USER_CREDENTIALS"
else:
self.metadata["cos_auth_type"] = "KUBERNETES_SECRET"
# save changes
MetadataManager(schemaspace="runtimes").update(self.name,
self,
for_migration=True)
return None
async def get(self, schemaspace):
schemaspace = url_unescape(schemaspace)
parent = self.settings.get("elyra")
try:
metadata_manager = MetadataManager(schemaspace=schemaspace, parent=parent)
metadata = metadata_manager.get_all()
except (ValidationError, ValueError) as err:
raise web.HTTPError(400, str(err)) from err
except MetadataNotFoundError as err:
raise web.HTTPError(404, str(err)) from err
except Exception as err:
raise web.HTTPError(500, repr(err)) from err
metadata_model = {schemaspace: [r.to_dict(trim=True) for r in metadata]}
self.set_header("Content-Type", "application/json")
self.finish(metadata_model)
def on_load(self, **kwargs: Any) -> None:
super().on_load(**kwargs)
update_required = False
if self.metadata.get("git_type") is None:
# Inject git_type property for metadata persisted using Elyra < 3.5:
self.metadata["git_type"] = SupportedGitTypes.GITHUB.name
update_required = True
if self.metadata.get("cos_auth_type") is None:
# Inject cos_auth_type property for metadata persisted using Elyra < 3.4:
# - cos_username and cos_password must be present
# - cos_secret may be present (above statement also applies in this case)
if self.metadata.get("cos_username") and self.metadata.get(
"cos_password"):
if len(self.metadata.get("cos_secret", "")) == 0:
self.metadata["cos_auth_type"] = "USER_CREDENTIALS"
else:
self.metadata["cos_auth_type"] = "KUBERNETES_SECRET"
update_required = True
if update_required:
# save changes
MetadataManager(schemaspace="runtimes").update(self.name,
self,
for_migration=True)
return None
def _determine_runtime(runtime_config: str) -> str:
"""Derives the runtime (processor) from the runtime_config."""
# If not present or 'local', treat as special case.
if not runtime_config or runtime_config.upper() == RuntimeProcessorType.LOCAL.name:
return RuntimeProcessorType.LOCAL.name.lower()
runtime_metadata = MetadataManager(schemaspace=Runtimes.RUNTIMES_SCHEMASPACE_ID).get(runtime_config)
return runtime_metadata.schema_name
async def delete(self, schemaspace, resource):
schemaspace = url_unescape(schemaspace)
resource = url_unescape(resource)
parent = self.settings.get("elyra")
try:
self.log.debug(
f"MetadataHandler: Deleting metadata instance '{resource}' in schemaspace '{schemaspace}'..."
)
metadata_manager = MetadataManager(schemaspace=schemaspace, parent=parent)
metadata_manager.remove(resource)
except (ValidationError, ValueError) as err:
raise web.HTTPError(400, str(err)) from err
except PermissionError as err:
raise web.HTTPError(403, str(err)) from err
except MetadataNotFoundError as err:
raise web.HTTPError(404, str(err)) from err
except Exception as err:
raise web.HTTPError(500, repr(err)) from err
self.set_status(204)
self.finish()
def tests_manager(jp_environ, schemaspace_location, request):
metadata_mgr = MetadataManager(schemaspace=METADATA_TEST_SCHEMASPACE,
metadata_store_class=request.param)
store_mgr = metadata_mgr.metadata_store
create_instance(store_mgr, schemaspace_location, "valid",
valid_metadata_json)
create_instance(store_mgr, schemaspace_location, "another",
another_metadata_json)
create_instance(store_mgr, schemaspace_location, "invalid",
invalid_metadata_json)
create_instance(store_mgr, schemaspace_location, "bad", invalid_json)
create_instance(store_mgr, schemaspace_location, "invalid_schema_name",
invalid_schema_name_json)
return metadata_mgr
def test_manager_add_invalid(tests_manager):
with pytest.raises(ValueError):
MetadataManager(schemaspace="invalid")
# Attempt with non Metadata instance
with pytest.raises(TypeError):
tests_manager.create(valid_metadata_json)
# and invalid parameters
with pytest.raises(TypeError):
tests_manager.create(None, invalid_no_display_name_json)
with pytest.raises(ValueError):
tests_manager.create("foo", None)
def refresh(self):
"""Triggers a refresh of all catalogs in the component cache.
Raises RefreshInProgressError if a complete refresh is in progress.
Note that we do not preclude non-server processes from performing a
complete refresh. In such cases, each of the catalog entries will be
written to the manifest, which will be placed into the update queue.
As a result, non-server applications could by-pass the "refresh in progress"
constraint, but we're assuming a CLI application won't be as likely to
"pound" refresh like a UI application can.
"""
if self.is_server_process and self.cache_manager.is_refreshing():
raise RefreshInProgressError()
catalogs = MetadataManager(schemaspace=ComponentCatalogs.COMPONENT_CATALOGS_SCHEMASPACE_ID).get_all()
for catalog in catalogs:
self._insert_request(self.refresh_queue, catalog, "modify")
def _determine_runtime_type(runtime_config: str) -> str:
"""Derives the runtime type (platform) from the runtime_config."""
# Pull the runtime_type (platform) from the runtime_config
# Need to special case 'local' runtime_config instances
if runtime_config.lower() == "local":
runtime_type = RuntimeProcessorType.LOCAL
else:
runtime_metadata = MetadataManager(schemaspace=Runtimes.RUNTIMES_SCHEMASPACE_ID).get(runtime_config)
runtime_type_name = runtime_metadata.metadata.get("runtime_type")
try:
runtime_type = RuntimeProcessorType.get_instance_by_name(runtime_type_name)
except (KeyError, TypeError):
raise ValueError(
f"Unsupported pipeline runtime: '{runtime_type_name}' " f"found in config '{runtime_config}'!"
)
return runtime_type.name
def airflow_runtime_instance():
"""Creates an airflow RTC and removes it after test."""
instance_name = "valid_airflow_test_config"
instance_config_file = Path(__file__).parent / "resources" / "runtime_configs" / f"{instance_name}.json"
with open(instance_config_file, "r") as fd:
instance_config = json.load(fd)
md_mgr = MetadataManager(schemaspace=Runtimes.RUNTIMES_SCHEMASPACE_ID)
# clean possible orphaned instance...
try:
md_mgr.remove(instance_name)
except Exception:
pass
runtime_instance = md_mgr.create(instance_name, Metadata(**instance_config))
yield runtime_instance.name
md_mgr.remove(runtime_instance.name)
def on_load(self, **kwargs: Any) -> None:
"""Perform any necessary adjustments, migrations when instance is loaded."""
# If there's no runtime_type property in the instance metadata, infer from schema_name
if "runtime_type" not in self.metadata:
if self.schema_name == "kfp":
self.metadata["runtime_type"] = "KUBEFLOW_PIPELINES"
elif self.schema_name == "airflow":
self.metadata["runtime_type"] = "APACHE_AIRFLOW"
elif self.schema_name == "argo":
self.metadata["runtime_type"] = "ARGO"
else:
raise ValueError(
f"Unknown Runtimes schema name detected: '{self.schema_name}'! Skipping..."
)
getLogger("ServerApp").info(
f"Upgrading runtime {self.schema_name} instance '{self.name}' "
f"to include runtime_type '{self.metadata['runtime_type']}'..."
)
MetadataManager(schemaspace="runtimes").update(self.name,
self,
for_migration=True)
async def put(self, catalog):
# Validate the body
cache_refresh = self.get_json_body()
if "action" not in cache_refresh or cache_refresh[
"action"] != "refresh":
raise web.HTTPError(
400, reason="A body of {'action': 'refresh'} is required.")
try:
# Ensure given catalog name is a metadata instance
catalog_instance = MetadataManager(
schemaspace=ComponentCatalogs.COMPONENT_CATALOGS_SCHEMASPACE_ID
).get(name=catalog)
except MetadataNotFoundError:
raise web.HTTPError(404, f"Catalog '{catalog}' cannot be found.")
self.log.debug(
f"Refreshing component cache for catalog with name '{catalog}'...")
ComponentCache.instance().update(catalog=catalog_instance,
action="modify")
self.set_status(204)
await self.finish()
def tests_hierarchy_manager(
setup_hierarchy): # Only uses FileMetadataStore for storage right now.
return MetadataManager(schemaspace=METADATA_TEST_SCHEMASPACE)
class SchemaspaceExport(SchemaspaceBase):
"""Handles the 'export' subcommand functionality for a specific schemaspace."""
schema_name_option = CliOption(
"--schema_name",
name="schema_name",
description="The schema name of the metadata instances to export",
required=False,
)
include_invalid_flag = Flag(
"--include-invalid",
name="include-invalid",
description="Export valid and invalid instances. "
"By default only valid instances are exported.",
default_value=False,
)
clean_flag = Flag("--clean",
name="clean",
description="Clear out contents of the export directory",
default_value=False)
directory_option = CliOption(
"--directory",
name="directory",
description=
"The local file system path where the exported metadata will be stored",
required=True,
)
# 'Export' flags
options: List[Option] = [
schema_name_option, include_invalid_flag, clean_flag, directory_option
]
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.metadata_manager = MetadataManager(schemaspace=self.schemaspace)
def start(self):
super().start() # process options
schema_name = self.schema_name_option.value
if schema_name:
schema_list = sorted(list(self.schemas.keys()))
if schema_name not in schema_list:
print(
f"Schema name '{schema_name}' is invalid. For the '{self.schemaspace}' schemaspace, "
f"the schema name must be one of {schema_list}")
self.exit(1)
include_invalid = self.include_invalid_flag.value
directory = self.directory_option.value
clean = self.clean_flag.value
try:
if self.schema_name_option is not None:
metadata_instances = self.metadata_manager.get_all(
include_invalid=include_invalid, of_schema=schema_name)
else:
metadata_instances = self.metadata_manager.get_all(
include_invalid=include_invalid)
except MetadataNotFoundError:
metadata_instances = None
if not metadata_instances:
print(
f"No metadata instances found for schemaspace '{self.schemaspace}'"
+ (f" and schema '{schema_name}'" if schema_name else ""))
print(f"Nothing exported to '{directory}'")
return
dest_directory = os.path.join(directory, self.schemaspace)
if not os.path.exists(dest_directory):
try:
print(f"Creating directory structure for '{dest_directory}'")
os.makedirs(dest_directory)
except OSError as e:
print(
f"Error creating directory structure for '{dest_directory}': {e.strerror}: '{e.filename}'"
)
self.exit(1)
else:
if clean:
files = [
os.path.join(dest_directory, f)
for f in os.listdir(dest_directory)
]
if len(files) > 0:
print(f"Cleaning out all files in '{dest_directory}'")
[os.remove(f) for f in files if os.path.isfile(f)]
print(
f"Exporting metadata instances for schemaspace '{self.schemaspace}'"
+ (f" and schema '{schema_name}'" if schema_name else "") +
(" (includes invalid)" if include_invalid else " (valid only)") +
f" to '{dest_directory}'")
num_valid_exported = 0
num_invalid_exported = 0
for instance in metadata_instances:
dict_metadata = instance.to_dict()
output_file = os.path.join(dest_directory,
f'{dict_metadata["name"]}.json')
if "reason" in dict_metadata and len(dict_metadata["reason"]) > 0:
num_invalid_exported += 1
else:
num_valid_exported += 1
with open(output_file, mode="w") as output_file:
json.dump(dict_metadata, output_file, indent=4)
total_exported = num_valid_exported + num_invalid_exported
print(f"Exported {total_exported} " +
("instances" if total_exported > 1 else "instance") +
f" ({num_invalid_exported} of which " +
("is" if num_invalid_exported == 1 else "are") + " invalid)")
class SchemaspaceInstall(SchemaspaceBase):
"""DEPRECATED (removed in v4.0):
Handles the 'install' subcommand functionality for a specific schemaspace.
"""
# Known options, others will be derived from schema based on schema_name...
replace_flag = Flag("--replace",
name="replace",
description="Replace an existing instance",
default_value=False)
name_option = CliOption(
"--name",
name="name",
description="The name of the metadata instance to install")
file_option = FileOption(
"--file",
name="file",
description="The filename containing the metadata instance to install. "
"Can be used to bypass individual property arguments.",
)
json_option = JSONOption(
"--json",
name="json",
description=
"The JSON string containing the metadata instance to install. "
"Can be used to bypass individual property arguments.",
)
# 'Install' options
options: List[Option] = [replace_flag, file_option, json_option
] # defer name option until after schema
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.complex_properties: List[str] = []
self.metadata_manager = MetadataManager(schemaspace=self.schemaspace)
# First, process the schema_name option so we can then load the appropriate schema
# file to build the schema-based options. If help is requested, give it to them.
# As an added benefit, if the schemaspace has one schema, got ahead and default that value.
# If multiple, add the list so proper messaging can be applied. As a result, we need to
# to build the option here since this is where we have access to the schemas.
schema_list = list(self.schemas.keys())
if len(schema_list) == 1:
self.schema_name_option = CliOption(
"--schema_name",
name="schema_name",
default_value=schema_list[0],
description="The schema_name of the metadata instance to "
f"install (defaults to '{schema_list[0]}')",
required=True,
)
else:
enum = schema_list
self.schema_name_option = CliOption(
"--schema_name",
name="schema_name",
enum=enum,
description=
"The schema_name of the metadata instance to install. "
f"Must be one of: {enum}",
required=True,
)
self.options.extend([self.schema_name_option, self.name_option])
# Since we need to know if the replace option is in use prior to normal option processing,
# go ahead and check for its existence on the command-line and process if present.
if self.replace_flag.cli_option in self.argv_mappings.keys():
self.process_cli_option(self.replace_flag)
# Determine if --json, --file, or --replace are in use and relax required properties if so.
bulk_metadata = self._process_json_based_options()
relax_required = bulk_metadata or self.replace_flag.value
# This needs to occur following json-based options since they may add it as an option
self.process_cli_option(self.schema_name_option, check_help=True)
# Schema appears to be a valid name, convert its properties to options and continue
schema = self.schemas[self.schema_name_option.value]
# Convert schema properties to options, gathering complex property names
schema_options = self._schema_to_options(schema, relax_required)
self.options.extend(schema_options)
def start(self):
super().start() # process options
# Get known options, then gather display_name and build metadata dict.
name = self.name_option.value
schema_name = self.schema_name_option.value
display_name = None
metadata = {}
# Walk the options looking for SchemaProperty instances. Any MetadataSchemaProperty instances go
# into the metadata dict. Note that we process JSONBasedOptions (--json or --file) prior to
# MetadataSchemaProperty types since the former will set the base metadata stanza and individual
# values can be used to override the former's content (like BYO authentication OVPs, for example).
for option in self.options:
if isinstance(option, MetadataSchemaProperty):
# skip adding any non required properties that have no value (unless its a null type).
if not option.required and not option.value and option.type != "null":
continue
metadata[option.name] = option.value
elif isinstance(option, SchemaProperty):
if option.name == "display_name": # Be sure we have a display_name
display_name = option.value
continue
elif isinstance(option, JSONBasedOption):
metadata.update(option.metadata)
if display_name is None and self.replace_flag.value is False: # Only require on create
self.log_and_exit(
f"Could not determine display_name from schema '{schema_name}'"
)
ex_msg = None
new_instance = None
try:
if self.replace_flag.value: # if replacing, fetch the instance so it can be updated
updated_instance = self.metadata_manager.get(name)
updated_instance.schema_name = schema_name
if display_name:
updated_instance.display_name = display_name
updated_instance.metadata.update(metadata)
new_instance = self.metadata_manager.update(
name, updated_instance)
else: # create a new instance
instance = Metadata(schema_name=schema_name,
name=name,
display_name=display_name,
metadata=metadata)
new_instance = self.metadata_manager.create(name, instance)
except Exception as ex:
ex_msg = str(ex)
if new_instance:
print(
f"Metadata instance '{new_instance.name}' for schema '{schema_name}' has been written "
f"to: {new_instance.resource}")
else:
if ex_msg:
self.log_and_exit(
f"The following exception occurred saving metadata instance "
f"for schema '{schema_name}': {ex_msg}",
display_help=False,
)
else:
self.log_and_exit(
f"A failure occurred saving metadata instance '{name}' for "
f"schema '{schema_name}'.",
display_help=False,
)
def _process_json_based_options(self) -> bool:
"""Process the file and json options to see if they have values (and those values can be loaded as JSON)
Then check payloads for schema_name, display_name and derive name options and add to argv mappings
if currently not specified.
If either option is set, indicate that the metadata stanza should be skipped (return True)
"""
bulk_metadata = False
self.process_cli_option(self.file_option, check_help=True)
self.process_cli_option(self.json_option, check_help=True)
# if both are set, raise error
if self.json_option.value is not None and self.file_option.value is not None:
self.log_and_exit(
"At most one of '--json' or '--file' can be set at a time.",
display_help=True)
elif self.json_option.value is not None:
bulk_metadata = True
self.json_option.transfer_names_to_argvs(self.argv,
self.argv_mappings)
elif self.file_option.value is not None:
bulk_metadata = True
self.file_option.transfer_names_to_argvs(self.argv,
self.argv_mappings)
# else, neither is set so metadata stanza will be considered
return bulk_metadata
def _schema_to_options(self,
schema: Dict,
relax_required: bool = False) -> List[Option]:
"""Takes a JSON schema and builds a list of SchemaProperty instances corresponding to each
property in the schema. There are two sections of properties, one that includes
schema_name and display_name and another within the metadata container - which
will be separated by class type - SchemaProperty vs. MetadataSchemaProperty.
If relax_required is true, a --json or --file option is in use and the primary metadata
comes from those options OR the --replace option is in use, in which case the primary
metadata comes from the existing instance (being replaced). In such cases, skip setting
required values since most will come from the JSON-based option or already be present
(in the case of replace). This allows CLI-specified metadata properties to override the
primary metadata (either in the JSON options or from the existing instance).
"""
options = {}
properties = schema["properties"]
for name, value in properties.items():
if name == "schema_name": # already have this option, skip
continue
if name != "metadata":
options[name] = SchemaProperty(name, value)
else: # convert first-level metadata properties to options...
metadata_properties = properties["metadata"]["properties"]
for md_name, md_value in metadata_properties.items():
msp = MetadataSchemaProperty(md_name, md_value)
# skip if this property was not specified on the command line and its a replace/bulk op
if msp.cli_option not in self.argv_mappings and relax_required:
continue
if msp.unsupported_meta_props: # if this option includes complex meta-props, note that.
self.complex_properties.append(md_name)
options[md_name] = msp
# Now set required-ness on MetadataProperties, but only when creation is using fine-grained property options
if not relax_required:
required_props = properties["metadata"].get("required")
for required in required_props:
options.get(required).required = True
# ... and top-level (schema) Properties if we're not replacing (updating)
if self.replace_flag.value is False:
required_props = set(schema.get("required")) - {
"schema_name", "metadata"
} # skip schema_name & metadata
for required in required_props:
options.get(required).required = True
return list(options.values())
def print_help(self):
super().print_help()
# If we gathered any complex properties, go ahead and note how behaviors might be affected, etc.
if self.complex_properties:
print(
f"Note: The following properties in this schema contain JSON keywords that are not supported "
f"by the tooling: {self.complex_properties}.")
print(
"This can impact the tool's ability to derive context from the schema, including a property's "
"type, description, or behaviors included in complex types like 'oneOf'."
)
print(
"It is recommended that options corresponding to these properties be set after understanding "
"the schema or indirectly using `--file` or `--json` options.")
print(
'If the property is of type "object" it can be set using a file containing only that property\'s '
"JSON.")
print(
f"The following are considered unsupported keywords: {SchemaProperty.unsupported_keywords}"
)
class SchemaspaceList(SchemaspaceBase):
"""Handles the 'list' subcommand functionality for a specific schemaspace."""
json_flag = Flag("--json",
name="json",
description="List complete instances as JSON",
default_value=False)
valid_only_flag = Flag(
"--valid-only",
name="valid-only",
description=
"Only list valid instances (default includes invalid instances)",
default_value=False,
)
# 'List' flags
options = [json_flag, valid_only_flag]
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.metadata_manager = MetadataManager(schemaspace=self.schemaspace)
def start(self):
super().start() # process options
include_invalid = not self.valid_only_flag.value
try:
metadata_instances = self.metadata_manager.get_all(
include_invalid=include_invalid)
except MetadataNotFoundError:
metadata_instances = None
if self.json_flag.value:
if metadata_instances is None:
metadata_instances = []
print(metadata_instances)
else:
if not metadata_instances:
print(f"No metadata instances found for {self.schemaspace}")
return
validity_clause = "includes invalid" if include_invalid else "valid only"
print(
f"Available metadata instances for {self.schemaspace} ({validity_clause}):"
)
sorted_instances = sorted(metadata_instances,
key=lambda inst:
(inst.schema_name, inst.name))
# pad to width of longest instance
max_schema_name_len = len("Schema")
max_name_len = len("Instance")
max_resource_len = len("Resource")
for instance in sorted_instances:
max_schema_name_len = max(len(instance.schema_name),
max_schema_name_len)
max_name_len = max(len(instance.name), max_name_len)
max_resource_len = max(len(instance.resource),
max_resource_len)
print()
print("%s %s %s " % (
"Schema".ljust(max_schema_name_len),
"Instance".ljust(max_name_len),
"Resource".ljust(max_resource_len),
))
print("%s %s %s " % (
"------".ljust(max_schema_name_len),
"--------".ljust(max_name_len),
"--------".ljust(max_resource_len),
))
for instance in sorted_instances:
invalid = ""
if instance.reason and len(instance.reason) > 0:
invalid = f"**INVALID** ({instance.reason})"
print("%s %s %s %s" % (
instance.schema_name.ljust(max_schema_name_len),
instance.name.ljust(max_name_len),
instance.resource.ljust(max_resource_len),
invalid,
))
class SchemaspaceImport(SchemaspaceBase):
"""Handles the 'import' subcommand functionality for a specific schemaspace."""
directory_option = CliOption(
"--directory",
name="directory",
description=
"The local file system path from where the metadata will be imported",
required=True,
)
overwrite_flag = Flag(
"--overwrite",
name="overwrite",
description="Overwrite existing metadata instance with the same name",
default_value=False,
)
# 'Import' flags
options: List[Option] = [directory_option, overwrite_flag]
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.metadata_manager = MetadataManager(schemaspace=self.schemaspace)
def start(self):
super().start() # process options
src_directory = self.directory_option.value
try:
json_files = [
f for f in os.listdir(src_directory) if f.endswith(".json")
]
except OSError as e:
print(
f"Unable to reach the '{src_directory}' directory: {e.strerror}: '{e.filename}'"
)
self.exit(1)
if len(json_files) == 0:
print(
f"No instances for import found in the '{src_directory}' directory"
)
return
metadata_file = None
non_imported_files = []
for file in json_files:
filepath = os.path.join(src_directory, file)
try:
with open(filepath) as f:
metadata_file = json.loads(f.read())
except OSError as e:
non_imported_files.append([file, e.strerror])
continue
name = os.path.splitext(file)[0]
try:
schema_name = metadata_file["schema_name"]
display_name = metadata_file["display_name"]
metadata = metadata_file["metadata"]
except KeyError as e:
non_imported_files.append([
file,
f"Could not find '{e.args[0]}' key in the import file '{filepath}'"
])
continue
try:
if self.overwrite_flag.value: # if overwrite flag is true
try: # try updating the existing instance
updated_instance = self.metadata_manager.get(name)
updated_instance.schema_name = schema_name
if display_name:
updated_instance.display_name = display_name
if name:
updated_instance.name = name
updated_instance.metadata.update(metadata)
self.metadata_manager.update(name, updated_instance)
except MetadataNotFoundError: # no existing instance - create new
instance = Metadata(schema_name=schema_name,
name=name,
display_name=display_name,
metadata=metadata)
self.metadata_manager.create(name, instance)
else:
instance = Metadata(schema_name=schema_name,
name=name,
display_name=display_name,
metadata=metadata)
self.metadata_manager.create(name, instance)
except Exception as e:
if isinstance(e, MetadataExistsError):
non_imported_files.append(
[file, f"{str(e)} Use '--overwrite' to update."])
else:
non_imported_files.append([file, str(e)])
instance_count_not_imported = len(non_imported_files)
instance_count_imported = len(json_files) - instance_count_not_imported
print(f"Imported {instance_count_imported} " +
("instance" if instance_count_imported == 1 else "instances"))
if instance_count_not_imported > 0:
print(f"{instance_count_not_imported} " +
("instance" if instance_count_not_imported ==
1 else "instances") + " could not be imported")
non_imported_files.sort(key=lambda x: x[0])
print("\nThe following files could not be imported: ")
# pad to width of longest file and reason
max_file_name_len = len("File")
max_reason_len = len("Reason")
for file in non_imported_files:
max_file_name_len = max(len(file[0]), max_file_name_len)
max_reason_len = max(len(file[1]), max_reason_len)
print(
f"{'File'.ljust(max_file_name_len)} {'Reason'.ljust(max_reason_len)}"
)
print(
f"{'----'.ljust(max_file_name_len)} {'------'.ljust(max_reason_len)}"
)
for file in non_imported_files:
print(
f"{file[0].ljust(max_file_name_len)} {file[1].ljust(max_reason_len)}"
)
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.metadata_manager = MetadataManager(schemaspace=self.schemaspace)