def initialize_settings(self):
self.log.info(f"Config {self.config}")
# Instantiate singletons with appropriate parent to enable configurability, and convey
# root_dir to PipelineProcessorManager.
PipelineProcessorRegistry.instance(root_dir=self.settings["server_root_dir"], parent=self)
PipelineProcessorManager.instance(root_dir=self.settings["server_root_dir"], parent=self)
PipelineValidationManager.instance(root_dir=self.settings["server_root_dir"], parent=self)
FileMetadataCache.instance(parent=self)
ComponentCache.instance(parent=self).load()
SchemaManager.instance(parent=self)
def validate_log_output(caplog: pytest.LogCaptureFixture, expected_entry: str) -> Schemaspace:
"""Common negative test pattern that validates expected log output."""
SchemaManager.clear_instance()
schema_mgr = SchemaManager.instance()
assert expected_entry in caplog.text
byo_ss = schema_mgr.get_schemaspace(BYOSchemaspace.BYO_SCHEMASPACE_ID)
# Ensure there are two valid schemas and make sure our bad one is not in the list.
assert len(byo_ss.schemas) == 2
for name, schema in byo_ss.schemas.items():
assert schema["name"] in ["byo-test-0", "byo-test-1"]
return byo_ss # in case caller wants to check other things.
def from_dict(cls: Type[M], schemaspace: str, metadata_dict: dict) -> M:
"""Creates an appropriate instance of Metadata from a dictionary instance"""
# Get the schema and look for metadata_class entry and use that, else Metadata.
metadata_class_name = "elyra.metadata.metadata.Metadata"
schema_name = metadata_dict.get("schema_name")
if schema_name:
schema = SchemaManager.instance().get_schema(
schemaspace, schema_name)
metadata_class_name = schema.get("metadata_class_name",
metadata_class_name)
metadata_class = import_item(metadata_class_name)
try:
instance = metadata_class(**metadata_dict)
if not isinstance(instance, Metadata):
raise ValueError(
f"The metadata_class_name ('{metadata_class_name}') for "
f"schema '{schema_name}' must be a subclass of '{cls.__name__}'!"
)
except TypeError as te:
raise ValueError(
f"The metadata_class_name ('{metadata_class_name}') for "
f"schema '{schema_name}' must be a subclass of '{cls.__name__}'!"
) from te
return instance
def test_schemaspace_case_sensitive_name_id(byo_schemaspaces, caplog):
"""Ensures that a schemaspace with case sensitive name and id has its instance properties unaltered."""
SchemaManager.clear_instance()
schema_mgr = SchemaManager.instance()
byo_ss_name = "byo-schemaspace_CaseSensitiveName"
byo_ss_id = "1b1e461a-c7fa-40f2-a3a3-bf1f2fd48EEA"
schema_mgr_ss_names = schema_mgr.get_schemaspace_names()
# Check schemaspace name is normalized in schema manager reference list
assert byo_ss_name.lower() in schema_mgr_ss_names
byo_ss_instance_name = schema_mgr.get_schemaspace_name(byo_ss_name)
assert byo_ss_instance_name == byo_ss_name
byo_ss_instance_name = schema_mgr.get_schemaspace_name(byo_ss_id)
assert byo_ss_instance_name == byo_ss_name
# Confirm this schemaspace produces an "empty schemaspace warning
assert "The following schemaspaces have no schemas: ['byo-schemaspace_CaseSensitiveName']" in caplog.text
def test_validate_factory_schemas():
# Test that each of our factory schemas meet the minimum requirements.
# This is accomplished by merely accessing the schemas of the schemaspace
# and ensuring their presence.
schema_mgr = SchemaManager.instance() # validation actually happens here
schemaspace_names = schema_mgr.get_schemaspace_names()
for schemaspace_name in schemaspace_names:
schemaspace = schema_mgr.get_schemaspace(schemaspace_name)
for name, schema in schemaspace.schemas.items():
print(f"Schema '{schemaspace_name}/{name}' is valid.")
def _is_compatible_pipeline(runtime_name: str, runtime_type: str):
"""Returns true if the pipeline's runtime name is compatible to its type."""
if runtime_type.lower() == "generic":
return True # TODO: this won't always be true as some runtime impls won't support generics
# We need to make the "local" runtimes a real runtime someday! Until then, we have this...
if runtime_name.lower() == "local":
runtime_type_from_schema = runtime_name.upper() # use the up-cased value since runtime_types are up-cased
else: # fetch the metadata instance corresponding to runtime_name and compare its runtime_type
runtime_schema = SchemaManager.instance().get_schema(Runtimes.RUNTIMES_SCHEMASPACE_ID, runtime_name)
runtime_type_from_schema = runtime_schema.get("runtime_type")
return runtime_type_from_schema == runtime_type
def test_schemaspace_display_name():
"""Ensures that display_name properly defaults from name (or not when provided itself)."""
schema_mgr = SchemaManager.instance()
# Only metadata-tests have matching name and display_name values
schemaspace = schema_mgr.get_schemaspace("metadata-tests")
assert schemaspace.name == "metadata-tests"
assert schemaspace.display_name == schemaspace.name
# All others have a separate name, we'll check runtime-images
schemaspace = schema_mgr.get_schemaspace("runtime-images")
assert schemaspace.name == "runtime-images"
assert schemaspace.display_name == "Runtime Images"
def test_schema_no_side_effect():
"""Ensures that schemas returned from get_schema_schemas can be altered and not side-effect the next access."""
schema_mgr = SchemaManager.instance()
schemas = schema_mgr.get_schemaspace_schemas(METADATA_TEST_SCHEMASPACE_ID)
for name, schema in schemas.items():
if name == "metadata-test":
orig_schema = copy.deepcopy(schema_mgr.get_schema(METADATA_TEST_SCHEMASPACE_ID, name)) # capture good copy
schema["metadata_class_name"] = "bad_class"
assert schema != orig_schema
fresh_schema = schema_mgr.get_schema(METADATA_TEST_SCHEMASPACE_ID, name)
assert fresh_schema == orig_schema
def _get_runtime_display_name(schema_name: Optional[str]) -> Optional[str]:
"""Return the display name for the specified runtime schema_name"""
if not schema_name or schema_name == "local":
# No schame name was specified or it is local.
# Cannot use metadata manager to determine the display name.
return schema_name
try:
schema_manager = SchemaManager.instance()
schema = schema_manager.get_schema(Runtimes.RUNTIMES_SCHEMASPACE_NAME, schema_name)
return schema["display_name"]
except Exception as e:
raise click.ClickException(f"Invalid runtime configuration: {schema_name}\n {e}")
async def get(self):
try:
schema_manager = SchemaManager.instance()
schemaspaces = schema_manager.get_schemaspace_names()
except (ValidationError, ValueError) as err:
raise web.HTTPError(404, str(err)) from err
except Exception as err:
raise web.HTTPError(500, repr(err)) from err
schemaspace_model = {"schemaspaces": schemaspaces}
self.set_header("Content-Type", "application/json")
self.finish(schemaspace_model)
async def get(self, schemaspace, resource):
schemaspace = url_unescape(schemaspace)
resource = url_unescape(resource)
try:
schema_manager = SchemaManager.instance()
schema = schema_manager.get_schema(schemaspace, resource)
except (ValidationError, ValueError, SchemaNotFoundError) as err:
raise web.HTTPError(404, str(err)) from err
except Exception as err:
raise web.HTTPError(500, repr(err)) from err
self.set_header("Content-Type", "application/json")
self.finish(schema)
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.schemaspace_schemas = {}
schema_mgr = SchemaManager.instance()
# Migration should include deprecated schemaspaces
include_deprecated = False
args = kwargs.get("argv", [])
if len(args) > 0:
# identify commands that can operate on deprecated schemaspaces
include_deprecated = args[0] not in ["install", "create", "update"]
schemaspace_names = schema_mgr.get_schemaspace_names(
include_deprecated=include_deprecated)
for name in schemaspace_names:
self.schemaspace_schemas[
name] = schema_mgr.get_schemaspace_schemas(name)
def start(self):
super().start() # process options
# Regardless of schemaspace, call migrate. If the schemaspace implementation doesn't
# require migration, an appropriate log statement will be produced.
schemaspace = SchemaManager.instance().get_schemaspace(
self.schemaspace)
migrated = schemaspace.migrate()
if migrated:
print(
f"The following {self.schemaspace} instances were migrated: {migrated}"
)
else:
print(
f"No instances of schemaspace {self.schemaspace} were migrated."
)
async def get(self, schemaspace):
try:
schema_manager = SchemaManager.instance()
schemaspace = schema_manager.get_schemaspace(schemaspace)
except (ValidationError, ValueError) as err:
raise web.HTTPError(404, str(err)) from err
except Exception as err:
raise web.HTTPError(500, repr(err)) from err
schemaspace_info_model = {
"name": schemaspace.name,
"id": schemaspace.id,
"display_name": schemaspace.display_name,
"description": schemaspace.description,
}
self.set_header("Content-Type", "application/json")
self.finish(schemaspace_info_model)
def byo_schemaspaces(monkeypatch):
"""Setup the BYO Schemaspaces and SchemasProviders, returning the SchemaManager instance."""
monkeypatch.setattr(SchemaManager, "_get_schemaspaces", mock_get_schemaspaces)
monkeypatch.setattr(SchemaManager, "_get_schemas_providers", mock_get_schemas_providers)
yield # We must clear the SchemaManager instance else follow-on tests will be side-effected
SchemaManager.clear_instance()
def schema_manager():
schema_manager = SchemaManager.instance()
yield schema_manager
SchemaManager.clear_instance()