def test_load_live_schemas(self):
MLSchema.populate_registry()
all_schema_paths = list(Path('mlspeclib').glob('schemas/**/*.yaml'))
for schema_path in all_schema_paths:
this_text = schema_path.read_text()
this_dict = convert_yaml_to_dict(this_text)
if this_dict['mlspec_schema_version']['meta'] == '0.0.1':
continue
print(schema_path)
loaded_schema = MLSchema.create_schema(this_text)
self.assertIsNotNone(loaded_schema.schema_name)
def test_try_create_missing_mlspec_version_and_type(self):
schema_string = """
mlspec_schema_version:
# Identifies the version of this schema
meta: 0.0.1
mlspec_schema_type:
# Identifies the type of this schema
meta: base
"""
no_version = convert_yaml_to_dict(schema_string)
no_version.pop("mlspec_schema_version")
with self.assertRaises(KeyError):
MLSchema.create_schema(no_version)
no_schema = convert_yaml_to_dict(schema_string)
no_schema.pop("mlspec_schema_type")
with self.assertRaises(KeyError):
MLSchema.create_schema(no_schema)
def populate_registry():
for schema_file in list(
Path(os.path.dirname(__file__)).glob('schemas/**/*.yaml')):
schema_text = schema_file.read_text()
loaded_schema = convert_yaml_to_dict(schema_text)
loaded_schema_name = MLSchema.build_schema_name_for_schema(
mlspec_schema_type=loaded_schema['mlspec_schema_type'],
mlspec_schema_version=loaded_schema['mlspec_schema_version'])
try:
marshmallow.class_registry.get_class(loaded_schema_name)
except RegistryError:
MLSchema.create_schema(loaded_schema)
def create_schema_type(raw_string: dict, schema_name: str = None):
""" Creates a new schema from a string of yaml. inheriting from MLSchema.\
Schema still needs to be instantiated before use.
e.g. this_schema = MLSchema.create_schema(raw_string)
schema = this_schema()
this_object = schema.load(object_submission_dict)
"""
schema_as_dict = convert_yaml_to_dict(raw_string)
schema_as_dict = MLSchema._augment_with_base_schema(schema_as_dict)
if schema_name is None:
# schema_name = foo_foo("a", "b")
schema_name = build_schema_name_for_schema(
mlspec_schema_version=schema_as_dict["mlspec_schema_version"],
mlspec_schema_type=schema_as_dict["mlspec_schema_type"],
)
fields_dict = {}
for field in schema_as_dict:
if "marshmallow.fields" in str(
type(schema_as_dict[field]
)) or "mlschemafields.MLSchemaFields" in str(
type(schema_as_dict[field])):
# In this case, the field has already been created an instantiated properly (because
# it comes from a base schema registered in marshmallow.class_registry). We can skip
# all of the below and just add it to the field dict. This includes nested fields.
fields_dict[field] = schema_as_dict[field]
elif schema_as_dict[field] is None:
raise AttributeError(
f"""It appears at the field '{field}' the yaml/dict \
is not formatted with attributes. Could it be an indentation error?"""
)
elif ("type" in schema_as_dict[field]
and schema_as_dict[field]["type"].lower() == "nested"):
nested_schema_type = MLSchema.create_schema_type(
schema_as_dict[field]["schema"],
schema_name + "_" + field.lower())
nested_schema_type.name = field
fields_dict[field] = fields.Nested(nested_schema_type)
else:
field_method = MLSchema._field_method_from_dict(
field, schema_as_dict[field])
if field_method:
fields_dict[field] = field_method
abstract_schema = MLSchema.from_dict(fields_dict)
if schema_name:
marshmallow.class_registry.register(schema_name, abstract_schema)
abstract_schema.schema_name = schema_name
return abstract_schema
def test_convert_dicts_to_sub_schema(self):
class UserSchema(Schema):
name = fields.String(required=True)
email = fields.Email(required=True)
class BlogSchema(Schema):
title = fields.String(required=True)
year = fields.Int(required=True)
author = fields.Nested(UserSchema, required=True)
marshmallow.class_registry.register("blog_author", \
UserSchema)
marshmallow.class_registry.register("blog", \
BlogSchema)
sub_schema_string = """
title: "Something Completely Different"
year: 1970
author:
name: "Monty"
email: "*****@*****.**"
"""
full_schema_data = convert_yaml_to_dict(sub_schema_string)
full_schema_loaded = BlogSchema().load(full_schema_data)
self.assertTrue(full_schema_loaded['title'] == full_schema_data['title'])
self.assertTrue(full_schema_loaded['author']['name'] == full_schema_data['author']['name'])
missing_author_name_data = convert_yaml_to_dict(sub_schema_string)
missing_author_name_data['author'].pop('name', None)
with self.assertRaises(ValidationError):
BlogSchema().load(missing_author_name_data)
missing_year_data = convert_yaml_to_dict(sub_schema_string)
missing_year_data.pop('year', None)
with self.assertRaises(ValidationError):
BlogSchema().load(missing_year_data)
def test_load_live_data(self):
MLSchema.populate_registry()
all_data_files = list(Path("tests").glob("data/**/*.yaml"))
for data_file in all_data_files:
this_text = data_file.read_text(encoding="utf-8")
this_dict = convert_yaml_to_dict(this_text)
if this_dict["schema_version"] == "0.0.1":
continue
# print(data_file)
loaded_object, errors = MLObject.create_object_from_file(data_file)
self.assertTrue(len(errors) == 0)
self.assertIsNotNone(loaded_object.get_schema())
def test_load_live_data(self):
MLSchema.populate_registry()
all_data_files = list(Path('tests').glob('data/**/*.yaml'))
for data_file in all_data_files:
this_text = data_file.read_text()
this_dict = convert_yaml_to_dict(this_text)
if this_dict['schema_version'] == '0.0.1':
continue
print(data_file)
loaded_object, errors = MLObject.create_object_from_file(data_file)
self.assertTrue(len(errors) == 0)
self.assertIsNotNone(loaded_object.get_schema())
def test_interfaces_mismatch_type(self):
instantiated_schema = MLSchema.create_schema(
self.wrap_schema_with_mlschema_info(
SampleSchema.TEST.INTERFACE)) # noqa
yaml_submission = convert_yaml_to_dict(
self.wrap_submission_with_mlschema_info(
SampleSubmissions.UNIT_TESTS.INTERFACE_INVALID_MISMATCH_TYPE)
) # noqa
with self.assertRaises(ValidationError) as context:
instantiated_schema.load(yaml_submission)
self.assertTrue(
'valid default' in context.exception.messages['inputs'][0][0])
def test_interfaces_type_unknown(self):
instantiated_schema = MLSchema.create_schema(
self.wrap_schema_with_mlschema_info(
SampleSchema.TEST.INTERFACE)) # noqa
yaml_submission = convert_yaml_to_dict(
self.wrap_submission_with_mlschema_info(
SampleSubmissions.UNIT_TESTS.INTERFACE_INVALID_TYPE_UNKNOWN_1)
) # noqa
with self.assertRaises(ValidationError) as context:
instantiated_schema.load(yaml_submission)
self.assertTrue(
'string or a dict' in context.exception.messages['inputs'][0][0])
def test_load_full_datapath_schema(self):
MLSchema.create_schema(SampleSchema.SCHEMAS.BASE)
instantiated_schema = MLSchema.create_schema(
SampleSchema.SCHEMAS.DATAPATH)
submission_dict = convert_yaml_to_dict(
SampleSubmissions.FULL_SUBMISSIONS.DATAPATH)
instantiated_object = instantiated_schema.load(submission_dict)
assert instantiated_object['run_date'].isoformat() == \
submission_dict['run_date'].isoformat()
assert instantiated_object['connection']['endpoint'] == \
submission_dict['connection']['endpoint']
submission_dict.pop('run_date', None)
with self.assertRaises(ValidationError):
instantiated_schema.load(submission_dict)
def test_merge_two_dicts_with_valid_base(self):
base_schema = MLSchema.create_schema(SampleSchema.SCHEMAS.BASE)
base_object = base_schema.load(SampleSubmissions.FULL_SUBMISSIONS.BASE)
datapath_schema = MLSchema.create_schema(SampleSchema.SCHEMAS.DATAPATH)
datapath_object = datapath_schema.load(
SampleSubmissions.FULL_SUBMISSIONS.DATAPATH
)
# Should not work - BASE did not merge with DATAPATH
with self.assertRaises(KeyError):
base_object[
"data_store"
] == "NULL_STRING_SHOULD_NOT_WORK" # pylint: disable=pointless-statement
base_object_dict = convert_yaml_to_dict(SampleSubmissions.FULL_SUBMISSIONS.BASE)
datapath_object_dict = convert_yaml_to_dict(
SampleSubmissions.FULL_SUBMISSIONS.DATAPATH
)
self.assertTrue(isinstance(base_object["run_id"], UUID))
self.assertTrue(base_object["run_id"] == UUID(base_object_dict["run_id"]))
# Should work - DATAPATH inherited from BASE
self.assertTrue(isinstance(datapath_object["data_store"], str))
self.assertTrue(
datapath_object["data_store"] == datapath_object_dict["data_store"]
)
self.assertTrue(isinstance(datapath_object["connection"]["access_key_id"], str))
self.assertTrue(
datapath_object["connection"]["access_key_id"]
== datapath_object_dict["connection"]["access_key_id"]
)
self.assertTrue(isinstance(datapath_object["run_id"], UUID))
self.assertTrue(
datapath_object["run_id"] == UUID(datapath_object_dict["run_id"])
)
def populate_registry():
""" Loads all the base schemas for the schema registry. """
rootLogger = logging.getLogger()
schemas_to_process = []
no_base = []
has_base = []
last = []
load_root = os.path.dirname(mlspeclib.__file__)
rootLogger.debug(f"Registry load root: {load_root}")
load_path = Path(load_root).glob("schemas/**/*.yaml")
rootLogger.debug(f"Registry load path: {load_path}")
load_list = list(load_path)
rootLogger.debug(f"Registry load list: {load_list}")
for schema_file in load_list:
schema_text = schema_file.read_text('utf-8')
schema_dict = convert_yaml_to_dict(schema_text)
if "last" in schema_dict["mlspec_schema_type"]:
last.append(schema_dict)
elif "mlspec_base_type" not in schema_dict:
no_base.append(schema_dict)
else:
has_base.append(schema_dict)
schemas_to_process = no_base + has_base + last
for loaded_schema in schemas_to_process:
loaded_schema_name = build_schema_name_for_schema(
mlspec_schema_type=loaded_schema["mlspec_schema_type"],
mlspec_schema_version=loaded_schema["mlspec_schema_version"],
)
try:
marshmallow.class_registry.get_class(loaded_schema_name)
except RegistryError:
MLSchema.create_schema(loaded_schema)
def create_object_from_string(file_contents: str):
""" Creates an MLObject based on a string. String must be valid yaml.
Returns Tuple MLObject and list of errors."""
contents_as_dict = convert_yaml_to_dict(file_contents)
# if (self.schema_type() is not None and self.version() is not None):
# schema_string = self.schema_type().name.lower()
# if (self.version() != contents_as_dict['schema_version'] or \
# schema_string != contents_as_dict['schema_type']):
# raise AttributeError("""The schema version and schema type were not in sync
# with those provided in the data. Rather than guessing which you want to use, we
# are throwing this error:
# Version Expected: %s
# Version Provided: %s
# Schema Type Expected: %s
# Schema Type Provided: %s """ % (self.version(), contents_as_dict['schema_version'], \
# schema_string, contents_as_dict['schema_type']))
ml_object = MLObject()
ml_object.set_type(schema_version=contents_as_dict["schema_version"],
schema_type=contents_as_dict['schema_type'])
MLObject.update_tree(ml_object, contents_as_dict)
errors = ml_object.validate()
return ml_object, errors
def return_base_schema_and_submission():
instantiated_schema = MLSchema.create_schema(SampleSchema.SCHEMAS.BASE)
yaml_submission = convert_yaml_to_dict(SampleSubmissions.FULL_SUBMISSIONS.BASE)
return instantiated_schema, yaml_submission
class ValidatorsTestSuite(unittest.TestCase):
"""Validators test cases."""
schema_schema_info = convert_yaml_to_dict("""
mlspec_schema_version:
# Identifies the version of this schema
meta: 0.0.1
mlspec_schema_type:
# Base schema type that this extends
meta: base
schema_version:
# Identifies version of MLSpec to use
type: semver
required: True
schema_type:
# Identifies version of MLSpec to use
type: allowed_schema_types
required: True
""")
submission_schema_info = {"schema_version": "0.0.1", "schema_type": "base"}
def test_semver_found(self):
assert MLSchemaValidators.validate_type_semver("0.0.1")
def test_semver_not_found(self):
assert not MLSchemaValidators.validate_type_semver("x.x.x")
def test_uuid_found(self):
instantiated_object = self.generic_schema_validator(
SampleSchema.TEST.UUID, SampleSubmissions.UNIT_TESTS.UUID_VALID)
self.assertTrue(instantiated_object["run_id"])
def test_uuid_not_found(self):
self.generic_schema_validator(
SampleSchema.TEST.UUID,
SampleSubmissions.UNIT_TESTS.UUID_INVALID,
ValidationError,
"UUID",
)
def test_uri_valid(self):
instantiated_object = self.generic_schema_validator(
SampleSchema.TEST.URI, SampleSubmissions.UNIT_TESTS.URI_VALID_1)
self.assertTrue(instantiated_object["endpoint"])
instantiated_object = self.generic_schema_validator(
SampleSchema.TEST.URI, SampleSubmissions.UNIT_TESTS.URI_VALID_2)
self.assertTrue(instantiated_object["endpoint"])
def test_uri_invalid(self):
self.generic_schema_validator(
SampleSchema.TEST.URI,
SampleSubmissions.UNIT_TESTS.URI_INVALID_1,
ValidationError,
"Invalid",
)
self.generic_schema_validator(
SampleSchema.TEST.URI,
SampleSubmissions.UNIT_TESTS.URI_INVALID_2,
ValidationError,
"valid string",
)
def test_regex_valid(self):
instantiated_object = self.generic_schema_validator(
SampleSchema.TEST.REGEX,
SampleSubmissions.UNIT_TESTS.REGEX_ALL_LETTERS)
self.assertTrue(instantiated_object["all_letters"])
self.generic_schema_validator(
SampleSchema.TEST.REGEX,
SampleSubmissions.UNIT_TESTS.REGEX_ALL_NUMBERS,
ValidationError,
"No match",
)
def test_regex_invalid(self):
self.generic_schema_validator(SampleSchema.TEST.INVALID_REGEX, None,
AssertionError, "valid regex")
# @unittest.skip("NYI")
# def test_path(self):
# self.assertTrue(False)
# @unittest.skip("NYI")
# def test_bucket(self):
# self.assertTrue(False)
def test_interfaces_valid(self):
instantiated_object = self.generic_schema_validator(
SampleSchema.TEST.INTERFACE,
SampleSubmissions.UNIT_TESTS.INTERFACE_VALID_UNNAMED,
)
self.assertTrue(len(instantiated_object["inputs"]) == 2)
instantiated_object = self.generic_schema_validator(
SampleSchema.TEST.INTERFACE,
SampleSubmissions.UNIT_TESTS.INTERFACE_VALID_NAMED,
)
self.assertTrue(len(instantiated_object["inputs"]) == 2)
# @unittest.skip("Type is not required in KFP (but it should be)")
# def test_interfaces_missing_type(self):
# self.generic_schema_validator(
# SampleSchema.TEST.INTERFACE,
# SampleSubmissions.UNIT_TESTS.INTERFACE_INVALID_MISSING_TYPE,
# ValidationError,
# "No type",
# )
def test_interfaces_mismatch_type(self):
self.generic_schema_validator(
SampleSchema.TEST.INTERFACE,
SampleSubmissions.UNIT_TESTS.INTERFACE_INVALID_MISMATCH_TYPE,
ValidationError,
"valid default",
)
def test_interfaces_type_unknown(self):
self.generic_schema_validator(
SampleSchema.TEST.INTERFACE,
SampleSubmissions.UNIT_TESTS.INTERFACE_INVALID_TYPE_UNKNOWN_1,
ValidationError,
"string or a dict",
)
def test_validate_constraints_constraint_valid_greater_equal(self):
this_schema = MLSchema.create_schema(
self.wrap_schema_with_mlschema_info(
SampleSchema.TEST.OPERATOR_VALID))
self.assertTrue(
isinstance(this_schema.declared_fields["num"], fields.Integer))
def test_validate_constraints_constraint_valid_modulo(self):
this_schema = MLSchema.create_schema(
self.wrap_schema_with_mlschema_info(
SampleSchema.TEST.OPERATOR_VALID_MODULO_2))
self.assertTrue(
isinstance(this_schema.declared_fields["num"], fields.Integer))
def test_validate_constraints_true_value(self):
instantiated_object = self.generic_schema_validator(
SampleSchema.TEST.OPERATOR_VALID,
SampleSubmissions.UNIT_TESTS.CONSTRAINT_VALID_MORE_THAN_1000,
None,
)
self.assertTrue(isinstance(instantiated_object["num"], int))
def test_validate_constraints_false_value(self):
self.generic_schema_validator(
SampleSchema.TEST.OPERATOR_VALID,
SampleSubmissions.UNIT_TESTS.CONSTRAINT_VALID_LESS_THAN_1000,
ValidationError,
"Invalid value",
)
def test_validate_constraints_modulo_true(self):
instantiated_object = self.generic_schema_validator(
SampleSchema.TEST.OPERATOR_VALID_MODULO_2,
SampleSubmissions.UNIT_TESTS.CONSTRAINT_VALID_MODULO_2_TRUE,
)
self.assertTrue(isinstance(instantiated_object["num"], int))
def test_validate_constraints_modulo_false(self):
self.generic_schema_validator(
SampleSchema.TEST.OPERATOR_VALID_MODULO_2,
SampleSubmissions.UNIT_TESTS.CONSTRAINT_VALID_MODULO_2_FALSE,
ValidationError,
"Invalid value",
)
def test_validate_constraints_constraint_invalid_type(self):
self.generic_schema_validator(
SampleSchema.TEST.OPERATOR_INVALID_TYPE,
None,
ValueError,
"Attempting to add",
)
def test_validate_constraints_constraint_no_valid_operator(self):
self.generic_schema_validator(
SampleSchema.TEST.OPERATOR_INVALID_NO_OPERATOR,
None,
ValidationError,
"No parsable lambda",
)
def test_validate_workflow_valid(self):
MLSchema.populate_registry()
this_schema = self.generic_schema_validator(
SampleSchema.TEST.WORKFLOW_STEP,
SampleSubmissions.UNIT_TESTS.WORKFLOW_VALID,
None,
None,
)
self.assertTrue(len(this_schema["steps"]["step_name"]) == 5)
def test_validate_workflow_missing_input(self):
MLSchema.populate_registry()
self.generic_schema_validator(
SampleSchema.TEST.WORKFLOW_STEP,
SampleSubmissions.UNIT_TESTS.WORKFLOW_BAD_INPUT,
ValidationError,
"class registry",
)
def test_validate_workflow_bad_semver(self):
MLSchema.populate_registry()
self.generic_schema_validator(
SampleSchema.TEST.WORKFLOW_STEP,
SampleSubmissions.UNIT_TESTS.WORKFLOW_BAD_SEMVER,
ValidationError,
"semver",
)
def test_validate_workflow_no_input(self):
MLSchema.populate_registry()
self.generic_schema_validator(
SampleSchema.TEST.WORKFLOW_STEP,
SampleSubmissions.UNIT_TESTS.WORKFLOW_NO_INPUT,
ValidationError,
"input",
)
def generic_schema_validator(self,
test_schema,
test_submission,
exception_type=None,
exception_string=None) -> MLObject:
error_string = None
try:
instantiated_schema = MLSchema.create_schema(
self.wrap_schema_with_mlschema_info(test_schema)) # noqa
except Exception as e:
self.assertTrue(isinstance(e, exception_type))
error_string = str(e)
if test_submission is not None:
yaml_submission = convert_yaml_to_dict(
self.wrap_submission_with_mlschema_info(
test_submission)) # noqa
if exception_type is not None:
with self.assertRaises(exception_type) as context:
instantiated_schema.load(yaml_submission)
if context is not None:
error_string = str(context.exception)
# if error string is not none, we threw an error, return
if error_string is not None:
if exception_string is not None:
self.assertTrue(exception_string in error_string)
else:
print(error_string) # Unexpected error, print it out
return # Raised an exception during loading dict, return
return instantiated_schema.load(yaml_submission)
def wrap_schema_with_mlschema_info(self, this_dict):
return merge_two_dicts(self.schema_schema_info,
convert_yaml_to_dict(this_dict))
def wrap_submission_with_mlschema_info(self, this_dict):
return merge_two_dicts(self.submission_schema_info,
convert_yaml_to_dict(this_dict))
def wrap_submission_with_mlschema_info(self, this_dict):
return merge_two_dicts(self.submission_schema_info,
convert_yaml_to_dict(this_dict))
def create_object(submission_text: str):
""" Creates an object that can be read and written to. """
submission_dict = convert_yaml_to_dict(submission_text)
schema = MLSchema.load_schema_from_registry(data=submission_dict)
return schema().load(submission_dict)
def append_schema_to_registry(load_path: Path) -> bool:
if isinstance(load_path, str):
load_path = Path(load_path)
if not isinstance(load_path, Path):
raise TypeError(
"Appending schemas to a registry expects a Path object.")
all_found_files = list(load_path.glob("**/*.yaml"))
if len(all_found_files) == 0:
raise FileNotFoundError(
f"No files ending in '.yaml' were found in the path '{load_path}'"
)
all_schemas = []
files_with_errors = []
no_base_schemas = []
schemas_with_base = []
last_schemas = []
for putative_schema_file in all_found_files:
this_text = putative_schema_file.read_text('utf-8')
try:
this_dict = convert_yaml_to_dict(this_text)
except ScannerError as se:
files_with_errors.append((
putative_schema_file.name,
f"Yaml could not be parsed. Error details: \n{str(se)}",
))
continue
if not contains_minimum_fields_for_schema(this_dict):
files_with_errors.append((
putative_schema_file.name,
"""Does not contain all of the minimum schema necessary as top level fields - list includes: mlspec_schema_version, mlspec_schema_version.meta, mlspec_base_type, mlspec_base_type.meta, mlspec_schema_type, mlspec_schema_type.meta, schema_version and schema_type""",
))
continue
if "last" in this_dict["mlspec_schema_type"]:
last_schemas.append(this_dict)
elif "mlspec_base_type" in this_dict:
schemas_with_base.append(this_dict)
else:
no_base_schemas.append(this_dict)
all_schemas = no_base_schemas + schemas_with_base + last_schemas
if len(files_with_errors) > 0:
rootLogger = logging.getLogger()
error_string = ""
for err in files_with_errors:
error_string += f"::CRITICAL - {err[0]}: {err[1]}\n"
# TODO: Move to root logger
rootLogger.critical(error_string)
return False
for schema_dict in all_schemas:
schema_name = build_schema_name_for_schema(
mlspec_schema_type=schema_dict["mlspec_schema_type"],
mlspec_schema_version=schema_dict["mlspec_schema_version"],
)
try:
marshmallow.class_registry.get_class(schema_name)
except RegistryError:
MLSchema.create_schema(schema_dict)
class ValidatorsTestSuite(unittest.TestCase):
"""Validators test cases."""
schema_schema_info = convert_yaml_to_dict("""
mlspec_schema_version:
# Identifies the version of this schema
meta: 0.0.1
mlspec_schema_type:
# Base schema type that this extends
meta: base
schema_version:
# Identifies version of MLSpec to use
type: semver
required: True
schema_type:
# Identifies version of MLSpec to use
type: allowed_schema_types
required: True
""")
submission_schema_info = {'schema_version': '0.0.1', 'schema_type': 'base'}
def test_semver_found(self):
assert MLSchemaValidators.validate_type_semver('0.0.1')
def test_semver_not_found(self):
assert not MLSchemaValidators.validate_type_semver('x.x.x')
def test_uuid_found(self):
instantiated_schema = MLSchema.create_schema(
self.wrap_schema_with_mlschema_info(
SampleSchema.TEST.UUID)) # noqa
yaml_submission = convert_yaml_to_dict(
self.wrap_submission_with_mlschema_info(
SampleSubmissions.UNIT_TESTS.UUID_VALID)) # noqa
instantiated_object = instantiated_schema.load(yaml_submission)
self.assertTrue(instantiated_object['run_id'])
def test_uuid_not_found(self):
instantiated_schema = MLSchema.create_schema(
self.wrap_schema_with_mlschema_info(
SampleSchema.TEST.UUID)) # noqa
yaml_submission = convert_yaml_to_dict(
self.wrap_submission_with_mlschema_info(
SampleSubmissions.UNIT_TESTS.UUID_INVALID)) # noqa
with self.assertRaises(ValidationError):
instantiated_schema.load(yaml_submission)
def test_uri_valid(self):
instantiated_schema = MLSchema.create_schema(
self.wrap_schema_with_mlschema_info(SampleSchema.TEST.URI)) # noqa
yaml_submission = convert_yaml_to_dict(
self.wrap_submission_with_mlschema_info(
SampleSubmissions.UNIT_TESTS.URI_VALID_1)) # noqa
instantiated_object = instantiated_schema.load(yaml_submission)
self.assertTrue(instantiated_object['endpoint'])
yaml_submission = convert_yaml_to_dict(
self.wrap_submission_with_mlschema_info(
SampleSubmissions.UNIT_TESTS.URI_VALID_2)) # noqa
instantiated_object = instantiated_schema.load(yaml_submission)
self.assertTrue(instantiated_object['endpoint'])
def test_uri_invalid(self):
instantiated_schema = MLSchema.create_schema(
self.wrap_schema_with_mlschema_info(SampleSchema.TEST.URI)) # noqa
yaml_submission = convert_yaml_to_dict(
self.wrap_submission_with_mlschema_info(
SampleSubmissions.UNIT_TESTS.URI_INVALID_1)) # noqa
with self.assertRaises(ValidationError):
instantiated_schema.load(yaml_submission)
yaml_submission = convert_yaml_to_dict(
self.wrap_submission_with_mlschema_info(
SampleSubmissions.UNIT_TESTS.URI_INVALID_2)) # noqa
with self.assertRaises(ValidationError):
instantiated_schema.load(yaml_submission)
def test_regex_valid(self):
instantiated_schema = MLSchema.create_schema(
self.wrap_schema_with_mlschema_info(
SampleSchema.TEST.REGEX)) # noqa
yaml_submission = convert_yaml_to_dict(
self.wrap_submission_with_mlschema_info(
SampleSubmissions.UNIT_TESTS.REGEX_ALL_LETTERS)) # noqa
instantiated_object = instantiated_schema.load(yaml_submission)
self.assertTrue(instantiated_object['all_letters'])
yaml_submission = convert_yaml_to_dict(
self.wrap_submission_with_mlschema_info(
SampleSubmissions.UNIT_TESTS.REGEX_ALL_NUMBERS)) # noqa
with self.assertRaises(ValidationError):
instantiated_schema.load(yaml_submission)
def test_regex_invalid(self):
with self.assertRaises(AssertionError):
MLSchema.create_schema(
self.wrap_schema_with_mlschema_info(
SampleSchema.TEST.INVALID_REGEX)) # noqa
@unittest.skip("NYI")
def test_path(self):
self.assertTrue(False)
@unittest.skip("NYI")
def test_bucket(self):
self.assertTrue(False)
def test_interfaces_valid(self):
instantiated_schema = MLSchema.create_schema(
self.wrap_schema_with_mlschema_info(
SampleSchema.TEST.INTERFACE)) # noqa
yaml_submission = convert_yaml_to_dict(
self.wrap_submission_with_mlschema_info(
SampleSubmissions.UNIT_TESTS.INTERFACE_VALID_UNNAMED)) # noqa
instantiated_object = instantiated_schema.load(yaml_submission)
self.assertTrue(len(instantiated_object['inputs']) == 2)
yaml_submission = convert_yaml_to_dict(
self.wrap_submission_with_mlschema_info(
SampleSubmissions.UNIT_TESTS.INTERFACE_VALID_NAMED)) # noqa
instantiated_object = instantiated_schema.load(yaml_submission)
self.assertTrue(len(instantiated_object['inputs']) == 2)
@unittest.skip("Type is not required in KFP (but it should be)")
def test_interfaces_missing_type(self):
instantiated_schema = MLSchema.create_schema(
self.wrap_schema_with_mlschema_info(
SampleSchema.TEST.INTERFACE)) # noqa
yaml_submission = convert_yaml_to_dict(
self.wrap_submission_with_mlschema_info(
SampleSubmissions.UNIT_TESTS.INTERFACE_INVALID_MISSING_TYPE)
) # noqa
with self.assertRaises(ValidationError) as context:
instantiated_schema.load(yaml_submission)
self.assertTrue(
'No type' in context.exception.messages['inputs'][0][0])
def test_interfaces_mismatch_type(self):
instantiated_schema = MLSchema.create_schema(
self.wrap_schema_with_mlschema_info(
SampleSchema.TEST.INTERFACE)) # noqa
yaml_submission = convert_yaml_to_dict(
self.wrap_submission_with_mlschema_info(
SampleSubmissions.UNIT_TESTS.INTERFACE_INVALID_MISMATCH_TYPE)
) # noqa
with self.assertRaises(ValidationError) as context:
instantiated_schema.load(yaml_submission)
self.assertTrue(
'valid default' in context.exception.messages['inputs'][0][0])
def test_interfaces_type_unknown(self):
instantiated_schema = MLSchema.create_schema(
self.wrap_schema_with_mlschema_info(
SampleSchema.TEST.INTERFACE)) # noqa
yaml_submission = convert_yaml_to_dict(
self.wrap_submission_with_mlschema_info(
SampleSubmissions.UNIT_TESTS.INTERFACE_INVALID_TYPE_UNKNOWN_1)
) # noqa
with self.assertRaises(ValidationError) as context:
instantiated_schema.load(yaml_submission)
self.assertTrue(
'string or a dict' in context.exception.messages['inputs'][0][0])
def wrap_schema_with_mlschema_info(self, this_dict):
return merge_two_dicts(self.schema_schema_info,
convert_yaml_to_dict(this_dict))
def wrap_submission_with_mlschema_info(self, this_dict):
return merge_two_dicts(self.submission_schema_info,
convert_yaml_to_dict(this_dict))
class ValidatorsTestSuite(unittest.TestCase):
"""Validators test cases."""
schema_schema_info = convert_yaml_to_dict("""
mlspec_schema_version:
# Identifies the version of this schema
meta: 0.0.1
mlspec_schema_type:
# Base schema type that this extends
meta: base
schema_version:
# Identifies version of MLSpec to use
type: semver
required: True
schema_type:
# Identifies version of MLSpec to use
type: allowed_schema_types
required: True
""")
submission_schema_info = {'schema_version': '0.0.1', 'schema_type': 'base'}
def test_semver_found(self):
assert MLSchemaValidators.validate_type_semver('0.0.1')
def test_semver_not_found(self):
assert not MLSchemaValidators.validate_type_semver('x.x.x')
def test_uuid_found(self):
instantiated_schema = MLSchema.create_schema(
self.wrap_schema_with_mlschema_info(
SampleSchema.TEST.UUID)) # noqa
yaml_submission = convert_yaml_to_dict(
self.wrap_submission_with_mlschema_info(
SampleSubmissions.UNIT_TESTS.UUID_VALID)) # noqa
instantiated_object = instantiated_schema.load(yaml_submission)
self.assertTrue(instantiated_object['run_id'])
def test_uuid_not_found(self):
instantiated_schema = MLSchema.create_schema(
self.wrap_schema_with_mlschema_info(
SampleSchema.TEST.UUID)) # noqa
yaml_submission = convert_yaml_to_dict(
self.wrap_submission_with_mlschema_info(
SampleSubmissions.UNIT_TESTS.UUID_INVALID)) # noqa
with self.assertRaises(ValidationError):
instantiated_schema.load(yaml_submission)
def test_uri_valid(self):
instantiated_schema = MLSchema.create_schema(
self.wrap_schema_with_mlschema_info(SampleSchema.TEST.URI)) # noqa
yaml_submission = convert_yaml_to_dict(
self.wrap_submission_with_mlschema_info(
SampleSubmissions.UNIT_TESTS.URI_VALID_1)) # noqa
instantiated_object = instantiated_schema.load(yaml_submission)
self.assertTrue(instantiated_object['endpoint'])
yaml_submission = convert_yaml_to_dict(
self.wrap_submission_with_mlschema_info(
SampleSubmissions.UNIT_TESTS.URI_VALID_2)) # noqa
instantiated_object = instantiated_schema.load(yaml_submission)
self.assertTrue(instantiated_object['endpoint'])
def test_uri_invalid(self):
instantiated_schema = MLSchema.create_schema(
self.wrap_schema_with_mlschema_info(SampleSchema.TEST.URI)) # noqa
yaml_submission = convert_yaml_to_dict(
self.wrap_submission_with_mlschema_info(
SampleSubmissions.UNIT_TESTS.URI_INVALID_1)) # noqa
with self.assertRaises(ValidationError):
instantiated_schema.load(yaml_submission)
yaml_submission = convert_yaml_to_dict(
self.wrap_submission_with_mlschema_info(
SampleSubmissions.UNIT_TESTS.URI_INVALID_2)) # noqa
with self.assertRaises(ValidationError):
instantiated_schema.load(yaml_submission)
def test_regex_valid(self):
instantiated_schema = MLSchema.create_schema(
self.wrap_schema_with_mlschema_info(
SampleSchema.TEST.REGEX)) # noqa
yaml_submission = convert_yaml_to_dict(
self.wrap_submission_with_mlschema_info(
SampleSubmissions.UNIT_TESTS.REGEX_ALL_LETTERS)) # noqa
instantiated_object = instantiated_schema.load(yaml_submission)
self.assertTrue(instantiated_object['all_letters'])
yaml_submission = convert_yaml_to_dict(
self.wrap_submission_with_mlschema_info(
SampleSubmissions.UNIT_TESTS.REGEX_ALL_NUMBERS)) # noqa
with self.assertRaises(ValidationError):
instantiated_schema.load(yaml_submission)
def test_regex_invalid(self):
with self.assertRaises(AssertionError):
MLSchema.create_schema(
self.wrap_schema_with_mlschema_info(
SampleSchema.TEST.INVALID_REGEX)) # noqa
@unittest.skip("NYI")
def test_path(self):
self.assertTrue(False)
@unittest.skip("NYI")
def test_bucket(self):
self.assertTrue(False)
def wrap_schema_with_mlschema_info(self, this_dict):
return merge_two_dicts(self.schema_schema_info,
convert_yaml_to_dict(this_dict))
def wrap_submission_with_mlschema_info(self, this_dict):
return merge_two_dicts(self.submission_schema_info,
convert_yaml_to_dict(this_dict))
