def test_merge_two_dicts_with_invalid_base(self):
# Should not work - trying to instantiate a schema with a base_type
# but the base_type has not been registered
try:
marshmallow.class_registry._registry.pop("0_0_1_base")
except KeyError:
# This is acceptable because we want to make sure the registry is empty.
pass
with self.assertRaises(RegistryError):
MLSchema.create_schema(SampleSchema.SCHEMAS.DATAPATH)
def test_load_full_datapath_schema(self):
MLSchema.create_schema(SampleSchema.SCHEMAS.BASE)
instantiated_schema = MLSchema.create_schema(
SampleSchema.SCHEMAS.DATAPATH)
submission_dict = convert_yaml_to_dict(
SampleSubmissions.FULL_SUBMISSIONS.DATAPATH)
instantiated_object = instantiated_schema.load(submission_dict)
assert instantiated_object['run_date'].isoformat() == \
submission_dict['run_date'].isoformat()
assert instantiated_object['connection']['endpoint'] == \
submission_dict['connection']['endpoint']
submission_dict.pop('run_date', None)
with self.assertRaises(ValidationError):
instantiated_schema.load(submission_dict)
def generic_schema_validator(self,
test_schema,
test_submission,
exception_type=None,
exception_string=None) -> MLObject:
error_string = None
try:
instantiated_schema = MLSchema.create_schema(
self.wrap_schema_with_mlschema_info(test_schema)) # noqa
except Exception as e:
self.assertTrue(isinstance(e, exception_type))
error_string = str(e)
if test_submission is not None:
yaml_submission = convert_yaml_to_dict(
self.wrap_submission_with_mlschema_info(
test_submission)) # noqa
if exception_type is not None:
with self.assertRaises(exception_type) as context:
instantiated_schema.load(yaml_submission)
if context is not None:
error_string = str(context.exception)
# if error string is not none, we threw an error, return
if error_string is not None:
if exception_string is not None:
self.assertTrue(exception_string in error_string)
else:
print(error_string) # Unexpected error, print it out
return # Raised an exception during loading dict, return
return instantiated_schema.load(yaml_submission)
def test_validate_constraints_constraint_valid_modulo(self):
this_schema = MLSchema.create_schema(
self.wrap_schema_with_mlschema_info(
SampleSchema.TEST.OPERATOR_VALID_MODULO_2))
self.assertTrue(
isinstance(this_schema.declared_fields["num"], fields.Integer))
def test_uuid_not_found(self):
instantiated_schema = MLSchema.create_schema(
self.wrap_schema_with_mlschema_info(
SampleSchema.TEST.UUID)) # noqa
yaml_submission = convert_yaml_to_dict(
self.wrap_submission_with_mlschema_info(
SampleSubmissions.UNIT_TESTS.UUID_INVALID)) # noqa
with self.assertRaises(ValidationError):
instantiated_schema.load(yaml_submission)
def test_create_nested_schema(self):
connection_text = """
mlspec_schema_version:
# Identifies the version of this schema
meta: 0.0.1
mlspec_schema_type:
# Identifies the type of this schema
meta: datapath
# Connection to datapath
schema_version:
type: semver
required: True
schema_type:
type: string
required: True
connection:
type: nested
schema:
# URI for the location of the data store
endpoint:
type: URI
required: True
one_more_field:
type: String
required: True"""
nested_schema = MLSchema.create_schema(connection_text, "0_0_1_datapath")
connection_submission = """
schema_version: 0.0.1
schema_type: datapath
connection:
endpoint: S3://mybucket/puppy.jpg
one_more_field: foobaz
"""
connection_submission_dict = convert_yaml_to_dict(connection_submission)
nested_object = nested_schema.load(connection_submission)
self.assertTrue(
nested_object["connection"]["endpoint"]
== connection_submission_dict["connection"]["endpoint"]
)
self.assertTrue(
nested_object["one_more_field"]
== connection_submission_dict["one_more_field"]
)
nested_missing_endpoint_dict = convert_yaml_to_dict(connection_submission)
nested_missing_endpoint_dict["connection"].pop("endpoint", None)
with self.assertRaises(ValidationError):
nested_schema.load(nested_missing_endpoint_dict)
missing_extra_dict = convert_yaml_to_dict(connection_submission)
missing_extra_dict.pop("one_more_field", None)
with self.assertRaises(ValidationError):
nested_schema.load(missing_extra_dict)
def test_incorrectly_indented_yaml(self):
bad_yaml_string = """
mlspec_schema_version:
# Identifies the version of this schema
meta: 0.0.1
mlspec_schema_type:
# Identifies the type of this schema
meta: datapath
connection:
type: nested
schema:
# URI for the location of the data store
endpoint:
type: URI
required: True"""
with self.assertRaises(AttributeError):
MLSchema.create_schema(bad_yaml_string)
def test_uuid_found(self):
instantiated_schema = MLSchema.create_schema(
self.wrap_schema_with_mlschema_info(
SampleSchema.TEST.UUID)) # noqa
yaml_submission = convert_yaml_to_dict(
self.wrap_submission_with_mlschema_info(
SampleSubmissions.UNIT_TESTS.UUID_VALID)) # noqa
instantiated_object = instantiated_schema.load(yaml_submission)
self.assertTrue(instantiated_object['run_id'])
def test_all_schemas(self):
MLSchema.populate_registry()
all_001_schemas = list(Path('mlspeclib').glob('schemas/0/0/1/*.yaml'))
self.assertTrue(len(all_001_schemas) > 1)
for schema in all_001_schemas:
this_text = schema.read_text()
loaded_schema = MLSchema.create_schema(this_text)
self.assertIsNotNone(loaded_schema.schema_name)
def test_load_full_base_schema(self):
instantiated_schema = MLSchema.create_schema(SampleSchema.SCHEMAS.BASE)
submission_dict = convert_yaml_to_dict(
SampleSubmissions.FULL_SUBMISSIONS.BASE)
instantiated_object = instantiated_schema.load(submission_dict)
assert (instantiated_object["run_date"].isoformat() ==
submission_dict["run_date"].isoformat())
submission_dict.pop("run_date", None)
with self.assertRaises(ValidationError):
instantiated_schema.load(submission_dict)
def test_load_live_schemas(self):
MLSchema.populate_registry()
all_schema_paths = list(Path("mlspeclib").glob("schemas/**/*.yaml"))
for schema_path in all_schema_paths:
this_text = schema_path.read_text(encoding="utf-8")
this_dict = convert_yaml_to_dict(this_text)
if this_dict["mlspec_schema_version"]["meta"] == "0.0.1":
continue
# print(schema_path)
loaded_schema = MLSchema.create_schema(this_text)
self.assertIsNotNone(loaded_schema.schema_name)
def test_try_create_missing_mlspec_version_and_type(self):
schema_string = """
mlspec_schema_version:
# Identifies the version of this schema
meta: 0.0.1
mlspec_schema_type:
# Identifies the type of this schema
meta: base
"""
no_version = convert_yaml_to_dict(schema_string)
no_version.pop("mlspec_schema_version")
with self.assertRaises(KeyError):
MLSchema.create_schema(no_version)
no_schema = convert_yaml_to_dict(schema_string)
no_schema.pop("mlspec_schema_type")
with self.assertRaises(KeyError):
MLSchema.create_schema(no_schema)
def test_load_live_schemas(self):
MLSchema.populate_registry()
all_schema_paths = list(Path('mlspeclib').glob('schemas/**/*.yaml'))
for schema_path in all_schema_paths:
this_text = schema_path.read_text()
this_dict = convert_yaml_to_dict(this_text)
if this_dict['mlspec_schema_version']['meta'] == '0.0.1':
continue
print(schema_path)
loaded_schema = MLSchema.create_schema(this_text)
self.assertIsNotNone(loaded_schema.schema_name)
def test_interfaces_type_unknown(self):
instantiated_schema = MLSchema.create_schema(
self.wrap_schema_with_mlschema_info(
SampleSchema.TEST.INTERFACE)) # noqa
yaml_submission = convert_yaml_to_dict(
self.wrap_submission_with_mlschema_info(
SampleSubmissions.UNIT_TESTS.INTERFACE_INVALID_TYPE_UNKNOWN_1)
) # noqa
with self.assertRaises(ValidationError) as context:
instantiated_schema.load(yaml_submission)
self.assertTrue(
'string or a dict' in context.exception.messages['inputs'][0][0])
def test_interfaces_mismatch_type(self):
instantiated_schema = MLSchema.create_schema(
self.wrap_schema_with_mlschema_info(
SampleSchema.TEST.INTERFACE)) # noqa
yaml_submission = convert_yaml_to_dict(
self.wrap_submission_with_mlschema_info(
SampleSubmissions.UNIT_TESTS.INTERFACE_INVALID_MISMATCH_TYPE)
) # noqa
with self.assertRaises(ValidationError) as context:
instantiated_schema.load(yaml_submission)
self.assertTrue(
'valid default' in context.exception.messages['inputs'][0][0])
def test_merge_two_dicts_with_valid_base(self):
base_schema = MLSchema.create_schema(SampleSchema.SCHEMAS.BASE)
base_object = base_schema.load(SampleSubmissions.FULL_SUBMISSIONS.BASE)
datapath_schema = MLSchema.create_schema(SampleSchema.SCHEMAS.DATAPATH)
datapath_object = datapath_schema.load(
SampleSubmissions.FULL_SUBMISSIONS.DATAPATH
)
# Should not work - BASE did not merge with DATAPATH
with self.assertRaises(KeyError):
base_object[
"data_store"
] == "NULL_STRING_SHOULD_NOT_WORK" # pylint: disable=pointless-statement
base_object_dict = convert_yaml_to_dict(SampleSubmissions.FULL_SUBMISSIONS.BASE)
datapath_object_dict = convert_yaml_to_dict(
SampleSubmissions.FULL_SUBMISSIONS.DATAPATH
)
self.assertTrue(isinstance(base_object["run_id"], UUID))
self.assertTrue(base_object["run_id"] == UUID(base_object_dict["run_id"]))
# Should work - DATAPATH inherited from BASE
self.assertTrue(isinstance(datapath_object["data_store"], str))
self.assertTrue(
datapath_object["data_store"] == datapath_object_dict["data_store"]
)
self.assertTrue(isinstance(datapath_object["connection"]["access_key_id"], str))
self.assertTrue(
datapath_object["connection"]["access_key_id"]
== datapath_object_dict["connection"]["access_key_id"]
)
self.assertTrue(isinstance(datapath_object["run_id"], UUID))
self.assertTrue(
datapath_object["run_id"] == UUID(datapath_object_dict["run_id"])
)
def test_regex_valid(self):
instantiated_schema = MLSchema.create_schema(
self.wrap_schema_with_mlschema_info(
SampleSchema.TEST.REGEX)) # noqa
yaml_submission = convert_yaml_to_dict(
self.wrap_submission_with_mlschema_info(
SampleSubmissions.UNIT_TESTS.REGEX_ALL_LETTERS)) # noqa
instantiated_object = instantiated_schema.load(yaml_submission)
self.assertTrue(instantiated_object['all_letters'])
yaml_submission = convert_yaml_to_dict(
self.wrap_submission_with_mlschema_info(
SampleSubmissions.UNIT_TESTS.REGEX_ALL_NUMBERS)) # noqa
with self.assertRaises(ValidationError):
instantiated_schema.load(yaml_submission)
def test_uri_valid(self):
instantiated_schema = MLSchema.create_schema(
self.wrap_schema_with_mlschema_info(SampleSchema.TEST.URI)) # noqa
yaml_submission = convert_yaml_to_dict(
self.wrap_submission_with_mlschema_info(
SampleSubmissions.UNIT_TESTS.URI_VALID_1)) # noqa
instantiated_object = instantiated_schema.load(yaml_submission)
self.assertTrue(instantiated_object['endpoint'])
yaml_submission = convert_yaml_to_dict(
self.wrap_submission_with_mlschema_info(
SampleSubmissions.UNIT_TESTS.URI_VALID_2)) # noqa
instantiated_object = instantiated_schema.load(yaml_submission)
self.assertTrue(instantiated_object['endpoint'])
def test_interfaces_valid(self):
instantiated_schema = MLSchema.create_schema(
self.wrap_schema_with_mlschema_info(
SampleSchema.TEST.INTERFACE)) # noqa
yaml_submission = convert_yaml_to_dict(
self.wrap_submission_with_mlschema_info(
SampleSubmissions.UNIT_TESTS.INTERFACE_VALID_UNNAMED)) # noqa
instantiated_object = instantiated_schema.load(yaml_submission)
self.assertTrue(len(instantiated_object['inputs']) == 2)
yaml_submission = convert_yaml_to_dict(
self.wrap_submission_with_mlschema_info(
SampleSubmissions.UNIT_TESTS.INTERFACE_VALID_NAMED)) # noqa
instantiated_object = instantiated_schema.load(yaml_submission)
self.assertTrue(len(instantiated_object['inputs']) == 2)
def return_base_schema_and_submission():
instantiated_schema = MLSchema.create_schema(SampleSchema.SCHEMAS.BASE)
yaml_submission = convert_yaml_to_dict(SampleSubmissions.FULL_SUBMISSIONS.BASE)
return instantiated_schema, yaml_submission
def test_regex_invalid(self):
with self.assertRaises(AssertionError):
MLSchema.create_schema(
self.wrap_schema_with_mlschema_info(
SampleSchema.TEST.INVALID_REGEX)) # noqa
def test_enter_schema_with_invalid_yaml(self):
with self.assertRaises(ScannerError):
MLSchema.create_schema(SampleSchema.TEST.INVALID_YAML)