def test_load_object_from_disk(self):
MLSchema.populate_registry()
file_path = Path('tests/data/0/0/1/datapath.yaml')
ml_object, _ = MLObject.create_object_from_file(file_path)
self.assertIsNotNone(ml_object.run_date)
self.assertIsNotNone(ml_object.connection.endpoint)
def test_all_data(self):
MLSchema.populate_registry()
all_data_files = list(Path('tests').glob('data/0/0/1/*.yaml'))
self.assertTrue(len(all_data_files) > 1)
for data_file in all_data_files:
print(data_file)
loaded_object, errors = MLObject.create_object_from_file(data_file)
self.assertTrue(len(errors) == 0)
self.assertIsNotNone(loaded_object.get_schema())
def test_load_live_data(self):
MLSchema.populate_registry()
all_data_files = list(Path("tests").glob("data/**/*.yaml"))
for data_file in all_data_files:
this_text = data_file.read_text(encoding="utf-8")
this_dict = convert_yaml_to_dict(this_text)
if this_dict["schema_version"] == "0.0.1":
continue
# print(data_file)
loaded_object, errors = MLObject.create_object_from_file(data_file)
self.assertTrue(len(errors) == 0)
self.assertIsNotNone(loaded_object.get_schema())
def test_load_live_data(self):
MLSchema.populate_registry()
all_data_files = list(Path('tests').glob('data/**/*.yaml'))
for data_file in all_data_files:
this_text = data_file.read_text()
this_dict = convert_yaml_to_dict(this_text)
if this_dict['schema_version'] == '0.0.1':
continue
print(data_file)
loaded_object, errors = MLObject.create_object_from_file(data_file)
self.assertTrue(len(errors) == 0)
self.assertIsNotNone(loaded_object.get_schema())
def test_load_and_save_file(self):
run_id = uuid.uuid4()
save_path = Path(self.test_dir.name) / str(run_id)
save_path.mkdir()
datapath_object = MLObject()
datapath_object.set_type('0.0.1', MLSchemaTypes.DATAPATH)
datapath_object.run_id = run_id
datapath_object.step_id = uuid.uuid4()
datapath_object.run_date = datetime.datetime.now()
datapath_object.data_store = None # This is an intentional bug
# This is an intentional bug (Should be AWS_BLOB)
datapath_object.storage_connection_type = 'AWS_BLOB_OBJECT'
datapath_object.connection.endpoint = None # Another intentional bug
datapath_object.connection.access_key_id = 'AKIAIOSFODNN7EXAMPLE'
datapath_object.connection.secret_access_key = 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY'
response, errors = datapath_object.save(save_path)
self.assertFalse(response)
self.assertTrue(len(errors) == 3)
self.assertTrue(len(list(Path(save_path).glob('*'))) == 0)
datapath_object.storage_connection_type = 'AWS_BLOB'
response, errors = datapath_object.save(save_path)
self.assertFalse(response)
self.assertTrue(len(errors) == 2)
self.assertTrue(len(list(Path(save_path).glob('*'))) == 0)
datapath_object.connection.endpoint = 'http://s3.amazon.com/BUCKET'
response, errors = datapath_object.save(save_path)
self.assertFalse(response)
self.assertTrue(len(errors) == 1)
self.assertTrue(len(list(Path(save_path).glob('*'))) == 0)
datapath_object.data_store = 'BUCKET NAME'
response, errors = datapath_object.save(save_path)
self.assertTrue(response)
self.assertTrue(len(errors) == 0)
path = Path(save_path)
all_files = list(path.glob('*'))
self.assertTrue(len(all_files) == 1)
ml_object, errors = MLObject.create_object_from_file(all_files[0])
self.assertTrue(len(ml_object) == 13)
self.assertTrue(len(errors) == 0)
self.assertTrue(datapath_object.data_store == ml_object.data_store)
self.assertTrue(datapath_object.storage_connection_type ==
ml_object.storage_connection_type)
self.assertTrue(datapath_object.connection.endpoint ==
ml_object.connection.endpoint)
