def test_load_object_from_disk(self): MLSchema.populate_registry() file_path = Path('tests/data/0/0/1/datapath.yaml') ml_object, _ = MLObject.create_object_from_file(file_path) self.assertIsNotNone(ml_object.run_date) self.assertIsNotNone(ml_object.connection.endpoint)
def test_all_data(self): MLSchema.populate_registry() all_data_files = list(Path('tests').glob('data/0/0/1/*.yaml')) self.assertTrue(len(all_data_files) > 1) for data_file in all_data_files: print(data_file) loaded_object, errors = MLObject.create_object_from_file(data_file) self.assertTrue(len(errors) == 0) self.assertIsNotNone(loaded_object.get_schema())
def test_load_live_data(self): MLSchema.populate_registry() all_data_files = list(Path("tests").glob("data/**/*.yaml")) for data_file in all_data_files: this_text = data_file.read_text(encoding="utf-8") this_dict = convert_yaml_to_dict(this_text) if this_dict["schema_version"] == "0.0.1": continue # print(data_file) loaded_object, errors = MLObject.create_object_from_file(data_file) self.assertTrue(len(errors) == 0) self.assertIsNotNone(loaded_object.get_schema())
def test_load_live_data(self): MLSchema.populate_registry() all_data_files = list(Path('tests').glob('data/**/*.yaml')) for data_file in all_data_files: this_text = data_file.read_text() this_dict = convert_yaml_to_dict(this_text) if this_dict['schema_version'] == '0.0.1': continue print(data_file) loaded_object, errors = MLObject.create_object_from_file(data_file) self.assertTrue(len(errors) == 0) self.assertIsNotNone(loaded_object.get_schema())
def test_load_and_save_file(self): run_id = uuid.uuid4() save_path = Path(self.test_dir.name) / str(run_id) save_path.mkdir() datapath_object = MLObject() datapath_object.set_type('0.0.1', MLSchemaTypes.DATAPATH) datapath_object.run_id = run_id datapath_object.step_id = uuid.uuid4() datapath_object.run_date = datetime.datetime.now() datapath_object.data_store = None # This is an intentional bug # This is an intentional bug (Should be AWS_BLOB) datapath_object.storage_connection_type = 'AWS_BLOB_OBJECT' datapath_object.connection.endpoint = None # Another intentional bug datapath_object.connection.access_key_id = 'AKIAIOSFODNN7EXAMPLE' datapath_object.connection.secret_access_key = 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY' response, errors = datapath_object.save(save_path) self.assertFalse(response) self.assertTrue(len(errors) == 3) self.assertTrue(len(list(Path(save_path).glob('*'))) == 0) datapath_object.storage_connection_type = 'AWS_BLOB' response, errors = datapath_object.save(save_path) self.assertFalse(response) self.assertTrue(len(errors) == 2) self.assertTrue(len(list(Path(save_path).glob('*'))) == 0) datapath_object.connection.endpoint = 'http://s3.amazon.com/BUCKET' response, errors = datapath_object.save(save_path) self.assertFalse(response) self.assertTrue(len(errors) == 1) self.assertTrue(len(list(Path(save_path).glob('*'))) == 0) datapath_object.data_store = 'BUCKET NAME' response, errors = datapath_object.save(save_path) self.assertTrue(response) self.assertTrue(len(errors) == 0) path = Path(save_path) all_files = list(path.glob('*')) self.assertTrue(len(all_files) == 1) ml_object, errors = MLObject.create_object_from_file(all_files[0]) self.assertTrue(len(ml_object) == 13) self.assertTrue(len(errors) == 0) self.assertTrue(datapath_object.data_store == ml_object.data_store) self.assertTrue(datapath_object.storage_connection_type == ml_object.storage_connection_type) self.assertTrue(datapath_object.connection.endpoint == ml_object.connection.endpoint)