def _initialize_entity(self, entity_type, git=GIT_PATH):
api.init('repository')
api.remote_add(entity_type, git)
api.storage_add(bucket_type=STORAGE_TYPE, bucket_name=BUCKET_NAME, credentials=PROFILE)
api.init(entity_type)
metadata_path = os.path.join(self.tmp_dir, ML_GIT_DIR, entity_type, 'metadata')
self.assertTrue(os.path.exists(metadata_path))
def _add_remote(self, entity_type):
api.init('repository')
api.remote_add(entity_type, os.path.join(self.tmp_dir, GIT_PATH))
with open(os.path.join(self.tmp_dir, ML_GIT_DIR, 'config.yaml'),
'r') as c:
config = yaml_processor.load(c)
self.assertEqual(os.path.join(self.tmp_dir, GIT_PATH),
config[entity_type]['git'])
def test_21_add_storage(self):
api.init('repository')
with open(os.path.join(self.tmp_dir, ML_GIT_DIR, 'config.yaml'), 'r') as c:
config = yaml_processor.load(c)
self.assertNotIn(S3H, config[STORAGE_CONFIG_KEY])
api.storage_add(bucket_name=BUCKET_NAME, credentials=PROFILE)
with open(os.path.join(self.tmp_dir, ML_GIT_DIR, 'config.yaml'), 'r') as c:
config = yaml_processor.load(c)
self.assertEqual(PROFILE, config[STORAGE_CONFIG_KEY][S3H][BUCKET_NAME]['aws-credentials']['profile'])
def test_22_add_storage_azure_type(self):
bucket_name = 'container_azure'
api.init('repository')
with open(os.path.join(self.tmp_dir, ML_GIT_DIR, 'config.yaml'), 'r') as c:
config = yaml_processor.load(c)
self.assertNotIn(AZUREBLOBH, config[STORAGE_CONFIG_KEY])
api.storage_add(bucket_name=bucket_name, bucket_type=AZUREBLOBH)
with open(os.path.join(self.tmp_dir, ML_GIT_DIR, 'config.yaml'), 'r') as c:
config = yaml_processor.load(c)
self.assertIn(bucket_name, config[STORAGE_CONFIG_KEY][AZUREBLOBH])
def test_23_add_storage_gdrive_type(self):
bucket_name = 'my-drive'
profile = 'path-to-credentials'
api.init('repository')
with open(os.path.join(self.tmp_dir, ML_GIT_DIR, 'config.yaml'), 'r') as c:
config = yaml_processor.load(c)
self.assertNotIn(GDRIVEH, config[STORAGE_CONFIG_KEY])
api.storage_add(bucket_name=bucket_name, bucket_type=GDRIVEH, credentials=profile)
with open(os.path.join(self.tmp_dir, ML_GIT_DIR, 'config.yaml'), 'r') as c:
config = yaml_processor.load(c)
self.assertEqual(profile, config[STORAGE_CONFIG_KEY][GDRIVEH][bucket_name]['credentials-path'])
def test_22_add_store_azure_type(self):
bucket_type = 'azureblobh'
bucket_name = 'container_azure'
api.init('repository')
with open(os.path.join(self.tmp_dir, ML_GIT_DIR, 'config.yaml'),
'r') as c:
config = yaml_processor.load(c)
self.assertNotIn(bucket_type, config['store'])
api.store_add(bucket_name=bucket_name, bucket_type=bucket_type)
with open(os.path.join(self.tmp_dir, ML_GIT_DIR, 'config.yaml'),
'r') as c:
config = yaml_processor.load(c)
self.assertIn(bucket_name, config['store'][bucket_type])
def test_32_add_storage_with_region(self):
bucket_region = 'my-bucket-region'
api.init('repository')
api.storage_add(bucket_name=BUCKET_NAME,
credentials=PROFILE,
region=bucket_region)
with open(os.path.join(self.tmp_dir, ML_GIT_DIR, 'config.yaml'),
'r') as c:
config = yaml_processor.load(c)
self.assertEqual(
PROFILE, config[STORAGE_CONFIG_KEY][S3H][BUCKET_NAME]
['aws-credentials']['profile'])
self.assertEqual(
bucket_region,
config[STORAGE_CONFIG_KEY][S3H][BUCKET_NAME]['region'])
def test_39_local_export_graph_without_entities(self):
api.init('repository')
self._clean_up_local_config()
local_manager = api.init_local_entity_manager()
local_manager.display_graph(export_path=os.getcwd())
output = ';'.join([record.message for record in self.caplog.records])
self.assertIn(
output_messages['WARN_REPOSITORY_NOT_FOUND_FOR_ENTITY'] %
'datasets', output)
self.assertIn(
output_messages['WARN_REPOSITORY_NOT_FOUND_FOR_ENTITY'] % 'labels',
output)
self.assertIn(
output_messages['WARN_REPOSITORY_NOT_FOUND_FOR_ENTITY'] % 'models',
output)
self.assertIn(output_messages['INFO_ENTITIES_RELATIONSHIPS_NOT_FOUND'],
output)
def test_17_init_repository(self):
config = os.path.join(self.tmp_dir, ML_GIT_DIR, 'config.yaml')
self.assertFalse(os.path.exists(config))
api.init('repository')
self.assertTrue(os.path.exists(config))