def test_mlgit_init_without_permission(self):
output = io.StringIO()
with mock.patch('os.mkdir', side_effect=PermissionError()):
with redirect_stdout(output):
init_mlgit()
self.assertIn('Permission denied.', output.getvalue())
def test_save_global_config_in_local(self):
remote_default = 'git_local_server.git'
new_remote = 'git_local_server2.git'
init_mlgit()
self.assertTrue(os.path.isdir('.ml-git'))
config = yaml_load('.ml-git/config.yaml')
self.assertEqual(config[DATASETS]['git'], remote_default)
global_conf = {
DATASETS: {
'git': 'url'
},
MODELS: {
'git': 'url'
},
LABELS: {
'git': new_remote
},
STORAGE_CONFIG_KEY: {}
}
with mock.patch('ml_git.config.global_config_load',
return_value=global_conf):
save_global_config_in_local()
config = yaml_load('.ml-git/config.yaml')
self.assertEqual(config[LABELS]['git'], new_remote)
def test_remote_add_global_config(self):
remote_default = 'git_local_server.git'
new_remote = 'git_local_server2.git'
dataset = 'dataset'
init_mlgit()
with mock.patch('pathlib.Path.home', return_value=self.tmp_dir):
remote_add(dataset, new_remote, global_conf=True)
self.assertTrue(os.path.exists('.mlgitconfig'))
config = yaml_load('.ml-git/config.yaml')
config_global = yaml_load('.mlgitconfig')
self.assertEqual(config_global['dataset']['git'], new_remote)
self.assertNotEqual(config['dataset']['git'], remote_default)
with mock.patch('pathlib.Path.home', return_value=self.tmp_dir):
remote_add(dataset, '', global_conf=True)
config_ = yaml_load('.mlgitconfig')
self.assertEqual(config_['dataset']['git'], '')
with mock.patch('pathlib.Path.home', return_value=self.tmp_dir):
remote_add(dataset, new_remote, global_conf=True)
config__ = yaml_load('.mlgitconfig')
self.assertEqual(config__['dataset']['git'], new_remote)
def test_save_global_config_in_local(self):
remote_default = 'git_local_server.git'
new_remote = 'git_local_server2.git'
init_mlgit()
self.assertTrue(os.path.isdir('.ml-git'))
config = yaml_load('.ml-git/config.yaml')
self.assertEqual(config['dataset']['git'], remote_default)
global_conf = {
'dataset': {
'git': 'url'
},
'model': {
'git': 'url'
},
'labels': {
'git': new_remote
},
'store': {}
}
with mock.patch('ml_git.config.global_config_load',
return_value=global_conf):
save_global_config_in_local()
config = yaml_load('.ml-git/config.yaml')
self.assertEqual(config['labels']['git'], new_remote)
def test_store_del(self):
init_mlgit()
store_add('s3', 'bucket_test', 'personal')
config_edit = yaml_load('.ml-git/config.yaml')
self.assertEqual(config_edit['store']['s3']['bucket_test']['aws-credentials']['profile'], 'personal')
store_del('s3', 'bucket_test')
config = yaml_load('.ml-git/config.yaml')
self.assertFalse('s3' in config['store'] and 'bucket_test' in config['store']['s3'])
def test_store_add(self):
init_mlgit()
store_add('s3', 'bucket_test', 'personal')
config_edit = yaml_load('.ml-git/config.yaml')
self.assertEqual(config_edit['store']['s3']['bucket_test']['aws-credentials']['profile'], 'personal')
self.assertEqual(config_edit['store']['s3']['bucket_test']['region'], None)
s = store_add('s4', 'bucket_test', 'personal')
self.assertEqual(s, None)
config = yaml_load('.ml-git/config.yaml')
self.assertTrue('s3' in config['store'])
def test_storage_del(self):
init_mlgit()
storage_add(S3, 'bucket_test', 'personal')
config_edit = yaml_load('.ml-git/config.yaml')
self.assertEqual(
config_edit[STORAGE_CONFIG_KEY][S3]['bucket_test']
['aws-credentials']['profile'], 'personal')
storage_del(S3, 'bucket_test')
config = yaml_load('.ml-git/config.yaml')
self.assertFalse(S3 in config[STORAGE_CONFIG_KEY]
and 'bucket_test' in config[STORAGE_CONFIG_KEY][S3])
def test_merged_config_load(self):
global_conf = {DATASETS: {'git': 'global-url'}, MODELS: {'git': 'url'}, PUSH_THREADS_COUNT: 10}
local_conf = {DATASETS: {'git': 'url'}}
init_mlgit()
with mock.patch('ml_git.config.mlgit_config_load', return_value=local_conf):
with mock.patch('ml_git.config.global_config_load', return_value=global_conf):
config_file = merged_config_load()
self.assertEqual(config_file[DATASETS]['git'], 'url')
self.assertEqual(config_file[MODELS]['git'], 'url')
self.assertEqual(config_file[PUSH_THREADS_COUNT], 10)
def test_remote_del(self):
remote_default = 'git_local_server.git'
init_mlgit()
config = yaml_load('.ml-git/config.yaml')
self.assertEqual(config[DATASETS]['git'], '')
remote_add(DATASETS, remote_default)
config = yaml_load('.ml-git/config.yaml')
self.assertEqual(config[DATASETS]['git'], remote_default)
remote_del(DATASETS)
config_ = yaml_load('.ml-git/config.yaml')
self.assertEqual(config_[DATASETS]['git'], '')
def test_remote_del(self):
remote_default = 'git_local_server.git'
dataset = 'dataset'
init_mlgit()
config = yaml_load('.ml-git/config.yaml')
self.assertEqual(config['dataset']['git'], '')
remote_add(dataset, remote_default)
config = yaml_load('.ml-git/config.yaml')
self.assertEqual(config['dataset']['git'], remote_default)
remote_del(dataset)
config_ = yaml_load('.ml-git/config.yaml')
self.assertEqual(config_['dataset']['git'], '')
def test_storage_add(self):
init_mlgit()
storage_add(S3, 'bucket_test', 'personal')
config_edit = yaml_load('.ml-git/config.yaml')
self.assertEqual(
config_edit[STORAGE_CONFIG_KEY][S3]['bucket_test']
['aws-credentials']['profile'], 'personal')
self.assertEqual(
config_edit[STORAGE_CONFIG_KEY][S3]['bucket_test']['region'], None)
s = storage_add('s4', 'bucket_test', 'personal')
self.assertEqual(s, None)
config = yaml_load('.ml-git/config.yaml')
self.assertTrue(S3 in config[STORAGE_CONFIG_KEY])
def test_store_del_global_config(self):
with mock.patch('pathlib.Path.home', return_value=self.tmp_dir):
init_mlgit()
store_add('s3', 'bucket_test', 'personal', global_conf=True)
config_edit = yaml_load('.mlgitconfig')
self.assertEqual(config_edit['store']['s3']['bucket_test']['aws-credentials']['profile'], 'personal')
with mock.patch('pathlib.Path.home', return_value=self.tmp_dir):
store_del('s3', 'bucket_test', global_conf=True)
config = yaml_load('.mlgitconfig')
self.assertFalse('s3' in config['store'] and 'bucket_test' in config['store']['s3'])
def _initialize_repository_on_the_fly(self):
if os.path.exists(get_global_config_path()):
log.info('Initializing the project with global settings',
class_name=REPOSITORY_CLASS_NAME)
init_mlgit()
save_global_config_in_local()
metadata_path = get_metadata_path(self.__config)
if not os.path.exists(metadata_path):
Metadata('', metadata_path, self.__config,
self.__repo_type).init()
return metadata_path
raise RootPathException(
'You are not in an initialized ml-git repository and do not have a global configuration.'
)
def test_remote_add(self):
remote_default = 'git_local_server.git'
new_remote = 'git_local_server2.git'
init_mlgit()
remote_add(DATASETS, new_remote)
self.assertTrue(os.path.isdir('.ml-git'))
config = yaml_load('.ml-git/config.yaml')
self.assertEqual(config[DATASETS]['git'], new_remote)
self.assertNotEqual(remote_default, new_remote)
remote_add(DATASETS, '')
config_ = yaml_load('.ml-git/config.yaml')
self.assertEqual(config_[DATASETS]['git'], '')
remote_add(DATASETS, new_remote)
self.assertTrue(os.path.isdir('.ml-git'))
config__ = yaml_load('.ml-git/config.yaml')
self.assertEqual(config__[DATASETS]['git'], new_remote)
def test_store_add_global_config(self):
init_mlgit()
with mock.patch('pathlib.Path.home', return_value=self.tmp_dir):
store_add('s3', 'bucket_test', 'personal', global_conf=True)
config_edit = yaml_load('.mlgitconfig')
self.assertEqual(
config_edit['store']['s3']['bucket_test']['aws-credentials']
['profile'], 'personal')
self.assertEqual(config_edit['store']['s3']['bucket_test']['region'],
'us-east-1')
with mock.patch('pathlib.Path.home', return_value=self.tmp_dir):
s = store_add('s4', 'bucket_test', 'personal', global_conf=True)
self.assertEqual(s, None)
config = yaml_load('.mlgitconfig')
self.assertTrue('s3' in config['store'])
def init(entity):
"""This command will start the ml-git entity.
Examples:
init('repository')
init('dataset')
Args:
entity (str): The type of entity that will be initialized (repository, dataset, labels or model).
"""
if entity == 'repository':
init_mlgit()
elif entity in EntityType.to_list():
repo = Repository(config_load(), entity)
repo.init()
else:
log.error('The type of entity entered is invalid. Valid types are: [repository, dataset, labels or model]')
def test_storage_add_global_config(self):
init_mlgit()
with mock.patch('pathlib.Path.home', return_value=self.tmp_dir):
storage_add(S3, 'bucket_test', 'personal', global_conf=True)
config_edit = yaml_load('.mlgitconfig')
self.assertEqual(
config_edit[STORAGE_CONFIG_KEY][S3]['bucket_test']
['aws-credentials']['profile'], 'personal')
self.assertEqual(
config_edit[STORAGE_CONFIG_KEY][S3]['bucket_test']['region'], None)
with mock.patch('pathlib.Path.home', return_value=self.tmp_dir):
s = storage_add('s4', 'bucket_test', 'personal', global_conf=True)
self.assertEqual(s, None)
config = yaml_load('.mlgitconfig')
self.assertTrue(S3 in config[STORAGE_CONFIG_KEY])
def init(entity):
"""This command will start the ml-git entity.
Examples:
init('repository')
init('datasets')
Args:
entity (str): The type of an ML entity. (datasets, labels or models)
"""
if entity == 'repository':
init_mlgit()
elif entity in EntityType.to_list():
repo = get_repository_instance(entity)
repo.init()
else:
log.error(output_messages['ERROR_INVALID_ENTERED_ENTITY_TYPE'])
def test_storage_add_check_type_azureblobh(self):
init_mlgit()
storage_type = AZUREBLOBH
container = 'azure'
self.check_storage(container, storage_type, self.tmp_dir)
def test_store_add_check_type_azureblobh(self):
init_mlgit()
store_type = 'azureblobh'
container = 'azure'
self.check_store(container, store_type, self.tmp_dir)
def init():
init_mlgit()
def test_mlgit_init(self):
init_mlgit()
self.assertTrue(os.path.isdir('.ml-git'))
