def test_mlgit_init_without_permission(self): output = io.StringIO() with mock.patch('os.mkdir', side_effect=PermissionError()): with redirect_stdout(output): init_mlgit() self.assertIn('Permission denied.', output.getvalue())
def test_save_global_config_in_local(self): remote_default = 'git_local_server.git' new_remote = 'git_local_server2.git' init_mlgit() self.assertTrue(os.path.isdir('.ml-git')) config = yaml_load('.ml-git/config.yaml') self.assertEqual(config[DATASETS]['git'], remote_default) global_conf = { DATASETS: { 'git': 'url' }, MODELS: { 'git': 'url' }, LABELS: { 'git': new_remote }, STORAGE_CONFIG_KEY: {} } with mock.patch('ml_git.config.global_config_load', return_value=global_conf): save_global_config_in_local() config = yaml_load('.ml-git/config.yaml') self.assertEqual(config[LABELS]['git'], new_remote)
def test_remote_add_global_config(self): remote_default = 'git_local_server.git' new_remote = 'git_local_server2.git' dataset = 'dataset' init_mlgit() with mock.patch('pathlib.Path.home', return_value=self.tmp_dir): remote_add(dataset, new_remote, global_conf=True) self.assertTrue(os.path.exists('.mlgitconfig')) config = yaml_load('.ml-git/config.yaml') config_global = yaml_load('.mlgitconfig') self.assertEqual(config_global['dataset']['git'], new_remote) self.assertNotEqual(config['dataset']['git'], remote_default) with mock.patch('pathlib.Path.home', return_value=self.tmp_dir): remote_add(dataset, '', global_conf=True) config_ = yaml_load('.mlgitconfig') self.assertEqual(config_['dataset']['git'], '') with mock.patch('pathlib.Path.home', return_value=self.tmp_dir): remote_add(dataset, new_remote, global_conf=True) config__ = yaml_load('.mlgitconfig') self.assertEqual(config__['dataset']['git'], new_remote)
def test_save_global_config_in_local(self): remote_default = 'git_local_server.git' new_remote = 'git_local_server2.git' init_mlgit() self.assertTrue(os.path.isdir('.ml-git')) config = yaml_load('.ml-git/config.yaml') self.assertEqual(config['dataset']['git'], remote_default) global_conf = { 'dataset': { 'git': 'url' }, 'model': { 'git': 'url' }, 'labels': { 'git': new_remote }, 'store': {} } with mock.patch('ml_git.config.global_config_load', return_value=global_conf): save_global_config_in_local() config = yaml_load('.ml-git/config.yaml') self.assertEqual(config['labels']['git'], new_remote)
def test_store_del(self): init_mlgit() store_add('s3', 'bucket_test', 'personal') config_edit = yaml_load('.ml-git/config.yaml') self.assertEqual(config_edit['store']['s3']['bucket_test']['aws-credentials']['profile'], 'personal') store_del('s3', 'bucket_test') config = yaml_load('.ml-git/config.yaml') self.assertFalse('s3' in config['store'] and 'bucket_test' in config['store']['s3'])
def test_store_add(self): init_mlgit() store_add('s3', 'bucket_test', 'personal') config_edit = yaml_load('.ml-git/config.yaml') self.assertEqual(config_edit['store']['s3']['bucket_test']['aws-credentials']['profile'], 'personal') self.assertEqual(config_edit['store']['s3']['bucket_test']['region'], None) s = store_add('s4', 'bucket_test', 'personal') self.assertEqual(s, None) config = yaml_load('.ml-git/config.yaml') self.assertTrue('s3' in config['store'])
def test_storage_del(self): init_mlgit() storage_add(S3, 'bucket_test', 'personal') config_edit = yaml_load('.ml-git/config.yaml') self.assertEqual( config_edit[STORAGE_CONFIG_KEY][S3]['bucket_test'] ['aws-credentials']['profile'], 'personal') storage_del(S3, 'bucket_test') config = yaml_load('.ml-git/config.yaml') self.assertFalse(S3 in config[STORAGE_CONFIG_KEY] and 'bucket_test' in config[STORAGE_CONFIG_KEY][S3])
def test_merged_config_load(self): global_conf = {DATASETS: {'git': 'global-url'}, MODELS: {'git': 'url'}, PUSH_THREADS_COUNT: 10} local_conf = {DATASETS: {'git': 'url'}} init_mlgit() with mock.patch('ml_git.config.mlgit_config_load', return_value=local_conf): with mock.patch('ml_git.config.global_config_load', return_value=global_conf): config_file = merged_config_load() self.assertEqual(config_file[DATASETS]['git'], 'url') self.assertEqual(config_file[MODELS]['git'], 'url') self.assertEqual(config_file[PUSH_THREADS_COUNT], 10)
def test_remote_del(self): remote_default = 'git_local_server.git' init_mlgit() config = yaml_load('.ml-git/config.yaml') self.assertEqual(config[DATASETS]['git'], '') remote_add(DATASETS, remote_default) config = yaml_load('.ml-git/config.yaml') self.assertEqual(config[DATASETS]['git'], remote_default) remote_del(DATASETS) config_ = yaml_load('.ml-git/config.yaml') self.assertEqual(config_[DATASETS]['git'], '')
def test_remote_del(self): remote_default = 'git_local_server.git' dataset = 'dataset' init_mlgit() config = yaml_load('.ml-git/config.yaml') self.assertEqual(config['dataset']['git'], '') remote_add(dataset, remote_default) config = yaml_load('.ml-git/config.yaml') self.assertEqual(config['dataset']['git'], remote_default) remote_del(dataset) config_ = yaml_load('.ml-git/config.yaml') self.assertEqual(config_['dataset']['git'], '')
def test_storage_add(self): init_mlgit() storage_add(S3, 'bucket_test', 'personal') config_edit = yaml_load('.ml-git/config.yaml') self.assertEqual( config_edit[STORAGE_CONFIG_KEY][S3]['bucket_test'] ['aws-credentials']['profile'], 'personal') self.assertEqual( config_edit[STORAGE_CONFIG_KEY][S3]['bucket_test']['region'], None) s = storage_add('s4', 'bucket_test', 'personal') self.assertEqual(s, None) config = yaml_load('.ml-git/config.yaml') self.assertTrue(S3 in config[STORAGE_CONFIG_KEY])
def test_store_del_global_config(self): with mock.patch('pathlib.Path.home', return_value=self.tmp_dir): init_mlgit() store_add('s3', 'bucket_test', 'personal', global_conf=True) config_edit = yaml_load('.mlgitconfig') self.assertEqual(config_edit['store']['s3']['bucket_test']['aws-credentials']['profile'], 'personal') with mock.patch('pathlib.Path.home', return_value=self.tmp_dir): store_del('s3', 'bucket_test', global_conf=True) config = yaml_load('.mlgitconfig') self.assertFalse('s3' in config['store'] and 'bucket_test' in config['store']['s3'])
def _initialize_repository_on_the_fly(self): if os.path.exists(get_global_config_path()): log.info('Initializing the project with global settings', class_name=REPOSITORY_CLASS_NAME) init_mlgit() save_global_config_in_local() metadata_path = get_metadata_path(self.__config) if not os.path.exists(metadata_path): Metadata('', metadata_path, self.__config, self.__repo_type).init() return metadata_path raise RootPathException( 'You are not in an initialized ml-git repository and do not have a global configuration.' )
def test_remote_add(self): remote_default = 'git_local_server.git' new_remote = 'git_local_server2.git' init_mlgit() remote_add(DATASETS, new_remote) self.assertTrue(os.path.isdir('.ml-git')) config = yaml_load('.ml-git/config.yaml') self.assertEqual(config[DATASETS]['git'], new_remote) self.assertNotEqual(remote_default, new_remote) remote_add(DATASETS, '') config_ = yaml_load('.ml-git/config.yaml') self.assertEqual(config_[DATASETS]['git'], '') remote_add(DATASETS, new_remote) self.assertTrue(os.path.isdir('.ml-git')) config__ = yaml_load('.ml-git/config.yaml') self.assertEqual(config__[DATASETS]['git'], new_remote)
def test_store_add_global_config(self): init_mlgit() with mock.patch('pathlib.Path.home', return_value=self.tmp_dir): store_add('s3', 'bucket_test', 'personal', global_conf=True) config_edit = yaml_load('.mlgitconfig') self.assertEqual( config_edit['store']['s3']['bucket_test']['aws-credentials'] ['profile'], 'personal') self.assertEqual(config_edit['store']['s3']['bucket_test']['region'], 'us-east-1') with mock.patch('pathlib.Path.home', return_value=self.tmp_dir): s = store_add('s4', 'bucket_test', 'personal', global_conf=True) self.assertEqual(s, None) config = yaml_load('.mlgitconfig') self.assertTrue('s3' in config['store'])
def init(entity): """This command will start the ml-git entity. Examples: init('repository') init('dataset') Args: entity (str): The type of entity that will be initialized (repository, dataset, labels or model). """ if entity == 'repository': init_mlgit() elif entity in EntityType.to_list(): repo = Repository(config_load(), entity) repo.init() else: log.error('The type of entity entered is invalid. Valid types are: [repository, dataset, labels or model]')
def test_storage_add_global_config(self): init_mlgit() with mock.patch('pathlib.Path.home', return_value=self.tmp_dir): storage_add(S3, 'bucket_test', 'personal', global_conf=True) config_edit = yaml_load('.mlgitconfig') self.assertEqual( config_edit[STORAGE_CONFIG_KEY][S3]['bucket_test'] ['aws-credentials']['profile'], 'personal') self.assertEqual( config_edit[STORAGE_CONFIG_KEY][S3]['bucket_test']['region'], None) with mock.patch('pathlib.Path.home', return_value=self.tmp_dir): s = storage_add('s4', 'bucket_test', 'personal', global_conf=True) self.assertEqual(s, None) config = yaml_load('.mlgitconfig') self.assertTrue(S3 in config[STORAGE_CONFIG_KEY])
def init(entity): """This command will start the ml-git entity. Examples: init('repository') init('datasets') Args: entity (str): The type of an ML entity. (datasets, labels or models) """ if entity == 'repository': init_mlgit() elif entity in EntityType.to_list(): repo = get_repository_instance(entity) repo.init() else: log.error(output_messages['ERROR_INVALID_ENTERED_ENTITY_TYPE'])
def test_storage_add_check_type_azureblobh(self): init_mlgit() storage_type = AZUREBLOBH container = 'azure' self.check_storage(container, storage_type, self.tmp_dir)
def test_store_add_check_type_azureblobh(self): init_mlgit() store_type = 'azureblobh' container = 'azure' self.check_store(container, store_type, self.tmp_dir)
def init(): init_mlgit()
def test_mlgit_init(self): init_mlgit() self.assertTrue(os.path.isdir('.ml-git'))