def test_store_del(self): init_mlgit() store_add('s3', 'bucket_test', 'personal') config_edit = yaml_load('.ml-git/config.yaml') self.assertEqual(config_edit['store']['s3']['bucket_test']['aws-credentials']['profile'], 'personal') store_del('s3', 'bucket_test') config = yaml_load('.ml-git/config.yaml') self.assertFalse('s3' in config['store'] and 'bucket_test' in config['store']['s3'])
def test_store_add(self): init_mlgit() store_add('s3', 'bucket_test', 'personal') config_edit = yaml_load('.ml-git/config.yaml') self.assertEqual(config_edit['store']['s3']['bucket_test']['aws-credentials']['profile'], 'personal') self.assertEqual(config_edit['store']['s3']['bucket_test']['region'], None) s = store_add('s4', 'bucket_test', 'personal') self.assertEqual(s, None) config = yaml_load('.ml-git/config.yaml') self.assertTrue('s3' in config['store'])
def test_store_del_global_config(self): with mock.patch('pathlib.Path.home', return_value=self.tmp_dir): init_mlgit() store_add('s3', 'bucket_test', 'personal', global_conf=True) config_edit = yaml_load('.mlgitconfig') self.assertEqual(config_edit['store']['s3']['bucket_test']['aws-credentials']['profile'], 'personal') with mock.patch('pathlib.Path.home', return_value=self.tmp_dir): store_del('s3', 'bucket_test', global_conf=True) config = yaml_load('.mlgitconfig') self.assertFalse('s3' in config['store'] and 'bucket_test' in config['store']['s3'])
def create(self, kwargs): artifact_name = kwargs['artifact_name'] categories = list(kwargs['category']) version = int(kwargs['version_number']) imported_dir = kwargs['import'] store_type = kwargs['store_type'] bucket_name = kwargs['bucket_name'] start_wizard = kwargs['wizard_config'] import_url = kwargs['import_url'] unzip_file = kwargs['unzip'] credentials_path = kwargs['credentials_path'] repo_type = self.__repo_type try: create_workspace_tree_structure(repo_type, artifact_name, categories, store_type, bucket_name, version, imported_dir, kwargs['mutability']) if start_wizard: has_new_store, store_type, bucket, profile, endpoint_url, git_repo = start_wizard_questions( repo_type) if has_new_store: store_add(store_type, bucket, profile, endpoint_url) update_store_spec(repo_type, artifact_name, store_type, bucket) remote_add(repo_type, git_repo) if import_url: self.create_config_store('gdrive', credentials_path) local = LocalRepository( self.__config, get_objects_path(self.__config, repo_type)) destine_path = os.path.join(repo_type, artifact_name, 'data') local.import_file_from_url(destine_path, import_url, StoreType.GDRIVE.value) if unzip_file: log.info('Unzipping files', CLASS_NAME=REPOSITORY_CLASS_NAME) data_path = os.path.join(get_root_path(), repo_type, artifact_name, 'data') unzip_files_in_directory(data_path) log.info("Project Created.", CLASS_NAME=REPOSITORY_CLASS_NAME) except Exception as e: if not isinstance(e, PermissionError): clear(os.path.join(repo_type, artifact_name)) if isinstance(e, KeyboardInterrupt): log.info("Create command aborted!", class_name=REPOSITORY_CLASS_NAME) else: log.error(e, CLASS_NAME=REPOSITORY_CLASS_NAME)
def test_store_add_global_config(self): init_mlgit() with mock.patch('pathlib.Path.home', return_value=self.tmp_dir): store_add('s3', 'bucket_test', 'personal', global_conf=True) config_edit = yaml_load('.mlgitconfig') self.assertEqual( config_edit['store']['s3']['bucket_test']['aws-credentials'] ['profile'], 'personal') self.assertEqual(config_edit['store']['s3']['bucket_test']['region'], 'us-east-1') with mock.patch('pathlib.Path.home', return_value=self.tmp_dir): s = store_add('s4', 'bucket_test', 'personal', global_conf=True) self.assertEqual(s, None) config = yaml_load('.mlgitconfig') self.assertTrue('s3' in config['store'])
def store_add(bucket_name, bucket_type=StoreType.S3H.value, credentials=None, global_configuration=False, endpoint_url=None): """This command will add a store to the ml-git project. Examples: store_add('my-bucket', type='s3h') Args: bucket_name (str): The name of the bucket in the storage. bucket_type (str, optional): Store type (s3h, azureblobh or gdriveh) [default: s3h]. credentials (str, optional): Name of the profile that stores the credentials or the path to the credentials. global_configuration (bool, optional): Use this option to set configuration at global level [default: False]. endpoint_url (str, optional): Store endpoint url. """ if bucket_type not in StoreType.to_list(): log.error('Aqui2') return admin.store_add(bucket_type, bucket_name, credentials, global_configuration, endpoint_url)
def check_store(self, container, store_type, tmpdir): store_add(store_type, container, 'personal') config_edit = yaml_load(os.path.join(tmpdir, '.ml-git/config.yaml')) self.assertIn(store_type, config_edit['store']) self.assertIn(container, config_edit['store'][store_type])
def storage_add(context, **kwargs): check_deprecated_command(context) admin.store_add(kwargs['type'], kwargs['bucket_name'], kwargs['credentials'], kwargs['global'], kwargs['endpoint_url'])
def store_add(**kwargs): admin.store_add(kwargs['type'], kwargs['bucket_name'], kwargs['credentials'], kwargs['global'], kwargs['endpoint_url'])