def test_store_del(self):
init_mlgit()
store_add('s3', 'bucket_test', 'personal')
config_edit = yaml_load('.ml-git/config.yaml')
self.assertEqual(config_edit['store']['s3']['bucket_test']['aws-credentials']['profile'], 'personal')
store_del('s3', 'bucket_test')
config = yaml_load('.ml-git/config.yaml')
self.assertFalse('s3' in config['store'] and 'bucket_test' in config['store']['s3'])
def test_store_add(self):
init_mlgit()
store_add('s3', 'bucket_test', 'personal')
config_edit = yaml_load('.ml-git/config.yaml')
self.assertEqual(config_edit['store']['s3']['bucket_test']['aws-credentials']['profile'], 'personal')
self.assertEqual(config_edit['store']['s3']['bucket_test']['region'], None)
s = store_add('s4', 'bucket_test', 'personal')
self.assertEqual(s, None)
config = yaml_load('.ml-git/config.yaml')
self.assertTrue('s3' in config['store'])
def test_store_del_global_config(self):
with mock.patch('pathlib.Path.home', return_value=self.tmp_dir):
init_mlgit()
store_add('s3', 'bucket_test', 'personal', global_conf=True)
config_edit = yaml_load('.mlgitconfig')
self.assertEqual(config_edit['store']['s3']['bucket_test']['aws-credentials']['profile'], 'personal')
with mock.patch('pathlib.Path.home', return_value=self.tmp_dir):
store_del('s3', 'bucket_test', global_conf=True)
config = yaml_load('.mlgitconfig')
self.assertFalse('s3' in config['store'] and 'bucket_test' in config['store']['s3'])
def create(self, kwargs):
artifact_name = kwargs['artifact_name']
categories = list(kwargs['category'])
version = int(kwargs['version_number'])
imported_dir = kwargs['import']
store_type = kwargs['store_type']
bucket_name = kwargs['bucket_name']
start_wizard = kwargs['wizard_config']
import_url = kwargs['import_url']
unzip_file = kwargs['unzip']
credentials_path = kwargs['credentials_path']
repo_type = self.__repo_type
try:
create_workspace_tree_structure(repo_type, artifact_name,
categories, store_type,
bucket_name, version, imported_dir,
kwargs['mutability'])
if start_wizard:
has_new_store, store_type, bucket, profile, endpoint_url, git_repo = start_wizard_questions(
repo_type)
if has_new_store:
store_add(store_type, bucket, profile, endpoint_url)
update_store_spec(repo_type, artifact_name, store_type, bucket)
remote_add(repo_type, git_repo)
if import_url:
self.create_config_store('gdrive', credentials_path)
local = LocalRepository(
self.__config, get_objects_path(self.__config, repo_type))
destine_path = os.path.join(repo_type, artifact_name, 'data')
local.import_file_from_url(destine_path, import_url,
StoreType.GDRIVE.value)
if unzip_file:
log.info('Unzipping files', CLASS_NAME=REPOSITORY_CLASS_NAME)
data_path = os.path.join(get_root_path(), repo_type,
artifact_name, 'data')
unzip_files_in_directory(data_path)
log.info("Project Created.", CLASS_NAME=REPOSITORY_CLASS_NAME)
except Exception as e:
if not isinstance(e, PermissionError):
clear(os.path.join(repo_type, artifact_name))
if isinstance(e, KeyboardInterrupt):
log.info("Create command aborted!",
class_name=REPOSITORY_CLASS_NAME)
else:
log.error(e, CLASS_NAME=REPOSITORY_CLASS_NAME)
def test_store_add_global_config(self):
init_mlgit()
with mock.patch('pathlib.Path.home', return_value=self.tmp_dir):
store_add('s3', 'bucket_test', 'personal', global_conf=True)
config_edit = yaml_load('.mlgitconfig')
self.assertEqual(
config_edit['store']['s3']['bucket_test']['aws-credentials']
['profile'], 'personal')
self.assertEqual(config_edit['store']['s3']['bucket_test']['region'],
'us-east-1')
with mock.patch('pathlib.Path.home', return_value=self.tmp_dir):
s = store_add('s4', 'bucket_test', 'personal', global_conf=True)
self.assertEqual(s, None)
config = yaml_load('.mlgitconfig')
self.assertTrue('s3' in config['store'])
def store_add(bucket_name, bucket_type=StoreType.S3H.value, credentials=None, global_configuration=False, endpoint_url=None):
"""This command will add a store to the ml-git project.
Examples:
store_add('my-bucket', type='s3h')
Args:
bucket_name (str): The name of the bucket in the storage.
bucket_type (str, optional): Store type (s3h, azureblobh or gdriveh) [default: s3h].
credentials (str, optional): Name of the profile that stores the credentials or the path to the credentials.
global_configuration (bool, optional): Use this option to set configuration at global level [default: False].
endpoint_url (str, optional): Store endpoint url.
"""
if bucket_type not in StoreType.to_list():
log.error('Aqui2')
return
admin.store_add(bucket_type, bucket_name, credentials, global_configuration, endpoint_url)
def check_store(self, container, store_type, tmpdir):
store_add(store_type, container, 'personal')
config_edit = yaml_load(os.path.join(tmpdir, '.ml-git/config.yaml'))
self.assertIn(store_type, config_edit['store'])
self.assertIn(container, config_edit['store'][store_type])
def storage_add(context, **kwargs):
check_deprecated_command(context)
admin.store_add(kwargs['type'], kwargs['bucket_name'],
kwargs['credentials'], kwargs['global'],
kwargs['endpoint_url'])
def store_add(**kwargs):
admin.store_add(kwargs['type'], kwargs['bucket_name'],
kwargs['credentials'], kwargs['global'],
kwargs['endpoint_url'])
