def test_28_checkout_entity_with_ignore_file(self):
entity = DATASETS
init_repository(entity, self)
workspace = os.path.join(self.tmp_dir, DATASETS, DATASET_NAME)
os.mkdir(os.path.join(workspace, 'data'))
create_file(workspace, 'image.png', '0')
create_file(workspace, 'file1', '0')
create_ignore_file(workspace)
self.assertNotIn(
ERROR_MESSAGE,
check_output(MLGIT_ADD %
(DATASETS, DATASET_NAME, '--bumpversion')))
self.assertNotIn(
ERROR_MESSAGE,
check_output(MLGIT_COMMIT % (DATASETS, DATASET_NAME, '')))
self.assertNotIn(ERROR_MESSAGE,
check_output(MLGIT_PUSH % (DATASETS, DATASET_NAME)))
clear(os.path.join(self.tmp_dir, ML_GIT_DIR, entity))
clear(workspace)
mlgit_ignore_file_path = os.path.join(workspace,
MLGIT_IGNORE_FILE_NAME)
self.assertNotIn(ERROR_MESSAGE,
check_output(MLGIT_ENTITY_INIT % entity))
self.assertNotIn(ERROR_MESSAGE,
check_output(MLGIT_CHECKOUT % (entity, DATASET_NAME)))
self.assertTrue(os.path.exists(mlgit_ignore_file_path))
self.assertTrue(
os.path.exists(os.path.join(workspace, 'data', 'file1')))
self.assertFalse(
os.path.exists(os.path.join(workspace, 'data', 'image.png')))
def _fsck_missing(self, entity):
init_repository(entity, self)
add_file(self, entity, '', 'new', file_content='2')
self.assertNotIn(
ERROR_MESSAGE,
check_output(MLGIT_COMMIT % (entity, entity + '-ex', '')))
self.assertNotIn(ERROR_MESSAGE,
check_output(MLGIT_PUSH % (entity, entity + '-ex')))
fsck_output = check_output(MLGIT_FSCK % entity)
self.assertIn(
output_messages['INFO_SUMMARY_FSCK_FILES'].format(
'corrupted', 0, ''), fsck_output)
self.assertIn(
output_messages['INFO_SUMMARY_FSCK_FILES'].format(
'missing', 0, ''), fsck_output)
self.assertIn(output_messages['INFO_FSCK_FIXED_FILES'].format(0, ''),
fsck_output)
os.remove(
os.path.join(self.tmp_dir, ML_GIT_DIR, entity, 'objects', 'hashfs',
'dr', 'vG',
'zdj7WdrvGPx9s8wmSB6KJGCmfCRNDQX6i8kVfFenQbWDQ1pmd'))
fsck_output = check_output(MLGIT_FSCK % entity)
self.assertIn(
output_messages['INFO_SUMMARY_FSCK_FILES'].format(
'corrupted', 0, ''), fsck_output)
self.assertIn(
output_messages['INFO_SUMMARY_FSCK_FILES'].format(
'missing', 1, ''), fsck_output)
self.assertIn(output_messages['INFO_FSCK_FIXED_FILES'].format(1, ''),
fsck_output)
def test_12_push_with_invalid_retry_number(self):
entity_type = DATASETS
init_repository(entity_type, self)
add_file(self, entity_type, '--bumpversion', 'new', file_content='0')
self.assertNotIn(ERROR_MESSAGE, check_output(MLGIT_COMMIT % (entity_type, DATASET_NAME, '')))
expected_error_message = '-2 is not in the valid range of 0 to 99999999.'
self.assertIn(expected_error_message, check_output(MLGIT_PUSH % (entity_type, entity_type+'-ex --retry=-2')))
def test_05_hard_with_data_in_subpath(self):
entity = DATASETS
subpath = 'data'
init_repository(entity, self)
first_commit_file_name = 'file1'
data_path = os.path.join(entity, entity+'-ex', subpath)
ensure_path_exists(data_path)
create_file(data_path, first_commit_file_name, '0', '')
self.assertNotIn(ERROR_MESSAGE, check_output(MLGIT_ADD % (entity, entity+'-ex', '--bumpversion')))
self.assertNotIn(ERROR_MESSAGE, check_output(MLGIT_COMMIT % (entity, entity+'-ex', '')))
second_commit_file_name = 'file2'
create_file(data_path, second_commit_file_name, '1', '')
self.assertNotIn(ERROR_MESSAGE, check_output(MLGIT_ADD % (entity, entity+'-ex', '--bumpversion')))
self.assertNotIn(ERROR_MESSAGE, check_output(MLGIT_COMMIT % (entity, entity+'-ex', '')))
self.assertTrue(os.path.exists(os.path.join(data_path, first_commit_file_name)))
self.assertTrue(os.path.exists(os.path.join(data_path, second_commit_file_name)))
self.assertIn(output_messages['INFO_INITIALIZING_RESET'] % ('--hard', 'HEAD~1'),
check_output(MLGIT_RESET % (entity, entity+'-ex') + ' --hard --reference=head~1'))
self.assertTrue(os.path.exists(os.path.join(data_path, first_commit_file_name)))
self.assertFalse(os.path.exists(os.path.join(data_path, second_commit_file_name)))
self.assertRegex(check_output(MLGIT_STATUS % (entity, entity+'-ex')),
r'Changes to be committed:\n\tNew file: datasets-ex.spec\n\nUntracked files:\n\nCorrupted files')
self._check_dir(self.dataset_tag)
def test_07_checkout_without_clear_workspace(self):
entity = DATASETS
init_repository(entity, self)
self._create_new_tag(entity, 'new')
with open(
os.path.join(self.tmp_dir, entity, entity + '-ex',
'newfile-tag2'), 'wt') as z:
z.write('0' * 100)
self.assertNotIn(
ERROR_MESSAGE,
check_output(MLGIT_ADD %
(entity, entity + '-ex', '--bumpversion')))
self.assertNotIn(
ERROR_MESSAGE,
check_output(MLGIT_COMMIT % (entity, entity + '-ex', '')))
self.assertNotIn(ERROR_MESSAGE,
check_output(MLGIT_PUSH % (entity, entity + '-ex')))
self.assertNotIn(
ERROR_MESSAGE,
check_output(MLGIT_CHECKOUT %
(entity, 'computer-vision__images__datasets-ex__2')))
file = os.path.join(self.tmp_dir, entity, DATASET_NAME, 'newfile0')
self.check_metadata()
self.check_amount_of_files(entity, 6)
self.assertTrue(os.path.exists(file))
def test_42_graph_order_by_version_number(self):
init_repository(DATASETS, self)
self.create_file_in_ws(DATASETS, 'file', '1')
api.add(DATASETS, DATASET_NAME, file_path=['file'])
api.commit(DATASETS, DATASET_NAME)
self.create_file_in_ws(DATASETS, 'file2', '1')
api.add(DATASETS, DATASET_NAME, bumpversion=True, file_path=['file2'])
api.commit(DATASETS, DATASET_NAME)
self.create_file_in_ws(DATASETS, 'file3', '1')
api.add(DATASETS, DATASET_NAME, file_path=['file3'])
self.assertNotIn(
ERROR_MESSAGE,
check_output(MLGIT_COMMIT %
(DATASETS, DATASET_NAME, '--version=10')))
local_manager = api.init_local_entity_manager()
entities_relationships = local_manager.get_project_entities_relationships(
export_type=FileType.DOT.value)
graph_path = local_manager.export_graph(entities_relationships)
self.assertTrue(os.path.exists(graph_path))
with open(graph_path, 'r') as graph_file:
content = graph_file.read()
self.assertIn(
'\\"{} (1)\\" -> \\"{} (2)\\"'.format(DATASET_NAME,
DATASET_NAME), content)
self.assertIn(
'\\"{} (2)\\" -> \\"{} (10)\\"'.format(DATASET_NAME,
DATASET_NAME), content)
def test_13_commit_files(self):
self.set_up_test()
self.set_up_add_test()
api.add(DATASETS,
DATASET_NAME,
bumpversion=True,
fsck=False,
file_path=['file'])
api.commit(DATASETS, DATASET_NAME)
HEAD = os.path.join(self.tmp_dir, ML_GIT_DIR, DATASETS, 'refs',
DATASET_NAME, 'HEAD')
self.assertTrue(os.path.exists(HEAD))
init_repository(LABELS, self)
self.create_file_in_ws(LABELS, 'file', '0')
api.add(LABELS,
'labels-ex',
bumpversion=True,
fsck=False,
file_path=['file'])
api.commit(LABELS, 'labels-ex', related_dataset=DATASET_NAME)
labels_metadata = os.path.join(self.tmp_dir, ML_GIT_DIR, LABELS,
'metadata')
with open(os.path.join(labels_metadata, 'labels-ex',
'labels-ex.spec')) as y:
spec = yaml_processor.load(y)
HEAD = os.path.join(self.tmp_dir, ML_GIT_DIR, LABELS, 'refs',
'labels-ex', 'HEAD')
self.assertTrue(os.path.exists(HEAD))
self.assertEqual('computer-vision__images__datasets-ex__11',
spec[LABELS_SPEC_KEY][DATASET_SPEC_KEY]['tag'])
def test_15_add_and_edit_file_with_same_hash(self):
entity_name = '{}-ex'.format(DATASETS)
init_repository(DATASETS, self, mutability=MUTABLE)
add_file(self, DATASETS, '')
self.assertNotIn(
ERROR_MESSAGE,
check_output(MLGIT_COMMIT % (DATASETS, entity_name, '')))
self.assertNotIn(ERROR_MESSAGE,
check_output(MLGIT_PUSH % (DATASETS, entity_name)))
data_folder = os.path.join(self.tmp_dir, DATASETS, DATASET_NAME)
file = os.path.join(data_folder, 'file1')
copyfile(file, os.path.join(data_folder, 'file1 - Copy'))
with open(file, 'wt') as f:
f.write('Modified file.')
self.assertNotIn(
ERROR_MESSAGE,
check_output(MLGIT_ADD %
(DATASETS, entity_name, ' --bumpversion')))
self.assertNotIn(
ERROR_MESSAGE,
check_output(MLGIT_COMMIT % (DATASETS, entity_name, '')))
self.assertNotIn(ERROR_MESSAGE,
check_output(MLGIT_PUSH % (DATASETS, entity_name)))
index = os.path.join(ML_GIT_DIR, DATASETS, 'index', 'metadata',
DATASET_NAME, 'INDEX.yaml')
self._check_index(index, ['file1', 'file1 - Copy'], [])
def test_10_checkout_with_unsaved_work(self):
entity = DATASETS
init_repository(entity, self)
self._create_new_tag(entity, 'tag1')
entity_dir = os.path.join(self.tmp_dir, DATASETS, DATASET_NAME)
with open(os.path.join(entity_dir, 'tag2'), 'wt') as z:
z.write('0' * 100)
self.assertNotIn(
ERROR_MESSAGE,
check_output(MLGIT_ADD % (DATASETS, DATASET_NAME, '')))
self.assertNotIn(
ERROR_MESSAGE,
check_output(MLGIT_COMMIT %
(entity, entity + '-ex', '--version=3')))
self.assertNotIn(ERROR_MESSAGE,
check_output(MLGIT_PUSH % (entity, entity + '-ex')))
unsaved_file_dir = os.path.join(self.tmp_dir, DATASETS, DATASET_NAME,
'folderA')
ensure_path_exists(unsaved_file_dir)
with open(os.path.join(unsaved_file_dir, 'test-unsaved-file'),
'wt') as z:
z.write('0' * 100)
output_command = check_output(
MLGIT_CHECKOUT % (DATASETS, DATASET_NAME + ' --version=2'))
self.assertIn(output_messages['ERROR_DISCARDED_LOCAL_CHANGES'],
output_command)
self.assertIn('test-unsaved-file', output_command)
def test_08_push_after_remote_del(self):
clear(
os.path.join(MINIO_BUCKET_PATH,
'zdj7WWjGAAJ8gdky5FKcVLfd63aiRUGb8fkc8We2bvsp9WW12'))
entity_type = DATASETS
init_repository(entity_type, self)
add_file(self, entity_type, '--bumpversion', 'new', file_content='0')
metadata_path = os.path.join(self.tmp_dir, ML_GIT_DIR, entity_type,
'metadata')
self.assertIn(
output_messages['INFO_COMMIT_REPO'] %
(metadata_path, entity_type + '-ex'),
check_output(MLGIT_COMMIT %
(entity_type, entity_type + '-ex', '')))
head_file = os.path.join(self.tmp_dir, ML_GIT_DIR, entity_type, 'refs',
entity_type + '-ex', 'HEAD')
self.assertTrue(os.path.exists(head_file))
self._remote_del(entity_type)
self.assertIn(
output_messages['ERROR_REMOTE_NOT_FOUND'],
check_output(MLGIT_PUSH % (entity_type, entity_type + '-ex')))
self.assertFalse(
os.path.exists(
os.path.join(
MINIO_BUCKET_PATH,
'zdj7WWjGAAJ8gdky5FKcVLfd63aiRUGb8fkc8We2bvsp9WW12')))
def _setup_update_entity(self, entity_type):
init_repository(entity_type, self)
add_file(self, entity_type, '', 'new')
metadata_path = os.path.join(self.tmp_dir, ML_GIT_DIR, entity_type, 'metadata')
self.assertIn(output_messages['INFO_COMMIT_REPO'] % (metadata_path, entity_type + '-ex'),
check_output(MLGIT_COMMIT % (entity_type, entity_type + '-ex', '')))
self.assertNotIn(ERROR_MESSAGE, check_output(MLGIT_PUSH % (entity_type, entity_type + '-ex')))
def test_04_commit_command_with_version(self):
init_repository(DATASETS, self)
create_spec(self, DATASETS, self.tmp_dir)
workspace = os.path.join(self.tmp_dir, DATASETS, DATASET_NAME)
os.makedirs(os.path.join(workspace, 'data'))
create_file(workspace, 'file1', '0')
self.assertIn(output_messages['INFO_ADDING_PATH'] % DATASETS,
check_output(MLGIT_ADD % (DATASETS, DATASET_NAME, "")))
self.assertIn(
output_messages['INFO_COMMIT_REPO'] % (os.path.join(
self.tmp_dir, ML_GIT_DIR, DATASETS, 'metadata'), DATASET_NAME),
check_output(MLGIT_COMMIT % (DATASETS, DATASET_NAME, '')))
create_file(workspace, 'file2', '1')
self.assertIn(output_messages['INFO_ADDING_PATH'] % DATASETS,
check_output(MLGIT_ADD % (DATASETS, DATASET_NAME, "")))
self.assertIn(
output_messages['ERROR_INVALID_VALUE_FOR'] % ('--version', '-10'),
check_output(MLGIT_COMMIT %
(DATASETS, DATASET_NAME, ' --version=-10')))
self.assertIn(
output_messages['ERROR_INVALID_VALUE_FOR'] % ('--version', 'test'),
check_output(MLGIT_COMMIT %
(DATASETS, DATASET_NAME, '--version=test')))
self.assertIn(
output_messages['INFO_COMMIT_REPO'] % (os.path.join(
self.tmp_dir, ML_GIT_DIR, DATASETS, 'metadata'), DATASET_NAME),
check_output(MLGIT_COMMIT %
(DATASETS, DATASET_NAME, '--version=2')))
def _push_entity(self, entity_type):
clear(
os.path.join(MINIO_BUCKET_PATH,
'zdj7WWjGAAJ8gdky5FKcVLfd63aiRUGb8fkc8We2bvsp9WW12'))
init_repository(entity_type, self)
add_file(self, entity_type, '--bumpversion', 'new', file_content='0')
metadata_path = os.path.join(self.tmp_dir, ML_GIT_DIR, entity_type,
'metadata')
self.assertIn(
messages[17] %
(metadata_path,
os.path.join('computer-vision', 'images', entity_type + '-ex')),
check_output(MLGIT_COMMIT %
(entity_type, entity_type + '-ex', '')))
HEAD = os.path.join(self.tmp_dir, ML_GIT_DIR, entity_type, 'refs',
entity_type + '-ex', 'HEAD')
self.assertTrue(os.path.exists(HEAD))
self.assertNotIn(
ERROR_MESSAGE,
check_output(MLGIT_PUSH % (entity_type, entity_type + '-ex')))
os.chdir(metadata_path)
self.assertTrue(
os.path.exists(
os.path.join(
MINIO_BUCKET_PATH,
'zdj7WWjGAAJ8gdky5FKcVLfd63aiRUGb8fkc8We2bvsp9WW12')))
self.assertIn('computer-vision__images__' + entity_type + '-ex__2',
check_output('git describe --tags'))
def export(self, repotype, entity):
init_repository(repotype, self)
add_file(self, repotype, '', repotype)
file_in_store = os.path.join(PATH_TEST, 'data', 'mlgit',
repotype + 'file0')
self.assertIn(
messages[17] %
(os.path.join(self.tmp_dir, ML_GIT_DIR, repotype, 'metadata'),
os.path.join('computer-vision', 'images', entity)),
check_output(MLGIT_COMMIT % (repotype, entity, '')))
self.assertFalse(os.path.exists(file_in_store))
check_output(MLGIT_PUSH % (repotype, entity))
store = 's3h://mlgit'
tag = 'computer-vision__images__%s__1' % entity
self.assertIn(
messages[66] % (tag, store, 's3://mlgit'),
check_output(
MLGIT_EXPORT %
(repotype,
' {} {} --credentials={} --endpoint=http://127.0.0.1:9000'.
format(tag, BUCKET_NAME, PROFILE))))
self.assertTrue(os.path.exists(file_in_store))
def test_08_push_without_default_config(self):
entity_type = 'dataset'
artifact_name = 'dataset-ex'
init_repository(entity_type, self, profile=None)
add_file(self, entity_type, '--bumpversion', 'new')
self.assertNotIn(ERROR_MESSAGE, check_output(MLGIT_COMMIT % (entity_type, artifact_name, '')))
self.assertIn('Unable to locate credentials', check_output(MLGIT_PUSH % (entity_type, artifact_name)))
def test_05_push_with_wrong_credentials_profile(self):
entity_type = 'dataset'
artifact_name = 'dataset-ex'
init_repository(entity_type, self, profile='personal2')
add_file(self, entity_type, '--bumpversion', 'new')
self.assertNotIn(ERROR_MESSAGE, check_output(MLGIT_COMMIT % (entity_type, artifact_name, '')))
self.assertIn(output_messages['ERROR_AWS_KEY_NOT_EXIST'], check_output(MLGIT_PUSH % (entity_type, artifact_name)))
def set_up_test(self, repo_type=MODELS):
self.TAG_TIMES = []
entity_name = '{}-ex'.format(repo_type)
init_repository(repo_type, self)
create_spec(self, repo_type, self.tmp_dir)
metrics_options = '--metric Accuracy 10 --metric Recall 10'
self.assertNotIn(
ERROR_MESSAGE,
check_output(MLGIT_ADD %
(repo_type, entity_name, metrics_options)))
self.assertNotIn(
ERROR_MESSAGE,
check_output(MLGIT_COMMIT % (repo_type, entity_name, '')))
self._git_commit_time()
metrics_options = '--metric Accuracy 20 --metric Recall 20'
workspace = os.path.join(self.tmp_dir, repo_type, entity_name)
os.makedirs(os.path.join(workspace, 'data'))
create_file(workspace, 'file1', '0')
self.assertNotIn(
ERROR_MESSAGE,
check_output(MLGIT_ADD %
(repo_type, entity_name, metrics_options)))
self.assertNotIn(
ERROR_MESSAGE,
check_output(MLGIT_COMMIT %
(repo_type, entity_name, ' --version=2')))
self._git_commit_time()
def test_05_hard_with_data_in_subpath(self):
entity = DATASETS
subpath = 'data'
init_repository(entity, self)
first_commit_file_name = 'file1'
data_path = os.path.join(entity, entity+'-ex', subpath)
ensure_path_exists(data_path)
create_file(data_path, first_commit_file_name, '0', '')
self.assertNotIn(ERROR_MESSAGE, check_output(MLGIT_ADD % (entity, entity+'-ex', '--bumpversion')))
self.assertNotIn(ERROR_MESSAGE, check_output(MLGIT_COMMIT % (entity, entity+'-ex', '')))
second_commit_file_name = 'file2'
create_file(data_path, second_commit_file_name, '1', '')
self.assertNotIn(ERROR_MESSAGE, check_output(MLGIT_ADD % (entity, entity+'-ex', '--bumpversion')))
self.assertNotIn(ERROR_MESSAGE, check_output(MLGIT_COMMIT % (entity, entity+'-ex', '')))
self.assertTrue(os.path.exists(os.path.join(data_path, first_commit_file_name)))
self.assertTrue(os.path.exists(os.path.join(data_path, second_commit_file_name)))
self.assertIn(output_messages['INFO_INITIALIZING_RESET'] % ('--hard', 'HEAD~1'),
check_output(MLGIT_RESET % (entity, entity+'-ex') + ' --hard --reference=head~1'))
self.assertTrue(os.path.exists(os.path.join(data_path, first_commit_file_name)))
self.assertFalse(os.path.exists(os.path.join(data_path, second_commit_file_name)))
self.assertRegex(check_output(MLGIT_STATUS % (entity, entity+'-ex')),
DATASET_UNPUBLISHED_COMMITS_INFO_REGEX.format(unpublished_commits=1, pluralize_char='') +
DATASET_PUSH_INFO_REGEX +
r'Changes to be committed:\s+'
r'New file: datasets-ex.spec')
self._check_dir(self.dataset_tag)
def test_04_fsck_with_full_option(self):
entity = DATASETS
init_repository(entity, self)
add_file(self, entity, '', 'new', file_content='2')
self.assertIn(
output_messages['INFO_SUMMARY_FSCK_FILES'].format(
'corrupted', 0, ''), check_output(MLGIT_FSCK % entity))
with open(
os.path.join(
self.tmp_dir, ML_GIT_DIR, entity, 'objects', 'hashfs',
'dr', 'vG',
'zdj7WdrvGPx9s8wmSB6KJGCmfCRNDQX6i8kVfFenQbWDQ1pmd'),
'wt') as file:
file.write('corrupting file')
output = check_output((MLGIT_FSCK % entity) + ' --full')
self.assertIn(
output_messages['INFO_SUMMARY_FSCK_FILES'].format(
'corrupted', 1,
'[\'zdj7WdrvGPx9s8wmSB6KJGCmfCRNDQX6i8kVfFenQbWDQ1pmd\']'),
output)
self.assertIn(
output_messages['INFO_SUMMARY_FSCK_FILES'].format(
'missing', 0, '[]'), output)
self.assertIn('zdj7WdrvGPx9s8wmSB6KJGCmfCRNDQX6i8kVfFenQbWDQ1pmd',
output)
def _create_entity_with_mutability(self, entity_type, mutability_type):
init_repository(entity_type, self)
workspace = os.path.join(self.tmp_dir, entity_type,
entity_type + '-ex')
create_spec(self, entity_type, self.tmp_dir, 1, mutability_type)
os.makedirs(os.path.join(workspace, 'data'))
create_file(workspace, 'file1', '0')
self.assertNotIn(
ERROR_MESSAGE,
check_output(MLGIT_ADD % (entity_type, entity_type + '-ex', '')))
self.assertIn(
messages[17] %
(os.path.join(self.tmp_dir, ML_GIT_DIR, entity_type, 'metadata'),
os.path.join('computer-vision', 'images', entity_type + '-ex')),
check_output(MLGIT_COMMIT %
(entity_type, entity_type + '-ex', '')))
self.assertNotIn(
ERROR_MESSAGE,
check_output(MLGIT_PUSH % (entity_type, entity_type + '-ex')))
clear(os.path.join(self.tmp_dir, ML_GIT_DIR))
clear(workspace)
clear(os.path.join(self.tmp_dir, entity_type))
def set_up_add_test(self, entity=DATASETS):
clear(os.path.join(self.tmp_dir, ML_GIT_DIR))
clear(os.path.join(self.tmp_dir, entity))
init_repository(entity, self)
self.create_file_in_ws(entity, 'file', '0')
self.create_file_in_ws(entity, 'file2', '1')
def test_03_import_when_credentials_does_not_exist(self):
init_repository('dataset', self)
self.assertIn(
messages[52] % 'anyone',
check_output(MLGIT_IMPORT % ('dataset', 'bucket', 'dataset-ex') +
' --credentials=anyone'))
self.check_amount_of_files('dataset', 1)
def _create_entity_with_mutability(self, entity_type, mutability_type):
init_repository(entity_type, self)
workspace = os.path.join(self.tmp_dir, entity_type, entity_type + '-ex')
create_spec(self, entity_type, self.tmp_dir, 1, mutability_type)
os.makedirs(os.path.join(workspace, 'data'))
create_file(workspace, 'file1', '0')
self._push_files(entity_type, '')
self._clear_path()
def set_up_push_without_profile(self,
entity_type=DATASETS,
artifact_name=DATASET_NAME):
init_repository(entity_type, self, profile=None)
add_file(self, entity_type, '--bumpversion', 'new')
self.assertNotIn(
ERROR_MESSAGE,
check_output(MLGIT_COMMIT % (entity_type, artifact_name, '')))
def test_01_import_with_wrong_credentials(self):
init_repository('dataset', self)
self.assertIn(
messages[54],
check_output(MLGIT_IMPORT % ('dataset', 'bucket', 'dataset-ex') +
' --credentials=personal2'))
self.check_amount_of_files('dataset', 1)
def test_11_push_with_wrong_minio_endpoint(self):
entity_type = DATASETS
artifact_name = DATASET_NAME
init_repository(entity_type, self)
add_file(self, entity_type, '--bumpversion', 'new')
self.assertNotIn(ERROR_MESSAGE, check_output(MLGIT_COMMIT % (entity_type, artifact_name, '')))
output = check_output(MLGIT_PUSH % (entity_type, artifact_name))
self.assertIn(ERROR_MESSAGE, output)
self.assertIn('There was an error checking if bucket \'{}\' exists.'.format(BUCKET_NAME), output)
def _checkout_entity(self, entity_type, tag=DATASET_TAG):
init_repository(entity_type, self)
self.assertIn(
output_messages['INFO_MLGIT_PULL'] %
(os.path.join(self.tmp_dir, ML_GIT_DIR, entity_type, 'metadata')),
check_output(MLGIT_UPDATE % entity_type))
self.assertNotIn(ERROR_MESSAGE,
check_output(MLGIT_CHECKOUT % (entity_type, tag)))
def test_06_commit_with_large_version_number(self):
init_repository(DATASETS, self)
create_spec(self, DATASETS, self.tmp_dir)
self.assertIn(output_messages['ERROR_INVALID_VALUE_FOR'] % ('--version', '9999999999'),
check_output(MLGIT_COMMIT % (DATASETS, DATASET_NAME, ' --version=9999999999')))
self.assertIn(output_messages['ERROR_INVALID_VALUE_FOR'] % ('--version', '9999999999'),
check_output(MLGIT_COMMIT % (MODELS, MODELS + '-ex', ' --version=9999999999')))
self.assertIn(output_messages['ERROR_INVALID_VALUE_FOR'] % ('--version', '9999999999'),
check_output(MLGIT_COMMIT % (LABELS, LABELS + '-ex', ' --version=9999999999')))
def test_02_import_with_wrong_bucket(self):
init_repository('dataset', self)
self.assertIn(
messages[51],
check_output(MLGIT_IMPORT %
('dataset', 'wrong-bucket', 'dataset-ex') +
' --object=test --credentials=' + PROFILE))
self.check_amount_of_files('dataset', 1)
def _checkout_entity(self, entity_type, tag='computer-vision__images__dataset-ex__1', bare=True):
init_repository(entity_type, self)
self.assertIn(messages[20] % (os.path.join(self.tmp_dir, ML_GIT_DIR, entity_type, 'metadata')),
check_output(MLGIT_UPDATE % entity_type))
if bare:
self.assertIn(messages[68], check_output(MLGIT_CHECKOUT % (entity_type, tag + ' --bare')))
else:
self.assertNotIn(ERROR_MESSAGE, check_output(MLGIT_CHECKOUT % (entity_type, tag)))
self.assertTrue(os.path.exists(os.path.join(self.tmp_dir, 'dataset', 'computer-vision', 'images', 'dataset-ex', 'data', 'file1')))