def _fsck_missing(self, entity):
init_repository(entity, self)
add_file(self, entity, '', 'new', file_content='2')
self.assertNotIn(
ERROR_MESSAGE,
check_output(MLGIT_COMMIT % (entity, entity + '-ex', '')))
self.assertNotIn(ERROR_MESSAGE,
check_output(MLGIT_PUSH % (entity, entity + '-ex')))
fsck_output = check_output(MLGIT_FSCK % entity)
self.assertIn(
output_messages['INFO_SUMMARY_FSCK_FILES'].format(
'corrupted', 0, ''), fsck_output)
self.assertIn(
output_messages['INFO_SUMMARY_FSCK_FILES'].format(
'missing', 0, ''), fsck_output)
self.assertIn(output_messages['INFO_FSCK_FIXED_FILES'].format(0, ''),
fsck_output)
os.remove(
os.path.join(self.tmp_dir, ML_GIT_DIR, entity, 'objects', 'hashfs',
'dr', 'vG',
'zdj7WdrvGPx9s8wmSB6KJGCmfCRNDQX6i8kVfFenQbWDQ1pmd'))
fsck_output = check_output(MLGIT_FSCK % entity)
self.assertIn(
output_messages['INFO_SUMMARY_FSCK_FILES'].format(
'corrupted', 0, ''), fsck_output)
self.assertIn(
output_messages['INFO_SUMMARY_FSCK_FILES'].format(
'missing', 1, ''), fsck_output)
self.assertIn(output_messages['INFO_FSCK_FIXED_FILES'].format(1, ''),
fsck_output)
def _push_entity(self, entity_type):
clear(
os.path.join(MINIO_BUCKET_PATH,
'zdj7WWjGAAJ8gdky5FKcVLfd63aiRUGb8fkc8We2bvsp9WW12'))
init_repository(entity_type, self)
add_file(self, entity_type, '--bumpversion', 'new', file_content='0')
metadata_path = os.path.join(self.tmp_dir, ML_GIT_DIR, entity_type,
'metadata')
self.assertIn(
messages[17] %
(metadata_path,
os.path.join('computer-vision', 'images', entity_type + '-ex')),
check_output(MLGIT_COMMIT %
(entity_type, entity_type + '-ex', '')))
HEAD = os.path.join(self.tmp_dir, ML_GIT_DIR, entity_type, 'refs',
entity_type + '-ex', 'HEAD')
self.assertTrue(os.path.exists(HEAD))
self.assertNotIn(
ERROR_MESSAGE,
check_output(MLGIT_PUSH % (entity_type, entity_type + '-ex')))
os.chdir(metadata_path)
self.assertTrue(
os.path.exists(
os.path.join(
MINIO_BUCKET_PATH,
'zdj7WWjGAAJ8gdky5FKcVLfd63aiRUGb8fkc8We2bvsp9WW12')))
self.assertIn('computer-vision__images__' + entity_type + '-ex__2',
check_output('git describe --tags'))
def _setup_update_entity(self, entity_type):
init_repository(entity_type, self)
add_file(self, entity_type, '', 'new')
metadata_path = os.path.join(self.tmp_dir, ML_GIT_DIR, entity_type, 'metadata')
self.assertIn(output_messages['INFO_COMMIT_REPO'] % (metadata_path, entity_type + '-ex'),
check_output(MLGIT_COMMIT % (entity_type, entity_type + '-ex', '')))
self.assertNotIn(ERROR_MESSAGE, check_output(MLGIT_PUSH % (entity_type, entity_type + '-ex')))
def _add_tag_entity(self, entity_type):
self.set_up_tag(entity_type)
add_file(self, entity_type, '--bumpversion', 'new')
self.assertIn(
messages[17] %
(os.path.join(self.tmp_dir, ML_GIT_DIR, entity_type, 'metadata'),
os.path.join('computer-vision', 'images', entity_type + '-ex')),
check_output(MLGIT_COMMIT %
(entity_type, entity_type + '-ex', '')))
self.assertNotIn(
ERROR_MESSAGE,
check_output(MLGIT_PUSH % (entity_type, entity_type + '-ex')))
with open(os.path.join(entity_type, entity_type + '-ex', 'file1'),
'wb') as z:
z.write(b'0' * 1024)
self.assertIn(
messages[53],
check_output(MLGIT_TAG_ADD %
(entity_type, entity_type + '-ex', 'test-tag')))
self.assertNotIn(
ERROR_MESSAGE,
check_output(MLGIT_PUSH % (entity_type, entity_type + '-ex')))
def set_up_push(self):
os.makedirs(self.workspace)
create_spec(self,
self.repo_type,
self.tmp_dir,
version=1,
mutability=STRICT,
storage_type=self.storage_type)
self.assertIn(
output_messages['INFO_INITIALIZED_PROJECT_IN'] % self.tmp_dir,
check_output(MLGIT_INIT))
self.assertIn(
output_messages['INFO_ADD_REMOTE'] % (GIT_PATH, self.repo_type),
check_output(MLGIT_REMOTE_ADD % (self.repo_type, GIT_PATH)))
self.assertIn(
output_messages['INFO_ADD_STORAGE_WITHOUT_PROFILE'] %
(self.storage_type, self.bucket),
check_output('ml-git repository storage add %s --type=%s' %
(self.bucket, self.storage_type)))
self.assertNotIn(ERROR_MESSAGE,
check_output(MLGIT_ENTITY_INIT % DATASETS))
add_file(self, self.repo_type, '', 'new')
metadata_path = os.path.join(ML_GIT_DIR, DATASETS, 'metadata')
self.assertIn(
output_messages['INFO_COMMIT_REPO'] %
(os.path.join(self.tmp_dir, metadata_path), DATASET_NAME),
check_output(MLGIT_COMMIT % (self.repo_type, DATASET_NAME, '')))
HEAD = os.path.join(ML_GIT_DIR, DATASETS, 'refs', DATASET_NAME, 'HEAD')
self.assertTrue(os.path.exists(HEAD))
def set_up_push(self):
os.makedirs(self.workspace)
create_spec(self,
self.repo_type,
self.tmp_dir,
version=1,
mutability='strict',
store_type=self.store_type)
self.assertIn(messages[0], check_output(MLGIT_INIT))
self.assertIn(
messages[2] % (GIT_PATH, self.repo_type),
check_output(MLGIT_REMOTE_ADD % (self.repo_type, GIT_PATH)))
self.assertIn(
messages[87] % (self.store_type, self.bucket),
check_output('ml-git repository store add %s --type=%s' %
(self.bucket, self.store_type)))
self.assertNotIn(ERROR_MESSAGE,
check_output(MLGIT_ENTITY_INIT % 'dataset'))
add_file(self, self.repo_type, '', 'new')
metadata_path = os.path.join(ML_GIT_DIR, 'dataset', 'metadata')
self.assertIn(
messages[17] %
(os.path.join(self.tmp_dir, metadata_path),
os.path.join('computer-vision', 'images', 'dataset-ex')),
check_output(MLGIT_COMMIT % (self.repo_type, 'dataset-ex', '')))
HEAD = os.path.join(ML_GIT_DIR, 'dataset', 'refs', 'dataset-ex',
'HEAD')
self.assertTrue(os.path.exists(HEAD))
def test_06_create_with_wizard_enabled(self):
entity_type = DATASETS
entity_init(entity_type, self)
add_file(self, entity_type, '--bumpversion', 'new')
runner = CliRunner()
result = runner.invoke(entity.datasets, ['create', 'ENTITY-NAME', '--wizard'], input='test\nstrict')
self.assertIn(prompt_msg.CATEGORIES_MESSAGE, result.output)
def test_08_push_after_remote_del(self):
clear(
os.path.join(MINIO_BUCKET_PATH,
'zdj7WWjGAAJ8gdky5FKcVLfd63aiRUGb8fkc8We2bvsp9WW12'))
entity_type = DATASETS
init_repository(entity_type, self)
add_file(self, entity_type, '--bumpversion', 'new', file_content='0')
metadata_path = os.path.join(self.tmp_dir, ML_GIT_DIR, entity_type,
'metadata')
self.assertIn(
output_messages['INFO_COMMIT_REPO'] %
(metadata_path, entity_type + '-ex'),
check_output(MLGIT_COMMIT %
(entity_type, entity_type + '-ex', '')))
head_file = os.path.join(self.tmp_dir, ML_GIT_DIR, entity_type, 'refs',
entity_type + '-ex', 'HEAD')
self.assertTrue(os.path.exists(head_file))
self._remote_del(entity_type)
self.assertIn(
output_messages['ERROR_REMOTE_NOT_FOUND'],
check_output(MLGIT_PUSH % (entity_type, entity_type + '-ex')))
self.assertFalse(
os.path.exists(
os.path.join(
MINIO_BUCKET_PATH,
'zdj7WWjGAAJ8gdky5FKcVLfd63aiRUGb8fkc8We2bvsp9WW12')))
def test_12_push_with_invalid_retry_number(self):
entity_type = DATASETS
init_repository(entity_type, self)
add_file(self, entity_type, '--bumpversion', 'new', file_content='0')
self.assertNotIn(ERROR_MESSAGE, check_output(MLGIT_COMMIT % (entity_type, DATASET_NAME, '')))
expected_error_message = '-2 is not in the valid range of 0 to 99999999.'
self.assertIn(expected_error_message, check_output(MLGIT_PUSH % (entity_type, entity_type+'-ex --retry=-2')))
def test_04_list_tags_without_similar_tags(self):
self._list_tag_entity('dataset')
entity_type = 'dataset'
similar_entity = 'dataset-ex2'
workspace = os.path.join('dataset', similar_entity)
os.makedirs(workspace, exist_ok=True)
create_spec(self,
'dataset',
self.tmp_dir,
artifact_name=similar_entity)
add_file(self,
'dataset',
'--bumpversion',
'new',
artifact_name=similar_entity)
self.assertIn(
messages[17] %
(os.path.join(self.tmp_dir, ML_GIT_DIR, 'dataset', 'metadata'),
os.path.join('computer-vision', 'images', similar_entity)),
check_output(MLGIT_COMMIT % ('dataset', similar_entity, '')))
check_output(MLGIT_PUSH % ('dataset', similar_entity))
self.assertNotIn(
similar_entity,
check_output(MLGIT_TAG_LIST % (entity_type, entity_type + '-ex')))
self.assertIn(
similar_entity,
check_output(MLGIT_TAG_LIST % (entity_type, similar_entity)))
def test_04_fsck_with_full_option(self):
entity = DATASETS
init_repository(entity, self)
add_file(self, entity, '', 'new', file_content='2')
self.assertIn(
output_messages['INFO_SUMMARY_FSCK_FILES'].format(
'corrupted', 0, ''), check_output(MLGIT_FSCK % entity))
with open(
os.path.join(
self.tmp_dir, ML_GIT_DIR, entity, 'objects', 'hashfs',
'dr', 'vG',
'zdj7WdrvGPx9s8wmSB6KJGCmfCRNDQX6i8kVfFenQbWDQ1pmd'),
'wt') as file:
file.write('corrupting file')
output = check_output((MLGIT_FSCK % entity) + ' --full')
self.assertIn(
output_messages['INFO_SUMMARY_FSCK_FILES'].format(
'corrupted', 1,
'[\'zdj7WdrvGPx9s8wmSB6KJGCmfCRNDQX6i8kVfFenQbWDQ1pmd\']'),
output)
self.assertIn(
output_messages['INFO_SUMMARY_FSCK_FILES'].format(
'missing', 0, '[]'), output)
self.assertIn('zdj7WdrvGPx9s8wmSB6KJGCmfCRNDQX6i8kVfFenQbWDQ1pmd',
output)
def test_15_add_and_edit_file_with_same_hash(self):
entity_name = '{}-ex'.format(DATASETS)
init_repository(DATASETS, self, mutability=MUTABLE)
add_file(self, DATASETS, '')
self.assertNotIn(
ERROR_MESSAGE,
check_output(MLGIT_COMMIT % (DATASETS, entity_name, '')))
self.assertNotIn(ERROR_MESSAGE,
check_output(MLGIT_PUSH % (DATASETS, entity_name)))
data_folder = os.path.join(self.tmp_dir, DATASETS, DATASET_NAME)
file = os.path.join(data_folder, 'file1')
copyfile(file, os.path.join(data_folder, 'file1 - Copy'))
with open(file, 'wt') as f:
f.write('Modified file.')
self.assertNotIn(
ERROR_MESSAGE,
check_output(MLGIT_ADD %
(DATASETS, entity_name, ' --bumpversion')))
self.assertNotIn(
ERROR_MESSAGE,
check_output(MLGIT_COMMIT % (DATASETS, entity_name, '')))
self.assertNotIn(ERROR_MESSAGE,
check_output(MLGIT_PUSH % (DATASETS, entity_name)))
index = os.path.join(ML_GIT_DIR, DATASETS, 'index', 'metadata',
DATASET_NAME, 'INDEX.yaml')
self._check_index(index, ['file1', 'file1 - Copy'], [])
def test_18_recover_corrupted_file_added(self):
entity_init(DATASETS, self)
add_file(self, DATASETS, '--bumpversion', 'new')
corrupted_file = os.path.join(self.tmp_dir, DATASETS, DATASET_NAME,
'newfile0')
with open(corrupted_file) as f:
content = f.read()
os.chmod(corrupted_file, S_IWUSR | S_IREAD)
with open(corrupted_file, 'wb') as z:
z.write(b'0' * 0)
command_output = check_output(
MLGIT_ADD % (DATASETS, DATASET_NAME, '--bumpversion'))
self.assertIn(output_messages['WARN_CORRUPTED_CANNOT_BE_ADD'],
command_output)
self.assertIn('newfile0', command_output)
os.chmod(corrupted_file, S_IWUSR | S_IREAD)
with open(corrupted_file, 'w') as z:
z.write(content)
command_output = check_output(
MLGIT_ADD % (DATASETS, DATASET_NAME, '--bumpversion'))
self.assertNotIn(output_messages['WARN_CORRUPTED_CANNOT_BE_ADD'],
command_output)
self.assertNotIn('newfile0', command_output)
def test_08_push_without_default_config(self):
entity_type = 'dataset'
artifact_name = 'dataset-ex'
init_repository(entity_type, self, profile=None)
add_file(self, entity_type, '--bumpversion', 'new')
self.assertNotIn(ERROR_MESSAGE, check_output(MLGIT_COMMIT % (entity_type, artifact_name, '')))
self.assertIn('Unable to locate credentials', check_output(MLGIT_PUSH % (entity_type, artifact_name)))
def _commit_entity(self, entity_type):
entity_init(entity_type, self)
add_file(self, entity_type, '--bumpversion', 'new')
self.assertIn(output_messages['INFO_COMMIT_REPO'] % (os.path.join(self.tmp_dir, ML_GIT_DIR, entity_type, 'metadata'), entity_type + '-ex'),
check_output(MLGIT_COMMIT % (entity_type, entity_type + '-ex', '')))
HEAD = os.path.join(self.tmp_dir, ML_GIT_DIR, entity_type, 'refs', entity_type + '-ex', 'HEAD')
self.assertTrue(os.path.exists(HEAD))
def test_05_push_with_wrong_credentials_profile(self):
entity_type = 'dataset'
artifact_name = 'dataset-ex'
init_repository(entity_type, self, profile='personal2')
add_file(self, entity_type, '--bumpversion', 'new')
self.assertNotIn(ERROR_MESSAGE, check_output(MLGIT_COMMIT % (entity_type, artifact_name, '')))
self.assertIn(output_messages['ERROR_AWS_KEY_NOT_EXIST'], check_output(MLGIT_PUSH % (entity_type, artifact_name)))
def _add_tag_entity(self, entity_type):
self.set_up_tag(entity_type)
add_file(self, entity_type, '--bumpversion', 'new')
self.assertIn(
output_messages['INFO_COMMIT_REPO'] %
(os.path.join(self.tmp_dir, ML_GIT_DIR, entity_type,
'metadata'), entity_type + '-ex'),
check_output(MLGIT_COMMIT %
(entity_type, entity_type + '-ex', '')))
self.assertNotIn(
ERROR_MESSAGE,
check_output(MLGIT_PUSH % (entity_type, entity_type + '-ex')))
with open(os.path.join(entity_type, entity_type + '-ex', 'file1'),
'wb') as z:
z.write(b'0' * 1024)
self.assertIn(
output_messages['INFO_CREATE_TAG_SUCCESS'],
check_output(MLGIT_TAG_ADD %
(entity_type, entity_type + '-ex', 'test-tag')))
self.assertNotIn(
ERROR_MESSAGE,
check_output(MLGIT_PUSH % (entity_type, entity_type + '-ex')))
def export(self, repotype, entity):
init_repository(repotype, self)
add_file(self, repotype, '', repotype)
file_in_store = os.path.join(PATH_TEST, 'data', 'mlgit',
repotype + 'file0')
self.assertIn(
messages[17] %
(os.path.join(self.tmp_dir, ML_GIT_DIR, repotype, 'metadata'),
os.path.join('computer-vision', 'images', entity)),
check_output(MLGIT_COMMIT % (repotype, entity, '')))
self.assertFalse(os.path.exists(file_in_store))
check_output(MLGIT_PUSH % (repotype, entity))
store = 's3h://mlgit'
tag = 'computer-vision__images__%s__1' % entity
self.assertIn(
messages[66] % (tag, store, 's3://mlgit'),
check_output(
MLGIT_EXPORT %
(repotype,
' {} {} --credentials={} --endpoint=http://127.0.0.1:9000'.
format(tag, BUCKET_NAME, PROFILE))))
self.assertTrue(os.path.exists(file_in_store))
def _create_new_tag(self, entity, file_name):
add_file(self, entity, '', file_name)
self.assertNotIn(
ERROR_MESSAGE,
check_output(MLGIT_COMMIT %
(entity, entity + '-ex', '--version=2')))
self.assertNotIn(ERROR_MESSAGE,
check_output(MLGIT_PUSH % (entity, entity + '-ex')))
def set_up_push_without_profile(self,
entity_type=DATASETS,
artifact_name=DATASET_NAME):
init_repository(entity_type, self, profile=None)
add_file(self, entity_type, '--bumpversion', 'new')
self.assertNotIn(
ERROR_MESSAGE,
check_output(MLGIT_COMMIT % (entity_type, artifact_name, '')))
def test_05_commit_with_wizard_enabled(self):
entity_type = LABELS
entity_init(entity_type, self)
add_file(self, entity_type, '--bumpversion', 'new')
runner = CliRunner()
result = runner.invoke(entity.labels, ['commit', entity_type + '-ex', '--wizard'], input='\n'.join(['', 'message']))
self.assertIn(prompt_msg.COMMIT_VERSION.format('labels', '1'), result.output)
self.assertIn(prompt_msg.COMMIT_MESSAGE, result.output)
def test_12_commit_with_empty_related_entity_name(self):
entity_type = MODELS
entity_name = entity_type + '-ex'
entity_init(entity_type, self)
add_file(self, entity_type, '--bumpversion', 'new')
self.assertIn(output_messages['ERROR_INVALID_VALUE_FOR'] % ('--labels', 'Value cannot be empty'),
check_output(MLGIT_COMMIT % (entity_type, entity_name, ' --labels=')))
HEAD = os.path.join(self.tmp_dir, ML_GIT_DIR, entity_type, 'refs', entity_name, 'HEAD')
self.assertFalse(os.path.exists(HEAD))
def test_03_fsck_corrupted_file_in_workspace(self):
entity = DATASETS
init_repository(entity, self)
add_file(self, entity, '', 'new', file_content='2')
corrupted_file = os.path.join(self.tmp_dir, entity, DATASET_NAME,
'newfile4')
with open(corrupted_file) as f:
original_content = f.read()
self.assertNotIn(
ERROR_MESSAGE,
check_output(MLGIT_COMMIT % (entity, entity + '-ex', '')))
self.assertNotIn(ERROR_MESSAGE,
check_output(MLGIT_PUSH % (entity, entity + '-ex')))
fsck_output = check_output(MLGIT_FSCK % entity)
self.assertIn(
output_messages['INFO_SUMMARY_FSCK_FILES'].format(
'corrupted', 0, ''), fsck_output)
self.assertIn(
output_messages['INFO_SUMMARY_FSCK_FILES'].format(
'missing', 0, ''), fsck_output)
self.assertIn(output_messages['INFO_FSCK_FIXED_FILES'].format(0, ''),
fsck_output)
os.chmod(corrupted_file, S_IWUSR | S_IREAD)
corrupted_content = b'0' * 0
with open(corrupted_file, 'wb') as z:
z.write(corrupted_content)
fsck_output = check_output(MLGIT_FSCK % entity)
self.assertIn(
output_messages['INFO_SUMMARY_FSCK_FILES'].format(
'corrupted', 1, ''), fsck_output)
self.assertIn(
output_messages['INFO_SUMMARY_FSCK_FILES'].format(
'missing', 0, ''), fsck_output)
self.assertIn(output_messages['INFO_FSCK_FIXED_FILES'].format(0, ''),
fsck_output)
with open(corrupted_file) as f:
content = f.read()
self.assertEquals(content, '')
fsck_output = check_output(MLGIT_FSCK % (entity) + ' --fix-workspace')
self.assertIn(
output_messages['INFO_SUMMARY_FSCK_FILES'].format(
'corrupted', 1, ''), fsck_output)
self.assertIn(
output_messages['INFO_SUMMARY_FSCK_FILES'].format(
'missing', 0, ''), fsck_output)
self.assertIn(output_messages['INFO_FSCK_FIXED_FILES'].format(1, ''),
fsck_output)
with open(corrupted_file) as f:
content = f.read()
self.assertEquals(content, original_content)
def test_11_push_with_wrong_minio_endpoint(self):
entity_type = DATASETS
artifact_name = DATASET_NAME
init_repository(entity_type, self)
add_file(self, entity_type, '--bumpversion', 'new')
self.assertNotIn(ERROR_MESSAGE, check_output(MLGIT_COMMIT % (entity_type, artifact_name, '')))
output = check_output(MLGIT_PUSH % (entity_type, artifact_name))
self.assertIn(ERROR_MESSAGE, output)
self.assertIn('There was an error checking if bucket \'{}\' exists.'.format(BUCKET_NAME), output)
def test_13_commit_with_invalid_related_entity_name(self):
entity_type = MODELS
entity_name = entity_type + '-ex'
entity_init(entity_type, self)
add_file(self, entity_type, '--bumpversion', 'new')
self.assertIn(output_messages['ERROR_ENTITY_NOT_FIND'].format('wrong-entity'),
check_output(MLGIT_COMMIT % (entity_type, entity_name, ' --labels=wrong-entity')))
HEAD = os.path.join(self.tmp_dir, ML_GIT_DIR, entity_type, 'refs', entity_name, 'HEAD')
self.assertFalse(os.path.exists(HEAD))
def test_19_commit_files_to_labels_with_wizard_enabled(self):
entity_type = LABELS
entity_init(entity_type, self)
add_file(self, entity_type, '--bumpversion', 'new')
runner = CliRunner()
result = runner.invoke(entity.labels, ['commit', entity_type + '-ex', '--wizard'], input='\n'.join(['', 'message', 'y', ' ', 'dataset-ex']))
self.assertIn(prompt_msg.COMMIT_VERSION.format('labels', '1'), result.output)
self.assertIn(prompt_msg.COMMIT_MESSAGE, result.output)
self.assertIn(prompt_msg.WANT_LINK_TO_LABEL_ENTITY.format(parse_entity_type_to_singular(DATASETS)), result.output)
self.assertIn(prompt_msg.DEFINE_LINKED_DATASET, result.output)
def set_up_checkout(self, entity):
init_repository(entity, self)
add_file(self, entity, '', 'new')
metadata_path = os.path.join(self.tmp_dir, ML_GIT_DIR, entity, 'metadata')
self.assertIn(output_messages['INFO_COMMIT_REPO'] % (metadata_path, os.path.join('computer-vision', 'images', entity + '-ex')),
check_output(MLGIT_COMMIT % (entity, entity + '-ex', '')))
head_path = os.path.join(self.tmp_dir, ML_GIT_DIR, entity, 'refs', entity + '-ex', 'HEAD')
self.assertTrue(os.path.exists(head_path))
self.assertNotIn(ERROR_MESSAGE, check_output(MLGIT_PUSH % (entity, entity + '-ex')))
self._clear_workspace(entity)
def test_03_checkout(self):
os.makedirs(self.workspace)
create_spec(self,
self.repo_type,
self.tmp_dir,
version=1,
mutability='strict',
store_type=self.store_type)
self.assertIn(messages[0], check_output(MLGIT_INIT))
self.assertIn(
messages[2] % (GIT_PATH, self.repo_type),
check_output(MLGIT_REMOTE_ADD % (self.repo_type, GIT_PATH)))
self.assertIn(
messages[87] % (self.store_type, self.bucket),
check_output('ml-git repository store add %s --type=%s' %
(self.bucket, self.store_type)))
self.assertNotIn(ERROR_MESSAGE,
check_output(MLGIT_ENTITY_INIT % 'dataset'))
add_file(self, self.repo_type, '', 'new')
metadata_path = os.path.join(ML_GIT_DIR, 'dataset', 'metadata')
self.assertIn(
messages[17] %
(os.path.join(self.tmp_dir, metadata_path),
os.path.join('computer-vision', 'images', 'dataset-ex')),
check_output(MLGIT_COMMIT % (self.repo_type, 'dataset-ex', '')))
HEAD = os.path.join(ML_GIT_DIR, 'dataset', 'refs', 'dataset-ex',
'HEAD')
self.assertTrue(os.path.exists(HEAD))
self.assertEqual(os.getenv('AZURE_STORAGE_CONNECTION_STRING'),
self.dev_store_account_)
self.assertNotIn(
ERROR_MESSAGE,
check_output(MLGIT_PUSH % (self.repo_type, 'dataset-ex')))
clear(self.workspace)
clear(os.path.join(ML_GIT_DIR, 'dataset'))
self.assertNotIn(ERROR_MESSAGE,
check_output(MLGIT_ENTITY_INIT % self.repo_type))
self.assertEqual(os.getenv('AZURE_STORAGE_CONNECTION_STRING'),
self.dev_store_account_)
self.assertNotIn(
ERROR_MESSAGE,
check_output(
MLGIT_CHECKOUT %
(self.repo_type, 'computer-vision__images__dataset-ex__1')))
ws_path = os.path.join(self.tmp_dir, 'dataset', 'computer-vision',
'images', 'dataset-ex')
self.assertTrue(os.path.isfile(os.path.join(ws_path, 'newfile0')))
self.assertTrue(os.path.isfile(os.path.join(ws_path, 'newfile1')))
self.assertTrue(os.path.isfile(os.path.join(ws_path, 'newfile2')))
self.assertTrue(os.path.isfile(os.path.join(ws_path, 'newfile3')))
self.assertTrue(os.path.isfile(os.path.join(ws_path, 'newfile4')))
def test_06_add_command_with_corrupted_file_added(self):
entity_init('dataset', self)
add_file(self, 'dataset', '--bumpversion', 'new')
corrupted_file = os.path.join(self.tmp_dir, 'dataset', 'dataset-ex', 'newfile0')
os.chmod(corrupted_file, S_IWUSR | S_IREAD)
with open(corrupted_file, 'wb') as z:
z.write(b'0' * 0)
self.assertIn(messages[67], check_output(MLGIT_ADD % ('dataset', 'dataset-ex', '--bumpversion')))
def test_10_push_without_commit(self):
entity_type = DATASETS
init_repository(entity_type, self)
add_file(self, entity_type, '--bumpversion', 'new', file_content='0')
output = check_output(MLGIT_PUSH % (entity_type, DATASET_NAME))
self.assertIn(output_messages['INFO_NO_BLOBS_TO_PUSH'], output)
self.assertIn(output_messages['ERROR_COMMIT_BEFORE_PUSH'], output)
self.assertNotIn('did not match any file(s) known to git', output)
metadata_path = os.path.join(self.tmp_dir, ML_GIT_DIR, entity_type, 'metadata')
os.chdir(metadata_path)
self.assertNotIn('computer-vision__images__' + entity_type + '-ex__2', check_output('git describe --tags'))