def test_increment_version_in_dataset_spec(self):
dataset = 'test_dataset'
dir1 = get_spec_file_dir(dataset)
dir2 = os.path.join('.ml-git', DATASETS, 'index', 'metadata',
dataset) # Linked path to the original
os.makedirs(os.path.join(self.tmp_dir, dir1))
os.makedirs(os.path.join(self.tmp_dir, dir2))
file1 = os.path.join(self.tmp_dir, dir1, '%s.spec' % dataset)
file2 = os.path.join(self.tmp_dir, dir2, '%s.spec' % dataset)
self.assertFalse(increment_version_in_spec(None))
self.assertFalse(
increment_version_in_spec(os.path.join(get_root_path(), dataset)))
spec = yaml_load(os.path.join(testdir, 'invalid2.spec'))
yaml_save(spec, file1)
self.assertFalse(
increment_version_in_spec(os.path.join(get_root_path(), dataset)))
spec = yaml_load(os.path.join(testdir, 'valid.spec'))
yaml_save(spec, file1)
os.link(file1, file2)
self.assertTrue(
increment_version_in_spec(
os.path.join(get_root_path(), self.tmp_dir, DATASETS, dataset,
dataset + '.spec')))
def test_set_version_in_spec(self):
tmpfile = os.path.join(self.tmp_dir, 'sample.spec')
file = os.path.join(testdir, 'sample.spec')
spec_hash = yaml_load(file)
yaml_save(spec_hash, tmpfile)
set_version_in_spec(3, tmpfile, DATASETS)
spec_hash = yaml_load(tmpfile)
self.assertEqual(spec_hash[DATASET_SPEC_KEY]['version'], 3)
def test_set_version_in_spec(self):
tmpfile = os.path.join(self.tmp_dir, 'sample.spec')
file = os.path.join(testdir, 'sample.spec')
spec_hash = yaml_load(file)
yaml_save(spec_hash, tmpfile)
set_version_in_spec(3, tmpfile, 'dataset')
spec_hash = yaml_load(tmpfile)
self.assertEqual(spec_hash['dataset']['version'], 3)
def test_incr_version(self):
tmpfile = os.path.join(self.tmp_dir, 'sample.spec')
file = os.path.join(testdir, 'sample.spec')
spec_hash = yaml_load(file)
yaml_save(spec_hash, tmpfile)
version = spec_hash['dataset']['version']
incr_version(tmpfile)
incremented_hash = yaml_load(tmpfile)
self.assertEqual(incremented_hash['dataset']['version'], version + 1)
incr_version('non-existent-file')
def test_update_store_spec(self):
spec_path = os.path.join(
os.getcwd(), os.sep.join([DATASETS, 'dataex', 'dataex.spec']))
update_storage_spec(DATASETS, 'dataex', S3H, 'fakestorage')
spec1 = yaml_load(spec_path)
self.assertEqual(spec1[DATASET_SPEC_KEY]['manifest'][STORAGE_SPEC_KEY],
's3h://fakestorage')
update_storage_spec(DATASETS, 'dataex', S3H, 'some-bucket-name')
spec2 = yaml_load(spec_path)
self.assertEqual(spec2[DATASET_SPEC_KEY]['manifest'][STORAGE_SPEC_KEY],
's3h://some-bucket-name')
def test_update_store_spec(self):
spec_path = os.path.join(
os.getcwd(), os.sep.join(['dataset', 'dataex', 'dataex.spec']))
update_store_spec('dataset', 'dataex', 's3h', 'fakestore')
spec1 = yaml_load(spec_path)
self.assertEqual(spec1['dataset']['manifest']['store'],
's3h://fakestore')
update_store_spec('dataset', 'dataex', 's3h', 'some-bucket-name')
spec2 = yaml_load(spec_path)
self.assertEqual(spec2['dataset']['manifest']['store'],
's3h://some-bucket-name')
def test_is_valid_version(self):
self.assertFalse(is_valid_version(None))
self.assertFalse(is_valid_version({}))
file = os.path.join(testdir, 'valid.spec')
spec_hash = yaml_load(file)
self.assertTrue(is_valid_version(spec_hash))
file = os.path.join(testdir, 'invalid1.spec')
spec_hash = yaml_load(file)
self.assertFalse(is_valid_version(spec_hash))
file = os.path.join(testdir, 'invalid2.spec')
spec_hash = yaml_load(file)
self.assertFalse(is_valid_version(spec_hash))
file = os.path.join(testdir, 'invalid3.spec')
spec_hash = yaml_load(file)
self.assertFalse(is_valid_version(spec_hash))
def test_validate_bucket_name(self):
config = yaml_load(os.path.join(os.getcwd(), '.ml-git', 'config.yaml'))
spec_with_wrong_bucket_name = yaml_load(
os.path.join(testdir, 'invalid4.spec'))
self.assertFalse(
validate_bucket_name(spec_with_wrong_bucket_name[DATASET_SPEC_KEY],
config))
spec_bucket_name_not_in_config = yaml_load(
os.path.join(testdir, 'valid.spec'))
self.assertFalse(
validate_bucket_name(
spec_bucket_name_not_in_config[DATASET_SPEC_KEY], config))
storage = config[STORAGE_CONFIG_KEY][S3]
del config[STORAGE_CONFIG_KEY][S3]
config[STORAGE_CONFIG_KEY][S3H] = storage
spec_with_bucket_in_config = yaml_load(
os.path.join(testdir, 'valid2.spec'))
self.assertTrue(
validate_bucket_name(spec_with_bucket_in_config[DATASET_SPEC_KEY],
config))
def test_update_storage_spec_with_entity_dir(self):
entity_name = 'dataex'
entity_type = EntityType.DATASETS.value
dataset_without_dir_path = os.path.join(
os.getcwd(), os.sep.join([entity_type, entity_name]))
dataset_path = os.path.join(
os.getcwd(), os.sep.join([entity_type, 'folderA', 'folderB']))
ensure_path_exists(dataset_path)
shutil.move(dataset_without_dir_path, dataset_path)
spec_path = os.path.join(dataset_path, entity_name,
entity_name + SPEC_EXTENSION)
entity_dir = os.sep.join(['folderA', 'folderB'])
update_storage_spec(entity_type, entity_name, StorageType.S3H.value,
'fakestorage', entity_dir)
spec = yaml_load(spec_path)
self.assertEqual(spec[DATASET_SPEC_KEY]['manifest'][STORAGE_SPEC_KEY],
's3h://fakestorage')
update_storage_spec(entity_type, entity_name, StorageType.S3H.value,
'some-bucket-name', entity_dir)
spec = yaml_load(spec_path)
self.assertEqual(spec[DATASET_SPEC_KEY]['manifest'][STORAGE_SPEC_KEY],
's3h://some-bucket-name')
def test_validate_bucket_name(self):
repo_type = 'dataset'
config = yaml_load(os.path.join(os.getcwd(), '.ml-git', 'config.yaml'))
spec_with_wrong_bucket_name = yaml_load(
os.path.join(testdir, 'invalid4.spec'))
self.assertFalse(
validate_bucket_name(spec_with_wrong_bucket_name[repo_type],
config))
spec_bucket_name_not_in_config = yaml_load(
os.path.join(testdir, 'valid.spec'))
self.assertFalse(
validate_bucket_name(spec_bucket_name_not_in_config[repo_type],
config))
store = config['store']['s3']
del config['store']['s3']
config['store']['s3h'] = store
spec_with_bucket_in_config = yaml_load(
os.path.join(testdir, 'valid2.spec'))
self.assertTrue(
validate_bucket_name(spec_with_bucket_in_config[repo_type],
config))