def commit(self,
spec,
specs,
version_number=None,
run_fsck=False,
msg=None):
# Move chunks from index to .ml-git/objects
repo_type = self.__repo_type
try:
index_path = get_index_path(self.__config, repo_type)
objects_path = get_objects_path(self.__config, repo_type)
metadata_path = get_metadata_path(self.__config, repo_type)
refs_path = get_refs_path(self.__config, repo_type)
repo = LocalRepository(self.__config, objects_path, repo_type)
mutability, check_mutability = repo.get_mutability_from_spec(
spec, repo_type)
if not mutability:
return
if not check_mutability:
log.error('Spec mutability cannot be changed.',
class_name=REPOSITORY_CLASS_NAME)
return
except Exception as e:
log.error(e, class_name=REPOSITORY_CLASS_NAME)
return
ref = Refs(refs_path, spec, repo_type)
tag, sha = ref.branch()
categories_path = get_path_with_categories(tag)
manifest_path = os.path.join(metadata_path, categories_path, spec,
'MANIFEST.yaml')
path, file = None, None
try:
path, file = search_spec_file(self.__repo_type, spec,
categories_path)
except Exception as e:
log.error(e, class_name=REPOSITORY_CLASS_NAME)
if path is None:
return None, None, None
spec_path = os.path.join(path, file)
idx = MultihashIndex(spec, index_path, objects_path)
if version_number:
set_version_in_spec(version_number, spec_path, self.__repo_type)
idx.add_metadata(path, file)
# Check tag before anything to avoid creating unstable state
log.debug('Check if tag already exists',
class_name=REPOSITORY_CLASS_NAME)
m = Metadata(spec, metadata_path, self.__config, repo_type)
if not m.check_exists():
log.error('The %s has not been initialized' % self.__repo_type,
class_name=REPOSITORY_CLASS_NAME)
return
full_metadata_path, categories_sub_path, metadata = m.tag_exists(
index_path)
if metadata is None:
return None
log.debug('%s -> %s' % (index_path, objects_path),
class_name=REPOSITORY_CLASS_NAME)
# commit objects in index to ml-git objects
o = Objects(spec, objects_path)
changed_files, deleted_files = o.commit_index(index_path, path)
bare_mode = os.path.exists(
os.path.join(index_path, 'metadata', spec, 'bare'))
if not bare_mode and len(deleted_files) > 0:
self.remove_deleted_files(idx, index_path, m, manifest_path, path,
spec, deleted_files)
elif bare_mode:
tag, _ = ref.branch()
self._checkout_ref(tag)
# update metadata spec & README.md
# option --dataset-spec --labels-spec
tag, sha = m.commit_metadata(index_path, specs, msg, changed_files,
mutability, path)
# update ml-git ref spec HEAD == to new SHA-1 / tag
if tag is None:
return None
ref = Refs(refs_path, spec, repo_type)
ref.update_head(tag, sha)
# Run file check
if run_fsck:
self.fsck()
return tag
def tag(self, spec, usr_tag):
repo_type = self.__repo_type
try:
metadata_path = get_metadata_path(self.__config, repo_type)
refs_path = get_refs_path(self.__config, repo_type)
r = Refs(refs_path, spec, repo_type)
curtag, sha = r.head()
except Exception as e:
log.error(e, class_name=REPOSITORY_CLASS_NAME)
return False
if curtag is None:
log.error('No current tag for [%s]. commit first.' % spec,
class_name=REPOSITORY_CLASS_NAME)
return False
utag = UsrTag(curtag, usr_tag)
# Check if usrtag exists before creating it
log.debug('Check if tag [%s] already exists' % utag,
class_name=REPOSITORY_CLASS_NAME)
m = Metadata(spec, metadata_path, self.__config, repo_type)
if m._usrtag_exists(utag) is True:
log.error('Tag [%s] already exists.' % utag,
class_name=REPOSITORY_CLASS_NAME)
return False
# ensure metadata repository is at the current tag/sha version
m = Metadata('', metadata_path, self.__config, repo_type)
m.checkout(curtag)
# TODO: format to something that could be used for a checkout:
# format: _._user_.._ + curtag + _.._ + usrtag
# at checkout with usrtag look for pattern _._ then find usrtag in the list (split on '_.._')
# adds usrtag to the metadata repository
m = Metadata(spec, metadata_path, self.__config, repo_type)
try:
m.tag_add(utag)
except Exception as e:
match = re.search("stderr: 'fatal:(.*)'$", e.stderr)
err = match.group(1)
log.error(err, class_name=REPOSITORY_CLASS_NAME)
return
log.info('Create Tag Successfull', class_name=REPOSITORY_CLASS_NAME)
# checkout at metadata repository at master version
m.checkout('master')
return True
def clone_config(self, url, folder=None, track=False):
if clone_config_repository(url, folder, track):
self.__config = config_load()
m = Metadata('', get_metadata_path(self.__config), self.__config)
m.clone_config_repo()
def test_metadata_tag(self):
m = Metadata(spec, index_path, config, DATASETS)
tag = m.metadata_tag(metadata_config)
self.assertEqual(tag, 'images__dataset_ex__1')
def test_init(self):
m = Metadata(spec, self.test_dir, config, DATASETS)
m.init()
self.assertTrue(m.check_exists())
clear(m.path)
def test_blank_remote_url(self):
config_cp = deepcopy(config)
config_cp[DATASETS]['git'] = ''
m = Metadata(spec, self.test_dir, config_cp, DATASETS)
self.assertRaises(GitError, m.validate_blank_remote_url)
clear(m.path)
def test_clone_config_repo(self):
m = Metadata('', self.test_dir, config, DATASETS)
m.clone_config_repo()
self.assertTrue(m.check_exists())
