def _add_associate_entity_metadata(self, metadata, specs):
if 'dataset' in specs and self.__repo_type in ['labels', 'model']:
d_spec = specs['dataset']
refs_path = get_refs_path(self.__config, 'dataset')
r = Refs(refs_path, d_spec, 'dataset')
tag, sha = r.head()
if tag is not None:
log.info('Associate dataset [%s]-[%s] to the %s.' %
(d_spec, tag, self.__repo_type),
class_name=LOCAL_REPOSITORY_CLASS_NAME)
metadata[self.__repo_type]['dataset'] = {}
metadata[self.__repo_type]['dataset']['tag'] = tag
metadata[self.__repo_type]['dataset']['sha'] = sha
if 'labels' in specs and self.__repo_type in ['model']:
l_spec = specs['labels']
refs_path = get_refs_path(self.__config, 'labels')
r = Refs(refs_path, l_spec, 'labels')
tag, sha = r.head()
if tag is not None:
log.info('Associate labels [%s]-[%s] to the %s.' %
(l_spec, tag, self.__repo_type),
class_name=LOCAL_REPOSITORY_CLASS_NAME)
metadata[self.__repo_type]['labels'] = {}
metadata[self.__repo_type]['labels']['tag'] = tag
metadata[self.__repo_type]['labels']['sha'] = sha
def _add_associate_entity_metadata(self, metadata, specs):
dataset = EntityType.DATASETS.value
labels = EntityType.LABELS.value
model = EntityType.MODELS.value
entity_spec_key = get_spec_key(self.__repo_type)
if dataset in specs and self.__repo_type in [labels, model]:
d_spec = specs[dataset]
refs_path = get_refs_path(self.__config, dataset)
r = Refs(refs_path, d_spec, dataset)
tag, sha = r.head()
if tag is not None:
log.info(output_messages['INFO_ASSOCIATE_DATASETS'] %
(d_spec, tag, self.__repo_type),
class_name=LOCAL_REPOSITORY_CLASS_NAME)
metadata[entity_spec_key][DATASET_SPEC_KEY] = {}
metadata[entity_spec_key][DATASET_SPEC_KEY]['tag'] = tag
metadata[entity_spec_key][DATASET_SPEC_KEY]['sha'] = sha
if labels in specs and self.__repo_type in [model]:
l_spec = specs[labels]
refs_path = get_refs_path(self.__config, labels)
r = Refs(refs_path, l_spec, labels)
tag, sha = r.head()
if tag is not None:
log.info('Associate labels [%s]-[%s] to the %s.' %
(l_spec, tag, self.__repo_type),
class_name=LOCAL_REPOSITORY_CLASS_NAME)
metadata[entity_spec_key][LABELS_SPEC_KEY] = {}
metadata[entity_spec_key][LABELS_SPEC_KEY]['tag'] = tag
metadata[entity_spec_key][LABELS_SPEC_KEY]['sha'] = sha
def test_head(self):
config = config_load()
spec_path = 'dataset-ex'
ml_dir = os.path.join(self.tmp_dir, config['mlgit_path'])
os.mkdir(ml_dir)
refs_dir = os.path.join(ml_dir, 'dataset', 'refs')
refs = Refs(refs_dir, spec_path)
sha = 'b569b7e4cd82206b451315123669057ef5f1ac3b'
tag = 'images__dataset_ex__1'
refs.update_head(tag, sha)
head = os.path.join(refs_dir, spec_path, 'HEAD')
self.assertEqual((tag, sha), refs.head())
os.remove(head)
self.assertEqual((None, None), refs.head())
def check_related_entities_tags(self, tags):
dataset = EntityType.DATASETS.value
labels = EntityType.LABELS.value
if dataset in tags:
d_spec = tags[dataset]
refs_path = get_refs_path(self.__config, dataset)
r = Refs(refs_path, d_spec, dataset)
tag, sha = r.head()
if tag is None:
log.error(
output_messages['ERROR_ENTITY_NOT_FIND'].format(d_spec))
return False
if labels in tags:
l_spec = tags[labels]
refs_path = get_refs_path(self.__config, labels)
r = Refs(refs_path, l_spec, labels)
tag, sha = r.head()
if tag is None:
log.error(
output_messages['ERROR_ENTITY_NOT_FIND'].format(l_spec))
return False
return True
def tag(self, spec, usr_tag):
repo_type = self.__repo_type
try:
metadata_path = get_metadata_path(self.__config, repo_type)
refs_path = get_refs_path(self.__config, repo_type)
r = Refs(refs_path, spec, repo_type)
curtag, sha = r.head()
except Exception as e:
log.error(e, class_name=REPOSITORY_CLASS_NAME)
return False
if curtag is None:
log.error('No current tag for [%s]. commit first.' % spec,
class_name=REPOSITORY_CLASS_NAME)
return False
utag = UsrTag(curtag, usr_tag)
# Check if usrtag exists before creating it
log.debug('Check if tag [%s] already exists' % utag,
class_name=REPOSITORY_CLASS_NAME)
m = Metadata(spec, metadata_path, self.__config, repo_type)
if m._usrtag_exists(utag) is True:
log.error('Tag [%s] already exists.' % utag,
class_name=REPOSITORY_CLASS_NAME)
return False
# ensure metadata repository is at the current tag/sha version
m = Metadata('', metadata_path, self.__config, repo_type)
m.checkout(curtag)
# TODO: format to something that could be used for a checkout:
# format: _._user_.._ + curtag + _.._ + usrtag
# at checkout with usrtag look for pattern _._ then find usrtag in the list (split on '_.._')
# adds usrtag to the metadata repository
m = Metadata(spec, metadata_path, self.__config, repo_type)
try:
m.tag_add(utag)
except Exception as e:
match = re.search("stderr: 'fatal:(.*)'$", e.stderr)
err = match.group(1)
log.error(err, class_name=REPOSITORY_CLASS_NAME)
return
log.info('Create Tag Successfull', class_name=REPOSITORY_CLASS_NAME)
# checkout at metadata repository at master version
m.checkout()
return True
def show(self, spec):
repo_type = self.__repo_type
try:
metadata_path = get_metadata_path(self.__config, repo_type)
refs_path = get_refs_path(self.__config, repo_type)
except Exception as e:
log.error(e, class_name=REPOSITORY_CLASS_NAME)
return
r = Refs(refs_path, spec, repo_type)
tag, sha = r.head()
if tag is None:
log.info('No HEAD for [%s]' % spec,
class_name=LOCAL_REPOSITORY_CLASS_NAME)
return
m = Metadata('', metadata_path, self.__config, repo_type)
m.checkout(tag)
m.show(spec)
m.checkout()