def _checkout_ref(self, ref):
repo_type = self.__repo_type
metadata_path = get_metadata_path(self.__config, repo_type)
# checkout
m = Metadata('', metadata_path, self.__config, repo_type)
m.checkout(ref)
def _checkout_ref(self, ref=None):
repo_type = self.__repo_type
metadata_path = get_metadata_path(self.__config, repo_type)
m = Metadata('', metadata_path, self.__config, repo_type)
if ref is None:
ref = m.get_default_branch()
m.checkout(ref)
def push(self, spec, retry=2, clear_on_fail=False):
repo_type = self.__repo_type
try:
objects_path = get_objects_path(self.__config, repo_type)
metadata_path = get_metadata_path(self.__config, repo_type)
refs_path = get_refs_path(self.__config, repo_type)
except Exception as e:
log.error(e, class_name=REPOSITORY_CLASS_NAME)
return
met = Metadata(spec, metadata_path, self.__config, repo_type)
fields = met.git_user_config()
if None in fields.values():
log.error(
'Your name and email address need to be configured in git. '
'Please see the commands below:',
class_name=REPOSITORY_CLASS_NAME)
log.error('git config --global user.name \'Your Name\'',
class_name=REPOSITORY_CLASS_NAME)
log.error('git config --global user.email [email protected]',
class_name=REPOSITORY_CLASS_NAME)
return
if met.fetch() is False:
return
ref = Refs(refs_path, spec, repo_type)
tag, sha = ref.branch()
categories_path = get_path_with_categories(tag)
spec_path, spec_file = None, None
try:
spec_path, spec_file = search_spec_file(self.__repo_type, spec,
categories_path)
except Exception as e:
log.error(e, class_name=REPOSITORY_CLASS_NAME)
if spec_path is None:
return
full_spec_path = os.path.join(spec_path, spec_file)
repo = LocalRepository(self.__config, objects_path, repo_type)
ret = repo.push(objects_path, full_spec_path, retry, clear_on_fail)
# ensure first we're on master !
met.checkout()
if ret == 0:
# push metadata spec to LocalRepository git repository
try:
met.push()
except Exception as e:
log.error(e, class_name=REPOSITORY_CLASS_NAME)
return
MultihashFS(objects_path).reset_log()
def tag(self, spec, usr_tag):
repo_type = self.__repo_type
try:
metadata_path = get_metadata_path(self.__config, repo_type)
refs_path = get_refs_path(self.__config, repo_type)
r = Refs(refs_path, spec, repo_type)
curtag, sha = r.head()
except Exception as e:
log.error(e, class_name=REPOSITORY_CLASS_NAME)
return False
if curtag is None:
log.error('No current tag for [%s]. commit first.' % spec,
class_name=REPOSITORY_CLASS_NAME)
return False
utag = UsrTag(curtag, usr_tag)
# Check if usrtag exists before creating it
log.debug('Check if tag [%s] already exists' % utag,
class_name=REPOSITORY_CLASS_NAME)
m = Metadata(spec, metadata_path, self.__config, repo_type)
if m._usrtag_exists(utag) is True:
log.error('Tag [%s] already exists.' % utag,
class_name=REPOSITORY_CLASS_NAME)
return False
# ensure metadata repository is at the current tag/sha version
m = Metadata('', metadata_path, self.__config, repo_type)
m.checkout(curtag)
# TODO: format to something that could be used for a checkout:
# format: _._user_.._ + curtag + _.._ + usrtag
# at checkout with usrtag look for pattern _._ then find usrtag in the list (split on '_.._')
# adds usrtag to the metadata repository
m = Metadata(spec, metadata_path, self.__config, repo_type)
try:
m.tag_add(utag)
except Exception as e:
match = re.search("stderr: 'fatal:(.*)'$", e.stderr)
err = match.group(1)
log.error(err, class_name=REPOSITORY_CLASS_NAME)
return
log.info('Create Tag Successfull', class_name=REPOSITORY_CLASS_NAME)
# checkout at metadata repository at master version
m.checkout()
return True
def list(self):
repo_type = self.__repo_type
try:
metadata_path = get_metadata_path(self.__config, repo_type)
m = Metadata('', metadata_path, self.__config, repo_type)
if not m.check_exists():
raise RuntimeError('The %s doesn\'t have been initialized.' %
self.__repo_type)
m.checkout()
m.list(title='ML ' + repo_type)
except GitError as g:
error_message = g.stderr
if 'did not match any file(s) known' in error_message:
error_message = 'You don\'t have any entity being managed.'
log.error(error_message, class_name=REPOSITORY_CLASS_NAME)
return
except Exception as e:
log.error(e, class_name=REPOSITORY_CLASS_NAME)
return
def fetch_tag(self, tag, samples, retries=2):
repo_type = self.__repo_type
try:
objects_path = get_objects_path(self.__config, repo_type)
metadata_path = get_metadata_path(self.__config, repo_type)
m = Metadata('', metadata_path, self.__config, repo_type)
m.checkout(tag)
fetch_success = self._fetch(tag, samples, retries)
if not fetch_success:
objs = Objects('', objects_path)
objs.fsck(remove_corrupted=True)
m.checkout()
except Exception as e:
log.error(e, class_name=REPOSITORY_CLASS_NAME)
return
# restore to master/head
self._checkout_ref()
def show(self, spec):
repo_type = self.__repo_type
try:
metadata_path = get_metadata_path(self.__config, repo_type)
refs_path = get_refs_path(self.__config, repo_type)
except Exception as e:
log.error(e, class_name=REPOSITORY_CLASS_NAME)
return
r = Refs(refs_path, spec, repo_type)
tag, sha = r.head()
if tag is None:
log.info('No HEAD for [%s]' % spec,
class_name=LOCAL_REPOSITORY_CLASS_NAME)
return
m = Metadata('', metadata_path, self.__config, repo_type)
m.checkout(tag)
m.show(spec)
m.checkout()
def add(self, spec, file_path, bump_version=False, run_fsck=False):
repo_type = self.__repo_type
is_shared_objects = 'objects_path' in self.__config[repo_type]
is_shared_cache = 'cache_path' in self.__config[repo_type]
if not validate_config_spec_hash(self.__config):
log.error(
'.ml-git/config.yaml invalid. It should look something like this:\n%s'
% get_yaml_str(
get_sample_config_spec('somebucket', 'someprofile',
'someregion')),
class_name=REPOSITORY_CLASS_NAME)
return None
path, file = None, None
try:
refs_path = get_refs_path(self.__config, repo_type)
index_path = get_index_path(self.__config, repo_type)
metadata_path = get_metadata_path(self.__config, repo_type)
cache_path = get_cache_path(self.__config, repo_type)
objects_path = get_objects_path(self.__config, repo_type)
repo = LocalRepository(self.__config, objects_path, repo_type)
mutability, check_mutability = repo.get_mutability_from_spec(
spec, repo_type)
sampling_flag = os.path.exists(
os.path.join(index_path, 'metadata', spec, 'sampling'))
if sampling_flag:
log.error(
'You cannot add new data to an entity that is based on a checkout with the --sampling option.',
class_name=REPOSITORY_CLASS_NAME)
return
if not mutability:
return
if not check_mutability:
log.error('Spec mutability cannot be changed.',
class_name=REPOSITORY_CLASS_NAME)
return
_, deleted, untracked_files, _, changed_files = repo.status(
spec, log_errors=False)
if deleted is None and untracked_files is None and changed_files is None:
return None
elif len(deleted) == 0 and len(untracked_files) == 0 and len(
changed_files) == 0:
log.info('There is no new data to add',
class_name=REPOSITORY_CLASS_NAME)
return None
ref = Refs(refs_path, spec, repo_type)
tag, sha = ref.branch()
categories_path = get_path_with_categories(tag)
path, file = search_spec_file(self.__repo_type, spec,
categories_path)
except Exception as e:
log.error(e, class_name=REPOSITORY_CLASS_NAME)
return
if path is None:
return
spec_path = os.path.join(path, file)
spec_file = yaml_load(spec_path)
if not validate_spec_hash(spec_file, self.__repo_type):
log.error(
'Invalid %s spec in %s. It should look something like this:\n%s'
% (self.__repo_type, spec_path,
get_sample_spec_doc('somebucket', self.__repo_type)),
class_name=REPOSITORY_CLASS_NAME)
return None
if not validate_bucket_name(spec_file[self.__repo_type],
self.__config):
return None
# Check tag before anything to avoid creating unstable state
log.debug('Repository: check if tag already exists',
class_name=REPOSITORY_CLASS_NAME)
m = Metadata(spec, metadata_path, self.__config, repo_type)
if not m.check_exists():
log.error('The %s has not been initialized' % self.__repo_type,
class_name=REPOSITORY_CLASS_NAME)
return
try:
m.update()
except Exception:
pass
# get version of current manifest file
manifest = ''
if tag is not None:
m.checkout(tag)
md_metadata_path = m.get_metadata_path(tag)
manifest = os.path.join(md_metadata_path, 'MANIFEST.yaml')
m.checkout('master')
try:
# adds chunks to ml-git Index
log.info('%s adding path [%s] to ml-git index' % (repo_type, path),
class_name=REPOSITORY_CLASS_NAME)
with change_mask_for_routine(is_shared_objects):
idx = MultihashIndex(spec, index_path, objects_path,
mutability, cache_path)
idx.add(path, manifest, file_path)
# create hard links in ml-git Cache
self.create_hard_links_in_cache(cache_path, index_path,
is_shared_cache, mutability, path,
spec)
except Exception as e:
log.error(e, class_name=REPOSITORY_CLASS_NAME)
return None
if bump_version and not increment_version_in_spec(
spec_path, self.__repo_type):
return None
idx.add_metadata(path, file)
self._check_corrupted_files(spec, repo)
# Run file check
if run_fsck:
self.fsck()
