def _checkout_ref(self, ref): repo_type = self.__repo_type metadata_path = get_metadata_path(self.__config, repo_type) # checkout m = Metadata('', metadata_path, self.__config, repo_type) m.checkout(ref)
def _checkout_ref(self, ref=None): repo_type = self.__repo_type metadata_path = get_metadata_path(self.__config, repo_type) m = Metadata('', metadata_path, self.__config, repo_type) if ref is None: ref = m.get_default_branch() m.checkout(ref)
def push(self, spec, retry=2, clear_on_fail=False): repo_type = self.__repo_type try: objects_path = get_objects_path(self.__config, repo_type) metadata_path = get_metadata_path(self.__config, repo_type) refs_path = get_refs_path(self.__config, repo_type) except Exception as e: log.error(e, class_name=REPOSITORY_CLASS_NAME) return met = Metadata(spec, metadata_path, self.__config, repo_type) fields = met.git_user_config() if None in fields.values(): log.error( 'Your name and email address need to be configured in git. ' 'Please see the commands below:', class_name=REPOSITORY_CLASS_NAME) log.error('git config --global user.name \'Your Name\'', class_name=REPOSITORY_CLASS_NAME) log.error('git config --global user.email [email protected]', class_name=REPOSITORY_CLASS_NAME) return if met.fetch() is False: return ref = Refs(refs_path, spec, repo_type) tag, sha = ref.branch() categories_path = get_path_with_categories(tag) spec_path, spec_file = None, None try: spec_path, spec_file = search_spec_file(self.__repo_type, spec, categories_path) except Exception as e: log.error(e, class_name=REPOSITORY_CLASS_NAME) if spec_path is None: return full_spec_path = os.path.join(spec_path, spec_file) repo = LocalRepository(self.__config, objects_path, repo_type) ret = repo.push(objects_path, full_spec_path, retry, clear_on_fail) # ensure first we're on master ! met.checkout() if ret == 0: # push metadata spec to LocalRepository git repository try: met.push() except Exception as e: log.error(e, class_name=REPOSITORY_CLASS_NAME) return MultihashFS(objects_path).reset_log()
def tag(self, spec, usr_tag): repo_type = self.__repo_type try: metadata_path = get_metadata_path(self.__config, repo_type) refs_path = get_refs_path(self.__config, repo_type) r = Refs(refs_path, spec, repo_type) curtag, sha = r.head() except Exception as e: log.error(e, class_name=REPOSITORY_CLASS_NAME) return False if curtag is None: log.error('No current tag for [%s]. commit first.' % spec, class_name=REPOSITORY_CLASS_NAME) return False utag = UsrTag(curtag, usr_tag) # Check if usrtag exists before creating it log.debug('Check if tag [%s] already exists' % utag, class_name=REPOSITORY_CLASS_NAME) m = Metadata(spec, metadata_path, self.__config, repo_type) if m._usrtag_exists(utag) is True: log.error('Tag [%s] already exists.' % utag, class_name=REPOSITORY_CLASS_NAME) return False # ensure metadata repository is at the current tag/sha version m = Metadata('', metadata_path, self.__config, repo_type) m.checkout(curtag) # TODO: format to something that could be used for a checkout: # format: _._user_.._ + curtag + _.._ + usrtag # at checkout with usrtag look for pattern _._ then find usrtag in the list (split on '_.._') # adds usrtag to the metadata repository m = Metadata(spec, metadata_path, self.__config, repo_type) try: m.tag_add(utag) except Exception as e: match = re.search("stderr: 'fatal:(.*)'$", e.stderr) err = match.group(1) log.error(err, class_name=REPOSITORY_CLASS_NAME) return log.info('Create Tag Successfull', class_name=REPOSITORY_CLASS_NAME) # checkout at metadata repository at master version m.checkout() return True
def list(self): repo_type = self.__repo_type try: metadata_path = get_metadata_path(self.__config, repo_type) m = Metadata('', metadata_path, self.__config, repo_type) if not m.check_exists(): raise RuntimeError('The %s doesn\'t have been initialized.' % self.__repo_type) m.checkout() m.list(title='ML ' + repo_type) except GitError as g: error_message = g.stderr if 'did not match any file(s) known' in error_message: error_message = 'You don\'t have any entity being managed.' log.error(error_message, class_name=REPOSITORY_CLASS_NAME) return except Exception as e: log.error(e, class_name=REPOSITORY_CLASS_NAME) return
def fetch_tag(self, tag, samples, retries=2): repo_type = self.__repo_type try: objects_path = get_objects_path(self.__config, repo_type) metadata_path = get_metadata_path(self.__config, repo_type) m = Metadata('', metadata_path, self.__config, repo_type) m.checkout(tag) fetch_success = self._fetch(tag, samples, retries) if not fetch_success: objs = Objects('', objects_path) objs.fsck(remove_corrupted=True) m.checkout() except Exception as e: log.error(e, class_name=REPOSITORY_CLASS_NAME) return # restore to master/head self._checkout_ref()
def show(self, spec): repo_type = self.__repo_type try: metadata_path = get_metadata_path(self.__config, repo_type) refs_path = get_refs_path(self.__config, repo_type) except Exception as e: log.error(e, class_name=REPOSITORY_CLASS_NAME) return r = Refs(refs_path, spec, repo_type) tag, sha = r.head() if tag is None: log.info('No HEAD for [%s]' % spec, class_name=LOCAL_REPOSITORY_CLASS_NAME) return m = Metadata('', metadata_path, self.__config, repo_type) m.checkout(tag) m.show(spec) m.checkout()
def add(self, spec, file_path, bump_version=False, run_fsck=False): repo_type = self.__repo_type is_shared_objects = 'objects_path' in self.__config[repo_type] is_shared_cache = 'cache_path' in self.__config[repo_type] if not validate_config_spec_hash(self.__config): log.error( '.ml-git/config.yaml invalid. It should look something like this:\n%s' % get_yaml_str( get_sample_config_spec('somebucket', 'someprofile', 'someregion')), class_name=REPOSITORY_CLASS_NAME) return None path, file = None, None try: refs_path = get_refs_path(self.__config, repo_type) index_path = get_index_path(self.__config, repo_type) metadata_path = get_metadata_path(self.__config, repo_type) cache_path = get_cache_path(self.__config, repo_type) objects_path = get_objects_path(self.__config, repo_type) repo = LocalRepository(self.__config, objects_path, repo_type) mutability, check_mutability = repo.get_mutability_from_spec( spec, repo_type) sampling_flag = os.path.exists( os.path.join(index_path, 'metadata', spec, 'sampling')) if sampling_flag: log.error( 'You cannot add new data to an entity that is based on a checkout with the --sampling option.', class_name=REPOSITORY_CLASS_NAME) return if not mutability: return if not check_mutability: log.error('Spec mutability cannot be changed.', class_name=REPOSITORY_CLASS_NAME) return _, deleted, untracked_files, _, changed_files = repo.status( spec, log_errors=False) if deleted is None and untracked_files is None and changed_files is None: return None elif len(deleted) == 0 and len(untracked_files) == 0 and len( changed_files) == 0: log.info('There is no new data to add', class_name=REPOSITORY_CLASS_NAME) return None ref = Refs(refs_path, spec, repo_type) tag, sha = ref.branch() categories_path = get_path_with_categories(tag) path, file = search_spec_file(self.__repo_type, spec, categories_path) except Exception as e: log.error(e, class_name=REPOSITORY_CLASS_NAME) return if path is None: return spec_path = os.path.join(path, file) spec_file = yaml_load(spec_path) if not validate_spec_hash(spec_file, self.__repo_type): log.error( 'Invalid %s spec in %s. It should look something like this:\n%s' % (self.__repo_type, spec_path, get_sample_spec_doc('somebucket', self.__repo_type)), class_name=REPOSITORY_CLASS_NAME) return None if not validate_bucket_name(spec_file[self.__repo_type], self.__config): return None # Check tag before anything to avoid creating unstable state log.debug('Repository: check if tag already exists', class_name=REPOSITORY_CLASS_NAME) m = Metadata(spec, metadata_path, self.__config, repo_type) if not m.check_exists(): log.error('The %s has not been initialized' % self.__repo_type, class_name=REPOSITORY_CLASS_NAME) return try: m.update() except Exception: pass # get version of current manifest file manifest = '' if tag is not None: m.checkout(tag) md_metadata_path = m.get_metadata_path(tag) manifest = os.path.join(md_metadata_path, 'MANIFEST.yaml') m.checkout('master') try: # adds chunks to ml-git Index log.info('%s adding path [%s] to ml-git index' % (repo_type, path), class_name=REPOSITORY_CLASS_NAME) with change_mask_for_routine(is_shared_objects): idx = MultihashIndex(spec, index_path, objects_path, mutability, cache_path) idx.add(path, manifest, file_path) # create hard links in ml-git Cache self.create_hard_links_in_cache(cache_path, index_path, is_shared_cache, mutability, path, spec) except Exception as e: log.error(e, class_name=REPOSITORY_CLASS_NAME) return None if bump_version and not increment_version_in_spec( spec_path, self.__repo_type): return None idx.add_metadata(path, file) self._check_corrupted_files(spec, repo) # Run file check if run_fsck: self.fsck()