def commit(self, spec, specs, version_number=None, run_fsck=False, msg=None): # Move chunks from index to .ml-git/objects repo_type = self.__repo_type try: index_path = get_index_path(self.__config, repo_type) objects_path = get_objects_path(self.__config, repo_type) metadata_path = get_metadata_path(self.__config, repo_type) refs_path = get_refs_path(self.__config, repo_type) repo = LocalRepository(self.__config, objects_path, repo_type) mutability, check_mutability = repo.get_mutability_from_spec( spec, repo_type) if not mutability: return if not check_mutability: log.error('Spec mutability cannot be changed.', class_name=REPOSITORY_CLASS_NAME) return except Exception as e: log.error(e, class_name=REPOSITORY_CLASS_NAME) return ref = Refs(refs_path, spec, repo_type) tag, sha = ref.branch() categories_path = get_path_with_categories(tag) manifest_path = os.path.join(metadata_path, categories_path, spec, 'MANIFEST.yaml') path, file = None, None try: path, file = search_spec_file(self.__repo_type, spec, categories_path) except Exception as e: log.error(e, class_name=REPOSITORY_CLASS_NAME) if path is None: return None, None, None spec_path = os.path.join(path, file) idx = MultihashIndex(spec, index_path, objects_path) if version_number: set_version_in_spec(version_number, spec_path, self.__repo_type) idx.add_metadata(path, file) # Check tag before anything to avoid creating unstable state log.debug('Check if tag already exists', class_name=REPOSITORY_CLASS_NAME) m = Metadata(spec, metadata_path, self.__config, repo_type) if not m.check_exists(): log.error('The %s has not been initialized' % self.__repo_type, class_name=REPOSITORY_CLASS_NAME) return full_metadata_path, categories_sub_path, metadata = m.tag_exists( index_path) if metadata is None: return None log.debug('%s -> %s' % (index_path, objects_path), class_name=REPOSITORY_CLASS_NAME) # commit objects in index to ml-git objects o = Objects(spec, objects_path) changed_files, deleted_files = o.commit_index(index_path, path) bare_mode = os.path.exists( os.path.join(index_path, 'metadata', spec, 'bare')) if not bare_mode and len(deleted_files) > 0: self.remove_deleted_files(idx, index_path, m, manifest_path, path, spec, deleted_files) elif bare_mode: tag, _ = ref.branch() self._checkout_ref(tag) # update metadata spec & README.md # option --dataset-spec --labels-spec tag, sha = m.commit_metadata(index_path, specs, msg, changed_files, mutability, path) # update ml-git ref spec HEAD == to new SHA-1 / tag if tag is None: return None ref = Refs(refs_path, spec, repo_type) ref.update_head(tag, sha) # Run file check if run_fsck: self.fsck() return tag
def tag(self, spec, usr_tag): repo_type = self.__repo_type try: metadata_path = get_metadata_path(self.__config, repo_type) refs_path = get_refs_path(self.__config, repo_type) r = Refs(refs_path, spec, repo_type) curtag, sha = r.head() except Exception as e: log.error(e, class_name=REPOSITORY_CLASS_NAME) return False if curtag is None: log.error('No current tag for [%s]. commit first.' % spec, class_name=REPOSITORY_CLASS_NAME) return False utag = UsrTag(curtag, usr_tag) # Check if usrtag exists before creating it log.debug('Check if tag [%s] already exists' % utag, class_name=REPOSITORY_CLASS_NAME) m = Metadata(spec, metadata_path, self.__config, repo_type) if m._usrtag_exists(utag) is True: log.error('Tag [%s] already exists.' % utag, class_name=REPOSITORY_CLASS_NAME) return False # ensure metadata repository is at the current tag/sha version m = Metadata('', metadata_path, self.__config, repo_type) m.checkout(curtag) # TODO: format to something that could be used for a checkout: # format: _._user_.._ + curtag + _.._ + usrtag # at checkout with usrtag look for pattern _._ then find usrtag in the list (split on '_.._') # adds usrtag to the metadata repository m = Metadata(spec, metadata_path, self.__config, repo_type) try: m.tag_add(utag) except Exception as e: match = re.search("stderr: 'fatal:(.*)'$", e.stderr) err = match.group(1) log.error(err, class_name=REPOSITORY_CLASS_NAME) return log.info('Create Tag Successfull', class_name=REPOSITORY_CLASS_NAME) # checkout at metadata repository at master version m.checkout('master') return True
def clone_config(self, url, folder=None, track=False): if clone_config_repository(url, folder, track): self.__config = config_load() m = Metadata('', get_metadata_path(self.__config), self.__config) m.clone_config_repo()
def test_metadata_tag(self): m = Metadata(spec, index_path, config, DATASETS) tag = m.metadata_tag(metadata_config) self.assertEqual(tag, 'images__dataset_ex__1')
def test_init(self): m = Metadata(spec, self.test_dir, config, DATASETS) m.init() self.assertTrue(m.check_exists()) clear(m.path)
def test_blank_remote_url(self): config_cp = deepcopy(config) config_cp[DATASETS]['git'] = '' m = Metadata(spec, self.test_dir, config_cp, DATASETS) self.assertRaises(GitError, m.validate_blank_remote_url) clear(m.path)
def test_clone_config_repo(self): m = Metadata('', self.test_dir, config, DATASETS) m.clone_config_repo() self.assertTrue(m.check_exists())