def test_get_metrics(self): repo_type = MODELS mdpath = os.path.join(self.test_dir, 'mdata', repo_type, 'metadata') specpath = os.path.join('vision-computer', 'images') entity = 'model-ex' m = Metadata(entity, self.test_dir, config, repo_type) m.init() ensure_path_exists(os.path.join(mdpath, specpath, entity)) spec_metadata_path = os.path.join(mdpath, specpath, entity, 'model-ex.spec') shutil.copy('hdata/dataset-ex.spec', spec_metadata_path) spec_file = yaml_load(spec_metadata_path) spec_file[MODEL_SPEC_KEY] = deepcopy(spec_file[DATASET_SPEC_KEY]) del spec_file[DATASET_SPEC_KEY] spec_file[MODEL_SPEC_KEY]['metrics'] = {'metric_1': 0, 'metric_2': 1} yaml_save(spec_file, spec_metadata_path) tag = 'vision-computer__images__model-ex__1' sha = m.commit(spec_metadata_path, specpath) m.tag_add(tag) metrics = m._get_metrics(entity, sha) test_table = PrettyTable() test_table.field_names = ['Name', 'Value'] test_table.align['Name'] = 'l' test_table.align['Value'] = 'l' test_table.add_row(['metric_1', 0]) test_table.add_row(['metric_2', 1]) test_metrics = '\nmetrics:\n{}'.format(test_table.get_string()) self.assertEqual(metrics, test_metrics)
def tag(self, spec, usr_tag): repo_type = self.__repo_type try: metadata_path = get_metadata_path(self.__config, repo_type) refs_path = get_refs_path(self.__config, repo_type) r = Refs(refs_path, spec, repo_type) curtag, sha = r.head() except Exception as e: log.error(e, class_name=REPOSITORY_CLASS_NAME) return False if curtag is None: log.error('No current tag for [%s]. commit first.' % spec, class_name=REPOSITORY_CLASS_NAME) return False utag = UsrTag(curtag, usr_tag) # Check if usrtag exists before creating it log.debug('Check if tag [%s] already exists' % utag, class_name=REPOSITORY_CLASS_NAME) m = Metadata(spec, metadata_path, self.__config, repo_type) if m._usrtag_exists(utag) is True: log.error('Tag [%s] already exists.' % utag, class_name=REPOSITORY_CLASS_NAME) return False # ensure metadata repository is at the current tag/sha version m = Metadata('', metadata_path, self.__config, repo_type) m.checkout(curtag) # TODO: format to something that could be used for a checkout: # format: _._user_.._ + curtag + _.._ + usrtag # at checkout with usrtag look for pattern _._ then find usrtag in the list (split on '_.._') # adds usrtag to the metadata repository m = Metadata(spec, metadata_path, self.__config, repo_type) try: m.tag_add(utag) except Exception as e: match = re.search("stderr: 'fatal:(.*)'$", e.stderr) err = match.group(1) log.error(err, class_name=REPOSITORY_CLASS_NAME) return log.info('Create Tag Successfull', class_name=REPOSITORY_CLASS_NAME) # checkout at metadata repository at master version m.checkout() return True
def test_get_specs_to_compare(self): mdpath = os.path.join(self.test_dir, 'mdata', DATASETS, 'metadata') specpath = 'dataset-ex' m = Metadata(specpath, self.test_dir, config, DATASETS) m.init() ensure_path_exists(os.path.join(mdpath, specpath)) spec_metadata_path = os.path.join(mdpath, specpath) + '/dataset-ex.spec' shutil.copy('hdata/dataset-ex.spec', spec_metadata_path) sha = m.commit(spec_metadata_path, specpath) m.tag_add(sha) specs = m.get_specs_to_compare(specpath) spec_file = yaml_load(spec_metadata_path) for c, v in specs: self.assertEqual(c, spec_file[DATASETS]['manifest']) self.assertIsNotNone(v, {DATASETS: {'manifest': {}}})
def test_get_metrics_without_metrics(self): repo_type = MODELS mdpath = os.path.join(self.test_dir, 'mdata', repo_type, 'metadata') specpath = os.path.join('vision-computer', 'images') entity = 'model-ex' m = Metadata(entity, self.test_dir, config, repo_type) m.init() ensure_path_exists(os.path.join(mdpath, specpath, entity)) spec_metadata_path = os.path.join(mdpath, specpath, entity) + '/model-ex.spec' shutil.copy('hdata/dataset-ex.spec', spec_metadata_path) spec_file = yaml_load(spec_metadata_path) spec_file[MODEL_SPEC_KEY] = deepcopy(spec_file[DATASET_SPEC_KEY]) del spec_file[DATASET_SPEC_KEY] yaml_save(spec_file, spec_metadata_path) tag = 'vision-computer__images__model-ex__1' sha = m.commit(spec_metadata_path, specpath) m.tag_add(tag) metrics = m._get_metrics(entity, sha) self.assertEqual(metrics, '')
def test_get_spec_content_from_ref(self): mdpath = os.path.join(self.test_dir, 'mdata', DATASETS, 'metadata') specpath = 'dataset-ex' m = Metadata(specpath, self.test_dir, config, DATASETS) m.init() ensure_path_exists(os.path.join(mdpath, specpath)) spec_metadata_path = os.path.join(mdpath, specpath) + '/dataset-ex.spec' shutil.copy('hdata/dataset-ex.spec', spec_metadata_path) sha = m.commit(spec_metadata_path, specpath) tag = m.tag_add(sha) path = 'dataset-ex/dataset-ex.spec' content = yaml_load_str(m._get_spec_content_from_ref(tag.commit, path)) spec_file = yaml_load(spec_metadata_path) self.assertEqual(content, spec_file)