def test_search_spec_file(self): categories_path = '' specpath = 'dataset-ex' spec_dir = os.path.join(self.tmp_dir, 'dataset') spec_dir_c = os.path.join(spec_dir, categories_path, specpath) os.mkdir(spec_dir) os.mkdir(spec_dir_c) os.mkdir(os.path.join(spec_dir_c, 'data')) spec_file = specpath + '.spec' f = open(os.path.join(spec_dir_c, spec_file), 'w') f.close() dir, spec = search_spec_file(spec_dir, specpath, categories_path) self.assertEqual(dir, spec_dir_c) self.assertEqual(spec, spec_file) os.remove(os.path.join(spec_dir_c, spec_file)) self.assertRaises( SearchSpecException, lambda: search_spec_file(spec_dir, specpath, categories_path)) shutil.rmtree(spec_dir) self.assertRaises( Exception, lambda: search_spec_file(spec_dir, specpath, categories_path))
def remote_fsck(self, spec, retries=2, thorough=False, paranoid=False): repo_type = self.__repo_type try: metadata_path = get_metadata_path(self.__config, repo_type) objects_path = get_objects_path(self.__config, repo_type) refs_path = get_refs_path(self.__config, repo_type) ref = Refs(refs_path, spec, repo_type) tag, sha = ref.branch() categories_path = get_path_with_categories(tag) self._checkout_ref(tag) spec_path, spec_file = search_spec_file(self.__repo_type, spec, categories_path) except Exception as e: log.error(e, class_name=REPOSITORY_CLASS_NAME) return if spec_path is None: return full_spec_path = os.path.join(spec_path, spec_file) r = LocalRepository(self.__config, objects_path, repo_type) r.remote_fsck(metadata_path, tag, full_spec_path, retries, thorough, paranoid) # ensure first we're on master ! self._checkout_ref()
def unlock_file(self, spec, file_path): repo_type = self.__repo_type if not validate_config_spec_hash(self.__config): log.error( '.ml-git/config.yaml invalid. It should look something like this:\n%s' % get_yaml_str( get_sample_config_spec('somebucket', 'someprofile', 'someregion')), class_name=REPOSITORY_CLASS_NAME) return None path, file = None, None try: refs_path = get_refs_path(self.__config, repo_type) objects_path = get_objects_path(self.__config, repo_type) index_path = get_index_path(self.__config, repo_type) cache_path = get_cache_path(self.__config, repo_type) ref = Refs(refs_path, spec, repo_type) tag, sha = ref.branch() categories_path = get_path_with_categories(tag) path, file = search_spec_file(self.__repo_type, spec, categories_path) except Exception as e: log.error(e, class_name=REPOSITORY_CLASS_NAME) return if path is None: return spec_path = os.path.join(path, file) spec_file = yaml_load(spec_path) try: mutability = spec_file[repo_type]['mutability'] if mutability not in Mutability.list(): log.error('Invalid mutability type.', class_name=REPOSITORY_CLASS_NAME) return except Exception: log.info( 'The spec does not have the \'mutability\' property set. Default: strict.', class_name=REPOSITORY_CLASS_NAME) return if mutability != Mutability.STRICT.value: try: local = LocalRepository(self.__config, objects_path, repo_type) local.unlock_file(path, file_path, index_path, objects_path, spec, cache_path) except Exception as e: log.error(e, class_name=REPOSITORY_CLASS_NAME) return else: log.error( 'You cannot use this command for this entity because mutability cannot be strict.', class_name=REPOSITORY_CLASS_NAME)
def checkout(entity, tag, sampling=None, retries=2, force=False, dataset=False, labels=False, version=-1): """This command allows retrieving the data of a specific version of an ML entity. Example: checkout('datasets', 'computer-vision__images3__imagenet__1') Args: entity (str): The type of an ML entity. (datasets, labels or models) tag (str): An ml-git tag to identify a specific version of an ML entity. sampling (dict): group: <amount>:<group> The group sample option consists of amount and group used to download a sample.\n range: <start:stop:step> The range sample option consists of start, stop and step used to download a sample. The start parameter can be equal or greater than zero. The stop parameter can be 'all', -1 or any integer above zero.\n random: <amount:frequency> The random sample option consists of amount and frequency used to download a sample. seed: The seed is used to initialize the pseudorandom numbers. retries (int, optional): Number of retries to download the files from the storage [default: 2]. force (bool, optional): Force checkout command to delete untracked/uncommitted files from the local repository [default: False]. dataset (bool, optional): If exist a dataset related with the model or labels, this one must be downloaded [default: False]. labels (bool, optional): If exist labels related with the model, they must be downloaded [default: False]. Returns: str: Return the path where the data was checked out. """ repo = get_repository_instance(entity) repo.update() if sampling is not None and not validate_sample(sampling): return None options = {} options['with_dataset'] = dataset options['with_labels'] = labels options['retry'] = retries options['force'] = force options['bare'] = False options['version'] = version repo.checkout(tag, sampling, options) spec_name = tag if re.search(RGX_TAG_FORMAT, tag): _, spec_name, _ = spec_parse(tag) spec_path, _ = search_spec_file(entity, spec_name) data_path = os.path.relpath(spec_path, get_root_path()) if not os.path.exists(data_path): data_path = None return data_path
def push(self, spec, retry=2, clear_on_fail=False): repo_type = self.__repo_type try: objects_path = get_objects_path(self.__config, repo_type) metadata_path = get_metadata_path(self.__config, repo_type) refs_path = get_refs_path(self.__config, repo_type) except Exception as e: log.error(e, class_name=REPOSITORY_CLASS_NAME) return met = Metadata(spec, metadata_path, self.__config, repo_type) fields = met.git_user_config() if None in fields.values(): log.error( 'Your name and email address need to be configured in git. ' 'Please see the commands below:', class_name=REPOSITORY_CLASS_NAME) log.error('git config --global user.name \'Your Name\'', class_name=REPOSITORY_CLASS_NAME) log.error('git config --global user.email [email protected]', class_name=REPOSITORY_CLASS_NAME) return if met.fetch() is False: return ref = Refs(refs_path, spec, repo_type) tag, sha = ref.branch() categories_path = get_path_with_categories(tag) spec_path, spec_file = None, None try: spec_path, spec_file = search_spec_file(self.__repo_type, spec, categories_path) except Exception as e: log.error(e, class_name=REPOSITORY_CLASS_NAME) if spec_path is None: return full_spec_path = os.path.join(spec_path, spec_file) repo = LocalRepository(self.__config, objects_path, repo_type) ret = repo.push(objects_path, full_spec_path, retry, clear_on_fail) # ensure first we're on master ! met.checkout() if ret == 0: # push metadata spec to LocalRepository git repository try: met.push() except Exception as e: log.error(e, class_name=REPOSITORY_CLASS_NAME) return MultihashFS(objects_path).reset_log()
def test_search_spec_file(self): spec_name = 'dataset-ex' entity_dir = os.path.join(self.tmp_dir, DATASETS) spec_path = os.path.join(entity_dir, spec_name) os.mkdir(entity_dir) os.mkdir(spec_path) os.mkdir(os.path.join(spec_path, 'data')) spec_file = spec_name + '.spec' f = open(os.path.join(spec_path, spec_file), 'w') f.close() dir, spec = search_spec_file(DATASETS, spec_name, entity_dir) self.assertEqual(dir, spec_path) self.assertEqual(spec, spec_file) os.remove(os.path.join(spec_path, spec_file)) self.assertRaises( SearchSpecException, lambda: search_spec_file(DATASETS, spec_name, entity_dir)) shutil.rmtree(entity_dir) self.assertRaises( Exception, lambda: search_spec_file(DATASETS, spec_name, entity_dir))
def reset(self, spec, reset_type, head): log.info(output_messages['INFO_INITIALIZING_RESET'] % (reset_type, head), class_name=REPOSITORY_CLASS_NAME) if (reset_type == '--soft' or reset_type == '--mixed') and head == HEAD: return try: repo_type = self.__repo_type metadata_path = get_metadata_path(self.__config, repo_type) index_path = get_index_path(self.__config, repo_type) refs_path = get_refs_path(self.__config, repo_type) object_path = get_objects_path(self.__config, repo_type) met = Metadata(spec, metadata_path, self.__config, repo_type) ref = Refs(refs_path, spec, repo_type) idx = MultihashIndex(spec, index_path, object_path) fidx = FullIndex(spec, index_path) except Exception as e: log.error(e, class_name=REPOSITORY_CLASS_NAME) return # get tag before reset tag = met.get_current_tag() categories_path = get_path_with_categories(str(tag)) # current manifest file before reset manifest_path = os.path.join(metadata_path, categories_path, spec, MANIFEST_FILE) _manifest = Manifest(manifest_path).load() if head == HEAD_1: # HEAD~1 try: # reset the repo met.reset() except Exception: return # get tag after reset tag_after_reset = met.get_current_tag() sha = met.sha_from_tag(tag_after_reset) # update ml-git ref HEAD ref.update_head(str(tag_after_reset), sha) # # get path to reset workspace in case of --hard path, file = None, None try: path, file = search_spec_file(self.__repo_type, spec, categories_path) except Exception as e: log.error(e, class_name=REPOSITORY_CLASS_NAME) if reset_type == '--hard' and path is None: return # get manifest from metadata after reset _manifest_changed = Manifest(manifest_path) hash_files, file_names = _manifest_changed.get_diff(_manifest) idx_mf = idx.get_index().load() if reset_type == '--soft': # add in index/metadata/<entity-name>/MANIFEST idx.update_index_manifest(idx_mf) idx.update_index_manifest(hash_files) fidx.update_index_status(file_names, Status.a.name) else: # --hard or --mixed # remove hash from index/hashsh/store.log file_names.update(*idx_mf.values()) objs = MultihashFS(index_path) for key_hash in hash_files: objs.remove_hash(key_hash) idx.remove_manifest() fidx.remove_from_index_yaml(file_names) fidx.remove_uncommitted() if reset_type == '--hard': # reset workspace remove_from_workspace(file_names, path, spec)
def add(self, spec, file_path, bump_version=False, run_fsck=False): repo_type = self.__repo_type is_shared_objects = 'objects_path' in self.__config[repo_type] is_shared_cache = 'cache_path' in self.__config[repo_type] if not validate_config_spec_hash(self.__config): log.error( '.ml-git/config.yaml invalid. It should look something like this:\n%s' % get_yaml_str( get_sample_config_spec('somebucket', 'someprofile', 'someregion')), class_name=REPOSITORY_CLASS_NAME) return None path, file = None, None try: refs_path = get_refs_path(self.__config, repo_type) index_path = get_index_path(self.__config, repo_type) metadata_path = get_metadata_path(self.__config, repo_type) cache_path = get_cache_path(self.__config, repo_type) objects_path = get_objects_path(self.__config, repo_type) repo = LocalRepository(self.__config, objects_path, repo_type) mutability, check_mutability = repo.get_mutability_from_spec( spec, repo_type) sampling_flag = os.path.exists( os.path.join(index_path, 'metadata', spec, 'sampling')) if sampling_flag: log.error( 'You cannot add new data to an entity that is based on a checkout with the --sampling option.', class_name=REPOSITORY_CLASS_NAME) return if not mutability: return if not check_mutability: log.error('Spec mutability cannot be changed.', class_name=REPOSITORY_CLASS_NAME) return if not self._has_new_data(repo, spec): return None ref = Refs(refs_path, spec, repo_type) tag, sha = ref.branch() categories_path = get_path_with_categories(tag) path, file = search_spec_file(self.__repo_type, spec, categories_path) except Exception as e: log.error(e, class_name=REPOSITORY_CLASS_NAME) return if path is None: return spec_path = os.path.join(path, file) if not self._is_spec_valid(spec_path): return None # Check tag before anything to avoid creating unstable state log.debug('Repository: check if tag already exists', class_name=REPOSITORY_CLASS_NAME) m = Metadata(spec, metadata_path, self.__config, repo_type) if not m.check_exists(): log.error('The %s has not been initialized' % self.__repo_type, class_name=REPOSITORY_CLASS_NAME) return try: m.update() except Exception: pass # get version of current manifest file manifest = self._get_current_manifest_file(m, tag) try: # adds chunks to ml-git Index log.info('%s adding path [%s] to ml-git index' % (repo_type, path), class_name=REPOSITORY_CLASS_NAME) with change_mask_for_routine(is_shared_objects): idx = MultihashIndex(spec, index_path, objects_path, mutability, cache_path) idx.add(path, manifest, file_path) # create hard links in ml-git Cache self.create_hard_links_in_cache(cache_path, index_path, is_shared_cache, mutability, path, spec) except Exception as e: log.error(e, class_name=REPOSITORY_CLASS_NAME) return None if bump_version and not increment_version_in_spec( spec_path, self.__repo_type): return None idx.add_metadata(path, file) self._check_corrupted_files(spec, repo) # Run file check if run_fsck: self.fsck()
def commit(self, spec, specs, version=None, run_fsck=False, msg=None): # Move chunks from index to .ml-git/objects repo_type = self.__repo_type try: index_path = get_index_path(self.__config, repo_type) objects_path = get_objects_path(self.__config, repo_type) metadata_path = get_metadata_path(self.__config, repo_type) refs_path = get_refs_path(self.__config, repo_type) repo = LocalRepository(self.__config, objects_path, repo_type) mutability, check_mutability = repo.get_mutability_from_spec( spec, repo_type) if not mutability: return if not check_mutability: log.error('Spec mutability cannot be changed.', class_name=REPOSITORY_CLASS_NAME) return except Exception as e: log.error(e, class_name=REPOSITORY_CLASS_NAME) return ref = Refs(refs_path, spec, repo_type) tag, sha = ref.branch() categories_path = get_path_with_categories(tag) manifest_path = os.path.join(metadata_path, categories_path, spec, MANIFEST_FILE) path, file = None, None try: path, file = search_spec_file(self.__repo_type, spec, categories_path) except Exception as e: log.error(e, class_name=REPOSITORY_CLASS_NAME) if path is None: return None, None, None spec_path = os.path.join(path, file) idx = MultihashIndex(spec, index_path, objects_path) if version: set_version_in_spec(version, spec_path, self.__repo_type) idx.add_metadata(path, file) # Check tag before anything to avoid creating unstable state log.debug('Check if tag already exists', class_name=REPOSITORY_CLASS_NAME) m = Metadata(spec, metadata_path, self.__config, repo_type) if not m.check_exists(): log.error('The %s has not been initialized' % self.__repo_type, class_name=REPOSITORY_CLASS_NAME) return full_metadata_path, categories_sub_path, metadata = m.tag_exists( index_path) if metadata is None: return None log.debug('%s -> %s' % (index_path, objects_path), class_name=REPOSITORY_CLASS_NAME) # commit objects in index to ml-git objects o = Objects(spec, objects_path) changed_files, deleted_files = o.commit_index(index_path, path) bare_mode = os.path.exists( os.path.join(index_path, 'metadata', spec, 'bare')) if not bare_mode: manifest = m.get_metadata_manifest(manifest_path) self._remove_deleted_files(idx, index_path, m, manifest, spec, deleted_files) m.remove_files_added_after_base_tag(manifest, path) else: tag, _ = ref.branch() self._checkout_ref(tag) # update metadata spec & README.md # option --dataset-spec --labels-spec tag, sha = m.commit_metadata(index_path, specs, msg, changed_files, mutability, path) # update ml-git ref spec HEAD == to new SHA-1 / tag if tag is None: return None ref = Refs(refs_path, spec, repo_type) ref.update_head(tag, sha) # Run file check if run_fsck: self.fsck() return tag
def _check_is_valid_entity(self, repo_type, spec): ref = Refs(get_refs_path(self.__config, repo_type), spec, repo_type) tag, _ = ref.branch() categories_path = get_path_with_categories(tag) search_spec_file(repo_type, spec, categories_path)