def _update_file_status(self, cache, filepath, fullpath, scid, st, value): status = Status.a.name prev_hash = value['hash'] scid_ret = scid is_flexible = self._mutability == MutabilityType.FLEXIBLE.value is_strict = self._mutability == MutabilityType.STRICT.value not_unlocked = value['mtime'] != st.st_mtime and 'untime' not in value bare_mode = os.path.exists( os.path.join(self._path, 'metadata', self._spec, 'bare')) if (is_flexible and not_unlocked) or is_strict: if value['status'] == Status.c.name and 'previous_hash' in value: prev_hash = value['previous_hash'] if scid == prev_hash: prev_hash = None status = Status.u.name log.debug(output_messages['DEBUG_RESTORED_FILE'].format( posix_path(filepath)), class_name=MULTI_HASH_CLASS_NAME) else: status = Status.c.name scid_ret = None file_path = Cache(cache).get_keypath(value['hash']) if os.path.exists(file_path): os.unlink(file_path) elif bare_mode and self._mutability == MutabilityType.MUTABLE.value: print('\n') log.warn(output_messages['WARN_FILE_EXISTS_IN_REPOSITORY'] % filepath, class_name=MULTI_HASH_CLASS_NAME) self.update_full_index(posix_path(filepath), fullpath, status, scid, prev_hash) return scid_ret
def _update_file_status(self, cache, filepath, fullpath, scid, st, value): status = Status.a.name prev_hash = value['hash'] scid_ret = scid is_flexible = self._mutability == Mutability.FLEXIBLE.value is_strict = self._mutability == Mutability.STRICT.value not_unlocked = value['mtime'] != st.st_mtime and 'untime' not in value bare_mode = os.path.exists( os.path.join(self._path, 'metadata', self._spec, 'bare')) if (is_flexible and not_unlocked) or is_strict: status = Status.c.name prev_hash = None scid_ret = None file_path = Cache(cache).get_keypath(value['hash']) if os.path.exists(file_path): os.unlink(file_path) elif bare_mode and self._mutability == Mutability.MUTABLE.value: print('\n') log.warn( 'The file %s already exists in the repository. If you commit, the' ' file will be overwritten.' % filepath, class_name=MULTI_HASH_CLASS_NAME) self.update_full_index(posix_path(filepath), fullpath, status, scid, prev_hash) return scid_ret
def _get_spec_content(self, spec, sha): entity_dir = get_entity_dir(self.__repo_type, spec, root_path=self.__path) spec_path = '/'.join([posix_path(entity_dir), spec + SPEC_EXTENSION]) return yaml_load_str(self._get_spec_content_from_ref(sha, spec_path))
def fsck(self, entity_path, hfs, cache): corrupted_files = {} f_index = self.get_index() for k, v in f_index.items(): expected_hash = v['hash'] file_path = os.path.join(entity_path, k) if os.path.exists(file_path): if v['status'] == Status.c.name and 'previous_hash' in v: expected_hash = v['previous_hash'] if hfs.get_scid(file_path) != expected_hash: check_file = f_index.get(posix_path(k)) self.check_and_update(k, check_file, hfs, posix_path(k), file_path, cache) corrupted_files[file_path] = { 'hash': expected_hash, 'key': k } self.save_manifest_index() return corrupted_files
def _add_file(self, basepath, filepath, f_index_file): fullpath = os.path.join(basepath, filepath) metadatapath = os.path.join(self._path, 'metadata', self._spec) ensure_path_exists(metadatapath) scid = None check_file = f_index_file.get(posix_path(filepath)) previous_hash = None if check_file is not None: if self._full_idx.check_and_update(filepath, check_file, self._hfs, posix_path(filepath), fullpath, self._cache): scid = self._hfs.put(fullpath) updated_check = f_index_file.get(posix_path(filepath)) if 'previous_hash' in updated_check: previous_hash = updated_check['previous_hash'] else: scid = self._hfs.put(fullpath) self._full_idx.update_full_index(posix_path(filepath), fullpath, Status.a.name, scid) return scid, filepath, previous_hash
def get_specs_to_compare(self, spec): entity = self.__repo_type spec_manifest_key = 'manifest' tags = self.list_tags(spec, True) entity_dir = get_entity_dir(entity, spec, root_path=self.__path) spec_path = '/'.join([posix_path(entity_dir), spec + SPEC_EXTENSION]) for tag in tags: current_ref = tag.commit parents = current_ref.parents base_spec = {entity: {spec_manifest_key: {}}} if parents: base_ref = parents[0] base_spec = yaml_load_str( self._get_spec_content_from_ref(base_ref, spec_path)) current_spec = yaml_load_str( self._get_spec_content_from_ref(current_ref, spec_path)) yield current_spec[entity][spec_manifest_key], base_spec[entity][ spec_manifest_key]
def check_and_update(self, key, value, hfs, filepath, fullpath, cache): st = os.stat(fullpath) if key == filepath and value['ctime'] == st.st_ctime and value['mtime'] == st.st_mtime: log.debug('File [%s] already exists in ml-git repository' % filepath, class_name=MULTI_HASH_CLASS_NAME) return None elif key == filepath and value['ctime'] != st.st_ctime or value['mtime'] != st.st_mtime: log.debug('File [%s] was modified' % filepath, class_name=MULTI_HASH_CLASS_NAME) scid = hfs.get_scid(fullpath) if value['hash'] != scid: status = Status.a.name prev_hash = value['hash'] scid_ret = scid is_flexible = self._mutability == Mutability.FLEXIBLE.value is_strict = self._mutability == Mutability.STRICT.value not_unlocked = value['mtime'] != st.st_mtime and 'untime' not in value bare_mode = os.path.exists(os.path.join(self._path, 'metadata', self._spec, 'bare')) if (is_flexible and not_unlocked) or is_strict: status = Status.c.name prev_hash = None scid_ret = None file_path = Cache(cache).get_keypath(value['hash']) if os.path.exists(file_path): os.unlink(file_path) elif bare_mode and self._mutability == Mutability.MUTABLE.value: print('\n') log.warn('The file %s already exists in the repository. If you commit, the' ' file will be overwritten.' % filepath, class_name=MULTI_HASH_CLASS_NAME) self.update_full_index(posix_path(filepath), fullpath, status, scid, prev_hash) return scid_ret return None
def update_index(self, objectkey, filename, previous_hash=None): self._mf.add(objectkey, posix_path(filename), previous_hash)