def search_uuid(root, uuid, direction=DOWN, depth=0, skip_root=False): '''Like search but finds a uuid file. This is a low-level function used to lookup missing entries. Arguments: root (str): Directory to search uuid (str): UUID to match direction (int): Direction to search (fsfs.UP or fsfs.DOWN) depth (int): Maximum depth of search skip_root (bool): Skip search in root directory Returns: generator yielding (root, data_root, uuid_file) ''' if direction == DOWN: base_level = root.count(os.sep) for root, subdirs, _ in walk(root): level = root.count(os.sep) - base_level if depth and level == depth: del subdirs[:] subdirs[:] = [d for d in subdirs if not d == api.get_data_root()] if skip_root and level == 0: continue root = util.unipath(root) data_root = root + '/' + api.get_data_root() uuid_file = root + '/' + api.get_data_root() + '/' + 'uuid_' + uuid if os.path.isfile(uuid_file): yield root, data_root, uuid_file if direction == UP: level = -1 next_root = util.unipath(root) while True: root = next_root level += 1 if depth and level > depth: break if skip_root and level == 0: next_root = os.path.dirname(root) continue data_root = root + '/' + api.get_data_root() uuid_path = root + '/' + api.get_data_root() + '/' + 'uuid_' + uuid if os.path.isfile(uuid_path): yield root, data_root, uuid_file next_root = os.path.dirname(root) if next_root == root: break
def test_move_entry_raises(tempdir): '''Copy or Move Entry to existing location raises''' entry_path = util.unipath(tempdir, 'entry') dest_path = util.unipath(tempdir, 'move') os.makedirs(dest_path) entry = fsfs.get_entry(entry_path) entry.tag('generic') assert_raises(OSError, entry.move, dest_path) assert_raises(OSError, entry.copy, dest_path)
def test_copy_entry(tempdir): '''Copy Entry with files''' entry_path = util.unipath(tempdir, 'entry') entry_copy_path = util.unipath(tempdir, 'copy') entry, manifest, data_manifest = random_entry(entry_path) new_entry = entry.copy(entry_copy_path) assert new_entry is fsfs.get_entry(entry_copy_path) assert new_entry.uuid != entry.uuid for file in manifest + data_manifest: assert os.path.exists(util.unipath(new_entry.path, file))
def test_new_uuid_after_copy(tempdir): '''Copy Entry creates a new uuid''' entry_path = util.unipath(tempdir, 'entry') dest_path = util.unipath(tempdir, 'copy') entry = fsfs.get_entry(entry_path) entry.tag('generic') new_entry = entry.copy(dest_path) assert new_entry.path == dest_path assert new_entry.uuid != entry.uuid assert len(glob.glob(new_entry.data.path + '/uuid_*')) == 1
def delete(self, remove_root=False): '''Delete an entry Arguments: remove_root (bool): Remove root directory and all it's contents not just the data directory that marks this as an Entry ''' self.data.delete() self.data = EntryData(self, util.unipath(self.path, api.get_data_root())) if remove_root: # Delete children depth-first making sure we send all # appropriate signals along the way for child in list(self.children())[::-1]: child.delete(remove_root=remove_root) try: shutil.rmtree(self.path) except OSError as e: if e.errno != errno.EEXIST: raise self.deleted.send(self)
def select_from_tree(root, selector, sep=DEFAULT_SELECTOR_SEP, direction=DOWN, depth=None, skip_root=False, data_root=None): '''This method is used under the hood by the Search class, you shouldn't need to call it manually. Walk tree up or down yielding entries that match the selector. A selector is a string representing a hierarchy of names. The selector `my_parent/my_entry` would match an Entry named `my_entry` within an Entry named `my_parent`. You can use a custom separator by passing the keyword argument `sep`. Arguments: root (str): Directory to search selector (str): Hierarchy of names separated by sep sep (str): Separator used to split selector direction (int): Direction to search (fsfs.UP or fsfs.DOWN) depth (int): Maximum depth of search skip_root (bool): Skip search in root directory Returns: generator: yielding :class:`models.Entry` matches ''' selector = selector.split(sep) root = util.unipath(root) data_root = data_root or api.get_data_root() if direction == DOWN: depth = depth or DEFAULT_SEARCH_DN_DEPTH return _select_tree_dn(root, selector, data_root, depth) elif direction == UP: depth = depth or DEFAULT_SEARCH_UP_DEPTH return _select_tree_up(root, selector, data_root, depth) else: raise RuntimeError('Invalid direction: ' + str(direction))
def _select_tree_dn(root, selector, data_root, depth, gap=0): dirs = { e.name: e.path for e in safe_scandir(root) if e.is_dir() } if dirs.pop(data_root, None): gap = 0 sel = selector[0] if sel in os.path.basename(root): selector.pop(0) if not selector: yield api.get_entry(util.unipath(root)) return if gap == depth: return for dir in dirs.values(): yield _select_tree_dn( dir, list(selector), data_root, depth, gap + 1 )
def search(root, direction=DOWN, depth=None, levels=None, skip_root=False): '''Search a root directory yielding Entry objects. You can specify a direction to search (fsfs.UP or fsfs.DOWN) and a maximum search depth. Arguments: root (str): Directory to search direction (int): Direction to search (fsfs.UP or fsfs.DOWN) depth (int): Maximum depth of search skip_root (bool): Skip search in root directory Returns: generator: yielding :class:`models.Entry` matches ''' kwargs = dict( root=util.unipath(root), data_root=api.get_data_root(), skip_root=skip_root ) if direction == DOWN: kwargs['depth'] = depth or DEFAULT_SEARCH_DN_DEPTH kwargs['levels'] = levels or DEFAULT_SEARCH_DN_LEVELS return _search_dn(**kwargs) elif direction == UP: kwargs['levels'] = levels or DEFAULT_SEARCH_UP_LEVELS return _search_up(**kwargs) else: raise RuntimeError('Invalid direction: ' + str(direction))
def _search_dn(root, depth=DEFAULT_SEARCH_DN_DEPTH, gap=0, levels=DEFAULT_SEARCH_DN_LEVELS, level=0, skip_root=False, at_root=True, data_root=None): dirs = { e.name: e.path for e in safe_scandir(root) if e.is_dir() } if dirs.pop(data_root, None): gap = 0 if not (skip_root and at_root): level += 1 yield api.get_entry(util.unipath(root)) if gap == depth or (levels and level == levels): return for dir in dirs.values(): yield _search_dn( dir, depth, gap + 1, levels, level, skip_root, False, data_root )
def test_move_entry(tempdir): '''Move Entry''' entry_path = util.unipath(tempdir, 'entry') entry_move_path = util.unipath(tempdir, 'move') entry, manifest, data_manifest = random_entry(entry_path) old_uuid = entry.uuid entry.move(entry_move_path) assert entry.name == 'move' assert entry.path != entry_path assert entry.path == entry_move_path assert entry is fsfs.get_entry(entry_move_path) assert entry.uuid == old_uuid for file in manifest + data_manifest: assert os.path.exists(util.unipath(entry.path, file))
def quick_select(root, selector, sep=DEFAULT_SELECTOR_SEP, first_depth=2, depth=4, skip_root=False): '''Use this method to quickly find one Entry using a selector string. Unlike search, this method returns one Entry, not a generator yielding all matches. Arguments: root: Directory to search within selector: List of partial names to select from tree sep: Separator used to split selector into parts first_depth: Max directoy depth of first selector depth: Max directory depth to search for the rest of the selectors ''' if skip_root: root = util.unipath(root, '*') parts = selector.split(sep) depth = first_depth matches = [] while parts: if matches: root = matches[-1] part = parts.pop(0) # glob recursively until we find a match # root/*, root/*/* ... for i in range(depth): pattern = util.unipath( root, '*/' * i, '*' + part + '*', get_data_root() ) entries = glob(pattern) if entries: match = min(entries, key=len)[:-6] matches.append(match) break else: return return get_entry(util.unipath(matches[-1]))
def _set_path(self, path, uuid=None, uuid_file=None): self.path = path self.blobs_path = util.unipath(self.path, 'blobs') self.files_path = util.unipath(self.path, 'files') self.file = util.unipath(self.path, api.get_data_file()) if not uuid or not uuid_file: self.uuid = None self.uuid_file = None self._find_uuid() else: self.uuid = uuid self.uuid_file = uuid_file if self._lock and self._lock.acquired: self._lock.release() self._lock = lockfile.LockFile(util.unipath(self.path, '.lock'))
def write_blob(self, key, data): if not os.path.exists(self.blobs_path): os.makedirs(self.blobs_path) blob_name = key + '.blob' with open(util.unipath(self.blobs_path, blob_name), 'wb') as f: f.write(data) self._write(**dict(blobs={key: blob_name})) self.parent.data_changed.send(self.parent, dict(self._data))
def _set_path(self, path, uuid=None, uuid_file=None): '''Sets this Entry's path. Used by relink to update this Entry's path, name, and it's associated data path. This shouldn't be necessary during normal usage. ''' self.path = path self.name = os.path.basename(path) data_path = util.unipath(path, api.get_data_root()) self.data._set_path(data_path, uuid, uuid_file)
def write_file(self, key, file): if not os.path.exists(self.files_path): os.makedirs(self.files_path) file_name = os.path.basename(file) file_path = util.unipath(self.files_path, file_name) util.copy_file(file, file_path) self._write(**dict(files={key: file_name})) self.parent.data_changed.send(self.parent, dict(self._data))
def test_entryfactory(tempdir): '''Custom EntryFactory''' entry_path = util.unipath(tempdir, 'entry') fsfs.set_entry_factory(CustomFactory) entry = fsfs.get_entry(entry_path) # Factory returns cache entry obj assert entry is fsfs.get_entry(entry_path) # No tag == default entry assert type(entry) == CustomFactory.EntryProxy assert type(entry.obj()) == CustomFactory.Entry # Add project tag, now we get a Project instance entry.tag('project') assert type(entry) == CustomFactory.EntryProxy assert type(entry.obj()) == CustomFactory.get_type('project') assert hasattr(entry, 'project_method') # Remove project tag entry.untag('project') assert type(entry) == CustomFactory.EntryProxy assert type(entry.obj()) == CustomFactory.Entry assert not hasattr(entry, 'project_method') # Add asset tag now we get asset methods entry.tag('asset') assert type(entry) == CustomFactory.EntryProxy assert type(entry.obj()) == CustomFactory.get_type('asset') assert hasattr(entry, 'asset_method') # Relinked when moved new_entry_path = util.unipath(tempdir, 'supercool') assert samefile(entry.path, entry_path) os.rename(entry.path, new_entry_path) entry.read() # Triggers entry project to relink entry assert samefile(entry.path, new_entry_path) # Restore DefaultFactory fsfs.set_entry_factory(fsfs.DefaultFactory)
def make_tag_path(root, tag): '''Tag name to tag filepath Arguments: root (str): Directory tag (str): tag str Returns: str: {root}/{get_data_path()}/tag_{tag} ''' return util.unipath(root, get_data_root(), 'tag_' + tag)
def tag(root, *tags): '''Tag a directory as an Entry with the provided tags. Arguments: root (str): Directory to tag *tags (List[str]): Tags to add to directory like: 'asset' or 'project' ''' if not tags: raise Exception('Must provide at least one tag.') entry = get_entry(util.unipath(root)) entry.tag(*tags)
def untag(root, *tags): '''Remove a tag from a directory. Arguments: root (str): Directory to remove tags from *tags (List[str]): Tags to remove like: 'asset' or 'project' ''' if not tags: raise Exception('Must provide at least one tag.') entry = get_entry(util.unipath(root)) entry.untag(*tags)
def random_entry(entry_path): files = [fake_name() for _ in range(4)] data_globs = ['/'.join([fsfs.get_data_root(), 'globs', fake_name()]) for _ in range(4)] data_files = ['/'.join([fsfs.get_data_root(), 'files', fake_name()]) for _ in range(4)] children = { fake_name(): { 'files': [fake_name() for _ in range(4)], 'data_globs': [fake_name() for _ in range(4)], 'data_files': [fake_name() for _ in range(4)], } for _ in range(4) } data_manifest = data_globs + data_files manifest = files for child, data in children.items(): for f in data['files']: manifest.append('/'.join([child, f])) for f in data['data_files'] + data['data_globs']: data_manifest.append('/'.join([child, fsfs.get_data_root(), f])) entry = fsfs.get_entry(entry_path) entry.tag('parent') for child_entry_path in children: child_entry = fsfs.get_entry( util.unipath(entry.path, child_entry_path) ) child_entry.tag('child') for file in manifest + data_manifest: util.touch(util.unipath(entry.path, file)) return entry, manifest, data_manifest
def test_custom_uuid(tempdir): '''Assign custom uuid using Entry.uuid setter''' path = util.unipath(tempdir, 'entry') entry = fsfs.get_entry(path) entry.tag('generic') old_uuid = entry.uuid new_uuid = 'custom_uuid' entry.uuid = new_uuid assert entry.uuid != old_uuid assert entry.uuid == new_uuid
def get_tags(root): '''Get the directory's tags Arguments: root (str): Directory to get tags from Returns: list: List of tags ''' tags = [] path = util.unipath(root, get_data_root()) if os.path.isdir(path): for entry in scandir(path): if entry.name.startswith('tag_'): tags.append(entry.name.replace('tag_', '')) return tags
def test_id_generator(tempdir): '''Custom id generator''' def make_id(count=[0]): _id = str(count[0]) count[0] += 1 return _id fsfs.set_id_generator(make_id) generated = [] for i in range(10): e = fsfs.get_entry(util.unipath(tempdir, 'entry_' + str(i))) e.tag('generic') generated.append(e) for i, e in enumerate(generated): assert e.uuid == str(i) fsfs.set_default_policy()
def copy(self, dest, only_data=False): '''Copy this Entry and it's children to a new location Arguments: dest (str): Destination path for new Entry only_data (bool): Copy only Entry data, includes no files outside the Entry's data directories Raises: OSError: Raised when dest already exists or copy_tree fails. Entry is left in original location, any files partially copied will be removed. ''' if os.path.exists(dest): raise OSError('Can not copy Entry to existing location...') try: if not only_data: util.copy_tree(self.path, dest, force=True, overwrite=True) else: hierarchy = [self] + list(self.children()) for entry in hierarchy: old_data_path = entry.data.path rel_data_path = os.path.relpath(old_data_path, self.path) new_data_path = util.unipath(dest, rel_data_path) util.copy_tree(old_data_path, new_data_path) except: if os.path.exists(dest): shutil.rmtree(dest) raise # Update uuids and send EntryCreated signals new_entry = api.get_entry(dest) new_entry.data._set_uuid() new_entry.created.send(new_entry) for child in new_entry.children(): child.data._set_uuid() child.created.send(child) return new_entry
def _make_uuid_path(self, uuid): return util.unipath(self.path, 'uuid_' + uuid)
def __init__(self, path): self.path = path self.name = os.path.basename(path) self.data = EntryData(self, util.unipath(path, api.get_data_root()))
def read_file(self, key): data = self._read() files = data.setdefault('files', {}) file_path = util.unipath(self.files_path, files[key]) return types.File(file_path, mode='rb')
def _make_tag_path(self, tag): return util.unipath(self.path, 'tag_' + tag)
def read_blob(self, key): data = self._read() blobs = data.setdefault('blobs', {}) blob_path = util.unipath(self.blobs_path, blobs[key]) return types.File(blob_path, mode='rb')