def search_uuid(root, uuid, direction=DOWN, depth=0, skip_root=False):
'''Like search but finds a uuid file. This is a low-level function used
to lookup missing entries.
Arguments:
root (str): Directory to search
uuid (str): UUID to match
direction (int): Direction to search (fsfs.UP or fsfs.DOWN)
depth (int): Maximum depth of search
skip_root (bool): Skip search in root directory
Returns:
generator yielding (root, data_root, uuid_file)
'''
if direction == DOWN:
base_level = root.count(os.sep)
for root, subdirs, _ in walk(root):
level = root.count(os.sep) - base_level
if depth and level == depth:
del subdirs[:]
subdirs[:] = [d for d in subdirs if not d == api.get_data_root()]
if skip_root and level == 0:
continue
root = util.unipath(root)
data_root = root + '/' + api.get_data_root()
uuid_file = root + '/' + api.get_data_root() + '/' + 'uuid_' + uuid
if os.path.isfile(uuid_file):
yield root, data_root, uuid_file
if direction == UP:
level = -1
next_root = util.unipath(root)
while True:
root = next_root
level += 1
if depth and level > depth:
break
if skip_root and level == 0:
next_root = os.path.dirname(root)
continue
data_root = root + '/' + api.get_data_root()
uuid_path = root + '/' + api.get_data_root() + '/' + 'uuid_' + uuid
if os.path.isfile(uuid_path):
yield root, data_root, uuid_file
next_root = os.path.dirname(root)
if next_root == root:
break
def test_move_entry_raises(tempdir):
'''Copy or Move Entry to existing location raises'''
entry_path = util.unipath(tempdir, 'entry')
dest_path = util.unipath(tempdir, 'move')
os.makedirs(dest_path)
entry = fsfs.get_entry(entry_path)
entry.tag('generic')
assert_raises(OSError, entry.move, dest_path)
assert_raises(OSError, entry.copy, dest_path)
def test_copy_entry(tempdir):
'''Copy Entry with files'''
entry_path = util.unipath(tempdir, 'entry')
entry_copy_path = util.unipath(tempdir, 'copy')
entry, manifest, data_manifest = random_entry(entry_path)
new_entry = entry.copy(entry_copy_path)
assert new_entry is fsfs.get_entry(entry_copy_path)
assert new_entry.uuid != entry.uuid
for file in manifest + data_manifest:
assert os.path.exists(util.unipath(new_entry.path, file))
def test_new_uuid_after_copy(tempdir):
'''Copy Entry creates a new uuid'''
entry_path = util.unipath(tempdir, 'entry')
dest_path = util.unipath(tempdir, 'copy')
entry = fsfs.get_entry(entry_path)
entry.tag('generic')
new_entry = entry.copy(dest_path)
assert new_entry.path == dest_path
assert new_entry.uuid != entry.uuid
assert len(glob.glob(new_entry.data.path + '/uuid_*')) == 1
def delete(self, remove_root=False):
'''Delete an entry
Arguments:
remove_root (bool): Remove root directory and all it's contents not
just the data directory that marks this as an Entry
'''
self.data.delete()
self.data = EntryData(self, util.unipath(self.path,
api.get_data_root()))
if remove_root:
# Delete children depth-first making sure we send all
# appropriate signals along the way
for child in list(self.children())[::-1]:
child.delete(remove_root=remove_root)
try:
shutil.rmtree(self.path)
except OSError as e:
if e.errno != errno.EEXIST:
raise
self.deleted.send(self)
def select_from_tree(root, selector, sep=DEFAULT_SELECTOR_SEP, direction=DOWN,
depth=None, skip_root=False, data_root=None):
'''This method is used under the hood by the Search class, you shouldn't
need to call it manually.
Walk tree up or down yielding entries that match the selector.
A selector is a string representing a hierarchy of names. The selector
`my_parent/my_entry` would match an Entry named `my_entry` within an Entry
named `my_parent`. You can use a custom separator by passing the
keyword argument `sep`.
Arguments:
root (str): Directory to search
selector (str): Hierarchy of names separated by sep
sep (str): Separator used to split selector
direction (int): Direction to search (fsfs.UP or fsfs.DOWN)
depth (int): Maximum depth of search
skip_root (bool): Skip search in root directory
Returns:
generator: yielding :class:`models.Entry` matches
'''
selector = selector.split(sep)
root = util.unipath(root)
data_root = data_root or api.get_data_root()
if direction == DOWN:
depth = depth or DEFAULT_SEARCH_DN_DEPTH
return _select_tree_dn(root, selector, data_root, depth)
elif direction == UP:
depth = depth or DEFAULT_SEARCH_UP_DEPTH
return _select_tree_up(root, selector, data_root, depth)
else:
raise RuntimeError('Invalid direction: ' + str(direction))
def _select_tree_dn(root, selector, data_root, depth, gap=0):
dirs = {
e.name: e.path
for e in safe_scandir(root) if e.is_dir()
}
if dirs.pop(data_root, None):
gap = 0
sel = selector[0]
if sel in os.path.basename(root):
selector.pop(0)
if not selector:
yield api.get_entry(util.unipath(root))
return
if gap == depth:
return
for dir in dirs.values():
yield _select_tree_dn(
dir,
list(selector),
data_root,
depth,
gap + 1
)
def search(root, direction=DOWN, depth=None, levels=None, skip_root=False):
'''Search a root directory yielding Entry objects. You can specify a
direction to search (fsfs.UP or fsfs.DOWN) and a maximum search depth.
Arguments:
root (str): Directory to search
direction (int): Direction to search (fsfs.UP or fsfs.DOWN)
depth (int): Maximum depth of search
skip_root (bool): Skip search in root directory
Returns:
generator: yielding :class:`models.Entry` matches
'''
kwargs = dict(
root=util.unipath(root),
data_root=api.get_data_root(),
skip_root=skip_root
)
if direction == DOWN:
kwargs['depth'] = depth or DEFAULT_SEARCH_DN_DEPTH
kwargs['levels'] = levels or DEFAULT_SEARCH_DN_LEVELS
return _search_dn(**kwargs)
elif direction == UP:
kwargs['levels'] = levels or DEFAULT_SEARCH_UP_LEVELS
return _search_up(**kwargs)
else:
raise RuntimeError('Invalid direction: ' + str(direction))
def _search_dn(root, depth=DEFAULT_SEARCH_DN_DEPTH, gap=0,
levels=DEFAULT_SEARCH_DN_LEVELS, level=0,
skip_root=False, at_root=True, data_root=None):
dirs = {
e.name: e.path
for e in safe_scandir(root) if e.is_dir()
}
if dirs.pop(data_root, None):
gap = 0
if not (skip_root and at_root):
level += 1
yield api.get_entry(util.unipath(root))
if gap == depth or (levels and level == levels):
return
for dir in dirs.values():
yield _search_dn(
dir,
depth,
gap + 1,
levels,
level,
skip_root,
False,
data_root
)
def test_move_entry(tempdir):
'''Move Entry'''
entry_path = util.unipath(tempdir, 'entry')
entry_move_path = util.unipath(tempdir, 'move')
entry, manifest, data_manifest = random_entry(entry_path)
old_uuid = entry.uuid
entry.move(entry_move_path)
assert entry.name == 'move'
assert entry.path != entry_path
assert entry.path == entry_move_path
assert entry is fsfs.get_entry(entry_move_path)
assert entry.uuid == old_uuid
for file in manifest + data_manifest:
assert os.path.exists(util.unipath(entry.path, file))
def quick_select(root, selector, sep=DEFAULT_SELECTOR_SEP,
first_depth=2, depth=4, skip_root=False):
'''Use this method to quickly find one Entry using a selector string.
Unlike search, this method returns one Entry, not a generator yielding
all matches.
Arguments:
root: Directory to search within
selector: List of partial names to select from tree
sep: Separator used to split selector into parts
first_depth: Max directoy depth of first selector
depth: Max directory depth to search for the rest of the selectors
'''
if skip_root:
root = util.unipath(root, '*')
parts = selector.split(sep)
depth = first_depth
matches = []
while parts:
if matches:
root = matches[-1]
part = parts.pop(0)
# glob recursively until we find a match
# root/*, root/*/* ...
for i in range(depth):
pattern = util.unipath(
root,
'*/' * i,
'*' + part + '*',
get_data_root()
)
entries = glob(pattern)
if entries:
match = min(entries, key=len)[:-6]
matches.append(match)
break
else:
return
return get_entry(util.unipath(matches[-1]))
def _set_path(self, path, uuid=None, uuid_file=None):
self.path = path
self.blobs_path = util.unipath(self.path, 'blobs')
self.files_path = util.unipath(self.path, 'files')
self.file = util.unipath(self.path, api.get_data_file())
if not uuid or not uuid_file:
self.uuid = None
self.uuid_file = None
self._find_uuid()
else:
self.uuid = uuid
self.uuid_file = uuid_file
if self._lock and self._lock.acquired:
self._lock.release()
self._lock = lockfile.LockFile(util.unipath(self.path, '.lock'))
def write_blob(self, key, data):
if not os.path.exists(self.blobs_path):
os.makedirs(self.blobs_path)
blob_name = key + '.blob'
with open(util.unipath(self.blobs_path, blob_name), 'wb') as f:
f.write(data)
self._write(**dict(blobs={key: blob_name}))
self.parent.data_changed.send(self.parent, dict(self._data))
def _set_path(self, path, uuid=None, uuid_file=None):
'''Sets this Entry's path. Used by relink to update this Entry's
path, name, and it's associated data path. This shouldn't be necessary
during normal usage.
'''
self.path = path
self.name = os.path.basename(path)
data_path = util.unipath(path, api.get_data_root())
self.data._set_path(data_path, uuid, uuid_file)
def write_file(self, key, file):
if not os.path.exists(self.files_path):
os.makedirs(self.files_path)
file_name = os.path.basename(file)
file_path = util.unipath(self.files_path, file_name)
util.copy_file(file, file_path)
self._write(**dict(files={key: file_name}))
self.parent.data_changed.send(self.parent, dict(self._data))
def test_entryfactory(tempdir):
'''Custom EntryFactory'''
entry_path = util.unipath(tempdir, 'entry')
fsfs.set_entry_factory(CustomFactory)
entry = fsfs.get_entry(entry_path)
# Factory returns cache entry obj
assert entry is fsfs.get_entry(entry_path)
# No tag == default entry
assert type(entry) == CustomFactory.EntryProxy
assert type(entry.obj()) == CustomFactory.Entry
# Add project tag, now we get a Project instance
entry.tag('project')
assert type(entry) == CustomFactory.EntryProxy
assert type(entry.obj()) == CustomFactory.get_type('project')
assert hasattr(entry, 'project_method')
# Remove project tag
entry.untag('project')
assert type(entry) == CustomFactory.EntryProxy
assert type(entry.obj()) == CustomFactory.Entry
assert not hasattr(entry, 'project_method')
# Add asset tag now we get asset methods
entry.tag('asset')
assert type(entry) == CustomFactory.EntryProxy
assert type(entry.obj()) == CustomFactory.get_type('asset')
assert hasattr(entry, 'asset_method')
# Relinked when moved
new_entry_path = util.unipath(tempdir, 'supercool')
assert samefile(entry.path, entry_path)
os.rename(entry.path, new_entry_path)
entry.read() # Triggers entry project to relink entry
assert samefile(entry.path, new_entry_path)
# Restore DefaultFactory
fsfs.set_entry_factory(fsfs.DefaultFactory)
def make_tag_path(root, tag):
'''Tag name to tag filepath
Arguments:
root (str): Directory
tag (str): tag str
Returns:
str: {root}/{get_data_path()}/tag_{tag}
'''
return util.unipath(root, get_data_root(), 'tag_' + tag)
def tag(root, *tags):
'''Tag a directory as an Entry with the provided tags.
Arguments:
root (str): Directory to tag
*tags (List[str]): Tags to add to directory like: 'asset' or 'project'
'''
if not tags:
raise Exception('Must provide at least one tag.')
entry = get_entry(util.unipath(root))
entry.tag(*tags)
def untag(root, *tags):
'''Remove a tag from a directory.
Arguments:
root (str): Directory to remove tags from
*tags (List[str]): Tags to remove like: 'asset' or 'project'
'''
if not tags:
raise Exception('Must provide at least one tag.')
entry = get_entry(util.unipath(root))
entry.untag(*tags)
def random_entry(entry_path):
files = [fake_name() for _ in range(4)]
data_globs = ['/'.join([fsfs.get_data_root(), 'globs', fake_name()])
for _ in range(4)]
data_files = ['/'.join([fsfs.get_data_root(), 'files', fake_name()])
for _ in range(4)]
children = {
fake_name(): {
'files': [fake_name() for _ in range(4)],
'data_globs': [fake_name() for _ in range(4)],
'data_files': [fake_name() for _ in range(4)],
}
for _ in range(4)
}
data_manifest = data_globs + data_files
manifest = files
for child, data in children.items():
for f in data['files']:
manifest.append('/'.join([child, f]))
for f in data['data_files'] + data['data_globs']:
data_manifest.append('/'.join([child, fsfs.get_data_root(), f]))
entry = fsfs.get_entry(entry_path)
entry.tag('parent')
for child_entry_path in children:
child_entry = fsfs.get_entry(
util.unipath(entry.path, child_entry_path)
)
child_entry.tag('child')
for file in manifest + data_manifest:
util.touch(util.unipath(entry.path, file))
return entry, manifest, data_manifest
def test_custom_uuid(tempdir):
'''Assign custom uuid using Entry.uuid setter'''
path = util.unipath(tempdir, 'entry')
entry = fsfs.get_entry(path)
entry.tag('generic')
old_uuid = entry.uuid
new_uuid = 'custom_uuid'
entry.uuid = new_uuid
assert entry.uuid != old_uuid
assert entry.uuid == new_uuid
def get_tags(root):
'''Get the directory's tags
Arguments:
root (str): Directory to get tags from
Returns:
list: List of tags
'''
tags = []
path = util.unipath(root, get_data_root())
if os.path.isdir(path):
for entry in scandir(path):
if entry.name.startswith('tag_'):
tags.append(entry.name.replace('tag_', ''))
return tags
def test_id_generator(tempdir):
'''Custom id generator'''
def make_id(count=[0]):
_id = str(count[0])
count[0] += 1
return _id
fsfs.set_id_generator(make_id)
generated = []
for i in range(10):
e = fsfs.get_entry(util.unipath(tempdir, 'entry_' + str(i)))
e.tag('generic')
generated.append(e)
for i, e in enumerate(generated):
assert e.uuid == str(i)
fsfs.set_default_policy()
def copy(self, dest, only_data=False):
'''Copy this Entry and it's children to a new location
Arguments:
dest (str): Destination path for new Entry
only_data (bool): Copy only Entry data, includes no files outside
the Entry's data directories
Raises:
OSError: Raised when dest already exists or copy_tree fails.
Entry is left in original location, any files partially copied
will be removed.
'''
if os.path.exists(dest):
raise OSError('Can not copy Entry to existing location...')
try:
if not only_data:
util.copy_tree(self.path, dest, force=True, overwrite=True)
else:
hierarchy = [self] + list(self.children())
for entry in hierarchy:
old_data_path = entry.data.path
rel_data_path = os.path.relpath(old_data_path, self.path)
new_data_path = util.unipath(dest, rel_data_path)
util.copy_tree(old_data_path, new_data_path)
except:
if os.path.exists(dest):
shutil.rmtree(dest)
raise
# Update uuids and send EntryCreated signals
new_entry = api.get_entry(dest)
new_entry.data._set_uuid()
new_entry.created.send(new_entry)
for child in new_entry.children():
child.data._set_uuid()
child.created.send(child)
return new_entry
def _make_uuid_path(self, uuid):
return util.unipath(self.path, 'uuid_' + uuid)
def __init__(self, path):
self.path = path
self.name = os.path.basename(path)
self.data = EntryData(self, util.unipath(path, api.get_data_root()))
def read_file(self, key):
data = self._read()
files = data.setdefault('files', {})
file_path = util.unipath(self.files_path, files[key])
return types.File(file_path, mode='rb')
def _make_tag_path(self, tag):
return util.unipath(self.path, 'tag_' + tag)
def read_blob(self, key):
data = self._read()
blobs = data.setdefault('blobs', {})
blob_path = util.unipath(self.blobs_path, blobs[key])
return types.File(blob_path, mode='rb')
