def scan_directory(self, directory_key="") -> dict:
url = self.url(directory_key)
if not self.s3.exists(url):
raise Exception(f"S3 url '{url}' does not exist.")
s3key_map = dict(
(self.remove_bucket(filepath), "s3://" + filepath)
for filepath in sorted(self.s3.ls(url))
)
def is_annotation_dir(dir_s3key):
return (
dir_s3key.endswith(self.annotation_dir_suffix)
and self.convert_annotation_key_to_h5ad(dir_s3key) in h5ad_keys
)
h5ad_keys = [
filepath
for filepath, item_url in s3key_map.items()
if self.is_h5ad_url(item_url)
]
subdir_keys = [
filepath
for filepath, item_url in s3key_map.items()
if self.s3.isdir(item_url) and not is_annotation_dir(filepath)
]
items = [self.make_s3item_from_key(basename(key), key) for key in h5ad_keys]
branches = None
if len(subdir_keys) > 0:
branches = [self.scan_directory(key) for key in subdir_keys]
return ItemTree(directory_key, items, branches)
def test_GIVEN_deep_nested_dirs_THEN_includes_dirs_in_output(self, item_source):
item_source.name = "FakeSource"
item_tree = ItemTree("foo/bar/baz", [], [])
rendered = render_item_tree(item_tree, item_source)
self.assertEqual(
rendered,
"<li><a href='/filecrawl/foo/bar/baz?source=FakeSource'>baz</a><ul></ul></li>",
)
def test_GIVEN_some_filter_THEN_includes_filterpart_in_heading(self, item_source):
item_source.name = "FakeSource"
item_source.list_items.return_value = ItemTree("rootdir", [], [])
rendered = render_item_source(item_source, "some_filter")
self.assertEqual(
rendered,
"<h6><a href='/filecrawl.html?source=FakeSource'>FakeSource</a>:some_filter</h6><li><a href='/filecrawl/rootdir?source=FakeSource'>rootdir</a><ul></ul></li>",
)
def scan_directory(self, subpath="") -> dict:
base_path = os.path.join(self.base_path, subpath)
if not os.path.exists(base_path):
raise Exception(f"Path for local files '{base_path}' does not exist.")
filepath_map = dict(
(filepath, os.path.join(base_path, filepath))
for filepath in sorted(os.listdir(base_path))
)
def is_annotation_dir(dir):
return (
dir.endswith(self.annotation_dir_suffix)
and self.convert_annotation_path_to_h5ad(dir) in h5ad_paths
)
h5ad_paths = [
filepath
for filepath, full_path in filepath_map.items()
if self.is_h5ad_file(full_path)
]
subdirs = [
filepath
for filepath, full_path in filepath_map.items()
if os.path.isdir(full_path) and not is_annotation_dir(filepath)
]
items = [
self.make_fileitem_from_path(filename, subpath) for filename in h5ad_paths
]
branches = None
if len(subdirs) > 0:
branches = [
self.scan_directory(os.path.join(subpath, subdir)) for subdir in subdirs
]
return ItemTree(subpath, items, branches)
def scan_directory(self, subpath="") -> dict:
url = self.url(subpath)
if not self.s3.exists(url):
raise Exception(f"S3 url '{url}' does not exist.")
s3key_map = dict(
(filepath[len(self.bucket):].lstrip("/"), "s3://" + filepath)
for filepath in sorted(self.s3.ls(url)))
def is_annotation_dir(dir_s3key):
return (dir_s3key.endswith(self.annotation_dir_suffix)
and self.convert_annotation_key_to_h5ad(dir_s3key)
in h5ad_paths)
h5ad_paths = [
filepath for filepath, item_url in s3key_map.items()
if self.is_h5ad_url(item_url)
]
subdirs = [
filepath for filepath, item_url in s3key_map.items()
if self.s3.isdir(item_url) and not is_annotation_dir(filepath)
]
items = [
self.make_s3item_from_key(filename, join(subpath, filename))
for filename in h5ad_paths
]
branches = None
if len(subdirs) > 0:
branches = [
self.scan_directory(join(subpath, subdir))
for subdir in subdirs
]
return ItemTree(subpath, items, branches)