def scan_directory(self, directory_key="") -> dict: url = self.url(directory_key) if not self.s3.exists(url): raise Exception(f"S3 url '{url}' does not exist.") s3key_map = dict( (self.remove_bucket(filepath), "s3://" + filepath) for filepath in sorted(self.s3.ls(url)) ) def is_annotation_dir(dir_s3key): return ( dir_s3key.endswith(self.annotation_dir_suffix) and self.convert_annotation_key_to_h5ad(dir_s3key) in h5ad_keys ) h5ad_keys = [ filepath for filepath, item_url in s3key_map.items() if self.is_h5ad_url(item_url) ] subdir_keys = [ filepath for filepath, item_url in s3key_map.items() if self.s3.isdir(item_url) and not is_annotation_dir(filepath) ] items = [self.make_s3item_from_key(basename(key), key) for key in h5ad_keys] branches = None if len(subdir_keys) > 0: branches = [self.scan_directory(key) for key in subdir_keys] return ItemTree(directory_key, items, branches)
def test_GIVEN_deep_nested_dirs_THEN_includes_dirs_in_output(self, item_source): item_source.name = "FakeSource" item_tree = ItemTree("foo/bar/baz", [], []) rendered = render_item_tree(item_tree, item_source) self.assertEqual( rendered, "<li><a href='/filecrawl/foo/bar/baz?source=FakeSource'>baz</a><ul></ul></li>", )
def test_GIVEN_some_filter_THEN_includes_filterpart_in_heading(self, item_source): item_source.name = "FakeSource" item_source.list_items.return_value = ItemTree("rootdir", [], []) rendered = render_item_source(item_source, "some_filter") self.assertEqual( rendered, "<h6><a href='/filecrawl.html?source=FakeSource'>FakeSource</a>:some_filter</h6><li><a href='/filecrawl/rootdir?source=FakeSource'>rootdir</a><ul></ul></li>", )
def scan_directory(self, subpath="") -> dict: base_path = os.path.join(self.base_path, subpath) if not os.path.exists(base_path): raise Exception(f"Path for local files '{base_path}' does not exist.") filepath_map = dict( (filepath, os.path.join(base_path, filepath)) for filepath in sorted(os.listdir(base_path)) ) def is_annotation_dir(dir): return ( dir.endswith(self.annotation_dir_suffix) and self.convert_annotation_path_to_h5ad(dir) in h5ad_paths ) h5ad_paths = [ filepath for filepath, full_path in filepath_map.items() if self.is_h5ad_file(full_path) ] subdirs = [ filepath for filepath, full_path in filepath_map.items() if os.path.isdir(full_path) and not is_annotation_dir(filepath) ] items = [ self.make_fileitem_from_path(filename, subpath) for filename in h5ad_paths ] branches = None if len(subdirs) > 0: branches = [ self.scan_directory(os.path.join(subpath, subdir)) for subdir in subdirs ] return ItemTree(subpath, items, branches)
def scan_directory(self, subpath="") -> dict: url = self.url(subpath) if not self.s3.exists(url): raise Exception(f"S3 url '{url}' does not exist.") s3key_map = dict( (filepath[len(self.bucket):].lstrip("/"), "s3://" + filepath) for filepath in sorted(self.s3.ls(url))) def is_annotation_dir(dir_s3key): return (dir_s3key.endswith(self.annotation_dir_suffix) and self.convert_annotation_key_to_h5ad(dir_s3key) in h5ad_paths) h5ad_paths = [ filepath for filepath, item_url in s3key_map.items() if self.is_h5ad_url(item_url) ] subdirs = [ filepath for filepath, item_url in s3key_map.items() if self.s3.isdir(item_url) and not is_annotation_dir(filepath) ] items = [ self.make_s3item_from_key(filename, join(subpath, filename)) for filename in h5ad_paths ] branches = None if len(subdirs) > 0: branches = [ self.scan_directory(join(subpath, subdir)) for subdir in subdirs ] return ItemTree(subpath, items, branches)