def test_render_tmpfs_ok(report_data, account_factory, report_factory): tmp_fs = TempFS() tmp_fs.makedirs('package/report') with tmp_fs.open('package/report/template.html.j2', 'w') as fp: fp.write(''' <html> <head><title>PDF Report</title></head> <body> <ul> {% for item in data %} <li>{{item[0]}} {{item[1]}}</li> {% endfor %} </ul> </body> </html> ''') renderer = PDFRenderer( 'runtime', tmp_fs.root_path, account_factory(), report_factory(), template='package/report/template.html.j2', ) data = report_data(2, 2) path_to_output = f'{tmp_fs.root_path}/package/report/report' output_file = renderer.render(data, path_to_output) assert output_file == f'{path_to_output}.zip' with ZipFile(output_file) as zip_file: assert sorted(zip_file.namelist()) == ['report.pdf', 'summary.json'] with zip_file.open('report.pdf', 'r') as fp: assert 'PDF Report' in str(fp.read())
def test_render_tmpfs_ok(account_factory, report_factory, report_data): tmp_fs = TempFS() tmp_fs.makedirs('package/report') wb = Workbook() ws = wb.active ws.title = 'Data' ws.cell(1, 1, value='Name') ws.cell(1, 2, value='Description') wb.save(f'{tmp_fs.root_path}/package/report/template.xlsx') renderer = XLSXRenderer( 'runtime', tmp_fs.root_path, account_factory(), report_factory(), template='package/report/template.xlsx', ) data = report_data(2, 2) path_to_output = f'{tmp_fs.root_path}/package/report/report' output_file = renderer.render(data, path_to_output, start_time=datetime.now()) wb = load_workbook(output_file) ws = wb['Data'] assert output_file == f'{path_to_output}.xlsx' assert data == [[ws[f'A{item}'].value, ws[f'B{item}'].value] for item in range(2, 4)]
def _get_tmpfs_with_readme_and_entry(entrypoint): tmp_fs = TempFS() tmp_fs.create('readme.md') *dirpath, filename = entrypoint.split('.') package_path = '/'.join(dirpath) script_path = f'{package_path}/{filename}.py' tmp_fs.makedirs(package_path) tmp_fs.create(script_path) return tmp_fs
def test_validate_tmpfs_css_missing(): tmp_fs = TempFS() tmp_fs.makedirs('package/report') tmp_fs.create('package/report/template.html.j2') definition = RendererDefinition( root_path=tmp_fs.root_path, id='renderer_id', type='pdf', description='description', template='package/report/template.html.j2', args={'css_file': 'package/report/css_file.css'}, ) errors = PDFRenderer.validate(definition) assert f"css_file `{definition.args['css_file']}` not found." == errors[0]
def test_validate_tmpfs_template_wrong_name(): tmp_fs = TempFS() tmp_fs.makedirs('package/report') tmp_fs.create('package/report/template.html.j3') tmp_fs.create('css_file.css') definition = RendererDefinition( root_path=tmp_fs.root_path, id='renderer_id', type='pdf', description='description', template='package/report/template.html.j3', args={'css_file': 'css_file.css'}, ) errors = PDFRenderer.validate(definition) assert f"invalid template name: `{definition.template}`" in errors[0]
def test_render_tempfs_ok(report_data, account_factory, report_factory, extra_context): tmp_fs = TempFS() template_filename = 'template.csv.j2' directory_structure = 'package/report' tmp_fs.makedirs(directory_structure) path_to_template = f'{tmp_fs.root_path}/{directory_structure}/{template_filename}' with open(path_to_template, 'w') as fp: fp.write('"name";"description"\n') fp.write('{% for item in data %}') fp.write('"{{item[0]}}";"{{item[1]}}"\n') fp.write('{% endfor %}') account = account_factory() report = report_factory() renderer = Jinja2Renderer( 'runtime', tmp_fs.root_path, account, report, f'{directory_structure}/{template_filename}', ) # report_data method generates a matrix with this aspect: # [['row_0_col_0', 'row_0_col_1'], ['row_1_col_0', 'row_1_col_1']] data = report_data(2, 2) renderer.set_extra_context(extra_context) ctx = renderer.get_context(data) path_to_output_file = f'{tmp_fs.root_path}/{directory_structure}/report' output_file = renderer.render(data, path_to_output_file) with ZipFile(output_file) as repzip: assert sorted(repzip.namelist()) == ['report.csv', 'summary.json'] with TextIOWrapper(repzip.open('report.csv'), encoding="utf-8") as fp: csv_reader = csv.reader(fp, delimiter=';') content = [row for row in csv_reader] assert output_file == f'{path_to_output_file}.zip' assert data[0] == content[1] assert data[1] == content[2] if extra_context: assert 'name' in ctx['extra_context'] assert 'desc' in ctx['extra_context']
SHA = "" else: repo = git.Repo(os.path.dirname(os.path.realpath(__file__)), search_parent_directories=True) SHA = repo.head.object.hexsha[0:10] CACHED_CONFIG_FILE = None SERVER_CONFIG_FILE = "mswms_settings.py" MSCOLAB_CONFIG_FILE = "mscolab_settings.py" ROOT_FS = TempFS(identifier=f"msui{SHA}") OSFS_URL = ROOT_FS.geturl("", purpose="fs") ROOT_DIR = ROOT_FS.getsyspath("") if not ROOT_FS.exists("msui/testdata"): ROOT_FS.makedirs("msui/testdata") SERVER_CONFIG_FS = fs.open_fs(fs.path.join(ROOT_DIR, "msui")) DATA_FS = fs.open_fs(fs.path.join(ROOT_DIR, "msui/testdata")) MSUI_CONFIG_PATH = OSFS_URL # MSUI_CONFIG_PATH = SERVER_CONFIG_FS.getsyspath("") would use a none osfs path os.environ["MSUI_CONFIG_PATH"] = MSUI_CONFIG_PATH SERVER_CONFIG_FILE_PATH = fs.path.join(SERVER_CONFIG_FS.getsyspath(""), SERVER_CONFIG_FILE) # we keep DATA_DIR until we move netCDF4 files to pyfilesystem2 DATA_DIR = DATA_FS.getsyspath("") # deployed mscolab url MSCOLAB_URL = "http://localhost:8083" # mscolab test server's url
class TestCOWFS(FSTestCases, unittest.TestCase): def make_fs(self) -> COWFS: self.tempfs = TempFS() return COWFS(self.tempfs) def test_makedir_bug(self) -> None: # These two commands disclosed a bug in COWFS.makedir(). This # serves as a regression test. self.tempfs.makedir("/b$") self.fs.makedir("/b$/c$") self.fs.invariant() def test_openbin_bug(self) -> None: # These two commands disclosed a bug in COWFS.openbin(). This # serves as a regression test. self.tempfs.makedir("/b$") self.fs.writetext("/b$/c.txt", "Buggy?") self.fs.invariant() def test_listdir_bug(self) -> None: # Removing a file disclosed a bug in COWFS.listdir(). This # serves as a regression test. self.tempfs.makedirs("/b/d") self.tempfs.writetext("/b/d/c1.txt", "hmmm") self.tempfs.writetext("/b/d/c2.txt", "hmmm") self.tempfs.writetext("/b/d/c3.txt", "hmmm") self.fs.remove("/b/d/c1.txt") self.assertEqual({"c2.txt", "c3.txt"}, set(self.fs.listdir("/b/d"))) def test_listdir(self) -> None: fs = MemoryFS() fs.makedirs("/b$") fs.makedirs("/b$/dir1") fs.makedirs("/b$/dir2") fs.writetext("/b$/file1.txt", "file1") fs.writetext("/b$/file2.txt", "file2") fs.writetext("/b$/dir1/file1.txt", "file1") fs.writetext("/b$/dir1/file2.txt", "file2") fs.writetext("/b$/dir2/file1.txt", "file1") fs.writetext("/b$/dir2/file2.txt", "file2") c = COWFS(fs) path = "/b$/dir1/file2.txt" c.writetext(path, "xxxx") # Now the COW version is different. But it should still have # the old unchanged files. self.assertTrue(c.exists("/b$/dir1/file1.txt")) # Yes, but... self.assertEqual({"dir1", "dir2", "file1.txt", "file2.txt"}, set(c.listdir("/b$"))) def test_getsyspath(self) -> None: dirpath = "/b$/dir1" self.tempfs.makedirs(dirpath) filepath = fs.path.join(dirpath, "foo.txt") self.tempfs.writetext(filepath, "original contents") # syspath for a filepath is the same as the syspath in the # basefs. self.assertEqual(self.fs.base_fs.getsyspath(filepath), self.fs.getsyspath(filepath)) # After writing to it, the syspath is now the same as the # syspath in the additions_fs. self.fs.writetext(filepath, "replacement contents") self.assertEqual(self.fs.additions_fs.getsyspath(filepath), self.fs.getsyspath(filepath)) # root raises an exception with self.assertRaises(fs.errors.NoSysPath): self.fs.getsyspath("/")
class DocuService: def __init__(self, file_system, root_path='/'): "docstring" self.fs = file_system.opendir(root_path) self.temp_fs = TempFS() self.fillers = {} self._temp_files = {} def _load_filler(self, key): module = importlib.import_module(_FILLERS[key]) self.fillers[key] = module.Filler(self) def fill(self, data, template_name, docu_path): template_type = template_name.split('.')[-1] doc_type = docu_path.split('.')[-1] filler_key = '{}2{}'.format(template_type, doc_type) if filler_key not in self.fillers: # Lazzy loading self._load_filler(filler_key) filler = self.fillers[filler_key] filler.fill(data, template_name, docu_path) logger.info('filled document {}'.format(docu_path)) def getsyspath(self, path): if path in self._temp_files: return self._temp_files[path] if self.fs.hassyspath(path): return self.fs.getsyspath(path) else: dirname = fs.path.dirname(path) if not self.temp_fs.isdir(dirname): self.temp_fs.makedirs(dirname, recreate=True) fs.copy.copy_file(self.fs, path, self.temp_fs, path) logger.info('Copied {} file to temporary fs'.format(path)) self._temp_files[path] = self.temp_fs.getsyspath(path) return self._temp_files[path] def print_to_cups_printer(self, printer_name, file_path, media='A4', quality=5): """Print to cups printer using command line """ path = self.getsyspath(file_path) command = 'lp -d {} -o media={} -o print-quality={} {}'.format( printer_name, media, quality, path) subprocess.check_call(command, shell=True) def export(self, source_path, fs, destination_path=None): destination_path = destination_path if destination_path else source_path fs.copy.copy_file(self.fs, source_path, fs, destination_path) def __del__(self): self.temp_fs.close() self.fs.close()
class COWFS(FS): def __init__( self, base_fs: FS, additions_fs: Optional[FS] = None, deletions_fs: Optional[FS] = None, ) -> None: FS.__init__(self) if additions_fs: self.additions_fs = additions_fs else: self.additions_fs = TempFS() if deletions_fs: _deletions_invariant(deletions_fs) self.deletions_fs = deletions_fs else: self.deletions_fs = TempFS() self.original_base_fs = base_fs self.base_fs = fs.wrap.read_only(base_fs) self.invariant() @staticmethod def create_cowfs(base_fs: FS, read_write_layer: FS, recreate: bool = False) -> "COWFS": additions_fs = read_write_layer.makedir("/additions", recreate=recreate) deletions_fs = read_write_layer.makedir("/deletions", recreate=recreate) return COWFS(base_fs, additions_fs, deletions_fs) def __str__(self) -> str: return (f"COWFS({self.original_base_fs}, " f"{self.additions_fs}, " f"{self.deletions_fs})") def __repr__(self) -> str: return (f"COWFS({self.original_base_fs!r}, " f"{self.additions_fs!r}, " f"{self.deletions_fs!r})") ############################################################ def invariant(self) -> bool: if not self.additions_fs: raise ValueError(f"Invalid additions_fs: {self.additions_fs}.") if not self.deletions_fs: raise ValueError(f"Invalid deletions_fs: {self.additions_fs}.") if not self.base_fs: raise ValueError(f"Invalid base_fs: {self.base_fs}.") _deletions_invariant(self.deletions_fs) additions_paths = set(paths(self.additions_fs)) deletions_paths = { fs.path.dirname(file) for file in self.deletions_fs.walk.files() } if additions_paths > deletions_paths: raise ValueError(f"Additions_paths {additions_paths} " + "is not a subset of deletions_path " + f"{deletions_paths}. Extras are " + f"{additions_paths - deletions_paths}.") return True def is_deletion(self, path: str) -> bool: """ Is the path marked in the deletions_fs" """ return self.deletions_fs.exists(del_path(path)) def mark_deletion(self, path: str) -> None: """ Mark the path in the deletions_fs. """ self.deletions_fs.makedirs(path, None, True) self.deletions_fs.touch(del_path(path)) def makedirs_mark_deletion( self, path: str, permissions: Optional[Permissions] = None, recreate: bool = False, ) -> None: for p in fs.path.recursepath(path)[:-1]: self.additions_fs.makedirs(p, permissions=permissions, recreate=True) self.mark_deletion(p) self.additions_fs.makedir(path, permissions=permissions, recreate=recreate) self.mark_deletion(path) def layer(self, path: str) -> int: """ Get the layer on which the file lives, or ROOT_LAYER if it's the root path. """ if path == "/": return ROOT_LAYER if self.additions_fs.exists(path): return ADD_LAYER elif self.is_deletion(path): return NO_LAYER elif self.base_fs.exists(path): return BASE_LAYER else: return NO_LAYER def copy_up(self, path: str) -> None: """ Copy the file from the base_fs to additions_fs. """ self.makedirs_mark_deletion(fs.path.dirname(path)) self.mark_deletion(path) fs.copy.copy_file(self.base_fs, path, self.additions_fs, path) def triple_tree(self) -> None: print("base_fs ------------------------------") self.base_fs.tree() print("additions_fs ------------------------------") self.additions_fs.tree() print("deletions_fs ------------------------------") self.deletions_fs.tree() ############################################################ def getmeta(self, namespace: str = "standard") -> Mapping[str, object]: return self.base_fs.getmeta(namespace) def getinfo(self, path: str, namespaces: Optional[Collection[str]] = None) -> Info: self.check() self.validatepath(path) layer = self.layer(path) if layer == NO_LAYER: raise fs.errors.ResourceNotFound(path) elif layer == BASE_LAYER: return self.base_fs.getinfo(path, namespaces) elif layer == ADD_LAYER: return self.additions_fs.getinfo(path, namespaces) elif layer == ROOT_LAYER: # TODO implement this raw_info = {} if namespaces is None or "basic" in namespaces: raw_info["basic"] = {"name": "", "is_dir": True} return Info(raw_info) else: raise RuntimeError(f"Unknown layer {layer}.") def getsyspath(self, path: str) -> str: self.check() # self.validatepath(path) layer = self.layer(path) if layer == NO_LAYER: raise fs.errors.NoSysPath(path=path) elif layer == BASE_LAYER: return self.base_fs.getsyspath(path) elif layer == ADD_LAYER: return self.additions_fs.getsyspath(path) elif layer == ROOT_LAYER: raise fs.errors.NoSysPath(path=path) else: raise RuntimeError(f"Unknown layer {layer}.") def listdir(self, path: str) -> List[str]: self.check() self.validatepath(path) layer = self.layer(path) if layer == NO_LAYER: raise fs.errors.ResourceNotFound(path) elif layer == BASE_LAYER: return [ name for name in self.base_fs.listdir(path) if self.layer(fs.path.join(path, name)) != NO_LAYER ] elif layer == ADD_LAYER: # Get the listing on the additions layer names = set(self.additions_fs.listdir(path)) # Add in the listing on the base layer (if it exists) if self.base_fs.isdir(path): names |= set(self.base_fs.listdir(path)) # Return the entries that actually exist return [ name for name in list(names) if self.layer(fs.path.join(path, name)) != NO_LAYER ] elif layer == ROOT_LAYER: # Get the listing of the root on the additions layer and # the base layer. names = set(self.additions_fs.listdir("/")) names |= set(self.base_fs.listdir("/")) # Return the entries that actually exist. return [ name for name in list(names) if self.layer(name) != NO_LAYER ] else: raise RuntimeError(f"Unknown layer {layer}.") def makedir( self, path: str, permissions: Optional[Permissions] = None, recreate: bool = False, ) -> SubFS["COWFS"]: self.check() self.validatepath(path) # Check if it *can* be created. # get a normalized parent_dir path. parent_dir = fs.path.dirname(fs.path.forcedir(path)[:-1]) if not parent_dir: parent_dir = "/" if not self.isdir(parent_dir): raise fs.errors.ResourceNotFound(path) layer = self.layer(path) if layer == NO_LAYER: self.makedirs_mark_deletion(path, permissions=permissions, recreate=recreate) return SubFS(self, path) elif layer in [BASE_LAYER, ADD_LAYER, ROOT_LAYER]: if recreate: return SubFS(self, path) else: # I think this is wrong. What if it's a file? raise fs.errors.DirectoryExists(path) else: raise RuntimeError(f"Unknown layer {layer}.") def openbin(self, path: str, mode: str = "r", buffering: int = -1, **options: Any) -> BinaryIO: self.check() self.validatepath(path) parent_dir = fs.path.dirname(fs.path.forcedir(path)[:-1]) if not parent_dir: parent_dir = "/" if not self.isdir(parent_dir): raise fs.errors.ResourceNotFound(path) mode_obj = Mode(mode) layer = self.layer(path) if layer == NO_LAYER: if mode_obj.create: for p in fs.path.recursepath(path)[:-1]: self.additions_fs.makedirs(p, recreate=True) self.mark_deletion(p) self.mark_deletion(path) return self.additions_fs.openbin(path, mode, buffering, **options) else: raise fs.errors.ResourceNotFound(path) elif layer == ADD_LAYER: self.mark_deletion(path) return self.additions_fs.openbin(path, mode, buffering, **options) elif layer == BASE_LAYER: if mode_obj.writing: self.copy_up(path) return self.additions_fs.openbin(path, mode, buffering, **options) else: return self.base_fs.openbin(path, mode, buffering, **options) elif layer == ROOT_LAYER: raise fs.errors.FileExpected(path) else: raise RuntimeError(f"Unknown layer {layer}.") def remove(self, path: str) -> None: self.check() self.validatepath(path) layer = self.layer(path) if layer == NO_LAYER: raise fs.errors.ResourceNotFound(path) elif layer == BASE_LAYER: if self.base_fs.isfile(path): self.mark_deletion(path) else: raise fs.errors.FileExpected(path) elif layer == ADD_LAYER: self.additions_fs.remove(path) self.mark_deletion(path) elif layer == ROOT_LAYER: raise fs.errors.FileExpected(path) else: raise RuntimeError(f"Unknown layer {layer}.") def removedir(self, path: str) -> None: self.check() layer = self.layer(path) if layer == NO_LAYER: raise fs.errors.ResourceNotFound(path) elif layer == BASE_LAYER: if self.base_fs.isdir(path): self.mark_deletion(path) else: raise fs.errors.FileExpected(path) elif layer == ADD_LAYER: if self.additions_fs.isdir(path): self.additions_fs.removedir(path) self.mark_deletion(path) else: raise fs.errors.DirectoryExpected(path) elif layer == ROOT_LAYER: raise fs.errors.RemoveRootError(path) else: raise RuntimeError(f"Unknown layer {layer}.") def setinfo(self, path: str, info: _INFO_DICT) -> None: self.check() self.validatepath(path) layer = self.layer(path) if layer == NO_LAYER: raise fs.errors.ResourceNotFound(path) elif layer == BASE_LAYER: self.copy_up(path) self.additions_fs.setinfo(path, info) elif layer == ADD_LAYER: self.additions_fs.setinfo(path, info) elif layer == ROOT_LAYER: pass else: raise RuntimeError(f"Unknown layer {layer}.") ############################################################ def makedirs( self, path: str, permissions: Optional[Permissions] = None, recreate: bool = False, ) -> SubFS[FS]: return FS.makedirs(self, path, permissions=permissions, recreate=recreate)