def test_render_tmpfs_ok(report_data, account_factory, report_factory):
tmp_fs = TempFS()
tmp_fs.makedirs('package/report')
with tmp_fs.open('package/report/template.html.j2', 'w') as fp:
fp.write('''
<html>
<head><title>PDF Report</title></head>
<body>
<ul>
{% for item in data %}
<li>{{item[0]}} {{item[1]}}</li>
{% endfor %}
</ul>
</body>
</html>
''')
renderer = PDFRenderer(
'runtime',
tmp_fs.root_path,
account_factory(),
report_factory(),
template='package/report/template.html.j2',
)
data = report_data(2, 2)
path_to_output = f'{tmp_fs.root_path}/package/report/report'
output_file = renderer.render(data, path_to_output)
assert output_file == f'{path_to_output}.zip'
with ZipFile(output_file) as zip_file:
assert sorted(zip_file.namelist()) == ['report.pdf', 'summary.json']
with zip_file.open('report.pdf', 'r') as fp:
assert 'PDF Report' in str(fp.read())
def test_render_tmpfs_ok(account_factory, report_factory, report_data):
tmp_fs = TempFS()
tmp_fs.makedirs('package/report')
wb = Workbook()
ws = wb.active
ws.title = 'Data'
ws.cell(1, 1, value='Name')
ws.cell(1, 2, value='Description')
wb.save(f'{tmp_fs.root_path}/package/report/template.xlsx')
renderer = XLSXRenderer(
'runtime',
tmp_fs.root_path,
account_factory(),
report_factory(),
template='package/report/template.xlsx',
)
data = report_data(2, 2)
path_to_output = f'{tmp_fs.root_path}/package/report/report'
output_file = renderer.render(data,
path_to_output,
start_time=datetime.now())
wb = load_workbook(output_file)
ws = wb['Data']
assert output_file == f'{path_to_output}.xlsx'
assert data == [[ws[f'A{item}'].value, ws[f'B{item}'].value]
for item in range(2, 4)]
def _get_tmpfs_with_readme_and_entry(entrypoint):
tmp_fs = TempFS()
tmp_fs.create('readme.md')
*dirpath, filename = entrypoint.split('.')
package_path = '/'.join(dirpath)
script_path = f'{package_path}/{filename}.py'
tmp_fs.makedirs(package_path)
tmp_fs.create(script_path)
return tmp_fs
def test_validate_tmpfs_css_missing():
tmp_fs = TempFS()
tmp_fs.makedirs('package/report')
tmp_fs.create('package/report/template.html.j2')
definition = RendererDefinition(
root_path=tmp_fs.root_path,
id='renderer_id',
type='pdf',
description='description',
template='package/report/template.html.j2',
args={'css_file': 'package/report/css_file.css'},
)
errors = PDFRenderer.validate(definition)
assert f"css_file `{definition.args['css_file']}` not found." == errors[0]
def test_validate_tmpfs_template_wrong_name():
tmp_fs = TempFS()
tmp_fs.makedirs('package/report')
tmp_fs.create('package/report/template.html.j3')
tmp_fs.create('css_file.css')
definition = RendererDefinition(
root_path=tmp_fs.root_path,
id='renderer_id',
type='pdf',
description='description',
template='package/report/template.html.j3',
args={'css_file': 'css_file.css'},
)
errors = PDFRenderer.validate(definition)
assert f"invalid template name: `{definition.template}`" in errors[0]
def test_render_tempfs_ok(report_data, account_factory, report_factory,
extra_context):
tmp_fs = TempFS()
template_filename = 'template.csv.j2'
directory_structure = 'package/report'
tmp_fs.makedirs(directory_structure)
path_to_template = f'{tmp_fs.root_path}/{directory_structure}/{template_filename}'
with open(path_to_template, 'w') as fp:
fp.write('"name";"description"\n')
fp.write('{% for item in data %}')
fp.write('"{{item[0]}}";"{{item[1]}}"\n')
fp.write('{% endfor %}')
account = account_factory()
report = report_factory()
renderer = Jinja2Renderer(
'runtime',
tmp_fs.root_path,
account,
report,
f'{directory_structure}/{template_filename}',
)
# report_data method generates a matrix with this aspect:
# [['row_0_col_0', 'row_0_col_1'], ['row_1_col_0', 'row_1_col_1']]
data = report_data(2, 2)
renderer.set_extra_context(extra_context)
ctx = renderer.get_context(data)
path_to_output_file = f'{tmp_fs.root_path}/{directory_structure}/report'
output_file = renderer.render(data, path_to_output_file)
with ZipFile(output_file) as repzip:
assert sorted(repzip.namelist()) == ['report.csv', 'summary.json']
with TextIOWrapper(repzip.open('report.csv'), encoding="utf-8") as fp:
csv_reader = csv.reader(fp, delimiter=';')
content = [row for row in csv_reader]
assert output_file == f'{path_to_output_file}.zip'
assert data[0] == content[1]
assert data[1] == content[2]
if extra_context:
assert 'name' in ctx['extra_context']
assert 'desc' in ctx['extra_context']
SHA = ""
else:
repo = git.Repo(os.path.dirname(os.path.realpath(__file__)),
search_parent_directories=True)
SHA = repo.head.object.hexsha[0:10]
CACHED_CONFIG_FILE = None
SERVER_CONFIG_FILE = "mswms_settings.py"
MSCOLAB_CONFIG_FILE = "mscolab_settings.py"
ROOT_FS = TempFS(identifier=f"msui{SHA}")
OSFS_URL = ROOT_FS.geturl("", purpose="fs")
ROOT_DIR = ROOT_FS.getsyspath("")
if not ROOT_FS.exists("msui/testdata"):
ROOT_FS.makedirs("msui/testdata")
SERVER_CONFIG_FS = fs.open_fs(fs.path.join(ROOT_DIR, "msui"))
DATA_FS = fs.open_fs(fs.path.join(ROOT_DIR, "msui/testdata"))
MSUI_CONFIG_PATH = OSFS_URL
# MSUI_CONFIG_PATH = SERVER_CONFIG_FS.getsyspath("") would use a none osfs path
os.environ["MSUI_CONFIG_PATH"] = MSUI_CONFIG_PATH
SERVER_CONFIG_FILE_PATH = fs.path.join(SERVER_CONFIG_FS.getsyspath(""),
SERVER_CONFIG_FILE)
# we keep DATA_DIR until we move netCDF4 files to pyfilesystem2
DATA_DIR = DATA_FS.getsyspath("")
# deployed mscolab url
MSCOLAB_URL = "http://localhost:8083"
# mscolab test server's url
class TestCOWFS(FSTestCases, unittest.TestCase):
def make_fs(self) -> COWFS:
self.tempfs = TempFS()
return COWFS(self.tempfs)
def test_makedir_bug(self) -> None:
# These two commands disclosed a bug in COWFS.makedir(). This
# serves as a regression test.
self.tempfs.makedir("/b$")
self.fs.makedir("/b$/c$")
self.fs.invariant()
def test_openbin_bug(self) -> None:
# These two commands disclosed a bug in COWFS.openbin(). This
# serves as a regression test.
self.tempfs.makedir("/b$")
self.fs.writetext("/b$/c.txt", "Buggy?")
self.fs.invariant()
def test_listdir_bug(self) -> None:
# Removing a file disclosed a bug in COWFS.listdir(). This
# serves as a regression test.
self.tempfs.makedirs("/b/d")
self.tempfs.writetext("/b/d/c1.txt", "hmmm")
self.tempfs.writetext("/b/d/c2.txt", "hmmm")
self.tempfs.writetext("/b/d/c3.txt", "hmmm")
self.fs.remove("/b/d/c1.txt")
self.assertEqual({"c2.txt", "c3.txt"}, set(self.fs.listdir("/b/d")))
def test_listdir(self) -> None:
fs = MemoryFS()
fs.makedirs("/b$")
fs.makedirs("/b$/dir1")
fs.makedirs("/b$/dir2")
fs.writetext("/b$/file1.txt", "file1")
fs.writetext("/b$/file2.txt", "file2")
fs.writetext("/b$/dir1/file1.txt", "file1")
fs.writetext("/b$/dir1/file2.txt", "file2")
fs.writetext("/b$/dir2/file1.txt", "file1")
fs.writetext("/b$/dir2/file2.txt", "file2")
c = COWFS(fs)
path = "/b$/dir1/file2.txt"
c.writetext(path, "xxxx")
# Now the COW version is different. But it should still have
# the old unchanged files.
self.assertTrue(c.exists("/b$/dir1/file1.txt")) # Yes, but...
self.assertEqual({"dir1", "dir2", "file1.txt", "file2.txt"},
set(c.listdir("/b$")))
def test_getsyspath(self) -> None:
dirpath = "/b$/dir1"
self.tempfs.makedirs(dirpath)
filepath = fs.path.join(dirpath, "foo.txt")
self.tempfs.writetext(filepath, "original contents")
# syspath for a filepath is the same as the syspath in the
# basefs.
self.assertEqual(self.fs.base_fs.getsyspath(filepath),
self.fs.getsyspath(filepath))
# After writing to it, the syspath is now the same as the
# syspath in the additions_fs.
self.fs.writetext(filepath, "replacement contents")
self.assertEqual(self.fs.additions_fs.getsyspath(filepath),
self.fs.getsyspath(filepath))
# root raises an exception
with self.assertRaises(fs.errors.NoSysPath):
self.fs.getsyspath("/")
class DocuService:
def __init__(self, file_system, root_path='/'):
"docstring"
self.fs = file_system.opendir(root_path)
self.temp_fs = TempFS()
self.fillers = {}
self._temp_files = {}
def _load_filler(self, key):
module = importlib.import_module(_FILLERS[key])
self.fillers[key] = module.Filler(self)
def fill(self, data, template_name, docu_path):
template_type = template_name.split('.')[-1]
doc_type = docu_path.split('.')[-1]
filler_key = '{}2{}'.format(template_type, doc_type)
if filler_key not in self.fillers: # Lazzy loading
self._load_filler(filler_key)
filler = self.fillers[filler_key]
filler.fill(data, template_name, docu_path)
logger.info('filled document {}'.format(docu_path))
def getsyspath(self, path):
if path in self._temp_files:
return self._temp_files[path]
if self.fs.hassyspath(path):
return self.fs.getsyspath(path)
else:
dirname = fs.path.dirname(path)
if not self.temp_fs.isdir(dirname):
self.temp_fs.makedirs(dirname, recreate=True)
fs.copy.copy_file(self.fs, path, self.temp_fs, path)
logger.info('Copied {} file to temporary fs'.format(path))
self._temp_files[path] = self.temp_fs.getsyspath(path)
return self._temp_files[path]
def print_to_cups_printer(self,
printer_name,
file_path,
media='A4',
quality=5):
"""Print to cups printer using command line
"""
path = self.getsyspath(file_path)
command = 'lp -d {} -o media={} -o print-quality={} {}'.format(
printer_name, media, quality, path)
subprocess.check_call(command, shell=True)
def export(self, source_path, fs, destination_path=None):
destination_path = destination_path if destination_path else source_path
fs.copy.copy_file(self.fs, source_path, fs, destination_path)
def __del__(self):
self.temp_fs.close()
self.fs.close()
class COWFS(FS):
def __init__(
self,
base_fs: FS,
additions_fs: Optional[FS] = None,
deletions_fs: Optional[FS] = None,
) -> None:
FS.__init__(self)
if additions_fs:
self.additions_fs = additions_fs
else:
self.additions_fs = TempFS()
if deletions_fs:
_deletions_invariant(deletions_fs)
self.deletions_fs = deletions_fs
else:
self.deletions_fs = TempFS()
self.original_base_fs = base_fs
self.base_fs = fs.wrap.read_only(base_fs)
self.invariant()
@staticmethod
def create_cowfs(base_fs: FS,
read_write_layer: FS,
recreate: bool = False) -> "COWFS":
additions_fs = read_write_layer.makedir("/additions",
recreate=recreate)
deletions_fs = read_write_layer.makedir("/deletions",
recreate=recreate)
return COWFS(base_fs, additions_fs, deletions_fs)
def __str__(self) -> str:
return (f"COWFS({self.original_base_fs}, "
f"{self.additions_fs}, "
f"{self.deletions_fs})")
def __repr__(self) -> str:
return (f"COWFS({self.original_base_fs!r}, "
f"{self.additions_fs!r}, "
f"{self.deletions_fs!r})")
############################################################
def invariant(self) -> bool:
if not self.additions_fs:
raise ValueError(f"Invalid additions_fs: {self.additions_fs}.")
if not self.deletions_fs:
raise ValueError(f"Invalid deletions_fs: {self.additions_fs}.")
if not self.base_fs:
raise ValueError(f"Invalid base_fs: {self.base_fs}.")
_deletions_invariant(self.deletions_fs)
additions_paths = set(paths(self.additions_fs))
deletions_paths = {
fs.path.dirname(file)
for file in self.deletions_fs.walk.files()
}
if additions_paths > deletions_paths:
raise ValueError(f"Additions_paths {additions_paths} " +
"is not a subset of deletions_path " +
f"{deletions_paths}. Extras are " +
f"{additions_paths - deletions_paths}.")
return True
def is_deletion(self, path: str) -> bool:
"""
Is the path marked in the deletions_fs"
"""
return self.deletions_fs.exists(del_path(path))
def mark_deletion(self, path: str) -> None:
"""
Mark the path in the deletions_fs.
"""
self.deletions_fs.makedirs(path, None, True)
self.deletions_fs.touch(del_path(path))
def makedirs_mark_deletion(
self,
path: str,
permissions: Optional[Permissions] = None,
recreate: bool = False,
) -> None:
for p in fs.path.recursepath(path)[:-1]:
self.additions_fs.makedirs(p,
permissions=permissions,
recreate=True)
self.mark_deletion(p)
self.additions_fs.makedir(path,
permissions=permissions,
recreate=recreate)
self.mark_deletion(path)
def layer(self, path: str) -> int:
"""
Get the layer on which the file lives, or ROOT_LAYER if it's the
root path.
"""
if path == "/":
return ROOT_LAYER
if self.additions_fs.exists(path):
return ADD_LAYER
elif self.is_deletion(path):
return NO_LAYER
elif self.base_fs.exists(path):
return BASE_LAYER
else:
return NO_LAYER
def copy_up(self, path: str) -> None:
"""
Copy the file from the base_fs to additions_fs.
"""
self.makedirs_mark_deletion(fs.path.dirname(path))
self.mark_deletion(path)
fs.copy.copy_file(self.base_fs, path, self.additions_fs, path)
def triple_tree(self) -> None:
print("base_fs ------------------------------")
self.base_fs.tree()
print("additions_fs ------------------------------")
self.additions_fs.tree()
print("deletions_fs ------------------------------")
self.deletions_fs.tree()
############################################################
def getmeta(self, namespace: str = "standard") -> Mapping[str, object]:
return self.base_fs.getmeta(namespace)
def getinfo(self,
path: str,
namespaces: Optional[Collection[str]] = None) -> Info:
self.check()
self.validatepath(path)
layer = self.layer(path)
if layer == NO_LAYER:
raise fs.errors.ResourceNotFound(path)
elif layer == BASE_LAYER:
return self.base_fs.getinfo(path, namespaces)
elif layer == ADD_LAYER:
return self.additions_fs.getinfo(path, namespaces)
elif layer == ROOT_LAYER:
# TODO implement this
raw_info = {}
if namespaces is None or "basic" in namespaces:
raw_info["basic"] = {"name": "", "is_dir": True}
return Info(raw_info)
else:
raise RuntimeError(f"Unknown layer {layer}.")
def getsyspath(self, path: str) -> str:
self.check()
# self.validatepath(path)
layer = self.layer(path)
if layer == NO_LAYER:
raise fs.errors.NoSysPath(path=path)
elif layer == BASE_LAYER:
return self.base_fs.getsyspath(path)
elif layer == ADD_LAYER:
return self.additions_fs.getsyspath(path)
elif layer == ROOT_LAYER:
raise fs.errors.NoSysPath(path=path)
else:
raise RuntimeError(f"Unknown layer {layer}.")
def listdir(self, path: str) -> List[str]:
self.check()
self.validatepath(path)
layer = self.layer(path)
if layer == NO_LAYER:
raise fs.errors.ResourceNotFound(path)
elif layer == BASE_LAYER:
return [
name for name in self.base_fs.listdir(path)
if self.layer(fs.path.join(path, name)) != NO_LAYER
]
elif layer == ADD_LAYER:
# Get the listing on the additions layer
names = set(self.additions_fs.listdir(path))
# Add in the listing on the base layer (if it exists)
if self.base_fs.isdir(path):
names |= set(self.base_fs.listdir(path))
# Return the entries that actually exist
return [
name for name in list(names)
if self.layer(fs.path.join(path, name)) != NO_LAYER
]
elif layer == ROOT_LAYER:
# Get the listing of the root on the additions layer and
# the base layer.
names = set(self.additions_fs.listdir("/"))
names |= set(self.base_fs.listdir("/"))
# Return the entries that actually exist.
return [
name for name in list(names) if self.layer(name) != NO_LAYER
]
else:
raise RuntimeError(f"Unknown layer {layer}.")
def makedir(
self,
path: str,
permissions: Optional[Permissions] = None,
recreate: bool = False,
) -> SubFS["COWFS"]:
self.check()
self.validatepath(path)
# Check if it *can* be created.
# get a normalized parent_dir path.
parent_dir = fs.path.dirname(fs.path.forcedir(path)[:-1])
if not parent_dir:
parent_dir = "/"
if not self.isdir(parent_dir):
raise fs.errors.ResourceNotFound(path)
layer = self.layer(path)
if layer == NO_LAYER:
self.makedirs_mark_deletion(path,
permissions=permissions,
recreate=recreate)
return SubFS(self, path)
elif layer in [BASE_LAYER, ADD_LAYER, ROOT_LAYER]:
if recreate:
return SubFS(self, path)
else:
# I think this is wrong. What if it's a file?
raise fs.errors.DirectoryExists(path)
else:
raise RuntimeError(f"Unknown layer {layer}.")
def openbin(self,
path: str,
mode: str = "r",
buffering: int = -1,
**options: Any) -> BinaryIO:
self.check()
self.validatepath(path)
parent_dir = fs.path.dirname(fs.path.forcedir(path)[:-1])
if not parent_dir:
parent_dir = "/"
if not self.isdir(parent_dir):
raise fs.errors.ResourceNotFound(path)
mode_obj = Mode(mode)
layer = self.layer(path)
if layer == NO_LAYER:
if mode_obj.create:
for p in fs.path.recursepath(path)[:-1]:
self.additions_fs.makedirs(p, recreate=True)
self.mark_deletion(p)
self.mark_deletion(path)
return self.additions_fs.openbin(path, mode, buffering,
**options)
else:
raise fs.errors.ResourceNotFound(path)
elif layer == ADD_LAYER:
self.mark_deletion(path)
return self.additions_fs.openbin(path, mode, buffering, **options)
elif layer == BASE_LAYER:
if mode_obj.writing:
self.copy_up(path)
return self.additions_fs.openbin(path, mode, buffering,
**options)
else:
return self.base_fs.openbin(path, mode, buffering, **options)
elif layer == ROOT_LAYER:
raise fs.errors.FileExpected(path)
else:
raise RuntimeError(f"Unknown layer {layer}.")
def remove(self, path: str) -> None:
self.check()
self.validatepath(path)
layer = self.layer(path)
if layer == NO_LAYER:
raise fs.errors.ResourceNotFound(path)
elif layer == BASE_LAYER:
if self.base_fs.isfile(path):
self.mark_deletion(path)
else:
raise fs.errors.FileExpected(path)
elif layer == ADD_LAYER:
self.additions_fs.remove(path)
self.mark_deletion(path)
elif layer == ROOT_LAYER:
raise fs.errors.FileExpected(path)
else:
raise RuntimeError(f"Unknown layer {layer}.")
def removedir(self, path: str) -> None:
self.check()
layer = self.layer(path)
if layer == NO_LAYER:
raise fs.errors.ResourceNotFound(path)
elif layer == BASE_LAYER:
if self.base_fs.isdir(path):
self.mark_deletion(path)
else:
raise fs.errors.FileExpected(path)
elif layer == ADD_LAYER:
if self.additions_fs.isdir(path):
self.additions_fs.removedir(path)
self.mark_deletion(path)
else:
raise fs.errors.DirectoryExpected(path)
elif layer == ROOT_LAYER:
raise fs.errors.RemoveRootError(path)
else:
raise RuntimeError(f"Unknown layer {layer}.")
def setinfo(self, path: str, info: _INFO_DICT) -> None:
self.check()
self.validatepath(path)
layer = self.layer(path)
if layer == NO_LAYER:
raise fs.errors.ResourceNotFound(path)
elif layer == BASE_LAYER:
self.copy_up(path)
self.additions_fs.setinfo(path, info)
elif layer == ADD_LAYER:
self.additions_fs.setinfo(path, info)
elif layer == ROOT_LAYER:
pass
else:
raise RuntimeError(f"Unknown layer {layer}.")
############################################################
def makedirs(
self,
path: str,
permissions: Optional[Permissions] = None,
recreate: bool = False,
) -> SubFS[FS]:
return FS.makedirs(self,
path,
permissions=permissions,
recreate=recreate)
