def get_all_result_keys(serializer: MongoResultSerializer,
limit: int = 0,
force_reload: bool = False) -> List[Tuple[str, str]]:
all_keys = get_cache(("all_result_keys", limit))
if not all_keys or force_reload:
all_keys = serializer.get_all_result_keys(limit=limit)
set_cache(("all_result_keys", limit), all_keys, timeout=1)
return all_keys
def _get_output_path_hex() -> str:
if python_template_dir() and not NOTEBOOKER_DISABLE_GIT:
logger.info("Pulling latest notebook templates from git.")
try:
latest_sha = _git_pull_templates()
if get_cache("latest_sha") != latest_sha:
logger.info("Change detected in notebook template master!")
set_cache("latest_sha", latest_sha)
logger.info("Git pull done.")
except Exception as e:
logger.exception(e)
return get_cache("latest_sha") or "OLD"
else:
return str(uuid.uuid4())
def test_generate_ipynb_from_py():
python_dir = tempfile.mkdtemp()
try:
set_cache("latest_sha", "fake_sha_early")
os.mkdir(python_dir + "/extra_path")
with open(os.path.join(python_dir, "extra_path", "test_report.py"), "w") as f:
f.write("#hello world\n")
report_path = os.sep.join(["extra_path", "test_report"])
with mock.patch("notebooker.utils.conversion._git_pull_templates") as pull:
conversion.python_template_dir = lambda *a, **kw: python_dir
pull.return_value = "fake_sha_early"
conversion.generate_ipynb_from_py(python_dir, report_path)
pull.return_value = "fake_sha_later"
conversion.generate_ipynb_from_py(python_dir, report_path)
conversion.generate_ipynb_from_py(python_dir, report_path)
assert get_cache("latest_sha") == "fake_sha_later"
expected_filename = _output_ipynb_name(report_path)
expected_ipynb_path = os.path.join(python_dir, "fake_sha_early", expected_filename)
assert os.path.exists(expected_ipynb_path), f".ipynb at {expected_ipynb_path} was not generated as expected!"
expected_ipynb_path = os.path.join(python_dir, "fake_sha_later", expected_filename)
assert os.path.exists(expected_ipynb_path), ".ipynb was not generated as expected!"
with mock.patch("notebooker.utils.conversion.uuid.uuid4") as uuid4:
with mock.patch("notebooker.utils.conversion.pkg_resources.resource_filename") as resource_filename:
conversion.python_template_dir = lambda *a, **kw: None
uuid4.return_value = "uuid"
resource_filename.return_value = python_dir + "/extra_path/test_report.py"
conversion.generate_ipynb_from_py(python_dir, "extra_path/test_report")
expected_ipynb_path = os.path.join(python_dir, "uuid", expected_filename)
assert os.path.exists(expected_ipynb_path), f".ipynb at {expected_ipynb_path} was not generated as expected!"
with mock.patch("notebooker.utils.conversion.uuid.uuid4") as uuid4:
conversion.python_template_dir = lambda *a, **kw: python_dir
conversion.NOTEBOOKER_DISABLE_GIT = True
uuid4.return_value = "uuid_nogit"
conversion.generate_ipynb_from_py(python_dir, "extra_path/test_report")
expected_ipynb_path = os.path.join(python_dir, "uuid_nogit", expected_filename)
assert os.path.exists(expected_ipynb_path), ".ipynb was not generated as expected!"
finally:
_cleanup_dirs()
shutil.rmtree(python_dir)
def _get_preview(template_name: str, warn_on_local: Optional[bool] = True) -> str:
""" Returns an HTML render of a report template, with parameters highlighted. """
cached = get_cache(("preview", template_name))
if cached:
logger.info("Getting %s preview from cache.", template_name)
return cached
nb = template_name_to_notebook_node(template_name, warn_on_local=warn_on_local)
parameters_idx = _get_parameters_cell_idx(nb)
conf = Config()
if parameters_idx is not None:
# Use this template to highlight the cell with parameters
conf.HTMLExporter.template_file = pkg_resources.resource_filename(
__name__, "../nbtemplates/notebook_preview.tpl"
)
exporter = HTMLExporter(config=conf)
html, _ = exporter.from_notebook_node(nb) if nb["cells"] else ("", "")
set_cache(("preview", template_name), html, timeout=30)
return html
def test_generate_ipynb_from_py(setup_and_cleanup_notebooker_filesystem,
webapp_config, flask_app):
python_dir = webapp_config.PY_TEMPLATE_BASE_DIR
with flask_app.app_context():
set_cache("latest_sha", "fake_sha_early")
os.mkdir(python_dir + "/extra_path")
with open(os.path.join(python_dir, "extra_path", "test_report.py"),
"w") as f:
f.write("#hello world\n")
report_path = os.sep.join(["extra_path", "test_report"])
with mock.patch("notebooker.utils.conversion.git.repo.Repo") as repo:
repo().commit().hexsha = "fake_sha_early"
conversion.generate_ipynb_from_py(python_dir, report_path, False,
python_dir)
repo().commit().hexsha = "fake_sha_later"
conversion.generate_ipynb_from_py(python_dir, report_path, False,
python_dir)
conversion.generate_ipynb_from_py(python_dir, report_path, False,
python_dir)
expected_filename = _output_ipynb_name(report_path)
expected_ipynb_path = os.path.join(python_dir, "fake_sha_early",
expected_filename)
assert os.path.exists(
expected_ipynb_path
), f".ipynb at {expected_ipynb_path} was not generated as expected!"
expected_ipynb_path = os.path.join(python_dir, "fake_sha_later",
expected_filename)
assert os.path.exists(
expected_ipynb_path), ".ipynb was not generated as expected!"
with mock.patch("notebooker.utils.conversion.uuid.uuid4") as uuid4:
with mock.patch(
"notebooker.utils.conversion.pkg_resources.resource_filename"
) as resource_filename:
conversion.python_template_dir = lambda *a, **kw: None
uuid4.return_value = "uuid"
resource_filename.return_value = python_dir + "/extra_path/test_report.py"
conversion.generate_ipynb_from_py(python_dir,
"extra_path/test_report",
False,
py_template_dir="")
expected_ipynb_path = os.path.join(python_dir, "uuid",
expected_filename)
assert os.path.exists(
expected_ipynb_path
), f".ipynb at {expected_ipynb_path} was not generated as expected!"
with mock.patch("notebooker.utils.conversion.uuid.uuid4") as uuid4:
conversion.python_template_dir = lambda *a, **kw: python_dir
conversion.NOTEBOOKER_DISABLE_GIT = True
uuid4.return_value = "uuid_nogit"
conversion.generate_ipynb_from_py(python_dir,
"extra_path/test_report",
True,
py_template_dir=python_dir)
expected_ipynb_path = os.path.join(python_dir, "uuid_nogit",
expected_filename)
assert os.path.exists(
expected_ipynb_path), ".ipynb was not generated as expected!"