def test_generate_ipynb_from_py():
python_dir = tempfile.mkdtemp()
try:
set_cache("latest_sha", "fake_sha_early")
os.mkdir(python_dir + "/extra_path")
with open(os.path.join(python_dir, "extra_path", "test_report.py"), "w") as f:
f.write("#hello world\n")
report_path = os.sep.join(["extra_path", "test_report"])
with mock.patch("notebooker.utils.conversion._git_pull_templates") as pull:
conversion.python_template_dir = lambda *a, **kw: python_dir
pull.return_value = "fake_sha_early"
conversion.generate_ipynb_from_py(python_dir, report_path)
pull.return_value = "fake_sha_later"
conversion.generate_ipynb_from_py(python_dir, report_path)
conversion.generate_ipynb_from_py(python_dir, report_path)
assert get_cache("latest_sha") == "fake_sha_later"
expected_filename = _output_ipynb_name(report_path)
expected_ipynb_path = os.path.join(python_dir, "fake_sha_early", expected_filename)
assert os.path.exists(expected_ipynb_path), f".ipynb at {expected_ipynb_path} was not generated as expected!"
expected_ipynb_path = os.path.join(python_dir, "fake_sha_later", expected_filename)
assert os.path.exists(expected_ipynb_path), ".ipynb was not generated as expected!"
with mock.patch("notebooker.utils.conversion.uuid.uuid4") as uuid4:
with mock.patch("notebooker.utils.conversion.pkg_resources.resource_filename") as resource_filename:
conversion.python_template_dir = lambda *a, **kw: None
uuid4.return_value = "uuid"
resource_filename.return_value = python_dir + "/extra_path/test_report.py"
conversion.generate_ipynb_from_py(python_dir, "extra_path/test_report")
expected_ipynb_path = os.path.join(python_dir, "uuid", expected_filename)
assert os.path.exists(expected_ipynb_path), f".ipynb at {expected_ipynb_path} was not generated as expected!"
with mock.patch("notebooker.utils.conversion.uuid.uuid4") as uuid4:
conversion.python_template_dir = lambda *a, **kw: python_dir
conversion.NOTEBOOKER_DISABLE_GIT = True
uuid4.return_value = "uuid_nogit"
conversion.generate_ipynb_from_py(python_dir, "extra_path/test_report")
expected_ipynb_path = os.path.join(python_dir, "uuid_nogit", expected_filename)
assert os.path.exists(expected_ipynb_path), ".ipynb was not generated as expected!"
finally:
_cleanup_dirs()
shutil.rmtree(python_dir)
def test_generate_ipynb_from_py(setup_and_cleanup_notebooker_filesystem,
webapp_config, flask_app):
python_dir = webapp_config.PY_TEMPLATE_BASE_DIR
with flask_app.app_context():
set_cache("latest_sha", "fake_sha_early")
os.mkdir(python_dir + "/extra_path")
with open(os.path.join(python_dir, "extra_path", "test_report.py"),
"w") as f:
f.write("#hello world\n")
report_path = os.sep.join(["extra_path", "test_report"])
with mock.patch("notebooker.utils.conversion.git.repo.Repo") as repo:
repo().commit().hexsha = "fake_sha_early"
conversion.generate_ipynb_from_py(python_dir, report_path, False,
python_dir)
repo().commit().hexsha = "fake_sha_later"
conversion.generate_ipynb_from_py(python_dir, report_path, False,
python_dir)
conversion.generate_ipynb_from_py(python_dir, report_path, False,
python_dir)
expected_filename = _output_ipynb_name(report_path)
expected_ipynb_path = os.path.join(python_dir, "fake_sha_early",
expected_filename)
assert os.path.exists(
expected_ipynb_path
), f".ipynb at {expected_ipynb_path} was not generated as expected!"
expected_ipynb_path = os.path.join(python_dir, "fake_sha_later",
expected_filename)
assert os.path.exists(
expected_ipynb_path), ".ipynb was not generated as expected!"
with mock.patch("notebooker.utils.conversion.uuid.uuid4") as uuid4:
with mock.patch(
"notebooker.utils.conversion.pkg_resources.resource_filename"
) as resource_filename:
conversion.python_template_dir = lambda *a, **kw: None
uuid4.return_value = "uuid"
resource_filename.return_value = python_dir + "/extra_path/test_report.py"
conversion.generate_ipynb_from_py(python_dir,
"extra_path/test_report",
False,
py_template_dir="")
expected_ipynb_path = os.path.join(python_dir, "uuid",
expected_filename)
assert os.path.exists(
expected_ipynb_path
), f".ipynb at {expected_ipynb_path} was not generated as expected!"
with mock.patch("notebooker.utils.conversion.uuid.uuid4") as uuid4:
conversion.python_template_dir = lambda *a, **kw: python_dir
conversion.NOTEBOOKER_DISABLE_GIT = True
uuid4.return_value = "uuid_nogit"
conversion.generate_ipynb_from_py(python_dir,
"extra_path/test_report",
True,
py_template_dir=python_dir)
expected_ipynb_path = os.path.join(python_dir, "uuid_nogit",
expected_filename)
assert os.path.exists(
expected_ipynb_path), ".ipynb was not generated as expected!"
def _run_checks(
job_id: str,
job_start_time: datetime.datetime,
template_name: str,
report_title: str,
output_base_dir: str,
template_base_dir: str,
overrides: Dict[AnyStr, Any],
generate_pdf_output: Optional[bool] = True,
mailto: Optional[str] = "",
prepare_only: Optional[bool] = False,
) -> NotebookResultComplete:
"""
This is the actual method which executes a notebook, whether running in the webapp or via the entrypoint.
If this crashes, an exception is raised (and should be caught by run_checks().)
Parameters
----------
job_id : `str`
The unique ID of this report.
job_start_time : `datetime.datetime`
The UTC start time of this report.
template_name : `str`
The name of the template which we are running. NB this should be a path relative to the python_template_dir()
report_title : `str`
The user-specified optional title of the report. Defaults to the template name.
output_base_dir : `str`
Internal use. The temp directory where output is being saved, local to the executor.
template_base_dir : `str`
Internal use. The temp directory where the .py->.ipynb converted templates reside, local to the executor.
overrides : Dict[AnyStr, Any]
The dictionary of overrides which parametrize our Notebook Template.
generate_pdf_output : `Optional[bool]`
Whether to generate PDF output or not. NB this requires xelatex to be installed on the executor.
mailto : `Optional[str]`
Comma-separated email addresses to send on completion (or error).
prepare_only : `Optional[bool]`
Internal usage. Whether we want to do everything apart from executing the notebook.
Returns
-------
NotebookResultComplete
Raises
------
Exception()
"""
output_dir = _output_dir(output_base_dir, template_name, job_id)
output_ipynb = _output_ipynb_name(template_name)
if not os.path.isdir(output_dir):
logger.info("Making dir @ {}".format(output_dir))
os.makedirs(output_dir)
ipynb_raw_path = generate_ipynb_from_py(template_base_dir, template_name)
ipynb_executed_path = os.path.join(output_dir, output_ipynb)
logger.info("Executing notebook at {} using parameters {} --> {}".format(
ipynb_raw_path, overrides, output_ipynb))
pm.execute_notebook(ipynb_raw_path,
ipynb_executed_path,
parameters=overrides,
log_output=True,
prepare_only=prepare_only)
with open(ipynb_executed_path, "r") as f:
raw_executed_ipynb = f.read()
logger.info(
"Saving output notebook as HTML from {}".format(ipynb_executed_path))
html, resources = ipython_to_html(ipynb_executed_path, job_id)
pdf = ipython_to_pdf(raw_executed_ipynb,
report_title) if generate_pdf_output else ""
notebook_result = NotebookResultComplete(
job_id=job_id,
job_start_time=job_start_time,
job_finish_time=datetime.datetime.now(),
raw_html_resources=resources,
raw_ipynb_json=raw_executed_ipynb,
raw_html=html,
mailto=mailto,
pdf=pdf,
generate_pdf_output=generate_pdf_output,
report_name=template_name,
report_title=report_title,
overrides=overrides,
)
return notebook_result