def test_conversion_doesnt_fail(template_name, template_dir):
# Test conversion to ipynb - this will throw if stuff goes wrong
generate_ipynb_from_py(template_dir,
template_name,
notebooker_disable_git=True,
py_template_dir="",
warn_on_local=False)
def sanity_check(template_dir):
logger.info("Starting sanity check")
os.environ["PY_TEMPLATE_DIR"] = template_dir
try:
for template_name in templates._all_templates():
logger.info(
"========================[ Sanity checking {} ]========================"
.format(template_name))
# Test conversion to ipynb - this will throw if stuff goes wrong
generate_ipynb_from_py(filesystem.get_template_dir(),
template_name,
warn_on_local=False)
# Test that each template has parameters as expected
nb = templates.template_name_to_notebook_node(template_name,
warn_on_local=False)
param_idx = templates._get_parameters_cell_idx(nb)
if param_idx is None:
logger.warning(
'Template {} does not have a "parameters"-tagged cell.'.
format(template_name))
# Test that we can generate a preview from the template
preview = templates._get_preview(template_name,
warn_on_local=False)
# Previews in HTML are gigantic since they include all jupyter css and js.
assert len(
preview
) > 1000, "Preview was not properly generated for {}".format(
template_name)
logger.info(
"========================[ {} PASSED ]========================"
.format(template_name))
finally:
filesystem._cleanup_dirs()
def test_template_has_parameters(template_name):
generate_ipynb_from_py(get_template_dir(),
template_name,
warn_on_local=False)
nb = template_name_to_notebook_node(template_name, warn_on_local=False)
metadata_idx = _get_parameters_cell_idx(nb)
assert metadata_idx is not None, 'Template {} does not have a "parameters"-tagged cell.'.format(
template_name)
def test_template_has_parameters(template_name, template_dir, flask_app):
flask_app.config["PY_TEMPLATE_DIR"] = ""
with flask_app.app_context():
generate_ipynb_from_py(template_dir,
template_name,
notebooker_disable_git=True,
py_template_dir="",
warn_on_local=False)
nb = template_name_to_notebook_node(template_name,
notebooker_disable_git=True,
py_template_dir="",
warn_on_local=False)
metadata_idx = _get_parameters_cell_idx(nb)
assert metadata_idx is not None, 'Template {} does not have a "parameters"-tagged cell.'.format(
template_name)
def run_report_http(report_name):
"""
The "Run Report" interface is generated by this method.
:param report_name: The parameter here should be a "/"-delimited string which mirrors the directory structure of \
the notebook templates.
:returns: An HTML template which is the Run Report interface.
"""
report_name = convert_report_name_url_to_path(report_name)
json_params = request.args.get("json_params")
initial_python_parameters = json_to_python(json_params) or ""
try:
path = generate_ipynb_from_py(get_template_dir(), report_name)
except FileNotFoundError as e:
logger.exception(e)
return "", 404
nb = nbformat.read(path, as_version=nbformat.v4.nbformat)
metadata_idx = _get_parameters_cell_idx(nb)
parameters_as_html = ""
has_prefix = has_suffix = False
if metadata_idx is not None:
metadata = nb["cells"][metadata_idx]
parameters_as_html = metadata["source"].strip()
has_prefix, has_suffix = (bool(nb["cells"][:metadata_idx]), bool(nb["cells"][metadata_idx + 1 :]))
logger.info("initial_python_parameters = {}".format(initial_python_parameters))
return render_template(
"run_report.html",
parameters_as_html=parameters_as_html,
has_prefix=has_prefix,
has_suffix=has_suffix,
report_name=report_name,
all_reports=get_all_possible_templates(),
initialPythonParameters=initial_python_parameters,
)
def test_generate_ipynb_from_py():
python_dir = tempfile.mkdtemp()
try:
set_cache("latest_sha", "fake_sha_early")
os.mkdir(python_dir + "/extra_path")
with open(os.path.join(python_dir, "extra_path", "test_report.py"), "w") as f:
f.write("#hello world\n")
report_path = os.sep.join(["extra_path", "test_report"])
with mock.patch("notebooker.utils.conversion._git_pull_templates") as pull:
conversion.python_template_dir = lambda *a, **kw: python_dir
pull.return_value = "fake_sha_early"
conversion.generate_ipynb_from_py(python_dir, report_path)
pull.return_value = "fake_sha_later"
conversion.generate_ipynb_from_py(python_dir, report_path)
conversion.generate_ipynb_from_py(python_dir, report_path)
assert get_cache("latest_sha") == "fake_sha_later"
expected_filename = _output_ipynb_name(report_path)
expected_ipynb_path = os.path.join(python_dir, "fake_sha_early", expected_filename)
assert os.path.exists(expected_ipynb_path), f".ipynb at {expected_ipynb_path} was not generated as expected!"
expected_ipynb_path = os.path.join(python_dir, "fake_sha_later", expected_filename)
assert os.path.exists(expected_ipynb_path), ".ipynb was not generated as expected!"
with mock.patch("notebooker.utils.conversion.uuid.uuid4") as uuid4:
with mock.patch("notebooker.utils.conversion.pkg_resources.resource_filename") as resource_filename:
conversion.python_template_dir = lambda *a, **kw: None
uuid4.return_value = "uuid"
resource_filename.return_value = python_dir + "/extra_path/test_report.py"
conversion.generate_ipynb_from_py(python_dir, "extra_path/test_report")
expected_ipynb_path = os.path.join(python_dir, "uuid", expected_filename)
assert os.path.exists(expected_ipynb_path), f".ipynb at {expected_ipynb_path} was not generated as expected!"
with mock.patch("notebooker.utils.conversion.uuid.uuid4") as uuid4:
conversion.python_template_dir = lambda *a, **kw: python_dir
conversion.NOTEBOOKER_DISABLE_GIT = True
uuid4.return_value = "uuid_nogit"
conversion.generate_ipynb_from_py(python_dir, "extra_path/test_report")
expected_ipynb_path = os.path.join(python_dir, "uuid_nogit", expected_filename)
assert os.path.exists(expected_ipynb_path), ".ipynb was not generated as expected!"
finally:
_cleanup_dirs()
shutil.rmtree(python_dir)
def template_name_to_notebook_node(
template_name: str, notebooker_disable_git: bool, py_template_dir: str, warn_on_local: Optional[bool] = True
) -> nbformat.NotebookNode:
path = generate_ipynb_from_py(
get_template_dir(), template_name, notebooker_disable_git, py_template_dir, warn_on_local=warn_on_local
)
nb = nbformat.read(path, as_version=nbformat.v4.nbformat)
return nb
def test_generate_ipynb_from_py(setup_and_cleanup_notebooker_filesystem,
webapp_config, flask_app):
python_dir = webapp_config.PY_TEMPLATE_BASE_DIR
with flask_app.app_context():
set_cache("latest_sha", "fake_sha_early")
os.mkdir(python_dir + "/extra_path")
with open(os.path.join(python_dir, "extra_path", "test_report.py"),
"w") as f:
f.write("#hello world\n")
report_path = os.sep.join(["extra_path", "test_report"])
with mock.patch("notebooker.utils.conversion.git.repo.Repo") as repo:
repo().commit().hexsha = "fake_sha_early"
conversion.generate_ipynb_from_py(python_dir, report_path, False,
python_dir)
repo().commit().hexsha = "fake_sha_later"
conversion.generate_ipynb_from_py(python_dir, report_path, False,
python_dir)
conversion.generate_ipynb_from_py(python_dir, report_path, False,
python_dir)
expected_filename = _output_ipynb_name(report_path)
expected_ipynb_path = os.path.join(python_dir, "fake_sha_early",
expected_filename)
assert os.path.exists(
expected_ipynb_path
), f".ipynb at {expected_ipynb_path} was not generated as expected!"
expected_ipynb_path = os.path.join(python_dir, "fake_sha_later",
expected_filename)
assert os.path.exists(
expected_ipynb_path), ".ipynb was not generated as expected!"
with mock.patch("notebooker.utils.conversion.uuid.uuid4") as uuid4:
with mock.patch(
"notebooker.utils.conversion.pkg_resources.resource_filename"
) as resource_filename:
conversion.python_template_dir = lambda *a, **kw: None
uuid4.return_value = "uuid"
resource_filename.return_value = python_dir + "/extra_path/test_report.py"
conversion.generate_ipynb_from_py(python_dir,
"extra_path/test_report",
False,
py_template_dir="")
expected_ipynb_path = os.path.join(python_dir, "uuid",
expected_filename)
assert os.path.exists(
expected_ipynb_path
), f".ipynb at {expected_ipynb_path} was not generated as expected!"
with mock.patch("notebooker.utils.conversion.uuid.uuid4") as uuid4:
conversion.python_template_dir = lambda *a, **kw: python_dir
conversion.NOTEBOOKER_DISABLE_GIT = True
uuid4.return_value = "uuid_nogit"
conversion.generate_ipynb_from_py(python_dir,
"extra_path/test_report",
True,
py_template_dir=python_dir)
expected_ipynb_path = os.path.join(python_dir, "uuid_nogit",
expected_filename)
assert os.path.exists(
expected_ipynb_path), ".ipynb was not generated as expected!"
def _run_checks(
job_id: str,
job_start_time: datetime.datetime,
template_name: str,
report_title: str,
output_base_dir: str,
template_base_dir: str,
overrides: Dict[AnyStr, Any],
generate_pdf_output: Optional[bool] = True,
mailto: Optional[str] = "",
prepare_only: Optional[bool] = False,
) -> NotebookResultComplete:
"""
This is the actual method which executes a notebook, whether running in the webapp or via the entrypoint.
If this crashes, an exception is raised (and should be caught by run_checks().)
Parameters
----------
job_id : `str`
The unique ID of this report.
job_start_time : `datetime.datetime`
The UTC start time of this report.
template_name : `str`
The name of the template which we are running. NB this should be a path relative to the python_template_dir()
report_title : `str`
The user-specified optional title of the report. Defaults to the template name.
output_base_dir : `str`
Internal use. The temp directory where output is being saved, local to the executor.
template_base_dir : `str`
Internal use. The temp directory where the .py->.ipynb converted templates reside, local to the executor.
overrides : Dict[AnyStr, Any]
The dictionary of overrides which parametrize our Notebook Template.
generate_pdf_output : `Optional[bool]`
Whether to generate PDF output or not. NB this requires xelatex to be installed on the executor.
mailto : `Optional[str]`
Comma-separated email addresses to send on completion (or error).
prepare_only : `Optional[bool]`
Internal usage. Whether we want to do everything apart from executing the notebook.
Returns
-------
NotebookResultComplete
Raises
------
Exception()
"""
output_dir = _output_dir(output_base_dir, template_name, job_id)
output_ipynb = _output_ipynb_name(template_name)
if not os.path.isdir(output_dir):
logger.info("Making dir @ {}".format(output_dir))
os.makedirs(output_dir)
ipynb_raw_path = generate_ipynb_from_py(template_base_dir, template_name)
ipynb_executed_path = os.path.join(output_dir, output_ipynb)
logger.info("Executing notebook at {} using parameters {} --> {}".format(
ipynb_raw_path, overrides, output_ipynb))
pm.execute_notebook(ipynb_raw_path,
ipynb_executed_path,
parameters=overrides,
log_output=True,
prepare_only=prepare_only)
with open(ipynb_executed_path, "r") as f:
raw_executed_ipynb = f.read()
logger.info(
"Saving output notebook as HTML from {}".format(ipynb_executed_path))
html, resources = ipython_to_html(ipynb_executed_path, job_id)
pdf = ipython_to_pdf(raw_executed_ipynb,
report_title) if generate_pdf_output else ""
notebook_result = NotebookResultComplete(
job_id=job_id,
job_start_time=job_start_time,
job_finish_time=datetime.datetime.now(),
raw_html_resources=resources,
raw_ipynb_json=raw_executed_ipynb,
raw_html=html,
mailto=mailto,
pdf=pdf,
generate_pdf_output=generate_pdf_output,
report_name=template_name,
report_title=report_title,
overrides=overrides,
)
return notebook_result
def test_conversion_doesnt_fail(template_name, template_dir):
# Test conversion to ipynb - this will throw if stuff goes wrong
generate_ipynb_from_py(template_dir, template_name, warn_on_local=False)
