def test_send_result_email(test_name, job_id, report_name, report_title,
expected_title):
body_in = "<body><h1>hello 😆 😉 😊 😋 😎</h1></body>"
result = NotebookResultComplete(
job_id=job_id,
job_start_time=datetime.datetime.now(),
job_finish_time=datetime.datetime.now(),
raw_html_resources={},
raw_ipynb_json={},
raw_html=body_in,
pdf="",
report_name=report_name,
report_title=report_title,
)
to_email = "∫åññîsté®@test-email.com"
with mock.patch("notebooker.utils.notebook_execution.mail") as mail:
notebook_execution.send_result_email(result, to_email)
email_sent = mail.mock_calls[0][1]
assert len(email_sent
) == 4, "mail() was not called with the correct number of args"
from_address = email_sent[0]
to_address = email_sent[1]
title = email_sent[2]
body = email_sent[3]
assert from_address == "*****@*****.**"
assert to_address == to_email
assert title == expected_title
assert body == [
"Please either activate HTML emails, or see the PDF attachment.",
body_in
]
def test_send_result_email_fallback_to_raw_html():
body_in = "<body><h1>hello 😆 😉 😊 😋 😎</h1></body>"
to_email = "hello@you"
result = NotebookResultComplete(
job_id="fallback",
job_start_time=datetime.datetime.now(),
job_finish_time=datetime.datetime.now(),
mailto=to_email,
raw_html_resources={},
raw_ipynb_json={},
raw_html=body_in,
pdf="",
report_name="fallback",
report_title="fallback",
)
with mock.patch("notebooker.utils.notebook_execution.mail") as mail:
notebook_execution.send_result_email(result)
email_sent = mail.mock_calls[0][1]
assert len(email_sent
) == 4, "mail() was not called with the correct number of args"
from_address = email_sent[0]
to_address = email_sent[1]
title = email_sent[2]
body = email_sent[3]
assert from_address == "*****@*****.**"
assert to_address == to_email
assert title == "Notebooker: fallback report completed with status: Checks done!"
assert body == [
"Please either activate HTML emails, or see the PDF attachment.",
body_in
]
def _run_checks(
job_id: str,
job_start_time: datetime.datetime,
template_name: str,
report_title: str,
output_base_dir: str,
template_base_dir: str,
overrides: Dict[AnyStr, Any],
generate_pdf_output: Optional[bool] = True,
mailto: Optional[str] = "",
prepare_only: Optional[bool] = False,
) -> NotebookResultComplete:
"""
This is the actual method which executes a notebook, whether running in the webapp or via the entrypoint.
If this crashes, an exception is raised (and should be caught by run_checks().)
Parameters
----------
job_id : `str`
The unique ID of this report.
job_start_time : `datetime.datetime`
The UTC start time of this report.
template_name : `str`
The name of the template which we are running. NB this should be a path relative to the python_template_dir()
report_title : `str`
The user-specified optional title of the report. Defaults to the template name.
output_base_dir : `str`
Internal use. The temp directory where output is being saved, local to the executor.
template_base_dir : `str`
Internal use. The temp directory where the .py->.ipynb converted templates reside, local to the executor.
overrides : Dict[AnyStr, Any]
The dictionary of overrides which parametrize our Notebook Template.
generate_pdf_output : `Optional[bool]`
Whether to generate PDF output or not. NB this requires xelatex to be installed on the executor.
mailto : `Optional[str]`
Comma-separated email addresses to send on completion (or error).
prepare_only : `Optional[bool]`
Internal usage. Whether we want to do everything apart from executing the notebook.
Returns
-------
NotebookResultComplete
Raises
------
Exception()
"""
output_dir = _output_dir(output_base_dir, template_name, job_id)
output_ipynb = _output_ipynb_name(template_name)
if not os.path.isdir(output_dir):
logger.info("Making dir @ {}".format(output_dir))
os.makedirs(output_dir)
ipynb_raw_path = generate_ipynb_from_py(template_base_dir, template_name)
ipynb_executed_path = os.path.join(output_dir, output_ipynb)
logger.info("Executing notebook at {} using parameters {} --> {}".format(
ipynb_raw_path, overrides, output_ipynb))
pm.execute_notebook(ipynb_raw_path,
ipynb_executed_path,
parameters=overrides,
log_output=True,
prepare_only=prepare_only)
with open(ipynb_executed_path, "r") as f:
raw_executed_ipynb = f.read()
logger.info(
"Saving output notebook as HTML from {}".format(ipynb_executed_path))
html, resources = ipython_to_html(ipynb_executed_path, job_id)
pdf = ipython_to_pdf(raw_executed_ipynb,
report_title) if generate_pdf_output else ""
notebook_result = NotebookResultComplete(
job_id=job_id,
job_start_time=job_start_time,
job_finish_time=datetime.datetime.now(),
raw_html_resources=resources,
raw_ipynb_json=raw_executed_ipynb,
raw_html=html,
mailto=mailto,
pdf=pdf,
generate_pdf_output=generate_pdf_output,
report_name=template_name,
report_title=report_title,
overrides=overrides,
)
return notebook_result
def test_report_hunter_pending_to_done(bson_library, mongo_host, test_db_name,
test_lib_name):
job_id = str(uuid.uuid4())
report_name = str(uuid.uuid4())
serializer = PyMongoNotebookResultSerializer(
database_name=test_db_name,
mongo_host=mongo_host,
result_collection_name=test_lib_name)
with freezegun.freeze_time(datetime.datetime(2018, 1, 12, 2, 30)):
serializer.save_check_stub(job_id,
report_name,
status=JobStatus.SUBMITTED)
_report_hunter(
Serializer.PYMONGO.value,
mongo_host=mongo_host,
database_name=test_db_name,
result_collection_name=test_lib_name,
run_once=True,
)
expected = NotebookResultPending(
job_id=job_id,
report_name=report_name,
report_title=report_name,
status=JobStatus.SUBMITTED,
update_time=datetime.datetime(2018, 1, 12, 2, 30),
job_start_time=datetime.datetime(2018, 1, 12, 2, 30),
)
assert get_report_cache(report_name, job_id) == expected
with freezegun.freeze_time(datetime.datetime(2018, 1, 12, 2, 32)):
serializer.update_check_status(job_id, JobStatus.PENDING)
_report_hunter(
Serializer.PYMONGO.value,
mongo_host=mongo_host,
database_name=test_db_name,
result_collection_name=test_lib_name,
run_once=True,
)
expected = NotebookResultPending(
job_id=job_id,
report_name=report_name,
report_title=report_name,
status=JobStatus.PENDING,
update_time=datetime.datetime(2018, 1, 12, 2, 32),
job_start_time=datetime.datetime(2018, 1, 12, 2, 30),
)
assert get_report_cache(report_name, job_id) == expected
with freezegun.freeze_time(datetime.datetime(2018, 1, 12, 2, 37)):
serializer.update_check_status(
job_id,
JobStatus.DONE,
raw_html_resources={"outputs": {}},
job_finish_time=datetime.datetime.now(),
pdf="",
raw_ipynb_json="[]",
raw_html="",
)
_report_hunter(
Serializer.PYMONGO.value,
mongo_host=mongo_host,
database_name=test_db_name,
result_collection_name=test_lib_name,
run_once=True,
)
expected = NotebookResultComplete(
job_id=job_id,
report_name=report_name,
report_title=report_name,
status=JobStatus.DONE,
update_time=datetime.datetime(2018, 1, 12, 2, 37),
job_start_time=datetime.datetime(2018, 1, 12, 2, 30),
job_finish_time=datetime.datetime(2018, 1, 12, 2, 37),
raw_html="",
raw_html_resources={"outputs": {}},
raw_ipynb_json="[]",
)
assert get_report_cache(report_name, job_id) == expected
def _convert_result(
self,
result: Dict,
load_payload: bool = True
) -> Union[NotebookResultError, NotebookResultComplete,
NotebookResultPending, None]:
if not result:
return None
status = result.get("status", "")
job_status = JobStatus.from_string(status)
if job_status is None:
return None
cls = {
JobStatus.CANCELLED: NotebookResultError,
JobStatus.DONE: NotebookResultComplete,
JobStatus.PENDING: NotebookResultPending,
JobStatus.ERROR: NotebookResultError,
JobStatus.SUBMITTED: NotebookResultPending,
JobStatus.TIMEOUT: NotebookResultError,
JobStatus.DELETED: None,
}.get(job_status)
if cls is None:
return None
if load_payload and job_status == JobStatus.DONE:
def read_file(path):
try:
return self.result_data_store.get_last_version(path).read()
except NoFile:
logger.error("Could not find file %s in %s", path,
self.result_data_store)
return ""
outputs = {
path: read_file(path)
for path in result.get("raw_html_resources", {}).get(
"outputs", [])
}
result["raw_html_resources"]["outputs"] = outputs
if result.get("generate_pdf_output"):
pdf_filename = _pdf_filename(result["job_id"])
result["pdf"] = read_file(pdf_filename)
if cls == NotebookResultComplete:
return NotebookResultComplete(
job_id=result["job_id"],
job_start_time=result["job_start_time"],
report_name=result["report_name"],
status=job_status,
update_time=result["update_time"],
job_finish_time=result["job_finish_time"],
raw_html_resources=result.get("raw_html_resources", {}),
raw_ipynb_json=result.get("raw_ipynb_json"),
raw_html=result.get("raw_html"),
pdf=result.get("pdf", ""),
overrides=result.get("overrides", {}),
generate_pdf_output=result.get("generate_pdf_output", True),
report_title=result.get("report_title", result["report_name"]),
mailto=result.get("mailto", ""),
stdout=result.get("stdout", []),
)
elif cls == NotebookResultPending:
return NotebookResultPending(
job_id=result["job_id"],
job_start_time=result["job_start_time"],
report_name=result["report_name"],
status=job_status,
update_time=result["update_time"],
overrides=result.get("overrides", {}),
generate_pdf_output=result.get("generate_pdf_output", True),
report_title=result.get("report_title", result["report_name"]),
mailto=result.get("mailto", ""),
stdout=result.get("stdout", []),
)
elif cls == NotebookResultError:
return NotebookResultError(
job_id=result["job_id"],
job_start_time=result["job_start_time"],
report_name=result["report_name"],
status=job_status,
update_time=result["update_time"],
error_info=result["error_info"],
overrides=result.get("overrides", {}),
generate_pdf_output=result.get("generate_pdf_output", True),
report_title=result.get("report_title", result["report_name"]),
mailto=result.get("mailto", ""),
stdout=result.get("stdout", []),
)
else:
raise ValueError(
"Could not deserialise {} into result object.".format(result))
def test_report_hunter_pending_to_done(bson_library, webapp_config):
job_id = str(uuid.uuid4())
report_name = str(uuid.uuid4())
serializer = initialize_serializer_from_config(webapp_config)
with freezegun.freeze_time(datetime.datetime(2018, 1, 12, 2, 30)):
serializer.save_check_stub(job_id,
report_name,
status=JobStatus.SUBMITTED)
_report_hunter(webapp_config=webapp_config, run_once=True)
expected = NotebookResultPending(
job_id=job_id,
report_name=report_name,
report_title=report_name,
status=JobStatus.SUBMITTED,
update_time=datetime.datetime(2018, 1, 12, 2, 30),
job_start_time=datetime.datetime(2018, 1, 12, 2, 30),
)
assert get_report_cache(report_name,
job_id,
cache_dir=webapp_config.CACHE_DIR) == expected
with freezegun.freeze_time(datetime.datetime(2018, 1, 12, 2, 32)):
serializer.update_check_status(job_id, JobStatus.PENDING)
_report_hunter(webapp_config=webapp_config, run_once=True)
expected = NotebookResultPending(
job_id=job_id,
report_name=report_name,
report_title=report_name,
status=JobStatus.PENDING,
update_time=datetime.datetime(2018, 1, 12, 2, 32),
job_start_time=datetime.datetime(2018, 1, 12, 2, 30),
)
assert get_report_cache(report_name,
job_id,
cache_dir=webapp_config.CACHE_DIR) == expected
with freezegun.freeze_time(datetime.datetime(2018, 1, 12, 2, 37)):
serializer.update_check_status(
job_id,
JobStatus.DONE,
raw_html_resources={"outputs": {}},
job_finish_time=datetime.datetime.now(),
pdf="",
raw_ipynb_json="[]",
raw_html="",
)
_report_hunter(webapp_config=webapp_config, run_once=True)
expected = NotebookResultComplete(
job_id=job_id,
report_name=report_name,
report_title=report_name,
status=JobStatus.DONE,
update_time=datetime.datetime(2018, 1, 12, 2, 37),
job_start_time=datetime.datetime(2018, 1, 12, 2, 30),
job_finish_time=datetime.datetime(2018, 1, 12, 2, 37),
raw_html="",
raw_html_resources={"outputs": {}},
raw_ipynb_json="[]",
)
assert get_report_cache(report_name,
job_id,
cache_dir=webapp_config.CACHE_DIR) == expected