def test_send_result_email(test_name, job_id, report_name, report_title, expected_title): body_in = "<body><h1>hello 😆 😉 😊 😋 😎</h1></body>" result = NotebookResultComplete( job_id=job_id, job_start_time=datetime.datetime.now(), job_finish_time=datetime.datetime.now(), raw_html_resources={}, raw_ipynb_json={}, raw_html=body_in, pdf="", report_name=report_name, report_title=report_title, ) to_email = "∫åññîsté®@test-email.com" with mock.patch("notebooker.utils.notebook_execution.mail") as mail: notebook_execution.send_result_email(result, to_email) email_sent = mail.mock_calls[0][1] assert len(email_sent ) == 4, "mail() was not called with the correct number of args" from_address = email_sent[0] to_address = email_sent[1] title = email_sent[2] body = email_sent[3] assert from_address == "*****@*****.**" assert to_address == to_email assert title == expected_title assert body == [ "Please either activate HTML emails, or see the PDF attachment.", body_in ]
def test_send_result_email_fallback_to_raw_html(): body_in = "<body><h1>hello 😆 😉 😊 😋 😎</h1></body>" to_email = "hello@you" result = NotebookResultComplete( job_id="fallback", job_start_time=datetime.datetime.now(), job_finish_time=datetime.datetime.now(), mailto=to_email, raw_html_resources={}, raw_ipynb_json={}, raw_html=body_in, pdf="", report_name="fallback", report_title="fallback", ) with mock.patch("notebooker.utils.notebook_execution.mail") as mail: notebook_execution.send_result_email(result) email_sent = mail.mock_calls[0][1] assert len(email_sent ) == 4, "mail() was not called with the correct number of args" from_address = email_sent[0] to_address = email_sent[1] title = email_sent[2] body = email_sent[3] assert from_address == "*****@*****.**" assert to_address == to_email assert title == "Notebooker: fallback report completed with status: Checks done!" assert body == [ "Please either activate HTML emails, or see the PDF attachment.", body_in ]
def _run_checks( job_id: str, job_start_time: datetime.datetime, template_name: str, report_title: str, output_base_dir: str, template_base_dir: str, overrides: Dict[AnyStr, Any], generate_pdf_output: Optional[bool] = True, mailto: Optional[str] = "", prepare_only: Optional[bool] = False, ) -> NotebookResultComplete: """ This is the actual method which executes a notebook, whether running in the webapp or via the entrypoint. If this crashes, an exception is raised (and should be caught by run_checks().) Parameters ---------- job_id : `str` The unique ID of this report. job_start_time : `datetime.datetime` The UTC start time of this report. template_name : `str` The name of the template which we are running. NB this should be a path relative to the python_template_dir() report_title : `str` The user-specified optional title of the report. Defaults to the template name. output_base_dir : `str` Internal use. The temp directory where output is being saved, local to the executor. template_base_dir : `str` Internal use. The temp directory where the .py->.ipynb converted templates reside, local to the executor. overrides : Dict[AnyStr, Any] The dictionary of overrides which parametrize our Notebook Template. generate_pdf_output : `Optional[bool]` Whether to generate PDF output or not. NB this requires xelatex to be installed on the executor. mailto : `Optional[str]` Comma-separated email addresses to send on completion (or error). prepare_only : `Optional[bool]` Internal usage. Whether we want to do everything apart from executing the notebook. Returns ------- NotebookResultComplete Raises ------ Exception() """ output_dir = _output_dir(output_base_dir, template_name, job_id) output_ipynb = _output_ipynb_name(template_name) if not os.path.isdir(output_dir): logger.info("Making dir @ {}".format(output_dir)) os.makedirs(output_dir) ipynb_raw_path = generate_ipynb_from_py(template_base_dir, template_name) ipynb_executed_path = os.path.join(output_dir, output_ipynb) logger.info("Executing notebook at {} using parameters {} --> {}".format( ipynb_raw_path, overrides, output_ipynb)) pm.execute_notebook(ipynb_raw_path, ipynb_executed_path, parameters=overrides, log_output=True, prepare_only=prepare_only) with open(ipynb_executed_path, "r") as f: raw_executed_ipynb = f.read() logger.info( "Saving output notebook as HTML from {}".format(ipynb_executed_path)) html, resources = ipython_to_html(ipynb_executed_path, job_id) pdf = ipython_to_pdf(raw_executed_ipynb, report_title) if generate_pdf_output else "" notebook_result = NotebookResultComplete( job_id=job_id, job_start_time=job_start_time, job_finish_time=datetime.datetime.now(), raw_html_resources=resources, raw_ipynb_json=raw_executed_ipynb, raw_html=html, mailto=mailto, pdf=pdf, generate_pdf_output=generate_pdf_output, report_name=template_name, report_title=report_title, overrides=overrides, ) return notebook_result
def test_report_hunter_pending_to_done(bson_library, mongo_host, test_db_name, test_lib_name): job_id = str(uuid.uuid4()) report_name = str(uuid.uuid4()) serializer = PyMongoNotebookResultSerializer( database_name=test_db_name, mongo_host=mongo_host, result_collection_name=test_lib_name) with freezegun.freeze_time(datetime.datetime(2018, 1, 12, 2, 30)): serializer.save_check_stub(job_id, report_name, status=JobStatus.SUBMITTED) _report_hunter( Serializer.PYMONGO.value, mongo_host=mongo_host, database_name=test_db_name, result_collection_name=test_lib_name, run_once=True, ) expected = NotebookResultPending( job_id=job_id, report_name=report_name, report_title=report_name, status=JobStatus.SUBMITTED, update_time=datetime.datetime(2018, 1, 12, 2, 30), job_start_time=datetime.datetime(2018, 1, 12, 2, 30), ) assert get_report_cache(report_name, job_id) == expected with freezegun.freeze_time(datetime.datetime(2018, 1, 12, 2, 32)): serializer.update_check_status(job_id, JobStatus.PENDING) _report_hunter( Serializer.PYMONGO.value, mongo_host=mongo_host, database_name=test_db_name, result_collection_name=test_lib_name, run_once=True, ) expected = NotebookResultPending( job_id=job_id, report_name=report_name, report_title=report_name, status=JobStatus.PENDING, update_time=datetime.datetime(2018, 1, 12, 2, 32), job_start_time=datetime.datetime(2018, 1, 12, 2, 30), ) assert get_report_cache(report_name, job_id) == expected with freezegun.freeze_time(datetime.datetime(2018, 1, 12, 2, 37)): serializer.update_check_status( job_id, JobStatus.DONE, raw_html_resources={"outputs": {}}, job_finish_time=datetime.datetime.now(), pdf="", raw_ipynb_json="[]", raw_html="", ) _report_hunter( Serializer.PYMONGO.value, mongo_host=mongo_host, database_name=test_db_name, result_collection_name=test_lib_name, run_once=True, ) expected = NotebookResultComplete( job_id=job_id, report_name=report_name, report_title=report_name, status=JobStatus.DONE, update_time=datetime.datetime(2018, 1, 12, 2, 37), job_start_time=datetime.datetime(2018, 1, 12, 2, 30), job_finish_time=datetime.datetime(2018, 1, 12, 2, 37), raw_html="", raw_html_resources={"outputs": {}}, raw_ipynb_json="[]", ) assert get_report_cache(report_name, job_id) == expected
def _convert_result( self, result: Dict, load_payload: bool = True ) -> Union[NotebookResultError, NotebookResultComplete, NotebookResultPending, None]: if not result: return None status = result.get("status", "") job_status = JobStatus.from_string(status) if job_status is None: return None cls = { JobStatus.CANCELLED: NotebookResultError, JobStatus.DONE: NotebookResultComplete, JobStatus.PENDING: NotebookResultPending, JobStatus.ERROR: NotebookResultError, JobStatus.SUBMITTED: NotebookResultPending, JobStatus.TIMEOUT: NotebookResultError, JobStatus.DELETED: None, }.get(job_status) if cls is None: return None if load_payload and job_status == JobStatus.DONE: def read_file(path): try: return self.result_data_store.get_last_version(path).read() except NoFile: logger.error("Could not find file %s in %s", path, self.result_data_store) return "" outputs = { path: read_file(path) for path in result.get("raw_html_resources", {}).get( "outputs", []) } result["raw_html_resources"]["outputs"] = outputs if result.get("generate_pdf_output"): pdf_filename = _pdf_filename(result["job_id"]) result["pdf"] = read_file(pdf_filename) if cls == NotebookResultComplete: return NotebookResultComplete( job_id=result["job_id"], job_start_time=result["job_start_time"], report_name=result["report_name"], status=job_status, update_time=result["update_time"], job_finish_time=result["job_finish_time"], raw_html_resources=result.get("raw_html_resources", {}), raw_ipynb_json=result.get("raw_ipynb_json"), raw_html=result.get("raw_html"), pdf=result.get("pdf", ""), overrides=result.get("overrides", {}), generate_pdf_output=result.get("generate_pdf_output", True), report_title=result.get("report_title", result["report_name"]), mailto=result.get("mailto", ""), stdout=result.get("stdout", []), ) elif cls == NotebookResultPending: return NotebookResultPending( job_id=result["job_id"], job_start_time=result["job_start_time"], report_name=result["report_name"], status=job_status, update_time=result["update_time"], overrides=result.get("overrides", {}), generate_pdf_output=result.get("generate_pdf_output", True), report_title=result.get("report_title", result["report_name"]), mailto=result.get("mailto", ""), stdout=result.get("stdout", []), ) elif cls == NotebookResultError: return NotebookResultError( job_id=result["job_id"], job_start_time=result["job_start_time"], report_name=result["report_name"], status=job_status, update_time=result["update_time"], error_info=result["error_info"], overrides=result.get("overrides", {}), generate_pdf_output=result.get("generate_pdf_output", True), report_title=result.get("report_title", result["report_name"]), mailto=result.get("mailto", ""), stdout=result.get("stdout", []), ) else: raise ValueError( "Could not deserialise {} into result object.".format(result))
def test_report_hunter_pending_to_done(bson_library, webapp_config): job_id = str(uuid.uuid4()) report_name = str(uuid.uuid4()) serializer = initialize_serializer_from_config(webapp_config) with freezegun.freeze_time(datetime.datetime(2018, 1, 12, 2, 30)): serializer.save_check_stub(job_id, report_name, status=JobStatus.SUBMITTED) _report_hunter(webapp_config=webapp_config, run_once=True) expected = NotebookResultPending( job_id=job_id, report_name=report_name, report_title=report_name, status=JobStatus.SUBMITTED, update_time=datetime.datetime(2018, 1, 12, 2, 30), job_start_time=datetime.datetime(2018, 1, 12, 2, 30), ) assert get_report_cache(report_name, job_id, cache_dir=webapp_config.CACHE_DIR) == expected with freezegun.freeze_time(datetime.datetime(2018, 1, 12, 2, 32)): serializer.update_check_status(job_id, JobStatus.PENDING) _report_hunter(webapp_config=webapp_config, run_once=True) expected = NotebookResultPending( job_id=job_id, report_name=report_name, report_title=report_name, status=JobStatus.PENDING, update_time=datetime.datetime(2018, 1, 12, 2, 32), job_start_time=datetime.datetime(2018, 1, 12, 2, 30), ) assert get_report_cache(report_name, job_id, cache_dir=webapp_config.CACHE_DIR) == expected with freezegun.freeze_time(datetime.datetime(2018, 1, 12, 2, 37)): serializer.update_check_status( job_id, JobStatus.DONE, raw_html_resources={"outputs": {}}, job_finish_time=datetime.datetime.now(), pdf="", raw_ipynb_json="[]", raw_html="", ) _report_hunter(webapp_config=webapp_config, run_once=True) expected = NotebookResultComplete( job_id=job_id, report_name=report_name, report_title=report_name, status=JobStatus.DONE, update_time=datetime.datetime(2018, 1, 12, 2, 37), job_start_time=datetime.datetime(2018, 1, 12, 2, 30), job_finish_time=datetime.datetime(2018, 1, 12, 2, 37), raw_html="", raw_html_resources={"outputs": {}}, raw_ipynb_json="[]", ) assert get_report_cache(report_name, job_id, cache_dir=webapp_config.CACHE_DIR) == expected