def save_check_stub( self, job_id: str, report_name: str, report_title: Optional[str] = "", job_start_time: Optional[datetime.datetime] = None, status: JobStatus = JobStatus.PENDING, overrides: Optional[Dict] = None, mailto: str = "", generate_pdf_output: bool = True, ) -> None: """ Call this when we are just starting a check. Saves a "pending" job into storage. """ job_start_time = job_start_time or datetime.datetime.now() report_title = report_title or report_name pending_result = NotebookResultPending( job_id=job_id, status=status, report_title=report_title, job_start_time=job_start_time, report_name=report_name, mailto=mailto, generate_pdf_output=generate_pdf_output, overrides=overrides or {}, ) self._save_to_db(pending_result)
def test_report_hunter_with_one(bson_library, mongo_host, test_db_name, test_lib_name): serializer = PyMongoNotebookResultSerializer( database_name=test_db_name, mongo_host=mongo_host, result_collection_name=test_lib_name) job_id = str(uuid.uuid4()) report_name = str(uuid.uuid4()) serializer.save_check_stub(job_id, report_name) _report_hunter( Serializer.PYMONGO.value, mongo_host=mongo_host, database_name=test_db_name, result_collection_name=test_lib_name, run_once=True, ) expected = NotebookResultPending( job_id=job_id, report_name=report_name, report_title=report_name, update_time=datetime.datetime(2018, 1, 12), job_start_time=datetime.datetime(2018, 1, 12), ) assert get_report_cache(report_name, job_id) == expected
def test_report_hunter_timeout(bson_library, mongo_host, status, time_later, should_timeout, test_db_name, test_lib_name): job_id = str(uuid.uuid4()) report_name = str(uuid.uuid4()) serializer = PyMongoNotebookResultSerializer( database_name=test_db_name, mongo_host=mongo_host, result_collection_name=test_lib_name) start_time = time_now = datetime.datetime(2018, 1, 12, 2, 30) with freezegun.freeze_time(time_now): serializer.save_check_stub(job_id, report_name, status=status) _report_hunter( Serializer.PYMONGO.value, mongo_host=mongo_host, database_name=test_db_name, result_collection_name=test_lib_name, run_once=True, ) expected = NotebookResultPending( job_id=job_id, report_name=report_name, report_title=report_name, status=status, update_time=time_now, job_start_time=start_time, ) assert get_report_cache(report_name, job_id) == expected time_now += time_later with freezegun.freeze_time(time_now): _report_hunter( Serializer.PYMONGO.value, mongo_host=mongo_host, database_name=test_db_name, result_collection_name=test_lib_name, run_once=True, ) if should_timeout: mins = (time_later.total_seconds() / 60) - 1 expected = NotebookResultError( job_id=job_id, report_name=report_name, report_title=report_name, status=JobStatus.TIMEOUT, update_time=time_now, job_start_time=start_time, error_info= "This request timed out while being submitted to run. " "Please try again! " "Timed out after {:.0f} minutes 0 seconds.".format(mins), ) else: # expected does not change pass assert get_report_cache(report_name, job_id) == expected
def test_report_hunter_with_status_change(bson_library, mongo_host, test_db_name, test_lib_name): initialise_base_dirs() serializer = PyMongoNotebookResultSerializer( database_name=test_db_name, mongo_host=mongo_host, result_collection_name=test_lib_name) job_id = str(uuid.uuid4()) report_name = str(uuid.uuid4()) with freezegun.freeze_time(datetime.datetime(2018, 1, 12, 2, 30)): serializer.save_check_stub(job_id, report_name) _report_hunter( Serializer.PYMONGO.value, mongo_host=mongo_host, database_name=test_db_name, result_collection_name=test_lib_name, run_once=True, ) expected = NotebookResultPending( job_id=job_id, report_name=report_name, report_title=report_name, update_time=datetime.datetime(2018, 1, 12, 2, 30), job_start_time=datetime.datetime(2018, 1, 12, 2, 30), ) assert get_report_cache(report_name, job_id) == expected with freezegun.freeze_time(datetime.datetime(2018, 1, 12, 2, 32)): serializer.update_check_status(job_id, JobStatus.CANCELLED, error_info="This was cancelled!") _report_hunter( Serializer.PYMONGO.value, mongo_host=mongo_host, database_name=test_db_name, result_collection_name=test_lib_name, run_once=True, ) expected = NotebookResultError( job_id=job_id, report_name=report_name, report_title=report_name, status=JobStatus.CANCELLED, update_time=datetime.datetime(2018, 1, 12, 2, 32), job_start_time=datetime.datetime(2018, 1, 12, 2, 30), error_info="This was cancelled!", ) assert get_report_cache(report_name, job_id) == expected
def test_report_hunter_timeout(bson_library, status, time_later, should_timeout, webapp_config): job_id = str(uuid.uuid4()) report_name = str(uuid.uuid4()) serializer = initialize_serializer_from_config(webapp_config) start_time = time_now = datetime.datetime(2018, 1, 12, 2, 30) with freezegun.freeze_time(time_now): serializer.save_check_stub(job_id, report_name, status=status) _report_hunter(webapp_config=webapp_config, run_once=True) expected = NotebookResultPending( job_id=job_id, report_name=report_name, report_title=report_name, status=status, update_time=time_now, job_start_time=start_time, ) assert get_report_cache(report_name, job_id, cache_dir=webapp_config.CACHE_DIR) == expected time_now += time_later with freezegun.freeze_time(time_now): _report_hunter(webapp_config=webapp_config, run_once=True) if should_timeout: mins = (time_later.total_seconds() / 60) - 1 expected = NotebookResultError( job_id=job_id, report_name=report_name, report_title=report_name, status=JobStatus.TIMEOUT, update_time=time_now, job_start_time=start_time, error_info= "This request timed out while being submitted to run. " "Please try again! " "Timed out after {:.0f} minutes 0 seconds.".format(mins), ) else: # expected does not change pass assert get_report_cache(report_name, job_id, cache_dir=webapp_config.CACHE_DIR) == expected
def test_report_hunter_with_one(bson_library, webapp_config): serializer = initialize_serializer_from_config(webapp_config) job_id = str(uuid.uuid4()) report_name = str(uuid.uuid4()) serializer.save_check_stub(job_id, report_name) _report_hunter(webapp_config=webapp_config, run_once=True) expected = NotebookResultPending( job_id=job_id, report_name=report_name, report_title=report_name, update_time=datetime.datetime(2018, 1, 12), job_start_time=datetime.datetime(2018, 1, 12), ) assert get_report_cache(report_name, job_id, cache_dir=webapp_config.CACHE_DIR) == expected
def test_report_hunter_with_status_change(bson_library, webapp_config): initialise_base_dirs(webapp_config=webapp_config) serializer = initialize_serializer_from_config(webapp_config) job_id = str(uuid.uuid4()) report_name = str(uuid.uuid4()) with freezegun.freeze_time(datetime.datetime(2018, 1, 12, 2, 30)): serializer.save_check_stub(job_id, report_name) _report_hunter(webapp_config=webapp_config, run_once=True) expected = NotebookResultPending( job_id=job_id, report_name=report_name, report_title=report_name, update_time=datetime.datetime(2018, 1, 12, 2, 30), job_start_time=datetime.datetime(2018, 1, 12, 2, 30), ) assert get_report_cache(report_name, job_id, cache_dir=webapp_config.CACHE_DIR) == expected with freezegun.freeze_time(datetime.datetime(2018, 1, 12, 2, 32)): serializer.update_check_status(job_id, JobStatus.CANCELLED, error_info="This was cancelled!") _report_hunter(webapp_config=webapp_config, run_once=True) expected = NotebookResultError( job_id=job_id, report_name=report_name, report_title=report_name, status=JobStatus.CANCELLED, update_time=datetime.datetime(2018, 1, 12, 2, 32), job_start_time=datetime.datetime(2018, 1, 12, 2, 30), error_info="This was cancelled!", ) assert get_report_cache(report_name, job_id, cache_dir=webapp_config.CACHE_DIR) == expected
def test_report_hunter_pending_to_done(bson_library, mongo_host, test_db_name, test_lib_name): job_id = str(uuid.uuid4()) report_name = str(uuid.uuid4()) serializer = PyMongoNotebookResultSerializer( database_name=test_db_name, mongo_host=mongo_host, result_collection_name=test_lib_name) with freezegun.freeze_time(datetime.datetime(2018, 1, 12, 2, 30)): serializer.save_check_stub(job_id, report_name, status=JobStatus.SUBMITTED) _report_hunter( Serializer.PYMONGO.value, mongo_host=mongo_host, database_name=test_db_name, result_collection_name=test_lib_name, run_once=True, ) expected = NotebookResultPending( job_id=job_id, report_name=report_name, report_title=report_name, status=JobStatus.SUBMITTED, update_time=datetime.datetime(2018, 1, 12, 2, 30), job_start_time=datetime.datetime(2018, 1, 12, 2, 30), ) assert get_report_cache(report_name, job_id) == expected with freezegun.freeze_time(datetime.datetime(2018, 1, 12, 2, 32)): serializer.update_check_status(job_id, JobStatus.PENDING) _report_hunter( Serializer.PYMONGO.value, mongo_host=mongo_host, database_name=test_db_name, result_collection_name=test_lib_name, run_once=True, ) expected = NotebookResultPending( job_id=job_id, report_name=report_name, report_title=report_name, status=JobStatus.PENDING, update_time=datetime.datetime(2018, 1, 12, 2, 32), job_start_time=datetime.datetime(2018, 1, 12, 2, 30), ) assert get_report_cache(report_name, job_id) == expected with freezegun.freeze_time(datetime.datetime(2018, 1, 12, 2, 37)): serializer.update_check_status( job_id, JobStatus.DONE, raw_html_resources={"outputs": {}}, job_finish_time=datetime.datetime.now(), pdf="", raw_ipynb_json="[]", raw_html="", ) _report_hunter( Serializer.PYMONGO.value, mongo_host=mongo_host, database_name=test_db_name, result_collection_name=test_lib_name, run_once=True, ) expected = NotebookResultComplete( job_id=job_id, report_name=report_name, report_title=report_name, status=JobStatus.DONE, update_time=datetime.datetime(2018, 1, 12, 2, 37), job_start_time=datetime.datetime(2018, 1, 12, 2, 30), job_finish_time=datetime.datetime(2018, 1, 12, 2, 37), raw_html="", raw_html_resources={"outputs": {}}, raw_ipynb_json="[]", ) assert get_report_cache(report_name, job_id) == expected
def _convert_result( self, result: Dict, load_payload: bool = True ) -> Union[NotebookResultError, NotebookResultComplete, NotebookResultPending, None]: if not result: return None status = result.get("status", "") job_status = JobStatus.from_string(status) if job_status is None: return None cls = { JobStatus.CANCELLED: NotebookResultError, JobStatus.DONE: NotebookResultComplete, JobStatus.PENDING: NotebookResultPending, JobStatus.ERROR: NotebookResultError, JobStatus.SUBMITTED: NotebookResultPending, JobStatus.TIMEOUT: NotebookResultError, JobStatus.DELETED: None, }.get(job_status) if cls is None: return None if load_payload and job_status == JobStatus.DONE: def read_file(path): try: return self.result_data_store.get_last_version(path).read() except NoFile: logger.error("Could not find file %s in %s", path, self.result_data_store) return "" outputs = { path: read_file(path) for path in result.get("raw_html_resources", {}).get( "outputs", []) } result["raw_html_resources"]["outputs"] = outputs if result.get("generate_pdf_output"): pdf_filename = _pdf_filename(result["job_id"]) result["pdf"] = read_file(pdf_filename) if cls == NotebookResultComplete: return NotebookResultComplete( job_id=result["job_id"], job_start_time=result["job_start_time"], report_name=result["report_name"], status=job_status, update_time=result["update_time"], job_finish_time=result["job_finish_time"], raw_html_resources=result.get("raw_html_resources", {}), raw_ipynb_json=result.get("raw_ipynb_json"), raw_html=result.get("raw_html"), pdf=result.get("pdf", ""), overrides=result.get("overrides", {}), generate_pdf_output=result.get("generate_pdf_output", True), report_title=result.get("report_title", result["report_name"]), mailto=result.get("mailto", ""), stdout=result.get("stdout", []), ) elif cls == NotebookResultPending: return NotebookResultPending( job_id=result["job_id"], job_start_time=result["job_start_time"], report_name=result["report_name"], status=job_status, update_time=result["update_time"], overrides=result.get("overrides", {}), generate_pdf_output=result.get("generate_pdf_output", True), report_title=result.get("report_title", result["report_name"]), mailto=result.get("mailto", ""), stdout=result.get("stdout", []), ) elif cls == NotebookResultError: return NotebookResultError( job_id=result["job_id"], job_start_time=result["job_start_time"], report_name=result["report_name"], status=job_status, update_time=result["update_time"], error_info=result["error_info"], overrides=result.get("overrides", {}), generate_pdf_output=result.get("generate_pdf_output", True), report_title=result.get("report_title", result["report_name"]), mailto=result.get("mailto", ""), stdout=result.get("stdout", []), ) else: raise ValueError( "Could not deserialise {} into result object.".format(result))
def test_report_hunter_pending_to_done(bson_library, webapp_config): job_id = str(uuid.uuid4()) report_name = str(uuid.uuid4()) serializer = initialize_serializer_from_config(webapp_config) with freezegun.freeze_time(datetime.datetime(2018, 1, 12, 2, 30)): serializer.save_check_stub(job_id, report_name, status=JobStatus.SUBMITTED) _report_hunter(webapp_config=webapp_config, run_once=True) expected = NotebookResultPending( job_id=job_id, report_name=report_name, report_title=report_name, status=JobStatus.SUBMITTED, update_time=datetime.datetime(2018, 1, 12, 2, 30), job_start_time=datetime.datetime(2018, 1, 12, 2, 30), ) assert get_report_cache(report_name, job_id, cache_dir=webapp_config.CACHE_DIR) == expected with freezegun.freeze_time(datetime.datetime(2018, 1, 12, 2, 32)): serializer.update_check_status(job_id, JobStatus.PENDING) _report_hunter(webapp_config=webapp_config, run_once=True) expected = NotebookResultPending( job_id=job_id, report_name=report_name, report_title=report_name, status=JobStatus.PENDING, update_time=datetime.datetime(2018, 1, 12, 2, 32), job_start_time=datetime.datetime(2018, 1, 12, 2, 30), ) assert get_report_cache(report_name, job_id, cache_dir=webapp_config.CACHE_DIR) == expected with freezegun.freeze_time(datetime.datetime(2018, 1, 12, 2, 37)): serializer.update_check_status( job_id, JobStatus.DONE, raw_html_resources={"outputs": {}}, job_finish_time=datetime.datetime.now(), pdf="", raw_ipynb_json="[]", raw_html="", ) _report_hunter(webapp_config=webapp_config, run_once=True) expected = NotebookResultComplete( job_id=job_id, report_name=report_name, report_title=report_name, status=JobStatus.DONE, update_time=datetime.datetime(2018, 1, 12, 2, 37), job_start_time=datetime.datetime(2018, 1, 12, 2, 30), job_finish_time=datetime.datetime(2018, 1, 12, 2, 37), raw_html="", raw_html_resources={"outputs": {}}, raw_ipynb_json="[]", ) assert get_report_cache(report_name, job_id, cache_dir=webapp_config.CACHE_DIR) == expected