def test_report_hunter_timeout(bson_library, mongo_host, status, time_later,
should_timeout, test_db_name, test_lib_name):
job_id = str(uuid.uuid4())
report_name = str(uuid.uuid4())
serializer = PyMongoNotebookResultSerializer(
database_name=test_db_name,
mongo_host=mongo_host,
result_collection_name=test_lib_name)
start_time = time_now = datetime.datetime(2018, 1, 12, 2, 30)
with freezegun.freeze_time(time_now):
serializer.save_check_stub(job_id, report_name, status=status)
_report_hunter(
Serializer.PYMONGO.value,
mongo_host=mongo_host,
database_name=test_db_name,
result_collection_name=test_lib_name,
run_once=True,
)
expected = NotebookResultPending(
job_id=job_id,
report_name=report_name,
report_title=report_name,
status=status,
update_time=time_now,
job_start_time=start_time,
)
assert get_report_cache(report_name, job_id) == expected
time_now += time_later
with freezegun.freeze_time(time_now):
_report_hunter(
Serializer.PYMONGO.value,
mongo_host=mongo_host,
database_name=test_db_name,
result_collection_name=test_lib_name,
run_once=True,
)
if should_timeout:
mins = (time_later.total_seconds() / 60) - 1
expected = NotebookResultError(
job_id=job_id,
report_name=report_name,
report_title=report_name,
status=JobStatus.TIMEOUT,
update_time=time_now,
job_start_time=start_time,
error_info=
"This request timed out while being submitted to run. "
"Please try again! "
"Timed out after {:.0f} minutes 0 seconds.".format(mins),
)
else:
# expected does not change
pass
assert get_report_cache(report_name, job_id) == expected
def test_report_hunter_with_status_change(bson_library, mongo_host,
test_db_name, test_lib_name):
initialise_base_dirs()
serializer = PyMongoNotebookResultSerializer(
database_name=test_db_name,
mongo_host=mongo_host,
result_collection_name=test_lib_name)
job_id = str(uuid.uuid4())
report_name = str(uuid.uuid4())
with freezegun.freeze_time(datetime.datetime(2018, 1, 12, 2, 30)):
serializer.save_check_stub(job_id, report_name)
_report_hunter(
Serializer.PYMONGO.value,
mongo_host=mongo_host,
database_name=test_db_name,
result_collection_name=test_lib_name,
run_once=True,
)
expected = NotebookResultPending(
job_id=job_id,
report_name=report_name,
report_title=report_name,
update_time=datetime.datetime(2018, 1, 12, 2, 30),
job_start_time=datetime.datetime(2018, 1, 12, 2, 30),
)
assert get_report_cache(report_name, job_id) == expected
with freezegun.freeze_time(datetime.datetime(2018, 1, 12, 2, 32)):
serializer.update_check_status(job_id,
JobStatus.CANCELLED,
error_info="This was cancelled!")
_report_hunter(
Serializer.PYMONGO.value,
mongo_host=mongo_host,
database_name=test_db_name,
result_collection_name=test_lib_name,
run_once=True,
)
expected = NotebookResultError(
job_id=job_id,
report_name=report_name,
report_title=report_name,
status=JobStatus.CANCELLED,
update_time=datetime.datetime(2018, 1, 12, 2, 32),
job_start_time=datetime.datetime(2018, 1, 12, 2, 30),
error_info="This was cancelled!",
)
assert get_report_cache(report_name, job_id) == expected
def test_report_hunter_timeout(bson_library, status, time_later,
should_timeout, webapp_config):
job_id = str(uuid.uuid4())
report_name = str(uuid.uuid4())
serializer = initialize_serializer_from_config(webapp_config)
start_time = time_now = datetime.datetime(2018, 1, 12, 2, 30)
with freezegun.freeze_time(time_now):
serializer.save_check_stub(job_id, report_name, status=status)
_report_hunter(webapp_config=webapp_config, run_once=True)
expected = NotebookResultPending(
job_id=job_id,
report_name=report_name,
report_title=report_name,
status=status,
update_time=time_now,
job_start_time=start_time,
)
assert get_report_cache(report_name,
job_id,
cache_dir=webapp_config.CACHE_DIR) == expected
time_now += time_later
with freezegun.freeze_time(time_now):
_report_hunter(webapp_config=webapp_config, run_once=True)
if should_timeout:
mins = (time_later.total_seconds() / 60) - 1
expected = NotebookResultError(
job_id=job_id,
report_name=report_name,
report_title=report_name,
status=JobStatus.TIMEOUT,
update_time=time_now,
job_start_time=start_time,
error_info=
"This request timed out while being submitted to run. "
"Please try again! "
"Timed out after {:.0f} minutes 0 seconds.".format(mins),
)
else:
# expected does not change
pass
assert get_report_cache(report_name,
job_id,
cache_dir=webapp_config.CACHE_DIR) == expected
def test_report_hunter_with_status_change(bson_library, webapp_config):
initialise_base_dirs(webapp_config=webapp_config)
serializer = initialize_serializer_from_config(webapp_config)
job_id = str(uuid.uuid4())
report_name = str(uuid.uuid4())
with freezegun.freeze_time(datetime.datetime(2018, 1, 12, 2, 30)):
serializer.save_check_stub(job_id, report_name)
_report_hunter(webapp_config=webapp_config, run_once=True)
expected = NotebookResultPending(
job_id=job_id,
report_name=report_name,
report_title=report_name,
update_time=datetime.datetime(2018, 1, 12, 2, 30),
job_start_time=datetime.datetime(2018, 1, 12, 2, 30),
)
assert get_report_cache(report_name,
job_id,
cache_dir=webapp_config.CACHE_DIR) == expected
with freezegun.freeze_time(datetime.datetime(2018, 1, 12, 2, 32)):
serializer.update_check_status(job_id,
JobStatus.CANCELLED,
error_info="This was cancelled!")
_report_hunter(webapp_config=webapp_config, run_once=True)
expected = NotebookResultError(
job_id=job_id,
report_name=report_name,
report_title=report_name,
status=JobStatus.CANCELLED,
update_time=datetime.datetime(2018, 1, 12, 2, 32),
job_start_time=datetime.datetime(2018, 1, 12, 2, 30),
error_info="This was cancelled!",
)
assert get_report_cache(report_name,
job_id,
cache_dir=webapp_config.CACHE_DIR) == expected
def run_report(
job_submit_time,
report_name,
overrides,
result_serializer,
report_title="",
job_id=None,
output_base_dir=None,
template_base_dir=None,
attempts_remaining=2,
mailto="",
generate_pdf_output=True,
prepare_only=False,
):
job_id = job_id or str(uuid.uuid4())
stop_execution = os.getenv("NOTEBOOKER_APP_STOPPING")
if stop_execution:
logger.info(
"Aborting attempt to run %s, jobid=%s as app is shutting down.",
report_name, job_id)
result_serializer.update_check_status(job_id,
JobStatus.CANCELLED,
error_info=CANCEL_MESSAGE)
return
try:
logger.info(
"Calculating a new %s ipynb with parameters: %s (attempts remaining: %s)",
report_name,
overrides,
attempts_remaining,
)
result_serializer.update_check_status(job_id,
report_name=report_name,
job_start_time=job_submit_time,
status=JobStatus.PENDING)
result = _run_checks(
job_id,
job_submit_time,
report_name,
report_title,
output_base_dir,
template_base_dir,
overrides,
mailto=mailto,
generate_pdf_output=generate_pdf_output,
prepare_only=prepare_only,
)
logger.info("Successfully got result.")
result_serializer.save_check_result(result)
logger.info("Saved result to mongo successfully.")
except Exception:
error_info = traceback.format_exc()
logger.exception("%s report failed! (job id=%s)", report_name, job_id)
result = NotebookResultError(
job_id=job_id,
job_start_time=job_submit_time,
report_name=report_name,
report_title=report_title,
error_info=error_info,
overrides=overrides,
mailto=mailto,
generate_pdf_output=generate_pdf_output,
)
logger.error(
"Report run failed. Saving error result to mongo library %s@%s...",
result_serializer.database_name,
result_serializer.mongo_host,
)
result_serializer.save_check_result(result)
logger.info("Error result saved to mongo successfully.")
if attempts_remaining > 0:
logger.info("Retrying report.")
return run_report(
job_submit_time,
report_name,
overrides,
result_serializer,
report_title=report_title,
job_id=job_id,
output_base_dir=output_base_dir,
template_base_dir=template_base_dir,
attempts_remaining=attempts_remaining - 1,
mailto=mailto,
generate_pdf_output=generate_pdf_output,
prepare_only=prepare_only,
)
else:
logger.info(
"Abandoning attempt to run report. It failed too many times.")
return result
def _convert_result(
self,
result: Dict,
load_payload: bool = True
) -> Union[NotebookResultError, NotebookResultComplete,
NotebookResultPending, None]:
if not result:
return None
status = result.get("status", "")
job_status = JobStatus.from_string(status)
if job_status is None:
return None
cls = {
JobStatus.CANCELLED: NotebookResultError,
JobStatus.DONE: NotebookResultComplete,
JobStatus.PENDING: NotebookResultPending,
JobStatus.ERROR: NotebookResultError,
JobStatus.SUBMITTED: NotebookResultPending,
JobStatus.TIMEOUT: NotebookResultError,
JobStatus.DELETED: None,
}.get(job_status)
if cls is None:
return None
if load_payload and job_status == JobStatus.DONE:
def read_file(path):
try:
return self.result_data_store.get_last_version(path).read()
except NoFile:
logger.error("Could not find file %s in %s", path,
self.result_data_store)
return ""
outputs = {
path: read_file(path)
for path in result.get("raw_html_resources", {}).get(
"outputs", [])
}
result["raw_html_resources"]["outputs"] = outputs
if result.get("generate_pdf_output"):
pdf_filename = _pdf_filename(result["job_id"])
result["pdf"] = read_file(pdf_filename)
if cls == NotebookResultComplete:
return NotebookResultComplete(
job_id=result["job_id"],
job_start_time=result["job_start_time"],
report_name=result["report_name"],
status=job_status,
update_time=result["update_time"],
job_finish_time=result["job_finish_time"],
raw_html_resources=result.get("raw_html_resources", {}),
raw_ipynb_json=result.get("raw_ipynb_json"),
raw_html=result.get("raw_html"),
pdf=result.get("pdf", ""),
overrides=result.get("overrides", {}),
generate_pdf_output=result.get("generate_pdf_output", True),
report_title=result.get("report_title", result["report_name"]),
mailto=result.get("mailto", ""),
stdout=result.get("stdout", []),
)
elif cls == NotebookResultPending:
return NotebookResultPending(
job_id=result["job_id"],
job_start_time=result["job_start_time"],
report_name=result["report_name"],
status=job_status,
update_time=result["update_time"],
overrides=result.get("overrides", {}),
generate_pdf_output=result.get("generate_pdf_output", True),
report_title=result.get("report_title", result["report_name"]),
mailto=result.get("mailto", ""),
stdout=result.get("stdout", []),
)
elif cls == NotebookResultError:
return NotebookResultError(
job_id=result["job_id"],
job_start_time=result["job_start_time"],
report_name=result["report_name"],
status=job_status,
update_time=result["update_time"],
error_info=result["error_info"],
overrides=result.get("overrides", {}),
generate_pdf_output=result.get("generate_pdf_output", True),
report_title=result.get("report_title", result["report_name"]),
mailto=result.get("mailto", ""),
stdout=result.get("stdout", []),
)
else:
raise ValueError(
"Could not deserialise {} into result object.".format(result))