def blast_it(func, *args, **kwargs):
try:
initialise_base_dirs()
result = func(*args, **kwargs)
return result
finally:
_cleanup_dirs()
def setup_app(flask_app: Flask, web_config: WebappConfig):
# Setup environment
initialise_base_dirs(web_config)
logging.basicConfig(level=logging.getLevelName(web_config.LOGGING_LEVEL))
flask_app.config.from_object(web_config)
flask_app.config.update(TEMPLATES_AUTO_RELOAD=web_config.DEBUG,
EXPLAIN_TEMPLATE_LOADING=True,
DEBUG=web_config.DEBUG)
return flask_app
def setup_app(flask_app):
# Setup environment
setup_env_vars()
initialise_base_dirs()
logging.basicConfig(level=logging.getLevelName(os.getenv("LOGGING_LEVEL", "INFO")))
flask_app.config.update(
TEMPLATES_AUTO_RELOAD=bool(os.environ["DEBUG"]), EXPLAIN_TEMPLATE_LOADING=True, DEBUG=bool(os.environ["DEBUG"])
)
start_app(os.environ["NOTEBOOK_SERIALIZER"])
return flask_app
def test_report_hunter_with_status_change(bson_library, mongo_host,
test_db_name, test_lib_name):
initialise_base_dirs()
serializer = PyMongoNotebookResultSerializer(
database_name=test_db_name,
mongo_host=mongo_host,
result_collection_name=test_lib_name)
job_id = str(uuid.uuid4())
report_name = str(uuid.uuid4())
with freezegun.freeze_time(datetime.datetime(2018, 1, 12, 2, 30)):
serializer.save_check_stub(job_id, report_name)
_report_hunter(
Serializer.PYMONGO.value,
mongo_host=mongo_host,
database_name=test_db_name,
result_collection_name=test_lib_name,
run_once=True,
)
expected = NotebookResultPending(
job_id=job_id,
report_name=report_name,
report_title=report_name,
update_time=datetime.datetime(2018, 1, 12, 2, 30),
job_start_time=datetime.datetime(2018, 1, 12, 2, 30),
)
assert get_report_cache(report_name, job_id) == expected
with freezegun.freeze_time(datetime.datetime(2018, 1, 12, 2, 32)):
serializer.update_check_status(job_id,
JobStatus.CANCELLED,
error_info="This was cancelled!")
_report_hunter(
Serializer.PYMONGO.value,
mongo_host=mongo_host,
database_name=test_db_name,
result_collection_name=test_lib_name,
run_once=True,
)
expected = NotebookResultError(
job_id=job_id,
report_name=report_name,
report_title=report_name,
status=JobStatus.CANCELLED,
update_time=datetime.datetime(2018, 1, 12, 2, 32),
job_start_time=datetime.datetime(2018, 1, 12, 2, 30),
error_info="This was cancelled!",
)
assert get_report_cache(report_name, job_id) == expected
def test_report_hunter_with_status_change(bson_library, webapp_config):
initialise_base_dirs(webapp_config=webapp_config)
serializer = initialize_serializer_from_config(webapp_config)
job_id = str(uuid.uuid4())
report_name = str(uuid.uuid4())
with freezegun.freeze_time(datetime.datetime(2018, 1, 12, 2, 30)):
serializer.save_check_stub(job_id, report_name)
_report_hunter(webapp_config=webapp_config, run_once=True)
expected = NotebookResultPending(
job_id=job_id,
report_name=report_name,
report_title=report_name,
update_time=datetime.datetime(2018, 1, 12, 2, 30),
job_start_time=datetime.datetime(2018, 1, 12, 2, 30),
)
assert get_report_cache(report_name,
job_id,
cache_dir=webapp_config.CACHE_DIR) == expected
with freezegun.freeze_time(datetime.datetime(2018, 1, 12, 2, 32)):
serializer.update_check_status(job_id,
JobStatus.CANCELLED,
error_info="This was cancelled!")
_report_hunter(webapp_config=webapp_config, run_once=True)
expected = NotebookResultError(
job_id=job_id,
report_name=report_name,
report_title=report_name,
status=JobStatus.CANCELLED,
update_time=datetime.datetime(2018, 1, 12, 2, 32),
job_start_time=datetime.datetime(2018, 1, 12, 2, 30),
error_info="This was cancelled!",
)
assert get_report_cache(report_name,
job_id,
cache_dir=webapp_config.CACHE_DIR) == expected
def main(
report_name,
overrides_as_json,
iterate_override_values_of,
report_title,
n_retries,
mongo_db_name,
mongo_host,
mongo_user,
mongo_password,
result_collection_name,
notebook_kernel_name,
job_id,
output_base_dir,
template_base_dir,
mailto,
pdf_output,
serializer_cls,
prepare_notebook_only,
):
if report_name is None:
raise ValueError("Error! Please provide a --report-name.")
mongo_db_name = mongo_db_name or os.environ.get("DATABASE_NAME",
"notebooker")
mongo_host = mongo_host or os.environ.get("MONGO_HOST", "research")
mongo_user = mongo_user if mongo_user is not None else os.environ.get(
"MONGO_USER")
if mongo_user is None:
os.environ.pop("MONGO_USER", None)
else:
os.environ["MONGO_USER"] = mongo_user
mongo_password = mongo_password if mongo_password is not None else os.environ.get(
"MONGO_PASSWORD")
if mongo_password is None:
os.environ.pop("MONGO_PASSWORD", None)
else:
os.environ[
"MONGO_PASSWORD"] = mongo_password # FIXME: rather insecure..
result_collection_name = result_collection_name or os.environ.get(
"RESULT_COLLECTION_NAME", "NOTEBOOK_OUTPUT")
if notebook_kernel_name:
os.environ["NOTEBOOK_KERNEL_NAME"] = notebook_kernel_name
report_title = report_title or report_name
output_dir, template_dir, _ = initialise_base_dirs(
output_dir=output_base_dir, template_dir=template_base_dir)
all_overrides = _get_overrides(overrides_as_json,
iterate_override_values_of)
start_time = datetime.datetime.now()
logger.info("Running a report with these parameters:")
logger.info("report_name = %s", report_name)
logger.info("overrides_as_json = %s", overrides_as_json)
logger.info("iterate_override_values_of = %s", iterate_override_values_of)
logger.info("report_title = %s", report_title)
logger.info("n_retries = %s", n_retries)
logger.info("mongo_db_name = %s", mongo_db_name)
logger.info("mongo_host = %s", mongo_host)
logger.info("mongo_user = %s", mongo_user)
logger.info("mongo_password = %s", "*******")
logger.info("result_collection_name = %s", result_collection_name)
logger.info("job_id = %s", job_id)
logger.info("output_dir = %s", output_dir)
logger.info("template_dir = %s", template_dir)
logger.info("mailto = %s", mailto)
logger.info("pdf_output = %s", pdf_output)
logger.info("prepare_notebook_only = %s", prepare_notebook_only)
logger.info("Calculated overrides are: %s", str(all_overrides))
result_serializer = get_serializer_from_cls(
serializer_cls,
database_name=mongo_db_name,
mongo_host=mongo_host,
result_collection_name=result_collection_name,
user=mongo_user,
password=mongo_password,
)
results = []
for overrides in all_overrides:
result = run_report(
start_time,
report_name,
overrides,
result_serializer,
report_title=report_title,
job_id=job_id,
output_base_dir=output_dir,
template_base_dir=template_dir,
attempts_remaining=n_retries - 1,
mailto=mailto,
generate_pdf_output=pdf_output,
prepare_only=prepare_notebook_only,
)
if mailto:
send_result_email(result, mailto)
if isinstance(result, NotebookResultError):
logger.warning("Notebook execution failed! Output was:")
logger.warning(repr(result))
raise Exception(result.error_info)
results.append(result)
return results
def execute_notebook_entrypoint(
config: BaseConfig,
report_name: str,
overrides_as_json: str,
iterate_override_values_of: Union[List[str], str],
report_title: str,
n_retries: int,
job_id: str,
mailto: str,
error_mailto: str,
email_subject: str,
pdf_output: bool,
hide_code: bool,
prepare_notebook_only: bool,
):
report_title = report_title or report_name
output_dir, template_dir, _ = initialise_base_dirs(
output_dir=config.OUTPUT_DIR, template_dir=config.TEMPLATE_DIR)
all_overrides = _get_overrides(overrides_as_json,
iterate_override_values_of)
notebooker_disable_git = config.NOTEBOOKER_DISABLE_GIT
py_template_base_dir = config.PY_TEMPLATE_BASE_DIR
py_template_subdir = config.PY_TEMPLATE_SUBDIR
start_time = datetime.datetime.now()
logger.info("Running a report with these parameters:")
logger.info("report_name = %s", report_name)
logger.info("overrides_as_json = %s", overrides_as_json)
logger.info("iterate_override_values_of = %s", iterate_override_values_of)
logger.info("report_title = %s", report_title)
logger.info("n_retries = %s", n_retries)
logger.info("job_id = %s", job_id)
logger.info("output_dir = %s", output_dir)
logger.info("template_dir = %s", template_dir)
logger.info("mailto = %s", mailto)
logger.info("error_mailto = %s", error_mailto)
logger.info("email_subject = %s", email_subject)
logger.info("pdf_output = %s", pdf_output)
logger.info("hide_code = %s", hide_code)
logger.info("prepare_notebook_only = %s", prepare_notebook_only)
logger.info("notebooker_disable_git = %s", notebooker_disable_git)
logger.info("py_template_base_dir = %s", py_template_base_dir)
logger.info("py_template_subdir = %s", py_template_subdir)
logger.info("serializer_cls = %s", config.SERIALIZER_CLS)
logger.info("serializer_config = %s", config.SERIALIZER_CONFIG)
logger.info("Calculated overrides are: %s", str(all_overrides))
result_serializer = get_serializer_from_cls(config.SERIALIZER_CLS,
**config.SERIALIZER_CONFIG)
results = []
for overrides in all_overrides:
result = run_report(
start_time,
report_name,
overrides,
result_serializer,
report_title=report_title,
job_id=job_id,
output_base_dir=output_dir,
template_base_dir=template_dir,
attempts_remaining=n_retries - 1,
mailto=mailto,
error_mailto=error_mailto,
email_subject=email_subject,
generate_pdf_output=pdf_output,
hide_code=hide_code,
prepare_only=prepare_notebook_only,
notebooker_disable_git=notebooker_disable_git,
py_template_base_dir=py_template_base_dir,
py_template_subdir=py_template_subdir,
)
if result.mailto:
send_result_email(result)
if isinstance(result, NotebookResultError):
logger.warning("Notebook execution failed! Output was:")
logger.warning(repr(result))
raise Exception(result.error_info)
results.append(result)
return results
def setup_and_cleanup_notebooker_filesystem(webapp_config):
try:
initialise_base_dirs(webapp_config=webapp_config)
yield
finally:
_cleanup_dirs(webapp_config)