def test_execution_of_templates(template_name):
try:
_run_checks(
"job_id_{}".format(str(uuid.uuid4())[:6]),
datetime.datetime.now(),
template_name,
template_name,
get_output_dir(),
get_template_dir(),
{},
generate_pdf_output=False,
)
finally:
_cleanup_dirs()
def regression_test(template_dir):
logger.info("Starting regression test")
os.environ["PY_TEMPLATE_DIR"] = template_dir
try:
attempted_templates, failed_templates = [], set()
for template_name in templates._all_templates():
logger.info(
"============================[ Testing {} ]============================"
.format(template_name))
try:
attempted_templates.append(template_name)
_run_checks(
"job_id_{}".format(str(uuid.uuid4())[:6]),
datetime.datetime.now(),
template_name,
template_name,
filesystem.get_output_dir(),
filesystem.get_template_dir(),
{},
generate_pdf_output=False,
)
logger.info(
"===============================[ SUCCESS ]=============================="
)
except Exception:
failed_templates.add(template_name)
logger.info(
"===============================[ FAILED ]==============================="
)
logger.exception(
"Failed to execute template {}".format(template_name))
for template in attempted_templates:
logger.info("{}: {}".format(
template,
"FAILED" if template in failed_templates else "PASSED"))
if len(failed_templates) > 0:
raise NotebookRunException(
"The following templates failed to execute with no parameters:\n{}"
.format("\n".join(failed_templates)))
finally:
filesystem._cleanup_dirs()
def run_report(report_name,
report_title,
mailto,
overrides,
hide_code=False,
generate_pdf_output=False,
prepare_only=False):
"""
Actually run the report in earnest.
Uses a subprocess to execute the report asynchronously, which is identical to the non-webapp entrypoint.
:param report_name: `str` The report which we are executing
:param report_title: `str` The user-specified title of the report
:param mailto: `Optional[str]` Who the results will be emailed to
:param overrides: `Optional[Dict[str, Any]]` The parameters to be passed into the report
:param generate_pdf_output: `bool` Whether we're generating a PDF. Defaults to False.
:param prepare_only: `bool` Whether to do everything except execute the notebook. Useful for testing.
:return: The unique job_id.
"""
job_id = str(uuid.uuid4())
job_start_time = datetime.datetime.now()
result_serializer = get_serializer()
result_serializer.save_check_stub(
job_id,
report_name,
report_title=report_title,
job_start_time=job_start_time,
status=JobStatus.SUBMITTED,
overrides=overrides,
mailto=mailto,
generate_pdf_output=generate_pdf_output,
hide_code=hide_code,
)
app_config = current_app.config
p = subprocess.Popen(
[
"notebooker-cli",
"--output-base-dir",
get_output_dir(),
"--template-base-dir",
get_template_dir(),
"--py-template-base-dir",
app_config["PY_TEMPLATE_BASE_DIR"],
"--py-template-subdir",
app_config["PY_TEMPLATE_SUBDIR"],
] + (["--notebooker-disable-git"]
if app_config["NOTEBOOKER_DISABLE_GIT"] else []) +
["--serializer-cls", result_serializer.__class__.__name__] +
result_serializer.serializer_args_to_cmdline_args() + [
"execute-notebook",
"--job-id",
job_id,
"--report-name",
report_name,
"--report-title",
report_title,
"--mailto",
mailto,
"--overrides-as-json",
json.dumps(overrides),
"--pdf-output" if generate_pdf_output else "--no-pdf-output",
"--hide-code" if hide_code else "--show-code",
] + (["--prepare-notebook-only"] if prepare_only else []),
stderr=subprocess.PIPE,
)
stderr_thread = threading.Thread(
target=_monitor_stderr,
args=(p, job_id, current_app.config["SERIALIZER_CLS"],
current_app.config["SERIALIZER_CONFIG"]),
)
stderr_thread.daemon = True
stderr_thread.start()
return job_id
def run_report(report_name, report_title, mailto, overrides, generate_pdf_output=False, prepare_only=False):
"""
Actually run the report in earnest.
Uses a subprocess to execute the report asynchronously, which is identical to the non-webapp entrypoint.
:param report_name: `str` The report which we are executing
:param report_title: `str` The user-specified title of the report
:param mailto: `Optional[str]` Who the results will be emailed to
:param overrides: `Optional[Dict[str, Any]]` The parameters to be passed into the report
:param generate_pdf_output: `bool` Whether we're generating a PDF. Defaults to False.
:param prepare_only: `bool` Whether to do everything except execute the notebook. Useful for testing.
:return: The unique job_id.
"""
job_id = str(uuid.uuid4())
job_start_time = datetime.datetime.now()
result_serializer = get_serializer()
result_serializer.save_check_stub(
job_id,
report_name,
report_title=report_title,
job_start_time=job_start_time,
status=JobStatus.SUBMITTED,
overrides=overrides,
mailto=mailto,
generate_pdf_output=generate_pdf_output,
)
p = subprocess.Popen(
[
sys.executable,
"-m",
execute_notebook.__name__,
"--job-id",
job_id,
"--output-base-dir",
get_output_dir(),
"--template-base-dir",
get_template_dir(),
"--report-name",
report_name,
"--report-title",
report_title,
"--mailto",
mailto,
"--overrides-as-json",
json.dumps(overrides),
"--mongo-db-name",
result_serializer.database_name,
"--mongo-host",
result_serializer.mongo_host,
*(
("--mongo-user", result_serializer.user)
if result_serializer.user is not None else
()
),
*(
("--mongo-password", result_serializer.password)
if result_serializer.password is not None else
()
),
"--result-collection-name",
result_serializer.result_collection_name,
"--pdf-output" if generate_pdf_output else "--no-pdf-output",
"--serializer-cls",
result_serializer.__class__.__name__,
]
+ (["--prepare-notebook-only"] if prepare_only else []),
stderr=subprocess.PIPE,
)
stderr_thread = threading.Thread(target=_monitor_stderr, args=(p, job_id))
stderr_thread.daemon = True
stderr_thread.start()
return job_id