def run_report_http(report_name):
"""
The "Run Report" interface is generated by this method.
:param report_name: The parameter here should be a "/"-delimited string which mirrors the directory structure of \
the notebook templates.
:returns: An HTML template which is the Run Report interface.
"""
report_name = convert_report_name_url_to_path(report_name)
json_params = request.args.get("json_params")
initial_python_parameters = json_to_python(json_params) or ""
try:
path = generate_ipynb_from_py(get_template_dir(), report_name)
except FileNotFoundError as e:
logger.exception(e)
return "", 404
nb = nbformat.read(path, as_version=nbformat.v4.nbformat)
metadata_idx = _get_parameters_cell_idx(nb)
parameters_as_html = ""
has_prefix = has_suffix = False
if metadata_idx is not None:
metadata = nb["cells"][metadata_idx]
parameters_as_html = metadata["source"].strip()
has_prefix, has_suffix = (bool(nb["cells"][:metadata_idx]), bool(nb["cells"][metadata_idx + 1 :]))
logger.info("initial_python_parameters = {}".format(initial_python_parameters))
return render_template(
"run_report.html",
parameters_as_html=parameters_as_html,
has_prefix=has_prefix,
has_suffix=has_suffix,
report_name=report_name,
all_reports=get_all_possible_templates(),
initialPythonParameters=initial_python_parameters,
)
def sanity_check(template_dir):
logger.info("Starting sanity check")
os.environ["PY_TEMPLATE_DIR"] = template_dir
try:
for template_name in templates._all_templates():
logger.info(
"========================[ Sanity checking {} ]========================"
.format(template_name))
# Test conversion to ipynb - this will throw if stuff goes wrong
generate_ipynb_from_py(filesystem.get_template_dir(),
template_name,
warn_on_local=False)
# Test that each template has parameters as expected
nb = templates.template_name_to_notebook_node(template_name,
warn_on_local=False)
param_idx = templates._get_parameters_cell_idx(nb)
if param_idx is None:
logger.warning(
'Template {} does not have a "parameters"-tagged cell.'.
format(template_name))
# Test that we can generate a preview from the template
preview = templates._get_preview(template_name,
warn_on_local=False)
# Previews in HTML are gigantic since they include all jupyter css and js.
assert len(
preview
) > 1000, "Preview was not properly generated for {}".format(
template_name)
logger.info(
"========================[ {} PASSED ]========================"
.format(template_name))
finally:
filesystem._cleanup_dirs()
def template_name_to_notebook_node(
template_name: str, notebooker_disable_git: bool, py_template_dir: str, warn_on_local: Optional[bool] = True
) -> nbformat.NotebookNode:
path = generate_ipynb_from_py(
get_template_dir(), template_name, notebooker_disable_git, py_template_dir, warn_on_local=warn_on_local
)
nb = nbformat.read(path, as_version=nbformat.v4.nbformat)
return nb
def test_template_has_parameters(template_name):
generate_ipynb_from_py(get_template_dir(),
template_name,
warn_on_local=False)
nb = template_name_to_notebook_node(template_name, warn_on_local=False)
metadata_idx = _get_parameters_cell_idx(nb)
assert metadata_idx is not None, 'Template {} does not have a "parameters"-tagged cell.'.format(
template_name)
def test_execution_of_templates(template_name):
try:
_run_checks(
"job_id_{}".format(str(uuid.uuid4())[:6]),
datetime.datetime.now(),
template_name,
template_name,
get_output_dir(),
get_template_dir(),
{},
generate_pdf_output=False,
)
finally:
_cleanup_dirs()
def regression_test(template_dir):
logger.info("Starting regression test")
os.environ["PY_TEMPLATE_DIR"] = template_dir
try:
attempted_templates, failed_templates = [], set()
for template_name in templates._all_templates():
logger.info(
"============================[ Testing {} ]============================"
.format(template_name))
try:
attempted_templates.append(template_name)
_run_checks(
"job_id_{}".format(str(uuid.uuid4())[:6]),
datetime.datetime.now(),
template_name,
template_name,
filesystem.get_output_dir(),
filesystem.get_template_dir(),
{},
generate_pdf_output=False,
)
logger.info(
"===============================[ SUCCESS ]=============================="
)
except Exception:
failed_templates.add(template_name)
logger.info(
"===============================[ FAILED ]==============================="
)
logger.exception(
"Failed to execute template {}".format(template_name))
for template in attempted_templates:
logger.info("{}: {}".format(
template,
"FAILED" if template in failed_templates else "PASSED"))
if len(failed_templates) > 0:
raise NotebookRunException(
"The following templates failed to execute with no parameters:\n{}"
.format("\n".join(failed_templates)))
finally:
filesystem._cleanup_dirs()
def run_report(report_name,
report_title,
mailto,
overrides,
hide_code=False,
generate_pdf_output=False,
prepare_only=False):
"""
Actually run the report in earnest.
Uses a subprocess to execute the report asynchronously, which is identical to the non-webapp entrypoint.
:param report_name: `str` The report which we are executing
:param report_title: `str` The user-specified title of the report
:param mailto: `Optional[str]` Who the results will be emailed to
:param overrides: `Optional[Dict[str, Any]]` The parameters to be passed into the report
:param generate_pdf_output: `bool` Whether we're generating a PDF. Defaults to False.
:param prepare_only: `bool` Whether to do everything except execute the notebook. Useful for testing.
:return: The unique job_id.
"""
job_id = str(uuid.uuid4())
job_start_time = datetime.datetime.now()
result_serializer = get_serializer()
result_serializer.save_check_stub(
job_id,
report_name,
report_title=report_title,
job_start_time=job_start_time,
status=JobStatus.SUBMITTED,
overrides=overrides,
mailto=mailto,
generate_pdf_output=generate_pdf_output,
hide_code=hide_code,
)
app_config = current_app.config
p = subprocess.Popen(
[
"notebooker-cli",
"--output-base-dir",
get_output_dir(),
"--template-base-dir",
get_template_dir(),
"--py-template-base-dir",
app_config["PY_TEMPLATE_BASE_DIR"],
"--py-template-subdir",
app_config["PY_TEMPLATE_SUBDIR"],
] + (["--notebooker-disable-git"]
if app_config["NOTEBOOKER_DISABLE_GIT"] else []) +
["--serializer-cls", result_serializer.__class__.__name__] +
result_serializer.serializer_args_to_cmdline_args() + [
"execute-notebook",
"--job-id",
job_id,
"--report-name",
report_name,
"--report-title",
report_title,
"--mailto",
mailto,
"--overrides-as-json",
json.dumps(overrides),
"--pdf-output" if generate_pdf_output else "--no-pdf-output",
"--hide-code" if hide_code else "--show-code",
] + (["--prepare-notebook-only"] if prepare_only else []),
stderr=subprocess.PIPE,
)
stderr_thread = threading.Thread(
target=_monitor_stderr,
args=(p, job_id, current_app.config["SERIALIZER_CLS"],
current_app.config["SERIALIZER_CONFIG"]),
)
stderr_thread.daemon = True
stderr_thread.start()
return job_id
def test_conversion_doesnt_fail(template_name):
# Test conversion to ipynb - this will throw if stuff goes wrong
generate_ipynb_from_py(get_template_dir(),
template_name,
warn_on_local=False)
def run_report(report_name, report_title, mailto, overrides, generate_pdf_output=False, prepare_only=False):
"""
Actually run the report in earnest.
Uses a subprocess to execute the report asynchronously, which is identical to the non-webapp entrypoint.
:param report_name: `str` The report which we are executing
:param report_title: `str` The user-specified title of the report
:param mailto: `Optional[str]` Who the results will be emailed to
:param overrides: `Optional[Dict[str, Any]]` The parameters to be passed into the report
:param generate_pdf_output: `bool` Whether we're generating a PDF. Defaults to False.
:param prepare_only: `bool` Whether to do everything except execute the notebook. Useful for testing.
:return: The unique job_id.
"""
job_id = str(uuid.uuid4())
job_start_time = datetime.datetime.now()
result_serializer = get_serializer()
result_serializer.save_check_stub(
job_id,
report_name,
report_title=report_title,
job_start_time=job_start_time,
status=JobStatus.SUBMITTED,
overrides=overrides,
mailto=mailto,
generate_pdf_output=generate_pdf_output,
)
p = subprocess.Popen(
[
sys.executable,
"-m",
execute_notebook.__name__,
"--job-id",
job_id,
"--output-base-dir",
get_output_dir(),
"--template-base-dir",
get_template_dir(),
"--report-name",
report_name,
"--report-title",
report_title,
"--mailto",
mailto,
"--overrides-as-json",
json.dumps(overrides),
"--mongo-db-name",
result_serializer.database_name,
"--mongo-host",
result_serializer.mongo_host,
*(
("--mongo-user", result_serializer.user)
if result_serializer.user is not None else
()
),
*(
("--mongo-password", result_serializer.password)
if result_serializer.password is not None else
()
),
"--result-collection-name",
result_serializer.result_collection_name,
"--pdf-output" if generate_pdf_output else "--no-pdf-output",
"--serializer-cls",
result_serializer.__class__.__name__,
]
+ (["--prepare-notebook-only"] if prepare_only else []),
stderr=subprocess.PIPE,
)
stderr_thread = threading.Thread(target=_monitor_stderr, args=(p, job_id))
stderr_thread.daemon = True
stderr_thread.start()
return job_id
