def task_results_latest(report_name):
"""
Renders the full results page for a report_name. This searches the database
for the last completed report for the given report_name.
:param report_name: The name of the report
:return: The HTML rendering of the results page for the latest successful execution of the given report_name.
"""
report_name = convert_report_name_url_to_path(report_name)
params = _params_from_request_args(request.args)
result = get_latest_job_results(report_name, params, get_serializer())
job_id = result.job_id
return _render_results(job_id, report_name, result)
def latest_task_results_as_of(report_name, as_of):
"""This URL will ignore all paramterisation of the report and get the latest of any run for a given report name, \
Up to a given as_of date. In this method, we either:
- present the HTML results, if the job has finished
- present the error, if the job has failed
- present the user with some info detailing the progress of the job, if it is still running.
:param report_name: The name of the template which we want to get the latest version of, up to and including as_of.
:param as_of: The maximum date of reports which we want to see.
:return: The HTML render of the absolute-latest run of a report, regardless of parametrization.
"""
return _process_result_or_abort(get_latest_job_results(report_name, None, get_serializer(), as_of=as_of))
def task_results_html(job_id, report_name):
"""
Returns the HTML render of the .ipynb output of notebook execution. In the webapp this is rendered within an \
iframe. In this method, we either:
- present the HTML results, if the job has finished
- present the error, if the job has failed
- present the user with some info detailing the progress of the job, if it is still running.
:param job_id: The UUID of the report which we are accessing.
:param report_name: The name of the report
:return: The HTML rendering of the .ipynb for the given report_name & job_id.
"""
return _process_result_or_abort(_get_job_results(job_id, report_name, get_serializer()))
def _rerun_report(job_id, prepare_only=False):
result = get_serializer().get_check_result(job_id)
if not result:
abort(404)
prefix = "Rerun of "
title = result.report_title if result.report_title.startswith(prefix) else (prefix + result.report_title)
new_job_id = run_report(
result.report_name,
title,
result.mailto,
result.overrides,
generate_pdf_output=result.generate_pdf_output,
prepare_only=prepare_only,
)
return new_job_id
def task_latest_status(report_name):
"""
Searches for the latest status of the given report_name/override args combination, and returns the status with
a redirect URL or stdout.
:param report_name: The name of the report which we are searching for the latest status of.
:return: A JSON which contains "status" and either stdout in "run_output" or a URL to results in "results_url".
"""
params = _params_from_request_args(request.args)
result = get_latest_job_results(report_name, params, get_serializer())
job_id = result.job_id
if job_id:
return jsonify(_get_job_status(job_id, report_name))
return jsonify({"status": "Job not found for given overrides"})
def task_result_resources_html(job_id, resource, report_name):
"""
Returns resources, such as stylesheets and images, which are requested by the HTML rendering of the .ipynb.
:param report_name: The name of the report.
:param resource: The relative path to the resource, as saved on disk during execution and saved into storage.
:param job_id: The UUID of the report.
:return: A download of the data as requested. 404s if not found.
"""
result = _get_job_results(job_id, report_name, get_serializer())
if isinstance(result, NotebookResultComplete):
html_resources = result.raw_html_resources
resource_path = os.path.join(get_resources_dir(job_id), resource)
if resource_path in html_resources.get("outputs", {}):
return html_resources["outputs"][resource_path]
abort(404)
def download_pdf_result(job_id, report_name):
"""
Allows a user to download the PDF output from storage.
:param report_name: The name of the report.
:param job_id: The UUID of the report.
:return: A download of the PDF as requested. 404s if not found.
"""
result = _get_job_results(job_id, report_name, get_serializer())
if isinstance(result, NotebookResultComplete):
return Response(
result.pdf,
mimetype="application/pdf",
headers={"Content-Disposition": "attachment;filename={}".format(_pdf_filename(job_id))},
)
else:
abort(404)
def latest_successful_task_results_html(report_name):
"""
Returns the HTML render of the .ipynb output of notebook execution. In the webapp this is rendered within an \
iframe. Searches the database for the last successful execution of this report_name. \
Notebook parameters can be specified as request args, \
e.g. ?ticker=AAPL. In this method, we either:
- present the HTML results, if the job has finished
- present the error, if the job has failed
- present the user with some info detailing the progress of the job, if it is still running.
:param report_name: The name of the report.
:return: The HTML rendering of the .ipynb for the latest successful execution of the given report_name.
"""
params = _params_from_request_args(request.args)
result = get_latest_successful_job_results(report_name, params, get_serializer())
return _process_result_or_abort(result)
def test_run_report_and_rerun(bson_library, flask_app,
setup_and_cleanup_notebooker_filesystem,
setup_workspace):
with flask_app.app_context():
serialiser = get_serializer()
overrides = {"n_points": 5}
report_name = "fake/report"
report_title = "my report title"
mailto = "*****@*****.**"
job_id = run_report(report_name,
report_title,
mailto,
overrides,
generate_pdf_output=False,
prepare_only=True)
_check_report_output(
job_id,
serialiser,
overrides=overrides,
report_name=report_name,
report_title=report_title,
mailto=mailto,
generate_pdf_output=False,
)
new_job_id = _rerun_report(job_id, prepare_only=True)
_check_report_output(
new_job_id,
serialiser,
overrides=overrides,
report_name=report_name,
report_title="Rerun of " + report_title,
mailto=mailto,
generate_pdf_output=False,
)
assert new_job_id == serialiser.get_latest_job_id_for_name_and_params(
report_name, overrides)
assert not {job_id, new_job_id} - set(
serialiser.get_all_job_ids_for_name_and_params(
report_name, overrides))
assert new_job_id == serialiser.get_latest_successful_job_id_for_name_and_params(
report_name, overrides)
assert job_id != serialiser.get_latest_successful_job_id_for_name_and_params(
report_name, overrides)
def latest_parameterised_task_results_as_of(report_name, as_of):
"""
Returns the HTML render of the .ipynb output of notebook execution. In the webapp this is rendered within an \
iframe. Searches the database for the last result as of the given date, regardless of status. \
Notebook parameters can be specified as request args, \
e.g. ?ticker=AAPL. In this method, we either:
- present the HTML results, if the job has finished
- present the error, if the job has failed
- present the user with some info detailing the progress of the job, if it is still running.
:param report_name: The name of the report.
:param as_of: The maximum date up to which we are searching for any executions.
:return: The HTML rendering of the .ipynb for the latest execution of the given report_name.
"""
params = _params_from_request_args(request.args)
result = get_latest_job_results(report_name, params, get_serializer(), as_of=as_of)
return _process_result_or_abort(result)
def index():
"""
The index page which returns a blank table which is async populated by /core/all_available_results.
Async populating the table from a different URL means that we can lock down the "core" blueprint to
only users with correct privileges.
"""
username = request.headers.get("X-Auth-Username")
all_reports = get_all_possible_templates()
with current_app.app_context():
result = render_template(
"index.html",
all_jobs_url=url_for("core_bp.all_available_results"),
all_reports=all_reports,
n_results_available=get_serializer().n_all_results(),
donevalue=JobStatus.
DONE, # needed so we can check if a result is available
username=username,
)
return result
def test_run_report(bson_library, environ):
flask_app = create_app()
with flask_app.app_context() as c:
serialiser = get_serializer()
overrides = {"n_points": 5}
report_name = "fake/report"
report_title = "my report title"
mailto = "*****@*****.**"
job_id = run_report(report_name, report_title, mailto, overrides, generate_pdf_output=False, prepare_only=True)
_check_report_output(
job_id,
serialiser,
overrides=overrides,
report_name=report_name,
report_title=report_title,
mailto=mailto,
)
assert job_id == serialiser.get_latest_job_id_for_name_and_params(report_name, overrides)
assert job_id == serialiser.get_latest_job_id_for_name_and_params(report_name, None)
assert job_id == serialiser.get_latest_successful_job_id_for_name_and_params(report_name, overrides)
assert job_id == serialiser.get_latest_successful_job_id_for_name_and_params(report_name, None)
def test_run_report(bson_library, mongo_host, workspace, test_db_name,
test_lib_name):
_setup_workspace(workspace)
for k, v in _environ(mongo_host, workspace, test_db_name,
test_lib_name).items():
os.environ[k] = v
try:
flask_app = create_app()
with flask_app.app_context() as c:
serialiser = get_serializer()
overrides = {"n_points": 5}
report_name = "fake/report"
report_title = "my report title"
mailto = "*****@*****.**"
job_id = run_report(report_name,
report_title,
mailto,
overrides,
generate_pdf_output=False,
prepare_only=True)
_check_report_output(
job_id,
serialiser,
overrides=overrides,
report_name=report_name,
report_title=report_title,
mailto=mailto,
)
assert job_id == serialiser.get_latest_job_id_for_name_and_params(
report_name, overrides)
assert job_id == serialiser.get_latest_job_id_for_name_and_params(
report_name, None)
assert job_id == serialiser.get_latest_successful_job_id_for_name_and_params(
report_name, overrides)
assert job_id == serialiser.get_latest_successful_job_id_for_name_and_params(
report_name, None)
finally:
for k, __ in _environ(mongo_host, workspace, test_db_name,
test_lib_name).items():
del os.environ[k]
def run_report(report_name,
report_title,
mailto,
overrides,
hide_code=False,
generate_pdf_output=False,
prepare_only=False):
"""
Actually run the report in earnest.
Uses a subprocess to execute the report asynchronously, which is identical to the non-webapp entrypoint.
:param report_name: `str` The report which we are executing
:param report_title: `str` The user-specified title of the report
:param mailto: `Optional[str]` Who the results will be emailed to
:param overrides: `Optional[Dict[str, Any]]` The parameters to be passed into the report
:param generate_pdf_output: `bool` Whether we're generating a PDF. Defaults to False.
:param prepare_only: `bool` Whether to do everything except execute the notebook. Useful for testing.
:return: The unique job_id.
"""
job_id = str(uuid.uuid4())
job_start_time = datetime.datetime.now()
result_serializer = get_serializer()
result_serializer.save_check_stub(
job_id,
report_name,
report_title=report_title,
job_start_time=job_start_time,
status=JobStatus.SUBMITTED,
overrides=overrides,
mailto=mailto,
generate_pdf_output=generate_pdf_output,
hide_code=hide_code,
)
app_config = current_app.config
p = subprocess.Popen(
[
"notebooker-cli",
"--output-base-dir",
get_output_dir(),
"--template-base-dir",
get_template_dir(),
"--py-template-base-dir",
app_config["PY_TEMPLATE_BASE_DIR"],
"--py-template-subdir",
app_config["PY_TEMPLATE_SUBDIR"],
] + (["--notebooker-disable-git"]
if app_config["NOTEBOOKER_DISABLE_GIT"] else []) +
["--serializer-cls", result_serializer.__class__.__name__] +
result_serializer.serializer_args_to_cmdline_args() + [
"execute-notebook",
"--job-id",
job_id,
"--report-name",
report_name,
"--report-title",
report_title,
"--mailto",
mailto,
"--overrides-as-json",
json.dumps(overrides),
"--pdf-output" if generate_pdf_output else "--no-pdf-output",
"--hide-code" if hide_code else "--show-code",
] + (["--prepare-notebook-only"] if prepare_only else []),
stderr=subprocess.PIPE,
)
stderr_thread = threading.Thread(
target=_monitor_stderr,
args=(p, job_id, current_app.config["SERIALIZER_CLS"],
current_app.config["SERIALIZER_CONFIG"]),
)
stderr_thread.daemon = True
stderr_thread.start()
return job_id
def run_report(report_name, report_title, mailto, overrides, generate_pdf_output=False, prepare_only=False):
"""
Actually run the report in earnest.
Uses a subprocess to execute the report asynchronously, which is identical to the non-webapp entrypoint.
:param report_name: `str` The report which we are executing
:param report_title: `str` The user-specified title of the report
:param mailto: `Optional[str]` Who the results will be emailed to
:param overrides: `Optional[Dict[str, Any]]` The parameters to be passed into the report
:param generate_pdf_output: `bool` Whether we're generating a PDF. Defaults to False.
:param prepare_only: `bool` Whether to do everything except execute the notebook. Useful for testing.
:return: The unique job_id.
"""
job_id = str(uuid.uuid4())
job_start_time = datetime.datetime.now()
result_serializer = get_serializer()
result_serializer.save_check_stub(
job_id,
report_name,
report_title=report_title,
job_start_time=job_start_time,
status=JobStatus.SUBMITTED,
overrides=overrides,
mailto=mailto,
generate_pdf_output=generate_pdf_output,
)
p = subprocess.Popen(
[
sys.executable,
"-m",
execute_notebook.__name__,
"--job-id",
job_id,
"--output-base-dir",
get_output_dir(),
"--template-base-dir",
get_template_dir(),
"--report-name",
report_name,
"--report-title",
report_title,
"--mailto",
mailto,
"--overrides-as-json",
json.dumps(overrides),
"--mongo-db-name",
result_serializer.database_name,
"--mongo-host",
result_serializer.mongo_host,
*(
("--mongo-user", result_serializer.user)
if result_serializer.user is not None else
()
),
*(
("--mongo-password", result_serializer.password)
if result_serializer.password is not None else
()
),
"--result-collection-name",
result_serializer.result_collection_name,
"--pdf-output" if generate_pdf_output else "--no-pdf-output",
"--serializer-cls",
result_serializer.__class__.__name__,
]
+ (["--prepare-notebook-only"] if prepare_only else []),
stderr=subprocess.PIPE,
)
stderr_thread = threading.Thread(target=_monitor_stderr, args=(p, job_id))
stderr_thread.daemon = True
stderr_thread.start()
return job_id
