def run_report_http(report_name): """ The "Run Report" interface is generated by this method. :param report_name: The parameter here should be a "/"-delimited string which mirrors the directory structure of \ the notebook templates. :returns: An HTML template which is the Run Report interface. """ report_name = convert_report_name_url_to_path(report_name) json_params = request.args.get("json_params") initial_python_parameters = json_to_python(json_params) or "" try: path = generate_ipynb_from_py(get_template_dir(), report_name) except FileNotFoundError as e: logger.exception(e) return "", 404 nb = nbformat.read(path, as_version=nbformat.v4.nbformat) metadata_idx = _get_parameters_cell_idx(nb) parameters_as_html = "" has_prefix = has_suffix = False if metadata_idx is not None: metadata = nb["cells"][metadata_idx] parameters_as_html = metadata["source"].strip() has_prefix, has_suffix = (bool(nb["cells"][:metadata_idx]), bool(nb["cells"][metadata_idx + 1 :])) logger.info("initial_python_parameters = {}".format(initial_python_parameters)) return render_template( "run_report.html", parameters_as_html=parameters_as_html, has_prefix=has_prefix, has_suffix=has_suffix, report_name=report_name, all_reports=get_all_possible_templates(), initialPythonParameters=initial_python_parameters, )
def sanity_check(template_dir): logger.info("Starting sanity check") os.environ["PY_TEMPLATE_DIR"] = template_dir try: for template_name in templates._all_templates(): logger.info( "========================[ Sanity checking {} ]========================" .format(template_name)) # Test conversion to ipynb - this will throw if stuff goes wrong generate_ipynb_from_py(filesystem.get_template_dir(), template_name, warn_on_local=False) # Test that each template has parameters as expected nb = templates.template_name_to_notebook_node(template_name, warn_on_local=False) param_idx = templates._get_parameters_cell_idx(nb) if param_idx is None: logger.warning( 'Template {} does not have a "parameters"-tagged cell.'. format(template_name)) # Test that we can generate a preview from the template preview = templates._get_preview(template_name, warn_on_local=False) # Previews in HTML are gigantic since they include all jupyter css and js. assert len( preview ) > 1000, "Preview was not properly generated for {}".format( template_name) logger.info( "========================[ {} PASSED ]========================" .format(template_name)) finally: filesystem._cleanup_dirs()
def template_name_to_notebook_node( template_name: str, notebooker_disable_git: bool, py_template_dir: str, warn_on_local: Optional[bool] = True ) -> nbformat.NotebookNode: path = generate_ipynb_from_py( get_template_dir(), template_name, notebooker_disable_git, py_template_dir, warn_on_local=warn_on_local ) nb = nbformat.read(path, as_version=nbformat.v4.nbformat) return nb
def test_template_has_parameters(template_name): generate_ipynb_from_py(get_template_dir(), template_name, warn_on_local=False) nb = template_name_to_notebook_node(template_name, warn_on_local=False) metadata_idx = _get_parameters_cell_idx(nb) assert metadata_idx is not None, 'Template {} does not have a "parameters"-tagged cell.'.format( template_name)
def test_execution_of_templates(template_name): try: _run_checks( "job_id_{}".format(str(uuid.uuid4())[:6]), datetime.datetime.now(), template_name, template_name, get_output_dir(), get_template_dir(), {}, generate_pdf_output=False, ) finally: _cleanup_dirs()
def regression_test(template_dir): logger.info("Starting regression test") os.environ["PY_TEMPLATE_DIR"] = template_dir try: attempted_templates, failed_templates = [], set() for template_name in templates._all_templates(): logger.info( "============================[ Testing {} ]============================" .format(template_name)) try: attempted_templates.append(template_name) _run_checks( "job_id_{}".format(str(uuid.uuid4())[:6]), datetime.datetime.now(), template_name, template_name, filesystem.get_output_dir(), filesystem.get_template_dir(), {}, generate_pdf_output=False, ) logger.info( "===============================[ SUCCESS ]==============================" ) except Exception: failed_templates.add(template_name) logger.info( "===============================[ FAILED ]===============================" ) logger.exception( "Failed to execute template {}".format(template_name)) for template in attempted_templates: logger.info("{}: {}".format( template, "FAILED" if template in failed_templates else "PASSED")) if len(failed_templates) > 0: raise NotebookRunException( "The following templates failed to execute with no parameters:\n{}" .format("\n".join(failed_templates))) finally: filesystem._cleanup_dirs()
def run_report(report_name, report_title, mailto, overrides, hide_code=False, generate_pdf_output=False, prepare_only=False): """ Actually run the report in earnest. Uses a subprocess to execute the report asynchronously, which is identical to the non-webapp entrypoint. :param report_name: `str` The report which we are executing :param report_title: `str` The user-specified title of the report :param mailto: `Optional[str]` Who the results will be emailed to :param overrides: `Optional[Dict[str, Any]]` The parameters to be passed into the report :param generate_pdf_output: `bool` Whether we're generating a PDF. Defaults to False. :param prepare_only: `bool` Whether to do everything except execute the notebook. Useful for testing. :return: The unique job_id. """ job_id = str(uuid.uuid4()) job_start_time = datetime.datetime.now() result_serializer = get_serializer() result_serializer.save_check_stub( job_id, report_name, report_title=report_title, job_start_time=job_start_time, status=JobStatus.SUBMITTED, overrides=overrides, mailto=mailto, generate_pdf_output=generate_pdf_output, hide_code=hide_code, ) app_config = current_app.config p = subprocess.Popen( [ "notebooker-cli", "--output-base-dir", get_output_dir(), "--template-base-dir", get_template_dir(), "--py-template-base-dir", app_config["PY_TEMPLATE_BASE_DIR"], "--py-template-subdir", app_config["PY_TEMPLATE_SUBDIR"], ] + (["--notebooker-disable-git"] if app_config["NOTEBOOKER_DISABLE_GIT"] else []) + ["--serializer-cls", result_serializer.__class__.__name__] + result_serializer.serializer_args_to_cmdline_args() + [ "execute-notebook", "--job-id", job_id, "--report-name", report_name, "--report-title", report_title, "--mailto", mailto, "--overrides-as-json", json.dumps(overrides), "--pdf-output" if generate_pdf_output else "--no-pdf-output", "--hide-code" if hide_code else "--show-code", ] + (["--prepare-notebook-only"] if prepare_only else []), stderr=subprocess.PIPE, ) stderr_thread = threading.Thread( target=_monitor_stderr, args=(p, job_id, current_app.config["SERIALIZER_CLS"], current_app.config["SERIALIZER_CONFIG"]), ) stderr_thread.daemon = True stderr_thread.start() return job_id
def test_conversion_doesnt_fail(template_name): # Test conversion to ipynb - this will throw if stuff goes wrong generate_ipynb_from_py(get_template_dir(), template_name, warn_on_local=False)
def run_report(report_name, report_title, mailto, overrides, generate_pdf_output=False, prepare_only=False): """ Actually run the report in earnest. Uses a subprocess to execute the report asynchronously, which is identical to the non-webapp entrypoint. :param report_name: `str` The report which we are executing :param report_title: `str` The user-specified title of the report :param mailto: `Optional[str]` Who the results will be emailed to :param overrides: `Optional[Dict[str, Any]]` The parameters to be passed into the report :param generate_pdf_output: `bool` Whether we're generating a PDF. Defaults to False. :param prepare_only: `bool` Whether to do everything except execute the notebook. Useful for testing. :return: The unique job_id. """ job_id = str(uuid.uuid4()) job_start_time = datetime.datetime.now() result_serializer = get_serializer() result_serializer.save_check_stub( job_id, report_name, report_title=report_title, job_start_time=job_start_time, status=JobStatus.SUBMITTED, overrides=overrides, mailto=mailto, generate_pdf_output=generate_pdf_output, ) p = subprocess.Popen( [ sys.executable, "-m", execute_notebook.__name__, "--job-id", job_id, "--output-base-dir", get_output_dir(), "--template-base-dir", get_template_dir(), "--report-name", report_name, "--report-title", report_title, "--mailto", mailto, "--overrides-as-json", json.dumps(overrides), "--mongo-db-name", result_serializer.database_name, "--mongo-host", result_serializer.mongo_host, *( ("--mongo-user", result_serializer.user) if result_serializer.user is not None else () ), *( ("--mongo-password", result_serializer.password) if result_serializer.password is not None else () ), "--result-collection-name", result_serializer.result_collection_name, "--pdf-output" if generate_pdf_output else "--no-pdf-output", "--serializer-cls", result_serializer.__class__.__name__, ] + (["--prepare-notebook-only"] if prepare_only else []), stderr=subprocess.PIPE, ) stderr_thread = threading.Thread(target=_monitor_stderr, args=(p, job_id)) stderr_thread.daemon = True stderr_thread.start() return job_id