示例#1
0
def test_generate_ipynb_from_py():
    python_dir = tempfile.mkdtemp()
    try:
        set_cache("latest_sha", "fake_sha_early")

        os.mkdir(python_dir + "/extra_path")
        with open(os.path.join(python_dir, "extra_path", "test_report.py"), "w") as f:
            f.write("#hello world\n")
        report_path = os.sep.join(["extra_path", "test_report"])
        with mock.patch("notebooker.utils.conversion._git_pull_templates") as pull:
            conversion.python_template_dir = lambda *a, **kw: python_dir
            pull.return_value = "fake_sha_early"
            conversion.generate_ipynb_from_py(python_dir, report_path)
            pull.return_value = "fake_sha_later"
            conversion.generate_ipynb_from_py(python_dir, report_path)
            conversion.generate_ipynb_from_py(python_dir, report_path)

        assert get_cache("latest_sha") == "fake_sha_later"
        expected_filename = _output_ipynb_name(report_path)
        expected_ipynb_path = os.path.join(python_dir, "fake_sha_early", expected_filename)
        assert os.path.exists(expected_ipynb_path), f".ipynb at {expected_ipynb_path} was not generated as expected!"
        expected_ipynb_path = os.path.join(python_dir, "fake_sha_later", expected_filename)
        assert os.path.exists(expected_ipynb_path), ".ipynb was not generated as expected!"

        with mock.patch("notebooker.utils.conversion.uuid.uuid4") as uuid4:
            with mock.patch("notebooker.utils.conversion.pkg_resources.resource_filename") as resource_filename:
                conversion.python_template_dir = lambda *a, **kw: None
                uuid4.return_value = "uuid"
                resource_filename.return_value = python_dir + "/extra_path/test_report.py"
                conversion.generate_ipynb_from_py(python_dir, "extra_path/test_report")

        expected_ipynb_path = os.path.join(python_dir, "uuid", expected_filename)
        assert os.path.exists(expected_ipynb_path), f".ipynb at {expected_ipynb_path} was not generated as expected!"

        with mock.patch("notebooker.utils.conversion.uuid.uuid4") as uuid4:
            conversion.python_template_dir = lambda *a, **kw: python_dir
            conversion.NOTEBOOKER_DISABLE_GIT = True
            uuid4.return_value = "uuid_nogit"
            conversion.generate_ipynb_from_py(python_dir, "extra_path/test_report")

        expected_ipynb_path = os.path.join(python_dir, "uuid_nogit", expected_filename)
        assert os.path.exists(expected_ipynb_path), ".ipynb was not generated as expected!"

    finally:
        _cleanup_dirs()
        shutil.rmtree(python_dir)
示例#2
0
def test_generate_ipynb_from_py(setup_and_cleanup_notebooker_filesystem,
                                webapp_config, flask_app):
    python_dir = webapp_config.PY_TEMPLATE_BASE_DIR
    with flask_app.app_context():
        set_cache("latest_sha", "fake_sha_early")

        os.mkdir(python_dir + "/extra_path")
        with open(os.path.join(python_dir, "extra_path", "test_report.py"),
                  "w") as f:
            f.write("#hello world\n")
        report_path = os.sep.join(["extra_path", "test_report"])
        with mock.patch("notebooker.utils.conversion.git.repo.Repo") as repo:
            repo().commit().hexsha = "fake_sha_early"
            conversion.generate_ipynb_from_py(python_dir, report_path, False,
                                              python_dir)
            repo().commit().hexsha = "fake_sha_later"
            conversion.generate_ipynb_from_py(python_dir, report_path, False,
                                              python_dir)
            conversion.generate_ipynb_from_py(python_dir, report_path, False,
                                              python_dir)

        expected_filename = _output_ipynb_name(report_path)
        expected_ipynb_path = os.path.join(python_dir, "fake_sha_early",
                                           expected_filename)
        assert os.path.exists(
            expected_ipynb_path
        ), f".ipynb at {expected_ipynb_path} was not generated as expected!"
        expected_ipynb_path = os.path.join(python_dir, "fake_sha_later",
                                           expected_filename)
        assert os.path.exists(
            expected_ipynb_path), ".ipynb was not generated as expected!"

        with mock.patch("notebooker.utils.conversion.uuid.uuid4") as uuid4:
            with mock.patch(
                    "notebooker.utils.conversion.pkg_resources.resource_filename"
            ) as resource_filename:
                conversion.python_template_dir = lambda *a, **kw: None
                uuid4.return_value = "uuid"
                resource_filename.return_value = python_dir + "/extra_path/test_report.py"
                conversion.generate_ipynb_from_py(python_dir,
                                                  "extra_path/test_report",
                                                  False,
                                                  py_template_dir="")

        expected_ipynb_path = os.path.join(python_dir, "uuid",
                                           expected_filename)
        assert os.path.exists(
            expected_ipynb_path
        ), f".ipynb at {expected_ipynb_path} was not generated as expected!"

        with mock.patch("notebooker.utils.conversion.uuid.uuid4") as uuid4:
            conversion.python_template_dir = lambda *a, **kw: python_dir
            conversion.NOTEBOOKER_DISABLE_GIT = True
            uuid4.return_value = "uuid_nogit"
            conversion.generate_ipynb_from_py(python_dir,
                                              "extra_path/test_report",
                                              True,
                                              py_template_dir=python_dir)

        expected_ipynb_path = os.path.join(python_dir, "uuid_nogit",
                                           expected_filename)
        assert os.path.exists(
            expected_ipynb_path), ".ipynb was not generated as expected!"
示例#3
0
def _run_checks(
    job_id: str,
    job_start_time: datetime.datetime,
    template_name: str,
    report_title: str,
    output_base_dir: str,
    template_base_dir: str,
    overrides: Dict[AnyStr, Any],
    generate_pdf_output: Optional[bool] = True,
    mailto: Optional[str] = "",
    prepare_only: Optional[bool] = False,
) -> NotebookResultComplete:
    """
    This is the actual method which executes a notebook, whether running in the webapp or via the entrypoint.
    If this crashes, an exception is raised (and should be caught by run_checks().)

    Parameters
    ----------
    job_id : `str`
        The unique ID of this report.
    job_start_time : `datetime.datetime`
        The UTC start time of this report.
    template_name : `str`
        The name of the template which we are running. NB this should be a path relative to the python_template_dir()
    report_title : `str`
        The user-specified optional title of the report. Defaults to the template name.
    output_base_dir : `str`
        Internal use. The temp directory where output is being saved, local to the executor.
    template_base_dir : `str`
        Internal use. The temp directory where the .py->.ipynb converted templates reside, local to the executor.
    overrides : Dict[AnyStr, Any]
        The dictionary of overrides which parametrize our Notebook Template.
    generate_pdf_output : `Optional[bool]`
        Whether to generate PDF output or not. NB this requires xelatex to be installed on the executor.
    mailto : `Optional[str]`
        Comma-separated email addresses to send on completion (or error).
    prepare_only : `Optional[bool]`
        Internal usage. Whether we want to do everything apart from executing the notebook.


    Returns
    -------
    NotebookResultComplete

    Raises
    ------
    Exception()

    """

    output_dir = _output_dir(output_base_dir, template_name, job_id)
    output_ipynb = _output_ipynb_name(template_name)

    if not os.path.isdir(output_dir):
        logger.info("Making dir @ {}".format(output_dir))
        os.makedirs(output_dir)

    ipynb_raw_path = generate_ipynb_from_py(template_base_dir, template_name)
    ipynb_executed_path = os.path.join(output_dir, output_ipynb)

    logger.info("Executing notebook at {} using parameters {} --> {}".format(
        ipynb_raw_path, overrides, output_ipynb))
    pm.execute_notebook(ipynb_raw_path,
                        ipynb_executed_path,
                        parameters=overrides,
                        log_output=True,
                        prepare_only=prepare_only)
    with open(ipynb_executed_path, "r") as f:
        raw_executed_ipynb = f.read()

    logger.info(
        "Saving output notebook as HTML from {}".format(ipynb_executed_path))
    html, resources = ipython_to_html(ipynb_executed_path, job_id)
    pdf = ipython_to_pdf(raw_executed_ipynb,
                         report_title) if generate_pdf_output else ""

    notebook_result = NotebookResultComplete(
        job_id=job_id,
        job_start_time=job_start_time,
        job_finish_time=datetime.datetime.now(),
        raw_html_resources=resources,
        raw_ipynb_json=raw_executed_ipynb,
        raw_html=html,
        mailto=mailto,
        pdf=pdf,
        generate_pdf_output=generate_pdf_output,
        report_name=template_name,
        report_title=report_title,
        overrides=overrides,
    )
    return notebook_result