def _get_python_template_path(report_path: str, warn_on_local: bool) -> str: if python_template_dir(): return _python_template(report_path) else: if warn_on_local: logger.warning( "Loading from notebooker default templates. This is only expected if you are running locally." ) return pkg_resources.resource_filename( __name__, "../notebook_templates_example/{}.py".format(report_path))
def get_all_possible_templates(warn_on_local=True): if python_template_dir(): all_checks = get_directory_structure() else: if warn_on_local: logger.warning("Fetching all possible checks from local repo. New updates will not be retrieved from git.") # Only import here because we don't actually want to import these if the app is working properly. from .. import notebook_templates_example all_checks = get_directory_structure(os.path.abspath(notebook_templates_example.__path__[0])) return all_checks
def _get_output_path_hex() -> str: if python_template_dir() and not NOTEBOOKER_DISABLE_GIT: logger.info("Pulling latest notebook templates from git.") try: latest_sha = _git_pull_templates() if get_cache("latest_sha") != latest_sha: logger.info("Change detected in notebook template master!") set_cache("latest_sha", latest_sha) logger.info("Git pull done.") except Exception as e: logger.exception(e) return get_cache("latest_sha") or "OLD" else: return str(uuid.uuid4())
def get_directory_structure(starting_point: Optional[str] = None) -> Dict[str, Union[Dict, None]]: """ Creates a nested dictionary that represents the folder structure of rootdir """ starting_point = starting_point or python_template_dir() all_dirs = {} rootdir = starting_point.rstrip(os.sep) start = rootdir.rfind(os.sep) + 1 for path, dirs, files in os.walk(rootdir): if not _valid_dirname(path): continue folders = path[start:].split(os.sep) subdir = {os.sep.join(folders[1:] + [f.replace(".py", "")]): None for f in files if _valid_filename(f)} parent = reduce(dict.get, folders[:-1], all_dirs) parent[folders[-1]] = subdir return all_dirs[rootdir[start:]]
def _run_checks( job_id: str, job_start_time: datetime.datetime, template_name: str, report_title: str, output_base_dir: str, template_base_dir: str, overrides: Dict[AnyStr, Any], generate_pdf_output: Optional[bool] = True, hide_code: Optional[bool] = False, mailto: Optional[str] = "", error_mailto: Optional[str] = "", email_subject: Optional[str] = "", prepare_only: Optional[bool] = False, notebooker_disable_git: bool = False, py_template_base_dir: str = "", py_template_subdir: str = "", ) -> NotebookResultComplete: """ This is the actual method which executes a notebook, whether running in the webapp or via the entrypoint. If this crashes, an exception is raised (and should be caught by run_checks().) Parameters ---------- job_id : `str` The unique ID of this report. job_start_time : `datetime.datetime` The UTC start time of this report. template_name : `str` The name of the template which we are running. NB this should be a path relative to the python_template_dir() report_title : `str` The user-specified optional title of the report. Defaults to the template name. output_base_dir : `str` Internal use. The temp directory where output is being saved, local to the executor. template_base_dir : `str` Internal use. The temp directory where the .py->.ipynb converted templates reside, local to the executor. overrides : Dict[AnyStr, Any] The dictionary of overrides which parametrize our Notebook Template. generate_pdf_output : `Optional[bool]` Whether to generate PDF output or not. NB this requires xelatex to be installed on the executor. mailto : `Optional[str]` Comma-separated email addresses to send on completion (or error). prepare_only : `Optional[bool]` Internal usage. Whether we want to do everything apart from executing the notebook. Returns ------- NotebookResultComplete Raises ------ Exception() """ output_dir = _output_dir(output_base_dir, template_name, job_id) output_ipynb = _output_ipynb_name(template_name) if not os.path.isdir(output_dir): logger.info("Making dir @ {}".format(output_dir)) os.makedirs(output_dir) py_template_dir = python_template_dir(py_template_base_dir, py_template_subdir) ipynb_raw_path = generate_ipynb_from_py(template_base_dir, template_name, notebooker_disable_git, py_template_dir) ipynb_executed_path = os.path.join(output_dir, output_ipynb) logger.info("Executing notebook at {} using parameters {} --> {}".format( ipynb_raw_path, overrides, output_ipynb)) pm.execute_notebook(ipynb_raw_path, ipynb_executed_path, parameters=overrides, log_output=True, prepare_only=prepare_only) with open(ipynb_executed_path, "r") as f: raw_executed_ipynb = f.read() logger.info( "Saving output notebook as HTML from {}".format(ipynb_executed_path)) html, resources = ipython_to_html(ipynb_executed_path, job_id) email_html, resources = ipython_to_html(ipynb_executed_path, job_id, hide_code=hide_code) pdf = ipython_to_pdf(raw_executed_ipynb, report_title, hide_code=hide_code) if generate_pdf_output else "" notebook_result = NotebookResultComplete( job_id=job_id, job_start_time=job_start_time, job_finish_time=datetime.datetime.now(), raw_html_resources=resources, raw_ipynb_json=raw_executed_ipynb, raw_html=html, email_html=email_html, mailto=mailto, email_subject=email_subject, pdf=pdf, generate_pdf_output=generate_pdf_output, report_name=template_name, report_title=report_title, overrides=overrides, ) return notebook_result
def _get_python_template_dir() -> str: return python_template_dir(current_app.config["PY_TEMPLATE_BASE_DIR"], current_app.config["PY_TEMPLATE_SUBDIR"])
def _python_template(report_path: AnyStr) -> AnyStr: file_name = "{}.py".format(report_path) return os.path.join(python_template_dir(), file_name)