def _get_python_template_path(report_path: str, warn_on_local: bool) -> str:
if python_template_dir():
return _python_template(report_path)
else:
if warn_on_local:
logger.warning(
"Loading from notebooker default templates. This is only expected if you are running locally."
)
return pkg_resources.resource_filename(
__name__,
"../notebook_templates_example/{}.py".format(report_path))
def get_all_possible_templates(warn_on_local=True):
if python_template_dir():
all_checks = get_directory_structure()
else:
if warn_on_local:
logger.warning("Fetching all possible checks from local repo. New updates will not be retrieved from git.")
# Only import here because we don't actually want to import these if the app is working properly.
from .. import notebook_templates_example
all_checks = get_directory_structure(os.path.abspath(notebook_templates_example.__path__[0]))
return all_checks
def _get_output_path_hex() -> str:
if python_template_dir() and not NOTEBOOKER_DISABLE_GIT:
logger.info("Pulling latest notebook templates from git.")
try:
latest_sha = _git_pull_templates()
if get_cache("latest_sha") != latest_sha:
logger.info("Change detected in notebook template master!")
set_cache("latest_sha", latest_sha)
logger.info("Git pull done.")
except Exception as e:
logger.exception(e)
return get_cache("latest_sha") or "OLD"
else:
return str(uuid.uuid4())
def get_directory_structure(starting_point: Optional[str] = None) -> Dict[str, Union[Dict, None]]:
"""
Creates a nested dictionary that represents the folder structure of rootdir
"""
starting_point = starting_point or python_template_dir()
all_dirs = {}
rootdir = starting_point.rstrip(os.sep)
start = rootdir.rfind(os.sep) + 1
for path, dirs, files in os.walk(rootdir):
if not _valid_dirname(path):
continue
folders = path[start:].split(os.sep)
subdir = {os.sep.join(folders[1:] + [f.replace(".py", "")]): None for f in files if _valid_filename(f)}
parent = reduce(dict.get, folders[:-1], all_dirs)
parent[folders[-1]] = subdir
return all_dirs[rootdir[start:]]
def _run_checks(
job_id: str,
job_start_time: datetime.datetime,
template_name: str,
report_title: str,
output_base_dir: str,
template_base_dir: str,
overrides: Dict[AnyStr, Any],
generate_pdf_output: Optional[bool] = True,
hide_code: Optional[bool] = False,
mailto: Optional[str] = "",
error_mailto: Optional[str] = "",
email_subject: Optional[str] = "",
prepare_only: Optional[bool] = False,
notebooker_disable_git: bool = False,
py_template_base_dir: str = "",
py_template_subdir: str = "",
) -> NotebookResultComplete:
"""
This is the actual method which executes a notebook, whether running in the webapp or via the entrypoint.
If this crashes, an exception is raised (and should be caught by run_checks().)
Parameters
----------
job_id : `str`
The unique ID of this report.
job_start_time : `datetime.datetime`
The UTC start time of this report.
template_name : `str`
The name of the template which we are running. NB this should be a path relative to the python_template_dir()
report_title : `str`
The user-specified optional title of the report. Defaults to the template name.
output_base_dir : `str`
Internal use. The temp directory where output is being saved, local to the executor.
template_base_dir : `str`
Internal use. The temp directory where the .py->.ipynb converted templates reside, local to the executor.
overrides : Dict[AnyStr, Any]
The dictionary of overrides which parametrize our Notebook Template.
generate_pdf_output : `Optional[bool]`
Whether to generate PDF output or not. NB this requires xelatex to be installed on the executor.
mailto : `Optional[str]`
Comma-separated email addresses to send on completion (or error).
prepare_only : `Optional[bool]`
Internal usage. Whether we want to do everything apart from executing the notebook.
Returns
-------
NotebookResultComplete
Raises
------
Exception()
"""
output_dir = _output_dir(output_base_dir, template_name, job_id)
output_ipynb = _output_ipynb_name(template_name)
if not os.path.isdir(output_dir):
logger.info("Making dir @ {}".format(output_dir))
os.makedirs(output_dir)
py_template_dir = python_template_dir(py_template_base_dir,
py_template_subdir)
ipynb_raw_path = generate_ipynb_from_py(template_base_dir, template_name,
notebooker_disable_git,
py_template_dir)
ipynb_executed_path = os.path.join(output_dir, output_ipynb)
logger.info("Executing notebook at {} using parameters {} --> {}".format(
ipynb_raw_path, overrides, output_ipynb))
pm.execute_notebook(ipynb_raw_path,
ipynb_executed_path,
parameters=overrides,
log_output=True,
prepare_only=prepare_only)
with open(ipynb_executed_path, "r") as f:
raw_executed_ipynb = f.read()
logger.info(
"Saving output notebook as HTML from {}".format(ipynb_executed_path))
html, resources = ipython_to_html(ipynb_executed_path, job_id)
email_html, resources = ipython_to_html(ipynb_executed_path,
job_id,
hide_code=hide_code)
pdf = ipython_to_pdf(raw_executed_ipynb, report_title,
hide_code=hide_code) if generate_pdf_output else ""
notebook_result = NotebookResultComplete(
job_id=job_id,
job_start_time=job_start_time,
job_finish_time=datetime.datetime.now(),
raw_html_resources=resources,
raw_ipynb_json=raw_executed_ipynb,
raw_html=html,
email_html=email_html,
mailto=mailto,
email_subject=email_subject,
pdf=pdf,
generate_pdf_output=generate_pdf_output,
report_name=template_name,
report_title=report_title,
overrides=overrides,
)
return notebook_result
def _get_python_template_dir() -> str:
return python_template_dir(current_app.config["PY_TEMPLATE_BASE_DIR"],
current_app.config["PY_TEMPLATE_SUBDIR"])
def _python_template(report_path: AnyStr) -> AnyStr:
file_name = "{}.py".format(report_path)
return os.path.join(python_template_dir(), file_name)
