def start_client(self):
# setup the execution current working directory
cwd_context: ContextManager[str]
if self.nb_config.execution_in_temp:
cwd_context = TemporaryDirectory()
else:
if self.path is None:
raise ValueError(
"Input source must exist as file, if execution_in_temp=False"
)
cwd_context = nullcontext(str(self.path.parent))
# execute in the context of the current working directory
with cwd_context as cwd:
cwd = os.path.abspath(cwd)
self.logger.info("Executing notebook using " + (
"temporary" if self.nb_config.execution_in_temp else "local") +
" CWD")
result = single_nb_execution(
self.notebook,
cwd=cwd,
allow_errors=self.nb_config.execution_allow_errors,
timeout=self.nb_config.execution_timeout,
meta_override=True, # TODO still support this?
)
if result.err is not None:
if self.nb_config.execution_raise_on_error:
raise ExecutionError(str(self.path)) from result.err
msg = f"Executing notebook failed: {result.err.__class__.__name__}"
if self.nb_config.execution_show_tb:
msg += f"\n{result.exc_string}"
self.logger.warning(msg, subtype="exec")
else:
self.logger.info(f"Executed notebook in {result.time:.2f} seconds")
self.exec_metadata = {
"mtime": datetime.now().timestamp(),
"runtime": result.time,
"method": self.nb_config.execution_mode,
"succeeded": False if result.err else True,
"error":
f"{result.err.__class__.__name__}" if result.err else None,
"traceback": result.exc_string if result.err else None,
}
def execute_single(self, nb_bundle, uri, cwd, timeout, allow_errors, asset_files):
result = single_nb_execution(
nb_bundle.nb,
cwd=cwd,
timeout=timeout,
allow_errors=allow_errors,
)
if result.err:
self.logger.error("Execution Failed: {}".format(uri))
return _create_bundle(
nb_bundle,
cwd,
asset_files,
result.time,
result.exc_string,
)
self.logger.info("Execution Succeeded: {}".format(uri))
return _create_bundle(nb_bundle, cwd, asset_files, result.time, None)
def generate_notebook_outputs(
env: BuildEnvironment,
ntbk: nbf.NotebookNode,
file_path: Optional[str] = None,
show_traceback: bool = False,
) -> nbf.NotebookNode:
"""
Add outputs to a NotebookNode by pulling from cache.
Function to get the database instance. Get the cached output of the notebook
and merge it with the original notebook. If there is no cached output,
checks if there was error during execution, then saves the traceback to a log file.
"""
# check if the file is of a format that may be associated with outputs
if not is_valid_exec_file(env, env.docname):
return ntbk
# If we have a jupyter_cache, see if there's a cache for this notebook
file_path = file_path or env.doc2path(env.docname)
execution_method = env.config["jupyter_execute_notebooks"] # type: str
path_to_cache = env.nb_path_to_cache if "cache" in execution_method else None
if not path_to_cache and "off" in execution_method:
return ntbk
if not path_to_cache:
if execution_method == "auto" and nb_has_all_output(file_path):
LOGGER.info(
"Did not execute %s. "
"Set jupyter_execute_notebooks to `force` to execute",
env.docname,
)
else:
if env.config["execution_in_temp"]:
with tempfile.TemporaryDirectory() as tmpdirname:
LOGGER.info("Executing: %s in temporary directory",
env.docname)
result = single_nb_execution(
ntbk,
cwd=tmpdirname,
timeout=env.config["execution_timeout"],
allow_errors=env.config["execution_allow_errors"],
)
else:
cwd = Path(file_path).parent
LOGGER.info("Executing: %s in: %s", env.docname, cwd)
result = single_nb_execution(
ntbk,
cwd=cwd,
timeout=env.config["execution_timeout"],
allow_errors=env.config["execution_allow_errors"],
)
report_path = None
if result.err:
report_path, message = _report_exec_fail(
env,
Path(file_path).name,
result.exc_string,
show_traceback,
"Execution Failed with traceback saved in {}",
)
LOGGER.error(message)
ntbk = result.nb
env.nb_execution_data_changed = True
env.nb_execution_data[env.docname] = {
"mtime": datetime.now().timestamp(),
"runtime": result.time,
"method": execution_method,
"succeeded": False if result.err else True,
}
if report_path:
env.nb_execution_data[env.docname]["error_log"] = report_path
return ntbk
cache_base = get_cache(path_to_cache)
# Use relpath here in case Sphinx is building from a non-parent folder
r_file_path = Path(os.path.relpath(file_path, Path().resolve()))
# default execution data
runtime = None
succeeded = False
report_path = None
try:
pk, ntbk = cache_base.merge_match_into_notebook(ntbk)
except KeyError:
message = (
f"Couldn't find cache key for notebook file {str(r_file_path)}. "
"Outputs will not be inserted.")
try:
stage_record = cache_base.get_staged_record(file_path)
except KeyError:
stage_record = None
if stage_record and stage_record.traceback:
report_path, suffix = _report_exec_fail(
env,
r_file_path.name,
stage_record.traceback,
show_traceback,
"\n Last execution failed with traceback saved in {}",
)
message += suffix
LOGGER.error(message)
else:
LOGGER.verbose("Merged cached outputs into %s", str(r_file_path))
succeeded = True
try:
runtime = cache_base.get_cache_record(pk).data.get(
"execution_seconds", None)
except Exception:
pass
env.nb_execution_data_changed = True
env.nb_execution_data[env.docname] = {
"mtime": datetime.now().timestamp(),
"runtime": runtime,
"method": execution_method,
"succeeded": succeeded,
}
if report_path:
env.nb_execution_data[env.docname]["error_log"] = report_path
return ntbk
def start_client(self):
# setup the cache
cache = get_cache(self.nb_config.execution_cache_path
or ".jupyter_cache")
# TODO config on what notebook/cell metadata to hash/merge
# attempt to match the notebook to one in the cache
cache_record = None
with suppress(KeyError):
cache_record = cache.match_cache_notebook(self.notebook)
# use the cached notebook if it exists
if cache_record is not None:
self.logger.info(f"Using cached notebook: ID={cache_record.pk}")
_, self._notebook = cache.merge_match_into_notebook(self.notebook)
self.exec_metadata = {
"mtime": cache_record.created.timestamp(),
"runtime": cache_record.data.get("execution_seconds", None),
"method": self.nb_config.execution_mode,
"succeeded": True,
"error": None,
"traceback": None,
}
return
if self.path is None:
raise ValueError(
"Input source must exist as file, if execution_mode is 'cache'"
)
# attempt to execute the notebook
read_fmt = self._kwargs.get("read_fmt", None)
if read_fmt is not None:
stage_record = cache.add_nb_to_project(str(self.path),
read_data=read_fmt)
else:
stage_record = cache.add_nb_to_project(str(self.path))
# TODO do in try/except, in case of db write errors
NbProjectRecord.remove_tracebacks([stage_record.pk], cache.db)
cwd_context: ContextManager[str] = (
TemporaryDirectory() # type: ignore
if self.nb_config.execution_in_temp else nullcontext(
str(self.path.parent)))
with cwd_context as cwd:
cwd = os.path.abspath(cwd)
self.logger.info("Executing notebook using " + (
"temporary" if self.nb_config.execution_in_temp else "local") +
" CWD")
result = single_nb_execution(
self.notebook,
cwd=cwd,
allow_errors=self.nb_config.execution_allow_errors,
timeout=self.nb_config.execution_timeout,
meta_override=True, # TODO still support this?
)
# handle success / failure cases
# TODO do in try/except to be careful (in case of database write errors?
if result.err is not None:
if self.nb_config.execution_raise_on_error:
raise ExecutionError(str(self.path)) from result.err
msg = f"Executing notebook failed: {result.err.__class__.__name__}"
if self.nb_config.execution_show_tb:
msg += f"\n{result.exc_string}"
self.logger.warning(msg, subtype="exec")
NbProjectRecord.set_traceback(stage_record.uri, result.exc_string,
cache.db)
else:
self.logger.info(f"Executed notebook in {result.time:.2f} seconds")
cache_record = cache.cache_notebook_bundle(
CacheBundleIn(
self.notebook,
stage_record.uri,
data={"execution_seconds": result.time},
),
check_validity=False,
overwrite=True,
)
self.logger.info(f"Cached executed notebook: ID={cache_record.pk}")
self.exec_metadata = {
"mtime": datetime.now().timestamp(),
"runtime": result.time,
"method": self.nb_config.execution_mode,
"succeeded": False if result.err else True,
"error":
f"{result.err.__class__.__name__}" if result.err else None,
"traceback": result.exc_string if result.err else None,
}