def process_notebook(notebook_filename, execute=True):
'''Checks if an IPython notebook runs without error from start to finish. If so, writes the notebook to HTML (with outputs) and overwrites the .ipynb file (without outputs).
'''
with open(notebook_filename) as f:
nb = nbformat.read(f, as_version=4)
clear = ClearOutputPreprocessor()
try:
if execute:
# Check that the notebook runs
ep = ExecutePreprocessor(timeout=600, kernel_name='python3')
ep.preprocess(nb, {'metadata': {'path': ''}})
msg = ''
except CellExecutionError:
msg = f'\n Error executing the notebook "{notebook_filename}".\n'
msg += f' See notebook "{notebook_filename}" for the traceback.'
# Clear notebook outputs and save as .ipynb
cleared = clear.preprocess(nb, {})
with open(notebook_filename, mode='w', encoding='utf-8') as f:
nbformat.write(nb, f)
print(f"Processed {notebook_filename}{msg}")
return
def clear_notebooks(root):
"""Clear the outputs of documentation notebooks."""
preprocessor = ClearOutputPreprocessor()
for dirpath, dirnames, filenames in os.walk(root):
for filename in sorted(filenames):
if os.path.splitext(filename)[1] == '.ipynb':
# read in the notebook
pth = os.path.join(dirpath, filename)
with open(pth, 'r') as fh:
orig_nb = read(fh, 4)
# copy the original notebook
new_nb = deepcopy(orig_nb)
# check outputs of all the cells
new_nb = preprocessor.preprocess(new_nb, {})[0]
# clear metadata
new_nb.metadata = {}
# write the notebook back to disk
with open(pth, 'w') as fh:
write(new_nb, fh, 4)
def NB(notebook, test_notebook):
working_dir = '.'
with open(notebook, 'r') as f:
nb = nbformat.read(f, as_version=4)
with open(test_notebook, 'r') as f:
test_nb = nbformat.read(f, as_version=4)
test_start = len(nb.cells)
nb.cells.extend(test_nb.cells)
# run the notebook
meta = dict(metadata=dict(path=working_dir))
p1 = ClearOutputPreprocessor()
p2 = ExecutePreprocessor(allow_errors=True)
p1.preprocess(nb, meta)
p2.preprocess(nb, meta)
try:
output_notebook = os.path.join(os.environ['REPORT_DIR'],
os.environ['DRIVER_NAME'] + '.ipynb')
with open(output_notebook, 'w') as f:
nbformat.write(nb, f)
except:
pass
tests = []
for cell in nb.cells[test_start:]:
if cell.cell_type == 'code':
tests.append(CellTest(cell))
return tests
def clear_notebook(path,
kwargs={
"SCOPETYPE": "OPENADC",
"PLATFORM": "CWLITEARM",
"VERSION": "HARDWARE"
}):
real_path = Path(path)
body = ""
with open(real_path, "r", encoding="utf-8") as nbfile:
nb = nbformat.read(nbfile, as_version=4)
# special case: if the top block isn't a parameter block, nbparameterise will:
# * error out if invalid python syntax
# * replace code with a parameter block (whatever we passed in with kwargs, including an empty block)
# so if no parameters being changed, don't run nbparameterise
if len(kwargs) > 0:
orig_parameters = extract_parameters(nb)
params = parameter_values(orig_parameters, **kwargs)
new_nb = replace_definitions(nb, params, execute=False)
else:
new_nb = nb
co = ClearOutputPreprocessor()
exporter = NotebookExporter()
node, resources = co.preprocess(new_nb, {'metadata': {'path': './'}})
body, resources = exporter.from_notebook_node(node, resources)
with open(real_path, "w", encoding="utf-8") as nbfile:
nbfile.write(body)
def run_notebook(self, name, event=None):
"""
run a given notebook immediately.
the job parameter is the name of the job script as in ipynb.
Inserts and returns the Metadata document for the job.
"""
notebook = self.get(name)
meta_job = self.metadata(name)
ts = datetime.datetime.now()
# execute
ep_kwargs = {'timeout': None}
ep_kwargs.update(meta_job.kind_meta.get('ep_kwargs', {}))
try:
if not meta_job.kind_meta.get('keep_output', False):
resources = {
} # https://nbconvert.readthedocs.io/en/latest/api/preprocessors.html
cp = ClearOutputPreprocessor()
cp.preprocess(notebook, resources)
ep = ExecutePreprocessor(**ep_kwargs)
ep.preprocess(notebook, {'metadata': {'path': '/'}})
except Exception as e:
status = 'ERROR'
message = str(e)
else:
status = 'OK'
message = ''
# record results
meta_results = self.put(notebook,
'results/{name}_{ts}'.format(**locals()))
meta_results.attributes['source_job'] = name
meta_results.save()
job_results = meta_job.attributes.get('job_results', [])
job_results.append(meta_results.name)
meta_job.attributes['job_results'] = job_results
# record final job status
job_runs = meta_job.attributes.get('job_runs', [])
runstate = {
'status': status,
'ts': ts,
'message': message,
'results': meta_results.name if status == 'OK' else None
}
job_runs.append(runstate)
meta_job.attributes['job_runs'] = job_runs
# set event run state if event was specified
if event:
attrs = meta_job.attributes
triggers = attrs['triggers'] = attrs.get('triggers', [])
scheduled = (trigger for trigger in triggers
if trigger['event-kind'] == 'scheduled')
for trigger in scheduled:
if event == trigger['event']:
trigger['status'] = status
trigger['ts'] = ts
meta_job.save()
return meta_results
def clear_notebooks(root):
"""Clear the outputs of documentation notebooks."""
# cleanup ignored files
run(['git', 'clean', '-fdX', root])
# remove release/autograded/feedback
if os.path.exists(os.path.join(root, "user_guide", "release")):
shutil.rmtree(os.path.join(root, "user_guide", "release"))
if os.path.exists(os.path.join(root, "user_guide", "autograded")):
shutil.rmtree(os.path.join(root, "user_guide", "autograded"))
if os.path.exists(os.path.join(root, "user_guide", "feedback")):
shutil.rmtree(os.path.join(root, "user_guide", "feedback"))
if os.path.exists(
os.path.join(root, "user_guide", "downloaded", "ps1",
"extracted")):
shutil.rmtree(
os.path.join(root, "user_guide", "downloaded", "ps1", "extracted"))
print("Clearing outputs of notebooks in '{}'...".format(
os.path.abspath(root)))
preprocessor = ClearOutputPreprocessor()
for dirpath, dirnames, filenames in os.walk(root):
is_submitted = _check_if_directory_in_path(dirpath, 'submitted')
for filename in sorted(filenames):
if os.path.splitext(filename)[1] == '.ipynb':
# read in the notebook
pth = os.path.join(dirpath, filename)
with open(pth, 'r') as fh:
orig_nb = read(fh, 4)
# copy the original notebook
new_nb = deepcopy(orig_nb)
# check outputs of all the cells
if not is_submitted:
new_nb = preprocessor.preprocess(new_nb, {})[0]
# clear metadata
new_nb.metadata = {
"kernelspec": {
"display_name": "Python",
"language": "python",
"name": "python"
}
}
# write the notebook back to disk
with open(pth, 'w') as fh:
write(new_nb, fh, 4)
if orig_nb != new_nb:
print("Cleared '{}'".format(pth))
def clear_notebook(path):
real_path = Path(path)
body = ""
with open(real_path, "r", encoding="utf-8") as nbfile:
nb = nbformat.read(nbfile, as_version=4)
co = ClearOutputPreprocessor()
exporter = NotebookExporter()
node, resources = co.preprocess(nb, {'metadata': {'path': './'}})
body, resources = exporter.from_notebook_node(node, resources)
with open(real_path, "w", encoding="utf-8") as nbfile:
nbfile.write(body)
def clear(notebook_filename, nbconvert_filename):
with open(notebook_filename, 'r') as fh:
nb = nbformat.read(fh, 4)
# check outputs of all the cells
preprocessor = ClearOutputPreprocessor()
clear_nb = preprocessor.preprocess(nb, {})[0]
# clear metadata
clear_nb.metadata = {}
# write the notebook back to disk
with open(nbconvert_filename, 'w') as fh:
nbformat.write(clear_nb, fh, 4)
def test_func(self):
passing = True
print("\n--------------- Testing {0} ---------------".format(nbname))
print(" {0}".format(nbpath))
if nbname in py2Ignore and sys.version_info[0] == 2:
print(" Skipping {}".format(nbname))
return
ep = ClearOutputPreprocessor()
with open(nbpath) as f:
nb = nbformat.read(f, as_version=4)
ep.preprocess(nb, {})
ex = ExecutePreprocessor(
timeout=600,
kernel_name='python{}'.format(sys.version_info[0]),
allow_errors=True
)
out = ex.preprocess(nb, {})
for cell in out[0]['cells']:
if 'outputs' in cell.keys():
for output in cell['outputs']:
if output['output_type'] == 'error':
passing = False
err_msg = []
for o in output['traceback']:
err_msg += ["{}".format(o)]
err_msg = "\n".join(err_msg)
msg = """
\n <<<<< {} FAILED >>>>> \n
{} in cell [{}] \n-----------\n{}\n-----------\n
----------------- >> begin Traceback << ----------------- \n
{}\n
\n----------------- >> end Traceback << -----------------\n
""".format(
nbname, output['ename'],
cell['execution_count'], cell['source'],
err_msg
)
assert passing, msg
print("\n ..... {0} Passed ..... \n".format(nbname))
def execute(input, to_html, to_self, to_clear):
print('Processing ' + Fore.LIGHTGREEN_EX + input)
with open(input, 'rb') as f:
notebook = nbformat.read(f, as_version=4)
if to_html or to_self:
print(" Executing")
ep = ExecutePreprocessor(timeout=600, kernel_name='python3')
ep.preprocess(notebook, {'metadata': {'path': './'}})
if to_html:
html_output = input + ".html"
print(" Saving to HTML")
exportHTML = HTMLExporter()
(body, resources) = exportHTML.from_notebook_node(notebook)
with open(html_output, 'wt') as f:
f.write(body)
if to_clear:
print(" Clearing")
ep = ClearOutputPreprocessor()
ep.preprocess(notebook, {'metadata': {'path': './'}})
if to_self or to_clear:
stringio = io.StringIO()
nbformat.write(notebook, stringio)
self_output = input
print(" Saving to self")
with open(self_output, 'wb') as f:
s = stringio.getvalue().encode('utf-8')
f.write(s)
def clear_notebook(path):
real_path = Path(path)
body = ""
with open(real_path, "r", encoding="utf-8") as nbfile:
nb = nbformat.read(nbfile, as_version=4)
orig_parameters = extract_parameters(nb)
params = parameter_values(orig_parameters,
SCOPETYPE="OPENADC",
PLATFORM="CWLITEARM")
new_nb = replace_definitions(nb, params, execute=False)
co = ClearOutputPreprocessor()
exporter = NotebookExporter()
node, resources = co.preprocess(new_nb, {'metadata': {'path': './'}})
body, resources = exporter.from_notebook_node(node, resources)
with open(real_path, "w", encoding="utf-8") as nbfile:
nbfile.write(body)
def executenb(nb, cwd=None, km=None, **kwargs):
resources = {}
if cwd is not None:
resources['metadata'] = {'path': cwd} # pragma: no cover
# Clear any stale output, in case of exception
nb, resources = ClearOutputPreprocessor().preprocess(nb, resources)
ep = ExecutePreprocessorWithOutputWidget(**kwargs)
return ep.preprocess(nb, resources, km=km)[0]
def executenb(nb, cwd=None, km=None, **kwargs):
resources = {}
if cwd is not None:
resources['metadata'] = {'path': cwd} # pragma: no cover
# Clear any stale output, in case of exception
nb, resources = ClearOutputPreprocessor().preprocess(nb, resources)
executor = VoilaExecutor(nb, km=km, **kwargs)
return executor.execute(nb, resources, km=km)
async def _jinja_cell_generator(self, nb, kernel_id):
"""Generator that will execute a single notebook cell at a time"""
nb, resources = ClearOutputPreprocessor().preprocess(
nb, {'metadata': {
'path': self.cwd
}})
for cell_idx, input_cell in enumerate(nb.cells):
try:
task = asyncio.ensure_future(
self.executor.execute_cell(input_cell,
None,
cell_idx,
store_history=False))
while True:
done, pending = await asyncio.wait(
{task},
timeout=self.voila_configuration.
http_keep_alive_timeout)
if pending:
# If not done within the timeout, we send a heartbeat
# this is fundamentally to avoid browser/proxy read-timeouts, but
# can be used in a template to give feedback to a user
self.write("<script>voila_heartbeat()</script>\n")
self.flush()
continue
output_cell = await task
break
except TimeoutError:
output_cell = input_cell
break
except Exception as e:
self.log.exception('Error at server while executing cell: %r',
input_cell)
output_cell = nbformat.v4.new_code_cell()
if self.executor.should_strip_error():
output_cell.outputs = [{
"output_type":
"stream",
"name":
"stderr",
"text":
"An exception occurred at the server (not the notebook). {}"
.format(self.executor.cell_error_instruction),
}]
else:
output_cell.outputs = [{
'output_type':
'error',
'ename':
type(e).__name__,
'evalue':
str(e),
'traceback':
traceback.format_exception(*sys.exc_info()),
}]
finally:
yield output_cell
def clearNbOutput():
cwd = os.getcwd()
# walk the test directory and find all notebooks
for dirname, dirnames, filenames in os.walk(nbdir):
if dirname != ".ipynb_checkpoints":
for filename in filenames:
if (filename.endswith(".ipynb")
and not filename.endswith("-checkpoint.ipynb")):
with open(os.path.join(dirname, filename)) as f:
print("Clearing output from {}".format(filename))
nb = nbformat.read(f, as_version=4)
ep = ClearOutputPreprocessor()
ep.preprocess(nb, {
"kernel_name":
"python{}".format(sys.version_info[0])
})
print(" ... done\n")
return True
def generate_exercises():
p = Path(*EXERCISES_DIR)
exporter = HTMLExporter()
exporter.register_preprocessor(ClearOutputPreprocessor(), enabled=True)
for exercise in p.iterdir():
if exercise.suffix == '.ipynb':
html, _ = exporter.from_file(exercise.open())
with open(exercise.with_suffix('.html').name, 'w') as f:
f.write(html)
def unexecute(self):
"""
Unexecutes the notebook i.e. removes all output cells
"""
_logger.info(f"Cleaning up notebook {self.nb} in {self.nb_dir}")
if not self.executed_nb_path.exists():
_logger.warning(
f"{self.executed_nb_path} not found, nothing to clean")
return
with open(self.executed_nb_path) as f:
nb = nbformat.read(f, as_version=IPYTHON_VERSION)
if nb['metadata'].get('docs_executed', None):
nb['metadata'].pop('docs_executed')
clear_executor = ClearOutputPreprocessor()
clear_executor.preprocess(nb, {})
with open(self.executed_nb_path, 'w') as f:
nbformat.write(nb, f)
def clear_notebook(path):
"""Clears output from a notebook
Parameters
----------
path: str (optional)
path to a Jupyter Notebook
Returns
-------
None
"""
with open(_get_path(path), encoding="utf-8") as f:
nb = nbformat.read(f, as_version=4)
pp = ClearOutputPreprocessor()
pp.preprocess(nb, {})
with open(path, "w", encoding="utf-8") as f:
nbformat.write(nb, f)
def _jinja_cell_generator(self, nb, kernel_id):
"""Generator that will execute a single notebook cell at a time"""
km = self.kernel_manager.get_kernel(kernel_id)
nb, resources = ClearOutputPreprocessor().preprocess(nb, {'metadata': {'path': self.cwd}})
ep = VoilaExecutePreprocessor(config=self.traitlet_config)
with ep.setup_preprocessor(nb, resources, km=km):
for cell_idx, cell in enumerate(nb.cells):
res = ep.preprocess_cell(cell, resources, cell_idx, store_history=False)
yield res[0]
async def _jinja_cell_generator(self, nb, kernel_id):
"""Generator that will execute a single notebook cell at a time"""
nb, _ = ClearOutputPreprocessor().preprocess(
nb, {'metadata': {
'path': self.cwd
}})
for cell_idx, input_cell in enumerate(nb.cells):
try:
output_cell = await self.executor.execute_cell(
input_cell, None, cell_idx, store_history=False)
except TimeoutError:
output_cell = input_cell
break
except CellExecutionError:
self.log.exception('Error at server while executing cell: %r',
input_cell)
if self.executor.should_strip_error():
strip_code_cell_warnings(input_cell)
self.executor.strip_code_cell_errors(input_cell)
output_cell = input_cell
break
except Exception as e:
self.log.exception('Error at server while executing cell: %r',
input_cell)
output_cell = nbformat.v4.new_code_cell()
if self.executor.should_strip_error():
output_cell.outputs = [{
'output_type':
'stream',
'name':
'stderr',
'text':
'An exception occurred at the server (not the notebook). {}'
.format(self.executor.cell_error_instruction),
}]
else:
output_cell.outputs = [{
'output_type':
'error',
'ename':
type(e).__name__,
'evalue':
str(e),
'traceback':
traceback.format_exception(*sys.exc_info()),
}]
finally:
yield output_cell
await self._cleanup_resources()
def nb_exec(fname, name):
ver = make_version(name)
dirpath = os.path.join(data_dir, name, ver)
execpath = os.path.join(dirpath, 'src')
src_nb = os.path.join(execpath, 'src.ipynb')
ensure_dir(dirpath)
ensure_dir(execpath)
logger.info(f'Copying {fname} to {src_nb}')
shutil.copy2(fname, src_nb)
with open(src_nb) as f:
nb = nbformat.read(f, as_version=4)
co = ClearOutputPreprocessor()
nb, _ = co.preprocess(nb, {})
ep = ExecutePreprocessor(timeout=-1, kernel_name='python3')
try:
ep.preprocess(nb, {'metadata': {'path': execpath}})
except CellExecutionError as e:
print(e)
with open(os.path.join(dirpath, 'error.log'), 'w') as f:
f.write(str(e))
print('Stack trace saved')
with open(os.path.join(dirpath, 'output.ipynb'), 'w',
encoding='utf-8') as f:
nbformat.write(nb, f)
exp = HTMLExporter()
body, res = exp.from_notebook_node(nb)
with open(os.path.join(dirpath, 'output.html'), 'w',
encoding='utf-8') as f:
f.write(body)
def clear_notebooks(root):
"""Clear the outputs of documentation notebooks."""
# cleanup ignored files
run(['git', 'clean', '-fdX', root])
print("Clearing outputs of notebooks in '{}'...".format(
os.path.abspath(root)))
preprocessor = ClearOutputPreprocessor()
for dirpath, dirnames, filenames in os.walk(root):
is_submitted = _check_if_directory_in_path(dirpath, 'submitted')
for filename in sorted(filenames):
if os.path.splitext(filename)[1] == '.ipynb':
# read in the notebook
pth = os.path.join(dirpath, filename)
with open(pth, 'r') as fh:
orig_nb = read(fh, 4)
# copy the original notebook
new_nb = deepcopy(orig_nb)
# check outputs of all the cells
if not is_submitted:
new_nb = preprocessor.preprocess(new_nb, {})[0]
# clear metadata
new_nb.metadata = {}
# write the notebook back to disk
with open(pth, 'w') as fh:
write(new_nb, fh, 4)
if orig_nb != new_nb:
print("Cleared '{}'".format(pth))
def unexecute(self, remove_gh=False):
"""
Unexecutes the notebook i.e. removes all output cells. If remove_gh=True looks to see if
notebook metadata contains an executed tag. If it doesn't it means the notebook either
errored or was not run (for case when only specific notebooks chosen to build examples) and
removes the notebooks so old ones can be used
"""
_logger.info(f"Cleaning up notebook {self.nb} in {self.nb_dir}")
if not self.executed_nb_path.exists():
_logger.warning(
f"{self.executed_nb_path} not found, nothing to clean")
return
with open(self.executed_nb_path, encoding='utf-8') as f:
nb = nbformat.read(f, as_version=IPYTHON_VERSION)
if not remove_gh:
if nb['metadata'].get('docs_executed', None):
nb['metadata'].pop('docs_executed')
clear_executor = ClearOutputPreprocessor()
clear_executor.preprocess(nb, {})
with open(self.executed_nb_path, 'w', encoding='utf-8') as f:
nbformat.write(nb, f)
elif remove_gh:
executed_flag = nb['metadata'].get('docs_executed', None)
if executed_flag != 'executed':
_logger.warning(f"Notebook {self.nb} not executed or errored, "
f"version already on website will be used")
os.remove(self.executed_nb_path)
os.remove(self.output_path.joinpath(self.nb_name + '.html'))
else:
_logger.info(f"Notebook {self.nb} executed, "
f"new version will be uploaded to website")
clear_executor = ClearOutputPreprocessor()
clear_executor.preprocess(nb, {})
with open(self.executed_nb_path, 'w', encoding='utf-8') as f:
nbformat.write(nb, f)
async def _jinja_cell_generator(self, nb, kernel_id):
"""Generator that will execute a single notebook cell at a time"""
nb, resources = ClearOutputPreprocessor().preprocess(
nb, {'metadata': {
'path': self.cwd
}})
stop_execution = False
for cell_idx, cell in enumerate(nb.cells):
if stop_execution:
break
try:
res = await self.executor.execute_cell(cell,
None,
cell_idx,
store_history=False)
except TimeoutError:
res = cell
stop_execution = True
yield res
def clean():
_delete_generated_files()
config = Config()
config.NotebookExporter.preprocessors = [ClearOutputPreprocessor()]
exporter = NotebookExporter(config=config)
for notebook in get_files("ipynb"):
with open(notebook, encoding="utf8") as nb_file:
nb = nbformat.read(nb_file, as_version=4)
if not nb.cells[-1]["source"]:
nb.cells.pop()
format_code_cells(nb)
for cell_id, cell in enumerate(nb.cells):
cell["id"] = f"{notebook.stem}-{cell_id}"
ipynb, _ = exporter.from_notebook_node(nb)
with open(f"{notebook.stem}.ipynb", "w", encoding="utf8") as writable:
writable.write(ipynb)
def open_nb_and_strip_output(fname):
cop = ClearOutputPreprocessor()
with open(fname) as f:
nb = nbformat.read(f, as_version=4)
cop.preprocess(nb, dict())
return nb
def convert_notebook_to_assets(notebook_file_name, base_name, output_prefix):
# define and create output folder
output_folder = output_prefix + '/' + base_name
os.makedirs(output_folder, exist_ok=True)
# open file
print('Converting Notebook: ' + notebook_file_name + ' ...')
nb_file = open(notebook_file_name, 'r').read()
nb = nbformat.reads(nb_file, as_version=4)
# 1. clear output
print(" - clearing output")
ep = ClearOutputPreprocessor()
ep.preprocess(nb, {})
# 2. generate fresh charts by running the notebook
print(" - executing")
ep = ExecutePreprocessor(timeout=600,
kernel_name='python3',
allow_errors=False)
try:
ep.preprocess(nb, {'metadata': {'path': output_folder}})
except Exception as e:
print('ERROR: Execution of the notebook ' + notebook_file_name +
' stopped, likely for missing some py libs.')
print(
' Please check the output/exception and add those to the requirements.'
)
print(e)
exit(1)
# 3. export HTML
print(" - generating html")
cleaner_config = Config({
"HTMLExporter": {
"exclude_input": True,
"exclude_input_prompt": True,
"exclude_output_prompt": True,
"preprocessors":
['nbconvert.preprocessors.ExtractOutputPreprocessor']
},
})
local_templates = DictLoader({
'our-html.tpl': stand_alone_tpl,
'react-glue.tpl': react_glue_tpl,
})
exporter = HTMLExporter(config=cleaner_config,
extra_loaders=[local_templates])
exporter.template_file = 'our-html.tpl'
(html_body, html_resources) = exporter.from_notebook_node(nb)
notebook_convert_time = current_utc_time()
# save html output file, with local reference to the pictures
local_html = []
output_html_file_name = output_folder + '/' + "index.html"
print(" - saving html: " + output_html_file_name)
with open(output_html_file_name, 'wt') as the_file:
the_file.write(html_body)
local_html.append({
'notebook': base_name,
'notebook_html': output_html_file_name,
'convert_time': notebook_convert_time,
})
# save js file for react inclusion (local ref to the pictures)
# exporter.template_file = 'react-glue.tpl'
# (react_body, react_resources) = exporter.from_notebook_node(nb)
# output_react_file_name = output_folder + '/' + "index.js"
# print(" - saving react js: " + output_react_file_name)
# with open(output_react_file_name, 'wt') as the_file:
# the_file.write(react_body)
# save all the figures
local_figures = []
figures = html_resources['outputs']
figures_count = len(figures)
figure_index = 1
for figure_file in figures:
output_figure_file_name = output_folder + '/' + figure_file
print(" - saving png " + str(figure_index) + " of " +
str(figures_count) + ": " + output_figure_file_name)
if not figure_file.endswith('.png'):
print("WARNING: figure is not a PNG file")
continue
with open(output_figure_file_name, 'wb') as the_file:
the_file.write(figures[figure_file])
local_figures.append({
'figure': figure_file,
'file': output_figure_file_name,
'notebook': base_name,
'notebook_html': output_html_file_name,
'convert_time': notebook_convert_time,
})
# create an empty 'custom.css'
custom_css_file_name = output_folder + '/' + 'custom.css'
with open(custom_css_file_name, 'wt') as the_file:
the_file.write("")
# return a recap of all assets
return local_html, local_figures
def test_func(self):
cwd = os.getcwd()
passing = True
print("\n---------------------"
" Testing {0}.ipynb "
"---------------------".format(nbname))
if (nbname in self.ignore) or (nbname in self.py2_ignore
and sys.version_info[0] == 2):
print(" Skipping {}".format(nbname))
return
run_path = os.path.sep.join(nbpath.split(os.path.sep)[:-1])
os.chdir(run_path)
clear_output = ClearOutputPreprocessor()
with open(nbpath) as nbfile:
notebook = nbformat.read(nbfile, as_version=4)
clear_output.preprocess(notebook, {})
execute = ExecutePreprocessor(
timeout=timeout,
kernel_name="python{}".format(sys.version_info[0]),
allow_errors=True,
)
out = execute.preprocess(notebook, {})
os.chdir(cwd)
for cell in out[0]["cells"]:
if "outputs" in cell.keys():
for output in cell["outputs"]:
if output["output_type"] == "error":
passing = False
err_msg = []
for traceback in output["traceback"]:
err_msg += ["{}".format(traceback)]
err_msg = "\n".join(err_msg)
msg = """
\n ... {} FAILED \n
{} in cell [{}] \n-----------\n{}\n-----------\n
""".format(
nbname,
output["ename"],
cell["execution_count"],
cell["source"],
)
traceback = """
----------------- >> begin Traceback << ----------------- \n
{}\n
\n----------------- >> end Traceback << -----------------\n
""".format(err_msg)
print(u"{}".format(msg + traceback))
assert passing, msg
print(" ... {0} Passed \n".format(nbname))
def test_func(self):
cwd = os.getcwd()
passing = True
print("\n---------------------"
" Testing {0}.ipynb "
"---------------------".format(nbname))
if ((nbname in self.ignore)
or (nbname in self.py2_ignore and sys.version_info[0] == 2)):
print(" Skipping {}".format(nbname))
return
run_path = os.path.sep.join(nbpath.split(os.path.sep)[:-1])
os.chdir(run_path)
ep = ClearOutputPreprocessor(
resources={'metadata': {
'path': run_path
}})
with open(nbpath) as f:
nb = nbformat.read(f, as_version=4)
ep.preprocess(nb, {})
ex = ExecutePreprocessor(
timeout=timeout,
kernel_name='python{}'.format(sys.version_info[0]),
allow_errors=True,
resources={'metadata': {
'path': run_path
}})
out = ex.preprocess(nb, {})
os.chdir(cwd)
for cell in out[0]['cells']:
if 'outputs' in cell.keys():
for output in cell['outputs']:
if output['output_type'] == 'error':
passing = False
err_msg = []
for o in output['traceback']:
err_msg += ["{}".format(o)]
err_msg = "\n".join(err_msg)
msg = """
\n ... {} FAILED \n
{} in cell [{}] \n-----------\n{}\n-----------\n
""".format(
nbname,
output['ename'],
cell['execution_count'],
cell['source'],
)
traceback = """
----------------- >> begin Traceback << ----------------- \n
{}\n
\n----------------- >> end Traceback << -----------------\n
""".format(err_msg)
print(u"{}".format(msg + traceback))
assert passing, msg
print(" ... {0} Passed \n".format(nbname))
def run_notebook(self, name, event=None, timeout=None):
""" run a given notebook immediately.
Args:
name (str): the name of the jobfile
event (str): an event name
timeout (int): timeout in seconds
Returns:
Metadata of results
See Also:
* nbconvert https://nbconvert.readthedocs.io/en/latest/execute_api.html
"""
notebook = self.get(name)
meta_job = self.metadata(name)
ts = datetime.datetime.now()
# execute kwargs
# -- see ExecuteProcessor class
# -- see https://nbconvert.readthedocs.io/en/latest/execute_api.html
ep_kwargs = {
# avoid timeouts to stop kernel
'timeout': timeout,
# avoid kernel at exit functions
# -- this stops ipykernel AttributeError 'send_multipart'
'shutdown_kernel': 'immediate',
# set kernel name, blank is default
# -- e.g. python3, ir
# -- see https://stackoverflow.com/a/47053020/890242
'kernel_name': '',
}
# other interesting options
ep_kwargs.update(meta_job.kind_meta.get('ep_kwargs', {}))
try:
resources = {
'metadata': {
'path': self.defaults.OMEGA_TMP,
}
}
if not meta_job.kind_meta.get('keep_output', False):
# https://nbconvert.readthedocs.io/en/latest/api/preprocessors.html
cp = ClearOutputPreprocessor()
cp.preprocess(notebook, resources)
ep = ExecutePreprocessor(**ep_kwargs)
ep.preprocess(notebook, resources)
except Exception as e:
status = 'ERROR'
message = str(e)
else:
status = 'OK'
message = ''
finally:
del ep
# record results
meta_results = self.put(notebook,
'results/{name}_{ts}'.format(**locals()))
meta_results.attributes['source_job'] = name
meta_results.save()
job_results = meta_job.attributes.get('job_results', [])
job_results.append(meta_results.name)
meta_job.attributes['job_results'] = job_results
# record final job status
job_runs = meta_job.attributes.get('job_runs', [])
runstate = {
'status': status,
'ts': ts,
'message': message,
'results': meta_results.name if status == 'OK' else None
}
job_runs.append(runstate)
meta_job.attributes['job_runs'] = job_runs
# set event run state if event was specified
if event:
attrs = meta_job.attributes
triggers = attrs['triggers'] = attrs.get('triggers', [])
scheduled = (trigger for trigger in triggers
if trigger['event-kind'] == 'scheduled')
for trigger in scheduled:
if event == trigger['event']:
trigger['status'] = status
trigger['ts'] = ts
meta_job.save()
return meta_results
