def NB(notebook, test_notebook):
working_dir = '.'
with open(notebook, 'r') as f:
nb = nbformat.read(f, as_version=4)
with open(test_notebook, 'r') as f:
test_nb = nbformat.read(f, as_version=4)
test_start = len(nb.cells)
nb.cells.extend(test_nb.cells)
# run the notebook
meta = dict(metadata=dict(path=working_dir))
p1 = ClearOutputPreprocessor()
p2 = ExecutePreprocessor(allow_errors=True)
p1.preprocess(nb, meta)
p2.preprocess(nb, meta)
try:
output_notebook = os.path.join(os.environ['REPORT_DIR'],
os.environ['DRIVER_NAME'] + '.ipynb')
with open(output_notebook, 'w') as f:
nbformat.write(nb, f)
except:
pass
tests = []
for cell in nb.cells[test_start:]:
if cell.cell_type == 'code':
tests.append(CellTest(cell))
return tests
def run_notebook(self, name, event=None):
"""
run a given notebook immediately.
the job parameter is the name of the job script as in ipynb.
Inserts and returns the Metadata document for the job.
"""
notebook = self.get(name)
meta_job = self.metadata(name)
ts = datetime.datetime.now()
# execute
ep_kwargs = {'timeout': None}
ep_kwargs.update(meta_job.kind_meta.get('ep_kwargs', {}))
try:
if not meta_job.kind_meta.get('keep_output', False):
resources = {
} # https://nbconvert.readthedocs.io/en/latest/api/preprocessors.html
cp = ClearOutputPreprocessor()
cp.preprocess(notebook, resources)
ep = ExecutePreprocessor(**ep_kwargs)
ep.preprocess(notebook, {'metadata': {'path': '/'}})
except Exception as e:
status = 'ERROR'
message = str(e)
else:
status = 'OK'
message = ''
# record results
meta_results = self.put(notebook,
'results/{name}_{ts}'.format(**locals()))
meta_results.attributes['source_job'] = name
meta_results.save()
job_results = meta_job.attributes.get('job_results', [])
job_results.append(meta_results.name)
meta_job.attributes['job_results'] = job_results
# record final job status
job_runs = meta_job.attributes.get('job_runs', [])
runstate = {
'status': status,
'ts': ts,
'message': message,
'results': meta_results.name if status == 'OK' else None
}
job_runs.append(runstate)
meta_job.attributes['job_runs'] = job_runs
# set event run state if event was specified
if event:
attrs = meta_job.attributes
triggers = attrs['triggers'] = attrs.get('triggers', [])
scheduled = (trigger for trigger in triggers
if trigger['event-kind'] == 'scheduled')
for trigger in scheduled:
if event == trigger['event']:
trigger['status'] = status
trigger['ts'] = ts
meta_job.save()
return meta_results
def test_func(self):
passing = True
print("\n--------------- Testing {0} ---------------".format(nbname))
print(" {0}".format(nbpath))
if nbname in py2Ignore and sys.version_info[0] == 2:
print(" Skipping {}".format(nbname))
return
ep = ClearOutputPreprocessor()
with open(nbpath) as f:
nb = nbformat.read(f, as_version=4)
ep.preprocess(nb, {})
ex = ExecutePreprocessor(
timeout=600,
kernel_name='python{}'.format(sys.version_info[0]),
allow_errors=True
)
out = ex.preprocess(nb, {})
for cell in out[0]['cells']:
if 'outputs' in cell.keys():
for output in cell['outputs']:
if output['output_type'] == 'error': #in output.keys():
passing = False
err_msg = []
for o in output['traceback']:
err_msg += ["{}".format(o)]
err_msg = "\n".join(err_msg)
msg = """
\n <<<<< {} FAILED >>>>> \n
{} in cell [{}] \n-----------\n{}\n-----------\n
----------------- >> begin Traceback << ----------------- \n
{}\n
\n----------------- >> end Traceback << -----------------\n
""".format(
nbname, output['ename'],
cell['execution_count'], cell['source'],
err_msg
)
assert passing, msg
print("\n ..... {0} Passed ..... \n".format(nbname))
def test_func(self):
passing = True
print("\n--------------- Testing {0} ---------------".format(nbname))
print(" {0}".format(nbpath))
if nbname in py2Ignore and sys.version_info[0] == 2:
print(" Skipping {}".format(nbname))
return
ep = ClearOutputPreprocessor()
with open(nbpath) as f:
nb = nbformat.read(f, as_version=4)
ep.preprocess(nb, {})
ex = ExecutePreprocessor(
timeout=600,
kernel_name='python{}'.format(sys.version_info[0]),
allow_errors=True
)
out = ex.preprocess(nb, {})
for cell in out[0]['cells']:
if 'outputs' in cell.keys():
for output in cell['outputs']:
if output['output_type'] == 'error':
passing = False
err_msg = []
for o in output['traceback']:
err_msg += ["{}".format(o)]
err_msg = "\n".join(err_msg)
msg = """
\n <<<<< {} FAILED >>>>> \n
{} in cell [{}] \n-----------\n{}\n-----------\n
----------------- >> begin Traceback << ----------------- \n
{}\n
\n----------------- >> end Traceback << -----------------\n
""".format(
nbname, output['ename'],
cell['execution_count'], cell['source'],
err_msg
)
assert passing, msg
print("\n ..... {0} Passed ..... \n".format(nbname))
def process_notebook(notebook_filename, execute=True):
'''Checks if an IPython notebook runs without error from start to finish. If so, writes the notebook to HTML (with outputs) and overwrites the .ipynb file (without outputs).
'''
with open(notebook_filename) as f:
nb = nbformat.read(f, as_version=4)
clear = ClearOutputPreprocessor()
try:
if execute:
# Check that the notebook runs
ep = ExecutePreprocessor(timeout=600, kernel_name='python3')
ep.preprocess(nb, {'metadata': {'path': ''}})
msg = ''
except CellExecutionError:
msg = f'\n Error executing the notebook "{notebook_filename}".\n'
msg += f' See notebook "{notebook_filename}" for the traceback.'
# Clear notebook outputs and save as .ipynb
cleared = clear.preprocess(nb, {})
with open(notebook_filename, mode='w', encoding='utf-8') as f:
nbformat.write(nb, f)
print(f"Processed {notebook_filename}{msg}")
return
def clear_notebooks(root):
"""Clear the outputs of documentation notebooks."""
preprocessor = ClearOutputPreprocessor()
for dirpath, dirnames, filenames in os.walk(root):
for filename in sorted(filenames):
if os.path.splitext(filename)[1] == '.ipynb':
# read in the notebook
pth = os.path.join(dirpath, filename)
with open(pth, 'r') as fh:
orig_nb = read(fh, 4)
# copy the original notebook
new_nb = deepcopy(orig_nb)
# check outputs of all the cells
new_nb = preprocessor.preprocess(new_nb, {})[0]
# clear metadata
new_nb.metadata = {}
# write the notebook back to disk
with open(pth, 'w') as fh:
write(new_nb, fh, 4)
def clear_notebook(path,
kwargs={
"SCOPETYPE": "OPENADC",
"PLATFORM": "CWLITEARM",
"VERSION": "HARDWARE"
}):
real_path = Path(path)
body = ""
with open(real_path, "r", encoding="utf-8") as nbfile:
nb = nbformat.read(nbfile, as_version=4)
# special case: if the top block isn't a parameter block, nbparameterise will:
# * error out if invalid python syntax
# * replace code with a parameter block (whatever we passed in with kwargs, including an empty block)
# so if no parameters being changed, don't run nbparameterise
if len(kwargs) > 0:
orig_parameters = extract_parameters(nb)
params = parameter_values(orig_parameters, **kwargs)
new_nb = replace_definitions(nb, params, execute=False)
else:
new_nb = nb
co = ClearOutputPreprocessor()
exporter = NotebookExporter()
node, resources = co.preprocess(new_nb, {'metadata': {'path': './'}})
body, resources = exporter.from_notebook_node(node, resources)
with open(real_path, "w", encoding="utf-8") as nbfile:
nbfile.write(body)
def clear_notebooks(root):
"""Clear the outputs of documentation notebooks."""
# cleanup ignored files
run(['git', 'clean', '-fdX', root])
print("Clearing outputs of notebooks in '{}'...".format(os.path.abspath(root)))
preprocessor = ClearOutputPreprocessor()
for dirpath, dirnames, filenames in os.walk(root):
is_submitted = _check_if_directory_in_path(dirpath, 'submitted')
for filename in sorted(filenames):
if os.path.splitext(filename)[1] == '.ipynb':
# read in the notebook
pth = os.path.join(dirpath, filename)
with open(pth, 'r') as fh:
orig_nb = read(fh, 4)
# copy the original notebook
new_nb = deepcopy(orig_nb)
# check outputs of all the cells
if not is_submitted:
new_nb = preprocessor.preprocess(new_nb, {})[0]
# clear metadata
new_nb.metadata = {}
# write the notebook back to disk
with open(pth, 'w') as fh:
write(new_nb, fh, 4)
if orig_nb != new_nb:
print("Cleared '{}'".format(pth))
def clearNbOutput():
cwd = os.getcwd()
# walk the test directory and find all notebooks
for dirname, dirnames, filenames in os.walk(nbdir):
if dirname != ".ipynb_checkpoints":
for filename in filenames:
if (filename.endswith(".ipynb")
and not filename.endswith("-checkpoint.ipynb")):
with open(os.path.join(dirname, filename)) as f:
print("Clearing output from {}".format(filename))
nb = nbformat.read(f, as_version=4)
ep = ClearOutputPreprocessor()
ep.preprocess(nb, {
"kernel_name":
"python{}".format(sys.version_info[0])
})
print(" ... done\n")
return True
def clear_notebooks(root):
"""Clear the outputs of documentation notebooks."""
# cleanup ignored files
run(['git', 'clean', '-fdX', root])
# remove release/autograded/feedback
if os.path.exists(os.path.join(root, "user_guide", "release")):
shutil.rmtree(os.path.join(root, "user_guide", "release"))
if os.path.exists(os.path.join(root, "user_guide", "autograded")):
shutil.rmtree(os.path.join(root, "user_guide", "autograded"))
if os.path.exists(os.path.join(root, "user_guide", "feedback")):
shutil.rmtree(os.path.join(root, "user_guide", "feedback"))
if os.path.exists(
os.path.join(root, "user_guide", "downloaded", "ps1",
"extracted")):
shutil.rmtree(
os.path.join(root, "user_guide", "downloaded", "ps1", "extracted"))
print("Clearing outputs of notebooks in '{}'...".format(
os.path.abspath(root)))
preprocessor = ClearOutputPreprocessor()
for dirpath, dirnames, filenames in os.walk(root):
is_submitted = _check_if_directory_in_path(dirpath, 'submitted')
for filename in sorted(filenames):
if os.path.splitext(filename)[1] == '.ipynb':
# read in the notebook
pth = os.path.join(dirpath, filename)
with open(pth, 'r') as fh:
orig_nb = read(fh, 4)
# copy the original notebook
new_nb = deepcopy(orig_nb)
# check outputs of all the cells
if not is_submitted:
new_nb = preprocessor.preprocess(new_nb, {})[0]
# clear metadata
new_nb.metadata = {
"kernelspec": {
"display_name": "Python",
"language": "python",
"name": "python"
}
}
# write the notebook back to disk
with open(pth, 'w') as fh:
write(new_nb, fh, 4)
if orig_nb != new_nb:
print("Cleared '{}'".format(pth))
def clear_notebook(path):
real_path = Path(path)
body = ""
with open(real_path, "r", encoding="utf-8") as nbfile:
nb = nbformat.read(nbfile, as_version=4)
co = ClearOutputPreprocessor()
exporter = NotebookExporter()
node, resources = co.preprocess(nb, {'metadata': {'path': './'}})
body, resources = exporter.from_notebook_node(node, resources)
with open(real_path, "w", encoding="utf-8") as nbfile:
nbfile.write(body)
def unexecute(self):
"""
Unexecutes the notebook i.e. removes all output cells
"""
_logger.info(f"Cleaning up notebook {self.nb} in {self.nb_dir}")
if not self.executed_nb_path.exists():
_logger.warning(
f"{self.executed_nb_path} not found, nothing to clean")
return
with open(self.executed_nb_path) as f:
nb = nbformat.read(f, as_version=IPYTHON_VERSION)
if nb['metadata'].get('docs_executed', None):
nb['metadata'].pop('docs_executed')
clear_executor = ClearOutputPreprocessor()
clear_executor.preprocess(nb, {})
with open(self.executed_nb_path, 'w') as f:
nbformat.write(nb, f)
def clear_notebook(path):
"""Clears output from a notebook
Parameters
----------
path: str (optional)
path to a Jupyter Notebook
Returns
-------
None
"""
with open(_get_path(path), encoding="utf-8") as f:
nb = nbformat.read(f, as_version=4)
pp = ClearOutputPreprocessor()
pp.preprocess(nb, {})
with open(path, "w", encoding="utf-8") as f:
nbformat.write(nb, f)
def unexecute(self, remove_gh=False):
"""
Unexecutes the notebook i.e. removes all output cells. If remove_gh=True looks to see if
notebook metadata contains an executed tag. If it doesn't it means the notebook either
errored or was not run (for case when only specific notebooks chosen to build examples) and
removes the notebooks so old ones can be used
"""
_logger.info(f"Cleaning up notebook {self.nb} in {self.nb_dir}")
if not self.executed_nb_path.exists():
_logger.warning(
f"{self.executed_nb_path} not found, nothing to clean")
return
with open(self.executed_nb_path, encoding='utf-8') as f:
nb = nbformat.read(f, as_version=IPYTHON_VERSION)
if not remove_gh:
if nb['metadata'].get('docs_executed', None):
nb['metadata'].pop('docs_executed')
clear_executor = ClearOutputPreprocessor()
clear_executor.preprocess(nb, {})
with open(self.executed_nb_path, 'w', encoding='utf-8') as f:
nbformat.write(nb, f)
elif remove_gh:
executed_flag = nb['metadata'].get('docs_executed', None)
if executed_flag != 'executed':
_logger.warning(f"Notebook {self.nb} not executed or errored, "
f"version already on website will be used")
os.remove(self.executed_nb_path)
os.remove(self.output_path.joinpath(self.nb_name + '.html'))
else:
_logger.info(f"Notebook {self.nb} executed, "
f"new version will be uploaded to website")
clear_executor = ClearOutputPreprocessor()
clear_executor.preprocess(nb, {})
with open(self.executed_nb_path, 'w', encoding='utf-8') as f:
nbformat.write(nb, f)
def clear(notebook_filename, nbconvert_filename):
with open(notebook_filename, 'r') as fh:
nb = nbformat.read(fh, 4)
# check outputs of all the cells
preprocessor = ClearOutputPreprocessor()
clear_nb = preprocessor.preprocess(nb, {})[0]
# clear metadata
clear_nb.metadata = {}
# write the notebook back to disk
with open(nbconvert_filename, 'w') as fh:
nbformat.write(clear_nb, fh, 4)
def clearNbOutput():
cwd = os.getcwd()
# walk the test directory and find all notebooks
for dirname, dirnames, filenames in os.walk(nbdir):
if dirname != ".ipynb_checkpoints":
for filename in filenames:
if (
filename.endswith(".ipynb") and not
filename.endswith("-checkpoint.ipynb")
):
with open(os.path.join(dirname, filename)) as f:
print("Clearing output from {}".format(filename))
nb = nbformat.read(f, as_version=4)
ep = ClearOutputPreprocessor()
ep.preprocess(
nb, {
"kernel_name": "python{}".format(
sys.version_info[0]
)
}
)
print(" ... done\n")
return True
def clear_notebook(path):
real_path = Path(path)
body = ""
with open(real_path, "r", encoding="utf-8") as nbfile:
nb = nbformat.read(nbfile, as_version=4)
orig_parameters = extract_parameters(nb)
params = parameter_values(orig_parameters,
SCOPETYPE="OPENADC",
PLATFORM="CWLITEARM")
new_nb = replace_definitions(nb, params, execute=False)
co = ClearOutputPreprocessor()
exporter = NotebookExporter()
node, resources = co.preprocess(new_nb, {'metadata': {'path': './'}})
body, resources = exporter.from_notebook_node(node, resources)
with open(real_path, "w", encoding="utf-8") as nbfile:
nbfile.write(body)
def clear_notebooks(root):
"""Clear the outputs of documentation notebooks."""
# cleanup ignored files
run(['git', 'clean', '-fdX', root])
# remove release/autograded/feedback
if os.path.exists(os.path.join(root, "user_guide", "release")):
shutil.rmtree(os.path.join(root, "user_guide", "release"))
if os.path.exists(os.path.join(root, "user_guide", "autograded")):
shutil.rmtree(os.path.join(root, "user_guide", "autograded"))
if os.path.exists(os.path.join(root, "user_guide", "feedback")):
shutil.rmtree(os.path.join(root, "user_guide", "feedback"))
if os.path.exists(os.path.join(root, "user_guide", "downloaded", "ps1", "extracted")):
shutil.rmtree(os.path.join(root, "user_guide", "downloaded", "ps1", "extracted"))
print("Clearing outputs of notebooks in '{}'...".format(os.path.abspath(root)))
preprocessor = ClearOutputPreprocessor()
for dirpath, dirnames, filenames in os.walk(root):
is_submitted = _check_if_directory_in_path(dirpath, 'submitted')
for filename in sorted(filenames):
if os.path.splitext(filename)[1] == '.ipynb':
# read in the notebook
pth = os.path.join(dirpath, filename)
with io.open(pth, encoding='utf-8') as fh:
orig_nb = read(fh, 4)
# copy the original notebook
new_nb = clean_notebook(orig_nb)
# check outputs of all the cells
if not is_submitted:
new_nb = preprocessor.preprocess(new_nb, {})[0]
# write the notebook back to disk
with io.open(pth, mode='w', encoding='utf-8') as fh:
write(new_nb, fh, 4)
if orig_nb != new_nb:
print("Cleared '{}'".format(pth))
def nb_exec(fname, name):
ver = make_version(name)
dirpath = os.path.join(data_dir, name, ver)
execpath = os.path.join(dirpath, 'src')
src_nb = os.path.join(execpath, 'src.ipynb')
ensure_dir(dirpath)
ensure_dir(execpath)
logger.info(f'Copying {fname} to {src_nb}')
shutil.copy2(fname, src_nb)
with open(src_nb) as f:
nb = nbformat.read(f, as_version=4)
co = ClearOutputPreprocessor()
nb, _ = co.preprocess(nb, {})
ep = ExecutePreprocessor(timeout=-1, kernel_name='python3')
try:
ep.preprocess(nb, {'metadata': {'path': execpath}})
except CellExecutionError as e:
print(e)
with open(os.path.join(dirpath, 'error.log'), 'w') as f:
f.write(str(e))
print('Stack trace saved')
with open(os.path.join(dirpath, 'output.ipynb'), 'w',
encoding='utf-8') as f:
nbformat.write(nb, f)
exp = HTMLExporter()
body, res = exp.from_notebook_node(nb)
with open(os.path.join(dirpath, 'output.html'), 'w',
encoding='utf-8') as f:
f.write(body)
def clear_notebooks(root):
"""Clear the outputs of documentation notebooks."""
# cleanup ignored files
run(['git', 'clean', '-fdX', root])
print("Clearing outputs of notebooks in '{}'...".format(
os.path.abspath(root)))
preprocessor = ClearOutputPreprocessor()
for dirpath, dirnames, filenames in os.walk(root):
is_submitted = _check_if_directory_in_path(dirpath, 'submitted')
for filename in sorted(filenames):
if os.path.splitext(filename)[1] == '.ipynb':
# read in the notebook
pth = os.path.join(dirpath, filename)
with open(pth, 'r') as fh:
orig_nb = read(fh, 4)
# copy the original notebook
new_nb = deepcopy(orig_nb)
# check outputs of all the cells
if not is_submitted:
new_nb = preprocessor.preprocess(new_nb, {})[0]
# clear metadata
new_nb.metadata = {}
# write the notebook back to disk
with open(pth, 'w') as fh:
write(new_nb, fh, 4)
if orig_nb != new_nb:
print("Cleared '{}'".format(pth))
def execute(self, force=False):
"""
Executes the specified notebook file, and writes the executed notebook to a
new file.
Parameters
----------
force : bool, optional
To force rerun notebook even if it has already been executed
Returns
-------
executed_nb_path : str, ``None``
The path to the executed notebook path, or ``None`` if ``write=False``.
status: bool
Whether the notebook executed without errors or not, 0 = ran without error, 1 = error
"""
with open(self.nb_path, encoding='utf-8') as f:
nb = nbformat.read(f, as_version=IPYTHON_VERSION)
is_executed = nb['metadata'].get('docs_executed')
if is_executed == 'executed' and not force:
_logger.warning(
f"Notebook {self.nb} in {self.nb_dir} already executed, skipping,"
f"to force execute, parse argument -f")
status = 0
else:
# Execute the notebook
_logger.info(f"Executing notebook {self.nb} in {self.nb_dir}")
t0 = time.time()
clear_executor = ClearOutputPreprocessor()
executor = ExecutePreprocessor(**self.execute_kwargs)
# First clean up the notebook and remove any cells that have been run
clear_executor.preprocess(nb, {})
try:
executor.preprocess(nb, {'metadata': {'path': self.nb_dir}})
execute_dict = {'docs_executed': 'executed'}
nb['metadata'].update(execute_dict)
status = 0
except CellExecutionError as err:
execute_dict = {'docs_executed': 'errored'}
nb['metadata'].update(execute_dict)
_logger.error(f"Error executing notebook {self.nb}")
_logger.error(err)
status = 1
_logger.info(f"Finished running notebook ({time.time() - t0})")
_logger.info(
f"Writing executed notebook to {self.executed_nb_path}")
# Makes sure original notebook isn't left blank in case of error during writing
if self.overwrite:
with open(self.temp_nb_path, 'w', encoding='utf-8') as f:
nbformat.write(nb, f)
shutil.copyfile(self.temp_nb_path, self.executed_nb_path)
os.remove(self.temp_nb_path)
else:
with open(self.executed_nb_path, 'w', encoding='utf-8') as f:
nbformat.write(nb, f)
return self.executed_nb_path, status
def open_nb_and_strip_output(fname):
cop = ClearOutputPreprocessor()
with open(fname) as f:
nb = nbformat.read(f, as_version=4)
cop.preprocess(nb, dict())
return nb
def convert_notebook_to_assets(notebook_file_name, base_name, output_prefix):
# define and create output folder
output_folder = output_prefix + '/' + base_name
os.makedirs(output_folder, exist_ok=True)
# open file
print('Converting Notebook: ' + notebook_file_name + ' ...')
nb_file = open(notebook_file_name, 'r').read()
nb = nbformat.reads(nb_file, as_version=4)
# 1. clear output
print(" - clearing output")
ep = ClearOutputPreprocessor()
ep.preprocess(nb, {})
# 2. generate fresh charts by running the notebook
print(" - executing")
ep = ExecutePreprocessor(timeout=600,
kernel_name='python3',
allow_errors=False)
try:
ep.preprocess(nb, {'metadata': {'path': output_folder}})
except Exception as e:
print('ERROR: Execution of the notebook ' + notebook_file_name +
' stopped, likely for missing some py libs.')
print(
' Please check the output/exception and add those to the requirements.'
)
print(e)
exit(1)
# 3. export HTML
print(" - generating html")
cleaner_config = Config({
"HTMLExporter": {
"exclude_input": True,
"exclude_input_prompt": True,
"exclude_output_prompt": True,
"preprocessors":
['nbconvert.preprocessors.ExtractOutputPreprocessor']
},
})
local_templates = DictLoader({
'our-html.tpl': stand_alone_tpl,
'react-glue.tpl': react_glue_tpl,
})
exporter = HTMLExporter(config=cleaner_config,
extra_loaders=[local_templates])
exporter.template_file = 'our-html.tpl'
(html_body, html_resources) = exporter.from_notebook_node(nb)
notebook_convert_time = current_utc_time()
# save html output file, with local reference to the pictures
local_html = []
output_html_file_name = output_folder + '/' + "index.html"
print(" - saving html: " + output_html_file_name)
with open(output_html_file_name, 'wt') as the_file:
the_file.write(html_body)
local_html.append({
'notebook': base_name,
'notebook_html': output_html_file_name,
'convert_time': notebook_convert_time,
})
# save js file for react inclusion (local ref to the pictures)
# exporter.template_file = 'react-glue.tpl'
# (react_body, react_resources) = exporter.from_notebook_node(nb)
# output_react_file_name = output_folder + '/' + "index.js"
# print(" - saving react js: " + output_react_file_name)
# with open(output_react_file_name, 'wt') as the_file:
# the_file.write(react_body)
# save all the figures
local_figures = []
figures = html_resources['outputs']
figures_count = len(figures)
figure_index = 1
for figure_file in figures:
output_figure_file_name = output_folder + '/' + figure_file
print(" - saving png " + str(figure_index) + " of " +
str(figures_count) + ": " + output_figure_file_name)
if not figure_file.endswith('.png'):
print("WARNING: figure is not a PNG file")
continue
with open(output_figure_file_name, 'wb') as the_file:
the_file.write(figures[figure_file])
local_figures.append({
'figure': figure_file,
'file': output_figure_file_name,
'notebook': base_name,
'notebook_html': output_html_file_name,
'convert_time': notebook_convert_time,
})
# create an empty 'custom.css'
custom_css_file_name = output_folder + '/' + 'custom.css'
with open(custom_css_file_name, 'wt') as the_file:
the_file.write("")
# return a recap of all assets
return local_html, local_figures
def test_func(self):
cwd = os.getcwd()
passing = True
print("\n---------------------"
" Testing {0}.ipynb "
"---------------------".format(nbname))
if (nbname in self.ignore) or (nbname in self.py2_ignore
and sys.version_info[0] == 2):
print(" Skipping {}".format(nbname))
return
run_path = os.path.sep.join(nbpath.split(os.path.sep)[:-1])
os.chdir(run_path)
clear_output = ClearOutputPreprocessor()
with open(nbpath) as nbfile:
notebook = nbformat.read(nbfile, as_version=4)
clear_output.preprocess(notebook, {})
execute = ExecutePreprocessor(
timeout=timeout,
kernel_name="python{}".format(sys.version_info[0]),
allow_errors=True,
)
out = execute.preprocess(notebook, {})
os.chdir(cwd)
for cell in out[0]["cells"]:
if "outputs" in cell.keys():
for output in cell["outputs"]:
if output["output_type"] == "error":
passing = False
err_msg = []
for traceback in output["traceback"]:
err_msg += ["{}".format(traceback)]
err_msg = "\n".join(err_msg)
msg = """
\n ... {} FAILED \n
{} in cell [{}] \n-----------\n{}\n-----------\n
""".format(
nbname,
output["ename"],
cell["execution_count"],
cell["source"],
)
traceback = """
----------------- >> begin Traceback << ----------------- \n
{}\n
\n----------------- >> end Traceback << -----------------\n
""".format(err_msg)
print(u"{}".format(msg + traceback))
assert passing, msg
print(" ... {0} Passed \n".format(nbname))
def test_func(self):
cwd = os.getcwd()
passing = True
print("\n---------------------"
" Testing {0}.ipynb "
"---------------------".format(nbname))
if ((nbname in self.ignore)
or (nbname in self.py2_ignore and sys.version_info[0] == 2)):
print(" Skipping {}".format(nbname))
return
run_path = os.path.sep.join(nbpath.split(os.path.sep)[:-1])
os.chdir(run_path)
ep = ClearOutputPreprocessor(
resources={'metadata': {
'path': run_path
}})
with open(nbpath) as f:
nb = nbformat.read(f, as_version=4)
ep.preprocess(nb, {})
ex = ExecutePreprocessor(
timeout=timeout,
kernel_name='python{}'.format(sys.version_info[0]),
allow_errors=True,
resources={'metadata': {
'path': run_path
}})
out = ex.preprocess(nb, {})
os.chdir(cwd)
for cell in out[0]['cells']:
if 'outputs' in cell.keys():
for output in cell['outputs']:
if output['output_type'] == 'error':
passing = False
err_msg = []
for o in output['traceback']:
err_msg += ["{}".format(o)]
err_msg = "\n".join(err_msg)
msg = """
\n ... {} FAILED \n
{} in cell [{}] \n-----------\n{}\n-----------\n
""".format(
nbname,
output['ename'],
cell['execution_count'],
cell['source'],
)
traceback = """
----------------- >> begin Traceback << ----------------- \n
{}\n
\n----------------- >> end Traceback << -----------------\n
""".format(err_msg)
print(u"{}".format(msg + traceback))
assert passing, msg
print(" ... {0} Passed \n".format(nbname))
def run_notebook(self, name, event=None, timeout=None):
""" run a given notebook immediately.
Args:
name (str): the name of the jobfile
event (str): an event name
timeout (int): timeout in seconds
Returns:
Metadata of results
See Also:
* nbconvert https://nbconvert.readthedocs.io/en/latest/execute_api.html
"""
notebook = self.get(name)
meta_job = self.metadata(name)
ts = datetime.datetime.now()
# execute kwargs
# -- see ExecuteProcessor class
# -- see https://nbconvert.readthedocs.io/en/latest/execute_api.html
ep_kwargs = {
# avoid timeouts to stop kernel
'timeout': timeout,
# avoid kernel at exit functions
# -- this stops ipykernel AttributeError 'send_multipart'
'shutdown_kernel': 'immediate',
# set kernel name, blank is default
# -- e.g. python3, ir
# -- see https://stackoverflow.com/a/47053020/890242
'kernel_name': '',
}
# other interesting options
ep_kwargs.update(meta_job.kind_meta.get('ep_kwargs', {}))
try:
resources = {
'metadata': {
'path': self.defaults.OMEGA_TMP,
}
}
if not meta_job.kind_meta.get('keep_output', False):
# https://nbconvert.readthedocs.io/en/latest/api/preprocessors.html
cp = ClearOutputPreprocessor()
cp.preprocess(notebook, resources)
ep = ExecutePreprocessor(**ep_kwargs)
ep.preprocess(notebook, resources)
except Exception as e:
status = 'ERROR'
message = str(e)
else:
status = 'OK'
message = ''
finally:
del ep
# record results
meta_results = self.put(notebook,
'results/{name}_{ts}'.format(**locals()))
meta_results.attributes['source_job'] = name
meta_results.save()
job_results = meta_job.attributes.get('job_results', [])
job_results.append(meta_results.name)
meta_job.attributes['job_results'] = job_results
# record final job status
job_runs = meta_job.attributes.get('job_runs', [])
runstate = {
'status': status,
'ts': ts,
'message': message,
'results': meta_results.name if status == 'OK' else None
}
job_runs.append(runstate)
meta_job.attributes['job_runs'] = job_runs
# set event run state if event was specified
if event:
attrs = meta_job.attributes
triggers = attrs['triggers'] = attrs.get('triggers', [])
scheduled = (trigger for trigger in triggers
if trigger['event-kind'] == 'scheduled')
for trigger in scheduled:
if event == trigger['event']:
trigger['status'] = status
trigger['ts'] = ts
meta_job.save()
return meta_results
