def preprocess_cell(self, cell, resources, cell_index):
"""
Executes cells without 'skip' tag only.
If the tag is not found cell is not executed.
"""
tags = cell.metadata.get('tags')
if tags is not None and 'skip' in tags:
return cell, resources
else:
try:
cell, resources = super().preprocess_cell(
cell, resources, cell_index)
# cell output to STDOUT of this process
if hasattr(cell, 'outputs'):
for output in cell.outputs:
output_text = ''
# support multiple types of output:
# output['text'] (for output['output_type']=='stream')
# output['data']['text/plain'] (for output['output_type']=='execute_result')
# Note this means application/json and image/png are currently not supported for logging.
if 'text' in output:
output_text = output['text']
elif 'data' in output and 'text/plain' in output[
'data']:
output_text = output['data']['text/plain']
if not output_text.endswith('\n'):
output_text = ''.join([output_text, '\n'])
# process output text with ansi2html to prep for output
# in html log viewer
output_text = ansi2html(output_text)
self.log_file.write(
"[%i] %s" % (cell['execution_count'], output_text))
self.log_file.flush()
except CellExecutionError as e:
self.log_file.write("%s" % ansi2html(e))
self.log_file.flush()
# raise CellExecutionError to avoid execution next cells
raise e
# else:
# # TODO: Evaluate whether we want to output anything for no output cells
# self.log_file.write("%s" % ('<No output cell: %s >\n' % cell['cell_type']))
return cell, resources
def log_output_message(self, output):
if self.current_cell is None:
raise Exception(
"log_output_message should not be called if there is no current notebook cell"
)
# cell output to STDOUT of this process
output_text = ''
# support multiple types of output:
# output['text'] (for output['output_type']=='stream')
# output['data']['text/plain'] (for output['output_type']=='execute_result')
# Note this means application/json and image/png are currently not supported for logging.
if 'text' in output:
output_text = output['text']
elif 'data' in output and 'text/plain' in output['data']:
output_text = output['data']['text/plain']
if not output_text.endswith('\n'):
output_text = ''.join([output_text, '\n'])
# process output text with ansi2html to prep for output
# in html log viewer
output_text = ansi2html(output_text)
prefix = "[%i] " % self.current_cell["execution_count"]
if self.current_cell["execution_count"] in self.printed_indices:
prefix = ""
else:
self.printed_indices.add(self.current_cell["execution_count"])
self.log_file.write("".join([prefix, output_text]))
self.log_file.flush()
def validate(self, filename):
self.log.info("Validating '{}'".format(os.path.abspath(filename)))
nb = self._preprocess(filename)
changed = self._get_changed_cells(nb)
passed = self._get_passed_cells(nb)
failed = self._get_failed_cells(nb)
results = {}
if not self.ignore_checksums and len(changed) > 0:
results['changed'] = [{
"source": cell.source.strip()
} for cell in changed]
elif self.invert:
if len(passed) > 0:
results['passed'] = [{
"source": cell.source.strip()
} for cell in passed]
else:
if len(failed) > 0:
results['failed'] = [{
"source": cell.source.strip(),
"error": ansi2html(self._extract_error(cell)),
"raw_error": self._extract_error(cell)
} for cell in failed]
return results
def preprocess(self, nb, resources):
if 'nbgrader' not in resources:
resources['nbgrader'] = {}
resources['nbgrader']['failed_cells'] = []
resources['nbgrader']['passed_cells'] = []
resources['nbgrader']['checksum_mismatch'] = []
nb, resources = super(DisplayAutoGrades,
self).preprocess(nb, resources)
changed = resources['nbgrader']['checksum_mismatch']
failed = resources['nbgrader']['failed_cells']
passed = resources['nbgrader']['passed_cells']
json_dict = {}
if not self.ignore_checksums and len(changed) > 0:
if self.as_json:
json_dict['changed'] = [{
"source": cell.source.strip()
} for cell in changed]
else:
self._print_num_changed(len(changed))
for cell in changed:
self._print_changed(cell)
elif self.invert:
if self.as_json:
if len(passed) > 0:
json_dict['passed'] = [{
"source": cell.source.strip()
} for cell in passed]
else:
self._print_num_passed(len(passed))
for cell in passed:
self._print_pass(cell)
else:
if self.as_json:
if len(failed) > 0:
json_dict['failed'] = [{
"source":
cell.source.strip(),
"error":
ansi2html(self._extract_error(cell))
} for cell in failed]
else:
self._print_num_failed(len(failed))
for cell in failed:
self._print_error(cell)
if self.as_json:
self.stream.write(json.dumps(json_dict))
return nb, resources
def validate(
self, filename: str
) -> typing.Dict[str, typing.List[typing.Dict[str, str]]]:
self.log.info("Validating '{}'".format(os.path.abspath(filename)))
basename = os.path.basename(filename)
dirname = os.path.dirname(filename)
with utils.chdir(dirname):
nb = read_nb(basename, as_version=current_nbformat)
type_changed = self._get_type_changed_cells(nb)
if len(type_changed) > 0:
results = {}
results['type_changed'] = [{
"source":
cell.source.strip(),
"old_type":
cell.cell_type,
"new_type":
cell.metadata.nbgrader.cell_type
} for cell in type_changed]
return results
with utils.chdir(dirname):
nb = self._preprocess(nb)
changed = self._get_changed_cells(nb)
passed = self._get_passed_cells(nb)
failed = self._get_failed_cells(nb)
results = {}
if not self.ignore_checksums and len(changed) > 0:
results['changed'] = [{
"source": cell.source.strip()
} for cell in changed]
elif self.invert:
if len(passed) > 0:
results['passed'] = [{
"source": cell.source.strip()
} for cell in passed]
else:
if len(failed) > 0:
results['failed'] = [{
"source":
cell.source.strip(),
"error":
ansi2html(self._extract_error(cell)),
"raw_error":
self._extract_error(cell)
} for cell in failed]
return results
def preprocess(self, nb, resources):
if 'nbgrader' not in resources:
resources['nbgrader'] = {}
resources['nbgrader']['failed_cells'] = []
resources['nbgrader']['passed_cells'] = []
resources['nbgrader']['checksum_mismatch'] = []
nb, resources = super(DisplayAutoGrades, self).preprocess(nb, resources)
changed = resources['nbgrader']['checksum_mismatch']
failed = resources['nbgrader']['failed_cells']
passed = resources['nbgrader']['passed_cells']
json_dict = {}
if not self.ignore_checksums and len(changed) > 0:
if self.as_json:
json_dict['changed'] = [{
"source": cell.source.strip()
} for cell in changed]
else:
self._print_num_changed(len(changed))
for cell in changed:
self._print_changed(cell)
elif self.invert:
if self.as_json:
if len(passed) > 0:
json_dict['passed'] = [{
"source": cell.source.strip()
} for cell in passed]
else:
self._print_num_passed(len(passed))
for cell in passed:
self._print_pass(cell)
else:
if self.as_json:
if len(failed) > 0:
json_dict['failed'] = [{
"source": cell.source.strip(),
"error": ansi2html(self._extract_error(cell))
} for cell in failed]
else:
self._print_num_failed(len(failed))
for cell in failed:
self._print_error(cell)
if self.as_json:
self.stream.write(json.dumps(json_dict))
return nb, resources
def validate(self, filename):
self.log.info("Validating '{}'".format(os.path.abspath(filename)))
basename = os.path.basename(filename)
dirname = os.path.dirname(filename)
with utils.chdir(dirname):
nb = read_nb(basename, as_version=current_nbformat)
type_changed = self._get_type_changed_cells(nb)
if len(type_changed) > 0:
results = {}
results['type_changed'] = [{
"source": cell.source.strip(),
"old_type": cell.cell_type,
"new_type": cell.metadata.nbgrader.cell_type
} for cell in type_changed]
return results
# Changing directories to enable validation to run in the same
# environment as the original notebook.
with utils.chdir(dirname):
nb = self._preprocess(nb)
changed = self._get_changed_cells(nb)
passed = self._get_passed_cells(nb)
failed = self._get_failed_cells(nb)
results = {}
if not self.ignore_checksums and len(changed) > 0:
results['changed'] = [{
"source": cell.source.strip()
} for cell in changed]
elif self.invert:
if len(passed) > 0:
results['passed'] = [{
"source": cell.source.strip()
} for cell in passed]
else:
if len(failed) > 0:
results['failed'] = [{
"source": cell.source.strip(),
"error": ansi2html(self._extract_error(cell)),
"raw_error": self._extract_error(cell)
} for cell in failed]
return results
def validate(self, filename):
self.log.info("Validating '{}'".format(os.path.abspath(filename)))
basename = os.path.basename(filename)
dirname = os.path.dirname(filename)
with utils.chdir(dirname):
nb = read_nb(basename, as_version=current_nbformat)
type_changed = self._get_type_changed_cells(nb)
if len(type_changed) > 0:
results = {}
results['type_changed'] = [{
"source": cell.source.strip(),
"old_type": cell.cell_type,
"new_type": cell.metadata.nbgrader.cell_type
} for cell in type_changed]
return results
with utils.chdir(dirname):
nb = self._preprocess(nb)
changed = self._get_changed_cells(nb)
passed = self._get_passed_cells(nb)
failed = self._get_failed_cells(nb)
results = {}
if not self.ignore_checksums and len(changed) > 0:
results['changed'] = [{
"source": cell.source.strip()
} for cell in changed]
elif self.invert:
if len(passed) > 0:
results['passed'] = [{
"source": cell.source.strip()
} for cell in passed]
else:
if len(failed) > 0:
results['failed'] = [{
"source": cell.source.strip(),
"error": ansi2html(self._extract_error(cell)),
"raw_error": self._extract_error(cell)
} for cell in failed]
return results
def _notebook_run(path, SCOPETYPE='OPENADC', PLATFORM='CWLITEARM', **kwargs):
"""Execute a notebook via nbconvert and collect output.
:returns (parsed nb object, execution errors)
"""
html_path = Path("html/" + path + "-{}-{}".format(SCOPETYPE, PLATFORM) +
".html")
real_path = Path(path)
with open(real_path) as nbfile:
nb = nbformat.read(nbfile, as_version=4)
orig_parameters = extract_parameters(nb)
params = parameter_values(orig_parameters,
SCOPETYPE=SCOPETYPE,
PLATFORM=PLATFORM,
**kwargs)
new_nb = replace_definitions(nb, params, execute=False)
ep = ExecutePreprocessor(timeout=None,
kernel_name='python3',
allow_errors=True)
ep.preprocess(new_nb, {'metadata': {'path': './'}})
errors = [[i+1,output] for i,cell in enumerate(new_nb.cells) if "outputs" in cell
for output in cell["outputs"]\
if output.output_type == "error"]
with open(html_path, "w", encoding='utf-8') as html_file:
html_exporter = HTMLExporter()
body, res = html_exporter.from_notebook_node(new_nb)
body = ansi2html(body)
html_file.write(body)
return nb, errors
def highlight_ansi_and_escape(self, text):
return filters.ansi2html(text)
def highlight_ansi_and_escape(self, text):
return filters.ansi2html(text)
def ansi2html(lst):
text = ''.join(lst)
return filters.ansi2html(text)