def preprocess(self, nb, resources):
logging.info('adding ipub defaults to notebook')
for keys, val in flatten(self.nb_defaults).items():
dct = nb.metadata
for key in keys[:-1]:
if key not in dct:
dct[key] = NotebookNode({})
dct = dct[key]
if keys[-1] not in dct:
dct[keys[-1]] = val
elif self.overwrite:
dct[keys[-1]] = val
for cell in nb.cells:
for keys, val in flatten(self.cell_defaults).items():
dct = cell.metadata
for key in keys[:-1]:
if key not in dct:
dct[key] = NotebookNode({})
dct = dct[key]
if keys[-1] not in dct:
dct[keys[-1]] = val
elif self.overwrite:
dct[keys[-1]] = val
return nb, resources
def preprocess_cell(self,
cell: NotebookNode,
resources: ResourcesDict,
cell_index: int
) -> Tuple[NotebookNode, ResourcesDict]:
# replace solution regions with the relevant stubs
language = resources["language"]
replaced_solution = self._replace_solution_region(cell, language)
# determine whether the cell is a solution/grade cell
is_solution = utils.is_solution(cell)
# check that it is marked as a solution cell if we replaced a solution
# region -- if it's not, then this is a problem, because the cell needs
# to be given an id
if not is_solution and replaced_solution:
if self.enforce_metadata:
raise RuntimeError(
"Solution region detected in a non-solution cell; please make sure "
"all solution regions are within solution cells."
)
# replace solution cells with the code/text stub -- but not if
# we already replaced a solution region, because that means
# there are parts of the cells that should be preserved
if is_solution and not replaced_solution:
if cell.cell_type == 'code':
cell.source = self.code_stub[language]
else:
cell.source = self.text_stub
return cell, resources
def postprocess_output(self, outputs):
"""
Postprocesses output and maps mime types to ones accepted by R.
"""
res = []
for output in outputs:
msg_type = output.output_type
content = output
out = NotebookNode(output_type=msg_type)
if msg_type in ('display_data', 'execute_result'):
for mime, data in content['data'].items():
try:
attr = self.MIME_MAP[mime]
if attr == 'text':
tmpval = RClansiconv(data)
else:
tmpval = data
setattr(out, attr, tmpval)
except KeyError:
raise NotImplementedError('unhandled mime type: %s' %
mime)
elif msg_type == 'stream':
setattr(out, 'text', RClansiconv(content['text']))
elif msg_type == 'error':
setattr(out, 'html',
RClansiconv('\n'.join(content['traceback']) + '\n'))
else:
if _debugging: logging.info('Unsupported: ' + msg_type)
raise NotImplementedError('unhandled result: %s' % msg_type)
if _debugging:
logging.info(
'Sending: msg_type: [{}]; HTML: [{}]; TEXT: [{}]'.format(
msg_type, out.get('html', ''), out.get('text', '')))
res.append(out)
return res # upstream process will handle it [e.g. send as an oob message]
def update_cell_type(self, cell: NotebookNode, cell_type: str) -> None:
if cell.cell_type == cell_type:
return
elif cell_type == 'code':
cell.cell_type = 'code'
cell.outputs = []
cell.execution_count = None
validate(cell, 'code_cell')
elif cell_type == 'markdown':
cell.cell_type = 'markdown'
if 'outputs' in cell:
del cell['outputs']
if 'execution_count' in cell:
del cell['execution_count']
validate(cell, 'markdown_cell')
def postprocess_output(self, outputs):
"""
Postprocesses output and maps mime types to ones accepted by R.
"""
res = []
for output in outputs:
msg_type = output.output_type
content = output
out = NotebookNode(output_type = msg_type)
if msg_type in ('display_data', 'execute_result'):
for mime, data in content['data'].items():
try:
attr = self.MIME_MAP[mime]
if attr == 'text':
tmpval = RClansiconv(data)
else:
tmpval = data
setattr(out, attr, tmpval)
except KeyError:
raise NotImplementedError('unhandled mime type: %s' % mime)
elif msg_type == 'stream':
setattr(out, 'text', RClansiconv(content['text']))
elif msg_type == 'error':
setattr(out, 'html', RClansiconv('\n'.join(content['traceback']) + '\n'))
else:
if _debugging: logging.info('Unsupported: ' + msg_type)
raise NotImplementedError('unhandled result: %s' % msg_type)
if _debugging: logging.info('Sending: msg_type: [{}]; HTML: [{}]; TEXT: [{}]'.format(msg_type, out.get('html', ''), out.get('text', '') ))
res.append(out)
return res # upstream process will handle it [e.g. send as an oob message]
def preprocess(
self, nb: NotebookNode,
resources: ResourcesDict) -> Tuple[NotebookNode, ResourcesDict]:
"""Concatenates the cells from the header and footer notebooks to the
given cells.
"""
new_cells = []
# header
if self.header:
with io.open(self.header, encoding='utf-8') as fh:
header_nb = read_nb(fh, as_version=current_nbformat)
new_cells.extend(header_nb.cells)
# body
new_cells.extend(nb.cells)
# footer
if self.footer:
with io.open(self.footer, encoding='utf-8') as fh:
footer_nb = read_nb(fh, as_version=current_nbformat)
new_cells.extend(footer_nb.cells)
nb.cells = new_cells
super(IncludeHeaderFooter, self).preprocess(nb, resources)
return nb, resources
def preprocess(
self, nb: NotebookNode,
resources: ResourcesDict) -> Tuple[NotebookNode, ResourcesDict]:
# keep track of grade ids encountered so far
self.grade_ids = set([])
# reverse cell order
nb.cells = nb.cells[::-1]
# process each cell in reverse order
nb, resources = super(DeduplicateIds, self).preprocess(nb, resources)
# unreverse cell order
nb.cells = nb.cells[::-1]
return nb, resources
def new_cell(idents, filename, lineno, source=None):
cell_types = ['markdown', 'code']
slide_types = ['slide', 'subslide', 'fragment']
split_types = ['split']
cell = {'metadata': {},
'cell_type': 'markdown'}
if source is not None:
if source and source[-1] == "\n":
source = source[:-1]
cell['source'] = source
for ident in idents:
if ident in cell_types:
cell['cell_type'] = ident
if ident == 'code':
cell['outputs'] = []
cell['execution_count'] = None
elif ident in slide_types:
if 'slideshow' not in cell['metadata']:
cell['metadata']['slideshow'] = {}
cell['metadata']['slideshow']['slide_type'] = ident
elif ident in split_types:
cell['metadata']['cell_style'] = ident
else:
print("{}:{} - ignored directive `{}' in separator"
.format(filename, lineno, ident))
return NotebookNode(cell)
def embed_html(self, cell, path):
""" a new cell, based on embedded html file
"""
logging.info('embedding html in notebook from: {}'.format(path))
height = int(cell.metadata.ipub.embed_html.get('height', 0.5) * 100)
width = int(cell.metadata.ipub.embed_html.get('width', 0.5) * 100)
embed_code = """
<iframe style="display:block; margin: 0 auto; height:{height}vh; width:{width}vw; overflow:auto; resize:both" {src}="{path}" frameborder="0" allowfullscreen></iframe>
""".format(src=self.src_name, path=path, height=height, width=width)
# add to the exising output or create a new one
if cell.outputs:
cell.outputs[0]["data"]["text/html"] = embed_code
else:
cell.outputs.append(
NotebookNode({
"data": {
"text/html": embed_code
},
"execution_count": 0,
"metadata": {},
"output_type": "execute_result"
}))
return cell
def preprocess_cell(self, cell, resources, cell_index, store_history=True):
"""
Need to override preprocess_cell to check reply for errors
"""
# Copied from nbconvert ExecutePreprocessor
if cell.cell_type != 'code' or not cell.source.strip():
return cell, resources
reply, outputs = self.run_cell(cell, cell_index, store_history)
# Backwards compatibility for processes that wrap run_cell
cell.outputs = outputs
cell_allows_errors = (self.allow_errors
or "raises-exception" in cell.metadata.get(
"tags", []))
if self.force_raise_errors or not cell_allows_errors:
if (reply is not None) and reply['content']['status'] == 'error':
raise CellExecutionError.from_cell_and_msg(
cell, reply['content'])
# Ensure errors are recorded to prevent false positives when autograding
if (reply is None) or reply['content']['status'] == 'error':
error_recorded = False
for output in cell.outputs:
if output.output_type == 'error':
error_recorded = True
if not error_recorded:
error_output = NotebookNode(output_type='error')
if reply is None:
# Occurs when
# IPython.core.interactiveshell.InteractiveShell.showtraceback
# = None
error_output.ename = "CellTimeoutError"
error_output.evalue = ""
error_output.traceback = ["ERROR: No reply from kernel"]
else:
# Occurs when
# IPython.core.interactiveshell.InteractiveShell.showtraceback
# = lambda *args, **kwargs : None
error_output.ename = reply['content']['ename']
error_output.evalue = reply['content']['evalue']
error_output.traceback = reply['content']['traceback']
if error_output.traceback == []:
error_output.traceback = [
"ERROR: An error occurred while"
" showtraceback was disabled"
]
cell.outputs.append(error_output)
return cell, resources
def _replace_solution_region(self, cell: NotebookNode,
language: str) -> bool:
"""Find a region in the cell that is delimeted by
`self.begin_solution_delimeter` and `self.end_solution_delimeter` (e.g.
### BEGIN SOLUTION and ### END SOLUTION). Replace that region either
with the code stub or text stub, depending the cell type.
This modifies the cell in place, and then returns True if a
solution region was replaced, and False otherwise.
"""
# pull out the cell input/source
lines = cell.source.split("\n")
if cell.cell_type == "code":
stub_lines = self.code_stub[language].split("\n")
else:
stub_lines = self.text_stub.split("\n")
new_lines = []
in_solution = False
replaced_solution = False
for line in lines:
# begin the solution area
if self.begin_solution_delimeter in line:
# check to make sure this isn't a nested BEGIN
# SOLUTION region
if in_solution:
raise RuntimeError(
"encountered nested begin solution statements")
in_solution = True
replaced_solution = True
# replace it with the stub, indented as necessary
indent = re.match(r"\s*", line).group(0)
for stub_line in stub_lines:
new_lines.append(indent + stub_line)
# end the solution area
elif self.end_solution_delimeter in line:
in_solution = False
# add lines as long as it's not in the solution area
elif not in_solution:
new_lines.append(line)
# we finished going through all the lines, but didn't find a
# matching END SOLUTION statment
if in_solution:
raise RuntimeError("no end solution statement found")
# replace the cell source
cell.source = "\n".join(new_lines)
return replaced_solution
def preprocess(self, nb, resources):
logging.info('extracting caption cells')
# extract captions
final_cells = []
captions = {}
for cell in nb.cells:
if hasattr(cell.metadata, 'ipub'):
if hasattr(cell.metadata.ipub.get('equation',False),'get'):
if hasattr(cell.metadata.ipub.equation.get('environment',False),'startswith'):
if cell.metadata.ipub.equation.environment.startswith('breqn'):
if "ipub" not in nb.metadata:
nb.metadata["ipub"] = NotebookNode({'enable_breqn':True})
else:
nb.metadata.ipub['enable_breqn'] = True
if hasattr(cell.metadata.ipub, 'caption'):
if cell.cell_type == 'markdown':
capt = cell.source.split(r'\n')[0]
captions[cell.metadata.ipub.caption] = capt
continue
elif cell.cell_type == 'code':
if not cell.outputs:
pass
elif "text/latex" in cell.outputs[0].get('data',{}):
capt = cell.outputs[0].data["text/latex"].split(r'\n')[0]
captions[cell.metadata.ipub.caption] = capt
continue
elif "text/plain" in cell.outputs[0].get('data',{}):
capt = cell.outputs[0].data["text/plain"].split(r'\n')[0]
captions[cell.metadata.ipub.caption] = capt
continue
final_cells.append(cell)
nb.cells = final_cells
# replace captions
for cell in nb.cells:
if hasattr(cell.metadata, 'ipub'):
for key in cell.metadata.ipub:
if hasattr(cell.metadata.ipub[key], 'label'):
if cell.metadata.ipub[key]['label'] in captions:
logging.debug('replacing caption for: {}'.format(cell.metadata.ipub[key]['label']))
cell.metadata.ipub[key]['caption'] = captions[cell.metadata.ipub[key]['label']]
# add float type/number prefix to caption, if required
if self.add_prefix:
if hasattr(cell.metadata.ipub[key], 'caption'):
if hasattr(cell.metadata.ipub[key], 'caption_prefix'):
newcaption = cell.metadata.ipub[key].caption_prefix + cell.metadata.ipub[key].caption
cell.metadata.ipub[key].caption = newcaption
return nb, resources
def mkdcell(self, source, metadata, slidetype):
meta = copy.deepcopy(metadata)
meta.ipyslides = slidetype
self.append(
NotebookNode({
"cell_type": "markdown",
"source": '\n'.join(source),
"metadata": meta
}))
def preprocess_cell(self,
cell: NotebookNode,
resources: ResourcesDict,
cell_index: int
) -> Tuple[NotebookNode, ResourcesDict]:
grade_id = cell.metadata.get('nbgrader', {}).get('grade_id', None)
if grade_id is None:
return cell, resources
try:
source_cell = self.gradebook.find_source_cell(
grade_id,
self.notebook_id,
self.assignment_id)
except MissingEntry:
self.log.warning("Cell '{}' does not exist in the database".format(grade_id))
del cell.metadata.nbgrader['grade_id']
return cell, resources
# check that the cell type hasn't changed
if cell.cell_type != source_cell.cell_type:
self.report_change(grade_id, "cell_type", source_cell.cell_type, cell.cell_type)
self.update_cell_type(cell, source_cell.cell_type)
# check that the locked status hasn't changed
if utils.is_locked(cell) != source_cell.locked:
self.report_change(grade_id, "locked", source_cell.locked, utils.is_locked(cell))
cell.metadata.nbgrader["locked"] = source_cell.locked
# if it's a grade cell, check that the max score hasn't changed
if utils.is_grade(cell):
grade_cell = self.gradebook.find_graded_cell(
grade_id,
self.notebook_id,
self.assignment_id)
old_points = float(grade_cell.max_score)
new_points = float(cell.metadata.nbgrader["points"])
if old_points != new_points:
self.report_change(grade_id, "points", old_points, new_points)
cell.metadata.nbgrader["points"] = old_points
# always update the checksum, just in case
cell.metadata.nbgrader["checksum"] = source_cell.checksum
# if it's locked, check that the checksum hasn't changed
if source_cell.locked:
old_checksum = source_cell.checksum
new_checksum = utils.compute_checksum(cell)
if old_checksum != new_checksum:
self.report_change(grade_id, "checksum", old_checksum, new_checksum)
cell.source = source_cell.source
# double check the the checksum is correct now
if utils.compute_checksum(cell) != source_cell.checksum:
raise RuntimeError("Inconsistent checksums for cell {}".format(source_cell.name))
return cell, resources
def preprocess_cell(self, cell, resources, cell_index):
"""Also extracts attachments"""
from nbformat.notebooknode import NotebookNode
attach_names = []
# Just move the attachment into an output
for k, attach in cell.get('attachments', {}).items():
for mime_type in self.extract_output_types:
if mime_type in attach:
if 'outputs' not in cell:
cell['outputs'] = []
o = NotebookNode({
'data': NotebookNode({mime_type: attach[mime_type]}),
'metadata': NotebookNode({
'filenames': {mime_type: k} # Will get re-written
}),
'output_type': 'display_data'
})
cell['outputs'].append(o)
attach_names.append((mime_type, k))
nb, resources = super().preprocess_cell(cell, resources, cell_index)
output_names = list(resources.get('outputs', {}).keys())
if attach_names:
# We're going to assume that attachments are only on Markdown cells, and Markdown cells
# can't generate output, so all of the outputs were added.
# reverse + zip matches the last len(attach_names) elements from output_names
for output_name, (mimetype, an) in zip(reversed(output_names), reversed(attach_names)):
# We'll post process to set the final output directory
cell.source = re.sub(r'\(attachment:{}\)'.format(an),
'(__IMGDIR__/{})'.format(output_name), cell.source)
return nb, resources
def preprocess(self, nb, resources):
if not self.split:
return nb, resources
logging.info("splitting outputs into separate cells")
final_cells = []
for cell in nb.cells:
if not cell.cell_type == "code":
final_cells.append(cell)
continue
outputs = cell.pop("outputs")
cell.outputs = []
final_cells.append(cell)
for output in outputs:
meta = copy.deepcopy(cell.metadata)
# don't need the code to output
meta.get("ipub", NotebookNode({})).code = False
# don't create a new slide for each output,
# unless specified in output level metadata
if "slide" in meta.get("ipub", NotebookNode({})):
if meta.ipub.slide == "new":
meta.ipub.slide = True
else:
meta.ipub.slide = meta.ipub.slide
meta = merge(meta, output.get("metadata", {}))
new = NotebookNode(
{
"cell_type": "code",
"source": "",
"execution_count": None,
"metadata": meta,
"outputs": [output],
}
)
final_cells.append(new)
nb.cells = final_cells
return nb, resources
def preprocess(self, nb, resources):
logging.info("adding ipub defaults to notebook")
for keys, val in flatten(self.nb_defaults).items():
dct = nb.metadata
for key in keys[:-1]:
if key not in dct:
dct[key] = NotebookNode({})
dct = dct[key]
if keys[-1] not in dct:
dct[keys[-1]] = val
elif self.overwrite:
dct[keys[-1]] = val
for cell in nb.cells:
for keys, val in flatten(self.cell_defaults).items():
dct = cell.metadata
leaf_not_dict = False
for key in keys[:-1]:
if key not in dct:
dct[key] = NotebookNode({})
elif dct[key] is False and self.overwrite:
dct[key] = NotebookNode({})
elif dct[key] is True:
dct[key] = NotebookNode({})
elif not hasattr(dct[key], "items"):
leaf_not_dict = True
break
dct = dct[key]
if leaf_not_dict:
pass
elif keys[-1] not in dct:
dct[keys[-1]] = val
elif self.overwrite:
dct[keys[-1]] = val
return nb, resources
def edit_notebook(self, nb):
"""
Inject the code needed to setup and shutdown spark and sc magic variables.
"""
from nbformat.notebooknode import NotebookNode
from textwrap import dedent
preamble_node = NotebookNode(cell_type="code",
source=dedent("""
from pyspark.sql import SparkSession
spark = SparkSession.builder.appName("NotebookTestSuite").master("local[*]").getOrCreate()
globals()["spark"] = spark
globals()["sc"] = spark.sparkContext
"""))
epilogue_node = NotebookNode(cell_type="code",
source=dedent("""
try:
spark.stop()
except:
pass
"""))
nb.cells.insert(0, preamble_node)
nb.cells.append(epilogue_node)
return nb
def ensure_title(self, licence, authors, logo_path):
"""
make sure the first cell is a author + licence cell
"""
# the title cell has 3 parts that are equidistant
# xxx it looks like this <style> tag somehow gets
# trimmed away when rendered inside of edx
# so I had to add it in nbhosting's custom.css as well
def title_cell(licence, authors, logo_path):
cell = ''
cell += f'<div class="licence">\n'
cell += f'<span>{licence}</span>\n'
if authors:
cell += f'<span>{" & ".join(authors)}</span>\n'
if logo_path:
cell += f'<span><img src="{logo_path}" /></span>\n'
cell += f'</div>'
return cell
# a bit rustic but good enough
def is_title_cell(cell):
# for legacy - notebooks tweaked with older versions
# of this tool, we want to consider first cells that have
# Licence as being our title cell as well
return cell['cell_type'] == 'markdown' \
and (cell['source'].find("title-slide") >= 0
or cell['source'].lower().find("licence") >= 0)
# when opened interactively and then saved again, this is how the result looks like
expected_title_cell = title_cell(licence, authors, logo_path)
title_lines = [line + "\n" for line in expected_title_cell.split("\n")]
# remove last \n
title_lines[-1] = title_lines[-1][:-1]
first_cell = self.cells()[0]
# cell.source is a list of strings
if is_title_cell(first_cell):
# licence cell already here, just overwrite contents to latest version
first_cell['source'] = title_lines
else:
self.cells().insert(
0,
NotebookNode({
"cell_type": "markdown",
"metadata": {},
"source": title_lines,
}))
def upgrade_cell_metadata(self, cell: NotebookNode) -> NotebookNode:
if 'nbgrader' not in cell.metadata:
return cell
if 'schema_version' not in cell.metadata['nbgrader']:
cell.metadata['nbgrader']['schema_version'] = 0
if cell.metadata['nbgrader']['schema_version'] == 0:
cell = self._upgrade_v0_to_v1(cell)
if 'nbgrader' not in cell.metadata:
return cell
self._remove_extra_keys(cell)
return cell
def preprocess_cell(self,
cell: NotebookNode,
resources: ResourcesDict,
cell_index: int
) -> Tuple[NotebookNode, ResourcesDict]:
if (self.lock_solution_cells or self.lock_grade_cells) and utils.is_solution(cell) and utils.is_grade(cell):
cell.metadata['deletable'] = False
elif self.lock_solution_cells and utils.is_solution(cell):
cell.metadata['deletable'] = False
elif self.lock_grade_cells and utils.is_grade(cell):
cell.metadata['deletable'] = False
cell.metadata['editable'] = False
elif self.lock_readonly_cells and utils.is_locked(cell):
cell.metadata['deletable'] = False
cell.metadata['editable'] = False
elif self.lock_all_cells:
cell.metadata['deletable'] = False
cell.metadata['editable'] = False
return cell, resources
def _remove_hidden_test_region(self, cell: NotebookNode) -> bool:
"""Find a region in the cell that is delimeted by
`self.begin_test_delimeter` and `self.end_test_delimeter` (e.g. ###
BEGIN HIDDEN TESTS and ### END HIDDEN TESTS). Remove that region
depending the cell type.
This modifies the cell in place, and then returns True if a
hidden test region was removed, and False otherwise.
"""
# pull out the cell input/source
lines = cell.source.split("\n")
new_lines = []
in_test = False
removed_test = False
for line in lines:
# begin the test area
if self.begin_test_delimeter in line:
# check to make sure this isn't a nested BEGIN HIDDEN TESTS
# region
if in_test:
raise RuntimeError(
"Encountered nested begin hidden tests statements")
in_test = True
removed_test = True
# end the solution area
elif self.end_test_delimeter in line:
in_test = False
# add lines as long as it's not in the hidden tests region
elif not in_test:
new_lines.append(line)
# we finished going through all the lines, but didn't find a
# matching END HIDDEN TESTS statment
if in_test:
raise RuntimeError("No end hidden tests statement found")
# replace the cell source
cell.source = "\n".join(new_lines)
return removed_test
def preprocess(
self, nb: NotebookNode,
resources: ResourcesDict) -> Tuple[NotebookNode, ResourcesDict]:
# pull information from the resources
notebook_id = resources['nbgrader']['notebook']
assignment_id = resources['nbgrader']['assignment']
db_url = resources['nbgrader']['db_url']
with Gradebook(db_url) as gb:
kernelspec = json.loads(
gb.find_notebook(notebook_id, assignment_id).kernelspec)
self.log.debug("Source notebook kernelspec: {}".format(kernelspec))
self.log.debug("Submitted notebook kernelspec: {}"
"".format(nb.metadata.get('kernelspec', None)))
if kernelspec:
self.log.debug("Overwriting submitted notebook kernelspec: {}"
"".format(kernelspec))
nb.metadata['kernelspec'] = kernelspec
return nb, resources
def run_cmd(self, cmd, kernel_name=None):
"""
Runs python command string.
"""
if _debugging:
logging.info('Running command: ' + cmd + ' using kernel: ' +
kernel_name)
notebook = nbformat.v4.new_notebook()
my_cell = nbformat.v4.new_code_cell(source=cmd)
notebook.cells = [my_cell]
if kernel_name:
notebook.metadata['kernelspec'] = {'name': kernel_name}
try:
self.executePreprocessor.preprocess(notebook,
{'metadata': {
'path': '.'
}})
if _debugging:
logging.info('Result notebook: ' +
nbformat.v4.writes_json(notebook))
if len(notebook.cells) < 1 or len(notebook.cells[0].outputs) < 1:
return None
return self.postprocess_output(notebook.cells[0].outputs)
except:
exc_type, exc_obj, exc_tb = sys.exc_info()
msg = None
if _debugging:
msg = '\n'.join(
traceback.format_exception_only(exc_type, exc_obj) +
traceback.format_tb(exc_tb))
else:
msg = '\n'.join(
traceback.format_exception_only(exc_type, exc_obj))
out = NotebookNode(output_type='error',
html=RClansiconv(msg + '\n'))
return [out]
def _limit_stream_output(self, cell: NotebookNode) -> NotebookNode:
if self.max_lines == -1 or cell.cell_type != "code":
return cell
length = 0
new_outputs = []
for output in cell.outputs:
if output.output_type == 'stream':
if length == self.max_lines:
continue
text = output.text.split("\n")
if (len(text) + length) > self.max_lines:
text = text[:(self.max_lines - length - 1)]
text.append("... Output truncated ...")
length += len(text)
output.text = "\n".join(text)
new_outputs.append(output)
cell.outputs = new_outputs
return cell
def on_cell_executed(self, **kwargs):
cell = kwargs['cell']
cell_index = kwargs['cell_index']
reply = kwargs['execute_reply']
if reply['content']['status'] == 'error':
error_recorded = False
for output in cell.outputs:
if output.output_type == 'error':
error_recorded = True
if not error_recorded:
# Occurs when
# IPython.core.interactiveshell.InteractiveShell.showtraceback
# = lambda *args, **kwargs : None
error_output = NotebookNode(output_type='error')
error_output.ename = reply['content']['ename']
error_output.evalue = reply['content']['evalue']
error_output.traceback = reply['content']['traceback']
if error_output.traceback == []:
error_output.traceback = ["ERROR: An error occurred while"
" showtraceback was disabled"]
cell.outputs.append(error_output)
def preprocess(self, nb, resources):
logger.info("extracting caption cells")
# extract captions
final_cells = []
captions = {}
for cell in nb.cells:
if hasattr(cell.metadata, "ipub"):
if hasattr(cell.metadata.ipub.get("equation", False), "get"):
if hasattr(
cell.metadata.ipub.equation.get(
"environment", False),
"startswith",
):
if cell.metadata.ipub.equation.environment.startswith(
"breqn"): # noqa: E501
if "ipub" not in nb.metadata:
nb.metadata["ipub"] = NotebookNode(
{"enable_breqn": True})
else:
nb.metadata.ipub["enable_breqn"] = True
if hasattr(cell.metadata.ipub, "caption"):
if cell.cell_type == "markdown":
capt = cell.source.split(r"\n")[0]
captions[cell.metadata.ipub.caption] = capt
continue
elif cell.cell_type == "code":
if not cell.outputs:
pass
elif "text/latex" in cell.outputs[0].get("data", {}):
capt = cell.outputs[0].data["text/latex"].split(
r"\n")[0]
captions[cell.metadata.ipub.caption] = capt
continue
elif "text/plain" in cell.outputs[0].get("data", {}):
capt = cell.outputs[0].data["text/plain"].split(
r"\n")[0]
captions[cell.metadata.ipub.caption] = capt
continue
final_cells.append(cell)
nb.cells = final_cells
# replace captions
for cell in nb.cells:
if hasattr(cell.metadata, "ipub"):
for key in cell.metadata.ipub:
if hasattr(cell.metadata.ipub[key], "label"):
if cell.metadata.ipub[key]["label"] in captions:
logger.debug("replacing caption for: {}".format(
cell.metadata.ipub[key]["label"]))
cell.metadata.ipub[key]["caption"] = captions[
cell.metadata.ipub[key]["label"]] # noqa: E501
# add float type/number prefix to caption, if required
if self.add_prefix:
if hasattr(cell.metadata.ipub[key], "caption"):
if hasattr(cell.metadata.ipub[key],
"caption_prefix"):
newcaption = (
cell.metadata.ipub[key].caption_prefix +
cell.metadata.ipub[key].caption)
cell.metadata.ipub[key].caption = newcaption
return nb, resources
def preprocess(self, nb, resources):
logging.info(
'creating slides based on markdown and existing slide tags')
latexdoc_tags = [
'code', 'error', 'table', 'equation', 'figure', 'text'
]
# break up titles
cells_in_slide = 0
header_levels = []
final_cells = FinalCells(self.header_slide)
for i, cell in enumerate(nb.cells):
# Make sure every cell has an ipub meta tag
cell.metadata.ipub = cell.metadata.get('ipub', NotebookNode())
if cell.metadata.ipub.get('ignore', False):
cell.metadata.ipyslides = 'skip'
final_cells.append(cell)
continue
if cell.metadata.ipub.get('slide', False) == 'notes':
cell.metadata.ipyslides = 'notes'
final_cells.append(cell)
continue
if not cell.cell_type == "markdown":
# TODO this doesn't test if the data is actually available to be output
if not any([
cell.metadata.ipub.get(typ, False)
for typ in latexdoc_tags
]):
cell.metadata.ipyslides = 'skip'
final_cells.append(cell)
continue
if cells_in_slide > self.max_cells and self.max_cells:
cell.metadata.ipyslides = 'verticalbreak_after'
cells_in_slide = 1
elif cell.metadata.ipub.get('slide', False) == 'new':
cell.metadata.ipyslides = 'verticalbreak_after'
cells_in_slide = 1
else:
cell.metadata.ipyslides = 'normal'
cells_in_slide += 1
final_cells.append(cell)
continue
nonheader_lines = []
for line in cell.source.split('\n'):
if is_header(line, 0) and self.autonumbering:
line, header_levels = number_title(line, header_levels[:])
if is_header(line, self.column_level):
if nonheader_lines and cell.metadata.ipub.get(
'slide', False):
if (cells_in_slide > self.max_cells and self.max_cells
) or cell.metadata.ipub.slide == 'new':
final_cells.mkdcell(nonheader_lines, cell.metadata,
'verticalbreak_after')
cells_in_slide = 1
else:
cells_in_slide += 1
final_cells.mkdcell(nonheader_lines, cell.metadata,
'normal')
current_lines = []
if self.header_slide:
final_cells.mkdcell(
[line], cell.metadata,
'horizontalbreak_after_plusvertical')
else:
final_cells.mkdcell([line], cell.metadata,
'horizontalbreak_after')
cells_in_slide = 1
elif is_header(line, self.row_level):
if nonheader_lines and cell.metadata.ipub.get(
'slide', False):
if (cells_in_slide > self.max_cells and self.max_cells
) or cell.metadata.ipub.slide == 'new':
final_cells.mkdcell(nonheader_lines, cell.metadata,
'verticalbreak_after')
cells_in_slide = 1
else:
cells_in_slide += 1
final_cells.mkdcell(nonheader_lines, cell.metadata,
'normal')
current_lines = []
final_cells.mkdcell([line], cell.metadata,
'verticalbreak_after')
cells_in_slide = 1
else:
nonheader_lines.append(line)
if nonheader_lines and cell.metadata.ipub.get('slide', False):
if (cells_in_slide > self.max_cells and
self.max_cells) or cell.metadata.ipub.slide == 'new':
final_cells.mkdcell(nonheader_lines, cell.metadata,
'verticalbreak_after')
cells_in_slide = 1
else:
cells_in_slide += 1
final_cells.mkdcell(nonheader_lines, cell.metadata,
'normal')
if not final_cells.finalize():
logging.warning('no cells available for slideshow')
nb.cells = final_cells.cells
return nb, resources
def preprocess(self, nb, resources):
logging.info(
"creating slides based on markdown and existing slide tags")
latexdoc_tags = [
"code", "error", "table", "equation", "figure", "text"
]
# break up titles
cells_in_slide = 0
final_cells = FinalCells(self.header_slide)
header_levels = []
try:
base_numbering = nb.metadata.toc.base_numbering
header_levels = list(
map(lambda x: int(x), base_numbering.split(".")))
header_levels[0] -= 1
logging.debug("base_numbering = " + base_numbering)
logging.debug("header_levels = " + str(header_levels))
except ValueError:
logging.warning("Invalid toc.base_numbering in notebook metadata")
except AttributeError:
logging.debug(
"No toc.base_numbering in notebook metadata; starting at 1")
for i, cell in enumerate(nb.cells):
# Make sure every cell has an ipub meta tag
cell.metadata.ipub = cell.metadata.get("ipub", NotebookNode())
if cell.metadata.ipub.get("ignore", False):
cell.metadata.ipyslides = "skip"
final_cells.append(cell)
continue
if cell.metadata.ipub.get("slide", False) == "notes":
cell.metadata.ipyslides = "notes"
final_cells.append(cell)
continue
if not cell.cell_type == "markdown":
# TODO this doesn't test if the data is actually available
# to be output
if not any([
cell.metadata.ipub.get(typ, False)
for typ in latexdoc_tags
]):
cell.metadata.ipyslides = "skip"
final_cells.append(cell)
continue
if cells_in_slide > self.max_cells and self.max_cells:
cell.metadata.ipyslides = "verticalbreak_after"
cells_in_slide = 1
elif cell.metadata.ipub.get("slide", False) == "new":
cell.metadata.ipyslides = "verticalbreak_after"
cells_in_slide = 1
else:
cell.metadata.ipyslides = "normal"
cells_in_slide += 1
final_cells.append(cell)
continue
nonheader_lines = []
for line in cell.source.split("\n"):
if is_header(line, 0) and self.autonumbering:
line, header_levels = number_title(line, header_levels[:])
if is_header(line, self.column_level):
if nonheader_lines and cell.metadata.ipub.get(
"slide", False):
if (cells_in_slide > self.max_cells and self.max_cells
) or cell.metadata.ipub.slide == "new":
final_cells.mkdcell(nonheader_lines, cell.metadata,
"verticalbreak_after")
cells_in_slide = 1
else:
cells_in_slide += 1
final_cells.mkdcell(nonheader_lines, cell.metadata,
"normal")
# current_lines = []
if self.header_slide:
final_cells.mkdcell(
[line], cell.metadata,
"horizontalbreak_after_plusvertical")
else:
final_cells.mkdcell([line], cell.metadata,
"horizontalbreak_after")
cells_in_slide = 1
elif is_header(line, self.row_level):
if nonheader_lines and cell.metadata.ipub.get(
"slide", False):
if (cells_in_slide > self.max_cells and self.max_cells
) or cell.metadata.ipub.slide == "new":
final_cells.mkdcell(nonheader_lines, cell.metadata,
"verticalbreak_after")
cells_in_slide = 1
else:
cells_in_slide += 1
final_cells.mkdcell(nonheader_lines, cell.metadata,
"normal")
# current_lines = []
final_cells.mkdcell([line], cell.metadata,
"verticalbreak_after")
cells_in_slide = 1
else:
nonheader_lines.append(line)
if nonheader_lines and cell.metadata.ipub.get("slide", False):
if (cells_in_slide > self.max_cells and
self.max_cells) or cell.metadata.ipub.slide == "new":
final_cells.mkdcell(nonheader_lines, cell.metadata,
"verticalbreak_after")
cells_in_slide = 1
else:
cells_in_slide += 1
final_cells.mkdcell(nonheader_lines, cell.metadata,
"normal")
if not final_cells.finalize():
logging.warning("no cells available for slideshow")
nb.cells = final_cells.cells
return nb, resources
def ensure_title(self, licence, authors, logo_path):
"""
make sure the first cell is a author + licence cell
"""
# the title cell has 3 parts that are equidistant
# xxx it looks like this <style> tag somehow gets
# trimmed away when rendered inside of edx
# so I had to add it in nbhosting's custom.css as well
title_style = '''<style>
div.title-slide {
width: 100%;
display: flex;
flex-direction: row; /* default value; can be omitted */
flex-wrap: nowrap; /* default value; can be omitted */
justify-content: space-between;
}
</style>
'''
title_format = '''<div class="title-slide">
<span style="float:left;">{licence}</span>
<span>{html_authors}</span>
<span>{html_image}</span>
</div>'''
title_image_format = '<img src="{logo_path}" style="display:inline" />'
html_image = "" if not logo_path else \
title_image_format.format(logo_path=logo_path)
# a bit rustic but good enough
def is_title_cell(cell):
# for legacy - notebooks tweaked with older versions
# of this tool, we want to consider first cells that have
# Licence as being our title cell as well
return cell['cell_type'] == 'markdown' \
and (cell['source'].find("title-slide") >= 0
or cell['source'].find("Licence") >= 0)
html_authors = "" if not authors \
else " & ".join(authors)
title_line = title_style.replace("\n", "") \
+ title_format.format(
licence=licence,
html_authors=html_authors,
html_image=html_image)
# when opened interactively and then saved again, this is how the result looks like
title_lines = [line + "\n" for line in title_line.split("\n")]
# remove last \n
title_lines[-1] = title_lines[-1][:-1]
first_cell = self.cells()[0]
# cell.source is a list of strings
if is_title_cell(first_cell):
# licence cell already here, just overwrite contents to latest version
first_cell['source'] = title_lines
else:
self.cells().insert(
0,
NotebookNode({
"cell_type": "markdown",
"metadata": {},
"source": title_lines,
}))