def setUp(self):
nbdir = self.notebook_dir
if not os.path.isdir(pjoin(nbdir, 'foo')):
subdir = pjoin(nbdir, 'foo')
os.mkdir(subdir)
# Make sure that we clean this up when we're done.
# By using addCleanup this will happen correctly even if we fail
# later in setUp.
@self.addCleanup
def cleanup_dir():
shutil.rmtree(subdir, ignore_errors=True)
nb = new_notebook()
nb.cells.append(new_markdown_cell(u'Created by test ³'))
cc1 = new_code_cell(source=u'print(2*6)')
cc1.outputs.append(new_output(output_type="stream", text=u'12'))
cc1.outputs.append(new_output(output_type="execute_result",
data={'image/png' : png_green_pixel},
execution_count=1,
))
nb.cells.append(cc1)
with io.open(pjoin(nbdir, 'foo', 'testnb.ipynb'), 'w',
encoding='utf-8') as f:
write(nb, f, version=4)
self.nbconvert_api = NbconvertAPI(self.request)
def test_pretty_print_code_cell():
cell = v4.new_code_cell(source='def foo():\n return 4',
execution_count=3,
outputs=[
v4.new_output('stream', name='stdout', text='some\ntext'),
v4.new_output('display_data', {'text/plain': 'hello display'}),
]
)
io = StringIO()
pp.pretty_print_value_at(cell, "/cells/0", "+", io)
text = io.getvalue()
lines = text.splitlines()
assert lines == [
'+code cell:',
'+ execution_count: 3',
'+ source:',
'+ def foo():',
'+ return 4',
'+ outputs:',
'+ output 0:',
'+ output_type: stream',
'+ name: stdout',
'+ text:',
'+ some',
'+ text',
'+ output 1:',
'+ output_type: display_data',
'+ data:',
'+ text/plain: hello display',
]
def test_present_code_cell():
cell = v4.new_code_cell(source='def foo()',
outputs=[
v4.new_output('stream', name='stdout', text='some\ntext'),
v4.new_output('display_data', {'text/plain': 'hello display'}),
]
)
lines = pp.present_value('+ ', cell)
assert lines[0] == ''
assert lines[1] == '+ code cell:'
def test_determine_grade_code_grade():
cell = create_grade_cell('print("test")', "code", "foo", 10)
cell.outputs = []
assert utils.determine_grade(cell) == (10, 10)
cell.outputs = [new_output('error', ename="NotImplementedError", evalue="", traceback=["error"])]
assert utils.determine_grade(cell) == (0, 10)
def _add_error(self, cell):
cell.outputs.append(new_output(
"error",
ename="Error",
evalue="oh noes, an error occurred!",
traceback=["oh noes, an error occurred!"]
))
return cell
def test_determine_grade_code_grade_and_solution():
cell = create_grade_and_solution_cell('test', "code", "foo", 10)
cell.metadata.nbgrader['checksum'] = utils.compute_checksum(cell)
cell.outputs = []
assert utils.determine_grade(cell) == (0, 10)
cell.outputs = [new_output('error', ename="NotImplementedError", evalue="", traceback=["error"])]
cell.source = 'test!'
assert utils.determine_grade(cell) == (None, 10)
def test_present_stream_output():
output = v4.new_output('stream', name='stdout', text='some\ntext')
lines = pp.present_value('+ ', output)
assert lines == [
'+ output_type: stream',
"+ name: stdout",
"+ text:",
"+ some",
"+ text",
]
def build_notebook(self):
"""Build a reveal slides notebook in memory for use with tests.
Overrides base in PreprocessorTestsBase"""
outputs = [nbformat.new_output(output_type='display_data',
data={'image/svg+xml':self.simple_svg})
]
cells=[nbformat.new_code_cell(source="", execution_count=1, outputs=outputs)]
return nbformat.new_notebook(cells=cells)
def test_output_sanitizing(self):
"""Test that outputs are also sanitized properly"""
preprocessor = self.build_preprocessor()
nb = self.build_notebook()
outputs = [
nbformat.new_output("display_data", data={
'text/plain': 'b',
'text/html': '<script>more evil</script>',
'text/css': '<style> * {display:none}</style>'
}),
nbformat.new_output('stream', name='stdout', text="wat"),
nbformat.new_output('stream', name='stdout', text="<script>Evil tag</script>")
]
nb.cells[0].outputs = outputs
res = self.build_resources()
nb, res = preprocessor(nb, res)
expected_output = [
{
'data': {
'text/html': '<script>more evil</script>',
'text/plain': 'b'
},
'metadata': {},
'output_type': 'display_data',
},
{
'name': 'stdout',
'output_type': 'stream',
'text': 'wat'
},
{
'name': 'stdout',
'output_type':
'stream', 'text': '<script>Evil tag</script>'
}
]
self.assertEqual(nb.cells[0].outputs, expected_output)
def test_present_display_data():
output = v4.new_output('display_data', {
'text/plain': 'text',
'image/png': b64text(1024),
})
lines = pp.present_value('+ ', output)
text = '\n'.join(lines)
assert 'output_type: display_data' in text
assert len(text) < 500
assert 'snip base64' in text
assert 'image/png' in text
assert "text/plain: text" in text
assert all(line.startswith('+ ') for line in lines if line)
def setUp(self):
nbdir = self.notebook_dir.name
if not os.path.isdir(pjoin(nbdir, 'foo')):
os.mkdir(pjoin(nbdir, 'foo'))
nb = new_notebook()
nb.cells.append(new_markdown_cell(u'Created by test ³'))
cc1 = new_code_cell(source=u'print(2*6)')
cc1.outputs.append(new_output(output_type="stream", text=u'12'))
cc1.outputs.append(new_output(output_type="execute_result",
data={'image/png' : png_green_pixel},
execution_count=1,
))
nb.cells.append(cc1)
with io.open(pjoin(nbdir, 'foo', 'testnb.ipynb'), 'w',
encoding='utf-8') as f:
write(nb, f, version=4)
self.nbconvert_api = NbconvertAPI(self.base_url())
def new_code_cell(self, node, index=None):
# Get the code cell input: the first child of the CodeCell block.
input_block = node.children[0]
assert input_block.name == 'CodeBlock'
cell = new_code_cell(input_block.children[0],
execution_count=self.execution_count,
)
# Next we need to add the outputs: the next children in the CodeCell.
for child in node.children[1:]:
# Outputs can be code blocks or Markdown paragraphs containing
# an image.
if child.name == 'CodeBlock':
# The output is a code block.
# What is the output's type? It depends on the code block's
# name. It can be: `stdout`, `stderr`, `result`.
output_type = child.lang or 'result'
assert output_type in ('stdout', 'stderr', 'result')
contents = child.children[0]
# NOTE: append new lines at the end of every line in stdout
# and stderr contents, to match with the Jupyter Notebook.
if output_type != 'result':
contents = _append_newlines(contents)
if output_type == 'result':
kwargs = dict(execution_count=self.execution_count,
data={'text/plain': contents})
# Output type to pass to nbformat.
output_type = 'execute_result'
elif output_type in ('stdout', 'stderr'):
# Standard output or error.
kwargs = dict(text=contents, name=output_type)
# Output type to pass to nbformat.
output_type = 'stream'
elif child.name == 'Para':
img = child.children[0]
assert img.name == 'Image'
fn = img.url
caption = self._md.write(img.children[0])
output_type = 'display_data'
data = {} # Dictionary {mimetype: data_buffer}.
# Infer the mime type of the file, from its filename and
# extension.
mime_type = guess_type(fn)[0]
assert mime_type # unknown extension: this shouldn't happen!
data[mime_type] = self._get_b64_resource(fn)
assert data[mime_type] # TODO
data['text/plain'] = caption
kwargs = dict(data=data)
output = new_output(output_type, **kwargs)
cell.outputs.append(output)
self.execution_count += 1
return cell
def build_notebook(self):
"""
Build a notebook to have metadata tags for cells, output_areas, and
individual outputs.
"""
notebook = super(TestTagRemove, self).build_notebook()
# Add a few empty cells
notebook.cells[0].outputs.extend(
[nbformat.new_output("display_data",
data={'text/plain': 'i'},
metadata={'tags': ["hide_one_output"]}
),
])
outputs_to_be_removed = [
nbformat.new_output("display_data",
data={'text/plain': "remove_my_output"}),
]
outputs_to_be_kept = [
nbformat.new_output("stream",
name="stdout",
text="remove_my_output",
),
]
notebook.cells.extend(
[nbformat.new_code_cell(source="display('remove_my_output')",
execution_count=2,
outputs=outputs_to_be_removed,
metadata={"tags": ["hide_all_outputs"]}),
nbformat.new_code_cell(source="print('remove this cell')",
execution_count=3,
outputs=outputs_to_be_kept,
metadata={"tags": ["hide_this_cell"]}),
]
)
return notebook
def test_save_incorrect_code(self, preprocessors, gradebook, resources):
"""Is a failing code cell correctly graded?"""
cell = create_grade_cell("hello", "code", "foo", 1)
cell.metadata.nbgrader['checksum'] = compute_checksum(cell)
cell.outputs = [new_output('error', ename="NotImplementedError", evalue="", traceback=["error"])]
nb = new_notebook()
nb.cells.append(cell)
preprocessors[0].preprocess(nb, resources)
gradebook.add_submission("ps0", "bar")
preprocessors[1].preprocess(nb, resources)
preprocessors[2].preprocess(nb, resources)
assert cell.metadata.nbgrader['score'] == 0
assert cell.metadata.nbgrader['points'] == 1
assert 'comment' not in cell.metadata.nbgrader
def build_notebook(self):
"""Build a notebook in memory for use with preprocessor tests"""
outputs = [
nbformat.new_output("stream", name="stdout", text="a"),
nbformat.new_output("display_data", data={'text/plain': 'b'}),
nbformat.new_output("stream", name="stdout", text="c"),
nbformat.new_output("stream", name="stdout", text="d"),
nbformat.new_output("stream", name="stderr", text="e"),
nbformat.new_output("stream", name="stderr", text="f"),
nbformat.new_output("display_data", data={'image/png': 'Zw=='}), # g
nbformat.new_output("display_data", data={'application/pdf': 'aA=='}), # h
]
cells=[nbformat.new_code_cell(source="$ e $", execution_count=1, outputs=outputs),
nbformat.new_markdown_cell(source="$ e $")]
return nbformat.new_notebook(cells=cells)
def test_coalesce_sequenced_streams(self):
"""Can the coalesce streams preprocessor merge a sequence of streams?"""
outputs = [nbformat.new_output(output_type="stream", name="stdout", text="0"),
nbformat.new_output(output_type="stream", name="stdout", text="1"),
nbformat.new_output(output_type="stream", name="stdout", text="2"),
nbformat.new_output(output_type="stream", name="stdout", text="3"),
nbformat.new_output(output_type="stream", name="stdout", text="4"),
nbformat.new_output(output_type="stream", name="stdout", text="5"),
nbformat.new_output(output_type="stream", name="stdout", text="6"),
nbformat.new_output(output_type="stream", name="stdout", text="7")]
cells=[nbformat.new_code_cell(source="# None", execution_count=1,outputs=outputs)]
nb = nbformat.new_notebook(cells=cells)
res = self.build_resources()
nb, res = coalesce_streams(nb, res)
outputs = nb.cells[0].outputs
self.assertEqual(outputs[0].text, u'01234567')
def test_javascript_output(self):
nb = v4.new_notebook(
cells=[
v4.new_code_cell(
outputs=[v4.new_output(
output_type='display_data',
data={
'application/javascript': "javascript_output();"
}
)]
)
]
)
(output, resources) = HTMLExporter(template_file='basic').from_notebook_node(nb)
self.assertIn('javascript_output', output)
def setup_class(cls):
"""Make a test notebook. Borrowed from nbconvert test. Assumes the class
teardown will clean it up in the end."""
super(BundleAPITest, cls).setup_class()
nbdir = cls.notebook_dir.name
nb = new_notebook()
nb.cells.append(new_markdown_cell(u'Created by test'))
cc1 = new_code_cell(source=u'print(2*6)')
cc1.outputs.append(new_output(output_type="stream", text=u'12'))
nb.cells.append(cc1)
with io.open(pjoin(nbdir, 'testnb.ipynb'), 'w',
encoding='utf-8') as f:
write(nb, f, version=4)
def test_pretty_print_stream_output():
output = v4.new_output('stream', name='stdout', text='some\ntext')
io = StringIO()
pp.pretty_print_value_at(output, "/cells/2/outputs/3", "+", io)
text = io.getvalue()
lines = text.splitlines()
assert lines == [
'+output:',
'+ output_type: stream',
"+ name: stdout",
"+ text:",
"+ some",
"+ text",
]
def build_notebook(self):
"""Build a reveal slides notebook in memory for use with tests.
Overrides base in PreprocessorTestsBase"""
outputs = [nbformat.new_output(output_type="stream", name="stdout", text="a")]
slide_metadata = {'slideshow' : {'slide_type': 'slide'}}
subslide_metadata = {'slideshow' : {'slide_type': 'subslide'}}
cells=[nbformat.new_code_cell(source="", execution_count=1, outputs=outputs),
nbformat.new_markdown_cell(source="", metadata=slide_metadata),
nbformat.new_code_cell(source="", execution_count=2, outputs=outputs),
nbformat.new_markdown_cell(source="", metadata=slide_metadata),
nbformat.new_markdown_cell(source="", metadata=subslide_metadata)]
return nbformat.new_notebook(cells=cells)
def test_pretty_print_display_data():
output = v4.new_output('display_data', {
'text/plain': 'text',
'image/png': b64text(1024),
})
io = StringIO()
pp.pretty_print_value_at(output, "/cells/1/outputs/2", "+", io)
text = io.getvalue()
lines = text.splitlines()
assert 'output_type: display_data' in text
assert len(text) < 500
assert 'snip base64' in text
assert 'image/png' in text
assert "text/plain: text" in text
assert all(line.startswith('+') for line in lines if line)
def test_grade_incorrect_code(self, preprocessors, gradebook, resources):
"""Is a failing code cell correctly graded?"""
cell = create_grade_cell("hello", "code", "foo", 1)
cell.metadata.nbgrader['checksum'] = compute_checksum(cell)
cell.outputs = [new_output('error', ename="NotImplementedError", evalue="", traceback=["error"])]
nb = new_notebook()
nb.cells.append(cell)
preprocessors[0].preprocess(nb, resources)
gradebook.add_submission("ps0", "bar")
preprocessors[1].preprocess(nb, resources)
grade_cell = gradebook.find_grade("foo", "test", "ps0", "bar")
assert grade_cell.score == 0
assert grade_cell.max_score == 1
assert grade_cell.auto_score == 0
assert grade_cell.manual_score == None
assert not grade_cell.needs_manual_grade
def test_contents_manager(self):
"make sure ContentsManager returns right files (ipynb, bin, txt)."
nbdir = self.notebook_dir.name
base = self.base_url()
nb = new_notebook(
cells=[
new_markdown_cell(u'Created by test ³'),
new_code_cell("print(2*6)", outputs=[
new_output("stream", text="12"),
])
]
)
with io.open(pjoin(nbdir, 'testnb.ipynb'), 'w',
encoding='utf-8') as f:
write(nb, f, version=4)
with io.open(pjoin(nbdir, 'test.bin'), 'wb') as f:
f.write(b'\xff' + os.urandom(5))
f.close()
with io.open(pjoin(nbdir, 'test.txt'), 'w') as f:
f.write(u'foobar')
f.close()
r = requests.get(url_path_join(base, 'files', 'testnb.ipynb'))
self.assertEqual(r.status_code, 200)
self.assertIn('print(2*6)', r.text)
json.loads(r.text)
r = requests.get(url_path_join(base, 'files', 'test.bin'))
self.assertEqual(r.status_code, 200)
self.assertEqual(r.headers['content-type'], 'application/octet-stream')
self.assertEqual(r.content[:1], b'\xff')
self.assertEqual(len(r.content), 6)
r = requests.get(url_path_join(base, 'files', 'test.txt'))
self.assertEqual(r.status_code, 200)
self.assertEqual(r.headers['content-type'], 'text/plain')
self.assertEqual(r.text, 'foobar')
def test_contents_manager(self):
"make sure ContentsManager returns right files (ipynb, bin, txt)."
nbdir = self.notebook_dir.name
nb = new_notebook(
cells=[
new_markdown_cell(u'Created by test ³'),
new_code_cell("print(2*6)", outputs=[
new_output("stream", text="12"),
])
]
)
with io.open(pjoin(nbdir, 'testnb.ipynb'), 'w',
encoding='utf-8') as f:
write(nb, f, version=4)
with io.open(pjoin(nbdir, 'test.bin'), 'wb') as f:
f.write(b'\xff' + os.urandom(5))
f.close()
with io.open(pjoin(nbdir, 'test.txt'), 'w') as f:
f.write(u'foobar')
f.close()
r = self.request('GET', 'files/testnb.ipynb')
self.assertEqual(r.status_code, 200)
self.assertIn('print(2*6)', r.text)
json.loads(r.text)
r = self.request('GET', 'files/test.bin')
self.assertEqual(r.status_code, 200)
self.assertEqual(r.headers['content-type'], 'application/octet-stream')
self.assertEqual(r.content[:1], b'\xff')
self.assertEqual(len(r.content), 6)
r = self.request('GET', 'files/test.txt')
self.assertEqual(r.status_code, 200)
self.assertEqual(r.headers['content-type'], 'text/plain')
self.assertEqual(r.text, 'foobar')
def test_coalesce_replace_streams(self):
"""Are \\r characters handled?"""
outputs = [nbformat.new_output(output_type="stream", name="stdout", text="z"),
nbformat.new_output(output_type="stream", name="stdout", text="\ra"),
nbformat.new_output(output_type="stream", name="stdout", text="\nz\rb"),
nbformat.new_output(output_type="stream", name="stdout", text="\nz"),
nbformat.new_output(output_type="stream", name="stdout", text="\rc\n"),
nbformat.new_output(output_type="stream", name="stdout", text="z\rz\rd")]
cells=[nbformat.new_code_cell(source="# None", execution_count=1,outputs=outputs)]
nb = nbformat.new_notebook(cells=cells)
res = self.build_resources()
nb, res = coalesce_streams(nb, res)
outputs = nb.cells[0].outputs
self.assertEqual(outputs[0].text, u'a\nb\nc\nd')
def cells(self):
for cell in self.nb.cells:
if isinstance(cell, TextCell):
yield new_markdown_cell(source=cell.input, )
elif isinstance(cell, ComputeCell):
# SageNB's counting starts at 0 but IPython starts at 1
count = cell.index + 1
yield new_code_cell(source=cell.ipython_input(),
execution_count=count,
outputs=[
new_output(
output_type=u'execute_result',
data={
'text/plain':
cell.plain_text_output(),
},
execution_count=count,
)
])
else:
log.critical('unknown cell: {0}'.format(cell))
def test_grade_incorrect_code(self, preprocessors, gradebook, resources):
"""Is a failing code cell correctly graded?"""
cell = create_grade_cell("hello", "code", "foo", 1)
cell.metadata.nbgrader['checksum'] = compute_checksum(cell)
cell.outputs = [
new_output('error',
ename="NotImplementedError",
evalue="",
traceback=["error"])
]
nb = new_notebook()
nb.cells.append(cell)
preprocessors[0].preprocess(nb, resources)
gradebook.add_submission("ps0", "bar")
preprocessors[1].preprocess(nb, resources)
grade_cell = gradebook.find_grade("foo", "test", "ps0", "bar")
assert grade_cell.score == 0
assert grade_cell.max_score == 1
assert grade_cell.auto_score == 0
assert grade_cell.manual_score == None
assert not grade_cell.needs_manual_grade
def cells(self):
for cell in self.nb.cells:
if isinstance(cell, TextCell):
yield new_markdown_cell(
source=cell.input,
)
elif isinstance(cell, ComputeCell):
yield new_code_cell(
source=cell.input,
execution_count=cell.index,
outputs=[
new_output(
output_type=u'execute_result',
data={
'text/plain': cell.output,
},
execution_count=cell.index,
)
]
)
else:
log.critical('unknown cell: {0}'.format(cell))
def build_notebook(self):
"""Build a notebook in memory for use with preprocessor tests"""
outputs = [
nbformat.new_output("stream", name="stdout", text="a"),
nbformat.new_output("display_data", data={'text/plain': 'b'}),
nbformat.new_output("stream", name="stdout", text="c"),
nbformat.new_output("stream", name="stdout", text="d"),
nbformat.new_output("stream", name="stderr", text="e"),
nbformat.new_output("stream", name="stderr", text="f"),
nbformat.new_output("display_data", data={'image/png':
'Zw=='}), # g
nbformat.new_output("display_data",
data={'application/pdf': 'aA=='}), # h
]
cells = [
nbformat.new_code_cell(source="$ e $",
execution_count=1,
outputs=outputs),
nbformat.new_markdown_cell(source="$ e $")
]
return nbformat.new_notebook(cells=cells)
def build_notebook(self):
"""Build a reveal slides notebook in memory for use with tests.
Overrides base in PreprocessorTestsBase"""
outputs = [
nbformat.new_output(output_type="stream", name="stdout", text="a")
]
slide_metadata = {'slideshow': {'slide_type': 'slide'}}
subslide_metadata = {'slideshow': {'slide_type': 'subslide'}}
cells = [
nbformat.new_code_cell(source="",
execution_count=1,
outputs=outputs),
nbformat.new_markdown_cell(source="", metadata=slide_metadata),
nbformat.new_code_cell(source="",
execution_count=2,
outputs=outputs),
nbformat.new_markdown_cell(source="", metadata=slide_metadata),
nbformat.new_markdown_cell(source="", metadata=subslide_metadata)
]
return nbformat.new_notebook(cells=cells)
# "image/svg+xml": [
# "PD94bCjwvc3ZnPgo="
# ]
# }
# },
# "cell_type": "markdown",
# "metadata": {},
# "source": [
# "\n",
# "\n",
# ""
# ]
# },
output = new_output('execute_result',
data={'text/plain': '2'},
execution_count=1)
cell.outputs.append(output)
####################################################################################################
source = '''
import numpy as np
import matplotlib.pyplot as plt
figure = plt.figure(1, (20, 10))
x = np.arange(1, 10, .1)
y = np.sin(x)
plt.plot(x, y)
plt.show()
'''
def build_notebook(self, with_json_outputs=False):
"""Build a notebook in memory for use with preprocessor tests"""
outputs = [
nbformat.new_output("stream", name="stdout", text="a"),
nbformat.new_output("display_data", data={"text/plain": "b"}),
nbformat.new_output("stream", name="stdout", text="c"),
nbformat.new_output("stream", name="stdout", text="d"),
nbformat.new_output("stream", name="stderr", text="e"),
nbformat.new_output("stream", name="stderr", text="f"),
nbformat.new_output("display_data", data={"image/png":
"Zw=="}), # g
nbformat.new_output("display_data",
data={"application/pdf": "aA=="}), # h
]
if with_json_outputs:
outputs.extend([
nbformat.new_output("display_data",
data={"application/json": [1, 2, 3]}), # j
nbformat.new_output(
"display_data",
data={"application/json": {
"a": 1,
"c": {
"b": 2
}
}}), # k
nbformat.new_output("display_data",
data={"application/json": "abc"}), # l
nbformat.new_output("display_data",
data={"application/json": 15.03}), # m
])
cells = [
nbformat.new_code_cell(source="$ e $",
execution_count=1,
outputs=outputs),
nbformat.new_markdown_cell(source="$ e $"),
]
return nbformat.new_notebook(cells=cells)
def add_code_cell(self, nb):
output = nbformat.new_output("display_data", {'application/javascript': "alert('hi');"})
cell = nbformat.new_code_cell("print('hi')", outputs=[output])
nb.cells.append(cell)
def save_history(self):
"""This saves all cell executions in the current session as a new notebook"""
try:
from nbformat import write, v4, validator
except ImportError:
logger.error("Run pip install nbformat to save notebook history")
return
# TODO: some tests didn't patch ipython properly?
if self.shell is None:
return
cells = []
hist = list(self.shell.history_manager.get_range(output=True))
if len(hist) <= 1 or not self.settings.save_code:
logger.info("not saving jupyter history")
return
try:
for _, execution_count, exc in hist:
if exc[1]:
# TODO: capture stderr?
outputs = [
v4.new_output(output_type="stream",
name="stdout",
text=exc[1])
]
else:
outputs = []
if self.outputs.get(execution_count):
for out in self.outputs[execution_count]:
outputs.append(
v4.new_output(
output_type="display_data",
data=out["data"],
metadata=out["metadata"] or {},
))
cells.append(
v4.new_code_cell(execution_count=execution_count,
source=exc[0],
outputs=outputs))
if hasattr(self.shell, "kernel"):
language_info = self.shell.kernel.language_info
else:
language_info = {"name": "python", "version": sys.version}
logger.info("saving %i cells to _session_history.ipynb",
len(cells))
nb = v4.new_notebook(
cells=cells,
metadata={
"kernelspec": {
"display_name": "Python %i" % sys.version_info[0],
"name": "python%i" % sys.version_info[0],
"language": "python",
},
"language_info": language_info,
},
)
state_path = os.path.join("code", "_session_history.ipynb")
wandb.run._set_config_wandb("session_history", state_path)
wandb.util.mkdir_exists_ok(os.path.join(wandb.run.dir, "code"))
with open(
os.path.join(self.settings._tmp_code_dir,
"_session_history.ipynb"),
"w",
encoding="utf-8",
) as f:
write(nb, f, version=4)
with open(os.path.join(wandb.run.dir, state_path),
"w",
encoding="utf-8") as f:
write(nb, f, version=4)
except (OSError, validator.NotebookValidationError) as e:
logger.error("Unable to save ipython session history:\n%s", e)
pass
description)
code = open("{}.py".format(filename), encoding="utf-8").read()
code = re.sub(
'\.writeEPSfile\(("[a-z]+")?\)\n.*writePDFfile\(("[a-z]+")?\)\n.*writeSVGfile\(("[a-z]+")?\)\n',
"", code)
nb = nbf.new_notebook()
cells = []
cells.append(nbf.new_markdown_cell(source="# " + title))
cells.append(
nbf.new_code_cell(source=code,
execution_count=1,
outputs=[
nbf.new_output(
output_type=u'execute_result',
execution_count=1,
data={
'image/png':
base64.encodebytes(
open("{}.png".format(filename),
"rb").read()).decode("ascii"),
'image/svg+xml':
open("{}.svg".format(filename),
"r",
encoding="utf-8").read()
})
]))
cells.append(nbf.new_markdown_cell(source=description))
nb = nbf.new_notebook(cells=cells, metadata={'language': 'python'})
open("{}.ipynb".format(filename), "w").write(nbf.writes(nb))
def test_get_notebook_outputs(self):
output = new_output(
output_type='display_data', data={}, metadata={'papermill': {'name': 'test'}}
)
nb = new_notebook(cells=[new_code_cell('test', outputs=[output])])
assert _get_notebook_outputs(nb) == {'test': output}
from nbformat.v4 import new_notebook, new_code_cell, new_output, new_markdown_cell
import recombinecm
w_output_1 = new_notebook(cells = [
new_code_cell("print('foo')",
execution_count=1,
outputs = [
new_output('stream', text='foo\n'),
]),
new_markdown_cell('This is some *markdown*'),
new_code_cell("a=1\na",
execution_count=2,
outputs = [
new_output('execute_result',
{'text/plain': '1'},
execution_count=1)
]),
new_code_cell("print('fob')",
execution_count=4,
outputs = [
new_output('stream', text='fob\n'),
]),
])
clean_1 = new_notebook(cells = [
new_code_cell("print('foo')"),
new_markdown_cell('This is some *markdown*'),
new_code_cell("a=1\na"),
new_code_cell("print('fob')"),
])
def save_history(self):
"""This saves all cell executions in the current session as a new notebook"""
try:
from nbformat import write, v4, validator
except ImportError:
logger.error("Run pip install nbformat to save notebook history")
return
# TODO: some tests didn't patch ipython properly?
if self.shell == None:
return
cells = []
hist = list(self.shell.history_manager.get_range(output=True))
if len(hist) <= 1 or not env.should_save_code():
return
try:
for session, execution_count, exc in hist:
if exc[1]:
# TODO: capture stderr?
outputs = [
v4.new_output(output_type="stream",
name="stdout",
text=exc[1])
]
else:
outputs = []
if self.outputs.get(execution_count):
for out in self.outputs[execution_count]:
outputs.append(
v4.new_output(output_type="display_data",
data=out["data"],
metadata=out["metadata"] or {}))
cells.append(
v4.new_code_cell(execution_count=execution_count,
source=exc[0],
outputs=outputs))
if hasattr(self.shell, "kernel"):
language_info = self.shell.kernel.language_info
else:
language_info = {'name': "python", "version": sys.version}
nb = v4.new_notebook(cells=cells,
metadata={
'kernelspec': {
'display_name':
'Python %i' % sys.version_info[0],
'name':
'python%i' % sys.version_info[0],
'language':
'python'
},
'language_info': language_info
})
state_path = os.path.join("code", "_session_history.ipynb")
wandb.run.config._set_wandb("session_history", state_path)
wandb.run.config.persist()
wandb.util.mkdir_exists_ok(os.path.join(wandb.run.dir, "code"))
with open(os.path.join(wandb.run.dir, state_path),
'w',
encoding='utf-8') as f:
write(nb, f, version=4)
except (OSError, validator.NotebookValidationError) as e:
logger.error("Unable to save ipython session history:\n%s", e)
pass
def test_empty_metadata(self):
output = new_output(output_type='display_data', data={}, metadata={})
nb = new_notebook(cells=[new_code_cell('test', outputs=[output])])
assert _get_notebook_outputs(nb) == {}
def new_code_cell(self, node, index=None):
# Get the code cell input: the first child of the CodeCell block.
input_block = node.children[0]
assert input_block.name == 'CodeBlock'
cell = new_code_cell(input_block.children[0],
execution_count=self.execution_count,
)
# Next we need to add the outputs: the next children in the CodeCell.
for child in node.children[1:]:
# Outputs can be code blocks or Markdown paragraphs containing
# an image.
if child.name == 'CodeBlock':
# The output is a code block.
# What is the output's type? It depends on the code block's
# name. It can be: `stdout`, `stderr`, `result`.
output_type = child.lang or '{output:result}'
assert output_type.startswith('{output')
contents = child.children[0]
# NOTE: append new lines at the end of every line in stdout
# and stderr contents, to match with the Jupyter Notebook.
if output_type != '{output:result}':
contents = _append_newlines(contents)
if output_type == '{output:result}':
kwargs = dict(execution_count=self.execution_count,
data={'text/plain': contents})
# Output type to pass to nbformat.
output_type = 'execute_result'
elif output_type in ('{output:stdout}', '{output:stderr}'):
# Standard output or error.
# NOTE: strip {output } and only keep stdout/stderr in name.
kwargs = dict(text=contents, name=output_type[8:-1])
# Output type to pass to nbformat.
output_type = 'stream'
elif child.name == 'Para':
img = child.children[0]
assert img.name == 'Image'
fn = img.url
caption = self._md.write(img.children[0])
output_type = 'display_data'
data = {} # Dictionary {mimetype: data_buffer}.
# Infer the mime type of the file, from its filename and
# extension.
mime_type = guess_type(fn)[0]
assert mime_type # unknown extension: this shouldn't happen!
# Get the resource data.
if self._dir_path:
image_path = op.join(self._dir_path, fn)
# The image path could be absolute.
elif op.isabs(fn):
image_path = fn
else: # pragma: no cover
image_path = None
# If the image path exists, open it.
if image_path and op.exists(image_path):
with open(image_path, 'rb') as f:
data[mime_type] = _get_b64_resource(f.read())
else: # pragma: no cover
logger.debug("File `%s` doesn't exist.", image_path)
# Save the caption in the output text.
data['text/plain'] = caption
# Save the caption in the cell metadata too, so that it is not lost when
# executing the notebook.
if 'podoc' not in cell.metadata:
cell.metadata['podoc'] = {}
cell.metadata['podoc'].update({'output_text': caption})
kwargs = dict(data=data)
assert not output_type.startswith('{output')
output = new_output(output_type, **kwargs)
cell.outputs.append(output)
self.execution_count += 1
return cell
def add_code_cell(notebook):
output = nbformat.new_output("display_data",
{"application/javascript": "alert('hi');"})
cell = nbformat.new_code_cell("print('hi')", outputs=[output])
notebook.cells.append(cell)
def add_invalid_cell(notebook):
output = nbformat.new_output("display_data",
{"application/javascript": "alert('hi');"})
cell = nbformat.new_code_cell("print('hi')", outputs=[output])
cell.pop("source") # Remove source to invaliate
notebook.cells.append(cell)
def test_trim_output(self):
trim_output = IPyTail()._trim_output
output = new_output('stream', text=['1', '2', '3', '4', '5', '6'])
output2 = trim_output(output, 4)
assert output2 == new_output('stream',
text=['1', '2', '...\n', '5', '6'])
def test_papermill_metadata_but_empty(self):
output = new_output(output_type='display_data',
metadata={'papermill': {}})
nb = new_notebook(cells=[new_code_cell('test', outputs=[output])])
assert _get_notebook_outputs(nb) == {}
def build_notebook(self, with_json_outputs=False):
"""Build a notebook in memory for use with preprocessor tests"""
outputs = [
nbformat.new_output("stream", name="stdout", text="a"),
nbformat.new_output("display_data", data={'text/plain': 'b'}),
nbformat.new_output("stream", name="stdout", text="c"),
nbformat.new_output("stream", name="stdout", text="d"),
nbformat.new_output("stream", name="stderr", text="e"),
nbformat.new_output("stream", name="stderr", text="f"),
nbformat.new_output("display_data", data={'image/png':
'Zw=='}), # g
nbformat.new_output("display_data",
data={'application/pdf': 'aA=='}), # h
]
if with_json_outputs:
outputs.extend([
nbformat.new_output("display_data",
data={'application/json': [1, 2, 3]}), # j
nbformat.new_output(
"display_data",
data={'application/json': {
'a': 1,
'c': {
'b': 2
}
}}), # k
nbformat.new_output("display_data",
data={'application/json': 'abc'}), # l
nbformat.new_output("display_data",
data={'application/json': 15.03}), # m
])
cells = [
nbformat.new_code_cell(source="$ e $",
execution_count=1,
outputs=outputs),
nbformat.new_markdown_cell(source="$ e $")
]
return nbformat.new_notebook(cells=cells)
def build_notebook(self, with_json_outputs=False):
"""Build a notebook in memory for use with preprocessor tests"""
outputs = [
nbformat.new_output("stream", name="stdout", text="a"),
nbformat.new_output("display_data", data={'text/plain': 'b'}),
nbformat.new_output("stream", name="stdout", text="c"),
nbformat.new_output("stream", name="stdout", text="d"),
nbformat.new_output("stream", name="stderr", text="e"),
nbformat.new_output("stream", name="stderr", text="f"),
nbformat.new_output("display_data", data={'image/png': 'Zw=='}), # g
nbformat.new_output("display_data", data={'application/pdf': 'aA=='}), # h
]
if with_json_outputs:
outputs.extend([
nbformat.new_output(
"display_data", data={'application/json': [1, 2, 3]}
), # j
nbformat.new_output(
"display_data", data={'application/json': {'a': 1, 'c': {'b': 2}}}
), # k
nbformat.new_output(
"display_data", data={'application/json': 'abc'}
), # l
nbformat.new_output(
"display_data", data={'application/json': 15.03}
), # m
])
cells=[nbformat.new_code_cell(source="$ e $", execution_count=1, outputs=outputs),
nbformat.new_markdown_cell(source="$ e $")]
return nbformat.new_notebook(cells=cells)
def test_metadata_but_empty_content():
output = new_output(output_type="display_data", metadata={"scrapbook": {}})
raw_nb = new_notebook(cells=[new_code_cell("test", outputs=[output])])
nb = Notebook(raw_nb)
assert nb.scraps == collections.OrderedDict()
