def lines_to_notebook(lines, name=None):
"""
Convert the lines of an m file into an IPython notebook
Parameters
----------
lines : list
A list of strings. Each element is a line in the m file
Returns
-------
notebook : an IPython NotebookNode class instance, containing the
information required to create a file
"""
source = []
md = np.empty(len(lines), dtype=object)
new_cell = np.empty(len(lines), dtype=object)
for idx, l in enumerate(lines):
new_cell[idx], md[idx], this_source = format_line(l)
# Transitions between markdown and code and vice-versa merit a new
# cell, even if no newline, or "%%" is found. Make sure not to do this
# check for the very first line!
if idx>1 and not new_cell[idx]:
if md[idx] != md[idx-1]:
new_cell[idx] = True
source.append(this_source)
# This defines the breaking points between cells:
new_cell_idx = np.hstack([np.where(new_cell)[0], -1])
# Listify the sources:
cell_source = [source[new_cell_idx[i]:new_cell_idx[i+1]]
for i in range(len(new_cell_idx)-1)]
cell_md = [md[new_cell_idx[i]] for i in range(len(new_cell_idx)-1)]
cells = []
# Append the notebook with loading matlab magic extension
notebook_head = "import pymatbridge as pymat\n" + "ip = get_ipython()\n" \
+ "pymat.load_ipython_extension(ip)"
cells.append(nbformat.new_code_cell(notebook_head, language='python'))
for cell_idx, cell_s in enumerate(cell_source):
if cell_md[cell_idx]:
cells.append(nbformat.new_text_cell('markdown', cell_s))
else:
cell_s.insert(0, '%%matlab\n')
cells.append(nbformat.new_code_cell(cell_s, language='matlab'))
ws = nbformat.new_worksheet(cells=cells)
notebook = nbformat.new_notebook(metadata=nbformat.new_metadata(),
worksheets=[ws])
return notebook
def build_notebook(self):
"""Build a reveal slides notebook in memory for use with tests.
Overrides base in TransformerTestsBase"""
outputs = [
nbformat.new_output(output_type="stream",
stream="stdout",
output_text="a")
]
slide_metadata = {'slideshow': {'slide_type': 'slide'}}
subslide_metadata = {'slideshow': {'slide_type': 'subslide'}}
cells = [
nbformat.new_code_cell(input="", prompt_number=1, outputs=outputs),
nbformat.new_text_cell('markdown',
source="",
metadata=slide_metadata),
nbformat.new_code_cell(input="", prompt_number=2, outputs=outputs),
nbformat.new_text_cell('markdown',
source="",
metadata=slide_metadata),
nbformat.new_text_cell('markdown',
source="",
metadata=subslide_metadata)
]
worksheets = [nbformat.new_worksheet(name="worksheet1", cells=cells)]
return nbformat.new_notebook(name="notebook1", worksheets=worksheets)
def convert(py_file, ipynb_file):
cells = []
imports = "" # needed for each file
for cell, line_type, line in parse_blocks(py_file):
if line_type == file_type:
# write cell to file
try:
fname = line.split()[1]
except:
raise Exception(
"Markdown for file output must be the "
+ "following format\n#to_file filename.py: "
+ "{}\n".format(line)
)
sys.exit(0)
with open(fname, "w") as f:
f.write(cell)
new_cell = "%run {}".format(fname)
ipynb_cell = nbf.new_code_cell(new_cell)
cells.append(ipynb_cell)
else:
# convert cell to ipynb cell
ipynb_cell = nbf.new_code_cell(cell)
cells.append(ipynb_cell)
# create new notebook
nb = nbf.new_notebook()
nb["worksheets"].append(nbf.new_worksheet(cells=cells))
with open(ipynb_file, "w") as f:
nbf.write(nb, f, "ipynb")
def test_match_tests_double_test(self):
"""Is an error raised when a test id is used twice?"""
cell1 = new_code_cell()
cell1.metadata = dict(assignment=dict(cell_type="grade", id="foo", points=""))
cell2 = new_code_cell()
cell2.metadata = dict(assignment=dict(cell_type="autograder", id="foo_test"))
cell3 = new_text_cell("markdown")
cell3.metadata = dict(assignment=dict(cell_type="autograder", id="foo_test"))
cells = [cell1, cell2, cell3]
assert_raises(RuntimeError, self.preprocessor._match_tests, cells)
def test_basic_notebook_merge():
notebook = nbformat.new_notebook()
code_cell = nbformat.new_code_cell(input=['a', 'b'])
notebook['worksheets'] = [{'cells': [code_cell]}]
notebook2 = nbformat.new_notebook()
notebook3 = nbformat.new_notebook()
code_cell = nbformat.new_code_cell(input=['a', 'b'])
notebook3['worksheets'] = [{'cells': [code_cell]}]
result = notebook_merge(notebook, notebook2, notebook3)
result_cells = result['worksheets'][0]['cells']
state = result_cells[0]['metadata']['state']
assert state == 'added'
def generateCells(self, crossTabList, grpFieldString, datasetName,
grpField):
for fieldList in crossTabList:
fieldListString, fieldListDfString, k_anon_size = self.generateFieldListStrings(
fieldList, "fieldListString", "fieldList")
crossStuff = grpField + "\n" + fieldListString + "\n" + fieldListDfString + "\n" + k_anon_size + "\n"
crossStuff = crossStuff + "crossBy, single_cell_info = makeCrossTabInfo( grpField, grpFieldList, fieldList, fieldListString, datasetName, k_anon_size)"
crossStuff = crossStuff + "\n" + "single_cell_info"
cell = nbf.new_code_cell(crossStuff)
self.cells.append(cell)
cell = nbf.new_code_cell("crossBy")
self.cells.append(cell)
def visit_literal_block(self, node):
raw_text = node.astext()
#only include lines that begin with <<<<
#we want the example code and not the example output
processed_text = '\n'.join([line.split('>>>')[1][1:] for line in raw_text.split('\n') if line.startswith('>>>')])
c = nbformat.new_code_cell(input=processed_text)
self.add_cell(c)
def _create_code_cell():
source = """print("something")
### BEGIN SOLUTION
print("hello")
### END SOLUTION"""
cell = new_code_cell(input=source)
return cell
def build_notebook(self):
"""Build a notebook in memory for use with preprocessor tests"""
outputs = [
nbformat.new_output(output_type="stream",
stream="stdout",
output_text="a"),
nbformat.new_output(output_type="text", output_text="b"),
nbformat.new_output(output_type="stream",
stream="stdout",
output_text="c"),
nbformat.new_output(output_type="stream",
stream="stdout",
output_text="d"),
nbformat.new_output(output_type="stream",
stream="stderr",
output_text="e"),
nbformat.new_output(output_type="stream",
stream="stderr",
output_text="f"),
nbformat.new_output(output_type="png", output_png=b'Zw==')
] #g
cells = [
nbformat.new_code_cell(input="$ e $",
prompt_number=1,
outputs=outputs),
nbformat.new_text_cell('markdown', source="$ e $")
]
worksheets = [nbformat.new_worksheet(name="worksheet1", cells=cells)]
return nbformat.new_notebook(name="notebook1", worksheets=worksheets)
def test_basic_notebook_merge():
notebook = nbformat.new_notebook()
code_cell = nbformat.new_code_cell(input=['a', 'b'])
notebook['worksheets'] = [
{'cells': [code_cell]}
]
notebook2 = nbformat.new_notebook()
notebook3 = nbformat.new_notebook()
code_cell = nbformat.new_code_cell(input=['a', 'b'])
notebook3['worksheets'] = [
{'cells': [code_cell]}
]
result = notebook_merge(notebook, notebook2, notebook3)
result_cells = result['worksheets'][0]['cells']
state = result_cells[0]['metadata']['state']
assert state == 'added'
def test_coalesce_sequenced_streams(self):
"""Can the coalesce streams preprocessor merge a sequence of streams?"""
outputs = [nbformat.new_output(output_type="stream", stream="stdout", output_text="0"),
nbformat.new_output(
output_type="stream", stream="stdout", output_text="1"),
nbformat.new_output(
output_type="stream", stream="stdout", output_text="2"),
nbformat.new_output(
output_type="stream", stream="stdout", output_text="3"),
nbformat.new_output(
output_type="stream", stream="stdout", output_text="4"),
nbformat.new_output(
output_type="stream", stream="stdout", output_text="5"),
nbformat.new_output(
output_type="stream", stream="stdout", output_text="6"),
nbformat.new_output(output_type="stream", stream="stdout", output_text="7")]
cells = [nbformat.new_code_cell(
input="# None", prompt_number=1, outputs=outputs)]
worksheets = [nbformat.new_worksheet(name="worksheet1", cells=cells)]
nb = nbformat.new_notebook(name="notebook1", worksheets=worksheets)
res = self.build_resources()
nb, res = coalesce_streams(nb, res)
outputs = nb.worksheets[0].cells[0].outputs
self.assertEqual(outputs[0].text, u'01234567')
def add_code_cell(self, nb):
output = current.new_output("display_data",
output_javascript="alert('hi');")
cell = current.new_code_cell("print('hi')", outputs=[output])
if not nb.worksheets:
nb.worksheets.append(current.new_worksheet())
nb.worksheets[0].cells.append(cell)
def test_coalesce_replace_streams(self):
"""Are \\r characters handled?"""
outputs = [
nbformat.new_output(output_type="stream",
stream="stdout",
output_text="z"),
nbformat.new_output(output_type="stream",
stream="stdout",
output_text="\ra"),
nbformat.new_output(output_type="stream",
stream="stdout",
output_text="\nz\rb"),
nbformat.new_output(output_type="stream",
stream="stdout",
output_text="\nz"),
nbformat.new_output(output_type="stream",
stream="stdout",
output_text="\rc\n"),
nbformat.new_output(output_type="stream",
stream="stdout",
output_text="z\rz\rd")
]
cells = [
nbformat.new_code_cell(input="# None",
prompt_number=1,
outputs=outputs)
]
worksheets = [nbformat.new_worksheet(name="worksheet1", cells=cells)]
nb = nbformat.new_notebook(name="notebook1", worksheets=worksheets)
res = self.build_resources()
nb, res = coalesce_streams(nb, res)
outputs = nb.worksheets[0].cells[0].outputs
self.assertEqual(outputs[0].text, u'a\nb\nc\nd')
def build_notebook(self):
"""Build a reveal slides notebook in memory for use with tests.
Overrides base in TransformerTestsBase"""
outputs = [nbformat.new_output(output_type="stream", stream="stdout", output_text="a")]
slide_metadata = {'slideshow' : {'slide_type': 'slide'}}
subslide_metadata = {'slideshow' : {'slide_type': 'subslide'}}
cells=[nbformat.new_code_cell(input="", prompt_number=1, outputs=outputs),
nbformat.new_text_cell('markdown', source="", metadata=slide_metadata),
nbformat.new_code_cell(input="", prompt_number=2, outputs=outputs),
nbformat.new_text_cell('markdown', source="", metadata=slide_metadata),
nbformat.new_text_cell('markdown', source="", metadata=subslide_metadata)]
worksheets = [nbformat.new_worksheet(name="worksheet1", cells=cells)]
return nbformat.new_notebook(name="notebook1", worksheets=worksheets)
def _create_code_cell():
source = """# YOUR CODE HERE
{% if solution %}
print "hello"
{% endif %}
"""
cell = new_code_cell(input=source, prompt_number=2, outputs=["foo"])
return cell
def test_replace_test_source_code(self):
"""Is the code source properly replaced?"""
cell = new_code_cell("\n", metadata=dict(
assignment=dict(id="test1_for_problem1")))
self.preprocessor._load_autograder_tests_file()
self.preprocessor._replace_test_source(cell)
assert cell.input == "# blah blah blah"
assert cell.metadata['assignment']['weight'] == 0.3333333333333333
assert cell.metadata['assignment']['points'] == 1
def test_get_score_ok(self):
"""Is the score zero when there was no error?"""
cell = new_code_cell("\n", metadata=dict(
assignment=dict(id="test1_for_problem1")))
cell.outputs = []
self.preprocessor._load_autograder_tests_file()
score, total_score = self.preprocessor._get_score(cell)
assert score == 1
assert total_score == 1
def test_get_score_error(self):
"""Is the score zero when there was an error?"""
cell = new_code_cell("\n", metadata=dict(
assignment=dict(id="test1_for_problem1")))
cell.outputs = [NotebookNode()]
cell.outputs[0]['output_type'] = 'pyerr'
self.preprocessor._load_autograder_tests_file()
score, total_score = self.preprocessor._get_score(cell)
assert score == 0
assert total_score == 1
def visit_literal_block(self, node):
raw_text = node.astext()
#only include lines that begin with >>>
#we want the example code and not the example output
processed_text = '\n'.join([
line.split('>>>')[1][1:] for line in raw_text.split('\n')
if line.startswith('>>>')
])
c = nbformat.new_code_cell(input=processed_text)
self.add_cell(c)
def build_notebook(self):
"""Build a reveal slides notebook in memory for use with tests.
Overrides base in TransformerTestsBase"""
outputs = [nbformat.new_output(output_type="svg", output_svg=self.simple_svg)]
slide_metadata = {'slideshow' : {'slide_type': 'slide'}}
subslide_metadata = {'slideshow' : {'slide_type': 'subslide'}}
cells=[nbformat.new_code_cell(input="", prompt_number=1, outputs=outputs)]
worksheets = [nbformat.new_worksheet(name="worksheet1", cells=cells)]
return nbformat.new_notebook(name="notebook1", worksheets=worksheets)
def build_notebook(self):
"""Build a reveal slides notebook in memory for use with tests.
Overrides base in PreprocessorTestsBase"""
outputs = [nbformat.new_output(output_type="svg", output_svg=self.simple_svg)]
slide_metadata = {'slideshow' : {'slide_type': 'slide'}}
subslide_metadata = {'slideshow' : {'slide_type': 'subslide'}}
cells=[nbformat.new_code_cell(input="", prompt_number=1, outputs=outputs)]
worksheets = [nbformat.new_worksheet(name="worksheet1", cells=cells)]
return nbformat.new_notebook(name="notebook1", worksheets=worksheets)
def test_empty_code_cell(self):
"""No empty code cells in rst"""
nbname = self._get_notebook()
with io.open(nbname, encoding='utf8') as f:
nb = current.read(f, 'json')
exporter = self.exporter_class()
(output, resources) = exporter.from_notebook_node(nb)
# add an empty code cell
nb.worksheets[0].cells.append(current.new_code_cell(input=""))
(output2, resources) = exporter.from_notebook_node(nb)
# adding an empty code cell shouldn't change output
self.assertEqual(output.strip(), output2.strip())
def generateCrossTabsReport(self, datasetName, src_dir, fnName,
crossTabList, grpFieldString):
dfcell = 'df, titleStuff = makeOutput(datasetName, src_dir , fnName )'
grpFieldString, grpField = self.generateGrpListStrings(
grpFieldString, "grpFieldList", "grpField")
stuff = '''
datasetName = "%s"
src_dir = '%s'
fnName = '%s'
''' % (datasetName, src_dir, fnName)
stuff = stuff + "\n" + grpFieldString + "\n" + dfcell + "\n" + "titleStuff"
cell = nbf.new_code_cell(stuff)
self.cells.append(cell)
self.generateCells(crossTabList, grpFieldString, datasetName, grpField)
def test_run_nb(self):
"""Test %run notebook.ipynb"""
from IPython.nbformat import current
nb = current.new_notebook(worksheets=[
current.new_worksheet(cells=[
current.new_text_cell("The Ultimate Question of Everything"),
current.new_code_cell("answer=42")
])
])
src = current.writes(nb, 'json')
self.mktmp(src, ext='.ipynb')
_ip.magic("run %s" % self.fname)
nt.assert_equal(_ip.user_ns['answer'], 42)
def build_notebook(self):
"""Build a notebook in memory for use with preprocessor tests"""
outputs = [nbformat.new_output(output_type="stream", stream="stdout", output_text="a"),
nbformat.new_output(output_type="text", output_text="b"),
nbformat.new_output(output_type="stream", stream="stdout", output_text="c"),
nbformat.new_output(output_type="stream", stream="stdout", output_text="d"),
nbformat.new_output(output_type="stream", stream="stderr", output_text="e"),
nbformat.new_output(output_type="stream", stream="stderr", output_text="f"),
nbformat.new_output(output_type="png", output_png='Zw==')] #g
cells=[nbformat.new_code_cell(input="$ e $", prompt_number=1,outputs=outputs),
nbformat.new_text_cell('markdown', source="$ e $")]
worksheets = [nbformat.new_worksheet(name="worksheet1", cells=cells)]
return nbformat.new_notebook(name="notebook1", worksheets=worksheets)
def test_coalesce_replace_streams(self):
"""Are \\r characters handled?"""
outputs = [nbformat.new_output(output_type="stream", stream="stdout", output_text="z"),
nbformat.new_output(output_type="stream", stream="stdout", output_text="\ra"),
nbformat.new_output(output_type="stream", stream="stdout", output_text="\nz\rb"),
nbformat.new_output(output_type="stream", stream="stdout", output_text="\nz"),
nbformat.new_output(output_type="stream", stream="stdout", output_text="\rc\n"),
nbformat.new_output(output_type="stream", stream="stdout", output_text="z\rz\rd")]
cells=[nbformat.new_code_cell(input="# None", prompt_number=1,outputs=outputs)]
worksheets = [nbformat.new_worksheet(name="worksheet1", cells=cells)]
nb = nbformat.new_notebook(name="notebook1", worksheets=worksheets)
res = self.build_resources()
nb, res = coalesce_streams(nb, res)
outputs = nb.worksheets[0].cells[0].outputs
self.assertEqual(outputs[0].text, u'a\nb\nc\nd')
def test_empty_code_cell(self):
"""No empty code cells in rst"""
nbname = self._get_notebook()
with io.open(nbname, encoding='utf8') as f:
nb = current.read(f, 'json')
exporter = self.exporter_class()
(output, resources) = exporter.from_notebook_node(nb)
# add an empty code cell
nb.worksheets[0].cells.append(
current.new_code_cell(input="")
)
(output2, resources) = exporter.from_notebook_node(nb)
# adding an empty code cell shouldn't change output
self.assertEqual(output.strip(), output2.strip())
def test_run_nb(self):
"""Test %run notebook.ipynb"""
from IPython.nbformat import current
nb = current.new_notebook(
worksheets=[
current.new_worksheet(cells=[
current.new_code_cell("answer=42")
])
]
)
src = current.writes(nb, 'json')
self.mktmp(src, ext='.ipynb')
_ip.magic("run %s" % self.fname)
nt.assert_equal(_ip.user_ns['answer'], 42)
def write_notebook(flow, options):
"""See http://nbviewer.ipython.org/gist/fperez/9716279"""
from IPython.nbformat import current as nbf
nb = nbf.new_notebook()
cells = [
#nbf.new_text_cell('heading', "This is an auto-generated notebook for %s" % os.path.basename(pseudopath)),
nbf.new_code_cell("""\
##%%javascript
##IPython.OutputArea.auto_scroll_threshold = 9999;
from __future__ import print_function
from abipy import abilab
%matplotlib inline
mpld3 = abilab.mpld3_enable_notebook()
import pylab
pylab.rcParams['figure.figsize'] = (25.0, 10.0)
import seaborn as sns
#sns.set(style="dark", palette="Set2")
sns.set(style='ticks', palette='Set2')"""),
nbf.new_code_cell("flow = abilab.Flow.pickle_load('%s')" % flow.workdir),
nbf.new_code_cell("flow.show_dependencies()"),
nbf.new_code_cell("flow.check_status(show=True, verbose=0)"),
nbf.new_code_cell("flow.show_inputs(nids=None, wslice=None)"),
nbf.new_code_cell("flow.inspect(nids=None, wslice=None)"),
nbf.new_code_cell("flow.show_abierrors()"),
nbf.new_code_cell("flow.show_qouts()"),
]
# Now that we have the cells, we can make a worksheet with them and add it to the notebook:
nb['worksheets'].append(nbf.new_worksheet(cells=cells))
# Next, we write it to a file on disk that we can then open as a new notebook.
# Note: This should be as easy as: nbf.write(nb, fname), but the current api is a little more verbose and needs a real file-like object.
import tempfile
_, tmpfname = tempfile.mkstemp(suffix='.ipynb', text=True)
with open(tmpfname, 'w') as fh:
nbf.write(nb, fh, 'ipynb')
os.system("ipython notebook %s" % tmpfname)
def to_notebook(infile, hr_separated=False):
"""Given markdown, returns an ipynb compliant JSON string"""
parser = markdown.DocParser()
ast = json.loads(markdown.ASTtoJSON(
parser.parse(infile.read())))
cells = [current.new_text_cell('markdown', '')]
for block in ast.get('children', []):
if block['t'] in ["IndentedCode", "FencedCode"]:
cells.append(current.new_code_cell(
block['string_content'].rstrip()
))
elif block['t'] in ['SetextHeader', 'ATXHeader']:
src = '{} {}'.format(
'#' * block.get('level', 1),
''.join(block['strings'])
).rstrip()
if hr_separated and cells[-1]['cell_type'] is 'markdown':
cells[-1]['source'] += '\n\n{}'.format(src)
else:
cells.append(current.new_text_cell('markdown', src))
elif block['t'] in ['HorizontalRule']:
# We don't render horizontal rules
if hr_separated:
cells.append(current.new_text_cell('markdown', ''))
else:
src = '\n'.join(block['strings']).rstrip()
if hr_separated and cells[-1]['cell_type'] is 'markdown':
cells[-1]['source'] += '\n\n{}'.format(src)
else:
cells.append(current.new_text_cell('markdown', src))
cells = tidy_notebook(cells[:])
worksheet = current.new_worksheet(cells=cells)
nb = current.new_notebook(
basename(infile.name).split('.')[:-1],
worksheets=[worksheet]
)
# using the indent option leaves a bunch of trailing whitespace. No thanks!
return json.dumps(nb, indent=2).replace(' \n', '\n')
def test_coalesce_sequenced_streams(self):
"""Can the coalesce streams preprocessor merge a sequence of streams?"""
outputs = [nbformat.new_output(output_type="stream", stream="stdout", output_text="0"),
nbformat.new_output(output_type="stream", stream="stdout", output_text="1"),
nbformat.new_output(output_type="stream", stream="stdout", output_text="2"),
nbformat.new_output(output_type="stream", stream="stdout", output_text="3"),
nbformat.new_output(output_type="stream", stream="stdout", output_text="4"),
nbformat.new_output(output_type="stream", stream="stdout", output_text="5"),
nbformat.new_output(output_type="stream", stream="stdout", output_text="6"),
nbformat.new_output(output_type="stream", stream="stdout", output_text="7")]
cells=[nbformat.new_code_cell(input="# None", prompt_number=1,outputs=outputs)]
worksheets = [nbformat.new_worksheet(cells=cells)]
nb = nbformat.new_notebook(name="notebook1", worksheets=worksheets)
res = self.build_resources()
nb, res = coalesce_streams(nb, res)
outputs = nb.worksheets[0].cells[0].outputs
self.assertEqual(outputs[0].text, u'01234567')
def notebook(self, s):
"""Export and convert IPython notebooks.
This function can export the current IPython history to a notebook file
or can convert an existing notebook file into a different format. For
example, to export the history to "foo.ipynb" do "%notebook -e foo.ipynb".
To export the history to "foo.py" do "%notebook -e foo.py". To convert
"foo.ipynb" to "foo.json" do "%notebook -f json foo.ipynb". Possible
formats include (json/ipynb, py).
"""
args = magic_arguments.parse_argstring(self.notebook, s)
from IPython.nbformat import current
args.filename = unquote_filename(args.filename)
if args.export:
fname, name, format = current.parse_filename(args.filename)
cells = []
hist = list(self.shell.history_manager.get_range())
for session, prompt_number, input in hist[:-1]:
cells.append(
current.new_code_cell(prompt_number=prompt_number,
input=input))
worksheet = current.new_worksheet(cells=cells)
nb = current.new_notebook(name=name, worksheets=[worksheet])
with io.open(fname, 'w', encoding='utf-8') as f:
current.write(nb, f, format)
elif args.format is not None:
old_fname, old_name, old_format = current.parse_filename(
args.filename)
new_format = args.format
if new_format == u'xml':
raise ValueError('Notebooks cannot be written as xml.')
elif new_format == u'ipynb' or new_format == u'json':
new_fname = old_name + u'.ipynb'
new_format = u'json'
elif new_format == u'py':
new_fname = old_name + u'.py'
else:
raise ValueError('Invalid notebook format: %s' % new_format)
with io.open(old_fname, 'r', encoding='utf-8') as f:
nb = current.read(f, old_format)
with io.open(new_fname, 'w', encoding='utf-8') as f:
current.write(nb, f, new_format)
def test_contents_manager(self):
"make sure ContentsManager returns right files (ipynb, bin, txt)."
nbdir = self.notebook_dir.name
base = self.base_url()
nb = new_notebook(name='testnb')
ws = new_worksheet()
nb.worksheets = [ws]
ws.cells.append(new_heading_cell(u'Created by test ³'))
cc1 = new_code_cell(input=u'print(2*6)')
cc1.outputs.append(new_output(output_text=u'12', output_type='stream'))
ws.cells.append(cc1)
with io.open(pjoin(nbdir, 'testnb.ipynb'), 'w',
encoding='utf-8') as f:
write(nb, f, format='ipynb')
with io.open(pjoin(nbdir, 'test.bin'), 'wb') as f:
f.write(b'\xff' + os.urandom(5))
f.close()
with io.open(pjoin(nbdir, 'test.txt'), 'w') as f:
f.write(u'foobar')
f.close()
r = requests.get(url_path_join(base, 'files', 'testnb.ipynb'))
self.assertEqual(r.status_code, 200)
self.assertIn('print(2*6)', r.text)
json.loads(r.text)
r = requests.get(url_path_join(base, 'files', 'test.bin'))
self.assertEqual(r.status_code, 200)
self.assertEqual(r.headers['content-type'], 'application/octet-stream')
self.assertEqual(r.content[:1], b'\xff')
self.assertEqual(len(r.content), 6)
r = requests.get(url_path_join(base, 'files', 'test.txt'))
self.assertEqual(r.status_code, 200)
self.assertEqual(r.headers['content-type'], 'text/plain')
self.assertEqual(r.text, 'foobar')
def __init__(self):
self.fxns = '''
# functions used throughout analysis
from __future__ import division
from IPython.core.display import HTML
import numpy as np
import pandas as pd
from collections import Counter
from scipy.stats import mode
import sys
sys.path.append('/home/ubuntu/workspace/pydev/')
from CrossTabUtils import *
pd.set_option('display.max_colwidth', 1)
'''
self.nb = nbf.new_notebook()
self.cells = []
cell = nbf.new_code_cell(self.fxns)
self.cells.append(cell)
def test_contents_manager(self):
"make sure ContentsManager returns right files (ipynb, bin, txt)."
nbdir = self.notebook_dir.name
base = self.base_url()
nb = new_notebook(name='testnb')
ws = new_worksheet()
nb.worksheets = [ws]
ws.cells.append(new_heading_cell(u'Created by test ³'))
cc1 = new_code_cell(input=u'print(2*6)')
cc1.outputs.append(new_output(output_text=u'12', output_type='stream'))
ws.cells.append(cc1)
with io.open(pjoin(nbdir, 'testnb.ipynb'), 'w', encoding='utf-8') as f:
write(nb, f, format='ipynb')
with io.open(pjoin(nbdir, 'test.bin'), 'wb') as f:
f.write(b'\xff' + os.urandom(5))
f.close()
with io.open(pjoin(nbdir, 'test.txt'), 'w') as f:
f.write(u'foobar')
f.close()
r = requests.get(url_path_join(base, 'files', 'testnb.ipynb'))
self.assertEqual(r.status_code, 200)
self.assertIn('print(2*6)', r.text)
json.loads(r.text)
r = requests.get(url_path_join(base, 'files', 'test.bin'))
self.assertEqual(r.status_code, 200)
self.assertEqual(r.headers['content-type'], 'application/octet-stream')
self.assertEqual(r.content[:1], b'\xff')
self.assertEqual(len(r.content), 6)
r = requests.get(url_path_join(base, 'files', 'test.txt'))
self.assertEqual(r.status_code, 200)
self.assertEqual(r.headers['content-type'], 'text/plain')
self.assertEqual(r.text, 'foobar')
def notebook(self, s):
"""Export and convert IPython notebooks.
This function can export the current IPython history to a notebook file
or can convert an existing notebook file into a different format. For
example, to export the history to "foo.ipynb" do "%notebook -e foo.ipynb".
To export the history to "foo.py" do "%notebook -e foo.py". To convert
"foo.ipynb" to "foo.json" do "%notebook -f json foo.ipynb". Possible
formats include (json/ipynb, py).
"""
args = magic_arguments.parse_argstring(self.notebook, s)
from IPython.nbformat import current
args.filename = unquote_filename(args.filename)
if args.export:
fname, name, format = current.parse_filename(args.filename)
cells = []
hist = list(self.shell.history_manager.get_range())
for session, prompt_number, input in hist[:-1]:
cells.append(current.new_code_cell(prompt_number=prompt_number,
input=input))
worksheet = current.new_worksheet(cells=cells)
nb = current.new_notebook(name=name,worksheets=[worksheet])
with io.open(fname, 'w', encoding='utf-8') as f:
current.write(nb, f, format)
elif args.format is not None:
old_fname, old_name, old_format = current.parse_filename(args.filename)
new_format = args.format
if new_format == u'xml':
raise ValueError('Notebooks cannot be written as xml.')
elif new_format == u'ipynb' or new_format == u'json':
new_fname = old_name + u'.ipynb'
new_format = u'json'
elif new_format == u'py':
new_fname = old_name + u'.py'
else:
raise ValueError('Invalid notebook format: %s' % new_format)
with io.open(old_fname, 'r', encoding='utf-8') as f:
nb = current.read(f, old_format)
with io.open(new_fname, 'w', encoding='utf-8') as f:
current.write(nb, f, new_format)
def setUp(self):
nbdir = self.notebook_dir.name
if not os.path.isdir(pjoin(nbdir, 'foo')):
os.mkdir(pjoin(nbdir, 'foo'))
nb = new_notebook(name='testnb')
ws = new_worksheet()
nb.worksheets = [ws]
ws.cells.append(new_heading_cell(u'Created by test ³'))
cc1 = new_code_cell(input=u'print(2*6)')
cc1.outputs.append(new_output(output_text=u'12'))
cc1.outputs.append(new_output(output_png=png_green_pixel, output_type='pyout'))
ws.cells.append(cc1)
with io.open(pjoin(nbdir, 'foo', 'testnb.ipynb'), 'w',
encoding='utf-8') as f:
write(nb, f, format='ipynb')
self.nbconvert_api = NbconvertAPI(self.base_url())
def save_intermediate(km, ws, cell, save_name=None):
"""
Conditionally save __save__ variables on error, and perhaps add a cell to
load them up again into the worksheet.
This will be utilized via a command-line flag.
"""
saving_tmpl = """#error happend here, execute this cell to reload state
_load = False
_name = '%s'
if '__save__' in locals():
d = {}
for k in __save__:
try:
d[k] = locals()[k]
# for things that are meant to be saved but that haven't executed yet
except KeyError:
d[k] = None
np.savez(_name, **d)
print "saved " + _name + '.npz'
if _load:
loaded = np.load(_name+'.npz')
print loaded.keys(), "now injected into this namespace"
locals().update(loaded)
"""
if not save_name:
save_name = ''
save_name += '_error'
save_cell = cur.new_code_cell(saving_tmpl%save_name,"-1")
if hasattr(km, 'saveonerror') and km.saveonerror:
idx = ws.cells.index(cell)
outs, exec_count = run_cell(km, save_cell)
save_cell.outputs = outs
# XXX: the problem with this approach is that after the first error,
# the whole notebook must be regenerated in order to not have this
# saving code in it
ws.cells.insert(idx+1, save_cell)
pass
def new_notebook_from_string(notebookname, filename, sourcestring):
root = ast.parse(sourcestring, filename=filename, mode='exec')
x = DetermineBlocks()
for child in ast.iter_child_nodes(root):
print(child.lineno, child)
x.visit(child)
x.end()
sourcelines = sourcestring.splitlines()
cells = []
for block in x.blocks:
print(block)
blocklines = sourcelines[block[1] - 1:block[2]]
blocksrc = '\n'.join(blocklines)
if len(blocksrc) > 0:
cell = notebook_format.new_code_cell(input=blocksrc)
cells.append(cell)
ws = notebook_format.new_worksheet(cells=cells)
result = notebook_format.new_notebook(worksheets=[ws])
result.metadata.name = notebookname
return result
def new_notebook_from_string(notebookname, filename, sourcestring):
root = ast.parse(sourcestring, filename=filename, mode='exec')
x = DetermineBlocks()
for child in ast.iter_child_nodes(root):
print child.lineno, child
x.visit(child)
x.end()
sourcelines = sourcestring.splitlines()
cells = []
for block in x.blocks:
print block
blocklines = sourcelines[block[1]-1:block[2]]
blocksrc = '\n'.join(blocklines)
if len(blocksrc) > 0:
cell = notebook_format.new_code_cell(input=blocksrc)
cells.append(cell)
ws = notebook_format.new_worksheet(cells=cells)
result = notebook_format.new_notebook(worksheets=[ws])
result.metadata.name = notebookname
return result
def remote_solutions(nb):
"""Generate a version of the notebook with stripped exercises solutions"""
for ws in nb.worksheets:
inside_solution = False
cells_to_remove = []
for i, cell in enumerate(ws.cells):
if cell.cell_type == 'heading':
inside_solution = False
elif cell.cell_type == 'markdown':
first_line = cell.source.split("\n")[0].strip()
if first_line.lower() in ("**exercise:**", "**exercise**:"):
inside_solution = True
# Insert a new code cell to work on the exercise
ws.cells.insert(i + 1, current.new_code_cell())
continue
if inside_solution:
if cell.cell_type == 'code' and not hasattr(cell, 'input'):
# Leave blank code cells
continue
cells_to_remove.append(cell)
for cell in cells_to_remove:
ws.cells.remove(cell)
def remote_solutions(nb):
"""Generate a version of the notebook with stripped exercises solutions"""
for ws in nb.worksheets:
inside_solution = False
cells_to_remove = []
for i, cell in enumerate(ws.cells):
if cell.cell_type == 'heading':
inside_solution = False
elif cell.cell_type == 'markdown':
first_line = cell.source.split("\n")[0].strip()
if first_line.lower() in ("**exercise:**", "**exercise**:"):
inside_solution = True
# Insert a new code cell to work on the exercise
ws.cells.insert(i + 1, current.new_code_cell())
continue
if inside_solution:
if cell.cell_type == 'code' and not hasattr(cell, 'input'):
# Leave blank code cells
continue
cells_to_remove.append(cell)
for cell in cells_to_remove:
ws.cells.remove(cell)
def setUp(self):
nbdir = self.notebook_dir.name
if not os.path.isdir(pjoin(nbdir, 'foo')):
os.mkdir(pjoin(nbdir, 'foo'))
nb = new_notebook(name='testnb')
ws = new_worksheet()
nb.worksheets = [ws]
ws.cells.append(new_heading_cell(u'Created by test ³'))
cc1 = new_code_cell(input=u'print(2*6)')
cc1.outputs.append(new_output(output_text=u'12'))
cc1.outputs.append(
new_output(output_png=png_green_pixel, output_type='pyout'))
ws.cells.append(cc1)
with io.open(pjoin(nbdir, 'foo', 'testnb.ipynb'),
'w',
encoding='utf-8') as f:
write(nb, f, format='ipynb')
self.nbconvert_api = NbconvertAPI(self.base_url())
def to_ipython_cell(self, df):
code = []
outputs = []
local = {"data": self.data, "parameters": self.parameters}
for line in self.code.split("\n"):
if line.endswith("# INPUT"):
a, b = line.split("=")
a, b = a.strip(), b.strip()
value = eval(b, None, local)
line = '{a} = {value}'.format(a=a, value=repr(value))
code.append(line)
elif line.endswith("# OUTPUT"):
a, b = line.split("=")
outputs += [x.strip() for x in a.split(",")]
code.append(line[4:-len("# OUTPUT")].rstrip())
elif "def" not in line and not line.startswith("@") and "return" not in line:
code.append(line[4:]) # get rid of first indent
code = "\n".join(code)
gs = {"bz": bz, "pd": pd, "df": df, "stats": stats, "np":np, "StatisticalTest":StatisticalTest}
ls = {}
exec(code, gs, ls)
output_cells = []
for output in outputs:
kwargs = {}
if hasattr(ls[output], "to_html"):
kwargs["output_html"] = ls[output].to_html()
o = current.new_output("display_data", str(ls[output]), **kwargs)
output_cells.append(o)
code += "\ndisplay({})".format(output)
return current.new_code_cell(code, outputs=output_cells)
def add_code_cell(self, nb):
output = current.new_output("display_data", output_javascript="alert('hi');")
cell = current.new_code_cell("print('hi')", outputs=[output])
if not nb.worksheets:
nb.worksheets.append(current.new_worksheet())
nb.worksheets[0].cells.append(cell)
def add_code_cell(self, lines):
c = nbformat.new_code_cell(input='\n'.join(lines))
self.add_cell(c)
def _init_cells(self):
self._cells = [
nbf.new_code_cell('%load_ext rmagic',
metadata=META_SKIP)
]
def run(self):
path = os.path.abspath(os.path.expanduser(self.args.path))
final_dir = os.path.abspath(os.path.expanduser(self.args.final_dir))
if not path.endswith(".yaml") and not path.endswith(".yml"):
raise ValueError
filename = os.path.basename(path)
new_filename = "Mission" + filename.replace(".yml", ".ipynb").replace(".yaml", ".ipynb")
final_dest = os.path.join(final_dir, new_filename)
mission, screens = mission_loader(path)
nb = nbf.new_notebook()
mission_cell = nbf.new_text_cell('markdown', self.assemble_mission_cell(mission).strip())
cells = [mission_cell]
for screen in screens:
text = self.assemble_screen_meta(screen)
text += "\n\n"
if screen["type"] == "code":
text += "# " + screen["name"]
text += "\n\n"
text += screen["left_text"]
if "instructions" in screen:
text += "\n\n"
text += "## Instructions\n\n"
text += screen["instructions"]
if "hint" in screen:
text += "\n\n"
text += "## Hint\n\n"
text += screen["hint"]
elif screen["type"] == "video":
text += "# " + screen["name"]
text += "\n\n"
text += screen["video"]
elif screen["type"] == "text":
text += "# " + screen["name"]
text += "\n\n"
text += screen["text"]
cell = nbf.new_text_cell('markdown', text.strip())
cells.append(cell)
if screen["type"] == "code":
text = ""
if "initial" not in screen and "answer" not in screen:
text += screen["initial_display"]
else:
items = [
{"key": "initial", "name": "## Initial"},
{"key": "initial_display", "name": "## Display"},
{"key": "answer", "name": "## Answer"},
{"key": "check_val", "name": "## Check val"},
{"key": "check_vars", "name": "## Check vars"},
{"key": "check_code_run", "name": "## Check code run"}
]
for item in items:
if item["key"] in screen and len(str(screen[item["key"]]).strip()) > 0:
if item["key"] == "check_vars" and len(screen[item["key"]]) == 0:
continue
text += item["name"] + "\n\n"
if item["key"] == "check_val":
text += '"' + str(screen[item["key"]]).strip().replace("\n", "\\n") + '"'
else:
text += str(screen[item["key"]]).strip()
text += "\n\n"
cell = nbf.new_code_cell(input=text.strip())
cells.append(cell)
nb['worksheets'].append(nbf.new_worksheet(cells=cells))
with open(final_dest, 'w+') as f:
nbf.write(nb, f, 'ipynb')
# Copy any associated files over
original_dir = os.path.dirname(path)
for f in os.listdir(original_dir):
full_path = os.path.join(original_dir, f)
if os.path.isfile(full_path):
if not f.endswith(".yaml") and not f.endswith(".yml") and not f.endswith(".ipynb"):
shutil.copy2(full_path, os.path.join(final_dir, f))
def add_code_cell(self, lines):
c = nbformat.new_code_cell(input='\n'.join(lines))
self.add_cell(c)
def generate_report(reviews, dataset_name, file_name, load_reviews_code):
nb = nbf.new_notebook()
title = '# ' + dataset_name + ' Dataset Analysis'
title_cell = nbf.new_text_cell(u'markdown', title)
rda = ReviewsDatasetAnalyzer(reviews)
num_reviews = len(rda.reviews)
num_users = len(rda.user_ids)
num_items = len(rda.item_ids)
user_avg_reviews = float(num_reviews) / num_users
item_avg_reviews = float(num_reviews) / num_items
sparsity = rda.calculate_sparsity_approx()
fact_sheet_text =\
'## Fact Sheet\n' +\
'The ' + dataset_name + ' contains:\n' +\
'* ' + str(num_reviews) + ' reviews\n' +\
'* Made by ' + str(num_users) + ' users\n' +\
'* About ' + str(num_items) + ' items\n' +\
'* It has an approximated sparsity of ' + str(sparsity) + '\n' +\
'\nNow we are going to analyze the number of reviews per user and ' \
'per item'
fact_sheet_cell = nbf.new_text_cell(u'markdown', fact_sheet_text)
reviews_analysis_code =\
'import sys\n' +\
'sys.path.append(\'/Users/fpena/UCC/Thesis/projects/yelp/source/python\')\n' +\
'from etl import ETLUtils\n\n' +\
'from etl.reviews_dataset_analyzer import ReviewsDatasetAnalyzer\n' +\
'\n# Load reviews\n' + load_reviews_code + '\n' +\
'rda = ReviewsDatasetAnalyzer(reviews)\n'
reviews_analysis_cell = nbf.new_code_cell(reviews_analysis_code)
user_analysis_text =\
'## Users Reviews Analysis\n' +\
'* The average number of reviews per user is ' + str(user_avg_reviews) + '\n' +\
'* The minimum number of reviews a user has is ' + str(min(rda.users_count)) + '\n' +\
'* The maximum number of reviews a user has is ' + str(max(rda.users_count))
user_analysis_cell = nbf.new_text_cell(u'markdown', user_analysis_text)
counts_per_user_code =\
'# Number of reviews per user\n' +\
'users_summary = rda.summarize_reviews_by_field(\'user_id\')\n' +\
'print(\'Average number of reviews per user\', float(rda.num_reviews)/rda.num_users)\n' +\
'users_summary.plot(kind=\'line\', rot=0)'
counts_per_user_cell = nbf.new_code_cell(counts_per_user_code)
item_analysis_text =\
'## Items Reviews Analysis\n' +\
'* The average number of reviews per item is ' + str(item_avg_reviews) + '\n' +\
'* The minimum number of reviews an item has is ' + str(min(rda.items_count)) + '\n' +\
'* The maximum number of reviews an item has is ' + str(max(rda.items_count))
item_analysis_cell = nbf.new_text_cell(u'markdown', item_analysis_text)
counts_per_item_code =\
'# Number of reviews per item\n' +\
'items_summary = rda.summarize_reviews_by_field(\'offering_id\')\n' +\
'print(\'Average number of reviews per item\', float(rda.num_reviews)/rda.num_items)\n' +\
'items_summary.plot(kind=\'line\', rot=0)'
counts_per_item_cell = nbf.new_code_cell(counts_per_item_code)
common_items_text =\
'## Number of items 2 users have in common\n' +\
'In this section we are going to count the number of items two ' \
'users have in common'
common_items_text_cell = nbf.new_text_cell(u'markdown', common_items_text)
common_items_code =\
'# Number of items 2 users have in common\n' +\
'common_item_counts = rda.count_items_in_common()\n' +\
'plt.plot(common_item_counts.keys(), common_item_counts.values())\n'
common_items_code_cell = nbf.new_code_cell(common_items_code)
common_items_box_code =\
'from pylab import boxplot\n' +\
'my_data = [key for key, value in common_item_counts.iteritems() for i in xrange(value)]\n' +\
'mean_common_items = float(sum(my_data))/len(my_data)\n' +\
'print(\'Average number of common items between two users:\', mean_common_items)\n' +\
'boxplot(my_data)'
common_items_box_cell = nbf.new_code_cell(common_items_box_code)
cells = []
cells.append(title_cell)
cells.append(fact_sheet_cell)
cells.append(reviews_analysis_cell)
cells.append(user_analysis_cell)
cells.append(counts_per_user_cell)
cells.append(item_analysis_cell)
cells.append(counts_per_item_cell)
cells.append(common_items_text_cell)
cells.append(common_items_code_cell)
cells.append(common_items_box_cell)
nb['worksheets'].append(nbf.new_worksheet(cells=cells))
with open(file_name, 'w') as f:
nbf.write(nb, f, 'ipynb')
def test_match_tests_no_match(self):
"""Is an error raised when an autograding cell cannot be matched?"""
cell = new_code_cell()
cell.metadata = dict(assignment=dict(cell_type="autograder"))
cells = [cell]
assert_raises(RuntimeError, self.preprocessor._match_tests, cells)
def _init_cells(self):
self._cells = [
nbf.new_code_cell('%load_ext rmagic', metadata=META_SKIP)
]
def new_code_cell():
nlines = random.randint(0, 30)
input = [str(random.random()) for i in range(nlines)]
code_cell = nbformat.new_code_cell(input=input)
return code_cell
def process_codeblock(self, codeblock):
""" Process a code block from pandoc json """
attrs, contents = codeblock
if self.guess_language(contents) == 'R':
contents = '%%R\n' + contents
self._add_cell(nbf.new_code_cell(contents))