def lines_to_notebook(lines, name=None):
"""
Convert the lines of an m file into an IPython notebook
Parameters
----------
lines : list
A list of strings. Each element is a line in the m file
Returns
-------
notebook : an IPython NotebookNode class instance, containing the
information required to create a file
"""
source = []
md = np.empty(len(lines), dtype=object)
new_cell = np.empty(len(lines), dtype=object)
for idx, l in enumerate(lines):
new_cell[idx], md[idx], this_source = format_line(l)
# Transitions between markdown and code and vice-versa merit a new
# cell, even if no newline, or "%%" is found. Make sure not to do this
# check for the very first line!
if idx>1 and not new_cell[idx]:
if md[idx] != md[idx-1]:
new_cell[idx] = True
source.append(this_source)
# This defines the breaking points between cells:
new_cell_idx = np.hstack([np.where(new_cell)[0], -1])
# Listify the sources:
cell_source = [source[new_cell_idx[i]:new_cell_idx[i+1]]
for i in range(len(new_cell_idx)-1)]
cell_md = [md[new_cell_idx[i]] for i in range(len(new_cell_idx)-1)]
cells = []
# Append the notebook with loading matlab magic extension
notebook_head = "import pymatbridge as pymat\n" + "ip = get_ipython()\n" \
+ "pymat.load_ipython_extension(ip)"
cells.append(nbformat.new_code_cell(notebook_head, language='python'))
for cell_idx, cell_s in enumerate(cell_source):
if cell_md[cell_idx]:
cells.append(nbformat.new_text_cell('markdown', cell_s))
else:
cell_s.insert(0, '%%matlab\n')
cells.append(nbformat.new_code_cell(cell_s, language='matlab'))
ws = nbformat.new_worksheet(cells=cells)
notebook = nbformat.new_notebook(metadata=nbformat.new_metadata(),
worksheets=[ws])
return notebook
def test_basic_notebook_merge():
notebook = nbformat.new_notebook()
code_cell = nbformat.new_code_cell(input=['a', 'b'])
notebook['worksheets'] = [{'cells': [code_cell]}]
notebook2 = nbformat.new_notebook()
notebook3 = nbformat.new_notebook()
code_cell = nbformat.new_code_cell(input=['a', 'b'])
notebook3['worksheets'] = [{'cells': [code_cell]}]
result = notebook_merge(notebook, notebook2, notebook3)
result_cells = result['worksheets'][0]['cells']
state = result_cells[0]['metadata']['state']
assert state == 'added'
def build_notebook(self):
"""Build a reveal slides notebook in memory for use with tests.
Overrides base in TransformerTestsBase"""
outputs = [
nbformat.new_output(output_type="stream",
stream="stdout",
output_text="a")
]
slide_metadata = {'slideshow': {'slide_type': 'slide'}}
subslide_metadata = {'slideshow': {'slide_type': 'subslide'}}
cells = [
nbformat.new_code_cell(input="", prompt_number=1, outputs=outputs),
nbformat.new_text_cell('markdown',
source="",
metadata=slide_metadata),
nbformat.new_code_cell(input="", prompt_number=2, outputs=outputs),
nbformat.new_text_cell('markdown',
source="",
metadata=slide_metadata),
nbformat.new_text_cell('markdown',
source="",
metadata=subslide_metadata)
]
worksheets = [nbformat.new_worksheet(name="worksheet1", cells=cells)]
return nbformat.new_notebook(name="notebook1", worksheets=worksheets)
def test_very_long_cells(self):
"""
Torture test that long cells do not cause issues
"""
lorem_ipsum_text = textwrap.dedent("""\
Lorem ipsum dolor sit amet, consectetur adipiscing elit. Donec
dignissim, ipsum non facilisis tempus, dui felis tincidunt metus,
nec pulvinar neque odio eget risus. Nulla nisi lectus, cursus
suscipit interdum at, ultrices sit amet orci. Mauris facilisis
imperdiet elit, vitae scelerisque ipsum dignissim non. Integer
consequat malesuada neque sit amet pulvinar. Curabitur pretium
ut turpis eget aliquet. Maecenas sagittis lacus sed lectus
volutpat, eu adipiscing purus pulvinar. Maecenas consequat
luctus urna, eget cursus quam mollis a. Aliquam vitae ornare
erat, non hendrerit urna. Sed eu diam nec massa egestas pharetra
at nec tellus. Fusce feugiat lacus quis urna sollicitudin volutpat.
Quisque at sapien non nibh feugiat tempus ac ultricies purus.
""")
lorem_ipsum_text = lorem_ipsum_text.replace("\n"," ") + "\n\n"
large_lorem_ipsum_text = "".join([lorem_ipsum_text]*3000)
notebook_name = "lorem_ipsum_long.ipynb"
tex_name = "lorem_ipsum_long.tex"
with self.create_temp_cwd([]):
nb = current.new_notebook(
worksheets=[
current.new_worksheet(cells=[
current.new_text_cell('markdown',source=large_lorem_ipsum_text)
])
]
)
with open(notebook_name, 'w') as f: current.write(nb, f, 'ipynb')
self.call('nbconvert --to latex --log-level 0 ' +
os.path.join(notebook_name))
assert os.path.isfile(tex_name)
def test_preprocess_autograder_cells_show(self):
"""Are autograder cells properly preprocessed and shown?"""
self.preprocessor.hide_autograder_cells = False
self.preprocessor.solution = False
cell1 = self._create_code_cell()
cell1.metadata = dict(assignment=dict(cell_type="grade", id="foo", points=1))
cell2 = self._create_code_cell()
cell2.input = "# hello"
cell2.metadata = dict(assignment=dict(cell_type="autograder", id="foo"))
cell3 = self._create_text_cell()
cell3.source = "goodbye"
cell3.metadata = dict(assignment=dict(cell_type="autograder", id="bar"))
nb = new_notebook(worksheets=[NotebookNode()])
nb.worksheets[0].cells = [cell1, cell2, cell3]
nb, resources = self.preprocessor.preprocess(nb, {})
cell1, cell2, cell3 = nb.worksheets[0].cells
assert cell2.input == "# hello"
assert cell3.source == "goodbye"
tests = resources["tests"]
assert tests == {
"foo": dict(weight=0.5, points=0.5, problem="foo", source="# hello", cell_type="code"),
"bar": dict(weight=0.5, points=0.5, problem="foo", source="goodbye", cell_type="markdown"),
}
def convert(py_file, ipynb_file):
cells = []
imports = "" # needed for each file
for cell, line_type, line in parse_blocks(py_file):
if line_type == file_type:
# write cell to file
try:
fname = line.split()[1]
except:
raise Exception(
"Markdown for file output must be the "
+ "following format\n#to_file filename.py: "
+ "{}\n".format(line)
)
sys.exit(0)
with open(fname, "w") as f:
f.write(cell)
new_cell = "%run {}".format(fname)
ipynb_cell = nbf.new_code_cell(new_cell)
cells.append(ipynb_cell)
else:
# convert cell to ipynb cell
ipynb_cell = nbf.new_code_cell(cell)
cells.append(ipynb_cell)
# create new notebook
nb = nbf.new_notebook()
nb["worksheets"].append(nbf.new_worksheet(cells=cells))
with open(ipynb_file, "w") as f:
nbf.write(nb, f, "ipynb")
def test_very_long_cells(self):
"""
Torture test that long cells do not cause issues
"""
lorem_ipsum_text = textwrap.dedent("""\
Lorem ipsum dolor sit amet, consectetur adipiscing elit. Donec
dignissim, ipsum non facilisis tempus, dui felis tincidunt metus,
nec pulvinar neque odio eget risus. Nulla nisi lectus, cursus
suscipit interdum at, ultrices sit amet orci. Mauris facilisis
imperdiet elit, vitae scelerisque ipsum dignissim non. Integer
consequat malesuada neque sit amet pulvinar. Curabitur pretium
ut turpis eget aliquet. Maecenas sagittis lacus sed lectus
volutpat, eu adipiscing purus pulvinar. Maecenas consequat
luctus urna, eget cursus quam mollis a. Aliquam vitae ornare
erat, non hendrerit urna. Sed eu diam nec massa egestas pharetra
at nec tellus. Fusce feugiat lacus quis urna sollicitudin volutpat.
Quisque at sapien non nibh feugiat tempus ac ultricies purus.
""")
lorem_ipsum_text = lorem_ipsum_text.replace("\n", " ") + "\n\n"
large_lorem_ipsum_text = "".join([lorem_ipsum_text] * 3000)
notebook_name = "lorem_ipsum_long.ipynb"
tex_name = "lorem_ipsum_long.tex"
with self.create_temp_cwd([]):
nb = current.new_notebook(worksheets=[
current.new_worksheet(cells=[
current.new_text_cell('markdown',
source=large_lorem_ipsum_text)
])
])
with open(notebook_name, 'w') as f:
current.write(nb, f, 'ipynb')
self.call('nbconvert --to latex --log-level 0 ' +
os.path.join(notebook_name))
assert os.path.isfile(tex_name)
def build_notebook(self):
"""Build a notebook in memory for use with preprocessor tests"""
outputs = [
nbformat.new_output(output_type="stream",
stream="stdout",
output_text="a"),
nbformat.new_output(output_type="text", output_text="b"),
nbformat.new_output(output_type="stream",
stream="stdout",
output_text="c"),
nbformat.new_output(output_type="stream",
stream="stdout",
output_text="d"),
nbformat.new_output(output_type="stream",
stream="stderr",
output_text="e"),
nbformat.new_output(output_type="stream",
stream="stderr",
output_text="f"),
nbformat.new_output(output_type="png", output_png=b'Zw==')
] #g
cells = [
nbformat.new_code_cell(input="$ e $",
prompt_number=1,
outputs=outputs),
nbformat.new_text_cell('markdown', source="$ e $")
]
worksheets = [nbformat.new_worksheet(name="worksheet1", cells=cells)]
return nbformat.new_notebook(name="notebook1", worksheets=worksheets)
def split_into_units(nb_name):
"""Split notebook into units."""
try:
with io.open(nb_name, 'r', encoding='utf-8') as f:
nb = current.read(f, 'json')
except IOError as e:
if e.errno == 2:
print('File not found: {0}'.format(nb_name))
return []
else:
raise e
indexes = []
cells = nb.worksheets[0].cells
for (i, cell) in enumerate(cells):
if cell.cell_type == 'heading' and cell.level == 1:
indexes.append(i)
separated_cells = [cells[i:j] for i, j in zip(indexes, indexes[1:]+[None])]
worksheets = map(lambda cells: current.new_worksheet(name=cells[0].source,
cells=cells),
separated_cells)
units = map(lambda worksheet: current.new_notebook(name=worksheet.name,
worksheets=[worksheet]),
worksheets)
return units
def new_notebook(self):
"""Create a new notebook and return its notebook_id."""
name = self.increment_filename('Untitled')
metadata = current.new_metadata(name=name)
nb = current.new_notebook(metadata=metadata)
notebook_id = self.write_notebook_object(nb)
return notebook_id
def new_notebook(self):
"""Create a new notebook and return its notebook_id."""
name = self.increment_filename('Untitled')
metadata = current.new_metadata(name=name)
nb = current.new_notebook(metadata=metadata)
notebook_id = self.write_notebook_object(nb)
return notebook_id
def test_basic_notebook_merge():
notebook = nbformat.new_notebook()
code_cell = nbformat.new_code_cell(input=['a', 'b'])
notebook['worksheets'] = [
{'cells': [code_cell]}
]
notebook2 = nbformat.new_notebook()
notebook3 = nbformat.new_notebook()
code_cell = nbformat.new_code_cell(input=['a', 'b'])
notebook3['worksheets'] = [
{'cells': [code_cell]}
]
result = notebook_merge(notebook, notebook2, notebook3)
result_cells = result['worksheets'][0]['cells']
state = result_cells[0]['metadata']['state']
assert state == 'added'
def setUp(self):
nbdir = self.notebook_dir.name
self.blob = os.urandom(100)
self.b64_blob = base64.encodestring(self.blob).decode('ascii')
for d in (self.dirs + self.hidden_dirs):
d.replace('/', os.sep)
if not os.path.isdir(pjoin(nbdir, d)):
os.mkdir(pjoin(nbdir, d))
for d, name in self.dirs_nbs:
d = d.replace('/', os.sep)
# create a notebook
with io.open(pjoin(nbdir, d, '%s.ipynb' % name), 'w',
encoding='utf-8') as f:
nb = new_notebook(name=name)
write(nb, f, format='ipynb')
# create a text file
with io.open(pjoin(nbdir, d, '%s.txt' % name), 'w',
encoding='utf-8') as f:
f.write(self._txt_for_name(name))
# create a binary file
with io.open(pjoin(nbdir, d, '%s.blob' % name), 'wb') as f:
f.write(self._blob_for_name(name))
self.api = API(self.base_url())
def create_file(self, model=None, path='', ext='.ipynb'):
"""Create a new file or directory and return its model with no content."""
path = path.strip('/')
if model is None:
model = {}
if 'content' not in model and model.get('type', None) != 'directory':
if ext == '.ipynb':
metadata = current.new_metadata(name=u'')
model['content'] = current.new_notebook(metadata=metadata)
model['type'] = 'notebook'
model['format'] = 'json'
else:
model['content'] = ''
model['type'] = 'file'
model['format'] = 'text'
if 'name' not in model:
if model['type'] == 'directory':
untitled = self.untitled_directory
elif model['type'] == 'notebook':
untitled = self.untitled_notebook
elif model['type'] == 'file':
untitled = self.untitled_file
else:
raise HTTPError(400,
"Unexpected model type: %r" % model['type'])
model['name'] = self.increment_filename(untitled + ext, path)
model['path'] = path
model = self.save(model, model['name'], model['path'])
return model
def create_file(self, model=None, path='', ext='.ipynb'):
"""Create a new file or directory and return its model with no content."""
path = path.strip('/')
if model is None:
model = {}
if 'content' not in model and model.get('type', None) != 'directory':
if ext == '.ipynb':
metadata = current.new_metadata(name=u'')
model['content'] = current.new_notebook(metadata=metadata)
model['type'] = 'notebook'
model['format'] = 'json'
else:
model['content'] = ''
model['type'] = 'file'
model['format'] = 'text'
if 'name' not in model:
if model['type'] == 'directory':
untitled = self.untitled_directory
elif model['type'] == 'notebook':
untitled = self.untitled_notebook
elif model['type'] == 'file':
untitled = self.untitled_file
else:
raise HTTPError(400, "Unexpected model type: %r" % model['type'])
model['name'] = self.increment_filename(untitled + ext, path)
model['path'] = path
model = self.save(model, model['name'], model['path'])
return model
def mdstrip(paths):
for path in paths:
if os.path.isdir(path):
files = glob(os.path.join(path, "*.ipynb"))
else:
files = [path]
for in_file in files:
input_nb_name = basename(in_file)
slug = input_nb_name[:-6]
title = slug.replace("_", " ")
actual_title_nb = io.StringIO(
title_data.replace("{{ title }}", title))
title_nb = nbf.read(actual_title_nb, "ipynb")
title_cell = title_nb.worksheets[0].cells[0]
input_nb = nbf.read(open(in_file), "ipynb")
worksheet = input_nb.worksheets[0]
# add graphic here & append to cell_list
cell_list = [title_cell]
for cell in worksheet.cells:
if cell.cell_type == ("code"):
cell.outputs = []
cell_list.append(cell)
elif cell.cell_type == "heading":
cell_list.append(cell)
output_nb = nbf.new_notebook()
output_nb_name = slug+".prod.ipynb"
output_nb.worksheets.append(nbf.new_worksheet(cells=cell_list))
with open(output_nb_name, 'w') as f:
nbf.write(output_nb, f, "ipynb")
def setUp(self):
nbdir = self.notebook_dir.name
self.blob = os.urandom(100)
self.b64_blob = base64.encodestring(self.blob).decode('ascii')
for d in (self.dirs + self.hidden_dirs):
d.replace('/', os.sep)
if not os.path.isdir(pjoin(nbdir, d)):
os.mkdir(pjoin(nbdir, d))
for d, name in self.dirs_nbs:
d = d.replace('/', os.sep)
# create a notebook
with io.open(pjoin(nbdir, d, '%s.ipynb' % name),
'w',
encoding='utf-8') as f:
nb = new_notebook(name=name)
write(nb, f, format='ipynb')
# create a text file
with io.open(pjoin(nbdir, d, '%s.txt' % name),
'w',
encoding='utf-8') as f:
f.write(self._txt_for_name(name))
# create a binary file
with io.open(pjoin(nbdir, d, '%s.blob' % name), 'wb') as f:
f.write(self._blob_for_name(name))
self.api = API(self.base_url())
def test_upload(self):
nb = new_notebook(name=u'ignored')
nbmodel = {'content': nb}
resp = self.nb_api.upload(u'Upload tést.ipynb',
path=u'å b',
body=json.dumps(nbmodel))
self._check_nb_created(resp, u'Upload tést.ipynb', u'å b')
def test_coalesce_replace_streams(self):
"""Are \\r characters handled?"""
outputs = [
nbformat.new_output(output_type="stream",
stream="stdout",
output_text="z"),
nbformat.new_output(output_type="stream",
stream="stdout",
output_text="\ra"),
nbformat.new_output(output_type="stream",
stream="stdout",
output_text="\nz\rb"),
nbformat.new_output(output_type="stream",
stream="stdout",
output_text="\nz"),
nbformat.new_output(output_type="stream",
stream="stdout",
output_text="\rc\n"),
nbformat.new_output(output_type="stream",
stream="stdout",
output_text="z\rz\rd")
]
cells = [
nbformat.new_code_cell(input="# None",
prompt_number=1,
outputs=outputs)
]
worksheets = [nbformat.new_worksheet(name="worksheet1", cells=cells)]
nb = nbformat.new_notebook(name="notebook1", worksheets=worksheets)
res = self.build_resources()
nb, res = coalesce_streams(nb, res)
outputs = nb.worksheets[0].cells[0].outputs
self.assertEqual(outputs[0].text, u'a\nb\nc\nd')
def test_coalesce_sequenced_streams(self):
"""Can the coalesce streams preprocessor merge a sequence of streams?"""
outputs = [nbformat.new_output(output_type="stream", stream="stdout", output_text="0"),
nbformat.new_output(
output_type="stream", stream="stdout", output_text="1"),
nbformat.new_output(
output_type="stream", stream="stdout", output_text="2"),
nbformat.new_output(
output_type="stream", stream="stdout", output_text="3"),
nbformat.new_output(
output_type="stream", stream="stdout", output_text="4"),
nbformat.new_output(
output_type="stream", stream="stdout", output_text="5"),
nbformat.new_output(
output_type="stream", stream="stdout", output_text="6"),
nbformat.new_output(output_type="stream", stream="stdout", output_text="7")]
cells = [nbformat.new_code_cell(
input="# None", prompt_number=1, outputs=outputs)]
worksheets = [nbformat.new_worksheet(name="worksheet1", cells=cells)]
nb = nbformat.new_notebook(name="notebook1", worksheets=worksheets)
res = self.build_resources()
nb, res = coalesce_streams(nb, res)
outputs = nb.worksheets[0].cells[0].outputs
self.assertEqual(outputs[0].text, u'01234567')
def test_clear_metadata(self):
"""Is the metadata properly cleared?"""
nb = new_notebook()
nb.metadata['disable_assignment_toolbar'] = True
nb.metadata['hide_autograder_cells'] = True
self.preprocessor._clear_metadata(nb)
assert 'disable_assignment_toolbar' not in nb.metadata
assert 'hide_autograder_cells' not in nb.metadata
def new_notebook_files(name='default.ipynb'):
# make a new file ugh
metadata = current.new_metadata(name=name)
nb = current.new_notebook(metadata=metadata)
content = current.writes(nb, format=u'json')
file = github.InputFileContent(content)
files = {name: file}
return files
def mini_markdown_nb(markdown):
"create a single text cell notebook with markdown in it"
nb = new_notebook()
wks = new_worksheet()
cell = new_text_cell('markdown', source=markdown)
nb['worksheets'].append(wks)
nb['worksheets'][0]['cells'].append(cell)
return nb
def new_notebook(self):
"""Create a new notebook and return its notebook_id."""
path, name = self.increment_filename('Untitled')
notebook_id = self.new_notebook_id(name)
metadata = current.new_metadata(name=name)
nb = current.new_notebook(metadata=metadata)
with open(path,'w') as f:
current.write(nb, f, u'json')
return notebook_id
def new_notebook(self):
"""
Create an empty notebook and push it into the workspace without an object_id
or name, so that the WSS generates a new object ID for us. Then return
that.
This will likely only be called as a developer tool from http://<base url>/narrative
or from starting up locally.
"""
wsclient = self.wsclient()
user_id = self.get_userid()
# Verify that our own home workspace exists, note that we aren't doing this
# as a general thing for other workspaces
#
# This is needed for running locally - a workspace is required.
try:
(homews, homews_id) = ws_util.check_homews(wsclient, user_id)
except Exception as e:
raise web.HTTPError(401, u'User must be logged in to access their workspaces')
# Have IPython create a new, empty notebook
nb = current.new_notebook()
new_name = normalize('NFC', u"Untitled %s" % (datetime.datetime.now().strftime("%y%m%d_%H%M%S")))
new_name = self._clean_id(new_name)
# Add in basic metadata to the ipynb object
try:
nb.metadata.ws_name = os.environ.get('KB_WORKSPACE_ID', homews)
nb.metadata.creator = user_id
nb.metadata.type = self.ws_type
nb.metadata.description = ''
nb.metadata.name = new_name
nb.metadata.data_dependencies = []
nb.metadata.format = self.node_format
except Exception as e:
raise web.HTTPError(400, u'Unexpected error setting notebook attributes: %s' %e)
try:
wsobj = {
'type' : self.ws_type,
'data' : nb,
'provenance' : [],
'meta' : nb.metadata.copy(),
}
wsid = homews_id
wsobj['meta']['data_dependencies'] = json.dumps(wsobj['meta']['data_dependencies'])
self.log.debug("calling ws_util.put_wsobj")
res = ws_util.put_wsobj(wsclient, wsid, wsobj)
self.log.debug("save_object returned %s" % res)
except Exception as e:
raise web.HTTPError(500, u'%s saving Narrative: %s' % (type(e),e))
# use "ws.ws_id.obj.object_id" as the identifier
id = "ws.%s.obj.%s" % (res['wsid'], res['objid'])
self._set_narrative_env(id)
# update the mapping
self.list_notebooks()
return id
def setUp(self):
nbdir = self.notebook_dir.name
os.mkdir(pjoin(nbdir, 'foo'))
with io.open(pjoin(nbdir, 'foo', 'nb1.ipynb'), 'w') as f:
nb = new_notebook(name='nb1')
write(nb, f, format='ipynb')
self.sess_api = SessionAPI(self.base_url())
def new_notebook(self):
"""Create a new notebook and return its notebook_id."""
path, name = self.increment_filename('Untitled')
notebook_id = self.new_notebook_id(name)
metadata = current.new_metadata(name=name)
nb = current.new_notebook(metadata=metadata)
with open(path, 'w') as f:
current.write(nb, f, u'json')
return notebook_id
def convert_normal_cells(normal_cells):
""" Convert normal_cells into html. """
for cell in normal_cells:
if cell.cell_type == 'markdown':
cell.source = re.sub(r'\\begin\{ *equation *\}', '\[', cell.source)
cell.source = re.sub(r'\\end\{ *equation *\}', '\]', cell.source)
worksheet = current.new_worksheet(cells=normal_cells)
tmp = current.new_notebook(worksheets=[worksheet])
html = export_unit_to_html(tmp)
return html
def __init__(self, document):
nodes.NodeVisitor.__init__(self, document)
self.settings = settings = document.settings
lcode = settings.language_code
self.language = languages.get_language(lcode, document.reporter)
# A heterogenous stack used in conjunction with the tree traversal.
# Make sure that the pops correspond to the pushes:
self.context = []
self.body = []
ws = nbformat.new_worksheet()
self.nb = nbformat.new_notebook(worksheets=[ws])
def new_notebook(self): """Create a new notebook and return its notebook_id.""" n = Notebook() n.id = str(uuid.uuid4()) n.name = 'New Notebook' metadata = current.new_metadata(name=n.name) nb = current.new_notebook(metadata=metadata) data = current.writes(nb, u'json') n.content = data n.save(force_insert=True) return n.id
def __init__(self, document):
nodes.NodeVisitor.__init__(self, document)
self.settings = settings = document.settings
lcode = settings.language_code
self.language = languages.get_language(lcode, document.reporter)
# A heterogenous stack used in conjunction with the tree traversal.
# Make sure that the pops correspond to the pushes:
self.context = []
self.body = []
self.section_level = 0
ws = nbformat.new_worksheet()
self.nb = nbformat.new_notebook(worksheets=[ws])
def build_notebook(self):
"""Build a reveal slides notebook in memory for use with tests.
Overrides base in TransformerTestsBase"""
outputs = [nbformat.new_output(output_type="svg", output_svg=self.simple_svg)]
slide_metadata = {'slideshow' : {'slide_type': 'slide'}}
subslide_metadata = {'slideshow' : {'slide_type': 'subslide'}}
cells=[nbformat.new_code_cell(input="", prompt_number=1, outputs=outputs)]
worksheets = [nbformat.new_worksheet(name="worksheet1", cells=cells)]
return nbformat.new_notebook(name="notebook1", worksheets=worksheets)
def build_notebook(self):
"""Build a reveal slides notebook in memory for use with tests.
Overrides base in PreprocessorTestsBase"""
outputs = [nbformat.new_output(output_type="svg", output_svg=self.simple_svg)]
slide_metadata = {'slideshow' : {'slide_type': 'slide'}}
subslide_metadata = {'slideshow' : {'slide_type': 'subslide'}}
cells=[nbformat.new_code_cell(input="", prompt_number=1, outputs=outputs)]
worksheets = [nbformat.new_worksheet(name="worksheet1", cells=cells)]
return nbformat.new_notebook(name="notebook1", worksheets=worksheets)
def create_notebook_model(self, model=None, path=''):
"""Create a new notebook and return its model with no content."""
path = path.strip('/')
if model is None:
model = {}
if 'content' not in model:
metadata = current.new_metadata(name=u'')
model['content'] = current.new_notebook(metadata=metadata)
if 'name' not in model:
model['name'] = self.increment_filename('Untitled', path)
model['path'] = path
model = self.save_notebook_model(model, model['name'], model['path'])
return model
def setUp(self):
nbdir = self.notebook_dir.name
try:
os.mkdir(pjoin(nbdir, "foo"))
except OSError as e:
# Deleting the folder in an earlier test may have failed
if e.errno != errno.EEXIST:
raise
with io.open(pjoin(nbdir, "foo", "nb1.ipynb"), "w", encoding="utf-8") as f:
nb = new_notebook(name="nb1")
write(nb, f, format="ipynb")
self.sess_api = SessionAPI(self.base_url())
def setUp(self):
nbdir = self.notebook_dir.name
try:
os.mkdir(pjoin(nbdir, 'foo'))
except OSError as e:
# Deleting the folder in an earlier test may have failed
if e.errno != errno.EEXIST:
raise
with io.open(pjoin(nbdir, 'foo', 'nb1.ipynb'), 'w') as f:
nb = new_notebook(name='nb1')
write(nb, f, format='ipynb')
self.sess_api = SessionAPI(self.base_url())
def setUp(self):
nbdir = self.notebook_dir.name
try:
os.mkdir(pjoin(nbdir, 'foo'))
except OSError as e:
# Deleting the folder in an earlier test may have failed
if e.errno != errno.EEXIST:
raise
with io.open(pjoin(nbdir, 'foo', 'nb1.ipynb'), 'w') as f:
nb = new_notebook(name='nb1')
write(nb, f, format='ipynb')
self.sess_api = SessionAPI(self.base_url())
def create_notebook_model(self, model=None, path=''):
"""Create a new notebook and return its model with no content."""
path = path.strip('/')
if model is None:
model = {}
if 'content' not in model:
metadata = current.new_metadata(name=u'')
model['content'] = current.new_notebook(metadata=metadata)
if 'name' not in model:
model['name'] = self.increment_filename('Untitled', path)
model['path'] = path
model = self.save_notebook_model(model, model['name'], model['path'])
return model
def setUp(self):
nbdir = self.notebook_dir.name
for d in self.dirs:
d.replace('/', os.sep)
if not os.path.isdir(pjoin(nbdir, d)):
os.mkdir(pjoin(nbdir, d))
for d, name in self.dirs_nbs:
d = d.replace('/', os.sep)
with io.open(pjoin(nbdir, d, '%s.ipynb' % name), 'w') as f:
nb = new_notebook(name=name)
write(nb, f, format='ipynb')
self.nb_api = NBAPI(self.base_url())
def test_run_nb(self):
"""Test %run notebook.ipynb"""
from IPython.nbformat import current
nb = current.new_notebook(worksheets=[
current.new_worksheet(cells=[
current.new_text_cell("The Ultimate Question of Everything"),
current.new_code_cell("answer=42")
])
])
src = current.writes(nb, 'json')
self.mktmp(src, ext='.ipynb')
_ip.magic("run %s" % self.fname)
nt.assert_equal(_ip.user_ns['answer'], 42)
def setUp(self):
nbdir = self.notebook_dir.name
for d in self.dirs:
d.replace('/', os.sep)
if not os.path.isdir(pjoin(nbdir, d)):
os.mkdir(pjoin(nbdir, d))
for d, name in self.dirs_nbs:
d = d.replace('/', os.sep)
with io.open(pjoin(nbdir, d, '%s.ipynb' % name), 'w') as f:
nb = new_notebook(name=name)
write(nb, f, format='ipynb')
self.nb_api = NBAPI(self.base_url())
def build_notebook(self):
"""Build a notebook in memory for use with preprocessor tests"""
outputs = [nbformat.new_output(output_type="stream", stream="stdout", output_text="a"),
nbformat.new_output(output_type="text", output_text="b"),
nbformat.new_output(output_type="stream", stream="stdout", output_text="c"),
nbformat.new_output(output_type="stream", stream="stdout", output_text="d"),
nbformat.new_output(output_type="stream", stream="stderr", output_text="e"),
nbformat.new_output(output_type="stream", stream="stderr", output_text="f"),
nbformat.new_output(output_type="png", output_png='Zw==')] #g
cells=[nbformat.new_code_cell(input="$ e $", prompt_number=1,outputs=outputs),
nbformat.new_text_cell('markdown', source="$ e $")]
worksheets = [nbformat.new_worksheet(name="worksheet1", cells=cells)]
return nbformat.new_notebook(name="notebook1", worksheets=worksheets)
def new_notebook(self):
"""Create a new notebook and returns its notebook_id."""
i = 0
while True:
name = u'Untitled%i' % i
path = self.get_path_by_name(name)
if not os.path.isfile(path):
break
else:
i = i + 1
notebook_id = self.new_notebook_id(name)
metadata = current.new_metadata(name=name)
nb = current.new_notebook(metadata=metadata)
with open(path, 'w') as f:
current.write(nb, f, u'json')
return notebook_id
def test_run_nb(self):
"""Test %run notebook.ipynb"""
from IPython.nbformat import current
nb = current.new_notebook(
worksheets=[
current.new_worksheet(cells=[
current.new_code_cell("answer=42")
])
]
)
src = current.writes(nb, 'json')
self.mktmp(src, ext='.ipynb')
_ip.magic("run %s" % self.fname)
nt.assert_equal(_ip.user_ns['answer'], 42)
def test_coalesce_replace_streams(self):
"""Are \\r characters handled?"""
outputs = [nbformat.new_output(output_type="stream", stream="stdout", output_text="z"),
nbformat.new_output(output_type="stream", stream="stdout", output_text="\ra"),
nbformat.new_output(output_type="stream", stream="stdout", output_text="\nz\rb"),
nbformat.new_output(output_type="stream", stream="stdout", output_text="\nz"),
nbformat.new_output(output_type="stream", stream="stdout", output_text="\rc\n"),
nbformat.new_output(output_type="stream", stream="stdout", output_text="z\rz\rd")]
cells=[nbformat.new_code_cell(input="# None", prompt_number=1,outputs=outputs)]
worksheets = [nbformat.new_worksheet(name="worksheet1", cells=cells)]
nb = nbformat.new_notebook(name="notebook1", worksheets=worksheets)
res = self.build_resources()
nb, res = coalesce_streams(nb, res)
outputs = nb.worksheets[0].cells[0].outputs
self.assertEqual(outputs[0].text, u'a\nb\nc\nd')
def new_notebook(self):
"""Create a new notebook and returns its notebook_id."""
i = 0
while True:
name = u"Untitled%i" % i
path = self.get_path_by_name(name)
if not os.path.isfile(path):
break
else:
i = i + 1
notebook_id = self.new_notebook_id(name)
metadata = current.new_metadata(name=name)
nb = current.new_notebook(metadata=metadata)
with open(path, "w") as f:
current.write(nb, f, u"json")
return notebook_id
def build_notebook(self):
"""Build a reveal slides notebook in memory for use with tests.
Overrides base in TransformerTestsBase"""
outputs = [nbformat.new_output(output_type="stream", stream="stdout", output_text="a")]
slide_metadata = {'slideshow' : {'slide_type': 'slide'}}
subslide_metadata = {'slideshow' : {'slide_type': 'subslide'}}
cells=[nbformat.new_code_cell(input="", prompt_number=1, outputs=outputs),
nbformat.new_text_cell('markdown', source="", metadata=slide_metadata),
nbformat.new_code_cell(input="", prompt_number=2, outputs=outputs),
nbformat.new_text_cell('markdown', source="", metadata=slide_metadata),
nbformat.new_text_cell('markdown', source="", metadata=subslide_metadata)]
worksheets = [nbformat.new_worksheet(name="worksheet1", cells=cells)]
return nbformat.new_notebook(name="notebook1", worksheets=worksheets)
def test_coalesce_sequenced_streams(self):
"""Can the coalesce streams preprocessor merge a sequence of streams?"""
outputs = [nbformat.new_output(output_type="stream", stream="stdout", output_text="0"),
nbformat.new_output(output_type="stream", stream="stdout", output_text="1"),
nbformat.new_output(output_type="stream", stream="stdout", output_text="2"),
nbformat.new_output(output_type="stream", stream="stdout", output_text="3"),
nbformat.new_output(output_type="stream", stream="stdout", output_text="4"),
nbformat.new_output(output_type="stream", stream="stdout", output_text="5"),
nbformat.new_output(output_type="stream", stream="stdout", output_text="6"),
nbformat.new_output(output_type="stream", stream="stdout", output_text="7")]
cells=[nbformat.new_code_cell(input="# None", prompt_number=1,outputs=outputs)]
worksheets = [nbformat.new_worksheet(cells=cells)]
nb = nbformat.new_notebook(name="notebook1", worksheets=worksheets)
res = self.build_resources()
nb, res = coalesce_streams(nb, res)
outputs = nb.worksheets[0].cells[0].outputs
self.assertEqual(outputs[0].text, u'01234567')
def notebook(self, s):
"""Export and convert IPython notebooks.
This function can export the current IPython history to a notebook file
or can convert an existing notebook file into a different format. For
example, to export the history to "foo.ipynb" do "%notebook -e foo.ipynb".
To export the history to "foo.py" do "%notebook -e foo.py". To convert
"foo.ipynb" to "foo.json" do "%notebook -f json foo.ipynb". Possible
formats include (json/ipynb, py).
"""
args = magic_arguments.parse_argstring(self.notebook, s)
from IPython.nbformat import current
args.filename = unquote_filename(args.filename)
if args.export:
fname, name, format = current.parse_filename(args.filename)
cells = []
hist = list(self.shell.history_manager.get_range())
for session, prompt_number, input in hist[:-1]:
cells.append(
current.new_code_cell(prompt_number=prompt_number,
input=input))
worksheet = current.new_worksheet(cells=cells)
nb = current.new_notebook(name=name, worksheets=[worksheet])
with io.open(fname, 'w', encoding='utf-8') as f:
current.write(nb, f, format)
elif args.format is not None:
old_fname, old_name, old_format = current.parse_filename(
args.filename)
new_format = args.format
if new_format == u'xml':
raise ValueError('Notebooks cannot be written as xml.')
elif new_format == u'ipynb' or new_format == u'json':
new_fname = old_name + u'.ipynb'
new_format = u'json'
elif new_format == u'py':
new_fname = old_name + u'.py'
else:
raise ValueError('Invalid notebook format: %s' % new_format)
with io.open(old_fname, 'r', encoding='utf-8') as f:
nb = current.read(f, old_format)
with io.open(new_fname, 'w', encoding='utf-8') as f:
current.write(nb, f, new_format)
def write_notebook(flow, options):
"""See http://nbviewer.ipython.org/gist/fperez/9716279"""
from IPython.nbformat import current as nbf
nb = nbf.new_notebook()
cells = [
#nbf.new_text_cell('heading', "This is an auto-generated notebook for %s" % os.path.basename(pseudopath)),
nbf.new_code_cell("""\
##%%javascript
##IPython.OutputArea.auto_scroll_threshold = 9999;
from __future__ import print_function
from abipy import abilab
%matplotlib inline
mpld3 = abilab.mpld3_enable_notebook()
import pylab
pylab.rcParams['figure.figsize'] = (25.0, 10.0)
import seaborn as sns
#sns.set(style="dark", palette="Set2")
sns.set(style='ticks', palette='Set2')"""),
nbf.new_code_cell("flow = abilab.Flow.pickle_load('%s')" % flow.workdir),
nbf.new_code_cell("flow.show_dependencies()"),
nbf.new_code_cell("flow.check_status(show=True, verbose=0)"),
nbf.new_code_cell("flow.show_inputs(nids=None, wslice=None)"),
nbf.new_code_cell("flow.inspect(nids=None, wslice=None)"),
nbf.new_code_cell("flow.show_abierrors()"),
nbf.new_code_cell("flow.show_qouts()"),
]
# Now that we have the cells, we can make a worksheet with them and add it to the notebook:
nb['worksheets'].append(nbf.new_worksheet(cells=cells))
# Next, we write it to a file on disk that we can then open as a new notebook.
# Note: This should be as easy as: nbf.write(nb, fname), but the current api is a little more verbose and needs a real file-like object.
import tempfile
_, tmpfname = tempfile.mkstemp(suffix='.ipynb', text=True)
with open(tmpfname, 'w') as fh:
nbf.write(nb, fh, 'ipynb')
os.system("ipython notebook %s" % tmpfname)
def __init__(self):
self.fxns = '''
# functions used throughout analysis
from __future__ import division
from IPython.core.display import HTML
import numpy as np
import pandas as pd
from collections import Counter
from scipy.stats import mode
import sys
sys.path.append('/home/ubuntu/workspace/pydev/')
from CrossTabUtils import *
pd.set_option('display.max_colwidth', 1)
'''
self.nb = nbf.new_notebook()
self.cells = []
cell = nbf.new_code_cell(self.fxns)
self.cells.append(cell)
def test_contents_manager(self):
"make sure ContentsManager returns right files (ipynb, bin, txt)."
nbdir = self.notebook_dir.name
base = self.base_url()
nb = new_notebook(name='testnb')
ws = new_worksheet()
nb.worksheets = [ws]
ws.cells.append(new_heading_cell(u'Created by test ³'))
cc1 = new_code_cell(input=u'print(2*6)')
cc1.outputs.append(new_output(output_text=u'12', output_type='stream'))
ws.cells.append(cc1)
with io.open(pjoin(nbdir, 'testnb.ipynb'), 'w', encoding='utf-8') as f:
write(nb, f, format='ipynb')
with io.open(pjoin(nbdir, 'test.bin'), 'wb') as f:
f.write(b'\xff' + os.urandom(5))
f.close()
with io.open(pjoin(nbdir, 'test.txt'), 'w') as f:
f.write(u'foobar')
f.close()
r = requests.get(url_path_join(base, 'files', 'testnb.ipynb'))
self.assertEqual(r.status_code, 200)
self.assertIn('print(2*6)', r.text)
json.loads(r.text)
r = requests.get(url_path_join(base, 'files', 'test.bin'))
self.assertEqual(r.status_code, 200)
self.assertEqual(r.headers['content-type'], 'application/octet-stream')
self.assertEqual(r.content[:1], b'\xff')
self.assertEqual(len(r.content), 6)
r = requests.get(url_path_join(base, 'files', 'test.txt'))
self.assertEqual(r.status_code, 200)
self.assertEqual(r.headers['content-type'], 'text/plain')
self.assertEqual(r.text, 'foobar')
def new_notebook_from_string(notebookname, filename, sourcestring):
root = ast.parse(sourcestring, filename=filename, mode='exec')
x = DetermineBlocks()
for child in ast.iter_child_nodes(root):
print(child.lineno, child)
x.visit(child)
x.end()
sourcelines = sourcestring.splitlines()
cells = []
for block in x.blocks:
print(block)
blocklines = sourcelines[block[1] - 1:block[2]]
blocksrc = '\n'.join(blocklines)
if len(blocksrc) > 0:
cell = notebook_format.new_code_cell(input=blocksrc)
cells.append(cell)
ws = notebook_format.new_worksheet(cells=cells)
result = notebook_format.new_notebook(worksheets=[ws])
result.metadata.name = notebookname
return result
def setUp(self):
nbdir = self.notebook_dir.name
if not os.path.isdir(pjoin(nbdir, 'foo')):
os.mkdir(pjoin(nbdir, 'foo'))
nb = new_notebook(name='testnb')
ws = new_worksheet()
nb.worksheets = [ws]
ws.cells.append(new_heading_cell(u'Created by test ³'))
cc1 = new_code_cell(input=u'print(2*6)')
cc1.outputs.append(new_output(output_text=u'12'))
cc1.outputs.append(
new_output(output_png=png_green_pixel, output_type='pyout'))
ws.cells.append(cc1)
with io.open(pjoin(nbdir, 'foo', 'testnb.ipynb'),
'w',
encoding='utf-8') as f:
write(nb, f, format='ipynb')
self.nbconvert_api = NbconvertAPI(self.base_url())
def parse(self, pdj, name=''):
meta, body = pdj
self._markdown_buffer = ''
self._init_cells()
for cell in body:
t, c = cell['t'], cell['c']
if t == 'Header':
self.process_header(c)
continue
elif t == 'CodeBlock':
self.process_codeblock(c)
continue
elif t in ('Para', 'Plain'):
res = self.process_inline(c)
elif t == 'RawBlock':
res = c[1]
elif t == 'BlockQuote':
res = self.process_blockquote(c)
elif t == 'BulletList':
res = self.process_bulletlist(c)
elif t == 'OrderedList':
res = self.process_orderedlist(c)
elif t == 'Table':
res = self.process_table(c)
elif t == 'DefinitionList':
res = self.process_definitionlist(c)
else:
raise ValueError('Not yet for ' + t)
if res != '':
self._markdown_buffer = '\n'.join((self._markdown_buffer, res))
self.flush_markdown()
self._post_cells()
nb = nbf.new_notebook(name=name)
ws = nbf.new_worksheet()
ws['cells'] += self._cells[:]
nb['worksheets'].append(ws)
return nb
def new_notebook(metadata, filepath):
content = current.new_notebook(metadata=metadata)
nb = current.to_notebook_json(content)
with io.open(filepath, 'w', encoding='utf-8') as f:
current.write(nb, f, u'json')
