def test_save(self):
resp = self.api.read('a.ipynb', 'foo')
nbcontent = json.loads(resp.text)['content']
nb = from_dict(nbcontent)
nb.cells.append(new_markdown_cell(u'Created by test ³'))
nbmodel= {'name': 'a.ipynb', 'path':'foo', 'content': nb, 'type': 'notebook'}
resp = self.api.save('a.ipynb', path='foo', body=json.dumps(nbmodel))
nbfile = pjoin(self.notebook_dir.name, 'foo', 'a.ipynb')
with io.open(nbfile, 'r', encoding='utf-8') as f:
newnb = read(f, as_version=4)
self.assertEqual(newnb.cells[0].source,
u'Created by test ³')
nbcontent = self.api.read('a.ipynb', 'foo').json()['content']
newnb = from_dict(nbcontent)
self.assertEqual(newnb.cells[0].source,
u'Created by test ³')
# Save and rename
nbmodel= {'name': 'a2.ipynb', 'path':'foo/bar', 'content': nb, 'type': 'notebook'}
resp = self.api.save('a.ipynb', path='foo', body=json.dumps(nbmodel))
saved = resp.json()
self.assertEqual(saved['name'], 'a2.ipynb')
self.assertEqual(saved['path'], 'foo/bar')
assert os.path.isfile(pjoin(self.notebook_dir.name,'foo','bar','a2.ipynb'))
assert not os.path.isfile(pjoin(self.notebook_dir.name, 'foo', 'a.ipynb'))
with assert_http_error(404):
self.api.read('a.ipynb', 'foo')
def test_very_long_cells(self):
"""
Torture test that long cells do not cause issues
"""
lorem_ipsum_text = textwrap.dedent("""\
Lorem ipsum dolor sit amet, consectetur adipiscing elit. Donec
dignissim, ipsum non facilisis tempus, dui felis tincidunt metus,
nec pulvinar neque odio eget risus. Nulla nisi lectus, cursus
suscipit interdum at, ultrices sit amet orci. Mauris facilisis
imperdiet elit, vitae scelerisque ipsum dignissim non. Integer
consequat malesuada neque sit amet pulvinar. Curabitur pretium
ut turpis eget aliquet. Maecenas sagittis lacus sed lectus
volutpat, eu adipiscing purus pulvinar. Maecenas consequat
luctus urna, eget cursus quam mollis a. Aliquam vitae ornare
erat, non hendrerit urna. Sed eu diam nec massa egestas pharetra
at nec tellus. Fusce feugiat lacus quis urna sollicitudin volutpat.
Quisque at sapien non nibh feugiat tempus ac ultricies purus.
""")
lorem_ipsum_text = lorem_ipsum_text.replace("\n"," ") + "\n\n"
large_lorem_ipsum_text = "".join([lorem_ipsum_text]*3000)
notebook_name = "lorem_ipsum_long.ipynb"
nb = v4.new_notebook(
cells=[
v4.new_markdown_cell(source=large_lorem_ipsum_text)
]
)
with TemporaryDirectory() as td:
nbfile = os.path.join(td, notebook_name)
with open(nbfile, 'w') as f:
write(nb, f, 4)
(output, resources) = LatexExporter(template_file='article').from_filename(nbfile)
assert len(output) > 0
def test_checkpoints_follow_file(self):
# Read initial file state
orig = self.api.read('foo/a.ipynb')
# Create a checkpoint of initial state
r = self.api.new_checkpoint('foo/a.ipynb')
cp1 = r.json()
# Modify file and save
nbcontent = json.loads(orig.text)['content']
nb = from_dict(nbcontent)
hcell = new_markdown_cell('Created by test')
nb.cells.append(hcell)
nbmodel = {'content': nb, 'type': 'notebook'}
self.api.save('foo/a.ipynb', body=json.dumps(nbmodel))
# Rename the file.
self.api.rename('foo/a.ipynb', 'foo/z.ipynb')
# Looking for checkpoints in the old location should yield no results.
self.assertEqual(self.api.get_checkpoints('foo/a.ipynb').json(), [])
# Looking for checkpoints in the new location should work.
cps = self.api.get_checkpoints('foo/z.ipynb').json()
self.assertEqual(cps, [cp1])
# Delete the file. The checkpoint should be deleted as well.
self.api.delete('foo/z.ipynb')
cps = self.api.get_checkpoints('foo/z.ipynb').json()
self.assertEqual(cps, [])
def test_very_long_cells(self):
"""
Torture test that long cells do not cause issues
"""
lorem_ipsum_text = textwrap.dedent("""\
Lorem ipsum dolor sit amet, consectetur adipiscing elit. Donec
dignissim, ipsum non facilisis tempus, dui felis tincidunt metus,
nec pulvinar neque odio eget risus. Nulla nisi lectus, cursus
suscipit interdum at, ultrices sit amet orci. Mauris facilisis
imperdiet elit, vitae scelerisque ipsum dignissim non. Integer
consequat malesuada neque sit amet pulvinar. Curabitur pretium
ut turpis eget aliquet. Maecenas sagittis lacus sed lectus
volutpat, eu adipiscing purus pulvinar. Maecenas consequat
luctus urna, eget cursus quam mollis a. Aliquam vitae ornare
erat, non hendrerit urna. Sed eu diam nec massa egestas pharetra
at nec tellus. Fusce feugiat lacus quis urna sollicitudin volutpat.
Quisque at sapien non nibh feugiat tempus ac ultricies purus.
""")
lorem_ipsum_text = lorem_ipsum_text.replace("\n", " ") + "\n\n"
large_lorem_ipsum_text = "".join([lorem_ipsum_text] * 3000)
notebook_name = "lorem_ipsum_long.ipynb"
nb = v4.new_notebook(
cells=[v4.new_markdown_cell(source=large_lorem_ipsum_text)])
with TemporaryDirectory() as td:
nbfile = os.path.join(td, notebook_name)
with open(nbfile, 'w') as f:
write(nb, f, 4)
(output, resources) = LatexExporter(
template_file='article').from_filename(nbfile)
assert len(output) > 0
def test_checkpoints_follow_file(self):
# Read initial file state
orig = self.api.read('foo/a.ipynb')
# Create a checkpoint of initial state
r = self.api.new_checkpoint('foo/a.ipynb')
cp1 = r.json()
# Modify file and save
nbcontent = json.loads(orig.text)['content']
nb = from_dict(nbcontent)
hcell = new_markdown_cell('Created by test')
nb.cells.append(hcell)
nbmodel = {'content': nb, 'type': 'notebook'}
self.api.save('foo/a.ipynb', body=json.dumps(nbmodel))
# Rename the file.
self.api.rename('foo/a.ipynb', 'foo/z.ipynb')
# Looking for checkpoints in the old location should yield no results.
self.assertEqual(self.api.get_checkpoints('foo/a.ipynb').json(), [])
# Looking for checkpoints in the new location should work.
cps = self.api.get_checkpoints('foo/z.ipynb').json()
self.assertEqual(cps, [cp1])
# Delete the file. The checkpoint should be deleted as well.
self.api.delete('foo/z.ipynb')
cps = self.api.get_checkpoints('foo/z.ipynb').json()
self.assertEqual(cps, [])
def build_notebook(self):
"""Build a reveal slides notebook in memory for use with tests.
Overrides base in PreprocessorTestsBase"""
outputs = [nbformat.new_output(output_type="stream", name="stdout", text="a")]
slide_metadata = {'slideshow' : {'slide_type': 'slide'}}
subslide_metadata = {'slideshow' : {'slide_type': 'subslide'}}
cells=[nbformat.new_code_cell(source="", execution_count=1, outputs=outputs),
nbformat.new_markdown_cell(source="", metadata=slide_metadata),
nbformat.new_code_cell(source="", execution_count=2, outputs=outputs),
nbformat.new_markdown_cell(source="", metadata=slide_metadata),
nbformat.new_markdown_cell(source="", metadata=subslide_metadata)]
return nbformat.new_notebook(cells=cells)
def setUp(self):
nbdir = self.notebook_dir.name
if not os.path.isdir(pjoin(nbdir, 'foo')):
os.mkdir(pjoin(nbdir, 'foo'))
nb = new_notebook()
nb.cells.append(new_markdown_cell(u'Created by test ³'))
cc1 = new_code_cell(source=u'print(2*6)')
cc1.outputs.append(new_output(output_type="stream", text=u'12'))
cc1.outputs.append(
new_output(
output_type="execute_result",
data={'image/png': png_green_pixel},
execution_count=1,
))
nb.cells.append(cc1)
with io.open(pjoin(nbdir, 'foo', 'testnb.ipynb'),
'w',
encoding='utf-8') as f:
write(nb, f, version=4)
self.nbconvert_api = NbconvertAPI(self.base_url())
def lines_to_notebook(lines, name=None):
"""
Convert the lines of an m file into an IPython notebook
Parameters
----------
lines : list
A list of strings. Each element is a line in the m file
Returns
-------
notebook : an IPython NotebookNode class instance, containing the
information required to create a file
"""
source = []
md = np.empty(len(lines), dtype=object)
new_cell = np.empty(len(lines), dtype=object)
for idx, l in enumerate(lines):
new_cell[idx], md[idx], this_source = format_line(l)
# Transitions between markdown and code and vice-versa merit a new
# cell, even if no newline, or "%%" is found. Make sure not to do this
# check for the very first line!
if idx > 1 and not new_cell[idx]:
if md[idx] != md[idx - 1]:
new_cell[idx] = True
source.append(this_source)
# This defines the breaking points between cells:
new_cell_idx = np.hstack([np.where(new_cell)[0], -1])
# Listify the sources:
cell_source = [
source[new_cell_idx[i]:new_cell_idx[i + 1]]
for i in range(len(new_cell_idx) - 1)
]
cell_md = [md[new_cell_idx[i]] for i in range(len(new_cell_idx) - 1)]
cells = []
# Append the notebook with loading matlab magic extension
notebook_head = "import pymatbridge as pymat\n" + "ip = get_ipython()\n" \
+ "pymat.load_ipython_extension(ip)"
cells.append(nbformat.new_code_cell(notebook_head)) #, language='python'))
for cell_idx, cell_s in enumerate(cell_source):
if cell_md[cell_idx]:
cells.append(nbformat.new_markdown_cell(cell_s))
else:
cell_s.insert(0, '%%matlab\n')
cells.append(
nbformat.new_code_cell(cell_s)) #, language='matlab'))
#ws = nbformat.new_worksheet(cells=cells)
notebook = nbformat.new_notebook(cells=cells)
return notebook
def add_markdown_cell(self, path):
# Load and update
model = self.contents.get(path=path)
model['content'].cells.append(
new_markdown_cell('Created by test: ' + path)
)
# Save and checkpoint again.
self.contents.save(model, path=path)
return model
def lines_to_notebook(lines, name=None):
"""
Convert the lines of an m file into an IPython notebook
Parameters
----------
lines : list
A list of strings. Each element is a line in the m file
Returns
-------
notebook : an IPython NotebookNode class instance, containing the
information required to create a file
"""
source = []
md = np.empty(len(lines), dtype=object)
new_cell = np.empty(len(lines), dtype=object)
for idx, l in enumerate(lines):
new_cell[idx], md[idx], this_source = format_line(l)
# Transitions between markdown and code and vice-versa merit a new
# cell, even if no newline, or "%%" is found. Make sure not to do this
# check for the very first line!
if idx>1 and not new_cell[idx]:
if md[idx] != md[idx-1]:
new_cell[idx] = True
source.append(this_source)
# This defines the breaking points between cells:
new_cell_idx = np.hstack([np.where(new_cell)[0], -1])
# Listify the sources:
cell_source = [source[new_cell_idx[i]:new_cell_idx[i+1]]
for i in range(len(new_cell_idx)-1)]
cell_md = [md[new_cell_idx[i]] for i in range(len(new_cell_idx)-1)]
cells = []
# Append the notebook with loading matlab magic extension
notebook_head = "import pymatbridge as pymat\n" + "ip = get_ipython()\n" \
+ "pymat.load_ipython_extension(ip)"
cells.append(nbformat.new_code_cell(notebook_head))#, language='python'))
for cell_idx, cell_s in enumerate(cell_source):
if cell_md[cell_idx]:
cells.append(nbformat.new_markdown_cell(cell_s))
else:
cell_s.insert(0, '%%matlab\n')
cells.append(nbformat.new_code_cell(cell_s))#, language='matlab'))
#ws = nbformat.new_worksheet(cells=cells)
notebook = nbformat.new_notebook(cells=cells)
return notebook
def test_save(self):
resp = self.api.read('foo/a.ipynb')
nbcontent = json.loads(resp.text)['content']
nb = from_dict(nbcontent)
nb.cells.append(new_markdown_cell(u'Created by test ³'))
nbmodel = {'content': nb, 'type': 'notebook'}
resp = self.api.save('foo/a.ipynb', body=json.dumps(nbmodel))
nbcontent = self.api.read('foo/a.ipynb').json()['content']
newnb = from_dict(nbcontent)
self.assertEqual(newnb.cells[0].source, u'Created by test ³')
def test_save(self):
resp = self.api.read('foo/a.ipynb')
nbcontent = json.loads(resp.text)['content']
nb = from_dict(nbcontent)
nb.cells.append(new_markdown_cell(u'Created by test ³'))
nbmodel= {'content': nb, 'type': 'notebook'}
resp = self.api.save('foo/a.ipynb', body=json.dumps(nbmodel))
nbcontent = self.api.read('foo/a.ipynb').json()['content']
newnb = from_dict(nbcontent)
self.assertEqual(newnb.cells[0].source,
u'Created by test ³')
def flush_cell(self):
if len(self._code_lines) > 0 and len(self._text_lines) > 0:
raise Exception('only text or only code can be flushed')
if len(self._code_lines) == 0 and len(self._text_lines) == 0:
raise Exception('nothing to flush')
if len(self._code_lines) > 0:
self.cells.append(nbf.new_code_cell('\n'.join(self._code_lines)))
else:
self.cells.append(nbf.new_markdown_cell('\n'.join(self._text_lines)))
self._code_lines = []
self._text_lines = []
def _create_solution_cell(source, cell_type):
if cell_type == "markdown":
cell = new_markdown_cell(source=source)
elif cell_type == "code":
cell = new_code_cell(source=source)
else:
raise ValueError("invalid cell type: {}".format(cell_type))
cell.metadata.nbgrader = {}
cell.metadata.nbgrader["solution"] = True
cell.metadata.nbgrader["checksum"] = compute_checksum(cell)
return cell
def create_locked_cell(source, cell_type, grade_id):
if cell_type == "markdown":
cell = new_markdown_cell(source=source)
elif cell_type == "code":
cell = new_code_cell(source=source)
else:
raise ValueError("invalid cell type: {}".format(cell_type))
cell.metadata.nbgrader = {}
cell.metadata.nbgrader["locked"] = True
cell.metadata.nbgrader["grade_id"] = grade_id
return cell
def create_locked_cell(source, cell_type, grade_id):
if cell_type == "markdown":
cell = new_markdown_cell(source=source)
elif cell_type == "code":
cell = new_code_cell(source=source)
else:
raise ValueError("invalid cell type: {}".format(cell_type))
cell.metadata.nbgrader = {}
cell.metadata.nbgrader["locked"] = True
cell.metadata.nbgrader["grade_id"] = grade_id
return cell
def test_run_nb(self):
"""Test %run notebook.ipynb"""
from IPython.nbformat import v4, writes
nb = v4.new_notebook(cells=[
v4.new_markdown_cell("The Ultimate Question of Everything"),
v4.new_code_cell("answer=42")
])
src = writes(nb, version=4)
self.mktmp(src, ext='.ipynb')
_ip.magic("run %s" % self.fname)
nt.assert_equal(_ip.user_ns['answer'], 42)
def flush_cell(self):
if len(self._code_lines) > 0 and len(self._text_lines) > 0:
raise Exception('only text or only code can be flushed')
if len(self._code_lines) == 0 and len(self._text_lines) == 0:
raise Exception('nothing to flush')
if len(self._code_lines) > 0:
self.cells.append(nbf.new_code_cell('\n'.join(self._code_lines)))
else:
self.cells.append(nbf.new_markdown_cell('\n'.join(self._text_lines)))
self._code_lines = []
self._text_lines = []
def test_checkpoint_all(self):
"""
Test that checkpoint_all correctly makes a checkpoint for all files.
"""
paths = populate(self.contents)
original_content_minus_trust = {
# Remove metadata that we expect to have dropped
path: strip_transient(self.contents.get(path)['content'])
for path in paths
}
original_cps = {}
for path in paths:
# Create a checkpoint, then update the file.
original_cps[path] = self.contents.create_checkpoint(path)
self.add_markdown_cell(path)
# Verify that we still have the old version checkpointed.
cp_content = {
path: self.checkpoints.get_notebook_checkpoint(
cp['id'],
path,
)['content']
for path, cp in iteritems(original_cps)
}
self.assertEqual(original_content_minus_trust, cp_content)
new_cps = checkpoint_all(
self.checkpoints.db_url,
self.td.name,
self.checkpoints.user_id,
)
new_cp_content = {
path: self.checkpoints.get_notebook_checkpoint(
cp['id'],
path,
)['content']
for path, cp in iteritems(new_cps)
}
for path, new_content in iteritems(new_cp_content):
old_content = original_content_minus_trust[_norm_unicode(path)]
self.assertEqual(
new_content['cells'][:-1],
old_content['cells'],
)
self.assertEqual(
new_content['cells'][-1],
new_markdown_cell('Created by test: ' + _norm_unicode(path)),
)
def create_grade_and_solution_cell(source, cell_type, grade_id, points):
if cell_type == "markdown":
cell = new_markdown_cell(source=source)
elif cell_type == "code":
cell = new_code_cell(source=source)
else:
raise ValueError("invalid cell type: {}".format(cell_type))
cell.metadata.nbgrader = {}
cell.metadata.nbgrader["solution"] = True
cell.metadata.nbgrader["grade"] = True
cell.metadata.nbgrader["grade_id"] = grade_id
cell.metadata.nbgrader["points"] = points
return cell
def create_grade_and_solution_cell(source, cell_type, grade_id, points):
if cell_type == "markdown":
cell = new_markdown_cell(source=source)
elif cell_type == "code":
cell = new_code_cell(source=source)
else:
raise ValueError("invalid cell type: {}".format(cell_type))
cell.metadata.nbgrader = {}
cell.metadata.nbgrader["solution"] = True
cell.metadata.nbgrader["grade"] = True
cell.metadata.nbgrader["grade_id"] = grade_id
cell.metadata.nbgrader["points"] = points
return cell
def test_run_nb(self):
"""Test %run notebook.ipynb"""
from IPython.nbformat import v4, writes
nb = v4.new_notebook(
cells=[
v4.new_markdown_cell("The Ultimate Question of Everything"),
v4.new_code_cell("answer=42")
]
)
src = writes(nb, version=4)
self.mktmp(src, ext='.ipynb')
_ip.magic("run %s" % self.fname)
nt.assert_equal(_ip.user_ns['answer'], 42)
def build_notebook(self):
"""Build a reveal slides notebook in memory for use with tests.
Overrides base in PreprocessorTestsBase"""
outputs = [
nbformat.new_output(output_type="stream", name="stdout", text="a")
]
slide_metadata = {'slideshow': {'slide_type': 'slide'}}
subslide_metadata = {'slideshow': {'slide_type': 'subslide'}}
cells = [
nbformat.new_code_cell(source="",
execution_count=1,
outputs=outputs),
nbformat.new_markdown_cell(source="", metadata=slide_metadata),
nbformat.new_code_cell(source="",
execution_count=2,
outputs=outputs),
nbformat.new_markdown_cell(source="", metadata=slide_metadata),
nbformat.new_markdown_cell(source="", metadata=subslide_metadata)
]
return nbformat.new_notebook(cells=cells)
def write(cells, nb_version=4):
"""Turn cells list into valid IPython notebook code."""
# Use IPython.nbformat functionality for writing the notebook
if nb_version == 3:
from IPython.nbformat.v3 import (
new_code_cell,
new_text_cell,
new_worksheet,
new_notebook,
new_metadata,
new_author,
)
nb = new_worksheet()
elif nb_version == 4:
from IPython.nbformat.v4 import new_code_cell, new_markdown_cell, new_notebook
nb_cells = []
for cell_tp, language, block in cells:
if cell_tp == "markdown":
if nb_version == 3:
nb.cells.append(new_text_cell(u"markdown", source=block))
elif nb_version == 4:
nb_cells.append(new_markdown_cell(source=block))
elif cell_tp == "codecell":
if nb_version == 3:
nb.cells.append(new_code_cell(input=block))
elif nb_version == 4:
nb_cells.append(new_code_cell(source=block))
if nb_version == 3:
nb = new_notebook(worksheets=[nb], metadata=new_metadata())
# Let us make v4 notebook here by upgrading
from IPython.nbformat.v4 import upgrade
nb = upgrade(nb)
import IPython.nbformat.v4.nbjson as nbjson
# Convert nb to json format
filestr = nbjson.writes(nb)
elif nb_version == 4:
nb = new_notebook(cells=nb_cells)
from IPython.nbformat import writes
filestr = writes(nb, version=4)
return filestr
def test_save(self):
resp = self.api.read('foo/a.ipynb')
nbcontent = json.loads(resp.text)['content']
nb = from_dict(nbcontent)
nb.cells.append(new_markdown_cell(u'Created by test ³'))
nbmodel = {'content': nb, 'type': 'notebook'}
resp = self.api.save('foo/a.ipynb', body=json.dumps(nbmodel))
nbfile = pjoin(self.notebook_dir.name, 'foo', 'a.ipynb')
with io.open(nbfile, 'r', encoding='utf-8') as f:
newnb = read(f, as_version=4)
self.assertEqual(newnb.cells[0].source, u'Created by test ³')
nbcontent = self.api.read('foo/a.ipynb').json()['content']
newnb = from_dict(nbcontent)
self.assertEqual(newnb.cells[0].source, u'Created by test ³')
def test_save(self):
resp = self.api.read("foo/a.ipynb")
nbcontent = json.loads(resp.text)["content"]
nb = from_dict(nbcontent)
nb.cells.append(new_markdown_cell(u"Created by test ³"))
nbmodel = {"content": nb, "type": "notebook"}
resp = self.api.save("foo/a.ipynb", body=json.dumps(nbmodel))
nbfile = pjoin(self.notebook_dir.name, "foo", "a.ipynb")
with io.open(nbfile, "r", encoding="utf-8") as f:
newnb = read(f, as_version=4)
self.assertEqual(newnb.cells[0].source, u"Created by test ³")
nbcontent = self.api.read("foo/a.ipynb").json()["content"]
newnb = from_dict(nbcontent)
self.assertEqual(newnb.cells[0].source, u"Created by test ³")
def write(cells):
"""Turn cells list into valid IPython notebook code."""
# Use IPython.nbformat functionality for writing the notebook
from IPython.nbformat.v4 import (
new_code_cell, new_markdown_cell, new_notebook)
nb_cells = []
for cell_tp, language, block in cells:
if cell_tp == 'markdown':
nb_cells.append(new_markdown_cell(source=block))
elif cell_tp == 'codecell':
nb_cells.append(new_code_cell(source=block))
nb = new_notebook(cells=nb_cells)
from IPython.nbformat import writes
filestr = writes(nb, version=4)
return filestr
def write(cells):
"""Turn cells list into valid IPython notebook code."""
# Use IPython.nbformat functionality for writing the notebook
from IPython.nbformat.v4 import (
new_code_cell, new_markdown_cell, new_notebook)
nb_cells = []
for cell_tp, language, block in cells:
if cell_tp == 'markdown':
nb_cells.append(new_markdown_cell(source=block))
elif cell_tp == 'codecell':
nb_cells.append(new_code_cell(source=block))
nb = new_notebook(cells=nb_cells)
from IPython.nbformat import writes
filestr = writes(nb, version=4)
return filestr
def test_save(self):
resp = self.api.read('foo/a.ipynb')
nbcontent = json.loads(resp.text)['content']
nb = from_dict(nbcontent)
nb.cells.append(new_markdown_cell(u'Created by test ³'))
nbmodel= {'content': nb, 'type': 'notebook'}
resp = self.api.save('foo/a.ipynb', body=json.dumps(nbmodel))
nbfile = pjoin(self.notebook_dir.name, 'foo', 'a.ipynb')
with io.open(nbfile, 'r', encoding='utf-8') as f:
newnb = read(f, as_version=4)
self.assertEqual(newnb.cells[0].source,
u'Created by test ³')
nbcontent = self.api.read('foo/a.ipynb').json()['content']
newnb = from_dict(nbcontent)
self.assertEqual(newnb.cells[0].source,
u'Created by test ³')
def build_notebook(self):
"""Build a notebook in memory for use with preprocessor tests"""
outputs = [
nbformat.new_output("stream", name="stdout", text="a"),
nbformat.new_output("display_data", data={'text/plain': 'b'}),
nbformat.new_output("stream", name="stdout", text="c"),
nbformat.new_output("stream", name="stdout", text="d"),
nbformat.new_output("stream", name="stderr", text="e"),
nbformat.new_output("stream", name="stderr", text="f"),
nbformat.new_output("display_data", data={'image/png': 'Zw=='}), # g
nbformat.new_output("display_data", data={'application/pdf': 'aA=='}), # h
]
cells=[nbformat.new_code_cell(source="$ e $", execution_count=1, outputs=outputs),
nbformat.new_markdown_cell(source="$ e $")]
return nbformat.new_notebook(cells=cells)
def test_contents_manager(self):
"make sure ContentsManager returns right files (ipynb, bin, txt)."
nbdir = self.notebook_dir.name
base = self.base_url()
nb = new_notebook(
cells=[
new_markdown_cell(u'Created by test ³'),
new_code_cell("print(2*6)", outputs=[
new_output("stream", text="12"),
])
]
)
with io.open(pjoin(nbdir, 'testnb.ipynb'), 'w',
encoding='utf-8') as f:
write(nb, f, version=4)
with io.open(pjoin(nbdir, 'test.bin'), 'wb') as f:
f.write(b'\xff' + os.urandom(5))
f.close()
with io.open(pjoin(nbdir, 'test.txt'), 'w') as f:
f.write(u'foobar')
f.close()
r = requests.get(url_path_join(base, 'files', 'testnb.ipynb'))
self.assertEqual(r.status_code, 200)
self.assertIn('print(2*6)', r.text)
json.loads(r.text)
r = requests.get(url_path_join(base, 'files', 'test.bin'))
self.assertEqual(r.status_code, 200)
self.assertEqual(r.headers['content-type'], 'application/octet-stream')
self.assertEqual(r.content[:1], b'\xff')
self.assertEqual(len(r.content), 6)
r = requests.get(url_path_join(base, 'files', 'test.txt'))
self.assertEqual(r.status_code, 200)
self.assertEqual(r.headers['content-type'], 'text/plain')
self.assertEqual(r.text, 'foobar')
def test_contents_manager(self):
"make sure ContentsManager returns right files (ipynb, bin, txt)."
nbdir = self.notebook_dir.name
base = self.base_url()
nb = new_notebook(
cells=[
new_markdown_cell(u'Created by test ³'),
new_code_cell("print(2*6)", outputs=[
new_output("stream", text="12"),
])
]
)
with io.open(pjoin(nbdir, 'testnb.ipynb'), 'w',
encoding='utf-8') as f:
write(nb, f, version=4)
with io.open(pjoin(nbdir, 'test.bin'), 'wb') as f:
f.write(b'\xff' + os.urandom(5))
f.close()
with io.open(pjoin(nbdir, 'test.txt'), 'w') as f:
f.write(u'foobar')
f.close()
r = requests.get(url_path_join(base, 'files', 'testnb.ipynb'))
self.assertEqual(r.status_code, 200)
self.assertIn('print(2*6)', r.text)
json.loads(r.text)
r = requests.get(url_path_join(base, 'files', 'test.bin'))
self.assertEqual(r.status_code, 200)
self.assertEqual(r.headers['content-type'], 'application/octet-stream')
self.assertEqual(r.content[:1], b'\xff')
self.assertEqual(len(r.content), 6)
r = requests.get(url_path_join(base, 'files', 'test.txt'))
self.assertEqual(r.status_code, 200)
self.assertEqual(r.headers['content-type'], 'text/plain')
self.assertEqual(r.text, 'foobar')
def setUp(self):
nbdir = self.notebook_dir.name
if not os.path.isdir(pjoin(nbdir, "foo")):
os.mkdir(pjoin(nbdir, "foo"))
nb = new_notebook()
nb.cells.append(new_markdown_cell(u"Created by test ³"))
cc1 = new_code_cell(source=u"print(2*6)")
cc1.outputs.append(new_output(output_type="stream", text=u"12"))
cc1.outputs.append(
new_output(output_type="execute_result", data={"image/png": png_green_pixel}, execution_count=1)
)
nb.cells.append(cc1)
with io.open(pjoin(nbdir, "foo", "testnb.ipynb"), "w", encoding="utf-8") as f:
write(nb, f, version=4)
self.nbconvert_api = NbconvertAPI(self.base_url())
def setUp(self):
nbdir = self.notebook_dir.name
if not os.path.isdir(pjoin(nbdir, 'foo')):
os.mkdir(pjoin(nbdir, 'foo'))
nb = new_notebook()
nb.cells.append(new_markdown_cell(u'Created by test ³'))
cc1 = new_code_cell(source=u'print(2*6)')
cc1.outputs.append(new_output(output_type="stream", text=u'12'))
cc1.outputs.append(new_output(output_type="execute_result",
data={'image/png': png_green_pixel},
execution_count=1,
))
nb.cells.append(cc1)
with io.open(pjoin(nbdir, 'foo', 'testnb.ipynb'), 'w',
encoding='utf-8') as f:
write(nb, f, version=4)
self.nbconvert_api = NbconvertAPI(self.base_url())
def write(cells, nb_version=4):
"""Turn cells list into valid IPython notebook code."""
# Use IPython.nbformat functionality for writing the notebook
if nb_version == 3:
from IPython.nbformat.v3 import (new_code_cell, new_text_cell,
new_worksheet, new_notebook,
new_metadata, new_author)
nb = new_worksheet()
elif nb_version == 4:
from IPython.nbformat.v4 import (new_code_cell, new_markdown_cell,
new_notebook)
nb_cells = []
for cell_tp, language, block in cells:
if cell_tp == 'markdown':
if nb_version == 3:
nb.cells.append(new_text_cell(u'markdown', source=block))
elif nb_version == 4:
nb_cells.append(new_markdown_cell(source=block))
elif cell_tp == 'codecell':
if nb_version == 3:
nb.cells.append(new_code_cell(input=block))
elif nb_version == 4:
nb_cells.append(new_code_cell(source=block))
if nb_version == 3:
nb = new_notebook(worksheets=[nb], metadata=new_metadata())
# Let us make v4 notebook here by upgrading
from IPython.nbformat.v4 import upgrade
nb = upgrade(nb)
import IPython.nbformat.v4.nbjson as nbjson
# Convert nb to json format
filestr = nbjson.writes(nb)
elif nb_version == 4:
nb = new_notebook(cells=nb_cells)
from IPython.nbformat import writes
filestr = writes(nb, version=4)
return filestr
def test_checkpoints(self):
resp = self.api.read("foo/a.ipynb")
r = self.api.new_checkpoint("foo/a.ipynb")
self.assertEqual(r.status_code, 201)
cp1 = r.json()
self.assertEqual(set(cp1), {"id", "last_modified"})
self.assertEqual(r.headers["Location"].split("/")[-1], cp1["id"])
# Modify it
nbcontent = json.loads(resp.text)["content"]
nb = from_dict(nbcontent)
hcell = new_markdown_cell("Created by test")
nb.cells.append(hcell)
# Save
nbmodel = {"content": nb, "type": "notebook"}
resp = self.api.save("foo/a.ipynb", body=json.dumps(nbmodel))
# List checkpoints
cps = self.api.get_checkpoints("foo/a.ipynb").json()
self.assertEqual(cps, [cp1])
nbcontent = self.api.read("foo/a.ipynb").json()["content"]
nb = from_dict(nbcontent)
self.assertEqual(nb.cells[0].source, "Created by test")
# Restore cp1
r = self.api.restore_checkpoint("foo/a.ipynb", cp1["id"])
self.assertEqual(r.status_code, 204)
nbcontent = self.api.read("foo/a.ipynb").json()["content"]
nb = from_dict(nbcontent)
self.assertEqual(nb.cells, [])
# Delete cp1
r = self.api.delete_checkpoint("foo/a.ipynb", cp1["id"])
self.assertEqual(r.status_code, 204)
cps = self.api.get_checkpoints("foo/a.ipynb").json()
self.assertEqual(cps, [])
def test_checkpoints(self):
resp = self.api.read('foo/a.ipynb')
r = self.api.new_checkpoint('foo/a.ipynb')
self.assertEqual(r.status_code, 201)
cp1 = r.json()
self.assertEqual(set(cp1), {'id', 'last_modified'})
self.assertEqual(r.headers['Location'].split('/')[-1], cp1['id'])
# Modify it
nbcontent = json.loads(resp.text)['content']
nb = from_dict(nbcontent)
hcell = new_markdown_cell('Created by test')
nb.cells.append(hcell)
# Save
nbmodel = {'content': nb, 'type': 'notebook'}
resp = self.api.save('foo/a.ipynb', body=json.dumps(nbmodel))
# List checkpoints
cps = self.api.get_checkpoints('foo/a.ipynb').json()
self.assertEqual(cps, [cp1])
nbcontent = self.api.read('foo/a.ipynb').json()['content']
nb = from_dict(nbcontent)
self.assertEqual(nb.cells[0].source, 'Created by test')
# Restore cp1
r = self.api.restore_checkpoint('foo/a.ipynb', cp1['id'])
self.assertEqual(r.status_code, 204)
nbcontent = self.api.read('foo/a.ipynb').json()['content']
nb = from_dict(nbcontent)
self.assertEqual(nb.cells, [])
# Delete cp1
r = self.api.delete_checkpoint('foo/a.ipynb', cp1['id'])
self.assertEqual(r.status_code, 204)
cps = self.api.get_checkpoints('foo/a.ipynb').json()
self.assertEqual(cps, [])
def test_checkpoints(self):
resp = self.api.read('foo/a.ipynb')
r = self.api.new_checkpoint('foo/a.ipynb')
self.assertEqual(r.status_code, 201)
cp1 = r.json()
self.assertEqual(set(cp1), {'id', 'last_modified'})
self.assertEqual(r.headers['Location'].split('/')[-1], cp1['id'])
# Modify it
nbcontent = json.loads(resp.text)['content']
nb = from_dict(nbcontent)
hcell = new_markdown_cell('Created by test')
nb.cells.append(hcell)
# Save
nbmodel= {'content': nb, 'type': 'notebook'}
resp = self.api.save('foo/a.ipynb', body=json.dumps(nbmodel))
# List checkpoints
cps = self.api.get_checkpoints('foo/a.ipynb').json()
self.assertEqual(cps, [cp1])
nbcontent = self.api.read('foo/a.ipynb').json()['content']
nb = from_dict(nbcontent)
self.assertEqual(nb.cells[0].source, 'Created by test')
# Restore cp1
r = self.api.restore_checkpoint('foo/a.ipynb', cp1['id'])
self.assertEqual(r.status_code, 204)
nbcontent = self.api.read('foo/a.ipynb').json()['content']
nb = from_dict(nbcontent)
self.assertEqual(nb.cells, [])
# Delete cp1
r = self.api.delete_checkpoint('foo/a.ipynb', cp1['id'])
self.assertEqual(r.status_code, 204)
cps = self.api.get_checkpoints('foo/a.ipynb').json()
self.assertEqual(cps, [])
def build_notebook(self):
"""Build a notebook in memory for use with preprocessor tests"""
outputs = [
nbformat.new_output("stream", name="stdout", text="a"),
nbformat.new_output("display_data", data={'text/plain': 'b'}),
nbformat.new_output("stream", name="stdout", text="c"),
nbformat.new_output("stream", name="stdout", text="d"),
nbformat.new_output("stream", name="stderr", text="e"),
nbformat.new_output("stream", name="stderr", text="f"),
nbformat.new_output("display_data", data={'image/png':
'Zw=='}), # g
nbformat.new_output("display_data",
data={'application/pdf': 'aA=='}), # h
]
cells = [
nbformat.new_code_cell(source="$ e $",
execution_count=1,
outputs=outputs),
nbformat.new_markdown_cell(source="$ e $")
]
return nbformat.new_notebook(cells=cells)
def get(self, user, filename):
## filename can have a path on it
next = "/hub/%s/public/%s" % (user, filename)
filesystem_path = "/home/%s/Public/%s" % (user, filename)
if os.path.isfile(filesystem_path): # download, raw, or view notebook
command = "view"
if len(self.get_arguments("view")) > 0:
command = "view"
elif len(self.get_arguments("download")) > 0:
command = "download"
elif len(self.get_arguments("copy")) > 0:
command = "copy"
elif len(self.get_arguments("pdf")) > 0:
command = "pdf"
elif len(self.get_arguments("raw")) > 0:
command = "raw"
# else: view
if filename.endswith(".ipynb"):
if command in ["view", "pdf"]:
# first, make a notebook html:
#with open("/home/%s/Public/%s" % (user, filename)) as fp:
# notebook_content = fp.read()
if command == "view":
exporter = HTMLExporter(template_file='full')
else:
exporter = PDFExporter(latex_count=1)
nb_json = nbformat.read("/home/%s/Public/%s" % (user, filename), as_version=4)
#if command == "pdf":
# # If pdf, remove heading numbering:
# for cell in nb_json["worksheets"][0]["cells"]:
# if cell["cell_type"] == "heading":
# cell["source"] = re.sub("^([0-9]+\.?)+\s", "", cell["source"])
# where to get css, images?
if command == "pdf":
self.set_header('Content-Type', "application/pdf")
base_filename = os.path.basename(filename)
self.set_header('Content-Disposition', 'attachment; filename="%s"' % base_filename)
else: # render as HTML
# add header/footer:
path = "/hub/%s/public" % user
parts = [(path, path)]
for part in filename.split("/")[:-1]:
path += "/" + part
parts.append((path, part))
breadcrumbs = " / ".join(map(lambda pair: '<a href="%s" target="_blank">%s</a>' % pair, parts))
env = {
"breadcrumbs": breadcrumbs,
"url": path + "/" + filename + "?download"
}
cell = new_markdown_cell(source="""
<table width="100%" style="border: none;">
<tr style="border: none;">
<td style="border: none;" width="100px">
<img src="https://serendip.brynmawr.edu/oneworld/files/styles/thumbnail/public/pictures/SerendipStudioAvatar.png?itok=48Z_omRv"/>
</td>
<td style="border: none;" width="50%">
<h2><a href="https://serendip.brynmawr.edu/oneworld/tides/explore">TIDES: Teaching to Increase Diversity and Equity in STEM</a></h2>
</td>
<td style="border: none;">
<a href="http://jupyter.physics.brynmawr.edu/hub/dblank/public/Jupyter%20Help.ipynb" title="Help">
<span class='fa fa-info-circle fa-2x menu-icon'></span>
<span class='menu-text'>Help</span>
</a>
</td>
<td style="border: none;">
<a href="{url}" title="Download Notebook" download>
<span class='fa fa-download fa-2x menu-icon'></span>
<span class='menu-text'>Download Notebook</span>
</a>
</td>
</tr>
<tr style="border: none;">
<td colspan="4" style="border: none;">
<b>Public notebooks:</b> {breadcrumbs}
</td>
</tr>
</table>
<hr style="background-color: #534f9a; height: 5px; border: 0; ">
""".format(**env))
nb_json["cells"].insert(0, cell)
(body, resources) = exporter.from_notebook_node(nb_json)
self.write(body)
elif command == "download": # download notebook json
self.download(user, filename, "text/plain")
elif command == "copy": # copy notebook json, if logged in
if self.get_current_user_name():
self.copy_file(user, filename, self.get_current_user_name())
else:
self.write("Please <a href=\"/hub/login?next=%s\">login</a> to allow copy." % next)
else: # raw, just get file contents
with open("/home/%s/Public/%s" % (user, filename), "rb") as fp:
self.write(fp.read())
else: # some other kind of file
# FIXME: how to get all of custom stuff?
if command == "copy":
if self.get_current_user_name():
self.copy_file(user, filename, self.get_current_user_name())
else:
self.write("Please <a href=\"/hub/login?next=%s\">login</a> to allow copy." % next)
else: # whatever, just get or download it
base_filename = os.path.basename(filename)
base, ext = os.path.splitext(base_filename)
app_log.info("extension is: %s" % ext)
if base_filename == "custom.css":
file_path = "/home/%s/.ipython/profile_default/static/custom/custom.css" % user
self.set_header('Content-Type', "text/css")
with open(file_path, "rb") as fp:
self.write(fp.read())
elif ext in [".txt", ".html", ".js", ".css", ".pdf", ".gif", ".jpeg", ".jpg", ".png"]: # show in browser
app_log.info("mime: %s" % str(mimetypes.guess_type(filename)[0]))
self.set_header('Content-Type', mimetypes.guess_type(filename)[0])
with open("/home/%s/Public/%s" % (user, filename), "rb") as fp:
self.write(fp.read())
else:
self.download(user, filename)
else: # not a file; directory listing
# filename can have a full path, and might be empty
url_path = "/hub/%s/public" % user
# could be: images, images/, images/subdir, images/subdir/
if not filename.endswith("/") and filename.strip() != "":
filename += "/"
files = glob.glob("/home/%s/Public/%s*" % (user, filename))
self.write("<h1>Jupyter Project at Bryn Mawr College</h1>\n")
self.write("[<a href=\"/hub/login\">Home</a>] ")
if self.get_current_user_name():
self.write("[<a href=\"/user/%(current_user)s/tree\">%(current_user)s</a>] " % {"current_user": self.get_current_user_name()})
self.write("<p/>\n")
self.write("<p>Public files for <b>/hub/%s/public/%s</b>:</p>\n" % (user, filename))
self.write("<ol>\n")
for absolute_filename in sorted(files):
if os.path.isdir(absolute_filename):
dir_path = absolute_filename.split("/")
dir_name = dir_path[-1]
public_path = "/".join(dir_path[dir_path.index("Public") + 1:])
self.write("<li><a href=\"%(url_path)s/%(public_path)s\">%(dir_name)s</a></li>\n" % {"url_path": url_path,
"dir_name": dir_name,
"public_path": public_path})
else:
file_path, filename = absolute_filename.rsplit("/", 1)
dir_path = absolute_filename.split("/")
public_path = "/".join(dir_path[dir_path.index("Public") + 1:])
variables = {"user": user, "filename": filename, "url_path": url_path, "next": next,
"public_path": public_path}
if filename.endswith(".ipynb"):
if self.get_current_user_name():
self.write(("<li><a href=\"%(url_path)s/%(public_path)s\">%(filename)s</a> "+
"(<a href=\"%(url_path)s/%(public_path)s?raw\">raw</a>, " +
"<a href=\"%(url_path)s/%(public_path)s?download\">download</a>, " +
"<a href=\"%(url_path)s/%(public_path)s?copy\">copy</a>" +
((", <a href=\"/user/%s/notebooks/Public/%s\">edit</a>" % (user, filename)) if self.get_current_user_name() == user else ", edit") +
")</li>\n") % variables)
else:
self.write(("<li><a href=\"%(url_path)s/%(public_path)s\">%(filename)s</a> "+
"(<a href=\"%(url_path)s/%(public_path)s?raw\">raw</a>, " +
"<a href=\"%(url_path)s/%(public_path)s?download\">download</a>, " +
"copy, edit) " +
"[<a href=\"/hub/login?next=%(next)s\">login</a> to copy]</li>\n") % variables)
else:
# some other kind of file (eg, .zip, .css):
self.write("<li><a href=\"%(url_path)s/%(public_path)s\">%(filename)s</a></li>\n" % variables)
self.write("</ol>\n")
self.write("<hr>\n")
self.write("<p><i>Please see <a href=\"/hub/dblank/public/Jupyter Help.ipynb\">Jupyter Help</a> for more information about this server.</i></p>\n")
import base64, os, re, sys
import IPython.nbformat.v4 as nbf
filename = os.path.splitext(sys.argv[1])[0]
try:
title, description = open("{}.txt".format(filename), encoding="utf-8").read().split('\n\n', 1)
except IOError:
title, description = filename, ""
description = description.replace("...", "").replace("'''", "**").replace("''", "*")
bendpattern = re.compile("^!+", re.MULTILINE)
bendcode = '<img src="http://pyx.sourceforge.net/bend.png" align="left">'
description = re.sub(bendpattern, lambda m: bendcode*(m.end()-m.start()), description)
code = open("{}.py".format(filename), encoding="utf-8").read()
code = re.sub('\.writeEPSfile\(("[a-z]+")?\)\n.*writePDFfile\(("[a-z]+")?\)\n.*writeSVGfile\(("[a-z]+")?\)\n', "", code)
nb = nbf.new_notebook()
cells = []
cells.append(nbf.new_markdown_cell(source="# " + title))
cells.append(nbf.new_code_cell(source=code, execution_count=1,
outputs=[nbf.new_output(output_type=u'execute_result', execution_count=1,
data={'image/png': base64.encodebytes(open("{}.png".format(filename), "rb").read()).decode("ascii"),
'image/svg+xml': open("{}.svg".format(filename), "r", encoding="utf-8").read()})]))
cells.append(nbf.new_markdown_cell(source=description))
nb = nbf.new_notebook(cells=cells, metadata={'language': 'python'})
open("{}.ipynb".format(filename), "w").write(nbf.writes(nb))
def create_text_cell():
source = "this is the answer!\n"
cell = new_markdown_cell(source=source)
return cell
def ipynb_code(filestr, code_blocks, code_block_types, tex_blocks, format):
"""
# We expand all newcommands now
from html import embed_newcommands
newcommands = embed_newcommands(filestr)
if newcommands:
filestr = newcommands + filestr
"""
# Fix pandoc citations to normal internal links: [[key]](#key)
filestr = re.sub(r'\[@(.+?)\]', r'[[\g<1>]](#\g<1>)', filestr)
# filestr becomes json list after this function so we must typeset
# envirs here. All envirs are typeset as pandoc_quote.
from common import _CODE_BLOCK, _MATH_BLOCK
envir_format = option('ipynb_admon=', 'paragraph')
# Remove all !bpop-!epop environments (they cause only problens and
# have no use)
for envir in 'pop', 'slidecell':
filestr = re.sub('^<!-- !b%s .*\n' % envir,
'',
filestr,
flags=re.MULTILINE)
filestr = re.sub('^<!-- !e%s .*\n' % envir,
'',
filestr,
flags=re.MULTILINE)
filestr = re.sub('^<!-- !bnotes.*?<!-- !enotes -->\n',
'',
filestr,
flags=re.DOTALL | re.MULTILINE)
filestr = re.sub('^<!-- !split -->\n', '', filestr, flags=re.MULTILINE)
from doconce import doconce_envirs
envirs = doconce_envirs()[8:-2]
for envir in envirs:
pattern = r'^!b%s(.*?)\n(.+?)\s*^!e%s' % (envir, envir)
if envir_format in ('quote', 'paragraph', 'hrule'):
def subst(m):
title = m.group(1).strip()
# Text size specified in parenthesis?
m2 = re.search('^\s*\((.+?)\)', title)
if title == '' and envir not in ('block', 'quote'):
title = envir.capitalize() + '.'
elif title.lower() == 'none':
title == ''
elif m2:
text_size = m2.group(1).lower()
title = title.replace('(%s)' % text_size, '').strip()
elif title and title[-1] not in ('.', ':', '!', '?'):
# Make sure the title ends with puncuation
title += '.'
# Recall that this formatting is called very late
# so native format must be used!
if title:
title = '**' + title + '**\n'
# Could also consider subsubsection formatting
block = m.group(2)
# Always use quote typesetting for quotes
if envir_format == 'quote' or envir == 'quote':
# Make Markdown quote of the block: lines start with >
lines = []
for line in block.splitlines():
# Just quote plain text
if not (_MATH_BLOCK in line or _CODE_BLOCK in line
or line.startswith('FIGURE:')
or line.startswith('MOVIE:')
or line.startswith('|')):
lines.append('> ' + line)
else:
lines.append('\n' + line + '\n')
block = '\n'.join(lines) + '\n\n'
# Add quote and a blank line after title
if title:
title = '> ' + title + '>\n'
else:
# Add a blank line after title
if title:
title += '\n'
if envir_format == 'hrule':
# Native ------ does not work, use <hr/>
#text = '\n\n----------\n' + title + '----------\n' + \
# block + '\n----------\n\n'
text = '\n\n<hr/>\n' + title + \
block + '\n<hr/>\n\n'
else:
text = title + block + '\n\n'
return text
else:
print '*** error: --ipynb_admon=%s is not supported' % envir_format
filestr = re.sub(pattern,
subst,
filestr,
flags=re.DOTALL | re.MULTILINE)
# Fix pyshell and ipy interactive sessions: remove prompt and output.
# or split in multiple cells such that output comes out at the end of a cell
# Fix sys environments and use run prog.py so programs can be run in cell
# Insert %matplotlib inline in the first block using matplotlib
# Only typeset Python code as blocks, otherwise !bc environmens
# become plain indented Markdown.
from doconce import dofile_basename
from sets import Set
ipynb_tarfile = 'ipynb-%s-src.tar.gz' % dofile_basename
src_paths = Set()
mpl_inline = False
split_pyshell = option('ipynb_split_pyshell=', 'on')
if split_pyshell is None:
split_pyshell = False
elif split_pyshell in ('no', 'False', 'off'):
split_pyshell = False
else:
split_pyshell = True
ipynb_code_tp = [None] * len(code_blocks)
for i in range(len(code_blocks)):
# Check if continuation lines are in the code block, because
# doconce.py inserts a blank after the backslash
if '\\ \n' in code_blocks[i]:
code_blocks[i] = code_blocks[i].replace('\\ \n', '\\\n')
if not mpl_inline and (
re.search(r'import +matplotlib', code_blocks[i]) or \
re.search(r'from +matplotlib', code_blocks[i]) or \
re.search(r'import +scitools', code_blocks[i]) or \
re.search(r'from +scitools', code_blocks[i])):
code_blocks[i] = '%matplotlib inline\n\n' + code_blocks[i]
mpl_inline = True
tp = code_block_types[i]
if tp.endswith('-t'):
# Standard Markdown code with pandoc/github extension
language = tp[:-2]
language_spec = language2pandoc.get(language, '')
#code_blocks[i] = '\n' + indent_lines(code_blocks[i], format) + '\n'
code_blocks[i] = "```%s\n" % language_spec + \
indent_lines(code_blocks[i].strip(), format) + \
"```"
ipynb_code_tp[i] = 'markdown'
elif tp.startswith('pyshell') or tp.startswith('ipy'):
lines = code_blocks[i].splitlines()
last_cell_end = -1
if split_pyshell:
new_code_blocks = []
# Split for each output an put in separate cell
for j in range(len(lines)):
if lines[j].startswith('>>>') or lines[j].startswith(
'... '):
lines[j] = lines[j][4:]
elif lines[j].startswith('In ['): # IPython
lines[j] = ':'.join(lines[j].split(':')[1:]).strip()
elif lines[j].startswith(' ...: '): # IPython
lines[j] = lines[j][8:]
else:
# output (no prefix or Out)
lines[j] = ''
new_code_blocks.append('\n'.join(lines[last_cell_end +
1:j + 1]))
last_cell_end = j
code_blocks[i] = new_code_blocks
ipynb_code_tp[i] = 'cell'
else:
# Remove prompt and output lines; leave code executable in cell
for j in range(len(lines)):
if lines[j].startswith('>>> ') or lines[j].startswith(
'... '):
lines[j] = lines[j][4:]
elif lines[j].startswith('In ['):
lines[j] = ':'.join(lines[j].split(':')[1:]).strip()
else:
# output
lines[j] = ''
for j in range(lines.count('')):
lines.remove('')
code_blocks[i] = '\n'.join(lines)
ipynb_code_tp[i] = 'cell'
elif tp.startswith('sys'):
# Do we find execution of python file? If so, copy the file
# to separate subdir and make a run file command in a cell.
# Otherwise, it is just a plain verbatim Markdown block.
found_unix_lines = False
lines = code_blocks[i].splitlines()
for j in range(len(lines)):
m = re.search(r'(.+?>|\$) *python +([A-Za-z_0-9]+?\.py)',
lines[j])
if m:
name = m.group(2).strip()
if os.path.isfile(name):
src_paths.add(os.path.dirname(name))
lines[j] = '%%run "%s"' % fullpath
else:
found_unix_lines = True
src_paths = list(src_paths)
if src_paths and not found_unix_lines:
# This is a sys block with run commands only
code_blocks[i] = '\n'.join(lines)
ipynb_code_tp[i] = 'cell'
else:
# Standard Markdown code
code_blocks[i] = '\n'.join(lines)
code_blocks[i] = indent_lines(code_blocks[i], format)
ipynb_code_tp[i] = 'markdown'
elif tp.endswith('hid'):
ipynb_code_tp[i] = 'cell_hidden'
elif tp.startswith('py'):
ipynb_code_tp[i] = 'cell'
else:
# Should support other languages as well, but not for now
code_blocks[i] = indent_lines(code_blocks[i], format)
ipynb_code_tp[i] = 'markdown'
# figure_files and movie_files are global variables and contain
# all figures and movies referred to
src_paths = list(src_paths)
if figure_files:
src_paths += figure_files
if movie_files:
src_paths += movie_files
if src_paths:
# Make tar file with all the source dirs with files
# that need to be executed
os.system('tar cfz %s %s' % (ipynb_tarfile, ' '.join(src_paths)))
print 'collected all required additional files in', ipynb_tarfile, 'which must be distributed with the notebook'
elif os.path.isfile(ipynb_tarfile):
os.remove(ipynb_tarfile)
# Parse document into markdown text, code blocks, and tex blocks.
# Store in nested list notebook_blocks.
notebook_blocks = [[]]
authors = ''
for line in filestr.splitlines():
if line.startswith('authors = [new_author(name='): # old author method
authors = line[10:]
elif _CODE_BLOCK in line:
code_block_tp = line.split()[-1]
if code_block_tp in (
'pyhid', ) or not code_block_tp.endswith('hid'):
notebook_blocks[-1] = '\n'.join(notebook_blocks[-1]).strip()
notebook_blocks.append(line)
# else: hidden block to be dropped (may include more languages
# with time in the above tuple)
elif _MATH_BLOCK in line:
notebook_blocks[-1] = '\n'.join(notebook_blocks[-1]).strip()
notebook_blocks.append(line)
else:
if not isinstance(notebook_blocks[-1], list):
notebook_blocks.append([])
notebook_blocks[-1].append(line)
if isinstance(notebook_blocks[-1], list):
notebook_blocks[-1] = '\n'.join(notebook_blocks[-1]).strip()
# Add block type info
pattern = r'(\d+) +%s'
for i in range(len(notebook_blocks)):
if re.match(pattern % _CODE_BLOCK, notebook_blocks[i]):
m = re.match(pattern % _CODE_BLOCK, notebook_blocks[i])
idx = int(m.group(1))
if ipynb_code_tp[idx] == 'cell':
notebook_blocks[i] = ['cell', notebook_blocks[i]]
elif ipynb_code_tp[idx] == 'cell_hidden':
notebook_blocks[i] = ['cell_hidden', notebook_blocks[i]]
else:
notebook_blocks[i] = ['text', notebook_blocks[i]]
elif re.match(pattern % _MATH_BLOCK, notebook_blocks[i]):
notebook_blocks[i] = ['math', notebook_blocks[i]]
else:
notebook_blocks[i] = ['text', notebook_blocks[i]]
# Go through tex_blocks and wrap in $$
# (doconce.py runs align2equations so there are no align/align*
# environments in tex blocks)
label2tag = {}
tag_counter = 1
for i in range(len(tex_blocks)):
# Extract labels and add tags
labels = re.findall(r'label\{(.+?)\}', tex_blocks[i])
for label in labels:
label2tag[label] = tag_counter
# Insert tag to get labeled equation
tex_blocks[i] = tex_blocks[i].replace(
'label{%s}' % label,
'label{%s} \\tag{%s}' % (label, tag_counter))
tag_counter += 1
# Remove \[ and \] or \begin/end{equation*} in single equations
tex_blocks[i] = tex_blocks[i].replace(r'\[', '')
tex_blocks[i] = tex_blocks[i].replace(r'\]', '')
tex_blocks[i] = tex_blocks[i].replace(r'\begin{equation*}', '')
tex_blocks[i] = tex_blocks[i].replace(r'\end{equation*}', '')
# Check for illegal environments
m = re.search(r'\\begin\{(.+?)\}', tex_blocks[i])
if m:
envir = m.group(1)
if envir not in ('equation', 'equation*', 'align*', 'align',
'array'):
print """\
*** warning: latex envir \\begin{%s} does not work well in Markdown.
Stick to \\[ ... \\], equation, equation*, align, or align*
environments in math environments.
""" % envir
eq_type = 'heading' # or '$$'
eq_type = '$$'
# Markdown: add $$ on each side of the equation
if eq_type == '$$':
# Make sure there are no newline after equation
tex_blocks[i] = '$$\n' + tex_blocks[i].strip() + '\n$$'
# Here: use heading (###) and simple formula (remove newline
# in math expressions to keep everything within a heading) as
# the equation then looks bigger
elif eq_type == 'heading':
tex_blocks[i] = '### $ ' + ' '.join(
tex_blocks[i].splitlines()) + ' $'
# Add labels for the eqs above the block (for reference)
if labels:
#label_tp = '<a name="%s"></a>'
label_tp = '<div id="%s"></div>'
tex_blocks[i] = '<!-- Equation labels as ordinary links -->\n' + \
' '.join([label_tp % label
for label in labels]) + '\n\n' + \
tex_blocks[i]
# blocks is now a list of text chunks in markdown and math/code line
# instructions. Insert code and tex blocks
for i in range(len(notebook_blocks)):
if _CODE_BLOCK in notebook_blocks[i][
1] or _MATH_BLOCK in notebook_blocks[i][1]:
words = notebook_blocks[i][1].split()
# start of notebook_blocks[i]: number block-indicator code-type
n = int(words[0])
if _CODE_BLOCK in notebook_blocks[i][1]:
notebook_blocks[i][1] = code_blocks[n] # can be list!
if _MATH_BLOCK in notebook_blocks[i][1]:
notebook_blocks[i][1] = tex_blocks[n]
# Make IPython structures
nb_version = int(option('ipynb_version=', '3'))
if nb_version == 3:
from IPython.nbformat.v3 import (new_code_cell, new_text_cell,
new_worksheet, new_notebook,
new_metadata, new_author)
nb = new_worksheet()
elif nb_version == 4:
from IPython.nbformat.v4 import (new_code_cell, new_markdown_cell,
new_notebook)
cells = []
mdstr = [] # plain md format of the notebook
prompt_number = 1
for block_tp, block in notebook_blocks:
if (block_tp == 'text' or block_tp == 'math') and block != '':
# Pure comments between math/code and math/code come
# out as empty blocks, should detect that situation
# (challenging - can have multiple lines of comments,
# or begin and end comment lines with important things between)
if nb_version == 3:
nb.cells.append(new_text_cell(u'markdown', source=block))
elif nb_version == 4:
cells.append(new_markdown_cell(source=block))
mdstr.append(('markdown', block))
elif block_tp == 'cell' and block != '' and block != []:
if isinstance(block, list):
for block_ in block:
block_ = block_.rstrip()
if block_ != '':
if nb_version == 3:
nb.cells.append(
new_code_cell(input=block_,
prompt_number=prompt_number,
collapsed=False))
elif nb_version == 4:
cells.append(
new_code_cell(source=block_,
execution_count=prompt_number))
prompt_number += 1
mdstr.append(('codecell', block_))
else:
block = block.rstrip()
if block != '':
if nb_version == 3:
nb.cells.append(
new_code_cell(input=block,
prompt_number=prompt_number,
collapsed=False))
elif nb_version == 4:
cells.append(
new_code_cell(source=block,
execution_count=prompt_number))
prompt_number += 1
mdstr.append(('codecell', block))
elif block_tp == 'cell_hidden' and block != '':
block = block.rstrip()
if nb_version == 3:
nb.cells.append(
new_code_cell(input=block,
prompt_number=prompt_number,
collapsed=True))
elif nb_version == 4:
cells.append(
new_code_cell(source=block, execution_count=prompt_number))
prompt_number += 1
mdstr.append(('codecell', block))
"""
# Dump the notebook cells in a simple ASCII format
# (doc/src/ipynb/ipynb_generator.py can translate it back to .ipynb file)
f = open(dofile_basename + '.md-ipynb', 'w')
for cell_tp, block in mdstr:
if cell_tp == 'markdown':
f.write('\n-----\n\n')
elif cell_tp == 'codecell':
f.write('\n-----py\n\n')
f.write(block)
f.close()
"""
if nb_version == 3:
# Catch the title as the first heading
m = re.search(r'^#+\s*(.+)$', filestr, flags=re.MULTILINE)
title = m.group(1).strip() if m else ''
# md below is not used for anything
if authors:
authors = eval(authors)
md = new_metadata(name=title, authors=authors)
else:
md = new_metadata(name=title)
nb = new_notebook(worksheets=[nb], metadata=new_metadata())
# Let us make v4 notebook here by upgrading
from IPython.nbformat.v4 import upgrade
nb = upgrade(nb)
import IPython.nbformat.v4.nbjson as nbjson
# Convert nb to json format
filestr = nbjson.writes(nb)
elif nb_version == 4:
nb = new_notebook(cells=cells)
from IPython.nbformat import writes
filestr = writes(nb, version=4)
# Check that there are no empty cells:
if '"input": []' in filestr:
print '*** error: empty cells in notebook - report bug in DocOnce'
_abort()
# must do the replacements here at the very end when json is written out
# \eqref and labels will not work, but labels (only in math) do no harm
filestr = re.sub(r'([^\\])label\{',
r'\g<1>\\\\label{',
filestr,
flags=re.MULTILINE)
# \\eqref{} just gives (???) link at this stage - future versions
# will probably support labels
#filestr = re.sub(r'\(ref\{(.+?)\}\)', r'\\eqref{\g<1>}', filestr)
# Now we use explicit references to tags
def subst(m):
label = m.group(1)
try:
return r'[(%s)](#%s)' % (label2tag[label], label)
except KeyError as e:
print '*** error: label "%s" is not defined' % str(e)
filestr = re.sub(r'\(ref\{(.+?)\}\)', subst, filestr)
"""
# MathJax reference to tag (recall that the equations have both label
# and tag (know that tag only works well in HTML, but this mjx-eqn-no
# label does not work in ipynb)
filestr = re.sub(r'\(ref\{(.+?)\}\)',
lambda m: r'[(%s)](#mjx-eqn-%s)' % (label2tag[m.group(1)], label2tag[m.group(1)]), filestr)
"""
#filestr = re.sub(r'\(ref\{(.+?)\}\)', r'Eq (\g<1>)', filestr)
'''
# Final fixes: replace all text between cells by markdown code cells
# Note: the patterns are overlapping so a plain re.sub will not work,
# here we run through all blocks found and subsitute the first remaining
# one, one by one.
pattern = r' \},\n(.+?)\{\n "cell_type":'
begin_pattern = r'^(.+?)\{\n "cell_type":'
remaining_block_begin = re.findall(begin_pattern, filestr, flags=re.DOTALL)
remaining_blocks = re.findall(pattern, filestr, flags=re.DOTALL)
import string
for block in remaining_block_begin + remaining_blocks:
filestr = string.replace(filestr, block, json_markdown(block) + ' ',
maxreplace=1)
filestr_end = re.sub(r' \{\n "cell_type": .+?\n \},\n', '', filestr,
flags=re.DOTALL)
filestr = filestr.replace(filestr_end, json_markdown(filestr_end))
filestr = """{
"metadata": {
"name": "SOME NAME"
},
"nbformat": 3,
"nbformat_minor": 0,
"worksheets": [
{
"cells": [
""" + filestr.rstrip() + '\n'+ \
json_pycode('', final_prompt_no+1, 'python').rstrip()[:-1] + """
],
"metadata": {}
}
]
}"""
'''
return filestr
except IOError:
title, description = filename, ""
description = description.replace("...", "").replace("'''",
"**").replace("''", "*")
bendpattern = re.compile("^!+", re.MULTILINE)
bendcode = '<img src="http://pyx.sourceforge.net/bend.png" align="left">'
description = re.sub(bendpattern, lambda m: bendcode * (m.end() - m.start()),
description)
code = open("{}.py".format(filename), encoding="utf-8").read()
code = re.sub(
'\.writeEPSfile\(("[a-z]+")?\)\n.*writePDFfile\(("[a-z]+")?\)\n.*writeSVGfile\(("[a-z]+")?\)\n',
"", code)
nb = nbf.new_notebook()
cells = []
cells.append(nbf.new_markdown_cell(source="# " + title))
cells.append(
nbf.new_code_cell(source=code,
execution_count=1,
outputs=[
nbf.new_output(
output_type=u'execute_result',
execution_count=1,
data={
'image/png':
base64.encodebytes(
open("{}.png".format(filename),
"rb").read()).decode("ascii"),
'image/svg+xml':
open("{}.svg".format(filename),
"r",
def post(self, convert=False):
# Check if this is a convert operation
if convert:
# Get JSON payload
json_data = tornado.escape.json_decode(self.request.body)
# Go through all the assignments / bundles in the project payload
for x in json_data['bundles']:
# Create a notebook version4 object
nb = nbfv4.new_notebook()
# Create the task description
task_description = '## Task \n' + x['description'] + '\n'\
'___ \n' \
'#### '+ x['owner'] + '\n' \
'___ \n'
# Create the general description string
common_description ='#### Temporary folder \n' \
'Set your working dir to following folder '+ json_data['gid']+'. Upload your csv/data files into '\
'this directoy to use them.<br/>'\
'`ftp://pycard.ifi.uzh.ch/data/'+json_data['gid']+'`'\
'<br/><br/>'\
'Use with R Kernel <br/>' \
'`setwd("./'+ json_data['gid']+'")` <br/><br/>' \
'Use with Python Kernel <br/> ' \
'`import os` <br/>' \
'`os.chdir("./'+ json_data['gid']+'")` \n' \
'___ \n' \
'#### Notes board \n' \
'In order to avoid conflicts between notebooks and have a clean transition from one step to another, use the shared notes file ' \
'shared.txt . The contents of the file will be loaded and made in every notebook, so it is a good place to register variable names used in the different steps, or to provide feedback after each iteration. <br/><br/>'
# Add the task_description as a markdown cell
heading = nbfv4.new_markdown_cell(task_description)
# Set the task_description as read-only
heading['metadata'].run_control = dict()
heading['metadata'].run_control['read_only'] = True
# Append cell to notebook
nb['cells'].append(heading)
# Add the common description cell as a markdown cell
common = nbfv4.new_markdown_cell(common_description)
# Set the common description cell as read only
common['metadata'].run_control = dict()
common['metadata']['common'] = True
common['metadata'].run_control['read_only'] = True
# Add the cell to the notebook
nb['cells'].append(common)
# Create a markdown cell for the note board, set the variable_cell metadata to true
variablesh = nbfv4.new_markdown_cell()
variablesh['metadata']['variable_cell'] = True
nb['cells'].append(variablesh)
# Set the notebook kernel in metadata
nb['metadata']['language'] = json_data['kernel']
# Set cell toolbar to Side Comments in metadata
nb['metadata']['celltoolbar'] = "Side Comments"
# Set project ID in metadata
nb['metadata']['pgid'] = json_data['gid']
# Set id of notes board for this project (shared_notes.txt file)
nb['metadata']['variablesid'] = json_data['variablesid']
# Set Google ID for this notebook
nb['metadata']['id'] = x['gid']
# Set the worker assigned to this task
nb['metadata']['bundle-owner'] = x['owner']
# Go through all the actions in the assignment
for a in x['actions']:
# Create action description text
text = '#### This is the description of the actions that need to be implemented.' \
'\n' \
'### ' + a['name'] + '\n' \
'Description: ' + a['description'] + '<br>' \
'Input: ' + a['input'] + '<br>' \
'Output: ' + a['output']
code = "# Enter implementation here."
# Add the description cell as a markdown cell, set it read-only
desc = nbfv4.new_markdown_cell(text)
desc['metadata'].run_control = dict()
desc['metadata'].run_control['read_only'] = True
nb['cells'].append(desc)
# Create a cell for logging the work
log_text = '# Log and description \n' \
'Please record here all information needed to reproduce and understand your work: \n' \
'- Algorithms used\n' \
'- Things tried but discarded\n' \
'- Explanation **why** you have solved the problem the way you solved it.\n'
# Add the description cell as a markdown cell, set it read-only
log_desc = nbfv4.new_markdown_cell(log_text)
nb['cells'].append(log_desc)
# Create the cell code for this action
code_cell = nbfv4.new_code_cell(code)
code_cell['metadata'].side_comments = dict()
# Create the cell code for this action, set the section id needed
# for the SideComments extension as metadata
code_cell['metadata'].side_comments['id'] = ''.join(
random.SystemRandom().choice(string.ascii_uppercase +
string.digits)
for _ in range(20))
logger.info(code_cell['metadata'])
# Add cell to notebook
nb['cells'].append(code_cell)
# Add the contents of the created notebook a the bundle {notebook} property in the project payload
x['notebook'] = nbf.writes(nb, version=4)
json_data['variables'] = html2text.html2text(
json_data['variables'])
# Send the answer back to client
self.write(json.dumps(json_data, cls=Encoder))
self.set_status(201)
else:
# Get the project payload from the client
json_data = tornado.escape.json_decode(self.request.body)
# Add the project to the database
ret = db.addProject(json_data)
if ret['ok'] == 1.0:
# If project was added, create temporary work folder for the project
# which can be accessed through ftp
if not os.path.exists('/ftp/ipython/data/' + json_data['gid']):
os.makedirs('/ftp/ipython/data/' + json_data['gid'])
os.chmod('/ftp/ipython/data/' + json_data['gid'], 0o755)
self.set_status(201)
def post(self, convert=False):
# Check if this is a convert operation
if convert:
# Get JSON payload
json_data = tornado.escape.json_decode(self.request.body)
# Go through all the assignments / bundles in the project payload
for x in json_data['bundles']:
# Create a notebook version4 object
nb = nbfv4.new_notebook()
# Create the task description
task_description = '## Task \n' + x['description'] + '\n'\
'___ \n' \
'#### '+ x['owner'] + '\n' \
'___ \n'
# Create the general description string
common_description ='#### Temporary folder \n' \
'Set your working dir to following folder '+ json_data['gid']+'. Upload your csv/data files into '\
'this directoy to use them.<br/>'\
'`ftp://pycard.ifi.uzh.ch/data/'+json_data['gid']+'`'\
'<br/><br/>'\
'Use with R Kernel <br/>' \
'`setwd("./'+ json_data['gid']+'")` <br/><br/>' \
'Use with Python Kernel <br/> ' \
'`import os` <br/>' \
'`os.chdir("./'+ json_data['gid']+'")` \n' \
'___ \n' \
'#### Notes board \n' \
'In order to avoid conflicts between notebooks and have a clean transition from one step to another, use the shared notes file ' \
'shared.txt . The contents of the file will be loaded and made in every notebook, so it is a good place to register variable names used in the different steps, or to provide feedback after each iteration. <br/><br/>'
# Add the task_description as a markdown cell
heading = nbfv4.new_markdown_cell(task_description)
# Set the task_description as read-only
heading['metadata'].run_control = dict()
heading['metadata'].run_control['read_only'] = True
# Append cell to notebook
nb['cells'].append(heading)
# Add the common description cell as a markdown cell
common = nbfv4.new_markdown_cell(common_description)
# Set the common description cell as read only
common['metadata'].run_control = dict()
common['metadata']['common'] = True
common['metadata'].run_control['read_only'] = True
# Add the cell to the notebook
nb['cells'].append(common)
# Create a markdown cell for the note board, set the variable_cell metadata to true
variablesh = nbfv4.new_markdown_cell()
variablesh['metadata']['variable_cell'] = True
nb['cells'].append(variablesh)
# Set the notebook kernel in metadata
nb['metadata']['language'] = json_data['kernel']
# Set cell toolbar to Side Comments in metadata
nb['metadata']['celltoolbar'] = "Side Comments"
# Set project ID in metadata
nb['metadata']['pgid'] = json_data['gid']
# Set id of notes board for this project (shared_notes.txt file)
nb['metadata']['variablesid'] = json_data['variablesid']
# Set Google ID for this notebook
nb['metadata']['id'] = x['gid']
# Go through all the actions in the assignment
for a in x['actions']:
# Create action description text
text = '#### This is the description of the actions that need to be implemented.' \
'\n' \
'### ' + a['name'] + '\n' \
'Description: ' + a['description'] + '<br>' \
'Input: ' + a['input'] + '<br>' \
'Output: ' + a['output']
code = "# Enter implementation here."
# Add the description cell as a markdown cell, set it read-only
desc = nbfv4.new_markdown_cell(text)
desc['metadata'].run_control = dict()
desc['metadata'].run_control['read_only'] = True
nb['cells'].append(desc)
# Create the cell code for this action
code_cell = nbfv4.new_code_cell(code)
code_cell['metadata'].side_comments = dict()
# Create the cell code for this action, set the section id needed
# for the SideComments extension as metadata
code_cell['metadata'].side_comments['id'] = ''.join(random.SystemRandom().choice(string.ascii_uppercase + string.digits) for _ in range(20))
logger.info(code_cell['metadata'])
# Add cell to notebook
nb['cells'].append(code_cell)
# Add the contents of the created notebook a the bundle {notebook} property in the project payload
x['notebook'] = nbf.writes(nb,version=4)
json_data['variables'] = html2text.html2text(json_data['variables'])
# Send the answer back to client
self.write(json.dumps(json_data, cls=Encoder))
self.set_status(201)
else:
# Get the project payload from the client
json_data = tornado.escape.json_decode(self.request.body)
# Add the project to the database
ret = db.addProject(json_data)
if ret['ok'] == 1.0:
# If project was added, create temporary work folder for the project
# which can be accessed through ftp
if not os.path.exists('/ftp/ipython/data/' + json_data['gid']):
os.makedirs('/ftp/ipython/data/' + json_data['gid'])
os.chmod('/ftp/ipython/data/' + json_data['gid'], 0o755)
self.set_status(201)
def _create_text_cell():
source = "this is the answer!\n"
cell = new_markdown_cell(source=source)
return cell
def ipynb_code(filestr, code_blocks, code_block_types,
tex_blocks, format):
"""
# We expand all newcommands now
from html import embed_newcommands
newcommands = embed_newcommands(filestr)
if newcommands:
filestr = newcommands + filestr
"""
# Fix pandoc citations to normal internal links: [[key]](#key)
filestr = re.sub(r'\[@(.+?)\]', r'[[\g<1>]](#\g<1>)', filestr)
# filestr becomes json list after this function so we must typeset
# envirs here. All envirs are typeset as pandoc_quote.
from common import _CODE_BLOCK, _MATH_BLOCK
envir_format = option('ipynb_admon=', 'paragraph')
# Remove all !bpop-!epop environments (they cause only problens and
# have no use)
for envir in 'pop', 'slidecell':
filestr = re.sub('^<!-- !b%s .*\n' % envir, '', filestr,
flags=re.MULTILINE)
filestr = re.sub('^<!-- !e%s .*\n' % envir, '', filestr,
flags=re.MULTILINE)
filestr = re.sub('^<!-- !bnotes.*?<!-- !enotes -->\n', '', filestr,
flags=re.DOTALL|re.MULTILINE)
filestr = re.sub('^<!-- !split -->\n', '', filestr, flags=re.MULTILINE)
from doconce import doconce_envirs
envirs = doconce_envirs()[8:-2]
for envir in envirs:
pattern = r'^!b%s(.*?)\n(.+?)\s*^!e%s' % (envir, envir)
if envir_format in ('quote', 'paragraph', 'hrule'):
def subst(m):
title = m.group(1).strip()
# Text size specified in parenthesis?
m2 = re.search('^\s*\((.+?)\)', title)
if title == '' and envir not in ('block', 'quote'):
title = envir.capitalize() + '.'
elif title.lower() == 'none':
title == ''
elif m2:
text_size = m2.group(1).lower()
title = title.replace('(%s)' % text_size, '').strip()
elif title and title[-1] not in ('.', ':', '!', '?'):
# Make sure the title ends with puncuation
title += '.'
# Recall that this formatting is called very late
# so native format must be used!
if title:
title = '**' + title + '**\n'
# Could also consider subsubsection formatting
block = m.group(2)
# Always use quote typesetting for quotes
if envir_format == 'quote' or envir == 'quote':
# Make Markdown quote of the block: lines start with >
lines = []
for line in block.splitlines():
# Just quote plain text
if not (_MATH_BLOCK in line or
_CODE_BLOCK in line or
line.startswith('FIGURE:') or
line.startswith('MOVIE:') or
line.startswith('|')):
lines.append('> ' + line)
else:
lines.append('\n' + line + '\n')
block = '\n'.join(lines) + '\n\n'
# Add quote and a blank line after title
if title:
title = '> ' + title + '>\n'
else:
# Add a blank line after title
if title:
title += '\n'
if envir_format == 'hrule':
# Native ------ does not work, use <hr/>
#text = '\n\n----------\n' + title + '----------\n' + \
# block + '\n----------\n\n'
text = '\n\n<hr/>\n' + title + \
block + '\n<hr/>\n\n'
else:
text = title + block + '\n\n'
return text
else:
print '*** error: --ipynb_admon=%s is not supported' % envir_format
filestr = re.sub(pattern, subst, filestr,
flags=re.DOTALL | re.MULTILINE)
# Fix pyshell and ipy interactive sessions: remove prompt and output.
# or split in multiple cells such that output comes out at the end of a cell
# Fix sys environments and use run prog.py so programs can be run in cell
# Insert %matplotlib inline in the first block using matplotlib
# Only typeset Python code as blocks, otherwise !bc environmens
# become plain indented Markdown.
from doconce import dofile_basename
from sets import Set
ipynb_tarfile = 'ipynb-%s-src.tar.gz' % dofile_basename
src_paths = Set()
mpl_inline = False
split_pyshell = option('ipynb_split_pyshell=', 'on')
if split_pyshell is None:
split_pyshell = False
elif split_pyshell in ('no', 'False', 'off'):
split_pyshell = False
else:
split_pyshell = True
ipynb_code_tp = [None]*len(code_blocks)
for i in range(len(code_blocks)):
# Check if continuation lines are in the code block, because
# doconce.py inserts a blank after the backslash
if '\\ \n' in code_blocks[i]:
code_blocks[i] = code_blocks[i].replace('\\ \n', '\\\n')
if not mpl_inline and (
re.search(r'import +matplotlib', code_blocks[i]) or \
re.search(r'from +matplotlib', code_blocks[i]) or \
re.search(r'import +scitools', code_blocks[i]) or \
re.search(r'from +scitools', code_blocks[i])):
code_blocks[i] = '%matplotlib inline\n\n' + code_blocks[i]
mpl_inline = True
tp = code_block_types[i]
if tp.endswith('-t'):
# Standard Markdown code with pandoc/github extension
language = tp[:-2]
language_spec = language2pandoc.get(language, '')
#code_blocks[i] = '\n' + indent_lines(code_blocks[i], format) + '\n'
code_blocks[i] = "```%s\n" % language_spec + \
indent_lines(code_blocks[i].strip(), format) + \
"```"
ipynb_code_tp[i] = 'markdown'
elif tp.startswith('pyshell') or tp.startswith('ipy'):
lines = code_blocks[i].splitlines()
last_cell_end = -1
if split_pyshell:
new_code_blocks = []
# Split for each output an put in separate cell
for j in range(len(lines)):
if lines[j].startswith('>>>') or lines[j].startswith('... '):
lines[j] = lines[j][4:]
elif lines[j].startswith('In ['): # IPython
lines[j] = ':'.join(lines[j].split(':')[1:]).strip()
elif lines[j].startswith(' ...: '): # IPython
lines[j] = lines[j][8:]
else:
# output (no prefix or Out)
lines[j] = ''
new_code_blocks.append(
'\n'.join(lines[last_cell_end+1:j+1]))
last_cell_end = j
code_blocks[i] = new_code_blocks
ipynb_code_tp[i] = 'cell'
else:
# Remove prompt and output lines; leave code executable in cell
for j in range(len(lines)):
if lines[j].startswith('>>> ') or lines[j].startswith('... '):
lines[j] = lines[j][4:]
elif lines[j].startswith('In ['):
lines[j] = ':'.join(lines[j].split(':')[1:]).strip()
else:
# output
lines[j] = ''
for j in range(lines.count('')):
lines.remove('')
code_blocks[i] = '\n'.join(lines)
ipynb_code_tp[i] = 'cell'
elif tp.startswith('sys'):
# Do we find execution of python file? If so, copy the file
# to separate subdir and make a run file command in a cell.
# Otherwise, it is just a plain verbatim Markdown block.
found_unix_lines = False
lines = code_blocks[i].splitlines()
for j in range(len(lines)):
m = re.search(r'(.+?>|\$) *python +([A-Za-z_0-9]+?\.py)',
lines[j])
if m:
name = m.group(2).strip()
if os.path.isfile(name):
src_paths.add(os.path.dirname(name))
lines[j] = '%%run "%s"' % fullpath
else:
found_unix_lines = True
src_paths = list(src_paths)
if src_paths and not found_unix_lines:
# This is a sys block with run commands only
code_blocks[i] = '\n'.join(lines)
ipynb_code_tp[i] = 'cell'
else:
# Standard Markdown code
code_blocks[i] = '\n'.join(lines)
code_blocks[i] = indent_lines(code_blocks[i], format)
ipynb_code_tp[i] = 'markdown'
elif tp.endswith('hid'):
ipynb_code_tp[i] = 'cell_hidden'
elif tp.startswith('py'):
ipynb_code_tp[i] = 'cell'
else:
# Should support other languages as well, but not for now
code_blocks[i] = indent_lines(code_blocks[i], format)
ipynb_code_tp[i] = 'markdown'
# figure_files and movie_files are global variables and contain
# all figures and movies referred to
src_paths = list(src_paths)
if figure_files:
src_paths += figure_files
if movie_files:
src_paths += movie_files
if src_paths:
# Make tar file with all the source dirs with files
# that need to be executed
os.system('tar cfz %s %s' % (ipynb_tarfile, ' '.join(src_paths)))
print 'collected all required additional files in', ipynb_tarfile, 'which must be distributed with the notebook'
elif os.path.isfile(ipynb_tarfile):
os.remove(ipynb_tarfile)
# Parse document into markdown text, code blocks, and tex blocks.
# Store in nested list notebook_blocks.
notebook_blocks = [[]]
authors = ''
for line in filestr.splitlines():
if line.startswith('authors = [new_author(name='): # old author method
authors = line[10:]
elif _CODE_BLOCK in line:
code_block_tp = line.split()[-1]
if code_block_tp in ('pyhid',) or not code_block_tp.endswith('hid'):
notebook_blocks[-1] = '\n'.join(notebook_blocks[-1]).strip()
notebook_blocks.append(line)
# else: hidden block to be dropped (may include more languages
# with time in the above tuple)
elif _MATH_BLOCK in line:
notebook_blocks[-1] = '\n'.join(notebook_blocks[-1]).strip()
notebook_blocks.append(line)
else:
if not isinstance(notebook_blocks[-1], list):
notebook_blocks.append([])
notebook_blocks[-1].append(line)
if isinstance(notebook_blocks[-1], list):
notebook_blocks[-1] = '\n'.join(notebook_blocks[-1]).strip()
# Add block type info
pattern = r'(\d+) +%s'
for i in range(len(notebook_blocks)):
if re.match(pattern % _CODE_BLOCK, notebook_blocks[i]):
m = re.match(pattern % _CODE_BLOCK, notebook_blocks[i])
idx = int(m.group(1))
if ipynb_code_tp[idx] == 'cell':
notebook_blocks[i] = ['cell', notebook_blocks[i]]
elif ipynb_code_tp[idx] == 'cell_hidden':
notebook_blocks[i] = ['cell_hidden', notebook_blocks[i]]
else:
notebook_blocks[i] = ['text', notebook_blocks[i]]
elif re.match(pattern % _MATH_BLOCK, notebook_blocks[i]):
notebook_blocks[i] = ['math', notebook_blocks[i]]
else:
notebook_blocks[i] = ['text', notebook_blocks[i]]
# Go through tex_blocks and wrap in $$
# (doconce.py runs align2equations so there are no align/align*
# environments in tex blocks)
label2tag = {}
tag_counter = 1
for i in range(len(tex_blocks)):
# Extract labels and add tags
labels = re.findall(r'label\{(.+?)\}', tex_blocks[i])
for label in labels:
label2tag[label] = tag_counter
# Insert tag to get labeled equation
tex_blocks[i] = tex_blocks[i].replace(
'label{%s}' % label, 'label{%s} \\tag{%s}' % (label, tag_counter))
tag_counter += 1
# Remove \[ and \] or \begin/end{equation*} in single equations
tex_blocks[i] = tex_blocks[i].replace(r'\[', '')
tex_blocks[i] = tex_blocks[i].replace(r'\]', '')
tex_blocks[i] = tex_blocks[i].replace(r'\begin{equation*}', '')
tex_blocks[i] = tex_blocks[i].replace(r'\end{equation*}', '')
# Check for illegal environments
m = re.search(r'\\begin\{(.+?)\}', tex_blocks[i])
if m:
envir = m.group(1)
if envir not in ('equation', 'equation*', 'align*', 'align',
'array'):
print """\
*** warning: latex envir \\begin{%s} does not work well in Markdown.
Stick to \\[ ... \\], equation, equation*, align, or align*
environments in math environments.
""" % envir
eq_type = 'heading' # or '$$'
eq_type = '$$'
# Markdown: add $$ on each side of the equation
if eq_type == '$$':
# Make sure there are no newline after equation
tex_blocks[i] = '$$\n' + tex_blocks[i].strip() + '\n$$'
# Here: use heading (###) and simple formula (remove newline
# in math expressions to keep everything within a heading) as
# the equation then looks bigger
elif eq_type == 'heading':
tex_blocks[i] = '### $ ' + ' '.join(tex_blocks[i].splitlines()) + ' $'
# Add labels for the eqs above the block (for reference)
if labels:
#label_tp = '<a name="%s"></a>'
label_tp = '<div id="%s"></div>'
tex_blocks[i] = '<!-- Equation labels as ordinary links -->\n' + \
' '.join([label_tp % label
for label in labels]) + '\n\n' + \
tex_blocks[i]
# blocks is now a list of text chunks in markdown and math/code line
# instructions. Insert code and tex blocks
for i in range(len(notebook_blocks)):
if _CODE_BLOCK in notebook_blocks[i][1] or _MATH_BLOCK in notebook_blocks[i][1]:
words = notebook_blocks[i][1].split()
# start of notebook_blocks[i]: number block-indicator code-type
n = int(words[0])
if _CODE_BLOCK in notebook_blocks[i][1]:
notebook_blocks[i][1] = code_blocks[n] # can be list!
if _MATH_BLOCK in notebook_blocks[i][1]:
notebook_blocks[i][1] = tex_blocks[n]
# Make IPython structures
nb_version = int(option('ipynb_version=', '3'))
if nb_version == 3:
from IPython.nbformat.v3 import (
new_code_cell, new_text_cell, new_worksheet,
new_notebook, new_metadata, new_author)
nb = new_worksheet()
elif nb_version == 4:
from IPython.nbformat.v4 import (
new_code_cell, new_markdown_cell, new_notebook)
cells = []
mdstr = [] # plain md format of the notebook
prompt_number = 1
for block_tp, block in notebook_blocks:
if (block_tp == 'text' or block_tp == 'math') and block != '':
# Pure comments between math/code and math/code come
# out as empty blocks, should detect that situation
# (challenging - can have multiple lines of comments,
# or begin and end comment lines with important things between)
if nb_version == 3:
nb.cells.append(new_text_cell(u'markdown', source=block))
elif nb_version == 4:
cells.append(new_markdown_cell(source=block))
mdstr.append(('markdown', block))
elif block_tp == 'cell' and block != '' and block != []:
if isinstance(block, list):
for block_ in block:
block_ = block_.rstrip()
if block_ != '':
if nb_version == 3:
nb.cells.append(new_code_cell(
input=block_,
prompt_number=prompt_number,
collapsed=False))
elif nb_version == 4:
cells.append(new_code_cell(
source=block_,
execution_count=prompt_number))
prompt_number += 1
mdstr.append(('codecell', block_))
else:
block = block.rstrip()
if block != '':
if nb_version == 3:
nb.cells.append(new_code_cell(
input=block,
prompt_number=prompt_number,
collapsed=False))
elif nb_version == 4:
cells.append(new_code_cell(
source=block,
execution_count=prompt_number))
prompt_number += 1
mdstr.append(('codecell', block))
elif block_tp == 'cell_hidden' and block != '':
block = block.rstrip()
if nb_version == 3:
nb.cells.append(new_code_cell(
input=block, prompt_number=prompt_number, collapsed=True))
elif nb_version == 4:
cells.append(new_code_cell(
source=block, execution_count=prompt_number))
prompt_number += 1
mdstr.append(('codecell', block))
"""
# Dump the notebook cells in a simple ASCII format
# (doc/src/ipynb/ipynb_generator.py can translate it back to .ipynb file)
f = open(dofile_basename + '.md-ipynb', 'w')
for cell_tp, block in mdstr:
if cell_tp == 'markdown':
f.write('\n-----\n\n')
elif cell_tp == 'codecell':
f.write('\n-----py\n\n')
f.write(block)
f.close()
"""
if nb_version == 3:
# Catch the title as the first heading
m = re.search(r'^#+\s*(.+)$', filestr, flags=re.MULTILINE)
title = m.group(1).strip() if m else ''
# md below is not used for anything
if authors:
authors = eval(authors)
md = new_metadata(name=title, authors=authors)
else:
md = new_metadata(name=title)
nb = new_notebook(worksheets=[nb], metadata=new_metadata())
# Let us make v4 notebook here by upgrading
from IPython.nbformat.v4 import upgrade
nb = upgrade(nb)
import IPython.nbformat.v4.nbjson as nbjson
# Convert nb to json format
filestr = nbjson.writes(nb)
elif nb_version == 4:
nb = new_notebook(cells=cells)
from IPython.nbformat import writes
filestr = writes(nb, version=4)
# Check that there are no empty cells:
if '"input": []' in filestr:
print '*** error: empty cells in notebook - report bug in DocOnce'
_abort()
# must do the replacements here at the very end when json is written out
# \eqref and labels will not work, but labels (only in math) do no harm
filestr = re.sub(r'([^\\])label\{', r'\g<1>\\\\label{', filestr,
flags=re.MULTILINE)
# \\eqref{} just gives (???) link at this stage - future versions
# will probably support labels
#filestr = re.sub(r'\(ref\{(.+?)\}\)', r'\\eqref{\g<1>}', filestr)
# Now we use explicit references to tags
def subst(m):
label = m.group(1)
try:
return r'[(%s)](#%s)' % (label2tag[label], label)
except KeyError as e:
print '*** error: label "%s" is not defined' % str(e)
filestr = re.sub(r'\(ref\{(.+?)\}\)', subst, filestr)
"""
# MathJax reference to tag (recall that the equations have both label
# and tag (know that tag only works well in HTML, but this mjx-eqn-no
# label does not work in ipynb)
filestr = re.sub(r'\(ref\{(.+?)\}\)',
lambda m: r'[(%s)](#mjx-eqn-%s)' % (label2tag[m.group(1)], label2tag[m.group(1)]), filestr)
"""
#filestr = re.sub(r'\(ref\{(.+?)\}\)', r'Eq (\g<1>)', filestr)
'''
# Final fixes: replace all text between cells by markdown code cells
# Note: the patterns are overlapping so a plain re.sub will not work,
# here we run through all blocks found and subsitute the first remaining
# one, one by one.
pattern = r' \},\n(.+?)\{\n "cell_type":'
begin_pattern = r'^(.+?)\{\n "cell_type":'
remaining_block_begin = re.findall(begin_pattern, filestr, flags=re.DOTALL)
remaining_blocks = re.findall(pattern, filestr, flags=re.DOTALL)
import string
for block in remaining_block_begin + remaining_blocks:
filestr = string.replace(filestr, block, json_markdown(block) + ' ',
maxreplace=1)
filestr_end = re.sub(r' \{\n "cell_type": .+?\n \},\n', '', filestr,
flags=re.DOTALL)
filestr = filestr.replace(filestr_end, json_markdown(filestr_end))
filestr = """{
"metadata": {
"name": "SOME NAME"
},
"nbformat": 3,
"nbformat_minor": 0,
"worksheets": [
{
"cells": [
""" + filestr.rstrip() + '\n'+ \
json_pycode('', final_prompt_no+1, 'python').rstrip()[:-1] + """
],
"metadata": {}
}
]
}"""
'''
return filestr