def test_no_execute(self, course_dir):
with open("nbgrader_config.py", "a") as fh:
fh.write(
"""c.NbGrader.db_assignments = [dict(name='ps1', duedate='2015-02-02 14:58:23.948203 PST')]\n"""
)
fh.write("""c.NbGrader.db_students = [dict(id="foo")]""")
self._copy_file(join("files", "test.ipynb"),
join(course_dir, "source", "ps1", "p1.ipynb"))
run_nbgrader(["assign", "ps1"])
self._copy_file(
join("files", "test-with-output.ipynb"),
join(course_dir, "submitted", "foo", "ps1", "p1.ipynb"))
with open(
join(os.path.dirname(__file__), "files",
"test-with-output.ipynb"), "r") as fh:
orig_contents = reads(fh.read())
run_nbgrader(["autograde", "ps1"])
with open(join(course_dir, "autograded", "foo", "ps1", "p1.ipynb"),
"r") as fh:
new_contents = reads(fh.read())
different = False
for i in range(len(orig_contents.cells)):
orig_cell = orig_contents.cells[i]
new_cell = new_contents.cells[i]
if 'outputs' in orig_cell:
if orig_cell.outputs != new_cell.outputs:
different = True
break
elif 'outputs' in new_cell:
different = True
assert different
run_nbgrader(["autograde", "ps1", "--force", "--no-execute"])
with open(join(course_dir, "autograded", "foo", "ps1", "p1.ipynb"),
"r") as fh:
new_contents = reads(fh.read())
for i in range(len(orig_contents.cells)):
orig_cell = orig_contents.cells[i]
new_cell = new_contents.cells[i]
if 'outputs' in orig_cell:
assert orig_cell.outputs == new_cell.outputs
else:
assert 'outputs' not in new_cell
def main():
for fname in sys.argv[1:]:
nb = reads(io.open(fname, 'r').read())
remove_outputs(nb)
base, ext = os.path.splitext(fname)
new_ipynb = "%s-no-output%s" % (base, ext)
io.open(new_ipynb, 'w', encoding='utf8').write(writes(nb))
def main():
for fname in sys.argv[1:]:
nb = reads(io.open(fname, 'r').read())
remove_outputs(nb)
base, ext = os.path.splitext(fname)
new_ipynb = "%s-no-output%s" % (base, ext)
io.open(new_ipynb, 'w', encoding='utf8').write(writes(nb))
def get_dependencies(self, dirnames):
dependencies = {}
for dirname in dirnames:
files = glob.glob("{}/*.ipynb".format(dirname))
for filename in files:
modname = os.path.splitext(os.path.basename(filename))[0]
with open(filename, "r") as fh:
nb = reads(fh.read())
params = self.extract_parameters(nb)
if '__depends__' not in params:
continue
if '__dest__' not in params:
raise ValueError("__dest__ is not defined in {}".format(filename))
# get sources that are specified in the file
sources = [self.resolve_path(filename, x) for x in params['__depends__']]
targets = params['__dest__']
if not hasattr(targets, '__iter__'):
if targets is None:
targets = []
else:
targets = [targets]
targets = [self.resolve_path(filename, x) for x in targets]
dependencies[os.path.join(dirname, '{}.ipynb'.format(modname))] = {
'targets': targets,
'sources': sources
}
return json.dumps(dependencies, indent=2)
def test_notebook_translator(testdata, parse_rst, name):
"""Ensure that the notebook translator can correctly convert an rst doctree into
the correct sequence of notebook cells."""
rst = testdata(pathlib.Path("tutorial", name + ".rst")).decode("utf8")
json = testdata(pathlib.Path("tutorial", name + ".ipynb")).decode("utf8")
nb = nbformat.reads(json)
# Don't worry about different minor version numbers
nb.nbformat_minor = None
doctree = parse_rst(rst)
translator = tutorial.NotebookTranslator(doctree)
doctree.walkabout(translator)
notebook = translator.asnotebook()
notebook.nbformat_minor = None
# Don't worry about cell ids
for cell in notebook.cells:
cell.id = ""
actual = nbformat.writes(notebook)
expected = nbformat.writes(nb)
assert expected == actual
def test_no_execute(self, course_dir):
with open("nbgrader_config.py", "a") as fh:
fh.write("""c.NbGrader.db_assignments = [dict(name='ps1', duedate='2015-02-02 14:58:23.948203 PST')]\n""")
fh.write("""c.NbGrader.db_students = [dict(id="foo")]""")
self._copy_file(join("files", "test.ipynb"), join(course_dir, "source", "ps1", "p1.ipynb"))
run_nbgrader(["assign", "ps1"])
self._copy_file(
join("files", "test-with-output.ipynb"), join(course_dir, "submitted", "foo", "ps1", "p1.ipynb")
)
with open(join(os.path.dirname(__file__), "files", "test-with-output.ipynb"), "r") as fh:
orig_contents = reads(fh.read())
run_nbgrader(["autograde", "ps1"])
with open(join(course_dir, "autograded", "foo", "ps1", "p1.ipynb"), "r") as fh:
new_contents = reads(fh.read())
different = False
for i in range(len(orig_contents.cells)):
orig_cell = orig_contents.cells[i]
new_cell = new_contents.cells[i]
if "outputs" in orig_cell:
if orig_cell.outputs != new_cell.outputs:
different = True
break
elif "outputs" in new_cell:
different = True
assert different
run_nbgrader(["autograde", "ps1", "--force", "--no-execute"])
with open(join(course_dir, "autograded", "foo", "ps1", "p1.ipynb"), "r") as fh:
new_contents = reads(fh.read())
for i in range(len(orig_contents.cells)):
orig_cell = orig_contents.cells[i]
new_cell = new_contents.cells[i]
if "outputs" in orig_cell:
assert orig_cell.outputs == new_cell.outputs
else:
assert "outputs" not in new_cell
def get_dependencies(self, dirnames):
dependencies = {}
for dirname in dirnames:
files = glob.glob("{}/*.ipynb".format(dirname))
for filename in files:
modname = os.path.splitext(os.path.basename(filename))[0]
with open(filename, "r") as fh:
nb = reads(fh.read())
params = self.extract_parameters(nb)
if '__depends__' not in params:
continue
if '__dest__' not in params:
raise ValueError(
"__dest__ is not defined in {}".format(filename))
# get sources that are specified in the file
sources = [
self.resolve_path(filename, x)
for x in params['__depends__']
]
targets = params['__dest__']
if not hasattr(targets, '__iter__'):
if targets is None:
targets = []
else:
targets = [targets]
targets = [self.resolve_path(filename, x) for x in targets]
dependencies[os.path.join(dirname,
'{}.ipynb'.format(modname))] = {
'targets': targets,
'sources': sources
}
return json.dumps(dependencies, indent=2)
def test_notebook_translator(testdata, name):
"""Ensure that the notebook translator can correctly convert an rst doctree into
the correct sequence of notebook cells."""
rst = testdata(os.path.join("doc", name + ".rst")).decode("utf8")
json = testdata(os.path.join("doc", name + ".ipynb")).decode("utf8")
nb = nbf.reads(json)
# Getting docutils to generate a doctree for us appears to require some gymnastics
# perhaps there is a better way?
parser = Parser()
settings = _get_mock_settings()
reader = Reader()
doctree = reader.read(StringInput(rst), parser, settings)
translator = NotebookTranslator(doctree)
doctree.walkabout(translator)
actual = nbf.writes(translator.asnotebook())
expected = nbf.writes(nb)
assert expected == actual
"language": "python",
"name": "python3"},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3"}}
for cell in nb.cells:
if 'outputs' in cell:
cell['outputs'] = []
if 'execution_count' in cell:
cell['execution_count'] = None
if 'metadata' in cell:
cell['metadata'] = {}
return nb
if __name__ == '__main__':
nb = v4.reads(sys.stdin.read())
nb = strip_output(nb)
output = v4.writes(nb)
if type(output) == str and PY2:
output = output.encode('utf-8')
sys.stdout.write(output)
# Start from scratch to avoid left-over files due to renamed tutorials, changed
# cell numbers, etc.
if os.path.exists(target_dir):
shutil.rmtree(target_dir)
os.mkdir(target_dir)
tutorials = []
for fname in sorted(glob.glob1(src_dir, '*.ipynb')):
basename = fname[:-6]
output_ipynb_fname = os.path.join(target_dir, fname)
output_rst_fname = os.path.join(target_dir, basename + '.rst')
print 'Running', fname
with open(os.path.join(src_dir, fname), 'r') as f:
notebook = reads(f.read())
# The first line of the tutorial file should give the title
title = notebook.cells[0]['source'].split('\n')[0].strip('# ')
tutorials.append((basename, title))
# Execute the notebook
preprocessor = ExecutePreprocessor()
preprocessor.allow_errors = True
notebook, _ = preprocessor.preprocess(notebook,
{'metadata': {'path': src_dir}})
print 'Saving notebook and converting to RST'
exporter = NotebookExporter()
output, _ = exporter.from_notebook_node(notebook)
with codecs.open(output_ipynb_fname, 'w', encoding='utf-8') as f:
Opens a notebook, strips its output, and writes the outputless version to the original file.
Useful mainly as a git filter or pre-commit hook for users who don't want to track output in VCS.
This does mostly the same thing as the `Clear All Output` command in the notebook UI.
LICENSE: Public Domain
"""
import io
import sys
from nbformat import v4
def strip_output(nb):
"""strip the outputs from a notebook object"""
nb.metadata.pop('signature', None)
for cell in nb.cells:
if 'outputs' in cell:
cell['outputs'] = []
if 'prompt_number' in cell:
cell['prompt_number'] = None
return nb
if __name__ == '__main__':
nb = v4.reads(sys.stdin.read())
nb = strip_output(nb)
sys.stdout.write(v4.writes(nb))
target_dir = os.path.abspath('../../../docs_sphinx/resources/tutorials')
# Start from scratch to avoid left-over files due to renamed tutorials, changed
# cell numbers, etc.
if os.path.exists(target_dir):
shutil.rmtree(target_dir)
os.mkdir(target_dir)
tutorials = []
for fname in sorted(glob.glob1(src_dir, '*.ipynb')):
basename = fname[:-6]
output_ipynb_fname = os.path.join(target_dir, fname)
output_rst_fname = os.path.join(target_dir, basename + '.rst')
print 'Running', fname
notebook = reads(open(os.path.join(src_dir, fname), 'r').read())
# The first line of the tutorial file should give the title
title = notebook.cells[0]['source'].split('\n')[0].strip('# ')
tutorials.append((basename, title))
# Execute the notebook
preprocessor = ExecutePreprocessor()
preprocessor.allow_errors = True
notebook, _ = preprocessor.preprocess(notebook, {})
print 'Saving notebook and converting to RST'
exporter = NotebookExporter()
output, _ = exporter.from_notebook_node(notebook)
codecs.open(output_ipynb_fname, 'w', encoding='utf-8').write(output)
def parse_line(line):
if not line.startswith('#'):
return None
ilevel = 0
for char in line:
if char == '#': ilevel += 1
else: break
name = line[ilevel:].strip()
return ilevel, name
if __name__ == '__main__':
with open(sys.argv[1], 'r') as nbfile:
nb = v4.reads(nbfile.read())
print('Contents\n=======\n---')
for cell in nb.cells:
if cell['cell_type'] == 'markdown':
for line in cell['source'].splitlines():
header = parse_line(line)
if header is None: continue
ilevel, name = header
print(' '*(ilevel-1) + ' - [%s](#%s)'%(name, name.replace(' ','-')))
# Start from scratch to avoid left-over files due to renamed tutorials, changed
# cell numbers, etc.
if os.path.exists(target_dir):
shutil.rmtree(target_dir)
os.mkdir(target_dir)
tutorials = []
for fname in sorted(glob.glob1(src_dir, '*.ipynb')):
basename = fname[:-6]
output_ipynb_fname = os.path.join(target_dir, fname)
output_rst_fname = os.path.join(target_dir, basename + '.rst')
print 'Running', fname
with open(os.path.join(src_dir, fname), 'r') as f:
notebook = reads(f.read())
# The first line of the tutorial file should give the title
title = notebook.cells[0]['source'].split('\n')[0].strip('# ')
tutorials.append((basename, title))
# Execute the notebook
preprocessor = ExecutePreprocessor()
preprocessor.allow_errors = True
notebook, _ = preprocessor.preprocess(notebook,
{'metadata': {
'path': src_dir
}})
print 'Saving notebook and converting to RST'
exporter = NotebookExporter()
(<button class='fa fa-plus icon-plus btn btn-xs btn-default'></button>), or pressing `SHIFT+ENTER` while this cell
is selected.
<div class="alert alert-block alert-warning" role="alert" style="margin: 10px">
<p><b>WARNING</b></p>
<p>Don't rely on this server for anything you want to last - your session will be
deleted after a short period of inactivity.</p>
</div>
This notebook is running on [mybinder.org](http://mybinder.org) created by the
[Freeman lab](https://www.janelia.org/lab/freeman-lab).
'''
if not os.path.exists('tutorials'):
os.mkdir('tutorials')
for notebook in sorted(glob.glob('_tutorials/*.ipynb')):
with open(notebook, 'r') as f:
content = reads(f.read())
title = content.cells[0]['source'].split('\n')[0].strip('# ')
all_tutorials.append((title, notebook[1:].replace('\\', '/')))
# Insert a note about Jupyter notebooks at the top with a download link
content.cells.insert(1, new_markdown_cell(note))
(path, filename) = os.path.split(notebook)
with open('tutorials/' + filename, 'w') as f:
nbf.write(content, f)
shutil.rmtree('_tutorials')
###################### GENERATE EXAMPLES NOTEBOOKS ############################
def do_stripping():
nb = v4.reads(sys.stdin.read())
nb = strip_output(nb)
sys.stdout.write(v4.writes(nb))
def parse_line(line):
if not line.startswith('#'):
return None
ilevel = 0
for char in line:
if char == '#': ilevel += 1
else: break
name = line[ilevel:].strip()
return ilevel, name
if __name__ == '__main__':
with open(sys.argv[1], 'r') as nbfile:
nb = v4.reads(nbfile.read())
print 'Contents\n=======\n---'
for cell in nb.cells:
if cell['cell_type'] == 'markdown':
for line in cell['source'].splitlines():
header = parse_line(line)
if header is None: continue
ilevel, name = header
print ' ' * (ilevel - 1) + ' - [%s](#%s)' % (
name, name.replace(' ', '-'))
def load_notebook(file_name):
""" a convenience wrapper around a "safe" notebook load
"""
file_path = check_path(file_name)
return reads(file_path.read_text())
To run the code below:
1. Click on the cell to select it.
2. Press `SHIFT+ENTER` on your keyboard or press the play button (<button class='fa fa-play icon-play btn btn-xs btn-default'></button>) in the toolbar above.
Feel free to create new cells using the plus button (<button class='fa fa-plus icon-plus btn btn-xs btn-default'></button>), or pressing `SHIFT+ENTER` while this cell is selected.
<div class="alert alert-warning" role="alert" style="margin: 10px">
<p>**WARNING**</p>
<p>Don't rely on this server for anything you want to last - your server will be *deleted after 10 minutes of inactivity*.</p>
</div>
'''
if not os.path.exists('tutorials'):
os.mkdir('tutorials')
for notebook in glob.glob('_tutorials/*.ipynb'):
with open(notebook, 'r') as f:
content = reads(f.read())
# Insert a note about Jupyter notebooks at the top with a download link
content.cells.insert(1, new_markdown_cell(note))
(path, filename) = os.path.split(notebook) #[-1]
with open('tutorials/' + filename, 'w') as f:
nbf.write(content, f)
shutil.rmtree('_tutorials')
magic = '''%matplotlib notebook\n'''
if not os.path.exists('examples'):
os.mkdir('examples')
for root, subfolders, files in os.walk('_examples'):
def parse(path):
source = read_text_file(path)
result = {}
if path.endswith('.html'):
if utils.detect_blogofile(source):
heading, info, other_html = utils.convert_blogofile(source)
parts = utils.parse_rst(heading)
body_html = parts['docinfo'] + other_html
body_html = utils.pygmentize_pre_blocks(body_html)
body_html = body_html.replace('\n</pre>', '</pre>')
result['title'] = utils.html_parser.unescape(parts['title'])
result['needs_disqus'] = True
result['date'] = info['date']
result['tags'] = info['tags']
else:
result['title'] = utils.find_title_in_html(source)
body_html = SimpleTemplate(source)
result['needs_disqus'] = False
result['date'] = None
result['tags'] = ()
result['body'] = body_html
result['next_link'] = None
result['previous_link'] = None
result['tags'] = [tag for tag in result['tags'] if tag]
elif path.endswith('.md'):
if utils.detect_blogofile(source):
heading, info, body = utils.convert_blogofile(source)
source = body
result['date'] = info['date']
result['title'] = info['title']
result['needs_disqus'] = True
else:
result['needs_disqus'] = False
result['body'] = commonmark(source)
elif path.endswith('.rst'):
if utils.detect_blogofile(source):
heading, info, body = utils.convert_blogofile(source)
source = heading + body
result['title'] = info['title']
del heading, info, body
result['needs_disqus'] = True
else:
result['needs_disqus'] = False
doctree = publish_doctree(source)
docinfos = doctree.traverse(nodes.docinfo)
docinfo = {c.tagname: str(c.children[0])
for i in docinfos for c in i.children}
parts = utils.parse_rst(source)
# parts = publish_from_doctree(source, writer_name='html',
# settings_overrides={'initial_header_level': 2})
body = parts['docinfo'] + utils.pygmentize_pre_blocks(parts['fragment'])
result['body'] = body
result['date'] = datetime.strptime(
docinfo.get('date'), '%d %B %Y').date()
if 'title' not in result:
result['title'] = parts['title']
elif path.endswith('.ipynb'):
notebook = nbformat.reads(source)
docinfo = utils.build_docinfo_block_for_notebook(notebook)
exporter = HTMLExporter(config=None, extra_loaders=[dl],
filters=filters)
exporter.template_file = 'brandon.tpl'
#notebook = nbformat.convert(notebook, nbformat.current_nbformat)
body, resources = exporter.from_notebook_node(notebook)
body = body.replace('\n</pre>', '</pre>')
body = body.replace('</h1>', '</h1>\n' + docinfo.rstrip())
date = notebook['metadata'].get('date')
if date is not None:
date = datetime.strptime(date, '%d %B %Y').date()
result['body'] = body
result['date'] = date
result['needs_disqus'] = notebook['metadata'].get('needs_disqus')
result['title'] = (notebook['metadata'].get('name', None)
or utils.find_title_in_html(body))
else:
raise ValueError('unrecognized path: {}'.format(path))
return result
def parse(path):
source = read_text_file(path)
result = {}
if path.endswith('.html'):
if utils.detect_blogofile(source):
heading, info, other_html = utils.convert_blogofile(source)
parts = utils.parse_rst(heading)
body_html = parts['docinfo'] + other_html
body_html = utils.pygmentize_pre_blocks(body_html)
body_html = body_html.replace('\n</pre>', '</pre>')
result['title'] = utils.html_parser.unescape(parts['title'])
result['needs_disqus'] = True
result['date'] = info['date']
result['tags'] = info['tags']
else:
result['title'] = utils.find_title_in_html(source)
body_html = SimpleTemplate(source)
result['needs_disqus'] = False
result['date'] = None
result['tags'] = ()
result['body'] = body_html
result['next_link'] = None
result['previous_link'] = None
result['tags'] = [tag for tag in result['tags'] if tag]
elif path.endswith('.md'):
if utils.detect_blogofile(source):
heading, info, body = utils.convert_blogofile(source)
source = body
result['date'] = info['date']
result['title'] = info['title']
result['needs_disqus'] = True
else:
result['needs_disqus'] = False
result['body'] = commonmark(source)
elif path.endswith('.rst'):
if utils.detect_blogofile(source):
heading, info, body = utils.convert_blogofile(source)
source = heading + body
result['title'] = info['title']
del heading, info, body
result['needs_disqus'] = True
else:
result['needs_disqus'] = False
doctree = publish_doctree(source)
docinfos = doctree.traverse(nodes.docinfo)
docinfo = {
c.tagname: str(c.children[0])
for i in docinfos for c in i.children
}
parts = utils.parse_rst(source)
# parts = publish_from_doctree(source, writer_name='html',
# settings_overrides={'initial_header_level': 2})
body = parts['docinfo'] + utils.pygmentize_pre_blocks(
parts['fragment'])
result['body'] = body
result['date'] = datetime.strptime(docinfo.get('date'),
'%d %B %Y').date()
if 'title' not in result:
result['title'] = parts['title']
elif path.endswith('.ipynb'):
notebook = nbformat.reads(source)
docinfo = utils.build_docinfo_block_for_notebook(notebook)
exporter = HTMLExporter(config=None,
extra_loaders=[dl],
filters=filters)
exporter.template_file = 'brandon.tpl'
#notebook = nbformat.convert(notebook, nbformat.current_nbformat)
body, resources = exporter.from_notebook_node(notebook)
body = body.replace('\n</pre>', '</pre>')
body = body.replace('</h1>', '</h1>\n' + docinfo.rstrip())
date = notebook['metadata'].get('date')
if date is not None:
date = datetime.strptime(date, '%d %B %Y').date()
result['body'] = body
result['date'] = date
result['needs_disqus'] = notebook['metadata'].get('needs_disqus')
result['title'] = (notebook['metadata'].get('name', None)
or utils.find_title_in_html(body))
else:
raise ValueError('unrecognized path: {}'.format(path))
return result
