def test_no_execute(self, course_dir): with open("nbgrader_config.py", "a") as fh: fh.write( """c.NbGrader.db_assignments = [dict(name='ps1', duedate='2015-02-02 14:58:23.948203 PST')]\n""" ) fh.write("""c.NbGrader.db_students = [dict(id="foo")]""") self._copy_file(join("files", "test.ipynb"), join(course_dir, "source", "ps1", "p1.ipynb")) run_nbgrader(["assign", "ps1"]) self._copy_file( join("files", "test-with-output.ipynb"), join(course_dir, "submitted", "foo", "ps1", "p1.ipynb")) with open( join(os.path.dirname(__file__), "files", "test-with-output.ipynb"), "r") as fh: orig_contents = reads(fh.read()) run_nbgrader(["autograde", "ps1"]) with open(join(course_dir, "autograded", "foo", "ps1", "p1.ipynb"), "r") as fh: new_contents = reads(fh.read()) different = False for i in range(len(orig_contents.cells)): orig_cell = orig_contents.cells[i] new_cell = new_contents.cells[i] if 'outputs' in orig_cell: if orig_cell.outputs != new_cell.outputs: different = True break elif 'outputs' in new_cell: different = True assert different run_nbgrader(["autograde", "ps1", "--force", "--no-execute"]) with open(join(course_dir, "autograded", "foo", "ps1", "p1.ipynb"), "r") as fh: new_contents = reads(fh.read()) for i in range(len(orig_contents.cells)): orig_cell = orig_contents.cells[i] new_cell = new_contents.cells[i] if 'outputs' in orig_cell: assert orig_cell.outputs == new_cell.outputs else: assert 'outputs' not in new_cell
def main(): for fname in sys.argv[1:]: nb = reads(io.open(fname, 'r').read()) remove_outputs(nb) base, ext = os.path.splitext(fname) new_ipynb = "%s-no-output%s" % (base, ext) io.open(new_ipynb, 'w', encoding='utf8').write(writes(nb))
def get_dependencies(self, dirnames): dependencies = {} for dirname in dirnames: files = glob.glob("{}/*.ipynb".format(dirname)) for filename in files: modname = os.path.splitext(os.path.basename(filename))[0] with open(filename, "r") as fh: nb = reads(fh.read()) params = self.extract_parameters(nb) if '__depends__' not in params: continue if '__dest__' not in params: raise ValueError("__dest__ is not defined in {}".format(filename)) # get sources that are specified in the file sources = [self.resolve_path(filename, x) for x in params['__depends__']] targets = params['__dest__'] if not hasattr(targets, '__iter__'): if targets is None: targets = [] else: targets = [targets] targets = [self.resolve_path(filename, x) for x in targets] dependencies[os.path.join(dirname, '{}.ipynb'.format(modname))] = { 'targets': targets, 'sources': sources } return json.dumps(dependencies, indent=2)
def test_notebook_translator(testdata, parse_rst, name): """Ensure that the notebook translator can correctly convert an rst doctree into the correct sequence of notebook cells.""" rst = testdata(pathlib.Path("tutorial", name + ".rst")).decode("utf8") json = testdata(pathlib.Path("tutorial", name + ".ipynb")).decode("utf8") nb = nbformat.reads(json) # Don't worry about different minor version numbers nb.nbformat_minor = None doctree = parse_rst(rst) translator = tutorial.NotebookTranslator(doctree) doctree.walkabout(translator) notebook = translator.asnotebook() notebook.nbformat_minor = None # Don't worry about cell ids for cell in notebook.cells: cell.id = "" actual = nbformat.writes(notebook) expected = nbformat.writes(nb) assert expected == actual
def test_no_execute(self, course_dir): with open("nbgrader_config.py", "a") as fh: fh.write("""c.NbGrader.db_assignments = [dict(name='ps1', duedate='2015-02-02 14:58:23.948203 PST')]\n""") fh.write("""c.NbGrader.db_students = [dict(id="foo")]""") self._copy_file(join("files", "test.ipynb"), join(course_dir, "source", "ps1", "p1.ipynb")) run_nbgrader(["assign", "ps1"]) self._copy_file( join("files", "test-with-output.ipynb"), join(course_dir, "submitted", "foo", "ps1", "p1.ipynb") ) with open(join(os.path.dirname(__file__), "files", "test-with-output.ipynb"), "r") as fh: orig_contents = reads(fh.read()) run_nbgrader(["autograde", "ps1"]) with open(join(course_dir, "autograded", "foo", "ps1", "p1.ipynb"), "r") as fh: new_contents = reads(fh.read()) different = False for i in range(len(orig_contents.cells)): orig_cell = orig_contents.cells[i] new_cell = new_contents.cells[i] if "outputs" in orig_cell: if orig_cell.outputs != new_cell.outputs: different = True break elif "outputs" in new_cell: different = True assert different run_nbgrader(["autograde", "ps1", "--force", "--no-execute"]) with open(join(course_dir, "autograded", "foo", "ps1", "p1.ipynb"), "r") as fh: new_contents = reads(fh.read()) for i in range(len(orig_contents.cells)): orig_cell = orig_contents.cells[i] new_cell = new_contents.cells[i] if "outputs" in orig_cell: assert orig_cell.outputs == new_cell.outputs else: assert "outputs" not in new_cell
def get_dependencies(self, dirnames): dependencies = {} for dirname in dirnames: files = glob.glob("{}/*.ipynb".format(dirname)) for filename in files: modname = os.path.splitext(os.path.basename(filename))[0] with open(filename, "r") as fh: nb = reads(fh.read()) params = self.extract_parameters(nb) if '__depends__' not in params: continue if '__dest__' not in params: raise ValueError( "__dest__ is not defined in {}".format(filename)) # get sources that are specified in the file sources = [ self.resolve_path(filename, x) for x in params['__depends__'] ] targets = params['__dest__'] if not hasattr(targets, '__iter__'): if targets is None: targets = [] else: targets = [targets] targets = [self.resolve_path(filename, x) for x in targets] dependencies[os.path.join(dirname, '{}.ipynb'.format(modname))] = { 'targets': targets, 'sources': sources } return json.dumps(dependencies, indent=2)
def test_notebook_translator(testdata, name): """Ensure that the notebook translator can correctly convert an rst doctree into the correct sequence of notebook cells.""" rst = testdata(os.path.join("doc", name + ".rst")).decode("utf8") json = testdata(os.path.join("doc", name + ".ipynb")).decode("utf8") nb = nbf.reads(json) # Getting docutils to generate a doctree for us appears to require some gymnastics # perhaps there is a better way? parser = Parser() settings = _get_mock_settings() reader = Reader() doctree = reader.read(StringInput(rst), parser, settings) translator = NotebookTranslator(doctree) doctree.walkabout(translator) actual = nbf.writes(translator.asnotebook()) expected = nbf.writes(nb) assert expected == actual
"language": "python", "name": "python3"}, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3}, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3"}} for cell in nb.cells: if 'outputs' in cell: cell['outputs'] = [] if 'execution_count' in cell: cell['execution_count'] = None if 'metadata' in cell: cell['metadata'] = {} return nb if __name__ == '__main__': nb = v4.reads(sys.stdin.read()) nb = strip_output(nb) output = v4.writes(nb) if type(output) == str and PY2: output = output.encode('utf-8') sys.stdout.write(output)
# Start from scratch to avoid left-over files due to renamed tutorials, changed # cell numbers, etc. if os.path.exists(target_dir): shutil.rmtree(target_dir) os.mkdir(target_dir) tutorials = [] for fname in sorted(glob.glob1(src_dir, '*.ipynb')): basename = fname[:-6] output_ipynb_fname = os.path.join(target_dir, fname) output_rst_fname = os.path.join(target_dir, basename + '.rst') print 'Running', fname with open(os.path.join(src_dir, fname), 'r') as f: notebook = reads(f.read()) # The first line of the tutorial file should give the title title = notebook.cells[0]['source'].split('\n')[0].strip('# ') tutorials.append((basename, title)) # Execute the notebook preprocessor = ExecutePreprocessor() preprocessor.allow_errors = True notebook, _ = preprocessor.preprocess(notebook, {'metadata': {'path': src_dir}}) print 'Saving notebook and converting to RST' exporter = NotebookExporter() output, _ = exporter.from_notebook_node(notebook) with codecs.open(output_ipynb_fname, 'w', encoding='utf-8') as f:
Opens a notebook, strips its output, and writes the outputless version to the original file. Useful mainly as a git filter or pre-commit hook for users who don't want to track output in VCS. This does mostly the same thing as the `Clear All Output` command in the notebook UI. LICENSE: Public Domain """ import io import sys from nbformat import v4 def strip_output(nb): """strip the outputs from a notebook object""" nb.metadata.pop('signature', None) for cell in nb.cells: if 'outputs' in cell: cell['outputs'] = [] if 'prompt_number' in cell: cell['prompt_number'] = None return nb if __name__ == '__main__': nb = v4.reads(sys.stdin.read()) nb = strip_output(nb) sys.stdout.write(v4.writes(nb))
target_dir = os.path.abspath('../../../docs_sphinx/resources/tutorials') # Start from scratch to avoid left-over files due to renamed tutorials, changed # cell numbers, etc. if os.path.exists(target_dir): shutil.rmtree(target_dir) os.mkdir(target_dir) tutorials = [] for fname in sorted(glob.glob1(src_dir, '*.ipynb')): basename = fname[:-6] output_ipynb_fname = os.path.join(target_dir, fname) output_rst_fname = os.path.join(target_dir, basename + '.rst') print 'Running', fname notebook = reads(open(os.path.join(src_dir, fname), 'r').read()) # The first line of the tutorial file should give the title title = notebook.cells[0]['source'].split('\n')[0].strip('# ') tutorials.append((basename, title)) # Execute the notebook preprocessor = ExecutePreprocessor() preprocessor.allow_errors = True notebook, _ = preprocessor.preprocess(notebook, {}) print 'Saving notebook and converting to RST' exporter = NotebookExporter() output, _ = exporter.from_notebook_node(notebook) codecs.open(output_ipynb_fname, 'w', encoding='utf-8').write(output)
def parse_line(line): if not line.startswith('#'): return None ilevel = 0 for char in line: if char == '#': ilevel += 1 else: break name = line[ilevel:].strip() return ilevel, name if __name__ == '__main__': with open(sys.argv[1], 'r') as nbfile: nb = v4.reads(nbfile.read()) print('Contents\n=======\n---') for cell in nb.cells: if cell['cell_type'] == 'markdown': for line in cell['source'].splitlines(): header = parse_line(line) if header is None: continue ilevel, name = header print(' '*(ilevel-1) + ' - [%s](#%s)'%(name, name.replace(' ','-')))
# Start from scratch to avoid left-over files due to renamed tutorials, changed # cell numbers, etc. if os.path.exists(target_dir): shutil.rmtree(target_dir) os.mkdir(target_dir) tutorials = [] for fname in sorted(glob.glob1(src_dir, '*.ipynb')): basename = fname[:-6] output_ipynb_fname = os.path.join(target_dir, fname) output_rst_fname = os.path.join(target_dir, basename + '.rst') print 'Running', fname with open(os.path.join(src_dir, fname), 'r') as f: notebook = reads(f.read()) # The first line of the tutorial file should give the title title = notebook.cells[0]['source'].split('\n')[0].strip('# ') tutorials.append((basename, title)) # Execute the notebook preprocessor = ExecutePreprocessor() preprocessor.allow_errors = True notebook, _ = preprocessor.preprocess(notebook, {'metadata': { 'path': src_dir }}) print 'Saving notebook and converting to RST' exporter = NotebookExporter()
(<button class='fa fa-plus icon-plus btn btn-xs btn-default'></button>), or pressing `SHIFT+ENTER` while this cell is selected. <div class="alert alert-block alert-warning" role="alert" style="margin: 10px"> <p><b>WARNING</b></p> <p>Don't rely on this server for anything you want to last - your session will be deleted after a short period of inactivity.</p> </div> This notebook is running on [mybinder.org](http://mybinder.org) created by the [Freeman lab](https://www.janelia.org/lab/freeman-lab). ''' if not os.path.exists('tutorials'): os.mkdir('tutorials') for notebook in sorted(glob.glob('_tutorials/*.ipynb')): with open(notebook, 'r') as f: content = reads(f.read()) title = content.cells[0]['source'].split('\n')[0].strip('# ') all_tutorials.append((title, notebook[1:].replace('\\', '/'))) # Insert a note about Jupyter notebooks at the top with a download link content.cells.insert(1, new_markdown_cell(note)) (path, filename) = os.path.split(notebook) with open('tutorials/' + filename, 'w') as f: nbf.write(content, f) shutil.rmtree('_tutorials') ###################### GENERATE EXAMPLES NOTEBOOKS ############################
def do_stripping(): nb = v4.reads(sys.stdin.read()) nb = strip_output(nb) sys.stdout.write(v4.writes(nb))
def parse_line(line): if not line.startswith('#'): return None ilevel = 0 for char in line: if char == '#': ilevel += 1 else: break name = line[ilevel:].strip() return ilevel, name if __name__ == '__main__': with open(sys.argv[1], 'r') as nbfile: nb = v4.reads(nbfile.read()) print 'Contents\n=======\n---' for cell in nb.cells: if cell['cell_type'] == 'markdown': for line in cell['source'].splitlines(): header = parse_line(line) if header is None: continue ilevel, name = header print ' ' * (ilevel - 1) + ' - [%s](#%s)' % ( name, name.replace(' ', '-'))
def load_notebook(file_name): """ a convenience wrapper around a "safe" notebook load """ file_path = check_path(file_name) return reads(file_path.read_text())
To run the code below: 1. Click on the cell to select it. 2. Press `SHIFT+ENTER` on your keyboard or press the play button (<button class='fa fa-play icon-play btn btn-xs btn-default'></button>) in the toolbar above. Feel free to create new cells using the plus button (<button class='fa fa-plus icon-plus btn btn-xs btn-default'></button>), or pressing `SHIFT+ENTER` while this cell is selected. <div class="alert alert-warning" role="alert" style="margin: 10px"> <p>**WARNING**</p> <p>Don't rely on this server for anything you want to last - your server will be *deleted after 10 minutes of inactivity*.</p> </div> ''' if not os.path.exists('tutorials'): os.mkdir('tutorials') for notebook in glob.glob('_tutorials/*.ipynb'): with open(notebook, 'r') as f: content = reads(f.read()) # Insert a note about Jupyter notebooks at the top with a download link content.cells.insert(1, new_markdown_cell(note)) (path, filename) = os.path.split(notebook) #[-1] with open('tutorials/' + filename, 'w') as f: nbf.write(content, f) shutil.rmtree('_tutorials') magic = '''%matplotlib notebook\n''' if not os.path.exists('examples'): os.mkdir('examples') for root, subfolders, files in os.walk('_examples'):
def parse(path): source = read_text_file(path) result = {} if path.endswith('.html'): if utils.detect_blogofile(source): heading, info, other_html = utils.convert_blogofile(source) parts = utils.parse_rst(heading) body_html = parts['docinfo'] + other_html body_html = utils.pygmentize_pre_blocks(body_html) body_html = body_html.replace('\n</pre>', '</pre>') result['title'] = utils.html_parser.unescape(parts['title']) result['needs_disqus'] = True result['date'] = info['date'] result['tags'] = info['tags'] else: result['title'] = utils.find_title_in_html(source) body_html = SimpleTemplate(source) result['needs_disqus'] = False result['date'] = None result['tags'] = () result['body'] = body_html result['next_link'] = None result['previous_link'] = None result['tags'] = [tag for tag in result['tags'] if tag] elif path.endswith('.md'): if utils.detect_blogofile(source): heading, info, body = utils.convert_blogofile(source) source = body result['date'] = info['date'] result['title'] = info['title'] result['needs_disqus'] = True else: result['needs_disqus'] = False result['body'] = commonmark(source) elif path.endswith('.rst'): if utils.detect_blogofile(source): heading, info, body = utils.convert_blogofile(source) source = heading + body result['title'] = info['title'] del heading, info, body result['needs_disqus'] = True else: result['needs_disqus'] = False doctree = publish_doctree(source) docinfos = doctree.traverse(nodes.docinfo) docinfo = {c.tagname: str(c.children[0]) for i in docinfos for c in i.children} parts = utils.parse_rst(source) # parts = publish_from_doctree(source, writer_name='html', # settings_overrides={'initial_header_level': 2}) body = parts['docinfo'] + utils.pygmentize_pre_blocks(parts['fragment']) result['body'] = body result['date'] = datetime.strptime( docinfo.get('date'), '%d %B %Y').date() if 'title' not in result: result['title'] = parts['title'] elif path.endswith('.ipynb'): notebook = nbformat.reads(source) docinfo = utils.build_docinfo_block_for_notebook(notebook) exporter = HTMLExporter(config=None, extra_loaders=[dl], filters=filters) exporter.template_file = 'brandon.tpl' #notebook = nbformat.convert(notebook, nbformat.current_nbformat) body, resources = exporter.from_notebook_node(notebook) body = body.replace('\n</pre>', '</pre>') body = body.replace('</h1>', '</h1>\n' + docinfo.rstrip()) date = notebook['metadata'].get('date') if date is not None: date = datetime.strptime(date, '%d %B %Y').date() result['body'] = body result['date'] = date result['needs_disqus'] = notebook['metadata'].get('needs_disqus') result['title'] = (notebook['metadata'].get('name', None) or utils.find_title_in_html(body)) else: raise ValueError('unrecognized path: {}'.format(path)) return result
def parse(path): source = read_text_file(path) result = {} if path.endswith('.html'): if utils.detect_blogofile(source): heading, info, other_html = utils.convert_blogofile(source) parts = utils.parse_rst(heading) body_html = parts['docinfo'] + other_html body_html = utils.pygmentize_pre_blocks(body_html) body_html = body_html.replace('\n</pre>', '</pre>') result['title'] = utils.html_parser.unescape(parts['title']) result['needs_disqus'] = True result['date'] = info['date'] result['tags'] = info['tags'] else: result['title'] = utils.find_title_in_html(source) body_html = SimpleTemplate(source) result['needs_disqus'] = False result['date'] = None result['tags'] = () result['body'] = body_html result['next_link'] = None result['previous_link'] = None result['tags'] = [tag for tag in result['tags'] if tag] elif path.endswith('.md'): if utils.detect_blogofile(source): heading, info, body = utils.convert_blogofile(source) source = body result['date'] = info['date'] result['title'] = info['title'] result['needs_disqus'] = True else: result['needs_disqus'] = False result['body'] = commonmark(source) elif path.endswith('.rst'): if utils.detect_blogofile(source): heading, info, body = utils.convert_blogofile(source) source = heading + body result['title'] = info['title'] del heading, info, body result['needs_disqus'] = True else: result['needs_disqus'] = False doctree = publish_doctree(source) docinfos = doctree.traverse(nodes.docinfo) docinfo = { c.tagname: str(c.children[0]) for i in docinfos for c in i.children } parts = utils.parse_rst(source) # parts = publish_from_doctree(source, writer_name='html', # settings_overrides={'initial_header_level': 2}) body = parts['docinfo'] + utils.pygmentize_pre_blocks( parts['fragment']) result['body'] = body result['date'] = datetime.strptime(docinfo.get('date'), '%d %B %Y').date() if 'title' not in result: result['title'] = parts['title'] elif path.endswith('.ipynb'): notebook = nbformat.reads(source) docinfo = utils.build_docinfo_block_for_notebook(notebook) exporter = HTMLExporter(config=None, extra_loaders=[dl], filters=filters) exporter.template_file = 'brandon.tpl' #notebook = nbformat.convert(notebook, nbformat.current_nbformat) body, resources = exporter.from_notebook_node(notebook) body = body.replace('\n</pre>', '</pre>') body = body.replace('</h1>', '</h1>\n' + docinfo.rstrip()) date = notebook['metadata'].get('date') if date is not None: date = datetime.strptime(date, '%d %B %Y').date() result['body'] = body result['date'] = date result['needs_disqus'] = notebook['metadata'].get('needs_disqus') result['title'] = (notebook['metadata'].get('name', None) or utils.find_title_in_html(body)) else: raise ValueError('unrecognized path: {}'.format(path)) return result