def mdstrip(paths): for path in paths: if os.path.isdir(path): files = glob(os.path.join(path, "*.ipynb")) else: files = [path] for in_file in files: input_nb_name = basename(in_file) slug = input_nb_name[:-6] title = slug.replace("_", " ") actual_title_nb = io.StringIO( title_data.replace("{{ title }}", title)) title_nb = nbf.read(actual_title_nb, "ipynb") title_cell = title_nb.worksheets[0].cells[0] input_nb = nbf.read(open(in_file), "ipynb") worksheet = input_nb.worksheets[0] # add graphic here & append to cell_list cell_list = [title_cell] for cell in worksheet.cells: if cell.cell_type == ("code"): cell.outputs = [] cell_list.append(cell) elif cell.cell_type == "heading": cell_list.append(cell) output_nb = nbf.new_notebook() output_nb_name = slug+".prod.ipynb" output_nb.worksheets.append(nbf.new_worksheet(cells=cell_list)) with open(output_nb_name, 'w') as f: nbf.write(output_nb, f, "ipynb")
def convert_nb(nbname): rst_name = "%s.rst" % nbname nbname = "%s.ipynb" % nbname # Do nothing if already built. if os.path.exists(rst_name) and \ os.path.getmtime(rst_name) >= os.path.getmtime(nbname): print("\t%s is up to date; nothing to do." % rst_name) return os.system("runipy --o %s --matplotlib --quiet" % nbname) with io.open(nbname, 'r', encoding='utf8') as f: nb = current.read(f, 'json') nb = clean_for_doc(nb) print("Writing to", nbname) with io.open(nbname, 'w', encoding='utf8') as f: current.write(nb, f, 'json') # Convert to rst. os.system("ipython nbconvert --to rst %s" % nbname) with io.open(nbname, 'r', encoding='utf8') as f: nb = current.read(f, 'json') nb = strip_output(nb) print("Writing to", nbname) with io.open(nbname, 'w', encoding='utf8') as f: current.write(nb, f, 'json')
def testRunNotebooks(self): notebook_dir = path.join('tests', 'input') for notebook_path in glob(path.join(notebook_dir, '*.ipynb')): notebook_file = path.basename(notebook_path) print notebook_file expected_file = path.join('tests', 'expected', notebook_file) runner = NotebookRunner(read(open(notebook_path), 'json'), working_dir=notebook_dir) runner.run_notebook(True) expected = read(open(expected_file), 'json') self.assert_notebooks_equal(expected, runner.nb)
def testRunNotebooks(self): input_glob = path.join('tests', 'input', '*.ipynb') for notebook_file in glob(input_glob): notebook_file_base = path.basename(notebook_file) print notebook_file_base expected_file = path.join('tests', 'expected', notebook_file_base) runner = NotebookRunner(read(open(notebook_file), 'json')) runner.run_notebook(True) expected = read(open(expected_file), 'json') self.assert_notebooks_equal(expected, runner.nb)
def restore_checkpoint(self, checkpoint_id, name, path=""): """restore a notebook to a checkpointed state""" path = path.strip("/") self.log.info("restoring Notebook %s from checkpoint %s", name, checkpoint_id) nb_path = self._get_os_path(name, path) cp_path = self.get_checkpoint_path(checkpoint_id, name, path) if not os.path.isfile(cp_path): self.log.debug("checkpoint file does not exist: %s", cp_path) raise web.HTTPError(404, u"Notebook checkpoint does not exist: %s-%s" % (name, checkpoint_id)) # ensure notebook is readable (never restore from an unreadable notebook) with io.open(cp_path, "r", encoding="utf-8") as f: current.read(f, u"json") self._copy(cp_path, nb_path) self.log.debug("copying %s -> %s", cp_path, nb_path)
def testRunNotebooks(self): notebook_dir = path.join("tests", "input") for notebook_path in glob(path.join(notebook_dir, "*.ipynb")): notebook_file = path.basename(notebook_path) print(notebook_file) expected_file = path.join("tests", "expected", notebook_file) notebook = "" with open(notebook_path) as notebook_file: notebook = read(notebook_file, "json") runner = NotebookRunner(notebook, working_dir=notebook_dir) runner.run_notebook(True) expected = "" with open(expected_file) as notebook_file: expected = read(notebook_file, "json") self.assert_notebooks_equal(expected, runner.nb)
def export_latex(fname): with open(fname) as f: nb = current.read(f, 'json') lines = '' for cell in nb.worksheets[0].cells: if cell.cell_type == u'code': lines += '\\begin{verbatim}\n' lines += '%s\n' % cell.input lines += '\\end{verbatim}\n' for output in cell.outputs: if output.output_type == u'pyout': if hasattr(output, 'latex'): lines += '%s\n' % output.latex else: lines += '\n' lines += '\\begin{verbatim}\n' lines += '%s\n' % output.text lines += '\\end{verbatim}\n' lines += '\n' if cell.cell_type == u'markdown': paragraphs = wrap_paragraphs(cell.source) for p in paragraphs: lines += p lines += '\n\n' newfname = os.path.splitext(fname)[0] + '.tex' with open(newfname,'w') as f: f.write(lines.encode('utf8'))
def from_filename(self, filename, resources=None, **kw): """ Convert a notebook from a notebook file. Parameters ---------- filename : str Full filename of the notebook file to open and convert. """ # Pull the metadata from the filesystem. if resources is None: resources = ResourcesDict() if not 'metadata' in resources or resources['metadata'] == '': resources['metadata'] = ResourcesDict() basename = os.path.basename(filename) notebook_name = basename[:basename.rfind('.')] resources['metadata']['name'] = notebook_name modified_date = datetime.datetime.fromtimestamp( os.path.getmtime(filename)) resources['metadata'][ 'modified_date'] = modified_date.strftime(text.date_format) with io.open(filename, encoding='utf-8') as f: return self.from_notebook_node(nbformat.read(f, 'json'), resources=resources, **kw)
def test_save(self): resp = self.nb_api.read('a.ipynb', 'foo') nbcontent = json.loads(resp.text)['content'] nb = to_notebook_json(nbcontent) ws = new_worksheet() nb.worksheets = [ws] ws.cells.append(new_heading_cell(u'Created by test ³')) nbmodel = {'name': 'a.ipynb', 'path': 'foo', 'content': nb} resp = self.nb_api.save( 'a.ipynb', path='foo', body=json.dumps(nbmodel)) nbfile = pjoin(self.notebook_dir.name, 'foo', 'a.ipynb') with io.open(nbfile, 'r', encoding='utf-8') as f: newnb = read(f, format='ipynb') self.assertEqual(newnb.worksheets[0].cells[0].source, u'Created by test ³') nbcontent = self.nb_api.read('a.ipynb', 'foo').json()['content'] newnb = to_notebook_json(nbcontent) self.assertEqual(newnb.worksheets[0].cells[0].source, u'Created by test ³') # Save and rename nbmodel = {'name': 'a2.ipynb', 'path': 'foo/bar', 'content': nb} resp = self.nb_api.save( 'a.ipynb', path='foo', body=json.dumps(nbmodel)) saved = resp.json() self.assertEqual(saved['name'], 'a2.ipynb') self.assertEqual(saved['path'], 'foo/bar') assert os.path.isfile( pjoin(self.notebook_dir.name, 'foo', 'bar', 'a2.ipynb')) assert not os.path.isfile( pjoin(self.notebook_dir.name, 'foo', 'a.ipynb')) with assert_http_error(404): self.nb_api.read('a.ipynb', 'foo')
def process_html_notebook(nb_path, src_dir=None, start=None, end=None, **kwargs): with open(nb_path) as f: n = nbformat.read(f, 'ipynb') fullhtml = FullHtmlStaticExporter() (nbc,resources) = fullhtml._preprocess(n, resources={}) if src_dir: nbc = _prepend_srcdir(nbc, src_dir) # made the decision to have subset after image genration # the images are done via an iteration key. So we make sure to generate # the keys for the entire notebook and then grab the cells after wrds. # this is to keep image names from clashing. image 33 will be image 33 regardless # of cells range nbc = _subset_cells(nbc, start, end) output_only = kwargs.get('output_only', False) if output_only: cells = nbc.worksheets[0].cells for cell in cells: del cell['input'] del cell['prompt_number'] # render to html template = fullhtml.environment.get_template(fullhtml.template_file+fullhtml.template_extension) body = template.render(nb=nbc, resources=resources) return body, resources
def run_notebook(self, nb_in, skip_exceptions=False, autosave=None): ''' Run all the cells of a notebook in order and update the outputs in-place. If ``skip_exceptions`` is set, then if exceptions occur in a cell, the subsequent cells are run (by default, the notebook execution stops). ''' self.nb = read(open(nb_in), 'json') for cell in self.iter_code_cells(): cell['outputs'] = [] if 'prompt_number' in cell: del cell['prompt_number'] if autosave is not None: self.save_notebook(autosave) for cell in self.iter_code_cells(): try: self.run_cell(cell, autosave = autosave) except NotebookError: if not skip_exceptions: raise if autosave is not None: self.save_notebook(autosave)
def load_module(self, fullname): """import a notebook as a module""" path = find_notebook(fullname, self.path) disp = "importing IPython notebook from " disp += "<a href='./{}' target='_blank'>{}</a>".format(path, path[:-6]) display_html(disp, raw=True) # print disp # load the notebook object with io.open(path, "r", encoding="utf-8") as f: nb = current.read(f, "json") # create the module and add it to sys.modules # if name in sys.modules: # return sys.modules[name] mod = types.ModuleType(fullname) mod.__file__ = path mod.__loader__ = self sys.modules[fullname] = mod # extra work to ensure that magics that would affect the user_ns # actually affect the notebook module's ns save_user_ns = self.shell.user_ns self.shell.user_ns = mod.__dict__ try: for cell in nb.worksheets[0].cells: if cell.cell_type == "code" and cell.language == "python": # transform the input to executable Python code = self.shell.input_transformer_manager.transform_cell(cell.input) # run the code in themodule exec code in mod.__dict__ finally: self.shell.user_ns = save_user_ns return mod
def notebook(notebook): try: notebook = nbf.read(open('notebooks/%s' % notebook, 'r'), 'ipynb') except IOError: abort(418) html_notebook= convert_nb_html(notebook) return render_template('notebook.html', content=html_notebook)
def generate_data_files(self, ip, notebook, ref_dir, data_dir, regen): """ Generate both testand new reference data (if needed) using NBRunner. """ msg ='' basename = os.path.split(notebook)[1].rsplit('.ipynb')[0] nb = current.read(open(notebook,'r'), 'ipynb') # Reference data not found - regenerate it and exit if regen: if os.path.isdir(ref_dir): shutil.rmtree(ref_dir) os.mkdir(ref_dir) reference_runner = NBRunner(ip, basename, nb, ref_dir, reference=True) reference_runner.run() return '' elif not os.path.isdir(ref_dir): return False # Remove any pre-existing test data. if os.path.isdir(data_dir): shutil.rmtree(data_dir) # Generate the test data os.mkdir(data_dir) NBRunner(ip, basename, nb, data_dir, reference=False).run() return msg
def __init__(self, root, ipynb_name): self.root = root with io.open(ipynb_name, 'rb') as f: self.ipynb = v3.read(f, 'json') self.ipynb_name = ipynb_name self.ipynb_full = '/Users/bussonniermatthias/ipynb2fs/'+self.ipynb_name self.buffers = dict({})
def main(): parser = ArgumentParser(description=DESCRIP, epilog=EPILOG, formatter_class=RawDescriptionHelpFormatter) parser.add_argument('searchpath', type=str, help='directory from which to search') args = parser.parse_args() for dirpath, dirnames, filenames in os.walk(args.searchpath): # Omit directories beginning with dots and underscores dirnames[:] = [d for d in dirnames if not d.startswith('.') and not d.startswith('_')] for fname in filenames: if fname.startswith('.'): continue if not fname.endswith('.ipynb'): continue fullpath = os.path.join(dirpath, fname) with io.open(fullpath, 'r') as f: nb = current.read(f, 'json') for cell in cellgen(nb, 'code'): if hasattr(cell, 'prompt_number'): sys.stderr.write( 'cell prompt number {0} in {1}\n'.format( cell.prompt_number, fname)) sys.exit(1) if not cell.outputs == []: sys.stderr.write('cell output in {0}\n'.format(fname)) sys.exit(1)
def split_into_units(nb_name): """Split notebook into units.""" try: with io.open(nb_name, 'r', encoding='utf-8') as f: nb = current.read(f, 'json') except IOError as e: if e.errno == 2: print('File not found: {0}'.format(nb_name)) return [] else: raise e indexes = [] cells = nb.worksheets[0].cells for (i, cell) in enumerate(cells): if cell.cell_type == 'heading' and cell.level == 1: indexes.append(i) separated_cells = [cells[i:j] for i, j in zip(indexes, indexes[1:]+[None])] worksheets = map(lambda cells: current.new_worksheet(name=cells[0].source, cells=cells), separated_cells) units = map(lambda worksheet: current.new_notebook(name=worksheet.name, worksheets=[worksheet]), worksheets) return units
def get_notebook_params(path): from runipy.notebook_runner import NotebookRunner from IPython.nbformat.current import read # Load the notebook: with open(path) as f: notebook = read(f, 'json') r = NotebookRunner(notebook) def callback(cell_idx): try: # Has the %params_done magic been called yet? has_params = pull(r.shell, "get_ipython().magics_manager.registry['ParameterMagics']" ".has_params") except: # The magic probably hasn't been loaded yet. pass else: if has_params: raise StopIteration() # Yes, we're done! # Run the notebook, checking whether we're done after every cell: try: r.run_notebook(progress_callback=callback) except StopIteration: pass # Pull the param declarations: try: params = pull(r.shell, "get_ipython().magics_manager.registry['ParameterMagics']" ".params") except: params = dict() return params
def process_notebook_file(fname, actions=('clean',), asciidoc_fname=None): print("Performing '{}' on: {}".format("', '".join(actions), fname)) orig_wd = os.getcwd() os.chdir(os.path.dirname(fname)) with io.open(fname, 'rb') as f: nb = current.read(f, 'json') if 'uuid' in actions: check_uuid(nb) if 'render' in actions: run_notebook(nb) elif 'check' in actions: run_notebook(deepcopy(nb)) if 'clean' in actions: remove_outputs(nb) if 'merge' in actions: doc = AsciidDoc(asciidoc_fname) doc.merge_code_from(nb) doc.save() elif 'convert' in actions: doc = AsciidDoc() doc.convert_from(nb) doc.save(asciidoc_fname) os.chdir(orig_wd) with io.open(fname, 'wb') as f: current.write(nb, f, 'json')
def load_module(self, fullname): """import a notebook as a module""" path = find_notebook(fullname, self.path) print ("importing Jupyter notebook from %s" % path) # load the notebook object with io.open(path, 'r', encoding='utf-8') as f: nb = current.read(f, 'json') # create the module and add it to sys.modules # if name in sys.modules: # return sys.modules[name] mod = types.ModuleType(fullname) mod.__file__ = path mod.__loader__ = self mod.__dict__['get_ipython'] = get_ipython sys.modules[fullname] = mod # extra work to ensure that magics that would affect the user_ns # actually affect the notebook module's ns save_user_ns = self.shell.user_ns self.shell.user_ns = mod.__dict__ try: for cell in nb.worksheets[0].cells: if cell.cell_type == 'code' and cell.language == 'python': # transform the input to executable Python code = self.shell.input_transformer_manager.transform_cell(cell.input) # run the code in themodule exec(code, mod.__dict__) finally: self.shell.user_ns = save_user_ns return mod
def run(self): from IPython.nbformat.current import read, write with open('template.rst', 'r') as f: template = f.read() toc = "" from urllib import quote for notebook in (glob.glob('lectures/*.ipynb')): with open(notebook, 'r') as f: nb = read(f, 'json') for ws in nb.worksheets: for cell in ws.cells: if cell.cell_type == 'heading': if cell['level'] > 1: continue toc += (" " * (cell['level'] - 1) + "* `{0} <_static/{1}.html#{2}>`__\n".format(cell['source'].replace('`', ''), quote(os.path.basename(notebook)).replace('.ipynb', ''), strip_punctuation(cell['source']).replace(' ', '-'))) with open('docs/index.rst', 'w') as f: f.write(template.format(lectures_toc=toc))
def evaluate_notebook(nb_path, dest_path=None, skip_exceptions=False): # Create evaluated version and save it to the dest path. # Always use --pylab so figures appear inline # perhaps this is questionable? notebook = read(open(nb_path), 'json') cwd = os.getcwd() filedir, filename = os.path.split(nb_path) os.chdir(filedir) nb_runner = NotebookRunner(notebook, pylab=False) try: nb_runner.run_notebook(skip_exceptions=skip_exceptions) except NotebookError as e: print('') print(e) # Return the traceback, filtering out ANSI color codes. # http://stackoverflow.com/questions/13506033/filtering-out-ansi-escape-sequences return 'Notebook conversion failed with the following traceback: \n%s' % \ re.sub(r'\\033[\[\]]([0-9]{1,2}([;@][0-9]{0,2})*)*[mKP]?', '', str(e)) os.chdir(cwd) write(nb_runner.nb, open(dest_path, 'w'), 'json') ret = nb_to_html(dest_path) if dest_path.endswith('temp_evaluated.ipynb'): os.remove(dest_path) return ret
def __init__(self, nb_in=None, pylab=False, mpl_inline=False, nb=None): self.km = KernelManager() if pylab: self.km.start_kernel(extra_arguments=['--pylab=inline']) elif mpl_inline: self.km.start_kernel(extra_arguments=['--matplotlib=inline']) else: self.km.start_kernel() if platform.system() == 'Darwin': # There is sometimes a race condition where the first # execute command hits the kernel before it's ready. # It appears to happen only on Darwin (Mac OS) and an # easy (but clumsy) way to mitigate it is to sleep # for a second. sleep(1) self.kc = self.km.client() self.kc.start_channels() self.shell = self.kc.shell_channel self.iopub = self.kc.iopub_channel logging.info('Reading notebook %s', nb_in) self.nb = nb if not self.nb: self.nb = read(open(nb_in), 'json')
def test_nbs(): path = os.path.join(pyfolio_root(), 'examples', '*.ipynb') for ipynb in glob.glob(path): with open(ipynb) as f: nb = read(f, 'json') nb_runner = NotebookRunner(nb) nb_runner.run_notebook(skip_exceptions=False)
def evaluate_notebook(nb_path, dest_path=None, skip_exceptions=True): # Create evaluated version and save it to the dest path. # Always use --pylab so figures appear inline # perhaps this is questionable? curr_dir = os.getcwd() os.chdir(os.path.dirname(nb_path)) notebook = read(open(nb_path), "json") nb_runner = NotebookRunner(notebook, pylab=True) try: nb_runner.run_notebook(skip_exceptions=skip_exceptions) except NotebookError as e: print "" print e # Return the traceback, filtering out ANSI color codes. # http://stackoverflow.com/questions/13506033/filtering-out-ansi-escape-sequences return "Notebook conversion failed with the following traceback: \n%s" % re.sub( r"\\033[\[\]]([0-9]{1,2}([;@][0-9]{0,2})*)*[mKP]?", "", str(e) ) if dest_path is None: dest_path = "temp_evaluated.ipynb" write(nb_runner.nb, open(dest_path, "w"), "json") ret = nb_to_html(dest_path) if dest_path is "temp_evaluated.ipynb": os.remove(dest_path) os.chdir(curr_dir) return ret
def test_export_nbnode(self): """ Can a notebook be exported by a notebook node handle? """ with open(self._get_notebook(), 'r') as f: notebook = nbformat.read(f, 'json') (output, resources) = export_by_name('python', notebook) assert len(output) > 0
def execute_notebook(nbfile): with io.open(nbfile) as f: nb = current.read(f,'json') ip = get_ipython() for cell in nb.worksheets[0].cells: if cell.cell_type != 'code': continue ip.run_cell(cell.input)
def restore_checkpoint(self, checkpoint_id, name, path=''): """restore a file to a checkpointed state""" path = path.strip('/') self.log.info("restoring %s from checkpoint %s", name, checkpoint_id) nb_path = self._get_os_path(name, path) cp_path = self.get_checkpoint_path(checkpoint_id, name, path) if not os.path.isfile(cp_path): self.log.debug("checkpoint file does not exist: %s", cp_path) raise web.HTTPError(404, u'checkpoint does not exist: %s-%s' % (name, checkpoint_id) ) # ensure notebook is readable (never restore from an unreadable notebook) if cp_path.endswith('.ipynb'): with io.open(cp_path, 'r', encoding='utf-8') as f: current.read(f, u'json') self._copy(cp_path, nb_path) self.log.debug("copying %s -> %s", cp_path, nb_path)
def notebook_content(self): filepath = os.path.join(self.bundle_path, self.name) with io.open(filepath, 'r', encoding='utf-8') as f: try: nb = current.read(f, u'json') except Exception as e: nb = None return nb
def parse (self): try: with open(self.filename) as f: self.notebook = current_notebook.read(f,'ipynb') except: print("Could not parse {}".format(self.filename)) import traceback traceback.print_exc()
def main(): parser = ArgumentParser(description=DESCRIP, epilog=EPILOG, formatter_class=RawDescriptionHelpFormatter) parser.add_argument('filename', type=str, nargs='+', help='notebook filenames') args = parser.parse_args() for fname in args.filename: with io.open(fname, 'r') as f: nb = current.read(f, 'json') for cell in cellgen(nb, 'code'): if hasattr(cell, 'prompt_number'): del cell['prompt_number'] cell.outputs = [] with io.open(fname, 'w') as f: current.write(nb, f, 'ipynb')
def run(self): from IPython.nbformat.current import read, write for notebook in glob.glob('notebooks/*.ipynb'): with open(notebook, 'r') as f: nb = read(f, 'json') for ws in nb.worksheets: for cell in ws.cells: if cell.cell_type == 'code': cell.outputs = [] if 'prompt_number' in cell: cell.pop('prompt_number') with open(notebook, 'w') as f: write(nb, f, 'json')
def remove_outputs(fname): """ remove the outputs from a notebook "fname" and create a new notebook """ with io.open(fname, 'r') as f: nb = read(f, 'json') for ws in nb.worksheets: for cell in ws.cells: if cell.cell_type == 'code': cell.outputs = [] base, ext = os.path.splitext(fname) new_ipynb = "%s_removed%s" % (base, ext) with io.open(new_ipynb, 'w', encoding='utf8') as f: write(nb, f, 'json') print "wrote %s" % new_ipynb return "Done"
def extract_exercises(inputs, odir='exercises', inplace=False): """ Strip output of notebooks. Parameters ---------- inputs : list of string Path to the notebooks to be processed. inplace : bool If this is `True`, outputs in the input files will be deleted. Default is `False`. """ for inpath in inputs: with file(inpath) as fp: nb = nbformat.read(fp, 'ipynb') clear_everything_but_exercises(nb) if odir: nbformat.write(nb, file(os.path.join(odir, os.path.basename(inpath)), 'w'), 'ipynb')
def merge_notebooks(filenames): merged = None added_appendix = False for fname in filenames: with io.open(fname, 'r', encoding='utf-8') as f: nb = current.read(f, u'json') remove_formatting(nb) if not added_appendix and fname[0:8] == 'Appendix': remove_links_add_appendix(nb) added_appendix = True else: remove_links(nb) if merged is None: merged = nb else: merged.worksheets[0].cells.extend(nb.worksheets[0].cells) merged.metadata.name += "_merged" print(current.writes(merged, u'json'))
def notebook(self, s): """Export and convert IPython notebooks. This function can export the current IPython history to a notebook file or can convert an existing notebook file into a different format. For example, to export the history to "foo.ipynb" do "%notebook -e foo.ipynb". To export the history to "foo.py" do "%notebook -e foo.py". To convert "foo.ipynb" to "foo.json" do "%notebook -f json foo.ipynb". Possible formats include (json/ipynb, py). """ args = magic_arguments.parse_argstring(self.notebook, s) from IPython.nbformat import current args.filename = unquote_filename(args.filename) if args.export: fname, name, format = current.parse_filename(args.filename) cells = [] hist = list(self.shell.history_manager.get_range()) for session, prompt_number, input in hist[:-1]: cells.append( current.new_code_cell(prompt_number=prompt_number, input=input)) worksheet = current.new_worksheet(cells=cells) nb = current.new_notebook(name=name, worksheets=[worksheet]) with io.open(fname, 'w', encoding='utf-8') as f: current.write(nb, f, format) elif args.format is not None: old_fname, old_name, old_format = current.parse_filename( args.filename) new_format = args.format if new_format == u'xml': raise ValueError('Notebooks cannot be written as xml.') elif new_format == u'ipynb' or new_format == u'json': new_fname = old_name + u'.ipynb' new_format = u'json' elif new_format == u'py': new_fname = old_name + u'.py' else: raise ValueError('Invalid notebook format: %s' % new_format) with io.open(old_fname, 'r', encoding='utf-8') as f: nb = current.read(f, old_format) with io.open(new_fname, 'w', encoding='utf-8') as f: current.write(nb, f, new_format)
def nb_to_html(nb_path): """convert notebook to html""" c = Config({'ExtractOutputPreprocessor': {'enabled': True}}) exporter = html.HTMLExporter(template_file='full', config=c) notebook = nbformat.read(open(nb_path), 'json') output, resources = exporter.from_notebook_node(notebook) header = output.split('<head>', 1)[1].split('</head>', 1)[0] body = output.split('<body>', 1)[1].split('</body>', 1)[0] # http://imgur.com/eR9bMRH header = header.replace('<style', '<style scoped="scoped"') header = header.replace( 'body {\n overflow: visible;\n padding: 8px;\n}\n', '') header = header.replace("code,pre{", "code{") # Filter out styles that conflict with the sphinx theme. filter_strings = [ 'navbar', 'body{', 'alert{', 'uneditable-input{', 'collapse{', ] filter_strings.extend(['h%s{' % (i + 1) for i in range(6)]) line_begin = ['pre{', 'p{margin'] filterfunc = lambda x: not any([s in x for s in filter_strings]) header_lines = filter(filterfunc, header.split('\n')) filterfunc = lambda x: not any([x.startswith(s) for s in line_begin]) header_lines = filter(filterfunc, header_lines) header = '\n'.join(header_lines) # concatenate raw html lines lines = ['<div class="ipynotebook">'] lines.append(header) lines.append(body) lines.append('</div>') return '\n'.join(lines), resources
def main(): parser = ArgumentParser(description=DESCRIP, epilog=EPILOG, formatter_class=RawDescriptionHelpFormatter) parser.add_argument('in_fname', type=str, help='input notebook filename') parser.add_argument('out_fname', type=str, help='output notebook filename') args = parser.parse_args() out_name = splitext(basename(args.out_fname))[0] with io.open(args.in_fname, 'r') as f: nb = current.read(f, 'json') nb.name = out_name for cell in cellgen(nb, 'code'): if hasattr(cell, 'prompt_number'): del cell['prompt_number'] cell.outputs = [] if hasattr(cell, 'input'): cell['input'] = strip_after_comments(cell['input']) with io.open(args.out_fname, 'w') as f: current.write(nb, f, 'ipynb')
def test_save(self): resp = self.api.read('a.ipynb', 'foo') nbcontent = json.loads(resp.text)['content'] nb = to_notebook_json(nbcontent) ws = new_worksheet() nb.worksheets = [ws] ws.cells.append(new_heading_cell(u'Created by test ³')) nbmodel = { 'name': 'a.ipynb', 'path': 'foo', 'content': nb, 'type': 'notebook' } resp = self.api.save('a.ipynb', path='foo', body=json.dumps(nbmodel)) nbfile = pjoin(self.notebook_dir.name, 'foo', 'a.ipynb') with io.open(nbfile, 'r', encoding='utf-8') as f: newnb = read(f, format='ipynb') self.assertEqual(newnb.worksheets[0].cells[0].source, u'Created by test ³') nbcontent = self.api.read('a.ipynb', 'foo').json()['content'] newnb = to_notebook_json(nbcontent) self.assertEqual(newnb.worksheets[0].cells[0].source, u'Created by test ³') # Save and rename nbmodel = { 'name': 'a2.ipynb', 'path': 'foo/bar', 'content': nb, 'type': 'notebook' } resp = self.api.save('a.ipynb', path='foo', body=json.dumps(nbmodel)) saved = resp.json() self.assertEqual(saved['name'], 'a2.ipynb') self.assertEqual(saved['path'], 'foo/bar') assert os.path.isfile( pjoin(self.notebook_dir.name, 'foo', 'bar', 'a2.ipynb')) assert not os.path.isfile( pjoin(self.notebook_dir.name, 'foo', 'a.ipynb')) with assert_http_error(404): self.api.read('a.ipynb', 'foo')
def run(self): # Now convert the lecture notes, problem sets, and practice problems to # HTML notebooks. from runipy.notebook_runner import NotebookRunner from IPython.nbformat.current import read, write start_dir = os.path.abspath('.') for notebook in glob.glob('notebooks/*.ipynb'): print("Running {0}...".format(notebook)) os.chdir(os.path.dirname(notebook)) with open(os.path.basename(notebook)) as f: r = NotebookRunner(read(f, 'json'), pylab=False) r.run_notebook(skip_exceptions=True) with open(os.path.basename(notebook), 'w') as f: write(r.nb, f, 'json') os.chdir(start_dir)
def _notebook_model(self, name, path='', content=True): """Build a notebook model if content is requested, the notebook content will be populated as a JSON structure (not double-serialized) """ model = self._base_model(name, path) model['type'] = 'notebook' if content: os_path = self._get_os_path(name, path) with io.open(os_path, 'r', encoding='utf-8') as f: try: nb = current.read(f, u'json') except Exception as e: raise web.HTTPError(400, u"Unreadable Notebook: %s %s" % (os_path, e)) self.mark_trusted_cells(nb, name, path) model['content'] = nb model['format'] = 'json' return model
def execute_notebook(path): ''' * Função execute_notebook(path) executa todas as celulas de um notebook, que contêm declarações de funções (def) em seu conteúdo Entradas: -path: String que contêm o diretório do arquivo .ipynb Saída: - Sem saídas. ''' with io.open(path) as f: nb = current.read(f, 'json') ip = get_ipython() for cell in nb.worksheets[0].cells: if cell.cell_type != 'code': continue if find_word(cell.input, 'def'): ip.run_cell(cell.input) else: continue
def main(): parser = ArgumentParser(description=DESCRIP, epilog=EPILOG, formatter_class=RawDescriptionHelpFormatter) parser.add_argument('filename', type=str, nargs='+', help='notebook filenames') args = parser.parse_args() for fname in args.filename: with io.open(fname, 'r') as f: nb = current.read(f, 'json') for cell in cellgen(nb, 'code'): if hasattr(cell, 'prompt_number'): sys.stderr.write('cell prompt number {0} in {1}\n'.format( cell.prompt_number, fname)) sys.exit(1) if not cell.outputs == []: sys.stderr.write('cell output in {0}\n'.format(fname)) sys.exit(1)
def get_notebook(self, name, path='', content=True): """ Takes a path and name for a notebook and returns its model Parameters ---------- name : str the name of the notebook path : str the URL path that describes the relative path for the notebook Returns ------- model : dict the notebook model. If contents=True, returns the 'contents' dict in the model as well. """ path = path.strip('/') if not self.notebook_exists(name=name, path=path): raise web.HTTPError(404, u'Notebook does not exist: %s' % name) os_path = self._get_os_path(name, path) info = os.stat(os_path) last_modified = tz.utcfromtimestamp(info.st_mtime) created = tz.utcfromtimestamp(info.st_ctime) # Create the notebook model. model = {} model['name'] = name model['path'] = path model['last_modified'] = last_modified model['created'] = created model['type'] = 'notebook' if content: with io.open(os_path, 'r', encoding='utf-8') as f: try: nb = current.read(f, u'json') except Exception as e: raise web.HTTPError( 400, u"Unreadable Notebook: %s %s" % (os_path, e)) self.mark_trusted_cells(nb, name, path) model['content'] = nb return model
def notebooks(env, loader, args): nb_path = env.resolve('notebooks') res_path = env.resolve('results') for filename in os.listdir(nb_path): if filename.endswith('.ipynb'): nb_name = filename.split('.')[0] print('running', nb_name) src_path = os.path.join(nb_path, filename) html_path = os.path.join(res_path, nb_name + '.html') with open(src_path, 'r') as f: notebook = read(f, 'json') r = NotebookRunner(notebook, working_dir=nb_path) r.run_notebook() dest_path = tempfile.mkstemp() t = tempfile.NamedTemporaryFile() with open(t.name, 'w') as g: write(r.nb, g) exporter = nbconvert.HTMLExporter() body, resources = exporter.from_filename(t.name) with open(html_path, 'wb') as g: g.write(body.encode('utf-8'))
def evaluate_notebook(nb_path, dest_path=None, skip_exceptions=False): # Create evaluated version and save it to the dest path. # Always use --pylab so figures appear inline # perhaps this is questionable? notebook = read(open(nb_path), 'json') nb_runner = NotebookRunner(notebook, pylab=True, mpl_inline=True) try: nb_runner.run_notebook(skip_exceptions=skip_exceptions) except NotebookError as e: print('\n', '-' * 80) print(e) print('-' * 80) raise if dest_path is None: dest_path = 'temp_evaluated.ipynb' write(nb_runner.nb, open(dest_path, 'w'), 'json') ret = nb_to_html(dest_path) if dest_path is 'temp_evaluated.ipynb': os.remove(dest_path) return ret
def import_notebook(nbname, loader=None): fullpath = "{}/{}.ipynb".format(os.getcwd(), nbname) if not os.path.exists(fullpath): raise Exception( "Failed to find ipython notebook: {}.ipynb".format(nbname)) if nbname in sys.modules: mod = sys.modules[nbname] return mod moduleName = nbname nbFilename = nbname + ".ipynb" print "Importing ipynb file : {}".format(nbFilename) with io.open(nbFilename) as f: nb = current.read(f, 'json') newModule = imp.new_module(moduleName) try: for cell in nb.worksheets[0].cells: if cell.cell_type != 'code': continue exec cell.input in newModule.__dict__ except: print "Error in importing ipython notebook." print_exc() raise # Set a few properties required by PEP 302 newModule.__file__ = nbname newModule.__name__ = nbname newModule.__path__ = [nbname] newModule.__loader__ = loader newModule.__package__ = '' # no package sys.modules[moduleName] = newModule return newModule
def evaluate_notebook(nb_path, dest_path=None, skip_exceptions=False): # Create evaluated version and save it to the dest path. # Always use --pylab so figures appear inline # perhaps this is questionable? notebook = read(open(nb_path), 'json') nb_runner = NotebookRunner(notebook, pylab=False) try: nb_runner.run_notebook(skip_exceptions=skip_exceptions) except NotebookError as e: print '' print e # Return the traceback, filtering out ANSI color codes. # http://stackoverflow.com/questions/13506033/filtering-out-ansi-escape-sequences return 'Notebook conversion failed with the following traceback: \n%s' % \ re.sub(r'\\033[\[\]]([0-9]{1,2}([;@][0-9]{0,2})*)*[mKP]?', '', str(e)) if dest_path is None: dest_path = 'temp_evaluated.ipynb' write(nb_runner.nb, open(dest_path, 'w'), 'json') ret = nb_to_html(dest_path) if dest_path is 'temp_evaluated.ipynb': os.remove(dest_path) return ret
def stripoutput(inputs, inplace=False): """ Strip output of notebooks. Parameters ---------- inputs : list of string Path to the notebooks to be processed. inplace : bool If this is `True`, outputs in the input files will be deleted. Default is `False`. """ for inpath in inputs: with file(inpath) as fp: nb = nbformat.read(fp, 'ipynb') clear_outputs(nb) if inplace: with file(inpath, 'w') as fp: nbformat.write(nb, fp, 'ipynb') else: nbformat.write(nb, sys.stdout, 'ipynb')
def process_notebook_file(fname, action='clean', output_fname=None): print("Performing '{}' on: {}".format(action, fname)) orig_wd = os.getcwd() with io.open(fname, 'rb') as f: nb = current.read(f, 'json') if action == 'check': os.chdir(os.path.dirname(fname)) run_notebook(nb) remove_outputs(nb) elif action == 'render': os.chdir(os.path.dirname(fname)) run_notebook(nb) else: # Clean by default remove_outputs(nb) os.chdir(orig_wd) if output_fname is None: output_fname = fname with io.open(output_fname, 'wb') as f: nb = current.write(nb, f, 'json')
def walk_through_tutorials(only_published=True, selected_nb_re=None): """ Generator for walking through the tutorials directory structure. This returns tuples of (full_tutorial_path, tutorial_name) for each tutorial. If published is set to True, this will only return the published tutorials. """ nbre = re.compile(selected_nb_re) if selected_nb_re else None current_directory = os.getcwd() tutorials_base = os.path.join(current_directory, 'tutorials') if not os.path.exists(tutorials_base): err = ("Can't find 'tutorials' path! You must run this script from the" " top-level astropy-tutorials directory.") raise IOError(err) # walk through each directory in tutorials/ to find all .ipynb file for tutorial_name in os.listdir(tutorials_base): tutorial_path = os.path.join(tutorials_base, tutorial_name) if not os.path.isdir(tutorial_path): # skip files / things that are not directories continue for filename in os.listdir(tutorial_path): base, ext = os.path.splitext(filename) if ext.lower() == ".ipynb" and "checkpoint" not in base: full_filename = os.path.join(tutorial_path, filename) notebook = read(open(full_filename), 'json') is_published = notebook['metadata']['astropy-tutorials'].get( 'published', False) if not is_published and only_published: continue if nbre and nbre.match(base) is None: continue yield full_filename, notebook
def __init__(self, nb_in, pylab=False): self.km = KernelManager() if pylab: self.km.start_kernel(extra_arguments=['--pylab=inline']) else: self.km.start_kernel() if platform.system() == 'Darwin': # There is sometimes a race condition where the first # execute command hits the kernel before it's ready. # It appears to happen only on Darwin (Mac OS) and an # easy (but clumsy) way to mitigate it is to sleep # for a second. sleep(1) self.kc = self.km.client() self.kc.start_channels() self.shell = self.kc.shell_channel self.iopub = self.kc.iopub_channel logging.info('Reading notebook %s', nb_in) self.nb = read(open(nb_in), 'json')
def load_module(self, fullname): """import a notebook as a module""" path = find_notebook(fullname, self.path) disp = "importing IPython notebook from " disp += "<a href='./{}' target='_blank'>{}</a>".format(path, path[:-6]) display_html(disp, raw=True) #print disp # load the notebook object with io.open(path, 'r', encoding='utf-8') as f: nb = current.read(f, 'json') # create the module and add it to sys.modules # if name in sys.modules: # return sys.modules[name] mod = types.ModuleType(fullname) mod.__file__ = path mod.__loader__ = self sys.modules[fullname] = mod # extra work to ensure that magics that would affect the user_ns # actually affect the notebook module's ns save_user_ns = self.shell.user_ns self.shell.user_ns = mod.__dict__ try: for cell in nb.worksheets[0].cells: if cell.cell_type == 'code' and cell.language == 'python': # transform the input to executable Python code = self.shell.input_transformer_manager.transform_cell( cell.input) # run the code in themodule exec code in mod.__dict__ finally: self.shell.user_ns = save_user_ns return mod
def test_notebook(filename): nb = read(open(filename), 'json') expected = deepcopy(nb['worksheets'][0]['cells']) # TODO: figure out how to set the log-level for IPython here r = NotebookRunner(nb) r.run_notebook() actual = r.nb['worksheets'][0]['cells'] failed = [] for exp, act in zip(expected, actual): if exp['cell_type'] == 'code': #print("comparing outputs for ", exp['input']) for eo, ao in zip(exp['outputs'], act['outputs']): #print("\t", eo['text'], ao['text'], eo['text']==ao['text']) eotext = [ '\n'.join(l for l in eo['text'].split('\n') if _filter_exceptions(l)) ] aotext = [ '\n'.join(l for l in ao['text'].split('\n') if _filter_exceptions(l)) ] if eotext != aotext: #print("\tFAILED") failed.append({ 'prompt_number': exp['prompt_number'], 'input': exp['input'], 'expected_output': eotext, 'actual_output': aotext }) return len(failed) == 0, failed, r.nb
def main(): parser = ArgumentParser(description=DESCRIP, epilog=EPILOG, formatter_class=RawDescriptionHelpFormatter) parser.add_argument('searchpath', type=str, help='directory from which to search') args = parser.parse_args() for dirpath, dirnames, filenames in os.walk(args.searchpath): dirnames[:] = [d for d in dirnames if not d.startswith('.')] for fname in filenames: if fname.startswith('.'): continue if not fname.endswith('.ipynb'): continue fullpath = os.path.join(dirpath, fname) with io.open(fullpath, 'r') as f: nb = current.read(f, 'json') for cell in cellgen(nb, 'code'): if hasattr(cell, 'prompt_number'): del cell['prompt_number'] cell.outputs = [] with io.open(fullpath, 'w') as f: current.write(nb, f, 'ipynb')
def from_filename(self, filename, resources=None, **kw): """ Convert a notebook from a notebook file. Parameters ---------- filename : str Full filename of the notebook file to open and convert. """ #Pull the metadata from the filesystem. if resources is None: resources = ResourcesDict() if not 'metadata' in resources or resources['metadata'] == '': resources['metadata'] = ResourcesDict() basename = os.path.basename(filename) notebook_name = basename[:basename.rfind('.')] resources['metadata']['name'] = notebook_name modified_date = datetime.datetime.fromtimestamp(os.path.getmtime(filename)) resources['metadata']['modified_date'] = modified_date.strftime(text.date_format) with io.open(filename) as f: return self.from_notebook_node(nbformat.read(f, 'json'), resources=resources,**kw)
def read_nb(fp): return current.read(fp, 'json')
""" Usage: python remove_output.py notebook.ipynb [ > without_output.ipynb ] Modified from remove_output by Minrk """ import sys import io import os from IPython.nbformat.current import read, write def remove_outputs(nb): """remove the outputs from a notebook""" for ws in nb.worksheets: for cell in ws.cells: if cell.cell_type == 'code': cell.outputs = [] if __name__ == '__main__': fname = sys.argv[1] with io.open(fname, 'r') as f: nb = read(f, 'json') remove_outputs(nb) base, ext = os.path.splitext(fname) new_ipynb = "%s%s" % (base, ext) # "%s_removed%s" % (base, ext) with io.open(new_ipynb, 'w', encoding='utf8') as f: write(nb, f, 'json') print("wrote %s" % new_ipynb)
def parse(root): """returns list of the NB in root """ dirs = sorted([dn for dn in os.listdir(root) if '00' < dn < '99']) r = [] for sub_dir in dirs: f = sorted([ os.path.join(root, sub_dir, fn) for fn in os.listdir(os.path.join(root, sub_dir)) if ('00' < fn < '99' or fn.startswith('content')) and fn.endswith('.ipynb') ]) r.extend(f) return r nb_filenames = ["Index.ipynb", "Index-labs1-6.ipynb"] nb_filenames.extend(parse('./LABS')) nb_filenames.extend(parse('.')) for nb_filename in nb_filenames: print('*' * 80) print(nb_filename) print('*' * 80) notebook = read(open(nb_filename), 'json') r = NotebookRunner(notebook) r.run_notebook() r.shutdown_kernel()
def strip_output(nb): for ws in nb.worksheets: for cell in ws.cells: if hasattr(cell, "outputs"): cell.outputs = [] if hasattr(cell, "prompt_number"): del cell["prompt_number"] if __name__ == "__main__": from sys import stdin, stdout from IPython.nbformat.current import read, write nb = read(stdin, "ipynb") strip_output(nb) write(nb, stdout, "ipynb") stdout.write("\n")