def _load_notebook(self, uri): """ Loads a local or remote notebook. Raises RuntimeError if no installed kernel can handle the language specified in the notebook. Otherwise, returns the notebook object. """ parts = urlparse(self.seed_uri) if parts.netloc == "" or parts.netloc == "file": # Local file with open(parts.path) as nb_fh: notebook = nbformat.read(nb_fh, 4) else: # Remote file import requests resp = requests.get(uri) resp.raise_for_status() notebook = nbformat.reads(resp.text, 4) # Error if no kernel spec can handle the language requested kernel_name = notebook["metadata"]["kernelspec"]["name"] self.kernel_spec_manager.get_kernel_spec(kernel_name) return notebook
def check_one_notebook(filepath): folder, filename = os.path.split(filepath) os.chdir(folder) with open(filename) as f: nb = nbformat.reads(f.read(), nbformat.NO_CONVERT) run_notebook(nb) os.chdir('../../../')
def downloadIntroNotebooks(self): #download the Intro.txt file that contains all the notebooks to download response = requests.get("https://github.com/ibm-watson-data-lab/pixiedust/raw/master/notebook/Intro.txt") if not response.ok: raise Exception("Unable to read the list of Intro Notebooks") notebookNames = response.content.decode().split("\n") introNotebooksUrls = [ "https://github.com/ibm-watson-data-lab/pixiedust/raw/master/notebook/" + n for n in notebookNames if n != "" ] for url in introNotebooksUrls: print("...{0}".format(url)) try: path = self.downloadFileToDir(url, targetDir=self.pixiedust_notebooks_dir) #update kernel name and display_name f = open(path, 'r') contents = f.read() f.close() nb=nbformat.reads(contents, as_version=4) nb.metadata.kernelspec.name=self.kernelInternalName nb.metadata.kernelspec.display_name = self.kernelName f = open(path, 'w') f.write(json.dumps(nb)) f.close() print("\033[F\033[F") print("...{0} : {1}".format(url, self.hilite("done"))) except Exception as e: print("\033[F\033[F") print("...{0} : {1}".format(url, self.hilite("Error {}".format(e))))
def get_hyperopt_model_string(model, data, functions, notebook_name, verbose, stack): model_string = inspect.getsource(model) model_string = remove_imports(model_string) if notebook_name: notebook_path = os.getcwd() + "/{}.ipynb".format(notebook_name) with open(notebook_path, 'r') as f: notebook = nbformat.reads(f.read(), nbformat.NO_CONVERT) exporter = PythonExporter() source, _ = exporter.from_notebook_node(notebook) else: calling_script_file = os.path.abspath(inspect.stack()[stack][1]) with open(calling_script_file, 'r') as f: source = f.read() cleaned_source = remove_all_comments(source) imports = extract_imports(cleaned_source, verbose) parts = hyperparameter_names(model_string) aug_parts = augmented_names(parts) hyperopt_params = get_hyperparameters(model_string) space = get_hyperopt_space(parts, hyperopt_params, verbose) functions_string = retrieve_function_string(functions, verbose) data_string = retrieve_data_string(data, verbose) model = hyperopt_keras_model(model_string, parts, aug_parts, verbose) temp_str = temp_string(imports, model, data_string, functions_string, space) return temp_str
def read_notebook(self, arg): # Currently assuming arg is a filename relative to # where the server was started from, later we may # want to accept urls or full notebooks as well. if not isinstance(arg, string_types): raise web.HTTPError(400, 'Expecting a filename or a URL.') try: # Check that file exists if arg == EXPLICIT_MISSING_FILE: path = arg else: path = os.path.join(self.curdir, arg) if not os.path.exists(path): if '://' not in arg: raise ValueError('Supplied argument cannot be read: %r' % arg) # Assume file is URI r = requests.get(arg) # Let nbformat do the reading and validation if os.path.exists(path): nb = nbformat.read(path, as_version=4) elif path == EXPLICIT_MISSING_FILE: nb = nbformat.v4.new_notebook() else: nb = nbformat.reads(r.text, as_version=4) except Exception as e: self.log.exception(e) raise web.HTTPError(422, 'Invalid notebook: %s' % arg) return nb
def nb_to_html(root, template='basic', version=4, timeout=600, kernel='python3'): ''' This functions executes a Jupyter notebook and creates the related HTML file. Args: root (str): name of the file without the .ipynb extension template (str): name of the template (to be in the current folder as template.tpl) version (int): version of the notebook timeout (float): maximum time spent per cell kernel (str) Returns: None The function executes root.ipynb into root_exe.ipynb and creates the file root.html. ''' with open(root + '.ipynb') as f: nb = nbformat.read(f, as_version=version) ep = ExecutePreprocessor(timeout=timeout, kernel_name=kernel) ep.preprocess(nb, {'metadata': {'path': '.'}}) with open(root + '_exe.ipynb', 'wt') as f: nbformat.write(nb, f) html_exporter = HTMLExporter() html_exporter.template_file = template with open(root + '_exe.ipynb', mode='r') as f: notebook = nbformat.reads(''.join(f.readlines()), as_version=version) (body, _) = html_exporter.from_notebook_node(notebook) codecs.open(root + '.html', 'w', encoding='utf-8').write(body)
def write_notebook(self, include_html=True): suffix = "_responses_with_names" if self.include_usernames else "_responses" nb_name = self.nb_name_stem + suffix output_file = os.path.join(PROCESSED_NOTEBOOK_DIR, nb_name + '.ipynb') html_output = os.path.join(PROCESSED_NOTEBOOK_DIR, nb_name + '.html') remove_duplicate_answers = not self.include_usernames filtered_cells = [] for prompt in self.question_prompts: filtered_cells += prompt.cells answers = prompt.answers_without_duplicates if remove_duplicate_answers else prompt.answers for gh_username, response_cells in answers.items(): if self.include_usernames: filtered_cells.append( NotebookUtils.markdown_heading_cell(self.gh_username_to_fullname(gh_username), 4)) filtered_cells.extend(response_cells) answer_book = deepcopy(self.template) answer_book['cells'] = filtered_cells nb = nbformat.from_dict(answer_book) print "Writing", output_file with io.open(output_file, 'wt') as fp: nbformat.write(nb, fp, version=4) if include_html: # TODO why is the following necessary? nb = nbformat.reads(nbformat.writes(nb, version=4), as_version=4) html_content, _ = nbconvert.export_html(nb) print "Writing", html_output with io.open(html_output, 'w') as fp: fp.write(html_content)
def run_ipython_notebook(notebook_str): """ References: https://github.com/paulgb/runipy >>> from utool.util_ipynb import * # NOQA """ from runipy.notebook_runner import NotebookRunner import nbformat import logging log_format = '%(asctime)s %(levelname)s: %(message)s' log_datefmt = '%m/%d/%Y %I:%M:%S %p' logging.basicConfig( level=logging.INFO, format=log_format, datefmt=log_datefmt ) #fpath = 'tmp.ipynb' #notebook_str = ut.readfrom(fpath) #nb3 = IPython.nbformat.reads(notebook_str, 3) #cell = nb4.cells[1] #self = runner #runner = NotebookRunner(nb3, mpl_inline=True) print('Executing IPython notebook') nb4 = nbformat.reads(notebook_str, 4) runner = NotebookRunner(nb4) runner.run_notebook(skip_exceptions=False) run_nb = runner.nb return run_nb
def get_notebook_argument(self, argname): # Assuming a request on the form "{'argname':arg}" body = json.loads(escape.to_unicode(self.request.body)) arg = body[argname] # Currently assuming arg is a filename relative to # where the server was started from, later we may # want to accept urls or full notebooks as well. if not isinstance(arg, string_types): raise web.HTTPError(400, "Expecting a filename.") # Check that file exists path = os.path.join(self.params["cwd"], arg) if not os.path.exists(path): # Assume file is URI r = requests.get(arg) # Let nbformat do the reading and validation try: if os.path.exists(path): nb = nbformat.read(path, as_version=4) else: nb = nbformat.reads(r.text, as_version=4) except: raise web.HTTPError(400, "Invalid notebook: %s" % truncate_filename(arg)) return nb
def bundle_notebook(vid, fileid): """Return a file from the bundle""" from ambry.orm.file import File import nbformat from traitlets.config import Config from nbconvert import HTMLExporter b = aac.library.bundle(vid) nbfile = b.build_source_files.file_by_id(fileid) notebook = nbformat.reads(nbfile.unpacked_contents, as_version=4) html_exporter = HTMLExporter() html_exporter.template_file = 'basic' (body, resources) = html_exporter.from_notebook_node(notebook) cxt = dict( vid=vid, b=b, fileid=fileid, nbfile=nbfile, notebooks=b.build_source_files.list_records(File.BSFILE.NOTEBOOK), notebook=notebook, notebook_html=body, **aac.cc ) return aac.render('bundle/notebook.html', **cxt)
def update_jupyter(self, s, keywords): '''Update @jupyter node in the vr pane.''' pc = self c = pc.c if pc.must_change_widget(QtWebKitWidgets.QWebView): # g.trace('===== instantiating QWebView') w = QtWebKitWidgets.QWebView() n = c.config.getInt('qweb_view_font_size') if n: settings = w.settings() settings.setFontSize(settings.DefaultFontSize, n) pc.embed_widget(w) assert(w == pc.w) else: w = pc.w url = g.getUrlFromNode(c.p) if url and nbformat: s = urlopen(url).read().decode() try: nb = nbformat.reads(s, as_version=4) e = HTMLExporter() (s, junk_resources) = e.from_notebook_node(nb) except nbformat.reader.NotJSONError: # Assume the result is html. pass elif url: s = 'can not import nbformt: %r' % url else: s = g.u('') if isQt5: w.hide() # This forces a proper update. w.setHtml(s) w.show() c.bodyWantsFocusNow()
def parse(self, inputstring, document): """Parse `inputstring`, write results to `document`.""" nb = nbformat.reads(inputstring, as_version=_ipynbversion) env = document.settings.env srcdir = os.path.dirname(env.doc2path(env.docname)) auxdir = os.path.join(env.doctreedir, 'nbsphinx') sphinx.util.ensuredir(auxdir) resources = {} # Working directory for ExecutePreprocessor resources['metadata'] = {'path': srcdir} # Sphinx doesn't accept absolute paths in images etc. resources['output_files_dir'] = os.path.relpath(auxdir, srcdir) resources['unique_key'] = env.docname.replace('/', '_') exporter = Exporter(allow_errors=env.config.nbsphinx_allow_errors, timeout=env.config.nbsphinx_timeout, codecell_lexer=env.config.nbsphinx_codecell_lexer) try: rststring, resources = exporter.from_notebook_node(nb, resources) except NotebookError as e: env.warn(env.docname, str(e)) return # document is unchanged (i.e. empty) # Create additional output files (figures etc.), # see nbconvert.writers.FilesWriter.write() for filename, data in resources.get('outputs', {}).items(): dest = os.path.normpath(os.path.join(srcdir, filename)) with open(dest, 'wb') as f: f.write(data) rst.Parser.parse(self, rststring, document)
def _load_notebook(self, uri): """Loads a notebook from the local filesystem or HTTP URL. Raises ------ RuntimeError if no installed kernel can handle the language specified in the notebook. Returns ------- object Notebook object from nbformat """ parts = urlparse(uri) if parts.netloc == '' or parts.netloc == 'file': # Local file with open(parts.path) as nb_fh: notebook = nbformat.read(nb_fh, 4) else: # Remote file import requests resp = requests.get(uri) resp.raise_for_status() notebook = nbformat.reads(resp.text, 4) # Error if no kernel spec can handle the language requested kernel_name = notebook['metadata']['kernelspec']['name'] self.kernel_spec_manager.get_kernel_spec(kernel_name) return notebook
def parse(self, inputstring, document): """Parse `inputstring`, write results to `document`.""" nb = nbformat.reads(inputstring, as_version=_ipynbversion) nbsphinx_metadata = nb.metadata.get('nbsphinx', {}) resources = {} env = document.settings.env srcdir = os.path.dirname(env.doc2path(env.docname)) auxdir = os.path.join(env.doctreedir, 'nbsphinx') sphinx.util.ensuredir(auxdir) # Execute notebook only if there are no outputs: if not any(c.outputs for c in nb.cells if 'outputs' in c): resources.setdefault('metadata', {})['path'] = srcdir allow_errors = nbsphinx_metadata.get('allow_errors', False) pp = nbconvert.preprocessors.ExecutePreprocessor( allow_errors=allow_errors) nb, resources = pp.preprocess(nb, resources) # Remove hidden cells nb.cells[:] = (cell for cell in nb.cells if cell.metadata.get('nbsphinx') != 'hidden') # Sphinx doesn't accept absolute paths in images etc. resources['output_files_dir'] = os.path.relpath(auxdir, srcdir) resources['unique_key'] = env.docname.replace('/', '_') def get_empty_lines(s): """Get number of empty lines before and after code.""" before = 0 lines = s.split('\n') for line in lines: if line.strip(): break before += 1 after = 0 for line in reversed(lines[before:]): if line.strip(): break after += 1 return before, after resources['get_empty_lines'] = get_empty_lines loader = jinja2.DictLoader({'nbsphinx-rst.tpl': RST_TEMPLATE}) exporter = nbconvert.RSTExporter(template_file='nbsphinx-rst', extra_loaders=[loader]) rststring, resources = exporter.from_notebook_node(nb, resources) if nbsphinx_metadata.get('orphan', False): rststring = ':orphan:\n\n' + rststring # Create additional output files (figures etc.), # see nbconvert.writers.FilesWriter.write() for filename, data in resources.get('outputs', {}).items(): dest = os.path.normpath(os.path.join(srcdir, filename)) with open(dest, 'wb') as f: f.write(data) rst.Parser.parse(self, rststring, document)
def finish_notebook(self, json_notebook, download_url, provider_url=None, provider_icon=None, provider_label=None, msg=None, breadcrumbs=None, public=False, format=None, request=None): """render a notebook from its JSON body. download_url is required, provider_url is not. msg is extra information for the log message when rendering fails. """ if msg is None: msg = download_url try: nb = reads(json_notebook, current_nbformat) except ValueError: app_log.error("Failed to render %s", msg, exc_info=True) raise web.HTTPError(400, "Error reading JSON notebook") try: app_log.debug("Requesting render of %s", download_url) with time_block("Rendered %s" % download_url): app_log.info("rendering %d B notebook from %s", len(json_notebook), download_url) nbhtml, config = yield self.pool.submit(render_notebook, self.formats[format], nb, download_url, config=self.config, ) except NbFormatError as e: app_log.error("Invalid notebook %s: %s", msg, e) raise web.HTTPError(400, str(e)) except Exception as e: app_log.error("Failed to render %s", msg, exc_info=True) raise web.HTTPError(400, str(e)) else: app_log.debug("Finished render of %s", download_url) html = self.render_template( "formats/%s.html" % format, body=nbhtml, nb=nb, download_url=download_url, provider_url=provider_url, provider_label=provider_label, provider_icon=provider_icon, format=self.format, default_format=self.default_format, format_prefix=format_prefix, formats=dict(self.filter_formats(nb, json_notebook)), format_base=self.request.uri.replace(self.format_prefix, ""), date=datetime.utcnow().strftime(date_fmt), breadcrumbs=breadcrumbs, **config) if 'content_type' in self.formats[format]: self.set_header('Content-Type', self.formats[format]['content_type']) yield self.cache_and_finish(html) # Index notebook self.index.index_notebook(download_url, nb, public)
def nb2rst(filepath): with open(filepath) as fh: nb = nbformat.reads(fh.read(), nbformat.NO_CONVERT) exporter = RSTExporter() # source is a tuple of python source code # meta contains metadata source, meta = exporter.from_notebook_node(nb) return [line + '\n' for line in source.split('\n')]
def convertNotebook(notebookPath): with open(notebookPath) as fh: nb = nbformat.reads(fh.read(), nbformat.NO_CONVERT) exporter = PythonExporter() source, meta = exporter.from_notebook_node(nb) lines = source.split("\n") for line in lines: print(line)
def create_post(self, path, **kw): """Create a new post.""" if flag is None: req_missing(["ipython[notebook]>=2.0.0"], "build this site (compile ipynb)") content = kw.pop("content", None) onefile = kw.pop("onefile", False) kernel = kw.pop("ipython_kernel", None) # is_page is not needed to create the file kw.pop("is_page", False) metadata = {} metadata.update(self.default_metadata) metadata.update(kw) makedirs(os.path.dirname(path)) if content.startswith("{"): # imported .ipynb file, guaranteed to start with "{" because it’s JSON. nb = nbformat.reads(content, current_nbformat) else: if ipy_modern: nb = nbformat.v4.new_notebook() nb["cells"] = [nbformat.v4.new_markdown_cell(content)] else: nb = nbformat.new_notebook() nb["worksheets"] = [nbformat.new_worksheet(cells=[nbformat.new_text_cell("markdown", [content])])] if kernelspec is not None: if kernel is None: kernel = self.default_kernel self.logger.notice('No kernel specified, assuming "{0}".'.format(kernel)) IPYNB_KERNELS = {} ksm = kernelspec.KernelSpecManager() for k in ksm.find_kernel_specs(): IPYNB_KERNELS[k] = ksm.get_kernel_spec(k).to_dict() IPYNB_KERNELS[k]["name"] = k del IPYNB_KERNELS[k]["argv"] if kernel not in IPYNB_KERNELS: self.logger.error('Unknown kernel "{0}". Maybe you mispelled it?'.format(kernel)) self.logger.info("Available kernels: {0}".format(", ".join(sorted(IPYNB_KERNELS)))) raise Exception('Unknown kernel "{0}"'.format(kernel)) nb["metadata"]["kernelspec"] = IPYNB_KERNELS[kernel] else: # Older IPython versions don’t need kernelspecs. pass if onefile: nb["metadata"]["nikola"] = metadata with io.open(path, "w+", encoding="utf8") as fd: if ipy_modern: nbformat.write(nb, fd, 4) else: nbformat.write(nb, fd, "ipynb")
def load_ipynb_url(url, proxies): response = retryrequests.get(url, proxies=proxies) response.raise_for_status() try: return (nbformat.reads(response.text, as_version=4), len(response.content)) except IOError as e: _schema_not_found_error_handler(e) raise
def processed_notebook(assignment_id): with open('processed_notebooks/%s_reading_journal_responses.ipynb' % assignment_id) as f: nb = nbformat.reads(f.read(), as_version=4) str, _ = nbconvert.export_html(nb) assignment_name = assignments[assignment_id][1] return flask.render_template( 'processed_notebook.html', course_name=COURSE_NAME, title=' '.join([assignment_name, 'Processed Notebook']), nb_html=str)
def parse_ipynb(fname): """Use nbformat.reads to parse a .ipynb file.""" # Figure out the version number. with open(fname) as fh: contents = fh.read() version = json.loads(contents)['nbformat'] # Parse and return. parsed = reads(contents, version) return parsed
def run(): content = sys.stdin.read() version = json.loads(content)['nbformat'] content = reads(content, version) for command in SHEET_COMMANDS: command(content) for cell in content.cells: for command in CELL_COMMANDS: command(cell) write(content, sys.stdout, version)
def notebook_node_from_string_list(string_list): """ Reads a notebook from a string list and returns the NotebookNode object. :param string_list: The notebook file contents as list of strings (linewise). :return: The notebook as NotebookNode. """ return nbformat.reads(''.join(string_list), nbformat.NO_CONVERT)
def read(self, filepath, as_version=4): """ Read a notebook from Storage :param filepath: The path on the Storage to the notebook to read :param as_version: Version of the notebook return A NotebookNode objet """ self.log.debug("Read the notebook store at '%s'" % filepath); content = self.do_read(filepath); return nbformat.reads(content, as_version);
def get(self, *args): self.log.info("hide_code: Starting Latex PDF export for {}".format(args[-1])) with open(ipynb_file_name(args)) as f: nb = nbformat.reads(f.read(), as_version=4) exporter = HideCodePDFExporter() output, resources = exporter.from_notebook_node(nb, resources={"metadata": {"name": notebook_name(args)}}) self.set_header('Content-Disposition', 'attachment; filename=' + notebook_name(args) + '.pdf') self.flush() self.write(output) self.log.info("hide_code: Finished Latex PDF export for {}".format(args[-1])) self.finish()
def __getitem__(self, name): if isinstance(name, int): name = self.names[name] # Cache file reads nbs = self.cache.get(name) if nbs is None: with open(os.path.join(self.filespath, name + ".ipynb")) as f: nbs = f.read() self.cache[name] = nbs # But return a new notebook copy every time return nbformat.reads(nbs, as_version=4)
def get(self, *args): self.log.info("hide_code: Starting HTML export for {}".format(args[-1])) with open(ipynb_file_name(args)) as f: nb = nbformat.reads(f.read(), as_version=4) exporter = HideCodeHTMLExporter() output_html, resources = exporter.from_notebook_node(nb) self.set_header('Content-Type', 'text/html') self.set_header('Content-Disposition', 'attachment; filename=' + notebook_name(args) + '.html') self.flush() self.write(output_html) self.log.info("hide_code: Finished HTML export for {}".format(args[-1])) self.finish()
def render(file): """Generate the result HTML.""" fp = file.open() content = fp.read() fp.close() notebook = nbformat.reads(content.decode('utf-8'), as_version=4) html_exporter = HTMLExporter() html_exporter.template_file = 'basic' (body, resources) = html_exporter.from_notebook_node(notebook) return body, resources
def test_codefolding(): """ Test codefolding preprocessor """ nb_name='tests/data/codefolding.ipynb' with open(nb_name, 'r') as f: notebook_json = f.read() notebook = nbformat.reads(notebook_json, as_version=4) c.RSTExporter.preprocessors = ["pre_codefolding.CodeFoldingPreprocessor"] c.NbConvertApp.export_format = 'rst' rst_exporter = RSTExporter(config=c) body = rst_exporter.from_notebook_node(notebook) assert 'AXYZ12AXY' not in body[0] # firstline fold assert 'GR4CX32ZT' not in body[0] # function fold
def parse(self, fn): if g.os_path_exists(fn): with open(fn) as f: # payload_source = f.name payload = f.read() nb = nbformat.reads(payload, as_version=4) # nbformat.NO_CONVERT: no conversion # as_version=4: Require IPython 4. return nb else: g.es_print('not found', fn) return None
def get(self, *args): self.log.info("hide_code: Starting HTML export for {}".format( args[-1])) with open(ipynb_file_name(args), encoding="utf-8") as f: nb = nbformat.reads(f.read(), as_version=4) exporter = HideCodeHTMLExporter() output_html, resources = exporter.from_notebook_node(nb) self.set_header('Content-Type', 'text/html') self.set_header('Content-Disposition', 'attachment; filename=' + notebook_name(args, 'html')) self.flush() self.write(output_html) self.log.info("hide_code: Finished HTML export for {}".format( args[-1])) self.finish()
def sample_perf(nb, n=30): samples = pd.DataFrame( pd.np.NaN, index=pd.MultiIndex.from_product( (range(n), ['nbformat'] + JUPYTEXT_FORMATS), names=['sample', 'implementation']), columns=pd.Index(['size', 'read', 'write'], name='measure')) for i, fmt in samples.index: t0 = time.time() if fmt == 'nbformat': text = nbformat.writes(nb) else: text = jupytext.writes(nb, fmt) t1 = time.time() samples.loc[(i, fmt), 'write'] = t1 - t0 samples.loc[(i, fmt), 'size'] = len(text) t0 = time.time() if fmt == 'nbformat': nbformat.reads(text, as_version=4) else: jupytext.reads(text, fmt) t1 = time.time() samples.loc[(i, fmt), 'read'] = t1 - t0 return samples
def exec_module(self, module: ModuleType) -> ModuleType: """Exceute each cell in the module attaching everything to the module. """ path = Path(self.path) source = path.read_text() nb = (load_markdown(source) if path.suffix in ('.markdown', '.md') else reads(source, 4)) for cell in nb.cells: try: if cell['cell_type'] == 'code': exec(self.transformer(cell.source, ns=module.__dict__), module.__dict__) except: raise ImportError(cell.source) return module
def main(args): args = parse_args(args) tmpfile = tempfile.mktemp(suffix='.ipynb') run_notebook(tmpfile, clfname=args.clf) url = 'file://' + tmpfile response = urlopen(url).read().decode() jake_notebook = nbformat.reads(response, as_version=4) # save_notebook() # save_notebook() # time.sleep(2) # current_file = 'GMM.ipynb' # output_file = 'output_file.html' output_HTML(jake_notebook, output_file=args.clf + '.html')
def __init__(self, filename, refs): logger.info('Operating on notebook {}'.format(filename)) with open(filename, 'r') as f: self.contents = nbformat.reads(f.read(), as_version=4) self.cells = [JupyterCell(**c) for c in self.contents['cells']] self.metadata = self.contents['metadata'] self.keys = [] self.refs = refs self.filename = filename # make backup split = filename.split('/') backup_name = '/'.join(split[:-1] + ['.' + split[-1]]) shutil.copyfile(filename, backup_name) self.err = None self.nglview = False
def read_notebook(self, arg, fail_on_empty=True): # Currently assuming arg is a filename relative to # where the server was started from, later we may # want to accept urls or full notebooks as well. if not isinstance(arg, str): raise web.HTTPError(400, 'Expecting a filename or a URL.') try: # Check that file exists if arg == EXPLICIT_MISSING_FILE: path = arg else: path = os.path.join(self.curdir, arg) if not os.path.exists(path): if '://' not in arg: raise ValueError( 'Supplied argument cannot be read: %r' % arg) # Assume file is URI r = requests.get(arg) r.raise_for_status() # Let nbformat do the reading and validation if path == EXPLICIT_MISSING_FILE: nb = nbformat.v4.new_notebook() elif os.path.exists(path): try: nb = nbformat.read(path, as_version=4) except nbformat.reader.NotJSONError: if fail_on_empty: raise # Handle empty notebook file if isinstance(path, str): with io.open(path, encoding='utf-8') as fo: if len(fo.read(10)) != 0: raise nb = nbformat.v4.new_notebook() else: nb = nbformat.reads(r.text, as_version=4) except requests.exceptions.HTTPError as e: self.log.exception(e) raise web.HTTPError( 422, 'Invalid notebook: %s, received http error: %s' % (arg, str(e))) except Exception as e: self.log.exception(e) raise web.HTTPError(422, 'Invalid notebook: %s' % arg) return nb
def main(app): static_dir = os.path.join(app.builder.srcdir, '_static') target_dir = os.path.join(app.builder.srcdir, 'notebooks') source_dir = os.path.abspath( os.path.join(app.builder.srcdir, '..', 'notebooks')) rendered_dir = os.path.join(target_dir, 'rendered') if not os.path.exists(static_dir): os.makedirs(static_dir) if not os.path.exists(target_dir): os.makedirs(target_dir) if not os.path.exists(rendered_dir): os.makedirs(rendered_dir) nbroots = [] nbtitles = [] exporter = HTMLExporter(template_file='full') for nb_src in glob.glob(os.path.join(source_dir, '*.ipynb')): print("converting notebook {0}".format(nb_src)) basedir, nbname = os.path.split(nb_src) nb_dest = os.path.join(target_dir, nbname) shutil.copyfile(nb_src, nb_dest) with open(nb_dest, 'r') as f: nb_json = nbformat.reads(f.read(), as_version=4) (body, resources) = exporter.from_notebook_node(nb_json) root, ext = os.path.splitext(nbname) nb_html_dest = os.path.join(rendered_dir, root + '.html') with open(nb_html_dest, 'w') as f: f.write(body) nbroots.append(root) nbtitles.append(get_notebook_title(nb_json, root)) for nbroot, nbtitle in zip(nbroots, nbtitles): with open(os.path.join(target_dir, nbroot + '.rst'), 'w') as f: f.write(RST_TEMPLATE.render(title=nbtitle, nbroot=nbroot)) with open(os.path.join(target_dir, 'index.rst'), 'w') as f: f.write( INDEX_TEMPLATE.render(notebooks=nbroots, sphinx_tag='notebook-examples'))
def convert_tohtml(download_path): try: from nbconvert import HTMLExporter import nbformat f = open(download_path) con = f.read() con = nbformat.reads(con, as_version=4) con['nbformat_minor'] = 4 html_exporter = HTMLExporter() html_exporter.template_name = 'basic' generated_html = html_exporter.from_notebook_node(con)[0] except Exception as e: raise e return '<pre>nbconvert parsing error</pre>' return generated_html
def ipython_to_pdf(raw_executed_ipynb: str, report_title: str, hide_code: bool = False) -> AnyStr: c = Config() c.PDFExporter.exclude_input = hide_code c.PDFExporter.exclude_output_prompt = hide_code c.HTMLExporter.template_file = pkg_resources.resource_filename( __name__, "../nbtemplates/notebooker_pdf_output.tplx") pdf_exporter = PDFExporter(c) resources = ResourcesDict() resources["metadata"] = ResourcesDict() resources["metadata"]["name"] = report_title pdf, _ = pdf_exporter.from_notebook_node(nbformat.reads( raw_executed_ipynb, as_version=nbformat.v4.nbformat), resources=resources) return pdf
def notebook_details(request, notebook_id): notebook = SharedNotebook.objects.get(pk=notebook_id) liked = False if request.user.is_authenticated: if notebook.notebooklike_set.filter(oh_member=request.user.oh_member): liked = True format_notebook = nbformat.reads(notebook.notebook_content, as_version=nbformat.NO_CONVERT) html_exporter = nbconvert.HTMLExporter() html_exporter.template_file = 'basic' (body, resources) = html_exporter.from_notebook_node(format_notebook) return render(request, 'main/notebook_details.html', { 'notebook': notebook, 'notebook_preview': body, 'liked': liked })
def collect(self): with self.fspath.open() as f: payload = f.read() self.nb = nbformat.reads(payload, 3) # kernel needs to start from the same dir the ipynb is in notebook_dir = self.fspath.dirname cwd = os.getcwd() if cwd != notebook_dir: os.chdir(notebook_dir) self.runner = NotebookRunner(self.nb) os.chdir(cwd) cell_num = 1 for cell in self.runner.iter_code_cells(): yield IPyNbCell.from_parent(self, name=self.name, cell_num=cell_num, cell=cell) cell_num += 1
def nblint(nb_paths): """lint a number of notebook paths""" len_paths = len(nb_paths) for i, nb_path in enumerate(nb_paths): nb_text = nb_path.read_text(encoding="utf-8") if len_paths > 1: print(f"[{i + 1} of {len_paths}] {nb_path}", flush=True) nb_node = nblint_one(nbformat.reads(nb_text, 4)) with nb_path.open("w", encoding="utf-8") as fpt: nbformat.write(nb_node, fpt) return 0
def notebook_view(request_args): check.dict_param(request_args, "request_args") # This currently provides open access to your file system - the very least we can # do is limit it to notebook files until we create a more permanent solution. path = request_args["path"] if not path.endswith(".ipynb"): return "Invalid Path", 400 with open(os.path.abspath(path)) as f: read_data = f.read() notebook = nbformat.reads(read_data, as_version=4) html_exporter = HTMLExporter() html_exporter.template_file = "basic" (body, resources) = html_exporter.from_notebook_node(notebook) return "<style>" + resources["inlining"]["css"][0] + "</style>" + body, 200
def list_to_cells(lst): '''convert list of cells to notebook form list should be of the form: [[list of strings representing python code for cell]] ''' cells = '"cells": [' for cell in lst: to_add = '{"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": ["' + '\\n","'.join( cell) + '"]},' cells += to_add cells = cells[:-1] + '],' nb = '{' + cells + '"metadata": {"header": "HEADLESS", "kernelspec": {"display_name" : "python", "language": "", "name": "python"}, "language":"python"' return nbformat.writes(nbformat.reads(nb, as_version=4)).encode('utf-8')
def ipython_to_html(ipynb_path: str, job_id: str) -> (nbformat.NotebookNode, Dict[str, Any]): c = Config() c.HTMLExporter.preprocessors = [ "nbconvert.preprocessors.ExtractOutputPreprocessor" ] c.HTMLExporter.template_file = pkg_resources.resource_filename( __name__, "../nbtemplates/notebooker_html_output.tpl") html_exporter_with_figs = HTMLExporter(config=c) with open(ipynb_path, "r") as nb_file: nb = nbformat.reads(nb_file.read(), as_version=nbformat.v4.nbformat) resources_dir = get_resources_dir(job_id) html, resources = html_exporter_with_figs.from_notebook_node( nb, resources={"output_files_dir": resources_dir}) return html, resources
def _read_notebook(self, os_path, as_version=4): """Read a notebook from an os path.""" with self.open(os_path, 'r', encoding='utf-8') as f: try: if ftdetect(os_path) == 'notebook': return nbformat.read(f, as_version=as_version) elif ftdetect(os_path) == 'markdown': nbjson = self.convert(os_path, informat='markdown', outformat='notebook') return nbformat.reads(nbjson, as_version=as_version) except Exception as e: raise web.HTTPError( 400, u"Unreadable Notebook: %s %r" % (os_path, e), )
def notebook_view(request_args): check.dict_param(request_args, 'request_args') # This currently provides open access to your file system - the very least we can # do is limit it to notebook files until we create a more permanent solution. path = request_args['path'] if not path.endswith('.ipynb'): return 'Invalid Path', 400 with open(os.path.abspath(path)) as f: read_data = f.read() notebook = nbformat.reads(read_data, as_version=4) html_exporter = HTMLExporter() html_exporter.template_file = 'basic' (body, resources) = html_exporter.from_notebook_node(notebook) return '<style>' + resources['inlining']['css'][0] + '</style>' + body, 200
def get(self, path, content=True, type=None, format=None): """Get the model of a file or directory with or without content.""" path = path.strip('/') model = base_model(path, path) if self.exists(path) and type != u'directory': #It's a narrative object, so try to fetch it. obj_ref = self._parse_path(path) if not obj_ref: raise HTTPError(404, u'Unknown Narrative "{}"'.format(path)) try: nar_obj = self.read_narrative(u'{}/{}'.format(obj_ref[u'wsid'], obj_ref[u'objid']), content) model[u'type'] = u'notebook' user = self.get_userid() if content: model['format'] = u'json' nb = nbformat.reads(json.dumps(nar_obj['data']), 4) nb['metadata'].pop('orig_nbformat', None) self.mark_trusted_cells(nb, path) model['content'] = nb model['name'] = nar_obj['data']['metadata'].get('name', 'Untitled') util.kbase_env.narrative = 'ws.{}.obj.{}'.format(obj_ref['wsid'], obj_ref['objid']) util.kbase_env.workspace = model['content'].metadata.ws_name if user is not None: model['writable'] = self.narrative_writable(u'{}/{}'.format(obj_ref['wsid'], obj_ref['objid']), user) self.log.info(u'Got narrative {}'.format(model['name'])) except HTTPError: raise except PermissionsError as e: raise HTTPError(403, e) except Exception as e: raise HTTPError(500, u'An error occurred while fetching your narrative: {}'.format(e)) if not path or type == 'directory': #if it's the empty string, look up all narratives, treat them as a dir self.log.info(u'Getting narrative list') model['type'] = type model['format'] = u'json' if content: contents = [] nar_list = self.list_narratives() self.log.info('Found {} narratives'.format(len(nar_list))) for nar in nar_list: contents.append(self._wsobj_to_model(nar, content=False)) model['content'] = contents return model
def gen_tutorials(repo_dir: str) -> None: """Generate HTML tutorials for Docusaurus site from Jupyter notebooks. Also create ipynb and py versions of tutorial in Docusaurus site for download. """ with open(os.path.join(repo_dir, "website", "tutorials.json"), "r") as infile: tutorial_config = json.loads(infile.read()) tutorial_ids = {x["id"] for v in tutorial_config.values() for x in v} for tid in tutorial_ids: print("Generating {} tutorial".format(tid)) # convert notebook to HTML ipynb_in_path = os.path.join(repo_dir, "tutorials", "{}.ipynb".format(tid)) with open(ipynb_in_path, "r") as infile: nb_str = infile.read() nb = nbformat.reads(nb_str, nbformat.NO_CONVERT) # displayname is absent from notebook metadata nb["metadata"]["kernelspec"]["display_name"] = "python3" exporter = HTMLExporter(template_name="classic") html, meta = exporter.from_notebook_node(nb) # pull out html div for notebook soup = BeautifulSoup(html, "html.parser") nb_meat = soup.find("div", {"id": "notebook-container"}) del nb_meat.attrs["id"] nb_meat.attrs["class"] = ["notebook"] html_out = JS_SCRIPTS + str(nb_meat) # generate html file html_out_path = os.path.join( repo_dir, "website", "tutorials", "{}.html".format(tid) ) with open(html_out_path, "w") as html_outfile: html_outfile.write(html_out) # generate JS file script = TEMPLATE.format(tid) js_out_path = os.path.join( repo_dir, "website", "pages", "tutorials", "{}.js".format(tid) ) with open(js_out_path, "w") as js_outfile: js_outfile.write(script)
def reads(text, fmt, as_version=nbformat.NO_CONVERT, config=None, **kwargs): """ Read a notebook from a string :param text: the text representation of the notebook :param fmt: (optional) the jupytext format like `md`, `py:percent`, ... :param as_version: see nbformat.reads :param config: (optional) a Jupytext configuration object :param kwargs: (not used) additional parameters for nbformat.reads :return: the notebook """ fmt = copy(fmt) if fmt else divine_format(text) fmt = long_form_one_format(fmt) ext = fmt["extension"] if ext == ".ipynb": nb = nbformat.reads(text, as_version, **kwargs) (version, version_minor) = nbformat.reader.get_version(nb) if version != 4: warnings.warn( f"Notebooks in nbformat version {version}.{version_minor} are not supported by Jupytext. " f"Please consider converting them to nbformat version 4.x with " f"'jupyter nbconvert --to notebook --inplace'" ) return nb format_name = read_format_from_metadata(text, ext) or fmt.get("format_name") if format_name: format_options = {} else: format_name, format_options = guess_format(text, ext) if format_name: fmt["format_name"] = format_name fmt.update(format_options) reader = TextNotebookConverter(fmt, config) notebook = reader.reads(text, **kwargs) rearrange_jupytext_metadata(notebook.metadata) if format_name and insert_or_test_version_number(): notebook.metadata.setdefault("jupytext", {}).setdefault( "text_representation", {} ).update({"extension": ext, "format_name": format_name}) return notebook
def get(self, path, content=True, type=None, format=None): """Get the model of a file or directory with or without content.""" path = path.strip("/") model = base_model(path, path) try: if self.exists(path) and type != "directory": # It's a narrative object, so try to fetch it. ref = self._parse_path(path) if not ref: raise HTTPError(404, 'Unknown Narrative "{}"'.format(path)) nar_obj = self.read_narrative(ref, content=content) model["type"] = "notebook" user = self.get_userid() if content: model["format"] = "json" nb = nbformat.reads(json.dumps(nar_obj["data"]), 4) nb["metadata"].pop("orig_nbformat", None) self.mark_trusted_cells(nb, nar_obj["info"][5], path) model["content"] = nb model["name"] = nar_obj["data"]["metadata"].get("name", "Untitled") util.kbase_env.narrative = "ws.{}.obj.{}".format( ref.wsid, ref.objid ) util.kbase_env.workspace = model["content"].metadata.ws_name self.narrative_logger.narrative_open( "{}/{}".format(ref.wsid, ref.objid), nar_obj["info"][4] ) if user is not None: model["writable"] = self.narrative_writable(ref, user) self.log.info("Got narrative {}".format(model["name"])) except WorkspaceError as e: raise HTTPError(e.http_code, e.message) if not path or type == "directory": # if it's the empty string, look up all narratives, treat them as a dir self.log.info("Getting narrative list") model["type"] = type model["format"] = "json" if content: contents = [] nar_list = self.list_narratives() self.log.info("Found {} narratives".format(len(nar_list))) for nar in nar_list: contents.append(self._wsobj_to_model(nar, content=False)) model["content"] = contents return model
def __init__(self, node_or_path): if isinstance(node_or_path, string_types): path = urlparse(node_or_path).path if not os.path.splitext(path)[-1].endswith('ipynb'): raise Warning( "Requires an '.ipynb' file extension. Provided path: '{}'". format(node_or_path)) self.path = node_or_path self.node = nbformat.reads(papermill_io.read(node_or_path), as_version=4) else: self.path = "" self.node = node_or_path # Memoized traits self._scraps = None self._outputs = None
def get_source(path, basedir="."): import nbformat source = None if not path.startswith("http") and not path.startswith("git+file"): if path.startswith("file://"): path = path[7:] elif path.startswith("file:"): path = path[5:] if not os.path.isabs(path): path = os.path.abspath(os.path.join(basedir, path)) path = "file://" + path path = format(path, stepout=1) if path.startswith("file://"): sourceurl = "file:" + pathname2url(path[7:]) elif path.startswith("git+file"): source = git_content(path) (root_path, file_path, version) = split_git_path(path) path = path.rstrip("@" + version) else: sourceurl = path if source is None: with urlopen(sourceurl) as source: source = source.read() language = None if path.endswith(".py"): language = "python" elif path.endswith(".ipynb"): language = "jupyter" elif path.endswith(".R"): language = "r" elif path.endswith(".Rmd"): language = "rmarkdown" elif path.endswith(".jl"): language = "julia" # detect kernel language for Jupyter Notebooks if language == "jupyter": nb = nbformat.reads(source, as_version=nbformat.NO_CONVERT) kernel_language = nb["metadata"]["language_info"]["name"] language += "_" + kernel_language.lower() return path, source, language
def __init__(self, node_or_path): if isinstance(node_or_path, string_types): if not node_or_path.endswith(".ipynb"): raise ValueError( "Requires an '.ipynb' file extension. Provided path: '{}'".format( node_or_path ) ) self.path = node_or_path self.node = nbformat.reads(papermill_io.read(node_or_path), as_version=4) else: self.path = "" self.node = node_or_path # Memoized traits self._scraps = None self._outputs = None
def execute_test(file_path, result_path): """Executes a single notebook. Args: file_path: Path to the notebook to execute. result_path: Path to store the resulting notebook. Returns: bool: True if the notebook does not have any errors, False otherwise. Raises: Exception if an unexpected error occurs executing the notebook. """ try: with open(file_path, 'r') as f: filedata = f.read() if FLAGS.override_pip_install_agents: # Replaces pip install tf-agents with a noop. If this gets any bigger, # refactor filedata = filedata.replace( 'pip install tf-agents-nightly[reverb]', 'pip --version') filedata = filedata.replace('pip install tf-agents-nightly', 'pip --version') filedata = filedata.replace('pip install tf-agents[reverb]', 'pip --version') filedata = filedata.replace( 'pip install --pre tf-agents[reverb]', 'pip --version') filedata = filedata.replace('pip install tf-agents', 'pip --version') filedata = filedata.replace('pip install --pre tf-agents', 'pip --version') nb = nbformat.reads(filedata, as_version=4) ep = ExecutePreprocessor(timeout=3600, kernel_name='python3') try: ep.preprocess(nb, {'metadata': {'path': FLAGS.output_dir}}) except CellExecutionError as cex: logging.error('ERROR executing:%s', file_path) logging.error(cex) return False with open(result_path, 'w', encoding='utf-8') as fo: nbformat.write(nb, fo) return True except Exception as e: # pylint: disable=W0703 logging.error('Unexpected ERROR: in %s', file_path) logging.error(e)
def get(self, *args): self.log.info("hide_code: Starting Latex PDF export for {}".format( args[-1])) with open(ipynb_file_name(args)) as f: nb = nbformat.reads(f.read(), as_version=4) exporter = HideCodePDFExporter() output, resources = exporter.from_notebook_node( nb, resources={"metadata": { "name": notebook_name(args) }}) self.set_header('Content-Disposition', 'attachment; filename=' + notebook_name(args) + '.pdf') self.flush() self.write(output) self.log.info("hide_code: Finished Latex PDF export for {}".format( args[-1])) self.finish()
def parse(self, inputstring, document): """Parse `inputstring`, write results to `document`.""" nb = nbformat.reads(inputstring, as_version=_ipynbversion) env = document.settings.env srcdir = os.path.dirname(env.doc2path(env.docname)) auxdir = os.path.join(env.doctreedir, 'nbsphinx') sphinx.util.ensuredir(auxdir) resources = {} # Working directory for ExecutePreprocessor resources['metadata'] = {'path': srcdir} # Sphinx doesn't accept absolute paths in images etc. resources['output_files_dir'] = os.path.relpath(auxdir, srcdir) resources['unique_key'] = re.sub('[/ ]', '_', env.docname) exporter = Exporter( execute=env.config.nbsphinx_execute, kernel_name=env.config.nbsphinx_kernel_name, execute_arguments=env.config.nbsphinx_execute_arguments, allow_errors=env.config.nbsphinx_allow_errors, timeout=env.config.nbsphinx_timeout, codecell_lexer=env.config.nbsphinx_codecell_lexer, ) try: rststring, resources = exporter.from_notebook_node(nb, resources) except nbconvert.preprocessors.execute.CellExecutionError as e: lines = str(e).split('\n') lines[0] = 'CellExecutionError in {}:'.format( env.doc2path(env.docname, base=None)) lines.append("You can ignore this error by setting the following " "in conf.py:\n\n nbsphinx_allow_errors = True\n") raise NotebookError('\n'.join(lines)) except Exception as e: raise NotebookError( type(e).__name__ + ' in ' + env.doc2path(env.docname, base=None) + ':\n' + str(e)) # Create additional output files (figures etc.), # see nbconvert.writers.FilesWriter.write() for filename, data in resources.get('outputs', {}).items(): dest = os.path.normpath(os.path.join(srcdir, filename)) with open(dest, 'wb') as f: f.write(data) rst.Parser.parse(self, rststring, document)
def run_notebook_test(notebook_path, parameters=None): # Ensure workload identity is ready. # TODO(jlewi): Need to skip this when not running on GCP. gcp_util.get_gcp_credentials() output_path = execute_notebook(notebook_path, parameters=parameters) logging.info(f"Reading notebook {output_path}") with open(output_path, "r") as hf: actual_output = hf.read() logging.info("Converting notebook to html") nb = nbformat.reads(actual_output, as_version=4) html_exporter = nbconvert.HTMLExporter() (html_output, _) = html_exporter.from_notebook_node(nb) gcs_path = os.getenv("OUTPUT_GCS") logging.info(f"Uploading notebook to {gcs_path}") _upload_notebook_html(html_output, gcs_path)
def notebookExportToHtml(outputFilePath=None): """Export current notebook to HTML. If outputFilePath is not specified then filename will be generated from the notebook filename with current timestamp appended. It returns full path of the saved html file. It requires nbformat and nbconvert packages. You can use this command to install them:: pip_install("nbformat nbconvert") """ try: import nbformat from nbconvert import HTMLExporter except ModuleNotFoundError: import logging logging.error( "notebookExportToHtml requires nbformat and nbconvert. They can be installed by running this command:\n\n pip_install('nbformat nbconvert')\n" ) import datetime, json, os notebook_path = notebookPath() # Generate output file path from notebook name and timestamp (if not specified) if not outputFilePath: this_notebook_name = os.path.splitext( os.path.basename(notebook_path))[0] save_timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S') save_file_name = this_notebook_name + "_" + save_timestamp + ".html" notebooks_save_path = os.path.dirname(notebook_path) outputFilePath = os.path.join(notebooks_save_path, save_file_name) with open(notebook_path, mode="r") as f: file_json = json.load(f) notebook_content = nbformat.reads(json.dumps(file_json), as_version=4) html_exporter = HTMLExporter() (body, resources) = html_exporter.from_notebook_node(notebook_content) f = open(outputFilePath, 'wb') f.write(body.encode()) f.close() return outputFilePath
def create_post(self, path, **kw): """Create a new post.""" self._req_missing_ipynb() content = kw.pop('content', None) onefile = kw.pop('onefile', False) kernel = kw.pop('jupyter_kernel', None) # is_page is not needed to create the file kw.pop('is_page', False) metadata = {} metadata.update(self.default_metadata) metadata.update(kw) makedirs(os.path.dirname(path)) if content.startswith("{"): # imported .ipynb file, guaranteed to start with "{" because it’s JSON. nb = nbformat.reads(content, current_nbformat) else: nb = nbformat.v4.new_notebook() nb["cells"] = [nbformat.v4.new_markdown_cell(content)] if kernel is None: kernel = self.default_kernel self.logger.notice('No kernel specified, assuming "{0}".'.format(kernel)) IPYNB_KERNELS = {} ksm = kernelspec.KernelSpecManager() for k in ksm.find_kernel_specs(): IPYNB_KERNELS[k] = ksm.get_kernel_spec(k).to_dict() IPYNB_KERNELS[k]['name'] = k del IPYNB_KERNELS[k]['argv'] if kernel not in IPYNB_KERNELS: self.logger.error('Unknown kernel "{0}". Maybe you mispelled it?'.format(kernel)) self.logger.info("Available kernels: {0}".format(", ".join(sorted(IPYNB_KERNELS)))) raise Exception('Unknown kernel "{0}"'.format(kernel)) nb["metadata"]["kernelspec"] = IPYNB_KERNELS[kernel] if onefile: nb["metadata"]["nikola"] = metadata with io.open(path, "w+", encoding="utf8") as fd: nbformat.write(nb, fd, 4)