def __init__(self, path: Path): self.path: Path = path # Read the notebook with nbformat with open(self.path) as f: nb_raw = f.read() self.nb_dict: NotebookNode = nbformat.reads(nb_raw, as_version=4) # Populate the list of Cells self.cells: List[Cell] = [ Cell(cell_index, cell_dict) for cell_index, cell_dict in enumerate(self.nb_dict.cells) ] self.non_executed = all( [cell.non_executed for cell in self.code_cells]) # Convert the notebook to a Python script python_exporter = nbconvert.PythonExporter() self.script, _ = python_exporter.from_notebook_node(self.nb_dict) # Extract the Python abstract syntax tree # (or set `has_invalid_python_syntax` to True) self.has_invalid_python_syntax: bool = False try: self.ast = ast.parse(self.script) except SyntaxError: self.has_invalid_python_syntax = True
def main(): args = parser.parse_args() full_path = args.full_path force = args.force if full_path is None: full_path = os.path.join(BASE_PATH, "..", "examples", "notebooks") notebook = args.notebook if notebook is None: notebooks = glob.glob(os.path.join(full_path, "*.ipynb")) else: if not notebook.endswith(".ipynb"): notebook = notebook + ".ipynb" notebook = os.path.abspath(os.path.join(full_path, notebook)) if not os.path.exists(notebook): raise FileNotFoundError("Notebook {0} not found.".format(notebook)) notebooks = [notebook] if not notebooks: import warnings warnings.warn("No notebooks found", UserWarning) for nb in notebooks: nb_full_name = os.path.split(nb)[1] nb_name = os.path.splitext(nb_full_name)[0] py_name = nb_name + ".py" # Get base directory to notebook out_file = os.path.split(nb)[0] # Write to ../python out_file = os.path.join(out_file, "..", "python", py_name) if is_newer(out_file, nb) and not force: logger.info("Skipping {0}, exported version newer than " "notebook".format(nb_name)) continue logger.info("Converting {0}".format(nb_name)) with open(nb, "r", encoding="utf8") as nb_file: converter = nbconvert.PythonExporter() python = converter.from_file(nb_file) code = python[0].split("\n") code_out = [] for i, block in enumerate(code): if "get_ipython" in block: continue elif block.startswith("# In[ ]:"): continue if block.startswith("#"): # Wrap comments from Markdown block = textwrap.fill(block, width=74) block = block.replace("\n", "\n# ") code_out.append(block) if not code_out[0]: code_out = code_out[1:] loc = 0 for i, line in enumerate(code_out): if "# coding: utf" in line: loc = i + 1 break code_out.insert(loc, DO_NOT_EDIT.format(notebook=nb_full_name)) code_out = "\n".join(code_out) code_out, success = FormatCode(code_out, style_config="pep8") with open(out_file, "w", encoding="utf8", newline="\n") as of: of.write(code_out)
def test_example_notebooks(example, examples_directory): import nbconvert import nbformat exporter = nbconvert.PythonExporter() notebooks_folder = examples_directory / "notebooks" env = os.environ.copy() env["PYTHONWARNINGS"] = "ignore" with open(notebooks_folder / example) as fp: nbcode = nbformat.reads(fp.read(), as_version=4) (code, _) = exporter.from_notebook_node(nbcode) script = "import matplotlib\nmatplotlib.use('Agg')\n" + code process = subprocess.run( ["python", "-c", script], capture_output=True, cwd=str(notebooks_folder), env=env, ) if b"SpiceSolverError" in process.stderr: skip("No SPICE solver found.") elif b"SmartsSolverError" in process.stderr: skip("No SMARTS solver found.") elif b"RCWASolverError" in process.stderr: skip("No RCWA solver found.") elif process.stderr != b"": raise Exception(process.stderr.decode())
def jup2py(jup_filename, py_filename=None, save=True): # Converting jupyter to py through nbconvertor py_code = nbconvert.export(nbconvert.PythonExporter(), jup_filename) mycode = py_code[0].split("\n") clean_code = [] for index, line in enumerate(mycode): # Removing which are either "\n" or denotes the "Input" Tag for jupyter notebook if line == "\n" or ( line.startswith("# In[")) or line is None or line == "": pass # Commenting the magic functions or the command line elif line.startswith("get_ipython()"): line = "#" + line clean_code.append(line) # Remove the jup2py line itself elif "jup2py" in line: pass else: clean_code.append(line) clean_code = "\n".join(clean_code) # Saving back the .py file if save: save_pyfile(clean_code, py_filename) else: return clean_code
def nb_to_py(nb_file: str, location: str) -> str: """Convery .ipynb to temporary .py file.""" try: import nbformat import nbconvert except ImportError as e: raise pe.ScriptImportError( f"{e}\nCannot import notebook conversion libraries." + "Please add `jupyter` (or `nbformat` and `nbconvert`)" + " to your dependencies.", ) handle, filename = tempfile.mkstemp(text=True, suffix=".py") with os.fdopen(handle, "w") as tf: with open(os.path.join(location, nb_file)) as fh: nb = nbformat.reads(fh.read(), nbformat.NO_CONVERT) exporter = nbconvert.PythonExporter() src, _ = exporter.from_notebook_node(nb) tf.writelines(src) # delete file on exit atexit.register(functools.partial(os.remove, filename)) return os.path.basename(filename), os.path.dirname(filename)
def extract_py_notebook(in_file: Path) -> str: """Extract the python code from a given notebook""" # we use config for importing c = TConfig() # c.PythonExporter.preprocessors = [] # load the notebook notebook: nbformat.NotebookNode = nbformat.read( str(in_file), as_version=nbformat.NO_CONVERT) # TODO - any preprocessing here # convert it conv = nbconvert.PythonExporter(config=c) (body, resources) = conv.from_notebook_node(notebook) # (body, resources) = conv.from_filename(str(in_file)) # write the notebook # writer = nbconvert.writers.FilesWriter() # writer.write(body, resources, "output_notebook") # TODO - use nbconvert jinja support once v6 supports custom templates properly # postprocess body # we need to mock get_ipython as nbconvert doesn't comment it out header = """# inject get_ipython mock for magic functions from unittest.mock import Mock get_ipython = Mock() """ script = header + body return script
def upload_file(self, source, mkdir=False, append=False, binary=False, quiet=True, root=""): if os.path.exists(source) and os.path.isfile(source): destination = source root_len = len(root) if root != "" and root_len + 1 < len( source) and source[:root_len] == root: destination = destination[root_len + 1:] if destination.endswith(".ipynb"): notebook = nbformat.read(source, as_version=4) py_exporter = nbconvert.PythonExporter() output, resources = py_exporter.from_notebook_node(notebook) file_contents = output.replace("get_ipython().run_line_magic", "# %") destination = destination.replace(".ipynb", ".py") else: file_contents = open(source, "r").read() self.sres("\n\nuploading '{0}'\n".format(destination)) self.dc.send_to_file(destination, mkdir=mkdir, append=append, binary=binary, quiet=quiet, file_contents=file_contents) else: self.sres("'{0}' is not a file\n\n".format(source))
def extract_extrametadata(notebook, override=None): base = notebook.metadata.get('celltests', {}) override = override or {} base['lines'] = 0 # TODO: is this used? base['kernelspec'] = notebook.metadata.get('kernelspec', {}) # "python code" things (e.g. number of function definitions)... # note: no attempt to be clever here (so e.g. "%time def f: pass" would be missed, as would the contents of # a cell using %%capture cell magics; possible to handle those scenarios but would take more effort) code = nbconvert.PythonExporter( exclude_raw=True).from_notebook_node(notebook)[0] # notebooks start with "coding: utf-8" parsed_source = ast.parse( code.encode('utf8') if sys.version_info[0] == 2 else code) fn_def_counter = FnDefCounter() fn_def_counter.visit(parsed_source) base['functions'] = fn_def_counter.count class_counter = ClassDefCounter() class_counter.visit(parsed_source) base['classes'] = class_counter.count # alternative to doing it this way would be to check the ipython # souce for %magic, %%magics before it's converted to regular python magics_recorder = MagicsRecorder() magics_recorder.visit(parsed_source) base['magics'] = magics_recorder.seen # "notebook structure" things... base['cell_count'] = 0 base['cell_tested'] = [] base['test_count'] = 0 base['cell_lines'] = [] for c in notebook.cells: if c.get('cell_type') in ( 'markdown', 'raw', ): continue base['cell_lines'].append(0) base['cell_tested'].append(False) base['cell_count'] += 1 for line in c['source'].split('\n'): base['lines'] += 1 base['cell_lines'][-1] += 1 for t in c['metadata'].get('tests', []): if t.strip().startswith('%cell'): base['test_count'] += 1 base['cell_tested'][-1] = True break # in case you want to override notebook settings if override: base.update(override) return base
def apply_preprocessors(preprocessors, nbname): notebooks_path = os.path.join(os.path.split(__file__)[0], 'notebooks') with open(os.path.join(notebooks_path, nbname)) as f: nb = nbformat.read(f, nbformat.NO_CONVERT) exporter = nbconvert.PythonExporter() for preprocessor in preprocessors: exporter.register_preprocessor(preprocessor) source, meta = exporter.from_notebook_node(nb) return source
def preprocess(self): exporter = nbconvert.PythonExporter() exporter.register_preprocessor(self.notebook_preprocessor, enabled=True) contents, _ = exporter.from_filename(self.notebook_file) converted_notebook = Path(self.notebook_file).with_suffix('.py') with open(converted_notebook, 'w') as f: f.write(contents) self.executable = converted_notebook return [converted_notebook]
def export_as_python(filename): with io.open(filename, encoding='utf8') as nbfile: nb = nbformat.read(nbfile, as_version=4) output, _ = nbconvert.PythonExporter().from_notebook_node(nb) if sys.version_info[0] == 2: # notebooks will start with "coding: utf-8", but output already unicode # (alternative hack would be to find the coding line and remove it) output = output.encode('utf8') return output
def main(): args = parser.parse_args() full_path = args.full_path force = args.force if full_path is None: full_path = os.path.join(BASE_PATH, '..', 'examples', 'notebooks') notebook = args.notebook if notebook is None: notebooks = glob.glob(os.path.join(full_path, '*.ipynb')) else: if not notebook.endswith('.ipynb'): notebook = notebook + '.ipynb' notebook = os.path.abspath(os.path.join(full_path, notebook)) if not os.path.exists(notebook): raise FileNotFoundError('Notebook {0} not found.'.format(notebook)) notebooks = [notebook] if not notebooks: import warnings warnings.warn('No notebooks found', UserWarning) for nb in notebooks: nb_full_name = os.path.split(nb)[1] nb_name = os.path.splitext(nb_full_name)[0] py_name = nb_name + '.py' # Get base directory to notebook out_file = os.path.split(nb)[0] # Write to ../python out_file = os.path.join(out_file, '..', 'python', py_name) if is_newer(out_file, nb) and not force: logger.info('Skipping {0}, exported version newer than ' 'notebook'.format(nb_name)) continue logger.info('Converting {0}'.format(nb_name)) with open(nb, 'r') as nb_file: converter = nbconvert.PythonExporter() python = converter.from_file(nb_file) code = python[0].split('\n') code_out = [] for i, block in enumerate(code): if 'get_ipython' in block: continue elif block.startswith('# In[ ]:'): continue if block.startswith('#'): # Wrap comments from Markdown block = textwrap.fill(block, width=74) block = block.replace('\n', '\n# ') code_out.append(block) if not code_out[0]: code_out = code_out[1:] loc = 0 if 'coding' in code_out[0]: loc = 1 code_out.insert(loc, DO_NOT_EDIT.format(notebook=nb_full_name)) code_out = '\n'.join(code_out) code_out, success = FormatCode(code_out, style_config='pep8') with open(out_file, 'w') as of: of.write(code_out)
def runtest(self): with io.open(self.name, encoding='utf8') as nbfile: nb = nbformat.read(nbfile, as_version=4) magics_processor = magics.Processor( extra_line_blacklist=_get_list_from_conf( 'nbsmoke_flakes_line_magics_blacklist', self.parent.parent.config), extra_cell_blacklist=_get_list_from_conf( 'nbsmoke_flakes_cell_magics_blacklist', self.parent.parent.config)) magics_processor.insert_get_ipython(nb) ipy, _ = nbconvert.PythonExporter().from_notebook_node(nb) debug = self.config.option.nbsmoke_lint_debug filenames = [] self._write_debug_file(debug, ipy, self.name, "pre", filenames) py = magics_processor.ipython_to_python_for_flake_checks(ipy) self._write_debug_file(debug, py, self.name, "post", filenames) flake_result = _pyflakes.flake_check( # notebooks will start with "coding: utf-8", but py already unicode py.encode('utf8') if sys.version_info[0] == 2 else py, self.name) ### remove flakes by regex ### _user_flakes_to_ignore = self.parent.parent.config.getini( 'nbsmoke_flakes_to_ignore') if _user_flakes_to_ignore != '': _user_flakes_to_ignore = _user_flakes_to_ignore.splitlines() for pattern in set(flakes_to_ignore) | set(_user_flakes_to_ignore): flake_result['messages'] = [ msg for msg in flake_result['messages'] if not re.search(pattern, msg) ] ############################## if flake_result['messages']: msg = "%s\n** " % self.name if self.config.option.nbsmoke_lint_onlywarn and 'message_for_onlywarn' in flake_result: msg += flake_result['message_for_onlywarn'] else: msg += "\n** ".join(flake_result['messages']) msg += "\n" + "To see python source that was checked by pyflakes: %s" % filenames[ 1] msg += "\n" + "To see python source before magics were handled by nbsmoke: %s" % filenames[ 0] if self.config.option.nbsmoke_lint_onlywarn: warnings.warn("Flakes (or error) detected:\n" + msg + "\n") else: raise NBLintError(msg)
def export_notebook_to_tar_gz(notebook_file, output_filename, converted_filename=DEFAULT_CONVERTED_FILENAME): if notebook_file is None: notebook_file = get_notebook_name() exporter = nbconvert.PythonExporter() contents, _ = exporter.from_filename(notebook_file) with open(converted_filename, "w+") as f: f.write(contents) generate_context_tarball(converted_filename, output_filename)
def runtest(self): with io.open(self.name, encoding='utf8') as nbfile: nb = nbformat.read(nbfile, as_version=4) _insert_get_ipython(nb) py, resources = nbconvert.PythonExporter().from_notebook_node(nb) py = insert_ipython_magic_content(py) if sys.version_info[0] == 2: # notebooks will start with "coding: utf-8", but py already unicode py = py.encode('utf8') if flake_check(py, self.name) != 0: raise AssertionError
def _notebook_to_py(notebook): exporter = nbconvert.PythonExporter() exporter.exclude_markdown = True exporter.exclude_output = True exporter.exclude_raw = True exporter.exclude_unknown = True exporter.exclude_header = True body, resources = exporter.from_file(notebook) # Remove magic commands. body = body.replace("get_ipython()", "# get_ipython()") # The last two elements are a lint added by nbconvert (extra lines). return body[:-2]
def _reformat_notebook(script): """convert a notebook to a script and remove jupyter magics Parameters ---------- script : str The path to the Jupyter notebook Returns ------- code : str The notebook as a python script. """ nb = nbformat.read(script, nbformat.NO_CONVERT) code = nbconvert.export(nbconvert.PythonExporter(), nb)[0] new_code = [] for line in code.split('\n'): # Exclude magics - HACK - TODO # Matthew R. Becker 2/27/2017 # So this one is tough. The following magics # # %matplotlib inline # %matplotlib notebook # # are only available in jupyter. # When notebooks are converted to scripts currently (2/17/2017 w/ # version 5.1.1 of nbconvert), these magics are left in the notebook # as normal ipython magics. However, they cannot be executed by # ipython and cause an error. There is an open issue in nbconvert to # exclude these things # (see https://github.com/jupyter/nbconvert/pull/507). Until this # issue is addressed, we have to parse them out by hand. # The block of code here does the following. # 1) finds jupyter magics at any indentation level (they CAN be # indented!) # 2) replaces them with `pass` in python (which does nothing) at the # same indentation level if not (line.strip().startswith("get_ipython().magic(" "'matplotlib") or line.strip().startswith("get_ipython().run_line_magic(" "'matplotlib") or # python 2 - ugh line.strip().startswith("get_ipython().magic(" "u'matplotlib") or line.strip().startswith("get_ipython().run_line_magic(" "u'matplotlib")): new_code.append(line) else: new_code.append(line[:-len(line.lstrip())] + 'pass') return '\n'.join(new_code)
def export_to_python(filename=None, preprocessors=[OptsMagicProcessor(), OutputMagicProcessor(), StripMagicsProcessor()]): filename = filename if filename else sys.argv[1] with open(filename) as f: nb = nbformat.read(f, nbformat.NO_CONVERT) exporter = nbconvert.PythonExporter() for preprocessor in preprocessors: exporter.register_preprocessor(preprocessor) source, meta = exporter.from_notebook_node(nb) return source
def preprocess(self): exporter = nbconvert.PythonExporter() exporter.register_preprocessor(self.notebook_preprocessor, enabled=True) contents, _ = exporter.from_filename(self.notebook_file) converted_notebook = Path(self.notebook_file).with_suffix('.py') if converted_notebook.exists() and not self.overwrite: raise Exception('Default path {} exists but overwrite flag\ is False'.format(converted_notebook)) with open(converted_notebook, 'w') as f: logging.info('Converting {} to {}'.format(self.notebook_file, converted_notebook)) f.write(contents) self.executable = converted_notebook return [converted_notebook]
def preprocess(self): exporter = nbconvert.PythonExporter() exporter.register_preprocessor(self.notebook_preprocessor, enabled=True) contents, _ = exporter.from_filename(self.notebook_file) converted_notebook = Path(self.notebook_file).with_suffix('.py') if converted_notebook.exists(): dir_name = Path(self.notebook_file).parent _, file_name = tempfile.mkstemp(suffix=".py", dir=dir_name) converted_notebook = Path(file_name[file_name.find(str(dir_name)):]) with open(converted_notebook, 'w') as f: f.write(contents) self.executable = converted_notebook return [converted_notebook]
def test_example_notebooks(example, examples_directory, monkeypatch): from solcore.spice import SpiceSolverError from solcore.light_source import SmartsSolverError from solcore.absorption_calculator import RCWASolverError import nbconvert import nbformat import sys notebooks_folder = examples_directory / "notebooks" os.chdir(notebooks_folder) monkeypatch.setattr("builtins.input", lambda *x, **y: "y") script = str(notebooks_folder / Path(example)) exporter = nbconvert.PythonExporter() try: nbcode = nbformat.reads(open(script).read(), as_version=4) (code, _) = exporter.from_notebook_node(nbcode) with open(script) as fp: code = compile(code, script, "exec") # try to emulate __main__ namespace as much as possible # What trace python module uses https://docs.python.org/3.7/library/trace.html globs = { "__file__": script, "__name__": "__main__", "__package__": None, "__cached__": None, } exec(code, globs, globs) os.chdir(cwd) except OSError as err: os.chdir(cwd) skip("Cannot run file %r because: %s" % (sys.argv[0], err)) except SystemExit: os.chdir(cwd) pass except SpiceSolverError: os.chdir(cwd) skip("No SPICE solver found.") except SmartsSolverError: os.chdir(cwd) skip("No SMARTS solver found.") except RCWASolverError: os.chdir(cwd) skip("No RCWA solver found.")
def extract_py_notebook(in_file: Path) -> str: """Extract the python code from a given notebook""" # we use config for importing c = TConfig() c.PythonExporter.preprocessors = [] # load the notebook notebook: nbformat.NotebookNode = nbformat.read(str(in_file), as_version=4) # TODO - any preprocessing here # convert it conv = nbconvert.PythonExporter(config=c) (body, resources) = conv.from_notebook_node(notebook) # (body, resources) = conv.from_filename(str(in_file)) # write the notebook # writer = nbconvert.writers.FilesWriter() # writer.write(body, resources, "output_notebook") return body
def overwrite_module_from_notebook(): """Overwrite graphql_example.py from same-named notebook""" nbfilename = pkg_resources.resource_filename('graphql_example', 'graphql_example.ipynb') notebook = nbformat.read(nbfilename, as_version=4) exporter = nbconvert.PythonExporter() body, *_ = exporter.from_notebook_node(notebook) module_filename = pkg_resources.resource_filename('graphql_example', 'graphql_example.py') with open(module_filename, 'w') as main_module: main_module.write(body)
def run_test(project, zone, cluster, new_values): # pylint: disable=too-many-locals # TODO([email protected]): Need to configure the notebook and test to build # using GCB. dirname = os.path.dirname(__file__) if not dirname: logging.info("__file__ doesn't apper to be absolute path.") dirname = os.getcwd() notebook_path = os.path.join(dirname, "TF on GKE.ipynb") logging.info("Reading notebook %s", notebook_path) if not os.path.exists(notebook_path): raise ValueError("%s does not exist" % notebook_path) with open(notebook_path) as hf: node = nbformat.read(hf, nbformat.NO_CONVERT) exporter = nbconvert.PythonExporter() raw, _ = nbconvert.export(exporter, node) credentials = GoogleCredentials.get_application_default() gke = discovery.build("container", "v1", credentials=credentials) lines = raw.splitlines() modified = replace_vars(lines, new_values) modified = strip_appendix(modified) modified = strip_unexecutable(modified) with tempfile.NamedTemporaryFile(suffix="notebook.py", prefix="tmpGke", mode="w", delete=False) as hf: code_path = hf.name hf.write("\n".join(modified)) logging.info("Wrote notebook to: %s", code_path) with open(code_path) as hf: contents = hf.read() logging.info("Notebook contents:\n%s", contents) try: runpy.run_path(code_path) finally: logging.info("Deleting cluster; project=%s, zone=%s, name=%s", project, zone, cluster) util.delete_cluster(gke, cluster, project, zone)
def preprocess(self): """Preprocessor the Notebook. :return: results: the preprocessed notebook list. """ exporter = nbconvert.PythonExporter() exporter.register_preprocessor(self.notebook_preprocessor, enabled=True) processed, _ = exporter.from_filename(self.notebook_file) lines = [] for l in processed.splitlines(): # Get rid of multiple blank lines if not l.strip(): if lines: if not lines[-1]: # last line is already blank don't add another one continue # strip in statements if l.startswith("# In["): continue lines.append(l) contents = "\n".join(lines) converted_notebook = Path(self.notebook_file).with_suffix('.py') if converted_notebook.exists() and not self.overwrite: raise Exception('Default path {} exists but overwrite flag\ is False'.format(converted_notebook)) with open(converted_notebook, 'w') as f: logging.info('Converting {} to {}'.format(self.notebook_file, converted_notebook)) f.write(contents) f.write("\n") logging.info('Creating entry point for the class name {}'.format( self.class_name)) f.write(""" if __name__ == "__main__": import fire import logging logging.basicConfig(format='%(message)s') logging.getLogger().setLevel(logging.INFO) fire.Fire({0}) """.format(self.class_name)) self.executable = converted_notebook results = [converted_notebook] results.extend(self.input_files) return results
def export_for_spyder(nb): for i, cell in enumerate(nb["cells"]): if cell["cell_type"] in ("markdown", "code"): cell["source"] = "#%%\n" + cell["source"] exporter = nbconvert.PythonExporter() exporter.exclude_input_prompt = True exporter.exclude_output_prompt = True exporter.exclude_output = True py = exporter.from_notebook_node(nb) code = py[0].split("\n") code = [line if line != "# #%%" else "#%%" for line in code] imported_get_python = False for i, line in enumerate(code): if line.startswith("get_ipython()") and not imported_get_python: code.insert(i, "from IPython import get_ipython") imported_get_python = True return "\n".join(code)
def notebook_run(ipynb): with open(ipynb) as fh: nb = nbformat.reads(fh.read(), 4) exporter = nbconvert.PythonExporter() try: os.mkdir('figure') except: pass # source is a tuple of python source code # meta contains metadata source, meta = exporter.from_notebook_node(nb) try: exec(source.encode()) plt.close('all') shutil.rmtree('figure', ignore_errors=True) return [] except: shutil.rmtree('figure', ignore_errors=True) return traceback.print_exc()
def _load_ipynb_modules(ipynb_path: str): """ First converts data_prep.ipynb to a temp .py file Then it loads the functions from the .py file Returns the loaded modules """ assert ipynb_path.endswith(".ipynb") basename, _ = os.path.splitext(ipynb_path) # read ipynb file and get the text with open(ipynb_path) as ipynb_file: nb = nbformat.reads(s=ipynb_file.read(), as_version=nbformat.NO_CONVERT) assert isinstance(nb, nbformat.notebooknode.NotebookNode) # convert the .ipynb text to a string .py format pyexporter = nbconvert.PythonExporter() source, meta = pyexporter.from_notebook_node(nb=nb) assert isinstance(source, str) # parse the .py string to pick out only 'class', 'import' and 'def function's parsed_code = ast.parse(source=source) for node in parsed_code.body[:]: if node.__class__ not in [ ast.ClassDef, ast.FunctionDef, ast.Import, ast.ImportFrom, ]: parsed_code.body.remove(node) assert parsed_code.body # make sure there is a non-empty list # import modules from the parsed .py string module = types.ModuleType(basename) code = compile(source=parsed_code, filename=f"{basename}_ipynb.py", mode="exec") exec(code, module.__dict__) return module
def preprocess(self): exporter = nbconvert.PythonExporter() exporter.register_preprocessor(self.notebook_preprocessor, enabled=True) processed, _ = exporter.from_filename(self.notebook_file) lines = [] for l in processed.splitlines(): # Get rid of multiple blank lines if not l.strip(): if lines: if not lines[-1]: # last line is already blank don't add another one continue # strip in statements if l.startswith("# In["): continue lines.append(l) contents = "\n".join(lines) converted_notebook = Path(self.notebook_file).with_suffix('.py') if converted_notebook.exists(): dir_name = Path(self.notebook_file).parent _, file_name = tempfile.mkstemp(suffix=".py", dir=dir_name) converted_notebook = Path(file_name[file_name.find(str(dir_name)):]) with open(converted_notebook, 'w') as f: f.write(contents) f.write("\n") f.write(""" if __name__ == "__main__": import fire import logging logging.basicConfig(format='%(message)s') logging.getLogger().setLevel(logging.INFO) fire.Fire({0}) """.format(self.class_name)) self.executable = converted_notebook results = [converted_notebook] results.extend(self.input_files) return results
def export_for_spyder(nb): for i, cell in enumerate(nb["cells"]): if cell["cell_type"] in ("markdown", "code"): cell["source"] = "#%%\n" + cell["source"] exporter = nbconvert.PythonExporter() exporter.exclude_input_prompt = True exporter.exclude_output_prompt = True exporter.exclude_output = True py = exporter.from_notebook_node(nb) code = py[0].split("\n") code = [line if line != "# #%%" else "#%%" for line in code] imported_get_python = False for i, line in enumerate(code): if line.startswith("get_ipython()") and not imported_get_python: code.insert(i, "from IPython import get_ipython") imported_get_python = True mode = bm.Mode([bm.TargetVersion.PY38, bm.TargetVersion.PY39]) code = black.format_file_contents("\n".join(code), mode=mode, fast=False) code = isort.api.sort_code_string(code) return code