def test_tutorials(testfolder, tutorials=('ligand-binding-analysis', 'protein-folding-analysis')): startdir = os.path.dirname(os.path.realpath(__file__)) pyexp = PythonExporter() pyexp.template_file = os.path.join(startdir, 'simplepython.tpl') # For each tutorial noteboook for tut in glob(os.path.join(startdir, '*.ipynb')): name = os.path.splitext(os.path.basename(tut))[0] if name not in tutorials: continue testsubf = os.path.abspath(os.path.join(testfolder, name)) if not os.path.exists(testsubf): os.makedirs(testsubf) # Read notebook in with open(tut, 'r') as fi: notebook = nbformat.read(fi, nbformat.NO_CONVERT) output, resources = pyexp.from_notebook_node(notebook, ) # Write it out in .py format with open(os.path.join(testsubf, 'test_{}.py'.format(name)), 'w') as fo: fo.writelines(format_script(output, testsubf, name)) return pytest.main([testfolder])
def tokenize_lambda_func(val, within, notebook_name=None): if notebook_name: notebook_path = os.getcwd() + "/{}.ipynb".format(notebook_name) with open(notebook_path, 'r') as f: notebook = nbformat.reads(f.read(), nbformat.NO_CONVERT) exporter = PythonExporter() source, _ = exporter.from_notebook_node(notebook) readline = io.StringIO(source).readline else: readline = open(val.__code__.co_filename).readline first_line = val.__code__.co_firstlineno flag = False source = "" for t in tokenize.generate_tokens(readline): t_type, t_string, (r_start, c_start), (r_end, c_end), line = t t_name = token.tok_name[t_type] if r_start == first_line: if t_name == 'NAME' and t_string == within: flag = True res = t_string start = 0 # number of parenthesis continue if flag: source += ' ' + t_string if t_name == 'OP': if t_string == '(': start += 1 elif t_string == ')': start -= 1 if start == 0: break return source.strip()
def locals_from_notebook(notebook_ipynb): """Read, run, return locals of notebook""" banned_commands = ['HTML'] ipynb = json.load(open(notebook_ipynb)) for cell in ipynb['cells']: # erase test cells, this is optional and useful for debugging # to avoid recursions when developing if any('## TEST ##' in line for line in cell['source']): cell['source'] = [] # filter out all the lines with banned commands if banned_commands is not None: cell['source'] = [ line for line in cell['source'] if not any(command in line for command in banned_commands) ] exporter = PythonExporter() source, meta = exporter.from_notebook_node( nbformat.reads(json.dumps(ipynb), nbformat.NO_CONVERT)) with open('./cleaned_notebook.py', 'w') as fh: fh.write(source) notebook_locals = run_path('cleaned_notebook.py') os.system('rm cleaned_notebook.py') return notebook_locals
def parse_ipynb(file: Path) -> str: with open(file, "r") as nb_file: nb_str = nb_file.read() nb = nbformat.reads(nb_str, nbformat.NO_CONVERT) exporter = PythonExporter() script, _ = exporter.from_notebook_node(nb) return script
def convertNotebook(notebookPath, modulePath): with open(notebookPath) as fh: nb = nbformat.reads(fh.read(), nbformat.NO_CONVERT) exporter = PythonExporter() source, meta = exporter.from_notebook_node(nb) with open(modulePath, 'w+') as fh: fh.writelines(source) # .encode('utf-8'))
def test_run_tutorial(): """ There are a lot of examples in tutorial, and all of them should be working. Notes ----- IPython notebook is converted to Python script, so tqdm bars and plotting is removed. """ # pylint: disable=exec-used tutorials_dir = './../../examples/tutorials/' notebook = '09_solving_PDE_with_NN.ipynb' file = tutorials_dir + notebook with warnings.catch_warnings(): warnings.simplefilter("ignore") from nbconvert import PythonExporter code, _ = PythonExporter().from_filename(file) code_ = [] for line in code.split('\n'): if not line.startswith('#'): if not any(bs in line for bs in BAD_PREFIXES): code_.append(line.replace('in tqdm_notebook', 'in ')) code = '\n'.join(code_) exec(code, {})
def parse_ipynb_file(filename): """A simple parser for extracting the names of imported modules / packages from an IPython (Jupyter) notebook. Parameters ---------- filename : str Path to an IPython notebook file. Returns ------- packages : set A unique list of all (root) packages imported by the specified notebook file. """ import nbformat from nbconvert import PythonExporter with open(filename) as _file: nb_stuff = nbformat.reads(_file.read(), as_version=4) exporter = PythonExporter() (body, _) = exporter.from_notebook_node(nb_stuff) return parse_py_module(body)
def test_run_tutorial(path): """ There are a lot of examples in tutorial, and all of them should be working. Notes ----- IPython notebook is converted to Python script, so tqdm bars and plotting is removed. """ # pylint: disable=exec-used with warnings.catch_warnings(): warnings.simplefilter("ignore") from nbconvert import PythonExporter # pylint: disable=import-outside-toplevel code, _ = PythonExporter().from_filename(path) code_ = [] for line in code.split('\n'): if not line.startswith('#'): if not any(bs in line for bs in BAD_PREFIXES): line = line.replace("bar='notebook'", "bar=False") line = line.replace("in tqdm_notebook", "in ") code_.append(line) code = '\n'.join(code_) exec(code, {})
def test_notebooks(): """Run all notebooks in /docs/tutorials/ as tests.""" # Get the notebook names root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) path = os.path.join(root, 'docs', 'tutorials') notebooks = glob.glob(os.path.join(path, '*.ipynb')) # Convert them to python scripts exporter = PythonExporter() for notebook in notebooks: # Get the script as a string script, _ = exporter.from_filename(notebook) # Get rid of %matplotlib inline commands script = script.replace("get_ipython().magic('matplotlib inline')", "") script = script.replace( "get_ipython().run_line_magic('matplotlib', 'inline')", "") # Get rid of %run commands script = re.sub("get_ipython\(\).magic\('run (.*)'\)", r"#", script) script = re.sub("get_ipython\(\).run_line_magic\('run', '(.*)'\)", r"#", script) # Remove the %time wrappers script = re.sub("get_ipython\(\).magic\('time (.*)'\)", r"\1", script) script = re.sub("get_ipython\(\).run_line_magic\('time', '(.*)'\)", r"\1", script) # Remove calls to map.show() script = re.sub("(.*)\.show()(.*)", r"#", script) # Run it print("Running %s..." % os.path.basename(notebook)) exec(script, globals(), globals()) pl.close('all')
def read_source_script(filepath): """Read the contents of `filepath` Parameters ---------- filepath: Str Absolute path to a Python file. Expected to end with one of the following extensions: '.py', '.ipynb' Returns ------- source: Str The contents of `filepath`""" if filepath.endswith('.ipynb'): with open(filepath, 'r') as f: from nbconvert import PythonExporter import nbformat notebook = nbformat.reads(f.read(), nbformat.NO_CONVERT) exporter = PythonExporter() source, _ = exporter.from_notebook_node(notebook) else: with open(filepath, 'r') as f: source = f.read() return source
def get_hyperopt_model_string(model, data, functions, notebook_name, verbose, stack, data_args): model_string = inspect.getsource(model) model_string = remove_imports(model_string) if notebook_name: notebook_path = os.getcwd() + "/{}.ipynb".format(notebook_name) with open(notebook_path, 'r') as f: notebook = nbformat.reads(f.read(), nbformat.NO_CONVERT) exporter = PythonExporter() source, _ = exporter.from_notebook_node(notebook) else: calling_script_file = os.path.abspath(inspect.stack()[stack][1]) with open(calling_script_file, 'r') as f: source = f.read() cleaned_source = remove_all_comments(source) imports = extract_imports(cleaned_source, verbose) parts = hyperparameter_names(model_string) aug_parts = augmented_names(parts) hyperopt_params = get_hyperparameters(model_string) space = get_hyperopt_space(parts, hyperopt_params, verbose) functions_string = retrieve_function_string(functions, verbose) data_string = retrieve_data_string(data, verbose, data_args) model = hyperopt_keras_model(model_string, parts, aug_parts, verbose) temp_str = temp_string(imports, model, data_string, functions_string, space) return temp_str
def get_hyperopt_model_string(model, data, functions, notebook_name, verbose, stack): model_string = inspect.getsource(model) model_string = remove_imports(model_string) if notebook_name: notebook_path = os.getcwd() + "/{}.ipynb".format(notebook_name) with open(notebook_path, 'r') as f: notebook = nbformat.reads(f.read(), nbformat.NO_CONVERT) exporter = PythonExporter() source, _ = exporter.from_notebook_node(notebook) else: calling_script_file = os.path.abspath(inspect.stack()[stack][1]) with open(calling_script_file, 'r') as f: source = f.read() cleaned_source = remove_all_comments(source) imports = extract_imports(cleaned_source, verbose) parts = hyperparameter_names(model_string) aug_parts = augmented_names(parts) hyperopt_params = get_hyperparameters(model_string) space = get_hyperopt_space(parts, hyperopt_params, verbose) functions_string = retrieve_function_string(functions, verbose) data_string = retrieve_data_string(data, verbose) model = hyperopt_keras_model(model_string, parts, aug_parts, verbose) temp_str = temp_string(imports, model, data_string, functions_string, space) return temp_str
def convertNotebook(notebookPath): with open(notebookPath, encoding='UTF-8') as fh: nb = nbformat.reads(fh.read(), nbformat.NO_CONVERT) exporter = PythonExporter() source, meta = exporter.from_notebook_node(nb) return source
def update_script(name): python_exporter = PythonExporter() with open("experiments/{name}/{name}.ipynb".format(name=name)) as f: notebook = nbformat.reads(f.read(), 4) test = python_exporter.from_notebook_node(notebook) path = "experiments/{name}/{name}.py".format(name=name) with open(path, "w") as f_out: f_out.write(test[0]) return path
def convert_ipynb_to_py(notebook_path, output_path): with open(notebook_path) as fh: nb = nbformat.reads(fh.read(), nbformat.NO_CONVERT) exporter = PythonExporter() source, meta = exporter.from_notebook_node(nb) with open(output_path, 'w+') as fh: fh.writelines(source.encode('utf-8'))
def collect(self): exporter = PythonExporter() exporter.exclude_markdown = True exporter.exclude_input_prompt = True notebook_contents = self.fspath.open() notebook = nbformat.read(notebook_contents, 4) code, _ = exporter.from_notebook_node(notebook) yield IPyNbTest(self.name, self, code)
def convertNotebook(notebookPath): with open(notebookPath) as fh: nb = nbformat.reads(fh.read(), nbformat.NO_CONVERT) exporter = PythonExporter() source, meta = exporter.from_notebook_node(nb) lines = source.split("\n") for line in lines: print(line)
def __init__(self, *args, **kwargs): if 'filename' not in kwargs: raise ValueError('Must pass a filename to NotebookHandler') with open(kwargs['filename']) as f: nb = nbformat.read(f, nbformat.NO_CONVERT) exporter = PythonExporter() source, meta = exporter.from_notebook_node(nb) kwargs['source'] = source super(NotebookHandler, self).__init__(*args, **kwargs)
def convert_notebook(notebook_path, module_path): with open(notebook_path) as fh: nb = nbformat.reads(fh.read(), nbformat.NO_CONVERT) exporter = PythonExporter() source, _ = exporter.from_notebook_node(nb) with open(module_path, 'w+') as fh: fh.write(source) print("\t> Converted {} to {}".format(notebook_path, module_path))
def test_nb_pipeline_runs(): """ Tests whether the .ipynb version of the pipeline works """ with open(FILE_NB) as file: notebook = nbformat.reads(file.read(), nbformat.NO_CONVERT) exporter = PythonExporter() code, _ = exporter.from_notebook_node(notebook) parsed_ast = ast.parse(code) exec(compile(parsed_ast, filename="<ast>", mode="exec"))
def _get_pycode(cls, nb_node: str, removed_tags: Set[str] = set()) -> str: c = Config() c.TagRemovePreprocessor.remove_cell_tags = removed_tags c.TemplateExporter.exclude_input_prompt = True c.TemplateExporter.exclude_output_prompt = True c.TemplateExporter.exclude_markdown = True exporter = PythonExporter(config=c) code, _ = exporter.from_notebook_node(nb_node) code = re.sub('\n+', '\n', code) return code
def load_module(self, fullname): url = self.try_url(fullname) if fullname in sys.modules: return try: # The importer protocol requires the loader create a new module # object, set certain attributes on it, then add it to # `sys.modules` before executing the code inside the module (which # is when the "module" actually gets code inside it) m = types.ModuleType(fullname, 'remote notebook: {}'.format(fullname)) m.__file__ = '<notebook {}>'.format(fullname) m.__name__ = url m.__loader__ = self sys.modules[url] = m # Try to load notebook resp = urlopen(url) body = resp.read().decode() notebook = nbformat.reads(body, as_version=4) exporter = PythonExporter() (code, resources) = exporter.from_notebook_node(notebook) code = re.sub(r'%.+', "", code) tree = ast.parse(code) extractor = Extractor() extractor.visit(tree) exec(extractor.code(), sys.modules[url].__dict__) sys.modules[fullname] = sys.modules[url] return m except Exception as e: # If it fails, we need to reset sys.modules to it's old state. This # is good practice in general, but also a mandatory part of the # spec, likely to keep the import statement idempotent and free of # side-effects across imports. # Delete the entry we might've created; use LBYL to avoid nested # exception handling if sys.modules.get(url): del sys.modules[url] raise e if sys.modules.get(fullname): del sys.modules[fullname] raise e
def convert_nb(file_name): export_path = f'{os.getcwd()}/{os.path.split(file_name)[1][:-5]}py' print(export_path) with open(file_name, encoding="utf8") as fn: nb = nbformat.read(fn, as_version=4) exporter = PythonExporter() # source is a tuple of python source code # meta contains metadata source, meta = exporter.from_notebook_node(nb) with open(export_path, 'w+', encoding="utf8") as fh: fh.writelines(source)
def save_tasks_py(nb): """Select cells appropriate for `tasks.py`.""" shouldnt_start_with = '# Put in a file named run-readout-scan.py' tasks_py = nb.copy() tasks_py.cells = [ cell for cell in tasks_py.cells if not cell.source.startswith(shouldnt_start_with) ] exporter = PythonExporter() exporter.raw_template = TEMPLATE output, resources = exporter.from_notebook_node(tasks_py) with open(f'{REPO_DIR}/recirq/readout_scan/tasks.py', 'w') as f: f.write(output)
def collect(self): exporter = PythonExporter() exporter.exclude_markdown = True exporter.exclude_input_prompt = True notebook_contents = self.fspath.open(encoding='utf-8') with warnings.catch_warnings(): warnings.filterwarnings("ignore", "IPython.core.inputsplitter is deprecated") notebook = nbformat.read(notebook_contents, 4) code, _ = exporter.from_notebook_node(notebook) if pytest_version >= 50403: yield IPyNbTest.from_parent(name=self.name, parent=self, code=code) else: yield IPyNbTest(self.name, self, code)
def convert_nb2inject() -> str: ''' Converts the current notebook to an executable .py file Returns ------- str : the full path and filename of the converted file ''' try: import ipykernel import notebook.notebookapp except ImportError as e: #log.exception('ImportError : This only runs in a Jupyter Notebook environment ' + str(e)) return from nbconvert import PythonExporter from nbconvert.writers import FilesWriter import nbformat #from traitlets.config import Config exporter = PythonExporter() nbfile = get_notebook_name() nb = nbformat.read(nbfile, nbformat.NO_CONVERT) start = 0 # Don't use the last cell nb.cells = nb.cells[start:-2] (output, resources) = exporter.from_notebook_node(nb) filename = nbfile.split('/')[-1].split('.')[0] user = os.getenv("JUPYTERHUB_USER") # FIX: This should not be hardcoded tmpdir = f"/tmp/cloudflow/inject/{user}" if not os.path.exists(tmpdir): os.makedirs(tmpdir) outfile = f"{tmpdir}/{filename}" # Save to file writer = FilesWriter() writer.write(output, resources, outfile) return outfile + ".py"
def write_files(export_list, nb_node, file_name): """ Export and write files from a notebook node. Args: export_list (list of strings) -- name of valid nbconvert exporters nb_node(nbformat node object) -- notebook to be my_converted file_name (str) -- base name of file to be written Returns: None """ try: # export and write file. for export in export_list: if export == "html": exporter = HTMLExporter() elif export == "py": exporter = PythonExporter() (body, resources) = exporter.from_notebook_node(nb_node) write_file = FilesWriter() write_file.write( output=body, resources=resources, notebook_name=file_name ) except Exception as e: print(f"There was a problem exporting the file(s): {e}") return None
def _nb_sample_to_py(notebook_path: str, output_path: str): """nb_sample_to_py converts notebook kfp sample to a python file. Cells with tag "skip-in-test" will be omitted.""" with open(notebook_path, 'r') as f: nb = nbformat.read(f, as_version=4) # Cells with skip-in-test tag will be omitted. # Example code that needs the tag: # kfp.Client().create_run_from_pipeline_func() # so that we won't submit pipelines when compiling them. nb.cells = [ cell for cell in nb.cells if 'skip-in-test' not in cell.get('metadata', {}).get('tags', []) ] py_exporter = PythonExporter() (py_code, res) = py_exporter.from_notebook_node(nb) with open(output_path, 'w') as out: out.write(py_code)
def convert_notebook(name, notebook, output_dir): c = Config() c.ExtractOutputPreprocessor.output_filename_template = name + '/{unique_key}_{cell_index}_{index}{extension}' md_exporter = MarkdownExporter(config=c) py_exporter = PythonExporter() (body, _) = py_exporter.from_notebook_node(notebook) with open(output_dir / f'{name}.py', 'w') as fp: fp.write(body) (body, resources) = md_exporter.from_notebook_node(notebook) with open(output_dir / f'{name}.md', 'w') as fp: fp.write(post_process_markdown(body)) for resource, value in resources['outputs'].items(): resource_path = output_dir / resource resource_path.parent.mkdir(exist_ok=True, parents=True) with open(resource_path, 'wb') as fp: fp.write(value)
def _get_kaggle_notebook_content(): """Returns the kaggle notebook python code contents.""" if PythonExporter is None: raise RuntimeError( # This should never occur. # `nbconvert` is always installed on Kaggle. "Please make sure you have installed `nbconvert` package.") from kaggle_session import UserSessionClient # pylint: disable=g-import-not-at-top # pytype: disable=import-error kaggle_session_client = UserSessionClient() try: response = kaggle_session_client.get_exportable_ipynb() ipynb_stream = io.StringIO(response["source"]) py_content, _ = PythonExporter().from_file(ipynb_stream) return py_content.splitlines(keepends=True) except: raise RuntimeError("Unable to get the notebook contents.")
def load_source_code(notebook_path, python_path, python_code): """ Load the pipeline source code from the specified source """ source_code = "" sources = [notebook_path, python_path, python_code] assert sum(source is not None for source in sources) == 1 if python_path is not None: with open(python_path) as file: source_code = file.read() elif notebook_path is not None: with open(notebook_path) as file: notebook = nbformat.reads(file.read(), nbformat.NO_CONVERT) exporter = PythonExporter() source_code, _ = exporter.from_notebook_node(notebook) elif python_code is not None: source_code = python_code return source_code
def convert_notebook_to_py(nb_fn: Path, py_fn: Path) -> None: """ From https://stackoverflow.com/questions/17077494/how-do-i-convert-a-ipython-notebook-into-a-python-file-via-commandline """ import nbformat from nbconvert import PythonExporter with open(nb_fn) as fh: nb = nbformat.reads(fh.read(), nbformat.NO_CONVERT) exporter = PythonExporter() source, meta = exporter.from_notebook_node(nb) # Skip the magic, which gets converted to `get_ipython()` source.replace("get_ipython", "# get_ipython") with open(py_fn, "w+") as fh: fh.writelines(source)
def convert_notebook(notebook_path): write_folder = cache.get_parent_folder(notebook_path) file_name = cache.get_file_name(notebook_path) write_file = os.path.join(write_folder, "%s.%s" % (file_name, props.TYPE_PYTHON)) if cache.file_exists(write_file): LOGGER.info("'%s' already converted. Moving on ... " % file_name) return else: LOGGER.info("Converting filename '%s' ... " % file_name) with open(notebook_path) as fh: nb = nbformat.reads(fh.read(), nbformat.NO_CONVERT) exporter = PythonExporter() try: source, meta = exporter.from_notebook_node(nb) except nbformat.validator.NotebookValidationError: LOGGER.error("Validation error while converting '%s'." % notebook_path) return with open(write_file, 'w+') as fh: fh.writelines(source.encode('utf-8'))
def export_python(wd, name): nb = _read(wd, name) exporter = PythonExporter() body, resources = exporter.from_notebook_node(nb) with open("{}/{}.py".format(wd, name), 'w') as f: f.write(body)
def convert_to_python(notebook, template_file=None): exporter = PythonExporter() exporter.template_file = template_file notebook_code, meta = exporter.from_notebook_node(notebook) return notebook_code, meta
def export_python(nb, destfn): exporter = PythonExporter() body, resources = exporter.from_notebook_node(nb) with open(destfn, 'w') as f: f.write(body)