def tokenize_lambda_func(val, within, notebook_name=None): if notebook_name: notebook_path = os.getcwd() + "/{}.ipynb".format(notebook_name) with open(notebook_path, 'r') as f: notebook = nbformat.reads(f.read(), nbformat.NO_CONVERT) exporter = PythonExporter() source, _ = exporter.from_notebook_node(notebook) readline = io.StringIO(source).readline else: readline = open(val.__code__.co_filename).readline first_line = val.__code__.co_firstlineno flag = False source = "" for t in tokenize.generate_tokens(readline): t_type, t_string, (r_start, c_start), (r_end, c_end), line = t t_name = token.tok_name[t_type] if r_start == first_line: if t_name == 'NAME' and t_string == within: flag = True res = t_string start = 0 # number of parenthesis continue if flag: source += ' ' + t_string if t_name == 'OP': if t_string == '(': start += 1 elif t_string == ')': start -= 1 if start == 0: break return source.strip()
def get_hyperopt_model_string(model, data, functions, notebook_name, verbose, stack): model_string = inspect.getsource(model) model_string = remove_imports(model_string) if notebook_name: notebook_path = os.getcwd() + "/{}.ipynb".format(notebook_name) with open(notebook_path, 'r') as f: notebook = nbformat.reads(f.read(), nbformat.NO_CONVERT) exporter = PythonExporter() source, _ = exporter.from_notebook_node(notebook) else: calling_script_file = os.path.abspath(inspect.stack()[stack][1]) with open(calling_script_file, 'r') as f: source = f.read() cleaned_source = remove_all_comments(source) imports = extract_imports(cleaned_source, verbose) parts = hyperparameter_names(model_string) aug_parts = augmented_names(parts) hyperopt_params = get_hyperparameters(model_string) space = get_hyperopt_space(parts, hyperopt_params, verbose) functions_string = retrieve_function_string(functions, verbose) data_string = retrieve_data_string(data, verbose) model = hyperopt_keras_model(model_string, parts, aug_parts, verbose) temp_str = temp_string(imports, model, data_string, functions_string, space) return temp_str
def locals_from_notebook(notebook_ipynb): """Read, run, return locals of notebook""" banned_commands = ['HTML'] ipynb = json.load(open(notebook_ipynb)) for cell in ipynb['cells']: # erase test cells, this is optional and useful for debugging # to avoid recursions when developing if any('## TEST ##' in line for line in cell['source']): cell['source'] = [] # filter out all the lines with banned commands if banned_commands is not None: cell['source'] = [ line for line in cell['source'] if not any(command in line for command in banned_commands) ] exporter = PythonExporter() source, meta = exporter.from_notebook_node( nbformat.reads(json.dumps(ipynb), nbformat.NO_CONVERT)) with open('./cleaned_notebook.py', 'w') as fh: fh.write(source) notebook_locals = run_path('cleaned_notebook.py') os.system('rm cleaned_notebook.py') return notebook_locals
def convertNotebook(notebookPath, modulePath): with open(notebookPath) as fh: nb = nbformat.reads(fh.read(), nbformat.NO_CONVERT) exporter = PythonExporter() source, meta = exporter.from_notebook_node(nb) with open(modulePath, 'w+') as fh: fh.writelines(source) # .encode('utf-8'))
def parse_ipynb_file(filename): """A simple parser for extracting the names of imported modules / packages from an IPython (Jupyter) notebook. Parameters ---------- filename : str Path to an IPython notebook file. Returns ------- packages : set A unique list of all (root) packages imported by the specified notebook file. """ import nbformat from nbconvert import PythonExporter with open(filename) as _file: nb_stuff = nbformat.reads(_file.read(), as_version=4) exporter = PythonExporter() (body, _) = exporter.from_notebook_node(nb_stuff) return parse_py_module(body)
def parse_ipynb(file: Path) -> str: with open(file, "r") as nb_file: nb_str = nb_file.read() nb = nbformat.reads(nb_str, nbformat.NO_CONVERT) exporter = PythonExporter() script, _ = exporter.from_notebook_node(nb) return script
def read_source_script(filepath): """Read the contents of `filepath` Parameters ---------- filepath: Str Absolute path to a Python file. Expected to end with one of the following extensions: '.py', '.ipynb' Returns ------- source: Str The contents of `filepath`""" if filepath.endswith('.ipynb'): with open(filepath, 'r') as f: from nbconvert import PythonExporter import nbformat notebook = nbformat.reads(f.read(), nbformat.NO_CONVERT) exporter = PythonExporter() source, _ = exporter.from_notebook_node(notebook) else: with open(filepath, 'r') as f: source = f.read() return source
def test_tutorials(testfolder, tutorials=('ligand-binding-analysis', 'protein-folding-analysis')): startdir = os.path.dirname(os.path.realpath(__file__)) pyexp = PythonExporter() pyexp.template_file = os.path.join(startdir, 'simplepython.tpl') # For each tutorial noteboook for tut in glob(os.path.join(startdir, '*.ipynb')): name = os.path.splitext(os.path.basename(tut))[0] if name not in tutorials: continue testsubf = os.path.abspath(os.path.join(testfolder, name)) if not os.path.exists(testsubf): os.makedirs(testsubf) # Read notebook in with open(tut, 'r') as fi: notebook = nbformat.read(fi, nbformat.NO_CONVERT) output, resources = pyexp.from_notebook_node(notebook, ) # Write it out in .py format with open(os.path.join(testsubf, 'test_{}.py'.format(name)), 'w') as fo: fo.writelines(format_script(output, testsubf, name)) return pytest.main([testfolder])
def test_tutorials(testfolder, tutorials=('ligand-binding-analysis', 'protein-folding-analysis')): startdir = os.path.dirname(os.path.realpath(__file__)) pyexp = PythonExporter() pyexp.template_file = os.path.join(startdir, 'simplepython.tpl') # For each tutorial noteboook for tut in glob(os.path.join(startdir, '*.ipynb')): name = os.path.splitext(os.path.basename(tut))[0] if name not in tutorials: continue testsubf = os.path.abspath(os.path.join(testfolder, name)) if not os.path.exists(testsubf): os.makedirs(testsubf) # Read notebook in with open(tut, 'r') as fi: notebook = nbformat.read(fi, nbformat.NO_CONVERT) output, resources = pyexp.from_notebook_node(notebook, ) # Write it out in .py format with open(os.path.join(testsubf, 'test_{}.py'.format(name)), 'w') as fo: fo.writelines(format_script(output, testsubf, name)) return pytest.main([testfolder])
def get_hyperopt_model_string(model, data, functions, notebook_name, verbose, stack, data_args): model_string = inspect.getsource(model) model_string = remove_imports(model_string) if notebook_name: notebook_path = os.getcwd() + "/{}.ipynb".format(notebook_name) with open(notebook_path, 'r') as f: notebook = nbformat.reads(f.read(), nbformat.NO_CONVERT) exporter = PythonExporter() source, _ = exporter.from_notebook_node(notebook) else: calling_script_file = os.path.abspath(inspect.stack()[stack][1]) with open(calling_script_file, 'r') as f: source = f.read() cleaned_source = remove_all_comments(source) imports = extract_imports(cleaned_source, verbose) parts = hyperparameter_names(model_string) aug_parts = augmented_names(parts) hyperopt_params = get_hyperparameters(model_string) space = get_hyperopt_space(parts, hyperopt_params, verbose) functions_string = retrieve_function_string(functions, verbose) data_string = retrieve_data_string(data, verbose, data_args) model = hyperopt_keras_model(model_string, parts, aug_parts, verbose) temp_str = temp_string(imports, model, data_string, functions_string, space) return temp_str
def convertNotebook(notebookPath): with open(notebookPath, encoding='UTF-8') as fh: nb = nbformat.reads(fh.read(), nbformat.NO_CONVERT) exporter = PythonExporter() source, meta = exporter.from_notebook_node(nb) return source
def update_script(name): python_exporter = PythonExporter() with open("experiments/{name}/{name}.ipynb".format(name=name)) as f: notebook = nbformat.reads(f.read(), 4) test = python_exporter.from_notebook_node(notebook) path = "experiments/{name}/{name}.py".format(name=name) with open(path, "w") as f_out: f_out.write(test[0]) return path
def convertNotebook(notebookPath): with open(notebookPath) as fh: nb = nbformat.reads(fh.read(), nbformat.NO_CONVERT) exporter = PythonExporter() source, meta = exporter.from_notebook_node(nb) lines = source.split("\n") for line in lines: print(line)
def collect(self): exporter = PythonExporter() exporter.exclude_markdown = True exporter.exclude_input_prompt = True notebook_contents = self.fspath.open() notebook = nbformat.read(notebook_contents, 4) code, _ = exporter.from_notebook_node(notebook) yield IPyNbTest(self.name, self, code)
def convert_ipynb_to_py(notebook_path, output_path): with open(notebook_path) as fh: nb = nbformat.reads(fh.read(), nbformat.NO_CONVERT) exporter = PythonExporter() source, meta = exporter.from_notebook_node(nb) with open(output_path, 'w+') as fh: fh.writelines(source.encode('utf-8'))
def convert_notebook(notebook_path, module_path): with open(notebook_path) as fh: nb = nbformat.reads(fh.read(), nbformat.NO_CONVERT) exporter = PythonExporter() source, _ = exporter.from_notebook_node(nb) with open(module_path, 'w+') as fh: fh.write(source) print("\t> Converted {} to {}".format(notebook_path, module_path))
def test_nb_pipeline_runs(): """ Tests whether the .ipynb version of the pipeline works """ with open(FILE_NB) as file: notebook = nbformat.reads(file.read(), nbformat.NO_CONVERT) exporter = PythonExporter() code, _ = exporter.from_notebook_node(notebook) parsed_ast = ast.parse(code) exec(compile(parsed_ast, filename="<ast>", mode="exec"))
def _get_pycode(cls, nb_node: str, removed_tags: Set[str] = set()) -> str: c = Config() c.TagRemovePreprocessor.remove_cell_tags = removed_tags c.TemplateExporter.exclude_input_prompt = True c.TemplateExporter.exclude_output_prompt = True c.TemplateExporter.exclude_markdown = True exporter = PythonExporter(config=c) code, _ = exporter.from_notebook_node(nb_node) code = re.sub('\n+', '\n', code) return code
def __init__(self, *args, **kwargs): if 'filename' not in kwargs: raise ValueError('Must pass a filename to NotebookHandler') with open(kwargs['filename']) as f: nb = nbformat.read(f, nbformat.NO_CONVERT) exporter = PythonExporter() source, meta = exporter.from_notebook_node(nb) kwargs['source'] = source super(NotebookHandler, self).__init__(*args, **kwargs)
def parse_as_python_nb(code): try: nb = nbformat.reads(code, as_version=4) exporter = PythonExporter() sources, resources = exporter.from_notebook_node(nb) source = sources except: try: nb = nbformat.reads(code, as_version=3) exporter = PythonExporter() sources, resources = exporter.from_notebook_node(nb) source = sources except: source = None print('failed to parse as Notebook') # print(code) # replace any awful ipython notebook style commands if source: source = re.sub(r'^%.*\n', '', source, flags=re.M) source = re.sub(r'^!.*\n', '', source, flags=re.M) return source
def load_module(self, fullname): url = self.try_url(fullname) if fullname in sys.modules: return try: # The importer protocol requires the loader create a new module # object, set certain attributes on it, then add it to # `sys.modules` before executing the code inside the module (which # is when the "module" actually gets code inside it) m = types.ModuleType(fullname, 'remote notebook: {}'.format(fullname)) m.__file__ = '<notebook {}>'.format(fullname) m.__name__ = url m.__loader__ = self sys.modules[url] = m # Try to load notebook resp = urlopen(url) body = resp.read().decode() notebook = nbformat.reads(body, as_version=4) exporter = PythonExporter() (code, resources) = exporter.from_notebook_node(notebook) code = re.sub(r'%.+', "", code) tree = ast.parse(code) extractor = Extractor() extractor.visit(tree) exec(extractor.code(), sys.modules[url].__dict__) sys.modules[fullname] = sys.modules[url] return m except Exception as e: # If it fails, we need to reset sys.modules to it's old state. This # is good practice in general, but also a mandatory part of the # spec, likely to keep the import statement idempotent and free of # side-effects across imports. # Delete the entry we might've created; use LBYL to avoid nested # exception handling if sys.modules.get(url): del sys.modules[url] raise e if sys.modules.get(fullname): del sys.modules[fullname] raise e
def convert_nb(file_name): export_path = f'{os.getcwd()}/{os.path.split(file_name)[1][:-5]}py' print(export_path) with open(file_name, encoding="utf8") as fn: nb = nbformat.read(fn, as_version=4) exporter = PythonExporter() # source is a tuple of python source code # meta contains metadata source, meta = exporter.from_notebook_node(nb) with open(export_path, 'w+', encoding="utf8") as fh: fh.writelines(source)
def save_tasks_py(nb): """Select cells appropriate for `tasks.py`.""" shouldnt_start_with = '# Put in a file named run-readout-scan.py' tasks_py = nb.copy() tasks_py.cells = [ cell for cell in tasks_py.cells if not cell.source.startswith(shouldnt_start_with) ] exporter = PythonExporter() exporter.raw_template = TEMPLATE output, resources = exporter.from_notebook_node(tasks_py) with open(f'{REPO_DIR}/recirq/readout_scan/tasks.py', 'w') as f: f.write(output)
def collect(self): exporter = PythonExporter() exporter.exclude_markdown = True exporter.exclude_input_prompt = True notebook_contents = self.fspath.open(encoding='utf-8') with warnings.catch_warnings(): warnings.filterwarnings("ignore", "IPython.core.inputsplitter is deprecated") notebook = nbformat.read(notebook_contents, 4) code, _ = exporter.from_notebook_node(notebook) if pytest_version >= 50403: yield IPyNbTest.from_parent(name=self.name, parent=self, code=code) else: yield IPyNbTest(self.name, self, code)
def convert_nb2inject() -> str: ''' Converts the current notebook to an executable .py file Returns ------- str : the full path and filename of the converted file ''' try: import ipykernel import notebook.notebookapp except ImportError as e: #log.exception('ImportError : This only runs in a Jupyter Notebook environment ' + str(e)) return from nbconvert import PythonExporter from nbconvert.writers import FilesWriter import nbformat #from traitlets.config import Config exporter = PythonExporter() nbfile = get_notebook_name() nb = nbformat.read(nbfile, nbformat.NO_CONVERT) start = 0 # Don't use the last cell nb.cells = nb.cells[start:-2] (output, resources) = exporter.from_notebook_node(nb) filename = nbfile.split('/')[-1].split('.')[0] user = os.getenv("JUPYTERHUB_USER") # FIX: This should not be hardcoded tmpdir = f"/tmp/cloudflow/inject/{user}" if not os.path.exists(tmpdir): os.makedirs(tmpdir) outfile = f"{tmpdir}/{filename}" # Save to file writer = FilesWriter() writer.write(output, resources, outfile) return outfile + ".py"
def convert_notebook(name, notebook, output_dir): c = Config() c.ExtractOutputPreprocessor.output_filename_template = name + '/{unique_key}_{cell_index}_{index}{extension}' md_exporter = MarkdownExporter(config=c) py_exporter = PythonExporter() (body, _) = py_exporter.from_notebook_node(notebook) with open(output_dir / f'{name}.py', 'w') as fp: fp.write(body) (body, resources) = md_exporter.from_notebook_node(notebook) with open(output_dir / f'{name}.md', 'w') as fp: fp.write(post_process_markdown(body)) for resource, value in resources['outputs'].items(): resource_path = output_dir / resource resource_path.parent.mkdir(exist_ok=True, parents=True) with open(resource_path, 'wb') as fp: fp.write(value)
def _nb_sample_to_py(notebook_path: str, output_path: str): """nb_sample_to_py converts notebook kfp sample to a python file. Cells with tag "skip-in-test" will be omitted.""" with open(notebook_path, 'r') as f: nb = nbformat.read(f, as_version=4) # Cells with skip-in-test tag will be omitted. # Example code that needs the tag: # kfp.Client().create_run_from_pipeline_func() # so that we won't submit pipelines when compiling them. nb.cells = [ cell for cell in nb.cells if 'skip-in-test' not in cell.get('metadata', {}).get('tags', []) ] py_exporter = PythonExporter() (py_code, res) = py_exporter.from_notebook_node(nb) with open(output_path, 'w') as out: out.write(py_code)
def load_source_code(notebook_path, python_path, python_code): """ Load the pipeline source code from the specified source """ source_code = "" sources = [notebook_path, python_path, python_code] assert sum(source is not None for source in sources) == 1 if python_path is not None: with open(python_path) as file: source_code = file.read() elif notebook_path is not None: with open(notebook_path) as file: notebook = nbformat.reads(file.read(), nbformat.NO_CONVERT) exporter = PythonExporter() source_code, _ = exporter.from_notebook_node(notebook) elif python_code is not None: source_code = python_code return source_code
def convert_notebook_to_py(nb_fn: Path, py_fn: Path) -> None: """ From https://stackoverflow.com/questions/17077494/how-do-i-convert-a-ipython-notebook-into-a-python-file-via-commandline """ import nbformat from nbconvert import PythonExporter with open(nb_fn) as fh: nb = nbformat.reads(fh.read(), nbformat.NO_CONVERT) exporter = PythonExporter() source, meta = exporter.from_notebook_node(nb) # Skip the magic, which gets converted to `get_ipython()` source.replace("get_ipython", "# get_ipython") with open(py_fn, "w+") as fh: fh.writelines(source)
def convert_notebook(notebook_path): write_folder = cache.get_parent_folder(notebook_path) file_name = cache.get_file_name(notebook_path) write_file = os.path.join(write_folder, "%s.%s" % (file_name, props.TYPE_PYTHON)) if cache.file_exists(write_file): LOGGER.info("'%s' already converted. Moving on ... " % file_name) return else: LOGGER.info("Converting filename '%s' ... " % file_name) with open(notebook_path) as fh: nb = nbformat.reads(fh.read(), nbformat.NO_CONVERT) exporter = PythonExporter() try: source, meta = exporter.from_notebook_node(nb) except nbformat.validator.NotebookValidationError: LOGGER.error("Validation error while converting '%s'." % notebook_path) return with open(write_file, 'w+') as fh: fh.writelines(source.encode('utf-8'))
def nbconverter(notebookPath,directionPath=None): """method name is the same as jupyter default converter name : Nbconvert(notebookPath,directionPath) :param notebookPath: source path of the ipynb file you want to convert :param direction Path : direction path of the formatted .py file """ try: with open(notebookPath,'r',encoding='utf-8') as fh: nb = nbformat.reads(fh.read(), nbformat.NO_CONVERT) except FileNotFoundError: if '.ipynb' not in notebookPath: notebookPath += '.ipynb' with open(notebookPath,'r',encoding='utf-8') as fh: nb = nbformat.reads(fh.read(), nbformat.NO_CONVERT) if directionPath == None: directionPath = notebookPath.replace('.ipynb','.py') pattern_input = r'# In\[[\d\s]*\]:' pattern_comment = '^#.+' pattern_variables = r'^[a-zA-Z0-9_]+\[?[ |0-9]*\]? *$' pattern_square_bracket = r'^\[.*\]$' pattern_string = r"^[\'|\"].*[\'|\"] *$" pattern_number = r'^\d+ *[\+|\-|\*|\/]? *\d* *$' def print_match(matched): return 'print({})'.format(matched.group().strip()) exporter = PythonExporter() source, meta = exporter.from_notebook_node(nb) source = source.split('\n')[2:] source = [ t for t in source if t and not re.match(pattern_input,t)] source = [ '\n' + t if re.match(pattern_comment,t) else t for t in source ] source = [re.sub(pattern=pattern_variables,repl=print_match,string=t) for t in source] source = [re.sub(pattern=pattern_square_bracket,repl=print_match,string=t) for t in source] source = [re.sub(pattern=pattern_string,repl=print_match,string=t) for t in source] source = '\n'.join(source) + '\n\n' with open(directionPath, 'w+',encoding='utf-8') as fh: fh.write(source) print('{} has been saved'.format(directionPath))
def scan_jupyter_imports(filename): ''' Auxiliary function to get Python imports from Jupyter notebooks ''' # check input is correct if not os.access(filename, os.R_OK): raise IOError("File {} can't be read\n".format(filename)) logging.debug('Python scan for file: {}'.format(filename)) deps = set() try: # https://nbformat.readthedocs.io/en/latest/api.html ipynb = nbformat.read(filename, as_version=nbformat.NO_CONVERT) # https://nbconvert.readthedocs.io/en/latest/nbconvert_library.html python_exporter = PythonExporter() (body, resources) = python_exporter.from_notebook_node(ipynb) tree = ast.parse(body) for node in ast.walk(tree): modules = is_import(node) if modules is not None: for m in modules: if not is_python_std(m): orig = cleanup_import(m) tran = translate_python_import(orig) if tran != "ignore" and tran not in PY_LOCAL: deps.add(tran) logging.debug('Translating Python dependency {} into {}'.format(orig, tran)) else: logging.debug('Ignoring Python dependency: {}'.format(orig)) except BaseException: logging.warning("Could not parse file: {}".format(filename)) return deps
def convert_notebook() -> str: ''' Converts the current notebook to an executable .py file Returns ------- str : the full path and filename of the converted file ''' try: import ipykernel import notebook.notebookapp except ImportError as e: #log.exception('ImportError : This only runs in a Jupyter Notebook environment ' + str(e)) return from nbconvert import PythonExporter from nbconvert.writers import FilesWriter import nbformat exporter = PythonExporter() nbfile = get_notebook_name() nb = nbformat.read(nbfile, nbformat.NO_CONVERT) start = 0 # Don't use the last cell nb.cells = nb.cells[start:-2] (output, resources) = exporter.from_notebook_node(nb) filename = nbfile.split('/')[-1].split('.')[0] outfile = filename # Save to file writer = FilesWriter() writer.write(output, resources, outfile) return outfile + ".py"
def save_run_py(nb): """Select cells appropriate for `run-readout-scan.py`.""" should_start_with = '# Put in a file named run-readout-scan.py' run_py = nb.copy() run_py.cells = [ cell for cell in run_py.cells if cell.source.startswith(should_start_with) ] # Fix-up the content first_cell_lines = run_py.cells[0].source.splitlines() new_first_cell_lines = [] added_imports = False for line in first_cell_lines: if line.startswith(should_start_with): # Ignore this comment, which is out of place in the exported py file continue if line.startswith('import') and not added_imports: # Add imports that aren't necessary when it's in one file. new_first_cell_lines += [ 'from recirq.readout_scan.tasks import ReadoutScanTask, run_readout_scan' ] added_imports = True # Rest of the lines are unmodified new_first_cell_lines += [line] # Mutate the original cell run_py.cells[0].source = '\n'.join(new_first_cell_lines) exporter = PythonExporter() exporter.raw_template = TEMPLATE output, resources = exporter.from_notebook_node(run_py) with open(f'{REPO_DIR}/recirq/readout_scan/run-readout-scan.py', 'w') as f: f.write(output)
def export_python(wd, name): nb = _read(wd, name) exporter = PythonExporter() body, resources = exporter.from_notebook_node(nb) with open("{}/{}.py".format(wd, name), 'w') as f: f.write(body)
def convert_to_python(notebook, template_file=None): exporter = PythonExporter() exporter.template_file = template_file notebook_code, meta = exporter.from_notebook_node(notebook) return notebook_code, meta
def export_python(nb, destfn): exporter = PythonExporter() body, resources = exporter.from_notebook_node(nb) with open(destfn, 'w') as f: f.write(body)