def test_tutorials(testfolder,
tutorials=('ligand-binding-analysis',
'protein-folding-analysis')):
startdir = os.path.dirname(os.path.realpath(__file__))
pyexp = PythonExporter()
pyexp.template_file = os.path.join(startdir, 'simplepython.tpl')
# For each tutorial noteboook
for tut in glob(os.path.join(startdir, '*.ipynb')):
name = os.path.splitext(os.path.basename(tut))[0]
if name not in tutorials:
continue
testsubf = os.path.abspath(os.path.join(testfolder, name))
if not os.path.exists(testsubf):
os.makedirs(testsubf)
# Read notebook in
with open(tut, 'r') as fi:
notebook = nbformat.read(fi, nbformat.NO_CONVERT)
output, resources = pyexp.from_notebook_node(notebook, )
# Write it out in .py format
with open(os.path.join(testsubf, 'test_{}.py'.format(name)),
'w') as fo:
fo.writelines(format_script(output, testsubf, name))
return pytest.main([testfolder])
def tokenize_lambda_func(val, within, notebook_name=None):
if notebook_name:
notebook_path = os.getcwd() + "/{}.ipynb".format(notebook_name)
with open(notebook_path, 'r') as f:
notebook = nbformat.reads(f.read(), nbformat.NO_CONVERT)
exporter = PythonExporter()
source, _ = exporter.from_notebook_node(notebook)
readline = io.StringIO(source).readline
else:
readline = open(val.__code__.co_filename).readline
first_line = val.__code__.co_firstlineno
flag = False
source = ""
for t in tokenize.generate_tokens(readline):
t_type, t_string, (r_start, c_start), (r_end, c_end), line = t
t_name = token.tok_name[t_type]
if r_start == first_line:
if t_name == 'NAME' and t_string == within:
flag = True
res = t_string
start = 0 # number of parenthesis
continue
if flag:
source += ' ' + t_string
if t_name == 'OP':
if t_string == '(':
start += 1
elif t_string == ')':
start -= 1
if start == 0:
break
return source.strip()
def locals_from_notebook(notebook_ipynb):
"""Read, run, return locals of notebook"""
banned_commands = ['HTML']
ipynb = json.load(open(notebook_ipynb))
for cell in ipynb['cells']:
# erase test cells, this is optional and useful for debugging
# to avoid recursions when developing
if any('## TEST ##' in line for line in cell['source']):
cell['source'] = []
# filter out all the lines with banned commands
if banned_commands is not None:
cell['source'] = [
line for line in cell['source']
if not any(command in line for command in banned_commands)
]
exporter = PythonExporter()
source, meta = exporter.from_notebook_node(
nbformat.reads(json.dumps(ipynb), nbformat.NO_CONVERT))
with open('./cleaned_notebook.py', 'w') as fh:
fh.write(source)
notebook_locals = run_path('cleaned_notebook.py')
os.system('rm cleaned_notebook.py')
return notebook_locals
def parse_ipynb(file: Path) -> str:
with open(file, "r") as nb_file:
nb_str = nb_file.read()
nb = nbformat.reads(nb_str, nbformat.NO_CONVERT)
exporter = PythonExporter()
script, _ = exporter.from_notebook_node(nb)
return script
def convertNotebook(notebookPath, modulePath):
with open(notebookPath) as fh:
nb = nbformat.reads(fh.read(), nbformat.NO_CONVERT)
exporter = PythonExporter()
source, meta = exporter.from_notebook_node(nb)
with open(modulePath, 'w+') as fh:
fh.writelines(source) # .encode('utf-8'))
def test_run_tutorial():
""" There are a lot of examples in tutorial, and all of them
should be working.
Notes
-----
IPython notebook is converted to Python script, so tqdm bars and
plotting is removed.
"""
# pylint: disable=exec-used
tutorials_dir = './../../examples/tutorials/'
notebook = '09_solving_PDE_with_NN.ipynb'
file = tutorials_dir + notebook
with warnings.catch_warnings():
warnings.simplefilter("ignore")
from nbconvert import PythonExporter
code, _ = PythonExporter().from_filename(file)
code_ = []
for line in code.split('\n'):
if not line.startswith('#'):
if not any(bs in line for bs in BAD_PREFIXES):
code_.append(line.replace('in tqdm_notebook', 'in '))
code = '\n'.join(code_)
exec(code, {})
def parse_ipynb_file(filename):
"""A simple parser for extracting the names of imported modules / packages
from an IPython (Jupyter) notebook.
Parameters
----------
filename : str
Path to an IPython notebook file.
Returns
-------
packages : set
A unique list of all (root) packages imported by the specified notebook
file.
"""
import nbformat
from nbconvert import PythonExporter
with open(filename) as _file:
nb_stuff = nbformat.reads(_file.read(), as_version=4)
exporter = PythonExporter()
(body, _) = exporter.from_notebook_node(nb_stuff)
return parse_py_module(body)
def test_run_tutorial(path):
""" There are a lot of examples in tutorial, and all of them
should be working.
Notes
-----
IPython notebook is converted to Python script, so tqdm bars and
plotting is removed.
"""
# pylint: disable=exec-used
with warnings.catch_warnings():
warnings.simplefilter("ignore")
from nbconvert import PythonExporter # pylint: disable=import-outside-toplevel
code, _ = PythonExporter().from_filename(path)
code_ = []
for line in code.split('\n'):
if not line.startswith('#'):
if not any(bs in line for bs in BAD_PREFIXES):
line = line.replace("bar='notebook'", "bar=False")
line = line.replace("in tqdm_notebook", "in ")
code_.append(line)
code = '\n'.join(code_)
exec(code, {})
def test_notebooks():
"""Run all notebooks in /docs/tutorials/ as tests."""
# Get the notebook names
root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
path = os.path.join(root, 'docs', 'tutorials')
notebooks = glob.glob(os.path.join(path, '*.ipynb'))
# Convert them to python scripts
exporter = PythonExporter()
for notebook in notebooks:
# Get the script as a string
script, _ = exporter.from_filename(notebook)
# Get rid of %matplotlib inline commands
script = script.replace("get_ipython().magic('matplotlib inline')", "")
script = script.replace(
"get_ipython().run_line_magic('matplotlib', 'inline')", "")
# Get rid of %run commands
script = re.sub("get_ipython\(\).magic\('run (.*)'\)", r"#", script)
script = re.sub("get_ipython\(\).run_line_magic\('run', '(.*)'\)",
r"#", script)
# Remove the %time wrappers
script = re.sub("get_ipython\(\).magic\('time (.*)'\)", r"\1", script)
script = re.sub("get_ipython\(\).run_line_magic\('time', '(.*)'\)",
r"\1", script)
# Remove calls to map.show()
script = re.sub("(.*)\.show()(.*)", r"#", script)
# Run it
print("Running %s..." % os.path.basename(notebook))
exec(script, globals(), globals())
pl.close('all')
def read_source_script(filepath):
"""Read the contents of `filepath`
Parameters
----------
filepath: Str
Absolute path to a Python file. Expected to end with one of the following extensions: '.py', '.ipynb'
Returns
-------
source: Str
The contents of `filepath`"""
if filepath.endswith('.ipynb'):
with open(filepath, 'r') as f:
from nbconvert import PythonExporter
import nbformat
notebook = nbformat.reads(f.read(), nbformat.NO_CONVERT)
exporter = PythonExporter()
source, _ = exporter.from_notebook_node(notebook)
else:
with open(filepath, 'r') as f:
source = f.read()
return source
def test_tutorials(testfolder, tutorials=('ligand-binding-analysis', 'protein-folding-analysis')):
startdir = os.path.dirname(os.path.realpath(__file__))
pyexp = PythonExporter()
pyexp.template_file = os.path.join(startdir, 'simplepython.tpl')
# For each tutorial noteboook
for tut in glob(os.path.join(startdir, '*.ipynb')):
name = os.path.splitext(os.path.basename(tut))[0]
if name not in tutorials:
continue
testsubf = os.path.abspath(os.path.join(testfolder, name))
if not os.path.exists(testsubf):
os.makedirs(testsubf)
# Read notebook in
with open(tut, 'r') as fi:
notebook = nbformat.read(fi, nbformat.NO_CONVERT)
output, resources = pyexp.from_notebook_node(notebook, )
# Write it out in .py format
with open(os.path.join(testsubf, 'test_{}.py'.format(name)), 'w') as fo:
fo.writelines(format_script(output, testsubf, name))
return pytest.main([testfolder])
def get_hyperopt_model_string(model, data, functions, notebook_name, verbose,
stack, data_args):
model_string = inspect.getsource(model)
model_string = remove_imports(model_string)
if notebook_name:
notebook_path = os.getcwd() + "/{}.ipynb".format(notebook_name)
with open(notebook_path, 'r') as f:
notebook = nbformat.reads(f.read(), nbformat.NO_CONVERT)
exporter = PythonExporter()
source, _ = exporter.from_notebook_node(notebook)
else:
calling_script_file = os.path.abspath(inspect.stack()[stack][1])
with open(calling_script_file, 'r') as f:
source = f.read()
cleaned_source = remove_all_comments(source)
imports = extract_imports(cleaned_source, verbose)
parts = hyperparameter_names(model_string)
aug_parts = augmented_names(parts)
hyperopt_params = get_hyperparameters(model_string)
space = get_hyperopt_space(parts, hyperopt_params, verbose)
functions_string = retrieve_function_string(functions, verbose)
data_string = retrieve_data_string(data, verbose, data_args)
model = hyperopt_keras_model(model_string, parts, aug_parts, verbose)
temp_str = temp_string(imports, model, data_string, functions_string,
space)
return temp_str
def get_hyperopt_model_string(model, data, functions, notebook_name, verbose, stack):
model_string = inspect.getsource(model)
model_string = remove_imports(model_string)
if notebook_name:
notebook_path = os.getcwd() + "/{}.ipynb".format(notebook_name)
with open(notebook_path, 'r') as f:
notebook = nbformat.reads(f.read(), nbformat.NO_CONVERT)
exporter = PythonExporter()
source, _ = exporter.from_notebook_node(notebook)
else:
calling_script_file = os.path.abspath(inspect.stack()[stack][1])
with open(calling_script_file, 'r') as f:
source = f.read()
cleaned_source = remove_all_comments(source)
imports = extract_imports(cleaned_source, verbose)
parts = hyperparameter_names(model_string)
aug_parts = augmented_names(parts)
hyperopt_params = get_hyperparameters(model_string)
space = get_hyperopt_space(parts, hyperopt_params, verbose)
functions_string = retrieve_function_string(functions, verbose)
data_string = retrieve_data_string(data, verbose)
model = hyperopt_keras_model(model_string, parts, aug_parts, verbose)
temp_str = temp_string(imports, model, data_string, functions_string, space)
return temp_str
def convertNotebook(notebookPath):
with open(notebookPath, encoding='UTF-8') as fh:
nb = nbformat.reads(fh.read(), nbformat.NO_CONVERT)
exporter = PythonExporter()
source, meta = exporter.from_notebook_node(nb)
return source
def update_script(name):
python_exporter = PythonExporter()
with open("experiments/{name}/{name}.ipynb".format(name=name)) as f:
notebook = nbformat.reads(f.read(), 4)
test = python_exporter.from_notebook_node(notebook)
path = "experiments/{name}/{name}.py".format(name=name)
with open(path, "w") as f_out:
f_out.write(test[0])
return path
def convert_ipynb_to_py(notebook_path, output_path):
with open(notebook_path) as fh:
nb = nbformat.reads(fh.read(), nbformat.NO_CONVERT)
exporter = PythonExporter()
source, meta = exporter.from_notebook_node(nb)
with open(output_path, 'w+') as fh:
fh.writelines(source.encode('utf-8'))
def collect(self):
exporter = PythonExporter()
exporter.exclude_markdown = True
exporter.exclude_input_prompt = True
notebook_contents = self.fspath.open()
notebook = nbformat.read(notebook_contents, 4)
code, _ = exporter.from_notebook_node(notebook)
yield IPyNbTest(self.name, self, code)
def convertNotebook(notebookPath):
with open(notebookPath) as fh:
nb = nbformat.reads(fh.read(), nbformat.NO_CONVERT)
exporter = PythonExporter()
source, meta = exporter.from_notebook_node(nb)
lines = source.split("\n")
for line in lines:
print(line)
def __init__(self, *args, **kwargs):
if 'filename' not in kwargs:
raise ValueError('Must pass a filename to NotebookHandler')
with open(kwargs['filename']) as f:
nb = nbformat.read(f, nbformat.NO_CONVERT)
exporter = PythonExporter()
source, meta = exporter.from_notebook_node(nb)
kwargs['source'] = source
super(NotebookHandler, self).__init__(*args, **kwargs)
def convert_notebook(notebook_path, module_path):
with open(notebook_path) as fh:
nb = nbformat.reads(fh.read(), nbformat.NO_CONVERT)
exporter = PythonExporter()
source, _ = exporter.from_notebook_node(nb)
with open(module_path, 'w+') as fh:
fh.write(source)
print("\t> Converted {} to {}".format(notebook_path, module_path))
def test_nb_pipeline_runs():
"""
Tests whether the .ipynb version of the pipeline works
"""
with open(FILE_NB) as file:
notebook = nbformat.reads(file.read(), nbformat.NO_CONVERT)
exporter = PythonExporter()
code, _ = exporter.from_notebook_node(notebook)
parsed_ast = ast.parse(code)
exec(compile(parsed_ast, filename="<ast>", mode="exec"))
def _get_pycode(cls, nb_node: str, removed_tags: Set[str] = set()) -> str:
c = Config()
c.TagRemovePreprocessor.remove_cell_tags = removed_tags
c.TemplateExporter.exclude_input_prompt = True
c.TemplateExporter.exclude_output_prompt = True
c.TemplateExporter.exclude_markdown = True
exporter = PythonExporter(config=c)
code, _ = exporter.from_notebook_node(nb_node)
code = re.sub('\n+', '\n', code)
return code
def load_module(self, fullname):
url = self.try_url(fullname)
if fullname in sys.modules:
return
try:
# The importer protocol requires the loader create a new module
# object, set certain attributes on it, then add it to
# `sys.modules` before executing the code inside the module (which
# is when the "module" actually gets code inside it)
m = types.ModuleType(fullname,
'remote notebook: {}'.format(fullname))
m.__file__ = '<notebook {}>'.format(fullname)
m.__name__ = url
m.__loader__ = self
sys.modules[url] = m
# Try to load notebook
resp = urlopen(url)
body = resp.read().decode()
notebook = nbformat.reads(body, as_version=4)
exporter = PythonExporter()
(code, resources) = exporter.from_notebook_node(notebook)
code = re.sub(r'%.+', "", code)
tree = ast.parse(code)
extractor = Extractor()
extractor.visit(tree)
exec(extractor.code(), sys.modules[url].__dict__)
sys.modules[fullname] = sys.modules[url]
return m
except Exception as e:
# If it fails, we need to reset sys.modules to it's old state. This
# is good practice in general, but also a mandatory part of the
# spec, likely to keep the import statement idempotent and free of
# side-effects across imports.
# Delete the entry we might've created; use LBYL to avoid nested
# exception handling
if sys.modules.get(url):
del sys.modules[url]
raise e
if sys.modules.get(fullname):
del sys.modules[fullname]
raise e
def convert_nb(file_name):
export_path = f'{os.getcwd()}/{os.path.split(file_name)[1][:-5]}py'
print(export_path)
with open(file_name, encoding="utf8") as fn:
nb = nbformat.read(fn, as_version=4)
exporter = PythonExporter()
# source is a tuple of python source code
# meta contains metadata
source, meta = exporter.from_notebook_node(nb)
with open(export_path, 'w+', encoding="utf8") as fh:
fh.writelines(source)
def save_tasks_py(nb):
"""Select cells appropriate for `tasks.py`."""
shouldnt_start_with = '# Put in a file named run-readout-scan.py'
tasks_py = nb.copy()
tasks_py.cells = [
cell for cell in tasks_py.cells
if not cell.source.startswith(shouldnt_start_with)
]
exporter = PythonExporter()
exporter.raw_template = TEMPLATE
output, resources = exporter.from_notebook_node(tasks_py)
with open(f'{REPO_DIR}/recirq/readout_scan/tasks.py', 'w') as f:
f.write(output)
def collect(self):
exporter = PythonExporter()
exporter.exclude_markdown = True
exporter.exclude_input_prompt = True
notebook_contents = self.fspath.open(encoding='utf-8')
with warnings.catch_warnings():
warnings.filterwarnings("ignore", "IPython.core.inputsplitter is deprecated")
notebook = nbformat.read(notebook_contents, 4)
code, _ = exporter.from_notebook_node(notebook)
if pytest_version >= 50403:
yield IPyNbTest.from_parent(name=self.name, parent=self, code=code)
else:
yield IPyNbTest(self.name, self, code)
def convert_nb2inject() -> str:
''' Converts the current notebook to an executable .py file
Returns
-------
str : the full path and filename of the converted file
'''
try:
import ipykernel
import notebook.notebookapp
except ImportError as e:
#log.exception('ImportError : This only runs in a Jupyter Notebook environment ' + str(e))
return
from nbconvert import PythonExporter
from nbconvert.writers import FilesWriter
import nbformat
#from traitlets.config import Config
exporter = PythonExporter()
nbfile = get_notebook_name()
nb = nbformat.read(nbfile, nbformat.NO_CONVERT)
start = 0
# Don't use the last cell
nb.cells = nb.cells[start:-2]
(output, resources) = exporter.from_notebook_node(nb)
filename = nbfile.split('/')[-1].split('.')[0]
user = os.getenv("JUPYTERHUB_USER")
# FIX: This should not be hardcoded
tmpdir = f"/tmp/cloudflow/inject/{user}"
if not os.path.exists(tmpdir):
os.makedirs(tmpdir)
outfile = f"{tmpdir}/{filename}"
# Save to file
writer = FilesWriter()
writer.write(output, resources, outfile)
return outfile + ".py"
def write_files(export_list, nb_node, file_name):
"""
Export and write files from a notebook node.
Args:
export_list (list of strings) -- name of valid nbconvert exporters
nb_node(nbformat node object) -- notebook to be my_converted
file_name (str) -- base name of file to be written
Returns:
None
"""
try:
# export and write file.
for export in export_list:
if export == "html":
exporter = HTMLExporter()
elif export == "py":
exporter = PythonExporter()
(body, resources) = exporter.from_notebook_node(nb_node)
write_file = FilesWriter()
write_file.write(
output=body,
resources=resources,
notebook_name=file_name
)
except Exception as e:
print(f"There was a problem exporting the file(s): {e}")
return None
def _nb_sample_to_py(notebook_path: str, output_path: str):
"""nb_sample_to_py converts notebook kfp sample to a python file. Cells with tag "skip-in-test" will be omitted."""
with open(notebook_path, 'r') as f:
nb = nbformat.read(f, as_version=4)
# Cells with skip-in-test tag will be omitted.
# Example code that needs the tag:
# kfp.Client().create_run_from_pipeline_func()
# so that we won't submit pipelines when compiling them.
nb.cells = [
cell for cell in nb.cells
if 'skip-in-test' not in cell.get('metadata', {}).get('tags', [])
]
py_exporter = PythonExporter()
(py_code, res) = py_exporter.from_notebook_node(nb)
with open(output_path, 'w') as out:
out.write(py_code)
def convert_notebook(name, notebook, output_dir):
c = Config()
c.ExtractOutputPreprocessor.output_filename_template = name + '/{unique_key}_{cell_index}_{index}{extension}'
md_exporter = MarkdownExporter(config=c)
py_exporter = PythonExporter()
(body, _) = py_exporter.from_notebook_node(notebook)
with open(output_dir / f'{name}.py', 'w') as fp:
fp.write(body)
(body, resources) = md_exporter.from_notebook_node(notebook)
with open(output_dir / f'{name}.md', 'w') as fp:
fp.write(post_process_markdown(body))
for resource, value in resources['outputs'].items():
resource_path = output_dir / resource
resource_path.parent.mkdir(exist_ok=True, parents=True)
with open(resource_path, 'wb') as fp:
fp.write(value)
def _get_kaggle_notebook_content():
"""Returns the kaggle notebook python code contents."""
if PythonExporter is None:
raise RuntimeError(
# This should never occur.
# `nbconvert` is always installed on Kaggle.
"Please make sure you have installed `nbconvert` package.")
from kaggle_session import UserSessionClient # pylint: disable=g-import-not-at-top # pytype: disable=import-error
kaggle_session_client = UserSessionClient()
try:
response = kaggle_session_client.get_exportable_ipynb()
ipynb_stream = io.StringIO(response["source"])
py_content, _ = PythonExporter().from_file(ipynb_stream)
return py_content.splitlines(keepends=True)
except:
raise RuntimeError("Unable to get the notebook contents.")
def load_source_code(notebook_path, python_path, python_code):
"""
Load the pipeline source code from the specified source
"""
source_code = ""
sources = [notebook_path, python_path, python_code]
assert sum(source is not None for source in sources) == 1
if python_path is not None:
with open(python_path) as file:
source_code = file.read()
elif notebook_path is not None:
with open(notebook_path) as file:
notebook = nbformat.reads(file.read(), nbformat.NO_CONVERT)
exporter = PythonExporter()
source_code, _ = exporter.from_notebook_node(notebook)
elif python_code is not None:
source_code = python_code
return source_code
def convert_notebook_to_py(nb_fn: Path, py_fn: Path) -> None:
"""
From https://stackoverflow.com/questions/17077494/how-do-i-convert-a-ipython-notebook-into-a-python-file-via-commandline
"""
import nbformat
from nbconvert import PythonExporter
with open(nb_fn) as fh:
nb = nbformat.reads(fh.read(), nbformat.NO_CONVERT)
exporter = PythonExporter()
source, meta = exporter.from_notebook_node(nb)
# Skip the magic, which gets converted to `get_ipython()`
source.replace("get_ipython", "# get_ipython")
with open(py_fn, "w+") as fh:
fh.writelines(source)
def convert_notebook(notebook_path):
write_folder = cache.get_parent_folder(notebook_path)
file_name = cache.get_file_name(notebook_path)
write_file = os.path.join(write_folder, "%s.%s" % (file_name, props.TYPE_PYTHON))
if cache.file_exists(write_file):
LOGGER.info("'%s' already converted. Moving on ... " % file_name)
return
else:
LOGGER.info("Converting filename '%s' ... " % file_name)
with open(notebook_path) as fh:
nb = nbformat.reads(fh.read(), nbformat.NO_CONVERT)
exporter = PythonExporter()
try:
source, meta = exporter.from_notebook_node(nb)
except nbformat.validator.NotebookValidationError:
LOGGER.error("Validation error while converting '%s'." % notebook_path)
return
with open(write_file, 'w+') as fh:
fh.writelines(source.encode('utf-8'))
def export_python(wd, name):
nb = _read(wd, name)
exporter = PythonExporter()
body, resources = exporter.from_notebook_node(nb)
with open("{}/{}.py".format(wd, name), 'w') as f:
f.write(body)
def convert_to_python(notebook, template_file=None):
exporter = PythonExporter()
exporter.template_file = template_file
notebook_code, meta = exporter.from_notebook_node(notebook)
return notebook_code, meta
def export_python(nb, destfn):
exporter = PythonExporter()
body, resources = exporter.from_notebook_node(nb)
with open(destfn, 'w') as f:
f.write(body)
