def __init__(self, path: Path):
self.path: Path = path
# Read the notebook with nbformat
with open(self.path) as f:
nb_raw = f.read()
self.nb_dict: NotebookNode = nbformat.reads(nb_raw, as_version=4)
# Populate the list of Cells
self.cells: List[Cell] = [
Cell(cell_index, cell_dict)
for cell_index, cell_dict in enumerate(self.nb_dict.cells)
]
self.non_executed = all(
[cell.non_executed for cell in self.code_cells])
# Convert the notebook to a Python script
python_exporter = nbconvert.PythonExporter()
self.script, _ = python_exporter.from_notebook_node(self.nb_dict)
# Extract the Python abstract syntax tree
# (or set `has_invalid_python_syntax` to True)
self.has_invalid_python_syntax: bool = False
try:
self.ast = ast.parse(self.script)
except SyntaxError:
self.has_invalid_python_syntax = True
def main():
args = parser.parse_args()
full_path = args.full_path
force = args.force
if full_path is None:
full_path = os.path.join(BASE_PATH, "..", "examples", "notebooks")
notebook = args.notebook
if notebook is None:
notebooks = glob.glob(os.path.join(full_path, "*.ipynb"))
else:
if not notebook.endswith(".ipynb"):
notebook = notebook + ".ipynb"
notebook = os.path.abspath(os.path.join(full_path, notebook))
if not os.path.exists(notebook):
raise FileNotFoundError("Notebook {0} not found.".format(notebook))
notebooks = [notebook]
if not notebooks:
import warnings
warnings.warn("No notebooks found", UserWarning)
for nb in notebooks:
nb_full_name = os.path.split(nb)[1]
nb_name = os.path.splitext(nb_full_name)[0]
py_name = nb_name + ".py"
# Get base directory to notebook
out_file = os.path.split(nb)[0]
# Write to ../python
out_file = os.path.join(out_file, "..", "python", py_name)
if is_newer(out_file, nb) and not force:
logger.info("Skipping {0}, exported version newer than "
"notebook".format(nb_name))
continue
logger.info("Converting {0}".format(nb_name))
with open(nb, "r", encoding="utf8") as nb_file:
converter = nbconvert.PythonExporter()
python = converter.from_file(nb_file)
code = python[0].split("\n")
code_out = []
for i, block in enumerate(code):
if "get_ipython" in block:
continue
elif block.startswith("# In[ ]:"):
continue
if block.startswith("#"):
# Wrap comments from Markdown
block = textwrap.fill(block, width=74)
block = block.replace("\n", "\n# ")
code_out.append(block)
if not code_out[0]:
code_out = code_out[1:]
loc = 0
for i, line in enumerate(code_out):
if "# coding: utf" in line:
loc = i + 1
break
code_out.insert(loc, DO_NOT_EDIT.format(notebook=nb_full_name))
code_out = "\n".join(code_out)
code_out, success = FormatCode(code_out, style_config="pep8")
with open(out_file, "w", encoding="utf8", newline="\n") as of:
of.write(code_out)
def test_example_notebooks(example, examples_directory):
import nbconvert
import nbformat
exporter = nbconvert.PythonExporter()
notebooks_folder = examples_directory / "notebooks"
env = os.environ.copy()
env["PYTHONWARNINGS"] = "ignore"
with open(notebooks_folder / example) as fp:
nbcode = nbformat.reads(fp.read(), as_version=4)
(code, _) = exporter.from_notebook_node(nbcode)
script = "import matplotlib\nmatplotlib.use('Agg')\n" + code
process = subprocess.run(
["python", "-c", script],
capture_output=True,
cwd=str(notebooks_folder),
env=env,
)
if b"SpiceSolverError" in process.stderr:
skip("No SPICE solver found.")
elif b"SmartsSolverError" in process.stderr:
skip("No SMARTS solver found.")
elif b"RCWASolverError" in process.stderr:
skip("No RCWA solver found.")
elif process.stderr != b"":
raise Exception(process.stderr.decode())
def jup2py(jup_filename, py_filename=None, save=True):
# Converting jupyter to py through nbconvertor
py_code = nbconvert.export(nbconvert.PythonExporter(), jup_filename)
mycode = py_code[0].split("\n")
clean_code = []
for index, line in enumerate(mycode):
# Removing which are either "\n" or denotes the "Input" Tag for jupyter notebook
if line == "\n" or (
line.startswith("# In[")) or line is None or line == "":
pass
# Commenting the magic functions or the command line
elif line.startswith("get_ipython()"):
line = "#" + line
clean_code.append(line)
# Remove the jup2py line itself
elif "jup2py" in line:
pass
else:
clean_code.append(line)
clean_code = "\n".join(clean_code)
# Saving back the .py file
if save:
save_pyfile(clean_code, py_filename)
else:
return clean_code
def nb_to_py(nb_file: str, location: str) -> str:
"""Convery .ipynb to temporary .py file."""
try:
import nbformat
import nbconvert
except ImportError as e:
raise pe.ScriptImportError(
f"{e}\nCannot import notebook conversion libraries." +
"Please add `jupyter` (or `nbformat` and `nbconvert`)" +
" to your dependencies.", )
handle, filename = tempfile.mkstemp(text=True, suffix=".py")
with os.fdopen(handle, "w") as tf:
with open(os.path.join(location, nb_file)) as fh:
nb = nbformat.reads(fh.read(), nbformat.NO_CONVERT)
exporter = nbconvert.PythonExporter()
src, _ = exporter.from_notebook_node(nb)
tf.writelines(src)
# delete file on exit
atexit.register(functools.partial(os.remove, filename))
return os.path.basename(filename), os.path.dirname(filename)
def extract_py_notebook(in_file: Path) -> str:
"""Extract the python code from a given notebook"""
# we use config for importing
c = TConfig()
# c.PythonExporter.preprocessors = []
# load the notebook
notebook: nbformat.NotebookNode = nbformat.read(
str(in_file), as_version=nbformat.NO_CONVERT)
# TODO - any preprocessing here
# convert it
conv = nbconvert.PythonExporter(config=c)
(body, resources) = conv.from_notebook_node(notebook)
# (body, resources) = conv.from_filename(str(in_file))
# write the notebook
# writer = nbconvert.writers.FilesWriter()
# writer.write(body, resources, "output_notebook")
# TODO - use nbconvert jinja support once v6 supports custom templates properly
# postprocess body
# we need to mock get_ipython as nbconvert doesn't comment it out
header = """# inject get_ipython mock for magic functions
from unittest.mock import Mock
get_ipython = Mock()
"""
script = header + body
return script
def upload_file(self,
source,
mkdir=False,
append=False,
binary=False,
quiet=True,
root=""):
if os.path.exists(source) and os.path.isfile(source):
destination = source
root_len = len(root)
if root != "" and root_len + 1 < len(
source) and source[:root_len] == root:
destination = destination[root_len + 1:]
if destination.endswith(".ipynb"):
notebook = nbformat.read(source, as_version=4)
py_exporter = nbconvert.PythonExporter()
output, resources = py_exporter.from_notebook_node(notebook)
file_contents = output.replace("get_ipython().run_line_magic",
"# %")
destination = destination.replace(".ipynb", ".py")
else:
file_contents = open(source, "r").read()
self.sres("\n\nuploading '{0}'\n".format(destination))
self.dc.send_to_file(destination,
mkdir=mkdir,
append=append,
binary=binary,
quiet=quiet,
file_contents=file_contents)
else:
self.sres("'{0}' is not a file\n\n".format(source))
def extract_extrametadata(notebook, override=None):
base = notebook.metadata.get('celltests', {})
override = override or {}
base['lines'] = 0 # TODO: is this used?
base['kernelspec'] = notebook.metadata.get('kernelspec', {})
# "python code" things (e.g. number of function definitions)...
# note: no attempt to be clever here (so e.g. "%time def f: pass" would be missed, as would the contents of
# a cell using %%capture cell magics; possible to handle those scenarios but would take more effort)
code = nbconvert.PythonExporter(
exclude_raw=True).from_notebook_node(notebook)[0]
# notebooks start with "coding: utf-8"
parsed_source = ast.parse(
code.encode('utf8') if sys.version_info[0] == 2 else code)
fn_def_counter = FnDefCounter()
fn_def_counter.visit(parsed_source)
base['functions'] = fn_def_counter.count
class_counter = ClassDefCounter()
class_counter.visit(parsed_source)
base['classes'] = class_counter.count
# alternative to doing it this way would be to check the ipython
# souce for %magic, %%magics before it's converted to regular python
magics_recorder = MagicsRecorder()
magics_recorder.visit(parsed_source)
base['magics'] = magics_recorder.seen
# "notebook structure" things...
base['cell_count'] = 0
base['cell_tested'] = []
base['test_count'] = 0
base['cell_lines'] = []
for c in notebook.cells:
if c.get('cell_type') in (
'markdown',
'raw',
):
continue
base['cell_lines'].append(0)
base['cell_tested'].append(False)
base['cell_count'] += 1
for line in c['source'].split('\n'):
base['lines'] += 1
base['cell_lines'][-1] += 1
for t in c['metadata'].get('tests', []):
if t.strip().startswith('%cell'):
base['test_count'] += 1
base['cell_tested'][-1] = True
break
# in case you want to override notebook settings
if override:
base.update(override)
return base
def apply_preprocessors(preprocessors, nbname):
notebooks_path = os.path.join(os.path.split(__file__)[0], 'notebooks')
with open(os.path.join(notebooks_path, nbname)) as f:
nb = nbformat.read(f, nbformat.NO_CONVERT)
exporter = nbconvert.PythonExporter()
for preprocessor in preprocessors:
exporter.register_preprocessor(preprocessor)
source, meta = exporter.from_notebook_node(nb)
return source
def preprocess(self):
exporter = nbconvert.PythonExporter()
exporter.register_preprocessor(self.notebook_preprocessor, enabled=True)
contents, _ = exporter.from_filename(self.notebook_file)
converted_notebook = Path(self.notebook_file).with_suffix('.py')
with open(converted_notebook, 'w') as f:
f.write(contents)
self.executable = converted_notebook
return [converted_notebook]
def export_as_python(filename):
with io.open(filename, encoding='utf8') as nbfile:
nb = nbformat.read(nbfile, as_version=4)
output, _ = nbconvert.PythonExporter().from_notebook_node(nb)
if sys.version_info[0] == 2:
# notebooks will start with "coding: utf-8", but output already unicode
# (alternative hack would be to find the coding line and remove it)
output = output.encode('utf8')
return output
def main():
args = parser.parse_args()
full_path = args.full_path
force = args.force
if full_path is None:
full_path = os.path.join(BASE_PATH, '..', 'examples', 'notebooks')
notebook = args.notebook
if notebook is None:
notebooks = glob.glob(os.path.join(full_path, '*.ipynb'))
else:
if not notebook.endswith('.ipynb'):
notebook = notebook + '.ipynb'
notebook = os.path.abspath(os.path.join(full_path, notebook))
if not os.path.exists(notebook):
raise FileNotFoundError('Notebook {0} not found.'.format(notebook))
notebooks = [notebook]
if not notebooks:
import warnings
warnings.warn('No notebooks found', UserWarning)
for nb in notebooks:
nb_full_name = os.path.split(nb)[1]
nb_name = os.path.splitext(nb_full_name)[0]
py_name = nb_name + '.py'
# Get base directory to notebook
out_file = os.path.split(nb)[0]
# Write to ../python
out_file = os.path.join(out_file, '..', 'python', py_name)
if is_newer(out_file, nb) and not force:
logger.info('Skipping {0}, exported version newer than '
'notebook'.format(nb_name))
continue
logger.info('Converting {0}'.format(nb_name))
with open(nb, 'r') as nb_file:
converter = nbconvert.PythonExporter()
python = converter.from_file(nb_file)
code = python[0].split('\n')
code_out = []
for i, block in enumerate(code):
if 'get_ipython' in block:
continue
elif block.startswith('# In[ ]:'):
continue
if block.startswith('#'):
# Wrap comments from Markdown
block = textwrap.fill(block, width=74)
block = block.replace('\n', '\n# ')
code_out.append(block)
if not code_out[0]:
code_out = code_out[1:]
loc = 0
if 'coding' in code_out[0]:
loc = 1
code_out.insert(loc, DO_NOT_EDIT.format(notebook=nb_full_name))
code_out = '\n'.join(code_out)
code_out, success = FormatCode(code_out, style_config='pep8')
with open(out_file, 'w') as of:
of.write(code_out)
def runtest(self):
with io.open(self.name, encoding='utf8') as nbfile:
nb = nbformat.read(nbfile, as_version=4)
magics_processor = magics.Processor(
extra_line_blacklist=_get_list_from_conf(
'nbsmoke_flakes_line_magics_blacklist',
self.parent.parent.config),
extra_cell_blacklist=_get_list_from_conf(
'nbsmoke_flakes_cell_magics_blacklist',
self.parent.parent.config))
magics_processor.insert_get_ipython(nb)
ipy, _ = nbconvert.PythonExporter().from_notebook_node(nb)
debug = self.config.option.nbsmoke_lint_debug
filenames = []
self._write_debug_file(debug, ipy, self.name, "pre", filenames)
py = magics_processor.ipython_to_python_for_flake_checks(ipy)
self._write_debug_file(debug, py, self.name, "post", filenames)
flake_result = _pyflakes.flake_check(
# notebooks will start with "coding: utf-8", but py already unicode
py.encode('utf8') if sys.version_info[0] == 2 else py,
self.name)
### remove flakes by regex ###
_user_flakes_to_ignore = self.parent.parent.config.getini(
'nbsmoke_flakes_to_ignore')
if _user_flakes_to_ignore != '':
_user_flakes_to_ignore = _user_flakes_to_ignore.splitlines()
for pattern in set(flakes_to_ignore) | set(_user_flakes_to_ignore):
flake_result['messages'] = [
msg for msg in flake_result['messages']
if not re.search(pattern, msg)
]
##############################
if flake_result['messages']:
msg = "%s\n** " % self.name
if self.config.option.nbsmoke_lint_onlywarn and 'message_for_onlywarn' in flake_result:
msg += flake_result['message_for_onlywarn']
else:
msg += "\n** ".join(flake_result['messages'])
msg += "\n" + "To see python source that was checked by pyflakes: %s" % filenames[
1]
msg += "\n" + "To see python source before magics were handled by nbsmoke: %s" % filenames[
0]
if self.config.option.nbsmoke_lint_onlywarn:
warnings.warn("Flakes (or error) detected:\n" + msg + "\n")
else:
raise NBLintError(msg)
def export_notebook_to_tar_gz(notebook_file,
output_filename,
converted_filename=DEFAULT_CONVERTED_FILENAME):
if notebook_file is None:
notebook_file = get_notebook_name()
exporter = nbconvert.PythonExporter()
contents, _ = exporter.from_filename(notebook_file)
with open(converted_filename, "w+") as f:
f.write(contents)
generate_context_tarball(converted_filename, output_filename)
def runtest(self):
with io.open(self.name, encoding='utf8') as nbfile:
nb = nbformat.read(nbfile, as_version=4)
_insert_get_ipython(nb)
py, resources = nbconvert.PythonExporter().from_notebook_node(nb)
py = insert_ipython_magic_content(py)
if sys.version_info[0] == 2:
# notebooks will start with "coding: utf-8", but py already unicode
py = py.encode('utf8')
if flake_check(py, self.name) != 0:
raise AssertionError
def _notebook_to_py(notebook):
exporter = nbconvert.PythonExporter()
exporter.exclude_markdown = True
exporter.exclude_output = True
exporter.exclude_raw = True
exporter.exclude_unknown = True
exporter.exclude_header = True
body, resources = exporter.from_file(notebook)
# Remove magic commands.
body = body.replace("get_ipython()", "# get_ipython()")
# The last two elements are a lint added by nbconvert (extra lines).
return body[:-2]
def _reformat_notebook(script):
"""convert a notebook to a script and remove jupyter magics
Parameters
----------
script : str
The path to the Jupyter notebook
Returns
-------
code : str
The notebook as a python script.
"""
nb = nbformat.read(script, nbformat.NO_CONVERT)
code = nbconvert.export(nbconvert.PythonExporter(), nb)[0]
new_code = []
for line in code.split('\n'):
# Exclude magics - HACK - TODO
# Matthew R. Becker 2/27/2017
# So this one is tough. The following magics
#
# %matplotlib inline
# %matplotlib notebook
#
# are only available in jupyter.
# When notebooks are converted to scripts currently (2/17/2017 w/
# version 5.1.1 of nbconvert), these magics are left in the notebook
# as normal ipython magics. However, they cannot be executed by
# ipython and cause an error. There is an open issue in nbconvert to
# exclude these things
# (see https://github.com/jupyter/nbconvert/pull/507). Until this
# issue is addressed, we have to parse them out by hand.
# The block of code here does the following.
# 1) finds jupyter magics at any indentation level (they CAN be
# indented!)
# 2) replaces them with `pass` in python (which does nothing) at the
# same indentation level
if not (line.strip().startswith("get_ipython().magic("
"'matplotlib") or
line.strip().startswith("get_ipython().run_line_magic("
"'matplotlib") or
# python 2 - ugh
line.strip().startswith("get_ipython().magic("
"u'matplotlib") or
line.strip().startswith("get_ipython().run_line_magic("
"u'matplotlib")):
new_code.append(line)
else:
new_code.append(line[:-len(line.lstrip())] + 'pass')
return '\n'.join(new_code)
def export_to_python(filename=None,
preprocessors=[OptsMagicProcessor(),
OutputMagicProcessor(),
StripMagicsProcessor()]):
filename = filename if filename else sys.argv[1]
with open(filename) as f:
nb = nbformat.read(f, nbformat.NO_CONVERT)
exporter = nbconvert.PythonExporter()
for preprocessor in preprocessors:
exporter.register_preprocessor(preprocessor)
source, meta = exporter.from_notebook_node(nb)
return source
def preprocess(self):
exporter = nbconvert.PythonExporter()
exporter.register_preprocessor(self.notebook_preprocessor, enabled=True)
contents, _ = exporter.from_filename(self.notebook_file)
converted_notebook = Path(self.notebook_file).with_suffix('.py')
if converted_notebook.exists() and not self.overwrite:
raise Exception('Default path {} exists but overwrite flag\
is False'.format(converted_notebook))
with open(converted_notebook, 'w') as f:
logging.info('Converting {} to {}'.format(self.notebook_file, converted_notebook))
f.write(contents)
self.executable = converted_notebook
return [converted_notebook]
def preprocess(self):
exporter = nbconvert.PythonExporter()
exporter.register_preprocessor(self.notebook_preprocessor, enabled=True)
contents, _ = exporter.from_filename(self.notebook_file)
converted_notebook = Path(self.notebook_file).with_suffix('.py')
if converted_notebook.exists():
dir_name = Path(self.notebook_file).parent
_, file_name = tempfile.mkstemp(suffix=".py", dir=dir_name)
converted_notebook = Path(file_name[file_name.find(str(dir_name)):])
with open(converted_notebook, 'w') as f:
f.write(contents)
self.executable = converted_notebook
return [converted_notebook]
def test_example_notebooks(example, examples_directory, monkeypatch):
from solcore.spice import SpiceSolverError
from solcore.light_source import SmartsSolverError
from solcore.absorption_calculator import RCWASolverError
import nbconvert
import nbformat
import sys
notebooks_folder = examples_directory / "notebooks"
os.chdir(notebooks_folder)
monkeypatch.setattr("builtins.input", lambda *x, **y: "y")
script = str(notebooks_folder / Path(example))
exporter = nbconvert.PythonExporter()
try:
nbcode = nbformat.reads(open(script).read(), as_version=4)
(code, _) = exporter.from_notebook_node(nbcode)
with open(script) as fp:
code = compile(code, script, "exec")
# try to emulate __main__ namespace as much as possible
# What trace python module uses https://docs.python.org/3.7/library/trace.html
globs = {
"__file__": script,
"__name__": "__main__",
"__package__": None,
"__cached__": None,
}
exec(code, globs, globs)
os.chdir(cwd)
except OSError as err:
os.chdir(cwd)
skip("Cannot run file %r because: %s" % (sys.argv[0], err))
except SystemExit:
os.chdir(cwd)
pass
except SpiceSolverError:
os.chdir(cwd)
skip("No SPICE solver found.")
except SmartsSolverError:
os.chdir(cwd)
skip("No SMARTS solver found.")
except RCWASolverError:
os.chdir(cwd)
skip("No RCWA solver found.")
def extract_py_notebook(in_file: Path) -> str:
"""Extract the python code from a given notebook"""
# we use config for importing
c = TConfig()
c.PythonExporter.preprocessors = []
# load the notebook
notebook: nbformat.NotebookNode = nbformat.read(str(in_file), as_version=4)
# TODO - any preprocessing here
# convert it
conv = nbconvert.PythonExporter(config=c)
(body, resources) = conv.from_notebook_node(notebook)
# (body, resources) = conv.from_filename(str(in_file))
# write the notebook
# writer = nbconvert.writers.FilesWriter()
# writer.write(body, resources, "output_notebook")
return body
def overwrite_module_from_notebook():
"""Overwrite graphql_example.py from same-named notebook"""
nbfilename = pkg_resources.resource_filename('graphql_example',
'graphql_example.ipynb')
notebook = nbformat.read(nbfilename, as_version=4)
exporter = nbconvert.PythonExporter()
body, *_ = exporter.from_notebook_node(notebook)
module_filename = pkg_resources.resource_filename('graphql_example',
'graphql_example.py')
with open(module_filename, 'w') as main_module:
main_module.write(body)
def run_test(project, zone, cluster, new_values): # pylint: disable=too-many-locals
# TODO([email protected]): Need to configure the notebook and test to build
# using GCB.
dirname = os.path.dirname(__file__)
if not dirname:
logging.info("__file__ doesn't apper to be absolute path.")
dirname = os.getcwd()
notebook_path = os.path.join(dirname, "TF on GKE.ipynb")
logging.info("Reading notebook %s", notebook_path)
if not os.path.exists(notebook_path):
raise ValueError("%s does not exist" % notebook_path)
with open(notebook_path) as hf:
node = nbformat.read(hf, nbformat.NO_CONVERT)
exporter = nbconvert.PythonExporter()
raw, _ = nbconvert.export(exporter, node)
credentials = GoogleCredentials.get_application_default()
gke = discovery.build("container", "v1", credentials=credentials)
lines = raw.splitlines()
modified = replace_vars(lines, new_values)
modified = strip_appendix(modified)
modified = strip_unexecutable(modified)
with tempfile.NamedTemporaryFile(suffix="notebook.py",
prefix="tmpGke",
mode="w",
delete=False) as hf:
code_path = hf.name
hf.write("\n".join(modified))
logging.info("Wrote notebook to: %s", code_path)
with open(code_path) as hf:
contents = hf.read()
logging.info("Notebook contents:\n%s", contents)
try:
runpy.run_path(code_path)
finally:
logging.info("Deleting cluster; project=%s, zone=%s, name=%s",
project, zone, cluster)
util.delete_cluster(gke, cluster, project, zone)
def preprocess(self):
"""Preprocessor the Notebook.
:return: results: the preprocessed notebook list.
"""
exporter = nbconvert.PythonExporter()
exporter.register_preprocessor(self.notebook_preprocessor,
enabled=True)
processed, _ = exporter.from_filename(self.notebook_file)
lines = []
for l in processed.splitlines():
# Get rid of multiple blank lines
if not l.strip():
if lines:
if not lines[-1]:
# last line is already blank don't add another one
continue
# strip in statements
if l.startswith("# In["):
continue
lines.append(l)
contents = "\n".join(lines)
converted_notebook = Path(self.notebook_file).with_suffix('.py')
if converted_notebook.exists() and not self.overwrite:
raise Exception('Default path {} exists but overwrite flag\
is False'.format(converted_notebook))
with open(converted_notebook, 'w') as f:
logging.info('Converting {} to {}'.format(self.notebook_file,
converted_notebook))
f.write(contents)
f.write("\n")
logging.info('Creating entry point for the class name {}'.format(
self.class_name))
f.write("""
if __name__ == "__main__":
import fire
import logging
logging.basicConfig(format='%(message)s')
logging.getLogger().setLevel(logging.INFO)
fire.Fire({0})
""".format(self.class_name))
self.executable = converted_notebook
results = [converted_notebook]
results.extend(self.input_files)
return results
def export_for_spyder(nb):
for i, cell in enumerate(nb["cells"]):
if cell["cell_type"] in ("markdown", "code"):
cell["source"] = "#%%\n" + cell["source"]
exporter = nbconvert.PythonExporter()
exporter.exclude_input_prompt = True
exporter.exclude_output_prompt = True
exporter.exclude_output = True
py = exporter.from_notebook_node(nb)
code = py[0].split("\n")
code = [line if line != "# #%%" else "#%%" for line in code]
imported_get_python = False
for i, line in enumerate(code):
if line.startswith("get_ipython()") and not imported_get_python:
code.insert(i, "from IPython import get_ipython")
imported_get_python = True
return "\n".join(code)
def notebook_run(ipynb):
with open(ipynb) as fh:
nb = nbformat.reads(fh.read(), 4)
exporter = nbconvert.PythonExporter()
try:
os.mkdir('figure')
except:
pass
# source is a tuple of python source code
# meta contains metadata
source, meta = exporter.from_notebook_node(nb)
try:
exec(source.encode())
plt.close('all')
shutil.rmtree('figure', ignore_errors=True)
return []
except:
shutil.rmtree('figure', ignore_errors=True)
return traceback.print_exc()
def _load_ipynb_modules(ipynb_path: str):
"""
First converts data_prep.ipynb to a temp .py file
Then it loads the functions from the .py file
Returns the loaded modules
"""
assert ipynb_path.endswith(".ipynb")
basename, _ = os.path.splitext(ipynb_path)
# read ipynb file and get the text
with open(ipynb_path) as ipynb_file:
nb = nbformat.reads(s=ipynb_file.read(),
as_version=nbformat.NO_CONVERT)
assert isinstance(nb, nbformat.notebooknode.NotebookNode)
# convert the .ipynb text to a string .py format
pyexporter = nbconvert.PythonExporter()
source, meta = pyexporter.from_notebook_node(nb=nb)
assert isinstance(source, str)
# parse the .py string to pick out only 'class', 'import' and 'def function's
parsed_code = ast.parse(source=source)
for node in parsed_code.body[:]:
if node.__class__ not in [
ast.ClassDef,
ast.FunctionDef,
ast.Import,
ast.ImportFrom,
]:
parsed_code.body.remove(node)
assert parsed_code.body # make sure there is a non-empty list
# import modules from the parsed .py string
module = types.ModuleType(basename)
code = compile(source=parsed_code,
filename=f"{basename}_ipynb.py",
mode="exec")
exec(code, module.__dict__)
return module
def preprocess(self):
exporter = nbconvert.PythonExporter()
exporter.register_preprocessor(self.notebook_preprocessor, enabled=True)
processed, _ = exporter.from_filename(self.notebook_file)
lines = []
for l in processed.splitlines():
# Get rid of multiple blank lines
if not l.strip():
if lines:
if not lines[-1]:
# last line is already blank don't add another one
continue
# strip in statements
if l.startswith("# In["):
continue
lines.append(l)
contents = "\n".join(lines)
converted_notebook = Path(self.notebook_file).with_suffix('.py')
if converted_notebook.exists():
dir_name = Path(self.notebook_file).parent
_, file_name = tempfile.mkstemp(suffix=".py", dir=dir_name)
converted_notebook = Path(file_name[file_name.find(str(dir_name)):])
with open(converted_notebook, 'w') as f:
f.write(contents)
f.write("\n")
f.write("""
if __name__ == "__main__":
import fire
import logging
logging.basicConfig(format='%(message)s')
logging.getLogger().setLevel(logging.INFO)
fire.Fire({0})
""".format(self.class_name))
self.executable = converted_notebook
results = [converted_notebook]
results.extend(self.input_files)
return results
def export_for_spyder(nb):
for i, cell in enumerate(nb["cells"]):
if cell["cell_type"] in ("markdown", "code"):
cell["source"] = "#%%\n" + cell["source"]
exporter = nbconvert.PythonExporter()
exporter.exclude_input_prompt = True
exporter.exclude_output_prompt = True
exporter.exclude_output = True
py = exporter.from_notebook_node(nb)
code = py[0].split("\n")
code = [line if line != "# #%%" else "#%%" for line in code]
imported_get_python = False
for i, line in enumerate(code):
if line.startswith("get_ipython()") and not imported_get_python:
code.insert(i, "from IPython import get_ipython")
imported_get_python = True
mode = bm.Mode([bm.TargetVersion.PY38, bm.TargetVersion.PY39])
code = black.format_file_contents("\n".join(code), mode=mode, fast=False)
code = isort.api.sort_code_string(code)
return code
