def test_notebooks():
"""Run all notebooks in /docs/tutorials/ as tests."""
# Get the notebook names
root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
path = os.path.join(root, 'docs', 'tutorials')
notebooks = glob.glob(os.path.join(path, '*.ipynb'))
# Convert them to python scripts
exporter = PythonExporter()
for notebook in notebooks:
# Get the script as a string
script, _ = exporter.from_filename(notebook)
# Get rid of %matplotlib inline commands
script = script.replace("get_ipython().magic('matplotlib inline')", "")
script = script.replace(
"get_ipython().run_line_magic('matplotlib', 'inline')", "")
# Get rid of %run commands
script = re.sub("get_ipython\(\).magic\('run (.*)'\)", r"#", script)
script = re.sub("get_ipython\(\).run_line_magic\('run', '(.*)'\)",
r"#", script)
# Remove the %time wrappers
script = re.sub("get_ipython\(\).magic\('time (.*)'\)", r"\1", script)
script = re.sub("get_ipython\(\).run_line_magic\('time', '(.*)'\)",
r"\1", script)
# Remove calls to map.show()
script = re.sub("(.*)\.show()(.*)", r"#", script)
# Run it
print("Running %s..." % os.path.basename(notebook))
exec(script, globals(), globals())
pl.close('all')
def get_converted_script(input_notebook_path: str, conf: MlVToolConf) -> str:
"""
Extract notebook python content using nbconvert
"""
exporter = PythonExporter(get_config(TEMPLATE_PATH))
exporter.register_filter(name='filter_trailing_cells',
jinja_filter=filter_trailing_cells)
exporter.register_filter(name='get_formatted_cells',
jinja_filter=get_formatted_cells)
exporter.register_filter(name='get_data_from_docstring',
jinja_filter=get_data_from_docstring)
exporter.register_filter(name='sanitize_method_name',
jinja_filter=to_method_name)
resources = {'ignore_keys': conf.ignore_keys}
logging.debug(f'Template info {resources}')
try:
script_content, _ = exporter.from_filename(input_notebook_path,
resources=resources)
except Exception as e:
raise MlVToolException(e) from e
return script_content
class Extractor:
def __init__(self) -> None:
self.instructions = None
self.source = None
self.meta = None
self.python_exporter = PythonExporter()
self.imports = defaultdict(list)
def extract(self, notebook_path: str) -> str:
(self.source,
self.meta) = self.python_exporter.from_filename(notebook_path)
return self.source
def save_script(self, dest_path: str) -> None:
with open(dest_path, "w") as outfile:
outfile.writelines(self.source)
def get_imports(self, source: str) -> defaultdict:
self.instructions = dis.get_instructions(source)
for instruction in self.instructions:
if "IMPORT_NAME" in instruction.opname or "IMPORT_FROM" in instruction.opname:
self.imports[instruction.opname].append(instruction.argval)
return self.imports
def export_to_script(input_notebook_path: str, output_path: str,
conf: MlVToolConf):
"""
Export a notebook to a parameterize Python 3 script
using Jinja templates
"""
logging.info(
f'Generate Python script {output_path} from Jupyter Notebook {input_notebook_path}'
)
logging.debug(f'Global Configuration: {conf}')
logging.debug(f'Template path {TEMPLATE_PATH}')
exporter = PythonExporter(get_config(TEMPLATE_PATH))
exporter.register_filter(name='filter_trailing_cells',
jinja_filter=filter_trailing_cells)
exporter.register_filter(name='get_formatted_cells',
jinja_filter=get_formatted_cells)
exporter.register_filter(name='get_data_from_docstring',
jinja_filter=get_data_from_docstring)
exporter.register_filter(name='sanitize_method_name',
jinja_filter=to_method_name)
resources = {'ignore_keys': conf.ignore_keys}
logging.debug(f'Template info {resources}')
try:
script_content, _ = exporter.from_filename(input_notebook_path,
resources=resources)
except Exception as e:
raise MlVToolException(e) from e
if not script_content:
logging.warning('Empty notebook provided. Nothing to do.')
return
write_python_script(script_content, output_path)
logging.log(
logging.WARNING + 1,
f'Python script successfully generated in {abspath(output_path)}')
submission_path = '../Submission.html'
if sys.version_info[0] == 3 and sys.version_info[
1] >= 8 and sys.platform.startswith('win'):
asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
# Clean up
if os.path.exists(export_path):
os.remove(export_path)
# Start Notebook -> Python conversion
print("Converting to python: {0}".format(assignment_path))
exporter = PythonExporter()
# source is a tuple of python source code, meta contains metadata
(source, meta) = exporter.from_filename(assignment_path)
# skip plotting calls in the python code to speed up loading
new_source_lines = []
new_source_lines.append("#!/usr/bin/env python")
run_eval = False
for line in source.split('\n'):
if line.startswith("stop_training = False"):
print(
"Verification error. Set stop_training = True in your notebook and try again."
)
sys.exit()
if line.startswith("base_dir"):
line = "base_dir = '../'"
if line.startswith("plot_"):
line = "# {}".format(line)
def nb_to_python(nb_path):
"""convert notebook to python script"""
exporter = PythonExporter()
output, resources = exporter.from_filename(nb_path)
return output
