def convert_nb(nbname):
"""
Open up a Jupyter notebook, and clean outputs
"""
# Remove file extension
nbname = os.path.splitext(nbname)[0]
os.system(f"runipy --o {nbname}.ipynb --matplotlib --quiet")
os.system("rm -rf ./index_files")
filename = f"{nbname}.ipynb"
with io.open(filename, 'r', encoding='utf8') as f:
nb = current.read(f, 'json')
nb = clean_for_doc(nb)
print("Writing to", filename)
with io.open(filename, 'w', encoding='utf8') as f:
current.write(nb, f, 'json')
os.system(f"jupyter nbconvert --to rst {nbname}.ipynb")
filename = f"{nbname}.ipynb"
with io.open(filename, 'r', encoding='utf8') as f:
nb = current.read(f, 'json')
nb = strip_output(nb)
print("Writing to", filename)
with io.open(filename, 'w', encoding='utf8') as f:
current.write(nb, f, 'json')
def create_model_comparison_notebook():
nb = nbf.new_notebook()
cells = []
create_headline_cell('Model Report', cells)
create_import_cell(cells)
df_scores_all_test = pd.DataFrame()
df_scores_all_train = pd.DataFrame()
for model_dir in next(os.walk(BASE_MODEL_PATH))[1]:
if model_dir.startswith('.'):
continue
create_headline_cell(model_dir, cells, level=2)
create_headline_cell('Scores', cells, level=3)
df_scores = pd.read_csv(get_model_path(model_dir, 'scores.csv'))
df_scores.index = [model_dir]
create_score_view_cell(df_scores, cells)
df_scores_all_train = df_scores_all_train.append(df_scores.filter(regex=("_train")))
df_scores_all_test = df_scores_all_test.append(df_scores.filter(regex=("_test")))
create_headline_cell('Params', cells, level=3)
df_scores = pd.read_csv(get_model_path(model_dir, 'params.csv'))
df_scores.index = [model_dir]
create_score_view_cell(df_scores, cells)
create_headline_cell('Prediction comparison', cells, level=3)
create_headline_cell('Train', cells, level=4)
create_pdf_view_cell(get_model_path(model_dir, 'complete_comp_train.pdf'), cells)
create_pdf_view_cell(get_model_path(model_dir, 'zoomed_comp_train.pdf'), cells)
create_headline_cell('Test', cells, level=4)
create_pdf_view_cell(get_model_path(model_dir, 'complete_comp_test.pdf'), cells)
create_pdf_view_cell(get_model_path(model_dir, 'zoomed_comp_train.pdf'), cells)
create_headline_cell('Overall score comparison', cells, level=2)
create_headline_cell('Test', cells, level=3)
create_score_view_cell(df_scores_all_test, cells)
create_headline_cell('Train', cells, level=3)
create_score_view_cell(df_scores_all_train, cells)
cells = cells[:2] + cells[-5:] + cells[2:-5]
nb['worksheets'].append(nbf.new_worksheet(cells=cells))
with open(NOTEBOOK_NAME, 'w') as f:
nbf.write(nb, f, 'ipynb')
def write_report(conf_markdown_store,
conf_code_store,
markdown_store,
code_store,
report_name='my_notebook.ipynb'):
#print 'code_store', code_store
from nbformat import current as nbf
nb = nbf.new_notebook()
cells = []
# Write conf markdown
text = ''.join(each for each in conf_markdown_store)
conf_markdown_cell = nbf.new_text_cell('markdown', text)
cells.append(conf_markdown_cell)
# Write conf code
text = ''.join(each for each in conf_code_store)
conf_code_cell = nbf.new_code_cell(text)
cells.append(conf_code_cell)
# Write main/report markdown
text = ''.join(each for each in markdown_store)
report_markdown = nbf.new_text_cell('markdown', text)
cells.append(report_markdown)
# Write main /report code
if code_store:
for each in code_store:
text = '\r\n' + each
report_code = nbf.new_code_cell(text)
cells.append(report_code)
nb['worksheets'].append(nbf.new_worksheet(cells=cells))
# Writing to the notebook
with open(report_name, 'w') as f:
nbf.write(nb, f, 'ipynb')
def process_notebook_file(fname, action='clean', output_fname=None):
print("Performing '{}' on: {}".format(action, fname))
orig_wd = os.getcwd()
with io.open(fname, 'r') as f:
nb = current.read(f, 'json')
if action == 'check':
os.chdir(os.path.dirname(fname))
run_notebook(nb)
remove_outputs(nb)
remove_signature(nb)
else:
# Clean by default
remove_outputs(nb)
remove_signature(nb)
os.chdir(orig_wd)
if output_fname is None:
output_fname = fname
with io.open(output_fname, 'w') as f:
nb = current.write(nb, f, 'json')
def process_notebook_file(fname, action='clean', output_fname=None):
print("Performing '{}' on: {}".format(action, fname))
orig_wd = os.getcwd()
with io.open(fname, 'r') as f:
nb = current.read(f, 'json')
if action == 'check':
os.chdir(os.path.dirname(fname))
run_notebook(nb)
remove_outputs(nb)
remove_signature(nb)
elif action == 'render':
os.chdir(os.path.dirname(fname))
run_notebook(nb)
else:
# Clean by default
remove_outputs(nb)
remove_signature(nb)
os.chdir(orig_wd)
if output_fname is None:
output_fname = fname
with io.open(output_fname, 'w') as f:
nb = current.write(nb, f, 'json')
"""
Usage: python remove_output.py notebook.ipynb [ > without_output.ipynb ]
Modified from remove_output by Minrk
"""
import sys
import io
import os
from nbformat.current import read, write
def remove_outputs(nb):
"""remove the outputs from a notebook"""
for ws in nb.worksheets:
for cell in ws.cells:
if cell.cell_type == 'code':
cell.outputs = []
if __name__ == '__main__':
fname = sys.argv[1]
with io.open(fname, 'r') as f:
nb = read(f, 'json')
remove_outputs(nb)
base, ext = os.path.splitext(fname)
new_ipynb = "%s_removed%s" % (base, ext)
with io.open(new_ipynb, 'w', encoding='utf8') as f:
write(nb, f, version=4)
print "wrote %s" % new_ipynb
for cell in template['cells']:
cell_type = cell['cell_type']
source = cell['source']
if cell_type == 'markdown':
out_cells.append(nbf.new_text_cell('markdown', source))
else:
out_cells.append(nbf.new_code_cell(source))
def make_title(s):
return s.replace('-', ' ').replace('_', ' ').title()
for style_str in sorted(plt.style.available):
out_cells.append(
nbf.new_text_cell(
'markdown', "### {} (`'{}'`)".format(make_title(style_str),
style_str)))
out_cells.append(
nbf.new_code_cell("""\
with plt.style.context(('{0}')):
make_plots('{0}')\
""".format(style_str).splitlines(keepends=True)))
nb['worksheets'].append(nbf.new_worksheet(cells=out_cells))
with open(output_path, 'w') as f:
nbf.write(nb, f, 'ipynb')
# subprocess.call(["ipython", "-c", "%run {}".format(filename)])
#!/usr/bin/env python
def strip_output(nb):
for ws in nb.worksheets:
for cell in ws.cells:
if hasattr(cell, "outputs"):
cell.outputs = []
if hasattr(cell, "prompt_number"):
del cell["prompt_number"]
if __name__ == "__main__":
from sys import stdin, stdout
from nbformat.current import read, write
nb = read(stdin, "ipynb")
strip_output(nb)
write(nb, stdout, "ipynb")
stdout.write("\n")
out_cells = []
for cell in template['cells']:
cell_type = cell['cell_type']
source = cell['source']
if cell_type == 'markdown':
out_cells.append(nbf.new_text_cell('markdown', source))
else:
out_cells.append(nbf.new_code_cell(source))
def make_title(s):
return s.replace('-', ' ').replace('_', ' ').title()
for style_str in sorted(plt.style.available):
out_cells.append(nbf.new_text_cell('markdown', "### {} (`'{}'`)".format(make_title(style_str), style_str)))
out_cells.append(nbf.new_code_cell(
"""\
with plt.style.context(('{0}')):
make_plots('{0}')\
""".format(style_str).splitlines(keepends=True)
))
nb['worksheets'].append(nbf.new_worksheet(cells=out_cells))
with open(output_path, 'w') as f:
nbf.write(nb, f, 'ipynb')
# subprocess.call(["ipython", "-c", "%run {}".format(filename)])
"""
Usage: python remove_output.py notebook.ipynb [ > without_output.ipynb ]
Modified from remove_output by Minrk
"""
import sys
import io
import os
from nbformat.current import read, write
def remove_outputs(nb):
"""remove the outputs from a notebook"""
for ws in nb.worksheets:
for cell in ws.cells:
if cell.cell_type == 'code':
cell.outputs = []
for key in 'execution_count', 'prompt_number':
if key in cell: del cell[key]
if __name__ == '__main__':
for fname in sys.argv[1:]:
bakname = fname+'.bak'
os.rename(fname, bakname)
with io.open(bakname, 'r') as f:
nb = read(f, 'json')
remove_outputs(nb)
#base, ext = os.path.splitext(fname)
with io.open(fname, 'w', encoding='utf8') as f:
write(nb, f, 'json', version=4)
print("wrote cleaned %s and backward copy %s" % (fname, bakname))
"""
Usage: python remove_output.py notebook.ipynb [ > without_output.ipynb ]
Modified from remove_output by Minrk
"""
import sys
import io
import os
from nbformat.current import read, write
def remove_outputs(nb):
"""remove the outputs from a notebook"""
for ws in nb.worksheets:
for cell in ws.cells:
if cell.cell_type == 'code':
cell.outputs = []
if __name__ == '__main__':
fname = sys.argv[1]
with io.open(fname, 'r') as f:
nb = read(f, 'json')
remove_outputs(nb)
base, ext = os.path.splitext(fname)
new_ipynb = "%s_removed%s" % (base, ext)
with io.open(new_ipynb, 'w', encoding='utf8') as f:
write(nb, f, 'json')
print "wrote %s" % new_ipynb
"""
Usage: python remove_output.py notebook.ipynb [ > without_output.ipynb ]
Modified from remove_output by Minrk
"""
import sys
import io
import os
from nbformat.current import read, write
def remove_outputs(nb):
"""remove the outputs from a notebook"""
for ws in nb.worksheets:
for cell in ws.cells:
if cell.cell_type == 'code':
cell.outputs = []
if __name__ == '__main__':
for fname in sys.argv[1:]:
with io.open(fname, 'r') as f:
nb = read(f, 'json')
remove_outputs(nb)
#base, ext = os.path.splitext(fname)
#new_ipynb = "%s_removed%s" % (base, ext)
with io.open(fname, 'w', encoding='utf8') as f:
write(nb, f, 'json')
print "wrote %s" % fname