예제 #1
0
파일: convert.py 프로젝트: bch0w/pyatoa
def convert_nb(nbname):
    """
    Open up a Jupyter notebook, and clean outputs
    """
    # Remove file extension
    nbname = os.path.splitext(nbname)[0]

    os.system(f"runipy --o {nbname}.ipynb --matplotlib --quiet")
    os.system("rm -rf ./index_files")

    filename = f"{nbname}.ipynb"
    with io.open(filename, 'r', encoding='utf8') as f:
        nb = current.read(f, 'json')
    nb = clean_for_doc(nb)
    print("Writing to", filename)
    with io.open(filename, 'w', encoding='utf8') as f:
        current.write(nb, f, 'json')

    os.system(f"jupyter nbconvert --to rst {nbname}.ipynb")

    filename = f"{nbname}.ipynb"
    with io.open(filename, 'r', encoding='utf8') as f:
        nb = current.read(f, 'json')
    nb = strip_output(nb)
    print("Writing to", filename)
    with io.open(filename, 'w', encoding='utf8') as f:
        current.write(nb, f, 'json')
예제 #2
0
def create_model_comparison_notebook():
    nb = nbf.new_notebook()
    cells = []

    create_headline_cell('Model Report', cells)
    create_import_cell(cells)

    df_scores_all_test = pd.DataFrame()
    df_scores_all_train = pd.DataFrame()

    for model_dir in next(os.walk(BASE_MODEL_PATH))[1]:
        if model_dir.startswith('.'):
            continue
        create_headline_cell(model_dir, cells, level=2)

        create_headline_cell('Scores', cells, level=3)
        df_scores = pd.read_csv(get_model_path(model_dir, 'scores.csv'))
        df_scores.index = [model_dir]
        create_score_view_cell(df_scores, cells)
        df_scores_all_train = df_scores_all_train.append(df_scores.filter(regex=("_train")))
        df_scores_all_test = df_scores_all_test.append(df_scores.filter(regex=("_test")))

        create_headline_cell('Params', cells, level=3)
        df_scores = pd.read_csv(get_model_path(model_dir, 'params.csv'))
        df_scores.index = [model_dir]
        create_score_view_cell(df_scores, cells)

        create_headline_cell('Prediction comparison', cells, level=3)

        create_headline_cell('Train', cells, level=4)
        create_pdf_view_cell(get_model_path(model_dir, 'complete_comp_train.pdf'), cells)
        create_pdf_view_cell(get_model_path(model_dir, 'zoomed_comp_train.pdf'), cells)
        create_headline_cell('Test', cells, level=4)
        create_pdf_view_cell(get_model_path(model_dir, 'complete_comp_test.pdf'), cells)
        create_pdf_view_cell(get_model_path(model_dir, 'zoomed_comp_train.pdf'), cells)

    create_headline_cell('Overall score comparison', cells, level=2)
    create_headline_cell('Test', cells, level=3)
    create_score_view_cell(df_scores_all_test, cells)
    create_headline_cell('Train', cells, level=3)
    create_score_view_cell(df_scores_all_train, cells)

    cells = cells[:2] + cells[-5:] + cells[2:-5]

    nb['worksheets'].append(nbf.new_worksheet(cells=cells))

    with open(NOTEBOOK_NAME, 'w') as f:
        nbf.write(nb, f, 'ipynb')
    def write_report(conf_markdown_store,
                     conf_code_store,
                     markdown_store,
                     code_store,
                     report_name='my_notebook.ipynb'):

        #print 'code_store', code_store
        from nbformat import current as nbf
        nb = nbf.new_notebook()
        cells = []

        # Write conf markdown
        text = ''.join(each for each in conf_markdown_store)
        conf_markdown_cell = nbf.new_text_cell('markdown', text)
        cells.append(conf_markdown_cell)

        # Write conf code
        text = ''.join(each for each in conf_code_store)
        conf_code_cell = nbf.new_code_cell(text)
        cells.append(conf_code_cell)

        # Write main/report markdown
        text = ''.join(each for each in markdown_store)
        report_markdown = nbf.new_text_cell('markdown', text)
        cells.append(report_markdown)

        # Write main /report code
        if code_store:
            for each in code_store:
                text = '\r\n' + each
                report_code = nbf.new_code_cell(text)
                cells.append(report_code)
        nb['worksheets'].append(nbf.new_worksheet(cells=cells))
        # Writing to the notebook
        with open(report_name, 'w') as f:
            nbf.write(nb, f, 'ipynb')
예제 #4
0
def process_notebook_file(fname, action='clean', output_fname=None):
    print("Performing '{}' on: {}".format(action, fname))
    orig_wd = os.getcwd()
    with io.open(fname, 'r') as f:
        nb = current.read(f, 'json')

    if action == 'check':
        os.chdir(os.path.dirname(fname))
        run_notebook(nb)
        remove_outputs(nb)
        remove_signature(nb)
    else:
        # Clean by default
        remove_outputs(nb)
        remove_signature(nb)

    os.chdir(orig_wd)
    if output_fname is None:
        output_fname = fname
    with io.open(output_fname, 'w') as f:
        nb = current.write(nb, f, 'json')
예제 #5
0
def process_notebook_file(fname, action='clean', output_fname=None):
    print("Performing '{}' on: {}".format(action, fname))
    orig_wd = os.getcwd()
    with io.open(fname, 'r') as f:
        nb = current.read(f, 'json')

    if action == 'check':
        os.chdir(os.path.dirname(fname))
        run_notebook(nb)
        remove_outputs(nb)
        remove_signature(nb)
    elif action == 'render':
        os.chdir(os.path.dirname(fname))
        run_notebook(nb)
    else:
        # Clean by default
        remove_outputs(nb)
        remove_signature(nb)

    os.chdir(orig_wd)
    if output_fname is None:
        output_fname = fname
    with io.open(output_fname, 'w') as f:
        nb = current.write(nb, f, 'json')
예제 #6
0
"""
Usage: python remove_output.py notebook.ipynb [ > without_output.ipynb ]
Modified from remove_output by Minrk

"""
import sys
import io
import os
from nbformat.current import read, write


def remove_outputs(nb):
    """remove the outputs from a notebook"""
    for ws in nb.worksheets:
        for cell in ws.cells:
            if cell.cell_type == 'code':
                cell.outputs = []

if __name__ == '__main__':
    fname = sys.argv[1]
    with io.open(fname, 'r') as f:
        nb = read(f, 'json')
    remove_outputs(nb)
    base, ext = os.path.splitext(fname)
    new_ipynb = "%s_removed%s" % (base, ext)
    with io.open(new_ipynb, 'w', encoding='utf8') as f:
        write(nb, f, version=4)
    print "wrote %s" % new_ipynb
예제 #7
0
for cell in template['cells']:
    cell_type = cell['cell_type']
    source = cell['source']
    if cell_type == 'markdown':
        out_cells.append(nbf.new_text_cell('markdown', source))
    else:
        out_cells.append(nbf.new_code_cell(source))


def make_title(s):
    return s.replace('-', ' ').replace('_', ' ').title()


for style_str in sorted(plt.style.available):
    out_cells.append(
        nbf.new_text_cell(
            'markdown', "### {} (`'{}'`)".format(make_title(style_str),
                                                 style_str)))
    out_cells.append(
        nbf.new_code_cell("""\
with plt.style.context(('{0}')):
    make_plots('{0}')\
        """.format(style_str).splitlines(keepends=True)))

nb['worksheets'].append(nbf.new_worksheet(cells=out_cells))

with open(output_path, 'w') as f:
    nbf.write(nb, f, 'ipynb')

# subprocess.call(["ipython", "-c", "%run {}".format(filename)])
예제 #8
0
#!/usr/bin/env python


def strip_output(nb):
    for ws in nb.worksheets:
        for cell in ws.cells:
            if hasattr(cell, "outputs"):
                cell.outputs = []
            if hasattr(cell, "prompt_number"):
                del cell["prompt_number"]


if __name__ == "__main__":
    from sys import stdin, stdout
    from nbformat.current import read, write

    nb = read(stdin, "ipynb")
    strip_output(nb)
    write(nb, stdout, "ipynb")
    stdout.write("\n")
예제 #9
0
out_cells = []

for cell in template['cells']:
    cell_type = cell['cell_type']
    source = cell['source']
    if cell_type == 'markdown':
        out_cells.append(nbf.new_text_cell('markdown', source))
    else:
        out_cells.append(nbf.new_code_cell(source))


def make_title(s):
    return s.replace('-', ' ').replace('_', ' ').title()


for style_str in sorted(plt.style.available):
    out_cells.append(nbf.new_text_cell('markdown', "### {} (`'{}'`)".format(make_title(style_str), style_str)))
    out_cells.append(nbf.new_code_cell(
        """\
with plt.style.context(('{0}')):
    make_plots('{0}')\
        """.format(style_str).splitlines(keepends=True)
    ))

nb['worksheets'].append(nbf.new_worksheet(cells=out_cells))

with open(output_path, 'w') as f:
    nbf.write(nb, f, 'ipynb')

# subprocess.call(["ipython", "-c", "%run {}".format(filename)])
예제 #10
0
"""
Usage: python remove_output.py notebook.ipynb [ > without_output.ipynb ]
Modified from remove_output by Minrk

"""
import sys
import io
import os
from nbformat.current import read, write


def remove_outputs(nb):
    """remove the outputs from a notebook"""
    for ws in nb.worksheets:
        for cell in ws.cells:
            if cell.cell_type == 'code':
                cell.outputs = []
                for key in 'execution_count', 'prompt_number':
                    if key  in cell: del cell[key]

if __name__ == '__main__':
    for fname in sys.argv[1:]:
        bakname = fname+'.bak'
        os.rename(fname, bakname)
        with io.open(bakname, 'r') as f:
            nb = read(f, 'json')
        remove_outputs(nb)
        #base, ext = os.path.splitext(fname)
        with io.open(fname, 'w', encoding='utf8') as f:
            write(nb, f, 'json', version=4)
        print("wrote cleaned %s and backward copy %s" % (fname, bakname))
예제 #11
0
"""
Usage: python remove_output.py notebook.ipynb [ > without_output.ipynb ]
Modified from remove_output by Minrk

"""
import sys
import io
import os
from nbformat.current import read, write


def remove_outputs(nb):
    """remove the outputs from a notebook"""
    for ws in nb.worksheets:
        for cell in ws.cells:
            if cell.cell_type == 'code':
                cell.outputs = []


if __name__ == '__main__':
    fname = sys.argv[1]
    with io.open(fname, 'r') as f:
        nb = read(f, 'json')
    remove_outputs(nb)
    base, ext = os.path.splitext(fname)
    new_ipynb = "%s_removed%s" % (base, ext)
    with io.open(new_ipynb, 'w', encoding='utf8') as f:
        write(nb, f, 'json')
    print "wrote %s" % new_ipynb
"""
Usage: python remove_output.py notebook.ipynb [ > without_output.ipynb ]
Modified from remove_output by Minrk

"""
import sys
import io
import os
from nbformat.current import read, write


def remove_outputs(nb):
    """remove the outputs from a notebook"""
    for ws in nb.worksheets:
        for cell in ws.cells:
            if cell.cell_type == 'code':
                cell.outputs = []

if __name__ == '__main__':
    for fname in sys.argv[1:]:
        with io.open(fname, 'r') as f:
            nb = read(f, 'json')
        remove_outputs(nb)
        #base, ext = os.path.splitext(fname)
        #new_ipynb = "%s_removed%s" % (base, ext)
        with io.open(fname, 'w', encoding='utf8') as f:
            write(nb, f, 'json')
        print "wrote %s" % fname