예제 #1
0
    def test_no_execute(self, course_dir):
        with open("nbgrader_config.py", "a") as fh:
            fh.write(
                """c.NbGrader.db_assignments = [dict(name='ps1', duedate='2015-02-02 14:58:23.948203 PST')]\n"""
            )
            fh.write("""c.NbGrader.db_students = [dict(id="foo")]""")

        self._copy_file(join("files", "test.ipynb"),
                        join(course_dir, "source", "ps1", "p1.ipynb"))
        run_nbgrader(["assign", "ps1"])

        self._copy_file(
            join("files", "test-with-output.ipynb"),
            join(course_dir, "submitted", "foo", "ps1", "p1.ipynb"))
        with open(
                join(os.path.dirname(__file__), "files",
                     "test-with-output.ipynb"), "r") as fh:
            orig_contents = reads(fh.read())

        run_nbgrader(["autograde", "ps1"])
        with open(join(course_dir, "autograded", "foo", "ps1", "p1.ipynb"),
                  "r") as fh:
            new_contents = reads(fh.read())

        different = False
        for i in range(len(orig_contents.cells)):
            orig_cell = orig_contents.cells[i]
            new_cell = new_contents.cells[i]
            if 'outputs' in orig_cell:
                if orig_cell.outputs != new_cell.outputs:
                    different = True
                    break
            elif 'outputs' in new_cell:
                different = True

        assert different

        run_nbgrader(["autograde", "ps1", "--force", "--no-execute"])
        with open(join(course_dir, "autograded", "foo", "ps1", "p1.ipynb"),
                  "r") as fh:
            new_contents = reads(fh.read())

        for i in range(len(orig_contents.cells)):
            orig_cell = orig_contents.cells[i]
            new_cell = new_contents.cells[i]
            if 'outputs' in orig_cell:
                assert orig_cell.outputs == new_cell.outputs
            else:
                assert 'outputs' not in new_cell
예제 #2
0
def main():
    for fname in sys.argv[1:]:
        nb = reads(io.open(fname, 'r').read())
        remove_outputs(nb)
        base, ext = os.path.splitext(fname)
        new_ipynb = "%s-no-output%s" % (base, ext)
        io.open(new_ipynb, 'w', encoding='utf8').write(writes(nb))
예제 #3
0
def main():
    for fname in sys.argv[1:]:
        nb = reads(io.open(fname, 'r').read())
        remove_outputs(nb)
        base, ext = os.path.splitext(fname)
        new_ipynb = "%s-no-output%s" % (base, ext)
        io.open(new_ipynb, 'w', encoding='utf8').write(writes(nb))
예제 #4
0
    def get_dependencies(self, dirnames):
        dependencies = {}

        for dirname in dirnames:
            files = glob.glob("{}/*.ipynb".format(dirname))

            for filename in files:
                modname = os.path.splitext(os.path.basename(filename))[0]
                with open(filename, "r") as fh:
                    nb = reads(fh.read())

                params = self.extract_parameters(nb)
                if '__depends__' not in params:
                    continue
                if '__dest__' not in params:
                    raise ValueError("__dest__ is not defined in {}".format(filename))

                # get sources that are specified in the file
                sources = [self.resolve_path(filename, x) for x in params['__depends__']]

                targets = params['__dest__']
                if not hasattr(targets, '__iter__'):
                    if targets is None:
                        targets = []
                    else:
                        targets = [targets]
                targets = [self.resolve_path(filename, x) for x in targets]

                dependencies[os.path.join(dirname, '{}.ipynb'.format(modname))] = {
                    'targets': targets,
                    'sources': sources
                }

        return json.dumps(dependencies, indent=2)
예제 #5
0
def test_notebook_translator(testdata, parse_rst, name):
    """Ensure that the notebook translator can correctly convert an rst doctree into
    the correct sequence of notebook cells."""

    rst = testdata(pathlib.Path("tutorial", name + ".rst")).decode("utf8")
    json = testdata(pathlib.Path("tutorial", name + ".ipynb")).decode("utf8")

    nb = nbformat.reads(json)

    # Don't worry about different minor version numbers
    nb.nbformat_minor = None

    doctree = parse_rst(rst)

    translator = tutorial.NotebookTranslator(doctree)
    doctree.walkabout(translator)

    notebook = translator.asnotebook()
    notebook.nbformat_minor = None

    # Don't worry about cell ids
    for cell in notebook.cells:
        cell.id = ""

    actual = nbformat.writes(notebook)
    expected = nbformat.writes(nb)

    assert expected == actual
예제 #6
0
    def test_no_execute(self, course_dir):
        with open("nbgrader_config.py", "a") as fh:
            fh.write("""c.NbGrader.db_assignments = [dict(name='ps1', duedate='2015-02-02 14:58:23.948203 PST')]\n""")
            fh.write("""c.NbGrader.db_students = [dict(id="foo")]""")

        self._copy_file(join("files", "test.ipynb"), join(course_dir, "source", "ps1", "p1.ipynb"))
        run_nbgrader(["assign", "ps1"])

        self._copy_file(
            join("files", "test-with-output.ipynb"), join(course_dir, "submitted", "foo", "ps1", "p1.ipynb")
        )
        with open(join(os.path.dirname(__file__), "files", "test-with-output.ipynb"), "r") as fh:
            orig_contents = reads(fh.read())

        run_nbgrader(["autograde", "ps1"])
        with open(join(course_dir, "autograded", "foo", "ps1", "p1.ipynb"), "r") as fh:
            new_contents = reads(fh.read())

        different = False
        for i in range(len(orig_contents.cells)):
            orig_cell = orig_contents.cells[i]
            new_cell = new_contents.cells[i]
            if "outputs" in orig_cell:
                if orig_cell.outputs != new_cell.outputs:
                    different = True
                    break
            elif "outputs" in new_cell:
                different = True

        assert different

        run_nbgrader(["autograde", "ps1", "--force", "--no-execute"])
        with open(join(course_dir, "autograded", "foo", "ps1", "p1.ipynb"), "r") as fh:
            new_contents = reads(fh.read())

        for i in range(len(orig_contents.cells)):
            orig_cell = orig_contents.cells[i]
            new_cell = new_contents.cells[i]
            if "outputs" in orig_cell:
                assert orig_cell.outputs == new_cell.outputs
            else:
                assert "outputs" not in new_cell
예제 #7
0
파일: extractor.py 프로젝트: henfee/nbflow
    def get_dependencies(self, dirnames):
        dependencies = {}

        for dirname in dirnames:
            files = glob.glob("{}/*.ipynb".format(dirname))

            for filename in files:
                modname = os.path.splitext(os.path.basename(filename))[0]
                with open(filename, "r") as fh:
                    nb = reads(fh.read())

                params = self.extract_parameters(nb)
                if '__depends__' not in params:
                    continue
                if '__dest__' not in params:
                    raise ValueError(
                        "__dest__ is not defined in {}".format(filename))

                # get sources that are specified in the file
                sources = [
                    self.resolve_path(filename, x)
                    for x in params['__depends__']
                ]

                targets = params['__dest__']
                if not hasattr(targets, '__iter__'):
                    if targets is None:
                        targets = []
                    else:
                        targets = [targets]
                targets = [self.resolve_path(filename, x) for x in targets]

                dependencies[os.path.join(dirname,
                                          '{}.ipynb'.format(modname))] = {
                                              'targets': targets,
                                              'sources': sources
                                          }

        return json.dumps(dependencies, indent=2)
예제 #8
0
def test_notebook_translator(testdata, name):
    """Ensure that the notebook translator can correctly convert an rst doctree into
    the correct sequence of notebook cells."""

    rst = testdata(os.path.join("doc", name + ".rst")).decode("utf8")

    json = testdata(os.path.join("doc", name + ".ipynb")).decode("utf8")
    nb = nbf.reads(json)

    # Getting docutils to generate a doctree for us appears to require some gymnastics
    # perhaps there is a better way?
    parser = Parser()
    settings = _get_mock_settings()

    reader = Reader()
    doctree = reader.read(StringInput(rst), parser, settings)

    translator = NotebookTranslator(doctree)
    doctree.walkabout(translator)

    actual = nbf.writes(translator.asnotebook())
    expected = nbf.writes(nb)
    assert expected == actual
                                  "language": "python",
                                  "name": "python3"},
                   "language_info": {
                       "codemirror_mode": {
                           "name": "ipython",
                           "version": 3},
                       "file_extension": ".py",
                       "mimetype": "text/x-python",
                       "name": "python",
                       "nbconvert_exporter": "python",
                       "pygments_lexer": "ipython3"}}

    for cell in nb.cells:
        if 'outputs' in cell:
            cell['outputs'] = []
        if 'execution_count' in cell:
            cell['execution_count'] = None
        if 'metadata' in cell:
            cell['metadata'] = {}
    return nb

if __name__ == '__main__':
    nb = v4.reads(sys.stdin.read())
    nb = strip_output(nb)
    output = v4.writes(nb)
    if type(output) == str and PY2:
        output = output.encode('utf-8')
    sys.stdout.write(output)


예제 #10
0
# Start from scratch to avoid left-over files due to renamed tutorials, changed
# cell numbers, etc.
if os.path.exists(target_dir):
    shutil.rmtree(target_dir)
os.mkdir(target_dir)

tutorials = []
for fname in sorted(glob.glob1(src_dir, '*.ipynb')):
    basename = fname[:-6]
    output_ipynb_fname = os.path.join(target_dir, fname)
    output_rst_fname = os.path.join(target_dir, basename + '.rst')

    print 'Running', fname
    with open(os.path.join(src_dir, fname), 'r') as f:
        notebook = reads(f.read())

    # The first line of the tutorial file should give the title
    title = notebook.cells[0]['source'].split('\n')[0].strip('# ')
    tutorials.append((basename, title))

    # Execute the notebook
    preprocessor = ExecutePreprocessor()
    preprocessor.allow_errors = True
    notebook, _ = preprocessor.preprocess(notebook,
                                          {'metadata': {'path': src_dir}})

    print 'Saving notebook and converting to RST'
    exporter = NotebookExporter()
    output, _ = exporter.from_notebook_node(notebook)
    with codecs.open(output_ipynb_fname, 'w', encoding='utf-8') as f:
예제 #11
0
Opens a notebook, strips its output, and writes the outputless version to the original file.

Useful mainly as a git filter or pre-commit hook for users who don't want to track output in VCS.

This does mostly the same thing as the `Clear All Output` command in the notebook UI.

LICENSE: Public Domain
"""

import io
import sys
from nbformat import v4


def strip_output(nb):
    """strip the outputs from a notebook object"""
    nb.metadata.pop('signature', None)
    for cell in nb.cells:
        if 'outputs' in cell:
            cell['outputs'] = []
        if 'prompt_number' in cell:
            cell['prompt_number'] = None
    return nb


if __name__ == '__main__':
    nb = v4.reads(sys.stdin.read())
    nb = strip_output(nb)
    sys.stdout.write(v4.writes(nb))
예제 #12
0
target_dir = os.path.abspath('../../../docs_sphinx/resources/tutorials')

# Start from scratch to avoid left-over files due to renamed tutorials, changed
# cell numbers, etc.
if os.path.exists(target_dir):
    shutil.rmtree(target_dir)
os.mkdir(target_dir)

tutorials = []
for fname in sorted(glob.glob1(src_dir, '*.ipynb')):
    basename = fname[:-6]
    output_ipynb_fname = os.path.join(target_dir, fname)
    output_rst_fname = os.path.join(target_dir, basename + '.rst')

    print 'Running', fname
    notebook = reads(open(os.path.join(src_dir, fname), 'r').read())

    # The first line of the tutorial file should give the title
    title = notebook.cells[0]['source'].split('\n')[0].strip('# ')
    tutorials.append((basename, title))

    # Execute the notebook
    preprocessor = ExecutePreprocessor()
    preprocessor.allow_errors = True
    notebook, _ = preprocessor.preprocess(notebook, {})

    print 'Saving notebook and converting to RST'
    exporter = NotebookExporter()
    output, _ = exporter.from_notebook_node(notebook)
    codecs.open(output_ipynb_fname, 'w', encoding='utf-8').write(output)
예제 #13
0
def parse_line(line):
    if not line.startswith('#'):
        return None

    ilevel = 0
    for char in line:
        if char == '#': ilevel += 1
        else: break

    name = line[ilevel:].strip()
    return ilevel, name


if __name__ == '__main__':
    with open(sys.argv[1], 'r') as nbfile:
        nb = v4.reads(nbfile.read())

    print('Contents\n=======\n---')

    for cell in nb.cells:
        if cell['cell_type'] == 'markdown':
            for line in cell['source'].splitlines():
                header = parse_line(line)
                if header is None: continue

                ilevel, name = header
                print('  '*(ilevel-1) + ' - [%s](#%s)'%(name, name.replace(' ','-')))



예제 #14
0
# Start from scratch to avoid left-over files due to renamed tutorials, changed
# cell numbers, etc.
if os.path.exists(target_dir):
    shutil.rmtree(target_dir)
os.mkdir(target_dir)

tutorials = []
for fname in sorted(glob.glob1(src_dir, '*.ipynb')):
    basename = fname[:-6]
    output_ipynb_fname = os.path.join(target_dir, fname)
    output_rst_fname = os.path.join(target_dir, basename + '.rst')

    print 'Running', fname
    with open(os.path.join(src_dir, fname), 'r') as f:
        notebook = reads(f.read())

    # The first line of the tutorial file should give the title
    title = notebook.cells[0]['source'].split('\n')[0].strip('# ')
    tutorials.append((basename, title))

    # Execute the notebook
    preprocessor = ExecutePreprocessor()
    preprocessor.allow_errors = True
    notebook, _ = preprocessor.preprocess(notebook,
                                          {'metadata': {
                                              'path': src_dir
                                          }})

    print 'Saving notebook and converting to RST'
    exporter = NotebookExporter()
예제 #15
0
(<button class='fa fa-plus icon-plus btn btn-xs btn-default'></button>), or pressing `SHIFT+ENTER` while this cell
is selected.
<div class="alert alert-block alert-warning" role="alert" style="margin: 10px">
<p><b>WARNING</b></p>
<p>Don't rely on this server for anything you want to last - your session will be
deleted after a short period of inactivity.</p>
</div>

This notebook is running on [mybinder.org](http://mybinder.org) created by the
[Freeman lab](https://www.janelia.org/lab/freeman-lab).
'''
if not os.path.exists('tutorials'):
    os.mkdir('tutorials')
for notebook in sorted(glob.glob('_tutorials/*.ipynb')):
    with open(notebook, 'r') as f:
        content = reads(f.read())
    title = content.cells[0]['source'].split('\n')[0].strip('# ')
    all_tutorials.append((title, notebook[1:].replace('\\', '/')))

    # Insert a note about Jupyter notebooks at the top with a download link
    content.cells.insert(1, new_markdown_cell(note))
    (path, filename) = os.path.split(notebook)

    with open('tutorials/' + filename, 'w') as f:
        nbf.write(content, f)

shutil.rmtree('_tutorials')


###################### GENERATE EXAMPLES NOTEBOOKS ############################
예제 #16
0
def do_stripping():
    nb = v4.reads(sys.stdin.read())
    nb = strip_output(nb)
    sys.stdout.write(v4.writes(nb))

def parse_line(line):
    if not line.startswith('#'):
        return None

    ilevel = 0
    for char in line:
        if char == '#': ilevel += 1
        else: break

    name = line[ilevel:].strip()
    return ilevel, name


if __name__ == '__main__':
    with open(sys.argv[1], 'r') as nbfile:
        nb = v4.reads(nbfile.read())

    print 'Contents\n=======\n---'

    for cell in nb.cells:
        if cell['cell_type'] == 'markdown':
            for line in cell['source'].splitlines():
                header = parse_line(line)
                if header is None: continue

                ilevel, name = header
                print '  ' * (ilevel - 1) + ' - [%s](#%s)' % (
                    name, name.replace(' ', '-'))
예제 #18
0
파일: utils.py 프로젝트: deathbeds/ktop
def load_notebook(file_name):
    """ a convenience wrapper around a "safe" notebook load
    """
    file_path = check_path(file_name)

    return reads(file_path.read_text())
예제 #19
0
To run the code below:

1. Click on the cell to select it.
2. Press `SHIFT+ENTER` on your keyboard or press the play button (<button class='fa fa-play icon-play btn btn-xs btn-default'></button>) in the toolbar above.

Feel free to create new cells using the plus button (<button class='fa fa-plus icon-plus btn btn-xs btn-default'></button>), or pressing `SHIFT+ENTER` while this cell is selected.
<div class="alert alert-warning" role="alert" style="margin: 10px">
<p>**WARNING**</p>
<p>Don't rely on this server for anything you want to last - your server will be *deleted after 10 minutes of inactivity*.</p>
</div>
'''
if not os.path.exists('tutorials'):
    os.mkdir('tutorials')
for notebook in glob.glob('_tutorials/*.ipynb'):
    with open(notebook, 'r') as f:
        content = reads(f.read())

    # Insert a note about Jupyter notebooks at the top with a download link
    content.cells.insert(1, new_markdown_cell(note))
    (path, filename) = os.path.split(notebook) #[-1]

    with open('tutorials/' + filename, 'w') as f:
        nbf.write(content, f)

shutil.rmtree('_tutorials')


magic = '''%matplotlib notebook\n'''
if not os.path.exists('examples'):
    os.mkdir('examples')
for root, subfolders, files in os.walk('_examples'):
예제 #20
0
def parse(path):
    source = read_text_file(path)
    result = {}
    if path.endswith('.html'):
        if utils.detect_blogofile(source):
            heading, info, other_html = utils.convert_blogofile(source)
            parts = utils.parse_rst(heading)
            body_html = parts['docinfo'] + other_html
            body_html = utils.pygmentize_pre_blocks(body_html)
            body_html = body_html.replace('\n</pre>', '</pre>')
            result['title'] = utils.html_parser.unescape(parts['title'])
            result['needs_disqus'] = True
            result['date'] = info['date']
            result['tags'] = info['tags']
        else:
            result['title'] = utils.find_title_in_html(source)
            body_html = SimpleTemplate(source)
            result['needs_disqus'] = False
            result['date'] = None
            result['tags'] = ()

        result['body'] = body_html
        result['next_link'] = None
        result['previous_link'] = None
        result['tags'] = [tag for tag in result['tags'] if tag]

    elif path.endswith('.md'):
        if utils.detect_blogofile(source):
            heading, info, body = utils.convert_blogofile(source)
            source = body
            result['date'] = info['date']
            result['title'] = info['title']
            result['needs_disqus'] = True
        else:
            result['needs_disqus'] = False
        result['body'] = commonmark(source)

    elif path.endswith('.rst'):
        if utils.detect_blogofile(source):
            heading, info, body = utils.convert_blogofile(source)
            source = heading + body

            result['title'] = info['title']
            del heading, info, body
            result['needs_disqus'] = True
        else:
            result['needs_disqus'] = False
        doctree = publish_doctree(source)
        docinfos = doctree.traverse(nodes.docinfo)
        docinfo = {c.tagname: str(c.children[0])
                   for i in docinfos for c in i.children}

        parts = utils.parse_rst(source)
        # parts = publish_from_doctree(source, writer_name='html',
        #                       settings_overrides={'initial_header_level': 2})
        body = parts['docinfo'] + utils.pygmentize_pre_blocks(parts['fragment'])
        result['body'] = body
        result['date'] = datetime.strptime(
            docinfo.get('date'), '%d %B %Y').date()
        if 'title' not in result:
            result['title'] = parts['title']

    elif path.endswith('.ipynb'):
        notebook = nbformat.reads(source)
        docinfo = utils.build_docinfo_block_for_notebook(notebook)
        exporter = HTMLExporter(config=None, extra_loaders=[dl],
                                filters=filters)
        exporter.template_file = 'brandon.tpl'
        #notebook = nbformat.convert(notebook, nbformat.current_nbformat)
        body, resources = exporter.from_notebook_node(notebook)
        body = body.replace('\n</pre>', '</pre>')
        body = body.replace('</h1>', '</h1>\n' + docinfo.rstrip())

        date = notebook['metadata'].get('date')
        if date is not None:
            date = datetime.strptime(date, '%d %B %Y').date()

        result['body'] = body
        result['date'] = date
        result['needs_disqus'] = notebook['metadata'].get('needs_disqus')
        result['title'] = (notebook['metadata'].get('name', None)
                           or utils.find_title_in_html(body))

    else:
        raise ValueError('unrecognized path: {}'.format(path))

    return result
예제 #21
0
def parse(path):
    source = read_text_file(path)
    result = {}
    if path.endswith('.html'):
        if utils.detect_blogofile(source):
            heading, info, other_html = utils.convert_blogofile(source)
            parts = utils.parse_rst(heading)
            body_html = parts['docinfo'] + other_html
            body_html = utils.pygmentize_pre_blocks(body_html)
            body_html = body_html.replace('\n</pre>', '</pre>')
            result['title'] = utils.html_parser.unescape(parts['title'])
            result['needs_disqus'] = True
            result['date'] = info['date']
            result['tags'] = info['tags']
        else:
            result['title'] = utils.find_title_in_html(source)
            body_html = SimpleTemplate(source)
            result['needs_disqus'] = False
            result['date'] = None
            result['tags'] = ()

        result['body'] = body_html
        result['next_link'] = None
        result['previous_link'] = None
        result['tags'] = [tag for tag in result['tags'] if tag]

    elif path.endswith('.md'):
        if utils.detect_blogofile(source):
            heading, info, body = utils.convert_blogofile(source)
            source = body
            result['date'] = info['date']
            result['title'] = info['title']
            result['needs_disqus'] = True
        else:
            result['needs_disqus'] = False
        result['body'] = commonmark(source)

    elif path.endswith('.rst'):
        if utils.detect_blogofile(source):
            heading, info, body = utils.convert_blogofile(source)
            source = heading + body

            result['title'] = info['title']
            del heading, info, body
            result['needs_disqus'] = True
        else:
            result['needs_disqus'] = False
        doctree = publish_doctree(source)
        docinfos = doctree.traverse(nodes.docinfo)
        docinfo = {
            c.tagname: str(c.children[0])
            for i in docinfos for c in i.children
        }

        parts = utils.parse_rst(source)
        # parts = publish_from_doctree(source, writer_name='html',
        #                       settings_overrides={'initial_header_level': 2})
        body = parts['docinfo'] + utils.pygmentize_pre_blocks(
            parts['fragment'])
        result['body'] = body
        result['date'] = datetime.strptime(docinfo.get('date'),
                                           '%d %B %Y').date()
        if 'title' not in result:
            result['title'] = parts['title']

    elif path.endswith('.ipynb'):
        notebook = nbformat.reads(source)
        docinfo = utils.build_docinfo_block_for_notebook(notebook)
        exporter = HTMLExporter(config=None,
                                extra_loaders=[dl],
                                filters=filters)
        exporter.template_file = 'brandon.tpl'
        #notebook = nbformat.convert(notebook, nbformat.current_nbformat)
        body, resources = exporter.from_notebook_node(notebook)
        body = body.replace('\n</pre>', '</pre>')
        body = body.replace('</h1>', '</h1>\n' + docinfo.rstrip())

        date = notebook['metadata'].get('date')
        if date is not None:
            date = datetime.strptime(date, '%d %B %Y').date()

        result['body'] = body
        result['date'] = date
        result['needs_disqus'] = notebook['metadata'].get('needs_disqus')
        result['title'] = (notebook['metadata'].get('name', None)
                           or utils.find_title_in_html(body))

    else:
        raise ValueError('unrecognized path: {}'.format(path))

    return result