def copy_cells(from_notebook='', from_cells=(0, 0), to_notebook='', at_cell=0):

    if '.ipynb' not in from_notebook:
        from_notebook += '.ipynb'
    if '.ipynb' not in to_notebook:
        to_notebook += '.ipynb'

    with open(from_notebook) as nb1_f:
        nb1_raw = nb1_f.read()

    with open(to_notebook) as nb2_f:
        nb2_raw = nb2_f.read()

    nb1 = nbformat.reads(nb1_raw, as_version=ipy_ver)
    nb2 = nbformat.reads(nb2_raw, as_version=ipy_ver)

    start_id, end_id = from_cells
    copied_cells = nb1['worksheets'][0]['cells'][start_id:end_id]

    active_cells = nb2['worksheets'][0]['cells']
    nb2['worksheets'][0][
        'cells'] = active_cells[:at_cell] + copied_cells + active_cells[
            at_cell:]

    nb2_modified_raw = nbformat.writes(nb2, as_version=ipy_ver)

    with open(to_notebook, 'w') as nb2_f:
        nb2_f.write(nb2_modified_raw)
def copy_cells(from_notebook='', from_cells=(0, 0), to_notebook='', at_cell=0):

    if '.ipynb' not in from_notebook:
        from_notebook += '.ipynb'
    if '.ipynb' not in to_notebook:
        to_notebook += '.ipynb'

    with open(from_notebook) as nb1_f:
        nb1_raw = nb1_f.read()

    with open(to_notebook) as nb2_f:
        nb2_raw = nb2_f.read()

    nb1 = nbformat.reads(nb1_raw, as_version=ipy_ver)
    nb2 = nbformat.reads(nb2_raw, as_version=ipy_ver)

    start_id, end_id = from_cells
    copied_cells = nb1['worksheets'][0]['cells'][start_id:end_id]

    active_cells = nb2['worksheets'][0]['cells']
    nb2['worksheets'][0]['cells'] = active_cells[:at_cell] + copied_cells + active_cells[at_cell:]

    nb2_modified_raw = nbformat.writes(nb2, as_version=ipy_ver)

    with open(to_notebook, 'w') as nb2_f:
        nb2_f.write(nb2_modified_raw)
Example #3
0
 def testRunNotebooks(self):
     notebook_dir = path.join('tests', 'input')
     for notebook_path in glob(path.join(notebook_dir, '*.ipynb')):
         notebook_file = path.basename(notebook_path)
         print(notebook_file)
         expected_file = path.join('tests', 'expected', notebook_file)
         notebook = ""
         with open(notebook_path) as notebook_file:
             notebook = notebook_file.read()
         try:
             # IPython 3
             notebook = reads(notebook, 3)
         except (TypeError, NBFormatError):
             # IPython 2
             notebook = reads(notebook, 'json')
         runner = NotebookRunner(notebook, working_dir=notebook_dir)
         runner.run_notebook(True)
         expected = ""
         with open(expected_file) as notebook_file:
             expected = notebook_file.read()
         try:
             # IPython 3
             expected = reads(expected, 3)
         except (TypeError, NBFormatError):
             # IPython 2
             expected = reads(expected, 'json')
         self.assert_notebooks_equal(expected, runner.nb)
Example #4
0
 def testRunNotebooks(self):
     notebook_dir = path.join('tests', 'input')
     for notebook_path in glob(path.join(notebook_dir, '*.ipynb')):
         notebook_file = path.basename(notebook_path)
         print(notebook_file)
         expected_file = path.join('tests', 'expected', notebook_file)
         notebook = ""
         with open(notebook_path) as notebook_file:
             notebook = notebook_file.read()
         try:
             # IPython 3
             notebook = reads(notebook, 3)
         except (TypeError, NBFormatError):
             # IPython 2
             notebook = reads(notebook, 'json')
         runner = NotebookRunner(notebook, working_dir=notebook_dir)
         runner.run_notebook(True)
         expected = ""
         with open(expected_file) as notebook_file:
             expected = notebook_file.read()
         try:
             # IPython 3
             expected = reads(expected, 3)
         except (TypeError, NBFormatError):
             # IPython 2
             expected = reads(expected, 'json')
         self.assert_notebooks_equal(expected, runner.nb)
Example #5
0
    def diff_code(self, revision1, revision2):
        self.keeper._checkout(revision1)

        nb1 = current.reads(
            open(
                os.path.join(
                    self.keeper.work_dir,
                    self.keeper.project_name,
                    self.keeper.project_config['internal-path'],
                    self.notebook,
                    "notebook" + ".ipynb"),
                "r").read(),
            "ipynb")

        self.keeper._checkout(revision2)
        nb2 = current.reads(
            open(
                os.path.join(
                    self.keeper.work_dir,
                    self.keeper.project_name,
                    self.keeper.project_config['internal-path'],
                    self.notebook,
                    "notebook" + ".ipynb"),
                "r").read(),
            "ipynb")

        result_nb = notebook_diff(nb1, nb2)
        filename_placeholder = "{}-{}: {} and {}".format(
            self.keeper.project_name, self.notebook, revision1, revision2)
        app.add_notebook(result_nb, filename_placeholder)
        print "This diff have number {}".format(len(app.notebooks) - 1)
Example #6
0
def export_nb(nb_json, format):
    import IPython.nbconvert as nbconvert
    import IPython.nbformat as nbformat

    nb = nbformat.reads(nb_json, nbformat.NO_CONVERT)
    output = nbconvert.export_by_name(format, nb)
    print(output[0])
Example #7
0
def check_single_notebook(notebook_filename, timeout=500):
    """
    Checks single notebook being given its full name
    (executes cells one-by-one checking there are no exceptions, nothing more is guaranteed)
    """
    # do we need validation of JSON?
    with open(notebook_filename) as notebook_file:
        notebook_content = nbformat.reads(notebook_file.read(), as_version=nbformat.current_nbformat)
        os.chdir(os.path.dirname(notebook_filename))
        _, client = manager.start_new_kernel()

        for cell in notebook_content.cells:
            if cell.cell_type == 'code':
                message_id = client.execute(cell.source)
                try:
                    message = client.get_shell_msg(message_id, timeout=timeout)
                except Empty:
                    raise RuntimeError("Cell timed out: \n {}".format(cell.source))

                if message['content']['status'] != 'ok':
                    traceback = message['content']['traceback']
                    description = "Cell failed: '{}'\n\n Traceback:\n{}".format(cell.source, '\n'.join(traceback))
                    raise RuntimeError(description)

        client.stop_channels()
Example #8
0
def check_single_notebook(notebook_filename, timeout=500):
    """
    Checks single notebook being given its full name
    (executes cells one-by-one checking there are no exceptions, nothing more is guaranteed)
    """
    # do we need validation of JSON?
    with open(notebook_filename) as notebook_file:
        notebook_content = nbformat.reads(notebook_file.read(),
                                          as_version=nbformat.current_nbformat)
        os.chdir(os.path.dirname(notebook_filename))
        _, client = manager.start_new_kernel()

        for cell in notebook_content.cells:
            if cell.cell_type == 'code':
                message_id = client.execute(cell.source)
                try:
                    message = client.get_shell_msg(message_id, timeout=timeout)
                except Empty:
                    raise RuntimeError("Cell timed out: \n {}".format(
                        cell.source))

                if message['content']['status'] != 'ok':
                    traceback = message['content']['traceback']
                    description = "Cell failed: '{}'\n\n Traceback:\n{}".format(
                        cell.source, '\n'.join(traceback))
                    raise RuntimeError(description)

        client.stop_channels()
Example #9
0
    def _notebook_model(self, path, content=True):
        """Build a notebook model from shock node

        if content is requested, the notebook content will be populated
        as a JSON structure (not double-serialized)
        """
        self._get_notebook_list()
        model = self._base_model(path)
        if model['name'] not in self.nb_list:
            raise web.HTTPError(400, u"Unavailable Notebook: %s" % (model['name']))
        
        model['type'] = 'notebook'
        model['last_modified'] = dateutil.parser.parse(self.nb_list[model['name']]['attributes']['last_modified'])
        model['last_modified'].replace(tzinfo=tz.UTC)
        model['created'] = dateutil.parser.parse(self.nb_list[model['name']]['attributes']['created'])
        model['created'].replace(tzinfo=tz.UTC)
        
        if content:
            try:
                node_path = "/%s?download" %self.nb_list[model['name']]['id']
                node_data = self._get_shock_node(node_path, 'data')
            except:
                raise web.HTTPError(500, u'Notebook cannot be read')
            try:
                nb = nbformat.reads(node_data, as_version=4)
            except Exception as e:
                raise web.HTTPError(400, u"Unreadable Notebook: %s %r" % (model['name'], e))
            self.mark_trusted_cells(nb, model['name'])
            model['content'] = nb
            model['format'] = 'json'
            self.validate_notebook_model(model)
        return model
Example #10
0
    def collect(self):
        with self.fspath.open() as f:
            payload = f.read()
        self.notebook_folder = self.fspath.dirname
        try:
            # Ipython 3
            self.nb = reads(payload, 3)
        except (TypeError, NBFormatError):
            # Ipython 2
            self.nb = reads(payload, 'json')
        self.runner = NotebookRunner(self.nb)

        cell_num = 1

        for cell in self.runner.iter_code_cells():
            yield IPyNbCell(self.name, self, cell_num, cell)
            cell_num += 1
Example #11
0
    def collect(self):
        with self.fspath.open() as f:
            payload = f.read()
        self.notebook_folder = self.fspath.dirname
        try:
            # Ipython 3
            self.nb = reads(payload, 3)
        except (TypeError, NBFormatError):
            # Ipython 2
            self.nb = reads(payload, 'json')
        self.runner = NotebookRunner(self.nb)

        cell_num = 1

        for cell in self.runner.iter_code_cells():
            yield IPyNbCell(self.name, self, cell_num, cell)
            cell_num += 1
Example #12
0
    def create_post(self, path, **kw):
        """Create a new post."""
        if flag is None:
            req_missing(['ipython[notebook]>=2.0.0'], 'build this site (compile ipynb)')
        content = kw.pop('content', None)
        onefile = kw.pop('onefile', False)
        kernel = kw.pop('ipython_kernel', None)
        # is_page is not needed to create the file
        kw.pop('is_page', False)

        metadata = {}
        metadata.update(self.default_metadata)
        metadata.update(kw)

        makedirs(os.path.dirname(path))

        if content.startswith("{"):
            # imported .ipynb file, guaranteed to start with "{" because it’s JSON.
            nb = nbformat.reads(content, current_nbformat)
        else:
            if IPython.version_info[0] >= 3:
                nb = nbformat.v4.new_notebook()
                nb["cells"] = [nbformat.v4.new_markdown_cell(content)]
            else:
                nb = nbformat.new_notebook()
                nb["worksheets"] = [nbformat.new_worksheet(cells=[nbformat.new_text_cell('markdown', [content])])]

            if kernelspec is not None:
                if kernel is None:
                    kernel = self.default_kernel
                    self.logger.notice('No kernel specified, assuming "{0}".'.format(kernel))

                IPYNB_KERNELS = {}
                ksm = kernelspec.KernelSpecManager()
                for k in ksm.find_kernel_specs():
                    IPYNB_KERNELS[k] = ksm.get_kernel_spec(k).to_dict()
                    IPYNB_KERNELS[k]['name'] = k
                    del IPYNB_KERNELS[k]['argv']

                if kernel not in IPYNB_KERNELS:
                    self.logger.error('Unknown kernel "{0}". Maybe you mispelled it?'.format(kernel))
                    self.logger.info("Available kernels: {0}".format(", ".join(sorted(IPYNB_KERNELS))))
                    raise Exception('Unknown kernel "{0}"'.format(kernel))

                nb["metadata"]["kernelspec"] = IPYNB_KERNELS[kernel]
            else:
                # Older IPython versions don’t need kernelspecs.
                pass

        if onefile:
            nb["metadata"]["nikola"] = metadata

        with io.open(path, "w+", encoding="utf8") as fd:
            if IPython.version_info[0] >= 3:
                nbformat.write(nb, fd, 4)
            else:
                nbformat.write(nb, fd, 'ipynb')
Example #13
0
    def finish_notebook(self, json_notebook, download_url, provider_url=None, 
                        provider_icon=None, provider_label=None, msg=None,
                        breadcrumbs=None, public=False, format=None, request=None):
        """render a notebook from its JSON body.

        download_url is required, provider_url is not.

        msg is extra information for the log message when rendering fails.
        """

        if msg is None:
            msg = download_url

        try:
            nb = reads(json_notebook, current_nbformat)
        except ValueError:
            app_log.error("Failed to render %s", msg, exc_info=True)
            raise web.HTTPError(400, "Error reading JSON notebook")

        try:
            app_log.debug("Requesting render of %s", download_url)
            with self.time_block("Rendered %s" % download_url):
                app_log.info("rendering %d B notebook from %s", len(json_notebook), download_url)
                nbhtml, config = yield self.pool.submit(render_notebook,
                    self.formats[format], nb, download_url,
                    config=self.config,
                )
        except NbFormatError as e:
            app_log.error("Invalid notebook %s: %s", msg, e)
            raise web.HTTPError(400, str(e))
        except Exception as e:
            app_log.error("Failed to render %s", msg, exc_info=True)
            raise web.HTTPError(400, str(e))
        else:
            app_log.debug("Finished render of %s", download_url)

        html = self.render_template(
            "formats/%s.html" % format,
            body=nbhtml,
            nb=nb,
            download_url=download_url,
            provider_url=provider_url,
            provider_label=provider_label,
            provider_icon=provider_icon,
            format=self.format,
            default_format=self.default_format,
            format_prefix=format_prefix,
            formats=dict(self.filter_formats(nb, json_notebook)),
            format_base=self.request.uri.replace(self.format_prefix, ""),
            date=datetime.utcnow().strftime(date_fmt),
            breadcrumbs=breadcrumbs,
            **config)

        yield self.cache_and_finish(html)

        # Index notebook
        self.index.index_notebook(download_url, nb, public)
Example #14
0
    def finish_notebook(self, json_notebook, download_url, provider_url=None, 
                        provider_icon=None, provider_label=None, msg=None,
                        breadcrumbs=None, public=False, format=None, request=None):
        """render a notebook from its JSON body.

        download_url is required, provider_url is not.

        msg is extra information for the log message when rendering fails.
        """

        if msg is None:
            msg = download_url

        try:
            nb = reads(json_notebook, current_nbformat)
        except ValueError:
            app_log.error("Failed to render %s", msg, exc_info=True)
            raise web.HTTPError(400, "Error reading JSON notebook")

        try:
            app_log.debug("Requesting render of %s", download_url)
            with self.time_block("Rendered %s" % download_url):
                app_log.info("rendering %d B notebook from %s", len(json_notebook), download_url)
                nbhtml, config = yield self.pool.submit(render_notebook,
                    self.formats[format], nb, download_url,
                    config=self.config,
                )
        except NbFormatError as e:
            app_log.error("Invalid notebook %s: %s", msg, e)
            raise web.HTTPError(400, str(e))
        except Exception as e:
            app_log.error("Failed to render %s", msg, exc_info=True)
            raise web.HTTPError(400, str(e))
        else:
            app_log.debug("Finished render of %s", download_url)

        html = self.render_template(
            "formats/%s.html" % format,
            body=nbhtml,
            nb=nb,
            download_url=download_url,
            provider_url=provider_url,
            provider_label=provider_label,
            provider_icon=provider_icon,
            format=self.format,
            default_format=self.default_format,
            format_prefix=format_prefix,
            formats=dict(self.filter_formats(nb, json_notebook)),
            format_base=self.request.uri.replace(self.format_prefix, ""),
            date=datetime.utcnow().strftime(date_fmt),
            breadcrumbs=breadcrumbs,
            **config)

        yield self.cache_and_finish(html)

        # Index notebook
        self.index.index_notebook(download_url, nb, public)
Example #15
0
 def testUseCLI(self):
     notebook_dir = path.join('tests', 'expected')
     for notebook_path in glob(path.join(notebook_dir, '*.ipynb')):
         notebook_file = path.basename(notebook_path)
         print(notebook_file)
         expected = ""
         with open(notebook_path) as notebook_file:
             expected = notebook_file.read()
         try:
             # IPython 3
             expected = reads(expected, 3)
         except (TypeError, NBFormatError):
             # IPython 2
             expected = reads(expected, 'json')
         exit_code = 1
         argv = sys.argv
         stdout = sys.stdout
         stderr = sys.stderr
         try:
             with open(devnull, "w") as devnull_filehandle:
                 sys.stdout = sys.stderr = devnull_filehandle
                 sys.argv = [
                     "runipy",
                     "-o", notebook_path,
                     "--html", notebook_path.replace(".ipynb", ".html")
                 ]
                 main()
         except SystemExit as e:
             exit_code = e.code
         finally:
             sys.argv = argv
             sys.stdout = stdout
             sys.stderr = stderr
         notebook = ""
         with open(notebook_path) as notebook_file:
             notebook = notebook_file.read()
         try:
             # IPython 3
             notebook = reads(notebook, 3)
         except (TypeError, NBFormatError):
             # IPython 2
             notebook = reads(notebook, 'json')
         self.assert_notebooks_equal(expected, notebook)
Example #16
0
 def testUseCLI(self):
     notebook_dir = path.join('tests', 'expected')
     for notebook_path in glob(path.join(notebook_dir, '*.ipynb')):
         notebook_file = path.basename(notebook_path)
         print(notebook_file)
         expected = ""
         with open(notebook_path) as notebook_file:
             expected = notebook_file.read()
         try:
             # IPython 3
             expected = reads(expected, 3)
         except (TypeError, NBFormatError):
             # IPython 2
             expected = reads(expected, 'json')
         exit_code = 1
         argv = sys.argv
         stdout = sys.stdout
         stderr = sys.stderr
         try:
             with open(devnull, "w") as devnull_filehandle:
                 sys.stdout = sys.stderr = devnull_filehandle
                 sys.argv = [
                     "runipy", "-o", notebook_path, "--html",
                     notebook_path.replace(".ipynb", ".html")
                 ]
                 main()
         except SystemExit as e:
             exit_code = e.code
         finally:
             sys.argv = argv
             sys.stdout = stdout
             sys.stderr = stderr
         notebook = ""
         with open(notebook_path) as notebook_file:
             notebook = notebook_file.read()
         try:
             # IPython 3
             notebook = reads(notebook, 3)
         except (TypeError, NBFormatError):
             # IPython 2
             notebook = reads(notebook, 'json')
         self.assert_notebooks_equal(expected, notebook)
Example #17
0
 def _read_notebook(self, os_path, as_version=4):
     """Read a notebook from an os path."""
     with self.open(os_path, "r", encoding="utf-8") as f:
         try:
             if ftdetect(os_path) == "notebook":
                 return nbformat.read(f, as_version=as_version)
             elif ftdetect(os_path) == "markdown":
                 nbjson = convert(os_path, informat="markdown", outformat="notebook")
                 return nbformat.reads(nbjson, as_version=as_version)
         except Exception as e:
             raise web.HTTPError(400, u"Unreadable Notebook: %s %r" % (os_path, e))
Example #18
0
 def _read_notebook(self, os_path, as_version=4):
     """Read a notebook from an os path."""
     with self.open(os_path, 'r', encoding='utf-8') as f:
         try:
             if ftdetect(os_path) == 'notebook':
                 return nbformat.read(f, as_version=as_version)
             elif ftdetect(os_path) == 'markdown':
                 nbjson = convert(os_path,
                                  informat='markdown',
                                  outformat='notebook')
                 return nbformat.reads(nbjson, as_version=as_version)
         except Exception as e:
             raise web.HTTPError(
                 400,
                 u"Unreadable Notebook: %s %r" % (os_path, e),
             )
Example #19
0
    def render(self):
        try:
            with open(self.file_path, 'r') as file_pointer:
                notebook = nbformat.reads(file_pointer.read(), as_version=4)
        except ValueError:
            raise exceptions.InvalidFormat('Could not read ipython notebook file.')

        exporter = HTMLExporter(config=Config({
            'HTMLExporter': {
                'template_file': 'basic',
            },
            'CSSHtmlHeaderTransformer': {
                'enabled': False,
            },
        }))
        (body, _) = exporter.from_notebook_node(notebook)
        return self.TEMPLATE.render(base=self.assets_url, body=body)
Example #20
0
    def get_notebook(self, name, path='', content=True, **kwargs):
        """ Takes a path and name for a notebook and returns its model

        Parameters
        ----------
        name : str
            the name of the notebook
        path : str
            the URL path that describes the relative path for
            the notebook

        Returns
        -------
        model : dict
            the notebook model. If contents=True, returns the 'contents'
            dict in the model as well.
        """
        path = path.strip('/')
        if not self.notebook_exists(name=name, path=path):
            raise web.HTTPError(404, u'Notebook does not exist: %s' % name)
        gist = self._get_gist(name, path)
        # Create the notebook model.
        model = {}
        model['name'] = name
        model['path'] = path
        model['last_modified'] = gist.updated_at
        model['created'] = gist.created_at
        model['type'] = 'notebook'
        model['format'] = None
        model['writable'] = True
        model['mimetype'] = None
        if content:
            model['format'] = 'json'
            notebook_content = gist.notebook_content
            try:
                nb = nbformat.reads(notebook_content, as_version=4)
            except Exception as e:
                raise web.HTTPError(
                    400, u"Unreadable Notebook: %s %s %s" % (path, name, e))
            self.mark_trusted_cells(nb, self.fullpath(path, name))

            # add gist id if public.
            if gist.public:
                nb['metadata']['gist_id'] = gist.id
            model['content'] = nb
        return model
Example #21
0
    def get_notebook(self, name, path='', content=True, **kwargs):
        """ Takes a path and name for a notebook and returns its model

        Parameters
        ----------
        name : str
            the name of the notebook
        path : str
            the URL path that describes the relative path for
            the notebook

        Returns
        -------
        model : dict
            the notebook model. If contents=True, returns the 'contents'
            dict in the model as well.
        """
        path = path.strip('/')
        if not self.notebook_exists(name=name, path=path):
            raise web.HTTPError(404, u'Notebook does not exist: %s' % name)
        gist = self._get_gist(name, path)
        # Create the notebook model.
        model ={}
        model['name'] = name
        model['path'] = path
        model['last_modified'] = gist.updated_at
        model['created'] = gist.created_at
        model['type'] = 'notebook'
        model['format'] = None
        model['writable'] = True
        model['mimetype'] = None
        if content:
            model['format'] = 'json'
            notebook_content = gist.notebook_content
            try:
                nb = nbformat.reads(notebook_content, as_version=4)
            except Exception as e:
                raise web.HTTPError(400, u"Unreadable Notebook: %s %s %s" % (path, name, e))
            self.mark_trusted_cells(nb, self.fullpath(path, name))

            # add gist id if public.
            if gist.public:
                nb['metadata']['gist_id'] = gist.id
            model['content'] = nb
        return model
Example #22
0
    def render(self):
        try:
            with open(self.file_path, 'r') as file_pointer:
                notebook = nbformat.reads(file_pointer.read(), as_version=4)
        except ValueError:
            raise exceptions.InvalidFormat(
                'Could not read ipython notebook file.')

        exporter = HTMLExporter(config=Config({
            'HTMLExporter': {
                'template_file': 'basic',
            },
            'CSSHtmlHeaderTransformer': {
                'enabled': False,
            },
        }))
        (body, _) = exporter.from_notebook_node(notebook)
        return self.TEMPLATE.render(base=self.assets_url, body=body)
Example #23
0
def test_roundtrip():
    """Run nbconvert using our custom markdown template to recover
    original markdown from a notebook.
    """
    # create a notebook from the markdown
    mr = notedown.MarkdownReader()
    roundtrip_notebook = mr.to_notebook(roundtrip_markdown)

    # write the notebook into json
    notebook_json = nbformat.writes(roundtrip_notebook)

    # write the json back into notebook
    notebook = nbformat.reads(notebook_json, as_version=4)

    # convert notebook to markdown
    mw = notedown.MarkdownWriter(template_file='notedown/templates/markdown.tpl', strip_outputs=True)
    markdown = mw.writes(notebook)

    nt.assert_multi_line_equal(roundtrip_markdown, markdown)
Example #24
0
def test_roundtrip():
    """Run nbconvert using our custom markdown template to recover
    original markdown from a notebook.
    """
    # create a notebook from the markdown
    mr = notedown.MarkdownReader()
    roundtrip_notebook = mr.to_notebook(roundtrip_markdown)

    # write the notebook into json
    notebook_json = nbformat.writes(roundtrip_notebook)

    # write the json back into notebook
    notebook = nbformat.reads(notebook_json, as_version=4)

    # convert notebook to markdown
    mw = notedown.MarkdownWriter(
        template_file='notedown/templates/markdown.tpl', strip_outputs=True)
    markdown = mw.writes(notebook)

    nt.assert_multi_line_equal(roundtrip_markdown, markdown)
Example #25
0
    def collect(self):
        with self.fspath.open() as f:
            # self.nb = reads(f.read(), 'json')
            self.nb = reads(f.read(), 4)

            # Start the cell count
            cell_num = 0

            # Worksheets are NOT used anymore::
            # Currently there is only 1 worksheet (it seems in newer versions
            # of IPython, they are going to get rid of this option)
            # For every worksheet, read every cell associated to it

            for cell in self.nb.cells:
                # Skip the cells that have text, headings or related stuff
                # Only test code cells
                if cell.cell_type == 'code':
                    # If the code is a notebook magic cell, do not run
                    # i.e. cell code starts with '%%'
                    # Also ignore the cells that start with the
                    # comment string PYTEST_VALIDATE_IGNORE_OUTPUT
                    # NOTE: This actually skips execution, which probably isn't what we want!
                    #       It is typically helpful to execute the cell (to make sure that at
                    #       least the code doesn't fail) but then discard the result.
                    if not (cell.source.startswith('%%') or
                            cell.source.startswith(r'# PYTEST_VALIDATE_IGNORE_OUTPUT') or
                            cell.source.startswith(r'#PYTEST_VALIDATE_IGNORE_OUTPUT')):

                        yield IPyNbCell(self.name, self, cell_num, cell)

                    else:
                        # Skipped cells will not be counted
                        continue

                # Update 'code' cell count
                cell_num += 1
    def collect(self):
        with self.fspath.open() as f:
            # self.nb = reads(f.read(), 'json')
            self.nb = reads(f.read(), 4)

            # Start the cell count
            cell_num = 0

            # Worksheets are NOT used anymore::
            # Currently there is only 1 worksheet (it seems in newer versions
            # of IPython, they are going to get rid of this option)
            # For every worksheet, read every cell associated to it

            for cell in self.nb.cells:
                # Skip the cells that have text, headings or related stuff
                # Only test code cells
                if cell.cell_type == 'code':
                    # If the code is a notebook magic cell, do not run
                    # i.e. cell code starts with '%%'
                    # Also ignore the cells that start with the
                    # comment string PYTEST_VALIDATE_IGNORE_OUTPUT
                    # NOTE: This actually skips execution, which probably isn't what we want!
                    #       It is typically helpful to execute the cell (to make sure that at
                    #       least the code doesn't fail) but then discard the result.
                    if not (cell.source.startswith('%%') or
                            cell.source.startswith(r'# PYTEST_VALIDATE_IGNORE_OUTPUT') or
                            cell.source.startswith(r'#PYTEST_VALIDATE_IGNORE_OUTPUT')):

                        yield IPyNbCell(self.name, self, cell_num, cell)

                    else:
                        # Skipped cells will not be counted
                        continue

                # Update 'code' cell count
                cell_num += 1
def export_nb(nb_json, format):
    import IPython.nbconvert as nbconvert
    import IPython.nbformat as nbformat
    nb = nbformat.reads(nb_json, nbformat.NO_CONVERT)
    output = nbconvert.export_by_name(format, nb)
    print(output[0])
 def readjson(json_str):
     data = current.reads(json_str, 4)
     return data
Example #29
0
def reads_base64(nb, as_version=NBFORMAT_VERSION):
    """
    Read a notebook from base64.
    """
    return reads(b64decode(nb).decode('utf-8'), as_version=as_version)
    for ws in nb.worksheets:
        for cell in ws.cells:
            if cell.cell_type != 'code':
                continue
            shell.execute(cell.input)
            reply = shell.get_msg(timeout=20000)['content']
            if reply['status'] == 'error':
                failures += 1
                print("\nFAILURE:")
                print(cell.input)
                print('-----')
                print("raised:")
                print('\n'.join(reply['traceback']))
            cells += 1
            sys.stdout.write('.')

    print("ran notebook %s" % nb.metadata.name)
    print("    ran %3i cells" % cells)
    if failures:
        print("    %3i cells raised exceptions" % failures)
    kc.stop_channels()
    km.shutdown_kernel()
    del km

if __name__ == '__main__':
    for ipynb in sys.argv[1:]:
        print("running %s" % ipynb)
        with open(ipynb) as f:
            nb = reads(f.read(), 'json')
        run_notebook(nb)
Example #31
0
def main():
    log_format = "%(asctime)s %(levelname)s: %(message)s"
    log_datefmt = "%m/%d/%Y %I:%M:%S %p"

    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--version", "-v", action="version", version=runipy.__version__, help="print version information"
    )
    parser.add_argument("input_file", nargs="?", help=".ipynb file to run (or stdin)")
    parser.add_argument("output_file", nargs="?", help=".ipynb file to save cell output to")
    parser.add_argument("--quiet", "-q", action="store_true", help="don't print anything unless things go wrong")
    parser.add_argument(
        "--overwrite", "-o", action="store_true", help="write notebook output back to original notebook"
    )
    format_grp = parser.add_mutually_exclusive_group()
    format_grp.add_argument("--html", nargs="?", default=False, help="output an HTML snapshot of the notebook")
    format_grp.add_argument("--pdf", nargs="?", default=False, help="output a PDF snapshot of the notebook")
    parser.add_argument("--template", nargs="?", default=False, help="template to use for HTML output")
    parser.add_argument("--pylab", action="store_true", help="start notebook with pylab enabled")
    parser.add_argument("--matplotlib", action="store_true", help="start notebook with matplotlib inlined")
    parser.add_argument(
        "--skip-exceptions",
        "-s",
        action="store_true",
        help="if an exception occurs in a cell," + " continue running the subsequent cells",
    )
    parser.add_argument("--stdout", action="store_true", help="print notebook to stdout (or use - as output_file")
    parser.add_argument("--stdin", action="store_true", help="read notebook from stdin (or use - as input_file)")
    parser.add_argument(
        "--no-chdir", action="store_true", help="do not change directory to notebook's at kernel startup"
    )
    parser.add_argument("--profile-dir", help="set the profile location directly")
    args = parser.parse_args()

    if args.overwrite:
        if args.output_file is not None:
            print("Error: output_filename must not be provided if " "--overwrite (-o) given", file=stderr)
            exit(1)
        else:
            args.output_file = args.input_file

    if not args.quiet:
        logging.basicConfig(level=logging.INFO, format=log_format, datefmt=log_datefmt)

    working_dir = None

    payload_source = ""
    payload = ""
    if args.input_file == "-" or args.stdin:  # force stdin
        payload_source = stdin.name
        payload = stdin.read()
    elif not args.input_file and stdin.isatty():  # no force, empty stdin
        parser.print_help()
        exit()
    elif not args.input_file:  # no file -> default stdin
        payload_source = stdin.name
        payload = stdin.read()
    else:  # must have specified normal input_file
        with open(args.input_file) as input_file:
            payload_source = input_file.name
            payload = input_file.read()
        working_dir = os.path.dirname(args.input_file)

    if args.no_chdir:
        working_dir = None

    if args.profile_dir:
        profile_dir = os.path.expanduser(args.profile_dir)
    else:
        profile_dir = None

    logging.info("Reading notebook %s", payload_source)
    try:
        # Ipython 3
        nb = reads(payload, 3)
    except (TypeError, NBFormatError):
        # Ipython 2
        nb = reads(payload, "json")
    nb_runner = NotebookRunner(nb, args.pylab, args.matplotlib, profile_dir, working_dir)

    exit_status = 0
    try:
        nb_runner.run_notebook(skip_exceptions=args.skip_exceptions)
    except NotebookError:
        exit_status = 1

    if args.output_file and args.output_file != "-":
        logging.info("Saving to %s", args.output_file)
        with open(args.output_file, "w") as output_filehandle:
            try:
                # Ipython 3
                write(nb_runner.nb, output_filehandle, 3)
            except (TypeError, NBFormatError):
                # Ipython 2
                write(nb_runner.nb, output_filehandle, "json")

    if args.stdout or args.output_file == "-":
        try:
            # Ipython 3
            write(nb_runner.nb, stdout, 3)
        except (TypeError, NBFormatError):
            # Ipython 2
            write(nb_runner.nb, stdout, "json")
        print()

    if args.html is not False:
        if args.html is None:
            # if --html is given but no filename is provided,
            # come up with a sane output name based on the
            # input filename
            if args.input_file.endswith(".ipynb"):
                args.html = args.input_file[:-6] + ".html"
            else:
                args.html = args.input_file + ".html"

        if args.template is False:
            exporter = HTMLExporter()
        else:
            exporter = HTMLExporter(
                config=Config({"HTMLExporter": {"template_file": args.template, "template_path": [".", "/"]}})
            )

        logging.info("Saving HTML snapshot to %s" % args.html)
        output, resources = exporter.from_notebook_node(convert(nb_runner.nb, current_nbformat))
        codecs.open(args.html, "w", encoding="utf-8").write(output)

    elif args.pdf is not False:
        if args.pdf is None:
            # if --html is given but no filename is provided,
            # come up with a sane output name based on the
            # input filename
            if args.input_file.endswith(".ipynb"):
                args.pdf = args.input_file[:-6] + ".pdf"
            else:
                args.pdf = args.input_file + ".pdf"

        if args.template is False:
            exporter = PDFExporter()
        else:
            exporter = PDFExporter(
                config=Config({"PDFExporter": {"template_file": args.template, "template_path": [".", "/"]}})
            )

        logging.info("Saving PDF snapshot to %s" % args.pdf)
        output, resources = exporter.from_notebook_node(convert(nb_runner.nb, current_nbformat))
        writer = FilesWriter()
        writer.build_directory = os.path.dirname(args.pdf)
        writer.write(output, resources, os.path.splitext(os.path.basename(args.pdf))[0])

    nb_runner.shutdown_kernel()

    if exit_status != 0:
        logging.warning("Exiting with nonzero exit status")
    exit(exit_status)
Example #32
0
def main():
    log_format = '%(asctime)s %(levelname)s: %(message)s'
    log_datefmt = '%m/%d/%Y %I:%M:%S %p'

    parser = argparse.ArgumentParser()
    parser.add_argument(
        '--version', '-v', action='version',
        version=runipy.__version__,
        help='print version information'
    )
    parser.add_argument(
        'input_file', nargs='?',
        help='.ipynb file to run (or stdin)'
    )
    parser.add_argument(
        'output_file', nargs='?',
        help='.ipynb file to save cell output to'
    )
    parser.add_argument(
        '--quiet', '-q', action='store_true',
        help='don\'t print anything unless things go wrong'
    )
    parser.add_argument(
        '--overwrite', '-o', action='store_true',
        help='write notebook output back to original notebook'
    )
    parser.add_argument(
        '--html', nargs='?', default=False,
        help='output an HTML snapshot of the notebook'
    )
    parser.add_argument(
        '--template', nargs='?', default=False,
        help='template to use for HTML output'
    )
    parser.add_argument(
        '--pylab', action='store_true',
        help='start notebook with pylab enabled'
    )
    parser.add_argument(
        '--matplotlib', action='store_true',
        help='start notebook with matplotlib inlined'
    )
    parser.add_argument(
        '--skip-exceptions', '-s', action='store_true',
        help='if an exception occurs in a cell,' +
             ' continue running the subsequent cells'
    )
    parser.add_argument(
        '--stdout', action='store_true',
        help='print notebook to stdout (or use - as output_file'
    )
    parser.add_argument(
        '--stdin', action='store_true',
        help='read notebook from stdin (or use - as input_file)'
    )
    parser.add_argument(
        '--no-chdir', action='store_true',
        help="do not change directory to notebook's at kernel startup"
    )
    parser.add_argument(
        '--profile-dir',
        help="set the profile location directly"
    )
    args = parser.parse_args()

    if args.overwrite:
        if args.output_file is not None:
            print('Error: output_filename must not be provided if '
                  '--overwrite (-o) given', file=stderr)
            exit(1)
        else:
            args.output_file = args.input_file

    if not args.quiet:
        logging.basicConfig(
            level=logging.INFO, format=log_format, datefmt=log_datefmt
        )

    working_dir = None

    payload_source = ""
    payload = ""
    if args.input_file == '-' or args.stdin:  # force stdin
        payload_source = stdin.name
        payload = stdin.read()
    elif not args.input_file and stdin.isatty():  # no force, empty stdin
        parser.print_help()
        exit()
    elif not args.input_file:  # no file -> default stdin
        payload_source = stdin.name
        payload = stdin.read()
    else:  # must have specified normal input_file
        with open(args.input_file) as input_file:
            payload_source = input_file.name
            payload = input_file.read()
        working_dir = os.path.dirname(args.input_file)

    if args.no_chdir:
        working_dir = None

    if args.profile_dir:
        profile_dir = os.path.expanduser(args.profile_dir)
    else:
        profile_dir = None

    logging.info('Reading notebook %s', payload_source)
    try:
        # Ipython 3
        nb = reads(payload, 3)
    except (TypeError, NBFormatError):
        # Ipython 2
        nb = reads(payload, 'json')
    nb_runner = NotebookRunner(
        nb, args.pylab, args.matplotlib, profile_dir, working_dir
    )

    exit_status = 0
    try:
        nb_runner.run_notebook(skip_exceptions=args.skip_exceptions)
    except NotebookError:
        exit_status = 1

    if args.output_file and args.output_file != '-':
        logging.info('Saving to %s', args.output_file)
        with open(args.output_file, 'w') as output_filehandle:
            try:
                # Ipython 3
                write(nb_runner.nb, output_filehandle, 3)
            except (TypeError, NBFormatError):
                # Ipython 2
                write(nb_runner.nb, output_filehandle, 'json')

    if args.stdout or args.output_file == '-':
        try:
            # Ipython 3
            write(nb_runner.nb, stdout, 3)
        except (TypeError, NBFormatError):
            # Ipython 2
            write(nb_runner.nb, stdout, 'json')
        print()

    if args.html is not False:
        if args.html is None:
            # if --html is given but no filename is provided,
            # come up with a sane output name based on the
            # input filename
            if args.input_file.endswith('.ipynb'):
                args.html = args.input_file[:-6] + '.html'
            else:
                args.html = args.input_file + '.html'

        if args.template is False:
            exporter = HTMLExporter()
        else:
            exporter = HTMLExporter(
                config=Config({
                    'HTMLExporter': {
                        'template_file': args.template,
                        'template_path': ['.', '/']
                    }
                })
            )

        logging.info('Saving HTML snapshot to %s' % args.html)
        output, resources = exporter.from_notebook_node(nb_runner.nb)
        codecs.open(args.html, 'w', encoding='utf-8').write(output)

    nb_runner.shutdown_kernel()

    if exit_status != 0:
        logging.warning('Exiting with nonzero exit status')
    exit(exit_status)
Example #33
0
def main():
    log_format = '%(asctime)s %(levelname)s: %(message)s'
    log_datefmt = '%m/%d/%Y %I:%M:%S %p'

    parser = argparse.ArgumentParser()
    parser.add_argument('--version',
                        '-v',
                        action='version',
                        version=runipy.__version__,
                        help='print version information')
    parser.add_argument('input_file',
                        nargs='?',
                        help='.ipynb file to run (or stdin)')
    parser.add_argument('output_file',
                        nargs='?',
                        help='.ipynb file to save cell output to')
    parser.add_argument('--quiet',
                        '-q',
                        action='store_true',
                        help='don\'t print anything unless things go wrong')
    parser.add_argument('--overwrite',
                        '-o',
                        action='store_true',
                        help='write notebook output back to original notebook')
    parser.add_argument('--html',
                        nargs='?',
                        default=False,
                        help='output an HTML snapshot of the notebook')
    parser.add_argument('--template',
                        nargs='?',
                        default=False,
                        help='template to use for HTML output')
    parser.add_argument('--pylab',
                        action='store_true',
                        help='start notebook with pylab enabled')
    parser.add_argument('--matplotlib',
                        action='store_true',
                        help='start notebook with matplotlib inlined')
    parser.add_argument('--skip-exceptions',
                        '-s',
                        action='store_true',
                        help='if an exception occurs in a cell,' +
                        ' continue running the subsequent cells')
    parser.add_argument(
        '--stdout',
        action='store_true',
        help='print notebook to stdout (or use - as output_file')
    parser.add_argument(
        '--stdin',
        action='store_true',
        help='read notebook from stdin (or use - as input_file)')
    parser.add_argument(
        '--no-chdir',
        action='store_true',
        help="do not change directory to notebook's at kernel startup")
    parser.add_argument('--profile-dir',
                        help="set the profile location directly")
    args = parser.parse_args()

    if args.overwrite:
        if args.output_file is not None:
            print(
                'Error: output_filename must not be provided if '
                '--overwrite (-o) given',
                file=stderr)
            exit(1)
        else:
            args.output_file = args.input_file

    if not args.quiet:
        logging.basicConfig(level=logging.INFO,
                            format=log_format,
                            datefmt=log_datefmt)

    working_dir = None

    payload_source = ""
    payload = ""
    if args.input_file == '-' or args.stdin:  # force stdin
        payload_source = stdin.name
        payload = stdin.read()
    elif not args.input_file and stdin.isatty():  # no force, empty stdin
        parser.print_help()
        exit()
    elif not args.input_file:  # no file -> default stdin
        payload_source = stdin.name
        payload = stdin.read()
    else:  # must have specified normal input_file
        with open(args.input_file) as input_file:
            payload_source = input_file.name
            payload = input_file.read()
        working_dir = os.path.dirname(args.input_file)

    if args.no_chdir:
        working_dir = None

    if args.profile_dir:
        profile_dir = os.path.expanduser(args.profile_dir)
    else:
        profile_dir = None

    logging.info('Reading notebook %s', payload_source)
    try:
        # Ipython 3
        nb = reads(payload, 3)
    except (TypeError, NBFormatError):
        # Ipython 2
        nb = reads(payload, 'json')
    nb_runner = NotebookRunner(nb, args.pylab, args.matplotlib, profile_dir,
                               working_dir)

    exit_status = 0
    try:
        nb_runner.run_notebook(skip_exceptions=args.skip_exceptions)
    except NotebookError:
        exit_status = 1

    if args.output_file and args.output_file != '-':
        logging.info('Saving to %s', args.output_file)
        with open(args.output_file, 'w') as output_filehandle:
            try:
                # Ipython 3
                write(nb_runner.nb, output_filehandle, 3)
            except (TypeError, NBFormatError):
                # Ipython 2
                write(nb_runner.nb, output_filehandle, 'json')

    if args.stdout or args.output_file == '-':
        try:
            # Ipython 3
            write(nb_runner.nb, stdout, 3)
        except (TypeError, NBFormatError):
            # Ipython 2
            write(nb_runner.nb, stdout, 'json')
        print()

    if args.html is not False:
        if args.html is None:
            # if --html is given but no filename is provided,
            # come up with a sane output name based on the
            # input filename
            if args.input_file.endswith('.ipynb'):
                args.html = args.input_file[:-6] + '.html'
            else:
                args.html = args.input_file + '.html'

        if args.template is False:
            exporter = HTMLExporter()
        else:
            exporter = HTMLExporter(config=Config({
                'HTMLExporter': {
                    'template_file': args.template,
                    'template_path': ['.', '/']
                }
            }))

        logging.info('Saving HTML snapshot to %s' % args.html)
        output, resources = exporter.from_notebook_node(
            convert(nb_runner.nb, current_nbformat))
        codecs.open(args.html, 'w', encoding='utf-8').write(output)

    nb_runner.shutdown_kernel()

    if exit_status != 0:
        logging.warning('Exiting with nonzero exit status')
    exit(exit_status)
        for cell in ws.cells:
            if cell.cell_type != 'code':
                continue
            shell.execute(cell.input)
            reply = shell.get_msg(timeout=20000)['content']
            if reply['status'] == 'error':
                failures += 1
                print("\nFAILURE:")
                print(cell.input)
                print('-----')
                print("raised:")
                print('\n'.join(reply['traceback']))
            cells += 1
            sys.stdout.write('.')

    print("ran notebook %s" % nb.metadata.name)
    print("    ran %3i cells" % cells)
    if failures:
        print("    %3i cells raised exceptions" % failures)
    kc.stop_channels()
    km.shutdown_kernel()
    del km


if __name__ == '__main__':
    for ipynb in sys.argv[1:]:
        print("running %s" % ipynb)
        with open(ipynb) as f:
            nb = reads(f.read(), 'json')
        run_notebook(nb)
Example #35
0
    def create_post(self, path, **kw):
        """Create a new post."""
        if flag is None:
            req_missing(['ipython[notebook]>=2.0.0'],
                        'build this site (compile ipynb)')
        content = kw.pop('content', None)
        onefile = kw.pop('onefile', False)
        kernel = kw.pop('ipython_kernel', None)
        # is_page is not needed to create the file
        kw.pop('is_page', False)

        metadata = {}
        metadata.update(self.default_metadata)
        metadata.update(kw)

        makedirs(os.path.dirname(path))

        if content.startswith("{"):
            # imported .ipynb file, guaranteed to start with "{" because it’s JSON.
            nb = nbformat.reads(content, current_nbformat)
        else:
            if IPython.version_info[0] >= 3:
                nb = nbformat.v4.new_notebook()
                nb["cells"] = [nbformat.v4.new_markdown_cell(content)]
            else:
                nb = nbformat.new_notebook()
                nb["worksheets"] = [
                    nbformat.new_worksheet(
                        cells=[nbformat.new_text_cell('markdown', [content])])
                ]

            if kernelspec is not None:
                if kernel is None:
                    kernel = self.default_kernel
                    self.logger.notice(
                        'No kernel specified, assuming "{0}".'.format(kernel))

                IPYNB_KERNELS = {}
                ksm = kernelspec.KernelSpecManager()
                for k in ksm.find_kernel_specs():
                    IPYNB_KERNELS[k] = ksm.get_kernel_spec(k).to_dict()
                    IPYNB_KERNELS[k]['name'] = k
                    del IPYNB_KERNELS[k]['argv']

                if kernel not in IPYNB_KERNELS:
                    self.logger.error(
                        'Unknown kernel "{0}". Maybe you mispelled it?'.format(
                            kernel))
                    self.logger.info("Available kernels: {0}".format(", ".join(
                        sorted(IPYNB_KERNELS))))
                    raise Exception('Unknown kernel "{0}"'.format(kernel))

                nb["metadata"]["kernelspec"] = IPYNB_KERNELS[kernel]
            else:
                # Older IPython versions don’t need kernelspecs.
                pass

        if onefile:
            nb["metadata"]["nikola"] = metadata

        with io.open(path, "w+", encoding="utf8") as fd:
            if IPython.version_info[0] >= 3:
                nbformat.write(nb, fd, 4)
            else:
                nbformat.write(nb, fd, 'ipynb')
Example #36
0
def reads_base64(nb, as_version=NBFORMAT_VERSION):
    """
    Read a notebook from base64.
    """
    return reads(b64decode(nb).decode('utf-8'), as_version=as_version)