Ejemplo n.º 1
0
def notebooks_to_rst(app):
    from glob import glob

    try:
        # post "big-split", nbconvert is a separate namespace
        from nbconvert.nbconvertapp import NbConvertApp
        from nbconvert.writers import FilesWriter
    except ImportError:
        from IPython.nbconvert.nbconvertapp import NbConvertApp
        from IPython.nbconvert.writers import FilesWriter

    class OrphanizerWriter(FilesWriter):
        def write(self, output, resources, **kwargs):
            output = ':orphan:\n\n' + output
            FilesWriter.write(self, output, resources, **kwargs)

    olddir = os.path.abspath(os.curdir)
    try:
        srcdir = os.path.abspath(os.path.split(__file__)[0])
        os.chdir(os.path.join(srcdir, 'astroimtools', 'notebooks'))
        nbs = glob('*.ipynb')

        app = NbConvertApp()
        app.initialize(argv=[])
        app.writer = OrphanizerWriter()

        app.export_format = 'rst'
        app.notebooks = nbs

        app.start()
    except:
        pass
    finally:
        os.chdir(olddir)
Ejemplo n.º 2
0
def notebooks_to_rst(app):
    from glob import glob

    # post "big-split", nbconvert is a separate namespace
    from nbconvert.nbconvertapp import NbConvertApp
    from nbconvert.writers import FilesWriter

    class OrphanizerWriter(FilesWriter):
        def write(self, output, resources, **kwargs):
            output = ':orphan:\n\n' + output
            FilesWriter.write(self, output, resources, **kwargs)

    olddir = os.path.abspath(os.curdir)
    try:
        srcdir = os.path.abspath(os.path.split(__file__)[0])
        os.chdir(os.path.join(srcdir, 'stginga', 'notebooks'))
        nbs = glob('*.ipynb')

        app = NbConvertApp()
        app.initialize(argv=[])
        app.writer = OrphanizerWriter()

        app.export_format = 'rst'
        app.notebooks = nbs

        app.start()
    except:
        pass
    finally:
        os.chdir(olddir)
Ejemplo n.º 3
0
def post_save(model, os_path, contents_manager):
    """post-save hook for converting notebooks to .py scripts and html
       in a separate folder with the same name
    """
    # only do this for notebooks
    if model['type'] != 'notebook':
        return

    # only do this if we've added the special indicator file to the working directory
    cwd = os.path.dirname(os_path)
    save_procress_indicator = os.path.join(cwd, SAVE_PROCRESS_INDICATOR_FILE)
    should_convert = os.path.exists(save_procress_indicator)

    if should_convert:
        d, fname = os.path.split(os_path)
        subfolder = os.path.splitext(fname)[0]

        converter = NbConvertApp()
        converter.postprocessor = CopyToSubfolderPostProcessor(subfolder=subfolder)

        converter.export_format = 'script'
        converter.initialize(argv=[])
        converter.notebooks = [os_path]
        converter.convert_notebooks()

        converter.export_format = 'html'
        converter.initialize(argv=[])
        converter.notebooks = [os_path]
        converter.convert_notebooks()
Ejemplo n.º 4
0
    def from_notebook_node(self, nb, resources=None, **kw):
        # track conversion time.
        start_time = time()

        # Preprocess the notebook
        nb, resources = self._preprocess(nb, resources)

        # nbconvert options embedded in the metadata as a dict or a list.
        convert = nb.metadata.pop('nbconvert', {})
        if isinstance(convert, dict):
            convert = convert.items()

        for exporter, config in convert:
            app = NbConvertApp(config=Config(config))
            app.initialize([])
            app.exporter = get_exporter(exporter)(config=app.config)
            app.convert_single_notebook(self.filename)

        # How long did the conversions take?
        if convert:
            app.log.info(
                "Conversions completed in {} seconds.".format(time() -
                                                              start_time))

        return
Ejemplo n.º 5
0
def export_notebook(notebook_path: Path, config: NbAutoexportConfig):
    """Export a given notebook file given configuration.

    Args:
        notebook_path (Path): path to notebook to export with nbconvert
        config (NbAutoexportConfig): configuration
    """
    logger.info(f"nbautoexport | Exporting {notebook_path} ...")
    logger.debug(
        f"nbautoexport | Using export configuration:\n{config.json(indent=2)}")
    with cleared_argv():
        converter = NbConvertApp()
        converter.log.handlers = logger.handlers
        converter.log.setLevel(logger.level)

        for export_format in config.export_formats:
            if config.organize_by == "notebook":
                subfolder = notebook_path.stem
            elif config.organize_by == "extension":
                subfolder = export_format.value

            converter.postprocessor = CopyToSubfolderPostProcessor(
                subfolder=subfolder, export_format=export_format)
            converter.export_format = export_format.value
            converter.initialize()
            converter.notebooks = [str(notebook_path)]
            converter.convert_notebooks()
Ejemplo n.º 6
0
def convert_tour_nb_to_document():
    app = NbConvertApp()
    app.initialize()
    app.notebooks = ['../Skills-ML Tour.ipynb']
    app.export_format = 'markdown'
    app.output_base = 'skills_ml_tour.md'
    app.writer.build_directory = 'sources/'
    app.convert_notebooks()
Ejemplo n.º 7
0
def main():
    import argparse
    parser = argparse.ArgumentParser(description='''
        Make (and run) a jupyter notebook visualizing the results of an output
        file from fixed_run.py.'''.strip())
    parser.add_argument('fname')
    parser.add_argument('--key', default='df')
    parser.add_argument('out_name', nargs='?')
    parser.add_argument('-n', type=int)
    parser.add_argument('--level', type=float, default=.1)
    parser.add_argument('--kernel-name')
    parser.add_argument('--force', '-f', action='store_true')
    args = parser.parse_args()

    if args.out_name is None:
        assert args.fname.endswith('.h5')
        base = args.fname[:-3]
        args.out_name = base + '.ipynb'
    else:
        d = os.path.dirname(args.out_name)
        if d and not os.path.isdir(d):
            os.makedirs(d)

    if not args.force and os.path.exists(args.out_name):
        parser.exit(
            "Output {} already exists; use --force to overwrite.".format(
                args.out_name))
    del args.force

    if args.n is None:
        match = re.search('/n(\d+)\.h5$', args.fname)
        if match:
            args.n = int(match.group(1))
        else:
            parser.error('-n must be specified unless filename has it')

    make_notebook(**vars(args))
    nbc = NbConvertApp()
    nbc.initialize([
        '--to=notebook', '--execute', '--output',
        os.path.basename(args.out_name), args.out_name
    ])
    nbc.convert_notebooks()
Ejemplo n.º 8
0
def export_notebook(notebook_path: Path, config: NbAutoexportConfig):
    """Export a given notebook file given configuration.

    Args:
        notebook_path (Path): path to notebook to export with nbconvert
        config (NbAutoexportConfig): configuration
    """
    with cleared_argv():
        converter = NbConvertApp()

        for export_format in config.export_formats:
            if config.organize_by == "notebook":
                subfolder = notebook_path.stem
            elif config.organize_by == "extension":
                subfolder = export_format.value

            converter.postprocessor = CopyToSubfolderPostProcessor(
                subfolder=subfolder, export_format=export_format)
            converter.export_format = export_format.value
            converter.initialize()
            converter.notebooks = [str(notebook_path)]
            converter.convert_notebooks()
Ejemplo n.º 9
0
def notebooks_to_rst(app):
    from glob import glob

    try:
        # post "big-split", nbconvert is a separate namespace
        from nbconvert.nbconvertapp import NbConvertApp
        from nbconvert.writers import FilesWriter
        from nbconvert.preprocessors import Preprocessor, ExecutePreprocessor, execute
        from nbconvert.exporters import RSTExporter
        from nbformat import NotebookNode
    except ImportError:
        try:
            from IPython.nbconvert.nbconvertapp import NbConvertApp
            from IPython.nbconvert.writers import FilesWriter
            from IPython.nbconvert.preprocessors import Preprocessor, ExecutePreprocessor, execute
            from IPython.nbconvert.exporters import RSTExporter
            from IPython.nbformat import NotebookNode
        except ImportError:
            raise ImportError(
                "Failed to find Jupyter or IPython. Cannot build "
                "the notebooks embedded in the docs. Proceeding "
                "the rest of the doc build, but additional "
                "warnings are likely."
            )
            return

    class OrphanizerWriter(FilesWriter):
        def write(self, output, resources, **kwargs):
            output = ":orphan:\n\n" + output
            FilesWriter.write(self, output, resources, **kwargs)

    class AddSysPath(Preprocessor):
        """
        Adds the local system path to the top of the notebook.  This makes sure
        when build_sphinx is invoked that the notebook actually runs with the
        current build.
        """

        def preprocess(self, nb, resources):
            syspathstr = "sys.path = {} + sys.path".format(str(sys.path))
            cell = {
                "cell_type": "code",
                "execution_count": None,
                "metadata": {},
                "outputs": [],
                "source": "import sys\n" + syspathstr,
            }
            nb.cells.insert(0, NotebookNode(cell))
            return nb, resources

    class RemoveSysPath(Preprocessor):
        """
        Removes the sys.path cell added by AddSysPath
        """

        def preprocess(self, nb, resources):
            if "sys.path" in nb.cells[0].source:
                del nb.cells[0]
            return nb, resources

    class MonkeypatchCellExecutionError(execute.CellExecutionError):
        def __str__(self):
            sstr = super(MonkeypatchCellExecutionError, self).__str__()
            return sstr + " Traceback:\n" + str(self.traceback)

    execute.CellExecutionError = MonkeypatchCellExecutionError

    olddir = os.path.abspath(os.curdir)
    try:
        srcdir = os.path.abspath(os.path.split(__file__)[0])
        if os.path.isdir("notebooks"):
            os.chdir(os.path.join(srcdir, "notebooks"))
            nbs = glob("*.ipynb")

            app.info("Executing and converting these notebooks to sphinx files: " + str(nbs))

            nbc_app = NbConvertApp()
            nbc_app.initialize(argv=[])

            nbc_app.writer = OrphanizerWriter()

            nbc_app.export_format = "rst"

            pps = RSTExporter().default_preprocessors
            pps.insert(0, AddSysPath)
            pps.append(RemoveSysPath)
            nbc_app.config.RSTExporter.preprocessors = pps

            nbc_app.notebooks = nbs

            nbc_app.start()
        else:
            app.info("No notebook directory found in docs so not converting any notebooks.")
    except:
        e = sys.exc_info()[0]
        app.warn("Failed to convert notebooks to RST (see above): " + str(e))
    finally:
        os.chdir(olddir)
Ejemplo n.º 10
0
def main():
    p = ArgumentParser()
    p.add_argument("ipynb_notebook_file")
    p.add_argument("-o", "--out")
    p.add_argument("-d", "--default-slide-type", default="skip")
    p.add_argument("--include-input-prompt", action="store_true")
    p.add_argument("--include-output-prompt", action="store_true")
    args = p.parse_args()
    # https://nbconvert.readthedocs.io/en/latest/config_options.html?highlight=TemplateExporter.exclude
    TemplateExporter.exclude_input_prompt = not args.include_input_prompt
    TemplateExporter.exclude_output_prompt = not args.include_output_prompt

    with open(args.ipynb_notebook_file) as nb, FakeSysArgv(
    ), tempfile.NamedTemporaryFile(suffix=".ipynb", mode="w") as out_nb:
        nb_content = json.load(nb)
        for i, cell in enumerate(nb_content["cells"]):
            nb_content["cells"][i].setdefault("metadata", {})
            nb_content["cells"][i]["metadata"].setdefault("slideshow", {})
            nb_content["cells"][i]["metadata"].setdefault("tags", [])
            if cell["cell_type"] == "markdown":
                regex = MARKDOWN_CODE_REGEX
            elif cell["cell_type"] == "code":
                regex = PY_CODE_REGEX
            slide_type = args.default_slide_type
            tags = []
            # check the first line
            if not cell["source"]:
                continue
            m = regex.match(cell["source"][0])
            if m:
                slideshow_config = m.group(1)
                slideshow_config_items = slideshow_config.split()
                slide_type = slideshow_config_items[0]
                if slide_type not in ALLOWED_SLIDE_TYPES:
                    raise ValueError(
                        "unknown slide type: {}".format(slide_type))
                # find tags in format "tags=tag1,tag2,..."
                for item in slideshow_config_items:
                    if not item.startswith("tags="):
                        continue
                    item_tags = item[len("tags="):].split(",")
                    for tag in item_tags:
                        # add only new tags, just in case
                        if tag not in cell["metadata"].get("tags", []):
                            tags.append(tag)
                nb_content["cells"][i]["source"] = cell["source"][
                    1:]  # remove the first line

            nb_content["cells"][i]["metadata"]["slideshow"][
                "slide_type"] = slide_type
            nb_content["cells"][i]["metadata"]["tags"] += tags
        json.dump(nb_content, out_nb)
        out_nb.flush()
        converter = NbConvertApp()
        converter.notebooks = [out_nb.name]
        converter.export_format = "slides"
        if not args.out:
            converter.postprocessor_class = "serve"
        converter.initialize()
        converter.convert_notebooks()

        if args.out:
            base = os.path.splitext(out_nb.name)[0]
            shutil.copy(base + ".slides.html", args.out)
Ejemplo n.º 11
0
def notebooks_to_rst(app):
    from glob import glob

    try:
        # post "big-split", nbconvert is a separate namespace
        from nbconvert.nbconvertapp import NbConvertApp
        from nbconvert.writers import FilesWriter
        from nbconvert.preprocessors import Preprocessor, ExecutePreprocessor, execute
        from nbconvert.exporters import RSTExporter
        from nbformat import NotebookNode
    except ImportError:
        try:
            from IPython.nbconvert.nbconvertapp import NbConvertApp
            from IPython.nbconvert.writers import FilesWriter
            from IPython.nbconvert.preprocessors import Preprocessor, ExecutePreprocessor, execute
            from IPython.nbconvert.exporters import RSTExporter
            from IPython.nbformat import NotebookNode
        except ImportError:
            raise ImportError(
                'Failed to find Jupyter or IPython. Cannot build '
                'the notebooks embedded in the docs. Proceeding '
                'the rest of the doc build, but additional '
                'warnings are likely.')
            return

    class OrphanizerWriter(FilesWriter):
        def write(self, output, resources, **kwargs):
            output = ':orphan:\n\n' + output
            FilesWriter.write(self, output, resources, **kwargs)

    class AddSysPath(Preprocessor):
        """
        Adds the local system path to the top of the notebook.  This makes sure
        when build_sphinx is invoked that the notebook actually runs with the
        current build.
        """
        def preprocess(self, nb, resources):
            syspathstr = 'sys.path = {} + sys.path'.format(str(sys.path))
            cell = {
                'cell_type': 'code',
                'execution_count': None,
                'metadata': {},
                'outputs': [],
                'source': 'import sys\n' + syspathstr
            }
            nb.cells.insert(0, NotebookNode(cell))
            return nb, resources

    class RemoveSysPath(Preprocessor):
        """
        Removes the sys.path cell added by AddSysPath
        """
        def preprocess(self, nb, resources):
            if 'sys.path' in nb.cells[0].source:
                del nb.cells[0]
            return nb, resources

    class MonkeypatchCellExecutionError(execute.CellExecutionError):
        def __str__(self):
            sstr = super(MonkeypatchCellExecutionError, self).__str__()
            return sstr + ' Traceback:\n' + str(self.traceback)

    execute.CellExecutionError = MonkeypatchCellExecutionError

    olddir = os.path.abspath(os.curdir)
    try:
        srcdir = os.path.abspath(os.path.split(__file__)[0])
        if os.path.isdir('notebooks'):
            os.chdir(os.path.join(srcdir, 'notebooks'))
            nbs = glob('*.ipynb')

            app.info(
                "Executing and converting these notebooks to sphinx files: " +
                str(nbs))

            nbc_app = NbConvertApp()
            nbc_app.initialize(argv=[])

            nbc_app.writer = OrphanizerWriter()

            nbc_app.export_format = 'rst'

            pps = RSTExporter().default_preprocessors
            pps.insert(0, AddSysPath)
            pps.append(RemoveSysPath)
            nbc_app.config.RSTExporter.preprocessors = pps

            nbc_app.notebooks = nbs

            nbc_app.start()
        else:
            app.info(
                'No notebook directory found in docs so not converting any notebooks.'
            )
    except:
        e = sys.exc_info()[0]
        app.warn('Failed to convert notebooks to RST (see above): ' + str(e))
    finally:
        os.chdir(olddir)