def display_run_notebook(filename):
"""display function for running a notebook"""
import nbconvert
print("\nNotebook output:")
output = nbconvert.exporters.export(nbconvert.MarkdownExporter(), filename)[0]
print(output)
def notebook2md(fname='index.ipynb'):
"Convert a notebook to README.md in the current working directory"
print('Converting', fname, 'to README.md')
converter = nbconvert.MarkdownExporter()
md, resources = converter.from_filename(fname)
with open('README.md', 'w') as f:
f.write(md)
def extract_from_notebook(fname, outdir=None):
if outdir is None:
outdir = notebook_dir(fname)
_, resources = nbconvert.MarkdownExporter().from_filename(fname)
if "outputs" in resources:
for resource, data in resources["outputs"].items():
path = os.path.join(outdir, resource)
print("{}:{} → {}".format(fname, resource, path))
with open(path, "wb") as handle:
handle.write(data)
def convert(path: Path) -> str:
filename = path.name
log.info(f"convert to markdown: {filename}")
cwd = path.parent
nb = nbformat.reads(path.read_text(), as_version=4)
oldCwd = os.getcwd()
try:
os.chdir(cwd)
ArticleExecutePreprocessor().preprocess(nb)
finally:
os.chdir(oldCwd)
return nbconvert.MarkdownExporter().from_notebook_node(nb)[0]
def _nb2md(self, nb, out_folder, out_base):
"""Run nbconvert on notebook, output to the output folder
"""
c = Config()
c.Application.log_level = 'CRITICAL'
md_exporter = nbc.MarkdownExporter(config=c)
md_exporter.template_file = self.template_path
# Image output path - remove _ so Jekyll will copy them over
out_files_dir = op.join(self.images_folder, out_folder.lstrip('_'))
# Set base name via unique_key as seed for resources dict.
body, resources = md_exporter.from_notebook_node(
nb, resources= {'unique_key': out_base,
'output_files_dir': out_files_dir})
c.FilesWriter.build_directory = out_folder
FilesWriter(config=c).write(body, resources, notebook_name=out_base)
def convert(script, rst, markdown):
"""
Launch Jupyter Notebook (using Docker).
"""
if not script and not rst and not markdown:
all = True
else:
all = False
project_root = get_project_root()
notebooks = os.path.join(project_root, NOTEBOOKS_DIR)
if os.path.isdir(notebooks):
for dirpath, dirnames, filenames in os.walk(notebooks):
for filename in filenames:
if filename.endswith(".ipynb") and os.path.basename(
dirpath) != '.ipynb_checkpoints':
parentpath = os.path.relpath(dirpath, notebooks)
nb = nbf.read(open(os.path.join(dirpath, filename), 'r'),
as_version=4)
if rst or all:
#EXPORT RST
exporter = nbc.RSTExporter()
dir = os.path.join(project_root, RST_DIR, parentpath)
logging.info("Converting the " + filename +
" notebook to an RST format.")
_export(nb, exporter, project_root, dir, parentpath,
filename, '.rst')
if markdown or all:
#EXPORT Markdown
exporter = nbc.MarkdownExporter()
dir = os.path.join(project_root, DOCS_DIR, parentpath)
logging.info("Converting the " + filename +
" notebook to an Markdown format.")
_export(nb, exporter, project_root, dir, parentpath,
filename, '.md')
if script or all:
#EXPORT SCRIPT
exporter = nbc.ScriptExporter()
dir = os.path.join(project_root, SCRIPTS_DIR,
parentpath)
logging.info("Converting the " + filename +
" notebook to a python script.")
_export(nb, exporter, project_root, dir, parentpath,
filename, '.py')
elif filename.endswith(".Rmd"):
#TBD add processing for R notebooks. s
print("R", filename)
def to_github_markdown(self):
"""Export the notebook to Github optimized Markdown.
- Removes header enumeration prefixes from link targets.
('#1.-section-one' --> '#section-one')
"""
check_requirements()
# import on demand to be not required for module import
import nbconvert
from .jinja import FILTERS, LOADER
exporter = nbconvert.MarkdownExporter()
exporter.filters = FILTERS
exporter.extra_loaders = [LOADER]
# not a real file. will be loaded from string by LOADER
exporter.template_file = 'github_markdown'
markdown = exporter.from_filename(self)[0]
# github_markdown template puts code cell input and output
# in single blocks, but can't prevent empty lines in between
#==> Remove them:
markdown = re.sub(
# ```python
# >>> input
#
# output
# ```
r'(\n>>> .+\n)\s*\n([^>`])',
# ```python
# >>> input
# output
# ```
r'\1\2',
markdown)
# And also remove newlines after ```python and before ```:
markdown = re.sub(r'(\n```python.*\n)\s*', r'\1', markdown)
markdown = re.sub(r'\n\s*(```\s*\n)', r'\n\1', markdown)
return markdown
"""
converts jupyter notebooks to markdown using nbconvert
though for now the cli tool is good enough.
Still, this allows more customization.
"""
from traitlets.config import Config
import nbconvert
# Setup config
c = Config()
c.MarkdownExporter.preprocessors = [
"nbconvert.preprocessors.RegexRemovePreprocessor",
"nbconvert.preprocessors.TagRemovePreprocessor",
]
exporter = nbconvert.MarkdownExporter(config=c)
def _kp_read_ref(self, path, reference, revision=None):
print '_kp_read_ref'
print "here:", path, reference
try:
if reference == "knowledge.md":
from jinja2 import DictLoader
from traitlets.config import Config
c = Config()
c.ExtractOutputPreprocessor.output_filename_template = 'images/{unique_key}_{cell_index}_{index}{extension}'
m = nbconvert.MarkdownExporter(config=c, )
title = path.split('.kp')[0]
i = [
d for d in kg.themis.NotebookHub().search(title)
if title in d.get('title')
][0]
r = requests.get(i.get('url'))
(markdown, resource) = m.from_file(
cStringIO.StringIO(
json.dumps(r.json()['notebook']['content'])))
thumbnail = 0
if resource.get('outputs', {}):
thumbnail = resource.get('outputs', {}).keys()[0]
for fn, data in resource.get('outputs', {}).iteritems():
try:
os.makedirs('/data/kb/{}/images'.format(path))
except Exception as e:
print e
with open('/data/kb/{}/{}'.format(path, fn), 'wb') as fout:
fout.write(data)
header = """---
title: "{title}"
authors:
- {author}
tags:
{tags}
created_at: {created}
updated_at: {created}
thumbnail: {thumbnail}
tldr: |
{tldr}
---
""".format(title=title,
author=i.get('owner'),
tags='\n'.join(['- {}'.format(l) for l in i.get('tags')]
or ["- none"]),
created=arrow.get(i.get('update_date')).format('YYYY-MM-DD'),
thumbnail=thumbnail,
tldr='\n'.join([
' {}'.format(l)
for l in (i.get('description', '') or '').split('\n')
]))
return header + markdown
if 'images' in reference:
with open('/data/kb/{}/{}'.format(path, reference),
'rb') as fin:
return fin.read()
except Exception as e:
import traceback
traceback.print_exc()
print e
raise e
import nbconvert
import nbformat
from . import html
from . import markdown
from . import misc
# %% Constants
NOTEBOOK_VERSION = 4
OUTPUT_ENCODING = 'utf-8'
# %% Exporters
md_exporter = nbconvert.MarkdownExporter()
html_exporter = nbconvert.HTMLExporter()
exporters = {
'md': md_exporter,
'html': html_exporter,
}
# %% Read
def read(filename):
"""Read a notebook from file.
"""
return nbformat.read(filename, as_version=NOTEBOOK_VERSION)
{% block stream %}
<div class="output_subarea output_stream output_stdout output_text">
<pre>{{ output.text | ansi2html }}</pre>
</div>
{% endblock stream %}
{% block data_text scoped %}
<div class="output_subarea output_stream output_stdout output_text">
<pre>{{ output.data['text/plain'] | ansi2html }}</pre>
</div>
{% endblock data_text %}
"""
})
for infile in glob("nb/*.ipynb"):
basename, _ = os.path.splitext(os.path.basename(infile))
outfile = "src/%s.md" % basename
print("%s -> %s" % (infile, outfile))
with open(infile) as f:
nb = nbformat.read(f, 4)
ep = nbconvert.preprocessors.ExecutePreprocessor(timeout=300)
ep.preprocess(nb, {'metadata': {'path': 'nb/'}})
m = nbconvert.MarkdownExporter(extra_loaders=[dl])
m.template_file = "markdown2.tpl"
body, resources = m.from_notebook_node(nb)
with open(outfile, "w") as f:
f.write(body)