def export_notebook(nb, cid):
nb = nbformat.from_dict(nb)
html_exporter = HTMLExporter()
html_exporter.template_file = 'basic'
body = html_exporter.from_notebook_node(nb)[0]
soup = BeautifulSoup(body, 'html.parser')
# mark cells with special name for toggling, and
# TODO make element id's unique by appending cid (for ingester)
for div in soup.find_all('div', 'output_wrapper'):
script = div.find('script')
if script:
script = script.contents[0]
if script.startswith('render_json'):
div['name'] = 'HData'
elif script.startswith('render_table'):
div['name'] = 'Table'
elif script.startswith('render_plot'):
div['name'] = 'Graph'
else:
pre = div.find('pre')
if pre and pre.contents[0].startswith('Structure'):
div['name'] = 'Structure'
# name divs for toggling code_cells
for div in soup.find_all('div', 'input'):
div['name'] = 'Code'
# separate script
script = []
for s in soup.find_all('script'):
script.append(s.text)
s.extract() # remove javascript
return soup.prettify(), '\n'.join(script)
def md2html(md):
"Convert markdown `md` to HTML code"
import nbconvert
if nbconvert.__version__ < '5.5.0': return HTMLExporter().markdown2html(md)
else:
return HTMLExporter().markdown2html(
collections.defaultdict(lambda: collections.defaultdict(dict)), md)
def get_ipynb_content_and_metadata(path, delimeter="---"):
"""
Returns the html of a given ipynb file
"""
contents = path.read_text()
nb = json.loads(contents)
cells = []
for cell in nb["cells"]:
if ("tags" not in cell["metadata"]
or "post_metadata" not in cell["metadata"]["tags"]
and "ignore" not in cell["metadata"]["tags"]):
cells.append(cell)
elif "post_metadata" in cell["metadata"]["tags"]:
metadata = yaml.load("".join(cell["source"]))
nb["cells"] = cells
temporary_nb = tempfile.NamedTemporaryFile()
with open(temporary_nb.name, "w") as f:
f.write(json.dumps(nb))
html_exporter = HTMLExporter()
html_exporter.template_file = "basic"
return html_exporter.from_file(temporary_nb)[0], metadata
def get(self):
path = self.get_argument("path")
html_exporter = HTMLExporter()
html_exporter.template_name = "classic"
notebook_node = nbformat.from_dict(self.get_notebook(path).get("content", {}))
html, _ = html_exporter.from_notebook_node(notebook_node)
self.finish(html)
def bundle_notebook(vid, fileid):
"""Return a file from the bundle"""
from ambry.orm.file import File
import nbformat
from traitlets.config import Config
from nbconvert import HTMLExporter
b = aac.library.bundle(vid)
nbfile = b.build_source_files.file_by_id(fileid)
notebook = nbformat.reads(nbfile.unpacked_contents, as_version=4)
html_exporter = HTMLExporter()
html_exporter.template_file = 'basic'
(body, resources) = html_exporter.from_notebook_node(notebook)
cxt = dict(
vid=vid,
b=b,
fileid=fileid,
nbfile=nbfile,
notebooks=b.build_source_files.list_records(File.BSFILE.NOTEBOOK),
notebook=notebook,
notebook_html=body,
**aac.cc
)
return aac.render('bundle/notebook.html', **cxt)
def export_html(wd, name):
nb = _read(wd, name)
config = {
'Exporter': {
'template_file': 'embed',
'template_path': ['./sphinxext/']
},
'ExecutePreprocessor': {
'enabled': True
},
'ExtractOutputPreprocessor': {
'enabled': True
},
'CSSHTMLHeaderPreprocessor': {
'enabled': True
}
}
exporter = HTMLExporter(config)
try:
body, resources = exporter.from_notebook_node(nb)
for fn, data in resources['outputs'].items():
with open("{}/{}".format(wd, fn), 'wb') as f:
f.write(data)
return body
except Exception as e:
return str(e)
def convert_eda(eda_in_path, eda_out_path, **kwargs):
curdir = os.path.abspath(os.getcwd())
indir, _ = os.path.split(eda_in_path)
outdir, _ = os.path.split(eda_out_path)
os.makedirs(outdir, exist_ok=True)
config = {
"ExecutePreprocessor": {
"enabled": True
},
"TemplateExporter": {
"exclude_output_prompt": True,
"exclude_input": True,
"exclude_input_prompt": True
},
}
nb = nbformat.read(open(eda_in_path), as_version=4)
html_exporter = HTMLExporter(config=config)
# change dir to notebook dir, to execute notebook
os.chdir(indir)
body, resources = (html_exporter.from_notebook_node(nb))
# change back to original directory
os.chdir(curdir)
with open(eda_out_path, 'w') as fh:
fh.write(body)
def preview(request, notebook_url):
try:
notebook_response = urlopen(notebook_url).read().decode()
if notebook_url.endswith('.ipynb'):
html_exporter = HTMLExporter()
html_exporter.template_file = 'full'
notebook_data = nbformat.reads(notebook_response, as_version=4)
(body, resources) = html_exporter.from_notebook_node(notebook_data)
context = {
'notebook_url': notebook_url,
'notebook_preview': body,
}
return render(request, 'upload/preview.html', context)
else:
context = {
'notebook_url': notebook_url,
'notebook_preview': '<pre>' + notebook_response + '</pre>',
}
return render(request, 'upload/preview.html', context)
except HTTPError as error:
context = {
'notebook_url': notebook_url,
'error_msg': error,
}
return render(request, 'upload/preview_error.html', context)
def display_report(name):
name = secure_filename(name)
# Search for the report
report_dir = get_report_dir(name)
if not os.path.isdir(report_dir):
abort(404)
# Get notebook name
report_nb = next(n for n in os.listdir(report_dir) if n.endswith(".ipynb"))
# Execute the notebook
# TODO / NOTE: the notebooks may require some libraries to be installed
# https://nbconvert.readthedocs.io/en/latest/execute_api.html
# https://nbconvert.readthedocs.io/en/latest/nbconvert_library.html
with open(report_dir + '/' + report_nb) as f:
nb = nbformat.read(f, as_version=4)
ep = ExecutePreprocessor(timeout=600, kernel_name='python3')
ep.preprocess(nb, {'metadata': {'path': report_dir}})
html_exporter = HTMLExporter()
# html_exporter.template_file = 'basic'
# Export the notebook into HTML
(body, resources) = html_exporter.from_notebook_node(nb)
return body
def tohtml(working_directory, input, output=None):
prevcwd = os.getcwd()
try:
os.chdir(working_directory)
if output is None:
output = input + ".html"
print("Converting %s --> %s" % (input, output))
ep = ExecutePreprocessor(timeout=300, kernel_name='python3')
with open(input) as f:
nb = nbformat.read(f, as_version=4)
ep.preprocess(nb, {'metadata': {'path': './'}})
#filename_output = input + ".executed.ipynb"
#with open(filename_output, 'wt') as f:
# nbformat.write(nb, f)
exportHTML = HTMLExporter()
(body, resources) = exportHTML.from_notebook_node(nb)
with open(output, 'wt') as f:
f.write(body)
print("Done")
finally:
os.chdir(prevcwd)
def nb_to_html(root, template='basic', version=4, timeout=600, kernel='python3'):
'''
This functions executes a Jupyter notebook and creates the related
HTML file.
Args:
root (str): name of the file without the .ipynb extension
template (str): name of the template (to be in the current folder as template.tpl)
version (int): version of the notebook
timeout (float): maximum time spent per cell
kernel (str)
Returns:
None
The function executes root.ipynb into root_exe.ipynb and creates the file root.html.
'''
with open(root + '.ipynb') as f:
nb = nbformat.read(f, as_version=version)
ep = ExecutePreprocessor(timeout=timeout, kernel_name=kernel)
ep.preprocess(nb, {'metadata': {'path': '.'}})
with open(root + '_exe.ipynb', 'wt') as f:
nbformat.write(nb, f)
html_exporter = HTMLExporter()
html_exporter.template_file = template
with open(root + '_exe.ipynb', mode='r') as f:
notebook = nbformat.reads(''.join(f.readlines()), as_version=version)
(body, _) = html_exporter.from_notebook_node(notebook)
codecs.open(root + '.html', 'w', encoding='utf-8').write(body)
def extract_ipynb(file_, exclude_output: bool):
"""
parse and extract ipynb files
Args:
file_ - file-like object opened in binary mode (+b)
Returns:
html - html version of notebook
info - unmodified (is also passed in)
"""
# local import reduces amortized latency, saves memory
from nbconvert import HTMLExporter
import nbformat
# get the file size
file_.seek(0, os.SEEK_END)
size = file_.tell()
if size > LAMBDA_MAX_OUT:
exclude_output = True
# rewind
file_.seek(0, os.SEEK_SET)
html_exporter = HTMLExporter()
html_exporter.template_file = 'basic'
html_exporter.exclude_output = exclude_output
notebook = nbformat.read(file_, 4)
html, _ = html_exporter.from_notebook_node(notebook)
return html, {}
def generate_data(self,
request,
experiment_output,
experiment,
output_file=None,
output_dir=None):
# use papermill to generate the output notebook
output_file_path = os.path.realpath(output_file.name)
pm.execute_notebook(
os.path.join(BASE_DIR, "path", "to", "notebook.ipynb"),
# TODO: use TemporaryFile instead
'/tmp/output.ipynb',
parameters=dict(
experiment_output={},
experiment={},
output_file=output_file_path,
output_dir=output_dir
)
)
# TODO: convert the output notebook into html format
output_notebook = nbformat.read('/tmp/output.ipynb', as_version=4)
html_exporter = HTMLExporter()
(body, resources) = html_exporter.from_notebook_node(output_notebook)
# TODO: return the HTML output as the output key
return {
'output': body
}
def update_jupyter(self, s, keywords):
'''Update @jupyter node in the vr pane.'''
pc = self
c = pc.c
if pc.must_change_widget(QtWebKitWidgets.QWebView):
# g.trace('===== instantiating QWebView')
w = QtWebKitWidgets.QWebView()
n = c.config.getInt('qweb_view_font_size')
if n:
settings = w.settings()
settings.setFontSize(settings.DefaultFontSize, n)
pc.embed_widget(w)
assert(w == pc.w)
else:
w = pc.w
url = g.getUrlFromNode(c.p)
if url and nbformat:
s = urlopen(url).read().decode()
try:
nb = nbformat.reads(s, as_version=4)
e = HTMLExporter()
(s, junk_resources) = e.from_notebook_node(nb)
except nbformat.reader.NotJSONError:
# Assume the result is html.
pass
elif url:
s = 'can not import nbformt: %r' % url
else:
s = g.u('')
if isQt5:
w.hide() # This forces a proper update.
w.setHtml(s)
w.show()
c.bodyWantsFocusNow()
def execute(self):
print("Cleaning lowfat/reports/html ...")
old_reports = os.listdir("lowfat/reports/html")
for old_report in old_reports:
print("- Removing lowfat/reports/html/{}".format(old_report))
os.remove("lowfat/reports/html/{}".format(old_report))
print("Cleaning of lowfat/reports/html is complete.")
notebook_filenames = os.listdir("lowfat/reports")
for notebook_filename in notebook_filenames:
if not notebook_filename.endswith(".ipynb"):
continue
print("Processing lowfat/reports/{}".format(notebook_filename))
# Based on Executing notebooks, nbconvert Documentation by Jupyter Development Team.
# https://nbconvert.readthedocs.io/en/latest/execute_api.html
with open("lowfat/reports/{}".format(notebook_filename)) as file_:
notebook = nbformat.read(file_, as_version=4)
# Kernel is provided by https://github.com/django-extensions/django-extensions/
execute_preprocessor = ExecutePreprocessor(timeout=600, kernel_name='django_extensions')
execute_preprocessor.preprocess(notebook, {'metadata': {'path': '.'}})
html_exporter = HTMLExporter()
html_exporter.template_file = 'basic'
(body, dummy_resources) = html_exporter.from_notebook_node(notebook)
with open('lowfat/reports/html/{}.html'.format(notebook_filename), 'wt') as file_:
file_.write(body)
def convert_html(nb_path):
"""
Convert a notebook to html
"""
html_exporter = HTMLExporter()
html_exporter.template_file = "basic"
return html_exporter.from_file(str(nb_path))
def ipython_render(tupelo_dir, index_dict, dst_folder, doc_update):
"""
A very rough and temperarary solution to the weird ipython nbconvert API
NOT PRETTY
"""
## make it to forward slash
tupelo_dir_f = _href_link_path(tupelo_dir)
dst_folder_f = _href_link_path(dst_folder)
nav_links = []
for category in index_dict.keys():
nav_links.append(JUPYTER_FULL['navlinks'].format(category = category))
template = ''.join([
JUPYTER_FULL['header'],
JUPYTER_FULL['headerjs'].format(tupelo_dir = tupelo_dir_f),
JUPYTER_FULL['bodyheader'],
JUPYTER_FULL['bodylogo'].format(tupelo_dir = tupelo_dir_f, dst_folder = dst_folder_f),
*nav_links,
JUPYTER_FULL['content'],
JUPYTER_FULL['bodyjs'].format(tupelo_dir = tupelo_dir_f),
JUPYTER_FULL['footer']
])
jupyter_template = DictLoader({'full.tpl': template})
with open(doc_update['doc_src'], 'r') as ipython_doc:
nbdoc = nbformat.reads(ipython_doc.read(), as_version = 4)
exportHTML = HTMLExporter(extra_loaders = [jupyter_temp])
(body, resources) = exportHTML.from_notebook_node(nbdoc)
with open(doc_update['doc_dst'], 'w+') as ipython_html:
ipython_html.write(body)
## incorrecct index
def export_html(wd, name):
nb = _read(wd, name)
config = {
"Exporter": {
"template_file": "embed",
"template_path": ["./sphinxext/"],
},
# 'ExecutePreprocessor': {'enabled': True},
"ExecutePreprocessor": {"enabled": False},
"ExtractOutputPreprocessor": {"enabled": True},
"CSSHTMLHeaderPreprocessor": {"enabled": True},
}
exporter = HTMLExporter(config)
try:
body, resources = exporter.from_notebook_node(nb)
for fn, data in resources["outputs"].items():
with open("{}/{}".format(wd, fn), "wb") as f:
f.write(data)
return body
except Exception as e:
return str(e)
def export_notebook(nb, cid):
nb = nbformat.from_dict(nb)
html_exporter = HTMLExporter()
html_exporter.template_file = "basic"
body = html_exporter.from_notebook_node(nb)[0]
soup = BeautifulSoup(body, "html.parser")
# mark cells with special name for toggling, and
# TODO make element id's unique by appending cid (for ingester)
for div in soup.find_all("div", "output_wrapper"):
script = div.find("script")
if script:
script = script.contents[0]
if script.startswith("render_json"):
div["name"] = "HData"
elif script.startswith("render_table"):
div["name"] = "Tables"
elif script.startswith("render_plot"):
div["name"] = "Graphs"
else:
pre = div.find("pre")
if pre and pre.contents[0].startswith("Structure"):
div["name"] = "Structures"
# name divs for toggling code_cells
for div in soup.find_all("div", "input"):
div["name"] = "Code"
# separate script
script = []
for s in soup.find_all("script"):
script.append(s.text)
s.extract() # remove javascript
return soup.prettify(), "\n".join(script)
def gen_exporter():
config = TraitletsConfig()
config.htmlexporter.preprocessors = [
'nbconvert.preprocessors.extractoutputpreprocessor']
html_exporter = HTMLExporter(config=config)
html_exporter.template_file = 'basic'
return html_exporter
def _exporter():
exporter = HTMLExporter(Config())
exporter.exclude_input_prompt = True
exporter.exclude_output_prompt = True
exporter.template_file = 'jekyll.tpl'
exporter.template_path.append(str(Path(__file__).parent))
return exporter
def export_notebook_to_html(nb=None,
nb_input_file=None,
nb_output_file=None,
template=None):
if nb is None:
if nb_input_file is None:
raise Exception('Please provide either nb or nb_input_file !')
#
nb = read_notebook(nb_input_file=nb_input_file)
#
exporter = HTMLExporter(
template_file=template) if template else HTMLExporter()
html, resources = exporter.from_notebook_node(nb)
if nb_output_file is None:
if nb_input_file is not None:
nb_output_file = nb_input_file.replace('.ipynb', '.html')
#
#
if nb_output_file is not None:
with open(nb_output_file, 'w', encoding='utf-8') as fh:
fh.write(html)
logger.info(f'notebook exported to {nb_output_file}')
#
#
return html
def execute(self):
print("Cleaning lowfat/reports/html ...")
old_reports = os.listdir("lowfat/reports/html")
for old_report in old_reports:
print("- Removing lowfat/reports/html/{}".format(old_report))
os.remove("lowfat/reports/html/{}".format(old_report))
print("Cleaning of lowfat/reports/html is complete.")
notebook_filenames = os.listdir("lowfat/reports")
for notebook_filename in notebook_filenames:
if not notebook_filename.endswith(".ipynb"):
continue
print("Processing lowfat/reports/{}".format(notebook_filename))
# Based on Executing notebooks, nbconvert Documentation by Jupyter Development Team.
# https://nbconvert.readthedocs.io/en/latest/execute_api.html
with open("lowfat/reports/{}".format(notebook_filename)) as file_:
notebook = nbformat.read(file_, as_version=4)
# Kernel is provided by https://github.com/django-extensions/django-extensions/
execute_preprocessor = ExecutePreprocessor(timeout=600, kernel_name='django_extensions')
execute_preprocessor.preprocess(notebook, {'metadata': {'path': '.'}})
html_exporter = HTMLExporter()
html_exporter.template_file = 'basic'
(body, dummy_resources) = html_exporter.from_notebook_node(notebook)
with open('lowfat/reports/html/{}.html'.format(notebook_filename), 'wt') as file_:
file_.write(body)
def notebook_to_html(notebook_path, output_path):
'''Convert a Jupyter/IPython notebook to an HTML file
Parameters
----------
notebook_path: str
output_path: str
Output
------
None
'''
with open(notebook_path, 'r') as fp:
notebook = nbformat.read(fp, nbformat.NO_CONVERT)
# Instantiate the exporter
html_exporter = HTMLExporter()
# html_exporter.template_file = 'basic'
# Exclude code cells from the HTML output
html_exporter.exclude_input = True
# Generate HTML
(body, resources) = html_exporter.from_notebook_node(notebook)
# print("Resources:", resources.keys())
# print("Metadata:", resources['metadata'].keys())
# print("Inlining:", resources['inlining'].keys())
# print("Extension:", resources['output_extension'])
with open(output_path, 'w') as fp:
fp.write(body)
def export_notebook(nb, nb_path, output_dir, SCOPETYPE=None, PLATFORM=None):
"""Takes a notebook node and exports it to ReST and HTML
Args:
nb (notebook): The notebook returned by execute_notebook.
nb_path (str): Path to intput notebook file. Used to generate the
name of the output file.
output_dir (str): The output directory for the ReST and HTML file.
SCOPETYPE (str): Used to generate the output file name.
PLATFORM (str): Used to generate the output file name.
"""
notebook_dir, file_name = os.path.split(nb_path)
file_name_root, _ = os.path.splitext(file_name)
base_path = os.path.join(
output_dir, file_name_root + "-{}-{}".format(SCOPETYPE, PLATFORM))
rst_path = os.path.abspath(base_path + ".rst")
html_path = os.path.abspath(base_path + ".html")
with open(rst_path, "w", encoding='utf-8') as rst_file:
rst_exporter = RSTExporter()
body, res = rst_exporter.from_notebook_node(nb)
rst_file.write(body)
print('Wrote to: ', rst_path)
with open(html_path, 'w', encoding='utf-8') as html_file:
html_exporter = HTMLExporter()
body, res = html_exporter.from_notebook_node(nb)
html_file.write(body)
print('Wrote to: ', html_path)
def run_nb_test(nb_filename, output_fname):
"""test run"""
print(f'(1) Run notebook: {nb_filename}')
ye_notebook = nbformat.read(nb_filename, as_version=4)
client = NotebookClient(ye_notebook,
timeout=600,
kernel_name='python3',
resources={'metadata': {
'path': '.'
}})
client.execute()
if not os.path.isdir('test_output'):
os.makedirs('test_output')
output_file = f'test_output/{output_fname}'
nbformat.write(ye_notebook, output_file)
print(f'(2) file written: {output_file}')
ye_notebook2 = nbformat.read(output_file, as_version=4)
# 2. Instantiate the exporter. We use the `classic` template for now; we'll get into more details
# later about how to customize the exporter further.
html_exporter = HTMLExporter()
html_exporter.template_name = 'classic'
# 3. Process the notebook we loaded earlier
(body, resources) = html_exporter.from_notebook_node(ye_notebook)
html_filename = f'{output_file}.html'
open(html_filename, 'w').write(body)
print(f'file written: {html_filename}')
def notebook_to_html(content, htmlfile):
""" Convert notebook to html file.
Parameters
----------
content : nbformat.NotebookNode
A dict-like node of the notebook with attribute-access
htmlfile : str
Filename for the notebook exported as html
"""
# prepare html exporter, anchor_link_text=' ' suppress anchors being shown
html_exporter = HTMLExporter(anchor_link_text=' ',
exclude_input_prompt=True,
exclude_output_prompt=True)
# export to html
content, _ = html_exporter.from_notebook_node(content)
# check if export path exists
if os.path.dirname(htmlfile) != '' and not os.path.isdir(
os.path.dirname(htmlfile)):
raise FileNotFoundError(
f'Path to html-file does not exist: {os.path.dirname(htmlfile)}')
# write content to html file
with open(htmlfile, 'w', encoding='utf-8') as file:
file.write(content)
def readNotebook(self):
with open(self.filename) as src:
chapter_raw = src.read().decode()
chapter_nb = nbformat.reads(chapter_raw, as_version=4)
html_exporter = HTMLExporter()
html_exporter.template_file = 'basic'
self.notebook = html_exporter.from_notebook_node(chapter_nb)[0]
def _build_exporter(self, exported_data: dict, ws_id: int) -> HTMLExporter:
"""
This builds the HTMLExporter used to export the Notebook (i.e. Narrative) to
HTML. Data is passed into the exporter by configuration with various specific
keys set in the config traitlet.
The NarrativePreprocessor is used to process cells for templating, and consumes
the various elements in the narrative_session property of the config.
This expects to see the set of data exported from the Narrative as part of
its input - this gets passed along to the preprocessor, then to the template for
export.
:param exported_data: Dict - the exported data in the Narrative.
"""
c = Config()
c.HTMLExporter.preprocessors = [preprocessor.NarrativePreprocessor]
# all the static files (css, fonts, etc.) are relative to this dir.
base_path = os.path.dirname(os.path.abspath(__file__))
service_endpt = self.exporter_cfg["kbase-endpoint"]
endpt_parsed = urlparse(service_endpt)
netloc = endpt_parsed.netloc
# kinda hacky. dealing with it.
if netloc.startswith("kbase.us"):
netloc = "narrative." + netloc
host = (endpt_parsed.scheme or "https") + "://" + netloc
c.TemplateExporter.template_path = [
'.', os.path.join(base_path, "static", "templates")
]
c.CSSHTMLHeaderPreprocessor.enabled = True
c.NarrativePreprocessor.enabled = True
c.ClearMetadataPreprocessor.enabled = False
c.narrative_session.token = self.token
c.narrative_session.user_id = self.user_id
c.narrative_session.ws_url = self.exporter_cfg["workspace-url"]
c.narrative_session.nms_url = self.exporter_cfg["nms-url"]
c.narrative_session.nms_image_url = self.exporter_cfg["nms-image-url"]
c.narrative_session.profile_page_url = host + self.exporter_cfg[
"profile-page-path"]
c.narrative_session.auth_url = self.exporter_cfg["auth-url"]
c.narrative_session.assets_base_url = self.exporter_cfg[
"assets-base-url"]
c.narrative_session.service_wizard_url = self.exporter_cfg[
"srv-wiz-url"]
c.narrative_session.data_ie_url = self.exporter_cfg["data-ie-url"]
c.narrative_session.host = host
c.narrative_session.base_path = base_path
c.narrative_session.data_file_path = exported_data["path"]
c.narrative_session.narrative_data = exported_data
c.narrative_session.assets_version = self.exporter_cfg[
"assets-version"]
c.narrative_session.ws_id = ws_id
html_exporter = HTMLExporter(config=c)
html_exporter.template_file = NARRATIVE_TEMPLATE_FILE
return html_exporter
def notebook_to_html(path: Union[Path, str]) -> str:
"""
Convert jupyter notebook to html
Args:
path: path to notebook
Returns: full html page
"""
with open(Path(path)) as fp:
notebook = nbformat.read(fp, as_version=4)
assert 'front-matter' in notebook[
'metadata'], "You must have a front-matter field in the notebook's metadata"
front_matter_dict = dict(notebook['metadata']['front-matter'])
html_exporter = HTMLExporter()
html, _ = html_exporter.from_notebook_node(notebook)
with open(Path(path.parents[0], "front-matter.toml")) as fm:
front_matter_overload = toml.load(fm)
for key in front_matter_overload.keys():
front_matter_dict[key] = front_matter_overload[key]
front_matter = json.dumps(front_matter_dict, indent=2)
return html, front_matter
def build_iab_main(input_dir, output_dir, out_format, ext, css=None):
""" Convert md sources to readable book content, maintaining dir structure.
A few additional processing steps happen here:
* Add Table of Contents to the top of each section.
* Create links from sha1 aliases.
Parameters
----------
input_dir : str
Root path for the markdown files.
output_dir : str
Root path for the output files.
out_format : str
The ipymd format that output files should be written in (for example,
``notebook``).
ext : str
The extension to use for output files.
"""
# Walk the input root directory. We only care about root and files
# inside this loop (nothing happens with dirs).
for unit_number, (unit, chapters) in enumerate(input_dir):
# Iterate over the files in the current root.
if unit_number == 0:
unit_path = ''
else:
unit_path = str(unit_number) + '/'
for chapter_number, content_md in enumerate(chapters):
if chapter_number == 0:
chapter_path = 'index'
else:
chapter_path = str(chapter_number)
path = '%s%s' % (unit_path, chapter_path)
# Convert it from markdown
output_s = ipymd.convert(content_md, from_='markdown', to='notebook')
# define the output filepath
output_fp = get_output_fp(output_dir, path, ext)
try:
os.makedirs(os.path.split(output_fp)[0])
except OSError:
pass
# write the output ipynb
nbformat.write(output_s, output_fp)
if out_format == 'html' or out_format == 's3':
c = Config()
c.ExecutePreprocessor.timeout = 600
html_exporter = HTMLExporter(preprocessors=['nbconvert.preprocessors.execute.ExecutePreprocessor'],
config=c)
for root, dirs, files in os.walk(output_dir):
if css:
shutil.copy(css, os.path.join(root, 'custom.css'))
for f in files:
html_out, _ = html_exporter.from_filename(os.path.join(root, f))
output_fn = os.path.splitext(f)[0] + ext
output_fp = os.path.join(root, output_fn)
open(output_fp, 'w').write(html_out)
def update_jupyter(self, s, keywords):
'''Update @jupyter node in the vr pane.'''
pc = self
c = pc.c
if pc.must_change_widget(QtWebKitWidgets.QWebView):
# g.trace('===== instantiating QWebView')
w = QtWebKitWidgets.QWebView()
n = c.config.getInt('qweb_view_font_size')
if n:
settings = w.settings()
settings.setFontSize(settings.DefaultFontSize, n)
pc.embed_widget(w)
assert(w == pc.w)
else:
w = pc.w
url = g.getUrlFromNode(c.p)
if url and nbformat:
s = urlopen(url).read().decode()
try:
nb = nbformat.reads(s, as_version=4)
e = HTMLExporter()
(s, junk_resources) = e.from_notebook_node(nb)
except nbformat.reader.NotJSONError:
# Assume the result is html.
pass
elif url:
s = 'can not import nbformt: %r' % url
else:
s = g.u('')
if isQt5:
w.hide() # This forces a proper update.
w.setHtml(s)
w.show()
c.bodyWantsFocusNow()
def execute_report(execute_path, nb_path, html_path):
"""Execute the report and write the results as a Jupyter notebook and HTML.
:param execute_path: path to Jupyter notebook to execute
:type execute_path: str
:param nb_path: path to Jupyter notebook to generate
:type nb_path: str
:param html_path: path to HTML to generate
:type html_path: str
:return: tuple containing executed notebook and HTML
:rtype: tuple
"""
with open(execute_path, 'r') as f:
nb = nbformat.read(f, as_version=4)
ep = ExecutePreprocessor(timeout=600)
ep.preprocess(nb)
with open(nb_path, 'w') as f:
nbformat.write(nb, f)
with open(html_path, 'w') as f:
html_exporter = HTMLExporter()
html, resources = html_exporter.from_notebook_node(nb)
f.write(html)
return nb_path, html_path
def read(self, source_path):
'''Parse content and metadata for ipynb files'''
exporter = HTMLExporter(template_file='basic',
preprocessors=config_pres(self.settings))
content, info = exporter.from_filename(source_path)
# Math Support
summary = ""
text = 0
soup = BeautifulSoup(content, 'html.parser')
for x in soup.findAll('a', class_="anchor-link"):
p = x.parent
x.extract()
x.string = "#"
p.insert(0, " ")
p.insert(0, x)
penalty = self.settings.get('CELL_PENALTY', self.DEFAULT_CELL_PENALTY)
summary_size = self.settings.get('SUMMARY_SIZE', self.DEFAULT_SUMMARY_SIZE)
for cell in soup.find_all('div', recursive=False):
delta = len(cell.get_text())# penalty for each cell
delta += penalty * len(cell.find_all('div', ["input", 'output_wrapper'], recursive=False))
if text and text+delta >= summary_size*1.1 or text > summary_size:
break
text += delta
summary += str(cell)
metadata = {'title': get_file_name(source_path), 'summary': summary}
metadata.update(Metadata.data)
metadata['summary'] = summary
# Change Metadata.data to standard pelican metadata
for k, v in metadata.items():
metadata[k] = self.process_metadata(k, v)
return str(soup), metadata
def bundle_notebook(vid, fileid):
"""Return a file from the bundle"""
from ambry.orm.file import File
import nbformat
from traitlets.config import Config
from nbconvert import HTMLExporter
b = aac.library.bundle(vid)
nbfile = b.build_source_files.file_by_id(fileid)
notebook = nbformat.reads(nbfile.unpacked_contents, as_version=4)
html_exporter = HTMLExporter()
html_exporter.template_file = 'basic'
(body, resources) = html_exporter.from_notebook_node(notebook)
cxt = dict(vid=vid,
b=b,
fileid=fileid,
nbfile=nbfile,
notebooks=b.build_source_files.list_records(
File.BSFILE.NOTEBOOK),
notebook=notebook,
notebook_html=body,
**aac.cc)
return aac.render('bundle/notebook.html', **cxt)
def export_notebook_to_html(notebook_fp, output_dir):
nb = read_in_notebook(notebook_fp)
html_exporter = HTMLExporter()
body, resources = html_exporter.from_notebook_node(nb)
_, notebook_name, _ = get_file_name_pieces(notebook_fp)
out_fp = make_file_path(output_dir, notebook_name, ".html")
with open(out_fp, "w", encoding="utf8") as f:
f.write(body)
def export_notebook_to_html(notebook, datestamp, mark_as_latest=True):
html_exporter = HTMLExporter()
html, _resources = html_exporter.from_notebook_node(notebook)
output_path = get_monitoring_notebook_output_path(datestamp, ext='html')
with open(output_path, 'wt') as outfile:
outfile.write(html)
if mark_as_latest:
latest_notebook_path = get_monitoring_notebook_output_path('latest', ext='html')
copyfile(output_path, latest_notebook_path)
def output_HTML(read_file, output_file):
from nbconvert import HTMLExporter
import codecs
import nbformat
exporter = HTMLExporter()
# read_file is '.ipynb', output_file is '.html'
output_notebook = nbformat.read(read_file, as_version=4)
output, resources = exporter.from_notebook_node(output_notebook)
codecs.open(output_file, 'w', encoding='utf-8').write(output)
def write_html(self):
print("writing", self.html)
html_exporter = HTMLExporter()
html_exporter.template_file = 'full'
# do a deeo copy to any chance of overwriting the original notebooks
content = copy.deepcopy(self.content)
content.cells = content.cells[2:-1]
content.cells[0].source = "# " + self.numbered_title
(body, resources) = html_exporter.from_notebook_node(content)
with open(self.html, 'w') as f:
f.write(body)
def convert_nb_html(nb):
"""
Convert a notebooks output to HTML
"""
nb = run_notebook(nb)
config = Config({'HTMLExporter': {'default_template': 'basic'}})
exportHtml = HTMLExporter(config=config)
html, resources = exportHtml.from_notebook_node(nb)
soup = BeautifulSoup(html)
filters = ["output", "text_cell_render border-box-sizing rendered_html"]
return ''.join(map(str, soup.findAll("div", {"class": filters})))
def render(file):
"""Generate the result HTML."""
fp = file.open()
content = fp.read()
fp.close()
notebook = nbformat.reads(content.decode('utf-8'), as_version=4)
html_exporter = HTMLExporter()
html_exporter.template_file = 'basic'
(body, resources) = html_exporter.from_notebook_node(notebook)
return body, resources
def preview(self, filepath):
"""
Preview a notebook store in the Storage
:param filepath: Path to the notebook to preview on Storage
:return a notebook in html format
"""
self.log.debug("Make a Html preview of notebook '%s'" % filepath);
nb = self.read(filepath);
html_conveter = HTMLExporter()
(body, resources) = html_conveter.from_notebook_node(nb)
return body;
def _nbconvert_to_html(cls, doc):
'''Use nbconvert to render a notebook as HTML.
Strip the headings from the first markdown cell to avoid showing the
page title twice on the blog.
'''
if doc.cells and doc.cells[0].cell_type == 'markdown':
source = doc.cells[0].source
doc.cells[0].source = re.sub('^# .*\n', '', source)
e = HTMLExporter()
e.template_file = 'basic'
return e.from_notebook_node(doc)[0]
def _nbconvert_to_html(cls, page):
"""Use nbconvert to render a notebook as HTML.
Strip the headings from the first markdown cell to avoid showing the
page title twice on the blog.
"""
if page["cells"] and page["cells"][0]["cell_type"] == "markdown":
source = page["cells"][0]["source"]
page["cells"][0]["source"] = re.sub("#+.*\n", "", source, re.MULTILINE)
e = HTMLExporter()
e.template_file = "basic"
return e.from_notebook_node(page)[0]
def post(self):
id=self.get_argument("path")
print id
db=create_engine('postgresql://*****:*****@localhost/ishtar')
fileContent=reads_base64(pgquery.get_file(db, "share", id, include_content=True)['content'])
#notebook= nbformat.reads(fileContent, as_version=4)
notebook=fileContent
db.dispose()
html_exporter = HTMLExporter()
html_exporter.template_file = 'basic'
(body, resources) = html_exporter.from_notebook_node(notebook)
self.write(body)
def as_html(self, file):
with File().open(file) as fp:
notebook = nbformat.reads(fp.read().decode(), as_version=4)
exporter = HTMLExporter()
#exporter.template_file = 'basic'
(body, resources) = exporter.from_notebook_node(notebook)
def stream():
cherrypy.response.headers['Content-Type'] = 'text/html'
yield body
return stream
def nb2html(nb_filepath):
"""
Convert notebook to html string.
Args:
nb_filepath (str): Path of notbook file
Returns:
(str): HMTL of converted notebook.
"""
# Save notebook
exporter = HTMLExporter()
exporter.template_file = 'basic'
output, resources = exporter.from_filename(nb_filepath)
return output
def main(ipynb, kernel_name):
print("running %s" % ipynb)
nb = read(ipynb, as_version=4)
config = get_config()
config.BokehExecutePreprocessor.kernel_name = kernel_name
print(config)
ep = BokehExecutePreprocessor(config=config)
ep.preprocess(nb, {'metadata': {'path': './'}})
exportHtml = HTMLExporter()
(body, resources) = exportHtml.from_notebook_node(nb)
outfile = ipynb + ".html"
open(outfile, 'w').write(body)
print("wrote %s" % outfile)
class NarrativeExporter():
def __init__(self):
c = Config()
c.HTMLExporter.preprocessors = [NarrativePreprocessor]
c.TemplateExporter.template_path = ['.', self._narrative_template_path()]
c.CSSHTMLHeaderPreprocessor.enabled = True
self.html_exporter = HTMLExporter(config=c)
self.html_exporter.template_file = 'narrative'
self.ws_client = Workspace(URLS.workspace)
self.narr_fetcher = NarrativeIO()
def _narrative_template_path(self):
return os.path.join(os.environ.get('NARRATIVE_DIR', '.'), 'src', 'biokbase', 'narrative', 'exporter', 'templates')
def export_narrative(self, narrative_ref, output_file):
nar = self.narr_fetcher.read_narrative(narrative_ref)
nar = nar['data']
# # 1. Get the narrative object
# # (put in try/except)
# # (should also raise an error if narrative is not public)
# nar = self.ws_client.get_objects([{'ref': narrative_ref}])
# # put in separate try/except
# nar = nar[0]['data']
# 2. Convert to a notebook object
kb_notebook = nbformat.reads(json.dumps(nar), as_version=4)
# 3. make the thing
(body, resources) = self.html_exporter.from_notebook_node(kb_notebook)
with open(output_file, 'w') as output_html:
output_html.write(body)
def post_save(model, os_path, contents_manager):
"""post-save hook for converting notebooks to .py scripts"""
if model['type'] != 'notebook':
return # only do this for notebooks
if 'Untitled' in os_path:
return # do not save untitled notebooks
output_file_path = construct_output_py_file_path(os_path, skip_if_exists=False)
output_html_path = construct_output_html_file_path(os_path, skip_if_exists=False)
notebook_data = get_notebook_data(os_path)
write_notebook_data_to_py(notebook_data, output_file_path)
exporter = HTMLExporter()
output_notebook = nbformat.read(os_path, as_version=4)
output, resources = exporter.from_notebook_node(output_notebook)
codecs.open(output_html_path, 'w', encoding='utf-8').write(output)
print (output_file_path, "was successfully saved")
def notebook2html(index: dict, in_dir: str, templates: dict, host: str,
out_dir: str, content_dirname: str):
"""
Convert jupyter notebook to html. See relevant docs here:
https://nbconvert.readthedocs.io/en/latest/nbconvert_library.html#Quick-overview
Possible enhancements:
- extract images from notebook
- render from url
"""
new_index = []
for item in index:
if item.get('format') == 'ipynb':
# Render notebook as html
in_fp = f'{in_dir}/{item["in_pth"]}'
notebook = nbformat.read(in_fp, as_version=4)
html_exporter = HTMLExporter()
html_exporter.template_file = 'basic'
nb_html, resources = html_exporter.from_notebook_node(notebook)
# Render navbar
navbar = templates['navbar'].render()
# Render comments section
filename = ntpath.basename(in_fp)[:-len('.ipynb')]
page = {'url': f'{host}/{content_dirname}/{filename}.html',
'identifier': filename}
comments = templates['comments'].render(page=page)
# Render entire page
html = {'navbar': navbar, 'notebook': nb_html, 'comments': comments}
body = templates['notebook'].render(html=html)
# Write html to file
out_fp = f'{out_dir}/{filename}.html'
data2file(body, out_fp)
# Add html path to index
out_pth = f'./{content_dirname}/{filename}.html'
item_new = add2dict('out_pth', out_pth, item)
else:
item_new = item
new_index.append(item_new)
return new_index
def __init__(self):
c = Config()
c.HTMLExporter.preprocessors = [NarrativePreprocessor]
c.TemplateExporter.template_path = ['.', self._narrative_template_path()]
c.CSSHTMLHeaderPreprocessor.enabled = True
self.html_exporter = HTMLExporter(config=c)
self.html_exporter.template_file = 'narrative'
self.ws_client = Workspace(URLS.workspace)
self.narr_fetcher = NarrativeIO()
def render_nb():
# need ipynb v3 to play nice with runipy
notebook = read(open("stock_infos.ipynb"), 3)
nb = NotebookRunner(notebook, pylab=True)
nb.run_notebook()
# need ipynb v4 to play nice with Jupyter
nb = nbformat.convert(nb.nb, 4)
html_exporter = HTMLExporter()
body, resources = html_exporter.from_notebook_node(nb)
html_file= open("static/stock_infos.html","w")
html_file.write(body.encode('utf8', 'ignore'))
html_file.close()
return app.send_static_file('stock_infos.html')
def export_html(nb, f):
config = {
'Exporter': {'template_file': 'embed',
'template_path': ['./sphinxext/']},
'ExtractOutputPreprocessor': {'enabled': True},
'CSSHTMLHeaderPreprocessor': {'enabled': True}
}
exporter = HTMLExporter(config)
body, resources = exporter.from_notebook_node(
nb, resources={'output_files_dir': f['nbname']})
for fn, data in resources['outputs'].items():
bfn = os.path.basename(fn)
with open("{destdir}/{fn}".format(fn=bfn, **f), 'wb') as res_f:
res_f.write(data)
return body
def export_unit_to_html(unit):
"""Export unit into html format."""
slider_start = '// Slider JS Block START'
slider_end = '// Slider JS Block END'
replacement_start = '(function (requirejs, require, define) {'
replacement_end = ('}(RequireJS.requirejs, RequireJS.require, '
'RequireJS.define));')
bootstrap_css = ('<link rel="stylesheet" '
'href="/static/bootstrap.edx.css">\n')
hvcss = ('<link rel="stylesheet" '
'href="/static/holoviews.edx.css">\n')
path = os.path.dirname(os.path.realpath(__file__))
cfg = Config({'HTMLExporter':{'template_file':'no_code',
'template_path':['.',path],
'filters':{'markdown2html':
markdown2html_pandoc}}})
exportHtml = HTMLExporter(config=cfg)
(body, resources) = exportHtml.from_notebook_node(unit)
body = re.sub(r'\\begin\{ *equation *\}', '\[', body)
body = re.sub(r'\\end\{ *equation *\}', '\]', body)
if slider_start in body:
soup = bs4.BeautifulSoup(body, 'lxml')
labels = [strong for form in soup.find_all('form',
attrs={'class':'holoform'})
for strong in form.find_all('strong')]
for label in labels:
new = re.sub(r'[$\\{}]', '', label.contents[0])
label.contents[0].replace_with(new)
body = soup.__str__()
body = body.replace('hololayout', 'bootstrap-wrapper')
body = body.replace('span9 col-xs-8 col-md-9', '')
body = body.replace('span3 col-xs-4 col-md-3',
'col-md-6 col-md-offset-3 center-widget')
body = body.replace(slider_start, replacement_start)
body = body.replace(slider_end, replacement_end)
body = hvjs + bootstrap_css + hvcss + body
return body
def export_notebook(nb, cid):
nb = nbformat.from_dict(nb)
html_exporter = HTMLExporter()
html_exporter.template_file = 'basic'
body = html_exporter.from_notebook_node(nb)[0]
soup = BeautifulSoup(body, 'html.parser')
# mark cells with special name for toggling, and
# TODO make element id's unique by appending cid
for div in soup.find_all('div', 'output_wrapper'):
tag = div.find('h2')
div['name'] = tag.text.split()[0]
# name divs for toggling code_cells
for div in soup.find_all('div', 'input'):
div['name'] = 'Input'
# separate script
script = []
for s in soup.find_all('script'):
script.append(s.text)
s.extract() # remove javascript
return soup.prettify(), '\n'.join(script)
def md_to_html(input_fp, output_fp, ignore_badges=True):
""" Convert a markdown file to HTML.
This is useful for converting the README.md to an index.html.
"""
md_lines = []
for line in open(input_fp):
line = line.strip('\n')
if ignore_badges and line.startswith('[!'):
continue
md_lines.append(line)
nb_s = ipymd.convert('\n'.join(md_lines),
from_='markdown', to='notebook')
print(type(nb_s))
html_exporter = HTMLExporter()
html_out, _ = html_exporter.from_notebook_node(nb_s)
#html = markdown2.markdown('\n'.join(md_lines))
open(output_fp, 'w').write(html_out)
def export_html(wd, name):
nb = _read(wd, name)
config = {
'Exporter': {'template_file': 'embed',
'template_path': ['./sphinxext/']},
'ExecutePreprocessor': {'enabled': True},
'ExtractOutputPreprocessor': {'enabled': True},
'CSSHTMLHeaderPreprocessor': {'enabled': True}
}
exporter = HTMLExporter(config)
body, resources = exporter.from_notebook_node(nb)
for fn, data in resources['outputs'].items():
with open("{}/{}".format(wd, fn), 'wb') as f:
f.write(data)
return body
def convert_notebooks(in_directory, html_directory, static_directory):
dl = DictLoader({
'post.tpl':
"""
{%- extends 'basic.tpl' -%}
{% block body %}---
title: {{nb.metadata['title']}}
notebook: {{resources['metadata']['path']}}/{{resources['metadata']['name']}}.ipynb
date: {{nb.metadata['date']}}
---
{{ super() }}
{% endblock body %}
"""
})
c = Config()
c.HTMLExporter.preprocessors = [
'nbconvert.preprocessors.ExtractOutputPreprocessor'
]
html_exporter = HTMLExporter(config=c, extra_loaders=[dl])
html_exporter.template_file = 'post.tpl'
writer = FilesWriter(build_directory=html_directory)
for notebook_file in glob(path.join(in_directory, '*.ipynb')):
out_name, _ = path.splitext(path.basename(notebook_file))
out_name = out_name.lower().replace(' ', '-')
print('Converting {}'.format(notebook_file))
(body, resources) = html_exporter.from_filename(
notebook_file,
resources={'output_files_dir': out_name})
writer.write(body, resources, notebook_name=out_name)
shutil.rmtree(path.join(static_directory, out_name), True)
rename(path.join(html_directory, out_name),
path.join(static_directory, out_name))
def notebook_to_html(text, record=None):
# construct full path to the notebook (in same dir)
directory,contents_lr = os.path.split(record.source_filename)
notebook_path = os.path.join(directory,text)
# verify that the named notebook is among the attachments
for att in record.attachments:
if att.attachment_filename == notebook_path:
break
else:
raise RuntimeError("Couldn't find notebook file")
# render it
with open(notebook_path) as fl:
nb = nbformat.read(fl, as_version=4)
exporter = HTMLExporter()
exporter.template_file = 'basic'
body,resources = exporter.from_notebook_node(nb)
return body,resources