def get_html_from_filepath(filepath):
"""Convert ipython notebook to html
Return: html content of the converted notebook
"""
config = Config(
{
"CSSHTMLHeaderTransformer": {
"enabled": True,
"highlight_class": ".highlight-ipynb",
},
}
)
config.HTMLExporter.preprocessors = [HtmlLinksPreprocessor]
config.HTMLExporter.exclude_input = True
config.HTMLExporter.exclude_output_prompt = True
config.HtmlLinksPreprocessor["enabled"] = True
path = os.path.dirname(os.path.realpath(__file__))
exporter = HTMLExporter(
config=config,
template_file="no_code",
template_path=[".", path + "/../../../scripts/"],
filters={"highlight2html": custom_highlighter},
)
content, info = exporter.from_filename(filepath)
return content, info
def get_html_from_filepath(filepath, start=0, end=None):
"""Convert ipython notebook to html
Return: html content of the converted notebook
"""
config = Config({
"CSSHTMLHeaderTransformer": {
"enabled": True,
"highlight_class": ".highlight-ipynb",
},
"SubCell": {
"enabled": True,
"start": start,
"end": end
},
})
exporter = HTMLExporter(
config=config,
template_file="basic",
filters={"highlight2html": custom_highlighter},
preprocessors=[SubCell],
)
content, info = exporter.from_filename(filepath)
if BeautifulSoup:
soup = BeautifulSoup(content, "html.parser")
for i in soup.findAll("div", {"class": "input"}):
if i.findChildren()[1].find(text="#ignore") is not None:
i.extract()
content = soup.decode(formatter=None)
return content, info
def get_html_from_filepath(
filepath, start=0, end=None, preprocessors=[], template=None
):
"""Return the HTML from a Jupyter Notebook
"""
template_file = "basic"
extra_loaders = []
if template:
extra_loaders.append(jinja2.FileSystemLoader([os.path.dirname(template)]))
template_file = os.path.basename(template)
config = get_config()
config.update(
{
"CSSHTMLHeaderTransformer": {
"enabled": True,
"highlight_class": ".highlight-ipynb",
},
"SubCell": {"enabled": True, "start": start, "end": end},
}
)
exporter = HTMLExporter(
config=config,
template_file=template_file,
extra_loaders=extra_loaders,
filters={"highlight2html": custom_highlighter},
preprocessors=[SubCell] + preprocessors,
)
config.CSSHTMLHeaderPreprocessor.highlight_class = " .highlight pre "
content, info = exporter.from_filename(filepath)
return content, info
def _build_head() -> str:
import nbformat
from nbconvert.exporters import HTMLExporter
path = importlib_resources.files(
"miyadaiku.themes.ipynb") / EMPTY_IPYNB # type: ignore
src = path.read_bytes()
json = nbformat.reads(src, nbformat.current_nbformat)
html, _ = HTMLExporter({}).from_notebook_node(json)
soup = BeautifulSoup(html, "html.parser")
soup.head.title.extract()
soup.body.extract()
soup.head.unwrap()
soup.html.unwrap()
for x in soup.children:
if isinstance(x, Doctype):
x.extract()
if x.name == "meta":
x.extract()
return str(soup)
def get_html_from_filepath(filepath,
start=0,
end=None,
preprocessors=[],
template=None):
"""Return the HTML from a Jupyter Notebook
"""
template_file = 'basic'
extra_loaders = []
if template:
extra_loaders.append(
jinja2.FileSystemLoader([os.path.dirname(template)]))
template_file = os.path.basename(template)
config = Config({
'CSSHTMLHeaderTransformer': {
'enabled': True,
'highlight_class': '.highlight-ipynb'
},
'SubCell': {
'enabled': True,
'start': start,
'end': end
}
})
exporter = HTMLExporter(config=config,
template_file=template_file,
extra_loaders=extra_loaders,
filters={'highlight2html': custom_highlighter},
preprocessors=[SubCell] + preprocessors)
config.CSSHTMLHeaderPreprocessor.highlight_class = " .highlight pre "
content, info = exporter.from_filename(filepath)
return content, info
def convert_ipynb_to_html(notebook_file, html_file):
"""
Convert the given Jupyter notebook file (``.ipynb``)
to HTML and write it out as the given ``.html`` file.
Parameters
----------
notebook_file : str
Path to input Jupyter notebook file.
html_file : str
Path to output HTML file.
Note
----
This function is also exposed as the
:ref:`render_notebook <render_notebook>` command-line utility.
"""
# set a high timeout for datasets with a large number of features
report_config = Config({
'ExecutePreprocessor': {
'enabled': True,
'timeout': 3600
},
'HTMLExporter': {
'template_path': [template_path],
'template_file': 'report.tpl'
}
})
exportHtml = HTMLExporter(config=report_config)
output, _ = exportHtml.from_filename(notebook_file)
open(html_file, mode='w', encoding='utf-8').write(output)
def init_tornado_settings(self, webapp):
# Init jinja environment
jinja_env = Environment(
loader=FileSystemLoader([handlers.template_path]))
course_dir = self.coursedir.root
notebook_dir = self.parent.notebook_dir
relpath = os.path.relpath(course_dir, notebook_dir)
if relpath.startswith("../"):
nbgrader_bad_setup = True
self.log.error(
"The course directory root is not a subdirectory of the notebook "
"server root. This means that nbgrader will not work correctly. "
"If you want to use nbgrader, please ensure the course directory "
"root is in a subdirectory of the notebook root: %s",
notebook_dir)
else:
nbgrader_bad_setup = False
# Configure the formgrader settings
tornado_settings = dict(
nbgrader_url_prefix=os.path.relpath(self.coursedir.root,
self.parent.notebook_dir),
nbgrader_coursedir=self.coursedir,
nbgrader_exporter=HTMLExporter(config=self.config),
nbgrader_gradebook=None,
nbgrader_db_url=self.coursedir.db_url,
nbgrader_jinja2_env=jinja_env,
nbgrader_bad_setup=nbgrader_bad_setup)
webapp.settings.update(tornado_settings)
def get_html_from_filepath(filepath, start=0, end=None):
"""Convert ipython notebook to html
Return: html content of the converted notebook
"""
config = Config({
'CSSHTMLHeaderTransformer': {
'enabled': True,
'highlight_class': '.highlight-ipynb'
},
'SubCell': {
'enabled': True,
'start': start,
'end': end
}
})
exporter = HTMLExporter(config=config,
template_file='basic',
filters={'highlight2html': custom_highlighter},
preprocessors=[SubCell])
content, info = exporter.from_filename(filepath)
if BeautifulSoup:
soup = BeautifulSoup(content, 'html.parser')
for i in soup.findAll('div', {'class': 'input'}):
if i.findChildren()[1].find(text='#ignore') is not None:
i.extract()
content = soup.decode(formatter="minimal")
return content, info
def start(self):
super(FormgradeApp, self).start()
# Init authenticator.
self.authenticator_instance = self.authenticator_class(
app,
self.ip,
self.port,
self.base_directory,
parent=self)
app.auth = self.authenticator_instance
# now launch the formgrader
app.notebook_dir = self.base_directory
app.notebook_dir_format = self.directory_structure
app.nbgrader_step = self.autograded_directory
app.exporter = HTMLExporter(config=self.config)
app.mathjax_url = self.mathjax_url
url = "http://{:s}:{:d}/".format(self.ip, self.port)
self.log.info("Form grader running at {}".format(url))
self.log.info("Use Control-C to stop this server")
app.gradebook = Gradebook(self.db_url)
app.run(host=self.ip, port=self.port, debug=True, use_reloader=False)
def _process(self):
config = Config()
config.HTMLExporter.preprocessors = [CppHighlighter]
config.HTMLExporter.template_file = 'basic'
with self.attachment.file.open() as f:
notebook = nbformat.read(f, as_version=4)
html_exporter = HTMLExporter(config=config)
body, resources = html_exporter.from_notebook_node(notebook)
css_code = '\n'.join(resources['inlining'].get('css', []))
nonce = str(uuid4())
html = render_template('previewer_jupyter:ipynb_preview.html',
attachment=self.attachment,
html_code=body,
css_code=css_code,
nonce=nonce)
response = current_app.response_class(html)
# Use CSP to restrict access to possibly malicious scripts or inline JS
csp_header = "script-src cdn.mathjax.org 'nonce-{}';".format(nonce)
response.headers['Content-Security-Policy'] = csp_header
response.headers['X-Webkit-CSP'] = csp_header
# IE10 doesn't have proper CSP support, so we need to be more strict
response.headers[
'X-Content-Security-Policy'] = "sandbox allow-same-origin;"
return response
async def render(notebook_id: str = ID_VALIDATOR):
traitlets_config = {
"Highlight2HTML": {
"extra_formatter_options": {
"linenos": "table"
}
}
}
exporter = HTMLExporter(
# Input / output prompts are empty left gutter space
# Let's remove them. If we want gutters, we can CSS them.
exclude_input_prompt=True,
exclude_output_prompt=True,
extra_template_basedirs=[BASE_PATH],
template_name="nbconvert-template",
config=traitlets_config,
)
data, metadata = await backend.get(notebook_id)
if data is None:
# No data found
raise HTTPException(status_code=404)
if metadata.format == "html":
# R notebooks
# I HATE THIS BUT WHAT TO DO?
# We need notebook.js inside the iframe to respond to messages,
# and resize iframe appropriately. We don't have control over the HTML
# and I don't want to parse it. Instead, we just shove this in there.
# VALID HTML!? WHO KNOWS?!
output = "<script src='/static/notebook.js'></script>\n" + data.decode(
)
else:
if metadata.format == "ipynb":
notebook = nbformat.reads(data.decode(), as_version=4)
else:
config = JupytextConfiguration()
# Don't put Rmd and other 'front matter' as a raw cell in the notebook output
# Ideally we'd be able to parse this and display it nicely, but lacking that
# let's not output some raw YAML in the display.
config.root_level_metadata_as_raw_cell = False
notebook = jupytext.reads(data.decode(),
metadata.format,
config=config)
output, resources = exporter.from_notebook_node(
notebook, {"object_metadata": metadata})
return HTMLResponse(
output,
headers={
# Disable embedding our rendered notebook in other websites
# Don't want folks hotlinking our renders.
"Content-Security-Policy": "frame-ancestors 'self';",
"X-Frame-Options": "SAMEORIGIN",
# Intensely cache everything here.
# We can cache bust by purging everything with the cloudflare API,
# or with query params. This is much simpler than caching on
# the server side
"Cache-Control": "public, max-age=604800, immutable",
},
)
def to_html(self):
cb_name = self.nb_buffer.name.split('/')[-1]
notebook_name = cb_name.split('.')[0]
self.write_buffer()
html_exporter = HTMLExporter(config=self.c)
html_exporter.template_file = 'full'
body, resources = html_exporter.from_filename(cb_name)
self.writer.write(body, resources, notebook_name = notebook_name)
def render_ipynb(full_path, format):
"""
Render a given ipynb file
"""
exporter = HTMLExporter()
with open(full_path, encoding="utf-8") as file_handle:
html, _ = exporter.from_file(file_handle)
return Response(html, mimetype="text/html")
def render_ipynb(full_path):
"""
Render a given ipynb file
"""
exporter = HTMLExporter()
with open(full_path, encoding='utf-8') as file_handle:
html, res = exporter.from_file(file_handle)
return Response(html, mimetype='text/html')
def _compile_string(self, nb_json):
"""Export notebooks as HTML strings."""
self._req_missing_ipynb()
c = Config(self.site.config['IPYNB_CONFIG'])
c.update(get_default_jupyter_config())
exportHtml = HTMLExporter(config=c)
body, _ = exportHtml.from_notebook_node(nb_json)
return body
def test_widgets_from_htmlexporter(self):
"""Check jupyter widgets URL"""
with self.create_temp_cwd(["Widget_List.ipynb"]) as tmpdir:
with open(os.path.join(tmpdir, 'Widget_List.ipynb')) as f:
nb = nbformat.read(f, 4)
output, _ = HTMLExporter().from_notebook_node(nb)
assert "var widgetRendererSrc = 'https://unpkg.com/@jupyter-widgets/html-manager@*/dist/embed-amd.js';" in output
def _convert(self, tmpdir: Path, entry: Path, outdir: Path, depth: int):
"""Convert a notebook.
Args:
tmpdir: Temporary working directory
entry: notebook to convert
outdir: output directory for .html and .rst files
depth: depth below root, for fixing image paths
"""
test_mode = self.s.get("test_mode")
# strip special cells.
if self._has_tagged_cells(entry, set(self._cell_tags.values())):
_log.debug(f"notebook '{entry.name}' has test cell(s)")
orig_entry, entry = entry, self._strip_tagged_cells(
tmpdir, entry, ("remove", "exercise"), "testing")
notify(f"Stripped tags from: {orig_entry.name}", 3)
else:
# copy to temporary directory just to protect from output cruft
tmp_entry = tmpdir / entry.name
shutil.copy(entry, tmp_entry)
orig_entry, entry = entry, tmp_entry
# convert all tag-stripped versions of the notebook.
# before running, check if converted result is newer than source file
if self._already_converted(orig_entry, entry, outdir):
notify(
f"Skip notebook conversion, output is newer, for: {entry.name}",
3)
self._results.cached.append(entry)
return
notify(f"Running notebook: {entry.name}", 3)
nb = self._parse_and_execute(entry)
if test_mode: # don't do conversion in test mode
return
notify(f"Exporting notebook '{entry.name}' to directory {outdir}", 3)
wrt = FilesWriter()
# export each notebook into multiple target formats
created_wrapper = False
for (exp, postprocess_func, pp_args) in (
(RSTExporter(), self._postprocess_rst, ()),
(HTMLExporter(), self._postprocess_html, (depth, )),
):
_log.debug(
f"export '{orig_entry}' with {exp} to notebook '{entry}'")
(body, resources) = exp.from_notebook_node(nb)
body = postprocess_func(body, *pp_args)
wrt.build_directory = str(outdir)
wrt.write(body, resources, notebook_name=entry.stem)
# create a 'wrapper' page
if not created_wrapper:
_log.debug(
f"create wrapper page for '{entry.name}' in '{outdir}'")
self._create_notebook_wrapper_page(entry.stem, outdir)
created_wrapper = True
# move notebooks into docs directory
_log.debug(f"move notebook '{entry} to output directory: {outdir}")
shutil.copy(entry, outdir / entry.name)
def _compile_string(self, nb_json):
"""Export notebooks as HTML strings."""
if flag is None:
req_missing(['ipython[notebook]>=2.0.0'], 'build this site (compile ipynb)')
c = Config(self.site.config['IPYNB_CONFIG'])
c.update(get_default_jupyter_config())
exportHtml = HTMLExporter(config=c)
body, _ = exportHtml.from_notebook_node(nb_json)
return body
def get_html_from_filepath(filepath,
start=0,
end=None,
template=None,
execute=False):
"""Return the HTML from a Jupyter Notebook
"""
preprocessors_ = [SubCell]
template_file = "basic"
extra_loaders = []
if template:
extra_loaders.append(
jinja2.FileSystemLoader([os.path.dirname(template)]))
template_file = os.path.basename(template)
# Load the user's nbconvert configuration
app = NbConvertApp()
app.load_config_file()
app.config.update({
# This Preprocessor changes the pygments css prefixes
# from .highlight to .highlight-ipynb
"CSSHTMLHeaderPreprocessor": {
"enabled": True,
"highlight_class": ".highlight-ipynb",
},
"SubCell": {
"enabled": True,
"start": start,
"end": end
},
# "ExecutePreprocessor": {
# "enabled": execute,
# "store_widget_state": True
# }
})
# Overwrite Custom jinja filters
# This is broken right now so needs fix from below
# https://github.com/jupyter/nbconvert/pull/877
# TODO: is this fixed and released?
filters = {
"highlight_code": custom_highlight_code,
}
exporter = HTMLExporter(
config=app.config,
template_file=template_file,
preprocessors=preprocessors_,
extra_loaders=extra_loaders,
filters=filters,
)
content, info = exporter.from_filename(filepath)
return content, info
def generate_exercises():
p = Path(*EXERCISES_DIR)
exporter = HTMLExporter()
exporter.register_preprocessor(ClearOutputPreprocessor(), enabled=True)
for exercise in p.iterdir():
if exercise.suffix == '.ipynb':
html, _ = exporter.from_file(exercise.open())
with open(exercise.with_suffix('.html').name, 'w') as f:
f.write(html)
def _compile_string(self, nb_json):
"""Export notebooks as HTML strings."""
self._req_missing_ipynb()
c = Config(get_default_jupyter_config())
c.merge(Config(self.site.config['IPYNB_CONFIG']))
if 'template_file' not in self.site.config['IPYNB_CONFIG'].get('Exporter', {}):
c['Exporter']['template_file'] = 'basic.tpl' # not a typo
exportHtml = HTMLExporter(config=c)
body, _ = exportHtml.from_notebook_node(nb_json)
return body
def nb2html(nb_path, start=0, end=None, execute=False, kernel_name=""):
"""Convert a notebook and return html"""
logger.info(f"Convert notebook {nb_path}")
# Load the user's nbconvert configuration
app = NbConvertApp()
app.load_config_file()
app.config.update(
{
# This Preprocessor changes the pygments css prefixes
# from .highlight to .highlight-ipynb
"CSSHTMLHeaderPreprocessor": {
"enabled": True,
"highlight_class": ".highlight-ipynb",
},
"SubCell": {"enabled": True, "start": start, "end": end},
"ExecutePreprocessor": {
"enabled": execute,
"store_widget_state": True,
"kernel_name": kernel_name,
},
}
)
preprocessors_ = [SubCell]
filters = {
"highlight_code": custom_highlight_code,
}
template_file = "mkdocs_html/notebook.html.j2"
# template_file = "lab/index.html.j2"
exporter = HTMLExporter(
config=app.config,
template_file=template_file,
# uncomment this line when new nbconvert is released
# https://github.com/jupyter/nbconvert/pull/1429
extra_template_paths=[os.path.join(THIS_DIR, "templates")],
preprocessors=preprocessors_,
filters=filters,
)
# Delete this block when nbconvert is released
# https://github.com/jupyter/nbconvert/pull/1429
# exporter.template_paths.append(os.path.join(THIS_DIR, "templates"))
# print(exporter.template_paths)
# End block
html, info = exporter.from_filename(nb_path)
# HTML and CSS fixes
# html = html_fixes(html, info, fix_css=True, ignore_css=False)
return GENERATED_MD.format(html=html)
def get_html_from_filepath(filepath, start=0, end=None, template=None):
"""Return the HTML from a Jupyter Notebook
"""
preprocessors_ = [SubCell]
template_file = "basic"
extra_loaders = []
if template:
extra_loaders.append(
jinja2.FileSystemLoader([os.path.dirname(template)]))
template_file = os.path.basename(template)
# Load the user's nbconvert configuration
app = NbConvertApp()
app.load_config_file()
app.config.update({
# This Preprocessor changes the pygments css prefixes
# from .highlight to .highlight-ipynb
"CSSHTMLHeaderPreprocessor": {
"enabled": True,
"highlight_class": ".highlight-ipynb",
},
"SubCell": {
"enabled": True,
"start": start,
"end": end
},
})
# Overwrite Custom jinja filters
# This is broken right now so needs fix from below
# https://github.com/jupyter/nbconvert/pull/877
filters = {"highlight_code": custom_highlight_code}
exporter = HTMLExporter(
config=app.config,
template_file=template_file,
extra_loaders=extra_loaders,
filters=filters,
preprocessors=preprocessors_,
)
content, info = exporter.from_filename(filepath)
# Fix for nbconvert bug
content = content.replace("<pre>",
'<pre class="highlight highlight-ipynb">')
# end-fix
# Since we make a Markdown file we need to remove empty lines and strip
content = "\n".join(
[line.strip() for line in content.split("\n") if line.strip()])
return content, info
def compile_string(self, source, is_two_file=True):
"""Export notebooks as HTML strings."""
if flag is None:
req_missing(['ipython[notebook]>=2.0.0'],
'build this site (compile ipynb)')
c = Config(self.site.config['IPYNB_CONFIG'])
exportHtml = HTMLExporter(config=c)
with io.open(source, "r", encoding="utf8") as in_file:
nb_json = nbformat.read(in_file, current_nbformat)
(body, resources) = exportHtml.from_notebook_node(nb_json)
return body
def trace_get_func(job, func):
if func == "custom.css":
return ""
from nbconvert.exporters import HTMLExporter
exporter = HTMLExporter()
fn = os.path.join(tempfile.gettempdir(), job, func+"_output.ipynb")
output, resources = exporter.from_filename(fn)
return output
def notebook_tester(fname, kernelspec='python'):
raw_nb = Exporter().from_filename(fname)
raw_nb[0].metadata.setdefault('kernelspec', {})['name'] = kernelspec
preproc = ExecutePreprocessor(timeout=-1)
try:
exec_nb = preproc.preprocess(*raw_nb)
except Exception as e:
return '[Failed]\n{}'.format(e)
out_nb = HTMLExporter().from_notebook_node(*exec_nb)
fout = fname.replace('.ipynb', '.html')
with io.open(fout, 'w') as f:
f.write(out_nb[0])
return '[Passed]'
def post(self):
defined = []
for v in self.application.parameters:
if v.type is bool:
inp = v.with_value(
self.get_argument(v.name, default='off') == 'on')
else:
inp = v.with_value(v.type(self.get_argument(v.name)))
defined.append(inp)
nb = replace_definitions(self.application.nb, defined)
nb = execute(nb, cwd=os.path.dirname(self.application.path))
output, _ = HTMLExporter().from_notebook_node(nb)
self.write(output)
def main(app):
static_dir = os.path.join(app.builder.srcdir, '_static')
target_dir = os.path.join(app.builder.srcdir, 'notebooks')
source_dir = os.path.abspath(
os.path.join(app.builder.srcdir, '..', 'notebooks'))
rendered_dir = os.path.join(target_dir, 'rendered')
if not os.path.exists(static_dir):
os.makedirs(static_dir)
if not os.path.exists(target_dir):
os.makedirs(target_dir)
if not os.path.exists(rendered_dir):
os.makedirs(rendered_dir)
nbroots = []
nbtitles = []
exporter = HTMLExporter(template_file='full')
for nb_src in glob.glob(os.path.join(source_dir, '*.ipynb')):
print("converting notebook {0}".format(nb_src))
basedir, nbname = os.path.split(nb_src)
nb_dest = os.path.join(target_dir, nbname)
shutil.copyfile(nb_src, nb_dest)
with open(nb_dest, 'r') as f:
nb_json = nbformat.reads(f.read(), as_version=4)
(body, resources) = exporter.from_notebook_node(nb_json)
root, ext = os.path.splitext(nbname)
nb_html_dest = os.path.join(rendered_dir, root + '.html')
with open(nb_html_dest, 'w') as f:
f.write(body)
nbroots.append(root)
nbtitles.append(get_notebook_title(nb_json, root))
for nbroot, nbtitle in zip(nbroots, nbtitles):
with open(os.path.join(target_dir, nbroot + '.rst'), 'w') as f:
f.write(RST_TEMPLATE.render(title=nbtitle, nbroot=nbroot))
with open(os.path.join(target_dir, 'index.rst'), 'w') as f:
f.write(
INDEX_TEMPLATE.render(notebooks=nbroots,
sphinx_tag='notebook-examples'))
def init_tornado_settings(self, webapp):
# Init jinja environment
jinja_env = Environment(
loader=FileSystemLoader([handlers.template_path]))
# Configure the formgrader settings
tornado_settings = dict(
nbgrader_url_prefix=os.path.relpath(self.coursedir.root),
nbgrader_coursedir=self.coursedir,
nbgrader_exporter=HTMLExporter(config=self.config),
nbgrader_gradebook=None,
nbgrader_db_url=self.coursedir.db_url,
nbgrader_jinja2_env=jinja_env,
)
webapp.settings.update(tornado_settings)
def init_tornado_settings(self, webapp):
# Init jinja environment
jinja_env = Environment(loader=FileSystemLoader([handlers.template_path]))
# Configure the formgrader settings
tornado_settings = dict(
nbgrader_notebook_dir=self.coursedir.root,
nbgrader_notebook_dir_format=self.coursedir.directory_structure,
nbgrader_step=self.coursedir.autograded_directory,
nbgrader_exporter=HTMLExporter(config=self.config),
nbgrader_gradebook=Gradebook(self.coursedir.db_url),
nbgrader_jinja2_env=jinja_env,
nbgrader_notebook_url_prefix=os.path.relpath(self.coursedir.root)
)
webapp.settings.update(tornado_settings)
