def nb_to_html(nb, exclude_input=True, exclude_prompts=True):
"""
convert notebook to html
@input nb : notebook
@input exclude_input (bool default:True) : exclude input cells from output
@input exclude_prompts (bool default:True) : exclude prompts from ouput
@returns string : html body
"""
html_exporter = HTMLExporter()
html_exporter.exclude_input = exclude_input
html_exporter.exclude_input_prompt = exclude_prompts
html_exporter.exclude_output_prompt = exclude_prompts
(body, _) = html_exporter.from_notebook_node(nb)
return body
def notebook_view(request_args):
check.dict_param(request_args, 'request_args')
# This currently provides open access to your file system - the very least we can
# do is limit it to notebook files until we create a more permanent solution.
path = request_args['path']
if not path.endswith('.ipynb'):
return 'Invalid Path', 400
with open(os.path.abspath(path)) as f:
read_data = f.read()
notebook = nbformat.reads(read_data, as_version=4)
html_exporter = HTMLExporter()
html_exporter.template_file = 'basic'
(body, resources) = html_exporter.from_notebook_node(notebook)
return '<style>' + resources['inlining']['css'][0] + '</style>' + body, 200
def notebook_view(request_args):
check.dict_param(request_args, "request_args")
# This currently provides open access to your file system - the very least we can
# do is limit it to notebook files until we create a more permanent solution.
path = request_args["path"]
if not path.endswith(".ipynb"):
return "Invalid Path", 400
with open(os.path.abspath(path)) as f:
read_data = f.read()
notebook = nbformat.reads(read_data, as_version=4)
html_exporter = HTMLExporter()
html_exporter.template_file = "basic"
(body, resources) = html_exporter.from_notebook_node(notebook)
return "<style>" + resources["inlining"]["css"][0] + "</style>" + body, 200
def gen_tutorials(repo_dir: str) -> None:
"""Generate HTML tutorials for Docusaurus site from Jupyter notebooks.
Also create ipynb and py versions of tutorial in Docusaurus site for
download.
"""
with open(os.path.join(repo_dir, "website", "tutorials.json"), "r") as infile:
tutorial_config = json.loads(infile.read())
tutorial_ids = {x["id"] for v in tutorial_config.values() for x in v}
for tid in tutorial_ids:
print("Generating {} tutorial".format(tid))
# convert notebook to HTML
ipynb_in_path = os.path.join(repo_dir, "tutorials", "{}.ipynb".format(tid))
with open(ipynb_in_path, "r") as infile:
nb_str = infile.read()
nb = nbformat.reads(nb_str, nbformat.NO_CONVERT)
# displayname is absent from notebook metadata
nb["metadata"]["kernelspec"]["display_name"] = "python3"
exporter = HTMLExporter(template_name="classic")
html, meta = exporter.from_notebook_node(nb)
# pull out html div for notebook
soup = BeautifulSoup(html, "html.parser")
nb_meat = soup.find("div", {"id": "notebook-container"})
del nb_meat.attrs["id"]
nb_meat.attrs["class"] = ["notebook"]
html_out = JS_SCRIPTS + str(nb_meat)
# generate html file
html_out_path = os.path.join(
repo_dir, "website", "tutorials", "{}.html".format(tid)
)
with open(html_out_path, "w") as html_outfile:
html_outfile.write(html_out)
# generate JS file
script = TEMPLATE.format(tid)
js_out_path = os.path.join(
repo_dir, "website", "pages", "tutorials", "{}.js".format(tid)
)
with open(js_out_path, "w") as js_outfile:
js_outfile.write(script)
def notebook_html_get(pipeline_uuid, step_uuid):
pipeline_dir = None
if "pipeline_run_uuid" in request.args:
pipeline_dir = get_pipeline_directory_by_uuid(
pipeline_uuid,
pipeline_run_uuid=request.args.get("pipeline_run_uuid"))
else:
pipeline_dir = get_pipeline_directory_by_uuid(pipeline_uuid)
pipeline_json_path = os.path.join(pipeline_dir,
_config.PIPELINE_DESCRIPTION_PATH)
if os.path.isfile(pipeline_json_path):
with open(pipeline_json_path, "r") as json_file:
pipeline_json = json.load(json_file)
try:
notebook_path = os.path.join(
pipeline_dir,
pipeline_json["steps"][step_uuid]["file_path"])
except Exception as e:
logging.info(e)
return return_404("Invalid JSON for pipeline %s error: %e" %
(pipeline_uuid, e))
else:
return return_404("Could not find pipeline.json for pipeline %s" %
pipeline_uuid)
if os.path.isfile(notebook_path):
try:
with open(notebook_path, "r") as file:
nb = nbformat.read(file, nbformat.NO_CONVERT)
html_exporter = HTMLExporter()
(body, resources) = html_exporter.from_notebook_node(nb)
return body
except IOError as error:
logging.info("Error opening notebook file %s error: %s" %
(notebook_path, error))
return return_404("Could not find notebook file %s" %
notebook_path)
def main(ipynb, kernel_name):
print("running %s" % ipynb)
nb = read(ipynb, as_version=4)
config = get_config()
config.BokehExecutePreprocessor.kernel_name = kernel_name
config.BokehExecutePreprocessor.allow_errors = True
print(config)
ep = BokehExecutePreprocessor(config=config)
ep.preprocess(nb, {'metadata': {'path': './'}})
exportHtml = HTMLExporter()
(body, resources) = exportHtml.from_notebook_node(nb)
outfile = ipynb + ".html"
open(outfile, 'w').write(body)
print("wrote %s" % outfile)
def convert_to_html(result_notebook: Path) -> Path:
"""
:param result_notebook: The path to the result notebook
:return: Path with output extension
"""
print(f"Running conversion to HTML for {result_notebook}")
with result_notebook.open() as f:
notebook = nbformat.read(f, as_version=4)
html_exporter = HTMLExporter()
html_exporter.exclude_input = True
(body, resources) = html_exporter.from_notebook_node(notebook)
write_file = FilesWriter()
write_file.build_directory = str(result_notebook.parent)
write_file.write(output=body,
resources=resources,
notebook_name=str(result_notebook.stem))
return result_notebook.with_suffix(resources['output_extension'])
def create_html(filename):
while True:
file_name = filename.split(".")
htmlfile = os.path.join(
ipynbdir,
file_name[0] + "-" + str(randint(45448, 58595859)) + ".html")
if not os.path.exists(htmlfile):
break
nb = nbformat.read(filename, as_version=4)
html_exporter = HTMLExporter()
# html_exporter.template_file = 'basic'
(body, resources) = html_exporter.from_notebook_node(nb)
print(resources)
with open(htmlfile, "w") as f:
f.write(body)
f.close()
return htmlfile
def doc(elt):
"Show `show_doc` info in preview window along with link to full docs."
global use_relative_links
use_relative_links = False
elt = getattr(elt, '__func__', elt)
md = show_doc(elt, markdown=False)
if is_fastai_class(elt):
md += f'\n\n<a href="{get_fn_link(elt)}" target="_blank" rel="noreferrer noopener">Show in docs</a>'
output = HTMLExporter().markdown2html(md)
use_relative_links = True
try:
page.page({'text/html': output})
except:
try:
get_ipython().run_cell_magic(u'HTML', u'', output)
except:
display(Markdown(md))
def convertNotebooktoHTML(notebookPath,
outfilePath='nb_out',
template='md_not_converted'):
REG_nb = re.compile(r'(\d\d)\.(\d\d)-(.*)\.ipynb')
notebook_basename = os.path.basename(notebookPath)
if REG_nb.match(notebook_basename):
with open(notebookPath) as fh:
nbnode = nbformat.read(notebookPath, as_version=4)
exporter = HTMLExporter()
exporter.template_file = template # leaves markdown not converted, converts input and output cells
exporter.file_extension = '.md'
body, resources = exporter.from_notebook_node(nbnode)
writer = FilesWriter()
writer.write(
body, resources,
notebook_name=outfilePath) # will end up with .html extension
def __init__(self, builderSelf):
self.htmldir = builderSelf.outdir + "/html" #html directory
for path in [self.htmldir]:
ensuredir(path)
self.html_exporter = HTMLExporter()
templateFolder = builderSelf.config['tojupyter_template_path']
if os.path.exists(templateFolder):
pass
else:
builderSelf.logger.warning("template directory not found")
exit()
self.html_exporter.template_file = templateFolder + "/" + builderSelf.config["tojupyter_html_template"]
def nb2html(nb):
"""Helper function to convert a notebook to html for parsing
Parameters
----------
nb : InstructorNotebook
Input Notebook
Returns
-------
(string, string)
body and resurcers from teh html_export file
"""
html_exporter = HTMLExporter()
html_exporter.template_name = 'classic'
(body, resources) = html_exporter.from_notebook_node(nb.contents)
return (body, resources)
def get(self, uuid4):
path = os.path.join('jobs', uuid4, 'output.ipynb')
assert os.path.exists(path)
notebook = nbformat.read(open(path), as_version=4)
mode = self.get_argument('mode', None)
params = notebook.get('metadata', {}).get('extensions',
{}).get('jupyter_dashboards')
if params is None or mode == 'notebook':
exporter = HTMLExporter()
exporter.template_file = 'full'
resources = {}
else:
exporter = get_dashboard_exporter()
hasNotebookMetadata = True
activeView = params['activeView']
if activeView == 'grid_default' or mode == 'grid':
dashboard = params['views']['grid_default']
dashboardLayout = 'grid'
elif activeView == 'report_default' or mode == 'report':
dashboard = params['views']['report_default']
dashboardLayout = 'report'
resources = dict(
params=params,
hasNotebookMetadata=hasNotebookMetadata,
activeView=activeView,
dashboard=dashboard,
dashboardLayout=dashboardLayout,
)
(body, resources) = exporter.from_notebook_node(notebook, resources)
j = dict(html=body)
yml = ruamel.yaml.load(
open('dappled.yml').read(), ruamel.yaml.RoundTripLoader)
if 'results' in yml:
j['results'] = yml['results']
self.finish(json.dumps(j))
def get(self, path=None):
if path:
path += '.ipynb' # when used as a jupyter server extension, we don't use the extension
# if the handler got a notebook_path argument, always serve that
notebook_path = self.notebook_path or path
model = self.contents_manager.get(path=notebook_path)
if 'content' in model:
notebook = model['content']
else:
raise tornado.web.HTTPError(404, 'file not found')
# Fetch kernel name from the notebook metadata
kernel_name = notebook.metadata.get('kernelspec', {}).get(
'name', self.kernel_manager.default_kernel_name)
# Launch kernel and execute notebook
kernel_id = yield tornado.gen.maybe_future(
self.kernel_manager.start_kernel(kernel_name=kernel_name))
km = self.kernel_manager.get_kernel(kernel_id)
result = executenb(notebook, km=km)
# render notebook to html
resources = {'kernel_id': kernel_id, 'base_url': self.base_url}
# Filter out empty cells
# Could this be handled by nbconvert?
if self.strip_sources:
def filter_empty_output(cell):
return cell.cell_type != 'code' or len(cell.outputs) > 0
result.cells = list(filter(filter_empty_output, result.cells))
html, resources = HTMLExporter(
template_file='voila.tpl',
template_path=self.template_path,
exclude_input=self.strip_sources,
exclude_output_prompt=self.strip_sources,
exclude_input_prompt=self.strip_sources).from_notebook_node(
result, resources=resources)
# Compose reply
self.set_header('Content-Type', 'text/html')
self.write(html)
def run(self, parent_data_objs):
self._logger.info("Running stand alone python component")
cmdline = assemble_cmdline_from_args(self._params)
self._logger.debug("cmdline: {}".format(cmdline))
orig_cmdline = sys.argv[1:]
sys.argv[1:] = cmdline
module_main_prog = main_component_module(self._dag_node.comp_desc())
self._logger.info("Main program: {}".format(module_main_prog))
self._logger.info("Dag Node {}".format(self._dag_node))
try:
if self._dag_node.comp_language() == ComponentLanguage.PYTHON:
# Running the module as a script, using the __main__ as the run_name
runpy.run_module(str(module_main_prog),
run_name="__main__",
alter_sys=True)
elif self._dag_node.comp_language() == ComponentLanguage.JUPYTER:
# run the jupyter notebook
self._logger.info("Jupyter Notebook runner")
comp_dir = self._dag_node.comp_root_path()
self._logger.debug("Component directory {} -> {}".format(
comp_dir, os.listdir(comp_dir)))
notebook_filename = os.path.join(comp_dir,
self._dag_node.comp_program())
with open(notebook_filename) as f:
nb = nbformat.read(f, as_version=4)
# TODO this must come from EE - the type of kernel needed
ep = ExecutePreprocessor(timeout=600, kernel_name='python2')
ep.preprocess(nb, {'metadata': {'path': comp_dir}})
html_exporter = HTMLExporter()
# 3. Process the notebook we loaded earlier
(body, resources) = html_exporter.from_notebook_node(nb)
if mlops_loaded:
html = HTML().name("Jupyter Output").data(body)
mlops.set_stat(html)
else:
self._logger.error(
"Jupyter: mlops not loaded, so no output available")
finally:
sys.argv[1:] = orig_cmdline
def notebookExportToHtml(outputFilePath=None):
"""Export current notebook to HTML.
If outputFilePath is not specified then filename will be generated from the notebook filename
with current timestamp appended.
It returns full path of the saved html file.
It requires nbformat and nbconvert packages. You can use this command to install them::
pip_install("nbformat nbconvert")
"""
try:
import nbformat
from nbconvert import HTMLExporter
except ModuleNotFoundError:
import logging
logging.error(
"notebookExportToHtml requires nbformat and nbconvert. They can be installed by running this command:\n\n pip_install('nbformat nbconvert')\n"
)
import datetime, json, os
notebook_path = notebookPath()
# Generate output file path from notebook name and timestamp (if not specified)
if not outputFilePath:
this_notebook_name = os.path.splitext(
os.path.basename(notebook_path))[0]
save_timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
save_file_name = this_notebook_name + "_" + save_timestamp + ".html"
notebooks_save_path = os.path.dirname(notebook_path)
outputFilePath = os.path.join(notebooks_save_path, save_file_name)
with open(notebook_path, mode="r") as f:
file_json = json.load(f)
notebook_content = nbformat.reads(json.dumps(file_json), as_version=4)
html_exporter = HTMLExporter()
(body, resources) = html_exporter.from_notebook_node(notebook_content)
f = open(outputFilePath, 'wb')
f.write(body.encode())
f.close()
return outputFilePath
def do_execute(self, script, kernel, timeout=600):
cells = [nbformat.v4.new_code_cell(source=script)]
nb = nbformat.v4.new_notebook(cells=cells,
metadata={'language': 'python'})
# Execute the script
executor = ExecutePreprocessor(timeout=timeout, kernel_name=kernel)
nb, metadata = executor.preprocess(nb, {'metadata': {}})
# Export html from the notebook
html_exporter = HTMLExporter()
html_exporter.template_file = 'basic'
body, _ = html_exporter.from_notebook_node(nb)
# Extract the result part of the html
output = ScriptExecutor.extract_output(body, kernel)
return output
def _get_preview(template_name: str, warn_on_local: Optional[bool] = True) -> str:
""" Returns an HTML render of a report template, with parameters highlighted. """
cached = get_cache(("preview", template_name))
if cached:
logger.info("Getting %s preview from cache.", template_name)
return cached
nb = template_name_to_notebook_node(template_name, warn_on_local=warn_on_local)
parameters_idx = _get_parameters_cell_idx(nb)
conf = Config()
if parameters_idx is not None:
# Use this template to highlight the cell with parameters
conf.HTMLExporter.template_file = pkg_resources.resource_filename(
__name__, "../nbtemplates/notebook_preview.tpl"
)
exporter = HTMLExporter(config=conf)
html, _ = exporter.from_notebook_node(nb) if nb["cells"] else ("", "")
set_cache(("preview", template_name), html, timeout=30)
return html
def export_html(self, pathname):
"""
Export notebook to .html file
:param pathname: output filename
:return:
"""
html_exporter = HTMLExporter()
(body, resources) = html_exporter.from_notebook_node(self.nb)
if pathname == '-':
sys.__stdout__.write(body)
else:
with open(pathname, 'w') as f:
f.write(body)
logging.info("HTML notebook exported to '{}'".format(pathname))
async def download_notebook(self, request: Request):
context = self.make_request_context(request)
repo_location_name = request.query_params["repoLocName"]
nb_path = request.query_params["path"]
if not nb_path.endswith(".ipynb"):
return PlainTextResponse("Invalid Path", status_code=400)
# get ipynb content from grpc call
notebook_content = context.get_external_notebook_data(repo_location_name, nb_path)
check.inst_param(notebook_content, "notebook_content", bytes)
# parse content to HTML
notebook = nbformat.reads(notebook_content, as_version=4)
html_exporter = HTMLExporter()
html_exporter.template_file = "basic"
(body, resources) = html_exporter.from_notebook_node(notebook)
return HTMLResponse("<style>" + resources["inlining"]["css"][0] + "</style>" + body)
def execute_notebook_io(in_notebook, out_html):
paper_folder, _ = os.path.split(out_html)
os.makedirs(paper_folder, exist_ok=True)
with open(in_notebook, 'r') as notebook_file:
notebook = nbformat.reads(notebook_file.read(), as_version=4)
ep = ExecutePreprocessor(timeout=600,
kernel_name='python3',
allow_errors=True)
ep.preprocess(notebook, {'metadata': {'path': './'}})
html_exporter = HTMLExporter()
(body, _) = html_exporter.from_notebook_node(notebook)
with open(out_html, 'w') as html:
html.write(body)
return notebook
def _convert_to_html_nbconvert(nb_path, out_file):
"""convert executed ipynb file to html document using nbconvert. """
with open(nb_path) as f:
nb = nbformat.read(f, as_version=4)
html_exporter = HTMLExporter()
tag_remove_preprocessor = TagRemovePreprocessor(
remove_cell_tags=["remove_cell"],
remove_all_outputs_tags=["hide_output"],
remove_input_tags=["hide_input"],
)
html_exporter.template_file = "full"
html_exporter.register_preprocessor(tag_remove_preprocessor, enabled=True)
html, resources = html_exporter.from_notebook_node(nb)
with open(out_file, "w") as f:
f.write(html)
def render_nb():
# need ipynb v3 to play nice with runipy
notebook = read(open("stock_infos.ipynb"), 3)
nb = NotebookRunner(notebook, pylab=True)
nb.run_notebook()
# need ipynb v4 to play nice with Jupyter
nb = nbformat.convert(nb.nb, 4)
html_exporter = HTMLExporter()
body, resources = html_exporter.from_notebook_node(nb)
html_file = open("static/stock_infos.html", "w")
html_file.write(body.encode('utf8', 'ignore'))
html_file.close()
return app.send_static_file('stock_infos.html')
def get(self, path):
path = path.strip('/')
cm = content_manager(self) if content_manager else self.contents_manager
log.info("Viewing notebook %s" % path)
try:
model = cm.get(path, content=True)
except web.HTTPError as e:
raise
if model['type'] != 'notebook':
# not a notebook, redirect to files
return FilesRedirectHandler.redirect_to_files(self, path)
html_exporter = HTMLExporter()
html_exporter.template_name = 'classic'
(body, resources) = html_exporter.from_notebook_node(model['content'])
name = path.rsplit('/', 1)[-1]
original_path = path
if 'clone_url' in model:
path = model['clone_url']
if model['clone_url'] and self.get_argument("clone_folder", False):
path = path.split('/')
path.pop(-1)
path = "/".join(path)
self.write(self.render_template('notebook_view.html',
notebook_name=name,
path=original_path,
notebook=body,
resources=resources,
clonable=show_clone,
voila=has_voila,
clone_url=path,
base_url=self.base_url
)
)
self.finish()
def generate_notebook(template_notebook: Path, notebook_params: Dict,
result_notebook: Path) -> Path:
print(f"Writing report to {result_notebook}")
papermill.execute_notebook(input_path=str(template_notebook),
output_path=str(result_notebook),
parameters=notebook_params,
progress_bar=False)
print(f"Running conversion to HTML for {result_notebook}")
with result_notebook.open() as f:
notebook = nbformat.read(f, as_version=4)
html_exporter = HTMLExporter()
html_exporter.exclude_input = True
(body, resources) = html_exporter.from_notebook_node(notebook)
write_file = FilesWriter()
write_file.build_directory = str(result_notebook.parent)
write_file.write(output=body,
resources=resources,
notebook_name=str(result_notebook.stem))
return result_notebook.with_suffix(resources['output_extension'])
def generate_html_from_notebook(self, nb: NotebookNode) -> Text:
"""Converts a provided NotebookNode to HTML.
Args:
nb: NotebookNode that should be converted to HTML.
Returns:
HTML from converted NotebookNode as a string.
"""
# HTML generator and exporter object
html_exporter = HTMLExporter()
template_file = "templates/{}.tpl".format(self.template_type.value)
html_exporter.template_file = str(Path.cwd() / template_file)
# Output generator
self.ep.preprocess(nb, {"metadata": {"path": Path.cwd()}}, self.km)
# Export all html and outputs
body, _ = html_exporter.from_notebook_node(nb)
return body
def runtest(self):
from traitlets.config import Config
from nbconvert import HTMLExporter
from nbconvert.writers import FilesWriter
c = Config()
c.FilesWriter.build_directory = 'outputs/' + self.builddir
c.HTMLExporter.preprocessors = [
'nbconvert.preprocessors.ExecutePreprocessor',
'nbconvert.preprocessors.ExtractOutputPreprocessor'
]
exporter = HTMLExporter(config=c)
os.makedirs(c.FilesWriter.build_directory, exist_ok=True)
(body, resources) = exporter.from_notebook_node(self.nb)
writer = FilesWriter(config=c)
writer.write(body, resources, notebook_name=self.name)
def setUp(self):
self.path_test_nb = path.join(PATH_FIXTURES, self.name + ".ipynb")
self.path_test_html = path.join(PATH_FIXTURES, self.name + ".html")
self.path_test_js = path.join(PATH_JS_TESTS, self.name + ".js")
self.kernel_name = kernelspec.KERNEL_NAME
with open(self.path_test_nb, "r") as f:
self.nb = nbformat.read(f, as_version=4)
self.ep = ExecutePreprocessor(timeout=600, kernel_name=self.kernel_name)
self.html_exporter = HTMLExporter()
self.ep.preprocess(self.nb, {"metadata": {"path": "."}})
(self.body, _) = self.html_exporter.from_notebook_node(self.nb)
with open(self.path_test_html, "w") as f:
f.write(self.body)
def render_notebook(archived_path: str, check_cache=True):
html_path = archived_path.split(".ipynb")[0] + ".html"
if os.path.exists(html_path):
if check_cache:
# file already exists
return html_path
else:
os.remove(html_path)
with open(os.path.abspath(archived_path)) as f:
read_data = f.read()
notebook = nbformat.reads(read_data, as_version=4)
html_exporter = HTMLExporter()
html_exporter.template_file = "basic"
(body, resources) = html_exporter.from_notebook_node(notebook)
html_file = "<style>" + resources["inlining"]["css"][0] + "</style>" + body
with open(html_path, "w") as destination:
destination.write(html_file)
return html_path
def convert_github_file_to_html(file_path, content):
file_content = base64.b64decode(content)
if file_path.endswith('.ipynb'):
html_exporter = HTMLExporter()
html_exporter.template_file = 'basic'
notebook = nbformat.reads(file_content, as_version=4)
(body, _) = html_exporter.from_notebook_node(notebook)
else:
lexer = guess_lexer_for_filename(file_path, '', stripall=True)
formatter = HtmlFormatter(linenos='inline',
full=True,
noclasses=False,
nobackground=True,
lineseparator="<br/>",
classprefix='koopera-code-viewer')
body = highlight(file_content, lexer, formatter)
return body
