def restore_checkpoint(self, checkpoint_id, name, path=''):
self.log.info("restoring Notebook %s from checkpoint %s", name, checkpoint_id)
nb_path = self._get_s3_path(name, path)
cp_path = self.get_checkpoint_path(checkpoint_id, name, path)
if not self.path_exists(cp_path):
self.log.debug("checkpoint file does not exist: %s", cp_path)
raise web.HTTPError(404,
u'Notebook checkpoint does not exist: %s-%s' % (name, checkpoint_id)
)
# ensure notebook is readable (never restore from an unreadable notebook)
key = self.bucket.get_key(cp_path)
nb = key.get_contents_as_string()
current.reads_json(nb)
self._copy(cp_path, nb_path)
self.log.debug("copying %s -> %s", cp_path, nb_path)
def render_content(content, url=None, forced_theme=None):
try:
nb = nbformat.reads_json(content)
except ValueError:
raise NbFormatError('Error reading json notebook')
css_theme = nb.get('metadata', {}).get('_nbviewer', {}).get('css', None)
if css_theme and not re.match('\w', css_theme):
css_theme = None
if forced_theme and forced_theme != 'None':
css_theme = forced_theme
# get the notebook title
try:
name = nb.metadata.name
except AttributeError:
name = None
if not name:
name = url.rsplit('/')[-1]
if not name.endswith(".ipynb"):
name = name + ".ipynb"
config = {
'download_url': url,
'download_name': name,
'css_theme': css_theme,
'mathjax_conf': None,
}
return body_render(config, body=C.from_notebook_node(nb)[0])
def render_content(content, url=None, forced_theme=None):
try :
nb = nbformat.reads_json(content)
except ValueError:
raise NbFormatError('Error reading json notebook')
css_theme = nb.get('metadata', {}).get('_nbviewer', {}).get('css', None)
if css_theme and not re.match('\w', css_theme):
css_theme = None
if forced_theme and forced_theme != 'None' :
css_theme = forced_theme
# get the notebook title
try:
name = nb.metadata.name
except AttributeError:
name = url.rsplit('/')[-1]
if not name.endswith(".ipynb"):
name = name + ".ipynb"
config = {
'download_url': url,
'download_name': name,
'css_theme': css_theme,
'mathjax_conf': None,
}
return body_render(config, body=C.convert(nb)[0])#body_render(config, body)
def get_files(filepath):
"""
Returns a list of the Python files in a directory, and
converts IPython notebooks into Python source code and
includes them with the Python files.
"""
os.chdir(filepath)
files = []
for root, dirnames, filenames in os.walk("."):
for filename in fnmatch.filter(filenames, "*.py"):
files.append(os.path.join(root, filename))
for filename in fnmatch.filter(filenames, "*.ipynb"):
try:
with open(filename) as fh:
nb = nbformat.reads_json(fh.read())
export_path = filename.replace(".ipynb", ".py")
exporter = PythonExporter()
source, meta = exporter.from_notebook_node(nb)
with open(export_path, "w+") as fh:
fh.writelines(source)
files.append()
except Exception as e:
raise RepoLoaderWarning(
f"May have issues with JSON encoding. The error message is: {e}"
)
return files
def convert(exporter, nb_file, suffix=''):
nb_md = modification_date(nb_file)
out_file = nb_file.replace('.ipynb', suffix+'.html').replace(' ','_')
if os.path.exists(out_file):
out_md = modification_date(out_file)
if out_md > nb_md:
return None
#print "NBConvert: {}".format(nb_file)
with open(nb_file) as f:
notebook = nbformat.reads_json(f.read())
## I use basic here to have less boilerplate and headers in the HTML.
## we'll see later how to pass config to exporters.
(body,resources) = exporter.from_notebook_node(notebook)
with open(out_file,'w') as f:
f.write(body.encode('utf8'))
return out_file
def nbConverter(self, exporter, extension):
self.infile = self.titleHistory[-1] + '.ipynb'
self.notebook = open(self.infile).read()
self.nb_json = nbformat.reads_json(self.notebook)
self.exportHtml = exporter
(body, resources) = self.exportHtml.from_notebook_node(self.nb_json)
self.outfile = self.titleHistory[-1] + extension
open(self.outfile, 'w').write(body.encode('utf-8'))
def nbConverter(self, exporter, extension):
self.infile = self.titleHistory[-1] + '.ipynb'
self.notebook = open(self.infile).read()
self.nb_json = nbformat.reads_json(self.notebook)
self.exportHtml = exporter
(body, resources) = self.exportHtml.from_notebook_node(self.nb_json)
self.outfile = self.titleHistory[-1] + extension
open(self.outfile, 'w').write(body.encode('utf-8'))
def summerize_pickles(notebook_path):
from IPython.nbformat import current as nbformat
import re
with open(notebook_path) as fh:
nb = nbformat.reads_json(fh.read())
list_of_input_pickles = []
list_of_output_pickles = []
for cell in nb["worksheets"][0]["cells"]:
# This confirms there is at least one pickle in it.
if cell["cell_type"] != "code" or cell["input"].find(
"pickle"
) == -1: # Skipping over those cells which aren't code or those cells with code but which don't reference "pickle
continue
cell_content = cell["input"]
# In case there are multiple lines, it iterates line by line.
for line in cell_content.splitlines():
if line.find(
"pickle"
) == -1: # Skips over lines w/ no mention of "pickle" to potentially reduce the number of times it's searched.
continue
############################ ############################ ############################ ############################
code_type = str()
if line.find("pickle.dump") != -1 or line.find(".to_pickle") != -1:
code_type = "output"
elif line.find("pickle.load") != -1 or line.find(
".read_pickle") != -1:
code_type = "input"
else:
continue # This tells the code to skip over lines like "import cpickle as pickle"
############################ ############################ ############################ ############################
filename = re.findall(
r'"(.*?)"', line
) # This gets all the content between the quotes. See: http://stackoverflow.com/questions/171480/regex-grabbing-values-between-quotation-marks
if filename == []:
line = str(line)
start = line.find("(")
end = line.find(",")
extracted = line[start + 1:end] + ".pickle"
filename = [extracted]
############################ ############################ ############################ ############################
if code_type == "input":
list_of_input_pickles.append(filename[0])
elif code_type == "output":
list_of_output_pickles.append(filename[0])
pickles_dict = {
"input_pickles": list_of_input_pickles,
"output_pickles": list_of_output_pickles
}
return pickles_dict
def finish_notebook(self,
json_notebook,
download_url,
home_url=None,
msg=None,
breadcrumbs=None,
public=False):
"""render a notebook from its JSON body.
download_url is required, home_url is not.
msg is extra information for the log message when rendering fails.
"""
if msg is None:
msg = download_url
try:
nb = reads_json(json_notebook)
except ValueError:
app_log.error("Failed to render %s", msg, exc_info=True)
raise web.HTTPError(400, "Error reading JSON notebook")
try:
app_log.debug("Requesting render of %s", download_url)
with self.time_block("Rendered %s" % download_url):
app_log.info("rendering %d B notebook from %s",
len(json_notebook), download_url)
nbhtml, config = yield self.pool.submit(
render_notebook,
self.exporter,
nb,
download_url,
config=self.config,
)
except NbFormatError as e:
app_log.error("Invalid notebook %s: %s", msg, e)
raise web.HTTPError(400, str(e))
except Exception as e:
app_log.error("Failed to render %s", msg, exc_info=True)
raise web.HTTPError(400, str(e))
else:
app_log.debug("Finished render of %s", download_url)
html = self.render_template('notebook.html',
body=nbhtml,
download_url=download_url,
home_url=home_url,
date=datetime.utcnow().strftime(date_fmt),
breadcrumbs=breadcrumbs,
login=True if self.current_user else False,
**config)
yield self.cache_and_finish(html)
# Index notebook
self.index.index_notebook(download_url, nb, public)
def main():
"""The main called from the commandline"""
arguments = docopt.docopt(__doc__)
infile_name = arguments["<input>"]
outfile_name = arguments["<output>"]
json_string = open(infile_name).read()
data = nbformat.reads_json(json_string)
stripped_data = strip_solutions(data)
with open(outfile_name, "w") as outfile:
json.dump(stripped_data, outfile)
def finish_notebook(self, json_notebook, download_url, home_url=None, msg=None,
breadcrumbs=None, public=False, format=None, request=None):
"""render a notebook from its JSON body.
download_url is required, home_url is not.
msg is extra information for the log message when rendering fails.
"""
if msg is None:
msg = download_url
try:
nb = reads_json(json_notebook)
except ValueError:
app_log.error("Failed to render %s", msg, exc_info=True)
raise web.HTTPError(400, "Error reading JSON notebook")
try:
app_log.debug("Requesting render of %s", download_url)
with self.time_block("Rendered %s" % download_url):
app_log.info("rendering %d B notebook from %s", len(json_notebook), download_url)
nbhtml, config = yield self.pool.submit(render_notebook,
self.formats[format]["exporter"], nb, download_url,
config=self.config,
)
except NbFormatError as e:
app_log.error("Invalid notebook %s: %s", msg, e)
raise web.HTTPError(400, str(e))
except Exception as e:
app_log.error("Failed to render %s", msg, exc_info=True)
raise web.HTTPError(400, str(e))
else:
app_log.debug("Finished render of %s", download_url)
html = self.render_template(
"formats/%s.html" % format,
body=nbhtml,
download_url=download_url,
home_url=home_url,
format=self.format,
format_prefix=self.format_prefix,
formats=dict(self.filter_formats(nb, json_notebook)),
format_base=self.request.uri.replace(self.format_prefix, ""),
date=datetime.utcnow().strftime(date_fmt),
breadcrumbs=breadcrumbs,
**config)
yield self.cache_and_finish(html)
# Index notebook
self.index.index_notebook(download_url, nb, public)
def compile_html(self, source, dest, is_two_file=True):
if flag is None:
req_missing(['ipython>=1.1.0'], 'build this site (compile ipynb)')
makedirs(os.path.dirname(dest))
HTMLExporter.default_template = 'basic'
c = Config(self.site.config['IPYNB_CONFIG'])
exportHtml = HTMLExporter(config=c)
with codecs.open(dest, "w+", "utf8") as out_file:
with codecs.open(source, "r", "utf8") as in_file:
nb = in_file.read()
nb_json = nbformat.reads_json(nb)
(body, resources) = exportHtml.from_notebook_node(nb_json)
out_file.write(body)
def compile_html(self, source, dest, is_two_file=True):
if flag is None:
req_missing(['ipython>=1.0.0'], 'build this site (compile ipynb)')
makedirs(os.path.dirname(dest))
HTMLExporter.default_template = 'basic'
c = Config(self.site.config['IPYNB_CONFIG'])
exportHtml = HTMLExporter(config=c)
with codecs.open(dest, "w+", "utf8") as out_file:
with codecs.open(source, "r", "utf8") as in_file:
nb = in_file.read()
nb_json = nbformat.reads_json(nb)
(body, resources) = exportHtml.from_notebook_node(nb_json)
out_file.write(body)
def compile_html(self, source, dest, is_two_file=True):
if flag is None:
raise Exception('To build this site, you need '
'to install IPython 1.0.')
makedirs(os.path.dirname(dest))
HTMLExporter.default_template = 'basic'
exportHtml = HTMLExporter()
with codecs.open(dest, "w+", "utf8") as out_file:
with codecs.open(source, "r", "utf8") as in_file:
nb = in_file.read()
nb_json = nbformat.reads_json(nb)
(body, resources) = exportHtml.from_notebook_node(nb_json)
out_file.write(body)
def render_notebook(exporter,
json_notebook,
url=None,
forced_theme=None,
config=None):
app_log.info("rendering %d B notebook from %s", len(json_notebook), url)
if not isinstance(exporter, Exporter):
# allow exporter to be passed as a class, rather than instance
# because Exporter instances cannot be passed across multiprocessing boundaries
# instances are cached by class to avoid repeated instantiation of duplicates
exporter_cls = exporter
if exporter_cls not in exporters:
app_log.info("instantiating %s" % exporter_cls.__name__)
exporters[exporter_cls] = exporter_cls(config=config, log=app_log)
exporter = exporters[exporter_cls]
try:
nb = reads_json(json_notebook)
except ValueError:
raise NbFormatError('Error reading JSON notebook')
css_theme = nb.get('metadata', {}).get('_nbviewer', {}).get('css', None)
if not css_theme or not css_theme.strip():
# whitespace
css_theme = None
if forced_theme:
css_theme = forced_theme
# get the notebook title, if any
try:
name = nb.metadata.name
except AttributeError:
name = ''
if not name and url is not None:
name = url.rsplit('/')[-1]
if not name.endswith(".ipynb"):
name = name + ".ipynb"
html, resources = exporter.from_notebook_node(nb)
config = {
'download_name': name,
'css_theme': css_theme,
}
return html, config
def display(user, path):
full_path = os.path.join('/home', user, 'notebooks', path + '.ipynb')
if not os.path.exists(full_path):
return "No such notebook", 404
mtime = str(os.stat(full_path).st_mtime)
key = cache_key(user, path, mtime)
cached = redis.get(key)
if cached:
return cached
exportHtml = HTMLExporter(config=Config({'HTMLExporter':{'default_template':'basic'}}))
notebook = nbformat.reads_json(open(full_path).read())
body, res = exportHtml.from_notebook_node(notebook)
redis.set(key, body)
redis.expire(key, app.config['REDIS_EXPIRY_SECONDS'])
return body
def main(app):
static_dir = os.path.join(app.builder.srcdir, '_static')
target_dir = os.path.join(app.builder.srcdir, 'notebooks')
source_dir = os.path.abspath(
os.path.join(app.builder.srcdir, '..', 'notebooks'))
rendered_dir = os.path.join(target_dir, 'rendered')
if not os.path.exists(static_dir):
os.makedirs(static_dir)
if not os.path.exists(target_dir):
os.makedirs(target_dir)
if not os.path.exists(rendered_dir):
os.makedirs(rendered_dir)
nbroots = []
nbtitles = []
exporter = HTMLExporter(template_file='full')
for nb_src in glob.glob(os.path.join(source_dir, '*.ipynb')):
print("converting notebook {0}".format(nb_src))
basedir, nbname = os.path.split(nb_src)
nb_dest = os.path.join(target_dir, nbname)
shutil.copyfile(nb_src, nb_dest)
with open(nb_dest, 'r') as f:
nb_json = nbformat.reads_json(f.read())
(body, resources) = exporter.from_notebook_node(nb_json)
root, ext = os.path.splitext(nbname)
nb_html_dest = os.path.join(rendered_dir, root + '.html')
with open(nb_html_dest, 'w') as f:
f.write(body)
nbroots.append(root)
nbtitles.append(get_notebook_title(nb_json, root))
for nbroot, nbtitle in zip(nbroots, nbtitles):
with open(os.path.join(target_dir, nbroot + '.rst'), 'w') as f:
f.write(RST_TEMPLATE.render(title=nbtitle, nbroot=nbroot))
with open(os.path.join(target_dir, 'index.rst'), 'w') as f:
f.write(
INDEX_TEMPLATE.render(notebooks=nbroots,
sphinx_tag='notebook-examples'))
def compile_html(self, source, dest, is_two_file=True):
if flag is None:
raise Exception('To build this site, you need '
'to install IPython 1.0.')
try:
os.makedirs(os.path.dirname(dest))
except:
pass
HTMLExporter.default_template = 'basic'
exportHtml = HTMLExporter()
with codecs.open(dest, "w+", "utf8") as out_file:
with codecs.open(source, "r", "utf8") as in_file:
nb = in_file.read()
nb_json = nbformat.reads_json(nb)
(body, resources) = exportHtml.from_notebook_node(nb_json)
out_file.write(body)
def main(app):
static_dir = os.path.join(app.builder.srcdir, '_static')
target_dir = os.path.join(app.builder.srcdir, 'notebooks')
source_dir = os.path.abspath(os.path.join(app.builder.srcdir,
'..', 'notebooks'))
rendered_dir = os.path.join(target_dir, 'rendered')
if not os.path.exists(static_dir):
os.makedirs(static_dir)
if not os.path.exists(target_dir):
os.makedirs(target_dir)
if not os.path.exists(rendered_dir):
os.makedirs(rendered_dir)
nbroots = []
nbtitles = []
exporter = HTMLExporter(template_file='full')
for nb_src in glob.glob(os.path.join(source_dir, '*.ipynb')):
print("converting notebook {0}".format(nb_src))
basedir, nbname = os.path.split(nb_src)
nb_dest = os.path.join(target_dir, nbname)
shutil.copyfile(nb_src, nb_dest)
with open(nb_dest, 'r') as f:
nb_json = nbformat.reads_json(f.read())
(body, resources) = exporter.from_notebook_node(nb_json)
root, ext = os.path.splitext(nbname)
nb_html_dest = os.path.join(rendered_dir, root + '.html')
with open(nb_html_dest, 'w') as f:
f.write(body)
nbroots.append(root)
nbtitles.append(get_notebook_title(nb_json, root))
for nbroot, nbtitle in zip(nbroots, nbtitles):
with open(os.path.join(target_dir, nbroot + '.rst'), 'w') as f:
f.write(RST_TEMPLATE.render(title=nbtitle, nbroot=nbroot))
with open(os.path.join(target_dir, 'index.rst'), 'w') as f:
f.write(INDEX_TEMPLATE.render(notebooks=nbroots,
sphinx_tag='notebook-examples'))
def render_notebook(exporter, json_notebook, url=None, forced_theme=None, config=None):
app_log.info("rendering %d B notebook from %s", len(json_notebook), url)
if not isinstance(exporter, Exporter):
# allow exporter to be passed as a class, rather than instance
# because Exporter instances cannot be passed across multiprocessing boundaries
# instances are cached by class to avoid repeated instantiation of duplicates
exporter_cls = exporter
if exporter_cls not in exporters:
app_log.info("instantiating %s" % exporter_cls.__name__)
exporters[exporter_cls] = exporter_cls(config=config, log=app_log)
exporter = exporters[exporter_cls]
try:
nb = reads_json(json_notebook)
except ValueError:
raise NbFormatError('Error reading JSON notebook')
css_theme = nb.get('metadata', {}).get('_nbviewer', {}).get('css', None)
if not css_theme or not css_theme.strip():
# whitespace
css_theme = None
if forced_theme:
css_theme = forced_theme
# get the notebook title, if any
try:
name = nb.metadata.name
except AttributeError:
name = ''
if not name and url is not None:
name = url.rsplit('/')[-1]
if not name.endswith(".ipynb"):
name = name + ".ipynb"
html, resources = exporter.from_notebook_node(nb)
config = {
'download_name': name,
'css_theme': css_theme,
}
return html, config
def parse(path):
source = read_text_file(path)
if path.endswith('.rst'):
doctree = publish_doctree(source)
docinfos = doctree.traverse(nodes.docinfo)
docinfo = {c.tagname: str(c.children[0])
for i in docinfos for c in i.children}
parts = publish_parts(source, writer_name='html')
return {'body': parts['body'],
'date': docinfo.get('date'),
'title': parts['title']}
elif path.endswith('.ipynb'):
notebook = nbformat.reads_json(source)
exporter = HTMLExporter(config=None, extra_loaders=[dl])
body, resources = exporter.from_notebook_node(notebook)
return {'body': body,
'date': notebook['metadata']['date'],
'title': notebook['metadata']['name']}
def summerize_pickles(notebook_path):
from IPython.nbformat import current as nbformat
import re
with open(notebook_path) as fh:
nb = nbformat.reads_json(fh.read())
list_of_input_pickles = []
list_of_output_pickles = []
for cell in nb["worksheets"][0]["cells"]:
# This confirms there is at least one pickle in it.
if cell["cell_type"] != "code" or cell["input"].find("pickle") == -1: # Skipping over those cells which aren't code or those cells with code but which don't reference "pickle
continue
cell_content = cell["input"]
# In case there are multiple lines, it iterates line by line.
for line in cell_content.splitlines():
if line.find("pickle") == -1: # Skips over lines w/ no mention of "pickle" to potentially reduce the number of times it's searched.
continue
############################ ############################ ############################ ############################
code_type = str()
if line.find("pickle.dump") != -1 or line.find(".to_pickle")!= -1:
code_type = "output"
elif line.find("pickle.load") != -1 or line.find(".read_pickle")!= -1:
code_type = "input"
else:
continue # This tells the code to skip over lines like "import cpickle as pickle"
############################ ############################ ############################ ############################
filename = re.findall(r'"(.*?)"', line) # This gets all the content between the quotes. See: http://stackoverflow.com/questions/171480/regex-grabbing-values-between-quotation-marks
############################ ############################ ############################ ############################
if code_type == "input":
list_of_input_pickles.append(filename[0])
elif code_type == "output":
list_of_output_pickles.append(filename[0])
pickles_dict = {"input_pickles":list_of_input_pickles,
"output_pickles":list_of_output_pickles }
return pickles_dict
def guide_worksheet(path):
require_login()
from IPython.nbformat import current as nbformat
from IPython.nbconvert.exporters import HTMLExporter
from IPython.config import Config
try:
nb = nbformat.reads_json(file("notebooks/"+path).read())
except IOError:
return "Файл %s не найден" % (path)
config = Config()
config.HTMLExporter.template_file = 'basic'
config.NbconvertApp.fileext = 'html'
config.CSSHTMLHeaderTransformer.enabled = False
# don't strip the files prefix - we use it for redirects
config.Exporter.filters = {'strip_files_prefix': lambda s: s}
C = HTMLExporter(config=config)
body=C.from_notebook_node(nb)
return dict(content=body[0])
def guide_worksheet(path):
require_login()
from IPython.nbformat import current as nbformat
from IPython.nbconvert.exporters import RevealExporter
from IPython.config import Config
try:
nb = nbformat.reads_json(file("notebooks/"+path).read())
except IOError:
return "Файл %s не найден" % (path)
c = Config({
'RevealHelpTransformer':{
'enabled':True,
'url_prefix':'reveal.js',
},
})
exportHtml = RevealExporter(config=c)
(body,resources) = exportHtml.from_notebook_node(nb)
return body.encode('utf-8')
def _build(infile=None, outfile=None,
reveal_prefix='reveal.js',
template_file='templates/slides_reveal_local'):
if infile is None:
infile = "talk.ipynb" # load the name of your slideshow
if outfile is None:
outfile = "talk.slides.html"
print('building {} to {}'.format(infile, outfile))
notebook = open(infile).read()
notebook_json = nbformat.reads_json(notebook)
# This is the config object I talked before:
# After the 'url_prefix', you can set the location of your
# local reveal.js library, i.e. if the reveal.js is located
# in the same directory as your talk.slides.html, then
# set 'url_prefix':'reveal.js'.
c = Config({
'RevealHelpPreprocessor':{
'enabled':True,
'url_prefix':reveal_prefix,
},
})
exportHtml = SlidesExporter(config=c,
template_file=template_file)
(body,resources) = exportHtml.from_notebook_node(notebook_json)
# table -> bootstrap table
body = body.replace('<table>',
'<table class="table table-striped">')
# for all code blocks after "show me the code", no "input_hidden"
show = body.find('数据预处理-训练集与特征')
body = body[:show] + body[show:].replace('input_hidden', 'input')
open(outfile, 'w').write(body)
def get_files(filepath):
os.chdir(filepath)
files = []
for root, dirnames, filenames in os.walk("."):
for filename in fnmatch.filter(filenames, '*.py'):
files.append(os.path.join(root, filename))
for filename in fnmatch.filter(filenames, '*.ipynb'):
try:
with open(filename) as fh:
nb = nbformat.reads_json(fh.read())
export_path = filename.replace(".ipynb", ".py")
exporter = PythonExporter()
source, meta = exporter.from_notebook_node(nb)
with open(export_path, 'w+') as fh:
fh.writelines(source)
files.append()
except: #may have issues with JSON encoding
pass
return files
def get_files(filepath):
os.chdir(filepath)
files = []
for root, dirnames, filenames in os.walk("."):
for filename in fnmatch.filter(filenames, '*.py'):
files.append(os.path.join(root, filename))
for filename in fnmatch.filter(filenames, '*.ipynb'):
try:
with open(filename) as fh:
nb = nbformat.reads_json(fh.read())
export_path = filename.replace(".ipynb", ".py")
exporter = PythonExporter()
source, meta = exporter.from_notebook_node(nb)
with open(export_path, 'w+') as fh:
fh.writelines(source)
files.append()
except: #may have issues with JSON encoding
pass
return files
def main():
"""
Render STDIN as a notebook.
"""
exporter = HTMLExporter()
json_as_string = sys.stdin.read().decode("utf-8")
try:
notebook_node = reads_json(json_as_string)
except Exception:
logging.exception("Unable to parse JSON.")
html, _ = exporter.from_notebook_node(notebook_node)
sys.stderr.write("JSON was {:,} byte(s); html is {:,} byte(s).\n".format(
len(json_as_string), len(html)
))
sys.stdout.write(html.encode("utf-8"))
sys.stderr.flush()
sys.stdout.flush()
def render_notebook(exporter, json_notebook, url=None, forced_theme=None):
app_log.info("rendering %d B notebook from %s", len(json_notebook), url)
try:
nb = reads_json(json_notebook)
except ValueError:
raise NbFormatError('Error reading JSON notebook')
css_theme = nb.get('metadata', {}).get('_nbviewer', {}).get('css', None)
if not css_theme or not css_theme.strip():
# whitespace
css_theme = None
if forced_theme:
css_theme = forced_theme
# get the notebook title, if any
try:
name = nb.metadata.name
except AttributeError:
name = ''
if not name and url is not None:
name = url.rsplit('/')[-1]
if not name.endswith(".ipynb"):
name = name + ".ipynb"
html, resources = exporter.from_notebook_node(nb)
config = {
'download_name': name,
'css_theme': css_theme,
}
return html, config
def _parse_ipynb(self):
notebook = nbformat.reads_json(self.source)
config = Config({'HTMLExporter': {'default_template': 'basic'}})
exporter = HTMLExporter(config=config)
body, resources = exporter.from_notebook_node(notebook)
body = body[body.index('\n<div class="cell'):-36] # excise template
body = self._decapitate(body)
body = body.replace('\n</pre>', '</pre>')
self.add_mathjax = r'\(' in body
fields = notebook['metadata']
if 'date' in fields:
self.date = datetime.strptime(fields['date'], '%d %B %Y').date()
else:
self.date = datetime.now().date()
self.tags = set()
if 'tags' in fields:
self.tags.update('-'.join(tag.strip().lower().split())
for tag in fields['tags'].split(','))
if self.date and self.tags:
heading = ':Date: {}\n:Tags: {}\n'.format(
self.date.strftime('%d %B %Y').lstrip('0'),
', '.join(sorted(self.tags)),
)
parts = parse_rst(heading)
body = parts['docinfo'] + body
self.add_disqus = fields.get('add_disqus', False)
self.body_html = body
self.next_link = None
self.previous_link = None
self.add_title = True
def _parse_ipynb(self):
notebook = nbformat.reads_json(self.source)
config = Config({'HTMLExporter': {'default_template': 'basic'}})
exporter = HTMLExporter(config=config)
body, resources = exporter.from_notebook_node(notebook)
body = body[body.index('\n<div class="cell'):-36] # excise template
body = self._decapitate(body)
body = body.replace('\n</pre>', '</pre>')
self.add_mathjax = r'\(' in body
fields = notebook['metadata']
if 'date' in fields:
self.date = datetime.strptime(fields['date'], '%d %B %Y').date()
else:
self.date = datetime.now().date()
self.tags = set()
if 'tags' in fields:
self.tags.update('-'.join(tag.strip().lower().split())
for tag in fields['tags'].split(','))
if self.date and self.tags:
heading = ':Date: {}\n:Tags: {}\n'.format(
self.date.strftime('%d %B %Y').lstrip('0'),
', '.join(sorted(self.tags)),
)
parts = parse_rst(heading)
body = parts['docinfo'] + body
self.add_disqus = fields.get('add_disqus', False)
self.body_html = body
self.next_link = None
self.previous_link = None
self.add_title = True
def render_notebook(exporter, json_notebook, url=None, forced_theme=None):
app_log.info("rendering %d B notebook from %s", len(json_notebook), url)
try:
nb = reads_json(json_notebook)
except ValueError:
raise NbFormatError('Error reading JSON notebook')
css_theme = nb.get('metadata', {}).get('_nbviewer', {}).get('css', None)
if not css_theme or not css_theme.strip():
# whitespace
css_theme = None
if forced_theme:
css_theme = forced_theme
# get the notebook title, if any
try:
name = nb.metadata.name
except AttributeError:
name = ''
if not name and url is not None:
name = url.rsplit('/')[-1]
if not name.endswith(".ipynb"):
name = name + ".ipynb"
html, resources = exporter.from_notebook_node(nb)
config = {
'download_name': name,
'css_theme': css_theme,
}
return html, config
def parse(path):
source = read_text_file(path)
if path.endswith('.rst'):
doctree = publish_doctree(source)
docinfos = doctree.traverse(nodes.docinfo)
docinfo = {
c.tagname: str(c.children[0])
for i in docinfos for c in i.children
}
parts = publish_parts(source, writer_name='html')
return {
'body': parts['body'],
'date': docinfo.get('date'),
'title': parts['title']
}
elif path.endswith('.ipynb'):
notebook = nbformat.reads_json(source)
exporter = HTMLExporter(config=None, extra_loaders=[dl])
body, resources = exporter.from_notebook_node(notebook)
return {
'body': body,
'date': notebook['metadata']['date'],
'title': notebook['metadata']['name']
}
def finish_notebook(self, nbfile, url, msg=None):
"""render a notebook from its JSON body.
msg is extra information for the log message when rendering fails.
"""
if msg is None:
msg = url
with io.open(nbfile, encoding='utf-8') as f:
json_notebook = f.read()
nb = reads_json(json_notebook)
try:
app_log.debug("Requesting render of %s", url)
with self.time_block("Rendered %s" % url):
app_log.info(
"rendering %d B notebook from %s",
len(json_notebook), url)
nbhtml, config = yield self.pool.submit(
render_notebook, self.exporter, nbfile, nb, url,
config=self.config,
)
except NbFormatError as e:
app_log.error("Invalid notebook %s: %s", msg, e)
raise web.HTTPError(400, str(e))
except Exception as e:
app_log.error("Failed to render %s", msg, exc_info=True)
raise web.HTTPError(400, str(e))
else:
app_log.debug("Finished render of %s", url)
html = self.render_template('notebook.html',
body=nbhtml,
url=url,
date=datetime.utcnow().strftime(date_fmt),
**config)
yield self._finish(html)
def write_from_json(self, notebook_json):
notebook = nbformat.reads_json(notebook_json)
return self.write(notebook)
def render_notebook(exporter, json_notebook):
nb = reads_json(json_notebook)
return exporter.from_notebook_node(nb)
def notebook(preprocessor, tag, markup):
match = FORMAT.search(markup)
if match:
argdict = match.groupdict()
src = argdict['src']
start = argdict['start']
end = argdict['end']
else:
raise ValueError("Error processing input, "
"expected syntax: {0}".format(SYNTAX))
if start:
start = int(start)
else:
start = 0
if end:
end = int(end)
else:
end = None
settings = preprocessor.configs.config['settings']
nb_dir = settings.get('NOTEBOOK_DIR', 'notebooks')
nb_path = os.path.join('content', nb_dir, src)
if not os.path.exists(nb_path):
raise ValueError("File {0} could not be found".format(nb_path))
# Create the custom notebook converter
c = Config({'CSSHTMLHeaderTransformer':
{'enabled':True, 'highlight_class':'.highlight-ipynb'},
'SubCell':
{'enabled':True, 'start':start, 'end':end}})
exporter = HTMLExporter(config=c,
template_file='basic',
filters={'highlight2html': custom_highlighter},
transformers=[SubCell],
extra_loaders=[pelican_loader])
# read and parse the notebook
with open(nb_path) as f:
nb_text = f.read()
nb_json = nbformat.reads_json(nb_text)
(body, resources) = exporter.from_notebook_node(nb_json)
# if we haven't already saved the header, save it here.
if not notebook.header_saved:
print ("\n ** Writing styles to _nb_header.html: "
"this should be included in the theme. **\n")
header = '\n'.join(CSS_WRAPPER.format(css_line)
for css_line in resources['inlining']['css'])
header += JS_INCLUDE
with open('_nb_header.html', 'w') as f:
f.write(header)
notebook.header_saved = True
# this will stash special characters so that they won't be transformed
# by subsequent processes.
body = preprocessor.configs.htmlStash.store(body, safe=True)
return body
"""Convert IPython notebook into HTML reveal.js slides.
Overrides some standard functionality:
- Uses a local reveal.js with customized CSS.
- Avoids writing out the standard IPython reveal CSS overrides in favor of the default style.
"""
from IPython.nbconvert.exporters import RevealExporter
from IPython.config import Config
from IPython.nbformat import current as nbformat
infile = "chapmanb_bosc2013_bcbio.ipynb"
outfile = "chapmanb_bosc2013_bcbio.html"
notebook = open(infile).read()
notebook_json = nbformat.reads_json(notebook)
c = Config({'RevealHelpTransformer': {'enabled': True,
'url_prefix':'../reveal.js',},
"CSSHTMLHeaderTransformer": {'enabled': False}
})
exportHtml = RevealExporter(config=c)
(body,resources) = exportHtml.from_notebook_node(notebook_json)
with open(outfile, "w") as out_handle:
in_css_override = False
for line in body.encode('utf-8').split("\n"):
if line.startswith("/* Overrides of notebook CSS"):
in_css_override = True
if in_css_override:
parser.add_argument('--day', type=int, help=strings_help[9], nargs='?', default=datetime.datetime.now().day, required=False)
args = parser.parse_args()
host = default.host + "/blog"
exportHtml = HTMLExporter(config=Config({'HTMLExporter':{'default_template':'basic'}}))
f = open(args.rsakey,'r')
private_key = RSA.importKey(f.read())
signer = PKCS1_v1_5.new(private_key)
url = args.url
if url == "":
url = '-'.join((args.title).lower().split(" "))
notebook = nbformat.reads_json(open(os.path.join(args.directory, args.file), "r").read())
(body,resources) = exportHtml.from_notebook_node(notebook)
content = ((body.split("</body>")[0]).split('<div class="container" id="notebook-container">')[1])[0:-16]
category = ""
if args.title == "root":
category = "static"
else:
category = args.category
data = {"title": args.title,
"url": url,
"content": content,
"author": args.author,
"year": datetime.datetime.now().year,
def parse_json(content):
try:
nb = nbformat.reads_json(content)
except ValueError:
raise NbFormatError('Error reading json notebook')
return nb
import os, sys
path = '/Users/kougg/bash/Studying/Programming/IPython/reveal.js_3.1.js/lib'
sys.path.append(path)
from IPython.nbconvert.exporters import SlidesExporter
from IPython.config import Config
from IPython.nbformat import current as nbformat
infile = "Portfolio_Management.ipynb" # load the name of your slideshow
outfile = "Portfolio_Management.slides.html"
notebook = open(infile).read()
notebook_json = nbformat.reads_json(notebook)
# This is the config object I talked before:
# After the 'url_prefix', you can set the location of your
# local reveal.js library, i.e. if the reveal.js is located
# in the same directory as your talk.slides.html, then
# set 'url_prefix':'reveal.js'.
c = Config({
'RevealHelpTransformer':{
'enabled':True,
'url_prefix':'reveal.js',
},
})
print c
exportHtml = SlidesExporter(config=c)
def notebook(preprocessor, tag, markup):
match = FORMAT.search(markup)
if match:
argdict = match.groupdict()
src = argdict['src']
start = argdict['start']
end = argdict['end']
else:
raise ValueError("Error processing input, "
"expected syntax: {0}".format(SYNTAX))
if start:
start = int(start)
else:
start = 0
if end:
end = int(end)
else:
end = None
settings = preprocessor.configs.config['settings']
nb_dir = settings.get('NOTEBOOK_DIR', 'notebooks')
nb_path = os.path.join('content', nb_dir, src)
if not os.path.exists(nb_path):
raise ValueError("File {0} could not be found".format(nb_path))
# Create the custom notebook converter
c = Config({
'CSSHTMLHeaderTransformer': {
'enabled': True,
'highlight_class': '.highlight-ipynb'
},
'SubCell': {
'enabled': True,
'start': start,
'end': end
}
})
exporter = HTMLExporter(config=c,
template_file='basic',
filters={'highlight2html': custom_highlighter},
transformers=[SubCell],
extra_loaders=[pelican_loader])
# read and parse the notebook
with open(nb_path) as f:
nb_text = f.read()
nb_json = nbformat.reads_json(nb_text)
(body, resources) = exporter.from_notebook_node(nb_json)
# if we haven't already saved the header, save it here.
if not notebook.header_saved:
print("\n ** Writing styles to _nb_header.html: "
"this should be included in the theme. **\n")
header = '\n'.join(
CSS_WRAPPER.format(css_line)
for css_line in resources['inlining']['css'])
header += JS_INCLUDE
with open('_nb_header.html', 'w') as f:
f.write(header)
notebook.header_saved = True
# this will stash special characters so that they won't be transformed
# by subsequent processes.
body = preprocessor.configs.htmlStash.store(body, safe=True)
return body
build_dir = os.path.dirname(full_path)
rel_build_dir = os.path.relpath(build_dir, source_dir)
resources = {}
nb_name = os.path.splitext(os.path.basename(full_path))[0]
nb_output_dirs = nb_name + args.outputs_dir_suffix
resources['output_files_dir'] = nb_output_dirs
# Clear old output dir path
if os.path.isdir(os.path.join(build_dir, nb_output_dirs)):
shutil.rmtree(os.path.join(build_dir, nb_output_dirs))
exporter = RSTExporter()
nb = nbformat.reads_json(open(full_path).read())
if execute:
log.info("Execute notebook '{}'".format(rel_path))
nb = execute_notebook(nb, resources)
if overwrite and len(nbformat.validate(nb)) == 0:
with open(full_path, 'w') as f:
nbformat.write(nb, f, 'ipynb')
elif overwrite and len(nbformat.validate(nb)) > 0:
log.error("Executed notebook is not a valid format. "
"Original notebook won't be overwritten.")
log.info("Export notebook '{}'".format(rel_path))
(output, resources) = exporter.from_notebook_node(nb, resources=resources)
def notebook(preprocessor, tag, markup):
match = FORMAT.search(markup)
if match:
argdict = match.groupdict()
src = argdict['src']
start = argdict['start']
end = argdict['end']
else:
raise ValueError("Error processing input, "
"expected syntax: {0}".format(SYNTAX))
if start:
start = int(start)
else:
start = 0
if end:
end = int(end)
else:
end = None
settings = preprocessor.configs.config['settings']
nb_dir = settings.get('NOTEBOOK_DIR', 'notebooks')
nb_path = os.path.join('content', nb_dir, src)
if not os.path.exists(nb_path):
raise ValueError("File {0} could not be found".format(nb_path))
# Create the custom notebook converter
c = Config({'CSSHTMLHeaderTransformer':
{'enabled':True, 'highlight_class':'.highlight-ipynb'},
'SubCell':
{'enabled':True, 'start':start, 'end':end}})
template_file = 'basic'
if LooseVersion(IPython.__version__) >= '2.0':
if os.path.exists('pelicanhtml_2.tpl'):
template_file = 'pelicanhtml_2'
else:
if os.path.exists('pelicanhtml_1.tpl'):
template_file = 'pelicanhtml_1'
if LooseVersion(IPython.__version__) >= '2.0':
subcell_kwarg = dict(preprocessors=[SubCell])
else:
subcell_kwarg = dict(transformers=[SubCell])
exporter = HTMLExporter(config=c,
template_file=template_file,
filters={'highlight2html': custom_highlighter},
**subcell_kwarg)
# read and parse the notebook
with open(nb_path) as f:
nb_text = f.read()
nb_json = nbformat.reads_json(nb_text)
(body, resources) = exporter.from_notebook_node(nb_json)
for h in '123456':
body = body.replace('<h%s' % h, '<h%s class="ipynb"' % h)
body = '<div class="ipynb">\n\n' + body + "\n\n</div>"
# if we haven't already saved the header, save it here.
if not notebook.header_saved:
print ("\n ** Writing styles to _nb_header.html: "
"this should be included in the theme. **\n")
header = '\n'.join(CSS_WRAPPER.format(css_line)
for css_line in resources['inlining']['css'])
header += JS_INCLUDE
# # replace the highlight tags
header = header.replace('highlight', 'highlight-ipynb')
header = header.replace('html, body', '\n'.join(('pre.ipynb {',
' color: black;',
' background: #f7f7f7;',
' border: 0;',
' box-shadow: none;',
' margin-bottom: 0;',
' padding: 0;'
'}\n',
'html, body')))
# # create a special div for notebook
header = header.replace('body {', 'div.ipynb {')
header = header.replace('body{margin:0;font-family:"Helvetica Neue",Helvetica,Arial,sans-serif;font-size:13px;line-height:20px;color:#000;background-color:#fff}', '')
# # specialize headers
header = header.replace('html, body,',
'\n'.join((('h1.ipynb h2.ipynb h3.ipynb '
'h4.ipynb h5.ipynb h6.ipynb {'),
'h1.ipynb h2.ipynb ... {',
' margin: 0;',
' padding: 0;',
' border: 0;',
' font-size: 100%;',
' font: inherit;',
' vertical-align: baseline;',
'}\n',
'html, body,')))
#
#
# # comment out document-level formatting
header = header.replace('html, body,',
'/*html, body,*/')
header = header.replace('body{background-color:#fff}',
'/*body{background-color:#fff}*/')
header = header.replace('body{background-color:#fff;position:absolute;left:0;right:0;top:0;bottom:0;overflow:visible}',
'/*body{background-color:#fff;position:absolute;left:0;right:0;top:0;bottom:0;overflow:visible}*/')
header = header.replace('h1, h2, h3, h4, h5, h6,',
'/*h1, h2, h3, h4, h5, h6,*/')
with open('_nb_header.html', 'w') as f:
f.write(header)
notebook.header_saved = True
# this will stash special characters so that they won't be transformed
# by subsequent processes.
body = preprocessor.configs.htmlStash.store(body, safe=True)
return body
from IPython.nbformat import current as nbformat
from IPython.nbconvert import PythonExporter
filepath = "Huasong.Shan.HW1.ipynb" #'HW1_T2.ipynb'
export_path = "Huasong.Shan.HW1.ipynb.py"
with open(filepath) as fh:
nb = nbformat.reads_json(fh.read())
exporter = PythonExporter()
# source is a tuple of python source code
# meta contains metadata
source, meta = exporter.from_notebook_node(nb)
with open(export_path, "w+") as fh:
fh.writelines(source)
def notebook(preprocessor, tag, markup):
match = FORMAT.search(markup)
if match:
argdict = match.groupdict()
src = argdict['src']
start = argdict['start']
end = argdict['end']
language = argdict['language']
else:
raise ValueError("Error processing input, "
"expected syntax: {0}".format(SYNTAX))
if start:
start = int(start)
else:
start = 0
if end:
end = int(end)
else:
end = None
language_applied_highlighter = partial(custom_highlighter,
language=language)
nb_dir = preprocessor.configs.getConfig('NOTEBOOK_DIR')
nb_path = os.path.join('content', nb_dir, src)
if not os.path.exists(nb_path):
raise ValueError("File {0} could not be found".format(nb_path))
# Create the custom notebook converter
c = Config({
'CSSHTMLHeaderTransformer': {
'enabled': True,
'highlight_class': '.highlight-ipynb'
},
'SubCell': {
'enabled': True,
'start': start,
'end': end
}
})
template_file = 'basic'
if LooseVersion(IPython.__version__) >= '2.0':
if os.path.exists('pelicanhtml_2.tpl'):
template_file = 'pelicanhtml_2'
else:
if os.path.exists('pelicanhtml_1.tpl'):
template_file = 'pelicanhtml_1'
if LooseVersion(IPython.__version__) >= '2.0':
subcell_kwarg = dict(preprocessors=[SubCell])
else:
subcell_kwarg = dict(transformers=[SubCell])
exporter = HTMLExporter(
config=c,
template_file=template_file,
filters={'highlight2html': language_applied_highlighter},
**subcell_kwarg)
# read and parse the notebook
with open(nb_path) as f:
nb_text = f.read()
nb_json = nbformat.reads_json(nb_text)
(body, resources) = exporter.from_notebook_node(nb_json)
# To make the higlighting work with Ipython 2.X
resources['inlining']['css'][1] = resources['inlining']['css'][1].replace(
'highlight', 'highlight-ipynb')
# if we haven't already saved the header, save it here.
if not notebook.header_saved:
print("\n ** Writing styles to _nb_header.html: "
"this should be included in the theme. **\n")
header = '\n'.join(
CSS_WRAPPER.format(css_line)
for css_line in resources['inlining']['css'])
header += JS_INCLUDE
with open('_nb_header.html', 'w') as f:
f.write(header)
notebook.header_saved = True
# this will stash special characters so that they won't be transformed
# by subsequent processes.
body = preprocessor.configs.htmlStash.store(body, safe=True)
return body
from IPython.nbformat import current as nbformat
from IPython.nbconvert import PythonExporter
filepath = 'E:\\Chrome-Downloads\\NewProject\\SignatureVerification\\Siamese_Network\\predict.ipynb'
export_path = '.'
with open(filepath) as fh:
nb = nbformat.reads_json(fh.read())
exporter = PythonExporter()
# source is a tuple of python source code
# meta contains metadata
source, meta = exporter.from_notebook_node(nb)
with open(export_path, 'w+') as fh:
fh.writelines(source)
build_dir = os.path.dirname(full_path)
rel_build_dir = os.path.relpath(build_dir, source_dir)
resources = {}
nb_name = os.path.splitext(os.path.basename(full_path))[0]
nb_output_dirs = nb_name + args.outputs_dir_suffix[0]
resources['output_files_dir'] = nb_output_dirs
# Clear old output dir path
if os.path.isdir(os.path.join(build_dir, nb_output_dirs)):
shutil.rmtree(os.path.join(build_dir, nb_output_dirs))
exporter = RSTExporter()
nb = nbformat.reads_json(open(full_path).read())
if execute:
log.info("Execute notebook '{}'".format(rel_path))
nb = execute_notebook(nb, resources)
if overwrite and len(nbformat.validate(nb)) == 0:
with open(full_path, 'w') as f:
nbformat.write(nb, f, 'ipynb')
elif overwrite and len(nbformat.validate(nb)) > 0:
log.error("Executed notebook is not a valid format. "
"Original notebook won't be overwritten.")
log.info("Export notebook '{}'".format(rel_path))
(output,
resources) = exporter.from_notebook_node(nb, resources=resources)
def _parse_json(self, content):
try:
nb = nbformat.reads_json(content)
except ValueError:
raise NbFormatError('Error reading json notebook')
return nb
