def test_tutorial_nb(file_path):
"""Run tutorial jupyter notebook to catch any execution error.
Parameters
----------
file_path : str
path of tutorial markdown file
"""
tutorial_name = os.path.basename(file_path)
notebook = nbformat.read(file_path + '.ipynb', as_version=4)
eprocessor = ExecutePreprocessor(timeout=1800)
try:
eprocessor.preprocess(notebook, {'metadata': {}})
except Exception as err:
err_msg = str(err)
fail_dict[tutorial_name] = err_msg
finally:
output_nb = open("output.txt", mode='w')
nbformat.write(notebook, output_nb)
output_nb.close()
output_nb = open("output.txt", mode='r')
for line in output_nb:
if "Warning:" in line:
fail_dict[tutorial_name] = "%s has warning." % (tutorial_name)
return
def setUp(self):
if self.metadata is None:
metadata = self.default_metadata
else:
metadata = self.metadata
self.orig = nbformat.from_dict({
"nbformat": NBFORMAT_VERSION,
"nbformat_minor": 0,
"metadata": metadata,
"cells": self.cells
})
with tempfile.TemporaryDirectory() as d:
ipynb0_name = d + "/0"
rmd_name = d + "/1"
ipynb1_name = d + "/2"
with open(ipynb0_name, "w") as f:
nbformat.write(self.orig, f)
if self.use_rmd:
ipyrmd.ipynb_to_rmd(ipynb0_name, rmd_name)
ipyrmd.rmd_to_ipynb(rmd_name, ipynb1_name)
else:
ipyrmd.ipynb_to_spin(ipynb0_name, rmd_name)
ipyrmd.spin_to_ipynb(rmd_name, ipynb1_name)
with open(rmd_name) as f:
self.rmd = f.read()
with open(ipynb1_name) as f:
self.roundtrip = nbformat.read(f, NBFORMAT_VERSION)
def make_open_notebook(options):
"""
Generate an ipython notebook and open it in the browser.
Return system exit code.
Raise:
RuntimeError if jupyther is not in $PATH
"""
import nbformat
nbf = nbformat.v4
nb = nbf.new_notebook()
nb.cells.extend([
nbf.new_markdown_cell("# This is an auto-generated notebook for %s" % os.path.relpath(filepath)),
nbf.new_code_cell("""\
from __future__ import print_function, division, unicode_literals, absolute_import
%matplotlib notebook
#import numpy as np
#import seaborn as sns
from abipy import abilab\
"""),
nbf.new_code_cell("abifile = abilab.abiopen('%s')" % options.filepath)
])
_, nbpath = tempfile.mkstemp(suffix='.ipynb', text=True)
with io.open(nbpath, 'wt', encoding="utf8") as f:
nbformat.write(nb, f)
if which("jupyter") is None:
raise RuntimeError("Cannot find jupyter in PATH. Install it with `pip install`")
return os.system("jupyter notebook %s" % nbpath)
def test_very_long_cells(self):
"""
Torture test that long cells do not cause issues
"""
lorem_ipsum_text = textwrap.dedent("""\
Lorem ipsum dolor sit amet, consectetur adipiscing elit. Donec
dignissim, ipsum non facilisis tempus, dui felis tincidunt metus,
nec pulvinar neque odio eget risus. Nulla nisi lectus, cursus
suscipit interdum at, ultrices sit amet orci. Mauris facilisis
imperdiet elit, vitae scelerisque ipsum dignissim non. Integer
consequat malesuada neque sit amet pulvinar. Curabitur pretium
ut turpis eget aliquet. Maecenas sagittis lacus sed lectus
volutpat, eu adipiscing purus pulvinar. Maecenas consequat
luctus urna, eget cursus quam mollis a. Aliquam vitae ornare
erat, non hendrerit urna. Sed eu diam nec massa egestas pharetra
at nec tellus. Fusce feugiat lacus quis urna sollicitudin volutpat.
Quisque at sapien non nibh feugiat tempus ac ultricies purus.
""")
lorem_ipsum_text = lorem_ipsum_text.replace("\n"," ") + "\n\n"
large_lorem_ipsum_text = "".join([lorem_ipsum_text]*3000)
notebook_name = "lorem_ipsum_long.ipynb"
nb = v4.new_notebook(
cells=[
v4.new_markdown_cell(source=large_lorem_ipsum_text)
]
)
with TemporaryDirectory() as td:
nbfile = os.path.join(td, notebook_name)
with open(nbfile, 'w') as f:
write(nb, f, 4)
(output, resources) = LatexExporter(template_file='article').from_filename(nbfile)
assert len(output) > 0
def _runTest(self):
kernel = 'python%d' % sys.version_info[0]
cur_dir = os.path.dirname(self.nbfile)
with open(self.nbfile) as f:
nb = nbformat.read(f, as_version=4)
if self.cov:
covdict = {'cell_type': 'code', 'execution_count': 1,
'metadata': {'collapsed': True}, 'outputs': [],
'nbsphinx': 'hidden',
'source': 'import coverage\n'
'coverage.process_startup()\n'
'import sys\n'
'sys.path.append("{0}")\n'.format(cur_dir)
}
nb['cells'].insert(0, nbformat.from_dict(covdict))
exproc = ExecutePreprocessor(kernel_name=kernel, timeout=600)
try:
run_dir = os.getenv('WRADLIB_BUILD_DIR', cur_dir)
exproc.preprocess(nb, {'metadata': {'path': run_dir}})
except CellExecutionError as e:
raise e
if self.cov:
nb['cells'].pop(0)
with io.open(self.nbfile, 'wt') as f:
nbformat.write(nb, f)
self.assertTrue(True)
def execute_nb(src, dst, allow_errors=False, timeout=1000, kernel_name=None):
'''
Execute notebook in `src` and write the output to `dst`
Parameters
----------
src, dst: str
path to notebook
allow_errors: bool
timeout: int
kernel_name: str
defualts to value set in notebook metadata
Returns
-------
dst: str
'''
with io.open(src, encoding='utf-8') as f:
nb = nbformat.read(f, as_version=4)
ep = ExecutePreprocessor(allow_errors=False,
timeout=timeout,
kernel_name=kernel_name)
ep.preprocess(nb, {'metadata': {'path': SOURCE_DIR}})
with io.open(dst, 'wt', encoding='utf-8') as f:
nbformat.write(nb, f)
return dst
def nb_to_html(root, template='basic', version=4, timeout=600, kernel='python3'):
'''
This functions executes a Jupyter notebook and creates the related
HTML file.
Args:
root (str): name of the file without the .ipynb extension
template (str): name of the template (to be in the current folder as template.tpl)
version (int): version of the notebook
timeout (float): maximum time spent per cell
kernel (str)
Returns:
None
The function executes root.ipynb into root_exe.ipynb and creates the file root.html.
'''
with open(root + '.ipynb') as f:
nb = nbformat.read(f, as_version=version)
ep = ExecutePreprocessor(timeout=timeout, kernel_name=kernel)
ep.preprocess(nb, {'metadata': {'path': '.'}})
with open(root + '_exe.ipynb', 'wt') as f:
nbformat.write(nb, f)
html_exporter = HTMLExporter()
html_exporter.template_file = template
with open(root + '_exe.ipynb', mode='r') as f:
notebook = nbformat.reads(''.join(f.readlines()), as_version=version)
(body, _) = html_exporter.from_notebook_node(notebook)
codecs.open(root + '.html', 'w', encoding='utf-8').write(body)
def setUp(self):
nbdir = self.notebook_dir
if not os.path.isdir(pjoin(nbdir, 'foo')):
subdir = pjoin(nbdir, 'foo')
os.mkdir(subdir)
# Make sure that we clean this up when we're done.
# By using addCleanup this will happen correctly even if we fail
# later in setUp.
@self.addCleanup
def cleanup_dir():
shutil.rmtree(subdir, ignore_errors=True)
nb = new_notebook()
nb.cells.append(new_markdown_cell(u'Created by test ³'))
cc1 = new_code_cell(source=u'print(2*6)')
cc1.outputs.append(new_output(output_type="stream", text=u'12'))
cc1.outputs.append(new_output(output_type="execute_result",
data={'image/png' : png_green_pixel},
execution_count=1,
))
nb.cells.append(cc1)
with io.open(pjoin(nbdir, 'foo', 'testnb.ipynb'), 'w',
encoding='utf-8') as f:
write(nb, f, version=4)
self.nbconvert_api = NbconvertAPI(self.request)
def write_notebook(self, include_html=True):
suffix = "_responses_with_names" if self.include_usernames else "_responses"
nb_name = self.nb_name_stem + suffix
output_file = os.path.join(PROCESSED_NOTEBOOK_DIR, nb_name + '.ipynb')
html_output = os.path.join(PROCESSED_NOTEBOOK_DIR, nb_name + '.html')
remove_duplicate_answers = not self.include_usernames
filtered_cells = []
for prompt in self.question_prompts:
filtered_cells += prompt.cells
answers = prompt.answers_without_duplicates if remove_duplicate_answers else prompt.answers
for gh_username, response_cells in answers.items():
if self.include_usernames:
filtered_cells.append(
NotebookUtils.markdown_heading_cell(self.gh_username_to_fullname(gh_username), 4))
filtered_cells.extend(response_cells)
answer_book = deepcopy(self.template)
answer_book['cells'] = filtered_cells
nb = nbformat.from_dict(answer_book)
print "Writing", output_file
with io.open(output_file, 'wt') as fp:
nbformat.write(nb, fp, version=4)
if include_html:
# TODO why is the following necessary?
nb = nbformat.reads(nbformat.writes(nb, version=4), as_version=4)
html_content, _ = nbconvert.export_html(nb)
print "Writing", html_output
with io.open(html_output, 'w') as fp:
fp.write(html_content)
def setUp(self):
nbdir = self.notebook_dir
subdir = pjoin(nbdir, 'foo')
try:
os.mkdir(subdir)
except OSError as e:
# Deleting the folder in an earlier test may have failed
if e.errno != errno.EEXIST:
raise
self.addCleanup(partial(shutil.rmtree, subdir, ignore_errors=True))
with io.open(pjoin(subdir, 'nb1.ipynb'), 'w', encoding='utf-8') as f:
nb = new_notebook()
write(nb, f, version=4)
self.sess_api = SessionAPI(self.request)
@self.addCleanup
def cleanup_sessions():
for session in self.sess_api.list().json():
self.sess_api.delete(session['id'])
# This is necessary in some situations on Windows: without it, it
# fails to delete the directory because something is still using
# it. I think there is a brief period after the kernel terminates
# where Windows still treats its working directory as in use. On my
# Windows VM, 0.01s is not long enough, but 0.1s appears to work
# reliably. -- TK, 15 December 2014
time.sleep(0.1)
def check_stuff_gets_embedded(self, nb, exporter_name, to_be_included=[]):
nb_basename = 'notebook'
nb_src_filename = nb_basename + '.ipynb'
with io.open(nb_src_filename, 'w', encoding='utf-8') as f:
write(nb, f, 4)
# convert with default exporter
self.nbconvert('--to {} "{}"'.format('html', nb_src_filename))
nb_dst_filename = nb_basename + '.html'
assert os.path.isfile(nb_dst_filename)
statinfo = os.stat(nb_dst_filename)
os.remove(nb_dst_filename)
# convert with embedding exporter
self.nbconvert('--to {} "{}"'.format(exporter_name, nb_src_filename))
statinfo_e = os.stat(nb_dst_filename)
assert os.path.isfile(nb_dst_filename)
assert statinfo_e.st_size > statinfo.st_size
with io.open(nb_dst_filename, 'r', encoding='utf-8') as f:
embedded_nb = f.read()
for txt in to_be_included:
assert txt in embedded_nb
def main():
arguments = docopt(__doc__, version='nbgen 2.0')
cmd = subprocess.run([arguments["<path>"]] + arguments["<arguments>"], stdout=subprocess.PIPE)
cmd.check_returncode()
cells = json.loads(cmd.stdout.decode("utf-8"))
nb_dict = {
"metadata": {},
"nbformat": 4,
"nbformat_minor": 0,
"cells": cells,
}
notebook = nbformat.from_dict(nb_dict)
ep = ExecutePreprocessor(timeout=600, kernel_name='python3')
ep.preprocess(notebook, {'metadata': {}})
if arguments["nb"]:
nbformat.write(notebook, "{}.ipynb".format(arguments["<name>"]))
elif arguments["slides"]:
config = Config()
reveal_cdn = "https://cdnjs.cloudflare.com/ajax/libs/reveal.js/3.3.0/"
config.SlidesExporter.reveal_url_prefix = (arguments["--reveal"] or reveal_cdn)
slides, __ = export_slides(nb=notebook, config=config)
with open("{}.html".format(arguments["<name>"]), "w") as html_file:
html_file.write(slides)
def build_iab_main(input_dir, output_dir, out_format, ext, css=None):
""" Convert md sources to readable book content, maintaining dir structure.
A few additional processing steps happen here:
* Add Table of Contents to the top of each section.
* Create links from sha1 aliases.
Parameters
----------
input_dir : str
Root path for the markdown files.
output_dir : str
Root path for the output files.
out_format : str
The ipymd format that output files should be written in (for example,
``notebook``).
ext : str
The extension to use for output files.
"""
# Walk the input root directory. We only care about root and files
# inside this loop (nothing happens with dirs).
for unit_number, (unit, chapters) in enumerate(input_dir):
# Iterate over the files in the current root.
if unit_number == 0:
unit_path = ''
else:
unit_path = str(unit_number) + '/'
for chapter_number, content_md in enumerate(chapters):
if chapter_number == 0:
chapter_path = 'index'
else:
chapter_path = str(chapter_number)
path = '%s%s' % (unit_path, chapter_path)
# Convert it from markdown
output_s = ipymd.convert(content_md, from_='markdown', to='notebook')
# define the output filepath
output_fp = get_output_fp(output_dir, path, ext)
try:
os.makedirs(os.path.split(output_fp)[0])
except OSError:
pass
# write the output ipynb
nbformat.write(output_s, output_fp)
if out_format == 'html' or out_format == 's3':
c = Config()
c.ExecutePreprocessor.timeout = 600
html_exporter = HTMLExporter(preprocessors=['nbconvert.preprocessors.execute.ExecutePreprocessor'],
config=c)
for root, dirs, files in os.walk(output_dir):
if css:
shutil.copy(css, os.path.join(root, 'custom.css'))
for f in files:
html_out, _ = html_exporter.from_filename(os.path.join(root, f))
output_fn = os.path.splitext(f)[0] + ext
output_fp = os.path.join(root, output_fn)
open(output_fp, 'w').write(html_out)
def main_patch(args):
bfn = args.base
dfn = args.patch
afn = args.output
for fn in (bfn, dfn):
if not os.path.exists(fn):
print("Missing file {}".format(fn))
return 1
before = nbformat.read(bfn, as_version=4)
with open(dfn) as df:
d = json.load(df)
d = to_diffentry_dicts(d)
# TODO: Split lines here? Must be consistent with the diff for patch_notebook to work correctly!?
#before = split_lines(before)
after = patch_notebook(before, d)
if afn:
nbformat.write(after, afn)
else:
print(after)
return 0
def dojo_nbcompare(self, what="all", **kwargs):
"""
Generate an ipython notebook to compare the results in the dojoreport (calls dojo_compare).
"""
paths = [p.path for p in self]
import nbformat
nbf = nbformat.v4
nb = nbf.new_notebook()
nb.cells.extend([
#nbf.new_markdown_cell("# This is an auto-generated notebook for %s" % os.path.basename(pseudopath)),
nbf.new_code_cell("""\
from __future__ import print_function, division, unicode_literals
%matplotlib notebook"""),
nbf.new_code_cell("""\
from pseudo_dojo.core.pseudos import DojoTable
pseudos = DojoTable(%s)""" % str(paths)),
nbf.new_code_cell("pseudos.dojo_compare(what='%s')" % what),
])
_, nbpath = tempfile.mkstemp(suffix='.ipynb', text=True)
import io
with io.open(nbpath, 'wt', encoding="utf8") as f:
nbformat.write(nb, f)
if which("jupyter") is None:
raise RuntimeError("Cannot find jupyter in PATH. Install it with `pip install`")
return os.system("jupyter notebook %s" % nbpath)
def main():
if len(sys.argv) > 1:
if sys.argv[1] in ['help', '-h', '--help']:
print(__doc__, file=sys.stderr)
sys.exit(1)
if sys.argv[1] in ['install', '--install']:
sys.exit(install())
force = False
filenames = sys.argv[1:]
if filenames[0] in ['-f', '--force']:
force = True
filenames.pop(0)
for filename in filenames:
if not force and not filename.endswith('.ipynb'):
continue
try:
with io.open(filename, 'r', encoding='utf8') as f:
nb = read(f, as_version=NO_CONVERT)
nb = strip_output(nb)
with io.open(filename, 'w', encoding='utf8') as f:
write(nb, f)
except Exception:
# Ignore exceptions for non-notebook files.
print("Could not strip '{}'".format(filename))
raise
else:
write(strip_output(read(sys.stdin, as_version=NO_CONVERT)), sys.stdout)
def execute_nb(src, dst, allow_errors=False, timeout=1000, kernel_name=''):
"""
Execute notebook in `src` and write the output to `dst`
Parameters
----------
src, dst: str
path to notebook
allow_errors: bool
timeout: int
kernel_name: str
defualts to value set in notebook metadata
Returns
-------
dst: str
"""
import nbformat
from nbconvert.preprocessors import ExecutePreprocessor
with io.open(src, encoding='utf-8') as f:
nb = nbformat.read(f, as_version=4)
ep = ExecutePreprocessor(allow_errors=allow_errors,
timeout=timeout,
kernel_name=kernel_name)
ep.preprocess(nb, resources={})
with io.open(dst, 'wt', encoding='utf-8') as f:
nbformat.write(nb, f)
return dst
def copy_notebooks():
nblist = sorted(nb for nb in os.listdir(NB_SOURCE_DIR)
if nb.endswith('.ipynb'))
name_map = {nb: nb.rsplit('.', 1)[0].lower() + '.html'
for nb in nblist}
figsource = abspath_from_here('..', 'notebooks', 'figures')
figdest = abspath_from_here('content', 'figures')
if os.path.exists(figdest):
shutil.rmtree(figdest)
shutil.copytree(figsource, figdest)
figurelist = os.listdir(abspath_from_here('content', 'figures'))
figure_map = {os.path.join('figures', fig) : os.path.join('/PythonDataScienceHandbook/figures', fig)
for fig in figurelist}
for nb in nblist:
base, ext = os.path.splitext(nb)
print('-', nb)
content = nbformat.read(os.path.join(NB_SOURCE_DIR, nb),
as_version=4)
if nb == 'Index.ipynb':
cells = '1:'
template = 'page'
title = 'Python Data Science Handbook'
content.cells[2].source = INTRO_TEXT
else:
cells = '2:'
template = 'booksection'
title = content.cells[2].source
if not title.startswith('#') or len(title.splitlines()) > 1:
raise ValueError('title not found in third cell')
title = title.lstrip('#').strip()
# put nav below title
content.cells[0], content.cells[1], content.cells[2] = content.cells[2], content.cells[0], content.cells[1]
# Replace internal URLs and figure links in notebook
for cell in content.cells:
if cell.cell_type == 'markdown':
for nbname, htmlname in name_map.items():
if nbname in cell.source:
cell.source = cell.source.replace(nbname, htmlname)
for figname, newfigname in figure_map.items():
if figname in cell.source:
cell.source = cell.source.replace(figname, newfigname)
nbformat.write(content, os.path.join(NB_DEST_DIR, nb))
pagefile = os.path.join(PAGE_DEST_DIR, base + '.md')
htmlfile = base.lower() + '.html'
with open(pagefile, 'w') as f:
f.write(PAGEFILE.format(title=title,
htmlfile=htmlfile,
notebook_file=nb,
template=template,
cells=cells))
def execute_nb(src, dst, allow_errors=False, timeout=1000, kernel_name=None):
"""
Execute notebook in `src` and write the output to `dst`
Parameters
----------
src, dst: str
path to notebook
allow_errors: bool
timeout: int
kernel_name: str
defualts to value set in notebook metadata
Returns
-------
dst: str
"""
with io.open(src, encoding="utf-8") as f:
nb = nbformat.read(f, as_version=4)
ep = ExecutePreprocessor(allow_errors=allow_errors, timeout=timeout, kernel_name=kernel_name)
ep.preprocess(nb, {"metadta": {"path": "notebooks/"}})
with io.open(dst, "wt", encoding="utf-8") as f:
nbformat.write(nb, f)
return dst
def clear_notebooks(root):
"""Clear the outputs of documentation notebooks."""
# cleanup ignored files
run(['git', 'clean', '-fdX', root])
print("Clearing outputs of notebooks in '{}'...".format(os.path.abspath(root)))
preprocessor = ClearOutputPreprocessor()
for dirpath, dirnames, filenames in os.walk(root):
is_submitted = _check_if_directory_in_path(dirpath, 'submitted')
for filename in sorted(filenames):
if os.path.splitext(filename)[1] == '.ipynb':
# read in the notebook
pth = os.path.join(dirpath, filename)
with open(pth, 'r') as fh:
orig_nb = read(fh, 4)
# copy the original notebook
new_nb = deepcopy(orig_nb)
# check outputs of all the cells
if not is_submitted:
new_nb = preprocessor.preprocess(new_nb, {})[0]
# clear metadata
new_nb.metadata = {}
# write the notebook back to disk
with open(pth, 'w') as fh:
write(new_nb, fh, 4)
if orig_nb != new_nb:
print("Cleared '{}'".format(pth))
def notebook(self, s):
"""Export and convert IPython notebooks.
This function can export the current IPython history to a notebook file.
For example, to export the history to "foo.ipynb" do "%notebook foo.ipynb".
The -e or --export flag is deprecated in IPython 5.2, and will be
removed in the future.
"""
args = magic_arguments.parse_argstring(self.notebook, s)
from nbformat import write, v4
cells = []
hist = list(self.shell.history_manager.get_range())
if(len(hist)<=1):
raise ValueError('History is empty, cannot export')
for session, execution_count, source in hist[:-1]:
cells.append(v4.new_code_cell(
execution_count=execution_count,
source=source
))
nb = v4.new_notebook(cells=cells)
with io.open(args.filename, 'w', encoding='utf-8') as f:
write(nb, f, version=4)
def make_report(self, outcome):
"""Make report in form of two notebooks.
Use nbdime diff-web to present the difference between reference
cells and test cells.
"""
failures = self.getreports('failed')
if not failures:
return
for rep in failures:
# Check if this is a notebook node
msg = self._getfailureheadline(rep)
self.section(msg, rep.longrepr.splitlines()[1])
self._outrep_summary(rep)
tmpdir = tempfile.mkdtemp()
try:
ref_file = os.path.join(tmpdir, 'reference.ipynb')
test_file = os.path.join(tmpdir, 'test_result.ipynb')
with io.open(ref_file, "w", encoding="utf8") as f:
nbformat.write(self.nb_ref, f)
with io.open(test_file, "w", encoding="utf8") as f:
nbformat.write(self.nb_test, f)
run_server(
port=0, # Run on random port
cwd=tmpdir,
closable=True,
on_port=lambda port: browse(
port, ref_file, test_file, None))
finally:
shutil.rmtree(tmpdir)
def create_notebook(name, cells):
nb = new_notebook()
for cell in cells:
nb.cells.append(new_code_cell(source=cell))
with open(name, "w") as fh:
write(nb, fh, 4)
def clean_notebook_metadata(root):
"""Cleans the metadata of documentation notebooks."""
print("Cleaning the metadata of notebooks in '{}'...".format(os.path.abspath(root)))
for dirpath, dirnames, filenames in os.walk(root):
is_submitted = _check_if_directory_in_path(dirpath, 'submitted')
for filename in sorted(filenames):
if os.path.splitext(filename)[1] == '.ipynb':
# read in the notebook
pth = os.path.join(dirpath, filename)
with io.open(pth, encoding='utf-8') as fh:
orig_nb = read(fh, 4)
# copy the original notebook
new_nb = clean_notebook(orig_nb)
# write the notebook back to disk
os.chmod(pth, stat.S_IRUSR | stat.S_IWUSR)
with io.open(pth, mode='w', encoding='utf-8') as fh:
write(new_nb, fh, 4)
if orig_nb != new_nb:
print("Cleaned '{}'".format(pth))
def clear_notebooks(root):
"""Clear the outputs of documentation notebooks."""
preprocessor = ClearOutputPreprocessor()
for dirpath, dirnames, filenames in os.walk(root):
for filename in sorted(filenames):
if os.path.splitext(filename)[1] == '.ipynb':
# read in the notebook
pth = os.path.join(dirpath, filename)
with open(pth, 'r') as fh:
orig_nb = read(fh, 4)
# copy the original notebook
new_nb = deepcopy(orig_nb)
# check outputs of all the cells
new_nb = preprocessor.preprocess(new_nb, {})[0]
# clear metadata
new_nb.metadata = {}
# write the notebook back to disk
with open(pth, 'w') as fh:
write(new_nb, fh, 4)
def _notebook_run(path):
"""Execute a notebook via nbconvert and collect output.
:returns (parsed nb object, execution errors)
"""
kernel_name = 'python%d' % sys.version_info[0]
this_file_directory = os.path.dirname(__file__)
errors = []
with tempfile.NamedTemporaryFile(suffix=".ipynb", mode='wt') as fout:
with open(path) as f:
nb = nbformat.read(f, as_version=4)
nb.metadata.get('kernelspec', {})['name'] = kernel_name
ep = ExecutePreprocessor(kernel_name=kernel_name, timeout=10)
try:
ep.preprocess(nb, {'metadata': {'path': this_file_directory}})
except CellExecutionError as e:
if "SKIP" in e.traceback:
print(str(e.traceback).split("\n")[-2])
else:
raise e
except RuntimeError as e:
print(e)
finally:
nbformat.write(nb, fout)
return nb, errors
def _notebook_run(path):
"""Execute a notebook via nbconvert and collect output.
:returns (parsed nb object, execution errors)
"""
kernel_name = "python%d" % sys.version_info[0]
this_file_directory = os.path.dirname(__file__)
errors = []
with tempfile.NamedTemporaryFile(suffix=".ipynb", mode="wt") as fout:
with open(path) as f:
nb = nbformat.read(f, as_version=4)
nb.metadata.get("kernelspec", {})["name"] = kernel_name
ep = ExecutePreprocessor(kernel_name=kernel_name, timeout=10)
try:
ep.preprocess(nb, {"metadata": {"path": this_file_directory}})
except CellExecutionError as e:
if "SKIP" in e.traceback:
print(str(e.traceback).split("\n")[-2])
else:
raise e
except TimeoutError as e:
print(e)
finally:
nbformat.write(nb, fout)
# nb = nbformat.read(fout, nbformat.current_nbformat)
# errors = errors.extend(
# [output for cell in nb.cells if "outputs" in cell
# for output in cell["outputs"] if output.output_type == "error"])
return nb, errors
def main(args=None):
if args is None:
args = sys.argv[1:]
setup_std_streams()
nb = nbformat.read(sys.stdin, as_version=4)
opts = _build_arg_parser().parse_args(args)
do_filter(nb, opts)
nbformat.write(nb, sys.stdout)
def clear_notebook(old_ipynb, new_ipynb):
with io.open(old_ipynb, 'r') as f:
nb = nbformat.read(f, nbformat.NO_CONVERT)
remove_outputs(nb)
with io.open(new_ipynb, 'w', encoding='utf8') as f:
nbformat.write(nb, f, nbformat.NO_CONVERT)
def store_in_file(self, content, force=False):
nb = nbformat.reader.reads(open(self.notebook).read())
if force or 'environment' not in nb['metadata']:
nb['metadata']['environment'] = content
nbformat.write(nb, self.notebook)
return True
else:
raise EnvironmentAlreadyInNotebook(self.notebook)
def write(self, dest):
if isinstance(dest, str):
with open(dest, 'w') as w:
nbf.write(self.nb, w)
else:
nbf.write(self.nb, dest)
def save_notebook(nb, nbfile):
with open(nbfile, "w") as f:
nbformat.write(nb, f)
analysis_selection_list_name=analysis_pp.selection_list_name,
analysis_data_name=analysis_pp.data_name,
nb_log_name='simulate_log.ipynb')
print(simulate_pp.analysis_data_name, 'blah')
print(simulate_pp.data_name, 'blah2')
#simulate_pp.data_name = analysis_pp.data_name
print("Pre-simulation")
nb, resources = simulate_pp.preprocess(nb,
resources=resources,
km=analysis_pp.km)
print('indicators')
print(resources['indicators'])
# so we can run the log notebook
simulate_pp.nb_log.cells = (analysis_pp.nb_log.cells +
simulate_pp.nb_log.cells)
nbformat.write(simulate_pp.nb_log, open('simulate_log_final.ipynb', 'w'))
print('first pass done')
suff_stats = []
indicators = []
for i in range(n_simulations):
# Preprocess and save results
nb, resources = simulate_pp.preprocess(nb,
resources=resources,
km=simulate_pp.km)
indicators.append(resources['indicators'])
suff_stats.append(resources['suff_stat'])
print("Suff Stat:\n", suff_stats[-1], "\n")
print("\n-- SIMULATION %s COMPLETE --\n" % (i + 1))
def _execute(self, outname='out'):
nbf.write(self.nb, 'notebook/' + outname + '.ipynb')
import nbformat
##read the notebooks
first_nb = nbformat.read('test/notebook_a.ipynb', 4)
second_nb = nbformat.read('test/notebook_b.ipynb', 4)
##create a new nb
final_nb = nbformat.v4.new_notebook(metadata=first_nb.metadata)
##concatenate the nbs
final_nb.cells = first_nb.cells + second_nb.cells
##saving the final nb
nbformat.write(final_nb, './test/final_merged.ipynb')
def copy_notebooks():
if not os.path.exists(NB_DEST_DIR):
os.makedirs(NB_DEST_DIR)
if not os.path.exists(PAGE_DEST_DIR):
os.makedirs(PAGE_DEST_DIR)
nblist = sorted(nb for nb in os.listdir(NB_SOURCE_DIR)
if nb.endswith('.ipynb'))
name_map = {nb: nb.rsplit('.', 1)[0].lower() + '.html'
for nb in nblist}
figsource = abspath_from_here('..', 'notebooks', 'figures')
figdest = abspath_from_here('content', 'figures')
if os.path.exists(figdest):
shutil.rmtree(figdest)
shutil.copytree(figsource, figdest)
figurelist = os.listdir(abspath_from_here('content', 'figures'))
figure_map = {os.path.join('figures', fig) : os.path.join('/PythonDataScienceHandbook/figures', fig)
for fig in figurelist}
for nb in nblist:
base, ext = os.path.splitext(nb)
print('-', nb)
content = nbformat.read(os.path.join(NB_SOURCE_DIR, nb),
as_version=4)
if nb == 'Index.ipynb':
# content[0] is the title
# content[1] is the cover image
# content[2] is the license
cells = '1:'
template = 'page'
title = 'Python Data Science Handbook'
content.cells[2].source = INTRO_TEXT
else:
# content[0] is the book information
# content[1] is the navigation bar
# content[2] is the title
cells = '2:'
template = 'booksection'
title = content.cells[2].source
if not title.startswith('#') or len(title.splitlines()) > 1:
raise ValueError('title not found in third cell')
title = title.lstrip('#').strip()
# put nav below title
content.cells.insert(0, content.cells.pop(2))
# Replace internal URLs and figure links in notebook
for cell in content.cells:
if cell.cell_type == 'markdown':
for nbname, htmlname in name_map.items():
if nbname in cell.source:
cell.source = cell.source.replace(nbname, htmlname)
for figname, newfigname in figure_map.items():
if figname in cell.source:
cell.source = cell.source.replace(figname, newfigname)
if cell.source.startswith("<!--NAVIGATION-->"):
# Undo replacement of notebook link in the colab badge
cell.source = nb.join(cell.source.rsplit(name_map[nb], 1))
nbformat.write(content, os.path.join(NB_DEST_DIR, nb))
pagefile = os.path.join(PAGE_DEST_DIR, base + '.md')
htmlfile = base.lower() + '.html'
with open(pagefile, 'w') as f:
f.write(PAGEFILE.format(title=title,
htmlfile=htmlfile,
notebook_file=nb,
template=template,
cells=cells))
def create_analysis_notebook(nb_descriptors, ps, base_str, name_postfix=''):
import nbformat as nbf
import os
nb = nbf.v4.new_notebook()
cells = []
md_cell = ''
md_cell += '| Field | Value |\n'
md_cell += '|-|-|\n'
md_cell += "\n".join([
'| ' + name + ' | ' + des + ' |'
for (name, des) in nb_descriptors.items()
])
cells.append(nbf.v4.new_markdown_cell(md_cell))
cells.append(
nbf.v4.new_code_cell(
"import os,sys,inspect\ncurrentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))\nparentdir = os.path.dirname(currentdir)\nsys.path.insert(0,parentdir)"
))
cells.append(
nbf.v4.new_code_cell(
"%pylab inline\nimport cPickle\nfrom helper import PeriodicAccumulator, BooleanAccumulator\nfrom itertools import product\nplt.style.use('ggplot')"
))
pickler_cell_str = ""
pickler_cell_str += "def get(" + ", ".join(ps.keys()) + "):\n"
pickler_cell_str += " return cPickle.load(open(\'" + base_str + \
".p\'.format(" + ", ".join(ps.keys()) + "),\'rb\'))\n\n\n"
for name, vals in ps.items():
pickler_cell_str += name + "_s = [str(a) for a in " + repr(
vals) + "]\n"
names = [k + "_s" for k in ps.keys()]
pickler_cell_str += "\n\n"
pickler_cell_str += "params = list(product(" + ", ".join(names) + "))"
pickler_cell_str += "\n\n"
pickler_cell_str += "data = {tup:get(*tup) for tup in params}"
cells.append(nbf.v4.new_code_cell(pickler_cell_str))
cells.append(
nbf.v4.new_code_cell("from ipywidgets import interact, ToggleButtons"))
interact = ""
interact += "def show_plot(key," + ", ".join(
ps.keys()) + ",y_c,t_min,t_max):\n"
interact += " figure(figsize=(12,5))\n"
interact += " p = (" + ", ".join(ps.keys()) + ")\n"
interact += " curr = data[p][1][0]\n"
interact += " ts = curr.t\n"
interact += " mask = np.logical_and(ts>=t_min,ts<=t_max)\n"
interact += " if key=='y':\n"
interact += " plot(curr.t[mask],curr.res[key][mask,:int(y_c)+1])\n"
interact += " else:\n"
interact += " plot(curr.t[mask],curr.res[key][mask])\n"
cells.append(nbf.v4.new_code_cell(interact))
interact = ""
interact += "ts = data[params[0]][1][0].t\n"
interact += "i = interact(show_plot,\n"
interact += "key=ToggleButtons(description='key',options=['dendr_pred','weights','weight_updates', 'PIVs', 'y','h']),\n"
interact += "t_min=(0,int(np.round(ts[-1]))),\n"
interact += "t_max=(0,int(np.round(ts[-1]))),\n"
for name, vals in ps.items():
rep = repr(vals)
if rep[:6] == "array(":
rep = rep[6:-1]
interact += name + \
"=ToggleButtons(description=\'" + name + "\',options=" + name + "_s" + "),\n"
interact += "y_c=ToggleButtons(description='y_c',options=[str(a) for a in range(5)]))\n"
cells.append(nbf.v4.new_code_cell(interact))
nb['cells'] = cells
sim_file = nb_descriptors['simulation file'][:-3]
fname = sim_file + "_analysis" + name_postfix + ".ipynb"
if not os.path.exists(sim_file):
os.makedirs(sim_file)
with open(sim_file + '/' + fname, 'w') as f:
nbf.write(nb, f)
def main():
from subprocess import check_output, CalledProcessError
parser = ArgumentParser(epilog=__doc__,
formatter_class=RawDescriptionHelpFormatter)
task = parser.add_mutually_exclusive_group()
task.add_argument('--install',
action='store_true',
help="""Install nbstripout in the current repository (set
up the git filter and attributes)""")
task.add_argument('--uninstall',
action='store_true',
help="""Uninstall nbstripout from the current repository
(remove the git filter and attributes)""")
task.add_argument(
'--is-installed',
action='store_true',
help='Check if nbstripout is installed in current repository')
task.add_argument(
'--status',
action='store_true',
help=
'Print status of nbstripout installation in current repository and configuration summary if installed'
)
parser.add_argument('--keep-count',
action='store_true',
help='Do not strip the execution count/prompt number')
parser.add_argument('--keep-output',
action='store_true',
help='Do not strip output',
default=None)
parser.add_argument('--attributes',
metavar='FILEPATH',
help="""Attributes
file to add the filter to (in combination with --install/--uninstall),
defaults to .git/info/attributes""")
parser.add_argument('--global',
dest='_global',
action='store_true',
help='Use global git config (default is local config)')
task.add_argument('--version', action='store_true', help='Print version')
parser.add_argument(
'--force',
'-f',
action='store_true',
help='Strip output also from files with non ipynb extension')
parser.add_argument('--textconv',
'-t',
action='store_true',
help='Prints stripped files to STDOUT')
parser.add_argument('files', nargs='*', help='Files to strip output from')
args = parser.parse_args()
git_config = ['git', 'config'] + (['--global'] if args._global else [])
if args.install:
sys.exit(install(git_config, attrfile=args.attributes))
if args.uninstall:
sys.exit(uninstall(git_config, attrfile=args.attributes))
if args.is_installed:
sys.exit(status(git_config, verbose=False))
if args.status:
sys.exit(status(git_config, verbose=True))
if args.version:
print(__version__)
sys.exit(0)
try:
extra_keys = check_output(git_config +
['filter.nbstripout.extrakeys']).strip()
except CalledProcessError:
extra_keys = ''
input_stream = None
if sys.version_info < (3, 0):
import codecs
# Use UTF8 reader/writer for stdin/stdout
# http://stackoverflow.com/a/1169209
if sys.stdin:
input_stream = codecs.getreader('utf8')(sys.stdin)
output_stream = codecs.getwriter('utf8')(sys.stdout)
else:
# Wrap input/output stream in UTF-8 encoded text wrapper
# https://stackoverflow.com/a/16549381
if sys.stdin:
input_stream = io.TextIOWrapper(sys.stdin.buffer, encoding='utf-8')
output_stream = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8')
for filename in args.files:
if not (args.force or filename.endswith('.ipynb')):
continue
try:
with io.open(filename, 'r', encoding='utf8') as f:
nb = read(f, as_version=NO_CONVERT)
nb = strip_output(nb, args.keep_output, args.keep_count,
extra_keys)
if args.textconv:
write(nb, output_stream)
output_stream.flush()
else:
with io.open(filename, 'w', encoding='utf8') as f:
write(nb, f)
except NotJSONError:
print("'{}' is not a valid notebook".format(filename),
file=sys.stderr)
sys.exit(1)
except Exception:
# Ignore exceptions for non-notebook files.
print("Could not strip '{}'".format(filename), file=sys.stderr)
raise
if not args.files and input_stream:
try:
nb = strip_output(read(input_stream, as_version=NO_CONVERT),
args.keep_output, args.keep_count, extra_keys)
write(nb, output_stream)
output_stream.flush()
except NotJSONError:
print('No valid notebook detected', file=sys.stderr)
sys.exit(1)
def save_history(self):
"""This saves all cell executions in the current session as a new notebook"""
try:
from nbformat import write, v4, validator
except ImportError:
logger.error("Run pip install nbformat to save notebook history")
return
# TODO: some tests didn't patch ipython properly?
if self.shell is None:
return
cells = []
hist = list(self.shell.history_manager.get_range(output=True))
if len(hist) <= 1 or not self.settings.save_code:
logger.info("not saving jupyter history")
return
try:
for _, execution_count, exc in hist:
if exc[1]:
# TODO: capture stderr?
outputs = [
v4.new_output(output_type="stream",
name="stdout",
text=exc[1])
]
else:
outputs = []
if self.outputs.get(execution_count):
for out in self.outputs[execution_count]:
outputs.append(
v4.new_output(
output_type="display_data",
data=out["data"],
metadata=out["metadata"] or {},
))
cells.append(
v4.new_code_cell(execution_count=execution_count,
source=exc[0],
outputs=outputs))
if hasattr(self.shell, "kernel"):
language_info = self.shell.kernel.language_info
else:
language_info = {"name": "python", "version": sys.version}
logger.info("saving %i cells to _session_history.ipynb",
len(cells))
nb = v4.new_notebook(
cells=cells,
metadata={
"kernelspec": {
"display_name": "Python %i" % sys.version_info[0],
"name": "python%i" % sys.version_info[0],
"language": "python",
},
"language_info": language_info,
},
)
state_path = os.path.join("code", "_session_history.ipynb")
wandb.run.config["_wandb"]["session_history"] = state_path
wandb.run.config.persist()
wandb.util.mkdir_exists_ok(os.path.join(wandb.run.dir, "code"))
with open(
os.path.join(self.settings._tmp_code_dir,
"_session_history.ipynb"),
"w",
encoding="utf-8",
) as f:
write(nb, f, version=4)
with open(os.path.join(wandb.run.dir, state_path),
"w",
encoding="utf-8") as f:
write(nb, f, version=4)
except (OSError, validator.NotebookValidationError) as e:
logger.error("Unable to save ipython session history:\n%s", e)
pass
def convert_to_1d_psweep_nb(_model_path,
name=None,
settings=None,
dest_path=None):
user_dir = '/home/jovyan'
model_path = path.join(user_dir, _model_path)
file = model_path.split('/').pop()
if name is None:
name = get_file_name(file)
class_name = get_class_name(name)
if dest_path is None:
dest_path = model_path.split(file)[0]
# Collect .mdl Data
try:
with open(model_path, 'r') as json_file:
json_data = json.loads(json_file.read())
json_data['name'] = name
except FileNotFoundError as err:
raise ModelNotFoundError('Could not find model file: ' + str(err),
traceback.format_exc())
except JSONDecodeError as err:
raise ModelNotJSONFormatError(
"The model is not JSON decodable: " + str(err),
traceback.format_exc())
is_ode = json_data[
'defaultMode'] == "continuous" if settings is None else settings[
'simulationSettings']['algorithm'] == "ODE"
gillespy2_model = ModelFactory(json_data, is_ode).model
if settings is None or settings['simulationSettings']['isAutomatic']:
algorithm, solv_name = get_algorithm(gillespy2_model, is_psweep=True)
else:
algorithm, solv_name = get_algorithm(
gillespy2_model,
is_psweep=True,
algorithm=settings['simulationSettings']['algorithm'])
# Create new notebook
cells = []
# Create Markdown Cell with name
cells.append(nbf.new_markdown_cell('# {0}'.format(name)))
try:
# Create imports cell
if settings is None:
import_cell = generate_imports_cell(json_data, algorithm,
solv_name)
config_cell = generate_configure_simulation_cell(json_data,
algorithm,
solv_name,
is_psweep=True)
psweep_config_cell = generate_1D_psweep_config_cell(
json_data, class_name)
else:
import_cell = generate_imports_cell(
json_data,
algorithm,
solv_name,
settings=settings['simulationSettings'])
config_cell = generate_configure_simulation_cell(
json_data,
algorithm,
solv_name,
is_psweep=True,
settings=settings['simulationSettings'])
psweep_config_cell = generate_1D_psweep_config_cell(
json_data, class_name, settings=settings)
cells.append(nbf.new_code_cell(import_cell))
# Create Model Cell
cells.append(
nbf.new_code_cell(generate_model_cell(json_data, class_name)))
# Instantiate Model Cell
cells.append(nbf.new_code_cell('model = {0}()'.format(class_name)))
if solv_name == "VariableSSACSolver":
# Instantiate Solver Cell
cells.append(
nbf.new_code_cell(
'solver = model.get_best_solver()\nsolver = solver(model=model)'
))
# Configure Simulation Cell
cells.append(nbf.new_code_cell(config_cell))
# Feature Extraction cell
cells.append(nbf.new_code_cell(generate_feature_extraction_cell()))
# Feature Aggregate cell
cells.append(nbf.new_code_cell(generate_mean_std_aggregate_cell()))
# Parameter Sweep Class cell
cells.append(
nbf.new_code_cell(
generate_1D_parameter_sweep_class_cell(json_data, algorithm)))
# Parameter Sweep Config cell
cells.append(nbf.new_code_cell(psweep_config_cell))
# Parameter Sweep Execution cell
cells.append(
nbf.new_code_cell(
generate_parameter_sweep_run_cell(algorithm, settings)))
# Parameter Sweet Plot Cell
cells.append(nbf.new_code_cell('ps.plot()'))
# Parameter Sweet Plotly Cell
cells.append(nbf.new_code_cell('ps.plotplotly()'))
except KeyError as err:
raise JSONFileNotModelError(
"The JSON file is not formatted as a StochSS model " + str(err),
traceback.format_exc())
# Append cells to worksheet
nb = nbf.new_notebook(cells=cells)
# Open and write to file
dest_file = get_unique_file_name('{0}1dParamSweep.ipynb'.format(name),
dest_path)[0]
with open(dest_file, 'w') as f:
nbformat.write(nb, f, version=4)
f.close()
return {
"Message": '{0} successfully created'.format(dest_file),
"FilePath": dest_file.replace(user_dir + '/', ""),
"File": dest_file.split('/').pop()
}
import argparse
import nbformat as nbf
parser = argparse.ArgumentParser(
description='',
)
parser.add_argument('filenames', metavar='N', type=str, nargs='+', )
def read_file(filename):
with open(filename) as f:
return f.read()
nb = nbf.v4.new_notebook()
md_cell = nbf.v4.new_markdown_cell
code_cell = nbf.v4.new_code_cell
nb['cells'] = [code_cell(read_file(name))
if name.endswith('.py')
else md_cell(read_file(name))
for name in filenames]
nbf.write(nb, 'raw.ipynb')
import nbformat
import os
from nbconvert.preprocessors import ExecutePreprocessor
os.chdir('../docs/tutorials')
notebooks = ['sequencing', 'core', 'optimize']
for notebook in notebooks:
print('Testing %s notebook.' % notebook)
notebook_filename = '%s.ipynb' % notebook
with open(notebook_filename) as f:
nb = nbformat.read(f, as_version=4)
print('\tSuccessfully read notebook...')
ep = ExecutePreprocessor(timeout=600, kernel_name='python3')
ep.preprocess(nb, {})
print('\tSuccessfully executed notebook...')
with open('%s_executed.ipynb' % notebook, 'wt') as f:
nbformat.write(nb, f)
assert '%s_executed.ipynb' % notebook in os.listdir()
print('Notebook passed!')
os.remove('%s_executed.ipynb' % notebook)
def create_jupyter_notebook(filename, notebook_filename=None):
import nbformat as nbf
from fmpy import read_model_description
if notebook_filename is None:
notebook_filename, _ = os.path.splitext(filename)
notebook_filename += '.ipynb'
model_description = read_model_description(filename)
if model_description.defaultExperiment and model_description.defaultExperiment.stopTime:
stop_time = model_description.defaultExperiment.stopTime
else:
stop_time = 1
parameters = []
output_variables = []
max_name = 7
max_start = 7
max_unit = 3
max_output = 0
from .simulation import _get_output_variables
for variable in _get_output_variables(model_description):
max_output = max(max_output, len(variable.name))
output_variables.append((variable.name, variable.description))
for variable in model_description.modelVariables:
if variable.causality == 'parameter' and variable.variability in [
'fixed', 'tunable'
]:
name, start, unit, description = variable.name, variable.start, variable.unit, variable.description
if variable.type == 'String':
start = "'%s'" % start
elif variable.type == 'Boolean':
start = 'True' if start == 'true' else 'False'
if unit is None and variable.declaredType is not None:
unit = variable.declaredType.unit
max_name = max(max_name, len(name))
max_start = max(max_start, len(start))
max_unit = max(max_unit, len(unit)) if unit else max_unit
parameters.append((name, start, unit, description))
code = "import fmpy\n"
code += "from fmpy import *\n"
code += "\n"
code += "fmpy.plot_library = 'plotly' # experimental\n"
code += "\n"
code += "\n"
# use relative path if possible
if os.path.normpath(os.path.dirname(filename)) == os.path.normpath(
os.path.dirname(notebook_filename)):
code += "filename = '%s'\n" % os.path.basename(filename)
else:
code += "filename = r'%s'\n" % filename
code += "\n"
code += "start_values = {\n"
code += " " + "# variable".ljust(max_name + 3) + "start".rjust(
max_start + 2) + " unit".ljust(max_unit + 10) + "description\n"
for name, start, unit, description in parameters:
code += " " + ("'" + name + "':").ljust(max_name + 3) + " "
if unit:
code += ("(" + start).rjust(max_start + 1)
code += (", '" + unit + "'),").ljust(max_unit + 6)
else:
code += start.rjust(max_start + 1) + "," + " " * (max_unit + 5)
if description:
code += " # " + description
code += "\n"
code += "}\n"
code += "\n"
code += "output = [\n"
for name, description in output_variables:
code += " " + ("'%s'," % name).ljust(max_output + 3)
if description:
code += " # " + description
code += "\n"
code += "]\n"
code += "\n"
code += "result = simulate_fmu(filename, start_values=start_values, output=output, stop_time=%s)\n" % stop_time
code += "\n"
code += "plot_result(result)"
nb = nbf.v4.new_notebook()
cells = []
if model_description.description:
cells.append(nbf.v4.new_markdown_cell(model_description.description))
cells.append(nbf.v4.new_code_cell(code))
nb['cells'] = cells
with open(notebook_filename, 'w') as f:
nbf.write(nb, f)
os.makedirs(reports_dir, exist_ok=True)
report_file = os.path.join(reports_dir, 'report.ipynb')
with open(os.path.join(sys.path[0], 'spes_report.ipynb')) as f:
nb = nbformat.read(f, as_version=4)
# Get a list of Parameter objects
orig_parameters = extract_parameters(nb)
params = parameter_values(orig_parameters,
subj=subj,
subj_dir=SUBJECTS_DIR_NATIVE)
# Make a notebook object with these definitions, and execute it.
new_nb = replace_definitions(nb, params, execute=False)
with open(report_file, 'w') as f:
nbformat.write(new_nb, f)
subprocess.run([
'jupyter', 'nbconvert', '--execute',
'--ExecutePreprocessor.timeout=7200',
'--ExecutePreprocessor.kernel_name=python3', '--to=html', report_file
])
if send_notification_when_done:
try:
notifications.sendmail(to_address=send_notifications_to_email,
subject=subj + ' Report job done.',
message=subj + '\n')
except:
pass
def loop():
request = db.q.find_one({'isProcessed': False})
if request == None:
return
subprocess.run(shlex.split("ufw allow out to any port 443"),
env=os.environ,
errors=True)
subprocess.run(shlex.split("ufw deny out to any port 27017"),
env=os.environ,
errors=True)
print(request['_id'])
githubAcc = Github(request['githubTok'])
user = githubAcc.get_user()
repo = user.get_repo("plutons")
githubUserName = request['githubUser']
print(f"processing for: {githubUserName}")
qId = ObjectId(request['_id'])
fullPath = request['file']
notebook = gzip.decompress(request['notebook'])
tempFileName = plutoPath + fullPath[fullPath.rfind('/') + 1:]
print(tempFileName)
with open(tempFileName, mode='wb') as file:
file.write(notebook)
subprocess.run(shlex.split(f"chmod 666 {tempFileName}"),
env=os.environ,
errors=True)
insertRef(tempFileName)
subprocess.run(shlex.split("ufw deny out to any port 443"),
env=os.environ,
errors=True)
cmdLine = f"sudo -E -H -u pluto jupyter nbconvert --to notebook --execute {tempFileName} --inplace --allow-errors"
cpi = subprocess.run(shlex.split(cmdLine), env=os.environ, errors=True)
textLength = getOutputLength(tempFileName)
print(f"total output length: {textLength}")
if githubUserName != 'shyams80' and textLength > 10000:
cmdLine = f"jupyter nbconvert --ClearOutputPreprocessor.enabled=True --inplace {tempFileName}"
cpi = subprocess.run(shlex.split(cmdLine), env=os.environ, errors=True)
nbDoc = nbformat.read(tempFileName, as_version=4)
for nbCell in nbDoc['cells']:
if nbCell['cell_type'] != 'code' and nbCell['source'] != None:
continue
nbCell['execution_count'] = 1
outObj = nbformat.NotebookNode(
output_type='stream',
name='stderr',
text=[
'total output string length exceeded 10000 characters. please stay within the limit.'
])
nbCell['outputs'].append(outObj)
break
nbformat.write(nbDoc, tempFileName, version=4)
with open(tempFileName, mode='rb') as file:
outFileContent = file.read()
tooBig = False
subprocess.run(shlex.split("ufw allow out to any port 443"),
env=os.environ,
errors=True)
try:
fileContent = repo.get_contents(fullPath)
repo.update_file(fullPath, "response", outFileContent, fileContent.sha)
except Exception as exp:
print(exp)
if exp.data["errors"][0]['code'] == 'too_large':
tooBig = True
subprocess.run(shlex.split("ufw deny out to any port 443"),
env=os.environ,
errors=True)
if tooBig:
cmdLine = f"sudo -E -H -u pluto jupyter nbconvert --to markdown --execute {tempFileName} --allow-errors"
cpi = subprocess.run(shlex.split(cmdLine), env=os.environ, errors=True)
filePattern = tempFileName.replace(".ipynb", "")
subprocess.run(shlex.split("ufw allow out to any port 443"),
env=os.environ,
errors=True)
upsertGithub(filePattern + ".md", repo)
for fname in os.listdir(filePattern + "_files"):
upsertGithub(filePattern + "_files/" + fname, repo)
subprocess.run(shlex.split("ufw deny out to any port 443"),
env=os.environ,
errors=True)
os.remove(tempFileName)
subprocess.run(shlex.split("ufw allow out to any port 27017"),
env=os.environ,
errors=True)
db.q.update_one({'_id': qId}, {
'$set': {
'isProcessed': True,
'processedOn': datetime.now(),
'notebook': gzip.compress(outFileContent)
}
})
def rmd_to_ipynb(infile, outfile):
NN = nbformat.NotebookNode
node = NN(nbformat=4, nbformat_minor=0, metadata=NN(**METADATA), cells=[])
with open(infile) as f:
rmdlines = f.readlines()
# YAML front matter appears to be restricted to strictly ---\nYAML\n---
re_yaml_delim = re.compile(r"^---\s*$")
delim_lines = [i for i, l in enumerate(rmdlines) if re_yaml_delim.match(l)]
if len(delim_lines) >= 2 and delim_lines[1] - delim_lines[0] > 1:
yamltext = '\n'.join(rmdlines[delim_lines[0] + 1:delim_lines[1]])
try:
header = yaml.load(yamltext)
node.metadata["Rmd_header"] = header
except yaml.YAMLError as e:
print("Error reading document metadata block: {0}".format(e))
print("Trying to continue without header")
rmdlines = rmdlines[:delim_lines[0]] + rmdlines[delim_lines[1] + 1:]
# the behaviour of rmarkdown appears to be that a code block does not
# have to have matching numbers of start and end `s - just >=3
# and there can be any number of spaces before the {r, meta} block,
# but "r" must be the first character of that block
re_code_start = re.compile(r"^````*\s*{r(.*)}\s*$")
re_code_end = re.compile(r"^````*\s*$")
re_code_inline = re.compile(r"`r.+`")
MD, CODE = range(2)
def add_cell(celltype, celldata, **meta):
if celltype == MD:
cell = NN(cell_type="markdown", metadata=NN(**meta),
source=celldata)
else:
cell = NN(cell_type="code", execution_count=None, source=celldata,
metadata=NN(collapsed=True, autoscroll=False, **meta),
outputs=[])
node.cells.append(cell)
state = MD
celldata = []
meta = {}
for l in rmdlines:
if state == MD:
match = re_code_start.match(l)
if match:
state = CODE
# only add MD cells with non-whitespace content
if any([c.strip() for c in celldata]):
add_cell(MD, celldata, **meta)
celldata = []
meta = {}
if match.group(1):
chunk_opts = match.group(1).strip(" ,")
if chunk_opts:
meta['Rmd_chunk_options'] = chunk_opts
else:
if re_code_inline.search(l):
print("Inline R code detected - treated as text")
# cell.source in ipynb does not include implicit newlines
celldata.append(l.rstrip() + "\n")
else: # CODE
if re_code_end.match(l):
state = MD
# unconditionally add code blocks regardless of content
add_cell(CODE, celldata, **meta)
celldata = []
meta = {}
else:
if len(celldata) > 0:
celldata[-1] = celldata[-1] + "\n"
celldata.append(l.rstrip())
if state == CODE or celldata:
add_cell(state, celldata, **meta)
with open(outfile, "w") as f:
nbformat.write(node, outfile)
return True
def run_onboarding_data_assistant_result_jupyter_notebook_with_new_cell(
context: DataContext,
new_cell: str,
implicit: bool,
):
"""
To set this test up we:
- create a suite
- write code (as a string) for creating an OnboardingDataAssistantResult
- add a new cell to the notebook that was passed to this method
- write both cells to ipynb file
We then:
- load the notebook back from disk
- execute the notebook (Note: this will raise various errors like
CellExecutionError if any cell in the notebook fails)
"""
root_dir: str = context.root_directory
expectation_suite_name: str = "test_suite"
context.create_expectation_suite(
expectation_suite_name=expectation_suite_name, overwrite_existing=True)
notebook_path: str = os.path.join(root_dir,
f"run_onboarding_data_assistant.ipynb")
notebook_code_initialization: str = """
from typing import Optional, Union
import uuid
import great_expectations as ge
from great_expectations.data_context import BaseDataContext
from great_expectations.validator.validator import Validator
from great_expectations.rule_based_profiler.data_assistant import (
DataAssistant,
OnboardingDataAssistant,
)
from great_expectations.rule_based_profiler.types.data_assistant_result import DataAssistantResult
from great_expectations.rule_based_profiler.helpers.util import get_validator_with_expectation_suite
import great_expectations.exceptions as ge_exceptions
context = ge.get_context()
batch_request: dict = {
"datasource_name": "taxi_pandas",
"data_connector_name": "monthly",
"data_asset_name": "my_reports",
}
"""
explicit_instantiation_code: str = """
validator: Validator = get_validator_with_expectation_suite(
data_context=context,
batch_list=None,
batch_request=batch_request,
expectation_suite_name=None,
expectation_suite=None,
component_name="onboarding_data_assistant",
persist=False,
)
data_assistant: DataAssistant = OnboardingDataAssistant(
name="test_onboarding_data_assistant",
validator=validator,
)
data_assistant_result: DataAssistantResult = data_assistant.run()
"""
implicit_invocation_code: str = """
data_assistant_result: DataAssistantResult = context.assistants.onboarding.run(batch_request=batch_request)
"""
notebook_code: str
if implicit:
notebook_code = notebook_code_initialization + implicit_invocation_code
else:
notebook_code = notebook_code_initialization + explicit_instantiation_code
nb = nbformat.v4.new_notebook()
nb["cells"] = []
nb["cells"].append(nbformat.v4.new_code_cell(notebook_code))
nb["cells"].append(nbformat.v4.new_code_cell(new_cell))
# Write notebook to path and load it as NotebookNode
with open(notebook_path, "w") as f:
nbformat.write(nb, f)
nb: nbformat.notebooknode.NotebookNode = load_notebook_from_path(
notebook_path=notebook_path)
# Run notebook
ep: nbconvert.preprocessors.ExecutePreprocessor = (
nbconvert.preprocessors.ExecutePreprocessor(timeout=180,
kernel_name="python3"))
ep.preprocess(nb, {"metadata": {"path": root_dir}})
def mock_jupyter_notebook(args, check):
nb = jupytext.reads('2 + 2', fmt='py')
# args: "jupyter" {app} {path} {others, ...}
nbformat.write(nb, args[2])
def run(self):
# Setting os.environ["CI"] will disable interactive (blocking) mode in
# Jupyter notebooks
os.environ["CI"] = "true"
# Copy TestData directory to the tutorial folder
test_data_in_dir = (Path(self.current_file_dir).parent / "examples" /
"TestData")
test_data_out_dir = Path(self.current_file_dir) / "TestData"
if test_data_out_dir.exists():
shutil.rmtree(test_data_out_dir)
shutil.copytree(test_data_in_dir, test_data_out_dir)
# Copy and execute notebooks in the tutorial folder
nb_paths = []
example_dirs = ["Basic", "Advanced"]
for example_dir in example_dirs:
in_dir = (Path(self.current_file_dir).parent / "examples" /
"Python" / example_dir)
out_dir = Path(self.current_file_dir) / "tutorial" / example_dir
shutil.copy(
in_dir.parent / "open3d_tutorial.py",
out_dir.parent / "open3d_tutorial.py",
)
if self.clean_notebooks:
for nb_out_path in out_dir.glob("*.ipynb"):
print(f"Delete: {nb_out_path}")
nb_out_path.unlink()
for nb_in_path in in_dir.glob("*.ipynb"):
nb_out_path = out_dir / nb_in_path.name
if not nb_out_path.is_file():
print(f"Copy: {nb_in_path}\n -> {nb_out_path}")
shutil.copy(nb_in_path, nb_out_path)
else:
print(f"Copy skipped: {nb_out_path}")
nb_paths.append(nb_out_path)
# Execute Jupyter notebooks
for nb_path in nb_paths:
print(f"[Processing notebook {nb_path.name}]")
with open(nb_path) as f:
nb = nbformat.read(f, as_version=4)
# https://github.com/spatialaudio/nbsphinx/blob/master/src/nbsphinx.py
has_code = any(c.source for c in nb.cells if c.cell_type == "code")
has_output = any(
c.get("outputs") or c.get("execution_count") for c in nb.cells
if c.cell_type == "code")
execute = (self.execute_notebooks == "auto" and has_code and
not has_output) or self.execute_notebooks == "always"
print(
f"has_code: {has_code}, has_output: {has_output}, execute: {execute}"
)
if execute:
ep = nbconvert.preprocessors.ExecutePreprocessor(timeout=6000)
try:
ep.preprocess(nb, {"metadata": {"path": nb_path.parent}})
except nbconvert.preprocessors.execute.CellExecutionError:
print(
f"Execution of {nb_path.name} failed, this will cause Travis to fail."
)
if "TRAVIS" in os.environ:
raise
with open(nb_path, "w", encoding="utf-8") as f:
nbformat.write(nb, f)
# Note, this should trawl through subdirectories
# Also - where does this actually get executed
# And by which commands
fails = 0
for file in glob.glob("Mapping/*.ipynb"):
try:
with io.open(file, 'r', encoding='utf8') as f:
nb = read(f, as_version=NO_CONVERT)
nb2 = remove_outputs(nb)
shutil.copy(file, "{}.backup".format(file))
try:
with io.open("{}".format(file), 'w', encoding='utf8') as f:
write(nb2, f)
print 'stripped NB v {} file "{}"'.format(nb.nbformat, file)
except:
print "Unable to write file"
shutil.copy("{}.backup".format(file), file)
print json.dumps( nb2, indent=2 )
except:
print "Warning: Notebook {} was not stripped of output data, please check before committing".format(file)
print " Over-ride using 'git commit --no-verify' if this warning can be ignored "
fails += 1
if(fails):
sys.exit(-1)
def create_notebook_templates(self):
'''
Create notebook templates in current working directory.
The notebooks contain a skeleton to support the resumableds workflow.
'''
nb_defs = {
'''\
# Definitions
Define project variables, etc.''':
nbformat.v4.new_markdown_cell,
'''\
import resumableds''':
nbformat.v4.new_code_cell,
'''\
# DS project name
project = '%s'
# create project
rds = resumableds.RdsProject(project, 'defs')''' % self.project_name:
nbformat.v4.new_code_cell,
'''\
# your variables / definitions go here...
#rds.defs.a = 'a variable'
''':
nbformat.v4.new_code_cell,
'''\
# save defs to disk
rds.save('defs')''':
nbformat.v4.new_code_cell,
'''\
*(Notebook is based on resumableds template)*''':
nbformat.v4.new_markdown_cell,
}
nb_collection = {
'''\
# Data collection
Get raw data from data storages.''':
nbformat.v4.new_markdown_cell,
'''\
import resumableds''':
nbformat.v4.new_code_cell,
'''\
# DS project name
project = '%s'
# create project
rds = resumableds.RdsProject(project, 'raw')''' % self.project_name:
nbformat.v4.new_code_cell,
'''\
# your data retrieval here
#rds.raw.customer_details = pd.read_sql_table('customer_details', example_con)
''':
nbformat.v4.new_code_cell,
'''\
# save project
rds.save('raw')''':
nbformat.v4.new_code_cell,
'''\
*(Notebook is based on resumableds template)*''':
nbformat.v4.new_markdown_cell,
}
nb_processing = {
'''\
# Processing
Manipulate your data.''':
nbformat.v4.new_markdown_cell,
'''\
import resumableds''':
nbformat.v4.new_code_cell,
'''\
# DS project name
project = '%s'
# create project
rds = resumableds.RdsProject(project, ['raw', 'interim', 'processed'])''' % self.project_name:
nbformat.v4.new_code_cell,
'''\
# your data processing here
#rds.interim.german_customers = rds.raw.customer_details.loc[rds.raw.customer_details['country'] == 'Germany']
#rds.processed.customers_by_city = rds.interim.german_customers.groupby('city').customer_name.count()
''':
nbformat.v4.new_code_cell,
'''\
# save project
rds.save(['interim', 'processed'])''':
nbformat.v4.new_code_cell,
'''\
*(Notebook is based on resumableds template)*''':
nbformat.v4.new_markdown_cell,
}
nb_graphs = {
'''\
# Graphical output
Visualize your data.''':
nbformat.v4.new_markdown_cell,
'''\
import resumableds''':
nbformat.v4.new_code_cell,
'''\
# DS project name
project = '%s'
# create project
rds = resumableds.RdsProject(project, ['processed'])''' % self.project_name:
nbformat.v4.new_code_cell,
'''\
# your data visualization here
#rds.processed.customers_by_city.plot()
''':
nbformat.v4.new_code_cell,
'''\
# save project
rds.save('defs')''':
nbformat.v4.new_code_cell,
'''\
*(Notebook is based on resumableds template)*''':
nbformat.v4.new_markdown_cell,
}
nb_templates = {
'01_definitions.ipynb': nb_defs,
'10_collection.ipynb': nb_collection,
'20_processing.ipynb': nb_processing,
'30_graphs.ipynb': nb_graphs,
#'40_publication.ipynb': nb_publication,
}
for nb_name, nb_cells in nb_templates.items():
logging.debug('create notebook "%s" from template' % nb_name)
nb = nbformat.v4.new_notebook()
nb['cells'] = [f(arg) for arg, f in nb_cells.items()]
nbformat.write(nb, nb_name)
#!/usr/bin/env python
import nbformat
import sys
if len(sys.argv) != 3:
print("usage: %s INFILE OUTFILE" % sys.argv[0])
print(sys.argv)
sys.exit(1)
nb = nbformat.read(sys.argv[1], nbformat.NO_CONVERT)
cell = nbformat.v4.new_code_cell("""
import cloudpickle
validator = "validator" in globals() and globals()["validator"] or (lambda x: True)
with open("model.pickle", "wb") as f:
f.write(cloudpickle.dumps((validator, globals()["predictor"])))
with open("extra-requirements.txt", "w") as f:
reqs = "requirements" in globals() and globals()["requirements"] or []
f.write('\\n'.join(["%s==%s" % (k,v) for k,v in reqs]))
""")
nb.cells.append(cell)
nbformat.write(nb, sys.argv[2])
def main(arglist):
"""Process IPython notebooks from a list of files."""
args = parse_args(arglist)
# Filter paths from the git manifest
# - Only process .ipynb
# - Don't process student notebooks
# - Don't process deleted notebooks (which are paths in the git manifest)
def should_process(path):
return all([
path.endswith(".ipynb"),
"student/" not in path,
os.path.isfile(path),
])
nb_paths = [arg for arg in args.files if should_process(arg)]
if not nb_paths:
print("No notebook files found")
sys.exit(0)
# Set execution parameters. We allow NotImplementedError as that is raised
# by incomplete exercises and is unlikely to be otherwise encountered.
exec_kws = {"timeout": 600, "allow_error_names": ["NotImplementedError"]}
# Allow environment to override stored kernel name
if "NB_KERNEL" in os.environ:
exec_kws["kernel_name"] = os.environ["NB_KERNEL"]
# Defer failures until after processing all notebooks
errors = {}
notebooks = {}
for nb_path in nb_paths:
# Load the notebook structure
with open(nb_path) as f:
nb = nbformat.read(f, nbformat.NO_CONVERT)
if not sequentially_executed(nb):
if args.require_sequential:
err = (
"Notebook is not sequentially executed on a fresh kernel."
"\n"
"Please do 'Restart and run all' before pushing to Github."
)
errors[nb_path] = err
continue
# Clean whitespace from all code cells
clean_whitespace(nb)
# Run the notebook from top to bottom, catching errors
print(f"Executing {nb_path}")
executor = ExecutePreprocessor(**exec_kws)
try:
executor.preprocess(nb)
except Exception as err:
if args.raise_fast:
# Exit here (useful for debugging)
raise err
else:
# Log the error, but then continue
errors[nb_path] = err
else:
notebooks[nb_path] = nb
if errors or args.check_only:
exit(errors)
# Further filter the notebooks to run post-processing only on tutorials
tutorials = {
nb_path: nb
for nb_path, nb in notebooks.items()
if nb_path.startswith("tutorials")
}
# Post-process notebooks to remove solution code and write both versions
for nb_path, nb in tutorials.items():
# Extract components of the notebook path
nb_dir, nb_fname = os.path.split(nb_path)
nb_name, _ = os.path.splitext(nb_fname)
# Loop through the cells and fix any Colab badges we encounter
for cell in nb.get("cells", []):
if has_colab_badge(cell):
redirect_colab_badge_to_master_branch(cell)
# Ensure that Colab metadata dict exists and enforce some settings
add_colab_metadata(nb, nb_name)
# Clean the original notebook and save it to disk
print(f"Writing complete notebook to {nb_path}")
with open(nb_path, "w") as f:
nb_clean = clean_notebook(nb)
nbformat.write(nb_clean, f)
# Create subdirectories, if they don't exist
student_dir = make_sub_dir(nb_dir, "student")
static_dir = make_sub_dir(nb_dir, "static")
solutions_dir = make_sub_dir(nb_dir, "solutions")
# Generate the student version and save it to a subdirectory
print(f"Extracting solutions from {nb_path}")
processed = extract_solutions(nb, nb_dir, nb_name)
student_nb, static_images, solution_snippets = processed
# Loop through cells and point the colab badge at the student version
for cell in student_nb.get("cells", []):
if has_colab_badge(cell):
redirect_colab_badge_to_student_version(cell)
# Write the student version of the notebook
student_nb_path = os.path.join(student_dir, nb_fname)
print(f"Writing student notebook to {student_nb_path}")
with open(student_nb_path, "w") as f:
clean_student_nb = clean_notebook(student_nb)
nbformat.write(clean_student_nb, f)
# Write the images extracted from the solution cells
print(f"Writing solution images to {static_dir}")
for fname, image in static_images.items():
fname = fname.replace("static", static_dir)
image.save(fname)
# Write the solution snippets
print(f"Writing solution snippets to {solutions_dir}")
for fname, snippet in solution_snippets.items():
fname = fname.replace("solutions", solutions_dir)
with open(fname, "w") as f:
f.write(snippet)
exit(errors)
**{will.description}**
TODO"""
notes["cells"].extend(
[nbformat.v4.new_markdown_cell(text),
nbformat.v4.new_code_cell("")])
# Blessings
notes["cells"].append(nbformat.v4.new_markdown_cell(blessing_header))
for blessing in election.blessings:
text = f"""### {blessing.title}
**{blessing.description}**
TODO"""
code = f"""{str(getattr(blessing, "metadata", ""))}"""
notes["cells"].extend(
[nbformat.v4.new_markdown_cell(text),
nbformat.v4.new_code_cell(code)])
# Appendix
notes["cells"].append(nbformat.v4.new_markdown_cell(APPENDIX))
# Write file
with open(file_name, 'w') as fp:
nbformat.write(notes, fp)
print(f"Notes generated for Season {season}!")
hide = False
if len(sys.argv) > 1 and sys.argv[1] == 'hide':
hide = True
path = "./book/content/Exercises/"
print("The path can further be specified. Finish with Return.")
while True:
folder = input("Change all notebooks in " + path)
if folder is '':
break
path += folder + "/"
notebooks = glob(path + "**/*.ipynb", recursive=True)
# Search through each notebook and look for the loesung tag
for ipath in notebooks:
ntbk = nbf.read(ipath, nbf.NO_CONVERT)
for cell in ntbk.cells:
cell_tags = cell.get('metadata', {}).get('tags', [])
if 'loesung' in cell_tags:
if hide and 'remove_cell' not in cell_tags:
cell_tags.append('remove_cell')
print('Hid cell in ' + ipath)
elif not hide:
if 'remove_cell' in cell_tags:
cell_tags.remove('remove_cell')
print('Revealed cell in' + ipath)
if len(cell_tags) > 0:
cell['metadata']['tags'] = cell_tags
nbf.write(ntbk, ipath)
# We always start with at least one line of text
cell_type = "markdown"
cell = []
for line in lines:
# Ignore matplotlib plot directive
if ".. plot" in line or ":context:" in line:
continue
# Ignore blank lines
if not line:
continue
if line_type(line) != cell_type:
# We are on the first line of the next cell,
# so package up the last cell
add_cell(nb, cell, cell_type)
cell_type = line_type(line)
cell = []
if line_type(line) == "code":
line = re.sub(pat, "", line)
cell.append(line)
# Package the final cell
add_cell(nb, cell, cell_type)
nbformat.write(nb, f"docstrings/{name}.ipynb")
def write_notebook_to_disk(cls, notebook, notebook_file_path):
with open(notebook_file_path, "w") as f:
nbformat.write(notebook, f)
def reduce_specs():
"""Reduce specification parameters in JSON file a notebooks file."""
# Load notebook 01_spec_preparation
nb_path = '/home/neuro/notebooks/01_spec_preparation.ipynb'
with open(nb_path, 'rb') as nb_file:
nb_node = nbformat.reads(nb_file.read(), nbformat.NO_CONVERT)
# Rewrite notebook cells
for cell in nb_node['cells']:
if 'code' == cell['cell_type']:
if 'task_id = sorted(layout.get_tasks())' in cell['source']:
txt = cell['source']
txt = txt.replace('task_id = sorted(layout.get_tasks())',
'task_id = [sorted(layout.get_tasks())[1]]')
cell['source'] = txt
elif 'Voxel resolution of reference template' in cell['source']:
txt = cell['source']
txt = txt.replace('[1.0, 1.0, 1.0]', '[4.0, 4.0, 4.0]')
cell['source'] = txt
elif 'Should ANTs Normalization a \'fast\'' in cell['source']:
txt = cell['source']
txt = txt.replace('precise', 'fast')
cell['source'] = txt
elif 'Create contrasts (unique and versus rest)' in cell['source']:
txt = cell['source']
txt = txt.replace('df[\'condition\']', 'df[\'trial_type\']')
cell['source'] = txt
elif 'Specify which classifier to use' in cell['source']:
txt = cell['source']
txt = txt.replace('[\'SMLR\', \'LinearNuSVMC\']',
'[\'LinearNuSVMC\']')
cell['source'] = txt
elif 'Searchlight sphere radius' in cell['source']:
txt = cell['source']
txt = txt.replace(' = 3', ' = 2')
cell['source'] = txt
elif 'Number of step size to define' in cell['source']:
txt = cell['source']
txt = txt.replace(' = 3', ' = 1000')
cell['source'] = txt
elif 'Number of chunks to use' in cell['source']:
txt = cell['source']
txt = txt.replace(' = len(runs)', ' = 4')
cell['source'] = txt
elif 'Which classifications should be performed' in cell['source']:
txt = 'content_multivariate[\'tasks\'] = '
txt += '{u\'fingerfootlips\': [[[u\'Finger\', u\'Foot\'], '
txt += '[u\'Finger\', u\'Lips\']]]}'
cell['source'] = txt
elif 'Number of permutations to indicate group' in cell['source']:
txt = cell['source']
txt = txt.replace(' = 100', ' = 10')
cell['source'] = txt
elif 'Number of bootstrap samples to be' in cell['source']:
txt = cell['source']
txt = txt.replace(' = 100000', ' = 100')
cell['source'] = txt
elif 'Number of segments used to compute' in cell['source']:
txt = cell['source']
txt = txt.replace(' = 1000', ' = 10')
cell['source'] = txt
elif 'Feature-wise probability threshold per' in cell['source']:
txt = cell['source']
txt = txt.replace(' = 0.001', ' = 0.1')
cell['source'] = txt
# Overwrite notebook 01_spec_preparation with new changes
nbformat.write(nb_node, nb_path)
print('JSON specification file creation adapted to demo dataset.')
# Load notebook 06_analysis_multivariate
nb_path = '/home/neuro/notebooks/06_analysis_multivariate.ipynb'
with open(nb_path, 'rb') as nb_file:
nb_node = nbformat.reads(nb_file.read(), nbformat.NO_CONVERT)
# Rewrite notebook cells
for cell in nb_node['cells']:
if 'code' == cell['cell_type']:
if 'collects the relevant input files' in cell['source']:
txt = cell['source']
txt = txt.replace('glob(template_con.format',
'4 * glob(template_con.format')
cell['source'] = txt
elif '==%s\' % imgs_orig.shape[-1]' in cell['source']:
txt = cell['source']
txt = txt.replace('==%s\' % imgs_orig.shape[-1]', '!=0\'')
cell['source'] = txt
# Overwrite notebook 06_analysis_multivariate with new changes
nbformat.write(nb_node, nb_path)
print('Multivariate specification adapted to demo dataset.')
def parse_notebooks(folder, url_docs):
"""
Modifies raw and html-fixed notebooks so they will not have broken links
to other files in the documentation. Adds a box to the sphinx formatted
notebooks with info and links to the *.ipynb and *.py files.
"""
if "dev" in __version__:
release_number_binder = "master"
release_number_docs = "dev"
else:
release_number_docs = release_number_binder = __version__
DOWNLOAD_CELL = """
<div class="alert alert-info">
**This is a fixed-text formatted version of a Jupyter notebook**
- Try online [](https://mybinder.org/v2/gh/gammapy/gammapy-webpage/{release_number_binder}?urlpath=lab/tree/{nb_filename})
- You can contribute with your own notebooks in this
[GitHub repository](https://github.com/gammapy/gammapy/tree/master/tutorials).
- **Source files:**
[{nb_filename}](../_static/notebooks/{nb_filename}) |
[{py_filename}](../_static/notebooks/{py_filename})
</div>
"""
for nbpath in list(folder.glob("*.ipynb")):
if str(folder) == "notebooks":
# add binder cell
nb_filename = str(nbpath).replace("notebooks/", "")
py_filename = nb_filename.replace("ipynb", "py")
ctx = dict(
nb_filename=nb_filename,
py_filename=py_filename,
release_number_binder=release_number_binder,
)
strcell = DOWNLOAD_CELL.format(**ctx)
rawnb = nbformat.read(str(nbpath), as_version=nbformat.NO_CONVERT)
if "nbsphinx" not in rawnb.metadata:
rawnb.metadata["nbsphinx"] = {"orphan": bool("true")}
rawnb.cells.insert(0, new_markdown_cell(strcell))
# add latex format
for cell in rawnb.cells:
if "outputs" in cell.keys():
for output in cell["outputs"]:
if output["output_type"] == "execute_result":
if "text/latex" in output["data"].keys():
output["data"]["text/latex"] = output["data"][
"text/latex"
].replace("$", "$$")
nbformat.write(rawnb, str(nbpath))
# modif links to rst /html doc files
txt = nbpath.read_text(encoding="utf-8")
if str(folder) == "notebooks":
repl = r"..\/\1rst\2"
txt = re.sub(
pattern=url_docs + r"(.*?)html(\)|#)",
repl=repl,
string=txt,
flags=re.M | re.I,
)
else:
url_docs_release = url_docs.replace("dev", release_number_docs)
txt = txt.replace(url_docs, url_docs_release)
nbpath.write_text(txt, encoding="utf-8")
txt = f.read()
txt = txt.replace("{{{TOINUMBER}}}", "{0}".format(toi_number))
txt = txt.replace("{{{VERSIONNUMBER}}}", "{0}".format(__version__))
txt = re.sub(r"toi_num = [0-9]+", "toi_num = {0}".format(toi_number), txt)
with open(filename, "w") as f:
f.write(txt)
with open(filename) as f:
notebook = nbformat.read(f, as_version=4)
ep = ExecutePreprocessor(timeout=-1)
print("running: {0}".format(filename))
try:
ep.preprocess(notebook,
{"metadata": {
"path": "notebooks/{0}".format(__version__)
}})
except CellExecutionError as e:
msg = "error while running: {0}\n\n".format(filename)
msg += e.traceback
print(msg)
finally:
with open(filename, mode="wt") as f:
nbformat.write(notebook, f)
subprocess.check_call("git add {0}".format(filename), shell=True)
subprocess.check_call("git commit -m \"adding '{0}'\"".format(filename),
shell=True)
