def mdstrip(paths):
for path in paths:
if os.path.isdir(path):
files = glob(os.path.join(path, "*.ipynb"))
else:
files = [path]
for in_file in files:
input_nb_name = basename(in_file)
slug = input_nb_name[:-6]
title = slug.replace("_", " ")
actual_title_nb = io.StringIO(
title_data.replace("{{ title }}", title))
title_nb = nbf.read(actual_title_nb, "ipynb")
title_cell = title_nb.worksheets[0].cells[0]
input_nb = nbf.read(open(in_file), "ipynb")
worksheet = input_nb.worksheets[0]
# add graphic here & append to cell_list
cell_list = [title_cell]
for cell in worksheet.cells:
if cell.cell_type == ("code"):
cell.outputs = []
cell_list.append(cell)
elif cell.cell_type == "heading":
cell_list.append(cell)
output_nb = nbf.new_notebook()
output_nb_name = slug+".prod.ipynb"
output_nb.worksheets.append(nbf.new_worksheet(cells=cell_list))
with open(output_nb_name, 'w') as f:
nbf.write(output_nb, f, "ipynb")
def convert_nb(nbname):
rst_name = "%s.rst" % nbname
nbname = "%s.ipynb" % nbname
# Do nothing if already built.
if os.path.exists(rst_name) and \
os.path.getmtime(rst_name) >= os.path.getmtime(nbname):
print("\t%s is up to date; nothing to do." % rst_name)
return
os.system("runipy --o %s --matplotlib --quiet" % nbname)
with io.open(nbname, 'r', encoding='utf8') as f:
nb = current.read(f, 'json')
nb = clean_for_doc(nb)
print("Writing to", nbname)
with io.open(nbname, 'w', encoding='utf8') as f:
current.write(nb, f, 'json')
# Convert to rst.
os.system("ipython nbconvert --to rst %s" % nbname)
with io.open(nbname, 'r', encoding='utf8') as f:
nb = current.read(f, 'json')
nb = strip_output(nb)
print("Writing to", nbname)
with io.open(nbname, 'w', encoding='utf8') as f:
current.write(nb, f, 'json')
def testRunNotebooks(self):
notebook_dir = path.join('tests', 'input')
for notebook_path in glob(path.join(notebook_dir, '*.ipynb')):
notebook_file = path.basename(notebook_path)
print notebook_file
expected_file = path.join('tests', 'expected', notebook_file)
runner = NotebookRunner(read(open(notebook_path), 'json'), working_dir=notebook_dir)
runner.run_notebook(True)
expected = read(open(expected_file), 'json')
self.assert_notebooks_equal(expected, runner.nb)
def testRunNotebooks(self):
input_glob = path.join('tests', 'input', '*.ipynb')
for notebook_file in glob(input_glob):
notebook_file_base = path.basename(notebook_file)
print notebook_file_base
expected_file = path.join('tests', 'expected', notebook_file_base)
runner = NotebookRunner(read(open(notebook_file), 'json'))
runner.run_notebook(True)
expected = read(open(expected_file), 'json')
self.assert_notebooks_equal(expected, runner.nb)
def restore_checkpoint(self, checkpoint_id, name, path=""):
"""restore a notebook to a checkpointed state"""
path = path.strip("/")
self.log.info("restoring Notebook %s from checkpoint %s", name, checkpoint_id)
nb_path = self._get_os_path(name, path)
cp_path = self.get_checkpoint_path(checkpoint_id, name, path)
if not os.path.isfile(cp_path):
self.log.debug("checkpoint file does not exist: %s", cp_path)
raise web.HTTPError(404, u"Notebook checkpoint does not exist: %s-%s" % (name, checkpoint_id))
# ensure notebook is readable (never restore from an unreadable notebook)
with io.open(cp_path, "r", encoding="utf-8") as f:
current.read(f, u"json")
self._copy(cp_path, nb_path)
self.log.debug("copying %s -> %s", cp_path, nb_path)
def testRunNotebooks(self):
notebook_dir = path.join("tests", "input")
for notebook_path in glob(path.join(notebook_dir, "*.ipynb")):
notebook_file = path.basename(notebook_path)
print(notebook_file)
expected_file = path.join("tests", "expected", notebook_file)
notebook = ""
with open(notebook_path) as notebook_file:
notebook = read(notebook_file, "json")
runner = NotebookRunner(notebook, working_dir=notebook_dir)
runner.run_notebook(True)
expected = ""
with open(expected_file) as notebook_file:
expected = read(notebook_file, "json")
self.assert_notebooks_equal(expected, runner.nb)
def export_latex(fname):
with open(fname) as f:
nb = current.read(f, 'json')
lines = ''
for cell in nb.worksheets[0].cells:
if cell.cell_type == u'code':
lines += '\\begin{verbatim}\n'
lines += '%s\n' % cell.input
lines += '\\end{verbatim}\n'
for output in cell.outputs:
if output.output_type == u'pyout':
if hasattr(output, 'latex'):
lines += '%s\n' % output.latex
else:
lines += '\n'
lines += '\\begin{verbatim}\n'
lines += '%s\n' % output.text
lines += '\\end{verbatim}\n'
lines += '\n'
if cell.cell_type == u'markdown':
paragraphs = wrap_paragraphs(cell.source)
for p in paragraphs:
lines += p
lines += '\n\n'
newfname = os.path.splitext(fname)[0] + '.tex'
with open(newfname,'w') as f:
f.write(lines.encode('utf8'))
def from_filename(self, filename, resources=None, **kw):
"""
Convert a notebook from a notebook file.
Parameters
----------
filename : str
Full filename of the notebook file to open and convert.
"""
# Pull the metadata from the filesystem.
if resources is None:
resources = ResourcesDict()
if not 'metadata' in resources or resources['metadata'] == '':
resources['metadata'] = ResourcesDict()
basename = os.path.basename(filename)
notebook_name = basename[:basename.rfind('.')]
resources['metadata']['name'] = notebook_name
modified_date = datetime.datetime.fromtimestamp(
os.path.getmtime(filename))
resources['metadata'][
'modified_date'] = modified_date.strftime(text.date_format)
with io.open(filename, encoding='utf-8') as f:
return self.from_notebook_node(nbformat.read(f, 'json'), resources=resources, **kw)
def test_save(self):
resp = self.nb_api.read('a.ipynb', 'foo')
nbcontent = json.loads(resp.text)['content']
nb = to_notebook_json(nbcontent)
ws = new_worksheet()
nb.worksheets = [ws]
ws.cells.append(new_heading_cell(u'Created by test ³'))
nbmodel = {'name': 'a.ipynb', 'path': 'foo', 'content': nb}
resp = self.nb_api.save(
'a.ipynb', path='foo', body=json.dumps(nbmodel))
nbfile = pjoin(self.notebook_dir.name, 'foo', 'a.ipynb')
with io.open(nbfile, 'r', encoding='utf-8') as f:
newnb = read(f, format='ipynb')
self.assertEqual(newnb.worksheets[0].cells[0].source,
u'Created by test ³')
nbcontent = self.nb_api.read('a.ipynb', 'foo').json()['content']
newnb = to_notebook_json(nbcontent)
self.assertEqual(newnb.worksheets[0].cells[0].source,
u'Created by test ³')
# Save and rename
nbmodel = {'name': 'a2.ipynb', 'path': 'foo/bar', 'content': nb}
resp = self.nb_api.save(
'a.ipynb', path='foo', body=json.dumps(nbmodel))
saved = resp.json()
self.assertEqual(saved['name'], 'a2.ipynb')
self.assertEqual(saved['path'], 'foo/bar')
assert os.path.isfile(
pjoin(self.notebook_dir.name, 'foo', 'bar', 'a2.ipynb'))
assert not os.path.isfile(
pjoin(self.notebook_dir.name, 'foo', 'a.ipynb'))
with assert_http_error(404):
self.nb_api.read('a.ipynb', 'foo')
def process_html_notebook(nb_path, src_dir=None, start=None, end=None, **kwargs):
with open(nb_path) as f:
n = nbformat.read(f, 'ipynb')
fullhtml = FullHtmlStaticExporter()
(nbc,resources) = fullhtml._preprocess(n, resources={})
if src_dir:
nbc = _prepend_srcdir(nbc, src_dir)
# made the decision to have subset after image genration
# the images are done via an iteration key. So we make sure to generate
# the keys for the entire notebook and then grab the cells after wrds.
# this is to keep image names from clashing. image 33 will be image 33 regardless
# of cells range
nbc = _subset_cells(nbc, start, end)
output_only = kwargs.get('output_only', False)
if output_only:
cells = nbc.worksheets[0].cells
for cell in cells:
del cell['input']
del cell['prompt_number']
# render to html
template = fullhtml.environment.get_template(fullhtml.template_file+fullhtml.template_extension)
body = template.render(nb=nbc, resources=resources)
return body, resources
def run_notebook(self, nb_in, skip_exceptions=False, autosave=None):
'''
Run all the cells of a notebook in order and update
the outputs in-place.
If ``skip_exceptions`` is set, then if exceptions occur in a cell, the
subsequent cells are run (by default, the notebook execution stops).
'''
self.nb = read(open(nb_in), 'json')
for cell in self.iter_code_cells():
cell['outputs'] = []
if 'prompt_number' in cell:
del cell['prompt_number']
if autosave is not None:
self.save_notebook(autosave)
for cell in self.iter_code_cells():
try:
self.run_cell(cell, autosave = autosave)
except NotebookError:
if not skip_exceptions:
raise
if autosave is not None:
self.save_notebook(autosave)
def load_module(self, fullname):
"""import a notebook as a module"""
path = find_notebook(fullname, self.path)
disp = "importing IPython notebook from "
disp += "<a href='./{}' target='_blank'>{}</a>".format(path, path[:-6])
display_html(disp, raw=True)
# print disp
# load the notebook object
with io.open(path, "r", encoding="utf-8") as f:
nb = current.read(f, "json")
# create the module and add it to sys.modules
# if name in sys.modules:
# return sys.modules[name]
mod = types.ModuleType(fullname)
mod.__file__ = path
mod.__loader__ = self
sys.modules[fullname] = mod
# extra work to ensure that magics that would affect the user_ns
# actually affect the notebook module's ns
save_user_ns = self.shell.user_ns
self.shell.user_ns = mod.__dict__
try:
for cell in nb.worksheets[0].cells:
if cell.cell_type == "code" and cell.language == "python":
# transform the input to executable Python
code = self.shell.input_transformer_manager.transform_cell(cell.input)
# run the code in themodule
exec code in mod.__dict__
finally:
self.shell.user_ns = save_user_ns
return mod
def notebook(notebook):
try:
notebook = nbf.read(open('notebooks/%s' % notebook, 'r'), 'ipynb')
except IOError:
abort(418)
html_notebook= convert_nb_html(notebook)
return render_template('notebook.html', content=html_notebook)
def generate_data_files(self, ip, notebook, ref_dir, data_dir, regen):
"""
Generate both testand new reference data (if needed) using
NBRunner.
"""
msg =''
basename = os.path.split(notebook)[1].rsplit('.ipynb')[0]
nb = current.read(open(notebook,'r'), 'ipynb')
# Reference data not found - regenerate it and exit
if regen:
if os.path.isdir(ref_dir): shutil.rmtree(ref_dir)
os.mkdir(ref_dir)
reference_runner = NBRunner(ip, basename, nb, ref_dir, reference=True)
reference_runner.run()
return ''
elif not os.path.isdir(ref_dir):
return False
# Remove any pre-existing test data.
if os.path.isdir(data_dir):
shutil.rmtree(data_dir)
# Generate the test data
os.mkdir(data_dir)
NBRunner(ip, basename, nb, data_dir, reference=False).run()
return msg
def __init__(self, root, ipynb_name):
self.root = root
with io.open(ipynb_name, 'rb') as f:
self.ipynb = v3.read(f, 'json')
self.ipynb_name = ipynb_name
self.ipynb_full = '/Users/bussonniermatthias/ipynb2fs/'+self.ipynb_name
self.buffers = dict({})
def main():
parser = ArgumentParser(description=DESCRIP,
epilog=EPILOG,
formatter_class=RawDescriptionHelpFormatter)
parser.add_argument('searchpath', type=str,
help='directory from which to search')
args = parser.parse_args()
for dirpath, dirnames, filenames in os.walk(args.searchpath):
# Omit directories beginning with dots and underscores
dirnames[:] = [d for d in dirnames
if not d.startswith('.') and not d.startswith('_')]
for fname in filenames:
if fname.startswith('.'):
continue
if not fname.endswith('.ipynb'):
continue
fullpath = os.path.join(dirpath, fname)
with io.open(fullpath, 'r') as f:
nb = current.read(f, 'json')
for cell in cellgen(nb, 'code'):
if hasattr(cell, 'prompt_number'):
sys.stderr.write(
'cell prompt number {0} in {1}\n'.format(
cell.prompt_number,
fname))
sys.exit(1)
if not cell.outputs == []:
sys.stderr.write('cell output in {0}\n'.format(fname))
sys.exit(1)
def split_into_units(nb_name):
"""Split notebook into units."""
try:
with io.open(nb_name, 'r', encoding='utf-8') as f:
nb = current.read(f, 'json')
except IOError as e:
if e.errno == 2:
print('File not found: {0}'.format(nb_name))
return []
else:
raise e
indexes = []
cells = nb.worksheets[0].cells
for (i, cell) in enumerate(cells):
if cell.cell_type == 'heading' and cell.level == 1:
indexes.append(i)
separated_cells = [cells[i:j] for i, j in zip(indexes, indexes[1:]+[None])]
worksheets = map(lambda cells: current.new_worksheet(name=cells[0].source,
cells=cells),
separated_cells)
units = map(lambda worksheet: current.new_notebook(name=worksheet.name,
worksheets=[worksheet]),
worksheets)
return units
def get_notebook_params(path):
from runipy.notebook_runner import NotebookRunner
from IPython.nbformat.current import read
# Load the notebook:
with open(path) as f:
notebook = read(f, 'json')
r = NotebookRunner(notebook)
def callback(cell_idx):
try:
# Has the %params_done magic been called yet?
has_params = pull(r.shell,
"get_ipython().magics_manager.registry['ParameterMagics']"
".has_params")
except:
# The magic probably hasn't been loaded yet.
pass
else:
if has_params: raise StopIteration() # Yes, we're done!
# Run the notebook, checking whether we're done after every cell:
try: r.run_notebook(progress_callback=callback)
except StopIteration: pass
# Pull the param declarations:
try: params = pull(r.shell,
"get_ipython().magics_manager.registry['ParameterMagics']"
".params")
except: params = dict()
return params
def process_notebook_file(fname, actions=('clean',), asciidoc_fname=None):
print("Performing '{}' on: {}".format("', '".join(actions), fname))
orig_wd = os.getcwd()
os.chdir(os.path.dirname(fname))
with io.open(fname, 'rb') as f:
nb = current.read(f, 'json')
if 'uuid' in actions:
check_uuid(nb)
if 'render' in actions:
run_notebook(nb)
elif 'check' in actions:
run_notebook(deepcopy(nb))
if 'clean' in actions:
remove_outputs(nb)
if 'merge' in actions:
doc = AsciidDoc(asciidoc_fname)
doc.merge_code_from(nb)
doc.save()
elif 'convert' in actions:
doc = AsciidDoc()
doc.convert_from(nb)
doc.save(asciidoc_fname)
os.chdir(orig_wd)
with io.open(fname, 'wb') as f:
current.write(nb, f, 'json')
def load_module(self, fullname):
"""import a notebook as a module"""
path = find_notebook(fullname, self.path)
print ("importing Jupyter notebook from %s" % path)
# load the notebook object
with io.open(path, 'r', encoding='utf-8') as f:
nb = current.read(f, 'json')
# create the module and add it to sys.modules
# if name in sys.modules:
# return sys.modules[name]
mod = types.ModuleType(fullname)
mod.__file__ = path
mod.__loader__ = self
mod.__dict__['get_ipython'] = get_ipython
sys.modules[fullname] = mod
# extra work to ensure that magics that would affect the user_ns
# actually affect the notebook module's ns
save_user_ns = self.shell.user_ns
self.shell.user_ns = mod.__dict__
try:
for cell in nb.worksheets[0].cells:
if cell.cell_type == 'code' and cell.language == 'python':
# transform the input to executable Python
code = self.shell.input_transformer_manager.transform_cell(cell.input)
# run the code in themodule
exec(code, mod.__dict__)
finally:
self.shell.user_ns = save_user_ns
return mod
def run(self):
from IPython.nbformat.current import read, write
with open('template.rst', 'r') as f:
template = f.read()
toc = ""
from urllib import quote
for notebook in (glob.glob('lectures/*.ipynb')):
with open(notebook, 'r') as f:
nb = read(f, 'json')
for ws in nb.worksheets:
for cell in ws.cells:
if cell.cell_type == 'heading':
if cell['level'] > 1:
continue
toc += (" " * (cell['level'] - 1) +
"* `{0} <_static/{1}.html#{2}>`__\n".format(cell['source'].replace('`', ''),
quote(os.path.basename(notebook)).replace('.ipynb', ''),
strip_punctuation(cell['source']).replace(' ', '-')))
with open('docs/index.rst', 'w') as f:
f.write(template.format(lectures_toc=toc))
def evaluate_notebook(nb_path, dest_path=None, skip_exceptions=False):
# Create evaluated version and save it to the dest path.
# Always use --pylab so figures appear inline
# perhaps this is questionable?
notebook = read(open(nb_path), 'json')
cwd = os.getcwd()
filedir, filename = os.path.split(nb_path)
os.chdir(filedir)
nb_runner = NotebookRunner(notebook, pylab=False)
try:
nb_runner.run_notebook(skip_exceptions=skip_exceptions)
except NotebookError as e:
print('')
print(e)
# Return the traceback, filtering out ANSI color codes.
# http://stackoverflow.com/questions/13506033/filtering-out-ansi-escape-sequences
return 'Notebook conversion failed with the following traceback: \n%s' % \
re.sub(r'\\033[\[\]]([0-9]{1,2}([;@][0-9]{0,2})*)*[mKP]?', '', str(e))
os.chdir(cwd)
write(nb_runner.nb, open(dest_path, 'w'), 'json')
ret = nb_to_html(dest_path)
if dest_path.endswith('temp_evaluated.ipynb'):
os.remove(dest_path)
return ret
def __init__(self, nb_in=None, pylab=False, mpl_inline=False, nb=None):
self.km = KernelManager()
if pylab:
self.km.start_kernel(extra_arguments=['--pylab=inline'])
elif mpl_inline:
self.km.start_kernel(extra_arguments=['--matplotlib=inline'])
else:
self.km.start_kernel()
if platform.system() == 'Darwin':
# There is sometimes a race condition where the first
# execute command hits the kernel before it's ready.
# It appears to happen only on Darwin (Mac OS) and an
# easy (but clumsy) way to mitigate it is to sleep
# for a second.
sleep(1)
self.kc = self.km.client()
self.kc.start_channels()
self.shell = self.kc.shell_channel
self.iopub = self.kc.iopub_channel
logging.info('Reading notebook %s', nb_in)
self.nb = nb
if not self.nb:
self.nb = read(open(nb_in), 'json')
def test_nbs():
path = os.path.join(pyfolio_root(), 'examples', '*.ipynb')
for ipynb in glob.glob(path):
with open(ipynb) as f:
nb = read(f, 'json')
nb_runner = NotebookRunner(nb)
nb_runner.run_notebook(skip_exceptions=False)
def evaluate_notebook(nb_path, dest_path=None, skip_exceptions=True):
# Create evaluated version and save it to the dest path.
# Always use --pylab so figures appear inline
# perhaps this is questionable?
curr_dir = os.getcwd()
os.chdir(os.path.dirname(nb_path))
notebook = read(open(nb_path), "json")
nb_runner = NotebookRunner(notebook, pylab=True)
try:
nb_runner.run_notebook(skip_exceptions=skip_exceptions)
except NotebookError as e:
print ""
print e
# Return the traceback, filtering out ANSI color codes.
# http://stackoverflow.com/questions/13506033/filtering-out-ansi-escape-sequences
return "Notebook conversion failed with the following traceback: \n%s" % re.sub(
r"\\033[\[\]]([0-9]{1,2}([;@][0-9]{0,2})*)*[mKP]?", "", str(e)
)
if dest_path is None:
dest_path = "temp_evaluated.ipynb"
write(nb_runner.nb, open(dest_path, "w"), "json")
ret = nb_to_html(dest_path)
if dest_path is "temp_evaluated.ipynb":
os.remove(dest_path)
os.chdir(curr_dir)
return ret
def test_export_nbnode(self):
"""
Can a notebook be exported by a notebook node handle?
"""
with open(self._get_notebook(), 'r') as f:
notebook = nbformat.read(f, 'json')
(output, resources) = export_by_name('python', notebook)
assert len(output) > 0
def execute_notebook(nbfile):
with io.open(nbfile) as f:
nb = current.read(f,'json')
ip = get_ipython()
for cell in nb.worksheets[0].cells:
if cell.cell_type != 'code':
continue
ip.run_cell(cell.input)
def restore_checkpoint(self, checkpoint_id, name, path=''):
"""restore a file to a checkpointed state"""
path = path.strip('/')
self.log.info("restoring %s from checkpoint %s", name, checkpoint_id)
nb_path = self._get_os_path(name, path)
cp_path = self.get_checkpoint_path(checkpoint_id, name, path)
if not os.path.isfile(cp_path):
self.log.debug("checkpoint file does not exist: %s", cp_path)
raise web.HTTPError(404,
u'checkpoint does not exist: %s-%s' % (name, checkpoint_id)
)
# ensure notebook is readable (never restore from an unreadable notebook)
if cp_path.endswith('.ipynb'):
with io.open(cp_path, 'r', encoding='utf-8') as f:
current.read(f, u'json')
self._copy(cp_path, nb_path)
self.log.debug("copying %s -> %s", cp_path, nb_path)
def notebook_content(self):
filepath = os.path.join(self.bundle_path, self.name)
with io.open(filepath, 'r', encoding='utf-8') as f:
try:
nb = current.read(f, u'json')
except Exception as e:
nb = None
return nb
def parse (self):
try:
with open(self.filename) as f:
self.notebook = current_notebook.read(f,'ipynb')
except:
print("Could not parse {}".format(self.filename))
import traceback
traceback.print_exc()
def main():
parser = ArgumentParser(description=DESCRIP,
epilog=EPILOG,
formatter_class=RawDescriptionHelpFormatter)
parser.add_argument('filename',
type=str,
nargs='+',
help='notebook filenames')
args = parser.parse_args()
for fname in args.filename:
with io.open(fname, 'r') as f:
nb = current.read(f, 'json')
for cell in cellgen(nb, 'code'):
if hasattr(cell, 'prompt_number'):
del cell['prompt_number']
cell.outputs = []
with io.open(fname, 'w') as f:
current.write(nb, f, 'ipynb')
def run(self):
from IPython.nbformat.current import read, write
for notebook in glob.glob('notebooks/*.ipynb'):
with open(notebook, 'r') as f:
nb = read(f, 'json')
for ws in nb.worksheets:
for cell in ws.cells:
if cell.cell_type == 'code':
cell.outputs = []
if 'prompt_number' in cell:
cell.pop('prompt_number')
with open(notebook, 'w') as f:
write(nb, f, 'json')
def remove_outputs(fname):
"""
remove the outputs from a notebook "fname" and create a new notebook
"""
with io.open(fname, 'r') as f:
nb = read(f, 'json')
for ws in nb.worksheets:
for cell in ws.cells:
if cell.cell_type == 'code':
cell.outputs = []
base, ext = os.path.splitext(fname)
new_ipynb = "%s_removed%s" % (base, ext)
with io.open(new_ipynb, 'w', encoding='utf8') as f:
write(nb, f, 'json')
print "wrote %s" % new_ipynb
return "Done"
def extract_exercises(inputs, odir='exercises', inplace=False):
"""
Strip output of notebooks.
Parameters
----------
inputs : list of string
Path to the notebooks to be processed.
inplace : bool
If this is `True`, outputs in the input files will be deleted.
Default is `False`.
"""
for inpath in inputs:
with file(inpath) as fp:
nb = nbformat.read(fp, 'ipynb')
clear_everything_but_exercises(nb)
if odir:
nbformat.write(nb, file(os.path.join(odir, os.path.basename(inpath)), 'w'), 'ipynb')
def merge_notebooks(filenames):
merged = None
added_appendix = False
for fname in filenames:
with io.open(fname, 'r', encoding='utf-8') as f:
nb = current.read(f, u'json')
remove_formatting(nb)
if not added_appendix and fname[0:8] == 'Appendix':
remove_links_add_appendix(nb)
added_appendix = True
else:
remove_links(nb)
if merged is None:
merged = nb
else:
merged.worksheets[0].cells.extend(nb.worksheets[0].cells)
merged.metadata.name += "_merged"
print(current.writes(merged, u'json'))
def notebook(self, s):
"""Export and convert IPython notebooks.
This function can export the current IPython history to a notebook file
or can convert an existing notebook file into a different format. For
example, to export the history to "foo.ipynb" do "%notebook -e foo.ipynb".
To export the history to "foo.py" do "%notebook -e foo.py". To convert
"foo.ipynb" to "foo.json" do "%notebook -f json foo.ipynb". Possible
formats include (json/ipynb, py).
"""
args = magic_arguments.parse_argstring(self.notebook, s)
from IPython.nbformat import current
args.filename = unquote_filename(args.filename)
if args.export:
fname, name, format = current.parse_filename(args.filename)
cells = []
hist = list(self.shell.history_manager.get_range())
for session, prompt_number, input in hist[:-1]:
cells.append(
current.new_code_cell(prompt_number=prompt_number,
input=input))
worksheet = current.new_worksheet(cells=cells)
nb = current.new_notebook(name=name, worksheets=[worksheet])
with io.open(fname, 'w', encoding='utf-8') as f:
current.write(nb, f, format)
elif args.format is not None:
old_fname, old_name, old_format = current.parse_filename(
args.filename)
new_format = args.format
if new_format == u'xml':
raise ValueError('Notebooks cannot be written as xml.')
elif new_format == u'ipynb' or new_format == u'json':
new_fname = old_name + u'.ipynb'
new_format = u'json'
elif new_format == u'py':
new_fname = old_name + u'.py'
else:
raise ValueError('Invalid notebook format: %s' % new_format)
with io.open(old_fname, 'r', encoding='utf-8') as f:
nb = current.read(f, old_format)
with io.open(new_fname, 'w', encoding='utf-8') as f:
current.write(nb, f, new_format)
def nb_to_html(nb_path):
"""convert notebook to html"""
c = Config({'ExtractOutputPreprocessor': {'enabled': True}})
exporter = html.HTMLExporter(template_file='full', config=c)
notebook = nbformat.read(open(nb_path), 'json')
output, resources = exporter.from_notebook_node(notebook)
header = output.split('<head>', 1)[1].split('</head>', 1)[0]
body = output.split('<body>', 1)[1].split('</body>', 1)[0]
# http://imgur.com/eR9bMRH
header = header.replace('<style', '<style scoped="scoped"')
header = header.replace(
'body {\n overflow: visible;\n padding: 8px;\n}\n', '')
header = header.replace("code,pre{", "code{")
# Filter out styles that conflict with the sphinx theme.
filter_strings = [
'navbar',
'body{',
'alert{',
'uneditable-input{',
'collapse{',
]
filter_strings.extend(['h%s{' % (i + 1) for i in range(6)])
line_begin = ['pre{', 'p{margin']
filterfunc = lambda x: not any([s in x for s in filter_strings])
header_lines = filter(filterfunc, header.split('\n'))
filterfunc = lambda x: not any([x.startswith(s) for s in line_begin])
header_lines = filter(filterfunc, header_lines)
header = '\n'.join(header_lines)
# concatenate raw html lines
lines = ['<div class="ipynotebook">']
lines.append(header)
lines.append(body)
lines.append('</div>')
return '\n'.join(lines), resources
def main():
parser = ArgumentParser(description=DESCRIP,
epilog=EPILOG,
formatter_class=RawDescriptionHelpFormatter)
parser.add_argument('in_fname', type=str, help='input notebook filename')
parser.add_argument('out_fname', type=str, help='output notebook filename')
args = parser.parse_args()
out_name = splitext(basename(args.out_fname))[0]
with io.open(args.in_fname, 'r') as f:
nb = current.read(f, 'json')
nb.name = out_name
for cell in cellgen(nb, 'code'):
if hasattr(cell, 'prompt_number'):
del cell['prompt_number']
cell.outputs = []
if hasattr(cell, 'input'):
cell['input'] = strip_after_comments(cell['input'])
with io.open(args.out_fname, 'w') as f:
current.write(nb, f, 'ipynb')
def test_save(self):
resp = self.api.read('a.ipynb', 'foo')
nbcontent = json.loads(resp.text)['content']
nb = to_notebook_json(nbcontent)
ws = new_worksheet()
nb.worksheets = [ws]
ws.cells.append(new_heading_cell(u'Created by test ³'))
nbmodel = {
'name': 'a.ipynb',
'path': 'foo',
'content': nb,
'type': 'notebook'
}
resp = self.api.save('a.ipynb', path='foo', body=json.dumps(nbmodel))
nbfile = pjoin(self.notebook_dir.name, 'foo', 'a.ipynb')
with io.open(nbfile, 'r', encoding='utf-8') as f:
newnb = read(f, format='ipynb')
self.assertEqual(newnb.worksheets[0].cells[0].source,
u'Created by test ³')
nbcontent = self.api.read('a.ipynb', 'foo').json()['content']
newnb = to_notebook_json(nbcontent)
self.assertEqual(newnb.worksheets[0].cells[0].source,
u'Created by test ³')
# Save and rename
nbmodel = {
'name': 'a2.ipynb',
'path': 'foo/bar',
'content': nb,
'type': 'notebook'
}
resp = self.api.save('a.ipynb', path='foo', body=json.dumps(nbmodel))
saved = resp.json()
self.assertEqual(saved['name'], 'a2.ipynb')
self.assertEqual(saved['path'], 'foo/bar')
assert os.path.isfile(
pjoin(self.notebook_dir.name, 'foo', 'bar', 'a2.ipynb'))
assert not os.path.isfile(
pjoin(self.notebook_dir.name, 'foo', 'a.ipynb'))
with assert_http_error(404):
self.api.read('a.ipynb', 'foo')
def run(self):
# Now convert the lecture notes, problem sets, and practice problems to
# HTML notebooks.
from runipy.notebook_runner import NotebookRunner
from IPython.nbformat.current import read, write
start_dir = os.path.abspath('.')
for notebook in glob.glob('notebooks/*.ipynb'):
print("Running {0}...".format(notebook))
os.chdir(os.path.dirname(notebook))
with open(os.path.basename(notebook)) as f:
r = NotebookRunner(read(f, 'json'), pylab=False)
r.run_notebook(skip_exceptions=True)
with open(os.path.basename(notebook), 'w') as f:
write(r.nb, f, 'json')
os.chdir(start_dir)
def _notebook_model(self, name, path='', content=True):
"""Build a notebook model
if content is requested, the notebook content will be populated
as a JSON structure (not double-serialized)
"""
model = self._base_model(name, path)
model['type'] = 'notebook'
if content:
os_path = self._get_os_path(name, path)
with io.open(os_path, 'r', encoding='utf-8') as f:
try:
nb = current.read(f, u'json')
except Exception as e:
raise web.HTTPError(400, u"Unreadable Notebook: %s %s" % (os_path, e))
self.mark_trusted_cells(nb, name, path)
model['content'] = nb
model['format'] = 'json'
return model
def execute_notebook(path):
''' * Função execute_notebook(path) executa todas as celulas de um notebook, que contêm
declarações de funções (def) em seu conteúdo
Entradas:
-path: String que contêm o diretório do arquivo .ipynb
SaÃda:
- Sem saÃdas.
'''
with io.open(path) as f:
nb = current.read(f, 'json')
ip = get_ipython()
for cell in nb.worksheets[0].cells:
if cell.cell_type != 'code':
continue
if find_word(cell.input, 'def'):
ip.run_cell(cell.input)
else:
continue
def main():
parser = ArgumentParser(description=DESCRIP,
epilog=EPILOG,
formatter_class=RawDescriptionHelpFormatter)
parser.add_argument('filename',
type=str,
nargs='+',
help='notebook filenames')
args = parser.parse_args()
for fname in args.filename:
with io.open(fname, 'r') as f:
nb = current.read(f, 'json')
for cell in cellgen(nb, 'code'):
if hasattr(cell, 'prompt_number'):
sys.stderr.write('cell prompt number {0} in {1}\n'.format(
cell.prompt_number, fname))
sys.exit(1)
if not cell.outputs == []:
sys.stderr.write('cell output in {0}\n'.format(fname))
sys.exit(1)
def get_notebook(self, name, path='', content=True):
""" Takes a path and name for a notebook and returns its model
Parameters
----------
name : str
the name of the notebook
path : str
the URL path that describes the relative path for
the notebook
Returns
-------
model : dict
the notebook model. If contents=True, returns the 'contents'
dict in the model as well.
"""
path = path.strip('/')
if not self.notebook_exists(name=name, path=path):
raise web.HTTPError(404, u'Notebook does not exist: %s' % name)
os_path = self._get_os_path(name, path)
info = os.stat(os_path)
last_modified = tz.utcfromtimestamp(info.st_mtime)
created = tz.utcfromtimestamp(info.st_ctime)
# Create the notebook model.
model = {}
model['name'] = name
model['path'] = path
model['last_modified'] = last_modified
model['created'] = created
model['type'] = 'notebook'
if content:
with io.open(os_path, 'r', encoding='utf-8') as f:
try:
nb = current.read(f, u'json')
except Exception as e:
raise web.HTTPError(
400, u"Unreadable Notebook: %s %s" % (os_path, e))
self.mark_trusted_cells(nb, name, path)
model['content'] = nb
return model
def notebooks(env, loader, args):
nb_path = env.resolve('notebooks')
res_path = env.resolve('results')
for filename in os.listdir(nb_path):
if filename.endswith('.ipynb'):
nb_name = filename.split('.')[0]
print('running', nb_name)
src_path = os.path.join(nb_path, filename)
html_path = os.path.join(res_path, nb_name + '.html')
with open(src_path, 'r') as f:
notebook = read(f, 'json')
r = NotebookRunner(notebook, working_dir=nb_path)
r.run_notebook()
dest_path = tempfile.mkstemp()
t = tempfile.NamedTemporaryFile()
with open(t.name, 'w') as g:
write(r.nb, g)
exporter = nbconvert.HTMLExporter()
body, resources = exporter.from_filename(t.name)
with open(html_path, 'wb') as g:
g.write(body.encode('utf-8'))
def evaluate_notebook(nb_path, dest_path=None, skip_exceptions=False):
# Create evaluated version and save it to the dest path.
# Always use --pylab so figures appear inline
# perhaps this is questionable?
notebook = read(open(nb_path), 'json')
nb_runner = NotebookRunner(notebook, pylab=True, mpl_inline=True)
try:
nb_runner.run_notebook(skip_exceptions=skip_exceptions)
except NotebookError as e:
print('\n', '-' * 80)
print(e)
print('-' * 80)
raise
if dest_path is None:
dest_path = 'temp_evaluated.ipynb'
write(nb_runner.nb, open(dest_path, 'w'), 'json')
ret = nb_to_html(dest_path)
if dest_path is 'temp_evaluated.ipynb':
os.remove(dest_path)
return ret
def import_notebook(nbname, loader=None):
fullpath = "{}/{}.ipynb".format(os.getcwd(), nbname)
if not os.path.exists(fullpath):
raise Exception(
"Failed to find ipython notebook: {}.ipynb".format(nbname))
if nbname in sys.modules:
mod = sys.modules[nbname]
return mod
moduleName = nbname
nbFilename = nbname + ".ipynb"
print "Importing ipynb file : {}".format(nbFilename)
with io.open(nbFilename) as f:
nb = current.read(f, 'json')
newModule = imp.new_module(moduleName)
try:
for cell in nb.worksheets[0].cells:
if cell.cell_type != 'code':
continue
exec cell.input in newModule.__dict__
except:
print "Error in importing ipython notebook."
print_exc()
raise
# Set a few properties required by PEP 302
newModule.__file__ = nbname
newModule.__name__ = nbname
newModule.__path__ = [nbname]
newModule.__loader__ = loader
newModule.__package__ = '' # no package
sys.modules[moduleName] = newModule
return newModule
def evaluate_notebook(nb_path, dest_path=None, skip_exceptions=False):
# Create evaluated version and save it to the dest path.
# Always use --pylab so figures appear inline
# perhaps this is questionable?
notebook = read(open(nb_path), 'json')
nb_runner = NotebookRunner(notebook, pylab=False)
try:
nb_runner.run_notebook(skip_exceptions=skip_exceptions)
except NotebookError as e:
print ''
print e
# Return the traceback, filtering out ANSI color codes.
# http://stackoverflow.com/questions/13506033/filtering-out-ansi-escape-sequences
return 'Notebook conversion failed with the following traceback: \n%s' % \
re.sub(r'\\033[\[\]]([0-9]{1,2}([;@][0-9]{0,2})*)*[mKP]?', '', str(e))
if dest_path is None:
dest_path = 'temp_evaluated.ipynb'
write(nb_runner.nb, open(dest_path, 'w'), 'json')
ret = nb_to_html(dest_path)
if dest_path is 'temp_evaluated.ipynb':
os.remove(dest_path)
return ret
def stripoutput(inputs, inplace=False):
"""
Strip output of notebooks.
Parameters
----------
inputs : list of string
Path to the notebooks to be processed.
inplace : bool
If this is `True`, outputs in the input files will be deleted.
Default is `False`.
"""
for inpath in inputs:
with file(inpath) as fp:
nb = nbformat.read(fp, 'ipynb')
clear_outputs(nb)
if inplace:
with file(inpath, 'w') as fp:
nbformat.write(nb, fp, 'ipynb')
else:
nbformat.write(nb, sys.stdout, 'ipynb')
def process_notebook_file(fname, action='clean', output_fname=None):
print("Performing '{}' on: {}".format(action, fname))
orig_wd = os.getcwd()
with io.open(fname, 'rb') as f:
nb = current.read(f, 'json')
if action == 'check':
os.chdir(os.path.dirname(fname))
run_notebook(nb)
remove_outputs(nb)
elif action == 'render':
os.chdir(os.path.dirname(fname))
run_notebook(nb)
else:
# Clean by default
remove_outputs(nb)
os.chdir(orig_wd)
if output_fname is None:
output_fname = fname
with io.open(output_fname, 'wb') as f:
nb = current.write(nb, f, 'json')
def walk_through_tutorials(only_published=True, selected_nb_re=None):
""" Generator for walking through the tutorials directory structure.
This returns tuples of (full_tutorial_path, tutorial_name) for
each tutorial. If published is set to True, this will only return
the published tutorials.
"""
nbre = re.compile(selected_nb_re) if selected_nb_re else None
current_directory = os.getcwd()
tutorials_base = os.path.join(current_directory, 'tutorials')
if not os.path.exists(tutorials_base):
err = ("Can't find 'tutorials' path! You must run this script from the"
" top-level astropy-tutorials directory.")
raise IOError(err)
# walk through each directory in tutorials/ to find all .ipynb file
for tutorial_name in os.listdir(tutorials_base):
tutorial_path = os.path.join(tutorials_base, tutorial_name)
if not os.path.isdir(tutorial_path):
# skip files / things that are not directories
continue
for filename in os.listdir(tutorial_path):
base, ext = os.path.splitext(filename)
if ext.lower() == ".ipynb" and "checkpoint" not in base:
full_filename = os.path.join(tutorial_path, filename)
notebook = read(open(full_filename), 'json')
is_published = notebook['metadata']['astropy-tutorials'].get(
'published', False)
if not is_published and only_published:
continue
if nbre and nbre.match(base) is None:
continue
yield full_filename, notebook
def __init__(self, nb_in, pylab=False):
self.km = KernelManager()
if pylab:
self.km.start_kernel(extra_arguments=['--pylab=inline'])
else:
self.km.start_kernel()
if platform.system() == 'Darwin':
# There is sometimes a race condition where the first
# execute command hits the kernel before it's ready.
# It appears to happen only on Darwin (Mac OS) and an
# easy (but clumsy) way to mitigate it is to sleep
# for a second.
sleep(1)
self.kc = self.km.client()
self.kc.start_channels()
self.shell = self.kc.shell_channel
self.iopub = self.kc.iopub_channel
logging.info('Reading notebook %s', nb_in)
self.nb = read(open(nb_in), 'json')
def load_module(self, fullname):
"""import a notebook as a module"""
path = find_notebook(fullname, self.path)
disp = "importing IPython notebook from "
disp += "<a href='./{}' target='_blank'>{}</a>".format(path, path[:-6])
display_html(disp, raw=True)
#print disp
# load the notebook object
with io.open(path, 'r', encoding='utf-8') as f:
nb = current.read(f, 'json')
# create the module and add it to sys.modules
# if name in sys.modules:
# return sys.modules[name]
mod = types.ModuleType(fullname)
mod.__file__ = path
mod.__loader__ = self
sys.modules[fullname] = mod
# extra work to ensure that magics that would affect the user_ns
# actually affect the notebook module's ns
save_user_ns = self.shell.user_ns
self.shell.user_ns = mod.__dict__
try:
for cell in nb.worksheets[0].cells:
if cell.cell_type == 'code' and cell.language == 'python':
# transform the input to executable Python
code = self.shell.input_transformer_manager.transform_cell(
cell.input)
# run the code in themodule
exec code in mod.__dict__
finally:
self.shell.user_ns = save_user_ns
return mod
def test_notebook(filename):
nb = read(open(filename), 'json')
expected = deepcopy(nb['worksheets'][0]['cells'])
# TODO: figure out how to set the log-level for IPython here
r = NotebookRunner(nb)
r.run_notebook()
actual = r.nb['worksheets'][0]['cells']
failed = []
for exp, act in zip(expected, actual):
if exp['cell_type'] == 'code':
#print("comparing outputs for ", exp['input'])
for eo, ao in zip(exp['outputs'], act['outputs']):
#print("\t", eo['text'], ao['text'], eo['text']==ao['text'])
eotext = [
'\n'.join(l for l in eo['text'].split('\n')
if _filter_exceptions(l))
]
aotext = [
'\n'.join(l for l in ao['text'].split('\n')
if _filter_exceptions(l))
]
if eotext != aotext:
#print("\tFAILED")
failed.append({
'prompt_number': exp['prompt_number'],
'input': exp['input'],
'expected_output': eotext,
'actual_output': aotext
})
return len(failed) == 0, failed, r.nb
def main():
parser = ArgumentParser(description=DESCRIP,
epilog=EPILOG,
formatter_class=RawDescriptionHelpFormatter)
parser.add_argument('searchpath',
type=str,
help='directory from which to search')
args = parser.parse_args()
for dirpath, dirnames, filenames in os.walk(args.searchpath):
dirnames[:] = [d for d in dirnames if not d.startswith('.')]
for fname in filenames:
if fname.startswith('.'):
continue
if not fname.endswith('.ipynb'):
continue
fullpath = os.path.join(dirpath, fname)
with io.open(fullpath, 'r') as f:
nb = current.read(f, 'json')
for cell in cellgen(nb, 'code'):
if hasattr(cell, 'prompt_number'):
del cell['prompt_number']
cell.outputs = []
with io.open(fullpath, 'w') as f:
current.write(nb, f, 'ipynb')
def from_filename(self, filename, resources=None, **kw):
"""
Convert a notebook from a notebook file.
Parameters
----------
filename : str
Full filename of the notebook file to open and convert.
"""
#Pull the metadata from the filesystem.
if resources is None:
resources = ResourcesDict()
if not 'metadata' in resources or resources['metadata'] == '':
resources['metadata'] = ResourcesDict()
basename = os.path.basename(filename)
notebook_name = basename[:basename.rfind('.')]
resources['metadata']['name'] = notebook_name
modified_date = datetime.datetime.fromtimestamp(os.path.getmtime(filename))
resources['metadata']['modified_date'] = modified_date.strftime(text.date_format)
with io.open(filename) as f:
return self.from_notebook_node(nbformat.read(f, 'json'), resources=resources,**kw)
def read_nb(fp):
return current.read(fp, 'json')
"""
Usage: python remove_output.py notebook.ipynb [ > without_output.ipynb ]
Modified from remove_output by Minrk
"""
import sys
import io
import os
from IPython.nbformat.current import read, write
def remove_outputs(nb):
"""remove the outputs from a notebook"""
for ws in nb.worksheets:
for cell in ws.cells:
if cell.cell_type == 'code':
cell.outputs = []
if __name__ == '__main__':
fname = sys.argv[1]
with io.open(fname, 'r') as f:
nb = read(f, 'json')
remove_outputs(nb)
base, ext = os.path.splitext(fname)
new_ipynb = "%s%s" % (base, ext) # "%s_removed%s" % (base, ext)
with io.open(new_ipynb, 'w', encoding='utf8') as f:
write(nb, f, 'json')
print("wrote %s" % new_ipynb)
def parse(root):
"""returns list of the NB in root
"""
dirs = sorted([dn for dn in os.listdir(root) if '00' < dn < '99'])
r = []
for sub_dir in dirs:
f = sorted([
os.path.join(root, sub_dir, fn)
for fn in os.listdir(os.path.join(root, sub_dir))
if ('00' < fn < '99' or fn.startswith('content'))
and fn.endswith('.ipynb')
])
r.extend(f)
return r
nb_filenames = ["Index.ipynb", "Index-labs1-6.ipynb"]
nb_filenames.extend(parse('./LABS'))
nb_filenames.extend(parse('.'))
for nb_filename in nb_filenames:
print('*' * 80)
print(nb_filename)
print('*' * 80)
notebook = read(open(nb_filename), 'json')
r = NotebookRunner(notebook)
r.run_notebook()
r.shutdown_kernel()
def strip_output(nb):
for ws in nb.worksheets:
for cell in ws.cells:
if hasattr(cell, "outputs"):
cell.outputs = []
if hasattr(cell, "prompt_number"):
del cell["prompt_number"]
if __name__ == "__main__":
from sys import stdin, stdout
from IPython.nbformat.current import read, write
nb = read(stdin, "ipynb")
strip_output(nb)
write(nb, stdout, "ipynb")
stdout.write("\n")
