def test_very_long_cells(self):
"""
Torture test that long cells do not cause issues
"""
lorem_ipsum_text = textwrap.dedent("""\
Lorem ipsum dolor sit amet, consectetur adipiscing elit. Donec
dignissim, ipsum non facilisis tempus, dui felis tincidunt metus,
nec pulvinar neque odio eget risus. Nulla nisi lectus, cursus
suscipit interdum at, ultrices sit amet orci. Mauris facilisis
imperdiet elit, vitae scelerisque ipsum dignissim non. Integer
consequat malesuada neque sit amet pulvinar. Curabitur pretium
ut turpis eget aliquet. Maecenas sagittis lacus sed lectus
volutpat, eu adipiscing purus pulvinar. Maecenas consequat
luctus urna, eget cursus quam mollis a. Aliquam vitae ornare
erat, non hendrerit urna. Sed eu diam nec massa egestas pharetra
at nec tellus. Fusce feugiat lacus quis urna sollicitudin volutpat.
Quisque at sapien non nibh feugiat tempus ac ultricies purus.
""")
lorem_ipsum_text = lorem_ipsum_text.replace("\n", " ") + "\n\n"
large_lorem_ipsum_text = "".join([lorem_ipsum_text] * 3000)
notebook_name = "lorem_ipsum_long.ipynb"
nb = v4.new_notebook(
cells=[v4.new_markdown_cell(source=large_lorem_ipsum_text)])
with TemporaryDirectory() as td:
nbfile = os.path.join(td, notebook_name)
with open(nbfile, 'w') as f:
write(nb, f, 4)
(output, resources) = LatexExporter(
template_file='article').from_filename(nbfile)
assert len(output) > 0
def convert_to_v4(path):
nb = read(path, 3)
nb_new = convert(nb, 4)
nb_new["metadata"] ={
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.4.2"
}
}
validate(nb_new)
write(nb_new, path)
def test_very_long_cells(self):
"""
Torture test that long cells do not cause issues
"""
lorem_ipsum_text = textwrap.dedent("""\
Lorem ipsum dolor sit amet, consectetur adipiscing elit. Donec
dignissim, ipsum non facilisis tempus, dui felis tincidunt metus,
nec pulvinar neque odio eget risus. Nulla nisi lectus, cursus
suscipit interdum at, ultrices sit amet orci. Mauris facilisis
imperdiet elit, vitae scelerisque ipsum dignissim non. Integer
consequat malesuada neque sit amet pulvinar. Curabitur pretium
ut turpis eget aliquet. Maecenas sagittis lacus sed lectus
volutpat, eu adipiscing purus pulvinar. Maecenas consequat
luctus urna, eget cursus quam mollis a. Aliquam vitae ornare
erat, non hendrerit urna. Sed eu diam nec massa egestas pharetra
at nec tellus. Fusce feugiat lacus quis urna sollicitudin volutpat.
Quisque at sapien non nibh feugiat tempus ac ultricies purus.
""")
lorem_ipsum_text = lorem_ipsum_text.replace("\n"," ") + "\n\n"
large_lorem_ipsum_text = "".join([lorem_ipsum_text]*3000)
notebook_name = "lorem_ipsum_long.ipynb"
nb = v4.new_notebook(
cells=[
v4.new_markdown_cell(source=large_lorem_ipsum_text)
]
)
with TemporaryDirectory() as td:
nbfile = os.path.join(td, notebook_name)
with open(nbfile, 'w') as f:
write(nb, f, 4)
(output, resources) = LatexExporter(template_file='article').from_filename(nbfile)
assert len(output) > 0
def slides2pdf(fp,title=None,test=False):
# fp='group_meet_gened_b180605_v02_pdf_friendly.ipynb'
nb=nbformat.read(fp,as_version=nbformat.NO_CONVERT)
cells=[]
for cell in nb.cells:
if cell['metadata']!={}:
if 'slideshow' in cell['metadata']:
if cell['metadata']['slideshow']['slide_type']!="skip":
if cell['cell_type']=='markdown':
if 'source' in cell:
if '](' in cell['source']:
src=cell['source']
cell['source']=copyfilechangesrc(src)
if test:
if 'gene_exp_estradiol/sys.png' in src:
print(src)
print('\n')
print(copyfilechangesrc(src))
# break
cells.append(cell)
# break
# remo
cells=[cell for cell in cells if not (('thank you for your kind attention' in cell['source']) or ('#### Rohan Dandage' in cell['source']))]
# break
nb['cells']=cells
import datetime
if title is None:
fpout=fp+'.slides2.pdf'
else:
fpout='{}_{}.ipynb'.format(datetime.datetime.today().strftime('%y%m%d'),title.replace(' ','_'))
nbformat.write(nb,fpout)
system('jupyter nbconvert --to pdf --template ../../var/ipynb/pdf.tplx {}'.format(fpout))
logging.shutdown()
def setUp(self):
nbdir = self.notebook_dir.name
self.blob = os.urandom(100)
self.b64_blob = base64.encodestring(self.blob).decode("ascii")
for d in self.dirs + self.hidden_dirs:
d.replace("/", os.sep)
if not os.path.isdir(pjoin(nbdir, d)):
os.mkdir(pjoin(nbdir, d))
for d, name in self.dirs_nbs:
d = d.replace("/", os.sep)
# create a notebook
with io.open(pjoin(nbdir, d, "%s.ipynb" % name), "w", encoding="utf-8") as f:
nb = new_notebook()
write(nb, f, version=4)
# create a text file
with io.open(pjoin(nbdir, d, "%s.txt" % name), "w", encoding="utf-8") as f:
f.write(self._txt_for_name(name))
# create a binary file
with io.open(pjoin(nbdir, d, "%s.blob" % name), "wb") as f:
f.write(self._blob_for_name(name))
self.api = API(self.base_url())
def convert_to_v4(path):
nb = read(path, 3)
nb_new = convert(nb, 4)
nb_new["metadata"] = {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.4.2"
}
}
validate(nb_new)
write(nb_new, path)
def setUp(self):
nbdir = self.notebook_dir.name
self.blob = os.urandom(100)
self.b64_blob = base64.encodestring(self.blob).decode('ascii')
for d in (self.dirs + self.hidden_dirs):
d.replace('/', os.sep)
if not os.path.isdir(pjoin(nbdir, d)):
os.mkdir(pjoin(nbdir, d))
for d, name in self.dirs_nbs:
d = d.replace('/', os.sep)
# create a notebook
with io.open(pjoin(nbdir, d, '%s.ipynb' % name), 'w',
encoding='utf-8') as f:
nb = new_notebook()
write(nb, f, version=4)
# create a text file
with io.open(pjoin(nbdir, d, '%s.txt' % name), 'w',
encoding='utf-8') as f:
f.write(self._txt_for_name(name))
# create a binary file
with io.open(pjoin(nbdir, d, '%s.blob' % name), 'wb') as f:
f.write(self._blob_for_name(name))
self.api = API(self.base_url())
def setUp(self):
nbdir = self.notebook_dir.name
if not os.path.isdir(pjoin(nbdir, 'foo')):
os.mkdir(pjoin(nbdir, 'foo'))
nb = new_notebook()
nb.cells.append(new_markdown_cell(u'Created by test ³'))
cc1 = new_code_cell(source=u'print(2*6)')
cc1.outputs.append(new_output(output_type="stream", text=u'12'))
cc1.outputs.append(
new_output(
output_type="execute_result",
data={'image/png': png_green_pixel},
execution_count=1,
))
nb.cells.append(cc1)
with io.open(pjoin(nbdir, 'foo', 'testnb.ipynb'),
'w',
encoding='utf-8') as f:
write(nb, f, version=4)
self.nbconvert_api = NbconvertAPI(self.base_url())
def clear_docs(root='docs/source'):
"""Clear the outputs of documentation notebooks."""
# cleanup ignored files
run('git clean -fdX {}'.format(root))
echo("Clearing outputs of notebooks in '{}'...".format(os.path.abspath(root)))
preprocessor = ClearOutputPreprocessor()
for dirpath, dirnames, filenames in os.walk(root):
is_submitted = _check_if_directory_in_path(dirpath, 'submitted')
for filename in sorted(filenames):
if os.path.splitext(filename)[1] == '.ipynb':
# read in the notebook
pth = os.path.join(dirpath, filename)
with open(pth, 'r') as fh:
orig_nb = read(fh, 4)
# copy the original notebook
new_nb = deepcopy(orig_nb)
# check outputs of all the cells
if not is_submitted:
new_nb = preprocessor.preprocess(new_nb, {})[0]
# clear metadata
new_nb.metadata = {}
# write the notebook back to disk
with open(pth, 'w') as fh:
write(new_nb, fh, 4)
if orig_nb != new_nb:
print("Cleared '{}'".format(pth))
def store_in_file(self, content, force=False):
nb = nbformat.reader.reads(open(self.notebook).read())
if force or 'environment' not in nb['metadata']:
nb['metadata']['environment'] = content
nbformat.write(nb, self.notebook)
return True
else:
raise EnvironmentAlreadyInNotebook(self.notebook)
def store_in_file(self, content, force=False):
nb = nbformat.reader.reads(open(self.notebook).read())
if force or "environment" not in nb["metadata"]:
nb["metadata"]["environment"] = content
nbformat.write(nb, self.notebook)
return True
else:
raise EnvironmentAlreadyInNotebook(self.notebook)
def slides2pdf(fp, title=None, test=False):
# fp='group_meet_gened_b180605_v02_pdf_friendly.ipynb'
nb = nbformat.read(fp, as_version=nbformat.NO_CONVERT)
cells = []
for cell in nb.cells:
if cell['metadata'] != {}:
if 'slideshow' in cell['metadata']:
if cell['metadata']['slideshow']['slide_type'] != "skip":
if cell['cell_type'] == 'markdown':
if 'source' in cell:
if '](' in cell['source']:
src = cell['source']
cell['source'] = copyfilechangesrc(src)
if test:
if 'gene_exp_estradiol/sys.png' in src:
print(src)
print('\n')
print(copyfilechangesrc(src))
# break
if cell['cell_type'] == 'code':
if 'source' in cell:
# print(cell['source'].keys())
cell['source'] = ''
# del cell['source']
cells.append(cell)
# break
# remo
cells_ = []
for cell in cells:
if 'source' in cell:
if not (('thank you for your kind attention' in cell['source']) or
('#### Rohan Dandage' in cell['source'])):
cells_.append(cell)
else:
cells_.append(cell)
# break
nb['cells'] = cells_
# import datetime
# if title is None:
# fpout=fp+'.slides2.pdf'
# else:
# fpout='{}_{}.ipynb'.format(datetime.datetime.today().strftime('%y%m%d'),title.replace(' ','_'))
fpout = f"{fp}.cleaned.ipynb"
nbformat.write(nb, fpout)
system(
f"source activate pre; jupyter nbconvert {fpout} --to slides --reveal-prefix reveal.js-3.1.0"
)
cleanslideshtml(fp=f"{splitext(fpout)[0]}.slides.html")
system(f"source activate pre; jupyter nbconvert {fpout} --to html")
cleanslideshtml(fp=f"{splitext(fpout)[0]}.html")
system(
f"wkhtmltopdf {splitext(fpout)[0]}.html.cleaned.html {splitext(fpout)[0]}.html.cleaned.html.pdf"
)
logging.shutdown()
def _save_notebook(self, os_path, model, path=''):
"""save a notebook file"""
# Save the notebook file
nb = nbformat.from_dict(model['content'])
self.check_and_sign(nb, path)
with self.atomic_writing(os_path, encoding='utf-8') as f:
nbformat.write(nb, f, version=nbformat.NO_CONVERT)
def test_read_write_path(self):
"""read() and write() take filesystem paths"""
path = os.path.join(self._get_files_path(), u'test4.ipynb')
nb = read(path, as_version=4)
with TemporaryDirectory() as td:
dest = os.path.join(td, 'echidna.ipynb')
write(nb, dest)
assert os.path.isfile(dest)
def write_notebook(self, bundle_path, name, nb):
nb_path = os.path.join(bundle_path, name)
if "name" in nb["metadata"]:
nb["metadata"]["name"] = u""
try:
with io.open(nb_path, "w", encoding="utf-8") as f:
nbformat.write(nb, f, version=nbformat.NO_CONVERT)
except Exception as e:
raise Exception(u"Unexpected error while autosaving notebook: %s %s" % (nb_path, e))
def test_read_write_path(self):
"""read() and write() take filesystem paths"""
path = os.path.join(self._get_files_path(), u'test4.ipynb')
nb = read(path, as_version=4)
with TemporaryDirectory() as td:
dest = os.path.join(td, 'echidna.ipynb')
write(nb, dest)
assert os.path.isfile(dest)
def create_post(self, path, **kw):
"""Create a new post."""
if flag is None:
req_missing(['ipython[notebook]>=2.0.0'], 'build this site (compile ipynb)')
content = kw.pop('content', None)
onefile = kw.pop('onefile', False)
kernel = kw.pop('ipython_kernel', None)
# is_page is not needed to create the file
kw.pop('is_page', False)
metadata = {}
metadata.update(self.default_metadata)
metadata.update(kw)
makedirs(os.path.dirname(path))
if content.startswith("{"):
# imported .ipynb file, guaranteed to start with "{" because it’s JSON.
nb = nbformat.reads(content, current_nbformat)
else:
if IPython.version_info[0] >= 3:
nb = nbformat.v4.new_notebook()
nb["cells"] = [nbformat.v4.new_markdown_cell(content)]
else:
nb = nbformat.new_notebook()
nb["worksheets"] = [nbformat.new_worksheet(cells=[nbformat.new_text_cell('markdown', [content])])]
if kernelspec is not None:
if kernel is None:
kernel = self.default_kernel
self.logger.notice('No kernel specified, assuming "{0}".'.format(kernel))
IPYNB_KERNELS = {}
ksm = kernelspec.KernelSpecManager()
for k in ksm.find_kernel_specs():
IPYNB_KERNELS[k] = ksm.get_kernel_spec(k).to_dict()
IPYNB_KERNELS[k]['name'] = k
del IPYNB_KERNELS[k]['argv']
if kernel not in IPYNB_KERNELS:
self.logger.error('Unknown kernel "{0}". Maybe you mispelled it?'.format(kernel))
self.logger.info("Available kernels: {0}".format(", ".join(sorted(IPYNB_KERNELS))))
raise Exception('Unknown kernel "{0}"'.format(kernel))
nb["metadata"]["kernelspec"] = IPYNB_KERNELS[kernel]
else:
# Older IPython versions don’t need kernelspecs.
pass
if onefile:
nb["metadata"]["nikola"] = metadata
with io.open(path, "w+", encoding="utf8") as fd:
if IPython.version_info[0] >= 3:
nbformat.write(nb, fd, 4)
else:
nbformat.write(nb, fd, 'ipynb')
def _save_notebook(self, os_path, model, path=''):
"""save a notebook file"""
# Save the notebook file
nb = nbformat.from_dict(model['content'])
self.check_and_sign(nb, path)
with self.atomic_writing(os_path, encoding='utf-8') as f:
nbformat.write(nb, f, version=nbformat.NO_CONVERT)
def _save_notebook(self, os_path, nb):
"""Save a notebook to an os_path."""
with self.atomic_writing(os_path, encoding="utf-8") as f:
if ftdetect(os_path) == "notebook":
nbformat.write(nb, f, version=nbformat.NO_CONVERT)
elif ftdetect(os_path) == "markdown":
nbjson = nbformat.writes(nb, version=nbformat.NO_CONVERT)
markdown = convert(nbjson, informat="notebook", outformat="markdown")
f.write(markdown)
def run(self):
path = os.path.abspath(os.path.expanduser(self.args.file))
if not path.endswith(".ipynb"):
raise ValueError
with open(path, 'r') as f:
nb = nbformat.read(f, as_version=nbformat.NO_CONVERT)
self.strip_output(nb)
with open(path, 'w+') as f:
nbformat.write(nb, f)
def nbconcat(nbfiles):
with open(nbfiles[0]) as f:
nb = nbformat.read(f, as_version=v)
for nbf in nbfiles[1:]:
with open(nbf) as f:
nbb = nbformat.read(f, as_version=v)
nb.cells.extend(nbb.cells)
nbformat.write(nb, sys.stdout)
def convert(input_string, output_filename):
"""
Convert a preprocessed string object into notebook file
"""
# Read using v3
with StringIO(input_string) as fin:
nb = nbpy.read(fin)
# Write using the most recent version
with open(output_filename, 'w') as fout:
nbf.write(nb, fout, version=max(nbf.versions))
def _save_notebook(self, os_path, nb):
"""Save a notebook to an os_path."""
with self.atomic_writing(os_path, encoding='utf-8') as f:
if ftdetect(os_path) == 'notebook':
nbformat.write(nb, f, version=nbformat.NO_CONVERT)
elif ftdetect(os_path) == 'markdown':
nbjson = nbformat.writes(nb, version=nbformat.NO_CONVERT)
markdown = convert(nbjson,
informat='notebook',
outformat='markdown')
f.write(markdown)
def prep_for_html_conversion(filename):
added_appendix = False
with io.open(join('..', filename), 'r', encoding='utf-8') as f:
nb = nbformat.read(f, nbformat.NO_CONVERT)
remove_formatting(nb)
if not added_appendix and filename[0:8] == 'Appendix':
remove_links_add_appendix(nb)
added_appendix = True
else:
remove_links(nb)
nbformat.write(nb, join('html', filename))
def write_notebook(self, bundle_path, name, nb):
nb_path = os.path.join(bundle_path, name)
if 'name' in nb['metadata']:
nb['metadata']['name'] = u''
try:
with io.open(nb_path, 'w', encoding='utf-8') as f:
nbformat.write(nb, f, version=nbformat.NO_CONVERT)
except Exception as e:
raise Exception(
u'Unexpected error while autosaving notebook: %s %s' %
(nb_path, e))
def prep_for_html_conversion(filename):
added_appendix = False
with io.open(join('..', filename), 'r', encoding='utf-8') as f:
nb = nbformat.read(f, nbformat.NO_CONVERT)
remove_formatting(nb)
if not added_appendix and filename[0:8] == 'Appendix':
remove_links_add_appendix(nb)
added_appendix = True
else:
remove_links(nb)
nbformat.write(nb, join('html', filename))
def _save_notebook(self, os_path, model, name='', path=''):
"""save a notebook file"""
# Save the notebook file
nb = nbformat.from_dict(model['content'])
self.check_and_sign(nb, name, path)
if 'name' in nb['metadata']:
nb['metadata']['name'] = u''
with atomic_writing(os_path, encoding='utf-8') as f:
nbformat.write(nb, f, version=nbformat.NO_CONVERT)
def setUp(self):
nbdir = self.notebook_dir.name
d = os.path.join(nbdir, 'foo')
os.mkdir(d)
with io.open(os.path.join(d, 'bar.ipynb'), 'w', encoding='utf-8') as f:
nb = new_notebook()
write(nb, f, version=4)
with io.open(os.path.join(d, 'baz.txt'), 'w', encoding='utf-8') as f:
f.write(u'flamingo')
self.base_url()
def write_to_python_notebook():
cells = []
for index, text in enumerate(data):
cells.append(new_code_cell(
source=text,
execution_count=index,
))
nb0 = new_notebook(cells=cells, metadata={
'language': 'python',
})
f = codecs.open(python_notebook_name, encoding='utf-8', mode='w')
nbf.write(nb0, f, 4)
f.close()
def setUp(self):
nbdir = self.notebook_dir.name
d = os.path.join(nbdir, 'foo')
os.mkdir(d)
with io.open(os.path.join(d, 'bar.ipynb'), 'w', encoding='utf-8') as f:
nb = new_notebook()
write(nb, f, version=4)
with io.open(os.path.join(d, 'baz.txt'), 'w', encoding='utf-8') as f:
f.write(u'flamingo')
self.base_url()
def _save_notebook(self, path, nb):
self.log.debug('_save_notebook: {}'.format(locals()))
k = boto.s3.key.Key(self.bucket)
k.key = self._path_to_s3_key(path)
try:
with tempfile.NamedTemporaryFile() as f:
nbformat.write(nb, f, version=nbformat.NO_CONVERT)
f.seek(0)
k.set_contents_from_file(f)
except Exception as e:
raise web.HTTPError(400, u"Unexpected Error Writing Notebook: %s %s" % (path, e))
def main(paths):
failed = False
for path in paths:
with io.open(path, 'r', encoding='utf-8') as f:
nb = nbformat.read(f, 4)
try:
nbformat.validate(nb)
except ValidationError as e:
print(path, 'failed')
print(e)
else:
print(path)
with io.open(path, 'w', encoding='utf-8') as f:
nbformat.write(nb, f)
def _save_notebook(self, path, nb):
self.log.debug('_save_notebook: %s', locals())
k = boto.s3.key.Key(self.bucket)
k.key = self._path_to_s3_key(path)
try:
with tempfile.NamedTemporaryFile() as t, codecs.open(t.name, mode='w', encoding='utf-8') as f:
# write tempfile with utf-8 encoding
nbformat.write(nb, f, version=nbformat.NO_CONVERT)
# upload as bytes (t's fp didn't advance)
k.set_contents_from_file(t)
except Exception as e:
raise web.HTTPError(400, u"Unexpected Error Writing Notebook: %s %s" % (path, e))
def setUp(self):
nbdir = self.notebook_dir.name
try:
os.mkdir(pjoin(nbdir, "foo"))
except OSError as e:
# Deleting the folder in an earlier test may have failed
if e.errno != errno.EEXIST:
raise
with io.open(pjoin(nbdir, "foo", "nb1.ipynb"), "w", encoding="utf-8") as f:
nb = new_notebook()
write(nb, f, version=4)
self.sess_api = SessionAPI(self.base_url())
def setUp(self):
nbdir = self.notebook_dir.name
try:
os.mkdir(pjoin(nbdir, 'foo'))
except OSError as e:
# Deleting the folder in an earlier test may have failed
if e.errno != errno.EEXIST:
raise
with io.open(pjoin(nbdir, 'foo', 'nb1.ipynb'), 'w',
encoding='utf-8') as f:
nb = new_notebook()
write(nb, f, version=4)
self.sess_api = SessionAPI(self.base_url())
def process_notebook_file(fname, action='clean', output_fname=None):
print("Performing '{}' on: {}".format(action, fname))
orig_wd = os.getcwd()
# XXX: Ugly hack to preserve backward compat for now
cmd = ('ipython nbconvert --quiet --to notebook --nbformat 3'
' --output="%s" "%s"') % (fname, fname)
os.system(cmd)
with io.open(fname, 'r') as f:
nb = nbformat.read(f, nbformat.NO_CONVERT)
if action == 'check':
os.chdir(os.path.dirname(fname))
run_notebook(nb)
remove_outputs(nb)
remove_signature(nb)
elif action == 'render':
os.chdir(os.path.dirname(fname))
run_notebook(nb)
else:
# Clean by default
remove_outputs(nb)
remove_signature(nb)
os.chdir(orig_wd)
if output_fname is None:
output_fname = fname
with io.open(output_fname, 'w') as f:
nb = nbformat.write(nb, f, nbformat.NO_CONVERT)
def clean(in_handle, out_handle):
json_in = read(in_handle, NO_CONVERT)
if "signature" in json_in.metadata:
json_in.metadata.pop("signature")
for cell in json_in.cells:
if "outputs" in cell:
cell.outputs = []
if "execution_count" in cell:
cell.execution_count = None
if "prompt_number" in cell:
cell.pop("prompt_number")
if "metadata" in cell:
cell.metadata = {}
write(json_in, out_handle, NO_CONVERT)
def _save_notebook(self, path, nb):
self.log.debug('_save_notebook: %s', locals())
k = boto.s3.key.Key(self.bucket)
k.key = self._path_to_s3_key(path)
try:
with tempfile.NamedTemporaryFile() as t, codecs.open(
t.name, mode='w', encoding='utf-8') as f:
# write tempfile with utf-8 encoding
nbformat.write(nb, f, version=nbformat.NO_CONVERT)
# upload as bytes (t's fp didn't advance)
k.set_contents_from_file(t)
except Exception as e:
raise web.HTTPError(
400, u"Unexpected Error Writing Notebook: %s %s" % (path, e))
def process_notebook_file(fname, action='clean', output_fname=None):
print("Performing '{}' on: {}".format(action, fname))
orig_wd = os.getcwd()
# XXX: Ugly hack to preserve backward compat for now
cmd = ('ipython nbconvert --quiet --to notebook --nbformat 3'
' --output="%s" "%s"') % (fname, fname)
os.system(cmd)
with io.open(fname, 'r') as f:
nb = nbformat.read(f, nbformat.NO_CONVERT)
if action == 'check':
os.chdir(os.path.dirname(fname))
run_notebook(nb)
remove_outputs(nb)
remove_signature(nb)
elif action == 'render':
os.chdir(os.path.dirname(fname))
run_notebook(nb)
else:
# Clean by default
remove_outputs(nb)
remove_signature(nb)
os.chdir(orig_wd)
if output_fname is None:
output_fname = fname
with io.open(output_fname, 'w') as f:
nb = nbformat.write(nb, f, nbformat.NO_CONVERT)
def order_prompts(fname):
print(fname)
nb = read(fname, 4)
counter = 1
for cell in nb.cells:
if cell.cell_type != 'code':
continue
if cell.get('execution_count'):
cell.execution_count = counter
for output in cell.outputs:
if output.output_type == 'execute_result':
output.execution_count = counter
counter += 1
# remove trailing empty cell
while nb.cells and not nb.cells[-1].source:
nb.cells.pop(-1)
write(nb, fname, 4)
def test_contents_manager(self):
"make sure ContentsManager returns right files (ipynb, bin, txt)."
nbdir = self.notebook_dir.name
base = self.base_url()
nb = new_notebook(
cells=[
new_markdown_cell(u'Created by test ³'),
new_code_cell("print(2*6)", outputs=[
new_output("stream", text="12"),
])
]
)
with io.open(pjoin(nbdir, 'testnb.ipynb'), 'w',
encoding='utf-8') as f:
write(nb, f, version=4)
with io.open(pjoin(nbdir, 'test.bin'), 'wb') as f:
f.write(b'\xff' + os.urandom(5))
f.close()
with io.open(pjoin(nbdir, 'test.txt'), 'w') as f:
f.write(u'foobar')
f.close()
r = requests.get(url_path_join(base, 'files', 'testnb.ipynb'))
self.assertEqual(r.status_code, 200)
self.assertIn('print(2*6)', r.text)
json.loads(r.text)
r = requests.get(url_path_join(base, 'files', 'test.bin'))
self.assertEqual(r.status_code, 200)
self.assertEqual(r.headers['content-type'], 'application/octet-stream')
self.assertEqual(r.content[:1], b'\xff')
self.assertEqual(len(r.content), 6)
r = requests.get(url_path_join(base, 'files', 'test.txt'))
self.assertEqual(r.status_code, 200)
self.assertEqual(r.headers['content-type'], 'text/plain')
self.assertEqual(r.text, 'foobar')
def test_contents_manager(self):
"make sure ContentsManager returns right files (ipynb, bin, txt)."
nbdir = self.notebook_dir.name
base = self.base_url()
nb = new_notebook(
cells=[
new_markdown_cell(u'Created by test ³'),
new_code_cell("print(2*6)", outputs=[
new_output("stream", text="12"),
])
]
)
with io.open(pjoin(nbdir, 'testnb.ipynb'), 'w',
encoding='utf-8') as f:
write(nb, f, version=4)
with io.open(pjoin(nbdir, 'test.bin'), 'wb') as f:
f.write(b'\xff' + os.urandom(5))
f.close()
with io.open(pjoin(nbdir, 'test.txt'), 'w') as f:
f.write(u'foobar')
f.close()
r = requests.get(url_path_join(base, 'files', 'testnb.ipynb'))
self.assertEqual(r.status_code, 200)
self.assertIn('print(2*6)', r.text)
json.loads(r.text)
r = requests.get(url_path_join(base, 'files', 'test.bin'))
self.assertEqual(r.status_code, 200)
self.assertEqual(r.headers['content-type'], 'application/octet-stream')
self.assertEqual(r.content[:1], b'\xff')
self.assertEqual(len(r.content), 6)
r = requests.get(url_path_join(base, 'files', 'test.txt'))
self.assertEqual(r.status_code, 200)
self.assertEqual(r.headers['content-type'], 'text/plain')
self.assertEqual(r.text, 'foobar')
def setUp(self):
nbdir = self.notebook_dir.name
if not os.path.isdir(pjoin(nbdir, "foo")):
os.mkdir(pjoin(nbdir, "foo"))
nb = new_notebook()
nb.cells.append(new_markdown_cell(u"Created by test ³"))
cc1 = new_code_cell(source=u"print(2*6)")
cc1.outputs.append(new_output(output_type="stream", text=u"12"))
cc1.outputs.append(
new_output(output_type="execute_result", data={"image/png": png_green_pixel}, execution_count=1)
)
nb.cells.append(cc1)
with io.open(pjoin(nbdir, "foo", "testnb.ipynb"), "w", encoding="utf-8") as f:
write(nb, f, version=4)
self.nbconvert_api = NbconvertAPI(self.base_url())
def notebook(self, s):
"""Export and convert IPython notebooks.
This function can export the current IPython history to a notebook file.
For example, to export the history to "foo.ipynb" do "%notebook -e foo.ipynb".
To export the history to "foo.py" do "%notebook -e foo.py".
"""
args = magic_arguments.parse_argstring(self.notebook, s)
from IPython.nbformat import write, v4
args.filename = unquote_filename(args.filename)
if args.export:
cells = []
hist = list(self.shell.history_manager.get_range())
for session, execution_count, input in hist[:-1]:
cells.append(
v4.new_code_cell(execution_count=execution_count,
source=source))
nb = v4.new_notebook(cells=cells)
with io.open(args.filename, 'w', encoding='utf-8') as f:
write(nb, f, version=4)
def setUp(self):
nbdir = self.notebook_dir.name
if not os.path.isdir(pjoin(nbdir, 'foo')):
os.mkdir(pjoin(nbdir, 'foo'))
nb = new_notebook()
nb.cells.append(new_markdown_cell(u'Created by test ³'))
cc1 = new_code_cell(source=u'print(2*6)')
cc1.outputs.append(new_output(output_type="stream", text=u'12'))
cc1.outputs.append(new_output(output_type="execute_result",
data={'image/png': png_green_pixel},
execution_count=1,
))
nb.cells.append(cc1)
with io.open(pjoin(nbdir, 'foo', 'testnb.ipynb'), 'w',
encoding='utf-8') as f:
write(nb, f, version=4)
self.nbconvert_api = NbconvertAPI(self.base_url())
def notebook(self, s):
"""Export and convert IPython notebooks.
This function can export the current IPython history to a notebook file.
For example, to export the history to "foo.ipynb" do "%notebook -e foo.ipynb".
To export the history to "foo.py" do "%notebook -e foo.py".
"""
args = magic_arguments.parse_argstring(self.notebook, s)
from IPython.nbformat import write, v4
args.filename = unquote_filename(args.filename)
if args.export:
cells = []
hist = list(self.shell.history_manager.get_range())
for session, execution_count, input in hist[:-1]:
cells.append(v4.new_code_cell(
execution_count=execution_count,
source=source
))
nb = v4.new_notebook(cells=cells)
with io.open(args.filename, 'w', encoding='utf-8') as f:
write(nb, f, version=4)
def clear_notebooks(root):
"""Clear the outputs of documentation notebooks."""
# cleanup ignored files
run(['git', 'clean', '-fdX', root])
print("Clearing outputs of notebooks in '{}'...".format(
os.path.abspath(root)))
preprocessor = ClearOutputPreprocessor()
for dirpath, dirnames, filenames in os.walk(root):
is_submitted = _check_if_directory_in_path(dirpath, 'submitted')
for filename in sorted(filenames):
if os.path.splitext(filename)[1] == '.ipynb':
# read in the notebook
pth = os.path.join(dirpath, filename)
with open(pth, 'r') as fh:
orig_nb = read(fh, 4)
# copy the original notebook
new_nb = deepcopy(orig_nb)
# check outputs of all the cells
if not is_submitted:
new_nb = preprocessor.preprocess(new_nb, {})[0]
# clear metadata
new_nb.metadata = {}
# write the notebook back to disk
with open(pth, 'w') as fh:
write(new_nb, fh, 4)
if orig_nb != new_nb:
print("Cleared '{}'".format(pth))
import IPython.nbformat as nbf
nb = nbf.read(open('test.py', 'r'), 'py')
nbf.write(nb, open('test.ipynb', 'w'), 'ipynb')
def create_empty_notebook(self, path):
nb = v4.new_notebook()
with io.open(path, 'w', encoding='utf-8') as f:
write(nb, f, 4)
def create_empty_notebook(self, path):
nb = v4.new_notebook()
with io.open(path, 'w', encoding='utf-8') as f:
write(nb, f, 4)
import codecs
logging.basicConfig(format='%(levelname)s : %(message)s', level=logging.DEBUG)
logger = logging.getLogger(__name__)
parser = argparse.ArgumentParser()
parser.add_argument('resultsDir', help='directory containing experiment'
'result files.')
parser.add_argument('experimentName', help='name of the experiment')
parser.add_argument('outFile', help='file name to write IPython Notebook to '
'(*.ipynb)')
args = parser.parse_args()
resultsDir = args.resultsDir
experimentName = args.experimentName
outFile = args.outFile
with open('data/CPT_results_template.ipynb') as f:
nb = nbf.read(f, 4)
# cell 0 = title
nb['cells'][0]['source'] = nb['cells'][0]['source'].format(experimentName)
# cell 4 = set results dir
nb['cells'][4]['source'] = nb['cells'][4]['source'].format(resultsDir)
# save notebook
logger.info('writing notebook {}'.format(outFile))
with codecs.open(outFile, 'wb', encoding='utf8') as f:
nbf.write(nb, f)
def make_nb(self, api_path, nb):
"""Make a notebook file at a given api_path"""
os_path = self.to_os_path(api_path)
with io.open(os_path, 'w', encoding='utf-8') as f:
write(nb, f, version=4)
Usage: python remove_output.py notebook.ipynb [ > without_output.ipynb ]
Modified from remove_output by Minrk
Copied from https://gist.githubusercontent.com/damianavila/5305869
"""
import sys
import io
import os
from IPython.nbformat import read, write
def remove_outputs(nb):
"""remove the outputs from a notebook"""
for ws in nb.worksheets:
for cell in ws.cells:
if cell.cell_type == 'code':
cell.outputs = []
if __name__ == '__main__':
fname = sys.argv[1]
with io.open(fname, 'r') as f:
#nb = read(f, 'json')
nb = read(f, as_version=4)
remove_outputs(nb)
base, ext = os.path.splitext(fname)
new_ipynb = "%s_removed%s" % (base, ext)
with io.open(new_ipynb, 'w', encoding='utf8') as f:
#write(nb, f, 'json')
write(nb, f, version=nbformat.NO_CONVERT)
print "wrote %s" % new_ipynb
def main():
log_format = "%(asctime)s %(levelname)s: %(message)s"
log_datefmt = "%m/%d/%Y %I:%M:%S %p"
parser = argparse.ArgumentParser()
parser.add_argument(
"--version", "-v", action="version", version=runipy.__version__, help="print version information"
)
parser.add_argument("input_file", nargs="?", help=".ipynb file to run (or stdin)")
parser.add_argument("output_file", nargs="?", help=".ipynb file to save cell output to")
parser.add_argument("--quiet", "-q", action="store_true", help="don't print anything unless things go wrong")
parser.add_argument(
"--overwrite", "-o", action="store_true", help="write notebook output back to original notebook"
)
format_grp = parser.add_mutually_exclusive_group()
format_grp.add_argument("--html", nargs="?", default=False, help="output an HTML snapshot of the notebook")
format_grp.add_argument("--pdf", nargs="?", default=False, help="output a PDF snapshot of the notebook")
parser.add_argument("--template", nargs="?", default=False, help="template to use for HTML output")
parser.add_argument("--pylab", action="store_true", help="start notebook with pylab enabled")
parser.add_argument("--matplotlib", action="store_true", help="start notebook with matplotlib inlined")
parser.add_argument(
"--skip-exceptions",
"-s",
action="store_true",
help="if an exception occurs in a cell," + " continue running the subsequent cells",
)
parser.add_argument("--stdout", action="store_true", help="print notebook to stdout (or use - as output_file")
parser.add_argument("--stdin", action="store_true", help="read notebook from stdin (or use - as input_file)")
parser.add_argument(
"--no-chdir", action="store_true", help="do not change directory to notebook's at kernel startup"
)
parser.add_argument("--profile-dir", help="set the profile location directly")
args = parser.parse_args()
if args.overwrite:
if args.output_file is not None:
print("Error: output_filename must not be provided if " "--overwrite (-o) given", file=stderr)
exit(1)
else:
args.output_file = args.input_file
if not args.quiet:
logging.basicConfig(level=logging.INFO, format=log_format, datefmt=log_datefmt)
working_dir = None
payload_source = ""
payload = ""
if args.input_file == "-" or args.stdin: # force stdin
payload_source = stdin.name
payload = stdin.read()
elif not args.input_file and stdin.isatty(): # no force, empty stdin
parser.print_help()
exit()
elif not args.input_file: # no file -> default stdin
payload_source = stdin.name
payload = stdin.read()
else: # must have specified normal input_file
with open(args.input_file) as input_file:
payload_source = input_file.name
payload = input_file.read()
working_dir = os.path.dirname(args.input_file)
if args.no_chdir:
working_dir = None
if args.profile_dir:
profile_dir = os.path.expanduser(args.profile_dir)
else:
profile_dir = None
logging.info("Reading notebook %s", payload_source)
try:
# Ipython 3
nb = reads(payload, 3)
except (TypeError, NBFormatError):
# Ipython 2
nb = reads(payload, "json")
nb_runner = NotebookRunner(nb, args.pylab, args.matplotlib, profile_dir, working_dir)
exit_status = 0
try:
nb_runner.run_notebook(skip_exceptions=args.skip_exceptions)
except NotebookError:
exit_status = 1
if args.output_file and args.output_file != "-":
logging.info("Saving to %s", args.output_file)
with open(args.output_file, "w") as output_filehandle:
try:
# Ipython 3
write(nb_runner.nb, output_filehandle, 3)
except (TypeError, NBFormatError):
# Ipython 2
write(nb_runner.nb, output_filehandle, "json")
if args.stdout or args.output_file == "-":
try:
# Ipython 3
write(nb_runner.nb, stdout, 3)
except (TypeError, NBFormatError):
# Ipython 2
write(nb_runner.nb, stdout, "json")
print()
if args.html is not False:
if args.html is None:
# if --html is given but no filename is provided,
# come up with a sane output name based on the
# input filename
if args.input_file.endswith(".ipynb"):
args.html = args.input_file[:-6] + ".html"
else:
args.html = args.input_file + ".html"
if args.template is False:
exporter = HTMLExporter()
else:
exporter = HTMLExporter(
config=Config({"HTMLExporter": {"template_file": args.template, "template_path": [".", "/"]}})
)
logging.info("Saving HTML snapshot to %s" % args.html)
output, resources = exporter.from_notebook_node(convert(nb_runner.nb, current_nbformat))
codecs.open(args.html, "w", encoding="utf-8").write(output)
elif args.pdf is not False:
if args.pdf is None:
# if --html is given but no filename is provided,
# come up with a sane output name based on the
# input filename
if args.input_file.endswith(".ipynb"):
args.pdf = args.input_file[:-6] + ".pdf"
else:
args.pdf = args.input_file + ".pdf"
if args.template is False:
exporter = PDFExporter()
else:
exporter = PDFExporter(
config=Config({"PDFExporter": {"template_file": args.template, "template_path": [".", "/"]}})
)
logging.info("Saving PDF snapshot to %s" % args.pdf)
output, resources = exporter.from_notebook_node(convert(nb_runner.nb, current_nbformat))
writer = FilesWriter()
writer.build_directory = os.path.dirname(args.pdf)
writer.write(output, resources, os.path.splitext(os.path.basename(args.pdf))[0])
nb_runner.shutdown_kernel()
if exit_status != 0:
logging.warning("Exiting with nonzero exit status")
exit(exit_status)
def make_nb(self, api_path, nb):
"""Make a notebook file at a given api_path"""
os_path = self.to_os_path(api_path)
with io.open(os_path, 'w', encoding='utf-8') as f:
write(nb, f, version=4)
def compress(filename, output_filename=None, img_width=2048, img_format='png'):
"""Compress images in IPython notebooks.
Parameters
----------
filename : string
Notebook to compress. Will take any notebook format.
output_filename : string
If you do not want to overwrite your existing notebook, supply an
filename for the new compressed notebook.
img_width : int
Which width images should be resized to.
img_format : string
Which compression to use on the images, valid options are
*png* and *jpeg* (**requires libjpeg**).
Returns
-------
int
Size of new notebook in bytes.
"""
orig_filesize = stat(filename).st_size
# compress images
nb = read(filename, as_version=4)
outputs = [cell.get('outputs', []) for cell in nb['cells']]
# omit empty outputs
outputs = [o for o in outputs if len(o)]
# flatten
outputs = [o for lines in outputs for o in lines]
for output in outputs:
data = output.get('data', {})
if not data:
continue
keys = data.keys()
for key in keys:
if 'image' in key:
string = ''.join(data[key])
bytes_img = b64decode(string)
io_img = BytesIO(bytes_img)
img = Image.open(io_img)
factor = float(img_width) / img.size[0]
new_size = img.size
if factor < 1:
# only resize large images
new_size = [int(s * factor + 0.5) for s in img.size]
img = img.resize(new_size)
out = BytesIO()
if img.mode == 'RGBA':
bgimg = Image.new('RGBA', new_size, (255, 255, 255, 255))
bgimg.paste(img, mask=img)
img = bgimg.convert('RGB')
img.save(out, img_format)
out.seek(0)
mime = 'image/' + img_format
del data[key]
data[mime] = b64encode(out.read()).decode('ascii')
# save notebook
if not output_filename:
output_filename = filename
try:
output_format = nb.metadata.orig_nbformat
except AttributeError:
output_format = 4
write(nb, output_filename, version=output_format)
# calculate bytes saved
bytes_saved = orig_filesize - stat(output_filename).st_size
if bytes_saved <= 0:
print('%s: warning: no compression - %s bytes gained' %
(filename, -bytes_saved))
return bytes_saved
def main():
log_format = '%(asctime)s %(levelname)s: %(message)s'
log_datefmt = '%m/%d/%Y %I:%M:%S %p'
parser = argparse.ArgumentParser()
parser.add_argument('--version',
'-v',
action='version',
version=runipy.__version__,
help='print version information')
parser.add_argument('input_file',
nargs='?',
help='.ipynb file to run (or stdin)')
parser.add_argument('output_file',
nargs='?',
help='.ipynb file to save cell output to')
parser.add_argument('--quiet',
'-q',
action='store_true',
help='don\'t print anything unless things go wrong')
parser.add_argument('--overwrite',
'-o',
action='store_true',
help='write notebook output back to original notebook')
parser.add_argument('--html',
nargs='?',
default=False,
help='output an HTML snapshot of the notebook')
parser.add_argument('--template',
nargs='?',
default=False,
help='template to use for HTML output')
parser.add_argument('--pylab',
action='store_true',
help='start notebook with pylab enabled')
parser.add_argument('--matplotlib',
action='store_true',
help='start notebook with matplotlib inlined')
parser.add_argument('--skip-exceptions',
'-s',
action='store_true',
help='if an exception occurs in a cell,' +
' continue running the subsequent cells')
parser.add_argument(
'--stdout',
action='store_true',
help='print notebook to stdout (or use - as output_file')
parser.add_argument(
'--stdin',
action='store_true',
help='read notebook from stdin (or use - as input_file)')
parser.add_argument(
'--no-chdir',
action='store_true',
help="do not change directory to notebook's at kernel startup")
parser.add_argument('--profile-dir',
help="set the profile location directly")
args = parser.parse_args()
if args.overwrite:
if args.output_file is not None:
print(
'Error: output_filename must not be provided if '
'--overwrite (-o) given',
file=stderr)
exit(1)
else:
args.output_file = args.input_file
if not args.quiet:
logging.basicConfig(level=logging.INFO,
format=log_format,
datefmt=log_datefmt)
working_dir = None
payload_source = ""
payload = ""
if args.input_file == '-' or args.stdin: # force stdin
payload_source = stdin.name
payload = stdin.read()
elif not args.input_file and stdin.isatty(): # no force, empty stdin
parser.print_help()
exit()
elif not args.input_file: # no file -> default stdin
payload_source = stdin.name
payload = stdin.read()
else: # must have specified normal input_file
with open(args.input_file) as input_file:
payload_source = input_file.name
payload = input_file.read()
working_dir = os.path.dirname(args.input_file)
if args.no_chdir:
working_dir = None
if args.profile_dir:
profile_dir = os.path.expanduser(args.profile_dir)
else:
profile_dir = None
logging.info('Reading notebook %s', payload_source)
try:
# Ipython 3
nb = reads(payload, 3)
except (TypeError, NBFormatError):
# Ipython 2
nb = reads(payload, 'json')
nb_runner = NotebookRunner(nb, args.pylab, args.matplotlib, profile_dir,
working_dir)
exit_status = 0
try:
nb_runner.run_notebook(skip_exceptions=args.skip_exceptions)
except NotebookError:
exit_status = 1
if args.output_file and args.output_file != '-':
logging.info('Saving to %s', args.output_file)
with open(args.output_file, 'w') as output_filehandle:
try:
# Ipython 3
write(nb_runner.nb, output_filehandle, 3)
except (TypeError, NBFormatError):
# Ipython 2
write(nb_runner.nb, output_filehandle, 'json')
if args.stdout or args.output_file == '-':
try:
# Ipython 3
write(nb_runner.nb, stdout, 3)
except (TypeError, NBFormatError):
# Ipython 2
write(nb_runner.nb, stdout, 'json')
print()
if args.html is not False:
if args.html is None:
# if --html is given but no filename is provided,
# come up with a sane output name based on the
# input filename
if args.input_file.endswith('.ipynb'):
args.html = args.input_file[:-6] + '.html'
else:
args.html = args.input_file + '.html'
if args.template is False:
exporter = HTMLExporter()
else:
exporter = HTMLExporter(config=Config({
'HTMLExporter': {
'template_file': args.template,
'template_path': ['.', '/']
}
}))
logging.info('Saving HTML snapshot to %s' % args.html)
output, resources = exporter.from_notebook_node(
convert(nb_runner.nb, current_nbformat))
codecs.open(args.html, 'w', encoding='utf-8').write(output)
nb_runner.shutdown_kernel()
if exit_status != 0:
logging.warning('Exiting with nonzero exit status')
exit(exit_status)
def _save_notebook(self, os_path, nb):
"""Save a notebook to an os_path."""
with self.atomic_writing(os_path, encoding='utf-8') as f:
nbformat.write(nb, f, version=nbformat.NO_CONVERT)
'key': 'value'
},
'application/javascript': u'var i=0;'
},
),new_output(
output_type=u'error',
ename=u'NameError',
evalue=u'NameError was here',
traceback=[u'frame 0', u'frame 1', u'frame 2']
),new_output(
output_type=u'stream',
text='foo\rbar\r\n'
),new_output(
output_type=u'stream',
name='stderr',
text='\rfoo\rbar\n'
)]
))
nb0 = new_notebook(cells=cells,
metadata={
'language': 'python',
}
)
import IPython.nbformat as nbf
import codecs
f = codecs.open('test.ipynb', encoding='utf-8', mode='w')
nbf.write(nb0, f, 4)
f.close()
def _save_notebook(self, os_path, nb):
"""Save a notebook to an os_path."""
with self.atomic_writing(os_path, encoding='utf-8') as f:
nbformat.write(nb, f, version=nbformat.NO_CONVERT)