def testRunNotebooks(self):
notebook_dir = path.join('tests', 'input')
for notebook_path in glob(path.join(notebook_dir, '*.ipynb')):
notebook_file = path.basename(notebook_path)
print(notebook_file)
expected_file = path.join('tests', 'expected', notebook_file)
notebook = ""
try:
# IPython 3
with open(notebook_path) as notebook_file:
notebook = read(notebook_file, 3)
except (TypeError, NBFormatError):
# IPython 2
with open(notebook_path) as notebook_file:
notebook = read(notebook_file, 'json')
runner = NotebookRunner(notebook, working_dir=notebook_dir)
runner.run_notebook(True)
expected = ""
try:
# IPython 3
with open(expected_file) as notebook_file:
expected = read(notebook_file, 3)
except (TypeError, NBFormatError):
# IPython 2
with open(expected_file) as notebook_file:
expected = read(notebook_file, 'json')
self.assert_notebooks_equal(expected, runner.nb)
def setUp(self):
self.notary = sign.NotebookNotary(
secret=b'secret',
profile_dir=get_ipython().profile_dir
)
with self.fopen(u'test3.ipynb', u'r') as f:
self.nb = read(f, as_version=4)
with self.fopen(u'test3.ipynb', u'r') as f:
self.nb3 = read(f, as_version=3)
def restore_checkpoint(self, checkpoint_id, path):
"""restore a file to a checkpointed state"""
path = path.strip("/")
self.log.info("restoring %s from checkpoint %s", path, checkpoint_id)
nb_path = self._get_os_path(path)
cp_path = self.get_checkpoint_path(checkpoint_id, path)
if not os.path.isfile(cp_path):
self.log.debug("checkpoint file does not exist: %s", cp_path)
raise web.HTTPError(404, u"checkpoint does not exist: %s@%s" % (path, checkpoint_id))
# ensure notebook is readable (never restore from an unreadable notebook)
if cp_path.endswith(".ipynb"):
with io.open(cp_path, "r", encoding="utf-8") as f:
nbformat.read(f, as_version=4)
self._copy(cp_path, nb_path)
self.log.debug("copying %s -> %s", cp_path, nb_path)
def test_save(self):
resp = self.api.read('a.ipynb', 'foo')
nbcontent = json.loads(resp.text)['content']
nb = from_dict(nbcontent)
nb.cells.append(new_markdown_cell(u'Created by test ³'))
nbmodel= {'name': 'a.ipynb', 'path':'foo', 'content': nb, 'type': 'notebook'}
resp = self.api.save('a.ipynb', path='foo', body=json.dumps(nbmodel))
nbfile = pjoin(self.notebook_dir.name, 'foo', 'a.ipynb')
with io.open(nbfile, 'r', encoding='utf-8') as f:
newnb = read(f, as_version=4)
self.assertEqual(newnb.cells[0].source,
u'Created by test ³')
nbcontent = self.api.read('a.ipynb', 'foo').json()['content']
newnb = from_dict(nbcontent)
self.assertEqual(newnb.cells[0].source,
u'Created by test ³')
# Save and rename
nbmodel= {'name': 'a2.ipynb', 'path':'foo/bar', 'content': nb, 'type': 'notebook'}
resp = self.api.save('a.ipynb', path='foo', body=json.dumps(nbmodel))
saved = resp.json()
self.assertEqual(saved['name'], 'a2.ipynb')
self.assertEqual(saved['path'], 'foo/bar')
assert os.path.isfile(pjoin(self.notebook_dir.name,'foo','bar','a2.ipynb'))
assert not os.path.isfile(pjoin(self.notebook_dir.name, 'foo', 'a.ipynb'))
with assert_http_error(404):
self.api.read('a.ipynb', 'foo')
def from_filename(self, filename, resources=None, **kw):
"""
Convert a notebook from a notebook file.
Parameters
----------
filename : str
Full filename of the notebook file to open and convert.
"""
# Pull the metadata from the filesystem.
if resources is None:
resources = ResourcesDict()
if not 'metadata' in resources or resources['metadata'] == '':
resources['metadata'] = ResourcesDict()
path, basename = os.path.split(filename)
notebook_name = basename[:basename.rfind('.')]
resources['metadata']['name'] = notebook_name
resources['metadata']['path'] = path
modified_date = datetime.datetime.fromtimestamp(os.path.getmtime(filename))
resources['metadata']['modified_date'] = modified_date.strftime(text.date_format)
with io.open(filename, encoding='utf-8') as f:
return self.from_notebook_node(nbformat.read(f, as_version=4), resources=resources, **kw)
def convert_to_v4(path):
nb = read(path, 3)
nb_new = convert(nb, 4)
nb_new["metadata"] ={
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.4.2"
}
}
validate(nb_new)
write(nb_new, path)
def process_notebook_file(fname, action='clean', output_fname=None):
print("Performing '{}' on: {}".format(action, fname))
orig_wd = os.getcwd()
# XXX: Ugly hack to preserve backward compat for now
cmd = ('ipython nbconvert --quiet --to notebook --nbformat 3'
' --output="%s" "%s"') % (fname, fname)
os.system(cmd)
with io.open(fname, 'r') as f:
nb = nbformat.read(f, nbformat.NO_CONVERT)
if action == 'check':
os.chdir(os.path.dirname(fname))
run_notebook(nb)
remove_outputs(nb)
remove_signature(nb)
elif action == 'render':
os.chdir(os.path.dirname(fname))
run_notebook(nb)
else:
# Clean by default
remove_outputs(nb)
remove_signature(nb)
os.chdir(orig_wd)
if output_fname is None:
output_fname = fname
with io.open(output_fname, 'w') as f:
nb = nbformat.write(nb, f, nbformat.NO_CONVERT)
def test_run_notebooks(self):
"""Runs a series of test notebooks and compares them to their actual output"""
current_dir = os.path.dirname(__file__)
input_files = glob.glob(os.path.join(current_dir, 'files', '*.ipynb'))
for filename in input_files:
with io.open(os.path.join(current_dir, 'files', filename)) as f:
input_nb = nbformat.read(f, 4)
res = self.build_resources()
preprocessor = self.build_preprocessor()
cleaned_input_nb = copy.deepcopy(input_nb)
for cell in cleaned_input_nb.cells:
if 'execution_count' in cell:
del cell['execution_count']
cell['outputs'] = []
output_nb, _ = preprocessor(cleaned_input_nb, res)
if os.path.basename(filename) == "Disable Stdin.ipynb":
# We need to special-case this particular notebook, because the
# traceback contains machine-specific stuff like where IPython
# is installed. It is sufficient here to just check that an error
# was thrown, and that it was a StdinNotImplementedError
self.assertEqual(len(output_nb['cells']), 1)
self.assertEqual(len(output_nb['cells'][0]['outputs']), 1)
output = output_nb['cells'][0]['outputs'][0]
self.assertEqual(output['output_type'], 'error')
self.assertEqual(output['ename'], 'StdinNotImplementedError')
self.assertEqual(output['evalue'], 'raw_input was called, but this frontend does not support input requests.')
else:
self.assert_notebooks_equal(output_nb, input_nb)
def read_notebook(notebook_name):
from IPython.nbformat import read
with open(notebook_name, "rb") as f:
book = read(f, 4)
#remove cells before heading cell
for i, cell in enumerate(book.cells):
if cell.cell_type == "markdown" and cell.source.startswith("#"):
break
del book.cells[:i]
code_count = sum(1 for i, cell in enumerate(book.cells) if cell.cell_type == "code")
notebook_name_path = u"/".join(notebook_name.split(u"\\")[-2:])
cell = {u'source': u'''> **SOURCE**
> 与本节内容对应的Notebook为：`{}`'''.format(notebook_name_path),
u'cell_type': u'markdown', u'metadata': {}}
if code_count >= 5:
book.cells.insert(1, cell)
return book
def clear_docs(root='docs/source'):
"""Clear the outputs of documentation notebooks."""
# cleanup ignored files
run('git clean -fdX {}'.format(root))
echo("Clearing outputs of notebooks in '{}'...".format(os.path.abspath(root)))
preprocessor = ClearOutputPreprocessor()
for dirpath, dirnames, filenames in os.walk(root):
is_submitted = _check_if_directory_in_path(dirpath, 'submitted')
for filename in sorted(filenames):
if os.path.splitext(filename)[1] == '.ipynb':
# read in the notebook
pth = os.path.join(dirpath, filename)
with open(pth, 'r') as fh:
orig_nb = read(fh, 4)
# copy the original notebook
new_nb = deepcopy(orig_nb)
# check outputs of all the cells
if not is_submitted:
new_nb = preprocessor.preprocess(new_nb, {})[0]
# clear metadata
new_nb.metadata = {}
# write the notebook back to disk
with open(pth, 'w') as fh:
write(new_nb, fh, 4)
if orig_nb != new_nb:
print("Cleared '{}'".format(pth))
def test_export_nbnode(self):
"""
Can a notebook be exported by a notebook node handle?
"""
with open(self._get_notebook(), "r") as f:
notebook = nbformat.read(f, 4)
(output, resources) = export_by_name("python", notebook)
assert len(output) > 0
def run_notebook(nbfile):
with open(nbfile) as f:
nb = nbformat.read(f, 4)
ip = get_ipython()
for cell in nb.cells:
if cell.cell_type != 'code':
continue
ip.run_cell(cell.source)
def notebook_content(self):
filepath = os.path.join(self.bundle_path, self.name)
with io.open(filepath, 'r', encoding='utf-8') as f:
try:
nb = nbformat.read(f, as_version=4)
except Exception as e:
nb = None
return nb
def test_read_write_path(self):
"""read() and write() take filesystem paths"""
path = os.path.join(self._get_files_path(), u'test4.ipynb')
nb = read(path, as_version=4)
with TemporaryDirectory() as td:
dest = os.path.join(td, 'echidna.ipynb')
write(nb, dest)
assert os.path.isfile(dest)
def test_future(self):
"""Test than a notebook from the future with extra keys passes validation"""
with self.fopen(u'test4plus.ipynb', u'r') as f:
nb = read(f, as_version=4)
with self.assertRaises(ValidationError):
validate(nb, version=4)
self.assertEqual(isvalid(nb, version=4), False)
self.assertEqual(isvalid(nb), True)
def iter_notebook_paragraphs(folder):
number, name = path.basename(folder).split("-")
for notebook in iter_notebooks(folder):
basename = path.basename(notebook)
with open(notebook, "rb") as f:
book = read(f, 4)
for i, cell in enumerate(book.cells):
text = cell.source
yield basename, i, text
def from_file(self, file_stream, resources=None, **kw):
"""
Convert a notebook from a notebook file.
Parameters
----------
file_stream : file-like object
Notebook file-like object to convert.
"""
return self.from_notebook_node(nbformat.read(file_stream, as_version=4), resources=resources, **kw)
def test_write_downgrade_2(self):
"""dowgrade a v3 notebook to v2"""
# Open a version 3 notebook.
with self.fopen(u'test3.ipynb', 'r') as f:
nb = read(f, as_version=3)
jsons = writes(nb, version=2)
nb2 = json.loads(jsons)
(major, minor) = get_version(nb2)
self.assertEqual(major, 2)
def _read_notebook(self, os_path, as_version=4):
"""Read a notebook from an os path."""
with self.open(os_path, 'r', encoding='utf-8') as f:
try:
return nbformat.read(f, as_version=as_version)
except Exception as e:
raise HTTPError(
400,
u"Unreadable Notebook: %s %r" % (os_path, e),
)
def test_invalid(self):
"""Test than an invalid notebook does not pass validation"""
# this notebook has a few different errors:
# - one cell is missing its source
# - invalid cell type
# - invalid output_type
with self.fopen(u'invalid.ipynb', u'r') as f:
nb = read(f, as_version=4)
with self.assertRaises(ValidationError):
validate(nb)
self.assertEqual(isvalid(nb), False)
def prep_for_html_conversion(filename):
added_appendix = False
with io.open(join('..', filename), 'r', encoding='utf-8') as f:
nb = nbformat.read(f, nbformat.NO_CONVERT)
remove_formatting(nb)
if not added_appendix and filename[0:8] == 'Appendix':
remove_links_add_appendix(nb)
added_appendix = True
else:
remove_links(nb)
nbformat.write(nb, join('html', filename))
def _read_notebook(self, os_path, as_version=4):
"""Read a notebook from an os path."""
with self.open(os_path, "r", encoding="utf-8") as f:
try:
if ftdetect(os_path) == "notebook":
return nbformat.read(f, as_version=as_version)
elif ftdetect(os_path) == "markdown":
nbjson = convert(os_path, informat="markdown", outformat="notebook")
return nbformat.reads(nbjson, as_version=as_version)
except Exception as e:
raise web.HTTPError(400, u"Unreadable Notebook: %s %r" % (os_path, e))
def compile_html_string(self, source, is_two_file=True):
"""Export notebooks as HTML strings."""
if flag is None:
req_missing(['ipython[notebook]>=2.0.0'], 'build this site (compile ipynb)')
HTMLExporter.default_template = 'basic'
c = Config(self.site.config['IPYNB_CONFIG'])
exportHtml = HTMLExporter(config=c)
with io.open(source, "r", encoding="utf8") as in_file:
nb_json = nbformat.read(in_file, current_nbformat)
(body, resources) = exportHtml.from_notebook_node(nb_json)
return body
def restore_checkpoint(self, checkpoint_id, path):
"""restore a file to a checkpointed state"""
path = path.strip('/')
self.log.info("restoring %s from checkpoint %s", path, checkpoint_id)
nb_path = self._get_os_path(path)
cp_path = self.get_checkpoint_path(checkpoint_id, path)
if not os.path.isfile(cp_path):
self.log.debug("checkpoint file does not exist: %s", cp_path)
raise web.HTTPError(404,
u'checkpoint does not exist: %s@%s' % (
path, checkpoint_id)
)
# ensure notebook is readable (never restore from an unreadable
# notebook)
if cp_path.endswith('.ipynb'):
with self.open(cp_path, 'r', encoding='utf-8') as f:
nbformat.read(f, as_version=4)
self.log.debug("copying %s -> %s", cp_path, nb_path)
with self.perm_to_403():
self._copy(cp_path, nb_path)
def test_read(self):
"""Can older notebooks be opened and automatically converted to the current
nbformat?"""
# Open a version 2 notebook.
with self.fopen(u'test2.ipynb', 'r') as f:
nb = read(f, as_version=current_nbformat)
# Check that the notebook was upgraded to the latest version automatically.
(major, minor) = get_version(nb)
self.assertEqual(major, current_nbformat)
def get_references(abs_nb_path, version):
'''
Retrieves the raw references to files, folders, and exclusions
from a notebook's markdown cell comments.
'''
notebook = nbformat.read(abs_nb_path, version)
referenced_list = []
for cell in notebook.cells:
references = _get_references(cell)
if references:
referenced_list = referenced_list + references
return referenced_list
def compile_html(self, source, dest, is_two_file=True):
if flag is None:
req_missing(['ipython[notebook]>=2.0.0'], 'build this site (compile ipynb)')
makedirs(os.path.dirname(dest))
HTMLExporter.default_template = 'basic'
c = Config(self.site.config['IPYNB_CONFIG'])
exportHtml = HTMLExporter(config=c)
with io.open(dest, "w+", encoding="utf8") as out_file:
with io.open(source, "r", encoding="utf8") as in_file:
nb_json = nbformat.read(in_file, current_nbformat)
(body, resources) = exportHtml.from_notebook_node(nb_json)
out_file.write(body)
def read_metadata(self, post, file_metadata_regexp=None, unslugify_titles=False, lang=None):
"""read metadata directly from ipynb file.
As ipynb file support arbitrary metadata as json, the metadata used by Nikola
will be assume to be in the 'nikola' subfield.
"""
source = post.source_path
with io.open(source, "r", encoding="utf8") as in_file:
nb_json = nbformat.read(in_file, current_nbformat)
# metadata should always exist, but we never know if
# the user crafted the ipynb by hand and did not add it.
return nb_json.get('metadata', {}).get('nikola', {})
def main(ipynb):
print("running %s" % ipynb)
with io.open(ipynb, encoding='utf8') as f:
nb = read(f, NO_CONVERT)
test_notebook(nb)
base, ext = os.path.splitext(ipynb)
exportHtml = HTMLExporter()
(body, resources) = exportHtml.from_notebook_node(nb)
outfile = ipynb + ".html"
open(outfile, 'w').write(encode_utf8(body))
print("wrote %s" % outfile)
def processOneIPynbFile(infile, outfile, imagedir):
print('notebook={} latex={} imageDir={}'.format(infile, outfile, imagedir))
pdffile = outfile.replace('.tex', '.pdf')
bibfile = outfile.replace('.tex', '.bib')
# nb = ipnbcurrent.read(io.open(infile, encoding='utf-8'), 'json')
# if len(nb.worksheets) > 1:
# raise NotImplementedError("Only one worksheet allowed")
nb = nbformat.read(io.open(infile, encoding='utf-8'), nbformat.NO_CONVERT)
output = '\n'
# for cell_index, cell in enumerate(nb.worksheets[0].cells):
if 'cells' not in nb:
print("This notebook is probably not in Notebook 3 format.")
if len(nb.worksheets) > 1:
raise NotImplementedError("Only one worksheet allowed in Notebook 2 format.")
nbcells = nb.worksheets[0].cells
else:
nbcells = nb.cells
for cell_index, cell in enumerate(nbcells):
# add a default header, if not otherwise supplied
if cell_index==0:
if not 'raw' in cell.cell_type:
output += standardHeader
# print('\n********','cell.cell_type ={} cell={}'.format(cell.cell_type,cell))
if cell.cell_type not in fnTableCell:
raise NotImplementedError("Unknown cell type: >{}<.".format(cell.cell_type))
rtnString = fnTableCell[cell.cell_type](cell, cell_index, imagedir, infile)
output += rtnString
if len(bibtexlist):
output += '\n\n\\bibliographystyle{IEEEtran}\n'
output += '\\bibliography{{{0}}}\n\n'.format(bibfile.replace('.bib', ''))
output += r'\end{document}'+'\n\n'
# #move the document class line to the start of the file
# output = movedocumentclass(output)
with io.open(outfile, 'w', encoding='utf-8') as f:
f.write(unicode(output))
if len(bibtexlist):
with io.open(bibfile, 'w', encoding='utf-8') as f:
for bib in bibtexlist:
f.write(unicode(bib))
def clear_notebooks(root):
"""Clear the outputs of documentation notebooks."""
# cleanup ignored files
run(['git', 'clean', '-fdX', root])
print("Clearing outputs of notebooks in '{}'...".format(
os.path.abspath(root)))
preprocessor = ClearOutputPreprocessor()
for dirpath, dirnames, filenames in os.walk(root):
is_submitted = _check_if_directory_in_path(dirpath, 'submitted')
for filename in sorted(filenames):
if os.path.splitext(filename)[1] == '.ipynb':
# read in the notebook
pth = os.path.join(dirpath, filename)
with open(pth, 'r') as fh:
orig_nb = read(fh, 4)
# copy the original notebook
new_nb = deepcopy(orig_nb)
# check outputs of all the cells
if not is_submitted:
new_nb = preprocessor.preprocess(new_nb, {})[0]
# clear metadata
new_nb.metadata = {}
# write the notebook back to disk
with open(pth, 'w') as fh:
write(new_nb, fh, 4)
if orig_nb != new_nb:
print("Cleared '{}'".format(pth))
def from_filename(self, filename, resources=None, **kw):
"""
Convert a notebook from a notebook file.
Parameters
----------
filename : str
Full filename of the notebook file to open and convert.
"""
# Pull the metadata from the filesystem.
if resources is None:
resources = ResourcesDict()
if not 'metadata' in resources or resources['metadata'] == '':
resources['metadata'] = ResourcesDict()
basename = os.path.basename(filename)
notebook_name = basename[:basename.rfind('.')]
resources['metadata']['name'] = notebook_name
modified_date = datetime.datetime.fromtimestamp(os.path.getmtime(filename))
resources['metadata']['modified_date'] = modified_date.strftime(text.date_format)
with io.open(filename, encoding='utf-8') as f:
return self.from_notebook_node(nbformat.read(f, as_version=4), resources=resources, **kw)
def test_R():
"""Check that the R notebook generated from Rmd looks the same
as the reference (without output cells).
"""
knitr = notedown.Knitr()
with open('r-examples/r-example.Rmd') as rmd:
knitted_markdown_file = knitr.knit(rmd)
reader = notedown.MarkdownReader(precode=r"%load_ext rpy2.ipython",
magic=True)
notebook = reader.read(knitted_markdown_file)
with open('r-examples/r-example.ipynb') as f:
reference_notebook = nbformat.read(f, as_version=4)
notedown.main.strip(notebook)
notedown.main.strip(reference_notebook)
writer = nbformat
nbjson = writer.writes(notebook)
reference_nbjson = writer.writes(reference_notebook)
nt.assert_multi_line_equal(nbjson, reference_nbjson)
def test_nb4(self):
"""Test that a v4 notebook passes validation"""
with self.fopen(u'test4.ipynb', u'r') as f:
nb = read(f, as_version=4)
validate(nb)
self.assertEqual(isvalid(nb), True)
def read_nb(fp):
# Have to load as version 4 or running notebook fails
return nbformat.read(fp, 4)
def load_notebook(nb_path):
"""Load notebook from file."""
with io.open(nb_path, "r", encoding="utf-8") as f:
nb = nbformat.read(f, as_version=4)
return nb
def test_nb2(self):
"""Test that a v2 notebook converted to current passes validation"""
with self.fopen(u'test2.ipynb', u'r') as f:
nb = read(f, as_version=4)
validate(nb)
self.assertEqual(isvalid(nb), True)
def load_notebook(nb_path):
with io.open(nb_path, 'r', encoding='utf-8') as f:
nb = nbformat.read(f, as_version=4)
return nb
def load_notebook(filename):
"""load a notebook object from a filename"""
if not os.path.exists(filename) and not filename.endswith(".ipynb"):
filename = filename + ".ipynb"
with io.open(filename) as f:
return nbf.read(f, as_version=4)
"""
model = self._base_model(path)
model['type'] = 'notebook'
if content:
nb = self._read_notebook(path, as_version=4)
self.mark_trusted_cells(nb, path)
model['content'] = nb
model['format'] = 'json'
self.validate_notebook_model(model)
return model
def _read_notebook(self, path, as_version=4):
"""Read a notebook file from gridfs."""
fs = self._get_fs_instance()
file_id = fs.get_last_version(path)._id
try:
filename = fs.get(file_id)
except Exception, e:
raise web.HTTPError(
400,
u"An error occured while reading the Notebook \
on GridFS: %s %r" % (path, e),
)
try:
return nbformat.read(filename, as_version=as_version)
except Exception as e:
raise web.HTTPError(
400,
u"Unreadable Notebook: %s %r" % (path, e),
)
import codecs
logging.basicConfig(format='%(levelname)s : %(message)s', level=logging.DEBUG)
logger = logging.getLogger(__name__)
parser = argparse.ArgumentParser()
parser.add_argument('resultsDir', help='directory containing experiment'
'result files.')
parser.add_argument('experimentName', help='name of the experiment')
parser.add_argument('outFile', help='file name to write IPython Notebook to '
'(*.ipynb)')
args = parser.parse_args()
resultsDir = args.resultsDir
experimentName = args.experimentName
outFile = args.outFile
with open('data/CPT_results_template.ipynb') as f:
nb = nbf.read(f, 4)
# cell 0 = title
nb['cells'][0]['source'] = nb['cells'][0]['source'].format(experimentName)
# cell 4 = set results dir
nb['cells'][4]['source'] = nb['cells'][4]['source'].format(resultsDir)
# save notebook
logger.info('writing notebook {}'.format(outFile))
with codecs.open(outFile, 'wb', encoding='utf8') as f:
nbf.write(nb, f)
def compress(filename, output_filename=None, img_width=2048, img_format='png'):
"""Compress images in IPython notebooks.
Parameters
----------
filename : string
Notebook to compress. Will take any notebook format.
output_filename : string
If you do not want to overwrite your existing notebook, supply an
filename for the new compressed notebook.
img_width : int
Which width images should be resized to.
img_format : string
Which compression to use on the images, valid options are
*png* and *jpeg* (**requires libjpeg**).
Returns
-------
int
Size of new notebook in bytes.
"""
orig_filesize = stat(filename).st_size
# compress images
nb = read(filename, as_version=4)
outputs = [cell.get('outputs', []) for cell in nb['cells']]
# omit empty outputs
outputs = [o for o in outputs if len(o)]
# flatten
outputs = [o for lines in outputs for o in lines]
for output in outputs:
data = output.get('data', {})
if not data:
continue
keys = data.keys()
for key in keys:
if 'image' in key:
string = ''.join(data[key])
bytes_img = b64decode(string)
io_img = BytesIO(bytes_img)
img = Image.open(io_img)
factor = float(img_width) / img.size[0]
new_size = img.size
if factor < 1:
# only resize large images
new_size = [int(s * factor + 0.5) for s in img.size]
img = img.resize(new_size)
out = BytesIO()
if img.mode == 'RGBA':
bgimg = Image.new('RGBA', new_size, (255, 255, 255, 255))
bgimg.paste(img, mask=img)
img = bgimg.convert('RGB')
img.save(out, img_format)
out.seek(0)
mime = 'image/' + img_format
del data[key]
data[mime] = b64encode(out.read()).decode('ascii')
# save notebook
if not output_filename:
output_filename = filename
try:
output_format = nb.metadata.orig_nbformat
except AttributeError:
output_format = 4
write(nb, output_filename, version=output_format)
# calculate bytes saved
bytes_saved = orig_filesize - stat(output_filename).st_size
if bytes_saved <= 0:
print('%s: warning: no compression - %s bytes gained' %
(filename, -bytes_saved))
return bytes_saved
def main():
log_format = '%(asctime)s %(levelname)s: %(message)s'
log_datefmt = '%m/%d/%Y %I:%M:%S %p'
parser = argparse.ArgumentParser()
parser.add_argument('--version',
'-v',
action='version',
version=runipy.__version__,
help='print version information')
parser.add_argument('input_file',
nargs='?',
help='.ipynb file to run (or stdin)')
parser.add_argument('output_file',
nargs='?',
help='.ipynb file to save cell output to')
parser.add_argument('--quiet',
'-q',
action='store_true',
help='don\'t print anything unless things go wrong')
parser.add_argument('--overwrite',
'-o',
action='store_true',
help='write notebook output back to original notebook')
parser.add_argument('--html',
nargs='?',
default=False,
help='output an HTML snapshot of the notebook')
parser.add_argument('--template',
nargs='?',
default=False,
help='template to use for HTML output')
parser.add_argument('--pylab',
action='store_true',
help='start notebook with pylab enabled')
parser.add_argument('--matplotlib',
action='store_true',
help='start notebook with matplotlib inlined')
parser.add_argument('--skip-exceptions',
'-s',
action='store_true',
help='if an exception occurs in a cell,' +
' continue running the subsequent cells')
parser.add_argument(
'--stdout',
action='store_true',
help='print notebook to stdout (or use - as output_file')
parser.add_argument(
'--stdin',
action='store_true',
help='read notebook from stdin (or use - as input_file)')
parser.add_argument(
'--no-chdir',
action='store_true',
help="do not change directory to notebook's at kernel startup")
parser.add_argument('--profile-dir',
help="set the profile location directly")
args = parser.parse_args()
if args.overwrite:
if args.output_file is not None:
print(
'Error: output_filename must not be provided if '
'--overwrite (-o) given',
file=stderr)
exit(1)
else:
args.output_file = args.input_file
if not args.quiet:
logging.basicConfig(level=logging.INFO,
format=log_format,
datefmt=log_datefmt)
working_dir = None
if args.input_file == '-' or args.stdin: # force stdin
payload = stdin
elif not args.input_file and stdin.isatty(): # no force, empty stdin
parser.print_help()
exit()
elif not args.input_file: # no file -> default stdin
payload = stdin
else: # must have specified normal input_file
payload = open(args.input_file)
working_dir = os.path.dirname(args.input_file)
if args.no_chdir:
working_dir = None
if args.profile_dir:
profile_dir = os.path.expanduser(args.profile_dir)
else:
profile_dir = None
logging.info('Reading notebook %s', payload.name)
try:
# Ipython 3
nb = read(payload, 3)
except (TypeError, NBFormatError):
# Ipython 2
nb = read(payload, 'json')
nb_runner = NotebookRunner(nb, args.pylab, args.matplotlib, profile_dir,
working_dir)
exit_status = 0
try:
nb_runner.run_notebook(skip_exceptions=args.skip_exceptions)
except NotebookError:
exit_status = 1
if args.output_file and args.output_file != '-':
logging.info('Saving to %s', args.output_file)
with open(args.output_file, 'w') as output_filehandle:
write(nb_runner.nb, output_filehandle, 'json')
if args.stdout or args.output_file == '-':
write(nb_runner.nb, stdout, 'json')
print()
if args.html is not False:
if args.html is None:
# if --html is given but no filename is provided,
# come up with a sane output name based on the
# input filename
if args.input_file.endswith('.ipynb'):
args.html = args.input_file[:-6] + '.html'
else:
args.html = args.input_file + '.html'
if args.template is False:
exporter = HTMLExporter()
else:
exporter = HTMLExporter(config=Config({
'HTMLExporter': {
'template_file': args.template,
'template_path': ['.', '/']
}
}))
logging.info('Saving HTML snapshot to %s' % args.html)
output, resources = exporter.from_notebook_node(nb_runner.nb)
codecs.open(args.html, 'w', encoding='utf-8').write(output)
nb_runner.shutdown_kernel()
if exit_status != 0:
logging.warning('Exiting with nonzero exit status')
exit(exit_status)
import IPython.nbformat as nbf
nb = nbf.read(open('test.py', 'r'), 'py')
nbf.write(nb, open('test.ipynb', 'w'), 'ipynb')
def run_nbcells(nbfile, cell_ids, run_other=False, fileformat=3):
"""
Runs specific cells from another notebook. In the specified notebook, a cell label can be defined
in a comment like this:
# cell_ID 1c
This give the cell the label 1c
:args:
nbfile (str): name of the file to open (typically an ipython notebook)
cell_ids (list): a list of cell id's (strings) to be looked for
run_other (bool): run cells if not all required cells have been found
fileformat(int): default=3, the file format of the notebook
OLD: fileformat (str): 'json' (default), 'ipynb' or 'py' specifies the format of the file.
:returns:
None
:raises:
NameError if not all cells could be found and run_other is set to False
"""
# how to change
with open(nbfile) as f:
nb = nbformat.read(f, fileformat)
codecells = []
found_ids = []
ws = nb['worksheets'][
0] # the "worksheet" (what's the difference between worksheet and notebook?)
for cell in ws.cells:
# cell has cell_type, input ("code"), output
# get cell ID for every cell
if cell.cell_type == 'code':
for line_ in cell.input.splitlines():
id_found = False
if line_.startswith('#') and 'cell_ID' in line_:
m = re.search('cell_ID\s*([0-9A-Za-z][0-9A-Za-z._]*)',
line_)
if m is not None:
if len(m.groups()) > 0:
cell_id = m.groups(0)[0].strip()
id_found = True
if cell_id in cell_ids:
codecells.append(cell.input)
found_ids.append(cell_id)
allfound = set(found_ids) == set(cell_ids)
if allfound:
ip = get_ipython()
for code in codecells:
ip.run_cell(code)
elif run_other:
print "The following labels have not been found:"
print ', '.join(set(cell_ids) - set(found_ids))
ip = get_ipython()
for code in codecells:
ip.run_cell(code)
else:
print "The following labels have not been found:"
print ', '.join(set(cell_ids) - set(found_ids))
raise NameError("some cells could not be identified")
#! /usr/bin/env python3 import sys from IPython.nbformat import read, write, NO_CONVERT nb = read(sys.stdin, NO_CONVERT) for cell in nb.cells: if cell.cell_type == 'code': cell.outputs = [] cell.execution_count = None write(nb, sys.stdout, NO_CONVERT)
