def _preprocess(self, nb, resources):
"""
Preprocess the notebook before passing it into the Jinja engine.
To preprocess the notebook is to successively apply all the
enabled preprocessors. Output from each preprocessor is passed
along to the next one.
Parameters
----------
nb : notebook node
notebook that is being exported.
resources : a dict of additional resources that
can be accessed read/write by preprocessors
"""
# Do a copy.deepcopy first,
# we are never safe enough with what the preprocessors could do.
nbc = copy.deepcopy(nb)
resc = copy.deepcopy(resources)
# Run each preprocessor on the notebook. Carry the output along
# to each preprocessor
for preprocessor in self._preprocessors:
nbc, resc = preprocessor(nbc, resc)
try:
nbformat.validate(nbc, relax_add_props=True)
except nbformat.ValidationError:
self.log.error('Notebook is invalid after preprocessor %s',
preprocessor)
raise
return nbc, resc
def test_hg_mergedriver(hg_repo, filespath, reset_log):
# enable diff/merge drivers
write_local_hg_config(hg_repo)
# run merge with no conflicts
out = get_output('hg merge remote-no-conflict', err=True)
assert 'nbmergeapp' in out
with open('merge-no-conflict.ipynb') as f:
merged = f.read()
with open(os.path.join(filespath, 'multilevel-test-merged.ipynb')) as f:
expected = f.read()
# verify merge success
assert merged == expected
# reset
call('hg update --clean local')
# run merge with conflicts
with pytest.raises(CalledProcessError):
call('hg merge remote-conflict')
status = get_output('hg status')
assert 'merge-conflict.ipynb' in status
out = get_output('hg nbdiff', returncode=1)
assert 'nbdiff' in out
# verify that the conflicted result is a valid notebook
nb = nbformat.read('merge-conflict.ipynb', as_version=4)
nbformat.validate(nb)
def test_mergedriver(git_repo, filespath):
# enable diff/merge drivers
nbdime.gitdiffdriver.main(['config', '--enable'])
nbdime.gitmergedriver.main(['config', '--enable'])
# run merge with no conflicts
out = get_output('git merge remote-no-conflict', err=True)
assert 'nbmergeapp' in out
with open('merge-no-conflict.ipynb') as f:
merged = f.read()
with open(os.path.join(filespath, 'multilevel-test-merged.ipynb')) as f:
expected = f.read()
# verify merge success
assert merged == expected
# reset
call('git reset local --hard')
# run merge with conflicts
with pytest.raises(CalledProcessError):
call('git merge remote-conflict')
status = get_output('git status')
assert 'merge-conflict.ipynb' in status
out = get_output('git diff HEAD')
assert 'nbdiff' in out
# verify that the conflicted result is a valid notebook
nb = nbformat.read('merge-conflict.ipynb', as_version=4)
nbformat.validate(nb)
def test_diff_api_symlink(git_repo2, server_extension_app, needs_symlink):
root = server_extension_app['path']
subdir = pjoin(root, 'has space', 'subdir')
os.makedirs(subdir)
symlink = pjoin(subdir, 'link')
with pushd(subdir):
call(['git', 'init'])
with open('f', 'w') as f:
f.write('stuff')
call(['git', 'add', 'f'])
call(['git', 'commit', '-m', 'initial commit'])
os.symlink(git_repo2, symlink)
local_path = os.path.relpath(symlink, server_extension_app['path'])
url = 'http://127.0.0.1:%i/nbdime/api/diff' % server_extension_app['port']
r = requests.post(url,
headers=auth_header,
data=json.dumps({
'base':
'git:' + pjoin(local_path, 'diff.ipynb'),
}))
r.raise_for_status()
data = r.json()
nbformat.validate(data['base'])
assert data['diff']
assert len(data.keys()) == 2
def test_hg_mergedriver(hg_repo, filespath, reset_log):
# enable diff/merge drivers
write_local_hg_config(hg_repo)
# run merge with no conflicts
out = get_output('hg merge remote-no-conflict', err=True)
assert 'nbmergeapp' in out
with open('merge-no-conflict.ipynb') as f:
merged = f.read()
with open(os.path.join(filespath, 'multilevel-test-merged.ipynb')) as f:
expected = f.read()
# verify merge success
assert merged == expected
# reset
call('hg update --clean local')
# run merge with conflicts
with pytest.raises(CalledProcessError):
call('hg merge remote-conflict')
status = get_output('hg status')
assert 'merge-conflict.ipynb' in status
out = get_output('hg nbdiff', returncode=1)
assert 'nbdiff' in out
# verify that the conflicted result is a valid notebook
nb = nbformat.read('merge-conflict.ipynb', as_version=4)
nbformat.validate(nb)
def _validate_preprocessor(self, nbc, preprocessor):
try:
nbformat.validate(nbc, relax_add_props=True)
except nbformat.ValidationError:
self.log.error("Notebook is invalid after preprocessor %s",
preprocessor)
raise
def main_patch(args):
base_filename = args.base
patch_filename = args.patch
output_filename = args.output
for fn in (base_filename, patch_filename):
if not os.path.exists(fn) and fn != EXPLICIT_MISSING_FILE:
print("Missing file {}".format(fn))
return 1
before = read_notebook(base_filename, on_null='empty')
with io.open(patch_filename, encoding="utf8") as patch_file:
diff = json.load(patch_file)
diff = to_diffentry_dicts(diff)
after = patch_notebook(before, diff)
if output_filename:
nbformat.write(after, output_filename)
else:
try:
nbformat.validate(after, version=4)
except nbformat.ValidationError:
print("Patch result is not a valid notebook, printing as JSON:")
json.dump(after, sys.stdout)
else:
pretty_print_notebook(after)
return 0
def write(self, ast, context=None):
self.execution_count = 1
self._md = MarkdownPlugin()
# Add code cells in the AST.
ccw = CodeCellWrapper()
ast = ccw.wrap(ast)
# Find the directory containing the notebook file.
doc_path = (context or {}).get('path', None)
if doc_path:
self._dir_path = op.dirname(op.realpath(doc_path))
else:
logger.warn("No input path, unable to resolve the image relative paths.")
self._dir_path = None
# Create the notebook.
# new_output, new_code_cell, new_markdown_cell
# TODO: kernelspect
nb = new_notebook()
# Go through all top-level blocks.
for index, node in enumerate(ast.children):
# Determine the block type.
if node.name == 'CodeCell':
node_type = 'code'
else:
node_type = 'markdown'
# Create the notebook cell.
cell = getattr(self, 'new_{}_cell'.format(node_type))(node, index)
# Add it to the notebook.
nb.cells.append(cell)
nbformat.validate(nb)
return nb
def test_mergedriver(git_repo):
p = filespath()
# enable diff/merge drivers
nbdime.gitdiffdriver.main(['config', '--enable'])
nbdime.gitmergedriver.main(['config', '--enable'])
# run merge with no conflicts
out = get_output('git merge remote-no-conflict', err=True)
assert 'nbmergeapp' in out
with open('merge-no-conflict.ipynb') as f:
merged = f.read()
with open(os.path.join(p, 'multilevel-test-merged.ipynb')) as f:
expected = f.read()
# verify merge success
assert merged == expected
# reset
call('git reset local --hard')
# run merge with conflicts
with pytest.raises(CalledProcessError):
call('git merge remote-conflict')
status = get_output('git status')
assert 'merge-conflict.ipynb' in status
out = get_output('git diff HEAD')
assert 'nbdiff' in out
# verify that the conflicted result is a valid notebook
nb = nbformat.read('merge-conflict.ipynb', as_version=4)
nbformat.validate(nb)
def _warn_if_invalid(nb, version):
"""Log validation errors, if there are any."""
from nbformat import validate, ValidationError
try:
validate(nb, version=version)
except ValidationError as e:
get_logger().error("Notebook JSON is not valid v%i: %s", version, e)
def format_nb(notebook_filename, dry_run=False):
print('Formatting {}...'.format(notebook_filename), end='')
with open(notebook_filename, 'r') as f:
notebook = nbformat.read(f, as_version=nbformat.NO_CONVERT)
nbformat.validate(notebook)
changed = False
for cell in notebook.cells:
if cell['cell_type'] != 'code':
continue
src = cell['source']
lines = src.split('\n')
if len(lines) <= 0 or '# noqa' in lines[0]:
continue
formatted_src, did_change = FormatCode(src, style_config=style_file)
if formatted_src.endswith('\n'):
formatted_src = formatted_src[:-1] # remove last newline
did_change = True
if did_change:
cell['source'] = formatted_src
changed = True
if changed:
if dry_run:
print(' (reformatted)')
else:
with open(notebook_filename, 'w') as f:
nbformat.write(notebook, f, version=nbformat.NO_CONVERT)
print()
else:
print()
def nb_grade(path):
if os.path.isdir(path):
paths = [
os.path.join(path, x) for x in os.listdir(path)
if x.endswith('.ipynb')
]
elif os.path.exists(path):
paths = [path]
else:
print("Error: input path '{}' doesn't exist.".format(path))
for path in paths:
print("Converting '{}'...".format(path))
notebook = nb.read(path, nb.NO_CONVERT)
# Insert a grade cell before every exercise cell.
i = 0
cells = notebook['cells']
while i < len(cells):
if cells[i]['source'].startswith(EXER_HEADER):
# Exercise cell: insert a new grade cell before.
cells.insert(i, GRADE_CELL)
i += 2
elif cells[i]['source'] == GRADE_HEADER:
# Grade cell: skip past the subsequent exercise cell.
i += 2
else:
i += 1
nb.validate(notebook)
nb.write(notebook, path)
def test_diff_api_symlink(git_repo2, server_extension_app, needs_symlink):
root = server_extension_app['path']
subdir = pjoin(root, 'has space', 'subdir')
os.makedirs(subdir)
symlink = pjoin(subdir, 'link')
with pushd(subdir):
call(['git', 'init'])
with open('f', 'w') as f:
f.write('stuff')
call(['git', 'add', 'f'])
call(['git', 'commit', '-m', 'initial commit'])
os.symlink(git_repo2, symlink)
local_path = os.path.relpath(symlink, server_extension_app['path'])
url = 'http://127.0.0.1:%i/nbdime/api/diff' % server_extension_app['port']
r = requests.post(
url, headers=auth_header,
data=json.dumps({
'base': 'git:' + pjoin(local_path, 'diff.ipynb'),
}))
r.raise_for_status()
data = r.json()
nbformat.validate(data['base'])
assert data['diff']
assert len(data.keys()) == 2
def _preprocess(self, nb, resources):
"""
Preprocess the notebook before passing it into the Jinja engine.
To preprocess the notebook is to successively apply all the
enabled preprocessors. Output from each preprocessor is passed
along to the next one.
Parameters
----------
nb : notebook node
notebook that is being exported.
resources : a dict of additional resources that
can be accessed read/write by preprocessors
"""
# Do a copy.deepcopy first,
# we are never safe enough with what the preprocessors could do.
nbc = copy.deepcopy(nb)
resc = copy.deepcopy(resources)
#Run each preprocessor on the notebook. Carry the output along
#to each preprocessor
for preprocessor in self._preprocessors:
nbc, resc = preprocessor(nbc, resc)
try:
nbformat.validate(nbc, relax_add_props=True)
except nbformat.ValidationError:
self.log.error('Notebook is invalid after preprocessor {}',
preprocessor)
raise
return nbc, resc
def _check_or_apply_style(file_path, style_config, do_apply_style):
"""Returns true if style is valid.
Since there are common code for check and apply style, the two functions
are merged into one.
"""
# Ref: https://gist.github.com/oskopek/496c0d96c79fb6a13692657b39d7c709
with open(file_path, "r") as f:
notebook = nbformat.read(f, as_version=nbformat.NO_CONVERT)
nbformat.validate(notebook)
changed = False
for cell in notebook.cells:
if cell["cell_type"] != "code":
continue
src = cell["source"]
lines = src.split("\n")
if len(lines) <= 0 or "# noqa" in lines[0]:
continue
# yapf will puts a `\n` at the end of each cell, and if this is the
# only change, cell_changed is still False.
formatted_src, cell_changed = yapf.yapflib.yapf_api.FormatCode(
src, style_config=style_config)
if formatted_src.endswith("\n"):
formatted_src = formatted_src[:-1]
if cell_changed:
cell["source"] = formatted_src
changed = True
if do_apply_style:
with open(file_path, "w") as f:
nbformat.write(notebook, f, version=nbformat.NO_CONVERT)
return not changed
def _warn_if_invalid(nb, version):
"""Log validation errors, if there are any."""
from nbformat import validate, ValidationError
try:
validate(nb, version=version)
except ValidationError as e:
get_logger().error("Notebook JSON is not valid v%i: %s", version, e)
def test_from_version_4_4_upgrades(self):
notebook_name = 'nb_version_4.4.ipynb'
result_path = os.path.join(self.test_dir,
'output_{}'.format(notebook_name))
execute_notebook(get_notebook_path(notebook_name), result_path,
{'var': 'It works'})
nb = load_notebook_node(result_path)
validate(nb)
def test_no_v3_language_backport(self):
notebook_name = 'blank-vscode.ipynb'
result_path = os.path.join(self.test_dir,
'output_{}'.format(notebook_name))
execute_notebook(get_notebook_path(notebook_name), result_path,
{'var': 'It works'})
nb = load_notebook_node(result_path)
validate(nb)
def post(self, *args, **kwargs):
# Given a list of nb cells, save to ipynb format v4
filepath = self.get_argument('filepath')
if filepath[-6:] != '.ipynb':
filepath = '{}.ipynb'.format(filepath)
cells = json.loads(self.request.body)['cells']
nb_cells = []
for cell in cells:
cinput = cell.get('input', '')
coutput = cell.get('output', '')
ctype = cell.get('type')
tmp_cell = {
"cell_type": ctype,
"metadata": {
"collapsed" : False, # whether the output of the cell is collapsed
"autoscroll": "auto", # any of true, false or "auto"
},
"source": cinput,
}
if ctype=='code':
tmp_cell.update({
"execution_count": None,
"outputs": [{
"output_type" : "stream",
"name" : "stdout",
"text" : coutput,
}]
})
nb_cells.append(tmp_cell)
base_nb = {
'metadata': {
'kernelspec': {
'name': 'bash',
"display_name": "Bash",
"language": "bash"
},
"language_info": {
"codemirror_mode": "shell",
"file_extension": ".sh",
"mimetype": "text/x-sh",
"name": "bash"
}
},
'nbformat': 4,
'nbformat_minor': 0,
'cells': nb_cells
}
try:
nbformat.validate(base_nb,version=4)
nb = nbformat.from_dict(base_nb)
nbformat.write(nb,filepath)
self.write({'res':'File saved to {}'.format(filepath)})
except nbformat.ValidationError:
self.set_status(400)
return
def validate_notebook_model(self, model):
"""Add failed-validation message to model"""
try:
validate(model['content'])
except ValidationError as e:
model['message'] = u'Notebook Validation failed: {}:\n{}'.format(
e.message, json.dumps(e.instance, indent=1, default=lambda obj: '<UNKNOWN>'),
)
return model
def validate_notebook_model(self, model):
"""Add failed-validation message to model"""
try:
validate(model["content"])
except ValidationError as e:
model["message"] = u"Notebook Validation failed: {}:\n{}".format(
e.message, json.dumps(e.instance, indent=1, default=lambda obj: "<UNKNOWN>")
)
return model
def validate_notebook_model(self, model):
"""Add failed-validation message to model"""
try:
validate(model['content'])
except ValidationError as e:
model['message'] = 'Notebook Validation failed: {}:\n{}'.format(
e.message, json.dumps(e.instance, indent=1, default=lambda obj: '<UNKNOWN>'),
)
return model
def test_apply_filter_valid_filter(git_repo):
path = pjoin(git_repo, 'diff.ipynb')
gitattr = locate_gitattributes()
with io.open(gitattr, 'a', encoding="utf8") as f:
f.write(u'\n*.ipynb\tfilter=myfilter\n')
call('git config --local --add filter.myfilter.clean "cat"')
f = apply_possible_filter(path)
assert isinstance(f, StringIO)
# Read validates notebook:
nbformat.validate(nbformat.read(f, as_version=4))
def test_apply_filter_valid_filter(git_repo):
path = pjoin(git_repo, 'diff.ipynb')
gitattr = locate_gitattributes()
with io.open(gitattr, 'a', encoding="utf8") as f:
f.write(u'\n*.ipynb\tfilter=myfilter\n')
call('git config --local --add filter.myfilter.clean "cat"')
f = apply_possible_filter(path)
assert isinstance(f, StringIO)
# Read validates notebook:
nbformat.validate(nbformat.read(f, as_version=4))
def test_valid_nb():
"""Should only need to test a single Notebook for validity, this is exclusively a validity test"""
fname = 'digits-classification-df'
ext = '.ipynb'
nb = nbformat.read(os.path.join('./dfconvert/tests/example/', fname + ext),
nbformat.NO_CONVERT)
ipy.export_dfpynb(nb, in_fname=fname + ext)
new_nb = nbformat.read(os.path.join('./', fname + '_ipy.ipynb'),
nbformat.NO_CONVERT)
#Test will fail if this notebook does not validate
#Upon a validation failure nbformat will raise a ValidationError
nbformat.validate(new_nb)
def test_diff_api(git_repo2, server_extension_app):
local_path = os.path.relpath(git_repo2, server_extension_app['path'])
url = 'http://127.0.0.1:%i/nbdime/api/diff' % server_extension_app['port']
r = requests.post(
url, headers=auth_header,
data=json.dumps({
'base': 'git:' + pjoin(local_path, 'diff.ipynb'),
}))
r.raise_for_status()
data = r.json()
nbformat.validate(data['base'])
assert data['diff']
assert len(data.keys()) == 2
def test_upgrade_v4_to_4_dot_5():
here = os.path.dirname(__file__)
with open(os.path.join(here, os.pardir, "test4.ipynb"), encoding="utf-8") as f:
nb = reads(f.read())
assert nb["nbformat_minor"] == 0
validate(nb)
assert nb.cells[0].get("id") is None
nb_up = convert.upgrade(nb)
assert nb_up["nbformat_minor"] == 5
validate(nb_up)
assert nb_up.cells[0]["id"] is not None
def test_save_kernelspec(self, preprocessor, gradebook, resources):
kernelspec = dict(
display_name='blarg',
name='python3',
language='python',
)
nb = new_notebook()
nb.metadata['kernelspec'] = kernelspec
nb, resources = preprocessor.preprocess(nb, resources)
validate(nb)
notebook = gradebook.find_notebook("test", "ps0")
assert json.loads(notebook.kernelspec) == kernelspec
def test_upgrade_v4_to_4_dot_5():
here = os.path.dirname(__file__)
with io.open(os.path.join(here, os.pardir, os.pardir, 'tests',
"test4.ipynb"),
encoding='utf-8') as f:
nb = reads(f.read())
assert nb['nbformat_minor'] == 0
validate(nb)
assert nb.cells[0].get('id') is None
nb_up = convert.upgrade(nb)
assert nb_up['nbformat_minor'] == 5
validate(nb_up)
assert nb_up.cells[0]['id'] is not None
def test_apply_filter_valid_filter(git_repo):
try:
call('cat --help')
filter_cmd = 'cat'
except (CalledProcessError, FileNotFoundError):
filter_cmd = 'findstr x*'
path = pjoin(git_repo, 'diff.ipynb')
gitattr = locate_gitattributes()
with io.open(gitattr, 'a', encoding="utf8") as f:
f.write(u'\n*.ipynb\tfilter=myfilter\n')
call('git config --local --add filter.myfilter.clean "%s"' % filter_cmd)
f = apply_possible_filter(path)
assert isinstance(f, StringIO)
# Read validates notebook:
nbformat.validate(nbformat.read(f, as_version=4))
def update(self):
for smpl in self.walk_nbs():
if not smpl.file.exists():
print(f"No file found for {smpl} - skippiping")
continue
print("Fixing", smpl)
with open(smpl.file, 'rb') as f:
nb = nbformat.read(f, 4)
print("Backing up old file")
try:
new_name = smpl.file.rename(smpl.file.with_name(f'{smpl.name}-old.ipynb'))
except FileExistsError:
print(f"Already updated {smpl} - skipping")
continue
try:
print("Applying processors")
for cell in nb['cells']:
for processor in self.cell_processors:
processor(cell)
print("Success")
print("Errors: ", nbformat.validate(nb, version=4))
print("Writing new file")
with open(smpl.file, 'w', encoding='utf-8') as f:
nbformat.write(nb, f, 4)
print("Success")
except Exception:
print("Exception occured, rolling back")
new_name.rename(smpl.file.name)
def main(old, new, debug=False):
filename = old
assert filename[
-6:] == '.ipynb', "Error: the input file is not a .ipynb Jupyter Notebook file."
with open(filename, 'r') as file:
content = json.load(file)
# Check that it is a IOCaml notebook
assert content['metadata']['kernelspec'][
'name'] == "iocaml-kernel" and content['metadata']['kernelspec'][
'language'] == "ocaml" and content['metadata']['kernelspec'][
'display_name'] == "OCaml", "Error: the input notebook does not appear to have been produced by the IOCaml OCaml kernel."
# For each cell
for cell in content['cells']:
if cell['cell_type'] == "code":
outputs = cell['outputs']
# execution_count = cell['execution_count']
# No need
# if is_stderr_used(outputs):
# break
data_texthtml = get_data_texthtml(outputs)
# new_stderr_output = {
# "name": "stderr",
# "output_type": "stream",
# "text": data_texthtml,
# # "execution_count": execution_count,
# # "metadata": {}
# }
# if debug: pprint(new_stderr_output)
# cell['outputs'].append(new_stderr_output)
for output in outputs:
if 'data' in output:
output['data']['text/plain'] = data_texthtml
if debug: pprint(output['data'])
break # do not add twice the same output cell
# Check before changing the file
nbformat.validate(content) # raise an Error if not valid notebook
# Backup the input file by moving it to $input.ipynb~
# shutil.copy(filename, filename.replace('.ipynb', '.ipynb~'))
# Now write the JSON to the input file $input.ipynb
with open(new, 'w') as file:
json.dump(content, file, indent=2)
print("New notebook written to", new)
def _nb_node_from_nb_node_and_cells(nb_node: nbformat.NotebookNode,
cells: Union[nbformat.NotebookNode, List[nbformat.NotebookNode]]
) -> nbformat.NotebookNode:
if not isinstance(cells, list):
cells = [cells]
# get other data from original notebook
nb_dict = dict(
cells=cells,
metadata=nb_node['metadata'],
nbformat=nb_node['nbformat'],
nbformat_minor=nb_node['nbformat_minor']
)
nbformat.validate(nb_dict)
small_nb = nbformat.from_dict(nb_dict)
return small_nb
def test_overwrite_kernelspec(self, preprocessors, resources, gradebook):
kernelspec = dict(
display_name='blarg',
name='python3',
language='python',
)
nb = new_notebook()
nb.metadata['kernelspec'] = kernelspec
nb, resources = preprocessors[0].preprocess(nb, resources)
nb.metadata['kernelspec'] = {}
nb, resources = preprocessors[1].preprocess(nb, resources)
validate(nb)
notebook = gradebook.find_notebook("test", "ps0")
assert nb.metadata['kernelspec'] == kernelspec
assert json.loads(notebook.kernelspec) == kernelspec
def format_nb(notebook_filename, dry_run=False):
print('Formatting {}...'.format(notebook_filename), end='')
with open(notebook_filename, 'r') as f:
notebook = nbformat.read(f, as_version=nbformat.NO_CONVERT)
nbformat.validate(notebook)
changed = False
if notebook['nbformat'] != 4 or notebook['nbformat_minor'] != 0:
notebook['nbformat_minor'] = 0
changed = True
for token in ['colab', 'language_info']:
if token in notebook.metadata:
del notebook.metadata[token]
changed = True
for cell in notebook.cells:
if cell['metadata'] != {}:
cell['metadata'] = {}
changed = True
for cell in notebook.cells:
if cell['cell_type'] != 'code':
continue
src = cell['source']
lines = src.split('\n')
if len(lines) <= 0 or '# noqa' in lines[0]:
continue
formatted_src, did_change = FormatCode(src, style_config=style_file)
if did_change:
cell['source'] = formatted_src
changed = True
if changed:
if not dry_run:
with open(notebook_filename, 'w') as f:
nbformat.write(notebook, f, version=4)
print(' (reformatted)')
else:
print()
def validate_execution_data(request_data):
"""
This function validates the given request data.
:param request_data: The request data to validate
:raise BadRequest: If the request data is invalid
"""
try:
jsonschema.validate(request_data, request_schema)
except jsonschema.ValidationError as e:
raise BadRequest('Failed to validate request data. {}'.format(
str(e)))
for jupyter_notebook in request_data['jupyterNotebooks']:
try:
nbformat.validate(jupyter_notebook['data'])
except nbformat.ValidationError as e:
raise BadRequest(
'Failed to validate notebook "{}.\n{}"'.format(
jupyter_notebook['filename'], str(e)))
def test_git_diff_api(git_repo2, server_extension_app, filespath):
local_path = os.path.relpath(git_repo2, server_extension_app['path'])
url = 'http://127.0.0.1:%i/nbdime/api/gitdiff' % server_extension_app[
'port']
# Add a difference betweeen index and working tree:
shutil.copy(pjoin(filespath, 'foo--1.ipynb'),
pjoin(git_repo2, 'sub', 'subfile.ipynb'))
def _make_ref(key):
if key.lower() in ('working', 'index'):
return {'special': key}
return {'git': key}
# Test various diffs:
for args in (
('HEAD', 'WORKING', 'diff.ipynb'),
('HEAD', 'INDEX', 'diff.ipynb'),
('INDEX', 'HEAD', 'diff.ipynb'),
('INDEX', 'WORKING', 'sub/subfile.ipynb'),
('index', 'working', 'sub/subfile.ipynb'),
('iNdeX', 'WorKING', 'sub/subfile.ipynb'),
):
print(args)
r = requests.post(url,
headers=auth_header,
data=json.dumps({
'ref_local':
_make_ref(args[0]),
'ref_remote':
_make_ref(args[1]),
'file_path':
pjoin(local_path, args[2])
}))
r.raise_for_status()
data = r.json()
nbformat.validate(data['base'])
assert data['diff']
assert len(data.keys()) == 2
def main_patch(args):
base_filename = args.base
patch_filename = args.patch
output_filename = args.output
for fn in (base_filename, patch_filename):
if not os.path.exists(fn) and fn != EXPLICIT_MISSING_FILE:
print("Missing file {}".format(fn))
return 1
before = read_notebook(base_filename, on_null='empty')
with io.open(patch_filename, encoding="utf8") as patch_file:
diff = json.load(patch_file)
diff = to_diffentry_dicts(diff)
after = patch_notebook(before, diff)
if output_filename:
nbformat.write(after, output_filename)
else:
try:
nbformat.validate(after, version=4)
except nbformat.ValidationError:
print("Patch result is not a valid notebook, printing as JSON:")
json.dump(after, sys.stdout)
else:
# This printer is to keep the unit tests passing,
# some tests capture output with capsys which doesn't
# pick up on sys.stdout.write()
class Printer:
def write(self, text):
print(text, end="")
config = PrettyPrintConfig(out=Printer())
pretty_print_notebook(after, config=config)
return 0
def main_patch(args):
base_filename = args.base
patch_filename = args.patch
output_filename = args.output
for fn in (base_filename, patch_filename):
if not os.path.exists(fn) and fn != EXPLICIT_MISSING_FILE:
print("Missing file {}".format(fn))
return 1
before = read_notebook(base_filename, on_null='empty')
with io.open(patch_filename, encoding="utf8") as patch_file:
diff = json.load(patch_file)
diff = to_diffentry_dicts(diff)
after = patch_notebook(before, diff)
if output_filename:
nbformat.write(after, output_filename)
else:
try:
nbformat.validate(after, version=4)
except nbformat.ValidationError:
print("Patch result is not a valid notebook, printing as JSON:")
json.dump(after, sys.stdout)
else:
# This printer is to keep the unit tests passing,
# some tests capture output with capsys which doesn't
# pick up on sys.stdout.write()
class Printer:
def write(self, text):
print(text, end="")
config = PrettyPrintConfig(out=Printer())
pretty_print_notebook(after, config=config)
return 0
def write(self, ast, resources=None):
# Mapping {filename: data}.
self.resources = resources or {}
self.execution_count = 1
self._md = MarkdownPlugin()
# Add code cells in the AST.
ast = wrap_code_cells(ast)
# Create the notebook.
# new_output, new_code_cell, new_markdown_cell
nb = new_notebook()
# Go through all top-level blocks.
for index, node in enumerate(ast.children):
# Determine the block type.
if node.name == 'CodeCell':
node_type = 'code'
else:
node_type = 'markdown'
# Create the notebook cell.
cell = getattr(self, 'new_{}_cell'.format(node_type))(node, index)
# Add it to the notebook.
nb.cells.append(cell)
nbformat.validate(nb)
return nb
def find_notebooks(directory: Path) -> List[JupyterNotebook]:
"""Finds Jupyter notebooks in a directory. Not recursive.
Args:
directory (Path): directory to search for notebook files
Returns:
List[JupyterNotebook]: notebooks found
"""
notebooks = []
for subfile in directory.iterdir():
if subfile.is_file() and subfile.name:
try:
notebook = nbformat.read(str(subfile),
as_version=nbformat.NO_CONVERT)
nbformat.validate(notebook)
notebooks.append(
JupyterNotebook(path=subfile, metadata=notebook.metadata))
except Exception as e:
if subfile.suffix.lower() == ".ipynb":
warn(
f"Error reading {subfile.resolve()} as Jupyter Notebook: "
+ f"[{type(e).__name__}] {e}")
return notebooks
def test_diff_api_checkpoint(tmpdir, filespath, server_extension_app):
local_path = os.path.relpath(str(tmpdir), server_extension_app['path'])
# Create base
src = filespath
shutil.copy(pjoin(src, 'src-and-output--1.ipynb'), pjoin(str(tmpdir), 'diff.ipynb'))
url_path = pjoin(local_path, 'diff.ipynb')
if os.sep == '\\':
url_path = url_path.replace('\\', '/')
# Create checkpoint
url = 'http://127.0.0.1:%i/api/contents/%s/checkpoints' % (
server_extension_app['port'],
url_path,
)
r = requests.post(url, headers=auth_header)
r.raise_for_status()
# Overwrite:
shutil.copy(pjoin(src, 'src-and-output--2.ipynb'), pjoin(str(tmpdir), 'diff.ipynb'))
url = 'http://127.0.0.1:%i/nbdime/api/diff' % server_extension_app['port']
r = requests.post(
url, headers=auth_header,
data=json.dumps({
'base': 'checkpoint:' + url_path,
}))
r.raise_for_status()
data = r.json()
nbformat.validate(data['base'])
assert data['diff']
assert len(data.keys()) == 2
def test_downgrade_notebook():
nb04 = copy.deepcopy(nbexamples.nb0)
validate(nb04)
nb03 = convert.downgrade(nb04)
validate(nb03)
def test_upgrade_notebook():
nb03 = copy.deepcopy(v3examples.nb0)
validate(nb03)
nb04 = convert.upgrade(nb03)
validate(nb04)
import json
import nbformat
with open("test_graph.ipynb", 'r') as f:
nbjson = f.read()
print "validate", nbformat.validate(json.loads(nbjson))
node = nbformat.reads(nbjson, 4)
noded = nbformat.from_dict(json.loads(nbjson))
noded4 = nbformat.convert(noded, 4)
def test_downgrade_3(self):
exporter = self.exporter_class(nbformat_version=3)
(output, resources) = exporter.from_filename(self._get_notebook())
nb = json.loads(output)
validate(nb)
from nbformat import NotebookNode
test = 'what_is_data_1.ipynb'
out = 'test.ipynb'
data = nbformat.read(test, as_version=nbformat.NO_CONVERT)
new_metadata = data.metadata.copy()
cells = data.cells
document = NotebookNode(dict(author = 'Ewan Klein', licence = 'CC-BY', title="What is Data???"))
new_metadata['document'] = document
data.metadata = NotebookNode(new_metadata)
nbformat.validate(data)
header = cells[0]['source']
title = '# ' + document.title
author = document.author
header_lines = header.split('\n')
if header_lines[0].startswith('#'):
del header_lines[0]
header_lines = [title, author] + header_lines
header_string = '\n'.join(header_lines)
