def kernelspec_from_language(language):
"""Return the python kernel that matches the current env, or the first kernel that matches the given language"""
try:
if language == 'python':
# Return the kernel that matches the current Python executable
for name in find_kernel_specs():
kernel_specs = get_kernel_spec(name)
if kernel_specs.language == 'python' and kernel_specs.argv[
0] == sys.executable:
return {
'name': name,
'language': language,
'display_name': kernel_specs.display_name
}
for name in find_kernel_specs():
kernel_specs = get_kernel_spec(name)
if kernel_specs.language == language or (
language == 'c++'
and kernel_specs.language.startswith('C++')):
return {
'name': name,
'language': language,
'display_name': kernel_specs.display_name
}
except (KeyError, ValueError):
pass
return None
def kernelspec_from_language(language):
"""Return the python kernel that matches the current env, or the first kernel that matches the given language"""
if language == "python":
# Return the kernel that matches the current Python executable
for name in find_kernel_specs():
kernel_specs = get_kernel_spec(name)
cmd = kernel_specs.argv[0]
if (kernel_specs.language == "python" and os.path.isfile(cmd)
and os.path.samefile(cmd, sys.executable)):
return {
"name": name,
"language": language,
"display_name": kernel_specs.display_name,
}
raise ValueError(
"No kernel found that matches the current python executable {}\n".
format(sys.executable) +
"Install one with 'python -m ipykernel install --name kernel_name [--user]'"
)
for name in find_kernel_specs():
kernel_specs = get_kernel_spec(name)
if same_language(kernel_specs.language, language):
return {
"name": name,
"language": language,
"display_name": kernel_specs.display_name,
}
raise ValueError("No kernel found for the language {}".format(language))
def main():
kernel_name = args.kernel_name
kernel_env_kwargs = args.env_kwargs
kernelspecs = kernelspec.find_kernel_specs()
if not kernelspecs:
if args.verbose:
print("No kernelspecs were found: {}".format(kernelspecs))
exit(0)
if kernel_name not in kernelspecs:
print(
"The target kernel: {} does not exist on this system, the avilable are: {}"
.format(kernel_name, kernelspecs))
exit(0)
kernel_spec = kernelspec.get_kernel_spec(kernel_name)
# Update the kernel spec env with provided environment
if kernel_env_kwargs:
for key, value in kernel_env_kwargs.items():
# Expand value if environement variables are used
expanded_value = os.path.expandvars(value)
kernel_spec.env.update({key: expanded_value})
if update_kernel(kernel_spec) and args.verbose:
print("Updated kernel: {}".format(kernel_name))
def restart_jupyter_kernel(self):
"""Restart the julia jupyter kernel if it's already started. Otherwise, or if the julia version
has changed in settings, start a new jupyter kernel.
"""
kernel_name = self.julia_kernel_name()
if not kernel_name:
return
if self.kernel_manager and kernel_name == self.kernel_name:
# Restart current kernel
self.starting = True
self._toolbox.msg.emit("*** Restarting Julia REPL ***")
self.kernel_client.stop_channels()
self.kernel_manager.restart_kernel(now=True)
self.setup_client()
else:
# No kernel to restart (!) or julia has changed in settings. Start a new kernel
self.kernel_name = kernel_name
kernel_specs = find_kernel_specs()
julia_kernel_names = [x for x in kernel_specs if x.startswith('julia')]
if self.kernel_name in julia_kernel_names:
self.start_available_jupyter_kernel()
else:
self._toolbox.msg_error.emit("\tCouldn't find a Jupyter kernel "
"specification for {}".format(self.kernel_name))
self.handle_repl_failed_to_start()
def look_in_notebook_specs(self, keyname=None, speclist=None):
'''
find keyname in kernelspec
and return the first we find
store (specitem,keyname,keyvalue) in self.specitem
'''
keyname = keyname or self.keyname
keyvalue = None
speclist = speclist or kernelspec.find_kernel_specs().keys()
speclist = list(speclist)
# look over list
try:
for specitem in speclist:
try:
spec = kernelspec.get_kernel_spec(specitem)
env = spec.env
if keyname in env:
keyvalue = spec[keyname]
self.specitem = (specitem, keyname, keyvalue)
return keyvalue
except:
pass
except:
pass
return keyvalue
def notebook_generator(cells: Optional[List[NotebookNode]] = None) -> NotebookNode:
"""Generate an executable notebook.
The notebook cells are the one passed as arguments or the hard-coded cells
if no cells is provided.
"""
metadata = {}
for name in kernelspec.find_kernel_specs():
ks = kernelspec.get_kernel_spec(name)
metadata = {
'kernelspec': {
'name': name,
'language': ks.language,
'display_name': ks.display_name,
}
}
break
if cells is not None:
all_cells = cells
else: # Default cells
all_cells = [
new_code_cell('a = 2', metadata={"tags": []}),
new_code_cell('b=22\nb', metadata={"tags": ["test"]}),
new_code_cell(
"",
metadata={"tags": ["dummy-outputs"]},
outputs=[new_output('execute_result', data={"text/plain": "text"})],
),
]
return new_notebook(metadata=metadata, cells=all_cells)
def find_kernel(self):
"""Decide which kernel to use"""
kernel_specs = find_kernel_specs()
kernel = self.kernel
if kernel:
if kernel in kernel_specs:
return kernel
if self.force_fail:
self.report_exit("<No available kernel>",
"Kernel {} not found".format(kernel))
else:
vprint(self.vindex,
("Configured Kernel {} not found. "
"Trying to detect other kernel").format(self.kernel))
kernel = (self.notebook.get('metadata',
{}).get('kernelspec',
{}).get('name', 'python'))
if kernel in kernel_specs:
return kernel
kernel = 'python'
if kernel in kernel_specs:
return kernel
kernel = 'python{}'.format(sys.version_info[0])
if kernel in kernel_specs:
return kernel
if kernel_specs:
return next(iter(kernel_specs))
return None
def restart_jupyter_kernel(self):
"""Restarts the julia jupyter kernel if it's already started.
Starts a new kernel if none started or if the julia version has changed in Settings.
"""
kernel_name = self.julia_kernel_name()
if not kernel_name:
return
julia_project_path = self._toolbox.qsettings().value(
"appSettings/juliaProjectPath", defaultValue="")
if self.kernel_manager and kernel_name == self.kernel_name and julia_project_path == self.julia_project_path:
# Restart current kernel
self.starting = True
self._toolbox.msg.emit("*** Restarting Julia REPL ***")
self.kernel_client.stop_channels()
blackhole = open(os.devnull, 'w')
self.kernel_manager.restart_kernel(now=True,
stdout=blackhole,
stderr=blackhole)
self.setup_client()
else:
# No kernel to restart (!) or julia has changed in settings. Start a new kernel
self.julia_project_path = julia_project_path
kernel_specs = find_kernel_specs()
julia_kernel_names = [
x for x in kernel_specs if x.startswith('julia')
]
if kernel_name in julia_kernel_names:
self._do_start_jupyter_kernel(kernel_name)
else:
self._toolbox.msg_error.emit(
"\tCouldn't find Jupyter kernel specification for {}".
format(kernel_name))
self.handle_repl_failed_to_start()
def install_kernelspec(lib_dir, kernel_dir=None, user=True):
if kernel_dir is None:
kernel_dir = os.path.join(lib_dir, "kernel", "lambda-notebook")
if lib_dir:
# TODO: won't override an installed copy of lamb in site-packages (e.g.
# in the app), fix this
# the following line is to deal with a messy escaping situation for
# windows paths this may fail horribly on unix paths with backslashes,
# but I don't have a better workaround for now
lib_dir = lib_dir.replace("\\", "\\\\\\\\")
injection_path_opt = (
"--IPKernelApp.exec_lines=[\"import sys\",\"sys.path.insert(1,\\\"%s\\\")\", \"import lamb.lnsetup\", \"lamb.lnsetup.ipython_setup()\"]"
% lib_dir)
else:
injection_path_opt = "--IPKernelApp.exec_lines=[\"import lamb.lnsetup\", \"lamb.lnsetup.ipython_setup()\"]"
k_json = build_json()
k_json["argv"].append(injection_path_opt)
k_json_filename = os.path.join(kernel_dir, "kernel.json")
#print(k_json)
with open(k_json_filename, 'w') as k_json_file:
json.dump(k_json, k_json_file, sort_keys=True, indent=4)
kernelspec.install_kernel_spec(kernel_dir, user=user, replace=True)
location = kernelspec.find_kernel_specs()['lambda-notebook']
return location
def get_test_kernels():
test_kernels = [
k for k, v in kernelspec.find_kernel_specs().items()
if (k.startswith("ssh_")) and (
"TEST" in kernelspec.get_kernel_spec(k).display_name)
]
return test_kernels
def remove_kernelspecs():
km = kernelspec.KernelSpecManager()
kernel_id = None
while kernel_id != "done":
remote_kernels = {
kernelspec.get_kernel_spec(k).display_name: k
for k, v in kernelspec.find_kernel_specs().items()
if k.startswith("ssh_")
}
if remote_kernels == {}:
print_ok(" => No databricklabs_jupyterlab kernel spec left")
break
answer = question(
"kernel_name",
"Which kernel spec to delete (Ctrl-C to finish)?",
list(remote_kernels.keys()),
)
if answer is None:
break
kernel_name = answer["kernel_name"]
if kernel_id != "done":
answer = input("Really delete kernels spec '%s' (y/n) " %
kernel_name)
if answer.lower() == "y":
km.remove_kernel_spec(remote_kernels[kernel_name])
def write_notebook(self,specname=None,backup=None,keyname=None):
'''
write keyvalue into database notebook kernel file
This is an experimantal feature ... wouldnt advise using it
'''
verbose = self.verbose
# see https://jupyter-client.readthedocs.io/en/latest/kernels.html
specname = specname or self.database
backup = backup or self.backup
keyname = keyname or self.keyname
try:
# exists
spec = kernelspec.get_kernel_spec(specname)
except:
# doesnt exist
spec = kernelspec.get_kernel_spec(backup)
kernel_file = kernelspec.find_kernel_specs()[backup]
# create specname
# try as user then not
try:
kernelspec.install_kernel_spec(kernel_file,
kernel_name=specname, user=False,replace=True)
except:
kernelspec.install_kernel_spec(kernel_file,
kernel_name=specname,
user=True,replace=True)
try:
spec = kernelspec.get_kernel_spec(specname)
# now load specname
# make a name derived from specname
spec.display_name = ' '.join([i.capitalize() for i in specname.split('_')])
# make sure language is python
spec.language='python'
# add env variable if we can
if spec.has_trait('env'):
spec.env[keyname] = keyvalue
else:
# work out how to add env trait at some point
pass
# Serializing json
json_object = spec.to_json()
# write this out
kernel = Path(spec.resource_dir + '/kernel.json')
kernel.parent.mkdir(parents=True, exist_ok=True)
with open(kernel, 'w') as f:
if verbose:
print(f'jupyter kernel written to {kernel}')
f.write(json_object)
self.make_icons(spec.resource_dir)
# load again for luck
spec = kernelspec.get_kernel_spec(specname)
return spec
except:
return None
def kernelspec_from_language(language):
"""Return the python kernel that matches the current env, or the first kernel that matches the given language"""
try:
if language == "python":
# Return the kernel that matches the current Python executable
for name in find_kernel_specs():
kernel_specs = get_kernel_spec(name)
cmd = kernel_specs.argv[0]
if (
kernel_specs.language == "python"
and os.path.isfile(cmd)
and os.path.samefile(cmd, sys.executable)
):
return {
"name": name,
"language": language,
"display_name": kernel_specs.display_name,
}
# If none was found, return the first kernel that has 'python' as argv[0]
for name in find_kernel_specs():
kernel_specs = get_kernel_spec(name)
if (
kernel_specs.language == "python"
and kernel_specs.argv[0] == "python"
):
return {
"name": name,
"language": language,
"display_name": kernel_specs.display_name,
}
for name in find_kernel_specs():
kernel_specs = get_kernel_spec(name)
if same_language(kernel_specs.language, language):
return {
"name": name,
"language": language,
"display_name": kernel_specs.display_name,
}
except (KeyError, ValueError):
pass
return None
def kernelspec_from_language(language):
"""Return the kernel specification for the first kernel with a matching language"""
try:
for name in find_kernel_specs():
kernel_specs = get_kernel_spec(name)
if kernel_specs.language == language or (language == 'c++' and kernel_specs.language.startswith('C++')):
return {'name': name, 'language': language, 'display_name': kernel_specs.display_name}
except (KeyError, ValueError):
pass
return None
def get_kernel_path(cluster_id, with_spark):
def cond(k):
s = k.endswith("spark")
return s if with_spark else not s
test_kernels = [
(k, v) for k, v in kernelspec.find_kernel_specs().items()
if (k.startswith("ssh_")) and ("TEST" in kernelspec.get_kernel_spec(
k).display_name) and (cluster_id in k) and cond(k)
]
assert len(test_kernels) == 1
return test_kernels[0]
def kernel_names(self) -> Dict[str, list]:
"""Kernel name dictionary for languages. """
if self._kernel_names:
return self._kernel_names
kernel_specs = find_kernel_specs()
for kernel_name in kernel_specs:
kernel_spec = get_kernel_spec(kernel_name)
language = kernel_spec.language
if language not in self._kernel_names:
self._kernel_names[language] = [kernel_name]
else:
self._kernel_names[language].append(kernel_name)
return self._kernel_names
def notebook():
for name in kernelspec.find_kernel_specs():
ks = kernelspec.get_kernel_spec(name)
metadata = {
'kernelspec': {'name': name, 'language': ks.language, 'display_name': ks.display_name}
}
return nbformat.v4.new_notebook(
metadata=metadata,
cells=[
nbformat.v4.new_markdown_cell('This is a notebook with kernel: ' + ks.display_name)
],
)
raise EnvironmentError('No kernel found')
def test_set_kernel_with_name(py_file, tmpdir):
tmp_py = str(tmpdir.join('notebook.py'))
tmp_ipynb = str(tmpdir.join('notebook.ipynb'))
copyfile(py_file, tmp_py)
for kernel in find_kernel_specs():
jupytext(['--to', 'ipynb', tmp_py, '--set-kernel', kernel])
nb = read(tmp_ipynb)
assert nb.metadata['kernelspec']['name'] == kernel
with pytest.raises(KeyError):
jupytext(['--to', 'ipynb', tmp_py, '--set-kernel', 'non_existing_env'])
def test_set_kernel_with_name(py_file, tmpdir):
tmp_py = str(tmpdir.join("notebook.py"))
tmp_ipynb = str(tmpdir.join("notebook.ipynb"))
copyfile(py_file, tmp_py)
for kernel in find_kernel_specs():
jupytext(["--to", "ipynb", tmp_py, "--set-kernel", kernel])
nb = read(tmp_ipynb)
assert nb.metadata["kernelspec"]["name"] == kernel
with pytest.raises(KeyError):
jupytext(["--to", "ipynb", tmp_py, "--set-kernel", "non_existing_env"])
def kernelspec_from_language(language):
"""Return the python kernel that matches the current env, or the first kernel that matches the given language"""
try:
if language == 'python':
# Return the kernel that matches the current Python executable
for name in find_kernel_specs():
kernel_specs = get_kernel_spec(name)
if kernel_specs.language == 'python' and kernel_specs.argv[
0] == sys.executable:
return {
'name': name,
'language': language,
'display_name': kernel_specs.display_name
}
# If none was found, return the first kernel that has 'python' as argv[0]
for name in find_kernel_specs():
kernel_specs = get_kernel_spec(name)
if kernel_specs.language == 'python' and kernel_specs.argv[
0] == 'python':
return {
'name': name,
'language': language,
'display_name': kernel_specs.display_name
}
for name in find_kernel_specs():
kernel_specs = get_kernel_spec(name)
if same_language(kernel_specs.language, language):
return {
'name': name,
'language': language,
'display_name': kernel_specs.display_name
}
except (KeyError, ValueError):
pass
return None
def git_project(tmp_path):
git_ref = tmp_path / "ref"
for name in kernelspec.find_kernel_specs():
ks = kernelspec.get_kernel_spec(name)
if ks.language != "python":
continue
metadata = {
"kernelspec": {
"name": name,
"language": ks.language,
"display_name": ks.display_name,
}
}
break
fake_notebooks = {
git_ref / "notebook1.ipynb":
new_notebook(
cells=[
new_code_cell("a: int = 2 # Beautiful",
metadata={"tags": ["parameters"]}),
new_code_cell("a"),
],
metadata=metadata,
),
git_ref / "subfolder" / "notebook2.ipynb":
new_notebook(
cells=[
new_code_cell(
"b: str = 'The famous B' # Beautiful",
metadata={"tags": ["parameters"]},
),
new_markdown_cell("# Surprise"),
new_code_cell("f'{b}eaver'"),
],
metadata=metadata,
),
}
for path, nb in fake_notebooks.items():
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(nb))
# Initialize the git repository
check_call(["git", "init", str(git_ref)])
check_call(["git", "add", "-A"], cwd=str(git_ref))
check_call(["git", "commit", "-am", '"First commit"'], cwd=str(git_ref))
yield git_ref
def main(argv):
if len(argv) != 1:
print('Usage: {}'.format(argv[0]))
return 1
npm_output = subprocess.run(['npm', 'ls', 'socket.io'],
cwd='/datalab/web',
stdout=subprocess.PIPE)
socketio_version = npm_output.stdout.strip().decode('utf8').split('@')[-1]
for name, path in kernelspec.find_kernel_specs().items():
spec = kernelspec.get_kernel_spec(name)
colab_metadata = spec.metadata.setdefault('colab', {})
colab_metadata['socketio_version'] = socketio_version
with open(os.path.join(path, 'kernel.json'), 'wt') as f:
json.dump(spec.to_dict(), f, sort_keys=True, indent=1)
def init_myst_file(path, kernel, verbose=True):
"""Initialize a file with a Jupytext header that marks it as MyST markdown.
Parameters
----------
path : string
A path to a markdown file to be initialized for Jupytext
kernel : string
A kernel name to add to the markdown file. See a list of kernel names with
`jupyter kernelspec list`.
"""
try:
from jupytext.cli import jupytext
except ImportError:
raise ImportError(
"In order to use myst markdown features, " "please install jupytext first."
)
if not Path(path).exists():
raise FileNotFoundError(f"Markdown file not found: {path}")
kernels = list(find_kernel_specs().keys())
kernels_text = "\n".join(kernels)
if kernel is None:
if len(kernels) > 1:
_error(
"There are multiple kernel options, so you must give one manually."
" with `--kernel`\nPlease specify one of the following kernels.\n\n"
f"{kernels_text}"
)
else:
kernel = kernels[0]
if kernel not in kernels:
raise ValueError(
f"Did not find kernel: {kernel}\nPlease specify one of the "
f"installed kernels:\n\n{kernels_text}"
)
args = (str(path), "-q", "--set-kernel", kernel, "--set-formats", "myst")
jupytext(args)
if verbose:
print(f"Initialized file: {path}\nWith kernel: {kernel}")
def start_jupyter_kernel(self):
"""Start a Julia Jupyter kernel if available.
Returns:
True if the kernel is started, or in process of being started (installing/reconfiguring IJulia)
False if the kernel cannot be started and the user chooses not to install/reconfigure IJulia
"""
kernel_name = self.julia_kernel_name()
if not kernel_name:
return False
if self.kernel_manager and kernel_name == self.kernel_name:
self._toolbox.msg.emit("*** Using previously started Julia REPL ***")
return True
self.kernel_name = kernel_name
self.kernel_execution_state = None
kernel_specs = find_kernel_specs()
julia_kernel_names = [x for x in kernel_specs if x.startswith('julia')]
if self.kernel_name in julia_kernel_names:
return self.start_available_jupyter_kernel()
else:
self._toolbox.msg_error.emit("\tCouldn't find the {0} kernel specification".format(self.kernel_name))
return self.handle_repl_failed_to_start()
def start_jupyter_kernel(self):
"""Starts a Julia Jupyter kernel if available."""
kernel_name = self.julia_kernel_name()
if not kernel_name:
self.execution_failed.emit(-1)
return
julia_project_path = self._toolbox.qsettings().value("appSettings/juliaProjectPath", defaultValue="")
if julia_project_path == "":
julia_project_path = "@."
if self.kernel_manager and kernel_name == self.kernel_name and julia_project_path == self.julia_project_path:
self._toolbox.msg.emit("*** Using previously started Julia Console ***")
self.ready_to_execute.emit()
return
self.julia_project_path = julia_project_path
self.kernel_execution_state = None
kernel_specs = find_kernel_specs()
# logging.debug("kernel_specs:{0}".format(kernel_specs))
julia_kernel_names = [x for x in kernel_specs if x.startswith('julia')]
if kernel_name in julia_kernel_names:
self._do_start_jupyter_kernel(kernel_name)
return
self._toolbox.msg_error.emit("\tCouldn't find kernel specification {0}".format(kernel_name))
self.handle_repl_failed_to_start()
def install_kernelspec(lib_dir, kernel_dir=None, user=True):
if kernel_dir is None:
kernel_dir = os.path.join(lib_dir, "kernel", "lambda-notebook")
if lib_dir:
# TODO: won't override an installed copy of lamb in site-packages (e.g. in the app), fix this
# the following line is to deal with a messy escaping situation for windows paths
# this may fail horribly on unix paths with backslashes, but I don't have a better workaround for now
lib_dir = lib_dir.replace("\\", "\\\\\\\\")
injection_path_opt = "--IPKernelApp.exec_lines=[\"import sys\",\"sys.path.append(\\\"%s\\\")\", \"import lamb.lnsetup\", \"lamb.lnsetup.ipython_setup()\"]" % lib_dir
else:
injection_path_opt = "--IPKernelApp.exec_lines=[\"import lamb.lnsetup\", \"lamb.lnsetup.ipython_setup()\"]"
k_json = build_json()
k_json["argv"].append(injection_path_opt)
k_json_filename = os.path.join(kernel_dir, "kernel.json")
#print(k_json)
with open(k_json_filename, 'w') as k_json_file:
json.dump(k_json, k_json_file, sort_keys=True, indent=4)
kernelspec.install_kernel_spec(kernel_dir, user=user, replace=True)
location = kernelspec.find_kernel_specs()['lambda-notebook']
return location
def get_julia_env(settings):
"""
Args:
settings (QSettings, AppSettings)
Returns:
tuple, NoneType: (julia_exe, julia_project), or None if none found
"""
use_julia_kernel = settings.value("appSettings/useJuliaKernel",
defaultValue="2") == "2"
if use_julia_kernel:
kernel_name = settings.value("appSettings/juliaKernel",
defaultValue="")
resource_dir = find_kernel_specs().get(kernel_name)
if resource_dir is None:
return None
filepath = os.path.join(resource_dir, "kernel.json")
with open(filepath, "r") as fh:
try:
kernel_spec = json.load(fh)
except json.decoder.JSONDecodeError:
return None
julia = kernel_spec["argv"].pop(0)
project_arg = next(
(arg
for arg in kernel_spec["argv"] if arg.startswith("--project=")),
None)
project = "" if project_arg is None else project_arg.split(
"--project=")[1]
return julia, project
julia = settings.value("appSettings/juliaPath", defaultValue="")
if julia == "":
julia = resolve_executable_from_path(JULIA_EXECUTABLE)
if julia == "":
return None
project = settings.value("appSettings/juliaProjectPath", defaultValue="")
return julia, project
reason="flake8 not found")
requires_autopep8 = pytest.mark.skipif(not tool_version("autopep8"),
reason="autopep8 not found")
requires_nbconvert = pytest.mark.skipif(not tool_version("jupyter nbconvert"),
reason="nbconvert not found")
requires_sphinx_gallery = pytest.mark.skipif(
not rst2md, reason="sphinx_gallery is not available")
requires_pandoc = pytest.mark.skipif(
not is_pandoc_available() or sys.version_info < (3, ),
reason="pandoc>=2.7.2 is not available",
)
requires_no_pandoc = pytest.mark.skipif(is_pandoc_available(),
reason="Pandoc is installed")
requires_ir_kernel = pytest.mark.skipif(
not any(
get_kernel_spec(name).language == "R" for name in find_kernel_specs()),
reason="irkernel is not installed",
)
requires_myst = pytest.mark.skipif(not is_myst_available(),
reason="myst_parser not found")
requires_no_myst = pytest.mark.skipif(is_myst_available(),
reason="myst is available")
skip_on_windows = pytest.mark.skipif(sys.platform.startswith("win"),
reason="Issue 489")
def list_notebooks(path="ipynb", skip="World"):
"""All notebooks in the directory notebooks/path,
or in the package itself"""
if path == "ipynb":
return (list_notebooks("ipynb_julia", skip=skip) +
import sys
from jupyter_client import kernelspec
# If we get this far, vpnotebook ought to have been installed and
# we should have a vpython kernel.
assert 'vpython' in kernelspec.find_kernel_specs().keys()
# Make sure vpython is installed. The import is expected to fail because it
# tries to connect to a jupyter comm, and we are not running a notebook.
# The test is really that the error message expected if that is the
# failure is the actual error message.
try:
import vpython
except AttributeError as e:
assert "'NoneType' object has no attribute 'kernel'" in str(e)
except OSError as e:
if sys.platform.startswith('win'):
assert "The system cannot find the path specified" in str(e)
else:
assert "No such file or directory" in str(e)
def list_kernelspecs(cls):
"""Get the kernelspecs."""
_refresh_jupyter_path()
return find_kernel_specs()
def get(self):
specs = list(kernelspec.find_kernel_specs().keys())
km = self.kernel_manager
running = [make_kernel_model(km, kid) for kid in km.list_kernel_ids()]
self.finish(json.dumps({'available': specs, 'running': running}))
def jupytext(args=None):
"""Internal implementation of Jupytext command line"""
args = parse_jupytext_args(args)
def log(text):
if not args.quiet:
sys.stdout.write(text + '\n')
if args.version:
log(__version__)
return 0
if args.pre_commit:
if args.notebooks:
raise ValueError('--pre-commit takes notebooks from the git index. Do not pass any notebook here.')
args.notebooks = notebooks_in_git_index(args.input_format)
log('[jupytext] Notebooks in git index are:')
for nb_file in args.notebooks:
log(nb_file)
def writef_git_add(notebook_, nb_file_, fmt_):
writef(notebook_, nb_file_, fmt_)
if args.pre_commit:
system('git', 'add', nb_file_)
# Read notebook from stdin
if not args.notebooks:
if not args.pre_commit:
args.notebooks = ['-']
if args.set_formats is not None:
# Replace empty string with None
args.update_metadata = recursive_update(args.update_metadata,
{'jupytext': {'formats': args.set_formats or None}})
if args.paired_paths:
if len(args.notebooks) != 1:
raise ValueError('--paired-paths applies to a single notebook')
print_paired_paths(args.notebooks[0], args.input_format)
return 1
if not args.to and not args.output and not args.sync \
and not args.pipe and not args.check \
and not args.test and not args.test_strict \
and not args.update_metadata and not args.set_kernel:
raise ValueError('Please select an action')
if args.output and len(args.notebooks) != 1:
raise ValueError('Please input a single notebook when using --output')
if args.input_format:
args.input_format = long_form_one_format(args.input_format)
if args.to:
args.to = long_form_one_format(args.to)
set_format_options(args.to, args.format_options)
# Main loop
round_trip_conversion_errors = 0
# Wildcard extension on Windows #202
notebooks = []
for pattern in args.notebooks:
if '*' in pattern or '?' in pattern:
notebooks.extend(glob.glob(pattern))
else:
notebooks.append(pattern)
for nb_file in notebooks:
if nb_file == '-' and args.sync:
raise ValueError('Cannot sync a notebook on stdin')
nb_dest = args.output or (None if not args.to
else ('-' if nb_file == '-' else
full_path(base_path(nb_file, args.input_format), args.to)))
# Just acting on metadata / pipe => save in place
if not nb_dest and not args.sync:
nb_dest = nb_file
if nb_dest == '-':
args.quiet = True
# I. ### Read the notebook ###
fmt = copy(args.input_format) or {}
set_format_options(fmt, args.format_options)
log('[jupytext] Reading {}{}'.format(
nb_file if nb_file != '-' else 'stdin',
' in format {}'.format(short_form_one_format(fmt)) if 'extension' in fmt else ''))
notebook = readf(nb_file, fmt)
if not fmt:
text_representation = notebook.metadata.get('jupytext', {}).get('text_representation', {})
ext = os.path.splitext(nb_file)[1]
if text_representation.get('extension') == ext:
fmt = {key: text_representation[key] for key in text_representation if
key in ['extension', 'format_name']}
elif ext:
fmt = {'extension': ext}
# Compute actual extension when using script/auto, and update nb_dest if necessary
dest_fmt = args.to
if dest_fmt and dest_fmt['extension'] == '.auto':
auto_ext = auto_ext_from_metadata(notebook.metadata)
if not auto_ext:
raise ValueError('The notebook has no language information. '
'Please provide an explicit script extension.')
dest_fmt['extension'] = auto_ext
if not args.output and nb_file != '-':
nb_dest = full_path(base_path(nb_file, args.input_format), dest_fmt)
# Set the kernel
if args.set_kernel:
if args.set_kernel == '-':
language = notebook.metadata.get('jupytext', {}).get('main_language') \
or notebook.metadata['kernelspec']['language']
if not language:
raise ValueError('Cannot infer a kernel as notebook language is not defined')
kernelspec = kernelspec_from_language(language)
if not kernelspec:
raise ValueError('Found no kernel for {}'.format(language))
else:
try:
kernelspec = get_kernel_spec(args.set_kernel)
except KeyError:
raise KeyError('Please choose a kernel name among {}'
.format([name for name in find_kernel_specs()]))
kernelspec = {'name': args.set_kernel,
'language': kernelspec.language,
'display_name': kernelspec.display_name}
args.update_metadata['kernelspec'] = kernelspec
# Update the metadata
if args.update_metadata:
log("[jupytext] Updating notebook metadata with '{}'".format(json.dumps(args.update_metadata)))
# Are we updating a text file that has a metadata filter? #212
if fmt['extension'] != '.ipynb' and \
notebook.metadata.get('jupytext', {}).get('notebook_metadata_filter') == '-all':
notebook.metadata.get('jupytext', {}).pop('notebook_metadata_filter')
recursive_update(notebook.metadata, args.update_metadata)
if 'kernelspec' in args.update_metadata and 'main_language' in notebook.metadata.get('jupytext', {}):
notebook.metadata['jupytext'].pop('main_language')
# Read paired notebooks
if args.sync:
set_prefix_and_suffix(fmt, notebook, nb_file)
notebook, inputs_nb_file, outputs_nb_file = load_paired_notebook(notebook, fmt, nb_file, log)
# II. ### Apply commands onto the notebook ###
# Pipe the notebook into the desired commands
for cmd in args.pipe or []:
notebook = pipe_notebook(notebook, cmd, args.pipe_fmt)
# and/or test the desired commands onto the notebook
for cmd in args.check or []:
pipe_notebook(notebook, cmd, args.pipe_fmt, update=False)
# III. ### Possible actions ###
modified = args.update_metadata or args.pipe or args.set_kernel
# a. Test round trip conversion
if args.test or args.test_strict:
try:
test_round_trip_conversion(notebook, dest_fmt,
update=args.update,
allow_expected_differences=not args.test_strict,
stop_on_first_error=args.stop_on_first_error)
except NotebookDifference as err:
round_trip_conversion_errors += 1
sys.stdout.write('{}: {}'.format(nb_file, str(err)))
continue
# b. Output to the desired file or format
if nb_dest:
if nb_dest == nb_file and not dest_fmt:
dest_fmt = fmt
# Test consistency between dest name and output format
if dest_fmt and nb_dest != '-':
base_path(nb_dest, dest_fmt)
# Describe what jupytext is doing
if os.path.isfile(nb_dest) and args.update:
if not nb_dest.endswith('.ipynb'):
raise ValueError('--update is only for ipynb files')
action = ' (destination file updated)'
check_file_version(notebook, nb_file, nb_dest)
combine_inputs_with_outputs(notebook, readf(nb_dest), fmt)
elif os.path.isfile(nb_dest):
action = ' (destination file replaced)'
else:
action = ''
log('[jupytext] Writing {nb_dest}{format}{action}'
.format(nb_dest=nb_dest,
format=' in format ' + short_form_one_format(
dest_fmt) if dest_fmt and 'format_name' in dest_fmt else '',
action=action))
writef_git_add(notebook, nb_dest, dest_fmt)
# c. Synchronize paired notebooks
if args.sync:
# Also update the original notebook if the notebook was modified
if modified:
inputs_nb_file = outputs_nb_file = None
formats = notebook.metadata['jupytext']['formats']
for ipynb in [True, False]:
# Write first format last so that it is the most recent file
for alt_path, alt_fmt in paired_paths(nb_file, fmt, formats)[::-1]:
# Write ipynb first for compatibility with our contents manager
if alt_path.endswith('.ipynb') != ipynb:
continue
# Do not write the ipynb file if it was not modified
# But, always write text representations to make sure they are the most recent
if alt_path == inputs_nb_file and alt_path == outputs_nb_file:
continue
log("[jupytext] Updating '{}'".format(alt_path))
writef_git_add(notebook, alt_path, alt_fmt)
return round_trip_conversion_errors
