def create_kernel_json(self, location, interpreter):
python_lib_contents = listdir("{0}/python/lib".format(self.spark_home))
try:
py4j_zip = list(filter( lambda filename: "py4j" in filename, python_lib_contents))[0]
except:
self.log.warn('Unable to find py4j, installing without PySpark support.')
kernel_spec = KernelSpec()
interpreter_lang = INTERPRETER_LANGUAGES[interpreter]
kernel_spec.display_name = '{} - {}'.format(self.kernel_name, interpreter)
kernel_spec.language = interpreter_lang
kernel_spec.argv = [os.path.join(location, 'bin', 'run.sh'), '--profile', '{connection_file}']
kernel_spec.env = {
DEFAULT_INTERPRETER : interpreter,
# The SPARK_OPTS values are stored in TOREE_SPARK_OPTS to allow the two values to be merged when kernels
# are run. This allows values to be specified during install, but also during runtime.
TOREE_SPARK_OPTS : self.spark_opts,
SPARK_HOME : self.spark_home,
PYTHON_PATH : '{0}/python:{0}/python/lib/{1}'.format(self.spark_home, py4j_zip)
}
kernel_json_file = os.path.join(location, 'kernel.json')
self.log.debug('Creating kernel json file for {}'.format(interpreter))
with open(kernel_json_file, 'w+') as f:
json.dump(kernel_spec.to_dict(), f, indent=2)
def julia_spec(kernel_path=None, *, julia_path='/usr/bin/julia'):
if kernel_path is None:
kernel_path = os.path.join(os.path.dirname(find_ijulia()), "kernel.jl")
julia_argv = [
julia_path, '-i', '--startup-file=yes', '--color=yes', kernel_path,
'{connection_file}'
]
return KernelSpec(display_name="julia",
env={},
language="julia",
argv=julia_argv)
def create_kernel_json(self, location, interpreter):
python_lib_contents = listdir("{0}/python/lib".format(self.spark_home))
try:
py4j_zip = list(filter( lambda filename: "py4j" in filename, python_lib_contents))[0]
except:
self.log.warn('Unable to find py4j, installing without PySpark support.')
kernel_spec = KernelSpec()
interpreter_lang = INTERPRETER_LANGUAGES[interpreter]
kernel_spec.display_name = '{} - {}'.format(self.kernel_name, interpreter)
kernel_spec.language = interpreter_lang
kernel_spec.argv = [os.path.join(location, 'bin', 'run.sh'), '--profile', '{connection_file}']
kernel_spec.env = {
DEFAULT_INTERPRETER : interpreter,
# The SPARK_OPTS values are stored in TOREE_SPARK_OPTS to allow the two values to be merged when kernels
# are run. This allows values to be specified during install, but also during runtime.
TOREE_SPARK_OPTS : self.spark_opts,
TOREE_OPTS : self.toree_opts,
SPARK_HOME : self.spark_home,
PYTHON_PATH : '{0}/python:{0}/python/lib/{1}'.format(self.spark_home, py4j_zip),
PYTHON_EXEC : self.python_exec
}
kernel_json_file = os.path.join(location, 'kernel.json')
self.log.debug('Creating kernel json file for {}'.format(interpreter))
with open(kernel_json_file, 'w+') as f:
json.dump(kernel_spec.to_dict(), f, indent=2)
def venv_kernel_specs(self):
python_dirs = self.find_python_paths()
kspecs = {}
for venv_name, venv_dir in python_dirs.items():
exe_name = os.path.join(venv_dir, 'bin/python')
kspec_dict = {
"argv":
[exe_name, "-m", "IPython.kernel", "-f", "{connection_file}"],
"display_name":
"Environment ({})".format(venv_name),
"env": {}
}
kspecs.update({venv_name: KernelSpec(**kspec_dict)})
return kspecs
def _conda_kspecs(self):
""" Get (or refresh) the cache of conda kernels
"""
if self._conda_info is None:
return {}
expiry = self._conda_kernels_cache_expiry
if expiry is not None and expiry >= time.time():
return self._conda_kernels_cache
kspecs = {}
for name, info in self._all_specs().items():
kspecs[name] = KernelSpec(**info)
self._conda_kernels_cache_expiry = time.time() + CACHE_TIMEOUT
self._conda_kernels_cache = kspecs
return kspecs
def create_kernel_json(self, location, interpreter):
kernel_spec = KernelSpec()
interpreter_lang = INTERPRETER_LANGUAGES[interpreter]
kernel_spec.display_name = '{} - {}'.format(self.kernel_name, interpreter)
kernel_spec.language = interpreter_lang
kernel_spec.argv = [os.path.join(location, 'bin', 'run.sh'), '--profile', '{connection_file}']
kernel_spec.env = {
DEFAULT_INTERPRETER : interpreter,
# The SPARK_OPTS values are stored in TOREE_SPARK_OPTS to allow the two values to be merged when kernels
# are run. This allows values to be specified during install, but also during runtime.
TOREE_SPARK_OPTS : self.spark_opts,
TOREE_OPTS : self.toree_opts,
SPARK_HOME : self.spark_home,
}
kernel_json_file = os.path.join(location, 'kernel.json')
self.log.debug('Creating kernel json file for {}'.format(interpreter))
with open(kernel_json_file, 'w+') as f:
json.dump(kernel_spec.to_dict(), f, indent=2)
def _load_conda_kspecs(self):
""" Create a kernelspec for each of the envs where jupyter is installed
"""
kspecs = {}
for name, info in self._all_envs().items():
executable = info['executable']
display_name = info['display_name']
if info['language_key'] == 'py':
kspec = {
"argv":
[executable, "-m", "ipykernel", "-f", "{connection_file}"],
"display_name":
display_name,
"language":
"python",
"env": {},
"resource_dir":
join(dirname(abspath(__file__)), "logos", "python")
}
elif info['language_key'] == 'r':
kspec = {
"argv": [
executable, "--slave", "-e", "IRkernel::main()",
"--args", "{connection_file}"
],
"display_name":
display_name,
"language":
"R",
"env": {},
"resource_dir":
join(dirname(abspath(__file__)), "logos", "r")
}
kspecs.update({name: KernelSpec(**kspec)})
return kspecs
def test_connection_file_real_path():
"""Verify realpath is used when formatting connection file"""
with mock.patch("os.path.realpath") as patched_realpath:
patched_realpath.return_value = "foobar"
km = KernelManager(
connection_file=os.path.join(tempfile.gettempdir(), "kernel-test.json"),
kernel_name="test_kernel",
)
# KernelSpec and launch args have to be mocked as we don't have an actual kernel on disk
km._kernel_spec = KernelSpec(
resource_dir="test",
**{
"argv": ["python.exe", "-m", "test_kernel", "-f", "{connection_file}"],
"env": {},
"display_name": "test_kernel",
"language": "python",
"metadata": {},
},
)
km._launch_args = {}
cmds = km.format_kernel_cmd()
assert cmds[4] == "foobar"
def create_kernel_json(self, location, interpreter):
kernel_spec = KernelSpec()
interpreter_lang = INTERPRETER_LANGUAGES[interpreter]
kernel_spec.display_name = '{} - {}'.format(self.kernel_name, interpreter)
kernel_spec.language = interpreter_lang
kernel_spec.argv = [os.path.join(location, 'bin', 'run.sh'), '--profile', '{connection_file}']
kernel_spec.env = {
DEFAULT_INTERPRETER : interpreter,
# The SPARK_OPTS values are stored in TOREE_SPARK_OPTS to allow the two values to be merged when kernels
# are run. This allows values to be specified during install, but also during runtime.
TOREE_SPARK_OPTS : self.spark_opts,
SPARK_HOME : self.spark_home,
PYTHON_PATH : '{0}/python:{0}/python/lib/py4j-0.8.2.1-src.zip'.format(self.spark_home)
}
kernel_json_file = os.path.join(location, 'kernel.json')
self.log.debug('Creating kernel json file for {}'.format(interpreter))
with open(kernel_json_file, 'w+') as f:
json.dump(kernel_spec.to_dict(), f, indent=2)
def cache_item_to_kernel_spec(cache_item: CacheItemType) -> KernelSpec:
"""Converts a CacheItemType to a KernelSpec instance for user consumption."""
kernel_spec = KernelSpec(resource_dir=cache_item["resource_dir"], **cache_item["spec"])
return kernel_spec
def kernel_spec_to_cache_item(kernelspec: KernelSpec) -> CacheItemType:
"""Converts a KernelSpec instance to a CacheItemType for storage into the cache."""
cache_item = dict()
cache_item["spec"] = kernelspec.to_dict()
cache_item["resource_dir"] = kernelspec.resource_dir
return cache_item
def cache_item_to_kernel_spec(cache_item: CacheItemType) -> KernelSpec:
"""Converts a CacheItemType to a KernelSpec instance for user consumption."""
return KernelSpec.from_resource_dir(cache_item['resource_dir'])