def run_notebook(nb):
km = KernelManager()
km.start_kernel(stderr=open(os.devnull, 'w'))
kc = km.client()
kc.start_channels()
try:
kc.wait_for_ready()
except AttributeError:
# IPython < 3
kc.kernel_info()
while True:
msg = kc.get_shell_msg(block=True, timeout=30)
if msg['msg_type'] == 'kernel_info_reply':
break
# Flush IOPub channel
while True:
try:
msg = kc.get_iopub_msg(block=True, timeout=0.2)
except Empty:
break
# simple ping:
kc.execute("pass")
kc.get_shell_msg()
cells = 0
failures = 0
if hasattr(nb, 'worksheets'):
# nobody uses more than 1 worksheet
ws = nb.worksheets[0]
else:
# no more worksheet level in new format
ws = nb
for cell in ws.cells:
if cell.cell_type != 'code':
continue
outputs, failed = run_cell(kc, cell)
cell.outputs = outputs
cell['prompt_number'] = cells
failures += failed
cells += 1
sys.stdout.write('.')
sys.stdout.flush()
print()
print("ran %3i cells" % cells)
if failures:
print(" %3i cells raised exceptions" % failures)
kc.stop_channels()
km.shutdown_kernel()
del km
def start_kernel(
cls,
kernelspec_name=None,
connection_info=None,
connection_name=None,
):
"""Start kernel and return a `Kernel` instance."""
kernel_id = uuid.uuid4()
if kernelspec_name:
kernel_manager = KernelManager(kernel_name=kernelspec_name)
kernel_manager.start_kernel()
elif connection_info:
kernel_manager = KernelManager()
kernel_manager.load_connection_info(connection_info)
# `KernelManager.kernel_name` is not automatically set from connection info.
kernel_manager.kernel_name = connection_info.get("kernel_name", "")
else:
raise Exception(
"You must specify any of {`kernelspec_name`, `connection_info`}."
)
kernel = KernelConnection(kernel_id,
kernel_manager,
cls,
connection_name=connection_name,
logger=cls.logger)
cls.kernels[kernel_id] = kernel
return kernel
def test_connection_file_real_path():
""" Verify realpath is used when formatting connection file """
with mock.patch('os.path.realpath') as patched_realpath:
patched_realpath.return_value = 'foobar'
km = KernelManager(connection_file=os.path.join(
tempfile.gettempdir(), "kernel-test.json"),
kernel_name='test_kernel')
# KernelSpec and launch args have to be mocked as we don't have an actual kernel on disk
km._kernel_spec = KernelSpec(resource_dir='test',
**{
"argv": [
"python.exe",
"-m",
"test_kernel",
"-f",
"{connection_file}"
],
"env": {},
"display_name": "test_kernel",
"language": "python",
"metadata": {}
})
km._launch_args = {}
cmds = km.format_kernel_cmd()
assert cmds[4] is 'foobar'
def run_notebook(nb):
km = KernelManager()
km.start_kernel(stderr=open(os.devnull, 'w'))
kc = km.client()
kc.start_channels()
cells = failures = 0
for cell in nb.cells:
if cell.cell_type != 'code':
continue
kc.execute(cell.source)
# Wait to finish
try:
reply = kc.get_shell_msg()['content']
except Empty:
reply = {'status': 'error', 'traceback': ["Cell execution timed out!"]}
if reply['status'] == 'error':
failures += 1
print("\nFAILURE:")
print(cell.source)
print('-----')
print("Raised:")
print('\n'.join(reply['traceback']))
cells += 1
sys.stdout.write('.')
kc.stop_channels()
km.shutdown_kernel()
return cells, failures
def start(self):
self.km = KernelManager()
self.km.kernel_name = self.kernel_name
self.km.start_kernel(extra_arguments=[self.context])
self.kc = self.km.client()
self.kc.start_channels()
self.kc.wait_for_ready()
def start(self):
self.km = KernelManager()
self.km.kernel_name = self.kernel_name
self.km.start_kernel()
self.kc = self.km.client()
self.kc.start_channels()
self.kc.wait_for_ready()
def __enter__(self):
self.km = KernelManager()
self.km.start_kernel(extra_arguments=self.extra_arguments,
stderr=open(os.devnull, 'w'))
self.kc = self.km.client()
self.kc.start_channels()
self.iopub = self.kc.iopub_channel
self.shell = self.kc.shell_channel
# run %pylab inline, because some notebooks assume this
# even though they shouldn't
self.shell.send("pass")
self.shell.get_msg()
while True:
try:
self.iopub.get_msg(timeout=0.05)
except Exception as e:
if repr(e) == 'Empty()':
break
# we got a real error so raise it
raise
self.cmd_list = []
self.msg_list = {}
return self
def set_contents(self, contents):
self.contents = json.loads(contents)
self.contents['notebook']['metadata']['pixiedust'] = {}
kernel_spec = self.contents['notebook']['metadata']['kernelspec']
km = KernelManager(kernel_name=kernel_spec['name'])
with warnings.catch_warnings():
warnings.simplefilter("ignore")
self.kernel_spec = json.dumps(km.kernel_spec.to_dict(), indent=4, sort_keys=True)
def _make_kernel_manager(self, kernel_name, group_id):
"""Creates a new kernel manager for given kernel and group id if none exists, and returns it.
Args:
kernel_name (str): the kernel
group_id (str): item group that will execute using this kernel
Returns:
KernelManager
"""
if group_id is None:
# Execute in isolation
return KernelManager(kernel_name=kernel_name)
key = (kernel_name, group_id)
if key not in self._kernel_managers:
self._kernel_managers[key] = KernelManager(kernel_name=kernel_name)
return self._kernel_managers[key]
def start(self, kernel_name):
self.km = KernelManager()
self.km.kernel_name = kernel_name
self.km.start_kernel(extra_arguments=[self._context_base64()])
atexit.register(self.stop_kernel)
self.kc = self.km.client()
self.kc.start_channels()
try:
self.kc.wait_for_ready()
print("{} started successfully\n".format(kernel_name.capitalize()))
except AttributeError:
self._wait_for_ready_backport()
def __init__(self, shell):
super(GroovyMagics, self).__init__(shell)
self.km = KernelManager()
self.km.kernel_name = 'groovy'
self.km.start_kernel()
self.kc = self.km.client()
self.kc.start_channels()
try:
self.kc.wait_for_ready()
print("Groovy started successfully\n")
except AttributeError:
self._wait_for_ready_backport()
def start(self):
self.km = KernelManager()
self.km.kernel_name = 'groovy'
self.km.start_kernel()
atexit.register(self.stop_kernel)
self.kc = self.km.client()
self.kc.start_channels()
try:
self.kc.wait_for_ready()
print("Groovy started successfully\n")
except AttributeError:
self._wait_for_ready_backport()
def start():
self.manager = KernelManager(kernel_name=self.name)
self.manager.start_kernel()
self.client = self.manager.blocking_client()
self.client.start_channels()
try:
self.client.wait_for_ready(timeout=timeout)
except TimeoutError: # pragma: no cover
self.manager.shutdown_kernel()
return False
else:
self.client.execute_interactive(self.init_code)
return self.client
def createKernelSpecIfNeeded(kernelName=__TEST_KERNEL_NAME__, useSpark=False):
try:
km = KernelManager(kernel_name=kernelName)
km.kernel_spec
print("Kernal already exists!")
return None
except NoSuchKernel:
print("Creating new Kernel {} target: {}".format(kernelName, useSpark))
if useSpark:
sparkHome = os.environ["SPARK_HOME"]
overrides = {
"argv":
["python", "-m", "ipykernel", "-f", "{connection_file}"],
"env": {
"SCALA_HOME":
"{0}".format(os.environ["SCALA_HOME"]),
"SPARK_HOME":
"{0}".format(sparkHome),
"PYTHONPATH":
"{0}/python/:{0}/python/lib/py4j-0.9-src.zip".format(
sparkHome),
"PYTHONSTARTUP":
"{0}/python/pyspark/shell.py".format(sparkHome),
"PYSPARK_SUBMIT_ARGS":
"--driver-class-path {0}/data/libs/* --master local[10] pyspark-shell"
.format(
os.environ.get("PIXIEDUST_HOME",
os.path.expanduser('~'))),
"SPARK_DRIVER_MEMORY":
"10G",
"SPARK_LOCAL_IP":
"127.0.0.1"
}
}
else:
overrides = {
"argv":
["python", "-m", "ipykernel", "-f", "{connection_file}"],
"env": {
"PIXIEDUST_HOME":
os.environ.get("PIXIEDUST_HOME", os.path.expanduser('~'))
}
}
path = write_kernel_spec(overrides=overrides)
dest = KernelSpecManager().install_kernel_spec(path,
kernel_name=kernelName,
user=True)
# cleanup afterward
shutil.rmtree(path)
return dest
def start_ipython_kernel(self):
kernel_name = self._get_wrapped_kernel_name()
self.km = KernelManager(
kernel_name=kernel_name,
client_class='jupyter_client.blocking.BlockingKernelClient')
self.log.debug('kernel_manager: %s', str(self.km))
self.log.info('start wrapped kernel: %s', kernel_name)
self.km.start_kernel()
self.kc = self.km.client()
self.log.debug('kernel_client: %s', str(self.kc))
self.log.debug('start_channels')
self.kc.start_channels()
try:
self.log.debug('wait for ready of wrapped kernel')
self.kc.wait_for_ready(timeout=None)
except RuntimeError:
self.kc.stop_channels()
self.km.shutdown_kernel()
raise
for channel in self.proxy_channles:
stream = getattr(self, channel + '_socket')
thread = ChannelReaderThread(self, self.kc, stream, self.session,
channel)
thread.start()
self.threads[channel] = thread
for log_dir in self.log_dirs:
if self._is_writable_dir(log_dir):
self.log_path = log_dir
break
self.log.debug('log output directory: %s', self.log_path)
if self._find_default_keyword_pattern_file() is None:
self.log.info('default keyword pattern file "%s" not found',
IPYTHON_DEFAULT_PATTERN_FILE)
try:
self._generate_default_keyword_pattern_file()
except Exception as e:
self.log.exception(
"failed to generate default keyword pattern file: %s", e)
self.exec_info = None
self.notebook_path = self.get_notebook_path()
self.log.debug('notebook_path: %s', self.notebook_path)
def new_ipy(s=''):
"""Create a new IPython kernel (optionally with extra arguments)
XXX: Allow passing of profile information here
Examples
--------
new_ipy()
"""
from jupyter_client.manager import KernelManager
km = KernelManager()
km.start_kernel()
return km_from_string(km.connection_file)
def start_new_kernel(startup_timeout=60, kernel_name='python', spykernel=False,
**kwargs):
"""Start a new kernel, and return its Manager and Client"""
km = KernelManager(kernel_name=kernel_name)
if spykernel:
km._kernel_spec = SpyderKernelSpec()
km.start_kernel(**kwargs)
kc = km.client()
kc.start_channels()
try:
kc.wait_for_ready(timeout=startup_timeout)
except RuntimeError:
kc.stop_channels()
km.shutdown_kernel()
raise
return km, kc
def start_new_kernel(startup_timeout=60, kernel_name='python', **kwargs):
"""Start a new kernel, and return its Manager and Client"""
logger.debug('Starting new kernel: "%s"' % kernel_name)
km = KernelManager(kernel_name=kernel_name,
kernel_spec_manager=NbvalKernelspecManager())
km.start_kernel(**kwargs)
kc = km.client()
kc.start_channels()
try:
kc.wait_for_ready(timeout=startup_timeout)
except RuntimeError:
logger.exception('Failure starting kernel "%s"', kernel_name)
kc.stop_channels()
km.shutdown_kernel()
raise
return km, kc
def new_ipy(s=''):
"""Create a new IPython kernel (optionally with extra arguments)
XXX: Allow passing of profile information here
Examples
--------
new_ipy()
"""
# Modified by Bo Peng.
# This package has been deprecated. Need to import from ipykernel
from jupyter_client.manager import KernelManager
km = KernelManager()
km.start_kernel()
return km_from_string(km.connection_file)
def start_ipython_kernel(self):
kernel_name = self._get_wrapped_kernel_name()
self.km = KernelManager(
kernel_name=kernel_name,
client_class='jupyter_client.blocking.BlockingKernelClient')
self.log.debug('kernel_manager: %s', str(self.km))
self.log.info('start wrapped kernel: %s', kernel_name)
self.km.start_kernel()
self.kc = self.km.client()
self.log.debug('kernel_client: %s', str(self.kc))
self.log.debug('start_channels')
self.kc.start_channels()
self.flush_stream_event.set()
try:
self.log.debug('wait for ready of wrapped kernel')
self.kc.wait_for_ready(timeout=None)
except RuntimeError:
self.kc.stop_channels()
self.km.shutdown_kernel()
raise
for channel in self.proxy_channles:
stream = getattr(self, channel + '_socket')
thread = ChannelReaderThread(self, self.kc, stream, self.session,
channel)
thread.start()
self.threads[channel] = thread
self.notebook_path = self.get_notebook_path(self.kc)
self.log_path = os.path.join(self.notebook_path, u'.log')
if not os.path.exists(
os.path.join(self.notebook_path,
IPYTHON_DEFAULT_PATTERN_FILE)):
with open(
os.path.join(self.notebook_path,
IPYTHON_DEFAULT_PATTERN_FILE), 'w') as f:
f.write(IPYTHON_DEFAULT_PATTERN)
self.exec_info = None
self._init_log()
self.log.debug('notebook_path: %s', self.notebook_path)
def test_feature(kernel_name, feature):
print("...{}".format(feature))
manager = KernelManager(kernel_name=kernel_name)
manager.start_kernel()
client = manager.client()
with open(join(feature, "provided.json")) as fp:
provided = json.load(fp)
with open(join(feature, "expected.schema.json")) as fp:
expected = json.load(fp)
msg = client.session.msg(provided['header']['msg_type'],
provided['content'])
client.shell_channel.send(msg)
response = client.shell_channel.get_msg()
jsonschema.validate(instance=response, schema=expected)
def run_notebook(notebook):
"""Run the notebook"""
kernel_manager = KernelManager()
kernel_manager.start_kernel(stderr=open(os.devnull, 'w'))
kernel_client = kernel_manager.client()
kernel_client.start_channels()
for sheet in notebook.worksheets:
for (prompt_number, cell) in enumerate(sheet.cells, 1):
if cell.cell_type != "code":
continue
cell.outputs = run_cell(kernel_client, cell)
cell.prompt_number = prompt_number
if cell.outputs and cell.outputs[0]['output_type'] == 'pyout':
cell.outputs[0]["prompt_number"] = prompt_number
kernel_manager.shutdown_kernel()
def new_ipy(s=''):
"""Create a new IPython kernel (optionally with extra arguments)
XXX: Allow passing of profile information here
Examples
--------
new_ipy()
"""
try:
from jupyter_client.manager import KernelManager
except ImportError: # For compatibility with IPython 3 or lower
from IPython.kernel import KernelManager
km = KernelManager()
km.start_kernel()
return km_from_string(km.connection_file)
import pathlib from jupyter_client.manager import KernelManager from jupyter_client.connect import find_connection_file # forkする？ m = KernelManager() m.connection_file = "kernel-me.json" m.write_connection_file() info = m.get_connection_info() print(info) print(m.connection_file) print(find_connection_file()) print(find_connection_file(pathlib.Path(m.connection_file).name)) # m.load_connection_file() m.blocking_client()
def km_from_string(s=''):
"""create kernel manager from IPKernelApp string
such as '--shell=47378 --iopub=39859 --stdin=36778 --hb=52668' for IPython 0.11
or just 'kernel-12345.json' for IPython 0.12
"""
try:
import IPython
except ImportError:
raise ImportError("Could not find IPython. " + _install_instructions)
# Modified by BoPeng
#
#from IPython.config.loader import KeyValueConfigLoader
from traitlets.config.loader import KeyValueConfigLoader
try:
# Updated by Bo Peng for module names
from jupyter_client.manager import KernelManager
from jupyter_client import find_connection_file
except ImportError:
# IPython < 1.0
from IPython.zmq.blockingkernelmanager import BlockingKernelManager as KernelManager
from IPython.zmq.kernelapp import kernel_aliases
try:
from IPython.lib.kernel import find_connection_file
except ImportError:
# < 0.12, no find_connection_file
pass
global km, kc, send
s = s.replace('--existing', '')
if 'connection_file' in KernelManager.class_trait_names():
# 0.12 uses files instead of a collection of ports
# include default IPython search path
# filefind also allows for absolute paths, in which case the search
# is ignored
try:
# XXX: the following approach will be brittle, depending on what
# connection strings will end up looking like in the future, and
# whether or not they are allowed to have spaces. I'll have to sync
# up with the IPython team to address these issues -pi
if '--profile' in s:
k,p = s.split('--profile')
k = k.lstrip().rstrip() # kernel part of the string
p = p.lstrip().rstrip() # profile part of the string
fullpath = find_connection_file(k,p)
else:
fullpath = find_connection_file(s.lstrip().rstrip())
except IOError as e:
echo(":IPython " + s + " failed", "Info")
echo("^-- failed '" + s + "' not found", "Error")
return
km = KernelManager(connection_file = fullpath)
km.load_connection_file()
else:
if s == '':
echo(":IPython 0.11 requires the full connection string")
return
loader = KeyValueConfigLoader(s.split(), aliases=kernel_aliases)
cfg = loader.load_config()['KernelApp']
try:
km = KernelManager(
shell_address=(ip, cfg['shell_port']),
sub_address=(ip, cfg['iopub_port']),
stdin_address=(ip, cfg['stdin_port']),
hb_address=(ip, cfg['hb_port']))
except KeyError as e:
echo(":IPython " +s + " failed", "Info")
echo("^-- failed --"+e.message.replace('_port','')+" not specified", "Error")
return
try:
kc = km.client()
except AttributeError:
# 0.13
kc = km
kc.start_channels()
try:
send = kc.execute
except AttributeError:
# < 3.0
send = kc.shell_channel.execute
#XXX: backwards compatibility for IPython < 0.13
try:
import inspect
sc = kc.shell_channel
num_oinfo_args = len(inspect.getargspec(sc.object_info).args)
if num_oinfo_args == 2:
# patch the object_info method which used to only take one argument
klass = sc.__class__
klass._oinfo_orig = klass.object_info
klass.object_info = lambda s,x,y: s._oinfo_orig(x)
except Exception:
pass
#XXX: backwards compatibility for IPython < 1.0
if not hasattr(kc, 'iopub_channel'):
kc.iopub_channel = kc.sub_channel
# now that we're connect to an ipython kernel, activate completion
# machinery, but do so only for the local buffer if the user added the
# following line the vimrc:
# let g:ipy_completefunc = 'local'
vim.command("""
if g:ipy_completefunc == 'global'
set completefunc=CompleteIPython
elseif g:ipy_completefunc == 'local'
setl completefunc=CompleteIPython
endif
""")
# also activate GUI doc balloons if in gvim
vim.command("""
if has('balloon_eval')
set bexpr=IPythonBalloonExpr()
endif
""")
set_pid()
return km
def make_manager(self, name):
return KernelManager(kernel_spec_manager=self.cksm, kernel_name=name)
def parse_command_line(self, argv):
silent = "--silent" in argv
silent_spark_version = None
if silent:
argv.remove("--silent")
arg_str = None
for i, arg in enumerate(argv):
if arg.startswith("--spark"):
arg_str = arg
silent_spark_version = arg_str[len("--spark") + 1:]
if silent_spark_version not in self.spark_download_versions:
print("Invalid Spark version {}".format(
silent_spark_version))
self.exit(1)
break
if arg_str:
argv.remove(arg_str)
super(InstallKernelSpec, self).parse_command_line(argv)
self.pixiedust_home = os.environ.get(
"PIXIEDUST_HOME", "{}{}pixiedust".format(os.path.expanduser('~'),
os.sep))
if silent:
answer = 'y'
else:
answer = self.confirm("Step 1: PIXIEDUST_HOME: {0}".format(
self.pixiedust_home))
if answer != 'y':
self.pixiedust_home = input(
self.hilite("Please enter a PIXIEDUST_HOME location: "))
if not os.path.exists(self.pixiedust_home):
create = self.confirm(
"Directory {0} does not exist".format(self.pixiedust_home),
"Create")
if create != 'y':
self.exit(1)
else:
os.makedirs(self.pixiedust_home)
self.pixiedust_bin = os.path.join(self.pixiedust_home, "bin")
download_spark = False
if silent:
self.spark_home = os.environ.get("SPARK_HOME", None)
if not self.spark_home:
download_spark = True
self.spark_home = "{}{}spark".format(os.path.expanduser('~'),
os.sep)
else:
first_prompt = True
while True:
self.spark_home = os.environ.get(
"SPARK_HOME",
self.ensureDir(os.path.join(self.pixiedust_bin, "spark")))
if self.spark_home:
answer = self.confirm("Step 2: SPARK_HOME: {0}".format(
self.spark_home))
if answer != 'y':
self.spark_home = input(
self.hilite(
"Step 2: Please enter a SPARK_HOME location: ")
)
else:
if first_prompt:
first_prompt = False
prompt = "Step 2: Please enter a SPARK_HOME location: "
else:
prompt = "Please enter a SPARK_HOME location: "
self.spark_home = input(self.hilite(prompt))
while self.spark_home.rfind(
os.sep) == len(self.spark_home) - 1:
self.spark_home = self.spark_home[0:len(self.spark_home) -
1]
if not os.path.exists(self.spark_home):
create = self.confirm(
"Directory {0} does not exist".format(self.spark_home),
"Create")
if create != 'y':
continue
else:
os.makedirs(self.spark_home)
download_spark = True
break
elif not os.path.exists('{}{}bin{}pyspark'.format(
self.spark_home, os.sep, os.sep)):
existingInstalls = [
d for d in os.listdir(self.spark_home)
if os.path.exists(
os.path.join(self.spark_home, d, 'bin', 'pyspark'))
]
if len(existingInstalls) == 0:
download = self.confirm(
"Directory {0} does not contain a valid SPARK install"
.format(self.spark_home), "Download Spark")
if download == 'y':
download_spark = True
break
else:
existingInstalls.append("Create a new spark Install")
print(
"Select an existing spark install or create a new one"
)
message = "\n".join([
"{}. {}".format(k + 1, v)
for k, v in enumerate(existingInstalls)
])
while True:
try:
answer = int(
input(
self.hilite(message) +
"\n\tEnter your selection: ")) - 1
if answer < 0 or answer >= len(
existingInstalls):
raise Exception(
"Please pick a number within the specified values"
)
break
except Exception as e:
print("Invalid selection: {}".format(e))
download_spark = (answer == len(existingInstalls) - 1)
if not download_spark:
self.spark_home = os.path.join(
self.spark_home, existingInstalls[answer])
break
else:
break
if download_spark:
self.download_spark(silent, silent_spark_version)
scala_version = None
spark_version = self.get_spark_version()
if spark_version is None:
print("Unable to obtain Spark version")
self.exit(1)
elif spark_version[0] == 1:
scala_version = '2.10' # Spark 1.x = Scala 2.10
elif spark_version[0] == 2:
scala_version = '2.11' # Spark 2.x = Scala 2.11
else:
print("Invalid Spark version {}".format(spark_version))
self.exit(1)
#download spark-cloudant
sparkCloudantUrl = "https://github.com/cloudant-labs/spark-cloudant/releases/download/v1.6.4/cloudant-spark-v1.6.4-167.jar"
if spark_version[0] == 2:
sparkCloudantUrl = "https://github.com/cloudant-labs/spark-cloudant/releases/download/v2.0.0/cloudant-spark-v2.0.0-185.jar"
try:
self.sparkCloudantPath = self.downloadFileToDir(
sparkCloudantUrl, targetDir=self.pixiedust_bin)
print("downloaded spark cloudant jar: {0}".format(
self.sparkCloudantPath))
except Exception as e:
print(
"Error downloading Cloudant Jar: {}. Install will continue without the spark Cloudant connector"
.format(e))
#download spark csv connector if in spark 1.6
self.sparkCSVPath = None
self.commonsCSVPath = None
if spark_version[0] == 1:
try:
self.sparkCSVPath = self.downloadPackage(
"com.databricks:spark-csv_2.10:1.5.0")
self.commonsCSVPath = self.downloadPackage(
"org.apache.commons:commons-csv:0")
except Exception as e:
print(
"Error downloading csv connector Jar: {}. Install will continue without it"
.format(e))
download_scala = False
if silent:
self.scala_home = os.environ.get("SCALA_HOME", None)
if not self.scala_home:
download_scala = True
self.scala_home = "{}{}scala".format(os.path.expanduser('~'),
os.sep)
else:
first_prompt = True
while True:
self.scala_home = os.environ.get(
"SCALA_HOME",
self.ensureDir(os.path.join(self.pixiedust_bin, "scala")))
if self.scala_home:
answer = self.confirm("Step 3: SCALA_HOME: {0}".format(
self.scala_home))
if answer != 'y':
self.scala_home = input(
self.hilite(
"Step 3: Please enter a SCALA_HOME location: ")
)
else:
if first_prompt:
first_prompt = False
prompt = "Step 3: Please enter a SCALA_HOME location: "
else:
prompt = "Please enter a SCALA_HOME location: "
self.scala_home = input(self.hilite(prompt))
while self.scala_home.rfind(
os.sep) == len(self.scala_home) - 1:
self.scala_home = self.scala_home[0:len(self.scala_home) -
1]
if not os.path.exists(self.scala_home):
create = self.confirm(
"Directory {0} does not exist".format(self.scala_home),
"Create")
if create != 'y':
continue
else:
os.makedirs(self.scala_home)
download_scala = True
break
elif not os.path.exists('{}{}bin{}scala'.format(
self.scala_home, os.sep, os.sep)):
def acceptScalaVersion(dir):
version = self.get_scala_version_from_dir(
os.path.join(self.scala_home, dir))
return version is not None and str(
version[0]) + '.' + str(
version[1]) == scala_version
existingInstalls = [
d for d in os.listdir(self.scala_home)
if acceptScalaVersion(d)
]
if len(existingInstalls) == 0:
download = self.confirm(
"Directory {0} does not contain a valid scala install"
.format(self.scala_home), "Download Scala")
if download == 'y':
download_scala = True
break
else:
existingInstalls.append("Create a new scala Install")
print(
"Select an existing scala install or create a new one"
)
message = "\n".join([
"{}. {}".format(k + 1, v)
for k, v in enumerate(existingInstalls)
])
while True:
try:
answer = int(
input(
self.hilite(message) +
"\n\tEnter your selection: ")) - 1
if answer < 0 or answer >= len(
existingInstalls):
raise Exception(
"Please pick a number within the specified values"
)
break
except Exception as e:
print("Invalid selection: {}".format(e))
download_scala = (answer == len(existingInstalls) - 1)
if not download_scala:
self.scala_home = os.path.join(
self.scala_home, existingInstalls[answer])
break
else:
installed_scala_version = self.get_scala_version()
if not installed_scala_version or (
str(installed_scala_version[0]) + '.' +
str(installed_scala_version[1])) != scala_version:
print(
"A different version of Scala {0} is already installed in this directory."
.format(installed_scala_version))
continue
else:
break
if download_scala:
self.download_scala(scala_version)
self.kernelName = "Python with Pixiedust (Spark {}.{})".format(
spark_version[0], spark_version[1])
if silent:
answer = 'y'
else:
answer = self.confirm("Step 4: Kernel Name: {0}".format(
self.kernelName))
if answer != 'y':
self.kernelName = input(
self.hilite("Please enter a Kernel Name: "))
try:
km = KernelManager(kernel_name=self.kernelName)
km.kernel_spec
answer = self.confirm(
"Kernel '{0}' already exists".format(self.kernelName),
"Override")
if answer != 'y':
self.exit(1)
except NoSuchKernel:
pass
dest = self.createKernelSpec()
self.pixiedust_notebooks_dir = os.path.join(self.pixiedust_home,
"notebooks")
if not os.path.isdir(self.pixiedust_notebooks_dir):
os.makedirs(self.pixiedust_notebooks_dir)
print("Downloading intro notebooks into {}".format(
self.pixiedust_notebooks_dir))
self.downloadIntroNotebooks()
print("\n\n{}".format("#" * 100))
print("#\tCongratulations: Kernel {0} was successfully created in {1}".
format(self.kernelName, dest))
print(
"#\tYou can start the Notebook server with the following command:")
print("#\t\t{}".format(
self.hilite("jupyter notebook {}".format(
self.pixiedust_notebooks_dir))))
print("{}".format("#" * 100))
def parse_command_line(self, argv):
silent = "--silent" in argv
silent_spark_version = None
if silent:
argv.remove("--silent")
arg_str = None
for i, arg in enumerate(argv):
if arg.startswith("--spark"):
arg_str = arg
silent_spark_version = arg_str[len("--spark")+1:]
if silent_spark_version not in self.spark_download_versions:
print("Invalid Spark version {}".format(silent_spark_version))
self.exit(1)
break
if arg_str:
argv.remove(arg_str)
super(InstallKernelSpec, self).parse_command_line(argv)
self.pixiedust_home = os.environ.get("PIXIEDUST_HOME", "{}{}pixiedust".format(os.path.expanduser('~'), os.sep))
if silent:
answer = 'y'
else:
answer = self.confirm(
"Step 1: PIXIEDUST_HOME: {0}".format(self.pixiedust_home)
)
if answer != 'y':
self.pixiedust_home = input(self.hilite("Please enter a PIXIEDUST_HOME location: "))
if not os.path.exists(self.pixiedust_home):
create = self.confirm("Directory {0} does not exist".format(self.pixiedust_home), "Create")
if create != 'y':
self.exit(1)
else:
os.makedirs(self.pixiedust_home)
download_spark = False
if silent:
self.spark_home = os.environ.get("SPARK_HOME", None)
if not self.spark_home:
download_spark = True
self.spark_home = "{}{}spark".format(os.path.expanduser('~'), os.sep)
else:
while True:
self.spark_home = os.environ.get("SPARK_HOME", None)
if self.spark_home:
answer = self.confirm(
"Step 2: SPARK_HOME: {0}".format(self.spark_home)
)
if answer != 'y':
self.spark_home = input(self.hilite("Step 2: Please enter a SPARK_HOME location: "))
else:
self.spark_home = input(self.hilite("Step 2: Please enter a SPARK_HOME location: "))
while self.spark_home.rfind(os.sep) == len(self.spark_home) - 1:
self.spark_home = self.spark_home[0:len(self.spark_home)-1]
if not os.path.exists(self.spark_home):
create = self.confirm("Directory {0} does not exist".format(self.spark_home), "Create")
if create != 'y':
continue
else:
os.makedirs(self.spark_home)
download_spark = True
break
elif not os.path.exists('{}{}bin{}pyspark'.format(self.spark_home, os.sep, os.sep)):
download = self.confirm("Directory {0} does not contain a valid SPARK install".format(self.spark_home), "Download Spark")
if download == 'y':
download_spark = True
break
else:
break
if download_spark:
self.download_spark(silent, silent_spark_version)
scala_version = None
spark_version = self.get_spark_version()
if spark_version is None:
print("Unable to obtain Spark version")
self.exit(1)
elif spark_version[0] == 1:
scala_version = '2.10' # Spark 1.x = Scala 2.10
elif spark_version[0] == 2:
scala_version = '2.11' # Spark 2.x = Scala 2.11
else:
print("Invalid Spark version {}".format(spark_version))
self.exit(1)
download_scala = False
if silent:
self.scala_home = os.environ.get("SCALA_HOME", None)
if not self.scala_home:
download_scala = True
self.scala_home = "{}{}scala".format(os.path.expanduser('~'), os.sep)
else:
while True:
self.scala_home = os.environ.get("SCALA_HOME", None)
if self.scala_home:
answer = self.confirm(
"Step 3: SCALA_HOME: {0}".format(self.scala_home)
)
if answer != 'y':
self.scala_home = input(self.hilite("Step 3: Please enter a SCALA_HOME location: "))
else:
self.scala_home = input(self.hilite("Step 3: Please enter a SCALA_HOME location: "))
while self.scala_home.rfind(os.sep) == len(self.scala_home) - 1:
self.scala_home = self.scala_home[0:len(self.scala_home)-1]
if not os.path.exists(self.scala_home):
create = self.confirm("Directory {0} does not exist".format(self.scala_home), "Create")
if create != 'y':
continue
else:
os.makedirs(self.scala_home)
download_scala = True
break
elif not os.path.exists('{}{}bin{}scala'.format(self.scala_home, os.sep, os.sep)):
download = self.confirm(
"Directory {0} does not contain a valid SCALA install".format(self.scala_home),
"Download Scala"
)
if download == 'y':
download_scala = True
break
else:
installed_scala_version = self.get_scala_version()
if not installed_scala_version or (str(installed_scala_version[0]) + '.' + str(installed_scala_version[1])) != scala_version:
print("Invalid Scala version {0}".format(installed_scala_version))
continue
else:
break
if download_scala:
self.download_scala(scala_version)
self.kernelName = "Python with Pixiedust (Spark {}.{})".format(spark_version[0], spark_version[1])
if silent:
answer = 'y'
else:
answer = self.confirm(
"Step 4: Kernel Name: {0}".format(self.kernelName)
)
if answer != 'y':
self.kernelName = input(self.hilite("Step 4: Please enter a Kernel Name: "))
try:
km = KernelManager(kernel_name=self.kernelName)
km.kernel_spec
answer = self.confirm(
"Kernel '{0}' already exists".format(self.kernelName), "Override"
)
if answer != 'y':
self.exit(1)
except NoSuchKernel:
pass
dest = self.createKernelSpec()
print("Kernel {0} successfully created in {1}".format(self.kernelName, dest))
def jupyter_kernel(jupyter_manager, request):
return KernelManager(
kernel_spec_manager=jupyter_manager,
kernel_name=request.param,
)
