class SyntheticExchangeConfig(ExchangeConfig):
exchange_type = Instance(ExchangeType)
adversaries = List(default_value=[])
direction = Float(default_value=.5)
volatility = Float(default_value=200)
class KernelManager(ConnectionFileMixin):
"""Manages a single kernel in a subprocess on this host.
This version starts kernels with Popen.
"""
# The PyZMQ Context to use for communication with the kernel.
context = Instance(zmq.Context)
def _context_default(self):
return zmq.Context.instance()
# the class to create with our `client` method
client_class = DottedObjectName(
'jupyter_client.blocking.BlockingKernelClient')
client_factory = Type(klass='jupyter_client.KernelClient')
def _client_factory_default(self):
return import_item(self.client_class)
def _client_class_changed(self, name, old, new):
self.client_factory = import_item(str(new))
# The kernel process with which the KernelManager is communicating.
# generally a Popen instance
kernel = Any()
kernel_spec_manager = Instance(kernelspec.KernelSpecManager)
def _kernel_spec_manager_default(self):
return kernelspec.KernelSpecManager(data_dir=self.data_dir)
def _kernel_spec_manager_changed(self):
self._kernel_spec = None
shutdown_wait_time = Float(
5.0,
config=True,
help="Time to wait for a kernel to terminate before killing it, "
"in seconds.")
kernel_name = Unicode(kernelspec.NATIVE_KERNEL_NAME)
def _kernel_name_changed(self, name, old, new):
self._kernel_spec = None
if new == 'python':
self.kernel_name = kernelspec.NATIVE_KERNEL_NAME
_kernel_spec = None
@property
def kernel_spec(self):
if self._kernel_spec is None:
self._kernel_spec = self.kernel_spec_manager.get_kernel_spec(
self.kernel_name)
return self._kernel_spec
kernel_cmd = List(Unicode(),
config=True,
help="""DEPRECATED: Use kernel_name instead.
The Popen Command to launch the kernel.
Override this if you have a custom kernel.
If kernel_cmd is specified in a configuration file,
Jupyter does not pass any arguments to the kernel,
because it cannot make any assumptions about the
arguments that the kernel understands. In particular,
this means that the kernel does not receive the
option --debug if it given on the Jupyter command line.
""")
def _kernel_cmd_changed(self, name, old, new):
warnings.warn("Setting kernel_cmd is deprecated, use kernel_spec to "
"start different kernels.")
@property
def ipykernel(self):
return self.kernel_name in {'python', 'python2', 'python3'}
# Protected traits
_launch_args = Any()
_control_socket = Any()
_restarter = Any()
autorestart = Bool(True,
config=True,
help="""Should we autorestart the kernel if it dies.""")
def __del__(self):
self._close_control_socket()
self.cleanup_connection_file()
#--------------------------------------------------------------------------
# Kernel restarter
#--------------------------------------------------------------------------
def start_restarter(self):
pass
def stop_restarter(self):
pass
def add_restart_callback(self, callback, event='restart'):
"""register a callback to be called when a kernel is restarted"""
if self._restarter is None:
return
self._restarter.add_callback(callback, event)
def remove_restart_callback(self, callback, event='restart'):
"""unregister a callback to be called when a kernel is restarted"""
if self._restarter is None:
return
self._restarter.remove_callback(callback, event)
#--------------------------------------------------------------------------
# create a Client connected to our Kernel
#--------------------------------------------------------------------------
def client(self, **kwargs):
"""Create a client configured to connect to our kernel"""
kw = {}
kw.update(self.get_connection_info(session=True))
kw.update(dict(
connection_file=self.connection_file,
parent=self,
))
# add kwargs last, for manual overrides
kw.update(kwargs)
return self.client_factory(**kw)
#--------------------------------------------------------------------------
# Kernel management
#--------------------------------------------------------------------------
def format_kernel_cmd(self, extra_arguments=None):
"""replace templated args (e.g. {connection_file})"""
extra_arguments = extra_arguments or []
if self.kernel_cmd:
cmd = self.kernel_cmd + extra_arguments
else:
cmd = self.kernel_spec.argv + extra_arguments
if cmd and cmd[0] in {
'python',
'python%i' % sys.version_info[0],
'python%i.%i' % sys.version_info[:2]
}:
# executable is 'python' or 'python3', use sys.executable.
# These will typically be the same,
# but if the current process is in an env
# and has been launched by abspath without
# activating the env, python on PATH may not be sys.executable,
# but it should be.
cmd[0] = sys.executable
ns = dict(
connection_file=self.connection_file,
prefix=sys.prefix,
)
if self.kernel_spec:
ns["resource_dir"] = self.kernel_spec.resource_dir
ns.update(self._launch_args)
pat = re.compile(r'\{([A-Za-z0-9_]+)\}')
def from_ns(match):
"""Get the key out of ns if it's there, otherwise no change."""
return ns.get(match.group(1), match.group())
return [pat.sub(from_ns, arg) for arg in cmd]
def _launch_kernel(self, kernel_cmd, **kw):
"""actually launch the kernel
override in a subclass to launch kernel subprocesses differently
"""
return launch_kernel(kernel_cmd, **kw)
# Control socket used for polite kernel shutdown
def _connect_control_socket(self):
if self._control_socket is None:
self._control_socket = self.connect_control()
self._control_socket.linger = 100
def _close_control_socket(self):
if self._control_socket is None:
return
self._control_socket.close()
self._control_socket = None
def start_kernel(self, **kw):
"""Starts a kernel on this host in a separate process.
If random ports (port=0) are being used, this method must be called
before the channels are created.
Parameters
----------
`**kw` : optional
keyword arguments that are passed down to build the kernel_cmd
and launching the kernel (e.g. Popen kwargs).
"""
if self.transport == 'tcp' and not is_local_ip(self.ip):
raise RuntimeError(
"Can only launch a kernel on a local interface. "
"Make sure that the '*_address' attributes are "
"configured properly. "
"Currently valid addresses are: %s" % local_ips())
# write connection file / get default ports
self.write_connection_file()
# save kwargs for use in restart
self._launch_args = kw.copy()
# build the Popen cmd
extra_arguments = kw.pop('extra_arguments', [])
kernel_cmd = self.format_kernel_cmd(extra_arguments=extra_arguments)
env = kw.pop('env', os.environ).copy()
# Don't allow PYTHONEXECUTABLE to be passed to kernel process.
# If set, it can bork all the things.
env.pop('PYTHONEXECUTABLE', None)
if not self.kernel_cmd:
# If kernel_cmd has been set manually, don't refer to a kernel spec
# Environment variables from kernel spec are added to os.environ
env.update(self.kernel_spec.env or {})
# launch the kernel subprocess
self.log.debug("Starting kernel: %s", kernel_cmd)
self.kernel = self._launch_kernel(kernel_cmd, env=env, **kw)
self.start_restarter()
self._connect_control_socket()
def request_shutdown(self, restart=False):
"""Send a shutdown request via control channel
"""
content = dict(restart=restart)
msg = self.session.msg("shutdown_request", content=content)
# ensure control socket is connected
self._connect_control_socket()
self.session.send(self._control_socket, msg)
def finish_shutdown(self, waittime=None, pollinterval=0.1):
"""Wait for kernel shutdown, then kill process if it doesn't shutdown.
This does not send shutdown requests - use :meth:`request_shutdown`
first.
"""
if waittime is None:
waittime = max(self.shutdown_wait_time, 0)
for i in range(int(waittime / pollinterval)):
if self.is_alive():
time.sleep(pollinterval)
else:
break
else:
# OK, we've waited long enough.
if self.has_kernel:
self.log.debug("Kernel is taking too long to finish, killing")
self._kill_kernel()
def cleanup(self, connection_file=True):
"""Clean up resources when the kernel is shut down"""
if connection_file:
self.cleanup_connection_file()
self.cleanup_ipc_files()
self._close_control_socket()
def shutdown_kernel(self, now=False, restart=False):
"""Attempts to stop the kernel process cleanly.
This attempts to shutdown the kernels cleanly by:
1. Sending it a shutdown message over the shell channel.
2. If that fails, the kernel is shutdown forcibly by sending it
a signal.
Parameters
----------
now : bool
Should the kernel be forcible killed *now*. This skips the
first, nice shutdown attempt.
restart: bool
Will this kernel be restarted after it is shutdown. When this
is True, connection files will not be cleaned up.
"""
# Stop monitoring for restarting while we shutdown.
self.stop_restarter()
if now:
self._kill_kernel()
else:
self.request_shutdown(restart=restart)
# Don't send any additional kernel kill messages immediately, to give
# the kernel a chance to properly execute shutdown actions. Wait for at
# most 1s, checking every 0.1s.
self.finish_shutdown()
self.cleanup(connection_file=not restart)
def restart_kernel(self, now=False, newports=False, **kw):
"""Restarts a kernel with the arguments that were used to launch it.
Parameters
----------
now : bool, optional
If True, the kernel is forcefully restarted *immediately*, without
having a chance to do any cleanup action. Otherwise the kernel is
given 1s to clean up before a forceful restart is issued.
In all cases the kernel is restarted, the only difference is whether
it is given a chance to perform a clean shutdown or not.
newports : bool, optional
If the old kernel was launched with random ports, this flag decides
whether the same ports and connection file will be used again.
If False, the same ports and connection file are used. This is
the default. If True, new random port numbers are chosen and a
new connection file is written. It is still possible that the newly
chosen random port numbers happen to be the same as the old ones.
`**kw` : optional
Any options specified here will overwrite those used to launch the
kernel.
"""
if self._launch_args is None:
raise RuntimeError("Cannot restart the kernel. "
"No previous call to 'start_kernel'.")
else:
# Stop currently running kernel.
self.shutdown_kernel(now=now, restart=True)
if newports:
self.cleanup_random_ports()
# Start new kernel.
self._launch_args.update(kw)
self.start_kernel(**self._launch_args)
@property
def has_kernel(self):
"""Has a kernel been started that we are managing."""
return self.kernel is not None
def _kill_kernel(self):
"""Kill the running kernel.
This is a private method, callers should use shutdown_kernel(now=True).
"""
if self.has_kernel:
# Signal the kernel to terminate (sends SIGKILL on Unix and calls
# TerminateProcess() on Win32).
try:
if hasattr(signal, 'SIGKILL'):
self.signal_kernel(signal.SIGKILL)
else:
self.kernel.kill()
except OSError as e:
# In Windows, we will get an Access Denied error if the process
# has already terminated. Ignore it.
if sys.platform == 'win32':
if e.winerror != 5:
raise
# On Unix, we may get an ESRCH error if the process has already
# terminated. Ignore it.
else:
from errno import ESRCH
if e.errno != ESRCH:
raise
# Block until the kernel terminates.
self.kernel.wait()
self.kernel = None
else:
raise RuntimeError("Cannot kill kernel. No kernel is running!")
def interrupt_kernel(self):
"""Interrupts the kernel by sending it a signal.
Unlike ``signal_kernel``, this operation is well supported on all
platforms.
"""
if self.has_kernel:
interrupt_mode = self.kernel_spec.interrupt_mode
if interrupt_mode == 'signal':
if sys.platform == 'win32':
from .win_interrupt import send_interrupt
send_interrupt(self.kernel.win32_interrupt_event)
else:
self.signal_kernel(signal.SIGINT)
elif interrupt_mode == 'message':
msg = self.session.msg("interrupt_request", content={})
self._connect_control_socket()
self.session.send(self._control_socket, msg)
else:
raise RuntimeError(
"Cannot interrupt kernel. No kernel is running!")
def signal_kernel(self, signum):
"""Sends a signal to the process group of the kernel (this
usually includes the kernel and any subprocesses spawned by
the kernel).
Note that since only SIGTERM is supported on Windows, this function is
only useful on Unix systems.
"""
if self.has_kernel:
if hasattr(os, "getpgid") and hasattr(os, "killpg"):
try:
pgid = os.getpgid(self.kernel.pid)
os.killpg(pgid, signum)
return
except OSError:
pass
self.kernel.send_signal(signum)
else:
raise RuntimeError("Cannot signal kernel. No kernel is running!")
def is_alive(self):
"""Is the kernel process still running?"""
if self.has_kernel:
if self.kernel.poll() is None:
return True
else:
return False
else:
# we don't have a kernel
return False
class ZMQTerminalInteractiveShell(TerminalInteractiveShell):
"""A subclass of TerminalInteractiveShell that uses the 0MQ kernel"""
_executing = False
_execution_state = Unicode('')
_pending_clearoutput = False
kernel_banner = Unicode('')
kernel_timeout = Float(
60,
config=True,
help="""Timeout for giving up on a kernel (in seconds).
On first connect and restart, the console tests whether the
kernel is running and responsive by sending kernel_info_requests.
This sets the timeout in seconds for how long the kernel can take
before being presumed dead.
""")
image_handler = Enum(('PIL', 'stream', 'tempfile', 'callable'),
config=True,
allow_none=True,
help="""
Handler for image type output. This is useful, for example,
when connecting to the kernel in which pylab inline backend is
activated. There are four handlers defined. 'PIL': Use
Python Imaging Library to popup image; 'stream': Use an
external program to show the image. Image will be fed into
the STDIN of the program. You will need to configure
`stream_image_handler`; 'tempfile': Use an external program to
show the image. Image will be saved in a temporally file and
the program is called with the temporally file. You will need
to configure `tempfile_image_handler`; 'callable': You can set
any Python callable which is called with the image data. You
will need to configure `callable_image_handler`.
""")
stream_image_handler = List(config=True,
help="""
Command to invoke an image viewer program when you are using
'stream' image handler. This option is a list of string where
the first element is the command itself and reminders are the
options for the command. Raw image data is given as STDIN to
the program.
""")
tempfile_image_handler = List(config=True,
help="""
Command to invoke an image viewer program when you are using
'tempfile' image handler. This option is a list of string
where the first element is the command itself and reminders
are the options for the command. You can use {file} and
{format} in the string to represent the location of the
generated image file and image format.
""")
callable_image_handler = Any(config=True,
help="""
Callable object called via 'callable' image handler with one
argument, `data`, which is `msg["content"]["data"]` where
`msg` is the message from iopub channel. For exmaple, you can
find base64 encoded PNG data as `data['image/png']`.
""")
mime_preference = List(
default_value=['image/png', 'image/jpeg', 'image/svg+xml'],
config=True,
help="""
Preferred object representation MIME type in order. First
matched MIME type will be used.
""")
manager = Instance('jupyter_client.KernelManager', allow_none=True)
client = Instance('jupyter_client.KernelClient', allow_none=True)
def _client_changed(self, name, old, new):
self.session_id = new.session.session
session_id = Unicode()
def init_completer(self):
"""Initialize the completion machinery.
This creates completion machinery that can be used by client code,
either interactively in-process (typically triggered by the readline
library), programmatically (such as in test suites) or out-of-process
(typically over the network by remote frontends).
"""
from IPython.core.completerlib import (module_completer,
magic_run_completer,
cd_completer)
self.Completer = ZMQCompleter(self, self.client, config=self.config)
self.set_hook('complete_command', module_completer, str_key='import')
self.set_hook('complete_command', module_completer, str_key='from')
self.set_hook('complete_command', magic_run_completer, str_key='%run')
self.set_hook('complete_command', cd_completer, str_key='%cd')
# Only configure readline if we truly are using readline. IPython can
# do tab-completion over the network, in GUIs, etc, where readline
# itself may be absent
if self.has_readline:
self.set_readline_completer()
def run_cell(self, cell, store_history=True):
"""Run a complete IPython cell.
Parameters
----------
cell : str
The code (including IPython code such as %magic functions) to run.
store_history : bool
If True, the raw and translated cell will be stored in IPython's
history. For user code calling back into IPython's machinery, this
should be set to False.
"""
if (not cell) or cell.isspace():
# pressing enter flushes any pending display
self.handle_iopub()
return
# flush stale replies, which could have been ignored, due to missed heartbeats
while self.client.shell_channel.msg_ready():
self.client.shell_channel.get_msg()
# execute takes 'hidden', which is the inverse of store_hist
msg_id = self.client.execute(cell, not store_history)
# first thing is wait for any side effects (output, stdin, etc.)
self._executing = True
self._execution_state = "busy"
while self._execution_state != 'idle' and self.client.is_alive():
try:
self.handle_input_request(msg_id, timeout=0.05)
except Empty:
# display intermediate print statements, etc.
self.handle_iopub(msg_id)
except ZMQError as e:
# Carry on if polling was interrupted by a signal
if e.errno != errno.EINTR:
raise
# after all of that is done, wait for the execute reply
while self.client.is_alive():
try:
self.handle_execute_reply(msg_id, timeout=0.05)
except Empty:
pass
else:
break
self._executing = False
#-----------------
# message handlers
#-----------------
def handle_execute_reply(self, msg_id, timeout=None):
msg = self.client.shell_channel.get_msg(block=False, timeout=timeout)
if msg["parent_header"].get("msg_id", None) == msg_id:
self.handle_iopub(msg_id)
content = msg["content"]
status = content['status']
if status == 'aborted':
self.write('Aborted\n')
return
elif status == 'ok':
# handle payloads
for item in content.get("payload", []):
source = item['source']
if source == 'page':
page.page(item['data']['text/plain'])
elif source == 'set_next_input':
self.set_next_input(item['text'])
elif source == 'ask_exit':
self.keepkernel = item.get('keepkernel', False)
self.ask_exit()
elif status == 'error':
pass
self.execution_count = int(content["execution_count"] + 1)
include_other_output = Bool(False,
config=True,
help="""Whether to include output from clients
other than this one sharing the same kernel.
Outputs are not displayed until enter is pressed.
""")
other_output_prefix = Unicode(
"[remote] ",
config=True,
help="""Prefix to add to outputs coming from clients other than this one.
Only relevant if include_other_output is True.
""")
def from_here(self, msg):
"""Return whether a message is from this session"""
return msg['parent_header'].get("session",
self.session_id) == self.session_id
def include_output(self, msg):
"""Return whether we should include a given output message"""
from_here = self.from_here(msg)
if msg['msg_type'] == 'execute_input':
# only echo inputs not from here
return self.include_other_output and not from_here
if self.include_other_output:
return True
else:
return from_here
def handle_iopub(self, msg_id=''):
"""Process messages on the IOPub channel
This method consumes and processes messages on the IOPub channel,
such as stdout, stderr, execute_result and status.
It only displays output that is caused by this session.
"""
while self.client.iopub_channel.msg_ready():
sub_msg = self.client.iopub_channel.get_msg()
msg_type = sub_msg['header']['msg_type']
parent = sub_msg["parent_header"]
if self.include_output(sub_msg):
if msg_type == 'status':
self._execution_state = sub_msg["content"][
"execution_state"]
elif msg_type == 'stream':
if sub_msg["content"]["name"] == "stdout":
if self._pending_clearoutput:
print("\r", file=io.stdout, end="")
self._pending_clearoutput = False
print(sub_msg["content"]["text"],
file=io.stdout,
end="")
io.stdout.flush()
elif sub_msg["content"]["name"] == "stderr":
if self._pending_clearoutput:
print("\r", file=io.stderr, end="")
self._pending_clearoutput = False
print(sub_msg["content"]["text"],
file=io.stderr,
end="")
io.stderr.flush()
elif msg_type == 'execute_result':
if self._pending_clearoutput:
print("\r", file=io.stdout, end="")
self._pending_clearoutput = False
self.execution_count = int(
sub_msg["content"]["execution_count"])
if not self.from_here(sub_msg):
sys.stdout.write(self.other_output_prefix)
format_dict = sub_msg["content"]["data"]
self.handle_rich_data(format_dict)
# taken from DisplayHook.__call__:
hook = self.displayhook
hook.start_displayhook()
hook.write_output_prompt()
hook.write_format_data(format_dict)
hook.log_output(format_dict)
hook.finish_displayhook()
elif msg_type == 'display_data':
data = sub_msg["content"]["data"]
handled = self.handle_rich_data(data)
if not handled:
if not self.from_here(sub_msg):
sys.stdout.write(self.other_output_prefix)
# if it was an image, we handled it by now
if 'text/plain' in data:
print(data['text/plain'])
elif msg_type == 'execute_input':
content = sub_msg['content']
self.execution_count = content['execution_count']
if not self.from_here(sub_msg):
sys.stdout.write(self.other_output_prefix)
sys.stdout.write(self.prompt_manager.render('in'))
sys.stdout.write(content['code'])
elif msg_type == 'clear_output':
if sub_msg["content"]["wait"]:
self._pending_clearoutput = True
else:
print("\r", file=io.stdout, end="")
elif msg_type == 'error':
for frame in sub_msg["content"]["traceback"]:
print(frame, file=io.stderr)
_imagemime = {
'image/png': 'png',
'image/jpeg': 'jpeg',
'image/svg+xml': 'svg',
}
def handle_rich_data(self, data):
for mime in self.mime_preference:
if mime in data and mime in self._imagemime:
self.handle_image(data, mime)
return True
def handle_image(self, data, mime):
handler = getattr(self, 'handle_image_{0}'.format(self.image_handler),
None)
if handler:
handler(data, mime)
def handle_image_PIL(self, data, mime):
if mime not in ('image/png', 'image/jpeg'):
return
import PIL.Image
raw = base64.decodestring(data[mime].encode('ascii'))
img = PIL.Image.open(BytesIO(raw))
img.show()
def handle_image_stream(self, data, mime):
raw = base64.decodestring(data[mime].encode('ascii'))
imageformat = self._imagemime[mime]
fmt = dict(format=imageformat)
args = [s.format(**fmt) for s in self.stream_image_handler]
with open(os.devnull, 'w') as devnull:
proc = subprocess.Popen(args,
stdin=subprocess.PIPE,
stdout=devnull,
stderr=devnull)
proc.communicate(raw)
def handle_image_tempfile(self, data, mime):
raw = base64.decodestring(data[mime].encode('ascii'))
imageformat = self._imagemime[mime]
filename = 'tmp.{0}'.format(imageformat)
with NamedFileInTemporaryDirectory(filename) as f, \
open(os.devnull, 'w') as devnull:
f.write(raw)
f.flush()
fmt = dict(file=f.name, format=imageformat)
args = [s.format(**fmt) for s in self.tempfile_image_handler]
subprocess.call(args, stdout=devnull, stderr=devnull)
def handle_image_callable(self, data, mime):
self.callable_image_handler(data)
def handle_input_request(self, msg_id, timeout=0.1):
""" Method to capture raw_input
"""
req = self.client.stdin_channel.get_msg(timeout=timeout)
# in case any iopub came while we were waiting:
self.handle_iopub(msg_id)
if msg_id == req["parent_header"].get("msg_id"):
# wrap SIGINT handler
real_handler = signal.getsignal(signal.SIGINT)
def double_int(sig, frame):
# call real handler (forwards sigint to kernel),
# then raise local interrupt, stopping local raw_input
real_handler(sig, frame)
raise KeyboardInterrupt
signal.signal(signal.SIGINT, double_int)
content = req['content']
read = getpass if content.get('password', False) else input
try:
raw_data = read(content["prompt"])
except EOFError:
# turn EOFError into EOF character
raw_data = '\x04'
except KeyboardInterrupt:
sys.stdout.write('\n')
return
finally:
# restore SIGINT handler
signal.signal(signal.SIGINT, real_handler)
# only send stdin reply if there *was not* another request
# or execution finished while we were reading.
if not (self.client.stdin_channel.msg_ready()
or self.client.shell_channel.msg_ready()):
self.client.input(raw_data)
def mainloop(self, display_banner=False):
self.keepkernel = False
while True:
try:
self.interact(display_banner=display_banner)
#self.interact_with_readline()
# XXX for testing of a readline-decoupled repl loop, call
# interact_with_readline above
break
except KeyboardInterrupt:
# this should not be necessary, but KeyboardInterrupt
# handling seems rather unpredictable...
self.write("\nKeyboardInterrupt in interact()\n")
if self.keepkernel and not self.own_kernel:
print('keeping kernel alive')
elif self.keepkernel and self.own_kernel:
print("owning kernel, cannot keep it alive")
self.client.shutdown()
else:
print("Shutting down kernel")
self.client.shutdown()
def _banner1_default(self):
return "Jupyter Console {version}\n".format(version=__version__)
def compute_banner(self):
super(ZMQTerminalInteractiveShell, self).compute_banner()
if self.client and not self.kernel_banner:
msg_id = self.client.kernel_info()
while True:
try:
reply = self.client.get_shell_msg(timeout=1)
except Empty:
break
else:
if reply['parent_header'].get('msg_id') == msg_id:
self.kernel_banner = reply['content'].get('banner', '')
break
self.banner += self.kernel_banner
def wait_for_kernel(self, timeout=None):
"""method to wait for a kernel to be ready"""
tic = time.time()
self.client.hb_channel.unpause()
while True:
msg_id = self.client.kernel_info()
reply = None
while True:
try:
reply = self.client.get_shell_msg(timeout=1)
except Empty:
break
else:
if reply['parent_header'].get('msg_id') == msg_id:
return True
if timeout is not None \
and (time.time() - tic) > timeout \
and not self.client.hb_channel.is_beating():
# heart failed
return False
return True
def interact(self, display_banner=None):
"""Closely emulate the interactive Python console."""
# batch run -> do not interact
if self.exit_now:
return
if display_banner is None:
display_banner = self.display_banner
if isinstance(display_banner, string_types):
self.show_banner(display_banner)
elif display_banner:
self.show_banner()
more = False
# run a non-empty no-op, so that we don't get a prompt until
# we know the kernel is ready. This keeps the connection
# message above the first prompt.
if not self.wait_for_kernel(self.kernel_timeout):
error("Kernel did not respond\n")
return
if self.has_readline:
self.readline_startup_hook(self.pre_readline)
hlen_b4_cell = self.readline.get_current_history_length()
else:
hlen_b4_cell = 0
# exit_now is set by a call to %Exit or %Quit, through the
# ask_exit callback.
while not self.exit_now:
if not self.client.is_alive():
# kernel died, prompt for action or exit
action = "restart" if self.manager else "wait for restart"
ans = self.ask_yes_no("kernel died, %s ([y]/n)?" % action,
default='y')
if ans:
if self.manager:
self.manager.restart_kernel(True)
self.wait_for_kernel(self.kernel_timeout)
else:
self.exit_now = True
continue
try:
# protect prompt block from KeyboardInterrupt
# when sitting on ctrl-C
self.hooks.pre_prompt_hook()
if more:
try:
prompt = self.prompt_manager.render('in2')
except Exception:
self.showtraceback()
if self.autoindent:
self.rl_do_indent = True
else:
try:
prompt = self.separate_in + self.prompt_manager.render(
'in')
except Exception:
self.showtraceback()
line = self.raw_input(prompt)
if self.exit_now:
# quick exit on sys.std[in|out] close
break
if self.autoindent:
self.rl_do_indent = False
except KeyboardInterrupt:
#double-guard against keyboardinterrupts during kbdint handling
try:
self.write('\n' + self.get_exception_only())
source_raw = self.input_splitter.raw_reset()
hlen_b4_cell = self._replace_rlhist_multiline(
source_raw, hlen_b4_cell)
more = False
except KeyboardInterrupt:
pass
except EOFError:
if self.autoindent:
self.rl_do_indent = False
if self.has_readline:
self.readline_startup_hook(None)
self.write('\n')
self.exit()
except bdb.BdbQuit:
warn(
'The Python debugger has exited with a BdbQuit exception.\n'
'Because of how pdb handles the stack, it is impossible\n'
'for IPython to properly format this particular exception.\n'
'IPython will resume normal operation.')
except:
# exceptions here are VERY RARE, but they can be triggered
# asynchronously by signal handlers, for example.
self.showtraceback()
else:
try:
self.input_splitter.push(line)
more = self.input_splitter.push_accepts_more()
except SyntaxError:
# Run the code directly - run_cell takes care of displaying
# the exception.
more = False
if (self.SyntaxTB.last_syntax_error and self.autoedit_syntax):
self.edit_syntax_error()
if not more:
source_raw = self.input_splitter.raw_reset()
hlen_b4_cell = self._replace_rlhist_multiline(
source_raw, hlen_b4_cell)
self.run_cell(source_raw)
# Turn off the exit flag, so the mainloop can be restarted if desired
self.exit_now = False
def init_history(self):
"""Sets up the command history. """
self.history_manager = ZMQHistoryManager(client=self.client)
self.configurables.append(self.history_manager)
class Plot(widgets.DOMWidget):
"""
Main K3D widget.
Attributes:
antialias: `int`:
Enable antialiasing in WebGL renderer, changes have no effect after displaying.
height: `int`:
Height of the Widget in pixels, changes have no effect after displaying.
background_color: `int`.
Packed RGB color of the plot background (0xff0000 is red, 0xff is blue), -1 is for transparent.
camera_auto_fit: `bool`.
Enable automatic camera setting after adding, removing or changing a plot object.
grid_auto_fit: `bool`.
Enable automatic adjustment of the plot grid to contained objects.
grid_visible: `bool`.
Enable or disable grid.
screenshot_scale: `Float`.
Multipiler to screenshot resolution.
voxel_paint_color: `int`.
The (initial) int value to be inserted when editing voxels.
grid: `array_like`.
6-element tuple specifying the bounds of the plot grid (x0, y0, z0, x1, y1, z1).
camera: `array_like`.
9-element list or array specifying camera position.
camera_no_rotate: `Bool`.
Lock for camera rotation.
camera_no_zoom: `Bool`.
Lock for camera zoom.
camera_no_pan: `Bool`.
Lock for camera pan.
axes: `list`.
Axes labels for plot.
objects: `list`.
List of `k3d.objects.Drawable` currently included in the plot, not to be changed directly.
"""
_view_name = Unicode('PlotView').tag(sync=True)
_model_name = Unicode('PlotModel').tag(sync=True)
_view_module = Unicode('k3d').tag(sync=True)
_model_module = Unicode('k3d').tag(sync=True)
_view_module_version = Unicode(version).tag(sync=True)
_model_module_version = Unicode(version).tag(sync=True)
_backend_version = Unicode(version).tag(sync=True)
# readonly (specified at creation)
antialias = Int(min=0, max=5).tag(sync=True)
height = Int().tag(sync=True)
# readonly (not to be modified directly)
object_ids = List().tag(sync=True)
# read-write
camera_auto_fit = Bool(True).tag(sync=True)
lighting = Float().tag(sync=True)
grid_auto_fit = Bool(True).tag(sync=True)
grid_visible = Bool(True).tag(sync=True)
fps_meter = Bool(True).tag(sync=True)
menu_visibility = Bool(True).tag(sync=True)
screenshot_scale = Float().tag(sync=True)
time = Float().tag(sync=True)
grid = ListOrArray((-1, -1, -1, 1, 1, 1), minlen=6,
maxlen=6).tag(sync=True)
background_color = Int().tag(sync=True)
voxel_paint_color = Int().tag(sync=True)
camera = ListOrArray(minlen=9, maxlen=9, empty_ok=True).tag(sync=True)
camera_no_rotate = Bool(False).tag(sync=True)
camera_no_zoom = Bool(False).tag(sync=True)
camera_no_pan = Bool(False).tag(sync=True)
clipping_planes = ListOrArray(empty_ok=True).tag(sync=True)
colorbar_object_id = Int(-1).tag(sync=True)
rendering_steps = Int(1).tag(sync=True)
screenshot = Unicode().tag(sync=True)
axes = List(minlen=3, maxlen=3, default_value=['x', 'y',
'z']).tag(sync=True)
objects = []
def __init__(self,
antialias=3,
background_color=0xFFFFFF,
camera_auto_fit=True,
grid_auto_fit=True,
grid_visible=True,
height=512,
voxel_paint_color=0,
grid=(-1, -1, -1, 1, 1, 1),
screenshot_scale=2.0,
lighting=1.0,
time=0.0,
fps_meter=False,
menu_visibility=True,
colorbar_object_id=-1,
rendering_steps=1,
axes=['x', 'y', 'z'],
camera_no_rotate=False,
camera_no_zoom=False,
camera_no_pan=False,
*args,
**kwargs):
super(Plot, self).__init__()
self.antialias = antialias
self.camera_auto_fit = camera_auto_fit
self.grid_auto_fit = grid_auto_fit
self.fps_meter = fps_meter
self.grid = grid
self.grid_visible = grid_visible
self.background_color = background_color
self.voxel_paint_color = voxel_paint_color
self.screenshot_scale = screenshot_scale
self.height = height
self.lighting = lighting
self.time = time
self.menu_visibility = menu_visibility
self.colorbar_object_id = colorbar_object_id
self.rendering_steps = rendering_steps
self.camera_no_rotate = camera_no_rotate
self.camera_no_zoom = camera_no_zoom
self.camera_no_pan = camera_no_pan
self.axes = axes
self.camera = [4.5, 4.5, 4.5, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0]
self.object_ids = []
self.objects = []
self.outputs = []
def __iadd__(self, objs):
assert isinstance(objs, Drawable)
for obj in objs:
if obj.id not in self.object_ids:
self.object_ids = self.object_ids + [obj.id]
self.objects.append(obj)
return self
def __isub__(self, objs):
assert isinstance(objs, Drawable)
for obj in objs:
self.object_ids = [id_ for id_ in self.object_ids if id_ != obj.id]
if obj in self.objects:
self.objects.remove(obj)
return self
def display(self, **kwargs):
output = widgets.Output()
with output:
display(self, **kwargs)
self.outputs.append(output)
display(output)
def close(self):
for output in self.outputs:
output.clear_output()
self.outputs = []
def reset_camera(self):
self.send({'msg_type': 'reset_camera'})
def fetch_screenshot(self, only_canvas=False):
self.send({'msg_type': 'fetch_screenshot', 'only_canvas': only_canvas})
def yield_screenshots(self, generator_function):
"""Decorator for a generator function receiving screenshots via yield."""
@wraps(generator_function)
def inner():
generator = generator_function()
def send_new_value(change):
try:
generator.send(base64.b64decode(change.new))
except StopIteration:
self.unobserve(send_new_value, 'screenshot')
self.observe(send_new_value, 'screenshot')
# start the decorated generator
generator.send(None)
return inner
class Map(DOMWidget, InteractMixin):
_view_name = Unicode('LeafletMapView').tag(sync=True)
_model_name = Unicode('LeafletMapModel').tag(sync=True)
_view_module = Unicode('jupyter-leaflet').tag(sync=True)
_model_module = Unicode('jupyter-leaflet').tag(sync=True)
_view_module_version = Unicode(EXTENSION_VERSION).tag(sync=True)
_model_module_version = Unicode(EXTENSION_VERSION).tag(sync=True)
# Map options
center = List(def_loc).tag(sync=True, o=True)
zoom_start = Int(12).tag(sync=True, o=True)
zoom = Int(12).tag(sync=True, o=True)
max_zoom = Int(18).tag(sync=True, o=True)
min_zoom = Int(1).tag(sync=True, o=True)
interpolation = Unicode('bilinear').tag(sync=True, o=True)
crs = Enum(values=allowed_crs, default_value='EPSG3857').tag(sync=True)
# Specification of the basemap
basemap = Dict(default_value=dict(
url='https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png',
max_zoom=19,
attribution=
'Map data (c) <a href="https://openstreetmap.org">OpenStreetMap</a> contributors'
)).tag(sync=True, o=True)
modisdate = Unicode('yesterday').tag(sync=True)
# Interaction options
dragging = Bool(True).tag(sync=True, o=True)
touch_zoom = Bool(True).tag(sync=True, o=True)
scroll_wheel_zoom = Bool(False).tag(sync=True, o=True)
double_click_zoom = Bool(True).tag(sync=True, o=True)
box_zoom = Bool(True).tag(sync=True, o=True)
tap = Bool(True).tag(sync=True, o=True)
tap_tolerance = Int(15).tag(sync=True, o=True)
world_copy_jump = Bool(False).tag(sync=True, o=True)
close_popup_on_click = Bool(True).tag(sync=True, o=True)
bounce_at_zoom_limits = Bool(True).tag(sync=True, o=True)
keyboard = Bool(True).tag(sync=True, o=True)
keyboard_pan_offset = Int(80).tag(sync=True, o=True)
keyboard_zoom_offset = Int(1).tag(sync=True, o=True)
inertia = Bool(True).tag(sync=True, o=True)
inertia_deceleration = Int(3000).tag(sync=True, o=True)
inertia_max_speed = Int(1500).tag(sync=True, o=True)
# inertia_threshold = Int(?, o=True).tag(sync=True)
# fade_animation = Bool(?).tag(sync=True, o=True)
# zoom_animation = Bool(?).tag(sync=True, o=True)
zoom_animation_threshold = Int(4).tag(sync=True, o=True)
# marker_zoom_animation = Bool(?).tag(sync=True, o=True)
fullscreen = Bool(False).tag(sync=True, o=True)
options = List(trait=Unicode()).tag(sync=True)
style = InstanceDict(MapStyle).tag(sync=True, **widget_serialization)
default_style = InstanceDict(MapStyle).tag(sync=True,
**widget_serialization)
dragging_style = InstanceDict(MapStyle).tag(sync=True,
**widget_serialization)
zoom_control = Bool(True)
zoom_control_instance = ZoomControl()
attribution_control = Bool(True)
attribution_control_instance = AttributionControl(position='bottomright')
@default('dragging_style')
def _default_dragging_style(self):
return {'cursor': 'move'}
@default('options')
def _default_options(self):
return [name for name in self.traits(o=True)]
south = Float(def_loc[0], read_only=True).tag(sync=True)
north = Float(def_loc[0], read_only=True).tag(sync=True)
east = Float(def_loc[1], read_only=True).tag(sync=True)
west = Float(def_loc[1], read_only=True).tag(sync=True)
layers = Tuple().tag(trait=Instance(Layer),
sync=True,
**widget_serialization)
@default('layers')
def _default_layers(self):
return (basemap_to_tiles(self.basemap, self.modisdate, base=True), )
bounds = Tuple(read_only=True)
bounds_polygon = Tuple(read_only=True)
@observe('south', 'north', 'east', 'west')
def _observe_bounds(self, change):
self.set_trait('bounds',
((self.south, self.west), (self.north, self.east)))
self.set_trait('bounds_polygon',
((self.north, self.west), (self.north, self.east),
(self.south, self.east), (self.south, self.west)))
def __init__(self, **kwargs):
super(Map, self).__init__(**kwargs)
self.on_displayed(self._fire_children_displayed)
self.on_msg(self._handle_leaflet_event)
if self.zoom_control:
self.add_control(self.zoom_control_instance)
if self.attribution_control:
self.add_control(self.attribution_control_instance)
@observe('zoom_control')
def observe_zoom_control(self, change):
if change['new']:
self.add_control(self.zoom_control_instance)
else:
if self.zoom_control_instance in self.controls:
self.remove_control(self.zoom_control_instance)
@observe('attribution_control')
def observe_attribution_control(self, change):
if change['new']:
self.add_control(self.attribution_control_instance)
else:
if self.attribution_control_instance in self.controls:
self.remove_control(self.attribution_control_instance)
def _fire_children_displayed(self, widget, **kwargs):
for layer in self.layers:
layer._handle_displayed(**kwargs)
for control in self.controls:
control._handle_displayed(**kwargs)
_layer_ids = List()
@validate('layers')
def _validate_layers(self, proposal):
'''Validate layers list.
Makes sure only one instance of any given layer can exist in the
layers list.
'''
self._layer_ids = [l.model_id for l in proposal.value]
if len(set(self._layer_ids)) != len(self._layer_ids):
raise LayerException(
'duplicate layer detected, only use each layer once')
return proposal.value
def add_layer(self, layer):
if isinstance(layer, dict):
layer = basemap_to_tiles(layer)
if layer.model_id in self._layer_ids:
raise LayerException('layer already on map: %r' % layer)
self.layers = tuple([l for l in self.layers] + [layer])
def remove_layer(self, layer):
if layer.model_id not in self._layer_ids:
raise LayerException('layer not on map: %r' % layer)
self.layers = tuple(
[l for l in self.layers if l.model_id != layer.model_id])
def substitute_layer(self, old, new):
if isinstance(new, dict):
new = basemap_to_tiles(new)
if old.model_id not in self._layer_ids:
raise LayerException(
'Could not substitute layer: layer not on map.')
self.layers = tuple(
[new if l.model_id == old.model_id else l for l in self.layers])
def clear_layers(self):
self.layers = ()
controls = Tuple().tag(trait=Instance(Control),
sync=True,
**widget_serialization)
_control_ids = List()
@validate('controls')
def _validate_controls(self, proposal):
'''Validate controls list.
Makes sure only one instance of any given layer can exist in the
controls list.
'''
self._control_ids = [c.model_id for c in proposal.value]
if len(set(self._control_ids)) != len(self._control_ids):
raise ControlException(
'duplicate control detected, only use each control once')
return proposal.value
def add_control(self, control):
if control.model_id in self._control_ids:
raise ControlException('control already on map: %r' % control)
self.controls = tuple([c for c in self.controls] + [control])
def remove_control(self, control):
if control.model_id not in self._control_ids:
raise ControlException('control not on map: %r' % control)
self.controls = tuple(
[c for c in self.controls if c.model_id != control.model_id])
def clear_controls(self):
self.controls = ()
def __iadd__(self, item):
if isinstance(item, Layer):
self.add_layer(item)
elif isinstance(item, Control):
self.add_control(item)
return self
def __isub__(self, item):
if isinstance(item, Layer):
self.remove_layer(item)
elif isinstance(item, Control):
self.remove_control(item)
return self
def __add__(self, item):
if isinstance(item, Layer):
self.add_layer(item)
elif isinstance(item, Control):
self.add_control(item)
return self
# Event handling
_interaction_callbacks = Instance(CallbackDispatcher, ())
def _handle_leaflet_event(self, _, content, buffers):
if content.get('event', '') == 'interaction':
self._interaction_callbacks(**content)
def on_interaction(self, callback, remove=False):
self._interaction_callbacks.register_callback(callback, remove=remove)
class PodReflector(SingletonConfigurable):
request_timeout = Float(
60,
help="""
Network timeout for kubernetes watch.
Trigger watch reconnect when a given request is taking too long,
which can indicate network issues.
""",
config=True,
)
timeout_seconds = Int(
10,
help="""
Timeout (in seconds) for kubernetes watch.
Trigger watch reconnect when no watch event has been received.
This will cause a full reload of the currently existing resources
from the API server.
""",
config=True,
)
restart_seconds = Float(
30,
help="Maximum time (in seconds) before restarting a watch.",
config=True)
kube_client = Instance(kubernetes.client.CoreV1Api)
label_selector = Unicode()
namespace = Unicode()
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.first_load_future = get_running_loop().create_future()
self.stopped = False
self.start()
def __del__(self):
self.stop()
def _list_and_update(self):
initial_resources = self.kube_client.list_namespaced_pod(
self.namespace,
label_selector=self.label_selector,
_request_timeout=self.request_timeout,
)
self.pods = {p.metadata.name: p for p in initial_resources.items}
# return the resource version so we can hook up a watch
return initial_resources.metadata.resource_version
def _watch_and_update(self):
self.log.info("Starting pod watcher...")
cur_delay = 0.1
while not self.stopped:
start = time.monotonic()
watch = kubernetes.watch.Watch()
try:
resource_version = self._list_and_update()
if not self.first_load_future.done():
# signal that we've loaded our initial data
self.first_load_future.set_result(None)
kwargs = {
"namespace": self.namespace,
"label_selector": self.label_selector,
"resource_version": resource_version,
"_request_timeout": self.request_timeout,
"timeout_seconds": self.timeout_seconds,
}
for ev in watch.stream(self.kube_client.list_namespaced_pod,
**kwargs):
cur_delay = 0.1
pod = ev["object"]
if ev["type"] == "DELETED":
self.pods.pop(pod.metadata.name, None)
else:
self.pods[pod.metadata.name] = pod
if self.stopped:
# Check in inner loop to provide faster shutdown
break
watch_duration = time.monotonic() - start
if watch_duration >= self.restart_seconds:
self.log.debug(
"Restarting pod watcher after %.1f seconds",
watch_duration)
break
except ReadTimeoutError:
# network read time out, just continue and restart the watch
# this could be due to a network problem or just low activity
self.log.warning("Read timeout watching pods, reconnecting")
continue
except Exception as exc:
if cur_delay < 30:
cur_delay = cur_delay * 2
self.log.error(
"Error when watching pods, retrying in %.1f seconds...",
cur_delay,
exc_info=exc,
)
time.sleep(cur_delay)
continue
else:
# no events on watch, reconnect
self.log.debug("Pod watcher timeout, restarting")
finally:
watch.stop()
self.log.debug("Pod watcher stopped")
def start(self):
self.watch_thread = threading.Thread(target=self._watch_and_update,
daemon=True)
self.watch_thread.start()
def stop(self):
self.stopped = True
class TerminalInteractiveShell(InteractiveShell):
mime_renderers = Dict().tag(config=True)
space_for_menu = Integer(
6,
help='Number of line at the bottom of the screen '
'to reserve for the tab completion menu, '
'search history, ...etc, the height of '
'these menus will at most this value. '
'Increase it is you prefer long and skinny '
'menus, decrease for short and wide.').tag(config=True)
pt_app = None
debugger_history = None
debugger_history_file = Unicode(
"~/.pdbhistory",
help="File in which to store and read history").tag(config=True)
simple_prompt = Bool(
_use_simple_prompt,
help=
"""Use `raw_input` for the REPL, without completion and prompt colors.
Useful when controlling IPython as a subprocess, and piping STDIN/OUT/ERR. Known usage are:
IPython own testing machinery, and emacs inferior-shell integration through elpy.
This mode default to `True` if the `IPY_TEST_SIMPLE_PROMPT`
environment variable is set, or the current terminal is not a tty."""
).tag(config=True)
@property
def debugger_cls(self):
return Pdb if self.simple_prompt else TerminalPdb
confirm_exit = Bool(
True,
help="""
Set to confirm when you try to exit IPython with an EOF (Control-D
in Unix, Control-Z/Enter in Windows). By typing 'exit' or 'quit',
you can force a direct exit without any confirmation.""",
).tag(config=True)
editing_mode = Unicode(
'emacs',
help="Shortcut style to use at the prompt. 'vi' or 'emacs'.",
).tag(config=True)
emacs_bindings_in_vi_insert_mode = Bool(
True,
help="Add shortcuts from 'emacs' insert mode to 'vi' insert mode.",
).tag(config=True)
modal_cursor = Bool(
True,
help="""
Cursor shape changes depending on vi mode: beam in vi insert mode,
block in nav mode, underscore in replace mode.""",
).tag(config=True)
ttimeoutlen = Float(
0.01,
help="""The time in milliseconds that is waited for a key code
to complete.""",
).tag(config=True)
timeoutlen = Float(
0.5,
help="""The time in milliseconds that is waited for a mapped key
sequence to complete.""",
).tag(config=True)
autoformatter = Unicode(
None,
help=
"Autoformatter to reformat Terminal code. Can be `'black'` or `None`",
allow_none=True).tag(config=True)
mouse_support = Bool(
False,
help=
"Enable mouse support in the prompt\n(Note: prevents selecting text with the mouse)"
).tag(config=True)
# We don't load the list of styles for the help string, because loading
# Pygments plugins takes time and can cause unexpected errors.
highlighting_style = Union(
[Unicode('legacy'), Type(klass=Style)],
help="""The name or class of a Pygments style to use for syntax
highlighting. To see available styles, run `pygmentize -L styles`."""
).tag(config=True)
@validate('editing_mode')
def _validate_editing_mode(self, proposal):
if proposal['value'].lower() == 'vim':
proposal['value'] = 'vi'
elif proposal['value'].lower() == 'default':
proposal['value'] = 'emacs'
if hasattr(EditingMode, proposal['value'].upper()):
return proposal['value'].lower()
return self.editing_mode
@observe('editing_mode')
def _editing_mode(self, change):
if self.pt_app:
self.pt_app.editing_mode = getattr(EditingMode, change.new.upper())
@observe('autoformatter')
def _autoformatter_changed(self, change):
formatter = change.new
if formatter is None:
self.reformat_handler = lambda x: x
elif formatter == 'black':
self.reformat_handler = black_reformat_handler
else:
raise ValueError
@observe('highlighting_style')
@observe('colors')
def _highlighting_style_changed(self, change):
self.refresh_style()
def refresh_style(self):
self._style = self._make_style_from_name_or_cls(
self.highlighting_style)
highlighting_style_overrides = Dict(
help="Override highlighting format for specific tokens").tag(
config=True)
true_color = Bool(
False,
help="""Use 24bit colors instead of 256 colors in prompt highlighting.
If your terminal supports true color, the following command should
print ``TRUECOLOR`` in orange::
printf \"\\x1b[38;2;255;100;0mTRUECOLOR\\x1b[0m\\n\"
""",
).tag(config=True)
editor = Unicode(
get_default_editor(),
help="Set the editor used by IPython (default to $EDITOR/vi/notepad)."
).tag(config=True)
prompts_class = Type(
Prompts,
help='Class used to generate Prompt token for prompt_toolkit').tag(
config=True)
prompts = Instance(Prompts)
@default('prompts')
def _prompts_default(self):
return self.prompts_class(self)
# @observe('prompts')
# def _(self, change):
# self._update_layout()
@default('displayhook_class')
def _displayhook_class_default(self):
return RichPromptDisplayHook
term_title = Bool(
True, help="Automatically set the terminal title").tag(config=True)
term_title_format = Unicode(
"IPython: {cwd}",
help=
"Customize the terminal title format. This is a python format string. "
+ "Available substitutions are: {cwd}.").tag(config=True)
display_completions = Enum(
('column', 'multicolumn', 'readlinelike'),
help=
("Options for displaying tab completions, 'column', 'multicolumn', and "
"'readlinelike'. These options are for `prompt_toolkit`, see "
"`prompt_toolkit` documentation for more information."),
default_value='multicolumn').tag(config=True)
highlight_matching_brackets = Bool(
True,
help="Highlight matching brackets.",
).tag(config=True)
extra_open_editor_shortcuts = Bool(
False,
help=
"Enable vi (v) or Emacs (C-X C-E) shortcuts to open an external editor. "
"This is in addition to the F2 binding, which is always enabled.").tag(
config=True)
handle_return = Any(
None,
help="Provide an alternative handler to be called when the user presses "
"Return. This is an advanced option intended for debugging, which "
"may be changed or removed in later releases.").tag(config=True)
enable_history_search = Bool(
True,
help="Allows to enable/disable the prompt toolkit history search").tag(
config=True)
prompt_includes_vi_mode = Bool(
True,
help="Display the current vi mode (when using vi editing mode).").tag(
config=True)
@observe('term_title')
def init_term_title(self, change=None):
# Enable or disable the terminal title.
if self.term_title:
toggle_set_term_title(True)
set_term_title(self.term_title_format.format(cwd=abbrev_cwd()))
else:
toggle_set_term_title(False)
def restore_term_title(self):
if self.term_title:
restore_term_title()
def init_display_formatter(self):
super(TerminalInteractiveShell, self).init_display_formatter()
# terminal only supports plain text
self.display_formatter.active_types = ['text/plain']
# disable `_ipython_display_`
self.display_formatter.ipython_display_formatter.enabled = False
def init_prompt_toolkit_cli(self):
if self.simple_prompt:
# Fall back to plain non-interactive output for tests.
# This is very limited.
def prompt():
prompt_text = "".join(x[1]
for x in self.prompts.in_prompt_tokens())
lines = [input(prompt_text)]
prompt_continuation = "".join(
x[1] for x in self.prompts.continuation_prompt_tokens())
while self.check_complete('\n'.join(lines))[0] == 'incomplete':
lines.append(input(prompt_continuation))
return '\n'.join(lines)
self.prompt_for_code = prompt
return
# Set up keyboard shortcuts
key_bindings = create_ipython_shortcuts(self)
# Pre-populate history from IPython's history database
history = InMemoryHistory()
last_cell = u""
for __, ___, cell in self.history_manager.get_tail(
self.history_load_length, include_latest=True):
# Ignore blank lines and consecutive duplicates
cell = cell.rstrip()
if cell and (cell != last_cell):
history.append_string(cell)
last_cell = cell
self._style = self._make_style_from_name_or_cls(
self.highlighting_style)
self.style = DynamicStyle(lambda: self._style)
editing_mode = getattr(EditingMode, self.editing_mode.upper())
self.pt_loop = asyncio.new_event_loop()
self.pt_app = PromptSession(
auto_suggest=AutoSuggestFromHistory(),
editing_mode=editing_mode,
key_bindings=key_bindings,
history=history,
completer=IPythonPTCompleter(shell=self),
enable_history_search=self.enable_history_search,
style=self.style,
include_default_pygments_style=False,
mouse_support=self.mouse_support,
enable_open_in_editor=self.extra_open_editor_shortcuts,
color_depth=self.color_depth,
tempfile_suffix=".py",
**self._extra_prompt_options())
def _make_style_from_name_or_cls(self, name_or_cls):
"""
Small wrapper that make an IPython compatible style from a style name
We need that to add style for prompt ... etc.
"""
style_overrides = {}
if name_or_cls == 'legacy':
legacy = self.colors.lower()
if legacy == 'linux':
style_cls = get_style_by_name('monokai')
style_overrides = _style_overrides_linux
elif legacy == 'lightbg':
style_overrides = _style_overrides_light_bg
style_cls = get_style_by_name('pastie')
elif legacy == 'neutral':
# The default theme needs to be visible on both a dark background
# and a light background, because we can't tell what the terminal
# looks like. These tweaks to the default theme help with that.
style_cls = get_style_by_name('default')
style_overrides.update({
Token.Number:
'#ansigreen',
Token.Operator:
'noinherit',
Token.String:
'#ansiyellow',
Token.Name.Function:
'#ansiblue',
Token.Name.Class:
'bold #ansiblue',
Token.Name.Namespace:
'bold #ansiblue',
Token.Name.Variable.Magic:
'#ansiblue',
Token.Prompt:
'#ansigreen',
Token.PromptNum:
'#ansibrightgreen bold',
Token.OutPrompt:
'#ansired',
Token.OutPromptNum:
'#ansibrightred bold',
})
# Hack: Due to limited color support on the Windows console
# the prompt colors will be wrong without this
if os.name == 'nt':
style_overrides.update({
Token.Prompt: '#ansidarkgreen',
Token.PromptNum: '#ansigreen bold',
Token.OutPrompt: '#ansidarkred',
Token.OutPromptNum: '#ansired bold',
})
elif legacy == 'nocolor':
style_cls = _NoStyle
style_overrides = {}
else:
raise ValueError('Got unknown colors: ', legacy)
else:
if isinstance(name_or_cls, str):
style_cls = get_style_by_name(name_or_cls)
else:
style_cls = name_or_cls
style_overrides = {
Token.Prompt: '#ansigreen',
Token.PromptNum: '#ansibrightgreen bold',
Token.OutPrompt: '#ansired',
Token.OutPromptNum: '#ansibrightred bold',
}
style_overrides.update(self.highlighting_style_overrides)
style = merge_styles([
style_from_pygments_cls(style_cls),
style_from_pygments_dict(style_overrides),
])
return style
@property
def pt_complete_style(self):
return {
'multicolumn': CompleteStyle.MULTI_COLUMN,
'column': CompleteStyle.COLUMN,
'readlinelike': CompleteStyle.READLINE_LIKE,
}[self.display_completions]
@property
def color_depth(self):
return (ColorDepth.TRUE_COLOR if self.true_color else None)
def _extra_prompt_options(self):
"""
Return the current layout option for the current Terminal InteractiveShell
"""
def get_message():
return PygmentsTokens(self.prompts.in_prompt_tokens())
if self.editing_mode == 'emacs':
# with emacs mode the prompt is (usually) static, so we call only
# the function once. With VI mode it can toggle between [ins] and
# [nor] so we can't precompute.
# here I'm going to favor the default keybinding which almost
# everybody uses to decrease CPU usage.
# if we have issues with users with custom Prompts we can see how to
# work around this.
get_message = get_message()
options = {
'complete_in_thread':
False,
'lexer':
IPythonPTLexer(),
'reserve_space_for_menu':
self.space_for_menu,
'message':
get_message,
'prompt_continuation':
(lambda width, lineno, is_soft_wrap: PygmentsTokens(
self.prompts.continuation_prompt_tokens(width))),
'multiline':
True,
'complete_style':
self.pt_complete_style,
# Highlight matching brackets, but only when this setting is
# enabled, and only when the DEFAULT_BUFFER has the focus.
'input_processors': [
ConditionalProcessor(
processor=HighlightMatchingBracketProcessor(
chars='[](){}'),
filter=HasFocus(DEFAULT_BUFFER) & ~IsDone()
& Condition(lambda: self.highlight_matching_brackets))
],
}
if not PTK3:
options['inputhook'] = self.inputhook
return options
def prompt_for_code(self):
if self.rl_next_input:
default = self.rl_next_input
self.rl_next_input = None
else:
default = ''
# In order to make sure that asyncio code written in the
# interactive shell doesn't interfere with the prompt, we run the
# prompt in a different event loop.
# If we don't do this, people could spawn coroutine with a
# while/true inside which will freeze the prompt.
try:
old_loop = asyncio.get_event_loop()
except RuntimeError:
# This happens when the user used `asyncio.run()`.
old_loop = None
asyncio.set_event_loop(self.pt_loop)
try:
with patch_stdout(raw=True):
text = self.pt_app.prompt(default=default,
**self._extra_prompt_options())
finally:
# Restore the original event loop.
asyncio.set_event_loop(old_loop)
return text
def enable_win_unicode_console(self):
# Since IPython 7.10 doesn't support python < 3.6 and PEP 528, Python uses the unicode APIs for the Windows
# console by default, so WUC shouldn't be needed.
from warnings import warn
warn(
"`enable_win_unicode_console` is deprecated since IPython 7.10, does not do anything and will be removed in the future",
DeprecationWarning,
stacklevel=2)
def init_io(self):
if sys.platform not in {'win32', 'cli'}:
return
import colorama
colorama.init()
# For some reason we make these wrappers around stdout/stderr.
# For now, we need to reset them so all output gets coloured.
# https://github.com/ipython/ipython/issues/8669
# io.std* are deprecated, but don't show our own deprecation warnings
# during initialization of the deprecated API.
with warnings.catch_warnings():
warnings.simplefilter('ignore', DeprecationWarning)
io.stdout = io.IOStream(sys.stdout)
io.stderr = io.IOStream(sys.stderr)
def init_magics(self):
super(TerminalInteractiveShell, self).init_magics()
self.register_magics(TerminalMagics)
def init_alias(self):
# The parent class defines aliases that can be safely used with any
# frontend.
super(TerminalInteractiveShell, self).init_alias()
# Now define aliases that only make sense on the terminal, because they
# need direct access to the console in a way that we can't emulate in
# GUI or web frontend
if os.name == 'posix':
for cmd in ('clear', 'more', 'less', 'man'):
self.alias_manager.soft_define_alias(cmd, cmd)
def __init__(self, *args, **kwargs):
super(TerminalInteractiveShell, self).__init__(*args, **kwargs)
self.init_prompt_toolkit_cli()
self.init_term_title()
self.keep_running = True
def ask_exit(self):
self.keep_running = False
rl_next_input = None
def interact(self, display_banner=DISPLAY_BANNER_DEPRECATED):
if display_banner is not DISPLAY_BANNER_DEPRECATED:
warn(
'interact `display_banner` argument is deprecated since IPython 5.0. Call `show_banner()` if needed.',
DeprecationWarning,
stacklevel=2)
self.keep_running = True
while self.keep_running:
print(self.separate_in, end='')
try:
code = self.prompt_for_code()
except EOFError:
if (not self.confirm_exit) \
or self.ask_yes_no('Do you really want to exit ([y]/n)?','y','n'):
self.ask_exit()
else:
if code:
self.run_cell(code, store_history=True)
def mainloop(self, display_banner=DISPLAY_BANNER_DEPRECATED):
# An extra layer of protection in case someone mashing Ctrl-C breaks
# out of our internal code.
if display_banner is not DISPLAY_BANNER_DEPRECATED:
warn(
'mainloop `display_banner` argument is deprecated since IPython 5.0. Call `show_banner()` if needed.',
DeprecationWarning,
stacklevel=2)
while True:
try:
self.interact()
break
except KeyboardInterrupt as e:
print("\n%s escaped interact()\n" % type(e).__name__)
finally:
# An interrupt during the eventloop will mess up the
# internal state of the prompt_toolkit library.
# Stopping the eventloop fixes this, see
# https://github.com/ipython/ipython/pull/9867
if hasattr(self, '_eventloop'):
self._eventloop.stop()
self.restore_term_title()
# try to call some at-exit operation optimistically as some things can't
# be done during interpreter shutdown. this is technically inaccurate as
# this make mainlool not re-callable, but that should be a rare if not
# in existent use case.
self._atexit_once()
_inputhook = None
def inputhook(self, context):
if self._inputhook is not None:
self._inputhook(context)
active_eventloop = None
def enable_gui(self, gui=None):
if gui and (gui != 'inline'):
self.active_eventloop, self._inputhook =\
get_inputhook_name_and_func(gui)
else:
self.active_eventloop = self._inputhook = None
# For prompt_toolkit 3.0. We have to create an asyncio event loop with
# this inputhook.
if PTK3:
import asyncio
from prompt_toolkit.eventloop import new_eventloop_with_inputhook
if gui == 'asyncio':
# When we integrate the asyncio event loop, run the UI in the
# same event loop as the rest of the code. don't use an actual
# input hook. (Asyncio is not made for nesting event loops.)
self.pt_loop = asyncio.get_event_loop()
elif self._inputhook:
# If an inputhook was set, create a new asyncio event loop with
# this inputhook for the prompt.
self.pt_loop = new_eventloop_with_inputhook(self._inputhook)
else:
# When there's no inputhook, run the prompt in a separate
# asyncio event loop.
self.pt_loop = asyncio.new_event_loop()
# Run !system commands directly, not through pipes, so terminal programs
# work correctly.
system = InteractiveShell.system_raw
def auto_rewrite_input(self, cmd):
"""Overridden from the parent class to use fancy rewriting prompt"""
if not self.show_rewritten_input:
return
tokens = self.prompts.rewrite_prompt_tokens()
if self.pt_app:
print_formatted_text(PygmentsTokens(tokens),
end='',
style=self.pt_app.app.style)
print(cmd)
else:
prompt = ''.join(s for t, s in tokens)
print(prompt, cmd, sep='')
_prompts_before = None
def switch_doctest_mode(self, mode):
"""Switch prompts to classic for %doctest_mode"""
if mode:
self._prompts_before = self.prompts
self.prompts = ClassicPrompts(self)
elif self._prompts_before:
self.prompts = self._prompts_before
self._prompts_before = None
class Polyline(GMapsWidgetMixin, widgets.Widget):
"""
Widget representing a linear overlay of connected line segments on the map
Add this polyline to a map via the :func:`gmaps.drawing_layer`
function, or by passing it directly to the ``.features`` array
of a :class:`gmaps.Drawing` instance.
:Examples:
>>> fig = gmaps.figure()
>>> drawing = gmaps.drawing_layer(features=[
gmaps.Polyline(
[(46.72, 6.06), (46.48, 6.49), (46.79, 6.91)],
stroke_color='blue'
)
])
>>> fig.add_layer(drawing)
You can also add a polyline to an existing :class:`gmaps.Drawing`
instance:
>>> fig = gmaps.figure()
>>> drawing = gmaps.drawing_layer()
>>> fig.add_layer(drawing)
>>> fig # display the figure
You can now add polylines directly on the map:
>>> drawing.features = [
gmaps.Polyline(
[(46.72, 6.06), (46.48, 6.49), (46.79, 6.91)]
stroke_color='blue'
)
]
:param path:
List of (latitude, longitude) pairs denoting each point on the polyline.
Latitudes are expressed as a float between -90 (corresponding to 90
degrees south) and +90 (corresponding to 90 degrees north). Longitudes
are expressed as a float between -180 (corresponding to 180 degrees
west) and +180 (corresponding to 180 degrees east).
:type path: list of tuples of floats
{stroke_options_params}
{fill_options_params}
"""
_view_name = Unicode('PolylineView').tag(sync=True)
_model_name = Unicode('PolylineModel').tag(sync=True)
path = List(geotraitlets.Point(), minlen=3).tag(sync=True)
stroke_color = geotraitlets.ColorAlpha(
allow_none=False, default_value=DEFAULT_STROKE_COLOR).tag(sync=True)
stroke_weight = Float(min=0.0, allow_none=False,
default_value=2.0).tag(sync=True)
stroke_opacity = geotraitlets.StrokeOpacity().tag(sync=True)
def __init__(self,
path,
stroke_color=DEFAULT_STROKE_COLOR,
stroke_weight=2.0,
stroke_opacity=geotraitlets.StrokeOpacity.default_value):
kwargs = dict(
path=path,
stroke_color=stroke_color,
stroke_weight=stroke_weight,
stroke_opacity=stroke_opacity,
)
super(Polyline, self).__init__(**kwargs)
class Circle(GMapsWidgetMixin, widgets.Widget):
"""
Widget representing a closed circle on a map
Add this cicle to a map via the :func:`gmaps.drawing_layer`
function, or by passing it directly to the ``.features`` array
of a :class:`gmaps.Drawing` instance
:Examples:
>>> fig = gmaps.figure()
>>> drawing = gmaps.drawing_layer(features=[
gmaps.Circle(
radius=20000, # in meters
center=(46.656, 6.111),
stroke_color='red', fill_color=(255, 0, 132)
)
])
>>> fig.add_layer(drawing)
You can also add circles to an existing :class:`gmaps.Drawing`
instance:
>>> fig = gmaps.figure()
>>> drawing = gmaps.drawing_layer()
>>> fig.add_layer(drawing)
>>> fig # display the figure
You can now add polygons directly on the map:
>>> drawing.features = [
gmaps.Circle(
radius=20000, # in meters
center=(46.656, 6.111),
stroke_color='red', fill_color=(255, 0, 132)
)
]
:param center:
(latitude, longitude) pair denoting the center of the
circle. Latitudes are expressed as a float between -90 (
corresponding to 90 degrees south) and +90 (corresponding to
90 degrees north). Longitudes are expressed as a float between
-180 (corresponding to 180 degrees west) and +180
(corresponding to 180 degrees east).
:type center: pair of floats
:param radius:
Radius of the circle, in meters.
:type radius: float
{stroke_options_params}
{fill_options_params}
"""
_view_name = Unicode('CircleView').tag(sync=True)
_model_name = Unicode('CircleModel').tag(sync=True)
radius = Float(min=0.0).tag(sync=True)
center = geotraitlets.Point().tag(sync=True)
stroke_color = geotraitlets.ColorAlpha(
allow_none=False, default_value=DEFAULT_STROKE_COLOR).tag(sync=True)
stroke_weight = Float(min=0.0, allow_none=False,
default_value=2.0).tag(sync=True)
stroke_opacity = geotraitlets.StrokeOpacity().tag(sync=True)
fill_color = geotraitlets.ColorAlpha(
allow_none=False, default_value=DEFAULT_FILL_COLOR).tag(sync=True)
fill_opacity = geotraitlets.FillOpacity().tag(sync=True)
def __init__(self,
center,
radius,
stroke_color=DEFAULT_STROKE_COLOR,
stroke_weight=2.0,
stroke_opacity=geotraitlets.StrokeOpacity.default_value,
fill_color=DEFAULT_FILL_COLOR,
fill_opacity=geotraitlets.FillOpacity.default_value):
kwargs = dict(center=center,
radius=radius,
stroke_color=stroke_color,
stroke_weight=stroke_weight,
stroke_opacity=stroke_opacity,
fill_color=fill_color,
fill_opacity=fill_opacity)
super(Circle, self).__init__(**kwargs)
class IPEngineApp(BaseParallelApplication):
name = 'ipengine'
description = _description
examples = _examples
classes = List([ZMQInteractiveShell, ProfileDir, Session, EngineFactory, Kernel, MPI])
startup_script = Unicode(u'', config=True,
help='specify a script to be run at startup')
startup_command = Unicode('', config=True,
help='specify a command to be run at startup')
url_file = Unicode(u'', config=True,
help="""The full location of the file containing the connection information for
the controller. If this is not given, the file must be in the
security directory of the cluster directory. This location is
resolved using the `profile` or `profile_dir` options.""",
)
wait_for_url_file = Float(5, config=True,
help="""The maximum number of seconds to wait for url_file to exist.
This is useful for batch-systems and shared-filesystems where the
controller and engine are started at the same time and it
may take a moment for the controller to write the connector files.""")
url_file_name = Unicode(u'ipcontroller-engine.json', config=True)
def _cluster_id_changed(self, name, old, new):
if new:
base = 'ipcontroller-%s' % new
else:
base = 'ipcontroller'
self.url_file_name = "%s-engine.json" % base
log_url = Unicode('', config=True,
help="""The URL for the iploggerapp instance, for forwarding
logging to a central location.""")
# an IPKernelApp instance, used to setup listening for shell frontends
kernel_app = Instance(IPKernelApp, allow_none=True)
aliases = Dict(aliases)
flags = Dict(flags)
@property
def kernel(self):
"""allow access to the Kernel object, so I look like IPKernelApp"""
return self.engine.kernel
def find_url_file(self):
"""Set the url file.
Here we don't try to actually see if it exists for is valid as that
is hadled by the connection logic.
"""
# Find the actual controller key file
if not self.url_file:
self.url_file = os.path.join(
self.profile_dir.security_dir,
self.url_file_name
)
def load_connector_file(self):
"""load config from a JSON connector file,
at a *lower* priority than command-line/config files.
"""
self.log.info("Loading url_file %r", self.url_file)
config = self.config
with open(self.url_file) as f:
num_tries = 0
max_tries = 5
d = ""
while not d:
try:
d = json.loads(f.read())
except ValueError:
if num_tries > max_tries:
raise
num_tries += 1
time.sleep(0.5)
# allow hand-override of location for disambiguation
# and ssh-server
if 'EngineFactory.location' not in config:
config.EngineFactory.location = d['location']
if 'EngineFactory.sshserver' not in config:
config.EngineFactory.sshserver = d.get('ssh')
location = config.EngineFactory.location
proto, ip = d['interface'].split('://')
ip = disambiguate_ip_address(ip, location)
d['interface'] = '%s://%s' % (proto, ip)
# DO NOT allow override of basic URLs, serialization, or key
# JSON file takes top priority there
config.Session.key = cast_bytes(d['key'])
config.Session.signature_scheme = d['signature_scheme']
config.EngineFactory.url = d['interface'] + ':%i' % d['registration']
config.Session.packer = d['pack']
config.Session.unpacker = d['unpack']
self.log.debug("Config changed:")
self.log.debug("%r", config)
self.connection_info = d
def bind_kernel(self, **kwargs):
"""Promote engine to listening kernel, accessible to frontends."""
if self.kernel_app is not None:
return
self.log.info("Opening ports for direct connections as an IPython kernel")
kernel = self.kernel
kwargs.setdefault('config', self.config)
kwargs.setdefault('log', self.log)
kwargs.setdefault('profile_dir', self.profile_dir)
kwargs.setdefault('session', self.engine.session)
app = self.kernel_app = IPKernelApp(**kwargs)
# allow IPKernelApp.instance():
IPKernelApp._instance = app
app.init_connection_file()
# relevant contents of init_sockets:
app.shell_port = app._bind_socket(kernel.shell_streams[0], app.shell_port)
app.log.debug("shell ROUTER Channel on port: %i", app.shell_port)
iopub_socket = kernel.iopub_socket
# ipykernel 4.3 iopub_socket is an IOThread wrapper:
if hasattr(iopub_socket, 'socket'):
iopub_socket = iopub_socket.socket
app.iopub_port = app._bind_socket(iopub_socket, app.iopub_port)
app.log.debug("iopub PUB Channel on port: %i", app.iopub_port)
kernel.stdin_socket = self.engine.context.socket(zmq.ROUTER)
app.stdin_port = app._bind_socket(kernel.stdin_socket, app.stdin_port)
app.log.debug("stdin ROUTER Channel on port: %i", app.stdin_port)
# start the heartbeat, and log connection info:
app.init_heartbeat()
app.log_connection_info()
app.connection_dir = self.profile_dir.security_dir
app.write_connection_file()
def init_engine(self):
# This is the working dir by now.
sys.path.insert(0, '')
config = self.config
# print config
self.find_url_file()
# was the url manually specified?
keys = set(self.config.EngineFactory.keys())
keys = keys.union(set(self.config.RegistrationFactory.keys()))
if self.wait_for_url_file and not os.path.exists(self.url_file):
self.log.warn("url_file %r not found", self.url_file)
self.log.warn("Waiting up to %.1f seconds for it to arrive.", self.wait_for_url_file)
tic = time.time()
while not os.path.exists(self.url_file) and (time.time()-tic < self.wait_for_url_file):
# wait for url_file to exist, or until time limit
time.sleep(0.1)
if os.path.exists(self.url_file):
self.load_connector_file()
else:
self.log.fatal("Fatal: url file never arrived: %s", self.url_file)
self.exit(1)
exec_lines = []
for app in ('IPKernelApp', 'InteractiveShellApp'):
if '%s.exec_lines' % app in config:
exec_lines = config[app].exec_lines
break
exec_files = []
for app in ('IPKernelApp', 'InteractiveShellApp'):
if '%s.exec_files' % app in config:
exec_files = config[app].exec_files
break
config.IPKernelApp.exec_lines = exec_lines
config.IPKernelApp.exec_files = exec_files
if self.startup_script:
exec_files.append(self.startup_script)
if self.startup_command:
exec_lines.append(self.startup_command)
# Create the underlying shell class and Engine
# shell_class = import_item(self.master_config.Global.shell_class)
# print self.config
try:
self.engine = EngineFactory(config=config, log=self.log,
connection_info=self.connection_info,
)
except:
self.log.error("Couldn't start the Engine", exc_info=True)
self.exit(1)
def forward_logging(self):
if self.log_url:
self.log.info("Forwarding logging to %s", self.log_url)
context = self.engine.context
lsock = context.socket(zmq.PUB)
lsock.connect(self.log_url)
handler = EnginePUBHandler(self.engine, lsock)
handler.setLevel(self.log_level)
self.log.addHandler(handler)
def init_mpi(self):
global mpi
self.mpi = MPI(parent=self)
mpi_import_statement = self.mpi.init_script
if mpi_import_statement:
try:
self.log.info("Initializing MPI:")
self.log.info(mpi_import_statement)
exec(mpi_import_statement, globals())
except:
mpi = None
else:
mpi = None
@catch_config_error
def initialize(self, argv=None):
super(IPEngineApp, self).initialize(argv)
self.init_mpi()
self.init_engine()
self.forward_logging()
def start(self):
self.engine.start()
try:
self.engine.loop.start()
except KeyboardInterrupt:
self.log.critical("Engine Interrupted, shutting down...\n")
class Line(GMapsWidgetMixin, widgets.Widget):
"""
Widget representing a single line on a map
Add this line to a map via the :func:`gmaps.drawing_layer` function, or by
passing it directly to the ``.features`` array of an existing instance of
:class:`gmaps.Drawing`.
:Examples:
>>> fig = gmaps.figure()
>>> drawing = gmaps.drawing_layer(features=[
gmaps.Line((46.44, 5.24), (46.23, 5.86), stroke_color='green'),
gmaps.Line((48.44, 1.32), (47.13, 3.91), stroke_weight=5.0)
])
>>> fig.add_layer(drawing)
You can also add a line to an existing :class:`gmaps.Drawing`
instance:
>>> fig = gmaps.figure()
>>> drawing = gmaps.drawing_layer()
>>> fig.add_layer(drawing)
>>> fig # display the figure
You can now add lines directly on the map:
>>> drawing.features = [
gmaps.Line((46.44, 5.24), (46.23, 5.86), stroke_color='green'),
gmaps.Line((48.44, 1.32), (47.13, 3.91), stroke_weight=5.0)
]
:param start:
(latitude, longitude) pair denoting the start of the line. Latitudes
are expressed as a float between -90 (corresponding to 90 degrees
south) and +90 (corresponding to 90 degrees north). Longitudes are
expressed as a float between -180 (corresponding to 180 degrees west)
and +180 (corresponding to 180 degrees east).
:type start: tuple of floats
:param end:
(latitude, longitude) pair denoting the end of the line. Latitudes
are expressed as a float between -90 (corresponding to 90 degrees
south) and +90 (corresponding to 90 degrees north). Longitudes are
expressed as a float between -180 (corresponding to 180 degrees west)
and +180 (corresponding to 180 degrees east).
:type start: tuple of floats
{stroke_options_params}
"""
_view_name = Unicode('LineView').tag(sync=True)
_model_name = Unicode('LineModel').tag(sync=True)
start = geotraitlets.Point().tag(sync=True)
end = geotraitlets.Point().tag(sync=True)
stroke_color = geotraitlets.ColorAlpha(
allow_none=False, default_value=DEFAULT_STROKE_COLOR).tag(sync=True)
stroke_weight = Float(min=0.0, allow_none=False,
default_value=2.0).tag(sync=True)
stroke_opacity = geotraitlets.StrokeOpacity().tag(sync=True)
def __init__(self,
start,
end,
stroke_color=DEFAULT_STROKE_COLOR,
stroke_weight=2.0,
stroke_opacity=geotraitlets.StrokeOpacity.default_value):
kwargs = dict(start=start,
end=end,
stroke_color=stroke_color,
stroke_weight=stroke_weight,
stroke_opacity=stroke_opacity)
super(Line, self).__init__(**kwargs)
class JupyterHub(Application):
"""An Application for starting a Multi-User Jupyter Notebook server."""
name = 'jupyterhub'
version = jupyterhub.__version__
description = """Start a multi-user Jupyter Notebook server
Spawns a configurable-http-proxy and multi-user Hub,
which authenticates users and spawns single-user Notebook servers
on behalf of users.
"""
examples = """
generate default config file:
jupyterhub --generate-config -f /etc/jupyterhub/jupyterhub.py
spawn the server on 10.0.1.2:443 with https:
jupyterhub --ip 10.0.1.2 --port 443 --ssl-key my_ssl.key --ssl-cert my_ssl.cert
"""
aliases = Dict(aliases)
flags = Dict(flags)
subcommands = {'token': (NewToken, "Generate an API token for a user")}
classes = List([
Spawner,
LocalProcessSpawner,
Authenticator,
PAMAuthenticator,
])
config_file = Unicode(
'jupyterhub_config.py',
config=True,
help="The config file to load",
)
generate_config = Bool(
False,
config=True,
help="Generate default config file",
)
answer_yes = Bool(
False,
config=True,
help="Answer yes to any questions (e.g. confirm overwrite)")
pid_file = Unicode('',
config=True,
help="""File to write PID
Useful for daemonizing jupyterhub.
""")
cookie_max_age_days = Float(
14,
config=True,
help="""Number of days for a login cookie to be valid.
Default is two weeks.
""")
last_activity_interval = Integer(
300,
config=True,
help=
"Interval (in seconds) at which to update last-activity timestamps.")
proxy_check_interval = Integer(
30,
config=True,
help="Interval (in seconds) at which to check if the proxy is running."
)
data_files_path = Unicode(
DATA_FILES_PATH,
config=True,
help=
"The location of jupyterhub data files (e.g. /usr/local/share/jupyter/hub)"
)
template_paths = List(
config=True,
help="Paths to search for jinja templates.",
)
def _template_paths_default(self):
return [os.path.join(self.data_files_path, 'templates')]
confirm_no_ssl = Bool(
False,
config=True,
help="""Confirm that JupyterHub should be run without SSL.
This is **NOT RECOMMENDED** unless SSL termination is being handled by another layer.
""")
ssl_key = Unicode(
'',
config=True,
help="""Path to SSL key file for the public facing interface of the proxy
Use with ssl_cert
""")
ssl_cert = Unicode(
'',
config=True,
help=
"""Path to SSL certificate file for the public facing interface of the proxy
Use with ssl_key
""")
ip = Unicode(
'',
config=True,
help="The public facing ip of the whole application (the proxy)")
subdomain_host = Unicode(
'',
config=True,
help="""Run single-user servers on subdomains of this host.
This should be the full https://hub.domain.tld[:port]
Provides additional cross-site protections for javascript served by single-user servers.
Requires <username>.hub.domain.tld to resolve to the same host as hub.domain.tld.
In general, this is most easily achieved with wildcard DNS.
When using SSL (i.e. always) this also requires a wildcard SSL certificate.
""")
def _subdomain_host_changed(self, name, old, new):
if new and '://' not in new:
# host should include '://'
# if not specified, assume https: You have to be really explicit about HTTP!
self.subdomain_host = 'https://' + new
port = Integer(8000,
config=True,
help="The public facing port of the proxy")
base_url = URLPrefix('/',
config=True,
help="The base URL of the entire application")
logo_file = Unicode(
'',
config=True,
help=
"Specify path to a logo image to override the Jupyter logo in the banner."
)
def _logo_file_default(self):
return os.path.join(self.data_files_path, 'static', 'images',
'jupyter.png')
jinja_environment_options = Dict(
config=True,
help="Supply extra arguments that will be passed to Jinja environment."
)
proxy_cmd = Command('configurable-http-proxy',
config=True,
help="""The command to start the http proxy.
Only override if configurable-http-proxy is not on your PATH
""")
debug_proxy = Bool(False,
config=True,
help="show debug output in configurable-http-proxy")
proxy_auth_token = Unicode(config=True,
help="""The Proxy Auth token.
Loaded from the CONFIGPROXY_AUTH_TOKEN env variable by default.
""")
def _proxy_auth_token_default(self):
token = os.environ.get('CONFIGPROXY_AUTH_TOKEN', None)
if not token:
self.log.warn('\n'.join([
"",
"Generating CONFIGPROXY_AUTH_TOKEN. Restarting the Hub will require restarting the proxy.",
"Set CONFIGPROXY_AUTH_TOKEN env or JupyterHub.proxy_auth_token config to avoid this message.",
"",
]))
token = orm.new_token()
return token
proxy_api_ip = Unicode('127.0.0.1',
config=True,
help="The ip for the proxy API handlers")
proxy_api_port = Integer(config=True,
help="The port for the proxy API handlers")
def _proxy_api_port_default(self):
return self.port + 1
hub_port = Integer(8081, config=True, help="The port for this process")
hub_ip = Unicode('127.0.0.1', config=True, help="The ip for this process")
hub_prefix = URLPrefix(
'/hub/',
config=True,
help="The prefix for the hub server. Must not be '/'")
def _hub_prefix_default(self):
return url_path_join(self.base_url, '/hub/')
def _hub_prefix_changed(self, name, old, new):
if new == '/':
raise TraitError("'/' is not a valid hub prefix")
if not new.startswith(self.base_url):
self.hub_prefix = url_path_join(self.base_url, new)
cookie_secret = Bytes(config=True,
env='JPY_COOKIE_SECRET',
help="""The cookie secret to use to encrypt cookies.
Loaded from the JPY_COOKIE_SECRET env variable by default.
""")
cookie_secret_file = Unicode(
'jupyterhub_cookie_secret',
config=True,
help="""File in which to store the cookie secret.""")
authenticator_class = Type(PAMAuthenticator,
Authenticator,
config=True,
help="""Class for authenticating users.
This should be a class with the following form:
- constructor takes one kwarg: `config`, the IPython config object.
- is a tornado.gen.coroutine
- returns username on success, None on failure
- takes two arguments: (handler, data),
where `handler` is the calling web.RequestHandler,
and `data` is the POST form data from the login page.
""")
authenticator = Instance(Authenticator)
def _authenticator_default(self):
return self.authenticator_class(parent=self, db=self.db)
# class for spawning single-user servers
spawner_class = Type(
LocalProcessSpawner,
Spawner,
config=True,
help="""The class to use for spawning single-user servers.
Should be a subclass of Spawner.
""")
db_url = Unicode(
'sqlite:///jupyterhub.sqlite',
config=True,
help="url for the database. e.g. `sqlite:///jupyterhub.sqlite`")
def _db_url_changed(self, name, old, new):
if '://' not in new:
# assume sqlite, if given as a plain filename
self.db_url = 'sqlite:///%s' % new
db_kwargs = Dict(
config=True,
help="""Include any kwargs to pass to the database connection.
See sqlalchemy.create_engine for details.
""")
reset_db = Bool(False, config=True, help="Purge and reset the database.")
debug_db = Bool(
False,
config=True,
help="log all database transactions. This has A LOT of output")
session_factory = Any()
users = Instance(UserDict)
def _users_default(self):
assert self.tornado_settings
return UserDict(db_factory=lambda: self.db,
settings=self.tornado_settings)
admin_access = Bool(
False,
config=True,
help="""Grant admin users permission to access single-user servers.
Users should be properly informed if this is enabled.
""")
admin_users = Set(
config=True,
help="""DEPRECATED, use Authenticator.admin_users instead.""")
tornado_settings = Dict(config=True)
cleanup_servers = Bool(
True,
config=True,
help="""Whether to shutdown single-user servers when the Hub shuts down.
Disable if you want to be able to teardown the Hub while leaving the single-user servers running.
If both this and cleanup_proxy are False, sending SIGINT to the Hub will
only shutdown the Hub, leaving everything else running.
The Hub should be able to resume from database state.
""")
cleanup_proxy = Bool(
True,
config=True,
help="""Whether to shutdown the proxy when the Hub shuts down.
Disable if you want to be able to teardown the Hub while leaving the proxy running.
Only valid if the proxy was starting by the Hub process.
If both this and cleanup_servers are False, sending SIGINT to the Hub will
only shutdown the Hub, leaving everything else running.
The Hub should be able to resume from database state.
""")
handlers = List()
_log_formatter_cls = CoroutineLogFormatter
http_server = None
proxy_process = None
io_loop = None
def _log_level_default(self):
return logging.INFO
def _log_datefmt_default(self):
"""Exclude date from default date format"""
return "%Y-%m-%d %H:%M:%S"
def _log_format_default(self):
"""override default log format to include time"""
return "%(color)s[%(levelname)1.1s %(asctime)s.%(msecs).03d %(name)s %(module)s:%(lineno)d]%(end_color)s %(message)s"
extra_log_file = Unicode("",
config=True,
help="Set a logging.FileHandler on this file.")
extra_log_handlers = List(
Instance(logging.Handler),
config=True,
help="Extra log handlers to set on JupyterHub logger",
)
def init_logging(self):
# This prevents double log messages because tornado use a root logger that
# self.log is a child of. The logging module dipatches log messages to a log
# and all of its ancenstors until propagate is set to False.
self.log.propagate = False
if self.extra_log_file:
self.extra_log_handlers.append(
logging.FileHandler(self.extra_log_file))
_formatter = self._log_formatter_cls(
fmt=self.log_format,
datefmt=self.log_datefmt,
)
for handler in self.extra_log_handlers:
if handler.formatter is None:
handler.setFormatter(_formatter)
self.log.addHandler(handler)
# hook up tornado 3's loggers to our app handlers
for log in (app_log, access_log, gen_log):
# ensure all log statements identify the application they come from
log.name = self.log.name
logger = logging.getLogger('tornado')
logger.propagate = True
logger.parent = self.log
logger.setLevel(self.log.level)
def init_ports(self):
if self.hub_port == self.port:
raise TraitError(
"The hub and proxy cannot both listen on port %i" % self.port)
if self.hub_port == self.proxy_api_port:
raise TraitError(
"The hub and proxy API cannot both listen on port %i" %
self.hub_port)
if self.proxy_api_port == self.port:
raise TraitError(
"The proxy's public and API ports cannot both be %i" %
self.port)
@staticmethod
def add_url_prefix(prefix, handlers):
"""add a url prefix to handlers"""
for i, tup in enumerate(handlers):
lis = list(tup)
lis[0] = url_path_join(prefix, tup[0])
handlers[i] = tuple(lis)
return handlers
def init_handlers(self):
h = []
# load handlers from the authenticator
h.extend(self.authenticator.get_handlers(self))
# set default handlers
h.extend(handlers.default_handlers)
h.extend(apihandlers.default_handlers)
h.append((r'/logo', LogoHandler, {'path': self.logo_file}))
self.handlers = self.add_url_prefix(self.hub_prefix, h)
# some extra handlers, outside hub_prefix
self.handlers.extend([
(r"%s" % self.hub_prefix.rstrip('/'), web.RedirectHandler, {
"url": self.hub_prefix,
"permanent": False,
}),
(r"(?!%s).*" % self.hub_prefix, handlers.PrefixRedirectHandler),
(r'(.*)', handlers.Template404),
])
def _check_db_path(self, path):
"""More informative log messages for failed filesystem access"""
path = os.path.abspath(path)
parent, fname = os.path.split(path)
user = getuser()
if not os.path.isdir(parent):
self.log.error("Directory %s does not exist", parent)
if os.path.exists(parent) and not os.access(parent, os.W_OK):
self.log.error("%s cannot create files in %s", user, parent)
if os.path.exists(path) and not os.access(path, os.W_OK):
self.log.error("%s cannot edit %s", user, path)
def init_secrets(self):
trait_name = 'cookie_secret'
trait = self.traits()[trait_name]
env_name = trait.get_metadata('env')
secret_file = os.path.abspath(
os.path.expanduser(self.cookie_secret_file))
secret = self.cookie_secret
secret_from = 'config'
# load priority: 1. config, 2. env, 3. file
if not secret and os.environ.get(env_name):
secret_from = 'env'
self.log.info("Loading %s from env[%s]", trait_name, env_name)
secret = binascii.a2b_hex(os.environ[env_name])
if not secret and os.path.exists(secret_file):
secret_from = 'file'
perm = os.stat(secret_file).st_mode
if perm & 0o077:
self.log.error("Bad permissions on %s", secret_file)
else:
self.log.info("Loading %s from %s", trait_name, secret_file)
with open(secret_file) as f:
b64_secret = f.read()
try:
secret = binascii.a2b_base64(b64_secret)
except Exception as e:
self.log.error("%s does not contain b64 key: %s",
secret_file, e)
if not secret:
secret_from = 'new'
self.log.debug("Generating new %s", trait_name)
secret = os.urandom(SECRET_BYTES)
if secret_file and secret_from == 'new':
# if we generated a new secret, store it in the secret_file
self.log.info("Writing %s to %s", trait_name, secret_file)
b64_secret = binascii.b2a_base64(secret).decode('ascii')
with open(secret_file, 'w') as f:
f.write(b64_secret)
try:
os.chmod(secret_file, 0o600)
except OSError:
self.log.warn("Failed to set permissions on %s", secret_file)
# store the loaded trait value
self.cookie_secret = secret
# thread-local storage of db objects
_local = Instance(threading.local, ())
@property
def db(self):
if not hasattr(self._local, 'db'):
self._local.db = scoped_session(self.session_factory)()
return self._local.db
@property
def hub(self):
if not getattr(self._local, 'hub', None):
q = self.db.query(orm.Hub)
assert q.count() <= 1
self._local.hub = q.first()
if self.subdomain_host and self._local.hub:
self._local.hub.host = self.subdomain_host
return self._local.hub
@hub.setter
def hub(self, hub):
self._local.hub = hub
if hub and self.subdomain_host:
hub.host = self.subdomain_host
@property
def proxy(self):
if not getattr(self._local, 'proxy', None):
q = self.db.query(orm.Proxy)
assert q.count() <= 1
p = self._local.proxy = q.first()
if p:
p.auth_token = self.proxy_auth_token
return self._local.proxy
@proxy.setter
def proxy(self, proxy):
self._local.proxy = proxy
def init_db(self):
"""Create the database connection"""
self.log.debug("Connecting to db: %s", self.db_url)
try:
self.session_factory = orm.new_session_factory(self.db_url,
reset=self.reset_db,
echo=self.debug_db,
**self.db_kwargs)
# trigger constructing thread local db property
_ = self.db
except OperationalError as e:
self.log.error("Failed to connect to db: %s", self.db_url)
self.log.debug("Database error was:", exc_info=True)
if self.db_url.startswith('sqlite:///'):
self._check_db_path(self.db_url.split(':///', 1)[1])
self.exit(1)
def init_hub(self):
"""Load the Hub config into the database"""
self.hub = self.db.query(orm.Hub).first()
if self.hub is None:
self.hub = orm.Hub(server=orm.Server(
ip=self.hub_ip,
port=self.hub_port,
base_url=self.hub_prefix,
cookie_name='jupyter-hub-token',
))
self.db.add(self.hub)
else:
server = self.hub.server
server.ip = self.hub_ip
server.port = self.hub_port
server.base_url = self.hub_prefix
if self.subdomain_host:
if not self.subdomain_host:
raise ValueError(
"Must specify subdomain_host when using subdomains."
" This should be the public domain[:port] of the Hub.")
self.db.commit()
@gen.coroutine
def init_users(self):
"""Load users into and from the database"""
db = self.db
if self.admin_users and not self.authenticator.admin_users:
self.log.warn("\nJupyterHub.admin_users is deprecated."
"\nUse Authenticator.admin_users instead.")
self.authenticator.admin_users = self.admin_users
admin_users = [
self.authenticator.normalize_username(name)
for name in self.authenticator.admin_users
]
for username in admin_users:
if not self.authenticator.validate_username(username):
raise ValueError("username %r is not valid" % username)
if not admin_users:
self.log.warning(
"No admin users, admin interface will be unavailable.")
self.log.warning(
"Add any administrative users to `c.Authenticator.admin_users` in config."
)
new_users = []
for name in admin_users:
# ensure anyone specified as admin in config is admin in db
user = orm.User.find(db, name)
if user is None:
user = orm.User(name=name, admin=True)
new_users.append(user)
db.add(user)
else:
user.admin = True
# the admin_users config variable will never be used after this point.
# only the database values will be referenced.
whitelist = [
self.authenticator.normalize_username(name)
for name in self.authenticator.whitelist
]
for username in whitelist:
if not self.authenticator.validate_username(username):
raise ValueError("username %r is not valid" % username)
if not whitelist:
self.log.info(
"Not using whitelist. Any authenticated user will be allowed.")
# add whitelisted users to the db
for name in whitelist:
user = orm.User.find(db, name)
if user is None:
user = orm.User(name=name)
new_users.append(user)
db.add(user)
if whitelist:
# fill the whitelist with any users loaded from the db,
# so we are consistent in both directions.
# This lets whitelist be used to set up initial list,
# but changes to the whitelist can occur in the database,
# and persist across sessions.
for user in db.query(orm.User):
self.authenticator.whitelist.add(user.name)
# The whitelist set and the users in the db are now the same.
# From this point on, any user changes should be done simultaneously
# to the whitelist set and user db, unless the whitelist is empty (all users allowed).
db.commit()
for user in new_users:
yield gen.maybe_future(self.authenticator.add_user(user))
db.commit()
@gen.coroutine
def init_spawners(self):
db = self.db
user_summaries = ['']
def _user_summary(user):
parts = ['{0: >8}'.format(user.name)]
if user.admin:
parts.append('admin')
if user.server:
parts.append('running at %s' % user.server)
return ' '.join(parts)
@gen.coroutine
def user_stopped(user):
status = yield user.spawner.poll()
self.log.warn(
"User %s server stopped with exit code: %s",
user.name,
status,
)
yield self.proxy.delete_user(user)
yield user.stop()
for orm_user in db.query(orm.User):
self.users[orm_user.id] = user = User(orm_user,
self.tornado_settings)
if not user.state:
# without spawner state, server isn't valid
user.server = None
user_summaries.append(_user_summary(user))
continue
self.log.debug("Loading state for %s from db", user.name)
spawner = user.spawner
status = yield spawner.poll()
if status is None:
self.log.info("%s still running", user.name)
spawner.add_poll_callback(user_stopped, user)
spawner.start_polling()
else:
# user not running. This is expected if server is None,
# but indicates the user's server died while the Hub wasn't running
# if user.server is defined.
log = self.log.warn if user.server else self.log.debug
log("%s not running.", user.name)
user.server = None
user_summaries.append(_user_summary(user))
self.log.debug("Loaded users: %s", '\n'.join(user_summaries))
db.commit()
def init_proxy(self):
"""Load the Proxy config into the database"""
self.proxy = self.db.query(orm.Proxy).first()
if self.proxy is None:
self.proxy = orm.Proxy(
public_server=orm.Server(),
api_server=orm.Server(),
)
self.db.add(self.proxy)
self.db.commit()
self.proxy.auth_token = self.proxy_auth_token # not persisted
self.proxy.log = self.log
self.proxy.public_server.ip = self.ip
self.proxy.public_server.port = self.port
self.proxy.api_server.ip = self.proxy_api_ip
self.proxy.api_server.port = self.proxy_api_port
self.proxy.api_server.base_url = '/api/routes/'
self.db.commit()
@gen.coroutine
def start_proxy(self):
"""Actually start the configurable-http-proxy"""
# check for proxy
if self.proxy.public_server.is_up() or self.proxy.api_server.is_up():
# check for *authenticated* access to the proxy (auth token can change)
try:
yield self.proxy.get_routes()
except (HTTPError, OSError, socket.error) as e:
if isinstance(e, HTTPError) and e.code == 403:
msg = "Did CONFIGPROXY_AUTH_TOKEN change?"
else:
msg = "Is something else using %s?" % self.proxy.public_server.bind_url
self.log.error(
"Proxy appears to be running at %s, but I can't access it (%s)\n%s",
self.proxy.public_server.bind_url, e, msg)
self.exit(1)
return
else:
self.log.info("Proxy already running at: %s",
self.proxy.public_server.bind_url)
self.proxy_process = None
return
env = os.environ.copy()
env['CONFIGPROXY_AUTH_TOKEN'] = self.proxy.auth_token
cmd = self.proxy_cmd + [
'--ip',
self.proxy.public_server.ip,
'--port',
str(self.proxy.public_server.port),
'--api-ip',
self.proxy.api_server.ip,
'--api-port',
str(self.proxy.api_server.port),
'--default-target',
self.hub.server.host,
]
if self.subdomain_host:
cmd.append('--host-routing')
if self.debug_proxy:
cmd.extend(['--log-level', 'debug'])
if self.ssl_key:
cmd.extend(['--ssl-key', self.ssl_key])
if self.ssl_cert:
cmd.extend(['--ssl-cert', self.ssl_cert])
# Require SSL to be used or `--no-ssl` to confirm no SSL on
if ' --ssl' not in ' '.join(cmd):
if self.confirm_no_ssl:
self.log.warning(
"Running JupyterHub without SSL."
" There better be SSL termination happening somewhere else..."
)
else:
self.log.error(
"Refusing to run JuptyterHub without SSL."
" If you are terminating SSL in another layer,"
" pass --no-ssl to tell JupyterHub to allow the proxy to listen on HTTP."
)
self.exit(1)
self.log.info("Starting proxy @ %s", self.proxy.public_server.bind_url)
self.log.debug("Proxy cmd: %s", cmd)
try:
self.proxy_process = Popen(cmd, env=env)
except FileNotFoundError as e:
self.log.error(
"Failed to find proxy %r\n"
"The proxy can be installed with `npm install -g configurable-http-proxy`"
% self.proxy_cmd)
self.exit(1)
def _check():
status = self.proxy_process.poll()
if status is not None:
e = RuntimeError("Proxy failed to start with exit code %i" %
status)
# py2-compatible `raise e from None`
e.__cause__ = None
raise e
for server in (self.proxy.public_server, self.proxy.api_server):
for i in range(10):
_check()
try:
yield server.wait_up(1)
except TimeoutError:
continue
else:
break
yield server.wait_up(1)
self.log.debug("Proxy started and appears to be up")
@gen.coroutine
def check_proxy(self):
if self.proxy_process.poll() is None:
return
self.log.error(
"Proxy stopped with exit code %r", 'unknown'
if self.proxy_process is None else self.proxy_process.poll())
yield self.start_proxy()
self.log.info("Setting up routes on new proxy")
yield self.proxy.add_all_users(self.users)
self.log.info("New proxy back up, and good to go")
def init_tornado_settings(self):
"""Set up the tornado settings dict."""
base_url = self.hub.server.base_url
jinja_options = dict(autoescape=True, )
jinja_options.update(self.jinja_environment_options)
jinja_env = Environment(loader=FileSystemLoader(self.template_paths),
**jinja_options)
login_url = self.authenticator.login_url(base_url)
logout_url = self.authenticator.logout_url(base_url)
# if running from git, disable caching of require.js
# otherwise cache based on server start time
parent = os.path.dirname(os.path.dirname(jupyterhub.__file__))
if os.path.isdir(os.path.join(parent, '.git')):
version_hash = ''
else:
version_hash = datetime.now().strftime("%Y%m%d%H%M%S"),
subdomain_host = self.subdomain_host
domain = urlparse(subdomain_host).hostname
settings = dict(
log_function=log_request,
config=self.config,
log=self.log,
db=self.db,
proxy=self.proxy,
hub=self.hub,
admin_users=self.authenticator.admin_users,
admin_access=self.admin_access,
authenticator=self.authenticator,
spawner_class=self.spawner_class,
base_url=self.base_url,
cookie_secret=self.cookie_secret,
cookie_max_age_days=self.cookie_max_age_days,
login_url=login_url,
logout_url=logout_url,
static_path=os.path.join(self.data_files_path, 'static'),
static_url_prefix=url_path_join(self.hub.server.base_url,
'static/'),
static_handler_class=CacheControlStaticFilesHandler,
template_path=self.template_paths,
jinja2_env=jinja_env,
version_hash=version_hash,
subdomain_host=subdomain_host,
domain=domain,
)
# allow configured settings to have priority
settings.update(self.tornado_settings)
self.tornado_settings = settings
# constructing users requires access to tornado_settings
self.tornado_settings['users'] = self.users
def init_tornado_application(self):
"""Instantiate the tornado Application object"""
self.tornado_application = web.Application(self.handlers,
**self.tornado_settings)
def write_pid_file(self):
pid = os.getpid()
if self.pid_file:
self.log.debug("Writing PID %i to %s", pid, self.pid_file)
with open(self.pid_file, 'w') as f:
f.write('%i' % pid)
@gen.coroutine
@catch_config_error
def initialize(self, *args, **kwargs):
super().initialize(*args, **kwargs)
if self.generate_config or self.subapp:
return
self.load_config_file(self.config_file)
self.init_logging()
if 'JupyterHubApp' in self.config:
self.log.warn(
"Use JupyterHub in config, not JupyterHubApp. Outdated config:\n%s",
'\n'.join('JupyterHubApp.{key} = {value!r}'.format(key=key,
value=value)
for key, value in self.config.JupyterHubApp.items()))
cfg = self.config.copy()
cfg.JupyterHub.merge(cfg.JupyterHubApp)
self.update_config(cfg)
self.write_pid_file()
self.init_ports()
self.init_secrets()
self.init_db()
self.init_hub()
self.init_proxy()
yield self.init_users()
self.init_tornado_settings()
yield self.init_spawners()
self.init_handlers()
self.init_tornado_application()
@gen.coroutine
def cleanup(self):
"""Shutdown our various subprocesses and cleanup runtime files."""
futures = []
if self.cleanup_servers:
self.log.info("Cleaning up single-user servers...")
# request (async) process termination
for uid, user in self.users.items():
if user.spawner is not None:
futures.append(user.stop())
else:
self.log.info("Leaving single-user servers running")
# clean up proxy while SUS are shutting down
if self.cleanup_proxy:
if self.proxy_process:
self.log.info("Cleaning up proxy[%i]...",
self.proxy_process.pid)
if self.proxy_process.poll() is None:
try:
self.proxy_process.terminate()
except Exception as e:
self.log.error("Failed to terminate proxy process: %s",
e)
else:
self.log.info("I didn't start the proxy, I can't clean it up")
else:
self.log.info("Leaving proxy running")
# wait for the requests to stop finish:
for f in futures:
try:
yield f
except Exception as e:
self.log.error("Failed to stop user: %s", e)
self.db.commit()
if self.pid_file and os.path.exists(self.pid_file):
self.log.info("Cleaning up PID file %s", self.pid_file)
os.remove(self.pid_file)
# finally stop the loop once we are all cleaned up
self.log.info("...done")
def write_config_file(self):
"""Write our default config to a .py config file"""
if os.path.exists(self.config_file) and not self.answer_yes:
answer = ''
def ask():
prompt = "Overwrite %s with default config? [y/N]" % self.config_file
try:
return input(prompt).lower() or 'n'
except KeyboardInterrupt:
print('') # empty line
return 'n'
answer = ask()
while not answer.startswith(('y', 'n')):
print("Please answer 'yes' or 'no'")
answer = ask()
if answer.startswith('n'):
return
config_text = self.generate_config_file()
if isinstance(config_text, bytes):
config_text = config_text.decode('utf8')
print("Writing default config to: %s" % self.config_file)
with open(self.config_file, mode='w') as f:
f.write(config_text)
@gen.coroutine
def update_last_activity(self):
"""Update User.last_activity timestamps from the proxy"""
routes = yield self.proxy.get_routes()
for prefix, route in routes.items():
if 'user' not in route:
# not a user route, ignore it
continue
user = orm.User.find(self.db, route['user'])
if user is None:
self.log.warn("Found no user for route: %s", route)
continue
try:
dt = datetime.strptime(route['last_activity'], ISO8601_ms)
except Exception:
dt = datetime.strptime(route['last_activity'], ISO8601_s)
user.last_activity = max(user.last_activity, dt)
self.db.commit()
yield self.proxy.check_routes(self.users, routes)
@gen.coroutine
def start(self):
"""Start the whole thing"""
self.io_loop = loop = IOLoop.current()
if self.subapp:
self.subapp.start()
loop.stop()
return
if self.generate_config:
self.write_config_file()
loop.stop()
return
# start the webserver
self.http_server = tornado.httpserver.HTTPServer(
self.tornado_application, xheaders=True)
try:
self.http_server.listen(self.hub_port, address=self.hub_ip)
except Exception:
self.log.error("Failed to bind hub to %s",
self.hub.server.bind_url)
raise
else:
self.log.info("Hub API listening on %s", self.hub.server.bind_url)
# start the proxy
try:
yield self.start_proxy()
except Exception as e:
self.log.critical("Failed to start proxy", exc_info=True)
self.exit(1)
return
loop.add_callback(self.proxy.add_all_users, self.users)
if self.proxy_process:
# only check / restart the proxy if we started it in the first place.
# this means a restarted Hub cannot restart a Proxy that its
# predecessor started.
pc = PeriodicCallback(self.check_proxy,
1e3 * self.proxy_check_interval)
pc.start()
if self.last_activity_interval:
pc = PeriodicCallback(self.update_last_activity,
1e3 * self.last_activity_interval)
pc.start()
self.log.info("JupyterHub is now running at %s",
self.proxy.public_server.url)
# register cleanup on both TERM and INT
atexit.register(self.atexit)
self.init_signal()
def init_signal(self):
signal.signal(signal.SIGTERM, self.sigterm)
def sigterm(self, signum, frame):
self.log.critical("Received SIGTERM, shutting down")
self.io_loop.stop()
self.atexit()
_atexit_ran = False
def atexit(self):
"""atexit callback"""
if self._atexit_ran:
return
self._atexit_ran = True
# run the cleanup step (in a new loop, because the interrupted one is unclean)
IOLoop.clear_current()
loop = IOLoop()
loop.make_current()
loop.run_sync(self.cleanup)
def stop(self):
if not self.io_loop:
return
if self.http_server:
if self.io_loop._running:
self.io_loop.add_callback(self.http_server.stop)
else:
self.http_server.stop()
self.io_loop.add_callback(self.io_loop.stop)
@gen.coroutine
def launch_instance_async(self, argv=None):
try:
yield self.initialize(argv)
yield self.start()
except Exception as e:
self.log.exception("")
self.exit(1)
@classmethod
def launch_instance(cls, argv=None):
self = cls.instance()
loop = IOLoop.current()
loop.add_callback(self.launch_instance_async, argv)
try:
loop.start()
except KeyboardInterrupt:
print("\nInterrupted")
class MarketMap(DOMWidget):
"""Waffle wrapped map.
Attributes
----------
names: numpy.ndarray of strings (default: [])
The elements can also be objects convertible to string
primary key for the map data. A rectangle is created for each
unique entry in this array
groups: numpy.ndarray (default: [])
attribute on which the groupby is run. If this is an empty arrray, then
there is no group by for the map.
display_text: numpy.ndarray or None(default: None)
data to be displayed on each rectangle of the map.If this is empty it
defaults to the names attribute.
ref_data: pandas.DataDrame or None (default: None)
Additional data associated with each element of the map. The data in
this data frame can be displayed as a tooltip.
color: numpy.ndarray (default: [])
Data to represent the color for each of the cells. If the value of the
data is NaN for a cell, then the color of the cell is the color of the
group it belongs to in absence of data for color
scales: Dictionary of scales holding a scale for each data attribute
If the map has data being passed as color, then a corresponding color
scale is required
axes: List of axes
Ability to add an axis for the scales which are used to scale data
represented in the map
on_hover: custom event
This event is received when the mouse is hovering over a cell. Returns
the data of the cell and the ref_data associated with the cell.
tooltip_widget: Instance of a widget
Widget to be displayed as the tooltip. This can be combined with the
on_hover event to display the chart corresponding to the cell being
hovered on.
tooltip_fields: list
names of the fields from the ref_data dataframe which should be
displayed in the tooltip.
tooltip_formats: list
formats for each of the fields for the tooltip data. Order should match
the order of the tooltip_fields
show_groups: bool
attribute to determine if the groups should be displayed. If set to
True, the finer elements are blurred
Map Drawing Attributes
cols: int
Suggestion for no of columns in the map.If not specified, value is
inferred from the no of rows and no of cells
rows: int
No of rows in the map.If not specified, value is inferred from the no
of cells and no of columns.
If both rows and columns are not specified, then a square is
constructed basing on the no of cells.
The above two attributes are suggestions which are respected unless
they are not feasible. One required condition is that, the number of
columns is odd when row_groups is greater than 1.
row_groups: int
No of groups the rows should be divided into. This can be used to draw
more square cells for each of the groups
Layout Attributes
map_margin: dict (default: {top=50, bottom=50, left=50, right=50})
Dictionary containing the top, bottom, left and right margins. The user
is responsible for making sure that the width and height are greater
than the sum of the margins.
min_aspect_ratio: float
minimum width / height ratio of the figure
max_aspect_ratio: float
maximum width / height ratio of the figure
Display Attributes
colors: list of colors
Colors for each of the groups which are cycled over to cover all the
groups
title: string
Title of the Market Map
title_style: dict
CSS style for the title of the Market Map
stroke: color
Stroke of each of the cells of the market map
group_stroke: color
Stroke of the border for the group of cells corresponding to a group
selected_stroke: color
stroke for the selected cells
hovered_stroke: color
stroke for the cell being hovered on
font_style: dict
CSS style for the text of each cell
Other Attributes
enable_select: bool
boolean to control the ability to select the cells of the map by
clicking
enable_hover: bool
boolean to control if the map should be aware of which cell is being
hovered on. If it is set to False, tooltip will not be displayed
Note
----
The aspect ratios stand for width / height ratios.
- If the available space is within bounds in terms of min and max aspect
ratio, we use the entire available space.
- If the available space is too oblong horizontally, we use the client
height and the width that corresponds max_aspect_ratio (maximize width
under the constraints).
- If the available space is too oblong vertically, we use the client width
and the height that corresponds to min_aspect_ratio (maximize height
under the constraint).
This corresponds to maximizing the area under the constraints.
Default min and max aspect ratio are both equal to 16 / 9.
"""
names = Array([]).tag(sync=True, **array_serialization)
groups = Array([]).tag(sync=True, **array_serialization)
display_text = Array(None, allow_none=True).tag(sync=True,
**array_serialization)
ref_data = DataFrame(None, allow_none=True).tag(
sync=True, **dataframe_serialization).valid(dataframe_warn_indexname)
title = Unicode().tag(sync=True)
tooltip_fields = List().tag(sync=True)
tooltip_formats = List().tag(sync=True)
show_groups = Bool().tag(sync=True)
cols = Int(allow_none=True).tag(sync=True)
rows = Int(allow_none=True).tag(sync=True)
row_groups = Int(1).tag(sync=True)
colors = List(CATEGORY10).tag(sync=True)
scales = Dict().tag(sync=True, **widget_serialization)
axes = List().tag(sync=True, **widget_serialization)
color = Array([]).tag(sync=True, **array_serialization)
map_margin = Dict(dict(top=50, right=50, left=50,
bottom=50)).tag(sync=True)
layout = Instance(Layout, kw={
'min_width': '125px'
}, allow_none=True).tag(sync=True, **widget_serialization)
min_aspect_ratio = Float(1.0).tag(sync=True)
# Max aspect ratio is such that we can have 3 charts stacked vertically
# on a 16:9 monitor: 16/9*3 ~ 5.333
max_aspect_ratio = Float(6.0).tag(sync=True)
stroke = Color('white').tag(sync=True)
group_stroke = Color('black').tag(sync=True)
selected_stroke = Color('dodgerblue', allow_none=True).tag(sync=True)
hovered_stroke = Color('orangered', allow_none=True).tag(sync=True)
font_style = Dict().tag(sync=True)
title_style = Dict().tag(sync=True)
selected = List().tag(sync=True)
enable_hover = Bool(True).tag(sync=True)
enable_select = Bool(True).tag(sync=True)
tooltip_widget = Instance(DOMWidget, allow_none=True,
default_value=None).tag(sync=True,
**widget_serialization)
def __init__(self, **kwargs):
super(MarketMap, self).__init__(**kwargs)
self._hover_handlers = CallbackDispatcher()
self.on_msg(self._handle_custom_msgs)
def on_hover(self, callback, remove=False):
self._hover_handlers.register_callback(callback, remove=remove)
def _handle_custom_msgs(self, _, content, buffers=None):
if content.get('event', '') == 'hover':
self._hover_handlers(self, content)
_view_name = Unicode('MarketMap').tag(sync=True)
_model_name = Unicode('MarketMapModel').tag(sync=True)
_view_module = Unicode('bqplot').tag(sync=True)
_model_module = Unicode('bqplot').tag(sync=True)
_view_module_version = Unicode(__frontend_version__).tag(sync=True)
_model_module_version = Unicode(__frontend_version__).tag(sync=True)
class RiskConfig(HasTraits):
max_drawdown = Float(
default_value=100.0) # % Max drawdown before liquidation
max_risk = Float(default_value=100.0) # % Max to risk
total_funds = Float(default_value=0.0) # total funds available
trading_type = Instance(klass=TradingType, args=('NONE', ), kwargs={})
class Kernel(SingletonConfigurable):
#---------------------------------------------------------------------------
# Kernel interface
#---------------------------------------------------------------------------
# attribute to override with a GUI
eventloop = Any(None)
@observe('eventloop')
def _update_eventloop(self, change):
"""schedule call to eventloop from IOLoop"""
loop = ioloop.IOLoop.instance()
loop.add_callback(self.enter_eventloop)
session = Instance(Session, allow_none=True)
profile_dir = Instance('IPython.core.profiledir.ProfileDir',
allow_none=True)
shell_streams = List()
control_stream = Instance(ZMQStream, allow_none=True)
iopub_socket = Any()
iopub_thread = Any()
stdin_socket = Any()
log = Instance(logging.Logger, allow_none=True)
# identities:
int_id = Integer(-1)
ident = Unicode()
@default('ident')
def _default_ident(self):
return unicode_type(uuid.uuid4())
# This should be overridden by wrapper kernels that implement any real
# language.
language_info = {}
# any links that should go in the help menu
help_links = List()
# Private interface
_darwin_app_nap = Bool(
True,
help="""Whether to use appnope for compatiblity with OS X App Nap.
Only affects OS X >= 10.9.
""").tag(config=True)
# track associations with current request
_allow_stdin = Bool(False)
_parent_header = Dict()
_parent_ident = Any(b'')
# Time to sleep after flushing the stdout/err buffers in each execute
# cycle. While this introduces a hard limit on the minimal latency of the
# execute cycle, it helps prevent output synchronization problems for
# clients.
# Units are in seconds. The minimum zmq latency on local host is probably
# ~150 microseconds, set this to 500us for now. We may need to increase it
# a little if it's not enough after more interactive testing.
_execute_sleep = Float(0.0005).tag(config=True)
# Frequency of the kernel's event loop.
# Units are in seconds, kernel subclasses for GUI toolkits may need to
# adapt to milliseconds.
_poll_interval = Float(0.05).tag(config=True)
# If the shutdown was requested over the network, we leave here the
# necessary reply message so it can be sent by our registered atexit
# handler. This ensures that the reply is only sent to clients truly at
# the end of our shutdown process (which happens after the underlying
# IPython shell's own shutdown).
_shutdown_message = None
# This is a dict of port number that the kernel is listening on. It is set
# by record_ports and used by connect_request.
_recorded_ports = Dict()
# set of aborted msg_ids
aborted = Set()
# Track execution count here. For IPython, we override this to use the
# execution count we store in the shell.
execution_count = 0
msg_types = [
'execute_request',
'complete_request',
'inspect_request',
'history_request',
'comm_info_request',
'kernel_info_request',
'connect_request',
'shutdown_request',
'is_complete_request',
# deprecated:
'apply_request',
]
# add deprecated ipyparallel control messages
control_msg_types = msg_types + ['clear_request', 'abort_request']
def __init__(self, **kwargs):
super(Kernel, self).__init__(**kwargs)
# Build dict of handlers for message types
self.shell_handlers = {}
for msg_type in self.msg_types:
self.shell_handlers[msg_type] = getattr(self, msg_type)
self.control_handlers = {}
for msg_type in self.control_msg_types:
self.control_handlers[msg_type] = getattr(self, msg_type)
def dispatch_control(self, msg):
"""dispatch control requests"""
idents, msg = self.session.feed_identities(msg, copy=False)
try:
msg = self.session.deserialize(msg, content=True, copy=False)
except:
self.log.error("Invalid Control Message", exc_info=True)
return
self.log.debug("Control received: %s", msg)
# Set the parent message for side effects.
self.set_parent(idents, msg)
self._publish_status(u'busy')
header = msg['header']
msg_type = header['msg_type']
handler = self.control_handlers.get(msg_type, None)
if handler is None:
self.log.error("UNKNOWN CONTROL MESSAGE TYPE: %r", msg_type)
else:
try:
handler(self.control_stream, idents, msg)
except Exception:
self.log.error("Exception in control handler:", exc_info=True)
sys.stdout.flush()
sys.stderr.flush()
self._publish_status(u'idle')
def should_handle(self, stream, msg, idents):
"""Check whether a shell-channel message should be handled
Allows subclasses to prevent handling of certain messages (e.g. aborted requests).
"""
msg_id = msg['header']['msg_id']
if msg_id in self.aborted:
msg_type = msg['header']['msg_type']
# is it safe to assume a msg_id will not be resubmitted?
self.aborted.remove(msg_id)
reply_type = msg_type.split('_')[0] + '_reply'
status = {'status': 'aborted'}
md = {'engine': self.ident}
md.update(status)
self.session.send(stream,
reply_type,
metadata=md,
content=status,
parent=msg,
ident=idents)
return False
return True
def dispatch_shell(self, stream, msg):
"""dispatch shell requests"""
# flush control requests first
if self.control_stream:
self.control_stream.flush()
idents, msg = self.session.feed_identities(msg, copy=False)
try:
msg = self.session.deserialize(msg, content=True, copy=False)
except:
self.log.error("Invalid Message", exc_info=True)
return
# Set the parent message for side effects.
self.set_parent(idents, msg)
self._publish_status(u'busy')
header = msg['header']
msg_id = header['msg_id']
msg_type = msg['header']['msg_type']
# Print some info about this message and leave a '--->' marker, so it's
# easier to trace visually the message chain when debugging. Each
# handler prints its message at the end.
self.log.debug('\n*** MESSAGE TYPE:%s***', msg_type)
self.log.debug(' Content: %s\n --->\n ', msg['content'])
if not self.should_handle(stream, msg, idents):
return
handler = self.shell_handlers.get(msg_type, None)
if handler is None:
self.log.error("UNKNOWN MESSAGE TYPE: %r", msg_type)
else:
self.log.debug("%s: %s", msg_type, msg)
self.pre_handler_hook()
try:
handler(stream, idents, msg)
except Exception:
self.log.error("Exception in message handler:", exc_info=True)
finally:
self.post_handler_hook()
sys.stdout.flush()
sys.stderr.flush()
self._publish_status(u'idle')
def pre_handler_hook(self):
"""Hook to execute before calling message handler"""
# ensure default_int_handler during handler call
self.saved_sigint_handler = signal(SIGINT, default_int_handler)
def post_handler_hook(self):
"""Hook to execute after calling message handler"""
signal(SIGINT, self.saved_sigint_handler)
def enter_eventloop(self):
"""enter eventloop"""
self.log.info("entering eventloop %s", self.eventloop)
for stream in self.shell_streams:
# flush any pending replies,
# which may be skipped by entering the eventloop
stream.flush(zmq.POLLOUT)
# restore default_int_handler
signal(SIGINT, default_int_handler)
while self.eventloop is not None:
try:
self.eventloop(self)
except KeyboardInterrupt:
# Ctrl-C shouldn't crash the kernel
self.log.error("KeyboardInterrupt caught in kernel")
continue
else:
# eventloop exited cleanly, this means we should stop (right?)
self.eventloop = None
break
self.log.info("exiting eventloop")
def start(self):
"""register dispatchers for streams"""
if self.control_stream:
self.control_stream.on_recv(self.dispatch_control, copy=False)
def make_dispatcher(stream):
def dispatcher(msg):
return self.dispatch_shell(stream, msg)
return dispatcher
for s in self.shell_streams:
s.on_recv(make_dispatcher(s), copy=False)
# publish idle status
self._publish_status('starting')
def do_one_iteration(self):
"""step eventloop just once"""
if self.control_stream:
self.control_stream.flush()
for stream in self.shell_streams:
# handle at most one request per iteration
stream.flush(zmq.POLLIN, 1)
stream.flush(zmq.POLLOUT)
def record_ports(self, ports):
"""Record the ports that this kernel is using.
The creator of the Kernel instance must call this methods if they
want the :meth:`connect_request` method to return the port numbers.
"""
self._recorded_ports = ports
#---------------------------------------------------------------------------
# Kernel request handlers
#---------------------------------------------------------------------------
def _publish_execute_input(self, code, parent, execution_count):
"""Publish the code request on the iopub stream."""
self.session.send(self.iopub_socket,
u'execute_input', {
u'code': code,
u'execution_count': execution_count
},
parent=parent,
ident=self._topic('execute_input'))
def _publish_status(self, status, parent=None):
"""send status (busy/idle) on IOPub"""
self.session.send(
self.iopub_socket,
u'status',
{u'execution_state': status},
parent=parent or self._parent_header,
ident=self._topic('status'),
)
def set_parent(self, ident, parent):
"""Set the current parent_header
Side effects (IOPub messages) and replies are associated with
the request that caused them via the parent_header.
The parent identity is used to route input_request messages
on the stdin channel.
"""
self._parent_ident = ident
self._parent_header = parent
def send_response(self,
stream,
msg_or_type,
content=None,
ident=None,
buffers=None,
track=False,
header=None,
metadata=None):
"""Send a response to the message we're currently processing.
This accepts all the parameters of :meth:`jupyter_client.session.Session.send`
except ``parent``.
This relies on :meth:`set_parent` having been called for the current
message.
"""
return self.session.send(stream, msg_or_type, content,
self._parent_header, ident, buffers, track,
header, metadata)
def init_metadata(self, parent):
"""Initialize metadata.
Run at the beginning of execution requests.
"""
return {
'started': datetime.now(),
}
def finish_metadata(self, parent, metadata, reply_content):
"""Finish populating metadata.
Run after completing an execution request.
"""
return metadata
def execute_request(self, stream, ident, parent):
"""handle an execute_request"""
try:
content = parent[u'content']
code = py3compat.cast_unicode_py2(content[u'code'])
silent = content[u'silent']
store_history = content.get(u'store_history', not silent)
user_expressions = content.get('user_expressions', {})
allow_stdin = content.get('allow_stdin', False)
except:
self.log.error("Got bad msg: ")
self.log.error("%s", parent)
return
stop_on_error = content.get('stop_on_error', True)
metadata = self.init_metadata(parent)
# Re-broadcast our input for the benefit of listening clients, and
# start computing output
if not silent:
self.execution_count += 1
self._publish_execute_input(code, parent, self.execution_count)
reply_content = self.do_execute(code, silent, store_history,
user_expressions, allow_stdin)
# Flush output before sending the reply.
sys.stdout.flush()
sys.stderr.flush()
# FIXME: on rare occasions, the flush doesn't seem to make it to the
# clients... This seems to mitigate the problem, but we definitely need
# to better understand what's going on.
if self._execute_sleep:
time.sleep(self._execute_sleep)
# Send the reply.
reply_content = json_clean(reply_content)
metadata = self.finish_metadata(parent, metadata, reply_content)
reply_msg = self.session.send(stream,
u'execute_reply',
reply_content,
parent,
metadata=metadata,
ident=ident)
self.log.debug("%s", reply_msg)
if not silent and reply_msg['content'][
'status'] == u'error' and stop_on_error:
self._abort_queues()
def do_execute(self,
code,
silent,
store_history=True,
user_expressions=None,
allow_stdin=False):
"""Execute user code. Must be overridden by subclasses.
"""
raise NotImplementedError
def complete_request(self, stream, ident, parent):
content = parent['content']
code = content['code']
cursor_pos = content['cursor_pos']
matches = self.do_complete(code, cursor_pos)
matches = json_clean(matches)
completion_msg = self.session.send(stream, 'complete_reply', matches,
parent, ident)
self.log.debug("%s", completion_msg)
def do_complete(self, code, cursor_pos):
"""Override in subclasses to find completions.
"""
return {
'matches': [],
'cursor_end': cursor_pos,
'cursor_start': cursor_pos,
'metadata': {},
'status': 'ok'
}
def inspect_request(self, stream, ident, parent):
content = parent['content']
reply_content = self.do_inspect(content['code'], content['cursor_pos'],
content.get('detail_level', 0))
# Before we send this object over, we scrub it for JSON usage
reply_content = json_clean(reply_content)
msg = self.session.send(stream, 'inspect_reply', reply_content, parent,
ident)
self.log.debug("%s", msg)
def do_inspect(self, code, cursor_pos, detail_level=0):
"""Override in subclasses to allow introspection.
"""
return {'status': 'ok', 'data': {}, 'metadata': {}, 'found': False}
def history_request(self, stream, ident, parent):
content = parent['content']
reply_content = self.do_history(**content)
reply_content = json_clean(reply_content)
msg = self.session.send(stream, 'history_reply', reply_content, parent,
ident)
self.log.debug("%s", msg)
def do_history(self,
hist_access_type,
output,
raw,
session=None,
start=None,
stop=None,
n=None,
pattern=None,
unique=False):
"""Override in subclasses to access history.
"""
return {'history': []}
def connect_request(self, stream, ident, parent):
if self._recorded_ports is not None:
content = self._recorded_ports.copy()
else:
content = {}
msg = self.session.send(stream, 'connect_reply', content, parent,
ident)
self.log.debug("%s", msg)
@property
def kernel_info(self):
return {
'protocol_version': kernel_protocol_version,
'implementation': self.implementation,
'implementation_version': self.implementation_version,
'language_info': self.language_info,
'banner': self.banner,
'help_links': self.help_links,
}
def kernel_info_request(self, stream, ident, parent):
msg = self.session.send(stream, 'kernel_info_reply', self.kernel_info,
parent, ident)
self.log.debug("%s", msg)
def comm_info_request(self, stream, ident, parent):
content = parent['content']
target_name = content.get('target_name', None)
# Should this be moved to ipkernel?
if hasattr(self, 'comm_manager'):
comms = {
k: dict(target_name=v.target_name)
for (k, v) in self.comm_manager.comms.items()
if v.target_name == target_name or target_name is None
}
else:
comms = {}
reply_content = dict(comms=comms)
msg = self.session.send(stream, 'comm_info_reply', reply_content,
parent, ident)
self.log.debug("%s", msg)
def shutdown_request(self, stream, ident, parent):
content = self.do_shutdown(parent['content']['restart'])
self.session.send(stream,
u'shutdown_reply',
content,
parent,
ident=ident)
# same content, but different msg_id for broadcasting on IOPub
self._shutdown_message = self.session.msg(u'shutdown_reply', content,
parent)
self._at_shutdown()
# call sys.exit after a short delay
loop = ioloop.IOLoop.instance()
loop.add_timeout(time.time() + 0.1, loop.stop)
def do_shutdown(self, restart):
"""Override in subclasses to do things when the frontend shuts down the
kernel.
"""
return {'status': 'ok', 'restart': restart}
def is_complete_request(self, stream, ident, parent):
content = parent['content']
code = content['code']
reply_content = self.do_is_complete(code)
reply_content = json_clean(reply_content)
reply_msg = self.session.send(stream, 'is_complete_reply',
reply_content, parent, ident)
self.log.debug("%s", reply_msg)
def do_is_complete(self, code):
"""Override in subclasses to find completions.
"""
return {
'status': 'unknown',
}
#---------------------------------------------------------------------------
# Engine methods (DEPRECATED)
#---------------------------------------------------------------------------
def apply_request(self, stream, ident, parent):
self.log.warn(
"""apply_request is deprecated in kernel_base, moving to ipyparallel."""
)
try:
content = parent[u'content']
bufs = parent[u'buffers']
msg_id = parent['header']['msg_id']
except:
self.log.error("Got bad msg: %s", parent, exc_info=True)
return
md = self.init_metadata(parent)
reply_content, result_buf = self.do_apply(content, bufs, msg_id, md)
# flush i/o
sys.stdout.flush()
sys.stderr.flush()
md = self.finish_metadata(parent, md, reply_content)
self.session.send(stream,
u'apply_reply',
reply_content,
parent=parent,
ident=ident,
buffers=result_buf,
metadata=md)
def do_apply(self, content, bufs, msg_id, reply_metadata):
"""DEPRECATED"""
raise NotImplementedError
#---------------------------------------------------------------------------
# Control messages (DEPRECATED)
#---------------------------------------------------------------------------
def abort_request(self, stream, ident, parent):
"""abort a specific msg by id"""
self.log.warn(
"abort_request is deprecated in kernel_base. It os only part of IPython parallel"
)
msg_ids = parent['content'].get('msg_ids', None)
if isinstance(msg_ids, string_types):
msg_ids = [msg_ids]
if not msg_ids:
self._abort_queues()
for mid in msg_ids:
self.aborted.add(str(mid))
content = dict(status='ok')
reply_msg = self.session.send(stream,
'abort_reply',
content=content,
parent=parent,
ident=ident)
self.log.debug("%s", reply_msg)
def clear_request(self, stream, idents, parent):
"""Clear our namespace."""
self.log.warn(
"clear_request is deprecated in kernel_base. It os only part of IPython parallel"
)
content = self.do_clear()
self.session.send(stream,
'clear_reply',
ident=idents,
parent=parent,
content=content)
def do_clear(self):
"""DEPRECATED"""
raise NotImplementedError
#---------------------------------------------------------------------------
# Protected interface
#---------------------------------------------------------------------------
def _topic(self, topic):
"""prefixed topic for IOPub messages"""
base = "kernel.%s" % self.ident
return py3compat.cast_bytes("%s.%s" % (base, topic))
def _abort_queues(self):
for stream in self.shell_streams:
if stream:
self._abort_queue(stream)
def _abort_queue(self, stream):
poller = zmq.Poller()
poller.register(stream.socket, zmq.POLLIN)
while True:
idents, msg = self.session.recv(stream, zmq.NOBLOCK, content=True)
if msg is None:
return
self.log.info("Aborting:")
self.log.info("%s", msg)
msg_type = msg['header']['msg_type']
reply_type = msg_type.split('_')[0] + '_reply'
status = {'status': 'aborted'}
md = {'engine': self.ident}
md.update(status)
reply_msg = self.session.send(stream,
reply_type,
metadata=md,
content=status,
parent=msg,
ident=idents)
self.log.debug("%s", reply_msg)
# We need to wait a bit for requests to come in. This can probably
# be set shorter for true asynchronous clients.
poller.poll(50)
def _no_raw_input(self):
"""Raise StdinNotImplentedError if active frontend doesn't support
stdin."""
raise StdinNotImplementedError("raw_input was called, but this "
"frontend does not support stdin.")
def getpass(self, prompt=''):
"""Forward getpass to frontends
Raises
------
StdinNotImplentedError if active frontend doesn't support stdin.
"""
if not self._allow_stdin:
raise StdinNotImplementedError(
"getpass was called, but this frontend does not support input requests."
)
return self._input_request(
prompt,
self._parent_ident,
self._parent_header,
password=True,
)
def raw_input(self, prompt=''):
"""Forward raw_input to frontends
Raises
------
StdinNotImplentedError if active frontend doesn't support stdin.
"""
if not self._allow_stdin:
raise StdinNotImplementedError(
"raw_input was called, but this frontend does not support input requests."
)
return self._input_request(
prompt,
self._parent_ident,
self._parent_header,
password=False,
)
def _input_request(self, prompt, ident, parent, password=False):
# Flush output before making the request.
sys.stderr.flush()
sys.stdout.flush()
# flush the stdin socket, to purge stale replies
while True:
try:
self.stdin_socket.recv_multipart(zmq.NOBLOCK)
except zmq.ZMQError as e:
if e.errno == zmq.EAGAIN:
break
else:
raise
# Send the input request.
content = json_clean(dict(prompt=prompt, password=password))
self.session.send(self.stdin_socket,
u'input_request',
content,
parent,
ident=ident)
# Await a response.
while True:
try:
ident, reply = self.session.recv(self.stdin_socket, 0)
except Exception:
self.log.warn("Invalid Message:", exc_info=True)
except KeyboardInterrupt:
# re-raise KeyboardInterrupt, to truncate traceback
raise KeyboardInterrupt
else:
break
try:
value = py3compat.unicode_to_str(reply['content']['value'])
except:
self.log.error("Bad input_reply: %s", parent)
value = ''
if value == '\x04':
# EOF
raise EOFError
return value
def _at_shutdown(self):
"""Actions taken at shutdown by the kernel, called by python's atexit.
"""
# io.rprint("Kernel at_shutdown") # dbg
if self._shutdown_message is not None:
self.session.send(self.iopub_socket,
self._shutdown_message,
ident=self._topic('shutdown'))
self.log.debug("%s", self._shutdown_message)
[s.flush(zmq.POLLOUT) for s in self.shell_streams]
class BM25Para(Configurable):
"""
bm25's para
"""
k1 = Float(BM25_K1).tag(config=True)
b = Float(BM25_B).tag(config=True)
class KubeClusterManager(ClusterManager):
"""A cluster manager for deploying Dask on a Kubernetes cluster."""
namespace = Unicode(
"default",
help="""
Kubernetes namespace to launch pods in.
If running inside a kubernetes cluster with service accounts enabled,
defaults to the current namespace. If not, defaults to `default`
""",
config=True,
)
@default("namespace")
def _default_namespace(self):
"""
Set namespace default to current namespace if running in a k8s cluster
If not in a k8s cluster with service accounts enabled, default to
`default`
"""
ns_path = "/var/run/secrets/kubernetes.io/serviceaccount/namespace"
if os.path.exists(ns_path):
with open(ns_path) as f:
return f.read().strip()
return "default"
image = Unicode(
"jcrist/dask-gateway:latest",
help="Docker image to use for running user's containers.",
config=True,
)
image_pull_policy = Unicode(
"IfNotPresent",
help="The image pull policy of the docker image specified in ``image``",
config=True,
)
image_pull_secrets = List(
trait=Unicode(),
help="""
A list of secrets to use for pulling images from a private repository.
See `the Kubernetes documentation
<https://kubernetes.io/docs/concepts/containers/images/#specifying-imagepullsecrets-on-a-pod>`__
for more information.
""",
config=True,
)
working_dir = Unicode(
help="""
The working directory where the command will be run inside the
container. Default is the working directory defined in the Dockerfile.
""",
config=True,
)
common_labels = Dict(
{
"app.kubernetes.io/name": "dask-gateway",
"app.kubernetes.io/version": VERSION.replace("+", "_"),
},
help="Kubernetes labels to apply to all objects created by the gateway",
config=True,
)
common_annotations = Dict(
help=
"Kubernetes annotations to apply to all objects created by the gateway",
config=True,
)
# Kubernetes is a bit different in types/granularity of resource requests.
# We redefine these common fields here to support that.
worker_cores = Float(
1,
min=0,
help="""
Number of cpu-cores available for a dask worker.
""",
config=True,
)
worker_cores_limit = Float(
min=0,
help="""
Maximum number of cpu-cores available for a dask worker.
Defaults to ``worker_cores``.
""",
config=True,
)
@default("worker_cores_limit")
def _default_worker_cores_limit(self):
return self.worker_cores
worker_memory_limit = MemoryLimit(
help="""
Maximum number of bytes available for a dask worker. Allows the
following suffixes:
- K -> Kibibytes
- M -> Mebibytes
- G -> Gibibytes
- T -> Tebibytes
Defaults to ``worker_memory``.
""",
config=True,
)
@default("worker_memory_limit")
def _default_worker_memory_limit(self):
return self.worker_memory
scheduler_cores = Float(
1,
min=0,
help="""
Number of cpu-cores available for a dask scheduler.
""",
config=True,
)
scheduler_cores_limit = Float(
min=0,
help="""
Maximum number of cpu-cores available for a dask scheduler.
Defaults to ``scheduler_cores``.
""",
config=True,
)
@default("scheduler_cores_limit")
def _default_scheduler_cores_limit(self):
return self.scheduler_cores
scheduler_memory_limit = MemoryLimit(
help="""
Maximum number of bytes available for a dask scheduler. Allows the
following suffixes:
- K -> Kibibytes
- M -> Mebibytes
- G -> Gibibytes
- T -> Tebibytes
Defaults to ``scheduler_memory``.
""",
config=True,
)
@default("scheduler_memory_limit")
def _default_scheduler_memory_limit(self):
return self.scheduler_memory
# Internal fields
kube_client = Instance(kubernetes.client.CoreV1Api)
@default("kube_client")
def _default_kube_client(self):
configure_kubernetes_clients()
return kubernetes.client.CoreV1Api()
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
# Starts the pod reflector for the first instance only
self.pod_reflector = PodReflector.instance(
parent=self.parent or self,
kube_client=self.kube_client,
namespace=self.namespace,
label_selector=self.pod_label_selector,
)
def get_tls_paths(self):
"""Get the absolute paths to the tls cert and key files."""
return "/etc/dask-credentials/dask.crt", "/etc/dask-credentials/dask.pem"
@property
def worker_command(self):
"""The full command (with args) to launch a dask worker"""
return [
self.worker_cmd,
"--nthreads",
str(int(self.worker_cores_limit)),
"--memory-limit",
str(self.worker_memory_limit),
]
@property
def scheduler_command(self):
"""The full command (with args) to launch a dask scheduler"""
return [self.scheduler_cmd]
@property
def pod_label_selector(self):
"""A label selector for all pods started by dask-gateway"""
return ",".join("%s=%s" % (k, v)
for k, v in self.common_labels.items())
def get_labels_for(self, component, worker_name=None):
labels = self.common_labels.copy()
labels.update({
"app.kubernetes.io/component": component,
"cluster-name": self.cluster_name,
})
if worker_name:
labels["worker-name"] = worker_name
return labels
def make_secret_spec(self):
name = "dask-gateway-tls-%s" % self.cluster_name
labels = self.get_labels_for("dask-gateway-tls")
annotations = self.common_annotations
secret = V1Secret(
kind="Secret",
api_version="v1",
string_data={
"dask.crt": self.tls_cert.decode(),
"dask.pem": self.tls_key.decode(),
},
metadata=V1ObjectMeta(name=name,
labels=labels,
annotations=annotations),
)
return secret
def make_pod_spec(self, tls_secret, worker_name=None):
annotations = self.common_annotations
env = self.get_env()
if worker_name is not None:
# Worker
name = "dask-gateway-worker-%s" % worker_name
container_name = "dask-gateway-worker"
labels = self.get_labels_for("dask-gateway-worker",
worker_name=worker_name)
mem_req = self.worker_memory
mem_lim = self.worker_memory_limit
cpu_req = self.worker_cores
cpu_lim = self.worker_cores_limit
env["DASK_GATEWAY_WORKER_NAME"] = worker_name
cmd = self.worker_command
else:
# Scheduler
name = "dask-gateway-scheduler-%s" % self.cluster_name
container_name = "dask-gateway-scheduler"
labels = self.get_labels_for("dask-gateway-scheduler")
mem_req = self.scheduler_memory
mem_lim = self.scheduler_memory_limit
cpu_req = self.scheduler_cores
cpu_lim = self.scheduler_cores_limit
cmd = self.scheduler_command
volume = V1Volume(name="dask-credentials",
secret=V1SecretVolumeSource(secret_name=tls_secret))
container = V1Container(
name=container_name,
image=self.image,
args=cmd,
env=[V1EnvVar(k, v) for k, v in env.items()],
working_dir=self.working_dir or None,
image_pull_policy=self.image_pull_policy,
resources=V1ResourceRequirements(
requests={
"cpu": cpu_req,
"memory": mem_req
},
limits={
"cpu": cpu_lim,
"memory": mem_lim
},
),
volume_mounts=[
V1VolumeMount(
name=volume.name,
mount_path="/etc/dask-credentials/",
read_only=True,
)
],
)
pod = V1Pod(
kind="Pod",
api_version="v1",
metadata=V1ObjectMeta(name=name,
labels=labels,
annotations=annotations),
spec=V1PodSpec(containers=[container],
volumes=[volume],
restart_policy="Never"),
)
if self.image_pull_secrets:
pod.spec.image_pull_secrets = [
V1LocalObjectReference(name=s) for s in self.image_pull_secrets
]
# Ensure we don't accidentally give access to the kubernetes API
pod.spec.automount_service_account_token = False
return pod
async def start_cluster(self):
tls_secret = self.make_secret_spec()
secret_name = tls_secret.metadata.name
self.log.debug("Creating secret %s", secret_name)
loop = get_running_loop()
await loop.run_in_executor(None,
self.kube_client.create_namespaced_secret,
self.namespace, tls_secret)
yield {"secret_name": secret_name}
pod = self.make_pod_spec(secret_name)
self.log.debug("Starting pod %s", pod.metadata.name)
await loop.run_in_executor(None,
self.kube_client.create_namespaced_pod,
self.namespace, pod)
yield {"secret_name": secret_name, "pod_name": pod.metadata.name}
async def pod_status(self, pod_name, container_name):
if pod_name is None:
return
# Ensure initial data already loaded
if not self.pod_reflector.first_load_future.done():
await self.pod_reflector.first_load_future
pod = self.pod_reflector.pods.get(pod_name)
if pod is not None:
if pod.status.phase == "Pending":
return True
if pod.status.container_statuses is None:
return False
for c in pod.status.container_statuses:
if c.name == container_name:
if c.state.terminated:
msg = ("Container stopped with exit code %d" %
c.state.terminated.exit_code)
return False, msg
return True
# pod doesn't exist or has been deleted
return False, ("Pod %s already deleted" % pod_name)
async def cluster_status(self, cluster_state):
return await self.pod_status(cluster_state.get("pod_name"),
"dask-gateway-scheduler")
async def worker_status(self, worker_name, worker_state, cluster_state):
return await self.pod_status(worker_state.get("pod_name"),
"dask-gateway-worker")
async def stop_cluster(self, cluster_state):
loop = get_running_loop()
pod_name = cluster_state.get("pod_name")
if pod_name is not None:
await loop.run_in_executor(None,
self.kube_client.delete_namespaced_pod,
pod_name, self.namespace)
secret_name = cluster_state.get("secret_name")
if secret_name is not None:
await loop.run_in_executor(
None,
self.kube_client.delete_namespaced_secret,
secret_name,
self.namespace,
)
async def start_worker(self, worker_name, cluster_state):
secret_name = cluster_state["secret_name"]
pod = self.make_pod_spec(secret_name, worker_name)
self.log.debug("Starting pod %s", pod.metadata.name)
loop = get_running_loop()
await loop.run_in_executor(None,
self.kube_client.create_namespaced_pod,
self.namespace, pod)
yield {"pod_name": pod.metadata.name}
async def stop_worker(self, worker_name, worker_state, cluster_state):
pod_name = worker_state.get("pod_name")
if pod_name is not None:
loop = get_running_loop()
await loop.run_in_executor(None,
self.kube_client.delete_namespaced_pod,
pod_name, self.namespace)
class LmPara(Configurable):
dir_mu = Int(LM_DIR_MU).tag(config=True)
min_tf = Float(LM_MIN_TF).tag(config=True)
jm_lambda = Float(LM_JM_LAMBDA).tag(config=True)
class JupytextConfiguration(Configurable):
"""Jupytext Configuration's options"""
default_jupytext_formats = Unicode(
u"",
help="Save notebooks to these file extensions. "
"Can be any of ipynb,Rmd,md,jl,py,R,nb.jl,nb.py,nb.R "
"comma separated. If you want another format than the "
"default one, append the format name to the extension, "
"e.g. ipynb,py:percent to save the notebook to "
"hydrogen/spyder/vscode compatible scripts",
config=True,
)
preferred_jupytext_formats_save = Unicode(
u"",
help="Preferred format when saving notebooks as text, per extension. "
'Use "jl:percent,py:percent,R:percent" if you want to save '
"Julia, Python and R scripts in the double percent format and "
'only write "jupytext_formats": "py" in the notebook metadata.',
config=True,
)
preferred_jupytext_formats_read = Unicode(
u"",
help="Preferred format when reading notebooks from text, per "
'extension. Use "py:sphinx" if you want to read all python '
"scripts as Sphinx gallery scripts.",
config=True,
)
default_notebook_metadata_filter = Unicode(
u"",
help="Cell metadata that should be save in the text representations. "
"Examples: 'all', '-all', 'widgets,nteract', 'kernelspec,jupytext-all'",
config=True,
)
default_cell_metadata_filter = Unicode(
u"",
help=
"Notebook metadata that should be saved in the text representations. "
"Examples: 'all', 'hide_input,hide_output'",
config=True,
)
comment_magics = Enum(
values=[True, False],
allow_none=True,
help=
"Should Jupyter magic commands be commented out in the text representation?",
config=True,
)
split_at_heading = Bool(
False,
help=
"Split markdown cells on headings (Markdown and R Markdown formats only)",
config=True,
)
sphinx_convert_rst2md = Bool(
False,
help=
"When opening a Sphinx Gallery script, convert the reStructuredText to markdown",
config=True,
)
outdated_text_notebook_margin = Float(
1.0,
help="Refuse to overwrite inputs of a ipynb notebooks with those of a "
"text notebook when the text notebook plus margin is older than "
"the ipynb notebook (NB: This option is ignored by Jupytext CLI)",
config=True,
)
default_cell_markers = Unicode(
u"",
help=
'Start and end cell markers for the light format, comma separated. Use "{{{,}}}" to mark cells'
'as foldable regions in Vim, and "region,endregion" to mark cells as Vscode/PyCharm regions',
config=True,
)
notebook_extensions = Unicode(
u",".join(NOTEBOOK_EXTENSIONS),
help="A comma separated list of notebook extensions",
config=True,
)
def set_default_format_options(self, format_options, read=False):
"""Set default format option"""
if self.default_notebook_metadata_filter:
format_options.setdefault("notebook_metadata_filter",
self.default_notebook_metadata_filter)
if self.default_cell_metadata_filter:
format_options.setdefault("cell_metadata_filter",
self.default_cell_metadata_filter)
if self.comment_magics is not None:
format_options.setdefault("comment_magics", self.comment_magics)
if self.split_at_heading:
format_options.setdefault("split_at_heading",
self.split_at_heading)
if not read and self.default_cell_markers:
format_options.setdefault("cell_markers",
self.default_cell_markers)
if read and self.sphinx_convert_rst2md:
format_options.setdefault("rst2md", self.sphinx_convert_rst2md)
def default_formats(self, path):
"""Return the default formats, if they apply to the current path #157"""
formats = long_form_multiple_formats(self.default_jupytext_formats)
for fmt in formats:
try:
base_path(path, fmt)
return self.default_jupytext_formats
except InconsistentPath:
continue
return None
class Place(LockedXmlTraits, UrlContainer):
"""A place that can be visited."""
data_set_type = UseEnum(
DataSetType,
default_value=DataSetType.EARTH).tag(xml=XmlSer.attr("DataSetType"))
name = Unicode("").tag(xml=XmlSer.attr("Name"))
ra_hr = Float(0.0).tag(xml=XmlSer.attr("RA"), xml_if_sky_type_is=True)
dec_deg = Float(0.0).tag(xml=XmlSer.attr("Dec"), xml_if_sky_type_is=True)
latitude = Float(0.0).tag(xml=XmlSer.attr("Lat"), xml_if_sky_type_is=False)
longitude = Float(0.0).tag(xml=XmlSer.attr("Lng"),
xml_if_sky_type_is=False)
constellation = UseEnum(Constellation,
default_value=Constellation.UNSPECIFIED).tag(
xml=XmlSer.attr("Constellation"))
classification = UseEnum(Classification,
default_value=Classification.UNSPECIFIED).tag(
xml=XmlSer.attr("Classification"))
magnitude = Float(0.0).tag(xml=XmlSer.attr("Magnitude"))
distance = Float(0.0).tag(xml=XmlSer.attr("Distance"), xml_omit_zero=True)
angular_size = Float(0.0).tag(xml=XmlSer.attr("AngularSize"))
zoom_level = Float(0.0).tag(xml=XmlSer.attr("ZoomLevel"))
rotation_deg = Float(0.0).tag(xml=XmlSer.attr("Rotation"))
angle = Float(0.0).tag(xml=XmlSer.attr("Angle"))
opacity = Float(100.0).tag(xml=XmlSer.attr("Opacity"))
dome_alt = Float(0.0).tag(xml=XmlSer.attr("DomeAlt"), xml_omit_zero=True)
dome_az = Float(0.0).tag(xml=XmlSer.attr("DomeAz"), xml_omit_zero=True)
background_image_set = Instance(
ImageSet,
allow_none=True).tag(xml=XmlSer.wrapped_inner("BackgroundImageSet"))
foreground_image_set = Instance(
ImageSet,
allow_none=True).tag(xml=XmlSer.wrapped_inner("ForegroundImageSet"))
image_set = Instance(ImageSet,
allow_none=True).tag(xml=XmlSer.inner("ImageSet"))
thumbnail = Unicode("").tag(xml=XmlSer.attr("Thumbnail"))
description = Unicode("").tag(xml=XmlSer.text_elem("Description"))
"""
A description of the place, using HTML markup.
This field is not actually used in the stock WWT clients, but it is wired up
and loaded from the XML.
"""
annotation = Unicode("").tag(xml=XmlSer.attr("Annotation"))
"""
Annotation metadata for the place.
This field is only used in the web engine and web client app. The web client
app expects this field to contain a comma-separated list of key-value pairs,
where each pair is delimited with colons:
.. code-block::
key1:val1,key2:val2,key3:val3
The webclient includes some unfinished support for this field to be used to
create circular annotations with YouTube video links. If your WTML file will
not be viewed in the webclient, you can use this field to convey arbitrary
textual data to the WWT Web Engine JavaScript/TypeScript layer.
"""
msr_community_id = Int(0).tag(xml=XmlSer.attr("MSRCommunityId"),
xml_omit_zero=True)
"""The ID number of the WWT Community that this content came from."""
msr_component_id = Int(0).tag(xml=XmlSer.attr("MSRComponentId"),
xml_omit_zero=True)
"""The ID number of this content item on the WWT Communities system."""
permission = Int(0).tag(xml=XmlSer.attr("Permission"), xml_omit_zero=True)
"TBD."
xmeta = Instance(
Namespace,
args=(),
help=
"XML metadata - a namespace object for attaching arbitrary text to serialize",
).tag(xml=XmlSer.ns_to_attr("X"))
def _tag_name(self):
return "Place"
def mutate_urls(self, mutator):
if self.thumbnail:
self.thumbnail = mutator(self.thumbnail)
if self.background_image_set:
self.background_image_set.mutate_urls(mutator)
if self.foreground_image_set:
self.foreground_image_set.mutate_urls(mutator)
if self.image_set:
self.image_set.mutate_urls(mutator)
def as_imageset(self):
"""Return an ImageSet for this place if one is defined.
Returns
-------
Either :class:`wwt_data_formats.imageset.ImageSet` or None.
Notes
-----
If the :attr:`foreground_image_set` of this :class:`Place` is not
None, it is returned. Otherwise, if its :attr:`image_set` is not
None, that is returned. Otherwise, None is returned.
"""
if self.foreground_image_set is not None:
return self.foreground_image_set
return self.image_set
class MolViz3D(MessageWidget):
"""
Draws molecules in 3D with 3DMol.js
"""
height = Unicode(sync=True)
width = Unicode(sync=True)
_view_name = Unicode('MolWidget3DView').tag(sync=True)
_model_name = Unicode('MolWidget3DModel').tag(sync=True)
_view_module = Unicode('nbmolviz-js').tag(sync=True)
_model_module = Unicode('nbmolviz-js').tag(sync=True)
atom_labels_shown = Bool(False).tag(sync=True)
background_color = Unicode('#73757C').tag(sync=True)
background_opacity = Float(1.0).tag(sync=True)
model_data = Dict({}).tag(sync=True)
orbital = Dict({}).tag(sync=True)
selected_atom_indices = Set(set()).tag(sync=True)
selection_type = Unicode('Atom').tag(sync=True)
shapes = List([]).tag(sync=True)
styles = Dict({}).tag(sync=True)
SHAPE_NAMES = {
'SPHERE': 'Sphere',
'ARROW': 'Arrow',
'CYLINDER': 'Cylinder',
}
STYLE_NAMES = {
'vdw': 'sphere',
'licorice': 'stick',
'line': 'line',
'ribbon': 'cartoon',
None: None
}
STYLE_SYNONYMS = {
'vdw': 'vdw',
'sphere': 'vdw',
'cpk': 'vdw',
'licorice': 'licorice',
'stick': 'licorice',
'tube': 'licorice',
'ball_and_stick': 'ball_and_stick',
'line': 'line',
'cartoon': 'ribbon',
'ribbon': 'ribbon',
None: None,
'hide': None,
'invisible': None,
'remove': None
}
def __init__(self, mol=None, width=500, height=350, **kwargs):
kwargs.update(width=width, height=height)
super(MolViz3D, self).__init__(**kwargs)
self.height = in_pixels(height)
self.width = in_pixels(width)
# current state
self.num_frames = 1
self.current_frame = 0
self.current_orbital = None
# add the new molecule if necessary
if mol is not None: self.add_molecule(mol)
self.current_orbital = None
self.orbital_spec = {}
self._cached_cubefiles = {}
self._clicks_enabled = False
# Utilities
def convert_style_name(self, name):
canonical_name = self.STYLE_SYNONYMS[name]
if canonical_name in self.STYLE_NAMES:
return self.STYLE_NAMES[canonical_name]
else:
return canonical_name
# Standard view actions
def add_molecule(self, mol):
self.mol = mol
self.model_data = convert_to_json(self.mol)
def set_background_color(self, color, opacity=1.0):
color = translate_color(color)
self.background_color = color
self.background_opacity = opacity
def set_clipping(self, near, far):
self.viewer('setSlab', [float(near), float(far)])
def set_color(self, color, atoms):
self.styles = MolViz3D.get_styles_for_color(color, atoms, self.styles)
#some convenience synonyms
def sphere(self, **kwargs):
return self.add_style('vdw', **kwargs)
vdw = cpk = sphere
def ball_and_stick(self, **kwargs):
return self.add_style('ball_and_stick', **kwargs)
def licorice(self, **kwargs):
return self.add_style('licorice', **kwargs)
stick = tube = licorice
def line(self, **kwargs):
return self.add_style('line', **kwargs)
def ribbon(self, **kwargs):
return self.add_style('cartoon', **kwargs)
cartoon = ribbon
def hide(self, atoms=None):
return self.add_style(None, atoms=atoms)
invisible = hide
# Returns new styles after updating the given atoms with the given color
@staticmethod
def get_styles_for_color(color, atoms, styles):
styles = dict(styles)
if not atoms:
return styles
for atom in atoms:
if str(atom.index) in styles:
styles[str(atom.index)] = dict(styles[str(atom.index)])
else:
styles[str(atom.index)] = {}
styles[str(atom.index)]['color'] = color
return styles
def set_colors(self, colormap):
"""
Args:
colormap(Mapping[str,List[Atoms]]): mapping of colors to atoms
"""
styles = dict(self.styles)
for color, atoms in colormap.iteritems():
styles = MolViz3D.get_styles_for_color(color, atoms, styles)
self.styles = styles
def unset_color(self, atoms=None):
if atoms is None:
atom_json = {}
else:
atom_json = self._atoms_to_json(atoms)
self.viewer('unsetAtomColor',
[atom_json]) # TODO: remove calls to self.viewer
def set_style(self, style, atoms=None, **options):
self._change_style(style, atoms, True, options)
def add_style(self, style, atoms=None, **options):
self._change_style(style, atoms, False, options)
def _change_style(self, style_string, atoms, replace, options):
style = self.convert_style_name(style_string)
# No atoms passed means all atoms
if atoms is None:
atoms = self.mol.atoms
atoms = list(atoms)
if replace:
styles = dict()
else:
styles = dict(self.styles)
for i, atom in enumerate(self.mol.atoms):
for j in range(0, len(atoms)):
if (atoms[j] is atom):
newStyle = styles[str(
atom.index)].copy() if i in styles else {}
newStyle['visualization_type'] = style
if 'color' in options:
newStyle['color'] = options['color']
styles[str(atom.index)] = newStyle
atoms.remove(atoms[j])
break
self.styles = styles
def append_frame(self, positions=None):
if positions is None:
positions = self.get_positions()
positions = self._convert_units(positions)
try:
positions = positions.tolist()
except AttributeError:
pass
self.num_frames += 1
self.viewer('addFrameFromList', args=[positions])
self.show_frame(self.num_frames - 1)
def set_positions(self, positions=None):
from moldesign import units as u
if positions is not None:
for i, atom in enumerate(self.mol.atoms):
atom.position[0] = positions[i][0] * u.angstrom
atom.position[1] = positions[i][1] * u.angstrom
atom.position[2] = positions[i][2] * u.angstrom
self.model_data = convert_to_json(self.mol)
def show_frame(self, framenum):
self.viewer('setFrame', [framenum])
self.current_frame = framenum
if self.current_orbital is not None:
self.draw_orbital(self.current_orbital, **self.orbital_spec)
#Shapes
@staticmethod
def _list_to_jsvec(vec):
assert len(vec) == 3
return dict(x=vec[0], y=vec[1], z=vec[2])
# TODO this contains unused parameters and code due to removed functionality, is it needed?
def draw_sphere(self,
position,
radius=2.0,
color='red',
opacity=1.0,
clickable=False):
position = self._convert_units(position)
radius = self._convert_units(radius)
center = dict(x=position[0], y=position[1], z=position[2])
color = translate_color(color)
shape = {
'type': self.SHAPE_NAMES['SPHERE'],
'center': center,
'radius': radius,
'color': color,
'opacity': opacity,
}
shapes = list(self.shapes)
shapes.append(shape)
self.shapes = shapes
return shape
def draw_circle(self,
center,
normal,
radius,
color='red',
opacity=0.8,
clickable=False,
batch=False):
# TODO: this doesn't work! appears to be a bug in 3dmol.js
# return self._draw3dmol_cylinder(color, center,
# np.array(center) + np.array(normal) * 0.01,
# True, False,
# opacity,
# radius)
return self._draw3dmol_cylinder(
color, center,
np.array(center) + np.array(normal) * 0.01, True, True, opacity,
radius, clickable, batch)
def draw_cylinder(self,
start,
end,
radius,
color='red',
opacity=1.0,
clickable=False,
batch=False):
return self._draw3dmol_cylinder(color, start, end, True, True, opacity,
radius, clickable, batch)
def draw_tube(self,
start,
end,
radius,
color='red',
opacity=1.0,
clickable=False,
batch=False):
return self._draw3dmol_cylinder(color, start, end, False, False,
opacity, radius, clickable, batch)
# TODO this contains unused parameters and code due to removed functionality, is it needed?
def _draw3dmol_cylinder(self, color, start, end, draw_start_face,
draw_end_face, opacity, radius, clickable, batch):
color = translate_color(color)
facestart = self._convert_units(start)
faceend = self._convert_units(end)
radius = self._convert_units(radius)
spec = dict(start=self._list_to_jsvec(facestart),
end=self._list_to_jsvec(faceend),
radius=radius,
color=color,
alpha=opacity,
fromCap=draw_start_face,
toCap=draw_end_face)
shape = {
'type': self.SHAPE_NAMES['CYLINDER'],
'start': self._list_to_jsvec(facestart),
'end': self._list_to_jsvec(faceend),
'radius': radius,
'color': color,
'opacity': opacity,
}
shapes = list(self.shapes)
shapes.append(shape)
self.shapes = shapes
return shape
# TODO this contains unused parameters and code due to removed functionality, is it needed?
def draw_arrow(self,
start,
end=None,
vector=None,
radius=0.15,
color='red',
opacity=1.0,
clickable=False):
if (end is None) == (vector is None):
raise ValueError(
"Either 'end' or 'vector' should be passed, but not both.")
if end is None: end = np.array(start) + np.array(vector)
facestart = self._convert_units(start)
faceend = self._convert_units(end)
color = translate_color(color)
spec = dict(start=self._list_to_jsvec(facestart),
end=self._list_to_jsvec(faceend),
radius=radius,
color=color,
alpha=opacity)
shape = {
'type': self.SHAPE_NAMES['ARROW'],
'start': self._list_to_jsvec(facestart),
'end': self._list_to_jsvec(faceend),
'color': color,
'opacity': opacity,
}
shapes = list(self.shapes)
shapes.append(shape)
self.shapes = shapes
return shape
def remove_all_shapes(self):
self.shapes = list()
def remove(self, obj, batch=False):
if obj in self.shapes:
shapes = list(self.shapes)
shapes.remove(obj)
self.shapes = shapes
elif obj.type == 'label':
self.atom_labels_shown = False
else:
raise ValueError('Unknown object type %s' % obj['type'])
# Labels
def draw_label(self,
position,
text,
background='black',
border='black',
color='white',
fontsize=14,
opacity=1.0):
js_label = JSObject('label')
position = self._convert_units(position)
color = translate_color(color)
background = translate_color(background)
spec = dict(position=self._list_to_jsvec(position),
fontColor=color,
backgroundColor=background,
borderColor=border,
fontSize=fontsize,
backgroundOpacity=opacity)
self.viewer('renderPyLabel', [text, spec, js_label.id])
return js_label
def remove_all_labels(self):
self.viewer('removeAllLabels', [])
def select_residues(self, residues):
selected_atom_indices = set()
for residue in residues:
for atom in residues.atoms:
selected_atom_indices.add(atom.index)
self.selected_atom_indices = selected_atom_indices
def toggle_residues(self, residues):
selected_atom_indices = set(self.selected_atom_indices)
for residue in residues:
for atom in residue.atoms:
if atom.index in selected_atom_indices:
selected_atom_indices.remove(atom.index)
else:
selected_atom_indices.add(atom.index)
self.selected_atom_indices = selected_atom_indices
def get_selected_bonds(self, *args, **kwargs):
atomIndices = kwargs.get('atomIndices', self.selected_atom_indices)
bonds = set()
for bond in self.mol.bonds:
if bond.a1.index in atomIndices and bond.a2.index in atomIndices:
bonds.add(bond)
return bonds
def get_cubefile(self, orbname, npts, framenum):
orbital_key = (orbname, npts, framenum)
if orbital_key not in self._cached_cubefiles:
grid = self.calc_orb_grid(orbname, npts, framenum)
cubefile = self._grid_to_cube(grid)
self._cached_cubefiles[orbital_key] = cubefile
else:
cubefile = self._cached_cubefiles[orbital_key]
return cubefile
def draw_orbital(self,
orbname,
npts=50,
isoval=0.01,
opacity=0.8,
negative_color='red',
positive_color='blue'):
"""Display a molecular orbital
Args:
orbname: name of the orbital (interface dependent)
npts (int): resolution in each dimension
isoval (float): isosurface value to draw
opacity (float): opacity of the orbital (between 0 and 1)
positive_color (str or int): color of the positive isosurfaces
negative_color (str or int): color of the negative isosurfaces
"""
self.orbital_spec = dict(npts=npts,
isoval=isoval,
opacity=opacity,
negative_color=negative_color,
positive_color=positive_color)
self.current_orbital = orbname
positive_color = translate_color(positive_color)
negative_color = translate_color(negative_color)
orbidx = self.get_orbidx(orbname)
cubefile = self.get_cubefile(orbidx, npts, self.current_frame)
self.orbital = {
'cube_file': cubefile,
'iso_val': isoval,
'opacity': opacity
}
def get_orbidx(self, orbname):
try:
if orbname.lower().strip() == 'h**o':
orbname = self.homo_index()
elif orbname.lower().strip() == 'lumo':
orbname = self.homo_index() + 1
except AttributeError:
pass
return orbname
def get_orbnames(self):
raise NotImplementedError
@staticmethod
def _grid_to_cube(grid, f=None):
if f is None:
fobj = StringIO()
elif not hasattr(f, 'write'):
fobj = open(f, 'w')
else:
fobj = f
# First two header lines
print >> fobj, 'CUBE File\nGenerated by nbmolviz'
# third line: number of atoms (0, here) + origin of grid
print >> fobj, '-1 %f %f %f' % grid.origin()
# lines 4-7: number of points in each direction and basis vector for each
# basis vectors are negative to indicate angstroms
print >> fobj, '%d %f 0.0 0.0' % (-grid.npoints, grid.dx)
print >> fobj, '%d 0.0 %f 0.0' % (-grid.npoints, grid.dy)
print >> fobj, '%d 0.0 0.0 %f' % (-grid.npoints, grid.dz)
# Next is a line per atom
# We put just one atom here - it shouldn't be rendered
print >> fobj, '6 0.000 0.0 0.0 0.0'
# Next, indicate that there's just one orbital
print >> fobj, 1, 1
# finally, write out all the grid values
# ival = 0
for ix in xrange(grid.npoints):
for iy in xrange(grid.npoints):
for iz in xrange(grid.npoints):
print >> fobj, grid.fxyz[ix, iy, iz],
# ival += 1
# if ival%6 == 0: print >> fobj #newline
if iz % 6 == 5: print >> fobj
print >> fobj
if f is None:
fobj.seek(0)
return fobj.getvalue()
else:
fobj.close()
class DisplayHook(Configurable):
"""The custom IPython displayhook to replace sys.displayhook.
This class does many things, but the basic idea is that it is a callable
that gets called anytime user code returns a value.
"""
shell = Instance('IPython.core.interactiveshell.InteractiveShellABC',
allow_none=True)
exec_result = Instance('IPython.core.interactiveshell.ExecutionResult',
allow_none=True)
cull_fraction = Float(0.2)
def __init__(self, shell=None, cache_size=1000, **kwargs):
super(DisplayHook, self).__init__(shell=shell, **kwargs)
cache_size_min = 3
if cache_size <= 0:
self.do_full_cache = 0
cache_size = 0
elif cache_size < cache_size_min:
self.do_full_cache = 0
cache_size = 0
warn('caching was disabled (min value for cache size is %s).' %
cache_size_min,
stacklevel=3)
else:
self.do_full_cache = 1
self.cache_size = cache_size
# we need a reference to the user-level namespace
self.shell = shell
self._, self.__, self.___ = '', '', ''
# these are deliberately global:
to_user_ns = {'_': self._, '__': self.__, '___': self.___}
self.shell.user_ns.update(to_user_ns)
@property
def prompt_count(self):
return self.shell.execution_count
#-------------------------------------------------------------------------
# Methods used in __call__. Override these methods to modify the behavior
# of the displayhook.
#-------------------------------------------------------------------------
def check_for_underscore(self):
"""Check if the user has set the '_' variable by hand."""
# If something injected a '_' variable in __builtin__, delete
# ipython's automatic one so we don't clobber that. gettext() in
# particular uses _, so we need to stay away from it.
if '_' in builtin_mod.__dict__:
try:
user_value = self.shell.user_ns['_']
if user_value is not self._:
return
del self.shell.user_ns['_']
except KeyError:
pass
def quiet(self):
"""Should we silence the display hook because of ';'?"""
# do not print output if input ends in ';'
try:
cell = self.shell.history_manager.input_hist_parsed[-1]
except IndexError:
# some uses of ipshellembed may fail here
return False
sio = _io.StringIO(cell)
tokens = list(tokenize.generate_tokens(sio.readline))
for token in reversed(tokens):
if token[0] in (tokenize.ENDMARKER, tokenize.NL, tokenize.NEWLINE,
tokenize.COMMENT):
continue
if (token[0] == tokenize.OP) and (token[1] == ';'):
return True
else:
return False
def start_displayhook(self):
"""Start the displayhook, initializing resources."""
pass
def write_output_prompt(self):
"""Write the output prompt.
The default implementation simply writes the prompt to
``sys.stdout``.
"""
# Use write, not print which adds an extra space.
sys.stdout.write(self.shell.separate_out)
outprompt = 'Out[{}]: '.format(self.shell.execution_count)
if self.do_full_cache:
sys.stdout.write(outprompt)
def compute_format_data(self, result):
"""Compute format data of the object to be displayed.
The format data is a generalization of the :func:`repr` of an object.
In the default implementation the format data is a :class:`dict` of
key value pair where the keys are valid MIME types and the values
are JSON'able data structure containing the raw data for that MIME
type. It is up to frontends to determine pick a MIME to to use and
display that data in an appropriate manner.
This method only computes the format data for the object and should
NOT actually print or write that to a stream.
Parameters
----------
result : object
The Python object passed to the display hook, whose format will be
computed.
Returns
-------
(format_dict, md_dict) : dict
format_dict is a :class:`dict` whose keys are valid MIME types and values are
JSON'able raw data for that MIME type. It is recommended that
all return values of this should always include the "text/plain"
MIME type representation of the object.
md_dict is a :class:`dict` with the same MIME type keys
of metadata associated with each output.
"""
return self.shell.display_formatter.format(result)
# This can be set to True by the write_output_prompt method in a subclass
prompt_end_newline = False
def write_format_data(self, format_dict, md_dict=None) -> None:
"""Write the format data dict to the frontend.
This default version of this method simply writes the plain text
representation of the object to ``sys.stdout``. Subclasses should
override this method to send the entire `format_dict` to the
frontends.
Parameters
----------
format_dict : dict
The format dict for the object passed to `sys.displayhook`.
md_dict : dict (optional)
The metadata dict to be associated with the display data.
"""
if 'text/plain' not in format_dict:
# nothing to do
return
# We want to print because we want to always make sure we have a
# newline, even if all the prompt separators are ''. This is the
# standard IPython behavior.
result_repr = format_dict['text/plain']
if '\n' in result_repr:
# So that multi-line strings line up with the left column of
# the screen, instead of having the output prompt mess up
# their first line.
# We use the prompt template instead of the expanded prompt
# because the expansion may add ANSI escapes that will interfere
# with our ability to determine whether or not we should add
# a newline.
if not self.prompt_end_newline:
# But avoid extraneous empty lines.
result_repr = '\n' + result_repr
try:
print(result_repr)
except UnicodeEncodeError:
# If a character is not supported by the terminal encoding replace
# it with its \u or \x representation
print(
result_repr.encode(sys.stdout.encoding,
'backslashreplace').decode(
sys.stdout.encoding))
def update_user_ns(self, result):
"""Update user_ns with various things like _, __, _1, etc."""
# Avoid recursive reference when displaying _oh/Out
if self.cache_size and result is not self.shell.user_ns['_oh']:
if len(self.shell.user_ns['_oh']
) >= self.cache_size and self.do_full_cache:
self.cull_cache()
# Don't overwrite '_' and friends if '_' is in __builtin__
# (otherwise we cause buggy behavior for things like gettext). and
# do not overwrite _, __ or ___ if one of these has been assigned
# by the user.
update_unders = True
for unders in ['_' * i for i in range(1, 4)]:
if not unders in self.shell.user_ns:
continue
if getattr(self, unders) is not self.shell.user_ns.get(unders):
update_unders = False
self.___ = self.__
self.__ = self._
self._ = result
if ('_' not in builtin_mod.__dict__) and (update_unders):
self.shell.push({
'_': self._,
'__': self.__,
'___': self.___
},
interactive=False)
# hackish access to top-level namespace to create _1,_2... dynamically
to_main = {}
if self.do_full_cache:
new_result = '_%s' % self.prompt_count
to_main[new_result] = result
self.shell.push(to_main, interactive=False)
self.shell.user_ns['_oh'][self.prompt_count] = result
def fill_exec_result(self, result):
if self.exec_result is not None:
self.exec_result.result = result
def log_output(self, format_dict):
"""Log the output."""
if 'text/plain' not in format_dict:
# nothing to do
return
if self.shell.logger.log_output:
self.shell.logger.log_write(format_dict['text/plain'], 'output')
self.shell.history_manager.output_hist_reprs[self.prompt_count] = \
format_dict['text/plain']
def finish_displayhook(self):
"""Finish up all displayhook activities."""
sys.stdout.write(self.shell.separate_out2)
sys.stdout.flush()
def __call__(self, result=None):
"""Printing with history cache management.
This is invoked every time the interpreter needs to print, and is
activated by setting the variable sys.displayhook to it.
"""
self.check_for_underscore()
if result is not None and not self.quiet():
self.start_displayhook()
self.write_output_prompt()
format_dict, md_dict = self.compute_format_data(result)
self.update_user_ns(result)
self.fill_exec_result(result)
if format_dict:
self.write_format_data(format_dict, md_dict)
self.log_output(format_dict)
self.finish_displayhook()
def cull_cache(self):
"""Output cache is full, cull the oldest entries"""
oh = self.shell.user_ns.get('_oh', {})
sz = len(oh)
cull_count = max(int(sz * self.cull_fraction), 2)
warn('Output cache limit (currently {sz} entries) hit.\n'
'Flushing oldest {cull_count} entries.'.format(
sz=sz, cull_count=cull_count))
for i, n in enumerate(sorted(oh)):
if i >= cull_count:
break
self.shell.user_ns.pop('_%i' % n, None)
oh.pop(n, None)
def flush(self):
if not self.do_full_cache:
raise ValueError("You shouldn't have reached the cache flush "
"if full caching is not enabled!")
# delete auto-generated vars from global namespace
for n in range(1, self.prompt_count + 1):
key = '_' + repr(n)
try:
del self.shell.user_ns[key]
except:
pass
# In some embedded circumstances, the user_ns doesn't have the
# '_oh' key set up.
oh = self.shell.user_ns.get('_oh', None)
if oh is not None:
oh.clear()
# Release our own references to objects:
self._, self.__, self.___ = '', '', ''
if '_' not in builtin_mod.__dict__:
self.shell.user_ns.update({
'_': self._,
'__': self.__,
'___': self.___
})
import gc
# TODO: Is this really needed?
# IronPython blocks here forever
if sys.platform != "cli":
gc.collect()
class RasterLayer(Layer):
_view_name = Unicode('LeafletRasterLayerView').tag(sync=True)
_model_name = Unicode('LeafletRasterLayerModel').tag(sync=True)
opacity = Float(1.0, min=0.0, max=1.0).tag(sync=True)
visible = Bool(True).tag(sync=True)
class Figure(DOMWidget):
"""Main canvas for drawing a chart.
The Figure object holds the list of Marks and Axes. It also holds an
optional Interaction object that is responsible for figure-level mouse
interactions, the "interaction layer".
Besides, the Figure object has two reference scales, for positioning items
in an absolute fashion in the figure canvas.
Attributes
----------
title: string (default: '')
title of the figure
axes: List of Axes (default: [])
list containing the instances of the axes for the figure
marks: List of Marks (default: [])
list containing the marks which are to be appended to the figure
interaction: Interaction or None (default: )
optional interaction layer for the figure
scale_x: Scale
Scale representing the x values of the figure
scale_y: Scale
Scale representing the y values of the figure
padding_x: Float (default: 0.0)
Padding to be applied in the horizontal direction of the figure
around the data points, proportion of the horizontal length
padding_y: Float (default: 0.025)
Padding to be applied in the vertical direction of the figure
around the data points, proportion of the vertical length
legend_location: {'top-right', 'top', 'top-left', 'left', 'bottom-left', 'bottom', 'bottom-right', 'right'}
location of the legend relative to the center of the figure
background_style: Dict (default: {})
CSS style to be applied to the background of the figure
title_style: Dict (default: {})
CSS style to be applied to the title of the figure
animation_duration: nonnegative int (default: 0)
Duration of transition on change of data attributes, in milliseconds.
Layout Attributes
fig_margin: dict (default: {top=60, bottom=60, left=60, right=60})
Dictionary containing the top, bottom, left and right margins. The user
is responsible for making sure that the width and height are greater
than the sum of the margins.
min_aspect_ratio: float
minimum width / height ratio of the figure
max_aspect_ratio: float
maximum width / height ratio of the figure
Methods
-------
save_png:
Saves the figure as a png file
Note
----
The aspect ratios stand for width / height ratios.
- If the available space is within bounds in terms of min and max aspect
ratio, we use the entire available space.
- If the available space is too oblong horizontally, we use the client
height and the width that corresponds max_aspect_ratio (maximize width
under the constraints).
- If the available space is too oblong vertically, we use the client width
and the height that corresponds to min_aspect_ratio (maximize height
under the constraint).
This corresponds to maximizing the area under the constraints.
Default min and max aspect ratio are both equal to 16 / 9.
"""
title = Unicode().tag(sync=True, display_name='Title')
axes = List(Instance(Axis)).tag(sync=True, **widget_serialization)
marks = List(Instance(Mark)).tag(sync=True, **widget_serialization)
interaction = Instance(Interaction, default_value=None, allow_none=True).tag(sync=True,
**widget_serialization)
scale_x = Instance(Scale).tag(sync=True, **widget_serialization)
scale_y = Instance(Scale).tag(sync=True, **widget_serialization)
title_style = Dict(trait=Unicode()).tag(sync=True)
background_style = Dict().tag(sync=True)
layout = Instance(Layout, kw={
'flex': '1',
'align_self': 'stretch',
'min_width': '400px'
}, allow_none=True).tag(sync=True, **widget_serialization)
min_aspect_ratio = Float(16.0 / 9.0).tag(sync=True)
max_aspect_ratio = Float(16.0 / 9.0).tag(sync=True)
fig_margin = Dict(dict(top=60, bottom=60, left=60, right=60)).tag(sync=True)
padding_x = Float(0.0, min=0.0, max=1.0).tag(sync=True)
padding_y = Float(0.025, min=0.0, max=1.0).tag(sync=True)
legend_location = Enum(['top-right', 'top', 'top-left', 'left',
'bottom-left', 'bottom', 'bottom-right', 'right'],
default_value='top-right').tag(sync=True, display_name='Legend position')
animation_duration = Int().tag(sync=True, display_name='Animation duration')
@default('scale_x')
def _default_scale_x(self):
return LinearScale(min=0, max=1, allow_padding=False)
@default('scale_y')
def _default_scale_y(self):
return LinearScale(min=0, max=1, allow_padding=False)
def save_png(self):
self.send({"type": "save_png"})
@validate('min_aspect_ratio', 'max_aspect_ratio')
def _validate_aspect_ratio(self, proposal):
value = proposal['value']
if proposal['trait'].name == 'min_aspect_ratio' and value > self.max_aspect_ratio:
raise TraitError('setting min_aspect_ratio > max_aspect_ratio')
if proposal['trait'].name == 'max_aspect_ratio' and value < self.min_aspect_ratio:
raise TraitError('setting max_aspect_ratio < min_aspect_ratio')
return value
_view_name = Unicode('Figure').tag(sync=True)
_model_name = Unicode('FigureModel').tag(sync=True)
_view_module = Unicode('bqplot').tag(sync=True)
_model_module = Unicode('bqplot').tag(sync=True)
class ImageSlider(base.DOMWidget):
"""The backend python class for the custom ImageSlider widget.
This class declares and initializes all of the data that is synced
between the front- and back-ends of the widget code.
It also provides the majority of the calculation-based code
that runs the ImageSlider widget."""
_view_name = Unicode("ImgSliderView").tag(sync=True)
_model_name = Unicode("ImgSliderModel").tag(sync=True)
# public attrs
height = Integer().tag(sync=True)
width = Integer().tag(sync=True)
# index of current image in display
_img_index = Integer(0).tag(sync=True)
_N_images = Integer().tag(sync=True)
_b64value = Unicode().tag(sync=True)
_err = Unicode().tag(sync=True)
_format = Unicode("png").tag(sync=True)
_series_min = Float().tag(sync=True)
_series_max = Float().tag(sync=True)
_img_min = Float().tag(sync=True)
_img_max = Float().tag(sync=True)
_nrows = Integer().tag(sync=True)
_ncols = Integer().tag(sync=True)
_offsetX = Integer().tag(sync=True)
_offsetY = Integer().tag(sync=True)
_pix_val = Float().tag(sync=True)
# These variables were added to support zoom functionality
_ROI = Tuple((0, 0, 0, 0), sync=True) # Xtop, Ytop, Xbottom, Ybottom
_extrarows = Integer(0).tag(sync=True)
_extracols = Integer(0).tag(sync=True)
_nrows_currimg = Integer().tag(sync=True)
_ncols_currimg = Integer().tag(sync=True)
_xcoord_absolute = Integer(0).tag(sync=True)
_ycoord_absolute = Integer(0).tag(sync=True)
_vslide_reset = Integer(0).tag(sync=True)
_xcoord_max_roi = Integer().tag(sync=True)
_ycoord_max_roi = Integer().tag(sync=True)
def __init__(self, image_series, width, height):
"""Constructor method for setting the necessary member variables
that are synced between the front- and back-ends.
Creates the following non-synced member variables:
*image_series: the list containing the original series of image objects
passed by the image_series parameter.
This variable is not changed in the code to preserve the original data.
*current_img: the image object or corresponding numpy array of data
that is currently being displayed
*arr: a numpy array containing the data for the current image
that does not contain buffer rows/columns
*curr_img_data: a numpy array containing the data for the current image,
including buffer rows/columns
*xbuff and ybuff: the number of buffer rows in the previously displayed image
Parameters:
*image_series: a list of ImageFile objects (see
https://github.com/ornlneutronimaging/iMars3D/blob/master/python/imars3d/ImageFile.py
for more details).
This list is used to give the widget access
to the images that are to be viewed.
*width: an integer that is used to set the width of the image and UI elements.
*height: an integer that is used to set the height of the image and UI elements."""
super(ImageSlider, self).__init__()
assert len(image_series), "Image series cannot be empty"
self.image_series = list(image_series)
self.width = width
self.height = height
self._N_images = len(self.image_series)
self.current_img = self.image_series[self._img_index]
# image data array. need it to obtain the value at mouse p
self.arr = self.current_img.data.copy().astype("float")
# image data in the <img> tag. this may contains buffers at zoom, or may be altered due to
# intensity range limit
self.curr_img_data = self.arr.copy()
self._nrows, self._ncols = self.arr.shape
self._nrows_currimg, self._ncols_currimg = self.arr.shape
self._ycoord_max_roi, self._xcoord_max_roi = self.arr.shape
self.ybuff = 0
self.xbuff = 0
self._zoom = False
self.get_series_minmax()
self.update_image_div_data(None)
return
def get_series_minmax(self, sample_size=10):
"""Determines the absolute minimum and maximum image values of either
all the images in self.image_series or of
'sample_size' random images from self.image_series
Parameters:
*sample_size: the maximum number of images to use
in determining _img_min and _img_max.
By default, its value is 10."""
img_series = list(self.image_series)
N = len(img_series)
if N < sample_size:
data = [img.data for img in img_series]
else:
indexes = np.random.choice(N, sample_size, replace=False)
data = [img_series[i].data for i in indexes]
self._series_min = self._img_min = float(np.min(data))
self._series_max = self._img_max = float(np.max(data))
return
#This function is called when the values of _offsetX and/or _offsetY change
@observe("_offsetX", "_offsetY")
def get_val(self, change):
"""Tries to calculate the value of the image at the mouse position
and store the result in the member variable _pix_val
If an error occurs, this method calls the handle_error method
and stores the result in the member variable _err."""
try:
col = int(self._offsetX / self.width * self._ncols_currimg)
row = int(self._offsetY / self.height * self._nrows_currimg)
if self._extrarows != 0:
row = row - self.ybuff
if self._extracols != 0:
col = col - self.xbuff
if col >= self.arr.shape[1]:
col = self.arr.shape[1] - 1
if row >= self.arr.shape[0]:
row = self.arr.shape[0] - 1
self._pix_val = float(self.arr[row, col])
self._err = ""
except Exception:
self._pix_val = float(np.nan)
self._err = self.handle_error()
return
def getimg_bytes(self):
"""Encodes the data for the currently viewed image into Base64.
If _img_min and/or _img_max have been changed from their initial values,
this function will also change the image data to account for
this change before encoding the data into Base64."""
# force the intensity range limitation to be positive
if self._img_min >= self._img_max:
self._img_max = self._img_min + (self._series_max -
self._series_min) * 1e-5
# apply intensity range
self.curr_img_data[self.curr_img_data < self._img_min] = self._img_min
self.curr_img_data[self.curr_img_data > self._img_max] = self._img_max
img = ((self.curr_img_data - self._img_min) /
(self._img_max - self._img_min) * (2**8 - 1)).astype('uint8')
size = np.max(img.shape)
view_size = np.max((self.width, self.height))
# resample if necessary
resample_ratio = view_size / size
if resample_ratio != 1.:
import scipy.misc
img = scipy.misc.imresize(img, resample_ratio)
"""Allows the correct string IO module to be used
based on the version of Python.
Once support for Python 2.7 ends, this if-else statement
can be replaced by just the contents of the else statement."""
if sys.version_info < (3, 0):
from cStringIO import StringIO
f = StringIO()
else:
from io import BytesIO
f = BytesIO()
import PIL.Image, base64
PIL.Image.fromarray(img).save(f, self._format)
imgb64v = base64.b64encode(f.getvalue())
return imgb64v
def handle_error(self):
"""Creates and returns a custom error message if an error occurs in the get_val method."""
cla, exc, tb = sys.exc_info()
ex_name = cla.__name__
try:
ex_args = exc.__dict__["args"]
except KeyError:
ex_args = ("No args", )
ex_mess = str(ex_name)
for arg in ex_args:
ex_mess = ex_mess + str(arg)
return (ex_mess)
#This function is called when _img_index
@observe("_img_index")
def update_image_index(self, change):
"""
change the current_img member variable to the new desired image
and then update the image div
"""
self.current_img = self.image_series[self._img_index]
self.update_image_div_data(change)
return
@observe("_img_min", "_img_max")
def update_image_div_data(self, change):
"""update the image div data
This function is called whenever the image div needs to be updated.
It could be triggered directly by changes to _img_min and _img_max, but also
by function calls from handlers of other events such as image-index-change,
and ROI-change.
If the zoom is activated (see flag _zoom)
this function will call the update_image_div_data_with_zoom function to zoom into
the image and obtain the Base64 encoding.
Otherwise, this function calls the getimg_bytes method
to obtain the new Base64 encoding (of either the new or old image)
and stores this encoding in _b64value."""
self.arr = self.current_img.data.copy().astype("float")
self.curr_img_data = self.arr.copy()
if self._zoom:
self.update_image_div_data_with_zoom()
return
self._nrows, self._ncols = self.arr.shape
self._ycoord_max_roi, self._xcoord_max_roi = self.arr.shape
self._nrows_currimg, self._ncols_currimg = self.arr.shape
self._b64value = self.getimg_bytes()
return
#This function is called when _ROI changes.
@observe("_ROI")
def zoom_image(self, change):
"""Sets all values necessary for zooming into a Region of Interest
and then calls the update_image_div_data function."""
Xtop, Ytop, Xbottom, Ybottom = self._ROI
if Xtop < 0: # invalid ROI means reset
self._zoom = False
return self.reset_image()
self._zoom = True
self.left = int(Xtop / self.width * self._ncols_currimg)
self.right = int(Xbottom / self.width * self._ncols_currimg)
self.top = int(Ytop / self.height * self._nrows_currimg)
self.bottom = int(Ybottom / self.height * self._nrows_currimg)
self._xcoord_absolute += (self.left - self.xbuff)
self._ycoord_absolute += (self.top - self.ybuff)
self.update_image_div_data(change)
return
def reset_image(self):
"""Resets all variables that are involved in zooming to their default values.
After resetting, the update_image_div_data function is called."""
self._extrarows = 0
self._extracols = 0
self.xbuff = 0
self.ybuff = 0
self._xcoord_absolute = 0
self._ycoord_absolute = 0
self.get_series_minmax()
self._vslide_reset += 1
self.update_image_div_data(None)
return
def update_image_div_data_with_zoom(self):
"""The function that controlls zooming on a single image.
It splices the image data based on the left, right, bottom,
and top variables calculated in the zoom_image function.
The function then copies the data in ROI and adds
buffer rows/columns to the copy to insure the data
used to create the image is a square numpy array.
Then, the number of extra rows and/or columns is calculated.
Finally, the zoomed image data is converted to
a displayable image by calling the getimg_bytes function."""
select_width = self.right - self.left
select_height = self.bottom - self.top
if select_width == 0: select_width = 1
if select_height == 0: select_height = 1
self.arr = self.arr[self._ycoord_absolute:(self._ycoord_absolute +
select_height),
self._xcoord_absolute:(self._xcoord_absolute +
select_width)]
self._nrows, self._ncols = self.arr.shape
self.curr_img_data = self.arr.copy()
# calculate paddings
if self._ncols > self._nrows:
diff = self._ncols - self._nrows
if diff % 2 == 0:
addtop = diff // 2
addbottom = diff // 2
else:
addtop = diff // 2 + 1
addbottom = diff // 2
self.xbuff = 0
self.ybuff = addtop
self._nrows_currimg = self._ncols
self._ncols_currimg = self._ncols
self._extrarows = diff
self._extracols = 0
extrarows_top = np.full((addtop, self._ncols), 1)
extrarows_bottom = np.full((addbottom, self._ncols), 1)
self.curr_img_data = np.vstack(
(extrarows_top, self.curr_img_data, extrarows_bottom))
else:
diff = self._nrows - self._ncols
if diff % 2 == 0:
addleft = diff // 2
addright = diff // 2
else:
addleft = diff // 2 + 1
addright = diff // 2
self.xbuff = addleft
self.ybuff = 0
self._nrows_currimg = self._nrows
self._ncols_currimg = self._nrows
self._extrarows = 0
self._extracols = diff
extrarows_left = np.full((self._nrows, addleft), 1)
extrarows_right = np.full((self._nrows, addright), 1)
self.curr_img_data = np.hstack(
(extrarows_left, self.curr_img_data, extrarows_right))
self._xcoord_max_roi = self._xcoord_absolute + self._ncols_currimg - self._extracols
self._ycoord_max_roi = self._ycoord_absolute + self._nrows_currimg - self._extrarows
self._b64value = self.getimg_bytes()
return
class ClingKernel(Kernel):
"""Cling Kernel for Jupyter"""
implementation = 'cling_kernel'
implementation_version = __version__
language_version = 'X'
banner = Unicode()
def _banner_default(self):
return 'cling-%s' % self.language_version
return self._banner
language_info = {
'name': 'c++',
'codemirror_mode': 'c++',
'mimetype': 'text/x-c++src',
'file_extension': '.c++'
}
# Used in handle_input()
flush_interval = Float(0.25, config=True)
std = CaselessStrEnum(
default_value='c++11',
values=['c++11', 'c++14', 'c++1z', 'c++17'],
help="C++ standard to use, either c++17, c++1z, c++14 or c++11").tag(
config=True)
def __init__(self, **kwargs):
super(ClingKernel, self).__init__(**kwargs)
try:
whichCling = os.readlink(shutil.which('cling'))
except OSError as e:
#If cling is not a symlink try a regular file
#readlink returns POSIX error EINVAL (22) if the
#argument is not a symlink
if e.args[0] == 22:
whichCling = shutil.which('cling')
else:
raise e
except AttributeError:
from distutils.spawn import find_executable
whichCling = find_executable('cling')
if whichCling:
clingInstDir = os.path.dirname(os.path.dirname(whichCling))
llvmResourceDir = clingInstDir
else:
raise RuntimeError('Cannot find cling in $PATH. No cling, no fun.')
for ext in ['so', 'dylib', 'dll']:
libFilename = clingInstDir + "/lib/libclingJupyter." + ext
if os.access(libFilename, os.R_OK):
self.libclingJupyter = ctypes.CDLL(
clingInstDir + "/lib/libclingJupyter." + ext,
mode=ctypes.RTLD_GLOBAL)
break
if not getattr(self, 'libclingJupyter', None):
raise RuntimeError('Cannot find ' + clingInstDir +
'/lib/libclingJupyter.{so,dylib,dll}')
self.libclingJupyter.cling_create.restype = my_void_p
self.libclingJupyter.cling_eval.restype = my_void_p
#build -std=c++11 or -std=c++14 option
stdopt = ("-std=" + self.std).encode('utf-8')
self.log.info("Using {}".format(stdopt.decode('utf-8')))
#from IPython.utils import io
#io.rprint("DBG: Using {}".format(stdopt.decode('utf-8')))
strarr = ctypes.c_char_p * 5
argv = strarr(b"clingJupyter", stdopt,
b"-I" + clingInstDir.encode('utf-8') + b"/include/", b"",
b"")
llvmResourceDirCP = ctypes.c_char_p(llvmResourceDir.encode('utf8'))
# The sideband_pipe is used by cling::Jupyter::pushOutput() to publish MIME data to Jupyter.
self.sideband_pipe, pipe_in = os.pipe()
self.interp = self.libclingJupyter.cling_create(
5, argv, llvmResourceDirCP, pipe_in)
self.libclingJupyter.cling_complete_start.restype = my_void_p
self.libclingJupyter.cling_complete_next.restype = my_void_p #c_char_p
def _process_stdio_data(self, pipe, name):
"""Read from the pipe, send it to IOPub as name stream."""
data = os.read(pipe, 1024)
# send output
self.session.send(self.iopub_socket,
'stream', {
'name': name,
'text': data.decode('utf8', 'replace'),
},
parent=self._parent_header)
def _recv_dict(self, pipe):
"""Receive a serialized dict on a pipe
Returns the dictionary.
"""
# Wire format:
# // Pipe sees (all numbers are longs, except for the first):
# // - num bytes in a long (sent as a single unsigned char!)
# // - num elements of the MIME dictionary; Jupyter selects one to display.
# // For each MIME dictionary element:
# // - length of MIME type key
# // - MIME type key
# // - size of MIME data buffer (including the terminating 0 for
# // 0-terminated strings)
# // - MIME data buffer
data = {}
b1 = os.read(pipe, 1)
sizeof_long = struct.unpack('B', b1)[0]
if sizeof_long == 8:
fmt = 'Q'
else:
fmt = 'L'
buf = os.read(pipe, sizeof_long)
num_elements = struct.unpack(fmt, buf)[0]
for i in range(num_elements):
buf = os.read(pipe, sizeof_long)
len_key = struct.unpack(fmt, buf)[0]
key = os.read(pipe, len_key).decode('utf8')
buf = os.read(pipe, sizeof_long)
len_value = struct.unpack(fmt, buf)[0]
value = os.read(pipe, len_value).decode('utf8')
data[key] = value
return data
def _process_sideband_data(self):
"""publish display-data messages on IOPub.
"""
data = self._recv_dict(self.sideband_pipe)
self.session.send(
self.iopub_socket,
'display_data',
content={
'data': data,
'metadata': {},
},
parent=self._parent_header,
)
def forward_streams(self):
"""Put the forwarding pipes in place for stdout, stderr."""
self.replaced_streams = [FdReplacer("stdout"), FdReplacer("stderr")]
def handle_input(self):
"""Capture stdout, stderr and sideband. Forward them as stream messages."""
# create pipe for stdout, stderr
select_on = [self.sideband_pipe]
for rs in self.replaced_streams:
if rs:
select_on.append(rs.pipe_out)
r, w, x = select.select(select_on, [], [], self.flush_interval)
if not r:
# nothing to read, flush libc's stdout and check again
libc.fflush(c_stdout_p)
libc.fflush(c_stderr_p)
return False
for fd in r:
if fd == self.sideband_pipe:
self._process_sideband_data()
else:
if fd == self.replaced_streams[0].pipe_out:
rs = 0
else:
rs = 1
self._process_stdio_data(fd, self.replaced_streams[rs].name)
return True
def close_forwards(self):
"""Close the forwarding pipes."""
libc.fflush(c_stdout_p)
libc.fflush(c_stderr_p)
for rs in self.replaced_streams:
rs.restore()
self.replaced_streams = []
def run_cell(self, code, silent=False):
"""Run code in cling, storing the expression result or an empty string if there is none."""
self.stringResult = self.libclingJupyter.cling_eval(
self.interp, ctypes.c_char_p(code.encode('utf8')))
def do_execute(self,
code,
silent,
store_history=True,
user_expressions=None,
allow_stdin=False):
"""Runs code in cling and handles input; returns the evaluation result."""
if not code.strip():
return {
'status': 'ok',
'execution_count': self.execution_count,
'payload': [],
'user_expressions': {},
}
# Redirect stdout, stderr so handle_input() can pick it up.
self.forward_streams()
# Run code in cling in a thread.
run_cell_thread = threading.Thread(target=self.run_cell,
args=(
code,
silent,
))
run_cell_thread.start()
while True:
self.handle_input()
if not run_cell_thread.is_alive():
# self.run_cell() has returned.
break
run_cell_thread.join()
# Any leftovers?
while self.handle_input():
True
self.close_forwards()
status = 'ok'
if not self.stringResult:
status = 'error'
else:
# Execution has finished; we have a result.
self.session.send(
self.iopub_socket,
'execute_result',
content={
'data': {
'text/plain':
ctypes.cast(self.stringResult,
ctypes.c_char_p).value.decode(
'utf8', 'replace'),
},
'metadata': {},
'execution_count': self.execution_count,
},
parent=self._parent_header)
self.libclingJupyter.cling_eval_free(self.stringResult)
reply = {
'status': status,
'execution_count': self.execution_count,
}
if status == 'error':
err = {
'ename': 'ename',
'evalue': 'evalue',
'traceback': [],
}
self.send_response(self.iopub_socket, 'error', err)
reply.update(err)
elif status == 'ok':
reply.update({
'THIS DOES NOT WORK: payload': [{
'source':
'set_next_input',
'replace':
True,
'text':
'//THIS IS MAGIC\n' + code
}],
'user_expressions': {},
})
else:
raise ValueError("Invalid status: %r" % status)
return reply
def do_complete(self, code, cursor_pos):
"""Provide completions here"""
# if cursor_pos = cursor_start = cursor_end,
# matches should be a list of strings to be appended after the cursor
return {
'matches': [],
'cursor_end': cursor_pos,
'cursor_start': cursor_pos,
'metadata': {},
'status': 'ok'
}
class GatewayClient(SingletonConfigurable):
"""This class manages the configuration. It's its own singleton class so that we
can share these values across all objects. It also contains some helper methods
to build request arguments out of the various config options.
"""
url = Unicode(
default_value=None,
allow_none=True,
config=True,
help="""The url of the Kernel or Enterprise Gateway server where
kernel specifications are defined and kernel management takes place.
If defined, this Notebook server acts as a proxy for all kernel
management and kernel specification retrieval. (JUPYTER_GATEWAY_URL env var)
""")
url_env = 'JUPYTER_GATEWAY_URL'
@default('url')
def _url_default(self):
return os.environ.get(self.url_env)
@validate('url')
def _url_validate(self, proposal):
value = proposal['value']
# Ensure value, if present, starts with 'http'
if value is not None and len(value) > 0:
if not str(value).lower().startswith('http'):
raise TraitError(
"GatewayClient url must start with 'http': '%r'" % value)
return value
ws_url = Unicode(
default_value=None,
allow_none=True,
config=True,
help=
"""The websocket url of the Kernel or Enterprise Gateway server. If not provided, this value
will correspond to the value of the Gateway url with 'ws' in place of 'http'. (JUPYTER_GATEWAY_WS_URL env var)
""")
ws_url_env = 'JUPYTER_GATEWAY_WS_URL'
@default('ws_url')
def _ws_url_default(self):
default_value = os.environ.get(self.ws_url_env)
if default_value is None:
if self.gateway_enabled:
default_value = self.url.lower().replace('http', 'ws')
return default_value
@validate('ws_url')
def _ws_url_validate(self, proposal):
value = proposal['value']
# Ensure value, if present, starts with 'ws'
if value is not None and len(value) > 0:
if not str(value).lower().startswith('ws'):
raise TraitError(
"GatewayClient ws_url must start with 'ws': '%r'" % value)
return value
kernels_endpoint_default_value = '/api/kernels'
kernels_endpoint_env = 'JUPYTER_GATEWAY_KERNELS_ENDPOINT'
kernels_endpoint = Unicode(
default_value=kernels_endpoint_default_value,
config=True,
help=
"""The gateway API endpoint for accessing kernel resources (JUPYTER_GATEWAY_KERNELS_ENDPOINT env var)"""
)
@default('kernels_endpoint')
def _kernels_endpoint_default(self):
return os.environ.get(self.kernels_endpoint_env,
self.kernels_endpoint_default_value)
kernelspecs_endpoint_default_value = '/api/kernelspecs'
kernelspecs_endpoint_env = 'JUPYTER_GATEWAY_KERNELSPECS_ENDPOINT'
kernelspecs_endpoint = Unicode(
default_value=kernelspecs_endpoint_default_value,
config=True,
help=
"""The gateway API endpoint for accessing kernelspecs (JUPYTER_GATEWAY_KERNELSPECS_ENDPOINT env var)"""
)
@default('kernelspecs_endpoint')
def _kernelspecs_endpoint_default(self):
return os.environ.get(self.kernelspecs_endpoint_env,
self.kernelspecs_endpoint_default_value)
kernelspecs_resource_endpoint_default_value = '/kernelspecs'
kernelspecs_resource_endpoint_env = 'JUPYTER_GATEWAY_KERNELSPECS_RESOURCE_ENDPOINT'
kernelspecs_resource_endpoint = Unicode(
default_value=kernelspecs_resource_endpoint_default_value,
config=True,
help="""The gateway endpoint for accessing kernelspecs resources
(JUPYTER_GATEWAY_KERNELSPECS_RESOURCE_ENDPOINT env var)""")
@default('kernelspecs_resource_endpoint')
def _kernelspecs_resource_endpoint_default(self):
return os.environ.get(self.kernelspecs_resource_endpoint_env,
self.kernelspecs_resource_endpoint_default_value)
connect_timeout_default_value = 40.0
connect_timeout_env = 'JUPYTER_GATEWAY_CONNECT_TIMEOUT'
connect_timeout = Float(
default_value=connect_timeout_default_value,
config=True,
help=
"""The time allowed for HTTP connection establishment with the Gateway server.
(JUPYTER_GATEWAY_CONNECT_TIMEOUT env var)""")
@default('connect_timeout')
def connect_timeout_default(self):
return float(
os.environ.get('JUPYTER_GATEWAY_CONNECT_TIMEOUT',
self.connect_timeout_default_value))
request_timeout_default_value = 40.0
request_timeout_env = 'JUPYTER_GATEWAY_REQUEST_TIMEOUT'
request_timeout = Float(
default_value=request_timeout_default_value,
config=True,
help=
"""The time allowed for HTTP request completion. (JUPYTER_GATEWAY_REQUEST_TIMEOUT env var)"""
)
@default('request_timeout')
def request_timeout_default(self):
return float(
os.environ.get('JUPYTER_GATEWAY_REQUEST_TIMEOUT',
self.request_timeout_default_value))
client_key = Unicode(
default_value=None,
allow_none=True,
config=True,
help=
"""The filename for client SSL key, if any. (JUPYTER_GATEWAY_CLIENT_KEY env var)
""")
client_key_env = 'JUPYTER_GATEWAY_CLIENT_KEY'
@default('client_key')
def _client_key_default(self):
return os.environ.get(self.client_key_env)
client_cert = Unicode(
default_value=None,
allow_none=True,
config=True,
help=
"""The filename for client SSL certificate, if any. (JUPYTER_GATEWAY_CLIENT_CERT env var)
""")
client_cert_env = 'JUPYTER_GATEWAY_CLIENT_CERT'
@default('client_cert')
def _client_cert_default(self):
return os.environ.get(self.client_cert_env)
ca_certs = Unicode(
default_value=None,
allow_none=True,
config=True,
help=
"""The filename of CA certificates or None to use defaults. (JUPYTER_GATEWAY_CA_CERTS env var)
""")
ca_certs_env = 'JUPYTER_GATEWAY_CA_CERTS'
@default('ca_certs')
def _ca_certs_default(self):
return os.environ.get(self.ca_certs_env)
http_user = Unicode(
default_value=None,
allow_none=True,
config=True,
help=
"""The username for HTTP authentication. (JUPYTER_GATEWAY_HTTP_USER env var)
""")
http_user_env = 'JUPYTER_GATEWAY_HTTP_USER'
@default('http_user')
def _http_user_default(self):
return os.environ.get(self.http_user_env)
http_pwd = Unicode(
default_value=None,
allow_none=True,
config=True,
help=
"""The password for HTTP authentication. (JUPYTER_GATEWAY_HTTP_PWD env var)
""")
http_pwd_env = 'JUPYTER_GATEWAY_HTTP_PWD'
@default('http_pwd')
def _http_pwd_default(self):
return os.environ.get(self.http_pwd_env)
headers_default_value = '{}'
headers_env = 'JUPYTER_GATEWAY_HEADERS'
headers = Unicode(
default_value=headers_default_value,
allow_none=True,
config=True,
help=
"""Additional HTTP headers to pass on the request. This value will be converted to a dict.
(JUPYTER_GATEWAY_HEADERS env var)
""")
@default('headers')
def _headers_default(self):
return os.environ.get(self.headers_env, self.headers_default_value)
auth_token = Unicode(
default_value=None,
allow_none=True,
config=True,
help=
"""The authorization token used in the HTTP headers. The header will be formatted as:
{
'Authorization': '{auth_scheme} {auth_token}'
}
(JUPYTER_GATEWAY_AUTH_TOKEN env var)""")
auth_token_env = 'JUPYTER_GATEWAY_AUTH_TOKEN'
@default('auth_token')
def _auth_token_default(self):
return os.environ.get(self.auth_token_env, '')
auth_scheme = Unicode(
default_value=None,
allow_none=True,
config=True,
help=
"""The auth scheme, added as a prefix to the authorization token used in the HTTP headers.
(JUPYTER_GATEWAY_AUTH_SCHEME env var)""")
auth_scheme_env = 'JUPYTER_GATEWAY_AUTH_SCHEME'
@default('auth_scheme')
def _auth_scheme_default(self):
return os.environ.get(self.auth_scheme_env, 'token')
validate_cert_default_value = True
validate_cert_env = 'JUPYTER_GATEWAY_VALIDATE_CERT'
validate_cert = Bool(
default_value=validate_cert_default_value,
config=True,
help=
"""For HTTPS requests, determines if server's certificate should be validated or not.
(JUPYTER_GATEWAY_VALIDATE_CERT env var)""")
@default('validate_cert')
def validate_cert_default(self):
return bool(
os.environ.get(self.validate_cert_env,
str(self.validate_cert_default_value)) not in
['no', 'false'])
def __init__(self, **kwargs):
super().__init__(**kwargs)
self._static_args = {} # initialized on first use
env_whitelist_default_value = ''
env_whitelist_env = 'JUPYTER_GATEWAY_ENV_WHITELIST'
env_whitelist = Unicode(
default_value=env_whitelist_default_value,
config=True,
help=
"""A comma-separated list of environment variable names that will be included, along with
their values, in the kernel startup request. The corresponding `env_whitelist` configuration
value must also be set on the Gateway server - since that configuration value indicates which
environmental values to make available to the kernel. (JUPYTER_GATEWAY_ENV_WHITELIST env var)"""
)
@default('env_whitelist')
def _env_whitelist_default(self):
return os.environ.get(self.env_whitelist_env,
self.env_whitelist_default_value)
gateway_retry_interval_default_value = 1.0
gateway_retry_interval_env = 'JUPYTER_GATEWAY_RETRY_INTERVAL'
gateway_retry_interval = Float(
default_value=gateway_retry_interval_default_value,
config=True,
help=
"""The time allowed for HTTP reconnection with the Gateway server for the first time.
Next will be JUPYTER_GATEWAY_RETRY_INTERVAL multiplied by two in factor of numbers of retries
but less than JUPYTER_GATEWAY_RETRY_INTERVAL_MAX.
(JUPYTER_GATEWAY_RETRY_INTERVAL env var)""")
@default('gateway_retry_interval')
def gateway_retry_interval_default(self):
return float(
os.environ.get('JUPYTER_GATEWAY_RETRY_INTERVAL',
self.gateway_retry_interval_default_value))
gateway_retry_interval_max_default_value = 30.0
gateway_retry_interval_max_env = 'JUPYTER_GATEWAY_RETRY_INTERVAL_MAX'
gateway_retry_interval_max = Float(
default_value=gateway_retry_interval_max_default_value,
config=True,
help=
"""The maximum time allowed for HTTP reconnection retry with the Gateway server.
(JUPYTER_GATEWAY_RETRY_INTERVAL_MAX env var)""")
@default('gateway_retry_interval_max')
def gateway_retry_interval_max_default(self):
return float(
os.environ.get('JUPYTER_GATEWAY_RETRY_INTERVAL_MAX',
self.gateway_retry_interval_max_default_value))
gateway_retry_max_default_value = 5
gateway_retry_max_env = 'JUPYTER_GATEWAY_RETRY_MAX'
gateway_retry_max = Int(
default_value=gateway_retry_max_default_value,
config=True,
help=
"""The maximum retries allowed for HTTP reconnection with the Gateway server.
(JUPYTER_GATEWAY_RETRY_MAX env var)""")
@default('gateway_retry_max')
def gateway_retry_max_default(self):
return int(
os.environ.get('JUPYTER_GATEWAY_RETRY_MAX',
self.gateway_retry_max_default_value))
@property
def gateway_enabled(self):
return bool(self.url is not None and len(self.url) > 0)
# Ensure KERNEL_LAUNCH_TIMEOUT has a default value.
KERNEL_LAUNCH_TIMEOUT = int(os.environ.get('KERNEL_LAUNCH_TIMEOUT', 40))
def init_static_args(self):
"""Initialize arguments used on every request. Since these are static values, we'll
perform this operation once.
"""
# Ensure that request timeout and KERNEL_LAUNCH_TIMEOUT are the same, taking the
# greater value of the two.
if self.request_timeout < float(GatewayClient.KERNEL_LAUNCH_TIMEOUT):
self.request_timeout = float(GatewayClient.KERNEL_LAUNCH_TIMEOUT)
elif self.request_timeout > float(GatewayClient.KERNEL_LAUNCH_TIMEOUT):
GatewayClient.KERNEL_LAUNCH_TIMEOUT = int(self.request_timeout)
# Ensure any adjustments are reflected in env.
os.environ['KERNEL_LAUNCH_TIMEOUT'] = str(
GatewayClient.KERNEL_LAUNCH_TIMEOUT)
self._static_args['headers'] = json.loads(self.headers)
if 'Authorization' not in self._static_args['headers'].keys():
self._static_args['headers'].update({
'Authorization':
'{} {}'.format(self.auth_scheme, self.auth_token)
})
self._static_args['connect_timeout'] = self.connect_timeout
self._static_args['request_timeout'] = self.request_timeout
self._static_args['validate_cert'] = self.validate_cert
if self.client_cert:
self._static_args['client_cert'] = self.client_cert
self._static_args['client_key'] = self.client_key
if self.ca_certs:
self._static_args['ca_certs'] = self.ca_certs
if self.http_user:
self._static_args['auth_username'] = self.http_user
if self.http_pwd:
self._static_args['auth_password'] = self.http_pwd
def load_connection_args(self, **kwargs):
"""Merges the static args relative to the connection, with the given keyword arguments. If statics
have yet to be initialized, we'll do that here.
"""
if len(self._static_args) == 0:
self.init_static_args()
kwargs.update(self._static_args)
return kwargs
class Spawner(LoggingConfigurable):
"""Base class for spawning single-user notebook servers.
Subclass this, and override the following methods:
- load_state
- get_state
- start
- stop
- poll
"""
db = Any()
user = Any()
hub = Any()
authenticator = Any()
api_token = Unicode()
ip = Unicode('127.0.0.1',
help="The IP address (or hostname) the single-user server should listen on"
).tag(config=True)
start_timeout = Integer(60,
help="""Timeout (in seconds) before giving up on the spawner.
This is the timeout for start to return, not the timeout for the server to respond.
Callers of spawner.start will assume that startup has failed if it takes longer than this.
start should return when the server process is started and its location is known.
"""
).tag(config=True)
http_timeout = Integer(30,
help="""Timeout (in seconds) before giving up on a spawned HTTP server
Once a server has successfully been spawned, this is the amount of time
we wait before assuming that the server is unable to accept
connections.
"""
).tag(config=True)
poll_interval = Integer(30,
help="""Interval (in seconds) on which to poll the spawner."""
).tag(config=True)
_callbacks = List()
_poll_callback = Any()
debug = Bool(False,
help="Enable debug-logging of the single-user server"
).tag(config=True)
options_form = Unicode("", help="""
An HTML form for options a user can specify on launching their server.
The surrounding `<form>` element and the submit button are already provided.
For example:
Set your key:
<input name="key" val="default_key"></input>
<br>
Choose a letter:
<select name="letter" multiple="true">
<option value="A">The letter A</option>
<option value="B">The letter B</option>
</select>
""").tag(config=True)
def options_from_form(self, form_data):
"""Interpret HTTP form data
Form data will always arrive as a dict of lists of strings.
Override this function to understand single-values, numbers, etc.
This should coerce form data into the structure expected by self.user_options,
which must be a dict.
Instances will receive this data on self.user_options, after passing through this function,
prior to `Spawner.start`.
"""
return form_data
user_options = Dict(help="This is where form-specified options ultimately end up.")
env_keep = List([
'PATH',
'PYTHONPATH',
'CONDA_ROOT',
'CONDA_DEFAULT_ENV',
'VIRTUAL_ENV',
'LANG',
'LC_ALL',
],
help="Whitelist of environment variables for the subprocess to inherit"
).tag(config=True)
env = Dict(help="""Deprecated: use Spawner.get_env or Spawner.environment
- extend Spawner.get_env for adding required env in Spawner subclasses
- Spawner.environment for config-specified env
""")
environment = Dict(
help="""Environment variables to load for the Spawner.
Value could be a string or a callable. If it is a callable, it will
be called with one parameter, which will be the instance of the spawner
in use. It should quickly (without doing much blocking operations) return
a string that will be used as the value for the environment variable.
"""
).tag(config=True)
cmd = Command(['jupyterhub-singleuser'],
help="""The command used for starting notebooks."""
).tag(config=True)
args = List(Unicode(),
help="""Extra arguments to be passed to the single-user server"""
).tag(config=True)
notebook_dir = Unicode('',
help="""The notebook directory for the single-user server
`~` will be expanded to the user's home directory
`%U` will be expanded to the user's username
"""
).tag(config=True)
default_url = Unicode('',
help="""The default URL for the single-user server.
Can be used in conjunction with --notebook-dir=/ to enable
full filesystem traversal, while preserving user's homedir as
landing page for notebook
`%U` will be expanded to the user's username
"""
).tag(config=True)
disable_user_config = Bool(False,
help="""Disable per-user configuration of single-user servers.
This prevents any config in users' $HOME directories
from having an effect on their server.
"""
).tag(config=True)
def __init__(self, **kwargs):
super(Spawner, self).__init__(**kwargs)
if self.user.state:
self.load_state(self.user.state)
def load_state(self, state):
"""load state from the database
This is the extensible part of state
Override in a subclass if there is state to load.
Should call `super`.
See Also
--------
get_state, clear_state
"""
pass
def get_state(self):
"""store the state necessary for load_state
A black box of extra state for custom spawners.
Subclasses should call `super`.
Returns
-------
state: dict
a JSONable dict of state
"""
state = {}
return state
def clear_state(self):
"""clear any state that should be cleared when the process stops
State that should be preserved across server instances should not be cleared.
Subclasses should call super, to ensure that state is properly cleared.
"""
self.api_token = ''
def get_env(self):
"""Return the environment dict to use for the Spawner.
This applies things like `env_keep`, anything defined in `Spawner.environment`,
and adds the API token to the env.
Use this to access the env in Spawner.start to allow extension in subclasses.
"""
env = {}
if self.env:
warnings.warn("Spawner.env is deprecated, found %s" % self.env, DeprecationWarning)
env.update(self.env)
for key in self.env_keep:
if key in os.environ:
env[key] = os.environ[key]
# config overrides. If the value is a callable, it will be called with
# one parameter - the current spawner instance - and the return value
# will be assigned to the environment variable. This will be called at
# spawn time.
for key, value in self.environment.items():
if callable(value):
env[key] = value(self)
else:
env[key] = value
env['JPY_API_TOKEN'] = self.api_token
return env
def get_args(self):
"""Return the arguments to be passed after self.cmd"""
args = [
'--user=%s' % self.user.name,
'--port=%i' % self.user.server.port,
'--cookie-name=%s' % self.user.server.cookie_name,
'--base-url=%s' % self.user.server.base_url,
'--hub-host=%s' % self.hub.host,
'--hub-prefix=%s' % self.hub.server.base_url,
'--hub-api-url=%s' % self.hub.api_url,
]
if self.ip:
args.append('--ip=%s' % self.ip)
if self.notebook_dir:
self.notebook_dir = self.notebook_dir.replace("%U",self.user.name)
args.append('--notebook-dir=%s' % self.notebook_dir)
if self.default_url:
self.default_url = self.default_url.replace("%U",self.user.name)
args.append('--NotebookApp.default_url=%s' % self.default_url)
if self.debug:
args.append('--debug')
if self.disable_user_config:
args.append('--disable-user-config')
args.extend(self.args)
return args
@gen.coroutine
def start(self):
"""Start the single-user process"""
raise NotImplementedError("Override in subclass. Must be a Tornado gen.coroutine.")
@gen.coroutine
def stop(self, now=False):
"""Stop the single-user process"""
raise NotImplementedError("Override in subclass. Must be a Tornado gen.coroutine.")
@gen.coroutine
def poll(self):
"""Check if the single-user process is running
return None if it is, an exit status (0 if unknown) if it is not.
"""
raise NotImplementedError("Override in subclass. Must be a Tornado gen.coroutine.")
def add_poll_callback(self, callback, *args, **kwargs):
"""add a callback to fire when the subprocess stops
as noticed by periodic poll_and_notify()
"""
if args or kwargs:
cb = callback
callback = lambda : cb(*args, **kwargs)
self._callbacks.append(callback)
def stop_polling(self):
"""stop the periodic poll"""
if self._poll_callback:
self._poll_callback.stop()
self._poll_callback = None
def start_polling(self):
"""Start polling periodically
callbacks registered via `add_poll_callback` will fire
if/when the process stops.
Explicit termination via the stop method will not trigger the callbacks.
"""
if self.poll_interval <= 0:
self.log.debug("Not polling subprocess")
return
else:
self.log.debug("Polling subprocess every %is", self.poll_interval)
self.stop_polling()
self._poll_callback = PeriodicCallback(
self.poll_and_notify,
1e3 * self.poll_interval
)
self._poll_callback.start()
@gen.coroutine
def poll_and_notify(self):
"""Used as a callback to periodically poll the process,
and notify any watchers
"""
status = yield self.poll()
if status is None:
# still running, nothing to do here
return
self.stop_polling()
add_callback = IOLoop.current().add_callback
for callback in self._callbacks:
add_callback(callback)
death_interval = Float(0.1)
@gen.coroutine
def wait_for_death(self, timeout=10):
"""wait for the process to die, up to timeout seconds"""
loop = IOLoop.current()
for i in range(int(timeout / self.death_interval)):
status = yield self.poll()
if status is not None:
break
else:
yield gen.sleep(self.death_interval)
class KernelRestarterBase(LoggingConfigurable):
"""Monitor and autorestart a kernel."""
debug = Bool(
False,
config=True,
help="""Whether to include every poll event in debugging output.
Has to be set explicitly, because there will be *a lot* of output.
""")
time_to_dead = Float(3.0,
config=True,
help="""Kernel heartbeat interval in seconds.""")
restart_limit = Integer(
5,
config=True,
help=
"""The number of consecutive autorestarts before the kernel is presumed dead."""
)
_restarting = False
_restart_count = 0
def __init__(self, kernel_manager, kernel_type, kernel_finder=None, **kw):
super(KernelRestarterBase, self).__init__(**kw)
self.kernel_manager = kernel_manager
self.kernel_type = kernel_type
self.kernel_finder = kernel_finder or KernelFinder.from_entrypoints()
self.callbacks = dict(died=[], restarted=[], failed=[])
def start(self):
"""Start monitoring the kernel."""
raise NotImplementedError("Must be implemented in a subclass")
def stop(self):
"""Stop monitoring."""
raise NotImplementedError("Must be implemented in a subclass")
def add_callback(self, f, event):
"""
Register a callback to fire on a particular event.
Possible values for event:
'died': the monitored kernel has died
'restarted': a restart has been attempted (this does not necessarily mean that the new kernel is usable).
'failed': *restart_limit* attempts have failed in quick succession, and the restarter is giving up.
"""
self.callbacks[event].append(f)
def remove_callback(self, f, event):
"""Unregister a callback from a particular event
Possible values for *event* are the same as in :meth:`add_callback`.
"""
try:
self.callbacks[event].remove(f)
except ValueError:
pass
def _fire_callbacks(self, event, data):
"""fire our callbacks for a particular event"""
for callback in self.callbacks[event]:
try:
callback(data)
except Exception as e:
self.log.error("KernelRestarter: %s callback %r failed",
event,
callback,
exc_info=True)
async def do_restart(self, auto=False):
"""Called when the kernel has died"""
if auto and self._restarting:
self._restart_count += 1
else:
self._restart_count = 1
if self._restart_count >= self.restart_limit:
self.log.warning("KernelRestarter: restart failed")
self._fire_callbacks('failed', {
'restart_count': self._restart_count,
})
self._restarting = False
self._restart_count = 0
self.stop()
else:
cwd = getattr(self.kernel_manager, 'cwd', None) # :-/
self.log.info("KernelRestarter: starting new manager (%i/%i)",
self._restart_count, self.restart_limit)
await self.kernel_manager.cleanup()
conn_info, mgr = await self.kernel_finder.launch(
self.kernel_type, cwd)
self._fire_callbacks('restarted', {
'auto': auto,
'connection_info': conn_info,
'manager': mgr,
})
self.kernel_manager = mgr
self._restarting = True
async def poll(self):
if self.debug:
self.log.debug('Polling kernel...')
if not await self.kernel_manager.is_alive():
self._fire_callbacks('died', {})
await self.do_restart(auto=True)
else:
if self._restarting:
self.log.debug("KernelRestarter: restart apparently succeeded")
self._restarting = False
class ImageDataGraph(base.DOMWidget):
"""The backend python class for the custom ImageDataGraph widget.
This class declares and initializes all of the data that is synced
between the front- and back-ends of the widget code.
It also provides the majority of the mathematical calculations that run this widget."""
_view_name = Unicode("ImgDataGraphView").tag(sync=True)
_model_name = Unicode("ImgDataGraphModel").tag(sync=True)
_b64value = Unicode().tag(sync=True)
_graphb64 = Unicode().tag(sync=True)
_format = Unicode().tag(sync=True)
_nrows = Integer().tag(sync=True)
_ncols = Integer().tag(sync=True)
_offsetX1 = Float().tag(sync=True)
_offsetY1 = Float().tag(sync=True)
_offsetX2 = Float().tag(sync=True)
_offsetY2 = Float().tag(sync=True)
_img_min = Float().tag(sync=True)
_img_max = Float().tag(sync=True)
_graph_click = Integer(0).tag(sync=True)
_linepix_width = Float(1.0).tag(sync=True)
_num_bins = Integer(1).tag(sync=True)
width = Integer().tag(sync=True)
height = Integer().tag(sync=True)
def __init__(self, image, width, height, uformat="png"):
"""Constructor method for setting the necessary
member variables (including synced ones).
This function also calls the getimg_bytes() method
to create provide the image data to create the widget.
Parameters:
* image: an ImageFile object (see
https://github.com/ornlneutronimaging/iMars3D/blob/master/python/imars3d/ImageFile.py
for more details) that stores the data
for the image to be used in this widget.
* width: an integer that is used to set the width of the image and UI elements.
* height: an integer that is used to set the height of the image and UI elements.
* uformat: a string indicating the type of image
that the displayed image and graph will be.
By default, this is set to "png"."""
self.img = image
self.img_data = image.data.copy()
self.width = width
self.height = height
self._format = uformat
self._nrows, self._ncols = self.img_data.shape
self._img_min, self._img_max = int(np.min(self.img_data)), int(np.max(self.img_data))
self._b64value = self.getimg_bytes()
super(ImageDataGraph, self).__init__()
return
def getimg_bytes(self):
"""Encodes the image's data into Base64."""
img = ((self.img_data-self._img_min)/(self._img_max-self._img_min)*(2**8-1)).astype("uint8")
size = np.max(img.shape)
view_size = np.max((self.width, self.height))
if size > view_size:
downsample_ratio = view_size/size
import scipy.misc
img = scipy.misc.imresize(img, downsample_ratio)
else:
upsample_ratio = view_size/size
import scipy.misc
img = scipy.misc.imresize(img, upsample_ratio)
if sys.version_info < (3, 0):
from cStringIO import StringIO
f = StringIO()
else:
from io import BytesIO
f = BytesIO()
import PIL.Image, base64
PIL.Image.fromarray(img).save(f, self._format)
imgb64v = base64.b64encode(f.getvalue())
return imgb64v
#This function is called when the value of _graph_click changes
@observe("_graph_click")
def graph_data(self, change):
"""Determines whether the graph calculations should
include width or not and calls the appropriate function."""
if self._linepix_width == 1:
self._graphb64 = self.nowidth_graph()
else:
self._graphb64 = self.width_graph()
return
def nowidth_graph(self):
"""Collects the data for a line with no width.
Then, creates a matplotlib graph of the data,
and encodes the graph into Base64 for the JavaScript code to display."""
p1x_abs = self._offsetX1/self.width * self._ncols
p1y_abs = self._offsetY1/self.height * self._nrows
p2x_abs = self._offsetX2/self.width * self._ncols
p2y_abs = self._offsetY2/self.height * self._nrows
if p1x_abs > p2x_abs:
tempx = p2x_abs
tempy = p2y_abs
p2x_abs = p1x_abs
p2y_abs = p1y_abs
p1x_abs = tempx
p1y_abs = tempy
xcoords = []
ycoords = []
dists = []
vals = []
curr_x_abs = p1x_abs
curr_y_abs = p1y_abs
curr_x = int(curr_x_abs)
curr_y = int(curr_y_abs)
xcoords.append(curr_x)
ycoords.append(curr_y)
vals.append(self.img_data[curr_y, curr_x])
if p2y_abs == p1y_abs and p2x_abs != p1x_abs:
while curr_x_abs < p2x_abs:
curr_x_abs += 1
curr_x = int(curr_x_abs)
curr_y = int(curr_y_abs)
xcoords.append(curr_x)
ycoords.append(curr_y)
vals.append(self.img_data[curr_y, curr_x])
elif p2x_abs == p1x_abs and p2y_abs != p1y_abs:
while curr_y_abs < p2y_abs:
curr_y_abs += 1
curr_x = int(curr_x_abs)
curr_y = int(curr_y_abs)
xcoords.append(curr_x)
ycoords.append(curr_y)
vals.append(self.img_data[curr_y, curr_x])
else:
while curr_x_abs < p2x_abs:
slope = (p2y_abs - p1y_abs) / (p2x_abs - p1x_abs)
curr_x_abs += 1
curr_y_abs += slope
curr_x = int(curr_x_abs)
curr_y = int(curr_y_abs)
if curr_x_abs < p2x_abs:
xcoords.append(curr_x)
ycoords.append(curr_y)
vals.append(self.img_data[curr_y, curr_x])
curr_x = int(p2x_abs)
curr_y = int(p2y_abs)
xcoords.append(curr_x)
ycoords.append(curr_y)
vals.append(self.img_data[curr_x, curr_y])
for x, y in np.nditer([xcoords, ycoords]):
dist = np.sqrt(((x - xcoords[0])**2 + (y - ycoords[0])**2))
dists.append(dist)
plt.plot(dists, vals)
plt.xlim(np.min(dists) * 0.75, np.max(dists))
plt.ylim(np.min(vals) * 0.75, np.max(vals) * 1.25)
plt.xlabel("Distance from Initial Point")
plt.ylabel("Value")
graph = plt.gcf()
if sys.version_info < (3, 0):
from StringIO import StringIO
graphdata = StringIO()
else:
from io import BytesIO
graphdata = BytesIO()
graph.savefig(graphdata, format=self._format)
graphdata.seek(0)
import base64
gb64v = base64.b64encode(graphdata.read())
plt.clf()
return gb64v
def width_graph(self):
"""Creates the graph for a line with width.
First, it calculates the endpoints of the drawn line.
Then, depending on whether the line is horizontal,
vertical, or diagonal, it calls the corresponding function
to get the data needed for graphing. Finally, it creates
the matplotlib graph and encodes it into Base64."""
p1x_abs = self._offsetX1/self.width * self._ncols
p1y_abs = self._offsetY1/self.height * self._nrows
p2x_abs = self._offsetX2/self.width * self._ncols
p2y_abs = self._offsetY2/self.height * self._nrows
dists = []
vals = []
if p1y_abs == p2y_abs and p1x_abs != p2x_abs:
dists, vals, bar_width = self.get_data_horizontal(p1x_abs, p1y_abs, p2x_abs)
elif p1y_abs != p2y_abs and p1x_abs == p2x_abs:
dists, vals, bar_width = self.get_data_vertical(p1x_abs, p1y_abs, p2y_abs)
else:
dists, vals, bar_width = self.get_data_diagonal(p1x_abs, p1y_abs, p2x_abs, p2y_abs)
plt.bar(dists, vals, width=bar_width)
plt.xlabel("Distance from Initial Point")
plt.ylabel("Value")
graph = plt.gcf()
if sys.version_info < (3, 0):
from StringIO import StringIO
graphdata = StringIO()
else:
from io import BytesIO
graphdata = BytesIO()
graph.savefig(graphdata, format=self._format)
graphdata.seek(0)
import base64
gb64v = base64.b64encode(graphdata.read())
plt.clf()
return gb64v
def get_data_horizontal(self, x_init, y_init, x_fin):
"""Calculates the graphing data for a horizontal line with width.
Parameters:
* x_init: a float containing the exact mathematical
x coordinate of the first point of the line.
* y_init: a float containing the exact mathematical
y coordinate of the first point of the line.
* x_fin: a float containing the exact mathematical
x coordinate of the last point of the line."""
vals = []
x0 = x_init
x1 = x_fin
if x0 > x1:
tempx = x1
x1 = x0
x0 = tempx
wid = self._linepix_width//self.height * self._nrows
top = y_init - wid//2
if int(top) < 0:
top = 0
bottom = y_init + wid//2 + 1
if int(bottom) > self._nrows - 1:
bottom = self._nrows - 1
max_dist = np.sqrt((x1 - x0)**2)
bin_step = max_dist / self._num_bins
bins = [0]
curr_bin_max = 0
for i in range(self._num_bins):
curr_bin_max += bin_step
bins.append(curr_bin_max)
intensities = np.zeros(len(bins))
num_binvals = np.zeros(len(bins))
Y, X = np.mgrid[top:bottom, x0:(x1+1)]
for x, y in np.nditer([X, Y]):
for b in bins:
ind = bins.index(b)
if ind < len(bins) - 1:
if x >= b + x0 and x < bins[ind+1] + x0:
intensities[ind] = intensities[ind] + self.img_data[int(y), int(x)]
num_binvals[ind] = num_binvals[ind] + 1
break
for val, num in np.nditer([intensities, num_binvals]):
if num == 0:
vals.append(0)
else:
vals.append(val/num)
return bins, vals, bin_step
def get_data_vertical(self, x_init, y_init, y_fin):
"""Calculates the graphing data for a vertical line with width.
Parameters:
* x_init: a float containing the exact mathematical
x coordinate of the first point of the line.
* y_init: a float containing the exact mathematical
y coordinate of the first point of the line.
* y_fin: a float containing the exact mathematical
y coordinate of the last point of the line."""
vals = []
y0 = y_init
y1 = y_fin
if y0 > y1:
tempy = y1
y1 = y0
y0 = tempy
wid = self._linepix_width//self.width * self._ncols
left = x_init - wid//2
if int(left) < 0:
left = 0
right = x_init + wid//2 + 1
if int(right) > self._ncols - 1:
right = self._ncols - 1
max_dist = np.sqrt((y1 - y0)**2)
bin_step = max_dist / self._num_bins
bins = [0]
curr_bin_max = 0
for i in range(self._num_bins):
curr_bin_max += bin_step
bins.append(curr_bin_max)
intensities = np.zeros(len(bins))
num_binvals = np.zeros(len(bins))
Y, X = np.mgrid[y0:(y1+1), left:right]
for x, y in np.nditer([X, Y]):
for b in bins:
ind = bins.index(b)
if ind < len(bins) - 1:
if y >= b + y0 and y < bins[ind+1] + y0:
intensities[ind] = intensities[ind] + self.img_data[int(y), int(x)]
num_binvals[ind] = num_binvals[ind] + 1
break
for val, num in np.nditer([intensities, num_binvals]):
if num == 0:
vals.append(0)
else:
vals.append(val/num)
return bins, vals, bin_step
def get_data_diagonal(self, x_init, y_init, x_fin, y_fin):
"""Calculates the graphing data for a vertical line with width.
Parameters:
* x_init: a float containing the exact mathematical
x coordinate of the first point of the line.
* y_init: a float containing the exact mathematical
y coordinate of the first point of the line.
* x_fin: a float containing the exact mathematical
x coordinate of the last point of the line.
* y_fin: a float containing the exact mathematical
y coordinate of the last point of the line."""
bins = []
vals = []
x0 = x_init
x1 = x_fin
y0 = y_init
y1 = y_fin
if x0 > x1:
tempx = x1
tempy = y1
x1 = x0
y1 = y0
x0 = tempx
y0 = tempy
slope = (y1 - y0) / (x1 - x0)
angle = np.arctan(slope)
wid_x = abs((self._linepix_width * np.cos(angle))/self.width * self._ncols)
wid_y = abs((self._linepix_width * np.sin(angle))/self.height * self._nrows)
wid = np.sqrt((wid_x)**2 + (wid_y)**2)
left = x0 - (wid_x // 2)
right = x1 + (wid_x // 2) + 1
if slope > 0:
bottom = y0 - (wid_y // 2)
top = y1 + (wid_y // 2) + 1
else:
bottom = y1 - (wid_y // 2)
top = y0 + (wid_y // 2)
if int(bottom) < 0:
bottom = 0
if int(top) > self._nrows - 1:
top = self._nrows - 1
if int(left) < 0:
left = 0
if int(right) > self._ncols - 1:
right = self._ncols - 1
Y, X = np.mgrid[bottom:top, left:right]
h_x = X - x0
h_y = Y - y0
norm_x = (y0 - y1) / np.sqrt((y0 - y1)**2 + (x1 - x0)**2)
norm_y = (x1 - x0) / np.sqrt((y0 - y1)**2 + (x1 - x0)**2)
e_x = (x1 - x0) / np.sqrt((x1 - x0)**2 + (y1 - y0)**2)
e_y = (y1 - y0) / np.sqrt((x1 - x0)**2 + (y1 - y0)**2)
dist = h_x*norm_x + h_y*norm_y
pos = h_x*e_x + h_y*e_y
max_dist = np.sqrt((x1 - x0)**2 + (y1 - y0)**2)
bin_step = max_dist / self._num_bins
curr_bin_max = 0
bin_borders = [0]
for i in range(self._num_bins):
curr_bin_max += bin_step
bin_borders.append(curr_bin_max)
intensities = np.zeros(len(bin_borders))
num_binvals = np.zeros(len(bin_borders))
for x, y, d, p in np.nditer([X, Y, dist, pos]):
if d <= wid / 2:
if p < 0 or p > max_dist:
continue
else:
for b in bin_borders:
ind = bin_borders.index(b)
if ind < len(bin_borders) - 1:
if p >= b and p < bin_borders[ind + 1]:
intensities[ind] = intensities[ind] + self.img_data[int(y), int(x)]
num_binvals[ind] = num_binvals[ind] + 1
break
for i, n in np.nditer([intensities, num_binvals]):
ind = np.where(intensities == i)
if n == 0:
vals.append(0)
else:
vals.append(i/n)
bins = bin_borders
return bins, vals, bin_step
