class ClusterManager(LoggingConfigurable):
profiles = Dict()
delay = CFloat(1., config=True,
help="delay (in s) between starting the controller and the engines")
loop = Instance('zmq.eventloop.ioloop.IOLoop')
def _loop_default(self):
from zmq.eventloop.ioloop import IOLoop
return IOLoop.instance()
def build_launchers(self, profile_dir):
starter = DummyIPClusterStart(log=self.log)
starter.initialize(['--profile-dir', profile_dir])
cl = starter.controller_launcher
esl = starter.engine_launcher
n = starter.n
return cl, esl, n
def get_profile_dir(self, name, path):
p = ProfileDir.find_profile_dir_by_name(path,name=name)
return p.location
def update_profiles(self):
"""List all profiles in the ipython_dir and cwd.
"""
for path in [get_ipython_dir(), os.getcwdu()]:
for profile in list_profiles_in(path):
pd = self.get_profile_dir(profile, path)
if profile not in self.profiles:
self.log.debug("Adding cluster profile '%s'" % profile)
self.profiles[profile] = {
'profile': profile,
'profile_dir': pd,
'status': 'stopped'
}
def list_profiles(self):
self.update_profiles()
result = [self.profile_info(p) for p in sorted(self.profiles.keys())]
return result
def check_profile(self, profile):
if profile not in self.profiles:
raise web.HTTPError(404, u'profile not found')
def profile_info(self, profile):
self.check_profile(profile)
result = {}
data = self.profiles.get(profile)
result['profile'] = profile
result['profile_dir'] = data['profile_dir']
result['status'] = data['status']
if 'n' in data:
result['n'] = data['n']
return result
def start_cluster(self, profile, n=None):
"""Start a cluster for a given profile."""
self.check_profile(profile)
data = self.profiles[profile]
if data['status'] == 'running':
raise web.HTTPError(409, u'cluster already running')
cl, esl, default_n = self.build_launchers(data['profile_dir'])
n = n if n is not None else default_n
def clean_data():
data.pop('controller_launcher',None)
data.pop('engine_set_launcher',None)
data.pop('n',None)
data['status'] = 'stopped'
def engines_stopped(r):
self.log.debug('Engines stopped')
if cl.running:
cl.stop()
clean_data()
esl.on_stop(engines_stopped)
def controller_stopped(r):
self.log.debug('Controller stopped')
if esl.running:
esl.stop()
clean_data()
cl.on_stop(controller_stopped)
dc = ioloop.DelayedCallback(lambda: cl.start(), 0, self.loop)
dc.start()
dc = ioloop.DelayedCallback(lambda: esl.start(n), 1000*self.delay, self.loop)
dc.start()
self.log.debug('Cluster started')
data['controller_launcher'] = cl
data['engine_set_launcher'] = esl
data['n'] = n
data['status'] = 'running'
return self.profile_info(profile)
def stop_cluster(self, profile):
"""Stop a cluster for a given profile."""
self.check_profile(profile)
data = self.profiles[profile]
if data['status'] == 'stopped':
raise web.HTTPError(409, u'cluster not running')
data = self.profiles[profile]
cl = data['controller_launcher']
esl = data['engine_set_launcher']
if cl.running:
cl.stop()
if esl.running:
esl.stop()
# Return a temp info dict, the real one is updated in the on_stop
# logic above.
result = {
'profile': data['profile'],
'profile_dir': data['profile_dir'],
'status': 'stopped'
}
return result
def stop_all_clusters(self):
for p in self.profiles.keys():
self.stop_cluster(p)
class _BoundedFloatRange(_FloatRange):
step = CFloat(
1.0,
help="Minimum step that the value can take (ignored by some views)",
sync=True)
max = CFloat(100.0, help="Max value", sync=True)
min = CFloat(0.0, help="Min value", sync=True)
def __init__(self, *pargs, **kwargs):
any_value_given = 'value' in kwargs or 'upper' in kwargs or 'lower' in kwargs
_FloatRange.__init__(self, *pargs, **kwargs)
# ensure a minimal amount of sanity
if self.min > self.max:
raise ValueError("min must be <= max")
if any_value_given:
# if a value was given, clamp it within (min, max)
self._validate("value", None, self.value)
else:
# otherwise, set it to 25-75% to avoid the handles overlapping
self.value = (0.75 * self.min + 0.25 * self.max,
0.25 * self.min + 0.75 * self.max)
# callback already set for 'value', 'lower', 'upper'
self.on_trait_change(self._validate, ['min', 'max'])
def _validate(self, name, old, new):
if name == "min":
if new > self.max:
raise ValueError("setting min > max")
self.min = new
elif name == "max":
if new < self.min:
raise ValueError("setting max < min")
self.max = new
low, high = self.value
if name == "value":
low, high = min(new), max(new)
elif name == "upper":
if new < self.lower:
raise ValueError("setting upper < lower")
high = new
elif name == "lower":
if new > self.upper:
raise ValueError("setting lower > upper")
low = new
low = max(self.min, min(low, self.max))
high = min(self.max, max(high, self.min))
# determine the order in which we should update the
# lower, upper traits to avoid a temporary inverted overlap
lower_first = high < self.lower
self.value = (low, high)
if lower_first:
self.lower = low
self.upper = high
else:
self.upper = high
self.lower = low
class IPClusterStart(IPClusterEngines):
name = u'ipcluster'
description = start_help
examples = _start_examples
default_log_level = logging.INFO
auto_create = Bool(True, config=True,
help="whether to create the profile_dir if it doesn't exist")
classes = List()
def _classes_default(self,):
from IPython.parallel.apps import launcher
return [ProfileDir] + [IPClusterEngines] + launcher.all_launchers
clean_logs = Bool(True, config=True,
help="whether to cleanup old logs before starting")
delay = CFloat(1., config=True,
help="delay (in s) between starting the controller and the engines")
controller_launcher = Any(config=True, help="Deprecated, use controller_launcher_class")
def _controller_launcher_changed(self, name, old, new):
if isinstance(new, basestring):
# old 0.11-style config
self.log.warn("WARNING: %s.controller_launcher is deprecated as of 0.12,"
" use controller_launcher_class" % self.__class__.__name__)
self.controller_launcher_class = new
controller_launcher_class = DottedObjectName('LocalControllerLauncher',
config=True,
help="""The class for launching a Controller. Change this value if you want
your controller to also be launched by a batch system, such as PBS,SGE,MPIExec,etc.
Each launcher class has its own set of configuration options, for making sure
it will work in your environment.
Examples include:
LocalControllerLauncher : start engines locally as subprocesses
MPIExecControllerLauncher : use mpiexec to launch engines in an MPI universe
PBSControllerLauncher : use PBS (qsub) to submit engines to a batch queue
SGEControllerLauncher : use SGE (qsub) to submit engines to a batch queue
LSFControllerLauncher : use LSF (bsub) to submit engines to a batch queue
SSHControllerLauncher : use SSH to start the controller
WindowsHPCControllerLauncher : use Windows HPC
If you are using one of IPython's builtin launchers, you can specify just the
prefix, e.g:
c.IPClusterStart.controller_launcher_class = 'SSH'
or:
ipcluster start --controller 'MPIExec'
"""
)
reset = Bool(False, config=True,
help="Whether to reset config files as part of '--create'."
)
# flags = Dict(flags)
aliases = Dict(start_aliases)
def init_launchers(self):
self.controller_launcher = self.build_launcher(self.controller_launcher_class, 'Controller')
self.engine_launcher = self.build_launcher(self.engine_launcher_class, 'EngineSet')
self.controller_launcher.on_stop(self.stop_launchers)
def engines_stopped(self, r):
"""prevent parent.engines_stopped from stopping everything on engine shutdown"""
pass
def start_controller(self):
self.controller_launcher.start()
def stop_controller(self):
# self.log.info("In stop_controller")
if self.controller_launcher and self.controller_launcher.running:
return self.controller_launcher.stop()
def stop_launchers(self, r=None):
if not self._stopping:
self.stop_controller()
super(IPClusterStart, self).stop_launchers()
def start(self):
"""Start the app for the start subcommand."""
# First see if the cluster is already running
try:
pid = self.get_pid_from_file()
except PIDFileError:
pass
else:
if self.check_pid(pid):
self.log.critical(
'Cluster is already running with [pid=%s]. '
'use "ipcluster stop" to stop the cluster.' % pid
)
# Here I exit with a unusual exit status that other processes
# can watch for to learn how I existed.
self.exit(ALREADY_STARTED)
else:
self.remove_pid_file()
# Now log and daemonize
self.log.info(
'Starting ipcluster with [daemon=%r]' % self.daemonize
)
# TODO: Get daemonize working on Windows or as a Windows Server.
if self.daemonize:
if os.name=='posix':
daemonize()
dc = ioloop.DelayedCallback(self.start_controller, 0, self.loop)
dc.start()
dc = ioloop.DelayedCallback(self.start_engines, 1000*self.delay, self.loop)
dc.start()
# Now write the new pid file AFTER our new forked pid is active.
self.write_pid_file()
try:
self.loop.start()
except KeyboardInterrupt:
pass
except zmq.ZMQError as e:
if e.errno == errno.EINTR:
pass
else:
raise
finally:
self.remove_pid_file()
class LoadBalancedView(View):
"""An load-balancing View that only executes via the Task scheduler.
Load-balanced views can be created with the client's `view` method:
>>> v = client.load_balanced_view()
or targets can be specified, to restrict the potential destinations:
>>> v = client.client.load_balanced_view([1,3])
which would restrict loadbalancing to between engines 1 and 3.
"""
follow = Any()
after = Any()
timeout = CFloat()
retries = Integer(0)
_task_scheme = Any()
_flag_names = List(
['targets', 'block', 'track', 'follow', 'after', 'timeout', 'retries'])
def __init__(self, client=None, socket=None, **flags):
super(LoadBalancedView, self).__init__(client=client,
socket=socket,
**flags)
self._task_scheme = client._task_scheme
def _validate_dependency(self, dep):
"""validate a dependency.
For use in `set_flags`.
"""
if dep is None or isinstance(dep, string_types +
(AsyncResult, Dependency)):
return True
elif isinstance(dep, (list, set, tuple)):
for d in dep:
if not isinstance(d, string_types + (AsyncResult, )):
return False
elif isinstance(dep, dict):
if set(dep.keys()) != set(Dependency().as_dict().keys()):
return False
if not isinstance(dep['msg_ids'], list):
return False
for d in dep['msg_ids']:
if not isinstance(d, string_types):
return False
else:
return False
return True
def _render_dependency(self, dep):
"""helper for building jsonable dependencies from various input forms."""
if isinstance(dep, Dependency):
return dep.as_dict()
elif isinstance(dep, AsyncResult):
return dep.msg_ids
elif dep is None:
return []
else:
# pass to Dependency constructor
return list(Dependency(dep))
def set_flags(self, **kwargs):
"""set my attribute flags by keyword.
A View is a wrapper for the Client's apply method, but with attributes
that specify keyword arguments, those attributes can be set by keyword
argument with this method.
Parameters
----------
block : bool
whether to wait for results
track : bool
whether to create a MessageTracker to allow the user to
safely edit after arrays and buffers during non-copying
sends.
after : Dependency or collection of msg_ids
Only for load-balanced execution (targets=None)
Specify a list of msg_ids as a time-based dependency.
This job will only be run *after* the dependencies
have been met.
follow : Dependency or collection of msg_ids
Only for load-balanced execution (targets=None)
Specify a list of msg_ids as a location-based dependency.
This job will only be run on an engine where this dependency
is met.
timeout : float/int or None
Only for load-balanced execution (targets=None)
Specify an amount of time (in seconds) for the scheduler to
wait for dependencies to be met before failing with a
DependencyTimeout.
retries : int
Number of times a task will be retried on failure.
"""
super(LoadBalancedView, self).set_flags(**kwargs)
for name in ('follow', 'after'):
if name in kwargs:
value = kwargs[name]
if self._validate_dependency(value):
setattr(self, name, value)
else:
raise ValueError("Invalid dependency: %r" % value)
if 'timeout' in kwargs:
t = kwargs['timeout']
if not isinstance(t, (int, float, type(None))):
if (not PY3) and (not isinstance(t, long)):
raise TypeError("Invalid type for timeout: %r" % type(t))
if t is not None:
if t < 0:
raise ValueError("Invalid timeout: %s" % t)
self.timeout = t
@sync_results
@save_ids
def _really_apply(self,
f,
args=None,
kwargs=None,
block=None,
track=None,
after=None,
follow=None,
timeout=None,
targets=None,
retries=None):
"""calls f(*args, **kwargs) on a remote engine, returning the result.
This method temporarily sets all of `apply`'s flags for a single call.
Parameters
----------
f : callable
args : list [default: empty]
kwargs : dict [default: empty]
block : bool [default: self.block]
whether to block
track : bool [default: self.track]
whether to ask zmq to track the message, for safe non-copying sends
!!!!!! TODO: THE REST HERE !!!!
Returns
-------
if self.block is False:
returns AsyncResult
else:
returns actual result of f(*args, **kwargs) on the engine(s)
This will be a list of self.targets is also a list (even length 1), or
the single result if self.targets is an integer engine id
"""
# validate whether we can run
if self._socket.closed:
msg = "Task farming is disabled"
if self._task_scheme == 'pure':
msg += " because the pure ZMQ scheduler cannot handle"
msg += " disappearing engines."
raise RuntimeError(msg)
if self._task_scheme == 'pure':
# pure zmq scheme doesn't support extra features
msg = "Pure ZMQ scheduler doesn't support the following flags:"
"follow, after, retries, targets, timeout"
if (follow or after or retries or targets or timeout):
# hard fail on Scheduler flags
raise RuntimeError(msg)
if isinstance(f, dependent):
# soft warn on functional dependencies
warnings.warn(msg, RuntimeWarning)
# build args
args = [] if args is None else args
kwargs = {} if kwargs is None else kwargs
block = self.block if block is None else block
track = self.track if track is None else track
after = self.after if after is None else after
retries = self.retries if retries is None else retries
follow = self.follow if follow is None else follow
timeout = self.timeout if timeout is None else timeout
targets = self.targets if targets is None else targets
if not isinstance(retries, int):
raise TypeError('retries must be int, not %r' % type(retries))
if targets is None:
idents = []
else:
idents = self.client._build_targets(targets)[0]
# ensure *not* bytes
idents = [ident.decode() for ident in idents]
after = self._render_dependency(after)
follow = self._render_dependency(follow)
metadata = dict(after=after,
follow=follow,
timeout=timeout,
targets=idents,
retries=retries)
msg = self.client.send_apply_request(self._socket,
f,
args,
kwargs,
track=track,
metadata=metadata)
tracker = None if track is False else msg['tracker']
ar = AsyncResult(
self.client,
msg['header']['msg_id'],
fname=getname(f),
targets=None,
tracker=tracker,
owner=True,
)
if block:
try:
return ar.get()
except KeyboardInterrupt:
pass
return ar
@sync_results
@save_ids
def map(self, f, *sequences, **kwargs):
"""``view.map(f, *sequences, block=self.block, chunksize=1, ordered=True)`` => list|AsyncMapResult
Parallel version of builtin `map`, load-balanced by this View.
`block`, and `chunksize` can be specified by keyword only.
Each `chunksize` elements will be a separate task, and will be
load-balanced. This lets individual elements be available for iteration
as soon as they arrive.
Parameters
----------
f : callable
function to be mapped
*sequences: one or more sequences of matching length
the sequences to be distributed and passed to `f`
block : bool [default self.block]
whether to wait for the result or not
track : bool
whether to create a MessageTracker to allow the user to
safely edit after arrays and buffers during non-copying
sends.
chunksize : int [default 1]
how many elements should be in each task.
ordered : bool [default True]
Whether the results should be gathered as they arrive, or enforce
the order of submission.
Only applies when iterating through AsyncMapResult as results arrive.
Has no effect when block=True.
Returns
-------
if block=False
An :class:`~IPython.parallel.client.asyncresult.AsyncMapResult` instance.
An object like AsyncResult, but which reassembles the sequence of results
into a single list. AsyncMapResults can be iterated through before all
results are complete.
else
A list, the result of ``map(f,*sequences)``
"""
# default
block = kwargs.get('block', self.block)
chunksize = kwargs.get('chunksize', 1)
ordered = kwargs.get('ordered', True)
keyset = set(kwargs.keys())
extra_keys = keyset.difference_update(set(['block', 'chunksize']))
if extra_keys:
raise TypeError("Invalid kwargs: %s" % list(extra_keys))
assert len(sequences) > 0, "must have some sequences to map onto!"
pf = ParallelFunction(self,
f,
block=block,
chunksize=chunksize,
ordered=ordered)
return pf.map(*sequences)
class IntensityLight(Light):
_view_name = Unicode('PositionLight', sync=True)
intensity = CFloat(1, sync=True)
class HeartMonitor(LoggingConfigurable):
"""A basic HeartMonitor class
pingstream: a PUB stream
pongstream: an XREP stream
period: the period of the heartbeat in milliseconds"""
period = CFloat(
1000,
config=True,
help='The frequency at which the Hub pings the engines for heartbeats '
'(in ms)',
)
pingstream = Instance('zmq.eventloop.zmqstream.ZMQStream')
pongstream = Instance('zmq.eventloop.zmqstream.ZMQStream')
loop = Instance('zmq.eventloop.ioloop.IOLoop')
def _loop_default(self):
return ioloop.IOLoop.instance()
# not settable:
hearts = Set()
responses = Set()
on_probation = Set()
last_ping = CFloat(0)
_new_handlers = Set()
_failure_handlers = Set()
lifetime = CFloat(0)
tic = CFloat(0)
def __init__(self, **kwargs):
super(HeartMonitor, self).__init__(**kwargs)
self.pongstream.on_recv(self.handle_pong)
def start(self):
self.tic = time.time()
self.caller = ioloop.PeriodicCallback(self.beat, self.period,
self.loop)
self.caller.start()
def add_new_heart_handler(self, handler):
"""add a new handler for new hearts"""
self.log.debug("heartbeat::new_heart_handler: %s", handler)
self._new_handlers.add(handler)
def add_heart_failure_handler(self, handler):
"""add a new handler for heart failure"""
self.log.debug("heartbeat::new heart failure handler: %s", handler)
self._failure_handlers.add(handler)
def beat(self):
self.pongstream.flush()
self.last_ping = self.lifetime
toc = time.time()
self.lifetime += toc - self.tic
self.tic = toc
self.log.debug("heartbeat::sending %s", self.lifetime)
goodhearts = self.hearts.intersection(self.responses)
missed_beats = self.hearts.difference(goodhearts)
heartfailures = self.on_probation.intersection(missed_beats)
newhearts = self.responses.difference(goodhearts)
map(self.handle_new_heart, newhearts)
map(self.handle_heart_failure, heartfailures)
self.on_probation = missed_beats.intersection(self.hearts)
self.responses = set()
# print self.on_probation, self.hearts
# self.log.debug("heartbeat::beat %.3f, %i beating hearts", self.lifetime, len(self.hearts))
self.pingstream.send(asbytes(str(self.lifetime)))
def handle_new_heart(self, heart):
if self._new_handlers:
for handler in self._new_handlers:
handler(heart)
else:
self.log.info("heartbeat::yay, got new heart %s!", heart)
self.hearts.add(heart)
def handle_heart_failure(self, heart):
if self._failure_handlers:
for handler in self._failure_handlers:
try:
handler(heart)
except Exception as e:
self.log.error("heartbeat::Bad Handler! %s",
handler,
exc_info=True)
pass
else:
self.log.info("heartbeat::Heart %s failed :(", heart)
self.hearts.remove(heart)
def handle_pong(self, msg):
"a heart just beat"
current = asbytes(str(self.lifetime))
last = asbytes(str(self.last_ping))
if msg[1] == current:
delta = time.time() - self.tic
# self.log.debug("heartbeat::heart %r took %.2f ms to respond"%(msg[0], 1000*delta))
self.responses.add(msg[0])
elif msg[1] == last:
delta = time.time() - self.tic + (self.lifetime - self.last_ping)
self.log.warn(
"heartbeat::heart %r missed a beat, and took %.2f ms to respond",
msg[0], 1000 * delta)
self.responses.add(msg[0])
else:
self.log.warn(
"heartbeat::got bad heartbeat (possibly old?): %s (current=%.3f)",
msg[1], self.lifetime)
class EngineFactory(RegistrationFactory):
"""IPython engine"""
# configurables:
user_ns=Dict(config=True)
out_stream_factory=Type('IPython.zmq.iostream.OutStream', config=True)
display_hook_factory=Type('IPython.zmq.displayhook.DisplayHook', config=True)
location=Str(config=True)
timeout=CFloat(2,config=True)
# not configurable:
id=Int(allow_none=True)
registrar=Instance('zmq.eventloop.zmqstream.ZMQStream')
kernel=Instance(Kernel)
def __init__(self, **kwargs):
super(EngineFactory, self).__init__(**kwargs)
ctx = self.context
reg = ctx.socket(zmq.XREQ)
reg.setsockopt(zmq.IDENTITY, self.ident)
reg.connect(self.url)
self.registrar = zmqstream.ZMQStream(reg, self.loop)
def register(self):
"""send the registration_request"""
self.log.info("registering")
content = dict(queue=self.ident, heartbeat=self.ident, control=self.ident)
self.registrar.on_recv(self.complete_registration)
# print (self.session.key)
self.session.send(self.registrar, "registration_request",content=content)
def complete_registration(self, msg):
# print msg
self._abort_dc.stop()
ctx = self.context
loop = self.loop
identity = self.ident
idents,msg = self.session.feed_identities(msg)
msg = Message(self.session.unpack_message(msg))
if msg.content.status == 'ok':
self.id = int(msg.content.id)
# create Shell Streams (MUX, Task, etc.):
queue_addr = msg.content.mux
shell_addrs = [ str(queue_addr) ]
task_addr = msg.content.task
if task_addr:
shell_addrs.append(str(task_addr))
# Uncomment this to go back to two-socket model
# shell_streams = []
# for addr in shell_addrs:
# stream = zmqstream.ZMQStream(ctx.socket(zmq.XREP), loop)
# stream.setsockopt(zmq.IDENTITY, identity)
# stream.connect(disambiguate_url(addr, self.location))
# shell_streams.append(stream)
# Now use only one shell stream for mux and tasks
stream = zmqstream.ZMQStream(ctx.socket(zmq.XREP), loop)
stream.setsockopt(zmq.IDENTITY, identity)
shell_streams = [stream]
for addr in shell_addrs:
stream.connect(disambiguate_url(addr, self.location))
# end single stream-socket
# control stream:
control_addr = str(msg.content.control)
control_stream = zmqstream.ZMQStream(ctx.socket(zmq.XREP), loop)
control_stream.setsockopt(zmq.IDENTITY, identity)
control_stream.connect(disambiguate_url(control_addr, self.location))
# create iopub stream:
iopub_addr = msg.content.iopub
iopub_stream = zmqstream.ZMQStream(ctx.socket(zmq.PUB), loop)
iopub_stream.setsockopt(zmq.IDENTITY, identity)
iopub_stream.connect(disambiguate_url(iopub_addr, self.location))
# launch heartbeat
hb_addrs = msg.content.heartbeat
# print (hb_addrs)
# # Redirect input streams and set a display hook.
if self.out_stream_factory:
sys.stdout = self.out_stream_factory(self.session, iopub_stream, u'stdout')
sys.stdout.topic = 'engine.%i.stdout'%self.id
sys.stderr = self.out_stream_factory(self.session, iopub_stream, u'stderr')
sys.stderr.topic = 'engine.%i.stderr'%self.id
if self.display_hook_factory:
sys.displayhook = self.display_hook_factory(self.session, iopub_stream)
sys.displayhook.topic = 'engine.%i.pyout'%self.id
self.kernel = Kernel(config=self.config, int_id=self.id, ident=self.ident, session=self.session,
control_stream=control_stream, shell_streams=shell_streams, iopub_stream=iopub_stream,
loop=loop, user_ns = self.user_ns, logname=self.log.name)
self.kernel.start()
hb_addrs = [ disambiguate_url(addr, self.location) for addr in hb_addrs ]
heart = Heart(*map(str, hb_addrs), heart_id=identity)
# ioloop.DelayedCallback(heart.start, 1000, self.loop).start()
heart.start()
else:
self.log.fatal("Registration Failed: %s"%msg)
raise Exception("Registration Failed: %s"%msg)
self.log.info("Completed registration with id %i"%self.id)
def abort(self):
self.log.fatal("Registration timed out")
self.session.send(self.registrar, "unregistration_request", content=dict(id=self.id))
time.sleep(1)
sys.exit(255)
def start(self):
dc = ioloop.DelayedCallback(self.register, 0, self.loop)
dc.start()
self._abort_dc = ioloop.DelayedCallback(self.abort, self.timeout*1000, self.loop)
self._abort_dc.start()
class CircleGeometry(Geometry):
_view_name = Unicode('CircleGeometryView', sync=True)
radius = CFloat(1, sync=True)
segments = CFloat(8, sync=True)
thetaStart = CFloat(0, sync=True)
thetaLength = CFloat(2*math.pi, sync=True)
class ForceDirectedGraph(widgets.DOMWidget):
_view_module = Unicode('nbextensions/d3networkx/widget', sync=True)
_view_name = Unicode('D3ForceDirectedGraphView', sync=True)
width = CInt(400, sync=True)
height = CInt(300, sync=True)
charge = CFloat(270., sync=True)
distance = CInt(30., sync=True)
strength = CInt(0.3, sync=True)
def __init__(self, eventful_graph, *pargs, **kwargs):
widgets.DOMWidget.__init__(self, *pargs, **kwargs)
self._eventful_graph = eventful_graph
self._send_dict_changes(eventful_graph.graph, 'graph')
self._send_dict_changes(eventful_graph.node, 'node')
self._send_dict_changes(eventful_graph.adj, 'adj')
def _ipython_display_(self, *pargs, **kwargs):
# Show the widget, then send the current state
widgets.DOMWidget._ipython_display_(self, *pargs, **kwargs)
for (key, value) in self._eventful_graph.graph.items():
self.send({
'dict': 'graph',
'action': 'add',
'key': key,
'value': value
})
for (key, value) in self._eventful_graph.node.items():
self.send({
'dict': 'node',
'action': 'add',
'key': key,
'value': value
})
for (key, value) in self._eventful_graph.adj.items():
self.send({
'dict': 'adj',
'action': 'add',
'key': key,
'value': value
})
def _send_dict_changes(self, eventful_dict, dict_name):
def key_add(key, value):
self.send({
'dict': dict_name,
'action': 'add',
'key': key,
'value': value
})
def key_set(key, value):
self.send({
'dict': dict_name,
'action': 'set',
'key': key,
'value': value
})
def key_del(key):
self.send({'dict': dict_name, 'action': 'del', 'key': key})
eventful_dict.on_add(key_add)
eventful_dict.on_set(key_set)
eventful_dict.on_del(key_del)
class EngineFactory(RegistrationFactory):
"""IPython engine"""
# configurables:
out_stream_factory = Type('IPython.zmq.iostream.OutStream',
config=True,
help="""The OutStream for handling stdout/err.
Typically 'IPython.zmq.iostream.OutStream'""")
display_hook_factory = Type('IPython.zmq.displayhook.ZMQDisplayHook',
config=True,
help="""The class for handling displayhook.
Typically 'IPython.zmq.displayhook.ZMQDisplayHook'""")
location = Unicode(
config=True,
help="""The location (an IP address) of the controller. This is
used for disambiguating URLs, to determine whether
loopback should be used to connect or the public address.""")
timeout = CFloat(
2,
config=True,
help="""The time (in seconds) to wait for the Controller to respond
to registration requests before giving up.""")
sshserver = Unicode(
config=True,
help=
"""The SSH server to use for tunneling connections to the Controller."""
)
sshkey = Unicode(
config=True,
help=
"""The SSH private key file to use when tunneling connections to the Controller."""
)
paramiko = Bool(
sys.platform == 'win32',
config=True,
help="""Whether to use paramiko instead of openssh for tunnels.""")
# not configurable:
user_ns = Dict()
id = Integer(allow_none=True)
registrar = Instance('zmq.eventloop.zmqstream.ZMQStream')
kernel = Instance(Kernel)
bident = CBytes()
ident = Unicode()
def _ident_changed(self, name, old, new):
self.bident = cast_bytes(new)
using_ssh = Bool(False)
def __init__(self, **kwargs):
super(EngineFactory, self).__init__(**kwargs)
self.ident = self.session.session
def init_connector(self):
"""construct connection function, which handles tunnels."""
self.using_ssh = bool(self.sshkey or self.sshserver)
if self.sshkey and not self.sshserver:
# We are using ssh directly to the controller, tunneling localhost to localhost
self.sshserver = self.url.split('://')[1].split(':')[0]
if self.using_ssh:
if tunnel.try_passwordless_ssh(self.sshserver, self.sshkey,
self.paramiko):
password = False
else:
password = getpass("SSH Password for %s: " % self.sshserver)
else:
password = False
def connect(s, url):
url = disambiguate_url(url, self.location)
if self.using_ssh:
self.log.debug("Tunneling connection to %s via %s" %
(url, self.sshserver))
return tunnel.tunnel_connection(
s,
url,
self.sshserver,
keyfile=self.sshkey,
paramiko=self.paramiko,
password=password,
)
else:
return s.connect(url)
def maybe_tunnel(url):
"""like connect, but don't complete the connection (for use by heartbeat)"""
url = disambiguate_url(url, self.location)
if self.using_ssh:
self.log.debug("Tunneling connection to %s via %s" %
(url, self.sshserver))
url, tunnelobj = tunnel.open_tunnel(
url,
self.sshserver,
keyfile=self.sshkey,
paramiko=self.paramiko,
password=password,
)
return url
return connect, maybe_tunnel
def register(self):
"""send the registration_request"""
self.log.info("Registering with controller at %s" % self.url)
ctx = self.context
connect, maybe_tunnel = self.init_connector()
reg = ctx.socket(zmq.DEALER)
reg.setsockopt(zmq.IDENTITY, self.bident)
connect(reg, self.url)
self.registrar = zmqstream.ZMQStream(reg, self.loop)
content = dict(queue=self.ident,
heartbeat=self.ident,
control=self.ident)
self.registrar.on_recv(
lambda msg: self.complete_registration(msg, connect, maybe_tunnel))
# print (self.session.key)
self.session.send(self.registrar,
"registration_request",
content=content)
def complete_registration(self, msg, connect, maybe_tunnel):
# print msg
self._abort_dc.stop()
ctx = self.context
loop = self.loop
identity = self.bident
idents, msg = self.session.feed_identities(msg)
msg = Message(self.session.unserialize(msg))
if msg.content.status == 'ok':
self.id = int(msg.content.id)
# launch heartbeat
hb_addrs = msg.content.heartbeat
# possibly forward hb ports with tunnels
hb_addrs = [maybe_tunnel(addr) for addr in hb_addrs]
heart = Heart(*list(map(str, hb_addrs)), heart_id=identity)
heart.start()
# create Shell Streams (MUX, Task, etc.):
queue_addr = msg.content.mux
shell_addrs = [str(queue_addr)]
task_addr = msg.content.task
if task_addr:
shell_addrs.append(str(task_addr))
# Uncomment this to go back to two-socket model
# shell_streams = []
# for addr in shell_addrs:
# stream = zmqstream.ZMQStream(ctx.socket(zmq.ROUTER), loop)
# stream.setsockopt(zmq.IDENTITY, identity)
# stream.connect(disambiguate_url(addr, self.location))
# shell_streams.append(stream)
# Now use only one shell stream for mux and tasks
stream = zmqstream.ZMQStream(ctx.socket(zmq.ROUTER), loop)
stream.setsockopt(zmq.IDENTITY, identity)
shell_streams = [stream]
for addr in shell_addrs:
connect(stream, addr)
# end single stream-socket
# control stream:
control_addr = str(msg.content.control)
control_stream = zmqstream.ZMQStream(ctx.socket(zmq.ROUTER), loop)
control_stream.setsockopt(zmq.IDENTITY, identity)
connect(control_stream, control_addr)
# create iopub stream:
iopub_addr = msg.content.iopub
iopub_socket = ctx.socket(zmq.PUB)
iopub_socket.setsockopt(zmq.IDENTITY, identity)
connect(iopub_socket, iopub_addr)
# disable history:
self.config.HistoryManager.hist_file = ':memory:'
# Redirect input streams and set a display hook.
if self.out_stream_factory:
sys.stdout = self.out_stream_factory(self.session,
iopub_socket, 'stdout')
sys.stdout.topic = cast_bytes('engine.%i.stdout' % self.id)
sys.stderr = self.out_stream_factory(self.session,
iopub_socket, 'stderr')
sys.stderr.topic = cast_bytes('engine.%i.stderr' % self.id)
if self.display_hook_factory:
sys.displayhook = self.display_hook_factory(
self.session, iopub_socket)
sys.displayhook.topic = cast_bytes('engine.%i.pyout' % self.id)
self.kernel = Kernel(config=self.config,
int_id=self.id,
ident=self.ident,
session=self.session,
control_stream=control_stream,
shell_streams=shell_streams,
iopub_socket=iopub_socket,
loop=loop,
user_ns=self.user_ns,
log=self.log)
self.kernel.shell.display_pub.topic = cast_bytes(
'engine.%i.displaypub' % self.id)
# FIXME: This is a hack until IPKernelApp and IPEngineApp can be fully merged
app = IPKernelApp(config=self.config,
shell=self.kernel.shell,
kernel=self.kernel,
log=self.log)
app.init_profile_dir()
app.init_code()
self.kernel.start()
else:
self.log.fatal("Registration Failed: %s" % msg)
raise Exception("Registration Failed: %s" % msg)
self.log.info("Completed registration with id %i" % self.id)
def abort(self):
self.log.fatal("Registration timed out after %.1f seconds" %
self.timeout)
if self.url.startswith('127.'):
self.log.fatal("""
If the controller and engines are not on the same machine,
you will have to instruct the controller to listen on an external IP (in ipcontroller_config.py):
c.HubFactory.ip='*' # for all interfaces, internal and external
c.HubFactory.ip='192.168.1.101' # or any interface that the engines can see
or tunnel connections via ssh.
""")
self.session.send(self.registrar,
"unregistration_request",
content=dict(id=self.id))
time.sleep(1)
sys.exit(255)
def start(self):
dc = ioloop.DelayedCallback(self.register, 0, self.loop)
dc.start()
self._abort_dc = ioloop.DelayedCallback(self.abort,
self.timeout * 1000, self.loop)
self._abort_dc.start()
class Material(Widget):
_view_name = Unicode('MaterialView', sync=True)
color = Any('yellow', sync=True)
opacity = CFloat(1.0, sync=True)
wireframe = Bool(False, sync=True)
class SpotLight(PointLight):
_view_name = Unicode('SpotLight', sync=True)
angle = CFloat(10, sync=True)
exponent = CFloat(0.5, sync=True)
class PointLight(IntensityLight):
_view_name = Unicode('PointLight', sync=True)
distance = CFloat(10, sync=True)
class _Float(DOMWidget):
value = CFloat(0.0, help="Float value", sync=True)
disabled = Bool(False, help="Enable or disable user changes", sync=True)
description = Unicode(
help="Description of the value this widget represents", sync=True)
class LocalEngineSetLauncher(LocalEngineLauncher):
"""Launch a set of engines as regular external processes."""
delay = CFloat(
0.1,
config=True,
help="""delay (in seconds) between starting each engine after the first.
This can help force the engines to get their ids in order, or limit
process flood when starting many engines.""")
# launcher class
launcher_class = LocalEngineLauncher
launchers = Dict()
stop_data = Dict()
def __init__(self, work_dir=u'.', config=None, **kwargs):
super(LocalEngineSetLauncher, self).__init__(work_dir=work_dir,
config=config,
**kwargs)
self.stop_data = {}
def start(self, n):
"""Start n engines by profile or profile_dir."""
dlist = []
for i in range(n):
if i > 0:
time.sleep(self.delay)
el = self.launcher_class(
work_dir=self.work_dir,
config=self.config,
log=self.log,
profile_dir=self.profile_dir,
cluster_id=self.cluster_id,
)
# Copy the engine args over to each engine launcher.
el.engine_cmd = copy.deepcopy(self.engine_cmd)
el.engine_args = copy.deepcopy(self.engine_args)
el.on_stop(self._notice_engine_stopped)
d = el.start()
self.launchers[i] = el
dlist.append(d)
self.notify_start(dlist)
return dlist
def find_args(self):
return ['engine set']
def signal(self, sig):
dlist = []
for el in self.launchers.itervalues():
d = el.signal(sig)
dlist.append(d)
return dlist
def interrupt_then_kill(self, delay=1.0):
dlist = []
for el in self.launchers.itervalues():
d = el.interrupt_then_kill(delay)
dlist.append(d)
return dlist
def stop(self):
return self.interrupt_then_kill()
def _notice_engine_stopped(self, data):
pid = data['pid']
for idx, el in self.launchers.iteritems():
if el.process.pid == pid:
break
self.launchers.pop(idx)
self.stop_data[idx] = data
if not self.launchers:
self.notify_stop(self.stop_data)
class SphereGeometry(Geometry):
_view_name = Unicode('SphereGeometryView', sync=True)
radius = CFloat(1, sync=True)
class HeartMonitor(LoggingConfigurable):
"""A basic HeartMonitor class
pingstream: a PUB stream
pongstream: an ROUTER stream
period: the period of the heartbeat in milliseconds"""
debug = Bool(
False,
config=True,
help="""Whether to include every heartbeat in debugging output.
Has to be set explicitly, because there will be *a lot* of output.
""")
period = Integer(
3000,
config=True,
help='The frequency at which the Hub pings the engines for heartbeats '
'(in ms)',
)
max_heartmonitor_misses = Integer(
10,
config=True,
help=
'Allowed consecutive missed pings from controller Hub to engine before unregistering.',
)
pingstream = Instance('zmq.eventloop.zmqstream.ZMQStream')
pongstream = Instance('zmq.eventloop.zmqstream.ZMQStream')
loop = Instance('zmq.eventloop.ioloop.IOLoop')
def _loop_default(self):
return ioloop.IOLoop.instance()
# not settable:
hearts = Set()
responses = Set()
on_probation = Dict()
last_ping = CFloat(0)
_new_handlers = Set()
_failure_handlers = Set()
lifetime = CFloat(0)
tic = CFloat(0)
def __init__(self, **kwargs):
super(HeartMonitor, self).__init__(**kwargs)
self.pongstream.on_recv(self.handle_pong)
def start(self):
self.tic = time.time()
self.caller = ioloop.PeriodicCallback(self.beat, self.period,
self.loop)
self.caller.start()
def add_new_heart_handler(self, handler):
"""add a new handler for new hearts"""
self.log.debug("heartbeat::new_heart_handler: %s", handler)
self._new_handlers.add(handler)
def add_heart_failure_handler(self, handler):
"""add a new handler for heart failure"""
self.log.debug("heartbeat::new heart failure handler: %s", handler)
self._failure_handlers.add(handler)
def beat(self):
self.pongstream.flush()
self.last_ping = self.lifetime
toc = time.time()
self.lifetime += toc - self.tic
self.tic = toc
if self.debug:
self.log.debug("heartbeat::sending %s", self.lifetime)
goodhearts = self.hearts.intersection(self.responses)
missed_beats = self.hearts.difference(goodhearts)
newhearts = self.responses.difference(goodhearts)
for heart in newhearts:
self.handle_new_heart(heart)
heartfailures, on_probation = self._check_missed(
missed_beats, self.on_probation, self.hearts)
for failure in heartfailures:
self.handle_heart_failure(failure)
self.on_probation = on_probation
self.responses = set()
#print self.on_probation, self.hearts
# self.log.debug("heartbeat::beat %.3f, %i beating hearts", self.lifetime, len(self.hearts))
self.pingstream.send(str_to_bytes(str(self.lifetime)))
# flush stream to force immediate socket send
self.pingstream.flush()
def _check_missed(self, missed_beats, on_probation, hearts):
"""Update heartbeats on probation, identifying any that have too many misses.
"""
failures = []
new_probation = {}
for cur_heart in (b for b in missed_beats if b in hearts):
miss_count = on_probation.get(cur_heart, 0) + 1
self.log.info("heartbeat::missed %s : %s" %
(cur_heart, miss_count))
if miss_count > self.max_heartmonitor_misses:
failures.append(cur_heart)
else:
new_probation[cur_heart] = miss_count
return failures, new_probation
def handle_new_heart(self, heart):
if self._new_handlers:
for handler in self._new_handlers:
handler(heart)
else:
self.log.info("heartbeat::yay, got new heart %s!", heart)
self.hearts.add(heart)
def handle_heart_failure(self, heart):
if self._failure_handlers:
for handler in self._failure_handlers:
try:
handler(heart)
except Exception as e:
self.log.error("heartbeat::Bad Handler! %s",
handler,
exc_info=True)
pass
else:
self.log.info("heartbeat::Heart %s failed :(", heart)
self.hearts.remove(heart)
@log_errors
def handle_pong(self, msg):
"a heart just beat"
current = str_to_bytes(str(self.lifetime))
last = str_to_bytes(str(self.last_ping))
if msg[1] == current:
delta = time.time() - self.tic
if self.debug:
self.log.debug("heartbeat::heart %r took %.2f ms to respond",
msg[0], 1000 * delta)
self.responses.add(msg[0])
elif msg[1] == last:
delta = time.time() - self.tic + (self.lifetime - self.last_ping)
self.log.warn(
"heartbeat::heart %r missed a beat, and took %.2f ms to respond",
msg[0], 1000 * delta)
self.responses.add(msg[0])
else:
self.log.warn(
"heartbeat::got bad heartbeat (possibly old?): %s (current=%.3f)",
msg[1], self.lifetime)
class LatheGeometry(Geometry):
_view_name = Unicode('LatheGeometryView', sync=True)
points = List(vector3(), sync=True)
segments = CInt(12, sync=True)
phiStart = CFloat(0, sync=True)
phiLength = CFloat(2*math.pi, sync=True)
class Camera(Object3d):
_view_name = Unicode('CameraView', sync=True)
fov = CFloat(40, sync=True)
ratio = CFloat(600.0/400.0, sync=True)
