class ClusterManager(LoggingConfigurable): profiles = Dict() delay = CFloat(1., config=True, help="delay (in s) between starting the controller and the engines") loop = Instance('zmq.eventloop.ioloop.IOLoop') def _loop_default(self): from zmq.eventloop.ioloop import IOLoop return IOLoop.instance() def build_launchers(self, profile_dir): starter = DummyIPClusterStart(log=self.log) starter.initialize(['--profile-dir', profile_dir]) cl = starter.controller_launcher esl = starter.engine_launcher n = starter.n return cl, esl, n def get_profile_dir(self, name, path): p = ProfileDir.find_profile_dir_by_name(path,name=name) return p.location def update_profiles(self): """List all profiles in the ipython_dir and cwd. """ for path in [get_ipython_dir(), os.getcwdu()]: for profile in list_profiles_in(path): pd = self.get_profile_dir(profile, path) if profile not in self.profiles: self.log.debug("Adding cluster profile '%s'" % profile) self.profiles[profile] = { 'profile': profile, 'profile_dir': pd, 'status': 'stopped' } def list_profiles(self): self.update_profiles() result = [self.profile_info(p) for p in sorted(self.profiles.keys())] return result def check_profile(self, profile): if profile not in self.profiles: raise web.HTTPError(404, u'profile not found') def profile_info(self, profile): self.check_profile(profile) result = {} data = self.profiles.get(profile) result['profile'] = profile result['profile_dir'] = data['profile_dir'] result['status'] = data['status'] if 'n' in data: result['n'] = data['n'] return result def start_cluster(self, profile, n=None): """Start a cluster for a given profile.""" self.check_profile(profile) data = self.profiles[profile] if data['status'] == 'running': raise web.HTTPError(409, u'cluster already running') cl, esl, default_n = self.build_launchers(data['profile_dir']) n = n if n is not None else default_n def clean_data(): data.pop('controller_launcher',None) data.pop('engine_set_launcher',None) data.pop('n',None) data['status'] = 'stopped' def engines_stopped(r): self.log.debug('Engines stopped') if cl.running: cl.stop() clean_data() esl.on_stop(engines_stopped) def controller_stopped(r): self.log.debug('Controller stopped') if esl.running: esl.stop() clean_data() cl.on_stop(controller_stopped) dc = ioloop.DelayedCallback(lambda: cl.start(), 0, self.loop) dc.start() dc = ioloop.DelayedCallback(lambda: esl.start(n), 1000*self.delay, self.loop) dc.start() self.log.debug('Cluster started') data['controller_launcher'] = cl data['engine_set_launcher'] = esl data['n'] = n data['status'] = 'running' return self.profile_info(profile) def stop_cluster(self, profile): """Stop a cluster for a given profile.""" self.check_profile(profile) data = self.profiles[profile] if data['status'] == 'stopped': raise web.HTTPError(409, u'cluster not running') data = self.profiles[profile] cl = data['controller_launcher'] esl = data['engine_set_launcher'] if cl.running: cl.stop() if esl.running: esl.stop() # Return a temp info dict, the real one is updated in the on_stop # logic above. result = { 'profile': data['profile'], 'profile_dir': data['profile_dir'], 'status': 'stopped' } return result def stop_all_clusters(self): for p in self.profiles.keys(): self.stop_cluster(p)
class _BoundedFloatRange(_FloatRange): step = CFloat( 1.0, help="Minimum step that the value can take (ignored by some views)", sync=True) max = CFloat(100.0, help="Max value", sync=True) min = CFloat(0.0, help="Min value", sync=True) def __init__(self, *pargs, **kwargs): any_value_given = 'value' in kwargs or 'upper' in kwargs or 'lower' in kwargs _FloatRange.__init__(self, *pargs, **kwargs) # ensure a minimal amount of sanity if self.min > self.max: raise ValueError("min must be <= max") if any_value_given: # if a value was given, clamp it within (min, max) self._validate("value", None, self.value) else: # otherwise, set it to 25-75% to avoid the handles overlapping self.value = (0.75 * self.min + 0.25 * self.max, 0.25 * self.min + 0.75 * self.max) # callback already set for 'value', 'lower', 'upper' self.on_trait_change(self._validate, ['min', 'max']) def _validate(self, name, old, new): if name == "min": if new > self.max: raise ValueError("setting min > max") self.min = new elif name == "max": if new < self.min: raise ValueError("setting max < min") self.max = new low, high = self.value if name == "value": low, high = min(new), max(new) elif name == "upper": if new < self.lower: raise ValueError("setting upper < lower") high = new elif name == "lower": if new > self.upper: raise ValueError("setting lower > upper") low = new low = max(self.min, min(low, self.max)) high = min(self.max, max(high, self.min)) # determine the order in which we should update the # lower, upper traits to avoid a temporary inverted overlap lower_first = high < self.lower self.value = (low, high) if lower_first: self.lower = low self.upper = high else: self.upper = high self.lower = low
class IPClusterStart(IPClusterEngines): name = u'ipcluster' description = start_help examples = _start_examples default_log_level = logging.INFO auto_create = Bool(True, config=True, help="whether to create the profile_dir if it doesn't exist") classes = List() def _classes_default(self,): from IPython.parallel.apps import launcher return [ProfileDir] + [IPClusterEngines] + launcher.all_launchers clean_logs = Bool(True, config=True, help="whether to cleanup old logs before starting") delay = CFloat(1., config=True, help="delay (in s) between starting the controller and the engines") controller_launcher = Any(config=True, help="Deprecated, use controller_launcher_class") def _controller_launcher_changed(self, name, old, new): if isinstance(new, basestring): # old 0.11-style config self.log.warn("WARNING: %s.controller_launcher is deprecated as of 0.12," " use controller_launcher_class" % self.__class__.__name__) self.controller_launcher_class = new controller_launcher_class = DottedObjectName('LocalControllerLauncher', config=True, help="""The class for launching a Controller. Change this value if you want your controller to also be launched by a batch system, such as PBS,SGE,MPIExec,etc. Each launcher class has its own set of configuration options, for making sure it will work in your environment. Examples include: LocalControllerLauncher : start engines locally as subprocesses MPIExecControllerLauncher : use mpiexec to launch engines in an MPI universe PBSControllerLauncher : use PBS (qsub) to submit engines to a batch queue SGEControllerLauncher : use SGE (qsub) to submit engines to a batch queue LSFControllerLauncher : use LSF (bsub) to submit engines to a batch queue SSHControllerLauncher : use SSH to start the controller WindowsHPCControllerLauncher : use Windows HPC If you are using one of IPython's builtin launchers, you can specify just the prefix, e.g: c.IPClusterStart.controller_launcher_class = 'SSH' or: ipcluster start --controller 'MPIExec' """ ) reset = Bool(False, config=True, help="Whether to reset config files as part of '--create'." ) # flags = Dict(flags) aliases = Dict(start_aliases) def init_launchers(self): self.controller_launcher = self.build_launcher(self.controller_launcher_class, 'Controller') self.engine_launcher = self.build_launcher(self.engine_launcher_class, 'EngineSet') self.controller_launcher.on_stop(self.stop_launchers) def engines_stopped(self, r): """prevent parent.engines_stopped from stopping everything on engine shutdown""" pass def start_controller(self): self.controller_launcher.start() def stop_controller(self): # self.log.info("In stop_controller") if self.controller_launcher and self.controller_launcher.running: return self.controller_launcher.stop() def stop_launchers(self, r=None): if not self._stopping: self.stop_controller() super(IPClusterStart, self).stop_launchers() def start(self): """Start the app for the start subcommand.""" # First see if the cluster is already running try: pid = self.get_pid_from_file() except PIDFileError: pass else: if self.check_pid(pid): self.log.critical( 'Cluster is already running with [pid=%s]. ' 'use "ipcluster stop" to stop the cluster.' % pid ) # Here I exit with a unusual exit status that other processes # can watch for to learn how I existed. self.exit(ALREADY_STARTED) else: self.remove_pid_file() # Now log and daemonize self.log.info( 'Starting ipcluster with [daemon=%r]' % self.daemonize ) # TODO: Get daemonize working on Windows or as a Windows Server. if self.daemonize: if os.name=='posix': daemonize() dc = ioloop.DelayedCallback(self.start_controller, 0, self.loop) dc.start() dc = ioloop.DelayedCallback(self.start_engines, 1000*self.delay, self.loop) dc.start() # Now write the new pid file AFTER our new forked pid is active. self.write_pid_file() try: self.loop.start() except KeyboardInterrupt: pass except zmq.ZMQError as e: if e.errno == errno.EINTR: pass else: raise finally: self.remove_pid_file()
class LoadBalancedView(View): """An load-balancing View that only executes via the Task scheduler. Load-balanced views can be created with the client's `view` method: >>> v = client.load_balanced_view() or targets can be specified, to restrict the potential destinations: >>> v = client.client.load_balanced_view([1,3]) which would restrict loadbalancing to between engines 1 and 3. """ follow = Any() after = Any() timeout = CFloat() retries = Integer(0) _task_scheme = Any() _flag_names = List( ['targets', 'block', 'track', 'follow', 'after', 'timeout', 'retries']) def __init__(self, client=None, socket=None, **flags): super(LoadBalancedView, self).__init__(client=client, socket=socket, **flags) self._task_scheme = client._task_scheme def _validate_dependency(self, dep): """validate a dependency. For use in `set_flags`. """ if dep is None or isinstance(dep, string_types + (AsyncResult, Dependency)): return True elif isinstance(dep, (list, set, tuple)): for d in dep: if not isinstance(d, string_types + (AsyncResult, )): return False elif isinstance(dep, dict): if set(dep.keys()) != set(Dependency().as_dict().keys()): return False if not isinstance(dep['msg_ids'], list): return False for d in dep['msg_ids']: if not isinstance(d, string_types): return False else: return False return True def _render_dependency(self, dep): """helper for building jsonable dependencies from various input forms.""" if isinstance(dep, Dependency): return dep.as_dict() elif isinstance(dep, AsyncResult): return dep.msg_ids elif dep is None: return [] else: # pass to Dependency constructor return list(Dependency(dep)) def set_flags(self, **kwargs): """set my attribute flags by keyword. A View is a wrapper for the Client's apply method, but with attributes that specify keyword arguments, those attributes can be set by keyword argument with this method. Parameters ---------- block : bool whether to wait for results track : bool whether to create a MessageTracker to allow the user to safely edit after arrays and buffers during non-copying sends. after : Dependency or collection of msg_ids Only for load-balanced execution (targets=None) Specify a list of msg_ids as a time-based dependency. This job will only be run *after* the dependencies have been met. follow : Dependency or collection of msg_ids Only for load-balanced execution (targets=None) Specify a list of msg_ids as a location-based dependency. This job will only be run on an engine where this dependency is met. timeout : float/int or None Only for load-balanced execution (targets=None) Specify an amount of time (in seconds) for the scheduler to wait for dependencies to be met before failing with a DependencyTimeout. retries : int Number of times a task will be retried on failure. """ super(LoadBalancedView, self).set_flags(**kwargs) for name in ('follow', 'after'): if name in kwargs: value = kwargs[name] if self._validate_dependency(value): setattr(self, name, value) else: raise ValueError("Invalid dependency: %r" % value) if 'timeout' in kwargs: t = kwargs['timeout'] if not isinstance(t, (int, float, type(None))): if (not PY3) and (not isinstance(t, long)): raise TypeError("Invalid type for timeout: %r" % type(t)) if t is not None: if t < 0: raise ValueError("Invalid timeout: %s" % t) self.timeout = t @sync_results @save_ids def _really_apply(self, f, args=None, kwargs=None, block=None, track=None, after=None, follow=None, timeout=None, targets=None, retries=None): """calls f(*args, **kwargs) on a remote engine, returning the result. This method temporarily sets all of `apply`'s flags for a single call. Parameters ---------- f : callable args : list [default: empty] kwargs : dict [default: empty] block : bool [default: self.block] whether to block track : bool [default: self.track] whether to ask zmq to track the message, for safe non-copying sends !!!!!! TODO: THE REST HERE !!!! Returns ------- if self.block is False: returns AsyncResult else: returns actual result of f(*args, **kwargs) on the engine(s) This will be a list of self.targets is also a list (even length 1), or the single result if self.targets is an integer engine id """ # validate whether we can run if self._socket.closed: msg = "Task farming is disabled" if self._task_scheme == 'pure': msg += " because the pure ZMQ scheduler cannot handle" msg += " disappearing engines." raise RuntimeError(msg) if self._task_scheme == 'pure': # pure zmq scheme doesn't support extra features msg = "Pure ZMQ scheduler doesn't support the following flags:" "follow, after, retries, targets, timeout" if (follow or after or retries or targets or timeout): # hard fail on Scheduler flags raise RuntimeError(msg) if isinstance(f, dependent): # soft warn on functional dependencies warnings.warn(msg, RuntimeWarning) # build args args = [] if args is None else args kwargs = {} if kwargs is None else kwargs block = self.block if block is None else block track = self.track if track is None else track after = self.after if after is None else after retries = self.retries if retries is None else retries follow = self.follow if follow is None else follow timeout = self.timeout if timeout is None else timeout targets = self.targets if targets is None else targets if not isinstance(retries, int): raise TypeError('retries must be int, not %r' % type(retries)) if targets is None: idents = [] else: idents = self.client._build_targets(targets)[0] # ensure *not* bytes idents = [ident.decode() for ident in idents] after = self._render_dependency(after) follow = self._render_dependency(follow) metadata = dict(after=after, follow=follow, timeout=timeout, targets=idents, retries=retries) msg = self.client.send_apply_request(self._socket, f, args, kwargs, track=track, metadata=metadata) tracker = None if track is False else msg['tracker'] ar = AsyncResult( self.client, msg['header']['msg_id'], fname=getname(f), targets=None, tracker=tracker, owner=True, ) if block: try: return ar.get() except KeyboardInterrupt: pass return ar @sync_results @save_ids def map(self, f, *sequences, **kwargs): """``view.map(f, *sequences, block=self.block, chunksize=1, ordered=True)`` => list|AsyncMapResult Parallel version of builtin `map`, load-balanced by this View. `block`, and `chunksize` can be specified by keyword only. Each `chunksize` elements will be a separate task, and will be load-balanced. This lets individual elements be available for iteration as soon as they arrive. Parameters ---------- f : callable function to be mapped *sequences: one or more sequences of matching length the sequences to be distributed and passed to `f` block : bool [default self.block] whether to wait for the result or not track : bool whether to create a MessageTracker to allow the user to safely edit after arrays and buffers during non-copying sends. chunksize : int [default 1] how many elements should be in each task. ordered : bool [default True] Whether the results should be gathered as they arrive, or enforce the order of submission. Only applies when iterating through AsyncMapResult as results arrive. Has no effect when block=True. Returns ------- if block=False An :class:`~IPython.parallel.client.asyncresult.AsyncMapResult` instance. An object like AsyncResult, but which reassembles the sequence of results into a single list. AsyncMapResults can be iterated through before all results are complete. else A list, the result of ``map(f,*sequences)`` """ # default block = kwargs.get('block', self.block) chunksize = kwargs.get('chunksize', 1) ordered = kwargs.get('ordered', True) keyset = set(kwargs.keys()) extra_keys = keyset.difference_update(set(['block', 'chunksize'])) if extra_keys: raise TypeError("Invalid kwargs: %s" % list(extra_keys)) assert len(sequences) > 0, "must have some sequences to map onto!" pf = ParallelFunction(self, f, block=block, chunksize=chunksize, ordered=ordered) return pf.map(*sequences)
class IntensityLight(Light): _view_name = Unicode('PositionLight', sync=True) intensity = CFloat(1, sync=True)
class HeartMonitor(LoggingConfigurable): """A basic HeartMonitor class pingstream: a PUB stream pongstream: an XREP stream period: the period of the heartbeat in milliseconds""" period = CFloat( 1000, config=True, help='The frequency at which the Hub pings the engines for heartbeats ' '(in ms)', ) pingstream = Instance('zmq.eventloop.zmqstream.ZMQStream') pongstream = Instance('zmq.eventloop.zmqstream.ZMQStream') loop = Instance('zmq.eventloop.ioloop.IOLoop') def _loop_default(self): return ioloop.IOLoop.instance() # not settable: hearts = Set() responses = Set() on_probation = Set() last_ping = CFloat(0) _new_handlers = Set() _failure_handlers = Set() lifetime = CFloat(0) tic = CFloat(0) def __init__(self, **kwargs): super(HeartMonitor, self).__init__(**kwargs) self.pongstream.on_recv(self.handle_pong) def start(self): self.tic = time.time() self.caller = ioloop.PeriodicCallback(self.beat, self.period, self.loop) self.caller.start() def add_new_heart_handler(self, handler): """add a new handler for new hearts""" self.log.debug("heartbeat::new_heart_handler: %s", handler) self._new_handlers.add(handler) def add_heart_failure_handler(self, handler): """add a new handler for heart failure""" self.log.debug("heartbeat::new heart failure handler: %s", handler) self._failure_handlers.add(handler) def beat(self): self.pongstream.flush() self.last_ping = self.lifetime toc = time.time() self.lifetime += toc - self.tic self.tic = toc self.log.debug("heartbeat::sending %s", self.lifetime) goodhearts = self.hearts.intersection(self.responses) missed_beats = self.hearts.difference(goodhearts) heartfailures = self.on_probation.intersection(missed_beats) newhearts = self.responses.difference(goodhearts) map(self.handle_new_heart, newhearts) map(self.handle_heart_failure, heartfailures) self.on_probation = missed_beats.intersection(self.hearts) self.responses = set() # print self.on_probation, self.hearts # self.log.debug("heartbeat::beat %.3f, %i beating hearts", self.lifetime, len(self.hearts)) self.pingstream.send(asbytes(str(self.lifetime))) def handle_new_heart(self, heart): if self._new_handlers: for handler in self._new_handlers: handler(heart) else: self.log.info("heartbeat::yay, got new heart %s!", heart) self.hearts.add(heart) def handle_heart_failure(self, heart): if self._failure_handlers: for handler in self._failure_handlers: try: handler(heart) except Exception as e: self.log.error("heartbeat::Bad Handler! %s", handler, exc_info=True) pass else: self.log.info("heartbeat::Heart %s failed :(", heart) self.hearts.remove(heart) def handle_pong(self, msg): "a heart just beat" current = asbytes(str(self.lifetime)) last = asbytes(str(self.last_ping)) if msg[1] == current: delta = time.time() - self.tic # self.log.debug("heartbeat::heart %r took %.2f ms to respond"%(msg[0], 1000*delta)) self.responses.add(msg[0]) elif msg[1] == last: delta = time.time() - self.tic + (self.lifetime - self.last_ping) self.log.warn( "heartbeat::heart %r missed a beat, and took %.2f ms to respond", msg[0], 1000 * delta) self.responses.add(msg[0]) else: self.log.warn( "heartbeat::got bad heartbeat (possibly old?): %s (current=%.3f)", msg[1], self.lifetime)
class EngineFactory(RegistrationFactory): """IPython engine""" # configurables: user_ns=Dict(config=True) out_stream_factory=Type('IPython.zmq.iostream.OutStream', config=True) display_hook_factory=Type('IPython.zmq.displayhook.DisplayHook', config=True) location=Str(config=True) timeout=CFloat(2,config=True) # not configurable: id=Int(allow_none=True) registrar=Instance('zmq.eventloop.zmqstream.ZMQStream') kernel=Instance(Kernel) def __init__(self, **kwargs): super(EngineFactory, self).__init__(**kwargs) ctx = self.context reg = ctx.socket(zmq.XREQ) reg.setsockopt(zmq.IDENTITY, self.ident) reg.connect(self.url) self.registrar = zmqstream.ZMQStream(reg, self.loop) def register(self): """send the registration_request""" self.log.info("registering") content = dict(queue=self.ident, heartbeat=self.ident, control=self.ident) self.registrar.on_recv(self.complete_registration) # print (self.session.key) self.session.send(self.registrar, "registration_request",content=content) def complete_registration(self, msg): # print msg self._abort_dc.stop() ctx = self.context loop = self.loop identity = self.ident idents,msg = self.session.feed_identities(msg) msg = Message(self.session.unpack_message(msg)) if msg.content.status == 'ok': self.id = int(msg.content.id) # create Shell Streams (MUX, Task, etc.): queue_addr = msg.content.mux shell_addrs = [ str(queue_addr) ] task_addr = msg.content.task if task_addr: shell_addrs.append(str(task_addr)) # Uncomment this to go back to two-socket model # shell_streams = [] # for addr in shell_addrs: # stream = zmqstream.ZMQStream(ctx.socket(zmq.XREP), loop) # stream.setsockopt(zmq.IDENTITY, identity) # stream.connect(disambiguate_url(addr, self.location)) # shell_streams.append(stream) # Now use only one shell stream for mux and tasks stream = zmqstream.ZMQStream(ctx.socket(zmq.XREP), loop) stream.setsockopt(zmq.IDENTITY, identity) shell_streams = [stream] for addr in shell_addrs: stream.connect(disambiguate_url(addr, self.location)) # end single stream-socket # control stream: control_addr = str(msg.content.control) control_stream = zmqstream.ZMQStream(ctx.socket(zmq.XREP), loop) control_stream.setsockopt(zmq.IDENTITY, identity) control_stream.connect(disambiguate_url(control_addr, self.location)) # create iopub stream: iopub_addr = msg.content.iopub iopub_stream = zmqstream.ZMQStream(ctx.socket(zmq.PUB), loop) iopub_stream.setsockopt(zmq.IDENTITY, identity) iopub_stream.connect(disambiguate_url(iopub_addr, self.location)) # launch heartbeat hb_addrs = msg.content.heartbeat # print (hb_addrs) # # Redirect input streams and set a display hook. if self.out_stream_factory: sys.stdout = self.out_stream_factory(self.session, iopub_stream, u'stdout') sys.stdout.topic = 'engine.%i.stdout'%self.id sys.stderr = self.out_stream_factory(self.session, iopub_stream, u'stderr') sys.stderr.topic = 'engine.%i.stderr'%self.id if self.display_hook_factory: sys.displayhook = self.display_hook_factory(self.session, iopub_stream) sys.displayhook.topic = 'engine.%i.pyout'%self.id self.kernel = Kernel(config=self.config, int_id=self.id, ident=self.ident, session=self.session, control_stream=control_stream, shell_streams=shell_streams, iopub_stream=iopub_stream, loop=loop, user_ns = self.user_ns, logname=self.log.name) self.kernel.start() hb_addrs = [ disambiguate_url(addr, self.location) for addr in hb_addrs ] heart = Heart(*map(str, hb_addrs), heart_id=identity) # ioloop.DelayedCallback(heart.start, 1000, self.loop).start() heart.start() else: self.log.fatal("Registration Failed: %s"%msg) raise Exception("Registration Failed: %s"%msg) self.log.info("Completed registration with id %i"%self.id) def abort(self): self.log.fatal("Registration timed out") self.session.send(self.registrar, "unregistration_request", content=dict(id=self.id)) time.sleep(1) sys.exit(255) def start(self): dc = ioloop.DelayedCallback(self.register, 0, self.loop) dc.start() self._abort_dc = ioloop.DelayedCallback(self.abort, self.timeout*1000, self.loop) self._abort_dc.start()
class CircleGeometry(Geometry): _view_name = Unicode('CircleGeometryView', sync=True) radius = CFloat(1, sync=True) segments = CFloat(8, sync=True) thetaStart = CFloat(0, sync=True) thetaLength = CFloat(2*math.pi, sync=True)
class ForceDirectedGraph(widgets.DOMWidget): _view_module = Unicode('nbextensions/d3networkx/widget', sync=True) _view_name = Unicode('D3ForceDirectedGraphView', sync=True) width = CInt(400, sync=True) height = CInt(300, sync=True) charge = CFloat(270., sync=True) distance = CInt(30., sync=True) strength = CInt(0.3, sync=True) def __init__(self, eventful_graph, *pargs, **kwargs): widgets.DOMWidget.__init__(self, *pargs, **kwargs) self._eventful_graph = eventful_graph self._send_dict_changes(eventful_graph.graph, 'graph') self._send_dict_changes(eventful_graph.node, 'node') self._send_dict_changes(eventful_graph.adj, 'adj') def _ipython_display_(self, *pargs, **kwargs): # Show the widget, then send the current state widgets.DOMWidget._ipython_display_(self, *pargs, **kwargs) for (key, value) in self._eventful_graph.graph.items(): self.send({ 'dict': 'graph', 'action': 'add', 'key': key, 'value': value }) for (key, value) in self._eventful_graph.node.items(): self.send({ 'dict': 'node', 'action': 'add', 'key': key, 'value': value }) for (key, value) in self._eventful_graph.adj.items(): self.send({ 'dict': 'adj', 'action': 'add', 'key': key, 'value': value }) def _send_dict_changes(self, eventful_dict, dict_name): def key_add(key, value): self.send({ 'dict': dict_name, 'action': 'add', 'key': key, 'value': value }) def key_set(key, value): self.send({ 'dict': dict_name, 'action': 'set', 'key': key, 'value': value }) def key_del(key): self.send({'dict': dict_name, 'action': 'del', 'key': key}) eventful_dict.on_add(key_add) eventful_dict.on_set(key_set) eventful_dict.on_del(key_del)
class EngineFactory(RegistrationFactory): """IPython engine""" # configurables: out_stream_factory = Type('IPython.zmq.iostream.OutStream', config=True, help="""The OutStream for handling stdout/err. Typically 'IPython.zmq.iostream.OutStream'""") display_hook_factory = Type('IPython.zmq.displayhook.ZMQDisplayHook', config=True, help="""The class for handling displayhook. Typically 'IPython.zmq.displayhook.ZMQDisplayHook'""") location = Unicode( config=True, help="""The location (an IP address) of the controller. This is used for disambiguating URLs, to determine whether loopback should be used to connect or the public address.""") timeout = CFloat( 2, config=True, help="""The time (in seconds) to wait for the Controller to respond to registration requests before giving up.""") sshserver = Unicode( config=True, help= """The SSH server to use for tunneling connections to the Controller.""" ) sshkey = Unicode( config=True, help= """The SSH private key file to use when tunneling connections to the Controller.""" ) paramiko = Bool( sys.platform == 'win32', config=True, help="""Whether to use paramiko instead of openssh for tunnels.""") # not configurable: user_ns = Dict() id = Integer(allow_none=True) registrar = Instance('zmq.eventloop.zmqstream.ZMQStream') kernel = Instance(Kernel) bident = CBytes() ident = Unicode() def _ident_changed(self, name, old, new): self.bident = cast_bytes(new) using_ssh = Bool(False) def __init__(self, **kwargs): super(EngineFactory, self).__init__(**kwargs) self.ident = self.session.session def init_connector(self): """construct connection function, which handles tunnels.""" self.using_ssh = bool(self.sshkey or self.sshserver) if self.sshkey and not self.sshserver: # We are using ssh directly to the controller, tunneling localhost to localhost self.sshserver = self.url.split('://')[1].split(':')[0] if self.using_ssh: if tunnel.try_passwordless_ssh(self.sshserver, self.sshkey, self.paramiko): password = False else: password = getpass("SSH Password for %s: " % self.sshserver) else: password = False def connect(s, url): url = disambiguate_url(url, self.location) if self.using_ssh: self.log.debug("Tunneling connection to %s via %s" % (url, self.sshserver)) return tunnel.tunnel_connection( s, url, self.sshserver, keyfile=self.sshkey, paramiko=self.paramiko, password=password, ) else: return s.connect(url) def maybe_tunnel(url): """like connect, but don't complete the connection (for use by heartbeat)""" url = disambiguate_url(url, self.location) if self.using_ssh: self.log.debug("Tunneling connection to %s via %s" % (url, self.sshserver)) url, tunnelobj = tunnel.open_tunnel( url, self.sshserver, keyfile=self.sshkey, paramiko=self.paramiko, password=password, ) return url return connect, maybe_tunnel def register(self): """send the registration_request""" self.log.info("Registering with controller at %s" % self.url) ctx = self.context connect, maybe_tunnel = self.init_connector() reg = ctx.socket(zmq.DEALER) reg.setsockopt(zmq.IDENTITY, self.bident) connect(reg, self.url) self.registrar = zmqstream.ZMQStream(reg, self.loop) content = dict(queue=self.ident, heartbeat=self.ident, control=self.ident) self.registrar.on_recv( lambda msg: self.complete_registration(msg, connect, maybe_tunnel)) # print (self.session.key) self.session.send(self.registrar, "registration_request", content=content) def complete_registration(self, msg, connect, maybe_tunnel): # print msg self._abort_dc.stop() ctx = self.context loop = self.loop identity = self.bident idents, msg = self.session.feed_identities(msg) msg = Message(self.session.unserialize(msg)) if msg.content.status == 'ok': self.id = int(msg.content.id) # launch heartbeat hb_addrs = msg.content.heartbeat # possibly forward hb ports with tunnels hb_addrs = [maybe_tunnel(addr) for addr in hb_addrs] heart = Heart(*list(map(str, hb_addrs)), heart_id=identity) heart.start() # create Shell Streams (MUX, Task, etc.): queue_addr = msg.content.mux shell_addrs = [str(queue_addr)] task_addr = msg.content.task if task_addr: shell_addrs.append(str(task_addr)) # Uncomment this to go back to two-socket model # shell_streams = [] # for addr in shell_addrs: # stream = zmqstream.ZMQStream(ctx.socket(zmq.ROUTER), loop) # stream.setsockopt(zmq.IDENTITY, identity) # stream.connect(disambiguate_url(addr, self.location)) # shell_streams.append(stream) # Now use only one shell stream for mux and tasks stream = zmqstream.ZMQStream(ctx.socket(zmq.ROUTER), loop) stream.setsockopt(zmq.IDENTITY, identity) shell_streams = [stream] for addr in shell_addrs: connect(stream, addr) # end single stream-socket # control stream: control_addr = str(msg.content.control) control_stream = zmqstream.ZMQStream(ctx.socket(zmq.ROUTER), loop) control_stream.setsockopt(zmq.IDENTITY, identity) connect(control_stream, control_addr) # create iopub stream: iopub_addr = msg.content.iopub iopub_socket = ctx.socket(zmq.PUB) iopub_socket.setsockopt(zmq.IDENTITY, identity) connect(iopub_socket, iopub_addr) # disable history: self.config.HistoryManager.hist_file = ':memory:' # Redirect input streams and set a display hook. if self.out_stream_factory: sys.stdout = self.out_stream_factory(self.session, iopub_socket, 'stdout') sys.stdout.topic = cast_bytes('engine.%i.stdout' % self.id) sys.stderr = self.out_stream_factory(self.session, iopub_socket, 'stderr') sys.stderr.topic = cast_bytes('engine.%i.stderr' % self.id) if self.display_hook_factory: sys.displayhook = self.display_hook_factory( self.session, iopub_socket) sys.displayhook.topic = cast_bytes('engine.%i.pyout' % self.id) self.kernel = Kernel(config=self.config, int_id=self.id, ident=self.ident, session=self.session, control_stream=control_stream, shell_streams=shell_streams, iopub_socket=iopub_socket, loop=loop, user_ns=self.user_ns, log=self.log) self.kernel.shell.display_pub.topic = cast_bytes( 'engine.%i.displaypub' % self.id) # FIXME: This is a hack until IPKernelApp and IPEngineApp can be fully merged app = IPKernelApp(config=self.config, shell=self.kernel.shell, kernel=self.kernel, log=self.log) app.init_profile_dir() app.init_code() self.kernel.start() else: self.log.fatal("Registration Failed: %s" % msg) raise Exception("Registration Failed: %s" % msg) self.log.info("Completed registration with id %i" % self.id) def abort(self): self.log.fatal("Registration timed out after %.1f seconds" % self.timeout) if self.url.startswith('127.'): self.log.fatal(""" If the controller and engines are not on the same machine, you will have to instruct the controller to listen on an external IP (in ipcontroller_config.py): c.HubFactory.ip='*' # for all interfaces, internal and external c.HubFactory.ip='192.168.1.101' # or any interface that the engines can see or tunnel connections via ssh. """) self.session.send(self.registrar, "unregistration_request", content=dict(id=self.id)) time.sleep(1) sys.exit(255) def start(self): dc = ioloop.DelayedCallback(self.register, 0, self.loop) dc.start() self._abort_dc = ioloop.DelayedCallback(self.abort, self.timeout * 1000, self.loop) self._abort_dc.start()
class Material(Widget): _view_name = Unicode('MaterialView', sync=True) color = Any('yellow', sync=True) opacity = CFloat(1.0, sync=True) wireframe = Bool(False, sync=True)
class SpotLight(PointLight): _view_name = Unicode('SpotLight', sync=True) angle = CFloat(10, sync=True) exponent = CFloat(0.5, sync=True)
class PointLight(IntensityLight): _view_name = Unicode('PointLight', sync=True) distance = CFloat(10, sync=True)
class _Float(DOMWidget): value = CFloat(0.0, help="Float value", sync=True) disabled = Bool(False, help="Enable or disable user changes", sync=True) description = Unicode( help="Description of the value this widget represents", sync=True)
class LocalEngineSetLauncher(LocalEngineLauncher): """Launch a set of engines as regular external processes.""" delay = CFloat( 0.1, config=True, help="""delay (in seconds) between starting each engine after the first. This can help force the engines to get their ids in order, or limit process flood when starting many engines.""") # launcher class launcher_class = LocalEngineLauncher launchers = Dict() stop_data = Dict() def __init__(self, work_dir=u'.', config=None, **kwargs): super(LocalEngineSetLauncher, self).__init__(work_dir=work_dir, config=config, **kwargs) self.stop_data = {} def start(self, n): """Start n engines by profile or profile_dir.""" dlist = [] for i in range(n): if i > 0: time.sleep(self.delay) el = self.launcher_class( work_dir=self.work_dir, config=self.config, log=self.log, profile_dir=self.profile_dir, cluster_id=self.cluster_id, ) # Copy the engine args over to each engine launcher. el.engine_cmd = copy.deepcopy(self.engine_cmd) el.engine_args = copy.deepcopy(self.engine_args) el.on_stop(self._notice_engine_stopped) d = el.start() self.launchers[i] = el dlist.append(d) self.notify_start(dlist) return dlist def find_args(self): return ['engine set'] def signal(self, sig): dlist = [] for el in self.launchers.itervalues(): d = el.signal(sig) dlist.append(d) return dlist def interrupt_then_kill(self, delay=1.0): dlist = [] for el in self.launchers.itervalues(): d = el.interrupt_then_kill(delay) dlist.append(d) return dlist def stop(self): return self.interrupt_then_kill() def _notice_engine_stopped(self, data): pid = data['pid'] for idx, el in self.launchers.iteritems(): if el.process.pid == pid: break self.launchers.pop(idx) self.stop_data[idx] = data if not self.launchers: self.notify_stop(self.stop_data)
class SphereGeometry(Geometry): _view_name = Unicode('SphereGeometryView', sync=True) radius = CFloat(1, sync=True)
class HeartMonitor(LoggingConfigurable): """A basic HeartMonitor class pingstream: a PUB stream pongstream: an ROUTER stream period: the period of the heartbeat in milliseconds""" debug = Bool( False, config=True, help="""Whether to include every heartbeat in debugging output. Has to be set explicitly, because there will be *a lot* of output. """) period = Integer( 3000, config=True, help='The frequency at which the Hub pings the engines for heartbeats ' '(in ms)', ) max_heartmonitor_misses = Integer( 10, config=True, help= 'Allowed consecutive missed pings from controller Hub to engine before unregistering.', ) pingstream = Instance('zmq.eventloop.zmqstream.ZMQStream') pongstream = Instance('zmq.eventloop.zmqstream.ZMQStream') loop = Instance('zmq.eventloop.ioloop.IOLoop') def _loop_default(self): return ioloop.IOLoop.instance() # not settable: hearts = Set() responses = Set() on_probation = Dict() last_ping = CFloat(0) _new_handlers = Set() _failure_handlers = Set() lifetime = CFloat(0) tic = CFloat(0) def __init__(self, **kwargs): super(HeartMonitor, self).__init__(**kwargs) self.pongstream.on_recv(self.handle_pong) def start(self): self.tic = time.time() self.caller = ioloop.PeriodicCallback(self.beat, self.period, self.loop) self.caller.start() def add_new_heart_handler(self, handler): """add a new handler for new hearts""" self.log.debug("heartbeat::new_heart_handler: %s", handler) self._new_handlers.add(handler) def add_heart_failure_handler(self, handler): """add a new handler for heart failure""" self.log.debug("heartbeat::new heart failure handler: %s", handler) self._failure_handlers.add(handler) def beat(self): self.pongstream.flush() self.last_ping = self.lifetime toc = time.time() self.lifetime += toc - self.tic self.tic = toc if self.debug: self.log.debug("heartbeat::sending %s", self.lifetime) goodhearts = self.hearts.intersection(self.responses) missed_beats = self.hearts.difference(goodhearts) newhearts = self.responses.difference(goodhearts) for heart in newhearts: self.handle_new_heart(heart) heartfailures, on_probation = self._check_missed( missed_beats, self.on_probation, self.hearts) for failure in heartfailures: self.handle_heart_failure(failure) self.on_probation = on_probation self.responses = set() #print self.on_probation, self.hearts # self.log.debug("heartbeat::beat %.3f, %i beating hearts", self.lifetime, len(self.hearts)) self.pingstream.send(str_to_bytes(str(self.lifetime))) # flush stream to force immediate socket send self.pingstream.flush() def _check_missed(self, missed_beats, on_probation, hearts): """Update heartbeats on probation, identifying any that have too many misses. """ failures = [] new_probation = {} for cur_heart in (b for b in missed_beats if b in hearts): miss_count = on_probation.get(cur_heart, 0) + 1 self.log.info("heartbeat::missed %s : %s" % (cur_heart, miss_count)) if miss_count > self.max_heartmonitor_misses: failures.append(cur_heart) else: new_probation[cur_heart] = miss_count return failures, new_probation def handle_new_heart(self, heart): if self._new_handlers: for handler in self._new_handlers: handler(heart) else: self.log.info("heartbeat::yay, got new heart %s!", heart) self.hearts.add(heart) def handle_heart_failure(self, heart): if self._failure_handlers: for handler in self._failure_handlers: try: handler(heart) except Exception as e: self.log.error("heartbeat::Bad Handler! %s", handler, exc_info=True) pass else: self.log.info("heartbeat::Heart %s failed :(", heart) self.hearts.remove(heart) @log_errors def handle_pong(self, msg): "a heart just beat" current = str_to_bytes(str(self.lifetime)) last = str_to_bytes(str(self.last_ping)) if msg[1] == current: delta = time.time() - self.tic if self.debug: self.log.debug("heartbeat::heart %r took %.2f ms to respond", msg[0], 1000 * delta) self.responses.add(msg[0]) elif msg[1] == last: delta = time.time() - self.tic + (self.lifetime - self.last_ping) self.log.warn( "heartbeat::heart %r missed a beat, and took %.2f ms to respond", msg[0], 1000 * delta) self.responses.add(msg[0]) else: self.log.warn( "heartbeat::got bad heartbeat (possibly old?): %s (current=%.3f)", msg[1], self.lifetime)
class LatheGeometry(Geometry): _view_name = Unicode('LatheGeometryView', sync=True) points = List(vector3(), sync=True) segments = CInt(12, sync=True) phiStart = CFloat(0, sync=True) phiLength = CFloat(2*math.pi, sync=True)
class Camera(Object3d): _view_name = Unicode('CameraView', sync=True) fov = CFloat(40, sync=True) ratio = CFloat(600.0/400.0, sync=True)