def init_schedulers(self): children = self.children mq = import_item(str(self.mq_class)) f = self.factory ident = f.session.bsession # disambiguate url, in case of * monitor_url = disambiguate_url(f.monitor_url) # maybe_inproc = 'inproc://monitor' if self.use_threads else monitor_url # IOPub relay (in a Process) q = mq(zmq.PUB, zmq.SUB, zmq.PUB, b'N/A', b'iopub') q.bind_in(f.client_url('iopub')) q.setsockopt_in(zmq.IDENTITY, ident + b"_iopub") q.bind_out(f.engine_url('iopub')) q.setsockopt_out(zmq.SUBSCRIBE, b'') q.connect_mon(monitor_url) q.daemon = True children.append(q) # Multiplexer Queue (in a Process) q = mq(zmq.ROUTER, zmq.ROUTER, zmq.PUB, b'in', b'out') q.bind_in(f.client_url('mux')) q.setsockopt_in(zmq.IDENTITY, b'mux_in') q.bind_out(f.engine_url('mux')) q.setsockopt_out(zmq.IDENTITY, b'mux_out') q.connect_mon(monitor_url) q.daemon = True children.append(q) # Control Queue (in a Process) q = mq(zmq.ROUTER, zmq.ROUTER, zmq.PUB, b'incontrol', b'outcontrol') q.bind_in(f.client_url('control')) q.setsockopt_in(zmq.IDENTITY, b'control_in') q.bind_out(f.engine_url('control')) q.setsockopt_out(zmq.IDENTITY, b'control_out') q.connect_mon(monitor_url) q.daemon = True children.append(q) try: scheme = self.config.TaskScheduler.scheme_name except AttributeError: scheme = TaskScheduler.scheme_name.get_default_value() # Task Queue (in a Process) if scheme == 'pure': self.log.warn("task::using pure DEALER Task scheduler") q = mq(zmq.ROUTER, zmq.DEALER, zmq.PUB, b'intask', b'outtask') # q.setsockopt_out(zmq.HWM, hub.hwm) q.bind_in(f.client_url('task')) q.setsockopt_in(zmq.IDENTITY, b'task_in') q.bind_out(f.engine_url('task')) q.setsockopt_out(zmq.IDENTITY, b'task_out') q.connect_mon(monitor_url) q.daemon = True children.append(q) elif scheme == 'none': self.log.warn("task::using no Task scheduler") else: self.log.info("task::using Python %s Task scheduler" % scheme) sargs = ( f.client_url('task'), f.engine_url('task'), monitor_url, disambiguate_url(f.client_url('notification')), disambiguate_url(f.client_url('registration')), ) kwargs = dict(logname='scheduler', loglevel=self.log_level, log_url=self.log_url, config=dict(self.config)) if 'Process' in self.mq_class: # run the Python scheduler in a Process q = Process(target=launch_scheduler, args=sargs, kwargs=kwargs) q.daemon = True children.append(q) else: # single-threaded Controller kwargs['in_thread'] = True launch_scheduler(*sargs, **kwargs) # set unlimited HWM for all relay devices if hasattr(zmq, 'SNDHWM'): q = children[0] q.setsockopt_in(zmq.RCVHWM, 0) q.setsockopt_out(zmq.SNDHWM, 0) for q in children[1:]: if not hasattr(q, 'setsockopt_in'): continue q.setsockopt_in(zmq.SNDHWM, 0) q.setsockopt_in(zmq.RCVHWM, 0) q.setsockopt_out(zmq.SNDHWM, 0) q.setsockopt_out(zmq.RCVHWM, 0) q.setsockopt_mon(zmq.SNDHWM, 0)
def init_schedulers(self): children = self.children mq = import_item(str(self.mq_class)) hub = self.factory # disambiguate url, in case of * monitor_url = disambiguate_url(hub.monitor_url) # maybe_inproc = 'inproc://monitor' if self.use_threads else monitor_url # IOPub relay (in a Process) q = mq(zmq.PUB, zmq.SUB, zmq.PUB, b'N/A',b'iopub') q.bind_in(hub.client_info['iopub']) q.bind_out(hub.engine_info['iopub']) q.setsockopt_out(zmq.SUBSCRIBE, b'') q.connect_mon(monitor_url) q.daemon=True children.append(q) # Multiplexer Queue (in a Process) q = mq(zmq.ROUTER, zmq.ROUTER, zmq.PUB, b'in', b'out') q.bind_in(hub.client_info['mux']) q.setsockopt_in(zmq.IDENTITY, b'mux') q.bind_out(hub.engine_info['mux']) q.connect_mon(monitor_url) q.daemon=True children.append(q) # Control Queue (in a Process) q = mq(zmq.ROUTER, zmq.ROUTER, zmq.PUB, b'incontrol', b'outcontrol') q.bind_in(hub.client_info['control']) q.setsockopt_in(zmq.IDENTITY, b'control') q.bind_out(hub.engine_info['control']) q.connect_mon(monitor_url) q.daemon=True children.append(q) try: scheme = self.config.TaskScheduler.scheme_name except AttributeError: scheme = TaskScheduler.scheme_name.get_default_value() # Task Queue (in a Process) if scheme == 'pure': self.log.warn("task::using pure XREQ Task scheduler") q = mq(zmq.ROUTER, zmq.DEALER, zmq.PUB, b'intask', b'outtask') # q.setsockopt_out(zmq.HWM, hub.hwm) q.bind_in(hub.client_info['task'][1]) q.setsockopt_in(zmq.IDENTITY, b'task') q.bind_out(hub.engine_info['task']) q.connect_mon(monitor_url) q.daemon=True children.append(q) elif scheme == 'none': self.log.warn("task::using no Task scheduler") else: self.log.info("task::using Python %s Task scheduler"%scheme) sargs = (hub.client_info['task'][1], hub.engine_info['task'], monitor_url, disambiguate_url(hub.client_info['notification'])) kwargs = dict(logname='scheduler', loglevel=self.log_level, log_url = self.log_url, config=dict(self.config)) if 'Process' in self.mq_class: # run the Python scheduler in a Process q = Process(target=launch_scheduler, args=sargs, kwargs=kwargs) q.daemon=True children.append(q) else: # single-threaded Controller kwargs['in_thread'] = True launch_scheduler(*sargs, **kwargs)
def init_schedulers(self): children = self.children mq = import_item(str(self.mq_class)) f = self.factory ident = f.session.bsession # disambiguate url, in case of * monitor_url = disambiguate_url(f.monitor_url) # maybe_inproc = 'inproc://monitor' if self.use_threads else monitor_url # IOPub relay (in a Process) q = mq(zmq.PUB, zmq.SUB, zmq.PUB, b'N/A',b'iopub') q.bind_in(f.client_url('iopub')) q.setsockopt_in(zmq.IDENTITY, ident + b"_iopub") q.bind_out(f.engine_url('iopub')) q.setsockopt_out(zmq.SUBSCRIBE, b'') q.connect_mon(monitor_url) q.daemon=True children.append(q) # Multiplexer Queue (in a Process) q = mq(zmq.ROUTER, zmq.ROUTER, zmq.PUB, b'in', b'out') q.bind_in(f.client_url('mux')) q.setsockopt_in(zmq.IDENTITY, b'mux_in') q.bind_out(f.engine_url('mux')) q.setsockopt_out(zmq.IDENTITY, b'mux_out') q.connect_mon(monitor_url) q.daemon=True children.append(q) # Control Queue (in a Process) q = mq(zmq.ROUTER, zmq.ROUTER, zmq.PUB, b'incontrol', b'outcontrol') q.bind_in(f.client_url('control')) q.setsockopt_in(zmq.IDENTITY, b'control_in') q.bind_out(f.engine_url('control')) q.setsockopt_out(zmq.IDENTITY, b'control_out') q.connect_mon(monitor_url) q.daemon=True children.append(q) try: scheme = self.config.TaskScheduler.scheme_name except AttributeError: scheme = TaskScheduler.scheme_name.get_default_value() # Task Queue (in a Process) if scheme == 'pure': self.log.warn("task::using pure DEALER Task scheduler") q = mq(zmq.ROUTER, zmq.DEALER, zmq.PUB, b'intask', b'outtask') # q.setsockopt_out(zmq.HWM, hub.hwm) q.bind_in(f.client_url('task')) q.setsockopt_in(zmq.IDENTITY, b'task_in') q.bind_out(f.engine_url('task')) q.setsockopt_out(zmq.IDENTITY, b'task_out') q.connect_mon(monitor_url) q.daemon=True children.append(q) elif scheme == 'none': self.log.warn("task::using no Task scheduler") else: self.log.info("task::using Python %s Task scheduler"%scheme) sargs = (f.client_url('task'), f.engine_url('task'), monitor_url, disambiguate_url(f.client_url('notification')), disambiguate_url(f.client_url('registration')), ) kwargs = dict(logname='scheduler', loglevel=self.log_level, log_url = self.log_url, config=dict(self.config)) if 'Process' in self.mq_class: # run the Python scheduler in a Process q = Process(target=launch_scheduler, args=sargs, kwargs=kwargs) q.daemon=True children.append(q) else: # single-threaded Controller kwargs['in_thread'] = True launch_scheduler(*sargs, **kwargs) # set unlimited HWM for all relay devices if hasattr(zmq, 'SNDHWM'): q = children[0] q.setsockopt_in(zmq.RCVHWM, 0) q.setsockopt_out(zmq.SNDHWM, 0) for q in children[1:]: if not hasattr(q, 'setsockopt_in'): continue q.setsockopt_in(zmq.SNDHWM, 0) q.setsockopt_in(zmq.RCVHWM, 0) q.setsockopt_out(zmq.SNDHWM, 0) q.setsockopt_out(zmq.RCVHWM, 0) q.setsockopt_mon(zmq.SNDHWM, 0)
def init_schedulers(self): children = self.children mq = import_item(str(self.mq_class)) hub = self.factory # disambiguate url, in case of * monitor_url = disambiguate_url(hub.monitor_url) # maybe_inproc = 'inproc://monitor' if self.use_threads else monitor_url # IOPub relay (in a Process) q = mq(zmq.PUB, zmq.SUB, zmq.PUB, b'N/A', b'iopub') q.bind_in(hub.client_info['iopub']) q.bind_out(hub.engine_info['iopub']) q.setsockopt_out(zmq.SUBSCRIBE, b'') q.connect_mon(monitor_url) q.daemon = True children.append(q) # Multiplexer Queue (in a Process) q = mq(zmq.ROUTER, zmq.ROUTER, zmq.PUB, b'in', b'out') q.bind_in(hub.client_info['mux']) q.setsockopt_in(zmq.IDENTITY, b'mux') q.bind_out(hub.engine_info['mux']) q.connect_mon(monitor_url) q.daemon = True children.append(q) # Control Queue (in a Process) q = mq(zmq.ROUTER, zmq.ROUTER, zmq.PUB, b'incontrol', b'outcontrol') q.bind_in(hub.client_info['control']) q.setsockopt_in(zmq.IDENTITY, b'control') q.bind_out(hub.engine_info['control']) q.connect_mon(monitor_url) q.daemon = True children.append(q) try: scheme = self.config.TaskScheduler.scheme_name except AttributeError: scheme = TaskScheduler.scheme_name.get_default_value() # Task Queue (in a Process) if scheme == 'pure': self.log.warn("task::using pure XREQ Task scheduler") q = mq(zmq.ROUTER, zmq.DEALER, zmq.PUB, b'intask', b'outtask') # q.setsockopt_out(zmq.HWM, hub.hwm) q.bind_in(hub.client_info['task'][1]) q.setsockopt_in(zmq.IDENTITY, b'task') q.bind_out(hub.engine_info['task']) q.connect_mon(monitor_url) q.daemon = True children.append(q) elif scheme == 'none': self.log.warn("task::using no Task scheduler") else: self.log.info("task::using Python %s Task scheduler" % scheme) sargs = (hub.client_info['task'][1], hub.engine_info['task'], monitor_url, disambiguate_url(hub.client_info['notification'])) kwargs = dict(logname='scheduler', loglevel=self.log_level, log_url=self.log_url, config=dict(self.config)) if 'Process' in self.mq_class: # run the Python scheduler in a Process q = Process(target=launch_scheduler, args=sargs, kwargs=kwargs) q.daemon = True children.append(q) else: # single-threaded Controller kwargs['in_thread'] = True launch_scheduler(*sargs, **kwargs)
def init_schedulers(self): children = self.children mq = import_item(str(self.mq_class)) f = self.factory ident = f.session.bsession # disambiguate url, in case of * monitor_url = disambiguate_url(f.monitor_url) # maybe_inproc = 'inproc://monitor' if self.use_threads else monitor_url # IOPub relay (in a Process) q = mq(zmq.PUB, zmq.SUB, zmq.PUB, b"N/A", b"iopub") q.bind_in(f.client_url("iopub")) q.setsockopt_in(zmq.IDENTITY, ident + b"_iopub") q.bind_out(f.engine_url("iopub")) q.setsockopt_out(zmq.SUBSCRIBE, b"") q.connect_mon(monitor_url) q.daemon = True children.append(q) # Multiplexer Queue (in a Process) q = mq(zmq.ROUTER, zmq.ROUTER, zmq.PUB, b"in", b"out") q.bind_in(f.client_url("mux")) q.setsockopt_in(zmq.IDENTITY, b"mux_in") q.bind_out(f.engine_url("mux")) q.setsockopt_out(zmq.IDENTITY, b"mux_out") q.connect_mon(monitor_url) q.daemon = True children.append(q) # Control Queue (in a Process) q = mq(zmq.ROUTER, zmq.ROUTER, zmq.PUB, b"incontrol", b"outcontrol") q.bind_in(f.client_url("control")) q.setsockopt_in(zmq.IDENTITY, b"control_in") q.bind_out(f.engine_url("control")) q.setsockopt_out(zmq.IDENTITY, b"control_out") q.connect_mon(monitor_url) q.daemon = True children.append(q) try: scheme = self.config.TaskScheduler.scheme_name except AttributeError: scheme = TaskScheduler.scheme_name.get_default_value() # Task Queue (in a Process) if scheme == "pure": self.log.warn("task::using pure DEALER Task scheduler") q = mq(zmq.ROUTER, zmq.DEALER, zmq.PUB, b"intask", b"outtask") # q.setsockopt_out(zmq.HWM, hub.hwm) q.bind_in(f.client_url("task")) q.setsockopt_in(zmq.IDENTITY, b"task_in") q.bind_out(f.engine_url("task")) q.setsockopt_out(zmq.IDENTITY, b"task_out") q.connect_mon(monitor_url) q.daemon = True children.append(q) elif scheme == "none": self.log.warn("task::using no Task scheduler") else: self.log.info("task::using Python %s Task scheduler" % scheme) sargs = ( f.client_url("task"), f.engine_url("task"), monitor_url, disambiguate_url(f.client_url("notification")), disambiguate_url(f.client_url("registration")), ) kwargs = dict(logname="scheduler", loglevel=self.log_level, log_url=self.log_url, config=dict(self.config)) if "Process" in self.mq_class: # run the Python scheduler in a Process q = Process(target=launch_scheduler, args=sargs, kwargs=kwargs) q.daemon = True children.append(q) else: # single-threaded Controller kwargs["in_thread"] = True launch_scheduler(*sargs, **kwargs)