def _setup(self): """Setup all the attributes. Bind the server, connect to remote peers and initiate the timers. """ self._zmq_context = zmq.Context() self._socket_for_commands = self._zmq_context.socket(zmq.ROUTER) self._socket_for_consensus = self._zmq_context.socket(zmq.ROUTER) for remote_endpoint in self._remote_endpoints: remote_peer = peer.Peer(self._zmq_context, self._private_endpoint, remote_endpoint) peer_id = six.b(remote_endpoint) self._remote_peers[peer_id] = remote_peer self._server_state.init_indexes(peer_id) self._zmq_ioloop = ioloop.ZMQIOLoop().instance() self._zmq_ioloop.add_handler(self._socket_for_commands, self._process_command_message, zmq.POLLIN) self._zmq_ioloop.add_handler(self._socket_for_consensus, self._process_internal_message, zmq.POLLIN) self.checking_leader_timeout = ioloop.PeriodicCallback( self._election_timeout_task, random.randint(self._min_election_timeout, self._max_election_timeout), io_loop=self._zmq_ioloop) self.heartbeating = ioloop.PeriodicCallback( self.broadcast_append_entries, self._leader_heartbeat_interval, io_loop=self._zmq_ioloop)
def __init__(self): LoggerMixin.configure() db.configure() Compute.configure() APIContainer.configure() JBoxAsyncJob.configure() JBoxAsyncJob.init(JBoxAsyncJob.MODE_PUB) self.application = tornado.web.Application( handlers=[(r"^/", APIInfoHandler), (r"^/.*/.*", APIHandler)]) self.application.settings["cookie_secret"] = JBoxCfg.get('sesskey') self.application.listen(JBoxCfg.get('api.manager_port'), address=socket.gethostname()) self.application.listen(JBoxCfg.get('api.manager_port'), address='localhost') self.ioloop = ioloop.IOLoop.instance() # run container maintainence every 5 minutes run_interval = 5 * 60 * 1000 self.log_info("Container maintenance every " + str(run_interval / (60 * 1000)) + " minutes") self.ct = ioloop.PeriodicCallback(JBoxAPI.do_housekeeping, run_interval, self.ioloop) self.sigct = ioloop.PeriodicCallback(JBoxAPI.do_signals, 1000, self.ioloop)
def initialize(self): """Initialize method called at plugin start. The method register the periodic callback of the ping() method and fill the monitored_watchers dict. """ super(CircusAutorestartPlugin, self).initialize() self.fill_watchers(debug_output=True) self.periodic = ioloop.PeriodicCallback(self.ping, 1000, self.loop) self.periodic.start() self.periodic10 = ioloop.PeriodicCallback(self.fill_watchers, 10000, self.loop) self.periodic10.start()
def initialize(self): """Initialize method called at plugin start. The method register the periodic callback of the ping() method and fill the watchers set. """ super(CircusAutostart, self).initialize() self.periodic = ioloop.PeriodicCallback(self.ping, 1000, self.loop) self.periodic5 = ioloop.PeriodicCallback(self.ping_every5s, 5000, self.loop) self.periodic.start() self.periodic5.start() self.fill_watchers()
def test_simple(self): """simple IOLoop creation test""" loop = ioloop.IOLoop() dc = ioloop.PeriodicCallback(loop.stop, 200, loop) pc = ioloop.PeriodicCallback(lambda : None, 10, loop) pc.start() dc.start() t = Delay(loop.stop,1) t.start() loop.start() if t.isAlive(): t.abort() else: self.fail("IOLoop failed to exit")
def setup(self, pop_server_address, broker_server_address): super().setup() pop_subscription = self.context.socket(zmq.SUB) pop_subscription.connect("ipc://{}".format(pop_server_address)) logger.info('Connected to population subscription: {}'.format( pop_server_address)) pop_subscription.setsockopt(zmq.SUBSCRIBE, b"") self.population_subscription = zmqstream.ZMQStream( pop_subscription, self.loop) self.population_subscription.on_recv( PropagatorMessageHandler(None, self.propagator)) # Connection to broker broker_connection = self.context.socket(zmq.REQ) broker_connection.connect("ipc://{}".format(broker_server_address)) logger.info('Connected to broker: {}'.format(broker_server_address)) self.broker_connection = zmqstream.ZMQStream(broker_connection, self.loop) self.broker_connection.on_recv( PropagatorBrokerMessageHandler(self.broker_connection, self.propagator)) # Setup tick callback self.ticker = ioloop.PeriodicCallback(self.tick, 1000, self.loop)
def start(self): """ Start the Notebook server app, after initialization This method takes no arguments so all configuration and initialization must be done prior to calling this method.""" if not self.allow_root: # check if we are running as root, and abort if it's not allowed try: uid = os.geteuid() except AttributeError: uid = -1 # anything nonzero here, since we can't check UID assume non-root if uid == 0: self.log.critical("Running as root is not recommended. Use --allow-root to bypass.") self.exit(1) super(NotebookApp, self).start() info = self.log.info for line in self.notebook_info().split("\n"): info(line) info("Use Control-C to stop this server and shut down all kernels (twice to skip confirmation).") self.write_server_info_file() if self.open_browser or self.file_to_run: try: browser = webbrowser.get(self.browser or None) except webbrowser.Error as e: self.log.warning('No web browser found: %s.' % e) browser = None if self.file_to_run: if not os.path.exists(self.file_to_run): self.log.critical("%s does not exist" % self.file_to_run) self.exit(1) relpath = os.path.relpath(self.file_to_run, self.notebook_dir) uri = url_escape(url_path_join('notebooks', *relpath.split(os.sep))) else: # default_url contains base_url, but so does connection_url uri = self.default_url[len(self.base_url):] if browser: b = lambda : browser.open(url_path_join(self.connection_url, uri), new=2) threading.Thread(target=b).start() self.io_loop = ioloop.IOLoop.current() if sys.platform.startswith('win'): # add no-op to wake every 5s # to handle signals that may be ignored by the inner loop pc = ioloop.PeriodicCallback(lambda : None, 5000) pc.start() try: self.io_loop.start() except KeyboardInterrupt: info("Interrupted...") finally: self.cleanup_kernels() self.remove_server_info_file()
def __init__(self, socket_type, service_name, service_port=None, service_type=None): self.context = zmq.Context() self.socket = self.context.socket(socket_type) if not service_port: service_port = self.socket.bind_to_random_port('tcp://*', min_port=49152, max_port=65535, max_tries=100) else: self.socket.bind("tcp://*:%d" % service_port) print "Bound to port %d" % service_port self.stream = ZMQStream(self.socket) if not service_type: service_type = socket_type_to_service(socket_type) if socket_type == zmq.PUB: # TODO: how to handle this with ROUTER/DEALER combinations... self.heartbeat_timer = ioloop.PeriodicCallback( self._hearbeat, 1000) self.heartbeat_timer.start() if socket_type == zmq.ROUTER: self.stream.on_recv(self._method_callback_wrapper) bonjour_utilities.register_ioloop(ioloop.IOLoop.instance(), service_type, service_name, service_port)
def __init__(self, args=None, loop=None): if args is None: args = {} super(ExternalRunner, self).__init__(args) self._current_step = 0 self._step_started_at = None self._duration = self.args.get('duration') self._timeout = args.get('external_process_timeout', 2) self._processes = [] self._processes_pending_cleanup = [] # hits and users are lists that can be None. hits, users = [1], [1] if self.args.get('hits') is not None: hits = self.args['hits'] if self.args.get('users') is not None: users = self.args['users'] self.args['hits'] = hits self.args['users'] = users self._nb_steps = max(len(hits), len(users)) self._loop = loop or ioloop.IOLoop() # Check the status of the processes every so-often.(500ms) cb = ioloop.PeriodicCallback(self._check_processes, 500, self._loop) cb.start() self._receiver_socket = (self.args.get('zmq_receiver') or DEFAULT_EXTERNAL_RUNNER_RECEIVER)
def __init__(self, io_loop=None): self.io_loop = io_loop or ioloop.IOLoop.instance() self._sockets = {} # fd -> socket object self._started = False self._connections = {} self.checker = ioloop.PeriodicCallback( self.check_for_closed_connections, 30000)
def __init__(self, **kwargs): super(SQLiteDB, self).__init__(**kwargs) if sqlite3 is None: raise ImportError("SQLiteDB requires sqlite3") if not self.table: # use session, and prefix _, since starting with # is illegal self.table = '_' + self.session.replace('-', '_') if not self.location: # get current profile from IPython.core.application import BaseIPythonApplication if BaseIPythonApplication.initialized(): app = BaseIPythonApplication.instance() if app.profile_dir is not None: self.location = app.profile_dir.location else: self.location = u'.' else: self.location = u'.' self._init_db() # register db commit as 2s periodic callback # to prevent clogging pipes # assumes we are being run in a zmq ioloop app loop = ioloop.IOLoop.instance() pc = ioloop.PeriodicCallback(self._db.commit, 2000, loop) pc.start()
def main(): import optparse parser = optparse.OptionParser('usage: %prog') ZmqPublisher.addOptions(parser, 'zmqPublish') parser.add_option( '-p', '--prefix', default='', help= 'Prefix to prepend to incoming lines before publishing them (DO include trailing "." if you want it)' ) opts, args = parser.parse_args() if args: parser.error('expected no args') logging.basicConfig(level=logging.DEBUG) # set up networking p = ZmqPublisher(**ZmqPublisher.getOptionValues(opts)) p.start() # start publishing an arbitrary message that central should forward pubTimer = ioloop.PeriodicCallback(lambda: pubMessage(opts.prefix, p), 0.1) pubTimer.start() zmqLoop()
def start(self): super(FormgradeApp, self).start() if self.logfile: self.init_logging(logging.FileHandler, [self.logfile], color=False) else: self.init_logging() self.init_tornado_settings() self.init_handlers() self.init_tornado_application() # Create the application self.io_loop = ioloop.IOLoop.current() self.tornado_application.listen(self.port, address=self.ip) url = "http://{:s}:{:d}/".format(self.ip, self.port) self.log.info("Form grader running at {}".format(url)) self.log.info("Use Control-C to stop this server") if sys.platform.startswith('win'): # add no-op to wake every 1s # to handle signals that may be ignored by the inner loop pc = ioloop.PeriodicCallback(lambda: None, 1000) pc.start() # Start the loop self.io_loop.start()
def start(self): if self.state == 'before': self.process = Popen(self.args, stdout=PIPE, stderr=PIPE, stdin=PIPE, env=os.environ, cwd=self.work_dir) if WINDOWS: self.stdout = forward_read_events(self.process.stdout) self.stderr = forward_read_events(self.process.stderr) else: self.stdout = self.process.stdout.fileno() self.stderr = self.process.stderr.fileno() self.loop.add_handler(self.stdout, self.handle_stdout, self.loop.READ) self.loop.add_handler(self.stderr, self.handle_stderr, self.loop.READ) self.poller = ioloop.PeriodicCallback(self.poll, self.poll_frequency, self.loop) self.poller.start() self.notify_start(self.process.pid) else: s = 'The process was already started and has state: %r' % self.state raise ProcessStateError(s)
def start_hb(self, callback): """Start the heartbeating and call the callback if the kernel dies.""" if not self._beating: self._kernel_alive = True def ping_or_dead(): self.hb_stream.flush() if self._kernel_alive: self._kernel_alive = False self.hb_stream.send(b'ping') # flush stream to force immediate socket send self.hb_stream.flush() else: try: callback() except: pass finally: self.stop_hb() def beat_received(msg): self._kernel_alive = True self.hb_stream.on_recv(beat_received) loop = ioloop.IOLoop.instance() self._hb_periodic_callback = ioloop.PeriodicCallback( ping_or_dead, self.time_to_dead * 1000, loop) loop.add_timeout(time.time() + self.first_beat, self._really_start_hb) self._beating = True
def init_dynamic_configs(self) -> None: """ Initialize the set of configurables that should participate in dynamic updates. We should also log that we're performing dynamic configuration updates, along with the list of CLI options - that are not privy to dynamic updates. :return: """ if self.dynamic_config_interval > 0: self.add_dynamic_configurable("EnterpriseGatewayApp", self) self.add_dynamic_configurable("MappingKernelManager", self.kernel_manager) self.add_dynamic_configurable("KernelSpecManager", self.kernel_spec_manager) self.add_dynamic_configurable("KernelSessionManager", self.kernel_session_manager) self.log.info( "Dynamic updates have been configured. Checking every {} seconds." .format(self.dynamic_config_interval)) self.log.info( "The following configuration options will not be subject to dynamic updates " "(configured via CLI):") for config, options in self.cli_config.items(): for option, value in options.items(): self.log.info(f" '{config}.{option}': '{value}'") if self.dynamic_config_poller is None: self.dynamic_config_poller = ioloop.PeriodicCallback( self.update_dynamic_configurables, self.dynamic_config_interval * 1000) self.dynamic_config_poller.start()
def start_hb(self, callback): """Start the heartbeating and call the callback if the kernel dies.""" if not self._beating: self._kernel_alive = True def ping_or_dead(): if self._kernel_alive: self._kernel_alive = False self.hb_stream.send(b'ping') else: try: callback() except: pass finally: self._hb_periodic_callback.stop() def beat_received(msg): self._kernel_alive = True self.hb_stream.on_recv(beat_received) self._hb_periodic_callback = ioloop.PeriodicCallback( ping_or_dead, self.time_to_dead * 1000) self._hb_periodic_callback.start() self._beating = True
def __init__(self, loop, pingstream, pongstream, config): self.pingstream = pingstream self.pongstream = pongstream self.pongstream.on_recv(self.handle_input) self.hearts = set() self.responses = set() self.lifetime = 0 self.tic = time.time() self.logger = config.get_logger(__name__) self.checkin_count = config.get_checkin_count() self.caller = ioloop.PeriodicCallback( self.beat, config.get_ping_interval() * 1000, loop) self.action_poll_interval = config.get_action_poll_interval() self.last_action_poll = time.time() self.smdb = smdb.SMDB() self.changed_state = [] self.logger.info("Starting OSAD server.") self.caller.start()
def start(self): """Start the polling of the kernel.""" if self._pcallback is None: self._pcallback = ioloop.PeriodicCallback( self.poll, 1000*self.time_to_dead, ) self._pcallback.start()
def start(self, delay_start=0): """Start the client loop.""" # # Create a new instance of the ioloop. # This is important for example when running from an ipython notebook (which # is itself an ioloop.) # ioloop.IOLoop.clear_current() ioloop.IOLoop.clear_instance( ) # or del IOLoop._instance in tornado < 3.3 # # Set the (Tornado) loop # self.loop = ioloop.IOLoop().instance() # # Start the MDP client. # super(Client, self).start() # # Start the ioloop. # time.sleep(delay_start) ioloop.PeriodicCallback( partial(self.send_mmi, service=MDP.MMI_SERVICES, msg=[]), self.mmi_period, self.loop).start() self.loop.start()
def __init__(self, streamer, name, callback_time=1., io_loop=None): super(SocketStatsCollector, self).__init__(streamer, name, callback_time, io_loop) self._rstats = defaultdict(int) self.sockets = [sock for sock, address, fd in self.streamer.sockets] self._p = ioloop.PeriodicCallback(self._select, _LOOP_RES, io_loop=io_loop)
def run(self, func): context = zmq.Context() self.socket = context.socket(zmq.PAIR) self.socket.set_hwm(self.hwm) self.socket.bind(self.address) self.receive_timer = ioloop.PeriodicCallback(func, self.interval) self._start_timer()
def __init__(self, *args, **config): super(ApprcWatcher, self).__init__(*args, **config) self.loop_rate = config.get("loop_rate", 3) # in seconds self.apprc = config.get("apprc", "/home/application/apprc") self.port = config.get("port", "8888") self.period = ioloop.PeriodicCallback( FileWatcher(self.apprc, self.reload_env), self.loop_rate * 1000, self.loop)
def __init__(self, broker=DEFAULT_FRONTEND, ping_delay=10., ping_retries=3, params=None, timeout=DEFAULT_TIMEOUT_MOVF, max_age=DEFAULT_MAX_AGE, max_age_delta=DEFAULT_MAX_AGE_DELTA): logger.debug('Initializing the agent.') self.debug = logger.isEnabledFor(logging.DEBUG) self.params = params self.pid = os.getpid() self.agent_id = '%s-%s' % (get_hostname(), self.pid) self.timeout = timeout self.max_age = max_age self.max_age_delta = max_age_delta self.env = os.environ.copy() self.running = False self._workers = {} self._max_id = defaultdict(int) # Let's ask the broker its options self.broker = broker client = Client(self.broker) # this will timeout in case the broker is unreachable result = client.ping() self.endpoints = result['endpoints'] # Setup the zmq sockets self.loop = ioloop.IOLoop() self.ctx = zmq.Context() # backend socket - used to receive work from the broker self._backend = self.ctx.socket(zmq.ROUTER) self._backend.identity = self.agent_id self._backend.connect(self.endpoints['backend']) # register socket - used to register into the broker self._reg = self.ctx.socket(zmq.PUSH) self._reg.connect(self.endpoints['register']) # hearbeat socket - used to check if the broker is alive heartbeat = self.endpoints.get('heartbeat') if heartbeat is not None: logger.info("Hearbeat activated") self.ping = Stethoscope(heartbeat, onbeatlost=self.lost, delay=ping_delay, retries=ping_retries, ctx=self.ctx, io_loop=self.loop, onregister=self.register) else: self.ping = None # Setup the zmq streams. self._backstream = zmqstream.ZMQStream(self._backend, self.loop) self._backstream.on_recv(self._handle_recv_back) self._check = ioloop.PeriodicCallback(self._check_proc, ping_delay * 1000, io_loop=self.loop)
def __init__(self): super(mcp, self).__init__(METHODS_SERVICE_NAME, service_port=METHODS_PORT) # This is low-level ZMQStream, to be used only in special cases self.signals_stream = zmqdecorators.server_tracker.get_by_name(SIGNALS_SERVICE_NAME, zmq.PUB).stream # periodically reap dead workers from our registry self.reaper_pcb = ioloop_mod.PeriodicCallback(self.reap_dead_workers, 1000) self.reaper_pcb.start()
def registerAlarm(self, interval, callback, UUID=None): if UUID is None: UUID = uuid4() if UUID in self.alarms: raise KeyError("Alarm UUID already exist") self.alarms[UUID] = ioloop.PeriodicCallback(callback, interval) self.alarms[UUID].start() return UUID
def start(self): self.initialize() if self.check_delay > 0: # The specific case (check_delay < 0) # so with no period callback to manage_watchers # is probably "unit tests only" self.caller = ioloop.PeriodicCallback(self.manage_watchers, self.check_delay, self.loop) self.caller.start() self.started = True
def start(self): self.engine_stream.on_recv(self.dispatch_result, copy=False) self._notification_handlers = dict( registration_notification=self._register_engine, unregistration_notification=self._unregister_engine) self.notifier_stream.on_recv(self.dispatch_notification) self.auditor = ioloop.PeriodicCallback(self.audit_timeouts, 2e3, self.loop) # 1 Hz self.auditor.start() self.log.info("Scheduler started...%r" % self)
def handle_init(self): """Initialization of plugin - set the periodic call back for the process monitoring (at loop_rate) - create the listening UDP socket """ self.period = ioloop.PeriodicCallback(self.look_after, self.loop_rate * 1000, self.loop) self.period.start() self._bind_socket()
def start(self): """ Start the Notebook server app, after initialization This method takes no arguments so all configuration and initialization must be done prior to calling this method.""" super(NotebookApp, self).start() info = self.log.info for line in self.notebook_info().split("\n"): info(line) info( "Use Control-C to stop this server and shut down all kernels (twice to skip confirmation)." ) self.write_server_info_file() if self.open_browser or self.file_to_run: try: browser = webbrowser.get(self.browser or None) except webbrowser.Error as e: self.log.warn('No web browser found: %s.' % e) browser = None if self.file_to_run: if not os.path.exists(self.file_to_run): self.log.critical("%s does not exist" % self.file_to_run) self.exit(1) relpath = os.path.relpath(self.file_to_run, self.notebook_dir) uri = url_escape( url_path_join('notebooks', *relpath.split(os.sep))) else: # default_url contains base_url, but so does connection_url uri = self.default_url[len(self.base_url):] if self.one_time_token: uri = url_concat(uri, {'token': self.one_time_token}) if browser: b = lambda: browser.open( url_path_join(self.connection_url, uri), new=2) threading.Thread(target=b).start() self.io_loop = ioloop.IOLoop.current() if sys.platform.startswith('win'): # add no-op to wake every 5s # to handle signals that may be ignored by the inner loop pc = ioloop.PeriodicCallback(lambda: None, 5000) pc.start() try: self.io_loop.start() except KeyboardInterrupt: info("Interrupted...") finally: self.remove_server_info_file() self.cleanup_kernels()