def run(self): _period = 1000 * self.configreader["services"][self.name]["checkmail_interval"] beat = PeriodicCallback(self.cmd_processmails, _period, io_loop=self.ioloop) beat.start() Service.run(self)
def __init__(self, primary, local, remote): # initialize the Binary Star self.ctx = zmq.Context() # Our private context self.loop = IOLoop.instance() # Reactor loop self.state = STATE_PRIMARY if primary else STATE_BACKUP self.event = None # Current event self.peer_expiry = 0 # When peer is considered 'dead' self.voter_callback = None # Voting socket handler self.master_callback = None # Call when become master self.slave_callback = None # Call when become slave # Create publisher for state going to peer self.statepub = self.ctx.socket(zmq.PUB) self.statepub.bind(local) # Create subscriber for state coming from peer self.statesub = self.ctx.socket(zmq.SUB) self.statesub.setsockopt_string(zmq.SUBSCRIBE, u'') self.statesub.connect(remote) # wrap statesub in ZMQStream for event triggers self.statesub = ZMQStream(self.statesub, self.loop) # setup basic reactor events self.heartbeat = PeriodicCallback(self.send_state, HEARTBEAT, self.loop) self.statesub.on_recv(self.recv_state)
def __init__(self): self.stop = False self.conx = Context.instance() self.socket = self.conx.socket(zmq.ROUTER) self.socket.bind('tcp://*:5555') self.periodic = PeriodicCallback(self.timer, 4000) self.periodic.start()
def __init__(self, context, main_ep, opt_ep=None): """Init MDPBroker instance. """ l = logger.Logger('mq_broker') self.log = l.get_logger() self.log.info("MDP broker startup...") socket = ZmqSocket(context, zmq.ROUTER) socket.bind(main_ep) self.main_stream = ZMQStream(socket) self.main_stream.on_recv(self.on_message) if opt_ep: socket = ZmqSocket(context, zmq.ROUTER) socket.bind(opt_ep) self.client_stream = ZMQStream(socket) self.client_stream.on_recv(self.on_message) else: self.client_stream = self.main_stream self.log.debug("Socket created...") self._workers = {} # services contain the worker queue and the request queue self._services = {} self._worker_cmds = { b'\x01': self.on_ready, b'\x03': self.on_reply, b'\x04': self.on_heartbeat, b'\x05': self.on_disconnect, } self.log.debug("Launch the timer...") self.hb_check_timer = PeriodicCallback(self.on_timer, HB_INTERVAL) self.hb_check_timer.start() self.log.info("MDP broker started") return
def __init__(self, frontier, data_in_sock='ipc:///tmp/robot-data-w2m.sock', data_out_sock='ipc:///tmp/robot-data-m2w.sock', msg_in_sock='ipc:///tmp/robot-msg-w2m.sock', msg_out_sock='ipc:///tmp/robot-msg-m2w.sock', io_loop=None): self.identity = 'master:%s:%s' % (socket.gethostname(), os.getpid()) context = zmq.Context() self._io_loop = io_loop or IOLoop.instance() self._in_socket = context.socket(zmq.SUB) self._in_socket.setsockopt(zmq.SUBSCRIBE, '') self._in_socket.bind(data_in_sock) self._in_stream = ZMQStream(self._in_socket, io_loop) self._out_socket = context.socket(zmq.PUSH) self._out_socket.bind(data_out_sock) self._out_stream = ZMQStream(self._out_socket, io_loop) self._online_workers = set() self._running = False self._updater = PeriodicCallback(self._send_next, 100, io_loop=io_loop) self._reloader = PeriodicCallback(self.reload, 1000, io_loop=io_loop) self.frontier = frontier self.messenger = ServerMessenger(msg_in_sock, msg_out_sock, context, io_loop)
class WorkerRep(object): """Helper class to represent a worker in the broker. Instances of this class are used to track the state of the attached worker and carry the timers for incomming and outgoing heartbeats. :type wid: str :param wid: the worker id. :param service: service this worker serves :type service: str :param stream: the ZMQStream used to send messages :type stream: ZMQStream """ def __init__(self, wid, service, stream): self.id = wid self.service = service self.multicasts = [] self.curr_liveness = HB_LIVENESS self.stream = stream self.last_hb = 0 self.hb_out_timer = PeriodicCallback(self.send_hb, HB_INTERVAL) self.hb_out_timer.start() return def send_hb(self): """Called on every HB_INTERVAL. Decrements the current liveness by one. Sends heartbeat to worker. """ self.curr_liveness -= 1 msg = [ self.id, b'', MDP_WORKER_VERSION, b'\x05' ] self.stream.send_multipart(msg) return def on_heartbeat(self): """Called when a heartbeat message from the worker was received. Sets current liveness to HB_LIVENESS. """ self.curr_liveness = HB_LIVENESS return def is_alive(self): """Returns True when the worker is considered alive. """ return self.curr_liveness > 0 def shutdown(self): """Cleanup worker. Stops timer. """ self.hb_out_timer.stop() self.hb_out_timer = None self.stream = None return
class AppServer(object): # listen = "127.0.0.1" listen = "*" port = 5555 def __init__(self): self.ctx = zmq.Context() self.loop = IOLoop.instance() self.client_identities = {} self.server = self.ctx.socket(zmq.ROUTER) bind_addr = "tcp://%s:%s" % (self.listen, self.port) print("bind_addr=", bind_addr) self.server.bind(bind_addr) print("Server listening for new client connections at", bind_addr) self.server = ZMQStream(self.server) self.server.on_recv(self.on_recv) self.periodic = PeriodicCallback(self.periodictask, 1000) def start(self): self.periodic.start() try: self.loop.start() except KeyboardInterrupt: pass def periodictask(self): stale_clients = [] for client_id, last_seen in self.client_identities.items(): if last_seen + timedelta(seconds=10) < datetime.utcnow(): stale_clients.append(client_id) else: msg = HelloMessage() msg.send(self.server, client_id) for client_id in stale_clients: print( "Haven't received a HELO from cliient %s recently. Dropping from list of connected clients." % client_id) del self.client_identities[client_id] sys.stdout.write(".") sys.stdout.flush() def on_recv(self, msg): identity = msg[0] self.client_identities[identity] = datetime.utcnow() msg_type = msg[1] print("Received message of type %s from client ID %s!" % (msg_type, identity))
class WorkerRep(object): """Helper class to represent a worker in the broker. Instances of this class are used to track the state of the attached worker and carry the timers for incomming and outgoing heartbeats. :param proto: the worker protocol id. :type wid: str :param wid: the worker id. :type wid: str :param service: service this worker serves :type service: str :param stream: the ZMQStream used to send messages :type stream: ZMQStream """ def __init__(self, proto, wid, service, stream): self.proto = proto self.id = wid self.service = service self.curr_liveness = HB_LIVENESS self.stream = stream self.last_hb = 0 self.hb_out_timer = PeriodicCallback(self.send_hb, HB_INTERVAL) self.hb_out_timer.start() return def send_hb(self): """Called on every HB_INTERVAL. Decrements the current liveness by one. Sends heartbeat to worker. """ self.curr_liveness -= 1 msg = [self.id, b'', self.proto, b'\x04'] self.stream.send_multipart(msg) return def on_heartbeat(self): """Called when a heartbeat message from the worker was received. Sets current liveness to HB_LIVENESS. """ self.curr_liveness = HB_LIVENESS return def is_alive(self): """Returns True when the worker is considered alive. """ return self.curr_liveness > 0 def shutdown(self): """Cleanup worker. Stops timer. """ self.hb_out_timer.stop() self.hb_out_timer = None self.stream = None return
def main(pat): fname = find_connection_file(pat) with open(fname) as f: cfg = json.load(f) url = "%s://%s:%s" % (cfg.get('transport', 'tcp'), cfg['ip'], cfg['iopub_port']) session = Session(key=cfg['key']) ctx = zmq.Context.instance() sub = ctx.socket(zmq.SUB) sub.subscribe = b'' sub.connect(url) # import IPython # IPython.embed() # return stream = ZMQStream(sub) stream.on_recv(lambda msg_list: log_msg(session, msg_list)) pc = PeriodicCallback(print_time, 5 * 60 * 1000) pc.start() IOLoop.instance().start()
def __init__(self, port=5556): self.port = port self.ctx = zmq.Context() self.kvmap = {} self.loop = IOLoop.instance() # Set up our clone server sockets self.snapshot = self.ctx.socket(zmq.ROUTER) self.publisher = self.ctx.socket(zmq.PUB) self.collector = self.ctx.socket(zmq.PULL) self.snapshot.bind(f"tcp://*:{self.port:d}") self.publisher.bind(f"tcp://*:{self.port + 1:d}") self.collector.bind(f"tcp://*:{self.port + 2:d}") # Wrap sockets in ZMQStreams for IOLoop handlers self.snapshot = ZMQStream(self.snapshot) self.publisher = ZMQStream(self.publisher) self.collector = ZMQStream(self.collector) # Register our handlers with reactor self.snapshot.on_recv(self.handle_snapshot) self.collector.on_recv(self.handle_collect) self.flush_callback = PeriodicCallback(self.flush_ttl, 1000) # basic log formatting: logging.basicConfig(format="%(asctime)s %(message)s", datefmt="%Y-%m-%d %H:%M:%S", level=logging.INFO)
class BufferModule(base.ZmqProcess): def __init__(self, recv_addr, send_addr, recv_title, send_title, state_handler, period=1000): super().__init__() self.sub_stream = None self.timer = None self.recv_addr = recv_addr self.recv_title = recv_title self.state_handler = state_handler self.period = period self.callback = Callback(Publisher(send_addr, send_title), self.state_handler) def setup(self): super().setup() self.sub_stream, _ = self.stream(zmq.SUB, self.recv_addr, bind=False, subscribe=self.recv_title.encode('utf-8')) self.sub_stream.on_recv(SubStreamHandler(self.sub_stream, self.stop, self.state_handler)) self.timer = PeriodicCallback(self.callback, self.period, self.loop) def run(self): self.setup() print('Start loop!') self.timer.start() self.loop.start() def stop(self): self.loop.stop()
class DeviceRep(object): """ Helper class to represent a device to a worker """ def __init__(self, device_id, state='unknown'): self.id = device_id self.state = state self.curr_liveness = CLIENT_HB_LIVENESS self.hb_timer = PeriodicCallback(self.heartbeat, CLIENT_HB_INTERVAL) self.hb_timer.start() return def heartbeat(self): if self.curr_liveness > 0: self.curr_liveness -= 1 if self.curr_liveness == 0: self.state = 'dead' return def on_message_received(self): self.curr_liveness = CLIENT_HB_LIVENESS return def is_alive(self): return self.curr_liveness > 0 def get_state(self): return self.state def shutdown(self): self.hb_timer.stop() self.hb_timer = None
def __init__(self, context, main_ep, opt_ep=None, service_q=None): """Init MDPBroker instance. """ if service_q is None: self.service_q = ServiceQueue else: self.service_q = service_q socket = context.socket(zmq.ROUTER) socket.bind(main_ep) self.main_stream = ZMQStream(socket) self.main_stream.on_recv(self.on_message) if opt_ep: socket = context.socket(zmq.ROUTER) socket.bind(opt_ep) self.client_stream = ZMQStream(socket) self.client_stream.on_recv(self.on_message) else: self.client_stream = self.main_stream self._workers = {} # services contain the service queue and the request queue self._services = {} self._worker_cmds = { b'\x01': self.on_ready, b'\x03': self.on_reply, b'\x04': self.on_heartbeat, b'\x05': self.on_disconnect, } self.hb_check_timer = PeriodicCallback(self.on_timer, HB_INTERVAL) self.hb_check_timer.start() return
def __init__(self, settings, identity, insocket, outsocket, mgmt, frontier, log_handler, log_level, io_loop): """ Initialize the master. """ LoggingMixin.__init__(self, log_handler, log_level) self._identity = identity self._io_loop = io_loop or IOLoop.instance() self._in_stream = ZMQStream(insocket, io_loop) self._out_stream = ZMQStream(outsocket, io_loop) self._mgmt = mgmt self._frontier = frontier self._running = False self._available_workers = [] # periodically check if there are pending URIs to crawl self._periodic_update = PeriodicCallback(self._send_next_uri, settings.MASTER_PERIODIC_UPDATE_INTERVAL, io_loop=io_loop) # start this periodic callback when you are waiting for the workers to # finish self._periodic_shutdown = PeriodicCallback(self._shutdown_wait, 500, io_loop=io_loop) self._shutdown_counter = 0 self._logger.debug("zmqmaster::initialized")
def __init__(self, led_count, hw_backend, port=6606): self.ctx = zmq.Context() self.led_count = led_count self.port = port self.loop = IOLoop.instance() self.caller = PeriodicCallback(self._on_next_frame, 1000/30) self.hw_communication = hw_backend self.hw_communication.connect() self.zmq_collector = GlinAppZmqCollector(self, self.ctx) self.zmq_publisher = GlinAppZmqPublisher(self, self.ctx) # server side configuration self.config = SimpleNamespace() self.config.max_fps = 60 # current state (somehow client side configuration) self.state = SimpleNamespace() self.state.animationClasses = [] self.state.activeSceneId = None self.state.activeAnimation = None self.state.scenes = {} self.state.brightness = 1.0 self.state.sceneIdCtr = 0 self.state.mainswitch = True self.state.target_fps = 0 self.state.lastFrameSent = None
def start_ip_address_checker(self): '''Checks for possible public IP change''' if self.ob_ctx.enable_ip_checker: self.caller = PeriodicCallback(self._ip_updater_periodic_callback, 5000, ioloop.IOLoop.instance()) self.caller.start() self.log.info( "IP_CHECKER_ENABLED: Periodic IP Address Checker started.")
def startup(self): """Setup sockets and handlers.""" super(CCServer, self).startup() self.log.info("C&C server version %s starting up..", self.__version__) self.xtx = CryptoContext(self.cf) self.zctx = zmq.Context(self.zmq_nthreads) self.ioloop = IOLoop.instance() self.local_url = self.cf.get('cc-socket') self.cur_role = self.cf.get('cc-role', 'insecure') if self.cur_role == 'insecure': self.log.warning( 'CC is running in insecure mode, please add "cc-role = local" or "cc-role = remote" option to config' ) self.stat_level = self.cf.getint('cc-stats', 1) if self.stat_level < 1: self.log.warning('CC statistics level too low: %d', self.stat_level) # initialize local listen socket s = self.zctx.socket(zmq.XREP) s.setsockopt(zmq.LINGER, self.zmq_linger) s.setsockopt(zmq.HWM, self.zmq_hwm) if self.zmq_rcvbuf > 0: s.setsockopt(zmq.RCVBUF, self.zmq_rcvbuf) if self.zmq_sndbuf > 0: s.setsockopt(zmq.SNDBUF, self.zmq_sndbuf) if self.zmq_tcp_keepalive > 0: if getattr(zmq, 'TCP_KEEPALIVE', -1) > 0: s.setsockopt(zmq.TCP_KEEPALIVE, self.zmq_tcp_keepalive) s.setsockopt(zmq.TCP_KEEPALIVE_INTVL, self.zmq_tcp_keepalive_intvl) s.setsockopt(zmq.TCP_KEEPALIVE_IDLE, self.zmq_tcp_keepalive_idle) s.setsockopt(zmq.TCP_KEEPALIVE_CNT, self.zmq_tcp_keepalive_cnt) else: self.log.info("TCP_KEEPALIVE not available") s.bind(self.local_url) self.local = CCStream(s, self.ioloop, qmaxsize=self.zmq_hwm) self.local.on_recv(self.handle_cc_recv) self.handlers = {} self.routes = {} rcf = skytools.Config('routes', self.cf.filename, ignore_defs=True) for r, hnames in rcf.cf.items('routes'): self.log.info('New route: %s = %s', r, hnames) for hname in [hn.strip() for hn in hnames.split(',')]: h = self.get_handler(hname) self.add_handler(r, h) self.stimer = PeriodicCallback(self.send_stats, 30 * 1000, self.ioloop) self.stimer.start()
class TaskState (object): """ Tracks task state (with help of watchdog) """ log = skytools.getLogger ('d:TaskState') def __init__ (self, uid, name, info, ioloop, cc, xtx): self.uid = uid self.name = name self.info = info self.pidfile = info['config']['pidfile'] self.ioloop = ioloop self.cc = cc self.xtx = xtx self.timer = None self.timer_tick = 1 self.heartbeat = False self.start_time = None self.dead_since = None def start (self): self.start_time = time.time() self.timer = PeriodicCallback (self.watchdog, self.timer_tick * 1000, self.ioloop) self.timer.start() def stop (self): try: self.log.info ('Signalling %s', self.name) skytools.signal_pidfile (self.pidfile, signal.SIGINT) except: self.log.exception ('signal_pidfile failed: %s', self.pidfile) def watchdog (self): live = skytools.signal_pidfile (self.pidfile, 0) if live: self.log.debug ('%s is alive', self.name) if self.heartbeat: self.send_reply ('running') else: self.log.info ('%s is over', self.name) self.dead_since = time.time() self.timer.stop() self.timer = None self.send_reply ('stopped') def ccpublish (self, msg): assert isinstance (msg, TaskReplyMessage) cmsg = self.xtx.create_cmsg (msg) cmsg.send_to (self.cc) def send_reply (self, status, feedback = {}): msg = TaskReplyMessage( req = 'task.reply.%s' % self.uid, handler = self.info['task']['task_handler'], task_id = self.info['task']['task_id'], status = status, feedback = feedback) self.ccpublish (msg)
def __init__ (self, hname, hcf, ccscript): super(ProxyHandler, self).__init__(hname, hcf, ccscript) self.ping_remote = self.cf.getbool ("ping", False) if self.ping_remote: self.echo_stats = EchoState (self.remote_url) self.echo_timer = PeriodicCallback (self.ping, self.ping_tick * 1000, self.ioloop) self.echo_timer.start() self.log.debug ("will ping %s", self.remote_url)
class TaskState(object): """ Tracks task state (with help of watchdog) """ log = skytools.getLogger('d:TaskState') def __init__(self, uid, name, info, ioloop, cc, xtx): self.uid = uid self.name = name self.info = info self.pidfile = info['config']['pidfile'] self.ioloop = ioloop self.cc = cc self.xtx = xtx self.timer = None self.timer_tick = 1 self.heartbeat = False self.start_time = None self.dead_since = None def start(self): self.start_time = time.time() self.timer = PeriodicCallback(self.watchdog, self.timer_tick * 1000, self.ioloop) self.timer.start() def stop(self): try: self.log.info('Signalling %s', self.name) skytools.signal_pidfile(self.pidfile, signal.SIGINT) except: self.log.exception('signal_pidfile failed: %s', self.pidfile) def watchdog(self): live = skytools.signal_pidfile(self.pidfile, 0) if live: self.log.debug('%s is alive', self.name) if self.heartbeat: self.send_reply('running') else: self.log.info('%s is over', self.name) self.dead_since = time.time() self.timer.stop() self.timer = None self.send_reply('stopped') def ccpublish(self, msg): assert isinstance(msg, TaskReplyMessage) cmsg = self.xtx.create_cmsg(msg) cmsg.send_to(self.cc) def send_reply(self, status, feedback={}): msg = TaskReplyMessage(req='task.reply.%s' % self.uid, handler=self.info['task']['task_handler'], task_id=self.info['task']['task_id'], status=status, feedback=feedback) self.ccpublish(msg)
class AppClient(object): # where to connect to - point to another IP or hostname:5556 if running server on another machine endpoint = "tcp://127.0.0.1:5556" # crypto = True means 'use CurveZMQ'. False means don't. Must match server. crypto = True def __init__(self): self.ctx = zmq.Context() self.loop = IOLoop.instance() self.client = self.ctx.socket(zmq.DEALER) if self.crypto: self.keymonkey = KeyMonkey("client") self.client = self.keymonkey.setupClient(self.client, self.endpoint, "server") self.client.connect(self.endpoint) print("Connecting to", self.endpoint) self.client = ZMQStream(self.client) self.client.on_recv(self.on_recv) self.periodic = PeriodicCallback(self.periodictask, 1000) self.last_recv = None def periodictask(self): if not self.last_recv or self.last_recv + timedelta( seconds=5) < datetime.utcnow(): print( "Hmmm... haven't heard from the server in 5 seconds... Server unresponsive." ) print("Sending HELLO to server") msg = HelloMessage() msg.send(self.client) def start(self): self.periodic.start() try: self.loop.start() except KeyboardInterrupt: pass def on_recv(self, msg): self.last_recv = datetime.utcnow() print("Received a message of type %s from server!" % msg[0])
def setup(self): super().setup() self.sub_stream, _ = self.stream( zmq.SUB, self.recv_addr, bind=False, subscribe=self.recv_title.encode('utf-8')) self.sub_stream.on_recv( SubStreamHandler(self.sub_stream, self.stop, self.state_handler)) self.timer = PeriodicCallback(self.callback, self.period, self.loop)
def __init__(self, proto, wid, service, stream): self.proto = proto self.id = wid self.service = service self.curr_liveness = HB_LIVENESS self.stream = stream self.last_hb = 0 self.hb_out_timer = PeriodicCallback(self.send_hb, HB_INTERVAL) self.hb_out_timer.start() return
class AppClient(object): def __init__(self): self.ctx = zmq.Context() self.loop = IOLoop.instance() self.endpoint = "tcp://127.0.0.1:5556" self.client = self.ctx.socket(zmq.DEALER) self.client.setsockopt( zmq.LINGER, 0 ) # Without linger and timeouts you might have problems when closing context self.client.setsockopt(zmq.RCVTIMEO, 5000) # 5s self.client.setsockopt(zmq.SNDTIMEO, 5000) print("Connecting to", self.endpoint) self.client.connect(self.endpoint) self.client = ZMQStream(self.client) self.client.on_recv(self.on_recv) self.periodic = PeriodicCallback(self.periodictask, 1000) self.last_recv = None def disconnect(self): if self.ctx is not None: try: self.periodic.stop() print("Closing socket and context") self.client.close() self.ctx.term() except Exception as e: print(e) def periodictask(self): if self.client is None: return if not self.last_recv or self.last_recv + timedelta( seconds=5) < datetime.utcnow(): print("No data from remote (5s)... [ping]") print("Sending HELLO to server") msg = HelloMessage() msg.send(self.client) def start(self): try: self.periodic.start() self.loop.start() msg = HelloMessage() msg.send(self.client) except KeyboardInterrupt: print("\n\nCtrl+C detected\n") except Exception as E: print("Error detected") print(str(E)) finally: self.disconnect() def on_recv(self, msg): self.last_recv = datetime.utcnow() print("Received a message of type %s from server!" % msg[0])
def __init__ (self, hname, hcf, ccscript): super(Delay, self).__init__(hname, hcf, ccscript) self.fwd_hname = self.cf.get ('forward-to') self.delay = self.cf.getint ('delay', 0) self.fwd_handler = ccscript.get_handler (self.fwd_hname) self.queue = collections.deque() self.timer = PeriodicCallback (self.process_queue, self.tick, self.ioloop) self.timer.start()
def __init__(self, *args): super(TaskRouter, self).__init__(*args) self.route_map = {} self.reply_map = {} # 1 hr? XXX self.route_lifetime = self.cf.getint ('route-lifetime', 1 * 60 * 60) self.reply_timeout = self.cf.getint ('reply-timeout', 5 * 60) self.maint_period = self.cf.getint ('maint-period', 1 * 60) self.timer = PeriodicCallback(self.do_maint, self.maint_period*1000, self.ioloop) self.timer.start()
class AppServer(object): def __init__(self): self.listen = "127.0.0.1" self.port = 5556 self.ctx = zmq.Context() self.loop = IOLoop.instance() self.client_identities = {} self.server = self.ctx.socket(zmq.ROUTER) self.server.setsockopt( zmq.LINGER, 0 ) # Without linger and timeouts you might have problems when closing context self.server.setsockopt(zmq.RCVTIMEO, 5000) # 5s self.server.setsockopt(zmq.SNDTIMEO, 5000) bind_addr = "tcp://%s:%s" % (self.listen, self.port) self.server.bind(bind_addr) print("Server listening for new client connections at", bind_addr) self.server = ZMQStream(self.server) self.server.on_recv(self.on_recv) self.periodic = PeriodicCallback(self.periodictask, 1000) def start(self): self.periodic.start() try: self.loop.start() except KeyboardInterrupt: self.periodic.stop() print("\nClosing socket and context\n") self.server.close() self.ctx.term() def periodictask(self): stale_clients = [] for client_id, last_seen in self.client_identities.items(): if last_seen + timedelta(seconds=10) < datetime.utcnow(): stale_clients.append(client_id) else: msg = HelloMessage() msg.send(self.server, client_id) for client_id in stale_clients: print( "\nHaven't received a HELO from client %s recently. Dropping from list of connected clients." % client_id) del self.client_identities[client_id] sys.stdout.write(".") sys.stdout.flush() def on_recv(self, msg): identity = msg[ 0] # This contains client id (socket handle), use to reply it back self.client_identities[identity] = datetime.utcnow() msg_type = msg[1] print("Received message of type %s from client ID %s!" % (msg_type, identity))
def __init__(self, context, main_ep, opt_ep=None, service_q=None, data_q=None): """Init MNBroker instance. """ if service_q is None: self.service_q = ServiceQueue else: self.service_q = service_q if data_q is None: self.data_q = ServiceQueue else: self.data_q = data_q socket = context.socket(zmq.ROUTER) socket.bind(main_ep) socket.setsockopt(zmq.IDENTITY, b'BROKER') self.main_stream = ZMQStream(socket) self.main_stream.on_recv(self.on_message) if opt_ep: socket = context.socket(zmq.ROUTER) socket.bind(opt_ep) self.client_stream = ZMQStream(socket) self.client_stream.on_recv(self.on_message) else: self.client_stream = self.main_stream # TODO: merge worker_tracker and info self._workers = {} self._workers_info = {} self._services = { } # TODO: each worker must have his own request queue self._worker_cmds = { MSG_READY: self.on_ready, MSG_REPLY: self.on_reply, MSG_HEARTBEAT: self.on_heartbeat, MSG_DISCONNECT: self.on_disconnect, } self._local_cmds = { MSG_WINFO: self.get_workers_info, } self.hb_check_timer = PeriodicCallback(self.on_timer, HB_INTERVAL) self.hb_check_timer.start() self.hb_get_winfo = PeriodicCallback(self.collect_workers_info, HB_INTERVAL) self.hb_get_winfo.start() self.register_worker_info(self.main_stream.getsockopt( zmq.IDENTITY)) # register this instance _LOG.info("Broker initialized and can be found at '%s'" % main_ep) return
def __init__(self, proto, wid, service, stream): self.proto = proto self.id = wid self.service = service self.curr_retries = HB_RETRIES self.stream = stream self.last_hb = 0 self.free_space = 0 self.data = [] self.hb_out_timer = PeriodicCallback(self.send_hb, HB_INTERVAL) self.hb_out_timer.start() return
def __init__(self, history_filepath=None): self._logger = logging.getLogger(__name__) if history_filepath is None: self._logger.info(' getting default history filepath.') history_filepath = _get_default_history_filepath() self._logger.info(' history filepath %s', history_filepath) self.history = FileHistory(history_filepath) self.messaging = _Messaging('shell', run_control_loop=True) # TODO: Cleanup this API access self.messaging._heartbeat_reciever.identity_callback = self._identity_callback self._thread = _Thread(target=self.messaging.start, daemon=True) self._bot_status_monitor = PeriodicCallback(self._monitor_bot_state, 1000) self.shebangs = [ '!', ] self.command_observer = CommandObserver(self.messaging, prompt=self) commands = self.command_observer._get_commands() self._word_completer = WordCompleter(commands, WORD=_WORD) self._services_completer = ServiceCompleter(self._word_completer) self.author_interface = AuthorInterface(self._word_completer, self.messaging) self.service_interface = ServiceInterface(self._word_completer, self.messaging) self.entity_interface = EntityInterface(self.author_interface, self.service_interface) self._bot_status = '' super().__init__(message='vexbot: ', history=self.history, completer=self._services_completer, enable_system_prompt=True, enable_suspend=True, enable_open_in_editor=True, complete_while_typing=False) self.print_observer = PrintObserver(self.app) self._print_subscription = self.messaging.chatter.subscribe( self.print_observer) self.messaging.chatter.subscribe(LogObserver()) self.messaging.command.subscribe(self.command_observer) self.messaging.command.subscribe( ServicesObserver(self._identity_setter, self._set_service_completion))
def _create_stream(self): """Helper to create the socket and the stream. """ socket = self.context.socket(zmq.DEALER) ioloop = IOLoop.instance() self.stream = ZMQStream(socket, ioloop) self.stream.on_recv(self._on_message) self.stream.socket.setsockopt(zmq.LINGER, 0) self.stream.connect(self.endpoint) self.ticker = PeriodicCallback(self._tick, self.HB_INTERVAL) self._send_ready() self.ticker.start() return
def loop(): self.__ioloop = IOLoop() queue_socket = self.__context.socket(zmq.PULL) queue_socket.connect(self.message_address) queue_stream = ZMQStream(queue_socket, self.__ioloop) worker_socket = self.__context.socket(zmq.DEALER) for address in self.active_connections: worker_socket.connect(address) worker_stream = ZMQStream(worker_socket, self.__ioloop) def receive_response(message, response_override=None): self.__queued_messages.pop(message[1], None) self.__message_timeouts.pop(message[1], None) callback = self.__callbacks.pop(message[1], None) if callback: try: callback(response_override or self.loads(self.decompress(message[2]))) except Exception as e: self.log_error(e) callback({'error': e}) worker_stream.on_recv(receive_response) def queue_message(message): if message[0]: if message[0] == WORKER_SOCKET_CONNECT and message[2] not in self.active_connections: self.active_connections.add(message[2]) worker_stream.socket.connect(message[2]) elif message[0] == WORKER_SOCKET_DISCONNECT and message[2] in self.active_connections: self.active_connections.remove(message[2]) worker_stream.socket.disconnect(message[2]) return self.__queued_messages[message[1]] = (time(), message) try: worker_stream.send_multipart(message) except Exception as e: self.log_error(e) queue_stream.on_recv(queue_message) def timeout_message(): now = time() for message, retry in [(item[1], self.__message_auto_retry.get(item[1][1], self.__auto_retry)) for item, t in ((i, self.__message_timeouts.get(i[1][1], self.__timeout)) for i in self.__queued_messages.itervalues()) if t >= 0 and (item[0] + t < now)]: if retry: logging.info('Worker timeout, requeuing ' + message[1]) queue_message(message) else: receive_response(('', message[1]), {'error': 'timeout'}) timeout_callback = PeriodicCallback(timeout_message, int(abs(self.__timeout * 1000.0)), io_loop = self.__ioloop) timeout_callback.start() self.__ioloop.start() self.__thread = None
def __init__(self): self.ctx = zmq.Context() self.loop = IOLoop.instance() self.client = self.ctx.socket(zmq.DEALER) if self.crypto: self.keymonkey = KeyMonkey("client") self.client = self.keymonkey.setupClient(self.client, self.endpoint, "server") self.client.connect(self.endpoint) print("Connecting to", self.endpoint) self.client = ZMQStream(self.client) self.client.on_recv(self.on_recv) self.periodic = PeriodicCallback(self.periodictask, 1000) self.last_recv = None
def __init__(self, context, main_ep, client_ep, hb_ep, service_q=None): """Init MDPBroker instance. """ if service_q is None: self.service_q = ServiceQueue else: self.service_q = service_q # # Setup the zmq sockets. # socket = context.socket(zmq.ROUTER) socket.bind(main_ep) self.main_stream = ZMQStream(socket) self.main_stream.on_recv(self.on_message) socket = context.socket(zmq.ROUTER) socket.bind(client_ep) self.client_stream = ZMQStream(socket) self.client_stream.on_recv(self.on_message) socket = context.socket(zmq.ROUTER) socket.bind(hb_ep) self.hb_stream = ZMQStream(socket) self.hb_stream.on_recv(self.on_message) self._workers = {} # # services contain the service queue and the request queue # self._services = {} # # Mapping of worker commands and callbacks. # self._worker_cmds = { W_READY: self.on_ready, W_REPLY: self.on_reply, W_HEARTBEAT: self.on_heartbeat, W_DISCONNECT: self.on_disconnect, } # # 'Cleanup' timer for workers without heartbeat. # self.hb_check_timer = PeriodicCallback(self.on_timer, HB_INTERVAL) self.hb_check_timer.start()
class DeviceApp(object): # endpoint = "tcp://127.0.0.1:5556" # endpoint = "tcp://localhost:5556" endpoint = "tcp://165.227.24.226:5556" def __init__(self): self.ctx = zmq.Context() self.loop = IOLoop.instance() self.client = self.ctx.socket(zmq.DEALER) self.client.connect(self.endpoint) print("Connecting to", self.endpoint) self.client = ZMQStream(self.client) self.client.on_recv(self.on_recv) self.periodic = PeriodicCallback(self.periodictask, 1000) self.last_recv = None def periodictask(self): if not (self.last_recv) or (self.last_recv + timedelta(seconds=5)) < datetime.utcnow(): print( "Hmmm... haven't heard from the server in 5 seconds... Server unresponsive." ) print("Sending HELLO to server") msg = HelloMessage() print("msg=", msg) msg.send(self.client) def start(self): self.periodic.start() try: self.loop.start() except KeyboardInterrupt: pass def on_recv(self, msg): self.last_recv = datetime.utcnow() print("Received a message of type %s from server!" % msg[0])
def __init__(self, primary=True, ports=(5556, 5566)): self.primary = primary if primary: self.port, self.peer = ports frontend = "tcp://*:5003" backend = "tcp://localhost:5004" self.kvmap = {} else: self.peer, self.port = ports frontend = "tcp://*:5004" backend = "tcp://localhost:5003" self.ctx = zmq.Context.instance() self.pending = [] self.bstar = BinaryStar(primary, frontend, backend) self.bstar.register_voter("tcp://*:%i" % self.port, zmq.ROUTER, self.handle_snapshot) # Set up our clone server sockets self.publisher = self.ctx.socket(zmq.PUB) self.collector = self.ctx.socket(zmq.SUB) self.collector.setsockopt(zmq.SUBSCRIBE, b'') self.publisher.bind("tcp://*:%d" % (self.port + 1)) self.collector.bind("tcp://*:%d" % (self.port + 2)) # Set up our own clone client interface to peer self.subscriber = self.ctx.socket(zmq.SUB) self.subscriber.setsockopt(zmq.SUBSCRIBE, b'') self.subscriber.connect("tcp://localhost:%d" % (self.peer + 1)) # Register state change handlers self.bstar.master_callback = self.become_master self.bstar.slave_callback = self.become_slave # Wrap sockets in ZMQStreams for IOLoop handlers self.publisher = ZMQStream(self.publisher) self.subscriber = ZMQStream(self.subscriber) self.collector = ZMQStream(self.collector) # Register our handlers with reactor self.collector.on_recv(self.handle_collect) self.flush_callback = PeriodicCallback(self.flush_ttl, 1000) self.hugz_callback = PeriodicCallback(self.send_hugz, 1000) # basic log formatting: logging.basicConfig(format="%(asctime)s %(message)s", datefmt="%Y-%m-%d %H:%M:%S", level=logging.INFO)
def start(self): loop = self.loop loop.add_handler(self.udp.handle.fileno(), self.handle_beacon, loop.READ) stream = ZMQStream(self.pipe, loop) stream.on_recv(self.control_message) pc = PeriodicCallback(self.send_ping, PING_INTERVAL * 1000, loop) pc.start() pc = PeriodicCallback(self.reap_peers, PING_INTERVAL * 1000, loop) pc.start() pc = PeriodicCallback(self.send_ports, PING_INTERVAL * 1000, loop) pc.start() loop.start()
def __init__(self, port=5556): self.port = port self.ctx = zmq.Context() self.kvmap = {} self.loop = IOLoop.instance() # Set up our clone server sockets self.snapshot = self.ctx.socket(zmq.ROUTER) self.publisher = self.ctx.socket(zmq.PUB) self.collector = self.ctx.socket(zmq.PULL) self.snapshot.bind("tcp://*:%d" % self.port) self.publisher.bind("tcp://*:%d" % (self.port + 1)) self.collector.bind("tcp://*:%d" % (self.port + 2)) # Wrap sockets in ZMQStreams for IOLoop handlers self.snapshot = ZMQStream(self.snapshot) self.publisher = ZMQStream(self.publisher) self.collector = ZMQStream(self.collector) # Register our handlers with reactor self.snapshot.on_recv(self.handle_snapshot) self.collector.on_recv(self.handle_collect) self.flush_callback = PeriodicCallback(self.flush_ttl, 1000) # basic log formatting: logging.basicConfig(format="%(asctime)s %(message)s", datefmt="%Y-%m-%d %H:%M:%S", level=logging.INFO)
def __init__(self, broker, service, io_loop=None): """Create and setup an MDP worker. @param broker A string containing the broker's URL @param service A string containing the service name @param io_loop An existing I/O loop object. If None, the default will be used. """ self.service=service self._broker = broker self.ctx = zmq.Context() sock = self.ctx.socket(zmq.DEALER) ZMQStream.__init__(self, sock, io_loop) # last watchdog timer tick self.watchdog = 0 # connection callback one-shot self._conncb = DelayedCallback(self.send_ready, 3000, self.io_loop) # heartbeat callback..runs continuous when connected self._hbcb = PeriodicCallback(self.send_heartbeat, 2000, self.io_loop) # number of connection attempts self._conn_attempt = 0 # waiting to connect state self._waiting_to_connect = True # have we been disconnected? (flags a reconnect attempt) self.disconnected = False # connect the socket and send a READY when the io_loop starts self.connect(self._broker) self._conncb.start()
class UdpListener (CCDaemon): """ UDP server to handle UDP stream. """ log = skytools.getLogger ('d:UdpListener') def reload (self): super(UdpListener, self).reload() self.listen_host = self.cf.get ('listen-host') self.listen_port = self.cf.getint ('listen-port') self.stats_period = self.cf.getint ('stats-period', 30) def startup (self): super(UdpListener, self).startup() # plugins should be ready before we start receiving udp stream self.init_plugins() self.listen_addr = (self.listen_host, self.listen_port) self.sock = socket.socket (socket.AF_INET, socket.SOCK_DGRAM) self.sock.setblocking (0) try: self.sock.bind (self.listen_addr) except Exception, e: self.log.exception ("failed to bind to %s - %s", self.listen_addr, e) raise self.ioloop = IOLoop.instance() callback = functools.partial (self.handle_udp, self.sock) self.ioloop.add_handler (self.sock.fileno(), callback, self.ioloop.READ) self.timer_stats = PeriodicCallback (self.send_stats, self.stats_period * 1000, self.ioloop) self.timer_stats.start()
def __init__ (self, hname, hcf, ccscript): super(TailWriter, self).__init__(hname, hcf, ccscript) self.files = {} self.workers = [] self.wparams = {} # passed to workers self.wparams['dstdir'] = self.cf.getfile ('dstdir') self.wparams['host_subdirs'] = self.cf.getbool ('host-subdirs', 0) self.wparams['maint_period'] = self.cf.getint ('maint-period', 3) self.wparams['write_compressed'] = self.cf.get ('write-compressed', '') assert self.wparams['write_compressed'] in [None, '', 'no', 'keep', 'yes'] if self.wparams['write_compressed'] in ('keep', 'yes'): self.log.info ("position checking not supported for compressed files") if self.wparams['write_compressed'] == 'yes': self.wparams['compression'] = self.cf.get ('compression', '') if self.wparams['compression'] not in ('gzip', 'bzip2'): self.log.error ("unsupported compression: %s", self.wparams['compression']) self.wparams['compression_level'] = self.cf.getint ('compression-level', '') self.wparams['buf_maxbytes'] = cc.util.hsize_to_bytes (self.cf.get ('buffer-bytes', '1 MB')) if self.wparams['buf_maxbytes'] < BUF_MINBYTES: self.log.info ("buffer-bytes too low, adjusting: %i -> %i", self.wparams['buf_maxbytes'], BUF_MINBYTES) self.wparams['buf_maxbytes'] = BUF_MINBYTES # initialise sockets for communication with workers self.dealer_stream, self.dealer_url = self.init_comm (zmq.XREQ, 'inproc://workers-dealer', self.dealer_on_recv) self.router_stream, self.router_url = self.init_comm (zmq.XREP, 'inproc://workers-router', self.router_on_recv) self.launch_workers() self.timer_maint = PeriodicCallback (self.do_maint, self.wparams['maint_period'] * 1000, self.ioloop) self.timer_maint.start()
def __init__(self, context, main_ep, opt_ep=None, worker_q=None): """Init MDPBroker instance. """ socket = context.socket(zmq.XREP) socket.bind(main_ep) self.main_stream = ZMQStream(socket) self.main_stream.on_recv(self.on_message) if opt_ep: socket = context.socket(zmq.XREP) socket.bind(opt_ep) self.client_stream = ZMQStream(socket) self.client_stream.on_recv(self.on_message) else: self.client_stream = self.main_stream self._workers = {} # services contain the worker queue and the request queue self._services = {} self._worker_cmds = { '\x01': self.on_ready, '\x03': self.on_reply, '\x04': self.on_heartbeat, '\x05': self.on_disconnect, } self.hb_check_timer = PeriodicCallback(self.on_timer, HB_INTERVAL) self.hb_check_timer.start() return
def __init__(self, device_id, state='unknown'): self.id = device_id self.state = state self.curr_liveness = CLIENT_HB_LIVENESS self.hb_timer = PeriodicCallback(self.heartbeat, CLIENT_HB_INTERVAL) self.hb_timer.start() return
def __init__(self, primary=True, ports=(5556,5566)): self.primary = primary if primary: self.port, self.peer = ports frontend = "tcp://*:5003" backend = "tcp://localhost:5004" self.kvmap = {} else: self.peer, self.port = ports frontend = "tcp://*:5004" backend = "tcp://localhost:5003" self.ctx = zmq.Context.instance() self.pending = [] self.bstar = BinaryStar(primary, frontend, backend) self.bstar.register_voter("tcp://*:%i" % self.port, zmq.ROUTER, self.handle_snapshot) # Set up our clone server sockets self.publisher = self.ctx.socket(zmq.PUB) self.collector = self.ctx.socket(zmq.SUB) self.collector.setsockopt(zmq.SUBSCRIBE, b'') self.publisher.bind("tcp://*:%d" % (self.port + 1)) self.collector.bind("tcp://*:%d" % (self.port + 2)) # Set up our own clone client interface to peer self.subscriber = self.ctx.socket(zmq.SUB) self.subscriber.setsockopt(zmq.SUBSCRIBE, b'') self.subscriber.connect("tcp://localhost:%d" % (self.peer + 1)) # Register state change handlers self.bstar.master_callback = self.become_master self.bstar.slave_callback = self.become_slave # Wrap sockets in ZMQStreams for IOLoop handlers self.publisher = ZMQStream(self.publisher) self.subscriber = ZMQStream(self.subscriber) self.collector = ZMQStream(self.collector) # Register our handlers with reactor self.collector.on_recv(self.handle_collect) self.flush_callback = PeriodicCallback(self.flush_ttl, 1000) self.hugz_callback = PeriodicCallback(self.send_hugz, 1000) # basic log formatting: logging.basicConfig(format="%(asctime)s %(message)s", datefmt="%Y-%m-%d %H:%M:%S", level=logging.INFO)
def setup(self): super(DemoApp, self).setup() self.pub, self.pub_addr = self.stream(zmq.PUB, 'tcp://127.0.0.1:%(port)s', True) self.sub, sub_addr = self.stream(zmq.SUB, self.pub_addr, False, callback=DemoHandler()) self.heartbeat = PeriodicCallback(self.ping, 1000, self.loop)
def run(self): self.received_count = 0 ctx, io_loop, self.socks, streams = get_run_context(self.sock_configs) self.io_loop = io_loop if self.request_delay > 0: periodic_callback = PeriodicCallback(self.do_request, self.request_delay * 1000., io_loop) io_loop.add_timeout(timedelta(seconds=0.1), lambda: periodic_callback.start()) else: for i in range(self.target_count): io_loop.add_callback(self.do_request) try: io_loop.start() except KeyboardInterrupt: pass
def __init__(self, my_ip, my_port, market_id, bm_user=None, bm_pass=None, bm_port=None, seed_mode=0, dev_mode=False): self._log = logging.getLogger('[%s] %s' % (market_id, self.__class__.__name__)) requests_log = logging.getLogger("requests") requests_log.setLevel(logging.WARNING) # Connect to database MONGODB_URI = 'mongodb://localhost:27017' _dbclient = MongoClient() self._db = _dbclient.openbazaar self._bitmessage_api = None if (bm_user, bm_pass, bm_port) != (None, None, None): if not self._connect_to_bitmessage(bm_user, bm_pass, bm_port): self._log.info('Bitmessage not available') self._market_id = market_id self.nick_mapping = {} self._uri = "tcp://%s:%s" % (my_ip, my_port) self._ip = my_ip self._nickname = "" # Set up self._setup_settings() self._dht = DHT(self, market_id, self.settings) self._myself = ec.ECC(pubkey=self.pubkey.decode('hex'), privkey=self.secret.decode('hex'), curve='secp256k1') TransportLayer.__init__(self, market_id, my_ip, my_port, self.guid, self._nickname) # Set up callbacks self.add_callback('hello', self._ping) self.add_callback('findNode', self._findNode) self.add_callback('findNodeResponse', self._findNodeResponse) self.add_callback('store', self._storeValue) self.listen(self.pubkey) def cb(): r = requests.get(r'http://icanhazip.com') if r and hasattr(r,'text'): ip = r.text ip = ip.strip(' \t\n\r') if ip != self._ip: self._ip = ip self._uri = 'tcp://%s:%s' % (self._ip, self._port) self.stream.close() self.listen(self.pubkey) else: self._log.error('Could not get ip') if seed_mode == 0 and not dev_mode: # Check IP periodically for changes self.caller = PeriodicCallback(cb, 5000, ioloop.IOLoop.instance()) self.caller.start()
def _create_stream(self): """Overidden _create_stream for MDPWorker Adds the device_timer to manage connected devices """ self.device_timer = PeriodicCallback(self.device_watcher, CLIENT_HB_INTERVAL) self.device_timer.start() super(DeviceServiceManager, self)._create_stream() return
class Delay (CCHandler): """ Delays all received messages, then dispatches them to another handler. """ CC_ROLES = ['local', 'remote'] log = skytools.getLogger ('h:Delay') tick = 250 # ms def __init__ (self, hname, hcf, ccscript): super(Delay, self).__init__(hname, hcf, ccscript) self.fwd_hname = self.cf.get ('forward-to') self.delay = self.cf.getint ('delay', 0) self.fwd_handler = ccscript.get_handler (self.fwd_hname) self.queue = collections.deque() self.timer = PeriodicCallback (self.process_queue, self.tick, self.ioloop) self.timer.start() def handle_msg (self, cmsg): """ Got message from client -- queue it """ self.queue.append ((time.time() + self.delay, cmsg)) def process_queue (self): now = time.time() try: while (self.queue[0][0] <= now): at, cmsg = self.queue.popleft() size = cmsg.get_size() try: self.fwd_handler.handle_msg (cmsg) stat = 'ok' except Exception: self.log.exception ('crashed, dropping msg: %s', cmsg.get_dest()) stat = 'crashed' self.stat_inc ('delay.count') self.stat_inc ('delay.bytes', size) self.stat_inc ('delay.count.%s' % stat) self.stat_inc ('delay.bytes.%s' % stat, size) except IndexError: pass
def __init__(self, wid, service, stream): self.id = wid self.service = service self.multicasts = [] self.curr_liveness = HB_LIVENESS self.stream = stream self.last_hb = 0 self.hb_out_timer = PeriodicCallback(self.send_hb, HB_INTERVAL) self.hb_out_timer.start() return
def start(self): self.frontend.on_recv(self.handle_frontend) self.backend.on_recv(self.handle_backend) self.period = PeriodicCallback(self.workers.purge, self.heartbeat*1000) self.period.start() try: self.frontend.send(PPP_READY) #self.loop.start() except KeyboardInterrupt: times_str('ctrlc1')
def startup(self): """Setup sockets and handlers.""" super(CCServer, self).startup() self.log.info("C&C server version %s starting up..", self.__version__) self.xtx = CryptoContext(self.cf) self.zctx = zmq.Context(self.zmq_nthreads) self.ioloop = IOLoop.instance() self.local_url = self.cf.get("cc-socket") self.cur_role = self.cf.get("cc-role", "insecure") if self.cur_role == "insecure": self.log.warning( 'CC is running in insecure mode, please add "cc-role = local" or "cc-role = remote" option to config' ) self.stat_level = self.cf.getint("cc-stats", 1) if self.stat_level < 1: self.log.warning("CC statistics level too low: %d", self.stat_level) # initialize local listen socket s = self.zctx.socket(zmq.XREP) s.setsockopt(zmq.LINGER, self.zmq_linger) s.setsockopt(zmq.HWM, self.zmq_hwm) if self.zmq_rcvbuf > 0: s.setsockopt(zmq.RCVBUF, self.zmq_rcvbuf) if self.zmq_sndbuf > 0: s.setsockopt(zmq.SNDBUF, self.zmq_sndbuf) if self.zmq_tcp_keepalive > 0: if getattr(zmq, "TCP_KEEPALIVE", -1) > 0: s.setsockopt(zmq.TCP_KEEPALIVE, self.zmq_tcp_keepalive) s.setsockopt(zmq.TCP_KEEPALIVE_INTVL, self.zmq_tcp_keepalive_intvl) s.setsockopt(zmq.TCP_KEEPALIVE_IDLE, self.zmq_tcp_keepalive_idle) s.setsockopt(zmq.TCP_KEEPALIVE_CNT, self.zmq_tcp_keepalive_cnt) else: self.log.info("TCP_KEEPALIVE not available") s.bind(self.local_url) self.local = CCStream(s, self.ioloop, qmaxsize=self.zmq_hwm) self.local.on_recv(self.handle_cc_recv) self.handlers = {} self.routes = {} rcf = skytools.Config("routes", self.cf.filename, ignore_defs=True) for r, hnames in rcf.cf.items("routes"): self.log.info("New route: %s = %s", r, hnames) for hname in [hn.strip() for hn in hnames.split(",")]: h = self.get_handler(hname) self.add_handler(r, h) self.stimer = PeriodicCallback(self.send_stats, 30 * 1000, self.ioloop) self.stimer.start()
def start(self): """Initialize and start the event loop. Listen for ZMQ control messages.""" ctx = zmq.Context() socket = ctx.socket(zmq.PAIR) socket.bind("tcp://*:{}".format(settings.ZMQ_CONTROL_PORT)) logserver = LogServer() logserver.listen(settings.TCP_LOGGING_PORT) loop = self.loop stream = ZMQStream(socket, loop) stream.on_recv(self.handle_ctrl_msg) # TODO: Stop loop and restart on CONFIG reread just to get possible new # EVENT_POLL_INTERVAL setting? pc = PeriodicCallback(self.chime, self.EVENT_POLL_INTERVAL * 1E3, loop) pc.start() loop.start()
def startup(self): super(TaskRunner, self).startup() self.ioloop = IOLoop.instance() self.connect_cc() self.ccs = CCStream (self.cc, self.ioloop, qmaxsize = self.zmq_hwm) self.ccs.on_recv(self.handle_cc_recv) self.local_id = self.cf.get('local-id', self.hostname) self.reg_period = self.cf.getint('reg-period', 5 * 60) self.maint_period = self.cf.getint('maint-period', 60) self.grace_period = self.cf.getint('task-grace-period', 15 * 60) self.task_heartbeat = self.cf.getboolean('task-heartbeat', False) self.tasks = {} self.periodic_reg() self.timer_reg = PeriodicCallback (self.periodic_reg, self.reg_period * 1000, self.ioloop) self.timer_reg.start() self.timer_maint = PeriodicCallback (self.do_maint, self.maint_period * 1000, self.ioloop) self.timer_maint.start()