def __init__(self, settings, identity, insocket, outsocket, mgmt, frontier, log_handler, log_level, io_loop): """ Initialize the master. """ LoggingMixin.__init__(self, log_handler, log_level) self._identity = identity self._io_loop = io_loop or IOLoop.instance() self._in_stream = ZMQStream(insocket, io_loop) self._out_stream = ZMQStream(outsocket, io_loop) self._mgmt = mgmt self._frontier = frontier self._running = False self._available_workers = [] # periodically check if there are pending URIs to crawl self._periodic_update = PeriodicCallback(self._send_next_uri, settings.MASTER_PERIODIC_UPDATE_INTERVAL, io_loop=io_loop) # start this periodic callback when you are waiting for the workers to # finish self._periodic_shutdown = PeriodicCallback(self._shutdown_wait, 500, io_loop=io_loop) self._shutdown_counter = 0 self._logger.debug("zmqmaster::initialized")
def __init__(self, frontier, data_in_sock='ipc:///tmp/robot-data-w2m.sock', data_out_sock='ipc:///tmp/robot-data-m2w.sock', msg_in_sock='ipc:///tmp/robot-msg-w2m.sock', msg_out_sock='ipc:///tmp/robot-msg-m2w.sock', io_loop=None): self.identity = 'master:%s:%s' % (socket.gethostname(), os.getpid()) context = zmq.Context() self._io_loop = io_loop or IOLoop.instance() self._in_socket = context.socket(zmq.SUB) self._in_socket.setsockopt(zmq.SUBSCRIBE, '') self._in_socket.bind(data_in_sock) self._in_stream = ZMQStream(self._in_socket, io_loop) self._out_socket = context.socket(zmq.PUSH) self._out_socket.bind(data_out_sock) self._out_stream = ZMQStream(self._out_socket, io_loop) self._online_workers = set() self._running = False self._updater = PeriodicCallback(self._send_next, 100, io_loop=io_loop) self._reloader = PeriodicCallback(self.reload, 1000, io_loop=io_loop) self.frontier = frontier self.messenger = ServerMessenger(msg_in_sock, msg_out_sock, context, io_loop)
def start(self): loop = self.loop loop.add_handler(self.udp.handle.fileno(), self.handle_beacon, loop.READ) stream = ZMQStream(self.pipe, loop) stream.on_recv(self.control_message) pc = PeriodicCallback(self.send_ping, PING_INTERVAL * 1000, loop) pc.start() pc = PeriodicCallback(self.reap_peers, PING_INTERVAL * 1000, loop) pc.start() loop.start()
def __init__(self, context, main_ep, opt_ep=None, service_q=None, data_q=None): """Init MNBroker instance. """ if service_q is None: self.service_q = ServiceQueue else: self.service_q = service_q if data_q is None: self.data_q = ServiceQueue else: self.data_q = data_q socket = context.socket(zmq.ROUTER) socket.bind(main_ep) socket.setsockopt(zmq.IDENTITY, b'BROKER') self.main_stream = ZMQStream(socket) self.main_stream.on_recv(self.on_message) if opt_ep: socket = context.socket(zmq.ROUTER) socket.bind(opt_ep) self.client_stream = ZMQStream(socket) self.client_stream.on_recv(self.on_message) else: self.client_stream = self.main_stream # TODO: merge worker_tracker and info self._workers = {} self._workers_info = {} self._services = { } # TODO: each worker must have his own request queue self._worker_cmds = { MSG_READY: self.on_ready, MSG_REPLY: self.on_reply, MSG_HEARTBEAT: self.on_heartbeat, MSG_DISCONNECT: self.on_disconnect, } self._local_cmds = { MSG_WINFO: self.get_workers_info, } self.hb_check_timer = PeriodicCallback(self.on_timer, HB_INTERVAL) self.hb_check_timer.start() self.hb_get_winfo = PeriodicCallback(self.collect_workers_info, HB_INTERVAL) self.hb_get_winfo.start() self.register_worker_info(self.main_stream.getsockopt( zmq.IDENTITY)) # register this instance _LOG.info("Broker initialized and can be found at '%s'" % main_ep) return
def __init__(self, primary=True, ports=(5556, 5566)): self.primary = primary if primary: self.port, self.peer = ports frontend = "tcp://*:5003" backend = "tcp://localhost:5004" self.kvmap = {} else: self.peer, self.port = ports frontend = "tcp://*:5004" backend = "tcp://localhost:5003" self.ctx = zmq.Context.instance() self.pending = [] self.bstar = BinaryStar(primary, frontend, backend) self.bstar.register_voter("tcp://*:%i" % self.port, zmq.ROUTER, self.handle_snapshot) # Set up our clone server sockets self.publisher = self.ctx.socket(zmq.PUB) self.collector = self.ctx.socket(zmq.SUB) self.collector.setsockopt(zmq.SUBSCRIBE, b'') self.publisher.bind("tcp://*:%d" % (self.port + 1)) self.collector.bind("tcp://*:%d" % (self.port + 2)) # Set up our own clone client interface to peer self.subscriber = self.ctx.socket(zmq.SUB) self.subscriber.setsockopt(zmq.SUBSCRIBE, b'') self.subscriber.connect("tcp://localhost:%d" % (self.peer + 1)) # Register state change handlers self.bstar.master_callback = self.become_master self.bstar.slave_callback = self.become_slave # Wrap sockets in ZMQStreams for IOLoop handlers self.publisher = ZMQStream(self.publisher) self.subscriber = ZMQStream(self.subscriber) self.collector = ZMQStream(self.collector) # Register our handlers with reactor self.collector.on_recv(self.handle_collect) self.flush_callback = PeriodicCallback(self.flush_ttl, 1000) self.hugz_callback = PeriodicCallback(self.send_hugz, 1000) # basic log formatting: logging.basicConfig(format="%(asctime)s %(message)s", datefmt="%Y-%m-%d %H:%M:%S", level=logging.INFO)
def __init__(self, port=5556): self.port = port self.ctx = zmq.Context() self.kvmap = {} self.loop = IOLoop.instance() # Set up our clone server sockets self.snapshot = self.ctx.socket(zmq.ROUTER) self.publisher = self.ctx.socket(zmq.PUB) self.collector = self.ctx.socket(zmq.PULL) self.snapshot.bind(f"tcp://*:{self.port:d}") self.publisher.bind(f"tcp://*:{self.port + 1:d}") self.collector.bind(f"tcp://*:{self.port + 2:d}") # Wrap sockets in ZMQStreams for IOLoop handlers self.snapshot = ZMQStream(self.snapshot) self.publisher = ZMQStream(self.publisher) self.collector = ZMQStream(self.collector) # Register our handlers with reactor self.snapshot.on_recv(self.handle_snapshot) self.collector.on_recv(self.handle_collect) self.flush_callback = PeriodicCallback(self.flush_ttl, 1000) # basic log formatting: logging.basicConfig(format="%(asctime)s %(message)s", datefmt="%Y-%m-%d %H:%M:%S", level=logging.INFO)
def __init__(self, led_count, hw_backend, port=6606): self.ctx = zmq.Context() self.led_count = led_count self.port = port self.loop = IOLoop.instance() self.caller = PeriodicCallback(self._on_next_frame, 1000/30) self.hw_communication = hw_backend self.hw_communication.connect() self.zmq_collector = GlinAppZmqCollector(self, self.ctx) self.zmq_publisher = GlinAppZmqPublisher(self, self.ctx) # server side configuration self.config = SimpleNamespace() self.config.max_fps = 60 # current state (somehow client side configuration) self.state = SimpleNamespace() self.state.animationClasses = [] self.state.activeSceneId = None self.state.activeAnimation = None self.state.scenes = {} self.state.brightness = 1.0 self.state.sceneIdCtr = 0 self.state.mainswitch = True self.state.target_fps = 0 self.state.lastFrameSent = None
def __init__(self): self.stop = False self.conx = Context.instance() self.socket = self.conx.socket(zmq.ROUTER) self.socket.bind('tcp://*:5555') self.periodic = PeriodicCallback(self.timer, 4000) self.periodic.start()
class UdpListener (CCDaemon): """ UDP server to handle UDP stream. """ log = skytools.getLogger ('d:UdpListener') def reload (self): super(UdpListener, self).reload() self.listen_host = self.cf.get ('listen-host') self.listen_port = self.cf.getint ('listen-port') self.stats_period = self.cf.getint ('stats-period', 30) def startup (self): super(UdpListener, self).startup() # plugins should be ready before we start receiving udp stream self.init_plugins() self.listen_addr = (self.listen_host, self.listen_port) self.sock = socket.socket (socket.AF_INET, socket.SOCK_DGRAM) self.sock.setblocking (0) try: self.sock.bind (self.listen_addr) except Exception, e: self.log.exception ("failed to bind to %s - %s", self.listen_addr, e) raise self.ioloop = IOLoop.instance() callback = functools.partial (self.handle_udp, self.sock) self.ioloop.add_handler (self.sock.fileno(), callback, self.ioloop.READ) self.timer_stats = PeriodicCallback (self.send_stats, self.stats_period * 1000, self.ioloop) self.timer_stats.start()
def __init__(self, context, main_ep, opt_ep=None, service_q=None): """Init MDPBroker instance. """ if service_q is None: self.service_q = ServiceQueue else: self.service_q = service_q socket = context.socket(zmq.ROUTER) socket.bind(main_ep) self.main_stream = ZMQStream(socket) self.main_stream.on_recv(self.on_message) if opt_ep: socket = context.socket(zmq.ROUTER) socket.bind(opt_ep) self.client_stream = ZMQStream(socket) self.client_stream.on_recv(self.on_message) else: self.client_stream = self.main_stream self._workers = {} # services contain the service queue and the request queue self._services = {} self._worker_cmds = { b'\x01': self.on_ready, b'\x03': self.on_reply, b'\x04': self.on_heartbeat, b'\x05': self.on_disconnect, } self.hb_check_timer = PeriodicCallback(self.on_timer, HB_INTERVAL) self.hb_check_timer.start() return
def __init__(self, context, main_ep, opt_ep=None): """Init MDPBroker instance. """ l = logger.Logger('mq_broker') self.log = l.get_logger() self.log.info("MDP broker startup...") socket = ZmqSocket(context, zmq.ROUTER) socket.bind(main_ep) self.main_stream = ZMQStream(socket) self.main_stream.on_recv(self.on_message) if opt_ep: socket = ZmqSocket(context, zmq.ROUTER) socket.bind(opt_ep) self.client_stream = ZMQStream(socket) self.client_stream.on_recv(self.on_message) else: self.client_stream = self.main_stream self.log.debug("Socket created...") self._workers = {} # services contain the worker queue and the request queue self._services = {} self._worker_cmds = { b'\x01': self.on_ready, b'\x03': self.on_reply, b'\x04': self.on_heartbeat, b'\x05': self.on_disconnect, } self.log.debug("Launch the timer...") self.hb_check_timer = PeriodicCallback(self.on_timer, HB_INTERVAL) self.hb_check_timer.start() self.log.info("MDP broker started") return
def __init__(self, primary, local, remote): # initialize the Binary Star self.ctx = zmq.Context() # Our private context self.loop = IOLoop.instance() # Reactor loop self.state = STATE_PRIMARY if primary else STATE_BACKUP self.event = None # Current event self.peer_expiry = 0 # When peer is considered 'dead' self.voter_callback = None # Voting socket handler self.master_callback = None # Call when become master self.slave_callback = None # Call when become slave # Create publisher for state going to peer self.statepub = self.ctx.socket(zmq.PUB) self.statepub.bind(local) # Create subscriber for state coming from peer self.statesub = self.ctx.socket(zmq.SUB) self.statesub.setsockopt_string(zmq.SUBSCRIBE, u'') self.statesub.connect(remote) # wrap statesub in ZMQStream for event triggers self.statesub = ZMQStream(self.statesub, self.loop) # setup basic reactor events self.heartbeat = PeriodicCallback(self.send_state, HEARTBEAT, self.loop) self.statesub.on_recv(self.recv_state)
def start_ip_address_checker(self): '''Checks for possible public IP change''' if self.ob_ctx.enable_ip_checker: self.caller = PeriodicCallback(self._ip_updater_periodic_callback, 5000, ioloop.IOLoop.instance()) self.caller.start() self.log.info( "IP_CHECKER_ENABLED: Periodic IP Address Checker started.")
def startup(self): """Setup sockets and handlers.""" super(CCServer, self).startup() self.log.info("C&C server version %s starting up..", self.__version__) self.xtx = CryptoContext(self.cf) self.zctx = zmq.Context(self.zmq_nthreads) self.ioloop = IOLoop.instance() self.local_url = self.cf.get('cc-socket') self.cur_role = self.cf.get('cc-role', 'insecure') if self.cur_role == 'insecure': self.log.warning( 'CC is running in insecure mode, please add "cc-role = local" or "cc-role = remote" option to config' ) self.stat_level = self.cf.getint('cc-stats', 1) if self.stat_level < 1: self.log.warning('CC statistics level too low: %d', self.stat_level) # initialize local listen socket s = self.zctx.socket(zmq.XREP) s.setsockopt(zmq.LINGER, self.zmq_linger) s.setsockopt(zmq.HWM, self.zmq_hwm) if self.zmq_rcvbuf > 0: s.setsockopt(zmq.RCVBUF, self.zmq_rcvbuf) if self.zmq_sndbuf > 0: s.setsockopt(zmq.SNDBUF, self.zmq_sndbuf) if self.zmq_tcp_keepalive > 0: if getattr(zmq, 'TCP_KEEPALIVE', -1) > 0: s.setsockopt(zmq.TCP_KEEPALIVE, self.zmq_tcp_keepalive) s.setsockopt(zmq.TCP_KEEPALIVE_INTVL, self.zmq_tcp_keepalive_intvl) s.setsockopt(zmq.TCP_KEEPALIVE_IDLE, self.zmq_tcp_keepalive_idle) s.setsockopt(zmq.TCP_KEEPALIVE_CNT, self.zmq_tcp_keepalive_cnt) else: self.log.info("TCP_KEEPALIVE not available") s.bind(self.local_url) self.local = CCStream(s, self.ioloop, qmaxsize=self.zmq_hwm) self.local.on_recv(self.handle_cc_recv) self.handlers = {} self.routes = {} rcf = skytools.Config('routes', self.cf.filename, ignore_defs=True) for r, hnames in rcf.cf.items('routes'): self.log.info('New route: %s = %s', r, hnames) for hname in [hn.strip() for hn in hnames.split(',')]: h = self.get_handler(hname) self.add_handler(r, h) self.stimer = PeriodicCallback(self.send_stats, 30 * 1000, self.ioloop) self.stimer.start()
def __init__ (self, hname, hcf, ccscript): super(ProxyHandler, self).__init__(hname, hcf, ccscript) self.ping_remote = self.cf.getbool ("ping", False) if self.ping_remote: self.echo_stats = EchoState (self.remote_url) self.echo_timer = PeriodicCallback (self.ping, self.ping_tick * 1000, self.ioloop) self.echo_timer.start() self.log.debug ("will ping %s", self.remote_url)
def __init__(self, proto, wid, service, stream): self.proto = proto self.id = wid self.service = service self.curr_liveness = HB_LIVENESS self.stream = stream self.last_hb = 0 self.hb_out_timer = PeriodicCallback(self.send_hb, HB_INTERVAL) self.hb_out_timer.start() return
def setup(self): super().setup() self.sub_stream, _ = self.stream( zmq.SUB, self.recv_addr, bind=False, subscribe=self.recv_title.encode('utf-8')) self.sub_stream.on_recv( SubStreamHandler(self.sub_stream, self.stop, self.state_handler)) self.timer = PeriodicCallback(self.callback, self.period, self.loop)
def __init__ (self, hname, hcf, ccscript): super(Delay, self).__init__(hname, hcf, ccscript) self.fwd_hname = self.cf.get ('forward-to') self.delay = self.cf.getint ('delay', 0) self.fwd_handler = ccscript.get_handler (self.fwd_hname) self.queue = collections.deque() self.timer = PeriodicCallback (self.process_queue, self.tick, self.ioloop) self.timer.start()
def __init__(self, proto, wid, service, stream): self.proto = proto self.id = wid self.service = service self.curr_retries = HB_RETRIES self.stream = stream self.last_hb = 0 self.free_space = 0 self.data = [] self.hb_out_timer = PeriodicCallback(self.send_hb, HB_INTERVAL) self.hb_out_timer.start() return
def __init__(self, *args): super(TaskRouter, self).__init__(*args) self.route_map = {} self.reply_map = {} # 1 hr? XXX self.route_lifetime = self.cf.getint ('route-lifetime', 1 * 60 * 60) self.reply_timeout = self.cf.getint ('reply-timeout', 5 * 60) self.maint_period = self.cf.getint ('maint-period', 1 * 60) self.timer = PeriodicCallback(self.do_maint, self.maint_period*1000, self.ioloop) self.timer.start()
def __init__(self, history_filepath=None): self._logger = logging.getLogger(__name__) if history_filepath is None: self._logger.info(' getting default history filepath.') history_filepath = _get_default_history_filepath() self._logger.info(' history filepath %s', history_filepath) self.history = FileHistory(history_filepath) self.messaging = _Messaging('shell', run_control_loop=True) # TODO: Cleanup this API access self.messaging._heartbeat_reciever.identity_callback = self._identity_callback self._thread = _Thread(target=self.messaging.start, daemon=True) self._bot_status_monitor = PeriodicCallback(self._monitor_bot_state, 1000) self.shebangs = [ '!', ] self.command_observer = CommandObserver(self.messaging, prompt=self) commands = self.command_observer._get_commands() self._word_completer = WordCompleter(commands, WORD=_WORD) self._services_completer = ServiceCompleter(self._word_completer) self.author_interface = AuthorInterface(self._word_completer, self.messaging) self.service_interface = ServiceInterface(self._word_completer, self.messaging) self.entity_interface = EntityInterface(self.author_interface, self.service_interface) self._bot_status = '' super().__init__(message='vexbot: ', history=self.history, completer=self._services_completer, enable_system_prompt=True, enable_suspend=True, enable_open_in_editor=True, complete_while_typing=False) self.print_observer = PrintObserver(self.app) self._print_subscription = self.messaging.chatter.subscribe( self.print_observer) self.messaging.chatter.subscribe(LogObserver()) self.messaging.command.subscribe(self.command_observer) self.messaging.command.subscribe( ServicesObserver(self._identity_setter, self._set_service_completion))
def _start_broker(self, do_reply=False): """Helper activating a fake broker in the ioloop. """ socket = self.context.socket(zmq.XREP) self.broker = ZMQStream(socket) self.broker.socket.setsockopt(zmq.LINGER, 0) self.broker.bind(self.endpoint) self.broker.on_recv(self._on_msg) self.broker.do_reply = do_reply self.broker.ticker = PeriodicCallback(self._tick, MyWorker.HB_INTERVAL) self.broker.ticker.start() self.target = None return
def _create_stream(self): """Helper to create the socket and the stream. """ socket = self.context.socket(zmq.DEALER) ioloop = IOLoop.instance() self.stream = ZMQStream(socket, ioloop) self.stream.on_recv(self._on_message) self.stream.socket.setsockopt(zmq.LINGER, 0) self.stream.connect(self.endpoint) self.ticker = PeriodicCallback(self._tick, self.HB_INTERVAL) self._send_ready() self.ticker.start() return
def __init__(self): self.ctx = zmq.Context() self.loop = IOLoop.instance() self.client = self.ctx.socket(zmq.DEALER) if self.crypto: self.keymonkey = KeyMonkey("client") self.client = self.keymonkey.setupClient(self.client, self.endpoint, "server") self.client.connect(self.endpoint) print("Connecting to", self.endpoint) self.client = ZMQStream(self.client) self.client.on_recv(self.on_recv) self.periodic = PeriodicCallback(self.periodictask, 1000) self.last_recv = None
def __init__(self, context, main_ep, client_ep, hb_ep, service_q=None): """Init MDPBroker instance. """ if service_q is None: self.service_q = ServiceQueue else: self.service_q = service_q # # Setup the zmq sockets. # socket = context.socket(zmq.ROUTER) socket.bind(main_ep) self.main_stream = ZMQStream(socket) self.main_stream.on_recv(self.on_message) socket = context.socket(zmq.ROUTER) socket.bind(client_ep) self.client_stream = ZMQStream(socket) self.client_stream.on_recv(self.on_message) socket = context.socket(zmq.ROUTER) socket.bind(hb_ep) self.hb_stream = ZMQStream(socket) self.hb_stream.on_recv(self.on_message) self._workers = {} # # services contain the service queue and the request queue # self._services = {} # # Mapping of worker commands and callbacks. # self._worker_cmds = { W_READY: self.on_ready, W_REPLY: self.on_reply, W_HEARTBEAT: self.on_heartbeat, W_DISCONNECT: self.on_disconnect, } # # 'Cleanup' timer for workers without heartbeat. # self.hb_check_timer = PeriodicCallback(self.on_timer, HB_INTERVAL) self.hb_check_timer.start()
def startup(self): super(TaskRunner, self).startup() self.ioloop = IOLoop.instance() self.connect_cc() self.ccs = CCStream(self.cc, self.ioloop, qmaxsize=self.zmq_hwm) self.ccs.on_recv(self.handle_cc_recv) self.local_id = self.cf.get('local-id', self.hostname) self.reg_period = self.cf.getint('reg-period', 5 * 60) self.maint_period = self.cf.getint('maint-period', 60) self.grace_period = self.cf.getint('task-grace-period', 15 * 60) self.task_heartbeat = self.cf.getboolean('task-heartbeat', False) self.tasks = {} self.periodic_reg() self.timer_reg = PeriodicCallback(self.periodic_reg, self.reg_period * 1000, self.ioloop) self.timer_reg.start() self.timer_maint = PeriodicCallback(self.do_maint, self.maint_period * 1000, self.ioloop) self.timer_maint.start()
def __init__( self ): self.ctx = zmq.Context() self.loop = IOLoop.instance() self.client_identities = {} self.server = self.ctx.socket(zmq.ROUTER) bind_addr = "tcp://%s:%s" % ( self.listen, self.port ) self.server.bind( bind_addr ) print("Server listening for new client connections at", bind_addr) self.server = ZMQStream(self.server) self.server.on_recv(self.on_recv) self.periodic = PeriodicCallback(self.periodictask, 1000)
def startup(self): # announce channel (for new files) self.sconn = self.zctx.socket(zmq.XREP) self.sconn.connect(self.shared_url) # direct channel (for grabbed files) self.dconn = self.zctx.socket(zmq.XREQ) self.dconn.connect(self.direct_url) # polling interface self.poller = zmq.Poller() self.poller.register(self.sconn, zmq.POLLIN) self.poller.register(self.dconn, zmq.POLLIN) # schedule regular maintenance self.timer_maint = PeriodicCallback(self.do_maint, self.maint_period * 1000, self.ioloop) self.timer_maint.start()
def __init__(self): self.ctx = zmq.Context() self.loop = IOLoop.instance() self.endpoint = "tcp://127.0.0.1:5556" self.client = self.ctx.socket(zmq.DEALER) self.client.setsockopt( zmq.LINGER, 0 ) # Without linger and timeouts you might have problems when closing context self.client.setsockopt(zmq.RCVTIMEO, 5000) # 5s self.client.setsockopt(zmq.SNDTIMEO, 5000) print("Connecting to", self.endpoint) self.client.connect(self.endpoint) self.client = ZMQStream(self.client) self.client.on_recv(self.on_recv) self.periodic = PeriodicCallback(self.periodictask, 1000) self.last_recv = None
def __init__(self): self.ctx = zmq.Context() self.loop = IOLoop.instance() self.client = self.ctx.socket(zmq.DEALER) self.client.connect(self.endpoint) print("Connecting to", self.endpoint) self.client = ZMQStream(self.client) self.client.on_recv(self.on_recv) self.periodic = PeriodicCallback(self.periodictask, 1000) self.last_recv = None