def _dump_data(self, loop=True): if self._data.empty() and loop: gevent.spawn_later(self.interval, self._dump_data) return data = { 'data_type': 'batch', 'agent_id': self.agent_id, 'hostname': get_hostname(), 'run_id': self.run_id, 'counts': defaultdict(list) } # grabbing what we have for _ in range(self._data.qsize()): data_type, message = self._data.get() data['counts'][data_type].append(message) while True: try: self._push.send(self.encoder.encode(data), zmq.NOBLOCK) break except zmq.ZMQError as e: if e.errno in (errno.EAGAIN, errno.EWOULDBLOCK): continue else: raise if loop: gevent.spawn_later(self.interval, self._dump_data)
def _dump_data(self, loop=True): if self._data.empty() and loop: gevent.spawn_later(self.interval, self._dump_data) return data = {'data_type': 'batch', 'agent_id': self.agent_id, 'hostname': get_hostname(), 'run_id': self.run_id, 'counts': defaultdict(list)} # grabbing what we have for _ in range(self._data.qsize()): data_type, message = self._data.get() data['counts'][data_type].append(message) while True: try: self._push.send(self.encoder.encode(data), zmq.NOBLOCK) break except zmq.ZMQError as e: if e.errno in (errno.EAGAIN, errno.EWOULDBLOCK): continue else: raise if loop: gevent.spawn_later(self.interval, self._dump_data)
def _handle_recv_back(self, msg): # do the message and send the result if self.debug: target = timed()(self._handle_commands) else: target = self._handle_commands duration = -1 broker_id = msg[2] if len(msg) == 7: client_id = msg[4] else: client_id = None data = msg[-1] try: res = target(Message.load_from_string(data)) if self.debug: duration, res = res res['hostname'] = get_hostname() res = json.dumps(res) # we're working with strings if isinstance(res, unicode): res = res.encode('utf8') except Exception, e: exc_type, exc_value, exc_traceback = sys.exc_info() exc = traceback.format_tb(exc_traceback) exc.insert(0, str(e)) res = {'error': {'agent_id': self.pid, 'error': '\n'.join(exc)}} logger.error(res)
def __init__(self, broker=DEFAULT_FRONTEND, ping_delay=10., ping_retries=3, params=None, timeout=DEFAULT_TIMEOUT_MOVF, max_age=DEFAULT_MAX_AGE, max_age_delta=DEFAULT_MAX_AGE_DELTA): logger.debug('Initializing the agent.') self.debug = logger.isEnabledFor(logging.DEBUG) self.params = params self.pid = os.getpid() self.agent_id = '%s-%s' % (get_hostname(), self.pid) self.timeout = timeout self.max_age = max_age self.max_age_delta = max_age_delta self.env = os.environ.copy() self.running = False self._workers = {} self._max_id = defaultdict(int) # Let's ask the broker its options self.broker = broker client = Client(self.broker) # this will timeout in case the broker is unreachable result = client.ping() self.endpoints = result['endpoints'] # Setup the zmq sockets self.loop = ioloop.IOLoop() self.ctx = zmq.Context() # backend socket - used to receive work from the broker self._backend = self.ctx.socket(zmq.ROUTER) self._backend.identity = self.agent_id self._backend.connect(self.endpoints['backend']) # register socket - used to register into the broker self._reg = self.ctx.socket(zmq.PUSH) self._reg.connect(self.endpoints['register']) # hearbeat socket - used to check if the broker is alive heartbeat = self.endpoints.get('heartbeat') if heartbeat is not None: logger.info("Hearbeat activated") self.ping = Stethoscope(heartbeat, onbeatlost=self.lost, delay=ping_delay, retries=ping_retries, ctx=self.ctx, io_loop=self.loop, onregister=self.register) else: self.ping = None # Setup the zmq streams. self._backstream = zmqstream.ZMQStream(self._backend, self.loop) self._backstream.on_recv(self._handle_recv_back) self._check = ioloop.PeriodicCallback(self._check_proc, ping_delay * 1000, io_loop=self.loop)
def push(self, data_type, **data): data.update({'data_type': data_type, 'agent_id': self.agent_id, 'hostname': get_hostname(), 'run_id': self.run_id}) while True: try: self._push.send(self.encoder.encode(data), zmq.NOBLOCK) return except zmq.ZMQError as e: if e.errno in (errno.EAGAIN, errno.EWOULDBLOCK): continue else: raise
def push(self, data_type, **data): data.update({ 'data_type': data_type, 'agent_id': self.agent_id, 'hostname': get_hostname(), 'run_id': self.run_id }) while True: try: self._push.send(self.encoder.encode(data), zmq.NOBLOCK) return except zmq.ZMQError as e: if e.errno in (errno.EAGAIN, errno.EWOULDBLOCK): continue else: raise
def register(self): # telling the broker we are ready data = {'pid': self.pid, 'hostname': get_hostname()} self._reg.send_multipart(['REGISTER', json.dumps(data)])