def _dump_data(self, loop=True):
if self._data.empty() and loop:
gevent.spawn_later(self.interval, self._dump_data)
return
data = {
'data_type': 'batch',
'agent_id': self.agent_id,
'hostname': get_hostname(),
'run_id': self.run_id,
'counts': defaultdict(list)
}
# grabbing what we have
for _ in range(self._data.qsize()):
data_type, message = self._data.get()
data['counts'][data_type].append(message)
while True:
try:
self._push.send(self.encoder.encode(data), zmq.NOBLOCK)
break
except zmq.ZMQError as e:
if e.errno in (errno.EAGAIN, errno.EWOULDBLOCK):
continue
else:
raise
if loop:
gevent.spawn_later(self.interval, self._dump_data)
def _dump_data(self, loop=True):
if self._data.empty() and loop:
gevent.spawn_later(self.interval, self._dump_data)
return
data = {'data_type': 'batch',
'agent_id': self.agent_id,
'hostname': get_hostname(),
'run_id': self.run_id,
'counts': defaultdict(list)}
# grabbing what we have
for _ in range(self._data.qsize()):
data_type, message = self._data.get()
data['counts'][data_type].append(message)
while True:
try:
self._push.send(self.encoder.encode(data), zmq.NOBLOCK)
break
except zmq.ZMQError as e:
if e.errno in (errno.EAGAIN, errno.EWOULDBLOCK):
continue
else:
raise
if loop:
gevent.spawn_later(self.interval, self._dump_data)
def _handle_recv_back(self, msg):
# do the message and send the result
if self.debug:
target = timed()(self._handle_commands)
else:
target = self._handle_commands
duration = -1
broker_id = msg[2]
if len(msg) == 7:
client_id = msg[4]
else:
client_id = None
data = msg[-1]
try:
res = target(Message.load_from_string(data))
if self.debug:
duration, res = res
res['hostname'] = get_hostname()
res = json.dumps(res)
# we're working with strings
if isinstance(res, unicode):
res = res.encode('utf8')
except Exception, e:
exc_type, exc_value, exc_traceback = sys.exc_info()
exc = traceback.format_tb(exc_traceback)
exc.insert(0, str(e))
res = {'error': {'agent_id': self.pid, 'error': '\n'.join(exc)}}
logger.error(res)
def _handle_recv_back(self, msg):
# do the message and send the result
if self.debug:
target = timed()(self._handle_commands)
else:
target = self._handle_commands
duration = -1
broker_id = msg[2]
if len(msg) == 7:
client_id = msg[4]
else:
client_id = None
data = msg[-1]
try:
res = target(Message.load_from_string(data))
if self.debug:
duration, res = res
res['hostname'] = get_hostname()
res = json.dumps(res)
# we're working with strings
if isinstance(res, unicode):
res = res.encode('utf8')
except Exception, e:
exc_type, exc_value, exc_traceback = sys.exc_info()
exc = traceback.format_tb(exc_traceback)
exc.insert(0, str(e))
res = {'error': {'agent_id': self.pid, 'error': '\n'.join(exc)}}
logger.error(res)
def __init__(self, broker=DEFAULT_FRONTEND,
ping_delay=10., ping_retries=3,
params=None, timeout=DEFAULT_TIMEOUT_MOVF,
max_age=DEFAULT_MAX_AGE, max_age_delta=DEFAULT_MAX_AGE_DELTA):
logger.debug('Initializing the agent.')
self.debug = logger.isEnabledFor(logging.DEBUG)
self.params = params
self.pid = os.getpid()
self.agent_id = '%s-%s' % (get_hostname(), self.pid)
self.timeout = timeout
self.max_age = max_age
self.max_age_delta = max_age_delta
self.env = os.environ.copy()
self.running = False
self._workers = {}
self._max_id = defaultdict(int)
# Let's ask the broker its options
self.broker = broker
client = Client(self.broker)
# this will timeout in case the broker is unreachable
result = client.ping()
self.endpoints = result['endpoints']
# Setup the zmq sockets
self.loop = ioloop.IOLoop()
self.ctx = zmq.Context()
# backend socket - used to receive work from the broker
self._backend = self.ctx.socket(zmq.ROUTER)
self._backend.identity = self.agent_id
self._backend.connect(self.endpoints['backend'])
# register socket - used to register into the broker
self._reg = self.ctx.socket(zmq.PUSH)
self._reg.connect(self.endpoints['register'])
# hearbeat socket - used to check if the broker is alive
heartbeat = self.endpoints.get('heartbeat')
if heartbeat is not None:
logger.info("Hearbeat activated")
self.ping = Stethoscope(heartbeat, onbeatlost=self.lost,
delay=ping_delay, retries=ping_retries,
ctx=self.ctx, io_loop=self.loop,
onregister=self.register)
else:
self.ping = None
# Setup the zmq streams.
self._backstream = zmqstream.ZMQStream(self._backend, self.loop)
self._backstream.on_recv(self._handle_recv_back)
self._check = ioloop.PeriodicCallback(self._check_proc,
ping_delay * 1000,
io_loop=self.loop)
def push(self, data_type, **data):
data.update({'data_type': data_type,
'agent_id': self.agent_id,
'hostname': get_hostname(),
'run_id': self.run_id})
while True:
try:
self._push.send(self.encoder.encode(data), zmq.NOBLOCK)
return
except zmq.ZMQError as e:
if e.errno in (errno.EAGAIN, errno.EWOULDBLOCK):
continue
else:
raise
def push(self, data_type, **data):
data.update({
'data_type': data_type,
'agent_id': self.agent_id,
'hostname': get_hostname(),
'run_id': self.run_id
})
while True:
try:
self._push.send(self.encoder.encode(data), zmq.NOBLOCK)
return
except zmq.ZMQError as e:
if e.errno in (errno.EAGAIN, errno.EWOULDBLOCK):
continue
else:
raise
def register(self):
# telling the broker we are ready
data = {'pid': self.pid, 'hostname': get_hostname()}
self._reg.send_multipart(['REGISTER', json.dumps(data)])
def register(self):
# telling the broker we are ready
data = {'pid': self.pid, 'hostname': get_hostname()}
self._reg.send_multipart(['REGISTER', json.dumps(data)])
