def put(self, item, block=True, timeout=None): self._lock.acquire() try: try: if 'message_id' in item: message_id = item['message_id'] if message_id in self._message_id_set: _g_logger.info("Skipping sending a retransmission " "of message id %s" % message_id) return else: _g_logger.debug("Adding the message with id %s " % message_id) self._message_id_set.add(message_id) if 'request_id' in item: request_id = item['request_id'] if request_id not in self._request_id_count: self._request_id_count[request_id] = 0 self._request_id_count[request_id] += 1 if self._request_id_count[request_id] >= self._max_req_id: msg = "TOO MANY MESSAGES FOR %s!" % request_id _g_logger.error(msg) agent_utils.build_assertion_exception(_g_logger, msg) if self._request_id_count[request_id] ==\ self._max_req_id: dcm_logger.log_to_dcm_console_overloaded(msg=msg) return except Exception as ex: _g_logger.warn("Exception checking if message is a retrans " "%s" % str(ex)) return self._q.put(item, block=block, timeout=timeout) finally: self._lock.release()
def _process_doc(self, incoming_doc): if incoming_doc is None: return with tracer.RequestTracer(incoming_doc['request_id']): self._call_reply_observers("incoming_message", incoming_doc) _g_logger.debug("New message type %s" % incoming_doc['type']) # if the agent is misbehaving the AM might tell it to kill itself. # cold. if incoming_doc["type"] == message_types.MessageTypes.HEMLOCK: _g_logger.error("HEMLOCK: DCM told me to kill myself.") os.killpg(0, signal.SIGKILL) sys.exit(10) # if it is a alert message short circuit if incoming_doc["type"] == message_types.MessageTypes.ALERT_ACK: if self._id_system: self._id_system.incoming_message(incoming_doc) return request_id = incoming_doc["request_id"] # is this request already in memory? if request_id in self._requests: _g_logger.debug("The message was found in the requests.") # send it through, state machine will deal with it req = self._requests[request_id] req.incoming_message(incoming_doc) return # if the request id has already been seen by the database db_record = self._db.lookup_req(request_id) if db_record: _g_logger.info("Inflating the record from the DB." + request_id) req = ReplyRPC( self, self._conf.agent_id, self._conn, request_id, incoming_doc, self._db, timeout=self._timeout, reply_doc=db_record.reply_doc, start_state=db_record.state) # this will probably be used in the near future so get it # on the memory list self._requests[request_id] = req req.incoming_message(incoming_doc) return if incoming_doc["type"] == message_types.MessageTypes.REQUEST: if len(list(self._requests.keys())) >=\ self._conf.messaging_max_at_once > -1: # short circuit the case where the agent is too busy dcm_logger.log_to_dcm_console_overloaded( msg="The new request was rejected because the agent has too many outstanding requests.") nack_doc = { 'type': message_types.MessageTypes.NACK, 'message_id': utils.new_message_id(), 'request_id': request_id, 'agent_id': self._conf.agent_id, 'error_message': ("The agent can only handle %d " "commands at once" % self._conf.messaging_max_at_once)} self._conn.send(nack_doc) return _g_logger.debug("A new request has come in.") req = ReplyRPC( self, self._conf.agent_id, self._conn, request_id, incoming_doc, self._db, timeout=self._timeout) self._call_reply_observers("new_message", req) # only add the message if processing was successful self._requests[request_id] = req try: self._dispatcher.incoming_request(req) except Exception: _g_logger.exception("The dispatcher could not handle a " "message.") del self._requests[request_id] dcm_logger.log_to_dcm_console_messaging_error( msg="The dispatcher could not handle the message.") raise else: # if we have never heard of the ID and this is not a new # request we return a courtesy error _g_logger.debug("Unknown message ID sending a NACK") nack_doc = {'type': message_types.MessageTypes.NACK, 'message_id': utils.new_message_id(), 'request_id': request_id, 'agent_id': self._conf.agent_id, 'error_message': "%s is an unknown ID" % request_id} self._conn.send(nack_doc)