def receive_message(self):
self.logger.debug('Waiting for incoming message.')
message = None
while not message:
message = self.__source_pipeline.receive()
if not message:
self.logger.warning(
'Empty message received. Some previous bot sent invalid data.'
)
continue
# handle a sighup which happened during blocking read
self.__handle_sighup()
try:
self.__current_message = libmessage.MessageFactory.unserialize(
message, harmonization=self.harmonization)
except exceptions.InvalidKey as exc:
# In case a incoming message is malformed an does not conform with the currently
# loaded harmonization, stop now as this will happen repeatedly without any change
raise exceptions.ConfigurationError('harmonization', exc.args[0])
if 'raw' in self.__current_message and len(
self.__current_message['raw']) > 400:
tmp_msg = self.__current_message.to_dict(hierarchical=False)
tmp_msg['raw'] = tmp_msg['raw'][:397] + '...'
else:
tmp_msg = self.__current_message
self.logger.debug('Received message %r.', tmp_msg)
return self.__current_message
def send_message(self, *messages, path="_default", auto_add=None,
path_permissive=False):
"""
Parameters:
messages: Instances of intelmq.lib.message.Message class
auto_add: ignored
"""
for message in messages:
if not message:
self.logger.warning("Ignoring empty message at sending. Possible bug in bot.")
continue
if not self.__destination_pipeline:
raise exceptions.ConfigurationError('pipeline', 'No destination pipeline given, '
'but needed')
self.logger.debug("Sending message.")
self.__message_counter["since"] += 1
self.__message_counter["path"][path] += 1
if not self.__message_counter["start"]:
self.__message_counter["start"] = datetime.now()
if self.__message_counter["since"] % self.parameters.log_processed_messages_count == 0 or \
datetime.now() - self.__message_counter["start"] > self.parameters.log_processed_messages_seconds:
self.logger.info("%s %d messages since last logging.",
self._message_processed_verb,
self.__message_counter["since"])
self.__message_counter["since"] = 0
self.__message_counter["start"] = datetime.now()
raw_message = libmessage.MessageFactory.serialize(message)
self.__destination_pipeline.send(raw_message, path=path,
path_permissive=path_permissive)
def send_message(self, *messages):
for message in messages:
if not message:
self.logger.warning(
"Ignoring empty message at sending. Possible bug in bot.")
continue
if not self.__destination_pipeline:
raise exceptions.ConfigurationError(
'pipeline', 'No destination pipeline given, '
'but needed')
self.stop()
self.logger.debug("Sending message.")
self.__message_counter += 1
if not self.__message_counter_start:
self.__message_counter_start = datetime.datetime.now()
if self.__message_counter % self.parameters.log_processed_messages_count == 0 or \
datetime.datetime.now() - self.__message_counter_start > self.parameters.log_processed_messages_seconds:
self.logger.info("Processed %d messages since last logging.",
self.__message_counter)
self.__message_counter = 0
self.__message_counter_start = datetime.datetime.now()
raw_message = libmessage.MessageFactory.serialize(message)
self.__destination_pipeline.send(raw_message)
def load_harmonization_configuration(self):
self.logger.debug("Loading Harmonization configuration.")
config = utils.load_configuration(HARMONIZATION_CONF_FILE)
for message_types in config.keys():
for key in config[message_types].keys():
for _key in config.keys():
if _key.startswith("%s." % key):
raise exceptions.ConfigurationError(
'harmonization', "Key %s is not valid." % _key)
def receive(self):
if self.source_queue is None:
raise exceptions.ConfigurationError('pipeline', 'No source queue given.')
try:
retval = self.pipe.lindex(self.internal_queue, -1) # returns None if no value
if not retval:
retval = self.pipe.brpoplpush(self.source_queue,
self.internal_queue, 0)
return utils.decode(retval)
except Exception as exc:
raise exceptions.PipelineError(exc)
def _receive(self) -> bytes:
if self.source_queue is None:
raise exceptions.ConfigurationError('pipeline', 'No source queue given.')
try:
method, header, body = next(self.channel.consume(self.source_queue))
if method:
self.delivery_tag = method.delivery_tag
except Exception as exc:
raise exceptions.PipelineError(exc)
else:
return body
def receive_message(self):
"""
If the bot is reloaded when waiting for an incoming message, the received message
will be rejected to the pipeline in the first place to get to a clean state.
Then, after reloading, the message will be retrieved again.
"""
if self.__current_message:
self.logger.debug("Reusing existing current message as incoming.")
return self.__current_message
self.logger.debug('Waiting for incoming message.')
message = None
while not message:
message = self.__source_pipeline.receive()
if not message:
self.logger.warning(
'Empty message received. Some previous bot sent invalid data.'
)
self.__handle_sighup()
continue
# * handle a sighup which happened during blocking read
# * re-queue the message before reloading
# https://github.com/certtools/intelmq/issues/1438
if self.__sighup.is_set():
self.__source_pipeline.reject_message()
self.__handle_sighup()
return self.receive_message()
try:
self.__current_message = libmessage.MessageFactory.unserialize(
message, harmonization=self.harmonization)
except exceptions.InvalidKey as exc:
# In case a incoming message is malformed an does not conform with the currently
# loaded harmonization, stop now as this will happen repeatedly without any change
raise exceptions.ConfigurationError('harmonization', exc.args[0])
if self.logger.isEnabledFor(logging.DEBUG):
if 'raw' in self.__current_message and len(
self.__current_message['raw']) > 400:
tmp_msg = self.__current_message.to_dict(hierarchical=False)
tmp_msg['raw'] = tmp_msg['raw'][:397] + '...'
else:
tmp_msg = self.__current_message
self.logger.debug('Received message %r.', tmp_msg)
return self.__current_message
def send_message(self, *messages):
for message in messages:
if not message:
self.logger.warning("Ignoring empty message at sending. Possible bug in bot.")
continue
if not self.__destination_pipeline:
raise exceptions.ConfigurationError('pipeline', 'No destination pipeline given, '
'but needed')
self.stop()
self.logger.debug("Sending message.")
self.__message_counter += 1
if self.__message_counter % 500 == 0:
self.logger.info("Processed %s messages.", self.__message_counter)
raw_message = libmessage.MessageFactory.serialize(message)
self.__destination_pipeline.send(raw_message)
def _receive(self) -> bytes:
if self.source_queue is None:
raise exceptions.ConfigurationError('pipeline', 'No source queue given.')
try:
while True:
try:
retval = self.pipe.lindex(self.internal_queue, -1) # returns None if no value
except redis.exceptions.BusyLoadingError: # Just wait at redis' startup #1334
time.sleep(1)
else:
break
if not retval:
retval = self.pipe.brpoplpush(self.source_queue,
self.internal_queue, 0)
except Exception as exc:
raise exceptions.PipelineError(exc)
else:
return retval
def __load_pipeline_configuration(self):
self.logger.debug("Loading pipeline configuration from %r.", PIPELINE_CONF_FILE)
config = utils.load_configuration(PIPELINE_CONF_FILE)
self.__source_queues = None
self.__destination_queues = None
if self.__bot_id in list(config.keys()):
if 'source-queue' in config[self.__bot_id].keys():
self.__source_queues = config[self.__bot_id]['source-queue']
if 'destination-queues' in config[self.__bot_id].keys():
self.__destination_queues = config[
self.__bot_id]['destination-queues']
# Convert old to new format here
else:
raise exceptions.ConfigurationError('pipeline', "no key "
"{!r}.".format(self.__bot_id))