def test_intelmq_exchange(self):
""" Test if correct Events have been produced, sent from a TCP Output of another IntelMQ instance.
We spawn independent process of the TCPOutput bot that sends a bunch of messages.
"""
bot = TestTCPOutputBot()
bot.setUpClass()
bot.input_message = []
msg_count = 100
for i in range(msg_count):
bot.input_message.append(
Event(INPUT1, harmonization=self.harmonization))
(Process(target=bot._delayed_start)).start()
self.run_bot()
self.assertOutputQueueLen(msg_count)
for i, msg in enumerate(self.get_output_queue()):
report = MessageFactory.unserialize(
msg, harmonization=self.harmonization, default_type='Event')
output = MessageFactory.unserialize(
utils.base64_decode(report["raw"]),
harmonization=self.harmonization,
default_type='Event')
self.assertDictEqual(output, INPUT1)
del report['time.observation']
del report['raw']
self.assertDictEqual(report, REPORT1)
def receive_message(self):
# if a message comes from a collector, it qualifies as
# a `Report`, otherwise - an `Event`
msg_type = self._get_input_messages_type()
msg = MessageFactory.unserialize(self.current_message,
default_type=msg_type)
return msg
def process(self):
report = self.receive_message()
event = MessageFactory.unserialize(base64_decode(report['raw']))
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
event = MessageFactory.unserialize(base64_decode(report['raw']))
self.send_message(event)
self.acknowledge_message()
def receive_message(self):
message = self.source_pipeline.receive()
if not message:
return None
self.current_message = MessageFactory.unserialize(message)
return self.current_message
def test_random_input(self):
""" Check how we handle a random input, coming from an unknown source. We should put all the data to report['raw']. """
thread = threading.Thread(target=Client().random_client)
thread.start()
self.run_bot()
self.assertOutputQueueLen(2)
generated_report = MessageFactory.unserialize(self.get_output_queue()[1], harmonization=self.harmonization,
default_type='Event')
self.assertEqual(base64_decode(generated_report['raw']), ORIGINAL_DATA.split(SEPARATOR)[1])
def Xtest_random_input(self):
""" Check how we handle a random input, coming from an unknown source. We should put all the data to report['raw']. """
thread = threading.Thread(target=Client().random_client)
thread.start()
self.input_message = None
self.run_bot()
self.assertOutputQueueLen(2)
generated_report = MessageFactory.unserialize(self.get_output_queue()[1], harmonization=self.harmonization,
default_type='Event')
self.assertEqual(base64_decode(generated_report['raw']), ORIGINAL_DATA.split(SEPARATOR)[1])
def receive_message(self):
self.logger.debug('Receiving Message.')
message = None
while not message:
message = self.source_pipeline.receive()
self.logger.debug('Receive message {!r}...'.format(message[:500]))
if not message:
self.logger.warning('Empty message received.')
self.current_message = MessageFactory.unserialize(message)
return self.current_message
def receive_message(self):
self.logger.debug('Receiving Message.')
message = None
while not message:
message = self.source_pipeline.receive()
self.logger.debug('Receive message {!r}...'.format(message[:500]))
if not message:
self.logger.warning('Empty message received.')
self.current_message = MessageFactory.unserialize(message)
return self.current_message
def arg2msg(self, msg):
try:
default_type = "Report" if self.runtime_configuration["group"] == "Parser" else "Event"
msg = MessageFactory.unserialize(msg, default_type=default_type)
except (Exception, KeyError, TypeError, ValueError) as exc:
if exists(msg):
with open(msg, "r") as f:
return self.arg2msg(f.read())
self.messageWizzard("Message can not be parsed from JSON: {}".format(error_message_from_exc(exc)))
sys.exit(1)
return msg
def arg2msg(self, msg):
try:
default_type = "Report" if self.runtime_configuration["group"] is "Parser" else "Event"
msg = MessageFactory.unserialize(msg, default_type=default_type)
except (Exception, KeyError, TypeError, ValueError) as exc:
if exists(msg):
with open(msg, "r") as f:
return self.arg2msg(f.read())
self.messageWizzard("Message can not be parsed from JSON: {}".format(error_message_from_exc(exc)))
exit(1)
return msg
def test_intelmq_exchange(self):
""" Test if correct Events have been produced, sent from a TCP Output of another IntelMQ instance.
We spawn independent process of the TCPOutput bot that sends a bunch of messages.
"""
bot = TestTCPOutputBot()
bot.setUpClass()
bot.input_message = []
msg_count = 100
for i in range(msg_count):
bot.input_message.append(Event(INPUT1, harmonization=self.harmonization))
(Process(target=bot._delayed_start)).start()
self.run_bot()
self.assertOutputQueueLen(msg_count)
for i, msg in enumerate(self.get_output_queue()):
report = MessageFactory.unserialize(msg, harmonization=self.harmonization, default_type='Event')
output = MessageFactory.unserialize(utils.base64_decode(report["raw"]), harmonization=self.harmonization, default_type='Event')
self.assertDictEqual(output, INPUT1)
del report['time.observation']
del report['raw']
self.assertDictEqual(report, REPORT1)
def arg2msg(self, msg):
default_type = "Report" if (
self.runtime_configuration.get("group", None) == "Parser"
or isinstance(self.instance, ParserBot)) else "Event"
try:
msg = MessageFactory.unserialize(msg, default_type=default_type)
except (Exception, KeyError, TypeError, ValueError) as exc:
if exists(msg):
with open(msg, "r") as f:
return self.arg2msg(f.read())
self.messageWizzard(
"Message can not be parsed from JSON: {}".format(
error_message_from_exc(exc)))
sys.exit(1)
return msg
def process(self):
report = self.receive_message()
if self.splitlines:
lines = base64_decode(report['raw']).splitlines()
else:
lines = [base64_decode(report['raw'])]
for line in lines:
new_event = MessageFactory.unserialize(
line, harmonization=self.harmonization, default_type='Event')
event = self.new_event(report)
event.update(new_event)
if 'raw' not in event:
event['raw'] = line
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if getattr(self.parameters, 'splitlines', False):
lines = base64_decode(report['raw']).splitlines()
else:
lines = [base64_decode(report['raw'])]
for line in lines:
new_event = MessageFactory.unserialize(line,
harmonization=self.harmonization)
event = self.new_event(report)
dict.update(event, new_event)
if 'raw' not in event:
event['raw'] = line
self.send_message(event)
self.acknowledge_message()
def receive_message(self):
self.logger.debug('Waiting for incoming message.')
message = None
while not message:
message = self.__source_pipeline.receive()
if not message:
self.logger.warning('Empty message received.')
continue
self.__current_message = MessageFactory.unserialize(message)
if 'raw' in self.__current_message and len(self.__current_message['raw']) > 400:
tmp_msg = self.__current_message.to_dict(hierarchical=False)
tmp_msg['raw'] = tmp_msg['raw'][:397] + '...'
else:
tmp_msg = self.__current_message
self.logger.debug('Received message {!r}.'.format(tmp_msg))
return self.__current_message
def receive_message(self):
self.logger.debug('Waiting for incoming message.')
message = None
while not message:
message = self.__source_pipeline.receive()
if not message:
self.logger.warning('Empty message received.')
continue
self.__current_message = MessageFactory.unserialize(message)
if 'raw' in self.__current_message and len(self.__current_message['raw']) > 400:
tmp_msg = self.__current_message.to_dict(hierarchical=False)
tmp_msg['raw'] = tmp_msg['raw'][:397] + '...'
else:
tmp_msg = self.__current_message
self.logger.debug('Received message {!r}.'.format(tmp_msg))
return self.__current_message
def receive_message(self):
self.logger.debug("Waiting for incoming message.")
message = None
while not message:
message = self.__source_pipeline.receive()
if not message:
self.logger.warning("Empty message received.")
continue
self.__current_message = MessageFactory.unserialize(message)
if "raw" in self.__current_message and len(self.__current_message["raw"]) > 400:
tmp_msg = self.__current_message.to_dict(hierarchical=False)
tmp_msg["raw"] = tmp_msg["raw"][:397] + "..."
else:
tmp_msg = self.__current_message
self.logger.debug("Received message {!r}.".format(tmp_msg))
return self.__current_message
def process(self):
''' Stop the Bot if cannot connect to AMQP Server after the defined connection attempts '''
# self.connection and self.channel can be None
if getattr(self.connection, 'is_closed', None) or getattr(self.channel, 'is_closed', None):
self.connect_server()
try:
method, header, body = next(self.channel.consume(self.queue_name))
except (pika.exceptions.ChannelError, pika.exceptions.AMQPChannelError):
self.logger.exception('Error receiving messages.')
else:
if self.expect_intelmq_message:
message = MessageFactory.unserialize(body.decode())
self.send_message(message, auto_add=False)
else:
report = self.new_report()
report['raw'] = body
self.send_message(report)
self.channel.basic_ack(delivery_tag=method.delivery_tag)
def receive_message(self):
self.logger.debug('Waiting for incoming message.')
message = None
while not message:
message = self.__source_pipeline.receive()
if not message:
self.logger.warning('Empty message received. Some previous bot sent invalid data.')
continue
self.__current_message = MessageFactory.unserialize(message)
if 'raw' in self.__current_message and len(self.__current_message['raw']) > 400:
tmp_msg = self.__current_message.to_dict(hierarchical=False)
tmp_msg['raw'] = tmp_msg['raw'][:397] + '...'
else:
tmp_msg = self.__current_message
self.logger.debug('Received message {!r}.'.format(tmp_msg))
# handle a sighup which happened during blocking read
self.__handle_sighup()
return self.__current_message
