def test_intelmq_exchange(self):
""" Test if correct Events have been produced, sent from a TCP Output of another IntelMQ instance.
We spawn independent process of the TCPOutput bot that sends a bunch of messages.
"""
bot = TestTCPOutputBot()
bot.setUpClass()
bot.input_message = []
msg_count = 100
for i in range(msg_count):
bot.input_message.append(
Event(INPUT1, harmonization=self.harmonization))
(Process(target=bot._delayed_start)).start()
self.run_bot()
self.assertOutputQueueLen(msg_count)
for i, msg in enumerate(self.get_output_queue()):
report = MessageFactory.unserialize(
msg, harmonization=self.harmonization, default_type='Event')
output = MessageFactory.unserialize(
utils.base64_decode(report["raw"]),
harmonization=self.harmonization,
default_type='Event')
self.assertDictEqual(output, INPUT1)
del report['time.observation']
del report['raw']
self.assertDictEqual(report, REPORT1)
def _process(self, dryrun, msg, show):
if msg:
msg = MessageFactory.serialize(self.arg2msg(msg))
if not self.instance._Bot__source_pipeline:
# is None if source pipeline does not exist
self.instance._Bot__source_pipeline = Pipeline(None)
self.instance._Bot__source_pipeline.receive = lambda: msg
self.instance.logger.info(
" * Message from cli will be used when processing.")
if dryrun:
self.instance.send_message = lambda msg, path="_default": self.instance.logger.info(
"DRYRUN: Message would be sent now{}!".format(
" to the {} path".format(path)
if (path != "_default") else ""))
self.instance.acknowledge_message = lambda: self.instance.logger.info(
"DRYRUN: Message would be acknowledged now!")
self.instance.logger.info(
" * Dryrun only, no message will be really sent through.")
if show:
fn = self.instance.send_message
self.instance.send_message = lambda msg, path="_default": [
self.pprint(msg), fn(msg, path=path)
]
self.instance.logger.info("Processing...")
self.instance.process()
def _process(self, dryrun, msg, show):
if msg:
msg = MessageFactory.serialize(self.arg2msg(msg))
if not self.instance._Bot__source_pipeline:
# is None if source pipeline does not exist
self.instance._Bot__source_pipeline = Pipeline(None)
self.instance._Bot__source_pipeline.receive = lambda *args, **kwargs: msg
self.instance._Bot__source_pipeline.acknowledge = lambda *args, **kwargs: None
self.instance.logger.info(
" * Message from cli will be used when processing.")
if dryrun:
self.instance.send_message = lambda *args, **kwargs: self.instance.logger.info(
"DRYRUN: Message would be sent now to %r!",
kwargs.get('path', "_default"))
self.instance.acknowledge_message = lambda *args, **kwargs: self.instance.logger.info(
"DRYRUN: Message would be acknowledged now!")
self.instance.logger.info(
" * Dryrun only, no message will be really sent through.")
if show:
fn = self.instance.send_message
self.instance.send_message = lambda *args, **kwargs: [
self.pprint(args or "No message generated"),
fn(*args, **kwargs)
]
self.instance.logger.info("Processing...")
self.instance.process()
def process(self):
report = self.receive_message()
event = MessageFactory.unserialize(base64_decode(report['raw']))
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
event = MessageFactory.unserialize(base64_decode(report['raw']))
self.send_message(event)
self.acknowledge_message()
def receive_message(self):
# if a message comes from a collector, it qualifies as
# a `Report`, otherwise - an `Event`
msg_type = self._get_input_messages_type()
msg = MessageFactory.unserialize(self.current_message,
default_type=msg_type)
return msg
def receive_message(self):
message = self.source_pipeline.receive()
if not message:
return None
self.current_message = MessageFactory.unserialize(message)
return self.current_message
def process(self):
message = self.receive_message()
if message:
message = MessageFactory.serialize(message)
self.logger.info(message)
self.conn.send(body=message, destination=self.exchange)
def test_random_input(self):
""" Check how we handle a random input, coming from an unknown source. We should put all the data to report['raw']. """
thread = threading.Thread(target=Client().random_client)
thread.start()
self.run_bot()
self.assertOutputQueueLen(2)
generated_report = MessageFactory.unserialize(self.get_output_queue()[1], harmonization=self.harmonization,
default_type='Event')
self.assertEqual(base64_decode(generated_report['raw']), ORIGINAL_DATA.split(SEPARATOR)[1])
def Xtest_random_input(self):
""" Check how we handle a random input, coming from an unknown source. We should put all the data to report['raw']. """
thread = threading.Thread(target=Client().random_client)
thread.start()
self.input_message = None
self.run_bot()
self.assertOutputQueueLen(2)
generated_report = MessageFactory.unserialize(self.get_output_queue()[1], harmonization=self.harmonization,
default_type='Event')
self.assertEqual(base64_decode(generated_report['raw']), ORIGINAL_DATA.split(SEPARATOR)[1])
def arg2msg(self, msg):
try:
default_type = "Report" if self.runtime_configuration["group"] == "Parser" else "Event"
msg = MessageFactory.unserialize(msg, default_type=default_type)
except (Exception, KeyError, TypeError, ValueError) as exc:
if exists(msg):
with open(msg, "r") as f:
return self.arg2msg(f.read())
self.messageWizzard("Message can not be parsed from JSON: {}".format(error_message_from_exc(exc)))
sys.exit(1)
return msg
def send_message(self, message):
if not message:
self.logger.warning("Empty message found.")
return False
self.message_counter += 1
if self.message_counter % 500 == 0:
self.logger.info("Processed %s messages" % self.message_counter)
raw_message = MessageFactory.serialize(message)
self.destination_pipeline.send(raw_message)
def receive_message(self):
self.logger.debug('Receiving Message.')
message = None
while not message:
message = self.source_pipeline.receive()
self.logger.debug('Receive message {!r}...'.format(message[:500]))
if not message:
self.logger.warning('Empty message received.')
self.current_message = MessageFactory.unserialize(message)
return self.current_message
def receive_message(self):
self.logger.debug('Receiving Message.')
message = None
while not message:
message = self.source_pipeline.receive()
self.logger.debug('Receive message {!r}...'.format(message[:500]))
if not message:
self.logger.warning('Empty message received.')
self.current_message = MessageFactory.unserialize(message)
return self.current_message
def arg2msg(self, msg):
try:
default_type = "Report" if self.runtime_configuration["group"] is "Parser" else "Event"
msg = MessageFactory.unserialize(msg, default_type=default_type)
except (Exception, KeyError, TypeError, ValueError) as exc:
if exists(msg):
with open(msg, "r") as f:
return self.arg2msg(f.read())
self.messageWizzard("Message can not be parsed from JSON: {}".format(error_message_from_exc(exc)))
exit(1)
return msg
def send_message(self, message):
if not message:
self.logger.warning("Sending Message: Empty message found.")
return False
self.logger.debug("Sending message.")
self.message_counter += 1
if self.message_counter % 500 == 0:
self.logger.info("Processed %s messages." % self.message_counter)
raw_message = MessageFactory.serialize(message)
self.destination_pipeline.send(raw_message)
def send_message(self, *messages):
for message in messages:
if not message:
self.logger.warning("Ignoring empty message at sending.")
continue
self.logger.debug("Sending message.")
self.__message_counter += 1
if self.__message_counter % 500 == 0:
self.logger.info("Processed %s messages." % self.__message_counter)
raw_message = MessageFactory.serialize(message)
self.__destination_pipeline.send(raw_message)
def _process(self, dryrun, msg):
if msg:
msg = MessageFactory.serialize(self.arg2msg(msg))
self.instance._Bot__source_pipeline.receive = lambda: msg
self.instance.logger.info(" * Message from cli will be used when processing.")
if dryrun:
self.instance.send_message = lambda msg: self.instance.logger.info("DRYRUN: Message would be sent now!")
self.instance.acknowledge_message = lambda: self.instance.logger.info("DRYRUN: Message would be acknowledged now!")
self.instance.logger.info(" * Dryrun only, no message will be really sent through.")
self.instance.logger.info("Processing...")
self.instance.process()
def _process(self, dryrun, msg):
if msg:
msg = MessageFactory.serialize(self.arg2msg(msg))
self.instance._Bot__source_pipeline.receive = lambda: msg
self.instance.logger.info(" * Message from cli will be used when processing.")
if dryrun:
self.instance.send_message = lambda msg: self.instance.logger.info("DRYRUN: Message would be sent now!")
self.instance.acknowledge_message = lambda: self.instance.logger.info("DRYRUN: Message would be acknowledged now!")
self.instance.logger.info(" * Dryrun only, no message will be really sent through.")
self.instance.logger.info("Processing...")
self.instance.process()
def send_message(self, *messages):
for message in messages:
if not message:
self.logger.warning("Ignoring empty message at sending.")
continue
self.logger.debug("Sending message.")
self.__message_counter += 1
if self.__message_counter % 500 == 0:
self.logger.info("Processed %s messages." % self.__message_counter)
raw_message = MessageFactory.serialize(message)
self.__destination_pipeline.send(raw_message)
def test_intelmq_exchange(self):
""" Test if correct Events have been produced, sent from a TCP Output of another IntelMQ instance.
We spawn independent process of the TCPOutput bot that sends a bunch of messages.
"""
bot = TestTCPOutputBot()
bot.setUpClass()
bot.input_message = []
msg_count = 100
for i in range(msg_count):
bot.input_message.append(Event(INPUT1, harmonization=self.harmonization))
(Process(target=bot._delayed_start)).start()
self.run_bot()
self.assertOutputQueueLen(msg_count)
for i, msg in enumerate(self.get_output_queue()):
report = MessageFactory.unserialize(msg, harmonization=self.harmonization, default_type='Event')
output = MessageFactory.unserialize(utils.base64_decode(report["raw"]), harmonization=self.harmonization, default_type='Event')
self.assertDictEqual(output, INPUT1)
del report['time.observation']
del report['raw']
self.assertDictEqual(report, REPORT1)
def arg2msg(self, msg):
default_type = "Report" if (
self.runtime_configuration.get("group", None) == "Parser"
or isinstance(self.instance, ParserBot)) else "Event"
try:
msg = MessageFactory.unserialize(msg, default_type=default_type)
except (Exception, KeyError, TypeError, ValueError) as exc:
if exists(msg):
with open(msg, "r") as f:
return self.arg2msg(f.read())
self.messageWizzard(
"Message can not be parsed from JSON: {}".format(
error_message_from_exc(exc)))
sys.exit(1)
return msg
def process(self):
report = self.receive_message()
if self.splitlines:
lines = base64_decode(report['raw']).splitlines()
else:
lines = [base64_decode(report['raw'])]
for line in lines:
new_event = MessageFactory.unserialize(
line, harmonization=self.harmonization, default_type='Event')
event = self.new_event(report)
event.update(new_event)
if 'raw' not in event:
event['raw'] = line
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if getattr(self.parameters, 'splitlines', False):
lines = base64_decode(report['raw']).splitlines()
else:
lines = [base64_decode(report['raw'])]
for line in lines:
new_event = MessageFactory.unserialize(line,
harmonization=self.harmonization)
event = self.new_event(report)
dict.update(event, new_event)
if 'raw' not in event:
event['raw'] = line
self.send_message(event)
self.acknowledge_message()
def receive_message(self):
self.logger.debug('Waiting for incoming message.')
message = None
while not message:
message = self.__source_pipeline.receive()
if not message:
self.logger.warning('Empty message received.')
continue
self.__current_message = MessageFactory.unserialize(message)
if 'raw' in self.__current_message and len(self.__current_message['raw']) > 400:
tmp_msg = self.__current_message.to_dict(hierarchical=False)
tmp_msg['raw'] = tmp_msg['raw'][:397] + '...'
else:
tmp_msg = self.__current_message
self.logger.debug('Received message {!r}.'.format(tmp_msg))
return self.__current_message
def receive_message(self):
self.logger.debug("Waiting for incoming message.")
message = None
while not message:
message = self.__source_pipeline.receive()
if not message:
self.logger.warning("Empty message received.")
continue
self.__current_message = MessageFactory.unserialize(message)
if "raw" in self.__current_message and len(self.__current_message["raw"]) > 400:
tmp_msg = self.__current_message.to_dict(hierarchical=False)
tmp_msg["raw"] = tmp_msg["raw"][:397] + "..."
else:
tmp_msg = self.__current_message
self.logger.debug("Received message {!r}.".format(tmp_msg))
return self.__current_message
def _process(self, dryrun, msg, show):
if msg:
msg = MessageFactory.serialize(self.arg2msg(msg))
self.instance._Bot__source_pipeline.receive = lambda: msg
self.instance.logger.info(" * Message from cli will be used when processing.")
if dryrun:
self.instance.send_message = lambda msg, path="_default": self.instance.logger.info(
"DRYRUN: Message would be sent now{}!".format(" to the {} path".format(path) if (path != "_default") else ""))
self.instance.acknowledge_message = lambda: self.instance.logger.info("DRYRUN: Message would be acknowledged now!")
self.instance.logger.info(" * Dryrun only, no message will be really sent through.")
if show:
fn = self.instance.send_message
self.instance.send_message = lambda msg, path="_default": [self.pprint(msg), fn(msg, path=path)]
self.instance.logger.info("Processing...")
self.instance.process()
def receive_message(self):
self.logger.debug('Waiting for incoming message.')
message = None
while not message:
message = self.__source_pipeline.receive()
if not message:
self.logger.warning('Empty message received.')
continue
self.__current_message = MessageFactory.unserialize(message)
if 'raw' in self.__current_message and len(self.__current_message['raw']) > 400:
tmp_msg = self.__current_message.to_dict(hierarchical=False)
tmp_msg['raw'] = tmp_msg['raw'][:397] + '...'
else:
tmp_msg = self.__current_message
self.logger.debug('Received message {!r}.'.format(tmp_msg))
return self.__current_message
def process(self):
''' Stop the Bot if cannot connect to AMQP Server after the defined connection attempts '''
# self.connection and self.channel can be None
if getattr(self.connection, 'is_closed', None) or getattr(self.channel, 'is_closed', None):
self.connect_server()
try:
method, header, body = next(self.channel.consume(self.queue_name))
except (pika.exceptions.ChannelError, pika.exceptions.AMQPChannelError):
self.logger.exception('Error receiving messages.')
else:
if self.expect_intelmq_message:
message = MessageFactory.unserialize(body.decode())
self.send_message(message, auto_add=False)
else:
report = self.new_report()
report['raw'] = body
self.send_message(report)
self.channel.basic_ack(delivery_tag=method.delivery_tag)
def receive_message(self):
self.logger.debug('Waiting for incoming message.')
message = None
while not message:
message = self.__source_pipeline.receive()
if not message:
self.logger.warning('Empty message received. Some previous bot sent invalid data.')
continue
self.__current_message = MessageFactory.unserialize(message)
if 'raw' in self.__current_message and len(self.__current_message['raw']) > 400:
tmp_msg = self.__current_message.to_dict(hierarchical=False)
tmp_msg['raw'] = tmp_msg['raw'][:397] + '...'
else:
tmp_msg = self.__current_message
self.logger.debug('Received message {!r}.'.format(tmp_msg))
# handle a sighup which happened during blocking read
self.__handle_sighup()
return self.__current_message
def process(self):
message = self.receive_message()
message = MessageFactory.serialize(message)
self.conn.send(body=message, destination=self.exchange)
self.acknowledge_message()
def submit():
parameters = handle_parameters(request.form)
temp_file = get_temp_file()
if not temp_file:
return create_response('No file')
destination_pipeline = PipelineFactory.create(PipelineParameters(),
logger=app.logger,
direction='destination')
if not CONFIG.get('destination_pipeline_queue_formatted', False):
destination_pipeline.set_queues(CONFIG['destination_pipeline_queue'], "destination")
destination_pipeline.connect()
time_observation = DateTime().generate_datetime_now()
successful_lines = 0
with open(temp_file[0], encoding='utf8') as handle:
reader = csv.reader(handle, delimiter=parameters['delimiter'],
quotechar=parameters['quotechar'],
skipinitialspace=parameters['skipInitialSpace'],
escapechar=parameters['escapechar'],
)
if parameters['has_header']:
next(reader)
for _ in range(parameters['skipInitialLines']):
next(reader)
for lineindex, line in enumerate(reader):
event = Event()
try:
for columnindex, (column, value) in \
enumerate(zip(parameters['columns'], line)):
if not column or not value:
continue
if column.startswith('time.'):
parsed = dateutil.parser.parse(value)
if not parsed.tzinfo:
value += parameters['timezone']
parsed = dateutil.parser.parse(value)
value = parsed.isoformat()
if column == 'extra':
value = handle_extra(value)
event.add(column, value)
for key, value in parameters.get('constant_fields', {}).items():
if key not in event:
event.add(key, value)
for key, value in request.form.items():
if not key.startswith('custom_'):
continue
key = key[7:]
if key not in event:
event.add(key, value)
if CONFIG.get('destination_pipeline_queue_formatted', False):
queue_name = CONFIG['destination_pipeline_queue'].format(ev=event)
destination_pipeline.set_queues(queue_name, "destination")
destination_pipeline.connect()
except Exception:
continue
if 'classification.type' not in event:
event.add('classification.type', parameters['classification.type'])
if 'classification.identifier' not in event:
event.add('classification.identifier', parameters['classification.identifier'])
if 'feed.code' not in event:
event.add('feed.code', parameters['feed.code'])
if 'time.observation' not in event:
event.add('time.observation', time_observation, sanitize=False)
raw_message = MessageFactory.serialize(event)
destination_pipeline.send(raw_message)
successful_lines += 1
return create_response('Successfully processed %s lines.' % successful_lines)
def process(self):
message = self.receive_message()
message = MessageFactory.serialize(message)
self.conn.send(body=message, destination=self.exchange)
self.acknowledge_message()