def test_intelmq_exchange(self): """ Test if correct Events have been produced, sent from a TCP Output of another IntelMQ instance. We spawn independent process of the TCPOutput bot that sends a bunch of messages. """ bot = TestTCPOutputBot() bot.setUpClass() bot.input_message = [] msg_count = 100 for i in range(msg_count): bot.input_message.append( Event(INPUT1, harmonization=self.harmonization)) (Process(target=bot._delayed_start)).start() self.run_bot() self.assertOutputQueueLen(msg_count) for i, msg in enumerate(self.get_output_queue()): report = MessageFactory.unserialize( msg, harmonization=self.harmonization, default_type='Event') output = MessageFactory.unserialize( utils.base64_decode(report["raw"]), harmonization=self.harmonization, default_type='Event') self.assertDictEqual(output, INPUT1) del report['time.observation'] del report['raw'] self.assertDictEqual(report, REPORT1)
def receive_message(self): # if a message comes from a collector, it qualifies as # a `Report`, otherwise - an `Event` msg_type = self._get_input_messages_type() msg = MessageFactory.unserialize(self.current_message, default_type=msg_type) return msg
def process(self): report = self.receive_message() event = MessageFactory.unserialize(base64_decode(report['raw'])) self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() event = MessageFactory.unserialize(base64_decode(report['raw'])) self.send_message(event) self.acknowledge_message()
def receive_message(self): message = self.source_pipeline.receive() if not message: return None self.current_message = MessageFactory.unserialize(message) return self.current_message
def test_random_input(self): """ Check how we handle a random input, coming from an unknown source. We should put all the data to report['raw']. """ thread = threading.Thread(target=Client().random_client) thread.start() self.run_bot() self.assertOutputQueueLen(2) generated_report = MessageFactory.unserialize(self.get_output_queue()[1], harmonization=self.harmonization, default_type='Event') self.assertEqual(base64_decode(generated_report['raw']), ORIGINAL_DATA.split(SEPARATOR)[1])
def Xtest_random_input(self): """ Check how we handle a random input, coming from an unknown source. We should put all the data to report['raw']. """ thread = threading.Thread(target=Client().random_client) thread.start() self.input_message = None self.run_bot() self.assertOutputQueueLen(2) generated_report = MessageFactory.unserialize(self.get_output_queue()[1], harmonization=self.harmonization, default_type='Event') self.assertEqual(base64_decode(generated_report['raw']), ORIGINAL_DATA.split(SEPARATOR)[1])
def receive_message(self): self.logger.debug('Receiving Message.') message = None while not message: message = self.source_pipeline.receive() self.logger.debug('Receive message {!r}...'.format(message[:500])) if not message: self.logger.warning('Empty message received.') self.current_message = MessageFactory.unserialize(message) return self.current_message
def receive_message(self): self.logger.debug('Receiving Message.') message = None while not message: message = self.source_pipeline.receive() self.logger.debug('Receive message {!r}...'.format(message[:500])) if not message: self.logger.warning('Empty message received.') self.current_message = MessageFactory.unserialize(message) return self.current_message
def arg2msg(self, msg): try: default_type = "Report" if self.runtime_configuration["group"] == "Parser" else "Event" msg = MessageFactory.unserialize(msg, default_type=default_type) except (Exception, KeyError, TypeError, ValueError) as exc: if exists(msg): with open(msg, "r") as f: return self.arg2msg(f.read()) self.messageWizzard("Message can not be parsed from JSON: {}".format(error_message_from_exc(exc))) sys.exit(1) return msg
def arg2msg(self, msg): try: default_type = "Report" if self.runtime_configuration["group"] is "Parser" else "Event" msg = MessageFactory.unserialize(msg, default_type=default_type) except (Exception, KeyError, TypeError, ValueError) as exc: if exists(msg): with open(msg, "r") as f: return self.arg2msg(f.read()) self.messageWizzard("Message can not be parsed from JSON: {}".format(error_message_from_exc(exc))) exit(1) return msg
def test_intelmq_exchange(self): """ Test if correct Events have been produced, sent from a TCP Output of another IntelMQ instance. We spawn independent process of the TCPOutput bot that sends a bunch of messages. """ bot = TestTCPOutputBot() bot.setUpClass() bot.input_message = [] msg_count = 100 for i in range(msg_count): bot.input_message.append(Event(INPUT1, harmonization=self.harmonization)) (Process(target=bot._delayed_start)).start() self.run_bot() self.assertOutputQueueLen(msg_count) for i, msg in enumerate(self.get_output_queue()): report = MessageFactory.unserialize(msg, harmonization=self.harmonization, default_type='Event') output = MessageFactory.unserialize(utils.base64_decode(report["raw"]), harmonization=self.harmonization, default_type='Event') self.assertDictEqual(output, INPUT1) del report['time.observation'] del report['raw'] self.assertDictEqual(report, REPORT1)
def arg2msg(self, msg): default_type = "Report" if ( self.runtime_configuration.get("group", None) == "Parser" or isinstance(self.instance, ParserBot)) else "Event" try: msg = MessageFactory.unserialize(msg, default_type=default_type) except (Exception, KeyError, TypeError, ValueError) as exc: if exists(msg): with open(msg, "r") as f: return self.arg2msg(f.read()) self.messageWizzard( "Message can not be parsed from JSON: {}".format( error_message_from_exc(exc))) sys.exit(1) return msg
def process(self): report = self.receive_message() if self.splitlines: lines = base64_decode(report['raw']).splitlines() else: lines = [base64_decode(report['raw'])] for line in lines: new_event = MessageFactory.unserialize( line, harmonization=self.harmonization, default_type='Event') event = self.new_event(report) event.update(new_event) if 'raw' not in event: event['raw'] = line self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() if getattr(self.parameters, 'splitlines', False): lines = base64_decode(report['raw']).splitlines() else: lines = [base64_decode(report['raw'])] for line in lines: new_event = MessageFactory.unserialize(line, harmonization=self.harmonization) event = self.new_event(report) dict.update(event, new_event) if 'raw' not in event: event['raw'] = line self.send_message(event) self.acknowledge_message()
def receive_message(self): self.logger.debug('Waiting for incoming message.') message = None while not message: message = self.__source_pipeline.receive() if not message: self.logger.warning('Empty message received.') continue self.__current_message = MessageFactory.unserialize(message) if 'raw' in self.__current_message and len(self.__current_message['raw']) > 400: tmp_msg = self.__current_message.to_dict(hierarchical=False) tmp_msg['raw'] = tmp_msg['raw'][:397] + '...' else: tmp_msg = self.__current_message self.logger.debug('Received message {!r}.'.format(tmp_msg)) return self.__current_message
def receive_message(self): self.logger.debug('Waiting for incoming message.') message = None while not message: message = self.__source_pipeline.receive() if not message: self.logger.warning('Empty message received.') continue self.__current_message = MessageFactory.unserialize(message) if 'raw' in self.__current_message and len(self.__current_message['raw']) > 400: tmp_msg = self.__current_message.to_dict(hierarchical=False) tmp_msg['raw'] = tmp_msg['raw'][:397] + '...' else: tmp_msg = self.__current_message self.logger.debug('Received message {!r}.'.format(tmp_msg)) return self.__current_message
def receive_message(self): self.logger.debug("Waiting for incoming message.") message = None while not message: message = self.__source_pipeline.receive() if not message: self.logger.warning("Empty message received.") continue self.__current_message = MessageFactory.unserialize(message) if "raw" in self.__current_message and len(self.__current_message["raw"]) > 400: tmp_msg = self.__current_message.to_dict(hierarchical=False) tmp_msg["raw"] = tmp_msg["raw"][:397] + "..." else: tmp_msg = self.__current_message self.logger.debug("Received message {!r}.".format(tmp_msg)) return self.__current_message
def process(self): ''' Stop the Bot if cannot connect to AMQP Server after the defined connection attempts ''' # self.connection and self.channel can be None if getattr(self.connection, 'is_closed', None) or getattr(self.channel, 'is_closed', None): self.connect_server() try: method, header, body = next(self.channel.consume(self.queue_name)) except (pika.exceptions.ChannelError, pika.exceptions.AMQPChannelError): self.logger.exception('Error receiving messages.') else: if self.expect_intelmq_message: message = MessageFactory.unserialize(body.decode()) self.send_message(message, auto_add=False) else: report = self.new_report() report['raw'] = body self.send_message(report) self.channel.basic_ack(delivery_tag=method.delivery_tag)
def receive_message(self): self.logger.debug('Waiting for incoming message.') message = None while not message: message = self.__source_pipeline.receive() if not message: self.logger.warning('Empty message received. Some previous bot sent invalid data.') continue self.__current_message = MessageFactory.unserialize(message) if 'raw' in self.__current_message and len(self.__current_message['raw']) > 400: tmp_msg = self.__current_message.to_dict(hierarchical=False) tmp_msg['raw'] = tmp_msg['raw'][:397] + '...' else: tmp_msg = self.__current_message self.logger.debug('Received message {!r}.'.format(tmp_msg)) # handle a sighup which happened during blocking read self.__handle_sighup() return self.__current_message