def test_intelmq_exchange(self): """ Test if correct Events have been produced, sent from a TCP Output of another IntelMQ instance. We spawn independent process of the TCPOutput bot that sends a bunch of messages. """ bot = TestTCPOutputBot() bot.setUpClass() bot.input_message = [] msg_count = 100 for i in range(msg_count): bot.input_message.append( Event(INPUT1, harmonization=self.harmonization)) (Process(target=bot._delayed_start)).start() self.run_bot() self.assertOutputQueueLen(msg_count) for i, msg in enumerate(self.get_output_queue()): report = MessageFactory.unserialize( msg, harmonization=self.harmonization, default_type='Event') output = MessageFactory.unserialize( utils.base64_decode(report["raw"]), harmonization=self.harmonization, default_type='Event') self.assertDictEqual(output, INPUT1) del report['time.observation'] del report['raw'] self.assertDictEqual(report, REPORT1)
def _process(self, dryrun, msg, show): if msg: msg = MessageFactory.serialize(self.arg2msg(msg)) if not self.instance._Bot__source_pipeline: # is None if source pipeline does not exist self.instance._Bot__source_pipeline = Pipeline(None) self.instance._Bot__source_pipeline.receive = lambda: msg self.instance.logger.info( " * Message from cli will be used when processing.") if dryrun: self.instance.send_message = lambda msg, path="_default": self.instance.logger.info( "DRYRUN: Message would be sent now{}!".format( " to the {} path".format(path) if (path != "_default") else "")) self.instance.acknowledge_message = lambda: self.instance.logger.info( "DRYRUN: Message would be acknowledged now!") self.instance.logger.info( " * Dryrun only, no message will be really sent through.") if show: fn = self.instance.send_message self.instance.send_message = lambda msg, path="_default": [ self.pprint(msg), fn(msg, path=path) ] self.instance.logger.info("Processing...") self.instance.process()
def _process(self, dryrun, msg, show): if msg: msg = MessageFactory.serialize(self.arg2msg(msg)) if not self.instance._Bot__source_pipeline: # is None if source pipeline does not exist self.instance._Bot__source_pipeline = Pipeline(None) self.instance._Bot__source_pipeline.receive = lambda *args, **kwargs: msg self.instance._Bot__source_pipeline.acknowledge = lambda *args, **kwargs: None self.instance.logger.info( " * Message from cli will be used when processing.") if dryrun: self.instance.send_message = lambda *args, **kwargs: self.instance.logger.info( "DRYRUN: Message would be sent now to %r!", kwargs.get('path', "_default")) self.instance.acknowledge_message = lambda *args, **kwargs: self.instance.logger.info( "DRYRUN: Message would be acknowledged now!") self.instance.logger.info( " * Dryrun only, no message will be really sent through.") if show: fn = self.instance.send_message self.instance.send_message = lambda *args, **kwargs: [ self.pprint(args or "No message generated"), fn(*args, **kwargs) ] self.instance.logger.info("Processing...") self.instance.process()
def process(self): report = self.receive_message() event = MessageFactory.unserialize(base64_decode(report['raw'])) self.send_message(event) self.acknowledge_message()
def receive_message(self): # if a message comes from a collector, it qualifies as # a `Report`, otherwise - an `Event` msg_type = self._get_input_messages_type() msg = MessageFactory.unserialize(self.current_message, default_type=msg_type) return msg
def receive_message(self): message = self.source_pipeline.receive() if not message: return None self.current_message = MessageFactory.unserialize(message) return self.current_message
def process(self): message = self.receive_message() if message: message = MessageFactory.serialize(message) self.logger.info(message) self.conn.send(body=message, destination=self.exchange)
def test_random_input(self): """ Check how we handle a random input, coming from an unknown source. We should put all the data to report['raw']. """ thread = threading.Thread(target=Client().random_client) thread.start() self.run_bot() self.assertOutputQueueLen(2) generated_report = MessageFactory.unserialize(self.get_output_queue()[1], harmonization=self.harmonization, default_type='Event') self.assertEqual(base64_decode(generated_report['raw']), ORIGINAL_DATA.split(SEPARATOR)[1])
def Xtest_random_input(self): """ Check how we handle a random input, coming from an unknown source. We should put all the data to report['raw']. """ thread = threading.Thread(target=Client().random_client) thread.start() self.input_message = None self.run_bot() self.assertOutputQueueLen(2) generated_report = MessageFactory.unserialize(self.get_output_queue()[1], harmonization=self.harmonization, default_type='Event') self.assertEqual(base64_decode(generated_report['raw']), ORIGINAL_DATA.split(SEPARATOR)[1])
def arg2msg(self, msg): try: default_type = "Report" if self.runtime_configuration["group"] == "Parser" else "Event" msg = MessageFactory.unserialize(msg, default_type=default_type) except (Exception, KeyError, TypeError, ValueError) as exc: if exists(msg): with open(msg, "r") as f: return self.arg2msg(f.read()) self.messageWizzard("Message can not be parsed from JSON: {}".format(error_message_from_exc(exc))) sys.exit(1) return msg
def send_message(self, message): if not message: self.logger.warning("Empty message found.") return False self.message_counter += 1 if self.message_counter % 500 == 0: self.logger.info("Processed %s messages" % self.message_counter) raw_message = MessageFactory.serialize(message) self.destination_pipeline.send(raw_message)
def receive_message(self): self.logger.debug('Receiving Message.') message = None while not message: message = self.source_pipeline.receive() self.logger.debug('Receive message {!r}...'.format(message[:500])) if not message: self.logger.warning('Empty message received.') self.current_message = MessageFactory.unserialize(message) return self.current_message
def arg2msg(self, msg): try: default_type = "Report" if self.runtime_configuration["group"] is "Parser" else "Event" msg = MessageFactory.unserialize(msg, default_type=default_type) except (Exception, KeyError, TypeError, ValueError) as exc: if exists(msg): with open(msg, "r") as f: return self.arg2msg(f.read()) self.messageWizzard("Message can not be parsed from JSON: {}".format(error_message_from_exc(exc))) exit(1) return msg
def send_message(self, message): if not message: self.logger.warning("Sending Message: Empty message found.") return False self.logger.debug("Sending message.") self.message_counter += 1 if self.message_counter % 500 == 0: self.logger.info("Processed %s messages." % self.message_counter) raw_message = MessageFactory.serialize(message) self.destination_pipeline.send(raw_message)
def send_message(self, *messages): for message in messages: if not message: self.logger.warning("Ignoring empty message at sending.") continue self.logger.debug("Sending message.") self.__message_counter += 1 if self.__message_counter % 500 == 0: self.logger.info("Processed %s messages." % self.__message_counter) raw_message = MessageFactory.serialize(message) self.__destination_pipeline.send(raw_message)
def _process(self, dryrun, msg): if msg: msg = MessageFactory.serialize(self.arg2msg(msg)) self.instance._Bot__source_pipeline.receive = lambda: msg self.instance.logger.info(" * Message from cli will be used when processing.") if dryrun: self.instance.send_message = lambda msg: self.instance.logger.info("DRYRUN: Message would be sent now!") self.instance.acknowledge_message = lambda: self.instance.logger.info("DRYRUN: Message would be acknowledged now!") self.instance.logger.info(" * Dryrun only, no message will be really sent through.") self.instance.logger.info("Processing...") self.instance.process()
def test_intelmq_exchange(self): """ Test if correct Events have been produced, sent from a TCP Output of another IntelMQ instance. We spawn independent process of the TCPOutput bot that sends a bunch of messages. """ bot = TestTCPOutputBot() bot.setUpClass() bot.input_message = [] msg_count = 100 for i in range(msg_count): bot.input_message.append(Event(INPUT1, harmonization=self.harmonization)) (Process(target=bot._delayed_start)).start() self.run_bot() self.assertOutputQueueLen(msg_count) for i, msg in enumerate(self.get_output_queue()): report = MessageFactory.unserialize(msg, harmonization=self.harmonization, default_type='Event') output = MessageFactory.unserialize(utils.base64_decode(report["raw"]), harmonization=self.harmonization, default_type='Event') self.assertDictEqual(output, INPUT1) del report['time.observation'] del report['raw'] self.assertDictEqual(report, REPORT1)
def arg2msg(self, msg): default_type = "Report" if ( self.runtime_configuration.get("group", None) == "Parser" or isinstance(self.instance, ParserBot)) else "Event" try: msg = MessageFactory.unserialize(msg, default_type=default_type) except (Exception, KeyError, TypeError, ValueError) as exc: if exists(msg): with open(msg, "r") as f: return self.arg2msg(f.read()) self.messageWizzard( "Message can not be parsed from JSON: {}".format( error_message_from_exc(exc))) sys.exit(1) return msg
def process(self): report = self.receive_message() if self.splitlines: lines = base64_decode(report['raw']).splitlines() else: lines = [base64_decode(report['raw'])] for line in lines: new_event = MessageFactory.unserialize( line, harmonization=self.harmonization, default_type='Event') event = self.new_event(report) event.update(new_event) if 'raw' not in event: event['raw'] = line self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() if getattr(self.parameters, 'splitlines', False): lines = base64_decode(report['raw']).splitlines() else: lines = [base64_decode(report['raw'])] for line in lines: new_event = MessageFactory.unserialize(line, harmonization=self.harmonization) event = self.new_event(report) dict.update(event, new_event) if 'raw' not in event: event['raw'] = line self.send_message(event) self.acknowledge_message()
def receive_message(self): self.logger.debug('Waiting for incoming message.') message = None while not message: message = self.__source_pipeline.receive() if not message: self.logger.warning('Empty message received.') continue self.__current_message = MessageFactory.unserialize(message) if 'raw' in self.__current_message and len(self.__current_message['raw']) > 400: tmp_msg = self.__current_message.to_dict(hierarchical=False) tmp_msg['raw'] = tmp_msg['raw'][:397] + '...' else: tmp_msg = self.__current_message self.logger.debug('Received message {!r}.'.format(tmp_msg)) return self.__current_message
def receive_message(self): self.logger.debug("Waiting for incoming message.") message = None while not message: message = self.__source_pipeline.receive() if not message: self.logger.warning("Empty message received.") continue self.__current_message = MessageFactory.unserialize(message) if "raw" in self.__current_message and len(self.__current_message["raw"]) > 400: tmp_msg = self.__current_message.to_dict(hierarchical=False) tmp_msg["raw"] = tmp_msg["raw"][:397] + "..." else: tmp_msg = self.__current_message self.logger.debug("Received message {!r}.".format(tmp_msg)) return self.__current_message
def _process(self, dryrun, msg, show): if msg: msg = MessageFactory.serialize(self.arg2msg(msg)) self.instance._Bot__source_pipeline.receive = lambda: msg self.instance.logger.info(" * Message from cli will be used when processing.") if dryrun: self.instance.send_message = lambda msg, path="_default": self.instance.logger.info( "DRYRUN: Message would be sent now{}!".format(" to the {} path".format(path) if (path != "_default") else "")) self.instance.acknowledge_message = lambda: self.instance.logger.info("DRYRUN: Message would be acknowledged now!") self.instance.logger.info(" * Dryrun only, no message will be really sent through.") if show: fn = self.instance.send_message self.instance.send_message = lambda msg, path="_default": [self.pprint(msg), fn(msg, path=path)] self.instance.logger.info("Processing...") self.instance.process()
def process(self): ''' Stop the Bot if cannot connect to AMQP Server after the defined connection attempts ''' # self.connection and self.channel can be None if getattr(self.connection, 'is_closed', None) or getattr(self.channel, 'is_closed', None): self.connect_server() try: method, header, body = next(self.channel.consume(self.queue_name)) except (pika.exceptions.ChannelError, pika.exceptions.AMQPChannelError): self.logger.exception('Error receiving messages.') else: if self.expect_intelmq_message: message = MessageFactory.unserialize(body.decode()) self.send_message(message, auto_add=False) else: report = self.new_report() report['raw'] = body self.send_message(report) self.channel.basic_ack(delivery_tag=method.delivery_tag)
def receive_message(self): self.logger.debug('Waiting for incoming message.') message = None while not message: message = self.__source_pipeline.receive() if not message: self.logger.warning('Empty message received. Some previous bot sent invalid data.') continue self.__current_message = MessageFactory.unserialize(message) if 'raw' in self.__current_message and len(self.__current_message['raw']) > 400: tmp_msg = self.__current_message.to_dict(hierarchical=False) tmp_msg['raw'] = tmp_msg['raw'][:397] + '...' else: tmp_msg = self.__current_message self.logger.debug('Received message {!r}.'.format(tmp_msg)) # handle a sighup which happened during blocking read self.__handle_sighup() return self.__current_message
def process(self): message = self.receive_message() message = MessageFactory.serialize(message) self.conn.send(body=message, destination=self.exchange) self.acknowledge_message()
def submit(): parameters = handle_parameters(request.form) temp_file = get_temp_file() if not temp_file: return create_response('No file') destination_pipeline = PipelineFactory.create(PipelineParameters(), logger=app.logger, direction='destination') if not CONFIG.get('destination_pipeline_queue_formatted', False): destination_pipeline.set_queues(CONFIG['destination_pipeline_queue'], "destination") destination_pipeline.connect() time_observation = DateTime().generate_datetime_now() successful_lines = 0 with open(temp_file[0], encoding='utf8') as handle: reader = csv.reader(handle, delimiter=parameters['delimiter'], quotechar=parameters['quotechar'], skipinitialspace=parameters['skipInitialSpace'], escapechar=parameters['escapechar'], ) if parameters['has_header']: next(reader) for _ in range(parameters['skipInitialLines']): next(reader) for lineindex, line in enumerate(reader): event = Event() try: for columnindex, (column, value) in \ enumerate(zip(parameters['columns'], line)): if not column or not value: continue if column.startswith('time.'): parsed = dateutil.parser.parse(value) if not parsed.tzinfo: value += parameters['timezone'] parsed = dateutil.parser.parse(value) value = parsed.isoformat() if column == 'extra': value = handle_extra(value) event.add(column, value) for key, value in parameters.get('constant_fields', {}).items(): if key not in event: event.add(key, value) for key, value in request.form.items(): if not key.startswith('custom_'): continue key = key[7:] if key not in event: event.add(key, value) if CONFIG.get('destination_pipeline_queue_formatted', False): queue_name = CONFIG['destination_pipeline_queue'].format(ev=event) destination_pipeline.set_queues(queue_name, "destination") destination_pipeline.connect() except Exception: continue if 'classification.type' not in event: event.add('classification.type', parameters['classification.type']) if 'classification.identifier' not in event: event.add('classification.identifier', parameters['classification.identifier']) if 'feed.code' not in event: event.add('feed.code', parameters['feed.code']) if 'time.observation' not in event: event.add('time.observation', time_observation, sanitize=False) raw_message = MessageFactory.serialize(event) destination_pipeline.send(raw_message) successful_lines += 1 return create_response('Successfully processed %s lines.' % successful_lines)