def prepare_bot(self, raise_on_connect=False): self.log_stream = io.StringIO() self.bot_id = 'test-bot' src_name = "{}-input".format(self.bot_id) dst_name = "{}-output".format(self.bot_id) self.mocked_config = test.mocked_config( self.bot_id, src_name, [dst_name], {"raise_on_connect": raise_on_connect}) logger = logging.getLogger(self.bot_id) logger.setLevel("DEBUG") console_formatter = logging.Formatter(utils.LOG_FORMAT) console_handler = logging.StreamHandler(self.log_stream) console_handler.setFormatter(console_formatter) logger.addHandler(console_handler) self.mocked_log = test.mocked_logger(logger) class Parameters(object): source_queue = src_name destination_queues = [dst_name] parameters = Parameters() pipe = pipeline.Pythonlist(parameters) pipe.set_queues(parameters.source_queue, "source") pipe.set_queues(parameters.destination_queues, "destination") with mock.patch('intelmq.lib.utils.load_configuration', new=self.mocked_config): with mock.patch('intelmq.lib.utils.log', self.mocked_log): self.bot = self.bot_reference(self.bot_id) self.pipe = pipe
def prepare_bot(self): """Reconfigures the bot with the changed attributes""" self.log_stream = io.StringIO() src_name = "{}-input".format(self.bot_id) dst_name = "{}-output".format(self.bot_id) self.mocked_config = mocked_config( self.bot_id, src_name, [dst_name], sysconfig=self.sysconfig, ) logger = logging.getLogger(self.bot_id) logger.setLevel("INFO") console_formatter = logging.Formatter(utils.LOG_FORMAT) console_handler = logging.StreamHandler(self.log_stream) console_handler.setFormatter(console_formatter) logger.addHandler(console_handler) self.mocked_log = mocked_logger(logger) logging.captureWarnings(True) warnings_logger = logging.getLogger("py.warnings") warnings_logger.addHandler(console_handler) class Parameters(object): source_queue = src_name destination_queues = [dst_name] parameters = Parameters() self.pipe = pipeline.Pythonlist(parameters) self.pipe.set_queues(parameters.source_queue, "source") self.pipe.set_queues(parameters.destination_queues, "destination") with mock.patch('intelmq.lib.utils.load_configuration', new=self.mocked_config): with mock.patch('intelmq.lib.utils.log', self.mocked_log): self.bot = self.bot_reference(self.bot_id) if self.input_message is not None: if type(self.input_message) is not list: self.input_message = [self.input_message] self.input_queue = [] for msg in self.input_message: if type(msg) is dict: self.input_queue.append(json.dumps(msg)) elif issubclass(type(msg), message.Message): self.input_queue.append(msg.serialize()) else: self.input_queue.append(msg) self.input_message = None else: if self.default_input_message: # None for collectors self.input_queue = [self.default_input_message]
def prepare_bot(self, parameters={}, destination_queues=None): """ Reconfigures the bot with the changed attributes. Parameters: parameters: optional bot parameters for this run, as dict destination_queues: optional definition of destination queues default: {"_default": "{}-output".format(self.bot_id)} """ self.log_stream = io.StringIO() src_name = "{}-input".format(self.bot_id) if not destination_queues: destination_queues = {"_default": "{}-output".format(self.bot_id)} else: destination_queues = {queue_name: "%s-%s-output" % (self.bot_id, queue_name.strip('_')) for queue_name in destination_queues} config = self.sysconfig.copy() config.update(parameters) self.mocked_config = mocked_config(self.bot_id, src_name, destination_queues, sysconfig=config, group=self.bot_type.title(), module=self.bot_reference.__module__, ) self.resulting_config = BOT_CONFIG.copy() self.resulting_config.update(self.sysconfig) self.resulting_config.update(parameters) self.logger = utils.log(self.bot_id, log_path=False, stream=self.log_stream, log_format_stream=utils.LOG_FORMAT, log_level=self.resulting_config['logging_level']) self.logger_handlers_backup = self.logger.handlers parameters = Parameters() setattr(parameters, 'source_queue', src_name) setattr(parameters, 'destination_queues', destination_queues) with mock.patch('intelmq.lib.utils.load_configuration', new=self.mocked_config): with mock.patch('intelmq.lib.utils.log', self.get_mocked_logger(self.logger)): self.bot = self.bot_reference(self.bot_id) self.bot._Bot__stats_cache = None self.pipe = pipeline.Pythonlist(parameters, logger=self.logger, bot=self.bot) self.pipe.set_queues(parameters.source_queue, "source") self.pipe.set_queues(parameters.destination_queues, "destination") if self.input_message is not None: if not isinstance(self.input_message, (list, tuple)): self.input_message = [self.input_message] self.input_queue = [] for msg in self.input_message: if type(msg) is dict: self.input_queue.append(json.dumps(msg)) elif issubclass(type(msg), message.Message): self.input_queue.append(msg.serialize()) else: self.input_queue.append(msg) self.input_message = None else: if self.default_input_message: # None for collectors self.input_queue = [self.default_input_message]
def prepare_bot(self, parameters={}, destination_queues=None): """ Reconfigures the bot with the changed attributes. Parameters: parameters: optional bot parameters for this run, as dict destination_queues: optional definition of destination queues default: {"_default": "{}-output".format(self.bot_id)} """ self.log_stream = io.StringIO() src_name = "{}-input".format(self.bot_id) if not destination_queues: destination_queues = {"_default": "{}-output".format(self.bot_id)} else: destination_queues = { queue_name: "%s-%s-output" % (self.bot_id, queue_name.strip('_')) for queue_name in destination_queues } config = self.sysconfig.copy() config.update(parameters) self.mocked_config = mocked_config( self.bot_id, src_name, destination_queues, sysconfig=config, group=self.bot_type.title(), module=self.bot_reference.__module__, ) logger = logging.getLogger(self.bot_id) logger.setLevel("INFO") console_formatter = logging.Formatter(utils.LOG_FORMAT) console_handler = logging.StreamHandler(self.log_stream) console_handler.setFormatter(console_formatter) logger.addHandler(console_handler) self.mocked_log = mocked_logger(logger) logging.captureWarnings(True) warnings_logger = logging.getLogger("py.warnings") warnings_logger.addHandler(console_handler) parameters = Parameters() setattr(parameters, 'source_queue', src_name) setattr(parameters, 'destination_queues', destination_queues) with mock.patch('intelmq.lib.utils.load_configuration', new=self.mocked_config): with mock.patch('intelmq.lib.utils.log', self.mocked_log): self.bot = self.bot_reference(self.bot_id) self.bot._Bot__stats_cache = None self.pipe = pipeline.Pythonlist(parameters, logger=logger, bot=self.bot) self.pipe.set_queues(parameters.source_queue, "source") self.pipe.set_queues(parameters.destination_queues, "destination") if self.input_message is not None: if type(self.input_message) is not list: self.input_message = [self.input_message] self.input_queue = [] for msg in self.input_message: if type(msg) is dict: self.input_queue.append(json.dumps(msg)) elif issubclass(type(msg), message.Message): self.input_queue.append(msg.serialize()) else: self.input_queue.append(msg) self.input_message = None else: if self.default_input_message: # None for collectors self.input_queue = [self.default_input_message]
def prepare_bot(self, parameters={}, destination_queues=None): """ Reconfigures the bot with the changed attributes. Parameters: parameters: optional bot parameters for this run, as dict destination_queues: optional definition of destination queues default: {"_default": "{}-output".format(self.bot_id)} """ self.log_stream = io.StringIO() src_name = "{}-input".format(self.bot_id) if not destination_queues: destination_queues = {"_default": "{}-output".format(self.bot_id)} else: destination_queues = { queue_name: "%s-%s-output" % (self.bot_id, queue_name.strip('_')) for queue_name in destination_queues } config = BOT_CONFIG.copy() config.update(self.sysconfig) config.update(parameters) config['destination_queues'] = destination_queues self.mocked_config = mocked_config( self.bot_id, sysconfig=config, group=self.bot_type.title(), module=self.bot_reference.__module__, ) self.logger = utils.log(self.bot_id, log_path=False, stream=self.log_stream, log_format_stream=utils.LOG_FORMAT, log_level=config['logging_level']) self.logger_handlers_backup = self.logger.handlers parameters = Parameters() setattr(parameters, 'source_queue', src_name) setattr(parameters, 'destination_queues', destination_queues) with mock.patch('intelmq.lib.utils.load_configuration', new=self.mocked_config): with mock.patch('intelmq.lib.utils.log', self.get_mocked_logger(self.logger)): with mock.patch('intelmq.lib.utils.get_global_settings', mocked_get_global_settings): self.bot = self.bot_reference(self.bot_id) self.bot._Bot__stats_cache = None pipeline_args = { key: getattr(self, key) for key in dir(self) if not inspect.ismethod(getattr(self, key)) and (key.startswith( 'source_pipeline_') or key.startswith('destination_pipeline')) } self.pipe = pipeline.Pythonlist( logger=self.logger, pipeline_args=pipeline_args, load_balance=self.bot.load_balance, is_multithreaded=self.bot.is_multithreaded) self.pipe.set_queues(parameters.source_queue, "source") self.pipe.set_queues(parameters.destination_queues, "destination") self.prepare_source_queue()