def prepare_bot(self, raise_on_connect=False):
self.log_stream = io.StringIO()
self.bot_id = 'test-bot'
src_name = "{}-input".format(self.bot_id)
dst_name = "{}-output".format(self.bot_id)
self.mocked_config = test.mocked_config(
self.bot_id, src_name, [dst_name],
{"raise_on_connect": raise_on_connect})
logger = logging.getLogger(self.bot_id)
logger.setLevel("DEBUG")
console_formatter = logging.Formatter(utils.LOG_FORMAT)
console_handler = logging.StreamHandler(self.log_stream)
console_handler.setFormatter(console_formatter)
logger.addHandler(console_handler)
self.mocked_log = test.mocked_logger(logger)
class Parameters(object):
source_queue = src_name
destination_queues = [dst_name]
parameters = Parameters()
pipe = pipeline.Pythonlist(parameters)
pipe.set_queues(parameters.source_queue, "source")
pipe.set_queues(parameters.destination_queues, "destination")
with mock.patch('intelmq.lib.utils.load_configuration',
new=self.mocked_config):
with mock.patch('intelmq.lib.utils.log', self.mocked_log):
self.bot = self.bot_reference(self.bot_id)
self.pipe = pipe
def prepare_bot(self):
"""Reconfigures the bot with the changed attributes"""
self.log_stream = io.StringIO()
src_name = "{}-input".format(self.bot_id)
dst_name = "{}-output".format(self.bot_id)
self.mocked_config = mocked_config(
self.bot_id,
src_name,
[dst_name],
sysconfig=self.sysconfig,
)
logger = logging.getLogger(self.bot_id)
logger.setLevel("INFO")
console_formatter = logging.Formatter(utils.LOG_FORMAT)
console_handler = logging.StreamHandler(self.log_stream)
console_handler.setFormatter(console_formatter)
logger.addHandler(console_handler)
self.mocked_log = mocked_logger(logger)
logging.captureWarnings(True)
warnings_logger = logging.getLogger("py.warnings")
warnings_logger.addHandler(console_handler)
class Parameters(object):
source_queue = src_name
destination_queues = [dst_name]
parameters = Parameters()
self.pipe = pipeline.Pythonlist(parameters)
self.pipe.set_queues(parameters.source_queue, "source")
self.pipe.set_queues(parameters.destination_queues, "destination")
with mock.patch('intelmq.lib.utils.load_configuration',
new=self.mocked_config):
with mock.patch('intelmq.lib.utils.log', self.mocked_log):
self.bot = self.bot_reference(self.bot_id)
if self.input_message is not None:
if type(self.input_message) is not list:
self.input_message = [self.input_message]
self.input_queue = []
for msg in self.input_message:
if type(msg) is dict:
self.input_queue.append(json.dumps(msg))
elif issubclass(type(msg), message.Message):
self.input_queue.append(msg.serialize())
else:
self.input_queue.append(msg)
self.input_message = None
else:
if self.default_input_message: # None for collectors
self.input_queue = [self.default_input_message]
def prepare_bot(self, parameters={}, destination_queues=None):
"""
Reconfigures the bot with the changed attributes.
Parameters:
parameters: optional bot parameters for this run, as dict
destination_queues: optional definition of destination queues
default: {"_default": "{}-output".format(self.bot_id)}
"""
self.log_stream = io.StringIO()
src_name = "{}-input".format(self.bot_id)
if not destination_queues:
destination_queues = {"_default": "{}-output".format(self.bot_id)}
else:
destination_queues = {queue_name: "%s-%s-output" % (self.bot_id,
queue_name.strip('_'))
for queue_name in destination_queues}
config = self.sysconfig.copy()
config.update(parameters)
self.mocked_config = mocked_config(self.bot_id,
src_name,
destination_queues,
sysconfig=config,
group=self.bot_type.title(),
module=self.bot_reference.__module__,
)
self.resulting_config = BOT_CONFIG.copy()
self.resulting_config.update(self.sysconfig)
self.resulting_config.update(parameters)
self.logger = utils.log(self.bot_id,
log_path=False, stream=self.log_stream,
log_format_stream=utils.LOG_FORMAT,
log_level=self.resulting_config['logging_level'])
self.logger_handlers_backup = self.logger.handlers
parameters = Parameters()
setattr(parameters, 'source_queue', src_name)
setattr(parameters, 'destination_queues', destination_queues)
with mock.patch('intelmq.lib.utils.load_configuration',
new=self.mocked_config):
with mock.patch('intelmq.lib.utils.log', self.get_mocked_logger(self.logger)):
self.bot = self.bot_reference(self.bot_id)
self.bot._Bot__stats_cache = None
self.pipe = pipeline.Pythonlist(parameters, logger=self.logger, bot=self.bot)
self.pipe.set_queues(parameters.source_queue, "source")
self.pipe.set_queues(parameters.destination_queues, "destination")
if self.input_message is not None:
if not isinstance(self.input_message, (list, tuple)):
self.input_message = [self.input_message]
self.input_queue = []
for msg in self.input_message:
if type(msg) is dict:
self.input_queue.append(json.dumps(msg))
elif issubclass(type(msg), message.Message):
self.input_queue.append(msg.serialize())
else:
self.input_queue.append(msg)
self.input_message = None
else:
if self.default_input_message: # None for collectors
self.input_queue = [self.default_input_message]
def prepare_bot(self, parameters={}, destination_queues=None):
"""
Reconfigures the bot with the changed attributes.
Parameters:
parameters: optional bot parameters for this run, as dict
destination_queues: optional definition of destination queues
default: {"_default": "{}-output".format(self.bot_id)}
"""
self.log_stream = io.StringIO()
src_name = "{}-input".format(self.bot_id)
if not destination_queues:
destination_queues = {"_default": "{}-output".format(self.bot_id)}
else:
destination_queues = {
queue_name:
"%s-%s-output" % (self.bot_id, queue_name.strip('_'))
for queue_name in destination_queues
}
config = self.sysconfig.copy()
config.update(parameters)
self.mocked_config = mocked_config(
self.bot_id,
src_name,
destination_queues,
sysconfig=config,
group=self.bot_type.title(),
module=self.bot_reference.__module__,
)
logger = logging.getLogger(self.bot_id)
logger.setLevel("INFO")
console_formatter = logging.Formatter(utils.LOG_FORMAT)
console_handler = logging.StreamHandler(self.log_stream)
console_handler.setFormatter(console_formatter)
logger.addHandler(console_handler)
self.mocked_log = mocked_logger(logger)
logging.captureWarnings(True)
warnings_logger = logging.getLogger("py.warnings")
warnings_logger.addHandler(console_handler)
parameters = Parameters()
setattr(parameters, 'source_queue', src_name)
setattr(parameters, 'destination_queues', destination_queues)
with mock.patch('intelmq.lib.utils.load_configuration',
new=self.mocked_config):
with mock.patch('intelmq.lib.utils.log', self.mocked_log):
self.bot = self.bot_reference(self.bot_id)
self.bot._Bot__stats_cache = None
self.pipe = pipeline.Pythonlist(parameters,
logger=logger,
bot=self.bot)
self.pipe.set_queues(parameters.source_queue, "source")
self.pipe.set_queues(parameters.destination_queues, "destination")
if self.input_message is not None:
if type(self.input_message) is not list:
self.input_message = [self.input_message]
self.input_queue = []
for msg in self.input_message:
if type(msg) is dict:
self.input_queue.append(json.dumps(msg))
elif issubclass(type(msg), message.Message):
self.input_queue.append(msg.serialize())
else:
self.input_queue.append(msg)
self.input_message = None
else:
if self.default_input_message: # None for collectors
self.input_queue = [self.default_input_message]
def prepare_bot(self, parameters={}, destination_queues=None):
"""
Reconfigures the bot with the changed attributes.
Parameters:
parameters: optional bot parameters for this run, as dict
destination_queues: optional definition of destination queues
default: {"_default": "{}-output".format(self.bot_id)}
"""
self.log_stream = io.StringIO()
src_name = "{}-input".format(self.bot_id)
if not destination_queues:
destination_queues = {"_default": "{}-output".format(self.bot_id)}
else:
destination_queues = {
queue_name:
"%s-%s-output" % (self.bot_id, queue_name.strip('_'))
for queue_name in destination_queues
}
config = BOT_CONFIG.copy()
config.update(self.sysconfig)
config.update(parameters)
config['destination_queues'] = destination_queues
self.mocked_config = mocked_config(
self.bot_id,
sysconfig=config,
group=self.bot_type.title(),
module=self.bot_reference.__module__,
)
self.logger = utils.log(self.bot_id,
log_path=False,
stream=self.log_stream,
log_format_stream=utils.LOG_FORMAT,
log_level=config['logging_level'])
self.logger_handlers_backup = self.logger.handlers
parameters = Parameters()
setattr(parameters, 'source_queue', src_name)
setattr(parameters, 'destination_queues', destination_queues)
with mock.patch('intelmq.lib.utils.load_configuration',
new=self.mocked_config):
with mock.patch('intelmq.lib.utils.log',
self.get_mocked_logger(self.logger)):
with mock.patch('intelmq.lib.utils.get_global_settings',
mocked_get_global_settings):
self.bot = self.bot_reference(self.bot_id)
self.bot._Bot__stats_cache = None
pipeline_args = {
key: getattr(self, key)
for key in dir(self)
if not inspect.ismethod(getattr(self, key)) and (key.startswith(
'source_pipeline_') or key.startswith('destination_pipeline'))
}
self.pipe = pipeline.Pythonlist(
logger=self.logger,
pipeline_args=pipeline_args,
load_balance=self.bot.load_balance,
is_multithreaded=self.bot.is_multithreaded)
self.pipe.set_queues(parameters.source_queue, "source")
self.pipe.set_queues(parameters.destination_queues, "destination")
self.prepare_source_queue()
