def __init__(self, bot_id):
self.__log_buffer = []
self.parameters = Parameters()
self.__error_retries_counter = 0
self.__source_pipeline = None
self.__destination_pipeline = None
self.logger = None
try:
version_info = sys.version.splitlines()[0].strip()
self.__log_buffer.append(('info',
'{} initialized with id {} and version '
'{} as process {}.'
''.format(self.__class__.__name__,
bot_id, version_info,
os.getpid())))
self.__log_buffer.append(('debug', 'Library path: %r.' % __file__))
self.__load_defaults_configuration()
self.__load_system_configuration()
self.__check_bot_id(bot_id)
self.__bot_id = bot_id
if self.parameters.logging_handler == 'syslog':
syslog = self.parameters.logging_syslog
else:
syslog = False
self.logger = utils.log(self.__bot_id, syslog=syslog,
log_path=self.parameters.logging_path,
log_level=self.parameters.logging_level)
except:
self.__log_buffer.append(('critical', traceback.format_exc()))
self.stop()
else:
for line in self.__log_buffer:
getattr(self.logger, line[0])(line[1])
try:
self.logger.info('Bot is starting.')
self.__load_runtime_configuration()
self.__load_pipeline_configuration()
self.__load_harmonization_configuration()
self.init()
self.__sighup = False
signal.signal(signal.SIGHUP, self.__handle_sighup_signal)
# system calls should not be interrupted, but restarted
signal.siginterrupt(signal.SIGHUP, False)
except Exception as exc:
if self.parameters.error_log_exception:
self.logger.exception('Bot initialization failed.')
else:
self.logger.error(utils.error_message_from_exc(exc))
self.logger.error('Bot initialization failed.')
self.stop()
raise
def __init__(self, bot_id):
self.__log_buffer = []
self.parameters = Parameters()
self.__error_retries_counter = 0
self.__source_pipeline = None
self.__destination_pipeline = None
self.logger = None
try:
version_info = sys.version.splitlines()[0].strip()
self.__log_buffer.append(
('info', '{} initialized with id {} and version '
'{} as process {}.'
''.format(self.__class__.__name__, bot_id, version_info,
os.getpid())))
self.__log_buffer.append(('debug', 'Library path: %r.' % __file__))
self.__load_defaults_configuration()
self.__load_system_configuration()
self.__check_bot_id(bot_id)
self.__bot_id = bot_id
if self.parameters.logging_handler == 'syslog':
syslog = self.parameters.logging_syslog
else:
syslog = False
self.logger = utils.log(self.__bot_id,
syslog=syslog,
log_path=self.parameters.logging_path,
log_level=self.parameters.logging_level)
except:
self.__log_buffer.append(('critical', traceback.format_exc()))
self.stop()
else:
for line in self.__log_buffer:
getattr(self.logger, line[0])(line[1])
try:
self.logger.info('Bot is starting.')
self.__load_runtime_configuration()
self.__load_pipeline_configuration()
self.__load_harmonization_configuration()
self.init()
self.__sighup = False
signal.signal(signal.SIGHUP, self.__handle_sighup_signal)
# system calls should not be interrupted, but restarted
signal.siginterrupt(signal.SIGHUP, False)
except Exception as exc:
if self.parameters.error_log_exception:
self.logger.exception('Bot initialization failed.')
else:
self.logger.error(utils.error_message_from_exc(exc))
self.logger.error('Bot initialization failed.')
self.stop()
raise
def __init__(self, bot_id: str):
self.__log_buffer = []
self.parameters = Parameters()
self.__error_retries_counter = 0
self.__source_pipeline = None
self.__destination_pipeline = None
self.logger = None
try:
version_info = sys.version.splitlines()[0].strip()
self.__log_buffer.append(
('info', '{bot} initialized with id {id} and intelmq {intelmq}'
' and python {python} as process {pid}.'
''.format(bot=self.__class__.__name__,
id=bot_id,
python=version_info,
pid=os.getpid(),
intelmq=__version__)))
self.__log_buffer.append(('debug', 'Library path: %r.' % __file__))
self.__load_defaults_configuration()
self.__check_bot_id(bot_id)
self.__bot_id = bot_id
self.__init_logger()
except Exception:
self.__log_buffer.append(('critical', traceback.format_exc()))
self.stop()
else:
for line in self.__log_buffer:
getattr(self.logger, line[0])(line[1])
try:
self.logger.info('Bot is starting.')
self.__load_runtime_configuration()
self.__load_pipeline_configuration()
self.__load_harmonization_configuration()
self.init()
self.__sighup = False
signal.signal(signal.SIGHUP, self.__handle_sighup_signal)
# system calls should not be interrupted, but restarted
signal.siginterrupt(signal.SIGHUP, False)
signal.signal(signal.SIGTERM, self.__handle_sigterm_signal)
except Exception as exc:
if self.parameters.error_log_exception:
self.logger.exception('Bot initialization failed.')
else:
self.logger.error(utils.error_message_from_exc(exc))
self.logger.error('Bot initialization failed.')
self.stop()
raise
def arg2msg(self, msg):
try:
default_type = "Report" if self.runtime_configuration["group"] is "Parser" else "Event"
msg = MessageFactory.unserialize(msg, default_type=default_type)
except (Exception, KeyError, TypeError, ValueError) as exc:
if exists(msg):
with open(msg, "r") as f:
return self.arg2msg(f.read())
self.messageWizzard("Message can not be parsed from JSON: {}".format(error_message_from_exc(exc)))
exit(1)
return msg
def arg2msg(self, msg):
try:
default_type = "Report" if self.runtime_configuration["group"] == "Parser" else "Event"
msg = MessageFactory.unserialize(msg, default_type=default_type)
except (Exception, KeyError, TypeError, ValueError) as exc:
if exists(msg):
with open(msg, "r") as f:
return self.arg2msg(f.read())
self.messageWizzard("Message can not be parsed from JSON: {}".format(error_message_from_exc(exc)))
sys.exit(1)
return msg
def arg2msg(self, msg):
default_type = "Report" if (
self.runtime_configuration.get("group", None) == "Parser"
or isinstance(self.instance, ParserBot)) else "Event"
try:
msg = MessageFactory.unserialize(msg, default_type=default_type)
except (Exception, KeyError, TypeError, ValueError) as exc:
if exists(msg):
with open(msg, "r") as f:
return self.arg2msg(f.read())
self.messageWizzard(
"Message can not be parsed from JSON: {}".format(
error_message_from_exc(exc)))
sys.exit(1)
return msg
def start(self,
starting=True,
error_on_pipeline=True,
error_on_message=False,
source_pipeline=None,
destination_pipeline=None):
self.__source_pipeline = source_pipeline
self.__destination_pipeline = destination_pipeline
self.logger.info('Bot starts processings.')
while True:
try:
if not starting and (error_on_pipeline or error_on_message):
self.logger.info('Bot will restart in %s seconds.' %
self.parameters.error_retry_delay)
time.sleep(self.parameters.error_retry_delay)
self.logger.info('Bot woke up')
self.logger.info('Trying to start processing again.')
if error_on_message:
error_on_message = False
if error_on_pipeline:
self.__connect_pipelines()
error_on_pipeline = False
if starting:
self.logger.info("Start processing.")
starting = False
self.__handle_sighup()
self.process()
self.__error_retries_counter = 0 # reset counter
if self.parameters.rate_limit:
self.__sleep()
except exceptions.PipelineError:
error_on_pipeline = True
if self.parameters.error_log_exception:
self.logger.exception('Pipeline failed.')
else:
self.logger.error('Pipeline failed.')
self.__disconnect_pipelines()
except Exception as exc:
error_on_message = sys.exc_info()
if self.parameters.error_log_exception:
self.logger.exception("Bot has found a problem.")
else:
self.logger.error(utils.error_message_from_exc(exc))
self.logger.error("Bot has found a problem.")
if self.parameters.error_log_message:
# Dump full message if explicitly requested by config
self.logger.info("Current Message(event): {!r}."
"".format(self.__current_message))
except KeyboardInterrupt:
self.logger.error("Received KeyboardInterrupt.")
self.stop(exitcode=0)
del self
break
finally:
if getattr(self.parameters, 'testing', False):
self.stop()
break
if error_on_message or error_on_pipeline:
self.__error_retries_counter += 1
# reached the maximum number of retries
if (self.__error_retries_counter >
self.parameters.error_max_retries):
if error_on_message:
if self.parameters.error_dump_message:
error_traceback = traceback.format_exception(
*error_on_message)
self._dump_message(
error_traceback,
message=self.__current_message)
self.__current_message = None
# remove message from pipeline
self.acknowledge_message()
# when bot acknowledge the message,
# dont need to wait again
error_on_message = False
# error_procedure: stop
if self.parameters.error_procedure == "stop":
self.stop()
# error_procedure: pass
else:
self.__error_retries_counter = 0 # reset counter
self.__handle_sighup()
def start(self,
starting: bool = True,
error_on_pipeline: bool = True,
error_on_message: bool = False,
source_pipeline: Optional[str] = None,
destination_pipeline: Optional[str] = None):
self.__source_pipeline = source_pipeline
self.__destination_pipeline = destination_pipeline
while True:
try:
if not starting and (error_on_pipeline or error_on_message):
self.logger.info('Bot will continue in %s seconds.',
self.parameters.error_retry_delay)
time.sleep(self.parameters.error_retry_delay)
if error_on_message:
error_on_message = False
if error_on_pipeline:
self.__connect_pipelines()
error_on_pipeline = False
if starting:
starting = False
self.__handle_sighup()
self.process()
self.__error_retries_counter = 0 # reset counter
if self.parameters.rate_limit and self.run_mode != 'scheduled':
self.__sleep()
except exceptions.PipelineError as exc:
error_on_pipeline = True
if self.parameters.error_log_exception:
self.logger.exception('Pipeline failed.')
else:
self.logger.error(utils.error_message_from_exc(exc))
self.logger.error('Pipeline failed.')
self.__disconnect_pipelines()
except Exception as exc:
# in case of serious system issues, exit immediately
if isinstance(exc, MemoryError):
self.logger.exception('Out of memory. Exit immediately.')
self.stop()
elif isinstance(exc, (IOError, OSError)) and exc.errno == 28:
self.logger.exception(
'Out of disk space. Exit immediately.')
self.stop()
error_on_message = sys.exc_info()
if self.parameters.error_log_exception:
self.logger.exception("Bot has found a problem.")
else:
self.logger.error(utils.error_message_from_exc(exc))
self.logger.error("Bot has found a problem.")
if self.parameters.error_log_message:
# Dump full message if explicitly requested by config
self.logger.info("Current Message(event): %r.",
self.__current_message)
# In case of permanent failures, stop now
if isinstance(exc, exceptions.ConfigurationError):
self.stop()
except KeyboardInterrupt:
self.logger.info("Received KeyboardInterrupt.")
self.stop(exitcode=0)
del self
break
finally:
if getattr(self.parameters, 'testing', False):
self.stop()
break
if error_on_message or error_on_pipeline:
self.__error_retries_counter += 1
# reached the maximum number of retries
if (self.__error_retries_counter >
self.parameters.error_max_retries):
if error_on_message:
if self.parameters.error_dump_message:
error_traceback = traceback.format_exception(
*error_on_message)
self._dump_message(
error_traceback,
message=self.__current_message)
self.__current_message = None
# remove message from pipeline
self.acknowledge_message()
# when bot acknowledge the message,
# don't need to wait again
error_on_message = False
# run_mode: scheduled
if self.run_mode == 'scheduled':
self.logger.info('Shutting down scheduled bot.')
self.stop()
# error_procedure: stop
elif self.parameters.error_procedure == "stop":
self.stop()
# error_procedure: pass
else:
self.__error_retries_counter = 0 # reset counter
# no errors, check for run mode: scheduled
elif self.run_mode == 'scheduled':
self.logger.info('Shutting down scheduled bot.')
self.stop()
self.__handle_sighup()
def __init__(self, bot_id: str, start=False, sighup_event=None,
disable_multithreading=None):
self.__log_buffer = []
self.parameters = Parameters()
self.__error_retries_counter = 0
self.__source_pipeline = None
self.__destination_pipeline = None
self.logger = None
self.__message_counter = {"since": 0, # messages since last logging
"start": None, # last login time
"success": 0, # total number since the beginning
"failure": 0, # total number since the beginning
"stats_timestamp": datetime.now(), # stamp of last report to redis
"path": defaultdict(int), # number of messages sent to queues since last report to redis
"path_total": defaultdict(int) # number of messages sent to queues since beginning
}
try:
version_info = sys.version.splitlines()[0].strip()
self.__log_buffer.append(('info',
'{bot} initialized with id {id} and intelmq {intelmq}'
' and python {python} as process {pid}.'
''.format(bot=self.__class__.__name__,
id=bot_id, python=version_info,
pid=os.getpid(), intelmq=__version__)))
self.__log_buffer.append(('debug', 'Library path: %r.' % __file__))
if not utils.drop_privileges():
raise ValueError('IntelMQ must not run as root. Dropping privileges did not work.')
self.__load_defaults_configuration()
self.__bot_id_full, self.__bot_id, self.__instance_id = self.__check_bot_id(bot_id)
if self.__instance_id:
self.is_multithreaded = True
self.__init_logger()
except Exception:
self.__log_buffer.append(('critical', traceback.format_exc()))
self.stop()
else:
for line in self.__log_buffer:
getattr(self.logger, line[0])(line[1])
try:
self.logger.info('Bot is starting.')
self.__load_runtime_configuration()
broker = getattr(self.parameters, "source_pipeline_broker",
getattr(self.parameters, "broker", "redis")).title()
if broker != 'Amqp':
self.is_multithreadable = False
""" Multithreading """
if (getattr(self.parameters, 'instances_threads', 0) > 1 and
not self.is_multithreaded and
self.is_multithreadable and
not disable_multithreading):
self.logger.handlers = []
num_instances = int(self.parameters.instances_threads)
instances = []
sighup_events = []
def handle_sighup_signal_threading(signum: int,
stack: Optional[object]):
for event in sighup_events:
event.set()
signal.signal(signal.SIGHUP, handle_sighup_signal_threading)
for i in range(num_instances):
sighup_events.append(threading.Event())
threadname = '%s.%d' % (bot_id, i)
instances.append(threading.Thread(target=self.__class__,
kwargs={'bot_id': threadname,
'start': True,
'sighup_event': sighup_events[-1]},
name=threadname,
daemon=False))
instances[i].start()
for i, thread in enumerate(instances):
thread.join()
return
elif (getattr(self.parameters, 'instances_threads', 1) > 1 and
not self.is_multithreadable):
self.logger.error('Multithreading is configured, but is not '
'available for this bot. Look at the FAQ '
'for a list of reasons for this. '
'https://github.com/certtools/intelmq/blob/master/docs/FAQ.md')
elif disable_multithreading:
self.logger.warning('Multithreading is configured, but is not '
'available for interactive runs.')
self.__load_pipeline_configuration()
self.__load_harmonization_configuration()
self.init()
if not self.__instance_id:
self.__sighup = threading.Event()
signal.signal(signal.SIGHUP, self.__handle_sighup_signal)
# system calls should not be interrupted, but restarted
signal.siginterrupt(signal.SIGHUP, False)
signal.signal(signal.SIGTERM, self.__handle_sigterm_signal)
signal.signal(signal.SIGINT, self.__handle_sigterm_signal)
else:
self.__sighup = sighup_event
@atexit.register
def catch_shutdown():
self.stop()
except Exception as exc:
if self.parameters.error_log_exception:
self.logger.exception('Bot initialization failed.')
else:
self.logger.error(utils.error_message_from_exc(exc))
self.logger.error('Bot initialization failed.')
self.stop()
raise
self.__stats_cache = cache.Cache(host=getattr(self.parameters,
"statistics_host",
"127.0.0.1"),
port=getattr(self.parameters,
"statistics_port", "6379"),
db=int(getattr(self.parameters,
"statistics_database", 3)),
password=getattr(self.parameters,
"statistics_password",
None),
ttl=None,
)
if start:
self.start()
def start(self, starting: bool = True, error_on_pipeline: bool = True,
error_on_message: bool = False, source_pipeline: Optional[str] = None,
destination_pipeline: Optional[str] = None):
self.__source_pipeline = source_pipeline
self.__destination_pipeline = destination_pipeline
while True:
try:
if not starting and (error_on_pipeline or error_on_message):
self.logger.info('Bot will continue in %s seconds.',
self.parameters.error_retry_delay)
time.sleep(self.parameters.error_retry_delay)
if error_on_message:
error_on_message = False
if error_on_pipeline:
try:
self.__connect_pipelines()
except Exception as exc:
raise exceptions.PipelineError(exc)
else:
error_on_pipeline = False
if starting:
starting = False
self.__handle_sighup()
self.process()
self.__error_retries_counter = 0 # reset counter
if self.parameters.rate_limit and self.run_mode != 'scheduled':
self.__sleep()
if self.collector_empty_process and self.run_mode != 'scheduled':
self.__sleep(1, log=False)
except exceptions.PipelineError as exc:
error_on_pipeline = True
if self.parameters.error_log_exception:
self.logger.exception('Pipeline failed.')
else:
self.logger.error(utils.error_message_from_exc(exc))
self.logger.error('Pipeline failed.')
self.__disconnect_pipelines()
except Exception as exc:
# in case of serious system issues, exit immediately
if isinstance(exc, MemoryError):
self.logger.exception('Out of memory. Exit immediately. Reason: %r.' % exc.args[0])
self.stop()
elif isinstance(exc, (IOError, OSError)) and exc.errno == 28:
self.logger.exception('Out of disk space. Exit immediately.')
self.stop()
error_on_message = sys.exc_info()
if self.parameters.error_log_exception:
self.logger.exception("Bot has found a problem.")
else:
self.logger.error(utils.error_message_from_exc(exc))
self.logger.error("Bot has found a problem.")
if self.parameters.error_log_message:
# Print full message if explicitly requested by config
self.logger.info("Current Message(event): %r.",
self.__current_message)
# In case of permanent failures, stop now
if isinstance(exc, exceptions.ConfigurationError):
self.stop()
except KeyboardInterrupt:
self.logger.info("Received KeyboardInterrupt.")
self.stop(exitcode=0)
finally:
if getattr(self.parameters, 'testing', False):
self.stop(exitcode=0)
break
if error_on_message or error_on_pipeline:
self.__message_counter["failure"] += 1
self.__error_retries_counter += 1
# reached the maximum number of retries
if (self.__error_retries_counter >
self.parameters.error_max_retries):
if error_on_message:
if self.parameters.error_dump_message:
error_traceback = traceback.format_exception(*error_on_message)
self._dump_message(error_traceback,
message=self.__current_message)
else:
warnings.warn("Message will be removed from the pipeline and not dumped to the disk. "
"Set `error_dump_message` to true to save the message on disk. "
"This warning is only shown once in the runtime of a bot.")
if self.__destination_queues and '_on_error' in self.__destination_queues:
self.send_message(self.__current_message, path='_on_error')
# remove message from pipeline
self.acknowledge_message()
# when bot acknowledge the message,
# don't need to wait again
error_on_message = False
# run_mode: scheduled
if self.run_mode == 'scheduled':
self.logger.info('Shutting down scheduled bot.')
self.stop(exitcode=0)
# error_procedure: stop
elif self.parameters.error_procedure == "stop":
self.stop()
# error_procedure: pass
elif not error_on_pipeline:
self.__error_retries_counter = 0 # reset counter
# error_procedure: pass and pipeline problem
else:
# retry forever, see https://github.com/certtools/intelmq/issues/1333
# https://lists.cert.at/pipermail/intelmq-users/2018-October/000085.html
pass
else:
self.__message_counter["success"] += 1
# no errors, check for run mode: scheduled
if self.run_mode == 'scheduled':
self.logger.info('Shutting down scheduled bot.')
self.stop(exitcode=0)
self.__stats()
self.__handle_sighup()
def test_error_message_from_exc(self):
"""Tests if error_message_from_exc correctly returns the error message."""
exc = IndexError('This is a test')
self.assertEqual(utils.error_message_from_exc(exc), 'This is a test')
def __init__(self, bot_id: str, start=False, sighup_event=None,
disable_multithreading=None):
self.__log_buffer = []
self.parameters = Parameters()
self.__error_retries_counter = 0
self.__source_pipeline = None
self.__destination_pipeline = None
self.logger = None
self.__message_counter = {"since": 0, # messages since last logging
"start": None, # last login time
"success": 0, # total number since the beginning
"failure": 0, # total number since the beginning
"stats_timestamp": datetime.now(), # stamp of last report to redis
"path": defaultdict(int), # number of messages sent to queues since last report to redis
"path_total": defaultdict(int) # number of messages sent to queues since beginning
}
try:
version_info = sys.version.splitlines()[0].strip()
self.__log_buffer.append(('info',
'{bot} initialized with id {id} and intelmq {intelmq}'
' and python {python} as process {pid}.'
''.format(bot=self.__class__.__name__,
id=bot_id, python=version_info,
pid=os.getpid(), intelmq=__version__)))
self.__log_buffer.append(('debug', 'Library path: %r.' % __file__))
if not utils.drop_privileges():
raise ValueError('IntelMQ must not run as root. Dropping privileges did not work.')
self.__load_defaults_configuration()
self.__bot_id_full, self.__bot_id, self.__instance_id = self.__check_bot_id(bot_id)
if self.__instance_id:
self.is_multithreaded = True
self.__init_logger()
except Exception:
self.__log_buffer.append(('critical', traceback.format_exc()))
self.stop()
else:
for line in self.__log_buffer:
getattr(self.logger, line[0])(line[1])
try:
self.logger.info('Bot is starting.')
self.__load_runtime_configuration()
broker = getattr(self.parameters, "source_pipeline_broker",
getattr(self.parameters, "broker", "redis")).title()
if broker != 'Amqp':
self.is_multithreadable = False
""" Multithreading """
if (getattr(self.parameters, 'instances_threads', 0) > 1 and
not self.is_multithreaded and
self.is_multithreadable and
not disable_multithreading):
self.logger.handlers = []
num_instances = int(self.parameters.instances_threads)
instances = []
sighup_events = []
def handle_sighup_signal_threading(signum: int,
stack: Optional[object]):
for event in sighup_events:
event.set()
signal.signal(signal.SIGHUP, handle_sighup_signal_threading)
for i in range(num_instances):
sighup_events.append(threading.Event())
threadname = '%s.%d' % (bot_id, i)
instances.append(threading.Thread(target=self.__class__,
kwargs={'bot_id': threadname,
'start': True,
'sighup_event': sighup_events[-1]},
name=threadname,
daemon=False))
instances[i].start()
for i, thread in enumerate(instances):
thread.join()
return
elif (getattr(self.parameters, 'instances_threads', 1) > 1 and
not self.is_multithreadable):
self.logger.error('Multithreading is configured, but is not '
'available for this bot. Look at the FAQ '
'for a list of reasons for this. '
'https://github.com/certtools/intelmq/blob/master/docs/FAQ.md')
elif disable_multithreading:
self.logger.warning('Multithreading is configured, but is not '
'available for interactive runs.')
self.__load_pipeline_configuration()
self.__load_harmonization_configuration()
self.init()
if not self.__instance_id:
self.__sighup = threading.Event()
signal.signal(signal.SIGHUP, self.__handle_sighup_signal)
# system calls should not be interrupted, but restarted
signal.siginterrupt(signal.SIGHUP, False)
signal.signal(signal.SIGTERM, self.__handle_sigterm_signal)
signal.signal(signal.SIGINT, self.__handle_sigterm_signal)
else:
self.__sighup = sighup_event
@atexit.register
def catch_shutdown():
self.stop()
except Exception as exc:
if self.parameters.error_log_exception:
self.logger.exception('Bot initialization failed.')
else:
self.logger.error(utils.error_message_from_exc(exc))
self.logger.error('Bot initialization failed.')
self.stop()
raise
self.__stats_cache = cache.Cache(host=getattr(self.parameters,
"statistics_host",
"127.0.0.1"),
port=getattr(self.parameters,
"statistics_port", "6379"),
db=int(getattr(self.parameters,
"statistics_database", 3)),
password=getattr(self.parameters,
"statistics_password",
None),
ttl=None,
)
if start:
self.start()
def start(self, starting: bool = True, error_on_pipeline: bool = True,
error_on_message: bool = False, source_pipeline: Optional[str] = None,
destination_pipeline: Optional[str] = None):
self.__source_pipeline = source_pipeline
self.__destination_pipeline = destination_pipeline
while True:
try:
if not starting and (error_on_pipeline or error_on_message):
self.logger.info('Bot will continue in %s seconds.',
self.parameters.error_retry_delay)
time.sleep(self.parameters.error_retry_delay)
if error_on_message:
error_on_message = False
if error_on_pipeline:
try:
self.__connect_pipelines()
except Exception as exc:
raise exceptions.PipelineError(exc)
else:
error_on_pipeline = False
if starting:
starting = False
self.__handle_sighup()
self.process()
self.__error_retries_counter = 0 # reset counter
if self.parameters.rate_limit and self.run_mode != 'scheduled':
self.__sleep()
if self.collector_empty_process and self.run_mode != 'scheduled':
self.__sleep(1, log=False)
except exceptions.PipelineError as exc:
error_on_pipeline = True
if self.parameters.error_log_exception:
self.logger.exception('Pipeline failed.')
else:
self.logger.error(utils.error_message_from_exc(exc))
self.logger.error('Pipeline failed.')
self.__disconnect_pipelines()
except Exception as exc:
# in case of serious system issues, exit immediately
if isinstance(exc, MemoryError):
self.logger.exception('Out of memory. Exit immediately. Reason: %r.' % exc.args[0])
self.stop()
elif isinstance(exc, (IOError, OSError)) and exc.errno == 28:
self.logger.exception('Out of disk space. Exit immediately.')
self.stop()
error_on_message = sys.exc_info()
if self.parameters.error_log_exception:
self.logger.exception("Bot has found a problem.")
else:
self.logger.error(utils.error_message_from_exc(exc))
self.logger.error("Bot has found a problem.")
if self.parameters.error_log_message:
# Print full message if explicitly requested by config
self.logger.info("Current Message(event): %r.",
self.__current_message)
# In case of permanent failures, stop now
if isinstance(exc, exceptions.ConfigurationError):
self.stop()
except KeyboardInterrupt:
self.logger.info("Received KeyboardInterrupt.")
self.stop(exitcode=0)
finally:
if getattr(self.parameters, 'testing', False):
self.stop(exitcode=0)
break
if error_on_message or error_on_pipeline:
self.__message_counter["failure"] += 1
self.__error_retries_counter += 1
# reached the maximum number of retries
if (self.__error_retries_counter >
self.parameters.error_max_retries):
if error_on_message:
if self.parameters.error_dump_message:
error_traceback = traceback.format_exception(*error_on_message)
self._dump_message(error_traceback,
message=self.__current_message)
else:
warnings.warn("Message will be removed from the pipeline and not dumped to the disk. "
"Set `error_dump_message` to true to save the message on disk. "
"This warning is only shown once in the runtime of a bot.")
if self.__destination_queues and '_on_error' in self.__destination_queues:
self.send_message(self.__current_message, path='_on_error')
# remove message from pipeline
self.acknowledge_message()
# when bot acknowledge the message,
# don't need to wait again
error_on_message = False
# run_mode: scheduled
if self.run_mode == 'scheduled':
self.logger.info('Shutting down scheduled bot.')
self.stop(exitcode=0)
# error_procedure: stop
elif self.parameters.error_procedure == "stop":
self.stop()
# error_procedure: pass
elif not error_on_pipeline:
self.__error_retries_counter = 0 # reset counter
# error_procedure: pass and pipeline problem
else:
# retry forever, see https://github.com/certtools/intelmq/issues/1333
# https://lists.cert.at/pipermail/intelmq-users/2018-October/000085.html
pass
else:
self.__message_counter["success"] += 1
# no errors, check for run mode: scheduled
if self.run_mode == 'scheduled':
self.logger.info('Shutting down scheduled bot.')
self.stop(exitcode=0)
self.__stats()
self.__handle_sighup()
def start(self, starting=True, error_on_pipeline=True,
error_on_message=False, source_pipeline=None,
destination_pipeline=None):
self.__source_pipeline = source_pipeline
self.__destination_pipeline = destination_pipeline
while True:
try:
if not starting and (error_on_pipeline or error_on_message):
self.logger.info('Bot will continue in %s seconds.' %
self.parameters.error_retry_delay)
time.sleep(self.parameters.error_retry_delay)
if error_on_message:
error_on_message = False
if error_on_pipeline:
self.__connect_pipelines()
error_on_pipeline = False
if starting:
starting = False
self.__handle_sighup()
self.process()
self.__error_retries_counter = 0 # reset counter
if self.parameters.rate_limit:
self.__sleep()
except exceptions.PipelineError as exc:
error_on_pipeline = True
if self.parameters.error_log_exception:
self.logger.exception('Pipeline failed.')
else:
self.logger.error(utils.error_message_from_exc(exc))
self.logger.error('Pipeline failed.')
self.__disconnect_pipelines()
except Exception as exc:
if isinstance(exc, MemoryError):
self.logger.exception('Out of memory. Exit immediately.')
self.stop()
elif isinstance(exc, (IOError, OSError)) and exc.errno == 28:
self.logger.exception('Out of disk space. Exit immediately.')
self.stop()
error_on_message = sys.exc_info()
if self.parameters.error_log_exception:
self.logger.exception("Bot has found a problem.")
else:
self.logger.error(utils.error_message_from_exc(exc))
self.logger.error("Bot has found a problem.")
if self.parameters.error_log_message:
# Dump full message if explicitly requested by config
self.logger.info("Current Message(event): {!r}."
"".format(self.__current_message))
except KeyboardInterrupt:
self.logger.error("Received KeyboardInterrupt.")
self.stop(exitcode=0)
del self
break
finally:
if getattr(self.parameters, 'testing', False):
self.stop()
break
if error_on_message or error_on_pipeline:
self.__error_retries_counter += 1
# reached the maximum number of retries
if (self.__error_retries_counter >
self.parameters.error_max_retries):
if error_on_message:
if self.parameters.error_dump_message:
error_traceback = traceback.format_exception(*error_on_message)
self._dump_message(error_traceback,
message=self.__current_message)
self.__current_message = None
# remove message from pipeline
self.acknowledge_message()
# when bot acknowledge the message,
# dont need to wait again
error_on_message = False
# error_procedure: stop
if self.parameters.error_procedure == "stop":
self.stop()
# error_procedure: pass
else:
self.__error_retries_counter = 0 # reset counter
self.__handle_sighup()
def test_error_message_from_exc(self):
"""Tests if error_message_from_exc correctly returns the error message."""
exc = IndexError('This is a test')
self.assertEqual(utils.error_message_from_exc(exc), 'This is a test')
def check(self, no_connections=False):
retval = 0
if RETURN_TYPE == 'json':
output = []
if QUIET:
logger.setLevel('WARNING')
# loading files and syntax check
files = {DEFAULTS_CONF_FILE: None, PIPELINE_CONF_FILE: None,
RUNTIME_CONF_FILE: None, BOTS_FILE: None,
HARMONIZATION_CONF_FILE: None}
if RETURN_TYPE == 'json':
output.append(['info', 'Reading configuration files.'])
else:
self.logger.info('Reading configuration files.')
for filename in files:
try:
with open(filename) as file_handle:
files[filename] = json.load(file_handle)
except (IOError, ValueError) as exc: # pragma: no cover
if RETURN_TYPE == 'json':
output.append(['error', 'Coud not load %r: %s.' % (filename, exc)])
else:
self.logger.error('Coud not load %r: %s.', filename, exc)
retval = 1
if retval:
if RETURN_TYPE == 'json':
return 1, {'status': 'error', 'lines': output}
else:
self.logger.error('Fatal errors occurred.')
return 1, retval
if RETURN_TYPE == 'json':
output.append(['info', 'Checking defaults configuration.'])
else:
self.logger.info('Checking defaults configuration.')
try:
with open(pkg_resources.resource_filename('intelmq', 'etc/defaults.conf')) as fh:
defaults = json.load(fh)
except FileNotFoundError:
pass
else:
keys = set(defaults.keys()) - set(files[DEFAULTS_CONF_FILE].keys())
if keys:
if RETURN_TYPE == 'json':
output.append(['error', "Keys missing in your 'defaults.conf' file: %r" % keys])
else:
self.logger.error("Keys missing in your 'defaults.conf' file: %r", keys)
if RETURN_TYPE == 'json':
output.append(['info', 'Checking runtime configuration.'])
else:
self.logger.info('Checking runtime configuration.')
http_proxy = files[DEFAULTS_CONF_FILE].get('http_proxy')
https_proxy = files[DEFAULTS_CONF_FILE].get('https_proxy')
# Either both are given or both are not given
if (not http_proxy or not https_proxy) and not (http_proxy == https_proxy):
if RETURN_TYPE == 'json':
output.append(['warning', 'Incomplete configuration: Both http and https proxies must be set.'])
else:
self.logger.warning('Incomplete configuration: Both http and https proxies must be set.')
retval = 1
if RETURN_TYPE == 'json':
output.append(['info', 'Checking runtime and pipeline configuration.'])
else:
self.logger.info('Checking runtime and pipeline configuration.')
all_queues = set()
for bot_id, bot_config in files[RUNTIME_CONF_FILE].items():
# pipeline keys
for field in ['description', 'group', 'module', 'name']:
if field not in bot_config:
if RETURN_TYPE == 'json':
output.append(['warning', 'Bot %r has no %r.' % (bot_id, field)])
else:
self.logger.warning('Bot %r has no %r.', bot_id, field)
retval = 1
if 'module' in bot_config and bot_config['module'] == 'bots.collectors.n6.collector_stomp':
if RETURN_TYPE == 'json':
output.append(['warning',
"The module 'bots.collectors.n6.collector_stomp' is deprecated and will be removed in "
"version 2.0. Please use intelmq.bots.collectors."
"stomp.collector instead for bot %r." % bot_id])
else:
self.logger.warning("The module 'bots.collectors.n6.collector_stomp' is deprecated and will be removed in "
"version 2.0. Please use intelmq.bots.collectors."
"stomp.collector instead for bot %r." % bot_id)
if 'run_mode' in bot_config and bot_config['run_mode'] not in ['continuous', 'scheduled']:
message = "Bot %r has invalid `run_mode` %r. Must be 'continuous' or 'scheduled'."
if RETURN_TYPE == 'json':
output.append(['warning', message % (bot_id, bot_config['run_mode'])])
else:
self.logger.warning(message, bot_id, bot_config['run_mode'])
retval = 1
if bot_id not in files[PIPELINE_CONF_FILE]:
if RETURN_TYPE == 'json':
output.append(['error', 'Misconfiguration: No pipeline configuration found for %r.' % bot_id])
else:
self.logger.error('Misconfiguration: No pipeline configuration found for %r.', bot_id)
retval = 1
else:
if ('group' in bot_config and
bot_config['group'] in ['Collector', 'Parser', 'Expert']):
if ('destination-queues' not in files[PIPELINE_CONF_FILE][bot_id] or
(not isinstance(files[PIPELINE_CONF_FILE][bot_id]['destination-queues'], list) or
len(files[PIPELINE_CONF_FILE][bot_id]['destination-queues']) < 1)):
if RETURN_TYPE == 'json':
output.append(['error', 'Misconfiguration: No destination queues for %r.' % bot_id])
else:
self.logger.error('Misconfiguration: No destination queues for %r.', bot_id)
retval = 1
else:
all_queues = all_queues.union(files[PIPELINE_CONF_FILE][bot_id]['destination-queues'])
if ('group' in bot_config and
bot_config['group'] in ['Parser', 'Expert', 'Output']):
if ('source-queue' not in files[PIPELINE_CONF_FILE][bot_id] or
not isinstance(files[PIPELINE_CONF_FILE][bot_id]['source-queue'], str)):
if RETURN_TYPE == 'json':
output.append(['error', 'Misconfiguration: No source queue for %r.' % bot_id])
else:
self.logger.error('Misconfiguration: No source queue for %r.', bot_id)
retval = 1
else:
all_queues.add(files[PIPELINE_CONF_FILE][bot_id]['source-queue'])
all_queues.add(files[PIPELINE_CONF_FILE][bot_id]['source-queue'] + '-internal')
if not no_connections:
try:
pipeline = PipelineFactory.create(self.parameters)
pipeline.set_queues(None, "source")
pipeline.connect()
orphan_queues = "', '".join({a.decode() for a in pipeline.pipe.keys()} - all_queues)
except Exception as exc:
error = utils.error_message_from_exc(exc)
if RETURN_TYPE == 'json':
output.append(['error',
'Could not connect to redis pipeline: %s' % error])
else:
self.logger.error('Could not connect to redis pipeline: %s', error)
retval = 1
else:
if orphan_queues:
if RETURN_TYPE == 'json':
output.append(['warning', "Orphaned queues found: '%s'." % orphan_queues])
else:
self.logger.warning("Orphaned queues found: '%s'.", orphan_queues)
if RETURN_TYPE == 'json':
output.append(['info', 'Checking harmonization configuration.'])
else:
self.logger.info('Checking harmonization configuration.')
for event_type, event_type_conf in files[HARMONIZATION_CONF_FILE].items():
for harm_type_name, harm_type in event_type_conf.items():
if "description" not in harm_type:
if RETURN_TYPE == 'json':
output.append(['warn', 'Missing description for type %r.' % harm_type_name])
else:
self.logger.warn('Missing description for type %r.', harm_type_name)
if "type" not in harm_type:
if RETURN_TYPE == 'json':
output.append(['error', 'Missing type for type %r.' % harm_type_name])
else:
self.logger.error('Missing type for type %r.', harm_type_name)
retval = 1
continue
if "regex" in harm_type:
try:
re.compile(harm_type['regex'])
except Exception as e:
if RETURN_TYPE == 'json':
output.append(['error', 'Invalid regex for type %r: %r.' % (harm_type_name, str(e))])
else:
self.logger.error('Invalid regex for type %r: %r.', harm_type_name, str(e))
retval = 1
continue
extra_type = files[HARMONIZATION_CONF_FILE].get('event', {}).get('extra', {}).get('type')
if extra_type != 'JSONDict':
if RETURN_TYPE == 'json':
output.append(['warning', "'extra' field needs to be of type 'JSONDict'."])
else:
self.logger.warning("'extra' field needs to be of type 'JSONDict'.")
retval = 1
if RETURN_TYPE == 'json':
output.append(['info', 'Checking for bots.'])
else:
self.logger.info('Checking for bots.')
for bot_id, bot_config in files[RUNTIME_CONF_FILE].items():
# importable module
try:
bot_module = importlib.import_module(bot_config['module'])
except ImportError as exc:
if RETURN_TYPE == 'json':
output.append(['error', 'Incomplete installation: Bot %r not importable: %r.' % (bot_id, exc)])
else:
self.logger.error('Incomplete installation: Bot %r not importable: %r.', bot_id, exc)
retval = 1
continue
bot = getattr(bot_module, 'BOT')
bot_parameters = files[DEFAULTS_CONF_FILE].copy()
bot_parameters.update(bot_config['parameters'])
bot_check = bot.check(bot_parameters)
if bot_check:
if RETURN_TYPE == 'json':
output.extend(bot_check)
else:
for log_line in bot_check:
getattr(self.logger, log_line[0])("Bot %r: %s" % (bot_id, log_line[1]))
for group in files[BOTS_FILE].values():
for bot_id, bot in group.items():
if subprocess.call(['which', bot['module']], stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL):
if RETURN_TYPE == 'json':
output.append(['error', 'Incomplete installation: Executable %r for %r not found.' %
(bot['module'], bot_id)])
else:
self.logger.error('Incomplete installation: Executable %r for %r not found.',
bot['module'], bot_id)
retval = 1
if RETURN_TYPE == 'json':
if retval:
return 0, {'status': 'error', 'lines': output}
else:
return 1, {'status': 'success', 'lines': output}
else:
if retval:
self.logger.error('Some issues have been found, please check the above output.')
return retval, 'error'
else:
self.logger.info('No issues found.')
return retval, 'success'