Esempio n. 1
0
    def __init__(self, bot_id):
        self.__log_buffer = []
        self.parameters = Parameters()

        self.__error_retries_counter = 0
        self.__source_pipeline = None
        self.__destination_pipeline = None
        self.logger = None

        try:
            version_info = sys.version.splitlines()[0].strip()
            self.__log_buffer.append(('info',
                                      '{} initialized with id {} and version '
                                      '{} as process {}.'
                                      ''.format(self.__class__.__name__,
                                                bot_id, version_info,
                                                os.getpid())))
            self.__log_buffer.append(('debug', 'Library path: %r.' % __file__))

            self.__load_defaults_configuration()
            self.__load_system_configuration()

            self.__check_bot_id(bot_id)
            self.__bot_id = bot_id

            if self.parameters.logging_handler == 'syslog':
                syslog = self.parameters.logging_syslog
            else:
                syslog = False
            self.logger = utils.log(self.__bot_id, syslog=syslog,
                                    log_path=self.parameters.logging_path,
                                    log_level=self.parameters.logging_level)
        except:
            self.__log_buffer.append(('critical', traceback.format_exc()))
            self.stop()
        else:
            for line in self.__log_buffer:
                getattr(self.logger, line[0])(line[1])

        try:
            self.logger.info('Bot is starting.')
            self.__load_runtime_configuration()
            self.__load_pipeline_configuration()
            self.__load_harmonization_configuration()

            self.init()

            self.__sighup = False
            signal.signal(signal.SIGHUP, self.__handle_sighup_signal)
            # system calls should not be interrupted, but restarted
            signal.siginterrupt(signal.SIGHUP, False)
        except Exception as exc:
            if self.parameters.error_log_exception:
                self.logger.exception('Bot initialization failed.')
            else:
                self.logger.error(utils.error_message_from_exc(exc))
                self.logger.error('Bot initialization failed.')

            self.stop()
            raise
Esempio n. 2
0
    def __init__(self, bot_id):
        self.__log_buffer = []
        self.parameters = Parameters()

        self.__error_retries_counter = 0
        self.__source_pipeline = None
        self.__destination_pipeline = None
        self.logger = None

        try:
            version_info = sys.version.splitlines()[0].strip()
            self.__log_buffer.append(
                ('info', '{} initialized with id {} and version '
                 '{} as process {}.'
                 ''.format(self.__class__.__name__, bot_id, version_info,
                           os.getpid())))
            self.__log_buffer.append(('debug', 'Library path: %r.' % __file__))

            self.__load_defaults_configuration()
            self.__load_system_configuration()

            self.__check_bot_id(bot_id)
            self.__bot_id = bot_id

            if self.parameters.logging_handler == 'syslog':
                syslog = self.parameters.logging_syslog
            else:
                syslog = False
            self.logger = utils.log(self.__bot_id,
                                    syslog=syslog,
                                    log_path=self.parameters.logging_path,
                                    log_level=self.parameters.logging_level)
        except:
            self.__log_buffer.append(('critical', traceback.format_exc()))
            self.stop()
        else:
            for line in self.__log_buffer:
                getattr(self.logger, line[0])(line[1])

        try:
            self.logger.info('Bot is starting.')
            self.__load_runtime_configuration()
            self.__load_pipeline_configuration()
            self.__load_harmonization_configuration()

            self.init()

            self.__sighup = False
            signal.signal(signal.SIGHUP, self.__handle_sighup_signal)
            # system calls should not be interrupted, but restarted
            signal.siginterrupt(signal.SIGHUP, False)
        except Exception as exc:
            if self.parameters.error_log_exception:
                self.logger.exception('Bot initialization failed.')
            else:
                self.logger.error(utils.error_message_from_exc(exc))
                self.logger.error('Bot initialization failed.')

            self.stop()
            raise
Esempio n. 3
0
    def __init__(self, bot_id: str):
        self.__log_buffer = []
        self.parameters = Parameters()

        self.__error_retries_counter = 0
        self.__source_pipeline = None
        self.__destination_pipeline = None
        self.logger = None

        try:
            version_info = sys.version.splitlines()[0].strip()
            self.__log_buffer.append(
                ('info', '{bot} initialized with id {id} and intelmq {intelmq}'
                 ' and python {python} as process {pid}.'
                 ''.format(bot=self.__class__.__name__,
                           id=bot_id,
                           python=version_info,
                           pid=os.getpid(),
                           intelmq=__version__)))
            self.__log_buffer.append(('debug', 'Library path: %r.' % __file__))

            self.__load_defaults_configuration()

            self.__check_bot_id(bot_id)
            self.__bot_id = bot_id

            self.__init_logger()
        except Exception:
            self.__log_buffer.append(('critical', traceback.format_exc()))
            self.stop()
        else:
            for line in self.__log_buffer:
                getattr(self.logger, line[0])(line[1])

        try:
            self.logger.info('Bot is starting.')
            self.__load_runtime_configuration()
            self.__load_pipeline_configuration()
            self.__load_harmonization_configuration()

            self.init()

            self.__sighup = False
            signal.signal(signal.SIGHUP, self.__handle_sighup_signal)
            # system calls should not be interrupted, but restarted
            signal.siginterrupt(signal.SIGHUP, False)
            signal.signal(signal.SIGTERM, self.__handle_sigterm_signal)
        except Exception as exc:
            if self.parameters.error_log_exception:
                self.logger.exception('Bot initialization failed.')
            else:
                self.logger.error(utils.error_message_from_exc(exc))
                self.logger.error('Bot initialization failed.')

            self.stop()
            raise
Esempio n. 4
0
 def arg2msg(self, msg):
     try:
         default_type = "Report" if self.runtime_configuration["group"] is "Parser" else "Event"
         msg = MessageFactory.unserialize(msg, default_type=default_type)
     except (Exception, KeyError, TypeError, ValueError) as exc:
         if exists(msg):
             with open(msg, "r") as f:
                 return self.arg2msg(f.read())
         self.messageWizzard("Message can not be parsed from JSON: {}".format(error_message_from_exc(exc)))
         exit(1)
     return msg
Esempio n. 5
0
 def arg2msg(self, msg):
     try:
         default_type = "Report" if self.runtime_configuration["group"] == "Parser" else "Event"
         msg = MessageFactory.unserialize(msg, default_type=default_type)
     except (Exception, KeyError, TypeError, ValueError) as exc:
         if exists(msg):
             with open(msg, "r") as f:
                 return self.arg2msg(f.read())
         self.messageWizzard("Message can not be parsed from JSON: {}".format(error_message_from_exc(exc)))
         sys.exit(1)
     return msg
Esempio n. 6
0
 def arg2msg(self, msg):
     default_type = "Report" if (
         self.runtime_configuration.get("group", None) == "Parser"
         or isinstance(self.instance, ParserBot)) else "Event"
     try:
         msg = MessageFactory.unserialize(msg, default_type=default_type)
     except (Exception, KeyError, TypeError, ValueError) as exc:
         if exists(msg):
             with open(msg, "r") as f:
                 return self.arg2msg(f.read())
         self.messageWizzard(
             "Message can not be parsed from JSON: {}".format(
                 error_message_from_exc(exc)))
         sys.exit(1)
     return msg
Esempio n. 7
0
    def start(self,
              starting=True,
              error_on_pipeline=True,
              error_on_message=False,
              source_pipeline=None,
              destination_pipeline=None):

        self.__source_pipeline = source_pipeline
        self.__destination_pipeline = destination_pipeline
        self.logger.info('Bot starts processings.')

        while True:
            try:
                if not starting and (error_on_pipeline or error_on_message):
                    self.logger.info('Bot will restart in %s seconds.' %
                                     self.parameters.error_retry_delay)
                    time.sleep(self.parameters.error_retry_delay)
                    self.logger.info('Bot woke up')
                    self.logger.info('Trying to start processing again.')

                if error_on_message:
                    error_on_message = False

                if error_on_pipeline:
                    self.__connect_pipelines()
                    error_on_pipeline = False

                if starting:
                    self.logger.info("Start processing.")
                    starting = False

                self.__handle_sighup()
                self.process()
                self.__error_retries_counter = 0  # reset counter

                if self.parameters.rate_limit:
                    self.__sleep()

            except exceptions.PipelineError:
                error_on_pipeline = True

                if self.parameters.error_log_exception:
                    self.logger.exception('Pipeline failed.')
                else:
                    self.logger.error('Pipeline failed.')
                self.__disconnect_pipelines()

            except Exception as exc:
                error_on_message = sys.exc_info()

                if self.parameters.error_log_exception:
                    self.logger.exception("Bot has found a problem.")
                else:
                    self.logger.error(utils.error_message_from_exc(exc))
                    self.logger.error("Bot has found a problem.")

                if self.parameters.error_log_message:
                    # Dump full message if explicitly requested by config
                    self.logger.info("Current Message(event): {!r}."
                                     "".format(self.__current_message))

            except KeyboardInterrupt:
                self.logger.error("Received KeyboardInterrupt.")
                self.stop(exitcode=0)
                del self
                break

            finally:
                if getattr(self.parameters, 'testing', False):
                    self.stop()
                    break

                if error_on_message or error_on_pipeline:
                    self.__error_retries_counter += 1

                    # reached the maximum number of retries
                    if (self.__error_retries_counter >
                            self.parameters.error_max_retries):

                        if error_on_message:

                            if self.parameters.error_dump_message:
                                error_traceback = traceback.format_exception(
                                    *error_on_message)
                                self._dump_message(
                                    error_traceback,
                                    message=self.__current_message)
                                self.__current_message = None

                            # remove message from pipeline
                            self.acknowledge_message()

                            # when bot acknowledge the message,
                            # dont need to wait again
                            error_on_message = False

                        # error_procedure: stop
                        if self.parameters.error_procedure == "stop":
                            self.stop()

                        # error_procedure: pass
                        else:
                            self.__error_retries_counter = 0  # reset counter
            self.__handle_sighup()
Esempio n. 8
0
    def start(self,
              starting: bool = True,
              error_on_pipeline: bool = True,
              error_on_message: bool = False,
              source_pipeline: Optional[str] = None,
              destination_pipeline: Optional[str] = None):

        self.__source_pipeline = source_pipeline
        self.__destination_pipeline = destination_pipeline

        while True:
            try:
                if not starting and (error_on_pipeline or error_on_message):
                    self.logger.info('Bot will continue in %s seconds.',
                                     self.parameters.error_retry_delay)
                    time.sleep(self.parameters.error_retry_delay)

                if error_on_message:
                    error_on_message = False

                if error_on_pipeline:
                    self.__connect_pipelines()
                    error_on_pipeline = False

                if starting:
                    starting = False

                self.__handle_sighup()
                self.process()
                self.__error_retries_counter = 0  # reset counter

                if self.parameters.rate_limit and self.run_mode != 'scheduled':
                    self.__sleep()

            except exceptions.PipelineError as exc:
                error_on_pipeline = True

                if self.parameters.error_log_exception:
                    self.logger.exception('Pipeline failed.')
                else:
                    self.logger.error(utils.error_message_from_exc(exc))
                    self.logger.error('Pipeline failed.')
                self.__disconnect_pipelines()

            except Exception as exc:
                # in case of serious system issues, exit immediately
                if isinstance(exc, MemoryError):
                    self.logger.exception('Out of memory. Exit immediately.')
                    self.stop()
                elif isinstance(exc, (IOError, OSError)) and exc.errno == 28:
                    self.logger.exception(
                        'Out of disk space. Exit immediately.')
                    self.stop()

                error_on_message = sys.exc_info()

                if self.parameters.error_log_exception:
                    self.logger.exception("Bot has found a problem.")
                else:
                    self.logger.error(utils.error_message_from_exc(exc))
                    self.logger.error("Bot has found a problem.")

                if self.parameters.error_log_message:
                    # Dump full message if explicitly requested by config
                    self.logger.info("Current Message(event): %r.",
                                     self.__current_message)

                # In case of permanent failures, stop now
                if isinstance(exc, exceptions.ConfigurationError):
                    self.stop()

            except KeyboardInterrupt:
                self.logger.info("Received KeyboardInterrupt.")
                self.stop(exitcode=0)
                del self
                break

            finally:
                if getattr(self.parameters, 'testing', False):
                    self.stop()
                    break

                if error_on_message or error_on_pipeline:
                    self.__error_retries_counter += 1

                    # reached the maximum number of retries
                    if (self.__error_retries_counter >
                            self.parameters.error_max_retries):

                        if error_on_message:

                            if self.parameters.error_dump_message:
                                error_traceback = traceback.format_exception(
                                    *error_on_message)
                                self._dump_message(
                                    error_traceback,
                                    message=self.__current_message)
                                self.__current_message = None

                            # remove message from pipeline
                            self.acknowledge_message()

                            # when bot acknowledge the message,
                            # don't need to wait again
                            error_on_message = False

                        # run_mode: scheduled
                        if self.run_mode == 'scheduled':
                            self.logger.info('Shutting down scheduled bot.')
                            self.stop()

                        # error_procedure: stop
                        elif self.parameters.error_procedure == "stop":
                            self.stop()

                        # error_procedure: pass
                        else:
                            self.__error_retries_counter = 0  # reset counter

                # no errors, check for run mode: scheduled
                elif self.run_mode == 'scheduled':
                    self.logger.info('Shutting down scheduled bot.')
                    self.stop()

            self.__handle_sighup()
Esempio n. 9
0
    def __init__(self, bot_id: str, start=False, sighup_event=None,
                 disable_multithreading=None):
        self.__log_buffer = []
        self.parameters = Parameters()

        self.__error_retries_counter = 0
        self.__source_pipeline = None
        self.__destination_pipeline = None
        self.logger = None

        self.__message_counter = {"since": 0,  # messages since last logging
                                  "start": None,  # last login time
                                  "success": 0,  # total number since the beginning
                                  "failure": 0,  # total number since the beginning
                                  "stats_timestamp": datetime.now(),  # stamp of last report to redis
                                  "path": defaultdict(int),  # number of messages sent to queues since last report to redis
                                  "path_total": defaultdict(int)  # number of messages sent to queues since beginning
                                  }

        try:
            version_info = sys.version.splitlines()[0].strip()
            self.__log_buffer.append(('info',
                                      '{bot} initialized with id {id} and intelmq {intelmq}'
                                      ' and python {python} as process {pid}.'
                                      ''.format(bot=self.__class__.__name__,
                                                id=bot_id, python=version_info,
                                                pid=os.getpid(), intelmq=__version__)))
            self.__log_buffer.append(('debug', 'Library path: %r.' % __file__))
            if not utils.drop_privileges():
                raise ValueError('IntelMQ must not run as root. Dropping privileges did not work.')

            self.__load_defaults_configuration()

            self.__bot_id_full, self.__bot_id, self.__instance_id = self.__check_bot_id(bot_id)
            if self.__instance_id:
                self.is_multithreaded = True
            self.__init_logger()
        except Exception:
            self.__log_buffer.append(('critical', traceback.format_exc()))
            self.stop()
        else:
            for line in self.__log_buffer:
                getattr(self.logger, line[0])(line[1])

        try:
            self.logger.info('Bot is starting.')
            self.__load_runtime_configuration()

            broker = getattr(self.parameters, "source_pipeline_broker",
                             getattr(self.parameters, "broker", "redis")).title()
            if broker != 'Amqp':
                self.is_multithreadable = False

            """ Multithreading """
            if (getattr(self.parameters, 'instances_threads', 0) > 1 and
                not self.is_multithreaded and
                    self.is_multithreadable and
                    not disable_multithreading):
                self.logger.handlers = []
                num_instances = int(self.parameters.instances_threads)
                instances = []
                sighup_events = []

                def handle_sighup_signal_threading(signum: int,
                                                   stack: Optional[object]):
                    for event in sighup_events:
                        event.set()
                signal.signal(signal.SIGHUP, handle_sighup_signal_threading)

                for i in range(num_instances):
                    sighup_events.append(threading.Event())
                    threadname = '%s.%d' % (bot_id, i)
                    instances.append(threading.Thread(target=self.__class__,
                                                      kwargs={'bot_id': threadname,
                                                              'start': True,
                                                              'sighup_event': sighup_events[-1]},
                                                      name=threadname,
                                                      daemon=False))
                    instances[i].start()
                for i, thread in enumerate(instances):
                    thread.join()
                return
            elif (getattr(self.parameters, 'instances_threads', 1) > 1 and
                  not self.is_multithreadable):
                self.logger.error('Multithreading is configured, but is not '
                                  'available for this bot. Look at the FAQ '
                                  'for a list of reasons for this. '
                                  'https://github.com/certtools/intelmq/blob/master/docs/FAQ.md')
            elif disable_multithreading:
                self.logger.warning('Multithreading is configured, but is not '
                                    'available for interactive runs.')

            self.__load_pipeline_configuration()
            self.__load_harmonization_configuration()

            self.init()

            if not self.__instance_id:
                self.__sighup = threading.Event()
                signal.signal(signal.SIGHUP, self.__handle_sighup_signal)
                # system calls should not be interrupted, but restarted
                signal.siginterrupt(signal.SIGHUP, False)
                signal.signal(signal.SIGTERM, self.__handle_sigterm_signal)
                signal.signal(signal.SIGINT, self.__handle_sigterm_signal)
            else:
                self.__sighup = sighup_event

                @atexit.register
                def catch_shutdown():
                    self.stop()
        except Exception as exc:
            if self.parameters.error_log_exception:
                self.logger.exception('Bot initialization failed.')
            else:
                self.logger.error(utils.error_message_from_exc(exc))
                self.logger.error('Bot initialization failed.')

            self.stop()
            raise

        self.__stats_cache = cache.Cache(host=getattr(self.parameters,
                                                      "statistics_host",
                                                      "127.0.0.1"),
                                         port=getattr(self.parameters,
                                                      "statistics_port", "6379"),
                                         db=int(getattr(self.parameters,
                                                        "statistics_database", 3)),
                                         password=getattr(self.parameters,
                                                          "statistics_password",
                                                          None),
                                         ttl=None,
                                         )
        if start:
            self.start()
Esempio n. 10
0
    def start(self, starting: bool = True, error_on_pipeline: bool = True,
              error_on_message: bool = False, source_pipeline: Optional[str] = None,
              destination_pipeline: Optional[str] = None):

        self.__source_pipeline = source_pipeline
        self.__destination_pipeline = destination_pipeline

        while True:
            try:
                if not starting and (error_on_pipeline or error_on_message):
                    self.logger.info('Bot will continue in %s seconds.',
                                     self.parameters.error_retry_delay)
                    time.sleep(self.parameters.error_retry_delay)

                if error_on_message:
                    error_on_message = False

                if error_on_pipeline:
                    try:
                        self.__connect_pipelines()
                    except Exception as exc:
                        raise exceptions.PipelineError(exc)
                    else:
                        error_on_pipeline = False

                if starting:
                    starting = False

                self.__handle_sighup()
                self.process()
                self.__error_retries_counter = 0  # reset counter

                if self.parameters.rate_limit and self.run_mode != 'scheduled':
                    self.__sleep()
                if self.collector_empty_process and self.run_mode != 'scheduled':
                    self.__sleep(1, log=False)

            except exceptions.PipelineError as exc:
                error_on_pipeline = True

                if self.parameters.error_log_exception:
                    self.logger.exception('Pipeline failed.')
                else:
                    self.logger.error(utils.error_message_from_exc(exc))
                    self.logger.error('Pipeline failed.')
                self.__disconnect_pipelines()

            except Exception as exc:
                # in case of serious system issues, exit immediately
                if isinstance(exc, MemoryError):
                    self.logger.exception('Out of memory. Exit immediately. Reason: %r.' % exc.args[0])
                    self.stop()
                elif isinstance(exc, (IOError, OSError)) and exc.errno == 28:
                    self.logger.exception('Out of disk space. Exit immediately.')
                    self.stop()

                error_on_message = sys.exc_info()

                if self.parameters.error_log_exception:
                    self.logger.exception("Bot has found a problem.")
                else:
                    self.logger.error(utils.error_message_from_exc(exc))
                    self.logger.error("Bot has found a problem.")

                if self.parameters.error_log_message:
                    # Print full message if explicitly requested by config
                    self.logger.info("Current Message(event): %r.",
                                     self.__current_message)

                # In case of permanent failures, stop now
                if isinstance(exc, exceptions.ConfigurationError):
                    self.stop()

            except KeyboardInterrupt:
                self.logger.info("Received KeyboardInterrupt.")
                self.stop(exitcode=0)

            finally:
                if getattr(self.parameters, 'testing', False):
                    self.stop(exitcode=0)
                    break

                if error_on_message or error_on_pipeline:
                    self.__message_counter["failure"] += 1
                    self.__error_retries_counter += 1

                    # reached the maximum number of retries
                    if (self.__error_retries_counter >
                            self.parameters.error_max_retries):

                        if error_on_message:

                            if self.parameters.error_dump_message:
                                error_traceback = traceback.format_exception(*error_on_message)
                                self._dump_message(error_traceback,
                                                   message=self.__current_message)
                            else:
                                warnings.warn("Message will be removed from the pipeline and not dumped to the disk. "
                                              "Set `error_dump_message` to true to save the message on disk. "
                                              "This warning is only shown once in the runtime of a bot.")
                            if self.__destination_queues and '_on_error' in self.__destination_queues:
                                self.send_message(self.__current_message, path='_on_error')

                            # remove message from pipeline
                            self.acknowledge_message()

                            # when bot acknowledge the message,
                            # don't need to wait again
                            error_on_message = False

                        # run_mode: scheduled
                        if self.run_mode == 'scheduled':
                            self.logger.info('Shutting down scheduled bot.')
                            self.stop(exitcode=0)

                        # error_procedure: stop
                        elif self.parameters.error_procedure == "stop":
                            self.stop()

                        # error_procedure: pass
                        elif not error_on_pipeline:
                            self.__error_retries_counter = 0  # reset counter
                        # error_procedure: pass and pipeline problem
                        else:
                            # retry forever, see https://github.com/certtools/intelmq/issues/1333
                            # https://lists.cert.at/pipermail/intelmq-users/2018-October/000085.html
                            pass
                else:
                    self.__message_counter["success"] += 1
                    # no errors, check for run mode: scheduled
                    if self.run_mode == 'scheduled':
                        self.logger.info('Shutting down scheduled bot.')
                        self.stop(exitcode=0)

            self.__stats()
            self.__handle_sighup()
Esempio n. 11
0
 def test_error_message_from_exc(self):
     """Tests if error_message_from_exc correctly returns the error message."""
     exc = IndexError('This is a test')
     self.assertEqual(utils.error_message_from_exc(exc), 'This is a test')
Esempio n. 12
0
    def __init__(self, bot_id: str, start=False, sighup_event=None,
                 disable_multithreading=None):
        self.__log_buffer = []
        self.parameters = Parameters()

        self.__error_retries_counter = 0
        self.__source_pipeline = None
        self.__destination_pipeline = None
        self.logger = None

        self.__message_counter = {"since": 0,  # messages since last logging
                                  "start": None,  # last login time
                                  "success": 0,  # total number since the beginning
                                  "failure": 0,  # total number since the beginning
                                  "stats_timestamp": datetime.now(),  # stamp of last report to redis
                                  "path": defaultdict(int),  # number of messages sent to queues since last report to redis
                                  "path_total": defaultdict(int)  # number of messages sent to queues since beginning
                                  }

        try:
            version_info = sys.version.splitlines()[0].strip()
            self.__log_buffer.append(('info',
                                      '{bot} initialized with id {id} and intelmq {intelmq}'
                                      ' and python {python} as process {pid}.'
                                      ''.format(bot=self.__class__.__name__,
                                                id=bot_id, python=version_info,
                                                pid=os.getpid(), intelmq=__version__)))
            self.__log_buffer.append(('debug', 'Library path: %r.' % __file__))
            if not utils.drop_privileges():
                raise ValueError('IntelMQ must not run as root. Dropping privileges did not work.')

            self.__load_defaults_configuration()

            self.__bot_id_full, self.__bot_id, self.__instance_id = self.__check_bot_id(bot_id)
            if self.__instance_id:
                self.is_multithreaded = True
            self.__init_logger()
        except Exception:
            self.__log_buffer.append(('critical', traceback.format_exc()))
            self.stop()
        else:
            for line in self.__log_buffer:
                getattr(self.logger, line[0])(line[1])

        try:
            self.logger.info('Bot is starting.')
            self.__load_runtime_configuration()

            broker = getattr(self.parameters, "source_pipeline_broker",
                             getattr(self.parameters, "broker", "redis")).title()
            if broker != 'Amqp':
                self.is_multithreadable = False

            """ Multithreading """
            if (getattr(self.parameters, 'instances_threads', 0) > 1 and
                not self.is_multithreaded and
                    self.is_multithreadable and
                    not disable_multithreading):
                self.logger.handlers = []
                num_instances = int(self.parameters.instances_threads)
                instances = []
                sighup_events = []

                def handle_sighup_signal_threading(signum: int,
                                                   stack: Optional[object]):
                    for event in sighup_events:
                        event.set()
                signal.signal(signal.SIGHUP, handle_sighup_signal_threading)

                for i in range(num_instances):
                    sighup_events.append(threading.Event())
                    threadname = '%s.%d' % (bot_id, i)
                    instances.append(threading.Thread(target=self.__class__,
                                                      kwargs={'bot_id': threadname,
                                                              'start': True,
                                                              'sighup_event': sighup_events[-1]},
                                                      name=threadname,
                                                      daemon=False))
                    instances[i].start()
                for i, thread in enumerate(instances):
                    thread.join()
                return
            elif (getattr(self.parameters, 'instances_threads', 1) > 1 and
                  not self.is_multithreadable):
                self.logger.error('Multithreading is configured, but is not '
                                  'available for this bot. Look at the FAQ '
                                  'for a list of reasons for this. '
                                  'https://github.com/certtools/intelmq/blob/master/docs/FAQ.md')
            elif disable_multithreading:
                self.logger.warning('Multithreading is configured, but is not '
                                    'available for interactive runs.')

            self.__load_pipeline_configuration()
            self.__load_harmonization_configuration()

            self.init()

            if not self.__instance_id:
                self.__sighup = threading.Event()
                signal.signal(signal.SIGHUP, self.__handle_sighup_signal)
                # system calls should not be interrupted, but restarted
                signal.siginterrupt(signal.SIGHUP, False)
                signal.signal(signal.SIGTERM, self.__handle_sigterm_signal)
                signal.signal(signal.SIGINT, self.__handle_sigterm_signal)
            else:
                self.__sighup = sighup_event

                @atexit.register
                def catch_shutdown():
                    self.stop()
        except Exception as exc:
            if self.parameters.error_log_exception:
                self.logger.exception('Bot initialization failed.')
            else:
                self.logger.error(utils.error_message_from_exc(exc))
                self.logger.error('Bot initialization failed.')

            self.stop()
            raise

        self.__stats_cache = cache.Cache(host=getattr(self.parameters,
                                                      "statistics_host",
                                                      "127.0.0.1"),
                                         port=getattr(self.parameters,
                                                      "statistics_port", "6379"),
                                         db=int(getattr(self.parameters,
                                                        "statistics_database", 3)),
                                         password=getattr(self.parameters,
                                                          "statistics_password",
                                                          None),
                                         ttl=None,
                                         )
        if start:
            self.start()
Esempio n. 13
0
    def start(self, starting: bool = True, error_on_pipeline: bool = True,
              error_on_message: bool = False, source_pipeline: Optional[str] = None,
              destination_pipeline: Optional[str] = None):

        self.__source_pipeline = source_pipeline
        self.__destination_pipeline = destination_pipeline

        while True:
            try:
                if not starting and (error_on_pipeline or error_on_message):
                    self.logger.info('Bot will continue in %s seconds.',
                                     self.parameters.error_retry_delay)
                    time.sleep(self.parameters.error_retry_delay)

                if error_on_message:
                    error_on_message = False

                if error_on_pipeline:
                    try:
                        self.__connect_pipelines()
                    except Exception as exc:
                        raise exceptions.PipelineError(exc)
                    else:
                        error_on_pipeline = False

                if starting:
                    starting = False

                self.__handle_sighup()
                self.process()
                self.__error_retries_counter = 0  # reset counter

                if self.parameters.rate_limit and self.run_mode != 'scheduled':
                    self.__sleep()
                if self.collector_empty_process and self.run_mode != 'scheduled':
                    self.__sleep(1, log=False)

            except exceptions.PipelineError as exc:
                error_on_pipeline = True

                if self.parameters.error_log_exception:
                    self.logger.exception('Pipeline failed.')
                else:
                    self.logger.error(utils.error_message_from_exc(exc))
                    self.logger.error('Pipeline failed.')
                self.__disconnect_pipelines()

            except Exception as exc:
                # in case of serious system issues, exit immediately
                if isinstance(exc, MemoryError):
                    self.logger.exception('Out of memory. Exit immediately. Reason: %r.' % exc.args[0])
                    self.stop()
                elif isinstance(exc, (IOError, OSError)) and exc.errno == 28:
                    self.logger.exception('Out of disk space. Exit immediately.')
                    self.stop()

                error_on_message = sys.exc_info()

                if self.parameters.error_log_exception:
                    self.logger.exception("Bot has found a problem.")
                else:
                    self.logger.error(utils.error_message_from_exc(exc))
                    self.logger.error("Bot has found a problem.")

                if self.parameters.error_log_message:
                    # Print full message if explicitly requested by config
                    self.logger.info("Current Message(event): %r.",
                                     self.__current_message)

                # In case of permanent failures, stop now
                if isinstance(exc, exceptions.ConfigurationError):
                    self.stop()

            except KeyboardInterrupt:
                self.logger.info("Received KeyboardInterrupt.")
                self.stop(exitcode=0)

            finally:
                if getattr(self.parameters, 'testing', False):
                    self.stop(exitcode=0)
                    break

                if error_on_message or error_on_pipeline:
                    self.__message_counter["failure"] += 1
                    self.__error_retries_counter += 1

                    # reached the maximum number of retries
                    if (self.__error_retries_counter >
                            self.parameters.error_max_retries):

                        if error_on_message:

                            if self.parameters.error_dump_message:
                                error_traceback = traceback.format_exception(*error_on_message)
                                self._dump_message(error_traceback,
                                                   message=self.__current_message)
                            else:
                                warnings.warn("Message will be removed from the pipeline and not dumped to the disk. "
                                              "Set `error_dump_message` to true to save the message on disk. "
                                              "This warning is only shown once in the runtime of a bot.")
                            if self.__destination_queues and '_on_error' in self.__destination_queues:
                                self.send_message(self.__current_message, path='_on_error')

                            # remove message from pipeline
                            self.acknowledge_message()

                            # when bot acknowledge the message,
                            # don't need to wait again
                            error_on_message = False

                        # run_mode: scheduled
                        if self.run_mode == 'scheduled':
                            self.logger.info('Shutting down scheduled bot.')
                            self.stop(exitcode=0)

                        # error_procedure: stop
                        elif self.parameters.error_procedure == "stop":
                            self.stop()

                        # error_procedure: pass
                        elif not error_on_pipeline:
                            self.__error_retries_counter = 0  # reset counter
                        # error_procedure: pass and pipeline problem
                        else:
                            # retry forever, see https://github.com/certtools/intelmq/issues/1333
                            # https://lists.cert.at/pipermail/intelmq-users/2018-October/000085.html
                            pass
                else:
                    self.__message_counter["success"] += 1
                    # no errors, check for run mode: scheduled
                    if self.run_mode == 'scheduled':
                        self.logger.info('Shutting down scheduled bot.')
                        self.stop(exitcode=0)

            self.__stats()
            self.__handle_sighup()
Esempio n. 14
0
    def start(self, starting=True, error_on_pipeline=True,
              error_on_message=False, source_pipeline=None,
              destination_pipeline=None):

        self.__source_pipeline = source_pipeline
        self.__destination_pipeline = destination_pipeline

        while True:
            try:
                if not starting and (error_on_pipeline or error_on_message):
                    self.logger.info('Bot will continue in %s seconds.' %
                                     self.parameters.error_retry_delay)
                    time.sleep(self.parameters.error_retry_delay)

                if error_on_message:
                    error_on_message = False

                if error_on_pipeline:
                    self.__connect_pipelines()
                    error_on_pipeline = False

                if starting:
                    starting = False

                self.__handle_sighup()
                self.process()
                self.__error_retries_counter = 0  # reset counter

                if self.parameters.rate_limit:
                    self.__sleep()

            except exceptions.PipelineError as exc:
                error_on_pipeline = True

                if self.parameters.error_log_exception:
                    self.logger.exception('Pipeline failed.')
                else:
                    self.logger.error(utils.error_message_from_exc(exc))
                    self.logger.error('Pipeline failed.')
                self.__disconnect_pipelines()

            except Exception as exc:
                if isinstance(exc, MemoryError):
                    self.logger.exception('Out of memory. Exit immediately.')
                    self.stop()
                elif isinstance(exc, (IOError, OSError)) and exc.errno == 28:
                    self.logger.exception('Out of disk space. Exit immediately.')
                    self.stop()

                error_on_message = sys.exc_info()

                if self.parameters.error_log_exception:
                    self.logger.exception("Bot has found a problem.")
                else:
                    self.logger.error(utils.error_message_from_exc(exc))
                    self.logger.error("Bot has found a problem.")

                if self.parameters.error_log_message:
                    # Dump full message if explicitly requested by config
                    self.logger.info("Current Message(event): {!r}."
                                     "".format(self.__current_message))

            except KeyboardInterrupt:
                self.logger.error("Received KeyboardInterrupt.")
                self.stop(exitcode=0)
                del self
                break

            finally:
                if getattr(self.parameters, 'testing', False):
                    self.stop()
                    break

                if error_on_message or error_on_pipeline:
                    self.__error_retries_counter += 1

                    # reached the maximum number of retries
                    if (self.__error_retries_counter >
                            self.parameters.error_max_retries):

                        if error_on_message:

                            if self.parameters.error_dump_message:
                                error_traceback = traceback.format_exception(*error_on_message)
                                self._dump_message(error_traceback,
                                                   message=self.__current_message)
                                self.__current_message = None

                            # remove message from pipeline
                            self.acknowledge_message()

                            # when bot acknowledge the message,
                            # dont need to wait again
                            error_on_message = False

                        # error_procedure: stop
                        if self.parameters.error_procedure == "stop":
                            self.stop()

                        # error_procedure: pass
                        else:
                            self.__error_retries_counter = 0  # reset counter
            self.__handle_sighup()
Esempio n. 15
0
 def test_error_message_from_exc(self):
     """Tests if error_message_from_exc correctly returns the error message."""
     exc = IndexError('This is a test')
     self.assertEqual(utils.error_message_from_exc(exc), 'This is a test')
Esempio n. 16
0
    def check(self, no_connections=False):
        retval = 0
        if RETURN_TYPE == 'json':
            output = []
        if QUIET:
            logger.setLevel('WARNING')

        # loading files and syntax check
        files = {DEFAULTS_CONF_FILE: None, PIPELINE_CONF_FILE: None,
                 RUNTIME_CONF_FILE: None, BOTS_FILE: None,
                 HARMONIZATION_CONF_FILE: None}
        if RETURN_TYPE == 'json':
            output.append(['info', 'Reading configuration files.'])
        else:
            self.logger.info('Reading configuration files.')
        for filename in files:
            try:
                with open(filename) as file_handle:
                    files[filename] = json.load(file_handle)
            except (IOError, ValueError) as exc:  # pragma: no cover
                if RETURN_TYPE == 'json':
                    output.append(['error', 'Coud not load %r: %s.' % (filename, exc)])
                else:
                    self.logger.error('Coud not load %r: %s.', filename, exc)
                retval = 1
        if retval:
            if RETURN_TYPE == 'json':
                return 1, {'status': 'error', 'lines': output}
            else:
                self.logger.error('Fatal errors occurred.')
                return 1, retval

        if RETURN_TYPE == 'json':
            output.append(['info', 'Checking defaults configuration.'])
        else:
            self.logger.info('Checking defaults configuration.')
        try:
            with open(pkg_resources.resource_filename('intelmq', 'etc/defaults.conf')) as fh:
                defaults = json.load(fh)
        except FileNotFoundError:
            pass
        else:
            keys = set(defaults.keys()) - set(files[DEFAULTS_CONF_FILE].keys())
            if keys:
                if RETURN_TYPE == 'json':
                    output.append(['error', "Keys missing in your 'defaults.conf' file: %r" % keys])
                else:
                    self.logger.error("Keys missing in your 'defaults.conf' file: %r", keys)

        if RETURN_TYPE == 'json':
            output.append(['info', 'Checking runtime configuration.'])
        else:
            self.logger.info('Checking runtime configuration.')
        http_proxy = files[DEFAULTS_CONF_FILE].get('http_proxy')
        https_proxy = files[DEFAULTS_CONF_FILE].get('https_proxy')
        # Either both are given or both are not given
        if (not http_proxy or not https_proxy) and not (http_proxy == https_proxy):
            if RETURN_TYPE == 'json':
                output.append(['warning', 'Incomplete configuration: Both http and https proxies must be set.'])
            else:
                self.logger.warning('Incomplete configuration: Both http and https proxies must be set.')
            retval = 1

        if RETURN_TYPE == 'json':
            output.append(['info', 'Checking runtime and pipeline configuration.'])
        else:
            self.logger.info('Checking runtime and pipeline configuration.')
        all_queues = set()
        for bot_id, bot_config in files[RUNTIME_CONF_FILE].items():
            # pipeline keys
            for field in ['description', 'group', 'module', 'name']:
                if field not in bot_config:
                    if RETURN_TYPE == 'json':
                        output.append(['warning', 'Bot %r has no %r.' % (bot_id, field)])
                    else:
                        self.logger.warning('Bot %r has no %r.', bot_id, field)
                    retval = 1
            if 'module' in bot_config and bot_config['module'] == 'bots.collectors.n6.collector_stomp':
                if RETURN_TYPE == 'json':
                    output.append(['warning',
                                   "The module 'bots.collectors.n6.collector_stomp' is deprecated and will be removed in "
                                   "version 2.0. Please use intelmq.bots.collectors."
                                   "stomp.collector instead for bot %r." % bot_id])
                else:
                    self.logger.warning("The module 'bots.collectors.n6.collector_stomp' is deprecated and will be removed in "
                                        "version 2.0. Please use intelmq.bots.collectors."
                                        "stomp.collector instead for bot %r." % bot_id)
            if 'run_mode' in bot_config and bot_config['run_mode'] not in ['continuous', 'scheduled']:
                message = "Bot %r has invalid `run_mode` %r. Must be 'continuous' or 'scheduled'."
                if RETURN_TYPE == 'json':
                    output.append(['warning', message % (bot_id, bot_config['run_mode'])])
                else:
                    self.logger.warning(message, bot_id, bot_config['run_mode'])
                    retval = 1
            if bot_id not in files[PIPELINE_CONF_FILE]:
                if RETURN_TYPE == 'json':
                    output.append(['error', 'Misconfiguration: No pipeline configuration found for %r.' % bot_id])
                else:
                    self.logger.error('Misconfiguration: No pipeline configuration found for %r.', bot_id)
                retval = 1
            else:
                if ('group' in bot_config and
                        bot_config['group'] in ['Collector', 'Parser', 'Expert']):
                    if ('destination-queues' not in files[PIPELINE_CONF_FILE][bot_id] or
                            (not isinstance(files[PIPELINE_CONF_FILE][bot_id]['destination-queues'], list) or
                             len(files[PIPELINE_CONF_FILE][bot_id]['destination-queues']) < 1)):
                        if RETURN_TYPE == 'json':
                            output.append(['error', 'Misconfiguration: No destination queues for %r.' % bot_id])
                        else:
                            self.logger.error('Misconfiguration: No destination queues for %r.', bot_id)
                        retval = 1
                    else:
                        all_queues = all_queues.union(files[PIPELINE_CONF_FILE][bot_id]['destination-queues'])
                if ('group' in bot_config and
                        bot_config['group'] in ['Parser', 'Expert', 'Output']):
                    if ('source-queue' not in files[PIPELINE_CONF_FILE][bot_id] or
                            not isinstance(files[PIPELINE_CONF_FILE][bot_id]['source-queue'], str)):
                        if RETURN_TYPE == 'json':
                            output.append(['error', 'Misconfiguration: No source queue for %r.' % bot_id])
                        else:
                            self.logger.error('Misconfiguration: No source queue for %r.', bot_id)
                        retval = 1
                    else:
                        all_queues.add(files[PIPELINE_CONF_FILE][bot_id]['source-queue'])
                        all_queues.add(files[PIPELINE_CONF_FILE][bot_id]['source-queue'] + '-internal')
        if not no_connections:
            try:
                pipeline = PipelineFactory.create(self.parameters)
                pipeline.set_queues(None, "source")
                pipeline.connect()
                orphan_queues = "', '".join({a.decode() for a in pipeline.pipe.keys()} - all_queues)
            except Exception as exc:
                error = utils.error_message_from_exc(exc)
                if RETURN_TYPE == 'json':
                    output.append(['error',
                                   'Could not connect to redis pipeline: %s' % error])
                else:
                    self.logger.error('Could not connect to redis pipeline: %s', error)
                retval = 1
            else:
                if orphan_queues:
                    if RETURN_TYPE == 'json':
                        output.append(['warning', "Orphaned queues found: '%s'." % orphan_queues])
                    else:
                        self.logger.warning("Orphaned queues found: '%s'.", orphan_queues)

        if RETURN_TYPE == 'json':
            output.append(['info', 'Checking harmonization configuration.'])
        else:
            self.logger.info('Checking harmonization configuration.')
        for event_type, event_type_conf in files[HARMONIZATION_CONF_FILE].items():
            for harm_type_name, harm_type in event_type_conf.items():
                if "description" not in harm_type:
                    if RETURN_TYPE == 'json':
                        output.append(['warn', 'Missing description for type %r.' % harm_type_name])
                    else:
                        self.logger.warn('Missing description for type %r.', harm_type_name)
                if "type" not in harm_type:
                    if RETURN_TYPE == 'json':
                        output.append(['error', 'Missing type for type %r.' % harm_type_name])
                    else:
                        self.logger.error('Missing type for type %r.', harm_type_name)
                    retval = 1
                    continue
                if "regex" in harm_type:
                    try:
                        re.compile(harm_type['regex'])
                    except Exception as e:
                        if RETURN_TYPE == 'json':
                            output.append(['error', 'Invalid regex for type %r: %r.' % (harm_type_name, str(e))])
                        else:
                            self.logger.error('Invalid regex for type %r: %r.', harm_type_name, str(e))
                        retval = 1
                        continue
        extra_type = files[HARMONIZATION_CONF_FILE].get('event', {}).get('extra', {}).get('type')
        if extra_type != 'JSONDict':
            if RETURN_TYPE == 'json':
                output.append(['warning', "'extra' field needs to be of type 'JSONDict'."])
            else:
                self.logger.warning("'extra' field needs to be of type 'JSONDict'.")
            retval = 1

        if RETURN_TYPE == 'json':
            output.append(['info', 'Checking for bots.'])
        else:
            self.logger.info('Checking for bots.')
        for bot_id, bot_config in files[RUNTIME_CONF_FILE].items():
            # importable module
            try:
                bot_module = importlib.import_module(bot_config['module'])
            except ImportError as exc:
                if RETURN_TYPE == 'json':
                    output.append(['error', 'Incomplete installation: Bot %r not importable: %r.' % (bot_id, exc)])
                else:
                    self.logger.error('Incomplete installation: Bot %r not importable: %r.', bot_id, exc)
                retval = 1
                continue
            bot = getattr(bot_module, 'BOT')
            bot_parameters = files[DEFAULTS_CONF_FILE].copy()
            bot_parameters.update(bot_config['parameters'])
            bot_check = bot.check(bot_parameters)
            if bot_check:
                if RETURN_TYPE == 'json':
                    output.extend(bot_check)
                else:
                    for log_line in bot_check:
                        getattr(self.logger, log_line[0])("Bot %r: %s" % (bot_id, log_line[1]))
        for group in files[BOTS_FILE].values():
            for bot_id, bot in group.items():
                if subprocess.call(['which', bot['module']], stdout=subprocess.DEVNULL,
                                   stderr=subprocess.DEVNULL):
                    if RETURN_TYPE == 'json':
                        output.append(['error', 'Incomplete installation: Executable %r for %r not found.' %
                                       (bot['module'], bot_id)])
                    else:
                        self.logger.error('Incomplete installation: Executable %r for %r not found.',
                                          bot['module'], bot_id)
                    retval = 1

        if RETURN_TYPE == 'json':
            if retval:
                return 0, {'status': 'error', 'lines': output}
            else:
                return 1, {'status': 'success', 'lines': output}
        else:
            if retval:
                self.logger.error('Some issues have been found, please check the above output.')
                return retval, 'error'
            else:
                self.logger.info('No issues found.')
                return retval, 'success'