def test_no_sensors_dont_quit(self):
     process_container = ProcessSensorContainer(None, poll_interval=0.1)
     process_container_thread = eventlet.spawn(process_container.run)
     eventlet.sleep(0.5)
     self.assertEqual(process_container.running(), 0)
     self.assertEqual(process_container.stopped, False)
     process_container.shutdown()
     process_container_thread.kill()
 def test_no_sensors_dont_quit(self):
     process_container = ProcessSensorContainer(None, poll_interval=0.1)
     process_container_thread = eventlet.spawn(process_container.run)
     eventlet.sleep(0.5)
     self.assertEqual(process_container.running(), 0)
     self.assertEqual(process_container.stopped(), False)
     process_container.shutdown()
     process_container_thread.kill()
Exemple #3
0
    def run_sensors(self, sensors):
        """
        :param sensors: A list of DB models of sensors to run.
        :type sensors: ``list``
        """
        LOG.info('Setting up container to run %d sensors.', len(sensors))

        sensors_to_run = []
        for sensor in sensors:
            # TODO: Directly pass DB object to the ProcessContainer
            file_path = sensor.artifact_uri.replace('file://', '')
            class_name = sensor.entry_point.split('.')[-1]

            sensor_obj = {
                'pack': sensor.pack,
                'file_path': file_path,
                'class_name': class_name,
                'trigger_types': sensor.trigger_types,
                'poll_interval': sensor.poll_interval
            }
            sensors_to_run.append(sensor_obj)

        LOG.info('(PID:%s) SensorContainer started.', os.getpid())
        sensor_container = ProcessSensorContainer(sensors=sensors_to_run)

        def sigterm_handler(signum=None, frame=None):
            # This will cause SystemExit to be throw and we call sensor_container.shutdown()
            # there which cleans things up.
            sys.exit(0)

        # Register a SIGTERM signal handler which calls sys.exit which causes SystemExit to
        # be thrown. We catch SystemExit and handle cleanup there.
        signal.signal(signal.SIGTERM, sigterm_handler)

        try:
            exit_code = sensor_container.run()
            LOG.info('(PID:%s) SensorContainer stopped. Reason - run ended.', os.getpid())
            return exit_code
        except (KeyboardInterrupt, SystemExit):
            sensor_container.shutdown()

            LOG.info('(PID:%s) SensorContainer stopped. Reason - %s', os.getpid(),
                     sys.exc_info()[0].__name__)
            return 0
Exemple #4
0
class SensorContainerManager(object):
    def __init__(self, sensors_partitioner):
        self._sensor_container = None
        self._sensors_watcher = SensorWatcher(
            create_handler=self._handle_create_sensor,
            update_handler=self._handle_update_sensor,
            delete_handler=self._handle_delete_sensor,
            queue_suffix="sensor_container",
        )
        self._container_thread = None
        if not sensors_partitioner:
            raise ValueError("sensors_partitioner should be non-None.")
        self._sensors_partitioner = sensors_partitioner

    def run_sensors(self):
        """
        Run all sensors as determined by sensors_partitioner.
        """
        sensors = self._sensors_partitioner.get_sensors()
        if sensors:
            LOG.info("Setting up container to run %d sensors.", len(sensors))
            LOG.info("\tSensors list - %s.", [self._get_sensor_ref(sensor) for sensor in sensors])

        sensors_to_run = []
        for sensor in sensors:
            # TODO: Directly pass DB object to the ProcessContainer
            sensors_to_run.append(self._to_sensor_object(sensor))

        LOG.info("(PID:%s) SensorContainer started.", os.getpid())
        self._setup_sigterm_handler()
        self._spin_container_and_wait(sensors_to_run)

    def _spin_container_and_wait(self, sensors):
        try:
            self._sensor_container = ProcessSensorContainer(sensors=sensors)
            self._container_thread = eventlet.spawn(self._sensor_container.run)
            LOG.debug("Starting sensor CUD watcher...")
            self._sensors_watcher.start()
            exit_code = self._container_thread.wait()
            LOG.error("Process container quit with exit_code %d.", exit_code)
            LOG.error("(PID:%s) SensorContainer stopped.", os.getpid())
        except (KeyboardInterrupt, SystemExit):
            self._sensor_container.shutdown()
            self._sensors_watcher.stop()

            LOG.info("(PID:%s) SensorContainer stopped. Reason - %s", os.getpid(), sys.exc_info()[0].__name__)

            eventlet.kill(self._container_thread)
            self._container_thread = None

            return 0

    def _setup_sigterm_handler(self):
        def sigterm_handler(signum=None, frame=None):
            # This will cause SystemExit to be throw and we call sensor_container.shutdown()
            # there which cleans things up.
            sys.exit(0)

        # Register a SIGTERM signal handler which calls sys.exit which causes SystemExit to
        # be thrown. We catch SystemExit and handle cleanup there.
        signal.signal(signal.SIGTERM, sigterm_handler)

    def _to_sensor_object(self, sensor_db):
        file_path = sensor_db.artifact_uri.replace("file://", "")
        class_name = sensor_db.entry_point.split(".")[-1]

        sensor_obj = {
            "pack": sensor_db.pack,
            "file_path": file_path,
            "class_name": class_name,
            "trigger_types": sensor_db.trigger_types,
            "poll_interval": sensor_db.poll_interval,
            "ref": self._get_sensor_ref(sensor_db),
        }

        return sensor_obj

    #################################################
    # Event handler methods for the sensor CUD events
    #################################################

    def _handle_create_sensor(self, sensor):
        if not self._sensors_partitioner.is_sensor_owner(sensor):
            LOG.info("sensor %s is not supported. Ignoring create.", self._get_sensor_ref(sensor))
            return
        if not sensor.enabled:
            LOG.info("sensor %s is not enabled.", self._get_sensor_ref(sensor))
            return
        LOG.info("Adding sensor %s.", self._get_sensor_ref(sensor))
        self._sensor_container.add_sensor(sensor=self._to_sensor_object(sensor))

    def _handle_update_sensor(self, sensor):
        if not self._sensors_partitioner.is_sensor_owner(sensor):
            LOG.info("sensor %s is not supported. Ignoring update.", self._get_sensor_ref(sensor))
            return
        sensor_ref = self._get_sensor_ref(sensor)
        LOG.info("Sensor %s updated. Reloading sensor.", sensor_ref)
        sensor_obj = self._to_sensor_object(sensor)
        try:
            self._sensor_container.remove_sensor(sensor=sensor_obj)
        except:
            LOG.exception("Failed to reload sensor %s", sensor_ref)
        else:
            self._sensor_container.add_sensor(sensor=sensor_obj)
            LOG.info("Sensor %s reloaded.", sensor_ref)

    def _handle_delete_sensor(self, sensor):
        if not self._sensors_partitioner.is_sensor_owner(sensor):
            LOG.info("sensor %s is not supported. Ignoring delete.", self._get_sensor_ref(sensor))
            return
        LOG.info("Unloading sensor %s.", self._get_sensor_ref(sensor))
        self._sensor_container.remove_sensor(sensor=self._to_sensor_object(sensor))

    def _get_sensor_ref(self, sensor):
        return ResourceReference.to_string_reference(pack=sensor.pack, name=sensor.name)
Exemple #5
0
class SensorContainerManager(object):
    # TODO: Load balancing for sensors.
    def __init__(self, max_containers=10):
        self._max_containers = max_containers
        self._sensor_container = None
        self._sensors_watcher = SensorWatcher(
            create_handler=self._handle_create_sensor,
            update_handler=self._handle_update_sensor,
            delete_handler=self._handle_delete_sensor,
            queue_suffix='sensor_container')
        self._container_thread = None

    def run_sensors(self, sensors):
        """
        :param sensors: A list of DB models of sensors to run.
        :type sensors: ``list``
        """
        if sensors:
            LOG.info('Setting up container to run %d sensors.', len(sensors))

        sensors_to_run = []
        for sensor in sensors:
            # TODO: Directly pass DB object to the ProcessContainer
            sensors_to_run.append(self._to_sensor_object(sensor))

        LOG.info('(PID:%s) SensorContainer started.', os.getpid())
        self._setup_sigterm_handler()
        self._spin_container_and_wait(sensors_to_run)

    def _spin_container_and_wait(self, sensors):
        try:
            self._sensor_container = ProcessSensorContainer(sensors=sensors)
            self._container_thread = eventlet.spawn(self._sensor_container.run)
            LOG.debug('Starting sensor CUD watcher...')
            self._sensors_watcher.start()
            exit_code = self._container_thread.wait()
            LOG.error('Process container quit with exit_code %d.', exit_code)
            LOG.error('(PID:%s) SensorContainer stopped.', os.getpid())
        except (KeyboardInterrupt, SystemExit):
            self._sensor_container.shutdown()
            self._sensors_watcher.stop()

            LOG.info('(PID:%s) SensorContainer stopped. Reason - %s',
                     os.getpid(),
                     sys.exc_info()[0].__name__)

            self._container_thread = eventlet.kill(self._container_thread)

            return 0

    def _setup_sigterm_handler(self):
        def sigterm_handler(signum=None, frame=None):
            # This will cause SystemExit to be throw and we call sensor_container.shutdown()
            # there which cleans things up.
            sys.exit(0)

        # Register a SIGTERM signal handler which calls sys.exit which causes SystemExit to
        # be thrown. We catch SystemExit and handle cleanup there.
        signal.signal(signal.SIGTERM, sigterm_handler)

    def _to_sensor_object(self, sensor_db):
        file_path = sensor_db.artifact_uri.replace('file://', '')
        class_name = sensor_db.entry_point.split('.')[-1]

        sensor_obj = {
            'pack': sensor_db.pack,
            'file_path': file_path,
            'class_name': class_name,
            'trigger_types': sensor_db.trigger_types,
            'poll_interval': sensor_db.poll_interval,
            'ref': self._get_sensor_ref(sensor_db)
        }

        return sensor_obj

    #################################################
    # Event handler methods for the sensor CUD events
    #################################################

    def _handle_create_sensor(self, sensor):
        LOG.info('Adding sensor %s.', self._get_sensor_ref(sensor))
        self._sensor_container.add_sensor(
            sensor=self._to_sensor_object(sensor))

    def _handle_update_sensor(self, sensor):
        sensor_ref = self._get_sensor_ref(sensor)
        LOG.info('Sensor %s updated. Reloading sensor.', sensor_ref)
        sensor_obj = self._to_sensor_object(sensor)
        try:
            self._sensor_container.remove_sensor(sensor=sensor_obj)
        except:
            LOG.exception('Failed to reload sensor %s', sensor_ref)
        else:
            self._sensor_container.add_sensor(sensor=sensor_obj)
            LOG.info('Sensor %s reloaded.', sensor_ref)

    def _handle_delete_sensor(self, sensor):
        LOG.info('Unloading sensor %s.', self._get_sensor_ref(sensor))
        self._sensor_container.remove_sensor(
            sensor=self._to_sensor_object(sensor))

    def _get_sensor_ref(self, sensor):
        return ResourceReference.to_string_reference(pack=sensor.pack,
                                                     name=sensor.name)
Exemple #6
0
class SensorContainerManager(object):
    def __init__(self, sensors_partitioner, single_sensor_mode=False):
        if not sensors_partitioner:
            raise ValueError('sensors_partitioner should be non-None.')

        self._sensors_partitioner = sensors_partitioner
        self._single_sensor_mode = single_sensor_mode

        self._sensor_container = None
        self._container_thread = None

        self._sensors_watcher = SensorWatcher(
            create_handler=self._handle_create_sensor,
            update_handler=self._handle_update_sensor,
            delete_handler=self._handle_delete_sensor,
            queue_suffix='sensor_container')

    def run_sensors(self):
        """
        Run all sensors as determined by sensors_partitioner.
        """
        sensors = self._sensors_partitioner.get_sensors()
        if sensors:
            LOG.info('Setting up container to run %d sensors.', len(sensors))
            LOG.info('\tSensors list - %s.',
                     [self._get_sensor_ref(sensor) for sensor in sensors])

        sensors_to_run = []
        for sensor in sensors:
            # TODO: Directly pass DB object to the ProcessContainer
            sensors_to_run.append(self._to_sensor_object(sensor))

        LOG.info('(PID:%s) SensorContainer started.', os.getpid())
        self._setup_sigterm_handler()

        exit_code = self._spin_container_and_wait(sensors_to_run)
        return exit_code

    def _spin_container_and_wait(self, sensors):
        exit_code = 0

        try:
            self._sensor_container = ProcessSensorContainer(
                sensors=sensors, single_sensor_mode=self._single_sensor_mode)
            self._container_thread = concurrency.spawn(
                self._sensor_container.run)

            LOG.debug('Starting sensor CUD watcher...')
            self._sensors_watcher.start()

            exit_code = self._container_thread.wait()
            LOG.error('Process container quit with exit_code %d.', exit_code)
            LOG.error('(PID:%s) SensorContainer stopped.', os.getpid())
        except (KeyboardInterrupt, SystemExit):
            self._sensor_container.shutdown()
            self._sensors_watcher.stop()

            LOG.info('(PID:%s) SensorContainer stopped. Reason - %s',
                     os.getpid(),
                     sys.exc_info()[0].__name__)

            concurrency.kill(self._container_thread)
            self._container_thread = None

            return exit_code

        return exit_code

    def _setup_sigterm_handler(self):
        def sigterm_handler(signum=None, frame=None):
            # This will cause SystemExit to be throw and we call sensor_container.shutdown()
            # there which cleans things up.
            sys.exit(0)

        # Register a SIGTERM signal handler which calls sys.exit which causes SystemExit to
        # be thrown. We catch SystemExit and handle cleanup there.
        signal.signal(signal.SIGTERM, sigterm_handler)

    def _to_sensor_object(self, sensor_db):
        file_path = sensor_db.artifact_uri.replace('file://', '')
        class_name = sensor_db.entry_point.split('.')[-1]

        sensor_obj = {
            'pack': sensor_db.pack,
            'file_path': file_path,
            'class_name': class_name,
            'trigger_types': sensor_db.trigger_types,
            'poll_interval': sensor_db.poll_interval,
            'ref': self._get_sensor_ref(sensor_db)
        }

        return sensor_obj

    #################################################
    # Event handler methods for the sensor CUD events
    #################################################

    def _handle_create_sensor(self, sensor):
        if not self._sensors_partitioner.is_sensor_owner(sensor):
            LOG.info('sensor %s is not supported. Ignoring create.',
                     self._get_sensor_ref(sensor))
            return
        if not sensor.enabled:
            LOG.info('sensor %s is not enabled.', self._get_sensor_ref(sensor))
            return
        LOG.info('Adding sensor %s.', self._get_sensor_ref(sensor))
        self._sensor_container.add_sensor(
            sensor=self._to_sensor_object(sensor))

    def _handle_update_sensor(self, sensor):
        if not self._sensors_partitioner.is_sensor_owner(sensor):
            LOG.info(
                'sensor %s is not assigned to this partition. Ignoring update. ',
                self._get_sensor_ref(sensor))
            return
        sensor_ref = self._get_sensor_ref(sensor)
        sensor_obj = self._to_sensor_object(sensor)

        # Handle disabling sensor
        if not sensor.enabled:
            LOG.info('Sensor %s disabled. Unloading sensor.', sensor_ref)
            self._sensor_container.remove_sensor(sensor=sensor_obj)
            return

        LOG.info('Sensor %s updated. Reloading sensor.', sensor_ref)
        try:
            self._sensor_container.remove_sensor(sensor=sensor_obj)
        except:
            LOG.exception('Failed to reload sensor %s', sensor_ref)
        else:
            self._sensor_container.add_sensor(sensor=sensor_obj)
            LOG.info('Sensor %s reloaded.', sensor_ref)

    def _handle_delete_sensor(self, sensor):
        if not self._sensors_partitioner.is_sensor_owner(sensor):
            LOG.info('sensor %s is not supported. Ignoring delete.',
                     self._get_sensor_ref(sensor))
            return
        LOG.info('Unloading sensor %s.', self._get_sensor_ref(sensor))
        self._sensor_container.remove_sensor(
            sensor=self._to_sensor_object(sensor))

    def _get_sensor_ref(self, sensor):
        return ResourceReference.to_string_reference(pack=sensor.pack,
                                                     name=sensor.name)
Exemple #7
0
class SensorContainerManager(object):
    # TODO: Load balancing for sensors.
    def __init__(self, max_containers=10):
        self._max_containers = max_containers
        self._sensor_container = None
        self._sensors_watcher = SensorWatcher(create_handler=self._handle_create_sensor,
                                              update_handler=self._handle_update_sensor,
                                              delete_handler=self._handle_delete_sensor,
                                              queue_suffix='sensor_container')
        self._container_thread = None

    def run_sensors(self, sensors):
        """
        :param sensors: A list of DB models of sensors to run.
        :type sensors: ``list``
        """
        if sensors:
            LOG.info('Setting up container to run %d sensors.', len(sensors))

        sensors_to_run = []
        for sensor in sensors:
            # TODO: Directly pass DB object to the ProcessContainer
            sensors_to_run.append(self._to_sensor_object(sensor))

        LOG.info('(PID:%s) SensorContainer started.', os.getpid())
        self._setup_sigterm_handler()
        self._spin_container_and_wait(sensors_to_run)

    def _spin_container_and_wait(self, sensors):
        try:
            self._sensor_container = ProcessSensorContainer(sensors=sensors)
            self._container_thread = eventlet.spawn(self._sensor_container.run)
            LOG.debug('Starting sensor CUD watcher...')
            self._sensors_watcher.start()
            exit_code = self._container_thread.wait()
            LOG.error('Process container quit with exit_code %d.', exit_code)
            LOG.error('(PID:%s) SensorContainer stopped.', os.getpid())
        except (KeyboardInterrupt, SystemExit):
            self._sensor_container.shutdown()
            self._sensors_watcher.stop()

            LOG.info('(PID:%s) SensorContainer stopped. Reason - %s', os.getpid(),
                     sys.exc_info()[0].__name__)

            self._container_thread = eventlet.kill(self._container_thread)

            return 0

    def _setup_sigterm_handler(self):

        def sigterm_handler(signum=None, frame=None):
            # This will cause SystemExit to be throw and we call sensor_container.shutdown()
            # there which cleans things up.
            sys.exit(0)

        # Register a SIGTERM signal handler which calls sys.exit which causes SystemExit to
        # be thrown. We catch SystemExit and handle cleanup there.
        signal.signal(signal.SIGTERM, sigterm_handler)

    def _to_sensor_object(self, sensor_db):
        file_path = sensor_db.artifact_uri.replace('file://', '')
        class_name = sensor_db.entry_point.split('.')[-1]

        sensor_obj = {
            'pack': sensor_db.pack,
            'file_path': file_path,
            'class_name': class_name,
            'trigger_types': sensor_db.trigger_types,
            'poll_interval': sensor_db.poll_interval,
            'ref': self._get_sensor_ref(sensor_db)
        }

        return sensor_obj

    #################################################
    # Event handler methods for the sensor CUD events
    #################################################

    def _handle_create_sensor(self, sensor):
        LOG.info('Adding sensor %s.', self._get_sensor_ref(sensor))
        self._sensor_container.add_sensor(sensor=self._to_sensor_object(sensor))

    def _handle_update_sensor(self, sensor):
        sensor_ref = self._get_sensor_ref(sensor)
        LOG.info('Sensor %s updated. Reloading sensor.', sensor_ref)
        sensor_obj = self._to_sensor_object(sensor)
        try:
            self._sensor_container.remove_sensor(sensor=sensor_obj)
        except:
            LOG.exception('Failed to reload sensor %s', sensor_ref)
        else:
            self._sensor_container.add_sensor(sensor=sensor_obj)
            LOG.info('Sensor %s reloaded.', sensor_ref)

    def _handle_delete_sensor(self, sensor):
        LOG.info('Unloading sensor %s.', self._get_sensor_ref(sensor))
        self._sensor_container.remove_sensor(sensor=self._to_sensor_object(sensor))

    def _get_sensor_ref(self, sensor):
        return ResourceReference.to_string_reference(pack=sensor.pack, name=sensor.name)