示例#1
0
    def test_common_lib_path_not_in_pythonpath_env_var(self, mock_create_token, mock_subproc_popen):
        process_mock = Mock()
        attrs = {'communicate.return_value': ('output', 'error')}
        process_mock.configure_mock(**attrs)
        mock_subproc_popen.return_value = process_mock

        mock_create_token = Mock()
        mock_create_token.return_value = 'WHOLETTHEDOGSOUT'

        mock_dispatcher = Mock()
        process_container = ProcessSensorContainer(None, poll_interval=0.1,
                                                   dispatcher=mock_dispatcher)
        sensor = {
            'class_name': 'wolfpack.StupidSensor',
            'ref': 'wolfpack.StupidSensor',
            'id': '567890',
            'trigger_types': ['some_trigga'],
            'pack': 'wolfpack',
            'file_path': '/opt/stackstorm/packs/wolfpack/sensors/stupid_sensor.py',
            'poll_interval': 5
        }

        process_container._enable_common_pack_libs = False
        process_container._sensors = {'pack.StupidSensor': sensor}
        process_container._spawn_sensor_process(sensor)

        _, call_kwargs = mock_subproc_popen.call_args
        actual_env = call_kwargs['env']
        self.assertTrue('PYTHONPATH' in actual_env)
        pack_common_lib_path = '/opt/stackstorm/packs/wolfpack/lib'
        self.assertTrue(pack_common_lib_path not in actual_env['PYTHONPATH'])
示例#2
0
    def run_sensors(self, sensors):
        """
        :param sensors: A list of DB models of sensors to run.
        :type sensors: ``list``
        """
        LOG.info('Setting up container to run %d sensors.', len(sensors))

        sensors_to_run = []
        for sensor in sensors:
            # TODO: Directly pass DB object to the ProcessContainer
            file_path = sensor.artifact_uri.replace('file://', '')
            class_name = sensor.entry_point.split('.')[-1]

            sensor_obj = {
                'pack': sensor.pack,
                'file_path': file_path,
                'class_name': class_name,
                'trigger_types': sensor.trigger_types,
                'poll_interval': sensor.poll_interval
            }
            sensors_to_run.append(sensor_obj)

        LOG.info('(PID:%s) SensorContainer started.', os.getpid())
        sensor_container = ProcessSensorContainer(sensors=sensors_to_run)
        try:
            exit_code = sensor_container.run()
            LOG.info('(PID:%s) SensorContainer stopped. Reason - run ended.', os.getpid())
            return exit_code
        except (KeyboardInterrupt, SystemExit):
            LOG.info('(PID:%s) SensorContainer stopped. Reason - %s', os.getpid(),
                     sys.exc_info()[0].__name__)
            return 0
示例#3
0
    def _spin_container_and_wait(self, sensors):
        exit_code = 0

        try:
            self._sensor_container = ProcessSensorContainer(
                sensors=sensors, single_sensor_mode=self._single_sensor_mode)
            self._container_thread = eventlet.spawn(self._sensor_container.run)

            LOG.debug('Starting sensor CUD watcher...')
            self._sensors_watcher.start()

            exit_code = self._container_thread.wait()
            LOG.error('Process container quit with exit_code %d.', exit_code)
            LOG.error('(PID:%s) SensorContainer stopped.', os.getpid())
        except (KeyboardInterrupt, SystemExit):
            self._sensor_container.shutdown()
            self._sensors_watcher.stop()

            LOG.info('(PID:%s) SensorContainer stopped. Reason - %s',
                     os.getpid(),
                     sys.exc_info()[0].__name__)

            eventlet.kill(self._container_thread)
            self._container_thread = None

            return exit_code

        return exit_code
示例#4
0
    def test_dispatch_triggers_on_spawn_exit(self):
        mock_dispatcher = Mock()
        process_container = ProcessSensorContainer(None, poll_interval=0.1,
                                                   dispatcher=mock_dispatcher)
        sensor = {
            'class_name': 'pack.StupidSensor'
        }
        process = Mock()
        process_attrs = {'pid': 1234}
        process.configure_mock(**process_attrs)
        cmd = 'sensor_wrapper.py --class-name pack.StupidSensor'

        process_container._dispatch_trigger_for_sensor_spawn(sensor, process, cmd)
        mock_dispatcher.dispatch.assert_called_with(
            'core.st2.sensor.process_spawn',
            payload={
                'timestamp': 1439441533,
                'cmd': 'sensor_wrapper.py --class-name pack.StupidSensor',
                'pid': 1234,
                'id': 'pack.StupidSensor'})

        process_container._dispatch_trigger_for_sensor_exit(sensor, 1)
        mock_dispatcher.dispatch.assert_called_with(
            'core.st2.sensor.process_exit',
            payload={
                'id': 'pack.StupidSensor',
                'timestamp': 1439441533,
                'exit_code': 1
            })
 def test_no_sensors_dont_quit(self):
     process_container = ProcessSensorContainer(None, poll_interval=0.1)
     process_container_thread = eventlet.spawn(process_container.run)
     eventlet.sleep(0.5)
     self.assertEqual(process_container.running(), 0)
     self.assertEqual(process_container.stopped, False)
     process_container.shutdown()
     process_container_thread.kill()
示例#6
0
 def test_no_sensors_dont_quit(self):
     process_container = ProcessSensorContainer(None, poll_interval=0.1)
     process_container_thread = eventlet.spawn(process_container.run)
     eventlet.sleep(0.5)
     self.assertEqual(process_container.running(), 0)
     self.assertEqual(process_container.stopped, False)
     process_container.shutdown()
     process_container_thread.kill()
示例#7
0
文件: manager.py 项目: lyandut/st2
    def _spin_container_and_wait(self, sensors):
        exit_code = 0

        try:
            self._sensor_container = ProcessSensorContainer(
                sensors=sensors,
                single_sensor_mode=self._single_sensor_mode)
            self._container_thread = eventlet.spawn(self._sensor_container.run)

            LOG.debug('Starting sensor CUD watcher...')
            self._sensors_watcher.start()

            exit_code = self._container_thread.wait()
            LOG.error('Process container quit with exit_code %d.', exit_code)
            LOG.error('(PID:%s) SensorContainer stopped.', os.getpid())
        except (KeyboardInterrupt, SystemExit):
            self._sensor_container.shutdown()
            self._sensors_watcher.stop()

            LOG.info('(PID:%s) SensorContainer stopped. Reason - %s', os.getpid(),
                     sys.exc_info()[0].__name__)

            eventlet.kill(self._container_thread)
            self._container_thread = None

            return exit_code

        return exit_code
示例#8
0
文件: manager.py 项目: miqui/st2
    def run_sensors(self, sensors):
        """
        :param sensors: A list of DB models of sensors to run.
        :type sensors: ``list``
        """
        LOG.info('Setting up container to run %d sensors.', len(sensors))

        sensors_to_run = []
        for sensor in sensors:
            # TODO: Directly pass DB object to the ProcessContainer
            file_path = sensor.artifact_uri.replace('file://', '')
            class_name = sensor.entry_point.split('.')[-1]

            sensor_obj = {
                'pack': sensor.pack,
                'file_path': file_path,
                'class_name': class_name,
                'trigger_types': sensor.trigger_types,
                'poll_interval': sensor.poll_interval
            }
            sensors_to_run.append(sensor_obj)

        LOG.info('(PID:%s) SensorContainer started.', os.getpid())
        sensor_container = ProcessSensorContainer(sensors=sensors_to_run)

        def sigterm_handler(signum=None, frame=None):
            # This will cause SystemExit to be throw and we call sensor_container.shutdown()
            # there which cleans things up.
            sys.exit(0)

        # Register a SIGTERM signal handler which calls sys.exit which causes SystemExit to
        # be thrown. We catch SystemExit and handle cleanup there.
        signal.signal(signal.SIGTERM, sigterm_handler)

        try:
            exit_code = sensor_container.run()
            LOG.info('(PID:%s) SensorContainer stopped. Reason - run ended.', os.getpid())
            return exit_code
        except (KeyboardInterrupt, SystemExit):
            sensor_container.shutdown()

            LOG.info('(PID:%s) SensorContainer stopped. Reason - %s', os.getpid(),
                     sys.exc_info()[0].__name__)
            return 0
    def test_dispatch_triggers_on_spawn_exit(self):
        mock_dispatcher = Mock()
        process_container = ProcessSensorContainer(None,
                                                   poll_interval=0.1,
                                                   dispatcher=mock_dispatcher)
        sensor = {'class_name': 'pack.StupidSensor'}
        process = Mock()
        process_attrs = {'pid': 1234}
        process.configure_mock(**process_attrs)
        cmd = 'sensor_wrapper.py --class-name pack.StupidSensor'

        process_container._dispatch_trigger_for_sensor_spawn(
            sensor, process, cmd)
        mock_dispatcher.dispatch.assert_called_with(
            'core.st2.sensor.process_spawn',
            payload={
                'timestamp': 1439441533,
                'cmd': 'sensor_wrapper.py --class-name pack.StupidSensor',
                'pid': 1234,
                'id': 'pack.StupidSensor'
            })

        process_container._dispatch_trigger_for_sensor_exit(sensor, 1)
        mock_dispatcher.dispatch.assert_called_with(
            'core.st2.sensor.process_exit',
            payload={
                'id': 'pack.StupidSensor',
                'timestamp': 1439441533,
                'exit_code': 1
            })
示例#10
0
    def test_common_lib_path_not_in_pythonpath_env_var(self, mock_create_token,
                                                       mock_subproc_popen):
        process_mock = Mock()
        attrs = {'communicate.return_value': ('output', 'error')}
        process_mock.configure_mock(**attrs)
        mock_subproc_popen.return_value = process_mock

        mock_create_token = Mock()
        mock_create_token.return_value = 'WHOLETTHEDOGSOUT'

        mock_dispatcher = Mock()
        process_container = ProcessSensorContainer(None,
                                                   poll_interval=0.1,
                                                   dispatcher=mock_dispatcher)
        sensor = {
            'class_name': 'wolfpack.StupidSensor',
            'ref': 'wolfpack.StupidSensor',
            'id': '567890',
            'trigger_types': ['some_trigga'],
            'pack': 'wolfpack',
            'file_path':
            '/opt/stackstorm/packs/wolfpack/sensors/stupid_sensor.py',
            'poll_interval': 5
        }

        process_container._enable_common_pack_libs = False
        process_container._sensors = {'pack.StupidSensor': sensor}
        process_container._spawn_sensor_process(sensor)

        _, call_kwargs = mock_subproc_popen.call_args
        actual_env = call_kwargs['env']
        self.assertIn('PYTHONPATH', actual_env)
        pack_common_lib_path = '/opt/stackstorm/packs/wolfpack/lib'
        self.assertNotIn(pack_common_lib_path, actual_env['PYTHONPATH'])
示例#11
0
    def test_common_lib_path_in_pythonpath_env_var(self, mock_create_token,
                                                   mock_subproc_popen):
        process_mock = Mock()
        attrs = {"communicate.return_value": ("output", "error")}
        process_mock.configure_mock(**attrs)
        mock_subproc_popen.return_value = process_mock

        mock_create_token = Mock()
        mock_create_token.return_value = "WHOLETTHEDOGSOUT"

        mock_dispatcher = Mock()
        process_container = ProcessSensorContainer(None,
                                                   poll_interval=0.1,
                                                   dispatcher=mock_dispatcher)
        sensor = {
            "class_name": "wolfpack.StupidSensor",
            "ref": "wolfpack.StupidSensor",
            "id": "567890",
            "trigger_types": ["some_trigga"],
            "pack": "wolfpack",
            "file_path":
            "/opt/stackstorm/packs/wolfpack/sensors/stupid_sensor.py",
            "poll_interval": 5,
        }

        process_container._enable_common_pack_libs = True
        process_container._sensors = {"pack.StupidSensor": sensor}
        process_container._spawn_sensor_process(sensor)

        _, call_kwargs = mock_subproc_popen.call_args
        actual_env = call_kwargs["env"]
        self.assertIn("PYTHONPATH", actual_env)
        pack_common_lib_path = "/opt/stackstorm/packs/wolfpack/lib"
        self.assertIn(pack_common_lib_path, actual_env["PYTHONPATH"])
示例#12
0
    def test_dispatch_triggers_on_spawn_exit(self):
        mock_dispatcher = Mock()
        process_container = ProcessSensorContainer(None,
                                                   poll_interval=0.1,
                                                   dispatcher=mock_dispatcher)
        sensor = {"class_name": "pack.StupidSensor"}
        process = Mock()
        process_attrs = {"pid": 1234}
        process.configure_mock(**process_attrs)
        cmd = "sensor_wrapper.py --class-name pack.StupidSensor"

        process_container._dispatch_trigger_for_sensor_spawn(
            sensor, process, cmd)
        mock_dispatcher.dispatch.assert_called_with(
            "core.st2.sensor.process_spawn",
            payload={
                "timestamp": 1439441533,
                "cmd": "sensor_wrapper.py --class-name pack.StupidSensor",
                "pid": 1234,
                "id": "pack.StupidSensor",
            },
        )

        process_container._dispatch_trigger_for_sensor_exit(sensor, 1)
        mock_dispatcher.dispatch.assert_called_with(
            "core.st2.sensor.process_exit",
            payload={
                "id": "pack.StupidSensor",
                "timestamp": 1439441533,
                "exit_code": 1,
            },
        )
示例#13
0
文件: manager.py 项目: ruslantum/st2
class SensorContainerManager(object):
    def __init__(self, sensors_partitioner):
        self._sensor_container = None
        self._sensors_watcher = SensorWatcher(
            create_handler=self._handle_create_sensor,
            update_handler=self._handle_update_sensor,
            delete_handler=self._handle_delete_sensor,
            queue_suffix="sensor_container",
        )
        self._container_thread = None
        if not sensors_partitioner:
            raise ValueError("sensors_partitioner should be non-None.")
        self._sensors_partitioner = sensors_partitioner

    def run_sensors(self):
        """
        Run all sensors as determined by sensors_partitioner.
        """
        sensors = self._sensors_partitioner.get_sensors()
        if sensors:
            LOG.info("Setting up container to run %d sensors.", len(sensors))
            LOG.info("\tSensors list - %s.", [self._get_sensor_ref(sensor) for sensor in sensors])

        sensors_to_run = []
        for sensor in sensors:
            # TODO: Directly pass DB object to the ProcessContainer
            sensors_to_run.append(self._to_sensor_object(sensor))

        LOG.info("(PID:%s) SensorContainer started.", os.getpid())
        self._setup_sigterm_handler()
        self._spin_container_and_wait(sensors_to_run)

    def _spin_container_and_wait(self, sensors):
        try:
            self._sensor_container = ProcessSensorContainer(sensors=sensors)
            self._container_thread = eventlet.spawn(self._sensor_container.run)
            LOG.debug("Starting sensor CUD watcher...")
            self._sensors_watcher.start()
            exit_code = self._container_thread.wait()
            LOG.error("Process container quit with exit_code %d.", exit_code)
            LOG.error("(PID:%s) SensorContainer stopped.", os.getpid())
        except (KeyboardInterrupt, SystemExit):
            self._sensor_container.shutdown()
            self._sensors_watcher.stop()

            LOG.info("(PID:%s) SensorContainer stopped. Reason - %s", os.getpid(), sys.exc_info()[0].__name__)

            eventlet.kill(self._container_thread)
            self._container_thread = None

            return 0

    def _setup_sigterm_handler(self):
        def sigterm_handler(signum=None, frame=None):
            # This will cause SystemExit to be throw and we call sensor_container.shutdown()
            # there which cleans things up.
            sys.exit(0)

        # Register a SIGTERM signal handler which calls sys.exit which causes SystemExit to
        # be thrown. We catch SystemExit and handle cleanup there.
        signal.signal(signal.SIGTERM, sigterm_handler)

    def _to_sensor_object(self, sensor_db):
        file_path = sensor_db.artifact_uri.replace("file://", "")
        class_name = sensor_db.entry_point.split(".")[-1]

        sensor_obj = {
            "pack": sensor_db.pack,
            "file_path": file_path,
            "class_name": class_name,
            "trigger_types": sensor_db.trigger_types,
            "poll_interval": sensor_db.poll_interval,
            "ref": self._get_sensor_ref(sensor_db),
        }

        return sensor_obj

    #################################################
    # Event handler methods for the sensor CUD events
    #################################################

    def _handle_create_sensor(self, sensor):
        if not self._sensors_partitioner.is_sensor_owner(sensor):
            LOG.info("sensor %s is not supported. Ignoring create.", self._get_sensor_ref(sensor))
            return
        if not sensor.enabled:
            LOG.info("sensor %s is not enabled.", self._get_sensor_ref(sensor))
            return
        LOG.info("Adding sensor %s.", self._get_sensor_ref(sensor))
        self._sensor_container.add_sensor(sensor=self._to_sensor_object(sensor))

    def _handle_update_sensor(self, sensor):
        if not self._sensors_partitioner.is_sensor_owner(sensor):
            LOG.info("sensor %s is not supported. Ignoring update.", self._get_sensor_ref(sensor))
            return
        sensor_ref = self._get_sensor_ref(sensor)
        LOG.info("Sensor %s updated. Reloading sensor.", sensor_ref)
        sensor_obj = self._to_sensor_object(sensor)
        try:
            self._sensor_container.remove_sensor(sensor=sensor_obj)
        except:
            LOG.exception("Failed to reload sensor %s", sensor_ref)
        else:
            self._sensor_container.add_sensor(sensor=sensor_obj)
            LOG.info("Sensor %s reloaded.", sensor_ref)

    def _handle_delete_sensor(self, sensor):
        if not self._sensors_partitioner.is_sensor_owner(sensor):
            LOG.info("sensor %s is not supported. Ignoring delete.", self._get_sensor_ref(sensor))
            return
        LOG.info("Unloading sensor %s.", self._get_sensor_ref(sensor))
        self._sensor_container.remove_sensor(sensor=self._to_sensor_object(sensor))

    def _get_sensor_ref(self, sensor):
        return ResourceReference.to_string_reference(pack=sensor.pack, name=sensor.name)
示例#14
0
文件: manager.py 项目: srenatus/st2
class SensorContainerManager(object):
    # TODO: Load balancing for sensors.
    def __init__(self, max_containers=10):
        self._max_containers = max_containers
        self._sensor_container = None
        self._sensors_watcher = SensorWatcher(
            create_handler=self._handle_create_sensor,
            update_handler=self._handle_update_sensor,
            delete_handler=self._handle_delete_sensor,
            queue_suffix='sensor_container')
        self._container_thread = None

    def run_sensors(self, sensors):
        """
        :param sensors: A list of DB models of sensors to run.
        :type sensors: ``list``
        """
        if sensors:
            LOG.info('Setting up container to run %d sensors.', len(sensors))

        sensors_to_run = []
        for sensor in sensors:
            # TODO: Directly pass DB object to the ProcessContainer
            sensors_to_run.append(self._to_sensor_object(sensor))

        LOG.info('(PID:%s) SensorContainer started.', os.getpid())
        self._setup_sigterm_handler()
        self._spin_container_and_wait(sensors_to_run)

    def _spin_container_and_wait(self, sensors):
        try:
            self._sensor_container = ProcessSensorContainer(sensors=sensors)
            self._container_thread = eventlet.spawn(self._sensor_container.run)
            LOG.debug('Starting sensor CUD watcher...')
            self._sensors_watcher.start()
            exit_code = self._container_thread.wait()
            LOG.error('Process container quit with exit_code %d.', exit_code)
            LOG.error('(PID:%s) SensorContainer stopped.', os.getpid())
        except (KeyboardInterrupt, SystemExit):
            self._sensor_container.shutdown()
            self._sensors_watcher.stop()

            LOG.info('(PID:%s) SensorContainer stopped. Reason - %s',
                     os.getpid(),
                     sys.exc_info()[0].__name__)

            self._container_thread = eventlet.kill(self._container_thread)

            return 0

    def _setup_sigterm_handler(self):
        def sigterm_handler(signum=None, frame=None):
            # This will cause SystemExit to be throw and we call sensor_container.shutdown()
            # there which cleans things up.
            sys.exit(0)

        # Register a SIGTERM signal handler which calls sys.exit which causes SystemExit to
        # be thrown. We catch SystemExit and handle cleanup there.
        signal.signal(signal.SIGTERM, sigterm_handler)

    def _to_sensor_object(self, sensor_db):
        file_path = sensor_db.artifact_uri.replace('file://', '')
        class_name = sensor_db.entry_point.split('.')[-1]

        sensor_obj = {
            'pack': sensor_db.pack,
            'file_path': file_path,
            'class_name': class_name,
            'trigger_types': sensor_db.trigger_types,
            'poll_interval': sensor_db.poll_interval,
            'ref': self._get_sensor_ref(sensor_db)
        }

        return sensor_obj

    #################################################
    # Event handler methods for the sensor CUD events
    #################################################

    def _handle_create_sensor(self, sensor):
        LOG.info('Adding sensor %s.', self._get_sensor_ref(sensor))
        self._sensor_container.add_sensor(
            sensor=self._to_sensor_object(sensor))

    def _handle_update_sensor(self, sensor):
        sensor_ref = self._get_sensor_ref(sensor)
        LOG.info('Sensor %s updated. Reloading sensor.', sensor_ref)
        sensor_obj = self._to_sensor_object(sensor)
        try:
            self._sensor_container.remove_sensor(sensor=sensor_obj)
        except:
            LOG.exception('Failed to reload sensor %s', sensor_ref)
        else:
            self._sensor_container.add_sensor(sensor=sensor_obj)
            LOG.info('Sensor %s reloaded.', sensor_ref)

    def _handle_delete_sensor(self, sensor):
        LOG.info('Unloading sensor %s.', self._get_sensor_ref(sensor))
        self._sensor_container.remove_sensor(
            sensor=self._to_sensor_object(sensor))

    def _get_sensor_ref(self, sensor):
        return ResourceReference.to_string_reference(pack=sensor.pack,
                                                     name=sensor.name)
示例#15
0
class SensorContainerManager(object):
    def __init__(self, sensors_partitioner, single_sensor_mode=False):
        if not sensors_partitioner:
            raise ValueError('sensors_partitioner should be non-None.')

        self._sensors_partitioner = sensors_partitioner
        self._single_sensor_mode = single_sensor_mode

        self._sensor_container = None
        self._container_thread = None

        self._sensors_watcher = SensorWatcher(
            create_handler=self._handle_create_sensor,
            update_handler=self._handle_update_sensor,
            delete_handler=self._handle_delete_sensor,
            queue_suffix='sensor_container')

    def run_sensors(self):
        """
        Run all sensors as determined by sensors_partitioner.
        """
        sensors = self._sensors_partitioner.get_sensors()
        if sensors:
            LOG.info('Setting up container to run %d sensors.', len(sensors))
            LOG.info('\tSensors list - %s.',
                     [self._get_sensor_ref(sensor) for sensor in sensors])

        sensors_to_run = []
        for sensor in sensors:
            # TODO: Directly pass DB object to the ProcessContainer
            sensors_to_run.append(self._to_sensor_object(sensor))

        LOG.info('(PID:%s) SensorContainer started.', os.getpid())
        self._setup_sigterm_handler()

        exit_code = self._spin_container_and_wait(sensors_to_run)
        return exit_code

    def _spin_container_and_wait(self, sensors):
        exit_code = 0

        try:
            self._sensor_container = ProcessSensorContainer(
                sensors=sensors, single_sensor_mode=self._single_sensor_mode)
            self._container_thread = concurrency.spawn(
                self._sensor_container.run)

            LOG.debug('Starting sensor CUD watcher...')
            self._sensors_watcher.start()

            exit_code = self._container_thread.wait()
            LOG.error('Process container quit with exit_code %d.', exit_code)
            LOG.error('(PID:%s) SensorContainer stopped.', os.getpid())
        except (KeyboardInterrupt, SystemExit):
            self._sensor_container.shutdown()
            self._sensors_watcher.stop()

            LOG.info('(PID:%s) SensorContainer stopped. Reason - %s',
                     os.getpid(),
                     sys.exc_info()[0].__name__)

            concurrency.kill(self._container_thread)
            self._container_thread = None

            return exit_code

        return exit_code

    def _setup_sigterm_handler(self):
        def sigterm_handler(signum=None, frame=None):
            # This will cause SystemExit to be throw and we call sensor_container.shutdown()
            # there which cleans things up.
            sys.exit(0)

        # Register a SIGTERM signal handler which calls sys.exit which causes SystemExit to
        # be thrown. We catch SystemExit and handle cleanup there.
        signal.signal(signal.SIGTERM, sigterm_handler)

    def _to_sensor_object(self, sensor_db):
        file_path = sensor_db.artifact_uri.replace('file://', '')
        class_name = sensor_db.entry_point.split('.')[-1]

        sensor_obj = {
            'pack': sensor_db.pack,
            'file_path': file_path,
            'class_name': class_name,
            'trigger_types': sensor_db.trigger_types,
            'poll_interval': sensor_db.poll_interval,
            'ref': self._get_sensor_ref(sensor_db)
        }

        return sensor_obj

    #################################################
    # Event handler methods for the sensor CUD events
    #################################################

    def _handle_create_sensor(self, sensor):
        if not self._sensors_partitioner.is_sensor_owner(sensor):
            LOG.info('sensor %s is not supported. Ignoring create.',
                     self._get_sensor_ref(sensor))
            return
        if not sensor.enabled:
            LOG.info('sensor %s is not enabled.', self._get_sensor_ref(sensor))
            return
        LOG.info('Adding sensor %s.', self._get_sensor_ref(sensor))
        self._sensor_container.add_sensor(
            sensor=self._to_sensor_object(sensor))

    def _handle_update_sensor(self, sensor):
        if not self._sensors_partitioner.is_sensor_owner(sensor):
            LOG.info(
                'sensor %s is not assigned to this partition. Ignoring update. ',
                self._get_sensor_ref(sensor))
            return
        sensor_ref = self._get_sensor_ref(sensor)
        sensor_obj = self._to_sensor_object(sensor)

        # Handle disabling sensor
        if not sensor.enabled:
            LOG.info('Sensor %s disabled. Unloading sensor.', sensor_ref)
            self._sensor_container.remove_sensor(sensor=sensor_obj)
            return

        LOG.info('Sensor %s updated. Reloading sensor.', sensor_ref)
        try:
            self._sensor_container.remove_sensor(sensor=sensor_obj)
        except:
            LOG.exception('Failed to reload sensor %s', sensor_ref)
        else:
            self._sensor_container.add_sensor(sensor=sensor_obj)
            LOG.info('Sensor %s reloaded.', sensor_ref)

    def _handle_delete_sensor(self, sensor):
        if not self._sensors_partitioner.is_sensor_owner(sensor):
            LOG.info('sensor %s is not supported. Ignoring delete.',
                     self._get_sensor_ref(sensor))
            return
        LOG.info('Unloading sensor %s.', self._get_sensor_ref(sensor))
        self._sensor_container.remove_sensor(
            sensor=self._to_sensor_object(sensor))

    def _get_sensor_ref(self, sensor):
        return ResourceReference.to_string_reference(pack=sensor.pack,
                                                     name=sensor.name)
示例#16
0
class SensorContainerManager(object):
    # TODO: Load balancing for sensors.
    def __init__(self, max_containers=10):
        self._max_containers = max_containers
        self._sensor_container = None
        self._sensors_watcher = SensorWatcher(create_handler=self._handle_create_sensor,
                                              update_handler=self._handle_update_sensor,
                                              delete_handler=self._handle_delete_sensor,
                                              queue_suffix='sensor_container')
        self._container_thread = None

    def run_sensors(self, sensors):
        """
        :param sensors: A list of DB models of sensors to run.
        :type sensors: ``list``
        """
        if sensors:
            LOG.info('Setting up container to run %d sensors.', len(sensors))

        sensors_to_run = []
        for sensor in sensors:
            # TODO: Directly pass DB object to the ProcessContainer
            sensors_to_run.append(self._to_sensor_object(sensor))

        LOG.info('(PID:%s) SensorContainer started.', os.getpid())
        self._setup_sigterm_handler()
        self._spin_container_and_wait(sensors_to_run)

    def _spin_container_and_wait(self, sensors):
        try:
            self._sensor_container = ProcessSensorContainer(sensors=sensors)
            self._container_thread = eventlet.spawn(self._sensor_container.run)
            LOG.debug('Starting sensor CUD watcher...')
            self._sensors_watcher.start()
            exit_code = self._container_thread.wait()
            LOG.error('Process container quit with exit_code %d.', exit_code)
            LOG.error('(PID:%s) SensorContainer stopped.', os.getpid())
        except (KeyboardInterrupt, SystemExit):
            self._sensor_container.shutdown()
            self._sensors_watcher.stop()

            LOG.info('(PID:%s) SensorContainer stopped. Reason - %s', os.getpid(),
                     sys.exc_info()[0].__name__)

            self._container_thread = eventlet.kill(self._container_thread)

            return 0

    def _setup_sigterm_handler(self):

        def sigterm_handler(signum=None, frame=None):
            # This will cause SystemExit to be throw and we call sensor_container.shutdown()
            # there which cleans things up.
            sys.exit(0)

        # Register a SIGTERM signal handler which calls sys.exit which causes SystemExit to
        # be thrown. We catch SystemExit and handle cleanup there.
        signal.signal(signal.SIGTERM, sigterm_handler)

    def _to_sensor_object(self, sensor_db):
        file_path = sensor_db.artifact_uri.replace('file://', '')
        class_name = sensor_db.entry_point.split('.')[-1]

        sensor_obj = {
            'pack': sensor_db.pack,
            'file_path': file_path,
            'class_name': class_name,
            'trigger_types': sensor_db.trigger_types,
            'poll_interval': sensor_db.poll_interval,
            'ref': self._get_sensor_ref(sensor_db)
        }

        return sensor_obj

    #################################################
    # Event handler methods for the sensor CUD events
    #################################################

    def _handle_create_sensor(self, sensor):
        LOG.info('Adding sensor %s.', self._get_sensor_ref(sensor))
        self._sensor_container.add_sensor(sensor=self._to_sensor_object(sensor))

    def _handle_update_sensor(self, sensor):
        sensor_ref = self._get_sensor_ref(sensor)
        LOG.info('Sensor %s updated. Reloading sensor.', sensor_ref)
        sensor_obj = self._to_sensor_object(sensor)
        try:
            self._sensor_container.remove_sensor(sensor=sensor_obj)
        except:
            LOG.exception('Failed to reload sensor %s', sensor_ref)
        else:
            self._sensor_container.add_sensor(sensor=sensor_obj)
            LOG.info('Sensor %s reloaded.', sensor_ref)

    def _handle_delete_sensor(self, sensor):
        LOG.info('Unloading sensor %s.', self._get_sensor_ref(sensor))
        self._sensor_container.remove_sensor(sensor=self._to_sensor_object(sensor))

    def _get_sensor_ref(self, sensor):
        return ResourceReference.to_string_reference(pack=sensor.pack, name=sensor.name)