Exemple #1
0
    def run_sensors(self, sensors):
        """
        :param sensors: A list of DB models of sensors to run.
        :type sensors: ``list``
        """
        LOG.info('Setting up container to run %d sensors.', len(sensors))

        sensors_to_run = []
        for sensor in sensors:
            # TODO: Directly pass DB object to the ProcessContainer
            file_path = sensor.artifact_uri.replace('file://', '')
            class_name = sensor.entry_point.split('.')[-1]

            sensor_obj = {
                'pack': sensor.pack,
                'file_path': file_path,
                'class_name': class_name,
                'trigger_types': sensor.trigger_types,
                'poll_interval': sensor.poll_interval
            }
            sensors_to_run.append(sensor_obj)

        LOG.info('(PID:%s) SensorContainer started.', os.getpid())
        sensor_container = ProcessSensorContainer(sensors=sensors_to_run)
        try:
            exit_code = sensor_container.run()
            LOG.info('(PID:%s) SensorContainer stopped. Reason - run ended.', os.getpid())
            return exit_code
        except (KeyboardInterrupt, SystemExit):
            LOG.info('(PID:%s) SensorContainer stopped. Reason - %s', os.getpid(),
                     sys.exc_info()[0].__name__)
            return 0
    def test_common_lib_path_not_in_pythonpath_env_var(self, mock_create_token, mock_subproc_popen):
        process_mock = Mock()
        attrs = {'communicate.return_value': ('output', 'error')}
        process_mock.configure_mock(**attrs)
        mock_subproc_popen.return_value = process_mock

        mock_create_token = Mock()
        mock_create_token.return_value = 'WHOLETTHEDOGSOUT'

        mock_dispatcher = Mock()
        process_container = ProcessSensorContainer(None, poll_interval=0.1,
                                                   dispatcher=mock_dispatcher)
        sensor = {
            'class_name': 'wolfpack.StupidSensor',
            'ref': 'wolfpack.StupidSensor',
            'id': '567890',
            'trigger_types': ['some_trigga'],
            'pack': 'wolfpack',
            'file_path': '/opt/stackstorm/packs/wolfpack/sensors/stupid_sensor.py',
            'poll_interval': 5
        }

        process_container._enable_common_pack_libs = False
        process_container._sensors = {'pack.StupidSensor': sensor}
        process_container._spawn_sensor_process(sensor)

        _, call_kwargs = mock_subproc_popen.call_args
        actual_env = call_kwargs['env']
        self.assertTrue('PYTHONPATH' in actual_env)
        pack_common_lib_path = '/opt/stackstorm/packs/wolfpack/lib'
        self.assertTrue(pack_common_lib_path not in actual_env['PYTHONPATH'])
    def test_dispatch_triggers_on_spawn_exit(self):
        mock_dispatcher = Mock()
        process_container = ProcessSensorContainer(None, poll_interval=0.1,
                                                   dispatcher=mock_dispatcher)
        sensor = {
            'class_name': 'pack.StupidSensor'
        }
        process = Mock()
        process_attrs = {'pid': 1234}
        process.configure_mock(**process_attrs)
        cmd = 'sensor_wrapper.py --class-name pack.StupidSensor'

        process_container._dispatch_trigger_for_sensor_spawn(sensor, process, cmd)
        mock_dispatcher.dispatch.assert_called_with(
            'core.st2.sensor.process_spawn',
            payload={
                'timestamp': 1439441533,
                'cmd': 'sensor_wrapper.py --class-name pack.StupidSensor',
                'pid': 1234,
                'id': 'pack.StupidSensor'})

        process_container._dispatch_trigger_for_sensor_exit(sensor, 1)
        mock_dispatcher.dispatch.assert_called_with(
            'core.st2.sensor.process_exit',
            payload={
                'id': 'pack.StupidSensor',
                'timestamp': 1439441533,
                'exit_code': 1
            })
 def test_no_sensors_dont_quit(self):
     process_container = ProcessSensorContainer(None, poll_interval=0.1)
     process_container_thread = eventlet.spawn(process_container.run)
     eventlet.sleep(0.5)
     self.assertEqual(process_container.running(), 0)
     self.assertEqual(process_container.stopped, False)
     process_container.shutdown()
     process_container_thread.kill()
Exemple #5
0
    def _spin_container_and_wait(self, sensors):
        exit_code = 0

        try:
            self._sensor_container = ProcessSensorContainer(
                sensors=sensors,
                single_sensor_mode=self._single_sensor_mode)
            self._container_thread = eventlet.spawn(self._sensor_container.run)

            LOG.debug('Starting sensor CUD watcher...')
            self._sensors_watcher.start()

            exit_code = self._container_thread.wait()
            LOG.error('Process container quit with exit_code %d.', exit_code)
            LOG.error('(PID:%s) SensorContainer stopped.', os.getpid())
        except (KeyboardInterrupt, SystemExit):
            self._sensor_container.shutdown()
            self._sensors_watcher.stop()

            LOG.info('(PID:%s) SensorContainer stopped. Reason - %s', os.getpid(),
                     sys.exc_info()[0].__name__)

            eventlet.kill(self._container_thread)
            self._container_thread = None

            return exit_code

        return exit_code
Exemple #6
0
class SensorContainerManager(object):
    def __init__(self, sensors_partitioner):
        self._sensor_container = None
        self._sensors_watcher = SensorWatcher(
            create_handler=self._handle_create_sensor,
            update_handler=self._handle_update_sensor,
            delete_handler=self._handle_delete_sensor,
            queue_suffix="sensor_container",
        )
        self._container_thread = None
        if not sensors_partitioner:
            raise ValueError("sensors_partitioner should be non-None.")
        self._sensors_partitioner = sensors_partitioner

    def run_sensors(self):
        """
        Run all sensors as determined by sensors_partitioner.
        """
        sensors = self._sensors_partitioner.get_sensors()
        if sensors:
            LOG.info("Setting up container to run %d sensors.", len(sensors))
            LOG.info("\tSensors list - %s.", [self._get_sensor_ref(sensor) for sensor in sensors])

        sensors_to_run = []
        for sensor in sensors:
            # TODO: Directly pass DB object to the ProcessContainer
            sensors_to_run.append(self._to_sensor_object(sensor))

        LOG.info("(PID:%s) SensorContainer started.", os.getpid())
        self._setup_sigterm_handler()
        self._spin_container_and_wait(sensors_to_run)

    def _spin_container_and_wait(self, sensors):
        try:
            self._sensor_container = ProcessSensorContainer(sensors=sensors)
            self._container_thread = eventlet.spawn(self._sensor_container.run)
            LOG.debug("Starting sensor CUD watcher...")
            self._sensors_watcher.start()
            exit_code = self._container_thread.wait()
            LOG.error("Process container quit with exit_code %d.", exit_code)
            LOG.error("(PID:%s) SensorContainer stopped.", os.getpid())
        except (KeyboardInterrupt, SystemExit):
            self._sensor_container.shutdown()
            self._sensors_watcher.stop()

            LOG.info("(PID:%s) SensorContainer stopped. Reason - %s", os.getpid(), sys.exc_info()[0].__name__)

            eventlet.kill(self._container_thread)
            self._container_thread = None

            return 0

    def _setup_sigterm_handler(self):
        def sigterm_handler(signum=None, frame=None):
            # This will cause SystemExit to be throw and we call sensor_container.shutdown()
            # there which cleans things up.
            sys.exit(0)

        # Register a SIGTERM signal handler which calls sys.exit which causes SystemExit to
        # be thrown. We catch SystemExit and handle cleanup there.
        signal.signal(signal.SIGTERM, sigterm_handler)

    def _to_sensor_object(self, sensor_db):
        file_path = sensor_db.artifact_uri.replace("file://", "")
        class_name = sensor_db.entry_point.split(".")[-1]

        sensor_obj = {
            "pack": sensor_db.pack,
            "file_path": file_path,
            "class_name": class_name,
            "trigger_types": sensor_db.trigger_types,
            "poll_interval": sensor_db.poll_interval,
            "ref": self._get_sensor_ref(sensor_db),
        }

        return sensor_obj

    #################################################
    # Event handler methods for the sensor CUD events
    #################################################

    def _handle_create_sensor(self, sensor):
        if not self._sensors_partitioner.is_sensor_owner(sensor):
            LOG.info("sensor %s is not supported. Ignoring create.", self._get_sensor_ref(sensor))
            return
        if not sensor.enabled:
            LOG.info("sensor %s is not enabled.", self._get_sensor_ref(sensor))
            return
        LOG.info("Adding sensor %s.", self._get_sensor_ref(sensor))
        self._sensor_container.add_sensor(sensor=self._to_sensor_object(sensor))

    def _handle_update_sensor(self, sensor):
        if not self._sensors_partitioner.is_sensor_owner(sensor):
            LOG.info("sensor %s is not supported. Ignoring update.", self._get_sensor_ref(sensor))
            return
        sensor_ref = self._get_sensor_ref(sensor)
        LOG.info("Sensor %s updated. Reloading sensor.", sensor_ref)
        sensor_obj = self._to_sensor_object(sensor)
        try:
            self._sensor_container.remove_sensor(sensor=sensor_obj)
        except:
            LOG.exception("Failed to reload sensor %s", sensor_ref)
        else:
            self._sensor_container.add_sensor(sensor=sensor_obj)
            LOG.info("Sensor %s reloaded.", sensor_ref)

    def _handle_delete_sensor(self, sensor):
        if not self._sensors_partitioner.is_sensor_owner(sensor):
            LOG.info("sensor %s is not supported. Ignoring delete.", self._get_sensor_ref(sensor))
            return
        LOG.info("Unloading sensor %s.", self._get_sensor_ref(sensor))
        self._sensor_container.remove_sensor(sensor=self._to_sensor_object(sensor))

    def _get_sensor_ref(self, sensor):
        return ResourceReference.to_string_reference(pack=sensor.pack, name=sensor.name)
Exemple #7
0
class SensorContainerManager(object):
    # TODO: Load balancing for sensors.
    def __init__(self, max_containers=10):
        self._max_containers = max_containers
        self._sensor_container = None
        self._sensors_watcher = SensorWatcher(create_handler=self._handle_create_sensor,
                                              update_handler=self._handle_update_sensor,
                                              delete_handler=self._handle_delete_sensor,
                                              queue_suffix='sensor_container')
        self._container_thread = None

    def run_sensors(self, sensors):
        """
        :param sensors: A list of DB models of sensors to run.
        :type sensors: ``list``
        """
        if sensors:
            LOG.info('Setting up container to run %d sensors.', len(sensors))

        sensors_to_run = []
        for sensor in sensors:
            # TODO: Directly pass DB object to the ProcessContainer
            sensors_to_run.append(self._to_sensor_object(sensor))

        LOG.info('(PID:%s) SensorContainer started.', os.getpid())
        self._setup_sigterm_handler()
        self._spin_container_and_wait(sensors_to_run)

    def _spin_container_and_wait(self, sensors):
        try:
            self._sensor_container = ProcessSensorContainer(sensors=sensors)
            self._container_thread = eventlet.spawn(self._sensor_container.run)
            LOG.debug('Starting sensor CUD watcher...')
            self._sensors_watcher.start()
            exit_code = self._container_thread.wait()
            LOG.error('Process container quit with exit_code %d.', exit_code)
            LOG.error('(PID:%s) SensorContainer stopped.', os.getpid())
        except (KeyboardInterrupt, SystemExit):
            self._sensor_container.shutdown()
            self._sensors_watcher.stop()

            LOG.info('(PID:%s) SensorContainer stopped. Reason - %s', os.getpid(),
                     sys.exc_info()[0].__name__)

            self._container_thread = eventlet.kill(self._container_thread)

            return 0

    def _setup_sigterm_handler(self):

        def sigterm_handler(signum=None, frame=None):
            # This will cause SystemExit to be throw and we call sensor_container.shutdown()
            # there which cleans things up.
            sys.exit(0)

        # Register a SIGTERM signal handler which calls sys.exit which causes SystemExit to
        # be thrown. We catch SystemExit and handle cleanup there.
        signal.signal(signal.SIGTERM, sigterm_handler)

    def _to_sensor_object(self, sensor_db):
        file_path = sensor_db.artifact_uri.replace('file://', '')
        class_name = sensor_db.entry_point.split('.')[-1]

        sensor_obj = {
            'pack': sensor_db.pack,
            'file_path': file_path,
            'class_name': class_name,
            'trigger_types': sensor_db.trigger_types,
            'poll_interval': sensor_db.poll_interval,
            'ref': self._get_sensor_ref(sensor_db)
        }

        return sensor_obj

    #################################################
    # Event handler methods for the sensor CUD events
    #################################################

    def _handle_create_sensor(self, sensor):
        LOG.info('Adding sensor %s.', self._get_sensor_ref(sensor))
        self._sensor_container.add_sensor(sensor=self._to_sensor_object(sensor))

    def _handle_update_sensor(self, sensor):
        sensor_ref = self._get_sensor_ref(sensor)
        LOG.info('Sensor %s updated. Reloading sensor.', sensor_ref)
        sensor_obj = self._to_sensor_object(sensor)
        try:
            self._sensor_container.remove_sensor(sensor=sensor_obj)
        except:
            LOG.exception('Failed to reload sensor %s', sensor_ref)
        else:
            self._sensor_container.add_sensor(sensor=sensor_obj)
            LOG.info('Sensor %s reloaded.', sensor_ref)

    def _handle_delete_sensor(self, sensor):
        LOG.info('Unloading sensor %s.', self._get_sensor_ref(sensor))
        self._sensor_container.remove_sensor(sensor=self._to_sensor_object(sensor))

    def _get_sensor_ref(self, sensor):
        return ResourceReference.to_string_reference(pack=sensor.pack, name=sensor.name)