def _get_action_instance(self): actions_cls = action_loader.register_plugin(Action, self._file_path) action_cls = actions_cls[0] if actions_cls and len( actions_cls) > 0 else None if not action_cls: raise Exception( 'File "%s" has no action or the file doesn\'t exist.' % (self._file_path)) config_parser = ContentPackConfigParser(pack_name=self._pack) config = config_parser.get_action_config( action_file_path=self._file_path) if config: LOG.info('Using config "%s" for action "%s"' % (config.file_path, self._file_path)) action_instance = action_cls(config=config.config) else: LOG.info('No config found for action "%s"' % (self._file_path)) action_instance = action_cls(config={}) # Setup action_instance proeprties action_instance.logger = self._set_up_logger(action_cls.__name__) action_instance.datastore = DatastoreService( logger=action_instance.logger, pack_name=self._pack, class_name=action_cls.__name__, api_username="******") return action_instance
def _get_action_instance(self): actions_cls = action_loader.register_plugin(Action, self._file_path) action_cls = actions_cls[0] if actions_cls and len(actions_cls) > 0 else None if not action_cls: raise Exception('File "%s" has no action or the file doesn\'t exist.' % (self._file_path)) config_parser = ContentPackConfigParser(pack_name=self._pack) config = config_parser.get_action_config(action_file_path=self._file_path) if config: LOG.info('Using config "%s" for action "%s"' % (config.file_path, self._file_path)) action_instance = action_cls(config=config.config) else: LOG.info('No config found for action "%s"' % (self._file_path)) action_instance = action_cls(config={}) # Setup action_instance proeprties action_instance.logger = self._set_up_logger(action_cls.__name__) action_instance.datastore = DatastoreService(logger=action_instance.logger, pack_name=self._pack, class_name=action_cls.__name__, api_username="******") return action_instance
def test_get_config_existing_config(self): pack_name = "dummy_pack_2" parser = ContentPackConfigParser(pack_name=pack_name) config = parser.get_config() self.assertEqual(config.config["section1"]["key1"], "value1") self.assertEqual(config.config["section2"]["key10"], "value10")
def _get_action_instance(self): actions_cls = action_loader.register_plugin(Action, self._file_path) action_cls = actions_cls[0] if actions_cls and len( actions_cls) > 0 else None if not action_cls: raise Exception( 'File "%s" has no action or the file doesn\'t exist.' % (self._file_path)) config_parser = ContentPackConfigParser(pack_name=self._pack) config = config_parser.get_action_config( action_file_path=self._file_path) if config: LOG.info('Using config "%s" for action "%s"' % (config.file_path, self._file_path)) config = config.config else: LOG.info('No config found for action "%s"' % (self._file_path)) config = None action_service = ActionService(action_wrapper=self) action_instance = get_action_class_instance( action_cls=action_cls, config=config, action_service=action_service) return action_instance
def test_get_config_existing_config(self): pack_name = 'dummy_pack_2' parser = ContentPackConfigParser(pack_name=pack_name) config = parser.get_config() self.assertEqual(config.config['section1']['key1'], 'value1') self.assertEqual(config.config['section2']['key10'], 'value10')
def test_get_action_and_sensor_config_no_config(self): pack_name = 'dummy_pack_1' parser = ContentPackConfigParser(pack_name=pack_name) config = parser.get_action_config(action_file_path='my_action.py') self.assertEqual(config, None) config = parser.get_sensor_config(sensor_file_path='my_sensor.py') self.assertEqual(config, None)
def test_get_action_and_sensor_config_existing_config(self): pack_name = 'dummy_pack_2' parser = ContentPackConfigParser(pack_name=pack_name) config = parser.get_action_config(action_file_path='my_action.py') self.assertEqual(config.config['section1']['key1'], 'value1') self.assertEqual(config.config['section2']['key10'], 'value10') config = parser.get_sensor_config(sensor_file_path='my_sensor.py') self.assertEqual(config.config['section1']['key1'], 'value1') self.assertEqual(config.config['section2']['key10'], 'value10')
def _get_sensor_config(self): config_parser = ContentPackConfigParser(pack_name=self._pack) config = config_parser.get_sensor_config(sensor_file_path=self._file_path) if config: self._logger.info('Using config "%s" for sensor "%s"' % (config.file_path, self._class_name)) return config.config else: self._logger.info('No config found for sensor "%s"' % (self._class_name)) return {}
def _load_action(self): actions_kls = action_loader.register_plugin(Action, self.entry_point) action_kls = actions_kls[0] if actions_kls and len(actions_kls) > 0 else None if not action_kls: raise Exception('%s has no action.' % self.entry_point) config_parser = ContentPackConfigParser(pack_name=self.pack) config = config_parser.get_action_config(action_file_path=self.entry_point) if config: LOG.info('Using config "%s" for action "%s"' % (config.file_path, self.entry_point)) return action_kls(config=config.config) else: LOG.info('No config found for action "%s"' % (self.entry_point)) return action_kls(config={})
def _get_action_instance(self): actions_cls = action_loader.register_plugin(Action, self._file_path) action_cls = actions_cls[0] if actions_cls and len(actions_cls) > 0 else None if not action_cls: raise Exception('File "%s" has no action or the file doesn\'t exist.' % (self._file_path)) config_parser = ContentPackConfigParser(pack_name=self._pack) config = config_parser.get_action_config(action_file_path=self._file_path) if config: LOG.info('Using config "%s" for action "%s"' % (config.file_path, self._file_path)) return action_cls(config=config.config) else: LOG.info('No config found for action "%s"' % (self._file_path)) return action_cls(config={})
def __init__(self, pack_name, user=None): self.pack_name = pack_name self.user = user or cfg.CONF.system_user.user self.pack_path = content_utils.get_pack_base_path(pack_name=pack_name) self._config_parser = ContentPackConfigParser(pack_name=pack_name)
class ContentPackConfigLoader(object): """ Class which loads and resolves all the config values and returns a dictionary of resolved values which can be passed to the resource. It loads and resolves values in the following order: 1. Static values from <pack path>/config.yaml file 2. Dynamic and or static values from /opt/stackstorm/configs/<pack name>.yaml file. Values are merged from left to right which means values from "<pack name>.yaml" file have precedence and override values from pack local config file. """ def __init__(self, pack_name, user=None): self.pack_name = pack_name self.user = user or cfg.CONF.system_user.user self.pack_path = content_utils.get_pack_base_path(pack_name=pack_name) self._config_parser = ContentPackConfigParser(pack_name=pack_name) def get_config(self): result = {} # 1. Retrieve values from pack local config.yaml file config = self._config_parser.get_config() if config: config = config.config or {} result.update(config) # Retrieve corresponding ConfigDB and ConfigSchemaDB object # Note: ConfigSchemaDB is optional right now. If it doesn't exist, we assume every value # is of a type string try: config_db = Config.get_by_pack(value=self.pack_name) except StackStormDBObjectNotFoundError: # Corresponding pack config doesn't exist. We set config_db to an empty config so # that the default values from config schema are still correctly applied even if # pack doesn't contain a config. config_db = ConfigDB(pack=self.pack_name, values={}) try: config_schema_db = ConfigSchema.get_by_pack(value=self.pack_name) except StackStormDBObjectNotFoundError: config_schema_db = None # 2. Retrieve values from "global" pack config file (if available) and resolve them if # necessary config = self._get_values_for_config(config_schema_db=config_schema_db, config_db=config_db) result.update(config) return result def _get_values_for_config(self, config_schema_db, config_db): schema_values = getattr(config_schema_db, 'attributes', {}) config_values = getattr(config_db, 'values', {}) config = copy.deepcopy(config_values) # Assign dynamic config values based on the values in the datastore config = self._assign_dynamic_config_values(schema=schema_values, config=config) # If config_schema is available we do a second pass and set default values for required # items which values are not provided / available in the config itself config = self._assign_default_values(schema=schema_values, config=config) return config def _assign_dynamic_config_values(self, schema, config, parent_keys=None): """ Assign dynamic config value for a particular config item if the ite utilizes a Jinja expression for dynamic config values. Note: This method mutates config argument in place. :rtype: ``dict`` """ parent_keys = parent_keys or [] for config_item_key, config_item_value in six.iteritems(config): schema_item = schema.get(config_item_key, {}) is_dictionary = isinstance(config_item_value, dict) # Inspect nested object properties if is_dictionary: parent_keys += [config_item_key] self._assign_dynamic_config_values( schema=schema_item.get('properties', {}), config=config[config_item_key], parent_keys=parent_keys) else: is_jinja_expression = jinja_utils.is_jinja_expression( value=config_item_value) if is_jinja_expression: # Resolve / render the Jinja template expression full_config_item_key = '.'.join(parent_keys + [config_item_key]) value = self._get_datastore_value_for_expression( key=full_config_item_key, value=config_item_value, config_schema_item=schema_item) config[config_item_key] = value else: # Static value, no resolution needed config[config_item_key] = config_item_value return config def _assign_default_values(self, schema, config): """ Assign default values for particular config if default values are provided in the config schema and a value is not specified in the config. Note: This method mutates config argument in place. :rtype: ``dict`` """ for schema_item_key, schema_item in six.iteritems(schema): default_value = schema_item.get('default', None) is_required = schema_item.get('required', False) is_object = schema_item.get('type', None) == 'object' has_properties = schema_item.get('properties', None) if is_required and default_value and not config.get( schema_item_key, None): config[schema_item_key] = default_value # Inspect nested object properties if is_object and has_properties: if not config.get(schema_item_key, None): config[schema_item_key] = {} self._assign_default_values(schema=schema_item['properties'], config=config[schema_item_key]) return config def _get_datastore_value_for_expression(self, key, value, config_schema_item=None): """ Retrieve datastore value by first resolving the datastore expression and then retrieving the value from the datastore. :param key: Full path to the config item key (e.g. "token" / "auth.settings.token", etc.) """ from st2common.services.config import deserialize_key_value config_schema_item = config_schema_item or {} secret = config_schema_item.get('secret', False) try: value = render_template_with_system_and_user_context( value=value, user=self.user) except Exception as e: # Throw a more user-friendly exception on failed render exc_class = type(e) original_msg = str(e) msg = ( 'Failed to render dynamic configuration value for key "%s" with value ' '"%s" for pack "%s" config: %s ' % (key, value, self.pack_name, original_msg)) raise exc_class(msg) if value: # Deserialize the value value = deserialize_key_value(value=value, secret=secret) else: value = None return value
class ContentPackConfigLoader(object): """ Class which loads and resolves all the config values and returns a dictionary of resolved values which can be passed to the resource. It loads and resolves values in the following order: 1. Static values from <pack path>/config.yaml file 2. Dynamic and or static values from /opt/stackstorm/configs/<pack name>.yaml file. Values are merged from left to right which means values from "<pack name>.yaml" file have precedence and override values from pack local config file. """ def __init__(self, pack_name, user=None): self.pack_name = pack_name self.user = user or cfg.CONF.system_user.user self.pack_path = content_utils.get_pack_base_path(pack_name=pack_name) self._config_parser = ContentPackConfigParser(pack_name=pack_name) def get_config(self): result = {} # 1. Retrieve values from pack local config.yaml file config = self._config_parser.get_config() if config: config = config.config or {} result.update(config) # Retrieve corresponding ConfigDB and ConfigSchemaDB object # Note: ConfigSchemaDB is optional right now. If it doesn't exist, we assume every value # is of a type string try: config_db = Config.get_by_pack(value=self.pack_name) except StackStormDBObjectNotFoundError: # Corresponding pack config doesn't exist, return early return result try: config_schema_db = ConfigSchema.get_by_pack(value=self.pack_name) except StackStormDBObjectNotFoundError: config_schema_db = None # 2. Retrieve values from "global" pack config file (if available) and resolve them if # necessary config = self._get_values_for_config(config_schema_db=config_schema_db, config_db=config_db) result.update(config) return result def _get_values_for_config(self, config_schema_db, config_db): schema_values = getattr(config_schema_db, 'attributes', {}) result = {} for config_item_key, config_item_value in six.iteritems(config_db.values): is_jinja_expression = jinja_utils.is_jinja_expression(value=config_item_value) if is_jinja_expression: config_schema_item = schema_values.get(config_item_key, {}) value = self._get_datastore_value_for_expression(value=config_item_value, config_schema_item=config_schema_item) result[config_item_key] = value else: # Static value, no resolution needed result[config_item_key] = config_item_value return result def _get_datastore_value_for_expression(self, value, config_schema_item=None): """ Retrieve datastore value by first resolving the datastore expression and then retrieving the value from the datastore. """ config_schema_item = config_schema_item or {} secret = config_schema_item.get('secret', False) # TODO: Get key name so we can throw a more friendly exception value = render_template_with_system_and_user_context(value=value, user=self.user) if value: # Deserialize the value value = deserialize_key_value(value=value, secret=secret) else: value = None return value
class ContentPackConfigLoader(object): """ Class which loads and resolves all the config values and returns a dictionary of resolved values which can be passed to the resource. It loads and resolves values in the following order: 1. Static values from <pack path>/config.yaml file 2. Dynamic and or static values from /opt/stackstorm/configs/<pack name>.yaml file. Values are merged from left to right which means values from "<pack name>.yaml" file have precedence and override values from pack local config file. """ def __init__(self, pack_name, user=None): self.pack_name = pack_name self.user = user or cfg.CONF.system_user.user self.pack_path = content_utils.get_pack_base_path(pack_name=pack_name) self._config_parser = ContentPackConfigParser(pack_name=pack_name) def get_config(self): result = {} # 1. Retrieve values from pack local config.yaml file config = self._config_parser.get_config() if config: config = config.config or {} result.update(config) # Retrieve corresponding ConfigDB and ConfigSchemaDB object # Note: ConfigSchemaDB is optional right now. If it doesn't exist, we assume every value # is of a type string try: config_db = Config.get_by_pack(value=self.pack_name) except StackStormDBObjectNotFoundError: # Corresponding pack config doesn't exist, return early return result try: config_schema_db = ConfigSchema.get_by_pack(value=self.pack_name) except StackStormDBObjectNotFoundError: config_schema_db = None # 2. Retrieve values from "global" pack config file (if available) and resolve them if # necessary config = self._get_values_for_config(config_schema_db=config_schema_db, config_db=config_db) result.update(config) return result def _get_values_for_config(self, config_schema_db, config_db): schema_values = getattr(config_schema_db, 'attributes', {}) result = {} for config_item_key, config_item_value in six.iteritems( config_db.values): is_jinja_expression = jinja_utils.is_jinja_expression( value=config_item_value) if is_jinja_expression: config_schema_item = schema_values.get(config_item_key, {}) value = self._get_datastore_value_for_expression( value=config_item_value, config_schema_item=config_schema_item) result[config_item_key] = value else: # Static value, no resolution needed result[config_item_key] = config_item_value return result def _get_datastore_value_for_expression(self, value, config_schema_item=None): """ Retrieve datastore value by first resolving the datastore expression and then retrieving the value from the datastore. """ config_schema_item = config_schema_item or {} secret = config_schema_item.get('secret', False) # TODO: Get key name so we can throw a more friendly exception value = render_template_with_system_and_user_context(value=value, user=self.user) if value: # Deserialize the value value = deserialize_key_value(value=value, secret=secret) else: value = None return value
def run_sensors(self, sensors_dict): LOG.info('Setting up container to run %d sensors.', len(sensors_dict)) container_service = ContainerService() sensors_to_run = [] # TODO: Once the API registration is in place, query DB for available # sensors here # TODO: Use trigger_types and description from sensors metadata for filename, sensors in six.iteritems(sensors_dict): for sensor_class in sensors: sensor_class_kwargs = {} class_name = sensor_class.__name__ # System sensors which are not located inside a content pack # don't and can't have custom config associated with them pack = getattr(sensor_class, 'pack', None) if pack: # TODO: Don't parse the same config multiple times when we # are referring to sensors from the same pack pack = validate_pack_name(name=pack) config_parser = ContentPackConfigParser(pack_name=pack) config = config_parser.get_sensor_config(sensor_file_path=filename) if config: sensor_class_kwargs['config'] = config.config LOG.info('Using config "%s" for sensor "%s"' % (config.file_path, class_name)) else: LOG.info('No config found for sensor "%s"' % (class_name)) sensor_class_kwargs['config'] = {} else: pack = SYSTEM_PACK_NAME try: sensor = sensor_class(container_service=container_service, **sensor_class_kwargs) except Exception as e: LOG.warning('Unable to create instance for sensor %s in file %s. Exception: %s', sensor_class, filename, e, exc_info=True) continue try: trigger_types = sensor.get_trigger_types() if not trigger_types: trigger_type_dbs = [] LOG.warning('No trigger type registered by sensor %s in file %s', sensor_class, filename) else: assert isinstance(trigger_types, (list, tuple)) trigger_type_dbs = container_utils.add_trigger_models( pack=pack, trigger_types=trigger_types) except TriggerTypeRegistrationException as e: LOG.warning('Unable to register trigger type for sensor %s in file %s.' + ' Exception: %s', sensor_class, filename, e, exc_info=True) continue # Populate sensors dict trigger_type_refs = [] for trigger_type_db, _ in trigger_type_dbs: ref_obj = trigger_type_db.get_reference() trigger_type_ref = ref_obj.ref self._trigger_sensors[trigger_type_ref] = sensor trigger_type_refs.append(trigger_type_ref) # Register sensor type in the DB sensor_obj = { 'filename': os.path.abspath(filename), 'name': class_name, 'class_name': class_name, 'trigger_types': trigger_type_refs } container_utils.add_sensor_model(pack=pack, sensor=sensor_obj) # Add good sensor to the run list sensors_to_run.append(sensor) for trigger in Trigger.get_all(): self._create_handler(trigger=trigger) self._trigger_watcher.start() LOG.info('Watcher started.') LOG.info('(PID:%s) SensorContainer started.', os.getpid()) sensor_container = SensorContainer(sensor_instances=sensors_to_run) try: exit_code = sensor_container.run() LOG.info('(PID:%s) SensorContainer stopped. Reason - run ended.', os.getpid()) return exit_code except (KeyboardInterrupt, SystemExit): LOG.info('(PID:%s) SensorContainer stopped. Reason - %s', os.getpid(), sys.exc_info()[0].__name__) return 0 finally: self._trigger_watcher.stop()
def test_get_config_no_config(self): pack_name = 'dummy_pack_1' parser = ContentPackConfigParser(pack_name=pack_name) config = parser.get_config() self.assertEqual(config, None)
class ContentPackConfigLoader(object): """ Class which loads and resolves all the config values and returns a dictionary of resolved values which can be passed to the resource. It loads and resolves values in the following order: 1. Static values from <pack path>/config.yaml file 2. Dynamic and or static values from /opt/stackstorm/configs/<pack name>.yaml file. Values are merged from left to right which means values from "<pack name>.yaml" file have precedence and override values from pack local config file. """ def __init__(self, pack_name, user=None): self.pack_name = pack_name self.user = user or cfg.CONF.system_user.user self.pack_path = content_utils.get_pack_base_path(pack_name=pack_name) self._config_parser = ContentPackConfigParser(pack_name=pack_name) def get_config(self): result = {} # 1. Retrieve values from pack local config.yaml file config = self._config_parser.get_config() if config: config = config.config or {} result.update(config) # Retrieve corresponding ConfigDB and ConfigSchemaDB object # Note: ConfigSchemaDB is optional right now. If it doesn't exist, we assume every value # is of a type string try: config_db = Config.get_by_pack(value=self.pack_name) except StackStormDBObjectNotFoundError: # Corresponding pack config doesn't exist, return early return result try: config_schema_db = ConfigSchema.get_by_pack(value=self.pack_name) except StackStormDBObjectNotFoundError: config_schema_db = None # 2. Retrieve values from "global" pack config file (if available) and resolve them if # necessary config = self._get_values_for_config(config_schema_db=config_schema_db, config_db=config_db) result.update(config) return result def _get_values_for_config(self, config_schema_db, config_db): schema_values = getattr(config_schema_db, 'attributes', {}) config_values = getattr(config_db, 'values', {}) config = copy.deepcopy(config_values) # Assign dynamic config values based on the values in the datastore config = self._assign_dynamic_config_values(schema=schema_values, config=config) # If config_schema is available we do a second pass and set default values for required # items which values are not provided / available in the config itself config = self._assign_default_values(schema=schema_values, config=config) return config def _assign_dynamic_config_values(self, schema, config, parent_keys=None): """ Assign dynamic config value for a particular config item if the ite utilizes a Jinja expression for dynamic config values. Note: This method mutates config argument in place. :rtype: ``dict`` """ parent_keys = parent_keys or [] for config_item_key, config_item_value in six.iteritems(config): schema_item = schema.get(config_item_key, {}) is_dictionary = isinstance(config_item_value, dict) # Inspect nested object properties if is_dictionary: parent_keys += [config_item_key] self._assign_dynamic_config_values(schema=schema_item.get('properties', {}), config=config[config_item_key], parent_keys=parent_keys) else: is_jinja_expression = jinja_utils.is_jinja_expression(value=config_item_value) if is_jinja_expression: # Resolve / render the Jinja template expression full_config_item_key = '.'.join(parent_keys + [config_item_key]) value = self._get_datastore_value_for_expression(key=full_config_item_key, value=config_item_value, config_schema_item=schema_item) config[config_item_key] = value else: # Static value, no resolution needed config[config_item_key] = config_item_value return config def _assign_default_values(self, schema, config): """ Assign default values for particular config if default values are provided in the config schema and a value is not specified in the config. Note: This method mutates config argument in place. :rtype: ``dict`` """ for schema_item_key, schema_item in six.iteritems(schema): default_value = schema_item.get('default', None) is_required = schema_item.get('required', False) is_object = schema_item.get('type', None) == 'object' has_properties = schema_item.get('properties', None) if is_required and default_value and not config.get(schema_item_key, None): config[schema_item_key] = default_value # Inspect nested object properties if is_object and has_properties: if not config.get(schema_item_key, None): config[schema_item_key] = {} self._assign_default_values(schema=schema_item['properties'], config=config[schema_item_key]) return config def _get_datastore_value_for_expression(self, key, value, config_schema_item=None): """ Retrieve datastore value by first resolving the datastore expression and then retrieving the value from the datastore. :param key: Full path to the config item key (e.g. "token" / "auth.settings.token", etc.) """ from st2common.services.config import deserialize_key_value config_schema_item = config_schema_item or {} secret = config_schema_item.get('secret', False) try: value = render_template_with_system_and_user_context(value=value, user=self.user) except Exception as e: # Throw a more user-friendly exception on failed render exc_class = type(e) original_msg = str(e) msg = ('Failed to render dynamic configuration value for key "%s" with value ' '"%s" for pack "%s" config: %s ' % (key, value, self.pack_name, original_msg)) raise exc_class(msg) if value: # Deserialize the value value = deserialize_key_value(value=value, secret=secret) else: value = None return value
def test_get_config_inexistent_pack(self): parser = ContentPackConfigParser(pack_name='inexistent') config = parser.get_config() self.assertEqual(config, None)
def test_get_config_for_unicode_char(self): pack_name = 'dummy_pack_18' parser = ContentPackConfigParser(pack_name=pack_name) config = parser.get_config() self.assertEqual(config.config['section1']['key1'], u'测试')
def test_get_config_for_unicode_char(self): pack_name = "dummy_pack_18" parser = ContentPackConfigParser(pack_name=pack_name) config = parser.get_config() self.assertEqual(config.config["section1"]["key1"], "测试")
def test_get_action_config_inexistent_pack(self): parser = ContentPackConfigParser(pack_name='inexistent') config = parser.get_action_config(action_file_path='test.py') self.assertEqual(config, None)