def _register_pack_config_schema_db(self, pack_name, pack_dir): config_schema_path = os.path.join(pack_dir, CONFIG_SCHEMA_FILE_NAME) if not os.path.isfile(config_schema_path): # Note: Config schema is optional return None values = self._meta_loader.load(config_schema_path) if not values: raise ValueError('Config schema "%s" is empty and invalid.' % (config_schema_path)) content = {} content['pack'] = pack_name content['attributes'] = values config_schema_api = ConfigSchemaAPI(**content) config_schema_api = config_schema_api.validate() config_schema_db = ConfigSchemaAPI.to_model(config_schema_api) try: config_schema_db.id = ConfigSchema.get_by_pack(pack_name).id except StackStormDBObjectNotFoundError: LOG.debug('Config schema for pack %s not found. Creating new one.', pack_name) config_schema_db = ConfigSchema.add_or_update(config_schema_db) LOG.debug('Config schema for pack %s registered.' % (pack_name)) return config_schema_db
def test_register_packs(self): # Verify DB is empty pack_dbs = Pack.get_all() config_schema_dbs = ConfigSchema.get_all() self.assertEqual(len(pack_dbs), 0) self.assertEqual(len(config_schema_dbs), 0) registrar = ResourceRegistrar(use_pack_cache=False) registrar._pack_loader.get_packs = mock.Mock() registrar._pack_loader.get_packs.return_value = { 'dummy_pack_1': PACK_PATH } packs_base_paths = content_utils.get_packs_base_paths() registrar.register_packs(base_dirs=packs_base_paths) # Verify pack and schema have been registered pack_dbs = Pack.get_all() config_schema_dbs = ConfigSchema.get_all() self.assertEqual(len(pack_dbs), 1) self.assertEqual(len(config_schema_dbs), 1) self.assertEqual(pack_dbs[0].name, 'dummy_pack_1') self.assertTrue('api_key' in config_schema_dbs[0].attributes) self.assertTrue('api_secret' in config_schema_dbs[0].attributes)
def test_register_packs(self): # Verify DB is empty pack_dbs = Pack.get_all() config_schema_dbs = ConfigSchema.get_all() self.assertEqual(len(pack_dbs), 0) self.assertEqual(len(config_schema_dbs), 0) registrar = ResourceRegistrar(use_pack_cache=False) registrar._pack_loader.get_packs = mock.Mock() registrar._pack_loader.get_packs.return_value = {'dummy_pack_1': PACK_PATH_1} packs_base_paths = content_utils.get_packs_base_paths() registrar.register_packs(base_dirs=packs_base_paths) # Verify pack and schema have been registered pack_dbs = Pack.get_all() config_schema_dbs = ConfigSchema.get_all() self.assertEqual(len(pack_dbs), 1) self.assertEqual(len(config_schema_dbs), 1) self.assertEqual(pack_dbs[0].name, 'dummy_pack_1') self.assertEqual(len(pack_dbs[0].contributors), 2) self.assertEqual(pack_dbs[0].contributors[0], 'John Doe1 <*****@*****.**>') self.assertEqual(pack_dbs[0].contributors[1], 'John Doe2 <*****@*****.**>') self.assertTrue('api_key' in config_schema_dbs[0].attributes) self.assertTrue('api_secret' in config_schema_dbs[0].attributes)
def _delete_config_schema_db_object(self, pack): try: config_schema_db = ConfigSchema.get_by_pack(value=pack) except StackStormDBObjectNotFoundError: self.logger.exception('ConfigSchemaDB object not found') return try: ConfigSchema.delete(config_schema_db) except: self.logger.exception('Failed to remove DB object %s.', config_schema_db)
def test_run(self): pack = 'dummy_pack_1' # Verify all the resources are there pack_dbs = Pack.query(ref=pack) action_dbs = Action.query(pack=pack) alias_dbs = ActionAlias.query(pack=pack) rule_dbs = Rule.query(pack=pack) sensor_dbs = Sensor.query(pack=pack) trigger_type_dbs = TriggerType.query(pack=pack) policy_dbs = Policy.query(pack=pack) config_schema_dbs = ConfigSchema.query(pack=pack) config_dbs = Config.query(pack=pack) self.assertEqual(len(pack_dbs), 1) self.assertEqual(len(action_dbs), 1) self.assertEqual(len(alias_dbs), 2) self.assertEqual(len(rule_dbs), 1) self.assertEqual(len(sensor_dbs), 3) self.assertEqual(len(trigger_type_dbs), 4) self.assertEqual(len(policy_dbs), 2) self.assertEqual(len(config_schema_dbs), 1) self.assertEqual(len(config_dbs), 1) # Run action action = self.get_action_instance() action.run(packs=[pack]) # Make sure all resources have been removed from the db pack_dbs = Pack.query(ref=pack) action_dbs = Action.query(pack=pack) alias_dbs = ActionAlias.query(pack=pack) rule_dbs = Rule.query(pack=pack) sensor_dbs = Sensor.query(pack=pack) trigger_type_dbs = TriggerType.query(pack=pack) policy_dbs = Policy.query(pack=pack) config_schema_dbs = ConfigSchema.query(pack=pack) config_dbs = Config.query(pack=pack) self.assertEqual(len(pack_dbs), 0) self.assertEqual(len(action_dbs), 0) self.assertEqual(len(alias_dbs), 0) self.assertEqual(len(rule_dbs), 0) self.assertEqual(len(sensor_dbs), 0) self.assertEqual(len(trigger_type_dbs), 0) self.assertEqual(len(policy_dbs), 0) self.assertEqual(len(config_schema_dbs), 0) self.assertEqual(len(config_dbs), 0)
def test_run(self): pack = 'dummy_pack_1' # Verify all the resources are there pack_dbs = Pack.query(ref=pack) action_dbs = Action.query(pack=pack) alias_dbs = ActionAlias.query(pack=pack) rule_dbs = Rule.query(pack=pack) sensor_dbs = Sensor.query(pack=pack) trigger_type_dbs = TriggerType.query(pack=pack) policy_dbs = Policy.query(pack=pack) config_schema_dbs = ConfigSchema.query(pack=pack) config_dbs = Config.query(pack=pack) self.assertEqual(len(pack_dbs), 1) self.assertEqual(len(action_dbs), 1) self.assertEqual(len(alias_dbs), 3) self.assertEqual(len(rule_dbs), 1) self.assertEqual(len(sensor_dbs), 3) self.assertEqual(len(trigger_type_dbs), 4) self.assertEqual(len(policy_dbs), 2) self.assertEqual(len(config_schema_dbs), 1) self.assertEqual(len(config_dbs), 1) # Run action action = self.get_action_instance() action.run(packs=[pack]) # Make sure all resources have been removed from the db pack_dbs = Pack.query(ref=pack) action_dbs = Action.query(pack=pack) alias_dbs = ActionAlias.query(pack=pack) rule_dbs = Rule.query(pack=pack) sensor_dbs = Sensor.query(pack=pack) trigger_type_dbs = TriggerType.query(pack=pack) policy_dbs = Policy.query(pack=pack) config_schema_dbs = ConfigSchema.query(pack=pack) config_dbs = Config.query(pack=pack) self.assertEqual(len(pack_dbs), 0) self.assertEqual(len(action_dbs), 0) self.assertEqual(len(alias_dbs), 0) self.assertEqual(len(rule_dbs), 0) self.assertEqual(len(sensor_dbs), 0) self.assertEqual(len(trigger_type_dbs), 0) self.assertEqual(len(policy_dbs), 0) self.assertEqual(len(config_schema_dbs), 0) self.assertEqual(len(config_dbs), 0)
def _validate_config_values_against_schema(self): try: config_schema_db = ConfigSchema.get_by_pack(value=self.pack) except StackStormDBObjectNotFoundError: # Config schema is optional return # Note: We are doing optional validation so for now, we do allow additional properties instance = self.values or {} schema = config_schema_db.attributes schema = util_schema.get_schema_for_resource_parameters( parameters_schema=schema, allow_additional_properties=True) try: cleaned = util_schema.validate(instance=instance, schema=schema, cls=util_schema.CustomValidator, use_default=True, allow_default_none=True) except jsonschema.ValidationError as e: attribute = getattr(e, 'path', []) attribute = '.'.join(attribute) configs_path = os.path.join(cfg.CONF.system.base_path, 'configs/') config_path = os.path.join(configs_path, '%s.yaml' % (self.pack)) msg = ( 'Failed validating attribute "%s" in config for pack "%s" (%s): %s' % (attribute, self.pack, config_path, str(e))) raise jsonschema.ValidationError(msg) return cleaned
def get_config(self): result = {} # Retrieve corresponding ConfigDB and ConfigSchemaDB object # Note: ConfigSchemaDB is optional right now. If it doesn't exist, we assume every value # is of a type string try: config_db = Config.get_by_pack(value=self.pack_name) except StackStormDBObjectNotFoundError: # Corresponding pack config doesn't exist. We set config_db to an empty config so # that the default values from config schema are still correctly applied even if # pack doesn't contain a config. config_db = ConfigDB(pack=self.pack_name, values={}) try: config_schema_db = ConfigSchema.get_by_pack(value=self.pack_name) except StackStormDBObjectNotFoundError: config_schema_db = None # 2. Retrieve values from "global" pack config file (if available) and resolve them if # necessary config = self._get_values_for_config(config_schema_db=config_schema_db, config_db=config_db) result.update(config) return result
def _validate_config_values_against_schema(self): try: config_schema_db = ConfigSchema.get_by_pack(value=self.pack) except StackStormDBObjectNotFoundError: # Config schema is optional return # Note: We are doing optional validation so for now, we do allow additional properties instance = self.values or {} schema = config_schema_db.attributes schema = util_schema.get_schema_for_resource_parameters(parameters_schema=schema, allow_additional_properties=True) try: cleaned = util_schema.validate(instance=instance, schema=schema, cls=util_schema.CustomValidator, use_default=True, allow_default_none=True) except jsonschema.ValidationError as e: attribute = getattr(e, 'path', []) attribute = '.'.join(attribute) configs_path = os.path.join(cfg.CONF.system.base_path, 'configs/') config_path = os.path.join(configs_path, '%s.yaml' % (self.pack)) msg = ('Failed validating attribute "%s" in config for pack "%s" (%s): %s' % (attribute, self.pack, config_path, str(e))) raise jsonschema.ValidationError(msg) return cleaned
def get_config(self): result = {} # 1. Retrieve values from pack local config.yaml file config = self._config_parser.get_config() if config: config = config.config or {} result.update(config) # Retrieve corresponding ConfigDB and ConfigSchemaDB object # Note: ConfigSchemaDB is optional right now. If it doesn't exist, we assume every value # is of a type string try: config_db = Config.get_by_pack(value=self.pack_name) except StackStormDBObjectNotFoundError: # Corresponding pack config doesn't exist, return early return result try: config_schema_db = ConfigSchema.get_by_pack(value=self.pack_name) except StackStormDBObjectNotFoundError: config_schema_db = None # 2. Retrieve values from "global" pack config file (if available) and resolve them if # necessary config = self._get_values_for_config(config_schema_db=config_schema_db, config_db=config_db) result.update(config) return result
def get_config(self): result = {} # Retrieve corresponding ConfigDB and ConfigSchemaDB object # Note: ConfigSchemaDB is optional right now. If it doesn't exist, we assume every value # is of a type string try: config_db = Config.get_by_pack(value=self.pack_name) except StackStormDBObjectNotFoundError: # Corresponding pack config doesn't exist. We set config_db to an empty config so # that the default values from config schema are still correctly applied even if # pack doesn't contain a config. config_db = ConfigDB(pack=self.pack_name, values={}) try: config_schema_db = ConfigSchema.get_by_pack(value=self.pack_name) except StackStormDBObjectNotFoundError: config_schema_db = None # 2. Retrieve values from "global" pack config file (if available) and resolve them if # necessary config = self._get_values_for_config( config_schema_db=config_schema_db, config_db=config_db ) result.update(config) return result
def test_register_packs(self): # Verify DB is empty pack_dbs = Pack.get_all() config_schema_dbs = ConfigSchema.get_all() self.assertEqual(len(pack_dbs), 0) self.assertEqual(len(config_schema_dbs), 0) registrar = ResourceRegistrar(use_pack_cache=False) registrar._pack_loader.get_packs = mock.Mock() registrar._pack_loader.get_packs.return_value = { 'dummy_pack_1': PACK_PATH_1 } packs_base_paths = content_utils.get_packs_base_paths() registrar.register_packs(base_dirs=packs_base_paths) # Verify pack and schema have been registered pack_dbs = Pack.get_all() config_schema_dbs = ConfigSchema.get_all() self.assertEqual(len(pack_dbs), 1) self.assertEqual(len(config_schema_dbs), 1) pack_db = pack_dbs[0] config_schema_db = config_schema_dbs[0] self.assertEqual(pack_db.name, 'dummy_pack_1') self.assertEqual(len(pack_db.contributors), 2) self.assertEqual(pack_db.contributors[0], 'John Doe1 <*****@*****.**>') self.assertEqual(pack_db.contributors[1], 'John Doe2 <*****@*****.**>') self.assertTrue('api_key' in config_schema_db.attributes) self.assertTrue('api_secret' in config_schema_db.attributes) # Verify pack_db.files is correct and doesn't contain excluded files (*.pyc, .git/*, etc.) # Note: We can't test that .git/* files are excluded since git doesn't allow you to add # .git directory to existing repo index :/ excluded_files = [ '__init__.pyc', 'actions/dummy1.pyc', 'actions/dummy2.pyc', ] for excluded_file in excluded_files: self.assertTrue(excluded_file not in pack_db.files)
def test_register_packs(self): # Verify DB is empty pack_dbs = Pack.get_all() config_schema_dbs = ConfigSchema.get_all() self.assertEqual(len(pack_dbs), 0) self.assertEqual(len(config_schema_dbs), 0) registrar = ResourceRegistrar(use_pack_cache=False) registrar._pack_loader.get_packs = mock.Mock() registrar._pack_loader.get_packs.return_value = {'dummy_pack_1': PACK_PATH_1} packs_base_paths = content_utils.get_packs_base_paths() registrar.register_packs(base_dirs=packs_base_paths) # Verify pack and schema have been registered pack_dbs = Pack.get_all() config_schema_dbs = ConfigSchema.get_all() self.assertEqual(len(pack_dbs), 1) self.assertEqual(len(config_schema_dbs), 1) pack_db = pack_dbs[0] config_schema_db = config_schema_dbs[0] self.assertEqual(pack_db.name, 'dummy_pack_1') self.assertEqual(len(pack_db.contributors), 2) self.assertEqual(pack_db.contributors[0], 'John Doe1 <*****@*****.**>') self.assertEqual(pack_db.contributors[1], 'John Doe2 <*****@*****.**>') self.assertTrue('api_key' in config_schema_db.attributes) self.assertTrue('api_secret' in config_schema_db.attributes) # Verify pack_db.files is correct and doesn't contain excluded files (*.pyc, .git/*, etc.) # Note: We can't test that .git/* files are excluded since git doesn't allow you to add # .git directory to existing repo index :/ excluded_files = [ '__init__.pyc', 'actions/dummy1.pyc', 'actions/dummy2.pyc', ] for excluded_file in excluded_files: self.assertTrue(excluded_file not in pack_db.files)
def setUp(self): super(UnloadActionTestCase, self).setUp() # Register mock pack # Verify DB is empty pack_dbs = Pack.get_all() config_schema_dbs = ConfigSchema.get_all() config_dbs = Config.get_all() self.assertEqual(len(pack_dbs), 0) self.assertEqual(len(config_schema_dbs), 0) self.assertEqual(len(config_dbs), 0) # Register the pack with all the content # TODO: Don't use pack cache cfg.CONF.set_override(name='all', override=True, group='register') cfg.CONF.set_override(name='pack', override=PACK_PATH_1, group='register') cfg.CONF.set_override(name='no_fail_on_failure', override=True, group='register') register_content()
def _validate_config_values_against_schema(self): try: config_schema_db = ConfigSchema.get_by_pack(value=self.pack) except StackStormDBObjectNotFoundError: # Config schema is optional return # Note: We are doing optional validation so for now, we do allow additional properties instance = self.values or {} schema = config_schema_db.attributes or {} configs_path = os.path.join(cfg.CONF.system.base_path, 'configs/') config_path = os.path.join(configs_path, '%s.yaml' % (self.pack)) cleaned = validate_config_against_schema(config_schema=schema, config_object=instance, config_path=config_path, pack_name=self.pack) return cleaned
def _validate_config_values_against_schema(self): try: config_schema_db = ConfigSchema.get_by_pack(value=self.pack) except StackStormDBObjectNotFoundError: # Config schema is optional return # Note: We are doing optional validation so for now, we do allow additional properties instance = self.values or {} schema = config_schema_db.attributes schema = util_schema.get_schema_for_resource_parameters(parameters_schema=schema, allow_additional_properties=True) try: cleaned = util_schema.validate(instance=instance, schema=schema, cls=util_schema.CustomValidator, use_default=True, allow_default_none=True) except jsonschema.ValidationError as e: msg = 'Failed validating config for pack "%s": %s' % (self.pack, str(e)) raise jsonschema.ValidationError(msg) return cleaned