def _register_pack_config_schema_db(self, pack_name, pack_dir):
config_schema_path = os.path.join(pack_dir, CONFIG_SCHEMA_FILE_NAME)
if not os.path.isfile(config_schema_path):
# Note: Config schema is optional
return None
values = self._meta_loader.load(config_schema_path)
if not values:
raise ValueError('Config schema "%s" is empty and invalid.' %
(config_schema_path))
content = {}
content['pack'] = pack_name
content['attributes'] = values
config_schema_api = ConfigSchemaAPI(**content)
config_schema_api = config_schema_api.validate()
config_schema_db = ConfigSchemaAPI.to_model(config_schema_api)
try:
config_schema_db.id = ConfigSchema.get_by_pack(pack_name).id
except StackStormDBObjectNotFoundError:
LOG.debug('Config schema for pack %s not found. Creating new one.',
pack_name)
config_schema_db = ConfigSchema.add_or_update(config_schema_db)
LOG.debug('Config schema for pack %s registered.' % (pack_name))
return config_schema_db
def test_register_packs(self):
# Verify DB is empty
pack_dbs = Pack.get_all()
config_schema_dbs = ConfigSchema.get_all()
self.assertEqual(len(pack_dbs), 0)
self.assertEqual(len(config_schema_dbs), 0)
registrar = ResourceRegistrar(use_pack_cache=False)
registrar._pack_loader.get_packs = mock.Mock()
registrar._pack_loader.get_packs.return_value = {
'dummy_pack_1': PACK_PATH
}
packs_base_paths = content_utils.get_packs_base_paths()
registrar.register_packs(base_dirs=packs_base_paths)
# Verify pack and schema have been registered
pack_dbs = Pack.get_all()
config_schema_dbs = ConfigSchema.get_all()
self.assertEqual(len(pack_dbs), 1)
self.assertEqual(len(config_schema_dbs), 1)
self.assertEqual(pack_dbs[0].name, 'dummy_pack_1')
self.assertTrue('api_key' in config_schema_dbs[0].attributes)
self.assertTrue('api_secret' in config_schema_dbs[0].attributes)
def test_register_packs(self):
# Verify DB is empty
pack_dbs = Pack.get_all()
config_schema_dbs = ConfigSchema.get_all()
self.assertEqual(len(pack_dbs), 0)
self.assertEqual(len(config_schema_dbs), 0)
registrar = ResourceRegistrar(use_pack_cache=False)
registrar._pack_loader.get_packs = mock.Mock()
registrar._pack_loader.get_packs.return_value = {'dummy_pack_1': PACK_PATH_1}
packs_base_paths = content_utils.get_packs_base_paths()
registrar.register_packs(base_dirs=packs_base_paths)
# Verify pack and schema have been registered
pack_dbs = Pack.get_all()
config_schema_dbs = ConfigSchema.get_all()
self.assertEqual(len(pack_dbs), 1)
self.assertEqual(len(config_schema_dbs), 1)
self.assertEqual(pack_dbs[0].name, 'dummy_pack_1')
self.assertEqual(len(pack_dbs[0].contributors), 2)
self.assertEqual(pack_dbs[0].contributors[0], 'John Doe1 <*****@*****.**>')
self.assertEqual(pack_dbs[0].contributors[1], 'John Doe2 <*****@*****.**>')
self.assertTrue('api_key' in config_schema_dbs[0].attributes)
self.assertTrue('api_secret' in config_schema_dbs[0].attributes)
def _register_pack_config_schema_db(self, pack_name, pack_dir):
config_schema_path = os.path.join(pack_dir, CONFIG_SCHEMA_FILE_NAME)
if not os.path.isfile(config_schema_path):
# Note: Config schema is optional
return None
values = self._meta_loader.load(config_schema_path)
if not values:
raise ValueError('Config schema "%s" is empty and invalid.' % (config_schema_path))
content = {}
content['pack'] = pack_name
content['attributes'] = values
config_schema_api = ConfigSchemaAPI(**content)
config_schema_api = config_schema_api.validate()
config_schema_db = ConfigSchemaAPI.to_model(config_schema_api)
try:
config_schema_db.id = ConfigSchema.get_by_pack(pack_name).id
except StackStormDBObjectNotFoundError:
LOG.debug('Config schema for pack %s not found. Creating new one.', pack_name)
config_schema_db = ConfigSchema.add_or_update(config_schema_db)
LOG.debug('Config schema for pack %s registered.' % (pack_name))
return config_schema_db
def _delete_config_schema_db_object(self, pack):
try:
config_schema_db = ConfigSchema.get_by_pack(value=pack)
except StackStormDBObjectNotFoundError:
self.logger.exception('ConfigSchemaDB object not found')
return
try:
ConfigSchema.delete(config_schema_db)
except:
self.logger.exception('Failed to remove DB object %s.', config_schema_db)
def _delete_config_schema_db_object(self, pack):
try:
config_schema_db = ConfigSchema.get_by_pack(value=pack)
except StackStormDBObjectNotFoundError:
self.logger.exception('ConfigSchemaDB object not found')
return
try:
ConfigSchema.delete(config_schema_db)
except:
self.logger.exception('Failed to remove DB object %s.', config_schema_db)
def test_run(self):
pack = 'dummy_pack_1'
# Verify all the resources are there
pack_dbs = Pack.query(ref=pack)
action_dbs = Action.query(pack=pack)
alias_dbs = ActionAlias.query(pack=pack)
rule_dbs = Rule.query(pack=pack)
sensor_dbs = Sensor.query(pack=pack)
trigger_type_dbs = TriggerType.query(pack=pack)
policy_dbs = Policy.query(pack=pack)
config_schema_dbs = ConfigSchema.query(pack=pack)
config_dbs = Config.query(pack=pack)
self.assertEqual(len(pack_dbs), 1)
self.assertEqual(len(action_dbs), 1)
self.assertEqual(len(alias_dbs), 2)
self.assertEqual(len(rule_dbs), 1)
self.assertEqual(len(sensor_dbs), 3)
self.assertEqual(len(trigger_type_dbs), 4)
self.assertEqual(len(policy_dbs), 2)
self.assertEqual(len(config_schema_dbs), 1)
self.assertEqual(len(config_dbs), 1)
# Run action
action = self.get_action_instance()
action.run(packs=[pack])
# Make sure all resources have been removed from the db
pack_dbs = Pack.query(ref=pack)
action_dbs = Action.query(pack=pack)
alias_dbs = ActionAlias.query(pack=pack)
rule_dbs = Rule.query(pack=pack)
sensor_dbs = Sensor.query(pack=pack)
trigger_type_dbs = TriggerType.query(pack=pack)
policy_dbs = Policy.query(pack=pack)
config_schema_dbs = ConfigSchema.query(pack=pack)
config_dbs = Config.query(pack=pack)
self.assertEqual(len(pack_dbs), 0)
self.assertEqual(len(action_dbs), 0)
self.assertEqual(len(alias_dbs), 0)
self.assertEqual(len(rule_dbs), 0)
self.assertEqual(len(sensor_dbs), 0)
self.assertEqual(len(trigger_type_dbs), 0)
self.assertEqual(len(policy_dbs), 0)
self.assertEqual(len(config_schema_dbs), 0)
self.assertEqual(len(config_dbs), 0)
def test_run(self):
pack = 'dummy_pack_1'
# Verify all the resources are there
pack_dbs = Pack.query(ref=pack)
action_dbs = Action.query(pack=pack)
alias_dbs = ActionAlias.query(pack=pack)
rule_dbs = Rule.query(pack=pack)
sensor_dbs = Sensor.query(pack=pack)
trigger_type_dbs = TriggerType.query(pack=pack)
policy_dbs = Policy.query(pack=pack)
config_schema_dbs = ConfigSchema.query(pack=pack)
config_dbs = Config.query(pack=pack)
self.assertEqual(len(pack_dbs), 1)
self.assertEqual(len(action_dbs), 1)
self.assertEqual(len(alias_dbs), 3)
self.assertEqual(len(rule_dbs), 1)
self.assertEqual(len(sensor_dbs), 3)
self.assertEqual(len(trigger_type_dbs), 4)
self.assertEqual(len(policy_dbs), 2)
self.assertEqual(len(config_schema_dbs), 1)
self.assertEqual(len(config_dbs), 1)
# Run action
action = self.get_action_instance()
action.run(packs=[pack])
# Make sure all resources have been removed from the db
pack_dbs = Pack.query(ref=pack)
action_dbs = Action.query(pack=pack)
alias_dbs = ActionAlias.query(pack=pack)
rule_dbs = Rule.query(pack=pack)
sensor_dbs = Sensor.query(pack=pack)
trigger_type_dbs = TriggerType.query(pack=pack)
policy_dbs = Policy.query(pack=pack)
config_schema_dbs = ConfigSchema.query(pack=pack)
config_dbs = Config.query(pack=pack)
self.assertEqual(len(pack_dbs), 0)
self.assertEqual(len(action_dbs), 0)
self.assertEqual(len(alias_dbs), 0)
self.assertEqual(len(rule_dbs), 0)
self.assertEqual(len(sensor_dbs), 0)
self.assertEqual(len(trigger_type_dbs), 0)
self.assertEqual(len(policy_dbs), 0)
self.assertEqual(len(config_schema_dbs), 0)
self.assertEqual(len(config_dbs), 0)
def _validate_config_values_against_schema(self):
try:
config_schema_db = ConfigSchema.get_by_pack(value=self.pack)
except StackStormDBObjectNotFoundError:
# Config schema is optional
return
# Note: We are doing optional validation so for now, we do allow additional properties
instance = self.values or {}
schema = config_schema_db.attributes
schema = util_schema.get_schema_for_resource_parameters(
parameters_schema=schema, allow_additional_properties=True)
try:
cleaned = util_schema.validate(instance=instance,
schema=schema,
cls=util_schema.CustomValidator,
use_default=True,
allow_default_none=True)
except jsonschema.ValidationError as e:
attribute = getattr(e, 'path', [])
attribute = '.'.join(attribute)
configs_path = os.path.join(cfg.CONF.system.base_path, 'configs/')
config_path = os.path.join(configs_path, '%s.yaml' % (self.pack))
msg = (
'Failed validating attribute "%s" in config for pack "%s" (%s): %s'
% (attribute, self.pack, config_path, str(e)))
raise jsonschema.ValidationError(msg)
return cleaned
def get_config(self):
result = {}
# Retrieve corresponding ConfigDB and ConfigSchemaDB object
# Note: ConfigSchemaDB is optional right now. If it doesn't exist, we assume every value
# is of a type string
try:
config_db = Config.get_by_pack(value=self.pack_name)
except StackStormDBObjectNotFoundError:
# Corresponding pack config doesn't exist. We set config_db to an empty config so
# that the default values from config schema are still correctly applied even if
# pack doesn't contain a config.
config_db = ConfigDB(pack=self.pack_name, values={})
try:
config_schema_db = ConfigSchema.get_by_pack(value=self.pack_name)
except StackStormDBObjectNotFoundError:
config_schema_db = None
# 2. Retrieve values from "global" pack config file (if available) and resolve them if
# necessary
config = self._get_values_for_config(config_schema_db=config_schema_db,
config_db=config_db)
result.update(config)
return result
def _validate_config_values_against_schema(self):
try:
config_schema_db = ConfigSchema.get_by_pack(value=self.pack)
except StackStormDBObjectNotFoundError:
# Config schema is optional
return
# Note: We are doing optional validation so for now, we do allow additional properties
instance = self.values or {}
schema = config_schema_db.attributes
schema = util_schema.get_schema_for_resource_parameters(parameters_schema=schema,
allow_additional_properties=True)
try:
cleaned = util_schema.validate(instance=instance, schema=schema,
cls=util_schema.CustomValidator, use_default=True,
allow_default_none=True)
except jsonschema.ValidationError as e:
attribute = getattr(e, 'path', [])
attribute = '.'.join(attribute)
configs_path = os.path.join(cfg.CONF.system.base_path, 'configs/')
config_path = os.path.join(configs_path, '%s.yaml' % (self.pack))
msg = ('Failed validating attribute "%s" in config for pack "%s" (%s): %s' %
(attribute, self.pack, config_path, str(e)))
raise jsonschema.ValidationError(msg)
return cleaned
def get_config(self):
result = {}
# 1. Retrieve values from pack local config.yaml file
config = self._config_parser.get_config()
if config:
config = config.config or {}
result.update(config)
# Retrieve corresponding ConfigDB and ConfigSchemaDB object
# Note: ConfigSchemaDB is optional right now. If it doesn't exist, we assume every value
# is of a type string
try:
config_db = Config.get_by_pack(value=self.pack_name)
except StackStormDBObjectNotFoundError:
# Corresponding pack config doesn't exist, return early
return result
try:
config_schema_db = ConfigSchema.get_by_pack(value=self.pack_name)
except StackStormDBObjectNotFoundError:
config_schema_db = None
# 2. Retrieve values from "global" pack config file (if available) and resolve them if
# necessary
config = self._get_values_for_config(config_schema_db=config_schema_db,
config_db=config_db)
result.update(config)
return result
def get_config(self):
result = {}
# 1. Retrieve values from pack local config.yaml file
config = self._config_parser.get_config()
if config:
config = config.config or {}
result.update(config)
# Retrieve corresponding ConfigDB and ConfigSchemaDB object
# Note: ConfigSchemaDB is optional right now. If it doesn't exist, we assume every value
# is of a type string
try:
config_db = Config.get_by_pack(value=self.pack_name)
except StackStormDBObjectNotFoundError:
# Corresponding pack config doesn't exist, return early
return result
try:
config_schema_db = ConfigSchema.get_by_pack(value=self.pack_name)
except StackStormDBObjectNotFoundError:
config_schema_db = None
# 2. Retrieve values from "global" pack config file (if available) and resolve them if
# necessary
config = self._get_values_for_config(config_schema_db=config_schema_db,
config_db=config_db)
result.update(config)
return result
def get_config(self):
result = {}
# Retrieve corresponding ConfigDB and ConfigSchemaDB object
# Note: ConfigSchemaDB is optional right now. If it doesn't exist, we assume every value
# is of a type string
try:
config_db = Config.get_by_pack(value=self.pack_name)
except StackStormDBObjectNotFoundError:
# Corresponding pack config doesn't exist. We set config_db to an empty config so
# that the default values from config schema are still correctly applied even if
# pack doesn't contain a config.
config_db = ConfigDB(pack=self.pack_name, values={})
try:
config_schema_db = ConfigSchema.get_by_pack(value=self.pack_name)
except StackStormDBObjectNotFoundError:
config_schema_db = None
# 2. Retrieve values from "global" pack config file (if available) and resolve them if
# necessary
config = self._get_values_for_config(
config_schema_db=config_schema_db, config_db=config_db
)
result.update(config)
return result
def test_register_packs(self):
# Verify DB is empty
pack_dbs = Pack.get_all()
config_schema_dbs = ConfigSchema.get_all()
self.assertEqual(len(pack_dbs), 0)
self.assertEqual(len(config_schema_dbs), 0)
registrar = ResourceRegistrar(use_pack_cache=False)
registrar._pack_loader.get_packs = mock.Mock()
registrar._pack_loader.get_packs.return_value = {
'dummy_pack_1': PACK_PATH_1
}
packs_base_paths = content_utils.get_packs_base_paths()
registrar.register_packs(base_dirs=packs_base_paths)
# Verify pack and schema have been registered
pack_dbs = Pack.get_all()
config_schema_dbs = ConfigSchema.get_all()
self.assertEqual(len(pack_dbs), 1)
self.assertEqual(len(config_schema_dbs), 1)
pack_db = pack_dbs[0]
config_schema_db = config_schema_dbs[0]
self.assertEqual(pack_db.name, 'dummy_pack_1')
self.assertEqual(len(pack_db.contributors), 2)
self.assertEqual(pack_db.contributors[0],
'John Doe1 <*****@*****.**>')
self.assertEqual(pack_db.contributors[1],
'John Doe2 <*****@*****.**>')
self.assertTrue('api_key' in config_schema_db.attributes)
self.assertTrue('api_secret' in config_schema_db.attributes)
# Verify pack_db.files is correct and doesn't contain excluded files (*.pyc, .git/*, etc.)
# Note: We can't test that .git/* files are excluded since git doesn't allow you to add
# .git directory to existing repo index :/
excluded_files = [
'__init__.pyc',
'actions/dummy1.pyc',
'actions/dummy2.pyc',
]
for excluded_file in excluded_files:
self.assertTrue(excluded_file not in pack_db.files)
def test_register_packs(self):
# Verify DB is empty
pack_dbs = Pack.get_all()
config_schema_dbs = ConfigSchema.get_all()
self.assertEqual(len(pack_dbs), 0)
self.assertEqual(len(config_schema_dbs), 0)
registrar = ResourceRegistrar(use_pack_cache=False)
registrar._pack_loader.get_packs = mock.Mock()
registrar._pack_loader.get_packs.return_value = {'dummy_pack_1': PACK_PATH_1}
packs_base_paths = content_utils.get_packs_base_paths()
registrar.register_packs(base_dirs=packs_base_paths)
# Verify pack and schema have been registered
pack_dbs = Pack.get_all()
config_schema_dbs = ConfigSchema.get_all()
self.assertEqual(len(pack_dbs), 1)
self.assertEqual(len(config_schema_dbs), 1)
pack_db = pack_dbs[0]
config_schema_db = config_schema_dbs[0]
self.assertEqual(pack_db.name, 'dummy_pack_1')
self.assertEqual(len(pack_db.contributors), 2)
self.assertEqual(pack_db.contributors[0], 'John Doe1 <*****@*****.**>')
self.assertEqual(pack_db.contributors[1], 'John Doe2 <*****@*****.**>')
self.assertTrue('api_key' in config_schema_db.attributes)
self.assertTrue('api_secret' in config_schema_db.attributes)
# Verify pack_db.files is correct and doesn't contain excluded files (*.pyc, .git/*, etc.)
# Note: We can't test that .git/* files are excluded since git doesn't allow you to add
# .git directory to existing repo index :/
excluded_files = [
'__init__.pyc',
'actions/dummy1.pyc',
'actions/dummy2.pyc',
]
for excluded_file in excluded_files:
self.assertTrue(excluded_file not in pack_db.files)
def setUp(self):
super(UnloadActionTestCase, self).setUp()
# Register mock pack
# Verify DB is empty
pack_dbs = Pack.get_all()
config_schema_dbs = ConfigSchema.get_all()
config_dbs = Config.get_all()
self.assertEqual(len(pack_dbs), 0)
self.assertEqual(len(config_schema_dbs), 0)
self.assertEqual(len(config_dbs), 0)
# Register the pack with all the content
# TODO: Don't use pack cache
cfg.CONF.set_override(name='all', override=True, group='register')
cfg.CONF.set_override(name='pack', override=PACK_PATH_1, group='register')
cfg.CONF.set_override(name='no_fail_on_failure', override=True, group='register')
register_content()
def setUp(self):
super(UnloadActionTestCase, self).setUp()
# Register mock pack
# Verify DB is empty
pack_dbs = Pack.get_all()
config_schema_dbs = ConfigSchema.get_all()
config_dbs = Config.get_all()
self.assertEqual(len(pack_dbs), 0)
self.assertEqual(len(config_schema_dbs), 0)
self.assertEqual(len(config_dbs), 0)
# Register the pack with all the content
# TODO: Don't use pack cache
cfg.CONF.set_override(name='all', override=True, group='register')
cfg.CONF.set_override(name='pack', override=PACK_PATH_1, group='register')
cfg.CONF.set_override(name='no_fail_on_failure', override=True, group='register')
register_content()
def _validate_config_values_against_schema(self):
try:
config_schema_db = ConfigSchema.get_by_pack(value=self.pack)
except StackStormDBObjectNotFoundError:
# Config schema is optional
return
# Note: We are doing optional validation so for now, we do allow additional properties
instance = self.values or {}
schema = config_schema_db.attributes or {}
configs_path = os.path.join(cfg.CONF.system.base_path, 'configs/')
config_path = os.path.join(configs_path, '%s.yaml' % (self.pack))
cleaned = validate_config_against_schema(config_schema=schema,
config_object=instance,
config_path=config_path,
pack_name=self.pack)
return cleaned
def _validate_config_values_against_schema(self):
try:
config_schema_db = ConfigSchema.get_by_pack(value=self.pack)
except StackStormDBObjectNotFoundError:
# Config schema is optional
return
# Note: We are doing optional validation so for now, we do allow additional properties
instance = self.values or {}
schema = config_schema_db.attributes or {}
configs_path = os.path.join(cfg.CONF.system.base_path, 'configs/')
config_path = os.path.join(configs_path, '%s.yaml' % (self.pack))
cleaned = validate_config_against_schema(config_schema=schema,
config_object=instance,
config_path=config_path,
pack_name=self.pack)
return cleaned
def _validate_config_values_against_schema(self):
try:
config_schema_db = ConfigSchema.get_by_pack(value=self.pack)
except StackStormDBObjectNotFoundError:
# Config schema is optional
return
# Note: We are doing optional validation so for now, we do allow additional properties
instance = self.values or {}
schema = config_schema_db.attributes
schema = util_schema.get_schema_for_resource_parameters(parameters_schema=schema,
allow_additional_properties=True)
try:
cleaned = util_schema.validate(instance=instance, schema=schema,
cls=util_schema.CustomValidator, use_default=True,
allow_default_none=True)
except jsonschema.ValidationError as e:
msg = 'Failed validating config for pack "%s": %s' % (self.pack, str(e))
raise jsonschema.ValidationError(msg)
return cleaned
