def test_register_pack_pack_ref(self): # Verify DB is empty pack_dbs = Pack.get_all() self.assertEqual(len(pack_dbs), 0) registrar = ResourceRegistrar(use_pack_cache=False) registrar._pack_loader.get_packs = mock.Mock() registrar._pack_loader.get_packs.return_value = { 'dummy_pack_1': PACK_PATH_1, 'dummy_pack_6': PACK_PATH_6 } packs_base_paths = content_utils.get_packs_base_paths() registrar.register_packs(base_dirs=packs_base_paths) # Ref is provided pack_db = Pack.get_by_name('dummy_pack_6') self.assertEqual(pack_db.ref, 'dummy_pack_6_ref') self.assertEqual(len(pack_dbs[0].contributors), 0) # Ref is not provided, directory name should be used pack_db = Pack.get_by_name('dummy_pack_1') self.assertEqual(pack_db.ref, 'dummy_pack_1') # "ref" is not provided, but "name" is registrar._register_pack_db(pack_name=None, pack_dir=PACK_PATH_7) pack_db = Pack.get_by_name('dummy_pack_7_name') self.assertEqual(pack_db.ref, 'dummy_pack_7_name') # "ref" is not provided and "name" contains invalid characters expected_msg = 'contains invalid characters' self.assertRaisesRegexp(ValueError, expected_msg, registrar._register_pack_db, pack_name=None, pack_dir=PACK_PATH_8)
def _register_pack_db(self, pack_name, pack_dir): content = get_pack_metadata(pack_dir=pack_dir) # The rules for the pack ref are as follows: # 1. If ref attribute is available, we used that # 2. If pack_name is available we use that (this only applies to packs # 2hich are in sub-directories) # 2. If attribute is not available, but pack name is and pack name meets the valid name # criteria, we use that content['ref'] = get_pack_ref_from_metadata( metadata=content, pack_directory_name=pack_name) # Include a list of pack files pack_file_list = get_file_list(directory=pack_dir, exclude_patterns=EXCLUDE_FILE_PATTERNS) content['files'] = pack_file_list pack_api = PackAPI(**content) pack_api.validate() pack_db = PackAPI.to_model(pack_api) try: pack_db.id = Pack.get_by_ref(content['ref']).id except StackStormDBObjectNotFoundError: LOG.debug('Pack %s not found. Creating new one.', pack_name) pack_db = Pack.add_or_update(pack_db) LOG.debug('Pack %s registered.' % (pack_name)) return pack_db
def test_register_pack_pack_stackstorm_version_and_future_parameters(self): # Verify DB is empty pack_dbs = Pack.get_all() self.assertEqual(len(pack_dbs), 0) registrar = ResourceRegistrar(use_pack_cache=False) registrar._pack_loader.get_packs = mock.Mock() registrar._pack_loader.get_packs.return_value = {'dummy_pack_9': PACK_PATH_9} packs_base_paths = content_utils.get_packs_base_paths() registrar.register_packs(base_dirs=packs_base_paths) # Dependencies, stackstorm_version and future values pack_db = Pack.get_by_name('dummy_pack_9_deps') self.assertEqual(pack_db.dependencies, ['core=0.2.0']) self.assertEqual(pack_db.stackstorm_version, '>=1.6.0, <2.2.0') self.assertEqual(pack_db.system, {'centos': {'foo': '>= 1.0'}}) self.assertEqual(pack_db.python_versions, ['2', '3']) # Note: We only store parameters which are defined in the schema, all other custom user # defined attributes are ignored self.assertTrue(not hasattr(pack_db, 'future')) self.assertTrue(not hasattr(pack_db, 'this')) # Wrong characters in the required st2 version expected_msg = "'wrongstackstormversion' does not match" self.assertRaisesRegexp(ValidationError, expected_msg, registrar._register_pack_db, pack_name=None, pack_dir=PACK_PATH_10)
def _register_pack(self, pack_name, pack_dir): """ Register a pack (create a DB object in the system). Note: Pack registration now happens when registering the content and not when installing a pack using packs.install. Eventually this will be moved to the pack management API. """ manifest_path = os.path.join(pack_dir, MANIFEST_FILE_NAME) if not os.path.isfile(manifest_path): raise ValueError('Pack "%s" is missing %s file' % (pack_name, MANIFEST_FILE_NAME)) content = self._meta_loader.load(manifest_path) if not content: raise ValueError('Pack "%s" metadata file is empty' % (pack_name)) content['ref'] = pack_name pack_api = PackAPI(**content) pack_db = PackAPI.to_model(pack_api) try: pack_db.id = Pack.get_by_ref(pack_name).id except ValueError: LOG.debug('Pack %s not found. Creating new one.', pack_name) pack_db = Pack.add_or_update(pack_db) LOG.debug('Pack %s registered.' % (pack_name)) return pack_db
def test_register_packs(self): # Verify DB is empty pack_dbs = Pack.get_all() config_schema_dbs = ConfigSchema.get_all() self.assertEqual(len(pack_dbs), 0) self.assertEqual(len(config_schema_dbs), 0) registrar = ResourceRegistrar(use_pack_cache=False) registrar._pack_loader.get_packs = mock.Mock() registrar._pack_loader.get_packs.return_value = { 'dummy_pack_1': PACK_PATH } packs_base_paths = content_utils.get_packs_base_paths() registrar.register_packs(base_dirs=packs_base_paths) # Verify pack and schema have been registered pack_dbs = Pack.get_all() config_schema_dbs = ConfigSchema.get_all() self.assertEqual(len(pack_dbs), 1) self.assertEqual(len(config_schema_dbs), 1) self.assertEqual(pack_dbs[0].name, 'dummy_pack_1') self.assertTrue('api_key' in config_schema_dbs[0].attributes) self.assertTrue('api_secret' in config_schema_dbs[0].attributes)
def test_register_configs_for_all_packs(self): # Verify DB is empty pack_dbs = Pack.get_all() config_dbs = Config.get_all() self.assertEqual(len(pack_dbs), 0) self.assertEqual(len(config_dbs), 0) registrar = ConfigsRegistrar(use_pack_cache=False) registrar._pack_loader.get_packs = mock.Mock() registrar._pack_loader.get_packs.return_value = { 'dummy_pack_1': PACK_1_PATH } packs_base_paths = content_utils.get_packs_base_paths() registrar.register_configs_for_all_packs(base_dirs=packs_base_paths) # Verify pack and schema have been registered pack_dbs = Pack.get_all() config_dbs = Config.get_all() self.assertEqual(len(pack_dbs), 1) self.assertEqual(len(config_dbs), 1) config_db = config_dbs[0] self.assertEqual(config_db.values['api_key'], '{{user.api_key}}') self.assertEqual(config_db.values['api_secret'], '{{user.api_secret}}') self.assertEqual(config_db.values['region'], 'us-west-1')
def _register_pack_db(self, pack_name, pack_dir): manifest_path = os.path.join(pack_dir, MANIFEST_FILE_NAME) if not os.path.isfile(manifest_path): raise ValueError('Pack "%s" is missing %s file' % (pack_name, MANIFEST_FILE_NAME)) content = self._meta_loader.load(manifest_path) if not content: raise ValueError('Pack "%s" metadata file is empty' % (pack_name)) content['ref'] = pack_name # Include a list of pack files pack_file_list = get_file_list(directory=pack_dir, exclude_patterns=EXCLUDE_FILE_PATTERNS) content['files'] = pack_file_list pack_api = PackAPI(**content) pack_db = PackAPI.to_model(pack_api) try: pack_db.id = Pack.get_by_ref(pack_name).id except StackStormDBObjectNotFoundError: LOG.debug('Pack %s not found. Creating new one.', pack_name) pack_db = Pack.add_or_update(pack_db) LOG.debug('Pack %s registered.' % (pack_name)) return pack_db
def test_register_all_configs_invalid_config_no_config_schema(self): # verify_ configs is on, but ConfigSchema for the pack doesn't exist so # validation should proceed normally # Verify DB is empty pack_dbs = Pack.get_all() config_dbs = Config.get_all() self.assertEqual(len(pack_dbs), 0) self.assertEqual(len(config_dbs), 0) registrar = ConfigsRegistrar(use_pack_cache=False, validate_configs=False) registrar._pack_loader.get_packs = mock.Mock() registrar._pack_loader.get_packs.return_value = { "dummy_pack_6": PACK_6_PATH } packs_base_paths = content_utils.get_packs_base_paths() registrar.register_from_packs(base_dirs=packs_base_paths) # Verify pack and schema have been registered pack_dbs = Pack.get_all() config_dbs = Config.get_all() self.assertEqual(len(pack_dbs), 1) self.assertEqual(len(config_dbs), 1)
def _register_pack(self, pack_name, pack_dir): """ Register a pack (create a DB object in the system). Note: Pack registration now happens when registering the content and not when installing a pack using packs.install. Eventually this will be moved to the pack management API. """ manifest_path = os.path.join(pack_dir, MANIFEST_FILE_NAME) if not os.path.isfile(manifest_path): raise ValueError('Pack "%s" is missing %s file' % (pack_name, MANIFEST_FILE_NAME)) content = self._meta_loader.load(manifest_path) if not content: raise ValueError('Pack "%s" metadata file is empty' % (pack_name)) content['ref'] = pack_name # Include a list of pack files pack_file_list = get_file_list(directory=pack_dir, exclude_patterns=EXCLUDE_FILE_PATTERNS) content['files'] = pack_file_list pack_api = PackAPI(**content) pack_db = PackAPI.to_model(pack_api) try: pack_db.id = Pack.get_by_ref(pack_name).id except ValueError: LOG.debug('Pack %s not found. Creating new one.', pack_name) pack_db = Pack.add_or_update(pack_db) LOG.debug('Pack %s registered.' % (pack_name)) return pack_db
def _register_pack_db(self, pack_name, pack_dir): content = get_pack_metadata(pack_dir=pack_dir) # The rules for the pack ref are as follows: # 1. If ref attribute is available, we used that # 2. If pack_name is available we use that (this only applies to packs # 2hich are in sub-directories) # 2. If attribute is not available, but pack name is and pack name meets the valid name # criteria, we use that content['ref'] = get_pack_ref_from_metadata(metadata=content, pack_directory_name=pack_name) # Include a list of pack files pack_file_list = get_file_list(directory=pack_dir, exclude_patterns=EXCLUDE_FILE_PATTERNS) content['files'] = pack_file_list pack_api = PackAPI(**content) pack_api.validate() pack_db = PackAPI.to_model(pack_api) try: pack_db.id = Pack.get_by_ref(content['ref']).id except StackStormDBObjectNotFoundError: LOG.debug('Pack %s not found. Creating new one.', pack_name) pack_db = Pack.add_or_update(pack_db) LOG.debug('Pack %s registered.' % (pack_name)) return pack_db
def test_register_packs(self): # Verify DB is empty pack_dbs = Pack.get_all() config_schema_dbs = ConfigSchema.get_all() self.assertEqual(len(pack_dbs), 0) self.assertEqual(len(config_schema_dbs), 0) registrar = ResourceRegistrar(use_pack_cache=False) registrar._pack_loader.get_packs = mock.Mock() registrar._pack_loader.get_packs.return_value = {'dummy_pack_1': PACK_PATH_1} packs_base_paths = content_utils.get_packs_base_paths() registrar.register_packs(base_dirs=packs_base_paths) # Verify pack and schema have been registered pack_dbs = Pack.get_all() config_schema_dbs = ConfigSchema.get_all() self.assertEqual(len(pack_dbs), 1) self.assertEqual(len(config_schema_dbs), 1) self.assertEqual(pack_dbs[0].name, 'dummy_pack_1') self.assertEqual(len(pack_dbs[0].contributors), 2) self.assertEqual(pack_dbs[0].contributors[0], 'John Doe1 <*****@*****.**>') self.assertEqual(pack_dbs[0].contributors[1], 'John Doe2 <*****@*****.**>') self.assertTrue('api_key' in config_schema_dbs[0].attributes) self.assertTrue('api_secret' in config_schema_dbs[0].attributes)
def _delete_pack_db_object(self, pack): pack_db = None # 1. Try by ref try: pack_db = Pack.get_by_ref(value=pack) except StackStormDBObjectNotFoundError: pack_db = None # 2. Try by name (here for backward compatibility) # TODO: This shouldn't be needed in the future, remove it in v2.1 or similar if not pack_db: try: pack_db = Pack.get_by_name(value=pack) except StackStormDBObjectNotFoundError: pack_db = None if not pack_db: self.logger.exception('Pack DB object not found') return try: Pack.delete(pack_db) except: self.logger.exception('Failed to remove DB object %s.', pack_db)
def test_register_configs_for_all_packs(self): # Verify DB is empty pack_dbs = Pack.get_all() config_dbs = Config.get_all() self.assertEqual(len(pack_dbs), 0) self.assertEqual(len(config_dbs), 0) registrar = ConfigsRegistrar(use_pack_cache=False) registrar._pack_loader.get_packs = mock.Mock() registrar._pack_loader.get_packs.return_value = {"dummy_pack_1": PACK_PATH} packs_base_paths = content_utils.get_packs_base_paths() registrar.register_configs_for_all_packs(base_dirs=packs_base_paths) # Verify pack and schema have been registered pack_dbs = Pack.get_all() config_dbs = Config.get_all() self.assertEqual(len(pack_dbs), 1) self.assertEqual(len(config_dbs), 1) config_db = config_dbs[0] self.assertEqual(config_db.values["api_key"], "{{user.api_key}}") self.assertEqual(config_db.values["api_secret"], "{{user.api_secret}}") self.assertEqual(config_db.values["region"], "us-west-1")
def test_register_pack_pack_stackstorm_version_and_future_parameters(self): # Verify DB is empty pack_dbs = Pack.get_all() self.assertEqual(len(pack_dbs), 0) registrar = ResourceRegistrar(use_pack_cache=False) registrar._pack_loader.get_packs = mock.Mock() registrar._pack_loader.get_packs.return_value = { 'dummy_pack_9': PACK_PATH_9 } packs_base_paths = content_utils.get_packs_base_paths() registrar.register_packs(base_dirs=packs_base_paths) # Dependencies, stackstorm_version and future values pack_db = Pack.get_by_name('dummy_pack_9_deps') self.assertEqual(pack_db.dependencies, ['core=0.2.0']) self.assertEqual(pack_db.stackstorm_version, '>=1.6.0, <2.2.0') self.assertEqual(pack_db.system, {'centos': {'foo': '>= 1.0'}}) # Note: We only store paramters which are defined in the schema, all other custom user # defined attributes are ignored self.assertTrue(not hasattr(pack_db, 'future')) self.assertTrue(not hasattr(pack_db, 'this')) # Wrong characters in the required st2 version expected_msg = "'wrongstackstormversion' does not match" self.assertRaisesRegexp(ValidationError, expected_msg, registrar._register_pack_db, pack_name=None, pack_dir=PACK_PATH_10)
def test_register_configs_for_all_packs(self): # Verify DB is empty pack_dbs = Pack.get_all() config_dbs = Config.get_all() self.assertEqual(len(pack_dbs), 0) self.assertEqual(len(config_dbs), 0) registrar = ConfigsRegistrar(use_pack_cache=False) registrar._pack_loader.get_packs = mock.Mock() registrar._pack_loader.get_packs.return_value = {'dummy_pack_1': PACK_1_PATH} packs_base_paths = content_utils.get_packs_base_paths() registrar.register_from_packs(base_dirs=packs_base_paths) # Verify pack and schema have been registered pack_dbs = Pack.get_all() config_dbs = Config.get_all() self.assertEqual(len(pack_dbs), 1) self.assertEqual(len(config_dbs), 1) config_db = config_dbs[0] self.assertEqual(config_db.values['api_key'], '{{st2kv.user.api_key}}') self.assertEqual(config_db.values['api_secret'], SUPER_SECRET_PARAMETER) self.assertEqual(config_db.values['region'], 'us-west-1')
def test_register_configs_for_all_packs(self): # Verify DB is empty pack_dbs = Pack.get_all() config_dbs = Config.get_all() self.assertEqual(len(pack_dbs), 0) self.assertEqual(len(config_dbs), 0) registrar = ConfigsRegistrar(use_pack_cache=False) registrar._pack_loader.get_packs = mock.Mock() registrar._pack_loader.get_packs.return_value = { "dummy_pack_1": PACK_1_PATH } packs_base_paths = content_utils.get_packs_base_paths() registrar.register_from_packs(base_dirs=packs_base_paths) # Verify pack and schema have been registered pack_dbs = Pack.get_all() config_dbs = Config.get_all() self.assertEqual(len(pack_dbs), 1) self.assertEqual(len(config_dbs), 1) config_db = config_dbs[0] self.assertEqual(config_db.values["api_key"], "{{st2kv.user.api_key}}") self.assertEqual(config_db.values["api_secret"], SUPER_SECRET_PARAMETER) self.assertEqual(config_db.values["region"], "us-west-1")
def test_register_pack_pack_ref(self): # Verify DB is empty pack_dbs = Pack.get_all() self.assertEqual(len(pack_dbs), 0) registrar = ResourceRegistrar(use_pack_cache=False) registrar._pack_loader.get_packs = mock.Mock() registrar._pack_loader.get_packs.return_value = { 'dummy_pack_1': PACK_PATH_1, 'dummy_pack_6': PACK_PATH_6 } packs_base_paths = content_utils.get_packs_base_paths() registrar.register_packs(base_dirs=packs_base_paths) # Ref is provided pack_db = Pack.get_by_name('dummy_pack_6') self.assertEqual(pack_db.ref, 'dummy_pack_6_ref') self.assertEqual(len(pack_db.contributors), 0) # Ref is not provided, directory name should be used pack_db = Pack.get_by_name('dummy_pack_1') self.assertEqual(pack_db.ref, 'dummy_pack_1') # "ref" is not provided, but "name" is registrar._register_pack_db(pack_name=None, pack_dir=PACK_PATH_7) pack_db = Pack.get_by_name('dummy_pack_7_name') self.assertEqual(pack_db.ref, 'dummy_pack_7_name') # "ref" is not provided and "name" contains invalid characters expected_msg = 'contains invalid characters' self.assertRaisesRegexp(ValueError, expected_msg, registrar._register_pack_db, pack_name=None, pack_dir=PACK_PATH_8)
def setUpClass(cls): super(PacksControllerTestCase, cls).setUpClass() cls.pack_db_1 = PackDB(name='Pack1', description='foo', version='0.1.0', author='foo', email='*****@*****.**', ref='pack1') cls.pack_db_2 = PackDB(name='Pack2', description='foo', version='0.1.0', author='foo', email='*****@*****.**', ref='pack2') Pack.add_or_update(cls.pack_db_1) Pack.add_or_update(cls.pack_db_2)
def setUpClass(cls): super(PacksControllerTestCase, cls).setUpClass() cls.pack_db_1 = PackDB(name='pack1', description='foo', version='0.1.0', author='foo', email='*****@*****.**', ref='pack1') cls.pack_db_2 = PackDB(name='pack2', description='foo', version='0.1.0', author='foo', email='*****@*****.**', ref='pack2') Pack.add_or_update(cls.pack_db_1) Pack.add_or_update(cls.pack_db_2)
def _register_pack_db(self, pack_name, pack_dir): pack_name = pack_name or '' manifest_path = os.path.join(pack_dir, MANIFEST_FILE_NAME) if not os.path.isfile(manifest_path): raise ValueError('Pack "%s" is missing %s file' % (pack_name, MANIFEST_FILE_NAME)) content = self._meta_loader.load(manifest_path) if not content: raise ValueError('Pack "%s" metadata file is empty' % (pack_name)) # The rules for the pack ref are as follows: # 1. If ref attribute is available, we used that # 2. If pack_name is available we use that (this only applies to packs # 2hich are in sub-directories) # 2. If attribute is not available, but pack name is and pack name meets the valid name # criteria, we use that if content.get('ref', None): content['ref'] = content['ref'] elif re.match(PACK_REF_WHITELIST_REGEX, pack_name): content['ref'] = pack_name else: if re.match(PACK_REF_WHITELIST_REGEX, content['name']): content['ref'] = content['name'] else: raise ValueError( 'Pack name "%s" contains invalid characters and "ref" ' 'attribute is not available' % (content['name'])) # Note: We use a ref if available, if not we fall back to pack name # (pack directory name) content['ref'] = content.get('ref', pack_name) # Include a list of pack files pack_file_list = get_file_list(directory=pack_dir, exclude_patterns=EXCLUDE_FILE_PATTERNS) content['files'] = pack_file_list # Note: If some version values are not explicitly surrounded by quotes they are recognized # as numbers so we cast them to string if 'version' in content: content['version'] = str(content['version']) pack_api = PackAPI(**content) pack_api.validate() pack_db = PackAPI.to_model(pack_api) try: pack_db.id = Pack.get_by_ref(content['ref']).id except StackStormDBObjectNotFoundError: LOG.debug('Pack %s not found. Creating new one.', pack_name) pack_db = Pack.add_or_update(pack_db) LOG.debug('Pack %s registered.' % (pack_name)) return pack_db
def _insert_common_mock_resources(self): pack_1_db = PackDB(name='test_pack_1', ref='test_pack_1', description='', version='0.1.0', author='foo', email='*****@*****.**') pack_1_db = Pack.add_or_update(pack_1_db) self.resources['pack_1'] = pack_1_db pack_2_db = PackDB(name='test_pack_2', ref='test_pack_2', description='', version='0.1.0', author='foo', email='*****@*****.**') pack_2_db = Pack.add_or_update(pack_2_db) self.resources['pack_2'] = pack_2_db
def _delete_pack_db_object(self, pack): try: pack_db = Pack.get_by_name(value=pack) except ValueError: self.logger.exception('Pack DB object not found') return try: Pack.delete(pack_db) except: self.logger.exception('Failed to remove DB object %s.', pack_db)
def _delete_pack_db_object(self, pack): try: pack_db = Pack.get_by_name(value=pack) except StackStormDBObjectNotFoundError: self.logger.exception('Pack DB object not found') return try: Pack.delete(pack_db) except: self.logger.exception('Failed to remove DB object %s.', pack_db)
def test_run(self): pack = 'dummy_pack_1' # Verify all the resources are there pack_dbs = Pack.query(ref=pack) action_dbs = Action.query(pack=pack) alias_dbs = ActionAlias.query(pack=pack) rule_dbs = Rule.query(pack=pack) sensor_dbs = Sensor.query(pack=pack) trigger_type_dbs = TriggerType.query(pack=pack) policy_dbs = Policy.query(pack=pack) config_schema_dbs = ConfigSchema.query(pack=pack) config_dbs = Config.query(pack=pack) self.assertEqual(len(pack_dbs), 1) self.assertEqual(len(action_dbs), 1) self.assertEqual(len(alias_dbs), 2) self.assertEqual(len(rule_dbs), 1) self.assertEqual(len(sensor_dbs), 3) self.assertEqual(len(trigger_type_dbs), 4) self.assertEqual(len(policy_dbs), 2) self.assertEqual(len(config_schema_dbs), 1) self.assertEqual(len(config_dbs), 1) # Run action action = self.get_action_instance() action.run(packs=[pack]) # Make sure all resources have been removed from the db pack_dbs = Pack.query(ref=pack) action_dbs = Action.query(pack=pack) alias_dbs = ActionAlias.query(pack=pack) rule_dbs = Rule.query(pack=pack) sensor_dbs = Sensor.query(pack=pack) trigger_type_dbs = TriggerType.query(pack=pack) policy_dbs = Policy.query(pack=pack) config_schema_dbs = ConfigSchema.query(pack=pack) config_dbs = Config.query(pack=pack) self.assertEqual(len(pack_dbs), 0) self.assertEqual(len(action_dbs), 0) self.assertEqual(len(alias_dbs), 0) self.assertEqual(len(rule_dbs), 0) self.assertEqual(len(sensor_dbs), 0) self.assertEqual(len(trigger_type_dbs), 0) self.assertEqual(len(policy_dbs), 0) self.assertEqual(len(config_schema_dbs), 0) self.assertEqual(len(config_dbs), 0)
def test_run(self): pack = 'dummy_pack_1' # Verify all the resources are there pack_dbs = Pack.query(ref=pack) action_dbs = Action.query(pack=pack) alias_dbs = ActionAlias.query(pack=pack) rule_dbs = Rule.query(pack=pack) sensor_dbs = Sensor.query(pack=pack) trigger_type_dbs = TriggerType.query(pack=pack) policy_dbs = Policy.query(pack=pack) config_schema_dbs = ConfigSchema.query(pack=pack) config_dbs = Config.query(pack=pack) self.assertEqual(len(pack_dbs), 1) self.assertEqual(len(action_dbs), 1) self.assertEqual(len(alias_dbs), 3) self.assertEqual(len(rule_dbs), 1) self.assertEqual(len(sensor_dbs), 3) self.assertEqual(len(trigger_type_dbs), 4) self.assertEqual(len(policy_dbs), 2) self.assertEqual(len(config_schema_dbs), 1) self.assertEqual(len(config_dbs), 1) # Run action action = self.get_action_instance() action.run(packs=[pack]) # Make sure all resources have been removed from the db pack_dbs = Pack.query(ref=pack) action_dbs = Action.query(pack=pack) alias_dbs = ActionAlias.query(pack=pack) rule_dbs = Rule.query(pack=pack) sensor_dbs = Sensor.query(pack=pack) trigger_type_dbs = TriggerType.query(pack=pack) policy_dbs = Policy.query(pack=pack) config_schema_dbs = ConfigSchema.query(pack=pack) config_dbs = Config.query(pack=pack) self.assertEqual(len(pack_dbs), 0) self.assertEqual(len(action_dbs), 0) self.assertEqual(len(alias_dbs), 0) self.assertEqual(len(rule_dbs), 0) self.assertEqual(len(sensor_dbs), 0) self.assertEqual(len(trigger_type_dbs), 0) self.assertEqual(len(policy_dbs), 0) self.assertEqual(len(config_schema_dbs), 0) self.assertEqual(len(config_dbs), 0)
def test_register_pack_old_style_non_semver_version_is_normalized_to_valid_version(self): # Verify DB is empty pack_dbs = Pack.get_all() self.assertEqual(len(pack_dbs), 0) registrar = ResourceRegistrar(use_pack_cache=False) registrar._pack_loader.get_packs = mock.Mock() registrar._pack_loader.get_packs.return_value = {'dummy_pack_11': PACK_PATH_11} packs_base_paths = content_utils.get_packs_base_paths() registrar.register_packs(base_dirs=packs_base_paths) # Non-semver valid version 0.2 should be normalize to 0.2.0 pack_db = Pack.get_by_name('dummy_pack_11') self.assertEqual(pack_db.version, '0.2.0')
def test_get_pack_files_binary_files_are_excluded(self): binary_files = [ "icon.png", "etc/permissions.png", "etc/travisci.png", "etc/generate_new_token.png", ] pack_db = Pack.get_by_ref("dummy_pack_1") all_files_count = len(pack_db.files) non_binary_files_count = all_files_count - len(binary_files) resp = self.app.get("/v1/packs/views/files/dummy_pack_1") self.assertEqual(resp.status_int, http_client.OK) self.assertEqual(len(resp.json), non_binary_files_count) for file_path in binary_files: self.assertIn(file_path, pack_db.files) # But not in files controller response for file_path in binary_files: item = [ item for item in resp.json if item["file_path"] == file_path ] self.assertFalse(item)
def test_register_all_configs_with_config_schema_validation_validation_failure( self): # Verify DB is empty pack_dbs = Pack.get_all() config_dbs = Config.get_all() self.assertEqual(len(pack_dbs), 0) self.assertEqual(len(config_dbs), 0) registrar = ConfigsRegistrar(use_pack_cache=False, fail_on_failure=True, validate_configs=True) registrar._pack_loader.get_packs = mock.Mock() registrar._pack_loader.get_packs.return_value = { 'dummy_pack_6': PACK_6_PATH } # Register ConfigSchema for pack registrar._register_pack_db = mock.Mock() registrar._register_pack(pack_name='dummy_pack_5', pack_dir=PACK_6_PATH) packs_base_paths = content_utils.get_packs_base_paths() expected_msg = ( 'Failed validating attribute "regions" in config for pack "dummy_pack_6" ' '(.*?): 1000 is not of type u\'array\'') self.assertRaisesRegexp(ValueError, expected_msg, registrar.register_from_packs, base_dirs=packs_base_paths)
def test_uid_is_populated_on_save(self): pack_1_db = PackDB( ref="test_pack", name="test", description="foo", version="1.0.0", author="dev", email="*****@*****.**", ) pack_1_db = Pack.add_or_update(pack_1_db) pack_1_db.reload() self.assertEqual(pack_1_db.uid, "pack:test_pack") action_1_db = ActionDB( name="local", pack="core", ref="core.local", entry_point="", runner_type={"name": "local-shell-cmd"}, ) action_1_db = Action.add_or_update(action_1_db) action_1_db.reload() self.assertEqual(action_1_db.uid, "action:core:local")
def _update_pack_model(self, pack_name, data_files, written_file_paths): """ Update PackDB models (update files list). """ file_paths = [] # A list of paths relative to the pack directory for new files for file_path in written_file_paths: file_path = get_relative_path_to_pack(pack_name=pack_name, file_path=file_path) file_paths.append(file_path) pack_db = Pack.get_by_ref(pack_name) pack_db.files = set(pack_db.files) pack_db.files.update(set(file_paths)) pack_db.files = list(pack_db.files) pack_db = Pack.add_or_update(pack_db) return pack_db
def test_register_all_configs_with_config_schema_validation_validation_failure_1(self): # Verify DB is empty pack_dbs = Pack.get_all() config_dbs = Config.get_all() self.assertEqual(len(pack_dbs), 0) self.assertEqual(len(config_dbs), 0) registrar = ConfigsRegistrar(use_pack_cache=False, fail_on_failure=True, validate_configs=True) registrar._pack_loader.get_packs = mock.Mock() registrar._pack_loader.get_packs.return_value = {'dummy_pack_6': PACK_6_PATH} # Register ConfigSchema for pack registrar._register_pack_db = mock.Mock() registrar._register_pack(pack_name='dummy_pack_5', pack_dir=PACK_6_PATH) packs_base_paths = content_utils.get_packs_base_paths() if six.PY3: expected_msg = ('Failed validating attribute "regions" in config for pack ' '"dummy_pack_6" (.*?): 1000 is not of type \'array\'') else: expected_msg = ('Failed validating attribute "regions" in config for pack ' '"dummy_pack_6" (.*?): 1000 is not of type u\'array\'') self.assertRaisesRegexp(ValueError, expected_msg, registrar.register_from_packs, base_dirs=packs_base_paths)
def _update_pack_model(self, pack_ref, data_files, written_file_paths): """ Update PackDB models (update files list). """ file_paths = [] # A list of paths relative to the pack directory for new files for file_path in written_file_paths: file_path = get_relative_path_to_pack_file(pack_ref=pack_ref, file_path=file_path) file_paths.append(file_path) pack_db = Pack.get_by_ref(pack_ref) pack_db.files = set(pack_db.files) pack_db.files.update(set(file_paths)) pack_db.files = list(pack_db.files) pack_db = Pack.add_or_update(pack_db) return pack_db
def test_register_packs(self): # Verify DB is empty pack_dbs = Pack.get_all() config_schema_dbs = ConfigSchema.get_all() self.assertEqual(len(pack_dbs), 0) self.assertEqual(len(config_schema_dbs), 0) registrar = ResourceRegistrar(use_pack_cache=False) registrar._pack_loader.get_packs = mock.Mock() registrar._pack_loader.get_packs.return_value = { 'dummy_pack_1': PACK_PATH_1 } packs_base_paths = content_utils.get_packs_base_paths() registrar.register_packs(base_dirs=packs_base_paths) # Verify pack and schema have been registered pack_dbs = Pack.get_all() config_schema_dbs = ConfigSchema.get_all() self.assertEqual(len(pack_dbs), 1) self.assertEqual(len(config_schema_dbs), 1) pack_db = pack_dbs[0] config_schema_db = config_schema_dbs[0] self.assertEqual(pack_db.name, 'dummy_pack_1') self.assertEqual(len(pack_db.contributors), 2) self.assertEqual(pack_db.contributors[0], 'John Doe1 <*****@*****.**>') self.assertEqual(pack_db.contributors[1], 'John Doe2 <*****@*****.**>') self.assertTrue('api_key' in config_schema_db.attributes) self.assertTrue('api_secret' in config_schema_db.attributes) # Verify pack_db.files is correct and doesn't contain excluded files (*.pyc, .git/*, etc.) # Note: We can't test that .git/* files are excluded since git doesn't allow you to add # .git directory to existing repo index :/ excluded_files = [ '__init__.pyc', 'actions/dummy1.pyc', 'actions/dummy2.pyc', ] for excluded_file in excluded_files: self.assertTrue(excluded_file not in pack_db.files)
def test_post_include_files(self): # Verify initial state pack_db = Pack.get_by_ref(ACTION_12["pack"]) self.assertTrue("actions/filea.txt" not in pack_db.files) action = copy.deepcopy(ACTION_12) action["data_files"] = [{"file_path": "filea.txt", "content": "test content"}] post_resp = self.__do_post(action) # Verify file has been written on disk for file_path in self.to_delete_files: self.assertTrue(os.path.exists(file_path)) # Verify PackDB.files has been updated pack_db = Pack.get_by_ref(ACTION_12["pack"]) self.assertTrue("actions/filea.txt" in pack_db.files) self.__do_delete(self.__get_action_id(post_resp))
def test_post_include_files(self): # Verify initial state pack_db = Pack.get_by_ref(ACTION_12["pack"]) self.assertNotIn("actions/filea.txt", pack_db.files) action = copy.deepcopy(ACTION_12) action["data_files"] = [{"file_path": "filea.txt", "content": "test content"}] post_resp = self.__do_post(action) # Verify file has been written on disk for file_path in self.to_delete_files: self.assertTrue(os.path.exists(file_path)) # Verify PackDB.files has been updated pack_db = Pack.get_by_ref(ACTION_12["pack"]) self.assertIn("actions/filea.txt", pack_db.files) self.__do_delete(self.__get_action_id(post_resp))
def test_register_packs(self): # Verify DB is empty pack_dbs = Pack.get_all() config_schema_dbs = ConfigSchema.get_all() self.assertEqual(len(pack_dbs), 0) self.assertEqual(len(config_schema_dbs), 0) registrar = ResourceRegistrar(use_pack_cache=False) registrar._pack_loader.get_packs = mock.Mock() registrar._pack_loader.get_packs.return_value = {'dummy_pack_1': PACK_PATH_1} packs_base_paths = content_utils.get_packs_base_paths() registrar.register_packs(base_dirs=packs_base_paths) # Verify pack and schema have been registered pack_dbs = Pack.get_all() config_schema_dbs = ConfigSchema.get_all() self.assertEqual(len(pack_dbs), 1) self.assertEqual(len(config_schema_dbs), 1) pack_db = pack_dbs[0] config_schema_db = config_schema_dbs[0] self.assertEqual(pack_db.name, 'dummy_pack_1') self.assertEqual(len(pack_db.contributors), 2) self.assertEqual(pack_db.contributors[0], 'John Doe1 <*****@*****.**>') self.assertEqual(pack_db.contributors[1], 'John Doe2 <*****@*****.**>') self.assertTrue('api_key' in config_schema_db.attributes) self.assertTrue('api_secret' in config_schema_db.attributes) # Verify pack_db.files is correct and doesn't contain excluded files (*.pyc, .git/*, etc.) # Note: We can't test that .git/* files are excluded since git doesn't allow you to add # .git directory to existing repo index :/ excluded_files = [ '__init__.pyc', 'actions/dummy1.pyc', 'actions/dummy2.pyc', ] for excluded_file in excluded_files: self.assertTrue(excluded_file not in pack_db.files)
def test_post_include_files(self): # Verify initial state pack_db = Pack.get_by_ref(ACTION_12['pack']) self.assertTrue('actions/filea.txt' not in pack_db.files) action = copy.deepcopy(ACTION_12) action['data_files'] = [{ 'file_path': 'filea.txt', 'content': 'test content' }] post_resp = self.__do_post(action) # Verify file has been written on disk for file_path in self.to_delete_files: self.assertTrue(os.path.exists(file_path)) # Verify PackDB.files has been updated pack_db = Pack.get_by_ref(ACTION_12['pack']) self.assertTrue('actions/filea.txt' in pack_db.files) self.__do_delete(self.__get_action_id(post_resp))
def setUpClass(cls): super(PacksControllerTestCase, cls).setUpClass() cls.pack_db_1 = PackDB( name="pack1", description="foo", version="0.1.0", author="foo", email="*****@*****.**", ref="pack1", ) cls.pack_db_2 = PackDB( name="pack2", description="foo", version="0.1.0", author="foo", email="*****@*****.**", ref="pack2", ) cls.pack_db_3 = PackDB( name="pack3-name", ref="pack3-ref", description="foo", version="0.1.0", author="foo", email="*****@*****.**", ) Pack.add_or_update(cls.pack_db_1) Pack.add_or_update(cls.pack_db_2) Pack.add_or_update(cls.pack_db_3)
def _insert_common_mock_resources(self): pack_1_db = PackDB( name="test_pack_1", ref="test_pack_1", description="", version="0.1.0", author="foo", email="*****@*****.**", ) pack_1_db = Pack.add_or_update(pack_1_db) self.resources["pack_1"] = pack_1_db pack_2_db = PackDB( name="test_pack_2", ref="test_pack_2", description="", version="0.1.0", author="foo", email="*****@*****.**", ) pack_2_db = Pack.add_or_update(pack_2_db) self.resources["pack_2"] = pack_2_db
def test_post_include_files(self): # Verify initial state pack_db = Pack.get_by_ref(ACTION_12['pack']) self.assertTrue('actions/filea.txt' not in pack_db.files) action = copy.deepcopy(ACTION_12) action['data_files'] = [ { 'file_path': 'filea.txt', 'content': 'test content' } ] post_resp = self.__do_post(action) # Verify file has been written on disk for file_path in self.to_delete_files: self.assertTrue(os.path.exists(file_path)) # Verify PackDB.files has been updated pack_db = Pack.get_by_ref(ACTION_12['pack']) self.assertTrue('actions/filea.txt' in pack_db.files) self.__do_delete(self.__get_action_id(post_resp))
def _register_pack_db(self, pack_name, pack_dir): pack_name = pack_name or '' manifest_path = os.path.join(pack_dir, MANIFEST_FILE_NAME) if not os.path.isfile(manifest_path): raise ValueError('Pack "%s" is missing %s file' % (pack_name, MANIFEST_FILE_NAME)) content = self._meta_loader.load(manifest_path) if not content: raise ValueError('Pack "%s" metadata file is empty' % (pack_name)) # The rules for the pack ref are as follows: # 1. If ref attribute is available, we used that # 2. If pack_name is available we use that (this only applies to packs # 2hich are in sub-directories) # 2. If attribute is not available, but pack name is and pack name meets the valid name # criteria, we use that content['ref'] = get_pack_ref_from_metadata( metadata=content, pack_directory_name=pack_name) # Include a list of pack files pack_file_list = get_file_list(directory=pack_dir, exclude_patterns=EXCLUDE_FILE_PATTERNS) content['files'] = pack_file_list pack_api = PackAPI(**content) pack_api.validate() pack_db = PackAPI.to_model(pack_api) try: pack_db.id = Pack.get_by_ref(content['ref']).id except StackStormDBObjectNotFoundError: LOG.debug('Pack %s not found. Creating new one.', pack_name) pack_db = Pack.add_or_update(pack_db) LOG.debug('Pack %s registered.' % (pack_name)) return pack_db
def test_register_all_configs_invalid_config_no_config_schema(self): # verify_ configs is on, but ConfigSchema for the pack doesn't exist so # validation should proceed normally # Verify DB is empty pack_dbs = Pack.get_all() config_dbs = Config.get_all() self.assertEqual(len(pack_dbs), 0) self.assertEqual(len(config_dbs), 0) registrar = ConfigsRegistrar(use_pack_cache=False, validate_configs=False) registrar._pack_loader.get_packs = mock.Mock() registrar._pack_loader.get_packs.return_value = {'dummy_pack_6': PACK_6_PATH} packs_base_paths = content_utils.get_packs_base_paths() registrar.register_from_packs(base_dirs=packs_base_paths) # Verify pack and schema have been registered pack_dbs = Pack.get_all() config_dbs = Config.get_all() self.assertEqual(len(pack_dbs), 1) self.assertEqual(len(config_dbs), 1)
def test_uid_is_populated_on_save(self): pack_1_db = PackDB(ref='test_pack', name='test', description='foo', version='1.0.0', author='dev', email='*****@*****.**') pack_1_db = Pack.add_or_update(pack_1_db) pack_1_db.reload() self.assertEqual(pack_1_db.uid, 'pack:test_pack') action_1_db = ActionDB(name='local', pack='core', ref='core.local', entry_point='', runner_type={'name': 'local-shell-cmd'}) action_1_db = Action.add_or_update(action_1_db) action_1_db.reload() self.assertEqual(action_1_db.uid, 'action:core:local')
def test_uid_is_populated_on_save(self): pack_1_db = PackDB(ref='test_pack', name='test', description='foo', version='1.0', author='dev', email='*****@*****.**') pack_1_db = Pack.add_or_update(pack_1_db) pack_1_db.reload() self.assertEqual(pack_1_db.uid, 'pack:test_pack') action_1_db = ActionDB(name='local', pack='core', ref='core.local', entry_point='', runner_type={'name': 'local-shell-cmd'}) action_1_db = Action.add_or_update(action_1_db) action_1_db.reload() self.assertEqual(action_1_db.uid, 'action:core:local')
def _register_pack_db(self, pack_name, pack_dir): pack_name = pack_name or '' manifest_path = os.path.join(pack_dir, MANIFEST_FILE_NAME) if not os.path.isfile(manifest_path): raise ValueError('Pack "%s" is missing %s file' % (pack_name, MANIFEST_FILE_NAME)) content = self._meta_loader.load(manifest_path) if not content: raise ValueError('Pack "%s" metadata file is empty' % (pack_name)) # The rules for the pack ref are as follows: # 1. If ref attribute is available, we used that # 2. If pack_name is available we use that (this only applies to packs # 2hich are in sub-directories) # 2. If attribute is not available, but pack name is and pack name meets the valid name # criteria, we use that content['ref'] = get_pack_ref_from_metadata(metadata=content, pack_directory_name=pack_name) # Include a list of pack files pack_file_list = get_file_list(directory=pack_dir, exclude_patterns=EXCLUDE_FILE_PATTERNS) content['files'] = pack_file_list pack_api = PackAPI(**content) pack_api.validate() pack_db = PackAPI.to_model(pack_api) try: pack_db.id = Pack.get_by_ref(content['ref']).id except StackStormDBObjectNotFoundError: LOG.debug('Pack %s not found. Creating new one.', pack_name) pack_db = Pack.add_or_update(pack_db) LOG.debug('Pack %s registered.' % (pack_name)) return pack_db
def test_register_pack_arbitrary_properties_are_allowed(self): # Test registering a pack which has "arbitrary" properties in pack.yaml # We support this use-case (ignore properties which are not defined on the PackAPI model) # so we can add new attributes in a new version without breaking existing installations. registrar = ResourceRegistrar(use_pack_cache=False) registrar._pack_loader.get_packs = mock.Mock() registrar._pack_loader.get_packs.return_value = { 'dummy_pack_20': PACK_PATH_20, } packs_base_paths = content_utils.get_packs_base_paths() registrar.register_packs(base_dirs=packs_base_paths) # Ref is provided pack_db = Pack.get_by_name('dummy_pack_20') self.assertEqual(pack_db.ref, 'dummy_pack_20_ref') self.assertEqual(len(pack_db.contributors), 0)
def setUp(cls): super(PackControllerRBACTestCase, cls).setUp() cls.pack_db_1 = PackDB(name='pack1', description='foo', version='0.1.0', author='foo', email='*****@*****.**', ref='pack1') cls.pack_db_2 = PackDB(name='pack2', description='foo', version='0.1.0', author='foo', email='*****@*****.**', ref='pack2') cls.pack_db_3 = PackDB(name='pack3-name', ref='pack3-ref', description='foo', version='0.1.0', author='foo', email='*****@*****.**') Pack.add_or_update(cls.pack_db_1) Pack.add_or_update(cls.pack_db_2) Pack.add_or_update(cls.pack_db_3)
def setUp(self): super(UnloadActionTestCase, self).setUp() # Register mock pack # Verify DB is empty pack_dbs = Pack.get_all() config_schema_dbs = ConfigSchema.get_all() config_dbs = Config.get_all() self.assertEqual(len(pack_dbs), 0) self.assertEqual(len(config_schema_dbs), 0) self.assertEqual(len(config_dbs), 0) # Register the pack with all the content # TODO: Don't use pack cache cfg.CONF.set_override(name='all', override=True, group='register') cfg.CONF.set_override(name='pack', override=PACK_PATH_1, group='register') cfg.CONF.set_override(name='no_fail_on_failure', override=True, group='register') register_content()
def test_register_all_configs_with_config_schema_validation_validation_failure_2( self, ): # Verify DB is empty pack_dbs = Pack.get_all() config_dbs = Config.get_all() self.assertEqual(len(pack_dbs), 0) self.assertEqual(len(config_dbs), 0) registrar = ConfigsRegistrar(use_pack_cache=False, fail_on_failure=True, validate_configs=True) registrar._pack_loader.get_packs = mock.Mock() registrar._pack_loader.get_packs.return_value = { "dummy_pack_19": PACK_19_PATH } # Register ConfigSchema for pack registrar._register_pack_db = mock.Mock() registrar._register_pack(pack_name="dummy_pack_19", pack_dir=PACK_19_PATH) packs_base_paths = content_utils.get_packs_base_paths() if six.PY3: expected_msg = ( 'Failed validating attribute "instances.0.alias" in config for pack ' "\"dummy_pack_19\" (.*?): {'not': 'string'} is not of type " "'string'") else: expected_msg = ( 'Failed validating attribute "instances.0.alias" in config for pack ' "\"dummy_pack_19\" (.*?): {'not': 'string'} is not of type " "u'string'") self.assertRaisesRegexp( ValueError, expected_msg, registrar.register_from_packs, base_dirs=packs_base_paths, )
def test_get_pack_files_binary_files_are_excluded(self): binary_files = [ 'icon.png', 'etc/permissions.png', 'etc/travisci.png', 'etc/generate_new_token.png' ] pack_db = Pack.get_by_ref('dummy_pack_1') all_files_count = len(pack_db.files) non_binary_files_count = all_files_count - len(binary_files) resp = self.app.get('/v1/packs/views/files/dummy_pack_1') self.assertEqual(resp.status_int, httplib.OK) self.assertEqual(len(resp.json), non_binary_files_count) for file_path in binary_files: self.assertTrue(file_path in pack_db.files) # But not in files controller response for file_path in binary_files: item = [item for item in resp.json if item['file_path'] == file_path] self.assertFalse(item)
def get_pack_common_libs_path_for_pack_ref(pack_ref): pack_db = Pack.get_by_ref(pack_ref) pack_common_libs_path = get_pack_common_libs_path_for_pack_db(pack_db=pack_db) return pack_common_libs_path
def get_pack_by_ref(pack_ref): """ Retrieve PackDB by the provided reference. """ pack_db = Pack.get_by_ref(pack_ref) return pack_db