def _get_api_models_from_disk(artifact_type, pack_dir=None): loader = ContentPackLoader() artifacts = None if pack_dir: artifacts_dir = loader.get_content_from_pack(pack_dir, artifact_type) pack_name = os.path.basename(os.path.normpath(pack_dir)) artifacts = {pack_name: artifacts_dir} else: packs_dirs = content_utils.get_packs_base_paths() artifacts = loader.get_content(packs_dirs, artifact_type) artifacts_dict = {} for pack_name, pack_path in artifacts.items(): artifacts_paths = registrar.get_resources_from_pack(pack_path) for artifact_path in artifacts_paths: artifact = meta_loader.load(artifact_path) if artifact_type == "sensors": sensors_dir = os.path.dirname(artifact_path) sensor_file_path = os.path.join(sensors_dir, artifact["entry_point"]) artifact["artifact_uri"] = "file://" + sensor_file_path name = artifact.get("name", None) or artifact.get("class_name", None) if not artifact.get("pack", None): artifact["pack"] = pack_name ref = ResourceReference.to_string_reference(name=name, pack=pack_name) API_MODEL = API_MODELS_ARTIFACT_TYPES[artifact_type] # Following conversions are required because we add some fields with # default values in db model. If we don't do these conversions, # we'll see a unnecessary diff for those fields. artifact_api = API_MODEL(**artifact) artifact_db = API_MODEL.to_model(artifact_api) artifact_api = API_MODEL.from_model(artifact_db) artifacts_dict[ref] = artifact_api return artifacts_dict
def _get_action_alias_db_by_name(self, name): """ Retrieve ActionAlias DB object for the provided alias name. """ base_pack_path = self._get_base_pack_path() pack_yaml_path = os.path.join(base_pack_path, MANIFEST_FILE_NAME) if os.path.isfile(pack_yaml_path): # 1. 1st try to infer pack name from pack metadata file meta_loader = MetaLoader() pack_metadata = meta_loader.load(pack_yaml_path) pack = get_pack_ref_from_metadata(metadata=pack_metadata) else: # 2. If pack.yaml is not available, fail back to directory name # Note: For this to work, directory name needs to match pack name _, pack = os.path.split(base_pack_path) pack_loader = ContentPackLoader() registrar = AliasesRegistrar(use_pack_cache=False) aliases_path = pack_loader.get_content_from_pack(pack_dir=base_pack_path, content_type='aliases') aliases = registrar._get_aliases_from_pack(aliases_dir=aliases_path) for alias_path in aliases: action_alias_db = registrar._get_action_alias_db(pack=pack, action_alias=alias_path, ignore_metadata_file_error=True) if action_alias_db.name == name: return action_alias_db raise ValueError('Alias with name "%s" not found' % (name))
def test_get_sensors(self): packs_base_path = os.path.join(RESOURCES_DIR, "packs/") loader = ContentPackLoader() pack_sensors = loader.get_content( base_dirs=[packs_base_path], content_type="sensors" ) self.assertIsNotNone(pack_sensors.get("pack1", None))
def __init__(self, use_pack_cache=True, use_runners_cache=False, fail_on_failure=False): """ :param use_pack_cache: True to cache which packs have been registered in memory and making sure packs are only registered once. :type use_pack_cache: ``bool`` :param use_runners_cache: True to cache RunnerTypeDB objects in memory to reduce load on the database. :type use_runners_cache: ``bool`` :param fail_on_failure: Throw an exception if resource registration fails. :type fail_on_failure: ``bool`` """ self._use_pack_cache = use_pack_cache self._use_runners_cache = use_runners_cache self._fail_on_failure = fail_on_failure self._meta_loader = MetaLoader() self._pack_loader = ContentPackLoader() # Maps runner name -> RunnerTypeDB self._runner_type_db_cache = {}
def test_get_sensors(self): packs_base_path = os.path.join( os.path.dirname(os.path.realpath(__file__)), 'resources/packs/') loader = ContentPackLoader() pack_sensors = loader.get_content(base_dir=packs_base_path, content_type='sensors') self.assertTrue(pack_sensors.get('pack1', None) is not None)
def _get_user_sensors(): sensors_dict = defaultdict(list) pack_loader = ContentPackLoader() sensor_loader = SensorLoader() packs = pack_loader.get_content(base_dir=cfg.CONF.content.packs_base_path, content_type='sensors') for pack, sensor_dir in six.iteritems(packs): try: LOG.info('Loading sensors from pack: %s, dir: %s', pack, sensor_dir) if _is_requirements_ok(sensor_dir): base_dir = os.path.realpath(sensor_dir) pack_sensors = sensor_loader.get_sensors(base_dir=base_dir) # Include content pack name on the sensor class # TODO: This is nasty pack_sensors_augmented = defaultdict(list) for filename, sensors in pack_sensors.iteritems(): for sensor in sensors: sensor.pack = pack pack_sensors_augmented[filename].append(sensor) sensors_dict.update(pack_sensors_augmented) else: LOG.warning('Not registering sensors in sensor_dir: %s.', sensor_dir) except: LOG.exception('Failed loading sensors from dir: %s' % sensor_dir) return sensors_dict
def test_get_content_from_pack_no_sensors(self): loader = ContentPackLoader() pack_path = os.path.join(RESOURCES_DIR, "packs/pack2") result = loader.get_content_from_pack(pack_dir=pack_path, content_type="sensors") self.assertEqual(result, None)
def test_get_content_from_pack_success(self): loader = ContentPackLoader() pack_path = os.path.join(RESOURCES_DIR, "packs/pack1") sensors = loader.get_content_from_pack(pack_dir=pack_path, content_type="sensors") self.assertTrue(sensors.endswith("packs/pack1/sensors"))
def _get_action_alias_db_by_name(self, name): """ Retrieve ActionAlias DB object for the provided alias name. """ base_pack_path = self._get_base_pack_path() pack_yaml_path = os.path.join(base_pack_path, MANIFEST_FILE_NAME) if os.path.isfile(pack_yaml_path): # 1. 1st try to infer pack name from pack metadata file meta_loader = MetaLoader() pack_metadata = meta_loader.load(pack_yaml_path) pack = get_pack_ref_from_metadata(metadata=pack_metadata) else: # 2. If pack.yaml is not available, fail back to directory name # Note: For this to work, directory name needs to match pack name _, pack = os.path.split(base_pack_path) pack_loader = ContentPackLoader() registrar = AliasesRegistrar(use_pack_cache=False) aliases_path = pack_loader.get_content_from_pack(pack_dir=base_pack_path, content_type='aliases') aliases = registrar._get_aliases_from_pack(aliases_dir=aliases_path) for alias_path in aliases: action_alias_db = registrar._get_action_alias_db(pack=pack, action_alias=alias_path) if action_alias_db.name == name: return action_alias_db raise ValueError('Alias with name "%s" not found' % (name))
def test_invalid_content_type(self): packs_base_path = os.path.join( os.path.dirname(os.path.realpath(__file__)), 'resources/packs/') loader = ContentPackLoader() try: loader.get_content(base_dir=packs_base_path, content_type='stuff') self.fail('Asking for invalid content should have thrown.') except: pass
def __init__(self, use_pack_cache=True): """ :param use_pack_cache: True to cache which packs have been registered in memory and making sure packs are only registered once. :type use_pack_cache: ``bool`` """ self._use_pack_cache = use_pack_cache self._meta_loader = MetaLoader() self._pack_loader = ContentPackLoader()
def register_actions_from_packs(self, base_dir): pack_loader = ContentPackLoader() dirs = pack_loader.get_content(base_dir=base_dir, content_type='actions') for pack, actions_dir in six.iteritems(dirs): try: actions = self._get_actions_from_pack(actions_dir) self._register_actions_from_pack(pack, actions) except: LOG.exception('Failed registering all actions from pack: %s', actions_dir)
def test_get_sensors_pack_missing_sensors(self): loader = ContentPackLoader() fail_pack_path = os.path.join( os.path.dirname(os.path.realpath(__file__)), 'resources/packs/pack2') self.assertTrue(os.path.exists(fail_pack_path)) try: loader._get_sensors(fail_pack_path) self.fail('Empty packs must throw exception.') except: pass
def register_rules_from_packs(self, base_dir): pack_loader = ContentPackLoader() dirs = pack_loader.get_content(base_dir=base_dir, content_type='rules') for pack, rules_dir in six.iteritems(dirs): try: LOG.info('Registering rules from pack: %s', pack) rules = self._get_rules_from_pack(rules_dir) self._register_rules_from_pack(pack, rules) except: LOG.exception('Failed registering all rules from pack: %s', rules_dir)
def __init__(self, use_pack_cache=True, fail_on_failure=False): """ :param use_pack_cache: True to cache which packs have been registered in memory and making sure packs are only registered once. :type use_pack_cache: ``bool`` :param fail_on_failure: Throw an exception if resource registration fails. :type fail_on_failure: ``bool`` """ self._use_pack_cache = use_pack_cache self._fail_on_failure = fail_on_failure self._meta_loader = MetaLoader() self._pack_loader = ContentPackLoader()
def test_invalid_content_type(self): packs_base_path = os.path.join(RESOURCES_DIR, 'packs/') loader = ContentPackLoader() self.assertRaises(ValueError, loader.get_content, base_dirs=[packs_base_path], content_type='stuff')
def _load_config_schemas(): config_schemas = {} packs = ContentPackLoader().get_packs(content_utils.get_packs_base_paths()) for pack_name, pack_dir in six.iteritems(packs): config_schema_path = os.path.join(pack_dir, CONFIG_SCHEMA_FILE_NAME) if not os.path.isfile(config_schema_path): # Note: Config schema is optional continue values = MetaLoader().load(config_schema_path) if not values: raise ValueError('Config schema "%s" is empty and invalid.' % (config_schema_path)) content = {} content['pack'] = pack_name content['attributes'] = values config_schema_api = ConfigSchemaAPI(**content) config_schema_api = config_schema_api.validate() config_schemas[pack_name] = values return config_schemas
def test_get_content_from_pack_directory_doesnt_exist(self): loader = ContentPackLoader() pack_path = os.path.join(RESOURCES_DIR, 'packs/pack100') message_regex = 'Directory .*? doesn\'t exist' self.assertRaisesRegexp(ValueError, message_regex, loader.get_content_from_pack, pack_dir=pack_path, content_type='sensors')
def register_sensors_from_packs(self, base_dir): pack_loader = ContentPackLoader() dirs = pack_loader.get_content(base_dir=base_dir, content_type='sensors') # Add system sensors to the core pack dirs['core'] = {} dirs['core'] = SYSTEM_SENSORS_PATH for pack, sensors_dir in six.iteritems(dirs): try: LOG.info('Registering sensors from pack: %s', pack) sensors = self._get_sensors_from_pack(sensors_dir) self._register_sensors_from_pack(pack=pack, sensors=sensors) except Exception as e: LOG.exception('Failed registering all sensors from pack "%s": %s', sensors_dir, str(e))
def test_get_content_multiple_directories(self): packs_base_path_1 = os.path.join(RESOURCES_DIR, 'packs/') packs_base_path_2 = os.path.join(RESOURCES_DIR, 'packs2/') base_dirs = [packs_base_path_1, packs_base_path_2] LOG.warning = Mock() loader = ContentPackLoader() sensors = loader.get_content(base_dirs=base_dirs, content_type='sensors') self.assertTrue('pack1' in sensors) # from packs/ self.assertTrue('pack3' in sensors) # from packs2/ # Assert that a warning is emitted when a duplicated pack is found expected_msg = ('Pack "pack1" already found in ' '"%s/packs/", ignoring content from ' '"%s/packs2/"' % (RESOURCES_DIR, RESOURCES_DIR)) LOG.warning.assert_called_once_with(expected_msg)
def test_get_content_from_pack_no_sensors(self): loader = ContentPackLoader() pack_path = os.path.join(RESOURCES_DIR, 'packs/pack2') message_regex = 'No sensors found' self.assertRaisesRegexp(ValueError, message_regex, loader.get_content_from_pack, pack_dir=pack_path, content_type='sensors')
def _get_action_alias_db_by_name(self, name): """ Retrieve ActionAlias DB object for the provided alias name. """ base_pack_path = self._get_base_pack_path() _, pack = os.path.split(base_pack_path) pack_loader = ContentPackLoader() registrar = AliasesRegistrar(use_pack_cache=False) aliases_path = pack_loader.get_content_from_pack(pack_dir=base_pack_path, content_type='aliases') aliases = registrar._get_aliases_from_pack(aliases_dir=aliases_path) for alias_path in aliases: action_alias_db = registrar._get_action_alias_db(pack=pack, action_alias=alias_path) if action_alias_db.name == name: return action_alias_db raise ValueError('Alias with name "%s" not found' % (name))
def _get_api_models_from_disk(artifact_type, pack_dir=None): loader = ContentPackLoader() artifacts = None if pack_dir: artifacts_dir = loader.get_content_from_pack(pack_dir, artifact_type) pack_name = os.path.basename(os.path.normpath(pack_dir)) artifacts = {pack_name: artifacts_dir} else: packs_dirs = content_utils.get_packs_base_paths() artifacts = loader.get_content(packs_dirs, artifact_type) artifacts_dict = {} for pack_name, pack_path in artifacts.items(): artifacts_paths = registrar.get_resources_from_pack(pack_path) for artifact_path in artifacts_paths: artifact = meta_loader.load(artifact_path) if artifact_type == 'sensors': sensors_dir = os.path.dirname(artifact_path) sensor_file_path = os.path.join(sensors_dir, artifact['entry_point']) artifact['artifact_uri'] = 'file://' + sensor_file_path name = artifact.get('name', None) or artifact.get( 'class_name', None) if not artifact.get('pack', None): artifact['pack'] = pack_name ref = ResourceReference.to_string_reference(name=name, pack=pack_name) API_MODEL = API_MODELS_ARTIFACT_TYPES[artifact_type] # Following conversions are required because we add some fields with # default values in db model. If we don't do these conversions, # we'll see a unnecessary diff for those fields. artifact_api = API_MODEL(**artifact) artifact_db = API_MODEL.to_model(artifact_api) artifact_api = API_MODEL.from_model(artifact_db) artifacts_dict[ref] = artifact_api return artifacts_dict
def _load_actions(): actions = {} action_dirs = ContentPackLoader().get_content(content_utils.get_packs_base_paths(), 'actions') for pack in action_dirs: for action_path in ActionsRegistrar().get_resources_from_pack(action_dirs[pack]): content = MetaLoader().load(action_path) ref = pack + "." + content['name'] action_api = ActionAPI(pack=pack, **content) action_api.validate() # action_validator.validate_action(action_api) actions[ref] = ActionAPI.to_model(action_api) return actions
class ResourceRegistrar(object): ALLOWED_EXTENSIONS = [] def __init__(self, use_pack_cache=True, use_runners_cache=False, fail_on_failure=False): """ :param use_pack_cache: True to cache which packs have been registered in memory and making sure packs are only registered once. :type use_pack_cache: ``bool`` :param use_runners_cache: True to cache RunnerTypeDB objects in memory to reduce load on the database. :type use_runners_cache: ``bool`` :param fail_on_failure: Throw an exception if resource registration fails. :type fail_on_failure: ``bool`` """ self._use_pack_cache = use_pack_cache self._use_runners_cache = use_runners_cache self._fail_on_failure = fail_on_failure self._meta_loader = MetaLoader() self._pack_loader = ContentPackLoader() # Maps runner name -> RunnerTypeDB self._runner_type_db_cache = {} def get_resources_from_pack(self, resources_dir): resources = [] for ext in self.ALLOWED_EXTENSIONS: resources_glob = resources_dir if resources_dir.endswith('/'): resources_glob = resources_dir + ext else: resources_glob = resources_dir + '/*' + ext resource_files = glob.glob(resources_glob) resources.extend(resource_files) resources = sorted(resources) return resources def get_registered_packs(self): """ Return a list of registered packs. :rype: ``list`` """ return list(REGISTERED_PACKS_CACHE.keys()) def register_packs(self, base_dirs): """ Register packs in all the provided directories. """ packs = self._pack_loader.get_packs(base_dirs=base_dirs) registered_count = 0 for pack_name, pack_path in six.iteritems(packs): self.register_pack(pack_name=pack_name, pack_dir=pack_path) registered_count += 1 return registered_count def register_pack(self, pack_name, pack_dir): """ Register pack in the provided directory. """ if self._use_pack_cache and pack_name in REGISTERED_PACKS_CACHE: # This pack has already been registered during this register content run return LOG.debug('Registering pack: %s' % (pack_name)) REGISTERED_PACKS_CACHE[pack_name] = True try: pack_db, _ = self._register_pack(pack_name=pack_name, pack_dir=pack_dir) except Exception as e: if self._fail_on_failure: msg = 'Failed to register pack "%s": %s' % (pack_name, str(e)) raise ValueError(msg) LOG.exception('Failed to register pack "%s"' % (pack_name)) return None return pack_db def _register_pack(self, pack_name, pack_dir): """ Register a pack and corresponding pack config schema (create a DB object in the system). Note: Pack registration now happens when registering the content and not when installing a pack using packs.install. Eventually this will be moved to the pack management API. """ # 1. Register pack pack_db = self._register_pack_db(pack_name=pack_name, pack_dir=pack_dir) # Display a warning if pack contains deprecated config.yaml file. Support for those files # will be fully removed in v2.4.0. config_path = os.path.join(pack_dir, 'config.yaml') if os.path.isfile(config_path): LOG.error( 'Pack "%s" contains a deprecated config.yaml file (%s). ' 'Support for "config.yaml" files has been deprecated in StackStorm v1.6.0 ' 'in favor of config.schema.yaml config schema files and config files in ' '/opt/stackstorm/configs/ directory. Support for config.yaml files has ' 'been removed in the release (v2.4.0) so please migrate. For more ' 'information please refer to %s ' % (pack_db.name, config_path, 'https://docs.stackstorm.com/reference/pack_configs.html')) # 2. Register corresponding pack config schema config_schema_db = self._register_pack_config_schema_db( pack_name=pack_name, pack_dir=pack_dir) return pack_db, config_schema_db def _register_pack_db(self, pack_name, pack_dir): content = get_pack_metadata(pack_dir=pack_dir) # The rules for the pack ref are as follows: # 1. If ref attribute is available, we used that # 2. If pack_name is available we use that (this only applies to packs # 2hich are in sub-directories) # 2. If attribute is not available, but pack name is and pack name meets the valid name # criteria, we use that content['ref'] = get_pack_ref_from_metadata( metadata=content, pack_directory_name=pack_name) # Include a list of pack files pack_file_list = get_file_list(directory=pack_dir, exclude_patterns=EXCLUDE_FILE_PATTERNS) content['files'] = pack_file_list content['path'] = pack_dir pack_api = PackAPI(**content) pack_api.validate() pack_db = PackAPI.to_model(pack_api) try: pack_db.id = Pack.get_by_ref(content['ref']).id except StackStormDBObjectNotFoundError: LOG.debug('Pack %s not found. Creating new one.', pack_name) pack_db = Pack.add_or_update(pack_db) LOG.debug('Pack %s registered.' % (pack_name)) return pack_db def _register_pack_config_schema_db(self, pack_name, pack_dir): config_schema_path = os.path.join(pack_dir, CONFIG_SCHEMA_FILE_NAME) if not os.path.isfile(config_schema_path): # Note: Config schema is optional return None values = self._meta_loader.load(config_schema_path) if not values: raise ValueError('Config schema "%s" is empty and invalid.' % (config_schema_path)) content = {} content['pack'] = pack_name content['attributes'] = values config_schema_api = ConfigSchemaAPI(**content) config_schema_api = config_schema_api.validate() config_schema_db = ConfigSchemaAPI.to_model(config_schema_api) try: config_schema_db.id = ConfigSchema.get_by_pack(pack_name).id except StackStormDBObjectNotFoundError: LOG.debug('Config schema for pack %s not found. Creating new one.', pack_name) config_schema_db = ConfigSchema.add_or_update(config_schema_db) LOG.debug('Config schema for pack %s registered.' % (pack_name)) return config_schema_db
def test_get_sensors_pack_missing_sensors(self): loader = ContentPackLoader() fail_pack_path = os.path.join(RESOURCES_DIR, "packs/pack2") self.assertTrue(os.path.exists(fail_pack_path)) self.assertEqual(loader._get_sensors(fail_pack_path), None)
class ResourceRegistrar(object): ALLOWED_EXTENSIONS = [] def __init__(self, use_pack_cache=True, use_runners_cache=False, fail_on_failure=False): """ :param use_pack_cache: True to cache which packs have been registered in memory and making sure packs are only registered once. :type use_pack_cache: ``bool`` :param use_runners_cache: True to cache RunnerTypeDB objects in memory to reduce load on the database. :type use_runners_cache: ``bool`` :param fail_on_failure: Throw an exception if resource registration fails. :type fail_on_failure: ``bool`` """ self._use_pack_cache = use_pack_cache self._use_runners_cache = use_runners_cache self._fail_on_failure = fail_on_failure self._meta_loader = MetaLoader() self._pack_loader = ContentPackLoader() # Maps runner name -> RunnerTypeDB self._runner_type_db_cache = {} def get_resources_from_pack(self, resources_dir): resources = [] for ext in self.ALLOWED_EXTENSIONS: resources_glob = resources_dir if resources_dir.endswith('/'): resources_glob = resources_dir + ext else: resources_glob = resources_dir + '/*' + ext resource_files = glob.glob(resources_glob) resources.extend(resource_files) resources = sorted(resources) return resources def get_registered_packs(self): """ Return a list of registered packs. :rype: ``list`` """ return list(REGISTERED_PACKS_CACHE.keys()) def register_packs(self, base_dirs): """ Register packs in all the provided directories. """ packs = self._pack_loader.get_packs(base_dirs=base_dirs) registered_count = 0 for pack_name, pack_path in six.iteritems(packs): self.register_pack(pack_name=pack_name, pack_dir=pack_path) registered_count += 1 return registered_count def register_pack(self, pack_name, pack_dir): """ Register pack in the provided directory. """ if self._use_pack_cache and pack_name in REGISTERED_PACKS_CACHE: # This pack has already been registered during this register content run return LOG.debug('Registering pack: %s' % (pack_name)) REGISTERED_PACKS_CACHE[pack_name] = True try: pack_db, _ = self._register_pack(pack_name=pack_name, pack_dir=pack_dir) except Exception as e: if self._fail_on_failure: msg = 'Failed to register pack "%s": %s' % (pack_name, six.text_type(e)) raise ValueError(msg) LOG.exception('Failed to register pack "%s"' % (pack_name)) return None return pack_db def _register_pack(self, pack_name, pack_dir): """ Register a pack and corresponding pack config schema (create a DB object in the system). Note: Pack registration now happens when registering the content and not when installing a pack using packs.install. Eventually this will be moved to the pack management API. """ # 1. Register pack pack_db = self._register_pack_db(pack_name=pack_name, pack_dir=pack_dir) # Display a warning if pack contains deprecated config.yaml file. Support for those files # will be fully removed in v2.4.0. config_path = os.path.join(pack_dir, 'config.yaml') if os.path.isfile(config_path): LOG.error('Pack "%s" contains a deprecated config.yaml file (%s). ' 'Support for "config.yaml" files has been deprecated in StackStorm v1.6.0 ' 'in favor of config.schema.yaml config schema files and config files in ' '/opt/stackstorm/configs/ directory. Support for config.yaml files has ' 'been removed in the release (v2.4.0) so please migrate. For more ' 'information please refer to %s ' % (pack_db.name, config_path, 'https://docs.stackstorm.com/reference/pack_configs.html')) # 2. Register corresponding pack config schema config_schema_db = self._register_pack_config_schema_db(pack_name=pack_name, pack_dir=pack_dir) return pack_db, config_schema_db def _register_pack_db(self, pack_name, pack_dir): content = get_pack_metadata(pack_dir=pack_dir) # The rules for the pack ref are as follows: # 1. If ref attribute is available, we used that # 2. If pack_name is available we use that (this only applies to packs # 2hich are in sub-directories) # 2. If attribute is not available, but pack name is and pack name meets the valid name # criteria, we use that content['ref'] = get_pack_ref_from_metadata(metadata=content, pack_directory_name=pack_name) # Include a list of pack files pack_file_list = get_file_list(directory=pack_dir, exclude_patterns=EXCLUDE_FILE_PATTERNS) content['files'] = pack_file_list content['path'] = pack_dir pack_api = PackAPI(**content) pack_api.validate() pack_db = PackAPI.to_model(pack_api) try: pack_db.id = Pack.get_by_ref(content['ref']).id except StackStormDBObjectNotFoundError: LOG.debug('Pack %s not found. Creating new one.', pack_name) pack_db = Pack.add_or_update(pack_db) LOG.debug('Pack %s registered.' % (pack_name)) return pack_db def _register_pack_config_schema_db(self, pack_name, pack_dir): config_schema_path = os.path.join(pack_dir, CONFIG_SCHEMA_FILE_NAME) if not os.path.isfile(config_schema_path): # Note: Config schema is optional return None values = self._meta_loader.load(config_schema_path) if not values: raise ValueError('Config schema "%s" is empty and invalid.' % (config_schema_path)) content = {} content['pack'] = pack_name content['attributes'] = values config_schema_api = ConfigSchemaAPI(**content) config_schema_api = config_schema_api.validate() config_schema_db = ConfigSchemaAPI.to_model(config_schema_api) try: config_schema_db.id = ConfigSchema.get_by_pack(pack_name).id except StackStormDBObjectNotFoundError: LOG.debug('Config schema for pack %s not found. Creating new one.', pack_name) config_schema_db = ConfigSchema.add_or_update(config_schema_db) LOG.debug('Config schema for pack %s registered.' % (pack_name)) return config_schema_db
class ResourceRegistrar(object): ALLOWED_EXTENSIONS = [] def __init__(self, use_pack_cache=True, fail_on_failure=False): """ :param use_pack_cache: True to cache which packs have been registered in memory and making sure packs are only registered once. :type use_pack_cache: ``bool`` :param fail_on_failure: Throw an exception if resource registration fails. :type fail_on_failure: ``bool`` """ self._use_pack_cache = use_pack_cache self._fail_on_failure = fail_on_failure self._meta_loader = MetaLoader() self._pack_loader = ContentPackLoader() self._runner_loader = RunnersLoader() def get_resources_from_pack(self, resources_dir): resources = [] for ext in self.ALLOWED_EXTENSIONS: resources_glob = resources_dir if resources_dir.endswith('/'): resources_glob = resources_dir + ext else: resources_glob = resources_dir + '/*' + ext resource_files = glob.glob(resources_glob) resources.extend(resource_files) resources = sorted(resources) return resources def get_registered_packs(self): """ Return a list of registered packs. :rype: ``list`` """ return REGISTERED_PACKS_CACHE.keys() def register_packs(self, base_dirs): """ Register packs in all the provided directories. """ packs = self._pack_loader.get_packs(base_dirs=base_dirs) registered_count = 0 for pack_name, pack_path in six.iteritems(packs): self.register_pack(pack_name=pack_name, pack_dir=pack_path) registered_count += 1 return registered_count def register_pack(self, pack_name, pack_dir): """ Register pack in the provided directory. """ if self._use_pack_cache and pack_name in REGISTERED_PACKS_CACHE: # This pack has already been registered during this register content run return LOG.debug('Registering pack: %s' % (pack_name)) REGISTERED_PACKS_CACHE[pack_name] = True try: pack_db, _ = self._register_pack(pack_name=pack_name, pack_dir=pack_dir) except Exception: LOG.exception('Failed to register pack "%s"' % (pack_name)) return None return pack_db def _register_pack(self, pack_name, pack_dir): """ Register a pack and corresponding pack config schema (create a DB object in the system). Note: Pack registration now happens when registering the content and not when installing a pack using packs.install. Eventually this will be moved to the pack management API. """ # 1. Register pack pack_db = self._register_pack_db(pack_name=pack_name, pack_dir=pack_dir) # 2. Register corresponding pack config schema config_schema_db = self._register_pack_config_schema_db(pack_name=pack_name, pack_dir=pack_dir) return pack_db, config_schema_db def _register_pack_db(self, pack_name, pack_dir): pack_name = pack_name or '' manifest_path = os.path.join(pack_dir, MANIFEST_FILE_NAME) if not os.path.isfile(manifest_path): raise ValueError('Pack "%s" is missing %s file' % (pack_name, MANIFEST_FILE_NAME)) content = self._meta_loader.load(manifest_path) if not content: raise ValueError('Pack "%s" metadata file is empty' % (pack_name)) # The rules for the pack ref are as follows: # 1. If ref attribute is available, we used that # 2. If pack_name is available we use that (this only applies to packs # 2hich are in sub-directories) # 2. If attribute is not available, but pack name is and pack name meets the valid name # criteria, we use that content['ref'] = get_pack_ref_from_metadata(metadata=content, pack_directory_name=pack_name) # Include a list of pack files pack_file_list = get_file_list(directory=pack_dir, exclude_patterns=EXCLUDE_FILE_PATTERNS) content['files'] = pack_file_list pack_api = PackAPI(**content) pack_api.validate() pack_db = PackAPI.to_model(pack_api) try: pack_db.id = Pack.get_by_ref(content['ref']).id except StackStormDBObjectNotFoundError: LOG.debug('Pack %s not found. Creating new one.', pack_name) pack_db = Pack.add_or_update(pack_db) LOG.debug('Pack %s registered.' % (pack_name)) return pack_db def _register_pack_config_schema_db(self, pack_name, pack_dir): config_schema_path = os.path.join(pack_dir, CONFIG_SCHEMA_FILE_NAME) if not os.path.isfile(config_schema_path): # Note: Config schema is optional return None content = {} values = self._meta_loader.load(config_schema_path) content['pack'] = pack_name content['attributes'] = values config_schema_api = ConfigSchemaAPI(**content) config_schema_db = ConfigSchemaAPI.to_model(config_schema_api) try: config_schema_db.id = ConfigSchema.get_by_pack(pack_name).id except StackStormDBObjectNotFoundError: LOG.debug('Config schema for pack %s not found. Creating new one.', pack_name) config_schema_db = ConfigSchema.add_or_update(config_schema_db) LOG.debug('Config schema for pack %s registered.' % (pack_name)) return config_schema_db def register_runner(self): pass
def __init__(self): self._meta_loader = MetaLoader() self._pack_loader = ContentPackLoader()
def test_get_sensors_pack_missing_sensors(self): loader = ContentPackLoader() fail_pack_path = os.path.join(RESOURCES_DIR, 'packs/pack2') self.assertTrue(os.path.exists(fail_pack_path)) self.assertRaises(ValueError, loader._get_sensors, fail_pack_path)
def test_get_content_from_pack_success(self): loader = ContentPackLoader() pack_path = os.path.join(RESOURCES_DIR, 'packs/pack1') sensors = loader.get_content_from_pack(pack_dir=pack_path, content_type='sensors') self.assertTrue(sensors.endswith('packs/pack1/sensors'))
def test_get_content_from_pack_no_sensors(self): loader = ContentPackLoader() pack_path = os.path.join(RESOURCES_DIR, 'packs/pack2') result = loader.get_content_from_pack(pack_dir=pack_path, content_type='sensors') self.assertEqual(result, None)
class ResourceRegistrar(object): ALLOWED_EXTENSIONS = [] def __init__(self, use_pack_cache=True): """ :param use_pack_cache: True to cache which packs have been registered in memory and making sure packs are only registered once. :type use_pack_cache: ``bool`` """ self._use_pack_cache = use_pack_cache self._meta_loader = MetaLoader() self._pack_loader = ContentPackLoader() def get_resources_from_pack(self, resources_dir): resources = [] for ext in self.ALLOWED_EXTENSIONS: resources_glob = resources_dir if resources_dir.endswith("/"): resources_glob = resources_dir + ext else: resources_glob = resources_dir + "/*" + ext resource_files = glob.glob(resources_glob) resources.extend(resource_files) resources = sorted(resources) return resources def register_packs(self, base_dirs): """ Register packs in all the provided directories. """ packs = self._pack_loader.get_packs(base_dirs=base_dirs) registered_count = 0 for pack_name, pack_path in six.iteritems(packs): self.register_pack(pack_name=pack_name, pack_dir=pack_path) registered_count += 1 return registered_count def register_pack(self, pack_name, pack_dir): """ Register pack in the provided directory. """ if self._use_pack_cache and pack_name in REGISTERED_PACKS_CACHE: # This pack has already been registered during this register content run return LOG.debug("Registering pack: %s" % (pack_name)) REGISTERED_PACKS_CACHE[pack_name] = True try: pack_db = self._register_pack(pack_name=pack_name, pack_dir=pack_dir) except Exception: LOG.exception('Failed to register pack "%s"' % (pack_name)) return None return pack_db def _register_pack(self, pack_name, pack_dir): """ Register a pack (create a DB object in the system). Note: Pack registration now happens when registering the content and not when installing a pack using packs.install. Eventually this will be moved to the pack management API. """ manifest_path = os.path.join(pack_dir, MANIFEST_FILE_NAME) if not os.path.isfile(manifest_path): raise ValueError('Pack "%s" is missing %s file' % (pack_name, MANIFEST_FILE_NAME)) content = self._meta_loader.load(manifest_path) if not content: raise ValueError('Pack "%s" metadata file is empty' % (pack_name)) content["ref"] = pack_name # Include a list of pack files pack_file_list = get_file_list(directory=pack_dir, exclude_patterns=EXCLUDE_FILE_PATTERNS) content["files"] = pack_file_list pack_api = PackAPI(**content) pack_db = PackAPI.to_model(pack_api) try: pack_db.id = Pack.get_by_ref(pack_name).id except ValueError: LOG.debug("Pack %s not found. Creating new one.", pack_name) pack_db = Pack.add_or_update(pack_db) LOG.debug("Pack %s registered." % (pack_name)) return pack_db
def test_get_sensors(self): packs_base_path = os.path.join(RESOURCES_DIR, 'packs/') loader = ContentPackLoader() pack_sensors = loader.get_content(base_dirs=[packs_base_path], content_type='sensors') self.assertTrue(pack_sensors.get('pack1', None) is not None)
class ResourceRegistrar(object): ALLOWED_EXTENSIONS = [] def __init__(self): self._meta_loader = MetaLoader() self._pack_loader = ContentPackLoader() def get_resources_from_pack(self, resources_dir): resources = [] for ext in self.ALLOWED_EXTENSIONS: resources_glob = resources_dir if resources_dir.endswith('/'): resources_glob = resources_dir + ext else: resources_glob = resources_dir + '/*' + ext resource_files = glob.glob(resources_glob) resources.extend(resource_files) resources = sorted(resources) return resources def register_packs(self, base_dirs): """ Register packs in all the provided directories. """ packs = self._pack_loader.get_packs(base_dirs=base_dirs) registered_count = 0 for pack_name, pack_path in six.iteritems(packs): self.register_pack(pack_name=pack_name, pack_dir=pack_path) registered_count += 1 return registered_count def register_pack(self, pack_name, pack_dir): """ Register pack in the provided directory. """ if pack_name in REGISTERED_PACKS_CACHE: # This pack has already been registered during this register content run return LOG.debug('Registering pack: %s' % (pack_name)) REGISTERED_PACKS_CACHE[pack_name] = True try: pack_db = self._register_pack(pack_name=pack_name, pack_dir=pack_dir) except Exception: LOG.exception('Failed to register pack "%s"' % (pack_name)) return None return pack_db def _register_pack(self, pack_name, pack_dir): """ Register a pack (create a DB object in the system). Note: Pack registration now happens when registering the content and not when installing a pack using packs.install. Eventually this will be moved to the pack management API. """ manifest_path = os.path.join(pack_dir, MANIFEST_FILE_NAME) if not os.path.isfile(manifest_path): raise ValueError('Pack "%s" is missing %s file' % (pack_name, MANIFEST_FILE_NAME)) content = self._meta_loader.load(manifest_path) if not content: raise ValueError('Pack "%s" metadata file is empty' % (pack_name)) content['ref'] = pack_name pack_api = PackAPI(**content) pack_db = PackAPI.to_model(pack_api) try: pack_db.id = Pack.get_by_ref(pack_name).id except ValueError: LOG.debug('Pack %s not found. Creating new one.', pack_name) pack_db = Pack.add_or_update(pack_db) LOG.debug('Pack %s registered.' % (pack_name)) return pack_db
class ResourceRegistrar(object): ALLOWED_EXTENSIONS = [] def __init__(self, use_pack_cache=True): """ :param use_pack_cache: True to cache which packs have been registered in memory and making sure packs are only registered once. :type use_pack_cache: ``bool`` """ self._use_pack_cache = use_pack_cache self._meta_loader = MetaLoader() self._pack_loader = ContentPackLoader() def get_resources_from_pack(self, resources_dir): resources = [] for ext in self.ALLOWED_EXTENSIONS: resources_glob = resources_dir if resources_dir.endswith('/'): resources_glob = resources_dir + ext else: resources_glob = resources_dir + '/*' + ext resource_files = glob.glob(resources_glob) resources.extend(resource_files) resources = sorted(resources) return resources def register_packs(self, base_dirs): """ Register packs in all the provided directories. """ packs = self._pack_loader.get_packs(base_dirs=base_dirs) registered_count = 0 for pack_name, pack_path in six.iteritems(packs): self.register_pack(pack_name=pack_name, pack_dir=pack_path) registered_count += 1 return registered_count def register_pack(self, pack_name, pack_dir): """ Register pack in the provided directory. """ if self._use_pack_cache and pack_name in REGISTERED_PACKS_CACHE: # This pack has already been registered during this register content run return LOG.debug('Registering pack: %s' % (pack_name)) REGISTERED_PACKS_CACHE[pack_name] = True try: pack_db = self._register_pack(pack_name=pack_name, pack_dir=pack_dir) except Exception: LOG.exception('Failed to register pack "%s"' % (pack_name)) return None return pack_db def _register_pack(self, pack_name, pack_dir): """ Register a pack (create a DB object in the system). Note: Pack registration now happens when registering the content and not when installing a pack using packs.install. Eventually this will be moved to the pack management API. """ manifest_path = os.path.join(pack_dir, MANIFEST_FILE_NAME) if not os.path.isfile(manifest_path): raise ValueError('Pack "%s" is missing %s file' % (pack_name, MANIFEST_FILE_NAME)) content = self._meta_loader.load(manifest_path) if not content: raise ValueError('Pack "%s" metadata file is empty' % (pack_name)) content['ref'] = pack_name # Include a list of pack files pack_file_list = get_file_list(directory=pack_dir, exclude_patterns=EXCLUDE_FILE_PATTERNS) content['files'] = pack_file_list pack_api = PackAPI(**content) pack_db = PackAPI.to_model(pack_api) try: pack_db.id = Pack.get_by_ref(pack_name).id except ValueError: LOG.debug('Pack %s not found. Creating new one.', pack_name) pack_db = Pack.add_or_update(pack_db) LOG.debug('Pack %s registered.' % (pack_name)) return pack_db
class ResourceRegistrar(object): ALLOWED_EXTENSIONS = [] def __init__(self, use_pack_cache=True, fail_on_failure=False): """ :param use_pack_cache: True to cache which packs have been registered in memory and making sure packs are only registered once. :type use_pack_cache: ``bool`` :param fail_on_failure: Throw an exception if resource registration fails. :type fail_on_failure: ``bool`` """ self._use_pack_cache = use_pack_cache self._fail_on_failure = fail_on_failure self._meta_loader = MetaLoader() self._pack_loader = ContentPackLoader() self._runner_loader = RunnersLoader() def get_resources_from_pack(self, resources_dir): resources = [] for ext in self.ALLOWED_EXTENSIONS: resources_glob = resources_dir if resources_dir.endswith('/'): resources_glob = resources_dir + ext else: resources_glob = resources_dir + '/*' + ext resource_files = glob.glob(resources_glob) resources.extend(resource_files) resources = sorted(resources) return resources def get_registered_packs(self): """ Return a list of registered packs. :rype: ``list`` """ return REGISTERED_PACKS_CACHE.keys() def register_packs(self, base_dirs): """ Register packs in all the provided directories. """ packs = self._pack_loader.get_packs(base_dirs=base_dirs) registered_count = 0 for pack_name, pack_path in six.iteritems(packs): self.register_pack(pack_name=pack_name, pack_dir=pack_path) registered_count += 1 return registered_count def register_pack(self, pack_name, pack_dir): """ Register pack in the provided directory. """ if self._use_pack_cache and pack_name in REGISTERED_PACKS_CACHE: # This pack has already been registered during this register content run return LOG.debug('Registering pack: %s' % (pack_name)) REGISTERED_PACKS_CACHE[pack_name] = True try: pack_db, _ = self._register_pack(pack_name=pack_name, pack_dir=pack_dir) except Exception: LOG.exception('Failed to register pack "%s"' % (pack_name)) return None return pack_db def _register_pack(self, pack_name, pack_dir): """ Register a pack and corresponding pack config schema (create a DB object in the system). Note: Pack registration now happens when registering the content and not when installing a pack using packs.install. Eventually this will be moved to the pack management API. """ # 1. Register pack pack_db = self._register_pack_db(pack_name=pack_name, pack_dir=pack_dir) # 2. Register corresponding pack config schema config_schema_db = self._register_pack_config_schema_db( pack_name=pack_name, pack_dir=pack_dir) return pack_db, config_schema_db def _register_pack_db(self, pack_name, pack_dir): content = get_pack_metadata(pack_dir=pack_dir) # The rules for the pack ref are as follows: # 1. If ref attribute is available, we used that # 2. If pack_name is available we use that (this only applies to packs # 2hich are in sub-directories) # 2. If attribute is not available, but pack name is and pack name meets the valid name # criteria, we use that content['ref'] = get_pack_ref_from_metadata( metadata=content, pack_directory_name=pack_name) # Include a list of pack files pack_file_list = get_file_list(directory=pack_dir, exclude_patterns=EXCLUDE_FILE_PATTERNS) content['files'] = pack_file_list pack_api = PackAPI(**content) pack_api.validate() pack_db = PackAPI.to_model(pack_api) try: pack_db.id = Pack.get_by_ref(content['ref']).id except StackStormDBObjectNotFoundError: LOG.debug('Pack %s not found. Creating new one.', pack_name) pack_db = Pack.add_or_update(pack_db) LOG.debug('Pack %s registered.' % (pack_name)) return pack_db def _register_pack_config_schema_db(self, pack_name, pack_dir): config_schema_path = os.path.join(pack_dir, CONFIG_SCHEMA_FILE_NAME) if not os.path.isfile(config_schema_path): # Note: Config schema is optional return None content = {} values = self._meta_loader.load(config_schema_path) content['pack'] = pack_name content['attributes'] = values config_schema_api = ConfigSchemaAPI(**content) config_schema_db = ConfigSchemaAPI.to_model(config_schema_api) try: config_schema_db.id = ConfigSchema.get_by_pack(pack_name).id except StackStormDBObjectNotFoundError: LOG.debug('Config schema for pack %s not found. Creating new one.', pack_name) config_schema_db = ConfigSchema.add_or_update(config_schema_db) LOG.debug('Config schema for pack %s registered.' % (pack_name)) return config_schema_db def register_runner(self): pass
def test_get_sensors_pack_missing_sensors(self): loader = ContentPackLoader() fail_pack_path = os.path.join(RESOURCES_DIR, 'packs/pack2') self.assertTrue(os.path.exists(fail_pack_path)) self.assertEqual(loader._get_sensors(fail_pack_path), None)