Пример #1
0
def getBaseAppName():
    """Base App name, which this script belongs to.
    """
    appName = util.get_appname_from_path(__file__)
    if appName is None:
        raise Exception('Cannot get app name from file: %s' % __file__)
    return appName
Пример #2
0
    def __init__(self, namespace=None, default_level=logging.INFO):
        self._loggers = {}
        self._default_level = default_level
        if namespace is None:
            namespace = cutil.get_appname_from_path(op.abspath(__file__))

        if namespace:
            namespace = namespace.lower()
        self._namespace = namespace
 def __init__(self, conf_file, splunkd_uri, session_key, appname=None):
     if appname is None:
         appname = utils.get_appname_from_path(op.abspath(__file__))
     self._conf_file = conf.conf_file2name(conf_file)
     self._conf_mgr = conf.ConfManager(splunkd_uri, session_key, app_name=appname)
     self._cred_mgr = cred.CredentialManager(
         splunkd_uri, session_key, app=appname, owner="nobody", realm=appname
     )
     self._keys = None
Пример #4
0
 def __init__(self, conf_file, splunkd_uri, session_key, appname=None):
     if appname is None:
         appname = utils.get_appname_from_path(op.abspath(__file__))
     self._conf_file = conf.conf_file2name(conf_file)
     self._conf_mgr = conf.ConfManager(splunkd_uri, session_key,
                                       app_name=appname)
     self._cred_mgr = cred.CredentialManager(
         splunkd_uri, session_key, app=appname,
         owner="nobody", realm=appname)
     self._keys = None
Пример #5
0
    def __init__(self, namespace=None, default_level=logging.INFO):
        warnings.warn(
            "This class is deprecated. "
            "Please see https://github.com/splunk/addonfactory-ta-library-python/issues/38",
            DeprecationWarning,
            stacklevel=2,
        )
        self._loggers = {}
        self._default_level = default_level
        if namespace is None:
            namespace = cutil.get_appname_from_path(op.abspath(__file__))

        if namespace:
            namespace = namespace.lower()
        self._namespace = namespace
Пример #6
0
import logging
from inspect import ismethod
from os import path as op

from splunk import ResourceNotFound, RESTException, admin, entity, rest
from splunktalib.common import util as sc_util
from splunktalib.rest import splunkd_request

import splunktaucclib.common.log as stulog
from splunktaucclib.rest_handler.cred_mgmt import CredMgmt
from splunktaucclib.rest_handler.error_ctl import RestHandlerError as RH_Err
from splunktaucclib.rest_handler.util import makeConfItem

__all__ = ["user_caps", "BaseRestHandler", "BaseModel", "ResourceHandler"]

APP_NAME = sc_util.get_appname_from_path(op.abspath(__file__))


def get_entities(endpoint, session_key, user, app, get_args):
    url = rest.makeSplunkdUri() + "servicesNS/" + user + "/" + app + "/" + endpoint
    try:
        response, content = rest.simpleRequest(
            url,
            sessionKey=session_key,
            method="GET",
            getargs=get_args,
            raiseAllErrors=True,
        )
        res = json.loads(content)
        if "entry" in res:
            return {entry["name"]: entry["content"] for entry in res["entry"]}
Пример #7
0
class TaConfig(object):
    _current_hostname = socket.gethostname()
    _appname = util.get_appname_from_path(op.abspath(__file__))

    def __init__(self, meta_config, client_schema):
        self._meta_config = meta_config
        self._task_configs = []
        self._client_schema = client_schema
        self._server_info = sc.ServerInfo(meta_config[c.server_uri],
                                          meta_config[c.session_key])
        self._all_conf_contents = {}
        self._get_division_settings = {}
        self._load_task_configs()
        self._log_level = self._get_log_level()

    def is_shc_but_not_captain(self):
        return self._server_info.is_shc_member() and \
               not self._server_info.is_captain()

    def get_meta_config(self):
        return self._meta_config

    def get_task_configs(self):
        return self._task_configs

    def get_all_conf_contents(self):
        return self._all_conf_contents

    def get_divide_settings(self):
        return self._divide_settings

    def get_log_level(self):
        return self._log_level

    def _load_task_configs(self):
        config_handler = th.ConfigSchemaHandler(self._meta_config,
                                                self._client_schema)
        self._all_conf_contents = config_handler.get_all_conf_contents()
        self._divide_settings = config_handler.get_division_settings()
        assert self._divide_settings, "division is empty"
        self._generate_task_configs(self._all_conf_contents,
                                    self._divide_settings)

    def _generate_task_configs(self, all_conf_contents, divide_settings):
        all_task_configs = list()
        for division_endpoint, divide_setting in divide_settings.iteritems():
            task_configs = self._get_task_configs(all_conf_contents,
                                                  division_endpoint,
                                                  divide_setting)
            all_task_configs = all_task_configs + task_configs

        for task_config in all_task_configs:
            task_config[c.use_kv_store] = task_config.get(
                c.use_kv_store, False)
            task_config[c.appname] = TaConfig._appname
            task_config[c.index] = task_config.get(c.index, "default")
            if self._server_info.is_shc_member():
                task_config[c.use_kv_store] = True
            stulog.logger.debug("Task info: %s", task_config)
        self.process_task_configs(all_task_configs)
        #interval
        for task_config in all_task_configs:
            assert task_config.get(c.interval), "task config has no interval " \
                                                "field"
            task_config[c.interval] = int(task_config[c.interval])
        self._task_configs = all_task_configs
        stulog.logger.info("Totally generated {} task configs".format(
            len(self._task_configs)))

    # Override this method if some transforms or validations needs to be done
    # before task_configs is exposed
    def process_task_configs(self, task_configs):
        pass

    def _get_log_level(self):
        if not self._client_schema["basic"].get("config_meta"):
            return "INFO"
        if not self._client_schema["basic"]["config_meta"].get(
                "logging_setting"):
            return "INFO"
        paths = self._client_schema["basic"]["config_meta"][
            "logging_setting"].split(">")
        global_setting = self.get_all_conf_contents()[paths[0].strip()]
        if not global_setting:
            return "INFO"
        log_level = self.get_all_conf_contents()
        for i in xrange(len(paths)):
            log_level = log_level[paths[i].strip()]
        if not log_level:
            return "INFO"
        else:
            return log_level

    def _get_task_configs(self, all_conf_contents, division_endpoint,
                          divide_setting):
        task_configs = list()
        orig_task_configs = all_conf_contents.get(division_endpoint)
        for orig_task_config_stanza, orig_task_config_contents in \
                orig_task_configs.iteritems():
            if util.is_true(orig_task_config_contents.get(c.disabled, False)):
                stulog.logger.debug("Stanza %s is disabled",
                                    orig_task_config_contents)
                continue
            orig_task_config_contents[c.divide_endpoint] = division_endpoint
            divide_tasks = self._divide_task_config(orig_task_config_stanza,
                                                    orig_task_config_contents,
                                                    divide_setting,
                                                    all_conf_contents)
            task_configs = task_configs + divide_tasks
        return task_configs

    def _divide_task_config(self, task_config_stanza, task_config_contents,
                            divide_setting, all_conf_contents):
        task_config = dict()
        task_config[c.stanza_name] = [
            self._get_stanza_name(task_config_stanza)
        ]
        multi = 1
        for key, value in task_config_contents.iteritems():
            task_config[key] = [value]
            for divide_rule in divide_setting:
                if divide_rule.metric() == key:
                    if divide_rule.type() == th.ConfigSchemaHandler.TYPE_MULTI:
                        task_config[key] = value.split(divide_rule.separator())
                        multi = multi * len(task_config[key])
        scale_task_config = {}
        times = 0
        for key, value in task_config.iteritems():
            count = multi / len(value)
            scale_task_config[key] = value * count
            if len(value) == 1:
                continue
            times += 1
            if times % 2 == 0:
                scale_task_config[key].sort()
        return self._build_task_configs(scale_task_config, all_conf_contents,
                                        divide_setting, multi)

    def _build_task_configs(self, raw_task_config, all_conf_contents,
                            divide_setting, length):
        task_configs = list()
        # split task configs
        for i in xrange(length):
            task_config = dict()
            # handle endpoint config
            for raw_key, raw_value in raw_task_config.iteritems():
                value = raw_value[i]
                task_config[raw_key] = value
            # handle divide settings
            task_config[c.divide_key] = list()
            for divide_rule in divide_setting:
                task_config[c.divide_key].append(divide_rule.metric())
            task_config[c.divide_key].sort()
            task_configs.append(task_config)
        return task_configs

    def _get_stanza_name(self, input_item):
        if isinstance(input_item, basestring):
            in_name = input_item
        else:
            in_name = input_item[c.name]

        pos = in_name.find("://")
        if pos > 0:
            in_name = in_name[pos + 3:]
        return in_name
Пример #8
0
def get_appname_from_path(absolute_path):
    return scu.get_appname_from_path(absolute_path)
Пример #9
0
class ConfigSchemaHandler(object):
    _app_name = util.get_appname_from_path(op.abspath(__file__))
    # Division schema keys.
    TYPE = "type"

    TYPE_SINGLE = "single"
    TYPE_MULTI = "multi"
    REFER = "refer"
    SEPARATOR = "separator"

    def __init__(self, meta_configs, client_schema):
        self._config = sc.Config(splunkd_uri=meta_configs[c.server_uri],
                                 session_key=meta_configs[c.session_key],
                                 schema=json.dumps(client_schema[
                                                       c.config]),
                                 user="******",
                                 app=ConfigSchemaHandler._app_name)
        self._client_schema = client_schema
        self._all_conf_contents = {}
        self._load_conf_contents()
        self._division_settings = self._divide_settings()


    def get_endpoints(self):
        return self._config.get_endpoints()

    def get_all_conf_contents(self):
        return self._all_conf_contents

    def get_single_conf_contents(self, endpoint):
        return self._all_conf_contents.get(endpoint)

    def get_division_settings(self):
        return self._division_settings

    def _divide_settings(self):
        division_schema = self._client_schema[c.division]
        division_settings = dict()
        for division_endpoint, division_contents in division_schema.iteritems():
            division_settings[division_endpoint] = self._process_division(
                division_endpoint, division_contents)
        return division_settings


    def _load_conf_contents(self):
        self._all_conf_contents = self._config.load()

    def _process_division(self, division_endpoint, division_contents):
        division_metrics = []
        assert isinstance(division_contents, dict)
        for division_key, division_value in division_contents.iteritems():
            try:
                assert self.TYPE in division_value and \
                       division_value[self.TYPE] in \
                       [self.TYPE_SINGLE, self.TYPE_MULTI] and \
                       self.SEPARATOR in division_value if \
                    division_value[self.TYPE] == self.TYPE_MULTI else True
            except Exception:
                raise Exception("Invalid division schema")
            division_metrics.append(DivisionRule(division_endpoint,
                                                 division_key,
                                                 division_value[self.TYPE],
                                                 division_value.get(
                                                     self.SEPARATOR,
                                                 ),
                                                 division_value.get(
                                                     self.REFER,
                                                 )))
        return division_metrics
Пример #10
0
class TaConfig(object):
    _current_hostname = socket.gethostname()
    _appname = util.get_appname_from_path(op.abspath(__file__))

    def __init__(self, meta_config, client_schema, log_suffix=None,
                 stanza_name=None, input_type=None,
                 single_instance=True):
        self._meta_config = meta_config
        self._stanza_name = stanza_name
        self._input_type = input_type
        self._log_suffix = log_suffix
        self._single_instance = single_instance
        self._task_configs = []
        self._client_schema = client_schema
        self._server_info = sc.ServerInfo(meta_config[c.server_uri],
                                          meta_config[c.session_key])
        self._all_conf_contents = {}
        self._get_division_settings = {}
        self.set_logging()
        self._load_task_configs()

    def is_shc_member(self):
        return self._server_info.is_shc_member()

    def is_search_head(self):
        return self._server_info.is_search_head()

    def is_single_instance(self):
        return self._single_instance

    def get_meta_config(self):
        return self._meta_config

    def get_task_configs(self):
        return self._task_configs

    def get_all_conf_contents(self):
        if self._all_conf_contents:
            return self._all_conf_contents.get(c.inputs), \
                   self._all_conf_contents.get(c.all_configs), \
                   self._all_conf_contents.get(c.global_settings)

        inputs, configs, global_settings = th.get_all_conf_contents(
            self._meta_config[c.server_uri],
            self._meta_config[c.session_key],
            self._client_schema, self._input_type)
        self._all_conf_contents[c.inputs] = inputs
        self._all_conf_contents[c.all_configs] = configs
        self._all_conf_contents[c.global_settings] = global_settings
        return inputs, configs, global_settings

    def set_logging(self):
        # The default logger name is "cloud_connect_engine"
        if self._stanza_name and self._log_suffix:
            logger_name = self._log_suffix + "_" + th.format_name_for_file(
                self._stanza_name)
            stulog.reset_logger(logger_name)
        inputs, configs, global_settings = self.get_all_conf_contents()
        log_level = "INFO"
        for item in global_settings.get("settings"):
            if item.get(c.name) == "logging" and item.get("loglevel"):
                log_level = item["loglevel"]
                break
        stulog.set_log_level(log_level)
        stulog.logger.info("Set log_level={}".format(log_level))
        stulog.logger.info("Start {} task".format(self._stanza_name))

    def get_input_type(self):
        return self._input_type

    def _get_checkpoint_storage_type(self, config):
        cs_type = config.get(c.checkpoint_storage_type)
        stulog.logger.debug("Checkpoint storage type=%s", cs_type)

        cs_type = cs_type.strip() if cs_type else c.checkpoint_auto

        # Allow user configure 'auto' and 'file' only.
        if cs_type not in (c.checkpoint_auto, c.checkpoint_file):
            stulog.logger.warning(
                "Checkpoint storage type='%s' is invalid, change it to '%s'",
                cs_type, c.checkpoint_auto
            )
            cs_type = c.checkpoint_auto

        if cs_type == c.checkpoint_auto and self.is_search_head():
            stulog.logger.info(
                "Checkpoint storage type is '%s' and instance is "
                "search head, set checkpoint storage type to '%s'.",
                c.checkpoint_auto,
                c.checkpoint_kv_storage
            )
            cs_type = c.checkpoint_kv_storage
        return cs_type

    def _load_task_configs(self):
        inputs, configs, global_settings = self.get_all_conf_contents()
        if self._input_type:
            inputs = inputs.get(self._input_type)
        if not self._single_instance:
            inputs = [input for input in inputs if
                      input[c.name] == self._stanza_name]
        all_task_configs = []
        for input in inputs:
            task_config = {}
            task_config.update(input)
            task_config[c.configs] = configs
            task_config[c.settings] = \
                {item[c.name]: item for item in global_settings["settings"]}
            if self.is_single_instance():
                collection_interval = "collection_interval"
                task_config[c.interval] = task_config.get(collection_interval)
            task_config[c.interval] = int(task_config[c.interval])
            if task_config[c.interval] <= 0:
                raise ValueError(
                    "The interval value {} is invalid."
                    " It should be a positive integer".format(
                        task_config[c.interval]))

            task_config[c.checkpoint_storage_type] = \
                self._get_checkpoint_storage_type(task_config)

            task_config[c.appname] = TaConfig._appname
            task_config[c.mod_input_name] = self._input_type
            task_config[c.stanza_name] = task_config[c.name]

            all_task_configs.append(task_config)
        self._task_configs = all_task_configs

    # Override this method if some transforms or validations needs to be done
    # before task_configs is exposed
    def process_task_configs(self, task_configs):
        pass