Ejemplo n.º 1
0
    def __init__(self,
                 host="127.0.0.1",
                 port=6367,
                 password=None,
                 db=0,
                 match="*",
                 limit=0,
                 filters=None,
                 logger=None,
                 format="text"):
        self.logger = logger or logging.getLogger(__name__)
        self.logger.setLevel(logging.DEBUG)

        self.splitter = SimpleSplitter()
        self.isTextFormat = format == "text"
        self.reporter = TextReporter() if self.isTextFormat else JsonReporter()
        self.redis = connect_to_redis(host=host,
                                      port=port,
                                      db=db,
                                      password=password)

        self.match = match
        self.limit = limit if limit != 0 else sys.maxsize

        if 'types' in filters:
            self.types = list(map(redis_type_to_id, filters['types']))
        else:
            self.types = REDIS_TYPE_ID_ALL

        if 'behaviour' in filters:
            self.behaviour = filters['behaviour']
        else:
            self.behaviour = 'all'
Ejemplo n.º 2
0
    def __init__(self, host="127.0.0.1", port=6367, password=None, db=0, match="*", limit=0, filters=None, logger=None):
        self.logger = logger or logging.getLogger(__name__)

        self.splitter = SimpleSplitter()
        self.reporter = TextReporter()
        self.redis = connect_to_redis(host=host, port=port, db=db, password=password)

        self.match = match
        self.limit = limit if limit != 0 else sys.maxsize

        if 'types' in filters:
            self.types = list(map(redis_type_to_id, filters['types']))
        else:
            self.types = REDIS_TYPE_ID_ALL

        if 'behaviour' in filters:
            self.behaviour = filters['behaviour']
        else:
            self.behaviour = 'all'
Ejemplo n.º 3
0
class RmaApplication(object):
    globals = []

    types_rules = {
        REDIS_TYPE_ID_STRING: [],
        REDIS_TYPE_ID_HASH: [],
        REDIS_TYPE_ID_LIST: [],
        REDIS_TYPE_ID_SET: [],
        REDIS_TYPE_ID_ZSET: [],
    }

    def __init__(self, host="127.0.0.1", port=6367, password=None, db=0, match="*", limit=0, filters=None, logger=None):
        self.logger = logger or logging.getLogger(__name__)

        self.splitter = SimpleSplitter()
        self.reporter = TextReporter()
        self.redis = connect_to_redis(host=host, port=port, db=db, password=password)

        self.match = match
        self.limit = limit if limit != 0 else sys.maxsize

        if 'types' in filters:
            self.types = list(map(redis_type_to_id, filters['types']))
        else:
            self.types = REDIS_TYPE_ID_ALL

        if 'behaviour' in filters:
            self.behaviour = filters['behaviour']
        else:
            self.behaviour = 'all'

    def init_globals(self, redis):
        self.globals.append(GlobalKeySpace(redis=redis))

    def init_types_rules(self, redis):
        self.types_rules[REDIS_TYPE_ID_STRING].append(KeyString(redis=redis))
        self.types_rules[REDIS_TYPE_ID_STRING].append(ValueString(redis=redis))
        self.types_rules[REDIS_TYPE_ID_HASH].append(KeyString(redis=redis))
        self.types_rules[REDIS_TYPE_ID_HASH].append(Hash(redis=redis))
        self.types_rules[REDIS_TYPE_ID_LIST].append(KeyString(redis=redis))
        self.types_rules[REDIS_TYPE_ID_LIST].append(List(redis=redis))

        self.types_rules[REDIS_TYPE_ID_SET].append(KeyString(redis=redis))
        self.types_rules[REDIS_TYPE_ID_SET].append(Set(redis=redis))

        self.types_rules[REDIS_TYPE_ID_ZSET].append(KeyString(redis=redis))

    def run(self):
        self.init_types_rules(redis=self.redis)
        self.init_globals(redis=self.redis)

        str_res = []
        is_all = self.behaviour == 'all'
        with Scanner(redis=self.redis, match=self.match, accepted_types=self.types) as scanner:
            keys = defaultdict(list)
            for v in scanner.scan(limit=self.limit):
                keys[v["type"]].append(v)

            print("\r\nAggregating keys by pattern and type")
            keys = {k: self.get_pattern_aggregated_data(v) for k, v in keys.items()}

            print("\r\nApply rules")
            if self.behaviour == 'global' or is_all:
                str_res += self.do_globals()
            if self.behaviour == 'scanner' or is_all:
                str_res += self.do_scanner(self.redis, keys)
            if self.behaviour == 'ram' or is_all:
                str_res += self.do_ram(keys)

        self.reporter.print(str_res)

    def do_globals(self):
        res = ['Server stat']
        for glob in self.globals:
            res += glob.analyze()

        return res

    def do_scanner(self, r, res):
        key_stat = {
            'headers': ['Match', "Count", "Type", "%"],
            'data': []
        }
        total = min(r.dbsize(), self.limit)
        for key, aggregate_patterns in res.items():
            self.logger.debug("Processing type %s" % type_id_to_redis_type(key))
            r_type = type_id_to_redis_type(key)

            for k, v in aggregate_patterns.items():
                key_stat['data'].append([k, len(v), r_type, floored_percentage(len(v) / total, 2)])

        key_stat['data'].sort(key=lambda x: x[1], reverse=True)
        return ["Keys by types", key_stat]

    def do_ram(self, res):
        ret = []
        for key, aggregate_patterns in res.items():
            self.logger.debug("Processing type %s" % type_id_to_redis_type(key))
            if key in self.types_rules and key in self.types:
                ret.append("Processing {0}".format(type_id_to_redis_type(key)))
                for rule in self.types_rules[key]:
                    total_keys = sum(len(values) for key, values in aggregate_patterns.items())
                    ret += (rule.analyze(keys=aggregate_patterns, total=total_keys))

        return ret

    def get_pattern_aggregated_data(self, data):
        split_patterns = self.splitter.split((obj["name"] for obj in data))
        self.logger.debug(split_patterns)

        aggregate_patterns = {item: [] for item in split_patterns}
        for pattern in split_patterns:
            aggregate_patterns[pattern] = list(filter(lambda obj: fnmatch.fnmatch(obj["name"], pattern), data))

        return aggregate_patterns
Ejemplo n.º 4
0
class RmaApplication(object):
    globals = []

    types_rules = {
        REDIS_TYPE_ID_STRING: [],
        REDIS_TYPE_ID_HASH: [],
        REDIS_TYPE_ID_LIST: [],
        REDIS_TYPE_ID_SET: [],
        REDIS_TYPE_ID_ZSET: [],
    }

    def __init__(self, host="127.0.0.1", port=6367, password=None, db=0, match="*", limit=0, filters=None, logger=None):
        self.logger = logger or logging.getLogger(__name__)

        self.splitter = SimpleSplitter()
        self.reporter = TextReporter()
        self.redis = connect_to_redis(host=host, port=port, db=db, password=password)

        self.match = match
        self.limit = limit if limit != 0 else sys.maxsize

        if 'filters' in filters:
            if 'types' in filters['filters']:
                self.types = list(map(redis_type_to_id, filters['filters']['types']))
            else:
                self.types = REDIS_TYPE_ID_ALL
            if 'behaviour' in filters['filters']:
                self.behaviour = filters['filters']['behaviour']
            else:
                self.behaviour = 'all'
        else:
            self.types = REDIS_TYPE_ID_ALL
            self.behaviour = 'all'

    def init_globals(self, redis):
        self.globals.append(GlobalKeySpace(redis=redis))

    def init_types_rules(self, redis):
        self.types_rules[REDIS_TYPE_ID_STRING].append(KeyString(redis=redis))
        self.types_rules[REDIS_TYPE_ID_STRING].append(ValueString(redis=redis))
        self.types_rules[REDIS_TYPE_ID_HASH].append(KeyString(redis=redis))
        self.types_rules[REDIS_TYPE_ID_HASH].append(Hash(redis=redis))
        self.types_rules[REDIS_TYPE_ID_LIST].append(KeyString(redis=redis))
        self.types_rules[REDIS_TYPE_ID_LIST].append(List(redis=redis))

        self.types_rules[REDIS_TYPE_ID_SET].append(KeyString(redis=redis))
        self.types_rules[REDIS_TYPE_ID_SET].append(Set(redis=redis))

        self.types_rules[REDIS_TYPE_ID_ZSET].append(KeyString(redis=redis))

    def run(self):
        self.init_types_rules(redis=self.redis)
        self.init_globals(redis=self.redis)

        str_res = []
        is_all = self.behaviour == 'all'
        with Scanner(redis=self.redis, match=self.match, accepted_types=self.types) as scanner:
            keys = defaultdict(list)
            for v in scanner.scan(limit=self.limit):
                keys[v["type"]].append(v)

            print("Aggregating keys by pattern and type")
            keys = {k: self.get_pattern_aggregated_data(v) for k, v in keys.items()}

            print("Apply rules")
            if self.behaviour == 'global' or is_all:
                str_res += self.do_globals()
            if self.behaviour == 'scanner' or is_all:
                str_res += self.do_scanner(self.redis, keys)
            if self.behaviour == 'ram' or is_all:
                str_res += self.do_ram(keys)

        self.reporter.print(str_res)

    def do_globals(self):
        res = ['Server stat']
        for glob in self.globals:
            res += glob.analyze()

        return res

    def do_scanner(self, r, res):
        key_stat = {
            'headers': ['Match', "Count", "Type", "%"],
            'data': []
        }
        total = r.dbsize()
        for key, aggregate_patterns in res.items():
            self.logger.debug("Processing type %s" % type_id_to_redis_type(key))
            r_type = type_id_to_redis_type(key)

            for k, v in aggregate_patterns.items():
                key_stat['data'].append([k, len(v), r_type, floored_percentage(len(v) / total, 2)])

        key_stat['data'].sort(key=lambda x: x[1], reverse=True)
        return ["Keys by types", key_stat]

    def do_ram(self, res):
        ret = []
        for key, aggregate_patterns in res.items():
            self.logger.debug("Processing type %s" % type_id_to_redis_type(key))
            if key in self.types_rules and key in self.types:
                ret.append("Processing {0}".format(type_id_to_redis_type(key)))
                for rule in self.types_rules[key]:
                    ret += (rule.analyze(aggregate_patterns))

        return ret

    def get_pattern_aggregated_data(self, data):
        split_patterns = self.splitter.split((obj["name"] for obj in data))
        self.logger.debug(split_patterns)

        aggregate_patterns = {item: [] for item in split_patterns}
        for pattern in split_patterns:
            aggregate_patterns[pattern] = list(filter(lambda obj: fnmatch.fnmatch(obj["name"], pattern), data))

        return aggregate_patterns
Ejemplo n.º 5
0
class RmaApplication(object):
    globals = []

    types_rules = {
        REDIS_TYPE_ID_STRING: [],
        REDIS_TYPE_ID_HASH: [],
        REDIS_TYPE_ID_LIST: [],
        REDIS_TYPE_ID_SET: [],
        REDIS_TYPE_ID_ZSET: [],
    }

    def __init__(self,
                 host="127.0.0.1",
                 port=6367,
                 password=None,
                 db=0,
                 ssl=False,
                 match="*",
                 limit=0,
                 filters=None,
                 logger=None,
                 format="text",
                 separator=":"):
        self.logger = logger or logging.getLogger(__name__)

        self.splitter = SimpleSplitter(separator)
        self.isTextFormat = format == "text"
        self.reporter = TextReporter() if self.isTextFormat else JsonReporter()
        self.redis = connect_to_redis(host=host,
                                      port=port,
                                      db=db,
                                      password=password,
                                      ssl=ssl)

        self.match = match
        self.limit = limit if limit != 0 else sys.maxsize

        if 'types' in filters:
            self.types = list(map(redis_type_to_id, filters['types']))
        else:
            self.types = REDIS_TYPE_ID_ALL

        if 'behaviour' in filters:
            self.behaviour = filters['behaviour']
        else:
            self.behaviour = 'all'

    def init_globals(self, redis):
        self.globals.append(GlobalKeySpace(redis=redis))

    def init_types_rules(self, redis):
        self.types_rules[REDIS_TYPE_ID_STRING].append(KeyString(redis=redis))
        self.types_rules[REDIS_TYPE_ID_STRING].append(ValueString(redis=redis))
        self.types_rules[REDIS_TYPE_ID_HASH].append(KeyString(redis=redis))
        self.types_rules[REDIS_TYPE_ID_HASH].append(Hash(redis=redis))
        self.types_rules[REDIS_TYPE_ID_LIST].append(KeyString(redis=redis))
        self.types_rules[REDIS_TYPE_ID_LIST].append(List(redis=redis))

        self.types_rules[REDIS_TYPE_ID_SET].append(KeyString(redis=redis))
        self.types_rules[REDIS_TYPE_ID_SET].append(Set(redis=redis))

        self.types_rules[REDIS_TYPE_ID_ZSET].append(KeyString(redis=redis))

    def run(self):
        self.init_types_rules(redis=self.redis)
        self.init_globals(redis=self.redis)

        str_res = []
        is_all = self.behaviour == 'all'
        with Scanner(redis=self.redis,
                     match=self.match,
                     accepted_types=self.types) as scanner:
            keys = defaultdict(list)
            for v in scanner.scan(limit=self.limit):
                keys[v["type"]].append(v)

            if self.isTextFormat:
                print("\r\nAggregating keys by pattern and type")

            keys = {
                k: self.get_pattern_aggregated_data(v)
                for k, v in keys.items()
            }

            if self.isTextFormat:
                print("\r\nApply rules")

            if self.behaviour == 'global' or is_all:
                str_res.append(self.do_globals())
            if self.behaviour == 'scanner' or is_all:
                str_res.append(self.do_scanner(self.redis, keys))
            if self.behaviour == 'ram' or is_all:
                str_res.append(self.do_ram(keys))

        self.reporter.print(str_res)

    def do_globals(self):
        nodes = []
        for glob in self.globals:
            nodes.append(glob.analyze())

        return {"nodes": nodes}

    def do_scanner(self, r, res):
        keys = []
        total = min(r.dbsize(), self.limit)
        for key, aggregate_patterns in res.items():
            self.logger.debug("Processing type %s" %
                              type_id_to_redis_type(key))
            r_type = type_id_to_redis_type(key)

            for k, v in aggregate_patterns.items():
                keys.append(
                    [k,
                     len(v), r_type,
                     floored_percentage(len(v) / total, 2)])
                keys.sort(key=lambda x: x[1], reverse=True)

        return {
            "keys": {
                "data": keys,
                "headers": ['name', 'count', 'type', 'percent']
            }
        }

    def do_ram(self, res):
        ret = {}

        for key, aggregate_patterns in res.items():
            self.logger.debug("Processing type %s" %
                              type_id_to_redis_type(key))
            if key in self.types_rules and key in self.types:
                redis_type = type_id_to_redis_type(key)
                for rule in self.types_rules[key]:
                    total_keys = sum(
                        len(values)
                        for key, values in aggregate_patterns.items())
                    ret[redis_type] = rule.analyze(keys=aggregate_patterns,
                                                   total=total_keys)

        return {"stat": ret}

    def get_pattern_aggregated_data(self, data):
        split_patterns = self.splitter.split(
            (ptransform(obj["name"]) for obj in data))
        self.logger.debug(split_patterns)

        aggregate_patterns = {item: [] for item in split_patterns}
        for pattern in split_patterns:
            aggregate_patterns[pattern] = list(
                filter(
                    lambda obj: fnmatch.fnmatch(ptransform(obj["name"]),
                                                pattern), data))

        return aggregate_patterns