class RmaApplication(object): globals = [] types_rules = { REDIS_TYPE_ID_STRING: [], REDIS_TYPE_ID_HASH: [], REDIS_TYPE_ID_LIST: [], REDIS_TYPE_ID_SET: [], REDIS_TYPE_ID_ZSET: [], } def __init__(self, host="127.0.0.1", port=6367, password=None, db=0, match="*", limit=0, filters=None, logger=None): self.logger = logger or logging.getLogger(__name__) self.splitter = SimpleSplitter() self.reporter = TextReporter() self.redis = connect_to_redis(host=host, port=port, db=db, password=password) self.match = match self.limit = limit if limit != 0 else sys.maxsize if 'filters' in filters: if 'types' in filters['filters']: self.types = list(map(redis_type_to_id, filters['filters']['types'])) else: self.types = REDIS_TYPE_ID_ALL if 'behaviour' in filters['filters']: self.behaviour = filters['filters']['behaviour'] else: self.behaviour = 'all' else: self.types = REDIS_TYPE_ID_ALL self.behaviour = 'all' def init_globals(self, redis): self.globals.append(GlobalKeySpace(redis=redis)) def init_types_rules(self, redis): self.types_rules[REDIS_TYPE_ID_STRING].append(KeyString(redis=redis)) self.types_rules[REDIS_TYPE_ID_STRING].append(ValueString(redis=redis)) self.types_rules[REDIS_TYPE_ID_HASH].append(KeyString(redis=redis)) self.types_rules[REDIS_TYPE_ID_HASH].append(Hash(redis=redis)) self.types_rules[REDIS_TYPE_ID_LIST].append(KeyString(redis=redis)) self.types_rules[REDIS_TYPE_ID_LIST].append(List(redis=redis)) self.types_rules[REDIS_TYPE_ID_SET].append(KeyString(redis=redis)) self.types_rules[REDIS_TYPE_ID_SET].append(Set(redis=redis)) self.types_rules[REDIS_TYPE_ID_ZSET].append(KeyString(redis=redis)) def run(self): self.init_types_rules(redis=self.redis) self.init_globals(redis=self.redis) str_res = [] is_all = self.behaviour == 'all' with Scanner(redis=self.redis, match=self.match, accepted_types=self.types) as scanner: keys = defaultdict(list) for v in scanner.scan(limit=self.limit): keys[v["type"]].append(v) print("Aggregating keys by pattern and type") keys = {k: self.get_pattern_aggregated_data(v) for k, v in keys.items()} print("Apply rules") if self.behaviour == 'global' or is_all: str_res += self.do_globals() if self.behaviour == 'scanner' or is_all: str_res += self.do_scanner(self.redis, keys) if self.behaviour == 'ram' or is_all: str_res += self.do_ram(keys) self.reporter.print(str_res) def do_globals(self): res = ['Server stat'] for glob in self.globals: res += glob.analyze() return res def do_scanner(self, r, res): key_stat = { 'headers': ['Match', "Count", "Type", "%"], 'data': [] } total = r.dbsize() for key, aggregate_patterns in res.items(): self.logger.debug("Processing type %s" % type_id_to_redis_type(key)) r_type = type_id_to_redis_type(key) for k, v in aggregate_patterns.items(): key_stat['data'].append([k, len(v), r_type, floored_percentage(len(v) / total, 2)]) key_stat['data'].sort(key=lambda x: x[1], reverse=True) return ["Keys by types", key_stat] def do_ram(self, res): ret = [] for key, aggregate_patterns in res.items(): self.logger.debug("Processing type %s" % type_id_to_redis_type(key)) if key in self.types_rules and key in self.types: ret.append("Processing {0}".format(type_id_to_redis_type(key))) for rule in self.types_rules[key]: ret += (rule.analyze(aggregate_patterns)) return ret def get_pattern_aggregated_data(self, data): split_patterns = self.splitter.split((obj["name"] for obj in data)) self.logger.debug(split_patterns) aggregate_patterns = {item: [] for item in split_patterns} for pattern in split_patterns: aggregate_patterns[pattern] = list(filter(lambda obj: fnmatch.fnmatch(obj["name"], pattern), data)) return aggregate_patterns
class RmaApplication(object): globals = [] types_rules = { REDIS_TYPE_ID_STRING: [], REDIS_TYPE_ID_HASH: [], REDIS_TYPE_ID_LIST: [], REDIS_TYPE_ID_SET: [], REDIS_TYPE_ID_ZSET: [], } def __init__(self, host="127.0.0.1", port=6367, password=None, db=0, match="*", limit=0, filters=None, logger=None): self.logger = logger or logging.getLogger(__name__) self.splitter = SimpleSplitter() self.reporter = TextReporter() self.redis = connect_to_redis(host=host, port=port, db=db, password=password) self.match = match self.limit = limit if limit != 0 else sys.maxsize if 'types' in filters: self.types = list(map(redis_type_to_id, filters['types'])) else: self.types = REDIS_TYPE_ID_ALL if 'behaviour' in filters: self.behaviour = filters['behaviour'] else: self.behaviour = 'all' def init_globals(self, redis): self.globals.append(GlobalKeySpace(redis=redis)) def init_types_rules(self, redis): self.types_rules[REDIS_TYPE_ID_STRING].append(KeyString(redis=redis)) self.types_rules[REDIS_TYPE_ID_STRING].append(ValueString(redis=redis)) self.types_rules[REDIS_TYPE_ID_HASH].append(KeyString(redis=redis)) self.types_rules[REDIS_TYPE_ID_HASH].append(Hash(redis=redis)) self.types_rules[REDIS_TYPE_ID_LIST].append(KeyString(redis=redis)) self.types_rules[REDIS_TYPE_ID_LIST].append(List(redis=redis)) self.types_rules[REDIS_TYPE_ID_SET].append(KeyString(redis=redis)) self.types_rules[REDIS_TYPE_ID_SET].append(Set(redis=redis)) self.types_rules[REDIS_TYPE_ID_ZSET].append(KeyString(redis=redis)) def run(self): self.init_types_rules(redis=self.redis) self.init_globals(redis=self.redis) str_res = [] is_all = self.behaviour == 'all' with Scanner(redis=self.redis, match=self.match, accepted_types=self.types) as scanner: keys = defaultdict(list) for v in scanner.scan(limit=self.limit): keys[v["type"]].append(v) print("\r\nAggregating keys by pattern and type") keys = {k: self.get_pattern_aggregated_data(v) for k, v in keys.items()} print("\r\nApply rules") if self.behaviour == 'global' or is_all: str_res += self.do_globals() if self.behaviour == 'scanner' or is_all: str_res += self.do_scanner(self.redis, keys) if self.behaviour == 'ram' or is_all: str_res += self.do_ram(keys) self.reporter.print(str_res) def do_globals(self): res = ['Server stat'] for glob in self.globals: res += glob.analyze() return res def do_scanner(self, r, res): key_stat = { 'headers': ['Match', "Count", "Type", "%"], 'data': [] } total = min(r.dbsize(), self.limit) for key, aggregate_patterns in res.items(): self.logger.debug("Processing type %s" % type_id_to_redis_type(key)) r_type = type_id_to_redis_type(key) for k, v in aggregate_patterns.items(): key_stat['data'].append([k, len(v), r_type, floored_percentage(len(v) / total, 2)]) key_stat['data'].sort(key=lambda x: x[1], reverse=True) return ["Keys by types", key_stat] def do_ram(self, res): ret = [] for key, aggregate_patterns in res.items(): self.logger.debug("Processing type %s" % type_id_to_redis_type(key)) if key in self.types_rules and key in self.types: ret.append("Processing {0}".format(type_id_to_redis_type(key))) for rule in self.types_rules[key]: total_keys = sum(len(values) for key, values in aggregate_patterns.items()) ret += (rule.analyze(keys=aggregate_patterns, total=total_keys)) return ret def get_pattern_aggregated_data(self, data): split_patterns = self.splitter.split((obj["name"] for obj in data)) self.logger.debug(split_patterns) aggregate_patterns = {item: [] for item in split_patterns} for pattern in split_patterns: aggregate_patterns[pattern] = list(filter(lambda obj: fnmatch.fnmatch(obj["name"], pattern), data)) return aggregate_patterns
class RmaApplication(object): globals = [] types_rules = { REDIS_TYPE_ID_STRING: [], REDIS_TYPE_ID_HASH: [], REDIS_TYPE_ID_LIST: [], REDIS_TYPE_ID_SET: [], REDIS_TYPE_ID_ZSET: [], } def __init__(self, host="127.0.0.1", port=6367, password=None, db=0, ssl=False, match="*", limit=0, filters=None, logger=None, format="text", separator=":"): self.logger = logger or logging.getLogger(__name__) self.splitter = SimpleSplitter(separator) self.isTextFormat = format == "text" self.reporter = TextReporter() if self.isTextFormat else JsonReporter() self.redis = connect_to_redis(host=host, port=port, db=db, password=password, ssl=ssl) self.match = match self.limit = limit if limit != 0 else sys.maxsize if 'types' in filters: self.types = list(map(redis_type_to_id, filters['types'])) else: self.types = REDIS_TYPE_ID_ALL if 'behaviour' in filters: self.behaviour = filters['behaviour'] else: self.behaviour = 'all' def init_globals(self, redis): self.globals.append(GlobalKeySpace(redis=redis)) def init_types_rules(self, redis): self.types_rules[REDIS_TYPE_ID_STRING].append(KeyString(redis=redis)) self.types_rules[REDIS_TYPE_ID_STRING].append(ValueString(redis=redis)) self.types_rules[REDIS_TYPE_ID_HASH].append(KeyString(redis=redis)) self.types_rules[REDIS_TYPE_ID_HASH].append(Hash(redis=redis)) self.types_rules[REDIS_TYPE_ID_LIST].append(KeyString(redis=redis)) self.types_rules[REDIS_TYPE_ID_LIST].append(List(redis=redis)) self.types_rules[REDIS_TYPE_ID_SET].append(KeyString(redis=redis)) self.types_rules[REDIS_TYPE_ID_SET].append(Set(redis=redis)) self.types_rules[REDIS_TYPE_ID_ZSET].append(KeyString(redis=redis)) def run(self): self.init_types_rules(redis=self.redis) self.init_globals(redis=self.redis) str_res = [] is_all = self.behaviour == 'all' with Scanner(redis=self.redis, match=self.match, accepted_types=self.types) as scanner: keys = defaultdict(list) for v in scanner.scan(limit=self.limit): keys[v["type"]].append(v) if self.isTextFormat: print("\r\nAggregating keys by pattern and type") keys = { k: self.get_pattern_aggregated_data(v) for k, v in keys.items() } if self.isTextFormat: print("\r\nApply rules") if self.behaviour == 'global' or is_all: str_res.append(self.do_globals()) if self.behaviour == 'scanner' or is_all: str_res.append(self.do_scanner(self.redis, keys)) if self.behaviour == 'ram' or is_all: str_res.append(self.do_ram(keys)) self.reporter.print(str_res) def do_globals(self): nodes = [] for glob in self.globals: nodes.append(glob.analyze()) return {"nodes": nodes} def do_scanner(self, r, res): keys = [] total = min(r.dbsize(), self.limit) for key, aggregate_patterns in res.items(): self.logger.debug("Processing type %s" % type_id_to_redis_type(key)) r_type = type_id_to_redis_type(key) for k, v in aggregate_patterns.items(): keys.append( [k, len(v), r_type, floored_percentage(len(v) / total, 2)]) keys.sort(key=lambda x: x[1], reverse=True) return { "keys": { "data": keys, "headers": ['name', 'count', 'type', 'percent'] } } def do_ram(self, res): ret = {} for key, aggregate_patterns in res.items(): self.logger.debug("Processing type %s" % type_id_to_redis_type(key)) if key in self.types_rules and key in self.types: redis_type = type_id_to_redis_type(key) for rule in self.types_rules[key]: total_keys = sum( len(values) for key, values in aggregate_patterns.items()) ret[redis_type] = rule.analyze(keys=aggregate_patterns, total=total_keys) return {"stat": ret} def get_pattern_aggregated_data(self, data): split_patterns = self.splitter.split( (ptransform(obj["name"]) for obj in data)) self.logger.debug(split_patterns) aggregate_patterns = {item: [] for item in split_patterns} for pattern in split_patterns: aggregate_patterns[pattern] = list( filter( lambda obj: fnmatch.fnmatch(ptransform(obj["name"]), pattern), data)) return aggregate_patterns