def aggregate_group(request, response): """ Receives a list of results from the aggregate_host, and performs aggregation by group """ try: raw = yield request.read() tid, cfg, data = msgpack.unpackb(raw) logger = get_logger_adapter(tid) logger.debug("Unpack raw data successfully") payload = map(msgpack.unpackb, data) klass_name = cfg['class'] cfg['logger'] = logger result = _aggregate_group(klass_name, payload, cfg) except KeyError: response.error(-100, "There's no class named %s" % klass_name) logger.error("class %s is absent", klass_name) except Exception as err: # pylint: disable=broad-except response.error(100, repr(err)) logger.error("Error %s", err) else: logger.info("Result of group aggregation %s", str(result)) response.write(result) logger.info("Custom done") finally: response.close()
def aggregate_host(request, response): raw = yield request.read() TASK = msgpack.unpackb(raw) taskId = TASK['id'] logger = get_logger_adapter(taskId) logger.info("Handle task") cfg = TASK['config'] # config of aggregator token = TASK['token'] prtime = TASK['prevtime'] currtime = TASK['currtime'] dg = MysqlDG.get_service(DATABASEAPP) q = TABLEREGEX.sub(token, cfg['query']) q = TIMEREGEX.sub("1=1", q) logger.debug("QUERY: %s", q) pickled_res = yield dg.enqueue("query", msgpack.packb((token, q))) res = cPickle.loads(pickled_res) logger.debug(str(res)) try: ret = float(res[0][0]) # SELECT COUNT(*) logger.info("Result from DG %s", ret) if cfg.get('rps'): ret = ret / (currtime - prtime) except Exception: ret = 0 logger.info("%s", ret) response.write(msgpack.packb(ret)) response.close()
def aggregate_host(request, response): """ Gets the result of a single host, performs parsing and their aggregation """ try: raw = yield request.read() task = msgpack.unpackb(raw) tid = task['id'] logger = get_logger_adapter(tid) logger.info("Handle task") cfg = task['config'] klass_name = cfg['class'] cfg['logger'] = logger # Replace this name payload = task['token'] result = _aggregate_host(klass_name, payload, cfg, task) except KeyError: response.error(-100, "There's no class named %s" % klass_name) logger.error("class %s is absent", klass_name) except Exception as err: # pylint: disable=broad-except response.error(-3, "Exception during handling %s" % repr(err)) logger.error("Error %s", err) else: response.write(msgpack.packb(result)) finally: response.close() logger.info("Done")
def aggregate_group(request, response): raw = yield request.read() tid, cfg, data = msgpack.unpackb(raw) logger = get_logger_adapter(tid) logger.info("Raw data is received %s", str(data)) res = sum(map(msgpack.unpackb, data)) logger.info("Solved %s", str(res)) response.write(res) response.close()
def __init__(self, path, logger=None, extra_filter=None): # to allow relative imports from parsers # such as splitlines decorator if path not in sys.path: sys.path.insert(0, path) self.path = path # search path for pluigns self.available = None self.extra_filter = extra_filter self.logger = logger or get_logger_adapter("CORE")
def __init__(self, config): self.quantile = list(config.get("values", DEFAULT_QUANTILE_VALUES)) self.quantile.sort() # recalculate to rps? default yes self.rps = (config.get("rps", "yes") == "yes") # find timings by specified string. default '_timings' self.timings_is = config.get("timings_is", "_timings") # multiply on factor: default `1` factor = float(config.get("factor", 1)) self.factor = lambda item: factor * item # get prc. By default - No. Format: { ext_services: error/info } # get prc of errors in info from metrics which contents 'ext_services' self.get_prc = config.get("get_prc", False) self.log = config.get("logger", get_logger_adapter("Multimetrics"))
def aggregate_group(request, response): raw = yield request.read() tid, cfg, data = msgpack.unpackb(raw) logger = get_logger_adapter(tid) logger.debug("Unpack raw data successfully") payload = map(msgpack.unpackb, data) klass_name = cfg['class'] try: result = _aggregate_group(klass_name, payload, cfg) except KeyError: response.error(-100, "There's no class named %s" % klass_name) logger.error("class %s is absent", klass_name) except Exception as err: logger.error("%s", err) response.error(100, repr(err)) else: logger.info("Result of group aggreagtion %s", str(result)) response.write(result) response.close()
def parse(request, response): inc = yield request.read() task = ParserTask(inc) logger = get_logger_adapter(task.tid) try: result = yield apply_parse(task, PLUGINS, logger) response.write(msgpack.packb(result)) logger.info("%d items have been parsed by %s", len(result), task.parser_name) logger.debug("%s", result) logger.info("the parsing of data is done") except UnavailablePluginError as err: response.error(-100, "UnavailableParser: %s" % err) logger.error("Parser %s is absent", task.parser_name) except Exception as err: response.error(-3, "Exception in parsing %s" % repr(err)) logger.exception("Unknown error: %s", err) finally: response.close()
def aggregate_host(request, response): raw = yield request.read() TASK = msgpack.unpackb(raw) logger = get_logger_adapter(TASK['id']) logger.info("Handle task") cfg = TASK['config'] # config of aggregator token = TASK['token'] logger.debug(str(cfg)) dg = MysqlDG.get_service(DATABASEAPP) q = TABLEREGEX.sub(token, cfg['query']) q = TIMEREGEX.sub("1=1", q) logger.info("QUERY: %s", q) pickled_res = yield dg.enqueue("query", msgpack.packb((token, q))) res = cPickle.loads(pickled_res) logger.debug("%s", res) ret = quantile_packer(itertools.chain(*res)) logger.info("Return %s", str(ret)) response.write(msgpack.packb(ret)) response.close()
def aggregate_group(request, response): raw = yield request.read() tid, cfg, data = msgpack.unpackb(raw) logger = get_logger_adapter(tid) logger.debug("Unpack raw data successfully") raw_data = map(msgpack.unpackb, data) ret = merge(raw_data) logger.debug("Data has been merged %s", ret) qts = map(int, map(lambda x: float(ret["count"]) * x / 100, cfg.get("values", [75, 90, 93, 94, 95, 96, 97, 98, 99]))) try: ret = quants(qts, ret['data']) except Exception as err: logger.error(str(err)) response.error(100, repr(err)) else: logger.info("Result of group aggreagtion %s", ret) response.write(ret) response.close()
def aggregate_group(request, response): raw = yield request.read() tid, cfg, data = msgpack.unpackb(raw) logger = get_logger_adapter(tid) logger.debug("Unpack raw data successfully") raw_data = map(msgpack.unpackb, data) ret = merge(raw_data) logger.debug("Data has been merged %s", ret) qts = map( int, map(lambda x: float(ret["count"]) * x / 100, cfg.get("values", [75, 90, 93, 94, 95, 96, 97, 98, 99]))) try: ret = quants(qts, ret['data']) except Exception as err: logger.error(str(err)) response.error(100, repr(err)) else: logger.info("Result of group aggreagtion %s", ret) response.write(ret) response.close()
def aggregate_host(request, response): raw = yield request.read() task = msgpack.unpackb(raw) tid = task['id'] logger = get_logger_adapter(tid) logger.info("Handle task") cfg = task['config'] klass_name = cfg['class'] # Replace this name payload = task['token'] try: result = _aggregate_host(klass_name, payload, cfg, task) response.write(msgpack.packb(result)) logger.info("Done") except KeyError: response.error(-100, "There's no class named %s" % klass_name) logger.error("class %s is absent", klass_name) except Exception as err: response.error(-3, "Exception during handling %s" % repr(err)) logger.error("Error %s", err) finally: response.close()
def aggreagate(request, response): raw = yield request.read() task = AggregationTask(raw) logger = get_logger_adapter(task.Id) logger.info("task started") metahost = task.parsing_config.metahost hosts = task.hosts # read aggregation config passed to us aggcfg = task.aggregation_config logger.debug("aggregation config %s", aggcfg) logger.info("%s", hosts) # repack hosts by subgroups by dc # For example: # {"GROUP-DC": "hostname"} from {"DC": "hostname"} hosts = dict(("%s-%s" % (metahost, subgroup), v) for subgroup, v in hosts.iteritems()) result = {} for name, cfg in aggcfg.data.iteritems(): mapping = {} logger.info("Send to %s %s" % (name, cfg['type'])) app = cache.get(cfg['type']) if app is None: logger.info("Skip %s" % cfg['type']) continue result[name] = {} for subgroup, value in hosts.iteritems(): subgroup_data = list() for host in value: # Key specification key = "%s;%s;%s;%s;%s" % (host, task.parsing_config_name, task.aggregation_config_name, name, task.CurrTime) try: data = yield storage.read("combaine", key) subgroup_data.append(data) if cfg.get("perHost"): res = yield app.enqueue("aggregate_group", msgpack.packb((task.Id, cfg, [data]))) result[name][host] = res except Exception as err: if err.code != 2: logger.error("unable to read from cache %s %s", key, err) mapping[subgroup] = subgroup_data try: res = yield app.enqueue("aggregate_group", msgpack.packb((task.Id, cfg, subgroup_data))) logger.info("name %s subgroup %s result %s", name, subgroup, res) result[name][subgroup] = res except Exception as err: logger.error("unable to aggregte %s %s %s", name, subgroup, err) all_data = [] for v in mapping.itervalues(): all_data.extend(v) try: res = yield app.enqueue("aggregate_group", msgpack.packb((task.Id, cfg, all_data))) except Exception as err: logger.error("unable to aggreagate all: %s %s", name, err) logger.info("name %s ALL %s %d" % (name, res, len(all_data))) result[name][metahost] = res # Send data to various senders for name, item in aggcfg.senders.iteritems(): try: sender_type = item.get("type") if sender_type is None: logger.error("unable to detect sender type: %s", name) continue logger.info("Send to %s", sender_type) s = Service(sender_type) res = yield s.enqueue("send", msgpack.packb({"Config": item, "Data": result, "Id": task.Id})) logger.info("res for %s is %s", sender_type, res) except Exception as err: logger.error("unable to send to %s %s", name, err) logger.info("Result %s", result) response.write("Done") response.close()
def aggreagate(request, response): raw = yield request.read() task = AggregationTask(raw) logger = get_logger_adapter(task.Id) logger.info("task started") metahost = task.parsing_config.metahost hosts = task.hosts # read aggregation config passed to us aggcfg = task.aggregation_config logger.debug("aggregation config %s", aggcfg) logger.info("%s", hosts) # repack hosts by subgroups by dc # For example: # {"GROUP-DC": "hostname"} from {"DC": "hostname"} hosts = dict(("%s-%s" % (metahost, subgroup), v) for subgroup, v in hosts.iteritems()) result = {} for name, cfg in aggcfg.data.iteritems(): mapping = {} logger.info("Send to %s %s" % (name, cfg['type'])) app = cache.get(cfg['type']) if app is None: logger.info("Skip %s" % cfg['type']) continue result[name] = {} for subgroup, value in hosts.iteritems(): subgroup_data = list() for host in value: # Key specification key = "%s;%s;%s;%s;%s" % (host, task.parsing_config_name, task.aggregation_config_name, name, task.CurrTime) try: data = yield storage.read("combaine", key) subgroup_data.append(data) if cfg.get("perHost"): res = yield app.enqueue( "aggregate_group", msgpack.packb((task.Id, cfg, [data]))) result[name][host] = res except Exception as err: if err.code != 2: logger.error("unable to read from cache %s %s", key, err) mapping[subgroup] = subgroup_data try: res = yield app.enqueue( "aggregate_group", msgpack.packb((task.Id, cfg, subgroup_data))) logger.info("name %s subgroup %s result %s", name, subgroup, res) result[name][subgroup] = res except Exception as err: logger.error("unable to aggregte %s %s %s", name, subgroup, err) all_data = [] for v in mapping.itervalues(): all_data.extend(v) try: res = yield app.enqueue("aggregate_group", msgpack.packb((task.Id, cfg, all_data))) except Exception as err: logger.error("unable to aggreagate all: %s %s", name, err) logger.info("name %s ALL %s %d" % (name, res, len(all_data))) result[name][metahost] = res # Send data to various senders for name, item in aggcfg.senders.iteritems(): try: sender_type = item.get("type") if sender_type is None: logger.error("unable to detect sender type: %s", name) continue logger.info("Send to %s", sender_type) s = Service(sender_type) res = yield s.enqueue( "send", msgpack.packb({ "Config": item, "Data": result, "Id": task.Id })) logger.info("res for %s is %s", sender_type, res) except Exception as err: logger.error("unable to send to %s %s", name, err) logger.info("Result %s", result) response.write("Done") response.close()
import os import msgpack from cocaine.futures import chain from combaine.common.logger import get_logger_adapter from combaine.utils.pluginload import UnavailablePluginError from combaine.utils.pluginload import Plugins from combaine.common import ParserTask PATH = os.environ.get('PARSERS_PATH') or '/usr/lib/yandex/combaine/parsers' PLUGINS = Plugins(PATH, get_logger_adapter("CORE"), callable) @chain.concurrent def do_parse(parser, data): return [i.items() for i in parser(data) if i is not None] @chain.source def apply_parse(task, plugins, log): """ task - instance of ParserTask """ log.info("start parsing of data") func = plugins.get_plugin(task.parser_name) yield do_parse(func, task.data)