def aggregate_group(request, response):
"""
Receives a list of results from the aggregate_host,
and performs aggregation by group
"""
try:
raw = yield request.read()
tid, cfg, data = msgpack.unpackb(raw)
logger = get_logger_adapter(tid)
logger.debug("Unpack raw data successfully")
payload = map(msgpack.unpackb, data)
klass_name = cfg['class']
cfg['logger'] = logger
result = _aggregate_group(klass_name, payload, cfg)
except KeyError:
response.error(-100, "There's no class named %s" % klass_name)
logger.error("class %s is absent", klass_name)
except Exception as err: # pylint: disable=broad-except
response.error(100, repr(err))
logger.error("Error %s", err)
else:
logger.info("Result of group aggregation %s", str(result))
response.write(result)
logger.info("Custom done")
finally:
response.close()
def aggregate_host(request, response):
raw = yield request.read()
TASK = msgpack.unpackb(raw)
taskId = TASK['id']
logger = get_logger_adapter(taskId)
logger.info("Handle task")
cfg = TASK['config'] # config of aggregator
token = TASK['token']
prtime = TASK['prevtime']
currtime = TASK['currtime']
dg = MysqlDG.get_service(DATABASEAPP)
q = TABLEREGEX.sub(token, cfg['query'])
q = TIMEREGEX.sub("1=1", q)
logger.debug("QUERY: %s", q)
pickled_res = yield dg.enqueue("query",
msgpack.packb((token, q)))
res = cPickle.loads(pickled_res)
logger.debug(str(res))
try:
ret = float(res[0][0]) # SELECT COUNT(*)
logger.info("Result from DG %s", ret)
if cfg.get('rps'):
ret = ret / (currtime - prtime)
except Exception:
ret = 0
logger.info("%s", ret)
response.write(msgpack.packb(ret))
response.close()
def aggregate_host(request, response):
"""
Gets the result of a single host,
performs parsing and their aggregation
"""
try:
raw = yield request.read()
task = msgpack.unpackb(raw)
tid = task['id']
logger = get_logger_adapter(tid)
logger.info("Handle task")
cfg = task['config']
klass_name = cfg['class']
cfg['logger'] = logger
# Replace this name
payload = task['token']
result = _aggregate_host(klass_name, payload, cfg, task)
except KeyError:
response.error(-100, "There's no class named %s" % klass_name)
logger.error("class %s is absent", klass_name)
except Exception as err: # pylint: disable=broad-except
response.error(-3, "Exception during handling %s" % repr(err))
logger.error("Error %s", err)
else:
response.write(msgpack.packb(result))
finally:
response.close()
logger.info("Done")
def aggregate_group(request, response):
raw = yield request.read()
tid, cfg, data = msgpack.unpackb(raw)
logger = get_logger_adapter(tid)
logger.info("Raw data is received %s", str(data))
res = sum(map(msgpack.unpackb, data))
logger.info("Solved %s", str(res))
response.write(res)
response.close()
def __init__(self, path, logger=None, extra_filter=None):
# to allow relative imports from parsers
# such as splitlines decorator
if path not in sys.path:
sys.path.insert(0, path)
self.path = path # search path for pluigns
self.available = None
self.extra_filter = extra_filter
self.logger = logger or get_logger_adapter("CORE")
def __init__(self, path, logger=None, extra_filter=None):
# to allow relative imports from parsers
# such as splitlines decorator
if path not in sys.path:
sys.path.insert(0, path)
self.path = path # search path for pluigns
self.available = None
self.extra_filter = extra_filter
self.logger = logger or get_logger_adapter("CORE")
def __init__(self, config):
self.quantile = list(config.get("values", DEFAULT_QUANTILE_VALUES))
self.quantile.sort()
# recalculate to rps? default yes
self.rps = (config.get("rps", "yes") == "yes")
# find timings by specified string. default '_timings'
self.timings_is = config.get("timings_is", "_timings")
# multiply on factor: default `1`
factor = float(config.get("factor", 1))
self.factor = lambda item: factor * item
# get prc. By default - No. Format: { ext_services: error/info }
# get prc of errors in info from metrics which contents 'ext_services'
self.get_prc = config.get("get_prc", False)
self.log = config.get("logger", get_logger_adapter("Multimetrics"))
def aggregate_group(request, response):
raw = yield request.read()
tid, cfg, data = msgpack.unpackb(raw)
logger = get_logger_adapter(tid)
logger.debug("Unpack raw data successfully")
payload = map(msgpack.unpackb, data)
klass_name = cfg['class']
try:
result = _aggregate_group(klass_name, payload, cfg)
except KeyError:
response.error(-100, "There's no class named %s" % klass_name)
logger.error("class %s is absent", klass_name)
except Exception as err:
logger.error("%s", err)
response.error(100, repr(err))
else:
logger.info("Result of group aggreagtion %s", str(result))
response.write(result)
response.close()
def parse(request, response):
inc = yield request.read()
task = ParserTask(inc)
logger = get_logger_adapter(task.tid)
try:
result = yield apply_parse(task, PLUGINS, logger)
response.write(msgpack.packb(result))
logger.info("%d items have been parsed by %s",
len(result), task.parser_name)
logger.debug("%s", result)
logger.info("the parsing of data is done")
except UnavailablePluginError as err:
response.error(-100, "UnavailableParser: %s" % err)
logger.error("Parser %s is absent", task.parser_name)
except Exception as err:
response.error(-3, "Exception in parsing %s" % repr(err))
logger.exception("Unknown error: %s", err)
finally:
response.close()
def aggregate_host(request, response):
raw = yield request.read()
TASK = msgpack.unpackb(raw)
logger = get_logger_adapter(TASK['id'])
logger.info("Handle task")
cfg = TASK['config'] # config of aggregator
token = TASK['token']
logger.debug(str(cfg))
dg = MysqlDG.get_service(DATABASEAPP)
q = TABLEREGEX.sub(token, cfg['query'])
q = TIMEREGEX.sub("1=1", q)
logger.info("QUERY: %s", q)
pickled_res = yield dg.enqueue("query", msgpack.packb((token, q)))
res = cPickle.loads(pickled_res)
logger.debug("%s", res)
ret = quantile_packer(itertools.chain(*res))
logger.info("Return %s", str(ret))
response.write(msgpack.packb(ret))
response.close()
def aggregate_host(request, response):
raw = yield request.read()
TASK = msgpack.unpackb(raw)
logger = get_logger_adapter(TASK['id'])
logger.info("Handle task")
cfg = TASK['config'] # config of aggregator
token = TASK['token']
logger.debug(str(cfg))
dg = MysqlDG.get_service(DATABASEAPP)
q = TABLEREGEX.sub(token, cfg['query'])
q = TIMEREGEX.sub("1=1", q)
logger.info("QUERY: %s", q)
pickled_res = yield dg.enqueue("query",
msgpack.packb((token, q)))
res = cPickle.loads(pickled_res)
logger.debug("%s", res)
ret = quantile_packer(itertools.chain(*res))
logger.info("Return %s", str(ret))
response.write(msgpack.packb(ret))
response.close()
def aggregate_group(request, response):
raw = yield request.read()
tid, cfg, data = msgpack.unpackb(raw)
logger = get_logger_adapter(tid)
logger.debug("Unpack raw data successfully")
raw_data = map(msgpack.unpackb, data)
ret = merge(raw_data)
logger.debug("Data has been merged %s", ret)
qts = map(int,
map(lambda x: float(ret["count"]) * x / 100,
cfg.get("values", [75, 90, 93, 94, 95, 96, 97, 98, 99])))
try:
ret = quants(qts,
ret['data'])
except Exception as err:
logger.error(str(err))
response.error(100, repr(err))
else:
logger.info("Result of group aggreagtion %s", ret)
response.write(ret)
response.close()
def aggregate_group(request, response):
raw = yield request.read()
tid, cfg, data = msgpack.unpackb(raw)
logger = get_logger_adapter(tid)
logger.debug("Unpack raw data successfully")
raw_data = map(msgpack.unpackb, data)
ret = merge(raw_data)
logger.debug("Data has been merged %s", ret)
qts = map(
int,
map(lambda x: float(ret["count"]) * x / 100,
cfg.get("values", [75, 90, 93, 94, 95, 96, 97, 98, 99])))
try:
ret = quants(qts, ret['data'])
except Exception as err:
logger.error(str(err))
response.error(100, repr(err))
else:
logger.info("Result of group aggreagtion %s", ret)
response.write(ret)
response.close()
def aggregate_host(request, response):
raw = yield request.read()
task = msgpack.unpackb(raw)
tid = task['id']
logger = get_logger_adapter(tid)
logger.info("Handle task")
cfg = task['config']
klass_name = cfg['class']
# Replace this name
payload = task['token']
try:
result = _aggregate_host(klass_name, payload, cfg, task)
response.write(msgpack.packb(result))
logger.info("Done")
except KeyError:
response.error(-100, "There's no class named %s" % klass_name)
logger.error("class %s is absent", klass_name)
except Exception as err:
response.error(-3, "Exception during handling %s" % repr(err))
logger.error("Error %s", err)
finally:
response.close()
def aggreagate(request, response):
raw = yield request.read()
task = AggregationTask(raw)
logger = get_logger_adapter(task.Id)
logger.info("task started")
metahost = task.parsing_config.metahost
hosts = task.hosts
# read aggregation config passed to us
aggcfg = task.aggregation_config
logger.debug("aggregation config %s", aggcfg)
logger.info("%s", hosts)
# repack hosts by subgroups by dc
# For example:
# {"GROUP-DC": "hostname"} from {"DC": "hostname"}
hosts = dict(("%s-%s" % (metahost, subgroup), v)
for subgroup, v in hosts.iteritems())
result = {}
for name, cfg in aggcfg.data.iteritems():
mapping = {}
logger.info("Send to %s %s" % (name, cfg['type']))
app = cache.get(cfg['type'])
if app is None:
logger.info("Skip %s" % cfg['type'])
continue
result[name] = {}
for subgroup, value in hosts.iteritems():
subgroup_data = list()
for host in value:
# Key specification
key = "%s;%s;%s;%s;%s" % (host, task.parsing_config_name,
task.aggregation_config_name,
name,
task.CurrTime)
try:
data = yield storage.read("combaine", key)
subgroup_data.append(data)
if cfg.get("perHost"):
res = yield app.enqueue("aggregate_group",
msgpack.packb((task.Id, cfg, [data])))
result[name][host] = res
except Exception as err:
if err.code != 2:
logger.error("unable to read from cache %s %s",
key, err)
mapping[subgroup] = subgroup_data
try:
res = yield app.enqueue("aggregate_group",
msgpack.packb((task.Id, cfg, subgroup_data)))
logger.info("name %s subgroup %s result %s",
name, subgroup, res)
result[name][subgroup] = res
except Exception as err:
logger.error("unable to aggregte %s %s %s",
name, subgroup, err)
all_data = []
for v in mapping.itervalues():
all_data.extend(v)
try:
res = yield app.enqueue("aggregate_group",
msgpack.packb((task.Id, cfg, all_data)))
except Exception as err:
logger.error("unable to aggreagate all: %s %s", name, err)
logger.info("name %s ALL %s %d" % (name, res, len(all_data)))
result[name][metahost] = res
# Send data to various senders
for name, item in aggcfg.senders.iteritems():
try:
sender_type = item.get("type")
if sender_type is None:
logger.error("unable to detect sender type: %s", name)
continue
logger.info("Send to %s", sender_type)
s = Service(sender_type)
res = yield s.enqueue("send", msgpack.packb({"Config": item,
"Data": result,
"Id": task.Id}))
logger.info("res for %s is %s", sender_type, res)
except Exception as err:
logger.error("unable to send to %s %s", name, err)
logger.info("Result %s", result)
response.write("Done")
response.close()
def aggreagate(request, response):
raw = yield request.read()
task = AggregationTask(raw)
logger = get_logger_adapter(task.Id)
logger.info("task started")
metahost = task.parsing_config.metahost
hosts = task.hosts
# read aggregation config passed to us
aggcfg = task.aggregation_config
logger.debug("aggregation config %s", aggcfg)
logger.info("%s", hosts)
# repack hosts by subgroups by dc
# For example:
# {"GROUP-DC": "hostname"} from {"DC": "hostname"}
hosts = dict(("%s-%s" % (metahost, subgroup), v)
for subgroup, v in hosts.iteritems())
result = {}
for name, cfg in aggcfg.data.iteritems():
mapping = {}
logger.info("Send to %s %s" % (name, cfg['type']))
app = cache.get(cfg['type'])
if app is None:
logger.info("Skip %s" % cfg['type'])
continue
result[name] = {}
for subgroup, value in hosts.iteritems():
subgroup_data = list()
for host in value:
# Key specification
key = "%s;%s;%s;%s;%s" % (host, task.parsing_config_name,
task.aggregation_config_name, name,
task.CurrTime)
try:
data = yield storage.read("combaine", key)
subgroup_data.append(data)
if cfg.get("perHost"):
res = yield app.enqueue(
"aggregate_group",
msgpack.packb((task.Id, cfg, [data])))
result[name][host] = res
except Exception as err:
if err.code != 2:
logger.error("unable to read from cache %s %s", key,
err)
mapping[subgroup] = subgroup_data
try:
res = yield app.enqueue(
"aggregate_group",
msgpack.packb((task.Id, cfg, subgroup_data)))
logger.info("name %s subgroup %s result %s", name, subgroup,
res)
result[name][subgroup] = res
except Exception as err:
logger.error("unable to aggregte %s %s %s", name, subgroup,
err)
all_data = []
for v in mapping.itervalues():
all_data.extend(v)
try:
res = yield app.enqueue("aggregate_group",
msgpack.packb((task.Id, cfg, all_data)))
except Exception as err:
logger.error("unable to aggreagate all: %s %s", name, err)
logger.info("name %s ALL %s %d" % (name, res, len(all_data)))
result[name][metahost] = res
# Send data to various senders
for name, item in aggcfg.senders.iteritems():
try:
sender_type = item.get("type")
if sender_type is None:
logger.error("unable to detect sender type: %s", name)
continue
logger.info("Send to %s", sender_type)
s = Service(sender_type)
res = yield s.enqueue(
"send",
msgpack.packb({
"Config": item,
"Data": result,
"Id": task.Id
}))
logger.info("res for %s is %s", sender_type, res)
except Exception as err:
logger.error("unable to send to %s %s", name, err)
logger.info("Result %s", result)
response.write("Done")
response.close()
import os
import msgpack
from cocaine.futures import chain
from combaine.common.logger import get_logger_adapter
from combaine.utils.pluginload import UnavailablePluginError
from combaine.utils.pluginload import Plugins
from combaine.common import ParserTask
PATH = os.environ.get('PARSERS_PATH') or '/usr/lib/yandex/combaine/parsers'
PLUGINS = Plugins(PATH, get_logger_adapter("CORE"), callable)
@chain.concurrent
def do_parse(parser, data):
return [i.items() for i in parser(data) if i is not None]
@chain.source
def apply_parse(task, plugins, log):
"""
task - instance of ParserTask
"""
log.info("start parsing of data")
func = plugins.get_plugin(task.parser_name)
yield do_parse(func, task.data)
import os
import msgpack
from cocaine.futures import chain
from combaine.common.logger import get_logger_adapter
from combaine.utils.pluginload import UnavailablePluginError
from combaine.utils.pluginload import Plugins
from combaine.common import ParserTask
PATH = os.environ.get('PARSERS_PATH') or '/usr/lib/yandex/combaine/parsers'
PLUGINS = Plugins(PATH, get_logger_adapter("CORE"), callable)
@chain.concurrent
def do_parse(parser, data):
return [i.items() for i in parser(data) if i is not None]
@chain.source
def apply_parse(task, plugins, log):
"""
task - instance of ParserTask
"""
log.info("start parsing of data")
func = plugins.get_plugin(task.parser_name)
yield do_parse(func, task.data)
