def __init__(self, parsingconf, aggregation_config=None):
self.logger = CommonLogger()
self.logger.debug("read combaine config")
self.metahost = None
try:
_combaine = parse_common_cfg("combaine")#yaml.load(open('/etc/combaine/combaine.json'))
_parsing = parse_parsing_cfg(parsingconf)#yaml.load(open('/etc/combaine/parsing/%s.json' % parsingconf))
if aggregation_config is None:
_aggregations = [(parse_agg_cfg(agg_name), agg_name) for agg_name in _parsing["agg_configs"]]
else:
_aggregations = [(parse_agg_cfg(aggregation_config), aggregation_config), ]
self.metahost = _aggregations[0][0].get('metahost') or _parsing.get('metahost')
self.ds = _combaine["cloud_config"]["DistributedStorage"]
self.df = _combaine["cloud_config"]["DataFetcher"]
self.db = _combaine["cloud_config"]["LocalDatabase"]
_ds = _parsing.get("DistributedStorage")
_df = _parsing.get("DataFetcher")
_db = _parsing.get("LocalDatabase")
self.hosts_fetcher_http_hand = _combaine['Combainer'].get('HTTP_HAND')
self.parser = _parsing.get("parser")
if not _ds is None:
self.logger.debug("Update ds from parsing")
self.ds.update(_ds)
if not _df is None:
self.logger.debug("Update ds from parsing")
self.df.update(_df)
if not _db is None:
self.logger.debug("Update ds from parsing")
self.db.update(_db)
#===============
agg_bind = {
"summa" : "AverageAggregator",
"quant" : "QuantAggregator",
"average" : "AverageAggregator",
"uniq" : "UniqAggregator",
}
self.aggregators = []
self.resulthadlers = list()
for aggregator, _agg_name in _aggregations:
for name, dic in aggregator["data"].iteritems():
tmp = dict()
tmp["name"] = _agg_name + "@" + name
tmp["query"] = dic.get("query", "EMPTY")
tmp["type"] = dic["type"]
tmp.update(dic)
if dic["type"] == "quant":
tmp["values"] = dic["values"]
tmp["type"] = agg_bind.get(dic["type"]) #DIRTY HOOK!!!!!!!
if not tmp["type"] is None:
self.aggregators.append(tmp)
if aggregator.has_key("ResultHandlers"):
for name, dic in aggregator["ResultHandlers"].iteritems():
dic['type'] = name
dic['parsing_conf'] = _parsing
self.resulthadlers.append(dic)
except Exception as err:
self.logger.exception("Error in read confing")
raise
import collections
import itertools
import socket
import hashlib
from combaine.plugins.Aggregators import AggregatorFactory
from combaine.plugins.DistributedStorage import DistributedStorageFactory
from combaine.plugins.ResultHandler import ResultHandlerFactory
from combaine.common.configloader.parsingconfigurator import ParsingConfigurator
from combaine.common.loggers import AggregateLogger
from combaine.common.loggers import CommonLogger
from combaine.common.interfaces.aggresult import AggRes
from combaine.common.configloader import parse_common_cfg
try:
http_hand_url = parse_common_cfg("combaine")['Combainer']['Main']['HTTP_HAND']
except Exception as err:
print err
logger.error(str(err))
def split_hosts_by_dc(subgroups):
hosts = urllib.urlopen("%s%s?fields=root_datacenter_name,fqdn" % (http_hand_url, subgroups)).read()
if hosts == 'No groups found':
return []
host_dict = collections.defaultdict(list)
for item in hosts.splitlines():
dc, host = item.split('\t')
host_dict[dc].append(host)
return host_dict
def Main(groupname, config_name, agg_config_name, previous_time, current_time):
def parseConfig():
combaine_json = parse_common_cfg("combaine")["Combainer"]
combainer_dict = combaine_json["Main"]
lockserver_dict = combaine_json["Lockserver"]
storage_dict = combaine_json.get("Storage", DEFAULT_STORAGE)
return combainer_dict, lockserver_dict, storage_dict
app_log.addHandler(lhandler)
if logger["type"] == "STDOUT":
lhandler = logging.StreamHandler(sys.stdout)
lhandler.setFormatter(_format)
lhandler.setLevel(lvl)
app_log.addHandler(lhandler)
app_log.setLevel(lvl)
#app_log.info('Init logger successfully')
except Exception, err:
app_log.addHandler(logging.StreamHandler(sys.stdout))
print "Exception in logger init: "+str(err)
return False
return True
#init_logging((json.load(open('/etc/combaine/combaine.json','r'))["Combainer"]["logger"], ) )
init_logging([parse_common_cfg('combaine')["Combainer"]["logger"]])
class configChecker(object):
def __init__(self, config_path):
try:
self.raw_config = json.load(open(config_path),r)
except Exception, err:
print str(err)
raise
def checkOption(self, path, assertFunc):
option = self.raw_config
try:
for section in path.split('/'):
try:
from combaine.plugins.DistributedStorage import DistributedStorageFactory
from combaine.common.loggers import CommonLogger
from combaine.common.configloader import parse_common_cfg
##
#
#
# SHIT CODE: REWRITE!!!!
#
# 3
agave_headers = {"User-Agent": "Yandex/Agave", "Connection": "TE", "TE": "deflate,gzip;q=0.3"}
try:
agave_hosts = parse_common_cfg("combaine")["cloud_config"]["agave_hosts"]
except Exception as err:
print err
class Agave(AbstractResultHandler):
def __init__(self, **config):
self.logger = CommonLogger()
self.graph_name = config.get("graph_name")
self.graph_template = config.get("graph_template")
self.fields = config.get("Fields")
self.template_dict = {"template": self.graph_template, "title": self.graph_name, "graphname": self.graph_name}
self.logger.debug(self.template_dict)
def __makeUrls(self, frmt_dict):
self.template_dict.update(frmt_dict)
