def __init__(self, parsingconf, aggregation_config=None): self.logger = CommonLogger() self.logger.debug("read combaine config") self.metahost = None try: _combaine = parse_common_cfg("combaine")#yaml.load(open('/etc/combaine/combaine.json')) _parsing = parse_parsing_cfg(parsingconf)#yaml.load(open('/etc/combaine/parsing/%s.json' % parsingconf)) if aggregation_config is None: _aggregations = [(parse_agg_cfg(agg_name), agg_name) for agg_name in _parsing["agg_configs"]] else: _aggregations = [(parse_agg_cfg(aggregation_config), aggregation_config), ] self.metahost = _aggregations[0][0].get('metahost') or _parsing.get('metahost') self.ds = _combaine["cloud_config"]["DistributedStorage"] self.df = _combaine["cloud_config"]["DataFetcher"] self.db = _combaine["cloud_config"]["LocalDatabase"] _ds = _parsing.get("DistributedStorage") _df = _parsing.get("DataFetcher") _db = _parsing.get("LocalDatabase") self.hosts_fetcher_http_hand = _combaine['Combainer'].get('HTTP_HAND') self.parser = _parsing.get("parser") if not _ds is None: self.logger.debug("Update ds from parsing") self.ds.update(_ds) if not _df is None: self.logger.debug("Update ds from parsing") self.df.update(_df) if not _db is None: self.logger.debug("Update ds from parsing") self.db.update(_db) #=============== agg_bind = { "summa" : "AverageAggregator", "quant" : "QuantAggregator", "average" : "AverageAggregator", "uniq" : "UniqAggregator", } self.aggregators = [] self.resulthadlers = list() for aggregator, _agg_name in _aggregations: for name, dic in aggregator["data"].iteritems(): tmp = dict() tmp["name"] = _agg_name + "@" + name tmp["query"] = dic.get("query", "EMPTY") tmp["type"] = dic["type"] tmp.update(dic) if dic["type"] == "quant": tmp["values"] = dic["values"] tmp["type"] = agg_bind.get(dic["type"]) #DIRTY HOOK!!!!!!! if not tmp["type"] is None: self.aggregators.append(tmp) if aggregator.has_key("ResultHandlers"): for name, dic in aggregator["ResultHandlers"].iteritems(): dic['type'] = name dic['parsing_conf'] = _parsing self.resulthadlers.append(dic) except Exception as err: self.logger.exception("Error in read confing") raise
import collections import itertools import socket import hashlib from combaine.plugins.Aggregators import AggregatorFactory from combaine.plugins.DistributedStorage import DistributedStorageFactory from combaine.plugins.ResultHandler import ResultHandlerFactory from combaine.common.configloader.parsingconfigurator import ParsingConfigurator from combaine.common.loggers import AggregateLogger from combaine.common.loggers import CommonLogger from combaine.common.interfaces.aggresult import AggRes from combaine.common.configloader import parse_common_cfg try: http_hand_url = parse_common_cfg("combaine")['Combainer']['Main']['HTTP_HAND'] except Exception as err: print err logger.error(str(err)) def split_hosts_by_dc(subgroups): hosts = urllib.urlopen("%s%s?fields=root_datacenter_name,fqdn" % (http_hand_url, subgroups)).read() if hosts == 'No groups found': return [] host_dict = collections.defaultdict(list) for item in hosts.splitlines(): dc, host = item.split('\t') host_dict[dc].append(host) return host_dict def Main(groupname, config_name, agg_config_name, previous_time, current_time):
def parseConfig(): combaine_json = parse_common_cfg("combaine")["Combainer"] combainer_dict = combaine_json["Main"] lockserver_dict = combaine_json["Lockserver"] storage_dict = combaine_json.get("Storage", DEFAULT_STORAGE) return combainer_dict, lockserver_dict, storage_dict
app_log.addHandler(lhandler) if logger["type"] == "STDOUT": lhandler = logging.StreamHandler(sys.stdout) lhandler.setFormatter(_format) lhandler.setLevel(lvl) app_log.addHandler(lhandler) app_log.setLevel(lvl) #app_log.info('Init logger successfully') except Exception, err: app_log.addHandler(logging.StreamHandler(sys.stdout)) print "Exception in logger init: "+str(err) return False return True #init_logging((json.load(open('/etc/combaine/combaine.json','r'))["Combainer"]["logger"], ) ) init_logging([parse_common_cfg('combaine')["Combainer"]["logger"]]) class configChecker(object): def __init__(self, config_path): try: self.raw_config = json.load(open(config_path),r) except Exception, err: print str(err) raise def checkOption(self, path, assertFunc): option = self.raw_config try: for section in path.split('/'): try:
from combaine.plugins.DistributedStorage import DistributedStorageFactory from combaine.common.loggers import CommonLogger from combaine.common.configloader import parse_common_cfg ## # # # SHIT CODE: REWRITE!!!! # # 3 agave_headers = {"User-Agent": "Yandex/Agave", "Connection": "TE", "TE": "deflate,gzip;q=0.3"} try: agave_hosts = parse_common_cfg("combaine")["cloud_config"]["agave_hosts"] except Exception as err: print err class Agave(AbstractResultHandler): def __init__(self, **config): self.logger = CommonLogger() self.graph_name = config.get("graph_name") self.graph_template = config.get("graph_template") self.fields = config.get("Fields") self.template_dict = {"template": self.graph_template, "title": self.graph_name, "graphname": self.graph_name} self.logger.debug(self.template_dict) def __makeUrls(self, frmt_dict): self.template_dict.update(frmt_dict)