class Aggregator(AbstractResultHandler): def __init__(self, **config): self.logger = CommonLogger() try: self.expression = make_eval_string_safe(config["expression"]) self.logger.info("Evaluation expression: %s" % self.expression) except UnsafelyCodeError as err: self.logger.error(str(err)) raise except KeyError as err: self.logger.error("MathExp. Missing config parametr: %s" % str(err)) raise self._aggs = pattern.findall(self.expression) def send(self, data): interest_results = [_ for _ in data if _.aggname in self._aggs] for subgroup_name, subgroup_data in make_template_placeholders(interest_results): code = self.expression for key, value in subgroup_data.iteritems(): # re.subn(r"\${20x}","A","(${20x}+${30x})") code, n = re.subn(r"\${%s}" % key, str(value), code) self.logger.debug("After substitution %s" % code) try: res = eval(code) self.logger.info("MathExp: Result %s %s" % (subgroup_name, res)) except Exception as err: self.logger.error("Exception in evaluation %s: %s" % (code, err))
class Agave(AbstractResultHandler): def __init__(self, **config): self.logger = CommonLogger() self.graph_name = config.get("graph_name") self.graph_template = config.get("graph_template") self.fields = config.get("Fields") self.template_dict = {"template": self.graph_template, "title": self.graph_name, "graphname": self.graph_name} self.logger.debug(self.template_dict) def __makeUrls(self, frmt_dict): self.template_dict.update(frmt_dict) template = ( "/api/update/%(group)s/%(graphname)s?values=%(values)s&ts=%(time)i&template=%(template)s&title=%(title)s" % self.template_dict ) self.__send_point(template) def __send_point(self, url): for agv_host in agave_hosts: conn = httplib.HTTPConnection(agv_host, timeout=1) headers = agave_headers headers["Host"] = agv_host + ":80" try: conn.request("GET", url, None, headers) # print url, agv_host, conn.getresponse().read().splitlines()[0] _r = conn.getresponse() self.logger.info("%s %s %s %s %s" % (agv_host, _r.status, _r.reason, _r.read().strip("\r\n"), url)) except Exception as err: self.logger.exception("Unable to connect to one agave") else: _r.close() def send(self, data): for_send = collections.defaultdict(list) for aggres in data: for sbg_name, val in aggres.values: _sbg = sbg_name if sbg_name == aggres.groupname else "-".join((aggres.groupname, sbg_name)) if isinstance(val, types.ListType): # Quantile l = itertools.izip(self.fields[aggres.aggname], val) _value = "+".join(("%s:%s" % x for x in l)) else: # Simle single value _value = "%s:%s" % (aggres.aggname, val) for_send[_sbg].append(_value) time = aggres.time for name, val in for_send.iteritems(): frmt_dict = {"group": name, "values": "+".join(val), "time": time} self.__makeUrls(frmt_dict)
class Elliptics(AbstractDistributedStorage): def __init__(self, **config): self.logger = CommonLogger() cfg = [tuple(_i.split(":")) for _i in config["proxy_hosts"]] random.shuffle(cfg) self.hostsinfo = cfg self.read_timeout = config.get("read_timeout", 0.5) self.write_timeout = config.get("write_timeout", 0.5) self.read_url = string.Template("http://${HOST}:${R_PORT}/get/${KEY}?ioflags=3072") self.write_url = string.Template("http://${HOST}:${W_PORT}/upload/${KEY}?ioflags=3072") def connect(self, namespace): return True def insert(self, raw_key, data): key = hashlib.md5(raw_key).hexdigest() for host, r_port, w_port in self.hostsinfo[:]: try: r = requests.post(self.write_url.substitute(KEY=key, HOST=host, W_PORT=w_port), data=PACK(data), timeout=self.write_timeout) if r.status_code == 200: #because elliptics write cache bug self.logger.debug("Elliptics: insert key %s (%s) succesfully" % (key, raw_key)) return True except requests.exceptions.Timeout as err: self.hostsinfo.remove((host, r_port, w_port)) except requests.exceptions.ConnectionError as err: self.logger.debug("Elliptics hosts: %s" % self.hostsinfo) self.hostsinfo.remove((host, r_port, w_port)) self.logger.error("Elliptics: failed to insert key %s (%s)" % (key, raw_key)) return False def read(self, raw_key, cache=False): key = hashlib.md5(raw_key).hexdigest() for host, r_port, w_port in self.hostsinfo[:]: try: r = requests.get(self.read_url.substitute(KEY=key, HOST=host, R_PORT=r_port), timeout=self.read_timeout) if r.ok: self.logger.debug("Elliptics: read key %s (%s) succesfully" % (key, raw_key)) ret = UNPACK(r.content) r.close() return ret elif r.status_code == 404: self.logger.info("Elliptics: Key %s (%s) is missing" % (key, raw_key)) return None except requests.exceptions.Timeout as err: self.hostsinfo.remove((host, r_port, w_port)) except requests.exceptions.ConnectionError as err: self.hostsinfo.remove((host, r_port, w_port)) except Exception as err: self.logger.exception("Read error in elliptics proxy") self.logger.error("Elliptics: failed to read key %s (%s)" % (key, raw_key)) return None def remove(self, key): return "OK" def close(self): return True