class Aggregator(AbstractResultHandler):
def __init__(self, **config):
self.logger = CommonLogger()
try:
self.expression = make_eval_string_safe(config["expression"])
self.logger.info("Evaluation expression: %s" % self.expression)
except UnsafelyCodeError as err:
self.logger.error(str(err))
raise
except KeyError as err:
self.logger.error("MathExp. Missing config parametr: %s" % str(err))
raise
self._aggs = pattern.findall(self.expression)
def send(self, data):
interest_results = [_ for _ in data if _.aggname in self._aggs]
for subgroup_name, subgroup_data in make_template_placeholders(interest_results):
code = self.expression
for key, value in subgroup_data.iteritems():
# re.subn(r"\${20x}","A","(${20x}+${30x})")
code, n = re.subn(r"\${%s}" % key, str(value), code)
self.logger.debug("After substitution %s" % code)
try:
res = eval(code)
self.logger.info("MathExp: Result %s %s" % (subgroup_name, res))
except Exception as err:
self.logger.error("Exception in evaluation %s: %s" % (code, err))
class Agave(AbstractResultHandler):
def __init__(self, **config):
self.logger = CommonLogger()
self.graph_name = config.get("graph_name")
self.graph_template = config.get("graph_template")
self.fields = config.get("Fields")
self.template_dict = {"template": self.graph_template, "title": self.graph_name, "graphname": self.graph_name}
self.logger.debug(self.template_dict)
def __makeUrls(self, frmt_dict):
self.template_dict.update(frmt_dict)
template = (
"/api/update/%(group)s/%(graphname)s?values=%(values)s&ts=%(time)i&template=%(template)s&title=%(title)s"
% self.template_dict
)
self.__send_point(template)
def __send_point(self, url):
for agv_host in agave_hosts:
conn = httplib.HTTPConnection(agv_host, timeout=1)
headers = agave_headers
headers["Host"] = agv_host + ":80"
try:
conn.request("GET", url, None, headers)
# print url, agv_host, conn.getresponse().read().splitlines()[0]
_r = conn.getresponse()
self.logger.info("%s %s %s %s %s" % (agv_host, _r.status, _r.reason, _r.read().strip("\r\n"), url))
except Exception as err:
self.logger.exception("Unable to connect to one agave")
else:
_r.close()
def send(self, data):
for_send = collections.defaultdict(list)
for aggres in data:
for sbg_name, val in aggres.values:
_sbg = sbg_name if sbg_name == aggres.groupname else "-".join((aggres.groupname, sbg_name))
if isinstance(val, types.ListType): # Quantile
l = itertools.izip(self.fields[aggres.aggname], val)
_value = "+".join(("%s:%s" % x for x in l))
else: # Simle single value
_value = "%s:%s" % (aggres.aggname, val)
for_send[_sbg].append(_value)
time = aggres.time
for name, val in for_send.iteritems():
frmt_dict = {"group": name, "values": "+".join(val), "time": time}
self.__makeUrls(frmt_dict)
class Elliptics(AbstractDistributedStorage):
def __init__(self, **config):
self.logger = CommonLogger()
cfg = [tuple(_i.split(":")) for _i in config["proxy_hosts"]]
random.shuffle(cfg)
self.hostsinfo = cfg
self.read_timeout = config.get("read_timeout", 0.5)
self.write_timeout = config.get("write_timeout", 0.5)
self.read_url = string.Template("http://${HOST}:${R_PORT}/get/${KEY}?ioflags=3072")
self.write_url = string.Template("http://${HOST}:${W_PORT}/upload/${KEY}?ioflags=3072")
def connect(self, namespace):
return True
def insert(self, raw_key, data):
key = hashlib.md5(raw_key).hexdigest()
for host, r_port, w_port in self.hostsinfo[:]:
try:
r = requests.post(self.write_url.substitute(KEY=key, HOST=host, W_PORT=w_port), data=PACK(data), timeout=self.write_timeout)
if r.status_code == 200: #because elliptics write cache bug
self.logger.debug("Elliptics: insert key %s (%s) succesfully" % (key, raw_key))
return True
except requests.exceptions.Timeout as err:
self.hostsinfo.remove((host, r_port, w_port))
except requests.exceptions.ConnectionError as err:
self.logger.debug("Elliptics hosts: %s" % self.hostsinfo)
self.hostsinfo.remove((host, r_port, w_port))
self.logger.error("Elliptics: failed to insert key %s (%s)" % (key, raw_key))
return False
def read(self, raw_key, cache=False):
key = hashlib.md5(raw_key).hexdigest()
for host, r_port, w_port in self.hostsinfo[:]:
try:
r = requests.get(self.read_url.substitute(KEY=key, HOST=host, R_PORT=r_port), timeout=self.read_timeout)
if r.ok:
self.logger.debug("Elliptics: read key %s (%s) succesfully" % (key, raw_key))
ret = UNPACK(r.content)
r.close()
return ret
elif r.status_code == 404:
self.logger.info("Elliptics: Key %s (%s) is missing" % (key, raw_key))
return None
except requests.exceptions.Timeout as err:
self.hostsinfo.remove((host, r_port, w_port))
except requests.exceptions.ConnectionError as err:
self.hostsinfo.remove((host, r_port, w_port))
except Exception as err:
self.logger.exception("Read error in elliptics proxy")
self.logger.error("Elliptics: failed to read key %s (%s)" % (key, raw_key))
return None
def remove(self, key):
return "OK"
def close(self):
return True
