class _Socket(AbstractFetcher):
def __init__(self, **config):
self.log = CommonLogger()
try:
self.port = config.get('port', 89)
except Exception, err:
self.log.error("Error in init Socket getter: %s" % str(err))
raise Exception
class Tailer(AbstractFetcher):
def __init__(self, **config):
self.log = CommonLogger()
self.filter = lambda x: True
try:
self.port = config['port'] if config.has_key('port') else 89
except Exception, err:
print err
self.log.error("Error in init Tailer getter: %s" % str(err))
raise Exception
class _HTTP(AbstractFetcher):
def __init__(self, **config):
self.log = CommonLogger()
try:
url = config.get('url','')
self.port = config.get('port', 3132)
self.http_get_url = "%(url)s" % { 'url' : url}
except Exception, err:
self.log.error("Error in init HTTP Fetcher: %s" % str(err))
raise Exception
def aggregate_group(io):
"""Cloud wrapper """
message = ""
try:
message = io.read()
group_name, config_name, agg_config_name, prev_time, cur_time = message.split(';')
prev_time = int(prev_time)
cur_time = int(cur_time)
except Exception as err:
io.write("failed;Wrong message format:%s;%s;%s" % (message, socket.gethostname(), str(err)))
logger = CommonLogger()
logger.error("Wrong message %s" % message)
return
else:
try:
res = Main(group_name, config_name, agg_config_name, prev_time, cur_time)
except Exception as err:
res = 'failed;Error: %s' % err
finally:
io.write(';'.join((res, message, socket.gethostname())))
class Aggregator(AbstractResultHandler):
def __init__(self, **config):
self.logger = CommonLogger()
try:
self.expression = make_eval_string_safe(config["expression"])
self.logger.info("Evaluation expression: %s" % self.expression)
except UnsafelyCodeError as err:
self.logger.error(str(err))
raise
except KeyError as err:
self.logger.error("MathExp. Missing config parametr: %s" % str(err))
raise
self._aggs = pattern.findall(self.expression)
def send(self, data):
interest_results = [_ for _ in data if _.aggname in self._aggs]
for subgroup_name, subgroup_data in make_template_placeholders(interest_results):
code = self.expression
for key, value in subgroup_data.iteritems():
# re.subn(r"\${20x}","A","(${20x}+${30x})")
code, n = re.subn(r"\${%s}" % key, str(value), code)
self.logger.debug("After substitution %s" % code)
try:
res = eval(code)
self.logger.info("MathExp: Result %s %s" % (subgroup_name, res))
except Exception as err:
self.logger.error("Exception in evaluation %s: %s" % (code, err))
class Elliptics(AbstractDistributedStorage):
def __init__(self, **config):
self.logger = CommonLogger()
cfg = [tuple(_i.split(":")) for _i in config["proxy_hosts"]]
random.shuffle(cfg)
self.hostsinfo = cfg
self.read_timeout = config.get("read_timeout", 0.5)
self.write_timeout = config.get("write_timeout", 0.5)
self.read_url = string.Template("http://${HOST}:${R_PORT}/get/${KEY}?ioflags=3072")
self.write_url = string.Template("http://${HOST}:${W_PORT}/upload/${KEY}?ioflags=3072")
def connect(self, namespace):
return True
def insert(self, raw_key, data):
key = hashlib.md5(raw_key).hexdigest()
for host, r_port, w_port in self.hostsinfo[:]:
try:
r = requests.post(self.write_url.substitute(KEY=key, HOST=host, W_PORT=w_port), data=PACK(data), timeout=self.write_timeout)
if r.status_code == 200: #because elliptics write cache bug
self.logger.debug("Elliptics: insert key %s (%s) succesfully" % (key, raw_key))
return True
except requests.exceptions.Timeout as err:
self.hostsinfo.remove((host, r_port, w_port))
except requests.exceptions.ConnectionError as err:
self.logger.debug("Elliptics hosts: %s" % self.hostsinfo)
self.hostsinfo.remove((host, r_port, w_port))
self.logger.error("Elliptics: failed to insert key %s (%s)" % (key, raw_key))
return False
def read(self, raw_key, cache=False):
key = hashlib.md5(raw_key).hexdigest()
for host, r_port, w_port in self.hostsinfo[:]:
try:
r = requests.get(self.read_url.substitute(KEY=key, HOST=host, R_PORT=r_port), timeout=self.read_timeout)
if r.ok:
self.logger.debug("Elliptics: read key %s (%s) succesfully" % (key, raw_key))
ret = UNPACK(r.content)
r.close()
return ret
elif r.status_code == 404:
self.logger.info("Elliptics: Key %s (%s) is missing" % (key, raw_key))
return None
except requests.exceptions.Timeout as err:
self.hostsinfo.remove((host, r_port, w_port))
except requests.exceptions.ConnectionError as err:
self.hostsinfo.remove((host, r_port, w_port))
except Exception as err:
self.logger.exception("Read error in elliptics proxy")
self.logger.error("Elliptics: failed to read key %s (%s)" % (key, raw_key))
return None
def remove(self, key):
return "OK"
def close(self):
return True