def _run(self):
lredis = None
pb = None
pause = False
self.redis_prev_time = 0
while True:
try:
if pause:
gevent.sleep(2)
pause = False
lredis = StrictRedisWrapper(host=self._pi.ip_address,
port=self._pi.port,
password=self._rpass,
db=self._pi.redis_agg_db,
socket_timeout=30)
pb = lredis.pubsub()
inst = self._pi.instance_id
part = self._partno
pb.subscribe('AGPARTPUB:%s:%d' % (inst, part))
self.syncpart(lredis)
while True:
message = pb.get_message()
if not message:
gevent.sleep(0.001)
continue
if message["type"] != "message":
gevent.sleep(0)
continue
dataline = message["data"]
try:
elems = json.loads(dataline)
except:
self._logger.error("AggUVE Parsing failed: %s" %
str(message))
gevent.sleep(0)
continue
else:
self._logger.info("AggUVE loading: %s" % str(elems))
if self._content:
ppe = lredis.pipeline()
lelems = []
for elem in elems:
table, barekey = elem["key"].split(":", 1)
if self._tablefilt:
if not table in self._tablefilt:
gevent.sleep(0)
continue
if self._patterns:
kfilter_match = False
for pattern in self._patterns:
if pattern.match(barekey):
kfilter_match = True
break
if not kfilter_match:
gevent.sleep(0)
continue
if self._cfilter:
if elem["type"] not in self._cfilter:
gevent.sleep(0)
continue
lelems.append(elem)
if self._content:
# This UVE was deleted
if elem["type"] is None:
ppe.exists("AGPARTVALUES:%s:%d:%s" % \
(inst, part, elem["key"]))
else:
ppe.hget("AGPARTVALUES:%s:%d:%s" % \
(inst, part, elem["key"]), elem["type"])
# We need to execute this pipeline read only if we are
# keeping track of UVE contents (streaming case)
if self._content:
pperes = ppe.execute()
idx = 0
for elem in lelems:
key = elem["key"]
typ = elem["type"]
vdata = None
if not typ is None:
if self._content:
if not key in self._uvecache:
self._uvecache[key] = {}
vjson = pperes[idx]
if vjson is None:
vdata = None
if typ in self._uvecache[key]:
del self._uvecache[key][typ]
else:
vdata = json.loads(vjson)
self._uvecache[key][typ] = vjson
if self._token is not None:
if not self.is_uve_read_permitted(\
self._uvecache[key]):
gevent.sleep(0)
continue
else:
vdata = {}
self._cb(self._partno, self._pi, key, typ, vdata)
idx += 1
gevent.sleep(0)
except gevent.GreenletExit:
break
except (redis.exceptions.ConnectionError,
redis.exceptions.TimeoutError):
pass
except Exception as ex:
self.redis_cur_time = time.time()
if self.redis_prev_time == 0 or self.redis_cur_time - self.redis_prev_time > 60:
self.redis_prev_time = self.redis_cur_time
template = "Exception {0} in uve stream proc. Arguments:\n{1!r}"
messag = template.format(type(ex).__name__, ex.args)
self._logger.error("[%s:%d] AlarmGen %s,%d %s : traceback %s" % \
(self._pi.ip_address, self._pi.port, \
self._pi.instance_id, self._partno, \
messag, traceback.format_exc()))
else:
template = "Exception {0} in uve stream proc. Arguments:\n{1!r}"
messag = template.format(type(ex).__name__, ex.args)
self._logger.error("[%s:%d] AlarmGen %s,%d %s" % \
(self._pi.ip_address, self._pi.port, \
self._pi.instance_id, self._partno, \
messag))
finally:
lredis = None
if pb is not None:
pb.close()
pb = None
pause = True
return None
def _run(self):
lredis = None
pb = None
pause = False
self.redis_prev_time = 0
while True:
try:
if pause:
gevent.sleep(2)
pause = False
lredis = StrictRedisWrapper(
host=self._pi.ip_address,
port=self._pi.port,
password=self._rpass,
db=7, socket_timeout=30)
pb = lredis.pubsub()
inst = self._pi.instance_id
part = self._partno
pb.subscribe('AGPARTPUB:%s:%d' % (inst, part))
self.syncpart(lredis)
while True:
message = pb.get_message()
if not message:
gevent.sleep(0.001)
continue
if message["type"] != "message":
gevent.sleep(0)
continue
dataline = message["data"]
try:
elems = json.loads(dataline)
except:
self._logger.error("AggUVE Parsing failed: %s" % str(message))
gevent.sleep(0)
continue
else:
self._logger.info("AggUVE loading: %s" % str(elems))
if self._content:
ppe = lredis.pipeline()
lelems = []
for elem in elems:
table, barekey = elem["key"].split(":",1)
if self._tablefilt:
if not table in self._tablefilt:
gevent.sleep(0)
continue
if self._patterns:
kfilter_match = False
for pattern in self._patterns:
if pattern.match(barekey):
kfilter_match = True
break
if not kfilter_match:
gevent.sleep(0)
continue
if self._cfilter:
if elem["type"] not in self._cfilter:
gevent.sleep(0)
continue
lelems.append(elem)
if self._content:
# This UVE was deleted
if elem["type"] is None:
ppe.exists("AGPARTVALUES:%s:%d:%s" % \
(inst, part, elem["key"]))
else:
ppe.hget("AGPARTVALUES:%s:%d:%s" % \
(inst, part, elem["key"]), elem["type"])
# We need to execute this pipeline read only if we are
# keeping track of UVE contents (streaming case)
if self._content:
pperes = ppe.execute()
idx = 0
for elem in lelems:
key = elem["key"]
typ = elem["type"]
vdata = None
if not typ is None:
if self._content:
if not key in self._uvecache:
self._uvecache[key] = {}
vjson = pperes[idx]
if vjson is None:
vdata = None
if typ in self._uvecache[key]:
del self._uvecache[key][typ]
else:
vdata = json.loads(vjson)
self._uvecache[key][typ] = vjson
if self._token is not None:
if not self.is_uve_read_permitted(\
self._uvecache[key]):
gevent.sleep(0)
continue
else:
vdata = {}
self._cb(self._partno, self._pi, key, typ, vdata)
idx += 1
gevent.sleep(0)
except gevent.GreenletExit:
break
except (redis.exceptions.ConnectionError,
redis.exceptions.TimeoutError):
pass
except Exception as ex:
self.redis_cur_time = time.time()
if self.redis_prev_time == 0 or self.redis_cur_time - self.redis_prev_time > 60:
self.redis_prev_time = self.redis_cur_time
template = "Exception {0} in uve stream proc. Arguments:\n{1!r}"
messag = template.format(type(ex).__name__, ex.args)
self._logger.error("[%s:%d] AlarmGen %s,%d %s : traceback %s" % \
(self._pi.ip_address, self._pi.port, \
self._pi.instance_id, self._partno, \
messag, traceback.format_exc()))
else:
template = "Exception {0} in uve stream proc. Arguments:\n{1!r}"
messag = template.format(type(ex).__name__, ex.args)
self._logger.error("[%s:%d] AlarmGen %s,%d %s" % \
(self._pi.ip_address, self._pi.port, \
self._pi.instance_id, self._partno, \
messag))
finally:
lredis = None
if pb is not None:
pb.close()
pb = None
pause = True
return None
def _get_uve_content(self, table, barekeys, tfilter, ackfilter, keysonly):
brsp = {}
uveparts = {}
for barekey in barekeys:
part = self._uvedb[table][barekey]["__SOURCE__"]["partition"]
if not part in uveparts:
uveparts[part] = set()
uveparts[part].add(barekey)
for pkey, pvalue in uveparts.iteritems():
pi = self._agp[pkey]
lredis = StrictRedisWrapper(host=pi.ip_address,
port=pi.port,
password=self._rpass,
db=7,
socket_timeout=30)
ppe = lredis.pipeline()
luves = list(uveparts[pkey])
for elem in luves:
if len(tfilter) != 0:
ltypes = tfilter.keys()
ppe.hmget("AGPARTVALUES:%s:%d:%s:%s" % \
(pi.instance_id, pkey, table, elem),
*ltypes)
else:
ppe.hgetall("AGPARTVALUES:%s:%d:%s:%s" % \
(pi.instance_id, pkey, table, elem))
pperes = ppe.execute()
for uidx in range(0, len(luves)):
uvestruct = {}
if len(tfilter) != 0:
for tidx in range(0, len(ltypes)):
if not pperes[uidx][tidx]:
continue
afilter_list = tfilter[ltypes[tidx]]
ppeval = json.loads(pperes[uidx][tidx])
if len(afilter_list) == 0:
uvestruct[ltypes[tidx]] = ppeval
else:
for akey, aval in ppeval.iteritems():
if akey not in afilter_list:
continue
else:
if not ltypes[tidx] in uvestruct:
uvestruct[ltypes[tidx]] = {}
uvestruct[ltypes[tidx]][akey] = aval
else:
for tk, tv in pperes[uidx].iteritems():
uvestruct[tk] = json.loads(tv)
if ackfilter is not None:
if "UVEAlarms" in uvestruct and \
"alarms" in uvestruct["UVEAlarms"]:
alarms = []
for alarm in uvestruct["UVEAlarms"]["alarms"]:
ack = "false"
if "ack" in alarm:
if alarm["ack"]:
ack = "true"
else:
ack = "false"
if ack == ackfilter:
alarms.append(alarm)
if not len(alarms):
del uvestruct["UVEAlarms"]
else:
uvestruct["UVEAlarms"]["alarms"] = alarms
if len(uvestruct) != 0:
if keysonly:
brsp[luves[uidx]] = None
else:
brsp[luves[uidx]] = uvestruct
return brsp
def _get_uve_content(self, table, barekeys, tfilter, ackfilter, keysonly):
brsp = {}
uveparts = {}
for barekey in barekeys:
part = self._uvedb[table][barekey]["__SOURCE__"]["partition"]
if not part in uveparts:
uveparts[part] = set()
uveparts[part].add(barekey)
for pkey,pvalue in uveparts.iteritems():
pi = self._agp[pkey]
lredis = StrictRedisWrapper(
host=pi.ip_address,
port=pi.port,
password=self._rpass,
db=7, socket_timeout=30)
ppe = lredis.pipeline()
luves = list(uveparts[pkey])
for elem in luves:
if len(tfilter) != 0:
ltypes = tfilter.keys()
ppe.hmget("AGPARTVALUES:%s:%d:%s:%s" % \
(pi.instance_id, pkey, table, elem),
*ltypes)
else:
ppe.hgetall("AGPARTVALUES:%s:%d:%s:%s" % \
(pi.instance_id, pkey, table, elem))
pperes = ppe.execute()
for uidx in range(0,len(luves)):
uvestruct = {}
if len(tfilter) != 0:
for tidx in range(0,len(ltypes)):
if not pperes[uidx][tidx]:
continue
afilter_list = tfilter[ltypes[tidx]]
ppeval = json.loads(pperes[uidx][tidx])
if len(afilter_list) == 0:
uvestruct[ltypes[tidx]] = ppeval
else:
for akey, aval in ppeval.iteritems():
if akey not in afilter_list:
continue
else:
if not ltypes[tidx] in uvestruct:
uvestruct[ltypes[tidx]] = {}
uvestruct[ltypes[tidx]][akey] = aval
else:
for tk,tv in pperes[uidx].iteritems():
uvestruct[tk] = json.loads(tv)
if ackfilter is not None:
if "UVEAlarms" in uvestruct and \
"alarms" in uvestruct["UVEAlarms"]:
alarms = []
for alarm in uvestruct["UVEAlarms"]["alarms"]:
ack = "false"
if "ack" in alarm:
if alarm["ack"]:
ack = "true"
else:
ack = "false"
if ack == ackfilter:
alarms.append(alarm)
if not len(alarms):
del uvestruct["UVEAlarms"]
else:
uvestruct["UVEAlarms"]["alarms"] = alarms
if len(uvestruct) != 0:
if keysonly:
brsp[luves[uidx]] = None
else:
brsp[luves[uidx]] = uvestruct
return brsp