class ServiceConfigure(object):
'''
ServiceConfigure is meant to be a generic class for any service
which registers itself to, and gets configuration from UNIS. It
was originally developed for BLiPP, but BLiPP specific features
should be in the BlippConfigure class which extends
ServiceConfigure.
'''
def __init__(self, initial_config={}, node_id=None, urn=None):
if not node_id:
node_id = settings.UNIS_ID
self.node_id = node_id
self.urn = urn
self.config = initial_config
self.unis = UNISInstance(self.config)
self.service_setup = False
def initialize(self):
self._setup_node(self.node_id)
self._setup_service()
def refresh(self):
r = self.unis.get("/services/" + self.config["id"])
if not r:
logger.warn('refresh', msg="refresh failed")
else:
self.config = r
def _setup_node(self, node_id):
config = self.config
logger.debug('_setup_node', config=pprint.pformat(config))
hostname = settings.HOSTNAME
urn = settings.HOST_URN if not self.urn else self.urn
if node_id:
r = self.unis.get("/nodes/" + str(node_id))
if not r:
logger.warn('_setup_node', msg="node id %s not found" % node_id)
node_id = None
if not node_id:
r = self.unis.get("/nodes?urn=" + urn)
if r and len(r):
r = r[0]
logger.info('_setup_node',
msg="found node with our URN and id %s" % r["id"])
else:
r = self.unis.post("/nodes",
data={
"$schema": settings.SCHEMAS["nodes"],
"name": hostname,
"urn": urn})
if r:
self.node_id = r["id"]
if r:
config["runningOn"] = {
"href": r["selfRef"],
"rel": "full"}
self.node_setup = True
else:
config["runningOn"] = {"href": ""}
logger.warn('_setup_node', msg="Unable to set up node in UNIS")
def _setup_service(self):
config = self.config
logger.debug('_setup_service', config=pprint.pformat(config))
r = None
if config.get("id", None):
r = self.unis.get("/services/" + config["id"])
if not r:
logger.warn('_setup_service',
msg="service id not specified or not found "\
"unis instance ...querying for service")
rlist = self.unis.get("/services?name=" + config.get("name", None) +\
"&runningOn.href=" + config["runningOn"]["href"] + "&limit=2")
# loop over the returned services and find one that
# doesn't return 410 see
# https://uisapp2.iu.edu/jira-prd/browse/GEMINI-98
if rlist:
for i in range(len(rlist)):
r = self.unis.get('/services/' + rlist[i]["id"])
if r:
if isinstance(r, list):
logger.warn('_setup_service',
msg="id not unique... taking first result")
r = r[0]
logger.info('_setup_service',
msg="%s service found with id %s" % (config["name"], r["id"]))
break
else:
logger.warn('_setup_service',
msg="no service found by id or querying "\
"...creating new service")
if r:
merge_dicts(config, r)
# always update UNIS with the merged config
if config.get("id", None):
r = self.unis.put("/services/" + config["id"], data=config)
else:
r = self.unis.post("/services", data=config)
if r:
merge_dicts(config, r)
if r:
self.service_setup = True
else:
logger.warn('_setup_service', msg="unable to set up service in UNIS")
def get(self, key, default=None):
try:
return self.config[key]
except KeyError:
return default
def __getitem__(self, key):
'''
This allows an object which is an instance of this class to behave
like a dictionary when queried with [] syntax
'''
return self.config[key]
class Collector:
"""Collects reported measurements and aggregates them for
sending to MS at appropriate intervals.
Also does a bunch of other stuff which should probably be handled by separate classes.
Creates all the metadata objects, and the measurement object in UNIS for all data inserted.
Depends directly on the MS and UNIS... output could be far more modular.
"""
def __init__(self, service, measurement):
self.config = measurement["configuration"]
self.service = service
self.measurement = measurement
self.collections_created = False
self.ms = MSInstance(service, measurement)
self.dl = DataLogger(service, measurement)
self.mids = {} # {subject1: {metric1:mid, metric2:mid}, subj2: {...}}
# {mid: [{"ts": ts, "value": val}, {"ts": ts, "value": val}]}
self.mid_to_data = {}
self.mid_to_et = {}
self.unis = UNISInstance(service)
self.num_collected = 0
def insert(self, data, ts):
'''
Called (by probe_runner) to insert new data into this collector object.
'''
mids = self.mids
for subject, met_val in data.iteritems():
if "ts" in met_val:
ts = met_val["ts"]
del met_val["ts"]
for metric, value in met_val.iteritems():
if metric not in self.measurement["eventTypes"]:
self._add_et(metric)
if not metric in mids.get(subject, {}):
r = self.unis.find_or_create_metadata(subject,
metric,
self.measurement)
mids.setdefault(subject, {})[metric] = r["id"]
self.mid_to_data[r["id"]] = []
self.mid_to_et[mids[subject][metric]] = metric
self._insert_datum(mids[subject][metric], ts, value)
self.num_collected += 1
if self.num_collected >= self.config["reporting_params"]:
ret = self.report()
if ret:
self.num_collected = 0
def _insert_datum(self, mid, ts, val):
item = dict({"ts": ts * 10e5,
"value":val})
self.mid_to_data[mid].append(item)
def _add_et(self, metric):
self.measurement["eventTypes"].append(metric)
r = self.unis.put("/measurements/" +
self.measurement["id"],
data=self.measurement)
if r:
self.measurement = r
def report(self):
'''
Send all data collected so far, then clear stored data.
'''
post_data = []
for mid, data in self.mid_to_data.iteritems():
if len(data):
post_data.append({"mid":mid, "data":data})
ms_ret = self.ms.post_data(post_data)
dl_ret = self.dl.write_data(post_data, self.mid_to_et)
if not ms_ret and not dl_ret and self.num_collected < self.config["reporting_tolerance"] * self.config["reporting_params"]:
return None
self._clear_data()
return True
def _clear_data(self):
for mid in self.mid_to_data:
self.mid_to_data[mid]=[]
