def onSuccess(self, results, config):
# Create of map of ref to datasource.
datasources = dict(
(x.params['xenapi_ref'], x) for x in config.datasources)
data = self.new_data()
for ref, properties in results.iteritems():
datasource = datasources.get(ref)
if not datasource:
# We're not monitoring whatever this thing is. Skip it.
continue
points = dict((x.path, x) for x in datasource.points)
for path, point in points.items():
value = properties.get(path)
if value is None:
continue
if point.rpn:
try:
value = rpneval(value, point.rpn)
except Exception:
LOG.exception('Failed to evaluate RPN: %s', point.rpn)
continue
data['values'][datasource.component][point.id] = (value, 'N')
# Prune points so we know what's missing.
del(points[path])
if points:
LOG.debug(
"missing values for %s:%s:%s %s",
config.id,
datasource.component,
datasource.datasource,
points.keys())
# Prune datasources so we know what's missing.
del(datasources[ref])
if datasources:
LOG.debug(
"missing XenAPI data for %s:%s %s",
config.id,
config.datasources[0].params['xenapi_classname'],
datasources.keys())
LOG.debug(
'success for %s XenAPI %s',
config.id,
config.datasources[0].params['xenapi_classname'])
data['events'].append(get_event(config, 'successful collection', 0))
return data
def getGraphValue(self, context, getfunc):
"""when graphing use this so that rpn conversions are accounted for"""
val = getfunc(context)
if val is None or len(self.dsnames) == 0: return
dp = self.getRRDDataPoint(self.dsnames[0])
if dp and dp.rpn:
#When VDEF does full rpn
#val = "%s,%s" % (val, dp.rpn)
val = rpneval(val, dp.rpn)
return val
def onSuccess(self, results, config):
# Create of map of ref to datasource.
datasources = dict(
(x.params['xenapi_ref'], x) for x in config.datasources)
data = self.new_data()
for ref, properties in results.iteritems():
datasource = datasources.get(ref)
if not datasource:
# We're not monitoring whatever this thing is. Skip it.
continue
points = dict((x.path, x) for x in datasource.points)
for path, point in points.items():
value = properties.get(path)
if value is None:
continue
if point.rpn:
try:
value = rpneval(value, point.rpn)
except Exception:
LOG.exception('Failed to evaluate RPN: %s', point.rpn)
continue
data['values'][datasource.component][point.id] = (value, 'N')
# Prune points so we know what's missing.
del (points[path])
if points:
LOG.debug("missing values for %s:%s:%s %s", config.id,
datasource.component, datasource.datasource,
points.keys())
# Prune datasources so we know what's missing.
del (datasources[ref])
if datasources:
LOG.debug("missing XenAPI data for %s:%s %s", config.id,
config.datasources[0].params['xenapi_classname'],
datasources.keys())
LOG.debug('success for %s XenAPI %s', config.id,
config.datasources[0].params['xenapi_classname'])
data['events'].append(get_event(config, 'successful collection', 0))
return data
def collect_xen(self, config, ds0, client):
rrd_tree = collections.defaultdict(
lambda: collections.defaultdict(
lambda: collections.defaultdict(
list)))
for address in ds0.xenserver_addresses:
# We must check what time the host thinks it is so we be
# accurate and efficient about what and how much data we
# request.
server_time = None
time_check_result = yield client.rrd_updates(address, start=1e11)
for _, end in etree.iterparse(StringIO(time_check_result), tag='end'):
server_time = end.text
if not server_time:
continue
# Initialize {int: str} map of column indexes to
# their corresponding entry strings. To be used to match
# data to datapoints.
index_entries = None
start = int(server_time) - ds0.cycletime - 5
result = yield client.rrd_updates(
address, start=start, cf='AVERAGE', host=True)
for _, element in etree.iterparse(StringIO(result)):
if element.tag == 'meta':
step = int(element.findtext('step'))
if ds0.cycletime % step:
LOG.warn(
"%s:%s RRD interval (%s) not evenly divisible into datasource cycle (%s). Skipping collection",
config.id, address, step, ds0.cycletime)
continue
# Map (type, uuid, label) tuples to column indexes.
index_entries = dict(
(i, x.text.split(':')[1:]) for i, x in enumerate(
element.iter('entry')))
elif element.tag == 'row':
for i, v in enumerate(element.iter('v')):
try:
value = float(v.text)
except (TypeError, ValueError):
continue
if not math.isnan(value):
etype, euuid, elabel = index_entries[i]
rrd_tree[etype][euuid][elabel].append(value)
data = self.new_data()
missing_data = collections.defaultdict(list)
for datasource in config.datasources:
prefix = datasource.params.get('prefix')
if not prefix:
continue
datasource_data = rrd_tree[prefix[0]][prefix[1]]
for datapoint in datasource.points:
datapoint_data = []
for elabel in datasource_data.keys():
if elabel == prefix[2]:
datapoint_data.append(datasource_data[elabel])
elif elabel.startswith(prefix[2]):
remainder = elabel.replace(prefix[2], '', 1)
if re.search(datapoint.pattern, remainder):
datapoint_data.append(datasource_data[elabel])
if datapoint_data:
value = aggregate_values(datapoint, datapoint_data)
if datapoint.rpn:
try:
value = rpneval(value, datapoint.rpn)
except Exception:
LOG.exception(
'Failed to evaluate RPN: %s',
datapoint.rpn)
continue
data['values'][datasource.component][datapoint.id] = (
value, 'N')
else:
missing_data[datasource.component].append(datapoint.id)
for component, datapoint_ids in missing_data.items():
LOG.debug(
"missing RRD data for %s:%s %s",
config.id,
component,
datapoint_ids)
returnValue(data)
def onSuccess(self, results, config):
data = self.new_data()
root = etree.Element('root')
# Find and save the cluster tree
for result in results[0]:
tree = etree.parse(StringIO(result))
root.append(tree.getroot())
if tree.getroot().tag == 'clusters':
cluster_tree = tree
for result in results[0]:
result_tree = etree.parse(StringIO(result))
if result_tree.getroot().tag == 'storage_domains':
count = len(result_tree.getroot().getchildren())
addCount(root, [config.id], 'storagedomainCount', count)
elif result_tree.getroot().tag == 'clusters':
count = len(result_tree.getroot().getchildren())
addCount(root, [config.id], 'clusterCount', count)
data_centers = result_tree.xpath('//data_center/@id')
addCount(root, data_centers, 'clusterCount')
elif result_tree.getroot().tag == 'data_centers':
count = len(result_tree.getroot().getchildren())
addCount(root, [config.id], 'datacenterCount', count)
elif result_tree.getroot().tag == 'hosts':
count = len(result_tree.getroot().getchildren())
addCount(root, [config.id], 'hostCount', count)
clusters = result_tree.xpath('//cluster/@id')
addCount(root, clusters, 'hostCount')
for cluster in clusters:
datacenter = cluster_tree.xpath('//cluster[@id="%s"]/data_center/@id' % cluster)
addCount(root, datacenter, 'hostCount', 1)
elif result_tree.getroot().tag == 'vms':
count = len(result_tree.getroot().getchildren())
addCount(root, [config.id], 'vmCount', count)
clusters = result_tree.xpath('//cluster/@id')
addCount(root, clusters, 'vmCount')
hosts = result_tree.xpath('//host/@id')
addCount(root, hosts, 'vmCount')
for cluster in clusters:
datacenter = cluster_tree.xpath('//cluster[@id="%s"]/data_center/@id' % cluster)
addCount(root, datacenter, 'vmCount', 1)
for result_stat in results[1]:
root.append(etree.parse(StringIO(result_stat)).getroot())
# This is the general format ...
#root.xpath('//*[*/@id="368bf44e-7d29-483a-8c2e-9a79962b1e48"][name/text()="disk.read.latency"]/values/value/datum/text()')[0]
for ds in config.datasources:
if ds.component:
component_id = prepId(ds.component)
else:
component_id = None
for point in ds.points:
# Handle percentage custom datapoints
if "ovirt:" in point.xpath and point.rpn and 'xpath' in ds.params:
resultsDict = {}
try:
xpath = talesEvalStr(ds.params['xpath'], context=None, extra=ds.params['context'])
statdata = [(x.xpath('name/text()'), x.xpath('values/value/datum/text()')) for x in root.xpath(xpath) if x.tag == 'statistic']
for item, val in statdata:
resultsDict[item[0]] = val[0]
rpnstring = talesEvalStr(point.rpn, context=None, extra={'here': resultsDict})
results = rpneval(rpnstring.split(',', 1)[0], rpnstring.split(',', 1)[1])
data['values'][component_id][point.id] = (results, 'N')
except Exception:
pass
# Do the rest using xpath
elif 'xpath' in ds.params:
try:
# Some points may not exist in the xml, skip those...
xpath = talesEvalStr(ds.params['xpath'], context=None, extra=ds.params['context'])
results = root.xpath(xpath+point.xpath)
if 'Count' in point.xpath and not results:
results = ['0']
results = results[0]
# If rpn is defined, lets calculate the new results.
if point.rpn:
results = rpneval(
results, talesEvalStr(point.rpn, context=None, extra=ds.params['context']))
data['values'][component_id][point.id] = (results, 'N')
except Exception:
pass
data['events'].append({
'eventClassKey': 'oVirtCollectionSuccess',
'eventKey': eventKey(config),
'summary': 'ovirt: successful collection',
'eventClass': '/Status/Perf/',
'device': config.id,
'severity': 0,
})
return data
def collect_xen(self, config, ds0, client):
rrd_tree = collections.defaultdict(lambda: collections.defaultdict(
lambda: collections.defaultdict(list)))
for address in ds0.xenserver_addresses:
# We must check what time the host thinks it is so we be
# accurate and efficient about what and how much data we
# request.
server_time = None
time_check_result = yield client.rrd_updates(address, start=1e11)
for _, end in etree.iterparse(StringIO(time_check_result),
tag='end'):
server_time = end.text
if not server_time:
continue
# Initialize {int: str} map of column indexes to
# their corresponding entry strings. To be used to match
# data to datapoints.
index_entries = None
start = int(server_time) - ds0.cycletime - 5
result = yield client.rrd_updates(address,
start=start,
cf='AVERAGE',
host=True)
for _, element in etree.iterparse(StringIO(result)):
if element.tag == 'meta':
step = int(element.findtext('step'))
if ds0.cycletime % step:
LOG.warn(
"%s:%s RRD interval (%s) not evenly divisible into datasource cycle (%s). Skipping collection",
config.id, address, step, ds0.cycletime)
continue
# Map (type, uuid, label) tuples to column indexes.
index_entries = dict(
(i, x.text.split(':')[1:])
for i, x in enumerate(element.iter('entry')))
elif element.tag == 'row':
for i, v in enumerate(element.iter('v')):
try:
value = float(v.text)
except (TypeError, ValueError):
continue
if not math.isnan(value):
etype, euuid, elabel = index_entries[i]
rrd_tree[etype][euuid][elabel].append(value)
data = self.new_data()
missing_data = collections.defaultdict(list)
for datasource in config.datasources:
prefix = datasource.params.get('prefix')
if not prefix:
continue
datasource_data = rrd_tree[prefix[0]][prefix[1]]
for datapoint in datasource.points:
datapoint_data = []
for elabel in datasource_data.keys():
if elabel == prefix[2]:
datapoint_data.append(datasource_data[elabel])
elif elabel.startswith(prefix[2]):
remainder = elabel.replace(prefix[2], '', 1)
if re.search(datapoint.pattern, remainder):
datapoint_data.append(datasource_data[elabel])
if datapoint_data:
value = aggregate_values(datapoint, datapoint_data)
if datapoint.rpn:
try:
value = rpneval(value, datapoint.rpn)
except Exception:
LOG.exception('Failed to evaluate RPN: %s',
datapoint.rpn)
continue
data['values'][datasource.component][datapoint.id] = (
value, 'N')
else:
missing_data[datasource.component].append(datapoint.id)
for component, datapoint_ids in missing_data.items():
LOG.debug("missing RRD data for %s:%s %s", config.id, component,
datapoint_ids)
returnValue(data)