def collect(self, config, datasource, rrdcache, collectionTime): collectedEvents = [] collectedValues = {} expression = datasource.params.get('expression', None) debug = datasource.params.get('debug', None) if expression: # We will populate this with perf metrics and pass to eval() devdict = createDeviceDictionary(datasource.params['obj_attrs']) rrdValues = {} datapointDict = {} gotAllRRDValues = True for targetDatasource, targetDatapoint, targetRRA, targetAsRate in datasource.params['targetDatapoints']: try: value = yield rrdcache.getLastValue(targetDatasource, targetDatapoint, targetRRA, targetAsRate, datasource.cycletime*5, datasource.params['targets'][0]) except StandardError as ex: description = dsDescription(datasource, devdict) msg = "Failure before evaluation, %s" % description collectedEvents.append({ 'eventKey': 'calculatedDataSourcePlugin_result', 'severity': ZenEventClasses.Error if debug else ZenEventClasses.Debug, 'summary': msg, }) logMethod = log.error if debug else log.debug logMethod(msg + "\n%s", ex) log.exception(ex) continue # Datapoints can be specified in the following ways: # # 1. <dpname> # 2. <dsname>_<dpname> # 3. datapoint['<dpname>'] # 4. datapoint['<dsname>_<dpname>'] # # Option 1 and 3 can only be used in cases where the # referenced datapoint names are unique for the device # or component. # # Option 1 and 2 can only be used when the datapoint or # datasource_datapoint name are valid Python variable # names. # # Option 4 can only be used when there is not a # datapoint literally named "datapoint". This is most # likely the safest option if you can avoid naming your # datapoints "datapoint". if value is None: gotAllRRDValues = False else: fqdpn = '%s_%s' % (targetDatasource, targetDatapoint) # Syntax 1 rrdValues[targetDatapoint] = value # Syntax 2 rrdValues[fqdpn] = value # Syntax 3 datapointDict[targetDatapoint] = value # Syntax 4 datapointDict[fqdpn] = value if value is not None: rrdValues[targetDatapoint] = value rrdValues['%s_%s' % (targetDatasource, targetDatapoint)] = value result = None if gotAllRRDValues: devdict.update(rrdValues) devdict['datapoint'] = datapointDict description = dsDescription(datasource, devdict) try: result = eval(expression, devdict) if debug: log.debug("Evaluation successful, result is %s for %s" % (result, description)) except ZeroDivisionError: msg = "Evaluation failed due to attempted division by zero, %s" % description collectedEvents.append({ 'eventKey': 'calculatedDataSourcePlugin_result', 'severity': ZenEventClasses.Error if debug else ZenEventClasses.Debug, 'summary': msg, }) logMethod = log.warn if debug else log.debug logMethod(msg) except (TypeError, Exception) as ex: msg = "Evaluation failed due to %s, %s" % (ex.message, description) collectedEvents.append({ 'eventKey': 'calculatedDataSourcePlugin_result', 'severity': ZenEventClasses.Error if debug else ZenEventClasses.Debug, 'summary': msg, }) logMethod = log.exception if debug else log.debug logMethod(msg + "\n%s", ex) else: log.debug("Can't get RRD values for EXPR: %s --> DS: %s" % (expression, dsKey(datasource))) if result is not None: collectedValues.setdefault(datasource.component, {}) collectedValues[datasource.component]['_'.join((datasource.datasource, datasource.points[0].id))] = \ (result, collectionTime) returnValue({ 'events': collectedEvents, 'values': collectedValues, })
def collect(self, config): collectedEvents = [] collectedValues = {} collectedMaps = [] datasourcesByKey = {dsKey(ds): ds for ds in config.datasources} # if we are able prefetch all the metrics that we can if hasattr(self.rrdcache, "batchFetchMetrics"): datasources = [datasourcesByKey.get(ds) for ds in toposort(config.datasources, datasourcesByKey) if datasourcesByKey.get(ds)] yield self.rrdcache.batchFetchMetrics(datasources) startCollectTime = time.time() from collections import defaultdict sourcetypes = defaultdict(int) for dskey in toposort(config.datasources, datasourcesByKey): datasource = datasourcesByKey.get(dskey, None) if datasource is None or \ 'datasourceClassName' not in datasource.params or \ datasource.params['datasourceClassName'] not in DerivedProxyMap: #Not our datasource, it's a dependency from elsewhere #log.warn("not using ds: %s %s %s", dskey, datasource, datasource.params.__dict__) continue collectionTime = time.time() dsResult = yield DerivedProxyMap[datasource.params['datasourceClassName']].collect( config, datasource, self.rrdcache, collectionTime) if dsResult: # Data for this collection won't be written until the current task # is entirely complete. To allow derivations of derivations to complete in # a single collection cycle, we'll artificially cache the values here for # every possible RRA. These values may be slightly inaccurate, as we're # essentially always using the 'LAST' RRA. resultValues = dsResult.get('values', {}).get(datasource.component, {}) if resultValues: collectedPoints = (p for p in datasource.points if p.id in resultValues or \ '_'.join((datasource.datasource, p.id)) in resultValues) for datapoint in collectedPoints: rrdPath = datapoint.rrdPath.rsplit('/', 1)[0] # Datapoint metadata only exists in Zenoss 5. if hasattr(datapoint, 'metadata'): contextUUID = datapoint.metadata["contextUUID"] else: contextUUID = rrdPath value = (resultValues.get(datapoint.id, None) or resultValues.get('_'.join((datasource.datasource, datapoint.id))))[0] for rra in ('AVERAGE', 'MIN', 'MAX', 'LAST'): self.rrdcache.put(datasource.datasource, datapoint.id, rra, rrdPath, contextUUID, value) #incorporate results returned from the proxied method collectedEvents.extend(dsResult.get('events', [])) collectedMaps.extend(dsResult.get('maps', [])) collectedValues.setdefault(datasource.component, {}) collectedValues[datasource.component].update(resultValues) dsclassname = datasource.params['datasourceClassName'] sourcetypes[dsclassname] += 1 endCollectTime = time.time() timeTaken = endCollectTime - startCollectTime timeLogFn = log.debug if timeTaken > 60.0 : timeLogFn = log.warn timeLogFn(" Took %.1f seconds to collect datasources: %s", timeTaken, sourcetypes) returnValue({ 'events': collectedEvents, 'values': collectedValues, 'maps': collectedMaps })
def collect(self, config, datasource, rrdcache, collectionTime): collectedEvents = [] collectedValues = {} debug = datasource.params.get('debug', None) #Aggregate datasources only have one target datapoint config targetDatasource, targetDatapoint, targetRRA, targetAsRate = datasource.params['targetDatapoints'][0] targetValues, errors = yield self.getLastValues(rrdcache, targetDatasource, targetDatapoint, targetRRA, targetAsRate, datasource.cycletime, datasource.params['targets']) logMethod = log.error if debug else log.debug for ex, msg in errors: logMethod('%s: %s', msg, ex) if not targetValues: if datasource.params['targets']: msg = "No target values collected for datasource %s" % dsKey(datasource) collectedEvents.append({ 'summary': msg, 'eventKey': 'aggregatingDataSourcePlugin_novalues', 'severity': ZenEventClasses.Info if debug else ZenEventClasses.Debug, }) logMethod = log.info if debug else log.debug logMethod(msg) returnValue({ 'events': collectedEvents, 'values': collectedValues, }) for datapoint in datasource.points: try: aggregate, adjustedTargetValues = yield self.performAggregation( datapoint.operation, handleArguments(datasource.params['targetArgValues'][0], datapoint.arguments), targetValues) if debug: log.debug("Aggregate value %s calculated for datapoint %s_%s on %s:%s", str(aggregate), datasource.datasource, datapoint.id, datasource.device, datasource.component) except Exception as ex: msg = "Error calculating aggregation for %s_%s: %s" % ( targetDatasource, targetDatapoint, ex.message ) collectedEvents.append({ 'summary': msg, 'eventKey': 'aggregatingDataSourcePlugin_result', 'severity': ZenEventClasses.Error, }) log.exception(msg + "\n%s", ex) continue #stash values for the threshold to put in event details threshold_cache[getThresholdCacheKey(datasource, datapoint)] = adjustedTargetValues collectedValues.setdefault(datasource.component, {}) collectedValues[datasource.component]['_'.join((datasource.datasource, datapoint.id))] = \ (aggregate, collectionTime) returnValue({ 'events': collectedEvents, 'values': collectedValues, })