def getJobStats(where): ''' Get Job Statistics. Returns a list of of stat items, a stat item is a list (triplet) of identifier (metric name), value, timestamp. ''' messages = [] statement = """select * from JobStats %s""" % where logger.debug("Query db: %s" % statement) response = director.call(".sql query=\"%s\"" % statement) if not 'query' in response: return [] for row in response['query']: jobId = int(row['jobid']) if not jobId in jobIdMapping.keys(): statement = """select clientid,name from Job where jobid=%s""" % jobId logger.debug("Query db: %s" % statement) jobDetails = director.call(".sql query=\"%s\"" % statement) jobIdMapping[jobId] = jobDetails['query'][0]['clientid'] jobNameMapping[jobId] = jobDetails['query'][0]['name'] clientName = clientIdMapping[jobIdMapping[jobId]] jobName = jobNameMapping[jobId] timeObject = datetime.datetime.strptime(row['sampletime'], "%Y-%m-%d %H:%M:%S") timestamp = int(timeObject.strftime("%s")) logger.info("%s.clients.%s.jobs.%s.jobbytes %s at %d" % (metricPrefix, clientName, jobName, row['jobbytes'], timestamp)) messages.append(["%s.clients.%s.jobs.%s.jobbytes" % (metricPrefix, clientName, jobName), int(row['jobbytes']), timestamp]) messages.append(["%s.clients.%s.jobs.%s.jobfiles" % (metricPrefix, clientName, jobName), int(row['jobfiles']), timestamp]) return messages
def getDeviceStats(where): ''' Get DeviceStats in timeframe specified by where. Returns a list of of stat items, a stat item is a list (triplet) of identifier (metric name), value, timestamp. ''' messages = [] statement = """select * from DeviceStats %s""" % where logger.debug("Query db: %s" % statement) response = director.call(".sql query=\"%s\"" % statement) if not 'query' in response: return [] for row in response['query']: deviceId = row['deviceid'] if not deviceId in deviceIdMapping.keys(): logger.error("Device ID: %s unknown. Ignoring" % deviceId) continue deviceName = deviceIdMapping[deviceId] timeObject = datetime.datetime.strptime(row['sampletime'], "%Y-%m-%d %H:%M:%S") timestamp = int(timeObject.strftime("%s")) for dataItem in deviceStatKeys: logger.info("%s.devices.%s.%s %d at %d" % (metricPrefix, deviceName, dataItem, int( row[dataItem]), timestamp)) messages.append([ "%s.devices.%s.%s" % (metricPrefix, deviceName, dataItem), int(row[dataItem]), timestamp ]) return messages
def getDeviceStats(where): ''' Get DeviceStats in timeframe specified by where. Returns a list of of stat items, a stat item is a list (triplet) of identifier (metric name), value, timestamp. ''' messages = [] statement = """select * from DeviceStats %s""" % where logger.debug("Query db: %s" % statement) response = director.call(".sql query=\"%s\"" % statement) if not 'query' in response: return [] for row in response['query']: deviceId = row['deviceid'] if not deviceId in deviceIdMapping.keys(): logger.error("Device ID: %s unknown. Ignoring" % deviceId) continue deviceName = deviceIdMapping[deviceId] timeObject = datetime.datetime.strptime(row['sampletime'], "%Y-%m-%d %H:%M:%S") timestamp = int(timeObject.strftime("%s")) for dataItem in deviceStatKeys: logger.info("%s.devices.%s.%s %d at %d" % (metricPrefix, deviceName, dataItem, int(row[dataItem]), timestamp)) messages.append(["%s.devices.%s.%s" % (metricPrefix, deviceName, dataItem), int(row[dataItem]), timestamp]) return messages
def getJobStats(where): ''' Get Job Statistics. Returns a list of of stat items, a stat item is a list (triplet) of identifier (metric name), value, timestamp. ''' messages = [] statement = """select * from JobStats %s""" % where logger.debug("Query db: %s" % statement) response = director.call(".sql query=\"%s\"" % statement) if not 'query' in response: return [] for row in response['query']: jobId = int(row['jobid']) if not jobId in jobIdMapping.keys(): statement = """select clientid,name from Job where jobid=%s""" % jobId logger.debug("Query db: %s" % statement) jobDetails = director.call(".sql query=\"%s\"" % statement) jobIdMapping[jobId] = jobDetails['query'][0]['clientid'] jobNameMapping[jobId] = jobDetails['query'][0]['name'] clientName = clientIdMapping[jobIdMapping[jobId]] jobName = jobNameMapping[jobId] timeObject = datetime.datetime.strptime(row['sampletime'], "%Y-%m-%d %H:%M:%S") timestamp = int(timeObject.strftime("%s")) logger.info("%s.clients.%s.jobs.%s.jobbytes %s at %d" % (metricPrefix, clientName, jobName, row['jobbytes'], timestamp)) messages.append(["%s.clients.%s.jobs.%s.jobbytes" % (metricPrefix, clientName, jobName), int(row['jobbytes']), timestamp]) messages.append(["%s.clients.%s.jobs.%s.jobfiles" % (metricPrefix, clientName, jobName), int(row['jobfiles']), timestamp]) return messages
def getPoolStatistics(): ''' Get Statisctics regarding all defined pools ''' messages = [] now = int(time.time()) response = director.call('list volumes') for pool in response['volumes'].keys(): totalBytes = 0 numVolumes = 0 volStatus = dict() for volume in response['volumes'][pool]: totalBytes += int(volume['volbytes']) numVolumes += 1 status = volume['volstatus'] if status not in volStatus.keys(): volStatus[status] = 1 else: volStatus[status] += 1 logger.info("pool: %s, bytes: %d Volumes: %d" % (pool, totalBytes, numVolumes)) messages.append(["%s.pools.%s.totalBytes" % (metricPrefix, pool), totalBytes, now]) messages.append(["%s.pools.%s.totalVolumes" % (metricPrefix, pool), numVolumes, now]) for status in volStatus.keys(): messages.append(["%s.pools.%s.volumeStatus.%s" % (metricPrefix, pool, status), volStatus[status], now]) return messages
def getPoolStatistics(): ''' Get Statisctics regarding all defined pools ''' messages = [] now = int(time.time()) response = director.call('list volumes') for pool in response['volumes'].keys(): totalBytes = 0 numVolumes = 0 volStatus = dict() for volume in response['volumes'][pool]: totalBytes += int(volume['volbytes']) numVolumes += 1 status = volume['volstatus'] if status not in volStatus.keys(): volStatus[status] = 1 else: volStatus[status] += 1 logger.info("pool: %s, bytes: %d Volumes: %d" % (pool, totalBytes, numVolumes)) messages.append( ["%s.pools.%s.totalBytes" % (metricPrefix, pool), totalBytes, now]) messages.append([ "%s.pools.%s.totalVolumes" % (metricPrefix, pool), numVolumes, now ]) for status in volStatus.keys(): messages.append([ "%s.pools.%s.volumeStatus.%s" % (metricPrefix, pool, status), volStatus[status], now ]) return messages
def getJobTotals(): ''' Get total bytes / files per job ''' messages = [] now = int(time.time()) response = director.call("list jobtotals") totalFiles = response['jobtotals']['files'] totalBytes = response['jobtotals']['bytes'] totalJobs = response['jobtotals']['jobs'] messages.append([metricPrefix+".totalFiles", int(totalFiles), now]) messages.append([metricPrefix+".totalBytes", int(totalBytes), now]) messages.append([metricPrefix+".totalJobs", int(totalJobs), now]) for job in response['jobs']: jobFiles = job['files'] jobName = job['job'] jobBytes = job['bytes'] jobCount = job['jobs'] logger.info("job: %s, files: %s, count: %s, bytes: %s" % (jobName, jobFiles, jobCount, jobBytes)) messages.append(["%s.jobs.%s.totalFiles" % (metricPrefix, jobName), jobFiles, now]) messages.append(["%s.jobs.%s.totalBytes" % (metricPrefix, jobName), jobBytes, now]) messages.append(["%s.jobs.%s.jobCount" % (metricPrefix, jobName), jobCount, now]) return messages
logger.debug("Processing JobTotals") messages = getJobTotals() sentItemsToCarbon(messages, carbonHost, carbonPort) messages = [] logger.debug("Processing Pools") messages = getPoolStatistics() sentItemsToCarbon(messages, carbonHost, carbonPort) messages = [] # get number of running jobs response = director.call('llist jobs jobstatus=R count') numRunningJobs = response['jobs'][0]['count'] logger.info("runningJobs: %s" % (numRunningJobs)) messages.append(["%s.jobsRunning" % metricPrefix, numRunningJobs, int(now.strftime("%s"))]) sentItemsToCarbon(messages, carbonHost, carbonPort) messages = [] if args.events: logger.debug("Processing events") events = getEvents(whereJobs) sendEventsToCarbon(eventUrl, events) events = [] if args.devicestats: logger.debug("Processing devicestats") messages = getDeviceStats(where) sentItemsToCarbon(messages, carbonHost, carbonPort) messages = []
def getJobTotals(): ''' Get total bytes / files per job ''' messages = [] now = int(time.time()) response = director.call("list jobtotals") totalFiles = response['jobtotals']['files'] totalBytes = response['jobtotals']['bytes'] totalJobs = response['jobtotals']['jobs'] messages.append([metricPrefix + ".totalFiles", int(totalFiles), now]) messages.append([metricPrefix + ".totalBytes", int(totalBytes), now]) messages.append([metricPrefix + ".totalJobs", int(totalJobs), now]) for job in response['jobs']: jobFiles = job['files'] jobName = job['job'] jobBytes = job['bytes'] jobCount = job['jobs'] logger.info("job: %s, files: %s, count: %s, bytes: %s" % (jobName, jobFiles, jobCount, jobBytes)) messages.append( ["%s.jobs.%s.totalFiles" % (metricPrefix, jobName), jobFiles, now]) messages.append( ["%s.jobs.%s.totalBytes" % (metricPrefix, jobName), jobBytes, now]) messages.append( ["%s.jobs.%s.jobCount" % (metricPrefix, jobName), jobCount, now]) return messages
logger.debug("Processing JobTotals") messages = getJobTotals() sentItemsToCarbon(messages, carbonHost, carbonPort) messages = [] logger.debug("Processing Pools") messages = getPoolStatistics() sentItemsToCarbon(messages, carbonHost, carbonPort) messages = [] # get number of running jobs response = director.call('llist jobs jobstatus=R count') numRunningJobs = response['jobs'][0]['count'] logger.info("runningJobs: %s" % (numRunningJobs)) messages.append([ "%s.jobsRunning" % metricPrefix, numRunningJobs, int(now.strftime("%s")) ]) sentItemsToCarbon(messages, carbonHost, carbonPort) messages = [] if args.events: logger.debug("Processing events") events = getEvents(whereJobs) sendEventsToCarbon(eventUrl, events) events = [] if args.devicestats: logger.debug("Processing devicestats") messages = getDeviceStats(where) sentItemsToCarbon(messages, carbonHost, carbonPort)