def getJobStats(where):
'''
Get Job Statistics.
Returns a list of of stat items, a stat item is a list (triplet)
of identifier (metric name), value, timestamp.
'''
messages = []
statement = """select * from JobStats %s""" % where
logger.debug("Query db: %s" % statement)
response = director.call(".sql query=\"%s\"" % statement)
if not 'query' in response:
return []
for row in response['query']:
jobId = int(row['jobid'])
if not jobId in jobIdMapping.keys():
statement = """select clientid,name from Job where jobid=%s""" % jobId
logger.debug("Query db: %s" % statement)
jobDetails = director.call(".sql query=\"%s\"" % statement)
jobIdMapping[jobId] = jobDetails['query'][0]['clientid']
jobNameMapping[jobId] = jobDetails['query'][0]['name']
clientName = clientIdMapping[jobIdMapping[jobId]]
jobName = jobNameMapping[jobId]
timeObject = datetime.datetime.strptime(row['sampletime'], "%Y-%m-%d %H:%M:%S")
timestamp = int(timeObject.strftime("%s"))
logger.info("%s.clients.%s.jobs.%s.jobbytes %s at %d" %
(metricPrefix, clientName, jobName, row['jobbytes'], timestamp))
messages.append(["%s.clients.%s.jobs.%s.jobbytes" %
(metricPrefix, clientName, jobName), int(row['jobbytes']), timestamp])
messages.append(["%s.clients.%s.jobs.%s.jobfiles" %
(metricPrefix, clientName, jobName), int(row['jobfiles']), timestamp])
return messages
def getDeviceStats(where):
'''
Get DeviceStats in timeframe specified by where.
Returns a list of of stat items, a stat item is a list (triplet)
of identifier (metric name), value, timestamp.
'''
messages = []
statement = """select * from DeviceStats %s""" % where
logger.debug("Query db: %s" % statement)
response = director.call(".sql query=\"%s\"" % statement)
if not 'query' in response:
return []
for row in response['query']:
deviceId = row['deviceid']
if not deviceId in deviceIdMapping.keys():
logger.error("Device ID: %s unknown. Ignoring" % deviceId)
continue
deviceName = deviceIdMapping[deviceId]
timeObject = datetime.datetime.strptime(row['sampletime'],
"%Y-%m-%d %H:%M:%S")
timestamp = int(timeObject.strftime("%s"))
for dataItem in deviceStatKeys:
logger.info("%s.devices.%s.%s %d at %d" %
(metricPrefix, deviceName, dataItem, int(
row[dataItem]), timestamp))
messages.append([
"%s.devices.%s.%s" % (metricPrefix, deviceName, dataItem),
int(row[dataItem]), timestamp
])
return messages
def getDeviceStats(where):
'''
Get DeviceStats in timeframe specified by where.
Returns a list of of stat items, a stat item is a list (triplet)
of identifier (metric name), value, timestamp.
'''
messages = []
statement = """select * from DeviceStats %s""" % where
logger.debug("Query db: %s" % statement)
response = director.call(".sql query=\"%s\"" % statement)
if not 'query' in response:
return []
for row in response['query']:
deviceId = row['deviceid']
if not deviceId in deviceIdMapping.keys():
logger.error("Device ID: %s unknown. Ignoring" % deviceId)
continue
deviceName = deviceIdMapping[deviceId]
timeObject = datetime.datetime.strptime(row['sampletime'], "%Y-%m-%d %H:%M:%S")
timestamp = int(timeObject.strftime("%s"))
for dataItem in deviceStatKeys:
logger.info("%s.devices.%s.%s %d at %d"
% (metricPrefix, deviceName, dataItem, int(row[dataItem]), timestamp))
messages.append(["%s.devices.%s.%s" % (metricPrefix, deviceName, dataItem), int(row[dataItem]), timestamp])
return messages
def getJobStats(where):
'''
Get Job Statistics.
Returns a list of of stat items, a stat item is a list (triplet)
of identifier (metric name), value, timestamp.
'''
messages = []
statement = """select * from JobStats %s""" % where
logger.debug("Query db: %s" % statement)
response = director.call(".sql query=\"%s\"" % statement)
if not 'query' in response:
return []
for row in response['query']:
jobId = int(row['jobid'])
if not jobId in jobIdMapping.keys():
statement = """select clientid,name from Job where jobid=%s""" % jobId
logger.debug("Query db: %s" % statement)
jobDetails = director.call(".sql query=\"%s\"" % statement)
jobIdMapping[jobId] = jobDetails['query'][0]['clientid']
jobNameMapping[jobId] = jobDetails['query'][0]['name']
clientName = clientIdMapping[jobIdMapping[jobId]]
jobName = jobNameMapping[jobId]
timeObject = datetime.datetime.strptime(row['sampletime'], "%Y-%m-%d %H:%M:%S")
timestamp = int(timeObject.strftime("%s"))
logger.info("%s.clients.%s.jobs.%s.jobbytes %s at %d" %
(metricPrefix, clientName, jobName, row['jobbytes'], timestamp))
messages.append(["%s.clients.%s.jobs.%s.jobbytes" %
(metricPrefix, clientName, jobName), int(row['jobbytes']), timestamp])
messages.append(["%s.clients.%s.jobs.%s.jobfiles" %
(metricPrefix, clientName, jobName), int(row['jobfiles']), timestamp])
return messages
def getPoolStatistics():
'''
Get Statisctics regarding all defined pools
'''
messages = []
now = int(time.time())
response = director.call('list volumes')
for pool in response['volumes'].keys():
totalBytes = 0
numVolumes = 0
volStatus = dict()
for volume in response['volumes'][pool]:
totalBytes += int(volume['volbytes'])
numVolumes += 1
status = volume['volstatus']
if status not in volStatus.keys():
volStatus[status] = 1
else:
volStatus[status] += 1
logger.info("pool: %s, bytes: %d Volumes: %d" % (pool, totalBytes, numVolumes))
messages.append(["%s.pools.%s.totalBytes" % (metricPrefix, pool), totalBytes, now])
messages.append(["%s.pools.%s.totalVolumes" % (metricPrefix, pool), numVolumes, now])
for status in volStatus.keys():
messages.append(["%s.pools.%s.volumeStatus.%s" % (metricPrefix, pool, status), volStatus[status], now])
return messages
def getPoolStatistics():
'''
Get Statisctics regarding all defined pools
'''
messages = []
now = int(time.time())
response = director.call('list volumes')
for pool in response['volumes'].keys():
totalBytes = 0
numVolumes = 0
volStatus = dict()
for volume in response['volumes'][pool]:
totalBytes += int(volume['volbytes'])
numVolumes += 1
status = volume['volstatus']
if status not in volStatus.keys():
volStatus[status] = 1
else:
volStatus[status] += 1
logger.info("pool: %s, bytes: %d Volumes: %d" %
(pool, totalBytes, numVolumes))
messages.append(
["%s.pools.%s.totalBytes" % (metricPrefix, pool), totalBytes, now])
messages.append([
"%s.pools.%s.totalVolumes" % (metricPrefix, pool), numVolumes, now
])
for status in volStatus.keys():
messages.append([
"%s.pools.%s.volumeStatus.%s" % (metricPrefix, pool, status),
volStatus[status], now
])
return messages
def getJobTotals():
'''
Get total bytes / files per job
'''
messages = []
now = int(time.time())
response = director.call("list jobtotals")
totalFiles = response['jobtotals']['files']
totalBytes = response['jobtotals']['bytes']
totalJobs = response['jobtotals']['jobs']
messages.append([metricPrefix+".totalFiles", int(totalFiles), now])
messages.append([metricPrefix+".totalBytes", int(totalBytes), now])
messages.append([metricPrefix+".totalJobs", int(totalJobs), now])
for job in response['jobs']:
jobFiles = job['files']
jobName = job['job']
jobBytes = job['bytes']
jobCount = job['jobs']
logger.info("job: %s, files: %s, count: %s, bytes: %s" % (jobName, jobFiles, jobCount, jobBytes))
messages.append(["%s.jobs.%s.totalFiles" % (metricPrefix, jobName), jobFiles, now])
messages.append(["%s.jobs.%s.totalBytes" % (metricPrefix, jobName), jobBytes, now])
messages.append(["%s.jobs.%s.jobCount" % (metricPrefix, jobName), jobCount, now])
return messages
logger.debug("Processing JobTotals")
messages = getJobTotals()
sentItemsToCarbon(messages, carbonHost, carbonPort)
messages = []
logger.debug("Processing Pools")
messages = getPoolStatistics()
sentItemsToCarbon(messages, carbonHost, carbonPort)
messages = []
# get number of running jobs
response = director.call('llist jobs jobstatus=R count')
numRunningJobs = response['jobs'][0]['count']
logger.info("runningJobs: %s" % (numRunningJobs))
messages.append(["%s.jobsRunning" % metricPrefix, numRunningJobs, int(now.strftime("%s"))])
sentItemsToCarbon(messages, carbonHost, carbonPort)
messages = []
if args.events:
logger.debug("Processing events")
events = getEvents(whereJobs)
sendEventsToCarbon(eventUrl, events)
events = []
if args.devicestats:
logger.debug("Processing devicestats")
messages = getDeviceStats(where)
sentItemsToCarbon(messages, carbonHost, carbonPort)
messages = []
def getJobTotals():
'''
Get total bytes / files per job
'''
messages = []
now = int(time.time())
response = director.call("list jobtotals")
totalFiles = response['jobtotals']['files']
totalBytes = response['jobtotals']['bytes']
totalJobs = response['jobtotals']['jobs']
messages.append([metricPrefix + ".totalFiles", int(totalFiles), now])
messages.append([metricPrefix + ".totalBytes", int(totalBytes), now])
messages.append([metricPrefix + ".totalJobs", int(totalJobs), now])
for job in response['jobs']:
jobFiles = job['files']
jobName = job['job']
jobBytes = job['bytes']
jobCount = job['jobs']
logger.info("job: %s, files: %s, count: %s, bytes: %s" %
(jobName, jobFiles, jobCount, jobBytes))
messages.append(
["%s.jobs.%s.totalFiles" % (metricPrefix, jobName), jobFiles, now])
messages.append(
["%s.jobs.%s.totalBytes" % (metricPrefix, jobName), jobBytes, now])
messages.append(
["%s.jobs.%s.jobCount" % (metricPrefix, jobName), jobCount, now])
return messages
logger.debug("Processing JobTotals")
messages = getJobTotals()
sentItemsToCarbon(messages, carbonHost, carbonPort)
messages = []
logger.debug("Processing Pools")
messages = getPoolStatistics()
sentItemsToCarbon(messages, carbonHost, carbonPort)
messages = []
# get number of running jobs
response = director.call('llist jobs jobstatus=R count')
numRunningJobs = response['jobs'][0]['count']
logger.info("runningJobs: %s" % (numRunningJobs))
messages.append([
"%s.jobsRunning" % metricPrefix, numRunningJobs,
int(now.strftime("%s"))
])
sentItemsToCarbon(messages, carbonHost, carbonPort)
messages = []
if args.events:
logger.debug("Processing events")
events = getEvents(whereJobs)
sendEventsToCarbon(eventUrl, events)
events = []
if args.devicestats:
logger.debug("Processing devicestats")
messages = getDeviceStats(where)
sentItemsToCarbon(messages, carbonHost, carbonPort)
