def collectApp(self):
#获取所有的过去时段完成的app的列表
apps = self.getAppList()
if not apps or not apps["apps"]:
logger.info("no appid match")
return
startCollectTime = time.time()
#轮询app列表，获取每个app的详细信息
for app in apps["apps"]["app"]:
startTime = time.time()
appid = app["id"]
if app['state'] == 'FINISHED':
try:
jobid = util.appidToJobid(appid)
jobHistory = self.getJobHistory(jobid)
if jobHistory:
jobCounter = self.getJobCounter(jobid)
jobTasks = self.getJobAllTask(jobid)
self.updateWithAppid(app, jobHistory, jobCounter)
else:
logger.info(
"find some app run success but no history file:" +
appid)
except:
logger.exception("get error while doing app " + appid)
endTime = time.time()
else:
self.updateWithNotSuccAppid(app)
logger.info("getting appid: %s using %d ms" %
(appid, (endTime - startTime) * 1000))
endCollectTime = time.time()
logger.info("using %d ms to collect the data" %
((endCollectTime - startCollectTime) * 1000))
startFlushTime = time.time()
#提交数据
session = database.getSession()
for (appid, appRecordValue) in self.appList.items():
session.merge(appRecordValue)
session.commit()
logger.info("push %d appRecord into table" % (len(self.appList)))
for (key, nmRecordValue) in self.nmList.items():
session.merge(nmRecordValue)
session.commit()
logger.info("push %d nmRecord into table" % (len(self.nmList)))
for (key, rmRecordValue) in self.rmList.items():
session.merge(rmRecordValue)
session.commit()
logger.info("push %d rmRecord into table" % (len(self.rmList)))
endFlushTime = time.time()
logger.info("using %d ms to push to the db" %
((endFlushTime - startFlushTime) * 1000))
def collectApp(self):
#获取所有的过去时段完成的app的列表
apps = self.getAppList()
if not apps or not apps["apps"]:
logger.info("no appid match")
return
startCollectTime = time.time()
#轮询app列表，获取每个app的详细信息
for app in apps["apps"]["app"]:
startTime = time.time()
appid = app["id"]
if app['state'] == 'FINISHED':
try:
jobid = util.appidToJobid(appid)
jobHistory = self.getJobHistory(jobid)
if jobHistory:
jobCounter = self.getJobCounter(jobid)
jobTasks = self.getJobAllTask(jobid)
self.updateWithAppid(app,jobHistory,jobCounter)
else:
logger.info("find some app run success but no history file:"+appid)
except:
logger.exception("get error while doing app "+appid)
endTime = time.time()
else:
self.updateWithNotSuccAppid(app)
logger.info("getting appid: %s using %d ms" % (appid, (endTime - startTime)*1000))
endCollectTime = time.time()
logger.info("using %d ms to collect the data" % ((endCollectTime - startCollectTime)*1000) )
startFlushTime = time.time()
#提交数据
session = database.getSession()
for (appid,appRecordValue) in self.appList.items():
session.merge(appRecordValue)
session.commit()
logger.info("push %d appRecord into table" % (len(self.appList)))
for (key,nmRecordValue) in self.nmList.items():
session.merge(nmRecordValue)
session.commit()
logger.info("push %d nmRecord into table" % (len(self.nmList)))
for (key,rmRecordValue) in self.rmList.items():
session.merge(rmRecordValue)
session.commit()
logger.info("push %d rmRecord into table" % (len(self.rmList)))
endFlushTime = time.time()
logger.info("using %d ms to push to the db" % ((endFlushTime - startFlushTime)*1000))
def collectMetrics(self):
#获取当前集群的状态
metrics = self.getMetrics()
meRecord = metricsRecord(self.recordTime);
recordKey=["appsCompleted","appsPending","appsRunning",
"appsFailed","appsKilled","totalMB","allocatedMB",
"containersAllocated","containersReserved",
"containersPending","totalNodes","activeNodes"]
print metrics
for key in recordKey:
print metrics['clusterMetrics'][key]
meRecord.set(key,metrics['clusterMetrics'][key])
session = database.getSession()
session.merge(meRecord)
session.commit()
def collectMetrics(self):
#获取当前集群的状态
metrics = self.getMetrics()
meRecord = metricsRecord(self.recordTime)
recordKey = [
"appsCompleted", "appsPending", "appsRunning", "appsFailed",
"appsKilled", "totalMB", "allocatedMB", "containersAllocated",
"containersReserved", "containersPending", "totalNodes",
"activeNodes"
]
print metrics
for key in recordKey:
print metrics['clusterMetrics'][key]
meRecord.set(key, metrics['clusterMetrics'][key])
session = database.getSession()
session.merge(meRecord)
session.commit()
def collectMetrics(self):
#获取当前集群的状态
metrics = self.getMetrics()
#recordTime指向上个10分钟的开头时间，所以要往后移动一个10分钟
meRecord = metricsRecord(self.recordTime + config.collect_interval)
recordKey = [
"appsCompleted", "appsPending", "appsRunning", "appsFailed",
"appsKilled", "totalMB", "availableMB", "allocatedMB",
"containersAllocated", "totalNodes", "activeNodes"
]
#totalMB=availableMB+allocatedMB
temp = metrics['clusterMetrics']
temp["totalMB"] = int(temp["availableMB"]) + int(temp["allocatedMB"])
for key in recordKey:
meRecord.set(key, temp[key])
session = database.getSession()
session.merge(meRecord)
session.commit()
def collectMetrics(self):
#获取当前集群的状态
metrics = self.getMetrics()
#recordTime指向上个10分钟的开头时间，所以要往后移动一个10分钟
meRecord = metricsRecord(self.recordTime+config.collect_interval);
recordKey=["appsCompleted","appsPending","appsRunning",
"appsFailed","appsKilled",
"totalMB","availableMB",
"allocatedMB","containersAllocated",
"totalNodes","activeNodes"]
#totalMB=availableMB+allocatedMB
temp = metrics['clusterMetrics'];
temp["totalMB"] = int(temp["availableMB"]) + int (temp["allocatedMB"])
for key in recordKey:
meRecord.set(key,temp[key])
session = database.getSession()
session.merge(meRecord)
session.commit()
