def process(self):
perf = Performance()
for m in self.monitors:
rsp = m.pull()
if rsp == '':
continue
debug_print('pull response', rsp)
rsp = rsp.replace('reports:', '"reports":')
result = json.loads(rsp)
if len(result['reports']) == 0:
debug_print('WARN', 'reports is empty.')
return perf.str()
#取每个类别的最后一条记录，有时一个应答里同给类别会好几条report记录
#先把结果reverse下，在从前往后取，没有reverse之前，是按时间ASC排序的
#按时间DESC排序后只把最后的有效数据存在valid_category和valid_report，其他的丢弃
valid_category = []
valid_report = []
reports = result['reports']
debug_print('reports', reports)
reports.reverse()
debug_print('reports reverse', reports)
for report in reports:
if valid_category.count(report['report']['category']) == 0:
valid_category.append(report['report']['category'])
valid_report.append(report['report'])
debug_print('valid_report', valid_report)
debug_print('valid_category', valid_category)
for report in valid_report:
columns = []
for c in report['columns']:
columns.append(c['col']['name'])
rows = report['rows']
if(len(rows) == 0):
debug_print('WARN', 'rows is empty')
return perf.str()
group = Group(m.app_name + '_' + report['category'],'category')
for c in report['rows']:
monitor = Monitor(c['row']['instance'], 'test', 'FaeObserver', '127.0.0.1')
for i in range(0, len(columns)):
#判断计数器的值是不是数字，有个TX(error)显示的是最后的错误信息，在此忽略
if not re.match(r'\d{1,}\.\d{1,}',c['row']['data'][i]) and True or False:
debug_print('not digit',c['row']['data'][i])
continue
counter = Counter(columns[i], 'test', c['row']['data'][i])
monitor.counters.append(counter)
group.monitors.append(monitor)
debug_print('monitor', monitor.str())
debug_print('group', group.str())
perf.groups.append(group)
return perf.str()
def demo():
import time
start = time.time()
i = 0
while i < 10000:
rsp = u"{categorys:[{category:{name:'database-sp',instance:'5',columns:[{counter:{name:'tx(/sec.)'}},{counter:{name:'tx(concurrent.)'}},{counter:{name:'tx(failed.)'}},{counter:{name:'tx(cost ms.)'}},{counter:{name:'tx(error)'}}]}},{category:{name:'rpc:tcp-channel',instance:'2',columns:[{counter:{name:'request(bytes/sec)'}},{counter:{name:'request(/sec)'}},{counter:{name:'response(bytes/sec)'}},{counter:{name:'response(/sec)'}},{counter:{name:'connections'}},{counter:{name:'connect pendings'}},{counter:{name:'send pendings'}},{counter:{name:'errors'}}]}},{category:{name:'apps',instance:'8',columns:[{counter:{name:'tx(/sec.)'}},{counter:{name:'tx(concurrent.)'}},{counter:{name:'tx(failed.)'}},{counter:{name:'tx(cost ms.)'}},{counter:{name:'tx(error)'}}]}},{category:{name:'thread-pool',instance:'8',columns:[{counter:{name:'size'}},{counter:{name:'worker(/sec.)'}},{counter:{name:'worker(concurrent.)'}},{counter:{name:'worker(failed.)'}},{counter:{name:'worker(cost ms.)'}},{counter:{name:'worker(error)'}}]}},{category:{name:'logging',instance:'222',columns:[{counter:{name:'warning'}},{counter:{name:'error'}},{counter:{name:'last_exception'}}]}},{category:{name:'rpc-client',instance:'45',columns:[{counter:{name:'tx(/sec.)'}},{counter:{name:'tx(concurrent.)'}},{counter:{name:'tx(failed.)'}},{counter:{name:'tx(cost ms.)'}},{counter:{name:'tx(error)'}}]}},{category:{name:'rpc-server',instance:'7',columns:[{counter:{name:'tx(/sec.)'}},{counter:{name:'tx(concurrent.)'}},{counter:{name:'tx(failed.)'}},{counter:{name:'tx(cost ms.)'}},{counter:{name:'tx(error)'}}]}}]}"
rsp = rsp.replace('categorys:', '"categorys":').replace('category:', '"category":').replace('name', '"name"').replace('instance', '"instance"').replace('columns', '"columns"').replace('counter', '"counter"').replace("'", '"')
#result = json.loads(rsp)
i = i + 1
pass
print time.time() - start
rsp = u"{categorys:[{category:{name:'database-sp',instance:'5',columns:[{counter:{name:'tx(/sec.)'}},{counter:{name:'tx(concurrent.)'}},{counter:{name:'tx(failed.)'}},{counter:{name:'tx(cost ms.)'}},{counter:{name:'tx(error)'}}]}},{category:{name:'rpc:tcp-channel',instance:'2',columns:[{counter:{name:'request(bytes/sec)'}},{counter:{name:'request(/sec)'}},{counter:{name:'response(bytes/sec)'}},{counter:{name:'response(/sec)'}},{counter:{name:'connections'}},{counter:{name:'connect pendings'}},{counter:{name:'send pendings'}},{counter:{name:'errors'}}]}},{category:{name:'apps',instance:'8',columns:[{counter:{name:'tx(/sec.)'}},{counter:{name:'tx(concurrent.)'}},{counter:{name:'tx(failed.)'}},{counter:{name:'tx(cost ms.)'}},{counter:{name:'tx(error)'}}]}},{category:{name:'thread-pool',instance:'8',columns:[{counter:{name:'size'}},{counter:{name:'worker(/sec.)'}},{counter:{name:'worker(concurrent.)'}},{counter:{name:'worker(failed.)'}},{counter:{name:'worker(cost ms.)'}},{counter:{name:'worker(error)'}}]}},{category:{name:'logging',instance:'222',columns:[{counter:{name:'warning'}},{counter:{name:'error'}},{counter:{name:'last_exception'}}]}},{category:{name:'rpc-client',instance:'45',columns:[{counter:{name:'tx(/sec.)'}},{counter:{name:'tx(concurrent.)'}},{counter:{name:'tx(failed.)'}},{counter:{name:'tx(cost ms.)'}},{counter:{name:'tx(error)'}}]}},{category:{name:'rpc-server',instance:'7',columns:[{counter:{name:'tx(/sec.)'}},{counter:{name:'tx(concurrent.)'}},{counter:{name:'tx(failed.)'}},{counter:{name:'tx(cost ms.)'}},{counter:{name:'tx(error)'}}]}}]}"
rsp = rsp.replace('categorys:', '"categorys":').replace('category:', '"category":').replace('name', '"name"').replace('instance', '"instance"').replace('columns', '"columns"').replace('counter', '"counter"').replace("'", '"')
print rsp
result = json.loads(rsp)
#print result['categorys'][0]['category']['name']
for c in result['categorys']:
print c['category']['name']
print '*' * 20
rsp = '{reports:[{"report":{"category":"rpc-server","time":"2013-04-17 10:04:38","columns":[{"col":{"name":"tx(/sec.)","type":"DOUBLE"}},{"col":{"name":"tx(concurrent.)","type":"LONG"}},{"col":{"name":"tx(failed.)","type":"LONG"}},{"col":{"name":"tx(cost ms.)","type":"DOUBLE"}},{"col":{"name":"tx(error)","type":"TEXT"}}],"rows":[{"row":{"instance":"HAWorkerControlleeService.Start|null","data":[0.0,0,0,0.0,""]}},{"row":{"instance":"ISipcOverRpcInterface.requestReceived|SAP","data":[0.0,38674,0,0.0,""]}},{"row":{"instance":"FAE.user-MSProcessUserQuotaRemoteApp|user-SendMessageSipcApp","data":[0.0,0,0,0.0,""]}},{"row":{"instance":"FAE.user-MSSaveOfflineMessageRemoteApp|user-SendMessageSipcApp","data":[0.0,0,0,0.0,""]}},{"row":{"instance":"FAE.user-ValidateMachineCodeRemoteApp|user-SendMessageSipcApp","data":[0.0,0,0,0.0,""]}},{"row":{"instance":"FAE.user-SetUserSmsCountRemoteApp|user-SendMessageSipcApp","data":[0.0,0,0,0.0,""]}},{"row":{"instance":"FAE.user-MSGetUserDataRemoteApp|user-SendMessageSipcApp","data":[0.0,0,0,0.0,""]}}]}},{"report":{"category":"rpc-server","time":"2013-04-17 10:04:39","columns":[{"col":{"name":"tx(/sec.)","type":"DOUBLE"}},{"col":{"name":"tx(concurrent.)","type":"LONG"}},{"col":{"name":"tx(failed.)","type":"LONG"}},{"col":{"name":"tx(cost ms.)","type":"DOUBLE"}},{"col":{"name":"tx(error)","type":"TEXT"}}],"rows":[{"row":{"instance":"HAWorkerControlleeService.Start|null","data":[0.0,0,0,0.0,""]}},{"row":{"instance":"ISipcOverRpcInterface.requestReceived|SAP","data":[0.0,38674,0,0.0,""]}},{"row":{"instance":"FAE.user-MSProcessUserQuotaRemoteApp|user-SendMessageSipcApp","data":[0.0,0,0,0.0,""]}},{"row":{"instance":"FAE.user-MSSaveOfflineMessageRemoteApp|user-SendMessageSipcApp","data":[0.0,0,0,0.0,""]}},{"row":{"instance":"FAE.user-ValidateMachineCodeRemoteApp|user-SendMessageSipcApp","data":[0.0,0,0,0.0,""]}},{"row":{"instance":"FAE.user-SetUserSmsCountRemoteApp|user-SendMessageSipcApp","data":[0.0,0,0,0.0,""]}},{"row":{"instance":"FAE.user-MSGetUserDataRemoteApp|user-SendMessageSipcApp","data":[0.0,0,0,0.0,""]}}]}},{"report":{"category":"rpc-server","time":"2013-04-17 10:04:40","columns":[{"col":{"name":"tx(/sec.)","type":"DOUBLE"}},{"col":{"name":"tx(concurrent.)","type":"LONG"}},{"col":{"name":"tx(failed.)","type":"LONG"}},{"col":{"name":"tx(cost ms.)","type":"DOUBLE"}},{"col":{"name":"tx(error)","type":"TEXT"}}],"rows":[{"row":{"instance":"HAWorkerControlleeService.Start|null","data":[0.0,0,0,0.0,""]}},{"row":{"instance":"ISipcOverRpcInterface.requestReceived|SAP","data":[0.0,38674,0,0.0,""]}},{"row":{"instance":"FAE.user-MSProcessUserQuotaRemoteApp|user-SendMessageSipcApp","data":[0.0,0,0,0.0,""]}},{"row":{"instance":"FAE.user-MSSaveOfflineMessageRemoteApp|user-SendMessageSipcApp","data":[0.0,0,0,0.0,""]}},{"row":{"instance":"FAE.user-ValidateMachineCodeRemoteApp|user-SendMessageSipcApp","data":[0.0,0,0,0.0,""]}},{"row":{"instance":"FAE.user-SetUserSmsCountRemoteApp|user-SendMessageSipcApp","data":[0.0,0,0,0.0,""]}},{"row":{"instance":"FAE.user-MSGetUserDataRemoteApp|user-SendMessageSipcApp","data":[0.0,0,0,0.0,""]}}]}},{"report":{"category":"rpc-server","time":"2013-04-17 10:04:41","columns":[{"col":{"name":"tx(/sec.)","type":"DOUBLE"}},{"col":{"name":"tx(concurrent.)","type":"LONG"}},{"col":{"name":"tx(failed.)","type":"LONG"}},{"col":{"name":"tx(cost ms.)","type":"DOUBLE"}},{"col":{"name":"tx(error)","type":"TEXT"}}],"rows":[{"row":{"instance":"HAWorkerControlleeService.Start|null","data":[0.0,0,0,0.0,""]}},{"row":{"instance":"ISipcOverRpcInterface.requestReceived|SAP","data":[0.0,38674,0,0.0,""]}},{"row":{"instance":"FAE.user-MSProcessUserQuotaRemoteApp|user-SendMessageSipcApp","data":[0.0,0,0,0.0,""]}},{"row":{"instance":"FAE.user-MSSaveOfflineMessageRemoteApp|user-SendMessageSipcApp","data":[0.0,0,0,0.0,""]}},{"row":{"instance":"FAE.user-ValidateMachineCodeRemoteApp|user-SendMessageSipcApp","data":[0.0,0,0,0.0,""]}},{"row":{"instance":"FAE.user-SetUserSmsCountRemoteApp|user-SendMessageSipcApp","data":[0.0,0,0,0.0,""]}},{"row":{"instance":"FAE.user-MSGetUserDataRemoteApp|user-SendMessageSipcApp","data":[0.0,0,0,0.0,""]}}]}},{"report":{"category":"rpc-server","time":"2013-04-17 10:04:42","columns":[{"col":{"name":"tx(/sec.)","type":"DOUBLE"}},{"col":{"name":"tx(concurrent.)","type":"LONG"}},{"col":{"name":"tx(failed.)","type":"LONG"}},{"col":{"name":"tx(cost ms.)","type":"DOUBLE"}},{"col":{"name":"tx(error)","type":"TEXT"}}],"rows":[{"row":{"instance":"HAWorkerControlleeService.Start|null","data":[0.0,0,0,0.0,""]}},{"row":{"instance":"ISipcOverRpcInterface.requestReceived|SAP","data":[0.0,38674,0,0.0,""]}},{"row":{"instance":"FAE.user-MSProcessUserQuotaRemoteApp|user-SendMessageSipcApp","data":[0.0,0,0,0.0,""]}},{"row":{"instance":"FAE.user-MSSaveOfflineMessageRemoteApp|user-SendMessageSipcApp","data":[0.0,0,0,0.0,""]}},{"row":{"instance":"FAE.user-ValidateMachineCodeRemoteApp|user-SendMessageSipcApp","data":[0.0,0,0,0.0,""]}},{"row":{"instance":"FAE.user-SetUserSmsCountRemoteApp|user-SendMessageSipcApp","data":[0.0,0,0,0.0,""]}},{"row":{"instance":"FAE.user-MSGetUserDataRemoteApp|user-SendMessageSipcApp","data":[0.0,0,0,0.0,""]}}]}},{"report":{"category":"rpc-server","time":"2013-04-17 10:04:43","columns":[{"col":{"name":"tx(/sec.)","type":"DOUBLE"}},{"col":{"name":"tx(concurrent.)","type":"LONG"}},{"col":{"name":"tx(failed.)","type":"LONG"}},{"col":{"name":"tx(cost ms.)","type":"DOUBLE"}},{"col":{"name":"tx(error)","type":"TEXT"}}],"rows":[{"row":{"instance":"HAWorkerControlleeService.Start|null","data":[0.0,0,0,0.0,""]}},{"row":{"instance":"ISipcOverRpcInterface.requestReceived|SAP","data":[0.0,38674,0,0.0,""]}},{"row":{"instance":"FAE.user-MSProcessUserQuotaRemoteApp|user-SendMessageSipcApp","data":[0.0,0,0,0.0,""]}},{"row":{"instance":"FAE.user-MSSaveOfflineMessageRemoteApp|user-SendMessageSipcApp","data":[0.0,0,0,0.0,""]}},{"row":{"instance":"FAE.user-ValidateMachineCodeRemoteApp|user-SendMessageSipcApp","data":[0.0,0,0,0.0,""]}},{"row":{"instance":"FAE.user-SetUserSmsCountRemoteApp|user-SendMessageSipcApp","data":[0.0,0,0,0.0,""]}},{"row":{"instance":"FAE.user-MSGetUserDataRemoteApp|user-SendMessageSipcApp","data":[0.0,0,0,0.0,""]}}]}},{"report":{"category":"rpc-server","time":"2013-04-17 10:04:44","columns":[{"col":{"name":"tx(/sec.)","type":"DOUBLE"}},{"col":{"name":"tx(concurrent.)","type":"LONG"}},{"col":{"name":"tx(failed.)","type":"LONG"}},{"col":{"name":"tx(cost ms.)","type":"DOUBLE"}},{"col":{"name":"tx(error)","type":"TEXT"}}],"rows":[{"row":{"instance":"HAWorkerControlleeService.Start|null","data":[0.0,0,0,0.0,""]}},{"row":{"instance":"ISipcOverRpcInterface.requestReceived|SAP","data":[0.0,38674,0,0.0,""]}},{"row":{"instance":"FAE.user-MSProcessUserQuotaRemoteApp|user-SendMessageSipcApp","data":[0.0,0,0,0.0,""]}},{"row":{"instance":"FAE.user-MSSaveOfflineMessageRemoteApp|user-SendMessageSipcApp","data":[0.0,0,0,0.0,""]}},{"row":{"instance":"FAE.user-ValidateMachineCodeRemoteApp|user-SendMessageSipcApp","data":[0.0,0,0,0.0,""]}},{"row":{"instance":"FAE.user-SetUserSmsCountRemoteApp|user-SendMessageSipcApp","data":[0.0,0,0,0.0,""]}},{"row":{"instance":"FAE.user-MSGetUserDataRemoteApp|user-SendMessageSipcApp","data":[0.0,0,0,0.0,""]}}]}}]}'
rsp = rsp.replace('reports:', '"reports":')
result = json.loads(rsp)
for c in result['reports']:
print c['report']['category'], ' ', c['report']['time'], ' ', c['report']['columns'][0]['col']['name'], ' ', c['report']['rows'][2]['row']['data']
print '*' * 20
print result['reports'][0]
from entity.sitescope import Counter
columns = []
for c in result['reports'][0]['report']['columns']:
columns.append(c['col']['name'])
print c['col']['name']
print '*' * 20
from entity.sitescope import Monitor
for c in result['reports'][-1]['report']['rows']:
#print c['row']['instance'],c['row']['data']
monitor = Monitor(c['row']['instance'], 'test', 'FaeObserver', '127.0.0.1')
i = 0
for col in columns:
#print col,c['row']['data'][i]
counter = Counter(col, 'test', c['row']['data'][i])
monitor.counters.append(counter)
#print counter.str()
i += 1
print monitor.str()