def get(self):
unique_id = self.get_argument("job").encode('utf-8')
param={"id":unique_id}
mysql=mysqlLib()
datalist=mysql.query_task(param)
mysql.close()
#print datalist
datalist_info={}
datalist_info['submit'] = datalist[0][1]
datalist_info['url'] = datalist[0][2]
datalist_info['jobname'] = datalist[0][3]
datalist_info['status'] = datalist[0][5]
datalist_info['build_number'] = datalist[0][6]
#print datalist_info
cmdstr = "curl " + datalist_info['url'] + "job/" + datalist_info['jobname'] + "/" + str(datalist_info['build_number']) + "/logText/progressiveText?start=0"
#print cmdstr
if (datalist_info['status'] == 3 or datalist_info['status'] == 4 or datalist_info['status'] == 5):
status,output=cmd_execute(cmdstr)
jobname = datalist_info['jobname']
if (jobname == "pbdownload" or jobname == "addnode" or jobname == "delnode") :
output = "INFO, task execute success."
else:
tmp = output.split("]'\r\n")[1].split("Notifying")[0]
output = tmp
else:
output = "INFO, job is running now"
datalist_info['output'] = output
self.write(json.dumps(datalist_info))
def post(self):
post_param=urlparse.parse_qs(self.request.body,True)
limit=int(post_param['limit'][0])
offset=int(post_param['offset'][0])
belong=post_param['belong'][0]
param={"limit":limit,"offset":offset}
mysql=mysqlLib()
tasklist=mysql.query_task(param)
param={}
task_totalcnt=mysql.query_task_totalcnt(param)
mysql.close()
serverlist_info={}
serverlist_info['task_totalcnt']=task_totalcnt
ret_dict=[]
for index,item in enumerate(tasklist):
sub_ret_dict={}
sub_ret_dict['id']=str(int(item[0]))
sub_ret_dict['submit']=item[1]
sub_ret_dict['jenkinsurl']=item[2]
sub_ret_dict['jobname']=item[3]
sub_ret_dict['build_params']=item[4]
sub_ret_dict['status']=item[5]
sub_ret_dict['build_number']=item[6]
ret_dict.append(sub_ret_dict)
serverlist_info['ret_dict']=ret_dict
self.write(json.dumps(serverlist_info))
def post(self):
post_param=urlparse.parse_qs(self.request.body,True)
limit=int(post_param['limit'][0])
offset=int(post_param['offset'][0])
#belong=post_param['belong'][0]
param={"limit":limit,"offset":offset}
mysql=mysqlLib()
serverlist=mysql.query_server(param)
param={}
server_totalcnt=mysql.query_server_totalcnt(param)
mysql.close()
serverlist_info={}
serverlist_info['server_totalcnt']=server_totalcnt
ret_dict=[]
for index,item in enumerate(serverlist):
sub_ret_dict={}
sub_ret_dict['id']=str(int(item[0]))
sub_ret_dict['name']=item[1]
sub_ret_dict['exenum']=str(int(item[2]))
sub_ret_dict['workspace']=item[3]
sub_ret_dict['ip']=item[4]
sub_ret_dict['username']=item[5]
sub_ret_dict['password']=item[6]
sub_ret_dict['belong']=item[7]
sub_ret_dict['descpt']=item[8]
ret_dict.append(sub_ret_dict)
serverlist_info['ret_dict']=ret_dict
self.write(json.dumps(serverlist_info))
def monitor_alarm_proc(self):
logging.info("task task_id:" + str(self.last_id) + " monitor_alarm_proc start")
monitor_result = {}
for item in self.NewIndexInterface_conf:
desc = item["desc"]
interface = item["interface"]
ip_port = item["ip_port"]
url = item["url"]
method = item["method"]
params = item["params"]
expect = item["expect"]
compare = item["compare"]
flag = item["flag"]
data = None
monitor_result[interface] = {"status": 0, "msg": "ok"}
if flag == 1:
# print params
h = httpclientLib(ip_port, url, method=method, params=params)
status, data = h.request()
if status != 200:
monitor_result[interface] = {"status": -1, "msg": url + " access exception"}
continue
try:
print json.dumps(data, ensure_ascii=False)
print data
print expect
logging.info("****************" + desc + "****************")
logging.info(json.dumps(data, ensure_ascii=False))
logging.info(json.dumps(expect, ensure_ascii=False))
tmp = json.dumps(data, ensure_ascii=False)
# print tmp
values = find_values("url", json.loads(tmp))
error_item_list = []
for item in values:
# print item
h = httpclientLib(ip_port, url, method=method, params=params)
status, data = h.request()
if status != 200:
error_item_list.append(item)
if len(error_item_list) != 0:
monitor_result[interface] = {"status": -1, "msg": error_item_list}
except Exception as e:
monitor_result[interface] = {"status": -1, "msg": "response exception"}
# print monitor_result
logging.info(json.dumps(monitor_result))
total = 0
normal = 0
exception = 0
for item in monitor_result:
total += 1
if monitor_result[item]["status"] == 0:
normal += 1
else:
exception += 1
mysql = mysqlLib()
uptime = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
param = (total, normal, exception, json.dumps(monitor_result), uptime, "NewIndex")
mysql.update_tbl_seo(param)
mysql.close()
logging.info("task task_id:" + str(self.last_id) + " monitor_alarm_proc complete")
def main(self):
#1.获取账户信息
mysql = AccountInfo()
self.accountInfo =mysql.get_tbl_accountinfo()
mysql.save_tbl_accountinfo()
mysql.save_tbl_cid()
#2. 依次检查每一个账户
index = 0
while(index<len(self.accountInfo)):
keyword = self.GetReport(self.check_date, self.accountInfo[index])
each_dict = self.CheckKeyWordByAccount(self.accountInfo[index],keyword)
self.addCollect(each_dict)
index+=1
print index
#3. 写入数据库
mysql = mysqlLib()
ret_list = list()
for key in self.collect_dict:
param=[self.check_date,key.encode('UTF-8'),self.collect_dict[key][0],self.collect_dict[key][1],self.collect_dict[key][2]]
mysql.add_wordclass_data(param)
ret_list.append(param)
mysql.close()
print u'end'
return ret_list
def run(self):
print '*********************************'
print self.taskid
self.LoadAccountCid('accountinfoAllCid.txt')
self.LoadDeadDeal('deadtinyurl.txt')
#得到账户的关键词情况，日报
self.GetReport(self.dateStr, self.accountInfo)
#对每一个关键字检查对应的url,写入self.urlList
self.CheckKeyWord(self.accountInfo)
#统计数据存入本地数据库
mysql = mysqlLib()
param=[self.taskid,self.dateStr,self.accountInfo[0],self.totalurlNum,self.errorurlNum,self.deadDealNum,self.badDeadDealNum,self.openurlErrorNum,self.noListResultNum,self.errorCidNum,self.formatErrorNum,self.nullUrlNum]
mysql.add_url_data(param)
#详细情况存入本地数据库
#1.得到tbl_url_data的id
accountName=self.accountInfo[0]
param=[accountName,self.dateStr]
id=mysql.get_url_data_id(param)
#2.插入tbl_errorurl
for listInfo in self.resList:
if(len(listInfo) <= 0):
continue
param=[id,self.accountInfo[0],listInfo[2],listInfo[3],listInfo[4],listInfo[5],listInfo[7],listInfo[6],self.dateStr]
mysql.add_urlerror_data(param)
print '***'
mysql.close()
def post(self):
param={}
mysql=mysqlLib()
downloadstat_week=mysql.query_downloadstat(1,param)
downloadstat_all=mysql.query_downloadstat(0,param)
mysql.close()
datalist_info={}
ret_dict=[]
tmp = {}
for index,item in enumerate(downloadstat_week):
tmp[int(item[0])] = int(item[1])
for i in xrange(1,5):
if(i not in tmp):
ret_dict.append(0)
else:
ret_dict.append(tmp[i])
datalist_info['week']=ret_dict
ret_dict=[]
for index,item in enumerate(downloadstat_all):
ret_dict.append(int(item[1]))
datalist_info['all']=ret_dict
self.write(json.dumps(datalist_info))
def update_task_status(self, task_id, status):
try:
mysql = mysqlLib()
param=(status, task_id)
mysql.update_tbl_monitortask_status(param)
mysql.close()
logging.info("task task_id:" + str(task_id) + " database status modified to :" + str(status))
except Exception as e:
logging.error(str(e))
def add_monitor_data(self, task_id, status, info):
try:
mysql = mysqlLib()
param=(task_id, status, json.dumps(info))
n,last_id = mysql.add_tbl_monitordata(param)
mysql.close()
logging.info("add monitor data last_id:" + str(last_id) + " to database success")
except Exception as e:
logging.error(str(e))
def update_task_endtime(self, task_id, endtime):
try:
mysql = mysqlLib()
param=(endtime, task_id)
mysql.update_tbl_monitortask_endtime(param)
mysql.close()
logging.info("task task_id:" + str(task_id) + " database endtime update success")
except Exception as e:
logging.error(str(e))
def add_alarm(self, alarmConf_id, monitorTask_id, monitorObj_id, monitorMetrics_id, description,status, datetime):
try:
mysql = mysqlLib()
param=(alarmConf_id, monitorTask_id, monitorObj_id, monitorMetrics_id, description, status, datetime)
n,last_id = mysql.add_alarmobj(param)
mysql.close()
logging.info("alarm object last_id:" + str(last_id) + " added success")
except Exception as e:
logging.error(str(e))
def post(self):
post_param=urlparse.parse_qs(self.request.body,True)
server_id=post_param['id'][0]
param=(server_id,)
mysql=mysqlLib()
n=mysql.del_server(param)
mysql.close()
ret_dict={"errcode":n}
self.write(json.dumps(ret_dict))
def post(self):
#post_param=urlparse.parse_qs(self.request.body,True)
#print self.request.body
jsonobj = json.loads(self.request.body)
unique_id = jsonobj['build']['parameters']['UNIQUE_ID']
number = jsonobj['build']['number']
phase = jsonobj['build']['phase']
if (phase == "STARTED"):
mysql = mysqlLib()
param=(2, unique_id)
mysql.update_task_status(param)
print(u"#RUNNING, job running, 2")
param=(number, unique_id)
mysql.update_task_build_number(param)
mysql.close()
elif (phase == "FINALIZED" or phase == "COMPLETED"):
status = jsonobj['build']['status']
if (status == "SUCCESS"):
mysql = mysqlLib()
param=(3, unique_id)
mysql.update_task_status(param)
print(u"#SUCCESS, job complete, 3")
mysql.close()
elif (status == "ABORTED"):
mysql = mysqlLib()
param=(5, unique_id)
mysql.update_task_status(param)
print(u"#ABORTED, job aborted, 3")
mysql.close()
else:
mysql = mysqlLib()
param=(4, unique_id)
mysql.update_task_status(param)
print(u"#ERROR, job failed, 4")
mysql.close()
else:
mysql = mysqlLib()
param=(4, unique_id)
mysql.update_task_status(param)
print(u"#ERROR, job failed, 4")
mysql.close()
def post(self):
post_param=urlparse.parse_qs(self.request.body,True)
testdata_id=post_param['id'][0]
param=(testdata_id,)
mysql=mysqlLib()
n=mysql.del_testdata(param)
m=mysql.del_datafile(param)
mysql.close()
ret_dict={"errcoden":n,"errcodem":m}
self.write(json.dumps(ret_dict))
def monitor_alarm_proc(self):
logging.info("task task_id:" + str(self.last_id) + " monitor_alarm_proc start")
monitor_result = {}
for item in self.TSMInterface_conf:
desc = item['desc']
interface = item['interface']
ip_port = item['ip_port']
url = item['url']
method = item['method']
params = item['params']
expect = item['expect']
compare = item['compare']
flag = item['flag']
data = None
if(flag == 1):
#print params
h = httpclientLib(ip_port, url, method=method, params=params)
status,data = h.request()
'''print json.dumps(data, ensure_ascii=False)
print data
print expect'''
try:
logging.info("****************" + desc + "****************")
logging.info(json.dumps(data, ensure_ascii=False))
logging.info(json.dumps(expect, ensure_ascii=False))
if(compare == 1):
if(data['data'] == expect['data']):
monitor_result[interface] = {"status":0,"msg":"ok"}
else:
monitor_result[interface] = {"status":-1,"msg":"expect:\n"+json.dumps(expect, ensure_ascii=False)+"\ndata:\n"+json.dumps(data, ensure_ascii=False)}
else:
if(data['errno'] == 0):
monitor_result[interface] = {"status":0,"msg":"ok"}
else:
monitor_result[interface] = {"status":-1,"msg":"errno not 0:\n"+json.dumps(data, ensure_ascii=False)}
except Exception as e:
monitor_result[interface] = {"status":-1,"msg":"response data error"}
#print monitor_result
logging.info(json.dumps(monitor_result))
total = 0
normal = 0
exception = 0
for item in monitor_result:
total += 1
if(monitor_result[item]['status'] == 0):
normal += 1
else:
exception += 1
mysql = mysqlLib()
uptime = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
param = (total, normal, exception, json.dumps(monitor_result), uptime, "TSM")
mysql.update_tbl_seo(param)
mysql.close()
logging.info("task task_id:" + str(self.last_id) + " monitor_alarm_proc complete")
def get_monitorMetrics(self, monitorMetrics_id):
try:
mysql = mysqlLib()
param={"id":monitorMetrics_id}
retList = mysql.query_monitormetrics(param)
mysql.close()
ret_monitorMetrics_id = int(retList[0][0])
ret_monitorObj_id = int(retList[0][1])
ret_monitorMetrics_name = retList[0][2]
logging.info("get_monitorMetrics : " + str(monitorMetrics_id) + ", success")
return ret_monitorMetrics_id,ret_monitorObj_id,ret_monitorMetrics_name
except Exception as e:
logging.error(str(e))
def post(self):
post_param=urlparse.parse_qs(self.request.body,True)
srv_id=int(post_param['id'][0])
param={"id":srv_id}
mysql=mysqlLib()
serverlist=mysql.query_server(param)
mysql.close()
ret_dict={}
workspace=serverlist[0][3]
ret_dict['workspace']=workspace
self.write(json.dumps(ret_dict))
def post(self):
post_param=urlparse.parse_qs(self.request.body,True)
datatype=post_param['type'][0]
name=post_param['name'][0]
belong=post_param['belong'][0]
filenum=post_param['filenum'][0]
descpt=post_param['desc'][0]
param=(datatype,name,belong,filenum,descpt)
mysql=mysqlLib()
n,last_id=mysql.add_testdata(param)
mysql.close()
ret_dict={"errcode":n,"last_id":last_id}
self.write(json.dumps(ret_dict))
def add_job(self, monitorObj_id, monitorMetrics_id, monitorMetrics_name, taskrunner):
try:
mysql = mysqlLib()
param=(monitorObj_id, monitorMetrics_id, 0)
n,last_id = mysql.add_task(param)
mysql.close()
logging.info("add_job last_id:" + str(last_id) + ",monitorObj_id:" + str(monitorObj_id) + ",monitorMetrics_id:" + str(monitorMetrics_id) + " to database success")
jobname = self.jobname
jobparameters = {"last_id":last_id,"monitorObj_id":monitorObj_id,"monitorMetrics_id":monitorMetrics_id,"monitorMetrics_name":monitorMetrics_name,"taskrunner":taskrunner}
gearclient = gearClient()
gearclient.job_submit(str(jobname), jobparameters, wait_until_complete = False, background = True)
logging.info("gear job last_id:" + str(last_id) + ",monitorObj_id:" + str(monitorObj_id) + ",monitorMetrics_id:" + str(monitorMetrics_id) + " submited success")
self.update_task_status(last_id, 1)
except Exception as e:
logging.error(str(e))
def query_alarm_conf(self, monitorObj_id, monitorMetrics_id):
try:
mysql = mysqlLib()
param={"monitorObj_id":monitorObj_id,"monitorMetrics_id":monitorMetrics_id}
retList = mysql.query_tbl_alarmconf(param)
mysql.close()
alarm_conf_id = int(retList[0][0])
monitorObj_id = int(retList[0][1])
monitorMetrics_id = int(retList[0][2])
name = str(retList[0][3])
description = str(retList[0][4])
status = int(retList[0][5])
conf = json.loads(retList[0][6])
logging.info("query alarm conf monitorObj_id:" + str(monitorObj_id) + ",monitorMetrics_id:" + str(monitorMetrics_id) +" success")
return alarm_conf_id,monitorObj_id,monitorMetrics_id,name,description,status,conf
except Exception as e:
logging.error(str(e))
def post(self):
post_param=urlparse.parse_qs(self.request.body,True)
name=post_param['name'][0]
exenum=post_param['exenum'][0]
ip=post_param['ip'][0]
username=post_param['username'][0]
password=post_param['password'][0]
workspace=post_param['workspace'][0]
belong=post_param['belong'][0]
descpt=post_param['descpt'][0]
param=(name,exenum,workspace,ip,username,password,belong,descpt)
mysql=mysqlLib()
n,last_id=mysql.add_server(param)
mysql.close()
ret_dict={"errcode":n}
self.write(json.dumps(ret_dict))
def post(self):
post_param=urlparse.parse_qs(self.request.body,True)
json_postdata={}
try:
for i in post_param[' name']:
json_postdata[i.split("\r\n")[0].strip("\"")]=i.split("\r\n")[2]
except Exception:
pass
#print json_postdata
mysql=mysqlLib()
param=(json_postdata['last_id'],self.request.files['Filedata'][0]['filename'],self.request.files['Filedata'][0]['body'])
n,last_id=mysql.add_datastr(param)
mysql.close()
ret_dict={"errcode":n}
self.write(json.dumps(ret_dict))
'''file_dict_list = self.request.files['Filedata']
def post(self):
try:
post_param=urlparse.parse_qs(self.request.body,True)
limit=int(post_param['limit'][0])
offset=int(post_param['offset'][0])
param={"limit":limit,"offset":offset}
mysql=mysqlLib()
alarm_list=mysql.query_tbl_alarmobj(param)
param={}
alarm_totalcnt=mysql.query_alarm_totalcnt(param)
mysql.close()
alarm_list_info={}
alarm_list_info['alarm_totalcnt']=alarm_totalcnt
ret_dict=[]
for index,item in enumerate(alarm_list):
sub_ret_dict={}
sub_ret_dict['id']=str(int(item[0]))
sub_ret_dict['alarmConf_id']=str(int(item[1]))
sub_ret_dict['monitorTask_id']=str(int(item[2]))
sub_ret_dict['monitorObj_id']=str(int(item[3]))
sub_ret_dict['monitorMetrics_id']=str(int(item[4]))
sub_ret_dict['description']=item[5]
sub_ret_dict['status']=str(int(item[6]))
sub_ret_dict['datetime']=str(item[7])
sub_ret_dict['cause']=item[8]
sub_ret_dict['handleBy']=item[9]
sub_ret_dict['info']=item[10]
ret_dict.append(sub_ret_dict)
alarm_list_info['ret_dict']=ret_dict
self.write(json.dumps(alarm_list_info))
except Exception as e:
logging.error(str(e))
logging.info(traceback.print_exc())
def get(self):
userid=self.get_cookie_user()
unique_id = self.get_argument("job").encode('utf-8')
param={"id":unique_id}
mysql=mysqlLib()
datalist=mysql.query_task(param)
mysql.close()
#print datalist
datalist_info={}
datalist_info['submit'] = datalist[0][1]
datalist_info['url'] = datalist[0][2]
datalist_info['jobname'] = datalist[0][3]
datalist_info['status'] = datalist[0][5]
datalist_info['build_number'] = datalist[0][6]
#print datalist_info
cmdstr = "curl " + datalist_info['url'] + "job/" + datalist_info['jobname'] + "/" + str(datalist_info['build_number']) + "/logText/progressiveText?start=0"
#print cmdstr
if (datalist_info['status'] == 3 or datalist_info['status'] == 4 or datalist_info['status'] == 5):
status,output=cmd_execute(cmdstr)
jobname = datalist_info['jobname']
if (jobname == "pbdownload" or jobname == "addnode" or jobname == "delnode") :
output = "INFO, task execute success."
else:
tmp = output.split("]'\r\n")[1].split("Notifying")[0]
output = tmp
output = output.replace("\n","<br/>")
else:
output = "INFO, job is running now"
datalist_info['output'] = output
#self.write(json.dumps(datalist_info))
if(userid):
self.render("./taskmanagement/taskdetail.html",usrname=userid,datalist_info=datalist_info)
else:
redirect_url=CAS_SETTINGS[ 'cas_server' ] + '/login?service=' + CAS_SETTINGS[ 'service_url' ]
self.redirect(redirect_url)
def post(self):
post_param=urlparse.parse_qs(self.request.body,True)
limit=int(post_param['limit'][0])
offset=int(post_param['offset'][0])
belong=post_param['belong'][0]
datatype=post_param['datatype'][0]
if(datatype == -1):
param={"belong":belong,"limit":limit,"offset":offset}
else:
param={"belong":belong,"type":datatype,"limit":limit,"offset":offset}
mysql=mysqlLib()
datalist=mysql.query_testdata(param)
if(datatype == -1):
param={"belong":belong}
else:
param={"belong":belong,"type":datatype}
data_totalcnt=mysql.query_testdata_totalcnt(param)
mysql.close()
datalist_info={}
datalist_info['data_totalcnt']=data_totalcnt
ret_dict=[]
for index,item in enumerate(datalist):
sub_ret_dict={}
sub_ret_dict['id']=str(int(item[0]))
sub_ret_dict['name']=item[1]
sub_ret_dict['type']=item[2]
sub_ret_dict['belong']=item[3]
sub_ret_dict['filenum']=item[4]
sub_ret_dict['descpt']=item[5]
ret_dict.append(sub_ret_dict)
datalist_info['ret_dict']=ret_dict
self.write(json.dumps(datalist_info))
def monitor_alarm_proc(self):
logging.info("task task_id:" + str(self.last_id) + " monitor_alarm_proc start")
#监控数据上报
cutime=datetime.datetime.now().strftime("%y-%m-%d %H:%M:%S")
#1. 查询最近四周的数据，如果没有，就调用get_wordclass脚本
#function区分功能是 展示 impression,点击 click,消费 cost
now_time = datetime.datetime.now()
start_time = now_time + datetime.timedelta(days=-1)
index=0
while(index<=3):
tmp = index*7
day = start_time + datetime.timedelta(days=-tmp)#delay to index week ago
dateStr = day.strftime('%Y-%m-%d')
param = [dateStr]
mysql = mysqlLib()
ret = mysql.query_wordclass_data(param,function)
mysql.close()
if not ret:
wc = WordCalss(dateStr)
tmplist=wc.main()
mysql = mysqlLib()
ret = mysql.query_wordclass_data(param,function)
mysql.close()
#2 . 合并结果
self.makeFinalResult(ret)
index+=1
logging.info("task task_id:" + str(self.last_id) + " insert into db data_table end")
#3. 检查报警数据并写入数据库
#查询报警配置
alarm_conf_id,monitorObj_id,monitorMetrics_id,name,description,status,conf = self.query_alarm_conf(self.monitorObj_id, self.monitorMetrics_id)
#对比报警条件
self.CreateList()
print '2'
self.ComputeDiff(conf['threshold'])
print '3'
warnList=list()
for item in self.resList:
cutime=datetime.datetime.now().strftime("%y-%m-%d %H:%M:%S")
if(1 == item['flag']):
dsc = description + " : 词类:" + str(item['wordClass']) + ", 报警:降幅超过阈值 " +str(item[function])
print dsc
self.add_alarm(alarm_conf_id, self.last_id, monitorObj_id,monitorMetrics_id, dsc , 0, cutime)
warnList.append(dsc)
'''elif(2 == item['flag']):
dsc = description + " : 词类:" + str(item['wordClass']) + ", 报警:涨幅超过阈值 " +str(item[function])
print dsc
self.add_alarm(alarm_conf_id, self.last_id, monitorObj_id,monitorMetrics_id, dsc , 0, cutime)
warnList.append(dsc)'''
logging.info("task task_id:" + str(self.last_id) + " monitor_alarm_proc end")
#step 6: 发送报警邮件，短信
for item in warnList:
self.mail_send(self.subject+" "+description, item, self.from_mail_addr, self.to_mail_addr, self.mail_server)
logging.info("task task_id:" + str(self.last_id) + " mail send end")
#报警短信
for item in warnList:
self.msgSend(self.phone_list, item , mode="qapi")
logging.info("task task_id:" + str(self.last_id) + " msg send end")
def monitor_alarm_proc(self):
mysql = AccountInfo()
self.accountInfo =mysql.get_tbl_accountinfo()
mysql.save_tbl_accountinfo()
mysql.save_tbl_cid()
mysql.close()
logging.info("task task_id:" + str(self.last_id) + " monitor_alarm_proc start")
#监控数据上报
now_time = datetime.datetime.now()
day = now_time + datetime.timedelta(days=-1)
dateStr = day.strftime('%Y-%m-%d')
index=0
th_list = []
while(index<len(self.accountInfo)):
myMonitor = MyUrlMonitor(dateStr,self.accountInfo[index],self.last_id)
th_list.append(myMonitor)
myMonitor.start()
index+=1
for item in th_list:
item.join()
logging.info("task task_id:" + str(self.last_id) + " all thread run end")
#检查报警
mysql = mysqlLib()
alarm_conf_id,monitorObj_id,monitorMetrics_id,name,description,status,conf = self.query_alarm_conf(self.monitorObj_id, self.monitorMetrics_id)
#对比报警条件
print conf['threshold']
param=[dateStr,conf['threshold']]
warnList = mysql.query_urlalarm_data(param)
alarmList = list()
if(len(warnList)==0):
mysql.close()
return 'close'
for item in warnList:
accountName=item[2]
totalurlNum=item[3]
errorurlNum=item[4]
deadDealNum=item[5]
badDeadDealNum=item[6]
openurlErrorNum=item[7]
noListResultNum=item[8]
errorCidNum=item[9]
formatErrorNum=item[10]
nullUrlNum=item[11]
dsc = description + " 帐户名：" + str(accountName) + ",Url总数:"+str(totalurlNum)+",Error总数:"+str(errorurlNum)+",过期团单:"+str(deadDealNum)+",非合理过期团单:"+str(badDeadDealNum)+",访问错误:"+str(openurlErrorNum)+",无搜索结果:"+str(noListResultNum)+",非天兔Cid:"+str(errorCidNum)+",URL格式错误:"+str(formatErrorNum)+",空URL:"+str(nullUrlNum)
alarmList.append(dsc)
cutime=datetime.datetime.now().strftime("%y-%m-%d %H:%M:%S")
self.add_alarm(alarm_conf_id, self.last_id, monitorObj_id,monitorMetrics_id, dsc , 0, cutime)
logging.info("task task_id:" + str(self.last_id) + " insert into db alarm_table end")
mysql.close()
#step 6: 发送报警邮件，短信
for item in alarmList:
self.mail_send(self.subject+" "+description, str(item), self.from_mail_addr, self.to_mail_addr, self.mail_server)
logging.info("task task_id:" + str(self.last_id) + " mail send end")
#报警短信
for item in warnList:
self.msgSend(self.phone_list, str(item) , mode="qapi")
logging.info("task task_id:" + str(self.last_id) + " msg send end")
def run(self):
#获取logid请求参数
self.ftpPath = "ftp://" + self.hostname + "/home/tuan/odp/log/zhixin/zhixin.log." + self.datetime
self.line_list = None
try:
self.line_list = geFtpFile2Memory(self.ftpPath, self.keystr_request)
except Exception as e:
logging.error("", exc_info=1)
return
for line in self.line_list:
tmp_line_list = re.split("[][]", line)
logid = tmp_line_list[5]
req_param = tmp_line_list[22]
req_param_list = req_param.split(",")
try:
city_id = req_param_list[1].split(":")[1]
srcID = req_param_list[2].split(":")[1]
keyword = req_param_list[3].split(":")[1]
original_query = req_param_list[4].split(":")[1]
req_type = req_param_list[5].split(":")[1]
self.log_stat[logid] = {
"city_id":city_id,
"srcID":srcID,
#"keyword":keyword.decode('GBK').encode('UTF-8'),
#"original_query":original_query.decode('GBK').encode('UTF-8'),
#"req_type":req_type,
"as_null":0,
"as_delay":0,
"zhixin_null":0,
"zhixin_delay":0
}
except Exception as e:
logging.error("line : " + line)
logging.error("req_param : " + req_param)
#traceback.print_exc()
logging.error("", exc_info=1)
'''#logid维度获取各项指标
for logid in self.log_stat:
self.line_list = geFtpFile2Memory(self.ftpPath, logid)
for line in self.line_list:
if(self.keystr_as_total in line):
tmp_line_list = re.split("zhixin New as response,total:", line)
as_total = int(tmp_line_list[1])
if(as_total == 0):
self.log_stat[logid]['as_null'] = 1
elif(self.keystr_as_cost in line):
tmp_line_list = re.split("new as cost ", line)
as_delay = int(tmp_line_list[1].split(" ")[0])
self.log_stat[logid]['as_delay'] = as_delay
elif(self.keystr_zhixin_total in line):
tmp_line_list = re.split("zhixin_show", line)
if(tmp_line_list[1] == "[]"):
self.log_stat[logid]['zhixin_null'] = 1
elif(self.keystr_zhixin_cost in line):
tmp_line_list = re.split("Zhixin Query request cost ", line)
zhixin_delay = int(tmp_line_list[1].split(" ")[0])
self.log_stat[logid]['zhixin_delay'] = zhixin_delay'''
#logid维度获取各项指标-as_null指标
self.line_list = geFtpFile2Memory(self.ftpPath, self.keystr_as_total)
for logid in self.log_stat:
for line in self.line_list:
if(logid in line):
tmp_line_list = re.split("zhixin New as response,total:", line)
try:
as_total = int(tmp_line_list[1])
if(as_total == 0):
self.log_stat[logid]['as_null'] = 1
except Exception as e:
logging.error(str(e), exc_info=1)
#logid维度获取各项指标-as_delay指标
self.line_list = geFtpFile2Memory(self.ftpPath, self.keystr_as_cost)
for logid in self.log_stat:
for line in self.line_list:
if(logid in line):
tmp_line_list = re.split("new as cost ", line)
as_delay = int(tmp_line_list[1].split(" ")[0])
self.log_stat[logid]['as_delay'] = as_delay
#logid维度获取各项指标-as_delay指标
self.line_list = geFtpFile2Memory(self.ftpPath, self.keystr_zhixin_total)
for logid in self.log_stat:
for line in self.line_list:
if(logid in line):
tmp_line_list = re.split("zhixin_show", line)
if(tmp_line_list[1] == "[]"):
self.log_stat[logid]['zhixin_null'] = 1
#logid维度获取各项指标-as_delay指标
self.line_list = geFtpFile2Memory(self.ftpPath, self.keystr_zhixin_cost)
for logid in self.log_stat:
for line in self.line_list:
if(logid in line):
tmp_line_list = re.split("Zhixin Query request cost ", line)
zhixin_delay = int(tmp_line_list[1].split(" ")[0])
self.log_stat[logid]['zhixin_delay'] = zhixin_delay
'''print self.log_stat
total_len = 0
for item in self.log_stat:
total_len += 1
print total_len'''
#遍历log_stat, 给出最终hostname,date,cityID,srcID维度的统计数据——初始化
tmp_stat = {}
for item in self.log_stat:
data = self.log_stat[item]
tmp_key = self.hostname + self.datetime + data['city_id'] + data['srcID']
md5_data = hashlib.md5(tmp_key.encode("utf-8")).hexdigest()
tmp_stat[md5_data] = {
"hostname":self.hostname,
"datetime":self.datetime,
"city_id":data['city_id'],
"srcID":data['srcID'],
"flow":0,
"as_null":0,
"as_delay":0,
"zhixin_null":0,
"zhixin_delay":0
}
#遍历log_stat, 给出最终hostname,date,cityID,srcID维度的统计数据——统计计算
for item in self.log_stat:
data = self.log_stat[item]
tmp_key = self.hostname + self.datetime + data['city_id'] + data['srcID']
md5_data = hashlib.md5(tmp_key.encode("utf-8")).hexdigest()
tmp_stat[md5_data]['hostname'] = self.hostname
tmp_stat[md5_data]['datetime'] = self.datetime
tmp_stat[md5_data]['city_id'] = data['city_id']
tmp_stat[md5_data]['srcID'] = data['srcID']
tmp_stat[md5_data]['flow'] += 1
tmp_stat[md5_data]['as_null'] += data['as_null']
tmp_stat[md5_data]['as_delay'] += data['as_delay']
tmp_stat[md5_data]['zhixin_null'] += data['zhixin_null']
tmp_stat[md5_data]['zhixin_delay'] += data['zhixin_delay']
#print tmp_stat
#监控数据入库
mysql = mysqlLib()
for item in tmp_stat:
try:
hostname = tmp_stat[item]['hostname']
date = tmp_stat[item]['datetime']
a = time.strptime(date,'%Y%m%d%H')
timestamp = time.mktime(a)
localtime = time.localtime(timestamp)
date = time.strftime('%Y-%m-%d %H:%M:%S',localtime)
city_id = tmp_stat[item]['city_id']
srcID = tmp_stat[item]['srcID']
flow = tmp_stat[item]['flow']
as_null = tmp_stat[item]['as_null']
as_delay = tmp_stat[item]['as_delay']/float(flow)
zhixin_null = tmp_stat[item]['zhixin_null']
zhixin_delay = tmp_stat[item]['zhixin_delay']/float(flow)
param=(self.last_id, hostname, date, city_id, srcID, flow, as_null, as_delay, zhixin_null, zhixin_delay,0)
n,last_id = mysql.add_zhixin_monitordata(param)
except Exception as e:
logging.error(str(e), exc_info=1)
mysql.close()
def run(self):
#获取logid请求参数
self.ftpPath = "ftp://" + self.hostname + "/home/tuan/odp/log/zhixin/zhixin.log." + self.datetime
self.line_list = None
try:
self.line_list = geFtpFile2Memory(self.ftpPath, self.keystr_request)
except Exception as e:
logging.error("", exc_info=1)
return
for line in self.line_list:
tmp_line_list = re.split("[][]", line)
'''logid = tmp_line_list[5]
req_param = json.loads(tmp_line_list[20].split("Aladdin queryInfo: ")[1])'''
try:
logid = tmp_line_list[5]
req_param = json.loads(tmp_line_list[20].split("Aladdin queryInfo: ")[1])
city_id = str(req_param['city_id'])
srcID = str(req_param['SrcID'])
self.log_stat[logid] = {
"city_id":city_id,
"srcID":srcID,
#"keyword":keyword.decode('GBK').encode('UTF-8'),
#"original_query":original_query.decode('GBK').encode('UTF-8'),
#"req_type":req_type,
"as_null":0,
"as_delay":0,
"zhixin_null":0,
"zhixin_delay":0
}
except Exception as e:
logging.error("line : " + line)
logging.error("", exc_info=1)
logging.info(u"#获取logid请求参数")
'''#下载文件
dstFilePath = "zhixin.log." + self.datetime + "---" + self.hostname
downFtpFile(self.ftpPath, dstFilePath)
logging.info(u"#下载文件")
#logid维度获取各项指标-as_null指标
for logid in self.log_stat:
cmdstr = 'LC_ALL=C fgrep ' + logid + ' ' + dstFilePath + ' | fgrep "Aladdin query redis value :"'
status,line=cmd_execute(cmdstr)
if(line):
tmp_line_list = re.split("Aladdin query redis value :", line)
as_total = int(tmp_line_list[1])
if(as_total == 1):
self.log_stat[logid]['as_null'] = 1
logging.info(u"#logid维度获取各项指标-as_null指标")
#logid维度获取各项指标-as_delay指标
for logid in self.log_stat:
cmdstr = 'LC_ALL=C fgrep ' + logid + ' ' + dstFilePath + ' | fgrep "Aladdin nuomi query cost "'
status,line=cmd_execute(cmdstr)
if(line):
tmp_line_list = re.split("Aladdin nuomi query cost ", line)
as_delay = int(tmp_line_list[1].split(" ")[0])
self.log_stat[logid]['as_delay'] = as_delay
else:
self.log_stat[logid]['as_delay'] = 0
logging.info(u"#logid维度获取各项指标-as_delay指标")
#logid维度获取各项指标-as_delay指标
for logid in self.log_stat:
cmdstr = 'LC_ALL=C fgrep ' + logid + ' ' + dstFilePath + ' | fgrep "aladdin_show"'
status,line=cmd_execute(cmdstr)
if(line):
tmp_line_list = re.split("aladdin_show", line)
if(tmp_line_list[1] == "[]"):
self.log_stat[logid]['zhixin_null'] = 1
logging.info(u"##logid维度获取各项指标-as_delay指标")
#logid维度获取各项指标-as_delay指标
for logid in self.log_stat:
cmdstr = 'LC_ALL=C fgrep ' + logid + ' ' + dstFilePath + ' | fgrep "Aladdin Query request cost "'
status,line=cmd_execute(cmdstr)
if(line):
tmp_line_list = re.split("Aladdin Query request cost ", line)
zhixin_delay = int(tmp_line_list[1].split(" ")[0])
self.log_stat[logid]['zhixin_delay'] = zhixin_delay
logging.info(u"##logid维度获取各项指标-as_delay指标")'''
#logid维度获取各项指标-as_null指标
self.line_list = geFtpFile2Memory(self.ftpPath, self.keystr_as_total)
for logid in self.log_stat:
for line in self.line_list:
if(logid in line):
tmp_line_list = re.split("Aladdin query redis value :", line)
as_total = int(tmp_line_list[1])
if(as_total == 1):
self.log_stat[logid]['as_null'] = 1
logging.info(u"#logid维度获取各项指标-as_null指标")
#logid维度获取各项指标-as_delay指标
self.line_list = geFtpFile2Memory(self.ftpPath, self.keystr_as_cost)
for logid in self.log_stat:
for line in self.line_list:
if(logid in line):
tmp_line_list = re.split("Aladdin nuomi query cost ", line)
as_delay = int(tmp_line_list[1].split(" ")[0])
self.log_stat[logid]['as_delay'] = as_delay
else:
self.log_stat[logid]['as_delay'] = 0
logging.info(u"#logid维度获取各项指标-as_delay指标")
#logid维度获取各项指标-as_delay指标
self.line_list = geFtpFile2Memory(self.ftpPath, self.keystr_zhixin_total)
for logid in self.log_stat:
for line in self.line_list:
if(logid in line):
tmp_line_list = re.split("aladdin_show", line)
if(tmp_line_list[1] == "[]"):
self.log_stat[logid]['zhixin_null'] = 1
logging.info(u"##logid维度获取各项指标-as_delay指标")
#logid维度获取各项指标-as_delay指标
self.line_list = geFtpFile2Memory(self.ftpPath, self.keystr_zhixin_cost)
for logid in self.log_stat:
for line in self.line_list:
if(logid in line):
tmp_line_list = re.split("Aladdin Query request cost ", line)
zhixin_delay = int(tmp_line_list[1].split(" ")[0])
self.log_stat[logid]['zhixin_delay'] = zhixin_delay
logging.info(u"##logid维度获取各项指标-as_delay指标")
'''print self.log_stat
total_len = 0
for item in self.log_stat:
total_len += 1
print total_len'''
#遍历log_stat, 给出最终hostname,date,cityID,srcID维度的统计数据——初始化
tmp_stat = {}
for item in self.log_stat:
try:
data = self.log_stat[item]
tmp_key = self.hostname + self.datetime + data['city_id'] + data['srcID']
md5_data = hashlib.md5(tmp_key.encode("utf-8")).hexdigest()
tmp_stat[md5_data] = {
"hostname":self.hostname,
"datetime":self.datetime,
"city_id":data['city_id'],
"srcID":data['srcID'],
"flow":0,
"as_null":0,
"as_delay":0,
"zhixin_null":0,
"zhixin_delay":0
}
except Exception as e:
logging.error(str(e), exc_info=1)
logging.info(u"#遍历log_stat, 给出最终hostname,date,cityID,srcID维度的统计数据——初始化")
#遍历log_stat, 给出最终hostname,date,cityID,srcID维度的统计数据——统计计算
for item in self.log_stat:
try:
data = self.log_stat[item]
tmp_key = self.hostname + self.datetime + data['city_id'] + data['srcID']
md5_data = hashlib.md5(tmp_key.encode("utf-8")).hexdigest()
tmp_stat[md5_data]['hostname'] = self.hostname
tmp_stat[md5_data]['datetime'] = self.datetime
tmp_stat[md5_data]['city_id'] = data['city_id']
tmp_stat[md5_data]['srcID'] = data['srcID']
tmp_stat[md5_data]['flow'] += 1
tmp_stat[md5_data]['as_null'] += data['as_null']
tmp_stat[md5_data]['as_delay'] += data['as_delay']
tmp_stat[md5_data]['zhixin_null'] += data['zhixin_null']
tmp_stat[md5_data]['zhixin_delay'] += data['zhixin_delay']
except Exception as e:
logging.error(str(e), exc_info=1)
logging.info(u"#遍历log_stat, 给出最终hostname,date,cityID,srcID维度的统计数据——统计计算")
#print tmp_stat
#监控数据入库
mysql = mysqlLib()
for item in tmp_stat:
try:
hostname = tmp_stat[item]['hostname']
date = tmp_stat[item]['datetime']
a = time.strptime(date,'%Y%m%d%H')
timestamp = time.mktime(a)
localtime = time.localtime(timestamp)
date = time.strftime('%Y-%m-%d %H:%M:%S',localtime)
city_id = tmp_stat[item]['city_id']
srcID = tmp_stat[item]['srcID']
flow = tmp_stat[item]['flow']
as_null = tmp_stat[item]['as_null']
as_delay = tmp_stat[item]['as_delay']/float(flow)
zhixin_null = tmp_stat[item]['zhixin_null']
zhixin_delay = tmp_stat[item]['zhixin_delay']/float(flow)
param=(self.last_id, hostname, date, city_id, srcID, flow, as_null, as_delay, zhixin_null, zhixin_delay,1)
n,last_id = mysql.add_zhixin_monitordata(param)
except Exception as e:
logging.error(str(e), exc_info=1)
mysql.close()
logging.info(u"#监控数据入库")
def post(self):
post_param=urlparse.parse_qs(self.request.body,True)
tooltype=post_param['tooltype'][0]
proto=post_param['proto'][0]
srv=post_param['srv'][0]
path=post_param['path'][0]
belong=post_param['belong'][0]
cutime=datetime.datetime.now().strftime("%y-%m-%d %H:%M:%S")
param=(tooltype,belong,cutime)
mysql=mysqlLib()
n,last_id=mysql.add_downloadstat(param)
'''param={}
downloadstat=mysql.query_downloadstat(1,param)
print downloadstat
downloadstat=mysql.query_downloadstat(0,param)
print downloadstat'''
mysql.close()
if(int(tooltype) == 1 or int(tooltype) == 3):
param={"testdata_id":proto}
mysql=mysqlLib()
datafilelist=mysql.query_datafile(param)
mysql.close()
for item in datafilelist:
filename=item[0]
filestr=item[1]
#print filename
#print filestr
f = open("../proto/" + filename, "w")
f.write(filestr)
f.close()
cmdstr="cd ../proto/ && ./build.sh"
status,output=cmd_execute(cmdstr)
cmdstr="cd .. && comake2 && make"
status,output=cmd_execute(cmdstr)
if(int(tooltype) == 1):
cmdstr="cd .. && tar -czvf pb.tar.gz conf data log pbrpcbenchmark proto README"
status,output=cmd_execute(cmdstr)
if(int(tooltype) == 3):
cmdstr="cd .. && tar -czvf pb.tar.gz case conf data log pbrpcclient pbunittest.py proto pub README"
status,output=cmd_execute(cmdstr)
param={"id":srv}
mysql=mysqlLib()
srvlist=mysql.query_server(param)
ip=srvlist[0][2]
username=srvlist[0][6]
password=srvlist[0][7]
#workspace=srvlist[0][3]
mysql.close()
rootdir="/home/users/yangjun03/protobuf/workspace/app-test/search/lbs-stat/upps_test/jenkinsFramework/protobuf/"
filepath="pb.tar.gz"
remotepath=path + "/pb/"
ssh_cmd(ip,22,username,password,"mkdir -p " + remotepath)
put_file(ip,22,username,password,rootdir+filepath,remotepath+"/"+filepath)
ssh_cmd(ip,22,username,password,"cd " + remotepath + " && tar -xzvf " + filepath)
#删除tar包
ssh_cmd(ip,22,username,password,"cd " + remotepath + " && rm " + filepath)
#删除所有.svn
ssh_cmd(ip,22,username,password,"cd " + remotepath + " && find . -type d -name '.svn' | xargs rm -rf")
sub_ret_dict={}
ret=u"测试工具下载成功"
sub_ret_dict['msg']=ret
self.write(json.dumps(sub_ret_dict))
else:
if(int(tooltype) == 2):
cmdstr="cd ~/httpclient/ && tar -czvf http.tar.gz conf data dep httpbenchmark.jar log README simplehttpserver.jar"
status,output=cmd_execute(cmdstr)
if(int(tooltype) == 4):
cmdstr="cd ~/httpclient/ && tar -czvf http.tar.gz case com conf data dep httpclient.jar httpunittest.py log README simplehttpserver.jar"
status,output=cmd_execute(cmdstr)
param={"id":srv}
mysql=mysqlLib()
srvlist=mysql.query_server(param)
ip=srvlist[0][2]
username=srvlist[0][6]
password=srvlist[0][7]
#workspace=srvlist[0][3]
mysql.close()
rootdir="/home/users/yangjun03/httpclient/"
filepath="http.tar.gz"
remotepath=path + "/http/"
ssh_cmd(ip,22,username,password,"mkdir -p " + remotepath)
put_file(ip,22,username,password,rootdir+filepath,remotepath+"/"+filepath)
ssh_cmd(ip,22,username,password,"cd " + remotepath + " && tar -xzvf " + filepath)
#删除tar包
ssh_cmd(ip,22,username,password,"cd " + remotepath + " && rm " + filepath)
#删除所有.svn
ssh_cmd(ip,22,username,password,"cd " + remotepath + " && find . -type d -name '.svn' | xargs rm -rf")
sub_ret_dict={}
ret=u"测试工具下载成功"
sub_ret_dict['msg']=ret
self.write(json.dumps(sub_ret_dict))
