def get(self): unique_id = self.get_argument("job").encode('utf-8') param={"id":unique_id} mysql=mysqlLib() datalist=mysql.query_task(param) mysql.close() #print datalist datalist_info={} datalist_info['submit'] = datalist[0][1] datalist_info['url'] = datalist[0][2] datalist_info['jobname'] = datalist[0][3] datalist_info['status'] = datalist[0][5] datalist_info['build_number'] = datalist[0][6] #print datalist_info cmdstr = "curl " + datalist_info['url'] + "job/" + datalist_info['jobname'] + "/" + str(datalist_info['build_number']) + "/logText/progressiveText?start=0" #print cmdstr if (datalist_info['status'] == 3 or datalist_info['status'] == 4 or datalist_info['status'] == 5): status,output=cmd_execute(cmdstr) jobname = datalist_info['jobname'] if (jobname == "pbdownload" or jobname == "addnode" or jobname == "delnode") : output = "INFO, task execute success." else: tmp = output.split("]'\r\n")[1].split("Notifying")[0] output = tmp else: output = "INFO, job is running now" datalist_info['output'] = output self.write(json.dumps(datalist_info))
def post(self): post_param=urlparse.parse_qs(self.request.body,True) limit=int(post_param['limit'][0]) offset=int(post_param['offset'][0]) belong=post_param['belong'][0] param={"limit":limit,"offset":offset} mysql=mysqlLib() tasklist=mysql.query_task(param) param={} task_totalcnt=mysql.query_task_totalcnt(param) mysql.close() serverlist_info={} serverlist_info['task_totalcnt']=task_totalcnt ret_dict=[] for index,item in enumerate(tasklist): sub_ret_dict={} sub_ret_dict['id']=str(int(item[0])) sub_ret_dict['submit']=item[1] sub_ret_dict['jenkinsurl']=item[2] sub_ret_dict['jobname']=item[3] sub_ret_dict['build_params']=item[4] sub_ret_dict['status']=item[5] sub_ret_dict['build_number']=item[6] ret_dict.append(sub_ret_dict) serverlist_info['ret_dict']=ret_dict self.write(json.dumps(serverlist_info))
def post(self): post_param=urlparse.parse_qs(self.request.body,True) limit=int(post_param['limit'][0]) offset=int(post_param['offset'][0]) #belong=post_param['belong'][0] param={"limit":limit,"offset":offset} mysql=mysqlLib() serverlist=mysql.query_server(param) param={} server_totalcnt=mysql.query_server_totalcnt(param) mysql.close() serverlist_info={} serverlist_info['server_totalcnt']=server_totalcnt ret_dict=[] for index,item in enumerate(serverlist): sub_ret_dict={} sub_ret_dict['id']=str(int(item[0])) sub_ret_dict['name']=item[1] sub_ret_dict['exenum']=str(int(item[2])) sub_ret_dict['workspace']=item[3] sub_ret_dict['ip']=item[4] sub_ret_dict['username']=item[5] sub_ret_dict['password']=item[6] sub_ret_dict['belong']=item[7] sub_ret_dict['descpt']=item[8] ret_dict.append(sub_ret_dict) serverlist_info['ret_dict']=ret_dict self.write(json.dumps(serverlist_info))
def monitor_alarm_proc(self): logging.info("task task_id:" + str(self.last_id) + " monitor_alarm_proc start") monitor_result = {} for item in self.NewIndexInterface_conf: desc = item["desc"] interface = item["interface"] ip_port = item["ip_port"] url = item["url"] method = item["method"] params = item["params"] expect = item["expect"] compare = item["compare"] flag = item["flag"] data = None monitor_result[interface] = {"status": 0, "msg": "ok"} if flag == 1: # print params h = httpclientLib(ip_port, url, method=method, params=params) status, data = h.request() if status != 200: monitor_result[interface] = {"status": -1, "msg": url + " access exception"} continue try: print json.dumps(data, ensure_ascii=False) print data print expect logging.info("****************" + desc + "****************") logging.info(json.dumps(data, ensure_ascii=False)) logging.info(json.dumps(expect, ensure_ascii=False)) tmp = json.dumps(data, ensure_ascii=False) # print tmp values = find_values("url", json.loads(tmp)) error_item_list = [] for item in values: # print item h = httpclientLib(ip_port, url, method=method, params=params) status, data = h.request() if status != 200: error_item_list.append(item) if len(error_item_list) != 0: monitor_result[interface] = {"status": -1, "msg": error_item_list} except Exception as e: monitor_result[interface] = {"status": -1, "msg": "response exception"} # print monitor_result logging.info(json.dumps(monitor_result)) total = 0 normal = 0 exception = 0 for item in monitor_result: total += 1 if monitor_result[item]["status"] == 0: normal += 1 else: exception += 1 mysql = mysqlLib() uptime = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") param = (total, normal, exception, json.dumps(monitor_result), uptime, "NewIndex") mysql.update_tbl_seo(param) mysql.close() logging.info("task task_id:" + str(self.last_id) + " monitor_alarm_proc complete")
def main(self): #1.获取账户信息 mysql = AccountInfo() self.accountInfo =mysql.get_tbl_accountinfo() mysql.save_tbl_accountinfo() mysql.save_tbl_cid() #2. 依次检查每一个账户 index = 0 while(index<len(self.accountInfo)): keyword = self.GetReport(self.check_date, self.accountInfo[index]) each_dict = self.CheckKeyWordByAccount(self.accountInfo[index],keyword) self.addCollect(each_dict) index+=1 print index #3. 写入数据库 mysql = mysqlLib() ret_list = list() for key in self.collect_dict: param=[self.check_date,key.encode('UTF-8'),self.collect_dict[key][0],self.collect_dict[key][1],self.collect_dict[key][2]] mysql.add_wordclass_data(param) ret_list.append(param) mysql.close() print u'end' return ret_list
def run(self): print '*********************************' print self.taskid self.LoadAccountCid('accountinfoAllCid.txt') self.LoadDeadDeal('deadtinyurl.txt') #得到账户的关键词情况,日报 self.GetReport(self.dateStr, self.accountInfo) #对每一个关键字检查对应的url,写入self.urlList self.CheckKeyWord(self.accountInfo) #统计数据存入本地数据库 mysql = mysqlLib() param=[self.taskid,self.dateStr,self.accountInfo[0],self.totalurlNum,self.errorurlNum,self.deadDealNum,self.badDeadDealNum,self.openurlErrorNum,self.noListResultNum,self.errorCidNum,self.formatErrorNum,self.nullUrlNum] mysql.add_url_data(param) #详细情况存入本地数据库 #1.得到tbl_url_data的id accountName=self.accountInfo[0] param=[accountName,self.dateStr] id=mysql.get_url_data_id(param) #2.插入tbl_errorurl for listInfo in self.resList: if(len(listInfo) <= 0): continue param=[id,self.accountInfo[0],listInfo[2],listInfo[3],listInfo[4],listInfo[5],listInfo[7],listInfo[6],self.dateStr] mysql.add_urlerror_data(param) print '***' mysql.close()
def post(self): param={} mysql=mysqlLib() downloadstat_week=mysql.query_downloadstat(1,param) downloadstat_all=mysql.query_downloadstat(0,param) mysql.close() datalist_info={} ret_dict=[] tmp = {} for index,item in enumerate(downloadstat_week): tmp[int(item[0])] = int(item[1]) for i in xrange(1,5): if(i not in tmp): ret_dict.append(0) else: ret_dict.append(tmp[i]) datalist_info['week']=ret_dict ret_dict=[] for index,item in enumerate(downloadstat_all): ret_dict.append(int(item[1])) datalist_info['all']=ret_dict self.write(json.dumps(datalist_info))
def update_task_status(self, task_id, status): try: mysql = mysqlLib() param=(status, task_id) mysql.update_tbl_monitortask_status(param) mysql.close() logging.info("task task_id:" + str(task_id) + " database status modified to :" + str(status)) except Exception as e: logging.error(str(e))
def add_monitor_data(self, task_id, status, info): try: mysql = mysqlLib() param=(task_id, status, json.dumps(info)) n,last_id = mysql.add_tbl_monitordata(param) mysql.close() logging.info("add monitor data last_id:" + str(last_id) + " to database success") except Exception as e: logging.error(str(e))
def update_task_endtime(self, task_id, endtime): try: mysql = mysqlLib() param=(endtime, task_id) mysql.update_tbl_monitortask_endtime(param) mysql.close() logging.info("task task_id:" + str(task_id) + " database endtime update success") except Exception as e: logging.error(str(e))
def add_alarm(self, alarmConf_id, monitorTask_id, monitorObj_id, monitorMetrics_id, description,status, datetime): try: mysql = mysqlLib() param=(alarmConf_id, monitorTask_id, monitorObj_id, monitorMetrics_id, description, status, datetime) n,last_id = mysql.add_alarmobj(param) mysql.close() logging.info("alarm object last_id:" + str(last_id) + " added success") except Exception as e: logging.error(str(e))
def post(self): post_param=urlparse.parse_qs(self.request.body,True) server_id=post_param['id'][0] param=(server_id,) mysql=mysqlLib() n=mysql.del_server(param) mysql.close() ret_dict={"errcode":n} self.write(json.dumps(ret_dict))
def post(self): #post_param=urlparse.parse_qs(self.request.body,True) #print self.request.body jsonobj = json.loads(self.request.body) unique_id = jsonobj['build']['parameters']['UNIQUE_ID'] number = jsonobj['build']['number'] phase = jsonobj['build']['phase'] if (phase == "STARTED"): mysql = mysqlLib() param=(2, unique_id) mysql.update_task_status(param) print(u"#RUNNING, job running, 2") param=(number, unique_id) mysql.update_task_build_number(param) mysql.close() elif (phase == "FINALIZED" or phase == "COMPLETED"): status = jsonobj['build']['status'] if (status == "SUCCESS"): mysql = mysqlLib() param=(3, unique_id) mysql.update_task_status(param) print(u"#SUCCESS, job complete, 3") mysql.close() elif (status == "ABORTED"): mysql = mysqlLib() param=(5, unique_id) mysql.update_task_status(param) print(u"#ABORTED, job aborted, 3") mysql.close() else: mysql = mysqlLib() param=(4, unique_id) mysql.update_task_status(param) print(u"#ERROR, job failed, 4") mysql.close() else: mysql = mysqlLib() param=(4, unique_id) mysql.update_task_status(param) print(u"#ERROR, job failed, 4") mysql.close()
def post(self): post_param=urlparse.parse_qs(self.request.body,True) testdata_id=post_param['id'][0] param=(testdata_id,) mysql=mysqlLib() n=mysql.del_testdata(param) m=mysql.del_datafile(param) mysql.close() ret_dict={"errcoden":n,"errcodem":m} self.write(json.dumps(ret_dict))
def monitor_alarm_proc(self): logging.info("task task_id:" + str(self.last_id) + " monitor_alarm_proc start") monitor_result = {} for item in self.TSMInterface_conf: desc = item['desc'] interface = item['interface'] ip_port = item['ip_port'] url = item['url'] method = item['method'] params = item['params'] expect = item['expect'] compare = item['compare'] flag = item['flag'] data = None if(flag == 1): #print params h = httpclientLib(ip_port, url, method=method, params=params) status,data = h.request() '''print json.dumps(data, ensure_ascii=False) print data print expect''' try: logging.info("****************" + desc + "****************") logging.info(json.dumps(data, ensure_ascii=False)) logging.info(json.dumps(expect, ensure_ascii=False)) if(compare == 1): if(data['data'] == expect['data']): monitor_result[interface] = {"status":0,"msg":"ok"} else: monitor_result[interface] = {"status":-1,"msg":"expect:\n"+json.dumps(expect, ensure_ascii=False)+"\ndata:\n"+json.dumps(data, ensure_ascii=False)} else: if(data['errno'] == 0): monitor_result[interface] = {"status":0,"msg":"ok"} else: monitor_result[interface] = {"status":-1,"msg":"errno not 0:\n"+json.dumps(data, ensure_ascii=False)} except Exception as e: monitor_result[interface] = {"status":-1,"msg":"response data error"} #print monitor_result logging.info(json.dumps(monitor_result)) total = 0 normal = 0 exception = 0 for item in monitor_result: total += 1 if(monitor_result[item]['status'] == 0): normal += 1 else: exception += 1 mysql = mysqlLib() uptime = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") param = (total, normal, exception, json.dumps(monitor_result), uptime, "TSM") mysql.update_tbl_seo(param) mysql.close() logging.info("task task_id:" + str(self.last_id) + " monitor_alarm_proc complete")
def get_monitorMetrics(self, monitorMetrics_id): try: mysql = mysqlLib() param={"id":monitorMetrics_id} retList = mysql.query_monitormetrics(param) mysql.close() ret_monitorMetrics_id = int(retList[0][0]) ret_monitorObj_id = int(retList[0][1]) ret_monitorMetrics_name = retList[0][2] logging.info("get_monitorMetrics : " + str(monitorMetrics_id) + ", success") return ret_monitorMetrics_id,ret_monitorObj_id,ret_monitorMetrics_name except Exception as e: logging.error(str(e))
def post(self): post_param=urlparse.parse_qs(self.request.body,True) srv_id=int(post_param['id'][0]) param={"id":srv_id} mysql=mysqlLib() serverlist=mysql.query_server(param) mysql.close() ret_dict={} workspace=serverlist[0][3] ret_dict['workspace']=workspace self.write(json.dumps(ret_dict))
def post(self): post_param=urlparse.parse_qs(self.request.body,True) datatype=post_param['type'][0] name=post_param['name'][0] belong=post_param['belong'][0] filenum=post_param['filenum'][0] descpt=post_param['desc'][0] param=(datatype,name,belong,filenum,descpt) mysql=mysqlLib() n,last_id=mysql.add_testdata(param) mysql.close() ret_dict={"errcode":n,"last_id":last_id} self.write(json.dumps(ret_dict))
def add_job(self, monitorObj_id, monitorMetrics_id, monitorMetrics_name, taskrunner): try: mysql = mysqlLib() param=(monitorObj_id, monitorMetrics_id, 0) n,last_id = mysql.add_task(param) mysql.close() logging.info("add_job last_id:" + str(last_id) + ",monitorObj_id:" + str(monitorObj_id) + ",monitorMetrics_id:" + str(monitorMetrics_id) + " to database success") jobname = self.jobname jobparameters = {"last_id":last_id,"monitorObj_id":monitorObj_id,"monitorMetrics_id":monitorMetrics_id,"monitorMetrics_name":monitorMetrics_name,"taskrunner":taskrunner} gearclient = gearClient() gearclient.job_submit(str(jobname), jobparameters, wait_until_complete = False, background = True) logging.info("gear job last_id:" + str(last_id) + ",monitorObj_id:" + str(monitorObj_id) + ",monitorMetrics_id:" + str(monitorMetrics_id) + " submited success") self.update_task_status(last_id, 1) except Exception as e: logging.error(str(e))
def query_alarm_conf(self, monitorObj_id, monitorMetrics_id): try: mysql = mysqlLib() param={"monitorObj_id":monitorObj_id,"monitorMetrics_id":monitorMetrics_id} retList = mysql.query_tbl_alarmconf(param) mysql.close() alarm_conf_id = int(retList[0][0]) monitorObj_id = int(retList[0][1]) monitorMetrics_id = int(retList[0][2]) name = str(retList[0][3]) description = str(retList[0][4]) status = int(retList[0][5]) conf = json.loads(retList[0][6]) logging.info("query alarm conf monitorObj_id:" + str(monitorObj_id) + ",monitorMetrics_id:" + str(monitorMetrics_id) +" success") return alarm_conf_id,monitorObj_id,monitorMetrics_id,name,description,status,conf except Exception as e: logging.error(str(e))
def post(self): post_param=urlparse.parse_qs(self.request.body,True) name=post_param['name'][0] exenum=post_param['exenum'][0] ip=post_param['ip'][0] username=post_param['username'][0] password=post_param['password'][0] workspace=post_param['workspace'][0] belong=post_param['belong'][0] descpt=post_param['descpt'][0] param=(name,exenum,workspace,ip,username,password,belong,descpt) mysql=mysqlLib() n,last_id=mysql.add_server(param) mysql.close() ret_dict={"errcode":n} self.write(json.dumps(ret_dict))
def post(self): post_param=urlparse.parse_qs(self.request.body,True) json_postdata={} try: for i in post_param[' name']: json_postdata[i.split("\r\n")[0].strip("\"")]=i.split("\r\n")[2] except Exception: pass #print json_postdata mysql=mysqlLib() param=(json_postdata['last_id'],self.request.files['Filedata'][0]['filename'],self.request.files['Filedata'][0]['body']) n,last_id=mysql.add_datastr(param) mysql.close() ret_dict={"errcode":n} self.write(json.dumps(ret_dict)) '''file_dict_list = self.request.files['Filedata']
def post(self): try: post_param=urlparse.parse_qs(self.request.body,True) limit=int(post_param['limit'][0]) offset=int(post_param['offset'][0]) param={"limit":limit,"offset":offset} mysql=mysqlLib() alarm_list=mysql.query_tbl_alarmobj(param) param={} alarm_totalcnt=mysql.query_alarm_totalcnt(param) mysql.close() alarm_list_info={} alarm_list_info['alarm_totalcnt']=alarm_totalcnt ret_dict=[] for index,item in enumerate(alarm_list): sub_ret_dict={} sub_ret_dict['id']=str(int(item[0])) sub_ret_dict['alarmConf_id']=str(int(item[1])) sub_ret_dict['monitorTask_id']=str(int(item[2])) sub_ret_dict['monitorObj_id']=str(int(item[3])) sub_ret_dict['monitorMetrics_id']=str(int(item[4])) sub_ret_dict['description']=item[5] sub_ret_dict['status']=str(int(item[6])) sub_ret_dict['datetime']=str(item[7]) sub_ret_dict['cause']=item[8] sub_ret_dict['handleBy']=item[9] sub_ret_dict['info']=item[10] ret_dict.append(sub_ret_dict) alarm_list_info['ret_dict']=ret_dict self.write(json.dumps(alarm_list_info)) except Exception as e: logging.error(str(e)) logging.info(traceback.print_exc())
def get(self): userid=self.get_cookie_user() unique_id = self.get_argument("job").encode('utf-8') param={"id":unique_id} mysql=mysqlLib() datalist=mysql.query_task(param) mysql.close() #print datalist datalist_info={} datalist_info['submit'] = datalist[0][1] datalist_info['url'] = datalist[0][2] datalist_info['jobname'] = datalist[0][3] datalist_info['status'] = datalist[0][5] datalist_info['build_number'] = datalist[0][6] #print datalist_info cmdstr = "curl " + datalist_info['url'] + "job/" + datalist_info['jobname'] + "/" + str(datalist_info['build_number']) + "/logText/progressiveText?start=0" #print cmdstr if (datalist_info['status'] == 3 or datalist_info['status'] == 4 or datalist_info['status'] == 5): status,output=cmd_execute(cmdstr) jobname = datalist_info['jobname'] if (jobname == "pbdownload" or jobname == "addnode" or jobname == "delnode") : output = "INFO, task execute success." else: tmp = output.split("]'\r\n")[1].split("Notifying")[0] output = tmp output = output.replace("\n","<br/>") else: output = "INFO, job is running now" datalist_info['output'] = output #self.write(json.dumps(datalist_info)) if(userid): self.render("./taskmanagement/taskdetail.html",usrname=userid,datalist_info=datalist_info) else: redirect_url=CAS_SETTINGS[ 'cas_server' ] + '/login?service=' + CAS_SETTINGS[ 'service_url' ] self.redirect(redirect_url)
def post(self): post_param=urlparse.parse_qs(self.request.body,True) limit=int(post_param['limit'][0]) offset=int(post_param['offset'][0]) belong=post_param['belong'][0] datatype=post_param['datatype'][0] if(datatype == -1): param={"belong":belong,"limit":limit,"offset":offset} else: param={"belong":belong,"type":datatype,"limit":limit,"offset":offset} mysql=mysqlLib() datalist=mysql.query_testdata(param) if(datatype == -1): param={"belong":belong} else: param={"belong":belong,"type":datatype} data_totalcnt=mysql.query_testdata_totalcnt(param) mysql.close() datalist_info={} datalist_info['data_totalcnt']=data_totalcnt ret_dict=[] for index,item in enumerate(datalist): sub_ret_dict={} sub_ret_dict['id']=str(int(item[0])) sub_ret_dict['name']=item[1] sub_ret_dict['type']=item[2] sub_ret_dict['belong']=item[3] sub_ret_dict['filenum']=item[4] sub_ret_dict['descpt']=item[5] ret_dict.append(sub_ret_dict) datalist_info['ret_dict']=ret_dict self.write(json.dumps(datalist_info))
def monitor_alarm_proc(self): logging.info("task task_id:" + str(self.last_id) + " monitor_alarm_proc start") #监控数据上报 cutime=datetime.datetime.now().strftime("%y-%m-%d %H:%M:%S") #1. 查询最近四周的数据,如果没有,就调用get_wordclass脚本 #function区分功能是 展示 impression,点击 click,消费 cost now_time = datetime.datetime.now() start_time = now_time + datetime.timedelta(days=-1) index=0 while(index<=3): tmp = index*7 day = start_time + datetime.timedelta(days=-tmp)#delay to index week ago dateStr = day.strftime('%Y-%m-%d') param = [dateStr] mysql = mysqlLib() ret = mysql.query_wordclass_data(param,function) mysql.close() if not ret: wc = WordCalss(dateStr) tmplist=wc.main() mysql = mysqlLib() ret = mysql.query_wordclass_data(param,function) mysql.close() #2 . 合并结果 self.makeFinalResult(ret) index+=1 logging.info("task task_id:" + str(self.last_id) + " insert into db data_table end") #3. 检查报警数据并写入数据库 #查询报警配置 alarm_conf_id,monitorObj_id,monitorMetrics_id,name,description,status,conf = self.query_alarm_conf(self.monitorObj_id, self.monitorMetrics_id) #对比报警条件 self.CreateList() print '2' self.ComputeDiff(conf['threshold']) print '3' warnList=list() for item in self.resList: cutime=datetime.datetime.now().strftime("%y-%m-%d %H:%M:%S") if(1 == item['flag']): dsc = description + " : 词类:" + str(item['wordClass']) + ", 报警:降幅超过阈值 " +str(item[function]) print dsc self.add_alarm(alarm_conf_id, self.last_id, monitorObj_id,monitorMetrics_id, dsc , 0, cutime) warnList.append(dsc) '''elif(2 == item['flag']): dsc = description + " : 词类:" + str(item['wordClass']) + ", 报警:涨幅超过阈值 " +str(item[function]) print dsc self.add_alarm(alarm_conf_id, self.last_id, monitorObj_id,monitorMetrics_id, dsc , 0, cutime) warnList.append(dsc)''' logging.info("task task_id:" + str(self.last_id) + " monitor_alarm_proc end") #step 6: 发送报警邮件,短信 for item in warnList: self.mail_send(self.subject+" "+description, item, self.from_mail_addr, self.to_mail_addr, self.mail_server) logging.info("task task_id:" + str(self.last_id) + " mail send end") #报警短信 for item in warnList: self.msgSend(self.phone_list, item , mode="qapi") logging.info("task task_id:" + str(self.last_id) + " msg send end")
def monitor_alarm_proc(self): mysql = AccountInfo() self.accountInfo =mysql.get_tbl_accountinfo() mysql.save_tbl_accountinfo() mysql.save_tbl_cid() mysql.close() logging.info("task task_id:" + str(self.last_id) + " monitor_alarm_proc start") #监控数据上报 now_time = datetime.datetime.now() day = now_time + datetime.timedelta(days=-1) dateStr = day.strftime('%Y-%m-%d') index=0 th_list = [] while(index<len(self.accountInfo)): myMonitor = MyUrlMonitor(dateStr,self.accountInfo[index],self.last_id) th_list.append(myMonitor) myMonitor.start() index+=1 for item in th_list: item.join() logging.info("task task_id:" + str(self.last_id) + " all thread run end") #检查报警 mysql = mysqlLib() alarm_conf_id,monitorObj_id,monitorMetrics_id,name,description,status,conf = self.query_alarm_conf(self.monitorObj_id, self.monitorMetrics_id) #对比报警条件 print conf['threshold'] param=[dateStr,conf['threshold']] warnList = mysql.query_urlalarm_data(param) alarmList = list() if(len(warnList)==0): mysql.close() return 'close' for item in warnList: accountName=item[2] totalurlNum=item[3] errorurlNum=item[4] deadDealNum=item[5] badDeadDealNum=item[6] openurlErrorNum=item[7] noListResultNum=item[8] errorCidNum=item[9] formatErrorNum=item[10] nullUrlNum=item[11] dsc = description + " 帐户名:" + str(accountName) + ",Url总数:"+str(totalurlNum)+",Error总数:"+str(errorurlNum)+",过期团单:"+str(deadDealNum)+",非合理过期团单:"+str(badDeadDealNum)+",访问错误:"+str(openurlErrorNum)+",无搜索结果:"+str(noListResultNum)+",非天兔Cid:"+str(errorCidNum)+",URL格式错误:"+str(formatErrorNum)+",空URL:"+str(nullUrlNum) alarmList.append(dsc) cutime=datetime.datetime.now().strftime("%y-%m-%d %H:%M:%S") self.add_alarm(alarm_conf_id, self.last_id, monitorObj_id,monitorMetrics_id, dsc , 0, cutime) logging.info("task task_id:" + str(self.last_id) + " insert into db alarm_table end") mysql.close() #step 6: 发送报警邮件,短信 for item in alarmList: self.mail_send(self.subject+" "+description, str(item), self.from_mail_addr, self.to_mail_addr, self.mail_server) logging.info("task task_id:" + str(self.last_id) + " mail send end") #报警短信 for item in warnList: self.msgSend(self.phone_list, str(item) , mode="qapi") logging.info("task task_id:" + str(self.last_id) + " msg send end")
def run(self): #获取logid请求参数 self.ftpPath = "ftp://" + self.hostname + "/home/tuan/odp/log/zhixin/zhixin.log." + self.datetime self.line_list = None try: self.line_list = geFtpFile2Memory(self.ftpPath, self.keystr_request) except Exception as e: logging.error("", exc_info=1) return for line in self.line_list: tmp_line_list = re.split("[][]", line) logid = tmp_line_list[5] req_param = tmp_line_list[22] req_param_list = req_param.split(",") try: city_id = req_param_list[1].split(":")[1] srcID = req_param_list[2].split(":")[1] keyword = req_param_list[3].split(":")[1] original_query = req_param_list[4].split(":")[1] req_type = req_param_list[5].split(":")[1] self.log_stat[logid] = { "city_id":city_id, "srcID":srcID, #"keyword":keyword.decode('GBK').encode('UTF-8'), #"original_query":original_query.decode('GBK').encode('UTF-8'), #"req_type":req_type, "as_null":0, "as_delay":0, "zhixin_null":0, "zhixin_delay":0 } except Exception as e: logging.error("line : " + line) logging.error("req_param : " + req_param) #traceback.print_exc() logging.error("", exc_info=1) '''#logid维度获取各项指标 for logid in self.log_stat: self.line_list = geFtpFile2Memory(self.ftpPath, logid) for line in self.line_list: if(self.keystr_as_total in line): tmp_line_list = re.split("zhixin New as response,total:", line) as_total = int(tmp_line_list[1]) if(as_total == 0): self.log_stat[logid]['as_null'] = 1 elif(self.keystr_as_cost in line): tmp_line_list = re.split("new as cost ", line) as_delay = int(tmp_line_list[1].split(" ")[0]) self.log_stat[logid]['as_delay'] = as_delay elif(self.keystr_zhixin_total in line): tmp_line_list = re.split("zhixin_show", line) if(tmp_line_list[1] == "[]"): self.log_stat[logid]['zhixin_null'] = 1 elif(self.keystr_zhixin_cost in line): tmp_line_list = re.split("Zhixin Query request cost ", line) zhixin_delay = int(tmp_line_list[1].split(" ")[0]) self.log_stat[logid]['zhixin_delay'] = zhixin_delay''' #logid维度获取各项指标-as_null指标 self.line_list = geFtpFile2Memory(self.ftpPath, self.keystr_as_total) for logid in self.log_stat: for line in self.line_list: if(logid in line): tmp_line_list = re.split("zhixin New as response,total:", line) try: as_total = int(tmp_line_list[1]) if(as_total == 0): self.log_stat[logid]['as_null'] = 1 except Exception as e: logging.error(str(e), exc_info=1) #logid维度获取各项指标-as_delay指标 self.line_list = geFtpFile2Memory(self.ftpPath, self.keystr_as_cost) for logid in self.log_stat: for line in self.line_list: if(logid in line): tmp_line_list = re.split("new as cost ", line) as_delay = int(tmp_line_list[1].split(" ")[0]) self.log_stat[logid]['as_delay'] = as_delay #logid维度获取各项指标-as_delay指标 self.line_list = geFtpFile2Memory(self.ftpPath, self.keystr_zhixin_total) for logid in self.log_stat: for line in self.line_list: if(logid in line): tmp_line_list = re.split("zhixin_show", line) if(tmp_line_list[1] == "[]"): self.log_stat[logid]['zhixin_null'] = 1 #logid维度获取各项指标-as_delay指标 self.line_list = geFtpFile2Memory(self.ftpPath, self.keystr_zhixin_cost) for logid in self.log_stat: for line in self.line_list: if(logid in line): tmp_line_list = re.split("Zhixin Query request cost ", line) zhixin_delay = int(tmp_line_list[1].split(" ")[0]) self.log_stat[logid]['zhixin_delay'] = zhixin_delay '''print self.log_stat total_len = 0 for item in self.log_stat: total_len += 1 print total_len''' #遍历log_stat, 给出最终hostname,date,cityID,srcID维度的统计数据——初始化 tmp_stat = {} for item in self.log_stat: data = self.log_stat[item] tmp_key = self.hostname + self.datetime + data['city_id'] + data['srcID'] md5_data = hashlib.md5(tmp_key.encode("utf-8")).hexdigest() tmp_stat[md5_data] = { "hostname":self.hostname, "datetime":self.datetime, "city_id":data['city_id'], "srcID":data['srcID'], "flow":0, "as_null":0, "as_delay":0, "zhixin_null":0, "zhixin_delay":0 } #遍历log_stat, 给出最终hostname,date,cityID,srcID维度的统计数据——统计计算 for item in self.log_stat: data = self.log_stat[item] tmp_key = self.hostname + self.datetime + data['city_id'] + data['srcID'] md5_data = hashlib.md5(tmp_key.encode("utf-8")).hexdigest() tmp_stat[md5_data]['hostname'] = self.hostname tmp_stat[md5_data]['datetime'] = self.datetime tmp_stat[md5_data]['city_id'] = data['city_id'] tmp_stat[md5_data]['srcID'] = data['srcID'] tmp_stat[md5_data]['flow'] += 1 tmp_stat[md5_data]['as_null'] += data['as_null'] tmp_stat[md5_data]['as_delay'] += data['as_delay'] tmp_stat[md5_data]['zhixin_null'] += data['zhixin_null'] tmp_stat[md5_data]['zhixin_delay'] += data['zhixin_delay'] #print tmp_stat #监控数据入库 mysql = mysqlLib() for item in tmp_stat: try: hostname = tmp_stat[item]['hostname'] date = tmp_stat[item]['datetime'] a = time.strptime(date,'%Y%m%d%H') timestamp = time.mktime(a) localtime = time.localtime(timestamp) date = time.strftime('%Y-%m-%d %H:%M:%S',localtime) city_id = tmp_stat[item]['city_id'] srcID = tmp_stat[item]['srcID'] flow = tmp_stat[item]['flow'] as_null = tmp_stat[item]['as_null'] as_delay = tmp_stat[item]['as_delay']/float(flow) zhixin_null = tmp_stat[item]['zhixin_null'] zhixin_delay = tmp_stat[item]['zhixin_delay']/float(flow) param=(self.last_id, hostname, date, city_id, srcID, flow, as_null, as_delay, zhixin_null, zhixin_delay,0) n,last_id = mysql.add_zhixin_monitordata(param) except Exception as e: logging.error(str(e), exc_info=1) mysql.close()
def run(self): #获取logid请求参数 self.ftpPath = "ftp://" + self.hostname + "/home/tuan/odp/log/zhixin/zhixin.log." + self.datetime self.line_list = None try: self.line_list = geFtpFile2Memory(self.ftpPath, self.keystr_request) except Exception as e: logging.error("", exc_info=1) return for line in self.line_list: tmp_line_list = re.split("[][]", line) '''logid = tmp_line_list[5] req_param = json.loads(tmp_line_list[20].split("Aladdin queryInfo: ")[1])''' try: logid = tmp_line_list[5] req_param = json.loads(tmp_line_list[20].split("Aladdin queryInfo: ")[1]) city_id = str(req_param['city_id']) srcID = str(req_param['SrcID']) self.log_stat[logid] = { "city_id":city_id, "srcID":srcID, #"keyword":keyword.decode('GBK').encode('UTF-8'), #"original_query":original_query.decode('GBK').encode('UTF-8'), #"req_type":req_type, "as_null":0, "as_delay":0, "zhixin_null":0, "zhixin_delay":0 } except Exception as e: logging.error("line : " + line) logging.error("", exc_info=1) logging.info(u"#获取logid请求参数") '''#下载文件 dstFilePath = "zhixin.log." + self.datetime + "---" + self.hostname downFtpFile(self.ftpPath, dstFilePath) logging.info(u"#下载文件") #logid维度获取各项指标-as_null指标 for logid in self.log_stat: cmdstr = 'LC_ALL=C fgrep ' + logid + ' ' + dstFilePath + ' | fgrep "Aladdin query redis value :"' status,line=cmd_execute(cmdstr) if(line): tmp_line_list = re.split("Aladdin query redis value :", line) as_total = int(tmp_line_list[1]) if(as_total == 1): self.log_stat[logid]['as_null'] = 1 logging.info(u"#logid维度获取各项指标-as_null指标") #logid维度获取各项指标-as_delay指标 for logid in self.log_stat: cmdstr = 'LC_ALL=C fgrep ' + logid + ' ' + dstFilePath + ' | fgrep "Aladdin nuomi query cost "' status,line=cmd_execute(cmdstr) if(line): tmp_line_list = re.split("Aladdin nuomi query cost ", line) as_delay = int(tmp_line_list[1].split(" ")[0]) self.log_stat[logid]['as_delay'] = as_delay else: self.log_stat[logid]['as_delay'] = 0 logging.info(u"#logid维度获取各项指标-as_delay指标") #logid维度获取各项指标-as_delay指标 for logid in self.log_stat: cmdstr = 'LC_ALL=C fgrep ' + logid + ' ' + dstFilePath + ' | fgrep "aladdin_show"' status,line=cmd_execute(cmdstr) if(line): tmp_line_list = re.split("aladdin_show", line) if(tmp_line_list[1] == "[]"): self.log_stat[logid]['zhixin_null'] = 1 logging.info(u"##logid维度获取各项指标-as_delay指标") #logid维度获取各项指标-as_delay指标 for logid in self.log_stat: cmdstr = 'LC_ALL=C fgrep ' + logid + ' ' + dstFilePath + ' | fgrep "Aladdin Query request cost "' status,line=cmd_execute(cmdstr) if(line): tmp_line_list = re.split("Aladdin Query request cost ", line) zhixin_delay = int(tmp_line_list[1].split(" ")[0]) self.log_stat[logid]['zhixin_delay'] = zhixin_delay logging.info(u"##logid维度获取各项指标-as_delay指标")''' #logid维度获取各项指标-as_null指标 self.line_list = geFtpFile2Memory(self.ftpPath, self.keystr_as_total) for logid in self.log_stat: for line in self.line_list: if(logid in line): tmp_line_list = re.split("Aladdin query redis value :", line) as_total = int(tmp_line_list[1]) if(as_total == 1): self.log_stat[logid]['as_null'] = 1 logging.info(u"#logid维度获取各项指标-as_null指标") #logid维度获取各项指标-as_delay指标 self.line_list = geFtpFile2Memory(self.ftpPath, self.keystr_as_cost) for logid in self.log_stat: for line in self.line_list: if(logid in line): tmp_line_list = re.split("Aladdin nuomi query cost ", line) as_delay = int(tmp_line_list[1].split(" ")[0]) self.log_stat[logid]['as_delay'] = as_delay else: self.log_stat[logid]['as_delay'] = 0 logging.info(u"#logid维度获取各项指标-as_delay指标") #logid维度获取各项指标-as_delay指标 self.line_list = geFtpFile2Memory(self.ftpPath, self.keystr_zhixin_total) for logid in self.log_stat: for line in self.line_list: if(logid in line): tmp_line_list = re.split("aladdin_show", line) if(tmp_line_list[1] == "[]"): self.log_stat[logid]['zhixin_null'] = 1 logging.info(u"##logid维度获取各项指标-as_delay指标") #logid维度获取各项指标-as_delay指标 self.line_list = geFtpFile2Memory(self.ftpPath, self.keystr_zhixin_cost) for logid in self.log_stat: for line in self.line_list: if(logid in line): tmp_line_list = re.split("Aladdin Query request cost ", line) zhixin_delay = int(tmp_line_list[1].split(" ")[0]) self.log_stat[logid]['zhixin_delay'] = zhixin_delay logging.info(u"##logid维度获取各项指标-as_delay指标") '''print self.log_stat total_len = 0 for item in self.log_stat: total_len += 1 print total_len''' #遍历log_stat, 给出最终hostname,date,cityID,srcID维度的统计数据——初始化 tmp_stat = {} for item in self.log_stat: try: data = self.log_stat[item] tmp_key = self.hostname + self.datetime + data['city_id'] + data['srcID'] md5_data = hashlib.md5(tmp_key.encode("utf-8")).hexdigest() tmp_stat[md5_data] = { "hostname":self.hostname, "datetime":self.datetime, "city_id":data['city_id'], "srcID":data['srcID'], "flow":0, "as_null":0, "as_delay":0, "zhixin_null":0, "zhixin_delay":0 } except Exception as e: logging.error(str(e), exc_info=1) logging.info(u"#遍历log_stat, 给出最终hostname,date,cityID,srcID维度的统计数据——初始化") #遍历log_stat, 给出最终hostname,date,cityID,srcID维度的统计数据——统计计算 for item in self.log_stat: try: data = self.log_stat[item] tmp_key = self.hostname + self.datetime + data['city_id'] + data['srcID'] md5_data = hashlib.md5(tmp_key.encode("utf-8")).hexdigest() tmp_stat[md5_data]['hostname'] = self.hostname tmp_stat[md5_data]['datetime'] = self.datetime tmp_stat[md5_data]['city_id'] = data['city_id'] tmp_stat[md5_data]['srcID'] = data['srcID'] tmp_stat[md5_data]['flow'] += 1 tmp_stat[md5_data]['as_null'] += data['as_null'] tmp_stat[md5_data]['as_delay'] += data['as_delay'] tmp_stat[md5_data]['zhixin_null'] += data['zhixin_null'] tmp_stat[md5_data]['zhixin_delay'] += data['zhixin_delay'] except Exception as e: logging.error(str(e), exc_info=1) logging.info(u"#遍历log_stat, 给出最终hostname,date,cityID,srcID维度的统计数据——统计计算") #print tmp_stat #监控数据入库 mysql = mysqlLib() for item in tmp_stat: try: hostname = tmp_stat[item]['hostname'] date = tmp_stat[item]['datetime'] a = time.strptime(date,'%Y%m%d%H') timestamp = time.mktime(a) localtime = time.localtime(timestamp) date = time.strftime('%Y-%m-%d %H:%M:%S',localtime) city_id = tmp_stat[item]['city_id'] srcID = tmp_stat[item]['srcID'] flow = tmp_stat[item]['flow'] as_null = tmp_stat[item]['as_null'] as_delay = tmp_stat[item]['as_delay']/float(flow) zhixin_null = tmp_stat[item]['zhixin_null'] zhixin_delay = tmp_stat[item]['zhixin_delay']/float(flow) param=(self.last_id, hostname, date, city_id, srcID, flow, as_null, as_delay, zhixin_null, zhixin_delay,1) n,last_id = mysql.add_zhixin_monitordata(param) except Exception as e: logging.error(str(e), exc_info=1) mysql.close() logging.info(u"#监控数据入库")
def post(self): post_param=urlparse.parse_qs(self.request.body,True) tooltype=post_param['tooltype'][0] proto=post_param['proto'][0] srv=post_param['srv'][0] path=post_param['path'][0] belong=post_param['belong'][0] cutime=datetime.datetime.now().strftime("%y-%m-%d %H:%M:%S") param=(tooltype,belong,cutime) mysql=mysqlLib() n,last_id=mysql.add_downloadstat(param) '''param={} downloadstat=mysql.query_downloadstat(1,param) print downloadstat downloadstat=mysql.query_downloadstat(0,param) print downloadstat''' mysql.close() if(int(tooltype) == 1 or int(tooltype) == 3): param={"testdata_id":proto} mysql=mysqlLib() datafilelist=mysql.query_datafile(param) mysql.close() for item in datafilelist: filename=item[0] filestr=item[1] #print filename #print filestr f = open("../proto/" + filename, "w") f.write(filestr) f.close() cmdstr="cd ../proto/ && ./build.sh" status,output=cmd_execute(cmdstr) cmdstr="cd .. && comake2 && make" status,output=cmd_execute(cmdstr) if(int(tooltype) == 1): cmdstr="cd .. && tar -czvf pb.tar.gz conf data log pbrpcbenchmark proto README" status,output=cmd_execute(cmdstr) if(int(tooltype) == 3): cmdstr="cd .. && tar -czvf pb.tar.gz case conf data log pbrpcclient pbunittest.py proto pub README" status,output=cmd_execute(cmdstr) param={"id":srv} mysql=mysqlLib() srvlist=mysql.query_server(param) ip=srvlist[0][2] username=srvlist[0][6] password=srvlist[0][7] #workspace=srvlist[0][3] mysql.close() rootdir="/home/users/yangjun03/protobuf/workspace/app-test/search/lbs-stat/upps_test/jenkinsFramework/protobuf/" filepath="pb.tar.gz" remotepath=path + "/pb/" ssh_cmd(ip,22,username,password,"mkdir -p " + remotepath) put_file(ip,22,username,password,rootdir+filepath,remotepath+"/"+filepath) ssh_cmd(ip,22,username,password,"cd " + remotepath + " && tar -xzvf " + filepath) #删除tar包 ssh_cmd(ip,22,username,password,"cd " + remotepath + " && rm " + filepath) #删除所有.svn ssh_cmd(ip,22,username,password,"cd " + remotepath + " && find . -type d -name '.svn' | xargs rm -rf") sub_ret_dict={} ret=u"测试工具下载成功" sub_ret_dict['msg']=ret self.write(json.dumps(sub_ret_dict)) else: if(int(tooltype) == 2): cmdstr="cd ~/httpclient/ && tar -czvf http.tar.gz conf data dep httpbenchmark.jar log README simplehttpserver.jar" status,output=cmd_execute(cmdstr) if(int(tooltype) == 4): cmdstr="cd ~/httpclient/ && tar -czvf http.tar.gz case com conf data dep httpclient.jar httpunittest.py log README simplehttpserver.jar" status,output=cmd_execute(cmdstr) param={"id":srv} mysql=mysqlLib() srvlist=mysql.query_server(param) ip=srvlist[0][2] username=srvlist[0][6] password=srvlist[0][7] #workspace=srvlist[0][3] mysql.close() rootdir="/home/users/yangjun03/httpclient/" filepath="http.tar.gz" remotepath=path + "/http/" ssh_cmd(ip,22,username,password,"mkdir -p " + remotepath) put_file(ip,22,username,password,rootdir+filepath,remotepath+"/"+filepath) ssh_cmd(ip,22,username,password,"cd " + remotepath + " && tar -xzvf " + filepath) #删除tar包 ssh_cmd(ip,22,username,password,"cd " + remotepath + " && rm " + filepath) #删除所有.svn ssh_cmd(ip,22,username,password,"cd " + remotepath + " && find . -type d -name '.svn' | xargs rm -rf") sub_ret_dict={} ret=u"测试工具下载成功" sub_ret_dict['msg']=ret self.write(json.dumps(sub_ret_dict))