def readDataFromFile(logfilename):
logInfos = []
i = 0
try:
f = codecs.open(logfilename, 'r', encoding='gbk')
for line in f.readlines():
tj = str2Json(line[line.find('[') + 1:len(line) - 2])
i += 1
logInfos.append(tj)
if i == 2000:
break
#info = LogInfo()
#info.setAttr(tj)
f.close()
ret, ret1 = genInfo(tuple(logInfos), [], []) # to from
ret, ret1 = DataAnalysis(ret, ret1)
for i in range(0, len(ret)):
Deliver2ESWithpath(Dict2Strjson(ret[i]), "to1domain", "table1", i)
for i in range(0, len(ret)):
Deliver2ESWithpath(Dict2Strjson(ret[i]), "from1domain", "table2",
i)
#handle(logInfos)
except IOError:
g_loginst.logger.error("can't open such %s file" % logfilename)
def DataAnalysis(retAll, retFromAll):
#统计收件人域名的情况
OrderByToDomain = []
for part in retAll:
temp = {}
maxtime = 0 # 投递最长时间
mintime = 100000000 # 投递最短时间
averagetime = 0 # 投递的平均时长
temp["toDomain"] = part["to"] #收件人域名
for proxyInfo in part["proxy_chain"]:
averagetime += proxyInfo["optime"]
if maxtime <= proxyInfo["optime"]:
maxtime = proxyInfo["optime"]
if mintime >= proxyInfo["optime"]:
mintime = proxyInfo["optime"]
averagetime /= len(part["proxy_chain"])
temp["averagetime"] = averagetime
temp["maxtime"] = maxtime
temp["mintime"] = mintime
temp["proxycnt"] = len(part["proxy_chain"])
OrderByToDomain.append(temp)
#print json.dumps(OrderByToDomain,indent=4)
#统计发信人域名的情况
OrderByFromDomain = []
for part in retFromAll:
temp = {}
maxtime = 0 # 投递最长时间
mintime = 100000000 # 投递最短时间
averagetime = 0 # 投递的平均时长
temp["fromDomain"] = part["from"] #收件人域名
for proxyInfo in part["proxy_chain"]:
averagetime += proxyInfo["optime"]
if maxtime <= proxyInfo["optime"]:
maxtime = proxyInfo["optime"]
if mintime >= proxyInfo["optime"]:
mintime = proxyInfo["optime"]
averagetime /= len(part["proxy_chain"])
temp["averagetime"] = averagetime
temp["maxtime"] = maxtime
temp["mintime"] = mintime
temp["proxycnt"] = len(part["proxy_chain"])
OrderByFromDomain.append(temp)
# json.dumps(OrderByFromDomain,indent=4)
i = 0
for eachitem in OrderByToDomain:
Deliver2ESWithpath(eachitem, "todomain", "test_todmain", i)
i += 1
i = 0
for eachitem in OrderByFromDomain:
Deliver2ESWithpath(eachitem, "fromdomain", "test_domain", i)
i += 1
i = 0
return OrderByToDomain, OrderByFromDomain
def refreshTimeCount():
global g_logobj
g_logobj.logger.info("the g_logdict size=%d"%(len(g_logdict)))
for dkey,dvalue in g_logdict.items():
dvalue.CM_TimeCount+=1
if(dvalue.CM_status=="succeed" or dvalue.CM_status=="fail" or dvalue.CM_TimeCount>=g_logkeepTimeCountmax):
#投递给ES
#从g_logdict删除
if(dvalue.CM_TimeCount>=g_logkeepTimeCountmax):
dvalue.CM_status="overtime"
dvalue.country = updateIPmapping(dvalue.destip,g_mysqlInst)
Deliver2ESWithpath(dvalue.Data2Json(),"everymail","log",dvalue.CM_Index)
#业务功能 邮件走势量
StatmailInfo(dvalue)
global mailCount
g_logobj.logger.info("Delieve to everymail succeed------------------------------count=%d"%(mailCount))
print "Delieve succeed------------------------------count=%d"%(mailCount)
mailCount+=1
g_logdict.pop(dkey)
def StatEveryLog(sourceRecord):
global g_forCtyProxydict
global g_Erroecodedict
global g_proxydict
t_table = ProformanceTable(sourceRecord)
if g_proxydict.has_key(sourceRecord.bindip):
temp = g_proxydict[sourceRecord.bindip]
print "t_table.time=%s,temp.timestamp=%s" % (
time.strftime("%Y-%m-%dT%H:%M:%S.000Z",
time.localtime(t_table.timestamp)),
time.strftime("%Y-%m-%dT%H:%M:%S.000Z",
time.localtime(temp.timestamp)))
if (t_table.timestamp - temp.timestamp < 60):
t_table.did = temp.did
t_table.timestamp = temp.timestamp
t_table.SumofDAFailure += temp.SumofDAFailure
t_table.SumofDASucceed += temp.SumofDASucceed
t_table.SumofDADeliver += temp.SumofDADeliver
t_table.FailrateofDeliver = 1.0 - float(
t_table.SumofDASucceed) / float(t_table.SumofDADeliver)
t_table.DAproxydelay = temp.DAproxydelay * temp.SumofDADeliver / t_table.SumofDADeliver
else:
pass
g_proxydict[sourceRecord.bindip] = t_table
Deliver2ESWithpath(t_table.Data2SixJson(), "newpreformance", "doc",
t_table.did)
if (g_Erroecodedict.has_key(t_table.ErrorCode)):
temp = g_Erroecodedict[t_table.ErrorCode]
if (t_table.timestamp1 - temp.timestamp1 < 60):
t_table.did1 = temp.did1
t_table.timestamp1 = temp.timestamp1
t_table.NumofCode += temp.NumofCode
if (t_table.ErrorCode != 250):
t_table.detail += temp.detail
else:
pass
g_Erroecodedict[t_table.ErrorCode] = t_table
Deliver2ESWithpath(t_table.Data2FiveJson(), "errorcode", "doc",
t_table.did1)
strsize = ""
if (sourceRecord.size < 10 * 1024 * 1024):
strsize = "under10M"
t_table.size = "under10M"
else:
strsize = "up10M"
t_table.size = "up10M"
if (g_forCtyProxydict.has_key(t_table.proxyCountry + t_table.Proxyip +
strsize)):
temp = g_forCtyProxydict[t_table.proxyCountry + t_table.Proxyip +
strsize]
if (t_table.timestamp2 - temp.timestamp2 < 60):
t_table.did2 = temp.did2
t_table.timestamp2 = temp.timestamp2
t_table.ForCtySumofproxysucceedDeliver += temp.ForCtySumofproxysucceedDeliver
t_table.ForCtySumofproxyDelv += temp.ForCtySumofproxyDelv
t_table.ForCtyproxyDeliverRate = 1.0 - float(
t_table.ForCtySumofproxysucceedDeliver) / float(
t_table.ForCtySumofproxyDelv)
t_table.Proxydelay = (temp.Proxydelay * temp.ForCtySumofproxyDelv +
t_table.Proxydelay) / (
t_table.ForCtySumofproxyDelv)
else:
pass
g_forCtyProxydict[t_table.proxyCountry + t_table.Proxyip +
strsize] = t_table
Deliver2ESWithpath(t_table.Data2SevenJson(), "preformancepercountry",
"doc", t_table.did2)
def StatmailInfo(sourceRecord):
global g_rejectrateid
structdata = structreject(sourceRecord)
if g_recipdomaindict.has_key(structdata.recipDomain):
#if structdata.timestamp-rejectdict[structdata.SenderDomain].ZeroTime>86400:# 如果过了12点重置数据，重新计算该域名对应的收件量，发件量
temp = g_recipdomaindict[structdata.recipDomain]
if (structdata.timestamp - temp.timestamp < 60):
structdata.sendid = temp.sendid
structdata.timestamp = temp.timestamp
#temp=g_recipdomaindict[structdata.recipDomain]
structdata.ForrecipSendmail += temp.ForrecipSendmail
structdata.ForrecipaveDelay = (
temp.ForrecipSendmail * temp.ForrecipaveDelay +
structdata.ForrecipThisdelay) / structdata.ForrecipSendmail
if (structdata.Forrecipmaxdelay < temp.Forrecipmaxdelay):
structdata.Forrecipmaxdelay = temp.Forrecipmaxdelay
structdata.Forrecipreturn += temp.Forrecipreturn
structdata.ForrecipRejectrate = float(
structdata.Forrecipreturn) / float(structdata.ForrecipSendmail)
else:
pass
else:
pass
g_recipdomaindict[structdata.recipDomain] = structdata
Deliver2ESWithpath(structdata.Data2Onejson(), "recipdomainstat", "doc",
structdata.sendid)
if (g_recipdomaindict.has_key("GetTrendobj")):
temp = g_recipdomaindict["GetTrendobj"]
if (structdata.timestamp3 - temp.timestamp3 < 60):
structdata.timestamp3 = temp.timestamp3
structdata.sendid3 = temp.sendid3
structdata.Sumofreturn += temp.Sumofreturn
structdata.SumofDeliver += temp.SumofDeliver
structdata.SumofSucceed += temp.SumofSucceed
structdata.SumofFailrate = 1.0 - float(temp.SumofSucceed) / float(
temp.SumofDeliver)
else:
pass
g_recipdomaindict["GetTrendobj"] = structdata
Deliver2ESWithpath(structdata.Data2Fourjson(), "mailtrendstat", "doc",
structdata.sendid3)
if g_SenderDomaindict.has_key(structdata.SenderDomain):
temp = g_SenderDomaindict[structdata.SenderDomain]
if (structdata.timestamp1 - temp.timestamp1 < 60):
structdata.sendid1 = temp.sendid1
structdata.timestamp1 = temp.timestamp1
#temp=g_SenderDomaindict[structdata.SenderDomain]
structdata.ForSenderDelvier += temp.ForSenderDelvier
if (structdata.ForSendermaxdelay < temp.ForSendermaxdelay):
structdata.ForSendermaxdelay = temp.ForSendermaxdelay
else:
pass
g_SenderDomaindict[structdata.SenderDomain] = structdata
Deliver2ESWithpath(structdata.Data2Twojson(), "senderdomainstat", "doc",
structdata.sendid1)
if (g_Countrydict.has_key(structdata.ToCountry)):
temp = g_Countrydict[structdata.ToCountry]
if (structdata.timestamp2 - temp.timestamp2 < 60):
structdata.sendid2 = temp.sendid2
structdata.timestamp2 = temp.timestamp2
structdata.ForCtyNumofdlv += temp.ForCtyNumofdlv
structdata.ForCtyreturn += temp.ForCtyreturn
structdata.ForCtyNumofsucceed += temp.ForCtyNumofsucceed
structdata.ForCtyFailrate = 1.0 - float(
structdata.ForCtyNumofsucceed) / float(
structdata.ForCtyNumofdlv)
else:
pass
g_Countrydict[structdata.ToCountry] = structdata
Deliver2ESWithpath(structdata.Data2Eightjson(), "sdndeliverstat", "doc",
structdata.sendid2)