def run(property_dict) :
pid = os.getpid()
name = str(property_dict["key"])
#设置log输出logid
fastlog.set_logid(name + "_" + str(pid))
#信号处理函数
signal.signal(signal.SIGTERM, onsignal_term)
#根据配置中定义的队列服务器地址和key，连接队列服务器
get_queue_data = getqueuedata.GetQueuData(property_dict["servers"], str(property_dict["key"]))
##my data prepare
##------------------------START-----------------------------------
#configure message
cf = ConfigParser.ConfigParser()
cf.readfp(open('./conf/rinMethod.ini','r'))
c = configData.configData(cf)
#data parser
parser = stdSampleParser.stdSampleParser()
#init timeManager data
t = timeManager.timeManager(int(c.longTimePeriod),int(c.shortTimePeriod))
now = time.time()
t.setEndPoint(int(now))
effectMonitorInstance = effectMonitor.effectMonitor(c,t)
##------------------------END-----------------------------------
#循环从队列中获取数据
get_null_count = 0
while True :
#如果接收到kill信号，则会做如下退出处理
if break_flag :
print "break while loop in " + name
fastlog.info("break while loop in " + name)
##------------------------START-----------------------------------
##结束进程前的后续操作
##------------------------END-------------------------------------
break
try :
#从连接上的队列集群中，随机一台服务器中获取一条数据
#------------------------START-----------------------------------
(data, server_addr) = get_queue_data.randomgetonedata()
if data == None or data == "" :
get_null_count += 1
if get_null_count >= 10 :
get_null_count = 0
time.sleep(0.05)
continue
#fastlog.info("server: " + server_addr + " data: " + data)
get_null_count = 0
#------------------------END-------------------------------------
#------------------------START-----------------------------------
#print data
effectMonitorInstance.deal(parser.parse(data))
effectMonitorInstance.printStatsItemMsgToLog()
parser.clear()
#------------------------END-------------------------------------
except Exception, e:
fastlog.error("parser data Exception: " + str(e) + " data: " + str(data))
def test2():
import random
import time
import ConfigParser
import timeManager
import configData
import stdSampleParser
##my data prepare
##------------------------START-----------------------------------
#configure message
cf = ConfigParser.ConfigParser()
cf.readfp(open('./conf/rinMethod.ini','r'))
c = configData.configData(cf)
#data parser
parser = stdSampleParser.stdSampleParser()
#init timeManager data
t = timeManager.timeManager(int(c.longTimePeriod),int(c.shortTimePeriod))
now = int(time.time())
t.setEndPoint(now)
#key processer
effectMonitorInstance = effectMonitor(c,t)
intList = [-1,1]
#candtypeList = ["c001","c002"]
candtypeList = ["c001"]
#businessList = ["b001","b002"]
businessList = ["b001"]
for i in range(1,200):
#print ".................%d stats item data................"%(i)
#print "------------------------START------------------------------"
msg = ".................%d stats item data................"%(i)
effectMonitorInstance.logger.debug(msg)
effectMonitorInstance.logger.debug("------------------------START------------------------------")
achieve = random.choice(intList)
click = 1
if achieve == -1:
click = random.choice(intList)
createTime = random.randint(-4,10)+now
now = createTime
candtype = random.choice(candtypeList)
business = random.choice(businessList)
data = "expo_id:1234567|click:%d|achieve:%d|business_id:%s|candtype_id:%s|prim_temp_id:pt1|sec_temp_id:st1|expo_source:web|create_time:%d|timestamp:1435313246|weight:1"%(click,achieve,business,candtype,createTime)
#print data
effectMonitorInstance.logger.debug(data)
data = parser.parse(data)
effectMonitorInstance.deal(data)
parser.clear()
effectMonitorInstance.printStatsItemMsgToLog()
effectMonitorInstance.logger.debug("------------------------END------------------------------\n")
def test2():
import random
import time
import ConfigParser
import timeManager
import configData
import stdServLogParser
##my data prepare
##------------------------START-----------------------------------
#configure message
cf = ConfigParser.ConfigParser()
cf.readfp(open('./conf/rinMethod.ini','r'))
c = configData.configData(cf)
#data parser
parser = stdServLogParser.stdServLogParser(c)
#init timeManager data
now = int(time.time())
t = timeManager.timeManager(int(c.timePeriod),now)
#key processer
effectMonitorInstance = effectMonitor(c,t)
logTypeList = ["ERROR","INFO"]
retCodeList = [-1,0,1,2,3]
serverNodeNameList = ["10.33.96.33","10.33.96.32"]
for i in range(1,200):
msg = ".................%d stats item data................"%(i)
effectMonitorInstance.logger.debug(msg)
effectMonitorInstance.logger.debug("------------------------START------------------------------")
createTime = random.randint(-4,10)+now
logType = random.choice(logTypeList)
retCode = random.choice(retCodeList)
serverNodeName = random.choice(serverNodeNameList)
stdTimeStr = time.strftime("%Y-%m-%d %H:%M:%S",time.localtime(createTime))
data = "%s [%s.657] [logid:0] [tcpserver.cpp:152] | REQ_ID:10758252990973006192;SERV:Ranker;CMD:rank;RET_CODE:%d;FEAT_ROUTE:b_id_01->ct_id_01->pt_id_01->st_id_03;COST:3449(9-3421-3-13-1-2);EXT3:%s;"%(logType,stdTimeStr,retCode,serverNodeName)
effectMonitorInstance.logger.debug(data)
data = parser.parse(data)
effectMonitorInstance.deal(data)
parser.clear()
effectMonitorInstance.printStatsItemMsgToLog()
effectMonitorInstance.logger.debug("------------------------END------------------------------\n")
def test1():
import ConfigParser
import timeManager
import configData
import stdSampleParser
##my data prepare
##------------------------START-----------------------------------
#configure message
cf = ConfigParser.ConfigParser()
cf.readfp(open('./conf/rinMethod.ini','r'))
c = configData.configData(cf)
#data parser
parser = stdSampleParser.stdSampleParser()
#init timeManager data
t = timeManager.timeManager(int(c.longTimePeriod),int(c.shortTimePeriod))
t.setEndPoint(1435313248)
print ".................first stats item data................"
data = "expo_id:1234567|click:1|achieve:-1|business_id:userprofile|candtype_id:candtype1|prim_temp_id:pt1|sec_temp_id:st1|expo_source:web|create_time:1435313246|timestamp:1435313200|weight:1"
#key processer
data = parser.parse(data)
effectMonitorInstance = effectMonitor(c,t)
effectMonitorInstance.deal(data)
print(data)
parser.clear()
effectMonitorInstance.printStatsItemMsg()
print ".................second stats item data................"
data = "expo_id:12345678|click:1|achieve:1|business_id:userprofile|candtype_id:candtype2|prim_temp_id:pt1|sec_temp_id:st2|expo_source:web|create_time:1435313230|timestamp:1435313200|weight:1"
data = parser.parse(data)
effectMonitorInstance.deal(data)
print(data)
effectMonitorInstance.printStatsItemMsg()
parser.clear()
print ".................third stats item data................"
data = "expo_id:123456789|click:-1|achieve:-1|business_id:userprofile|candtype_id:candtype2|prim_temp_id:pt1|sec_temp_id:st2|expo_source:web|create_time:1435313480|timestamp:1435313200|weight:1"
data = parser.parse(data)
effectMonitorInstance.deal(data)
print(data)
effectMonitorInstance.printStatsItemMsg()
parser.clear()
print ".................forth stats item data................"
data = "expo_id:123456789|click:-1|achieve:-1|business_id:userprofile|candtype_id:candtype2|prim_temp_id:pt1|sec_temp_id:st1|expo_source:web|create_time:1435313480|timestamp:1435313200|weight:1"
data = parser.parse(data)
effectMonitorInstance.deal(data)
print(data)
effectMonitorInstance.printStatsItemMsg()
parser.clear()
print ".................fifth stats item data................"
data = "expo_id:123456789|click:1|achieve:-1|business_id:userprofile|candtype_id:candtype2|prim_temp_id:pt1|sec_temp_id:st1|expo_source:web|create_time:1435313000|timestamp:1435313200|weight:1"
data = parser.parse(data)
effectMonitorInstance.deal(data)
print(data)
effectMonitorInstance.printStatsItemMsg()
parser.clear()
print ".................sixth stats item data................"
data = "expo_id:123456789|click:1|achieve:-1|business_id:userprofile|candtype_id:candtype3|prim_temp_id:pt3|sec_temp_id:st1|expo_source:web|create_time:1435312700|timestamp:1435313200|weight:1"
data = parser.parse(data)
effectMonitorInstance.deal(data)
print(data)
effectMonitorInstance.printStatsItemMsg()
parser.clear()
print effectMonitorInstance.parseKeysDic
effectMonitorInstance.postDayStats()
dataNodeList = i.split(":")
temp_dic_data[dataNodeList[0]] = dataNodeList[-1]
costTimeKey = temp_dic_data.get(self.configData.costTimeKey,None)
if costTimeKey != None:
temp_dic_data[self.configData.costTimeKey] = costTimeKey[:costTimeKey.find("(")]
self.dic_data = temp_dic_data
return temp_dic_data
#def get(self,=""):
#return self.dic_data.get(,None)
def getDicData(self):
return self.dic_data
def clear(self):
self.dic_data.clear()
if __name__ == "__main__":
import configData
import ConfigParser
cf = ConfigParser.ConfigParser()
cf.readfp(open('./conf/rinMethod.ini','r'))
c = configData.configData(cf)
s = stdServLogParser(c)
data = "WARN [2015-05-03 11:13:46.998] [logid:0] [src/userrecomtest_work_interface.cpp:247] | REQ_ID:10758252990061920973;SERV:RecomServ;CMD:recom;RET_CODE:0;FEAT_ROUTE:b_id_01->ct_id_01->pt_id_01->st_id_03;COST:3449(9-3421-3-13-1-2);"
print s.parse(data)
data = "ERROR [2015-04-01 15:34:30.657] [logid:0] [tcpserver.cpp:152] | REQ_ID:10758252990973006192;SERV:Ranker;CMD:rank;RET_CODE:-3;"
print s.parse(data)
data = "INFO [2015-05-03 11:13:46.998] [logid:0] [src/userrecomtest_work_interface.cpp:312] | REQ_ID:10751920973825299006;SERV:RecomServ;CMD:recom;RET_CODE:24;FEAT_ROUTE:b_id_01->ct_id_01->pt_id_01->st_id_02;COST:37299(1154-2493-9257-3414-19115-1866);"
print s.parse(data)