def run(property_dict) : pid = os.getpid() name = str(property_dict["key"]) #设置log输出logid fastlog.set_logid(name + "_" + str(pid)) #信号处理函数 signal.signal(signal.SIGTERM, onsignal_term) #根据配置中定义的队列服务器地址和key,连接队列服务器 get_queue_data = getqueuedata.GetQueuData(property_dict["servers"], str(property_dict["key"])) ##my data prepare ##------------------------START----------------------------------- #configure message cf = ConfigParser.ConfigParser() cf.readfp(open('./conf/rinMethod.ini','r')) c = configData.configData(cf) #data parser parser = stdSampleParser.stdSampleParser() #init timeManager data t = timeManager.timeManager(int(c.longTimePeriod),int(c.shortTimePeriod)) now = time.time() t.setEndPoint(int(now)) effectMonitorInstance = effectMonitor.effectMonitor(c,t) ##------------------------END----------------------------------- #循环从队列中获取数据 get_null_count = 0 while True : #如果接收到kill信号,则会做如下退出处理 if break_flag : print "break while loop in " + name fastlog.info("break while loop in " + name) ##------------------------START----------------------------------- ##结束进程前的后续操作 ##------------------------END------------------------------------- break try : #从连接上的队列集群中,随机一台服务器中获取一条数据 #------------------------START----------------------------------- (data, server_addr) = get_queue_data.randomgetonedata() if data == None or data == "" : get_null_count += 1 if get_null_count >= 10 : get_null_count = 0 time.sleep(0.05) continue #fastlog.info("server: " + server_addr + " data: " + data) get_null_count = 0 #------------------------END------------------------------------- #------------------------START----------------------------------- #print data effectMonitorInstance.deal(parser.parse(data)) effectMonitorInstance.printStatsItemMsgToLog() parser.clear() #------------------------END------------------------------------- except Exception, e: fastlog.error("parser data Exception: " + str(e) + " data: " + str(data))
def test2(): import random import time import ConfigParser import timeManager import configData import stdSampleParser ##my data prepare ##------------------------START----------------------------------- #configure message cf = ConfigParser.ConfigParser() cf.readfp(open('./conf/rinMethod.ini','r')) c = configData.configData(cf) #data parser parser = stdSampleParser.stdSampleParser() #init timeManager data t = timeManager.timeManager(int(c.longTimePeriod),int(c.shortTimePeriod)) now = int(time.time()) t.setEndPoint(now) #key processer effectMonitorInstance = effectMonitor(c,t) intList = [-1,1] #candtypeList = ["c001","c002"] candtypeList = ["c001"] #businessList = ["b001","b002"] businessList = ["b001"] for i in range(1,200): #print ".................%d stats item data................"%(i) #print "------------------------START------------------------------" msg = ".................%d stats item data................"%(i) effectMonitorInstance.logger.debug(msg) effectMonitorInstance.logger.debug("------------------------START------------------------------") achieve = random.choice(intList) click = 1 if achieve == -1: click = random.choice(intList) createTime = random.randint(-4,10)+now now = createTime candtype = random.choice(candtypeList) business = random.choice(businessList) data = "expo_id:1234567|click:%d|achieve:%d|business_id:%s|candtype_id:%s|prim_temp_id:pt1|sec_temp_id:st1|expo_source:web|create_time:%d|timestamp:1435313246|weight:1"%(click,achieve,business,candtype,createTime) #print data effectMonitorInstance.logger.debug(data) data = parser.parse(data) effectMonitorInstance.deal(data) parser.clear() effectMonitorInstance.printStatsItemMsgToLog() effectMonitorInstance.logger.debug("------------------------END------------------------------\n")
def test1(): import ConfigParser import timeManager import configData import stdSampleParser ##my data prepare ##------------------------START----------------------------------- #configure message cf = ConfigParser.ConfigParser() cf.readfp(open('./conf/rinMethod.ini','r')) c = configData.configData(cf) #data parser parser = stdSampleParser.stdSampleParser() #init timeManager data t = timeManager.timeManager(int(c.longTimePeriod),int(c.shortTimePeriod)) t.setEndPoint(1435313248) print ".................first stats item data................" data = "expo_id:1234567|click:1|achieve:-1|business_id:userprofile|candtype_id:candtype1|prim_temp_id:pt1|sec_temp_id:st1|expo_source:web|create_time:1435313246|timestamp:1435313200|weight:1" #key processer data = parser.parse(data) effectMonitorInstance = effectMonitor(c,t) effectMonitorInstance.deal(data) print(data) parser.clear() effectMonitorInstance.printStatsItemMsg() print ".................second stats item data................" data = "expo_id:12345678|click:1|achieve:1|business_id:userprofile|candtype_id:candtype2|prim_temp_id:pt1|sec_temp_id:st2|expo_source:web|create_time:1435313230|timestamp:1435313200|weight:1" data = parser.parse(data) effectMonitorInstance.deal(data) print(data) effectMonitorInstance.printStatsItemMsg() parser.clear() print ".................third stats item data................" data = "expo_id:123456789|click:-1|achieve:-1|business_id:userprofile|candtype_id:candtype2|prim_temp_id:pt1|sec_temp_id:st2|expo_source:web|create_time:1435313480|timestamp:1435313200|weight:1" data = parser.parse(data) effectMonitorInstance.deal(data) print(data) effectMonitorInstance.printStatsItemMsg() parser.clear() print ".................forth stats item data................" data = "expo_id:123456789|click:-1|achieve:-1|business_id:userprofile|candtype_id:candtype2|prim_temp_id:pt1|sec_temp_id:st1|expo_source:web|create_time:1435313480|timestamp:1435313200|weight:1" data = parser.parse(data) effectMonitorInstance.deal(data) print(data) effectMonitorInstance.printStatsItemMsg() parser.clear() print ".................fifth stats item data................" data = "expo_id:123456789|click:1|achieve:-1|business_id:userprofile|candtype_id:candtype2|prim_temp_id:pt1|sec_temp_id:st1|expo_source:web|create_time:1435313000|timestamp:1435313200|weight:1" data = parser.parse(data) effectMonitorInstance.deal(data) print(data) effectMonitorInstance.printStatsItemMsg() parser.clear() print ".................sixth stats item data................" data = "expo_id:123456789|click:1|achieve:-1|business_id:userprofile|candtype_id:candtype3|prim_temp_id:pt3|sec_temp_id:st1|expo_source:web|create_time:1435312700|timestamp:1435313200|weight:1" data = parser.parse(data) effectMonitorInstance.deal(data) print(data) effectMonitorInstance.printStatsItemMsg() parser.clear() print effectMonitorInstance.parseKeysDic effectMonitorInstance.postDayStats()