def parse(self): try: self.userid = self.doc[4] self.ts = self.doc[9] self.mediaid = self.doc[10] if self.version == "1.1": self.retention = int(self.doc[14]) / 1000 self.vender = self.doc[12] elif self.version == "1.0": self.retention = int(self.doc[13]) / 1000 self.vender = self.doc[10] else: err('Log_5010_Entry::parse() invalid version %s' % self.version) return False if int(self.retention) > 10800: #if int(self.retention) > 86400: # invalid retention, maybe the retention was set to ts return False except Exception, e: import traceback debug(traceback.format_exc()) debug("exception meet for Log_5011_Entry %s" % self.doc) #print traceback.format_exc() #print "exception meet for Log_5011_Entry %s"%self.doc return False
def parse(self): try: self.userid = self.doc[4] self.ts = self.doc[9] self.mediaid = self.doc[10] if self.version == "1.1": self.retention = int(self.doc[14])/1000 self.vender = self.doc[12] elif self.version == "1.0": self.retention = int(self.doc[13])/1000 self.vender = self.doc[10] else: err('Log_5010_Entry::parse() invalid version %s'%self.version) return False if int(self.retention) > 10800: #if int(self.retention) > 86400: # invalid retention, maybe the retention was set to ts return False except Exception, e: import traceback debug(traceback.format_exc()) debug("exception meet for Log_5011_Entry %s"%self.doc) #print traceback.format_exc() #print "exception meet for Log_5011_Entry %s"%self.doc return False
def _start_workers(self): debug("GrabberManager::_start_worker() starting workers") for worker, data in self.workers_data.items(): wc = data['cl'](self.msgh, self.queue, worker, data['config']) wc.attach_calcs(self.calc_names) wc.start() if wc.is_alive(): data['status'] = True
def _start_workers(self): debug("GrabberManager::_start_worker() starting workers") for worker, data in self.workers_data.items(): wc = data['cl'](self.msgh, self.queue, worker, data['config']) wc.attach_calcs(self.calc_names) wc.start() if wc.is_alive(): data['status'] = True
def parse(self): try: self.userid = self.doc[4] self.ts = self.doc[9].strip() except Exception, e: import traceback debug(traceback.format_exc()) debug("exception meet for Log_5001_Entry %s" % self.doc) return False
def parse(self): try: self.userid = self.doc[4] self.ts = self.doc[9].strip() except Exception, e: import traceback debug(traceback.format_exc()) debug("exception meet for Log_5001_Entry %s"%self.doc) return False
def parse(self): try: self.userid = self.doc[4] self.ts = self.doc[9] self.typecode = self.doc[10] except Exception, e: import traceback debug(traceback.format_exc()) debug("exception meet for Log_5040_Entry %s" % self.doc) return False
def parse(self): try: self.userid = self.doc[4] self.ts = self.doc[9] self.typecode = self.doc[10] except Exception, e: import traceback debug(traceback.format_exc()) debug("exception meet for Log_5040_Entry %s"%self.doc) return False
def notice_grabber_start(self): debug("CalcManage::notice_grabber_start()") cs_msg = MsgCalcStart() for qname, data in self.workers_data.items(): if data['status'] is False: continue cs_msg.addQueue(qname) qid = self.msgh.findQueue("GrabberManager") grabber_mgr_q = self.msgh.getQueue(qid) self.queue.send(grabber_mgr_q, cs_msg)
def _init_timers(self): debug("GrabberWorker [%s] _init_timers()"%os.getpid()) self.eh.register_timer(self.report_intvl * 1000, TMtagsComm.REPORT_STATUS, True) self.eh.register_timer(self.alive_chk_intvl * 1000, TMtagsComm.GRABBER_ALIVE, True) self.eh.register_timer(self.rate_intvl * 1000, TMtagsComm.PROCESS_RATE, True)
def notice_grabber_start(self): debug("CalcManage::notice_grabber_start()") cs_msg = MsgCalcStart() for qname, data in self.workers_data.items(): if data['status'] is False: continue cs_msg.addQueue(qname) qid = self.msgh.findQueue("GrabberManager") grabber_mgr_q = self.msgh.getQueue(qid) self.queue.send(grabber_mgr_q, cs_msg)
def _process(self): self._init_timers() while self.finish is False: msg = self.eh.getEvent() if msg is None: debug("no message not, continue") evtype = msg.get_eventype() if evtype == EVENTYPE.TIMEREXPIRE: self._process_timer(msg) elif evtype == EVENTYPE.NORMALMSG: self._process_msg(msg)
def _process_msg(self, msg): debug("PlayUserCalcMgr::_process_msg() enter") msgtype = msg.get_msgtype() if msgtype == MsgType.MsgPlayUserReport: debug("PlayUserCalcMgr::_process_msg() got " "report message %s" % msg.get_body()) ptr_msg = MsgPlayUserReport() ptr_msg.cast(msg) data = json.loads(ptr_msg.get_report_info()) for user, play_info in data.items(): self.stat_user.merge_user_info(user, play_info) else: super(PlayUserCalcMgr, self)._process_msg(msg)
def _process_msg(self, msg): debug("TopMediasCalcMgr::_process_msg() enter") msgtype = msg.get_msgtype() if msgtype == MsgType.MsgTopMediasReport: debug("TopMediasCalcMgr::_process_msg() got " "report message %s"%msg.get_body()) ptr_msg = MsgTopMediasReport() ptr_msg.cast(msg) data = json.loads(ptr_msg.get_report_info()) for user, play_info in data.items(): self.stat_user.merge_user_info(user, play_info) else: super(TopMediasCalcMgr, self)._process_msg(msg)
def _grab(self): debug("GrabberWorker [%s] _grab() start"%os.getpid()) self.count = 70 while True: self.current += 1 time.sleep(0.1) if self.current > self.count: break tmsg = MsgTest() tmsg.set_text("a test message send from %s at %s"%\ (self.msgh_name, str(datetime.datetime.now()))) i = self.next_rrobin() self.queue.send(self.calc_queues[i], tmsg) debug("GrabberWorker [%s] _grab() finish"%os.getpid())
def parse(self): try: self.userid = self.doc[4] self.ts = self.doc[-5] if self.version == '1.0': self.vender = self.doc[-4] elif self.version == "1.1": self.vender = self.doc[-2] else: err('Log_5010_Entry::parse() invalid version %s'%self.version) return False except Exception, e: import traceback debug(traceback.format_exc()) debug("exception meet for Log_5010_Entry %s"%self.doc) return False
def _process_msg(self, msg): debug("AccessFreqCalcMgr::_process_msg() enter") msgtype = msg.get_msgtype() #print "--------------process msg", datetime.datetime.now() if msgtype == MsgType.MsgAccessFreqReport: debug("AccessFreqCalcMgr::_process_msg() got " "report message %s"%msg.get_body()) ptr_msg = MsgAccessFreqReport() ptr_msg.cast(msg) ptr_msg.parse() #print "------------", ptr_msg.body info = ptr_msg.get_freq_info() for ts, count in info: self.mark(ts, int(count)) else: super(AccessFreqCalcMgr, self)._process_msg(msg)
def _process_msg(self, msg): debug("AccessFreqCalcMgr::_process_msg() enter") msgtype = msg.get_msgtype() #print "--------------process msg", datetime.datetime.now() if msgtype == MsgType.MsgAccessFreqReport: debug("AccessFreqCalcMgr::_process_msg() got " "report message %s"%msg.get_body()) ptr_msg = MsgAccessFreqReport() ptr_msg.cast(msg) ptr_msg.parse() #print "------------", ptr_msg.body info = ptr_msg.get_freq_info() for ts, count in info: self.mark(ts, int(count)) else: super(AccessFreqCalcMgr, self)._process_msg(msg)
def parse(self): try: self.userid = self.doc[4] self.ts = self.doc[-5] if self.version == '1.0': self.vender = self.doc[-4] elif self.version == "1.1": self.vender = self.doc[-2] else: err('Log_5010_Entry::parse() invalid version %s'%self.version) return False except Exception, e: import traceback debug(traceback.format_exc()) debug("exception meet for Log_5010_Entry %s"%self.doc) return False
def _process_msg(self, msg): debug("PlayTimeCalcMgr::_process_msg() enter") msgtype = msg.get_msgtype() if msgtype == MsgType.MsgPlayTimeReport: debug("PlayTimeCalcMgr::_process_msg() got " "report message %s" % msg.get_body()) #print "PlayerStartupCalcMgr::_process_msg() got "\ # "report message %s"%msg.get_body() ptr_msg = MsgPlayTimeReport() ptr_msg.cast(msg) data = json.loads(ptr_msg.get_report_info()) for user, play_info in data.items(): self.stat_user.merge_user_info(user, play_info) #for vender, counts in data.items(): # self.collectors[vender].stat_time.merge(counts) else: super(PlayTimeCalcMgr, self)._process_msg(msg)
def _process_msg(self, msg): debug("PlayTimeCalcMgr::_process_msg() enter") msgtype = msg.get_msgtype() if msgtype == MsgType.MsgPlayTimeReport: debug("PlayTimeCalcMgr::_process_msg() got " "report message %s"%msg.get_body()) #print "PlayerStartupCalcMgr::_process_msg() got "\ # "report message %s"%msg.get_body() ptr_msg = MsgPlayTimeReport() ptr_msg.cast(msg) data = json.loads(ptr_msg.get_report_info()) for user, play_info in data.items(): self.stat_user.merge_user_info(user, play_info) #for vender, counts in data.items(): # self.collectors[vender].stat_time.merge(counts) else: super(PlayTimeCalcMgr, self)._process_msg(msg)
def _process(self): debug("GraberWorker::_process() enter") self._init_timers() self.thr = GrabberWorker.GrabberThread( self.msgh_name+"_thread", self._grab) self.thr.start() while self.finish is False: msg = self.eh.getEvent() if msg is None: debug("no message got, continue") continue evtype = msg.get_eventype() if evtype == EVENTYPE.TIMEREXPIRE: self._process_timer(msg) elif evtype == EVENTYPE.NORMALMSG: self._process_msg(msg)
def _process_msg(self, msg): debug("AccessUserCalcMgr::_process_msg() enter") msgtype = msg.get_msgtype() #print "--------------process msg", datetime.datetime.now() if msgtype == MsgType.MsgAccessUserReport: debug("AccessUserCalcMgr::_process_msg() got " "report message %s"%msg.get_body()) ptr_msg = MsgAccessUserReport() ptr_msg.cast(msg) ptr_msg.parse() #print ptr_msg.get_users_info() for user, ts in ptr_msg.get_users_info(): entry = self.dbsession.select({'user': user}) if entry is None: self.dbsession.insert({'user': user, 'timestamp': ts}) else: #print "entry----", entry exist_ts = entry[1][1] if ts < exist_ts: #print "ts < exists", ts, exist_ts self.dbsession.update({'user':user, 'ts': ts}) else: super(AccessUserCalcMgr, self)._process_msg(msg)
def _process_msg(self, msg): debug("AccessUserCalcMgr::_process_msg() enter") msgtype = msg.get_msgtype() #print "--------------process msg", datetime.datetime.now() if msgtype == MsgType.MsgAccessUserReport: debug("AccessUserCalcMgr::_process_msg() got " "report message %s" % msg.get_body()) ptr_msg = MsgAccessUserReport() ptr_msg.cast(msg) ptr_msg.parse() #print ptr_msg.get_users_info() for user, ts in ptr_msg.get_users_info(): entry = self.dbsession.select({'user': user}) if entry is None: self.dbsession.insert({'user': user, 'timestamp': ts}) else: #print "entry----", entry exist_ts = entry[1][1] if ts < exist_ts: #print "ts < exists", ts, exist_ts self.dbsession.update({'user': user, 'ts': ts}) else: super(AccessUserCalcMgr, self)._process_msg(msg)
def _process_msg(self, msg): debug("GrabberWorker [%s] _process_msg"%os.getpid())
def _final(self): debug("GrabberWorker [%s] _final() "%os.getpid()) self._report_status() wq_msg = MsgWorkerQuit() self.queue.send(self.mgrq, wq_msg)