def parse(self):
try:
self.userid = self.doc[4]
self.ts = self.doc[9]
self.mediaid = self.doc[10]
if self.version == "1.1":
self.retention = int(self.doc[14]) / 1000
self.vender = self.doc[12]
elif self.version == "1.0":
self.retention = int(self.doc[13]) / 1000
self.vender = self.doc[10]
else:
err('Log_5010_Entry::parse() invalid version %s' %
self.version)
return False
if int(self.retention) > 10800:
#if int(self.retention) > 86400:
# invalid retention, maybe the retention was set to ts
return False
except Exception, e:
import traceback
debug(traceback.format_exc())
debug("exception meet for Log_5011_Entry %s" % self.doc)
#print traceback.format_exc()
#print "exception meet for Log_5011_Entry %s"%self.doc
return False
def parse(self):
try:
self.userid = self.doc[4]
self.ts = self.doc[9]
self.mediaid = self.doc[10]
if self.version == "1.1":
self.retention = int(self.doc[14])/1000
self.vender = self.doc[12]
elif self.version == "1.0":
self.retention = int(self.doc[13])/1000
self.vender = self.doc[10]
else:
err('Log_5010_Entry::parse() invalid version %s'%self.version)
return False
if int(self.retention) > 10800:
#if int(self.retention) > 86400:
# invalid retention, maybe the retention was set to ts
return False
except Exception, e:
import traceback
debug(traceback.format_exc())
debug("exception meet for Log_5011_Entry %s"%self.doc)
#print traceback.format_exc()
#print "exception meet for Log_5011_Entry %s"%self.doc
return False
def _start_workers(self):
debug("GrabberManager::_start_worker() starting workers")
for worker, data in self.workers_data.items():
wc = data['cl'](self.msgh, self.queue, worker, data['config'])
wc.attach_calcs(self.calc_names)
wc.start()
if wc.is_alive():
data['status'] = True
def _start_workers(self):
debug("GrabberManager::_start_worker() starting workers")
for worker, data in self.workers_data.items():
wc = data['cl'](self.msgh, self.queue, worker, data['config'])
wc.attach_calcs(self.calc_names)
wc.start()
if wc.is_alive():
data['status'] = True
def parse(self):
try:
self.userid = self.doc[4]
self.ts = self.doc[9].strip()
except Exception, e:
import traceback
debug(traceback.format_exc())
debug("exception meet for Log_5001_Entry %s" % self.doc)
return False
def parse(self):
try:
self.userid = self.doc[4]
self.ts = self.doc[9].strip()
except Exception, e:
import traceback
debug(traceback.format_exc())
debug("exception meet for Log_5001_Entry %s"%self.doc)
return False
def parse(self):
try:
self.userid = self.doc[4]
self.ts = self.doc[9]
self.typecode = self.doc[10]
except Exception, e:
import traceback
debug(traceback.format_exc())
debug("exception meet for Log_5040_Entry %s" % self.doc)
return False
def parse(self):
try:
self.userid = self.doc[4]
self.ts = self.doc[9]
self.typecode = self.doc[10]
except Exception, e:
import traceback
debug(traceback.format_exc())
debug("exception meet for Log_5040_Entry %s"%self.doc)
return False
def notice_grabber_start(self):
debug("CalcManage::notice_grabber_start()")
cs_msg = MsgCalcStart()
for qname, data in self.workers_data.items():
if data['status'] is False:
continue
cs_msg.addQueue(qname)
qid = self.msgh.findQueue("GrabberManager")
grabber_mgr_q = self.msgh.getQueue(qid)
self.queue.send(grabber_mgr_q, cs_msg)
def _init_timers(self):
debug("GrabberWorker [%s] _init_timers()"%os.getpid())
self.eh.register_timer(self.report_intvl * 1000,
TMtagsComm.REPORT_STATUS,
True)
self.eh.register_timer(self.alive_chk_intvl * 1000,
TMtagsComm.GRABBER_ALIVE,
True)
self.eh.register_timer(self.rate_intvl * 1000,
TMtagsComm.PROCESS_RATE,
True)
def notice_grabber_start(self):
debug("CalcManage::notice_grabber_start()")
cs_msg = MsgCalcStart()
for qname, data in self.workers_data.items():
if data['status'] is False:
continue
cs_msg.addQueue(qname)
qid = self.msgh.findQueue("GrabberManager")
grabber_mgr_q = self.msgh.getQueue(qid)
self.queue.send(grabber_mgr_q, cs_msg)
def _process(self):
self._init_timers()
while self.finish is False:
msg = self.eh.getEvent()
if msg is None:
debug("no message not, continue")
evtype = msg.get_eventype()
if evtype == EVENTYPE.TIMEREXPIRE:
self._process_timer(msg)
elif evtype == EVENTYPE.NORMALMSG:
self._process_msg(msg)
def _process_msg(self, msg):
debug("PlayUserCalcMgr::_process_msg() enter")
msgtype = msg.get_msgtype()
if msgtype == MsgType.MsgPlayUserReport:
debug("PlayUserCalcMgr::_process_msg() got "
"report message %s" % msg.get_body())
ptr_msg = MsgPlayUserReport()
ptr_msg.cast(msg)
data = json.loads(ptr_msg.get_report_info())
for user, play_info in data.items():
self.stat_user.merge_user_info(user, play_info)
else:
super(PlayUserCalcMgr, self)._process_msg(msg)
def _process_msg(self, msg):
debug("TopMediasCalcMgr::_process_msg() enter")
msgtype = msg.get_msgtype()
if msgtype == MsgType.MsgTopMediasReport:
debug("TopMediasCalcMgr::_process_msg() got "
"report message %s"%msg.get_body())
ptr_msg = MsgTopMediasReport()
ptr_msg.cast(msg)
data = json.loads(ptr_msg.get_report_info())
for user, play_info in data.items():
self.stat_user.merge_user_info(user, play_info)
else:
super(TopMediasCalcMgr, self)._process_msg(msg)
def _grab(self):
debug("GrabberWorker [%s] _grab() start"%os.getpid())
self.count = 70
while True:
self.current += 1
time.sleep(0.1)
if self.current > self.count:
break
tmsg = MsgTest()
tmsg.set_text("a test message send from %s at %s"%\
(self.msgh_name, str(datetime.datetime.now())))
i = self.next_rrobin()
self.queue.send(self.calc_queues[i], tmsg)
debug("GrabberWorker [%s] _grab() finish"%os.getpid())
def parse(self):
try:
self.userid = self.doc[4]
self.ts = self.doc[-5]
if self.version == '1.0':
self.vender = self.doc[-4]
elif self.version == "1.1":
self.vender = self.doc[-2]
else:
err('Log_5010_Entry::parse() invalid version %s'%self.version)
return False
except Exception, e:
import traceback
debug(traceback.format_exc())
debug("exception meet for Log_5010_Entry %s"%self.doc)
return False
def _process_msg(self, msg):
debug("AccessFreqCalcMgr::_process_msg() enter")
msgtype = msg.get_msgtype()
#print "--------------process msg", datetime.datetime.now()
if msgtype == MsgType.MsgAccessFreqReport:
debug("AccessFreqCalcMgr::_process_msg() got "
"report message %s"%msg.get_body())
ptr_msg = MsgAccessFreqReport()
ptr_msg.cast(msg)
ptr_msg.parse()
#print "------------", ptr_msg.body
info = ptr_msg.get_freq_info()
for ts, count in info:
self.mark(ts, int(count))
else:
super(AccessFreqCalcMgr, self)._process_msg(msg)
def _process_msg(self, msg):
debug("AccessFreqCalcMgr::_process_msg() enter")
msgtype = msg.get_msgtype()
#print "--------------process msg", datetime.datetime.now()
if msgtype == MsgType.MsgAccessFreqReport:
debug("AccessFreqCalcMgr::_process_msg() got "
"report message %s"%msg.get_body())
ptr_msg = MsgAccessFreqReport()
ptr_msg.cast(msg)
ptr_msg.parse()
#print "------------", ptr_msg.body
info = ptr_msg.get_freq_info()
for ts, count in info:
self.mark(ts, int(count))
else:
super(AccessFreqCalcMgr, self)._process_msg(msg)
def parse(self):
try:
self.userid = self.doc[4]
self.ts = self.doc[-5]
if self.version == '1.0':
self.vender = self.doc[-4]
elif self.version == "1.1":
self.vender = self.doc[-2]
else:
err('Log_5010_Entry::parse() invalid version %s'%self.version)
return False
except Exception, e:
import traceback
debug(traceback.format_exc())
debug("exception meet for Log_5010_Entry %s"%self.doc)
return False
def _process_msg(self, msg):
debug("PlayTimeCalcMgr::_process_msg() enter")
msgtype = msg.get_msgtype()
if msgtype == MsgType.MsgPlayTimeReport:
debug("PlayTimeCalcMgr::_process_msg() got "
"report message %s" % msg.get_body())
#print "PlayerStartupCalcMgr::_process_msg() got "\
# "report message %s"%msg.get_body()
ptr_msg = MsgPlayTimeReport()
ptr_msg.cast(msg)
data = json.loads(ptr_msg.get_report_info())
for user, play_info in data.items():
self.stat_user.merge_user_info(user, play_info)
#for vender, counts in data.items():
# self.collectors[vender].stat_time.merge(counts)
else:
super(PlayTimeCalcMgr, self)._process_msg(msg)
def _process_msg(self, msg):
debug("PlayTimeCalcMgr::_process_msg() enter")
msgtype = msg.get_msgtype()
if msgtype == MsgType.MsgPlayTimeReport:
debug("PlayTimeCalcMgr::_process_msg() got "
"report message %s"%msg.get_body())
#print "PlayerStartupCalcMgr::_process_msg() got "\
# "report message %s"%msg.get_body()
ptr_msg = MsgPlayTimeReport()
ptr_msg.cast(msg)
data = json.loads(ptr_msg.get_report_info())
for user, play_info in data.items():
self.stat_user.merge_user_info(user, play_info)
#for vender, counts in data.items():
# self.collectors[vender].stat_time.merge(counts)
else:
super(PlayTimeCalcMgr, self)._process_msg(msg)
def _process(self):
debug("GraberWorker::_process() enter")
self._init_timers()
self.thr = GrabberWorker.GrabberThread(
self.msgh_name+"_thread", self._grab)
self.thr.start()
while self.finish is False:
msg = self.eh.getEvent()
if msg is None:
debug("no message got, continue")
continue
evtype = msg.get_eventype()
if evtype == EVENTYPE.TIMEREXPIRE:
self._process_timer(msg)
elif evtype == EVENTYPE.NORMALMSG:
self._process_msg(msg)
def _process_msg(self, msg):
debug("AccessUserCalcMgr::_process_msg() enter")
msgtype = msg.get_msgtype()
#print "--------------process msg", datetime.datetime.now()
if msgtype == MsgType.MsgAccessUserReport:
debug("AccessUserCalcMgr::_process_msg() got "
"report message %s"%msg.get_body())
ptr_msg = MsgAccessUserReport()
ptr_msg.cast(msg)
ptr_msg.parse()
#print ptr_msg.get_users_info()
for user, ts in ptr_msg.get_users_info():
entry = self.dbsession.select({'user': user})
if entry is None:
self.dbsession.insert({'user': user, 'timestamp': ts})
else:
#print "entry----", entry
exist_ts = entry[1][1]
if ts < exist_ts:
#print "ts < exists", ts, exist_ts
self.dbsession.update({'user':user, 'ts': ts})
else:
super(AccessUserCalcMgr, self)._process_msg(msg)
def _process_msg(self, msg):
debug("AccessUserCalcMgr::_process_msg() enter")
msgtype = msg.get_msgtype()
#print "--------------process msg", datetime.datetime.now()
if msgtype == MsgType.MsgAccessUserReport:
debug("AccessUserCalcMgr::_process_msg() got "
"report message %s" % msg.get_body())
ptr_msg = MsgAccessUserReport()
ptr_msg.cast(msg)
ptr_msg.parse()
#print ptr_msg.get_users_info()
for user, ts in ptr_msg.get_users_info():
entry = self.dbsession.select({'user': user})
if entry is None:
self.dbsession.insert({'user': user, 'timestamp': ts})
else:
#print "entry----", entry
exist_ts = entry[1][1]
if ts < exist_ts:
#print "ts < exists", ts, exist_ts
self.dbsession.update({'user': user, 'ts': ts})
else:
super(AccessUserCalcMgr, self)._process_msg(msg)
def _process_msg(self, msg):
debug("GrabberWorker [%s] _process_msg"%os.getpid())
def _final(self):
debug("GrabberWorker [%s] _final() "%os.getpid())
self._report_status()
wq_msg = MsgWorkerQuit()
self.queue.send(self.mgrq, wq_msg)