def __init__(self, msgh, config):
CalcManager.__init__(self, msgh)
self.report_fd = None
self.config = config
self.stat_user = StatUser(config['start'], config['end'])
self.rpt_print_intvl = 40
self.category_count = {1001: 0, # play
1002: 0, # zongyi
1003: 0,
1004: 0, # movie
1005: 0, # catoon
1006: 0,
1007: 0, # sport
1008: 0,
1009: 0,
1010: 0,
1011: 0,
1012: 0, # child
1013: 0,
1014: 0,
1015: 0,
1016: 0,
1017: 0,
1100: 0
}
self.country_count = {}
#self.eh.register_timer(self.rpt_print_intvl * 1000,
# TMTAGS.PRINT_REPORT, True)
self.db = Mydb()
self.db.connect('report')
def __init__(self, msgh, config):
CalcManager.__init__(self, msgh)
self.report_fd = None
self.collectors = {}
self.config = config
self.stat_user = StatUser(config['start'], config['end'])
self.rpt_print_intvl = 40
#self.eh.register_timer(self.rpt_print_intvl * 1000,
# TMTAGS.PRINT_REPORT, True)
self.db = Mydb()
self.db.connect('report')
def __init__(self, msgh, config):
CalcManager.__init__(self, msgh)
self.report_fd = None
self.collectors = {}
self.config = config
self.db = Mydb()
self.db.connect('report')
dbname = "freq_trends_" + dbname_dict[api]
self.dbsession = self.db.open(dbname)
self.data = {}
self.trends_data = {}
self.top_data = {}
def _print_report(self):
echo("going to generate report file")
echo("generating the count data for every seconds")
db = Mydb()
db.connect('report')
dbname = "freq_trends_" + dbname_dict[api]
dbsession = db.open(dbname)
d = {'timestamp': "", 'count': ''}
for ts, count in self.data.items():
d['timestamp'] = ts
d['count'] = count
self._mark_top(ts, count)
dbsession.insert(d)
dbsession.commit()
dbsession.close()
echo("generating the top count data for earch day")
db.connect('report')
dbname = "freq_tops_" + dbname_dict[api]
dbsession = db.open(dbname)
for ts, count in self.top_data.items():
d['timestamp'] = ts
d['count'] = count
dbsession.insert(d)
dbsession.commit()
dbsession.close()
def __init__(self, msgh, config):
CalcManager.__init__(self, msgh)
self.report_fd = None
self.collectors = {}
self.config = config
self.stat_user = StatUser(config['start'], config['end'])
self.analysis_time = StatTime(config['start_analysis'],
config['end_analysis'])
self.rpt_print_intvl = 40
self.commit_intvl = 30
self.eh.register_timer(self.commit_intvl * 1000, TMTAGS.DB_COMMIT,
True)
self.db = Mydb()
self.db.connect('repository')
self.dbsession = self.db.open('all_users')
def _print_report(self):
echo("going to generate report file")
echo("generating the count data for every seconds")
db = Mydb()
db.connect('report')
dbname = "freq_trends_" + dbname_dict[api]
dbsession = db.open(dbname)
d = {'timestamp': "",
'count': ''}
for ts, count in self.data.items():
d['timestamp'] = ts
d['count'] = count
self._mark_top(ts, count)
dbsession.insert(d)
dbsession.commit()
dbsession.close()
echo("generating the top count data for earch day")
db.connect('report')
dbname = "freq_tops_" + dbname_dict[api]
dbsession = db.open(dbname)
for ts, count in self.top_data.items():
d['timestamp'] = ts
d['count'] = count
dbsession.insert(d)
dbsession.commit()
dbsession.close()
def __init__(self, msgh, config):
CalcManager.__init__(self, msgh)
self.report_fd = None
self.collectors = {}
self.config = config
self.stat_user = StatUser(config['start'], config['end'])
self.rpt_print_intvl = 10
#self.eh.register_timer(self.rpt_print_intvl * 1000,
# TMTAGS.PRINT_REPORT, True)
self.db = Mydb()
self.db.connect('report')
def __init__(self, msgh, config):
CalcManager.__init__(self, msgh)
self.report_fd = None
self.collectors = {}
self.config = config
self.stat_user = StatUser(config['start'], config['end'])
self.analysis_time = StatTime(config['start_analysis'],
config['end_analysis'])
self.rpt_print_intvl = 40
self.commit_intvl = 30
self.eh.register_timer(self.commit_intvl * 1000,
TMTAGS.DB_COMMIT, True)
self.db = Mydb()
self.db.connect('repository')
self.dbsession = self.db.open('all_users')
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import urllib2
import json
import time
from core.mydb import Mydb
detail_page_url = "http://api.vod.jamdeocloud.com/medias/api/media/{0}"
# medias database
medias_db = Mydb()
medias_db.connect('repository')
medias_db_session = medias_db.open("all_medias")
class VodMedia():
def __init__(self, mediaid):
self.id = mediaid
self.data =None
self.title = 'anonymous'
self.category_id = None
self.country = 'None'
self._fill()
def _fill_with_local(self):
media = medias_db_session.select({"mediaid": self.id})
if media is None:
return False
self.title = media[1][1]
self.category_id = media[1][2]
self.country = media[1][3]
return True
class PlayTimeCalcMgr(CalcManager):
def __init__(self, msgh, config):
CalcManager.__init__(self, msgh)
self.report_fd = None
self.collectors = {}
self.config = config
self.stat_user = StatUser(config['start'], config['end'])
self.rpt_print_intvl = 10
#self.eh.register_timer(self.rpt_print_intvl * 1000,
# TMTAGS.PRINT_REPORT, True)
self.db = Mydb()
self.db.connect('report')
def set_config(self, config):
self.config = config
def _process_msg(self, msg):
debug("PlayTimeCalcMgr::_process_msg() enter")
msgtype = msg.get_msgtype()
if msgtype == MsgType.MsgPlayTimeReport:
debug("PlayTimeCalcMgr::_process_msg() got "
"report message %s"%msg.get_body())
#print "PlayerStartupCalcMgr::_process_msg() got "\
# "report message %s"%msg.get_body()
ptr_msg = MsgPlayTimeReport()
ptr_msg.cast(msg)
data = json.loads(ptr_msg.get_report_info())
for user, play_info in data.items():
self.stat_user.merge_user_info(user, play_info)
#for vender, counts in data.items():
# self.collectors[vender].stat_time.merge(counts)
else:
super(PlayTimeCalcMgr, self)._process_msg(msg)
def _process_timer(self, msg):
#tag = msg.get_tag()
#if tag == TMTAGS.PRINT_REPORT:
# self._print_report()
#else:
# super(PlayTimeCalcMgr, self)._process_timer(msg)
super(PlayTimeCalcMgr, self)._process_timer(msg)
def _print_report(self):
print "currently the total medias:", self.stat_user.count_user()
self._category_play_duration()
def _category_play_duration(self):
from common.media_type import media_types
from common.fetch_media_data import VodMedia
self.dbsession = self.db.open('play_top_media_types_duration')
data = {"seq": "",
'date':'%s_%s'%(self.config['start'],
self.config['end']),
'categoryid': "",
'category_name':'',
'count': 0}
media_type_count = {}
for mediaid, play_info in self.stat_user.users_info.users.items():
infos = play_info.count_with_index()
total = 0
for index, count in infos.items():
total += count
media = VodMedia(mediaid)
title = media.get_title()
print "id: %s, cid: %s, title: %s"%\
(mediaid, media.get_category_id(), unicode(title, 'utf-8'))
cid = media.get_category_id()
if cid is None:
continue
cid = int(cid)
if cid in media_type_count:
media_type_count[cid] += total
else:
media_type_count[cid] = total
sorted_types = sorted(media_type_count.items(), key=lambda d: d[1], reverse=True)
for i in range(len(sorted_types)):
data['seq'] = i
category_id = sorted_types[i][0]
data['categoryid'] = category_id
if category_id in media_types:
data['category_name'] = media_types[category_id]
else:
data['category_name'] = category_id
data['count'] = sorted_types[i][1]
self.dbsession.insert(data)
self.dbsession.commit()
self.dbsession.close()
def _final(self):
self._print_report()
super(PlayTimeCalcMgr, self)._final()
class AccessUserCalcMgr(CalcManager):
def __init__(self, msgh, config):
CalcManager.__init__(self, msgh)
self.report_fd = None
self.collectors = {}
self.config = config
self.stat_user = StatUser(config['start'], config['end'])
self.analysis_time = StatTime(config['start_analysis'],
config['end_analysis'])
self.rpt_print_intvl = 40
self.commit_intvl = 30
self.eh.register_timer(self.commit_intvl * 1000,
TMTAGS.DB_COMMIT, True)
self.db = Mydb()
self.db.connect('repository')
self.dbsession = self.db.open('all_users')
def set_config(self, config):
self.config = config
def _process_msg(self, msg):
debug("AccessUserCalcMgr::_process_msg() enter")
msgtype = msg.get_msgtype()
#print "--------------process msg", datetime.datetime.now()
if msgtype == MsgType.MsgAccessUserReport:
debug("AccessUserCalcMgr::_process_msg() got "
"report message %s"%msg.get_body())
ptr_msg = MsgAccessUserReport()
ptr_msg.cast(msg)
ptr_msg.parse()
#print ptr_msg.get_users_info()
for user, ts in ptr_msg.get_users_info():
entry = self.dbsession.select({'user': user})
if entry is None:
self.dbsession.insert({'user': user, 'timestamp': ts})
else:
#print "entry----", entry
exist_ts = entry[1][1]
if ts < exist_ts:
#print "ts < exists", ts, exist_ts
self.dbsession.update({'user':user, 'ts': ts})
else:
super(AccessUserCalcMgr, self)._process_msg(msg)
def _process_timer(self, msg):
tag = msg.get_tag()
if tag == TMTAGS.DB_COMMIT:
self.dbsession.commit()
else:
super(AccessUserCalcMgr, self)._process_timer(msg)
#super(AccessUserCalcMgr, self)._process_timer(msg)
def _print_report(self):
entries = self.dbsession.select("*")
for key, entry in entries:
userid = entry[0]
ts = entry[1]
self.analysis_time.stat_count(int(ts))
infos = self.analysis_time.show_info()
days_map = sorted(infos.items(), key=lambda info:info[0], reverse=False)
total = 0
ret = []
for day, count in days_map:
total += count
ret.append([day, total])
print ret
def _final(self):
echo("going to generate final report ...")
self._print_report()
self.dbsession.close()
echo("report generation completed")
super(AccessUserCalcMgr, self)._final()
class PlayTimeCalcMgr(CalcManager):
def __init__(self, msgh, config):
CalcManager.__init__(self, msgh)
self.report_fd = None
self.collectors = {}
self.config = config
self.stat_user = StatUser(config['start'], config['end'])
self.rpt_print_intvl = 10
#self.eh.register_timer(self.rpt_print_intvl * 1000,
# TMTAGS.PRINT_REPORT, True)
self.db = Mydb()
self.db.connect('report')
def set_config(self, config):
self.config = config
def _process_msg(self, msg):
debug("PlayTimeCalcMgr::_process_msg() enter")
msgtype = msg.get_msgtype()
if msgtype == MsgType.MsgPlayTimeReport:
debug("PlayTimeCalcMgr::_process_msg() got "
"report message %s" % msg.get_body())
#print "PlayerStartupCalcMgr::_process_msg() got "\
# "report message %s"%msg.get_body()
ptr_msg = MsgPlayTimeReport()
ptr_msg.cast(msg)
data = json.loads(ptr_msg.get_report_info())
for user, play_info in data.items():
self.stat_user.merge_user_info(user, play_info)
#for vender, counts in data.items():
# self.collectors[vender].stat_time.merge(counts)
else:
super(PlayTimeCalcMgr, self)._process_msg(msg)
def _process_timer(self, msg):
#tag = msg.get_tag()
#if tag == TMTAGS.PRINT_REPORT:
# self._print_report()
#else:
# super(PlayTimeCalcMgr, self)._process_timer(msg)
super(PlayTimeCalcMgr, self)._process_timer(msg)
def _print_report(self):
print "currently the total user:"******"vender": '', 'date': '', 'time': 0}
venders_time = self.stat_user.gen_stat_count_by_vender_per_day()
venders_user = self.stat_user.gen_stat_users_by_vender_per_day()
for vender, stat_time in venders_time.items():
print "--------", vender, stat_time.show_info()
for vender, stat_time in venders_user.items():
print "_duration_rate_day(): ", vender, stat_time.show_info()
if vender not in venders_time:
warn(
'_duration_rate_day():user vender %s is not in time venders[%s]'
% (vender, str(venders_time)))
continue
time_infos = venders_time[vender].show_info()
user_infos = stat_time.show_info()
total, rate = 0, 0
data['vender'] = vender
for day, count in user_infos.items():
if day not in time_infos:
warn(
"_duration_rate_day():user day %s is not in time days[%s]"
% (day, str(time_infos.keys())))
continue
rate = int(time_infos[day]) / int(count)
data['date'] = day
data['time'] = rate
self.dbsession.insert(data)
self.dbsession.commit()
total, rate = 0, 0
time_infos = self.stat_user.gen_stat_times().show_info()
user_infos = self.stat_user.gen_stat_users().show_info()
data['vender'] = 'HISENSE'
for day, count in user_infos.items():
if day not in time_infos:
warn('user ay %s is not in time days[%s]' %
(day, str(time_infos.keys())))
continue
rate = int(time_infos[day]) / int(count)
data['date'] = day
data['time'] = rate
self.dbsession.insert(data)
self.dbsession.commit()
self.dbsession.close()
def _duration_rate(self):
self.dbsession = self.db.open('play_duration_rate')
data = {
"vender": '',
'date': '%s_%s' % (self.config['start'], self.config['end']),
'time': 0
}
venders_time = self.stat_user.gen_stat_count_by_vender_per_day()
venders_user = self.stat_user.gen_stat_users_by_vender_per_day()
for vender, stat_time in venders_user.items():
if vender not in venders_time:
continue
time_infos = venders_time[vender].show_info()
user_infos = stat_time.show_info()
total, rate, u = 0, 0, 0
for day, count in user_infos.items():
if day not in time_infos:
continue
total += int(time_infos[day]) / int(count)
u += 1
if u == 0:
continue
rate = total / u
data['vender'] = vender
data['time'] = rate
self.dbsession.insert(data)
self.dbsession.commit()
total, rate, u = 0, 0, 0
time_infos = self.stat_user.gen_stat_times().show_info()
user_infos = self.stat_user.gen_stat_users().show_info()
for day, count in user_infos.items():
if day not in time_infos:
continue
total += int(time_infos[day]) / int(count)
u += 1
if u == 0:
return
rate = total / u
data['vender'] = 'HISENSE'
data['time'] = rate
self.dbsession.insert(data)
self.dbsession.commit()
self.dbsession.close()
def _duration_total_day(self):
self.dbsession = self.db.open('play_duration_day')
data = {"vender": '', 'date': '', 'time': 0}
venders_time = self.stat_user.gen_stat_count_by_vender_per_day()
for vender, stat_time in venders_time.items():
time_infos = stat_time.show_info()
data['vender'] = vender
for day, count in time_infos.items():
data['date'] = day
data['time'] = count
self.dbsession.insert(data)
self.dbsession.commit()
time_infos = self.stat_user.gen_stat_times().show_info()
for day, count in time_infos.items():
data['vender'] = 'HISENSE'
data['date'] = day
data['time'] = count
self.dbsession.insert(data)
self.dbsession.commit()
self.dbsession.close()
def _final(self):
self._print_report()
super(PlayTimeCalcMgr, self)._final()
class PlayTimeCalcMgr(CalcManager):
def __init__(self, msgh, config):
CalcManager.__init__(self, msgh)
self.report_fd = None
self.collectors = {}
self.config = config
self.stat_user = StatUser(config['start'], config['end'])
self.rpt_print_intvl = 10
#self.eh.register_timer(self.rpt_print_intvl * 1000,
# TMTAGS.PRINT_REPORT, True)
self.db = Mydb()
self.db.connect('report')
def set_config(self, config):
self.config = config
def _process_msg(self, msg):
debug("PlayTimeCalcMgr::_process_msg() enter")
msgtype = msg.get_msgtype()
if msgtype == MsgType.MsgPlayTimeReport:
debug("PlayTimeCalcMgr::_process_msg() got "
"report message %s"%msg.get_body())
#print "PlayerStartupCalcMgr::_process_msg() got "\
# "report message %s"%msg.get_body()
ptr_msg = MsgPlayTimeReport()
ptr_msg.cast(msg)
data = json.loads(ptr_msg.get_report_info())
for user, play_info in data.items():
self.stat_user.merge_user_info(user, play_info)
#for vender, counts in data.items():
# self.collectors[vender].stat_time.merge(counts)
else:
super(PlayTimeCalcMgr, self)._process_msg(msg)
def _process_timer(self, msg):
#tag = msg.get_tag()
#if tag == TMTAGS.PRINT_REPORT:
# self._print_report()
#else:
# super(PlayTimeCalcMgr, self)._process_timer(msg)
super(PlayTimeCalcMgr, self)._process_timer(msg)
def _print_report(self):
print "currently the total user:"******"vender": '',
'date': '',
'time': 0}
venders_time = self.stat_user.gen_stat_count_by_vender_per_day()
venders_user = self.stat_user.gen_stat_users_by_vender_per_day()
for vender, stat_time in venders_time.items():
print "--------", vender, stat_time.show_info()
for vender, stat_time in venders_user.items():
print "_duration_rate_day(): ", vender, stat_time.show_info()
if vender not in venders_time:
warn('_duration_rate_day():user vender %s is not in time venders[%s]'%(vender, str(venders_time)))
continue
time_infos = venders_time[vender].show_info()
user_infos = stat_time.show_info()
total, rate = 0, 0
data['vender'] = vender
for day, count in user_infos.items():
if day not in time_infos:
warn("_duration_rate_day():user day %s is not in time days[%s]"%(day, str(time_infos.keys())))
continue
rate = int(time_infos[day]) / int(count)
data['date'] = day
data['time'] = rate
self.dbsession.insert(data)
self.dbsession.commit()
total, rate = 0, 0
time_infos = self.stat_user.gen_stat_times().show_info()
user_infos = self.stat_user.gen_stat_users().show_info()
data['vender'] = 'HISENSE'
for day, count in user_infos.items():
if day not in time_infos:
warn('user ay %s is not in time days[%s]'%(day, str(time_infos.keys())))
continue
rate = int(time_infos[day]) / int(count)
data['date'] = day
data['time'] = rate
self.dbsession.insert(data)
self.dbsession.commit()
self.dbsession.close()
def _duration_rate(self):
self.dbsession = self.db.open('play_duration_rate')
data = {"vender": '',
'date': '%s_%s'%(self.config['start'],
self.config['end']),
'time': 0}
venders_time = self.stat_user.gen_stat_count_by_vender_per_day()
venders_user = self.stat_user.gen_stat_users_by_vender_per_day()
for vender, stat_time in venders_user.items():
if vender not in venders_time:
continue
time_infos = venders_time[vender].show_info()
user_infos = stat_time.show_info()
total, rate, u = 0, 0, 0
for day, count in user_infos.items():
if day not in time_infos:
continue
total += int(time_infos[day]) / int(count)
u += 1
if u == 0:
continue
rate = total / u
data['vender'] = vender
data['time'] = rate
self.dbsession.insert(data)
self.dbsession.commit()
total, rate, u = 0, 0, 0
time_infos = self.stat_user.gen_stat_times().show_info()
user_infos = self.stat_user.gen_stat_users().show_info()
for day, count in user_infos.items():
if day not in time_infos:
continue
total += int(time_infos[day]) / int(count)
u += 1
if u == 0:
return
rate = total / u
data['vender'] = 'HISENSE'
data['time'] = rate
self.dbsession.insert(data)
self.dbsession.commit()
self.dbsession.close()
def _duration_total_day(self):
self.dbsession = self.db.open('play_duration_day')
data = {"vender": '',
'date': '',
'time': 0}
venders_time = self.stat_user.gen_stat_count_by_vender_per_day()
for vender, stat_time in venders_time.items():
time_infos = stat_time.show_info()
data['vender'] = vender
for day, count in time_infos.items():
data['date'] = day
data['time'] = count
self.dbsession.insert(data)
self.dbsession.commit()
time_infos = self.stat_user.gen_stat_times().show_info()
for day, count in time_infos.items():
data['vender'] = 'HISENSE'
data['date'] = day
data['time'] = count
self.dbsession.insert(data)
self.dbsession.commit()
self.dbsession.close()
def _final(self):
self._print_report()
super(PlayTimeCalcMgr, self)._final()
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import urllib2
import json
import time
from core.mydb import Mydb
detail_page_url = "http://api.vod.jamdeocloud.com/medias/api/media/{0}"
# medias database
medias_db = Mydb()
medias_db.connect('repository')
medias_db_session = medias_db.open("all_medias")
class VodMedia():
def __init__(self, mediaid):
self.id = mediaid
self.data = None
self.title = 'anonymous'
self.category_id = None
self.country = 'None'
self._fill()
def _fill_with_local(self):
media = medias_db_session.select({"mediaid": self.id})
if media is None:
return False
self.title = media[1][1]
self.category_id = media[1][2]
self.country = media[1][3]
class PlayTimeCalcMgr(CalcManager):
def __init__(self, msgh, config):
CalcManager.__init__(self, msgh)
self.report_fd = None
self.collectors = {}
self.config = config
self.stat_user = StatUser(config['start'], config['end'])
self.rpt_print_intvl = 10
#self.eh.register_timer(self.rpt_print_intvl * 1000,
# TMTAGS.PRINT_REPORT, True)
self.db = Mydb()
self.db.connect('report')
def set_config(self, config):
self.config = config
def _process_msg(self, msg):
debug("PlayTimeCalcMgr::_process_msg() enter")
msgtype = msg.get_msgtype()
if msgtype == MsgType.MsgPlayTimeReport:
debug("PlayTimeCalcMgr::_process_msg() got "
"report message %s" % msg.get_body())
#print "PlayerStartupCalcMgr::_process_msg() got "\
# "report message %s"%msg.get_body()
ptr_msg = MsgPlayTimeReport()
ptr_msg.cast(msg)
data = json.loads(ptr_msg.get_report_info())
for user, play_info in data.items():
self.stat_user.merge_user_info(user, play_info)
#for vender, counts in data.items():
# self.collectors[vender].stat_time.merge(counts)
else:
super(PlayTimeCalcMgr, self)._process_msg(msg)
def _process_timer(self, msg):
#tag = msg.get_tag()
#if tag == TMTAGS.PRINT_REPORT:
# self._print_report()
#else:
# super(PlayTimeCalcMgr, self)._process_timer(msg)
super(PlayTimeCalcMgr, self)._process_timer(msg)
def _print_report(self):
print "currently the total medias:", self.stat_user.count_user()
self._category_play_duration()
def _category_play_duration(self):
from common.media_type import media_types
from common.fetch_media_data import VodMedia
self.dbsession = self.db.open('play_top_media_types_duration')
data = {
"seq": "",
'date': '%s_%s' % (self.config['start'], self.config['end']),
'categoryid': "",
'category_name': '',
'count': 0
}
media_type_count = {}
for mediaid, play_info in self.stat_user.users_info.users.items():
infos = play_info.count_with_index()
total = 0
for index, count in infos.items():
total += count
media = VodMedia(mediaid)
title = media.get_title()
print "id: %s, cid: %s, title: %s"%\
(mediaid, media.get_category_id(), unicode(title, 'utf-8'))
cid = media.get_category_id()
if cid is None:
continue
cid = int(cid)
if cid in media_type_count:
media_type_count[cid] += total
else:
media_type_count[cid] = total
sorted_types = sorted(media_type_count.items(),
key=lambda d: d[1],
reverse=True)
for i in range(len(sorted_types)):
data['seq'] = i
category_id = sorted_types[i][0]
data['categoryid'] = category_id
if category_id in media_types:
data['category_name'] = media_types[category_id]
else:
data['category_name'] = category_id
data['count'] = sorted_types[i][1]
self.dbsession.insert(data)
self.dbsession.commit()
self.dbsession.close()
def _final(self):
self._print_report()
super(PlayTimeCalcMgr, self)._final()
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from core import Application, make_app_wrappers, request
from core.bottle import template
from core.mydb import Mydb, MySession
#from datetime import datetime, timedelta
import datetime
import time
from time import mktime
app = Application()
get, post = make_app_wrappers(app)
db = Mydb()
db.connect('report')
@get('/')
def index():
return template('portal')
def compute_days(to_day, intvl):
''' here the input parameters changed by this function '''
intvl = int(intvl)
if to_day == '':
to_day = str(datetime.datetime.now())[:10]
d = datetime.datetime.strptime(to_day, '%Y-%m-%d')
pre_to_day = str(d - datetime.timedelta(days=intvl))[:10]
class PlayUserCalcMgr(CalcManager):
def __init__(self, msgh, config):
CalcManager.__init__(self, msgh)
self.report_fd = None
self.collectors = {}
self.config = config
self.stat_user = StatUser(config['start'], config['end'])
self.rpt_print_intvl = 40
#self.eh.register_timer(self.rpt_print_intvl * 1000,
# TMTAGS.PRINT_REPORT, True)
self.db = Mydb()
self.db.connect('report')
def set_config(self, config):
self.config = config
def _process_msg(self, msg):
debug("PlayUserCalcMgr::_process_msg() enter")
msgtype = msg.get_msgtype()
if msgtype == MsgType.MsgPlayUserReport:
debug("PlayUserCalcMgr::_process_msg() got "
"report message %s" % msg.get_body())
ptr_msg = MsgPlayUserReport()
ptr_msg.cast(msg)
data = json.loads(ptr_msg.get_report_info())
for user, play_info in data.items():
self.stat_user.merge_user_info(user, play_info)
else:
super(PlayUserCalcMgr, self)._process_msg(msg)
def _process_timer(self, msg):
#tag = msg.get_tag()
#if tag == TMTAGS.PRINT_REPORT:
# self._print_report()
#else:
# super(PlayUserCalcMgr, self)._process_timer(msg)
super(PlayUserCalcMgr, self)._process_timer(msg)
def _print_report(self):
self._user_report_total()
self._user_report_day()
self._times_report_total()
self._times_report_day()
def _final(self):
echo("going to generate final report ...")
self._print_report()
echo("report generation completed")
super(PlayUserCalcMgr, self)._final()
def _user_report_total(self):
venders = self.stat_user.gen_stat_users_by_vender()
self.dbsession = self.db.open('play_users_total')
data = {
"vender": "",
'date': '%s_%s' % (self.config['start'], self.config['end']),
'count': 0
}
for vender, count in venders.items():
data['vender'] = vender
data['count'] = count
self.dbsession.insert(data)
self.dbsession.commit()
data['vender'] = "HISENSE"
data['count'] = self.stat_user.count_user()
self.dbsession.insert(data)
self.dbsession.commit()
self.dbsession.close()
def _user_report_day(self):
venders = self.stat_user.gen_stat_users_by_vender_per_day()
self.dbsession = self.db.open('play_users_day')
data = {"vender": "", 'date': "", 'count': 0}
for vender, stat_time in venders.items():
infos = stat_time.show_info()
data['vender'] = vender
for day, count in infos.items():
data['date'] = day
data['count'] = count
self.dbsession.insert(data)
self.dbsession.commit()
infos = self.stat_user.gen_stat_users().show_info()
data['vender'] = 'HISENSE'
for day, count in infos.items():
data['date'] = day
data['count'] = count
self.dbsession.insert(data)
self.dbsession.commit()
self.dbsession.close()
def _times_report_total(self):
venders = self.stat_user.gen_stat_count_by_vender()
self.dbsession = self.db.open('play_times_total')
data = {
"vender": "",
'date': '%s_%s' % (self.config['start'], self.config['end']),
'count': 0
}
for vender, count in venders.items():
data['vender'] = vender
data['count'] = count
self.dbsession.insert(data)
self.dbsession.commit()
data['vender'] = "HISENSE"
data['count'] = self.stat_user.gen_stat_count()
self.dbsession.insert(data)
self.dbsession.commit()
self.dbsession.close()
def _times_report_day(self):
venders = self.stat_user.gen_stat_count_by_vender_per_day()
self.dbsession = self.db.open('play_times_day')
data = {"vender": "", 'date': "", 'count': 0}
for vender, stat_time in venders.items():
infos = stat_time.show_info()
data['vender'] = vender
for day, count in infos.items():
data['date'] = day
data['count'] = count
self.dbsession.insert(data)
self.dbsession.commit()
infos = self.stat_user.gen_stat_times().show_info()
data['vender'] = 'HISENSE'
for day, count in infos.items():
data['date'] = day
data['count'] = count
self.dbsession.insert(data)
self.dbsession.commit()
self.dbsession.close()
class ActiveUserCalcMgr(CalcManager):
def __init__(self, msgh, config):
CalcManager.__init__(self, msgh)
self.report_fd = None
self.collectors = {}
self.config = config
self.stat_user = StatUser(config['start'], config['end'])
self.rpt_print_intvl = 40
#self.eh.register_timer(self.rpt_print_intvl * 1000,
# TMTAGS.PRINT_REPORT, True)
self.db = Mydb()
self.db.connect('report')
def set_config(self, config):
self.config = config
def _process_msg(self, msg):
debug("ActiveUserCalcMgr::_process_msg() enter")
msgtype = msg.get_msgtype()
if msgtype == MsgType.MsgActiveUserReport:
debug("ActiveUserCalcMgr::_process_msg() got "
"report message %s"%msg.get_body())
ptr_msg = MsgActiveUserReport()
ptr_msg.cast(msg)
data = json.loads(ptr_msg.get_report_info())
for user, play_info in data.items():
self.stat_user.merge_user_info(user, play_info)
else:
super(ActiveUserCalcMgr, self)._process_msg(msg)
def _process_timer(self, msg):
#tag = msg.get_tag()
#if tag == TMTAGS.PRINT_REPORT:
# self._print_report()
#else:
# super(ActiveUserCalcMgr, self)._process_timer(msg)
super(ActiveUserCalcMgr, self)._process_timer(msg)
def _print_report(self):
self._user_report_total()
self._user_report_day()
def _final(self):
echo("going to generate final report ...")
self._print_report()
echo("report generation completed")
super(ActiveUserCalcMgr, self)._final()
def _user_report_total(self):
models = self.stat_user.gen_stat_users_by_model()
self.dbsession = self.db.open('active_users_total')
from common.models_list import models_hash
data = {"model": "",
'date': '%s_%s'%(self.config['start'],
self.config['end']),
'count': 0}
for model, count in models.items():
if model in models_hash:
data['model'] = models_hash[model]
data['count'] = count
else:
#print "%s --> %s"%(model, count)
data['model'] = model
data['count'] = count
self.dbsession.insert(data)
self.dbsession.commit()
data['model'] = "HISENSE"
data['count'] = self.stat_user.count_user()
self.dbsession.insert(data)
self.dbsession.commit()
self.dbsession.close()
def _user_report_day(self):
models = self.stat_user.gen_stat_users_by_model_per_day()
from common.models_list import models_hash
self.dbsession = self.db.open('active_users_day')
data = {"model": "",
'date': "",
'count': 0}
for m, stat_time in models.items():
infos = stat_time.show_info()
model = ""
if m in models_hash:
model = models_hash[m]
else:
model = m
data['model'] = model
for day, count in infos.items():
data['date'] = day
data['count'] = count
self.dbsession.insert(data)
self.dbsession.commit()
infos = self.stat_user.gen_stat_users().show_info()
data['model'] = 'HISENSE'
for day, count in infos.items():
data['date'] = day
data['count'] = count
self.dbsession.insert(data)
self.dbsession.commit()
self.dbsession.close()
class TopMediasCalcMgr(CalcManager):
def __init__(self, msgh, config):
CalcManager.__init__(self, msgh)
self.report_fd = None
self.config = config
self.stat_user = StatUser(config['start'], config['end'])
self.rpt_print_intvl = 40
self.category_count = {1001: 0, # play
1002: 0, # zongyi
1003: 0,
1004: 0, # movie
1005: 0, # catoon
1006: 0,
1007: 0, # sport
1008: 0,
1009: 0,
1010: 0,
1011: 0,
1012: 0, # child
1013: 0,
1014: 0,
1015: 0,
1016: 0,
1017: 0,
1100: 0
}
self.country_count = {}
#self.eh.register_timer(self.rpt_print_intvl * 1000,
# TMTAGS.PRINT_REPORT, True)
self.db = Mydb()
self.db.connect('report')
def set_config(self, config):
self.config = config
def _process_msg(self, msg):
debug("TopMediasCalcMgr::_process_msg() enter")
msgtype = msg.get_msgtype()
if msgtype == MsgType.MsgTopMediasReport:
debug("TopMediasCalcMgr::_process_msg() got "
"report message %s"%msg.get_body())
ptr_msg = MsgTopMediasReport()
ptr_msg.cast(msg)
data = json.loads(ptr_msg.get_report_info())
for user, play_info in data.items():
self.stat_user.merge_user_info(user, play_info)
else:
super(TopMediasCalcMgr, self)._process_msg(msg)
def _process_timer(self, msg):
#tag = msg.get_tag()
#if tag == TMTAGS.PRINT_REPORT:
# self._print_report()
#else:
# super(TopMediasCalcMgr, self)._process_timer(msg)
super(TopMediasCalcMgr, self)._process_timer(msg)
def _print_report(self):
self._top_medias()
self._top_media_type()
self._top_country()
self._category_times()
def _final(self):
echo("going to generate final report ...")
self._print_report()
echo("report generation completed")
super(TopMediasCalcMgr, self)._final()
def _count_medias_for_category(self, cid):
if cid not in self.category_count:
return
self.category_count[cid] += 1
def _count_media_for_country(self, country, count):
if country in self.country_count:
self.country_count[country] += count
else:
self.country_count[country] = count
def _top_medias(self):
top_list = self.stat_user.gen_stat_tops()
echo("there are totaly %s top medias"%len(top_list))
self.dbsession = self.db.open('play_top_medias')
data = {"seq": "",
'date':'%s_%s'%(self.config['start'],
self.config['end']),
'mediaid': "",
'media_name':'',
'count': 0}
i = 0
from common.fetch_media_data import VodMedia
for mediaid, count in top_list:
data['seq'] = i
data['mediaid'] = mediaid
media = VodMedia(mediaid)
title = media.get_title()
print "id: %s, cid: %s, title: %s"%\
(mediaid, media.get_category_id(), title)
#print "%s(%s) -- %s"%(title, mediaid, media.get_category_id())
data['media_name'] = title
data['count'] = count
self.dbsession.insert(data)
i += 1
self.dbsession.commit()
self.dbsession.close()
def _top_media_type(self):
from common.media_type import media_types
from common.fetch_media_data import VodMedia
self.dbsession = self.db.open('play_top_media_types_times')
data = {"seq": "",
'date':'%s_%s'%(self.config['start'],
self.config['end']),
'categoryid': "",
'category_name':'',
'count': 0}
media_type_count = {}
for mediaid, play_info in self.stat_user.users_info.users.items():
infos = play_info.count_with_index()
total = 0
for index, count in infos.items():
total += count
media = VodMedia(mediaid)
title = media.get_title()
print "id: %s, cid: %s, title: %s"%\
(mediaid, media.get_category_id(), unicode(title, 'utf-8'))
cid = media.get_category_id()
if cid is None:
continue
country = media.get_country()
self._count_medias_for_category(int(cid))
self._count_media_for_country(country, total)
# count the media play times
if cid in media_type_count:
media_type_count[cid] += total
else:
media_type_count[cid] = total
sorted_types = sorted(media_type_count.items(), key=lambda d: d[1], reverse=True)
for i in range(len(sorted_types)):
data['seq'] = i
category_id = sorted_types[i][0]
data['categoryid'] = category_id
if category_id in media_types:
data['category_name'] = media_types[category_id]
else:
data['category_name'] = category_id
data['count'] = sorted_types[i][1]
self.dbsession.insert(data)
self.dbsession.commit()
self.dbsession.close()
def _category_times(self):
print "### most popular categories play info ###"
from common.media_type import media_types
for cid, count in self.category_count.items():
print "%s : %s"%(media_types[cid], count)
def _top_country(self):
print "### most popular country play info ###"
for country, count in self.country_count.items():
print "%s : %s"%(country, count)
class PlayUserCalcMgr(CalcManager):
def __init__(self, msgh, config):
CalcManager.__init__(self, msgh)
self.report_fd = None
self.collectors = {}
self.config = config
self.stat_user = StatUser(config['start'], config['end'])
self.rpt_print_intvl = 40
#self.eh.register_timer(self.rpt_print_intvl * 1000,
# TMTAGS.PRINT_REPORT, True)
self.db = Mydb()
self.db.connect('report')
def set_config(self, config):
self.config = config
def _process_msg(self, msg):
debug("PlayUserCalcMgr::_process_msg() enter")
msgtype = msg.get_msgtype()
if msgtype == MsgType.MsgPlayUserReport:
debug("PlayUserCalcMgr::_process_msg() got "
"report message %s"%msg.get_body())
ptr_msg = MsgPlayUserReport()
ptr_msg.cast(msg)
data = json.loads(ptr_msg.get_report_info())
for user, play_info in data.items():
self.stat_user.merge_user_info(user, play_info)
else:
super(PlayUserCalcMgr, self)._process_msg(msg)
def _process_timer(self, msg):
#tag = msg.get_tag()
#if tag == TMTAGS.PRINT_REPORT:
# self._print_report()
#else:
# super(PlayUserCalcMgr, self)._process_timer(msg)
super(PlayUserCalcMgr, self)._process_timer(msg)
def _print_report(self):
self._user_report_total()
self._user_report_day()
self._times_report_total()
self._times_report_day()
def _final(self):
echo("going to generate final report ...")
self._print_report()
echo("report generation completed")
super(PlayUserCalcMgr, self)._final()
def _user_report_total(self):
venders = self.stat_user.gen_stat_users_by_vender()
self.dbsession = self.db.open('play_users_total')
data = {"vender": "",
'date': '%s_%s'%(self.config['start'],
self.config['end']),
'count': 0}
for vender, count in venders.items():
data['vender'] = vender
data['count'] = count
self.dbsession.insert(data)
self.dbsession.commit()
data['vender'] = "HISENSE"
data['count'] = self.stat_user.count_user()
self.dbsession.insert(data)
self.dbsession.commit()
self.dbsession.close()
def _user_report_day(self):
venders = self.stat_user.gen_stat_users_by_vender_per_day()
self.dbsession = self.db.open('play_users_day')
data = {"vender": "",
'date': "",
'count': 0}
for vender, stat_time in venders.items():
infos = stat_time.show_info()
data['vender'] = vender
for day, count in infos.items():
data['date'] = day
data['count'] = count
self.dbsession.insert(data)
self.dbsession.commit()
infos = self.stat_user.gen_stat_users().show_info()
data['vender'] = 'HISENSE'
for day, count in infos.items():
data['date'] = day
data['count'] = count
self.dbsession.insert(data)
self.dbsession.commit()
self.dbsession.close()
def _times_report_total(self):
venders = self.stat_user.gen_stat_count_by_vender()
self.dbsession = self.db.open('play_times_total')
data = {"vender": "",
'date': '%s_%s'%(self.config['start'],
self.config['end']),
'count': 0}
for vender, count in venders.items():
data['vender'] = vender
data['count'] = count
self.dbsession.insert(data)
self.dbsession.commit()
data['vender'] = "HISENSE"
data['count'] = self.stat_user.gen_stat_count()
self.dbsession.insert(data)
self.dbsession.commit()
self.dbsession.close()
def _times_report_day(self):
venders = self.stat_user.gen_stat_count_by_vender_per_day()
self.dbsession = self.db.open('play_times_day')
data = {"vender": "",
'date': "",
'count': 0}
for vender, stat_time in venders.items():
infos = stat_time.show_info()
data['vender'] = vender
for day, count in infos.items():
data['date'] = day
data['count'] = count
self.dbsession.insert(data)
self.dbsession.commit()
infos = self.stat_user.gen_stat_times().show_info()
data['vender'] = 'HISENSE'
for day, count in infos.items():
data['date'] = day
data['count'] = count
self.dbsession.insert(data)
self.dbsession.commit()
self.dbsession.close()
class AccessUserCalcMgr(CalcManager):
def __init__(self, msgh, config):
CalcManager.__init__(self, msgh)
self.report_fd = None
self.collectors = {}
self.config = config
self.stat_user = StatUser(config['start'], config['end'])
self.analysis_time = StatTime(config['start_analysis'],
config['end_analysis'])
self.rpt_print_intvl = 40
self.commit_intvl = 30
self.eh.register_timer(self.commit_intvl * 1000, TMTAGS.DB_COMMIT,
True)
self.db = Mydb()
self.db.connect('repository')
self.dbsession = self.db.open('all_users')
def set_config(self, config):
self.config = config
def _process_msg(self, msg):
debug("AccessUserCalcMgr::_process_msg() enter")
msgtype = msg.get_msgtype()
#print "--------------process msg", datetime.datetime.now()
if msgtype == MsgType.MsgAccessUserReport:
debug("AccessUserCalcMgr::_process_msg() got "
"report message %s" % msg.get_body())
ptr_msg = MsgAccessUserReport()
ptr_msg.cast(msg)
ptr_msg.parse()
#print ptr_msg.get_users_info()
for user, ts in ptr_msg.get_users_info():
entry = self.dbsession.select({'user': user})
if entry is None:
self.dbsession.insert({'user': user, 'timestamp': ts})
else:
#print "entry----", entry
exist_ts = entry[1][1]
if ts < exist_ts:
#print "ts < exists", ts, exist_ts
self.dbsession.update({'user': user, 'ts': ts})
else:
super(AccessUserCalcMgr, self)._process_msg(msg)
def _process_timer(self, msg):
tag = msg.get_tag()
if tag == TMTAGS.DB_COMMIT:
self.dbsession.commit()
else:
super(AccessUserCalcMgr, self)._process_timer(msg)
#super(AccessUserCalcMgr, self)._process_timer(msg)
def _print_report(self):
entries = self.dbsession.select("*")
for key, entry in entries:
userid = entry[0]
ts = entry[1]
self.analysis_time.stat_count(int(ts))
infos = self.analysis_time.show_info()
days_map = sorted(infos.items(),
key=lambda info: info[0],
reverse=False)
total = 0
ret = []
for day, count in days_map:
total += count
ret.append([day, total])
print ret
def _final(self):
echo("going to generate final report ...")
self._print_report()
self.dbsession.close()
echo("report generation completed")
super(AccessUserCalcMgr, self)._final()
class AccessFreqCalcMgr(CalcManager):
def __init__(self, msgh, config):
CalcManager.__init__(self, msgh)
self.report_fd = None
self.collectors = {}
self.config = config
self.db = Mydb()
self.db.connect('report')
dbname = "freq_trends_" + dbname_dict[api]
self.dbsession = self.db.open(dbname)
self.data = {}
self.trends_data = {}
self.top_data = {}
def set_config(self, config):
self.config = config
def mark(self, ts, count):
# convert to seconds from mseconds
ts = str(int(ts) / 1000)
if ts not in self.data:
self.data[ts] = count
else:
self.data[ts] += count
h = int(ts) / 3600000
if int(ts) % 3600000 == 0:
self.trends_data[ts] =
def _process_msg(self, msg):
debug("AccessFreqCalcMgr::_process_msg() enter")
msgtype = msg.get_msgtype()
#print "--------------process msg", datetime.datetime.now()
if msgtype == MsgType.MsgAccessFreqReport:
debug("AccessFreqCalcMgr::_process_msg() got "
"report message %s"%msg.get_body())
ptr_msg = MsgAccessFreqReport()
ptr_msg.cast(msg)
ptr_msg.parse()
#print "------------", ptr_msg.body
info = ptr_msg.get_freq_info()
for ts, count in info:
self.mark(ts, int(count))
else:
super(AccessFreqCalcMgr, self)._process_msg(msg)
def _process_timer(self, msg):
tag = msg.get_tag()
if tag == TMTAGS.DB_COMMIT:
#self.dbsession.commit()
pass
else:
super(AccessFreqCalcMgr, self)._process_timer(msg)
def _print_report(self):
print "going to generate report file"
d = {'timestamp': "",
'count': ''}
mmax = 0
mmax_ts = ''
for ts, count in self.data.items():
d['timestamp'] = ts
d['count'] = count
if count > mmax:
mmax = count
mmax_ts = ts
self.dbsession.insert(d)
self.dbsession.commit()
echo("report generation completed, max value is %d, related ts is %s"\
%(mmax, mmax_ts))
def _final(self):
self._print_report()
self.dbsession.close()
super(AccessFreqCalcMgr, self)._final()
#!/usr/bin/env python
from core.mydb import Mydb
db = Mydb()
db.connect('repository')
session = db.open("all_medias")
entries = session.select("*")
for key, values in entries:
print "--title--", values[0], values[1], values[2]
#!/usr/bin/env python
from core.mydb import Mydb
db = Mydb()
db.connect("repository")
session = db.open("all_medias")
entries = session.select("*")
for key, values in entries:
print "--title--", values[0], values[1], values[2]
