Пример #1
0
    def _print_report(self):
        echo("going to generate report file")
        echo("generating the count data for every seconds")
        db = Mydb()
        db.connect('report')
        dbname = "freq_trends_" + dbname_dict[api]
        dbsession = db.open(dbname)
        d = {'timestamp': "", 'count': ''}

        for ts, count in self.data.items():
            d['timestamp'] = ts
            d['count'] = count
            self._mark_top(ts, count)
            dbsession.insert(d)
        dbsession.commit()

        dbsession.close()
        echo("generating the top count data for earch day")
        db.connect('report')
        dbname = "freq_tops_" + dbname_dict[api]
        dbsession = db.open(dbname)
        for ts, count in self.top_data.items():
            d['timestamp'] = ts
            d['count'] = count
            dbsession.insert(d)
        dbsession.commit()
        dbsession.close()
Пример #2
0
    def _print_report(self):
        echo("going to generate report file")
        echo("generating the count data for every seconds")
        db = Mydb()
        db.connect('report')
        dbname = "freq_trends_" + dbname_dict[api]
        dbsession = db.open(dbname)
        d = {'timestamp': "",
             'count': ''}

        for ts, count in self.data.items():
            d['timestamp'] = ts
            d['count'] = count
            self._mark_top(ts, count)
            dbsession.insert(d)
        dbsession.commit()

        dbsession.close()
        echo("generating the top count data for earch day")
        db.connect('report')
        dbname = "freq_tops_" + dbname_dict[api]
        dbsession = db.open(dbname)
        for ts, count in self.top_data.items():
            d['timestamp'] = ts
            d['count'] = count
            dbsession.insert(d)
        dbsession.commit()
        dbsession.close()
Пример #3
0
class PlayTimeCalcMgr(CalcManager):
    def __init__(self, msgh, config):
        CalcManager.__init__(self, msgh)
        self.report_fd = None
        self.collectors = {}
        self.config = config
        self.stat_user = StatUser(config['start'], config['end'])
        self.rpt_print_intvl = 10
        #self.eh.register_timer(self.rpt_print_intvl * 1000,
        #                       TMTAGS.PRINT_REPORT, True)
        self.db = Mydb()
        self.db.connect('report')

    def set_config(self, config):
        self.config = config

    def _process_msg(self, msg):
        debug("PlayTimeCalcMgr::_process_msg() enter")
        msgtype = msg.get_msgtype()
        if msgtype == MsgType.MsgPlayTimeReport:
            debug("PlayTimeCalcMgr::_process_msg() got "
                  "report message %s"%msg.get_body())
            #print "PlayerStartupCalcMgr::_process_msg() got "\
            #      "report message %s"%msg.get_body()
            ptr_msg = MsgPlayTimeReport()
            ptr_msg.cast(msg)
            data = json.loads(ptr_msg.get_report_info())
            for user, play_info in data.items():
                self.stat_user.merge_user_info(user, play_info)
            #for vender, counts in data.items():
            #    self.collectors[vender].stat_time.merge(counts)
        else:
            super(PlayTimeCalcMgr, self)._process_msg(msg)

    def _process_timer(self, msg):
        #tag = msg.get_tag()
        #if tag == TMTAGS.PRINT_REPORT:
        #    self._print_report()
        #else:
        #    super(PlayTimeCalcMgr, self)._process_timer(msg)
        super(PlayTimeCalcMgr, self)._process_timer(msg)

    def _print_report(self):
        print "currently the total medias:", self.stat_user.count_user()
        self._category_play_duration()
        
    def _category_play_duration(self):
        from common.media_type import media_types
        from common.fetch_media_data import VodMedia
        self.dbsession = self.db.open('play_top_media_types_duration')

        data = {"seq": "",
                'date':'%s_%s'%(self.config['start'],
                                self.config['end']),
                'categoryid': "",
                'category_name':'',
                'count': 0}
        media_type_count = {}
        for mediaid, play_info in self.stat_user.users_info.users.items():
            infos = play_info.count_with_index()
            total = 0
            for index, count in infos.items():
                total += count
            media = VodMedia(mediaid)
            title = media.get_title()
            print "id: %s, cid: %s, title: %s"%\
                  (mediaid, media.get_category_id(), unicode(title, 'utf-8'))
            cid = media.get_category_id()
            if cid is None:
                continue
            cid = int(cid)
            if cid in media_type_count:
                media_type_count[cid] += total
            else:
                media_type_count[cid] = total

        sorted_types = sorted(media_type_count.items(), key=lambda d: d[1], reverse=True)
        for i in range(len(sorted_types)):
            data['seq'] = i
            category_id = sorted_types[i][0]
            data['categoryid'] = category_id
            if category_id in media_types:
                data['category_name'] = media_types[category_id]
            else:
                data['category_name'] = category_id
            data['count'] = sorted_types[i][1]
            self.dbsession.insert(data)
        self.dbsession.commit()
        self.dbsession.close()
                
    def _final(self):
        self._print_report()
        super(PlayTimeCalcMgr, self)._final()
Пример #4
0
class AccessUserCalcMgr(CalcManager):
    def __init__(self, msgh, config):
        CalcManager.__init__(self, msgh)
        self.report_fd = None
        self.collectors = {}
        self.config = config
        self.stat_user = StatUser(config['start'], config['end'])
        self.analysis_time = StatTime(config['start_analysis'],
                                      config['end_analysis'])
        self.rpt_print_intvl = 40
        self.commit_intvl = 30
        self.eh.register_timer(self.commit_intvl * 1000,
                               TMTAGS.DB_COMMIT, True)
        self.db = Mydb()
        self.db.connect('repository')
        self.dbsession = self.db.open('all_users')

    def set_config(self, config):
        self.config = config

    def _process_msg(self, msg):
        debug("AccessUserCalcMgr::_process_msg() enter")
        msgtype = msg.get_msgtype()
        #print "--------------process msg", datetime.datetime.now()
        if msgtype == MsgType.MsgAccessUserReport:
            debug("AccessUserCalcMgr::_process_msg() got "
                  "report message %s"%msg.get_body())
            ptr_msg = MsgAccessUserReport()
            ptr_msg.cast(msg)
            ptr_msg.parse()
            #print ptr_msg.get_users_info()
            for user, ts in ptr_msg.get_users_info():
                entry = self.dbsession.select({'user': user})
                if entry is None:
                    self.dbsession.insert({'user': user, 'timestamp': ts})
                else:
                    #print "entry----", entry
                    exist_ts = entry[1][1]
                    if ts < exist_ts:
                        #print "ts < exists", ts, exist_ts
                        self.dbsession.update({'user':user, 'ts': ts})
            
        else:
            super(AccessUserCalcMgr, self)._process_msg(msg)

    def _process_timer(self, msg):
        tag = msg.get_tag()
        if tag == TMTAGS.DB_COMMIT:
            self.dbsession.commit()
        else:
            super(AccessUserCalcMgr, self)._process_timer(msg)
        #super(AccessUserCalcMgr, self)._process_timer(msg)

    def _print_report(self):
        entries = self.dbsession.select("*")
        for key, entry in entries:
            userid = entry[0]
            ts = entry[1]
            self.analysis_time.stat_count(int(ts))
        infos = self.analysis_time.show_info()
        days_map = sorted(infos.items(), key=lambda info:info[0], reverse=False) 
        total = 0
        ret = []
        for day, count in days_map:
            total += count
            ret.append([day, total])
        print ret

    def _final(self):
        echo("going to generate final report ...")
        self._print_report()
        self.dbsession.close()
        echo("report generation completed")
        super(AccessUserCalcMgr, self)._final()
Пример #5
0
class PlayTimeCalcMgr(CalcManager):
    def __init__(self, msgh, config):
        CalcManager.__init__(self, msgh)
        self.report_fd = None
        self.collectors = {}
        self.config = config
        self.stat_user = StatUser(config['start'], config['end'])
        self.rpt_print_intvl = 10
        #self.eh.register_timer(self.rpt_print_intvl * 1000,
        #                       TMTAGS.PRINT_REPORT, True)
        self.db = Mydb()
        self.db.connect('report')

    def set_config(self, config):
        self.config = config

    def _process_msg(self, msg):
        debug("PlayTimeCalcMgr::_process_msg() enter")
        msgtype = msg.get_msgtype()
        if msgtype == MsgType.MsgPlayTimeReport:
            debug("PlayTimeCalcMgr::_process_msg() got "
                  "report message %s"%msg.get_body())
            #print "PlayerStartupCalcMgr::_process_msg() got "\
            #      "report message %s"%msg.get_body()
            ptr_msg = MsgPlayTimeReport()
            ptr_msg.cast(msg)
            data = json.loads(ptr_msg.get_report_info())
            for user, play_info in data.items():
                self.stat_user.merge_user_info(user, play_info)
            #for vender, counts in data.items():
            #    self.collectors[vender].stat_time.merge(counts)
        else:
            super(PlayTimeCalcMgr, self)._process_msg(msg)

    def _process_timer(self, msg):
        #tag = msg.get_tag()
        #if tag == TMTAGS.PRINT_REPORT:
        #    self._print_report()
        #else:
        #    super(PlayTimeCalcMgr, self)._process_timer(msg)
        super(PlayTimeCalcMgr, self)._process_timer(msg)

    def _print_report(self):
        print "currently the total user:"******"vender": '',
                'date': '',
                'time': 0}
        
        venders_time = self.stat_user.gen_stat_count_by_vender_per_day()
        venders_user = self.stat_user.gen_stat_users_by_vender_per_day()

        for vender, stat_time in venders_time.items():
            print "--------", vender, stat_time.show_info()

        for vender, stat_time in venders_user.items():
            print "_duration_rate_day(): ", vender, stat_time.show_info()
            if vender not in venders_time:
                warn('_duration_rate_day():user vender %s is not in time venders[%s]'%(vender, str(venders_time)))
                continue
            time_infos = venders_time[vender].show_info()
            user_infos = stat_time.show_info()
            total, rate = 0, 0
            data['vender'] = vender
            for day, count in user_infos.items():
                if day not in time_infos:
                    warn("_duration_rate_day():user day %s is not in time days[%s]"%(day, str(time_infos.keys())))
                    continue
                rate = int(time_infos[day]) / int(count)
                data['date'] = day
                data['time'] = rate
                self.dbsession.insert(data)
            self.dbsession.commit()

        total, rate = 0, 0
        time_infos = self.stat_user.gen_stat_times().show_info()
        user_infos = self.stat_user.gen_stat_users().show_info()
        data['vender'] = 'HISENSE'
        for day, count in user_infos.items():
            if day not in time_infos:
                warn('user ay %s is not in time days[%s]'%(day, str(time_infos.keys())))
                continue
            rate = int(time_infos[day]) / int(count)
            data['date'] = day
            data['time'] = rate
            self.dbsession.insert(data)
        self.dbsession.commit()
        self.dbsession.close()
        
    def _duration_rate(self):
        self.dbsession = self.db.open('play_duration_rate')
        data = {"vender": '',
                'date': '%s_%s'%(self.config['start'],
                                 self.config['end']),
                'time': 0}
        
        venders_time = self.stat_user.gen_stat_count_by_vender_per_day()
        venders_user = self.stat_user.gen_stat_users_by_vender_per_day()
        for vender, stat_time in venders_user.items():
            if vender not in venders_time:
                continue
            time_infos = venders_time[vender].show_info()
            user_infos = stat_time.show_info()
            total, rate, u = 0, 0, 0
            for day, count in user_infos.items():
                if day not in time_infos:
                    continue
                total += int(time_infos[day]) / int(count)
                u += 1
            if u == 0:
                continue
            rate = total / u
            data['vender'] = vender
            data['time'] = rate
            self.dbsession.insert(data)
        self.dbsession.commit()

        total, rate, u = 0, 0, 0
        time_infos = self.stat_user.gen_stat_times().show_info()
        user_infos = self.stat_user.gen_stat_users().show_info()
        for day, count in user_infos.items():
            if day not in time_infos:
                continue
            total += int(time_infos[day]) / int(count)
            u += 1
        if u == 0:
            return
        rate = total / u
        data['vender'] = 'HISENSE'
        data['time'] = rate
        self.dbsession.insert(data)
        self.dbsession.commit()
        self.dbsession.close()

    def _duration_total_day(self):
        self.dbsession = self.db.open('play_duration_day')
        data = {"vender": '',
                'date': '',
                'time': 0}
        venders_time = self.stat_user.gen_stat_count_by_vender_per_day()
        for vender, stat_time in venders_time.items():
            time_infos = stat_time.show_info()
            data['vender'] = vender
            for day, count in time_infos.items():
                data['date'] = day
                data['time'] = count
                self.dbsession.insert(data)
        self.dbsession.commit()
        
        time_infos = self.stat_user.gen_stat_times().show_info()
        for day, count in time_infos.items():
            data['vender'] = 'HISENSE'
            data['date'] = day
            data['time'] = count
            self.dbsession.insert(data)

        self.dbsession.commit()
        self.dbsession.close()

    def _final(self):
        self._print_report()
        super(PlayTimeCalcMgr, self)._final()
Пример #6
0
class AccessFreqCalcMgr(CalcManager):
    def __init__(self, msgh, config):
        CalcManager.__init__(self, msgh)
        self.report_fd = None
        self.collectors = {}
        self.config = config
        self.db = Mydb()
        self.db.connect('report')
        dbname = "freq_trends_" + dbname_dict[api]
        self.dbsession = self.db.open(dbname)
        self.data = {}
        self.trends_data = {}
        self.top_data = {}

    def set_config(self, config):
        self.config = config

    def mark(self, ts, count):
        # convert to seconds from mseconds
        ts = str(int(ts) / 1000)
        if ts not in self.data:
            self.data[ts] = count
        else:
            self.data[ts] += count
        
        h = int(ts) / 3600000
        if int(ts) % 3600000 == 0:
            self.trends_data[ts] = 

    def _process_msg(self, msg):
        debug("AccessFreqCalcMgr::_process_msg() enter")
        msgtype = msg.get_msgtype()
        #print "--------------process msg", datetime.datetime.now()
        if msgtype == MsgType.MsgAccessFreqReport:
            debug("AccessFreqCalcMgr::_process_msg() got "
                  "report message %s"%msg.get_body())
            ptr_msg = MsgAccessFreqReport()
            ptr_msg.cast(msg)
            ptr_msg.parse()
            #print "------------", ptr_msg.body
            info = ptr_msg.get_freq_info()
            for ts, count in info:
                self.mark(ts, int(count))
        else:
            super(AccessFreqCalcMgr, self)._process_msg(msg)

    def _process_timer(self, msg):
        tag = msg.get_tag()
        if tag == TMTAGS.DB_COMMIT:
            #self.dbsession.commit()
            pass
        else:
            super(AccessFreqCalcMgr, self)._process_timer(msg)

    def _print_report(self):
        print "going to generate report file"
        d = {'timestamp': "",
             'count': ''}
        mmax = 0
        mmax_ts = ''
        for ts, count in self.data.items():
            d['timestamp'] = ts
            d['count'] = count
            if count > mmax:
                mmax = count
                mmax_ts = ts
            self.dbsession.insert(d)
        self.dbsession.commit()
        echo("report generation completed, max value is %d, related ts is %s"\
             %(mmax, mmax_ts))

    def _final(self):
        self._print_report()
        self.dbsession.close()
        super(AccessFreqCalcMgr, self)._final()
Пример #7
0
#!/usr/bin/env python
# -*- coding: utf-8 -*-

import urllib2
import json
import time
from core.mydb import Mydb

detail_page_url = "http://api.vod.jamdeocloud.com/medias/api/media/{0}"
# medias database
medias_db = Mydb()
medias_db.connect('repository')
medias_db_session = medias_db.open("all_medias")

class VodMedia():
    def __init__(self, mediaid):
        self.id = mediaid
        self.data =None
        self.title = 'anonymous'
        self.category_id = None
        self.country = 'None'
        self._fill()

    def _fill_with_local(self):
        media = medias_db_session.select({"mediaid": self.id})
        if media is None:
            return False
        self.title = media[1][1]
        self.category_id = media[1][2]
        self.country = media[1][3]
        return True
Пример #8
0
class PlayTimeCalcMgr(CalcManager):
    def __init__(self, msgh, config):
        CalcManager.__init__(self, msgh)
        self.report_fd = None
        self.collectors = {}
        self.config = config
        self.stat_user = StatUser(config['start'], config['end'])
        self.rpt_print_intvl = 10
        #self.eh.register_timer(self.rpt_print_intvl * 1000,
        #                       TMTAGS.PRINT_REPORT, True)
        self.db = Mydb()
        self.db.connect('report')

    def set_config(self, config):
        self.config = config

    def _process_msg(self, msg):
        debug("PlayTimeCalcMgr::_process_msg() enter")
        msgtype = msg.get_msgtype()
        if msgtype == MsgType.MsgPlayTimeReport:
            debug("PlayTimeCalcMgr::_process_msg() got "
                  "report message %s" % msg.get_body())
            #print "PlayerStartupCalcMgr::_process_msg() got "\
            #      "report message %s"%msg.get_body()
            ptr_msg = MsgPlayTimeReport()
            ptr_msg.cast(msg)
            data = json.loads(ptr_msg.get_report_info())
            for user, play_info in data.items():
                self.stat_user.merge_user_info(user, play_info)
            #for vender, counts in data.items():
            #    self.collectors[vender].stat_time.merge(counts)
        else:
            super(PlayTimeCalcMgr, self)._process_msg(msg)

    def _process_timer(self, msg):
        #tag = msg.get_tag()
        #if tag == TMTAGS.PRINT_REPORT:
        #    self._print_report()
        #else:
        #    super(PlayTimeCalcMgr, self)._process_timer(msg)
        super(PlayTimeCalcMgr, self)._process_timer(msg)

    def _print_report(self):
        print "currently the total medias:", self.stat_user.count_user()
        self._category_play_duration()

    def _category_play_duration(self):
        from common.media_type import media_types
        from common.fetch_media_data import VodMedia
        self.dbsession = self.db.open('play_top_media_types_duration')

        data = {
            "seq": "",
            'date': '%s_%s' % (self.config['start'], self.config['end']),
            'categoryid': "",
            'category_name': '',
            'count': 0
        }
        media_type_count = {}
        for mediaid, play_info in self.stat_user.users_info.users.items():
            infos = play_info.count_with_index()
            total = 0
            for index, count in infos.items():
                total += count
            media = VodMedia(mediaid)
            title = media.get_title()
            print "id: %s, cid: %s, title: %s"%\
                  (mediaid, media.get_category_id(), unicode(title, 'utf-8'))
            cid = media.get_category_id()
            if cid is None:
                continue
            cid = int(cid)
            if cid in media_type_count:
                media_type_count[cid] += total
            else:
                media_type_count[cid] = total

        sorted_types = sorted(media_type_count.items(),
                              key=lambda d: d[1],
                              reverse=True)
        for i in range(len(sorted_types)):
            data['seq'] = i
            category_id = sorted_types[i][0]
            data['categoryid'] = category_id
            if category_id in media_types:
                data['category_name'] = media_types[category_id]
            else:
                data['category_name'] = category_id
            data['count'] = sorted_types[i][1]
            self.dbsession.insert(data)
        self.dbsession.commit()
        self.dbsession.close()

    def _final(self):
        self._print_report()
        super(PlayTimeCalcMgr, self)._final()
Пример #9
0
class TopMediasCalcMgr(CalcManager):
    def __init__(self, msgh, config):
        CalcManager.__init__(self, msgh)
        self.report_fd = None
        self.config = config
        self.stat_user = StatUser(config['start'], config['end'])
        self.rpt_print_intvl = 40
        self.category_count = {1001: 0, # play
                               1002: 0, # zongyi
                               1003: 0, 
                               1004: 0, # movie
                               1005: 0, # catoon
                               1006: 0,
                               1007: 0, # sport
                               1008: 0,
                               1009: 0,
                               1010: 0,
                               1011: 0,
                               1012: 0, # child
                               1013: 0,
                               1014: 0,
                               1015: 0,
                               1016: 0,
                               1017: 0,
                               1100: 0
                               }
        self.country_count = {}

        #self.eh.register_timer(self.rpt_print_intvl * 1000,
        #                       TMTAGS.PRINT_REPORT, True)
        self.db = Mydb()
        self.db.connect('report')

    def set_config(self, config):
        self.config = config

    def _process_msg(self, msg):
        debug("TopMediasCalcMgr::_process_msg() enter")
        msgtype = msg.get_msgtype()
        if msgtype == MsgType.MsgTopMediasReport:
            debug("TopMediasCalcMgr::_process_msg() got "
                  "report message %s"%msg.get_body())
            ptr_msg = MsgTopMediasReport()
            ptr_msg.cast(msg)
            data = json.loads(ptr_msg.get_report_info())
            for user, play_info in data.items():
                self.stat_user.merge_user_info(user, play_info)

        else:
            super(TopMediasCalcMgr, self)._process_msg(msg)

    def _process_timer(self, msg):
        #tag = msg.get_tag()
        #if tag == TMTAGS.PRINT_REPORT:
        #    self._print_report()
        #else:
        #    super(TopMediasCalcMgr, self)._process_timer(msg)
        super(TopMediasCalcMgr, self)._process_timer(msg)

    def _print_report(self):
        self._top_medias()
        self._top_media_type()
        self._top_country()
        self._category_times()

    def _final(self):
        echo("going to generate final report ...")
        self._print_report()
        echo("report generation completed")
        super(TopMediasCalcMgr, self)._final()

    def _count_medias_for_category(self, cid):
        if cid not in self.category_count:
            return
        self.category_count[cid] += 1

    def _count_media_for_country(self, country, count):
        if country in self.country_count:
            self.country_count[country] += count
        else:
            self.country_count[country] = count

    def _top_medias(self):
        top_list = self.stat_user.gen_stat_tops()
        echo("there are totaly %s top medias"%len(top_list))
        self.dbsession = self.db.open('play_top_medias')
        data = {"seq": "",
                'date':'%s_%s'%(self.config['start'],
                                self.config['end']),
                'mediaid': "",
                'media_name':'',
                'count': 0}
        i = 0
        from common.fetch_media_data import VodMedia
        for mediaid, count in top_list:
            data['seq'] = i
            data['mediaid'] = mediaid
            media = VodMedia(mediaid)
            title = media.get_title()
            print "id: %s, cid: %s, title: %s"%\
                  (mediaid, media.get_category_id(), title)
            #print "%s(%s) -- %s"%(title, mediaid, media.get_category_id())
            data['media_name'] = title
            data['count'] = count
            self.dbsession.insert(data)
            i += 1
        self.dbsession.commit()
        self.dbsession.close()
        
        
    def _top_media_type(self):
        from common.media_type import media_types
        from common.fetch_media_data import VodMedia
        self.dbsession = self.db.open('play_top_media_types_times')

        data = {"seq": "",
                'date':'%s_%s'%(self.config['start'],
                                self.config['end']),
                'categoryid': "",
                'category_name':'',
                'count': 0}
        media_type_count = {}
        for mediaid, play_info in self.stat_user.users_info.users.items():
            infos = play_info.count_with_index()
            total = 0
            for index, count in infos.items():
                total += count
            media = VodMedia(mediaid)
            title = media.get_title()
            print "id: %s, cid: %s, title: %s"%\
                  (mediaid, media.get_category_id(), unicode(title, 'utf-8'))
            cid = media.get_category_id()
            if cid is None:
                continue

            country = media.get_country()

            self._count_medias_for_category(int(cid))
            self._count_media_for_country(country, total)
            
            # count the media play times
            if cid in media_type_count:
                media_type_count[cid] += total
            else:
                media_type_count[cid] = total
                
        sorted_types = sorted(media_type_count.items(), key=lambda d: d[1], reverse=True)
        for i in range(len(sorted_types)):
            data['seq'] = i
            category_id = sorted_types[i][0]
            data['categoryid'] = category_id
            if category_id in media_types:
                data['category_name'] = media_types[category_id]
            else:
                data['category_name'] = category_id
            data['count'] = sorted_types[i][1]
            self.dbsession.insert(data)
        self.dbsession.commit()
        self.dbsession.close()

    def _category_times(self):
        print "### most popular categories play info ###"
        from common.media_type import media_types
        for cid, count in self.category_count.items():
            print "%s : %s"%(media_types[cid], count)
        
    def _top_country(self):
        print "### most popular country play info ###"
        for country, count in self.country_count.items():
            print "%s : %s"%(country, count)
Пример #10
0
class PlayUserCalcMgr(CalcManager):
    def __init__(self, msgh, config):
        CalcManager.__init__(self, msgh)
        self.report_fd = None
        self.collectors = {}
        self.config = config
        self.stat_user = StatUser(config['start'], config['end'])
        self.rpt_print_intvl = 40
        #self.eh.register_timer(self.rpt_print_intvl * 1000,
        #                       TMTAGS.PRINT_REPORT, True)
        self.db = Mydb()
        self.db.connect('report')

    def set_config(self, config):
        self.config = config

    def _process_msg(self, msg):
        debug("PlayUserCalcMgr::_process_msg() enter")
        msgtype = msg.get_msgtype()
        if msgtype == MsgType.MsgPlayUserReport:
            debug("PlayUserCalcMgr::_process_msg() got "
                  "report message %s"%msg.get_body())
            ptr_msg = MsgPlayUserReport()
            ptr_msg.cast(msg)
            data = json.loads(ptr_msg.get_report_info())
            for user, play_info in data.items():
                self.stat_user.merge_user_info(user, play_info)

        else:
            super(PlayUserCalcMgr, self)._process_msg(msg)

    def _process_timer(self, msg):
        #tag = msg.get_tag()
        #if tag == TMTAGS.PRINT_REPORT:
        #    self._print_report()
        #else:
        #    super(PlayUserCalcMgr, self)._process_timer(msg)
        super(PlayUserCalcMgr, self)._process_timer(msg)

    def _print_report(self):
        self._user_report_total()
        self._user_report_day()
        self._times_report_total()
        self._times_report_day()

    def _final(self):
        echo("going to generate final report ...")
        self._print_report()
        echo("report generation completed")
        super(PlayUserCalcMgr, self)._final()

    def _user_report_total(self):
        venders = self.stat_user.gen_stat_users_by_vender()
        self.dbsession = self.db.open('play_users_total')
        data = {"vender": "",
                'date': '%s_%s'%(self.config['start'],
                                 self.config['end']),
                'count': 0}
        for vender, count in venders.items():
            data['vender'] = vender
            data['count'] = count
            self.dbsession.insert(data)
        self.dbsession.commit()
        
        data['vender'] = "HISENSE"
        data['count'] = self.stat_user.count_user()
        self.dbsession.insert(data)
        self.dbsession.commit()
        self.dbsession.close()
        
    def _user_report_day(self):
        venders = self.stat_user.gen_stat_users_by_vender_per_day()
        self.dbsession = self.db.open('play_users_day')
        data = {"vender": "",
                'date': "",
                'count': 0}
        for vender, stat_time in venders.items():
            infos = stat_time.show_info()
            data['vender'] = vender
            for day, count in infos.items():
                data['date'] = day
                data['count'] = count
                self.dbsession.insert(data)
        self.dbsession.commit()

        infos = self.stat_user.gen_stat_users().show_info()
        data['vender'] = 'HISENSE'
        for day, count in infos.items():
            data['date'] = day
            data['count'] = count
            self.dbsession.insert(data)
        self.dbsession.commit()
        self.dbsession.close()

    def _times_report_total(self):
        venders = self.stat_user.gen_stat_count_by_vender()
        self.dbsession = self.db.open('play_times_total')
        data = {"vender": "",
                'date': '%s_%s'%(self.config['start'],
                                 self.config['end']),
                'count': 0}
        for vender, count in venders.items():
            data['vender'] = vender
            data['count'] = count
            self.dbsession.insert(data)
        self.dbsession.commit()

        data['vender'] = "HISENSE"
        data['count'] = self.stat_user.gen_stat_count()
        self.dbsession.insert(data)
        self.dbsession.commit()
        self.dbsession.close()

    def _times_report_day(self):
        venders = self.stat_user.gen_stat_count_by_vender_per_day()
        self.dbsession = self.db.open('play_times_day')
        data = {"vender": "",
                'date': "",
                'count': 0}
        for vender, stat_time in venders.items():
            infos = stat_time.show_info()
            data['vender'] = vender
            for day, count in infos.items():
                data['date'] = day
                data['count'] = count
                self.dbsession.insert(data)
        self.dbsession.commit()

        infos = self.stat_user.gen_stat_times().show_info()
        data['vender'] = 'HISENSE'
        for day, count in infos.items():
            data['date'] = day
            data['count'] = count
            self.dbsession.insert(data)
        self.dbsession.commit()
        self.dbsession.close()
Пример #11
0
class ActiveUserCalcMgr(CalcManager):
    def __init__(self, msgh, config):
        CalcManager.__init__(self, msgh)
        self.report_fd = None
        self.collectors = {}
        self.config = config
        self.stat_user = StatUser(config['start'], config['end'])
        self.rpt_print_intvl = 40
        #self.eh.register_timer(self.rpt_print_intvl * 1000,
        #                       TMTAGS.PRINT_REPORT, True)
        self.db = Mydb()
        self.db.connect('report')

    def set_config(self, config):
        self.config = config

    def _process_msg(self, msg):
        debug("ActiveUserCalcMgr::_process_msg() enter")
        msgtype = msg.get_msgtype()
        if msgtype == MsgType.MsgActiveUserReport:
            debug("ActiveUserCalcMgr::_process_msg() got "
                  "report message %s"%msg.get_body())
            ptr_msg = MsgActiveUserReport()
            ptr_msg.cast(msg)
            data = json.loads(ptr_msg.get_report_info())
            for user, play_info in data.items():
                self.stat_user.merge_user_info(user, play_info)

        else:
            super(ActiveUserCalcMgr, self)._process_msg(msg)

    def _process_timer(self, msg):
        #tag = msg.get_tag()
        #if tag == TMTAGS.PRINT_REPORT:
        #    self._print_report()
        #else:
        #    super(ActiveUserCalcMgr, self)._process_timer(msg)
        super(ActiveUserCalcMgr, self)._process_timer(msg)

    def _print_report(self):
        self._user_report_total()
        self._user_report_day()

    def _final(self):
        echo("going to generate final report ...")
        self._print_report()
        echo("report generation completed")
        super(ActiveUserCalcMgr, self)._final()

    def _user_report_total(self):
        models = self.stat_user.gen_stat_users_by_model()
        self.dbsession = self.db.open('active_users_total')
        from common.models_list import models_hash
        data = {"model": "",
                'date': '%s_%s'%(self.config['start'],
                                 self.config['end']),
                'count': 0}
        for model, count in models.items():
            if model in models_hash:
                data['model'] = models_hash[model]
                data['count'] = count
            else:
                #print "%s --> %s"%(model, count)
                data['model'] = model
                data['count'] = count
            self.dbsession.insert(data)
        self.dbsession.commit()
        
        data['model'] = "HISENSE"
        data['count'] = self.stat_user.count_user()
        self.dbsession.insert(data)
        self.dbsession.commit()
        self.dbsession.close()
        
    def _user_report_day(self):
        models = self.stat_user.gen_stat_users_by_model_per_day()
        from common.models_list import models_hash
        self.dbsession = self.db.open('active_users_day')
        data = {"model": "",
                'date': "",
                'count': 0}
        for m, stat_time in models.items():
            infos = stat_time.show_info()
            model = ""
            if m in models_hash:
                model = models_hash[m]
            else:
                model = m
            data['model'] = model
            for day, count in infos.items():
                data['date'] = day
                data['count'] = count
                self.dbsession.insert(data)
            self.dbsession.commit()

        infos = self.stat_user.gen_stat_users().show_info()
        data['model'] = 'HISENSE'
        for day, count in infos.items():
            data['date'] = day
            data['count'] = count
            self.dbsession.insert(data)
        self.dbsession.commit()
        self.dbsession.close()
Пример #12
0
#!/usr/bin/env python
# -*- coding: utf-8 -*-

import urllib2
import json
import time
from core.mydb import Mydb

detail_page_url = "http://api.vod.jamdeocloud.com/medias/api/media/{0}"
# medias database
medias_db = Mydb()
medias_db.connect('repository')
medias_db_session = medias_db.open("all_medias")


class VodMedia():
    def __init__(self, mediaid):
        self.id = mediaid
        self.data = None
        self.title = 'anonymous'
        self.category_id = None
        self.country = 'None'
        self._fill()

    def _fill_with_local(self):
        media = medias_db_session.select({"mediaid": self.id})
        if media is None:
            return False
        self.title = media[1][1]
        self.category_id = media[1][2]
        self.country = media[1][3]
Пример #13
0
class PlayTimeCalcMgr(CalcManager):
    def __init__(self, msgh, config):
        CalcManager.__init__(self, msgh)
        self.report_fd = None
        self.collectors = {}
        self.config = config
        self.stat_user = StatUser(config['start'], config['end'])
        self.rpt_print_intvl = 10
        #self.eh.register_timer(self.rpt_print_intvl * 1000,
        #                       TMTAGS.PRINT_REPORT, True)
        self.db = Mydb()
        self.db.connect('report')

    def set_config(self, config):
        self.config = config

    def _process_msg(self, msg):
        debug("PlayTimeCalcMgr::_process_msg() enter")
        msgtype = msg.get_msgtype()
        if msgtype == MsgType.MsgPlayTimeReport:
            debug("PlayTimeCalcMgr::_process_msg() got "
                  "report message %s" % msg.get_body())
            #print "PlayerStartupCalcMgr::_process_msg() got "\
            #      "report message %s"%msg.get_body()
            ptr_msg = MsgPlayTimeReport()
            ptr_msg.cast(msg)
            data = json.loads(ptr_msg.get_report_info())
            for user, play_info in data.items():
                self.stat_user.merge_user_info(user, play_info)
            #for vender, counts in data.items():
            #    self.collectors[vender].stat_time.merge(counts)
        else:
            super(PlayTimeCalcMgr, self)._process_msg(msg)

    def _process_timer(self, msg):
        #tag = msg.get_tag()
        #if tag == TMTAGS.PRINT_REPORT:
        #    self._print_report()
        #else:
        #    super(PlayTimeCalcMgr, self)._process_timer(msg)
        super(PlayTimeCalcMgr, self)._process_timer(msg)

    def _print_report(self):
        print "currently the total user:"******"vender": '', 'date': '', 'time': 0}

        venders_time = self.stat_user.gen_stat_count_by_vender_per_day()
        venders_user = self.stat_user.gen_stat_users_by_vender_per_day()

        for vender, stat_time in venders_time.items():
            print "--------", vender, stat_time.show_info()

        for vender, stat_time in venders_user.items():
            print "_duration_rate_day(): ", vender, stat_time.show_info()
            if vender not in venders_time:
                warn(
                    '_duration_rate_day():user vender %s is not in time venders[%s]'
                    % (vender, str(venders_time)))
                continue
            time_infos = venders_time[vender].show_info()
            user_infos = stat_time.show_info()
            total, rate = 0, 0
            data['vender'] = vender
            for day, count in user_infos.items():
                if day not in time_infos:
                    warn(
                        "_duration_rate_day():user day %s is not in time days[%s]"
                        % (day, str(time_infos.keys())))
                    continue
                rate = int(time_infos[day]) / int(count)
                data['date'] = day
                data['time'] = rate
                self.dbsession.insert(data)
            self.dbsession.commit()

        total, rate = 0, 0
        time_infos = self.stat_user.gen_stat_times().show_info()
        user_infos = self.stat_user.gen_stat_users().show_info()
        data['vender'] = 'HISENSE'
        for day, count in user_infos.items():
            if day not in time_infos:
                warn('user ay %s is not in time days[%s]' %
                     (day, str(time_infos.keys())))
                continue
            rate = int(time_infos[day]) / int(count)
            data['date'] = day
            data['time'] = rate
            self.dbsession.insert(data)
        self.dbsession.commit()
        self.dbsession.close()

    def _duration_rate(self):
        self.dbsession = self.db.open('play_duration_rate')
        data = {
            "vender": '',
            'date': '%s_%s' % (self.config['start'], self.config['end']),
            'time': 0
        }

        venders_time = self.stat_user.gen_stat_count_by_vender_per_day()
        venders_user = self.stat_user.gen_stat_users_by_vender_per_day()
        for vender, stat_time in venders_user.items():
            if vender not in venders_time:
                continue
            time_infos = venders_time[vender].show_info()
            user_infos = stat_time.show_info()
            total, rate, u = 0, 0, 0
            for day, count in user_infos.items():
                if day not in time_infos:
                    continue
                total += int(time_infos[day]) / int(count)
                u += 1
            if u == 0:
                continue
            rate = total / u
            data['vender'] = vender
            data['time'] = rate
            self.dbsession.insert(data)
        self.dbsession.commit()

        total, rate, u = 0, 0, 0
        time_infos = self.stat_user.gen_stat_times().show_info()
        user_infos = self.stat_user.gen_stat_users().show_info()
        for day, count in user_infos.items():
            if day not in time_infos:
                continue
            total += int(time_infos[day]) / int(count)
            u += 1
        if u == 0:
            return
        rate = total / u
        data['vender'] = 'HISENSE'
        data['time'] = rate
        self.dbsession.insert(data)
        self.dbsession.commit()
        self.dbsession.close()

    def _duration_total_day(self):
        self.dbsession = self.db.open('play_duration_day')
        data = {"vender": '', 'date': '', 'time': 0}
        venders_time = self.stat_user.gen_stat_count_by_vender_per_day()
        for vender, stat_time in venders_time.items():
            time_infos = stat_time.show_info()
            data['vender'] = vender
            for day, count in time_infos.items():
                data['date'] = day
                data['time'] = count
                self.dbsession.insert(data)
        self.dbsession.commit()

        time_infos = self.stat_user.gen_stat_times().show_info()
        for day, count in time_infos.items():
            data['vender'] = 'HISENSE'
            data['date'] = day
            data['time'] = count
            self.dbsession.insert(data)

        self.dbsession.commit()
        self.dbsession.close()

    def _final(self):
        self._print_report()
        super(PlayTimeCalcMgr, self)._final()
Пример #14
0
class AccessUserCalcMgr(CalcManager):
    def __init__(self, msgh, config):
        CalcManager.__init__(self, msgh)
        self.report_fd = None
        self.collectors = {}
        self.config = config
        self.stat_user = StatUser(config['start'], config['end'])
        self.analysis_time = StatTime(config['start_analysis'],
                                      config['end_analysis'])
        self.rpt_print_intvl = 40
        self.commit_intvl = 30
        self.eh.register_timer(self.commit_intvl * 1000, TMTAGS.DB_COMMIT,
                               True)
        self.db = Mydb()
        self.db.connect('repository')
        self.dbsession = self.db.open('all_users')

    def set_config(self, config):
        self.config = config

    def _process_msg(self, msg):
        debug("AccessUserCalcMgr::_process_msg() enter")
        msgtype = msg.get_msgtype()
        #print "--------------process msg", datetime.datetime.now()
        if msgtype == MsgType.MsgAccessUserReport:
            debug("AccessUserCalcMgr::_process_msg() got "
                  "report message %s" % msg.get_body())
            ptr_msg = MsgAccessUserReport()
            ptr_msg.cast(msg)
            ptr_msg.parse()
            #print ptr_msg.get_users_info()
            for user, ts in ptr_msg.get_users_info():
                entry = self.dbsession.select({'user': user})
                if entry is None:
                    self.dbsession.insert({'user': user, 'timestamp': ts})
                else:
                    #print "entry----", entry
                    exist_ts = entry[1][1]
                    if ts < exist_ts:
                        #print "ts < exists", ts, exist_ts
                        self.dbsession.update({'user': user, 'ts': ts})

        else:
            super(AccessUserCalcMgr, self)._process_msg(msg)

    def _process_timer(self, msg):
        tag = msg.get_tag()
        if tag == TMTAGS.DB_COMMIT:
            self.dbsession.commit()
        else:
            super(AccessUserCalcMgr, self)._process_timer(msg)
        #super(AccessUserCalcMgr, self)._process_timer(msg)

    def _print_report(self):
        entries = self.dbsession.select("*")
        for key, entry in entries:
            userid = entry[0]
            ts = entry[1]
            self.analysis_time.stat_count(int(ts))
        infos = self.analysis_time.show_info()
        days_map = sorted(infos.items(),
                          key=lambda info: info[0],
                          reverse=False)
        total = 0
        ret = []
        for day, count in days_map:
            total += count
            ret.append([day, total])
        print ret

    def _final(self):
        echo("going to generate final report ...")
        self._print_report()
        self.dbsession.close()
        echo("report generation completed")
        super(AccessUserCalcMgr, self)._final()
Пример #15
0
class PlayUserCalcMgr(CalcManager):
    def __init__(self, msgh, config):
        CalcManager.__init__(self, msgh)
        self.report_fd = None
        self.collectors = {}
        self.config = config
        self.stat_user = StatUser(config['start'], config['end'])
        self.rpt_print_intvl = 40
        #self.eh.register_timer(self.rpt_print_intvl * 1000,
        #                       TMTAGS.PRINT_REPORT, True)
        self.db = Mydb()
        self.db.connect('report')

    def set_config(self, config):
        self.config = config

    def _process_msg(self, msg):
        debug("PlayUserCalcMgr::_process_msg() enter")
        msgtype = msg.get_msgtype()
        if msgtype == MsgType.MsgPlayUserReport:
            debug("PlayUserCalcMgr::_process_msg() got "
                  "report message %s" % msg.get_body())
            ptr_msg = MsgPlayUserReport()
            ptr_msg.cast(msg)
            data = json.loads(ptr_msg.get_report_info())
            for user, play_info in data.items():
                self.stat_user.merge_user_info(user, play_info)

        else:
            super(PlayUserCalcMgr, self)._process_msg(msg)

    def _process_timer(self, msg):
        #tag = msg.get_tag()
        #if tag == TMTAGS.PRINT_REPORT:
        #    self._print_report()
        #else:
        #    super(PlayUserCalcMgr, self)._process_timer(msg)
        super(PlayUserCalcMgr, self)._process_timer(msg)

    def _print_report(self):
        self._user_report_total()
        self._user_report_day()
        self._times_report_total()
        self._times_report_day()

    def _final(self):
        echo("going to generate final report ...")
        self._print_report()
        echo("report generation completed")
        super(PlayUserCalcMgr, self)._final()

    def _user_report_total(self):
        venders = self.stat_user.gen_stat_users_by_vender()
        self.dbsession = self.db.open('play_users_total')
        data = {
            "vender": "",
            'date': '%s_%s' % (self.config['start'], self.config['end']),
            'count': 0
        }
        for vender, count in venders.items():
            data['vender'] = vender
            data['count'] = count
            self.dbsession.insert(data)
        self.dbsession.commit()

        data['vender'] = "HISENSE"
        data['count'] = self.stat_user.count_user()
        self.dbsession.insert(data)
        self.dbsession.commit()
        self.dbsession.close()

    def _user_report_day(self):
        venders = self.stat_user.gen_stat_users_by_vender_per_day()
        self.dbsession = self.db.open('play_users_day')
        data = {"vender": "", 'date': "", 'count': 0}
        for vender, stat_time in venders.items():
            infos = stat_time.show_info()
            data['vender'] = vender
            for day, count in infos.items():
                data['date'] = day
                data['count'] = count
                self.dbsession.insert(data)
        self.dbsession.commit()

        infos = self.stat_user.gen_stat_users().show_info()
        data['vender'] = 'HISENSE'
        for day, count in infos.items():
            data['date'] = day
            data['count'] = count
            self.dbsession.insert(data)
        self.dbsession.commit()
        self.dbsession.close()

    def _times_report_total(self):
        venders = self.stat_user.gen_stat_count_by_vender()
        self.dbsession = self.db.open('play_times_total')
        data = {
            "vender": "",
            'date': '%s_%s' % (self.config['start'], self.config['end']),
            'count': 0
        }
        for vender, count in venders.items():
            data['vender'] = vender
            data['count'] = count
            self.dbsession.insert(data)
        self.dbsession.commit()

        data['vender'] = "HISENSE"
        data['count'] = self.stat_user.gen_stat_count()
        self.dbsession.insert(data)
        self.dbsession.commit()
        self.dbsession.close()

    def _times_report_day(self):
        venders = self.stat_user.gen_stat_count_by_vender_per_day()
        self.dbsession = self.db.open('play_times_day')
        data = {"vender": "", 'date': "", 'count': 0}
        for vender, stat_time in venders.items():
            infos = stat_time.show_info()
            data['vender'] = vender
            for day, count in infos.items():
                data['date'] = day
                data['count'] = count
                self.dbsession.insert(data)
        self.dbsession.commit()

        infos = self.stat_user.gen_stat_times().show_info()
        data['vender'] = 'HISENSE'
        for day, count in infos.items():
            data['date'] = day
            data['count'] = count
            self.dbsession.insert(data)
        self.dbsession.commit()
        self.dbsession.close()
Пример #16
0
#!/usr/bin/env python

from core.mydb import Mydb

db = Mydb()
db.connect('repository')
session = db.open("all_medias")

entries = session.select("*")

for key, values in entries:
    print "--title--", values[0], values[1], values[2]
Пример #17
0
#!/usr/bin/env python

from core.mydb import Mydb

db = Mydb()
db.connect("repository")
session = db.open("all_medias")

entries = session.select("*")

for key, values in entries:
    print "--title--", values[0], values[1], values[2]