def run(self):
redispool = redis.ConnectionPool(host=self.redataIP, port=6379, db=0)
redata = redis.Redis(connection_pool=redispool)
mongoconn = MongoClient(self.mongodbIP, 27017)
date = self.datadate
while True:
try:
"""Obtain vid name for have data changs daily"""
vid = self.queue.get()
"""Obtain uid for the vid"""
uid = vid[0:4]
self.log.debug("handleTVIDSET -- Handling is %s" % vid)
"""Obtain kid from redis db for the vid"""
kid = redata.hget("T_video_kid_info", vid)
"""Obtain temp data from redis"""
redpipe = redata.pipeline()
"""Obtain the number of independent IP for the vid daily"""
redpipe.scard("T_IPS_" + vid + "_%s" % (date))
"""Obtain load numbers for the vid daily"""
redpipe.hget("T_loadtable_%s" % (date), vid)
"""Obtain play numbers for the vid daily"""
redpipe.hget("T_playtable_%s" % (date), vid)
"""Write uid to set for facilitate the processing after"""
redpipe.sadd("T_UIDS_%s" % (date), uid)
vidDataList = redpipe.execute()
indepipnums = int(vidDataList[0])
dailyload = int(vidDataList[1])
dailyplay = int(vidDataList[2])
"""--开始--"""
# --/
# 建立mongodb表连接
# --/
uid_stat_t = mongoconn["video"]["%s_stat" % (uid)]
uid_vdaily_t = mongoconn["video"]["%s_vdaily" % (uid)]
uid_daily_t = mongoconn["video"]["%s_daily" % (uid)]
uid_type_t = mongoconn["video"]["%s_type" % (uid)]
request = mongoconn.start_request()
# --/
# 保存该视频playid去mongodb
# --/
playid = redata.hget("vid_playid", vid)
uid_stat_t.update({"_id": vid},
{"$set": {
"PlayID": int(playid)
}},
upsert=True,
w=0)
# --/
# 统计视频每天播放信息
# --/
"""total loads of the vid daily"""
"""total plays of the vid daily"""
"""average click rate of the vid daily"""
"""the number of independent IP for the vid daily"""
"""avarage complete rate for the vid daily"""
uid_vdaily_t.update({"_id": date}, {
"$set": {
"%s.Load" % (vid):
dailyload,
"%s.Play" % (vid):
dailyplay,
"%s.Click" % (vid):
operator.itruediv(dailyplay, dailyload),
"%s.IP" % (vid):
indepipnums,
"%s.Engage" % (vid):
operator.itruediv(
float(
redata.hget("T_engagesum_%s_%s" %
(uid, date), vid)), dailyplay)
}
},
upsert=True,
w=0)
# --/
# 统计视频截至到目前总的播放信息
# --/
"""Load nums of total for the vid up to now"""
"""play nums of total for the vid up to now"""
"""independent IPs of total for the vid up to now"""
uid_stat_t.update({"_id": vid}, {
"$inc": {
"Load": dailyload,
"Play": dailyplay,
"IP": indepipnums
}
},
upsert=True,
w=0)
region_list = list(
redata.smembers("T_region_%s_%s" % (vid, date)))
for i in region_list:
LoadSum = redata.hget("T_regiontable_%s" % (date),
"%s%s" % (vid, i))
uid_stat_t.update(
{"_id": vid},
{"$inc": {
"Geo.%s.Load" % (i): int(LoadSum)
}},
upsert=True,
w=0)
uid_vdaily_t.update(
{"_id": date},
{"$inc": {
"%s.Geo.%s.Load" % (vid, i): int(LoadSum)
}},
upsert=True,
w=0)
"""获取目前为止该视频总的载入数,播放数和每次播放完成率之和"""
res = uid_stat_t.find_one({"_id": vid}, {
"_id": 0,
"Load": 1,
"Play": 1,
"EngageSum": 1
})
vidtotalload = int(res["Load"])
vidtotalplay = int(res["Play"])
videngagesum = float(res["EngageSum"])
"""准备数据为计算截至目前该视频播放的交互记录"""
def returnVal(dict, key):
if dict.has_key(key):
return dict[key]
else:
return 0
TotalSegFlagDict = uid_stat_t.find_one({"_id": vid}, {
"_id": 0,
"HeatMapSum": 1
})["HeatMapSum"]
totalinterdata = [[int(k) * 2 - 2,
int(k) * 2,
int(v)]
for k, v in TotalSegFlagDict.items()]
totalinterdata.sort()
self.log.debug("%s prehandle hmap is %s : " %
(vid, ujson.encode(totalinterdata)))
totalinterdata = mergerPlaySeg(totalinterdata)
self.log.debug("%s befhandle hmap is %s : " %
(vid, ujson.encode(totalinterdata)))
totalplayhotpointdict = uid_stat_t.find_one({"_id": vid}, {
"_id": 0,
"PlayMapSum": 1
})["PlayMapSum"]
totalplayhotdata = [[int(k) * 2 - 2,
int(k) * 2,
int(v)]
for k, v in totalplayhotpointdict.items()]
totalplayhotdata.sort()
self.log.debug("%s prehandle pmap is %s : " %
(vid, ujson.encode(totalplayhotdata)))
totalplayhotdata = mergerPlaySeg(totalplayhotdata)
self.log.debug("%s befhandle pmap is %s : " %
(vid, ujson.encode(totalplayhotdata)))
"""average click rate of total for the vid up to now"""
"""average complete rate of total for the vid up to now"""
"""interactive data(time segment) of total for the vid up to now"""
uid_stat_t.update({"_id": vid}, {
"$set": {
"Click": operator.itruediv(vidtotalplay, vidtotalload),
"Engage": operator.itruediv(videngagesum,
vidtotalplay),
"Htmap": ujson.encode(totalinterdata),
"Ptmap": ujson.encode(totalplayhotdata)
}
},
upsert=True,
w=0)
# --/
# 统计用户每天的播放信息
# --/
"""Load nums for the uid daily"""
"""play nums for the uid daily"""
"""the number of independent IP for the uid daily"""
uid_daily_t.update({"_id": date}, {
"$inc": {
"Load": dailyload,
"Play": dailyplay,
"IP": indepipnums
}
},
upsert=True,
w=0)
# --/
# 统计用户截至目前的播放信息
# --/
"""Load nums of total for the uid up to now"""
"""play nums of total for the uid up to now"""
"""independent IPs of total for the uid up to now"""
uid_stat_t.update({"_id": uid}, {
"$inc": {
"Load": dailyload,
"Play": dailyplay,
"IP": indepipnums
}
},
upsert=True,
w=0)
# --/
# 统计用户类别每天播放信息
# --/
"""Load nums for the uid's kid daily"""
"""play nums for the uid's kid daily"""
"""the number of independent IP for the uid's kid daily"""
if kid != "nokid":
uid_type_t.update({"_id": date}, {
"$inc": {
"%s.Load" % (kid): dailyload,
"%s.Play" % (kid): dailyplay,
"%s.IP" % (kid): indepipnums
}
},
upsert=True,
w=0)
# --/
# 统计用户类别截至目前播放信息
# --/
"""Load nums of total for the uid's kid up to now"""
"""play nums of total for the uid's kid up to now"""
"""independent IPs of total for the uid's kid up to now"""
if kid != "nokid":
for i in region_list:
LoadSum = redata.hget("T_regiontable_%s" % (date),
"%s%s" % (vid, i))
uid_type_t.update({"_id": date}, {
"$inc": {
"%s.Geo.%s.Load" % (kid, i): int(LoadSum)
}
},
upsert=True,
w=0)
request.end()
"""--结束--"""
# --/
# 删除T_VIDS_$date中已完成条目
# --/
redata.srem("T_VIDS_%s" % date, vid)
self.queue.task_done()
except Exception, e:
self.log.error("handleTVIDSET -- handle -- exception,%s" % (e))
break
def run(self):
redispool = redis.ConnectionPool(host=self.redataIP, port=6379, db=0)
redata = redis.Redis(connection_pool=redispool)
mongoconn = MongoClient(self.mongodbIP, 27017)
date = self.datadate
while True:
"""KeyName Format : vid_pid"""
KeyName = self.queue.get()
vid = KeyName[0:7]
uid = vid[0:4]
"""Obtain kid for the vid"""
kid = redata.hget("T_video_kid_info", vid)
if kid == None:
"Obtain kid for the vid from mysql db"
sql = '''
select b.kid from vinfo as a left join vsort as b on a.vsortid = b.id where a.vid = %s
'''
mysql_conn = MySQLdb.connect(host=self.mysqlIP,
user=self.mysqlUser,
passwd=self.mysqlPassword,
port=self.mysqlPort,
db=self.mysqlDbname,
charset="utf8")
cursor = mysql_conn.cursor()
param = vid
n = cursor.execute(sql, param)
if n == 1:
kid = cursor.fetchall()[0][0]
if kid == None: kid = "nokid"
redata.hset("T_video_kid_info", vid, kid)
if n == 0:
kid = "nokid"
redata.hset("T_video_kid_info", vid, kid)
self.log.debug("the %s kid is %s" % (vid, kid))
"""Obtain play data for the 'KeyName'"""
redpipe = redata.pipeline()
redpipe.get(KeyName + "_S")
redpipe.get(KeyName + "_J")
PlayDataList = redpipe.execute()
"""Base data for this a play"""
starttime = ujson.decode(PlayDataList[0])["starttime"]
country = ujson.decode(PlayDataList[0])["country"]
region = str(ujson.decode(PlayDataList[0])["region"])
city = ujson.decode(PlayDataList[0])["city"]
ip = ujson.decode(PlayDataList[0])["ip"]
os = ujson.decode(PlayDataList[0])["os"]
browser = ujson.decode(PlayDataList[0])["browser"]
flow = float(ujson.decode(PlayDataList[0])["flow"])
comprate = float(ujson.decode(PlayDataList[0])["comprate"])
interdata = ujson.decode(PlayDataList[1])
playdata = [[int(i[0]) * 2 - 2,
int(i[0]) * 2,
int(i[1])] for i in interdata]
playdata = mergerPlaySeg(playdata)
"""Temp data write to redis"""
redpipe = redata.pipeline()
"""ip for every play daily,but No duplicate ip"""
"""for the number of statistically independent ip"""
redpipe.sadd("T_IPS_" + vid + "_%s" % (date), ip)
"""To facilitate the processing of the video data changs"""
redpipe.sadd("T_VIDS_%s" % (date), vid)
"""To facilitate the processing of the video access region"""
redpipe.sadd("T_region_%s_%s" % (vid, date), region)
redpipe.sadd("T_region_%s_%s" % (uid, date), region)
redpipe.execute()
"""--开始--"""
# --/
# 建立mongodb表连接
# --/
uid_stat_t = mongoconn["video"]["%s_stat" % (uid)]
vid_play_t = mongoconn["video"]["%s_play" % (vid)]
uid_vdaily_t = mongoconn["video"]["%s_vdaily" % (uid)]
uid_daily_t = mongoconn["video"]["%s_daily" % (uid)]
uid_type_t = mongoconn["video"]["%s_type" % (uid)]
request = mongoconn.start_request()
# --/
# 为加快计算,将视频,用户,类别的每天的完成率和写到redis
# --/
#
redpipe = redata.pipeline()
redpipe.hincrbyfloat("T_engagesum_%s_%s" % (uid, date), vid,
float(comprate))
redpipe.hincrbyfloat("T_engagesum_%s_%s" % (uid, date), uid,
float(comprate))
if kid != "nokid":
redpipe.hincrbyfloat("T_engagesum_%s_%s" % (uid, date), kid,
float(comprate))
redpipe.execute()
# --/
# 统计视频每次播放信息
# --/
"""获取该vid自增长id值"""
if not redata.hexists("vid_playid", vid):
"""读mongodb数据库video表uid_stat获取该vid自增长id值"""
tmpval = uid_stat_t.find_one({"_id": vid}, {
"_id": 0,
"PlayID": 1
})
if tmpval == None or len(tmpval) == 0:
playid = redata.hincrby("vid_playid", vid, 1)
else:
playid = int(tmpval["PlayID"]) + 1
redata.hset("vid_playid", vid, playid)
else:
playid = redata.hincrby("vid_playid", vid, 1)
"""statistics of a play for the vid"""
vid_play_t.update({"_id": int(playid)}, {
"$set": {
"PlayTime": starttime,
"PIP": ip,
"Engage": comprate,
"Country": country,
"Provinces": region,
"PGeo": city,
"Htmap": ujson.encode(playdata),
"OS": os,
"Browser": browser
}
},
upsert=True,
w=0)
# --/
# 统计视频每天播放信息
# --/
"""total flow for the vid daily"""
"""region distribution for the vid daily"""
uid_vdaily_t.update({"_id": date}, {
"$inc": {
"%s.Traffic" % (vid): flow,
"%s.Geo.%s.ClickNum" % (vid, region): 1,
"%s.Geo.%s.Load" % (vid, region): 0,
"%s.Geo.%s.Traffic" % (vid, region): flow,
"%s.Geo.%s.EngageSum" % (vid, region): comprate,
"%s.EngageSum" % (vid): comprate
}
},
upsert=True,
w=0)
# --/
# 统计视频截至到目前总的播放信息
# --/
"""flow of the total for the vid up to now"""
"""region distribution for the vid up to now"""
"""average complete rate sum of the vid up to now"""
uid_stat_t.update({"_id": vid}, {
"$inc": {
"Traffic": flow,
"Geo.%s.ClickNum" % (region): 1,
"Geo.%s.Load" % (region): 0,
"Geo.%s.Traffic" % (region): flow,
"Geo.%s.EngageSum" % (region): comprate,
"EngageSum": comprate
}
},
upsert=True,
w=0)
"""interactive data(time point) of total for the vid up to now"""
for interseg in interdata:
uid_stat_t.update({"_id": vid}, {
"$inc": {
"HeatMapSum.%s" % (str(interseg[0])): interseg[1],
"PlayMapSum.%s" % (str(interseg[0])): 1
}
},
upsert=True,
w=0)
# --/
# 统计用户每天播放信息
# --/
"""total flow for the uid daily"""
"""region distribution for the uid daily"""
uid_daily_t.update({"_id": date}, {
"$inc": {
"Traffic": flow,
"Geo.%s.ClickNum" % (region): 1,
"Geo.%s.Load" % (region): 0,
"Geo.%s.Traffic" % (region): flow,
"Geo.%s.EngageSum" % (region): comprate,
"EngageSum": comprate
}
},
upsert=True,
w=0)
# --/
# 统计用户截至到目前总的播放信息
# --/
"""flow of the total for the uid up to now"""
"""region distribution for the uid up to now"""
uid_stat_t.update({"_id": uid}, {
"$inc": {
"Traffic": flow,
"Geo.%s.Load" % (region): 0,
"Geo.%s.ClickNum" % (region): 1,
"Geo.%s.Traffic" % (region): flow,
"Geo.%s.EngageSum" % (region): comprate,
"Mtraffic": flow,
"EngageSum": comprate
}
},
upsert=True,
w=0)
# --/
# 统计用户类别每天播放信息
# --/
"""total flow for the uid's kid daily"""
"""region distribution for the uid's kid daily"""
if kid != "nokid":
uid_type_t.update({"_id": date}, {
"$inc": {
"%s.Traffic" % (kid): flow,
"%s.Geo.%s.ClickNum" % (kid, region): 1,
"%s.Geo.%s.Load" % (kid, region): 0,
"%s.Geo.%s.Traffic" % (kid, region): flow,
"%s.Geo.%s.EngageSum" % (kid, region): comprate,
"%s.EngageSum" % (kid): comprate
}
},
upsert=True,
w=0)
request.end()
# -- /
# 删除T_endlist_$date中已处理完成的条目
# --/
redata.lrem("T_endlist_%s" % date, KeyName, 0)
"""--结束--"""
self.queue.task_done()
def run(self):
redispool = redis.ConnectionPool(host=self.redataIP,port=6379,db=0)
redata = redis.Redis(connection_pool=redispool)
mongoconn = MongoClient(self.mongodbIP,27017)
date = self.datadate
while True:
try:
"""Obtain vid name for have data changs daily"""
vid = self.queue.get()
"""Obtain uid for the vid"""
uid = vid[0:4]
self.log.debug("handleTVIDSET -- Handling is %s"%vid)
"""Obtain kid from redis db for the vid"""
kid = redata.hget("T_video_kid_info",vid)
"""Obtain temp data from redis"""
redpipe = redata.pipeline()
"""Obtain the number of independent IP for the vid daily"""
redpipe.scard("T_IPS_"+vid+"_%s"%(date))
"""Obtain load numbers for the vid daily"""
redpipe.hget("T_loadtable_%s"%(date),vid)
"""Obtain play numbers for the vid daily"""
redpipe.hget("T_playtable_%s"%(date),vid)
"""Write uid to set for facilitate the processing after"""
redpipe.sadd("T_UIDS_%s"%(date),uid)
vidDataList = redpipe.execute()
indepipnums = int(vidDataList[0])
dailyload = int(vidDataList[1])
dailyplay = int(vidDataList[2])
"""--开始--"""
# --/
# 建立mongodb表连接
# --/
uid_stat_t = mongoconn["video"]["%s_stat"%(uid)]
uid_vdaily_t = mongoconn["video"]["%s_vdaily"%(uid)]
uid_daily_t = mongoconn["video"]["%s_daily"%(uid)]
uid_type_t = mongoconn["video"]["%s_type"%(uid)]
request = mongoconn.start_request()
# --/
# 保存该视频playid去mongodb
# --/
playid = redata.hget("vid_playid",vid)
uid_stat_t.update({"_id":vid},{"$set":{"PlayID":int(playid)}},upsert=True,w=0)
# --/
# 统计视频每天播放信息
# --/
"""total loads of the vid daily"""
"""total plays of the vid daily"""
"""average click rate of the vid daily"""
"""the number of independent IP for the vid daily"""
"""avarage complete rate for the vid daily"""
uid_vdaily_t.update({"_id":date},{"$set":{
"%s.Load"%(vid):dailyload,
"%s.Play"%(vid):dailyplay,
"%s.Click"%(vid):operator.itruediv(dailyplay, dailyload),
"%s.IP"%(vid):indepipnums,
"%s.Engage"%(vid):operator.itruediv(float(redata.hget("T_engagesum_%s_%s"%(uid,date),vid)),dailyplay)
}},upsert=True,w=0)
# --/
# 统计视频截至到目前总的播放信息
# --/
"""Load nums of total for the vid up to now"""
"""play nums of total for the vid up to now"""
"""independent IPs of total for the vid up to now"""
uid_stat_t.update({"_id":vid},{"$inc":{
"Load":dailyload,
"Play":dailyplay,
"IP":indepipnums
}},upsert=True,w=0)
region_list = list(redata.smembers("T_region_%s_%s"%(vid,date)))
for i in region_list:
LoadSum = redata.hget("T_regiontable_%s"%(date),"%s%s"%(vid,i))
uid_stat_t.update({"_id":vid},{"$inc":{
"Geo.%s.Load"%(i):int(LoadSum)
}},upsert=True,w=0)
uid_vdaily_t.update({"_id":date},{"$inc":{
"%s.Geo.%s.Load"%(vid,i):int(LoadSum)
}},upsert=True,w=0)
"""获取目前为止该视频总的载入数,播放数和每次播放完成率之和"""
res = uid_stat_t.find_one({"_id":vid},{"_id":0,"Load":1,"Play":1,"EngageSum":1})
vidtotalload = int(res["Load"])
vidtotalplay = int(res["Play"])
videngagesum = float(res["EngageSum"])
"""准备数据为计算截至目前该视频播放的交互记录"""
def returnVal(dict,key):
if dict.has_key(key):
return dict[key]
else:
return 0
TotalSegFlagDict = uid_stat_t.find_one({"_id":vid},{"_id":0,"HeatMapSum":1})["HeatMapSum"]
totalinterdata = [[int(k)*2-2,int(k)*2,int(v)] for k,v in TotalSegFlagDict.items()]
totalinterdata.sort()
self.log.debug("%s prehandle hmap is %s : "%(vid,ujson.encode(totalinterdata)))
totalinterdata = mergerPlaySeg(totalinterdata)
self.log.debug("%s befhandle hmap is %s : "%(vid,ujson.encode(totalinterdata)))
totalplayhotpointdict = uid_stat_t.find_one({"_id":vid},{"_id":0,"PlayMapSum":1})["PlayMapSum"]
totalplayhotdata = [[int(k)*2-2,int(k)*2,int(v)] for k,v in totalplayhotpointdict.items()]
totalplayhotdata.sort()
self.log.debug("%s prehandle pmap is %s : "%(vid,ujson.encode(totalplayhotdata)))
totalplayhotdata = mergerPlaySeg(totalplayhotdata)
self.log.debug("%s befhandle pmap is %s : "%(vid,ujson.encode(totalplayhotdata)))
"""average click rate of total for the vid up to now"""
"""average complete rate of total for the vid up to now"""
"""interactive data(time segment) of total for the vid up to now"""
uid_stat_t.update({"_id":vid},{"$set":{
"Click":operator.itruediv(vidtotalplay,vidtotalload),
"Engage":operator.itruediv(videngagesum,vidtotalplay),
"Htmap":ujson.encode(totalinterdata),
"Ptmap":ujson.encode(totalplayhotdata)
}},upsert=True,w=0)
# --/
# 统计用户每天的播放信息
# --/
"""Load nums for the uid daily"""
"""play nums for the uid daily"""
"""the number of independent IP for the uid daily"""
uid_daily_t.update({"_id":date},{"$inc":{"Load":dailyload,"Play":dailyplay,"IP":indepipnums}},upsert=True,w=0)
# --/
# 统计用户截至目前的播放信息
# --/
"""Load nums of total for the uid up to now"""
"""play nums of total for the uid up to now"""
"""independent IPs of total for the uid up to now"""
uid_stat_t.update({"_id":uid},{"$inc":{"Load":dailyload,"Play":dailyplay,"IP":indepipnums}},upsert=True,w=0)
# --/
# 统计用户类别每天播放信息
# --/
"""Load nums for the uid's kid daily"""
"""play nums for the uid's kid daily"""
"""the number of independent IP for the uid's kid daily"""
if kid != "nokid":
uid_type_t.update({"_id":date},{"$inc":{"%s.Load"%(kid):dailyload,"%s.Play"%(kid):dailyplay,"%s.IP"%(kid):indepipnums}},upsert=True,w=0)
# --/
# 统计用户类别截至目前播放信息
# --/
"""Load nums of total for the uid's kid up to now"""
"""play nums of total for the uid's kid up to now"""
"""independent IPs of total for the uid's kid up to now"""
if kid != "nokid":
for i in region_list:
LoadSum = redata.hget("T_regiontable_%s"%(date),"%s%s"%(vid,i))
uid_type_t.update({"_id":date},{"$inc":{
"%s.Geo.%s.Load"%(kid,i):int(LoadSum)
}},upsert=True,w=0)
request.end()
"""--结束--"""
# --/
# 删除T_VIDS_$date中已完成条目
# --/
redata.srem("T_VIDS_%s"%date,vid)
self.queue.task_done()
except Exception,e:
self.log.error("handleTVIDSET -- handle -- exception,%s"%(e))
break
def run(self):
redispool = redis.ConnectionPool(host=self.redataIP,port=6379,db=0)
redata = redis.Redis(connection_pool=redispool)
mongoconn = MongoClient(self.mongodbIP,27017)
date = self.datadate
while True:
"""KeyName Format : vid_pid"""
KeyName = self.queue.get()
vid = KeyName[0:7]
uid = vid[0:4]
"""Obtain kid for the vid"""
kid = redata.hget("T_video_kid_info",vid)
if kid == None:
"Obtain kid for the vid from mysql db"
sql = '''
select b.kid from vinfo as a left join vsort as b on a.vsortid = b.id where a.vid = %s
'''
mysql_conn = MySQLdb.connect(host=self.mysqlIP,user=self.mysqlUser,passwd=self.mysqlPassword,port=self.mysqlPort,db=self.mysqlDbname,charset="utf8")
cursor = mysql_conn.cursor()
param = vid
n = cursor.execute(sql,param)
if n == 1:
kid = cursor.fetchall()[0][0]
if kid == None:kid = "nokid"
redata.hset("T_video_kid_info",vid,kid)
if n == 0:
kid = "nokid"
redata.hset("T_video_kid_info",vid,kid)
self.log.debug("the %s kid is %s" % (vid,kid))
"""Obtain play data for the 'KeyName'"""
redpipe = redata.pipeline()
redpipe.get(KeyName+"_S")
redpipe.get(KeyName+"_J")
PlayDataList = redpipe.execute()
"""Base data for this a play"""
starttime = ujson.decode(PlayDataList[0])["starttime"]
country = ujson.decode(PlayDataList[0])["country"]
region = str(ujson.decode(PlayDataList[0])["region"])
city = ujson.decode(PlayDataList[0])["city"]
ip = ujson.decode(PlayDataList[0])["ip"]
os = ujson.decode(PlayDataList[0])["os"]
browser = ujson.decode(PlayDataList[0])["browser"]
flow = float(ujson.decode(PlayDataList[0])["flow"])
comprate = float(ujson.decode(PlayDataList[0])["comprate"])
interdata = ujson.decode(PlayDataList[1])
playdata = [[int(i[0])*2-2,int(i[0])*2,int(i[1])] for i in interdata]
playdata = mergerPlaySeg(playdata)
"""Temp data write to redis"""
redpipe = redata.pipeline()
"""ip for every play daily,but No duplicate ip"""
"""for the number of statistically independent ip"""
redpipe.sadd("T_IPS_"+vid+"_%s"%(date),ip)
"""To facilitate the processing of the video data changs"""
redpipe.sadd("T_VIDS_%s"%(date),vid)
"""To facilitate the processing of the video access region"""
redpipe.sadd("T_region_%s_%s"%(vid,date),region)
redpipe.sadd("T_region_%s_%s"%(uid,date),region)
redpipe.execute()
"""--开始--"""
# --/
# 建立mongodb表连接
# --/
uid_stat_t = mongoconn["video"]["%s_stat"%(uid)]
vid_play_t = mongoconn["video"]["%s_play"%(vid)]
uid_vdaily_t = mongoconn["video"]["%s_vdaily"%(uid)]
uid_daily_t = mongoconn["video"]["%s_daily"%(uid)]
uid_type_t = mongoconn["video"]["%s_type"%(uid)]
request = mongoconn.start_request()
# --/
# 为加快计算,将视频,用户,类别的每天的完成率和写到redis
# --/
#
redpipe = redata.pipeline()
redpipe.hincrbyfloat("T_engagesum_%s_%s"%(uid,date),vid,float(comprate))
redpipe.hincrbyfloat("T_engagesum_%s_%s"%(uid,date),uid,float(comprate))
if kid != "nokid" :
redpipe.hincrbyfloat("T_engagesum_%s_%s"%(uid,date),kid,float(comprate))
redpipe.execute()
# --/
# 统计视频每次播放信息
# --/
"""获取该vid自增长id值"""
if not redata.hexists("vid_playid",vid):
"""读mongodb数据库video表uid_stat获取该vid自增长id值"""
tmpval = uid_stat_t.find_one({"_id":vid},{"_id":0,"PlayID":1})
if tmpval == None or len(tmpval) == 0:
playid = redata.hincrby("vid_playid",vid,1)
else:
playid = int(tmpval["PlayID"])+1
redata.hset("vid_playid",vid,playid)
else:
playid = redata.hincrby("vid_playid",vid,1)
"""statistics of a play for the vid"""
vid_play_t.update({"_id":int(playid)},{"$set":{
"PlayTime":starttime,
"PIP":ip,
"Engage":comprate,
"Country":country,
"Provinces":region,
"PGeo":city,
"Htmap":ujson.encode(playdata),
"OS":os,
"Browser":browser
}},upsert=True,w=0)
# --/
# 统计视频每天播放信息
# --/
"""total flow for the vid daily"""
"""region distribution for the vid daily"""
uid_vdaily_t.update({"_id":date},{"$inc":{
"%s.Traffic"%(vid):flow,
"%s.Geo.%s.ClickNum"%(vid,region):1,
"%s.Geo.%s.Load"%(vid,region):0,
"%s.Geo.%s.Traffic"%(vid,region):flow,
"%s.Geo.%s.EngageSum"%(vid,region):comprate,
"%s.EngageSum"%(vid):comprate
}},upsert=True,w=0)
# --/
# 统计视频截至到目前总的播放信息
# --/
"""flow of the total for the vid up to now"""
"""region distribution for the vid up to now"""
"""average complete rate sum of the vid up to now"""
uid_stat_t.update({"_id":vid},{"$inc":{
"Traffic":flow,
"Geo.%s.ClickNum"%(region):1,
"Geo.%s.Load"%(region):0,
"Geo.%s.Traffic"%(region):flow,
"Geo.%s.EngageSum"%(region):comprate,
"EngageSum":comprate
}},upsert=True,w=0)
"""interactive data(time point) of total for the vid up to now"""
for interseg in interdata:
uid_stat_t.update({"_id":vid},{"$inc":{"HeatMapSum.%s"%(str(interseg[0])):interseg[1],"PlayMapSum.%s"%(str(interseg[0])):1}},upsert=True,w=0)
# --/
# 统计用户每天播放信息
# --/
"""total flow for the uid daily"""
"""region distribution for the uid daily"""
uid_daily_t.update({"_id":date},{"$inc":{
"Traffic":flow,
"Geo.%s.ClickNum"%(region):1,
"Geo.%s.Load"%(region):0,
"Geo.%s.Traffic"%(region):flow,
"Geo.%s.EngageSum"%(region):comprate,
"EngageSum":comprate
}},upsert=True,w=0)
# --/
# 统计用户截至到目前总的播放信息
# --/
"""flow of the total for the uid up to now"""
"""region distribution for the uid up to now"""
uid_stat_t.update({"_id":uid},{"$inc":{
"Traffic":flow,
"Geo.%s.Load"%(region):0,
"Geo.%s.ClickNum"%(region):1,
"Geo.%s.Traffic"%(region):flow,
"Geo.%s.EngageSum"%(region):comprate,
"Mtraffic":flow,
"EngageSum":comprate
}},upsert=True,w=0)
# --/
# 统计用户类别每天播放信息
# --/
"""total flow for the uid's kid daily"""
"""region distribution for the uid's kid daily"""
if kid != "nokid":
uid_type_t.update({"_id":date},{"$inc":{
"%s.Traffic"%(kid):flow,
"%s.Geo.%s.ClickNum"%(kid,region):1,
"%s.Geo.%s.Load"%(kid,region):0,
"%s.Geo.%s.Traffic"%(kid,region):flow,
"%s.Geo.%s.EngageSum"%(kid,region):comprate,
"%s.EngageSum"%(kid):comprate
}},upsert=True,w=0)
request.end()
# -- /
# 删除T_endlist_$date中已处理完成的条目
# --/
redata.lrem("T_endlist_%s"%date,KeyName,0)
"""--结束--"""
self.queue.task_done()
def run(self):
redispool = redis.ConnectionPool(host=self.redataIP, port=6379, db=0)
redata = redis.Redis(connection_pool=redispool)
mongoconn = MongoClient(self.mongodbIP, 27017)
date = self.datadate
while True:
"""KeyName Format : vid_pid"""
KeyName = self.queue.get()
vid = KeyName[0:7]
uid = vid[0:4]
"""Obtain play data for the 'KeyName'"""
redpipe = redata.pipeline()
redpipe.get(KeyName + "_S")
redpipe.get(KeyName + "_J")
redpipe.get(KeyName + "_z1")
PlayDataList = redpipe.execute()
"""Base data for this a play"""
starttime = ujson.decode(PlayDataList[0])["starttime"]
country = ujson.decode(PlayDataList[0])["country"]
region = str(ujson.decode(PlayDataList[0])["region"])
city = ujson.decode(PlayDataList[0])["city"]
ip = ujson.decode(PlayDataList[0])["ip"]
os = ujson.decode(PlayDataList[0])["os"]
browser = ujson.decode(PlayDataList[0])["browser"]
flow = float(ujson.decode(PlayDataList[0])["flow"])
comprate = float(ujson.decode(PlayDataList[0])["comprate"])
interdata = ujson.decode(PlayDataList[1])
playdata = [[int(i[0]) * 2 - 2,
int(i[0]) * 2,
int(i[1])] for i in interdata]
playdata = mergerPlaySeg(playdata)
"""Temp data write to redis"""
redpipe = redata.pipeline()
"""ip for every play daily,but No duplicate ip"""
"""for the number of statistically independent ip"""
redpipe.sadd("T_IPS_" + vid + "_%s" % (date), ip)
"""To facilitate the processing of the video data changs"""
redpipe.sadd("T_VIDS_%s" % (date), vid)
"""To facilitate the processing of the video access region"""
redpipe.sadd("T_region_%s_%s" % (vid, date), region)
redpipe.sadd("T_region_%s_%s" % (uid, date), region)
redpipe.execute()
"""--开始--"""
# --/
# 建立mongodb表连接
# --/
uid_stat_t = mongoconn[self.mongodbName]["%s_stat" % (uid)]
vid_play_t = mongoconn[self.mongodbName]["%s_play" % (vid)]
uid_vdaily_t = mongoconn[self.mongodbName]["%s_vdaily" % (uid)]
uid_daily_t = mongoconn[self.mongodbName]["%s_daily" % (uid)]
uid_type_t = mongoconn[self.mongodbName]["%s_type" % (uid)]
request = mongoconn.start_request()
# --/
# 为加快计算,将视频,用户,类别的每天的完成率和写到redis
# --/
#
redpipe = redata.pipeline()
redpipe.hincrbyfloat("T_engagesum_%s_%s" % (uid, date), vid,
float(comprate))
redpipe.hincrbyfloat("T_engagesum_%s_%s" % (uid, date), uid,
float(comprate))
redpipe.execute()
# --/
# 统计视频每次播放信息
# --/
"""获取该vid自增长id值"""
if not redata.hexists("vid_playid", vid):
"""读mongodb数据库video表uid_stat获取该vid自增长id值"""
tmpval = uid_stat_t.find_one({"_id": vid}, {
"_id": 0,
"PlayID": 1
})
if tmpval == None or len(tmpval) == 0:
playid = redata.hincrby("vid_playid", vid, 1)
else:
playid = int(tmpval["PlayID"]) + 1
redata.hset("vid_playid", vid, playid)
else:
playid = redata.hincrby("vid_playid", vid, 1)
"""statistics of a play for the vid"""
vid_play_t.update({"_id": int(playid)}, {
"$set": {
"Date": date,
"PlayTime": starttime,
"PIP": ip,
"Engage": comprate,
"Country": country,
"Provinces": region,
"PGeo": city,
"Htmap": ujson.encode(playdata),
"OS": os,
"Browser": browser
}
},
upsert=True,
w=0)
# --/
# FUNCTION MODULE : COLLECT USER BASE INFORMATION
# WRITE USER INFO TO MONGODB
# ADD IN 20131210
# --/
if PlayDataList[2] != None:
zz_uinfo = PlayDataList[2]
vid_play_t.update(
{"_id": int(playid)},
{"$set": {
"zz_uinfo": ujson.encode(zz_uinfo)
}},
upsert=True,
w=0)
# --/
# 统计视频每天播放信息
# --/
"""total flow for the vid daily"""
"""region distribution for the vid daily"""
uid_vdaily_t.update({"_id": date}, {
"$inc": {
"%s.Traffic" % (vid): flow,
"%s.Geo.%s.ClickNum" % (vid, region): 1,
"%s.Geo.%s.Load" % (vid, region): 0,
"%s.Geo.%s.Traffic" % (vid, region): flow,
"%s.Geo.%s.EngageSum" % (vid, region): comprate,
"%s.EngageSum" % (vid): comprate
}
},
upsert=True,
w=0)
# --/
# 统计视频截至到目前总的播放信息
# --/
"""flow of the total for the vid up to now"""
"""region distribution for the vid up to now"""
"""average complete rate sum of the vid up to now"""
uid_stat_t.update({"_id": vid}, {
"$inc": {
"Traffic": flow,
"Geo.%s.ClickNum" % (region): 1,
"Geo.%s.Load" % (region): 0,
"Geo.%s.Traffic" % (region): flow,
"Geo.%s.EngageSum" % (region): comprate,
"EngageSum": comprate
}
},
upsert=True,
w=0)
"""interactive data(time point) of total for the vid up to now"""
for interseg in interdata:
uid_stat_t.update({"_id": vid}, {
"$inc": {
"HeatMapSum.%s" % (str(interseg[0])): interseg[1],
"PlayMapSum.%s" % (str(interseg[0])): 1
}
},
upsert=True,
w=0)
# --/
# 统计用户每天播放信息
# --/
"""total flow for the uid daily"""
"""region distribution for the uid daily"""
uid_daily_t.update({"_id": date}, {
"$inc": {
"Traffic": flow,
"Geo.%s.ClickNum" % (region): 1,
"Geo.%s.Load" % (region): 0,
"Geo.%s.Traffic" % (region): flow,
"Geo.%s.EngageSum" % (region): comprate,
"EngageSum": comprate
}
},
upsert=True,
w=0)
# --/
# 统计用户截至到目前总的播放信息
# --/
"""flow of the total for the uid up to now"""
"""region distribution for the uid up to now"""
uid_stat_t.update({"_id": uid}, {
"$inc": {
"Traffic": flow,
"Geo.%s.Load" % (region): 0,
"Geo.%s.ClickNum" % (region): 1,
"Geo.%s.Traffic" % (region): flow,
"Geo.%s.EngageSum" % (region): comprate,
"Mtraffic": flow,
"EngageSum": comprate
}
},
upsert=True,
w=0)
request.end()
# -- /
# 删除T_endlist_$date中已处理完成的条目
# --/
redata.lrem("T_endlist_%s" % date, KeyName, 0)
"""--结束--"""
self.queue.task_done()
def run(self):
redispool = redis.ConnectionPool(host=self.redataIP,port=6379,db=0)
redata = redis.Redis(connection_pool=redispool)
mongoconn = MongoClient(self.mongodbIP,27017)
date = self.datadate
while True:
"""KeyName Format : vid_pid"""
KeyName = self.queue.get()
vid = KeyName[0:7]
uid = vid[0:4]
"""Obtain play data for the 'KeyName'"""
redpipe = redata.pipeline()
redpipe.get(KeyName+"_S")
redpipe.get(KeyName+"_J")
redpipe.get(KeyName+"_z1")
PlayDataList = redpipe.execute()
"""Base data for this a play"""
starttime = ujson.decode(PlayDataList[0])["starttime"]
country = ujson.decode(PlayDataList[0])["country"]
region = str(ujson.decode(PlayDataList[0])["region"])
city = ujson.decode(PlayDataList[0])["city"]
ip = ujson.decode(PlayDataList[0])["ip"]
os = ujson.decode(PlayDataList[0])["os"]
browser = ujson.decode(PlayDataList[0])["browser"]
flow = float(ujson.decode(PlayDataList[0])["flow"])
comprate = float(ujson.decode(PlayDataList[0])["comprate"])
interdata = ujson.decode(PlayDataList[1])
playdata = [[int(i[0])*2-2,int(i[0])*2,int(i[1])] for i in interdata]
playdata = mergerPlaySeg(playdata)
"""Temp data write to redis"""
redpipe = redata.pipeline()
"""ip for every play daily,but No duplicate ip"""
"""for the number of statistically independent ip"""
redpipe.sadd("T_IPS_"+vid+"_%s"%(date),ip)
"""To facilitate the processing of the video data changs"""
redpipe.sadd("T_VIDS_%s"%(date),vid)
"""To facilitate the processing of the video access region"""
redpipe.sadd("T_region_%s_%s"%(vid,date),region)
redpipe.sadd("T_region_%s_%s"%(uid,date),region)
redpipe.execute()
"""--开始--"""
# --/
# 建立mongodb表连接
# --/
uid_stat_t = mongoconn[self.mongodbName]["%s_stat"%(uid)]
vid_play_t = mongoconn[self.mongodbName]["%s_play"%(vid)]
uid_vdaily_t = mongoconn[self.mongodbName]["%s_vdaily"%(uid)]
uid_daily_t = mongoconn[self.mongodbName]["%s_daily"%(uid)]
uid_type_t = mongoconn[self.mongodbName]["%s_type"%(uid)]
request = mongoconn.start_request()
# --/
# 为加快计算,将视频,用户,类别的每天的完成率和写到redis
# --/
#
redpipe = redata.pipeline()
redpipe.hincrbyfloat("T_engagesum_%s_%s"%(uid,date),vid,float(comprate))
redpipe.hincrbyfloat("T_engagesum_%s_%s"%(uid,date),uid,float(comprate))
redpipe.execute()
# --/
# 统计视频每次播放信息
# --/
"""获取该vid自增长id值"""
if not redata.hexists("vid_playid",vid):
"""读mongodb数据库video表uid_stat获取该vid自增长id值"""
tmpval = uid_stat_t.find_one({"_id":vid},{"_id":0,"PlayID":1})
if tmpval == None or len(tmpval) == 0:
playid = redata.hincrby("vid_playid",vid,1)
else:
playid = int(tmpval["PlayID"])+1
redata.hset("vid_playid",vid,playid)
else:
playid = redata.hincrby("vid_playid",vid,1)
"""statistics of a play for the vid"""
vid_play_t.update({"_id":int(playid)},{"$set":{
"Date":date,
"PlayTime":starttime,
"PIP":ip,
"Engage":comprate,
"Country":country,
"Provinces":region,
"PGeo":city,
"Htmap":ujson.encode(playdata),
"OS":os,
"Browser":browser
}},upsert=True,w=0)
# --/
# FUNCTION MODULE : COLLECT USER BASE INFORMATION
# WRITE USER INFO TO MONGODB
# ADD IN 20131210
# --/
if PlayDataList[2] != None:
zz_uinfo = PlayDataList[2]
vid_play_t.update({"_id":int(playid)},{"$set":{
"zz_uinfo":ujson.encode(zz_uinfo)
}},upsert=True,w=0)
# --/
# 统计视频每天播放信息
# --/
"""total flow for the vid daily"""
"""region distribution for the vid daily"""
uid_vdaily_t.update({"_id":date},{"$inc":{
"%s.Traffic"%(vid):flow,
"%s.Geo.%s.ClickNum"%(vid,region):1,
"%s.Geo.%s.Load"%(vid,region):0,
"%s.Geo.%s.Traffic"%(vid,region):flow,
"%s.Geo.%s.EngageSum"%(vid,region):comprate,
"%s.EngageSum"%(vid):comprate
}},upsert=True,w=0)
# --/
# 统计视频截至到目前总的播放信息
# --/
"""flow of the total for the vid up to now"""
"""region distribution for the vid up to now"""
"""average complete rate sum of the vid up to now"""
uid_stat_t.update({"_id":vid},{"$inc":{
"Traffic":flow,
"Geo.%s.ClickNum"%(region):1,
"Geo.%s.Load"%(region):0,
"Geo.%s.Traffic"%(region):flow,
"Geo.%s.EngageSum"%(region):comprate,
"EngageSum":comprate
}},upsert=True,w=0)
"""interactive data(time point) of total for the vid up to now"""
for interseg in interdata:
uid_stat_t.update({"_id":vid},{"$inc":{"HeatMapSum.%s"%(str(interseg[0])):interseg[1],"PlayMapSum.%s"%(str(interseg[0])):1}},upsert=True,w=0)
# --/
# 统计用户每天播放信息
# --/
"""total flow for the uid daily"""
"""region distribution for the uid daily"""
uid_daily_t.update({"_id":date},{"$inc":{
"Traffic":flow,
"Geo.%s.ClickNum"%(region):1,
"Geo.%s.Load"%(region):0,
"Geo.%s.Traffic"%(region):flow,
"Geo.%s.EngageSum"%(region):comprate,
"EngageSum":comprate
}},upsert=True,w=0)
# --/
# 统计用户截至到目前总的播放信息
# --/
"""flow of the total for the uid up to now"""
"""region distribution for the uid up to now"""
uid_stat_t.update({"_id":uid},{"$inc":{
"Traffic":flow,
"Geo.%s.Load"%(region):0,
"Geo.%s.ClickNum"%(region):1,
"Geo.%s.Traffic"%(region):flow,
"Geo.%s.EngageSum"%(region):comprate,
"Mtraffic":flow,
"EngageSum":comprate
}},upsert=True,w=0)
request.end()
# -- /
# 删除T_endlist_$date中已处理完成的条目
# --/
redata.lrem("T_endlist_%s"%date,KeyName,0)
"""--结束--"""
self.queue.task_done()
