def operator_itruediv(size):
a = Array(size, 'int32')
b = Array(size, 'int32')
for i in range(size):
a[i] = nb_types.int32(i+10)
b[i] = nb_types.int32(i+3)
operator.itruediv(a, b)
return a
def operator_itruediv2(size):
a = Array(size, 'double')
b = Array(size, 'double')
for i in range(size):
a[i] = nb_types.double(i+10)
b[i] = nb_types.double(i+3)
operator.itruediv(a, b)
return a
def test_inplace(self):
#operator = self.module
class C(object):
def __iadd__ (self, other): return "iadd"
def __iand__ (self, other): return "iand"
def __ifloordiv__(self, other): return "ifloordiv"
def __ilshift__ (self, other): return "ilshift"
def __imod__ (self, other): return "imod"
def __imul__ (self, other): return "imul"
def __ior__ (self, other): return "ior"
def __ipow__ (self, other): return "ipow"
def __irshift__ (self, other): return "irshift"
def __isub__ (self, other): return "isub"
def __itruediv__ (self, other): return "itruediv"
def __ixor__ (self, other): return "ixor"
def __getitem__(self, other): return 5 # so that C is a sequence
c = C()
self.assertEqual(operator.iadd (c, 5), "iadd")
self.assertEqual(operator.iand (c, 5), "iand")
self.assertEqual(operator.ifloordiv(c, 5), "ifloordiv")
self.assertEqual(operator.ilshift (c, 5), "ilshift")
self.assertEqual(operator.imod (c, 5), "imod")
self.assertEqual(operator.imul (c, 5), "imul")
self.assertEqual(operator.ior (c, 5), "ior")
self.assertEqual(operator.ipow (c, 5), "ipow")
self.assertEqual(operator.irshift (c, 5), "irshift")
self.assertEqual(operator.isub (c, 5), "isub")
self.assertEqual(operator.itruediv (c, 5), "itruediv")
self.assertEqual(operator.ixor (c, 5), "ixor")
self.assertEqual(operator.iconcat (c, c), "iadd")
def main():
x,y = 5,6
a,b = 7,8
# Incremental Add : x += y
iAdd = operator.iadd(5, 6)
print("iAdd: ", iAdd)
# Incremental Concatenate
iConCat = operator.iconcat("Hello", "World")
print("iConCat:", iConCat)
# Incremental Subtraction
iSub = operator.isub(5, 6)
print("iSub: ", iSub)
# Incremental Multiplication
iMul = operator.imul(5,6)
print("iMul:", iMul)
# Incremental Division
iDiv = operator.itruediv(10, 5)
print("iDiv: ", iDiv)
# Incremental Modulus
iMod = operator.imod(10, 6)
print("iMod: ", iMod)
# Incremental Exponential
iPow = operator.ipow(2, 4)
print("iPow: ", iPow)
def test_itruediv__on_int(self):
""" Verify that (true) /= works on proxy[int]. """
a = b = proxy(7)
b = operator.itruediv(b, 2)
self.assertEqual(a, 7)
self.assertEqual(b, 3.5)
self.assertTrue(issubclass(type(a), proxy))
self.assertTrue(issubclass(type(b), proxy))
def get_pair_stats(self, image_1, image_2):
"""Get the mean and varience from a pair of flats"""
fmean1 = self.mean(image_1)
fmean2 = self.mean(image_2)
fratio_im = afwImage.MaskedImageF(image_1, True)
fcopy_im = afwImage.MaskedImageF(image_2, True)
operator.itruediv(fratio_im, image_2)
fratio = self.mean(fratio_im)
fcopy_im *= fratio
fmean = (fmean1 + fmean2)/2.
fcorrmean = (fmean1 + self.mean(fcopy_im))/2.
fdiff = afwImage.MaskedImageF(image_1, True)
fdiff -= fcopy_im
fvar = self.var(fdiff)/2.
return (fratio, fmean, fcorrmean, fvar, fmean1, fmean2)
def test_inplace_truedivision(self, input_tuple, expected):
self.ureg.autoconvert_offset_to_baseunit = False
(q1v, q1u), (q2v, q2u) = input_tuple
# update input tuple with new values to have correct values on failure
input_tuple = ((np.array([q1v]*2, dtype=np.float), q1u),
(np.array([q2v]*2, dtype=np.float), q2u))
Q_ = self.Q_
qin1, qin2 = input_tuple
q1, q2 = Q_(*qin1), Q_(*qin2)
q1_cp = copy.copy(q1)
if expected == 'error':
self.assertRaises(OffsetUnitCalculusError, op.itruediv, q1_cp, q2)
else:
expected = np.array([expected[0]]*2, dtype=np.float), expected[1]
self.assertEqual(op.itruediv(q1_cp, q2).units, Q_(*expected).units)
q1_cp = copy.copy(q1)
self.assertQuantityAlmostEqual(op.itruediv(q1_cp, q2),
Q_(*expected), atol=0.01)
def test_inplace(self):
#operator = self.module
class C(object):
def __iadd__(self, other):
return "iadd"
def __iand__(self, other):
return "iand"
def __ifloordiv__(self, other):
return "ifloordiv"
def __ilshift__(self, other):
return "ilshift"
def __imod__(self, other):
return "imod"
def __imul__(self, other):
return "imul"
def __ior__(self, other):
return "ior"
def __ipow__(self, other):
return "ipow"
def __irshift__(self, other):
return "irshift"
def __isub__(self, other):
return "isub"
def __itruediv__(self, other):
return "itruediv"
def __ixor__(self, other):
return "ixor"
def __getitem__(self, other):
return 5 # so that C is a sequence
c = C()
self.assertEqual(operator.iadd(c, 5), "iadd")
self.assertEqual(operator.iand(c, 5), "iand")
self.assertEqual(operator.ifloordiv(c, 5), "ifloordiv")
self.assertEqual(operator.ilshift(c, 5), "ilshift")
self.assertEqual(operator.imod(c, 5), "imod")
self.assertEqual(operator.imul(c, 5), "imul")
self.assertEqual(operator.ior(c, 5), "ior")
self.assertEqual(operator.ipow(c, 5), "ipow")
self.assertEqual(operator.irshift(c, 5), "irshift")
self.assertEqual(operator.isub(c, 5), "isub")
self.assertEqual(operator.itruediv(c, 5), "itruediv")
self.assertEqual(operator.ixor(c, 5), "ixor")
self.assertEqual(operator.iconcat(c, c), "iadd")
def test_inplace_truedivision(self, input_tuple, expected):
self.ureg.autoconvert_offset_to_baseunit = False
(q1v, q1u), (q2v, q2u) = input_tuple
# update input tuple with new values to have correct values on failure
input_tuple = ((np.array([q1v] * 2, dtype=np.float), q1u),
(np.array([q2v] * 2, dtype=np.float), q2u))
Q_ = self.Q_
qin1, qin2 = input_tuple
q1, q2 = Q_(*qin1), Q_(*qin2)
q1_cp = copy.copy(q1)
if expected == 'error':
self.assertRaises(OffsetUnitCalculusError, op.itruediv, q1_cp, q2)
else:
expected = np.array([expected[0]] * 2, dtype=np.float), expected[1]
self.assertEqual(op.itruediv(q1_cp, q2).units, Q_(*expected).units)
q1_cp = copy.copy(q1)
self.assertQuantityAlmostEqual(op.itruediv(q1_cp, q2),
Q_(*expected),
atol=0.01)
def flat_pair_stats(ccd1, ccd2, amp, mask_files=(), bias_frame=None):
if ccd1.md.get('EXPTIME') != ccd2.md.get('EXPTIME'):
raise RuntimeError("Exposure times for files %s, %s do not match" %
(ccd1.imfile, ccd2.imfile))
#
# Mean and variance calculations that account for masks (via
# ccd1.stat_ctrl, which is the same for both MaskedImages).
#
def mean(im):
return afwMath.makeStatistics(im, afwMath.MEAN,
ccd1.stat_ctrl).getValue()
def var(im):
return afwMath.makeStatistics(im, afwMath.VARIANCE,
ccd1.stat_ctrl).getValue()
#
# Extract imaging region for segments of both CCDs.
#
image1 = ccd1.unbiased_and_trimmed_image(amp, bias_frame=bias_frame)
image2 = ccd2.unbiased_and_trimmed_image(amp, bias_frame=bias_frame)
if ccd1.imfile == ccd2.imfile:
# Don't have pairs of flats, so estimate noise and gain
# from a single frame, ignoring FPN.
fmean = mean(image1)
fvar = var(image1)
else:
#
# Make a deep copy since otherwise the pixel values in image1
# would be altered in the ratio calculation.
#
fratio_im = afwImage.MaskedImageF(image1, True)
operator.itruediv(fratio_im, image2)
fratio = mean(fratio_im)
image2 *= fratio
fmean = (mean(image1) + mean(image2)) / 2.
fdiff = afwImage.MaskedImageF(image1, True)
fdiff -= image2
fvar = var(fdiff) / 2.
return FlatPairStats(fmean, fvar)
def test_class_binary_inplace_operators(self):
class WithLotsOfOperators(Class):
def __iadd__(self, other):
return (self, "iadd", other)
def __isub__(self, other):
return (self, "isub", other)
def __imul__(self, other):
return (self, "imul", other)
def __imod__(self, other):
return (self, "imod", other)
def __itruediv__(self, other):
return (self, "itruediv", other)
def __ifloordiv__(self, other):
return (self, "ifloordiv", other)
def __ilshift__(self, other):
return (self, "ilshift", other)
def __irshift__(self, other):
return (self, "irshift", other)
def __ior__(self, other):
return (self, "ior", other)
def __iand__(self, other):
return (self, "iand", other)
def __ixor__(self, other):
return (self, "ixor", other)
def __imatmul__(self, other):
return (self, "imatmul", other)
c = WithLotsOfOperators()
self.assertEqual(operator.iadd(c, 0), (c, "iadd", 0))
self.assertEqual(operator.isub(c, 0), (c, "isub", 0))
self.assertEqual(operator.imul(c, 0), (c, "imul", 0))
self.assertEqual(operator.imod(c, 0), (c, "imod", 0))
self.assertEqual(operator.itruediv(c, 0), (c, "itruediv", 0))
self.assertEqual(operator.ifloordiv(c, 0), (c, "ifloordiv", 0))
self.assertEqual(operator.ilshift(c, 0), (c, "ilshift", 0))
self.assertEqual(operator.irshift(c, 0), (c, "irshift", 0))
self.assertEqual(operator.ior(c, 0), (c, "ior", 0))
self.assertEqual(operator.iand(c, 0), (c, "iand", 0))
self.assertEqual(operator.ixor(c, 0), (c, "ixor", 0))
self.assertEqual(operator.imatmul(c, 0), (c, "imatmul", 0))
def bpa1(goodslist, bpholdweight):
newlistpool = [[list[0], list[1], list[2],
itruediv(list[2], list[1])] for list in goodslist]
print(ujson.encode(newlistpool))
sortlistpool = sorted(newlistpool, key=itemgetter(3), reverse=1)
print(ujson.encode(sortlistpool))
remholdweight = bpholdweight
resultlist = []
for name, weight, value, pervalue in sortlistpool:
if weight <= remholdweight:
resultlist.append([name, weight, value, pervalue])
remholdweight = remholdweight - weight
print(ujson.encode(resultlist))
def handleVid(self, uid, vid):
uid_stat_t = self.mongoconn["video"]["%s_stat" % (uid)]
vid_play_t = self.mongoconn["video"]["%s_play" % (vid)]
uid_vdaily_t = self.mongoconn["video"]["%s_vdaily" % (uid)]
"""设置用户地域的载入次数"""
"""初始化vidinfo类"""
vidinfObj = VidInfo(self.mysql_conn, self.mongoconn, vid)
"""得到该视频本天统计数据"""
viddailydata = uid_vdaily_t.find_one({"_id": self.date}, {
"_id": 0,
vid: 1
})
Load = viddailydata[vid]["Load"]
Play = viddailydata[vid]["Play"]
IP = viddailydata[vid]["IP"]
Engage = viddailydata[vid]["Engage"]
Geo = viddailydata[vid]["Geo"]
Click = viddailydata[vid]["Click"]
Traffic = viddailydata[vid]["Traffic"]
"""计算地域分布数据中完成率"""
for region in Geo:
Geo[region]["Engage"] = operator.itruediv(
float(Geo[region]["EngageSum"]), int(Geo[region]["ClickNum"]))
"""得到该视频实时播放最后100个记录"""
plays_list = []
plays_sum = vid_play_t.count()
MaxSum = 100
if plays_sum > MaxSum:
plays_cursor = vid_play_t.find(
{"_id": {
"$gt": (plays_sum - MaxSum)
}})
else:
plays_cursor = vid_play_t.find().limit(MaxSum)
for play in plays_cursor:
vid_playtime = play["PlayTime"]
self.redata.zadd("%s_SD7" % (vid), ujson.encode(play),
vid_playtime)
"""得到该视频截至目前统计信息"""
vidnowstat = vidinfObj.get_vid_info(vid)
#vidnowstat = uid_stat_t.find_one({"_id":vid},{"_id":0,"EngageSum":0})
vidnowstat.pop("HeatMapSum")
nowGeo = vidnowstat["Geo"]
nowHtmap = json.loads(vidnowstat["Htmap"])
nowTraffic = vidnowstat["Traffic"]
nowPlay = vidnowstat["Play"]
nowClick = vidnowstat["Click"]
nowPtmap = vidnowstat["Ptmap"]
data = (nowPlay, nowClick, nowTraffic, vid)
Htmp_min = min([k for i, j, k in nowHtmap])
nowHtmap = [[i, j, k / Htmp_min] for i, j, k in nowHtmap]
"""时间序列已存储多少天数据"""
#days = redata.llen("%s_date"%(vid)) + 1
"""写数据到redis"""
redpipe = self.redata.pipeline()
redpipe.rpush("%s_date" % (vid), self.date)
redpipe.rpush("%s_SD1" % (vid), Load)
redpipe.rpush("%s_SD2" % (vid), Play)
redpipe.rpush("%s_SD3" % (vid), IP)
redpipe.rpush("%s_SD4" % (vid), Engage)
redpipe.rpush("%s_SD5" % (vid), Click)
redpipe.rpush("%s_SD6" % (vid), Traffic)
redpipe.rpush("%s_SDD" % (vid), ujson.encode(Geo))
redpipe.set("%s_PH" % (vid), ujson.encode(nowPtmap))
redpipe.set("%s_SH" % (vid), ujson.encode(vidnowstat))
redpipe.set("%s_SHH" % (vid), ujson.encode(nowGeo))
redpipe.set("%s_JH" % (vid), ujson.encode(nowHtmap))
"""
if days > self.daynums:
redpipe.lpop("%s_date"%(vid))
redpipe.lpop("%s_SD1"%(vid))
redpipe.lpop("%s_SD2"%(vid))
redpipe.lpop("%s_SD3"%(vid))
redpipe.lpop("%s_SD4"%(vid))
redpipe.lpop("%s_SD5"%(vid))
redpipe.lpop("%s_SD6"%(vid))
redpipe.lpop("%s_SDD"%(vid))
"""
redpipe.execute()
sql = '''
update vinfo set playcount=%s,click_rate=%s,traffic=%s where vid = %s
'''
self.cursor.execute(sql, data)
self.mysql_conn.commit()
vidinfObj.update_vsum(vid, vidnowstat)
vidinfObj.update_vgeo(vid, vidnowstat)
vidinfObj.update_vhmap(vid, vidnowstat)
b = operator.isub(a,1) print a print b #将与自身的值相乘之和的值赋给自身 同 *= #但是不改变自身的值，返回值返回相乘的结果 a = 4 b = operator.imul(a,5) print a print b #将与自身的值相除之和的值赋给自身 同 /= #这个除法是精确除法，不是取整 #但是不改变自身的值，返回值返回相除的结果 a = 9 b = operator.itruediv(a,2) print a print b #将与自身的值相与的值赋给自身 同 &= #但是不改变自身的值，返回值返回相与的结果 a = 8 b = operator.iand(a,1) print a print b #将与自身的值相或的值赋给自身 同 |= #但是不改变自身的值，返回值返回相或的结果 a = 8 b = operator.ior(a,1) print a
def handleVid(self,uid,vid):
uid_stat_t = self.mongoconn[self.mongodbName]["%s_stat"%(uid)]
vid_play_t = self.mongoconn[self.mongodbName]["%s_play"%(vid)]
uid_vdaily_t = self.mongoconn[self.mongodbName]["%s_vdaily"%(uid)]
"""设置用户地域的载入次数"""
"""初始化vidinfo类"""
vidinfObj = VidInfo(self.mysql_conn,self.mongoconn,vid)
"""得到该视频本天统计数据"""
viddailydata = uid_vdaily_t.find_one({"_id":self.date},{"_id":0,vid:1})
Load = viddailydata[vid]["Load"]
Play = viddailydata[vid]["Play"]
IP = viddailydata[vid]["IP"]
Engage = viddailydata[vid]["Engage"]
Geo = viddailydata[vid]["Geo"]
Click = viddailydata[vid]["Click"]
Traffic = viddailydata[vid]["Traffic"]
"""计算地域分布数据中完成率"""
for region in Geo:
Geo[region]["Engage"] = operator.itruediv(float(Geo[region]["EngageSum"]), int(Geo[region]["ClickNum"]))
"""得到该视频实时播放最后100个记录"""
plays_list = []
plays_sum = vid_play_t.count()
MaxSum = 100
if plays_sum > MaxSum:
plays_cursor = vid_play_t.find({"_id":{"$gt":(plays_sum-MaxSum)}})
else:
plays_cursor = vid_play_t.find().limit(MaxSum)
for play in plays_cursor:
vid_playtime = play["PlayTime"]
self.redata.zadd("%s_SD7"%(vid),ujson.encode(play),vid_playtime)
"""得到该视频截至目前统计信息"""
vidnowstat = vidinfObj.get_vid_info(vid)
vidnowstat.pop("HeatMapSum")
nowGeo = vidnowstat["Geo"]
nowHtmap = json.loads(vidnowstat["Htmap"])
nowTraffic = vidnowstat["Traffic"]
nowPlay = vidnowstat["Play"]
nowClick = vidnowstat["Click"]
nowPtmap = vidnowstat["Ptmap"]
data = (nowPlay,nowClick,nowTraffic,vid)
Htmp_min = min([k for i,j,k in nowHtmap])
nowHtmap = [[i,j,k/Htmp_min] for i,j,k in nowHtmap ]
"""时间序列已存储多少天数据"""
#days = redata.llen("%s_date"%(vid)) + 1
"""写数据到redis"""
redpipe = self.redata.pipeline()
redpipe.rpush("%s_date"%(vid), self.date)
redpipe.rpush("%s_SD1"%(vid),Load)
redpipe.rpush("%s_SD2"%(vid),Play)
redpipe.rpush("%s_SD3"%(vid),IP)
redpipe.rpush("%s_SD4"%(vid),Engage)
redpipe.rpush("%s_SD5"%(vid),Click)
redpipe.rpush("%s_SD6"%(vid),Traffic)
redpipe.rpush("%s_SDD"%(vid),ujson.encode(Geo))
redpipe.set("%s_PH"%(vid),ujson.encode(nowPtmap))
redpipe.set("%s_SH"%(vid),ujson.encode(vidnowstat))
redpipe.set("%s_SHH"%(vid),ujson.encode(nowGeo))
redpipe.set("%s_JH"%(vid),ujson.encode(nowHtmap))
"""
if days > self.daynums:
redpipe.lpop("%s_date"%(vid))
redpipe.lpop("%s_SD1"%(vid))
redpipe.lpop("%s_SD2"%(vid))
redpipe.lpop("%s_SD3"%(vid))
redpipe.lpop("%s_SD4"%(vid))
redpipe.lpop("%s_SD5"%(vid))
redpipe.lpop("%s_SD6"%(vid))
redpipe.lpop("%s_SDD"%(vid))
"""
redpipe.execute()
sql = '''
update video set playcount=%s,clickrate=%s,traffic=%s where vid = %s
'''
self.cursor.execute(sql,data)
self.mysql_conn.commit()
vidinfObj.update_vsum(vid,vidnowstat)
vidinfObj.update_vgeo(vid,vidnowstat)
vidinfObj.update_vhmap(vid,vidnowstat)
# --/
# FUNCTION MODULE : COLLECT USER BASE INFORMATION
# VIEW THE VIDEO OF THE CUSTOMER INFORMATION WRITTEN TO MYSQL
# ADD IN 20131210
# --/
play_zzuinfo_cursor = vid_play_t.find({"Date":self.date,"zz_uinfo":{"$exists":True}},{"PlayTime":1,"Engage":1,"PIP":1,"zz_uinfo":1})
if play_zzuinfo_cursor.count() != 0:
i=0
batchnum=10
values=[]
videoname = vidinfObj.get_video_name(vid)
for play in play_zzuinfo_cursor:
if i < batchnum:
i = i + 1
location = vidinfObj.get_video_play_location_from_api(play["PIP"])
playtime = time.strftime("%Y-%m-%d %H:%M:%S",time.localtime(int(play["PlayTime"])))
comprate = "%s"%int(float(play["Engage"])*100)+"%"
playuinfo = eval(eval(play["zz_uinfo"]))
tlist = [uid,vid,videoname,self.date,playtime,location,comprate,playuinfo["name"].decode("unicode_escape"),playuinfo["tel"],playuinfo["email"],playuinfo["qq"]]
values.append(tlist)
if i == batchnum:
vidinfObj.insert_vid_zzuinfo(values)
i=0
values=[]
if i > 0:
vidinfObj.insert_vid_zzuinfo(values)
such as strings, numbers and tuples.'''
import operator
# using iadd() to add and assign value
x = 2;y = 3
x = operator.iadd(x,y)
print("Addition",x)
x = 2;y = 3
x = operator.isub(x,y)
print("Subtraction", x)
x = 2;y = 3
x = operator.imul(x,y)
print("Multiply",x)
x = 10;y = 5
x = operator.itruediv(x,y)
print("Divide",x)
x = 10; y = 6
x = operator.imod(x,y)
print("Mod",x)
# initializing another values
y = 'geeks'
z = 'forgeeks'
# using iconcat() to concat the sequences
y = operator.iconcat(y,z)
print(y)
def exercise_array_inplace_ops_t(self, tp):
# There are a few inplace ops that don't seem to be utilized
# elsewhere in the test framework:
# array_inplace_bitwise_xor
# array_inplace_remainder
# array_inplace_right_shift
# array_inplace_floor_divide
# array_inplace_true_divide
# array_inplace_bitwise_and
# In this test, we'll try to work off this list.
a = np.arange(5, dtype=tp)
aref = np.arange(5, dtype=tp)
# array_inplace_bitwise_xor
a ^= a
assert_equal(a, np.zeros(5, tp))
# array_inplace_bitwise_or
a = np.zeros(5, dtype=tp)
a |= 7
assert_equal(a, [7, 7, 7, 7, 7])
# array_inplace_bitwise_and
a = np.arange(5, dtype=tp)
a &= 1
assert_equal(a, [0, 1, 0, 1, 0])
a = np.arange(5, dtype=tp)
# array_inplace_left_shift
a <<= 1
assert_equal(a, 2 * aref)
# array_inplace_right_shift
a >>= 1
assert_equal(a, aref)
# array_inplace_remainder
a = np.arange(5, dtype=tp)
a += 2
a %= 2
assert_equal(a, [0, 1, 0, 1, 0])
# array_inplace_floor_divide
a = np.arange(5, dtype=tp)
a //= 2
assert_equal(a, [0, 0, 1, 1, 2])
# # What does this do? Apparently not inplace_true_divide...
# a = np.arange(5)
# a <<= 1
# a /= 2
# assert_equal( a, aref )
# Now handle inplace_true_divide, which requires some subtlety.
from operator import itruediv
a = np.arange(5, dtype=tp)
itruediv(a, 2)
assert_equal(a, [0, 0, 1, 1, 2])
def itruediv_usecase(x, y):
return operator.itruediv(x, y)
def run(self):
redispool = redis.ConnectionPool(host=self.redataIP, port=6379, db=0)
redata = redis.Redis(connection_pool=redispool)
mongoconn = MongoClient(self.mongodbIP, 27017)
date = self.datadate
while True:
try:
"""Obtain vid name for have data changs daily"""
vid = self.queue.get()
"""Obtain uid for the vid"""
uid = vid[0:4]
self.log.debug("handleTVIDSET -- Handling is %s" % vid)
"""Obtain kid from redis db for the vid"""
kid = redata.hget("T_video_kid_info", vid)
"""Obtain temp data from redis"""
redpipe = redata.pipeline()
"""Obtain the number of independent IP for the vid daily"""
redpipe.scard("T_IPS_" + vid + "_%s" % (date))
"""Obtain load numbers for the vid daily"""
redpipe.hget("T_loadtable_%s" % (date), vid)
"""Obtain play numbers for the vid daily"""
redpipe.hget("T_playtable_%s" % (date), vid)
"""Write uid to set for facilitate the processing after"""
redpipe.sadd("T_UIDS_%s" % (date), uid)
vidDataList = redpipe.execute()
indepipnums = int(vidDataList[0])
dailyload = int(vidDataList[1])
dailyplay = int(vidDataList[2])
"""--开始--"""
# --/
# 建立mongodb表连接
# --/
uid_stat_t = mongoconn["video"]["%s_stat" % (uid)]
uid_vdaily_t = mongoconn["video"]["%s_vdaily" % (uid)]
uid_daily_t = mongoconn["video"]["%s_daily" % (uid)]
uid_type_t = mongoconn["video"]["%s_type" % (uid)]
request = mongoconn.start_request()
# --/
# 保存该视频playid去mongodb
# --/
playid = redata.hget("vid_playid", vid)
uid_stat_t.update({"_id": vid},
{"$set": {
"PlayID": int(playid)
}},
upsert=True,
w=0)
# --/
# 统计视频每天播放信息
# --/
"""total loads of the vid daily"""
"""total plays of the vid daily"""
"""average click rate of the vid daily"""
"""the number of independent IP for the vid daily"""
"""avarage complete rate for the vid daily"""
uid_vdaily_t.update({"_id": date}, {
"$set": {
"%s.Load" % (vid):
dailyload,
"%s.Play" % (vid):
dailyplay,
"%s.Click" % (vid):
operator.itruediv(dailyplay, dailyload),
"%s.IP" % (vid):
indepipnums,
"%s.Engage" % (vid):
operator.itruediv(
float(
redata.hget("T_engagesum_%s_%s" %
(uid, date), vid)), dailyplay)
}
},
upsert=True,
w=0)
# --/
# 统计视频截至到目前总的播放信息
# --/
"""Load nums of total for the vid up to now"""
"""play nums of total for the vid up to now"""
"""independent IPs of total for the vid up to now"""
uid_stat_t.update({"_id": vid}, {
"$inc": {
"Load": dailyload,
"Play": dailyplay,
"IP": indepipnums
}
},
upsert=True,
w=0)
region_list = list(
redata.smembers("T_region_%s_%s" % (vid, date)))
for i in region_list:
LoadSum = redata.hget("T_regiontable_%s" % (date),
"%s%s" % (vid, i))
uid_stat_t.update(
{"_id": vid},
{"$inc": {
"Geo.%s.Load" % (i): int(LoadSum)
}},
upsert=True,
w=0)
uid_vdaily_t.update(
{"_id": date},
{"$inc": {
"%s.Geo.%s.Load" % (vid, i): int(LoadSum)
}},
upsert=True,
w=0)
"""获取目前为止该视频总的载入数,播放数和每次播放完成率之和"""
res = uid_stat_t.find_one({"_id": vid}, {
"_id": 0,
"Load": 1,
"Play": 1,
"EngageSum": 1
})
vidtotalload = int(res["Load"])
vidtotalplay = int(res["Play"])
videngagesum = float(res["EngageSum"])
"""准备数据为计算截至目前该视频播放的交互记录"""
def returnVal(dict, key):
if dict.has_key(key):
return dict[key]
else:
return 0
TotalSegFlagDict = uid_stat_t.find_one({"_id": vid}, {
"_id": 0,
"HeatMapSum": 1
})["HeatMapSum"]
totalinterdata = [[int(k) * 2 - 2,
int(k) * 2,
int(v)]
for k, v in TotalSegFlagDict.items()]
totalinterdata.sort()
self.log.debug("%s prehandle hmap is %s : " %
(vid, ujson.encode(totalinterdata)))
totalinterdata = mergerPlaySeg(totalinterdata)
self.log.debug("%s befhandle hmap is %s : " %
(vid, ujson.encode(totalinterdata)))
totalplayhotpointdict = uid_stat_t.find_one({"_id": vid}, {
"_id": 0,
"PlayMapSum": 1
})["PlayMapSum"]
totalplayhotdata = [[int(k) * 2 - 2,
int(k) * 2,
int(v)]
for k, v in totalplayhotpointdict.items()]
totalplayhotdata.sort()
self.log.debug("%s prehandle pmap is %s : " %
(vid, ujson.encode(totalplayhotdata)))
totalplayhotdata = mergerPlaySeg(totalplayhotdata)
self.log.debug("%s befhandle pmap is %s : " %
(vid, ujson.encode(totalplayhotdata)))
"""average click rate of total for the vid up to now"""
"""average complete rate of total for the vid up to now"""
"""interactive data(time segment) of total for the vid up to now"""
uid_stat_t.update({"_id": vid}, {
"$set": {
"Click": operator.itruediv(vidtotalplay, vidtotalload),
"Engage": operator.itruediv(videngagesum,
vidtotalplay),
"Htmap": ujson.encode(totalinterdata),
"Ptmap": ujson.encode(totalplayhotdata)
}
},
upsert=True,
w=0)
# --/
# 统计用户每天的播放信息
# --/
"""Load nums for the uid daily"""
"""play nums for the uid daily"""
"""the number of independent IP for the uid daily"""
uid_daily_t.update({"_id": date}, {
"$inc": {
"Load": dailyload,
"Play": dailyplay,
"IP": indepipnums
}
},
upsert=True,
w=0)
# --/
# 统计用户截至目前的播放信息
# --/
"""Load nums of total for the uid up to now"""
"""play nums of total for the uid up to now"""
"""independent IPs of total for the uid up to now"""
uid_stat_t.update({"_id": uid}, {
"$inc": {
"Load": dailyload,
"Play": dailyplay,
"IP": indepipnums
}
},
upsert=True,
w=0)
# --/
# 统计用户类别每天播放信息
# --/
"""Load nums for the uid's kid daily"""
"""play nums for the uid's kid daily"""
"""the number of independent IP for the uid's kid daily"""
if kid != "nokid":
uid_type_t.update({"_id": date}, {
"$inc": {
"%s.Load" % (kid): dailyload,
"%s.Play" % (kid): dailyplay,
"%s.IP" % (kid): indepipnums
}
},
upsert=True,
w=0)
# --/
# 统计用户类别截至目前播放信息
# --/
"""Load nums of total for the uid's kid up to now"""
"""play nums of total for the uid's kid up to now"""
"""independent IPs of total for the uid's kid up to now"""
if kid != "nokid":
for i in region_list:
LoadSum = redata.hget("T_regiontable_%s" % (date),
"%s%s" % (vid, i))
uid_type_t.update({"_id": date}, {
"$inc": {
"%s.Geo.%s.Load" % (kid, i): int(LoadSum)
}
},
upsert=True,
w=0)
request.end()
"""--结束--"""
# --/
# 删除T_VIDS_$date中已完成条目
# --/
redata.srem("T_VIDS_%s" % date, vid)
self.queue.task_done()
except Exception, e:
self.log.error("handleTVIDSET -- handle -- exception,%s" % (e))
break
y = "geeks"
z = "forgeeks"
y = operator.iconcat(y, z)
print("the string after concatenation is : ", end="")
print(y)
x = operator.isub(2, 3)
print("the value after subtracting and assiging : ", end="")
print(x)
x = operator.imul(2, 3)
print("the value after multiplying and assiging : ", end="")
print(x)
x = operator.itruediv(2, 3)
print("the value after dividing and assign : ", end="")
print(x)
x = operator.ixor(10, 5)
print("the value after xoring and assigning : ", end="")
print(x)
x = operator.ipow(5, 4)
print("the value after exponentiating and assigning : ", end="")
print(x)
x = operator.ilshift(8, 2)
print("the value after bitwise left shift and assigning : ", end="")
print(x)
def test_cube_itruediv__uint(self):
with self.assertRaisesRegexp(ArithmeticError,
'Cannot perform inplace division'):
operator.itruediv(self.cube_1u, self.cube_2u)
b = operator.isub(a, 1) print a print b #将与自身的值相乘之和的值赋给自身 同 *= #但是不改变自身的值，返回值返回相乘的结果 a = 4 b = operator.imul(a, 5) print a print b #将与自身的值相除之和的值赋给自身 同 /= #这个除法是精确除法，不是取整 #但是不改变自身的值，返回值返回相除的结果 a = 9 b = operator.itruediv(a, 2) print a print b #将与自身的值相与的值赋给自身 同 &= #但是不改变自身的值，返回值返回相与的结果 a = 8 b = operator.iand(a, 1) print a print b #将与自身的值相或的值赋给自身 同 |= #但是不改变自身的值，返回值返回相或的结果 a = 8 b = operator.ior(a, 1) print a
def test_scalartype_ops(self):
int_types = [
np.byte, np.ubyte, np.short, np.ushort, np.intc, np.uintc, np.int_,
np.uint, np.longlong, np.ulonglong
]
for t in int_types:
x = t(7)
y = x ^ x
assert y == 0, "xor on scalartype"
x = t(1)
y = x << 1
assert y == 2, "left shift on scalartype"
# NOTE: y came back not the same type as t, so a right shift on
# y doesn't exercise the <t>_rshift function. To get the
# <t>_rshift, we have to go back to a <t> instance.
y = t(2)
z = y >> 1
assert z == 1, "right shift on scalartype"
assert np.invert(x) != 1, "invert on scalartype"
assert np.invert(np.invert(x)) == x, "invert on scalartype"
y = t(0)
z = x & y
assert z == 0, "bitwise and on scalartype"
z = x | y
assert z == 1, "bitwise or on scalartype"
assert z, "nonzero on scalartype"
x = t(0)
assert ~x, "Invert on numpy scalar types"
#x = t(5)
#y = x // 2
#assert y, "nonzero on numpy scalar types"
#assert y == 2, "floor divide on numpy scalar types"
for t in real_types:
x = t(5)
assert x, "nonzero on scalartype"
y = x // 2.
assert y == 2, "floor divide on scalartype"
y = t(2)
n, r = divmod(x, y)
assert n == t(2), "divmod on scalartype"
assert r == t(1), "divmod on scalartype"
for t in complex_types:
x = t(5)
assert x, "nonzero on complex scalartype"
y = x // 2
assert y == 2, "Floor divide on complex scalartype"
from operator import itruediv
itruediv(z, x)
for t in types[1:]:
z = t(5)
x = t(2)
itruediv(z, x)
x = t(5)
y = np.long(x)
assert y == x, "Cast scalartype to long"
y = np.int(x)
assert y == x, "Cast scalartype to int"
def __itruediv__(self, other):
pxy = self._pxy_get(copy=True)
proxied = pxy.deserialize(nbytes=self.__sizeof__())
pxy.obj = operator.itruediv(proxied, other)
self._pxy_set(pxy)
def handleUid(self, uid):
uid_stat_t = self.mongoconn["video"]["%s_stat" % (uid)]
uid_daily_t = self.mongoconn["video"]["%s_daily" % (uid)]
#初始化uidinfo类
uidinfObj = UidInfo(self.mysql_conn, self.mongoconn, uid)
"""得到该用户本天统计数据"""
uiddailystat = uid_daily_t.find_one({"_id": self.date})
Nvideo = uiddailystat["Nvideo"]
Load = uiddailystat["Load"]
Play = uiddailystat["Play"]
IP = uiddailystat["IP"]
Engage = uiddailystat["Engage"]
Click = uiddailystat["Click"]
Traffic = uiddailystat["Traffic"]
Geo = uiddailystat["Geo"]
"""计算地域分布数据中完成率"""
for region in Geo:
Geo[region]["Engage"] = operator.itruediv(
float(Geo[region]["EngageSum"]), int(Geo[region]["ClickNum"]))
"""得到该用户截至目前统计信息"""
uidnowstat = uidinfObj.get_uid_info(uid)
#uidnowstat = uid_stat_t.find_one({"_id":uid},{"_id":0,"EngageSum":0})
nowGeo = uidnowstat["Geo"]
nowTraffic = float(uidnowstat["Traffic"])
nowMtraffic = float(uidnowstat["Mtraffic"])
"""时间序列已存储多少天数据"""
#days = redata.llen("%s_date"%(uid)) + 1
"""写数据到redis"""
redpipe = self.redata.pipeline()
redpipe.rpush("%s_date" % (uid), self.date)
redpipe.rpush("%s_UD1" % (uid), Nvideo)
redpipe.rpush("%s_UD2" % (uid), Load)
redpipe.rpush("%s_UD3" % (uid), Play)
redpipe.rpush("%s_UD4" % (uid), IP)
redpipe.rpush("%s_UD5" % (uid), Engage)
redpipe.rpush("%s_UD6" % (uid), Click)
redpipe.rpush("%s_UD7" % (uid), Traffic)
redpipe.rpush("%s_UDD" % (uid), ujson.encode(Geo))
redpipe.set("%s_UH" % (uid), ujson.encode(uidnowstat))
redpipe.set("%s_UDH" % (uid), ujson.encode(nowGeo))
"""
if days > self.daynums:
redpipe.lpop("%s_date"%(uid))
redpipe.lpop("%s_UD1"%(uid))
redpipe.lpop("%s_UD2"%(uid))
redpipe.lpop("%s_UD3"%(uid))
redpipe.lpop("%s_UD4"%(uid))
redpipe.lpop("%s_UD5"%(uid))
redpipe.lpop("%s_UD6"%(uid))
redpipe.lpop("%s_UD7"%(uid))
redpipe.lpop("%s_UDD"%(uid))
"""
redpipe.execute()
uidinfObj.update_usum(uid, uidnowstat)
uidinfObj.update_ugeo(uid, uidnowstat)
b = 7 print(operator.isub(a, b)) """4.imul() :- This function is used to assign and multiply the current value. This operation does “a*=b” operation. Assigning is not performed in case of immutable containers, such as strings, numbers and tuples.""" a = 16 b = 6 print(operator.imul(a, b)) """5.itruediv() :- This function is used to assign and divide the current value. This operation does “a/=b” operation. Assigning is not performed in case of immutable containers, such as strings, numbers and tuples.""" a = 64 b = 8 print(operator.itruediv(a, b)) """6.imod()- This function is used to assign and return remainder. This operation does “a%=b” operation. Assigning is not performed in case of immutable containers, such as strings, numbers and tuples.""" a = 3 b = 2 print(operator.imod(a, b)) """7. ixor() :- This function is used to assign and xor the current value. This operation does “a^ = b” operation. Assigning is not performed in case of immutable containers, such as strings, numbers and tuples.""" a = 1 b = 0 print(operator.ixor(a, b))
def handleKid(self, uid, kid):
uid_stat_t = self.mongoconn["video"]["%s_stat" % (uid)]
uid_type_t = self.mongoconn["video"]["%s_type" % (uid)]
# 初始化kidinfo 类
kidinfObj = KidInfo(self.mysql_conn, self.mongoconn, kid)
"""得到用户本类本天统计数据"""
kiddailydata = uid_type_t.find_one({"_id": self.date}, {
"_id": 0,
kid: 1
})
Nvideo = kiddailydata[kid]["Nvideo"]
Load = kiddailydata[kid]["Load"]
Play = kiddailydata[kid]["Play"]
IP = kiddailydata[kid]["IP"]
Engage = kiddailydata[kid]["Engage"]
Click = kiddailydata[kid]["Click"]
Traffic = kiddailydata[kid]["Traffic"]
Geo = kiddailydata[kid]["Geo"]
#获取kid对应的视频id的列表
sql = '''
SELECT a.vid FROM vinfo AS a
LEFT JOIN vsort AS b ON a.vsortid = b.id
WHERE b.kid= %s
'''
param = (kid)
n = self.cursor.execute(sql, param)
kidinfo = uid_stat_t.find_one({"_id": kid}, {"_id": 0})
if kidinfo != None:
uid_stat_t.remove({"_id": kid})
vidlist = self.cursor.fetchall()
for i in vidlist:
vid = i[0]
vidtotalinfo = uid_stat_t.find_one({"_id": vid}, {"_id": 0})
if vidtotalinfo != None:
uid_stat_t.update({"_id": kid}, {
"$inc": {
"Click": 0,
"Engage": 0,
"EngageSum": vidtotalinfo["EngageSum"],
"IP": vidtotalinfo["IP"],
"Load": vidtotalinfo["Load"],
"Play": vidtotalinfo["Play"],
"Traffic": vidtotalinfo["Traffic"]
}
},
upsert=True,
w=0)
for geo in vidtotalinfo["Geo"].keys():
uid_stat_t.update({"_id": kid}, {
"$inc": {
"Geo.%s.ClickNum" % geo:
vidtotalinfo["Geo"][geo]["ClickNum"],
"Geo.%s.EngageSum" % geo:
vidtotalinfo["Geo"][geo]["EngageSum"],
"Geo.%s.Load" % geo:
vidtotalinfo["Geo"][geo]["Load"],
"Geo.%s.Traffic" % geo:
vidtotalinfo["Geo"][geo]["Traffic"]
}
},
upsert=True,
w=0)
kidtotalnow = uid_stat_t.find_one({"_id": kid}, {"_id": 0})
kidtotal_Load = kidtotalnow["Load"]
kidtotal_Play = kidtotalnow["Play"]
kidtotal_EngageSum = kidtotalnow["EngageSum"]
uid_stat_t.update({"_id": kid}, {
"$set": {
"Nvideo": n,
"Click": float(kidtotal_Play) / float(kidtotal_Load),
"Engage": float(kidtotal_EngageSum) / int(kidtotal_Play)
}
},
upset=True,
w=0)
kidlist = {}
for i in xrange(0, n):
vid = vidlist[i][0]
kidlist[str(i + 1)] = vid
print "kid geo update....."
"""计算地域分布数据中完成率"""
for region in Geo:
Geo[region]["Engage"] = operator.itruediv(
float(Geo[region]["EngageSum"]), int(Geo[region]["ClickNum"]))
"""得到该用户截至目前统计信息"""
kidnowstat = kidinfObj.get_kid_info(kid)
nowGeo = kidnowstat["Geo"]
nowTraffic = kidnowstat["Traffic"]
nowPlay = kidnowstat["Play"]
nowClick = kidnowstat["Click"]
data = (nowPlay, nowClick, nowTraffic, kid)
"""时间序列已存储多少天数据"""
#days = redata.llen("%s_%s_date"%(uid,kid)) + 1
"""写数据到redis"""
redpipe = self.redata.pipeline()
redpipe.rpush("%s_date" % (kid), self.date)
redpipe.rpush("%s_D1" % (kid), Nvideo)
redpipe.rpush("%s_D2" % (kid), Load)
redpipe.rpush("%s_D3" % (kid), Play)
redpipe.rpush("%s_D4" % (kid), IP)
redpipe.rpush("%s_D5" % (kid), Engage)
redpipe.rpush("%s_D6" % (kid), Click)
redpipe.rpush("%s_D7" % (kid), Traffic)
redpipe.rpush("%s_DD" % (kid), ujson.encode(Geo))
redpipe.set("%s_H" % (kid), ujson.encode(kidnowstat))
redpipe.set("%s_DH" % (kid), ujson.encode(nowGeo))
"写入kid的列表"
redpipe.set("%s_info" % (kid), ujson.encode(kidlist))
"""
if days > self.daynums:
redpipe.lpop("%s_%s_date"%(uid,kid))
redpipe.lpop("%s_%s_D1"%(uid,kid))
redpipe.lpop("%s_%s_D2"%(uid,kid))
redpipe.lpop("%s_%s_D3"%(uid,kid))
redpipe.lpop("%s_%s_D4"%(uid,kid))
redpipe.lpop("%s_%s_D5"%(uid,kid))
redpipe.lpop("%s_%s_D6"%(uid,kid))
redpipe.lpop("%s_%s_D7"%(uid,kid))
redpipe.lpop("%s_%s_DD"%(uid,kid))
"""
redpipe.execute()
sql = '''
update vsort set playcount=%s,click_rate=%s,traffic=%s where kid=%s
'''
self.cursor.execute(sql, data)
self.mysql_conn.commit()
kidinfObj.update_ksum(kid, kidnowstat)
kidinfObj.update_kgeo(kid, kidnowstat)
print "kid mysql update successful!!"
def test_scalartype_ops( self ):
int_types = [ np.byte, np.ubyte, np.short, np.ushort,
np.intc, np.uintc,
np.int_, np.uint, np.longlong, np.ulonglong ]
for t in int_types:
x = t(7)
y = x ^ x
assert y == 0, "xor on scalartype"
x = t(1)
y = x << 1
assert y == 2, "left shift on scalartype"
# NOTE: y came back not the same type as t, so a right shift on
# y doesn't exercise the <t>_rshift function. To get the
# <t>_rshift, we have to go back to a <t> instance.
y = t(2)
z = y >> 1
assert z == 1, "right shift on scalartype"
assert np.invert(x) != 1, "invert on scalartype"
assert np.invert( np.invert( x ) ) == x, "invert on scalartype"
y = t(0)
z = x & y
assert z == 0, "bitwise and on scalartype"
z = x | y
assert z == 1, "bitwise or on scalartype"
assert z, "nonzero on scalartype"
x = t(0)
assert ~x, "Invert on numpy scalar types"
#x = t(5)
#y = x // 2
#assert y, "nonzero on numpy scalar types"
#assert y == 2, "floor divide on numpy scalar types"
for t in real_types:
x = t(5)
assert x, "nonzero on scalartype"
y = x // 2.
assert y == 2, "floor divide on scalartype"
y = t(2)
n,r = divmod( x, y )
assert n == t(2), "divmod on scalartype"
assert r == t(1), "divmod on scalartype"
for t in complex_types:
x = t(5)
assert x, "nonzero on complex scalartype"
y = x // 2
assert y == 2, "Floor divide on complex scalartype"
from operator import itruediv
itruediv( z, x )
for t in types[1:]:
z = t(5)
x = t(2)
itruediv( z, x )
x = t(5)
y = np.long(x)
assert y == x, "Cast scalartype to long"
y = np.int(x)
assert y == x, "Cast scalartype to int"
def run(self):
redispool = redis.ConnectionPool(host=self.redataIP, port=6379, db=0)
redata = redis.Redis(connection_pool=redispool)
mongoconn = MongoClient(self.mongodbIP, 27017)
date = self.datadate
while True:
try:
"""Obtain uid name for have data changs daily"""
uid = self.queue.get()
self.log.debug("Handling uid is %s" % uid)
"""Obtain video nums from redis db for the uid"""
uidvideonums = len(redata.keys("%s???_info" % uid))
"""--开始--"""
# --/
# 建立mongodb表连接
# --/
uid_stat_t = mongoconn["video"]["%s_stat" % (uid)]
uid_vdaily_t = mongoconn["video"]["%s_vdaily" % (uid)]
uid_daily_t = mongoconn["video"]["%s_daily" % (uid)]
uid_type_t = mongoconn["video"]["%s_type" % (uid)]
request = mongoconn.start_request()
# --/
# 统计用户每天播放信息
# --/
"""获取用户视频该天载入次数,播放次数"""
uiddailystat = uid_daily_t.find_one({"_id": date}, {
"_id": 0,
"Load": 1,
"Play": 1
})
uiddailyload = int(uiddailystat["Load"])
uiddailyplay = int(uiddailystat["Play"])
"""video nums for the uid"""
"""average complete rate for the uid daily"""
"""Calculate avarge click rate for the uid daily"""
uid_daily_t.update({"_id": date}, {
"$set": {
"Nvideo":
uidvideonums,
"Engage":
operator.itruediv(
float(
redata.hget("T_engagesum_%s_%s" %
(uid, date), uid)), uiddailyplay),
"Click":
operator.itruediv(uiddailyplay, uiddailyload)
}
},
upsert=True,
w=0)
# --/
# 统计用户截至到目前播放信息
# --/
region_list = list(
redata.smembers("T_region_%s_%s" % (uid, date)))
for i in region_list:
LoadSum = redata.hget("T_regiontable_%s" % (date),
"%s%s" % (uid, i))
uid_stat_t.update(
{"_id": uid},
{"$inc": {
"Geo.%s.Load" % (i): int(LoadSum)
}},
upsert=True,
w=0)
uid_daily_t.update(
{"_id": date},
{"$set": {
"Geo.%s.Load" % (i): int(LoadSum)
}},
upsert=True,
w=0)
"""获取用户截至到目前载入次数,播放次数,完成率之和"""
uidnowstat = uid_stat_t.find_one({"_id": uid}, {
"_id": 0,
"Load": 1,
"Play": 1,
"EngageSum": 1
})
uidnowload = int(uidnowstat["Load"])
uidnowplay = int(uidnowstat["Play"])
uidnowengagesum = float(uidnowstat["EngageSum"])
"""video nums for the uid up to now"""
"""average complete rate of total for the uid up to now"""
"""Calculate avarge click rate for the uid up to now"""
uid_stat_t.update({"_id": uid}, {
"$set": {
"Nvideo": uidvideonums,
"Engage": operator.itruediv(uidnowengagesum,
uidnowplay),
"Click": operator.itruediv(uidnowplay, uidnowload)
}
})
# --/
# 统计用户类别播放信息
# --/
"""获取该天有数据变动的类别名"""
kidsdict = uid_type_t.find_one({"_id": date}, {"_id": 0})
if kidsdict != None:
for kid in kidsdict:
'''获取该类别视频数目'''
sql = '''
SELECT a.vid FROM vinfo AS a
LEFT JOIN vsort AS b ON a.vsortid = b.id
WHERE b.kid= %s
'''
mysql_conn = MySQLdb.connect(host=self.mysqlIP,
user=self.mysqlUser,
passwd=self.mysqlPassword,
port=self.mysqlPort,
db=self.mysqlDbname,
charset="utf8")
cursor = mysql_conn.cursor()
param = (kid)
n = cursor.execute(sql, param)
kidvideonums = n
# --/
# 统计用户类别每天播放信息
# --/
"""获取用户该类别视频该天载入次数,播放次数"""
kiddailystat = uid_type_t.find_one({"_id": date}, {
"_id": 0,
kid: 1
})
kiddailyload = int(kiddailystat[kid]["Load"])
kiddailyplay = int(kiddailystat[kid]["Play"])
"""video nums for the uid's kid daily"""
"""Calculate avarge complete rate for the uid's kid daily"""
"""Calculate avarge click rate for the uid's kid daily"""
uid_type_t.update({"_id": date}, {
"$set": {
"%s.Nvideo" % (kid):
kidvideonums,
"%s.Engage" % (kid):
operator.itruediv(
float(
redata.hget(
"T_engagesum_%s_%s" %
(uid, date), kid)), kiddailyplay),
"%s.Click" % (kid):
operator.itruediv(kiddailyplay, kiddailyload)
}
})
request.end()
'''--结束--'''
# -- /
# 删除T_UIDS_%date中已完成条目
# -- /
redata.srem("T_UIDS_%s" % date, uid)
self.queue.task_done()
except IOError, e:
self.log.error("handleTUIDSET -- handle -- exception,%s" % (e))
break
#建立hadoop集群信息
hadoopinfo_dict = {"HadoopNNAddr":HadoopNNAddr,"HadoopBinDir":HadoopBinDir}
###############################################
"""
分割视频
"""
TaskTrackerNums = 3
BlockSize = 64
VideoSizeThreshold = ""
PerSecondSizeByte = operator.div(VideoSize,VideoDuration)
BlockSizeSecondNum = int(operator.div(BlockSize*1024*1024,PerSecondSizeByte))
PartNums = int(operator.itruediv(VideoDuration,BlockSizeSecondNum))+1
print VideoFileNameAlias
print PerSecondSizeByte
print BlockSizeSecondNum
print int(operator.itruediv(VideoDuration,BlockSizeSecondNum))+1
############################
cmd = "cd /tmp/%s/ && /bin/sh %s/handleoffset.sh %s %s %s %s %s %s"
cmdstr = cmd%(VideoFileNameAlias,myselfdir,VideoFilePath,VideoFileNameAlias,VideoFormat,int(operator.itruediv(VideoDuration,BlockSizeSecondNum))+1,BlockSizeSecondNum,VideoDuration)
logger.info("Excute command : %s "%cmdstr)
p = subprocess.Popen(cmdstr,shell=True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT)
p.wait()
if p.returncode != 0:
logger.error("Excute command %s fail,Error info : %s,Exit code : %s"%(cmd,p.stdout.read(),p.returncode))
# using isub() to subtract and assign value
x = operator.isub(2, 3)
# printing the modified value
print("The value after subtracting and assigning : ", end="")
print(x)
# using imul() to multiply and assign value
x = operator.imul(2, 3)
# printing the modified value
print("The value after multiplying and assigning : ", end="")
print(x)
# Python code to demonstrate the working of
# itruediv() and imod()
# using itruediv() to divide and assign value
x = operator.itruediv(10, 5)
# printing the modified value
print("The value after dividing and assigning : ", end="")
print(x)
# using imod() to modulus and assign value
x = operator.imod(10.0, 6)
# printing the modified value
print("The value after modulus and assigning : ", end="")
print(x)
'format': VideoFormat,
'width': VideoFrameWidth,
'height': VideoFrameHeight,
'bitrate': VideoBitRate
}
#建立hadoop集群信息
hadoopinfo_dict = {"HadoopNNAddr": HadoopNNAddr, "HadoopBinDir": HadoopBinDir}
###############################################
""" 分割视频 """
PerSecondSizeByte = operator.div(VideoSize, VideoDuration)
BlockSizeSecondNum = int(
operator.div(BlockSize * 1024 * 1024, PerSecondSizeByte))
PartNums = int(operator.itruediv(VideoDuration, BlockSizeSecondNum)) + 1
logger.debug("split video information : \n \
PerSecondSizeByte : %s \n \
BlockSizeSecondNum : %s \n \
VideoSegNums : %s \n \
" %
(PerSecondSizeByte, BlockSizeSecondNum,
int(operator.itruediv(VideoDuration, BlockSizeSecondNum)) + 1))
###############################################
""" shell脚本分割视频 """
cmd = "cd /tmp/%s/ && /bin/sh %s/handleoffset.sh %s %s %s %s %s %s"
cmdstr = cmd % (VideoFileNameAlias, myselfdir, VideoFilePath,
VideoFileNameAlias, VideoFormat,
int(operator.itruediv(VideoDuration, BlockSizeSecondNum)) + 1,
while True:
# --/
# 分割视频准备工作,依据指定块大小将计算视频文件将分成几块
# --/
try:
BlockSize = BlockSizeList.pop()
logger.info("Now BlockSize is %s"%(BlockSize))
except IndexError:
logger.error("Video files can not be processed!!!")
MysqlObj.writeStatus("fail",IsWaterMark,IsOverWrite,info="Video files can not be processed!!!")
sys.exit()
PerSecondSizeByte = operator.div(VideoSize,VideoDuration)
BlockSizeSecondNum = int(operator.div(BlockSize*1024*1024,PerSecondSizeByte))
PartNums = int(operator.itruediv(VideoDuration,BlockSizeSecondNum))+1
logger.debug("split video information : \n \
PerSecondSizeByte : %s \n \
BlockSizeSecondNum : %s \n \
VideoSegNums : %s \n \
"%(PerSecondSizeByte,BlockSizeSecondNum,PartNums))
# --/
# 调用handleoffset.sh脚本,对视频进行预处理
# --/
cmd = "cd /tmp/%s/ && /bin/sh %s/handleoffset.sh %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s"
cmdstr = cmd%(vid,myselfdir,VideoFilePath,vid,OriginVideoFormat,PartNums,BlockSizeSecondNum,VideoDuration,
VFlag,BitRate,VideoBitRate,VideoCode,AudioBitRate,AudioCode,AudioSampleRate,AudioChannels,myselfdir)
def exercise_array_inplace_ops_t( self, tp ):
# There are a few inplace ops that don't seem to be utilized
# elsewhere in the test framework:
# array_inplace_bitwise_xor
# array_inplace_remainder
# array_inplace_right_shift
# array_inplace_floor_divide
# array_inplace_true_divide
# array_inplace_bitwise_and
# In this test, we'll try to work off this list.
a = np.arange( 5, dtype=tp )
aref = np.arange( 5, dtype=tp )
# array_inplace_bitwise_xor
a ^= a
assert_equal( a, np.zeros( 5, tp ) )
# array_inplace_bitwise_or
a = np.zeros( 5, dtype=tp )
a |= 7
assert_equal( a, [7,7,7,7,7] )
# array_inplace_bitwise_and
a = np.arange( 5, dtype=tp )
a &= 1
assert_equal( a, [0,1,0,1,0] )
a = np.arange( 5, dtype=tp )
# array_inplace_left_shift
a <<= 1
assert_equal( a, 2 * aref )
# array_inplace_right_shift
a >>= 1
assert_equal( a, aref )
# array_inplace_remainder
a = np.arange( 5, dtype=tp )
a += 2
a %= 2
assert_equal( a, [0,1,0,1,0] )
# array_inplace_floor_divide
a = np.arange( 5, dtype=tp )
a //= 2
assert_equal( a, [0,0,1,1,2] )
# # What does this do? Apparently not inplace_true_divide...
# a = np.arange(5)
# a <<= 1
# a /= 2
# assert_equal( a, aref )
# Now handle inplace_true_divide, which requires some subtlety.
from operator import itruediv
a = np.arange( 5, dtype=tp )
itruediv( a, 2 )
assert_equal( a, [0,0,1,1,2] )
def __itruediv__(self, other):
return operator.itruediv(self._wrapped(), other)
def handleUid(self,uid):
uid_stat_t = self.mongoconn["video"]["%s_stat"%(uid)]
uid_daily_t = self.mongoconn["video"]["%s_daily"%(uid)]
#初始化uidinfo类
uidinfObj = UidInfo(self.mysql_conn,self.mongoconn,uid)
"""得到该用户本天统计数据"""
uiddailystat = uid_daily_t.find_one({"_id":self.date})
Nvideo = uiddailystat["Nvideo"]
Load = uiddailystat["Load"]
Play = uiddailystat["Play"]
IP = uiddailystat["IP"]
Engage = uiddailystat["Engage"]
Click = uiddailystat["Click"]
Traffic = uiddailystat["Traffic"]
Geo = uiddailystat["Geo"]
"""计算地域分布数据中完成率"""
for region in Geo:
Geo[region]["Engage"] = operator.itruediv(float(Geo[region]["EngageSum"]), int(Geo[region]["ClickNum"]))
"""得到该用户截至目前统计信息"""
uidnowstat = uidinfObj.get_uid_info(uid)
#uidnowstat = uid_stat_t.find_one({"_id":uid},{"_id":0,"EngageSum":0})
nowGeo = uidnowstat["Geo"]
nowTraffic = float(uidnowstat["Traffic"])
nowMtraffic = float(uidnowstat["Mtraffic"])
"""时间序列已存储多少天数据"""
#days = redata.llen("%s_date"%(uid)) + 1
"""写数据到redis"""
redpipe = self.redata.pipeline()
redpipe.rpush("%s_date"%(uid), self.date)
redpipe.rpush("%s_UD1"%(uid),Nvideo)
redpipe.rpush("%s_UD2"%(uid),Load)
redpipe.rpush("%s_UD3"%(uid),Play)
redpipe.rpush("%s_UD4"%(uid),IP)
redpipe.rpush("%s_UD5"%(uid),Engage)
redpipe.rpush("%s_UD6"%(uid),Click)
redpipe.rpush("%s_UD7"%(uid),Traffic)
redpipe.rpush("%s_UDD"%(uid),ujson.encode(Geo))
redpipe.set("%s_UH"%(uid),ujson.encode(uidnowstat))
redpipe.set("%s_UDH"%(uid),ujson.encode(nowGeo))
"""
if days > self.daynums:
redpipe.lpop("%s_date"%(uid))
redpipe.lpop("%s_UD1"%(uid))
redpipe.lpop("%s_UD2"%(uid))
redpipe.lpop("%s_UD3"%(uid))
redpipe.lpop("%s_UD4"%(uid))
redpipe.lpop("%s_UD5"%(uid))
redpipe.lpop("%s_UD6"%(uid))
redpipe.lpop("%s_UD7"%(uid))
redpipe.lpop("%s_UDD"%(uid))
"""
redpipe.execute()
uidinfObj.update_usum(uid,uidnowstat)
uidinfObj.update_ugeo(uid,uidnowstat)
def test_uint_cube_itruediv__scalar(self):
with self.assertRaisesRegexp(ArithmeticError,
'Cannot perform inplace division'):
operator.itruediv(self.cube_1u, 5)
def handleKid(self,uid,kid):
uid_stat_t = self.mongoconn["video"]["%s_stat"%(uid)]
uid_type_t = self.mongoconn["video"]["%s_type"%(uid)]
# 初始化kidinfo 类
kidinfObj = KidInfo(self.mysql_conn,self.mongoconn,kid)
"""得到用户本类本天统计数据"""
kiddailydata = uid_type_t.find_one({"_id":self.date},{"_id":0,kid:1})
Nvideo = kiddailydata[kid]["Nvideo"]
Load = kiddailydata[kid]["Load"]
Play = kiddailydata[kid]["Play"]
IP = kiddailydata[kid]["IP"]
Engage = kiddailydata[kid]["Engage"]
Click = kiddailydata[kid]["Click"]
Traffic = kiddailydata[kid]["Traffic"]
Geo = kiddailydata[kid]["Geo"]
#获取kid对应的视频id的列表
sql = '''
SELECT a.vid FROM vinfo AS a
LEFT JOIN vsort AS b ON a.vsortid = b.id
WHERE b.kid= %s
'''
param = (kid)
n = self.cursor.execute(sql,param)
kidinfo = uid_stat_t.find_one({"_id":kid},{"_id":0})
if kidinfo != None:
uid_stat_t.remove({"_id":kid})
vidlist = self.cursor.fetchall()
for i in vidlist:
vid = i[0]
vidtotalinfo = uid_stat_t.find_one({"_id":vid},{"_id":0})
if vidtotalinfo != None :
uid_stat_t.update({"_id":kid},{"$inc":{
"Click" : 0,
"Engage" : 0,
"EngageSum" : vidtotalinfo["EngageSum"],
"IP": vidtotalinfo["IP"],
"Load":vidtotalinfo["Load"],
"Play":vidtotalinfo["Play"],
"Traffic":vidtotalinfo["Traffic"]
}},upsert=True,w=0)
for geo in vidtotalinfo["Geo"].keys():
uid_stat_t.update({"_id":kid},{"$inc":{
"Geo.%s.ClickNum"%geo : vidtotalinfo["Geo"][geo]["ClickNum"],
"Geo.%s.EngageSum"%geo : vidtotalinfo["Geo"][geo]["EngageSum"],
"Geo.%s.Load"%geo : vidtotalinfo["Geo"][geo]["Load"],
"Geo.%s.Traffic"%geo : vidtotalinfo["Geo"][geo]["Traffic"]
}},upsert=True,w=0)
kidtotalnow = uid_stat_t.find_one({"_id":kid},{"_id":0})
kidtotal_Load = kidtotalnow["Load"]
kidtotal_Play = kidtotalnow["Play"]
kidtotal_EngageSum = kidtotalnow["EngageSum"]
uid_stat_t.update({"_id":kid},{"$set":{
"Nvideo":n,
"Click" : float(kidtotal_Play)/float(kidtotal_Load),
"Engage": float(kidtotal_EngageSum)/int(kidtotal_Play)
}},upset=True,w=0)
kidlist = {}
for i in xrange(0,n):
vid = vidlist[i][0]
kidlist[str(i+1)] = vid
print "kid geo update....."
"""计算地域分布数据中完成率"""
for region in Geo:
Geo[region]["Engage"] = operator.itruediv(float(Geo[region]["EngageSum"]), int(Geo[region]["ClickNum"]))
"""得到该用户截至目前统计信息"""
kidnowstat = kidinfObj.get_kid_info(kid)
nowGeo = kidnowstat["Geo"]
nowTraffic = kidnowstat["Traffic"]
nowPlay = kidnowstat["Play"]
nowClick = kidnowstat["Click"]
data = (nowPlay,nowClick,nowTraffic,kid)
"""时间序列已存储多少天数据"""
#days = redata.llen("%s_%s_date"%(uid,kid)) + 1
"""写数据到redis"""
redpipe = self.redata.pipeline()
redpipe.rpush("%s_date"%(kid), self.date)
redpipe.rpush("%s_D1"%(kid),Nvideo)
redpipe.rpush("%s_D2"%(kid),Load)
redpipe.rpush("%s_D3"%(kid),Play)
redpipe.rpush("%s_D4"%(kid),IP)
redpipe.rpush("%s_D5"%(kid),Engage)
redpipe.rpush("%s_D6"%(kid),Click)
redpipe.rpush("%s_D7"%(kid),Traffic)
redpipe.rpush("%s_DD"%(kid),ujson.encode(Geo))
redpipe.set("%s_H"%(kid),ujson.encode(kidnowstat))
redpipe.set("%s_DH"%(kid),ujson.encode(nowGeo))
"写入kid的列表"
redpipe.set("%s_info"%(kid),ujson.encode(kidlist))
"""
if days > self.daynums:
redpipe.lpop("%s_%s_date"%(uid,kid))
redpipe.lpop("%s_%s_D1"%(uid,kid))
redpipe.lpop("%s_%s_D2"%(uid,kid))
redpipe.lpop("%s_%s_D3"%(uid,kid))
redpipe.lpop("%s_%s_D4"%(uid,kid))
redpipe.lpop("%s_%s_D5"%(uid,kid))
redpipe.lpop("%s_%s_D6"%(uid,kid))
redpipe.lpop("%s_%s_D7"%(uid,kid))
redpipe.lpop("%s_%s_DD"%(uid,kid))
"""
redpipe.execute()
sql = '''
update vsort set playcount=%s,click_rate=%s,traffic=%s where kid=%s
'''
self.cursor.execute(sql,data)
self.mysql_conn.commit()
kidinfObj.update_ksum(kid,kidnowstat)
kidinfObj.update_kgeo(kid,kidnowstat)
print "kid mysql update successful!!"
def test_uint_cube_itruediv__scalar(self):
with self.assertRaisesRegex(ArithmeticError,
"Cannot perform inplace division"):
operator.itruediv(self.cube_1u, 5)
def run(self):
redispool = redis.ConnectionPool(host=self.redataIP,port=6379,db=0)
redata = redis.Redis(connection_pool=redispool)
mongoconn = MongoClient(self.mongodbIP,27017)
date = self.datadate
while True:
try:
"""Obtain vid name for have data changs daily"""
vid = self.queue.get()
"""Obtain uid for the vid"""
uid = vid[0:4]
self.log.debug("handleTVIDSET -- Handling is %s"%vid)
"""Obtain kid from redis db for the vid"""
kid = redata.hget("T_video_kid_info",vid)
"""Obtain temp data from redis"""
redpipe = redata.pipeline()
"""Obtain the number of independent IP for the vid daily"""
redpipe.scard("T_IPS_"+vid+"_%s"%(date))
"""Obtain load numbers for the vid daily"""
redpipe.hget("T_loadtable_%s"%(date),vid)
"""Obtain play numbers for the vid daily"""
redpipe.hget("T_playtable_%s"%(date),vid)
"""Write uid to set for facilitate the processing after"""
redpipe.sadd("T_UIDS_%s"%(date),uid)
vidDataList = redpipe.execute()
indepipnums = int(vidDataList[0])
dailyload = int(vidDataList[1])
dailyplay = int(vidDataList[2])
"""--开始--"""
# --/
# 建立mongodb表连接
# --/
uid_stat_t = mongoconn["video"]["%s_stat"%(uid)]
uid_vdaily_t = mongoconn["video"]["%s_vdaily"%(uid)]
uid_daily_t = mongoconn["video"]["%s_daily"%(uid)]
uid_type_t = mongoconn["video"]["%s_type"%(uid)]
request = mongoconn.start_request()
# --/
# 保存该视频playid去mongodb
# --/
playid = redata.hget("vid_playid",vid)
uid_stat_t.update({"_id":vid},{"$set":{"PlayID":int(playid)}},upsert=True,w=0)
# --/
# 统计视频每天播放信息
# --/
"""total loads of the vid daily"""
"""total plays of the vid daily"""
"""average click rate of the vid daily"""
"""the number of independent IP for the vid daily"""
"""avarage complete rate for the vid daily"""
uid_vdaily_t.update({"_id":date},{"$set":{
"%s.Load"%(vid):dailyload,
"%s.Play"%(vid):dailyplay,
"%s.Click"%(vid):operator.itruediv(dailyplay, dailyload),
"%s.IP"%(vid):indepipnums,
"%s.Engage"%(vid):operator.itruediv(float(redata.hget("T_engagesum_%s_%s"%(uid,date),vid)),dailyplay)
}},upsert=True,w=0)
# --/
# 统计视频截至到目前总的播放信息
# --/
"""Load nums of total for the vid up to now"""
"""play nums of total for the vid up to now"""
"""independent IPs of total for the vid up to now"""
uid_stat_t.update({"_id":vid},{"$inc":{
"Load":dailyload,
"Play":dailyplay,
"IP":indepipnums
}},upsert=True,w=0)
region_list = list(redata.smembers("T_region_%s_%s"%(vid,date)))
for i in region_list:
LoadSum = redata.hget("T_regiontable_%s"%(date),"%s%s"%(vid,i))
uid_stat_t.update({"_id":vid},{"$inc":{
"Geo.%s.Load"%(i):int(LoadSum)
}},upsert=True,w=0)
uid_vdaily_t.update({"_id":date},{"$inc":{
"%s.Geo.%s.Load"%(vid,i):int(LoadSum)
}},upsert=True,w=0)
"""获取目前为止该视频总的载入数,播放数和每次播放完成率之和"""
res = uid_stat_t.find_one({"_id":vid},{"_id":0,"Load":1,"Play":1,"EngageSum":1})
vidtotalload = int(res["Load"])
vidtotalplay = int(res["Play"])
videngagesum = float(res["EngageSum"])
"""准备数据为计算截至目前该视频播放的交互记录"""
def returnVal(dict,key):
if dict.has_key(key):
return dict[key]
else:
return 0
TotalSegFlagDict = uid_stat_t.find_one({"_id":vid},{"_id":0,"HeatMapSum":1})["HeatMapSum"]
totalinterdata = [[int(k)*2-2,int(k)*2,int(v)] for k,v in TotalSegFlagDict.items()]
totalinterdata.sort()
self.log.debug("%s prehandle hmap is %s : "%(vid,ujson.encode(totalinterdata)))
totalinterdata = mergerPlaySeg(totalinterdata)
self.log.debug("%s befhandle hmap is %s : "%(vid,ujson.encode(totalinterdata)))
totalplayhotpointdict = uid_stat_t.find_one({"_id":vid},{"_id":0,"PlayMapSum":1})["PlayMapSum"]
totalplayhotdata = [[int(k)*2-2,int(k)*2,int(v)] for k,v in totalplayhotpointdict.items()]
totalplayhotdata.sort()
self.log.debug("%s prehandle pmap is %s : "%(vid,ujson.encode(totalplayhotdata)))
totalplayhotdata = mergerPlaySeg(totalplayhotdata)
self.log.debug("%s befhandle pmap is %s : "%(vid,ujson.encode(totalplayhotdata)))
"""average click rate of total for the vid up to now"""
"""average complete rate of total for the vid up to now"""
"""interactive data(time segment) of total for the vid up to now"""
uid_stat_t.update({"_id":vid},{"$set":{
"Click":operator.itruediv(vidtotalplay,vidtotalload),
"Engage":operator.itruediv(videngagesum,vidtotalplay),
"Htmap":ujson.encode(totalinterdata),
"Ptmap":ujson.encode(totalplayhotdata)
}},upsert=True,w=0)
# --/
# 统计用户每天的播放信息
# --/
"""Load nums for the uid daily"""
"""play nums for the uid daily"""
"""the number of independent IP for the uid daily"""
uid_daily_t.update({"_id":date},{"$inc":{"Load":dailyload,"Play":dailyplay,"IP":indepipnums}},upsert=True,w=0)
# --/
# 统计用户截至目前的播放信息
# --/
"""Load nums of total for the uid up to now"""
"""play nums of total for the uid up to now"""
"""independent IPs of total for the uid up to now"""
uid_stat_t.update({"_id":uid},{"$inc":{"Load":dailyload,"Play":dailyplay,"IP":indepipnums}},upsert=True,w=0)
# --/
# 统计用户类别每天播放信息
# --/
"""Load nums for the uid's kid daily"""
"""play nums for the uid's kid daily"""
"""the number of independent IP for the uid's kid daily"""
if kid != "nokid":
uid_type_t.update({"_id":date},{"$inc":{"%s.Load"%(kid):dailyload,"%s.Play"%(kid):dailyplay,"%s.IP"%(kid):indepipnums}},upsert=True,w=0)
# --/
# 统计用户类别截至目前播放信息
# --/
"""Load nums of total for the uid's kid up to now"""
"""play nums of total for the uid's kid up to now"""
"""independent IPs of total for the uid's kid up to now"""
if kid != "nokid":
for i in region_list:
LoadSum = redata.hget("T_regiontable_%s"%(date),"%s%s"%(vid,i))
uid_type_t.update({"_id":date},{"$inc":{
"%s.Geo.%s.Load"%(kid,i):int(LoadSum)
}},upsert=True,w=0)
request.end()
"""--结束--"""
# --/
# 删除T_VIDS_$date中已完成条目
# --/
redata.srem("T_VIDS_%s"%date,vid)
self.queue.task_done()
except Exception,e:
self.log.error("handleTVIDSET -- handle -- exception,%s"%(e))
break
def __itruediv__(self, other):
self.__wrapped__ = operator.itruediv(self.__wrapped__, other)
return self
def __itruediv__(self, other):
self.__wrapped__ = operator.itruediv(self.__wrapped__, other)
return self
def itruediv_usecase(x, y):
return operator.itruediv(x, y)
def run(self):
redispool = redis.ConnectionPool(host=self.redataIP,port=6379,db=0)
redata = redis.Redis(connection_pool=redispool)
mongoconn = MongoClient(self.mongodbIP,27017)
date = self.datadate
while True:
try:
"""Obtain uid name for have data changs daily"""
uid = self.queue.get()
self.log.debug("Handling uid is %s"%uid)
"""Obtain video nums from redis db for the uid"""
uidvideonums = len(redata.keys("%s???_info"%uid))
"""--开始--"""
# --/
# 建立mongodb表连接
# --/
uid_stat_t = mongoconn[self.mongodbName]["%s_stat"%(uid)]
uid_vdaily_t = mongoconn[self.mongodbName]["%s_vdaily"%(uid)]
uid_daily_t = mongoconn[self.mongodbName]["%s_daily"%(uid)]
uid_type_t = mongoconn[self.mongodbName]["%s_type"%(uid)]
request = mongoconn.start_request()
# --/
# 统计用户每天播放信息
# --/
"""获取用户视频该天载入次数,播放次数"""
uiddailystat = uid_daily_t.find_one({"_id":date},{"_id":0,"Load":1,"Play":1})
uiddailyload = int(uiddailystat["Load"])
uiddailyplay = int(uiddailystat["Play"])
"""video nums for the uid"""
"""average complete rate for the uid daily"""
"""Calculate avarge click rate for the uid daily"""
uid_daily_t.update({"_id":date},{"$set":{
"Nvideo":uidvideonums,
"Engage":operator.itruediv(float(redata.hget("T_engagesum_%s_%s"%(uid,date),uid)),uiddailyplay),
"Click":operator.itruediv(uiddailyplay,uiddailyload)
}},upsert=True,w=0)
# --/
# 统计用户截至到目前播放信息
# --/
region_list = list(redata.smembers("T_region_%s_%s"%(uid,date)))
for i in region_list:
LoadSum = redata.hget("T_regiontable_%s"%(date),"%s%s"%(uid,i))
if LoadSum == None:LoadSum = 0
uid_stat_t.update({"_id":uid},{"$inc":{
"Geo.%s.Load"%(i):int(LoadSum)
}},upsert=True,w=0)
uid_daily_t.update({"_id":date},{"$set":{
"Geo.%s.Load"%(i):int(LoadSum)
}},upsert=True,w=0)
"""获取用户截至到目前载入次数,播放次数,完成率之和"""
uidnowstat = uid_stat_t.find_one({"_id":uid},{"_id":0,"Load":1,"Play":1,"EngageSum":1})
uidnowload = int(uidnowstat["Load"])
uidnowplay = int(uidnowstat["Play"])
uidnowengagesum = float(uidnowstat["EngageSum"])
"""video nums for the uid up to now"""
"""average complete rate of total for the uid up to now"""
"""Calculate avarge click rate for the uid up to now"""
uid_stat_t.update({"_id":uid},{"$set":{
"Nvideo":uidvideonums,
"Engage":operator.itruediv(uidnowengagesum,uidnowplay),
"Click":operator.itruediv(uidnowplay,uidnowload)
}})
request.end()
'''--结束--'''
# -- /
# 删除T_UIDS_%date中已完成条目
# -- /
redata.srem("T_UIDS_%s"%date,uid)
self.queue.task_done()
except IOError,e:
self.log.error("handleTUIDSET -- handle -- exception,%s"%(e))
break
def run(self):
redispool = redis.ConnectionPool(host=self.redataIP,port=6379,db=0)
redata = redis.Redis(connection_pool=redispool)
mongoconn = MongoClient(self.mongodbIP,27017)
date = self.datadate
while True:
try:
"""Obtain uid name for have data changs daily"""
uid = self.queue.get()
self.log.debug("Handling uid is %s"%uid)
"""Obtain video nums from redis db for the uid"""
uidvideonums = len(redata.keys("%s???_info"%uid))
"""--开始--"""
# --/
# 建立mongodb表连接
# --/
uid_stat_t = mongoconn["video"]["%s_stat"%(uid)]
uid_vdaily_t = mongoconn["video"]["%s_vdaily"%(uid)]
uid_daily_t = mongoconn["video"]["%s_daily"%(uid)]
uid_type_t = mongoconn["video"]["%s_type"%(uid)]
request = mongoconn.start_request()
# --/
# 统计用户每天播放信息
# --/
"""获取用户视频该天载入次数,播放次数"""
uiddailystat = uid_daily_t.find_one({"_id":date},{"_id":0,"Load":1,"Play":1})
uiddailyload = int(uiddailystat["Load"])
uiddailyplay = int(uiddailystat["Play"])
"""video nums for the uid"""
"""average complete rate for the uid daily"""
"""Calculate avarge click rate for the uid daily"""
uid_daily_t.update({"_id":date},{"$set":{
"Nvideo":uidvideonums,
"Engage":operator.itruediv(float(redata.hget("T_engagesum_%s_%s"%(uid,date),uid)),uiddailyplay),
"Click":operator.itruediv(uiddailyplay,uiddailyload)
}},upsert=True,w=0)
# --/
# 统计用户截至到目前播放信息
# --/
region_list = list(redata.smembers("T_region_%s_%s"%(uid,date)))
for i in region_list:
LoadSum = redata.hget("T_regiontable_%s"%(date),"%s%s"%(uid,i))
uid_stat_t.update({"_id":uid},{"$inc":{
"Geo.%s.Load"%(i):int(LoadSum)
}},upsert=True,w=0)
uid_daily_t.update({"_id":date},{"$set":{
"Geo.%s.Load"%(i):int(LoadSum)
}},upsert=True,w=0)
"""获取用户截至到目前载入次数,播放次数,完成率之和"""
uidnowstat = uid_stat_t.find_one({"_id":uid},{"_id":0,"Load":1,"Play":1,"EngageSum":1})
uidnowload = int(uidnowstat["Load"])
uidnowplay = int(uidnowstat["Play"])
uidnowengagesum = float(uidnowstat["EngageSum"])
"""video nums for the uid up to now"""
"""average complete rate of total for the uid up to now"""
"""Calculate avarge click rate for the uid up to now"""
uid_stat_t.update({"_id":uid},{"$set":{
"Nvideo":uidvideonums,
"Engage":operator.itruediv(uidnowengagesum,uidnowplay),
"Click":operator.itruediv(uidnowplay,uidnowload)
}})
# --/
# 统计用户类别播放信息
# --/
"""获取该天有数据变动的类别名"""
kidsdict = uid_type_t.find_one({"_id":date},{"_id":0})
if kidsdict != None :
for kid in kidsdict:
'''获取该类别视频数目'''
sql = '''
SELECT a.vid FROM vinfo AS a
LEFT JOIN vsort AS b ON a.vsortid = b.id
WHERE b.kid= %s
'''
mysql_conn = MySQLdb.connect(host=self.mysqlIP,user=self.mysqlUser,passwd=self.mysqlPassword,port=self.mysqlPort,db=self.mysqlDbname,charset="utf8")
cursor = mysql_conn.cursor()
param = (kid)
n = cursor.execute(sql,param)
kidvideonums = n
# --/
# 统计用户类别每天播放信息
# --/
"""获取用户该类别视频该天载入次数,播放次数"""
kiddailystat = uid_type_t.find_one({"_id":date},{"_id":0,kid:1})
kiddailyload = int(kiddailystat[kid]["Load"])
kiddailyplay = int(kiddailystat[kid]["Play"])
"""video nums for the uid's kid daily"""
"""Calculate avarge complete rate for the uid's kid daily"""
"""Calculate avarge click rate for the uid's kid daily"""
uid_type_t.update({"_id":date},{"$set":{
"%s.Nvideo"%(kid):kidvideonums,
"%s.Engage"%(kid):operator.itruediv(float(redata.hget("T_engagesum_%s_%s"%(uid,date),kid)),kiddailyplay),
"%s.Click"%(kid):operator.itruediv(kiddailyplay,kiddailyload)
}})
request.end()
'''--结束--'''
# -- /
# 删除T_UIDS_%date中已完成条目
# -- /
redata.srem("T_UIDS_%s"%date,uid)
self.queue.task_done()
except IOError,e:
self.log.error("handleTUIDSET -- handle -- exception,%s"%(e))
break
def handleUid(self,uid):
uid_stat_t = self.mongoconn[self.mongodbName]["%s_stat"%(uid)]
uid_daily_t = self.mongoconn[self.mongodbName]["%s_daily"%(uid)]
#初始化uidinfo类
uidinfObj = UidInfo(self.mysql_conn,self.mongoconn,uid)
"""得到该用户本天统计数据"""
uiddailystat = uid_daily_t.find_one({"_id":self.date})
Nvideo = uiddailystat["Nvideo"]
Load = uiddailystat["Load"]
Play = uiddailystat["Play"]
IP = uiddailystat["IP"]
Engage = uiddailystat["Engage"]
Click = uiddailystat["Click"]
Traffic = uiddailystat["Traffic"]
Geo = uiddailystat["Geo"]
"""计算地域分布数据中完成率"""
for region in Geo:
Geo[region]["Engage"] = operator.itruediv(float(Geo[region]["EngageSum"]), int(Geo[region]["ClickNum"]))
"""得到该用户截至目前统计信息"""
uidnowstat = uidinfObj.get_uid_info(uid)
nowGeo = uidnowstat["Geo"]
nowTraffic = float(uidnowstat["Traffic"])
nowMtraffic = float(uidnowstat["Mtraffic"])
"""时间序列已存储多少天数据"""
#days = redata.llen("%s_date"%(uid)) + 1
"""写数据到redis"""
redpipe = self.redata.pipeline()
redpipe.rpush("%s_date"%(uid), self.date)
redpipe.rpush("%s_UD1"%(uid),Nvideo)
redpipe.rpush("%s_UD2"%(uid),Load)
redpipe.rpush("%s_UD3"%(uid),Play)
redpipe.rpush("%s_UD4"%(uid),IP)
redpipe.rpush("%s_UD5"%(uid),Engage)
redpipe.rpush("%s_UD6"%(uid),Click)
redpipe.rpush("%s_UD7"%(uid),Traffic)
redpipe.rpush("%s_UDD"%(uid),ujson.encode(Geo))
redpipe.set("%s_UH"%(uid),ujson.encode(uidnowstat))
redpipe.set("%s_UDH"%(uid),ujson.encode(nowGeo))
#为便于php进行套餐流量统计,将uid用户当天流量和写入redis
datadict = {"date":self.date,"traffic":Traffic}
redpipe.set("%s_traffic"%(uid),ujson.encode(datadict))
"""
if days > self.daynums:
redpipe.lpop("%s_date"%(uid))
redpipe.lpop("%s_UD1"%(uid))
redpipe.lpop("%s_UD2"%(uid))
redpipe.lpop("%s_UD3"%(uid))
redpipe.lpop("%s_UD4"%(uid))
redpipe.lpop("%s_UD5"%(uid))
redpipe.lpop("%s_UD6"%(uid))
redpipe.lpop("%s_UD7"%(uid))
redpipe.lpop("%s_UDD"%(uid))
"""
redpipe.execute()
uidinfObj.update_usum(uid,uidnowstat)
uidinfObj.update_ugeo(uid,uidnowstat)
# --/
# FUNCTION MODULE : COLLECT USER BASE INFORMATION
# USER BASE INFORMATION WRITE TO FILE OF EXECEL,TXT,PDF
# ADD IN 20131210
# --/
uidinfObj.write_zzuinfo_to_file()
def __itruediv__(self, other):
with self._obj_pxy_lock:
proxied = self._obj_pxy_deserialize()
self._obj_pxy["obj"] = operator.itruediv(proxied, other)
return self
def handleVid(self,uid,vid):
uid_stat_t = self.mongoconn["video"]["%s_stat"%(uid)]
vid_play_t = self.mongoconn["video"]["%s_play"%(vid)]
uid_vdaily_t = self.mongoconn["video"]["%s_vdaily"%(uid)]
"""设置用户地域的载入次数"""
"""初始化vidinfo类"""
vidinfObj = VidInfo(self.mysql_conn,self.mongoconn,vid)
"""得到该视频本天统计数据"""
viddailydata = uid_vdaily_t.find_one({"_id":self.date},{"_id":0,vid:1})
Load = viddailydata[vid]["Load"]
Play = viddailydata[vid]["Play"]
IP = viddailydata[vid]["IP"]
Engage = viddailydata[vid]["Engage"]
Geo = viddailydata[vid]["Geo"]
Click = viddailydata[vid]["Click"]
Traffic = viddailydata[vid]["Traffic"]
"""计算地域分布数据中完成率"""
for region in Geo:
Geo[region]["Engage"] = operator.itruediv(float(Geo[region]["EngageSum"]), int(Geo[region]["ClickNum"]))
"""得到该视频实时播放最后100个记录"""
plays_list = []
plays_sum = vid_play_t.count()
MaxSum = 100
if plays_sum > MaxSum:
plays_cursor = vid_play_t.find({"_id":{"$gt":(plays_sum-MaxSum)}})
else:
plays_cursor = vid_play_t.find().limit(MaxSum)
for play in plays_cursor:
vid_playtime = play["PlayTime"]
self.redata.zadd("%s_SD7"%(vid),ujson.encode(play),vid_playtime)
"""得到该视频截至目前统计信息"""
vidnowstat = vidinfObj.get_vid_info(vid)
#vidnowstat = uid_stat_t.find_one({"_id":vid},{"_id":0,"EngageSum":0})
vidnowstat.pop("HeatMapSum")
nowGeo = vidnowstat["Geo"]
nowHtmap = json.loads(vidnowstat["Htmap"])
nowTraffic = vidnowstat["Traffic"]
nowPlay = vidnowstat["Play"]
nowClick = vidnowstat["Click"]
nowPtmap = vidnowstat["Ptmap"]
data = (nowPlay,nowClick,nowTraffic,vid)
Htmp_min = min([k for i,j,k in nowHtmap])
nowHtmap = [[i,j,k/Htmp_min] for i,j,k in nowHtmap ]
"""时间序列已存储多少天数据"""
#days = redata.llen("%s_date"%(vid)) + 1
"""写数据到redis"""
redpipe = self.redata.pipeline()
redpipe.rpush("%s_date"%(vid), self.date)
redpipe.rpush("%s_SD1"%(vid),Load)
redpipe.rpush("%s_SD2"%(vid),Play)
redpipe.rpush("%s_SD3"%(vid),IP)
redpipe.rpush("%s_SD4"%(vid),Engage)
redpipe.rpush("%s_SD5"%(vid),Click)
redpipe.rpush("%s_SD6"%(vid),Traffic)
redpipe.rpush("%s_SDD"%(vid),ujson.encode(Geo))
redpipe.set("%s_PH"%(vid),ujson.encode(nowPtmap))
redpipe.set("%s_SH"%(vid),ujson.encode(vidnowstat))
redpipe.set("%s_SHH"%(vid),ujson.encode(nowGeo))
redpipe.set("%s_JH"%(vid),ujson.encode(nowHtmap))
"""
if days > self.daynums:
redpipe.lpop("%s_date"%(vid))
redpipe.lpop("%s_SD1"%(vid))
redpipe.lpop("%s_SD2"%(vid))
redpipe.lpop("%s_SD3"%(vid))
redpipe.lpop("%s_SD4"%(vid))
redpipe.lpop("%s_SD5"%(vid))
redpipe.lpop("%s_SD6"%(vid))
redpipe.lpop("%s_SDD"%(vid))
"""
redpipe.execute()
sql = '''
update vinfo set playcount=%s,click_rate=%s,traffic=%s where vid = %s
'''
self.cursor.execute(sql,data)
self.mysql_conn.commit()
vidinfObj.update_vsum(vid,vidnowstat)
vidinfObj.update_vgeo(vid,vidnowstat)
vidinfObj.update_vhmap(vid,vidnowstat)
