def main(): #主函数入口 Config = HandleConfig() today = (datetime.datetime.now() - datetime.timedelta(days=1)).strftime("%Y-%m-%d") redispool = redis.ConnectionPool(host=Config.RedisIp, port=6379, db=0) redata = redis.Redis(connection_pool=redispool) mongoconn = MongoClient(Config.MongodbIp, 27017) mysql_conn = MySQLdb.connect(host=Config.MysqldbIp, user=Config.MysqlUser, port=Config.MysqldbPort, passwd=Config.MysqlPassword, db=Config.MysqlDbname, charset="utf8") logger = getLog("Step6", logfile=Config.LogFile, loglevel=Config.LogLevel) logger.info("Step6 Handle Start") set_alluser = redata.smembers("user_info") list_user = list(set_alluser) for i in list_user: uidObj = UidInfo(mysql_conn, mongoconn, i) CheckSpace(Config, uidObj, Mail_list, redata, today, logger) logger.info("Step6 Handle End")
def main(): Config = HandleConfig() redispool = redis.ConnectionPool(host=Config.RedisIp,port=6379,db=0) redata = redis.Redis(connection_pool=redispool) mongoconn = MongoClient(Config.MongodbIp,27017) mysql_conn = MySQLdb.connect(host=Config.MysqldbIp,user=Config.MysqlUser,port=Config.MysqldbPort,passwd=Config.MysqlPassword,db=Config.MysqlDbname,charset="utf8") logger = getLog("Step6",logfile=Config.LogFile,loglevel=Config.LogLevel) logger.info("Step6 Handle Start") set_alluser = redata.smembers("user_info") if redata.exists("CheckFlow_User"): checked_user = redata.smembers("CheckFlow_User") set_user = set_alluser - checked_user list_user = list(set_user) else: list_user = list(set_alluser) for i in list_user: userObj = UidInfo(mysql_conn,mongoconn,i) CheckFlow(userObj,redata,Mail_list) redata.sadd("CheckFlow_User",i) if len(redata.smembers("CheckFlow_User")) == len(set_alluser): logger.info("Step6 Handle End") redata.delete("CheckFlow_User") else: logger.error("Step6 Handel error,please run it again!!!!")
def run(self): while True: uid = self.queue.get() userObj = UidInfo(self.mysql_conn,self.mongoconn,uid) CheckFlow(userObj,self.redata,Mail_list) self.queue.task_done()
def run(self): while True: uid = self.queue.get() userObj = UidInfo(self.mysql_conn, self.mongoconn, uid) playlist_info = userObj.get_uid_playlist(uid) CreatXml(uid, playlist_info, "adaptive", "PC") CreatXml(uid, playlist_info, "video", "IOS") CreatXml(uid, playlist_info, "video", "Android") "写入默认播放列表" defaultplaylist = {} for i in xrange(0, len(playlist_info)): vid_inlist = playlist_info[i][1] defaultplaylist[str(i + 1)] = vid_inlist self.redata.set("%sA0_info" % (uid), ujson.encode(defaultplaylist)) filename1 = uid + "A0.xml" filename2 = uid + "A05.xml" filename3 = uid + "A0A.xml" Base_Hdfs_Path = "/static/" + uid + "/" HdfsObj = WebHadoop(self.Config.HdfsHost, self.Config.HdfsPort, self.Config.HdfsUser, self.logger, self.Config.HdfsPrefix) HdfsObj.put_file("/tmp/%s" % (filename1), Base_Hdfs_Path + filename1, overwrite="true") HdfsObj.put_file("/tmp/%s" % (filename2), Base_Hdfs_Path + filename2, overwrite="true") HdfsObj.put_file("/tmp/%s" % (filename3), Base_Hdfs_Path + filename3, overwrite="true") refresh_url1 = "/%s/%sA0.xml" % (uid, uid) refresh_url2 = "/%s/%sA05.xml" % (uid, uid) refresh_url3 = "/%s/%sA0A.xml" % (uid, uid) self.redata.rpush("cacheurl_list", refresh_url1) self.redata.rpush("cacheurl_list", refresh_url2) self.redata.rpush("cacheurl_list", refresh_url3) self.queue.task_done()
def UserPackageInfo(self): self.UidObj = UidInfo(self.mysql_conn, self.mongoconn, self.uid) flow_info = self.UidObj.get_flow_info(self.uid) '''用户套餐所属类别ID''' self.Now_TypeID = flow_info[7] '''用户email地址''' self.Email = flow_info[8] '''用户套餐开始日期''' self.Now_StartDate = str(flow_info[0]) '''用户套餐结束日期''' self.Now_EndDate = str(flow_info[1]) '''用户套餐购买月数''' self.Now_Month_Count = flow_info[2] '''用户套餐月规定流量''' self.Now_Month_Flow = flow_info[3] '''用户该月已用流量''' self.Now_Month_Used_Flow = flow_info[4] '''用户该月已超流量''' self.Now_Month_Advance_Flow = flow_info[5] '''用户今天消耗流量''' self.Daily_Flow = self.UidObj.get_daily_flow(self.uid) / 1024 '''截至今天用户该月消耗总流量''' self.Month_Sum_Flow = self.Now_Month_Used_Flow + self.Now_Month_Advance_Flow + self.Daily_Flow '''Write to log file for debug''' self.logger.debug( "Now Package info : \n \ Uid : %s \n \ TypeID : %s \n \ Email : %s \n \ StartDate : %s \n \ EndDate : %s \n \ Month_Count : %s \n \ Month_Flow : %s \n \ Month_Used_Flow : %s \n \ Month_Advance_Flow : %s \n \ Daily_Flow : %s \n \ Month_Sum_Flow : %s \n \ " % (self.uid, self.Now_TypeID, self.Email, self.Now_StartDate, self.Now_EndDate, self.Now_Month_Count, self.Now_Month_Flow, self.Now_Month_Used_Flow, self.Now_Month_Advance_Flow, self.Daily_Flow, self.Month_Sum_Flow))
def handleUid(self, uid): uid_stat_t = self.mongoconn["video"]["%s_stat" % (uid)] uid_daily_t = self.mongoconn["video"]["%s_daily" % (uid)] #初始化uidinfo类 uidinfObj = UidInfo(self.mysql_conn, self.mongoconn, uid) """得到该用户本天统计数据""" uiddailystat = uid_daily_t.find_one({"_id": self.date}) Nvideo = uiddailystat["Nvideo"] Load = uiddailystat["Load"] Play = uiddailystat["Play"] IP = uiddailystat["IP"] Engage = uiddailystat["Engage"] Click = uiddailystat["Click"] Traffic = uiddailystat["Traffic"] Geo = uiddailystat["Geo"] """计算地域分布数据中完成率""" for region in Geo: Geo[region]["Engage"] = operator.itruediv( float(Geo[region]["EngageSum"]), int(Geo[region]["ClickNum"])) """得到该用户截至目前统计信息""" uidnowstat = uidinfObj.get_uid_info(uid) #uidnowstat = uid_stat_t.find_one({"_id":uid},{"_id":0,"EngageSum":0}) nowGeo = uidnowstat["Geo"] nowTraffic = float(uidnowstat["Traffic"]) nowMtraffic = float(uidnowstat["Mtraffic"]) """时间序列已存储多少天数据""" #days = redata.llen("%s_date"%(uid)) + 1 """写数据到redis""" redpipe = self.redata.pipeline() redpipe.rpush("%s_date" % (uid), self.date) redpipe.rpush("%s_UD1" % (uid), Nvideo) redpipe.rpush("%s_UD2" % (uid), Load) redpipe.rpush("%s_UD3" % (uid), Play) redpipe.rpush("%s_UD4" % (uid), IP) redpipe.rpush("%s_UD5" % (uid), Engage) redpipe.rpush("%s_UD6" % (uid), Click) redpipe.rpush("%s_UD7" % (uid), Traffic) redpipe.rpush("%s_UDD" % (uid), ujson.encode(Geo)) redpipe.set("%s_UH" % (uid), ujson.encode(uidnowstat)) redpipe.set("%s_UDH" % (uid), ujson.encode(nowGeo)) """ if days > self.daynums: redpipe.lpop("%s_date"%(uid)) redpipe.lpop("%s_UD1"%(uid)) redpipe.lpop("%s_UD2"%(uid)) redpipe.lpop("%s_UD3"%(uid)) redpipe.lpop("%s_UD4"%(uid)) redpipe.lpop("%s_UD5"%(uid)) redpipe.lpop("%s_UD6"%(uid)) redpipe.lpop("%s_UD7"%(uid)) redpipe.lpop("%s_UDD"%(uid)) """ redpipe.execute() uidinfObj.update_usum(uid, uidnowstat) uidinfObj.update_ugeo(uid, uidnowstat)
def handleUid(self, uid): uid_stat_t = self.mongoconn[self.mongodbName]["%s_stat" % (uid)] uid_daily_t = self.mongoconn[self.mongodbName]["%s_daily" % (uid)] #初始化uidinfo类 uidinfObj = UidInfo(self.mysql_conn, self.mongoconn, uid) """得到该用户本天统计数据""" uiddailystat = uid_daily_t.find_one({"_id": self.date}) Nvideo = uiddailystat["Nvideo"] Load = uiddailystat["Load"] Play = uiddailystat["Play"] IP = uiddailystat["IP"] Engage = uiddailystat["Engage"] Click = uiddailystat["Click"] Traffic = uiddailystat["Traffic"] Geo = uiddailystat["Geo"] """计算地域分布数据中完成率""" for region in Geo: Geo[region]["Engage"] = operator.itruediv( float(Geo[region]["EngageSum"]), int(Geo[region]["ClickNum"])) """得到该用户截至目前统计信息""" uidnowstat = uidinfObj.get_uid_info(uid) nowGeo = uidnowstat["Geo"] nowTraffic = float(uidnowstat["Traffic"]) nowMtraffic = float(uidnowstat["Mtraffic"]) """时间序列已存储多少天数据""" #days = redata.llen("%s_date"%(uid)) + 1 """写数据到redis""" redpipe = self.redata.pipeline() redpipe.rpush("%s_date" % (uid), self.date) redpipe.rpush("%s_UD1" % (uid), Nvideo) redpipe.rpush("%s_UD2" % (uid), Load) redpipe.rpush("%s_UD3" % (uid), Play) redpipe.rpush("%s_UD4" % (uid), IP) redpipe.rpush("%s_UD5" % (uid), Engage) redpipe.rpush("%s_UD6" % (uid), Click) redpipe.rpush("%s_UD7" % (uid), Traffic) redpipe.rpush("%s_UDD" % (uid), ujson.encode(Geo)) redpipe.set("%s_UH" % (uid), ujson.encode(uidnowstat)) redpipe.set("%s_UDH" % (uid), ujson.encode(nowGeo)) #为便于php进行套餐流量统计,将uid用户当天流量和写入redis datadict = {"date": self.date, "traffic": Traffic} redpipe.set("%s_traffic" % (uid), ujson.encode(datadict)) """ if days > self.daynums: redpipe.lpop("%s_date"%(uid)) redpipe.lpop("%s_UD1"%(uid)) redpipe.lpop("%s_UD2"%(uid)) redpipe.lpop("%s_UD3"%(uid)) redpipe.lpop("%s_UD4"%(uid)) redpipe.lpop("%s_UD5"%(uid)) redpipe.lpop("%s_UD6"%(uid)) redpipe.lpop("%s_UD7"%(uid)) redpipe.lpop("%s_UDD"%(uid)) """ redpipe.execute() uidinfObj.update_usum(uid, uidnowstat) uidinfObj.update_ugeo(uid, uidnowstat) # --/ # FUNCTION MODULE : COLLECT USER BASE INFORMATION # USER BASE INFORMATION WRITE TO FILE OF EXECEL,TXT,PDF # ADD IN 20131210 # --/ uidinfObj.write_zzuinfo_to_file()