def fetch_job(game_id=1007039):
fetcher.info('start fetch job,gameid:%s...' % game_id)
host, port = get_router(modid, cmdid)
if not (host and port):
print 'l5 get error'
return
udp_client = UDPClient(host=host, port=port)
hero_adjuster = HeroAdjuster(udp_client)
b = BaseDao()
video_dao = Video(b)
batch = 5
batch_num = 10
from_num = 0
try:
obj_arr = hero_adjuster.adjust(batch=batch,
batch_num=batch_num,
from_num=from_num,
game_id=game_id)
for item in obj_arr:
video_dao.add_item(item)
video_dao.add_video_game(item['vid'], game_id)
fetcher.info('fetch job finish, game_id:%s' % game_id)
except:
fetcher.info('fetch job error:%s, game_id:%s' %
(traceback.format_exc(), game_id))
finally:
udp_client.close()
b.close()
def get_urls(self, vids, video_infos):
from lib.config import read_l5_info
from l5.get_router import get_router
from transport.client import UDPClient
from adjuster.adjuster_tool import get_download_url_and_definition
modid,cmdid = read_l5_info()
host,port = get_router(modid, cmdid)
udp_client = UDPClient(host, port)
tmp = []
if video_infos and len(video_infos) and vids and len(vids):
for vid in vids:
filters = filter(lambda o:o.video_id == vid, video_infos)
if not len(filters):
continue
item = filters[0]
vid = item.video_id
game_id = item.game_id
if not (vid and game_id):
logger.warn("vid:%s or game_id:%s is null, skip.."%(vid, game_id))
continue
url,definition = get_download_url_and_definition(vid.encode('utf8'), int(game_id), udp_client=udp_client)
if url:
item.url = url
item.definition = definition
logger.info('get download url:%s of vid:%s, game_id:%s'%(url, vid, game_id))
tmp.append(item)
else:
logger.warn("vid:%s game_id:%s get download url is null, skip.."% (vid, game_id))
return tmp
def get_openid_by_uuid(uuid=0, uin=0):
#need ask them l5.
udp_client = UDPClient(host='10.239.196.159', port=15000)
try:
qt = QtHelper()
sess = qt.createSessInfo(pb_openid.CMD_TICKETSVR,
pb_openid.SUBCMD_PARSE_ID, 8031, uin)
req = pb_openid.ParseIdReq()
req.ids.extend([uuid])
res = qt.buildSendPkg(sess, req)
udp_client.sendData(buf=res.serialize())
bb = udp_client.recvData()
resp = qt.parseReceivePkg(bb)
resp.unserialize()
resp_str = str(buffer(resp.body_str)[:])
pb_resp = pb_openid.ParseIdRsp()
pb_resp.ParseFromString(resp_str)
return pb_resp
except:
logger.error('get error:%s' % traceback.format_exc())
finally:
udp_client.close()
def assemble_v_obj(info, game_id, appid, flag=True):
game_id = game_id or info.game_id
game_id = int(game_id)
v_obj = {}
v_obj['title'] = info.custom_title.title
v_obj['game_id'] = game_id
v_obj['pic_url'] = info.url
v_obj['duration'] = info.video_time * 1000
v_obj['share_time'] = info.share_time
v_obj['nickname'] = info.user_nick or ''
v_obj['user_icon'] = info.user_icon
v_obj['src_uid'] = info.uid
v_obj['vid'] = info.vid
v_obj['good_num'] = info.praise_num
v_obj['play_num'] = info.video_views
v_obj['nickname'] = info.user_nick or ''
v_obj['user_icon'] = info.user_icon or ''
v_obj['definition'] = 0
v_obj['v_url'] = ''
v_obj['category'] = ''
v_obj['second_c'] = ''
v_obj['appid'] = appid
v_obj['tags'] = []
try:
uid_type, src_open_id = _uuid_to_openid(info.uid)
#v_obj['uid'] = src_open_id
v_obj['uid_type'] = uid_type
v_obj['src_id'] = src_open_id
except:
logger.error('uuid to openid fail, ignore video item:%s' %
v_obj['vid'])
logger.error(traceback.format_exc())
return None
tag_info = pb_hero.GameTagInfo()
tag_info.ParseFromString(info.tag_info)
if len(tag_info.kv_list):
for kv in tag_info.kv_list:
v_obj.setdefault('tags', []).append(kv.key + kv.value)
if flag:
modid, cmdid = read_l5_info()
host, port = get_router(modid, cmdid)
if not (host and port):
logger.error('l5 get error')
return v_obj
udp_client = UDPClient(host=host, port=port)
v_url, definition = get_download_url_and_definition(
vid=info.vid, game_id=game_id, udp_client=udp_client)
#v_obj['v_url'] = v_url
v_obj['v_url'] = ''
v_obj['definition'] = definition
return v_obj
def update_job(game_id=1007039):
host, port = get_router(modid, cmdid)
if not (host and port):
print 'l5 get error'
return
b = BaseDao()
video_dao = Video(b)
udp_client = UDPClient(host=host, port=port)
try:
updater.info('start update job...')
up = Updater(video_dao, udp_client)
up.update_url(game_id=game_id)
updater.info('update job finish.')
except:
exstr = traceback.format_exc()
updater.error('update job error:%s' % exstr)
finally:
udp_client.close()
b.close()
def _get_herotime_client(self):
from l5.get_router import get_router
from transport.client import UDPClient
from lib.config import read_l5_info
modid, cmdid = read_l5_info()
host, port = get_router(modid, cmdid)
if not (host and port):
logger.error('l5 error')
return None
udp_client = UDPClient(host=host, port=port)
return udp_client
def _get_download_url(self, vid, game_id):
if not (vid and game_id):
logger.error('need game_id vid')
return ''
from lib.config import read_l5_info
from transport.client import UDPClient
from l5.get_router import get_router
from adjuster.adjuster_tool import get_download_info_req
udp_client = None
try:
modid, cmdid = read_l5_info()
host, port = get_router(modid, cmdid)
if not (host and port):
logger.error('l5 get error')
raise Exception('l5 get error')
udp_client = UDPClient(host=host, port=port)
resp = get_download_info_req(vid=vid.encode("utf-8"),
game_id=int(game_id),
udp_client=udp_client)
if not resp:
logger.warning('vid:%s download info is null.' % vid)
return ''
if len(resp.ul) and len(resp.cl):
url = resp.ul[0]
cl = resp.cl[0]
v_url = '%s%s?vkey=%s' % (url.url, resp.fn, cl.vkey)
return v_url
except:
logger.error(traceback.format_exc())
finally:
if udp_client:
udp_client.close()
return ''
def test_my_video():
from busi.dao.base import BaseDao
from busi.dao.data import Video
import time
t = time.localtime(time.time())
time1 = time.mktime(time.strptime(time.strftime('%Y-%m-%d 00:00:00',
t),'%Y-%m-%d %H:%M:%S'))
#ts = long(time1)
ts = 1519660800
modid,cmdid = read_l5_info()
host,port = get_router(modid, cmdid)
if not (host and port):
print 'l5 get error'
sys.exit(1)
udp_client = UDPClient(host=host, port=port)
hero_adjuster = HeroAdjuster(udp_client)
#resp = get_openid_by_uuid(uuid='LA-o6zB8wQxpM_kk0ns29fqsMh4s3lo')
#print resp
b = BaseDao()
video_dao = Video(b)
try:
#obj_arr = hero_adjuster.adjust(game_id=1007039)
obj_arr, t1, t2 = hero_adjuster.get_my_videos(game_id=1007058,uin=289296918, batch_num=50)
#obj_arr, t1, t2 = hero_adjuster.get_my_videos(game_id=1007058,openid='o6zB8wT0AIXgl6EFGS2sxzFsWyFg')
for obj in obj_arr:
print obj
#if obj['share_time'] >= ts:
# video_dao.add_item(obj)
# video_dao.add_video_game(obj['vid'], 1007039)
print t1, t2
except:
print traceback.format_exc()
finally:
udp_client.close()
def fix_job():
fixer.info('start fix job')
host, port = get_router(modid, cmdid)
if not (host and port):
print 'l5 get error'
return
udp_client = UDPClient(host=host, port=port)
b = BaseDao()
video_dao = Video(b)
try:
up = Updater(video_dao, udp_client)
up.fix_definition()
except:
exstr = traceback.format_exc()
updater.error('update fix error:%s' % exstr)
finally:
b.close()
fixer.info('fix job finish.')
def get_videos_by_batch_users(self, batch_num=5, game_ids=None, appid='wx9ce8f64a4c9b3308', uin=0, openids=[], uins=[], ip='', port=0):
#if not game_ids:
# game_ids = [2103041,1007058,1007039,1007044,1007042,1007053,1007060]
if not (ip and port):
modid,cmdid = read_l5_info()
ip,port = get_router(modid, cmdid)
if not (ip and port):
logger.error('ip or port is null')
return []
udp_client = UDPClient(host=ip, port=port)
end_sec = 0
end_usec = 0
obj_arr = []
try:
for item in openids:
uid = item.get('uid') or ''
if not uid:
continue
begin_sec = item.get('begin_sec') or 0
begin_usec = item.get('begin_usec') or 0
fetcher.info('get_videos_by_batch_users start, begin_sec:%s,begin_usec:%s,batch_num:%s,openid:%s'%(begin_sec,begin_usec,batch_num,uid))
buf = self.build_multi_game_send_buf(game_ids=game_ids,
begin_sec=begin_sec,begin_usec=begin_usec,batch_num=batch_num,
uin=uin,v_type=0,openid=uid)
udp_client.sendData(buf=buf)
for item in uins:
uid = item.get('uid') or ''
if not uid:
continue
begin_sec = item.get('begin_sec') or 0
begin_usec = item.get('begin_usec') or 0
fetcher.info('get_videos_by_batch_users start, begin_sec:%s,begin_usec:%s,batch_num:%s,uin:%s'%(begin_sec,begin_usec,batch_num,uid))
buf = self.build_multi_game_send_buf(game_ids=game_ids,
begin_sec=begin_sec,begin_usec=begin_usec,batch_num=batch_num,
uin=uid,v_type=0)
udp_client.sendData(buf=buf)
loops = len(openids) + len(uins)
while loops > 0:
uid_datas = {'uid':'','end_sec':0, 'end_usec':0, 'datas':[]}
loops -= 1
resp = self.recv_multi_game_resp(udp_client)
if not resp:
fetcher.error('get_videos_by_batch_users error, resp is null')
continue
if resp.result != 0:
fetcher.error('get_videos_by_batch_users error, resp.resp is not 0:%s'%resp)
continue
fetcher.info('get:%s %sitems of get_videos_by_batch_users in fact' % (resp.flag,len(resp.video_list)))
end_sec = resp.end_sec
end_usec = resp.end_usec
flag = resp.flag
for info in resp.video_list:
v_obj = assemble_v_obj(info, None, appid, flag=False)
if not v_obj:
continue
uid_datas['datas'].append(v_obj)
uid_datas['end_sec'] = end_sec
uid_datas['end_usec'] = end_usec
uid_datas['uid'] = flag
obj_arr.append(uid_datas)
except:
logger.error(traceback.format_exc())
finally:
udp_client.close()
return obj_arr
# logger.warn('[WARNING]vid:%s,game_id:%s download url is null'%(vid,game_id))
#def fix_uid(self):
# for o_uid in self.video_dao.get_old_uid():
# if not o_uid:
# continue
# n_uid = get_opedid_by_comid(o_uid)
# if not n_uid:
# logger.warn('[WARNING]get null new uid, old:%s'%n_uid)
# continue
# self.video_dao.update_videos_uid(o_uid, n_uid)
# self.video_dao.update_uid(o_uid, n_uid)
if __name__ == "__main__":
modid, cmdid = read_l5_info()
host, port = get_router(modid, cmdid)
print host, port
if not (host and port):
print 'l5 get error'
sys.exit(1)
udp_client = UDPClient(host, port)
vid = '61bbeb58abdf4813bb230837ae7df521'
game_id = 1007058
from adjuster.adjuster_tool import get_download_url_and_definition
print get_download_url_and_definition(vid=vid,
game_id=game_id,
udp_client=udp_client)
#updater.update_url(game_id)
from uploader.uploader import Uploader
from lib.config import read_l5_info
from l5.get_router import get_router
from transport.client import UDPClient
from busi.dao.base import BaseDao
from busi.dao.data import Video
from busi.dao.common import assemble
from busi.dao.common import valid
modid, cmdid = read_l5_info()
host, port = get_router(modid, cmdid)
if not (host and port):
logger.error('l5 get error')
sys.exit(1)
udp_client = UDPClient(host=host, port=port)
b = BaseDao()
try:
resolver = Resolver()
results = resolver.start()
processor = VideoProcessor()
for item in results:
candidates = item['candidates']
game_id = item['cmd']['game_id']
merge_info = processor.work(candidates)
if not (merge_info and merge_info.get('file_path')):
print 'merge fail, exit'
sys.exit(1)
def _merge_and_up(self, video_infos, jid, b, title, userinfo, game_id):
from video_processor.processor import VideoProcessor
from uploader.uploader import Uploader
from lib.config import read_l5_info
from l5.get_router import get_router
from transport.client import UDPClient
from busi.dao.common import assemble
from busi.dao.common import valid
vid = ''
fail_res = (False, vid)
succ_res = (True, vid)
if not (video_infos and len(video_infos)):
logger.error('video infos is null, jid:%s fail.'% jid)
return fail_res
try:
processor = VideoProcessor()
modid,cmdid = read_l5_info()
host,port = get_router(modid, cmdid)
if not (host and port):
logger.error('l5 get error')
return
merge_info = processor.work(video_infos)
file_path = merge_info.get('file_path')
workspace = merge_info.get('workspace')
if not (merge_info and file_path):
logger.error('merge fail, jid:%s fail.'% jid)
return fail_res
udp_client = UDPClient(host=host, port=port)
up = Uploader(file_path, 'mp4', udp_client)
up_info = up.upload(addrtype=2, game_id=game_id)
#up_info = up.upload(addrtype=2)
if workspace:
import shutil
logger.info('delete workspace:%s'%workspace)
shutil.rmtree(workspace)
if up_info['status'] != 0:
logger.error('upload merge_file fail, jid:%s fail.'%jid)
return fail_res
vid = up_info['vid']
if not vid:
logger.error('upload merge_file fail, jid:%s fail.'%jid)
return fail_res
assemble_obj = assemble(video_infos, up_info, udp_client)
if assemble_obj and valid(assemble_obj):
#assemble_obj['nickname'] = userinfo[0]
assemble_obj['uid'] = userinfo[1]
assemble_obj['title'] = title or assemble_obj['title']
self.video_dao = Video(b)
#print assemble_obj
self.video_dao.add_item(assemble_obj)
self.video_dao.add_video_game(vid, game_id)
#self.chuan_dao.update_chuan_status(jid=jid, status=1, vid=vid)
else:
logger.info('jid:%s assemble fail.'%jid)
return fail_res
logger.info('jid:%s finish success.'%jid)
succ_res = (True, vid)
return succ_res
except:
logger.error(traceback.format_exc())
return fail_res