def get_playlist(self, url):
if 'dayi.ca/' in url:
return []
hutf = self.get_hutf(url)
#m = re.search(U("通用版.+第(\d+)集"), p[3].text)
#if m:
# max_id = int(m.group(1))
#else:
# m = re.search(U("首播:.+共(\d+)集"), p[0].text) #, flags=re.M+re.U)
# max_id = int(m.group(1))
t = SelStr("h1.entry-title", hutf)[0]
m = re.search(U("(.+) .+第(\d+)集"), t.text)
if m:
title, max_id = m.group(1).strip(), int(m.group(2))
else:
m = re.search(u"(.+) (\d+)集全", t.text)
if m:
title, max_id = m.group(1).strip(), int(m.group(2))
#ps = SelStr("div.entry-content p", hutf)
#for a in p[1].select("a"):
al = SelStr("div.entry-content p a", hutf)
for a in al:
uo = urlparse.urlparse(a['href'])
qs = urlparse.parse_qs(uo.query)
if 'p' in qs and 'page' in qs:
pn = int(qs['p'][0])
break
us = [(U("%s_第%02d集") % (title, i),
"http://www.dayi.ca/ys/?p=%d&page=%d" % (pn, i))
for i in range(1, max_id + 1)]
debug(us)
return us
def detail_key(self, url):
ci = get_ci(debug())
qnrr = Queue()
def nrr(ci, msg):
url = msg['params']['response']['url']
if '/api/video/detail' in url:
qnrr.put(msg['params']['requestId'])
ci.reg("Network.responseReceived", nrr)
try:
ci.Page.navigate(url=url)
req_id = qnrr.get(timeout=ci.get_to())
ret = ci.Network.getResponseBody(requestId=req_id)
body = json.loads(ret['result']['body'])
bdi0 = body['data']['info'][0]
title, channel = bdi0['title'], bdi0['channel']
keys = [g['key'] for g in bdi0['guestSeriesList']]
debug("keys = ", keys)
return channel, title, keys
except Exception as e:
echo("detail_key out:", repr(e))
finally:
ci.close()
return None, None, []
def get_title(self, tvid, sect):
u = "https://www.kantv6.com/index.php/video/info"
u = "%s?tvid=%s&seo=%s" % (u, tvid, sect)
dat = self.get_hutf(u)
dat = json.loads(dat)
debug(json.dumps(dat, indent=2))
return dat['data']['title']
def query_info(self, url):
url = 'http://tv.sohu.com/20110220/n279432193.shtml'
url = 'http://tv.sohu.com/20150705/n416207533.shtml'
html = self.get_html(url)
vid = mg1(r'\Wvid\s*[\:=]\s*[\'"]?(\d+)[\'"]?', html)
echo('vid =', vid)
hutf = self.get_data_by_vid(vid)
debug(hutf)
data = json.loads(hutf)
for qtyp in ["oriVid", "superVid", "highVid", "norVid", "relativeId"]:
hqvid = data['data'][qtyp]
if hqvid != 0 and hqvid != vid:
break
debug(qtyp)
data = json.loads(self.get_data_by_vid(hqvid))
debug(data)
host = data['allot']
prot = data['prot']
tvid = data['tvid']
urls = []
data = data['data']
title = data['tvName']
size = sum(data['clipsBytes'])
ret = []
for new, cu, ck in zip(data['su'], data['clipsURL'], data['ck']):
urls.append(("%s%02d.mp4" % (title, len(ret) + 1),
self.real_url(host, vid, tvid, new, cu, ck)))
debug("title=%s, size=%d" % (title, size))
debug(urls)
def get_title(wo, obj):
req_url = obj['params']['response']['url']
for n in ('/v3/video/detail', '/v3/video/languagesplaylist'):
if n in req_url:
req_id = obj['params']['requestId']
debug("put", n, req_id)
cht.put((n, req_id))
def query_info(self, url):
url = 'http://tv.sohu.com/20110220/n279432193.shtml'
url = 'http://tv.sohu.com/20150705/n416207533.shtml'
html = self.get_html(url)
vid = mg1(r'\Wvid\s*[\:=]\s*[\'"]?(\d+)[\'"]?', html)
echo('vid =', vid)
hutf = self.get_data_by_vid(vid)
debug(hutf)
data = json.loads(hutf)
for qtyp in ["oriVid", "superVid", "highVid", "norVid", "relativeId"]:
hqvid = data['data'][qtyp]
if hqvid != 0 and hqvid != vid:
break
debug(qtyp)
data = json.loads(self.get_data_by_vid(hqvid))
debug(data)
host = data['allot']
prot = data['prot']
tvid = data['tvid']
urls = []
data = data['data']
title = data['tvName']
size = sum(data['clipsBytes'])
ret = []
for new, cu, ck in zip(data['su'], data['clipsURL'], data['ck']):
urls.append(("%s%02d.mp4" % (title, len(ret) + 1),
self.real_url(host, vid, tvid, new, cu, ck)))
debug("title=%s, size=%d" % (title, size))
debug(urls)
def getvkey(self, url, vid, vt, resolution, filename):
rand = random.random()
ckey = echo_ckeyv3(vid,
PLAYER_GUID,
rand,
player_version=PLAYER_VERSION,
platform=PLATFORM)
params = {
'guid': PLAYER_GUID,
'platform': PLATFORM,
'vt': vt,
'linkver': 2,
'vid': vid,
'lnk': vid,
'charge': 0,
'cKey': ckey,
'encryptVer': '5.4',
'otype': 'xml',
'filename': filename,
'ehost': url,
'format': resolution,
'appver': PLAYER_VERSION,
'ran': rand,
}
hutf = self.get_hutf('http://vv.video.qq.com/getvkey',
postdata=urllib.urlencode(params))
mp = MyHtmlParser(tidy=False)
mp.feed(hutf)
debug(str(mp.root_node))
return {
'filename': mp.select('filename')[0].text,
'br': float(mp.select('br')[0].text),
'key': mp.select('key')[0].text
}
def find_murl(self, url):
#url = 'https://www.olevod.com/index.php/vod/play/id/24986/sid/1/nid/76.html'
ci = get_ci()
murl = ""
qnrwb = Queue()
def nrwb(ci, msg):
u = msg['params']['request']['url']
debug("Network.requestWillBeSent url", url)
if 'master.m3u8' in u:
debug("got master.m3u8", u)
qnrwb.put(u)
ci.reg("Network.requestWillBeSent", nrwb)
try:
ci.Page.navigate(url=url)
murl = qnrwb.get(timeout=ci.get_to())
debug("murl = ", murl)
return murl
#except Exception as e:
# echo("key_m3u8 out:", repr(e))
finally:
ci.close()
def getvkey(self, url, vid, vt, resolution, filename):
rand = random.random()
ckey = echo_ckeyv3(vid, PLAYER_GUID, rand,
player_version=PLAYER_VERSION, platform=PLATFORM)
params = {
'guid': PLAYER_GUID,
'platform': PLATFORM,
'vt': vt,
'linkver': 2,
'vid': vid,
'lnk': vid,
'charge': 0,
'cKey': ckey,
'encryptVer': '5.4',
'otype': 'xml',
'filename': filename,
'ehost': url,
'format': resolution,
'appver': PLAYER_VERSION,
'ran': rand,
}
hutf = self.get_hutf('http://vv.video.qq.com/getvkey',
postdata=urllib.urlencode(params))
mp = MyHtmlParser(tidy=False)
mp.feed(hutf)
debug(str(mp.root_node))
return {
'filename': mp.select('filename')[0].text,
'br': float(mp.select('br')[0].text),
'key': mp.select('key')[0].text
}
def get_m3u8(self, url):
# https://www.kantv6.com/tvdrama/301948271219001-161948271219033
# https://www.kantv6.com/index.php/video/play?tvid=301948271219001&part_id=161948271219033&line=1&seo=tvdrama
sect, tvid, ptid = self.get_stp(url)
title = self.get_title(tvid, sect)
du = "https://www.kantv6.com/index.php/video/play"
if sect in ('movie', 'anime'):
du = "%s?tvid=%s&line=1&seo=%s" % (du, tvid, sect)
elif sect in ('tvdrama', 'show', 'liveshow', 'documentary'):
if not ptid:
echo("no ptid")
return
du = "%s?tvid=%s&part_id=%s&line=1&seo=%s" % (du, tvid, ptid, sect)
else:
echo("Unknown Sect", sect)
return
dat = self.get_hutf(du)
dat = json.loads(dat)
if sect in ('tvdrama', 'show', 'liveshow', 'documentary'):
title = title + "_" + dat['data']['part_title']
debug(json.dumps(dat, indent=2))
echo("title", title)
#return
#us = self.try_m3u8('https:' + dat['data']['url'])
#return title, None, us, None
return title, 'https:' + dat['data']['url']
def get_one(self, url, t=UTITLE, n=False):
if t == UTITLE:
hutf = self.get_hutf(url)
t = match1(hutf, '\<meta name="title" content="([^"]+)"\>')
#echo(url, t)
echo("download", t)
echo("")
#return
dn = os.path.dirname(os.path.abspath(__file__))
fn = os.path.abspath(os.path.join(dn, "../you-get/you-get"))
p = Popen([fn, "-o", self.out_dir, "--no-caption", url])
p.wait()
if not self.parsed_args.post_uri:
return
for i in range(1, len(t)):
n = t[:i] + "*"
debug("ls " + n)
ls = glob(n)
if len(ls) == 0:
break
if len(ls) == 1:
post_file(ls[0], self.parsed_args.post_uri)
return
raise Exception("can not find " + t)
def test1(self, argv):
mu = self.get_m3u8(argv.url)
echo(mu)
sect, tvid, ptid = self.get_stp(argv.url)
u = 'https://www.kantv6.com/index.php/video/part'
u = '%s?tvid=%s' % (u, tvid)
dat = self.get_hutf(u)
dat = json.loads(dat)
debug(json.dumps(dat, indent=2))
def query_info(self, url):
hutf = self.get_hutf(url)
dat = match1(hutf, r"var\s+player_data\s*\=\s*({[^}]+})")
debug(dat)
mu = self.last_m3u8(json.loads(dat)['url'])
#us = self.try_m3u8(u)
t = SelStr("h2.title", hutf)[0]
title = '_'.join(t.text.split())
return title, "m3u8", mu, None
def query_info(self, url):
# return title, ext, urls, size
vid = self.get_vid_from_url(url)
echo("vid =", vid)
api_url = 'http://play.youku.com/play/get.json?vid=%s&ct=12' % vid
hutf = self.get_hutf(api_url)
meta = json.loads(hutf)
data12 = meta['data']
data = data12
debug(data)
title = data['video']['title']
sec_ep = data12['security']['encrypt_string']
sec_ip = data12['security']['ip']
echo(title, sec_ep, sec_ip)
stype, ext, stm = self.get_stream(data)
# get url list
e_code = self.trans_e(self.f_code_1,
base64.b64decode(sec_ep.encode('ascii')))
#base64.b64decode(bytes(sec_ep, 'ascii')) )
sid, token = e_code.split('_')
echo(sid, token)
urls = []
streamfileid = stm['stream_fileid']
for no, seg in enumerate(stm['segs'], 0):
k = seg['key']
if k == -1:
raise # we hit the paywall; stop here
fileid = "%s%02X%s" % (streamfileid[0:8], no, streamfileid[10:])
ep = self.generate_ep(fileid, sid, token)
q = parse.urlencode(dict(
ctype = self.ctype,
ev = 1,
K = k,
ep = parse.unquote(ep),
oip = str(sec_ip),
token = token,
yxon = 1
))
u = 'http://k.youku.com/player/getFlvPath/sid/{sid}_00' \
'/st/{container}/fileid/{fileid}?{q}'.format(
sid = sid,
container = ext,
fileid = fileid,
q = q
)
html = self.get_html(u, True)
for i in json.loads(html.decode("utf8")):
echo(i['server'])
urls.append(i['server'])
k, size = self.get_total_size(urls)
return title, ext, urls, size
def sel_stream(self, data, stype="Auto"):
s = None
m = 1
for d in data['stream']:
if d['stream_type'] == stype:
s = d
break
#if d['total_size'] > m:
l = d.get('total_size', 0)
if l == 0:
l = d.get('size', 0)
if l > m:
m = l
s = d
debug("stream_type =", s['stream_type'],
", total_size =", s['total_size'])
return s
def sel_stream(self, data, stype="Auto"):
s = None
m = 1
for d in data['stream']:
if d['stream_type'] == stype:
s = d
break
#if d['total_size'] > m:
l = d.get('total_size', 0)
if l == 0:
l = d.get('size', 0)
if l > m:
m = l
s = d
debug("stream_type =", s['stream_type'], ", total_size =",
s['total_size'])
return s
def test1(self, args):
#url = 'https://www.ifsp.tv/play?id=lSqo26L8OME'
url = 'https://www.ifsp.tv/play?id=PBPhNoEyRaP'
ci = get_ci(debug=debug())
# "Network.requestWillBeSent"
# chunklist.m3u8
chm = Queue()
def get_murl(wo, obj):
u = obj['params']['request']['url']
if 'chunklist.m3u8' in u:
chm.put(u)
ci.reg("Network.requestWillBeSent", get_murl)
# "Network.responseReceived"
cht = Queue()
def get_title(wo, obj):
req_url = obj['params']['response']['url']
#/v3/video/detail # title
#/v3/video/languagesplaylist # play list
#if '/v3/video/play' not in req_url:
# return
for n in ('/v3/video/detail', '/v3/video/languagesplaylist'):
if n in req_url:
req_id = obj['params']['requestId']
debug("put", n, req_id)
cht.put((n, req_id))
ci.reg("Network.responseReceived", get_title)
ci.Page.navigate(url=url)
title, pl = "", []
for i in range(2):
n, req_id = cht.get()
debug("get", n, req_id)
info = ci.Network.getResponseBody(requestId=req_id)
dat = json.loads(info['result']['body'])['data']['info'][0]
print(n, json.dumps(dat, indent=2))
if 'detail' in n:
title = dat['title']
if 'playlist' in n:
pl = dat['playList']
if title and pl:
break
murl = chm.get()
ci.close()
debug("playlist", pl)
debug("title", title)
debug("murl", murl)
def getVMS(self, tvid, vid):
t = int(time.time() * 1000)
src = '76f90cbd92f94a2e925d83e8ccd22cb7'
key = 'd5fb4bd9d50c4be6948c97edd7254b0e'
msg = (str(t) + key + vid).encode('utf-8')
#sc = hashlib.new('md5',
# bytes(str(t) + key + vid,
# 'utf-8')).hexdigest()
sc = hashlib.new('md5', msg).hexdigest()
#u = 'http://cache.m.iqiyi.com/tmts' + \
# '/{0}/{1}/?t={2}&sc={3}&src={4}'.format(tvid,
# vid,
# t, sc, src)
u = "http://cache.m.iqiyi.com/tmts/%s/%s/?t=%d&sc=%s&src=%s" % (
tvid, vid, t, sc, src)
#data = self.get_hutf(vmsreq)
data = self.get_hutf(u)
debug(data)
return json.loads(data)
def getVMS(self, tvid, vid):
t = int(time.time() * 1000)
src = '76f90cbd92f94a2e925d83e8ccd22cb7'
key = 'd5fb4bd9d50c4be6948c97edd7254b0e'
msg = (str(t) + key + vid).encode('utf-8')
#sc = hashlib.new('md5',
# bytes(str(t) + key + vid,
# 'utf-8')).hexdigest()
sc = hashlib.new('md5', msg).hexdigest()
#u = 'http://cache.m.iqiyi.com/tmts' + \
# '/{0}/{1}/?t={2}&sc={3}&src={4}'.format(tvid,
# vid,
# t, sc, src)
u = "http://cache.m.iqiyi.com/tmts/%s/%s/?t=%d&sc=%s&src=%s" % (
tvid, vid, t, sc, src)
#data = self.get_hutf(vmsreq)
data = self.get_hutf(u)
debug(data)
return json.loads(data)
def query_info1(self, url):
#url = 'https://www.dnvod.eu/Movie/Readyplay.aspx?id=deYM01Pf0bo%3d'
hutf = self.get_hutf(url)
title = SelStr('span#bfy_title >', hutf)[0].data.strip()
debug('title =', title)
for script in SelStr('script', hutf):
txt = script.text
debug('txt =', txt)
if 'PlayerConfig' not in txt:
continue
debug('got PlayerConfig')
vid = match1(txt, "id:\s*'([^']+)',")
key = match1(txt, "key:\s*'([^']+)',")
debug('vid =', vid, ', key =', key)
break
u = "https://www.dnvod.eu/Movie/GetResource.ashx?id=%s&type=htm" % vid
self.extra_headers['Referer'] = url
durl = self.get_html(u, postdata="key=" + key)
debug(durl)
return title, None, [durl], None
def query_info1(self, url):
#url = 'https://www.dnvod.eu/Movie/Readyplay.aspx?id=deYM01Pf0bo%3d'
hutf = self.get_hutf(url)
title = SelStr('span#bfy_title >', hutf)[0].data.strip()
debug('title =', title)
for script in SelStr('script', hutf):
txt = script.text
debug('txt =', txt)
if 'PlayerConfig' not in txt:
continue
debug('got PlayerConfig')
vid = match1(txt, "id:\s*'([^']+)',")
key = match1(txt, "key:\s*'([^']+)',")
debug('vid =', vid, ', key =', key)
break
u = "https://www.dnvod.eu/Movie/GetResource.ashx?id=%s&type=htm" % vid
self.extra_headers['Referer'] = url
durl = self.get_html(u, postdata="key=" + key)
debug(durl)
return title, None, [durl], None
def get_vinfo_do(self, url, vid, fmt):
#getvinfo(target_dir, page_url, video_info['vid'])
rand = random.random()
ckey = echo_ckeyv3(vid,
PLAYER_GUID,
rand,
player_version=PLAYER_VERSION,
platform=PLATFORM)
params = {
'newplatform': PLATFORM,
'guid': PLAYER_GUID,
'pid': PLAYER_PID,
'speed': random.randint(5000, 9000),
'vids': vid,
'fp2p': 1,
'dtype': 3,
'linkver': 2,
'ehost': url,
'fhdswitch': 0,
'cKey': ckey,
'vid': vid,
'appver': PLAYER_VERSION,
'ran': '%.16f' % rand,
'utype': 0,
'encryptVer': '5.4',
'defnpayver': 1,
'charge': 0,
'ip': '',
'otype': 'xml',
'platform': PLATFORM,
}
if fmt:
params['defaultfmt'] = fmt
params['defn'] = fmt
hutf = self.get_html('http://vv.video.qq.com/getvinfo',
postdata=urllib.urlencode(params))
#echo(hutf)
mp = MyHtmlParser(tidy=False)
mp.feed(hutf)
debug(str(mp.root_node))
return mp
def get_playlist(self, url):
res = urlparse(url)
pre = "%s://%s" % (res.scheme, res.netloc)
debug("pre =", pre)
hutf = self.get_hutf(url)
urls = []
#for node in SelStr('a[_stat="videolist:click"]', hutf):
for node in SelStr('div.mod_episode > span.item > '
'a[_stat="videolist:click"][title]', hutf):
n = node.text.strip()
try:
cnt = int(n)
except ValueError:
continue
ii = node.select('i.mark_v > img[alt]')
if ii and ii[0]['alt'] == U('预告'):
continue
u = (n + node['title'], pre + node['href'])
debug(*u)
urls.append(u)
return urls
def title_murl(self, url):
ci = get_ci(debug=debug())
# "Network.requestWillBeSent"
# chunklist.m3u8
chm = Queue()
def get_murl(wo, obj):
u = obj['params']['request']['url']
if 'chunklist.m3u8' in u:
chm.put(u)
ci.reg("Network.requestWillBeSent", get_murl)
# "Network.responseReceived"
cht = Queue()
def get_title(wo, obj):
req_url = obj['params']['response']['url']
for n in ('/v3/video/detail', '/v3/video/languagesplaylist'):
if n in req_url:
req_id = obj['params']['requestId']
debug("put", n, req_id)
cht.put((n, req_id))
ci.reg("Network.responseReceived", get_title)
ci.Page.navigate(url=url)
title, pl = "", []
for i in range(2):
n, req_id = cht.get()
debug("get", n, req_id)
info = ci.Network.getResponseBody(requestId=req_id)
dat = json.loads(info['result']['body'])['data']['info'][0]
print(n, json.dumps(dat, indent=2))
if 'detail' in n:
title = dat['title']
if 'playlist' in n:
pl = dat['playList']
if title and pl:
break
murl = chm.get()
ci.close()
debug("playlist", pl)
debug("title", title)
debug("murl", murl)
return title, murl, [(p['key'], p['name']) for p in pl]
def query_info(self, url):
hutf = self.get_hutf(url)
debug(hutf)
title = SelStr('title', hutf)[0].text
k = None
if title.endswith('.mp4'):
title, k = title[:-4], 'mp4'
#url = "https://www.rapidvideo.com/embed/FUZ35WDLM7"
# https://www3731.playercdn.net/187/0/G4i-UJ6bQxIZI6FWc_F5dg/1536365722/180905/692FUZ37O792IXDCUZDFX.mp4
v = SelStr("video#videojs source", hutf)
if v:
u = v[0]["src"]
return title, k, [u], None
#url = 'https://www.rapidvideo.com/embed/ZsNSciBj'
# https://admkis.playercdn.net/85/1/sQ52oTwwZ6vCo3Vk7-RS2g/1482741547/161202/063k10VmKldzoX8.mp4
hutf = self.get_hutf(url, postdata='block=1')
data = match1(hutf, 'jwplayer\("home_video"\)\.setup\(([^\(\)]+)\);')
debug(data)
data = match1(data, '"sources":\s*(\[[^\[\]]+\])')
ml, u = 0, ''
for src in json.loads(data):
l = src['label']
if l not in self.labels:
echo("new label", l)
i = self.labels.index(l)
if i > ml:
ml, u = i, src['file']
debug(title, u)
return title, k, [u], None
def query_info(self, url):
hutf = self.get_hutf(url)
debug(hutf)
title = SelStr('title', hutf)[0].text
k = None
if title.endswith('.mp4'):
title, k = title[:-4], 'mp4'
#url = "https://www.rapidvideo.com/embed/FUZ35WDLM7"
# https://www3731.playercdn.net/187/0/G4i-UJ6bQxIZI6FWc_F5dg/1536365722/180905/692FUZ37O792IXDCUZDFX.mp4
v = SelStr("video#videojs source", hutf)
if v:
u = v[0]["src"]
return title, k, [u], None
#url = 'https://www.rapidvideo.com/embed/ZsNSciBj'
# https://admkis.playercdn.net/85/1/sQ52oTwwZ6vCo3Vk7-RS2g/1482741547/161202/063k10VmKldzoX8.mp4
hutf = self.get_hutf(url, postdata='block=1')
data = match1(hutf, 'jwplayer\("home_video"\)\.setup\(([^\(\)]+)\);')
debug(data)
data = match1(data, '"sources":\s*(\[[^\[\]]+\])')
ml, u = 0, ''
for src in json.loads(data):
l = src['label']
if l not in self.labels:
echo("new label", l)
i = self.labels.index(l)
if i > ml:
ml, u = i, src['file']
debug(title, u)
return title, k, [u], None
def query_info(self, url):
# url = 'http://www.acfun.cn/v/ac3526338'
hutf = self.get_hutf(url)
for s in SelStr("script", hutf):
t = s.text.strip()
if not t.startswith("var pageInfo = "):
continue
j = json.loads(t[15:])
vid = j['videoId']
title = j['title']
break
echo("vid=", vid, "title=", title)
info_url = 'http://www.acfun.cn/video/getVideo.aspx?id=%s' % vid
info = json.loads(self.get_hutf(info_url))
web = "http://aplay-vod.cn-beijing.aliyuncs.com/acfun/web?vid=%s&ct=85&ev=2&sign=%s&time=%d" % (
info['sourceId'], info['encode'], time.time() * 1000)
#1489616965963
#1490378161
d = json.loads(self.get_hutf(web))
#key = "328f45d8"
key = "2da3ca9e"
ext = "mp4"
data = json.loads(rc4(key, base64.b64decode(d['data'])))
stream = self.sel_stream(data, 'mp4hd3')
segs = stream.get('segs', [])
m3u8 = stream.get('m3u8', '')
styp = stream["stream_type"]
l = stream.get('total_size', 0)
if l == 0:
l = stream.get('size', 0)
if segs:
urls = [s['url'] for s in segs]
elif m3u8:
debug(m3u8)
urls = self.try_m3u8(m3u8)
ext = "ts"
# title, ext, urls, size
return title, ext, urls, l
def query_info1(self, url):
# https://openload.io/embed/igdtpdeGltM/
# https://openload.co/embed/isCWWnlsZLE/
# https://openload.io/embed/biw7ytfelzU/
# <span id="streamurl">isCWWnlsZLE~1481138074~208.91.0.0~g617lYdo</span>
echo("phantomjs wait 300 ...")
p = Popen(["./phantomjs", "dwm.js", "300", url], stdout=PIPE)
html = p.stdout.read()
hutf = html.decode('utf8')
p.wait()
debug(hutf)
n = SelStr('h6', hutf)
if n:
echo(n[0].text)
return self.title, None, [], None
#vid = match1(url, r'haiuken.com/theatre/([^/]+)/')
m = re.search('''openload.co/embed/([^/]+)/''', url)
if m:
uid = m.groups()[0]
echo(uid)
m = re.search('''<span id="streamurl">([^<>]+)</span>''', hutf)
vid = m.groups()[0]
if not vid.startswith(uid): # TODO, try to decode it
vid = uid + "~1497803146~64.180.0.0~eBodZDZa"
echo(vid)
url = "https://openload.co/stream/%s?mime=true" % vid
# "https://openload.co/embed/kUEfGclsU9o/"
n = SelStr("meta[name=og:title]", hutf)
if n and self.title == UTITLE:
self.title = n[0]['content'] # ="skyrim_no-audio_1080.mp4">"
# https://openload.co/stream/isCWWnlsZLE~1481139117~208.91.0.0~mcLfSy5C?mime=true
# video/mp4 584989307
k, tsize = get_kind_size(url)
k = k.split('/')[-1]
if self.title.endswith('.' + k):
self.title = self.title[:-4]
return self.title, k, [url], tsize
def query_info(self, url):
# url = 'http://www.acfun.cn/v/ac3526338'
hutf = self.get_hutf(url)
for s in SelStr("script", hutf):
t = s.text.strip()
if not t.startswith("var pageInfo = "):
continue
j = json.loads(t[15:])
vid = j['videoId']
title = j['title']
break
echo("vid=", vid, "title=", title)
info_url = 'http://www.acfun.cn/video/getVideo.aspx?id=%s' % vid
info = json.loads(self.get_hutf(info_url))
web = "http://aplay-vod.cn-beijing.aliyuncs.com/acfun/web?vid=%s&ct=85&ev=2&sign=%s&time=%d" % (
info['sourceId'], info['encode'], time.time() * 1000)
#1489616965963
#1490378161
d = json.loads(self.get_hutf(web))
#key = "328f45d8"
key = "2da3ca9e"
ext = "mp4"
data = json.loads(rc4(key, base64.b64decode(d['data'])))
stream = self.sel_stream(data, 'mp4hd3')
segs = stream.get('segs', [])
m3u8 = stream.get('m3u8', '')
styp = stream["stream_type"]
l = stream.get('total_size', 0)
if l == 0:
l = stream.get('size', 0)
if segs:
urls = [s['url'] for s in segs]
elif m3u8:
debug(m3u8)
urls = self.try_m3u8(m3u8)
ext = "ts"
# title, ext, urls, size
return title, ext, urls, l
def get_playlist(self, url):
res = urlparse(url)
pre = "%s://%s" % (res.scheme, res.netloc)
debug("pre =", pre)
hutf = self.get_hutf(url)
urls = []
#for node in SelStr('a[_stat="videolist:click"]', hutf):
for node in SelStr(
'div.mod_episode > span.item > '
'a[_stat="videolist:click"][title]', hutf):
n = node.text.strip()
try:
cnt = int(n)
except ValueError:
continue
ii = node.select('i.mark_v > img[alt]')
if ii and ii[0]['alt'] == U('预告'):
continue
u = (n + node['title'], pre + node['href'])
debug(*u)
urls.append(u)
return urls
def query_info1(self, url):
# https://openload.io/embed/igdtpdeGltM/
# https://openload.co/embed/isCWWnlsZLE/
# https://openload.io/embed/biw7ytfelzU/
# <span id="streamurl">isCWWnlsZLE~1481138074~208.91.0.0~g617lYdo</span>
echo("phantomjs wait 300 ...")
p = Popen(["./phantomjs", "dwm.js", "300", url], stdout=PIPE)
html = p.stdout.read()
hutf = html.decode('utf8')
p.wait()
debug(hutf)
n = SelStr('h6', hutf)
if n:
echo(n[0].text)
return self.title, None, [], None
#vid = match1(url, r'haiuken.com/theatre/([^/]+)/')
m = re.search('''openload.co/embed/([^/]+)/''', url)
if m:
uid = m.groups()[0]
echo(uid)
m = re.search('''<span id="streamurl">([^<>]+)</span>''', hutf)
vid = m.groups()[0]
if not vid.startswith(uid): # TODO, try to decode it
vid = uid + "~1497803146~64.180.0.0~eBodZDZa"
echo(vid)
url = "https://openload.co/stream/%s?mime=true" % vid
# "https://openload.co/embed/kUEfGclsU9o/"
n = SelStr("meta[name=og:title]", hutf)
if n and self.title == UTITLE:
self.title = n[0]['content'] # ="skyrim_no-audio_1080.mp4">"
# https://openload.co/stream/isCWWnlsZLE~1481139117~208.91.0.0~mcLfSy5C?mime=true
# video/mp4 584989307
k, tsize = get_kind_size(url)
k = k.split('/')[-1]
if self.title.endswith('.' + k):
self.title = self.title[:-4]
return self.title, k, [url], tsize
def get_vinfo_do(self, url, vid, fmt):
#getvinfo(target_dir, page_url, video_info['vid'])
rand = random.random()
ckey = echo_ckeyv3(vid, PLAYER_GUID, rand,
player_version=PLAYER_VERSION, platform=PLATFORM)
params = {
'newplatform': PLATFORM,
'guid': PLAYER_GUID,
'pid': PLAYER_PID,
'speed': random.randint(5000, 9000),
'vids': vid,
'fp2p': 1,
'dtype': 3,
'linkver': 2,
'ehost': url,
'fhdswitch': 0,
'cKey': ckey,
'vid': vid,
'appver': PLAYER_VERSION,
'ran': '%.16f' % rand,
'utype': 0,
'encryptVer': '5.4',
'defnpayver': 1,
'charge': 0,
'ip': '',
'otype': 'xml',
'platform': PLATFORM,
}
if fmt:
params['defaultfmt'] = fmt
params['defn'] = fmt
hutf = self.get_html('http://vv.video.qq.com/getvinfo',
postdata=urllib.urlencode(params))
#echo(hutf)
mp = MyHtmlParser(tidy=False)
mp.feed(hutf)
debug(str(mp.root_node))
return mp
def get_playlist(self, url):
sect, tvid, ptid = self.get_stp(url)
if sect not in ("tvdrama", "show", 'liveshow', 'documentary'):
return []
u = 'https://www.kantv6.com/index.php/video/part'
u = '%s?tvid=%s' % (u, tvid)
t = self.get_title(tvid, sect)
dat = self.get_hutf(u)
dat = json.loads(dat)
debug(json.dumps(dat, indent=2))
#bu = 'https://www.kantv6.com/%s/%s-' % (sect, tvid)
debug(t)
#return [(t + '_' + a['part_title'], "https:" + a['url']) for a in dat['data']['partList']]
urls = []
for a in dat['data']['partList']:
if sect in ("show", "liveshow"):
p = t + " " + a['part_title']
elif sect in ("tvdrama", "documentary"):
p = u"%s_第%02d集" % (t, a['part'])
u = "https://www.kantv6.com/%s/%s-%s" % (sect, tvid, a['part_id'])
#echo(p, u)
urls.append((p, u))
return urls
def query_info_dlt3(self, url, title, vid, mp):
#url = 'https://v.qq.com/x/cover/i200hs4ip5a6u7a.html'
#t, vid, mp = self.getvinfo(url) #, fmt='fhd')
mi, mz = None, 0
for fi in mp.select("fl > fi"):
sz = int(fi.select("fs")[0].text)
if sz > mz and fi.select('lmt')[0].text == '0':
mi = fi
mz = sz
debug("cname =", mi.select('cname')[0].text)
vlifs = int(mp.select('vl > vi > fs')[0].text)
fm = mi.select('name')[0].text.strip()
if vlifs < mz:
echo('need query again, current size=%d, best size=%d' % (vlifs,
mz))
echo('new query with fmt=' + fm)
t, vid, mp = self.getvinfo(url, fmt=fm)
for ui in mp.select("vl > vi > ul > ui"):
ur = ui.select('url')[0].text.strip()
if 'default' in ur:
break
#break
keyid = mp.select('vl > vi > keyid')[0].text.strip()
debug("ur =", ur)
pt = ui.select('hls > pt')[0].text.strip()
tp = ui.select('hls > ftype')[0].text.strip()
um = ur + pt + "&type=" + tp + "&fmt=" + fm
debug("um =", um)
hutf = self.get_hutf(um, raw=True)
#echo(hutf)
urls = []
for line in hutf.split('\n'):
if keyid in line:
urls.append(ur + line.strip())
debug('urls =', urls)
return title, tp, urls, mz
def query_info_dlt3(self, url, title, vid, mp):
#url = 'https://v.qq.com/x/cover/i200hs4ip5a6u7a.html'
#t, vid, mp = self.getvinfo(url) #, fmt='fhd')
mi, mz = None, 0
for fi in mp.select("fl > fi"):
sz = int(fi.select("fs")[0].text)
if sz > mz and fi.select('lmt')[0].text == '0':
mi = fi
mz = sz
debug("cname =", mi.select('cname')[0].text)
vlifs = int(mp.select('vl > vi > fs')[0].text)
fm = mi.select('name')[0].text.strip()
if vlifs < mz:
echo('need query again, current size=%d, best size=%d' %
(vlifs, mz))
echo('new query with fmt=' + fm)
t, vid, mp = self.getvinfo(url, fmt=fm)
for ui in mp.select("vl > vi > ul > ui"):
ur = ui.select('url')[0].text.strip()
if 'default' in ur:
break
#break
keyid = mp.select('vl > vi > keyid')[0].text.strip()
debug("ur =", ur)
pt = ui.select('hls > pt')[0].text.strip()
tp = ui.select('hls > ftype')[0].text.strip()
um = ur + pt + "&type=" + tp + "&fmt=" + fm
debug("um =", um)
hutf = self.get_hutf(um, raw=True)
#echo(hutf)
urls = []
for line in hutf.split('\n'):
if keyid in line:
urls.append(ur + line.strip())
debug('urls =', urls)
return title, tp, urls, mz
def title_murl(self, url):
#url = 'https://train.ifvod.tv/detail?id=pGhytibvDFN'
#url = 'https://train.ifvod.tv/play?id=p883vn6dt9F'
ci = get_ci(debug())
req_id, murl = "", ""
qnrr = Queue()
def nrr(ci, msg):
url = msg['params']['response']['url']
if '/api/video/play' in url:
qnrr.put(msg['params']['requestId'])
ci.reg("Network.responseReceived", nrr)
qnrwb = Queue()
def nrwb(ci, msg):
u = msg['params']['request']['url']
debug("Network.requestWillBeSent url", url)
if 'chunklist.m3u8' in u or ".mp4?" in u:
debug("got chunklist.m3u8")
qnrwb.put(u)
ci.reg("Network.requestWillBeSent", nrwb)
try:
ci.Page.navigate(url=url)
req_id = qnrr.get(timeout=ci.get_to())
debug("req_id = ", req_id)
#ret = ci.Network.getResponseBody(requestId=req_id)
ret = ci.wait("Network.getResponseBody", requestId=req_id)
debug("ret_grb =", json.dumps(ret, indent=2))
body = json.loads(ret['result']['body'])
debug("ret_body =", json.dumps(body, indent=2))
zero = body['data']['info'][0]
# find murl, this is not ready one
debug("url = ", zero['flvPathList'][-1]['result'])
title = zero['vl']['title']
debug("title=", title)
murl = qnrwb.get(timeout=ci.get_to())
debug("murl = ", murl)
return title, murl
#except Exception as e:
# echo("key_m3u8 out:", repr(e))
finally:
ci.close()
def query_info(self, url):
#url = 'http://bigdramas.net/video/維京傳奇第三季-第5集/'
#url = 'http://bigdramas.net/video/%E7%B6%AD%E4%BA%AC%E5%82%B3%E5%A5%87%E7%AC%AC%E4%B8%89%E5%AD%A3-%E7%AC%AC5%E9%9B%86/' # handle by google
hutf = self.get_hutf(url)
title = SelStr('div.video-info > h1', hutf)[0].text
#echo(title)
#echo(hutf)
dd = SelStr('div.sources > div.holder > a[data-data]', hutf)[0]['data-data']
dd = b64decode("".join(reversed(dd)))
debug(dd)
#{"source":"Bigdramas","ids":["VUVOM2EwdEdaM05TUldzelRsUmpjRlpVV1dwTlZsVjVUbFZTVWs5cFVrcE9SRVY2VWxaak1FMTViRmxQYWxsclZWUndVVmxIUVV0WlFXODkkJGRyYW1hMQ"]}
#{"source":"Dailymotion","ids":["k336RLStrzbIGzl96CY"]}
dd = json.loads(dd)
if dd['source'] == "Bigdramas":
self.extra_headers['Referer'] = url
ref = 'http://bigdramas.net/view/?ref=' + dd['ids'][0]
debug("ref =", ref)
hutf = self.get_hutf(ref)
#p = Popen(["./phantomjs", "dwm.js", "300", ref, url], stdout=PIPE)
#html = p.stdout.read()
#hutf = html.decode('utf8')
#p.wait()
sss = SelStr('div.video-wrapper > video > source', hutf)
if not sss:
echo("phantomjs wait 20 ...")
p = Popen(["./phantomjs", "dwm.js", "-20", ref, url], stdout=PIPE)
html = p.stdout.read()
hutf = html.decode('utf8')
p.wait()
sss = SelStr('div.video-wrapper > video > source', hutf)
if not sss:
debug('hutf size = ', len(hutf))
debug(hutf)
#sys.exit(1)
ss = sss[0]
urls = [ss['src']]
ext = ss['type'][-3:]
debug(urls)
return title, ext, urls, None
elif dd['source'] == "Dailymotion":
eurl = "http://www.dailymotion.com/embed/video/" + dd['ids'][0]
from dailymotion import DM
#return DM().query_info(eurl)
ret = DM().query_info(eurl)
return title, ret[1], ret[2], ret[3]
elif dd['source'] == "Youtube":
echo("dd =", dd)
echo("url: https://youtu.be/" + dd["ids"][0])
#sys.exit(1)
return None
elif dd['source'] == "Video66":
echo(dd)
from video66 import VIDEO66
v66 = VIDEO66()
urls = []
for uid in dd['ids']:
u = "http://video66.org/embed.php?w=798&h=449&vid=" + uid
echo(u)
t, e, us, s = v66.query_info(u)
urls.append(us[0])
return title, e, urls, None
else:
echo("found new source")
echo(dd)
return None
def query_info(self, url):
#url = 'http://bigdramas.net/video/維京傳奇第三季-第5集/'
#url = 'http://bigdramas.net/video/%E7%B6%AD%E4%BA%AC%E5%82%B3%E5%A5%87%E7%AC%AC%E4%B8%89%E5%AD%A3-%E7%AC%AC5%E9%9B%86/' # handle by google
hutf = self.get_hutf(url)
title = SelStr('div.video-info > h1', hutf)[0].text
#echo(title)
#echo(hutf)
dd = SelStr('div.sources > div.holder > a[data-data]',
hutf)[0]['data-data']
dd = b64decode("".join(reversed(dd)))
debug(dd)
#{"source":"Bigdramas","ids":["VUVOM2EwdEdaM05TUldzelRsUmpjRlpVV1dwTlZsVjVUbFZTVWs5cFVrcE9SRVY2VWxaak1FMTViRmxQYWxsclZWUndVVmxIUVV0WlFXODkkJGRyYW1hMQ"]}
#{"source":"Dailymotion","ids":["k336RLStrzbIGzl96CY"]}
dd = json.loads(dd)
if dd['source'] == "Bigdramas":
self.extra_headers['Referer'] = url
ref = 'http://bigdramas.net/view/?ref=' + dd['ids'][0]
debug("ref =", ref)
hutf = self.get_hutf(ref)
#p = Popen(["./phantomjs", "dwm.js", "300", ref, url], stdout=PIPE)
#html = p.stdout.read()
#hutf = html.decode('utf8')
#p.wait()
sss = SelStr('div.video-wrapper > video > source', hutf)
if not sss:
echo("phantomjs wait 20 ...")
p = Popen(["./phantomjs", "dwm.js", "-20", ref, url],
stdout=PIPE)
html = p.stdout.read()
hutf = html.decode('utf8')
p.wait()
sss = SelStr('div.video-wrapper > video > source', hutf)
if not sss:
debug('hutf size = ', len(hutf))
debug(hutf)
#sys.exit(1)
ss = sss[0]
urls = [ss['src']]
ext = ss['type'][-3:]
debug(urls)
return title, ext, urls, None
elif dd['source'] == "Dailymotion":
eurl = "http://www.dailymotion.com/embed/video/" + dd['ids'][0]
from dailymotion import DM
#return DM().query_info(eurl)
ret = DM().query_info(eurl)
return title, ret[1], ret[2], ret[3]
elif dd['source'] == "Youtube":
echo("dd =", dd)
echo("url: https://youtu.be/" + dd["ids"][0])
#sys.exit(1)
return None
elif dd['source'] == "Video66":
echo(dd)
from video66 import VIDEO66
v66 = VIDEO66()
urls = []
for uid in dd['ids']:
u = "http://video66.org/embed.php?w=798&h=449&vid=" + uid
echo(u)
t, e, us, s = v66.query_info(u)
urls.append(us[0])
return title, e, urls, None
else:
echo("found new source")
echo(dd)
return None
def query_info(self, url):
#url = 'https://v.qq.com/x/cover/ijilh0frmu96sbf/x0017evzp6n.html'
if 'iframe/player' in url:
title, vid, mp = self.getvinfo_iframe_player(url)
else:
title, vid, mp = self.getvinfo(url)
dlt = mp.select('dltype')[0].text.strip()
if dlt == '3':
return self.query_info_dlt3(url, title, vid, mp)
slid = None
mfs = 0
# <fi><sl>0</sl><br>1500</br><id>11401</id><name>shd</name><lmt>0</lmt><sb>1</sb><cname>超清;(720P)</cname><fs>304995197</fs></fi>
for ff in mp.select("fl > fi"):
#print ff
fs = int(ff.select("fs")[0].text)
if fs > mfs:
name = ff.select("name")[0].text
fiid = ff.select("id")[0].text
cname = ff.select("cname")[0].text
mfs = fs
slid = fiid
echo(cname)
for vi in mp.select('vl>vi'):
video_type = int(vi.select('videotype')[0].text)
if video_type == 1:
video_type = 'flv'
elif video_type == 2:
video_type = 'mp4'
else:
video_type = 'unknown'
video_id = vi.select('vid')[0].text
cdn_host = vi.select('ul>ui>url')[0].text
# this will much speed up
cdn_host = "http://videohy.tc.qq.com/vlive.qqvideo.tc.qq.com"
vt = vi.select('ul>ui>vt')[0].text
fn = vi.select('fn')[0].text
fs = int(vi.select('fs')[0].text)
# failed https://v.qq.com/x/cover/i200hs4ip5a6u7a.html
clfc = vi.select('cl>fc')
fc = 0
if clfc:
fc = int(clfc[0].text)
echo('video_id=', video_id)
echo('cdn_host=', cdn_host)
echo('fc=', fc)
urls = []
if fc == 0:
vkey = self.getvkey(url, vid, vt, slid, fn)
debug(vkey)
filename = vkey.get('filename')
target_file = os.path.join(target_dir, filename)
cdn_url = '%s/%s' % (cdn_host, filename)
key = vkey.get('key')
br = vkey.get('br')
urls.append(self.make_url(cdn_url, key, br, video_type, fs))
else:
for ci in vi.select('cl>ci'):
idx = int(ci.select('idx')[0].text)
cd = float(ci.select('cd')[0].text)
md5 = ci.select('cmd5')[0].text
vclip = self.getvclip(url, vid, vt, slid, idx)
filename = vclip['filename']
key = vclip['key']
cdn_url = '%s/%s' % (cdn_host, filename)
echo('cdn_url=', cdn_url)
urls.append(self.make_url(cdn_url, key, vclip['br'],
vclip['fmt'], vclip['fs']))
echo("urls =", urls)
#k, tsize = self.get_total_size(urls)
return title, video_type, urls, None
def nrwb(ci, msg):
u = msg['params']['request']['url']
debug("Network.requestWillBeSent url", url)
if 'chunklist.m3u8' in u or ".mp4?" in u:
debug("got chunklist.m3u8")
qnrwb.put(u)
def query_info(self, url):
murl = self.find_murl(url)
murl = self.last_m3u8(murl)
debug(murl)
title = self.get_title(url)
return title, "m3u8", murl, None
def query_info(self, url):
#url = 'https://v.qq.com/x/cover/ijilh0frmu96sbf/x0017evzp6n.html'
if 'iframe/player' in url:
title, vid, mp = self.getvinfo_iframe_player(url)
else:
title, vid, mp = self.getvinfo(url)
dlt = mp.select('dltype')[0].text.strip()
if dlt == '3':
return self.query_info_dlt3(url, title, vid, mp)
slid = None
mfs = 0
# <fi><sl>0</sl><br>1500</br><id>11401</id><name>shd</name><lmt>0</lmt><sb>1</sb><cname>超清;(720P)</cname><fs>304995197</fs></fi>
for ff in mp.select("fl > fi"):
#print ff
fs = int(ff.select("fs")[0].text)
if fs > mfs:
name = ff.select("name")[0].text
fiid = ff.select("id")[0].text
cname = ff.select("cname")[0].text
mfs = fs
slid = fiid
echo(cname)
for vi in mp.select('vl>vi'):
video_type = int(vi.select('videotype')[0].text)
if video_type == 1:
video_type = 'flv'
elif video_type == 2:
video_type = 'mp4'
else:
video_type = 'unknown'
video_id = vi.select('vid')[0].text
cdn_host = vi.select('ul>ui>url')[0].text
# this will much speed up
cdn_host = "http://videohy.tc.qq.com/vlive.qqvideo.tc.qq.com"
vt = vi.select('ul>ui>vt')[0].text
fn = vi.select('fn')[0].text
fs = int(vi.select('fs')[0].text)
# failed https://v.qq.com/x/cover/i200hs4ip5a6u7a.html
clfc = vi.select('cl>fc')
fc = 0
if clfc:
fc = int(clfc[0].text)
echo('video_id=', video_id)
echo('cdn_host=', cdn_host)
echo('fc=', fc)
urls = []
if fc == 0:
vkey = self.getvkey(url, vid, vt, slid, fn)
debug(vkey)
filename = vkey.get('filename')
target_file = os.path.join(target_dir, filename)
cdn_url = '%s/%s' % (cdn_host, filename)
key = vkey.get('key')
br = vkey.get('br')
urls.append(self.make_url(cdn_url, key, br, video_type, fs))
else:
for ci in vi.select('cl>ci'):
idx = int(ci.select('idx')[0].text)
cd = float(ci.select('cd')[0].text)
md5 = ci.select('cmd5')[0].text
vclip = self.getvclip(url, vid, vt, slid, idx)
filename = vclip['filename']
key = vclip['key']
cdn_url = '%s/%s' % (cdn_host, filename)
echo('cdn_url=', cdn_url)
urls.append(
self.make_url(cdn_url, key, vclip['br'], vclip['fmt'],
vclip['fs']))
echo("urls =", urls)
#k, tsize = self.get_total_size(urls)
return title, video_type, urls, None
def nrwb(ci, msg):
u = msg['params']['request']['url']
debug("Network.requestWillBeSent url", url)
if 'master.m3u8' in u:
debug("got master.m3u8", u)
qnrwb.put(u)
