def _get_video_query_url(resourceID):
# has to be like this
headers = FAKE_HEADERS.copy()
headers.update({
'DNT': '1',
'Referer': 'http://v.ucas.ac.cn/',
'Connection': 'keep-alive',
})
conn = http.client.HTTPConnection('210.76.211.10')
conn.request(
'GET',
'/vplus/remote.do?method=query2&loginname=videocas&pwd=af1c7a4c5f77'
'f790722f7cae474c37e281203765d423a23b&resource=%5B%7B%22resourceID%2'
'2%3A%22{}%22%2C%22on%22%3A1%2C%22time%22%3A600%2C%22eid%22%3A100%2C'
'%22w%22%3A800%2C%22h%22%3A600%7D%5D&timeStamp={}'.format(
resourceID, str(int(time()))
),
headers=headers
)
res = conn.getresponse()
data = res.read()
info = data.decode("utf-8")
return match1(info, r'video":"(.+)"')
def baidu_pan_download(url):
errno_patt = r'errno":([^"]+),'
refer_url = ''
fake_headers = FAKE_HEADERS.copy()
fake_headers.update({
'Host': 'pan.baidu.com',
'Origin': 'http://pan.baidu.com',
'Referer': refer_url,
})
if cookies:
print('Use user specified cookies')
else:
print('Generating cookies...')
fake_headers['Cookie'] = baidu_pan_gen_cookies(url)
refer_url = 'http://pan.baidu.com'
html = get_content(url, fake_headers, decoded=True)
isprotected = False
sign, timestamp, bdstoken, appid, primary_id, fs_id, uk = baidu_pan_parse(
html)
if sign is None:
if re.findall(r'verify-property', html):
isprotected = True
sign, timestamp, bdstoken, appid, primary_id, fs_id, uk, \
fake_headers, psk = baidu_pan_protected_share(url)
if not isprotected:
raise AssertionError('Share not found or canceled: {}'.format(url))
if bdstoken is None:
bdstoken = ''
if not isprotected:
sign, timestamp, bdstoken, appid, primary_id, fs_id, \
uk = baidu_pan_parse(html)
request_url = (
'http://pan.baidu.com/api/sharedownload?sign={}×tamp={}&'
'bdstoken={}&channel=chunlei&clienttype=0&web=1&app_id={}'.format(
sign, timestamp, bdstoken, appid))
refer_url = url
post_data = {
'encrypt': 0,
'product': 'share',
'uk': uk,
'primaryid': primary_id,
'fid_list': '[{}]'.format(fs_id)
}
if isprotected:
post_data['sekey'] = psk
response_content = post_content(request_url, fake_headers, post_data, True)
errno = match1(response_content, errno_patt)
if errno != '0':
raise AssertionError(
'Server refused to provide download link! (Errno:{})'.format(
errno))
real_url = match1(response_content,
r'dlink":"([^"]+)"').replace('\\/', '/')
title = match1(response_content, r'server_filename":"([^"]+)"')
assert real_url
_type, ext, size = url_info(real_url)
title_wrapped = json.loads(
'{{"wrapper":"{}"}}'.format(title)) # \u4ecb\u7ecd -> 介绍
title = title_wrapped['wrapper']
return real_url, title, ext, size
def fantasy_download_by_id_channelId(
id=0, channelId=0, output_dir='.', merge=True, info_only=False, **kwargs
):
api_url = (
'http://www.fantasy.tv/tv/playDetails.action?'
'myChannelId=1&id={id}&channelId={channelId}&t={t}'.format(
id=id,
channelId=channelId,
t=str(random.random())
)
)
html = get_content(api_url)
html = json.loads(html)
if int(html['status']) != 100000:
raise Exception('API error!')
title = html['data']['tv']['title']
video_url = html['data']['tv']['videoPath']
headers = FAKE_HEADERS.copy()
headers['Referer'] = api_url
type, ext, size = url_info(video_url, headers=headers)
print_info(site_info, title, type, size)
if not info_only:
download_urls(
[video_url], title, ext, size, output_dir, merge=merge,
headers=headers
)
def fc2video_download_by_upid(upid,
output_dir='.',
merge=True,
info_only=False,
**kwargs):
fake_headers = FAKE_HEADERS.copy()
fake_headers.update({
'DNT': '1',
'Accept-Encoding': 'gzip, deflate, sdch',
'Accept-Language': 'en-CA,en;q=0.8,en-US;q=0.6,zh-CN;q=0.4,zh;q=0.2',
'X-Requested-With': 'ShockwaveFlash/19.0.0.245',
'Connection': 'keep-alive',
})
api_base = ('https://video.fc2.com/ginfo.php?upid={upid}&mimi='
'{mimi}'.format(upid=upid, mimi=makeMimi(upid)))
html = get_content(api_base, headers=fake_headers)
video_url = match1(html, r'filepath=(.+)&sec')
video_url = video_url.replace('&mid', '?mid')
title = match1(html, r'&title=([^&]+)')
_type, ext, size = url_info(video_url, headers=fake_headers)
print_info(site_info, title, _type, size)
if not info_only:
download_urls([video_url],
title,
ext,
size,
output_dir,
merge=merge,
headers=fake_headers,
**kwargs)
def get_streams_by_id(account_number, video_id):
"""
int, int->list
Get the height of the videos.
Since brightcove is using 3 kinds of links: rtmp, http and https,
we will be using the HTTPS one to make it secure.
If somehow akamaihd.net is blocked by the Great F*****g Wall,
change the "startswith https" to http.
"""
endpoint = ('https://edge.api.brightcove.com/playback/v1/accounts/'
'{account_number}/videos/{video_id}'.format(
account_number=account_number, video_id=video_id))
fake_header_id = FAKE_HEADERS.copy()
# is this somehow related to the time? Magic....
fake_header_id['Accept'] = (
'application/json;pk=BCpkADawqM1cc6wmJQC2tvoXZt4mrB7bFfi6zGt9QnOzp'
'rPZcGLE9OMGJwspQwKfuFYuCjAAJ53JdjI8zGFx1ll4rxhYJ255AXH1BQ10rnm34w'
'eknpfG-sippyQ')
html = get_content(endpoint, headers=fake_header_id)
html_json = json.loads(html)
link_list = []
for i in html_json['sources']:
if 'src' in i and i.get('container') == 'MP4': # to avoid KeyError
if i['src'].startswith('https'):
link_list.append((str(i['height']), i['src']))
return link_list
def prepare(self, **kwargs):
headers = FAKE_HEADERS.copy()
if 'referer' in kwargs:
headers['Referer'] = kwargs['referer']
try:
page = get_content('https://vimeo.com/{}'.format(self.vid))
cfg_patt = r'clip_page_config\s*=\s*(\{.+?\});'
cfg = json.loads(match1(page, cfg_patt))
video_page = get_content(
cfg['player']['config_url'], headers=headers
)
self.title = cfg['clip']['title']
info = json.loads(video_page)
except Exception as e:
page = get_content('https://player.vimeo.com/video/{}'.format(
self.vid
))
self.title = match1(page, r'<title>([^<]+)</title>')
info = json.loads(match1(page, r'var t=(\{.+?\});'))
plain = info['request']['files']['progressive']
for s in plain:
meta = dict(src=[s['url']], container='mp4')
meta['video_profile'] = '{}x{}'.format(s['width'], s['height'])
for stream in self.__class__.stream_types:
if s['quality'] == stream['id']:
self.streams[s['quality']] = meta
self.master_m3u8 = info['request']['files']['hls']['cdns']
def baidu_pan_protected_share(url):
print('This share is protected by password!')
inpwd = input('Please provide unlock password: '******' ', '').replace('\t', '')
print('Please wait...')
post_pwd = {'pwd': inpwd, 'vcode': None, 'vstr': None}
from http import cookiejar
import time
cookiejar = cookiejar.CookieJar()
opener = request.build_opener(request.HTTPCookieProcessor(cookiejar))
resp = opener.open('http://pan.baidu.com')
resp = opener.open(url)
init_url = resp.geturl()
verify_url = ('http://pan.baidu.com/share/verify?{}&t={}&channel=chunlei&'
'clienttype=0&web=1'.format(
init_url.split('?', 1)[1], int(time.time())))
refer_url = init_url
fake_headers = FAKE_HEADERS.copy()
fake_headers.update({
'Host': 'pan.baidu.com',
'Origin': 'http://pan.baidu.com',
'Referer': refer_url,
})
opener.addheaders = dict2triplet(fake_headers)
pwd_resp = opener.open(verify_url, bytes(parse.urlencode(post_pwd),
'utf-8'))
pwd_resp_str = ungzip(pwd_resp.read()).decode('utf-8')
pwd_res = json.loads(pwd_resp_str)
if pwd_res['errno'] != 0:
raise AssertionError(
'Server returned an error: {} (Incorrect password?)'.format(
pwd_res['errno']))
pg_resp = opener.open('http://pan.baidu.com/share/link?{}'.format(
init_url.split('?', 1)[1]))
content = ungzip(pg_resp.read()).decode('utf-8')
sign, timestamp, bdstoken, appid, primary_id, fs_id, uk = baidu_pan_parse(
content)
psk = query_cookiejar(cookiejar, 'BDCLND')
psk = parse.unquote(psk)
fake_headers['Cookie'] = cookjar2hdr(cookiejar)
return (sign, timestamp, bdstoken, appid, primary_id, fs_id, uk,
fake_headers, psk)
def pixivision_download(url, output_dir='.', info_only=False, **kwargs):
html = get_content(url)
parser = get_parser(html)
title = parser.h1.text.strip()
output_dir = Path(output_dir) / title
imgs = parser.find_all('img', class_='am__work__illust')
print_info(site_info, title, 'jpg', 0)
if not info_only:
headers = FAKE_HEADERS.copy()
headers.update({'Referer': url})
for img in imgs:
img = img['src']
size = url_size(img, headers=headers)
filename, ext = img.split('/')[-1].split('.')
download_urls([img],
filename,
ext,
size,
output_dir,
refer=url,
**kwargs)
def url_save(
url, filepath, bar, refer=None, is_part=False, headers=None, timeout=None,
**kwargs
):
tmp_headers = headers.copy() if headers else FAKE_HEADERS.copy()
# When a referer specified with param refer,
# the key must be 'Referer' for the hack here
if refer:
tmp_headers['Referer'] = refer
file_size = url_size(url, headers=tmp_headers)
if os.path.exists(filepath):
if not force and file_size == os.path.getsize(filepath):
if not is_part:
if bar:
bar.done()
print(
'Skipping {}: file already exists'.format(
tr(os.path.basename(filepath))
)
)
else:
if bar:
bar.update_received(file_size)
return
else:
if not is_part:
if bar:
bar.done()
print('Overwriting %s' % tr(os.path.basename(filepath)), '...')
elif not os.path.exists(os.path.dirname(filepath)):
os.mkdir(os.path.dirname(filepath))
temp_filepath = filepath + '.download' if file_size != float('inf') \
else filepath
received = 0
if not force:
open_mode = 'ab'
if os.path.exists(temp_filepath):
received += os.path.getsize(temp_filepath)
if bar:
bar.update_received(os.path.getsize(temp_filepath))
else:
open_mode = 'wb'
if received < file_size:
if received:
tmp_headers['Range'] = 'bytes=' + str(received) + '-'
if refer:
tmp_headers['Referer'] = refer
kwargs = {
'headers': tmp_headers,
}
if timeout:
kwargs['timeout'] = timeout
response = urlopen_with_retry(url, **kwargs)
try:
range_start = int(
response.headers[
'content-range'
][6:].split('/')[0].split('-')[0]
)
end_length = int(
response.headers['content-range'][6:].split('/')[1]
)
range_length = end_length - range_start
except Exception:
content_length = response.headers['content-length']
range_length = int(content_length) if content_length \
else float('inf')
if file_size != received + range_length:
received = 0
if bar:
bar.received = 0
open_mode = 'wb'
with open(temp_filepath, open_mode) as output:
for chunk in response.iter_content(chunk_size=2048):
if chunk:
output.write(chunk)
received += len(chunk)
if bar:
bar.update_received(len(chunk))
assert received == os.path.getsize(temp_filepath), '{} == {} == {}'.format(
received, os.path.getsize(temp_filepath), temp_filepath
)
if os.access(filepath, os.W_OK):
# on Windows rename could fail if destination filepath exists
os.remove(filepath)
os.rename(temp_filepath, filepath)
request,
)
from lulu.common import (
match1,
url_info,
print_info,
get_content,
download_urls,
playlist_not_supported,
)
from lulu.config import FAKE_HEADERS
__all__ = ['nicovideo_download']
site_info = 'niconico nicovideo.jp'
fake_headers = FAKE_HEADERS.copy()
def nicovideo_login(user, password):
data = 'current_form=login&mail={}&password={}&login_submit=Log+In'.format(
user, password)
response = request.urlopen(
request.Request(
'https://secure.nicovideo.jp/secure/login?site=niconico',
headers=fake_headers,
data=data.encode('utf-8')))
return response.headers
def nicovideo_download(url, info_only=False, **kwargs):
import ssl