def _get_video_query_url(resourceID): # has to be like this headers = FAKE_HEADERS.copy() headers.update({ 'DNT': '1', 'Referer': 'http://v.ucas.ac.cn/', 'Connection': 'keep-alive', }) conn = http.client.HTTPConnection('210.76.211.10') conn.request( 'GET', '/vplus/remote.do?method=query2&loginname=videocas&pwd=af1c7a4c5f77' 'f790722f7cae474c37e281203765d423a23b&resource=%5B%7B%22resourceID%2' '2%3A%22{}%22%2C%22on%22%3A1%2C%22time%22%3A600%2C%22eid%22%3A100%2C' '%22w%22%3A800%2C%22h%22%3A600%7D%5D&timeStamp={}'.format( resourceID, str(int(time())) ), headers=headers ) res = conn.getresponse() data = res.read() info = data.decode("utf-8") return match1(info, r'video":"(.+)"')
def baidu_pan_download(url): errno_patt = r'errno":([^"]+),' refer_url = '' fake_headers = FAKE_HEADERS.copy() fake_headers.update({ 'Host': 'pan.baidu.com', 'Origin': 'http://pan.baidu.com', 'Referer': refer_url, }) if cookies: print('Use user specified cookies') else: print('Generating cookies...') fake_headers['Cookie'] = baidu_pan_gen_cookies(url) refer_url = 'http://pan.baidu.com' html = get_content(url, fake_headers, decoded=True) isprotected = False sign, timestamp, bdstoken, appid, primary_id, fs_id, uk = baidu_pan_parse( html) if sign is None: if re.findall(r'verify-property', html): isprotected = True sign, timestamp, bdstoken, appid, primary_id, fs_id, uk, \ fake_headers, psk = baidu_pan_protected_share(url) if not isprotected: raise AssertionError('Share not found or canceled: {}'.format(url)) if bdstoken is None: bdstoken = '' if not isprotected: sign, timestamp, bdstoken, appid, primary_id, fs_id, \ uk = baidu_pan_parse(html) request_url = ( 'http://pan.baidu.com/api/sharedownload?sign={}×tamp={}&' 'bdstoken={}&channel=chunlei&clienttype=0&web=1&app_id={}'.format( sign, timestamp, bdstoken, appid)) refer_url = url post_data = { 'encrypt': 0, 'product': 'share', 'uk': uk, 'primaryid': primary_id, 'fid_list': '[{}]'.format(fs_id) } if isprotected: post_data['sekey'] = psk response_content = post_content(request_url, fake_headers, post_data, True) errno = match1(response_content, errno_patt) if errno != '0': raise AssertionError( 'Server refused to provide download link! (Errno:{})'.format( errno)) real_url = match1(response_content, r'dlink":"([^"]+)"').replace('\\/', '/') title = match1(response_content, r'server_filename":"([^"]+)"') assert real_url _type, ext, size = url_info(real_url) title_wrapped = json.loads( '{{"wrapper":"{}"}}'.format(title)) # \u4ecb\u7ecd -> 介绍 title = title_wrapped['wrapper'] return real_url, title, ext, size
def fantasy_download_by_id_channelId( id=0, channelId=0, output_dir='.', merge=True, info_only=False, **kwargs ): api_url = ( 'http://www.fantasy.tv/tv/playDetails.action?' 'myChannelId=1&id={id}&channelId={channelId}&t={t}'.format( id=id, channelId=channelId, t=str(random.random()) ) ) html = get_content(api_url) html = json.loads(html) if int(html['status']) != 100000: raise Exception('API error!') title = html['data']['tv']['title'] video_url = html['data']['tv']['videoPath'] headers = FAKE_HEADERS.copy() headers['Referer'] = api_url type, ext, size = url_info(video_url, headers=headers) print_info(site_info, title, type, size) if not info_only: download_urls( [video_url], title, ext, size, output_dir, merge=merge, headers=headers )
def fc2video_download_by_upid(upid, output_dir='.', merge=True, info_only=False, **kwargs): fake_headers = FAKE_HEADERS.copy() fake_headers.update({ 'DNT': '1', 'Accept-Encoding': 'gzip, deflate, sdch', 'Accept-Language': 'en-CA,en;q=0.8,en-US;q=0.6,zh-CN;q=0.4,zh;q=0.2', 'X-Requested-With': 'ShockwaveFlash/19.0.0.245', 'Connection': 'keep-alive', }) api_base = ('https://video.fc2.com/ginfo.php?upid={upid}&mimi=' '{mimi}'.format(upid=upid, mimi=makeMimi(upid))) html = get_content(api_base, headers=fake_headers) video_url = match1(html, r'filepath=(.+)&sec') video_url = video_url.replace('&mid', '?mid') title = match1(html, r'&title=([^&]+)') _type, ext, size = url_info(video_url, headers=fake_headers) print_info(site_info, title, _type, size) if not info_only: download_urls([video_url], title, ext, size, output_dir, merge=merge, headers=fake_headers, **kwargs)
def get_streams_by_id(account_number, video_id): """ int, int->list Get the height of the videos. Since brightcove is using 3 kinds of links: rtmp, http and https, we will be using the HTTPS one to make it secure. If somehow akamaihd.net is blocked by the Great F*****g Wall, change the "startswith https" to http. """ endpoint = ('https://edge.api.brightcove.com/playback/v1/accounts/' '{account_number}/videos/{video_id}'.format( account_number=account_number, video_id=video_id)) fake_header_id = FAKE_HEADERS.copy() # is this somehow related to the time? Magic.... fake_header_id['Accept'] = ( 'application/json;pk=BCpkADawqM1cc6wmJQC2tvoXZt4mrB7bFfi6zGt9QnOzp' 'rPZcGLE9OMGJwspQwKfuFYuCjAAJ53JdjI8zGFx1ll4rxhYJ255AXH1BQ10rnm34w' 'eknpfG-sippyQ') html = get_content(endpoint, headers=fake_header_id) html_json = json.loads(html) link_list = [] for i in html_json['sources']: if 'src' in i and i.get('container') == 'MP4': # to avoid KeyError if i['src'].startswith('https'): link_list.append((str(i['height']), i['src'])) return link_list
def prepare(self, **kwargs): headers = FAKE_HEADERS.copy() if 'referer' in kwargs: headers['Referer'] = kwargs['referer'] try: page = get_content('https://vimeo.com/{}'.format(self.vid)) cfg_patt = r'clip_page_config\s*=\s*(\{.+?\});' cfg = json.loads(match1(page, cfg_patt)) video_page = get_content( cfg['player']['config_url'], headers=headers ) self.title = cfg['clip']['title'] info = json.loads(video_page) except Exception as e: page = get_content('https://player.vimeo.com/video/{}'.format( self.vid )) self.title = match1(page, r'<title>([^<]+)</title>') info = json.loads(match1(page, r'var t=(\{.+?\});')) plain = info['request']['files']['progressive'] for s in plain: meta = dict(src=[s['url']], container='mp4') meta['video_profile'] = '{}x{}'.format(s['width'], s['height']) for stream in self.__class__.stream_types: if s['quality'] == stream['id']: self.streams[s['quality']] = meta self.master_m3u8 = info['request']['files']['hls']['cdns']
def baidu_pan_protected_share(url): print('This share is protected by password!') inpwd = input('Please provide unlock password: '******' ', '').replace('\t', '') print('Please wait...') post_pwd = {'pwd': inpwd, 'vcode': None, 'vstr': None} from http import cookiejar import time cookiejar = cookiejar.CookieJar() opener = request.build_opener(request.HTTPCookieProcessor(cookiejar)) resp = opener.open('http://pan.baidu.com') resp = opener.open(url) init_url = resp.geturl() verify_url = ('http://pan.baidu.com/share/verify?{}&t={}&channel=chunlei&' 'clienttype=0&web=1'.format( init_url.split('?', 1)[1], int(time.time()))) refer_url = init_url fake_headers = FAKE_HEADERS.copy() fake_headers.update({ 'Host': 'pan.baidu.com', 'Origin': 'http://pan.baidu.com', 'Referer': refer_url, }) opener.addheaders = dict2triplet(fake_headers) pwd_resp = opener.open(verify_url, bytes(parse.urlencode(post_pwd), 'utf-8')) pwd_resp_str = ungzip(pwd_resp.read()).decode('utf-8') pwd_res = json.loads(pwd_resp_str) if pwd_res['errno'] != 0: raise AssertionError( 'Server returned an error: {} (Incorrect password?)'.format( pwd_res['errno'])) pg_resp = opener.open('http://pan.baidu.com/share/link?{}'.format( init_url.split('?', 1)[1])) content = ungzip(pg_resp.read()).decode('utf-8') sign, timestamp, bdstoken, appid, primary_id, fs_id, uk = baidu_pan_parse( content) psk = query_cookiejar(cookiejar, 'BDCLND') psk = parse.unquote(psk) fake_headers['Cookie'] = cookjar2hdr(cookiejar) return (sign, timestamp, bdstoken, appid, primary_id, fs_id, uk, fake_headers, psk)
def pixivision_download(url, output_dir='.', info_only=False, **kwargs): html = get_content(url) parser = get_parser(html) title = parser.h1.text.strip() output_dir = Path(output_dir) / title imgs = parser.find_all('img', class_='am__work__illust') print_info(site_info, title, 'jpg', 0) if not info_only: headers = FAKE_HEADERS.copy() headers.update({'Referer': url}) for img in imgs: img = img['src'] size = url_size(img, headers=headers) filename, ext = img.split('/')[-1].split('.') download_urls([img], filename, ext, size, output_dir, refer=url, **kwargs)
def url_save( url, filepath, bar, refer=None, is_part=False, headers=None, timeout=None, **kwargs ): tmp_headers = headers.copy() if headers else FAKE_HEADERS.copy() # When a referer specified with param refer, # the key must be 'Referer' for the hack here if refer: tmp_headers['Referer'] = refer file_size = url_size(url, headers=tmp_headers) if os.path.exists(filepath): if not force and file_size == os.path.getsize(filepath): if not is_part: if bar: bar.done() print( 'Skipping {}: file already exists'.format( tr(os.path.basename(filepath)) ) ) else: if bar: bar.update_received(file_size) return else: if not is_part: if bar: bar.done() print('Overwriting %s' % tr(os.path.basename(filepath)), '...') elif not os.path.exists(os.path.dirname(filepath)): os.mkdir(os.path.dirname(filepath)) temp_filepath = filepath + '.download' if file_size != float('inf') \ else filepath received = 0 if not force: open_mode = 'ab' if os.path.exists(temp_filepath): received += os.path.getsize(temp_filepath) if bar: bar.update_received(os.path.getsize(temp_filepath)) else: open_mode = 'wb' if received < file_size: if received: tmp_headers['Range'] = 'bytes=' + str(received) + '-' if refer: tmp_headers['Referer'] = refer kwargs = { 'headers': tmp_headers, } if timeout: kwargs['timeout'] = timeout response = urlopen_with_retry(url, **kwargs) try: range_start = int( response.headers[ 'content-range' ][6:].split('/')[0].split('-')[0] ) end_length = int( response.headers['content-range'][6:].split('/')[1] ) range_length = end_length - range_start except Exception: content_length = response.headers['content-length'] range_length = int(content_length) if content_length \ else float('inf') if file_size != received + range_length: received = 0 if bar: bar.received = 0 open_mode = 'wb' with open(temp_filepath, open_mode) as output: for chunk in response.iter_content(chunk_size=2048): if chunk: output.write(chunk) received += len(chunk) if bar: bar.update_received(len(chunk)) assert received == os.path.getsize(temp_filepath), '{} == {} == {}'.format( received, os.path.getsize(temp_filepath), temp_filepath ) if os.access(filepath, os.W_OK): # on Windows rename could fail if destination filepath exists os.remove(filepath) os.rename(temp_filepath, filepath)
request, ) from lulu.common import ( match1, url_info, print_info, get_content, download_urls, playlist_not_supported, ) from lulu.config import FAKE_HEADERS __all__ = ['nicovideo_download'] site_info = 'niconico nicovideo.jp' fake_headers = FAKE_HEADERS.copy() def nicovideo_login(user, password): data = 'current_form=login&mail={}&password={}&login_submit=Log+In'.format( user, password) response = request.urlopen( request.Request( 'https://secure.nicovideo.jp/secure/login?site=niconico', headers=fake_headers, data=data.encode('utf-8'))) return response.headers def nicovideo_download(url, info_only=False, **kwargs): import ssl