def vidto_download(url, output_dir='.', merge=True, info_only=False, **kwargs): html = get_content(url) params = {} r = re.findall( r'type="(?:hidden|submit)?"(?:.*?)name="(.+?)"\s* value="?(.+?)">', html) for name, value in r: params[name] = value data = parse.urlencode(params).encode('utf-8') req = request.Request(url) print("Please wait for 6 seconds...") time.sleep(6) print("Starting") new_html = request.urlopen(req, data).read().decode('utf-8', 'replace') new_stff = re.search('lnk_download" href="(.*?)">', new_html) if(new_stff): url = new_stff.group(1) title = params['fname'] type = "" ext = "" a, b, size = url_info(url) print_info(site_info, title, type, size) if not info_only: download_urls([url], title, ext, size, output_dir, merge=merge) else: write2buf("cannot find link, please review") pdb.set_trace()
def video_info(vid,**kwargs): url = 'http://api.letv.com/mms/out/video/playJson?id={}&platid=1&splatid=101&format=1&tkey={}&domain=www.letv.com'.format(vid,calcTimeKey(int(time.time()))) r = get_content(url, decoded=False) info=json.loads(str(r,"utf-8")) stream_id = None support_stream_id = info["playurl"]["dispatch"].keys() if "stream_id" in kwargs and kwargs["stream_id"].lower() in support_stream_id: stream_id = kwargs["stream_id"] else: write2buf("Current Video Supports:") for i in support_stream_id: write2buf("\t--format",i,"<URL>") if "1080p" in support_stream_id: stream_id = '1080p' elif "720p" in support_stream_id: stream_id = '720p' else: stream_id =sorted(support_stream_id,key= lambda i: int(i[1:]))[-1] url =info["playurl"]["domain"][0]+info["playurl"]["dispatch"][stream_id][0] ext = info["playurl"]["dispatch"][stream_id][1].split('.')[-1] url+="&ctv=pc&m3v=1&termid=1&format=1&hwtype=un&ostype=Linux&tag=letv&sign=letv&expect=3&tn={}&pay=0&iscpn=f9051&rateid={}".format(random.random(),stream_id) r2=get_content(url,decoded=False) info2=json.loads(str(r2,"utf-8")) # hold on ! more things to do # to decode m3u8 (encoded) m3u8 = get_content(info2["location"],decoded=False) m3u8_list = decode(m3u8) urls = re.findall(r'^[^#][^\r]*',m3u8_list,re.MULTILINE) return ext,urls
def url_save_chunked(url, filepath, bar, refer=None, is_part=False, faker=False, headers={}): if os.path.exists(filepath): if not force: if not is_part: if bar: bar.done() write2buf('Skipping %s: file already exists' % tr(os.path.basename(filepath))) set_exist(True) else: if bar: bar.update_received(os.path.getsize(filepath)) return else: if not is_part: if bar: bar.done() write2buf('Overwriting %s' % tr(os.path.basename(filepath)), '...') elif not os.path.exists(os.path.dirname(filepath)): os.mkdir(os.path.dirname(filepath)) temp_filepath = filepath + '.download' received = 0 if not force: open_mode = 'ab' if os.path.exists(temp_filepath): received += os.path.getsize(temp_filepath) if bar: bar.update_received(os.path.getsize(temp_filepath)) else: open_mode = 'wb' if faker: headers = fake_headers elif headers: headers = headers else: headers = {} if received: headers['Range'] = 'bytes=' + str(received) + '-' if refer: headers['Referer'] = refer response = request.urlopen(request.Request(url, headers=headers), None) with open(temp_filepath, open_mode) as output: while True: buffer = response.read(1024 * 256) if not buffer: break output.write(buffer) received += len(buffer) if bar: bar.update_received(len(buffer)) assert received == os.path.getsize(temp_filepath), '%s == %s == %s' % (received, os.path.getsize(temp_filepath)) if os.access(filepath, os.W_OK): os.remove(filepath) # on Windows rename could fail if destination filepath exists os.rename(temp_filepath, filepath)
def ffmpeg_play_stream(player, url, params={}): ffmpeg_params = [] # should these exist... if len(params) > 0: for k, v in params: ffmpeg_params.append(k) ffmpeg_params.append(v) write2buf('Playing streaming content with FFmpeg, press 1 to stop recording...') ffmpeg_params = [FFMPEG] + LOGLEVEL + ['-y', '-re', '-i'] ffmpeg_params.append(url) # not the same here!!!! if FFMPEG == 'avconv': # who cares? ffmpeg_params += ['-c', 'copy', '|'] else: ffmpeg_params += ['-c', 'copy', '-bsf:a', 'aac_adtstoasc', '|'] ffmpeg_params += [player, '-'] write2buf(' '.join(ffmpeg_params)) try: a = subprocess.Popen(ffmpeg_params, stdin=subprocess.PIPE) a.communicate() except KeyboardInterrupt: try: a.stdin.write('q'.encode('utf-8')) except: pass return True
def nicovideo_download(url, output_dir='.', merge=True, info_only=False, **kwargs): import ssl ssl_context = request.HTTPSHandler( context=ssl.SSLContext(ssl.PROTOCOL_TLSv1)) cookie_handler = request.HTTPCookieProcessor() opener = request.build_opener(ssl_context, cookie_handler) request.install_opener(opener) import netrc, getpass try: info = netrc.netrc().authenticators('nicovideo') except: info = None if info is None: user = input("User: "******"Password: "******"Logging in...") nicovideo_login(user, password) html = get_html(url) # necessary! title = unicodize(r1(r'<span class="videoHeaderTitle"[^>]*>([^<]+)</span>', html)) vid = url.split('/')[-1].split('?')[0] api_html = get_html('http://www.nicovideo.jp/api/getflv?v=%s' % vid) real_url = parse.unquote(r1(r'url=([^&]+)&', api_html)) type, ext, size = url_info(real_url) write2buf_info(site_info, title, type, size) if not info_only: download_urls([real_url], title, ext, size, output_dir, merge = merge)
def netease_lyric_download(song, lyric, output_dir='.', info_only=False): if info_only: return title = "%s. %s" % (song['position'], song['name']) filename = '%s.lrc' % get_filename(title) write2buf('Saving %s ...' % filename, end="", flush=True) with open(os.path.join(output_dir, filename), 'w', encoding='utf-8') as x: x.write(lyric) write2buf('Done.')
def play_rtmpdump_stream(player, url, params={}): cmdline = "rtmpdump -r '%s' " % url for key in params.keys(): cmdline += key + " " + params[key] if params[key] != None else "" + " " cmdline += " -o - | %s -" % player write2buf(cmdline) os.system(cmdline) # os.system("rtmpdump -r '%s' -y '%s' -o - | %s -" % (url, playpath, player)) return
def read_mp4(stream): write2buf(stream.name) atoms = parse_atoms(stream) moov = list(filter(lambda x: x.type == b'moov', atoms)) mdat = list(filter(lambda x: x.type == b'mdat', atoms)) assert len(moov) == 1 assert len(mdat) == 1 moov = moov[0] mdat = mdat[0] return atoms, moov, mdat
def netease_lyric_download(song, lyric, output_dir=".", info_only=False): if info_only: return title = "%s. %s" % (song["position"], song["name"]) filename = "%s.lrc" % get_filename(title) write2buf("Saving %s ..." % filename, end="", flush=True) with open(os.path.join(output_dir, filename), "w", encoding="utf-8") as x: x.write(lyric) write2buf("Done.")
def output(video_extractor, pretty_write2buf=True): ve = video_extractor out = {} out['url'] = ve.url out['title'] = ve.title out['site'] = ve.name out['streams'] = ve.streams if pretty_write2buf: write2buf(json.dumps(out, indent=4, sort_keys=True, ensure_ascii=False)) else: write2buf(json.dumps(out))
def concat_mp4(mp4s, output=None): assert mp4s, 'no mp4 file found' import os.path if not output: output = guess_output(mp4s) elif os.path.isdir(output): output = os.path.join(output, guess_output(mp4s)) write2buf('Merging video parts...') merge_mp4s(mp4s, output) return output
def concat_mp4(mp4s, output = None): assert mp4s, 'no mp4 file found' import os.path if not output: output = guess_output(mp4s) elif os.path.isdir(output): output = os.path.join(output, guess_output(mp4s)) write2buf('Merging video parts...') merge_mp4s(mp4s, output) return output
def tudou_download_playlist(url, output_dir='.', merge=True, info_only=False, **kwargs): videos = parse_plist(url) for i, (title, id) in enumerate(videos): write2buf('Processing %s of %s videos...' % (i + 1, len(videos))) tudou_download_by_iid(id, title, output_dir=output_dir, merge=merge, info_only=info_only)
def ffmpeg_concat_av(files, output, ext): write2buf('Merging video parts... ', end="", flush=True) params = [FFMPEG] + LOGLEVEL for file in files: if os.path.isfile(file): params.extend(['-i', file]) params.extend(['-c:v', 'copy']) if ext == 'mp4': params.extend(['-c:a', 'aac']) elif ext == 'webm': params.extend(['-c:a', 'vorbis']) params.extend(['-strict', 'experimental']) params.append(output) return subprocess.call(params)
def ffmpeg_concat_av(files, output, ext): write2buf("Merging video parts... ", end="", flush=True) params = [FFMPEG] + LOGLEVEL for file in files: if os.path.isfile(file): params.extend(["-i", file]) params.extend(["-c:v", "copy"]) if ext == "mp4": params.extend(["-c:a", "aac"]) elif ext == "webm": params.extend(["-c:a", "vorbis"]) params.extend(["-strict", "experimental"]) params.append(output) return subprocess.call(params)
def ffmpeg_concat_ts_to_mkv(files, output="output.mkv"): write2buf("Merging video parts... ", end="", flush=True) params = [FFMPEG] + LOGLEVEL + ["-isync", "-y", "-i"] params.append("concat:") for file in files: if os.path.isfile(file): params[-1] += file + "|" params += ["-f", "matroska", "-c", "copy", output] try: if subprocess.call(params) == 0: return True else: return False except: return False
def download_rtmp_url(url, title, ext, params={}, total_size=0, output_dir='.', refer=None, merge=True, faker=False): assert url if dry_run: write2buf('Real URL:\n%s\n' % [url]) if params.get("-y", False): # None or unset ->False write2buf('Real Playpath:\n%s\n' % [params.get("-y")]) return if player: from .processor.rtmpdump import play_rtmpdump_stream play_rtmpdump_stream(player, url, params) return from .processor.rtmpdump import has_rtmpdump_installed, download_rtmpdump_stream assert has_rtmpdump_installed(), "RTMPDump not installed." download_rtmpdump_stream(url, title, ext, params, output_dir)
def ffmpeg_concat_ts_to_mkv(files, output='output.mkv'): write2buf('Merging video parts... ', end="", flush=True) params = [FFMPEG] + LOGLEVEL + ['-isync', '-y', '-i'] params.append('concat:') for file in files: if os.path.isfile(file): params[-1] += file + '|' params += ['-f', 'matroska', '-c', 'copy', output] try: if subprocess.call(params) == 0: return True else: return False except: return False
def download_url_ffmpeg(url, title, ext, params={}, total_size=0, output_dir='.', refer=None, merge=True, faker=False): assert url if dry_run: write2buf('Real URL:\n%s\n' % [url]) if params.get("-y", False): # None or unset ->False write2buf('Real Playpath:\n%s\n' % [params.get("-y")]) return if player: from .processor.ffmpeg import ffmpeg_play_stream ffmpeg_play_stream(player, url, params) return from .processor.ffmpeg import has_ffmpeg_installed, ffmpeg_download_stream assert has_ffmpeg_installed(), "FFmpeg not installed." ffmpeg_download_stream(url, title, ext, params, output_dir)
def ffmpeg_concat_flv_to_mp4(files, output='output.mp4'): write2buf('Merging video parts... ', end="", flush=True) # Use concat demuxer on FFmpeg >= 1.1 if FFMPEG == 'ffmpeg' and (FFMPEG_VERSION[0] >= 2 or (FFMPEG_VERSION[0] == 1 and FFMPEG_VERSION[1] >= 1)): concat_list = open(output + '.txt', 'w', encoding="utf-8") for file in files: if os.path.isfile(file): # for escaping rules, see: # https://www.ffmpeg.org/ffmpeg-utils.html#Quoting-and-escaping concat_list.write("file %s\n" % parameterize(file)) concat_list.close() params = [FFMPEG] + LOGLEVEL + ['-f', 'concat', '-safe', '-1', '-y', '-i'] params.append(output + '.txt') params += ['-c', 'copy', output] subprocess.check_call(params) os.remove(output + '.txt') return True for file in files: if os.path.isfile(file): params = [FFMPEG] + LOGLEVEL + ['-y', '-i'] params.append(file) params += ['-map', '0', '-c', 'copy', '-f', 'mpegts', '-bsf:v', 'h264_mp4toannexb'] params.append(file + '.ts') subprocess.call(params) params = [FFMPEG] + LOGLEVEL + ['-y', '-i'] params.append('concat:') for file in files: f = file + '.ts' if os.path.isfile(f): params[-1] += f + '|' if FFMPEG == 'avconv': params += ['-c', 'copy', output] else: params += ['-c', 'copy', '-absf', 'aac_adtstoasc', output] if subprocess.call(params) == 0: for file in files: os.remove(file + '.ts') return True else: raise
def ffmpeg_concat_flv_to_mp4(files, output="output.mp4"): write2buf("Merging video parts... ", end="", flush=True) # Use concat demuxer on FFmpeg >= 1.1 if FFMPEG == "ffmpeg" and (FFMPEG_VERSION[0] >= 2 or (FFMPEG_VERSION[0] == 1 and FFMPEG_VERSION[1] >= 1)): concat_list = open(output + ".txt", "w", encoding="utf-8") for file in files: if os.path.isfile(file): # for escaping rules, see: # https://www.ffmpeg.org/ffmpeg-utils.html#Quoting-and-escaping concat_list.write("file %s\n" % parameterize(file)) concat_list.close() params = [FFMPEG] + LOGLEVEL + ["-f", "concat", "-safe", "-1", "-y", "-i"] params.append(output + ".txt") params += ["-c", "copy", output] subprocess.check_call(params) os.remove(output + ".txt") return True for file in files: if os.path.isfile(file): params = [FFMPEG] + LOGLEVEL + ["-y", "-i"] params.append(file) params += ["-map", "0", "-c", "copy", "-f", "mpegts", "-bsf:v", "h264_mp4toannexb"] params.append(file + ".ts") subprocess.call(params) params = [FFMPEG] + LOGLEVEL + ["-y", "-i"] params.append("concat:") for file in files: f = file + ".ts" if os.path.isfile(f): params[-1] += f + "|" if FFMPEG == "avconv": params += ["-c", "copy", output] else: params += ["-c", "copy", "-absf", "aac_adtstoasc", output] if subprocess.call(params) == 0: for file in files: os.remove(file + ".ts") return True else: raise
def concat_ts(ts_parts, output = None): assert ts_parts, 'no ts files found' import os.path if not output: output = guess_output(ts_parts) elif os.path.isdir(output): output = os.path.join(output, guess_output(ts_parts)) write2buf('Merging video parts...') ts_out_file = open(output, "wb") for ts_in in ts_parts: ts_in_file = open(ts_in, "rb") ts_in_data = ts_in_file.read() ts_in_file.close() ts_out_file.write(ts_in_data) ts_out_file.close() return output
def download_rtmpdump_stream(url, title, ext, params={}, output_dir='.'): filename = '%s.%s' % (title, ext) filepath = os.path.join(output_dir, filename) cmdline = [RTMPDUMP, '-r'] cmdline.append(url) cmdline.append('-o') cmdline.append(filepath) for key in params.keys(): cmdline.append(key) if params[key] != None: cmdline.append(params[key]) # cmdline.append('-y') # cmdline.append(playpath) write2buf("Call rtmpdump:\n" + " ".join(cmdline) + "\n") subprocess.call(cmdline) return
def concat_flv(flvs, output=None): assert flvs, 'no flv file found' import os.path if not output: output = guess_output(flvs) elif os.path.isdir(output): output = os.path.join(output, guess_output(flvs)) write2buf('Merging video parts...') ins = [open(flv, 'rb') for flv in flvs] for stream in ins: read_flv_header(stream) meta_tags = map(read_tag, ins) metas = list(map(read_meta_tag, meta_tags)) meta_types, metas = zip(*metas) assert len(set(meta_types)) == 1 meta_type = meta_types[0] # must merge fields: duration # TODO: check other meta info, update other meta info total_duration = sum(meta.get('duration') for meta in metas) meta_data = metas[0] meta_data.set('duration', total_duration) out = open(output, 'wb') write_flv_header(out) write_meta_tag(out, meta_type, meta_data) timestamp_start = 0 for stream in ins: while True: tag = read_tag(stream) if tag: data_type, timestamp, body_size, body, previous_tag_size = tag timestamp += timestamp_start tag = data_type, timestamp, body_size, body, previous_tag_size write_tag(out, tag) else: break timestamp_start = timestamp write_uint(out, previous_tag_size) return output
def concat_flv(flvs, output = None): assert flvs, 'no flv file found' import os.path if not output: output = guess_output(flvs) elif os.path.isdir(output): output = os.path.join(output, guess_output(flvs)) write2buf('Merging video parts...') ins = [open(flv, 'rb') for flv in flvs] for stream in ins: read_flv_header(stream) meta_tags = map(read_tag, ins) metas = list(map(read_meta_tag, meta_tags)) meta_types, metas = zip(*metas) assert len(set(meta_types)) == 1 meta_type = meta_types[0] # must merge fields: duration # TODO: check other meta info, update other meta info total_duration = sum(meta.get('duration') for meta in metas) meta_data = metas[0] meta_data.set('duration', total_duration) out = open(output, 'wb') write_flv_header(out) write_meta_tag(out, meta_type, meta_data) timestamp_start = 0 for stream in ins: while True: tag = read_tag(stream) if tag: data_type, timestamp, body_size, body, previous_tag_size = tag timestamp += timestamp_start tag = data_type, timestamp, body_size, body, previous_tag_size write_tag(out, tag) else: break timestamp_start = timestamp write_uint(out, previous_tag_size) return output
def ffmpeg_download_stream(files, title, ext, params={}, output_dir='.'): """str, str->True WARNING: NOT THE SAME PARMS AS OTHER FUNCTIONS!!!!!! You can basicly download anything with this function but better leave it alone with """ output = title + '.' + ext if not (output_dir == '.'): output = output_dir + output ffmpeg_params = [] # should these exist... if len(params) > 0: for k, v in params: ffmpeg_params.append(k) ffmpeg_params.append(v) write2buf('Downloading streaming content with FFmpeg, press q to stop recording...') ffmpeg_params = [FFMPEG] + ['-y', '-re', '-i'] ffmpeg_params.append(files) # not the same here!!!! if FFMPEG == 'avconv': # who cares? ffmpeg_params += ['-c', 'copy', output] else: ffmpeg_params += ['-c', 'copy', '-bsf:a', 'aac_adtstoasc'] ffmpeg_params.append(output) write2buf(' '.join(ffmpeg_params)) try: a = subprocess.Popen(ffmpeg_params, stdin=subprocess.PIPE) a.communicate() except KeyboardInterrupt: try: a.stdin.write('q'.encode('utf-8')) except: pass return True
def nicovideo_download(url, output_dir='.', merge=True, info_only=False, **kwargs): import ssl ssl_context = request.HTTPSHandler( context=ssl.SSLContext(ssl.PROTOCOL_TLSv1)) cookie_handler = request.HTTPCookieProcessor() opener = request.build_opener(ssl_context, cookie_handler) request.install_opener(opener) import netrc, getpass try: info = netrc.netrc().authenticators('nicovideo') except: info = None if info is None: user = input("User: "******"Password: "******"Logging in...") nicovideo_login(user, password) html = get_html(url) # necessary! title = unicodize( r1(r'<span class="videoHeaderTitle"[^>]*>([^<]+)</span>', html)) vid = url.split('/')[-1].split('?')[0] api_html = get_html('http://www.nicovideo.jp/api/getflv?v=%s' % vid) real_url = parse.unquote(r1(r'url=([^&]+)&', api_html)) type, ext, size = url_info(real_url) write2buf_info(site_info, title, type, size) if not info_only: download_urls([real_url], title, ext, size, output_dir, merge=merge)
def video_info(vid, **kwargs): url = 'http://api.letv.com/mms/out/video/playJson?id={}&platid=1&splatid=101&format=1&tkey={}&domain=www.letv.com'.format( vid, calcTimeKey(int(time.time()))) r = get_content(url, decoded=False) info = json.loads(str(r, "utf-8")) stream_id = None support_stream_id = info["playurl"]["dispatch"].keys() if "stream_id" in kwargs and kwargs["stream_id"].lower( ) in support_stream_id: stream_id = kwargs["stream_id"] else: write2buf("Current Video Supports:") for i in support_stream_id: write2buf("\t--format", i, "<URL>") if "1080p" in support_stream_id: stream_id = '1080p' elif "720p" in support_stream_id: stream_id = '720p' else: stream_id = sorted(support_stream_id, key=lambda i: int(i[1:]))[-1] url = info["playurl"]["domain"][0] + info["playurl"]["dispatch"][ stream_id][0] ext = info["playurl"]["dispatch"][stream_id][1].split('.')[-1] url += "&ctv=pc&m3v=1&termid=1&format=1&hwtype=un&ostype=Linux&tag=letv&sign=letv&expect=3&tn={}&pay=0&iscpn=f9051&rateid={}".format( random.random(), stream_id) r2 = get_content(url, decoded=False) info2 = json.loads(str(r2, "utf-8")) # hold on ! more things to do # to decode m3u8 (encoded) m3u8 = get_content(info2["location"], decoded=False) m3u8_list = decode(m3u8) urls = re.findall(r'^[^#][^\r]*', m3u8_list, re.MULTILINE) return ext, urls
def acfun_download_by_vid(vid, title, output_dir='.', merge=True, info_only=False, **kwargs): info = json.loads(get_html('http://www.acfun.tv/video/getVideo.aspx?id=' + vid)) sourceType = info['sourceType'] if 'sourceId' in info: sourceId = info['sourceId'] # danmakuId = info['danmakuId'] if sourceType == 'sina': sina_download_by_vid(sourceId, title, output_dir=output_dir, merge=merge, info_only=info_only) elif sourceType == 'youku': youku_download_by_vid(sourceId, title=title, output_dir=output_dir, merge=merge, info_only=info_only, **kwargs) elif sourceType == 'tudou': tudou_download_by_iid(sourceId, title, output_dir=output_dir, merge=merge, info_only=info_only) elif sourceType == 'qq': qq_download_by_vid(sourceId, title, output_dir=output_dir, merge=merge, info_only=info_only) elif sourceType == 'letv': letvcloud_download_by_vu(sourceId, '2d8c027396', title, output_dir=output_dir, merge=merge, info_only=info_only) elif sourceType == 'zhuzhan': a = 'http://api.aixifan.com/plays/%s' % vid s = json.loads(get_content(a, headers={'deviceType': '2'})) if s['data']['source'] == "zhuzhan-youku": sourceId = s['data']['sourceId'] youku_open_download_by_vid(client_id='908a519d032263f8', vid=sourceId, title=title, output_dir=output_dir, merge=merge, info_only=info_only, **kwargs) else: raise NotImplementedError(sourceType) if not info_only and not dry_run: if not kwargs['caption']: write2buf('Skipping danmaku.') return try: title = get_filename(title) write2buf('Downloading %s ...\n' % (title + '.cmt.json')) cmt = get_srt_json(vid) with open(os.path.join(output_dir, title + '.cmt.json'), 'w', encoding='utf-8') as x: x.write(cmt) except: pass
def p_i(self, stream_id): if stream_id in self.streams: stream = self.streams[stream_id] else: stream = self.dash_streams[stream_id] maybe_print(" - title: %s" % self.title) write2buf(" size: %s MiB (%s bytes)" % (round(stream['size'] / 1048576, 1), stream['size'])) write2buf(" url: %s" % self.url) write2buf()
def p_i(self, stream_id): if stream_id in self.streams: stream = self.streams[stream_id] else: stream = self.dash_streams[stream_id] maybe_write2buf(" - title: %s" % self.title) write2buf(" size: %s MiB (%s bytes)" % (round(stream['size'] / 1048576, 1), stream['size'])) write2buf(" url: %s" % self.url) write2buf()
def download_urls_chunked(urls, title, ext, total_size, output_dir='.', refer=None, merge=True, faker=False, headers={}): assert urls if dry_run: write2buf('Real URLs:\n%s\n' % urls) return if player: launch_player(player, urls) return title = tr(get_filename(title)) filename = '%s.%s' % (title, ext) filepath = os.path.join(output_dir, filename) if total_size and ext in ('ts'): if not force and os.path.exists(filepath[:-3] + '.mkv'): write2buf('Skipping %s: file already exists' % filepath[:-3] + '.mkv') write2buf() set_exist(True) return bar = SimpleProgressBar(total_size, len(urls)) else: bar = PiecesProgressBar(total_size, len(urls)) if len(urls) == 1: parts = [] url = urls[0] write2buf('Downloading %s ...' % tr(filename)) filepath = os.path.join(output_dir, filename) parts.append(filepath) url_save_chunked(url, filepath, bar, refer=refer, faker=faker, headers=headers) bar.done() if not merge: write2buf() return if ext == 'ts': from .processor.ffmpeg import has_ffmpeg_installed if has_ffmpeg_installed(): from .processor.ffmpeg import ffmpeg_convert_ts_to_mkv if ffmpeg_convert_ts_to_mkv(parts, os.path.join(output_dir, title + '.mkv')): for part in parts: os.remove(part) else: os.remove(os.path.join(output_dir, title + '.mkv')) else: write2buf('No ffmpeg is found. Conversion aborted.') else: write2buf("Can't convert %s files" % ext) else: parts = [] write2buf('Downloading %s.%s ...' % (tr(title), ext)) for i, url in enumerate(urls): filename = '%s[%02d].%s' % (title, i, ext) filepath = os.path.join(output_dir, filename) parts.append(filepath) # write2buf 'Downloading %s [%s/%s]...' % (tr(filename), i + 1, len(urls)) bar.update_piece(i + 1) url_save_chunked(url, filepath, bar, refer=refer, is_part=True, faker=faker, headers=headers) bar.done() if not merge: write2buf() return if ext == 'ts': from .processor.ffmpeg import has_ffmpeg_installed if has_ffmpeg_installed(): from .processor.ffmpeg import ffmpeg_concat_ts_to_mkv if ffmpeg_concat_ts_to_mkv(parts, os.path.join(output_dir, title + '.mkv')): for part in parts: os.remove(part) else: os.remove(os.path.join(output_dir, title + '.mkv')) else: write2buf('No ffmpeg is found. Merging aborted.') else: write2buf("Can't merge %s files" % ext) write2buf()
def download_urls(urls, title, ext, total_size, output_dir='.', refer=None, merge=True, faker=False, headers={}, **kwargs): assert urls if json_output: json_output_.download_urls(urls=urls, title=title, ext=ext, total_size=total_size, refer=refer) return if dry_run: write2buf('Real URLs:\n%s' % '\n'.join(urls)) return if player: launch_player(player, urls) return if not total_size: try: total_size = urls_size(urls, faker=faker, headers=headers) except Exception as e: mlog.exception(e) title = tr(get_filename(title)) output_filename = get_output_filename(urls, title, ext, output_dir, merge) output_filepath = os.path.join(output_dir, output_filename) if total_size: if not force and os.path.exists(output_filepath) and os.path.getsize(output_filepath) >= total_size * 0.9: write2buf('Skipping %s: file already exists' % output_filepath) write2buf() set_exist(True) return bar = SimpleProgressBar(total_size, len(urls)) else: bar = PiecesProgressBar(total_size, len(urls)) if len(urls) == 1: url = urls[0] write2buf('Downloading %s ...' % tr(output_filename)) bar.update() url_save(url, output_filepath, bar, refer=refer, faker=faker, headers=headers) bar.done() else: parts = [] write2buf('Downloading %s.%s ...' % (tr(title), ext)) bar.update() for i, url in enumerate(urls): filename = '%s[%02d].%s' % (title, i, ext) filepath = os.path.join(output_dir, filename) parts.append(filepath) # write2buf 'Downloading %s [%s/%s]...' % (tr(filename), i + 1, len(urls)) bar.update_piece(i + 1) url_save(url, filepath, bar, refer=refer, is_part=True, faker=faker, headers=headers) bar.done() if not merge: write2buf() return if 'av' in kwargs and kwargs['av']: from .processor.ffmpeg import has_ffmpeg_installed if has_ffmpeg_installed(): from .processor.ffmpeg import ffmpeg_concat_av ret = ffmpeg_concat_av(parts, output_filepath, ext) write2buf('Merged into %s' % output_filename) if ret == 0: for part in parts: os.remove(part) elif ext in ['flv', 'f4v']: try: from .processor.ffmpeg import has_ffmpeg_installed if has_ffmpeg_installed(): from .processor.ffmpeg import ffmpeg_concat_flv_to_mp4 ffmpeg_concat_flv_to_mp4(parts, output_filepath) else: from .processor.join_flv import concat_flv concat_flv(parts, output_filepath) write2buf('Merged into %s' % output_filename) except: raise else: for part in parts: os.remove(part) elif ext == 'mp4': try: from .processor.ffmpeg import has_ffmpeg_installed if has_ffmpeg_installed(): from .processor.ffmpeg import ffmpeg_concat_mp4_to_mp4 ffmpeg_concat_mp4_to_mp4(parts, output_filepath) else: from .processor.join_mp4 import concat_mp4 concat_mp4(parts, output_filepath) write2buf('Merged into %s' % output_filename) except: raise else: for part in parts: os.remove(part) elif ext == "ts": try: from .processor.ffmpeg import has_ffmpeg_installed if has_ffmpeg_installed(): from .processor.ffmpeg import ffmpeg_concat_ts_to_mkv ffmpeg_concat_ts_to_mkv(parts, output_filepath) else: from .processor.join_ts import concat_ts concat_ts(parts, output_filepath) write2buf('Merged into %s' % output_filename) except: raise else: for part in parts: os.remove(part) else: write2buf("Can't merge %s files" % ext) write2buf()
def done(self): if self.displayed: write2buf() self.displayed = False
def p_stream(self, stream_id): if stream_id in self.streams: stream = self.streams[stream_id] else: stream = self.dash_streams[stream_id] if 'itag' in stream: write2buf(" - itag: %s" % log.swrite2buf(stream_id, log.NEGATIVE)) else: write2buf(" - format: %s" % log.swrite2buf(stream_id, log.NEGATIVE)) if 'container' in stream: write2buf(" container: %s" % stream['container']) if 'video_profile' in stream: maybe_write2buf(" video-profile: %s" % stream['video_profile']) if 'quality' in stream: write2buf(" quality: %s" % stream['quality']) if 'size' in stream: write2buf(" size: %s MiB (%s bytes)" % (round(stream['size'] / 1048576, 1), stream['size'])) if 'itag' in stream: write2buf(" # download-with: %s" % log.swrite2buf("you-get --itag=%s [URL]" % stream_id, log.UNDERLINE)) else: write2buf(" # download-with: %s" % log.swrite2buf("you-get --format=%s [URL]" % stream_id, log.UNDERLINE)) write2buf()
def maybe_write2buf(*s): try: write2buf(*s) except: pass
def url_save_chunked(url, filepath, bar, refer=None, is_part=False, faker=False, headers={}): if os.path.exists(filepath): if not force: if not is_part: if bar: bar.done() write2buf('Skipping %s: file already exists' % tr(os.path.basename(filepath))) set_exist(True) else: if bar: bar.update_received(os.path.getsize(filepath)) return else: if not is_part: if bar: bar.done() write2buf('Overwriting %s' % tr(os.path.basename(filepath)), '...') elif not os.path.exists(os.path.dirname(filepath)): os.mkdir(os.path.dirname(filepath)) temp_filepath = filepath + '.download' received = 0 if not force: open_mode = 'ab' if os.path.exists(temp_filepath): received += os.path.getsize(temp_filepath) if bar: bar.update_received(os.path.getsize(temp_filepath)) else: open_mode = 'wb' if faker: headers = fake_headers elif headers: headers = headers else: headers = {} if received: headers['Range'] = 'bytes=' + str(received) + '-' if refer: headers['Referer'] = refer response = request.urlopen(request.Request(url, headers=headers), None) with open(temp_filepath, open_mode) as output: while True: buffer = response.read(1024 * 256) if not buffer: break output.write(buffer) received += len(buffer) if bar: bar.update_received(len(buffer)) assert received == os.path.getsize(temp_filepath), '%s == %s == %s' % ( received, os.path.getsize(temp_filepath)) if os.access(filepath, os.W_OK): os.remove( filepath ) # on Windows rename could fail if destination filepath exists os.rename(temp_filepath, filepath)
def write2buf_info(site_info, title, type, size): if json_output: json_output_.write2buf_info(site_info=site_info, title=title, type=type, size=size) return if type: type = type.lower() if type in ['3gp']: type = 'video/3gpp' elif type in ['asf', 'wmv']: type = 'video/x-ms-asf' elif type in ['flv', 'f4v']: type = 'video/x-flv' elif type in ['mkv']: type = 'video/x-matroska' elif type in ['mp3']: type = 'audio/mpeg' elif type in ['mp4']: type = 'video/mp4' elif type in ['mov']: type = 'video/quicktime' elif type in ['ts']: type = 'video/MP2T' elif type in ['webm']: type = 'video/webm' elif type in ['jpg']: type = 'image/jpeg' elif type in ['png']: type = 'image/png' elif type in ['gif']: type = 'image/gif' if type in ['video/3gpp']: type_info = "3GPP multimedia file (%s)" % type elif type in ['video/x-flv', 'video/f4v']: type_info = "Flash video (%s)" % type elif type in ['video/mp4', 'video/x-m4v']: type_info = "MPEG-4 video (%s)" % type elif type in ['video/MP2T']: type_info = "MPEG-2 transport stream (%s)" % type elif type in ['video/webm']: type_info = "WebM video (%s)" % type # elif type in ['video/ogg']: # type_info = "Ogg video (%s)" % type elif type in ['video/quicktime']: type_info = "QuickTime video (%s)" % type elif type in ['video/x-matroska']: type_info = "Matroska video (%s)" % type # elif type in ['video/x-ms-wmv']: # type_info = "Windows Media video (%s)" % type elif type in ['video/x-ms-asf']: type_info = "Advanced Systems Format (%s)" % type # elif type in ['video/mpeg']: # type_info = "MPEG video (%s)" % type elif type in ['audio/mp4']: type_info = "MPEG-4 audio (%s)" % type elif type in ['audio/mpeg']: type_info = "MP3 (%s)" % type elif type in ['image/jpeg']: type_info = "JPEG Image (%s)" % type elif type in ['image/png']: type_info = "Portable Network Graphics (%s)" % type elif type in ['image/gif']: type_info = "Graphics Interchange Format (%s)" % type else: type_info = "Unknown type (%s)" % type maybe_write2buf("Site: ", site_info) maybe_write2buf("Title: ", unescape_html(tr(title))) write2buf("Type: ", type_info) write2buf("Size: ", round(size / 1048576, 2), "MiB (" + str(size) + " Bytes)") write2buf()
def acfun_download_by_vid(vid, title, output_dir='.', merge=True, info_only=False, **kwargs): info = json.loads( get_html('http://www.acfun.tv/video/getVideo.aspx?id=' + vid)) sourceType = info['sourceType'] if 'sourceId' in info: sourceId = info['sourceId'] # danmakuId = info['danmakuId'] if sourceType == 'sina': sina_download_by_vid(sourceId, title, output_dir=output_dir, merge=merge, info_only=info_only) elif sourceType == 'youku': youku_download_by_vid(sourceId, title=title, output_dir=output_dir, merge=merge, info_only=info_only, **kwargs) elif sourceType == 'tudou': tudou_download_by_iid(sourceId, title, output_dir=output_dir, merge=merge, info_only=info_only) elif sourceType == 'qq': qq_download_by_vid(sourceId, title, output_dir=output_dir, merge=merge, info_only=info_only) elif sourceType == 'letv': letvcloud_download_by_vu(sourceId, '2d8c027396', title, output_dir=output_dir, merge=merge, info_only=info_only) elif sourceType == 'zhuzhan': a = 'http://api.aixifan.com/plays/%s' % vid s = json.loads(get_content(a, headers={'deviceType': '2'})) if s['data']['source'] == "zhuzhan-youku": sourceId = s['data']['sourceId'] youku_open_download_by_vid(client_id='908a519d032263f8', vid=sourceId, title=title, output_dir=output_dir, merge=merge, info_only=info_only, **kwargs) else: raise NotImplementedError(sourceType) if not info_only and not dry_run: if not kwargs['caption']: write2buf('Skipping danmaku.') return try: title = get_filename(title) write2buf('Downloading %s ...\n' % (title + '.cmt.json')) cmt = get_srt_json(vid) with open(os.path.join(output_dir, title + '.cmt.json'), 'w', encoding='utf-8') as x: x.write(cmt) except: pass
def p(self, stream_id=None): maybe_write2buf("site: %s" % self.__class__.name) maybe_write2buf("title: %s" % self.title) if stream_id: # write2buf the stream write2buf("stream:") self.p_stream(stream_id) elif stream_id is None: # write2buf stream with best quality write2buf("stream: # Best quality") stream_id = self.streams_sorted[0]['id'] if 'id' in self.streams_sorted[0] else self.streams_sorted[0]['itag'] self.p_stream(stream_id) elif stream_id == []: write2buf("streams: # Available quality and codecs") # write2buf DASH streams if self.dash_streams: write2buf(" [ DASH ] %s" % ('_' * 36)) itags = sorted(self.dash_streams, key=lambda i: -self.dash_streams[i]['size']) for stream in itags: self.p_stream(stream) # write2buf all other available streams write2buf(" [ DEFAULT ] %s" % ('_' * 33)) for stream in self.streams_sorted: self.p_stream(stream['id'] if 'id' in stream else stream['itag']) if self.audiolang: write2buf("audio-languages:") for i in self.audiolang: write2buf(" - lang: {}".format(i['lang'])) write2buf(" download-url: {}\n".format(i['url']))
def p_playlist(self, stream_id=None): maybe_write2buf("site: %s" % self.__class__.name) write2buf("playlist: %s" % self.title) write2buf("videos:")
def download_urls_chunked(urls, title, ext, total_size, output_dir='.', refer=None, merge=True, faker=False, headers={}): assert urls if dry_run: write2buf('Real URLs:\n%s\n' % urls) return if player: launch_player(player, urls) return title = tr(get_filename(title)) filename = '%s.%s' % (title, ext) filepath = os.path.join(output_dir, filename) if total_size and ext in ('ts'): if not force and os.path.exists(filepath[:-3] + '.mkv'): write2buf('Skipping %s: file already exists' % filepath[:-3] + '.mkv') write2buf() set_exist(True) return bar = SimpleProgressBar(total_size, len(urls)) else: bar = PiecesProgressBar(total_size, len(urls)) if len(urls) == 1: parts = [] url = urls[0] write2buf('Downloading %s ...' % tr(filename)) filepath = os.path.join(output_dir, filename) parts.append(filepath) url_save_chunked(url, filepath, bar, refer=refer, faker=faker, headers=headers) bar.done() if not merge: write2buf() return if ext == 'ts': from .processor.ffmpeg import has_ffmpeg_installed if has_ffmpeg_installed(): from .processor.ffmpeg import ffmpeg_convert_ts_to_mkv if ffmpeg_convert_ts_to_mkv( parts, os.path.join(output_dir, title + '.mkv')): for part in parts: os.remove(part) else: os.remove(os.path.join(output_dir, title + '.mkv')) else: write2buf('No ffmpeg is found. Conversion aborted.') else: write2buf("Can't convert %s files" % ext) else: parts = [] write2buf('Downloading %s.%s ...' % (tr(title), ext)) for i, url in enumerate(urls): filename = '%s[%02d].%s' % (title, i, ext) filepath = os.path.join(output_dir, filename) parts.append(filepath) # write2buf 'Downloading %s [%s/%s]...' % (tr(filename), i + 1, len(urls)) bar.update_piece(i + 1) url_save_chunked(url, filepath, bar, refer=refer, is_part=True, faker=faker, headers=headers) bar.done() if not merge: write2buf() return if ext == 'ts': from .processor.ffmpeg import has_ffmpeg_installed if has_ffmpeg_installed(): from .processor.ffmpeg import ffmpeg_concat_ts_to_mkv if ffmpeg_concat_ts_to_mkv( parts, os.path.join(output_dir, title + '.mkv')): for part in parts: os.remove(part) else: os.remove(os.path.join(output_dir, title + '.mkv')) else: write2buf('No ffmpeg is found. Merging aborted.') else: write2buf("Can't merge %s files" % ext) write2buf()
def bilibili_download(url, output_dir='.', merge=True, info_only=False, **kwargs): html = get_content(url) if re.match(r'https?://bangumi\.bilibili\.com/', url): # quick hack for bangumi URLs url = r1(r'"([^"]+)" class="v-av-link"', html) html = get_content(url) title = r1_of([ r'<meta name="title" content="([^<>]{1,999})" />', r'<h1[^>]*>([^<>]+)</h1>' ], html) if title: title = unescape_html(title) title = escape_file_path(title) flashvars = r1_of([ r'(cid=\d+)', r'(cid: \d+)', r'flashvars="([^"]+)"', r'"https://[a-z]+\.bilibili\.com/secure,(cid=\d+)(?:&aid=\d+)?"' ], html) assert flashvars flashvars = flashvars.replace(': ', '=') t, cid = flashvars.split('=', 1) cid = cid.split('&')[0] if t == 'cid': if re.match(r'https?://live\.bilibili\.com/', url): title = r1(r'<title>([^<>]+)</title>', html) bilibili_live_download_by_cid(cid, title, output_dir=output_dir, merge=merge, info_only=info_only) else: # multi-P cids = [] pages = re.findall('<option value=\'([^\']*)\'', html) titles = re.findall('<option value=.*>(.+)</option>', html) for i, page in enumerate(pages): html = get_html("http://www.bilibili.com%s" % page) flashvars = r1_of([ r'(cid=\d+)', r'flashvars="([^"]+)"', r'"https://[a-z]+\.bilibili\.com/secure,(cid=\d+)(?:&aid=\d+)?"' ], html) if flashvars: t, cid = flashvars.split('=', 1) cids.append(cid.split('&')[0]) if url.endswith(page): cids = [cid.split('&')[0]] titles = [titles[i]] break # no multi-P if not pages: cids = [cid] titles = [ r1(r'<option value=.* selected>(.+)</option>', html) or title ] for i in range(len(cids)): bilibili_download_by_cid(cids[i], titles[i], output_dir=output_dir, merge=merge, info_only=info_only) elif t == 'vid': sina_download_by_vid(cid, title=title, output_dir=output_dir, merge=merge, info_only=info_only) elif t == 'ykid': youku_download_by_vid(cid, title=title, output_dir=output_dir, merge=merge, info_only=info_only) elif t == 'uid': tudou_download_by_id(cid, title, output_dir=output_dir, merge=merge, info_only=info_only) else: raise NotImplementedError(flashvars) if not info_only and not dry_run: if not kwargs['caption']: write2buf('Skipping danmaku.') return title = get_filename(title) write2buf('Downloading %s ...\n' % (title + '.cmt.xml')) xml = get_srt_xml(cid) with open(os.path.join(output_dir, title + '.cmt.xml'), 'w', encoding='utf-8') as x: x.write(xml)
def download_urls(urls, title, ext, total_size, output_dir='.', refer=None, merge=True, faker=False, headers={}, **kwargs): assert urls if json_output: json_output_.download_urls(urls=urls, title=title, ext=ext, total_size=total_size, refer=refer) return if dry_run: write2buf('Real URLs:\n%s' % '\n'.join(urls)) return if player: launch_player(player, urls) return if not total_size: try: total_size = urls_size(urls, faker=faker, headers=headers) except Exception as e: mlog.exception(e) title = tr(get_filename(title)) output_filename = get_output_filename(urls, title, ext, output_dir, merge) output_filepath = os.path.join(output_dir, output_filename) if total_size: if not force and os.path.exists(output_filepath) and os.path.getsize( output_filepath) >= total_size * 0.9: write2buf('Skipping %s: file already exists' % output_filepath) write2buf() set_exist(True) return bar = SimpleProgressBar(total_size, len(urls)) else: bar = PiecesProgressBar(total_size, len(urls)) if len(urls) == 1: url = urls[0] write2buf('Downloading %s ...' % tr(output_filename)) bar.update() url_save(url, output_filepath, bar, refer=refer, faker=faker, headers=headers) bar.done() else: parts = [] write2buf('Downloading %s.%s ...' % (tr(title), ext)) bar.update() for i, url in enumerate(urls): filename = '%s[%02d].%s' % (title, i, ext) filepath = os.path.join(output_dir, filename) parts.append(filepath) # write2buf 'Downloading %s [%s/%s]...' % (tr(filename), i + 1, len(urls)) bar.update_piece(i + 1) url_save(url, filepath, bar, refer=refer, is_part=True, faker=faker, headers=headers) bar.done() if not merge: write2buf() return if 'av' in kwargs and kwargs['av']: from .processor.ffmpeg import has_ffmpeg_installed if has_ffmpeg_installed(): from .processor.ffmpeg import ffmpeg_concat_av ret = ffmpeg_concat_av(parts, output_filepath, ext) write2buf('Merged into %s' % output_filename) if ret == 0: for part in parts: os.remove(part) elif ext in ['flv', 'f4v']: try: from .processor.ffmpeg import has_ffmpeg_installed if has_ffmpeg_installed(): from .processor.ffmpeg import ffmpeg_concat_flv_to_mp4 ffmpeg_concat_flv_to_mp4(parts, output_filepath) else: from .processor.join_flv import concat_flv concat_flv(parts, output_filepath) write2buf('Merged into %s' % output_filename) except: raise else: for part in parts: os.remove(part) elif ext == 'mp4': try: from .processor.ffmpeg import has_ffmpeg_installed if has_ffmpeg_installed(): from .processor.ffmpeg import ffmpeg_concat_mp4_to_mp4 ffmpeg_concat_mp4_to_mp4(parts, output_filepath) else: from .processor.join_mp4 import concat_mp4 concat_mp4(parts, output_filepath) write2buf('Merged into %s' % output_filename) except: raise else: for part in parts: os.remove(part) elif ext == "ts": try: from .processor.ffmpeg import has_ffmpeg_installed if has_ffmpeg_installed(): from .processor.ffmpeg import ffmpeg_concat_ts_to_mkv ffmpeg_concat_ts_to_mkv(parts, output_filepath) else: from .processor.join_ts import concat_ts concat_ts(parts, output_filepath) write2buf('Merged into %s' % output_filename) except: raise else: for part in parts: os.remove(part) else: write2buf("Can't merge %s files" % ext) write2buf()
def usage(): write2buf('Usage: [python3] join_flv.py --output TARGET.flv flv...')
def url_save(url, filepath, bar, refer=None, is_part=False, faker=False, headers={}): file_size = url_size(url, faker=faker, headers=headers) if os.path.exists(filepath): if not force and file_size == os.path.getsize(filepath): if not is_part: if bar: bar.done() write2buf('Skipping %s: file already exists' % tr(os.path.basename(filepath))) set_exist(True) else: if bar: bar.update_received(file_size) return else: if not is_part: if bar: bar.done() write2buf('Overwriting %s' % tr(os.path.basename(filepath)), '...') elif not os.path.exists(os.path.dirname(filepath)): os.mkdir(os.path.dirname(filepath)) temp_filepath = filepath + '.download' if file_size != float('inf') else filepath received = 0 if not force: open_mode = 'ab' if os.path.exists(temp_filepath): received += os.path.getsize(temp_filepath) if bar: bar.update_received(os.path.getsize(temp_filepath)) else: open_mode = 'wb' if received < file_size: if faker: headers = fake_headers elif headers: headers = headers else: headers = {} if received: headers['Range'] = 'bytes=' + str(received) + '-' if refer: headers['Referer'] = refer response = request.urlopen(request.Request(url, headers=headers), None) try: range_start = int(response.headers['content-range'][6:].split('/')[0].split('-')[0]) end_length = end = int(response.headers['content-range'][6:].split('/')[1]) range_length = end_length - range_start except: content_length = response.headers['content-length'] range_length = int(content_length) if content_length != None else float('inf') if file_size != received + range_length: received = 0 if bar: bar.received = 0 open_mode = 'wb' with open(temp_filepath, open_mode) as output: while True: buffer = response.read(1024 * 256) if get_stop_thread(): # user maybe force to stop the downloading return if not buffer: if received == file_size: # Download finished break else: # Unexpected termination. Retry request headers['Range'] = 'bytes=' + str(received) + '-' response = request.urlopen(request.Request(url, headers=headers), None) output.write(buffer) received += len(buffer) if bar: bar.update_received(len(buffer)) assert received == os.path.getsize(temp_filepath), '%s == %s == %s' % ( received, os.path.getsize(temp_filepath), temp_filepath) if os.access(filepath, os.W_OK): os.remove(filepath) # on Windows rename could fail if destination filepath exists os.rename(temp_filepath, filepath)
def url_save(url, filepath, bar, refer=None, is_part=False, faker=False, headers={}): file_size = url_size(url, faker=faker, headers=headers) if os.path.exists(filepath): if not force and file_size == os.path.getsize(filepath): if not is_part: if bar: bar.done() write2buf('Skipping %s: file already exists' % tr(os.path.basename(filepath))) set_exist(True) else: if bar: bar.update_received(file_size) return else: if not is_part: if bar: bar.done() write2buf('Overwriting %s' % tr(os.path.basename(filepath)), '...') elif not os.path.exists(os.path.dirname(filepath)): os.mkdir(os.path.dirname(filepath)) temp_filepath = filepath + '.download' if file_size != float( 'inf') else filepath received = 0 if not force: open_mode = 'ab' if os.path.exists(temp_filepath): received += os.path.getsize(temp_filepath) if bar: bar.update_received(os.path.getsize(temp_filepath)) else: open_mode = 'wb' if received < file_size: if faker: headers = fake_headers elif headers: headers = headers else: headers = {} if received: headers['Range'] = 'bytes=' + str(received) + '-' if refer: headers['Referer'] = refer response = request.urlopen(request.Request(url, headers=headers), None) try: range_start = int(response.headers['content-range'][6:].split('/') [0].split('-')[0]) end_length = end = int( response.headers['content-range'][6:].split('/')[1]) range_length = end_length - range_start except: content_length = response.headers['content-length'] range_length = int( content_length) if content_length != None else float('inf') if file_size != received + range_length: received = 0 if bar: bar.received = 0 open_mode = 'wb' with open(temp_filepath, open_mode) as output: while True: buffer = response.read(1024 * 256) if get_stop_thread( ): # user maybe force to stop the downloading return if not buffer: if received == file_size: # Download finished break else: # Unexpected termination. Retry request headers['Range'] = 'bytes=' + str(received) + '-' response = request.urlopen( request.Request(url, headers=headers), None) output.write(buffer) received += len(buffer) if bar: bar.update_received(len(buffer)) assert received == os.path.getsize(temp_filepath), '%s == %s == %s' % ( received, os.path.getsize(temp_filepath), temp_filepath) if os.access(filepath, os.W_OK): os.remove( filepath ) # on Windows rename could fail if destination filepath exists os.rename(temp_filepath, filepath)
def usage(): write2buf('Usage: [python3] join_mp4.py --output TARGET.mp4 mp4...')
def huaban_download(url, output_dir='.', **kwargs): if re.match(r'http://huaban\.com/boards/\d+/', url): huaban_download_board(url, output_dir, **kwargs) else: write2buf('Only board (画板) pages are supported currently') write2buf('ex: http://huaban.com/boards/12345678/')
def download(self, **kwargs): """Override the original one Ugly ugly dirty hack""" if 'json_output' in kwargs and kwargs['json_output']: json_output.output(self) elif 'info_only' in kwargs and kwargs['info_only']: if 'stream_id' in kwargs and kwargs['stream_id']: # Display the stream stream_id = kwargs['stream_id'] if 'index' not in kwargs: self.p(stream_id) else: self.p_i(stream_id) else: # Display all available streams if 'index' not in kwargs: self.p([]) else: stream_id = self.streams_sorted[0][ 'id'] if 'id' in self.streams_sorted[ 0] else self.streams_sorted[0]['itag'] self.p_i(stream_id) else: if 'stream_id' in kwargs and kwargs['stream_id']: # Download the stream stream_id = kwargs['stream_id'] else: # Download stream with the best quality stream_id = self.streams_sorted[0][ 'id'] if 'id' in self.streams_sorted[ 0] else self.streams_sorted[0]['itag'] if 'index' not in kwargs: self.p(stream_id) else: self.p_i(stream_id) if stream_id in self.streams: urls = self.streams[stream_id]['src'] ext = self.streams[stream_id]['container'] total_size = self.streams[stream_id]['size'] else: urls = self.dash_streams[stream_id]['src'] ext = self.dash_streams[stream_id]['container'] total_size = self.dash_streams[stream_id]['size'] if not urls: log.wtf('[Failed] Cannot extract video source.') # For legacy main() #Here's the change!! download_url_ffmpeg( urls[0], self.title, 'mp4', output_dir=kwargs['output_dir'], merge=kwargs['merge'], ) if not kwargs['caption']: write2buf('Skipping captions.') return for lang in self.caption_tracks: filename = '%s.%s.srt' % (get_filename(self.title), lang) write2buf('Saving %s ... ' % filename, end="", flush=True) srt = self.caption_tracks[lang] with open(os.path.join(kwargs['output_dir'], filename), 'w', encoding='utf-8') as x: x.write(srt) write2buf('Done.')