def vidto_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
html = get_content(url)
params = {}
r = re.findall(
r'type="(?:hidden|submit)?"(?:.*?)name="(.+?)"\s* value="?(.+?)">', html)
for name, value in r:
params[name] = value
data = parse.urlencode(params).encode('utf-8')
req = request.Request(url)
print("Please wait for 6 seconds...")
time.sleep(6)
print("Starting")
new_html = request.urlopen(req, data).read().decode('utf-8', 'replace')
new_stff = re.search('lnk_download" href="(.*?)">', new_html)
if(new_stff):
url = new_stff.group(1)
title = params['fname']
type = ""
ext = ""
a, b, size = url_info(url)
print_info(site_info, title, type, size)
if not info_only:
download_urls([url], title, ext, size, output_dir, merge=merge)
else:
write2buf("cannot find link, please review")
pdb.set_trace()
def video_info(vid,**kwargs):
url = 'http://api.letv.com/mms/out/video/playJson?id={}&platid=1&splatid=101&format=1&tkey={}&domain=www.letv.com'.format(vid,calcTimeKey(int(time.time())))
r = get_content(url, decoded=False)
info=json.loads(str(r,"utf-8"))
stream_id = None
support_stream_id = info["playurl"]["dispatch"].keys()
if "stream_id" in kwargs and kwargs["stream_id"].lower() in support_stream_id:
stream_id = kwargs["stream_id"]
else:
write2buf("Current Video Supports:")
for i in support_stream_id:
write2buf("\t--format",i,"<URL>")
if "1080p" in support_stream_id:
stream_id = '1080p'
elif "720p" in support_stream_id:
stream_id = '720p'
else:
stream_id =sorted(support_stream_id,key= lambda i: int(i[1:]))[-1]
url =info["playurl"]["domain"][0]+info["playurl"]["dispatch"][stream_id][0]
ext = info["playurl"]["dispatch"][stream_id][1].split('.')[-1]
url+="&ctv=pc&m3v=1&termid=1&format=1&hwtype=un&ostype=Linux&tag=letv&sign=letv&expect=3&tn={}&pay=0&iscpn=f9051&rateid={}".format(random.random(),stream_id)
r2=get_content(url,decoded=False)
info2=json.loads(str(r2,"utf-8"))
# hold on ! more things to do
# to decode m3u8 (encoded)
m3u8 = get_content(info2["location"],decoded=False)
m3u8_list = decode(m3u8)
urls = re.findall(r'^[^#][^\r]*',m3u8_list,re.MULTILINE)
return ext,urls
def url_save_chunked(url, filepath, bar, refer=None, is_part=False, faker=False, headers={}):
if os.path.exists(filepath):
if not force:
if not is_part:
if bar:
bar.done()
write2buf('Skipping %s: file already exists' % tr(os.path.basename(filepath)))
set_exist(True)
else:
if bar:
bar.update_received(os.path.getsize(filepath))
return
else:
if not is_part:
if bar:
bar.done()
write2buf('Overwriting %s' % tr(os.path.basename(filepath)), '...')
elif not os.path.exists(os.path.dirname(filepath)):
os.mkdir(os.path.dirname(filepath))
temp_filepath = filepath + '.download'
received = 0
if not force:
open_mode = 'ab'
if os.path.exists(temp_filepath):
received += os.path.getsize(temp_filepath)
if bar:
bar.update_received(os.path.getsize(temp_filepath))
else:
open_mode = 'wb'
if faker:
headers = fake_headers
elif headers:
headers = headers
else:
headers = {}
if received:
headers['Range'] = 'bytes=' + str(received) + '-'
if refer:
headers['Referer'] = refer
response = request.urlopen(request.Request(url, headers=headers), None)
with open(temp_filepath, open_mode) as output:
while True:
buffer = response.read(1024 * 256)
if not buffer:
break
output.write(buffer)
received += len(buffer)
if bar:
bar.update_received(len(buffer))
assert received == os.path.getsize(temp_filepath), '%s == %s == %s' % (received, os.path.getsize(temp_filepath))
if os.access(filepath, os.W_OK):
os.remove(filepath) # on Windows rename could fail if destination filepath exists
os.rename(temp_filepath, filepath)
def ffmpeg_play_stream(player, url, params={}):
ffmpeg_params = []
# should these exist...
if len(params) > 0:
for k, v in params:
ffmpeg_params.append(k)
ffmpeg_params.append(v)
write2buf('Playing streaming content with FFmpeg, press 1 to stop recording...')
ffmpeg_params = [FFMPEG] + LOGLEVEL + ['-y', '-re', '-i']
ffmpeg_params.append(url) # not the same here!!!!
if FFMPEG == 'avconv': # who cares?
ffmpeg_params += ['-c', 'copy', '|']
else:
ffmpeg_params += ['-c', 'copy', '-bsf:a', 'aac_adtstoasc', '|']
ffmpeg_params += [player, '-']
write2buf(' '.join(ffmpeg_params))
try:
a = subprocess.Popen(ffmpeg_params, stdin=subprocess.PIPE)
a.communicate()
except KeyboardInterrupt:
try:
a.stdin.write('q'.encode('utf-8'))
except:
pass
return True
def nicovideo_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
import ssl
ssl_context = request.HTTPSHandler(
context=ssl.SSLContext(ssl.PROTOCOL_TLSv1))
cookie_handler = request.HTTPCookieProcessor()
opener = request.build_opener(ssl_context, cookie_handler)
request.install_opener(opener)
import netrc, getpass
try:
info = netrc.netrc().authenticators('nicovideo')
except:
info = None
if info is None:
user = input("User: "******"Password: "******"Logging in...")
nicovideo_login(user, password)
html = get_html(url) # necessary!
title = unicodize(r1(r'<span class="videoHeaderTitle"[^>]*>([^<]+)</span>', html))
vid = url.split('/')[-1].split('?')[0]
api_html = get_html('http://www.nicovideo.jp/api/getflv?v=%s' % vid)
real_url = parse.unquote(r1(r'url=([^&]+)&', api_html))
type, ext, size = url_info(real_url)
write2buf_info(site_info, title, type, size)
if not info_only:
download_urls([real_url], title, ext, size, output_dir, merge = merge)
def netease_lyric_download(song, lyric, output_dir='.', info_only=False):
if info_only: return
title = "%s. %s" % (song['position'], song['name'])
filename = '%s.lrc' % get_filename(title)
write2buf('Saving %s ...' % filename, end="", flush=True)
with open(os.path.join(output_dir, filename), 'w', encoding='utf-8') as x:
x.write(lyric)
write2buf('Done.')
def play_rtmpdump_stream(player, url, params={}):
cmdline = "rtmpdump -r '%s' " % url
for key in params.keys():
cmdline += key + " " + params[key] if params[key] != None else "" + " "
cmdline += " -o - | %s -" % player
write2buf(cmdline)
os.system(cmdline)
# os.system("rtmpdump -r '%s' -y '%s' -o - | %s -" % (url, playpath, player))
return
def play_rtmpdump_stream(player, url, params={}):
cmdline = "rtmpdump -r '%s' " % url
for key in params.keys():
cmdline += key + " " + params[key] if params[key] != None else "" + " "
cmdline += " -o - | %s -" % player
write2buf(cmdline)
os.system(cmdline)
# os.system("rtmpdump -r '%s' -y '%s' -o - | %s -" % (url, playpath, player))
return
def netease_lyric_download(song, lyric, output_dir='.', info_only=False):
if info_only: return
title = "%s. %s" % (song['position'], song['name'])
filename = '%s.lrc' % get_filename(title)
write2buf('Saving %s ...' % filename, end="", flush=True)
with open(os.path.join(output_dir, filename),
'w', encoding='utf-8') as x:
x.write(lyric)
write2buf('Done.')
def read_mp4(stream):
write2buf(stream.name)
atoms = parse_atoms(stream)
moov = list(filter(lambda x: x.type == b'moov', atoms))
mdat = list(filter(lambda x: x.type == b'mdat', atoms))
assert len(moov) == 1
assert len(mdat) == 1
moov = moov[0]
mdat = mdat[0]
return atoms, moov, mdat
def netease_lyric_download(song, lyric, output_dir=".", info_only=False):
if info_only:
return
title = "%s. %s" % (song["position"], song["name"])
filename = "%s.lrc" % get_filename(title)
write2buf("Saving %s ..." % filename, end="", flush=True)
with open(os.path.join(output_dir, filename), "w", encoding="utf-8") as x:
x.write(lyric)
write2buf("Done.")
def read_mp4(stream):
write2buf(stream.name)
atoms = parse_atoms(stream)
moov = list(filter(lambda x: x.type == b'moov', atoms))
mdat = list(filter(lambda x: x.type == b'mdat', atoms))
assert len(moov) == 1
assert len(mdat) == 1
moov = moov[0]
mdat = mdat[0]
return atoms, moov, mdat
def output(video_extractor, pretty_write2buf=True):
ve = video_extractor
out = {}
out['url'] = ve.url
out['title'] = ve.title
out['site'] = ve.name
out['streams'] = ve.streams
if pretty_write2buf:
write2buf(json.dumps(out, indent=4, sort_keys=True, ensure_ascii=False))
else:
write2buf(json.dumps(out))
def concat_mp4(mp4s, output=None):
assert mp4s, 'no mp4 file found'
import os.path
if not output:
output = guess_output(mp4s)
elif os.path.isdir(output):
output = os.path.join(output, guess_output(mp4s))
write2buf('Merging video parts...')
merge_mp4s(mp4s, output)
return output
def output(video_extractor, pretty_write2buf=True):
ve = video_extractor
out = {}
out['url'] = ve.url
out['title'] = ve.title
out['site'] = ve.name
out['streams'] = ve.streams
if pretty_write2buf:
write2buf(json.dumps(out, indent=4, sort_keys=True,
ensure_ascii=False))
else:
write2buf(json.dumps(out))
def concat_mp4(mp4s, output = None):
assert mp4s, 'no mp4 file found'
import os.path
if not output:
output = guess_output(mp4s)
elif os.path.isdir(output):
output = os.path.join(output, guess_output(mp4s))
write2buf('Merging video parts...')
merge_mp4s(mp4s, output)
return output
def tudou_download_playlist(url,
output_dir='.',
merge=True,
info_only=False,
**kwargs):
videos = parse_plist(url)
for i, (title, id) in enumerate(videos):
write2buf('Processing %s of %s videos...' % (i + 1, len(videos)))
tudou_download_by_iid(id,
title,
output_dir=output_dir,
merge=merge,
info_only=info_only)
def ffmpeg_concat_av(files, output, ext):
write2buf('Merging video parts... ', end="", flush=True)
params = [FFMPEG] + LOGLEVEL
for file in files:
if os.path.isfile(file): params.extend(['-i', file])
params.extend(['-c:v', 'copy'])
if ext == 'mp4':
params.extend(['-c:a', 'aac'])
elif ext == 'webm':
params.extend(['-c:a', 'vorbis'])
params.extend(['-strict', 'experimental'])
params.append(output)
return subprocess.call(params)
def ffmpeg_concat_av(files, output, ext):
write2buf("Merging video parts... ", end="", flush=True)
params = [FFMPEG] + LOGLEVEL
for file in files:
if os.path.isfile(file):
params.extend(["-i", file])
params.extend(["-c:v", "copy"])
if ext == "mp4":
params.extend(["-c:a", "aac"])
elif ext == "webm":
params.extend(["-c:a", "vorbis"])
params.extend(["-strict", "experimental"])
params.append(output)
return subprocess.call(params)
def ffmpeg_concat_ts_to_mkv(files, output="output.mkv"):
write2buf("Merging video parts... ", end="", flush=True)
params = [FFMPEG] + LOGLEVEL + ["-isync", "-y", "-i"]
params.append("concat:")
for file in files:
if os.path.isfile(file):
params[-1] += file + "|"
params += ["-f", "matroska", "-c", "copy", output]
try:
if subprocess.call(params) == 0:
return True
else:
return False
except:
return False
def download_rtmp_url(url, title, ext, params={}, total_size=0, output_dir='.', refer=None, merge=True, faker=False):
assert url
if dry_run:
write2buf('Real URL:\n%s\n' % [url])
if params.get("-y", False): # None or unset ->False
write2buf('Real Playpath:\n%s\n' % [params.get("-y")])
return
if player:
from .processor.rtmpdump import play_rtmpdump_stream
play_rtmpdump_stream(player, url, params)
return
from .processor.rtmpdump import has_rtmpdump_installed, download_rtmpdump_stream
assert has_rtmpdump_installed(), "RTMPDump not installed."
download_rtmpdump_stream(url, title, ext, params, output_dir)
def ffmpeg_concat_ts_to_mkv(files, output='output.mkv'):
write2buf('Merging video parts... ', end="", flush=True)
params = [FFMPEG] + LOGLEVEL + ['-isync', '-y', '-i']
params.append('concat:')
for file in files:
if os.path.isfile(file):
params[-1] += file + '|'
params += ['-f', 'matroska', '-c', 'copy', output]
try:
if subprocess.call(params) == 0:
return True
else:
return False
except:
return False
def download_url_ffmpeg(url, title, ext, params={}, total_size=0, output_dir='.', refer=None, merge=True, faker=False):
assert url
if dry_run:
write2buf('Real URL:\n%s\n' % [url])
if params.get("-y", False): # None or unset ->False
write2buf('Real Playpath:\n%s\n' % [params.get("-y")])
return
if player:
from .processor.ffmpeg import ffmpeg_play_stream
ffmpeg_play_stream(player, url, params)
return
from .processor.ffmpeg import has_ffmpeg_installed, ffmpeg_download_stream
assert has_ffmpeg_installed(), "FFmpeg not installed."
ffmpeg_download_stream(url, title, ext, params, output_dir)
def ffmpeg_concat_flv_to_mp4(files, output='output.mp4'):
write2buf('Merging video parts... ', end="", flush=True)
# Use concat demuxer on FFmpeg >= 1.1
if FFMPEG == 'ffmpeg' and (FFMPEG_VERSION[0] >= 2 or (FFMPEG_VERSION[0] == 1 and FFMPEG_VERSION[1] >= 1)):
concat_list = open(output + '.txt', 'w', encoding="utf-8")
for file in files:
if os.path.isfile(file):
# for escaping rules, see:
# https://www.ffmpeg.org/ffmpeg-utils.html#Quoting-and-escaping
concat_list.write("file %s\n" % parameterize(file))
concat_list.close()
params = [FFMPEG] + LOGLEVEL + ['-f', 'concat', '-safe', '-1', '-y', '-i']
params.append(output + '.txt')
params += ['-c', 'copy', output]
subprocess.check_call(params)
os.remove(output + '.txt')
return True
for file in files:
if os.path.isfile(file):
params = [FFMPEG] + LOGLEVEL + ['-y', '-i']
params.append(file)
params += ['-map', '0', '-c', 'copy', '-f', 'mpegts', '-bsf:v', 'h264_mp4toannexb']
params.append(file + '.ts')
subprocess.call(params)
params = [FFMPEG] + LOGLEVEL + ['-y', '-i']
params.append('concat:')
for file in files:
f = file + '.ts'
if os.path.isfile(f):
params[-1] += f + '|'
if FFMPEG == 'avconv':
params += ['-c', 'copy', output]
else:
params += ['-c', 'copy', '-absf', 'aac_adtstoasc', output]
if subprocess.call(params) == 0:
for file in files:
os.remove(file + '.ts')
return True
else:
raise
def ffmpeg_concat_flv_to_mp4(files, output="output.mp4"):
write2buf("Merging video parts... ", end="", flush=True)
# Use concat demuxer on FFmpeg >= 1.1
if FFMPEG == "ffmpeg" and (FFMPEG_VERSION[0] >= 2 or (FFMPEG_VERSION[0] == 1 and FFMPEG_VERSION[1] >= 1)):
concat_list = open(output + ".txt", "w", encoding="utf-8")
for file in files:
if os.path.isfile(file):
# for escaping rules, see:
# https://www.ffmpeg.org/ffmpeg-utils.html#Quoting-and-escaping
concat_list.write("file %s\n" % parameterize(file))
concat_list.close()
params = [FFMPEG] + LOGLEVEL + ["-f", "concat", "-safe", "-1", "-y", "-i"]
params.append(output + ".txt")
params += ["-c", "copy", output]
subprocess.check_call(params)
os.remove(output + ".txt")
return True
for file in files:
if os.path.isfile(file):
params = [FFMPEG] + LOGLEVEL + ["-y", "-i"]
params.append(file)
params += ["-map", "0", "-c", "copy", "-f", "mpegts", "-bsf:v", "h264_mp4toannexb"]
params.append(file + ".ts")
subprocess.call(params)
params = [FFMPEG] + LOGLEVEL + ["-y", "-i"]
params.append("concat:")
for file in files:
f = file + ".ts"
if os.path.isfile(f):
params[-1] += f + "|"
if FFMPEG == "avconv":
params += ["-c", "copy", output]
else:
params += ["-c", "copy", "-absf", "aac_adtstoasc", output]
if subprocess.call(params) == 0:
for file in files:
os.remove(file + ".ts")
return True
else:
raise
def concat_ts(ts_parts, output = None):
assert ts_parts, 'no ts files found'
import os.path
if not output:
output = guess_output(ts_parts)
elif os.path.isdir(output):
output = os.path.join(output, guess_output(ts_parts))
write2buf('Merging video parts...')
ts_out_file = open(output, "wb")
for ts_in in ts_parts:
ts_in_file = open(ts_in, "rb")
ts_in_data = ts_in_file.read()
ts_in_file.close()
ts_out_file.write(ts_in_data)
ts_out_file.close()
return output
def download_rtmpdump_stream(url, title, ext, params={}, output_dir='.'):
filename = '%s.%s' % (title, ext)
filepath = os.path.join(output_dir, filename)
cmdline = [RTMPDUMP, '-r']
cmdline.append(url)
cmdline.append('-o')
cmdline.append(filepath)
for key in params.keys():
cmdline.append(key)
if params[key] != None:
cmdline.append(params[key])
# cmdline.append('-y')
# cmdline.append(playpath)
write2buf("Call rtmpdump:\n" + " ".join(cmdline) + "\n")
subprocess.call(cmdline)
return
def download_rtmpdump_stream(url, title, ext, params={}, output_dir='.'):
filename = '%s.%s' % (title, ext)
filepath = os.path.join(output_dir, filename)
cmdline = [RTMPDUMP, '-r']
cmdline.append(url)
cmdline.append('-o')
cmdline.append(filepath)
for key in params.keys():
cmdline.append(key)
if params[key] != None:
cmdline.append(params[key])
# cmdline.append('-y')
# cmdline.append(playpath)
write2buf("Call rtmpdump:\n" + " ".join(cmdline) + "\n")
subprocess.call(cmdline)
return
def concat_flv(flvs, output=None):
assert flvs, 'no flv file found'
import os.path
if not output:
output = guess_output(flvs)
elif os.path.isdir(output):
output = os.path.join(output, guess_output(flvs))
write2buf('Merging video parts...')
ins = [open(flv, 'rb') for flv in flvs]
for stream in ins:
read_flv_header(stream)
meta_tags = map(read_tag, ins)
metas = list(map(read_meta_tag, meta_tags))
meta_types, metas = zip(*metas)
assert len(set(meta_types)) == 1
meta_type = meta_types[0]
# must merge fields: duration
# TODO: check other meta info, update other meta info
total_duration = sum(meta.get('duration') for meta in metas)
meta_data = metas[0]
meta_data.set('duration', total_duration)
out = open(output, 'wb')
write_flv_header(out)
write_meta_tag(out, meta_type, meta_data)
timestamp_start = 0
for stream in ins:
while True:
tag = read_tag(stream)
if tag:
data_type, timestamp, body_size, body, previous_tag_size = tag
timestamp += timestamp_start
tag = data_type, timestamp, body_size, body, previous_tag_size
write_tag(out, tag)
else:
break
timestamp_start = timestamp
write_uint(out, previous_tag_size)
return output
def concat_flv(flvs, output = None):
assert flvs, 'no flv file found'
import os.path
if not output:
output = guess_output(flvs)
elif os.path.isdir(output):
output = os.path.join(output, guess_output(flvs))
write2buf('Merging video parts...')
ins = [open(flv, 'rb') for flv in flvs]
for stream in ins:
read_flv_header(stream)
meta_tags = map(read_tag, ins)
metas = list(map(read_meta_tag, meta_tags))
meta_types, metas = zip(*metas)
assert len(set(meta_types)) == 1
meta_type = meta_types[0]
# must merge fields: duration
# TODO: check other meta info, update other meta info
total_duration = sum(meta.get('duration') for meta in metas)
meta_data = metas[0]
meta_data.set('duration', total_duration)
out = open(output, 'wb')
write_flv_header(out)
write_meta_tag(out, meta_type, meta_data)
timestamp_start = 0
for stream in ins:
while True:
tag = read_tag(stream)
if tag:
data_type, timestamp, body_size, body, previous_tag_size = tag
timestamp += timestamp_start
tag = data_type, timestamp, body_size, body, previous_tag_size
write_tag(out, tag)
else:
break
timestamp_start = timestamp
write_uint(out, previous_tag_size)
return output
def ffmpeg_download_stream(files, title, ext, params={}, output_dir='.'):
"""str, str->True
WARNING: NOT THE SAME PARMS AS OTHER FUNCTIONS!!!!!!
You can basicly download anything with this function
but better leave it alone with
"""
output = title + '.' + ext
if not (output_dir == '.'):
output = output_dir + output
ffmpeg_params = []
# should these exist...
if len(params) > 0:
for k, v in params:
ffmpeg_params.append(k)
ffmpeg_params.append(v)
write2buf('Downloading streaming content with FFmpeg, press q to stop recording...')
ffmpeg_params = [FFMPEG] + ['-y', '-re', '-i']
ffmpeg_params.append(files) # not the same here!!!!
if FFMPEG == 'avconv': # who cares?
ffmpeg_params += ['-c', 'copy', output]
else:
ffmpeg_params += ['-c', 'copy', '-bsf:a', 'aac_adtstoasc']
ffmpeg_params.append(output)
write2buf(' '.join(ffmpeg_params))
try:
a = subprocess.Popen(ffmpeg_params, stdin=subprocess.PIPE)
a.communicate()
except KeyboardInterrupt:
try:
a.stdin.write('q'.encode('utf-8'))
except:
pass
return True
def nicovideo_download(url,
output_dir='.',
merge=True,
info_only=False,
**kwargs):
import ssl
ssl_context = request.HTTPSHandler(
context=ssl.SSLContext(ssl.PROTOCOL_TLSv1))
cookie_handler = request.HTTPCookieProcessor()
opener = request.build_opener(ssl_context, cookie_handler)
request.install_opener(opener)
import netrc, getpass
try:
info = netrc.netrc().authenticators('nicovideo')
except:
info = None
if info is None:
user = input("User: "******"Password: "******"Logging in...")
nicovideo_login(user, password)
html = get_html(url) # necessary!
title = unicodize(
r1(r'<span class="videoHeaderTitle"[^>]*>([^<]+)</span>', html))
vid = url.split('/')[-1].split('?')[0]
api_html = get_html('http://www.nicovideo.jp/api/getflv?v=%s' % vid)
real_url = parse.unquote(r1(r'url=([^&]+)&', api_html))
type, ext, size = url_info(real_url)
write2buf_info(site_info, title, type, size)
if not info_only:
download_urls([real_url], title, ext, size, output_dir, merge=merge)
def video_info(vid, **kwargs):
url = 'http://api.letv.com/mms/out/video/playJson?id={}&platid=1&splatid=101&format=1&tkey={}&domain=www.letv.com'.format(
vid, calcTimeKey(int(time.time())))
r = get_content(url, decoded=False)
info = json.loads(str(r, "utf-8"))
stream_id = None
support_stream_id = info["playurl"]["dispatch"].keys()
if "stream_id" in kwargs and kwargs["stream_id"].lower(
) in support_stream_id:
stream_id = kwargs["stream_id"]
else:
write2buf("Current Video Supports:")
for i in support_stream_id:
write2buf("\t--format", i, "<URL>")
if "1080p" in support_stream_id:
stream_id = '1080p'
elif "720p" in support_stream_id:
stream_id = '720p'
else:
stream_id = sorted(support_stream_id, key=lambda i: int(i[1:]))[-1]
url = info["playurl"]["domain"][0] + info["playurl"]["dispatch"][
stream_id][0]
ext = info["playurl"]["dispatch"][stream_id][1].split('.')[-1]
url += "&ctv=pc&m3v=1&termid=1&format=1&hwtype=un&ostype=Linux&tag=letv&sign=letv&expect=3&tn={}&pay=0&iscpn=f9051&rateid={}".format(
random.random(), stream_id)
r2 = get_content(url, decoded=False)
info2 = json.loads(str(r2, "utf-8"))
# hold on ! more things to do
# to decode m3u8 (encoded)
m3u8 = get_content(info2["location"], decoded=False)
m3u8_list = decode(m3u8)
urls = re.findall(r'^[^#][^\r]*', m3u8_list, re.MULTILINE)
return ext, urls
def download_rtmp_url(url,
title,
ext,
params={},
total_size=0,
output_dir='.',
refer=None,
merge=True,
faker=False):
assert url
if dry_run:
write2buf('Real URL:\n%s\n' % [url])
if params.get("-y", False): # None or unset ->False
write2buf('Real Playpath:\n%s\n' % [params.get("-y")])
return
if player:
from .processor.rtmpdump import play_rtmpdump_stream
play_rtmpdump_stream(player, url, params)
return
from .processor.rtmpdump import has_rtmpdump_installed, download_rtmpdump_stream
assert has_rtmpdump_installed(), "RTMPDump not installed."
download_rtmpdump_stream(url, title, ext, params, output_dir)
def acfun_download_by_vid(vid, title, output_dir='.', merge=True, info_only=False, **kwargs):
info = json.loads(get_html('http://www.acfun.tv/video/getVideo.aspx?id=' + vid))
sourceType = info['sourceType']
if 'sourceId' in info: sourceId = info['sourceId']
# danmakuId = info['danmakuId']
if sourceType == 'sina':
sina_download_by_vid(sourceId, title, output_dir=output_dir, merge=merge, info_only=info_only)
elif sourceType == 'youku':
youku_download_by_vid(sourceId, title=title, output_dir=output_dir, merge=merge, info_only=info_only, **kwargs)
elif sourceType == 'tudou':
tudou_download_by_iid(sourceId, title, output_dir=output_dir, merge=merge, info_only=info_only)
elif sourceType == 'qq':
qq_download_by_vid(sourceId, title, output_dir=output_dir, merge=merge, info_only=info_only)
elif sourceType == 'letv':
letvcloud_download_by_vu(sourceId, '2d8c027396', title, output_dir=output_dir, merge=merge, info_only=info_only)
elif sourceType == 'zhuzhan':
a = 'http://api.aixifan.com/plays/%s' % vid
s = json.loads(get_content(a, headers={'deviceType': '2'}))
if s['data']['source'] == "zhuzhan-youku":
sourceId = s['data']['sourceId']
youku_open_download_by_vid(client_id='908a519d032263f8', vid=sourceId, title=title, output_dir=output_dir, merge=merge, info_only=info_only, **kwargs)
else:
raise NotImplementedError(sourceType)
if not info_only and not dry_run:
if not kwargs['caption']:
write2buf('Skipping danmaku.')
return
try:
title = get_filename(title)
write2buf('Downloading %s ...\n' % (title + '.cmt.json'))
cmt = get_srt_json(vid)
with open(os.path.join(output_dir, title + '.cmt.json'), 'w', encoding='utf-8') as x:
x.write(cmt)
except:
pass
def download_url_ffmpeg(url,
title,
ext,
params={},
total_size=0,
output_dir='.',
refer=None,
merge=True,
faker=False):
assert url
if dry_run:
write2buf('Real URL:\n%s\n' % [url])
if params.get("-y", False): # None or unset ->False
write2buf('Real Playpath:\n%s\n' % [params.get("-y")])
return
if player:
from .processor.ffmpeg import ffmpeg_play_stream
ffmpeg_play_stream(player, url, params)
return
from .processor.ffmpeg import has_ffmpeg_installed, ffmpeg_download_stream
assert has_ffmpeg_installed(), "FFmpeg not installed."
ffmpeg_download_stream(url, title, ext, params, output_dir)
def p_i(self, stream_id):
if stream_id in self.streams:
stream = self.streams[stream_id]
else:
stream = self.dash_streams[stream_id]
maybe_print(" - title: %s" % self.title)
write2buf(" size: %s MiB (%s bytes)" % (round(stream['size'] / 1048576, 1), stream['size']))
write2buf(" url: %s" % self.url)
write2buf()
def p_i(self, stream_id):
if stream_id in self.streams:
stream = self.streams[stream_id]
else:
stream = self.dash_streams[stream_id]
maybe_write2buf(" - title: %s" % self.title)
write2buf(" size: %s MiB (%s bytes)" % (round(stream['size'] / 1048576, 1), stream['size']))
write2buf(" url: %s" % self.url)
write2buf()
def download_urls_chunked(urls, title, ext, total_size, output_dir='.', refer=None, merge=True, faker=False,
headers={}):
assert urls
if dry_run:
write2buf('Real URLs:\n%s\n' % urls)
return
if player:
launch_player(player, urls)
return
title = tr(get_filename(title))
filename = '%s.%s' % (title, ext)
filepath = os.path.join(output_dir, filename)
if total_size and ext in ('ts'):
if not force and os.path.exists(filepath[:-3] + '.mkv'):
write2buf('Skipping %s: file already exists' % filepath[:-3] + '.mkv')
write2buf()
set_exist(True)
return
bar = SimpleProgressBar(total_size, len(urls))
else:
bar = PiecesProgressBar(total_size, len(urls))
if len(urls) == 1:
parts = []
url = urls[0]
write2buf('Downloading %s ...' % tr(filename))
filepath = os.path.join(output_dir, filename)
parts.append(filepath)
url_save_chunked(url, filepath, bar, refer=refer, faker=faker, headers=headers)
bar.done()
if not merge:
write2buf()
return
if ext == 'ts':
from .processor.ffmpeg import has_ffmpeg_installed
if has_ffmpeg_installed():
from .processor.ffmpeg import ffmpeg_convert_ts_to_mkv
if ffmpeg_convert_ts_to_mkv(parts, os.path.join(output_dir, title + '.mkv')):
for part in parts:
os.remove(part)
else:
os.remove(os.path.join(output_dir, title + '.mkv'))
else:
write2buf('No ffmpeg is found. Conversion aborted.')
else:
write2buf("Can't convert %s files" % ext)
else:
parts = []
write2buf('Downloading %s.%s ...' % (tr(title), ext))
for i, url in enumerate(urls):
filename = '%s[%02d].%s' % (title, i, ext)
filepath = os.path.join(output_dir, filename)
parts.append(filepath)
# write2buf 'Downloading %s [%s/%s]...' % (tr(filename), i + 1, len(urls))
bar.update_piece(i + 1)
url_save_chunked(url, filepath, bar, refer=refer, is_part=True, faker=faker, headers=headers)
bar.done()
if not merge:
write2buf()
return
if ext == 'ts':
from .processor.ffmpeg import has_ffmpeg_installed
if has_ffmpeg_installed():
from .processor.ffmpeg import ffmpeg_concat_ts_to_mkv
if ffmpeg_concat_ts_to_mkv(parts, os.path.join(output_dir, title + '.mkv')):
for part in parts:
os.remove(part)
else:
os.remove(os.path.join(output_dir, title + '.mkv'))
else:
write2buf('No ffmpeg is found. Merging aborted.')
else:
write2buf("Can't merge %s files" % ext)
write2buf()
def download_urls(urls, title, ext, total_size, output_dir='.', refer=None, merge=True, faker=False, headers={},
**kwargs):
assert urls
if json_output:
json_output_.download_urls(urls=urls, title=title, ext=ext, total_size=total_size, refer=refer)
return
if dry_run:
write2buf('Real URLs:\n%s' % '\n'.join(urls))
return
if player:
launch_player(player, urls)
return
if not total_size:
try:
total_size = urls_size(urls, faker=faker, headers=headers)
except Exception as e:
mlog.exception(e)
title = tr(get_filename(title))
output_filename = get_output_filename(urls, title, ext, output_dir, merge)
output_filepath = os.path.join(output_dir, output_filename)
if total_size:
if not force and os.path.exists(output_filepath) and os.path.getsize(output_filepath) >= total_size * 0.9:
write2buf('Skipping %s: file already exists' % output_filepath)
write2buf()
set_exist(True)
return
bar = SimpleProgressBar(total_size, len(urls))
else:
bar = PiecesProgressBar(total_size, len(urls))
if len(urls) == 1:
url = urls[0]
write2buf('Downloading %s ...' % tr(output_filename))
bar.update()
url_save(url, output_filepath, bar, refer=refer, faker=faker, headers=headers)
bar.done()
else:
parts = []
write2buf('Downloading %s.%s ...' % (tr(title), ext))
bar.update()
for i, url in enumerate(urls):
filename = '%s[%02d].%s' % (title, i, ext)
filepath = os.path.join(output_dir, filename)
parts.append(filepath)
# write2buf 'Downloading %s [%s/%s]...' % (tr(filename), i + 1, len(urls))
bar.update_piece(i + 1)
url_save(url, filepath, bar, refer=refer, is_part=True, faker=faker, headers=headers)
bar.done()
if not merge:
write2buf()
return
if 'av' in kwargs and kwargs['av']:
from .processor.ffmpeg import has_ffmpeg_installed
if has_ffmpeg_installed():
from .processor.ffmpeg import ffmpeg_concat_av
ret = ffmpeg_concat_av(parts, output_filepath, ext)
write2buf('Merged into %s' % output_filename)
if ret == 0:
for part in parts: os.remove(part)
elif ext in ['flv', 'f4v']:
try:
from .processor.ffmpeg import has_ffmpeg_installed
if has_ffmpeg_installed():
from .processor.ffmpeg import ffmpeg_concat_flv_to_mp4
ffmpeg_concat_flv_to_mp4(parts, output_filepath)
else:
from .processor.join_flv import concat_flv
concat_flv(parts, output_filepath)
write2buf('Merged into %s' % output_filename)
except:
raise
else:
for part in parts:
os.remove(part)
elif ext == 'mp4':
try:
from .processor.ffmpeg import has_ffmpeg_installed
if has_ffmpeg_installed():
from .processor.ffmpeg import ffmpeg_concat_mp4_to_mp4
ffmpeg_concat_mp4_to_mp4(parts, output_filepath)
else:
from .processor.join_mp4 import concat_mp4
concat_mp4(parts, output_filepath)
write2buf('Merged into %s' % output_filename)
except:
raise
else:
for part in parts:
os.remove(part)
elif ext == "ts":
try:
from .processor.ffmpeg import has_ffmpeg_installed
if has_ffmpeg_installed():
from .processor.ffmpeg import ffmpeg_concat_ts_to_mkv
ffmpeg_concat_ts_to_mkv(parts, output_filepath)
else:
from .processor.join_ts import concat_ts
concat_ts(parts, output_filepath)
write2buf('Merged into %s' % output_filename)
except:
raise
else:
for part in parts:
os.remove(part)
else:
write2buf("Can't merge %s files" % ext)
write2buf()
def done(self):
if self.displayed:
write2buf()
self.displayed = False
def p_stream(self, stream_id):
if stream_id in self.streams:
stream = self.streams[stream_id]
else:
stream = self.dash_streams[stream_id]
if 'itag' in stream:
write2buf(" - itag: %s" % log.swrite2buf(stream_id, log.NEGATIVE))
else:
write2buf(" - format: %s" % log.swrite2buf(stream_id, log.NEGATIVE))
if 'container' in stream:
write2buf(" container: %s" % stream['container'])
if 'video_profile' in stream:
maybe_write2buf(" video-profile: %s" % stream['video_profile'])
if 'quality' in stream:
write2buf(" quality: %s" % stream['quality'])
if 'size' in stream:
write2buf(" size: %s MiB (%s bytes)" % (round(stream['size'] / 1048576, 1), stream['size']))
if 'itag' in stream:
write2buf(" # download-with: %s" % log.swrite2buf("you-get --itag=%s [URL]" % stream_id, log.UNDERLINE))
else:
write2buf(" # download-with: %s" % log.swrite2buf("you-get --format=%s [URL]" % stream_id, log.UNDERLINE))
write2buf()
def maybe_write2buf(*s):
try:
write2buf(*s)
except:
pass
def url_save_chunked(url,
filepath,
bar,
refer=None,
is_part=False,
faker=False,
headers={}):
if os.path.exists(filepath):
if not force:
if not is_part:
if bar:
bar.done()
write2buf('Skipping %s: file already exists' %
tr(os.path.basename(filepath)))
set_exist(True)
else:
if bar:
bar.update_received(os.path.getsize(filepath))
return
else:
if not is_part:
if bar:
bar.done()
write2buf('Overwriting %s' % tr(os.path.basename(filepath)),
'...')
elif not os.path.exists(os.path.dirname(filepath)):
os.mkdir(os.path.dirname(filepath))
temp_filepath = filepath + '.download'
received = 0
if not force:
open_mode = 'ab'
if os.path.exists(temp_filepath):
received += os.path.getsize(temp_filepath)
if bar:
bar.update_received(os.path.getsize(temp_filepath))
else:
open_mode = 'wb'
if faker:
headers = fake_headers
elif headers:
headers = headers
else:
headers = {}
if received:
headers['Range'] = 'bytes=' + str(received) + '-'
if refer:
headers['Referer'] = refer
response = request.urlopen(request.Request(url, headers=headers), None)
with open(temp_filepath, open_mode) as output:
while True:
buffer = response.read(1024 * 256)
if not buffer:
break
output.write(buffer)
received += len(buffer)
if bar:
bar.update_received(len(buffer))
assert received == os.path.getsize(temp_filepath), '%s == %s == %s' % (
received, os.path.getsize(temp_filepath))
if os.access(filepath, os.W_OK):
os.remove(
filepath
) # on Windows rename could fail if destination filepath exists
os.rename(temp_filepath, filepath)
def write2buf_info(site_info, title, type, size):
if json_output:
json_output_.write2buf_info(site_info=site_info, title=title, type=type, size=size)
return
if type:
type = type.lower()
if type in ['3gp']:
type = 'video/3gpp'
elif type in ['asf', 'wmv']:
type = 'video/x-ms-asf'
elif type in ['flv', 'f4v']:
type = 'video/x-flv'
elif type in ['mkv']:
type = 'video/x-matroska'
elif type in ['mp3']:
type = 'audio/mpeg'
elif type in ['mp4']:
type = 'video/mp4'
elif type in ['mov']:
type = 'video/quicktime'
elif type in ['ts']:
type = 'video/MP2T'
elif type in ['webm']:
type = 'video/webm'
elif type in ['jpg']:
type = 'image/jpeg'
elif type in ['png']:
type = 'image/png'
elif type in ['gif']:
type = 'image/gif'
if type in ['video/3gpp']:
type_info = "3GPP multimedia file (%s)" % type
elif type in ['video/x-flv', 'video/f4v']:
type_info = "Flash video (%s)" % type
elif type in ['video/mp4', 'video/x-m4v']:
type_info = "MPEG-4 video (%s)" % type
elif type in ['video/MP2T']:
type_info = "MPEG-2 transport stream (%s)" % type
elif type in ['video/webm']:
type_info = "WebM video (%s)" % type
# elif type in ['video/ogg']:
# type_info = "Ogg video (%s)" % type
elif type in ['video/quicktime']:
type_info = "QuickTime video (%s)" % type
elif type in ['video/x-matroska']:
type_info = "Matroska video (%s)" % type
# elif type in ['video/x-ms-wmv']:
# type_info = "Windows Media video (%s)" % type
elif type in ['video/x-ms-asf']:
type_info = "Advanced Systems Format (%s)" % type
# elif type in ['video/mpeg']:
# type_info = "MPEG video (%s)" % type
elif type in ['audio/mp4']:
type_info = "MPEG-4 audio (%s)" % type
elif type in ['audio/mpeg']:
type_info = "MP3 (%s)" % type
elif type in ['image/jpeg']:
type_info = "JPEG Image (%s)" % type
elif type in ['image/png']:
type_info = "Portable Network Graphics (%s)" % type
elif type in ['image/gif']:
type_info = "Graphics Interchange Format (%s)" % type
else:
type_info = "Unknown type (%s)" % type
maybe_write2buf("Site: ", site_info)
maybe_write2buf("Title: ", unescape_html(tr(title)))
write2buf("Type: ", type_info)
write2buf("Size: ", round(size / 1048576, 2), "MiB (" + str(size) + " Bytes)")
write2buf()
def acfun_download_by_vid(vid,
title,
output_dir='.',
merge=True,
info_only=False,
**kwargs):
info = json.loads(
get_html('http://www.acfun.tv/video/getVideo.aspx?id=' + vid))
sourceType = info['sourceType']
if 'sourceId' in info: sourceId = info['sourceId']
# danmakuId = info['danmakuId']
if sourceType == 'sina':
sina_download_by_vid(sourceId,
title,
output_dir=output_dir,
merge=merge,
info_only=info_only)
elif sourceType == 'youku':
youku_download_by_vid(sourceId,
title=title,
output_dir=output_dir,
merge=merge,
info_only=info_only,
**kwargs)
elif sourceType == 'tudou':
tudou_download_by_iid(sourceId,
title,
output_dir=output_dir,
merge=merge,
info_only=info_only)
elif sourceType == 'qq':
qq_download_by_vid(sourceId,
title,
output_dir=output_dir,
merge=merge,
info_only=info_only)
elif sourceType == 'letv':
letvcloud_download_by_vu(sourceId,
'2d8c027396',
title,
output_dir=output_dir,
merge=merge,
info_only=info_only)
elif sourceType == 'zhuzhan':
a = 'http://api.aixifan.com/plays/%s' % vid
s = json.loads(get_content(a, headers={'deviceType': '2'}))
if s['data']['source'] == "zhuzhan-youku":
sourceId = s['data']['sourceId']
youku_open_download_by_vid(client_id='908a519d032263f8',
vid=sourceId,
title=title,
output_dir=output_dir,
merge=merge,
info_only=info_only,
**kwargs)
else:
raise NotImplementedError(sourceType)
if not info_only and not dry_run:
if not kwargs['caption']:
write2buf('Skipping danmaku.')
return
try:
title = get_filename(title)
write2buf('Downloading %s ...\n' % (title + '.cmt.json'))
cmt = get_srt_json(vid)
with open(os.path.join(output_dir, title + '.cmt.json'),
'w',
encoding='utf-8') as x:
x.write(cmt)
except:
pass
def p(self, stream_id=None):
maybe_write2buf("site: %s" % self.__class__.name)
maybe_write2buf("title: %s" % self.title)
if stream_id:
# write2buf the stream
write2buf("stream:")
self.p_stream(stream_id)
elif stream_id is None:
# write2buf stream with best quality
write2buf("stream: # Best quality")
stream_id = self.streams_sorted[0]['id'] if 'id' in self.streams_sorted[0] else self.streams_sorted[0]['itag']
self.p_stream(stream_id)
elif stream_id == []:
write2buf("streams: # Available quality and codecs")
# write2buf DASH streams
if self.dash_streams:
write2buf(" [ DASH ] %s" % ('_' * 36))
itags = sorted(self.dash_streams,
key=lambda i: -self.dash_streams[i]['size'])
for stream in itags:
self.p_stream(stream)
# write2buf all other available streams
write2buf(" [ DEFAULT ] %s" % ('_' * 33))
for stream in self.streams_sorted:
self.p_stream(stream['id'] if 'id' in stream else stream['itag'])
if self.audiolang:
write2buf("audio-languages:")
for i in self.audiolang:
write2buf(" - lang: {}".format(i['lang']))
write2buf(" download-url: {}\n".format(i['url']))
def p_playlist(self, stream_id=None):
maybe_write2buf("site: %s" % self.__class__.name)
write2buf("playlist: %s" % self.title)
write2buf("videos:")
def write2buf_info(site_info, title, type, size):
if json_output:
json_output_.write2buf_info(site_info=site_info,
title=title,
type=type,
size=size)
return
if type:
type = type.lower()
if type in ['3gp']:
type = 'video/3gpp'
elif type in ['asf', 'wmv']:
type = 'video/x-ms-asf'
elif type in ['flv', 'f4v']:
type = 'video/x-flv'
elif type in ['mkv']:
type = 'video/x-matroska'
elif type in ['mp3']:
type = 'audio/mpeg'
elif type in ['mp4']:
type = 'video/mp4'
elif type in ['mov']:
type = 'video/quicktime'
elif type in ['ts']:
type = 'video/MP2T'
elif type in ['webm']:
type = 'video/webm'
elif type in ['jpg']:
type = 'image/jpeg'
elif type in ['png']:
type = 'image/png'
elif type in ['gif']:
type = 'image/gif'
if type in ['video/3gpp']:
type_info = "3GPP multimedia file (%s)" % type
elif type in ['video/x-flv', 'video/f4v']:
type_info = "Flash video (%s)" % type
elif type in ['video/mp4', 'video/x-m4v']:
type_info = "MPEG-4 video (%s)" % type
elif type in ['video/MP2T']:
type_info = "MPEG-2 transport stream (%s)" % type
elif type in ['video/webm']:
type_info = "WebM video (%s)" % type
# elif type in ['video/ogg']:
# type_info = "Ogg video (%s)" % type
elif type in ['video/quicktime']:
type_info = "QuickTime video (%s)" % type
elif type in ['video/x-matroska']:
type_info = "Matroska video (%s)" % type
# elif type in ['video/x-ms-wmv']:
# type_info = "Windows Media video (%s)" % type
elif type in ['video/x-ms-asf']:
type_info = "Advanced Systems Format (%s)" % type
# elif type in ['video/mpeg']:
# type_info = "MPEG video (%s)" % type
elif type in ['audio/mp4']:
type_info = "MPEG-4 audio (%s)" % type
elif type in ['audio/mpeg']:
type_info = "MP3 (%s)" % type
elif type in ['image/jpeg']:
type_info = "JPEG Image (%s)" % type
elif type in ['image/png']:
type_info = "Portable Network Graphics (%s)" % type
elif type in ['image/gif']:
type_info = "Graphics Interchange Format (%s)" % type
else:
type_info = "Unknown type (%s)" % type
maybe_write2buf("Site: ", site_info)
maybe_write2buf("Title: ", unescape_html(tr(title)))
write2buf("Type: ", type_info)
write2buf("Size: ", round(size / 1048576, 2),
"MiB (" + str(size) + " Bytes)")
write2buf()
def download_urls_chunked(urls,
title,
ext,
total_size,
output_dir='.',
refer=None,
merge=True,
faker=False,
headers={}):
assert urls
if dry_run:
write2buf('Real URLs:\n%s\n' % urls)
return
if player:
launch_player(player, urls)
return
title = tr(get_filename(title))
filename = '%s.%s' % (title, ext)
filepath = os.path.join(output_dir, filename)
if total_size and ext in ('ts'):
if not force and os.path.exists(filepath[:-3] + '.mkv'):
write2buf('Skipping %s: file already exists' % filepath[:-3] +
'.mkv')
write2buf()
set_exist(True)
return
bar = SimpleProgressBar(total_size, len(urls))
else:
bar = PiecesProgressBar(total_size, len(urls))
if len(urls) == 1:
parts = []
url = urls[0]
write2buf('Downloading %s ...' % tr(filename))
filepath = os.path.join(output_dir, filename)
parts.append(filepath)
url_save_chunked(url,
filepath,
bar,
refer=refer,
faker=faker,
headers=headers)
bar.done()
if not merge:
write2buf()
return
if ext == 'ts':
from .processor.ffmpeg import has_ffmpeg_installed
if has_ffmpeg_installed():
from .processor.ffmpeg import ffmpeg_convert_ts_to_mkv
if ffmpeg_convert_ts_to_mkv(
parts, os.path.join(output_dir, title + '.mkv')):
for part in parts:
os.remove(part)
else:
os.remove(os.path.join(output_dir, title + '.mkv'))
else:
write2buf('No ffmpeg is found. Conversion aborted.')
else:
write2buf("Can't convert %s files" % ext)
else:
parts = []
write2buf('Downloading %s.%s ...' % (tr(title), ext))
for i, url in enumerate(urls):
filename = '%s[%02d].%s' % (title, i, ext)
filepath = os.path.join(output_dir, filename)
parts.append(filepath)
# write2buf 'Downloading %s [%s/%s]...' % (tr(filename), i + 1, len(urls))
bar.update_piece(i + 1)
url_save_chunked(url,
filepath,
bar,
refer=refer,
is_part=True,
faker=faker,
headers=headers)
bar.done()
if not merge:
write2buf()
return
if ext == 'ts':
from .processor.ffmpeg import has_ffmpeg_installed
if has_ffmpeg_installed():
from .processor.ffmpeg import ffmpeg_concat_ts_to_mkv
if ffmpeg_concat_ts_to_mkv(
parts, os.path.join(output_dir, title + '.mkv')):
for part in parts:
os.remove(part)
else:
os.remove(os.path.join(output_dir, title + '.mkv'))
else:
write2buf('No ffmpeg is found. Merging aborted.')
else:
write2buf("Can't merge %s files" % ext)
write2buf()
def bilibili_download(url,
output_dir='.',
merge=True,
info_only=False,
**kwargs):
html = get_content(url)
if re.match(r'https?://bangumi\.bilibili\.com/', url):
# quick hack for bangumi URLs
url = r1(r'"([^"]+)" class="v-av-link"', html)
html = get_content(url)
title = r1_of([
r'<meta name="title" content="([^<>]{1,999})" />',
r'<h1[^>]*>([^<>]+)</h1>'
], html)
if title:
title = unescape_html(title)
title = escape_file_path(title)
flashvars = r1_of([
r'(cid=\d+)', r'(cid: \d+)', r'flashvars="([^"]+)"',
r'"https://[a-z]+\.bilibili\.com/secure,(cid=\d+)(?:&aid=\d+)?"'
], html)
assert flashvars
flashvars = flashvars.replace(': ', '=')
t, cid = flashvars.split('=', 1)
cid = cid.split('&')[0]
if t == 'cid':
if re.match(r'https?://live\.bilibili\.com/', url):
title = r1(r'<title>([^<>]+)</title>', html)
bilibili_live_download_by_cid(cid,
title,
output_dir=output_dir,
merge=merge,
info_only=info_only)
else:
# multi-P
cids = []
pages = re.findall('<option value=\'([^\']*)\'', html)
titles = re.findall('<option value=.*>(.+)</option>', html)
for i, page in enumerate(pages):
html = get_html("http://www.bilibili.com%s" % page)
flashvars = r1_of([
r'(cid=\d+)', r'flashvars="([^"]+)"',
r'"https://[a-z]+\.bilibili\.com/secure,(cid=\d+)(?:&aid=\d+)?"'
], html)
if flashvars:
t, cid = flashvars.split('=', 1)
cids.append(cid.split('&')[0])
if url.endswith(page):
cids = [cid.split('&')[0]]
titles = [titles[i]]
break
# no multi-P
if not pages:
cids = [cid]
titles = [
r1(r'<option value=.* selected>(.+)</option>', html)
or title
]
for i in range(len(cids)):
bilibili_download_by_cid(cids[i],
titles[i],
output_dir=output_dir,
merge=merge,
info_only=info_only)
elif t == 'vid':
sina_download_by_vid(cid,
title=title,
output_dir=output_dir,
merge=merge,
info_only=info_only)
elif t == 'ykid':
youku_download_by_vid(cid,
title=title,
output_dir=output_dir,
merge=merge,
info_only=info_only)
elif t == 'uid':
tudou_download_by_id(cid,
title,
output_dir=output_dir,
merge=merge,
info_only=info_only)
else:
raise NotImplementedError(flashvars)
if not info_only and not dry_run:
if not kwargs['caption']:
write2buf('Skipping danmaku.')
return
title = get_filename(title)
write2buf('Downloading %s ...\n' % (title + '.cmt.xml'))
xml = get_srt_xml(cid)
with open(os.path.join(output_dir, title + '.cmt.xml'),
'w',
encoding='utf-8') as x:
x.write(xml)
def download_urls(urls,
title,
ext,
total_size,
output_dir='.',
refer=None,
merge=True,
faker=False,
headers={},
**kwargs):
assert urls
if json_output:
json_output_.download_urls(urls=urls,
title=title,
ext=ext,
total_size=total_size,
refer=refer)
return
if dry_run:
write2buf('Real URLs:\n%s' % '\n'.join(urls))
return
if player:
launch_player(player, urls)
return
if not total_size:
try:
total_size = urls_size(urls, faker=faker, headers=headers)
except Exception as e:
mlog.exception(e)
title = tr(get_filename(title))
output_filename = get_output_filename(urls, title, ext, output_dir, merge)
output_filepath = os.path.join(output_dir, output_filename)
if total_size:
if not force and os.path.exists(output_filepath) and os.path.getsize(
output_filepath) >= total_size * 0.9:
write2buf('Skipping %s: file already exists' % output_filepath)
write2buf()
set_exist(True)
return
bar = SimpleProgressBar(total_size, len(urls))
else:
bar = PiecesProgressBar(total_size, len(urls))
if len(urls) == 1:
url = urls[0]
write2buf('Downloading %s ...' % tr(output_filename))
bar.update()
url_save(url,
output_filepath,
bar,
refer=refer,
faker=faker,
headers=headers)
bar.done()
else:
parts = []
write2buf('Downloading %s.%s ...' % (tr(title), ext))
bar.update()
for i, url in enumerate(urls):
filename = '%s[%02d].%s' % (title, i, ext)
filepath = os.path.join(output_dir, filename)
parts.append(filepath)
# write2buf 'Downloading %s [%s/%s]...' % (tr(filename), i + 1, len(urls))
bar.update_piece(i + 1)
url_save(url,
filepath,
bar,
refer=refer,
is_part=True,
faker=faker,
headers=headers)
bar.done()
if not merge:
write2buf()
return
if 'av' in kwargs and kwargs['av']:
from .processor.ffmpeg import has_ffmpeg_installed
if has_ffmpeg_installed():
from .processor.ffmpeg import ffmpeg_concat_av
ret = ffmpeg_concat_av(parts, output_filepath, ext)
write2buf('Merged into %s' % output_filename)
if ret == 0:
for part in parts:
os.remove(part)
elif ext in ['flv', 'f4v']:
try:
from .processor.ffmpeg import has_ffmpeg_installed
if has_ffmpeg_installed():
from .processor.ffmpeg import ffmpeg_concat_flv_to_mp4
ffmpeg_concat_flv_to_mp4(parts, output_filepath)
else:
from .processor.join_flv import concat_flv
concat_flv(parts, output_filepath)
write2buf('Merged into %s' % output_filename)
except:
raise
else:
for part in parts:
os.remove(part)
elif ext == 'mp4':
try:
from .processor.ffmpeg import has_ffmpeg_installed
if has_ffmpeg_installed():
from .processor.ffmpeg import ffmpeg_concat_mp4_to_mp4
ffmpeg_concat_mp4_to_mp4(parts, output_filepath)
else:
from .processor.join_mp4 import concat_mp4
concat_mp4(parts, output_filepath)
write2buf('Merged into %s' % output_filename)
except:
raise
else:
for part in parts:
os.remove(part)
elif ext == "ts":
try:
from .processor.ffmpeg import has_ffmpeg_installed
if has_ffmpeg_installed():
from .processor.ffmpeg import ffmpeg_concat_ts_to_mkv
ffmpeg_concat_ts_to_mkv(parts, output_filepath)
else:
from .processor.join_ts import concat_ts
concat_ts(parts, output_filepath)
write2buf('Merged into %s' % output_filename)
except:
raise
else:
for part in parts:
os.remove(part)
else:
write2buf("Can't merge %s files" % ext)
write2buf()
def usage():
write2buf('Usage: [python3] join_flv.py --output TARGET.flv flv...')
def done(self):
if self.displayed:
write2buf()
self.displayed = False
def url_save(url, filepath, bar, refer=None, is_part=False, faker=False, headers={}):
file_size = url_size(url, faker=faker, headers=headers)
if os.path.exists(filepath):
if not force and file_size == os.path.getsize(filepath):
if not is_part:
if bar:
bar.done()
write2buf('Skipping %s: file already exists' % tr(os.path.basename(filepath)))
set_exist(True)
else:
if bar:
bar.update_received(file_size)
return
else:
if not is_part:
if bar:
bar.done()
write2buf('Overwriting %s' % tr(os.path.basename(filepath)), '...')
elif not os.path.exists(os.path.dirname(filepath)):
os.mkdir(os.path.dirname(filepath))
temp_filepath = filepath + '.download' if file_size != float('inf') else filepath
received = 0
if not force:
open_mode = 'ab'
if os.path.exists(temp_filepath):
received += os.path.getsize(temp_filepath)
if bar:
bar.update_received(os.path.getsize(temp_filepath))
else:
open_mode = 'wb'
if received < file_size:
if faker:
headers = fake_headers
elif headers:
headers = headers
else:
headers = {}
if received:
headers['Range'] = 'bytes=' + str(received) + '-'
if refer:
headers['Referer'] = refer
response = request.urlopen(request.Request(url, headers=headers), None)
try:
range_start = int(response.headers['content-range'][6:].split('/')[0].split('-')[0])
end_length = end = int(response.headers['content-range'][6:].split('/')[1])
range_length = end_length - range_start
except:
content_length = response.headers['content-length']
range_length = int(content_length) if content_length != None else float('inf')
if file_size != received + range_length:
received = 0
if bar:
bar.received = 0
open_mode = 'wb'
with open(temp_filepath, open_mode) as output:
while True:
buffer = response.read(1024 * 256)
if get_stop_thread(): # user maybe force to stop the downloading
return
if not buffer:
if received == file_size: # Download finished
break
else: # Unexpected termination. Retry request
headers['Range'] = 'bytes=' + str(received) + '-'
response = request.urlopen(request.Request(url, headers=headers), None)
output.write(buffer)
received += len(buffer)
if bar:
bar.update_received(len(buffer))
assert received == os.path.getsize(temp_filepath), '%s == %s == %s' % (
received, os.path.getsize(temp_filepath), temp_filepath)
if os.access(filepath, os.W_OK):
os.remove(filepath) # on Windows rename could fail if destination filepath exists
os.rename(temp_filepath, filepath)
def url_save(url,
filepath,
bar,
refer=None,
is_part=False,
faker=False,
headers={}):
file_size = url_size(url, faker=faker, headers=headers)
if os.path.exists(filepath):
if not force and file_size == os.path.getsize(filepath):
if not is_part:
if bar:
bar.done()
write2buf('Skipping %s: file already exists' %
tr(os.path.basename(filepath)))
set_exist(True)
else:
if bar:
bar.update_received(file_size)
return
else:
if not is_part:
if bar:
bar.done()
write2buf('Overwriting %s' % tr(os.path.basename(filepath)),
'...')
elif not os.path.exists(os.path.dirname(filepath)):
os.mkdir(os.path.dirname(filepath))
temp_filepath = filepath + '.download' if file_size != float(
'inf') else filepath
received = 0
if not force:
open_mode = 'ab'
if os.path.exists(temp_filepath):
received += os.path.getsize(temp_filepath)
if bar:
bar.update_received(os.path.getsize(temp_filepath))
else:
open_mode = 'wb'
if received < file_size:
if faker:
headers = fake_headers
elif headers:
headers = headers
else:
headers = {}
if received:
headers['Range'] = 'bytes=' + str(received) + '-'
if refer:
headers['Referer'] = refer
response = request.urlopen(request.Request(url, headers=headers), None)
try:
range_start = int(response.headers['content-range'][6:].split('/')
[0].split('-')[0])
end_length = end = int(
response.headers['content-range'][6:].split('/')[1])
range_length = end_length - range_start
except:
content_length = response.headers['content-length']
range_length = int(
content_length) if content_length != None else float('inf')
if file_size != received + range_length:
received = 0
if bar:
bar.received = 0
open_mode = 'wb'
with open(temp_filepath, open_mode) as output:
while True:
buffer = response.read(1024 * 256)
if get_stop_thread(
): # user maybe force to stop the downloading
return
if not buffer:
if received == file_size: # Download finished
break
else: # Unexpected termination. Retry request
headers['Range'] = 'bytes=' + str(received) + '-'
response = request.urlopen(
request.Request(url, headers=headers), None)
output.write(buffer)
received += len(buffer)
if bar:
bar.update_received(len(buffer))
assert received == os.path.getsize(temp_filepath), '%s == %s == %s' % (
received, os.path.getsize(temp_filepath), temp_filepath)
if os.access(filepath, os.W_OK):
os.remove(
filepath
) # on Windows rename could fail if destination filepath exists
os.rename(temp_filepath, filepath)
def usage():
write2buf('Usage: [python3] join_mp4.py --output TARGET.mp4 mp4...')
def huaban_download(url, output_dir='.', **kwargs):
if re.match(r'http://huaban\.com/boards/\d+/', url):
huaban_download_board(url, output_dir, **kwargs)
else:
write2buf('Only board (画板) pages are supported currently')
write2buf('ex: http://huaban.com/boards/12345678/')
def download(self, **kwargs):
"""Override the original one
Ugly ugly dirty hack"""
if 'json_output' in kwargs and kwargs['json_output']:
json_output.output(self)
elif 'info_only' in kwargs and kwargs['info_only']:
if 'stream_id' in kwargs and kwargs['stream_id']:
# Display the stream
stream_id = kwargs['stream_id']
if 'index' not in kwargs:
self.p(stream_id)
else:
self.p_i(stream_id)
else:
# Display all available streams
if 'index' not in kwargs:
self.p([])
else:
stream_id = self.streams_sorted[0][
'id'] if 'id' in self.streams_sorted[
0] else self.streams_sorted[0]['itag']
self.p_i(stream_id)
else:
if 'stream_id' in kwargs and kwargs['stream_id']:
# Download the stream
stream_id = kwargs['stream_id']
else:
# Download stream with the best quality
stream_id = self.streams_sorted[0][
'id'] if 'id' in self.streams_sorted[
0] else self.streams_sorted[0]['itag']
if 'index' not in kwargs:
self.p(stream_id)
else:
self.p_i(stream_id)
if stream_id in self.streams:
urls = self.streams[stream_id]['src']
ext = self.streams[stream_id]['container']
total_size = self.streams[stream_id]['size']
else:
urls = self.dash_streams[stream_id]['src']
ext = self.dash_streams[stream_id]['container']
total_size = self.dash_streams[stream_id]['size']
if not urls:
log.wtf('[Failed] Cannot extract video source.')
# For legacy main()
#Here's the change!!
download_url_ffmpeg(
urls[0],
self.title,
'mp4',
output_dir=kwargs['output_dir'],
merge=kwargs['merge'],
)
if not kwargs['caption']:
write2buf('Skipping captions.')
return
for lang in self.caption_tracks:
filename = '%s.%s.srt' % (get_filename(self.title), lang)
write2buf('Saving %s ... ' % filename, end="", flush=True)
srt = self.caption_tracks[lang]
with open(os.path.join(kwargs['output_dir'], filename),
'w',
encoding='utf-8') as x:
x.write(srt)
write2buf('Done.')
