def _download2(self, files, audio=False):
cookies = self.kwargs["cookies"]
if audio:
file_d = output(copy.copy(self.options), "m4a")
else:
file_d = output(self.options, self.options.other)
if hasattr(file_d, "read") is False:
return
eta = ETA(len(files))
n = 1
for i in files:
if self.options.output != "-" and not self.options.silent:
eta.increment()
progressbar(len(files), n, ''.join(['ETA: ', str(eta)]))
n += 1
data = self.http.request("get", i, cookies=cookies)
if data.status_code == 404:
break
data = data.content
file_d.write(data)
if self.options.output != "-":
file_d.close()
if not self.options.silent:
progress_stream.write('\n')
self.finished = True
def download(self):
""" Get the stream from HTTP """
data = self.http.request("get", self.url, stream=True)
try:
total_size = data.headers['content-length']
except KeyError:
total_size = 0
total_size = int(total_size)
bytes_so_far = 0
file_d = output(self.options, "mp4")
if hasattr(file_d, "read") is False:
return
eta = ETA(total_size)
for i in data.iter_content(8192):
bytes_so_far += len(i)
file_d.write(i)
if self.options.output != "-" and not self.options.silent:
eta.update(bytes_so_far)
progressbar(total_size, bytes_so_far, ''.join(["ETA: ", str(eta)]))
if self.options.output != "-":
file_d.close()
self.finished = True
def download_hds(options, url, swf=None):
data = get_http_data(url)
streams = {}
bootstrap = {}
xml = ET.XML(data)
prefix = xml.find("{http://ns.adobe.com/f4m/1.0}id").text
if sys.version_info < (2, 7):
bootstrapIter = xml.getiterator("{http://ns.adobe.com/f4m/1.0}bootstrapInfo")
mediaIter = xml.getiterator("{http://ns.adobe.com/f4m/1.0}media")
else:
bootstrapIter = xml.iter("{http://ns.adobe.com/f4m/1.0}bootstrapInfo")
mediaIter = xml.iter("{http://ns.adobe.com/f4m/1.0}media")
for i in bootstrapIter:
bootstrap[i.attrib["id"]] = i.text
for i in mediaIter:
streams[int(i.attrib["bitrate"])] = {"url": i.attrib["url"], "bootstrapInfoId": i.attrib["bootstrapInfoId"], "metadata": i.find("{http://ns.adobe.com/f4m/1.0}metadata").text}
test = select_quality(options, streams)
bootstrap = base64.b64decode(bootstrap[test["bootstrapInfoId"]])
box = readboxtype(bootstrap, 0)
if box[2] == "abst":
antal = readbox(bootstrap, box[0])
baseurl = url[0:url.rfind("/")]
i = 1
if options.output != "-":
extension = re.search("(\.[a-z0-9]+)$", options.output)
if not extension:
options.output = "%s.flv" % options.output
log.info("Outfile: %s", options.output)
file_d = open(options.output, "wb")
else:
file_d = sys.stdout
metasize = struct.pack(">L", len(base64.b64decode(test["metadata"])))[1:]
file_d.write(binascii.a2b_hex(b"464c560105000000090000000012"))
file_d.write(metasize)
file_d.write(binascii.a2b_hex(b"00000000000000"))
file_d.write(base64.b64decode(test["metadata"]))
file_d.write(binascii.a2b_hex(b"00000000"))
total = antal[1]["total"]
eta = ETA(total)
while i <= total:
url = "%s/%sSeg1-Frag%s" % (baseurl, test["url"], i)
if options.output != "-":
eta.update(i)
progressbar(total, i, ''.join(["ETA: ", str(eta)]))
data = get_http_data(url)
number = decode_f4f(i, data)
file_d.write(data[number:])
i += 1
if options.output != "-":
file_d.close()
progress_stream.write('\n')
def download(self):
""" Get the stream from HTTP """
data = self.http.request("get", self.url, stream=True)
try:
total_size = data.headers['content-length']
except KeyError:
total_size = 0
total_size = int(total_size)
bytes_so_far = 0
file_d = output(self.options, "mp4")
if hasattr(file_d, "read") is False:
return
eta = ETA(total_size)
for i in data.iter_content(8192):
bytes_so_far += len(i)
file_d.write(i)
if self.options.output != "-" and not self.options.silent:
eta.update(bytes_so_far)
progressbar(total_size, bytes_so_far,
''.join(["ETA: ", str(eta)]))
if self.options.output != "-":
file_d.close()
self.finished = True
def _download2(self, files, audio=False):
cookies = self.kwargs["cookies"]
if audio:
file_d = output(copy.copy(self.options), "m4a")
else:
file_d = output(self.options, self.options.other)
if file_d is None:
return
eta = ETA(len(files))
n = 1
for i in files:
if not self.options.silent:
eta.increment()
progressbar(len(files), n, ''.join(['ETA: ', str(eta)]))
n += 1
data = self.http.request("get", i, cookies=cookies)
if data.status_code == 404:
break
data = data.content
file_d.write(data)
file_d.close()
if not self.options.silent:
progress_stream.write('\n')
self.finished = True
def download(self):
if self.options.live and not self.options.force:
raise LiveHLSException(self.url)
cookies = self.kwargs["cookies"]
m3u8 = self.http.request("get", self.url, cookies=cookies).text
globaldata, files = parsem3u(m3u8)
encrypted = False
key = None
if "KEY" in globaldata:
keydata = globaldata["KEY"]
encrypted = True
if encrypted:
try:
from Crypto.Cipher import AES
except ImportError:
log.error("You need to install pycrypto to download encrypted HLS streams")
sys.exit(2)
match = re.search(r'URI="(https?://.*?)"', keydata)
if not match:
match = re.search(r'URI="([^"]+)"', keydata)
keyurl = _get_full_url(match.group(1), self.url)
key = self.http.request("get", keyurl, cookies=cookies).content
rand = os.urandom(16)
decryptor = AES.new(key, AES.MODE_CBC, rand)
file_d = output(self.options, "ts")
if file_d is None:
return
n = 1
eta = ETA(len(files))
for i in files:
item = _get_full_url(i[0], self.url)
if not self.options.silent:
eta.increment()
progressbar(len(files), n, ''.join(['ETA: ', str(eta)]))
n += 1
data = self.http.request("get", item, cookies=cookies)
if data.status_code == 404:
break
data = data.content
if encrypted:
data = decryptor.decrypt(data)
file_d.write(data)
file_d.close()
if not self.options.silent:
progress_stream.write('\n')
self.finished = True
def download(self):
if self.options.live and not self.options.force:
raise LiveHLSException(self.url)
m3u8 = get_http_data(self.url)
globaldata, files = parsem3u(m3u8)
encrypted = False
key = None
try:
keydata = globaldata["KEY"]
encrypted = True
except KeyError:
pass
if encrypted:
try:
from Crypto.Cipher import AES
except ImportError:
log.error("You need to install pycrypto to download encrypted HLS streams")
sys.exit(2)
match = re.search(r'URI="(https?://.*?)"', keydata)
key = get_http_data(match.group(1))
rand = os.urandom(16)
decryptor = AES.new(key, AES.MODE_CBC, rand)
if self.options.output != "-":
extension = re.search(r"(\.[a-z0-9]+)$", self.options.output)
if not extension:
self.options.output = "%s.ts" % self.options.output
log.info("Outfile: %s", self.options.output)
if os.path.isfile(self.options.output) and not self.options.force:
log.info("File already exists. use --force to overwrite")
return
file_d = open(self.options.output, "wb")
else:
file_d = sys.stdout
n = 0
eta = ETA(len(files))
for i in files:
item = _get_full_url(i[0], self.url)
if self.options.output != "-":
eta.increment()
progressbar(len(files), n, ''.join(['ETA: ', str(eta)]))
n += 1
data = get_http_data(item)
if encrypted:
data = decryptor.decrypt(data)
file_d.write(data)
if self.options.output != "-":
file_d.close()
progress_stream.write('\n')
def download(self):
if self.options.live and not self.options.force:
raise LiveHLSException(self.url)
error, m3u8 = get_http_data(self.url)
if error:
log.error("Cant get m3u8 file.")
return
globaldata, files = parsem3u(m3u8)
encrypted = False
key = None
if "KEY" in globaldata:
keydata = globaldata["KEY"]
encrypted = True
if encrypted:
try:
from Crypto.Cipher import AES
except ImportError:
log.error("You need to install pycrypto to download encrypted HLS streams")
sys.exit(2)
match = re.search(r'URI="(https?://.*?)"', keydata)
error, key = get_http_data(match.group(1))
if error:
log.error("Can't get crypto key to decode files.")
return
rand = os.urandom(16)
decryptor = AES.new(key, AES.MODE_CBC, rand)
file_d = output(self.options, "ts")
if hasattr(file_d, "read") is False:
return
n = 1
eta = ETA(len(files))
for i in files:
item = _get_full_url(i[0], self.url)
if self.options.output != "-":
eta.increment()
progressbar(len(files), n, "".join(["ETA: ", str(eta)]))
n += 1
error, data = get_http_data(item)
if error:
log.error("Missing segment in playlist")
return
if encrypted:
data = decryptor.decrypt(data)
file_d.write(data)
if self.options.output != "-":
file_d.close()
progress_stream.write("\n")
def download(self):
if self.options.live and not self.options.force:
raise LiveHLSException(self.url)
cookies = self.kwargs["cookies"]
m3u8 = self.http.request("get", self.url, cookies=cookies).text
globaldata, files = parsem3u(m3u8)
encrypted = False
key = None
if "KEY" in globaldata:
keydata = globaldata["KEY"]
encrypted = True
if encrypted:
try:
from Crypto.Cipher import AES
except ImportError:
log.error(
"You need to install pycrypto to download encrypted HLS streams"
)
sys.exit(2)
match = re.search(r'URI="(https?://.*?)"', keydata)
key = self.http.request("get", match.group(1)).content
rand = os.urandom(16)
decryptor = AES.new(key, AES.MODE_CBC, rand)
file_d = output(self.options, "ts")
if hasattr(file_d, "read") is False:
return
n = 1
eta = ETA(len(files))
for i in files:
item = _get_full_url(i[0], self.url)
if self.options.output != "-" and not self.options.silent:
eta.increment()
progressbar(len(files), n, ''.join(['ETA: ', str(eta)]))
n += 1
data = self.http.request("get", item, cookies=cookies)
if data.status_code == 404:
break
data = data.content
if encrypted:
data = decryptor.decrypt(data)
file_d.write(data)
if self.options.output != "-":
file_d.close()
if not self.options.silent:
progress_stream.write('\n')
self.finished = True
def download(self):
if self.options.live and not self.options.force:
raise LiveHDSException(self.url)
querystring = self.kwargs["querystring"]
cookies = self.kwargs["cookies"]
bootstrap = base64.b64decode(self.kwargs["bootstrap"])
box = readboxtype(bootstrap, 0)
antal = None
if box[2] == b"abst":
antal = readbox(bootstrap, box[0])
baseurl = self.kwargs["manifest"][0:self.kwargs["manifest"].rfind("/")]
file_d = output(self.options, "flv")
if hasattr(file_d, "read") is False:
return
metasize = struct.pack(">L",
len(base64.b64decode(
self.kwargs["metadata"])))[1:]
file_d.write(binascii.a2b_hex(b"464c560105000000090000000012"))
file_d.write(metasize)
file_d.write(binascii.a2b_hex(b"00000000000000"))
file_d.write(base64.b64decode(self.kwargs["metadata"]))
file_d.write(binascii.a2b_hex(b"00000000"))
i = 1
start = antal[1]["first"]
total = antal[1]["total"]
eta = ETA(total)
while i <= total:
url = "%s/%sSeg1-Frag%s?%s" % (baseurl, self.url, start,
querystring)
if self.options.output != "-" and not self.options.silent:
eta.update(i)
progressbar(total, i, ''.join(["ETA: ", str(eta)]))
data = self.http.request("get", url, cookies=cookies)
if data.status_code == 404:
break
data = data.content
number = decode_f4f(i, data)
file_d.write(data[number:])
i += 1
start += 1
if self.options.output != "-":
file_d.close()
if not self.options.silent:
progress_stream.write('\n')
self.finished = True
def download(self):
if self.options.live and not self.options.force:
raise LiveHDSException(self.url)
querystring = self.kwargs["querystring"]
bootstrap = base64.b64decode(self.kwargs["bootstrap"])
box = readboxtype(bootstrap, 0)
antal = None
if box[2] == b"abst":
antal = readbox(bootstrap, box[0])
baseurl = self.kwargs["manifest"][0:self.kwargs["manifest"].rfind("/")]
if self.options.output != "-":
extension = re.search(r"(\.[a-z0-9]+)$", self.options.output)
if not extension:
self.options.output = "%s.flv" % self.options.output
log.info("Outfile: %s", self.options.output)
if os.path.isfile(self.options.output) and not self.options.force:
log.info("File already exists. use --force to overwrite")
return
file_d = open(self.options.output, "wb")
else:
file_d = sys.stdout
metasize = struct.pack(">L", len(base64.b64decode(self.kwargs["metadata"])))[1:]
file_d.write(binascii.a2b_hex(b"464c560105000000090000000012"))
file_d.write(metasize)
file_d.write(binascii.a2b_hex(b"00000000000000"))
file_d.write(base64.b64decode(self.kwargs["metadata"]))
file_d.write(binascii.a2b_hex(b"00000000"))
i = 1
start = antal[1]["first"]
total = antal[1]["total"]
eta = ETA(total)
while i <= total:
url = "%s/%sSeg1-Frag%s?%s" % (baseurl, self.url, start, querystring)
if self.options.output != "-":
eta.update(i)
progressbar(total, i, ''.join(["ETA: ", str(eta)]))
data = get_http_data(url)
number = decode_f4f(i, data)
file_d.write(data[number:])
i += 1
start += 1
if self.options.output != "-":
file_d.close()
progress_stream.write('\n')
def download(self):
if self.options.live and not self.options.force:
raise LiveHLSException(self.url)
m3u8 = get_http_data(self.url)
globaldata, files = parsem3u(m3u8)
encrypted = False
key = None
try:
keydata = globaldata["KEY"]
encrypted = True
except KeyError:
pass
if encrypted:
try:
from Crypto.Cipher import AES
except ImportError:
log.error("You need to install pycrypto to download encrypted HLS streams")
sys.exit(2)
match = re.search(r'URI="(https?://.*?)"', keydata)
key = get_http_data(match.group(1))
rand = os.urandom(16)
decryptor = AES.new(key, AES.MODE_CBC, rand)
file_d = output(self.options, self.options.output, "ts")
if hasattr(file_d, "read") is False:
return
n = 1
eta = ETA(len(files))
for i in files:
item = _get_full_url(i[0], self.url)
if self.options.output != "-":
eta.increment()
progressbar(len(files), n, ''.join(['ETA: ', str(eta)]))
n += 1
data = get_http_data(item)
if encrypted:
data = decryptor.decrypt(data)
file_d.write(data)
if self.options.output != "-":
file_d.close()
progress_stream.write('\n')
def download(self):
if self.options.live and not self.options.force:
raise LiveHDSException(self.url)
querystring = self.kwargs["querystring"]
cookies = self.kwargs["cookies"]
bootstrap = base64.b64decode(self.kwargs["bootstrap"])
box = readboxtype(bootstrap, 0)
antal = None
if box[2] == b"abst":
antal = readbox(bootstrap, box[0])
baseurl = self.kwargs["manifest"][0:self.kwargs["manifest"].rfind("/")]
file_d = output(self.options, "flv")
if hasattr(file_d, "read") is False:
return
metasize = struct.pack(">L", len(base64.b64decode(self.kwargs["metadata"])))[1:]
file_d.write(binascii.a2b_hex(b"464c560105000000090000000012"))
file_d.write(metasize)
file_d.write(binascii.a2b_hex(b"00000000000000"))
file_d.write(base64.b64decode(self.kwargs["metadata"]))
file_d.write(binascii.a2b_hex(b"00000000"))
i = 1
start = antal[1]["first"]
total = antal[1]["total"]
eta = ETA(total)
while i <= total:
url = "%s/%sSeg1-Frag%s?%s" % (baseurl, self.url, start, querystring)
if self.options.output != "-" and not self.options.silent:
eta.update(i)
progressbar(total, i, ''.join(["ETA: ", str(eta)]))
data = self.http.request("get", url, cookies=cookies)
if data.status_code == 404:
break
data = data.content
number = decode_f4f(i, data)
file_d.write(data[number:])
i += 1
start += 1
if self.options.output != "-":
file_d.close()
if not self.options.silent:
progress_stream.write('\n')
self.finished = True
def _download_url(self, url, audio=False, total_size=None):
cookies = self.kwargs["cookies"]
data = self.http.request("get",
url,
cookies=cookies,
headers={'Range': 'bytes=0-8192'})
if not total_size:
try:
total_size = data.headers['Content-Range']
total_size = total_size[total_size.find("/") + 1:]
total_size = int(total_size)
except KeyError:
raise KeyError("Can't get the total size.")
bytes_so_far = 8192
if audio:
file_d = output(copy.copy(self.options), "m4a")
else:
file_d = output(self.options, self.options.other)
if file_d is None:
return
file_d.write(data.content)
eta = ETA(total_size)
while bytes_so_far < total_size:
if not self.options.silent:
eta.update(bytes_so_far)
progressbar(total_size, bytes_so_far,
''.join(["ETA: ", str(eta)]))
old = bytes_so_far + 1
bytes_so_far = total_size
bytes_range = "bytes={0}-{1}".format(old, bytes_so_far)
data = self.http.request("get",
url,
cookies=cookies,
headers={'Range': bytes_range})
file_d.write(data.content)
file_d.close()
progressbar(bytes_so_far, total_size, "ETA: complete")
progress_stream.write('\n')
self.finished = True
def _download(self, url, audio=False):
cookies = self.kwargs["cookies"]
data = self.http.request("get",
url,
cookies=cookies,
headers={'Range': 'bytes=0-8192'})
try:
total_size = data.headers['Content-Range']
total_size = total_size[total_size.find("/") + 1:]
except KeyError:
total_size = 0
total_size = int(total_size)
bytes_so_far = 8192
if audio:
file_d = output(copy.copy(self.options), "m4a")
else:
file_d = output(self.options, self.options.other)
if hasattr(file_d, "read") is False:
return
file_d.write(data.content)
eta = ETA(total_size)
while bytes_so_far < total_size:
old = bytes_so_far + 1
bytes_so_far = old + 1000000
if bytes_so_far > total_size:
bytes_so_far = total_size
bytes_range = "bytes=%s-%s" % (old, bytes_so_far)
data = self.http.request("get",
url,
cookies=cookies,
headers={'Range': bytes_range})
file_d.write(data.content)
if self.options.output != "-" and not self.options.silent:
eta.update(old)
progressbar(total_size, old, ''.join(["ETA: ", str(eta)]))
if self.options.output != "-":
file_d.close()
progressbar(bytes_so_far, total_size, "ETA: complete")
progress_stream.write('\n')
self.finished = True
def download(self):
if self.options.live and not self.options.force:
raise LiveHDSException(self.url)
querystring = self.kwargs["querystring"]
bootstrap = base64.b64decode(self.kwargs["bootstrap"])
box = readboxtype(bootstrap, 0)
antal = None
if box[2] == b"abst":
antal = readbox(bootstrap, box[0])
baseurl = self.kwargs["manifest"][0:self.kwargs["manifest"].rfind("/")]
file_d = output(self.options, "flv")
if hasattr(file_d, "read") is False:
return
metasize = struct.pack(">L", len(base64.b64decode(self.kwargs["metadata"])))[1:]
file_d.write(binascii.a2b_hex(b"464c560105000000090000000012"))
file_d.write(metasize)
file_d.write(binascii.a2b_hex(b"00000000000000"))
file_d.write(base64.b64decode(self.kwargs["metadata"]))
file_d.write(binascii.a2b_hex(b"00000000"))
i = 1
start = antal[1]["first"]
total = antal[1]["total"]
eta = ETA(total)
while i <= total:
url = "%s/%sSeg1-Frag%s?%s" % (baseurl, self.url, start, querystring)
if self.options.output != "-":
eta.update(i)
progressbar(total, i, ''.join(["ETA: ", str(eta)]))
error, data = get_http_data(url)
if error:
log.error("Missing segment in playlist")
return
number = decode_f4f(i, data)
file_d.write(data[number:])
i += 1
start += 1
if self.options.output != "-":
file_d.close()
progress_stream.write('\n')
def _download_url(self, url, audio=False, total_size=None):
cookies = self.kwargs["cookies"]
data = self.http.request("get", url, cookies=cookies, headers={'Range': 'bytes=0-8192'})
if not total_size:
try:
total_size = data.headers['Content-Range']
total_size = total_size[total_size.find("/") + 1:]
total_size = int(total_size)
except KeyError:
raise KeyError("Can't get the total size.")
bytes_so_far = 8192
if audio:
file_d = output(copy.copy(self.options), "m4a")
else:
file_d = output(self.options, self.options.other)
if file_d is None:
return
file_d.write(data.content)
eta = ETA(total_size)
while bytes_so_far < total_size:
if not self.options.silent:
eta.update(bytes_so_far)
progressbar(total_size, bytes_so_far, ''.join(["ETA: ", str(eta)]))
old = bytes_so_far + 1
bytes_so_far = total_size
bytes_range = "bytes={0}-{1}".format(old, bytes_so_far)
data = self.http.request("get", url, cookies=cookies, headers={'Range': bytes_range})
file_d.write(data.content)
file_d.close()
progressbar(bytes_so_far, total_size, "ETA: complete")
progress_stream.write('\n')
self.finished = True
def _download(self, url, audio=False):
cookies = self.kwargs["cookies"]
data = self.http.request(
"get", url, cookies=cookies, headers={'Range': 'bytes=0-8192'})
try:
total_size = data.headers['Content-Range']
total_size = total_size[total_size.find("/") + 1:]
except KeyError:
total_size = 0
total_size = int(total_size)
bytes_so_far = 8192
if audio:
file_d = output(copy.copy(self.options), "m4a")
else:
file_d = output(self.options, self.options.other)
if hasattr(file_d, "read") is False:
return
file_d.write(data.content)
eta = ETA(total_size)
while bytes_so_far < total_size:
old = bytes_so_far + 1
bytes_so_far = old + 1000000
if bytes_so_far > total_size:
bytes_so_far = total_size
bytes_range = "bytes=%s-%s" % (old, bytes_so_far)
data = self.http.request(
"get", url, cookies=cookies, headers={'Range': bytes_range})
file_d.write(data.content)
if self.options.output != "-" and not self.options.silent:
eta.update(old)
progressbar(total_size, old, ''.join(["ETA: ", str(eta)]))
if self.options.output != "-":
file_d.close()
progress_stream.write('\n')
self.finished = True
def _download(self, url, file_name):
cookies = self.kwargs["cookies"]
start_time = time.time()
m3u8 = M3U8(self.http.request("get", url, cookies=cookies).text)
key = None
if m3u8.encrypted:
try:
from Crypto.Cipher import AES
from Crypto import Random
except ImportError:
log.error(
"You need to install pycrypto to download encrypted HLS streams"
)
sys.exit(2)
file_d = output(file_name[0], file_name[1])
if file_d is None:
return
decryptor = None
size_media = len(m3u8.media_segment)
eta = ETA(size_media)
total_duration = 0
duration = 0
max_duration = 0
for index, i in enumerate(m3u8.media_segment):
if "duration" in i["EXTINF"]:
duration = i["EXTINF"]["duration"]
max_duration = max(max_duration, duration)
total_duration += duration
item = _get_full_url(i["URI"], url)
if not self.options.silent:
if self.options.live:
progressbar(
size_media, index + 1, ''.join([
'DU: ',
str(timedelta(seconds=int(total_duration)))
]))
else:
eta.increment()
progressbar(size_media, index + 1,
''.join(['ETA: ', str(eta)]))
data = self.http.request("get", item, cookies=cookies)
if data.status_code == 404:
break
data = data.content
if m3u8.encrypted:
headers = {}
if self.keycookie:
keycookies = self.keycookie
else:
keycookies = cookies
if self.authorization:
headers["authorization"] = self.authorization
# Update key/decryptor
if "EXT-X-KEY" in i:
keyurl = _get_full_url(i["EXT-X-KEY"]["URI"], url)
key = self.http.request("get",
keyurl,
cookies=keycookies,
headers=headers).content
vi = binascii.unhexlify(i["EXT-X-KEY"]["IV"][2:].zfill(
32)) if "IV" in i["EXT-X-KEY"] else Random.new().read(
AES.block_size)
decryptor = AES.new(key, AES.MODE_CBC, vi)
if decryptor:
data = decryptor.decrypt(data)
else:
raise ValueError(
"No decryptor found for encrypted hls steam.")
file_d.write(data)
if (self.options.capture_time >
0) and total_duration >= (self.options.capture_time * 60):
break
if (size_media == (index + 1)) and self.options.live:
sleep_int = (start_time + max_duration * 2) - time.time()
if sleep_int > 0:
time.sleep(sleep_int)
size_media_old = size_media
while size_media_old == size_media:
start_time = time.time()
if self.options.hls_time_stamp:
end_time_stamp = (
datetime.utcnow() -
timedelta(seconds=max_duration * 2)).replace(
microsecond=0)
start_time_stamp = end_time_stamp - timedelta(
minutes=1)
base_url = url.split(".m3u8")[0]
url = "{0}.m3u8?in={1}&out={2}?".format(
base_url, start_time_stamp.isoformat(),
end_time_stamp.isoformat())
new_m3u8 = M3U8(
self.http.request("get", url, cookies=cookies).text)
for n_m3u in new_m3u8.media_segment:
if not any(d["URI"] == n_m3u["URI"]
for d in m3u8.media_segment):
m3u8.media_segment.append(n_m3u)
size_media = len(m3u8.media_segment)
if size_media_old == size_media:
time.sleep(max_duration)
file_d.close()
if not self.options.silent:
progress_stream.write('\n')
self.finished = True
def _download(self, url, file_name):
cookies = self.kwargs["cookies"]
start_time = time.time()
m3u8 = M3U8(self.http.request("get", url, cookies=cookies).text)
key = None
if m3u8.encrypted:
try:
from Crypto.Cipher import AES
from Crypto import Random
except ImportError:
log.error("You need to install pycrypto to download encrypted HLS streams")
sys.exit(2)
file_d = output(file_name[0], file_name[1])
if file_d is None:
return
decryptor = None
size_media = len(m3u8.media_segment)
eta = ETA(size_media)
total_duration = 0
duration = 0
max_duration = 0
for index, i in enumerate(m3u8.media_segment):
if "duration" in i["EXTINF"]:
duration = i["EXTINF"]["duration"]
max_duration = max(max_duration, duration)
total_duration += duration
item = _get_full_url(i["URI"], url)
if not self.options.silent:
if self.options.live:
progressbar(size_media, index + 1, ''.join(['DU: ', str(timedelta(seconds=int(total_duration)))]))
else:
eta.increment()
progressbar(size_media, index + 1, ''.join(['ETA: ', str(eta)]))
data = self.http.request("get", item, cookies=cookies)
if data.status_code == 404:
break
data = data.content
if m3u8.encrypted:
headers = {}
if self.keycookie:
keycookies = self.keycookie
else:
keycookies = cookies
if self.authorization:
headers["authorization"] = self.authorization
# Update key/decryptor
if "EXT-X-KEY" in i:
keyurl = _get_full_url(i["EXT-X-KEY"]["URI"], url)
key = self.http.request("get", keyurl, cookies=keycookies, headers=headers).content
vi = binascii.unhexlify(i["EXT-X-KEY"]["IV"][2:].zfill(32)) if "IV" in i["EXT-X-KEY"] else Random.new().read(AES.block_size)
decryptor = AES.new(key, AES.MODE_CBC, vi)
if decryptor:
data = decryptor.decrypt(data)
else:
raise ValueError("No decryptor found for encrypted hls steam.")
file_d.write(data)
if (self.options.capture_time > 0) and total_duration >= (self.options.capture_time * 60):
break
if (size_media == (index + 1)) and self.options.live:
sleep_int = (start_time + max_duration * 2) - time.time()
if sleep_int > 0:
time.sleep(sleep_int)
size_media_old = size_media
while size_media_old == size_media:
start_time = time.time()
if self.options.hls_time_stamp:
end_time_stamp = (datetime.utcnow() - timedelta(seconds=max_duration * 2)).replace(microsecond=0)
start_time_stamp = end_time_stamp - timedelta(minutes=1)
base_url = url.split(".m3u8")[0]
url = "{0}.m3u8?in={1}&out={2}?".format(base_url, start_time_stamp.isoformat(), end_time_stamp.isoformat())
new_m3u8 = M3U8(self.http.request("get", url, cookies=cookies).text)
for n_m3u in new_m3u8.media_segment:
if not any(d["URI"] == n_m3u["URI"] for d in m3u8.media_segment):
m3u8.media_segment.append(n_m3u)
size_media = len(m3u8.media_segment)
if size_media_old == size_media:
time.sleep(max_duration)
file_d.close()
if not self.options.silent:
progress_stream.write('\n')
self.finished = True
def download_hls(options, url, baseurl=None):
data = get_http_data(url)
globaldata, files = parsem3u(data)
streams = {}
for i in files:
streams[int(i[1]["BANDWIDTH"])] = i[0]
test = select_quality(options, streams)
if baseurl and test[:4] != 'http':
test = "%s/%s" % (baseurl, test)
m3u8 = get_http_data(test)
globaldata, files = parsem3u(m3u8)
encrypted = False
key = None
try:
keydata = globaldata["KEY"]
encrypted = True
except:
pass
if encrypted:
try:
from Crypto.Cipher import AES
except ImportError:
log.error("You need to install pycrypto to download encrypted HLS streams")
sys.exit(2)
match = re.search("URI=\"(http://.*)\"", keydata)
key = get_http_data(match.group(1))
rand = os.urandom(16)
decryptor = AES.new(key, AES.MODE_CBC, rand)
n = 1
if options.output != "-":
extension = re.search("(\.[a-z0-9]+)$", options.output)
if not extension:
options.output = "%s.ts" % options.output
log.info("Outfile: %s", options.output)
file_d = open(options.output, "wb")
else:
file_d = sys.stdout
start = time.time()
estimated = ""
for i in files:
item = i[0]
if options.output != "-":
progressbar(len(files), n, estimated)
if item[0:5] != "http:":
item = "%s/%s" % (baseurl, item)
data = get_http_data(item)
if encrypted:
lots = StringIO(data)
plain = b""
crypt = lots.read(1024)
decrypted = decryptor.decrypt(crypt)
while decrypted:
plain += decrypted
crypt = lots.read(1024)
decrypted = decryptor.decrypt(crypt)
data = plain
file_d.write(data)
now = time.time()
dt = now - start
et = dt / (n + 1) * len(files)
rt = et - dt
td = timedelta(seconds = int(rt))
estimated = "Estimated Remaining: " + str(td)
n += 1
if options.output != "-":
file_d.close()
progress_stream.write('\n')
def _download2(self, files, audio=False):
cookies = self.kwargs["cookies"]
if audio:
file_d = output(copy.copy(self.options), "m4a")
else:
file_d = output(self.options, self.options.other)
if hasattr(file_d, "read") is False:
return
if file_d == None:
return
eta = ETA(len(files))
n = 1
# check if we can resume a past download
already_downloaded = os.fstat(file_d.fileno()).st_size
if self.options.resume and already_downloaded != 0:
print('Checking for existing segments...')
bytes_so_far = 0
resumed = False
data_size = 0
for i in files:
if self.options.output != "-" and not self.options.silent:
eta.increment()
progressbar(len(files), n, ''.join(['ETA: ', str(eta)]))
n += 1
# get segment size
if self.options.resume and already_downloaded != 0:
if bytes_so_far < already_downloaded:
head_req = self.http.request("head", i, cookies=cookies)
if head_req.status_code == 404:
break
data_size = int(head_req.headers['Content-Length'])
bytes_so_far += data_size
log.debug('Segment %d', n-1)
log.debug('So far: %d, segment size: %d, on disk: %d', bytes_so_far - data_size, data_size, already_downloaded)
if bytes_so_far >= already_downloaded:
if not resumed and self.options.resume:
print
print('Resuming download on segment ' + str(n-1))
log.debug('Truncating output file to %d', bytes_so_far - data_size)
file_d.truncate(bytes_so_far - data_size) # line up the file with a segment boundary
eta = ETA(len(files) - n)
resumed = True
data = self.http.request("get", i, cookies=cookies)
if data.status_code == 404:
break
data = data.content
file_d.write(data)
else:
data = self.http.request("get", i, cookies=cookies)
if data.status_code == 404:
break
data = data.content
file_d.write(data)
if self.options.output != "-":
file_d.close()
if not self.options.silent:
progress_stream.write('\n')
self.finished = True