def _download2(self, files, audio=False):
cookies = self.kwargs["cookies"]
if audio:
file_d = output(copy.copy(self.output),
self.config,
extension="m4a")
else:
file_d = output(self.output, self.config, extension="mp4")
if file_d is None:
return
eta = ETA(len(files))
n = 1
for i in files:
if not self.config.get("silent"):
eta.increment()
progressbar(len(files), n, "".join(["ETA: ", str(eta)]))
n += 1
data = self.http.request("get", i, cookies=cookies)
if data.status_code == 404:
break
data = data.content
file_d.write(data)
file_d.close()
if not self.config.get("silent"):
progress_stream.write("\n")
self.finished = True
def _download2(self, files, audio=False):
cookies = self.kwargs["cookies"]
if audio:
file_d = output(copy.copy(self.output), self.config, extension="m4a")
else:
file_d = output(self.output, self.config, extension="mp4")
if file_d is None:
return
eta = ETA(len(files))
n = 1
for i in files:
if not self.config.get("silent"):
eta.increment()
progressbar(len(files), n, ''.join(['ETA: ', str(eta)]))
n += 1
data = self.http.request("get", i, cookies=cookies)
if data.status_code == 404:
break
data = data.content
file_d.write(data)
file_d.close()
if not self.config.get("silent"):
progress_stream.write('\n')
self.finished = True
def save_file(self, data, subtype):
if platform.system() == "Windows":
file_d = output(self.output, self.config, subtype, mode="wt", encoding="utf-8")
else:
file_d = output(self.output, self.config, subtype, mode="wt")
if hasattr(file_d, "read") is False:
return
file_d.write(data)
file_d.close()
def save_file(self, data, subtype):
if platform.system() == "Windows":
file_d = output(self.output,
self.config,
subtype,
mode="wt",
encoding="utf-8")
else:
file_d = output(self.output, self.config, subtype, mode="wt")
if hasattr(file_d, "read") is False:
return
file_d.write(data)
file_d.close()
def download(self):
""" Get the stream from HTTP """
self.output_extention = "mp4" # this might be wrong..
data = self.http.request("get", self.url, stream=True)
try:
total_size = data.headers['content-length']
except KeyError:
total_size = 0
total_size = int(total_size)
bytes_so_far = 0
file_d = output(self.output, self.config, "mp4")
if file_d is None:
return
eta = ETA(total_size)
for i in data.iter_content(8192):
bytes_so_far += len(i)
file_d.write(i)
if not self.config.get("silent"):
eta.update(bytes_so_far)
progressbar(total_size, bytes_so_far, ''.join(["ETA: ", str(eta)]))
file_d.close()
self.finished = True
def download(self):
""" Get the stream from HTTP """
_, ext = os.path.splitext(self.url)
if ext == ".mp3":
self.output_extention = "mp3"
else:
self.output_extention = "mp4" # this might be wrong..
data = self.http.request("get", self.url, stream=True)
try:
total_size = data.headers["content-length"]
except KeyError:
total_size = 0
total_size = int(total_size)
bytes_so_far = 0
file_d = output(self.output, self.config, self.output_extention)
if file_d is None:
return
eta = ETA(total_size)
for i in data.iter_content(8192):
bytes_so_far += len(i)
file_d.write(i)
if not self.config.get("silent"):
eta.update(bytes_so_far)
progressbar(total_size, bytes_so_far,
"".join(["ETA: ", str(eta)]))
file_d.close()
self.finished = True
def download(self):
""" Get the stream from RTMP """
self.output_extention = "flv"
args = []
if self.config.get("live"):
args.append("-v")
if self.config.get("resume"):
args.append("-e")
file_d = output(self.output, self.config, "flv", False)
if file_d is None:
return
args += ["-o", formatname(self.output, self.config, "flv")]
if self.config.get("silent"):
args.append("-q")
if self.kwargs.get("other"):
args += shlex.split(self.kwargs.pop("other"))
if self.config.get("verbose"):
args.append("-V")
command = ["rtmpdump", "-r", self.url] + args
log.debug("Running: {0}".format(" ".join(command)))
try:
subprocess.call(command)
except OSError as e:
log.error("Could not execute rtmpdump: {0}".format(e.strerror))
return
self.finished = True
def _download_url(self, url, audio=False, total_size=None):
cookies = self.kwargs["cookies"]
data = self.http.request("get",
url,
cookies=cookies,
headers={'Range': 'bytes=0-8192'})
if not total_size:
try:
total_size = data.headers['Content-Range']
total_size = total_size[total_size.find("/") + 1:]
total_size = int(total_size)
except KeyError:
raise KeyError("Can't get the total size.")
bytes_so_far = 8192
if audio:
file_d = output(copy.copy(self.output), self.config, "m4a")
else:
file_d = output(self.output, self.config, "mp4")
if file_d is None:
return
file_d.write(data.content)
eta = ETA(total_size)
while bytes_so_far < total_size:
if not self.config.get("silent"):
eta.update(bytes_so_far)
progressbar(total_size, bytes_so_far,
''.join(["ETA: ", str(eta)]))
old = bytes_so_far + 1
bytes_so_far = total_size
bytes_range = "bytes={0}-{1}".format(old, bytes_so_far)
data = self.http.request("get",
url,
cookies=cookies,
headers={'Range': bytes_range})
file_d.write(data.content)
file_d.close()
progressbar(bytes_so_far, total_size, "ETA: complete")
# progress_stream.write('\n')
self.finished = True
def save_file(self, data, subtype):
file_d = output(self.output,
self.config,
subtype,
mode="w",
encoding="utf-8")
if hasattr(file_d, "read") is False:
return
file_d.write(data)
file_d.close()
def _download_url(self, url, audio=False, total_size=None):
cookies = self.kwargs["cookies"]
data = self.http.request("get", url, cookies=cookies, headers={'Range': 'bytes=0-8192'})
if not total_size:
try:
total_size = data.headers['Content-Range']
total_size = total_size[total_size.find("/") + 1:]
total_size = int(total_size)
except KeyError:
raise KeyError("Can't get the total size.")
bytes_so_far = 8192
if audio:
file_d = output(copy.copy(self.output), self.config, "m4a")
else:
file_d = output(self.output, self.config, "mp4")
if file_d is None:
return
file_d.write(data.content)
eta = ETA(total_size)
while bytes_so_far < total_size:
if not self.config.get("silent"):
eta.update(bytes_so_far)
progressbar(total_size, bytes_so_far, ''.join(["ETA: ", str(eta)]))
old = bytes_so_far + 1
bytes_so_far = total_size
bytes_range = "bytes={0}-{1}".format(old, bytes_so_far)
data = self.http.request("get", url, cookies=cookies, headers={'Range': bytes_range})
file_d.write(data.content)
file_d.close()
progressbar(bytes_so_far, total_size, "ETA: complete")
# progress_stream.write('\n')
self.finished = True
def download(self):
self.output_extention = "flv"
if self.config.get("live") and not self.config.get("force"):
raise LiveHDSException(self.url)
querystring = self.kwargs["querystring"]
cookies = self.kwargs["cookies"]
bootstrap = base64.b64decode(self.kwargs["bootstrap"])
box = readboxtype(bootstrap, 0)
antal = None
if box[2] == b"abst":
antal = readbox(bootstrap, box[0])
baseurl = self.url[0:self.url.rfind("/")]
file_d = output(self.output, self.config, "flv")
if file_d is None:
return
metasize = struct.pack(">L",
len(base64.b64decode(
self.kwargs["metadata"])))[1:]
file_d.write(binascii.a2b_hex(b"464c560105000000090000000012"))
file_d.write(metasize)
file_d.write(binascii.a2b_hex(b"00000000000000"))
file_d.write(base64.b64decode(self.kwargs["metadata"]))
file_d.write(binascii.a2b_hex(b"00000000"))
i = 1
start = antal[1]["first"]
total = antal[1]["total"]
eta = ETA(total)
while i <= total:
url = "{0}/{1}Seg1-Frag{2}?{3}".format(baseurl,
self.kwargs["url_id"],
start, querystring)
if not self.config.get("silent"):
eta.update(i)
progressbar(total, i, ''.join(["ETA: ", str(eta)]))
data = self.http.request("get", url, cookies=cookies)
if data.status_code == 404:
break
data = data.content
number = decode_f4f(i, data)
file_d.write(data[number:])
i += 1
start += 1
file_d.close()
if not self.config.get("silent"):
progress_stream.write('\n')
self.finished = True
def download(self):
self.output_extention = "flv"
if self.config.get("live") and not self.config.get("force"):
raise LiveHDSException(self.url)
querystring = self.kwargs["querystring"]
cookies = self.kwargs["cookies"]
bootstrap = base64.b64decode(self.kwargs["bootstrap"])
box = readboxtype(bootstrap, 0)
antal = None
if box[2] == b"abst":
antal = readbox(bootstrap, box[0])
baseurl = self.url[0:self.url.rfind("/")]
file_d = output(self.output, self.config, "flv")
if file_d is None:
return
metasize = struct.pack(">L", len(base64.b64decode(self.kwargs["metadata"])))[1:]
file_d.write(binascii.a2b_hex(b"464c560105000000090000000012"))
file_d.write(metasize)
file_d.write(binascii.a2b_hex(b"00000000000000"))
file_d.write(base64.b64decode(self.kwargs["metadata"]))
file_d.write(binascii.a2b_hex(b"00000000"))
i = 1
start = antal[1]["first"]
total = antal[1]["total"]
eta = ETA(total)
while i <= total:
url = "{0}/{1}Seg1-Frag{2}?{3}".format(baseurl, self.kwargs["url_id"], start, querystring)
if not self.config.get("silent"):
eta.update(i)
progressbar(total, i, ''.join(["ETA: ", str(eta)]))
data = self.http.request("get", url, cookies=cookies)
if data.status_code == 404:
break
data = data.content
number = decode_f4f(i, data)
file_d.write(data[number:])
i += 1
start += 1
file_d.close()
if not self.config.get("silent"):
progress_stream.write('\n')
self.finished = True
def _download(self, url, file_name):
cookies = self.kwargs.get("cookies", None)
start_time = time.time()
m3u8 = M3U8(self.http.request("get", url, cookies=cookies).text)
key = None
if m3u8.encrypted:
from Crypto.Cipher import AES
def random_iv():
try:
from Crypto import Random
return Random.new().read(AES.block_size)
except ImportError:
return os.urandom(16)
file_d = output(file_name[0], self.config, file_name[1])
if file_d is None:
return
hls_time_stamp = self.kwargs.pop("hls_time_stamp", False)
decryptor = None
size_media = len(m3u8.media_segment)
eta = ETA(size_media)
total_duration = 0
duration = 0
max_duration = 0
for index, i in enumerate(m3u8.media_segment):
if "duration" in i["EXTINF"]:
duration = i["EXTINF"]["duration"]
max_duration = max(max_duration, duration)
total_duration += duration
item = get_full_url(i["URI"], url)
if not self.config.get("silent"):
if self.config.get("live"):
progressbar(
size_media, index + 1, ''.join([
'DU: ',
str(timedelta(seconds=int(total_duration)))
]))
else:
eta.increment()
progressbar(size_media, index + 1,
''.join(['ETA: ', str(eta)]))
data = self.http.request("get", item, cookies=cookies)
if data.status_code == 404:
break
data = data.content
if m3u8.encrypted:
headers = {}
if self.keycookie:
keycookies = self.keycookie
else:
keycookies = cookies
if self.authorization:
headers["authorization"] = self.authorization
# Update key/decryptor
if "EXT-X-KEY" in i:
keyurl = get_full_url(i["EXT-X-KEY"]["URI"], url)
key = self.http.request("get",
keyurl,
cookies=keycookies,
headers=headers).content
iv = binascii.unhexlify(i["EXT-X-KEY"]["IV"][2:].zfill(
32)) if "IV" in i["EXT-X-KEY"] else random_iv()
decryptor = AES.new(key, AES.MODE_CBC, iv)
if decryptor:
data = decryptor.decrypt(data)
else:
raise ValueError(
"No decryptor found for encrypted hls steam.")
file_d.write(data)
if self.config.get(
"capture_time"
) > 0 and total_duration >= self.config.get("capture_time") * 60:
break
if (size_media == (index + 1)) and self.config.get("live"):
sleep_int = (start_time + max_duration * 2) - time.time()
if sleep_int > 0:
time.sleep(sleep_int)
size_media_old = size_media
while size_media_old == size_media:
start_time = time.time()
if hls_time_stamp:
end_time_stamp = (datetime.utcnow() - timedelta(
minutes=1, seconds=max_duration * 2)).replace(
microsecond=0)
start_time_stamp = end_time_stamp - timedelta(
minutes=1)
base_url = url.split(".m3u8")[0]
url = "{0}.m3u8?in={1}&out={2}?".format(
base_url, start_time_stamp.isoformat(),
end_time_stamp.isoformat())
new_m3u8 = M3U8(
self.http.request("get", url, cookies=cookies).text)
for n_m3u in new_m3u8.media_segment:
if not any(d["URI"] == n_m3u["URI"]
for d in m3u8.media_segment):
m3u8.media_segment.append(n_m3u)
size_media = len(m3u8.media_segment)
if size_media_old == size_media:
time.sleep(max_duration)
file_d.close()
if not self.config.get("silent"):
progress_stream.write('\n')
self.finished = True
def _download(self, url, file_name):
cookies = self.kwargs.get("cookies", None)
start_time = time.time()
m3u8 = M3U8(self.http.request("get", url, cookies=cookies).text)
key = None
def random_iv():
return os.urandom(16)
file_d = output(file_name[0], self.config, file_name[1])
if file_d is None:
return
hls_time_stamp = self.kwargs.pop("hls_time_stamp", False)
decryptor = None
size_media = len(m3u8.media_segment)
eta = ETA(size_media)
total_duration = 0
duration = 0
max_duration = 0
for index, i in enumerate(m3u8.media_segment):
if "duration" in i["EXTINF"]:
duration = i["EXTINF"]["duration"]
max_duration = max(max_duration, duration)
total_duration += duration
item = get_full_url(i["URI"], url)
if not self.config.get("silent"):
if self.config.get("live"):
progressbar(size_media, index + 1, ''.join(['DU: ', str(timedelta(seconds=int(total_duration)))]))
else:
eta.increment()
progressbar(size_media, index + 1, ''.join(['ETA: ', str(eta)]))
data = self.http.request("get", item, cookies=cookies)
if data.status_code == 404:
break
data = data.content
if m3u8.encrypted:
headers = {}
if self.keycookie:
keycookies = self.keycookie
else:
keycookies = cookies
if self.authorization:
headers["authorization"] = self.authorization
# Update key/decryptor
if "EXT-X-KEY" in i:
keyurl = get_full_url(i["EXT-X-KEY"]["URI"], url)
if keyurl and keyurl[:4] == "skd:":
raise HLSException(keyurl, "Can't decrypt beacuse of DRM")
key = self.http.request("get", keyurl, cookies=keycookies, headers=headers).content
iv = binascii.unhexlify(i["EXT-X-KEY"]["IV"][2:].zfill(32)) if "IV" in i["EXT-X-KEY"] else random_iv()
backend = default_backend()
cipher = Cipher(algorithms.AES(key), modes.CBC(iv), backend=backend)
decryptor = cipher.decryptor()
if decryptor:
data = decryptor.update(data)
else:
raise ValueError("No decryptor found for encrypted hls steam.")
file_d.write(data)
if self.config.get("capture_time") > 0 and total_duration >= self.config.get("capture_time") * 60:
break
if (size_media == (index + 1)) and self.config.get("live"):
sleep_int = (start_time + max_duration * 2) - time.time()
if sleep_int > 0:
time.sleep(sleep_int)
size_media_old = size_media
while size_media_old == size_media:
start_time = time.time()
if hls_time_stamp:
end_time_stamp = (datetime.utcnow() - timedelta(minutes=1,
seconds=max_duration * 2)).replace(microsecond=0)
start_time_stamp = end_time_stamp - timedelta(minutes=1)
base_url = url.split(".m3u8")[0]
url = "{0}.m3u8?in={1}&out={2}?".format(base_url, start_time_stamp.isoformat(), end_time_stamp.isoformat())
new_m3u8 = M3U8(self.http.request("get", url, cookies=cookies).text)
for n_m3u in new_m3u8.media_segment:
if not any(d["URI"] == n_m3u["URI"] for d in m3u8.media_segment):
m3u8.media_segment.append(n_m3u)
size_media = len(m3u8.media_segment)
if size_media_old == size_media:
time.sleep(max_duration)
file_d.close()
if not self.config.get("silent"):
progress_stream.write('\n')
self.finished = True
def _download(self, url, file_name):
cookies = self.kwargs.get("cookies", None)
start_time = time.time()
m3u8 = M3U8(self.http.request("get", url, cookies=cookies).text)
key = None
def random_iv():
return os.urandom(16)
file_d = output(file_name[0], self.config, file_name[1])
if file_d is None:
return
if "EXT-X-MAP" in m3u8.media_segment[0]:
entry = {
"URI": get_full_url(m3u8.media_segment[0]["EXT-X-MAP"]["URI"],
url),
"EXTINF": {
"duration": 0
}
}
if "EXT-X-KEY" in m3u8.media_segment[0]:
entry["EXT-X-KEY"] = {
"URI": m3u8.media_segment[0]["EXT-X-KEY"]["URI"]
}
m3u8.media_segment.insert(0, entry)
hls_time_stamp = self.kwargs.pop("hls_time_stamp", False)
decryptor = None
size_media = len(m3u8.media_segment)
eta = ETA(size_media)
total_duration = 0
duration = 0
max_duration = 0
for index, i in enumerate(m3u8.media_segment):
if "duration" in i["EXTINF"]:
duration = i["EXTINF"]["duration"]
max_duration = max(max_duration, duration)
total_duration += duration
item = get_full_url(i["URI"], url)
if not self.config.get("silent"):
if self.config.get("live"):
progressbar(
size_media, index + 1, "".join([
"DU: ",
str(timedelta(seconds=int(total_duration)))
]))
else:
eta.increment()
progressbar(size_media, index + 1,
"".join(["ETA: ", str(eta)]))
data = self.http.request("get", item, cookies=cookies)
if data.status_code == 404:
break
data = data.content
if m3u8.encrypted:
headers = {}
if self.keycookie:
keycookies = self.keycookie
else:
keycookies = cookies
if self.authorization:
headers["authorization"] = self.authorization
# Update key/decryptor
if "EXT-X-KEY" in i:
keyurl = get_full_url(i["EXT-X-KEY"]["URI"], url)
if keyurl and keyurl[:4] == "skd:":
raise HLSException(keyurl,
"Can't decrypt beacuse of DRM")
key = self.http.request("get",
keyurl,
cookies=keycookies,
headers=headers).content
iv = binascii.unhexlify(i["EXT-X-KEY"]["IV"][2:].zfill(
32)) if "IV" in i["EXT-X-KEY"] else random_iv()
backend = default_backend()
cipher = Cipher(algorithms.AES(key),
modes.CBC(iv),
backend=backend)
decryptor = cipher.decryptor()
# In some cases the playlist say its encrypted but the files is not.
# This happen on svtplay 5.1ch stream where it started with ID3..
# Adding the other ones is header for mpeg-ts files. third byte is 10 or 11..
if data[:
3] != b"ID3" and data[:
3] != b"\x47\x40\x11" and data[:3] != b"\x47\x40\x10" and data[
4:12] != b"ftypisom":
if decryptor:
data = decryptor.update(data)
else:
raise ValueError(
"No decryptor found for encrypted hls steam.")
file_d.write(data)
if self.config.get(
"capture_time"
) > 0 and total_duration >= self.config.get("capture_time") * 60:
break
if (size_media == (index + 1)) and self.config.get("live"):
sleep_int = (start_time + max_duration * 2) - time.time()
if sleep_int > 0:
time.sleep(sleep_int)
size_media_old = size_media
while size_media_old == size_media:
start_time = time.time()
if hls_time_stamp:
end_time_stamp = (datetime.utcnow() - timedelta(
minutes=1, seconds=max_duration * 2)).replace(
microsecond=0)
start_time_stamp = end_time_stamp - timedelta(
minutes=1)
base_url = url.split(".m3u8")[0]
url = f"{base_url}.m3u8?in={start_time_stamp.isoformat()}&out={end_time_stamp.isoformat()}?"
new_m3u8 = M3U8(
self.http.request("get", url, cookies=cookies).text)
for n_m3u in new_m3u8.media_segment:
if not any(d["URI"] == n_m3u["URI"]
for d in m3u8.media_segment):
m3u8.media_segment.append(n_m3u)
size_media = len(m3u8.media_segment)
if size_media_old == size_media:
time.sleep(max_duration)
file_d.close()
if not self.config.get("silent"):
progress_stream.write("\n")
self.finished = True
