def _download2(self, files, audio=False): cookies = self.kwargs["cookies"] if audio: file_d = output(copy.copy(self.output), self.config, extension="m4a") else: file_d = output(self.output, self.config, extension="mp4") if file_d is None: return eta = ETA(len(files)) n = 1 for i in files: if not self.config.get("silent"): eta.increment() progressbar(len(files), n, "".join(["ETA: ", str(eta)])) n += 1 data = self.http.request("get", i, cookies=cookies) if data.status_code == 404: break data = data.content file_d.write(data) file_d.close() if not self.config.get("silent"): progress_stream.write("\n") self.finished = True
def download(self): """ Get the stream from HTTP """ _, ext = os.path.splitext(self.url) if ext == ".mp3": self.output_extention = "mp3" else: self.output_extention = "mp4" # this might be wrong.. data = self.http.request("get", self.url, stream=True) try: total_size = data.headers["content-length"] except KeyError: total_size = 0 total_size = int(total_size) bytes_so_far = 0 file_d = output(self.output, self.config, self.output_extention) if file_d is None: return eta = ETA(total_size) for i in data.iter_content(8192): bytes_so_far += len(i) file_d.write(i) if not self.config.get("silent"): eta.update(bytes_so_far) progressbar(total_size, bytes_so_far, "".join(["ETA: ", str(eta)])) file_d.close() self.finished = True
def download(self): """ Get the stream from HTTP """ self.output_extention = "mp4" # this might be wrong.. data = self.http.request("get", self.url, stream=True) try: total_size = data.headers['content-length'] except KeyError: total_size = 0 total_size = int(total_size) bytes_so_far = 0 file_d = output(self.output, self.config, "mp4") if file_d is None: return eta = ETA(total_size) for i in data.iter_content(8192): bytes_so_far += len(i) file_d.write(i) if not self.config.get("silent"): eta.update(bytes_so_far) progressbar(total_size, bytes_so_far, ''.join(["ETA: ", str(eta)])) file_d.close() self.finished = True
def _download2(self, files, audio=False): cookies = self.kwargs["cookies"] if audio: file_d = output(copy.copy(self.output), self.config, extension="m4a") else: file_d = output(self.output, self.config, extension="mp4") if file_d is None: return eta = ETA(len(files)) n = 1 for i in files: if not self.config.get("silent"): eta.increment() progressbar(len(files), n, ''.join(['ETA: ', str(eta)])) n += 1 data = self.http.request("get", i, cookies=cookies) if data.status_code == 404: break data = data.content file_d.write(data) file_d.close() if not self.config.get("silent"): progress_stream.write('\n') self.finished = True
def download(self): self.output_extention = "flv" if self.config.get("live") and not self.config.get("force"): raise LiveHDSException(self.url) querystring = self.kwargs["querystring"] cookies = self.kwargs["cookies"] bootstrap = base64.b64decode(self.kwargs["bootstrap"]) box = readboxtype(bootstrap, 0) antal = None if box[2] == b"abst": antal = readbox(bootstrap, box[0]) baseurl = self.url[0:self.url.rfind("/")] file_d = output(self.output, self.config, "flv") if file_d is None: return metasize = struct.pack(">L", len(base64.b64decode( self.kwargs["metadata"])))[1:] file_d.write(binascii.a2b_hex(b"464c560105000000090000000012")) file_d.write(metasize) file_d.write(binascii.a2b_hex(b"00000000000000")) file_d.write(base64.b64decode(self.kwargs["metadata"])) file_d.write(binascii.a2b_hex(b"00000000")) i = 1 start = antal[1]["first"] total = antal[1]["total"] eta = ETA(total) while i <= total: url = "{0}/{1}Seg1-Frag{2}?{3}".format(baseurl, self.kwargs["url_id"], start, querystring) if not self.config.get("silent"): eta.update(i) progressbar(total, i, ''.join(["ETA: ", str(eta)])) data = self.http.request("get", url, cookies=cookies) if data.status_code == 404: break data = data.content number = decode_f4f(i, data) file_d.write(data[number:]) i += 1 start += 1 file_d.close() if not self.config.get("silent"): progress_stream.write('\n') self.finished = True
def _download_url(self, url, audio=False, total_size=None): cookies = self.kwargs["cookies"] data = self.http.request("get", url, cookies=cookies, headers={'Range': 'bytes=0-8192'}) if not total_size: try: total_size = data.headers['Content-Range'] total_size = total_size[total_size.find("/") + 1:] total_size = int(total_size) except KeyError: raise KeyError("Can't get the total size.") bytes_so_far = 8192 if audio: file_d = output(copy.copy(self.output), self.config, "m4a") else: file_d = output(self.output, self.config, "mp4") if file_d is None: return file_d.write(data.content) eta = ETA(total_size) while bytes_so_far < total_size: if not self.config.get("silent"): eta.update(bytes_so_far) progressbar(total_size, bytes_so_far, ''.join(["ETA: ", str(eta)])) old = bytes_so_far + 1 bytes_so_far = total_size bytes_range = "bytes={0}-{1}".format(old, bytes_so_far) data = self.http.request("get", url, cookies=cookies, headers={'Range': bytes_range}) file_d.write(data.content) file_d.close() progressbar(bytes_so_far, total_size, "ETA: complete") # progress_stream.write('\n') self.finished = True
def download(self): self.output_extention = "flv" if self.config.get("live") and not self.config.get("force"): raise LiveHDSException(self.url) querystring = self.kwargs["querystring"] cookies = self.kwargs["cookies"] bootstrap = base64.b64decode(self.kwargs["bootstrap"]) box = readboxtype(bootstrap, 0) antal = None if box[2] == b"abst": antal = readbox(bootstrap, box[0]) baseurl = self.url[0:self.url.rfind("/")] file_d = output(self.output, self.config, "flv") if file_d is None: return metasize = struct.pack(">L", len(base64.b64decode(self.kwargs["metadata"])))[1:] file_d.write(binascii.a2b_hex(b"464c560105000000090000000012")) file_d.write(metasize) file_d.write(binascii.a2b_hex(b"00000000000000")) file_d.write(base64.b64decode(self.kwargs["metadata"])) file_d.write(binascii.a2b_hex(b"00000000")) i = 1 start = antal[1]["first"] total = antal[1]["total"] eta = ETA(total) while i <= total: url = "{0}/{1}Seg1-Frag{2}?{3}".format(baseurl, self.kwargs["url_id"], start, querystring) if not self.config.get("silent"): eta.update(i) progressbar(total, i, ''.join(["ETA: ", str(eta)])) data = self.http.request("get", url, cookies=cookies) if data.status_code == 404: break data = data.content number = decode_f4f(i, data) file_d.write(data[number:]) i += 1 start += 1 file_d.close() if not self.config.get("silent"): progress_stream.write('\n') self.finished = True
def _download_url(self, url, audio=False, total_size=None): cookies = self.kwargs["cookies"] data = self.http.request("get", url, cookies=cookies, headers={'Range': 'bytes=0-8192'}) if not total_size: try: total_size = data.headers['Content-Range'] total_size = total_size[total_size.find("/") + 1:] total_size = int(total_size) except KeyError: raise KeyError("Can't get the total size.") bytes_so_far = 8192 if audio: file_d = output(copy.copy(self.output), self.config, "m4a") else: file_d = output(self.output, self.config, "mp4") if file_d is None: return file_d.write(data.content) eta = ETA(total_size) while bytes_so_far < total_size: if not self.config.get("silent"): eta.update(bytes_so_far) progressbar(total_size, bytes_so_far, ''.join(["ETA: ", str(eta)])) old = bytes_so_far + 1 bytes_so_far = total_size bytes_range = "bytes={0}-{1}".format(old, bytes_so_far) data = self.http.request("get", url, cookies=cookies, headers={'Range': bytes_range}) file_d.write(data.content) file_d.close() progressbar(bytes_so_far, total_size, "ETA: complete") # progress_stream.write('\n') self.finished = True
def _download(self, url, file_name): cookies = self.kwargs.get("cookies", None) start_time = time.time() m3u8 = M3U8(self.http.request("get", url, cookies=cookies).text) key = None if m3u8.encrypted: from Crypto.Cipher import AES def random_iv(): try: from Crypto import Random return Random.new().read(AES.block_size) except ImportError: return os.urandom(16) file_d = output(file_name[0], self.config, file_name[1]) if file_d is None: return hls_time_stamp = self.kwargs.pop("hls_time_stamp", False) decryptor = None size_media = len(m3u8.media_segment) eta = ETA(size_media) total_duration = 0 duration = 0 max_duration = 0 for index, i in enumerate(m3u8.media_segment): if "duration" in i["EXTINF"]: duration = i["EXTINF"]["duration"] max_duration = max(max_duration, duration) total_duration += duration item = get_full_url(i["URI"], url) if not self.config.get("silent"): if self.config.get("live"): progressbar( size_media, index + 1, ''.join([ 'DU: ', str(timedelta(seconds=int(total_duration))) ])) else: eta.increment() progressbar(size_media, index + 1, ''.join(['ETA: ', str(eta)])) data = self.http.request("get", item, cookies=cookies) if data.status_code == 404: break data = data.content if m3u8.encrypted: headers = {} if self.keycookie: keycookies = self.keycookie else: keycookies = cookies if self.authorization: headers["authorization"] = self.authorization # Update key/decryptor if "EXT-X-KEY" in i: keyurl = get_full_url(i["EXT-X-KEY"]["URI"], url) key = self.http.request("get", keyurl, cookies=keycookies, headers=headers).content iv = binascii.unhexlify(i["EXT-X-KEY"]["IV"][2:].zfill( 32)) if "IV" in i["EXT-X-KEY"] else random_iv() decryptor = AES.new(key, AES.MODE_CBC, iv) if decryptor: data = decryptor.decrypt(data) else: raise ValueError( "No decryptor found for encrypted hls steam.") file_d.write(data) if self.config.get( "capture_time" ) > 0 and total_duration >= self.config.get("capture_time") * 60: break if (size_media == (index + 1)) and self.config.get("live"): sleep_int = (start_time + max_duration * 2) - time.time() if sleep_int > 0: time.sleep(sleep_int) size_media_old = size_media while size_media_old == size_media: start_time = time.time() if hls_time_stamp: end_time_stamp = (datetime.utcnow() - timedelta( minutes=1, seconds=max_duration * 2)).replace( microsecond=0) start_time_stamp = end_time_stamp - timedelta( minutes=1) base_url = url.split(".m3u8")[0] url = "{0}.m3u8?in={1}&out={2}?".format( base_url, start_time_stamp.isoformat(), end_time_stamp.isoformat()) new_m3u8 = M3U8( self.http.request("get", url, cookies=cookies).text) for n_m3u in new_m3u8.media_segment: if not any(d["URI"] == n_m3u["URI"] for d in m3u8.media_segment): m3u8.media_segment.append(n_m3u) size_media = len(m3u8.media_segment) if size_media_old == size_media: time.sleep(max_duration) file_d.close() if not self.config.get("silent"): progress_stream.write('\n') self.finished = True
def _download(self, url, file_name): cookies = self.kwargs.get("cookies", None) start_time = time.time() m3u8 = M3U8(self.http.request("get", url, cookies=cookies).text) key = None def random_iv(): return os.urandom(16) file_d = output(file_name[0], self.config, file_name[1]) if file_d is None: return hls_time_stamp = self.kwargs.pop("hls_time_stamp", False) decryptor = None size_media = len(m3u8.media_segment) eta = ETA(size_media) total_duration = 0 duration = 0 max_duration = 0 for index, i in enumerate(m3u8.media_segment): if "duration" in i["EXTINF"]: duration = i["EXTINF"]["duration"] max_duration = max(max_duration, duration) total_duration += duration item = get_full_url(i["URI"], url) if not self.config.get("silent"): if self.config.get("live"): progressbar(size_media, index + 1, ''.join(['DU: ', str(timedelta(seconds=int(total_duration)))])) else: eta.increment() progressbar(size_media, index + 1, ''.join(['ETA: ', str(eta)])) data = self.http.request("get", item, cookies=cookies) if data.status_code == 404: break data = data.content if m3u8.encrypted: headers = {} if self.keycookie: keycookies = self.keycookie else: keycookies = cookies if self.authorization: headers["authorization"] = self.authorization # Update key/decryptor if "EXT-X-KEY" in i: keyurl = get_full_url(i["EXT-X-KEY"]["URI"], url) if keyurl and keyurl[:4] == "skd:": raise HLSException(keyurl, "Can't decrypt beacuse of DRM") key = self.http.request("get", keyurl, cookies=keycookies, headers=headers).content iv = binascii.unhexlify(i["EXT-X-KEY"]["IV"][2:].zfill(32)) if "IV" in i["EXT-X-KEY"] else random_iv() backend = default_backend() cipher = Cipher(algorithms.AES(key), modes.CBC(iv), backend=backend) decryptor = cipher.decryptor() if decryptor: data = decryptor.update(data) else: raise ValueError("No decryptor found for encrypted hls steam.") file_d.write(data) if self.config.get("capture_time") > 0 and total_duration >= self.config.get("capture_time") * 60: break if (size_media == (index + 1)) and self.config.get("live"): sleep_int = (start_time + max_duration * 2) - time.time() if sleep_int > 0: time.sleep(sleep_int) size_media_old = size_media while size_media_old == size_media: start_time = time.time() if hls_time_stamp: end_time_stamp = (datetime.utcnow() - timedelta(minutes=1, seconds=max_duration * 2)).replace(microsecond=0) start_time_stamp = end_time_stamp - timedelta(minutes=1) base_url = url.split(".m3u8")[0] url = "{0}.m3u8?in={1}&out={2}?".format(base_url, start_time_stamp.isoformat(), end_time_stamp.isoformat()) new_m3u8 = M3U8(self.http.request("get", url, cookies=cookies).text) for n_m3u in new_m3u8.media_segment: if not any(d["URI"] == n_m3u["URI"] for d in m3u8.media_segment): m3u8.media_segment.append(n_m3u) size_media = len(m3u8.media_segment) if size_media_old == size_media: time.sleep(max_duration) file_d.close() if not self.config.get("silent"): progress_stream.write('\n') self.finished = True
def _download(self, url, file_name): cookies = self.kwargs.get("cookies", None) start_time = time.time() m3u8 = M3U8(self.http.request("get", url, cookies=cookies).text) key = None def random_iv(): return os.urandom(16) file_d = output(file_name[0], self.config, file_name[1]) if file_d is None: return if "EXT-X-MAP" in m3u8.media_segment[0]: entry = { "URI": get_full_url(m3u8.media_segment[0]["EXT-X-MAP"]["URI"], url), "EXTINF": { "duration": 0 } } if "EXT-X-KEY" in m3u8.media_segment[0]: entry["EXT-X-KEY"] = { "URI": m3u8.media_segment[0]["EXT-X-KEY"]["URI"] } m3u8.media_segment.insert(0, entry) hls_time_stamp = self.kwargs.pop("hls_time_stamp", False) decryptor = None size_media = len(m3u8.media_segment) eta = ETA(size_media) total_duration = 0 duration = 0 max_duration = 0 for index, i in enumerate(m3u8.media_segment): if "duration" in i["EXTINF"]: duration = i["EXTINF"]["duration"] max_duration = max(max_duration, duration) total_duration += duration item = get_full_url(i["URI"], url) if not self.config.get("silent"): if self.config.get("live"): progressbar( size_media, index + 1, "".join([ "DU: ", str(timedelta(seconds=int(total_duration))) ])) else: eta.increment() progressbar(size_media, index + 1, "".join(["ETA: ", str(eta)])) data = self.http.request("get", item, cookies=cookies) if data.status_code == 404: break data = data.content if m3u8.encrypted: headers = {} if self.keycookie: keycookies = self.keycookie else: keycookies = cookies if self.authorization: headers["authorization"] = self.authorization # Update key/decryptor if "EXT-X-KEY" in i: keyurl = get_full_url(i["EXT-X-KEY"]["URI"], url) if keyurl and keyurl[:4] == "skd:": raise HLSException(keyurl, "Can't decrypt beacuse of DRM") key = self.http.request("get", keyurl, cookies=keycookies, headers=headers).content iv = binascii.unhexlify(i["EXT-X-KEY"]["IV"][2:].zfill( 32)) if "IV" in i["EXT-X-KEY"] else random_iv() backend = default_backend() cipher = Cipher(algorithms.AES(key), modes.CBC(iv), backend=backend) decryptor = cipher.decryptor() # In some cases the playlist say its encrypted but the files is not. # This happen on svtplay 5.1ch stream where it started with ID3.. # Adding the other ones is header for mpeg-ts files. third byte is 10 or 11.. if data[: 3] != b"ID3" and data[: 3] != b"\x47\x40\x11" and data[:3] != b"\x47\x40\x10" and data[ 4:12] != b"ftypisom": if decryptor: data = decryptor.update(data) else: raise ValueError( "No decryptor found for encrypted hls steam.") file_d.write(data) if self.config.get( "capture_time" ) > 0 and total_duration >= self.config.get("capture_time") * 60: break if (size_media == (index + 1)) and self.config.get("live"): sleep_int = (start_time + max_duration * 2) - time.time() if sleep_int > 0: time.sleep(sleep_int) size_media_old = size_media while size_media_old == size_media: start_time = time.time() if hls_time_stamp: end_time_stamp = (datetime.utcnow() - timedelta( minutes=1, seconds=max_duration * 2)).replace( microsecond=0) start_time_stamp = end_time_stamp - timedelta( minutes=1) base_url = url.split(".m3u8")[0] url = f"{base_url}.m3u8?in={start_time_stamp.isoformat()}&out={end_time_stamp.isoformat()}?" new_m3u8 = M3U8( self.http.request("get", url, cookies=cookies).text) for n_m3u in new_m3u8.media_segment: if not any(d["URI"] == n_m3u["URI"] for d in m3u8.media_segment): m3u8.media_segment.append(n_m3u) size_media = len(m3u8.media_segment) if size_media_old == size_media: time.sleep(max_duration) file_d.close() if not self.config.get("silent"): progress_stream.write("\n") self.finished = True