Example #1
0
    def _download2(self, files, audio=False):
        cookies = self.kwargs["cookies"]

        if audio:
            file_d = output(copy.copy(self.options), "m4a")
        else:
            file_d = output(self.options, self.options.other)
        if file_d is None:
            return
        eta = ETA(len(files))
        n = 1
        for i in files:
            if not self.options.silent:
                eta.increment()
                progressbar(len(files), n, ''.join(['ETA: ', str(eta)]))
                n += 1
            data = self.http.request("get", i, cookies=cookies)

            if data.status_code == 404:
                break
            data = data.content
            file_d.write(data)

        file_d.close()
        if not self.options.silent:
            progress_stream.write('\n')
        self.finished = True
Example #2
0
    def _download2(self, files, audio=False):
        cookies = self.kwargs["cookies"]

        if audio:
            file_d = output(copy.copy(self.options), "m4a")
        else:
            file_d = output(self.options, self.options.other)
        if hasattr(file_d, "read") is False:
            return
        eta = ETA(len(files))
        n = 1
        for i in files:
            if self.options.output != "-" and not self.options.silent:
                eta.increment()
                progressbar(len(files), n, ''.join(['ETA: ', str(eta)]))
                n += 1
            data = self.http.request("get", i, cookies=cookies)

            if data.status_code == 404:
                break
            data = data.content
            file_d.write(data)

        if self.options.output != "-":
            file_d.close()
            if not self.options.silent:
                progress_stream.write('\n')
            self.finished = True
Example #3
0
    def download(self):
        subdata = self.http.request("get", self.url, cookies=self.options.cookies)

        data = None
        if self.subtype == "tt":
            data = self.tt(subdata)
        if self.subtype == "json":
            data = self.json(subdata)
        if self.subtype == "sami":
            data = self.sami(subdata)
        if self.subtype == "smi":
            data = self.smi(subdata)
        if self.subtype == "wrst":
            data = self.wrst(subdata)
        if self.subtype == "raw":
            if is_py2:
                data = subdata.text.encode("utf-8")
            else:
                data = subdata.text

        if platform.system() == "Windows" and is_py3:
            file_d = output(self.options, "srt", mode="wt", encoding="utf-8")
        else:
            file_d = output(self.options, "srt", mode="wt")
        if hasattr(file_d, "read") is False:
            return
        file_d.write(data)
        file_d.close()
Example #4
0
    def download(self):
        subdata = self.http.request("get", self.url, cookies=self.options.cookies)

        data = None
        if self.subtype == "tt":
            data = self.tt(subdata)
        if self.subtype == "json":
            data = self.json(subdata)
        if self.subtype == "sami":
            data = self.sami(subdata)
        if self.subtype == "smi":
            data = self.smi(subdata)
        if self.subtype == "wrst":
            data = self.wrst(subdata)
        if self.subtype == "raw":
            if is_py2:
                data = subdata.text.encode("utf-8")
            else:
                data = subdata.text
        
        if self.subfix:
            self.options.output = self.options.output + self.subfix
            
        if platform.system() == "Windows" and is_py3:
            file_d = output(self.options, "srt", mode="wt", encoding="utf-8")
        else:
            file_d = output(self.options, "srt", mode="wt")
        if hasattr(file_d, "read") is False:
            return
        file_d.write(data)
        file_d.close()
Example #5
0
    def download(self):
        """ Get the stream from RTMP """
        args = []
        if self.options.live:
            args.append("-v")

        if self.options.resume:
            args.append("-e")

        output(self.options, self.options.output, "flv", False)
        args += ["-o", self.options.output]
        if self.options.silent or self.options.output == "-":
            args.append("-q")
        if self.options.other:
            if is_py2:
                args += shlex.split(self.options.other.encode("utf-8"))
            else:
                args += shlex.split(self.options.other)

        if self.options.verbose:
            args.append("-V")

        command = ["rtmpdump", "-r", self.url] + args
        log.debug("Running: %s", " ".join(command))
        try:
            subprocess.call(command)
        except OSError as e:
            log.error("Could not execute rtmpdump: " + e.strerror)
Example #6
0
 def save_file(self, data, subtype):
     if platform.system() == "Windows" and is_py3:
         file_d = output(self.options, subtype, mode="wt", encoding="utf-8")
     else:
         file_d = output(self.options, subtype, mode="wt")
     if hasattr(file_d, "read") is False:
         return
     file_d.write(data)
     file_d.close()
Example #7
0
 def save_file(self, data, subtype):
     if platform.system() == "Windows" and is_py3:
         file_d = output(self.options, subtype, mode="wt", encoding="utf-8")
     else:
         file_d = output(self.options, subtype, mode="wt")
     if hasattr(file_d, "read") is False:
         return
     file_d.write(data)
     file_d.close()
Example #8
0
    def download(self):
        """ Get the stream from RTMP """
        args = []
        if self.options.live:
            args.append("-v")

        if self.options.resume:
            args.append("-e")

        file_d = output(self.options, self.options.output, "flv", False)
        if file_d is None:
            return
        args += ["-o", self.options.output]
        if self.options.silent or self.options.output == "-":
            args.append("-q")
        if self.options.other:
            if is_py2:
                args += shlex.split(self.options.other.encode("utf-8"))
            else:
                args += shlex.split(self.options.other)

        if self.options.verbose:
            args.append("-V")

        command = ["rtmpdump", "-r", self.url] + args
        log.debug("Running: %s", " ".join(command))
        try:
            subprocess.call(command)
        except OSError as e:
            log.error("Could not execute rtmpdump: " + e.strerror)
Example #9
0
    def download(self):
        """ Get the stream from HTTP """
        data = self.http.request("get", self.url, stream=True)
        try:
            total_size = data.headers['content-length']
        except KeyError:
            total_size = 0
        total_size = int(total_size)
        bytes_so_far = 0

        file_d = output(self.options, "mp4")
        if hasattr(file_d, "read") is False:
            return

        eta = ETA(total_size)
        for i in data.iter_content(8192):
            bytes_so_far += len(i)
            file_d.write(i)
            if self.options.output != "-" and not self.options.silent:
                eta.update(bytes_so_far)
                progressbar(total_size, bytes_so_far, ''.join(["ETA: ", str(eta)]))

        if self.options.output != "-":
            file_d.close()
            self.finished = True
Example #10
0
    def download(self):
        """ Get the stream from HTTP """
        data = self.http.request("get", self.url, stream=True)
        try:
            total_size = data.headers['content-length']
        except KeyError:
            total_size = 0
        total_size = int(total_size)
        bytes_so_far = 0

        file_d = output(self.options, "mp4")
        if hasattr(file_d, "read") is False:
            return

        eta = ETA(total_size)
        for i in data.iter_content(8192):
            bytes_so_far += len(i)
            file_d.write(i)
            if self.options.output != "-" and not self.options.silent:
                eta.update(bytes_so_far)
                progressbar(total_size, bytes_so_far,
                            ''.join(["ETA: ", str(eta)]))

        if self.options.output != "-":
            file_d.close()
            self.finished = True
Example #11
0
    def download(self):
        """ Get the stream from HTTP """
        data = self.http.request("get", self.url, stream=True)
        try:
            total_size = data.headers['content-length']
        except KeyError:
            total_size = 0
        total_size = int(total_size)
        bytes_so_far = 0

        file_d = output(self.options, "mp4")
        if hasattr(file_d, "read") is False:
            return

        lastprogress = 0
        for i in data.iter_content(8192):
            bytes_so_far += len(i)
            file_d.write(i)
            if self.options.output != "-":
                now = time.time()
                if lastprogress + 1 < now:
                    lastprogress = now
                    progress(bytes_so_far, total_size)

        if self.options.output != "-":
            file_d.close()
Example #12
0
    def download(self):
        """ Get the stream from RTMP """
        args = []
        if self.options.live:
            args.append("-v")

        if self.options.resume:
            args.append("-e")

        file_d = output(self.options, "flv", False)
        if file_d is None:
            return
        args += ["-o", self.options.output]
        if self.options.silent:
            args.append("-q")
        if self.options.other:
            if is_py2:
                args += shlex.split(self.options.other.encode("utf-8"))
            else:
                args += shlex.split(self.options.other)

        if self.options.verbose:
            args.append("-V")

        command = ["rtmpdump", "-r", self.url] + args
        log.debug("Running: {0}".format(" ".join(command)))
        try:
            subprocess.call(command)
        except OSError as e:
            log.error("Could not execute rtmpdump: {0}".format(e.strerror))
            return
        self.finished = True
Example #13
0
    def _download_url(self, url, audio=False, total_size=None):
        cookies = self.kwargs["cookies"]
        data = self.http.request("get",
                                 url,
                                 cookies=cookies,
                                 headers={'Range': 'bytes=0-8192'})
        if not total_size:
            try:
                total_size = data.headers['Content-Range']
                total_size = total_size[total_size.find("/") + 1:]
                total_size = int(total_size)
            except KeyError:
                raise KeyError("Can't get the total size.")

        bytes_so_far = 8192
        if audio:
            file_d = output(copy.copy(self.options), "m4a")
        else:
            file_d = output(self.options, self.options.other)

        if file_d is None:
            return
        file_d.write(data.content)
        eta = ETA(total_size)
        while bytes_so_far < total_size:

            if not self.options.silent:
                eta.update(bytes_so_far)
                progressbar(total_size, bytes_so_far,
                            ''.join(["ETA: ", str(eta)]))

            old = bytes_so_far + 1
            bytes_so_far = total_size

            bytes_range = "bytes={0}-{1}".format(old, bytes_so_far)

            data = self.http.request("get",
                                     url,
                                     cookies=cookies,
                                     headers={'Range': bytes_range})
            file_d.write(data.content)

        file_d.close()
        progressbar(bytes_so_far, total_size, "ETA: complete")
        progress_stream.write('\n')
        self.finished = True
Example #14
0
    def _download(self, url, audio=False):
        cookies = self.kwargs["cookies"]
        data = self.http.request("get",
                                 url,
                                 cookies=cookies,
                                 headers={'Range': 'bytes=0-8192'})
        try:
            total_size = data.headers['Content-Range']
            total_size = total_size[total_size.find("/") + 1:]
        except KeyError:
            total_size = 0
        total_size = int(total_size)
        bytes_so_far = 8192
        if audio:
            file_d = output(copy.copy(self.options), "m4a")
        else:
            file_d = output(self.options, self.options.other)
        if hasattr(file_d, "read") is False:
            return
        file_d.write(data.content)
        eta = ETA(total_size)
        while bytes_so_far < total_size:
            old = bytes_so_far + 1
            bytes_so_far = old + 1000000
            if bytes_so_far > total_size:
                bytes_so_far = total_size

            bytes_range = "bytes=%s-%s" % (old, bytes_so_far)

            data = self.http.request("get",
                                     url,
                                     cookies=cookies,
                                     headers={'Range': bytes_range})
            file_d.write(data.content)
            if self.options.output != "-" and not self.options.silent:
                eta.update(old)
                progressbar(total_size, old, ''.join(["ETA: ", str(eta)]))

        if self.options.output != "-":
            file_d.close()
            progressbar(bytes_so_far, total_size, "ETA: complete")
            progress_stream.write('\n')
            self.finished = True
Example #15
0
    def download(self):
        if self.options.live and not self.options.force:
            raise LiveHLSException(self.url)

        cookies = self.kwargs["cookies"]
        m3u8 = self.http.request("get", self.url, cookies=cookies).text
        globaldata, files = parsem3u(m3u8)
        encrypted = False
        key = None
        if "KEY" in globaldata:
            keydata = globaldata["KEY"]
            encrypted = True

        if encrypted:
            try:
                from Crypto.Cipher import AES
            except ImportError:
                log.error("You need to install pycrypto to download encrypted HLS streams")
                sys.exit(2)

            match = re.search(r'URI="(https?://.*?)"', keydata)
            if not match:
                match = re.search(r'URI="([^"]+)"', keydata)
            keyurl = _get_full_url(match.group(1), self.url)
            key = self.http.request("get", keyurl, cookies=cookies).content

            rand = os.urandom(16)
            decryptor = AES.new(key, AES.MODE_CBC, rand)

        file_d = output(self.options, "ts")
        if file_d is None:
            return

        n = 1
        eta = ETA(len(files))
        for i in files:
            item = _get_full_url(i[0], self.url)

            if not self.options.silent:
                eta.increment()
                progressbar(len(files), n, ''.join(['ETA: ', str(eta)]))
                n += 1

            data = self.http.request("get", item, cookies=cookies)
            if data.status_code == 404:
                break
            data = data.content
            if encrypted:
                data = decryptor.decrypt(data)
            file_d.write(data)

        file_d.close()
        if not self.options.silent:
            progress_stream.write('\n')
        self.finished = True
Example #16
0
    def download(self):
        if self.options.live and not self.options.force:
            raise LiveHLSException(self.url)

        error, m3u8 = get_http_data(self.url)
        if error:
            log.error("Cant get m3u8 file.")
            return
        globaldata, files = parsem3u(m3u8)
        encrypted = False
        key = None
        if "KEY" in globaldata:
            keydata = globaldata["KEY"]
            encrypted = True

        if encrypted:
            try:
                from Crypto.Cipher import AES
            except ImportError:
                log.error("You need to install pycrypto to download encrypted HLS streams")
                sys.exit(2)

            match = re.search(r'URI="(https?://.*?)"', keydata)
            error, key = get_http_data(match.group(1))
            if error:
                log.error("Can't get crypto key to decode files.")
                return
            rand = os.urandom(16)
            decryptor = AES.new(key, AES.MODE_CBC, rand)

        file_d = output(self.options, "ts")
        if hasattr(file_d, "read") is False:
            return

        n = 1
        eta = ETA(len(files))
        for i in files:
            item = _get_full_url(i[0], self.url)

            if self.options.output != "-":
                eta.increment()
                progressbar(len(files), n, "".join(["ETA: ", str(eta)]))
                n += 1

            error, data = get_http_data(item)
            if error:
                log.error("Missing segment in playlist")
                return
            if encrypted:
                data = decryptor.decrypt(data)
            file_d.write(data)

        if self.options.output != "-":
            file_d.close()
            progress_stream.write("\n")
Example #17
0
    def download(self):
        if self.options.live and not self.options.force:
            raise LiveHLSException(self.url)

        cookies = self.kwargs["cookies"]
        m3u8 = self.http.request("get", self.url, cookies=cookies).text
        globaldata, files = parsem3u(m3u8)
        encrypted = False
        key = None
        if "KEY" in globaldata:
            keydata = globaldata["KEY"]
            encrypted = True

        if encrypted:
            try:
                from Crypto.Cipher import AES
            except ImportError:
                log.error(
                    "You need to install pycrypto to download encrypted HLS streams"
                )
                sys.exit(2)

            match = re.search(r'URI="(https?://.*?)"', keydata)
            key = self.http.request("get", match.group(1)).content
            rand = os.urandom(16)
            decryptor = AES.new(key, AES.MODE_CBC, rand)

        file_d = output(self.options, "ts")
        if hasattr(file_d, "read") is False:
            return

        n = 1
        eta = ETA(len(files))
        for i in files:
            item = _get_full_url(i[0], self.url)

            if self.options.output != "-" and not self.options.silent:
                eta.increment()
                progressbar(len(files), n, ''.join(['ETA: ', str(eta)]))
                n += 1

            data = self.http.request("get", item, cookies=cookies)
            if data.status_code == 404:
                break
            data = data.content
            if encrypted:
                data = decryptor.decrypt(data)
            file_d.write(data)

        if self.options.output != "-":
            file_d.close()
            if not self.options.silent:
                progress_stream.write('\n')
            self.finished = True
Example #18
0
    def _download_url(self, url, audio=False, total_size=None):
        cookies = self.kwargs["cookies"]
        data = self.http.request("get", url, cookies=cookies, headers={'Range': 'bytes=0-8192'})
        if not total_size:
            try:
                total_size = data.headers['Content-Range']
                total_size = total_size[total_size.find("/") + 1:]
                total_size = int(total_size)
            except KeyError:
                raise KeyError("Can't get the total size.")

        bytes_so_far = 8192
        if audio:
            file_d = output(copy.copy(self.options), "m4a")
        else:
            file_d = output(self.options, self.options.other)

        if file_d is None:
            return
        file_d.write(data.content)
        eta = ETA(total_size)
        while bytes_so_far < total_size:

            if not self.options.silent:
                eta.update(bytes_so_far)
                progressbar(total_size, bytes_so_far, ''.join(["ETA: ", str(eta)]))

            old = bytes_so_far + 1
            bytes_so_far = total_size

            bytes_range = "bytes={0}-{1}".format(old, bytes_so_far)

            data = self.http.request("get", url, cookies=cookies, headers={'Range': bytes_range})
            file_d.write(data.content)

        file_d.close()
        progressbar(bytes_so_far, total_size, "ETA: complete")
        progress_stream.write('\n')
        self.finished = True
Example #19
0
    def _download(self, url, audio=False):
        cookies = self.kwargs["cookies"]
        data = self.http.request(
            "get", url, cookies=cookies, headers={'Range': 'bytes=0-8192'})
        try:
            total_size = data.headers['Content-Range']
            total_size = total_size[total_size.find("/") + 1:]
        except KeyError:
            total_size = 0
        total_size = int(total_size)
        bytes_so_far = 8192
        if audio:
            file_d = output(copy.copy(self.options), "m4a")
        else:
            file_d = output(self.options, self.options.other)
        if hasattr(file_d, "read") is False:
            return
        file_d.write(data.content)
        eta = ETA(total_size)
        while bytes_so_far < total_size:
            old = bytes_so_far + 1
            bytes_so_far = old + 1000000
            if bytes_so_far > total_size:
                bytes_so_far = total_size

            bytes_range = "bytes=%s-%s" % (old, bytes_so_far)

            data = self.http.request(
                "get", url, cookies=cookies, headers={'Range': bytes_range})
            file_d.write(data.content)
            if self.options.output != "-" and not self.options.silent:
                eta.update(old)
                progressbar(total_size, old, ''.join(["ETA: ", str(eta)]))

        if self.options.output != "-":
            file_d.close()
            progress_stream.write('\n')
            self.finished = True
Example #20
0
    def download(self):
        if self.options.live and not self.options.force:
            raise LiveHDSException(self.url)

        querystring = self.kwargs["querystring"]
        cookies = self.kwargs["cookies"]
        bootstrap = base64.b64decode(self.kwargs["bootstrap"])
        box = readboxtype(bootstrap, 0)
        antal = None
        if box[2] == b"abst":
            antal = readbox(bootstrap, box[0])
        baseurl = self.kwargs["manifest"][0:self.kwargs["manifest"].rfind("/")]

        file_d = output(self.options, "flv")
        if hasattr(file_d, "read") is False:
            return

        metasize = struct.pack(">L",
                               len(base64.b64decode(
                                   self.kwargs["metadata"])))[1:]
        file_d.write(binascii.a2b_hex(b"464c560105000000090000000012"))
        file_d.write(metasize)
        file_d.write(binascii.a2b_hex(b"00000000000000"))
        file_d.write(base64.b64decode(self.kwargs["metadata"]))
        file_d.write(binascii.a2b_hex(b"00000000"))
        i = 1
        start = antal[1]["first"]
        total = antal[1]["total"]
        eta = ETA(total)
        while i <= total:
            url = "%s/%sSeg1-Frag%s?%s" % (baseurl, self.url, start,
                                           querystring)
            if self.options.output != "-" and not self.options.silent:
                eta.update(i)
                progressbar(total, i, ''.join(["ETA: ", str(eta)]))
            data = self.http.request("get", url, cookies=cookies)
            if data.status_code == 404:
                break
            data = data.content
            number = decode_f4f(i, data)
            file_d.write(data[number:])
            i += 1
            start += 1

        if self.options.output != "-":
            file_d.close()
            if not self.options.silent:
                progress_stream.write('\n')
            self.finished = True
Example #21
0
    def download(self):
        if self.options.live and not self.options.force:
            raise LiveHLSException(self.url)

        m3u8 = get_http_data(self.url)
        globaldata, files = parsem3u(m3u8)
        encrypted = False
        key = None
        try:
            keydata = globaldata["KEY"]
            encrypted = True
        except KeyError:
            pass

        if encrypted:
            try:
                from Crypto.Cipher import AES
            except ImportError:
                log.error("You need to install pycrypto to download encrypted HLS streams")
                sys.exit(2)

            match = re.search(r'URI="(https?://.*?)"', keydata)
            key = get_http_data(match.group(1))
            rand = os.urandom(16)
            decryptor = AES.new(key, AES.MODE_CBC, rand)

        file_d = output(self.options, self.options.output, "ts")
        if hasattr(file_d, "read") is False:
            return

        n = 1
        eta = ETA(len(files))
        for i in files:
            item = _get_full_url(i[0], self.url)

            if self.options.output != "-":
                eta.increment()
                progressbar(len(files), n, ''.join(['ETA: ', str(eta)]))
                n += 1

            data = get_http_data(item)
            if encrypted:
                data = decryptor.decrypt(data)
            file_d.write(data)

        if self.options.output != "-":
            file_d.close()
            progress_stream.write('\n')
Example #22
0
    def download(self):
        if self.options.live and not self.options.force:
            raise LiveHDSException(self.url)

        querystring = self.kwargs["querystring"]
        cookies = self.kwargs["cookies"]
        bootstrap = base64.b64decode(self.kwargs["bootstrap"])
        box = readboxtype(bootstrap, 0)
        antal = None
        if box[2] == b"abst":
            antal = readbox(bootstrap, box[0])
        baseurl = self.kwargs["manifest"][0:self.kwargs["manifest"].rfind("/")]

        file_d = output(self.options, "flv")
        if hasattr(file_d, "read") is False:
            return

        metasize = struct.pack(">L", len(base64.b64decode(self.kwargs["metadata"])))[1:]
        file_d.write(binascii.a2b_hex(b"464c560105000000090000000012"))
        file_d.write(metasize)
        file_d.write(binascii.a2b_hex(b"00000000000000"))
        file_d.write(base64.b64decode(self.kwargs["metadata"]))
        file_d.write(binascii.a2b_hex(b"00000000"))
        i = 1
        start = antal[1]["first"]
        total = antal[1]["total"]
        eta = ETA(total)
        while i <= total:
            url = "%s/%sSeg1-Frag%s?%s" % (baseurl, self.url, start, querystring)
            if self.options.output != "-" and not self.options.silent:
                eta.update(i)
                progressbar(total, i, ''.join(["ETA: ", str(eta)]))
            data = self.http.request("get", url, cookies=cookies)
            if data.status_code == 404:
                break
            data = data.content
            number = decode_f4f(i, data)
            file_d.write(data[number:])
            i += 1
            start += 1

        if self.options.output != "-":
            file_d.close()
            if not self.options.silent:
                progress_stream.write('\n')
            self.finished = True
Example #23
0
    def download(self):
        if self.options.live and not self.options.force:
            raise LiveHDSException(self.url)

        querystring = self.kwargs["querystring"]
        bootstrap = base64.b64decode(self.kwargs["bootstrap"])
        box = readboxtype(bootstrap, 0)
        antal = None
        if box[2] == b"abst":
            antal = readbox(bootstrap, box[0])
        baseurl = self.kwargs["manifest"][0:self.kwargs["manifest"].rfind("/")]

        file_d = output(self.options, "flv")
        if hasattr(file_d, "read") is False:
            return

        metasize = struct.pack(">L", len(base64.b64decode(self.kwargs["metadata"])))[1:]
        file_d.write(binascii.a2b_hex(b"464c560105000000090000000012"))
        file_d.write(metasize)
        file_d.write(binascii.a2b_hex(b"00000000000000"))
        file_d.write(base64.b64decode(self.kwargs["metadata"]))
        file_d.write(binascii.a2b_hex(b"00000000"))
        i = 1
        start = antal[1]["first"]
        total = antal[1]["total"]
        eta = ETA(total)
        while i <= total:
            url = "%s/%sSeg1-Frag%s?%s" % (baseurl, self.url, start, querystring)
            if self.options.output != "-":
                eta.update(i)
                progressbar(total, i, ''.join(["ETA: ", str(eta)]))
            error, data = get_http_data(url)
            if error:
                log.error("Missing segment in playlist")
                return
            number = decode_f4f(i, data)
            file_d.write(data[number:])
            i += 1
            start += 1

        if self.options.output != "-":
            file_d.close()
            progress_stream.write('\n')
Example #24
0
    def download(self):
        subdata = self.http.request("get", self.url, cookies=self.options.cookies)

        data = None
        if self.subtype == "tt":
            data = self.tt(subdata)
        if self.subtype == "json":
            data = self.json(subdata)
        if self.subtype == "sami":
            data = self.sami(subdata)
        if self.subtype == "smi":
            data = self.smi(subdata)
        if self.subtype == "wrst":
            data = self.wrst(subdata)

        file_d = output(self.options, "srt", mode="wt")
        if hasattr(file_d, "read") is False:
            return
        file_d.write(data)
        file_d.close()
Example #25
0
    def download(self):
        """ Get the stream from HTTP """
        request = Request(self.url)
        request.add_header(
            'User-Agent',
            'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.9.0.3) Gecko/2008092417 Firefox/3.0.3'
        )
        try:
            response = urlopen(request)
        except HTTPError as e:
            log.error("Something wrong with that url")
            log.error("Error code: %s", e.code)
            sys.exit(5)
        try:
            total_size = response.info()['Content-Length']
        except KeyError:
            total_size = 0
        total_size = int(total_size)
        bytes_so_far = 0

        file_d = output(self.options, self.options.output, "mp4")
        if hasattr(file_d, "read") is False:
            return

        lastprogress = 0
        while 1:
            chunk = response.read(8192)
            bytes_so_far += len(chunk)

            if not chunk:
                break

            file_d.write(chunk)
            if self.options.output != "-":
                now = time.time()
                if lastprogress + 1 < now:
                    lastprogress = now
                    progress(bytes_so_far, total_size)

        if self.options.output != "-":
            file_d.close()
Example #26
0
    def download(self):
        """ Get the stream from HTTP """
        request = Request(self.url)
        request.add_header('User-Agent', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.9.0.3) Gecko/2008092417 Firefox/3.0.3')
        try:
            response = urlopen(request)
        except HTTPError as e:
            log.error("Something wrong with that url")
            log.error("Error code: %s", e.code)
            sys.exit(5)
        try:
            total_size = response.info()['Content-Length']
        except KeyError:
            total_size = 0
        total_size = int(total_size)
        bytes_so_far = 0

        file_d = output(self.options, self.url, "mp4")
        if hasattr(file_d, "read") is False:
            return

        lastprogress = 0
        while 1:
            chunk = response.read(8192)
            bytes_so_far += len(chunk)

            if not chunk:
                break

            file_d.write(chunk)
            if self.options.output != "-":
                now = time.time()
                if lastprogress + 1 < now:
                    lastprogress = now
                    progress(bytes_so_far, total_size)

        if self.options.output != "-":
            file_d.close()
Example #27
0
    def download(self):
        error, subdata = get_http_data(self.url, cookiejar=self.options.cookies)
        if error:
            log.error("Can't download subtitle")
            return

        data = None
        if self.subtype == "tt":
            data = self.tt(subdata)
        if self.subtype == "json":
            data = self.json(subdata)
        if self.subtype == "sami":
            data = self.sami(subdata)
        if self.subtype == "smi":
            data = self.smi(subdata)
        if self.subtype == "wrst":
            data = self.wrst(subdata)

        file_d = output(self.options, "srt")
        if hasattr(file_d, "read") is False:
            return
        file_d.write(data)
        file_d.close()
Example #28
0
    def _download(self, url, file_name):
        cookies = self.kwargs["cookies"]
        start_time = time.time()
        m3u8 = M3U8(self.http.request("get", url, cookies=cookies).text)
        key = None

        if m3u8.encrypted:
            try:
                from Crypto.Cipher import AES
                from Crypto import Random
            except ImportError:
                log.error("You need to install pycrypto to download encrypted HLS streams")
                sys.exit(2)

        file_d = output(file_name[0], file_name[1])
        if file_d is None:
            return

        decryptor = None
        size_media = len(m3u8.media_segment)
        eta = ETA(size_media)
        total_duration = 0
        duration = 0
        max_duration = 0
        for index, i in enumerate(m3u8.media_segment):
            if "duration" in i["EXTINF"]:
                duration = i["EXTINF"]["duration"]
                max_duration = max(max_duration, duration)
                total_duration += duration
            item = _get_full_url(i["URI"], url)

            if not self.options.silent:
                if self.options.live:
                    progressbar(size_media, index + 1, ''.join(['DU: ', str(timedelta(seconds=int(total_duration)))]))
                else:
                    eta.increment()
                    progressbar(size_media, index + 1, ''.join(['ETA: ', str(eta)]))

            data = self.http.request("get", item, cookies=cookies)
            if data.status_code == 404:
                break
            data = data.content
            if m3u8.encrypted:
                headers = {}
                if self.keycookie:
                    keycookies = self.keycookie
                else:
                    keycookies = cookies
                if self.authorization:
                    headers["authorization"] = self.authorization

                # Update key/decryptor
                if "EXT-X-KEY" in i:
                    keyurl = _get_full_url(i["EXT-X-KEY"]["URI"], url)
                    key = self.http.request("get", keyurl, cookies=keycookies, headers=headers).content
                    vi = binascii.unhexlify(i["EXT-X-KEY"]["IV"][2:].zfill(32)) if "IV" in i["EXT-X-KEY"] else Random.new().read(AES.block_size)
                    decryptor = AES.new(key, AES.MODE_CBC, vi)

                if decryptor:
                    data = decryptor.decrypt(data)
                else:
                    raise ValueError("No decryptor found for encrypted hls steam.")

            file_d.write(data)

            if (self.options.capture_time > 0) and total_duration >= (self.options.capture_time * 60):
                break

            if (size_media == (index + 1)) and self.options.live:
                sleep_int = (start_time + max_duration * 2) - time.time()
                if sleep_int > 0:
                    time.sleep(sleep_int)

                size_media_old = size_media
                while size_media_old == size_media:
                    start_time = time.time()

                    if self.options.hls_time_stamp:

                        end_time_stamp = (datetime.utcnow() - timedelta(seconds=max_duration * 2)).replace(microsecond=0)
                        start_time_stamp = end_time_stamp - timedelta(minutes=1)

                        base_url = url.split(".m3u8")[0]
                        url = "{0}.m3u8?in={1}&out={2}?".format(base_url, start_time_stamp.isoformat(), end_time_stamp.isoformat())

                    new_m3u8 = M3U8(self.http.request("get", url, cookies=cookies).text)
                    for n_m3u in new_m3u8.media_segment:
                        if not any(d["URI"] == n_m3u["URI"] for d in m3u8.media_segment):
                            m3u8.media_segment.append(n_m3u)

                    size_media = len(m3u8.media_segment)

                    if size_media_old == size_media:
                        time.sleep(max_duration)

        file_d.close()
        if not self.options.silent:
            progress_stream.write('\n')
        self.finished = True
Example #29
0
    def _download(self, url, file_name):
        cookies = self.kwargs["cookies"]
        start_time = time.time()
        m3u8 = M3U8(self.http.request("get", url, cookies=cookies).text)
        key = None

        if m3u8.encrypted:
            try:
                from Crypto.Cipher import AES
                from Crypto import Random
            except ImportError:
                log.error(
                    "You need to install pycrypto to download encrypted HLS streams"
                )
                sys.exit(2)

        file_d = output(file_name[0], file_name[1])
        if file_d is None:
            return

        decryptor = None
        size_media = len(m3u8.media_segment)
        eta = ETA(size_media)
        total_duration = 0
        duration = 0
        max_duration = 0
        for index, i in enumerate(m3u8.media_segment):
            if "duration" in i["EXTINF"]:
                duration = i["EXTINF"]["duration"]
                max_duration = max(max_duration, duration)
                total_duration += duration
            item = _get_full_url(i["URI"], url)

            if not self.options.silent:
                if self.options.live:
                    progressbar(
                        size_media, index + 1, ''.join([
                            'DU: ',
                            str(timedelta(seconds=int(total_duration)))
                        ]))
                else:
                    eta.increment()
                    progressbar(size_media, index + 1,
                                ''.join(['ETA: ', str(eta)]))

            data = self.http.request("get", item, cookies=cookies)
            if data.status_code == 404:
                break
            data = data.content
            if m3u8.encrypted:
                headers = {}
                if self.keycookie:
                    keycookies = self.keycookie
                else:
                    keycookies = cookies
                if self.authorization:
                    headers["authorization"] = self.authorization

                # Update key/decryptor
                if "EXT-X-KEY" in i:
                    keyurl = _get_full_url(i["EXT-X-KEY"]["URI"], url)
                    key = self.http.request("get",
                                            keyurl,
                                            cookies=keycookies,
                                            headers=headers).content
                    vi = binascii.unhexlify(i["EXT-X-KEY"]["IV"][2:].zfill(
                        32)) if "IV" in i["EXT-X-KEY"] else Random.new().read(
                            AES.block_size)
                    decryptor = AES.new(key, AES.MODE_CBC, vi)

                if decryptor:
                    data = decryptor.decrypt(data)
                else:
                    raise ValueError(
                        "No decryptor found for encrypted hls steam.")

            file_d.write(data)

            if (self.options.capture_time >
                    0) and total_duration >= (self.options.capture_time * 60):
                break

            if (size_media == (index + 1)) and self.options.live:
                sleep_int = (start_time + max_duration * 2) - time.time()
                if sleep_int > 0:
                    time.sleep(sleep_int)

                size_media_old = size_media
                while size_media_old == size_media:
                    start_time = time.time()

                    if self.options.hls_time_stamp:

                        end_time_stamp = (
                            datetime.utcnow() -
                            timedelta(seconds=max_duration * 2)).replace(
                                microsecond=0)
                        start_time_stamp = end_time_stamp - timedelta(
                            minutes=1)

                        base_url = url.split(".m3u8")[0]
                        url = "{0}.m3u8?in={1}&out={2}?".format(
                            base_url, start_time_stamp.isoformat(),
                            end_time_stamp.isoformat())

                    new_m3u8 = M3U8(
                        self.http.request("get", url, cookies=cookies).text)
                    for n_m3u in new_m3u8.media_segment:
                        if not any(d["URI"] == n_m3u["URI"]
                                   for d in m3u8.media_segment):
                            m3u8.media_segment.append(n_m3u)

                    size_media = len(m3u8.media_segment)

                    if size_media_old == size_media:
                        time.sleep(max_duration)

        file_d.close()
        if not self.options.silent:
            progress_stream.write('\n')
        self.finished = True
Example #30
0
    def _download2(self, files, audio=False):
        cookies = self.kwargs["cookies"]

        if audio:
            file_d = output(copy.copy(self.options), "m4a")
        else:
            file_d = output(self.options, self.options.other)
        if hasattr(file_d, "read") is False:
            return

        if file_d == None:
            return
        eta = ETA(len(files))
        n = 1
        # check if we can resume a past download
        already_downloaded = os.fstat(file_d.fileno()).st_size
        if self.options.resume and already_downloaded != 0:
            print('Checking for existing segments...')
        bytes_so_far = 0
        resumed = False
        data_size = 0
        for i in files:
            if self.options.output != "-" and not self.options.silent:
                eta.increment()
                progressbar(len(files), n, ''.join(['ETA: ', str(eta)]))
                n += 1

            # get segment size
            if self.options.resume and already_downloaded != 0:
                if bytes_so_far < already_downloaded:
                    head_req = self.http.request("head", i, cookies=cookies)
                    if head_req.status_code == 404:
                        break
                    data_size = int(head_req.headers['Content-Length'])
                    bytes_so_far += data_size
                    log.debug('Segment %d', n-1)
                    log.debug('So far: %d, segment size: %d, on disk: %d', bytes_so_far - data_size, data_size, already_downloaded)
                if bytes_so_far >= already_downloaded:
                    if not resumed and self.options.resume:
                        print
                        print('Resuming download on segment ' + str(n-1))
                        log.debug('Truncating output file to %d', bytes_so_far - data_size)              
                        file_d.truncate(bytes_so_far - data_size) # line up the file with a segment boundary                              
                        eta = ETA(len(files) - n)          
                        resumed = True
                    data = self.http.request("get", i, cookies=cookies)
                    if data.status_code == 404:
                        break
                    data = data.content
                    file_d.write(data)
            else:
                data = self.http.request("get", i, cookies=cookies)
                if data.status_code == 404:
                    break
                data = data.content
                file_d.write(data)

        if self.options.output != "-":
            file_d.close()
            if not self.options.silent:
                progress_stream.write('\n')
            self.finished = True