예제 #1
0
    def __init__(self,
                 url,
                 filename,
                 get={},
                 post={},
                 referer=None,
                 cj=None,
                 bucket=None,
                 options={},
                 progress=None,
                 disposition=False):
        self.url = url
        self.filename = filename  #: complete file destination, not only name
        self.get = get
        self.post = post
        self.referer = referer
        self.cj = cj  #: cookiejar if cookies are needed
        self.bucket = bucket
        self.options = options
        self.disposition = disposition
        # all arguments

        self.abort = False
        self.size = 0
        self.nameDisposition = None  #: will be parsed from content disposition

        self.chunks = []

        self.log = logging.getLogger("log")

        try:
            self.info = ChunkInfo.load(filename)
            self.info.resume = True  #: resume is only possible with valid info file
            self.size = self.info.size
            self.infoSaved = True
        except IOError:
            self.info = ChunkInfo(filename)

        self.chunkSupport = True
        self.m = pycurl.CurlMulti()

        # needed for speed calculation
        self.lastArrived = []
        self.speeds = []
        self.lastSpeeds = [0, 0]

        self.progress = progress
예제 #2
0
    def __init__(self, url, filename, get={}, post={}, referer=None, cj=None, bucket=None,
                 options={}, progress=None, disposition=False):
        self.url = url
        self.filename = filename  #: complete file destination, not only name
        self.get = get
        self.post = post
        self.referer = referer
        self.cj = cj  #: cookiejar if cookies are needed
        self.bucket = bucket
        self.options = options
        self.disposition = disposition
        # all arguments

        self.abort = False
        self.size = 0
        self.nameDisposition = None  #: will be parsed from content disposition

        self.chunks = []

        self.log = logging.getLogger("log")

        try:
            self.info = ChunkInfo.load(filename)
            self.info.resume = True  #: resume is only possible with valid info file
            self.size = self.info.size
            self.infoSaved = True
        except IOError:
            self.info = ChunkInfo(filename)

        self.chunkSupport = True
        self.m = pycurl.CurlMulti()

        # needed for speed calculation
        self.lastArrived = []
        self.speeds = []
        self.lastSpeeds = [0, 0]

        self.progress = progress
예제 #3
0
class HTTPDownload(object):
    """ loads a url http + ftp """

    def __init__(self, url, filename, get={}, post={}, referer=None, cj=None, bucket=None,
                 options={}, progress=None, disposition=False):
        self.url = url
        self.filename = filename  #: complete file destination, not only name
        self.get = get
        self.post = post
        self.referer = referer
        self.cj = cj  #: cookiejar if cookies are needed
        self.bucket = bucket
        self.options = options
        self.disposition = disposition
        # all arguments

        self.abort = False
        self.size = 0
        self.nameDisposition = None  #: will be parsed from content disposition

        self.chunks = []

        self.log = logging.getLogger("log")

        try:
            self.info = ChunkInfo.load(filename)
            self.info.resume = True  #: resume is only possible with valid info file
            self.size = self.info.size
            self.infoSaved = True
        except IOError:
            self.info = ChunkInfo(filename)

        self.chunkSupport = True
        self.m = pycurl.CurlMulti()

        # needed for speed calculation
        self.lastArrived = []
        self.speeds = []
        self.lastSpeeds = [0, 0]

        self.progress = progress


    @property
    def speed(self):
        last = [sum(x) for x in self.lastSpeeds if x]
        return (sum(self.speeds) + sum(last)) / (1 + len(last))


    @property
    def arrived(self):
        return sum([c.arrived for c in self.chunks])


    @property
    def percent(self):
        return (self.arrived * 100) / self.size if self.size else 0

    def _copyChunks(self):
        init = fs_encode(self.info.getChunkName(0))  #: initial chunk name

        if self.info.getCount() > 1:
            with open(init, "rb+") as fo:  #: first chunkfile
                for i in xrange(1, self.info.getCount()):
                    # input file
                    fo.seek(
                        self.info.getChunkRange(i - 1)[1] + 1)  #: seek to beginning of chunk, to get rid of overlapping chunks
                    fname = fs_encode("%s.chunk%d" % (self.filename, i))
                    with open(fname, "rb") as fi:
                        buf = 32 * 1024
                        while True:  #: copy in chunks, consumes less memory
                            data = fi.read(buf)
                            if not data:
                                break
                            fo.write(data)
                    if fo.tell() < self.info.getChunkRange(i)[1]:
                        reshutil.move(init)
                        self.info.remove()  #: there are probably invalid chunks
                        raise Exception("Downloaded content was smaller than expected. Try to reduce download connections.")
                    reshutil.move(fname)  #: remove chunk

        if self.nameDisposition and self.disposition:
            self.filename = fs_join(os.path.dirname(self.filename), self.nameDisposition)

        shutil.move(init, fs_encode(self.filename))
        self.info.remove()  #: remove info file


    def download(self, chunks=1, resume=False):
        """ returns new filename or None """

        chunks = max(1, chunks)
        resume = self.info.resume and resume

        try:
            self._download(chunks, resume)
        except pycurl.error, e:
            # code 33 - no resume
            code = e.args[0]
            if resume is True and code == 33:
                # try again without resume
                self.log.debug("Errno 33 -> Restart without resume")

                # remove old handles
                for chunk in self.chunks:
                    self.closeChunk(chunk)

                return self._download(chunks, False)
            else:
                raise
        finally:
예제 #4
0
class HTTPDownload(object):
    """ loads a url http + ftp """
    def __init__(self,
                 url,
                 filename,
                 get={},
                 post={},
                 referer=None,
                 cj=None,
                 bucket=None,
                 options={},
                 progress=None,
                 disposition=False):
        self.url = url
        self.filename = filename  #: complete file destination, not only name
        self.get = get
        self.post = post
        self.referer = referer
        self.cj = cj  #: cookiejar if cookies are needed
        self.bucket = bucket
        self.options = options
        self.disposition = disposition
        # all arguments

        self.abort = False
        self.size = 0
        self.nameDisposition = None  #: will be parsed from content disposition

        self.chunks = []

        self.log = logging.getLogger("log")

        try:
            self.info = ChunkInfo.load(filename)
            self.info.resume = True  #: resume is only possible with valid info file
            self.size = self.info.size
            self.infoSaved = True
        except IOError:
            self.info = ChunkInfo(filename)

        self.chunkSupport = True
        self.m = pycurl.CurlMulti()

        # needed for speed calculation
        self.lastArrived = []
        self.speeds = []
        self.lastSpeeds = [0, 0]

        self.progress = progress

    @property
    def speed(self):
        last = [sum(x) for x in self.lastSpeeds if x]
        return (sum(self.speeds) + sum(last)) / (1 + len(last))

    @property
    def arrived(self):
        return sum([c.arrived for c in self.chunks])

    @property
    def percent(self):
        return (self.arrived * 100) / self.size if self.size else 0

    def _copyChunks(self):
        init = fs_encode(self.info.getChunkName(0))  #: initial chunk name

        if self.info.getCount() > 1:
            with open(init, "rb+") as fo:  #: first chunkfile
                for i in xrange(1, self.info.getCount()):
                    # input file
                    fo.seek(
                        self.info.getChunkRange(i - 1)[1] + 1
                    )  #: seek to beginning of chunk, to get rid of overlapping chunks
                    fname = fs_encode("%s.chunk%d" % (self.filename, i))
                    with open(fname, "rb") as fi:
                        buf = 32 * 1024
                        while True:  #: copy in chunks, consumes less memory
                            data = fi.read(buf)
                            if not data:
                                break
                            fo.write(data)
                    if fo.tell() < self.info.getChunkRange(i)[1]:
                        reshutil.move(init)
                        self.info.remove(
                        )  #: there are probably invalid chunks
                        raise Exception(
                            "Downloaded content was smaller than expected. Try to reduce download connections."
                        )
                    reshutil.move(fname)  #: remove chunk

        if self.nameDisposition and self.disposition:
            self.filename = fs_join(os.path.dirname(self.filename),
                                    self.nameDisposition)

        shutil.move(init, fs_encode(self.filename))
        self.info.remove()  #: remove info file

    def download(self, chunks=1, resume=False):
        """ returns new filename or None """

        chunks = max(1, chunks)
        resume = self.info.resume and resume

        try:
            self._download(chunks, resume)
        except pycurl.error, e:
            # code 33 - no resume
            code = e.args[0]
            if resume is True and code == 33:
                # try again without resume
                self.log.debug("Errno 33 -> Restart without resume")

                # remove old handles
                for chunk in self.chunks:
                    self.closeChunk(chunk)

                return self._download(chunks, False)
            else:
                raise
        finally: