def __init__(self, url, filename, get={}, post={}, referer=None, cj=None, bucket=None, options={}, progressNotify=None, disposition=False): self.url = url self.filename = filename #complete file destination, not only name self.get = get self.post = post self.referer = referer self.cj = cj #cookiejar if cookies are needed self.bucket = bucket self.options = options self.disposition = disposition # all arguments self.abort = False self.size = 0 self.nameDisposition = None #will be parsed from content disposition self.chunks = [] self.log = getLogger("log") try: self.info = ChunkInfo.load(filename) self.info.resume = True #resume is only possible with valid info file self.size = self.info.size self.infoSaved = True except IOError: self.info = ChunkInfo(filename) self.chunkSupport = None self.m = pycurl.CurlMulti() #needed for speed calculation self.lastArrived = [] self.speeds = [] self.lastSpeeds = [0, 0] self.progressNotify = progressNotify
def __init__(self, url, filename, get={}, post={}, referer=None, cj=None, bucket=None, options={}, disposition=False, callback=None, bf_callback=None, abort_callback=None, suspended=False, headers=None, gd=0): self.url = url self.filename = filename #complete file destination, not only name self.get = get self.post = post self.referer = referer self.cj = cj #cookiejar if cookies are needed self.bucket = bucket self.options = options self.disposition = disposition self.callback = callback self.bf_callback = bf_callback self.abort_callback = abort_callback self.suspended = suspended self.gd = gd # all arguments self.abort = False self.size = self.gd self.headers = headers self._name = ""# will be parsed from content disposition self.chunks = [] self.log = getLogger("log") try: self.info = ChunkInfo.load(filename) self.info.resume = True #resume is only possible with valid info file self.size = self.info.size self.infoSaved = True except IOError: self.info = ChunkInfo(filename) self.chunkSupport = None self.m = pycurl.CurlMulti() #needed for speed calculation self.lastArrived = [] self.speeds = [] self.lastSpeeds = [0, 0]
def __init__(self, url, filename, get=None, post=None, referer=None, cj=None, bucket=None, options=None, progressNotify=None, disposition=False): self.url = url self.filename = filename # complete file destination, not only name self.get = get or {} self.post = post or {} self.referer = referer self.cj = cj # cookiejar if cookies are needed self.bucket = bucket self.options = options or {} self.disposition = disposition # all arguments self.abort = False self.size = 0 self.nameDisposition = None # will be parsed from content disposition self.chunks = [] self.log = getLogger("log") try: self.info = ChunkInfo.load(filename) self.info.resume = True # resume is only possible with valid info file self.size = self.info.size self.infoSaved = True except IOError: self.info = ChunkInfo(filename) self.chunkSupport = None self.m = pycurl.CurlMulti() # needed for speed calculation self.lastArrived = [] self.speeds = [] self.lastSpeeds = [0, 0] self.progressNotify = progressNotify
class HTTPDownload(): """ loads a url http + ftp """ def __init__(self, url, filename, get={}, post={}, referer=None, cj=None, bucket=None, options={}, progressNotify=None, disposition=False): self.url = url self.filename = filename #complete file destination, not only name self.get = get self.post = post self.referer = referer self.cj = cj #cookiejar if cookies are needed self.bucket = bucket self.options = options self.disposition = disposition # all arguments self.abort = False self.size = 0 self.nameDisposition = None #will be parsed from content disposition self.chunks = [] self.log = getLogger("log") try: self.info = ChunkInfo.load(filename) self.info.resume = True #resume is only possible with valid info file self.size = self.info.size self.infoSaved = True except IOError: self.info = ChunkInfo(filename) self.chunkSupport = None self.m = pycurl.CurlMulti() #needed for speed calculation self.lastArrived = [] self.speeds = [] self.lastSpeeds = [0, 0] self.progressNotify = progressNotify @property def speed(self): last = [sum(x) for x in self.lastSpeeds if x] return (sum(self.speeds) + sum(last)) / (1 + len(last)) @property def arrived(self): return sum([c.arrived for c in self.chunks]) @property def percent(self): if not self.size: return 0 return (self.arrived * 100) / self.size def _copyChunks(self): init = fs_encode(self.info.getChunkName(0)) #initial chunk name if self.info.getCount() > 1: fo = open(init, "rb+") #first chunkfile for i in range(1, self.info.getCount()): #input file fo.seek( self.info.getChunkRange(i - 1)[1] + 1) #seek to beginning of chunk, to get rid of overlapping chunks fname = fs_encode("%s.chunk%d" % (self.filename, i)) fi = open(fname, "rb") buf = 32 * 1024 while True: #copy in chunks, consumes less memory data = fi.read(buf) if not data: break fo.write(data) fi.close() if fo.tell() < self.info.getChunkRange(i)[1]: fo.close() remove(init) self.info.remove() #there are probably invalid chunks raise Exception("Downloaded content was smaller than expected. Try to reduce download connections.") remove(fname) #remove chunk fo.close() if self.nameDisposition and self.disposition: self.filename = save_join(dirname(self.filename), self.nameDisposition) move(init, fs_encode(self.filename)) self.info.remove() #remove info file def download(self, chunks=1, resume=False): """ returns new filename or None """ chunks = max(1, chunks) resume = self.info.resume and resume try: self._download(chunks, resume) except pycurl.error, e: #code 33 - no resume code = e.args[0] if code == 33: # try again without resume self.log.debug("Errno 33 -> Restart without resume") #remove old handles for chunk in self.chunks: self.closeChunk(chunk) return self._download(chunks, False) else: raise finally:
class HTTPDownload(): """ loads a url http + ftp """ def __init__(self, url, filename, get={}, post={}, referer=None, cj=None, bucket=None, options={}, progressNotify=None, disposition=False): self.url = url self.filename = filename #complete file destination, not only name self.get = get self.post = post self.referer = referer self.cj = cj #cookiejar if cookies are needed self.bucket = bucket self.options = options self.disposition = disposition # all arguments self.abort = False self.size = 0 self.nameDisposition = None #will be parsed from content disposition self.chunks = [] self.log = getLogger("log") try: self.info = ChunkInfo.load(filename) self.info.resume = True #resume is only possible with valid info file self.size = self.info.size self.infoSaved = True except IOError: self.info = ChunkInfo(filename) self.chunkSupport = None self.m = pycurl.CurlMulti() #needed for speed calculation self.lastArrived = [] self.speeds = [] self.lastSpeeds = [0, 0] self.progressNotify = progressNotify @property def speed(self): last = [sum(x) for x in self.lastSpeeds if x] return (sum(self.speeds) + sum(last)) / (1 + len(last)) @property def arrived(self): return sum([c.arrived for c in self.chunks]) @property def percent(self): if not self.size: return 0 return (self.arrived * 100) / self.size def _copyChunks(self): init = fs_encode(self.info.getChunkName(0)) #initial chunk name if self.info.getCount() > 1: fo = open(init, "rb+") #first chunkfile for i in range(1, self.info.getCount()): #input file fo.seek( self.info.getChunkRange(i - 1)[1] + 1 ) #seek to beginning of chunk, to get rid of overlapping chunks fname = fs_encode("%s.chunk%d" % (self.filename, i)) fi = open(fname, "rb") buf = 32 * 1024 while True: #copy in chunks, consumes less memory data = fi.read(buf) if not data: break fo.write(data) fi.close() if fo.tell() < self.info.getChunkRange(i)[1]: fo.close() remove(init) self.info.remove() #there are probably invalid chunks raise Exception( "Downloaded content was smaller than expected. Try to reduce download connections." ) remove(fname) #remove chunk fo.close() if self.nameDisposition and self.disposition: self.filename = save_join(dirname(self.filename), self.nameDisposition) move(init, fs_encode(self.filename)) self.info.remove() #remove info file def download(self, chunks=1, resume=False): """ returns new filename or None """ chunks = max(1, chunks) resume = self.info.resume and resume try: self._download(chunks, resume) except pycurl.error, e: #code 33 - no resume code = e.args[0] if code == 33: # try again without resume self.log.debug("Errno 33 -> Restart without resume") #remove old handles for chunk in self.chunks: self.closeChunk(chunk) return self._download(chunks, False) else: raise finally: