def __init__(self,
url,
filename,
get={},
post={},
referer=None,
cj=None,
bucket=None,
options={},
progressNotify=None,
disposition=False):
self.url = url
self.filename = filename #complete file destination, not only name
self.get = get
self.post = post
self.referer = referer
self.cj = cj #cookiejar if cookies are needed
self.bucket = bucket
self.options = options
self.disposition = disposition
# all arguments
self.abort = False
self.size = 0
self.nameDisposition = None #will be parsed from content disposition
self.chunks = []
self.log = getLogger("log")
try:
self.info = ChunkInfo.load(filename)
self.info.resume = True #resume is only possible with valid info file
self.size = self.info.size
self.infoSaved = True
except IOError:
self.info = ChunkInfo(filename)
self.chunkSupport = None
self.m = pycurl.CurlMulti()
#needed for speed calculation
self.lastArrived = []
self.speeds = []
self.lastSpeeds = [0, 0]
self.progressNotify = progressNotify
def __init__(self, url, filename, get={}, post={}, referer=None, cj=None, bucket=None,
options={}, disposition=False, callback=None, bf_callback=None, abort_callback=None, suspended=False,
headers=None, gd=0):
self.url = url
self.filename = filename #complete file destination, not only name
self.get = get
self.post = post
self.referer = referer
self.cj = cj #cookiejar if cookies are needed
self.bucket = bucket
self.options = options
self.disposition = disposition
self.callback = callback
self.bf_callback = bf_callback
self.abort_callback = abort_callback
self.suspended = suspended
self.gd = gd
# all arguments
self.abort = False
self.size = self.gd
self.headers = headers
self._name = ""# will be parsed from content disposition
self.chunks = []
self.log = getLogger("log")
try:
self.info = ChunkInfo.load(filename)
self.info.resume = True #resume is only possible with valid info file
self.size = self.info.size
self.infoSaved = True
except IOError:
self.info = ChunkInfo(filename)
self.chunkSupport = None
self.m = pycurl.CurlMulti()
#needed for speed calculation
self.lastArrived = []
self.speeds = []
self.lastSpeeds = [0, 0]
def __init__(self, url, filename, get=None, post=None, referer=None, cj=None, bucket=None,
options=None, progressNotify=None, disposition=False):
self.url = url
self.filename = filename # complete file destination, not only name
self.get = get or {}
self.post = post or {}
self.referer = referer
self.cj = cj # cookiejar if cookies are needed
self.bucket = bucket
self.options = options or {}
self.disposition = disposition
# all arguments
self.abort = False
self.size = 0
self.nameDisposition = None # will be parsed from content disposition
self.chunks = []
self.log = getLogger("log")
try:
self.info = ChunkInfo.load(filename)
self.info.resume = True # resume is only possible with valid info file
self.size = self.info.size
self.infoSaved = True
except IOError:
self.info = ChunkInfo(filename)
self.chunkSupport = None
self.m = pycurl.CurlMulti()
# needed for speed calculation
self.lastArrived = []
self.speeds = []
self.lastSpeeds = [0, 0]
self.progressNotify = progressNotify
class HTTPDownload():
""" loads a url http + ftp """
def __init__(self, url, filename, get={}, post={}, referer=None, cj=None, bucket=None,
options={}, progressNotify=None, disposition=False):
self.url = url
self.filename = filename #complete file destination, not only name
self.get = get
self.post = post
self.referer = referer
self.cj = cj #cookiejar if cookies are needed
self.bucket = bucket
self.options = options
self.disposition = disposition
# all arguments
self.abort = False
self.size = 0
self.nameDisposition = None #will be parsed from content disposition
self.chunks = []
self.log = getLogger("log")
try:
self.info = ChunkInfo.load(filename)
self.info.resume = True #resume is only possible with valid info file
self.size = self.info.size
self.infoSaved = True
except IOError:
self.info = ChunkInfo(filename)
self.chunkSupport = None
self.m = pycurl.CurlMulti()
#needed for speed calculation
self.lastArrived = []
self.speeds = []
self.lastSpeeds = [0, 0]
self.progressNotify = progressNotify
@property
def speed(self):
last = [sum(x) for x in self.lastSpeeds if x]
return (sum(self.speeds) + sum(last)) / (1 + len(last))
@property
def arrived(self):
return sum([c.arrived for c in self.chunks])
@property
def percent(self):
if not self.size: return 0
return (self.arrived * 100) / self.size
def _copyChunks(self):
init = fs_encode(self.info.getChunkName(0)) #initial chunk name
if self.info.getCount() > 1:
fo = open(init, "rb+") #first chunkfile
for i in range(1, self.info.getCount()):
#input file
fo.seek(
self.info.getChunkRange(i - 1)[1] + 1) #seek to beginning of chunk, to get rid of overlapping chunks
fname = fs_encode("%s.chunk%d" % (self.filename, i))
fi = open(fname, "rb")
buf = 32 * 1024
while True: #copy in chunks, consumes less memory
data = fi.read(buf)
if not data:
break
fo.write(data)
fi.close()
if fo.tell() < self.info.getChunkRange(i)[1]:
fo.close()
remove(init)
self.info.remove() #there are probably invalid chunks
raise Exception("Downloaded content was smaller than expected. Try to reduce download connections.")
remove(fname) #remove chunk
fo.close()
if self.nameDisposition and self.disposition:
self.filename = save_join(dirname(self.filename), self.nameDisposition)
move(init, fs_encode(self.filename))
self.info.remove() #remove info file
def download(self, chunks=1, resume=False):
""" returns new filename or None """
chunks = max(1, chunks)
resume = self.info.resume and resume
try:
self._download(chunks, resume)
except pycurl.error, e:
#code 33 - no resume
code = e.args[0]
if code == 33:
# try again without resume
self.log.debug("Errno 33 -> Restart without resume")
#remove old handles
for chunk in self.chunks:
self.closeChunk(chunk)
return self._download(chunks, False)
else:
raise
finally:
class HTTPDownload():
""" loads a url http + ftp """
def __init__(self,
url,
filename,
get={},
post={},
referer=None,
cj=None,
bucket=None,
options={},
progressNotify=None,
disposition=False):
self.url = url
self.filename = filename #complete file destination, not only name
self.get = get
self.post = post
self.referer = referer
self.cj = cj #cookiejar if cookies are needed
self.bucket = bucket
self.options = options
self.disposition = disposition
# all arguments
self.abort = False
self.size = 0
self.nameDisposition = None #will be parsed from content disposition
self.chunks = []
self.log = getLogger("log")
try:
self.info = ChunkInfo.load(filename)
self.info.resume = True #resume is only possible with valid info file
self.size = self.info.size
self.infoSaved = True
except IOError:
self.info = ChunkInfo(filename)
self.chunkSupport = None
self.m = pycurl.CurlMulti()
#needed for speed calculation
self.lastArrived = []
self.speeds = []
self.lastSpeeds = [0, 0]
self.progressNotify = progressNotify
@property
def speed(self):
last = [sum(x) for x in self.lastSpeeds if x]
return (sum(self.speeds) + sum(last)) / (1 + len(last))
@property
def arrived(self):
return sum([c.arrived for c in self.chunks])
@property
def percent(self):
if not self.size: return 0
return (self.arrived * 100) / self.size
def _copyChunks(self):
init = fs_encode(self.info.getChunkName(0)) #initial chunk name
if self.info.getCount() > 1:
fo = open(init, "rb+") #first chunkfile
for i in range(1, self.info.getCount()):
#input file
fo.seek(
self.info.getChunkRange(i - 1)[1] + 1
) #seek to beginning of chunk, to get rid of overlapping chunks
fname = fs_encode("%s.chunk%d" % (self.filename, i))
fi = open(fname, "rb")
buf = 32 * 1024
while True: #copy in chunks, consumes less memory
data = fi.read(buf)
if not data:
break
fo.write(data)
fi.close()
if fo.tell() < self.info.getChunkRange(i)[1]:
fo.close()
remove(init)
self.info.remove() #there are probably invalid chunks
raise Exception(
"Downloaded content was smaller than expected. Try to reduce download connections."
)
remove(fname) #remove chunk
fo.close()
if self.nameDisposition and self.disposition:
self.filename = save_join(dirname(self.filename),
self.nameDisposition)
move(init, fs_encode(self.filename))
self.info.remove() #remove info file
def download(self, chunks=1, resume=False):
""" returns new filename or None """
chunks = max(1, chunks)
resume = self.info.resume and resume
try:
self._download(chunks, resume)
except pycurl.error, e:
#code 33 - no resume
code = e.args[0]
if code == 33:
# try again without resume
self.log.debug("Errno 33 -> Restart without resume")
#remove old handles
for chunk in self.chunks:
self.closeChunk(chunk)
return self._download(chunks, False)
else:
raise
finally:
