def __init__(self,
url,
filename,
get={},
post={},
referer=None,
cj=None,
bucket=None,
options={},
progress=None,
disposition=False):
self.url = url
self.filename = filename #: complete file destination, not only name
self.get = get
self.post = post
self.referer = referer
self.cj = cj #: cookiejar if cookies are needed
self.bucket = bucket
self.options = options
self.disposition = disposition
# all arguments
self.abort = False
self.size = 0
self.nameDisposition = None #: will be parsed from content disposition
self.chunks = []
self.log = logging.getLogger("log")
try:
self.info = ChunkInfo.load(filename)
self.info.resume = True #: resume is only possible with valid info file
self.size = self.info.size
self.infoSaved = True
except IOError:
self.info = ChunkInfo(filename)
self.chunkSupport = True
self.m = pycurl.CurlMulti()
# needed for speed calculation
self.lastArrived = []
self.speeds = []
self.lastSpeeds = [0, 0]
self.progress = progress
def __init__(self, url, filename, get={}, post={}, referer=None, cj=None, bucket=None,
options={}, progress=None, disposition=False):
self.url = url
self.filename = filename #: complete file destination, not only name
self.get = get
self.post = post
self.referer = referer
self.cj = cj #: cookiejar if cookies are needed
self.bucket = bucket
self.options = options
self.disposition = disposition
# all arguments
self.abort = False
self.size = 0
self.nameDisposition = None #: will be parsed from content disposition
self.chunks = []
self.log = logging.getLogger("log")
try:
self.info = ChunkInfo.load(filename)
self.info.resume = True #: resume is only possible with valid info file
self.size = self.info.size
self.infoSaved = True
except IOError:
self.info = ChunkInfo(filename)
self.chunkSupport = True
self.m = pycurl.CurlMulti()
# needed for speed calculation
self.lastArrived = []
self.speeds = []
self.lastSpeeds = [0, 0]
self.progress = progress
class HTTPDownload(object):
""" loads a url http + ftp """
def __init__(self, url, filename, get={}, post={}, referer=None, cj=None, bucket=None,
options={}, progress=None, disposition=False):
self.url = url
self.filename = filename #: complete file destination, not only name
self.get = get
self.post = post
self.referer = referer
self.cj = cj #: cookiejar if cookies are needed
self.bucket = bucket
self.options = options
self.disposition = disposition
# all arguments
self.abort = False
self.size = 0
self.nameDisposition = None #: will be parsed from content disposition
self.chunks = []
self.log = logging.getLogger("log")
try:
self.info = ChunkInfo.load(filename)
self.info.resume = True #: resume is only possible with valid info file
self.size = self.info.size
self.infoSaved = True
except IOError:
self.info = ChunkInfo(filename)
self.chunkSupport = True
self.m = pycurl.CurlMulti()
# needed for speed calculation
self.lastArrived = []
self.speeds = []
self.lastSpeeds = [0, 0]
self.progress = progress
@property
def speed(self):
last = [sum(x) for x in self.lastSpeeds if x]
return (sum(self.speeds) + sum(last)) / (1 + len(last))
@property
def arrived(self):
return sum([c.arrived for c in self.chunks])
@property
def percent(self):
return (self.arrived * 100) / self.size if self.size else 0
def _copyChunks(self):
init = fs_encode(self.info.getChunkName(0)) #: initial chunk name
if self.info.getCount() > 1:
with open(init, "rb+") as fo: #: first chunkfile
for i in xrange(1, self.info.getCount()):
# input file
fo.seek(
self.info.getChunkRange(i - 1)[1] + 1) #: seek to beginning of chunk, to get rid of overlapping chunks
fname = fs_encode("%s.chunk%d" % (self.filename, i))
with open(fname, "rb") as fi:
buf = 32 * 1024
while True: #: copy in chunks, consumes less memory
data = fi.read(buf)
if not data:
break
fo.write(data)
if fo.tell() < self.info.getChunkRange(i)[1]:
reshutil.move(init)
self.info.remove() #: there are probably invalid chunks
raise Exception("Downloaded content was smaller than expected. Try to reduce download connections.")
reshutil.move(fname) #: remove chunk
if self.nameDisposition and self.disposition:
self.filename = fs_join(os.path.dirname(self.filename), self.nameDisposition)
shutil.move(init, fs_encode(self.filename))
self.info.remove() #: remove info file
def download(self, chunks=1, resume=False):
""" returns new filename or None """
chunks = max(1, chunks)
resume = self.info.resume and resume
try:
self._download(chunks, resume)
except pycurl.error, e:
# code 33 - no resume
code = e.args[0]
if resume is True and code == 33:
# try again without resume
self.log.debug("Errno 33 -> Restart without resume")
# remove old handles
for chunk in self.chunks:
self.closeChunk(chunk)
return self._download(chunks, False)
else:
raise
finally:
class HTTPDownload(object):
""" loads a url http + ftp """
def __init__(self,
url,
filename,
get={},
post={},
referer=None,
cj=None,
bucket=None,
options={},
progress=None,
disposition=False):
self.url = url
self.filename = filename #: complete file destination, not only name
self.get = get
self.post = post
self.referer = referer
self.cj = cj #: cookiejar if cookies are needed
self.bucket = bucket
self.options = options
self.disposition = disposition
# all arguments
self.abort = False
self.size = 0
self.nameDisposition = None #: will be parsed from content disposition
self.chunks = []
self.log = logging.getLogger("log")
try:
self.info = ChunkInfo.load(filename)
self.info.resume = True #: resume is only possible with valid info file
self.size = self.info.size
self.infoSaved = True
except IOError:
self.info = ChunkInfo(filename)
self.chunkSupport = True
self.m = pycurl.CurlMulti()
# needed for speed calculation
self.lastArrived = []
self.speeds = []
self.lastSpeeds = [0, 0]
self.progress = progress
@property
def speed(self):
last = [sum(x) for x in self.lastSpeeds if x]
return (sum(self.speeds) + sum(last)) / (1 + len(last))
@property
def arrived(self):
return sum([c.arrived for c in self.chunks])
@property
def percent(self):
return (self.arrived * 100) / self.size if self.size else 0
def _copyChunks(self):
init = fs_encode(self.info.getChunkName(0)) #: initial chunk name
if self.info.getCount() > 1:
with open(init, "rb+") as fo: #: first chunkfile
for i in xrange(1, self.info.getCount()):
# input file
fo.seek(
self.info.getChunkRange(i - 1)[1] + 1
) #: seek to beginning of chunk, to get rid of overlapping chunks
fname = fs_encode("%s.chunk%d" % (self.filename, i))
with open(fname, "rb") as fi:
buf = 32 * 1024
while True: #: copy in chunks, consumes less memory
data = fi.read(buf)
if not data:
break
fo.write(data)
if fo.tell() < self.info.getChunkRange(i)[1]:
reshutil.move(init)
self.info.remove(
) #: there are probably invalid chunks
raise Exception(
"Downloaded content was smaller than expected. Try to reduce download connections."
)
reshutil.move(fname) #: remove chunk
if self.nameDisposition and self.disposition:
self.filename = fs_join(os.path.dirname(self.filename),
self.nameDisposition)
shutil.move(init, fs_encode(self.filename))
self.info.remove() #: remove info file
def download(self, chunks=1, resume=False):
""" returns new filename or None """
chunks = max(1, chunks)
resume = self.info.resume and resume
try:
self._download(chunks, resume)
except pycurl.error, e:
# code 33 - no resume
code = e.args[0]
if resume is True and code == 33:
# try again without resume
self.log.debug("Errno 33 -> Restart without resume")
# remove old handles
for chunk in self.chunks:
self.closeChunk(chunk)
return self._download(chunks, False)
else:
raise
finally:
