def getSize(cls, url):
if url.scheme.lower() == 'ftp':
return None
for attempt in retry_http():
with attempt:
with closing(urlopen(url.geturl())) as readable:
# just read the header for content length
return int(readable.info().get('content-length'))
def _readFromUrl(cls, url, writable):
for attempt in retry_http():
# We can only retry on errors that happen as responses to the request.
# If we start getting file data, and the connection drops, we fail.
# So we don't have to worry about writing the start of the file twice.
with attempt:
with closing(urlopen(url.geturl())) as readable:
# Make something to count the bytes we get
# We need to put the actual count in a container so our
# nested function can modify it without creating its own
# local with the same name.
size = [0]
def count(l):
size[0] += l
counter = WriteWatchingStream(writable)
counter.onWrite(count)
# Do the download
shutil.copyfileobj(readable, counter)
return size[0]
def _readFromUrl(cls, url, writable):
for attempt in retry_http():
with attempt:
with closing(urlopen(url.geturl())) as readable:
shutil.copyfileobj(readable, writable)
