def getSize(cls, url): if url.scheme.lower() == 'ftp': return None for attempt in retry_http(): with attempt: with closing(urlopen(url.geturl())) as readable: # just read the header for content length return int(readable.info().get('content-length'))
def _readFromUrl(cls, url, writable): for attempt in retry_http(): # We can only retry on errors that happen as responses to the request. # If we start getting file data, and the connection drops, we fail. # So we don't have to worry about writing the start of the file twice. with attempt: with closing(urlopen(url.geturl())) as readable: # Make something to count the bytes we get # We need to put the actual count in a container so our # nested function can modify it without creating its own # local with the same name. size = [0] def count(l): size[0] += l counter = WriteWatchingStream(writable) counter.onWrite(count) # Do the download shutil.copyfileobj(readable, counter) return size[0]
def _readFromUrl(cls, url, writable): for attempt in retry_http(): with attempt: with closing(urlopen(url.geturl())) as readable: shutil.copyfileobj(readable, writable)