def downloadFromList(alist, directory=".", timeout=10):
"""Get files from a list of urls.
return : list, contained the failure fetch"""
failure = []
for url in alist:
print alist.index(url),
stream = getStream(url, timeout=timeout)
file_name = getFilenameFromURL(url)
if not stream or not writeBinFile(stream, file_name, directory):
failure.append(url)
return failure
def downloadFromQueue(queue, failure, directory='.', timeout=10):
"""Get files from a list of urls.
return : list, contained the failure fetch"""
while not queue.empty():
url = queue.get()
stream = getStream(url, timeout=timeout)
file_name = getFilenameFromURL(url)
if stream and writeBinFile(stream, file_name, directory):
queue.task_done()
print "Fetching", url, 'done.'
continue
failure.append(url)
queue.task_done()
return failure
def run(self):
stream = getStream(self.url, timeout=self.timeout)
file_name = getFilenameFromURL(self.url)
if not stream or not writeBinFile(stream, file_name, self.directory):
self.failure.append(self.url)
self.finished = True
