def embedded_fetch(url):
name = utils.file_name_from_url(url)
path, handle = utils.file_path_and_handle(self.storage_dir, name)
body = urllib2.urlopen(url).read()
handle.write(body)
handle.close()
return name
def fetch_multiple(self, url_list):
multi_handle = pycurl.CurlMulti()
requests = []
files = []
for url in url_list:
file_name = utils.file_name_from_url(url)
file_path, file_handle = utils.file_path_and_handle(self.storage_dir, file_name)
files.append(file_path)
easy_handle = pycurl.Curl()
easy_handle.setopt(pycurl.URL, url)
easy_handle.setopt(pycurl.WRITEFUNCTION, file_handle.write)
req = (url, file_handle, easy_handle)
multi_handle.add_handle(req[2])
requests.append(req)
num_handles = len(requests)
while num_handles:
ret = multi_handle.select(SELECT_TIMEOUT)
if ret == -1:
continue
while True:
ret, num_handles = multi_handle.perform()
if ret != pycurl.E_CALL_MULTI_PERFORM:
break
for req in requests:
req[1].close()
return files