def download_in_thread(url, target_file, if_modified_since, notify_done):
try:
#print "Child downloading", url
if url.startswith('http:') or url.startswith('https:') or url.startswith('ftp:'):
req = urllib2.Request(url)
if url.startswith('http:') and if_modified_since:
req.add_header('If-Modified-Since', if_modified_since)
src = _my_urlopen.open(req)
else:
raise Exception(_('Unsupported URL protocol in: %s') % url)
try:
sock = src.fp._sock
except AttributeError:
sock = src.fp.fp._sock # Python 2.5 on FreeBSD
while True:
data = sock.recv(256)
if not data: break
target_file.write(data)
target_file.flush()
notify_done(download.RESULT_OK)
except (urllib2.HTTPError, urllib2.URLError, httplib.HTTPException, socket.error) as ex:
if isinstance(ex, urllib2.HTTPError) and ex.code == 304: # Not modified
notify_done(download.RESULT_NOT_MODIFIED)
else:
#print >>sys.stderr, "Error downloading '" + url + "': " + (str(ex) or str(ex.__class__.__name__))
__, ex, tb = sys.exc_info()
notify_done(download.RESULT_FAILED, (download.DownloadError(_('Error downloading {url}: {ex}').format(url = url, ex = ex)), tb))
except Redirect as ex:
notify_done(download.RESULT_REDIRECT, redirect = ex.req.get_full_url())
except Exception as ex:
__, ex, tb = sys.exc_info()
notify_done(download.RESULT_FAILED, (ex, tb))
def download(self, dl):
# (changed if we get redirected)
current_url = dl.url
redirections_remaining = 10
# Assign the Download to a Site based on its scheme, host and port. If the result is a redirect,
# reassign it to the appropriate new site. Note that proxy handling happens later; we want to group
# and limit by the target site, not treat everything as going to a single site (the proxy).
while True:
location_parts = urlparse.urlparse(current_url)
site_key = (location_parts.scheme, location_parts.hostname,
location_parts.port
or default_port.get(location_parts.scheme, None))
step = DownloadStep()
step.dl = dl
step.url = current_url
blocker = self._sites[site_key].download(step)
yield blocker
tasks.check(blocker)
if not step.redirect:
break
current_url = step.redirect
if redirections_remaining == 0:
raise download.DownloadError(
"Too many redirections {url} -> {current}".format(
url=dl.url, current=current_url))
redirections_remaining -= 1
def download(self, dl, timeout=None):
"""@type dl: L{zeroinstall.injector.download.Download}"""
# (changed if we get redirected)
current_url = dl.url
redirections_remaining = 10
original_exception = None
# Assign the Download to a Site based on its scheme, host and port. If the result is a redirect,
# reassign it to the appropriate new site. Note that proxy handling happens later; we want to group
# and limit by the target site, not treat everything as going to a single site (the proxy).
while True:
location_parts = urlparse.urlparse(current_url)
site_key = (location_parts.scheme, location_parts.hostname,
location_parts.port
or default_port.get(location_parts.scheme, None))
step = DownloadStep()
step.dl = dl
step.url = current_url
blocker = self._sites[site_key].download(step, timeout)
yield blocker
try:
tasks.check(blocker)
except download.DownloadError as ex:
if original_exception is None:
original_exception = ex
else:
logger.warning("%s (while trying mirror)", ex)
mirror_url = step.dl.get_next_mirror_url()
if mirror_url is None:
raise original_exception
# Try the mirror.
# There are actually two places where we try to use the mirror: this one
# looks to see if we have an exact copy of same file somewhere else. If this
# fails, Fetcher will also look for a different archive that would generate
# the required implementation.
logger.warning("%s: trying archive mirror at %s", ex,
mirror_url)
step.redirect = mirror_url
redirections_remaining = 10
if not step.redirect:
break
current_url = step.redirect
if redirections_remaining == 0:
raise download.DownloadError(
"Too many redirections {url} -> {current}".format(
url=dl.url, current=current_url))
redirections_remaining -= 1
#Read data 256 bytes at a time. This is an optmization in case of large packages
data = doc.read(256)
except IncompleteRead as ex:
data = ex.partial
if not data: break
target_file.write(data)
target_file.flush()
'''
data = doc.read()
target_file.write(data)
target_file.flush()
#End of Safe Download Code
notify_done(download.RESULT_OK)
except (urllib2.HTTPError, urllib2.URLError, HTTPException, socket.error) as ex:
if isinstance(ex, urllib2.HTTPError) and ex.code == 304: # Not modified
notify_done(download.RESULT_NOT_MODIFIED)
else:
#print >>sys.stderr, "Error downloading '" + url + "': " + (str(ex) or str(ex.__class__.__name__))
__, ex, tb = sys.exc_info()
notify_done(download.RESULT_FAILED, (download.DownloadError(_('Error downloading {url}: {ex}').format(url = url, ex = ex)), tb))
except Redirect as ex:
notify_done(download.RESULT_REDIRECT, redirect = ex.req.get_full_url())
except Exception as ex:
__, ex, tb = sys.exc_info()
notify_done(download.RESULT_FAILED, (ex, tb))
finally:
if src is not None:
src.close()