def gateway_download(url,
target_full_path,
md5sum,
progress_update_callback=None):
timeout = (context.remote_connect_timeout_secs,
context.remote_read_timeout_secs)
session = CondaSession()
resp = session.get(url,
stream=True,
proxies=session.proxies,
timeout=timeout)
resp.raise_for_status()
content_length = int(resp.headers.get('Content-Length', 0))
digest_builder = hashlib.new('md5')
with open(target_full_path, 'wb') as fh:
streamed_bytes = 0
for chunk in resp.iter_content(2**14):
# chunk could be the decompressed form of the real data
# but we want the exact number of bytes read till now
streamed_bytes = resp.raw.tell()
try:
fh.write(chunk)
except IOError as e:
message = ("Failed to write to %(target_path)s\n "
"errno: %(errno)d")
# TODO: make this CondaIOError
raise CondaError(message,
target_path=target_full_path,
errno=e.errno)
digest_builder.update(chunk)
if (content_length and 0 <= streamed_bytes <= content_length):
if progress_update_callback:
progress_update_callback(streamed_bytes /
content_length)
if content_length and streamed_bytes != content_length:
# TODO: needs to be a more-specific error type
message = dals("""
Downloaded bytes did not match Content-Length
url: %(url)s
target_path: %(target_path)s
Content-Length: %(content_length)d
downloaded bytes: %(downloaded_bytes)d
""")
raise CondaError(message,
url=url,
target_path=target_full_path,
content_length=content_length,
downloaded_bytes=streamed_bytes)
actual_md5sum = digest_builder.hexdigest()
if md5sum and actual_md5sum != md5sum:
raise MD5MismatchError(url, target_full_path, md5sum,
actual_md5sum)
class RequestsTransport(Transport):
"""
Drop in Transport for xmlrpclib that uses Requests instead of httplib
"""
# change our user agent to reflect Requests
user_agent = "Python XMLRPC with Requests (python-requests.org)"
# override this if you'd like to https
use_https = True
session = CondaSession()
def request(self, host, handler, request_body, verbose):
"""
Make an xmlrpc request.
"""
headers = {
'User-Agent': self.user_agent,
'Content-Type': 'text/xml',
}
url = self._build_url(host, handler)
try:
resp = self.session.post(url,
data=request_body,
headers=headers,
proxies=self.session.proxies)
resp.raise_for_status()
except requests.exceptions.HTTPError as e:
if e.response.status_code == 407: # Proxy Authentication Required
handle_proxy_407(url, self.session)
# Try again
return self.request(host, handler, request_body, verbose)
else:
raise
except requests.exceptions.ConnectionError as e:
# requests isn't so nice here. For whatever reason, https gives this
# error and http gives the above error. Also, there is no status_code
# attribute here. We have to just check if it looks like 407. See
# https://github.com/kennethreitz/requests/issues/2061.
if "407" in str(e): # Proxy Authentication Required
handle_proxy_407(url, self.session)
# Try again
return self.request(host, handler, request_body, verbose)
else:
raise
except requests.RequestException as e:
raise ProtocolError(url, resp.status_code, str(e), resp.headers)
else:
return self.parse_response(resp)
def parse_response(self, resp):
"""
Parse the xmlrpc response.
"""
p, u = self.getparser()
p.feed(resp.text.encode("utf-8"))
p.close()
ret = u.close()
return ret
def _build_url(self, host, handler):
"""
Build a url for our request based on the host, handler and use_http
property
"""
scheme = 'https' if self.use_https else 'http'
return '%s://%s/%s' % (scheme, host, handler)