def gateway_download(url, target_full_path, md5sum, progress_update_callback=None): timeout = (context.remote_connect_timeout_secs, context.remote_read_timeout_secs) session = CondaSession() resp = session.get(url, stream=True, proxies=session.proxies, timeout=timeout) resp.raise_for_status() content_length = int(resp.headers.get('Content-Length', 0)) digest_builder = hashlib.new('md5') with open(target_full_path, 'wb') as fh: streamed_bytes = 0 for chunk in resp.iter_content(2**14): # chunk could be the decompressed form of the real data # but we want the exact number of bytes read till now streamed_bytes = resp.raw.tell() try: fh.write(chunk) except IOError as e: message = ("Failed to write to %(target_path)s\n " "errno: %(errno)d") # TODO: make this CondaIOError raise CondaError(message, target_path=target_full_path, errno=e.errno) digest_builder.update(chunk) if (content_length and 0 <= streamed_bytes <= content_length): if progress_update_callback: progress_update_callback(streamed_bytes / content_length) if content_length and streamed_bytes != content_length: # TODO: needs to be a more-specific error type message = dals(""" Downloaded bytes did not match Content-Length url: %(url)s target_path: %(target_path)s Content-Length: %(content_length)d downloaded bytes: %(downloaded_bytes)d """) raise CondaError(message, url=url, target_path=target_full_path, content_length=content_length, downloaded_bytes=streamed_bytes) actual_md5sum = digest_builder.hexdigest() if md5sum and actual_md5sum != md5sum: raise MD5MismatchError(url, target_full_path, md5sum, actual_md5sum)
class RequestsTransport(Transport): """ Drop in Transport for xmlrpclib that uses Requests instead of httplib """ # change our user agent to reflect Requests user_agent = "Python XMLRPC with Requests (python-requests.org)" # override this if you'd like to https use_https = True session = CondaSession() def request(self, host, handler, request_body, verbose): """ Make an xmlrpc request. """ headers = { 'User-Agent': self.user_agent, 'Content-Type': 'text/xml', } url = self._build_url(host, handler) try: resp = self.session.post(url, data=request_body, headers=headers, proxies=self.session.proxies) resp.raise_for_status() except requests.exceptions.HTTPError as e: if e.response.status_code == 407: # Proxy Authentication Required handle_proxy_407(url, self.session) # Try again return self.request(host, handler, request_body, verbose) else: raise except requests.exceptions.ConnectionError as e: # requests isn't so nice here. For whatever reason, https gives this # error and http gives the above error. Also, there is no status_code # attribute here. We have to just check if it looks like 407. See # https://github.com/kennethreitz/requests/issues/2061. if "407" in str(e): # Proxy Authentication Required handle_proxy_407(url, self.session) # Try again return self.request(host, handler, request_body, verbose) else: raise except requests.RequestException as e: raise ProtocolError(url, resp.status_code, str(e), resp.headers) else: return self.parse_response(resp) def parse_response(self, resp): """ Parse the xmlrpc response. """ p, u = self.getparser() p.feed(resp.text.encode("utf-8")) p.close() ret = u.close() return ret def _build_url(self, host, handler): """ Build a url for our request based on the host, handler and use_http property """ scheme = 'https' if self.use_https else 'http' return '%s://%s/%s' % (scheme, host, handler)