def get_data_from_url_with_requests(self, url, json_data, method=None): import requests username, password = self._get_auth() auth = requests.auth.HTTPDigestAuth(username, password) if not hasattr(self, "session"): self.session = requests.Session() if method is None: method = self.METHOD if method == self.METHOD_POST: session_method = self.session.post else: session_method = self.session.get try: ret = session_method( url, json=json_data, verify=self._get_verify(), proxies={}, headers=self._get_headers(), auth=auth, timeout=self.timeout ) except requests.exceptions.SSLError as exc: if "unknown protocol" in str(exc) and url.startswith("https:"): # Server is using http rather than https, for some reason. sys.stderr.write(WARNING_NO_HTTPS_SUPPORT.format(exc)) return self.get_data_from_url_with_requests( url.replace("https:", "http:", 1), json_data) if cylc.flags.debug: import traceback traceback.print_exc() raise ConnectionError(url, exc) except Exception as exc: if cylc.flags.debug: import traceback traceback.print_exc() raise ConnectionError(url, exc) if ret.status_code == 401: raise ConnectionDeniedError(url, self.prog_name, self.ACCESS_DESCRIPTION) if ret.status_code >= 400: from cylc.network.https.util import get_exception_from_html exception_text = get_exception_from_html(ret.text) if exception_text: sys.stderr.write(exception_text) else: sys.stderr.write(ret.text) try: ret.raise_for_status() except Exception as exc: if cylc.flags.debug: import traceback traceback.print_exc() raise ConnectionError(url, exc) try: return ret.json() except ValueError: return ret.text
def get_data_from_url_with_urllib2(self, url, json_data, method=None): import json import urllib2 import ssl if hasattr(ssl, '_create_unverified_context'): ssl._create_default_https_context = ssl._create_unverified_context if method is None: method = self.METHOD orig_json_data = json_data username, password = self._get_auth() auth_manager = urllib2.HTTPPasswordMgrWithDefaultRealm() auth_manager.add_password(None, url, username, password) auth = urllib2.HTTPDigestAuthHandler(auth_manager) opener = urllib2.build_opener(auth, urllib2.HTTPSHandler()) headers_list = self._get_headers().items() if json_data: json_data = json.dumps(json_data) headers_list.append(('Accept', 'application/json')) json_headers = {'Content-Type': 'application/json', 'Content-Length': len(json_data)} else: json_data = None json_headers = {'Content-Length': 0} opener.addheaders = headers_list req = urllib2.Request(url, json_data, json_headers) # This is an unpleasant monkey patch, but there isn't an alternative. # urllib2 uses POST iff there is a data payload, but that is not the # correct criterion. The difference is basically that POST changes # server state and GET doesn't. req.get_method = lambda: method try: response = opener.open(req, timeout=self.timeout) except urllib2.URLError as exc: if "unknown protocol" in str(exc) and url.startswith("https:"): # Server is using http rather than https, for some reason. sys.stderr.write(WARNING_NO_HTTPS_SUPPORT.format(exc)) return self.get_data_from_url_with_urllib2( url.replace("https:", "http:", 1), orig_json_data) if cylc.flags.debug: import traceback traceback.print_exc() if "timed out" in str(exc): raise ConnectionTimeout(url, exc) else: raise ConnectionError(url, exc) except Exception as exc: if cylc.flags.debug: import traceback traceback.print_exc() raise ConnectionError(url, exc) if response.getcode() == 401: raise ConnectionDeniedError(url, self.prog_name, self.ACCESS_DESCRIPTION) response_text = response.read() if response.getcode() >= 400: from cylc.network.https.util import get_exception_from_html exception_text = get_exception_from_html(response_text) if exception_text: sys.stderr.write(exception_text) else: sys.stderr.write(response_text) raise ConnectionError(url, "%s HTTP return code" % response.getcode()) try: return json.loads(response_text) except ValueError: return response_text
def _get_data_from_url_with_urllib2(self, url, json_data, method=None): import json import urllib2 import ssl if hasattr(ssl, '_create_unverified_context'): ssl._create_default_https_context = ssl._create_unverified_context if method is None: method = self.METHOD orig_json_data = json_data username, password = self._get_auth() auth_manager = urllib2.HTTPPasswordMgrWithDefaultRealm() auth_manager.add_password(None, url, username, password) auth = urllib2.HTTPDigestAuthHandler(auth_manager) opener = urllib2.build_opener(auth, urllib2.HTTPSHandler()) headers_list = self._get_headers().items() if json_data: json_data = json.dumps(json_data) headers_list.append(('Accept', 'application/json')) json_headers = {'Content-Type': 'application/json', 'Content-Length': len(json_data)} else: json_data = None json_headers = {'Content-Length': 0} opener.addheaders = headers_list req = urllib2.Request(url, json_data, json_headers) # This is an unpleasant monkey patch, but there isn't an alternative. # urllib2 uses POST if there is a data payload, but that is not the # correct criterion. The difference is basically that POST changes # server state and GET doesn't. req.get_method = lambda: method try: response = opener.open(req, timeout=self.timeout) except urllib2.URLError as exc: if "unknown protocol" in str(exc) and url.startswith("https:"): # Server is using http rather than https, for some reason. sys.stderr.write(WARNING_NO_HTTPS_SUPPORT.format(exc)) return self._get_data_from_url_with_urllib2( url.replace("https:", "http:", 1), orig_json_data) if cylc.flags.debug: import traceback traceback.print_exc() if "timed out" in str(exc): raise ConnectionTimeout(url, exc) else: raise ConnectionError(url, exc) except Exception as exc: if cylc.flags.debug: import traceback traceback.print_exc() raise ConnectionError(url, exc) if response.getcode() == 401: raise ConnectionDeniedError(url, self.prog_name, self.ACCESS_DESCRIPTION) response_text = response.read() if response.getcode() >= 400: from cylc.network.https.util import get_exception_from_html exception_text = get_exception_from_html(response_text) if exception_text: sys.stderr.write(exception_text) else: sys.stderr.write(response_text) raise ConnectionError(url, "%s HTTP return code" % response.getcode()) if self.auth and self.auth[1] != NO_PASSPHRASE: self.srv_files_mgr.cache_passphrase( self.suite, self.owner, self.host, self.auth[1]) try: return json.loads(response_text) except ValueError: return response_text
def _get_data_from_url_with_requests(self, url, json_data, method=None): import requests username, password = self._get_auth() auth = requests.auth.HTTPDigestAuth(username, password) if not hasattr(self, "session"): self.session = requests.Session() if method is None: method = self.METHOD if method == self.METHOD_POST: session_method = self.session.post else: session_method = self.session.get try: ret = session_method( url, json=json_data, verify=self._get_verify(), proxies={}, headers=self._get_headers(), auth=auth, timeout=self.timeout ) except requests.exceptions.SSLError as exc: if "unknown protocol" in str(exc) and url.startswith("https:"): # Server is using http rather than https, for some reason. sys.stderr.write(WARNING_NO_HTTPS_SUPPORT.format(exc)) return self._get_data_from_url_with_requests( url.replace("https:", "http:", 1), json_data) if cylc.flags.debug: import traceback traceback.print_exc() raise ConnectionError(url, exc) except requests.exceptions.Timeout as exc: if cylc.flags.debug: import traceback traceback.print_exc() raise ConnectionTimeout(url, exc) except requests.exceptions.RequestException as exc: if cylc.flags.debug: import traceback traceback.print_exc() raise ConnectionError(url, exc) if ret.status_code == 401: raise ConnectionDeniedError(url, self.prog_name, self.ACCESS_DESCRIPTION) if ret.status_code >= 400: from cylc.network.https.util import get_exception_from_html exception_text = get_exception_from_html(ret.text) if exception_text: sys.stderr.write(exception_text) else: sys.stderr.write(ret.text) try: ret.raise_for_status() except requests.exceptions.HTTPError as exc: if cylc.flags.debug: import traceback traceback.print_exc() raise ConnectionError(url, exc) if self.auth and self.auth[1] != NO_PASSPHRASE: self.srv_files_mgr.cache_passphrase( self.suite, self.owner, self.host, self.auth[1]) try: return ret.json() except ValueError: return ret.text