def get_data_from_url_with_requests(self, url, json_data, method=None):
import requests
username, password = self._get_auth()
auth = requests.auth.HTTPDigestAuth(username, password)
if not hasattr(self, "session"):
self.session = requests.Session()
if method is None:
method = self.METHOD
if method == self.METHOD_POST:
session_method = self.session.post
else:
session_method = self.session.get
try:
ret = session_method(
url,
json=json_data,
verify=self._get_verify(),
proxies={},
headers=self._get_headers(),
auth=auth,
timeout=self.timeout
)
except requests.exceptions.SSLError as exc:
if "unknown protocol" in str(exc) and url.startswith("https:"):
# Server is using http rather than https, for some reason.
sys.stderr.write(WARNING_NO_HTTPS_SUPPORT.format(exc))
return self.get_data_from_url_with_requests(
url.replace("https:", "http:", 1), json_data)
if cylc.flags.debug:
import traceback
traceback.print_exc()
raise ConnectionError(url, exc)
except Exception as exc:
if cylc.flags.debug:
import traceback
traceback.print_exc()
raise ConnectionError(url, exc)
if ret.status_code == 401:
raise ConnectionDeniedError(url, self.prog_name,
self.ACCESS_DESCRIPTION)
if ret.status_code >= 400:
from cylc.network.https.util import get_exception_from_html
exception_text = get_exception_from_html(ret.text)
if exception_text:
sys.stderr.write(exception_text)
else:
sys.stderr.write(ret.text)
try:
ret.raise_for_status()
except Exception as exc:
if cylc.flags.debug:
import traceback
traceback.print_exc()
raise ConnectionError(url, exc)
try:
return ret.json()
except ValueError:
return ret.text
def get_data_from_url_with_urllib2(self, url, json_data, method=None):
import json
import urllib2
import ssl
if hasattr(ssl, '_create_unverified_context'):
ssl._create_default_https_context = ssl._create_unverified_context
if method is None:
method = self.METHOD
orig_json_data = json_data
username, password = self._get_auth()
auth_manager = urllib2.HTTPPasswordMgrWithDefaultRealm()
auth_manager.add_password(None, url, username, password)
auth = urllib2.HTTPDigestAuthHandler(auth_manager)
opener = urllib2.build_opener(auth, urllib2.HTTPSHandler())
headers_list = self._get_headers().items()
if json_data:
json_data = json.dumps(json_data)
headers_list.append(('Accept', 'application/json'))
json_headers = {'Content-Type': 'application/json',
'Content-Length': len(json_data)}
else:
json_data = None
json_headers = {'Content-Length': 0}
opener.addheaders = headers_list
req = urllib2.Request(url, json_data, json_headers)
# This is an unpleasant monkey patch, but there isn't an alternative.
# urllib2 uses POST iff there is a data payload, but that is not the
# correct criterion. The difference is basically that POST changes
# server state and GET doesn't.
req.get_method = lambda: method
try:
response = opener.open(req, timeout=self.timeout)
except urllib2.URLError as exc:
if "unknown protocol" in str(exc) and url.startswith("https:"):
# Server is using http rather than https, for some reason.
sys.stderr.write(WARNING_NO_HTTPS_SUPPORT.format(exc))
return self.get_data_from_url_with_urllib2(
url.replace("https:", "http:", 1), orig_json_data)
if cylc.flags.debug:
import traceback
traceback.print_exc()
if "timed out" in str(exc):
raise ConnectionTimeout(url, exc)
else:
raise ConnectionError(url, exc)
except Exception as exc:
if cylc.flags.debug:
import traceback
traceback.print_exc()
raise ConnectionError(url, exc)
if response.getcode() == 401:
raise ConnectionDeniedError(url, self.prog_name,
self.ACCESS_DESCRIPTION)
response_text = response.read()
if response.getcode() >= 400:
from cylc.network.https.util import get_exception_from_html
exception_text = get_exception_from_html(response_text)
if exception_text:
sys.stderr.write(exception_text)
else:
sys.stderr.write(response_text)
raise ConnectionError(url,
"%s HTTP return code" % response.getcode())
try:
return json.loads(response_text)
except ValueError:
return response_text
def _get_data_from_url_with_urllib2(self, url, json_data, method=None):
import json
import urllib2
import ssl
if hasattr(ssl, '_create_unverified_context'):
ssl._create_default_https_context = ssl._create_unverified_context
if method is None:
method = self.METHOD
orig_json_data = json_data
username, password = self._get_auth()
auth_manager = urllib2.HTTPPasswordMgrWithDefaultRealm()
auth_manager.add_password(None, url, username, password)
auth = urllib2.HTTPDigestAuthHandler(auth_manager)
opener = urllib2.build_opener(auth, urllib2.HTTPSHandler())
headers_list = self._get_headers().items()
if json_data:
json_data = json.dumps(json_data)
headers_list.append(('Accept', 'application/json'))
json_headers = {'Content-Type': 'application/json',
'Content-Length': len(json_data)}
else:
json_data = None
json_headers = {'Content-Length': 0}
opener.addheaders = headers_list
req = urllib2.Request(url, json_data, json_headers)
# This is an unpleasant monkey patch, but there isn't an alternative.
# urllib2 uses POST if there is a data payload, but that is not the
# correct criterion. The difference is basically that POST changes
# server state and GET doesn't.
req.get_method = lambda: method
try:
response = opener.open(req, timeout=self.timeout)
except urllib2.URLError as exc:
if "unknown protocol" in str(exc) and url.startswith("https:"):
# Server is using http rather than https, for some reason.
sys.stderr.write(WARNING_NO_HTTPS_SUPPORT.format(exc))
return self._get_data_from_url_with_urllib2(
url.replace("https:", "http:", 1), orig_json_data)
if cylc.flags.debug:
import traceback
traceback.print_exc()
if "timed out" in str(exc):
raise ConnectionTimeout(url, exc)
else:
raise ConnectionError(url, exc)
except Exception as exc:
if cylc.flags.debug:
import traceback
traceback.print_exc()
raise ConnectionError(url, exc)
if response.getcode() == 401:
raise ConnectionDeniedError(url, self.prog_name,
self.ACCESS_DESCRIPTION)
response_text = response.read()
if response.getcode() >= 400:
from cylc.network.https.util import get_exception_from_html
exception_text = get_exception_from_html(response_text)
if exception_text:
sys.stderr.write(exception_text)
else:
sys.stderr.write(response_text)
raise ConnectionError(url,
"%s HTTP return code" % response.getcode())
if self.auth and self.auth[1] != NO_PASSPHRASE:
self.srv_files_mgr.cache_passphrase(
self.suite, self.owner, self.host, self.auth[1])
try:
return json.loads(response_text)
except ValueError:
return response_text
def _get_data_from_url_with_requests(self, url, json_data, method=None):
import requests
username, password = self._get_auth()
auth = requests.auth.HTTPDigestAuth(username, password)
if not hasattr(self, "session"):
self.session = requests.Session()
if method is None:
method = self.METHOD
if method == self.METHOD_POST:
session_method = self.session.post
else:
session_method = self.session.get
try:
ret = session_method(
url,
json=json_data,
verify=self._get_verify(),
proxies={},
headers=self._get_headers(),
auth=auth,
timeout=self.timeout
)
except requests.exceptions.SSLError as exc:
if "unknown protocol" in str(exc) and url.startswith("https:"):
# Server is using http rather than https, for some reason.
sys.stderr.write(WARNING_NO_HTTPS_SUPPORT.format(exc))
return self._get_data_from_url_with_requests(
url.replace("https:", "http:", 1), json_data)
if cylc.flags.debug:
import traceback
traceback.print_exc()
raise ConnectionError(url, exc)
except requests.exceptions.Timeout as exc:
if cylc.flags.debug:
import traceback
traceback.print_exc()
raise ConnectionTimeout(url, exc)
except requests.exceptions.RequestException as exc:
if cylc.flags.debug:
import traceback
traceback.print_exc()
raise ConnectionError(url, exc)
if ret.status_code == 401:
raise ConnectionDeniedError(url, self.prog_name,
self.ACCESS_DESCRIPTION)
if ret.status_code >= 400:
from cylc.network.https.util import get_exception_from_html
exception_text = get_exception_from_html(ret.text)
if exception_text:
sys.stderr.write(exception_text)
else:
sys.stderr.write(ret.text)
try:
ret.raise_for_status()
except requests.exceptions.HTTPError as exc:
if cylc.flags.debug:
import traceback
traceback.print_exc()
raise ConnectionError(url, exc)
if self.auth and self.auth[1] != NO_PASSPHRASE:
self.srv_files_mgr.cache_passphrase(
self.suite, self.owner, self.host, self.auth[1])
try:
return ret.json()
except ValueError:
return ret.text
