def simple_post(domain, url, data, *, headers, auth_manager, verify,
notify_addresses=None, payload_id=None):
"""
POST with a cleaner API, and return the actual HTTPResponse object, so
that error codes can be interpreted.
"""
if isinstance(data, str):
# Encode as UTF-8, otherwise requests will send data containing
# non-ASCII characters as 'data:application/octet-stream;base64,...'
data = data.encode('utf-8')
default_headers = CaseInsensitiveDict({
"content-type": "text/xml",
"content-length": str(len(data)),
})
default_headers.update(headers)
requests = Requests(
domain,
base_url=url,
verify=verify,
auth_manager=auth_manager,
notify_addresses=notify_addresses,
payload_id=payload_id,
)
try:
response = requests.post(None, data=data, headers=default_headers)
except Exception as err:
requests.notify_error(str(err))
raise
if not 200 <= response.status_code < 300:
message = f'HTTP status code {response.status_code}: {response.text}'
requests.notify_error(message)
return response
def string_to_sign(self, request):
h = CaseInsensitiveDict()
h.update(request.headers)
# Try to use
if b'x-amz-date' in h or 'x-amz-date' in h:
date = ''
else:
date = h.get('Date') or self._get_date()
request.headers['Date'] = date
# Set the date header
request.headers['Date'] = date
# A fix for the content type header extraction in python 3
# This have to be done because requests will try to set application/www-url-encoded herader
# if we pass bytes as the content, and the content-type is set with a key that is b'Content-Type' and not
# 'Content-Type'
content_type = ''
if b'Content-Type' in request.headers:
# Fix content type
content_type = h.get(b'Content-Type')
del request.headers[b'Content-Type']
request.headers['Content-Type'] = content_type
msg = [
request.method,
h.get(b'Content-MD5', '') or h.get('Content-MD5', ''),
content_type or h.get('Content-Type', ''),
date,
self._get_canonicalized_amz_headers(h) + self._get_canonicalized_resource(request)
]
return '\n'.join(msg)
def request(self, method, url, data=None, files=None):
headers = CaseInsensitiveDict([
('Accept', 'text/html,image/webp,image/apng,*/*;q=0.8'),
('Accept-Encoding', 'gzip, deflate'),
('Accept-Language', 'en-US,en;q=0.8'),
('Accept-Charset', 'ISO-8859-1,utf-8;q=0.7,*;q=0.7'),
('Cache-Control', 'no-cache'), ('Connection', 'keep-alive'),
('Host', 'www.pinterest.com'),
('Origin', 'https://www.pinterest.com'), ('Referer', HOME_PAGE),
('X-Requested-With', 'XMLHttpRequest'),
('Accept', 'application/json'),
('Content-Type',
'application/x-www-form-urlencoded; charset=UTF-8'),
('User-Agent', AGENT_STRING)
])
csrftoken = self.http.cookies.get('csrftoken')
if csrftoken:
headers.update([('X-CSRFToken', csrftoken)])
response = self.http.request(method,
url,
data=data,
headers=headers,
files=files,
proxies=self.proxies)
response.raise_for_status()
self.registry.update(Registry.Key.COOKIES, response.cookies)
return response
def get_server_parameters(self, config):
parameters = config['parameters'].copy()
listen_addresses, port = split_host_port(config['listen'], 5432)
parameters.update(cluster_name=self._postgresql.scope,
listen_addresses=listen_addresses,
port=str(port))
if config.get('synchronous_mode', False):
if self._synchronous_standby_names is None:
if config.get('synchronous_mode_strict', False):
parameters['synchronous_standby_names'] = '*'
else:
parameters.pop('synchronous_standby_names', None)
else:
parameters[
'synchronous_standby_names'] = self._synchronous_standby_names
if self._postgresql.major_version >= 90600 and parameters[
'wal_level'] == 'hot_standby':
parameters['wal_level'] = 'replica'
ret = CaseInsensitiveDict({
k: v
for k, v in parameters.items()
if not self._postgresql.major_version or self._postgresql.
major_version >= self.CMDLINE_OPTIONS.get(k, (0, 1, 90100))[2]
})
ret.update({
k: os.path.join(self._config_dir, ret[k])
for k in ('hba_file', 'ident_file') if k in ret
})
return ret
def send_document(url, data, timeout=10, method="post", *args, **kwargs):
"""Helper method to send a document via POST.
Additional ``*args`` and ``**kwargs`` will be passed on to ``requests.post``.
:arg url: Full url to send to, including protocol
:arg data: Dictionary (will be form-encoded), bytes, or file-like object to send in the body
:arg timeout: Seconds to wait for response (defaults to 10)
:arg method: Method to use, defaults to post
:returns: Tuple of status code (int or None) and error (exception class instance or None)
"""
logger.debug("send_document: url=%s, data=%s, timeout=%s, method=%s", url, data, timeout, method)
if not method:
method = "post"
headers = CaseInsensitiveDict({
'User-Agent': USER_AGENT,
})
if "headers" in kwargs:
# Update from kwargs
headers.update(kwargs.get("headers"))
kwargs.update({
"data": data, "timeout": timeout, "headers": headers
})
request_func = getattr(requests, method)
try:
response = request_func(url, *args, **kwargs)
logger.debug("send_document: response status code %s", response.status_code)
return response.status_code, None
# TODO support rate limit 429 code
except RequestException as ex:
logger.debug("send_document: exception %s", ex)
return None, ex
class MockResponse(object):
"""
Mock response object with a status code and some content
"""
def __init__(self, status_code, content=None, headers=None):
self.status_code = status_code
self.content = content or ""
self.headers = CaseInsensitiveDict()
if headers:
self.headers.update(headers)
def raise_for_status(self):
http_error_msg = ""
if 400 <= self.status_code < 500:
http_error_msg = "%s Client Error: ..." % self.status_code
elif 500 <= self.status_code < 600:
http_error_msg = "%s Server Error: ..." % self.status_code
if http_error_msg:
raise requests.HTTPError(http_error_msg, response=self)
def json(self, **kwargs):
return json.loads(self.content)
class DDWRT(Router):
def __init__(self, conf, hostnames):
self.hostnames = CaseInsensitiveDict()
self.hostnames.update(hostnames)
self.conf = conf
self.auth = self.conf.auth()
def clients(self):
""" Receives all currently logged in users in a wifi network.
:rtype : list
:return: Returns a list of dicts, containing the following keys: mac, ipv4, seen, hostname
"""
clients = self._get_clients_raw()
clients_json = []
for client in clients:
client_hostname_from_router = client[0]
client_ipv4 = client[1].strip()
client_mac = client[2].strip().upper()
client_hostname = self.hostnames.get(client_mac, client_hostname_from_router).strip()
client_connections = int(client[3].strip())
# Clients with less than 20 connections are considered offline
if client_connections < 20:
continue
clients_json.append({
'mac': client_mac,
'ipv4': client_ipv4,
'seen': int(time.time()),
'hostname': client_hostname,
})
logger.debug('The router got us {} clients.'.format(len(clients_json)))
logger.debug(str(clients_json))
return clients_json
def _get_clients_raw(self):
info_page = self.conf.internal()
response = requests.get(info_page, auth=self.auth)
logger.info('Got response from router with code {}.'.format(response.status_code))
return DDWRT._convert_to_clients(response.text) or []
@staticmethod
def _convert_to_clients(router_info_all):
# Split router info in lines and filter empty info
router_info_lines = filter(None, router_info_all.split("\n"))
# Get key / value of router info
router_info_items = dict()
for item in router_info_lines:
key, value = item[1:-1].split("::") # Remove curly braces and split
router_info_items[key.strip()] = value.strip()
# Get client info as a list
arp_table = utils.groupn(router_info_items['arp_table'].replace("'", "").split(","), 4)
dhcp_leases = utils.groupn(router_info_items['dhcp_leases'].replace("'", "").split(","), 5)
return arp_table if (len(arp_table) > 0) else []
def string_to_sign(self, request):
"""
Generates the string we need to sign on.
Params:
- request The request object
Returns
String ready to be signed on
"""
# We'll use case insensitive dict to store the headers
h = CaseInsensitiveDict()
# Add the hearders
h.update(request.headers)
# If we have an 'x-amz-date' header,
# we'll try to use it instead of the date
if b'x-amz-date' in h or 'x-amz-date' in h:
date = ''
else:
# No x-amz-header, we'll generate a date
date = h.get('Date') or self._get_date()
# Set the date header
request.headers['Date'] = date
# A fix for the content type header extraction in python 3
# This have to be done because requests will try to set
# application/www-url-encoded header if we pass bytes as the content,
# and the content-type is set with a key that is b'Content-Type' and
# not 'Content-Type'
content_type = ''
if b'Content-Type' in request.headers:
# Fix content type
content_type = h.get(b'Content-Type')
del request.headers[b'Content-Type']
request.headers['Content-Type'] = content_type
# The string we're about to generate
# There's more information about it here:
# http://docs.aws.amazon.com/AmazonS3/latest/dev/
# RESTAuthentication.html#ConstructingTheAuthenticationHeader
msg = [
# HTTP Method
request.method,
# MD5 If provided
h.get(b'Content-MD5', '') or h.get('Content-MD5', ''),
# Content type if provided
content_type or h.get('Content-Type', ''),
# Date
date,
# Canonicalized special amazon headers and resource uri
self._get_canonicalized_amz_headers(h) +
self._get_canonicalized_resource(request)
]
# join with a newline and return
return '\n'.join(msg)
class WorkerRequest(WPSRequest):
"""
Extended :mod:`pywps` request with additional handling provided by :mod:`weaver`.
"""
_auth_headers = CaseInsensitiveDict({ # take advantage of case insensitive only, value don't care
"Authorization": None,
"X-Auth": None,
sd.XAuthVaultFileHeader.name: None,
})
def __init__(self, http_request=None, http_headers=None, **kwargs):
# type: (Optional[AnyRequestType], Optional[AnyHeadersContainer], **Any) -> None
if http_request and not isinstance(http_request, WerkzeugRequest):
http_request = extend_instance(http_request, WerkzeugRequest)
super(WorkerRequest, self).__init__(http_request, **kwargs)
self.auth_headers = CaseInsensitiveDict()
if http_request:
self.auth_headers.update(
self.parse_auth_headers(http_request.headers))
if http_headers:
self.auth_headers.update(self.parse_auth_headers(http_headers))
def parse_auth_headers(self, headers):
# type: (Optional[AnyHeadersContainer]) -> HeadersType
if not headers:
return {}
if isinstance(headers, list):
headers = dict(headers)
auth_headers = {}
for name, value in headers.items():
if name in self._auth_headers:
auth_headers[name] = value
return auth_headers
def request(self, method, url, data=None, files=None, extra_headers=None):
headers = CaseInsensitiveDict([
('Referer', HOME_PAGE), ('X-Requested-With', 'XMLHttpRequest'),
('Accept', 'application/json'),
('Content-Type',
'application/x-www-form-urlencoded; charset=UTF-8'),
('User-Agent', self.user_agent)
])
csrftoken = self.http.cookies.get('csrftoken')
if csrftoken:
headers.update([('X-CSRFToken', csrftoken)])
if extra_headers is not None:
for h in extra_headers:
headers.update([(h, extra_headers[h])])
response = self.http.request(method,
url,
data=data,
headers=headers,
files=files,
proxies=self.proxies)
response.raise_for_status()
self.registry.update(Registry.Key.COOKIES, response.cookies)
return response
def send_document(url, data, timeout=10, *args, **kwargs):
"""Helper method to send a document via POST.
Additional ``*args`` and ``**kwargs`` will be passed on to ``requests.post``.
:arg url: Full url to send to, including protocol
:arg data: POST data to send (dict)
:arg timeout: Seconds to wait for response (defaults to 10)
:returns: Tuple of status code (int or None) and error (exception class instance or None)
"""
logger.debug("send_document: url=%s, data=%s, timeout=%s", url, data, timeout)
headers = CaseInsensitiveDict({
'User-Agent': USER_AGENT,
})
if "headers" in kwargs:
# Update from kwargs
headers.update(kwargs.get("headers"))
kwargs.update({
"data": data, "timeout": timeout, "headers": headers
})
try:
response = requests.post(url, *args, **kwargs)
logger.debug("send_document: response status code %s", response.status_code)
return response.status_code, None
except RequestException as ex:
logger.debug("send_document: exception %s", ex)
return None, ex
class MockResponse(object):
"""
Mock response object with a status code and some content
"""
def __init__(self, status_code, content=None, headers=None):
self.status_code = status_code
self.content = content or ''
self.headers = CaseInsensitiveDict()
if headers:
self.headers.update(headers)
def raise_for_status(self):
http_error_msg = ''
if 400 <= self.status_code < 500:
http_error_msg = '%s Client Error: ...' % self.status_code
elif 500 <= self.status_code < 600:
http_error_msg = '%s Server Error: ...' % self.status_code
if http_error_msg:
raise requests.HTTPError(http_error_msg, response=self)
def json(self, **kwargs):
return json.loads(self.content)
def request(self, method, url, params=None, data=None, files=None, headers=None, ajax=False, stream=None):
"""
:rtype: requests.models.Response
"""
_headers = CaseInsensitiveDict([
('Accept', 'text/html,image/webp,image/apng,*/*;q=0.8'),
('Accept-Encoding', 'gzip, deflate'),
('Accept-Language', 'en-US,en;q=0.8'),
('Accept-Charset', 'ISO-8859-1,utf-8;q=0.7,*;q=0.7'),
('Cache-Control', 'no-cache'),
('Connection', 'keep-alive'),
('Host', 'www.pinterest.com'),
('Origin', 'https://www.pinterest.com'),
('Referer', self.home_page),
('User-Agent', self.registry.get(Registry.Key.USER_AGENT))])
if method.upper() == 'POST':
_headers.update([('Content-Type', 'application/x-www-form-urlencoded; charset=UTF-8')])
if ajax:
_headers.update([('Accept', 'application/json')])
csrftoken = self.http.cookies.get('csrftoken')
if csrftoken:
_headers.update([('X-CSRFToken', csrftoken)])
_headers.update([('X-Requested-With', 'XMLHttpRequest')])
if headers:
_headers.update(headers)
response = self.http.request(method, url, params=params, data=data, headers=_headers,
files=files, timeout=60, proxies=self.proxies, stream=stream)
response.raise_for_status()
self.registry.update(Registry.Key.COOKIES, response.cookies)
return response
def request(self, method, path, data=None, headers={}, query=None, response_type=None):
url = '{0}{1}'.format(ApiClient.__api_root, path)
req_headers = CaseInsensitiveDict({ 'User-Agent': 'sonic-cli' })
req_headers.update(headers)
body = None
if data is not None:
if 'Content-Type' not in req_headers:
req_headers['Content-Type'] = 'application/yang-data+json'
body = json.dumps(data)
try:
r = ApiClient.__session.request(
method,
url,
headers=req_headers,
data=body,
params=query,
verify=False)
return Response(r, response_type)
except requests.RequestException as e:
log_warning('cli_client request exception: ' + str(e))
#TODO have more specific error message based
msg = '%Error: Could not connect to Management REST Server'
return ApiClient.__new_error_response(msg)
def string_to_sign(self, request):
"""
Generates the string we need to sign on.
Params:
- request The request object
Returns
String ready to be signed on
"""
# We'll use case insensitive dict to store the headers
h = CaseInsensitiveDict()
# Add the hearders
h.update(request.headers)
# If we have an 'x-amz-date' header,
# we'll try to use it instead of the date
if b'x-amz-date' in h or 'x-amz-date' in h:
date = ''
else:
# No x-amz-header, we'll generate a date
date = h.get('Date') or self._get_date()
# Set the date header
request.headers['Date'] = date
# A fix for the content type header extraction in python 3
# This have to be done because requests will try to set
# application/www-url-encoded header if we pass bytes as the content,
# and the content-type is set with a key that is b'Content-Type' and
# not 'Content-Type'
content_type = ''
if b'Content-Type' in request.headers:
# Fix content type
content_type = h.get(b'Content-Type')
del request.headers[b'Content-Type']
request.headers['Content-Type'] = content_type
# The string we're about to generate
# There's more information about it here:
# http://docs.aws.amazon.com/AmazonS3/latest/dev/
# RESTAuthentication.html#ConstructingTheAuthenticationHeader
msg = [
# HTTP Method
request.method,
# MD5 If provided
h.get(b'Content-MD5', '') or h.get('Content-MD5', ''),
# Content type if provided
content_type or h.get('Content-Type', ''),
# Date
date,
# Canonicalized special amazon headers and resource uri
self._get_canonicalized_amz_headers(h) +
self._get_canonicalized_resource(request)
]
# join with a newline and return
return '\n'.join(msg)
def _check_creds(self, creds):
d = CaseInsensitiveDict()
if isinstance(creds, dict):
d.update(creds)
elif isinstance(creds, basestring):
if os.path.exists(creds):
creds = file(creds, "r").read()
for line in creds.splitlines():
if ":" in line:
k, v = line.split(":", 1)
d[k.strip()] = v.strip()
else:
raise TypeError("unsupported type for credentials data")
if "companyId" not in d and "CID" in d:
d["companyId"] = d["CID"]
if "companyId" in d and not "psk" in d:
raise ValueError("psk is required when companyId is provided")
elif "psk" in d and not "companyId" in d:
raise ValueError("companyId is required when psk is provided")
elif "companyId" in d and "psk" in d:
return {"companyId": int(d["companyId"]), "psk": str(d["psk"])}
elif "loginSessionId" in d and not "profileId" in d:
raise ValueError("profileId is required when loginSessionId is " "provided")
elif "profileId" in d and not "loginSessionId" in d:
raise ValueError("loginSessionId is required when profileId is " "provided")
elif "loginSessionId" in d and "profileId" in d:
return {"loginSessionId": str(d["loginSessionId"]), "profileId": int(d["profileId"])}
else:
raise ValueError("either companyId+psk or " "loginSessionId+profileId must be provided")
def request(self, method, url, data=None, files=None, extra_headers=None):
headers = CaseInsensitiveDict([
("Referer", HOME_PAGE),
("X-Requested-With", "XMLHttpRequest"),
("Accept", "application/json"),
("Content-Type",
"application/x-www-form-urlencoded; charset=UTF-8"),
("User-Agent", self.user_agent),
])
csrftoken = self.http.cookies.get("csrftoken")
if csrftoken:
headers.update([("X-CSRFToken", csrftoken)])
if extra_headers is not None:
for h in extra_headers:
headers.update([(h, extra_headers[h])])
response = self.http.request(method,
url,
data=data,
headers=headers,
files=files,
proxies=self.proxies)
response.raise_for_status()
return response
def test_preserve_last_key_case(self):
cid = CaseInsensitiveDict({"Accept": "application/json", "user-Agent": "requests"})
cid.update({"ACCEPT": "application/json"})
cid["USER-AGENT"] = "requests"
keyset = frozenset(["ACCEPT", "USER-AGENT"])
assert frozenset(i[0] for i in cid.items()) == keyset
assert frozenset(cid.keys()) == keyset
assert frozenset(cid) == keyset
def test_update(self):
cid = CaseInsensitiveDict()
cid['spam'] = 'blueval'
cid.update({'sPam': 'notblueval'})
assert cid['spam'] == 'notblueval'
cid = CaseInsensitiveDict({'Foo': 'foo','BAr': 'bar'})
cid.update({'fOO': 'anotherfoo', 'bAR': 'anotherbar'})
assert len(cid) == 2
assert cid['foo'] == 'anotherfoo'
assert cid['bar'] == 'anotherbar'
def test_update(self):
cid = CaseInsensitiveDict()
cid['spam'] = 'blueval'
cid.update({'sPam': 'notblueval'})
self.assertEqual(cid['spam'], 'notblueval')
cid = CaseInsensitiveDict({'Foo': 'foo', 'BAr': 'bar'})
cid.update({'fOO': 'anotherfoo', 'bAR': 'anotherbar'})
self.assertEqual(len(cid), 2)
self.assertEqual(cid['foo'], 'anotherfoo')
self.assertEqual(cid['bar'], 'anotherbar')
def test_update(self):
cid = CaseInsensitiveDict()
cid['spam'] = 'blueval'
cid.update({'sPam': 'notblueval'})
self.assertEqual(cid['spam'], 'notblueval')
cid = CaseInsensitiveDict({'Foo': 'foo','BAr': 'bar'})
cid.update({'fOO': 'anotherfoo', 'bAR': 'anotherbar'})
self.assertEqual(len(cid), 2)
self.assertEqual(cid['foo'], 'anotherfoo')
self.assertEqual(cid['bar'], 'anotherbar')
def test_update(self):
cid = CaseInsensitiveDict()
cid['spam'] = 'blueval'
cid.update({'sPam': 'notblueval'})
assert cid['spam'] == 'notblueval'
cid = CaseInsensitiveDict({'Foo': 'foo','BAr': 'bar'})
cid.update({'fOO': 'anotherfoo', 'bAR': 'anotherbar'})
assert len(cid) == 2
assert cid['foo'] == 'anotherfoo'
assert cid['bar'] == 'anotherbar'
def test_update(self):
cid = CaseInsensitiveDict()
cid["spam"] = "blueval"
cid.update({"sPam": "notblueval"})
assert cid["spam"] == "notblueval"
cid = CaseInsensitiveDict({"Foo": "foo", "BAr": "bar"})
cid.update({"fOO": "anotherfoo", "bAR": "anotherbar"})
assert len(cid) == 2
assert cid["foo"] == "anotherfoo"
assert cid["bar"] == "anotherbar"
def test_update(self):
cid = CaseInsensitiveDict()
cid["spam"] = "blueval"
cid.update({"sPam": "notblueval"})
self.assertEqual(cid["spam"], "notblueval")
cid = CaseInsensitiveDict({"Foo": "foo", "BAr": "bar"})
cid.update({"fOO": "anotherfoo", "bAR": "anotherbar"})
self.assertEqual(len(cid), 2)
self.assertEqual(cid["foo"], "anotherfoo")
self.assertEqual(cid["bar"], "anotherbar")
def __init__(self,
response=None,
status=None,
headers=None,
mimetype=None,
content_type=None,
direct_passthrough=False):
headers = CaseInsensitiveDict(headers) if headers is not None else None
if response is not None and isinstance(
response, BaseResponse) and response.headers is not None:
headers = CaseInsensitiveDict(response.headers)
if headers is None:
headers = CaseInsensitiveDict()
h = headers
h['Access-Control-Allow-Origin'] = headers.get(
'Access-Control-Allow-Origin', '*')
h['Access-Control-Allow-Methods'] = headers.get(
'Access-Control-Allow-Methods',
"GET, PUT, POST, HEAD, OPTIONS, DELETE")
h['Access-Control-Max-Age'] = headers.get('Access-Control-Max-Age',
"21600")
h['Cache-Control'] = headers.get(
'Cache-Control', "no-cache, must-revalidate, no-store")
if 'Access-Control-Allow-Headers' not in headers and len(
headers.keys()) > 0:
h['Access-Control-Allow-Headers'] = ', '.join(iterkeys(headers))
data = None
if response is not None and isinstance(response, string_types):
data = response
response = None
if response is not None and isinstance(response, BaseResponse):
new_response_headers = CaseInsensitiveDict(
response.headers if response.headers is not None else {})
new_response_headers.update(h)
response.headers = new_response_headers
headers = None
data = response.get_data()
else:
headers.update(h)
headers = dict(headers)
super(IppResponse,
self).__init__(response=response,
status=status,
headers=headers,
mimetype=mimetype,
content_type=content_type,
direct_passthrough=direct_passthrough)
if data is not None:
self.set_data(data)
def request(self, method, url, accept_json=False, headers=None, params=None, json=None, data=None, files=None,
**kwargs):
full_url = self.url + url
input_headers = _remove_null_values(headers) if headers else {}
headers = CaseInsensitiveDict({'user-agent': 'watson-developer-cloud-python-' + __version__})
if accept_json:
headers['accept'] = 'application/json'
headers.update(input_headers)
# Remove keys with None values
params = _remove_null_values(params)
json = _remove_null_values(json)
data = _remove_null_values(data)
files = _remove_null_values(files)
# Support versions of requests older than 2.4.2 without the json input
if not data and json is not None:
data = json_import.dumps(json)
headers.update({'content-type': 'application/json'})
auth = None
if self.username and self.password:
auth = (self.username, self.password)
if self.api_key is not None:
if params is None:
params = {}
if url.startswith('https://gateway-a.watsonplatform.net/calls'):
params['apikey'] = self.api_key
else:
params['api_key'] = self.api_key
response = requests.request(method=method, url=full_url, cookies=self.jar, auth=auth, headers=headers,
params=params, data=data, files=files, **kwargs)
if 200 <= response.status_code <= 299:
if accept_json:
response_json = response.json()
if 'status' in response_json and response_json['status'] == 'ERROR':
response.status_code = 400
error_message = 'Unknown error'
if 'statusInfo' in response_json:
error_message = response_json['statusInfo']
if error_message == 'invalid-api-key':
response.status_code = 401
raise WatsonException('Error: ' + error_message)
return response_json
return response
else:
if response.status_code == 401:
error_message = 'Unauthorized: Access is denied due to invalid credentials'
else:
error_message = self._get_error_message(response)
raise WatsonException(error_message)
def test_preserve_last_key_case(self):
cid = CaseInsensitiveDict({
'Accept': 'application/json',
'user-Agent': 'requests',
})
cid.update({'ACCEPT': 'application/json'})
cid['USER-AGENT'] = 'requests'
keyset = frozenset(['ACCEPT', 'USER-AGENT'])
assert frozenset(i[0] for i in cid.items()) == keyset
assert frozenset(cid.keys()) == keyset
assert frozenset(cid) == keyset
def test_preserve_last_key_case(self):
cid = CaseInsensitiveDict({
'Accept': 'application/json',
'user-Agent': 'requests',
})
cid.update({'ACCEPT': 'application/json'})
cid['USER-AGENT'] = 'requests'
keyset = frozenset(['ACCEPT', 'USER-AGENT'])
assert frozenset(i[0] for i in cid.items()) == keyset
assert frozenset(cid.keys()) == keyset
assert frozenset(cid) == keyset
def request(self, method, url, accept_json=False, headers=None, params=None, json=None, data=None, files=None,
**kwargs):
full_url = self.url + url
input_headers = _remove_null_values(headers) if headers else {}
headers = CaseInsensitiveDict({'user-agent': 'watson-developer-cloud-python-' + __version__})
if accept_json:
headers['accept'] = 'application/json'
headers.update(input_headers)
# Remove keys with None values
params = _remove_null_values(params)
json = _remove_null_values(json)
data = _remove_null_values(data)
files = _remove_null_values(files)
# Support versions of requests older than 2.4.2 without the json input
if not data and json is not None:
data = json_import.dumps(json)
headers.update({'content-type': 'application/json'})
auth = None
if self.username and self.password:
auth = (self.username, self.password)
if self.api_key is not None:
if params is None:
params = {}
params['apikey'] = self.api_key
response = requests.request(method=method, url=full_url, cookies=self.jar, auth=auth, headers=headers,
params=params, data=data, files=files, **kwargs)
if 200 <= response.status_code <= 299:
if accept_json:
response_json = response.json()
if 'status' in response_json and response_json['status'] == 'ERROR':
response.status_code = 400
error_message = 'Unknown error'
if 'statusInfo' in response_json:
error_message = response_json['statusInfo']
if error_message == 'invalid-api-key':
response.status_code = 401
raise WatsonException('Error: ' + error_message)
return response_json
return response
else:
if response.status_code == 401:
error_message = 'Unauthorized: Access is denied due to invalid credentials'
else:
error_message = self._get_error_message(response)
raise WatsonException(error_message)
def _prepare_request(self, command, json, opcode_name, fetch_list,
**kwargs):
kwargs = CaseInsensitiveDict(kwargs)
kwargs.update({
'apiKey': self.key,
opcode_name: command,
})
if json:
kwargs['response'] = 'json'
if 'page' in kwargs or fetch_list:
kwargs.setdefault('pagesize', 500)
kwarg = 'params' if self.method == 'get' else 'data'
return kwarg, dict(kwargs._store.values())
def _prepare_request(self, command, json, opcode_name, fetch_list,
**kwargs):
params = CaseInsensitiveDict(**kwargs)
params.update({
'apiKey': self.key,
opcode_name: command,
})
if json:
params['response'] = 'json'
if 'page' in kwargs or fetch_list:
params.setdefault('pagesize', PAGE_SIZE)
kind = 'params' if self.method == 'get' else 'data'
return kind, {k: v for k, v in params.items()}
def _configure_headers(self, additional_headers):
headers = CaseInsensitiveDict()
headers.update(requests.utils.default_headers())
headers["api-key"] = self._config.api_key
headers["content-type"] = "application/json"
headers["accept"] = "application/json"
headers["x-cdp-sdk"] = "CognitePythonSDK:{}".format(utils._auxiliary.get_current_sdk_version())
headers["x-cdp-app"] = self._config.client_name
if "User-Agent" in headers:
headers["User-Agent"] += " " + utils._auxiliary.get_user_agent()
else:
headers["User-Agent"] = utils._auxiliary.get_user_agent()
headers.update(additional_headers)
return headers
def http_get(self, url, headers=None):
"""
returns (status, header, content)
status: int
header: dict
content: string
"""
hdrs = CaseInsensitiveDict()
if headers:
hdrs.update(headers)
if not 'user-agent' in hdrs:
hdrs['user-agent'] = USER_AGENT
print "http_get:", url
r = requests.get(url, headers=hdrs)
return (r.status_code, r.headers, r.text)
def simple_request(domain,
url,
data,
*,
headers,
auth_manager,
verify,
method="POST",
notify_addresses=None,
payload_id=None):
if isinstance(data, str):
# Encode as UTF-8, otherwise requests will send data containing
# non-ASCII characters as 'data:application/octet-stream;base64,...'
data = data.encode('utf-8')
default_headers = CaseInsensitiveDict({
"content-type": "text/xml",
"content-length": str(len(data)),
})
default_headers.update(headers)
requests = Requests(
domain,
base_url=url,
verify=verify,
auth_manager=auth_manager,
notify_addresses=notify_addresses,
payload_id=payload_id,
)
request_methods = {
REQUEST_DELETE: requests.delete,
REQUEST_POST: requests.post,
REQUEST_PUT: requests.put,
}
try:
request_method = request_methods[method]
except KeyError:
raise ValueError(
f"Method must be one of {', '.join(request_methods.keys())}")
try:
response = request_method(None, data=data, headers=default_headers)
except Exception as err:
requests.notify_error(str(err))
raise
if not 200 <= response.status_code < 300:
message = f'HTTP status code {response.status_code}: {response.text}'
requests.notify_error(message)
return response
def _merge_headers(session, headers):
if headers is None:
headers = {}
if session.headers is None:
merged_headers = {}
else:
# Session headers are the default but local headers
# have priority and can override values
merged_headers = session.headers.copy()
# Make sure merged_headers are CaseIsensitiveDict
if not isinstance(merged_headers, CaseInsensitiveDict):
merged_headers = CaseInsensitiveDict(merged_headers)
merged_headers.update(headers)
return merged_headers
def generate_request(method, url, body):
"""
Generate our own custom request, so we can calculate digest auth.
"""
method = method.upper()
url = url
files = []
json_string = None
headers = CaseInsensitiveDict({
'Accept': 'application/json',
'Accept-Encoding': 'gzip, deflate',
'Connection': 'keep-alive',
'Content-Type': 'application/json',
'User-Agent': 'stormpath-flask/0.4.4 flask/0.10.1 stormpath-sdk-python/2.4.5 python/2.7.6 Linux/LinuxMint (Linux-3.13.0-37-generic-x86_64-with-LinuxMint-17.1-rebecca)'
})
if body:
headers.update({'Content-Length': str(len(json.dumps(body)))})
params = OrderedDict()
auth = Sauthc1Signer(
id=os.environ.get('STORMPATH_API_KEY_ID'),
secret=os.environ.get('STORMPATH_API_KEY_SECRET'))
cookies = RequestsCookieJar()
hooks = {'response': []}
pr = PreparedRequest()
if body:
json_body = json.dumps(body)
else:
json_body = None
pr.prepare(
method=method.upper(),
url=url,
files=files,
data=json_body,
json=json_string,
headers=headers,
params=params,
auth=auth,
cookies=cookies,
hooks=hooks,
)
return pr
def _prepare_request(self, command, json=True, opcode_name='command',
fetch_list=False, **kwargs):
params = CaseInsensitiveDict(**kwargs)
params.update({
'apiKey': self.key,
opcode_name: command,
})
if json:
params['response'] = 'json'
if 'page' in kwargs or fetch_list:
params.setdefault('pagesize', PAGE_SIZE)
if 'expires' not in params and self.expiration.total_seconds() >= 0:
params['signatureVersion'] = '3'
tz = pytz.utc
expires = tz.localize(datetime.utcnow() + self.expiration)
params['expires'] = expires.astimezone(tz).strftime(EXPIRES_FORMAT)
kind = 'params' if self.method == 'get' else 'data'
return kind, dict(params.items())
def request(url, params=None, data=None, headers=None, ajax=False, method=None):
if not method:
method = 'GET'
elif isinstance(method, bytes):
try:
method = method.decode('utf-8').upper()
except UnicodeDecodeError:
method = method.decode('iso-8859-1').upper()
else:
raise TypeError("Expected HTTP request method as a str or None type")
assert (method in ['GET', 'HEAD', 'POST', 'PUT', 'DELETE', 'CONNECT', 'OPTIONS', 'TRACE', 'PATCH']), 'Unsupported HTTP Request method.'
_headers = CaseInsensitiveDict([
('Accept', 'text/html,image/webp,image/apng,*/*;q=0.8'),
('Accept-Encoding', 'gzip, deflate'),
('Accept-Language', 'en-US,en;q=0.8'),
('Accept-Charset', 'ISO-8859-1,utf-8;q=0.7,*;q=0.7'),
('Cache-Control', 'no-cache'),
('Connection', 'keep-alive'),
('Host', ''),
('Origin', ''),
('Referer', ''),
('User-Agent', '')])
if data is not None:
if method == 'GET':
method = 'POST'
# If data is empty but not None. We will post with no body. So no Content-Type header set
if data:
_headers.update([('Content-Type', 'application/x-www-form-urlencoded; charset=UTF-8')])
if ajax:
_headers.update([('Accept', 'application/json')])
csrftoken = http.cookies.get('csrftoken')
if csrftoken:
_headers.update([('X-CSRFToken', csrftoken)])
_headers.update([('X-Requested-With', 'XMLHttpRequest')])
if headers:
_headers.update(headers)
try:
response = http.request(method, url, params=params, data=data, headers=_headers, timeout=60)
except Exception as ex:
raise Exception("Request Error, %s(%s) on %s" % (ex.__class__.__name__, ex.message, url))
raise_for_status(response)
return response
def http_get(self, url, headers=None):
"""
returns (status, header, content)
status: int
header: dict
content: string
"""
hdrs = CaseInsensitiveDict()
if headers:
hdrs.update(headers)
if not 'user-agent' in hdrs:
hdrs['user-agent'] = USER_AGENT
print "http_get:", url
try:
r = requests.get(url, headers=hdrs)
return (r.status_code, r.headers, r.text)
except requests.exceptions.ConnectionError, e:
# happens for example, when
return (500, None,
"requests.exceptions.ConnectionError getting %s" % url)
def handle_response(cls, response_code, response_data):
"""
Handles service response.
Returns the message, status or workflow id, if applicable
"""
client_error = response_code >= 400 and response_code < 500
success = response_code >= 200 and response_code < 300
workflow = CaseInsensitiveDict({"Message":"", "Id": 0, "Status": 0})
if client_error or success:
try:
response_content = json.loads(response_data)
workflow.update(response_content)
except ValueError:
workflow["Message"] = "Invalid response content"
else:
workflow["Message"] = "{0}: Internal server error".format(response_code)
return workflow
def _configure_headers(self, additional_headers):
headers = CaseInsensitiveDict()
headers.update(requests.utils.default_headers())
if self._config.token is None:
headers["api-key"] = self._config.api_key
elif isinstance(self._config.token, str):
headers["Authorization"] = "Bearer {}".format(self._config.token)
elif isinstance(self._config.token, Callable):
headers["Authorization"] = "Bearer {}".format(self._config.token())
else:
raise TypeError("'token' must be str, Callable, or None.")
headers["content-type"] = "application/json"
headers["accept"] = "application/json"
headers["x-cdp-sdk"] = "CognitePythonSDK:{}".format(utils._auxiliary.get_current_sdk_version())
headers["x-cdp-app"] = self._config.client_name
if "User-Agent" in headers:
headers["User-Agent"] += " " + utils._auxiliary.get_user_agent()
else:
headers["User-Agent"] = utils._auxiliary.get_user_agent()
headers.update(additional_headers)
return headers
def make_request(
self,
method, # type: str
url, # type: str
retry=False, # type: Union[bool, int]
cookies=None, # type: Optional[AnyCookiesContainer]
headers=None, # type: Optional[AnyHeadersContainer]
**kwargs, # type: Any
): # type: (...) -> AnyResponseType
"""
Sends the request with additional parameter handling for the current process definition.
"""
retries = int(retry) if retry is not None else 0
cookies = CaseInsensitiveDict(cookies or {})
headers = CaseInsensitiveDict(headers or {})
cookies.update(self.get_auth_cookies())
headers.update(self.headers.copy())
headers.update(self.get_auth_headers())
response = request_extra(method,
url=url,
settings=self.settings,
retries=retries,
headers=headers,
cookies=cookies,
**kwargs)
return response
def _get_email_headers_from_part(self, part, charset=None):
email_headers = list(part.items())
# TODO: the next 2 ifs can be condensed to use 'or'
if charset is None:
charset = part.get_content_charset()
if charset is None:
charset = 'utf8'
if not email_headers:
return {}
# Convert the header tuple into a dictionary
headers = CaseInsensitiveDict()
try:
[
headers.update({x[0]: self._get_string(x[1], charset)})
for x in email_headers
]
except Exception as e:
error_code, error_msg = self._get_error_message_from_exception(e)
err = "Error occurred while converting the header tuple into a dictionary"
self._base_connector.debug_print("{}. {}. {}".format(
err, error_code, error_msg))
# Handle received separately
try:
received_headers = [
self._get_string(x[1], charset) for x in email_headers
if x[0].lower() == 'received'
]
except Exception as e:
error_code, error_msg = self._get_error_message_from_exception(e)
err = "Error occurred while handling the received header tuple separately"
self._base_connector.debug_print("{}. {}. {}".format(
err, error_code, error_msg))
if received_headers:
headers['Received'] = received_headers
# handle the subject string, if required add a new key
subject = headers.get('Subject')
if subject:
try:
headers['decodedSubject'] = str(
make_header(decode_header(subject)))
except Exception:
headers['decodedSubject'] = self._decode_uni_string(
subject, subject)
return dict(headers)
def _check_creds(self, creds):
d = CaseInsensitiveDict()
if isinstance(creds, dict):
d.update(creds)
elif isinstance(creds, basestring):
if os.path.exists(creds):
creds = file(creds, 'r').read()
for line in creds.splitlines():
if ':' in line:
k, v = line.split(':', 1)
d[k.strip()] = v.strip()
else:
raise TypeError('unsupported type for credentials data')
if 'companyId' not in d and 'CID' in d:
d['companyId'] = d['CID']
if 'companyId' in d and 'psk' not in d:
raise ValueError('psk is required when companyId is provided')
elif 'psk' in d and 'companyId' not in d:
raise ValueError('companyId is required when psk is provided')
elif 'companyId' in d and 'psk' in d:
return {
'companyId': int(d['companyId']),
'psk': str(d['psk']),
}
elif 'loginSessionId' in d and 'profileId' not in d:
raise ValueError('profileId is required when loginSessionId is '
'provided')
elif 'profileId' in d and 'loginSessionId' not in d:
raise ValueError('loginSessionId is required when profileId is '
'provided')
elif 'loginSessionId' in d and 'profileId' in d:
return {
'loginSessionId': str(d['loginSessionId']),
'profileId': int(d['profileId']),
}
else:
raise ValueError('either companyId+psk or '
'loginSessionId+profileId must be provided')
def test_update_retains_unchanged(self):
cid = CaseInsensitiveDict({"foo": "foo", "bar": "bar"})
cid.update({"foo": "newfoo"})
assert cid["bar"] == "bar"
class Site(object):
"""
Public properties (member variables at the moment):
* url: Full url to site's api.php
* session: current request.session object
* log: an object that will be used for logging. ConsoleLog is created by default
"""
def __init__(self, url, headers=None, session=None, log=None):
self._loginOnDemand = False
self.session = session if session else requests.session()
self.log = log if log else ConsoleLog()
self.url = url
self.tokens = {}
self.noSSL = False # For non-ssl sites, it might be needed to avoid HTTPS
try:
script = os.path.abspath(sys.modules['__main__'].__file__)
except (KeyError, AttributeError):
script = sys.executable
path, f = os.path.split(script)
self.headers = CaseInsensitiveDict({u'User-Agent': u'%s-%s BareboneMWReq/0.1' % (os.path.basename(path), f)})
if headers:
self.headers.update(headers)
def __call__(self, action, **kwargs):
"""
Make an API call with any arguments provided as named values:
data = site('query', meta='siteinfo')
By default uses GET request to the default URL set in the Site constructor.
In case of an error, ApiError exception will be raised
Any warnings will be logged via the logging interface
:param action could also be
Several special "magic" parameters could be used to customize api call.
Special parameters must be all CAPS to avoid collisions with the server API:
:param POST: Use POST method when calling server API. Value is ignored.
:param HTTPS: Force https (ssl) protocol for this request. Value is ignored.
:param EXTRAS: Any extra parameters as passed to requests' session.request(). Value is a dict()
"""
# Magic CAPS parameters
method = 'POST' if 'POST' in kwargs or action in ['login', 'edit'] else 'GET'
forceSSL = not self.noSSL and (action == 'login' or 'SSL' in kwargs or 'HTTPS' in kwargs)
request_kw = dict() if 'EXTRAS' not in kwargs else kwargs['EXTRAS']
# Clean up magic CAPS params as they shouldn't be passed to the server
for k in ['POST', 'SSL', 'HTTPS', 'EXTRAS']:
if k in kwargs:
del kwargs[k]
for k, val in kwargs.items():
# Only support the well known types.
# Everything else should be client's responsibility
if isinstance(val, list) or isinstance(val, tuple):
kwargs[k] = '|'.join(val)
# Make server call
kwargs['action'] = action
kwargs['format'] = 'json'
if method == 'POST':
request_kw['data'] = kwargs
else:
request_kw['params'] = kwargs
if self._loginOnDemand and action != 'login':
self.login(self._loginOnDemand[0], self._loginOnDemand[1])
data = parseJson(self.request(method, forceSSL=forceSSL, **request_kw))
# Handle success and failure
if 'error' in data:
raise ApiError('Server API Error', data['error'])
if 'warnings' in data:
self.log(2, data['warnings'])
return data
def login(self, user, password, onDemand=False):
"""
:param user:
:param password:
:param onDemand: if True, will postpone login until an actual API request is made
:return:
"""
self.tokens = {}
if onDemand:
self._loginOnDemand = (user, password)
return
res = self('login', lgname=user, lgpassword=password)['login']
if res['result'] == 'NeedToken':
res = self('login', lgname=user, lgpassword=password, lgtoken=res['token'])['login']
if res['result'] != 'Success':
raise ApiError('Login failed', res)
self._loginOnDemand = False
def query(self, **kwargs):
"""
Call Query API with given parameters, and yield all results returned
by the server, properly handling result continuation.
"""
if 'rawcontinue' in kwargs:
raise ValueError("rawcontinue is not supported with query() function, use object's __call__()")
if 'continue' not in kwargs:
kwargs['continue'] = ''
req = kwargs
while True:
result = self('query', **req)
if 'query' in result:
yield result['query']
if 'continue' not in result:
break
# re-send all continue values in the next call
req = kwargs.copy()
req.update(result['continue'])
def queryPages(self, **kwargs):
"""
Query the server and return all page objects individually.
"""
incomplete = {}
changed = set()
for result in self.query(**kwargs):
if 'pages' not in result:
raise ApiError('Missing pages element in query result', result)
finished = incomplete.copy()
for pageId, page in result['pages'].items():
if pageId in changed:
continue
if pageId in incomplete:
del finished[pageId] # If server returned it => not finished
p = incomplete[pageId]
if 'lastrevid' in page and p['lastrevid'] != page['lastrevid']:
# someone else modified this page, it must be requested anew separately
changed.add(pageId)
del incomplete[pageId]
continue
self._mergePage(p, page)
else:
p = page
incomplete[pageId] = p
for pageId, page in finished.items():
if pageId not in changed:
yield page
for pageId, page in incomplete.items():
yield page
if changed:
# some pages have been changed between api calls, notify caller
raise ApiPagesModifiedError(list(changed))
def _mergePage(self, a, b):
"""
Recursively merge two page objects
"""
for k in b:
val = b[k]
if k in a:
if isinstance(val, dict):
self._mergePage(a[k], val)
elif isinstance(val, list):
a[k] = a[k] + val
else:
a[k] = val
else:
a[k] = val
def token(self, tokenType='csrf'):
if tokenType not in self.tokens:
self.tokens[tokenType] = next(self.query(meta='tokens', type=tokenType))['tokens'][tokenType + 'token']
return self.tokens[tokenType]
def request(self, method, forceSSL=False, headers=None, **request_kw):
"""Make a low level request to the server"""
url = self.url
if forceSSL:
parts = list(urlparse.urlparse(url))
parts[0] = 'https'
url = urlparse.urlunparse(parts)
if headers:
h = self.headers.copy()
h.update(headers)
headers = h
else:
headers = self.headers
r = self.session.request(method, url, headers=headers, **request_kw)
if not r.ok:
raise ApiError('Call failed', r)
if self.log.isEnabled(5):
dbg = [r.request.url, headers]
self.log(5, dbg)
return r
def proxy(identifier, in_method, in_headers, data):
# find endpoint
endpoint = pg.select1(
'endpoints',
what=['definition', 'method', 'pass_headers',
'headers', 'url', 'url_dynamic'],
where={'id': identifier}
)
if not endpoint:
return 'endpoint not found, create it at ' \
'<a href="/dashboard">dashboard</a>', 404, {}
event = {
'in': {
'time': time.time(),
'method': in_method,
'headers': dict(in_headers),
'body': data,
'replay': True
},
'out': {
'method': endpoint['method'],
'url': endpoint['url'],
'body': None,
'headers': {}
},
'response': {
'code': 0,
'body': ''
}
}
mutated, error = jq(endpoint['definition'], data=data)
if not mutated or error:
event['out']['error'] = error.decode('utf-8')
publish(identifier, event)
return 'transmutated into null and aborted', 201, {}
h = CaseInsensitiveDict({'Content-Type': 'application/json'})
if endpoint['pass_headers']:
h.update(in_headers)
h.update(endpoint['headers'])
event['out']['headers'] = dict(h)
# reformat the mutated data
mutatedjson = json.loads(mutated.decode('utf-8'))
if h.get('content-type') == 'application/x-www-form-urlencoded':
# oops, not json
mutated = urlencode(mutatedjson)
else:
mutated = json.dumps(mutatedjson)
event['out']['body'] = mutated
if endpoint['url_dynamic']:
urlb, error = jq(endpoint['url'], data=data)
print('URL', urlb, 'ERROR', error)
if not urlb:
event['out']['url_error'] = error.decode('utf-8')
publish(identifier, event)
return 'url building has failed', 200, {}
url = urlb.decode('utf-8')
event['out']['url'] = url
else:
url = endpoint['url']
# event['out'] is completed at this point
# and we all have everything needed to perform the request
if url and is_valid_url(url):
# we will only perform a request if there is an URL and it is valid
try:
s = requests.Session()
req = requests.Request(endpoint['method'], url,
data=mutated, headers=h).prepare()
resp = s.send(req, timeout=4)
if not resp.ok:
print('FAILED TO POST', resp.text, identifier, mutated)
except requests.exceptions.RequestException as e:
print(identifier, 'FAILED TO POST', mutated, 'TO URL', url)
print(e)
publish(identifier, event)
return "<request failed: '%s'>" % e, 503, {}
event['response']['code'] = resp.status_code
event['response']['body'] = resp.text
publish(identifier, event)
return resp.text, resp.status_code, dict(resp.headers)
else:
# no valid URL, just testing
publish(identifier, event)
return 'no URL to send this to', 201, {}
def test_update_retains_unchanged(self):
cid = CaseInsensitiveDict({'foo': 'foo', 'bar': 'bar'})
cid.update({'foo': 'newfoo'})
assert cid['bar'] == 'bar'
