def simple_post(domain, url, data, *, headers, auth_manager, verify, notify_addresses=None, payload_id=None): """ POST with a cleaner API, and return the actual HTTPResponse object, so that error codes can be interpreted. """ if isinstance(data, str): # Encode as UTF-8, otherwise requests will send data containing # non-ASCII characters as 'data:application/octet-stream;base64,...' data = data.encode('utf-8') default_headers = CaseInsensitiveDict({ "content-type": "text/xml", "content-length": str(len(data)), }) default_headers.update(headers) requests = Requests( domain, base_url=url, verify=verify, auth_manager=auth_manager, notify_addresses=notify_addresses, payload_id=payload_id, ) try: response = requests.post(None, data=data, headers=default_headers) except Exception as err: requests.notify_error(str(err)) raise if not 200 <= response.status_code < 300: message = f'HTTP status code {response.status_code}: {response.text}' requests.notify_error(message) return response
def string_to_sign(self, request): h = CaseInsensitiveDict() h.update(request.headers) # Try to use if b'x-amz-date' in h or 'x-amz-date' in h: date = '' else: date = h.get('Date') or self._get_date() request.headers['Date'] = date # Set the date header request.headers['Date'] = date # A fix for the content type header extraction in python 3 # This have to be done because requests will try to set application/www-url-encoded herader # if we pass bytes as the content, and the content-type is set with a key that is b'Content-Type' and not # 'Content-Type' content_type = '' if b'Content-Type' in request.headers: # Fix content type content_type = h.get(b'Content-Type') del request.headers[b'Content-Type'] request.headers['Content-Type'] = content_type msg = [ request.method, h.get(b'Content-MD5', '') or h.get('Content-MD5', ''), content_type or h.get('Content-Type', ''), date, self._get_canonicalized_amz_headers(h) + self._get_canonicalized_resource(request) ] return '\n'.join(msg)
def request(self, method, url, data=None, files=None): headers = CaseInsensitiveDict([ ('Accept', 'text/html,image/webp,image/apng,*/*;q=0.8'), ('Accept-Encoding', 'gzip, deflate'), ('Accept-Language', 'en-US,en;q=0.8'), ('Accept-Charset', 'ISO-8859-1,utf-8;q=0.7,*;q=0.7'), ('Cache-Control', 'no-cache'), ('Connection', 'keep-alive'), ('Host', 'www.pinterest.com'), ('Origin', 'https://www.pinterest.com'), ('Referer', HOME_PAGE), ('X-Requested-With', 'XMLHttpRequest'), ('Accept', 'application/json'), ('Content-Type', 'application/x-www-form-urlencoded; charset=UTF-8'), ('User-Agent', AGENT_STRING) ]) csrftoken = self.http.cookies.get('csrftoken') if csrftoken: headers.update([('X-CSRFToken', csrftoken)]) response = self.http.request(method, url, data=data, headers=headers, files=files, proxies=self.proxies) response.raise_for_status() self.registry.update(Registry.Key.COOKIES, response.cookies) return response
def get_server_parameters(self, config): parameters = config['parameters'].copy() listen_addresses, port = split_host_port(config['listen'], 5432) parameters.update(cluster_name=self._postgresql.scope, listen_addresses=listen_addresses, port=str(port)) if config.get('synchronous_mode', False): if self._synchronous_standby_names is None: if config.get('synchronous_mode_strict', False): parameters['synchronous_standby_names'] = '*' else: parameters.pop('synchronous_standby_names', None) else: parameters[ 'synchronous_standby_names'] = self._synchronous_standby_names if self._postgresql.major_version >= 90600 and parameters[ 'wal_level'] == 'hot_standby': parameters['wal_level'] = 'replica' ret = CaseInsensitiveDict({ k: v for k, v in parameters.items() if not self._postgresql.major_version or self._postgresql. major_version >= self.CMDLINE_OPTIONS.get(k, (0, 1, 90100))[2] }) ret.update({ k: os.path.join(self._config_dir, ret[k]) for k in ('hba_file', 'ident_file') if k in ret }) return ret
def send_document(url, data, timeout=10, method="post", *args, **kwargs): """Helper method to send a document via POST. Additional ``*args`` and ``**kwargs`` will be passed on to ``requests.post``. :arg url: Full url to send to, including protocol :arg data: Dictionary (will be form-encoded), bytes, or file-like object to send in the body :arg timeout: Seconds to wait for response (defaults to 10) :arg method: Method to use, defaults to post :returns: Tuple of status code (int or None) and error (exception class instance or None) """ logger.debug("send_document: url=%s, data=%s, timeout=%s, method=%s", url, data, timeout, method) if not method: method = "post" headers = CaseInsensitiveDict({ 'User-Agent': USER_AGENT, }) if "headers" in kwargs: # Update from kwargs headers.update(kwargs.get("headers")) kwargs.update({ "data": data, "timeout": timeout, "headers": headers }) request_func = getattr(requests, method) try: response = request_func(url, *args, **kwargs) logger.debug("send_document: response status code %s", response.status_code) return response.status_code, None # TODO support rate limit 429 code except RequestException as ex: logger.debug("send_document: exception %s", ex) return None, ex
class MockResponse(object): """ Mock response object with a status code and some content """ def __init__(self, status_code, content=None, headers=None): self.status_code = status_code self.content = content or "" self.headers = CaseInsensitiveDict() if headers: self.headers.update(headers) def raise_for_status(self): http_error_msg = "" if 400 <= self.status_code < 500: http_error_msg = "%s Client Error: ..." % self.status_code elif 500 <= self.status_code < 600: http_error_msg = "%s Server Error: ..." % self.status_code if http_error_msg: raise requests.HTTPError(http_error_msg, response=self) def json(self, **kwargs): return json.loads(self.content)
class DDWRT(Router): def __init__(self, conf, hostnames): self.hostnames = CaseInsensitiveDict() self.hostnames.update(hostnames) self.conf = conf self.auth = self.conf.auth() def clients(self): """ Receives all currently logged in users in a wifi network. :rtype : list :return: Returns a list of dicts, containing the following keys: mac, ipv4, seen, hostname """ clients = self._get_clients_raw() clients_json = [] for client in clients: client_hostname_from_router = client[0] client_ipv4 = client[1].strip() client_mac = client[2].strip().upper() client_hostname = self.hostnames.get(client_mac, client_hostname_from_router).strip() client_connections = int(client[3].strip()) # Clients with less than 20 connections are considered offline if client_connections < 20: continue clients_json.append({ 'mac': client_mac, 'ipv4': client_ipv4, 'seen': int(time.time()), 'hostname': client_hostname, }) logger.debug('The router got us {} clients.'.format(len(clients_json))) logger.debug(str(clients_json)) return clients_json def _get_clients_raw(self): info_page = self.conf.internal() response = requests.get(info_page, auth=self.auth) logger.info('Got response from router with code {}.'.format(response.status_code)) return DDWRT._convert_to_clients(response.text) or [] @staticmethod def _convert_to_clients(router_info_all): # Split router info in lines and filter empty info router_info_lines = filter(None, router_info_all.split("\n")) # Get key / value of router info router_info_items = dict() for item in router_info_lines: key, value = item[1:-1].split("::") # Remove curly braces and split router_info_items[key.strip()] = value.strip() # Get client info as a list arp_table = utils.groupn(router_info_items['arp_table'].replace("'", "").split(","), 4) dhcp_leases = utils.groupn(router_info_items['dhcp_leases'].replace("'", "").split(","), 5) return arp_table if (len(arp_table) > 0) else []
def string_to_sign(self, request): """ Generates the string we need to sign on. Params: - request The request object Returns String ready to be signed on """ # We'll use case insensitive dict to store the headers h = CaseInsensitiveDict() # Add the hearders h.update(request.headers) # If we have an 'x-amz-date' header, # we'll try to use it instead of the date if b'x-amz-date' in h or 'x-amz-date' in h: date = '' else: # No x-amz-header, we'll generate a date date = h.get('Date') or self._get_date() # Set the date header request.headers['Date'] = date # A fix for the content type header extraction in python 3 # This have to be done because requests will try to set # application/www-url-encoded header if we pass bytes as the content, # and the content-type is set with a key that is b'Content-Type' and # not 'Content-Type' content_type = '' if b'Content-Type' in request.headers: # Fix content type content_type = h.get(b'Content-Type') del request.headers[b'Content-Type'] request.headers['Content-Type'] = content_type # The string we're about to generate # There's more information about it here: # http://docs.aws.amazon.com/AmazonS3/latest/dev/ # RESTAuthentication.html#ConstructingTheAuthenticationHeader msg = [ # HTTP Method request.method, # MD5 If provided h.get(b'Content-MD5', '') or h.get('Content-MD5', ''), # Content type if provided content_type or h.get('Content-Type', ''), # Date date, # Canonicalized special amazon headers and resource uri self._get_canonicalized_amz_headers(h) + self._get_canonicalized_resource(request) ] # join with a newline and return return '\n'.join(msg)
class WorkerRequest(WPSRequest): """ Extended :mod:`pywps` request with additional handling provided by :mod:`weaver`. """ _auth_headers = CaseInsensitiveDict({ # take advantage of case insensitive only, value don't care "Authorization": None, "X-Auth": None, sd.XAuthVaultFileHeader.name: None, }) def __init__(self, http_request=None, http_headers=None, **kwargs): # type: (Optional[AnyRequestType], Optional[AnyHeadersContainer], **Any) -> None if http_request and not isinstance(http_request, WerkzeugRequest): http_request = extend_instance(http_request, WerkzeugRequest) super(WorkerRequest, self).__init__(http_request, **kwargs) self.auth_headers = CaseInsensitiveDict() if http_request: self.auth_headers.update( self.parse_auth_headers(http_request.headers)) if http_headers: self.auth_headers.update(self.parse_auth_headers(http_headers)) def parse_auth_headers(self, headers): # type: (Optional[AnyHeadersContainer]) -> HeadersType if not headers: return {} if isinstance(headers, list): headers = dict(headers) auth_headers = {} for name, value in headers.items(): if name in self._auth_headers: auth_headers[name] = value return auth_headers
def request(self, method, url, data=None, files=None, extra_headers=None): headers = CaseInsensitiveDict([ ('Referer', HOME_PAGE), ('X-Requested-With', 'XMLHttpRequest'), ('Accept', 'application/json'), ('Content-Type', 'application/x-www-form-urlencoded; charset=UTF-8'), ('User-Agent', self.user_agent) ]) csrftoken = self.http.cookies.get('csrftoken') if csrftoken: headers.update([('X-CSRFToken', csrftoken)]) if extra_headers is not None: for h in extra_headers: headers.update([(h, extra_headers[h])]) response = self.http.request(method, url, data=data, headers=headers, files=files, proxies=self.proxies) response.raise_for_status() self.registry.update(Registry.Key.COOKIES, response.cookies) return response
def send_document(url, data, timeout=10, *args, **kwargs): """Helper method to send a document via POST. Additional ``*args`` and ``**kwargs`` will be passed on to ``requests.post``. :arg url: Full url to send to, including protocol :arg data: POST data to send (dict) :arg timeout: Seconds to wait for response (defaults to 10) :returns: Tuple of status code (int or None) and error (exception class instance or None) """ logger.debug("send_document: url=%s, data=%s, timeout=%s", url, data, timeout) headers = CaseInsensitiveDict({ 'User-Agent': USER_AGENT, }) if "headers" in kwargs: # Update from kwargs headers.update(kwargs.get("headers")) kwargs.update({ "data": data, "timeout": timeout, "headers": headers }) try: response = requests.post(url, *args, **kwargs) logger.debug("send_document: response status code %s", response.status_code) return response.status_code, None except RequestException as ex: logger.debug("send_document: exception %s", ex) return None, ex
class MockResponse(object): """ Mock response object with a status code and some content """ def __init__(self, status_code, content=None, headers=None): self.status_code = status_code self.content = content or '' self.headers = CaseInsensitiveDict() if headers: self.headers.update(headers) def raise_for_status(self): http_error_msg = '' if 400 <= self.status_code < 500: http_error_msg = '%s Client Error: ...' % self.status_code elif 500 <= self.status_code < 600: http_error_msg = '%s Server Error: ...' % self.status_code if http_error_msg: raise requests.HTTPError(http_error_msg, response=self) def json(self, **kwargs): return json.loads(self.content)
def request(self, method, url, params=None, data=None, files=None, headers=None, ajax=False, stream=None): """ :rtype: requests.models.Response """ _headers = CaseInsensitiveDict([ ('Accept', 'text/html,image/webp,image/apng,*/*;q=0.8'), ('Accept-Encoding', 'gzip, deflate'), ('Accept-Language', 'en-US,en;q=0.8'), ('Accept-Charset', 'ISO-8859-1,utf-8;q=0.7,*;q=0.7'), ('Cache-Control', 'no-cache'), ('Connection', 'keep-alive'), ('Host', 'www.pinterest.com'), ('Origin', 'https://www.pinterest.com'), ('Referer', self.home_page), ('User-Agent', self.registry.get(Registry.Key.USER_AGENT))]) if method.upper() == 'POST': _headers.update([('Content-Type', 'application/x-www-form-urlencoded; charset=UTF-8')]) if ajax: _headers.update([('Accept', 'application/json')]) csrftoken = self.http.cookies.get('csrftoken') if csrftoken: _headers.update([('X-CSRFToken', csrftoken)]) _headers.update([('X-Requested-With', 'XMLHttpRequest')]) if headers: _headers.update(headers) response = self.http.request(method, url, params=params, data=data, headers=_headers, files=files, timeout=60, proxies=self.proxies, stream=stream) response.raise_for_status() self.registry.update(Registry.Key.COOKIES, response.cookies) return response
def request(self, method, path, data=None, headers={}, query=None, response_type=None): url = '{0}{1}'.format(ApiClient.__api_root, path) req_headers = CaseInsensitiveDict({ 'User-Agent': 'sonic-cli' }) req_headers.update(headers) body = None if data is not None: if 'Content-Type' not in req_headers: req_headers['Content-Type'] = 'application/yang-data+json' body = json.dumps(data) try: r = ApiClient.__session.request( method, url, headers=req_headers, data=body, params=query, verify=False) return Response(r, response_type) except requests.RequestException as e: log_warning('cli_client request exception: ' + str(e)) #TODO have more specific error message based msg = '%Error: Could not connect to Management REST Server' return ApiClient.__new_error_response(msg)
def string_to_sign(self, request): """ Generates the string we need to sign on. Params: - request The request object Returns String ready to be signed on """ # We'll use case insensitive dict to store the headers h = CaseInsensitiveDict() # Add the hearders h.update(request.headers) # If we have an 'x-amz-date' header, # we'll try to use it instead of the date if b'x-amz-date' in h or 'x-amz-date' in h: date = '' else: # No x-amz-header, we'll generate a date date = h.get('Date') or self._get_date() # Set the date header request.headers['Date'] = date # A fix for the content type header extraction in python 3 # This have to be done because requests will try to set # application/www-url-encoded header if we pass bytes as the content, # and the content-type is set with a key that is b'Content-Type' and # not 'Content-Type' content_type = '' if b'Content-Type' in request.headers: # Fix content type content_type = h.get(b'Content-Type') del request.headers[b'Content-Type'] request.headers['Content-Type'] = content_type # The string we're about to generate # There's more information about it here: # http://docs.aws.amazon.com/AmazonS3/latest/dev/ # RESTAuthentication.html#ConstructingTheAuthenticationHeader msg = [ # HTTP Method request.method, # MD5 If provided h.get(b'Content-MD5', '') or h.get('Content-MD5', ''), # Content type if provided content_type or h.get('Content-Type', ''), # Date date, # Canonicalized special amazon headers and resource uri self._get_canonicalized_amz_headers(h) + self._get_canonicalized_resource(request) ] # join with a newline and return return '\n'.join(msg)
def _check_creds(self, creds): d = CaseInsensitiveDict() if isinstance(creds, dict): d.update(creds) elif isinstance(creds, basestring): if os.path.exists(creds): creds = file(creds, "r").read() for line in creds.splitlines(): if ":" in line: k, v = line.split(":", 1) d[k.strip()] = v.strip() else: raise TypeError("unsupported type for credentials data") if "companyId" not in d and "CID" in d: d["companyId"] = d["CID"] if "companyId" in d and not "psk" in d: raise ValueError("psk is required when companyId is provided") elif "psk" in d and not "companyId" in d: raise ValueError("companyId is required when psk is provided") elif "companyId" in d and "psk" in d: return {"companyId": int(d["companyId"]), "psk": str(d["psk"])} elif "loginSessionId" in d and not "profileId" in d: raise ValueError("profileId is required when loginSessionId is " "provided") elif "profileId" in d and not "loginSessionId" in d: raise ValueError("loginSessionId is required when profileId is " "provided") elif "loginSessionId" in d and "profileId" in d: return {"loginSessionId": str(d["loginSessionId"]), "profileId": int(d["profileId"])} else: raise ValueError("either companyId+psk or " "loginSessionId+profileId must be provided")
def request(self, method, url, data=None, files=None, extra_headers=None): headers = CaseInsensitiveDict([ ("Referer", HOME_PAGE), ("X-Requested-With", "XMLHttpRequest"), ("Accept", "application/json"), ("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8"), ("User-Agent", self.user_agent), ]) csrftoken = self.http.cookies.get("csrftoken") if csrftoken: headers.update([("X-CSRFToken", csrftoken)]) if extra_headers is not None: for h in extra_headers: headers.update([(h, extra_headers[h])]) response = self.http.request(method, url, data=data, headers=headers, files=files, proxies=self.proxies) response.raise_for_status() return response
def test_preserve_last_key_case(self): cid = CaseInsensitiveDict({"Accept": "application/json", "user-Agent": "requests"}) cid.update({"ACCEPT": "application/json"}) cid["USER-AGENT"] = "requests" keyset = frozenset(["ACCEPT", "USER-AGENT"]) assert frozenset(i[0] for i in cid.items()) == keyset assert frozenset(cid.keys()) == keyset assert frozenset(cid) == keyset
def test_update(self): cid = CaseInsensitiveDict() cid['spam'] = 'blueval' cid.update({'sPam': 'notblueval'}) assert cid['spam'] == 'notblueval' cid = CaseInsensitiveDict({'Foo': 'foo','BAr': 'bar'}) cid.update({'fOO': 'anotherfoo', 'bAR': 'anotherbar'}) assert len(cid) == 2 assert cid['foo'] == 'anotherfoo' assert cid['bar'] == 'anotherbar'
def test_update(self): cid = CaseInsensitiveDict() cid['spam'] = 'blueval' cid.update({'sPam': 'notblueval'}) self.assertEqual(cid['spam'], 'notblueval') cid = CaseInsensitiveDict({'Foo': 'foo', 'BAr': 'bar'}) cid.update({'fOO': 'anotherfoo', 'bAR': 'anotherbar'}) self.assertEqual(len(cid), 2) self.assertEqual(cid['foo'], 'anotherfoo') self.assertEqual(cid['bar'], 'anotherbar')
def test_update(self): cid = CaseInsensitiveDict() cid['spam'] = 'blueval' cid.update({'sPam': 'notblueval'}) self.assertEqual(cid['spam'], 'notblueval') cid = CaseInsensitiveDict({'Foo': 'foo','BAr': 'bar'}) cid.update({'fOO': 'anotherfoo', 'bAR': 'anotherbar'}) self.assertEqual(len(cid), 2) self.assertEqual(cid['foo'], 'anotherfoo') self.assertEqual(cid['bar'], 'anotherbar')
def test_update(self): cid = CaseInsensitiveDict() cid['spam'] = 'blueval' cid.update({'sPam': 'notblueval'}) assert cid['spam'] == 'notblueval' cid = CaseInsensitiveDict({'Foo': 'foo','BAr': 'bar'}) cid.update({'fOO': 'anotherfoo', 'bAR': 'anotherbar'}) assert len(cid) == 2 assert cid['foo'] == 'anotherfoo' assert cid['bar'] == 'anotherbar'
def test_update(self): cid = CaseInsensitiveDict() cid["spam"] = "blueval" cid.update({"sPam": "notblueval"}) assert cid["spam"] == "notblueval" cid = CaseInsensitiveDict({"Foo": "foo", "BAr": "bar"}) cid.update({"fOO": "anotherfoo", "bAR": "anotherbar"}) assert len(cid) == 2 assert cid["foo"] == "anotherfoo" assert cid["bar"] == "anotherbar"
def test_update(self): cid = CaseInsensitiveDict() cid["spam"] = "blueval" cid.update({"sPam": "notblueval"}) self.assertEqual(cid["spam"], "notblueval") cid = CaseInsensitiveDict({"Foo": "foo", "BAr": "bar"}) cid.update({"fOO": "anotherfoo", "bAR": "anotherbar"}) self.assertEqual(len(cid), 2) self.assertEqual(cid["foo"], "anotherfoo") self.assertEqual(cid["bar"], "anotherbar")
def __init__(self, response=None, status=None, headers=None, mimetype=None, content_type=None, direct_passthrough=False): headers = CaseInsensitiveDict(headers) if headers is not None else None if response is not None and isinstance( response, BaseResponse) and response.headers is not None: headers = CaseInsensitiveDict(response.headers) if headers is None: headers = CaseInsensitiveDict() h = headers h['Access-Control-Allow-Origin'] = headers.get( 'Access-Control-Allow-Origin', '*') h['Access-Control-Allow-Methods'] = headers.get( 'Access-Control-Allow-Methods', "GET, PUT, POST, HEAD, OPTIONS, DELETE") h['Access-Control-Max-Age'] = headers.get('Access-Control-Max-Age', "21600") h['Cache-Control'] = headers.get( 'Cache-Control', "no-cache, must-revalidate, no-store") if 'Access-Control-Allow-Headers' not in headers and len( headers.keys()) > 0: h['Access-Control-Allow-Headers'] = ', '.join(iterkeys(headers)) data = None if response is not None and isinstance(response, string_types): data = response response = None if response is not None and isinstance(response, BaseResponse): new_response_headers = CaseInsensitiveDict( response.headers if response.headers is not None else {}) new_response_headers.update(h) response.headers = new_response_headers headers = None data = response.get_data() else: headers.update(h) headers = dict(headers) super(IppResponse, self).__init__(response=response, status=status, headers=headers, mimetype=mimetype, content_type=content_type, direct_passthrough=direct_passthrough) if data is not None: self.set_data(data)
def request(self, method, url, accept_json=False, headers=None, params=None, json=None, data=None, files=None, **kwargs): full_url = self.url + url input_headers = _remove_null_values(headers) if headers else {} headers = CaseInsensitiveDict({'user-agent': 'watson-developer-cloud-python-' + __version__}) if accept_json: headers['accept'] = 'application/json' headers.update(input_headers) # Remove keys with None values params = _remove_null_values(params) json = _remove_null_values(json) data = _remove_null_values(data) files = _remove_null_values(files) # Support versions of requests older than 2.4.2 without the json input if not data and json is not None: data = json_import.dumps(json) headers.update({'content-type': 'application/json'}) auth = None if self.username and self.password: auth = (self.username, self.password) if self.api_key is not None: if params is None: params = {} if url.startswith('https://gateway-a.watsonplatform.net/calls'): params['apikey'] = self.api_key else: params['api_key'] = self.api_key response = requests.request(method=method, url=full_url, cookies=self.jar, auth=auth, headers=headers, params=params, data=data, files=files, **kwargs) if 200 <= response.status_code <= 299: if accept_json: response_json = response.json() if 'status' in response_json and response_json['status'] == 'ERROR': response.status_code = 400 error_message = 'Unknown error' if 'statusInfo' in response_json: error_message = response_json['statusInfo'] if error_message == 'invalid-api-key': response.status_code = 401 raise WatsonException('Error: ' + error_message) return response_json return response else: if response.status_code == 401: error_message = 'Unauthorized: Access is denied due to invalid credentials' else: error_message = self._get_error_message(response) raise WatsonException(error_message)
def test_preserve_last_key_case(self): cid = CaseInsensitiveDict({ 'Accept': 'application/json', 'user-Agent': 'requests', }) cid.update({'ACCEPT': 'application/json'}) cid['USER-AGENT'] = 'requests' keyset = frozenset(['ACCEPT', 'USER-AGENT']) assert frozenset(i[0] for i in cid.items()) == keyset assert frozenset(cid.keys()) == keyset assert frozenset(cid) == keyset
def test_preserve_last_key_case(self): cid = CaseInsensitiveDict({ 'Accept': 'application/json', 'user-Agent': 'requests', }) cid.update({'ACCEPT': 'application/json'}) cid['USER-AGENT'] = 'requests' keyset = frozenset(['ACCEPT', 'USER-AGENT']) assert frozenset(i[0] for i in cid.items()) == keyset assert frozenset(cid.keys()) == keyset assert frozenset(cid) == keyset
def request(self, method, url, accept_json=False, headers=None, params=None, json=None, data=None, files=None, **kwargs): full_url = self.url + url input_headers = _remove_null_values(headers) if headers else {} headers = CaseInsensitiveDict({'user-agent': 'watson-developer-cloud-python-' + __version__}) if accept_json: headers['accept'] = 'application/json' headers.update(input_headers) # Remove keys with None values params = _remove_null_values(params) json = _remove_null_values(json) data = _remove_null_values(data) files = _remove_null_values(files) # Support versions of requests older than 2.4.2 without the json input if not data and json is not None: data = json_import.dumps(json) headers.update({'content-type': 'application/json'}) auth = None if self.username and self.password: auth = (self.username, self.password) if self.api_key is not None: if params is None: params = {} params['apikey'] = self.api_key response = requests.request(method=method, url=full_url, cookies=self.jar, auth=auth, headers=headers, params=params, data=data, files=files, **kwargs) if 200 <= response.status_code <= 299: if accept_json: response_json = response.json() if 'status' in response_json and response_json['status'] == 'ERROR': response.status_code = 400 error_message = 'Unknown error' if 'statusInfo' in response_json: error_message = response_json['statusInfo'] if error_message == 'invalid-api-key': response.status_code = 401 raise WatsonException('Error: ' + error_message) return response_json return response else: if response.status_code == 401: error_message = 'Unauthorized: Access is denied due to invalid credentials' else: error_message = self._get_error_message(response) raise WatsonException(error_message)
def _prepare_request(self, command, json, opcode_name, fetch_list, **kwargs): kwargs = CaseInsensitiveDict(kwargs) kwargs.update({ 'apiKey': self.key, opcode_name: command, }) if json: kwargs['response'] = 'json' if 'page' in kwargs or fetch_list: kwargs.setdefault('pagesize', 500) kwarg = 'params' if self.method == 'get' else 'data' return kwarg, dict(kwargs._store.values())
def _prepare_request(self, command, json, opcode_name, fetch_list, **kwargs): params = CaseInsensitiveDict(**kwargs) params.update({ 'apiKey': self.key, opcode_name: command, }) if json: params['response'] = 'json' if 'page' in kwargs or fetch_list: params.setdefault('pagesize', PAGE_SIZE) kind = 'params' if self.method == 'get' else 'data' return kind, {k: v for k, v in params.items()}
def _configure_headers(self, additional_headers): headers = CaseInsensitiveDict() headers.update(requests.utils.default_headers()) headers["api-key"] = self._config.api_key headers["content-type"] = "application/json" headers["accept"] = "application/json" headers["x-cdp-sdk"] = "CognitePythonSDK:{}".format(utils._auxiliary.get_current_sdk_version()) headers["x-cdp-app"] = self._config.client_name if "User-Agent" in headers: headers["User-Agent"] += " " + utils._auxiliary.get_user_agent() else: headers["User-Agent"] = utils._auxiliary.get_user_agent() headers.update(additional_headers) return headers
def http_get(self, url, headers=None): """ returns (status, header, content) status: int header: dict content: string """ hdrs = CaseInsensitiveDict() if headers: hdrs.update(headers) if not 'user-agent' in hdrs: hdrs['user-agent'] = USER_AGENT print "http_get:", url r = requests.get(url, headers=hdrs) return (r.status_code, r.headers, r.text)
def simple_request(domain, url, data, *, headers, auth_manager, verify, method="POST", notify_addresses=None, payload_id=None): if isinstance(data, str): # Encode as UTF-8, otherwise requests will send data containing # non-ASCII characters as 'data:application/octet-stream;base64,...' data = data.encode('utf-8') default_headers = CaseInsensitiveDict({ "content-type": "text/xml", "content-length": str(len(data)), }) default_headers.update(headers) requests = Requests( domain, base_url=url, verify=verify, auth_manager=auth_manager, notify_addresses=notify_addresses, payload_id=payload_id, ) request_methods = { REQUEST_DELETE: requests.delete, REQUEST_POST: requests.post, REQUEST_PUT: requests.put, } try: request_method = request_methods[method] except KeyError: raise ValueError( f"Method must be one of {', '.join(request_methods.keys())}") try: response = request_method(None, data=data, headers=default_headers) except Exception as err: requests.notify_error(str(err)) raise if not 200 <= response.status_code < 300: message = f'HTTP status code {response.status_code}: {response.text}' requests.notify_error(message) return response
def _merge_headers(session, headers): if headers is None: headers = {} if session.headers is None: merged_headers = {} else: # Session headers are the default but local headers # have priority and can override values merged_headers = session.headers.copy() # Make sure merged_headers are CaseIsensitiveDict if not isinstance(merged_headers, CaseInsensitiveDict): merged_headers = CaseInsensitiveDict(merged_headers) merged_headers.update(headers) return merged_headers
def generate_request(method, url, body): """ Generate our own custom request, so we can calculate digest auth. """ method = method.upper() url = url files = [] json_string = None headers = CaseInsensitiveDict({ 'Accept': 'application/json', 'Accept-Encoding': 'gzip, deflate', 'Connection': 'keep-alive', 'Content-Type': 'application/json', 'User-Agent': 'stormpath-flask/0.4.4 flask/0.10.1 stormpath-sdk-python/2.4.5 python/2.7.6 Linux/LinuxMint (Linux-3.13.0-37-generic-x86_64-with-LinuxMint-17.1-rebecca)' }) if body: headers.update({'Content-Length': str(len(json.dumps(body)))}) params = OrderedDict() auth = Sauthc1Signer( id=os.environ.get('STORMPATH_API_KEY_ID'), secret=os.environ.get('STORMPATH_API_KEY_SECRET')) cookies = RequestsCookieJar() hooks = {'response': []} pr = PreparedRequest() if body: json_body = json.dumps(body) else: json_body = None pr.prepare( method=method.upper(), url=url, files=files, data=json_body, json=json_string, headers=headers, params=params, auth=auth, cookies=cookies, hooks=hooks, ) return pr
def _prepare_request(self, command, json=True, opcode_name='command', fetch_list=False, **kwargs): params = CaseInsensitiveDict(**kwargs) params.update({ 'apiKey': self.key, opcode_name: command, }) if json: params['response'] = 'json' if 'page' in kwargs or fetch_list: params.setdefault('pagesize', PAGE_SIZE) if 'expires' not in params and self.expiration.total_seconds() >= 0: params['signatureVersion'] = '3' tz = pytz.utc expires = tz.localize(datetime.utcnow() + self.expiration) params['expires'] = expires.astimezone(tz).strftime(EXPIRES_FORMAT) kind = 'params' if self.method == 'get' else 'data' return kind, dict(params.items())
def request(url, params=None, data=None, headers=None, ajax=False, method=None): if not method: method = 'GET' elif isinstance(method, bytes): try: method = method.decode('utf-8').upper() except UnicodeDecodeError: method = method.decode('iso-8859-1').upper() else: raise TypeError("Expected HTTP request method as a str or None type") assert (method in ['GET', 'HEAD', 'POST', 'PUT', 'DELETE', 'CONNECT', 'OPTIONS', 'TRACE', 'PATCH']), 'Unsupported HTTP Request method.' _headers = CaseInsensitiveDict([ ('Accept', 'text/html,image/webp,image/apng,*/*;q=0.8'), ('Accept-Encoding', 'gzip, deflate'), ('Accept-Language', 'en-US,en;q=0.8'), ('Accept-Charset', 'ISO-8859-1,utf-8;q=0.7,*;q=0.7'), ('Cache-Control', 'no-cache'), ('Connection', 'keep-alive'), ('Host', ''), ('Origin', ''), ('Referer', ''), ('User-Agent', '')]) if data is not None: if method == 'GET': method = 'POST' # If data is empty but not None. We will post with no body. So no Content-Type header set if data: _headers.update([('Content-Type', 'application/x-www-form-urlencoded; charset=UTF-8')]) if ajax: _headers.update([('Accept', 'application/json')]) csrftoken = http.cookies.get('csrftoken') if csrftoken: _headers.update([('X-CSRFToken', csrftoken)]) _headers.update([('X-Requested-With', 'XMLHttpRequest')]) if headers: _headers.update(headers) try: response = http.request(method, url, params=params, data=data, headers=_headers, timeout=60) except Exception as ex: raise Exception("Request Error, %s(%s) on %s" % (ex.__class__.__name__, ex.message, url)) raise_for_status(response) return response
def http_get(self, url, headers=None): """ returns (status, header, content) status: int header: dict content: string """ hdrs = CaseInsensitiveDict() if headers: hdrs.update(headers) if not 'user-agent' in hdrs: hdrs['user-agent'] = USER_AGENT print "http_get:", url try: r = requests.get(url, headers=hdrs) return (r.status_code, r.headers, r.text) except requests.exceptions.ConnectionError, e: # happens for example, when return (500, None, "requests.exceptions.ConnectionError getting %s" % url)
def handle_response(cls, response_code, response_data): """ Handles service response. Returns the message, status or workflow id, if applicable """ client_error = response_code >= 400 and response_code < 500 success = response_code >= 200 and response_code < 300 workflow = CaseInsensitiveDict({"Message":"", "Id": 0, "Status": 0}) if client_error or success: try: response_content = json.loads(response_data) workflow.update(response_content) except ValueError: workflow["Message"] = "Invalid response content" else: workflow["Message"] = "{0}: Internal server error".format(response_code) return workflow
def _configure_headers(self, additional_headers): headers = CaseInsensitiveDict() headers.update(requests.utils.default_headers()) if self._config.token is None: headers["api-key"] = self._config.api_key elif isinstance(self._config.token, str): headers["Authorization"] = "Bearer {}".format(self._config.token) elif isinstance(self._config.token, Callable): headers["Authorization"] = "Bearer {}".format(self._config.token()) else: raise TypeError("'token' must be str, Callable, or None.") headers["content-type"] = "application/json" headers["accept"] = "application/json" headers["x-cdp-sdk"] = "CognitePythonSDK:{}".format(utils._auxiliary.get_current_sdk_version()) headers["x-cdp-app"] = self._config.client_name if "User-Agent" in headers: headers["User-Agent"] += " " + utils._auxiliary.get_user_agent() else: headers["User-Agent"] = utils._auxiliary.get_user_agent() headers.update(additional_headers) return headers
def make_request( self, method, # type: str url, # type: str retry=False, # type: Union[bool, int] cookies=None, # type: Optional[AnyCookiesContainer] headers=None, # type: Optional[AnyHeadersContainer] **kwargs, # type: Any ): # type: (...) -> AnyResponseType """ Sends the request with additional parameter handling for the current process definition. """ retries = int(retry) if retry is not None else 0 cookies = CaseInsensitiveDict(cookies or {}) headers = CaseInsensitiveDict(headers or {}) cookies.update(self.get_auth_cookies()) headers.update(self.headers.copy()) headers.update(self.get_auth_headers()) response = request_extra(method, url=url, settings=self.settings, retries=retries, headers=headers, cookies=cookies, **kwargs) return response
def _get_email_headers_from_part(self, part, charset=None): email_headers = list(part.items()) # TODO: the next 2 ifs can be condensed to use 'or' if charset is None: charset = part.get_content_charset() if charset is None: charset = 'utf8' if not email_headers: return {} # Convert the header tuple into a dictionary headers = CaseInsensitiveDict() try: [ headers.update({x[0]: self._get_string(x[1], charset)}) for x in email_headers ] except Exception as e: error_code, error_msg = self._get_error_message_from_exception(e) err = "Error occurred while converting the header tuple into a dictionary" self._base_connector.debug_print("{}. {}. {}".format( err, error_code, error_msg)) # Handle received separately try: received_headers = [ self._get_string(x[1], charset) for x in email_headers if x[0].lower() == 'received' ] except Exception as e: error_code, error_msg = self._get_error_message_from_exception(e) err = "Error occurred while handling the received header tuple separately" self._base_connector.debug_print("{}. {}. {}".format( err, error_code, error_msg)) if received_headers: headers['Received'] = received_headers # handle the subject string, if required add a new key subject = headers.get('Subject') if subject: try: headers['decodedSubject'] = str( make_header(decode_header(subject))) except Exception: headers['decodedSubject'] = self._decode_uni_string( subject, subject) return dict(headers)
def _check_creds(self, creds): d = CaseInsensitiveDict() if isinstance(creds, dict): d.update(creds) elif isinstance(creds, basestring): if os.path.exists(creds): creds = file(creds, 'r').read() for line in creds.splitlines(): if ':' in line: k, v = line.split(':', 1) d[k.strip()] = v.strip() else: raise TypeError('unsupported type for credentials data') if 'companyId' not in d and 'CID' in d: d['companyId'] = d['CID'] if 'companyId' in d and 'psk' not in d: raise ValueError('psk is required when companyId is provided') elif 'psk' in d and 'companyId' not in d: raise ValueError('companyId is required when psk is provided') elif 'companyId' in d and 'psk' in d: return { 'companyId': int(d['companyId']), 'psk': str(d['psk']), } elif 'loginSessionId' in d and 'profileId' not in d: raise ValueError('profileId is required when loginSessionId is ' 'provided') elif 'profileId' in d and 'loginSessionId' not in d: raise ValueError('loginSessionId is required when profileId is ' 'provided') elif 'loginSessionId' in d and 'profileId' in d: return { 'loginSessionId': str(d['loginSessionId']), 'profileId': int(d['profileId']), } else: raise ValueError('either companyId+psk or ' 'loginSessionId+profileId must be provided')
def test_update_retains_unchanged(self): cid = CaseInsensitiveDict({"foo": "foo", "bar": "bar"}) cid.update({"foo": "newfoo"}) assert cid["bar"] == "bar"
class Site(object): """ Public properties (member variables at the moment): * url: Full url to site's api.php * session: current request.session object * log: an object that will be used for logging. ConsoleLog is created by default """ def __init__(self, url, headers=None, session=None, log=None): self._loginOnDemand = False self.session = session if session else requests.session() self.log = log if log else ConsoleLog() self.url = url self.tokens = {} self.noSSL = False # For non-ssl sites, it might be needed to avoid HTTPS try: script = os.path.abspath(sys.modules['__main__'].__file__) except (KeyError, AttributeError): script = sys.executable path, f = os.path.split(script) self.headers = CaseInsensitiveDict({u'User-Agent': u'%s-%s BareboneMWReq/0.1' % (os.path.basename(path), f)}) if headers: self.headers.update(headers) def __call__(self, action, **kwargs): """ Make an API call with any arguments provided as named values: data = site('query', meta='siteinfo') By default uses GET request to the default URL set in the Site constructor. In case of an error, ApiError exception will be raised Any warnings will be logged via the logging interface :param action could also be Several special "magic" parameters could be used to customize api call. Special parameters must be all CAPS to avoid collisions with the server API: :param POST: Use POST method when calling server API. Value is ignored. :param HTTPS: Force https (ssl) protocol for this request. Value is ignored. :param EXTRAS: Any extra parameters as passed to requests' session.request(). Value is a dict() """ # Magic CAPS parameters method = 'POST' if 'POST' in kwargs or action in ['login', 'edit'] else 'GET' forceSSL = not self.noSSL and (action == 'login' or 'SSL' in kwargs or 'HTTPS' in kwargs) request_kw = dict() if 'EXTRAS' not in kwargs else kwargs['EXTRAS'] # Clean up magic CAPS params as they shouldn't be passed to the server for k in ['POST', 'SSL', 'HTTPS', 'EXTRAS']: if k in kwargs: del kwargs[k] for k, val in kwargs.items(): # Only support the well known types. # Everything else should be client's responsibility if isinstance(val, list) or isinstance(val, tuple): kwargs[k] = '|'.join(val) # Make server call kwargs['action'] = action kwargs['format'] = 'json' if method == 'POST': request_kw['data'] = kwargs else: request_kw['params'] = kwargs if self._loginOnDemand and action != 'login': self.login(self._loginOnDemand[0], self._loginOnDemand[1]) data = parseJson(self.request(method, forceSSL=forceSSL, **request_kw)) # Handle success and failure if 'error' in data: raise ApiError('Server API Error', data['error']) if 'warnings' in data: self.log(2, data['warnings']) return data def login(self, user, password, onDemand=False): """ :param user: :param password: :param onDemand: if True, will postpone login until an actual API request is made :return: """ self.tokens = {} if onDemand: self._loginOnDemand = (user, password) return res = self('login', lgname=user, lgpassword=password)['login'] if res['result'] == 'NeedToken': res = self('login', lgname=user, lgpassword=password, lgtoken=res['token'])['login'] if res['result'] != 'Success': raise ApiError('Login failed', res) self._loginOnDemand = False def query(self, **kwargs): """ Call Query API with given parameters, and yield all results returned by the server, properly handling result continuation. """ if 'rawcontinue' in kwargs: raise ValueError("rawcontinue is not supported with query() function, use object's __call__()") if 'continue' not in kwargs: kwargs['continue'] = '' req = kwargs while True: result = self('query', **req) if 'query' in result: yield result['query'] if 'continue' not in result: break # re-send all continue values in the next call req = kwargs.copy() req.update(result['continue']) def queryPages(self, **kwargs): """ Query the server and return all page objects individually. """ incomplete = {} changed = set() for result in self.query(**kwargs): if 'pages' not in result: raise ApiError('Missing pages element in query result', result) finished = incomplete.copy() for pageId, page in result['pages'].items(): if pageId in changed: continue if pageId in incomplete: del finished[pageId] # If server returned it => not finished p = incomplete[pageId] if 'lastrevid' in page and p['lastrevid'] != page['lastrevid']: # someone else modified this page, it must be requested anew separately changed.add(pageId) del incomplete[pageId] continue self._mergePage(p, page) else: p = page incomplete[pageId] = p for pageId, page in finished.items(): if pageId not in changed: yield page for pageId, page in incomplete.items(): yield page if changed: # some pages have been changed between api calls, notify caller raise ApiPagesModifiedError(list(changed)) def _mergePage(self, a, b): """ Recursively merge two page objects """ for k in b: val = b[k] if k in a: if isinstance(val, dict): self._mergePage(a[k], val) elif isinstance(val, list): a[k] = a[k] + val else: a[k] = val else: a[k] = val def token(self, tokenType='csrf'): if tokenType not in self.tokens: self.tokens[tokenType] = next(self.query(meta='tokens', type=tokenType))['tokens'][tokenType + 'token'] return self.tokens[tokenType] def request(self, method, forceSSL=False, headers=None, **request_kw): """Make a low level request to the server""" url = self.url if forceSSL: parts = list(urlparse.urlparse(url)) parts[0] = 'https' url = urlparse.urlunparse(parts) if headers: h = self.headers.copy() h.update(headers) headers = h else: headers = self.headers r = self.session.request(method, url, headers=headers, **request_kw) if not r.ok: raise ApiError('Call failed', r) if self.log.isEnabled(5): dbg = [r.request.url, headers] self.log(5, dbg) return r
def proxy(identifier, in_method, in_headers, data): # find endpoint endpoint = pg.select1( 'endpoints', what=['definition', 'method', 'pass_headers', 'headers', 'url', 'url_dynamic'], where={'id': identifier} ) if not endpoint: return 'endpoint not found, create it at ' \ '<a href="/dashboard">dashboard</a>', 404, {} event = { 'in': { 'time': time.time(), 'method': in_method, 'headers': dict(in_headers), 'body': data, 'replay': True }, 'out': { 'method': endpoint['method'], 'url': endpoint['url'], 'body': None, 'headers': {} }, 'response': { 'code': 0, 'body': '' } } mutated, error = jq(endpoint['definition'], data=data) if not mutated or error: event['out']['error'] = error.decode('utf-8') publish(identifier, event) return 'transmutated into null and aborted', 201, {} h = CaseInsensitiveDict({'Content-Type': 'application/json'}) if endpoint['pass_headers']: h.update(in_headers) h.update(endpoint['headers']) event['out']['headers'] = dict(h) # reformat the mutated data mutatedjson = json.loads(mutated.decode('utf-8')) if h.get('content-type') == 'application/x-www-form-urlencoded': # oops, not json mutated = urlencode(mutatedjson) else: mutated = json.dumps(mutatedjson) event['out']['body'] = mutated if endpoint['url_dynamic']: urlb, error = jq(endpoint['url'], data=data) print('URL', urlb, 'ERROR', error) if not urlb: event['out']['url_error'] = error.decode('utf-8') publish(identifier, event) return 'url building has failed', 200, {} url = urlb.decode('utf-8') event['out']['url'] = url else: url = endpoint['url'] # event['out'] is completed at this point # and we all have everything needed to perform the request if url and is_valid_url(url): # we will only perform a request if there is an URL and it is valid try: s = requests.Session() req = requests.Request(endpoint['method'], url, data=mutated, headers=h).prepare() resp = s.send(req, timeout=4) if not resp.ok: print('FAILED TO POST', resp.text, identifier, mutated) except requests.exceptions.RequestException as e: print(identifier, 'FAILED TO POST', mutated, 'TO URL', url) print(e) publish(identifier, event) return "<request failed: '%s'>" % e, 503, {} event['response']['code'] = resp.status_code event['response']['body'] = resp.text publish(identifier, event) return resp.text, resp.status_code, dict(resp.headers) else: # no valid URL, just testing publish(identifier, event) return 'no URL to send this to', 201, {}
def test_update_retains_unchanged(self): cid = CaseInsensitiveDict({'foo': 'foo', 'bar': 'bar'}) cid.update({'foo': 'newfoo'}) assert cid['bar'] == 'bar'