def sec_request(method, endpoint, *args, **kwargs): res = requests.__getattribute__(method)(URL+endpoint, *args, **kwargs).json() has_status(res, STATUS.NOT_LOGIN) kwargs['headers'] = HEADERS res = requests.__getattribute__(method)(URL+endpoint, *args, **kwargs).json() has_status(res, STATUS.SUCCESS) return res
def call_requests(resolve):
if not options.get("json"):
r = requests.__getattribute__(method)(
url=options["url"],
params=url.params,
headers=options["headers"])
else:
r = requests.__getattribute__(method)(
url=options["url"],
params=url.params,
headers=options["headers"],
data=js.dumps(options["json"]).encode('utf-8'))
resolve(r)
def hub(endpoint, payload, method="get"):
"""
make calls to hubspot
"""
base = "https://api.hubapi.com"
method = method.lower()
url = "{0}{1}".format(base, endpoint)
data = {"hapikey": os.environ.get('HAPIKEY', "demo")}
if method != "post":
data = {**data, **payload}
return requests.__getattribute__(method)(url,params=data)
else:
return requests.__getattribute__(method)(url, params=data, json=payload)
def _do_request(self, method, url, **kwargs):
attempt = kwargs.pop('attempt', 0)
try:
request_method = requests.__getattribute__(method)
if self.token:
response = request_method(url=url,
params={'access_token': self.token},
**kwargs)
else:
response = request_method(url=url, auth=self.auth, **kwargs)
if response.status_code not in (200, 201):
raise BadResponseError(response=response)
return response.json()
except (ConnectionError, ReadTimeout) as e:
if attempt < self.MAX_RETRIES:
attempt += 1
return self._do_request(url=url,
method=method,
attempt=attempt,
**kwargs)
logger.exception(e)
return {}
def request(self, scheme, url, data=None, params=None):
"""
Low-level request interface to mite. Takes a HTTP request scheme (lower
case!), a URL to request (relative), and optionally data to add to the
request. Either returns the JSON body of the request or raises a
HttpException.
"""
url = self.url.format(self.team, url)
headers = {
"X-MiteApikey": self.api_key,
"Content-Type": "application/json",
}
# this is a nice little hack to make the API nicer
# we pass the scheme as string, but have it as attributes in requests
fn = requests.__getattribute__(scheme)
res = fn(url, headers=headers, json=data, params=params)
if res.status_code >= 300:
self._raise_exception(res.status_code)
if not res.content:
return None
try:
return res.json()
except ValueError:
return res.content
def do_request(method, url, retries, **kwargs):
if retries > 0:
try:
return requests.__getattribute__(method)(url, **kwargs)
except:
time.sleep(1)
return do_request(method, url, retries - 1, **kwargs)
return None
def signAndRequestAPILatest(self, fn, base, query_params, post_data={}, file_data=False):
"""
This method signs the request with just the timestamp and
private key, which is what api v1.1 and 2 rely on.
fn - object mapping from mockdb describing POST, etc.
base - Base URL to ping.
query_params - Dictionary of data eventually getting sent over
to Gengo.
post_data - Any extra special post data to get sent over.
"""
# Encoding jobs becomes a bit different than any other method call,
# so we catch them and do a little
# JSON-dumping action. Catching them also allows us to provide some
# sense of portability between the various
# job-posting methods in that they can all safely rely on passing
# dictionaries around. Huzzah!
req_method = requests.__getattribute__(fn["method"].lower())
if fn["method"] == "POST" or fn["method"] == "PUT":
if "job" in post_data:
query_params["data"] = json.dumps(post_data["job"], separators=(",", ":"))
elif "jobs" in post_data:
query_params["data"] = json.dumps(post_data["jobs"], separators=(",", ":"))
elif "comment" in post_data:
query_params["data"] = json.dumps(post_data["comment"], separators=(",", ":"))
elif "action" in post_data:
query_params["data"] = json.dumps(post_data["action"], separators=(",", ":"))
query_hmac = hmac.new(self.private_key, Gengo.compatibletext(query_params["ts"]), sha1)
query_params["api_sig"] = query_hmac.hexdigest()
if self.debug is True:
print(query_params)
if not file_data:
return req_method(base, headers=self.headers, data=query_params)
else:
return req_method(base, headers=self.headers, files=file_data, data=query_params)
else:
query_string = urlencode(sorted(query_params.items(), key=itemgetter(0)))
if self.private_key is not None:
query_hmac = hmac.new(self.private_key, Gengo.compatibletext(query_params["ts"]), sha1)
query_params["api_sig"] = query_hmac.hexdigest()
query_string = urlencode(query_params)
if self.debug is True:
print(base + "?{0}".format(query_string))
return req_method(
base + "?{0}".format(query_string),
headers=self.headers,
# Don't know why but requests is trying to verify
# SSL here ...
verify=False,
)
def send_request(self, http_method, path, params=None, headers=None):
url = "{endpoint}/{path}?access_token={access_token}".format(
endpoint=self.api_endpoint,
path=path,
access_token=self.access_token)
response = requests.__getattribute__(http_method)(url,
params=params,
headers=headers)
return Response(response)
def _request(self,
request_method,
endpoint='',
url='',
data=None,
params=None,
use_api_key=False,
omit_api_version=False):
"""Perform a http request via the specified method to an API endpoint.
:param string request_method: Request method.
:param string endpoint: Target endpoint. (Optional).
:param string url: Override the endpoint and provide the full url (eg for pagination). (Optional).
:param dict params: Provide parameters to pass to the request. (Optional).
:param dict data: Data to pass to the post. (Optional).
:return: Response
:rtype: ``Response``
"""
if not data:
data = {}
if not params:
params = {}
if endpoint and omit_api_version and not url:
url = '%s/%s' % (self.base_url, endpoint)
if endpoint and not url:
url = '%s/%s/%s' % (self.base_url, settings.API_VERSION, endpoint)
if use_api_key:
headers = {
'Authorization': self.auth.get_api_key(),
'User-Agent': self.user_agent,
}
else:
headers = {
'Authorization': self.auth.get_authorization(),
'User-Agent': self.user_agent,
}
response = requests.__getattribute__(request_method)(
url=url,
hooks=settings.REQUEST_HOOK,
headers=headers,
json=data,
params=params)
if ((response.status_code != 200) and (response.status_code != 202)):
try:
response.raise_for_status()
except requests.exceptions.HTTPError as e:
raise PyCronofyRequestError(
request=e.request,
response=e.response,
)
return response
def make_api_request(method, url, params=None, payload=None, creds=None, access_token=None, *args, **kwargs):
'''Make an API call to twitter
Parameters:
-----------
method : str
'GET' or 'POST'
url : str
twitter url
params : dict
url parametres
payload :
POST request body
creds : dict
twitter credentials for signing
access_token : str
twitter access token
kwargs:
add_sign_to_params: (True, False)
include signature to the url parametres
'''
# Sign the request
sign = sign_request(method, twitter_rq_token, params, **creds, **payload)
sign = hmac_sha1(sign, c_secret, access_token or '')
# Add data
if 'add_sign_to_params' in kwargs and kwargs['add_sign_to_params'] == True:
params['oauth_signature'] = sign
if 'oauth_callback' in creds.keys():
creds['oauth_callback'] = quote(creds['oauth_callback'])
# sort parameters
params = {key: val for key, val in sorted(params.items())}
# sort headers
header_parts = list(creds.items())
header_parts.append(("oauth_signature", sign))
header_parts.sort(key=lambda x: x[0])
header = 'OAuth ' + ', '.join([f'{key}="{val}"' for key, val in header_parts])
headers = {
'Authorization': header,
'Content-type': 'application/x-www-form-urlencoded'
}
# sort payload
payload_parts = list(payload.items())
payload_parts.sort(key=lambda x: x[0])
bin_data = b'&'.join([(f'{key}={val}').encode() for key, val in payload_parts])
# Make request
http_method = rq.__getattribute__(method.lower())
resp = http_method(url, params=params, headers=headers, data=bin_data)
# pprint(resp)
# if not resp.ok:
# raise ValueError(resp.text)
txt = resp.text
# pprint(txt)
return txt
def targetprocess(resource, data={}, include='[Id]', method='get'):
headers = {
'Content-Type': 'application/json;charset=UTF-8',
}
params = {
'access_token': tp_cfg['access_token'],
'format': 'json',
'include': include
}
return requests.__getattribute__(method)(
tp_cfg['url'] + '/' + tp_cfg['api_uri'] + '/' + resource,
headers=headers,
params=params,
json=data).json()
def execute_request(
url: str,
method: str = 'get',
headers: Dict = {},
params: Dict = {},
json: Dict = {},
recorder: Optional[vcr.VCR] = recorder,
cassette_path: str = '',
logger: Optional[MyLogger] = None
) -> Tuple[requests.models.Response, bool]:
"""
Execute a HTTP request and return response defined by the `requests` package.
"""
if logger:
logger.logger.info('execute_request {} {}'.format(
recorder, cassette_path))
is_cached = False
method_func = requests.__getattribute__(method.lower())
if cassette_path and recorder:
from vcr.cassette import Cassette
from vcr.request import Request
logger.logger.info('imported vcr')
req = Request(method.upper(), url, 'irrelevant body?',
{'some': 'header'})
logger.logger.info('req ' + str(req))
c_path = os.path.join(recorder.cassette_library_dir, cassette_path)
logger.logger.info(c_path)
logger.logger.info(Cassette(None).load(path=c_path))
if req in Cassette(None).load(path=c_path):
is_cached = True
with recorder.use_cassette(c_path):
logger.logger.info('running...')
if json:
resp = method_func(url,
headers=headers,
params=params,
json=json)
else:
resp = method_func(url, headers=headers, params=params)
logger.logger.info('got it...')
else:
if json:
resp = method_func(url, headers=headers, params=params, json=json)
else:
resp = method_func(url, headers=headers, params=params)
# FIXME: requests.post('http://httpbin.org/post', json={"key": "value"})
return resp, is_cached
def signAndRequestAPILatest(self, fn, base, query_params, post_data = {},
file_data = False):
"""
Request signatures between API v1 and later versions of the API differ greatly in
how they're done, so they're kept in separate methods for now.
This method signs the request with just the timestamp and private key, which is what
api v1.1 relies on.
fn - object mapping from mockdb describing POST, etc.
base - Base URL to ping.
query_params - Dictionary of data eventually getting sent over to Gengo.
post_data - Any extra special post data to get sent over.
"""
# Encoding jobs becomes a bit different than any other method call, so we catch them and do a little
# JSON-dumping action. Catching them also allows us to provide some sense of portability between the various
# job-posting methods in that they can all safely rely on passing dictionaries around. Huzzah!
req_method = requests.__getattribute__( lower( fn['method'] ) )
if fn['method'] == 'POST' or fn['method'] == 'PUT':
if 'job' in post_data:
query_params['data'] = json.dumps(post_data['job'], separators = (',', ':'))
elif 'jobs' in post_data:
query_params['data'] = json.dumps(post_data['jobs'], separators = (',', ':'))
elif 'comment' in post_data:
query_params['data'] = json.dumps(post_data['comment'], separators = (',', ':'))
elif 'action' in post_data:
query_params['data'] = json.dumps(post_data['action'], separators = (',', ':'))
query_hmac = hmac.new(self.private_key, query_params['ts'], sha1)
query_params['api_sig'] = query_hmac.hexdigest()
if self.debug is True:
print query_data
if not file_data:
return req_method( base, headers = self.headers, data = query_params )
else:
return req_method( base, headers = self.headers, files = file_data, data = query_params )
else:
query_string = urlencode(sorted(query_params.items(), key = itemgetter(0)))
if self.private_key is not None:
query_hmac = hmac.new(self.private_key, query_params['ts'], sha1)
query_params['api_sig'] = query_hmac.hexdigest()
query_string = urlencode(query_params)
if self.debug is True:
print base + '?%s' % query_string
return req_method( base + '?%s' % query_string, headers = self.headers )
def NetTester(secret=None, claims=None):
headers = {}
if secret and claims:
token = jwt.encode(claims, secret, algorithm='HS256').decode("utf-8")
headers["Authorization"] = "Bearer " + token
class NT:
pass
x = NT()
for method in ["get", "post", "patch", "delete"]:
x.__setattr__(
method,
functools.partial(requests.__getattribute__(method),
headers=headers))
return x
def _do_request(self, method, url, **kwargs):
attempt = kwargs.pop('attempt', 0)
try:
request_method = requests.__getattribute__(method)
response = request_method(url=url, auth=self.auth, **kwargs)
if response.status_code not in (200, 201):
raise BadResponseError(response=response)
return response.json()
except (ConnectionError, ReadTimeout) as e:
if attempt < self.MAX_RETRIES:
attempt += 1
return self._do_request(url=url, method=method, attempt=attempt, **kwargs)
logger.exception(e)
return {}
def query(url, method, headers, data, params, stage):
if method == 'get' or method == 'post':
buffy = Cache(AWS_DEFAULT_REGION, stage + TABLE_NAME_SUF)
payload = dict(data.items() + params.items())
doc = buffy.get_doc(url, payload)
if doc is None:
doc = requests.__getattribute__(method)(url,
headers=headers,
data=data,
params=params).text
buffy.set_doc(url, payload, doc)
return doc
else:
return doc
if response.status_code != requests.codes.ok:
response.raise_for_status()
return response
def http_replay(method):
"""Replay an incoming request of type <method> against the parameter list of endpoints"""
endpoint_list = request.args.get("host", "")
endpoint_list = endpoint_list.split(";")
timeout = request.args.get("timeout", None) or 5
#_method = requests.__getattribute__(method.lower())
if not endpoint_list:
return jsonify(
status=500,
message=
"Expected parameters in the form of ?host=http://host/path;host=http://host2/path"
)
else:
# create an async model when we actually need it, not sooner
responses = []
for ep in endpoint_list:
try:
_r = requests.__getattribute__(method.lower())(ep,
timeout=timeout)
status_code = _r.status_code
except r.exceptions.Timeout:
status_code = 408 # request timeout
except r.exceptions.RequestException:
status_code = 520
responses.append(status_code)
_r_status = set(responses)
if len(_r_status) == 1 and _r_status.pop == 200:
# maybe 203 "non-authoritative" is better?
status = 200
else:
# 520 "unknown error" fine?
status = 520
return jsonify(status=status)
def __getattr__(self, name):
attr = r.__getattribute__(name)
if callable(attr):
def func(*args, **kwargs):
log.debug(args)
log.debug(kwargs)
if "json" in kwargs:
log.debug("\n" + json.dumps(kwargs["json"], indent=4))
response = attr(*args, **kwargs)
if isinstance(response, r.Response):
log.debug(response)
if response.headers.get(
"content-type") == "application/json":
log.debug("\n" +
json.dumps(response.json(), indent=4))
else:
log.debug(response.text)
return response
return func
return attr
def http_replay(method):
"""Replay an incoming request of type <method> against the parameter list of endpoints"""
endpoint_list = request.args.get("host", "")
endpoint_list = endpoint_list.split(";")
timeout = request.args.get("timeout", None) or 5
#_method = requests.__getattribute__(method.lower())
if not endpoint_list:
return jsonify(status=500,
message="Expected parameters in the form of ?host=http://host/path;host=http://host2/path")
else:
# create an async model when we actually need it, not sooner
responses = []
for ep in endpoint_list:
try:
_r = requests.__getattribute__(method.lower())(ep, timeout=timeout)
status_code = _r.status_code
except r.exceptions.Timeout:
status_code = 408 # request timeout
except r.exceptions.RequestException:
status_code = 520
responses.append(status_code)
_r_status = set(responses)
if len(_r_status) == 1 and _r_status.pop == 200:
# maybe 203 "non-authoritative" is better?
status = 200
else:
# 520 "unknown error" fine?
status = 520
return jsonify(status=status)
def _peticionar(metodo, url, datos=None, respuesta_tipo='json'):
"""La estructura base para todas las demás peticiones"""
verbo = requests.__getattribute__(metodo)
log.info('Peticionando: {} {}'.format(metodo.upper(), url))
peticion = verbo(url) if datos is None else verbo(url, json=datos)
respuesta = None
if respuesta_tipo == 'json':
respuesta = peticion.json()
else:
respuesta = [''] if peticion.text == '' else [
x for x in peticion.text.split('\n') if len(x) > 0
]
if peticion.status_code == 200:
return respuesta
else:
error = peticion.json()
mensaje = error['error']['type'] + ': ' + error['error']['reason']
log.error("Peticionando: Error {}".format(mensaje))
return {'acknowledged': False, 'error': mensaje}
def __getattr__(self, name):
if name in ('get', 'post', 'put', 'delete'):
return partial(self.handle_request, name)
return original_requests.__getattribute__(name)
def send_request(path, method='GET', headers=HEADERS, cookies=COOKIES): return requests.__getattribute__(method.lower())(path, headers=headers, cookies=cookies)
def func_headers(x):
nonlocal conf, opts, options, cb
if conf.get("id_token"):
x["Authorization"] = conf["id_token"]
Util.add_headers(x, opts, conf)
options = {"url": str(url.pathname), "headers": x}
Util.add_body(options, opts, conf)
if not options.get("json"):
r = requests.__getattribute__(method)(
url=options["url"],
params=url.params,
headers=options["headers"])
else:
r = requests.__getattribute__(method)(
url=options["url"],
params=url.params,
headers=options["headers"],
data=js.dumps(options["json"]).encode('utf-8'))
json = None
err = None
try:
json = r.json()
except:
json = r
try:
r.raise_for_status()
except requests.exceptions.HTTPError as errh:
print("Http Error:", errh)
err = errh
except requests.exceptions.ConnectionError as errc:
print("Error Connecting:", errc)
err = errc
except requests.exceptions.Timeout as errt:
print("Timeout Error:", errt)
err = errt
except requests.exceptions.RequestException as errr:
print("Request Error:", errr)
err = errr
if r.status_code != 200:
if r.status_code == 401 and not conf.get("retry"):
if conf.get("refresh_token") and conf.get("username"):
def func_token(tokens):
nonlocal conf
if tokens:
conf["retry"] = True
conf["id_token"] = tokens["id_token"]
conf["refresh_token"] = tokens["refresh_token"]
conf["token_source"] = "cognito"
Request.request(url, opts, conf, cb)
else:
conf["refresh_token"] = None
conf["id_token"] = None
cb(json, None)
Cognito.refresh_token(conf, func_token)
else:
conf["refresh_token"] = None
conf["id_token"] = None
conf["retry"] = True
Request.request(url, opts, conf, cb)
else:
cb(json, None)
else:
cb(err, json)
def std_request(method, endpoint, *args, **kwargs): kwargs['headers'] = HEADERS return requests.__getattribute__(method)(URL+endpoint, *args, **kwargs).json()
def http_patch_post(self,
mode: ['patch', 'post'],
endpoint: str,
jsonbody: Optional[Dict] = None,
urlparams: Optional[Dict] = None,
ids: Optional[Dict] = None) -> requests.Response:
"""Perform a HTTP Patch request."""
cls = self.__class__
# Select correct requests mode
if mode in ['patch', 'post']:
# pylint: disable=no-member
req = requests.__getattribute__(mode)
else:
raise NotImplementedError(
'HTTP mode "{}" not implemented'.format(mode))
# Validate endpoint and navigation
validate_urlpath(endpoint)
# Prepare dictionary for URL parameter
params = prepare_params_dict(urlparams)
# Prepare URI
uri = self.prepare_uri(endpoint, ids)
# Prepare JSON body
if jsonbody is None:
jsonbody = {}
# Prepare additional headers
headers = {'Accept': 'application/json'}
# If no CSRF token set, request one from base path of ODATA service
if self._csrftoken is None or self._cookies is None:
self.http_get('', fetch_csrf=True)
headers['X-CSRF-Token'] = self._csrftoken
# Basic authorization
if self._config.authorization == ODataConfig.AUTH_BASIC:
try:
resp = req(uri,
json=jsonbody,
params=params,
headers=headers,
cookies=self._cookies,
timeout=cls.TIMEOUT,
auth=(self._config.user, self._config.password))
except requests.ConnectionError as err:
_LOGGER.error(
'Connection error on OData %s request to URI %s: %s',
mode.upper(), uri, err)
self.odata_counter.labels( # pylint: disable=no-member
endpoint=endpoint,
result=err.__class__.__name__).inc()
raise ConnectionError(err)
except requests.Timeout as err:
_LOGGER.error('Timeout on OData %s request to URI %s: %s',
mode.upper(), uri, err)
self.odata_counter.labels( # pylint: disable=no-member
endpoint=endpoint,
result=err.__class__.__name__).inc()
raise TimeoutError(err)
except requests.RequestException as err:
_LOGGER.error('Error on OData %s request to URI %s: %s',
mode.upper(), uri, err)
self.odata_counter.labels( # pylint: disable=no-member
endpoint=endpoint,
result=err.__class__.__name__).inc()
raise IOError(err)
else:
# Invalid X-CSRF-Token returns 403 error.
# Refresh token and try again
if resp.status_code == 403:
# Request new token_csrftoken_csrftoken
self.http_get('', fetch_csrf=True)
headers['X-CSRF-Token'] = self._csrftoken
try:
resp = req(uri,
json=jsonbody,
params=params,
headers=headers,
cookies=self._cookies,
auth=(self._config.user,
self._config.password))
except requests.ConnectionError as err:
_LOGGER.error(
'Connection error on OData %s request: %s',
mode.upper(), err)
raise ConnectionError(err)
except requests.Timeout as err:
_LOGGER.error('Timeout on OData %s request: %s',
mode.upper(), err)
raise TimeoutError(err)
except requests.RequestException as err:
_LOGGER.error('Error on OData %s request: %s',
mode.upper(), err)
raise IOError(err)
return resp
else:
# Other authorization types not implemented yet
raise NotImplementedError
def signAndRequestAPILatest(self,
fn,
base,
query_params,
post_data={},
file_data=False):
"""
Request signatures between API v1 and later versions of the API differ greatly in
how they're done, so they're kept in separate methods for now.
This method signs the request with just the timestamp and private key, which is what
api v1.1 relies on.
fn - object mapping from mockdb describing POST, etc.
base - Base URL to ping.
query_params - Dictionary of data eventually getting sent over to Gengo.
post_data - Any extra special post data to get sent over.
"""
# Encoding jobs becomes a bit different than any other method call, so we catch them and do a little
# JSON-dumping action. Catching them also allows us to provide some sense of portability between the various
# job-posting methods in that they can all safely rely on passing dictionaries around. Huzzah!
req_method = requests.__getattribute__(lower(fn['method']))
if fn['method'] == 'POST' or fn['method'] == 'PUT':
if 'job' in post_data:
query_params['data'] = json.dumps(post_data['job'],
separators=(',', ':'))
elif 'jobs' in post_data:
query_params['data'] = json.dumps(post_data['jobs'],
separators=(',', ':'))
elif 'comment' in post_data:
query_params['data'] = json.dumps(post_data['comment'],
separators=(',', ':'))
elif 'action' in post_data:
query_params['data'] = json.dumps(post_data['action'],
separators=(',', ':'))
query_hmac = hmac.new(self.private_key, query_params['ts'], sha1)
query_params['api_sig'] = query_hmac.hexdigest()
if self.debug is True:
print query_data
if not file_data:
return req_method(base,
headers=self.headers,
data=query_params)
else:
return req_method(base,
headers=self.headers,
files=file_data,
data=query_params)
else:
query_string = urlencode(
sorted(query_params.items(), key=itemgetter(0)))
if self.private_key is not None:
query_hmac = hmac.new(self.private_key, query_params['ts'],
sha1)
query_params['api_sig'] = query_hmac.hexdigest()
query_string = urlencode(query_params)
if self.debug is True:
print base + '?%s' % query_string
return req_method(base + '?%s' % query_string,
headers=self.headers)
def method_with_etag_inner(
self, url: str, etag: str, data: Union[List, Dict], headers: Dict, method: str
) -> requests.Response:
"""method using etag"""
headers["If-Match"] = etag
return requests.__getattribute__(method)(url, data=data, headers=headers)