def request(method, url, **kwargs):
"""
Pass auth=HTTPKerberosAuth() kwarg
"""
auth = kwargs.get('auth')
headers = kwargs.get('headers', {})
# headers = headers.copy() # ? We do modify the dict in place here...
if isinstance(auth, TreqKerberosAuth):
del kwargs['auth']
if auth.force_preemptive:
# Save a round-trip and set the Negotiate header on the first req.
headers['Authorization'] = yield negotiate_header(url)
response = yield treq.request(method=method,
url=url,
headers=headers,
**kwargs)
# Retry if we got a 401 / Negotiate response.
if response.code == 401 and isinstance(auth, TreqKerberosAuth):
auth_mechs = response.headers.getRawHeaders('WWW-Authenticate')
if 'Negotiate' in auth_mechs:
headers['Authorization'] = yield negotiate_header(url)
response = yield treq.request(method=method,
url=url,
headers=headers,
**kwargs)
defer.returnValue(response)
def create(self, path='/'):
self.data['address'] = str(self.data['address'])
self.data['address_obj'] = urlparse(self.data['address'])
# Build the necessary request headers for contacting the HTTP resource
options = self.data['options']
if 'auth_method' in options:
if options['auth_method'].lower() == 'basic' and options['auth_username'] and options['auth_password']:
self._treq_options['auth'] = (options['auth_username'], options['auth_password'])
if 'timeout' in options:
try:
self._treq_options['timeout'] = int(options['timeout'])
except:
pass
if 'user-agent' in options:
self._treq_options['headers'] = {'User-Agent': [str(options['user-agent'])]}
# Fetch the first page and determine which template parser is needed
resp = yield request(method='GET', url=self._buildURL(path), **self._treq_options)
if resp.code != 200:
raise CrawlException('Response code was not 200: %s' % str(resp.code))
self._resp_headers = dict((key.lower(), value[0].lower()) for (key, value) in resp.headers._rawHeaders.items())
response_body = yield text_content(resp)
self._indexParser = HttpParse().get_parser(response_headers=self._resp_headers,
response_body=response_body)
if self._indexParser:
defer.returnValue(True)
defer.returnValue(False)
def _request(self, method, path, headers=None, **kwargs):
url = '%s/%s' % (self.endpoint, path)
headers = headers if headers else {}
if self.token:
headers['Authorization'] = 'Bearer %s' % (self.token, )
return treq.request(method, url, headers=headers,
**kwargs).addCallback(treq.json_content)
def _internal_call(self, method, url, payload, params):
args = dict(params=params)
if not url.startswith('http'):
url = self.prefix + url
headers = self._auth_headers()
headers['Content-Type'] = ['application/json']
if payload:
args["data"] = json.dumps(payload)
# TODO: treq can't handle unicode urls :(
url = str(url)
print("requesting %s" % url)
r_d = treq.request(method, url, headers=Headers(headers), **args)
r = yield r_d
response_json = yield r.json()
if self.trace: # pragma: no cover
print()
print(method, url)
if payload:
print("DATA", json.dumps(payload))
if r.code != 200:
raise SpotifyException(
r.code, -1,
u'%s:\n %s' % (url, response_json['error']['message']))
if self.trace: # pragma: no cover
print('RESP', response_json)
print()
defer.returnValue(response_json)
def a_case(fname):
env = treq.load_py(os.path.splitext(fname)[0] + ".py")
expect = env['request']
cfg = env['cfg']
req = treq.request(fname, expect)
for case in req.gen_cases(cfg):
case[0](*case[1:])
def _request(self, http_method, url, payload=None):
def _raise_error(txt):
logger.info("ERROR: %s", txt)
raise TwitterApiError(txt)
def _print_response(response):
logger.info("RESPONSE: ", response)
return response
def _read_response(response):
if response.code != 200:
return response.text().addCallback(_raise_error)
return treq.content().addCallback(json.loads)
headers = {}
body = None
client = self._oauth_client()
if payload:
body = urlencode(payload)
headers = {'Content-Type': 'application/x-www-form-urlencoded'}
xx, headers, body = client.sign(url, http_method=http_method,
headers=headers, body=body)
dfr = treq.request(http_method, url, headers=headers,
data=body)
return dfr.addCallback(_read_response)
def request(self, method, uri, data=b'', headers=None):
# A small wrapper around self.agent.request() so we can easily attach
# counters to it
outgoing_requests_counter.labels(method).inc()
# log request but strip `access_token` (AS requests for example include this)
logger.info("Sending request %s %s", method, redact_uri(uri))
try:
request_deferred = treq.request(method,
uri,
agent=self.agent,
data=data,
headers=headers)
request_deferred = timeout_deferred(
request_deferred,
60,
self.hs.get_reactor(),
cancelled_to_request_timed_out_error,
)
response = yield make_deferred_yieldable(request_deferred)
incoming_responses_counter.labels(method, response.code).inc()
logger.info("Received response to %s %s: %s", method,
redact_uri(uri), response.code)
defer.returnValue(response)
except Exception as e:
incoming_responses_counter.labels(method, "ERR").inc()
logger.info("Error sending request to %s %s: %s %s", method,
redact_uri(uri),
type(e).__name__, e.args[0])
raise
def _fetch_resource(self,
method: str,
suffix: str,
params: Dict[str, Any] = None) -> Deferred:
"""Post request data to API."""
if treq is None:
raise ImportError(
"treq is required for the IceCubedAsyncClient. Try installing the with `pip install ice3x[async]`"
)
if params is None:
params = {}
headers = {
"User-Agent": "Mozilla/4.0 (compatible; Ice3x Async Python client)"
}
if method == "post":
assert (
self.api_key is not None
), f"An API key is required in order to access the {suffix} resource."
headers["Key"] = self.api_key
headers["Sign"] = self.sign(params)
kwargs = {"params": params, "headers": headers}
url = f"{self.BASE_URI}{suffix}"
resp = yield treq.request(method, url, **kwargs)
data = yield resp.json()
return data
def test_request_invalid_param(self):
"""
`treq.request()` warns that it ignores unknown keyword arguments, but
this is deprecated.
This test verifies that stacklevel is set appropriately when issuing
the warning.
"""
self.failureResultOf(
treq.request(
"GET",
"https://foo.bar",
invalid=True,
pool=SyntacticAbominationHTTPConnectionPool(),
)
)
[w] = self.flushWarnings([self.test_request_invalid_param])
self.assertEqual(DeprecationWarning, w["category"])
self.assertEqual(
(
"Got unexpected keyword argument: 'invalid'."
" treq will ignore this argument,"
" but will raise TypeError in the next treq release."
),
w["message"],
)
def GET(self,
urlpath,
followRedirect=False,
return_response=False,
method="GET",
clientnum=0,
**kwargs):
# if return_response=True, this fires with (data, statuscode,
# respheaders) instead of just data.
assert not isinstance(urlpath, unicode)
url = self.client_baseurls[clientnum] + urlpath
response = yield treq.request(method,
url,
persistent=False,
allow_redirects=followRedirect,
**kwargs)
data = yield response.content()
if return_response:
# we emulate the old HTTPClientGetFactory-based response, which
# wanted a tuple of (bytestring of data, bytestring of response
# code like "200" or "404", and a
# twisted.web.http_headers.Headers instance). Fortunately treq's
# response.headers has one.
defer.returnValue((data, str(response.code), response.headers))
if 400 <= response.code < 600:
raise Error(response.code, response=data)
defer.returnValue(data)
def request(self, method, url_suffix, parser=None, **kw):
"""
Start the application, make an HTTP request and then shutdown
the application.
:param str url_suffix:
A path to make the request to.
:param str parser:
Response parser to use. Valid values are ``'bytes'``, ``'json'``,
``'json_lines'`` or ``None``. ``None`` indicates that the raw
response object should be returned. Otherwise the parsed data is
returned.
Other parameters are the same as for :func:`treq.request`.
"""
server = self.reactor.listenTCP(0, self.app, interface="127.0.0.1")
host = server.getHost()
# prefix the URL with the test server host and port
url = ('http://127.0.0.1:%d' % host.port) + url_suffix
# close the HTTP connection straight away so that the test
# reactor is clean when we leave this function
kw['persistent'] = False
response = yield treq.request(method, url, reactor=self.reactor, **kw)
if parser is not None:
parser_method = getattr(self, '_parse_' + parser)
response = yield parser_method(response)
yield server.stopListening()
server.loseConnection()
returnValue(response)
def _internal_call(self, method, url, payload, params):
args = dict(params=params)
if not url.startswith('http'):
url = self.prefix + url
headers = self._auth_headers()
headers['Content-Type'] = ['application/json']
if payload:
args["data"] = json.dumps(payload)
# TODO: treq can't handle unicode urls :(
url = str(url)
print("requesting %s" % url)
r_d = treq.request(method, url, headers=Headers(headers), **args)
r = yield r_d
response_json = yield r.json()
if self.trace: # pragma: no cover
print()
print(method, url)
if payload:
print("DATA", json.dumps(payload))
if r.code != 200:
raise SpotifyException(r.code,
-1, u'%s:\n %s' % (url, response_json['error']['message']))
if self.trace: # pragma: no cover
print('RESP', response_json)
print()
defer.returnValue(response_json)
def process_queue(self):
while True:
thing = yield self.queue_object.get()
if thing is None:
break
ch, method, properties, body = thing
if body:
path = body
print "PURGE %s %s" % (ua_map, path)
try:
response = yield treq.request(
"PURGE", "http://127.0.0.1" + path,
cookies={"foo": "bar"},
headers={"Host": "actual.host.com"},
timeout=10
)
except (ConnectError, DNSLookupError, CancelledError, ResponseFailed):
# Maybe better to do a blank except?
print "ERROR %s %s" (ua_map, path)
else:
print "RESULT %s %s" % (ua_map, path)
content = yield response.content()
print content
yield ch.basic_ack(delivery_tag=method.delivery_tag)
def with_signature(self,
meth,
object_path,
params=None,
data=None,
headers=None):
meth = meth.upper()
subresources = []
for key, value in params.items():
if key in self._subresource_key_set:
subresources.append((key, value))
subresources.sort(key=lambda p: p[0])
_subresources = '&'.join(
("=".join([k, v]) if v else k for k, v in subresources))
resources_string = '&'.join(
[f'/{self._bucket}/{object_path}', _subresources])
headers = headers or {}
headers['date'] = formatdate(None, usegmt=True)
headers_string = '\n'.join([
':'.join([k, v])
for k, v in sorted([(k.lower(), v) for k, v in headers.items()
if k.lower().startswith('x-oss-')],
key=lambda h: h[0])
])
if headers_string:
headers_string += '\n'
content_md5 = headers.get('content-md5', '')
content_type = headers.get('content-type', '')
_string_to_sign = '\n'.join([
meth, content_md5, content_type, headers['date'],
''.join([headers_string, resources_string])
])
h = hmac.new(self._sk.encode(), _string_to_sign.encode(), hashlib.sha1)
_signature = base64.b64encode(h.digest()).decode()
headers["Authorization"] = f"OSS {self._ak}:{_signature}"
params['OSSAccessKeyId'] = self._ak
params['Signature'] = _signature
url = '?'.join([
self._endpoint(object_path), '&'.join([
'='.join([quote(k, ''), quote(v, '')]) if v else quote(k, '')
for k, v in params.items()
])
])
response = yield treq.request(meth,
url,
data=data,
params=params,
headers=headers)
return response
def perform_effect(self, dispatcher):
import treq
headers = self.headers.copy() if self.headers is not None else {}
if 'user-agent' not in headers:
headers['user-agent'] = ['Effect example']
return treq.request(self.method.lower(), self.url, headers=headers,
data=self.data).addCallback(treq.content)
def _fetch_resource(self,
method: str,
suffix: str,
params: Dict = {}) -> Deferred:
"""Helper function to make API requests
Args:
method: The http verb i.e. get, post, put, delete
suffix: The uri suffix
params: A dict of query params
Returns:
A twisted deferred
"""
url = f"{self.BASE_URI}{suffix}"
headers = {"Content-Type": "application/x-www-form-urlencoded"}
auth = (self.api_key, self.secret)
resp = yield treq.request(method,
url,
params=params,
headers=headers,
auth=auth)
data = yield resp.json()
return data
def _call(self, method, call, **kwargs):
url = self.endpoint + call
headers = {
'content-type': 'application/json',
'accept': 'application/json'
}
if self.session_id is not None:
headers['coinsetter-client-session-id'] = self.session_id.encode(
'utf-8')
new_kwargs = {'headers': headers}
new_kwargs.update(kwargs)
if 'data' in kwargs:
new_kwargs['data'] = json.dumps(kwargs['data'])
try:
result = yield treq.request(method, url.encode('utf-8'),
**new_kwargs)
except (IOError, ConnectError) as e:
log.err(e)
self.emit("disconnect", self)
raise e
content = yield result.content()
if result.code != 200:
self.emit("disconnect", self)
raise Exception("Error code %d received" % result.code)
parsed = json.loads(content, parse_float=Decimal)
returnValue(parsed)
def _make_request(self, method, path, data=None, *args, **kwargs):
"""Parse and create the request object."""
data = json.dumps(data) if data is not None else None
headers = self._all_extra_headers()
new_headers = kwargs.pop("headers", {})
headers.update(new_headers)
if self.auth:
kwargs['auth'] = self.auth
if kwargs.get('params', {}):
params = kwargs.get('params', {})
for key, value in params.items():
value = utf8(value) if isinstance(value, basestring) else value
params[key] = value
if params:
kwargs['params'] = params
return treq.request(
method,
path,
headers=headers,
data=data,
**kwargs
).addCallbacks(
self._handle_response,
log.err
)
def _call(self, method, call, **kwargs):
url = self.endpoint + call
headers = {'content-type': 'application/json',
'accept': 'application/json'}
if self.session_id is not None:
headers['coinsetter-client-session-id'] = self.session_id.encode('utf-8')
new_kwargs = {'headers': headers}
new_kwargs.update(kwargs)
if 'data' in kwargs:
new_kwargs['data'] = json.dumps(kwargs['data'])
try:
result = yield treq.request(method, url.encode('utf-8'), **new_kwargs)
except (IOError, ConnectError) as e:
log.err(e)
self.emit("disconnect", self)
raise e
content = yield result.content()
if result.code != 200:
self.emit("disconnect", self)
raise Exception("Error code %d received" % result.code)
parsed = json.loads(content, parse_float=Decimal)
returnValue(parsed)
def _query_mailchimp(data_center, method_section, method_name, payload):
def _read_response(response):
logger.info("Received Data: %s", response)
if response.code != 200:
logger.info("Received Data with error")
raise MailChimpApiError("{}".format(response.phrase))
return treq.text_content(response).addCallback(json.loads)
def _check_for_error(response):
logger.info("Received Data: %s", response)
if "error" in response:
raise MailChimpApiError("{}:{}".format(response["name"],
response["error"]))
return response
logger.info("STARTING QUERY MAICHIMP WITH PAYLOAD {}".format(payload))
dfr = treq.request("POST",
MAILCHIMP_BASE_URL.format(dc=data_center,
section=method_section,
name=method_name).encode("utf-8"),
data=json.dumps(payload),
headers={"Content-Type": "application/json"})
return dfr.addCallback(_read_response).addCallback(_check_for_error)
def download_file(self, url: str, destination_filename: str):
"""
Downloads a file from the given url and saves it to the requested destination.
Returns a deferred!
:param url:
:param destination_filename:
:return:
"""
try:
treq_response = yield treq.request("GET", url)
except ConnectionRefusedError as e:
raise YomboWarning(f"Connection was refused to '{url}': {e}")
except ConnectError as e:
raise YomboWarning(f"Error connecting to '{url}': {e}")
except Exception as e:
logger.info("Requests download_file error: {error}", error=e)
logger.error(
"---------------==(Traceback)==--------------------------")
logger.error("{url}", url=url)
logger.error("{trace}", trace=traceback.format_exc())
logger.error(
"--------------------------------------------------------")
logger.warn(
"An exception of type {etype} occurred in yombo.lib.yomboapi:import_component. Message: {msg}",
etype=type(e),
msg=e)
logger.error(
"--------------------------------------------------------")
raise e
raw_content = yield treq.content(treq_response)
yield self._Files.save(destination_filename, raw_content)
def a_case(fname):
env = treq.load_py(os.path.splitext(fname)[0] + ".py")
expect = env['request']
cfg = env['cfg']
req = treq.request(fname, expect)
for case in req.gen_cases(cfg):
case[0](*case[1:])
def _make_request(token, method_name, method='get', params=None, data=None, files=None, timeout=10, **kwargs): request_url = API_URL + 'bot' + token + '/' + method_name params = _convert_utf8(params) resp = yield treq.request(method, request_url, params=params, data=data, files=files, timeout=timeout, **kwargs) result_json = yield _check_response(resp, method_name) returnValue(result_json)
def do_http(method, url, **kwargs):
response = yield treq.request(method, url, persistent=False, **kwargs)
body = yield treq.content(response)
# TODO: replace this with response.fail_for_status when
# https://github.com/twisted/treq/pull/159 has landed
if 400 <= response.code < 600:
raise Error(response.code, response=body)
defer.returnValue(body)
def request(self, method, path, data=None):
return treq.request(
method, 'http://localhost:%s%s' % (
self.listener_port,
path
),
data=data,
pool=self.pool)
def test_request_invalid_param(self):
"""
`treq.request()` warns that it ignores unknown keyword arguments, but
this is deprecated.
This test verifies that stacklevel is set appropriately when issuing
the warning.
"""
with self.assertRaises(TypeError) as c:
treq.request(
"GET",
"https://foo.bar",
invalid=True,
pool=SyntacticAbominationHTTPConnectionPool(),
)
self.assertIn("invalid", str(c.exception))
def do_http(method, url, **kwargs):
response = yield treq.request(method, url, persistent=False, **kwargs)
body = yield treq.content(response)
# TODO: replace this with response.fail_for_status when
# https://github.com/twisted/treq/pull/159 has landed
if 400 <= response.code < 600:
raise Error(response.code, response=body)
defer.returnValue(body)
def request(self, method, url, **kwargs):
"""
Make an HTTP request using treq. This basically uses treq, but parses the response
and attempts to decode the data if it's json or msgpack.
This must be called with "yield".
It returns a dictionary with these keys:
* content - The processed content. Convert JSON and msgpack to a dictionary.
* content_raw - The raw content from server, only passed through bytes to unicode.
* response - Raw treq response, with "all_headers" injected; which is a cleaned up headers version of
response.headers.
* content_type - NOT related to HTTP headers. This will be either "dict" if it's a dictionary, or "string".
* request - The original request object. Contains attributes such as: method, uri, and headers,
First two arguments:
* method (str) – HTTP method. Example: "GET", "HEAD". "PUT", "POST".
* url (str) – http or https URL, which may include query arguments.
Keyword arguments for fine tuning:
* headers (Headers or None) – Optional HTTP Headers to send with this request.
* params (dict w/ str or list/tuple of str values, list of 2-tuples, or None.) – Optional parameters to be append as the query string to the URL, any query string parameters in the URL already will be preserved.
* data (str, file-like, IBodyProducer, or None) – Optional request body.
* json (dict, list/tuple, int, string/unicode, bool, or None) – Optional JSON-serializable content to pass in body.
* persistent (bool) – Use persistent HTTP connections. Default: True
* allow_redirects (bool) – Follow HTTP redirects. Default: True
* auth (tuple of ("username", "password").) – HTTP Basic Authentication information.
* cookies (dict or CookieJar) – Cookies to send with this request. The HTTP kind, not the tasty kind.
* timeout (int) – Request timeout seconds. If a response is not received within this timeframe, a connection is aborted with CancelledError.
* browser_like_redirects (bool) – Use browser like redirects (i.e. Ignore RFC2616 section 10.3 and follow redirects from POST requests). Default: False
* unbuffered (bool) – Pass True to to disable response buffering. By default treq buffers the entire response body in memory.
:return:
"""
logger.debug("Request receive: {method} : {url}", method=method, url=url)
method = method.upper()
try:
treq_response = yield treq.request(method, url, **kwargs)
except ConnectionRefusedError as e:
raise YomboWarning(f"Connection was refused to '{url}': {e}")
except ConnectError as e:
raise YomboWarning(f"Error connecting to '{url}': {e}")
except Exception as e:
logger.info("Requests error: {error}", error=e)
logger.error("---------------==(Traceback)==--------------------------")
logger.error("{method} {url}", method=method, url=url)
logger.error("{trace}", trace=traceback.format_exc())
logger.error("--------------------------------------------------------")
logger.warn("An exception of type {etype} occurred in yombo.lib.yomboapi:import_component. Message: {msg}",
etype=type(e), msg=e)
logger.error("--------------------------------------------------------")
raise e
response = WebResponse(self)
yield response.process_response(treq_response)
return response
def test_http_parser():
for fname in glob.glob(os.path.join(reqdir, "*.http")):
if os.getenv("GUNS_BLAZING"):
expect = treq.load_py(os.path.splitext(fname)[0] + ".py")
req = treq.request(fname, expect)
for case in req.gen_cases():
yield case
else:
yield (a_case, fname)
def test_http_parser():
for fname in glob.glob(os.path.join(reqdir, "*.http")):
if os.getenv("GUNS_BLAZING"):
expect = treq.load_py(os.path.splitext(fname)[0] + ".py")
req = treq.request(fname, expect)
for case in req.gen_cases():
yield case
else:
yield (a_case, fname)
def check_if_started():
# Replace with ``GearClient.list`` as part of
# https://github.com/hybridlogic/flocker/issues/32
responded = request(
b"GET", b"http://127.0.0.1:%d/containers" % (GEAR_PORT,),
persistent=False)
responded.addCallback(content)
responded.addCallback(json.loads)
responded.addCallback(is_started)
return responded
def assertHTTPError(self, url, code, response_substring,
method="get", persistent=False,
**args):
response = yield treq.request(method, url, persistent=persistent,
**args)
body = yield response.content()
self.assertEquals(response.code, code)
if response_substring is not None:
self.assertIn(response_substring, body)
returnValue(body)
def assertHTTPError(self, url, code, response_substring,
method="get", persistent=False,
**args):
response = yield treq.request(method, url, persistent=persistent,
**args)
body = yield response.content()
self.assertEquals(response.code, code)
if response_substring is not None:
self.assertIn(response_substring, body)
returnValue(body)
def http_request(url,method='GET',headers=None,params=None,data=None,callback=None,json=True, code_only=False):
logging.critical('#####################################\n## THIS IS DEPRECATED - DO NOT USE ##\n#####################################')
request = treq.request(url=url,method=method,headers=headers,data=data)
if callback:
if code_only:
request.addCallback(callback)
if json:
request.addCallback(json_callback, callback)
else:
request.addCallback(text_callback, callback)
def bank_account_update(self, request):
id = request.args['member_id'][0]
name = request.args['name'][0]
contact_number = request.args['contact-number'][0]
address = request.args['address'][0]
price = request.args['price'][0]
branch_code = banks[request.args['bank'][0]][1]
account_number = request.args['bank-account'][0]
url = "/".join(request.prePathURL().split("/")[:3]) + "/notify"
q.update_account_number(main.con, id, name, price, contact_number,
address, branch_code, account_number)
treq.request("POST",
url,
headers={"Content-Type": "application/json"},
json={"procedure": "com.notify"})
# generate a PDF in a separate thread with all the information
return py.path.local(__file__).join(
'..', 'web',
'thankyou-bank.html').read().replace('{{gym_id}}',
request.args['gym_id'][0])
def doWebhook(self, wh):
self.log("Calling webhook: %s %s" % (wh['method'], wh['url']))
rsp = yield treq.request(wh['method'],
wh['url'],
data=wh['payload'],
persistent=False,
timeout=10)
rsp_content = yield rsp.content()
self.log("Webhook response (%s %s): %s %r" %
(wh['method'], wh['url'], rsp.code, rsp_content))
yield self.db.setWebhookResponse(wh['id'], rsp_content)
def _make_request(self, method_name, method='get', params=None, data=None, files=None, timeout=None, **kwargs):
request_url = API_URL + 'bot' + self.token + '/' + method_name
params = _convert_utf8(params)
if timeout is None:
timeout = self.timeout
resp = yield treq.request(method, request_url, params=params, data=data, files=files, timeout=timeout,
agent=self.agent, **kwargs)
result_json = yield _check_response(resp, method_name)
returnValue(result_json)
def perform_request_with_treq(dispatcher, http_request):
"""A performer for :obj:`HTTPRequest` that uses the ``treq`` library."""
headers = (http_request.headers.copy()
if http_request.headers is not None else {})
if 'user-agent' not in headers:
headers['user-agent'] = ['Effect example']
d = treq.request(http_request.method.lower(),
http_request.url,
headers=headers,
data=http_request.data).addCallback(treq.content)
return d
def process_queue(self):
while True:
thing = yield self.queue_object.get()
if thing is None:
break
ch, method, properties, body = thing
if body:
try:
di = json.loads(body)
except ValueError:
path = body
headers = {}
else:
path = di["path"]
headers = di["headers"]
self.log("Purging %s with headers %s" % (path, str(headers)))
domain = self.config.get("domain", None)
try:
if domain:
final_headers = {"Host": domain}
final_headers.update(headers)
response = yield treq.request(
"PURGE", "http://" \
+ self.config.get("proxy-address", "127.0.0.1") + path,
headers=final_headers,
timeout=10
)
else:
response = yield treq.request(
"PURGE", "http://" \
+ self.config.get("proxy-address", "127.0.0.1") + path,
timeout=10,
headers=headers
)
except Exception as exception:
msg = traceback.format_exc()
self.log("Error purging %s: %s" % (path, msg))
else:
content = yield response.content()
yield ch.basic_ack(delivery_tag=method.delivery_tag)
def _make_request(self, method, path, data=None, *args, **kwargs):
data = json.dumps(data) if data is not None else None
headers = self._all_extra_headers()
new_headers = kwargs.pop("headers", {})
headers.update(new_headers)
if self.auth:
kwargs['auth'] = self.auth
return treq.request(method, path, headers=headers, data=data,
**kwargs).addCallbacks(self._handle_response,
log.err)
def request(self, path, method="GET", headers=None, **kwargs):
if headers is None:
headers = Headers()
headers.addRawHeader("User-Agent", self.user_agent)
return request(
method,
headers=headers,
url=self.root_url + path,
params=[("fmt", "json")],
reactor=self.reactor,
**kwargs
).addCallback(json_content).addCallback(_check_for_errors)
def request(self, method, uri, data=None, headers=None):
"""
Args:
method (str): HTTP method to use.
uri (str): URI to query.
data (bytes): Data to send in the request body, if applicable.
headers (t.w.http_headers.Headers): Request headers.
Raises:
SynapseError: If the IP is blacklisted.
"""
# A small wrapper around self.agent.request() so we can easily attach
# counters to it
outgoing_requests_counter.labels(method).inc()
# log request but strip `access_token` (AS requests for example include this)
logger.info("Sending request %s %s", method, redact_uri(uri))
try:
body_producer = None
if data is not None:
body_producer = QuieterFileBodyProducer(BytesIO(data))
request_deferred = treq.request(method,
uri,
agent=self.agent,
data=body_producer,
headers=headers,
**self._extra_treq_args)
request_deferred = timeout_deferred(
request_deferred,
60,
self.hs.get_reactor(),
cancelled_to_request_timed_out_error,
)
response = yield make_deferred_yieldable(request_deferred)
incoming_responses_counter.labels(method, response.code).inc()
logger.info("Received response to %s %s: %s", method,
redact_uri(uri), response.code)
defer.returnValue(response)
except Exception as e:
incoming_responses_counter.labels(method, "ERR").inc()
logger.info(
"Error sending request to %s %s: %s %s",
method,
redact_uri(uri),
type(e).__name__,
e.args[0],
)
raise
def perform_request_with_treq(dispatcher, http_request):
"""A performer for :obj:`HTTPRequest` that uses the ``treq`` library."""
headers = (
http_request.headers.copy()
if http_request.headers is not None
else {})
if 'user-agent' not in headers:
headers['user-agent'] = ['Effect example']
d = treq.request(
http_request.method.lower(),
http_request.url,
headers=headers,
data=http_request.data).addCallback(treq.content)
return d
def do_http(method, url, **kwargs):
"""
Run HTTP query, return Deferred of body as bytes.
"""
response = yield treq.request(method, url, persistent=False, **kwargs)
body = yield treq.content(response)
# TODO: replace this with response.fail_for_status when
# https://github.com/twisted/treq/pull/159 has landed
if 400 <= response.code < 600:
raise VerboseError(
response.code,
response="For request {!r} to {!r}, got: {!r}".format(
method, url, body))
returnValue(body)
def _request(self, method, url, **kwargs):
if 'headers' in kwargs:
headers = self.config.base_headers.copy()
headers.update(kwargs['headers'])
kwargs['headers'] = headers
else:
kwargs['headers'] = self.config.base_headers
if self.config.api_key_secret:
kwargs['auth'] = (self.config.api_key_secret, '')
d = treq.request(method, url, **kwargs)
d.addCallback(self.deserialize)
return d
def proxy_devserver(request):
def stream_response(response):
request.setResponseCode(response.code)
for key, values in response.headers.getAllRawHeaders():
for value in values:
request.setHeader(key, value)
d = treq.collect(response, request.write)
d.addCallback(lambda _: request.finish())
return d
url = ELM_DEVSERVER.encode('utf-8') + request.uri[len('/dev'):]
d = treq.request(request.method.decode('ascii'), url)
d.addCallback(stream_response)
return d
def request(self, method, uri, data=b'', headers=None):
"""
Args:
method (str): HTTP method to use.
uri (str): URI to query.
data (bytes): Data to send in the request body, if applicable.
headers (t.w.http_headers.Headers): Request headers.
Raises:
SynapseError: If the IP is blacklisted.
"""
# A small wrapper around self.agent.request() so we can easily attach
# counters to it
outgoing_requests_counter.labels(method).inc()
# log request but strip `access_token` (AS requests for example include this)
logger.info("Sending request %s %s", method, redact_uri(uri))
try:
request_deferred = treq.request(
method,
uri,
agent=self.agent,
data=data,
headers=headers,
**self._extra_treq_args
)
request_deferred = timeout_deferred(
request_deferred,
60,
self.hs.get_reactor(),
cancelled_to_request_timed_out_error,
)
response = yield make_deferred_yieldable(request_deferred)
incoming_responses_counter.labels(method, response.code).inc()
logger.info(
"Received response to %s %s: %s", method, redact_uri(uri), response.code
)
defer.returnValue(response)
except Exception as e:
incoming_responses_counter.labels(method, "ERR").inc()
logger.info(
"Error sending request to %s %s: %s %s",
method,
redact_uri(uri),
type(e).__name__,
e.args[0],
)
raise
def _request(method, url, **kwargs):
def handle(req):
if req.code != 200:
req.content().addCallback(log.debug)
raise HTTPError("Bad status code: {} for URL '{}'".format(req.code, url), req.code)
else:
return req.content()
# set the User-Agent header if it isn't already set by the user
if 'headers' not in kwargs:
kwargs['headers'] = {}
if 'User-Agent' not in kwargs['headers']:
kwargs['headers']['User-Agent'] = 'exchangelib/{}'.format(VERSION)
return treq.request(method, url, **kwargs).addCallback(handle)
def _request(self, service, method='GET', params=None):
''' Make the actual request '''
if not self.config:
raise errors.CCBError('Internal Error: No CCB configuration')
data = None
if method == "POST" and params:
data = params.copy()
params = None
if not params:
params = {}
params['srv'] = service
auth = (self.config.username, self.config.password)
d = treq.request(method, self.config.url, params=params, data=data, auth=auth)
d.addCallback(self._get_content, params["srv"])
return d
def retry(req, **overrides):
req['attempts'] = req['attempts'] + 1
d = req['request']
opts = {
'method': encode(d['method']),
'url': encode(d['url']),
'data': encode(d.get('body')),
'headers': dict([
(encode(k), [encode(v) for v in values])
for k, values in d.get('headers', {}).iteritems()]),
}
opts.update(overrides)
return treq.request(**opts)
def request(self, request_type, method, payload={}):
def _read_response(response):
logger.info("Received Data: %s", response)
if response.code != 200:
raise DiscourseApiError("{}".format(response.phrase))
return response.content().addCallback(lambda x: pprint(x))
payload.update(dict(api_key=self.api_key,
api_username=self.api_username))
dfr = treq.request(request_type, "{}{}".format(self.host, method),
params=payload,
headers={"Content-Type": "application/json"})
return dfr.addCallback(_read_response)
def test_cached_pool(self):
"""
The first use of the module-level API populates the global connection
pool, which is used for all subsequent requests.
"""
pool = SyntacticAbominationHTTPConnectionPool()
self.patch(treq.api, "HTTPConnectionPool", lambda reactor, persistent: pool)
self.failureResultOf(treq.head("http://test.com"), TabError)
self.failureResultOf(treq.get("http://test.com"), TabError)
self.failureResultOf(treq.post("http://test.com"), TabError)
self.failureResultOf(treq.put("http://test.com"), TabError)
self.failureResultOf(treq.delete("http://test.com"), TabError)
self.failureResultOf(treq.request("OPTIONS", "http://test.com"), TabError)
self.assertEqual(pool.requests, 6)
def _call(self, method, url, data=None):
url = urlparse.urljoin(self.endpoint, url).encode('utf-8')
headers = {"Content-Type": "application/json"}
if self.token:
headers["Authorization"] = "Bearer %s" % self.token
if data:
data = json.dumps(data) # raises TypeError
if self.debug:
print "%s %s" % (method, url)
print "Headers: %s" % str(headers)
print "Data: %s" % data
response = yield treq.request(method, url, headers=headers, data=data)
code = response.code
content = yield treq.content(response)
if self.debug:
print "Got: %s" % content
try:
def encode(data):
encoded = {}
for key, value in data.iteritems():
if isinstance(key, unicode):
key = key.encode("utf-8")
if isinstance(value, unicode):
value = value.encode("utf-8")
encoded[key] = value
return encoded
content = json.loads(content, object_hook=encode)
except ValueError as e:
pass
if code == 200:
returnValue(content)
elif code == 400:
raise BadRequest(content)
elif code == 401:
raise Unauthorized(content)
elif code == 404:
raise MethodNotFound(content)
elif code == 406:
raise NotAcceptable(content)
else:
raise BitGoException(content)
def _request(self, method, path, data=None):
"""Send HTTP request to gear.
:param bytes method: The HTTP method to send, e.g. ``b"GET"``.
:param bytes path: Path to request.
:param data: ``None``, or object with a body for the request that
can be serialized to JSON.
:return: A ``Defered`` that fires with a response object.
"""
url = self._base_url + path
if data is not None:
data = json.dumps(data)
return request(method, url, data=data, persistent=False)
def _call(self, method, url, data=None):
url = urlparse.urljoin(self.endpoint, url).encode('utf-8')
headers = {"Content-Type": "application/json"}
if self.token:
headers["Authorization"] = "Bearer %s" % self.token
if data:
data = json.dumps(data) # raises TypeError
if self.debug:
print "%s %s" % (method, url)
print "Headers: %s" % str(headers)
print "Data: %s" % data
response = yield treq.request(method, url, headers=headers, data=data)
code = response.code
content = yield treq.content(response)
if self.debug:
print "Got: %s" % content
try:
def encode(data):
encoded = {}
for key, value in data.iteritems():
if isinstance(key, unicode):
key = key.encode("utf-8")
if isinstance(value, unicode):
value = value.encode("utf-8")
encoded[key] = value
return encoded
content = json.loads(content, object_hook=encode)
except ValueError as e:
pass
if code == 200:
returnValue(content)
elif code == 400:
raise BadRequest(content)
elif code == 401:
raise Unauthorized(content)
elif code == 404:
raise MethodNotFound(content)
elif code == 406:
raise NotAcceptable(content)
else:
raise BitGoException(content)
def __request(self, url, data, content_type):
headers = \
{"Content-Type": [content_type.encode("ascii", "ignore")], \
"X-Application": [self.app_key.encode("ascii", "ignore")], \
"X-Authentication": [self.session_token.encode("ascii", "ignore")]}
data = data.encode("ascii", "ignore")
self.logger.debug(url)
self.logger.debug(headers)
self.logger.debug(data)
request = treq.request(method="POST", \
url=url.encode("ascii", "ignore"), headers=headers, data=data)
request .addErrback(self.__error_handler)
resp = yield request
content = yield treq.content(resp)
returnValue(content)
def send_request():
"""
Send an HTTP request in a loop until the request is answered.
"""
response = request(b"GET", b"http://127.0.0.1:%d" % (port,), persistent=False)
def check_error(failure):
"""
Catch ConnectionRefused errors and response timeouts and return
False so that loop_until repeats the request.
Other error conditions will be passed down the errback chain.
"""
failure.trap(ConnectionRefusedError, ResponseNeverReceived)
return False
response.addErrback(check_error)
return response
def execute(self, method, path, body=None, params=None):
"""Execute a query against a server."""
headers = {b'Content-Type': [b'application/json']}
if not isinstance(body, basestring):
body = anyjson.serialize(body)
for attempt in range(self.max_retries + 1):
server = self.servers.get()
timeout = self.servers.timeout
url = _prepare_url(server, path, params)
response = None # Stop UnboundLocalError on uncaught exception
try:
response = yield treq.request(
method, url, data=body, pool=self.pool,
auth=self.http_auth, persistent=self.persistent,
timeout=timeout, headers=headers)
json_data = yield response.json()
exceptions.raise_exceptions(response.code, json_data)
except Exception as e:
retry = False
if isinstance(
e, (
ConnectError,
ResponseFailed,
RequestTransmissionFailed,
)
) or response and response.code in (503, 504):
retry = True
if retry:
self.servers.mark_dead(server)
if attempt == self.max_retries:
raise
continue
else:
raise
else:
defer.returnValue(json_data)
def _do_request(self, method, uri, *args, **kwargs):
"""
Modularized because API was broken.
Need to be able to inject Mocked response objects here.
"""
method = method.upper()
params = kwargs.pop('params', None)
if params:
uri += '?' + urlencode(params)
headers = kwargs.pop('headers', None)
if headers is not None:
headers = {key: [value, ] for (key, value) in headers.iteritems()}
headers = Headers(headers)
data = kwargs.pop('data', None)
response = yield treq.request(method, uri, headers=headers, data=data,)
handled = yield self._handle_response(response)
defer.returnValue(handled)
