def request(method, url, **kwargs): """ Pass auth=HTTPKerberosAuth() kwarg """ auth = kwargs.get('auth') headers = kwargs.get('headers', {}) # headers = headers.copy() # ? We do modify the dict in place here... if isinstance(auth, TreqKerberosAuth): del kwargs['auth'] if auth.force_preemptive: # Save a round-trip and set the Negotiate header on the first req. headers['Authorization'] = yield negotiate_header(url) response = yield treq.request(method=method, url=url, headers=headers, **kwargs) # Retry if we got a 401 / Negotiate response. if response.code == 401 and isinstance(auth, TreqKerberosAuth): auth_mechs = response.headers.getRawHeaders('WWW-Authenticate') if 'Negotiate' in auth_mechs: headers['Authorization'] = yield negotiate_header(url) response = yield treq.request(method=method, url=url, headers=headers, **kwargs) defer.returnValue(response)
def create(self, path='/'): self.data['address'] = str(self.data['address']) self.data['address_obj'] = urlparse(self.data['address']) # Build the necessary request headers for contacting the HTTP resource options = self.data['options'] if 'auth_method' in options: if options['auth_method'].lower() == 'basic' and options['auth_username'] and options['auth_password']: self._treq_options['auth'] = (options['auth_username'], options['auth_password']) if 'timeout' in options: try: self._treq_options['timeout'] = int(options['timeout']) except: pass if 'user-agent' in options: self._treq_options['headers'] = {'User-Agent': [str(options['user-agent'])]} # Fetch the first page and determine which template parser is needed resp = yield request(method='GET', url=self._buildURL(path), **self._treq_options) if resp.code != 200: raise CrawlException('Response code was not 200: %s' % str(resp.code)) self._resp_headers = dict((key.lower(), value[0].lower()) for (key, value) in resp.headers._rawHeaders.items()) response_body = yield text_content(resp) self._indexParser = HttpParse().get_parser(response_headers=self._resp_headers, response_body=response_body) if self._indexParser: defer.returnValue(True) defer.returnValue(False)
def _request(self, method, path, headers=None, **kwargs): url = '%s/%s' % (self.endpoint, path) headers = headers if headers else {} if self.token: headers['Authorization'] = 'Bearer %s' % (self.token, ) return treq.request(method, url, headers=headers, **kwargs).addCallback(treq.json_content)
def _internal_call(self, method, url, payload, params): args = dict(params=params) if not url.startswith('http'): url = self.prefix + url headers = self._auth_headers() headers['Content-Type'] = ['application/json'] if payload: args["data"] = json.dumps(payload) # TODO: treq can't handle unicode urls :( url = str(url) print("requesting %s" % url) r_d = treq.request(method, url, headers=Headers(headers), **args) r = yield r_d response_json = yield r.json() if self.trace: # pragma: no cover print() print(method, url) if payload: print("DATA", json.dumps(payload)) if r.code != 200: raise SpotifyException( r.code, -1, u'%s:\n %s' % (url, response_json['error']['message'])) if self.trace: # pragma: no cover print('RESP', response_json) print() defer.returnValue(response_json)
def a_case(fname): env = treq.load_py(os.path.splitext(fname)[0] + ".py") expect = env['request'] cfg = env['cfg'] req = treq.request(fname, expect) for case in req.gen_cases(cfg): case[0](*case[1:])
def _request(self, http_method, url, payload=None): def _raise_error(txt): logger.info("ERROR: %s", txt) raise TwitterApiError(txt) def _print_response(response): logger.info("RESPONSE: ", response) return response def _read_response(response): if response.code != 200: return response.text().addCallback(_raise_error) return treq.content().addCallback(json.loads) headers = {} body = None client = self._oauth_client() if payload: body = urlencode(payload) headers = {'Content-Type': 'application/x-www-form-urlencoded'} xx, headers, body = client.sign(url, http_method=http_method, headers=headers, body=body) dfr = treq.request(http_method, url, headers=headers, data=body) return dfr.addCallback(_read_response)
def request(self, method, uri, data=b'', headers=None): # A small wrapper around self.agent.request() so we can easily attach # counters to it outgoing_requests_counter.labels(method).inc() # log request but strip `access_token` (AS requests for example include this) logger.info("Sending request %s %s", method, redact_uri(uri)) try: request_deferred = treq.request(method, uri, agent=self.agent, data=data, headers=headers) request_deferred = timeout_deferred( request_deferred, 60, self.hs.get_reactor(), cancelled_to_request_timed_out_error, ) response = yield make_deferred_yieldable(request_deferred) incoming_responses_counter.labels(method, response.code).inc() logger.info("Received response to %s %s: %s", method, redact_uri(uri), response.code) defer.returnValue(response) except Exception as e: incoming_responses_counter.labels(method, "ERR").inc() logger.info("Error sending request to %s %s: %s %s", method, redact_uri(uri), type(e).__name__, e.args[0]) raise
def _fetch_resource(self, method: str, suffix: str, params: Dict[str, Any] = None) -> Deferred: """Post request data to API.""" if treq is None: raise ImportError( "treq is required for the IceCubedAsyncClient. Try installing the with `pip install ice3x[async]`" ) if params is None: params = {} headers = { "User-Agent": "Mozilla/4.0 (compatible; Ice3x Async Python client)" } if method == "post": assert ( self.api_key is not None ), f"An API key is required in order to access the {suffix} resource." headers["Key"] = self.api_key headers["Sign"] = self.sign(params) kwargs = {"params": params, "headers": headers} url = f"{self.BASE_URI}{suffix}" resp = yield treq.request(method, url, **kwargs) data = yield resp.json() return data
def test_request_invalid_param(self): """ `treq.request()` warns that it ignores unknown keyword arguments, but this is deprecated. This test verifies that stacklevel is set appropriately when issuing the warning. """ self.failureResultOf( treq.request( "GET", "https://foo.bar", invalid=True, pool=SyntacticAbominationHTTPConnectionPool(), ) ) [w] = self.flushWarnings([self.test_request_invalid_param]) self.assertEqual(DeprecationWarning, w["category"]) self.assertEqual( ( "Got unexpected keyword argument: 'invalid'." " treq will ignore this argument," " but will raise TypeError in the next treq release." ), w["message"], )
def GET(self, urlpath, followRedirect=False, return_response=False, method="GET", clientnum=0, **kwargs): # if return_response=True, this fires with (data, statuscode, # respheaders) instead of just data. assert not isinstance(urlpath, unicode) url = self.client_baseurls[clientnum] + urlpath response = yield treq.request(method, url, persistent=False, allow_redirects=followRedirect, **kwargs) data = yield response.content() if return_response: # we emulate the old HTTPClientGetFactory-based response, which # wanted a tuple of (bytestring of data, bytestring of response # code like "200" or "404", and a # twisted.web.http_headers.Headers instance). Fortunately treq's # response.headers has one. defer.returnValue((data, str(response.code), response.headers)) if 400 <= response.code < 600: raise Error(response.code, response=data) defer.returnValue(data)
def request(self, method, url_suffix, parser=None, **kw): """ Start the application, make an HTTP request and then shutdown the application. :param str url_suffix: A path to make the request to. :param str parser: Response parser to use. Valid values are ``'bytes'``, ``'json'``, ``'json_lines'`` or ``None``. ``None`` indicates that the raw response object should be returned. Otherwise the parsed data is returned. Other parameters are the same as for :func:`treq.request`. """ server = self.reactor.listenTCP(0, self.app, interface="127.0.0.1") host = server.getHost() # prefix the URL with the test server host and port url = ('http://127.0.0.1:%d' % host.port) + url_suffix # close the HTTP connection straight away so that the test # reactor is clean when we leave this function kw['persistent'] = False response = yield treq.request(method, url, reactor=self.reactor, **kw) if parser is not None: parser_method = getattr(self, '_parse_' + parser) response = yield parser_method(response) yield server.stopListening() server.loseConnection() returnValue(response)
def _internal_call(self, method, url, payload, params): args = dict(params=params) if not url.startswith('http'): url = self.prefix + url headers = self._auth_headers() headers['Content-Type'] = ['application/json'] if payload: args["data"] = json.dumps(payload) # TODO: treq can't handle unicode urls :( url = str(url) print("requesting %s" % url) r_d = treq.request(method, url, headers=Headers(headers), **args) r = yield r_d response_json = yield r.json() if self.trace: # pragma: no cover print() print(method, url) if payload: print("DATA", json.dumps(payload)) if r.code != 200: raise SpotifyException(r.code, -1, u'%s:\n %s' % (url, response_json['error']['message'])) if self.trace: # pragma: no cover print('RESP', response_json) print() defer.returnValue(response_json)
def process_queue(self): while True: thing = yield self.queue_object.get() if thing is None: break ch, method, properties, body = thing if body: path = body print "PURGE %s %s" % (ua_map, path) try: response = yield treq.request( "PURGE", "http://127.0.0.1" + path, cookies={"foo": "bar"}, headers={"Host": "actual.host.com"}, timeout=10 ) except (ConnectError, DNSLookupError, CancelledError, ResponseFailed): # Maybe better to do a blank except? print "ERROR %s %s" (ua_map, path) else: print "RESULT %s %s" % (ua_map, path) content = yield response.content() print content yield ch.basic_ack(delivery_tag=method.delivery_tag)
def with_signature(self, meth, object_path, params=None, data=None, headers=None): meth = meth.upper() subresources = [] for key, value in params.items(): if key in self._subresource_key_set: subresources.append((key, value)) subresources.sort(key=lambda p: p[0]) _subresources = '&'.join( ("=".join([k, v]) if v else k for k, v in subresources)) resources_string = '&'.join( [f'/{self._bucket}/{object_path}', _subresources]) headers = headers or {} headers['date'] = formatdate(None, usegmt=True) headers_string = '\n'.join([ ':'.join([k, v]) for k, v in sorted([(k.lower(), v) for k, v in headers.items() if k.lower().startswith('x-oss-')], key=lambda h: h[0]) ]) if headers_string: headers_string += '\n' content_md5 = headers.get('content-md5', '') content_type = headers.get('content-type', '') _string_to_sign = '\n'.join([ meth, content_md5, content_type, headers['date'], ''.join([headers_string, resources_string]) ]) h = hmac.new(self._sk.encode(), _string_to_sign.encode(), hashlib.sha1) _signature = base64.b64encode(h.digest()).decode() headers["Authorization"] = f"OSS {self._ak}:{_signature}" params['OSSAccessKeyId'] = self._ak params['Signature'] = _signature url = '?'.join([ self._endpoint(object_path), '&'.join([ '='.join([quote(k, ''), quote(v, '')]) if v else quote(k, '') for k, v in params.items() ]) ]) response = yield treq.request(meth, url, data=data, params=params, headers=headers) return response
def perform_effect(self, dispatcher): import treq headers = self.headers.copy() if self.headers is not None else {} if 'user-agent' not in headers: headers['user-agent'] = ['Effect example'] return treq.request(self.method.lower(), self.url, headers=headers, data=self.data).addCallback(treq.content)
def _fetch_resource(self, method: str, suffix: str, params: Dict = {}) -> Deferred: """Helper function to make API requests Args: method: The http verb i.e. get, post, put, delete suffix: The uri suffix params: A dict of query params Returns: A twisted deferred """ url = f"{self.BASE_URI}{suffix}" headers = {"Content-Type": "application/x-www-form-urlencoded"} auth = (self.api_key, self.secret) resp = yield treq.request(method, url, params=params, headers=headers, auth=auth) data = yield resp.json() return data
def _call(self, method, call, **kwargs): url = self.endpoint + call headers = { 'content-type': 'application/json', 'accept': 'application/json' } if self.session_id is not None: headers['coinsetter-client-session-id'] = self.session_id.encode( 'utf-8') new_kwargs = {'headers': headers} new_kwargs.update(kwargs) if 'data' in kwargs: new_kwargs['data'] = json.dumps(kwargs['data']) try: result = yield treq.request(method, url.encode('utf-8'), **new_kwargs) except (IOError, ConnectError) as e: log.err(e) self.emit("disconnect", self) raise e content = yield result.content() if result.code != 200: self.emit("disconnect", self) raise Exception("Error code %d received" % result.code) parsed = json.loads(content, parse_float=Decimal) returnValue(parsed)
def _make_request(self, method, path, data=None, *args, **kwargs): """Parse and create the request object.""" data = json.dumps(data) if data is not None else None headers = self._all_extra_headers() new_headers = kwargs.pop("headers", {}) headers.update(new_headers) if self.auth: kwargs['auth'] = self.auth if kwargs.get('params', {}): params = kwargs.get('params', {}) for key, value in params.items(): value = utf8(value) if isinstance(value, basestring) else value params[key] = value if params: kwargs['params'] = params return treq.request( method, path, headers=headers, data=data, **kwargs ).addCallbacks( self._handle_response, log.err )
def _call(self, method, call, **kwargs): url = self.endpoint + call headers = {'content-type': 'application/json', 'accept': 'application/json'} if self.session_id is not None: headers['coinsetter-client-session-id'] = self.session_id.encode('utf-8') new_kwargs = {'headers': headers} new_kwargs.update(kwargs) if 'data' in kwargs: new_kwargs['data'] = json.dumps(kwargs['data']) try: result = yield treq.request(method, url.encode('utf-8'), **new_kwargs) except (IOError, ConnectError) as e: log.err(e) self.emit("disconnect", self) raise e content = yield result.content() if result.code != 200: self.emit("disconnect", self) raise Exception("Error code %d received" % result.code) parsed = json.loads(content, parse_float=Decimal) returnValue(parsed)
def _query_mailchimp(data_center, method_section, method_name, payload): def _read_response(response): logger.info("Received Data: %s", response) if response.code != 200: logger.info("Received Data with error") raise MailChimpApiError("{}".format(response.phrase)) return treq.text_content(response).addCallback(json.loads) def _check_for_error(response): logger.info("Received Data: %s", response) if "error" in response: raise MailChimpApiError("{}:{}".format(response["name"], response["error"])) return response logger.info("STARTING QUERY MAICHIMP WITH PAYLOAD {}".format(payload)) dfr = treq.request("POST", MAILCHIMP_BASE_URL.format(dc=data_center, section=method_section, name=method_name).encode("utf-8"), data=json.dumps(payload), headers={"Content-Type": "application/json"}) return dfr.addCallback(_read_response).addCallback(_check_for_error)
def download_file(self, url: str, destination_filename: str): """ Downloads a file from the given url and saves it to the requested destination. Returns a deferred! :param url: :param destination_filename: :return: """ try: treq_response = yield treq.request("GET", url) except ConnectionRefusedError as e: raise YomboWarning(f"Connection was refused to '{url}': {e}") except ConnectError as e: raise YomboWarning(f"Error connecting to '{url}': {e}") except Exception as e: logger.info("Requests download_file error: {error}", error=e) logger.error( "---------------==(Traceback)==--------------------------") logger.error("{url}", url=url) logger.error("{trace}", trace=traceback.format_exc()) logger.error( "--------------------------------------------------------") logger.warn( "An exception of type {etype} occurred in yombo.lib.yomboapi:import_component. Message: {msg}", etype=type(e), msg=e) logger.error( "--------------------------------------------------------") raise e raw_content = yield treq.content(treq_response) yield self._Files.save(destination_filename, raw_content)
def _make_request(token, method_name, method='get', params=None, data=None, files=None, timeout=10, **kwargs): request_url = API_URL + 'bot' + token + '/' + method_name params = _convert_utf8(params) resp = yield treq.request(method, request_url, params=params, data=data, files=files, timeout=timeout, **kwargs) result_json = yield _check_response(resp, method_name) returnValue(result_json)
def do_http(method, url, **kwargs): response = yield treq.request(method, url, persistent=False, **kwargs) body = yield treq.content(response) # TODO: replace this with response.fail_for_status when # https://github.com/twisted/treq/pull/159 has landed if 400 <= response.code < 600: raise Error(response.code, response=body) defer.returnValue(body)
def request(self, method, path, data=None): return treq.request( method, 'http://localhost:%s%s' % ( self.listener_port, path ), data=data, pool=self.pool)
def test_request_invalid_param(self): """ `treq.request()` warns that it ignores unknown keyword arguments, but this is deprecated. This test verifies that stacklevel is set appropriately when issuing the warning. """ with self.assertRaises(TypeError) as c: treq.request( "GET", "https://foo.bar", invalid=True, pool=SyntacticAbominationHTTPConnectionPool(), ) self.assertIn("invalid", str(c.exception))
def request(self, method, url, **kwargs): """ Make an HTTP request using treq. This basically uses treq, but parses the response and attempts to decode the data if it's json or msgpack. This must be called with "yield". It returns a dictionary with these keys: * content - The processed content. Convert JSON and msgpack to a dictionary. * content_raw - The raw content from server, only passed through bytes to unicode. * response - Raw treq response, with "all_headers" injected; which is a cleaned up headers version of response.headers. * content_type - NOT related to HTTP headers. This will be either "dict" if it's a dictionary, or "string". * request - The original request object. Contains attributes such as: method, uri, and headers, First two arguments: * method (str) – HTTP method. Example: "GET", "HEAD". "PUT", "POST". * url (str) – http or https URL, which may include query arguments. Keyword arguments for fine tuning: * headers (Headers or None) – Optional HTTP Headers to send with this request. * params (dict w/ str or list/tuple of str values, list of 2-tuples, or None.) – Optional parameters to be append as the query string to the URL, any query string parameters in the URL already will be preserved. * data (str, file-like, IBodyProducer, or None) – Optional request body. * json (dict, list/tuple, int, string/unicode, bool, or None) – Optional JSON-serializable content to pass in body. * persistent (bool) – Use persistent HTTP connections. Default: True * allow_redirects (bool) – Follow HTTP redirects. Default: True * auth (tuple of ("username", "password").) – HTTP Basic Authentication information. * cookies (dict or CookieJar) – Cookies to send with this request. The HTTP kind, not the tasty kind. * timeout (int) – Request timeout seconds. If a response is not received within this timeframe, a connection is aborted with CancelledError. * browser_like_redirects (bool) – Use browser like redirects (i.e. Ignore RFC2616 section 10.3 and follow redirects from POST requests). Default: False * unbuffered (bool) – Pass True to to disable response buffering. By default treq buffers the entire response body in memory. :return: """ logger.debug("Request receive: {method} : {url}", method=method, url=url) method = method.upper() try: treq_response = yield treq.request(method, url, **kwargs) except ConnectionRefusedError as e: raise YomboWarning(f"Connection was refused to '{url}': {e}") except ConnectError as e: raise YomboWarning(f"Error connecting to '{url}': {e}") except Exception as e: logger.info("Requests error: {error}", error=e) logger.error("---------------==(Traceback)==--------------------------") logger.error("{method} {url}", method=method, url=url) logger.error("{trace}", trace=traceback.format_exc()) logger.error("--------------------------------------------------------") logger.warn("An exception of type {etype} occurred in yombo.lib.yomboapi:import_component. Message: {msg}", etype=type(e), msg=e) logger.error("--------------------------------------------------------") raise e response = WebResponse(self) yield response.process_response(treq_response) return response
def test_http_parser(): for fname in glob.glob(os.path.join(reqdir, "*.http")): if os.getenv("GUNS_BLAZING"): expect = treq.load_py(os.path.splitext(fname)[0] + ".py") req = treq.request(fname, expect) for case in req.gen_cases(): yield case else: yield (a_case, fname)
def check_if_started(): # Replace with ``GearClient.list`` as part of # https://github.com/hybridlogic/flocker/issues/32 responded = request( b"GET", b"http://127.0.0.1:%d/containers" % (GEAR_PORT,), persistent=False) responded.addCallback(content) responded.addCallback(json.loads) responded.addCallback(is_started) return responded
def assertHTTPError(self, url, code, response_substring, method="get", persistent=False, **args): response = yield treq.request(method, url, persistent=persistent, **args) body = yield response.content() self.assertEquals(response.code, code) if response_substring is not None: self.assertIn(response_substring, body) returnValue(body)
def http_request(url,method='GET',headers=None,params=None,data=None,callback=None,json=True, code_only=False): logging.critical('#####################################\n## THIS IS DEPRECATED - DO NOT USE ##\n#####################################') request = treq.request(url=url,method=method,headers=headers,data=data) if callback: if code_only: request.addCallback(callback) if json: request.addCallback(json_callback, callback) else: request.addCallback(text_callback, callback)
def bank_account_update(self, request): id = request.args['member_id'][0] name = request.args['name'][0] contact_number = request.args['contact-number'][0] address = request.args['address'][0] price = request.args['price'][0] branch_code = banks[request.args['bank'][0]][1] account_number = request.args['bank-account'][0] url = "/".join(request.prePathURL().split("/")[:3]) + "/notify" q.update_account_number(main.con, id, name, price, contact_number, address, branch_code, account_number) treq.request("POST", url, headers={"Content-Type": "application/json"}, json={"procedure": "com.notify"}) # generate a PDF in a separate thread with all the information return py.path.local(__file__).join( '..', 'web', 'thankyou-bank.html').read().replace('{{gym_id}}', request.args['gym_id'][0])
def doWebhook(self, wh): self.log("Calling webhook: %s %s" % (wh['method'], wh['url'])) rsp = yield treq.request(wh['method'], wh['url'], data=wh['payload'], persistent=False, timeout=10) rsp_content = yield rsp.content() self.log("Webhook response (%s %s): %s %r" % (wh['method'], wh['url'], rsp.code, rsp_content)) yield self.db.setWebhookResponse(wh['id'], rsp_content)
def _make_request(self, method_name, method='get', params=None, data=None, files=None, timeout=None, **kwargs): request_url = API_URL + 'bot' + self.token + '/' + method_name params = _convert_utf8(params) if timeout is None: timeout = self.timeout resp = yield treq.request(method, request_url, params=params, data=data, files=files, timeout=timeout, agent=self.agent, **kwargs) result_json = yield _check_response(resp, method_name) returnValue(result_json)
def perform_request_with_treq(dispatcher, http_request): """A performer for :obj:`HTTPRequest` that uses the ``treq`` library.""" headers = (http_request.headers.copy() if http_request.headers is not None else {}) if 'user-agent' not in headers: headers['user-agent'] = ['Effect example'] d = treq.request(http_request.method.lower(), http_request.url, headers=headers, data=http_request.data).addCallback(treq.content) return d
def process_queue(self): while True: thing = yield self.queue_object.get() if thing is None: break ch, method, properties, body = thing if body: try: di = json.loads(body) except ValueError: path = body headers = {} else: path = di["path"] headers = di["headers"] self.log("Purging %s with headers %s" % (path, str(headers))) domain = self.config.get("domain", None) try: if domain: final_headers = {"Host": domain} final_headers.update(headers) response = yield treq.request( "PURGE", "http://" \ + self.config.get("proxy-address", "127.0.0.1") + path, headers=final_headers, timeout=10 ) else: response = yield treq.request( "PURGE", "http://" \ + self.config.get("proxy-address", "127.0.0.1") + path, timeout=10, headers=headers ) except Exception as exception: msg = traceback.format_exc() self.log("Error purging %s: %s" % (path, msg)) else: content = yield response.content() yield ch.basic_ack(delivery_tag=method.delivery_tag)
def _make_request(self, method, path, data=None, *args, **kwargs): data = json.dumps(data) if data is not None else None headers = self._all_extra_headers() new_headers = kwargs.pop("headers", {}) headers.update(new_headers) if self.auth: kwargs['auth'] = self.auth return treq.request(method, path, headers=headers, data=data, **kwargs).addCallbacks(self._handle_response, log.err)
def request(self, path, method="GET", headers=None, **kwargs): if headers is None: headers = Headers() headers.addRawHeader("User-Agent", self.user_agent) return request( method, headers=headers, url=self.root_url + path, params=[("fmt", "json")], reactor=self.reactor, **kwargs ).addCallback(json_content).addCallback(_check_for_errors)
def request(self, method, uri, data=None, headers=None): """ Args: method (str): HTTP method to use. uri (str): URI to query. data (bytes): Data to send in the request body, if applicable. headers (t.w.http_headers.Headers): Request headers. Raises: SynapseError: If the IP is blacklisted. """ # A small wrapper around self.agent.request() so we can easily attach # counters to it outgoing_requests_counter.labels(method).inc() # log request but strip `access_token` (AS requests for example include this) logger.info("Sending request %s %s", method, redact_uri(uri)) try: body_producer = None if data is not None: body_producer = QuieterFileBodyProducer(BytesIO(data)) request_deferred = treq.request(method, uri, agent=self.agent, data=body_producer, headers=headers, **self._extra_treq_args) request_deferred = timeout_deferred( request_deferred, 60, self.hs.get_reactor(), cancelled_to_request_timed_out_error, ) response = yield make_deferred_yieldable(request_deferred) incoming_responses_counter.labels(method, response.code).inc() logger.info("Received response to %s %s: %s", method, redact_uri(uri), response.code) defer.returnValue(response) except Exception as e: incoming_responses_counter.labels(method, "ERR").inc() logger.info( "Error sending request to %s %s: %s %s", method, redact_uri(uri), type(e).__name__, e.args[0], ) raise
def perform_request_with_treq(dispatcher, http_request): """A performer for :obj:`HTTPRequest` that uses the ``treq`` library.""" headers = ( http_request.headers.copy() if http_request.headers is not None else {}) if 'user-agent' not in headers: headers['user-agent'] = ['Effect example'] d = treq.request( http_request.method.lower(), http_request.url, headers=headers, data=http_request.data).addCallback(treq.content) return d
def do_http(method, url, **kwargs): """ Run HTTP query, return Deferred of body as bytes. """ response = yield treq.request(method, url, persistent=False, **kwargs) body = yield treq.content(response) # TODO: replace this with response.fail_for_status when # https://github.com/twisted/treq/pull/159 has landed if 400 <= response.code < 600: raise VerboseError( response.code, response="For request {!r} to {!r}, got: {!r}".format( method, url, body)) returnValue(body)
def _request(self, method, url, **kwargs): if 'headers' in kwargs: headers = self.config.base_headers.copy() headers.update(kwargs['headers']) kwargs['headers'] = headers else: kwargs['headers'] = self.config.base_headers if self.config.api_key_secret: kwargs['auth'] = (self.config.api_key_secret, '') d = treq.request(method, url, **kwargs) d.addCallback(self.deserialize) return d
def proxy_devserver(request): def stream_response(response): request.setResponseCode(response.code) for key, values in response.headers.getAllRawHeaders(): for value in values: request.setHeader(key, value) d = treq.collect(response, request.write) d.addCallback(lambda _: request.finish()) return d url = ELM_DEVSERVER.encode('utf-8') + request.uri[len('/dev'):] d = treq.request(request.method.decode('ascii'), url) d.addCallback(stream_response) return d
def request(self, method, uri, data=b'', headers=None): """ Args: method (str): HTTP method to use. uri (str): URI to query. data (bytes): Data to send in the request body, if applicable. headers (t.w.http_headers.Headers): Request headers. Raises: SynapseError: If the IP is blacklisted. """ # A small wrapper around self.agent.request() so we can easily attach # counters to it outgoing_requests_counter.labels(method).inc() # log request but strip `access_token` (AS requests for example include this) logger.info("Sending request %s %s", method, redact_uri(uri)) try: request_deferred = treq.request( method, uri, agent=self.agent, data=data, headers=headers, **self._extra_treq_args ) request_deferred = timeout_deferred( request_deferred, 60, self.hs.get_reactor(), cancelled_to_request_timed_out_error, ) response = yield make_deferred_yieldable(request_deferred) incoming_responses_counter.labels(method, response.code).inc() logger.info( "Received response to %s %s: %s", method, redact_uri(uri), response.code ) defer.returnValue(response) except Exception as e: incoming_responses_counter.labels(method, "ERR").inc() logger.info( "Error sending request to %s %s: %s %s", method, redact_uri(uri), type(e).__name__, e.args[0], ) raise
def _request(method, url, **kwargs): def handle(req): if req.code != 200: req.content().addCallback(log.debug) raise HTTPError("Bad status code: {} for URL '{}'".format(req.code, url), req.code) else: return req.content() # set the User-Agent header if it isn't already set by the user if 'headers' not in kwargs: kwargs['headers'] = {} if 'User-Agent' not in kwargs['headers']: kwargs['headers']['User-Agent'] = 'exchangelib/{}'.format(VERSION) return treq.request(method, url, **kwargs).addCallback(handle)
def _request(self, service, method='GET', params=None): ''' Make the actual request ''' if not self.config: raise errors.CCBError('Internal Error: No CCB configuration') data = None if method == "POST" and params: data = params.copy() params = None if not params: params = {} params['srv'] = service auth = (self.config.username, self.config.password) d = treq.request(method, self.config.url, params=params, data=data, auth=auth) d.addCallback(self._get_content, params["srv"]) return d
def retry(req, **overrides): req['attempts'] = req['attempts'] + 1 d = req['request'] opts = { 'method': encode(d['method']), 'url': encode(d['url']), 'data': encode(d.get('body')), 'headers': dict([ (encode(k), [encode(v) for v in values]) for k, values in d.get('headers', {}).iteritems()]), } opts.update(overrides) return treq.request(**opts)
def request(self, request_type, method, payload={}): def _read_response(response): logger.info("Received Data: %s", response) if response.code != 200: raise DiscourseApiError("{}".format(response.phrase)) return response.content().addCallback(lambda x: pprint(x)) payload.update(dict(api_key=self.api_key, api_username=self.api_username)) dfr = treq.request(request_type, "{}{}".format(self.host, method), params=payload, headers={"Content-Type": "application/json"}) return dfr.addCallback(_read_response)
def test_cached_pool(self): """ The first use of the module-level API populates the global connection pool, which is used for all subsequent requests. """ pool = SyntacticAbominationHTTPConnectionPool() self.patch(treq.api, "HTTPConnectionPool", lambda reactor, persistent: pool) self.failureResultOf(treq.head("http://test.com"), TabError) self.failureResultOf(treq.get("http://test.com"), TabError) self.failureResultOf(treq.post("http://test.com"), TabError) self.failureResultOf(treq.put("http://test.com"), TabError) self.failureResultOf(treq.delete("http://test.com"), TabError) self.failureResultOf(treq.request("OPTIONS", "http://test.com"), TabError) self.assertEqual(pool.requests, 6)
def _call(self, method, url, data=None): url = urlparse.urljoin(self.endpoint, url).encode('utf-8') headers = {"Content-Type": "application/json"} if self.token: headers["Authorization"] = "Bearer %s" % self.token if data: data = json.dumps(data) # raises TypeError if self.debug: print "%s %s" % (method, url) print "Headers: %s" % str(headers) print "Data: %s" % data response = yield treq.request(method, url, headers=headers, data=data) code = response.code content = yield treq.content(response) if self.debug: print "Got: %s" % content try: def encode(data): encoded = {} for key, value in data.iteritems(): if isinstance(key, unicode): key = key.encode("utf-8") if isinstance(value, unicode): value = value.encode("utf-8") encoded[key] = value return encoded content = json.loads(content, object_hook=encode) except ValueError as e: pass if code == 200: returnValue(content) elif code == 400: raise BadRequest(content) elif code == 401: raise Unauthorized(content) elif code == 404: raise MethodNotFound(content) elif code == 406: raise NotAcceptable(content) else: raise BitGoException(content)
def _request(self, method, path, data=None): """Send HTTP request to gear. :param bytes method: The HTTP method to send, e.g. ``b"GET"``. :param bytes path: Path to request. :param data: ``None``, or object with a body for the request that can be serialized to JSON. :return: A ``Defered`` that fires with a response object. """ url = self._base_url + path if data is not None: data = json.dumps(data) return request(method, url, data=data, persistent=False)
def __request(self, url, data, content_type): headers = \ {"Content-Type": [content_type.encode("ascii", "ignore")], \ "X-Application": [self.app_key.encode("ascii", "ignore")], \ "X-Authentication": [self.session_token.encode("ascii", "ignore")]} data = data.encode("ascii", "ignore") self.logger.debug(url) self.logger.debug(headers) self.logger.debug(data) request = treq.request(method="POST", \ url=url.encode("ascii", "ignore"), headers=headers, data=data) request .addErrback(self.__error_handler) resp = yield request content = yield treq.content(resp) returnValue(content)
def send_request(): """ Send an HTTP request in a loop until the request is answered. """ response = request(b"GET", b"http://127.0.0.1:%d" % (port,), persistent=False) def check_error(failure): """ Catch ConnectionRefused errors and response timeouts and return False so that loop_until repeats the request. Other error conditions will be passed down the errback chain. """ failure.trap(ConnectionRefusedError, ResponseNeverReceived) return False response.addErrback(check_error) return response
def execute(self, method, path, body=None, params=None): """Execute a query against a server.""" headers = {b'Content-Type': [b'application/json']} if not isinstance(body, basestring): body = anyjson.serialize(body) for attempt in range(self.max_retries + 1): server = self.servers.get() timeout = self.servers.timeout url = _prepare_url(server, path, params) response = None # Stop UnboundLocalError on uncaught exception try: response = yield treq.request( method, url, data=body, pool=self.pool, auth=self.http_auth, persistent=self.persistent, timeout=timeout, headers=headers) json_data = yield response.json() exceptions.raise_exceptions(response.code, json_data) except Exception as e: retry = False if isinstance( e, ( ConnectError, ResponseFailed, RequestTransmissionFailed, ) ) or response and response.code in (503, 504): retry = True if retry: self.servers.mark_dead(server) if attempt == self.max_retries: raise continue else: raise else: defer.returnValue(json_data)
def _do_request(self, method, uri, *args, **kwargs): """ Modularized because API was broken. Need to be able to inject Mocked response objects here. """ method = method.upper() params = kwargs.pop('params', None) if params: uri += '?' + urlencode(params) headers = kwargs.pop('headers', None) if headers is not None: headers = {key: [value, ] for (key, value) in headers.iteritems()} headers = Headers(headers) data = kwargs.pop('data', None) response = yield treq.request(method, uri, headers=headers, data=data,) handled = yield self._handle_response(response) defer.returnValue(handled)