async def fetch(self, url, method='GET', headers=None, body=None):
"""Perform a HTTP request and return decoded JSON data"""
request_headers = self.prepare_request_headers(headers)
url = self.proxy + url
if self.verbose:
self.print("\nRequest:", method, url, headers, body)
self.logger.debug("%s %s, Request: %s %s", method, url, headers, body)
request_body = body
encoded_body = body.encode() if body else None
self.open()
session_method = getattr(self.session, method.lower())
http_response = None
http_status_code = None
http_status_text = None
json_response = None
try:
async with session_method(yarl.URL(url, encoded=True),
data=encoded_body,
headers=request_headers,
timeout=(self.timeout / 1000),
proxy=self.aiohttp_proxy) as response:
http_response = await response.text()
# CIMultiDictProxy
raw_headers = response.headers
headers = {}
for header in raw_headers:
if header in headers:
headers[header] = headers[header] + ', ' + raw_headers[header]
else:
headers[header] = raw_headers[header]
http_status_code = response.status
http_status_text = response.reason
http_response = self.on_rest_response(http_status_code, http_status_text, url, method, headers, http_response, request_headers, request_body)
json_response = self.parse_json(http_response)
if self.enableLastHttpResponse:
self.last_http_response = http_response
if self.enableLastResponseHeaders:
self.last_response_headers = headers
if self.enableLastJsonResponse:
self.last_json_response = json_response
if self.verbose:
self.print("\nResponse:", method, url, http_status_code, headers, http_response)
self.logger.debug("%s %s, Response: %s %s %s", method, url, http_status_code, headers, http_response)
except socket.gaierror as e:
details = ' '.join([self.id, method, url])
raise ExchangeNotAvailable(details) from e
except (concurrent.futures.TimeoutError, asyncio.TimeoutError) as e:
details = ' '.join([self.id, method, url])
raise RequestTimeout(details) from e
except aiohttp.ClientConnectionError as e:
details = ' '.join([self.id, method, url])
raise ExchangeNotAvailable(details) from e
except aiohttp.ClientError as e: # base exception class
details = ' '.join([self.id, method, url])
raise ExchangeError(details) from e
self.handle_errors(http_status_code, http_status_text, url, method, headers, http_response, json_response, request_headers, request_body)
self.handle_http_status_code(http_status_code, http_status_text, url, method, http_response)
if json_response is not None:
return json_response
if self.is_text_response(headers):
return http_response
return response.content
def fetch_requests(self, url, method='GET', headers=None, body=None):
"""Perform a HTTP request and return decoded JSON data"""
headers = headers or {}
headers.update(self.headers)
if self.userAgent:
if type(self.userAgent) is str:
headers.update({'User-Agent': self.userAgent})
elif (type(self.userAgent) is dict) and ('User-Agent'
in self.userAgent):
headers.update(self.userAgent)
if self.proxy:
headers.update({'Origin': '*'})
headers.update({'Accept-Encoding': 'gzip, deflate'})
url = self.proxy + url
if self.verbose:
print(method, url, "\nRequest:", headers, body)
if body:
body = body.encode()
self.session.cookies.clear()
# do something with prepped.body
# prepped.body = 'Seriously, send exactly these bytes.'
# do something with prepped.headers
# prepped.headers['Keep-Dead'] = 'parrot'
response = None
try:
response = self.session.request(method,
url,
data=body,
headers=headers,
timeout=int(self.timeout / 1000),
proxies=self.proxies)
self.last_http_response = response.text
response.raise_for_status()
except Timeout as e:
raise RequestTimeout(' '.join(
[self.id, method, url, 'request timeout']))
except ConnectionError as e:
self.raise_error(ExchangeNotAvailable, url, method, e)
except TooManyRedirects as e:
self.raise_error(ExchangeError, url, method, e)
except HTTPError as e:
self.handle_errors(response.status_code, response.reason, url,
method, None, self.last_http_response)
self.handle_rest_errors(e, response.status_code,
self.last_http_response, url, method)
self.raise_error(ExchangeError, url, method, e,
self.last_http_response)
except RequestException as e:
self.raise_error(ExchangeError, url, method, e)
if self.verbose:
print(method, url, "\nResponse:", str(response.headers),
self.last_http_response)
return self.handle_rest_response(self.last_http_response, url, method,
headers, body)
async def fetch(self, url, method='GET', headers=None, body=None):
"""Perform a HTTP request and return decoded JSON data"""
request_headers = self.prepare_request_headers(headers)
url = self.proxy + url
if self.verbose:
print("\nRequest:", method, url, headers, body)
self.logger.debug("%s %s, Request: %s %s", method, url, headers, body)
request_body = body
encoded_body = body.encode() if body else None
self.open()
session_method = getattr(self.session, method.lower())
http_response = None
http_status_code = None
http_status_text = None
json_response = None
try:
async with session_method(yarl.URL(url, encoded=True),
data=encoded_body,
headers=request_headers,
timeout=(self.timeout / 1000),
proxy=self.aiohttp_proxy) as response:
http_response = await response.text()
http_status_code = response.status
http_status_text = response.reason
json_response = self.parse_json(http_response)
headers = response.headers
if self.enableLastHttpResponse:
self.last_http_response = http_response
if self.enableLastResponseHeaders:
self.last_response_headers = headers
if self.enableLastJsonResponse:
self.last_json_response = json_response
if self.verbose:
print("\nResponse:", method, url, http_status_code,
headers, http_response)
self.logger.debug("%s %s, Response: %s %s %s", method, url,
http_status_code, headers, http_response)
except socket.gaierror as e:
raise ExchangeNotAvailable(method + ' ' + url)
except concurrent.futures._base.TimeoutError as e:
raise RequestTimeout(method + ' ' + url)
except aiohttp.client_exceptions.ClientConnectionError as e:
raise ExchangeNotAvailable(method + ' ' + url)
except aiohttp.client_exceptions.ClientError as e: # base exception class
raise ExchangeError(method + ' ' + url)
self.handle_errors(http_status_code, http_status_text, url, method,
headers, http_response, json_response,
request_headers, request_body)
self.handle_rest_errors(http_status_code, http_status_text,
http_response, url, method)
self.handle_rest_response(http_response, json_response, url, method)
if json_response is not None:
return json_response
if self.is_text_response(headers):
return http_response
return response.content
def _websocket_pong_timeout(self, contextId):
ex = RequestTimeout(self.id +
' does not received pong message after 30 seconds')
self.emit('err', ex, contextId)