def request_direct(self,
method,
url,
timeout=None,
**kwargs) -> requests.Response:
if timeout is None:
timeout = _DEFAULT_TIMEOUT
response = requests.request(method, url, **kwargs, timeout=timeout)
response.close()
requests_check_incomplete_response(response)
return response
def ask(self, message: ActorMessage):
self._init()
message, dst_url, headers, data = self._get_ask_request(message)
with self._sentry_message_scope(message):
try:
r = self.session.post(dst_url,
data=data,
headers=headers,
timeout=self.timeout)
requests_check_incomplete_response(r)
except requests.RequestException as ex:
LOG.warning(f'failed to send message to {dst_url}: {ex}')
raise
r.close()
r.raise_for_status()
return self._decode_ask_response(r.content, r.headers)
def _group_send(self, dst_url, messages):
with self._sentry_group_message_scope(dst_url):
LOG.info(f'send {len(messages)} messages to {dst_url}')
data = ActorMessage.batch_encode(messages, self.content_encoding)
try:
r = self.session.post(dst_url,
data=data,
headers=self.headers,
timeout=self.timeout)
requests_check_incomplete_response(r)
except requests.ConnectionError as ex:
LOG.warning(f'failed to send message to {dst_url}: {ex}')
return
except requests.RequestException as ex:
LOG.warning(f'failed to send message to {dst_url}: {ex}')
raise
r.close()
r.raise_for_status()
def _read_content(self, response: requests.Response):
content_length = response.headers.get('Content-Length')
if content_length:
content_length = int(content_length)
if content_length > self.max_content_length:
msg = 'content length {} larger than limit {}'.format(
content_length, self.max_content_length)
raise ContentTooLargeError(msg)
content_length = 0
content = bytearray()
for data in response.iter_content(chunk_size=64 * 1024):
content_length += len(data)
if content_length > self.max_content_length:
msg = 'content length larger than limit {}'.format(
self.max_content_length)
raise ContentTooLargeError(msg)
content.extend(data)
requests_check_incomplete_response(response)
return content
def _request_by_rss_proxy(self,
method,
url,
timeout=None,
**kwargs) -> requests.Response:
if timeout is None:
timeout = _DEFAULT_TIMEOUT
request: requests.PreparedRequest
request = requests.Request(method, url, **kwargs).prepare()
request_body = None
if request.body:
if isinstance(request.body, bytes):
request_body = request.body.decode('utf-8')
elif isinstance(request.body, str):
request_body = request.body
else:
msg = f'not support request body type {type(request.body).__name__}'
raise ValueError(msg)
headers = dict(request.headers)
headers['user-agent'] = _DEFAULT_USER_AGENT
proxy_data = {
'method': request.method,
'token': self.rss_proxy_token,
'url': request.url,
'headers': headers,
'body': request_body,
}
response = requests.post(self.rss_proxy_url,
json=proxy_data,
timeout=timeout)
response.close()
requests_check_incomplete_response(response)
proxy_status = response.headers.get('x-rss-proxy-status', None)
if response.status_code != 200 or proxy_status == 'ERROR':
msg = 'status={} {}'.format(response.status_code, response.text)
raise RSSProxyClientError(msg)
response.status_code = int(proxy_status)
return response