def test_to_dict_body(self):
request = Request()
request.body = Body()
request_dict = request.to_dict()
self.assertTrue(request_dict['body'])
request.body.close()
request = Request()
request.body = NotImplemented
request_dict = request.to_dict()
self.assertFalse(request_dict['body'])
response = Response()
response.body = Body()
response_dict = response.to_dict()
self.assertTrue(response_dict['body'])
response.body.close()
response = Response()
response.body = NotImplemented
response_dict = response.to_dict()
self.assertFalse(response_dict['body'])
def test_to_dict_body(self):
request = Request()
request.body = Body()
request_dict = request.to_dict()
self.assertTrue(request_dict['body'])
request.body.close()
request = Request()
request.body = NotImplemented
request_dict = request.to_dict()
self.assertFalse(request_dict['body'])
response = Response()
response.body = Body()
response_dict = response.to_dict()
self.assertTrue(response_dict['body'])
response.body.close()
response = Response()
response.body = NotImplemented
response_dict = response.to_dict()
self.assertFalse(response_dict['body'])
def _add_post_data(self, request: Request):
'''Add data to the payload.'''
if self._item_session.url_record.post_data:
data = wpull.string.to_bytes(self._item_session.url_record.post_data)
else:
data = wpull.string.to_bytes(
self._processor.fetch_params.post_data
)
request.method = 'POST'
request.fields['Content-Type'] = 'application/x-www-form-urlencoded'
request.fields['Content-Length'] = str(len(data))
_logger.debug('Posting with data {0}.', data)
if not request.body:
request.body = Body(io.BytesIO())
with wpull.util.reset_file_offset(request.body):
request.body.write(data)
def _add_post_data(self, request: Request):
'''Add data to the payload.'''
if self._item_session.url_record.post_data:
data = wpull.string.to_bytes(self._item_session.url_record.post_data)
else:
data = wpull.string.to_bytes(
self._processor.fetch_params.post_data
)
request.method = 'POST'
request.fields['Content-Type'] = 'application/x-www-form-urlencoded'
request.fields['Content-Length'] = str(len(data))
_logger.debug('Posting with data {0}.', data)
if not request.body:
request.body = Body(io.BytesIO())
with wpull.util.reset_file_offset(request.body):
request.body.write(data)
def _process_request(self, request: Request):
_logger.debug(__('Got request {0}', request))
if request.method == 'CONNECT':
self._reject_request('CONNECT is intentionally not supported')
return
if self._is_ssl_tunnel and request.url.startswith('http://'):
# Since we are spying under a SSL tunnel, assume processed requests
# are SSL
request.url = request.url.replace('http://', 'https://', 1)
if 'Upgrade' in request.fields.get('Connection', ''):
_logger.warning(__(
_('Connection Upgrade not supported for {}'),
request.url
))
self._reject_request('Upgrade not supported')
return
_logger.debug('Begin response.')
try:
action = self.hook_dispatcher.call(self.Event.client_request, request)
except HookDisconnected:
pass
else:
if not action:
_logger.debug('Proxy force reject request')
self._reject_request()
return
with self._http_client.session() as session:
if 'Content-Length' in request.fields:
request.body = self._reader
try:
response = yield from session.start(request)
except NetworkError as error:
_logger.debug('Upstream error', exc_info=True)
self._write_error_response()
self.event_dispatcher.notify(self.Event.server_response_error, error)
return
response.body = Body()
try:
action = self.hook_dispatcher.call(self.Event.server_begin_response, response)
except HookDisconnected:
pass
else:
if not action:
_logger.debug('Proxy force reject request via response')
self._reject_request()
return
try:
self._writer.write(response.to_bytes())
yield from self._writer.drain()
session.event_dispatcher.add_listener(
Session.Event.response_data,
self._writer.write
)
yield from session.download(file=response.body, raw=True)
yield from self._writer.drain()
except NetworkError as error:
_logger.debug('Upstream error', exc_info=True)
self.event_dispatcher.notify(self.Event.server_response_error, error)
raise
self.event_dispatcher.notify(self.Event.server_end_response, response)
_logger.debug('Response done.')
def _process_request(self, request: Request):
_logger.debug(__('Got request {0}', request))
if request.method == 'CONNECT':
yield from self._start_connect_tunnel()
return
if self._is_ssl_tunnel and request.url.startswith('http://'):
# Since we are spying under a SSL tunnel, assume processed requests
# are SSL
request.url = request.url.replace('http://', 'https://', 1)
if 'Upgrade' in request.fields.get('Connection', ''):
_logger.warning(__(
_('Connection Upgrade not supported for {}'),
request.url
))
self._reject_request('Upgrade not supported')
return
_logger.debug('Begin response.')
try:
action = self.hook_dispatcher.call(self.Event.client_request, request)
except HookDisconnected:
pass
else:
if not action:
_logger.debug('Proxy force reject request')
self._reject_request()
return
with self._http_client.session() as session:
if 'Content-Length' in request.fields:
request.body = self._reader
try:
response = yield from session.start(request)
except NetworkError as error:
_logger.debug('Upstream error', exc_info=True)
self._write_error_response()
self.event_dispatcher.notify(self.Event.server_response_error, error)
return
response.body = Body()
try:
action = self.hook_dispatcher.call(self.Event.server_begin_response, response)
except HookDisconnected:
pass
else:
if not action:
_logger.debug('Proxy force reject request via response')
self._reject_request()
return
try:
self._writer.write(response.to_bytes())
yield from self._writer.drain()
session.event_dispatcher.add_listener(
Session.Event.response_data,
self._writer.write
)
yield from session.download(file=response.body, raw=True)
yield from self._writer.drain()
except NetworkError as error:
_logger.debug('Upstream error', exc_info=True)
self.event_dispatcher.notify(self.Event.server_response_error, error)
raise
self.event_dispatcher.notify(self.Event.server_end_response, response)
_logger.debug('Response done.')