def test_deprecated_normalize_unicode(self):
# weblib.http.normalize_unicode is required by grab release
from weblib.http import normalize_unicode
self.assertEqual(
normalize_unicode(u'фыва'),
u'фыва'.encode('utf-8')
)
self.assertEqual(
normalize_unicode(u'фыва'.encode('utf-8')),
u'фыва'.encode('utf-8')
)
self.assertEqual(normalize_unicode(1), b'1')
def process_request_result(self, prepare_response_func=None):
"""
Process result of real request performed via transport extension.
"""
now = datetime.utcnow()
# TODO: move into separate method
if self.config['debug_post']:
post = self.config['post'] or self.config['multipart_post']
if isinstance(post, dict):
post = list(post.items())
if post:
if isinstance(post, six.string_types):
post = normalize_unicode(
post[:self.config['debug_post_limit']]) + b'...'
else:
items = normalize_http_values(
post, charset=self.config['charset'])
new_items = []
for key, value in items:
if len(value) > self.config['debug_post_limit']:
value = value[:self.
config['debug_post_limit']] + b'...'
else:
value = value
new_items.append((key, value))
post = '\n'.join('%-25s: %s' % x for x in new_items)
if post:
logger_network.debug('[%02d] POST request:\n%s\n',
self.request_counter, post)
# It's important to delete old POST data after request is performed.
# If POST data is not cleared then next request will try to use them
# again!
self.reset_temporary_options()
if prepare_response_func:
self.doc = prepare_response_func(self.transport, self)
else:
self.doc = self.transport.prepare_response(self)
# Workaround
if self.doc.grab is None:
self.doc.grab = weakref.proxy(self)
if self.config['reuse_cookies']:
self.cookies.update(self.doc.cookies)
self.doc.timestamp = now
self.config['charset'] = self.doc.charset
if self.config['log_file']:
with open(self.config['log_file'], 'wb') as out:
out.write(self.doc.body)
if self.config['cookiefile']:
self.cookies.save_to_file(self.config['cookiefile'])
if self.config['reuse_referer']:
self.config['referer'] = self.doc.url
self.copy_request_data()
# Should be called after `copy_request_data`
if self.config['log_dir']:
self.save_dumps()
return self.doc