def _browse(self,
url_in,
params_in=None,
description=None,
ok_codes=None,
method='post',
follow_redirect=True,
load_auto_links=True,
sleep=True):
"""Simulate a browser handle redirects, load/cache css and images."""
self._response = None
# Loop mode
if self._loop_mode:
if self.steps == self._loop_steps[0]:
self._loop_recording = True
self.logi('Loop mode start recording')
if self._loop_recording:
self._loop_records.append(
(url_in, params_in, description, ok_codes, method,
follow_redirect, load_auto_links, False))
# ok codes
if ok_codes is None:
ok_codes = self.ok_codes
if type(params_in) is DictType:
params_in = params_in.items()
params = []
if params_in:
if isinstance(params_in, Data):
params = params_in
else:
for key, value in params_in:
if type(value) is DictType:
for val, selected in value.items():
if selected:
params.append((key, val))
elif type(value) in (ListType, TupleType):
for val in value:
params.append((key, val))
else:
params.append((key, value))
if method == 'get' and params:
url = url_in + '?' + urlencode(params)
else:
url = url_in
if method == 'get':
params = None
if method == 'get':
if not self.in_bench_mode:
self.logd('GET: %s\n\tPage %i: %s ...' %
(url, self.steps, description or ''))
else:
url = url_in
if not self.in_bench_mode:
self.logd('%s: %s %s\n\tPage %i: %s ...' %
(method.upper(), url, str(params), self.steps,
description or ''))
# Fetching
response = self._connect(url, params, ok_codes, method, description)
# Check redirection
if follow_redirect and response.code in (301, 302, 303, 307):
max_redirect_count = 10
thread_sleep() # give a chance to other threads
while response.code in (301, 302, 303, 307) and max_redirect_count:
# Figure the location - which may be relative
newurl = response.headers['Location']
url = urljoin(url_in, newurl)
# Save the current url as the base for future redirects
url_in = url
self.logd(' Load redirect link: %s' % url)
# Use the appropriate method for redirection
if response.code in (302, 303):
method = 'get'
if response.code == 303:
# 303 is HTTP/1.1, make sure the connection
# is not in keep alive mode
self.setHeader('Connection', 'close')
response = self._connect(url,
None,
ok_codes,
rtype=method,
description=None,
redirect=True)
max_redirect_count -= 1
if not max_redirect_count:
self.logd(' WARNING Too many redirects give up.')
# Load auto links (css and images)
response.is_html = is_html(response.body)
if load_auto_links and response.is_html and not self._simple_fetch:
self.logd(' Load css and images...')
page = response.body
t_start = time.time()
c_start = self.total_time
try:
# pageImages is patched to call _log_response on all links
self._browser.pageImages(url, page, self)
except HTTPError, error:
if self._accept_invalid_links:
if not self.in_bench_mode:
self.logd(' ' + str(error))
else:
t_stop = time.time()
t_delta = t_stop - t_start
self.step_success = False
self.test_status = 'Failure'
self.logd(' Failed in ~ %.2fs' % t_delta)
# XXX The duration logged for this response is wrong
self._log_response(error.response,
'link',
None,
t_start,
t_stop,
log_body=True)
raise self.failureException, str(error)
c_stop = self.total_time
self.logd(' Done in %.3fs' % (c_stop - c_start))
def _browse(self, url_in, params_in=None,
description=None, ok_codes=None,
method='post',
follow_redirect=True, load_auto_links=True,
sleep=True):
"""Simulate a browser handle redirects, load/cache css and images."""
self._response = None
# Loop mode
if self._loop_mode:
if self.steps == self._loop_steps[0]:
self._loop_recording = True
self.logi('Loop mode start recording')
if self._loop_recording:
self._loop_records.append((url_in, params_in, description,
ok_codes, method, follow_redirect,
load_auto_links, False))
# ok codes
if ok_codes is None:
ok_codes = self.ok_codes
if type(params_in) is DictType:
params_in = params_in.items()
params = []
if params_in:
if isinstance(params_in, Data):
params = params_in
else:
for key, value in params_in:
if type(value) is DictType:
for val, selected in value.items():
if selected:
params.append((key, val))
elif type(value) in (ListType, TupleType):
for val in value:
params.append((key, val))
else:
params.append((key, value))
if method == 'get' and params:
url = url_in + '?' + urlencode(params)
else:
url = url_in
if method == 'get':
params = None
if method == 'get':
if not self.in_bench_mode:
self.logd('GET: %s\n\tPage %i: %s ...' % (url, self.steps,
description or ''))
else:
url = url_in
if not self.in_bench_mode:
self.logd('%s: %s %s\n\tPage %i: %s ...' % (
method.upper(), url, str(params),
self.steps, description or ''))
# Fetching
response = self._connect(url, params, ok_codes, method, description)
# Check redirection
if follow_redirect and response.code in (301, 302, 303, 307):
max_redirect_count = 10
thread_sleep() # give a chance to other threads
while response.code in (301, 302, 303, 307) and max_redirect_count:
# Figure the location - which may be relative
newurl = response.headers['Location']
url = urljoin(url_in, newurl)
# Save the current url as the base for future redirects
url_in = url
self.logd(' Load redirect link: %s' % url)
# Use the appropriate method for redirection
if response.code in (302, 303):
method = 'get'
if response.code == 303:
# 303 is HTTP/1.1, make sure the connection
# is not in keep alive mode
self.setHeader('Connection', 'close')
response = self._connect(url, None, ok_codes, rtype=method,
description=None, redirect=True)
max_redirect_count -= 1
if not max_redirect_count:
self.logd(' WARNING Too many redirects give up.')
# Load auto links (css and images)
response.is_html = is_html(response.body)
if load_auto_links and response.is_html and not self._simple_fetch:
self.logd(' Load css and images...')
page = response.body
t_start = time.time()
c_start = self.total_time
try:
# pageImages is patched to call _log_response on all links
self._browser.pageImages(url, page, self)
except HTTPError, error:
if self._accept_invalid_links:
if not self.in_bench_mode:
self.logd(' ' + str(error))
else:
t_stop = time.time()
t_delta = t_stop - t_start
self.step_success = False
self.test_status = 'Failure'
self.logd(' Failed in ~ %.2fs' % t_delta)
# XXX The duration logged for this response is wrong
self._log_response(error.response, 'link', None,
t_start, t_stop, log_body=True)
raise self.failureException, str(error)
c_stop = self.total_time
self.logd(' Done in %.3fs' % (c_stop - c_start))
def _browse(self, url_in, params_in=None,
description=None, ok_codes=None,
method='post',
follow_redirect=True, load_auto_links=True,
sleep=True):
"""Simulate a browser handle redirects, load/cache css and images."""
self._response = None
# Loop mode
if self._loop_mode:
if self.steps == self._loop_steps[0]:
self._loop_recording = True
self.logi('Loop mode start recording')
if self._loop_recording:
self._loop_records.append((url_in, params_in, description,
ok_codes, method, follow_redirect,
load_auto_links, False))
# ok codes
if ok_codes is None:
ok_codes = self.ok_codes
if type(params_in) is DictType:
params_in = params_in.items()
params = []
if params_in:
if isinstance(params_in, Data):
params = params_in
else:
for key, value in params_in:
if type(value) is DictType:
for val, selected in value.items():
if selected:
params.append((key, val))
elif type(value) in (ListType, TupleType):
for val in value:
params.append((key, val))
else:
params.append((key, value))
if method == 'get' and params:
url = url_in + '?' + urlencode(params)
else:
url = url_in
if method == 'get':
params = None
if method == 'get':
if not self.in_bench_mode:
self.logd('GET: %s\n\tPage %i: %s ...' % (url, self.steps,
description or ''))
else:
url = url_in
if not self.in_bench_mode:
self.logd('%s: %s %s\n\tPage %i: %s ...' % (
method.upper(), url, str(params),
self.steps, description or ''))
# Fetching
response = self._connect(url, params, ok_codes, method, description)
# Check redirection
if follow_redirect and response.code in (301, 302, 303, 307):
max_redirect_count = 10
thread_sleep() # give a chance to other threads
while response.code in (301, 302, 303, 307) and max_redirect_count:
# Figure the location - which may be relative
newurl = response.headers['Location']
url = urljoin(url_in, newurl)
# Save the current url as the base for future redirects
url_in = url
self.logd(' Load redirect link: %s' % url)
# Use the appropriate method for redirection
if response.code in (302, 303):
method = 'get'
if response.code == 303:
# 303 is HTTP/1.1, make sure the connection
# is not in keep alive mode
self.setHeader('Connection', 'close')
response = self._connect(url, None, ok_codes, rtype=method,
description=None, redirect=True)
max_redirect_count -= 1
if not max_redirect_count:
self.logd(' WARNING Too many redirects give up.')
# Load auto links (css and images)
response.is_html = is_html(response.body)
if load_auto_links and response.is_html and not self._simple_fetch:
self.logd(' Load css and images...')
page = response.body
# pageImages is patched to call self.record on all links
self._browser.pageImages(url, page, self)
if sleep:
self.sleep()
self._response = response
# Loop mode
if self._loop_mode and self.steps == self._loop_steps[-1]:
self._loop_recording = False
self.logi('Loop mode end recording.')
t_start = self.total_time
count = 0
for i in range(self._loop_number):
self.logi('Loop mode replay %i' % i)
for record in self._loop_records:
count += 1
self.steps += 1
self._browse(*record)
t_delta = self.total_time - t_start
text = ('End of loop: %d pages rendered in %.3fs, '
'avg of %.3fs per page, '
'%.3f SPPS without concurrency.' % (count, t_delta,
t_delta / count,
count/t_delta))
self.logi(text)
trace(text + '\n')
return response