def createThreads(self, cycle, number_of_threads): """Creates number_of_threads threads and returns as a list. NOTE: This method is not thread safe. Thread safety must be handled by the caller.""" threads = [] i = 0 for i in range(number_of_threads): thread_id = self.createThreadId() thread_signaller = ThreadSignaller() thread = LoopTestRunner(self.module_name, self.class_name, self.method_name, self.options, cycle, number_of_threads, thread_id, thread_signaller, self.sleep_time) trace(".") try: thread.start() except ThreadError: trace("\nERROR: Can not create more than %i threads, try a " "smaller stack size using: 'ulimit -s 2048' " "for example\n" % (i + 1)) raise thread_data = ThreadData(thread, thread_id, thread_signaller) threads.append(thread_data) thread_sleep(self.startup_delay) trace(' done.\n') return threads
def do_link(self, attributes): """Process link tag.""" newattributes = [('rel', 'stylesheet'), ('type', 'text/css')] for name, value in attributes: if name == 'href': # construct full url url = urlparse.urljoin(self.base, value) # make sure it's syntactically valid if not valid_url.match(url): continue # TODO: figure the re-write path # newattributes.append((name, path)) if not self.session.css.has_key(url): self.ftestcase.logdd(' link: %s ...' % url) t_start = time.time() self.session.css[url] = self.session.fetch(url) t_stop = time.time() self.ftestcase.logdd(' Done in %.3fs' % (t_stop - t_start)) self.session.history.append(('link', url)) self.ftestcase.total_time += (t_stop - t_start) self.ftestcase.total_links += 1 self.ftestcase._log_response(self.session.css[url], 'link', None, t_start, t_stop) thread_sleep() # give a chance to other threads else: newattributes.append((name, value)) # Write the link tag to file (with revised paths) self.unknown_starttag('link', newattributes)
def run(self): """Run a test in loop.""" while (self.thread_signaller.running()): test_result = unittest.TestResult() self.test.clearContext() self.test(test_result) if test_result.wasSuccessful(): if recording(): add_cycle_result('success') if self.color: trace(green_str('.')) else: trace('.') else: if len(test_result.errors): if recording(): add_cycle_result('error') if self.color: trace(red_str('E')) else: trace('E') else: if recording(): add_cycle_result('failure') if self.color: trace(red_str('F')) else: trace('F') if self.debug: for (test, error) in test_result.errors: trace("ERROR %s: %s" % (str(test), str(error))) for (test, error) in test_result.failures: trace("FAILURE %s: %s" % (str(test), str(error))) thread_sleep(self.sleep_time)
def do_link(self, attributes): """Process link tag.""" newattributes = [('rel', 'stylesheet'), ('type', 'text/css')] for name, value in attributes: if name == 'href': # construct full url url = urlparse.urljoin(self.base, value) # make sure it's syntactically valid if not valid_url.match(url): continue # TODO: figure the re-write path # newattributes.append((name, path)) if not self.session.css.has_key(url): self.ftestcase.logdd(' link: %s ...' % url) with self.ftestcase.record_response('link', url, None): try: self.session.css[url] = self.session.fetch(url) self.session.history.append(('link', url)) except HTTPError as error: if self.ftestcase._accept_invalid_links: if not self.ftestcase.in_bench_mode: self.ftestcase.logd(' ' + str(error)) else: raise thread_sleep() # give a chance to other threads else: newattributes.append((name, value)) # Write the link tag to file (with revised paths) self.unknown_starttag('link', newattributes)
def do_img(self, attributes): """Process img tag.""" newattributes = [] for name, value in attributes: if name == "src": # construct full url url = urlparse.urljoin(self.base, value) # make sure it's syntactically valid if not valid_url.match(url): continue # TODO: figure the re-write path # newattributes.append((name, path)) if not self.session.images.has_key(url): self.ftestcase.logdd(" img: %s ..." % url) t_start = time.time() self.session.images[url] = self.session.fetch(url) t_stop = time.time() self.ftestcase.logdd(" Done in %.3fs" % (t_stop - t_start)) self.session.history.append(("image", url)) self.ftestcase.total_time += t_stop - t_start self.ftestcase.total_images += 1 self.ftestcase._log_response(self.session.images[url], "image", None, t_start, t_stop) thread_sleep() # give a chance to other threads else: newattributes.append((name, value)) # Write the img tag to file (with revised paths) self.unknown_starttag("img", newattributes)
def run(self): """Run a test in loop.""" while (self.thread_signaller.running()): test_result = unittest.TestResult() self.test.clearContext() self.test(test_result) feedback = {} if test_result.wasSuccessful(): if recording(): feedback['count'] = add_cycle_result('success') if self.color: trace(green_str('.')) else: trace('.') feedback['result'] = 'success' else: if len(test_result.errors): if recording(): feedback['count'] = add_cycle_result('error') if self.color: trace(red_str('E')) else: trace('E') feedback['result'] = 'error' else: if recording(): feedback['count'] = add_cycle_result('failure') if self.color: trace(red_str('F')) else: trace('F') feedback['result'] = 'failure' if self.debug: feedback['errors'] = test_result.errors feedback['failures'] = test_result.failures for (test, error) in test_result.errors: trace("ERROR %s: %s" % (str(test), str(error))) for (test, error) in test_result.failures: trace("FAILURE %s: %s" % (str(test), str(error))) if self.feedback is not None: self.feedback.test_done(feedback) thread_sleep(self.sleep_time)
def run(self): """Run a test in loop.""" while self.thread_signaller.running(): test_result = unittest.TestResult() self.test.clearContext() self.test(test_result) feedback = {} if test_result.wasSuccessful(): if recording(): feedback["count"] = add_cycle_result("success") if self.color: trace(green_str(".")) else: trace(".") feedback["result"] = "success" else: if len(test_result.errors): if recording(): feedback["count"] = add_cycle_result("error") if self.color: trace(red_str("E")) else: trace("E") feedback["result"] = "error" else: if recording(): feedback["count"] = add_cycle_result("failure") if self.color: trace(red_str("F")) else: trace("F") feedback["result"] = "failure" if self.debug: feedback["errors"] = test_result.errors feedback["failures"] = test_result.failures for (test, error) in test_result.errors: trace("ERROR %s: %s" % (str(test), str(error))) for (test, error) in test_result.failures: trace("FAILURE %s: %s" % (str(test), str(error))) if self.feedback is not None: self.feedback.test_done(feedback) thread_sleep(self.sleep_time)
def sleep(self): """Sleeps a random amount of time. Between the predefined sleep_time_min and sleep_time_max values. """ if self._pause: raw_input("Press ENTER to continue ") return s_min = self.sleep_time_min s_max = self.sleep_time_max if s_max != s_min: s_val = s_min + abs(s_max - s_min) * random() else: s_val = s_min # we should always sleep something thread_sleep(s_val)
def _browse(self, url_in, params_in=None, description=None, ok_codes=None, method='post', follow_redirect=True, load_auto_links=True, sleep=True): """Simulate a browser handle redirects, load/cache css and images.""" self._response = None # Loop mode if self._loop_mode: if self.steps == self._loop_steps[0]: self._loop_recording = True self.logi('Loop mode start recording') if self._loop_recording: self._loop_records.append((url_in, params_in, description, ok_codes, method, follow_redirect, load_auto_links, False)) # ok codes if ok_codes is None: ok_codes = self.ok_codes if type(params_in) is DictType: params_in = params_in.items() params = [] if params_in: if isinstance(params_in, Data): params = params_in else: for key, value in params_in: if type(value) is DictType: for val, selected in value.items(): if selected: params.append((key, val)) elif type(value) in (ListType, TupleType): for val in value: params.append((key, val)) else: params.append((key, value)) if method == 'get' and params: url = url_in + '?' + urlencode(params) else: url = url_in if method == 'get': params = None if method == 'get': if not self.in_bench_mode: self.logd('GET: %s\n\tPage %i: %s ...' % (url, self.steps, description or '')) else: url = url_in if not self.in_bench_mode: self.logd('%s: %s %s\n\tPage %i: %s ...' % ( method.upper(), url, str(params), self.steps, description or '')) # Fetching response = self._connect(url, params, ok_codes, method, description) # Check redirection if follow_redirect and response.code in (301, 302, 303, 307): max_redirect_count = 10 thread_sleep() # give a chance to other threads while response.code in (301, 302, 303, 307) and max_redirect_count: # Figure the location - which may be relative newurl = response.headers['Location'] url = urljoin(url_in, newurl) # Save the current url as the base for future redirects url_in = url self.logd(' Load redirect link: %s' % url) # Use the appropriate method for redirection if response.code in (302, 303): method = 'get' if response.code == 303: # 303 is HTTP/1.1, make sure the connection # is not in keep alive mode self.setHeader('Connection', 'close') response = self._connect(url, None, ok_codes, rtype=method, description=None, redirect=True) max_redirect_count -= 1 if not max_redirect_count: self.logd(' WARNING Too many redirects give up.') # Load auto links (css and images) response.is_html = is_html(response.body) if load_auto_links and response.is_html and not self._simple_fetch: self.logd(' Load css and images...') page = response.body t_start = time.time() c_start = self.total_time try: # pageImages is patched to call _log_response on all links self._browser.pageImages(url, page, self) except HTTPError, error: if self._accept_invalid_links: if not self.in_bench_mode: self.logd(' ' + str(error)) else: t_stop = time.time() t_delta = t_stop - t_start self.step_success = False self.test_status = 'Failure' self.logd(' Failed in ~ %.2fs' % t_delta) # XXX The duration logged for this response is wrong self._log_response(error.response, 'link', None, t_start, t_stop, log_body=True) raise self.failureException, str(error) c_stop = self.total_time self.logd(' Done in %.3fs' % (c_stop - c_start))
def _browse(self, url_in, params_in=None, description=None, ok_codes=None, method='post', follow_redirect=True, load_auto_links=True, sleep=True): """Simulate a browser handle redirects, load/cache css and images.""" self._response = None # Loop mode if self._loop_mode: if self.steps == self._loop_steps[0]: self._loop_recording = True self.logi('Loop mode start recording') if self._loop_recording: self._loop_records.append( (url_in, params_in, description, ok_codes, method, follow_redirect, load_auto_links, False)) # ok codes if ok_codes is None: ok_codes = self.ok_codes if type(params_in) is DictType: params_in = params_in.items() params = [] if params_in: if isinstance(params_in, Data): params = params_in else: for key, value in params_in: if type(value) is DictType: for val, selected in value.items(): if selected: params.append((key, val)) elif type(value) in (ListType, TupleType): for val in value: params.append((key, val)) else: params.append((key, value)) if method == 'get' and params: url = url_in + '?' + urlencode(params) else: url = url_in if method == 'get': params = None if method == 'get': if not self.in_bench_mode: self.logd('GET: %s\n\tPage %i: %s ...' % (url, self.steps, description or '')) else: url = url_in if not self.in_bench_mode: self.logd('%s: %s %s\n\tPage %i: %s ...' % (method.upper(), url, str(params), self.steps, description or '')) # Fetching response = self._connect(url, params, ok_codes, method, description) # Check redirection if follow_redirect and response.code in (301, 302, 303, 307): max_redirect_count = 10 thread_sleep() # give a chance to other threads while response.code in (301, 302, 303, 307) and max_redirect_count: # Figure the location - which may be relative newurl = response.headers['Location'] url = urljoin(url_in, newurl) # Save the current url as the base for future redirects url_in = url self.logd(' Load redirect link: %s' % url) # Use the appropriate method for redirection if response.code in (302, 303): method = 'get' if response.code == 303: # 303 is HTTP/1.1, make sure the connection # is not in keep alive mode self.setHeader('Connection', 'close') response = self._connect(url, None, ok_codes, rtype=method, description=None, redirect=True) max_redirect_count -= 1 if not max_redirect_count: self.logd(' WARNING Too many redirects give up.') # Load auto links (css and images) response.is_html = is_html(response.body) if load_auto_links and response.is_html and not self._simple_fetch: self.logd(' Load css and images...') page = response.body t_start = time.time() c_start = self.total_time try: # pageImages is patched to call _log_response on all links self._browser.pageImages(url, page, self) except HTTPError, error: if self._accept_invalid_links: if not self.in_bench_mode: self.logd(' ' + str(error)) else: t_stop = time.time() t_delta = t_stop - t_start self.step_success = False self.test_status = 'Failure' self.logd(' Failed in ~ %.2fs' % t_delta) # XXX The duration logged for this response is wrong self._log_response(error.response, 'link', None, t_start, t_stop, log_body=True) raise self.failureException, str(error) c_stop = self.total_time self.logd(' Done in %.3fs' % (c_stop - c_start))
def _browse(self, url_in, params_in=None, description=None, ok_codes=None, method='post', follow_redirect=True, load_auto_links=True, sleep=True): """Simulate a browser handle redirects, load/cache css and images.""" self._response = None # Loop mode if self._loop_mode: if self.steps == self._loop_steps[0]: self._loop_recording = True self.logi('Loop mode start recording') if self._loop_recording: self._loop_records.append((url_in, params_in, description, ok_codes, method, follow_redirect, load_auto_links, False)) # ok codes if ok_codes is None: ok_codes = self.ok_codes if type(params_in) is DictType: params_in = params_in.items() params = [] if params_in: if isinstance(params_in, Data): params = params_in else: for key, value in params_in: if type(value) is DictType: for val, selected in value.items(): if selected: params.append((key, val)) elif type(value) in (ListType, TupleType): for val in value: params.append((key, val)) else: params.append((key, value)) if method == 'get' and params: url = url_in + '?' + urlencode(params) else: url = url_in if method == 'get': params = None if method == 'get': if not self.in_bench_mode: self.logd('GET: %s\n\tPage %i: %s ...' % (url, self.steps, description or '')) else: url = url_in if not self.in_bench_mode: self.logd('%s: %s %s\n\tPage %i: %s ...' % ( method.upper(), url, str(params), self.steps, description or '')) # Fetching response = self._connect(url, params, ok_codes, method, description) # Check redirection if follow_redirect and response.code in (301, 302, 303, 307): max_redirect_count = 10 thread_sleep() # give a chance to other threads while response.code in (301, 302, 303, 307) and max_redirect_count: # Figure the location - which may be relative newurl = response.headers['Location'] url = urljoin(url_in, newurl) # Save the current url as the base for future redirects url_in = url self.logd(' Load redirect link: %s' % url) # Use the appropriate method for redirection if response.code in (302, 303): method = 'get' if response.code == 303: # 303 is HTTP/1.1, make sure the connection # is not in keep alive mode self.setHeader('Connection', 'close') response = self._connect(url, None, ok_codes, rtype=method, description=None, redirect=True) max_redirect_count -= 1 if not max_redirect_count: self.logd(' WARNING Too many redirects give up.') # Load auto links (css and images) response.is_html = is_html(response.body) if load_auto_links and response.is_html and not self._simple_fetch: self.logd(' Load css and images...') page = response.body # pageImages is patched to call self.record on all links self._browser.pageImages(url, page, self) if sleep: self.sleep() self._response = response # Loop mode if self._loop_mode and self.steps == self._loop_steps[-1]: self._loop_recording = False self.logi('Loop mode end recording.') t_start = self.total_time count = 0 for i in range(self._loop_number): self.logi('Loop mode replay %i' % i) for record in self._loop_records: count += 1 self.steps += 1 self._browse(*record) t_delta = self.total_time - t_start text = ('End of loop: %d pages rendered in %.3fs, ' 'avg of %.3fs per page, ' '%.3f SPPS without concurrency.' % (count, t_delta, t_delta / count, count/t_delta)) self.logi(text) trace(text + '\n') return response