Exemplo n.º 1
0
    def createThreads(self, cycle, number_of_threads):
        """Creates number_of_threads threads and returns as a list.

        NOTE: This method is not thread safe. Thread safety must be
        handled by the caller."""
        threads = []
        i = 0
        for i in range(number_of_threads):
            thread_id = self.createThreadId()
            thread_signaller = ThreadSignaller()
            thread = LoopTestRunner(self.module_name, self.class_name,
                                    self.method_name, self.options,
                                    cycle, number_of_threads,
                                    thread_id, thread_signaller,
                                    self.sleep_time)
            trace(".")
            try:
                thread.start()
            except ThreadError:
                trace("\nERROR: Can not create more than %i threads, try a "
                      "smaller stack size using: 'ulimit -s 2048' "
                      "for example\n" % (i + 1))
                raise
            thread_data = ThreadData(thread, thread_id, thread_signaller)
            threads.append(thread_data)
            thread_sleep(self.startup_delay)
        trace(' done.\n')
        return threads
Exemplo n.º 2
0
 def do_link(self, attributes):
     """Process link tag."""
     newattributes = [('rel', 'stylesheet'), ('type', 'text/css')]
     for name, value in attributes:
         if name == 'href':
             # construct full url
             url = urlparse.urljoin(self.base, value)
             # make sure it's syntactically valid
             if not valid_url.match(url):
                 continue
             # TODO: figure the re-write path
             # newattributes.append((name, path))
             if not self.session.css.has_key(url):
                 self.ftestcase.logdd('    link: %s ...' % url)
                 t_start = time.time()
                 self.session.css[url] = self.session.fetch(url)
                 t_stop = time.time()
                 self.ftestcase.logdd('     Done in %.3fs' %
                                      (t_stop - t_start))
                 self.session.history.append(('link', url))
                 self.ftestcase.total_time += (t_stop - t_start)
                 self.ftestcase.total_links += 1
                 self.ftestcase._log_response(self.session.css[url], 'link',
                                              None, t_start, t_stop)
                 thread_sleep()  # give a chance to other threads
         else:
             newattributes.append((name, value))
     # Write the link tag to file (with revised paths)
     self.unknown_starttag('link', newattributes)
Exemplo n.º 3
0
 def run(self):
     """Run a test in loop."""
     while (self.thread_signaller.running()):
         test_result = unittest.TestResult()
         self.test.clearContext()
         self.test(test_result)
         if test_result.wasSuccessful():
             if recording():
                 add_cycle_result('success')
                 if self.color:
                     trace(green_str('.'))
                 else:
                     trace('.')
         else:
             if len(test_result.errors):
                 if recording():
                     add_cycle_result('error')
                 if self.color:
                     trace(red_str('E'))
                 else:
                     trace('E')
             else:
                 if recording():
                     add_cycle_result('failure')
                 if self.color:
                     trace(red_str('F'))
                 else:
                     trace('F')
             if self.debug:
                 for (test, error) in test_result.errors:
                     trace("ERROR %s: %s" % (str(test), str(error)))
                 for (test, error) in test_result.failures:
                     trace("FAILURE %s: %s" % (str(test), str(error)))
         thread_sleep(self.sleep_time)
Exemplo n.º 4
0
    def createThreads(self, cycle, number_of_threads):
        """Creates number_of_threads threads and returns as a list.

        NOTE: This method is not thread safe. Thread safety must be
        handled by the caller."""
        threads = []
        i = 0
        for i in range(number_of_threads):
            thread_id = self.createThreadId()
            thread_signaller = ThreadSignaller()
            thread = LoopTestRunner(self.module_name, self.class_name,
                                    self.method_name, self.options, cycle,
                                    number_of_threads, thread_id,
                                    thread_signaller, self.sleep_time)
            trace(".")
            try:
                thread.start()
            except ThreadError:
                trace("\nERROR: Can not create more than %i threads, try a "
                      "smaller stack size using: 'ulimit -s 2048' "
                      "for example\n" % (i + 1))
                raise
            thread_data = ThreadData(thread, thread_id, thread_signaller)
            threads.append(thread_data)
            thread_sleep(self.startup_delay)
        trace(' done.\n')
        return threads
Exemplo n.º 5
0
 def do_link(self, attributes):
     """Process link tag."""
     newattributes = [('rel', 'stylesheet'), ('type', 'text/css')]
     for name, value in attributes:
         if name == 'href':
             # construct full url
             url = urlparse.urljoin(self.base, value)
             # make sure it's syntactically valid
             if not valid_url.match(url):
                 continue
             # TODO: figure the re-write path
             # newattributes.append((name, path))
             if not self.session.css.has_key(url):
                 self.ftestcase.logdd('    link: %s ...' % url)
                 t_start = time.time()
                 self.session.css[url] = self.session.fetch(url)
                 t_stop = time.time()
                 self.ftestcase.logdd('     Done in %.3fs' %
                                      (t_stop - t_start))
                 self.session.history.append(('link', url))
                 self.ftestcase.total_time += (t_stop - t_start)
                 self.ftestcase.total_links += 1
                 self.ftestcase._log_response(self.session.css[url],
                                              'link', None, t_start, t_stop)
                 thread_sleep()      # give a chance to other threads
         else:
             newattributes.append((name, value))
     # Write the link tag to file (with revised paths)
     self.unknown_starttag('link', newattributes)
Exemplo n.º 6
0
 def do_link(self, attributes):
     """Process link tag."""
     newattributes = [('rel', 'stylesheet'), ('type', 'text/css')]
     for name, value in attributes:
         if name == 'href':
             # construct full url
             url = urlparse.urljoin(self.base, value)
             # make sure it's syntactically valid
             if not valid_url.match(url):
                 continue
             # TODO: figure the re-write path
             # newattributes.append((name, path))
             if not self.session.css.has_key(url):
                 self.ftestcase.logdd('    link: %s ...' % url)
                 with self.ftestcase.record_response('link', url, None):
                     try:
                         self.session.css[url] = self.session.fetch(url)
                         self.session.history.append(('link', url))
                     except HTTPError as error:
                         if self.ftestcase._accept_invalid_links:
                             if not self.ftestcase.in_bench_mode:
                                 self.ftestcase.logd('  ' + str(error))
                         else:
                             raise
                 thread_sleep()      # give a chance to other threads
         else:
             newattributes.append((name, value))
     # Write the link tag to file (with revised paths)
     self.unknown_starttag('link', newattributes)
Exemplo n.º 7
0
 def run(self):
     """Run a test in loop."""
     while (self.thread_signaller.running()):
         test_result = unittest.TestResult()
         self.test.clearContext()
         self.test(test_result)
         if test_result.wasSuccessful():
             if recording():
                 add_cycle_result('success')
                 if self.color:
                     trace(green_str('.'))
                 else:
                     trace('.')
         else:
             if len(test_result.errors):
                 if recording():
                     add_cycle_result('error')
                 if self.color:
                     trace(red_str('E'))
                 else:
                     trace('E')
             else:
                 if recording():
                     add_cycle_result('failure')
                 if self.color:
                     trace(red_str('F'))
                 else:
                     trace('F')
             if self.debug:
                 for (test, error) in test_result.errors:
                     trace("ERROR %s: %s" % (str(test), str(error)))
                 for (test, error) in test_result.failures:
                     trace("FAILURE %s: %s" % (str(test), str(error)))
         thread_sleep(self.sleep_time)
Exemplo n.º 8
0
 def do_img(self, attributes):
     """Process img tag."""
     newattributes = []
     for name, value in attributes:
         if name == "src":
             # construct full url
             url = urlparse.urljoin(self.base, value)
             # make sure it's syntactically valid
             if not valid_url.match(url):
                 continue
             # TODO: figure the re-write path
             # newattributes.append((name, path))
             if not self.session.images.has_key(url):
                 self.ftestcase.logdd("    img: %s ..." % url)
                 t_start = time.time()
                 self.session.images[url] = self.session.fetch(url)
                 t_stop = time.time()
                 self.ftestcase.logdd("     Done in %.3fs" % (t_stop - t_start))
                 self.session.history.append(("image", url))
                 self.ftestcase.total_time += t_stop - t_start
                 self.ftestcase.total_images += 1
                 self.ftestcase._log_response(self.session.images[url], "image", None, t_start, t_stop)
                 thread_sleep()  # give a chance to other threads
         else:
             newattributes.append((name, value))
     # Write the img tag to file (with revised paths)
     self.unknown_starttag("img", newattributes)
Exemplo n.º 9
0
    def run(self):
        """Run a test in loop."""
        while (self.thread_signaller.running()):
            test_result = unittest.TestResult()
            self.test.clearContext()
            self.test(test_result)
            feedback = {}

            if test_result.wasSuccessful():
                if recording():
                    feedback['count'] = add_cycle_result('success')

                if self.color:
                    trace(green_str('.'))
                else:
                    trace('.')

                feedback['result'] = 'success'
            else:
                if len(test_result.errors):
                    if recording():
                        feedback['count'] = add_cycle_result('error')

                    if self.color:
                        trace(red_str('E'))
                    else:
                        trace('E')

                    feedback['result'] = 'error'

                else:
                    if recording():
                        feedback['count'] = add_cycle_result('failure')

                    if self.color:
                        trace(red_str('F'))
                    else:
                        trace('F')

                    feedback['result'] = 'failure'

                if self.debug:
                    feedback['errors'] = test_result.errors
                    feedback['failures'] = test_result.failures

                    for (test, error) in test_result.errors:
                        trace("ERROR %s: %s" % (str(test), str(error)))
                    for (test, error) in test_result.failures:
                        trace("FAILURE %s: %s" % (str(test), str(error)))

            if self.feedback is not None:
                self.feedback.test_done(feedback)

            thread_sleep(self.sleep_time)
Exemplo n.º 10
0
    def run(self):
        """Run a test in loop."""
        while (self.thread_signaller.running()):
            test_result = unittest.TestResult()
            self.test.clearContext()
            self.test(test_result)
            feedback = {}

            if test_result.wasSuccessful():
                if recording():
                    feedback['count'] = add_cycle_result('success')

                if self.color:
                    trace(green_str('.'))
                else:
                    trace('.')

                feedback['result'] = 'success'
            else:
                if len(test_result.errors):
                    if recording():
                        feedback['count'] = add_cycle_result('error')

                    if self.color:
                        trace(red_str('E'))
                    else:
                        trace('E')

                    feedback['result'] = 'error'

                else:
                    if recording():
                        feedback['count'] = add_cycle_result('failure')

                    if self.color:
                        trace(red_str('F'))
                    else:
                        trace('F')

                    feedback['result'] = 'failure'

                if self.debug:
                    feedback['errors'] = test_result.errors
                    feedback['failures'] = test_result.failures

                    for (test, error) in test_result.errors:
                        trace("ERROR %s: %s" % (str(test), str(error)))
                    for (test, error) in test_result.failures:
                        trace("FAILURE %s: %s" % (str(test), str(error)))

            if self.feedback is not None:
                self.feedback.test_done(feedback)

            thread_sleep(self.sleep_time)
Exemplo n.º 11
0
    def run(self):
        """Run a test in loop."""
        while self.thread_signaller.running():
            test_result = unittest.TestResult()
            self.test.clearContext()
            self.test(test_result)
            feedback = {}

            if test_result.wasSuccessful():
                if recording():
                    feedback["count"] = add_cycle_result("success")

                if self.color:
                    trace(green_str("."))
                else:
                    trace(".")

                feedback["result"] = "success"
            else:
                if len(test_result.errors):
                    if recording():
                        feedback["count"] = add_cycle_result("error")

                    if self.color:
                        trace(red_str("E"))
                    else:
                        trace("E")

                    feedback["result"] = "error"

                else:
                    if recording():
                        feedback["count"] = add_cycle_result("failure")

                    if self.color:
                        trace(red_str("F"))
                    else:
                        trace("F")

                    feedback["result"] = "failure"

                if self.debug:
                    feedback["errors"] = test_result.errors
                    feedback["failures"] = test_result.failures

                    for (test, error) in test_result.errors:
                        trace("ERROR %s: %s" % (str(test), str(error)))
                    for (test, error) in test_result.failures:
                        trace("FAILURE %s: %s" % (str(test), str(error)))

            if self.feedback is not None:
                self.feedback.test_done(feedback)

            thread_sleep(self.sleep_time)
Exemplo n.º 12
0
    def sleep(self):
        """Sleeps a random amount of time.

        Between the predefined sleep_time_min and sleep_time_max values.
        """
        if self._pause:
            raw_input("Press ENTER to continue ")
            return
        s_min = self.sleep_time_min
        s_max = self.sleep_time_max
        if s_max != s_min:
            s_val = s_min + abs(s_max - s_min) * random()
        else:
            s_val = s_min
        # we should always sleep something
        thread_sleep(s_val)
Exemplo n.º 13
0
    def sleep(self):
        """Sleeps a random amount of time.

        Between the predefined sleep_time_min and sleep_time_max values.
        """
        if self._pause:
            raw_input("Press ENTER to continue ")
            return
        s_min = self.sleep_time_min
        s_max = self.sleep_time_max
        if s_max != s_min:
            s_val = s_min + abs(s_max - s_min) * random()
        else:
            s_val = s_min
        # we should always sleep something
        thread_sleep(s_val)
Exemplo n.º 14
0
    def _browse(self, url_in, params_in=None,
                description=None, ok_codes=None,
                method='post',
                follow_redirect=True, load_auto_links=True,
                sleep=True):
        """Simulate a browser handle redirects, load/cache css and images."""
        self._response = None
        # Loop mode
        if self._loop_mode:
            if self.steps == self._loop_steps[0]:
                self._loop_recording = True
                self.logi('Loop mode start recording')
            if self._loop_recording:
                self._loop_records.append((url_in, params_in, description,
                                           ok_codes, method, follow_redirect,
                                           load_auto_links, False))
        # ok codes
        if ok_codes is None:
            ok_codes = self.ok_codes
        if type(params_in) is DictType:
            params_in = params_in.items()
        params = []
        if params_in:
            if isinstance(params_in, Data):
                params = params_in
            else:
                for key, value in params_in:
                    if type(value) is DictType:
                        for val, selected in value.items():
                            if selected:
                                params.append((key, val))
                    elif type(value) in (ListType, TupleType):
                        for val in value:
                            params.append((key, val))
                    else:
                        params.append((key, value))

        if method == 'get' and params:
            url = url_in + '?' + urlencode(params)
        else:
            url = url_in
        if method == 'get':
            params = None

        if method == 'get':
            if not self.in_bench_mode:
                self.logd('GET: %s\n\tPage %i: %s ...' % (url, self.steps,
                                                          description or ''))
        else:
            url = url_in
            if not self.in_bench_mode:
                self.logd('%s: %s %s\n\tPage %i: %s ...' % (
                        method.upper(), url, str(params),
                        self.steps, description or ''))
            # Fetching
        response = self._connect(url, params, ok_codes, method, description)

        # Check redirection
        if follow_redirect and response.code in (301, 302, 303, 307):
            max_redirect_count = 10
            thread_sleep()              # give a chance to other threads
            while response.code in (301, 302, 303, 307) and max_redirect_count:
                # Figure the location - which may be relative
                newurl = response.headers['Location']
                url = urljoin(url_in, newurl)
                # Save the current url as the base for future redirects
                url_in = url
                self.logd(' Load redirect link: %s' % url)
                # Use the appropriate method for redirection
                if response.code in (302, 303):
                    method = 'get'
                if response.code == 303:
                    # 303 is HTTP/1.1, make sure the connection 
                    # is not in keep alive mode
                    self.setHeader('Connection', 'close')
                response = self._connect(url, None, ok_codes, rtype=method,
                                         description=None, redirect=True)
                max_redirect_count -= 1
            if not max_redirect_count:
                self.logd(' WARNING Too many redirects give up.')

        # Load auto links (css and images)
        response.is_html = is_html(response.body)
        if load_auto_links and response.is_html and not self._simple_fetch:
            self.logd(' Load css and images...')
            page = response.body
            t_start = time.time()
            c_start = self.total_time
            try:
                # pageImages is patched to call _log_response on all links
                self._browser.pageImages(url, page, self)
            except HTTPError, error:
                if self._accept_invalid_links:
                    if not self.in_bench_mode:
                        self.logd('  ' + str(error))
                else:
                    t_stop = time.time()
                    t_delta = t_stop - t_start
                    self.step_success = False
                    self.test_status = 'Failure'
                    self.logd('  Failed in ~ %.2fs' % t_delta)
                    # XXX The duration logged for this response is wrong
                    self._log_response(error.response, 'link', None,
                                       t_start, t_stop, log_body=True)
                    raise self.failureException, str(error)
            c_stop = self.total_time
            self.logd('  Done in %.3fs' % (c_stop - c_start))
Exemplo n.º 15
0
    def _browse(self,
                url_in,
                params_in=None,
                description=None,
                ok_codes=None,
                method='post',
                follow_redirect=True,
                load_auto_links=True,
                sleep=True):
        """Simulate a browser handle redirects, load/cache css and images."""
        self._response = None
        # Loop mode
        if self._loop_mode:
            if self.steps == self._loop_steps[0]:
                self._loop_recording = True
                self.logi('Loop mode start recording')
            if self._loop_recording:
                self._loop_records.append(
                    (url_in, params_in, description, ok_codes, method,
                     follow_redirect, load_auto_links, False))
        # ok codes
        if ok_codes is None:
            ok_codes = self.ok_codes
        if type(params_in) is DictType:
            params_in = params_in.items()
        params = []
        if params_in:
            if isinstance(params_in, Data):
                params = params_in
            else:
                for key, value in params_in:
                    if type(value) is DictType:
                        for val, selected in value.items():
                            if selected:
                                params.append((key, val))
                    elif type(value) in (ListType, TupleType):
                        for val in value:
                            params.append((key, val))
                    else:
                        params.append((key, value))

        if method == 'get' and params:
            url = url_in + '?' + urlencode(params)
        else:
            url = url_in
        if method == 'get':
            params = None

        if method == 'get':
            if not self.in_bench_mode:
                self.logd('GET: %s\n\tPage %i: %s ...' %
                          (url, self.steps, description or ''))
        else:
            url = url_in
            if not self.in_bench_mode:
                self.logd('%s: %s %s\n\tPage %i: %s ...' %
                          (method.upper(), url, str(params), self.steps,
                           description or ''))
            # Fetching
        response = self._connect(url, params, ok_codes, method, description)

        # Check redirection
        if follow_redirect and response.code in (301, 302, 303, 307):
            max_redirect_count = 10
            thread_sleep()  # give a chance to other threads
            while response.code in (301, 302, 303, 307) and max_redirect_count:
                # Figure the location - which may be relative
                newurl = response.headers['Location']
                url = urljoin(url_in, newurl)
                # Save the current url as the base for future redirects
                url_in = url
                self.logd(' Load redirect link: %s' % url)
                # Use the appropriate method for redirection
                if response.code in (302, 303):
                    method = 'get'
                if response.code == 303:
                    # 303 is HTTP/1.1, make sure the connection
                    # is not in keep alive mode
                    self.setHeader('Connection', 'close')
                response = self._connect(url,
                                         None,
                                         ok_codes,
                                         rtype=method,
                                         description=None,
                                         redirect=True)
                max_redirect_count -= 1
            if not max_redirect_count:
                self.logd(' WARNING Too many redirects give up.')

        # Load auto links (css and images)
        response.is_html = is_html(response.body)
        if load_auto_links and response.is_html and not self._simple_fetch:
            self.logd(' Load css and images...')
            page = response.body
            t_start = time.time()
            c_start = self.total_time
            try:
                # pageImages is patched to call _log_response on all links
                self._browser.pageImages(url, page, self)
            except HTTPError, error:
                if self._accept_invalid_links:
                    if not self.in_bench_mode:
                        self.logd('  ' + str(error))
                else:
                    t_stop = time.time()
                    t_delta = t_stop - t_start
                    self.step_success = False
                    self.test_status = 'Failure'
                    self.logd('  Failed in ~ %.2fs' % t_delta)
                    # XXX The duration logged for this response is wrong
                    self._log_response(error.response,
                                       'link',
                                       None,
                                       t_start,
                                       t_stop,
                                       log_body=True)
                    raise self.failureException, str(error)
            c_stop = self.total_time
            self.logd('  Done in %.3fs' % (c_stop - c_start))
Exemplo n.º 16
0
    def _browse(self, url_in, params_in=None,
                description=None, ok_codes=None,
                method='post',
                follow_redirect=True, load_auto_links=True,
                sleep=True):
        """Simulate a browser handle redirects, load/cache css and images."""
        self._response = None
        # Loop mode
        if self._loop_mode:
            if self.steps == self._loop_steps[0]:
                self._loop_recording = True
                self.logi('Loop mode start recording')
            if self._loop_recording:
                self._loop_records.append((url_in, params_in, description,
                                           ok_codes, method, follow_redirect,
                                           load_auto_links, False))
        # ok codes
        if ok_codes is None:
            ok_codes = self.ok_codes
        if type(params_in) is DictType:
            params_in = params_in.items()
        params = []
        if params_in:
            if isinstance(params_in, Data):
                params = params_in
            else:
                for key, value in params_in:
                    if type(value) is DictType:
                        for val, selected in value.items():
                            if selected:
                                params.append((key, val))
                    elif type(value) in (ListType, TupleType):
                        for val in value:
                            params.append((key, val))
                    else:
                        params.append((key, value))

        if method == 'get' and params:
            url = url_in + '?' + urlencode(params)
        else:
            url = url_in
        if method == 'get':
            params = None

        if method == 'get':
            if not self.in_bench_mode:
                self.logd('GET: %s\n\tPage %i: %s ...' % (url, self.steps,
                                                          description or ''))
        else:
            url = url_in
            if not self.in_bench_mode:
                self.logd('%s: %s %s\n\tPage %i: %s ...' % (
                        method.upper(), url, str(params),
                        self.steps, description or ''))
            # Fetching
        response = self._connect(url, params, ok_codes, method, description)

        # Check redirection
        if follow_redirect and response.code in (301, 302, 303, 307):
            max_redirect_count = 10
            thread_sleep()              # give a chance to other threads
            while response.code in (301, 302, 303, 307) and max_redirect_count:
                # Figure the location - which may be relative
                newurl = response.headers['Location']
                url = urljoin(url_in, newurl)
                # Save the current url as the base for future redirects
                url_in = url
                self.logd(' Load redirect link: %s' % url)
                # Use the appropriate method for redirection
                if response.code in (302, 303):
                    method = 'get'
                if response.code == 303:
                    # 303 is HTTP/1.1, make sure the connection 
                    # is not in keep alive mode
                    self.setHeader('Connection', 'close')
                response = self._connect(url, None, ok_codes, rtype=method,
                                         description=None, redirect=True)
                max_redirect_count -= 1
            if not max_redirect_count:
                self.logd(' WARNING Too many redirects give up.')

        # Load auto links (css and images)
        response.is_html = is_html(response.body)
        if load_auto_links and response.is_html and not self._simple_fetch:
            self.logd(' Load css and images...')
            page = response.body
            # pageImages is patched to call self.record on all links
            self._browser.pageImages(url, page, self)
        if sleep:
            self.sleep()
        self._response = response

        # Loop mode
        if self._loop_mode and self.steps == self._loop_steps[-1]:
            self._loop_recording = False
            self.logi('Loop mode end recording.')
            t_start = self.total_time
            count = 0
            for i in range(self._loop_number):
                self.logi('Loop mode replay %i' % i)
                for record in self._loop_records:
                    count += 1
                    self.steps += 1
                    self._browse(*record)
            t_delta = self.total_time - t_start
            text = ('End of loop: %d pages rendered in %.3fs, '
                    'avg of %.3fs per page, '
                    '%.3f SPPS without concurrency.' % (count, t_delta,
                                                        t_delta / count,
                                                        count/t_delta))
            self.logi(text)
            trace(text + '\n')

        return response