def get_port_for_run(args):
     options, parsed_args = run_webkit_tests.parse_args(args)
     test_port = ImageDiffTestPort(options=options,
                                   user=mocktool.MockUser())
     res = passing_run(args, port_obj=test_port, tests_included=True)
     self.assertTrue(res)
     return test_port
def parse_args(extra_args=None,
               record_results=False,
               tests_included=False,
               new_results=False,
               print_nothing=True):
    extra_args = extra_args or []
    if print_nothing:
        args = ['--print', 'nothing']
    else:
        args = []
    if not '--platform' in extra_args:
        args.extend(['--platform', 'test'])
    if not record_results:
        args.append('--no-record-results')
    if not new_results:
        args.append('--no-new-test-results')

    if not '--child-processes' in extra_args and not '--worker-model' in extra_args:
        args.extend(['--worker-model', 'inline'])
    args.extend(extra_args)
    if not tests_included:
        # We use the glob to test that globbing works.
        args.extend(
            ['passes', 'http/tests', 'websocket/tests', 'failures/expected/*'])
    return run_webkit_tests.parse_args(args)
Example #3
0
    def test_http_locking(tester):
        class LockCheckingManager(Manager):
            def __init__(self, port, options, printer):
                super(LockCheckingManager, self).__init__(port, options, printer)
                self._finished_list_called = False

            def handle_finished_list(self, source, list_name, num_tests, elapsed_time):
                if not self._finished_list_called:
                    tester.assertEquals(list_name, 'locked_tests')
                    tester.assertTrue(self._remaining_locked_shards)
                    tester.assertTrue(self._has_http_lock)

                super(LockCheckingManager, self).handle_finished_list(source, list_name, num_tests, elapsed_time)

                if not self._finished_list_called:
                    tester.assertEquals(self._remaining_locked_shards, [])
                    tester.assertFalse(self._has_http_lock)
                    self._finished_list_called = True

        options, args = run_webkit_tests.parse_args(['--platform=test', '--print=nothing', 'http/tests/passes', 'passes'])
        host = MockHost()
        port = host.port_factory.get(port_name=options.platform, options=options)
        run_webkit_tests._set_up_derived_options(port, options)
        printer = printing.Printer(port, options, StringIO.StringIO(), StringIO.StringIO(), configure_logging=False)
        manager = LockCheckingManager(port, options, printer)
        manager.collect_tests(args)
        manager.parse_expectations()
        result_summary = manager.set_up_run()
        num_unexpected_results = manager.run(result_summary)
        manager.clean_up_run()
        printer.cleanup()
        tester.assertEquals(num_unexpected_results, 0)
 def get_port_for_run(args):
     options, parsed_args = run_webkit_tests.parse_args(args)
     host = MockHost()
     test_port = ImageDiffTestPort(host, options=options)
     res = passing_run(args, port_obj=test_port, tests_included=True)
     self.assertTrue(res)
     return test_port
def logging_run(extra_args=None, port_obj=None, tests_included=False):
    extra_args = extra_args or []
    args = ['--no-record-results']
    if not '--platform' in extra_args:
        args.extend(['--platform', 'test'])
    if not '--child-processes' in extra_args:
        args.extend(['--worker-model', 'inline'])
    args.extend(extra_args)
    if not tests_included:
        args.extend(['passes',
                     'http/tests',
                     'websocket/tests',
                     'failures/expected/*'])

    oc = outputcapture.OutputCapture()
    try:
        oc.capture_output()
        options, parsed_args = run_webkit_tests.parse_args(args)
        user = MockUser()
        if not port_obj:
            port_obj = port.get(port_name=options.platform, options=options,
                                user=user)
        buildbot_output = array_stream.ArrayStream()
        regular_output = array_stream.ArrayStream()
        res = run_webkit_tests.run(port_obj, options, parsed_args,
                                   buildbot_output=buildbot_output,
                                   regular_output=regular_output)
    finally:
        oc.restore_output()
    return (res, buildbot_output, regular_output, user)
 def get_port_for_run(args):
     options, parsed_args = run_webkit_tests.parse_args(args)
     host = MockHost()
     test_port = ImageDiffTestPort(host, options=options)
     res = passing_run(args, port_obj=test_port, tests_included=True)
     self.assertTrue(res)
     return test_port
Example #7
0
    def _runner(self, port=None):
        # FIXME: we shouldn't have to use run_webkit_tests.py to get the options we need.
        options = run_webkit_tests.parse_args(['--platform', 'test-mac-mac10.11'])[0]
        options.child_processes = '1'

        host = MockHost()
        port = port or host.port_factory.get(options.platform, options=options)
        return LockCheckingRunner(port, options, FakePrinter(), self, True)
def passing_run(args, port_obj=None, logging_included=False):
    if not logging_included:
        args.extend(['--print', 'nothing'])
    options, args = run_webkit_tests.parse_args(args)
    if port_obj is None:
        port_obj = port.get(options.platform, options)
    res = run_webkit_tests.run(port_obj, options, args)
    return res == 0
    def _runner(self, port=None):
        # FIXME: we shouldn't have to use run_webkit_tests.py to get the options we need.
        options = run_webkit_tests.parse_args(['--platform', 'test-mac-snowleopard'])[0]
        options.child_processes = '1'

        host = MockHost()
        port = port or host.port_factory.get(options.platform, options=options)
        return LayoutTestRunner(options, port, FakePrinter(), port.results_directory(), lambda test_name: False)
    def _runner(self, port=None):
        # FIXME: we shouldn't have to use run_webkit_tests.py to get the options we need.
        options = run_webkit_tests.parse_args(['--platform', 'test-mac-snowleopard'])[0]
        options.child_processes = '1'

        host = MockHost()
        port = port or host.port_factory.get(options.platform, options=options)
        return LayoutTestRunner(options, port, FakePrinter(), port.results_directory())
    def _runner(self, port=None):
        # FIXME: we shouldn't have to use run_webkit_tests.py to get the options we need.
        options = run_webkit_tests.parse_args(['--platform', 'test-mac-snowleopard'])[0]
        options.child_processes = '1'

        host = MockHost()
        port = port or host.port_factory.get(options.platform, options=options)
        return LockCheckingRunner(port, options, FakePrinter(), self, True)
def passing_run(args, port_obj=None, logging_included=False):
    if not logging_included:
        args.extend(['--print', 'nothing'])
    options, args = run_webkit_tests.parse_args(args)
    if port_obj is None:
        port_obj = port.get(options.platform, options)
    res = run_webkit_tests.run(port_obj, options, args)
    return res == 0
def logging_run(args):
    options, args = run_webkit_tests.parse_args(args)
    port_obj = port.get(options.platform, options)
    buildbot_output = array_stream.ArrayStream()
    regular_output = array_stream.ArrayStream()
    res = run_webkit_tests.run(port_obj, options, args,
                               buildbot_output=buildbot_output,
                               regular_output=regular_output)
    return (res, buildbot_output, regular_output)
def logging_run(args):
    options, args = run_webkit_tests.parse_args(args)
    port_obj = port.get(options.platform, options)
    buildbot_output = array_stream.ArrayStream()
    regular_output = array_stream.ArrayStream()
    res = run_webkit_tests.run(port_obj, options, args,
                               buildbot_output=buildbot_output,
                               regular_output=regular_output)
    return (res, buildbot_output, regular_output)
Example #15
0
 def test_perf_locking(tester):
     options, args = run_webkit_tests.parse_args(['--platform=test', '--print=nothing', '--no-http', 'passes', 'perf/'])
     host = MockHost()
     port = host.port_factory.get(port_name=options.platform, options=options)
     run_webkit_tests._set_up_derived_options(port, options)
     printer = printing.Printer(port, options, StringIO.StringIO(), StringIO.StringIO())
     manager = LockCheckingManager(port, options, printer, tester, False)
     manager.collect_tests(args)
     manager.parse_expectations()
     num_unexpected_results = manager.run()
     printer.cleanup()
     tester.assertEquals(num_unexpected_results, 0)
Example #16
0
 def test_perf_locking(tester):
     options, args = run_webkit_tests.parse_args(['--platform=test', '--print=nothing', '--no-http', 'passes', 'perf/'])
     host = MockHost()
     port = host.port_factory.get(port_name=options.platform, options=options)
     run_webkit_tests._set_up_derived_options(port, options)
     printer = printing.Printer(port, options, StringIO.StringIO(), StringIO.StringIO())
     manager = LockCheckingManager(port, options, printer, tester, False)
     manager.collect_tests(args)
     manager.parse_expectations()
     num_unexpected_results = manager.run()
     printer.cleanup()
     tester.assertEquals(num_unexpected_results, 0)
def get_tests_run(extra_args=None, tests_included=False, flatten_batches=False):
    extra_args = extra_args or []
    args = [
        '--print', 'nothing',
        '--platform', 'test',
        '--no-record-results',
        '--worker-model', 'inline']
    args.extend(extra_args)
    if not tests_included:
        # Not including http tests since they get run out of order (that
        # behavior has its own test, see test_get_test_file_queue)
        args.extend(['passes', 'failures'])
    options, parsed_args = run_webkit_tests.parse_args(args)
    user = MockUser()

    test_batches = []

    class RecordingTestDriver(TestDriver):
        def __init__(self, port, worker_number):
            TestDriver.__init__(self, port, worker_number)
            self._current_test_batch = None

        def poll(self):
            # So that we don't create a new driver for every test
            return None

        def stop(self):
            self._current_test_batch = None

        def run_test(self, test_input):
            if self._current_test_batch is None:
                self._current_test_batch = []
                test_batches.append(self._current_test_batch)
            test_name = self._port.relative_test_filename(test_input.filename)
            self._current_test_batch.append(test_name)
            return TestDriver.run_test(self, test_input)

    class RecordingTestPort(TestPort):
        def create_driver(self, worker_number):
            return RecordingTestDriver(self, worker_number)

    recording_port = RecordingTestPort(options=options, user=user)
    logging_run(extra_args=args, port_obj=recording_port, tests_included=True)

    if flatten_batches:
        return list(itertools.chain(*test_batches))

    return test_batches
    def test_missing_and_unexpected_results_with_custom_exit_code(self):
        # Test that we update expectations in place. If the expectation
        # is missing, update the expected generic location.
        class CustomExitCodePort(test.TestPort):
            def exit_code_from_summarized_results(self, unexpected_results):
                return unexpected_results['num_regressions'] + unexpected_results['num_missing']

        host = MockHost()
        options, parsed_args = run_webkit_tests.parse_args(['--pixel-tests', '--no-new-test-results'])
        test_port = CustomExitCodePort(host, options=options)
        details, err, _ = logging_run(['--no-show-results',
            'failures/expected/missing_image.html',
            'failures/unexpected/missing_text.html',
            'failures/unexpected/text-image-checksum.html'],
            tests_included=True, host=host, port_obj=test_port)
        self.assertEqual(details.exit_code, 2)
    def test_missing_and_unexpected_results_with_custom_exit_code(self):
        # Test that we update expectations in place. If the expectation
        # is missing, update the expected generic location.
        class CustomExitCodePort(test.TestPort):
            def exit_code_from_summarized_results(self, unexpected_results):
                return unexpected_results['num_regressions'] + unexpected_results['num_missing']

        host = MockHost()
        options, parsed_args = run_webkit_tests.parse_args(['--pixel-tests', '--no-new-test-results'])
        test_port = CustomExitCodePort(host, options=options)
        details, err, _ = logging_run(['--no-show-results',
            'failures/expected/missing_image.html',
            'failures/unexpected/missing_text.html',
            'failures/unexpected/text-image-checksum.html'],
            tests_included=True, host=host, port_obj=test_port)
        self.assertEqual(details.exit_code, 2)
    def test_missing_and_unexpected_results_with_custom_exit_code(self):
        # Test that we update expectations in place. If the expectation
        # is missing, update the expected generic location.
        fs = unit_test_filesystem()

        class CustomExitCodePort(TestPort):
            def exit_code_from_summarized_results(self, unexpected_results):
                return unexpected_results['num_regressions'] + unexpected_results['num_missing']

        options, parsed_args = run_webkit_tests.parse_args(['--pixel-tests', '--no-new-test-results'])
        test_port = CustomExitCodePort(options=options, user=mocktool.MockUser())
        res, out, err, _ = logging_run(['--no-show-results',
            'failures/expected/missing_image.html',
            'failures/unexpected/missing_text.html',
            'failures/unexpected/text-image-checksum.html'],
            tests_included=True, filesystem=fs, record_results=True, port_obj=test_port)
        self.assertEquals(res, 2)
Example #21
0
def logging_run(args=[], tests_included=False):
    new_args = ['--no-record-results']
    if not '--platform' in args:
        new_args.extend(['--platform', 'test'])
    new_args.extend(args)
    if not tests_included:
        new_args.extend(['passes',
                         'http/tests'
                         'websocket/tests',
                         'failures/expected/*'])
    options, parsed_args = run_webkit_tests.parse_args(new_args)
    port_obj = port.get(options.platform, options)
    buildbot_output = array_stream.ArrayStream()
    regular_output = array_stream.ArrayStream()
    res = run_webkit_tests.run(port_obj, options, parsed_args,
                               buildbot_output=buildbot_output,
                               regular_output=regular_output)
    return (res, buildbot_output, regular_output)
def parse_args(extra_args=None, tests_included=False, new_results=False, print_nothing=True):
    extra_args = extra_args or []
    args = []
    if not '--platform' in extra_args:
        args.extend(['--platform', 'test'])
    if not new_results:
        args.append('--no-new-test-results')

    if not '--child-processes' in extra_args:
        args.extend(['--child-processes', 1])
    args.extend(extra_args)
    if not tests_included:
        # We use the glob to test that globbing works.
        args.extend(['passes',
                     'http/tests',
                     'websocket/tests',
                     'failures/expected/*'])
    return run_webkit_tests.parse_args(args)
Example #23
0
def passing_run(args=[], port_obj=None, record_results=False,
                tests_included=False):
    new_args = ['--print', 'nothing']
    if not '--platform' in args:
        new_args.extend(['--platform', 'test'])
    if not record_results:
        new_args.append('--no-record-results')
    new_args.extend(args)
    if not tests_included:
        # We use the glob to test that globbing works.
        new_args.extend(['passes',
                         'http/tests',
                         'websocket/tests',
                         'failures/expected/*'])
    options, parsed_args = run_webkit_tests.parse_args(new_args)
    if port_obj is None:
        port_obj = port.get(options.platform, options)
    res = run_webkit_tests.run(port_obj, options, parsed_args)
    return res == 0
Example #24
0
def passing_run(args=[],
                port_obj=None,
                record_results=False,
                tests_included=False):
    new_args = ['--print', 'nothing']
    if not '--platform' in args:
        new_args.extend(['--platform', 'test'])
    if not record_results:
        new_args.append('--no-record-results')
    new_args.extend(args)
    if not tests_included:
        # We use the glob to test that globbing works.
        new_args.extend(
            ['passes', 'http/tests', 'websocket/tests', 'failures/expected/*'])
    options, parsed_args = run_webkit_tests.parse_args(new_args)
    if port_obj is None:
        port_obj = port.get(options.platform, options)
    res = run_webkit_tests.run(port_obj, options, parsed_args)
    return res == 0
Example #25
0
    def test_http_locking(tester):
        class LockCheckingManager(Manager):
            def __init__(self, port, options, printer):
                super(LockCheckingManager,
                      self).__init__(port, options, printer)
                self._finished_list_called = False

            def handle_finished_list(self, source, list_name, num_tests,
                                     elapsed_time):
                if not self._finished_list_called:
                    tester.assertEquals(list_name, 'locked_tests')
                    tester.assertTrue(self._remaining_locked_shards)
                    tester.assertTrue(self._has_http_lock)

                super(LockCheckingManager,
                      self).handle_finished_list(source, list_name, num_tests,
                                                 elapsed_time)

                if not self._finished_list_called:
                    tester.assertEquals(self._remaining_locked_shards, [])
                    tester.assertFalse(self._has_http_lock)
                    self._finished_list_called = True

        options, args = run_webkit_tests.parse_args([
            '--platform=test', '--print=nothing', 'http/tests/passes', 'passes'
        ])
        host = MockHost()
        port = host.port_factory.get(port_name=options.platform,
                                     options=options)
        run_webkit_tests._set_up_derived_options(port, options)
        printer = printing.Printer(port,
                                   options,
                                   StringIO.StringIO(),
                                   StringIO.StringIO(),
                                   configure_logging=False)
        manager = LockCheckingManager(port, options, printer)
        manager.collect_tests(args)
        manager.parse_expectations()
        result_summary = manager.set_up_run()
        num_unexpected_results = manager.run(result_summary)
        manager.clean_up_run()
        printer.cleanup()
        tester.assertEquals(num_unexpected_results, 0)
Example #26
0
def logging_run(args=[], tests_included=False):
    new_args = ['--no-record-results']
    if not '--platform' in args:
        new_args.extend(['--platform', 'test'])
    new_args.extend(args)
    if not tests_included:
        new_args.extend(
            ['passes', 'http/tests'
             'websocket/tests', 'failures/expected/*'])
    options, parsed_args = run_webkit_tests.parse_args(new_args)
    port_obj = port.get(options.platform, options)
    buildbot_output = array_stream.ArrayStream()
    regular_output = array_stream.ArrayStream()
    res = run_webkit_tests.run(port_obj,
                               options,
                               parsed_args,
                               buildbot_output=buildbot_output,
                               regular_output=regular_output)
    return (res, buildbot_output, regular_output)
def parse_args(extra_args=None, record_results=False, tests_included=False, new_results=False, print_nothing=True):
    extra_args = extra_args or []
    if print_nothing:
        args = ["--print", "nothing"]
    else:
        args = []
    if not "--platform" in extra_args:
        args.extend(["--platform", "test"])
    if not record_results:
        args.append("--no-record-results")
    if not new_results:
        args.append("--no-new-test-results")

    if not "--child-processes" in extra_args and not "--worker-model" in extra_args:
        args.extend(["--worker-model", "inline"])
    args.extend(extra_args)
    if not tests_included:
        # We use the glob to test that globbing works.
        args.extend(["passes", "http/tests", "websocket/tests", "failures/expected/*"])
    return run_webkit_tests.parse_args(args)
Example #28
0
def parse_args(extra_args=None, record_results=False, tests_included=False,
               print_nothing=True):
    extra_args = extra_args or []
    if print_nothing:
        args = ['--print', 'nothing']
    else:
        args = []
    if not '--platform' in extra_args:
        args.extend(['--platform', 'test'])
    if not record_results:
        args.append('--no-record-results')
    if not '--child-processes' in extra_args:
        args.extend(['--worker-model', 'old-inline'])
    args.extend(extra_args)
    if not tests_included:
        # We use the glob to test that globbing works.
        args.extend(['passes',
                     'http/tests',
                     'websocket/tests',
                     'failures/expected/*'])
    return run_webkit_tests.parse_args(args)
    def test_missing_and_unexpected_results_with_custom_exit_code(self):
        # Test that we update expectations in place. If the expectation
        # is missing, update the expected generic location.
        class CustomExitCodePort(TestPort):
            def exit_code_from_summarized_results(self, unexpected_results):
                return unexpected_results["num_regressions"] + unexpected_results["num_missing"]

        host = MockHost()
        options, parsed_args = run_webkit_tests.parse_args(["--pixel-tests", "--no-new-test-results"])
        test_port = CustomExitCodePort(host, options=options)
        res, out, err, _ = logging_run(
            [
                "--no-show-results",
                "failures/expected/missing_image.html",
                "failures/unexpected/missing_text.html",
                "failures/unexpected/text-image-checksum.html",
            ],
            tests_included=True,
            host=host,
            record_results=True,
            port_obj=test_port,
        )
        self.assertEquals(res, 2)
def passing_run(extra_args=None, port_obj=None, record_results=False,
                tests_included=False):
    extra_args = extra_args or []
    args = ['--print', 'nothing']
    if not '--platform' in extra_args:
        args.extend(['--platform', 'test'])
    if not record_results:
        args.append('--no-record-results')
    if not '--child-processes' in extra_args:
        args.extend(['--worker-model', 'inline'])
    args.extend(extra_args)
    if not tests_included:
        # We use the glob to test that globbing works.
        args.extend(['passes',
                     'http/tests',
                     'websocket/tests',
                     'failures/expected/*'])
    options, parsed_args = run_webkit_tests.parse_args(args)
    if not port_obj:
        port_obj = port.get(port_name=options.platform, options=options,
                            user=MockUser())
    res = run_webkit_tests.run(port_obj, options, parsed_args)
    return res == 0
 def get_port_for_run(args):
     options, parsed_args = run_webkit_tests.parse_args(args)
     test_port = ImageDiffTestPort(options=options, user=mocktool.MockUser())
     res = passing_run(args, port_obj=test_port, tests_included=True)
     self.assertTrue(res)
     return test_port
 def get_port_for_run(args):
     options, parsed_args = run_webkit_tests.parse_args(args)
     test_port = ImageDiffTestPort(options=options, user=MockUser())
     passing_run(args, port_obj=test_port, tests_included=True)
     return test_port