예제 #1
0
    def test_join_results(self):
        """Can merge a full results dictionary into this result."""
        res = DatazillaResult({"suite1": {"test1a": [1]}})

        res.join_results(
            DatazillaResult({
                "suite1": {
                    "test1a": [2],
                    "test1b": [3]
                },
                "suite2": {
                    "test2a": [4]
                },
            }))

        self.assertEqual(
            res.results, {
                "suite1": {
                    "test1a": [1, 2],
                    "test1b": [3]
                },
                "suite2": {
                    "test2a": [4]
                },
            })
예제 #2
0
    def test_add_second_datazilla_result(self):
        """Adding a second DatazillaResult joins their results."""
        req = DatazillaRequest('http',
                               'host',
                               'project',
                               'key',
                               'secret',
                               branch='mozilla-try')
        res1 = DatazillaResult({'suite1': {'test': [1]}})
        res2 = DatazillaResult({'suite2': {'test': [2]}})

        req.add_datazilla_result(res1)
        req.add_datazilla_result(res2)

        self.assertEqual(
            req.results.results,
            {
                'suite1': {
                    'test': [1]
                },
                'suite2': {
                    'test': [2]
                }
            },
        )
예제 #3
0
 def test_add_xperf_results(self):
     """Can add xperf results to a suite"""
     res = DatazillaResult()
     res.add_xperf_results("suite", "name", [[1, 'hello'], [2, 'world'], 3])
     self.assertEqual(res.results_xperf,
                      {"suite": {
                          "name": [[1, 'hello'], [2, 'world'], 3]
                      }})
예제 #4
0
    def __call__(self):

        # platform
        machine = self.test_machine()

        # build information
        browser_config = self.results.browser_config

        # a place to put results
        res = DatazillaResult()

        for test in self.results.results:
            suite = "%s" % test.name()
            res.add_testsuite(suite, options=self.run_options(test))

            # serialize test results
            results = {}
            if not test.using_xperf:
                for result in test.results:
                    # XXX this will not work for manifests which list
                    # the same page name twice. It also ignores cycles
                    for page, val in result.raw_values():
                        if page == 'NULL':
                            results.setdefault(test.name(), []).extend(val)
                        else:
                            results.setdefault(page, []).extend(val)
                for result, values in results.items():
                    res.add_test_results(suite, result, values)

                # counters results_aux data
                for cd in test.all_counter_results:
                    for name, vals in cd.items():
                        res.add_talos_auxiliary(suite, name, vals)
            else:
                # specific xperf_aux data
                for cd in test.all_counter_results:
                    for name, vals in cd.items():
                        res.add_xperf_results(suite, name, vals)

        # make a datazilla test result collection
        collection = DatazillaResultsCollection(
            machine_name=machine['name'],
            os=machine['os'],
            os_version=machine['osversion'],
            platform=machine['platform'],
            build_name=browser_config['browser_name'],
            version=browser_config['browser_version'],
            revision=browser_config['sourcestamp'],
            branch=browser_config['branch_name'],
            id=browser_config['buildid'],
            test_date=self.results.date)
        collection.add_datazilla_result(res)
        return collection
예제 #5
0
    def test_add_datazilla_result(self):
        """Can add a DatazillaResult to a DatazillaRequest."""
        req = DatazillaRequest('http',
                               'host',
                               'project',
                               'key',
                               'secret',
                               branch='mozilla-try')
        res = DatazillaResult({'suite': {'test': [1, 2, 3]}})

        req.add_datazilla_result(res)

        self.assertEqual(req.results.results, res.results)
예제 #6
0
    def test_datasets(self):
        """Tests dataset creation for submission to datazilla"""

        req = DatazillaRequest(
            protocol='http',
            host='host',
            project='project',
            oauth_key='key',
            oauth_secret='secret',
            machine_name='qm-pxp01',
            os='linux',
            os_version='Ubuntu 11.10',
            platform='x86_64',
            build_name='Firefox',
            version='14.0a2',
            revision='785345035a3b',
            branch='Mozilla-Aurora',
            id='20120228122102',
        )

        results = {'suite1': {'test1': [1]}, 'suite2': {'test2': [2]}}
        req.add_datazilla_result(DatazillaResult(results))

        datasets = req.datasets()

        self.assertEqual(len(datasets), 2)

        for dataset in datasets:
            self.assertEqual(
                set(dataset.keys()),
                set(['results', 'test_build', 'test_machine', 'testrun']))
            self.assertEqual(
                dataset['test_build'], {
                    'branch': 'Mozilla-Aurora',
                    'id': '20120228122102',
                    'name': 'Firefox',
                    'revision': '785345035a3b',
                    'version': '14.0a2'
                })
            self.assertEqual(
                dataset['test_machine'], {
                    'name': 'qm-pxp01',
                    'os': 'linux',
                    'osversion': 'Ubuntu 11.10',
                    'platform': 'x86_64'
                })
            self.assertEqual(set(dataset['testrun'].keys()),
                             set(['suite', 'date']))
            suite = dataset['testrun']['suite']
            self.assertTrue(suite in results)
            self.assertEqual(dataset['results'], results[suite])
예제 #7
0
    def test_submit(self, mock_send):
        """Submits blob of JSON data for each test suite."""
        req = DatazillaRequest(
            protocol='http',
            host='host',
            project='project',
            oauth_key='key',
            oauth_secret='secret',
            machine_name='qm-pxp01',
            os='linux',
            os_version='Ubuntu 11.10',
            platform='x86_64',
            build_name='Firefox',
            version='14.0a2',
            revision='785345035a3b',
            branch='Mozilla-Aurora',
            id='20120228122102',
        )

        req.add_datazilla_result(
            DatazillaResult({
                'suite1': {
                    'test1': [1]
                },
                'suite2': {
                    'test2': [2]
                }
            }))

        req.submit()

        self.assertEqual(mock_send.call_count, 2)
        data1 = mock_send.call_args_list[0][0][0]
        data2 = mock_send.call_args_list[1][0][0]

        results = sorted([data1.pop('results'), data2.pop('results')])

        self.assertEqual(results, sorted([{'test1': [1]}, {'test2': [2]}]))

        suites = sorted(
            [data1['testrun'].pop('suite'), data2['testrun'].pop('suite')])

        self.assertEqual(suites, sorted(['suite1', 'suite2']))

        # aside from results and suite, datasets are the same
        self.assertEqual(data1, data2)

        self.assertEqual(
            data1['test_build'],
            {
                'branch': 'Mozilla-Aurora',
                'id': '20120228122102',
                'name': 'Firefox',
                'revision': '785345035a3b',
                'version': '14.0a2',
            },
        )

        self.assertEqual(
            data1['test_machine'],
            {
                'name': 'qm-pxp01',
                'os': 'linux',
                'osversion': 'Ubuntu 11.10',
                'platform': 'x86_64',
            },
        )

        self.assertEqual(data1['testrun']['date'], req.test_date)
예제 #8
0
    def test_init_with_results(self):
        """Can initialize with a dictionary of results."""
        data = {"suite": {"test": [1, 2, 3]}}
        res = DatazillaResult(data)

        self.assertEqual(res.results, data)
예제 #9
0
 def test_add_talos_auxiliary(self):
     """Can add auxiliary results to a suite"""
     res = DatazillaResult()
     res.add_talos_auxiliary("suite", "name", [1, 2, 3])
     self.assertEqual(res.talos_aux, {"suite": {"name": [1, 2, 3]}})
예제 #10
0
    def test_add_test_results(self):
        """Can add results to a suite."""
        res = DatazillaResult()
        res.add_test_results("suite", "test", [1, 2, 3])

        self.assertEqual(res.results, {"suite": {"test": [1, 2, 3]}})
예제 #11
0
    def test_add_testsuite(self):
        """Can add a test suite with tests and results."""
        res = DatazillaResult()
        res.add_testsuite("suite", {"test1": [1, 2, 3]})

        self.assertEqual(res.results, {"suite": {"test1": [1, 2, 3]}})
예제 #12
0
    def test_init_noargs(self):
        """Can initialize with no argument and get empty initial results."""
        res = DatazillaResult()

        self.assertEqual(res.results, {})