示例#1
0
    def test_from_file(self):
        project = self.create_project()
        build = self.create_build(project)
        job = self.create_job(build)
        jobphase = self.create_jobphase(job)
        jobstep = self.create_jobstep(jobphase)
        logger = logging.getLogger('xunit')

        path = os.path.join(os.path.dirname(__file__), 'fixtures',
                            'junit.xml.test')

        with open(path, 'rb') as fp:
            contents = fp.read()

        start = time.clock()
        handler = XunitHandler(jobstep)
        results = handler.get_tests(StringIO(contents))
        logger.info("XUnit handler ran in %f seconds." %
                    (time.clock() - start))

        assert len(results) == 675
        assert results[
            0].name == 'tests.changes.api.test_build_details.TestCase'
        assert results[0].result == Result.skipped
        assert results[
            674].name == 'tests.changes.web.test_auth.LogoutViewTest.test_simple'
        assert results[674].result == Result.passed
示例#2
0
    def test_invalid_junit(self):
        project = self.create_project()
        build = self.create_build(project)
        job = self.create_job(build)
        jobphase = self.create_jobphase(job)
        jobstep = self.create_jobstep(jobphase)
        artifact = self.create_artifact(jobstep, 'junit.xml')

        missing_name = """<?xml version="1.0" encoding="utf-8"?>
        <testsuite errors="1" failures="0" name="" skips="0" tests="0" time="0.077">
            <testcase classname="" time="0" owner="foo">
                <failure message="collection failure">tests/test_report.py:1: in &lt;module&gt;
        &gt;   import mock
        E   ImportError: No module named mock</failure>
            </testcase>
            <testcase classname="tests.test_report.ParseTestResultsTest" name="test_simple" time="0.001607" rerun="1"/>
        </testsuite>"""
        fp = StringIO(missing_name)

        handler = XunitHandler(jobstep)
        results = handler.process(fp, artifact)
        assert results == []
        reason = FailureReason.query.filter(
            FailureReason.step_id == jobstep.id).first()
        assert reason is not None
示例#3
0
def test_get_test_suite_multiple():
    jobstep = JobStep(
        id=uuid.uuid4(),
        project_id=uuid.uuid4(),
        job_id=uuid.uuid4(),
    )
    # needed for logging when a test suite has no duration
    jobstep.job = mock.MagicMock()

    fp = StringIO(SAMPLE_XUNIT_MULTIPLE_SUITES)

    handler = XunitHandler(jobstep)
    suites = handler.get_test_suites(fp)
    assert len(suites) == 3
    assert suites[0].name is None
    assert suites[0].duration is None
    assert suites[0].result == Result.failed
    assert suites[0].date_created is not None
    assert len(suites[0].test_results) == 3

    assert suites[1].name == 'suite2'
    assert suites[1].duration == 77
    assert suites[1].result == Result.failed
    assert suites[1].date_created is not None
    assert len(suites[1].test_results) == 3

    assert suites[2].name == ''
    assert suites[2].duration is None
    assert suites[2].result == Result.failed
    assert suites[2].date_created is not None
    assert len(suites[2].test_results) == 3

    tests = handler.aggregate_tests_from_suites(suites)
    assert len(tests) == 7  # 10 test cases, 3 of which are duplicates
示例#4
0
def test_result_generation():
    jobstep = JobStep(
        id=uuid.uuid4(),
        project_id=uuid.uuid4(),
        job_id=uuid.uuid4(),
    )

    fp = StringIO(SAMPLE_XUNIT)

    handler = XunitHandler(jobstep)
    results = handler.get_tests(fp)

    assert len(results) == 2

    r1 = results[0]
    assert type(r1) == TestResult
    assert r1.step == jobstep
    assert r1.package is None
    assert r1.name == 'tests.test_report'
    assert r1.duration == 0.0
    assert r1.result == Result.failed
    assert r1.message == """tests/test_report.py:1: in <module>
>   import mock
E   ImportError: No module named mock"""
    r2 = results[1]
    assert type(r2) == TestResult
    assert r2.step == jobstep
    assert r2.package is None
    assert r2.name == 'tests.test_report.ParseTestResultsTest.test_simple'
    assert r2.duration == 1.65796279907
    assert r2.result == Result.passed
    assert r2.message == ''
    assert r2.reruns == 1
示例#5
0
    def test_invalid_junit(self):
        project = self.create_project()
        build = self.create_build(project)
        job = self.create_job(build)
        jobphase = self.create_jobphase(job)
        jobstep = self.create_jobstep(jobphase)
        artifact = self.create_artifact(jobstep, 'junit.xml')

        missing_name = """<?xml version="1.0" encoding="utf-8"?>
        <testsuite errors="1" failures="0" name="" skips="0" tests="0" time="0.077">
            <testcase classname="" time="0" owner="foo">
                <failure message="collection failure">tests/test_report.py:1: in &lt;module&gt;
        &gt;   import mock
        E   ImportError: No module named mock</failure>
            </testcase>
            <testcase classname="tests.test_report.ParseTestResultsTest" name="test_simple" time="0.001607" rerun="1"/>
        </testsuite>"""
        fp = StringIO(missing_name)

        handler = XunitHandler(jobstep)
        results = handler.process(fp, artifact)
        assert results == []
        reason = FailureReason.query.filter(
            FailureReason.step_id == jobstep.id
        ).first()
        assert reason is not None
示例#6
0
def test_get_test_suite_multiple_empty(xml, result):
    jobstep = JobStep(
        id=uuid.uuid4(),
        project_id=uuid.uuid4(),
        job_id=uuid.uuid4(),
    )
    # needed for logging when a test suite has no duration
    jobstep.job = mock.MagicMock()

    fp = StringIO(xml)

    handler = XunitHandler(jobstep)
    suites = handler.get_test_suites(fp)
    assert len(suites) == 2
    assert suites[0].name == 'suite-name'
    assert suites[0].duration is None
    assert suites[0].result is result
    assert suites[0].date_created is not None
    assert len(suites[0].test_results) == 0

    assert suites[1].name == 'null'
    assert suites[1].duration is None
    assert suites[1].result == Result.passed
    assert suites[1].date_created is not None
    assert len(suites[1].test_results) == 1
示例#7
0
def test_result_generation():
    jobstep = JobStep(
        id=uuid.uuid4(),
        project_id=uuid.uuid4(),
        job_id=uuid.uuid4(),
    )

    fp = StringIO(SAMPLE_XUNIT)

    handler = XunitHandler(jobstep)
    results = handler.get_tests(fp)

    assert len(results) == 2

    r1 = results[0]
    assert type(r1) == TestResult
    assert r1.step == jobstep
    assert r1.package is None
    assert r1.name == 'tests.test_report'
    assert r1.duration == 0.0
    assert r1.result == Result.failed
    assert r1.message == """tests/test_report.py:1: in <module>
>   import mock
E   ImportError: No module named mock"""
    assert r1.owner == 'foo'
    r2 = results[1]
    assert type(r2) == TestResult
    assert r2.step == jobstep
    assert r2.package is None
    assert r2.name == 'tests.test_report.ParseTestResultsTest.test_simple'
    assert r2.duration == 1.65796279907
    assert r2.result == Result.passed
    assert r2.message == ''
    assert r2.reruns == 1
    assert r2.owner is None
示例#8
0
def test_result_generation_when_a_quarantined_test_has_two_cases():
    jobstep = JobStep(
        id=uuid.uuid4(),
        project_id=uuid.uuid4(),
        job_id=uuid.uuid4(),
    )

    XUNIT_STRING = (SAMPLE_XUNIT_DOUBLE_CASES.replace(
        '<testcase c', '<testcase quarantined="1" c',
        1).replace('<testcase c', '<testcase quarantined="1" c', 1))
    fp = StringIO(XUNIT_STRING)

    handler = XunitHandler(jobstep)
    results = handler.get_tests(fp)

    assert len(results) == 2

    r1 = results[0]
    assert type(r1) is TestResult
    assert r1.step == jobstep
    assert r1.package is None
    assert r1.name == 'test_simple.SampleTest.test_falsehood'
    assert r1.duration == 750.0
    assert r1.result == Result.quarantined_failed
    assert r1.message == ''
    assert len(r1.message_offsets) == 2
    label, start, length = r1.message_offsets[0]
    assert label == 'failure'
    assert XUNIT_STRING[start:start + length] == """\
test_simple.py:8: in test_falsehood
    assert False
E   AssertionError: assert False"""
    label, start, length = r1.message_offsets[1]
    assert label == 'error'
    assert XUNIT_STRING[start:start + length] == """\
test_simple.py:4: in tearDown
    1/0
E   ZeroDivisionError: integer division or modulo by zero"""
    assert r1.reruns == 3

    r2 = results[1]
    assert type(r2) is TestResult
    assert r2.step == jobstep
    assert r2.package is None
    assert r2.name == 'test_simple.SampleTest.test_truth'
    assert r2.duration == 1250.0
    assert r2.result == Result.failed
    assert r2.message is None
    assert len(r2.message_offsets) == 1
    label, start, length = r2.message_offsets[0]
    assert label == 'failure'
    assert XUNIT_STRING[start:start + length] == """\
test_simple.py:4: in tearDown
    1/0
E   ZeroDivisionError: integer division or modulo by zero"""
    assert r2.reruns == 0
示例#9
0
def test_result_generation_when_a_quarantined_test_has_two_cases():
    jobstep = JobStep(
        id=uuid.uuid4(),
        project_id=uuid.uuid4(),
        job_id=uuid.uuid4(),
    )

    XUNIT_STRING = (SAMPLE_XUNIT_DOUBLE_CASES
                    .replace('<testcase c', '<testcase quarantined="1" c', 1)
                    .replace('<testcase c', '<testcase quarantined="1" c', 1))
    fp = StringIO(XUNIT_STRING)

    handler = XunitHandler(jobstep)
    results = handler.get_tests(fp)

    assert len(results) == 2

    r1 = results[0]
    assert type(r1) is TestResult
    assert r1.step == jobstep
    assert r1.package is None
    assert r1.name == 'test_simple.SampleTest.test_falsehood'
    assert r1.duration == 750.0
    assert r1.result == Result.quarantined_failed
    assert r1.message == ''
    assert len(r1.message_offsets) == 2
    label, start, length = r1.message_offsets[0]
    assert label == 'failure'
    assert XUNIT_STRING[start:start + length] == """\
test_simple.py:8: in test_falsehood
    assert False
E   AssertionError: assert False"""
    label, start, length = r1.message_offsets[1]
    assert label == 'error'
    assert XUNIT_STRING[start:start + length] == """\
test_simple.py:4: in tearDown
    1/0
E   ZeroDivisionError: integer division or modulo by zero"""
    assert r1.reruns == 3

    r2 = results[1]
    assert type(r2) is TestResult
    assert r2.step == jobstep
    assert r2.package is None
    assert r2.name == 'test_simple.SampleTest.test_truth'
    assert r2.duration == 1250.0
    assert r2.result == Result.failed
    assert r2.message is None
    assert len(r2.message_offsets) == 1
    label, start, length = r2.message_offsets[0]
    assert label == 'failure'
    assert XUNIT_STRING[start:start + length] == """\
test_simple.py:4: in tearDown
    1/0
E   ZeroDivisionError: integer division or modulo by zero"""
    assert r2.reruns == 0
示例#10
0
def test_bad_encoding():
    jobstep = JobStep(
        id=uuid.uuid4(),
        project_id=uuid.uuid4(),
        job_id=uuid.uuid4(),
    )

    fp = StringIO(SAMPLE_XUNIT.replace('"utf-8"', '"utf8"'))

    handler = XunitHandler(jobstep)
    results = handler.get_tests(fp)

    assert len(results) == 3
示例#11
0
def test_bad_encoding():
    jobstep = JobStep(
        id=uuid.uuid4(),
        project_id=uuid.uuid4(),
        job_id=uuid.uuid4(),
    )

    fp = StringIO(SAMPLE_XUNIT.replace('"utf-8"', '"utf8"'))

    handler = XunitHandler(jobstep)
    results = handler.get_tests(fp)

    assert len(results) == 3
示例#12
0
def test_result_generation_when_a_quarantined_test_has_two_cases():
    jobstep = JobStep(id=uuid.uuid4(), project_id=uuid.uuid4(), job_id=uuid.uuid4())

    fp = StringIO(
        SAMPLE_XUNIT_DOUBLE_CASES.replace("<testcase c", '<testcase quarantined="1" c', 1).replace(
            "<testcase c", '<testcase quarantined="1" c', 1
        )
    )

    handler = XunitHandler(jobstep)
    results = handler.get_tests(fp)

    assert len(results) == 2

    r1 = results[0]
    assert type(r1) is TestResult
    assert r1.step == jobstep
    assert r1.package is None
    assert r1.name == "test_simple.SampleTest.test_falsehood"
    assert r1.duration == 750.0
    assert r1.result == Result.quarantined_failed
    assert (
        r1.message
        == """\
test_simple.py:8: in test_falsehood
    assert False
E   AssertionError: assert False

test_simple.py:4: in tearDown
    1/0
E   ZeroDivisionError: integer division or modulo by zero"""
    )
    assert r1.reruns == 3

    r2 = results[1]
    assert type(r2) is TestResult
    assert r2.step == jobstep
    assert r2.package is None
    assert r2.name == "test_simple.SampleTest.test_truth"
    assert r2.duration == 1250.0
    assert r2.result == Result.failed
    assert (
        r2.message
        == """\
test_simple.py:4: in tearDown
    1/0
E   ZeroDivisionError: integer division or modulo by zero"""
    )
    assert r2.reruns == 0
示例#13
0
    def _sync_artifact_as_xunit(self, artifact):
        jobstep = artifact.step
        resp = self.fetch_artifact(jobstep, artifact.data)

        # TODO(dcramer): requests doesnt seem to provide a non-binary file-like
        # API, so we're stuffing it into StringIO
        try:
            handler = XunitHandler(jobstep)
            handler.process(StringIO(resp.content))
        except Exception:
            db.session.rollback()
            self.logger.exception(
                'Failed to sync test results for job step %s', jobstep.id)
        else:
            db.session.commit()
示例#14
0
    def _sync_artifact_as_xunit(self, artifact):
        jobstep = artifact.step
        resp = self.fetch_artifact(jobstep, artifact.data)

        # TODO(dcramer): requests doesnt seem to provide a non-binary file-like
        # API, so we're stuffing it into StringIO
        try:
            handler = XunitHandler(jobstep)
            handler.process(StringIO(resp.content))
        except Exception:
            db.session.rollback()
            self.logger.exception(
                'Failed to sync test results for job step %s', jobstep.id)
        else:
            db.session.commit()
示例#15
0
def test_result_generation_when_one_test_has_two_cases():
    jobstep = JobStep(
        id=uuid.uuid4(),
        project_id=uuid.uuid4(),
        job_id=uuid.uuid4(),
    )

    fp = StringIO(SAMPLE_XUNIT_DOUBLE_CASES)

    handler = XunitHandler(jobstep)
    results = handler.get_tests(fp)

    assert len(results) == 2

    r1 = results[0]
    assert type(r1) is TestResult
    assert r1.step == jobstep
    assert r1.package is None
    assert r1.name == 'test_simple.SampleTest.test_falsehood'
    assert r1.duration == 750.0
    assert r1.result == Result.failed
    assert r1.message == """\
test_simple.py:8: in test_falsehood
    assert False
E   AssertionError: assert False

test_simple.py:4: in tearDown
    1/0
E   ZeroDivisionError: integer division or modulo by zero"""
    assert r1.reruns == 0

    r2 = results[1]
    assert type(r2) is TestResult
    assert r2.step == jobstep
    assert r2.package is None
    assert r2.name == 'test_simple.SampleTest.test_truth'
    assert r2.duration == 1250.0
    assert r2.result == Result.failed
    assert r2.message == """\
test_simple.py:4: in tearDown
    1/0
E   ZeroDivisionError: integer division or modulo by zero"""
    assert r2.reruns == 0
示例#16
0
    def test_from_file(self):
        project = self.create_project()
        build = self.create_build(project)
        job = self.create_job(build)
        jobphase = self.create_jobphase(job)
        jobstep = self.create_jobstep(jobphase)
        logger = logging.getLogger('xunit')

        path = os.path.join(os.path.dirname(__file__), 'fixtures', 'junit.xml.test')

        with open(path, 'rb') as fp:
            contents = fp.read()

        start = time.clock()
        handler = XunitHandler(jobstep)
        results = handler.get_tests(StringIO(contents))
        logger.info("XUnit handler ran in %f seconds." % (time.clock() - start))

        assert len(results) == 675
        assert results[0].name == 'tests.changes.api.test_build_details.TestCase'
        assert results[0].result == Result.skipped
        assert results[674].name == 'tests.changes.web.test_auth.LogoutViewTest.test_simple'
        assert results[674].result == Result.passed
示例#17
0
def test_get_test_single_suite(xml, suite_name, duration):
    jobstep = JobStep(
        id=uuid.uuid4(),
        project_id=uuid.uuid4(),
        job_id=uuid.uuid4(),
    )

    fp = StringIO(xml)

    handler = XunitHandler(jobstep)
    suites = handler.get_test_suites(fp)
    assert len(suites) == 1
    assert suites[0].name == suite_name
    assert suites[0].duration == duration
    assert suites[0].result == Result.failed
    assert suites[0].date_created is not None

    # test the equivalence of get_tests and get_test_suites in the case where
    # there is only one test suite, so that we can call get_tests directly
    # in the rest of this file.
    fp.seek(0)
    other_results = handler.get_tests(fp)

    results = suites[0].test_results
    assert len(results) == len(other_results)
    for i in range(len(results)):
        assert other_results[i].step == results[i].step
        assert other_results[i].step == results[i].step
        assert other_results[i]._name == results[i]._name
        assert other_results[i]._package == results[i]._package
        assert other_results[i].message == results[i].message
        assert other_results[i].result is results[i].result
        assert other_results[i].duration == results[i].duration
        assert other_results[i].reruns == results[i].reruns
        assert other_results[i].artifacts == results[i].artifacts
        assert other_results[i].owner == results[i].owner
        assert other_results[i].message_offsets == results[i].message_offsets
示例#18
0
    def sync_artifact(self, artifact, skip_checks=False):
        if not skip_checks:
            if artifact.name.endswith(".log") and not self.sync_log_artifacts:
                return

            elif XunitHandler.can_process(artifact.name) and not self.sync_xunit_artifacts:
                return

            elif CoverageHandler.can_process(artifact.name) and not self.sync_coverage_artifacts:
                return

            elif not self.sync_file_artifacts:
                return

        for cls in [XunitHandler, CoverageHandler, ManifestJsonHandler]:
            if cls.can_process(artifact.name):
                self._sync_artifact_as_file(artifact, handler_cls=cls)
                break
        else:
            if artifact.name.endswith(".log"):
                self._sync_artifact_as_log(artifact)

            else:
                self._sync_artifact_as_file(artifact)
示例#19
0
def test_result_generation():
    jobstep = JobStep(
        id=uuid.uuid4(),
        project_id=uuid.uuid4(),
        job_id=uuid.uuid4(),
    )

    fp = StringIO(SAMPLE_XUNIT)

    handler = XunitHandler(jobstep)
    results = handler.get_tests(fp)

    assert len(results) == 3

    r1 = results[0]
    assert type(r1) == TestResult
    assert r1.step == jobstep
    assert r1.package is None
    assert r1.name == 'tests.test_report'
    assert r1.duration == 0.0
    assert r1.result == Result.failed
    assert r1.message is None
    assert len(r1.message_offsets) == 1
    label, start, length = r1.message_offsets[0]
    assert label == 'failure'
    assert SAMPLE_XUNIT[start:start + length] == """\
tests/test_report.py:1: in &lt;module&gt;
&gt;   import mock
E   ImportError: No module named mock"""
    assert r1.owner == 'foo'
    r2 = results[1]
    assert type(r2) == TestResult
    assert r2.step == jobstep
    assert r2.package is None
    assert r2.name == 'tests.test_report.ParseTestResultsTest.test_simple'
    assert r2.duration == 1.65796279907
    assert r2.result == Result.passed
    assert r2.message is None
    assert r2.message_offsets == []
    assert r2.reruns == 1
    assert r2.owner is None
    r3 = results[2]
    assert type(r3) == TestResult
    assert r3.step == jobstep
    assert r3.package is None
    assert r3.name == 'test_simple.SampleTest.test_falsehood'
    assert r3.duration == 500.0
    assert r3.result == Result.passed
    assert r3.message is None
    assert len(r3.message_offsets) == 3
    label, start, length = r3.message_offsets[0]
    assert label == 'system-out'
    assert SAMPLE_XUNIT[start:start + length] == 'Running SampleTest'
    label, start, length = r3.message_offsets[1]
    assert label == 'error'
    assert SAMPLE_XUNIT[start:start + length] == """\
test_simple.py:4: in tearDown
    1/0
E   ZeroDivisionError: integer division or modulo by zero"""
    label, start, length = r3.message_offsets[2]
    assert label == 'system-out'
    assert SAMPLE_XUNIT[start:start + length] == 'Running SampleTest'
    assert r3.reruns == 3
    assert r3.owner is None
示例#20
0
def test_result_generation():
    jobstep = JobStep(
        id=uuid.uuid4(),
        project_id=uuid.uuid4(),
        job_id=uuid.uuid4(),
    )

    fp = StringIO(SAMPLE_XUNIT)

    handler = XunitHandler(jobstep)
    results = handler.get_tests(fp)

    assert len(results) == 3

    r1 = results[0]
    assert type(r1) == TestResult
    assert r1.step == jobstep
    assert r1.package is None
    assert r1.name == 'tests.test_report'
    assert r1.duration == 0.0
    assert r1.result == Result.failed
    assert r1.message is None
    assert len(r1.message_offsets) == 1
    label, start, length = r1.message_offsets[0]
    assert label == 'failure'
    assert SAMPLE_XUNIT[start:start + length] == """\
tests/test_report.py:1: in &lt;module&gt;
&gt;   import mock
E   ImportError: No module named mock"""
    assert r1.owner == 'foo'
    r2 = results[1]
    assert type(r2) == TestResult
    assert r2.step == jobstep
    assert r2.package is None
    assert r2.name == 'tests.test_report.ParseTestResultsTest.test_simple'
    assert r2.duration == 1.65796279907
    assert r2.result == Result.passed
    assert r2.message is None
    assert r2.message_offsets == []
    assert r2.reruns == 1
    assert r2.owner is None
    r3 = results[2]
    assert type(r3) == TestResult
    assert r3.step == jobstep
    assert r3.package is None
    assert r3.name == 'test_simple.SampleTest.test_falsehood'
    assert r3.duration == 500.0
    assert r3.result == Result.passed
    assert r3.message is None
    assert len(r3.message_offsets) == 3
    label, start, length = r3.message_offsets[0]
    assert label == 'system-out'
    assert SAMPLE_XUNIT[start:start + length] == 'Running SampleTest'
    label, start, length = r3.message_offsets[1]
    assert label == 'error'
    assert SAMPLE_XUNIT[start:start + length] == """\
test_simple.py:4: in tearDown
    1/0
E   ZeroDivisionError: integer division or modulo by zero"""
    label, start, length = r3.message_offsets[2]
    assert label == 'system-out'
    assert SAMPLE_XUNIT[start:start + length] == 'Running SampleTest'
    assert r3.reruns == 3
    assert r3.owner is None