Пример #1
0
    def process_test_data(self, test_data):
        """
        Convert JUnit output into a a list of report entries.

        :param test_data: JUnit test output.
        :type test_data: ``list``
        :return: list of sub reports.
        :rtype: ``list`` of (``TestGroupReport`` or ``TestCaseReport``)
        """
        result = []
        for suite in test_data:
            suite_name = suite.get("name", "JUnit testsuite")
            suite_report = TestGroupReport(
                name=suite_name,
                uid=suite_name,
                category=ReportCategories.TESTSUITE,
            )
            for case in suite.xpath("testcase"):
                case_name = case.get("name", "JUnit testcase")
                testcase_report = TestCaseReport(name=case_name, uid=case_name)
                for error in case.xpath("error"):
                    testcase_report.append(
                        registry.serialize(
                            assertions.Fail("Error executing test")
                        )
                    )
                    testcase_report.append(
                        registry.serialize(
                            CodeLog(
                                error.text,
                                language="java",
                                description="stacktrace",
                            )
                        )
                    )
                for failure in case.xpath("failure"):
                    message = failure.get("message", "testcase failure")
                    testcase_report.append(
                        registry.serialize(assertions.Fail(message))
                    )
                    if failure.text:
                        testcase_report.append(
                            registry.serialize(
                                CodeLog(
                                    failure.text,
                                    language="java",
                                    description="stacktrace",
                                )
                            )
                        )
                if not testcase_report.entries:
                    assertion_obj = assertions.RawAssertion(
                        description="Passed",
                        content="Testcase {} passed".format(case_name),
                        passed=True,
                    )
                    testcase_report.append(registry.serialize(assertion_obj))

                testcase_report.runtime_status = RuntimeStatus.FINISHED
                suite_report.append(testcase_report)
Пример #2
0
    def process_test_data(self, test_data):
        """
        XML output contains entries for skipped testcases
        as well, which are not included in the report.
        """
        result = []

        for suite in test_data.getchildren():
            suite_name = suite.attrib["name"]
            suite_report = TestGroupReport(
                name=suite_name,
                uid=suite_name,
                category=ReportCategories.TESTSUITE,
            )

            for testcase in suite.getchildren():
                if testcase.tag != "testcase":
                    continue

                testcase_classname = testcase.attrib["classname"]
                testcase_name = testcase.attrib["name"]
                testcase_prefix = testcase_classname.split(".")[-1]
                testcase_report = TestCaseReport(
                    name="{}::{}".format(testcase_prefix, testcase_name),
                    uid="{}::{}".format(testcase_classname.replace(".", "::"),
                                        testcase_name),
                )

                if not testcase.getchildren():
                    assertion_obj = RawAssertion(
                        description="Passed",
                        content="Testcase {} passed".format(testcase_name),
                        passed=True,
                    )
                    testcase_report.append(registry.serialize(assertion_obj))
                else:
                    for entry in testcase.getchildren():
                        assertion_obj = RawAssertion(
                            description=entry.tag,
                            content=entry.text,
                            passed=entry.tag not in ("failure", "error"),
                        )
                        testcase_report.append(
                            registry.serialize(assertion_obj))

                testcase_report.runtime_status = RuntimeStatus.FINISHED
                suite_report.append(testcase_report)

            if len(suite_report) > 0:
                result.append(suite_report)

        return result
Пример #3
0
    def process_test_data(self, test_data):
        """
        XML output contains entries for skipped testcases
        as well, which are not included in the report.
        """
        result = []

        for suite in test_data.getchildren():
            suite_name = suite.attrib["name"]
            suite_report = TestGroupReport(
                name=suite_name,
                uid=suite_name,
                category=ReportCategories.TESTSUITE,
            )
            suite_has_run = False

            for testcase in suite.getchildren():

                testcase_name = testcase.attrib["name"]
                testcase_report = TestCaseReport(
                    name=testcase_name, uid=testcase_name
                )

                if not testcase.getchildren():
                    assertion_obj = RawAssertion(
                        description="Passed",
                        content="Testcase {} passed".format(testcase_name),
                        passed=True,
                    )
                    testcase_report.append(registry.serialize(assertion_obj))
                else:
                    for entry in testcase.getchildren():
                        assertion_obj = RawAssertion(
                            description=entry.tag,
                            content=entry.text,
                            passed=entry.tag != "failure",
                        )
                        testcase_report.append(
                            registry.serialize(assertion_obj)
                        )

                testcase_report.runtime_status = RuntimeStatus.FINISHED

                if testcase.attrib["status"] != "notrun":
                    suite_report.append(testcase_report)
                    suite_has_run = True

            if suite_has_run:
                result.append(suite_report)

        return result
Пример #4
0
    def _process_data(self, data: Element) -> List[TestGroupReport]:
        """
        Processes data read from the source file.

        :param data: raw data as read by the importer
        """
        # NOTE: XML output contains skipped testcases which are ignored.
        result = []

        for suite in data.getchildren():
            suite_name = suite.attrib["name"]
            suite_report = TestGroupReport(
                name=suite_name,
                category=ReportCategories.TESTSUITE,
            )

            for testcase in suite.getchildren():
                if testcase.tag != "testcase":
                    continue

                testcase_classname = testcase.attrib["classname"]
                testcase_name = testcase.attrib["name"]
                testcase_prefix = testcase_classname.split(".")[-1]
                testcase_report = TestCaseReport(name="{}::{}".format(
                    testcase_prefix, testcase_name), )

                if not testcase.getchildren():
                    assertion_obj = RawAssertion(
                        description="Passed",
                        content=f"Testcase {testcase_name} passed",
                        passed=True,
                    )
                    testcase_report.append(registry.serialize(assertion_obj))
                else:
                    for entry in testcase.getchildren():
                        assertion_obj = RawAssertion(
                            description=entry.tag,
                            content=entry.text,
                            passed=entry.tag not in ("failure", "error"),
                        )
                        testcase_report.append(
                            registry.serialize(assertion_obj))

                testcase_report.runtime_status = RuntimeStatus.FINISHED
                suite_report.append(testcase_report)

            if len(suite_report) > 0:
                result.append(suite_report)

        return result
Пример #5
0
def test_create_pdf(tmpdir):
    """PDF exporter should generate a PDF file using the report data."""
    pdf_path = tmpdir.mkdir("reports").join("dummy_report.pdf").strpath

    assertion_entries = [
        assertions.Equal(1, 2),
        assertions.Greater(2, 1),
        assertions.IsFalse(True, "this should fail"),
        assertions.IsTrue(True, "this should pass"),
        assertions.Fail("Explicit failure"),
        base.Group(
            description="group description",
            entries=[
                assertions.NotEqual(2, 1),
                assertions.Contain(1, [1, 2, 3]),
            ],
        ),
    ]

    # Test large assertion
    for _ in range(10):
        assertion_entries.append(
            assertions.RegexMatch("Test.*", "Testplan\n" * 500))

    report = TestReport(
        name="my testplan",
        entries=[
            TestGroupReport(
                name="My Multitest",
                category=ReportCategories.MULTITEST,
                entries=[
                    TestGroupReport(
                        name="MySuite",
                        category=ReportCategories.TESTSUITE,
                        entries=[
                            TestCaseReport(
                                name="my_test_method",
                                entries=[
                                    registry.serialize(obj)
                                    for obj in assertion_entries
                                ],
                            )
                        ],
                    )
                ],
            )
        ],
    )

    exporter = PDFExporter(
        pdf_path=pdf_path,
        pdf_style=styles.Style(passing="assertion-detail",
                               failing="assertion-detail"),
    )

    with log_propagation_disabled(TESTPLAN_LOGGER):
        exporter.export(report)

    assert os.path.exists(pdf_path)
    assert os.stat(pdf_path).st_size > 0
Пример #6
0
    def pytest_runtest_logreport(self, report):
        """
        Hook called by pytest to report on the result of a test.

        :param report: the test report for the item just tested (see pytest
                       documentation)
        """
        if report.when == 'setup':
            if report.skipped and self._current_case_report is not None:
                # Status set to be SKIPPED if testcase is marked skip or xfail
                # lower versioned PyTest does not support this feature
                self._current_case_report.status_override = Status.SKIPPED

        elif report.when == 'call':
            if self._current_case_report is None:
                raise RuntimeError(
                    'Cannot store testcase results to report: no report '
                    'object was created.')

            # Add the assertion entry to the case report
            for entry in self._current_result_obj.entries:
                stdout_renderer = stdout_registry[entry]()
                stdout_header = stdout_renderer.get_header(entry)
                stdout_details = stdout_renderer.get_details(entry) or ''

                # Add 'stdout_header' and 'stdout_details' attributes to
                # serialized entries for standard output later
                serialized_entry = schema_registry.serialize(entry)
                serialized_entry.update(stdout_header=stdout_header,
                                        stdout_details=stdout_details)
                self._current_case_report.append(serialized_entry)
Пример #7
0
    def process_test_data(self, test_data):
        """
        XML output contains entries for skipped testcases
        as well, which are not included in the report.
        """
        result = []
        for suite in test_data.getchildren():
            suite_report = TestGroupReport(
                name=suite.attrib['name'],
                category='suite',
            )
            suite_has_run = False

            for testcase in suite.getchildren():

                testcase_report = TestCaseReport(name=testcase.attrib['name'])

                for entry in testcase.getchildren():
                    assertion_obj = RawAssertion(description=entry.tag,
                                                 content=entry.text,
                                                 passed=entry.tag != 'failure')
                    testcase_report.append(registry.serialize(assertion_obj))

                if testcase.attrib['status'] != 'notrun':
                    suite_report.append(testcase_report)
                    suite_has_run = True

            if suite_has_run:
                result.append(suite_report)
        return result
Пример #8
0
    def process_test_data(self, test_data):
        """
        JSON output contains entries for skipped testcases
        as well, which are not included in the report.
        """

        result = []
        for suite in test_data:
            suite_report = TestGroupReport(
                name=suite['name'],
                category='suite',
            )
            suite_has_run = False

            for testcase in suite['data']:
                if testcase['status'] != 'skipped':
                    suite_has_run = True

                    testcase_report = TestCaseReport(name=testcase['name'])
                    assertion_obj = RawAssertion(
                        passed=testcase['status'] == 'pass',
                        content=testcase['error'] or testcase['duration'],
                        description=testcase['name'])
                    testcase_report.append(registry.serialize(assertion_obj))
                    suite_report.append(testcase_report)

            if suite_has_run:
                result.append(suite_report)
        return result
Пример #9
0
    def process_test_data(self, test_data):
        """
        JSON output contains entries for skipped testcases
        as well, which are not included in the report.
        """

        result = []
        for suite in test_data:
            suite_report = TestGroupReport(name=suite["name"],
                                           uid=suite["name"],
                                           category="testsuite")
            suite_has_run = False

            for testcase in suite["data"]:
                if testcase["status"] != "skipped":
                    suite_has_run = True

                    testcase_report = TestCaseReport(
                        name=testcase["name"],
                        uid=testcase["name"],
                        suite_related=True,
                    )
                    assertion_obj = RawAssertion(
                        passed=testcase["status"] == "pass",
                        content=testcase["error"] or testcase["duration"],
                        description=testcase["name"],
                    )
                    testcase_report.append(registry.serialize(assertion_obj))
                    testcase_report.runtime_status = RuntimeStatus.FINISHED
                    suite_report.append(testcase_report)

            if suite_has_run:
                result.append(suite_report)
        return result
Пример #10
0
    def process_test_data(self, test_data):
        """
        XML output contains entries for skipped testcases
        as well, which are not included in the report.
        """
        result = []
        for suite in test_data.getchildren():
            suite_report = TestGroupReport(name=suite.attrib["name"],
                                           category="testsuite")
            suite_has_run = False

            for testcase in suite.getchildren():

                testcase_report = TestCaseReport(name=testcase.attrib["name"])

                if not testcase.getchildren():
                    testcase_report.status_override = Status.PASSED

                for entry in testcase.getchildren():
                    assertion_obj = RawAssertion(
                        description=entry.tag,
                        content=entry.text,
                        passed=entry.tag != "failure",
                    )
                    testcase_report.append(registry.serialize(assertion_obj))

                if testcase.attrib["status"] != "notrun":
                    suite_report.append(testcase_report)
                    suite_has_run = True

            if suite_has_run:
                result.append(suite_report)
        return result
Пример #11
0
def test_create_pdf(tmpdir):
    """PDF exporter should generate a PDF file using the report data."""
    pdf_path = tmpdir.mkdir('reports').join('dummy_report.pdf').strpath

    assertion_entries = [
        assertions.Equal(1, 2),
        assertions.Greater(2, 1),
        assertions.IsFalse(True, 'this should fail'),
        assertions.IsTrue(True, 'this should pass'),
        assertions.Fail('Explicit failure'),
        base.Group(
            description='group description',
            entries=[
                assertions.NotEqual(2, 1),
                assertions.Contain(1, [1, 2, 3]),
            ]
        )
    ]

    report = TestReport(
        name='my testplan',
        entries=[
            TestGroupReport(
                name='My Multitest',
                category='multitest',
                entries=[
                    TestGroupReport(
                        name='MySuite',
                        entries=[
                            TestCaseReport(
                                name='my_test_method',
                                entries=[
                                    registry.serialize(obj)
                                    for obj in assertion_entries
                                ]
                            )
                        ]
                    )
                ]
            )
        ]
    )

    exporter = PDFExporter(
        pdf_path=pdf_path,
        pdf_style=styles.Style(
            passing='assertion-detail',
            failing='assertion-detail'
        )
    )

    with log_propagation_disabled(TESTPLAN_LOGGER):
        exporter.export(report)

    assert os.path.exists(pdf_path)
    assert os.stat(pdf_path).st_size > 0
Пример #12
0
    def pytest_exception_interact(self, node, call, report):
        """
        Hook called when an exception raised and it can be handled. This hook
        is only called if the exception is not an PyTest internal exception.

        :param node: PyTest Function or Module object
        :param call: PyTest CallInfo object
        :param report: PyTest TestReport or CollectReport object
        """
        if call.when in ('memocollect', 'collect'):
            # Failed to collect tests: log to console and mark the report as
            # ERROR.
            self._report.logger.error(format_trace(
                inspect.getinnerframes(call.excinfo.tb), call.excinfo.value))
            self._report.status_override = Status.ERROR

        elif self._current_case_report is not None:
            # Log assertion errors or exceptions in testcase report
            traceback = call.excinfo.traceback[-1]
            message = getattr(call.excinfo.value, 'message', None) or \
                      getattr(call.excinfo.value, 'msg', None) or \
                     getattr(call.excinfo.value, 'args', None) or ''
            if isinstance(message, (tuple, list)):
                message = message[0]

            header = (('Assertion - Fail' if
                call.excinfo.typename == 'AssertionError' else
                'Exception raised') if call.when == 'call' else
                    '{} - Fail'.format(call.when))
            details = 'File: {}{}Line: {}{}{}: {}'.format(
                traceback.path.strpath,
                os.linesep,
                traceback.lineno + 1,
                os.linesep,
                call.excinfo.typename,
                message
            ) if call.excinfo.typename == 'AssertionError' else (
                report.longreprtext if hasattr(report, 'longreprtext') else
                    str(report.longrepr))

            assertion_obj = assertions.RawAssertion(
                description=header,
                content=details,
                passed=False
            )
            serialized_obj = schema_registry.serialize(assertion_obj)
            self._current_case_report.append(serialized_obj)
            self._current_case_report.status_override = Status.FAILED
        else:
            self._report.logger.error(
                'Exception occured outside of a testcase: during %s',
                call.when)
            self._report.logger.error(format_trace(
                inspect.getinnerframes(call.excinfo.tb), call.excinfo.value))
Пример #13
0
    def pytest_exception_interact(self, node, call, report):
        """
        Hook called when an exception raised and it can be handled. This hook
        is only called if the exception is not an PyTest internal exception.

        :param node: PyTest Function or Module object
        :param call: PyTest CallInfo object
        :param report: PyTest TestReport or CollectReport object
        """
        if call.when in ("memocollect", "collect"):
            # Failed to collect tests: log to console and mark the report as
            # ERROR.
            self._report.logger.error("".join(
                traceback.format_exception(call.excinfo.type,
                                           call.excinfo.value,
                                           call.excinfo.tb)))
            self._report.status_override = Status.ERROR

        elif self._current_case_report is not None:
            # Log assertion errors or exceptions in testcase report
            trace = call.excinfo.traceback[-1]
            message = (getattr(call.excinfo.value, "message", None)
                       or getattr(call.excinfo.value, "msg", None)
                       or getattr(call.excinfo.value, "args", None) or "")
            if isinstance(message, (tuple, list)):
                message = message[0]

            header = (("Assertion - Fail" if call.excinfo.typename
                       == "AssertionError" else "Exception raised") if
                      call.when == "call" else "{} - Fail".format(call.when))
            details = ("File: {}\nLine: {}\n{}: {}".format(
                trace.path.strpath,
                trace.lineno + 1,
                call.excinfo.typename,
                message,
            ) if call.excinfo.typename == "AssertionError" else
                       (report.longreprtext if hasattr(report, "longreprtext")
                        else str(report.longrepr)))

            assertion_obj = assertions.RawAssertion(description=header,
                                                    content=details,
                                                    passed=False)
            serialized_obj = schema_registry.serialize(assertion_obj)
            self._current_case_report.append(serialized_obj)
            self._current_case_report.status_override = Status.FAILED
        else:
            self._report.logger.error(
                "Exception occured outside of a testcase: during %s",
                call.when)
            self._report.logger.error("".join(
                traceback.format_exception(call.excinfo.type,
                                           call.excinfo.value,
                                           call.excinfo.tb)))
Пример #14
0
    def pytest_runtest_logreport(self, report):
        """
        Hook called by pytest to report on the result of a test.

        :param report: the test report for the item just tested (see pytest
                       documentation)
        """
        if report.when == "setup":
            if report.skipped:
                if self._current_case_report is None:
                    suite_name, case_name, case_params = _case_parse(
                        report.nodeid)
                    testcase_report = self.case_report(suite_name, case_name,
                                                       case_params)
                else:
                    testcase_report = self._current_case_report

                # Status set to be SKIPPED if testcase is marked skip or xfail
                # lower versioned PyTest does not support this feature
                testcase_report.status_override = Status.SKIPPED

        elif report.when == "call":
            if self._current_case_report is None:
                raise RuntimeError(
                    "Cannot store testcase results to report: no report "
                    "object was created.")

            if self._current_result_obj.entries:
                # Add the assertion entry to the case report
                for entry in self._current_result_obj.entries:
                    stdout_renderer = stdout_registry[entry]()
                    stdout_header = stdout_renderer.get_header(entry)
                    stdout_details = stdout_renderer.get_details(entry) or ""

                    # Add 'stdout_header' and 'stdout_details' attributes to
                    # serialized entries for standard output later
                    serialized_entry = schema_registry.serialize(entry)
                    serialized_entry.update(
                        stdout_header=stdout_header,
                        stdout_details=stdout_details,
                    )
                    self._current_case_report.append(serialized_entry)

            if report.failed:
                self._current_case_report.status_override = Status.FAILED
            else:
                self._current_case_report.pass_if_empty()
            self._current_case_report.runtime_status = RuntimeStatus.FINISHED

        elif report.when == "teardown":
            pass
Пример #15
0
    def _process_data(self, data: Element) -> List[TestGroupReport]:
        """
        Processes data read from the source file.

        :param data: raw data as read by the importer
        """
        result = []

        suites = data.getchildren() if data.tag == "testsuites" else [data]

        for suite in suites:
            suite_name = suite.attrib.get("name")
            suite_report = TestGroupReport(
                name=suite_name,
                category=ReportCategories.TESTSUITE,
            )

            for element in suite.getchildren():
                # Elements like properties, system-out, and system-err are
                # skipped.
                if element.tag != "testcase":
                    continue

                case_class = element.attrib.get("classname")
                case_name = element.attrib.get("name")

                if case_class is None:
                    if case_name == suite_report.name:
                        path = os.path.normpath(case_name)
                        suite_report.name = path.rpartition(os.sep)[-1]
                        # We use the name "Execution" to avoid collision of
                        # test suite and test case.
                        case_report_name = "Execution"
                    else:
                        case_report_name = case_name
                else:
                    case_report_name = (
                        f"{case_class.split('.')[-1]}::{case_name}")

                case_report = TestCaseReport(name=case_report_name)

                if not element.getchildren():
                    assertion = RawAssertion(
                        description="Passed",
                        content=f"Testcase {case_name} passed",
                        passed=True,
                    )
                    case_report.append(registry.serialize(assertion))
                else:
                    # Upon a failure, there will be a single testcase which is
                    # the first child.
                    content, tag, desc = "", None, None
                    for child in element.getchildren():
                        tag = tag or child.tag
                        msg = child.attrib.get("message") or child.text
                        # Approach: if it is a failure/error child, then use
                        # the message attribute directly.
                        # Otherwise, for instance if it is a system-err/out,
                        # then build up the content step by step from it.
                        if not desc and child.tag in ("failure", "error"):
                            desc = msg
                        else:
                            content += f"[{child.tag}]\n{msg}\n"
                    assertion = RawAssertion(
                        description=desc,
                        content=content,
                        passed=tag not in ("failure", "error"),
                    )
                    case_report.append(registry.serialize(assertion))

                suite_report.runtime_status = RuntimeStatus.FINISHED
                suite_report.append(case_report)

            if len(suite_report):
                result.append(suite_report)

        return result