def test_simplify(self) -> None:
self.assertEqual(
{
"": {
"Bar": {
"test_bar1": {
"seconds": 50.447,
"status": None
},
"test_bar2": {
"seconds": 3.742,
"status": "failed"
},
},
"Foo": {
"test_foo1": {
"seconds": 42,
"status": None
},
"test_foo2": {
"seconds": 56,
"status": None
},
},
"Grault": {
"test_grault0": {
"seconds": 4.78,
"status": "failed"
},
"test_grault2": {
"seconds": 1.473,
"status": "errored"
},
},
"Norf": {
"test_norf1": {
"seconds": 3,
"status": None
},
"test_norf3": {
"seconds": 3,
"status": None
},
"test_norf2": {
"seconds": 3,
"status": None
},
"test_norf4": {
"seconds": 3,
"status": None
},
},
"Qux": {
"test_qux1": {
"seconds": 0.001,
"status": "skipped"
},
"test_qux3": {
"seconds": 23.496,
"status": None
},
"test_qux4": {
"seconds": 7.158,
"status": "failed"
},
"test_qux5": {
"seconds": 11.968,
"status": None
},
"test_qux6": {
"seconds": 0.002,
"status": "skipped"
},
"test_qux7": {
"seconds": 0.003,
"status": "skipped"
},
},
},
},
print_test_stats.simplify(self.version1_report),
)
self.assertEqual(
{
"test_a": {
"Grault": {
"test_grault0": {
"seconds": 4.78,
"status": "failed"
},
"test_grault2": {
"seconds": 1.473,
"status": "errored"
},
},
"Qux": {
"test_qux1": {
"seconds": 0.001,
"status": "skipped"
},
"test_qux3": {
"seconds": 23.496,
"status": None
},
"test_qux4": {
"seconds": 7.158,
"status": "failed"
},
"test_qux6": {
"seconds": 0.002,
"status": "skipped"
},
"test_qux7": {
"seconds": 0.003,
"status": "skipped"
},
"test_qux8": {
"seconds": 11.968,
"status": None
},
},
},
"test_b": {
"Bar": {
"test_bar1": {
"seconds": 50.447,
"status": None
},
"test_bar2": {
"seconds": 3.742,
"status": "failed"
},
},
"Norf": {
"test_norf1": {
"seconds": 3,
"status": None
},
"test_norf2": {
"seconds": 3,
"status": None
},
"test_norf3": {
"seconds": 3,
"status": None
},
"test_norf4": {
"seconds": 3,
"status": None
},
},
},
"test_c": {
"Foo": {
"test_foo1": {
"seconds": 42,
"status": None
},
"test_foo2": {
"seconds": 56,
"status": None
},
},
},
},
print_test_stats.simplify(self.version2_report),
)
def test_analysis(self) -> None:
head_report = self.version1_report
base_reports: Dict[Commit, List[Report]] = {
# bbbb has no reports, so base is cccc instead
fakehash("b"): [],
fakehash("c"): [
make_report_v1({
"Baz": [
makecase("test_baz2", 13.605),
# no recent suites have & skip this test
makecase("test_baz1", 0.004, skipped=True),
],
"Foo": [
makecase("test_foo1", 43),
# test added since dddd
makecase("test_foo2", 57),
],
"Grault": [
makecase("test_grault0", 4.88, failed=True),
makecase("test_grault1", 11.967, failed=True),
makecase("test_grault2", 0.395, errored=True),
makecase("test_grault3", 30.460),
],
"Norf": [
makecase("test_norf1", 2),
makecase("test_norf2", 2),
makecase("test_norf3", 2),
makecase("test_norf4", 2),
],
"Qux": [
makecase("test_qux3", 4.978, errored=True),
makecase("test_qux7", 0.002, skipped=True),
makecase("test_qux2", 5.618),
makecase("test_qux4", 7.766, errored=True),
makecase("test_qux6", 23.589, failed=True),
],
}),
],
fakehash("d"): [
make_report_v1({
"Foo": [
makecase("test_foo1", 40),
# removed in cccc
makecase("test_foo3", 17),
],
"Baz": [
# not skipped, so not included in stdev
makecase("test_baz1", 3.14),
],
"Qux": [
makecase("test_qux7", 0.004, skipped=True),
makecase("test_qux2", 6.02),
makecase("test_qux4", 20.932),
],
"Norf": [
makecase("test_norf1", 3),
makecase("test_norf2", 3),
makecase("test_norf3", 3),
makecase("test_norf4", 3),
],
"Grault": [
makecase("test_grault0", 5, failed=True),
makecase("test_grault1", 14.325, failed=True),
makecase("test_grault2", 0.31, errored=True),
],
}),
],
fakehash("e"): [],
fakehash("f"): [
make_report_v1({
"Foo": [
makecase("test_foo3", 24),
makecase("test_foo1", 43),
],
"Baz": [
makecase("test_baz2", 16.857),
],
"Qux": [
makecase("test_qux2", 6.422),
makecase("test_qux4", 6.382, errored=True),
],
"Norf": [
makecase("test_norf1", 0.9),
makecase("test_norf3", 0.9),
makecase("test_norf2", 0.9),
makecase("test_norf4", 0.9),
],
"Grault": [
makecase("test_grault0", 4.7, failed=True),
makecase("test_grault1", 13.146, failed=True),
makecase("test_grault2", 0.48, errored=True),
],
}),
],
}
simpler_head = print_test_stats.simplify(head_report)
simpler_base = {}
for commit, reports in base_reports.items():
simpler_base[commit] = [
print_test_stats.simplify(r) for r in reports
]
analysis = print_test_stats.analyze(
head_report=simpler_head,
base_reports=simpler_base,
)
self.assertEqual(
"""\
- class Baz:
- # was 15.23s ± 2.30s
-
- def test_baz1: ...
- # was 0.004s (skipped)
-
- def test_baz2: ...
- # was 15.231s ± 2.300s
class Grault:
# was 48.86s ± 1.19s
# now 6.25s
- def test_grault1: ...
- # was 13.146s ± 1.179s (failed)
- def test_grault3: ...
- # was 30.460s
class Qux:
# was 41.66s ± 1.06s
# now 42.63s
- def test_qux2: ...
- # was 6.020s ± 0.402s
! def test_qux3: ...
! # was 4.978s (errored)
! # now 23.496s
! def test_qux4: ...
! # was 7.074s ± 0.979s (errored)
! # now 7.158s (failed)
! def test_qux6: ...
! # was 23.589s (failed)
! # now 0.002s (skipped)
+ def test_qux1: ...
+ # now 0.001s (skipped)
+ def test_qux5: ...
+ # now 11.968s
+ class Bar:
+ # now 54.19s
+
+ def test_bar1: ...
+ # now 50.447s
+
+ def test_bar2: ...
+ # now 3.742s (failed)
""",
print_test_stats.anomalies(analysis),
)
def test_simplify(self) -> None:
self.assertEqual(
{
'': {
'Bar': {
'test_bar1': {'seconds': 50.447, 'status': None},
'test_bar2': {'seconds': 3.742, 'status': 'failed'},
},
'Foo': {
'test_foo1': {'seconds': 42, 'status': None},
'test_foo2': {'seconds': 56, 'status': None},
},
'Grault': {
'test_grault0': {'seconds': 4.78, 'status': 'failed'},
'test_grault2': {'seconds': 1.473, 'status': 'errored'},
},
'Norf': {
'test_norf1': {'seconds': 3, 'status': None},
'test_norf3': {'seconds': 3, 'status': None},
'test_norf2': {'seconds': 3, 'status': None},
'test_norf4': {'seconds': 3, 'status': None},
},
'Qux': {
'test_qux1': {'seconds': 0.001, 'status': 'skipped'},
'test_qux3': {'seconds': 23.496, 'status': None},
'test_qux4': {'seconds': 7.158, 'status': 'failed'},
'test_qux5': {'seconds': 11.968, 'status': None},
'test_qux6': {'seconds': 0.002, 'status': 'skipped'},
'test_qux7': {'seconds': 0.003, 'status': 'skipped'},
},
},
},
print_test_stats.simplify(self.version1_report)
)
self.assertEqual(
{
'test_a': {
'Grault': {
'test_grault0': {'seconds': 4.78, 'status': 'failed'},
'test_grault2': {'seconds': 1.473, 'status': 'errored'},
},
'Qux': {
'test_qux1': {'seconds': 0.001, 'status': 'skipped'},
'test_qux3': {'seconds': 23.496, 'status': None},
'test_qux4': {'seconds': 7.158, 'status': 'failed'},
'test_qux6': {'seconds': 0.002, 'status': 'skipped'},
'test_qux7': {'seconds': 0.003, 'status': 'skipped'},
'test_qux8': {'seconds': 11.968, 'status': None},
},
},
'test_b': {
'Bar': {
'test_bar1': {'seconds': 50.447, 'status': None},
'test_bar2': {'seconds': 3.742, 'status': 'failed'},
},
'Norf': {
'test_norf1': {'seconds': 3, 'status': None},
'test_norf2': {'seconds': 3, 'status': None},
'test_norf3': {'seconds': 3, 'status': None},
'test_norf4': {'seconds': 3, 'status': None},
},
},
'test_c': {
'Foo': {
'test_foo1': {'seconds': 42, 'status': None},
'test_foo2': {'seconds': 56, 'status': None},
},
},
},
print_test_stats.simplify(self.version2_report),
)