def test_trybot_successful_run_count(self):
self.analyze_records(*self.trybot_records)
self.assertEquals(
self.create_count(
name='trybot-test_builder_ppp-pass-count',
description=
'Number of passing runs by the test_builder_ppp trybot.',
tally={PatchsetReference(1, 1): 1},
), self.get_stats('trybot-test_builder_ppp-pass-count'))
self.assertEquals(
self.create_count(
name='trybot-test_builder_fff-pass-count',
description=
'Number of passing runs by the test_builder_fff trybot.',
tally={PatchsetReference(1, 1): 0},
), self.get_stats('trybot-test_builder_fff-pass-count'))
self.assertEquals(
self.create_count(
name='trybot-test_builder_ffp-pass-count',
description=
'Number of passing runs by the test_builder_ffp trybot.',
tally={PatchsetReference(1, 1): 1},
), self.get_stats('trybot-test_builder_ffp-pass-count'))
self.assertEquals(
self.create_count(
name='trybot-pass-count',
description='Number of passing runs across all trybots.',
tally={
TrybotReference('test_master_a', 'test_builder_ppp'): 1,
TrybotReference('test_master_a', 'test_builder_fff'): 0,
TrybotReference('test_master_b', 'test_builder_ffp'): 1,
},
), self.get_stats('trybot-pass-count'))
def test_patchset_commit_count(self):
self.analyze_records(*self.patchset_commit_records)
self.assertEquals(
self.create_count(
name='patchset-commit-count',
description='Number of patchsets committed by the CQ.',
tally={
PatchsetReference(2, 1): 1,
PatchsetReference(3, 2): 1,
},
), self.get_stats('patchset-commit-count'))
def test_attempt_count(self):
self.analyze_records(*self.attempt_records)
self.assertEquals(
self.create_count(
name='attempt-count',
description='Number of CQ attempts made.',
tally={
PatchsetReference(1, 1): 2,
PatchsetReference(2, 1): 1,
},
), self.get_stats('attempt-count'))
def test_patchset_false_reject_count(self):
self.analyze_records(*self.rejected_patchset_records)
self.assertEquals(
self.create_count(
name='patchset-false-reject-count',
description=('Number of patchsets rejected by the trybots '
'that eventually passed.'),
tally={
PatchsetReference(1, 2): 1,
PatchsetReference(2, 2): 1,
},
), self.get_stats('patchset-false-reject-count'))
def test_patchset_total_commit_queue_durations(self):
self.analyze_records(*self.historical_records)
self.assertEquals(
self.create_list(
name='patchset-total-commit-queue-durations',
description='Total time spent in the CQ per patch.',
unit='seconds',
points=(
(hours(11), PatchsetReference(1, 2)),
(hours(5), PatchsetReference(1, 3)),
),
), self.get_stats('patchset-total-commit-queue-durations'))
def test_patchset_commit_durations(self):
self.analyze_records(*self.patchset_commit_records)
self.assertEquals(
self.create_list(
name='patchset-commit-durations',
description=('Time taken by the CQ to land a patch '
'after passing all checks.'),
unit='seconds',
points=(
(hours(1), PatchsetReference(2, 1)),
(hours(4), PatchsetReference(3, 2)),
),
), self.get_stats('patchset-commit-durations'))
def test_attempt_durations(self):
self.analyze_records(*self.attempt_records)
self.assertEquals(
self.create_list(
name='attempt-durations',
description='Total time spent per CQ attempt.',
unit='seconds',
points=(
(hours(2), PatchsetReference(1, 1)),
(hours(10), PatchsetReference(1, 1)),
(hours(12), PatchsetReference(2, 1)),
),
), self.get_stats('attempt-durations'))
def test_patchset_total_wall_time_durations(self):
self.analyze_records(*self.historical_records)
self.assertEquals(
self.create_list(
name='patchset-total-wall-time-durations',
description=
'Total time per patch since their commit box was checked.',
unit='seconds',
points=(
(hours(36), PatchsetReference(1, 2)),
(hours(5), PatchsetReference(1, 3)),
),
), self.get_stats('patchset-total-wall-time-durations'))
def test_patchset_attempts(self):
self.analyze_records(*self.issue_patchset_count_records)
self.assertEquals(
self.create_list(
name='patchset-attempts',
description='Number of CQ attempts per patchset.',
unit='attempts',
points=(
(1, PatchsetReference(1, 1)),
(1, PatchsetReference(2, 1)),
(1, PatchsetReference(3, 1)),
(2, PatchsetReference(3, 2)),
),
), self.get_stats('patchset-attempts'))
def test_tryjobverifier_simple_count(self):
counted_actions = (
('error', 'Number of tryjob verifier runs errored.'),
('fail', 'Number of tryjob verifier runs failed.'),
('pass', 'Number of tryjob verifier runs passed.'),
('retry', 'Number of tryjob verifier runs retried.'),
('skip', 'Number of tryjob verifier runs skipped.'),
('start', 'Number of tryjob verifier runs started.'),
('timeout', 'Number of tryjob verifier runs that timed out.'),
)
for action, description in counted_actions:
self.analyze_records(
(1, {
'issue': 1,
'patchset': 1,
'action': 'patch_start'
}),
(2, {
'issue': 1,
'patchset': 1,
'action': 'verifier_' + action
}),
(3, {
'issue': 1,
'patchset': 1,
'action': 'patch_stop'
}),
(4, {
'issue': 1,
'patchset': 2,
'action': 'patch_start'
}),
(5, {
'issue': 1,
'patchset': 2,
'action': 'patch_stop'
}),
(6, {
'issue': 2,
'patchset': 1,
'action': 'patch_start'
}),
(7, {
'issue': 2,
'patchset': 1,
'action': 'patch_stop'
}),
)
name = 'tryjobverifier-%s-count' % action
self.assertEquals(
self.create_count(
name=name,
description=description,
tally={PatchsetReference(1, 1): 1},
), self.get_stats(name))
def test_trybot_false_reject_count(self):
self.analyze_records(*self.trybot_records)
self.assertEquals(
self.create_count(
name='trybot-test_builder_ppp-false-reject-count',
description=(
'Number of false rejects by the test_builder_ppp trybot. '
'This counts any failed runs that also had passing runs '
'on the same patch.'),
tally={PatchsetReference(1, 1): 0},
), self.get_stats('trybot-test_builder_ppp-false-reject-count'))
self.assertEquals(
self.create_count(
name='trybot-test_builder_fff-false-reject-count',
description=(
'Number of false rejects by the test_builder_fff trybot. '
'This counts any failed runs that also had passing runs '
'on the same patch.'),
tally={PatchsetReference(1, 1): 0},
), self.get_stats('trybot-test_builder_fff-false-reject-count'))
self.assertEquals(
self.create_count(
name='trybot-test_builder_ffp-false-reject-count',
description=(
'Number of false rejects by the test_builder_ffp trybot. '
'This counts any failed runs that also had passing runs '
'on the same patch.'),
tally={PatchsetReference(1, 1): 2},
), self.get_stats('trybot-test_builder_ffp-false-reject-count'))
self.assertEquals(
self.create_count(
name='trybot-false-reject-count',
description=(
'Number of false rejects across all trybots. '
'This counts any failed runs that also had passing runs '
'on the same patch.'),
tally={
TrybotReference('test_master_b', 'test_builder_ffp'): 2
},
), self.get_stats('trybot-false-reject-count'))
def test_patchset_durations(self):
self.analyze_records(
(1, {
'issue': 1,
'patchset': 1,
'action': 'patch_start'
}),
(2, {
'issue': 1,
'patchset': 1,
'action': 'patch_stop'
}),
(3, {
'issue': 1,
'patchset': 2,
'action': 'patch_start'
}),
(5, {
'issue': 1,
'patchset': 2,
'action': 'patch_stop'
}),
(6, {
'issue': 2,
'patchset': 1,
'action': 'patch_start'
}),
(7, {
'issue': 2,
'patchset': 1,
'action': 'patch_stop'
}),
(8, {
'issue': 2,
'patchset': 1,
'action': 'patch_start'
}),
(10, {
'issue': 2,
'patchset': 1,
'action': 'patch_stop'
}),
(10, {
'issue': 2,
'patchset': 2,
'action': 'patch_start'
}),
(20, {
'issue': 2,
'patchset': 2,
'action': 'patch_stop'
}),
)
self.assertEquals(
self.create_list(
name='patchset-durations',
description=('Total time spent in the CQ per patchset, '
'counts multiple CQ attempts as one.'),
unit='seconds',
points=(
(hours(1), PatchsetReference(1, 1)),
(hours(2), PatchsetReference(1, 2)),
(hours(3), PatchsetReference(2, 1)),
(hours(10), PatchsetReference(2, 2)),
),
), self.get_stats('patchset-durations'))
def test_blocked_on_throttled_tree_durations(self):
self.analyze_records(
(1, {
'issue': 1,
'patchset': 1,
'action': 'patch_start'
}),
(2, {
'issue': 1,
'patchset': 1,
'action': 'patch_throttled'
}),
(3, {
'issue': 1,
'patchset': 1,
'action': 'patch_ready_to_commit'
}),
(4, {
'issue': 1,
'patchset': 1,
'action': 'patch_stop'
}),
(5, {
'issue': 2,
'patchset': 1,
'action': 'patch_start'
}),
(6, {
'issue': 2,
'patchset': 1,
'action': 'patch_throttled'
}),
(9, {
'issue': 2,
'patchset': 1,
'action': 'patch_ready_to_commit'
}),
(10, {
'issue': 2,
'patchset': 1,
'action': 'patch_stop'
}),
(11, {
'issue': 3,
'patchset': 2,
'action': 'patch_start'
}),
(12, {
'issue': 3,
'patchset': 2,
'action': 'patch_throttled'
}),
(14, {
'issue': 3,
'patchset': 2,
'action': 'patch_stop'
}),
(15, {
'issue': 4,
'patchset': 2,
'action': 'patch_start'
}),
(19, {
'issue': 4,
'patchset': 2,
'action': 'patch_ready_to_commit'
}),
(20, {
'issue': 4,
'patchset': 2,
'action': 'patch_stop'
}),
)
self.assertEquals(
self.create_list(
name='blocked-on-throttled-tree-durations',
description=('Time spent per committed patchset '
'blocked on a throttled tree.'),
unit='seconds',
points=(
(hours(1), PatchsetReference(1, 1)),
(hours(3), PatchsetReference(2, 1)),
(hours(0), PatchsetReference(4, 2)),
),
), self.get_stats('blocked-on-throttled-tree-durations'))
def test_attempt_false_reject_count(self):
self.analyze_records(*self.false_reject_attempt_records)
self.assertEquals(
self.create_count(
name='attempt-false-reject-count',
description=('Number of failed attempts on a committed '
'patch that passed presubmit, had all LGTMs '
'and were not manually cancelled.'),
tally={
PatchsetReference(1, 1): 1,
PatchsetReference(5, 1): 2,
PatchsetReference(7, 1): 1,
PatchsetReference(10, 1): 1,
PatchsetReference(14, 1): 1,
PatchsetReference(15, 1): 1,
PatchsetReference(16, 1): 1,
PatchsetReference(18, 1): 1,
PatchsetReference(19, 1): 1,
PatchsetReference(22, 1): 1,
},
), self.get_stats('attempt-false-reject-count'))
self.assertEquals(
self.create_count(
name='attempt-false-reject-commit-count',
description=
'Number of failed commit attempts on a committed patch.',
tally={PatchsetReference(15, 1): 1},
), self.get_stats('attempt-false-reject-commit-count'))
self.assertEquals(
self.create_count(
name='attempt-false-reject-cq-presubmit-count',
description=(
'Number of failed CQ presubmit checks on a committed '
'patch.'),
tally={PatchsetReference(16, 1): 1},
), self.get_stats('attempt-false-reject-cq-presubmit-count'))
self.assertEquals(
self.create_count(
name='attempt-false-reject-trigger-count',
description=(
'Number of failed job trigger attempts on a committed '
'patch.'),
tally={PatchsetReference(19, 1): 1},
), self.get_stats('attempt-false-reject-trigger-count'))
self.assertEquals(
self.create_count(
name='attempt-false-reject-tryjob-count',
description=
'Number of failed job attempts on a committed patch.',
tally={PatchsetReference(18, 1): 1},
), self.get_stats('attempt-false-reject-tryjob-count'))
def test_tryjobverifier_first_run_durations(self):
self.analyze_records(
(1, {
'issue': 1,
'patchset': 1,
'action': 'patch_start'
}),
(2, {
'issue': 1,
'patchset': 1,
'action': 'verifier_start'
}),
(3, {
'issue': 1,
'patchset': 1,
'action': 'verifier_retry'
}),
(4, {
'issue': 1,
'patchset': 1,
'action': 'verifier_pass'
}),
(5, {
'issue': 1,
'patchset': 1,
'action': 'patch_stop'
}),
(6, {
'issue': 2,
'patchset': 1,
'action': 'patch_start'
}),
(7, {
'issue': 2,
'patchset': 1,
'action': 'verifier_start'
}),
(8, {
'issue': 2,
'patchset': 1,
'action': 'verifier_timeout'
}),
(9, {
'issue': 2,
'patchset': 1,
'action': 'patch_stop'
}),
(10, {
'issue': 2,
'patchset': 1,
'action': 'patch_start'
}),
(11, {
'issue': 2,
'patchset': 1,
'action': 'verifier_start'
}),
(15, {
'issue': 2,
'patchset': 1,
'action': 'verifier_fail'
}),
(16, {
'issue': 2,
'patchset': 1,
'action': 'patch_stop'
}),
(17, {
'issue': 3,
'patchset': 1,
'action': 'patch_start'
}),
(18, {
'issue': 3,
'patchset': 1,
'action': 'verifier_start'
}),
(21, {
'issue': 3,
'patchset': 1,
'action': 'verifier_pass'
}),
(22, {
'issue': 3,
'patchset': 1,
'action': 'patch_stop'
}),
)
self.assertEquals(
self.create_list(
name='tryjobverifier-first-run-durations',
description='Time spent on each tryjob verifier first run.',
unit='seconds',
points=(
(hours(1), PatchsetReference(1, 1)),
(hours(1), PatchsetReference(2, 1)),
(hours(4), PatchsetReference(2, 1)),
(hours(3), PatchsetReference(3, 1)),
),
), self.get_stats('tryjobverifier-first-run-durations'))
def test_tryjobverifier_total_durations(self):
self.analyze_records(
(1, {
'issue': 1,
'patchset': 1,
'action': 'patch_start'
}),
(2, {
'issue': 1,
'patchset': 1,
'action': 'verifier_start'
}),
(3, {
'issue': 1,
'patchset': 1,
'action': 'verifier_pass'
}),
(4, {
'issue': 1,
'patchset': 1,
'action': 'patch_stop'
}),
(5, {
'issue': 1,
'patchset': 1,
'action': 'patch_start'
}),
(6, {
'issue': 1,
'patchset': 1,
'action': 'verifier_start'
}),
(10, {
'issue': 1,
'patchset': 1,
'action': 'verifier_fail'
}),
(11, {
'issue': 1,
'patchset': 1,
'action': 'patch_stop'
}),
(12, {
'issue': 1,
'patchset': 2,
'action': 'patch_start'
}),
(13, {
'issue': 1,
'patchset': 2,
'action': 'verifier_start'
}),
(14, {
'issue': 1,
'patchset': 2,
'action': 'verifier_retry'
}),
(20, {
'issue': 1,
'patchset': 2,
'action': 'verifier_timeout'
}),
(21, {
'issue': 1,
'patchset': 2,
'action': 'patch_stop'
}),
)
self.assertEquals(
self.create_list(
name='tryjobverifier-total-durations',
description=
'Total time spent per CQ attempt on tryjob verifier runs.',
unit='seconds',
points=(
(hours(1), PatchsetReference(1, 1)),
(hours(4), PatchsetReference(1, 1)),
(hours(7), PatchsetReference(1, 2)),
),
), self.get_stats('tryjobverifier-total-durations'))