def test_breakdown_resets_to_null_when_calculus_errors_out(betamax_recorder):
engineer_traction = EngineerTractionFormula(betamax_recorder.session)
test_moniker_A = ('build_metrics', 'build times')
test_moniker_B = ('nonexistent_framework', 'nonexistent_suite')
cassette_preffix_A = '-'.join(filter(None, test_moniker_A))
cassette_preffix_B = '-'.join(filter(None, test_moniker_B))
# run happy path calculus
with betamax_recorder.use_cassette(f'{cassette_preffix_A}',
serialize_with='prettyjson'):
engineer_traction(
*test_moniker_A) # let it perform calculus & cache breakdown
_ = engineer_traction.breakdown()
# now run alternated path calculus
with betamax_recorder.use_cassette(f'{cassette_preffix_B}',
serialize_with='prettyjson'):
with pytest.raises(NoFiledBugs):
engineer_traction(
*test_moniker_B) # intentionally blows up while doing calculus
# cached breakdown got invalidated & can no longer be obtained
with pytest.raises(RuntimeError):
_ = engineer_traction.breakdown()
def test_formula_counts_tracted_bugs(cooled_down_bugs, betamax_recorder):
engineer_traction = EngineerTractionFormula(betamax_recorder.session)
with betamax_recorder.use_cassette('cooled-down-bug-history',
serialize_with='prettyjson'):
tracted_bugs = engineer_traction._filter_tracted_bugs(cooled_down_bugs)
assert len(tracted_bugs) == 2
def test_formula_initializes_with_non_blockable_sessions(nonblock_session):
try:
_ = EngineerTractionFormula(nonblock_session)
except ValueError:
pytest.fail()
try:
_ = EngineerTractionFormula()
except ValueError:
pytest.fail()
def test_formula_filters_out_bugs_that_didnt_cool_down_yet(
framework, suite, test, betamax_recorder):
engineer_traction = EngineerTractionFormula(betamax_recorder.session)
cassette = '-'.join(filter(None, [framework, suite, test]))
with betamax_recorder.use_cassette(f'{cassette}',
serialize_with='prettyjson'):
engineer_traction(framework, suite,
test) # let it perform calculus & cache breakdown
# left with cooled down bugs only
all_filed_bugs, _ = engineer_traction.breakdown()
for bug in all_filed_bugs:
assert engineer_traction.has_cooled_down(bug)
def test_final_formula_confirms_non_sheriffed_tests(framework, suite,
betamax_recorder):
engineer_traction = EngineerTractionFormula(betamax_recorder.session)
with betamax_recorder.use_cassette(f'{framework}-{suite}',
serialize_with='prettyjson'):
assert engineer_traction(framework, suite) < 0.35
def handle(self, *args, **options):
if options.get('individually'):
return self._handle_individually(options)
quant_period = options['quantifying_period']
bug_cooldown = options['bug_cooldown']
multiprocessed = options['multiprocessing']
init_params = (None, quant_period, bug_cooldown)
formula_map = {
'EngineerTraction': EngineerTractionFormula(*init_params),
'FixRatio': FixRatioFormula(*init_params),
}
tracker = CriteriaTracker(formula_map, multiprocessed=multiprocessed)
tracker.load_records()
start = time.time()
tracker.update_records()
duration = time.time() - start
print(f'{self.INITIAL_PROMPT_MSG}', end='')
for record in tracker:
print(record)
print(f"Took {duration:.1f} seconds")
def handle(self, *args, **options):
framework = options['framework']
suite = options['suite']
test = options['test']
quant_period = options['quantifying_period']
bug_cooldown = options['bug_cooldown']
init_params = (None, quant_period, bug_cooldown)
targetted_test = (framework, suite, test)
engineer_traction = EngineerTractionFormula(*init_params)
fix_ratio = FixRatioFormula(*init_params)
print(f'\r{self.INITIAL_PROMPT_MSG}', end='')
compute_start = time.time()
eng_traction_result = engineer_traction(*targetted_test)
fix_ratio_result = fix_ratio(*targetted_test)
compute_duration = time.time() - compute_start
# turn into regular percentages
eng_traction_result *= 100
fix_ratio_result *= 100
self._display_results(eng_traction_result, fix_ratio_result, framework,
suite, test, compute_duration)
def test_formula_fetches_bugs_from_quantifying_period(framework, suite, test,
betamax_recorder):
engineer_traction = EngineerTractionFormula(betamax_recorder.session)
cassette = '-'.join(filter(None, [framework, suite, test]))
with betamax_recorder.use_cassette(f'{cassette}',
serialize_with='prettyjson'):
engineer_traction(framework, suite,
test) # let it perform calculus & cache breakdown
all_filed_bugs, except_new_bugs = engineer_traction.breakdown()
assert len(all_filed_bugs) > 0
for bug in all_filed_bugs:
creation_time = datetime.strptime(bug['creation_time'],
BZ_DATETIME_FORMAT)
assert creation_time >= engineer_traction.oldest_timestamp
def test_formula_errors_up_when_no_bugs_were_filed(betamax_recorder):
engineer_traction = EngineerTractionFormula(betamax_recorder.session)
nonexistent_framework = 'nonexistent_framework'
nonexistent_suite = 'nonexistent_suite'
with betamax_recorder.use_cassette(
f'{nonexistent_framework}-{nonexistent_suite}',
serialize_with='prettyjson'):
with pytest.raises(NoFiledBugs):
engineer_traction(nonexistent_framework, nonexistent_suite)
def _handle_individually(self, options):
framework = options['framework']
suite = options['suite']
test = options['test']
quant_period = options['quantifying_period']
bug_cooldown = options['bug_cooldown']
init_params = (None, quant_period, bug_cooldown)
targetted_test = (framework, suite, test)
engineer_traction = EngineerTractionFormula(*init_params)
fix_ratio = FixRatioFormula(*init_params)
print(f'\r{self.INITIAL_PROMPT_MSG}', end='')
compute_start = time.time()
eng_traction_result = engineer_traction(*targetted_test)
fix_ratio_result = fix_ratio(*targetted_test)
compute_duration = time.time() - compute_start
# turn into regular percentages
eng_traction_result *= 100
fix_ratio_result *= 100
# display results (inline)
test_moniker = ' '.join(filter(None, (suite, test)))
title = f'Perf Sheriffing Criteria for {framework} - {test_moniker}'
big_underline = '-' * len(title)
# & results headers
eng_traction_head = self.ENGINEER_TRACTION.capitalize()
fix_ratio_head = self.FIX_RATIO.capitalize()
justify_head = self.__get_head_justification(eng_traction_head,
fix_ratio_head)
# let's update 1st prompt line
print(f"\r{' ' * len(self.INITIAL_PROMPT_MSG)}", end='')
print(
f"\rComputing Perf Sheriffing Criteria... (took {compute_duration:{self.PRECISION}} seconds)"
)
# display title
print(big_underline)
print(title)
print(big_underline)
# & actual results
print(
f'{eng_traction_head:<{justify_head}}: {eng_traction_result:{self.PRECISION}}%'
)
print(
f'{fix_ratio_head:<{justify_head}}: {fix_ratio_result:{self.PRECISION}}%'
)
print(big_underline)
def test_breakdown_updates_between_calculations(betamax_recorder):
engineer_traction = EngineerTractionFormula(betamax_recorder.session)
test_moniker_A = ('build_metrics', 'build times')
test_moniker_B = ('talos', 'tp5n', 'nonmain_startup_fileio')
cassette_preffix_A = '-'.join(filter(None, test_moniker_A))
cassette_preffix_B = '-'.join(filter(None, test_moniker_B))
with betamax_recorder.use_cassette(f'{cassette_preffix_A}',
serialize_with='prettyjson'):
engineer_traction(
*test_moniker_A) # let it perform calculus & cache breakdown
breakdown_A = engineer_traction.breakdown()
with betamax_recorder.use_cassette(f'{cassette_preffix_B}',
serialize_with='prettyjson'):
engineer_traction(
*test_moniker_B) # let it perform calculus & cache breakdown
breakdown_B = engineer_traction.breakdown()
assert breakdown_A != breakdown_B
def test_formula_demands_at_least_framework_and_suite(betamax_recorder):
engineer_traction = EngineerTractionFormula(betamax_recorder.session)
with pytest.raises(TypeError):
engineer_traction('some_framework')
with pytest.raises(TypeError):
engineer_traction()
with betamax_recorder.use_cassette('awsy-JS', serialize_with='prettyjson'):
try:
engineer_traction('awsy', 'JS')
except TypeError:
pytest.fail()
def test_formula_cannot_be_initialized_with_a_regular_session(
unrecommended_session):
with pytest.raises(ValueError):
_ = EngineerTractionFormula(unrecommended_session)
def test_formula_detects_bugs_that_didnt_cool_down_yet(not_cooled_down_bug,
nonblock_session):
engineer_traction = EngineerTractionFormula(nonblock_session)
assert not engineer_traction.has_cooled_down(not_cooled_down_bug)
def test_formula_correctly_detects_cooled_down_bugs(cooled_down_bug,
nonblock_session):
engineer_traction = EngineerTractionFormula(nonblock_session)
assert engineer_traction.has_cooled_down(cooled_down_bug)
def test_formula_throws_adequate_error_for_bug(bad_structured_bug,
nonblock_session):
engineer_traction = EngineerTractionFormula(nonblock_session)
with pytest.raises(ValueError):
engineer_traction.has_cooled_down(bad_structured_bug)
def test_accessing_breakdown_without_prior_calculus_errors_out(
nonblock_session):
engineer_traction = EngineerTractionFormula(nonblock_session)
with pytest.raises(RuntimeError):
_ = engineer_traction.breakdown()
def formula_instances() -> List[BugzillaFormula]:
return [EngineerTractionFormula(), FixRatioFormula()]
def test_formula_exposes_oldest_timestamp(nonblock_session):
engineer_traction = EngineerTractionFormula(nonblock_session)
no_older_than = datetime.now() - timedelta(weeks=24, seconds=5)
assert engineer_traction.oldest_timestamp >= no_older_than
