def test_metrics_factory(): sample_data = TestData(perftest_data()) metric_collection_data = get_metric_collection_data() mmf = MetricsMethodFactory(metric_collection_data['initialization_data']) m_one = mmf.get_metric_method(metric_collection_data['testsuite_name']) #Should have one cached metric method instance assert len(mmf.metric_method_instances) == 1 #Retrieve metric method again, should still have one cached metric #method instance m_two = mmf.get_metric_method(metric_collection_data['testsuite_name']) assert len(mmf.metric_method_instances) == 1 #get_metric_method should return a TtestMethod class instance for #the sample data tm = TtestMethod(metric_collection_data['initialization_data']) assert m_one.__class__.__name__ == tm.__class__.__name__ assert m_two.__class__.__name__ == tm.__class__.__name__
def _test_metric_evaluations(setup_data, mtm, target_pass_count): """ The metrics data associated with the fail revision should evaluate to test failure. All other revisions should evaluate to test success. """ fail_revision = setup_data['fail_revision'] skip_revision = setup_data['skip_revision'] metric_collection_data = get_metric_collection_data() mmf = MetricsMethodFactory(metric_collection_data['initialization_data']) mm = mmf.get_metric_method(setup_data['testsuite_name']) metric_fail_count = 0 metric_pass_count = 0 metric_summary_fail_count = 0 metric_summary_pass_count = 0 for revision in setup_data['sample_revisions']: metrics_data = mtm.get_metrics_data(revision) if revision == skip_revision: #We should have no data for the skip revision assert metrics_data == {} continue for key in metrics_data: for data in metrics_data[key]['values']: test_result = {} test_result[data['metric_value_name']] = data['value'] if data['metric_value_name'] == 'h0_rejected': metric_evaluation = mm.evaluate_metric_result(test_result) if metric_evaluation == False: metric_fail_count += 1 #test evaluation should indicate failure assert revision == fail_revision else: metric_pass_count += 1 #all other tests should pass assert revision != fail_revision if data['metric_value_name'] == mm.SUMMARY_NAME: summary_evaluation = mm.evaluate_metric_summary_result( test_result) if summary_evaluation == False: metric_summary_fail_count += 1 #test evaluation should indicate failure assert revision == fail_revision else: metric_summary_pass_count += 1 #all other tests should pass assert revision != fail_revision target_fail_count = 3 assert metric_fail_count == target_fail_count assert metric_pass_count == target_pass_count assert metric_summary_fail_count == target_fail_count assert metric_summary_pass_count == target_pass_count