def test_get_all_query_performance_samples_passing(self):
b_p = agg.EdwBenchmarkPerformance(
total_iterations=2, expected_suite_queries=[Q1_NAME, Q2_NAME])
s1_p = agg.EdwSuitePerformance('suite_name', 'suite_seq_1', 2)
q11_p = agg.EdwQueryPerformance(Q1_NAME, 1.0,
{'job_id': 'q1_s1_job_id'})
s1_p.add_query_performance(q11_p)
q12_p = agg.EdwQueryPerformance(Q2_NAME, 2.0,
{'job_id': 'q2_s1_job_id'})
s1_p.add_query_performance(q12_p)
b_p.add_suite_performance('suite_seq_1', s1_p)
s2_p = agg.EdwSuitePerformance('suite_name', 'suite_seq_2', 2)
q21_p = agg.EdwQueryPerformance(Q1_NAME, 3.0,
{'job_id': 'q1_s2_job_id'})
s2_p.add_query_performance(q21_p)
q22_p = agg.EdwQueryPerformance(Q2_NAME, 4.0, {})
s2_p.add_query_performance(q22_p)
b_p.add_suite_performance('suite_seq_2', s2_p)
actual_sample_list = b_p.get_all_query_performance_samples({})
self.assertEqual(len(actual_sample_list), 6)
# 4 raw query samples and 2 aggregated samples
self.assertSameElements([x.metric for x in actual_sample_list], [
'edw_raw_query_time', 'edw_raw_query_time', 'edw_raw_query_time',
'edw_raw_query_time', 'edw_aggregated_query_time',
'edw_aggregated_query_time'
])
def test_get_all_query_performance(self):
s_p = agg.EdwSuitePerformance('suite_name', 'suite_seq', 10)
q1_p = agg.EdwQueryPerformance(Q1_NAME, Q1_PERFORMANCE, METADATA_EMPTY)
s_p.add_query_performance(q1_p)
q2_p = agg.EdwQueryPerformance(Q2_NAME, Q2_PERFORMANCE, METADATA_EMPTY)
s_p.add_query_performance(q2_p)
actual_all_query_performance = s_p.get_all_query_performance_samples(
METADATA_EMPTY)
self.assertEqual(len(actual_all_query_performance), 2)
self.assertListEqual([x.metric for x in actual_all_query_performance],
['edw_raw_query_time', 'edw_raw_query_time'])
self.assertSameElements(
[x.value for x in actual_all_query_performance],
[Q1_PERFORMANCE, Q2_PERFORMANCE])
self.assertListEqual([x.unit for x in actual_all_query_performance],
[SECS, SECS])
self.assertSameElements(
[x.metadata['query'] for x in actual_all_query_performance],
[Q1_NAME, Q2_NAME])
self.assertSameElements([
x.metadata['execution_status']
for x in actual_all_query_performance
], [
agg.EdwQueryExecutionStatus.SUCCESSFUL,
agg.EdwQueryExecutionStatus.SUCCESSFUL
])
def test_get_wall_time_performance_samples_passing(self):
b_p = agg.EdwBenchmarkPerformance(
total_iterations=2, expected_suite_queries=[Q1_NAME, Q2_NAME])
s1_p = agg.EdwSuitePerformance('suite_name', 'suite_seq_1', 2)
q11_p = agg.EdwQueryPerformance(Q1_NAME, 1.0, {'job_id': 'q1_s1_job_id'})
s1_p.add_query_performance(q11_p)
q12_p = agg.EdwQueryPerformance(Q2_NAME, 2.0, {'job_id': 'q2_s1_job_id'})
s1_p.add_query_performance(q12_p)
b_p.add_suite_performance('suite_seq_1', s1_p)
s2_p = agg.EdwSuitePerformance('suite_name', 'suite_seq_2', 2)
q21_p = agg.EdwQueryPerformance(Q1_NAME, 3.0, {'job_id': 'q1_s2_job_id'})
s2_p.add_query_performance(q21_p)
q22_p = agg.EdwQueryPerformance(Q2_NAME, 4.0, {})
s2_p.add_query_performance(q22_p)
b_p.add_suite_performance('suite_seq_2', s2_p)
actual_sample_list = b_p.get_wall_time_performance_samples(
{'benchmark_name': 'b_name'})
self.assertEqual(len(actual_sample_list), 3)
self.assertSameElements(
[x.metric for x in actual_sample_list],
['edw_raw_wall_time', 'edw_raw_wall_time', 'edw_aggregated_wall_time'])
raw_samples = list(
filter(lambda x: x.metric == 'edw_raw_wall_time', actual_sample_list))
actual_raw_samples_values = [x.value for x in raw_samples]
expected_raw_samples_values = [(1.0 + 2.0), (3.0 + 4.0)]
self.assertSameElements(actual_raw_samples_values,
expected_raw_samples_values)
aggregated_sample = list(
filter(lambda x: x.metric == 'edw_aggregated_wall_time',
actual_sample_list))[0]
self.assertEqual(aggregated_sample.value, (1.0 + 3.0) / 2 + (2.0 + 4.0) / 2)
def test_aggregated_query_metadata_passing(self):
b_p = agg.EdwBenchmarkPerformance(
total_iterations=2, expected_suite_queries=[Q1_NAME, Q2_NAME])
s1_p = agg.EdwSuitePerformance('suite_name', 'suite_seq_1', 2)
q11_p = agg.EdwQueryPerformance(Q1_NAME, 1.0, {'job_id': 'q1_s1_job_id'})
s1_p.add_query_performance(q11_p)
q12_p = agg.EdwQueryPerformance(Q2_NAME, 2.0, {'job_id': 'q2_s1_job_id'})
s1_p.add_query_performance(q12_p)
b_p.add_suite_performance('suite_seq_1', s1_p)
s2_p = agg.EdwSuitePerformance('suite_name', 'suite_seq_2', 2)
q21_p = agg.EdwQueryPerformance(Q1_NAME, 3.0, {'job_id': 'q1_s2_job_id'})
s2_p.add_query_performance(q21_p)
q22_p = agg.EdwQueryPerformance(Q2_NAME, 4.0, {})
s2_p.add_query_performance(q22_p)
b_p.add_suite_performance('suite_seq_2', s2_p)
actual_aggregated_query_metadata_q1 = b_p.aggregated_query_metadata(Q1_NAME)
expected_aggregated_query_metadata_q1 = {
'suite_seq_1' + '_runtime': 1.0,
'suite_seq_1' + '_job_id': 'q1_s1_job_id',
'suite_seq_2' + '_runtime': 3.0,
'suite_seq_2' + '_job_id': 'q1_s2_job_id'
}
self.assertDictEqual(actual_aggregated_query_metadata_q1,
expected_aggregated_query_metadata_q1)
actual_aggregated_query_metadata_q2 = b_p.aggregated_query_metadata(Q2_NAME)
expected_aggregated_query_metadata_q2 = {
'suite_seq_1' + '_runtime': 2.0,
'suite_seq_1' + '_job_id': 'q2_s1_job_id',
'suite_seq_2' + '_runtime': 4.0
}
self.assertDictEqual(actual_aggregated_query_metadata_q2,
expected_aggregated_query_metadata_q2)
def test_get_aggregated_geomean_performance_sample_passing(self):
b_p = agg.EdwBenchmarkPerformance(
total_iterations=2, expected_suite_queries=[Q1_NAME, Q2_NAME])
s1_p = agg.EdwSuitePerformance('suite_name', 'suite_seq_1', 2)
q11_p = agg.EdwQueryPerformance(Q1_NAME, 1.0, {'job_id': 'q1_s1_job_id'})
s1_p.add_query_performance(q11_p)
q12_p = agg.EdwQueryPerformance(Q2_NAME, 2.0, {'job_id': 'q2_s1_job_id'})
s1_p.add_query_performance(q12_p)
b_p.add_suite_performance('suite_seq_1', s1_p)
s2_p = agg.EdwSuitePerformance('suite_name', 'suite_seq_2', 2)
q21_p = agg.EdwQueryPerformance(Q1_NAME, 3.0, {'job_id': 'q1_s2_job_id'})
s2_p.add_query_performance(q21_p)
q22_p = agg.EdwQueryPerformance(Q2_NAME, 4.0, {})
s2_p.add_query_performance(q22_p)
b_p.add_suite_performance('suite_seq_2', s2_p)
actual_sample = b_p.get_aggregated_geomean_performance_sample(
{'benchmark_name': 'b_name'})
self.assertEqual(actual_sample.metric, 'edw_aggregated_geomean')
self.assertEqual(actual_sample.value,
agg.geometric_mean([(1.0 + 3.0) / 2, (2.0 + 4.0) / 2]))
self.assertEqual(actual_sample.unit, 'seconds')
expected_metadata = {
'benchmark_name': 'b_name',
'intra_query_aggregation_method': 'mean',
'inter_query_aggregation_method': 'geomean'
}
self.assertDictEqual(actual_sample.metadata, expected_metadata)
def test_has_query_performance(self):
s_p = agg.EdwSuitePerformance('suite_name', 'suite_seq', 10)
q1_p = agg.EdwQueryPerformance(Q1_NAME, Q1_PERFORMANCE, METADATA_EMPTY)
s_p.add_query_performance(q1_p)
self.assertTrue(s_p.has_query_performance(Q1_NAME))
q2_p = agg.EdwQueryPerformance(Q2_NAME, Q2_PERFORMANCE, METADATA_EMPTY)
self.assertFalse(s_p.has_query_performance(Q2_NAME))
s_p.add_query_performance(q2_p)
self.assertTrue(s_p.has_query_performance(Q2_NAME))
def test_is_query_successful(self):
s_p = agg.EdwSuitePerformance('suite_name', 'suite_seq', 10)
q_pass = agg.EdwQueryPerformance(Q1_NAME, Q1_PERFORMANCE, METADATA_EMPTY)
s_p.add_query_performance(q_pass)
self.assertTrue(s_p.is_query_successful(Q1_NAME))
q_fail = agg.EdwQueryPerformance(QFAIL_NAME, QFAIL_PERFORMANCE,
METADATA_EMPTY)
s_p.add_query_performance(q_fail)
self.assertFalse(s_p.is_query_successful(QFAIL_NAME))
def test_add_suite_performance(self):
b_p = agg.EdwBenchmarkPerformance(
total_iterations=2, expected_suite_queries=[Q1_NAME])
s1_p = agg.EdwSuitePerformance('suite_name', 'suite_seq_1', 1)
q11_p = agg.EdwQueryPerformance(Q1_NAME, Q1_PERFORMANCE, METADATA_EMPTY)
s1_p.add_query_performance(q11_p)
b_p.add_suite_performance('suite_seq_1', s1_p)
s2_p = agg.EdwSuitePerformance('suite_name', 'suite_seq_2', 1)
q21_p = agg.EdwQueryPerformance(Q1_NAME, Q1_PERFORMANCE, METADATA_EMPTY)
s2_p.add_query_performance(q21_p)
b_p.add_suite_performance('suite_seq_2', s2_p)
self.assertEqual(len(b_p.suite_performances), 2)
self.assertSameElements(b_p.suite_performances.keys(),
['suite_seq_1', 'suite_seq_2'])
def test_is_successful(self):
s_p = agg.EdwSuitePerformance('suite_name', 'suite_seq', 2)
q1_p = agg.EdwQueryPerformance(Q1_NAME, Q1_PERFORMANCE, METADATA_EMPTY)
s_p.add_query_performance(q1_p)
q2_p = agg.EdwQueryPerformance(Q2_NAME, Q2_PERFORMANCE, METADATA_EMPTY)
s_p.add_query_performance(q2_p)
self.assertTrue(s_p.is_successful())
s_p = agg.EdwSuitePerformance('suite_name', 'suite_seq', 2)
q1_p = agg.EdwQueryPerformance(Q1_NAME, Q1_PERFORMANCE, METADATA_EMPTY)
s_p.add_query_performance(q1_p)
qfail_p = agg.EdwQueryPerformance(QFAIL_NAME, QFAIL_PERFORMANCE,
METADATA_EMPTY)
s_p.add_query_performance(qfail_p)
self.assertFalse(s_p.is_successful())
def test_get_queries_geomean_performance(self):
s_p = agg.EdwSuitePerformance('suite_name', 'suite_seq', 2)
q1_p = agg.EdwQueryPerformance(Q1_NAME, Q1_PERFORMANCE, METADATA_EMPTY)
s_p.add_query_performance(q1_p)
q2_p = agg.EdwQueryPerformance(Q2_NAME, Q2_PERFORMANCE, METADATA_EMPTY)
s_p.add_query_performance(q2_p)
suite_wall_time_performance_sample = s_p.get_queries_geomean_performance_sample(
METADATA_EMPTY)
self.assertEqual(suite_wall_time_performance_sample.metric,
'edw_raw_geomean_time')
self.assertEqual(suite_wall_time_performance_sample.value,
agg.geometric_mean([Q1_PERFORMANCE, Q2_PERFORMANCE]))
self.assertEqual(suite_wall_time_performance_sample.unit, SECS)
self.assertDictEqual(suite_wall_time_performance_sample.metadata,
METADATA_EMPTY)
def test_get_aggregated_query_performance_sample_passing(self):
b_p = agg.EdwBenchmarkPerformance(
total_iterations=2, expected_suite_queries=[Q1_NAME, Q2_NAME])
s1_p = agg.EdwSuitePerformance('suite_name', 'suite_seq_1', 2)
q11_p = agg.EdwQueryPerformance(Q1_NAME, 1.0,
{'job_id': 'q1_s1_job_id'})
s1_p.add_query_performance(q11_p)
q12_p = agg.EdwQueryPerformance(Q2_NAME, 2.0,
{'job_id': 'q2_s1_job_id'})
s1_p.add_query_performance(q12_p)
b_p.add_suite_performance('suite_seq_1', s1_p)
s2_p = agg.EdwSuitePerformance('suite_name', 'suite_seq_2', 2)
q21_p = agg.EdwQueryPerformance(Q1_NAME, 3.0,
{'job_id': 'q1_s2_job_id'})
s2_p.add_query_performance(q21_p)
q22_p = agg.EdwQueryPerformance(Q2_NAME, 4.0, {})
s2_p.add_query_performance(q22_p)
b_p.add_suite_performance('suite_seq_2', s2_p)
actual_sample_q1 = b_p.get_aggregated_query_performance_sample(
Q1_NAME, {'benchmark_name': 'b_name'})
self.assertEqual(actual_sample_q1.metric, 'edw_aggregated_query_time')
self.assertEqual(actual_sample_q1.value, (1.0 + 3.0) / 2)
self.assertEqual(actual_sample_q1.unit, 'seconds')
expected_metadata_q1 = {
'suite_seq_1' + '_runtime': 1.0,
'suite_seq_1' + '_job_id': 'q1_s1_job_id',
'suite_seq_2' + '_runtime': 3.0,
'suite_seq_2' + '_job_id': 'q1_s2_job_id',
'query': Q1_NAME,
'aggregation_method': 'mean',
'execution_status': agg.EdwQueryExecutionStatus.SUCCESSFUL,
'benchmark_name': 'b_name'
}
self.assertDictEqual(actual_sample_q1.metadata, expected_metadata_q1)
actual_sample_q2 = b_p.get_aggregated_query_performance_sample(
Q2_NAME, {})
self.assertEqual(actual_sample_q2.metric, 'edw_aggregated_query_time')
self.assertEqual(actual_sample_q2.value, (2.0 + 4.0) / 2)
self.assertEqual(actual_sample_q2.unit, 'seconds')
expected_metadata_q2 = {
'suite_seq_1' + '_runtime': 2.0,
'suite_seq_1' + '_job_id': 'q2_s1_job_id',
'suite_seq_2' + '_runtime': 4.0,
'query': Q2_NAME,
'aggregation_method': 'mean',
'execution_status': agg.EdwQueryExecutionStatus.SUCCESSFUL
}
self.assertDictEqual(actual_sample_q2.metadata, expected_metadata_q2)
def test_aggregated_query_metadata_failing_query(self):
b_p = agg.EdwBenchmarkPerformance(
total_iterations=2, expected_suite_queries=[Q1_NAME, Q2_NAME])
s1_p = agg.EdwSuitePerformance('suite_name', 'suite_seq_1', 2)
q11_p = agg.EdwQueryPerformance(Q1_NAME, Q1_PERFORMANCE, METADATA_EMPTY)
s1_p.add_query_performance(q11_p)
q12_p = agg.EdwQueryPerformance(Q2_NAME, Q2_PERFORMANCE, METADATA_EMPTY)
s1_p.add_query_performance(q12_p)
b_p.add_suite_performance('suite_seq_1', s1_p)
s2_p = agg.EdwSuitePerformance('suite_name', 'suite_seq_2', 2)
q21_p = agg.EdwQueryPerformance(Q1_NAME, Q1_PERFORMANCE, METADATA_EMPTY)
s2_p.add_query_performance(q21_p)
q22_p = agg.EdwQueryPerformance(Q2_NAME, QFAIL_PERFORMANCE, METADATA_EMPTY)
s2_p.add_query_performance(q22_p)
b_p.add_suite_performance('suite_seq_2', s2_p)
with self.assertRaises(agg.EdwPerformanceAggregationError):
b_p.aggregated_query_metadata(Q2_NAME)
def test_aggregated_query_status_look_for_failing_query(self):
b_p = agg.EdwBenchmarkPerformance(
total_iterations=2, expected_suite_queries=[Q1_NAME, Q2_NAME])
s1_p = agg.EdwSuitePerformance('suite_name', 'suite_seq_1', 2)
q11_p = agg.EdwQueryPerformance(Q1_NAME, Q1_PERFORMANCE, METADATA_EMPTY)
s1_p.add_query_performance(q11_p)
q12_p = agg.EdwQueryPerformance(Q2_NAME, Q2_PERFORMANCE, METADATA_EMPTY)
s1_p.add_query_performance(q12_p)
b_p.add_suite_performance('suite_seq_1', s1_p)
s2_p = agg.EdwSuitePerformance('suite_name', 'suite_seq_2', 2)
q21_p = agg.EdwQueryPerformance(Q1_NAME, Q1_PERFORMANCE, METADATA_EMPTY)
s2_p.add_query_performance(q21_p)
q22_p = agg.EdwQueryPerformance(Q2_NAME, QFAIL_PERFORMANCE, METADATA_EMPTY)
s2_p.add_query_performance(q22_p)
b_p.add_suite_performance('suite_seq_2', s2_p)
self.assertTrue(b_p.aggregated_query_status(Q1_NAME))
self.assertFalse(b_p.aggregated_query_status(Q2_NAME))
def test_get_performance_simple(self):
q_p = agg.EdwQueryPerformance(Q1_NAME, Q1_PERFORMANCE, METADATA_EMPTY)
actual_sample = q_p.get_performance_sample(METADATA_EMPTY)
expected_sample = sample.Sample('edw_raw_query_time', Q1_PERFORMANCE,
SECS, METADATA_EMPTY)
self.assertEqual(actual_sample.metric, expected_sample.metric)
self.assertEqual(actual_sample.value, expected_sample.value)
self.assertEqual(actual_sample.unit, expected_sample.unit)
def test_add_query_performance(self):
s_p = agg.EdwSuitePerformance('suite_name', 'suite_seq', 10)
q_p = agg.EdwQueryPerformance(Q1_NAME, Q1_PERFORMANCE, METADATA_EMPTY)
s_p.add_query_performance(q_p)
actual_suite_performance = s_p.performance
expected_suite_performance = {Q1_NAME: q_p}
self.assertDictEqual(actual_suite_performance, expected_suite_performance)
self.assertEqual(s_p.total_count, 10)
self.assertEqual(s_p.successful_count, 1)
def Run(benchmark_spec):
"""Run phase executes the sql scripts on edw cluster and collects duration."""
results = []
edw_service_instance = benchmark_spec.edw_service
# Run a warm up query in case there are cold start issues.
edw_service_instance.GetClientInterface().WarmUpQuery()
# Default to executing just the sample query if no queries are provided.
queries_to_execute = FLAGS.queries_to_execute or [
os.path.basename(edw_service.SAMPLE_QUERY_PATH)
]
# Accumulator for the entire benchmark's performance
benchmark_performance = results_aggregator.EdwBenchmarkPerformance(
total_iterations=FLAGS.edw_suite_iterations,
expected_suite_queries=queries_to_execute)
# Multiple iterations of the suite are performed to avoid cold start penalty
for i in range(1, FLAGS.edw_suite_iterations + 1):
iteration = str(i)
# Accumulator for the current suite's performance
suite_performance = results_aggregator.EdwSuitePerformance(
suite_name='edw',
suite_sequence=iteration,
total_suite_queries=len(queries_to_execute))
for query in queries_to_execute:
time, metadata = edw_service_instance.GetClientInterface(
).ExecuteQuery(query)
edw_query_performance = results_aggregator.EdwQueryPerformance(
query_name=query, performance=time, metadata=metadata)
suite_performance.add_query_performance(edw_query_performance)
benchmark_performance.add_suite_performance(iteration,
suite_performance)
# Execution complete, generate results only if the benchmark was successful.
benchmark_metadata = {}
benchmark_metadata.update(edw_service_instance.GetMetadata())
if benchmark_performance.is_successful():
query_samples = benchmark_performance.get_all_query_performance_samples(
metadata=benchmark_metadata)
results.extend(query_samples)
wall_time_samples = benchmark_performance.get_wall_time_performance_samples(
metadata=benchmark_metadata)
results.extend(wall_time_samples)
geomean_samples = (
benchmark_performance.get_queries_geomean_performance_samples(
metadata=benchmark_metadata))
results.extend(geomean_samples)
else:
logging.error(
'At least one query failed, so not reporting any results.')
return results
def test_get_query_performance(self):
s_p = agg.EdwSuitePerformance('suite_name', 'suite_seq', 10)
q_p = agg.EdwQueryPerformance(Q1_NAME, Q1_PERFORMANCE, METADATA_EMPTY)
s_p.add_query_performance(q_p)
actual_query_performance = s_p.get_query_performance(Q1_NAME)
self.assertEqual(actual_query_performance.name, Q1_NAME)
self.assertEqual(actual_query_performance.performance, Q1_PERFORMANCE)
self.assertEqual(actual_query_performance.execution_status,
agg.EdwQueryExecutionStatus.SUCCESSFUL)
self.assertDictEqual(actual_query_performance.metadata, METADATA_EMPTY)
def test_aggregated_query_execution_time_passing(self):
b_p = agg.EdwBenchmarkPerformance(
total_iterations=2, expected_suite_queries=[Q1_NAME, Q2_NAME])
s1_p = agg.EdwSuitePerformance('suite_name', 'suite_seq_1', 2)
q11_p = agg.EdwQueryPerformance(Q1_NAME, 1.0, METADATA_EMPTY)
s1_p.add_query_performance(q11_p)
q12_p = agg.EdwQueryPerformance(Q2_NAME, 2.0, METADATA_EMPTY)
s1_p.add_query_performance(q12_p)
b_p.add_suite_performance('suite_seq_1', s1_p)
s2_p = agg.EdwSuitePerformance('suite_name', 'suite_seq_2', 2)
q21_p = agg.EdwQueryPerformance(Q1_NAME, 3.0, METADATA_EMPTY)
s2_p.add_query_performance(q21_p)
q22_p = agg.EdwQueryPerformance(Q2_NAME, 4.0, METADATA_EMPTY)
s2_p.add_query_performance(q22_p)
b_p.add_suite_performance('suite_seq_2', s2_p)
self.assertEqual(
b_p.aggregated_query_execution_time(Q1_NAME), (1.0 + 3.0) / 2)
self.assertEqual(
b_p.aggregated_query_execution_time(Q2_NAME), (2.0 + 4.0) / 2)
def test_add_query_performance(self):
i_p = agg.EdwPowerIterationPerformance('1', 2)
q1_p = agg.EdwQueryPerformance(Q1_NAME, Q1_PERFORMANCE, METADATA_EMPTY)
q2_p = agg.EdwQueryPerformance(Q2_NAME, Q2_PERFORMANCE, METADATA_EMPTY)
i_p.add_query_performance(Q1_NAME, Q1_PERFORMANCE, METADATA_EMPTY)
i_p.add_query_performance(Q2_NAME, Q2_PERFORMANCE, METADATA_EMPTY)
actual_iteration_performance = i_p.performance
expected_iteration_performance = {Q1_NAME: q1_p, Q2_NAME: q2_p}
self.assertSameElements(actual_iteration_performance.keys(),
expected_iteration_performance.keys())
self.assertEqual(actual_iteration_performance[Q1_NAME].performance,
expected_iteration_performance[Q1_NAME].performance)
self.assertDictEqual(actual_iteration_performance[Q1_NAME].metadata,
expected_iteration_performance[Q1_NAME].metadata)
self.assertEqual(actual_iteration_performance[Q2_NAME].performance,
expected_iteration_performance[Q2_NAME].performance)
self.assertDictEqual(actual_iteration_performance[Q2_NAME].metadata,
expected_iteration_performance[Q2_NAME].metadata)
self.assertEqual(i_p.total_count, 2)
self.assertEqual(i_p.successful_count, 2)
def test_get_performance_with_no_metadata(self):
q_p = agg.EdwQueryPerformance(Q1_NAME, Q1_PERFORMANCE, METADATA_EMPTY)
actual_sample = q_p.get_performance_sample(METADATA_EMPTY)
self.assertEqual(actual_sample.metric, 'edw_raw_query_time')
self.assertEqual(actual_sample.value, Q1_PERFORMANCE)
self.assertEqual(actual_sample.unit, SECS)
expected_metadata = {
'query': Q1_NAME,
'execution_status': agg.EdwQueryExecutionStatus.SUCCESSFUL
}
self.assertDictEqual(actual_sample.metadata, expected_metadata)
def test_aggregated_query_execution_time_missing_query(self):
b_p = agg.EdwBenchmarkPerformance(total_iterations=2)
s1_p = agg.EdwSuitePerformance('suite_name', 'suite_seq_1', 2)
q11_p = agg.EdwQueryPerformance(Q1_NAME, Q1_PERFORMANCE,
METADATA_EMPTY)
s1_p.add_query_performance(q11_p)
q12_p = agg.EdwQueryPerformance(Q2_NAME, Q2_PERFORMANCE,
METADATA_EMPTY)
s1_p.add_query_performance(q12_p)
b_p.add_suite_performance('suite_seq_1', s1_p)
s2_p = agg.EdwSuitePerformance('suite_name', 'suite_seq_2', 2)
q21_p = agg.EdwQueryPerformance(Q1_NAME, Q1_PERFORMANCE,
METADATA_EMPTY)
s2_p.add_query_performance(q21_p)
q22_p = agg.EdwQueryPerformance(Q2_NAME, Q2_PERFORMANCE,
METADATA_EMPTY)
s2_p.add_query_performance(q22_p)
b_p.add_suite_performance('suite_seq_2', s2_p)
with self.assertRaises(agg.EdwPerformanceAggregationError):
b_p.aggregated_query_execution_time(QFAIL_NAME)
def test_aggregated_query_status_passing(self):
b_p = agg.EdwBenchmarkPerformance(total_iterations=2)
s1_p = agg.EdwSuitePerformance('suite_name', 'suite_seq_1', 2)
q11_p = agg.EdwQueryPerformance(Q1_NAME, Q1_PERFORMANCE,
METADATA_EMPTY)
s1_p.add_query_performance(q11_p)
q12_p = agg.EdwQueryPerformance(Q2_NAME, Q2_PERFORMANCE,
METADATA_EMPTY)
s1_p.add_query_performance(q12_p)
b_p.add_suite_performance('suite_seq_1', s1_p)
s2_p = agg.EdwSuitePerformance('suite_name', 'suite_seq_2', 2)
q21_p = agg.EdwQueryPerformance(Q1_NAME, Q1_PERFORMANCE,
METADATA_EMPTY)
s2_p.add_query_performance(q21_p)
q22_p = agg.EdwQueryPerformance(Q2_NAME, Q2_PERFORMANCE,
METADATA_EMPTY)
s2_p.add_query_performance(q22_p)
b_p.add_suite_performance('suite_seq_2', s2_p)
self.assertTrue(b_p.aggregated_query_status(Q1_NAME))
self.assertTrue(b_p.aggregated_query_status(Q2_NAME))
def test_get_performance_with_query_and_suite_metadata(self):
q_p = agg.EdwQueryPerformance(QJOB_NAME, QJOB_PERFORMANCE, QJOB_METADATA)
actual_sample = q_p.get_performance_sample(SUITE_METATDATA)
self.assertEqual(actual_sample.metric, 'edw_raw_query_time')
self.assertEqual(actual_sample.value, QJOB_PERFORMANCE)
self.assertEqual(actual_sample.unit, SECS)
expected_metadata = {
'query': QJOB_NAME,
'execution_status': agg.EdwQueryExecutionStatus.SUCCESSFUL,
'job_id': QJOB_ID,
'suite_scale': 1
}
self.assertDictEqual(actual_sample.metadata, expected_metadata)
def test_is_successful_not_all_query_success(self):
b_p = agg.EdwBenchmarkPerformance(total_iterations=2)
s1_p = agg.EdwSuitePerformance('suite_name', 'suite_seq_1', 2)
q11_p = agg.EdwQueryPerformance(Q1_NAME, Q1_PERFORMANCE,
METADATA_EMPTY)
s1_p.add_query_performance(q11_p)
q12_p = agg.EdwQueryPerformance(Q2_NAME, Q2_PERFORMANCE,
METADATA_EMPTY)
s1_p.add_query_performance(q12_p)
b_p.add_suite_performance('suite_seq_1', s1_p)
s2_p = agg.EdwSuitePerformance('suite_name', 'suite_seq_2', 2)
q21_p = agg.EdwQueryPerformance(Q1_NAME, Q1_PERFORMANCE,
METADATA_EMPTY)
s2_p.add_query_performance(q21_p)
q22_p = agg.EdwQueryPerformance(Q2_NAME, QFAIL_PERFORMANCE,
METADATA_EMPTY)
s2_p.add_query_performance(q22_p)
b_p.add_suite_performance('suite_seq_2', s2_p)
self.assertEqual(len(b_p.suite_performances), 2)
self.assertFalse(b_p.is_successful())
self.assertSameElements(b_p.suite_performances.keys(),
['suite_seq_1', 'suite_seq_2'])
def test_is_successful(self):
q_p = agg.EdwQueryPerformance(Q1_NAME, Q1_PERFORMANCE, METADATA_EMPTY)
self.assertTrue(q_p.is_successful())
q_p = agg.EdwQueryPerformance(QFAIL_NAME, QFAIL_PERFORMANCE,
METADATA_EMPTY)
self.assertFalse(q_p.is_successful())
def test_get_performance_metadata(self):
q_p = agg.EdwQueryPerformance(Q1_NAME, Q1_PERFORMANCE, METADATA_EMPTY)
actual_md = q_p.get_performance_metadata()
self.assertDictEqual(actual_md, METADATA_EMPTY)
def test_get_performance_value(self):
q_p = agg.EdwQueryPerformance(Q1_NAME, Q1_PERFORMANCE, METADATA_EMPTY)
self.assertEqual(q_p.get_performance_value(), Q1_PERFORMANCE)
def build(self) -> agg.EdwQueryPerformance:
"""Builds an instance of agg.EdwQueryPerformance."""
return agg.EdwQueryPerformance(self.query_name, self.query_name,
self.metadata)