def test_get_queries_geomean_performance_samples_passing(self):
b_p = agg.EdwBenchmarkPerformance(total_iterations=2,
expected_queries=[Q1_NAME, Q2_NAME])
i1_p = agg.EdwPowerIterationPerformance('1', 2)
i1_p.add_query_performance(Q1_NAME, 1.0, {'job_id': 'q1_i1_job_id'})
i1_p.add_query_performance(Q2_NAME, 2.0, {'job_id': 'q2_i1_job_id'})
b_p.add_power_iteration_performance(i1_p)
i2_p = agg.EdwPowerIterationPerformance('2', 2)
i2_p.add_query_performance(Q1_NAME, 3.0, {'job_id': 'q1_i2_job_id'})
i2_p.add_query_performance(Q2_NAME, 4.0, {})
b_p.add_power_iteration_performance(i2_p)
actual_sample_list = b_p.get_queries_geomean_performance_samples(
{'benchmark_name': 'b_name'})
self.assertEqual(len(actual_sample_list), 3)
self.assertSameElements([x.metric for x in actual_sample_list], [
'edw_iteration_geomean_time', 'edw_iteration_geomean_time',
'edw_aggregated_geomean'
])
raw_samples = list(
filter(lambda x: x.metric == 'edw_iteration_geomean_time',
actual_sample_list))
actual_raw_samples_values = [x.value for x in raw_samples]
expected_raw_samples_values = [
agg.geometric_mean([1.0, 2.0]),
agg.geometric_mean([3.0, 4.0])
]
self.assertSameElements(actual_raw_samples_values,
expected_raw_samples_values)
aggregated_sample = list(
filter(lambda x: x.metric == 'edw_aggregated_geomean',
actual_sample_list))[0]
self.assertEqual(
aggregated_sample.value,
agg.geometric_mean([(1.0 + 3.0) / 2, (2.0 + 4.0) / 2]))
def build(self) -> agg.EdwBenchmarkPerformance:
"""Builds an instance of agg.EdwBenchmarkPerformance."""
benchmark_performance = agg.EdwBenchmarkPerformance(
self.total_iterations, self.expected_suite_queries)
for k, v in self.performance.items():
benchmark_performance.add_suite_performance(k, v)
return benchmark_performance
def test_aggregated_query_metadata_passing(self):
b_p = agg.EdwBenchmarkPerformance(
total_iterations=2, expected_suite_queries=[Q1_NAME, Q2_NAME])
s1_p = agg.EdwSuitePerformance('suite_name', 'suite_seq_1', 2)
q11_p = agg.EdwQueryPerformance(Q1_NAME, 1.0, {'job_id': 'q1_s1_job_id'})
s1_p.add_query_performance(q11_p)
q12_p = agg.EdwQueryPerformance(Q2_NAME, 2.0, {'job_id': 'q2_s1_job_id'})
s1_p.add_query_performance(q12_p)
b_p.add_suite_performance('suite_seq_1', s1_p)
s2_p = agg.EdwSuitePerformance('suite_name', 'suite_seq_2', 2)
q21_p = agg.EdwQueryPerformance(Q1_NAME, 3.0, {'job_id': 'q1_s2_job_id'})
s2_p.add_query_performance(q21_p)
q22_p = agg.EdwQueryPerformance(Q2_NAME, 4.0, {})
s2_p.add_query_performance(q22_p)
b_p.add_suite_performance('suite_seq_2', s2_p)
actual_aggregated_query_metadata_q1 = b_p.aggregated_query_metadata(Q1_NAME)
expected_aggregated_query_metadata_q1 = {
'suite_seq_1' + '_runtime': 1.0,
'suite_seq_1' + '_job_id': 'q1_s1_job_id',
'suite_seq_2' + '_runtime': 3.0,
'suite_seq_2' + '_job_id': 'q1_s2_job_id'
}
self.assertDictEqual(actual_aggregated_query_metadata_q1,
expected_aggregated_query_metadata_q1)
actual_aggregated_query_metadata_q2 = b_p.aggregated_query_metadata(Q2_NAME)
expected_aggregated_query_metadata_q2 = {
'suite_seq_1' + '_runtime': 2.0,
'suite_seq_1' + '_job_id': 'q2_s1_job_id',
'suite_seq_2' + '_runtime': 4.0
}
self.assertDictEqual(actual_aggregated_query_metadata_q2,
expected_aggregated_query_metadata_q2)
def test_get_all_query_performance_samples_passing(self):
b_p = agg.EdwBenchmarkPerformance(
total_iterations=2, expected_suite_queries=[Q1_NAME, Q2_NAME])
s1_p = agg.EdwSuitePerformance('suite_name', 'suite_seq_1', 2)
q11_p = agg.EdwQueryPerformance(Q1_NAME, 1.0,
{'job_id': 'q1_s1_job_id'})
s1_p.add_query_performance(q11_p)
q12_p = agg.EdwQueryPerformance(Q2_NAME, 2.0,
{'job_id': 'q2_s1_job_id'})
s1_p.add_query_performance(q12_p)
b_p.add_suite_performance('suite_seq_1', s1_p)
s2_p = agg.EdwSuitePerformance('suite_name', 'suite_seq_2', 2)
q21_p = agg.EdwQueryPerformance(Q1_NAME, 3.0,
{'job_id': 'q1_s2_job_id'})
s2_p.add_query_performance(q21_p)
q22_p = agg.EdwQueryPerformance(Q2_NAME, 4.0, {})
s2_p.add_query_performance(q22_p)
b_p.add_suite_performance('suite_seq_2', s2_p)
actual_sample_list = b_p.get_all_query_performance_samples({})
self.assertEqual(len(actual_sample_list), 6)
# 4 raw query samples and 2 aggregated samples
self.assertSameElements([x.metric for x in actual_sample_list], [
'edw_raw_query_time', 'edw_raw_query_time', 'edw_raw_query_time',
'edw_raw_query_time', 'edw_aggregated_query_time',
'edw_aggregated_query_time'
])
def test_get_wall_time_performance_samples_passing(self):
b_p = agg.EdwBenchmarkPerformance(
total_iterations=2, expected_suite_queries=[Q1_NAME, Q2_NAME])
s1_p = agg.EdwSuitePerformance('suite_name', 'suite_seq_1', 2)
q11_p = agg.EdwQueryPerformance(Q1_NAME, 1.0, {'job_id': 'q1_s1_job_id'})
s1_p.add_query_performance(q11_p)
q12_p = agg.EdwQueryPerformance(Q2_NAME, 2.0, {'job_id': 'q2_s1_job_id'})
s1_p.add_query_performance(q12_p)
b_p.add_suite_performance('suite_seq_1', s1_p)
s2_p = agg.EdwSuitePerformance('suite_name', 'suite_seq_2', 2)
q21_p = agg.EdwQueryPerformance(Q1_NAME, 3.0, {'job_id': 'q1_s2_job_id'})
s2_p.add_query_performance(q21_p)
q22_p = agg.EdwQueryPerformance(Q2_NAME, 4.0, {})
s2_p.add_query_performance(q22_p)
b_p.add_suite_performance('suite_seq_2', s2_p)
actual_sample_list = b_p.get_wall_time_performance_samples(
{'benchmark_name': 'b_name'})
self.assertEqual(len(actual_sample_list), 3)
self.assertSameElements(
[x.metric for x in actual_sample_list],
['edw_raw_wall_time', 'edw_raw_wall_time', 'edw_aggregated_wall_time'])
raw_samples = list(
filter(lambda x: x.metric == 'edw_raw_wall_time', actual_sample_list))
actual_raw_samples_values = [x.value for x in raw_samples]
expected_raw_samples_values = [(1.0 + 2.0), (3.0 + 4.0)]
self.assertSameElements(actual_raw_samples_values,
expected_raw_samples_values)
aggregated_sample = list(
filter(lambda x: x.metric == 'edw_aggregated_wall_time',
actual_sample_list))[0]
self.assertEqual(aggregated_sample.value, (1.0 + 3.0) / 2 + (2.0 + 4.0) / 2)
def test_aggregated_query_metadata_passing(self):
b_p = agg.EdwBenchmarkPerformance(total_iterations=2,
expected_queries=[Q1_NAME, Q2_NAME])
i1_p = agg.EdwPowerIterationPerformance('1', 2)
i1_p.add_query_performance(Q1_NAME, 1.0, {'job_id': 'q1_i1_job_id'})
i1_p.add_query_performance(Q2_NAME, 2.0, {'job_id': 'q2_i1_job_id'})
b_p.add_power_iteration_performance(i1_p)
i2_p = agg.EdwPowerIterationPerformance('2', 2)
i2_p.add_query_performance(Q1_NAME, 3.0, {'job_id': 'q1_i2_job_id'})
i2_p.add_query_performance(Q2_NAME, 4.0, {})
b_p.add_power_iteration_performance(i2_p)
actual_aggregated_query_metadata_q1 = b_p.aggregated_query_metadata(
Q1_NAME)
expected_aggregated_query_metadata_q1 = {
'1' + '_job_id': 'q1_i1_job_id',
'2' + '_job_id': 'q1_i2_job_id'
}
self.assertDictEqual(expected_aggregated_query_metadata_q1,
actual_aggregated_query_metadata_q1)
actual_aggregated_query_metadata_q2 = b_p.aggregated_query_metadata(
Q2_NAME)
expected_aggregated_query_metadata_q2 = {
'1' + '_job_id': 'q2_i1_job_id',
}
self.assertDictEqual(expected_aggregated_query_metadata_q2,
actual_aggregated_query_metadata_q2)
def test_get_aggregated_geomean_performance_sample_passing(self):
b_p = agg.EdwBenchmarkPerformance(
total_iterations=2, expected_suite_queries=[Q1_NAME, Q2_NAME])
s1_p = agg.EdwSuitePerformance('suite_name', 'suite_seq_1', 2)
q11_p = agg.EdwQueryPerformance(Q1_NAME, 1.0, {'job_id': 'q1_s1_job_id'})
s1_p.add_query_performance(q11_p)
q12_p = agg.EdwQueryPerformance(Q2_NAME, 2.0, {'job_id': 'q2_s1_job_id'})
s1_p.add_query_performance(q12_p)
b_p.add_suite_performance('suite_seq_1', s1_p)
s2_p = agg.EdwSuitePerformance('suite_name', 'suite_seq_2', 2)
q21_p = agg.EdwQueryPerformance(Q1_NAME, 3.0, {'job_id': 'q1_s2_job_id'})
s2_p.add_query_performance(q21_p)
q22_p = agg.EdwQueryPerformance(Q2_NAME, 4.0, {})
s2_p.add_query_performance(q22_p)
b_p.add_suite_performance('suite_seq_2', s2_p)
actual_sample = b_p.get_aggregated_geomean_performance_sample(
{'benchmark_name': 'b_name'})
self.assertEqual(actual_sample.metric, 'edw_aggregated_geomean')
self.assertEqual(actual_sample.value,
agg.geometric_mean([(1.0 + 3.0) / 2, (2.0 + 4.0) / 2]))
self.assertEqual(actual_sample.unit, 'seconds')
expected_metadata = {
'benchmark_name': 'b_name',
'intra_query_aggregation_method': 'mean',
'inter_query_aggregation_method': 'geomean'
}
self.assertDictEqual(actual_sample.metadata, expected_metadata)
def Run(benchmark_spec):
"""Run phase executes the sql scripts on edw cluster and collects duration."""
results = []
edw_service_instance = benchmark_spec.edw_service
# Run a warm up query in case there are cold start issues.
edw_service_instance.GetClientInterface().WarmUpQuery()
# Default to executing just the sample query if no queries are provided.
queries_to_execute = FLAGS.queries_to_execute or [
os.path.basename(edw_service.SAMPLE_QUERY_PATH)
]
# Accumulator for the entire benchmark's performance
benchmark_performance = results_aggregator.EdwBenchmarkPerformance(
total_iterations=FLAGS.edw_suite_iterations,
expected_suite_queries=queries_to_execute)
# Multiple iterations of the suite are performed to avoid cold start penalty
for i in range(1, FLAGS.edw_suite_iterations + 1):
iteration = str(i)
# Accumulator for the current suite's performance
suite_performance = results_aggregator.EdwSuitePerformance(
suite_name='edw',
suite_sequence=iteration,
total_suite_queries=len(queries_to_execute))
for query in queries_to_execute:
time, metadata = edw_service_instance.GetClientInterface(
).ExecuteQuery(query)
edw_query_performance = results_aggregator.EdwQueryPerformance(
query_name=query, performance=time, metadata=metadata)
suite_performance.add_query_performance(edw_query_performance)
benchmark_performance.add_suite_performance(iteration,
suite_performance)
# Execution complete, generate results only if the benchmark was successful.
benchmark_metadata = {}
benchmark_metadata.update(edw_service_instance.GetMetadata())
if benchmark_performance.is_successful():
query_samples = benchmark_performance.get_all_query_performance_samples(
metadata=benchmark_metadata)
results.extend(query_samples)
wall_time_samples = benchmark_performance.get_wall_time_performance_samples(
metadata=benchmark_metadata)
results.extend(wall_time_samples)
geomean_samples = (
benchmark_performance.get_queries_geomean_performance_samples(
metadata=benchmark_metadata))
results.extend(geomean_samples)
else:
logging.error(
'At least one query failed, so not reporting any results.')
return results
def test_add_power_iteration_performance(self):
b_p = agg.EdwBenchmarkPerformance(total_iterations=2,
expected_queries=[Q1_NAME])
i1_p = agg.EdwPowerIterationPerformance('1', 1)
i2_p = agg.EdwPowerIterationPerformance('2', 1)
i1_p.add_query_performance(Q1_NAME, Q1_PERFORMANCE, METADATA_EMPTY)
i2_p.add_query_performance(Q2_NAME, Q2_PERFORMANCE, METADATA_EMPTY)
b_p.add_power_iteration_performance(i1_p)
b_p.add_power_iteration_performance(i2_p)
self.assertEqual(len(b_p.iteration_performances), 2)
self.assertSameElements(b_p.iteration_performances.keys(), ['1', '2'])
def test_aggregated_query_metadata_failing_query(self):
b_p = agg.EdwBenchmarkPerformance(total_iterations=2,
expected_queries=[Q1_NAME, Q2_NAME])
i1_p = agg.EdwPowerIterationPerformance('1', 2)
i1_p.add_query_performance(Q1_NAME, Q1_PERFORMANCE, METADATA_EMPTY)
i1_p.add_query_performance(Q2_NAME, Q2_PERFORMANCE, METADATA_EMPTY)
b_p.add_power_iteration_performance(i1_p)
i2_p = agg.EdwPowerIterationPerformance('2', 2)
i2_p.add_query_performance(Q1_NAME, Q1_PERFORMANCE, METADATA_EMPTY)
i2_p.add_query_performance(Q2_NAME, QFAIL_PERFORMANCE, METADATA_EMPTY)
b_p.add_power_iteration_performance(i2_p)
with self.assertRaises(agg.EdwPerformanceAggregationError):
b_p.aggregated_query_metadata(Q2_NAME)
def test_aggregated_query_status_look_for_failing_query(self):
b_p = agg.EdwBenchmarkPerformance(total_iterations=2,
expected_queries=[Q1_NAME, Q2_NAME])
i1_p = agg.EdwPowerIterationPerformance('1', 2)
i1_p.add_query_performance(Q1_NAME, Q1_PERFORMANCE, METADATA_EMPTY)
i1_p.add_query_performance(Q2_NAME, Q2_PERFORMANCE, METADATA_EMPTY)
b_p.add_power_iteration_performance(i1_p)
i2_p = agg.EdwPowerIterationPerformance('2', 2)
i2_p.add_query_performance(Q1_NAME, Q1_PERFORMANCE, METADATA_EMPTY)
i2_p.add_query_performance(Q2_NAME, QFAIL_PERFORMANCE, METADATA_EMPTY)
b_p.add_power_iteration_performance(i2_p)
self.assertTrue(b_p.aggregated_query_status(Q1_NAME))
self.assertFalse(b_p.aggregated_query_status(Q2_NAME))
def test_add_suite_performance(self):
b_p = agg.EdwBenchmarkPerformance(
total_iterations=2, expected_suite_queries=[Q1_NAME])
s1_p = agg.EdwSuitePerformance('suite_name', 'suite_seq_1', 1)
q11_p = agg.EdwQueryPerformance(Q1_NAME, Q1_PERFORMANCE, METADATA_EMPTY)
s1_p.add_query_performance(q11_p)
b_p.add_suite_performance('suite_seq_1', s1_p)
s2_p = agg.EdwSuitePerformance('suite_name', 'suite_seq_2', 1)
q21_p = agg.EdwQueryPerformance(Q1_NAME, Q1_PERFORMANCE, METADATA_EMPTY)
s2_p.add_query_performance(q21_p)
b_p.add_suite_performance('suite_seq_2', s2_p)
self.assertEqual(len(b_p.suite_performances), 2)
self.assertSameElements(b_p.suite_performances.keys(),
['suite_seq_1', 'suite_seq_2'])
def test_aggregated_query_execution_time_passing(self):
b_p = agg.EdwBenchmarkPerformance(total_iterations=2,
expected_queries=[Q1_NAME, Q2_NAME])
i1_p = agg.EdwPowerIterationPerformance('1', 2)
i1_p.add_query_performance(Q1_NAME, Q1_PERFORMANCE, METADATA_EMPTY)
i1_p.add_query_performance(Q2_NAME, Q2_PERFORMANCE, METADATA_EMPTY)
b_p.add_power_iteration_performance(i1_p)
i2_p = agg.EdwPowerIterationPerformance('2', 2)
i2_p.add_query_performance(Q1_NAME, 3.0, METADATA_EMPTY)
i2_p.add_query_performance(Q2_NAME, 4.0, METADATA_EMPTY)
b_p.add_power_iteration_performance(i2_p)
self.assertEqual(b_p.aggregated_query_execution_time(Q1_NAME),
(1.0 + 3.0) / 2)
self.assertEqual(b_p.aggregated_query_execution_time(Q2_NAME),
(2.0 + 4.0) / 2)
def test_add_power_iteration_performance_duplicate_iteration(self):
"""Testing the scenario where a iteration with missing query is added."""
# Creating the bechmark performance
b_p = agg.EdwBenchmarkPerformance(total_iterations=2,
expected_queries=[Q1_NAME, Q2_NAME])
i1_p = agg.EdwPowerIterationPerformance('1', 1)
i1_p.add_query_performance(Q1_NAME, Q1_PERFORMANCE, METADATA_EMPTY)
i1_p.add_query_performance(Q2_NAME, Q2_PERFORMANCE, METADATA_EMPTY)
b_p.add_power_iteration_performance(i1_p)
i2_p = agg.EdwPowerIterationPerformance('1', 1)
i2_p.add_query_performance(Q1_NAME, Q1_PERFORMANCE, METADATA_EMPTY)
i2_p.add_query_performance(Q2_NAME, Q2_PERFORMANCE, METADATA_EMPTY)
# Expecting an error to be raised due duplicate iteration ID.
with self.assertRaises(agg.EdwPerformanceAggregationError):
b_p.add_power_iteration_performance(i2_p)
def test_get_aggregated_query_performance_sample_passing(self):
b_p = agg.EdwBenchmarkPerformance(
total_iterations=2, expected_suite_queries=[Q1_NAME, Q2_NAME])
s1_p = agg.EdwSuitePerformance('suite_name', 'suite_seq_1', 2)
q11_p = agg.EdwQueryPerformance(Q1_NAME, 1.0,
{'job_id': 'q1_s1_job_id'})
s1_p.add_query_performance(q11_p)
q12_p = agg.EdwQueryPerformance(Q2_NAME, 2.0,
{'job_id': 'q2_s1_job_id'})
s1_p.add_query_performance(q12_p)
b_p.add_suite_performance('suite_seq_1', s1_p)
s2_p = agg.EdwSuitePerformance('suite_name', 'suite_seq_2', 2)
q21_p = agg.EdwQueryPerformance(Q1_NAME, 3.0,
{'job_id': 'q1_s2_job_id'})
s2_p.add_query_performance(q21_p)
q22_p = agg.EdwQueryPerformance(Q2_NAME, 4.0, {})
s2_p.add_query_performance(q22_p)
b_p.add_suite_performance('suite_seq_2', s2_p)
actual_sample_q1 = b_p.get_aggregated_query_performance_sample(
Q1_NAME, {'benchmark_name': 'b_name'})
self.assertEqual(actual_sample_q1.metric, 'edw_aggregated_query_time')
self.assertEqual(actual_sample_q1.value, (1.0 + 3.0) / 2)
self.assertEqual(actual_sample_q1.unit, 'seconds')
expected_metadata_q1 = {
'suite_seq_1' + '_runtime': 1.0,
'suite_seq_1' + '_job_id': 'q1_s1_job_id',
'suite_seq_2' + '_runtime': 3.0,
'suite_seq_2' + '_job_id': 'q1_s2_job_id',
'query': Q1_NAME,
'aggregation_method': 'mean',
'execution_status': agg.EdwQueryExecutionStatus.SUCCESSFUL,
'benchmark_name': 'b_name'
}
self.assertDictEqual(actual_sample_q1.metadata, expected_metadata_q1)
actual_sample_q2 = b_p.get_aggregated_query_performance_sample(
Q2_NAME, {})
self.assertEqual(actual_sample_q2.metric, 'edw_aggregated_query_time')
self.assertEqual(actual_sample_q2.value, (2.0 + 4.0) / 2)
self.assertEqual(actual_sample_q2.unit, 'seconds')
expected_metadata_q2 = {
'suite_seq_1' + '_runtime': 2.0,
'suite_seq_1' + '_job_id': 'q2_s1_job_id',
'suite_seq_2' + '_runtime': 4.0,
'query': Q2_NAME,
'aggregation_method': 'mean',
'execution_status': agg.EdwQueryExecutionStatus.SUCCESSFUL
}
self.assertDictEqual(actual_sample_q2.metadata, expected_metadata_q2)
def test_aggregated_query_status_look_for_failing_query(self):
b_p = agg.EdwBenchmarkPerformance(
total_iterations=2, expected_suite_queries=[Q1_NAME, Q2_NAME])
s1_p = agg.EdwSuitePerformance('suite_name', 'suite_seq_1', 2)
q11_p = agg.EdwQueryPerformance(Q1_NAME, Q1_PERFORMANCE, METADATA_EMPTY)
s1_p.add_query_performance(q11_p)
q12_p = agg.EdwQueryPerformance(Q2_NAME, Q2_PERFORMANCE, METADATA_EMPTY)
s1_p.add_query_performance(q12_p)
b_p.add_suite_performance('suite_seq_1', s1_p)
s2_p = agg.EdwSuitePerformance('suite_name', 'suite_seq_2', 2)
q21_p = agg.EdwQueryPerformance(Q1_NAME, Q1_PERFORMANCE, METADATA_EMPTY)
s2_p.add_query_performance(q21_p)
q22_p = agg.EdwQueryPerformance(Q2_NAME, QFAIL_PERFORMANCE, METADATA_EMPTY)
s2_p.add_query_performance(q22_p)
b_p.add_suite_performance('suite_seq_2', s2_p)
self.assertTrue(b_p.aggregated_query_status(Q1_NAME))
self.assertFalse(b_p.aggregated_query_status(Q2_NAME))
def test_aggregated_query_metadata_failing_query(self):
b_p = agg.EdwBenchmarkPerformance(
total_iterations=2, expected_suite_queries=[Q1_NAME, Q2_NAME])
s1_p = agg.EdwSuitePerformance('suite_name', 'suite_seq_1', 2)
q11_p = agg.EdwQueryPerformance(Q1_NAME, Q1_PERFORMANCE, METADATA_EMPTY)
s1_p.add_query_performance(q11_p)
q12_p = agg.EdwQueryPerformance(Q2_NAME, Q2_PERFORMANCE, METADATA_EMPTY)
s1_p.add_query_performance(q12_p)
b_p.add_suite_performance('suite_seq_1', s1_p)
s2_p = agg.EdwSuitePerformance('suite_name', 'suite_seq_2', 2)
q21_p = agg.EdwQueryPerformance(Q1_NAME, Q1_PERFORMANCE, METADATA_EMPTY)
s2_p.add_query_performance(q21_p)
q22_p = agg.EdwQueryPerformance(Q2_NAME, QFAIL_PERFORMANCE, METADATA_EMPTY)
s2_p.add_query_performance(q22_p)
b_p.add_suite_performance('suite_seq_2', s2_p)
with self.assertRaises(agg.EdwPerformanceAggregationError):
b_p.aggregated_query_metadata(Q2_NAME)
def test_get_all_query_performance_samples_passing(self):
b_p = agg.EdwBenchmarkPerformance(total_iterations=2,
expected_queries=[Q1_NAME, Q2_NAME])
i1_p = agg.EdwPowerIterationPerformance('1', 2)
i1_p.add_query_performance(Q1_NAME, 1.0, {'job_id': 'q1_i1_job_id'})
i1_p.add_query_performance(Q2_NAME, 2.0, {'job_id': 'q2_i1_job_id'})
b_p.add_power_iteration_performance(i1_p)
i2_p = agg.EdwPowerIterationPerformance('2', 2)
i2_p.add_query_performance(Q1_NAME, 3.0, {'job_id': 'q1_i2_job_id'})
i2_p.add_query_performance(Q2_NAME, 4.0, {})
b_p.add_power_iteration_performance(i2_p)
actual_sample_list = b_p.get_all_query_performance_samples({})
self.assertEqual(len(actual_sample_list), 6)
# 4 raw query samples and 2 aggregated samples
self.assertSameElements([x.metric for x in actual_sample_list], [
'edw_raw_query_time', 'edw_raw_query_time', 'edw_raw_query_time',
'edw_raw_query_time', 'edw_aggregated_query_time',
'edw_aggregated_query_time'
])
def test_aggregated_query_execution_time_passing(self):
b_p = agg.EdwBenchmarkPerformance(
total_iterations=2, expected_suite_queries=[Q1_NAME, Q2_NAME])
s1_p = agg.EdwSuitePerformance('suite_name', 'suite_seq_1', 2)
q11_p = agg.EdwQueryPerformance(Q1_NAME, 1.0, METADATA_EMPTY)
s1_p.add_query_performance(q11_p)
q12_p = agg.EdwQueryPerformance(Q2_NAME, 2.0, METADATA_EMPTY)
s1_p.add_query_performance(q12_p)
b_p.add_suite_performance('suite_seq_1', s1_p)
s2_p = agg.EdwSuitePerformance('suite_name', 'suite_seq_2', 2)
q21_p = agg.EdwQueryPerformance(Q1_NAME, 3.0, METADATA_EMPTY)
s2_p.add_query_performance(q21_p)
q22_p = agg.EdwQueryPerformance(Q2_NAME, 4.0, METADATA_EMPTY)
s2_p.add_query_performance(q22_p)
b_p.add_suite_performance('suite_seq_2', s2_p)
self.assertEqual(
b_p.aggregated_query_execution_time(Q1_NAME), (1.0 + 3.0) / 2)
self.assertEqual(
b_p.aggregated_query_execution_time(Q2_NAME), (2.0 + 4.0) / 2)
def test_aggregated_query_status_passing(self):
b_p = agg.EdwBenchmarkPerformance(total_iterations=2)
s1_p = agg.EdwSuitePerformance('suite_name', 'suite_seq_1', 2)
q11_p = agg.EdwQueryPerformance(Q1_NAME, Q1_PERFORMANCE,
METADATA_EMPTY)
s1_p.add_query_performance(q11_p)
q12_p = agg.EdwQueryPerformance(Q2_NAME, Q2_PERFORMANCE,
METADATA_EMPTY)
s1_p.add_query_performance(q12_p)
b_p.add_suite_performance('suite_seq_1', s1_p)
s2_p = agg.EdwSuitePerformance('suite_name', 'suite_seq_2', 2)
q21_p = agg.EdwQueryPerformance(Q1_NAME, Q1_PERFORMANCE,
METADATA_EMPTY)
s2_p.add_query_performance(q21_p)
q22_p = agg.EdwQueryPerformance(Q2_NAME, Q2_PERFORMANCE,
METADATA_EMPTY)
s2_p.add_query_performance(q22_p)
b_p.add_suite_performance('suite_seq_2', s2_p)
self.assertTrue(b_p.aggregated_query_status(Q1_NAME))
self.assertTrue(b_p.aggregated_query_status(Q2_NAME))
def test_aggregated_query_execution_time_missing_query(self):
b_p = agg.EdwBenchmarkPerformance(total_iterations=2)
s1_p = agg.EdwSuitePerformance('suite_name', 'suite_seq_1', 2)
q11_p = agg.EdwQueryPerformance(Q1_NAME, Q1_PERFORMANCE,
METADATA_EMPTY)
s1_p.add_query_performance(q11_p)
q12_p = agg.EdwQueryPerformance(Q2_NAME, Q2_PERFORMANCE,
METADATA_EMPTY)
s1_p.add_query_performance(q12_p)
b_p.add_suite_performance('suite_seq_1', s1_p)
s2_p = agg.EdwSuitePerformance('suite_name', 'suite_seq_2', 2)
q21_p = agg.EdwQueryPerformance(Q1_NAME, Q1_PERFORMANCE,
METADATA_EMPTY)
s2_p.add_query_performance(q21_p)
q22_p = agg.EdwQueryPerformance(Q2_NAME, Q2_PERFORMANCE,
METADATA_EMPTY)
s2_p.add_query_performance(q22_p)
b_p.add_suite_performance('suite_seq_2', s2_p)
with self.assertRaises(agg.EdwPerformanceAggregationError):
b_p.aggregated_query_execution_time(QFAIL_NAME)
def test_is_successful_not_all_query_success(self):
b_p = agg.EdwBenchmarkPerformance(total_iterations=2)
s1_p = agg.EdwSuitePerformance('suite_name', 'suite_seq_1', 2)
q11_p = agg.EdwQueryPerformance(Q1_NAME, Q1_PERFORMANCE,
METADATA_EMPTY)
s1_p.add_query_performance(q11_p)
q12_p = agg.EdwQueryPerformance(Q2_NAME, Q2_PERFORMANCE,
METADATA_EMPTY)
s1_p.add_query_performance(q12_p)
b_p.add_suite_performance('suite_seq_1', s1_p)
s2_p = agg.EdwSuitePerformance('suite_name', 'suite_seq_2', 2)
q21_p = agg.EdwQueryPerformance(Q1_NAME, Q1_PERFORMANCE,
METADATA_EMPTY)
s2_p.add_query_performance(q21_p)
q22_p = agg.EdwQueryPerformance(Q2_NAME, QFAIL_PERFORMANCE,
METADATA_EMPTY)
s2_p.add_query_performance(q22_p)
b_p.add_suite_performance('suite_seq_2', s2_p)
self.assertEqual(len(b_p.suite_performances), 2)
self.assertFalse(b_p.is_successful())
self.assertSameElements(b_p.suite_performances.keys(),
['suite_seq_1', 'suite_seq_2'])
def test_get_aggregated_query_performance_sample_passing(self):
b_p = agg.EdwBenchmarkPerformance(total_iterations=2,
expected_queries=[Q1_NAME, Q2_NAME])
i1_p = agg.EdwPowerIterationPerformance('1', 2)
i1_p.add_query_performance(Q1_NAME, 1.0, {'job_id': 'q1_i1_job_id'})
i1_p.add_query_performance(Q2_NAME, 2.0, {'job_id': 'q2_i1_job_id'})
b_p.add_power_iteration_performance(i1_p)
i2_p = agg.EdwPowerIterationPerformance('2', 2)
i2_p.add_query_performance(Q1_NAME, 3.0, {'job_id': 'q1_i2_job_id'})
i2_p.add_query_performance(Q2_NAME, 4.0, {})
b_p.add_power_iteration_performance(i2_p)
actual_sample_q1 = b_p.get_aggregated_query_performance_sample(
Q1_NAME, {'benchmark_name': 'b_name'})
self.assertEqual(actual_sample_q1.metric, 'edw_aggregated_query_time')
self.assertEqual(actual_sample_q1.value, (1.0 + 3.0) / 2)
self.assertEqual(actual_sample_q1.unit, 'seconds')
expected_metadata_q1 = {
'1' + '_job_id': 'q1_i1_job_id',
'2' + '_job_id': 'q1_i2_job_id',
'query': Q1_NAME,
'aggregation_method': 'mean',
'execution_status': agg.EdwQueryExecutionStatus.SUCCESSFUL,
'benchmark_name': 'b_name'
}
self.assertDictEqual(actual_sample_q1.metadata, expected_metadata_q1)
actual_sample_q2 = b_p.get_aggregated_query_performance_sample(
Q2_NAME, {})
self.assertEqual(actual_sample_q2.metric, 'edw_aggregated_query_time')
self.assertEqual(actual_sample_q2.value, (2.0 + 4.0) / 2)
self.assertEqual(actual_sample_q2.unit, 'seconds')
expected_metadata_q2 = {
'1' + '_job_id': 'q2_i1_job_id',
'query': Q2_NAME,
'aggregation_method': 'mean',
'execution_status': agg.EdwQueryExecutionStatus.SUCCESSFUL
}
self.assertDictEqual(actual_sample_q2.metadata, expected_metadata_q2)
def test_get_aggregated_geomean_performance_sample_passing(self):
b_p = agg.EdwBenchmarkPerformance(total_iterations=2,
expected_queries=[Q1_NAME, Q2_NAME])
i1_p = agg.EdwPowerIterationPerformance('1', 2)
i1_p.add_query_performance(Q1_NAME, 1.0, {'job_id': 'q1_i1_job_id'})
i1_p.add_query_performance(Q2_NAME, 2.0, {'job_id': 'q2_i1_job_id'})
b_p.add_power_iteration_performance(i1_p)
i2_p = agg.EdwPowerIterationPerformance('2', 2)
i2_p.add_query_performance(Q1_NAME, 3.0, {'job_id': 'q1_i2_job_id'})
i2_p.add_query_performance(Q2_NAME, 4.0, {})
b_p.add_power_iteration_performance(i2_p)
actual_sample = b_p.get_aggregated_geomean_performance_sample(
{'benchmark_name': 'b_name'})
self.assertEqual(actual_sample.metric, 'edw_aggregated_geomean')
self.assertEqual(
actual_sample.value,
agg.geometric_mean([(1.0 + 3.0) / 2, (2.0 + 4.0) / 2]))
self.assertEqual(actual_sample.unit, 'seconds')
expected_metadata = {
'benchmark_name': 'b_name',
'intra_query_aggregation_method': 'mean',
'inter_query_aggregation_method': 'geomean'
}
self.assertDictEqual(actual_sample.metadata, expected_metadata)
