def test_aggregated_query_metadata_passing(self):
b_p = agg.EdwBenchmarkPerformance(total_iterations=2,
expected_queries=[Q1_NAME, Q2_NAME])
i1_p = agg.EdwPowerIterationPerformance('1', 2)
i1_p.add_query_performance(Q1_NAME, 1.0, {'job_id': 'q1_i1_job_id'})
i1_p.add_query_performance(Q2_NAME, 2.0, {'job_id': 'q2_i1_job_id'})
b_p.add_power_iteration_performance(i1_p)
i2_p = agg.EdwPowerIterationPerformance('2', 2)
i2_p.add_query_performance(Q1_NAME, 3.0, {'job_id': 'q1_i2_job_id'})
i2_p.add_query_performance(Q2_NAME, 4.0, {})
b_p.add_power_iteration_performance(i2_p)
actual_aggregated_query_metadata_q1 = b_p.aggregated_query_metadata(
Q1_NAME)
expected_aggregated_query_metadata_q1 = {
'1' + '_job_id': 'q1_i1_job_id',
'2' + '_job_id': 'q1_i2_job_id'
}
self.assertDictEqual(expected_aggregated_query_metadata_q1,
actual_aggregated_query_metadata_q1)
actual_aggregated_query_metadata_q2 = b_p.aggregated_query_metadata(
Q2_NAME)
expected_aggregated_query_metadata_q2 = {
'1' + '_job_id': 'q2_i1_job_id',
}
self.assertDictEqual(expected_aggregated_query_metadata_q2,
actual_aggregated_query_metadata_q2)
def test_get_queries_geomean_performance_samples_passing(self):
b_p = agg.EdwBenchmarkPerformance(total_iterations=2,
expected_queries=[Q1_NAME, Q2_NAME])
i1_p = agg.EdwPowerIterationPerformance('1', 2)
i1_p.add_query_performance(Q1_NAME, 1.0, {'job_id': 'q1_i1_job_id'})
i1_p.add_query_performance(Q2_NAME, 2.0, {'job_id': 'q2_i1_job_id'})
b_p.add_power_iteration_performance(i1_p)
i2_p = agg.EdwPowerIterationPerformance('2', 2)
i2_p.add_query_performance(Q1_NAME, 3.0, {'job_id': 'q1_i2_job_id'})
i2_p.add_query_performance(Q2_NAME, 4.0, {})
b_p.add_power_iteration_performance(i2_p)
actual_sample_list = b_p.get_queries_geomean_performance_samples(
{'benchmark_name': 'b_name'})
self.assertEqual(len(actual_sample_list), 3)
self.assertSameElements([x.metric for x in actual_sample_list], [
'edw_iteration_geomean_time', 'edw_iteration_geomean_time',
'edw_aggregated_geomean'
])
raw_samples = list(
filter(lambda x: x.metric == 'edw_iteration_geomean_time',
actual_sample_list))
actual_raw_samples_values = [x.value for x in raw_samples]
expected_raw_samples_values = [
agg.geometric_mean([1.0, 2.0]),
agg.geometric_mean([3.0, 4.0])
]
self.assertSameElements(actual_raw_samples_values,
expected_raw_samples_values)
aggregated_sample = list(
filter(lambda x: x.metric == 'edw_aggregated_geomean',
actual_sample_list))[0]
self.assertEqual(
aggregated_sample.value,
agg.geometric_mean([(1.0 + 3.0) / 2, (2.0 + 4.0) / 2]))
def test_add_power_iteration_performance(self):
b_p = agg.EdwBenchmarkPerformance(total_iterations=2,
expected_queries=[Q1_NAME])
i1_p = agg.EdwPowerIterationPerformance('1', 1)
i2_p = agg.EdwPowerIterationPerformance('2', 1)
i1_p.add_query_performance(Q1_NAME, Q1_PERFORMANCE, METADATA_EMPTY)
i2_p.add_query_performance(Q2_NAME, Q2_PERFORMANCE, METADATA_EMPTY)
b_p.add_power_iteration_performance(i1_p)
b_p.add_power_iteration_performance(i2_p)
self.assertEqual(len(b_p.iteration_performances), 2)
self.assertSameElements(b_p.iteration_performances.keys(), ['1', '2'])
def test_is_successful(self):
i_p = agg.EdwPowerIterationPerformance('1', 2)
i_p.add_query_performance(Q1_NAME, Q1_PERFORMANCE, METADATA_EMPTY)
i_p.add_query_performance(Q2_NAME, Q2_PERFORMANCE, METADATA_EMPTY)
self.assertTrue(i_p.is_successful(expected_queries=[Q1_NAME, Q2_NAME]))
i2_p = agg.EdwPowerIterationPerformance('2', 2)
i_p.add_query_performance(QFAIL_NAME, QFAIL_PERFORMANCE,
METADATA_EMPTY)
self.assertFalse(
i2_p.is_successful(
expected_queries=[Q1_NAME, Q2_NAME, QFAIL_NAME]))
def test_aggregated_query_status_look_for_failing_query(self):
b_p = agg.EdwBenchmarkPerformance(total_iterations=2,
expected_queries=[Q1_NAME, Q2_NAME])
i1_p = agg.EdwPowerIterationPerformance('1', 2)
i1_p.add_query_performance(Q1_NAME, Q1_PERFORMANCE, METADATA_EMPTY)
i1_p.add_query_performance(Q2_NAME, Q2_PERFORMANCE, METADATA_EMPTY)
b_p.add_power_iteration_performance(i1_p)
i2_p = agg.EdwPowerIterationPerformance('2', 2)
i2_p.add_query_performance(Q1_NAME, Q1_PERFORMANCE, METADATA_EMPTY)
i2_p.add_query_performance(Q2_NAME, QFAIL_PERFORMANCE, METADATA_EMPTY)
b_p.add_power_iteration_performance(i2_p)
self.assertTrue(b_p.aggregated_query_status(Q1_NAME))
self.assertFalse(b_p.aggregated_query_status(Q2_NAME))
def test_aggregated_query_metadata_failing_query(self):
b_p = agg.EdwBenchmarkPerformance(total_iterations=2,
expected_queries=[Q1_NAME, Q2_NAME])
i1_p = agg.EdwPowerIterationPerformance('1', 2)
i1_p.add_query_performance(Q1_NAME, Q1_PERFORMANCE, METADATA_EMPTY)
i1_p.add_query_performance(Q2_NAME, Q2_PERFORMANCE, METADATA_EMPTY)
b_p.add_power_iteration_performance(i1_p)
i2_p = agg.EdwPowerIterationPerformance('2', 2)
i2_p.add_query_performance(Q1_NAME, Q1_PERFORMANCE, METADATA_EMPTY)
i2_p.add_query_performance(Q2_NAME, QFAIL_PERFORMANCE, METADATA_EMPTY)
b_p.add_power_iteration_performance(i2_p)
with self.assertRaises(agg.EdwPerformanceAggregationError):
b_p.aggregated_query_metadata(Q2_NAME)
def test_add_power_iteration_performance_duplicate_iteration(self):
"""Testing the scenario where a iteration with missing query is added."""
# Creating the bechmark performance
b_p = agg.EdwBenchmarkPerformance(total_iterations=2,
expected_queries=[Q1_NAME, Q2_NAME])
i1_p = agg.EdwPowerIterationPerformance('1', 1)
i1_p.add_query_performance(Q1_NAME, Q1_PERFORMANCE, METADATA_EMPTY)
i1_p.add_query_performance(Q2_NAME, Q2_PERFORMANCE, METADATA_EMPTY)
b_p.add_power_iteration_performance(i1_p)
i2_p = agg.EdwPowerIterationPerformance('1', 1)
i2_p.add_query_performance(Q1_NAME, Q1_PERFORMANCE, METADATA_EMPTY)
i2_p.add_query_performance(Q2_NAME, Q2_PERFORMANCE, METADATA_EMPTY)
# Expecting an error to be raised due duplicate iteration ID.
with self.assertRaises(agg.EdwPerformanceAggregationError):
b_p.add_power_iteration_performance(i2_p)
def test_aggregated_query_execution_time_passing(self):
b_p = agg.EdwBenchmarkPerformance(total_iterations=2,
expected_queries=[Q1_NAME, Q2_NAME])
i1_p = agg.EdwPowerIterationPerformance('1', 2)
i1_p.add_query_performance(Q1_NAME, Q1_PERFORMANCE, METADATA_EMPTY)
i1_p.add_query_performance(Q2_NAME, Q2_PERFORMANCE, METADATA_EMPTY)
b_p.add_power_iteration_performance(i1_p)
i2_p = agg.EdwPowerIterationPerformance('2', 2)
i2_p.add_query_performance(Q1_NAME, 3.0, METADATA_EMPTY)
i2_p.add_query_performance(Q2_NAME, 4.0, METADATA_EMPTY)
b_p.add_power_iteration_performance(i2_p)
self.assertEqual(b_p.aggregated_query_execution_time(Q1_NAME),
(1.0 + 3.0) / 2)
self.assertEqual(b_p.aggregated_query_execution_time(Q2_NAME),
(2.0 + 4.0) / 2)
def test_has_query_performance(self):
i_p = agg.EdwPowerIterationPerformance('1', 2)
i_p.add_query_performance(Q1_NAME, Q1_PERFORMANCE, METADATA_EMPTY)
self.assertTrue(i_p.has_query_performance(Q1_NAME))
self.assertFalse(i_p.has_query_performance(Q2_NAME))
i_p.add_query_performance(Q2_NAME, Q2_PERFORMANCE, METADATA_EMPTY)
self.assertTrue(i_p.has_query_performance(Q2_NAME))
def test_get_all_query_performance_samples_passing(self):
b_p = agg.EdwBenchmarkPerformance(total_iterations=2,
expected_queries=[Q1_NAME, Q2_NAME])
i1_p = agg.EdwPowerIterationPerformance('1', 2)
i1_p.add_query_performance(Q1_NAME, 1.0, {'job_id': 'q1_i1_job_id'})
i1_p.add_query_performance(Q2_NAME, 2.0, {'job_id': 'q2_i1_job_id'})
b_p.add_power_iteration_performance(i1_p)
i2_p = agg.EdwPowerIterationPerformance('2', 2)
i2_p.add_query_performance(Q1_NAME, 3.0, {'job_id': 'q1_i2_job_id'})
i2_p.add_query_performance(Q2_NAME, 4.0, {})
b_p.add_power_iteration_performance(i2_p)
actual_sample_list = b_p.get_all_query_performance_samples({})
self.assertEqual(len(actual_sample_list), 6)
# 4 raw query samples and 2 aggregated samples
self.assertSameElements([x.metric for x in actual_sample_list], [
'edw_raw_query_time', 'edw_raw_query_time', 'edw_raw_query_time',
'edw_raw_query_time', 'edw_aggregated_query_time',
'edw_aggregated_query_time'
])
def test_get_queries_geomean_performance(self):
i_p = agg.EdwPowerIterationPerformance('1', 2)
i_p.add_query_performance(Q1_NAME, Q1_PERFORMANCE, METADATA_EMPTY)
i_p.add_query_performance(Q2_NAME, Q2_PERFORMANCE, METADATA_EMPTY)
iteration_geomean_performance_sample = i_p.get_queries_geomean_performance_sample(
expected_queries=[Q1_NAME, Q2_NAME], metadata=METADATA_EMPTY)
self.assertEqual(iteration_geomean_performance_sample.metric,
'edw_iteration_geomean_time')
self.assertEqual(iteration_geomean_performance_sample.value,
agg.geometric_mean([Q1_PERFORMANCE, Q2_PERFORMANCE]))
self.assertEqual(iteration_geomean_performance_sample.unit, SECS)
self.assertDictEqual(iteration_geomean_performance_sample.metadata,
METADATA_EMPTY)
def test_get_aggregated_query_performance_sample_passing(self):
b_p = agg.EdwBenchmarkPerformance(total_iterations=2,
expected_queries=[Q1_NAME, Q2_NAME])
i1_p = agg.EdwPowerIterationPerformance('1', 2)
i1_p.add_query_performance(Q1_NAME, 1.0, {'job_id': 'q1_i1_job_id'})
i1_p.add_query_performance(Q2_NAME, 2.0, {'job_id': 'q2_i1_job_id'})
b_p.add_power_iteration_performance(i1_p)
i2_p = agg.EdwPowerIterationPerformance('2', 2)
i2_p.add_query_performance(Q1_NAME, 3.0, {'job_id': 'q1_i2_job_id'})
i2_p.add_query_performance(Q2_NAME, 4.0, {})
b_p.add_power_iteration_performance(i2_p)
actual_sample_q1 = b_p.get_aggregated_query_performance_sample(
Q1_NAME, {'benchmark_name': 'b_name'})
self.assertEqual(actual_sample_q1.metric, 'edw_aggregated_query_time')
self.assertEqual(actual_sample_q1.value, (1.0 + 3.0) / 2)
self.assertEqual(actual_sample_q1.unit, 'seconds')
expected_metadata_q1 = {
'1' + '_job_id': 'q1_i1_job_id',
'2' + '_job_id': 'q1_i2_job_id',
'query': Q1_NAME,
'aggregation_method': 'mean',
'execution_status': agg.EdwQueryExecutionStatus.SUCCESSFUL,
'benchmark_name': 'b_name'
}
self.assertDictEqual(actual_sample_q1.metadata, expected_metadata_q1)
actual_sample_q2 = b_p.get_aggregated_query_performance_sample(
Q2_NAME, {})
self.assertEqual(actual_sample_q2.metric, 'edw_aggregated_query_time')
self.assertEqual(actual_sample_q2.value, (2.0 + 4.0) / 2)
self.assertEqual(actual_sample_q2.unit, 'seconds')
expected_metadata_q2 = {
'1' + '_job_id': 'q2_i1_job_id',
'query': Q2_NAME,
'aggregation_method': 'mean',
'execution_status': agg.EdwQueryExecutionStatus.SUCCESSFUL
}
self.assertDictEqual(actual_sample_q2.metadata, expected_metadata_q2)
def Run(benchmark_spec):
"""Run phase executes the sql scripts on edw cluster and collects duration."""
results = []
edw_service_instance = benchmark_spec.edw_service
client_interface = edw_service_instance.GetClientInterface()
# Run a warm up query in case there are cold start issues.
client_interface.WarmUpQuery()
# Default to executing just the sample query if no queries are provided.
all_queries = FLAGS.edw_power_queries.split(',') or [
os.path.basename(edw_service.SAMPLE_QUERY_PATH)
]
# Accumulator for the entire benchmark's performance
benchmark_performance = results_aggregator.EdwBenchmarkPerformance(
total_iterations=FLAGS.edw_suite_iterations,
expected_queries=all_queries)
# Multiple iterations of the suite are performed to avoid cold start penalty
for i in range(1, FLAGS.edw_suite_iterations + 1):
iteration = str(i)
# Accumulator for the current suite's performance
iteration_performance = results_aggregator.EdwPowerIterationPerformance(
iteration_id=iteration, total_queries=len(all_queries))
for query in all_queries:
execution_time, metadata = client_interface.ExecuteQuery(query)
iteration_performance.add_query_performance(
query, execution_time, metadata)
benchmark_performance.add_iteration_performance(iteration_performance)
# Execution complete, generate results only if the benchmark was successful.
benchmark_metadata = {}
benchmark_metadata.update(edw_service_instance.GetMetadata())
if benchmark_performance.is_successful():
query_samples = benchmark_performance.get_all_query_performance_samples(
metadata=benchmark_metadata)
results.extend(query_samples)
geomean_samples = (
benchmark_performance.get_queries_geomean_performance_samples(
metadata=benchmark_metadata))
results.extend(geomean_samples)
else:
logging.error(
'At least one query failed, so not reporting any results.')
return results
def test_get_aggregated_geomean_performance_sample_passing(self):
b_p = agg.EdwBenchmarkPerformance(total_iterations=2,
expected_queries=[Q1_NAME, Q2_NAME])
i1_p = agg.EdwPowerIterationPerformance('1', 2)
i1_p.add_query_performance(Q1_NAME, 1.0, {'job_id': 'q1_i1_job_id'})
i1_p.add_query_performance(Q2_NAME, 2.0, {'job_id': 'q2_i1_job_id'})
b_p.add_power_iteration_performance(i1_p)
i2_p = agg.EdwPowerIterationPerformance('2', 2)
i2_p.add_query_performance(Q1_NAME, 3.0, {'job_id': 'q1_i2_job_id'})
i2_p.add_query_performance(Q2_NAME, 4.0, {})
b_p.add_power_iteration_performance(i2_p)
actual_sample = b_p.get_aggregated_geomean_performance_sample(
{'benchmark_name': 'b_name'})
self.assertEqual(actual_sample.metric, 'edw_aggregated_geomean')
self.assertEqual(
actual_sample.value,
agg.geometric_mean([(1.0 + 3.0) / 2, (2.0 + 4.0) / 2]))
self.assertEqual(actual_sample.unit, 'seconds')
expected_metadata = {
'benchmark_name': 'b_name',
'intra_query_aggregation_method': 'mean',
'inter_query_aggregation_method': 'geomean'
}
self.assertDictEqual(actual_sample.metadata, expected_metadata)
def test_get_query_metadata(self):
i_p = agg.EdwPowerIterationPerformance('1', 4)
i_p.add_query_performance(Q1_NAME, Q1_PERFORMANCE,
{'job_id': 'q1_i1_job_id'})
i_p.add_query_performance(Q2_NAME, Q2_PERFORMANCE, {})
actual_aggregated_query_metadata_q1 = i_p.get_query_metadata(Q1_NAME)
expected_aggregated_query_metadata_q1 = {
'job_id': 'q1_i1_job_id',
}
self.assertDictEqual(actual_aggregated_query_metadata_q1,
expected_aggregated_query_metadata_q1)
actual_aggregated_query_metadata_q2 = i_p.get_query_metadata(Q2_NAME)
expected_aggregated_query_metadata_q2 = {}
self.assertDictEqual(actual_aggregated_query_metadata_q2,
expected_aggregated_query_metadata_q2)
def test_add_query_performance(self):
i_p = agg.EdwPowerIterationPerformance('1', 2)
q1_p = agg.EdwQueryPerformance(Q1_NAME, Q1_PERFORMANCE, METADATA_EMPTY)
q2_p = agg.EdwQueryPerformance(Q2_NAME, Q2_PERFORMANCE, METADATA_EMPTY)
i_p.add_query_performance(Q1_NAME, Q1_PERFORMANCE, METADATA_EMPTY)
i_p.add_query_performance(Q2_NAME, Q2_PERFORMANCE, METADATA_EMPTY)
actual_iteration_performance = i_p.performance
expected_iteration_performance = {Q1_NAME: q1_p, Q2_NAME: q2_p}
self.assertSameElements(actual_iteration_performance.keys(),
expected_iteration_performance.keys())
self.assertEqual(actual_iteration_performance[Q1_NAME].performance,
expected_iteration_performance[Q1_NAME].performance)
self.assertDictEqual(actual_iteration_performance[Q1_NAME].metadata,
expected_iteration_performance[Q1_NAME].metadata)
self.assertEqual(actual_iteration_performance[Q2_NAME].performance,
expected_iteration_performance[Q2_NAME].performance)
self.assertDictEqual(actual_iteration_performance[Q2_NAME].metadata,
expected_iteration_performance[Q2_NAME].metadata)
self.assertEqual(i_p.total_count, 2)
self.assertEqual(i_p.successful_count, 2)
def test_get_all_query_performance_samples(self):
i_p = agg.EdwPowerIterationPerformance('1', 10)
i_p.add_query_performance(Q1_NAME, Q1_PERFORMANCE, METADATA_EMPTY)
i_p.add_query_performance(Q2_NAME, Q2_PERFORMANCE, METADATA_EMPTY)
actual_all_query_performance = i_p.get_all_query_performance_samples(
METADATA_EMPTY)
self.assertEqual(len(actual_all_query_performance), 2)
self.assertListEqual([x.metric for x in actual_all_query_performance],
['edw_raw_query_time', 'edw_raw_query_time'])
self.assertSameElements(
[x.value for x in actual_all_query_performance],
[Q1_PERFORMANCE, Q2_PERFORMANCE])
self.assertListEqual([x.unit for x in actual_all_query_performance],
[SECS, SECS])
self.assertSameElements(
[x.metadata['query'] for x in actual_all_query_performance],
[Q1_NAME, Q2_NAME])
self.assertSameElements([
x.metadata['execution_status']
for x in actual_all_query_performance
], [
agg.EdwQueryExecutionStatus.SUCCESSFUL,
agg.EdwQueryExecutionStatus.SUCCESSFUL
])
def test_get_query_performance(self):
i_p = agg.EdwPowerIterationPerformance('1', 2)
i_p.add_query_performance(Q1_NAME, Q1_PERFORMANCE, METADATA_EMPTY)
actual_query_performance = i_p.get_query_performance(Q1_NAME)
self.assertEqual(actual_query_performance, Q1_PERFORMANCE)
