def test_vcpus_by_all_second_stage(self, usage_manager, setter_manager,
insert_manager, data_driven_specs_repo):
# load components
usage_manager.return_value = MockComponentManager.get_usage_cmpt_mgr()
setter_manager.return_value = \
MockComponentManager.get_setter_cmpt_mgr()
insert_manager.return_value = \
MockComponentManager.get_insert_pre_hourly_cmpt_mgr()
# init mock driver tables
data_driven_specs_repo.return_value = \
MockDataDrivenSpecsRepo(
self.spark_context,
self.get_pre_transform_specs_json_by_all(),
self.get_transform_specs_json_by_all())
# Create an RDD out of the mocked Monasca metrics
with open(SecondStageDataProvider.kafka_data_path_by_all) as f:
raw_lines = f.read().splitlines()
raw_tuple_list = [eval(raw_line) for raw_line in raw_lines]
vm_cpu_rdd = self.spark_context.parallelize(raw_tuple_list)
sql_context = SQLContext(self.spark_context)
vm_cpu_df = sql_context.read.json(vm_cpu_rdd)
PreHourlyProcessor.do_transform(vm_cpu_df)
metrics = DummyAdapter.adapter_impl.metric_list
vcpus_agg_metric = [
value for value in metrics
if value.get('metric').get('name') == 'vcpus_agg' and value.get(
'metric').get('dimensions').get('project_id') == 'all'
][0]
self.assertTrue(vcpus_agg_metric is not None)
self.assertEqual(7.0, vcpus_agg_metric.get('metric').get('value'))
self.assertEqual('useast', vcpus_agg_metric.get('meta').get('region'))
self.assertEqual(cfg.CONF.messaging.publish_kafka_project_id,
vcpus_agg_metric.get('meta').get('tenantId'))
self.assertEqual(
'all',
vcpus_agg_metric.get('metric').get('dimensions').get('host'))
self.assertEqual(
'prehourly',
vcpus_agg_metric.get('metric').get('dimensions').get(
'aggregation_period'))
self.assertEqual(
14.0,
vcpus_agg_metric.get('metric').get('value_meta').get(
'record_count'))
self.assertEqual(
'2016-01-20 16:40:00',
vcpus_agg_metric.get('metric').get('value_meta').get(
'firstrecord_timestamp_string'))
self.assertEqual(
'2016-01-20 16:40:46',
vcpus_agg_metric.get('metric').get('value_meta').get(
'lastrecord_timestamp_string'))
def test_fetch_quantity_sum(self, usage_manager, setter_manager,
insert_manager, data_driven_specs_repo):
# test operation
test_operation = "sum"
# load components
usage_manager.return_value = MockComponentManager.get_usage_cmpt_mgr()
setter_manager.return_value = \
MockComponentManager.get_setter_cmpt_mgr()
insert_manager.return_value = \
MockComponentManager.get_insert_cmpt_mgr()
# init mock driver tables
data_driven_specs_repo.return_value = \
MockDataDrivenSpecsRepo(self.spark_context,
self.get_pre_transform_specs_json(),
self.get_transform_specs_json_by_operation(
test_operation))
# Create an emulated set of Kafka messages (these were gathered
# by extracting Monasca messages from the Metrics queue on mini-mon).
# Create an RDD out of the mocked Monasca metrics
with open(DataProvider.kafka_data_path) as f:
raw_lines = f.read().splitlines()
raw_tuple_list = [eval(raw_line) for raw_line in raw_lines]
rdd_monasca = self.spark_context.parallelize(raw_tuple_list)
# decorate mocked RDD with dummy kafka offsets
myOffsetRanges = [OffsetRange("metrics", 1, 10,
20)] # mimic rdd.offsetRanges()
transform_context = TransformContextUtils.get_context(
offset_info=myOffsetRanges,
batch_time_info=self.get_dummy_batch_time())
rdd_monasca_with_offsets = rdd_monasca.map(
lambda x: RddTransformContext(x, transform_context))
try:
# Call the primary method in mon_metrics_kafka
MonMetricsKafkaProcessor.rdd_to_recordstore(
rdd_monasca_with_offsets)
self.assertTrue(False)
except FetchQuantityUtilException as e:
self.assertTrue("Operation sum is not supported" in e.value)
def test_missing_field_to_filter(self, usage_manager, setter_manager,
insert_manager, data_driven_specs_repo):
# load components
usage_manager.return_value = MockComponentManager.get_usage_cmpt_mgr()
setter_manager.return_value = \
MockComponentManager.get_setter_cmpt_mgr()
insert_manager.return_value = \
MockComponentManager.get_insert_cmpt_mgr()
# init mock driver tables
data_driven_specs_repo.return_value = \
MockDataDrivenSpecsRepo(
self.spark_context,
self.get_pre_transform_specs_json(),
self.get_invalid_filter_transform_specs_json("",
"-mgmt$",
"exclude"))
# Create an emulated set of Kafka messages (these were gathered
# by extracting Monasca messages from the Metrics queue on mini-mon).
# Create an RDD out of the mocked Monasca metrics
with open(DataProvider.fetch_quantity_data_path) as f:
raw_lines = f.read().splitlines()
raw_tuple_list = [eval(raw_line) for raw_line in raw_lines]
rdd_monasca = self.spark_context.parallelize(raw_tuple_list)
# decorate mocked RDD with dummy kafka offsets
myOffsetRanges = [OffsetRange("metrics", 1, 10,
20)] # mimic rdd.offsetRanges()
transform_context = TransformContextUtils.get_context(
offset_info=myOffsetRanges,
batch_time_info=self.get_dummy_batch_time())
rdd_monasca_with_offsets = rdd_monasca.map(
lambda x: RddTransformContext(x, transform_context))
try:
# Call the primary method in mon_metrics_kafka
MonMetricsKafkaProcessor.rdd_to_recordstore(
rdd_monasca_with_offsets)
# In this case, it's an error if no exception is caught
self.assertTrue(False)
except FetchQuantityException as e:
self.assertTrue("Encountered invalid filter details:" in e.value)
self.assertTrue("field to filter = ," in e.value)
def test_invalid_aggregated_metric_name(self, usage_manager,
setter_manager, insert_manager,
data_driven_specs_repo):
# load components
usage_manager.return_value = MockComponentManager.get_usage_cmpt_mgr()
setter_manager.return_value = \
MockComponentManager.get_setter_cmpt_mgr()
insert_manager.return_value = \
MockComponentManager.get_insert_cmpt_mgr()
# init mock driver tables
data_driven_specs_repo.return_value = \
MockDataDrivenSpecsRepo(
self.spark_context,
self.get_pre_transform_specs_json(),
self.get_transform_specs_json_invalid_name())
# Create an emulated set of Kafka messages (these were gathered
# by extracting Monasca messages from the Metrics queue on mini-mon).
# Create an RDD out of the mocked Monasca metrics
with open(DataProvider.fetch_quantity_data_path) as f:
raw_lines = f.read().splitlines()
raw_tuple_list = [eval(raw_line) for raw_line in raw_lines]
rdd_monasca = self.spark_context.parallelize(raw_tuple_list)
# decorate mocked RDD with dummy kafka offsets
myOffsetRanges = [OffsetRange("metrics", 1, 10,
20)] # mimic rdd.offsetRanges()
transform_context = TransformContextUtils.get_context(
offset_info=myOffsetRanges,
batch_time_info=self.get_dummy_batch_time())
rdd_monasca_with_offsets = rdd_monasca.map(
lambda x: RddTransformContext(x, transform_context))
# Call the primary method in mon_metrics_kafka
MonMetricsKafkaProcessor.rdd_to_recordstore(rdd_monasca_with_offsets)
# get the metrics that have been submitted to the dummy message adapter
metrics = DummyAdapter.adapter_impl.metric_list
# metrics should be empty
self.assertFalse(metrics)
def test_vcpus_by_all(self, usage_manager, setter_manager, insert_manager,
data_driven_specs_repo):
# load components
usage_manager.return_value = MockComponentManager.get_usage_cmpt_mgr()
setter_manager.return_value = \
MockComponentManager.get_setter_cmpt_mgr()
insert_manager.return_value = \
MockComponentManager.get_insert_pre_hourly_cmpt_mgr()
# init mock driver tables
data_driven_specs_repo.return_value = \
MockDataDrivenSpecsRepo(
self.spark_context,
self.get_pre_transform_specs_json_by_all(),
self.get_transform_specs_json_by_all())
# Create an RDD out of the mocked Monasca metrics
with open(DataProvider.kafka_data_path) as f:
raw_lines = f.read().splitlines()
raw_tuple_list = [eval(raw_line) for raw_line in raw_lines]
rdd_monasca = self.spark_context.parallelize(raw_tuple_list)
# decorate mocked RDD with dummy kafka offsets
myOffsetRanges = [OffsetRange("metrics", 1, 10,
20)] # mimic rdd.offsetRanges()
transform_context = TransformContextUtils.get_context(
offset_info=myOffsetRanges,
batch_time_info=self.get_dummy_batch_time())
rdd_monasca_with_offsets = rdd_monasca.map(
lambda x: RddTransformContext(x, transform_context))
# Call the primary method in mon_metrics_kafka
MonMetricsKafkaProcessor.rdd_to_recordstore(rdd_monasca_with_offsets)
# get the metrics that have been submitted to the dummy message adapter
metrics = DummyAdapter.adapter_impl.metric_list
vm_cpu_list = map(dump_as_ascii_string, metrics)
DummyAdapter.adapter_impl.metric_list = []
vm_cpu_rdd = self.spark_context.parallelize(vm_cpu_list)
sql_context = SQLContext(self.spark_context)
vm_cpu_df = sql_context.read.json(vm_cpu_rdd)
PreHourlyProcessor.do_transform(vm_cpu_df)
metrics = DummyAdapter.adapter_impl.metric_list
vcpus_agg_metric = [
value for value in metrics
if value.get('metric').get('name') == 'vcpus_agg' and value.get(
'metric').get('dimensions').get('project_id') == 'all'
][0]
self.assertTrue(vcpus_agg_metric is not None)
self.assertEqual(7.0, vcpus_agg_metric.get('metric').get('value'))
self.assertEqual('useast', vcpus_agg_metric.get('meta').get('region'))
self.assertEqual(cfg.CONF.messaging.publish_kafka_project_id,
vcpus_agg_metric.get('meta').get('tenantId'))
self.assertEqual(
'all',
vcpus_agg_metric.get('metric').get('dimensions').get('host'))
self.assertEqual(
'hourly',
vcpus_agg_metric.get('metric').get('dimensions').get(
'aggregation_period'))
self.assertEqual(
14.0,
vcpus_agg_metric.get('metric').get('value_meta').get(
'record_count'))
self.assertEqual(
'2016-01-20 16:40:00',
vcpus_agg_metric.get('metric').get('value_meta').get(
'firstrecord_timestamp_string'))
self.assertEqual(
'2016-01-20 16:40:46',
vcpus_agg_metric.get('metric').get('value_meta').get(
'lastrecord_timestamp_string'))
def test_fetch_quantity_sum_second_stage(self, usage_manager,
setter_manager, insert_manager,
data_driven_specs_repo):
# test operation
test_operation = "sum"
# load components
usage_manager.return_value = MockComponentManager.get_usage_cmpt_mgr()
setter_manager.return_value = \
MockComponentManager.get_setter_cmpt_mgr()
insert_manager.return_value = \
MockComponentManager.get_insert_pre_hourly_cmpt_mgr()
# init mock driver tables
data_driven_specs_repo.return_value = \
MockDataDrivenSpecsRepo(self.spark_context,
self.get_pre_transform_specs_json(),
self.get_transform_specs_json_by_operation(
test_operation))
# Create an emulated set of Kafka messages (these were gathered
# by extracting Monasca messages from the Metrics queue on mini-mon).
# Create an RDD out of the mocked Monasca metrics
with open(SecondStageDataProvider.fetch_quantity_data_path) as f:
raw_lines = f.read().splitlines()
raw_tuple_list = [eval(raw_line) for raw_line in raw_lines]
instance_usage_list = map(dump_as_ascii_string, raw_tuple_list)
# create a json RDD from instance_usage_list
instance_usage_rdd = self.spark_context.parallelize(
instance_usage_list)
sql_context = SQLContext(self.spark_context)
instance_usage_df = sql_context.read.json(instance_usage_rdd)
# call pre hourly processor
PreHourlyProcessor.do_transform(instance_usage_df)
metrics = DummyAdapter.adapter_impl.metric_list
mem_total_mb_agg_metric = [
value for value in metrics
if value.get('metric').get('name') == 'mem.total_mb_agg'
and value.get('metric').get('dimensions').get('host') == 'mini-mon'
][0]
self.assertTrue(mem_total_mb_agg_metric is not None)
self.assertEqual('mem.total_mb_agg',
mem_total_mb_agg_metric.get('metric').get('name'))
self.assertEqual(8679.0,
mem_total_mb_agg_metric.get('metric').get('value'))
self.assertEqual('useast',
mem_total_mb_agg_metric.get('meta').get('region'))
self.assertEqual(cfg.CONF.messaging.publish_kafka_project_id,
mem_total_mb_agg_metric.get('meta').get('tenantId'))
self.assertEqual(
'mini-mon',
mem_total_mb_agg_metric.get('metric').get('dimensions').get(
'host'))
self.assertEqual(
'all',
mem_total_mb_agg_metric.get('metric').get('dimensions').get(
'project_id'))
self.assertEqual(
'prehourly',
mem_total_mb_agg_metric.get('metric').get('dimensions').get(
'aggregation_period'))
self.assertEqual(
39.0,
mem_total_mb_agg_metric.get('metric').get('value_meta').get(
'record_count'))
self.assertEqual(
'2016-01-20 16:40:00',
mem_total_mb_agg_metric.get('metric').get('value_meta').get(
'firstrecord_timestamp_string'))
self.assertEqual(
'2016-01-20 17:40:00',
mem_total_mb_agg_metric.get('metric').get('value_meta').get(
'lastrecord_timestamp_string'))
def test_fetch_quantity_avg(self, usage_manager, setter_manager,
insert_manager, data_driven_specs_repo):
# test operation
test_operation = "avg"
# load components
usage_manager.return_value = MockComponentManager.get_usage_cmpt_mgr()
setter_manager.return_value = \
MockComponentManager.get_setter_cmpt_mgr()
insert_manager.return_value = \
MockComponentManager.get_insert_cmpt_mgr()
# init mock driver tables
data_driven_specs_repo.return_value = \
MockDataDrivenSpecsRepo(self.spark_context,
self.get_pre_transform_specs_json(),
self.get_transform_specs_json_by_operation(
test_operation))
# Create an emulated set of Kafka messages (these were gathered
# by extracting Monasca messages from the Metrics queue on mini-mon).
# Create an RDD out of the mocked Monasca metrics
with open(DataProvider.kafka_data_path) as f:
raw_lines = f.read().splitlines()
raw_tuple_list = [eval(raw_line) for raw_line in raw_lines]
rdd_monasca = self.spark_context.parallelize(raw_tuple_list)
# decorate mocked RDD with dummy kafka offsets
myOffsetRanges = [OffsetRange("metrics", 1, 10,
20)] # mimic rdd.offsetRanges()
transform_context = TransformContextUtils.get_context(
offset_info=myOffsetRanges,
batch_time_info=self.get_dummy_batch_time())
rdd_monasca_with_offsets = rdd_monasca.map(
lambda x: RddTransformContext(x, transform_context))
# Call the primary method in mon_metrics_kafka
MonMetricsKafkaProcessor.rdd_to_recordstore(rdd_monasca_with_offsets)
# get the metrics that have been submitted to the dummy message adapter
metrics = DummyAdapter.adapter_impl.metric_list
utilized_cpu_logical_agg_metric = [
value for value in metrics if value.get('metric').get('name') ==
'cpu.utilized_logical_cores_agg'
][0]
self.assertEqual(
7.134214285714285,
utilized_cpu_logical_agg_metric.get('metric').get('value'))
self.assertEqual(
'useast',
utilized_cpu_logical_agg_metric.get('meta').get('region'))
self.assertEqual(
cfg.CONF.messaging.publish_kafka_project_id,
utilized_cpu_logical_agg_metric.get('meta').get('tenantId'))
self.assertEqual(
'all',
utilized_cpu_logical_agg_metric.get('metric').get(
'dimensions').get('host'))
self.assertEqual(
'all',
utilized_cpu_logical_agg_metric.get('metric').get(
'dimensions').get('project_id'))
self.assertEqual(
'hourly',
utilized_cpu_logical_agg_metric.get('metric').get(
'dimensions').get('aggregation_period'))
self.assertEqual(
13.0,
utilized_cpu_logical_agg_metric.get('metric').get(
'value_meta').get('record_count'))
self.assertEqual(
'2016-03-07 16:09:23',
utilized_cpu_logical_agg_metric.get('metric').get(
'value_meta').get('firstrecord_timestamp_string'))
self.assertEqual(
'2016-03-07 16:10:38',
utilized_cpu_logical_agg_metric.get('metric').get(
'value_meta').get('lastrecord_timestamp_string'))
def test_fetch_quantity_avg_second_stage(self, usage_manager,
setter_manager, insert_manager,
data_driven_specs_repo):
# test operation
test_operation = "avg"
# load components
usage_manager.return_value = MockComponentManager.get_usage_cmpt_mgr()
setter_manager.return_value = \
MockComponentManager.get_setter_cmpt_mgr()
insert_manager.return_value = \
MockComponentManager.get_insert_pre_hourly_cmpt_mgr()
# init mock driver tables
data_driven_specs_repo.return_value = \
MockDataDrivenSpecsRepo(self.spark_context,
self.get_pre_transform_specs_json(),
self.get_transform_specs_json_by_operation(
test_operation, 'prehourly'))
# Create an RDD out of the mocked Monasca metrics
with open(SecondStageDataProvider.kafka_data_path) as f:
raw_lines = f.read().splitlines()
raw_tuple_list = [eval(raw_line) for raw_line in raw_lines]
util_list = map(dump_as_ascii_string, raw_tuple_list)
quantity_util_rdd = self.spark_context.parallelize(util_list)
sql_context = SQLContext(self.spark_context)
quantity_util_df = sql_context.read.json(quantity_util_rdd)
PreHourlyProcessor.do_transform(quantity_util_df)
metrics = DummyAdapter.adapter_impl.metric_list
utilized_cpu_logical_agg_metric = [
value for value in metrics if value.get('metric').get('name') ==
'cpu.utilized_logical_cores_agg'
][0]
self.assertEqual(
7.134214285714285,
utilized_cpu_logical_agg_metric.get('metric').get('value'))
self.assertEqual(
'useast',
utilized_cpu_logical_agg_metric.get('meta').get('region'))
self.assertEqual(
cfg.CONF.messaging.publish_kafka_project_id,
utilized_cpu_logical_agg_metric.get('meta').get('tenantId'))
self.assertEqual(
'all',
utilized_cpu_logical_agg_metric.get('metric').get(
'dimensions').get('host'))
self.assertEqual(
'all',
utilized_cpu_logical_agg_metric.get('metric').get(
'dimensions').get('project_id'))
self.assertEqual(
'hourly',
utilized_cpu_logical_agg_metric.get('metric').get(
'dimensions').get('aggregation_period'))
self.assertEqual(
13.0,
utilized_cpu_logical_agg_metric.get('metric').get(
'value_meta').get('record_count'))
self.assertEqual(
'2016-03-07 16:09:23',
utilized_cpu_logical_agg_metric.get('metric').get(
'value_meta').get('firstrecord_timestamp_string'))
self.assertEqual(
'2016-03-07 16:10:38',
utilized_cpu_logical_agg_metric.get('metric').get(
'value_meta').get('lastrecord_timestamp_string'))
def test_pod_net_in_usage_app(self, usage_manager, setter_manager,
insert_manager, data_driven_specs_repo):
# load components
usage_manager.return_value = MockComponentManager.get_usage_cmpt_mgr()
setter_manager.return_value = \
MockComponentManager.get_setter_cmpt_mgr()
insert_manager.return_value = \
MockComponentManager.get_insert_cmpt_mgr()
# init mock driver tables
data_driven_specs_repo.return_value = \
MockDataDrivenSpecsRepo(self.spark_context,
self.get_pre_transform_specs_json_app(),
self.get_transform_specs_json_app())
# Create an emulated set of Kafka messages (these were gathered
# by extracting Monasca messages from the Metrics queue on mini-mon).
# Create an RDD out of the mocked Monasca metrics
with open(DataProvider.fetch_quantity_data_path) as f:
raw_lines = f.read().splitlines()
raw_tuple_list = [eval(raw_line) for raw_line in raw_lines]
rdd_monasca = self.spark_context.parallelize(raw_tuple_list)
# decorate mocked RDD with dummy kafka offsets
myOffsetRanges = [OffsetRange("metrics", 1, 10,
20)] # mimic rdd.offsetRanges()
transform_context = TransformContextUtils.get_context(
offset_info=myOffsetRanges,
batch_time_info=self.get_dummy_batch_time())
rdd_monasca_with_offsets = rdd_monasca.map(
lambda x: RddTransformContext(x, transform_context))
# Call the primary method in mon_metrics_kafka
MonMetricsKafkaProcessor.rdd_to_recordstore(rdd_monasca_with_offsets)
# get the metrics that have been submitted to the dummy message adapter
metrics = DummyAdapter.adapter_impl.metric_list
pod_net_usage_agg_metric = [
value for value in metrics
if value.get('metric').get('name') == 'pod.net.in_bytes_sec_agg'
and value.get('metric').get('dimensions').get('app') == 'junk'
and value.get('metric').get('dimensions').get('namespace') == 'all'
and value.get('metric').get('dimensions').get('pod_name') == 'all'
][0]
self.assertTrue(pod_net_usage_agg_metric is not None)
self.assertEqual('pod.net.in_bytes_sec_agg',
pod_net_usage_agg_metric.get('metric').get('name'))
self.assertEqual(
'junk',
pod_net_usage_agg_metric.get("metric").get('dimensions').get(
'app'))
self.assertEqual(
'all',
pod_net_usage_agg_metric.get("metric").get('dimensions').get(
'namespace'))
self.assertEqual(
'all',
pod_net_usage_agg_metric.get("metric").get('dimensions').get(
'pod_name'))
self.assertEqual(122.94,
pod_net_usage_agg_metric.get('metric').get('value'))
self.assertEqual('useast',
pod_net_usage_agg_metric.get('meta').get('region'))
self.assertEqual(cfg.CONF.messaging.publish_kafka_project_id,
pod_net_usage_agg_metric.get('meta').get('tenantId'))
self.assertEqual(
'hourly',
pod_net_usage_agg_metric.get('metric').get('dimensions').get(
'aggregation_period'))
self.assertEqual(
3.0,
pod_net_usage_agg_metric.get('metric').get('value_meta').get(
'record_count'))
self.assertEqual(
'2017-01-24 20:14:47',
pod_net_usage_agg_metric.get('metric').get('value_meta').get(
'firstrecord_timestamp_string'))
self.assertEqual(
'2017-01-24 20:15:47',
pod_net_usage_agg_metric.get('metric').get('value_meta').get(
'lastrecord_timestamp_string'))