def process_metrics(transform_context, record_store_df):
"""start processing (aggregating) metrics
"""
#
# look in record_store_df for list of metrics to be processed
#
metric_ids_df = record_store_df.select("metric_id").distinct()
metric_ids_to_process = [row.metric_id
for row in metric_ids_df.collect()]
data_driven_specs_repo = DataDrivenSpecsRepoFactory.\
get_data_driven_specs_repo()
sqlc = SQLContext.getOrCreate(record_store_df.rdd.context)
transform_specs_df = data_driven_specs_repo.get_data_driven_specs(
sql_context=sqlc,
data_driven_spec_type=DataDrivenSpecsRepo.transform_specs_type)
for metric_id in metric_ids_to_process:
transform_spec_df = transform_specs_df.select(
["aggregation_params_map", "metric_id"]
).where(transform_specs_df.metric_id == metric_id)
source_record_store_df = record_store_df.select("*").where(
record_store_df.metric_id == metric_id)
# set transform_spec_df in TransformContext
transform_context = \
TransformContextUtils.get_context(
transform_context_info=transform_context,
transform_spec_df_info=transform_spec_df)
MonMetricsKafkaProcessor.process_metric(
transform_context, source_record_store_df)
def test_set_aggregated_metric_name(self):
record_store_df = RecordStoreUtils.create_df_from_json(
self.sql_context,
DataProvider.record_store_path)
transform_spec_df = TransformSpecsUtils.create_df_from_json(
self.sql_context,
DataProvider.transform_spec_path)
transform_context = TransformContextUtils.get_context(
transform_spec_df_info=transform_spec_df,
batch_time_info=self.get_dummy_batch_time())
instance_usage_df = FetchQuantity.usage(
transform_context, record_store_df)
instance_usage_df_1 = RollupQuantity.setter(
transform_context, instance_usage_df)
instance_usage_df_2 = SetAggregatedMetricName.setter(
transform_context, instance_usage_df_1)
result_list = [(row.usage_date, row.usage_hour,
row.tenant_id, row.host, row.quantity,
row.aggregated_metric_name)
for row in instance_usage_df_2.rdd.collect()]
expected_result = [
('2016-02-08', '18', 'all', 'all', 12946.0, 'mem.total_mb_agg')]
self.assertItemsEqual(result_list, expected_result)
def test_set_aggregated_metric_name(self):
record_store_df = RecordStoreUtils.create_df_from_json(
self.sql_context, DataProvider.record_store_path)
transform_spec_df = TransformSpecsUtils.create_df_from_json(
self.sql_context, DataProvider.transform_spec_path)
transform_context = TransformContextUtils.get_context(
transform_spec_df_info=transform_spec_df,
batch_time_info=self.get_dummy_batch_time())
instance_usage_df = FetchQuantity.usage(transform_context,
record_store_df)
instance_usage_df_1 = RollupQuantity.setter(transform_context,
instance_usage_df)
instance_usage_df_2 = SetAggregatedMetricName.setter(
transform_context, instance_usage_df_1)
result_list = [(row.usage_date, row.usage_hour, row.tenant_id,
row.host, row.quantity, row.aggregated_metric_name)
for row in instance_usage_df_2.rdd.collect()]
expected_result = [('2016-02-08', '18', 'all', 'all', 12946.0,
'mem.total_mb_agg')]
self.assertItemsEqual(result_list, expected_result)
def test_latest_quantity_all_hosts_component(self):
record_store_df = RecordStoreUtils.create_df_from_json(
self.sql_context,
DataProvider.record_store_path)
transform_spec_df = TransformSpecsUtils.create_df_from_json(
self.sql_context,
DataProvider.transform_spec_path)
transform_context = TransformContextUtils.get_context(
transform_spec_df_info=transform_spec_df,
batch_time_info=self.get_dummy_batch_time())
instance_usage_df = FetchQuantity.usage(
transform_context, record_store_df)
result_list = [(row.usage_date, row.usage_hour,
row.tenant_id, row.host, row.quantity)
for row in instance_usage_df.rdd.collect()]
expected_result = [
('2016-02-08', '18', 'NA', 'mini-mon', 5969.0),
('2016-02-08', '18', 'NA', 'devstack', 6977.0)]
self.assertItemsEqual(result_list, expected_result)
def process_metrics(transform_context, record_store_df):
"""start processing (aggregating) metrics"""
#
# look in record_store_df for list of metrics to be processed
#
metric_ids_df = record_store_df.select("metric_id").distinct()
metric_ids_to_process = [row.metric_id
for row in metric_ids_df.collect()]
data_driven_specs_repo = DataDrivenSpecsRepoFactory.\
get_data_driven_specs_repo()
sqlc = SQLContext.getOrCreate(record_store_df.rdd.context)
transform_specs_df = data_driven_specs_repo.get_data_driven_specs(
sql_context=sqlc,
data_driven_spec_type=DataDrivenSpecsRepo.transform_specs_type)
for metric_id in metric_ids_to_process:
transform_spec_df = transform_specs_df.select(
["aggregation_params_map", "metric_id"]
).where(transform_specs_df.metric_id == metric_id)
source_record_store_df = record_store_df.select("*").where(
record_store_df.metric_id == metric_id)
# set transform_spec_df in TransformContext
transform_context = \
TransformContextUtils.get_context(
transform_context_info=transform_context,
transform_spec_df_info=transform_spec_df)
try:
agg_inst_usage_df = (
MonMetricsKafkaProcessor.process_metric(
transform_context, source_record_store_df))
# if running in debug mode, write out the aggregated metric
# name just processed (along with the count of how many of
# these were aggregated) to the application log.
if log.isEnabledFor(logging.DEBUG):
agg_inst_usage_collection = agg_inst_usage_df.collect()
collection_len = len(agg_inst_usage_collection)
if collection_len > 0:
agg_inst_usage_dict = (
agg_inst_usage_collection[0].asDict())
log.debug("Submitted pre-hourly aggregated metric: "
"%s (%s)",
agg_inst_usage_dict[
"aggregated_metric_name"],
str(collection_len))
except FetchQuantityException:
raise
except FetchQuantityUtilException:
raise
except Exception as e:
MonMetricsKafkaProcessor.log_debug(
"Exception raised in metric processing for metric: " +
str(metric_id) + ". Error: " + str(e))
def process_metrics(transform_context, record_store_df):
"""start processing (aggregating) metrics"""
#
# look in record_store_df for list of metrics to be processed
#
metric_ids_df = record_store_df.select("metric_id").distinct()
metric_ids_to_process = [row.metric_id
for row in metric_ids_df.collect()]
data_driven_specs_repo = DataDrivenSpecsRepoFactory.\
get_data_driven_specs_repo()
sqlc = SQLContext.getOrCreate(record_store_df.rdd.context)
transform_specs_df = data_driven_specs_repo.get_data_driven_specs(
sql_context=sqlc,
data_driven_spec_type=DataDrivenSpecsRepo.transform_specs_type)
for metric_id in metric_ids_to_process:
transform_spec_df = transform_specs_df.select(
["aggregation_params_map", "metric_id"]
).where(transform_specs_df.metric_id == metric_id)
source_record_store_df = record_store_df.select("*").where(
record_store_df.metric_id == metric_id)
# set transform_spec_df in TransformContext
transform_context = \
TransformContextUtils.get_context(
transform_context_info=transform_context,
transform_spec_df_info=transform_spec_df)
try:
agg_inst_usage_df = (
MonMetricsKafkaProcessor.process_metric(
transform_context, source_record_store_df))
# if running in debug mode, write out the aggregated metric
# name just processed (along with the count of how many of
# these were aggregated) to the application log.
if log.isEnabledFor(logging.DEBUG):
agg_inst_usage_collection = agg_inst_usage_df.collect()
collection_len = len(agg_inst_usage_collection)
if collection_len > 0:
agg_inst_usage_dict = (
agg_inst_usage_collection[0].asDict())
log.debug("Submitted pre-hourly aggregated metric: "
"%s (%s)",
agg_inst_usage_dict[
"aggregated_metric_name"],
str(collection_len))
except FetchQuantityException:
raise
except FetchQuantityUtilException:
raise
except Exception as e:
MonMetricsKafkaProcessor.log_debug(
"Exception raised in metric processing for metric: " +
str(metric_id) + ". Error: " + str(e))
def test_fetch_quantity_sum(self,
usage_manager,
setter_manager,
insert_manager,
data_driven_specs_repo):
# test operation
test_operation = "sum"
# load components
usage_manager.return_value = MockComponentManager.get_usage_cmpt_mgr()
setter_manager.return_value = \
MockComponentManager.get_setter_cmpt_mgr()
insert_manager.return_value = \
MockComponentManager.get_insert_cmpt_mgr()
# init mock driver tables
data_driven_specs_repo.return_value = \
MockDataDrivenSpecsRepo(self.spark_context,
self.get_pre_transform_specs_json(),
self.get_transform_specs_json_by_operation(
test_operation))
# Create an emulated set of Kafka messages (these were gathered
# by extracting Monasca messages from the Metrics queue on mini-mon).
# Create an RDD out of the mocked Monasca metrics
with open(DataProvider.kafka_data_path) as f:
raw_lines = f.read().splitlines()
raw_tuple_list = [eval(raw_line) for raw_line in raw_lines]
rdd_monasca = self.spark_context.parallelize(raw_tuple_list)
# decorate mocked RDD with dummy kafka offsets
myOffsetRanges = [
OffsetRange("metrics", 1, 10, 20)] # mimic rdd.offsetRanges()
transform_context = TransformContextUtils.get_context(
offset_info=myOffsetRanges,
batch_time_info=self.get_dummy_batch_time())
rdd_monasca_with_offsets = rdd_monasca.map(
lambda x: RddTransformContext(x, transform_context))
try:
# Call the primary method in mon_metrics_kafka
MonMetricsKafkaProcessor.rdd_to_recordstore(
rdd_monasca_with_offsets)
self.assertTrue(False)
except FetchQuantityUtilException as e:
self.assertTrue("Operation sum is not supported" in
e.value)
def test_missing_field_to_filter(self,
usage_manager,
setter_manager,
insert_manager,
data_driven_specs_repo):
# load components
usage_manager.return_value = MockComponentManager.get_usage_cmpt_mgr()
setter_manager.return_value = \
MockComponentManager.get_setter_cmpt_mgr()
insert_manager.return_value = \
MockComponentManager.get_insert_cmpt_mgr()
# init mock driver tables
data_driven_specs_repo.return_value = \
MockDataDrivenSpecsRepo(
self.spark_context,
self.get_pre_transform_specs_json(),
self.get_invalid_filter_transform_specs_json("",
"-mgmt$",
"exclude"))
# Create an emulated set of Kafka messages (these were gathered
# by extracting Monasca messages from the Metrics queue on mini-mon).
# Create an RDD out of the mocked Monasca metrics
with open(DataProvider.fetch_quantity_data_path) as f:
raw_lines = f.read().splitlines()
raw_tuple_list = [eval(raw_line) for raw_line in raw_lines]
rdd_monasca = self.spark_context.parallelize(raw_tuple_list)
# decorate mocked RDD with dummy kafka offsets
myOffsetRanges = [
OffsetRange("metrics", 1, 10, 20)] # mimic rdd.offsetRanges()
transform_context = TransformContextUtils.get_context(
offset_info=myOffsetRanges,
batch_time_info=self.get_dummy_batch_time())
rdd_monasca_with_offsets = rdd_monasca.map(
lambda x: RddTransformContext(x, transform_context))
try:
# Call the primary method in mon_metrics_kafka
MonMetricsKafkaProcessor.rdd_to_recordstore(
rdd_monasca_with_offsets)
# In this case, it's an error if no exception is caught
self.assertTrue(False)
except FetchQuantityException as e:
self.assertTrue("Encountered invalid filter details:" in e.value)
self.assertTrue("field to filter = ," in e.value)
def test_invalid_aggregated_metric_name(self,
usage_manager,
setter_manager,
insert_manager,
data_driven_specs_repo):
# load components
usage_manager.return_value = MockComponentManager.get_usage_cmpt_mgr()
setter_manager.return_value = \
MockComponentManager.get_setter_cmpt_mgr()
insert_manager.return_value = \
MockComponentManager.get_insert_cmpt_mgr()
# init mock driver tables
data_driven_specs_repo.return_value = \
MockDataDrivenSpecsRepo(
self.spark_context,
self.get_pre_transform_specs_json(),
self.get_transform_specs_json_invalid_name())
# Create an emulated set of Kafka messages (these were gathered
# by extracting Monasca messages from the Metrics queue on mini-mon).
# Create an RDD out of the mocked Monasca metrics
with open(DataProvider.fetch_quantity_data_path) as f:
raw_lines = f.read().splitlines()
raw_tuple_list = [eval(raw_line) for raw_line in raw_lines]
rdd_monasca = self.spark_context.parallelize(raw_tuple_list)
# decorate mocked RDD with dummy kafka offsets
myOffsetRanges = [
OffsetRange("metrics", 1, 10, 20)] # mimic rdd.offsetRanges()
transform_context = TransformContextUtils.get_context(
offset_info=myOffsetRanges,
batch_time_info=self.get_dummy_batch_time())
rdd_monasca_with_offsets = rdd_monasca.map(
lambda x: RddTransformContext(x, transform_context))
# Call the primary method in mon_metrics_kafka
MonMetricsKafkaProcessor.rdd_to_recordstore(
rdd_monasca_with_offsets)
# get the metrics that have been submitted to the dummy message adapter
metrics = DummyAdapter.adapter_impl.metric_list
# metrics should be empty
self.assertFalse(metrics)
def store_offset_ranges(batch_time, rdd):
if rdd.isEmpty():
MonMetricsKafkaProcessor.log_debug(
"storeOffsetRanges: nothing to process...")
return rdd
else:
my_offset_ranges = rdd.offsetRanges()
transform_context = \
TransformContextUtils.get_context(offset_info=my_offset_ranges,
batch_time_info=batch_time
)
rdd_transform_context = \
rdd.map(lambda x: RddTransformContext(x, transform_context))
return rdd_transform_context
def store_offset_ranges(batch_time, rdd):
if rdd.isEmpty():
MonMetricsKafkaProcessor.log_debug(
"storeOffsetRanges: nothing to process...")
return rdd
else:
my_offset_ranges = rdd.offsetRanges()
transform_context = \
TransformContextUtils.get_context(offset_info=my_offset_ranges,
batch_time_info=batch_time
)
rdd_transform_context = \
rdd.map(lambda x: RddTransformContext(x, transform_context))
return rdd_transform_context
def test_fetch_quantity_sum(self, usage_manager, setter_manager,
insert_manager, data_driven_specs_repo):
# test operation
test_operation = "sum"
# load components
usage_manager.return_value = MockComponentManager.get_usage_cmpt_mgr()
setter_manager.return_value = \
MockComponentManager.get_setter_cmpt_mgr()
insert_manager.return_value = \
MockComponentManager.get_insert_cmpt_mgr()
# init mock driver tables
data_driven_specs_repo.return_value = \
MockDataDrivenSpecsRepo(self.spark_context,
self.get_pre_transform_specs_json(),
self.get_transform_specs_json_by_operation(
test_operation))
# Create an emulated set of Kafka messages (these were gathered
# by extracting Monasca messages from the Metrics queue on mini-mon).
# Create an RDD out of the mocked Monasca metrics
with open(DataProvider.kafka_data_path) as f:
raw_lines = f.read().splitlines()
raw_tuple_list = [eval(raw_line) for raw_line in raw_lines]
rdd_monasca = self.spark_context.parallelize(raw_tuple_list)
# decorate mocked RDD with dummy kafka offsets
myOffsetRanges = [OffsetRange("metrics", 1, 10,
20)] # mimic rdd.offsetRanges()
transform_context = TransformContextUtils.get_context(
offset_info=myOffsetRanges,
batch_time_info=self.get_dummy_batch_time())
rdd_monasca_with_offsets = rdd_monasca.map(
lambda x: RddTransformContext(x, transform_context))
try:
# Call the primary method in mon_metrics_kafka
MonMetricsKafkaProcessor.rdd_to_recordstore(
rdd_monasca_with_offsets)
self.assertTrue(False)
except FetchQuantityUtilException as e:
self.assertTrue("Operation sum is not supported" in e.value)
def test_missing_field_to_filter(self, usage_manager, setter_manager,
insert_manager, data_driven_specs_repo):
# load components
usage_manager.return_value = MockComponentManager.get_usage_cmpt_mgr()
setter_manager.return_value = \
MockComponentManager.get_setter_cmpt_mgr()
insert_manager.return_value = \
MockComponentManager.get_insert_cmpt_mgr()
# init mock driver tables
data_driven_specs_repo.return_value = \
MockDataDrivenSpecsRepo(
self.spark_context,
self.get_pre_transform_specs_json(),
self.get_invalid_filter_transform_specs_json("",
"-mgmt$",
"exclude"))
# Create an emulated set of Kafka messages (these were gathered
# by extracting Monasca messages from the Metrics queue on mini-mon).
# Create an RDD out of the mocked Monasca metrics
with open(DataProvider.fetch_quantity_data_path) as f:
raw_lines = f.read().splitlines()
raw_tuple_list = [eval(raw_line) for raw_line in raw_lines]
rdd_monasca = self.spark_context.parallelize(raw_tuple_list)
# decorate mocked RDD with dummy kafka offsets
myOffsetRanges = [OffsetRange("metrics", 1, 10,
20)] # mimic rdd.offsetRanges()
transform_context = TransformContextUtils.get_context(
offset_info=myOffsetRanges,
batch_time_info=self.get_dummy_batch_time())
rdd_monasca_with_offsets = rdd_monasca.map(
lambda x: RddTransformContext(x, transform_context))
try:
# Call the primary method in mon_metrics_kafka
MonMetricsKafkaProcessor.rdd_to_recordstore(
rdd_monasca_with_offsets)
# In this case, it's an error if no exception is caught
self.assertTrue(False)
except FetchQuantityException as e:
self.assertTrue("Encountered invalid filter details:" in e.value)
self.assertTrue("field to filter = ," in e.value)
def test_invalid_aggregated_metric_name(self, usage_manager,
setter_manager, insert_manager,
data_driven_specs_repo):
# load components
usage_manager.return_value = MockComponentManager.get_usage_cmpt_mgr()
setter_manager.return_value = \
MockComponentManager.get_setter_cmpt_mgr()
insert_manager.return_value = \
MockComponentManager.get_insert_cmpt_mgr()
# init mock driver tables
data_driven_specs_repo.return_value = \
MockDataDrivenSpecsRepo(
self.spark_context,
self.get_pre_transform_specs_json(),
self.get_transform_specs_json_invalid_name())
# Create an emulated set of Kafka messages (these were gathered
# by extracting Monasca messages from the Metrics queue on mini-mon).
# Create an RDD out of the mocked Monasca metrics
with open(DataProvider.fetch_quantity_data_path) as f:
raw_lines = f.read().splitlines()
raw_tuple_list = [eval(raw_line) for raw_line in raw_lines]
rdd_monasca = self.spark_context.parallelize(raw_tuple_list)
# decorate mocked RDD with dummy kafka offsets
myOffsetRanges = [OffsetRange("metrics", 1, 10,
20)] # mimic rdd.offsetRanges()
transform_context = TransformContextUtils.get_context(
offset_info=myOffsetRanges,
batch_time_info=self.get_dummy_batch_time())
rdd_monasca_with_offsets = rdd_monasca.map(
lambda x: RddTransformContext(x, transform_context))
# Call the primary method in mon_metrics_kafka
MonMetricsKafkaProcessor.rdd_to_recordstore(rdd_monasca_with_offsets)
# get the metrics that have been submitted to the dummy message adapter
metrics = DummyAdapter.adapter_impl.metric_list
# metrics should be empty
self.assertFalse(metrics)
def do_transform(instance_usage_df):
"""start processing (aggregating) metrics"""
#
# look in instance_usage_df for list of metrics to be processed
#
metric_ids_df = instance_usage_df.select(
"processing_meta.metric_id").distinct()
metric_ids_to_process = [row.metric_id
for row in metric_ids_df.collect()]
data_driven_specs_repo = (
DataDrivenSpecsRepoFactory.get_data_driven_specs_repo())
sqlc = SQLContext.getOrCreate(instance_usage_df.rdd.context)
transform_specs_df = data_driven_specs_repo.get_data_driven_specs(
sql_context=sqlc,
data_driven_spec_type=DataDrivenSpecsRepo.transform_specs_type)
for metric_id in metric_ids_to_process:
transform_spec_df = transform_specs_df.select(
["aggregation_params_map", "metric_id"]
).where(transform_specs_df.metric_id == metric_id)
source_instance_usage_df = instance_usage_df.select("*").where(
instance_usage_df.processing_meta.metric_id == metric_id)
# set transform_spec_df in TransformContext
transform_context = TransformContextUtils.get_context(
transform_spec_df_info=transform_spec_df)
agg_inst_usage_df = PreHourlyProcessor.process_instance_usage(
transform_context, source_instance_usage_df)
# if running in debug mode, write out the aggregated metric
# name just processed (along with the count of how many of these
# were aggregated) to the application log.
if log.isEnabledFor(logging.DEBUG):
agg_inst_usage_collection = agg_inst_usage_df.collect()
collection_len = len(agg_inst_usage_collection)
if collection_len > 0:
agg_inst_usage_dict = agg_inst_usage_collection[0].asDict()
log.debug("Submitted hourly aggregated metric: %s (%s)",
agg_inst_usage_dict["aggregated_metric_name"],
str(collection_len))
def test_transform_builder(self,
usage_manager,
setter_manager,
insert_manager):
usage_manager.return_value = MockComponentManager.get_usage_cmpt_mgr()
setter_manager.return_value = \
MockComponentManager.get_setter_cmpt_mgr()
insert_manager.return_value = \
MockComponentManager.get_insert_cmpt_mgr()
record_store_json_path = DataProvider.record_store_path
metric_proc_json_path = DataProvider.transform_spec_path
sql_context = SQLContext.getOrCreate(self.spark_context)
record_store_df = \
RecordStoreUtils.create_df_from_json(sql_context,
record_store_json_path)
transform_spec_df = TransformSpecsUtils.create_df_from_json(
sql_context, metric_proc_json_path)
transform_context = TransformContextUtils.get_context(
transform_spec_df_info=transform_spec_df,
batch_time_info=self.get_dummy_batch_time())
# invoke the generic transformation builder
instance_usage_df = GenericTransformBuilder.do_transform(
transform_context, record_store_df)
result_list = [(row.usage_date, row.usage_hour,
row.tenant_id, row.host, row.quantity,
row.aggregated_metric_name)
for row in instance_usage_df.rdd.collect()]
expected_result = [('2016-02-08', '18', 'all',
'all', 12946.0,
'mem.total_mb_agg')]
self.assertItemsEqual(result_list, expected_result)
def test_transform_builder(self, usage_manager, setter_manager,
insert_manager):
usage_manager.return_value = MockComponentManager.get_usage_cmpt_mgr()
setter_manager.return_value = \
MockComponentManager.get_setter_cmpt_mgr()
insert_manager.return_value = \
MockComponentManager.get_insert_cmpt_mgr()
record_store_json_path = DataProvider.record_store_path
metric_proc_json_path = DataProvider.transform_spec_path
sql_context = SQLContext.getOrCreate(self.spark_context)
record_store_df = \
RecordStoreUtils.create_df_from_json(sql_context,
record_store_json_path)
transform_spec_df = TransformSpecsUtils.create_df_from_json(
sql_context, metric_proc_json_path)
transform_context = TransformContextUtils.get_context(
transform_spec_df_info=transform_spec_df,
batch_time_info=self.get_dummy_batch_time())
# invoke the generic transformation builder
instance_usage_df = GenericTransformBuilder.do_transform(
transform_context, record_store_df)
result_list = [(row.usage_date, row.usage_hour, row.tenant_id,
row.host, row.quantity, row.aggregated_metric_name)
for row in instance_usage_df.rdd.collect()]
expected_result = [('2016-02-08', '18', 'all', 'all', 12946.0,
'mem.total_mb_agg')]
self.assertItemsEqual(result_list, expected_result)
def test_vcpus_by_all(self, usage_manager, setter_manager, insert_manager,
data_driven_specs_repo):
# load components
usage_manager.return_value = MockComponentManager.get_usage_cmpt_mgr()
setter_manager.return_value = \
MockComponentManager.get_setter_cmpt_mgr()
insert_manager.return_value = \
MockComponentManager.get_insert_pre_hourly_cmpt_mgr()
# init mock driver tables
data_driven_specs_repo.return_value = \
MockDataDrivenSpecsRepo(
self.spark_context,
self.get_pre_transform_specs_json_by_all(),
self.get_transform_specs_json_by_all())
# Create an RDD out of the mocked Monasca metrics
with open(DataProvider.kafka_data_path) as f:
raw_lines = f.read().splitlines()
raw_tuple_list = [eval(raw_line) for raw_line in raw_lines]
rdd_monasca = self.spark_context.parallelize(raw_tuple_list)
# decorate mocked RDD with dummy kafka offsets
myOffsetRanges = [OffsetRange("metrics", 1, 10,
20)] # mimic rdd.offsetRanges()
transform_context = TransformContextUtils.get_context(
offset_info=myOffsetRanges,
batch_time_info=self.get_dummy_batch_time())
rdd_monasca_with_offsets = rdd_monasca.map(
lambda x: RddTransformContext(x, transform_context))
# Call the primary method in mon_metrics_kafka
MonMetricsKafkaProcessor.rdd_to_recordstore(rdd_monasca_with_offsets)
# get the metrics that have been submitted to the dummy message adapter
metrics = DummyAdapter.adapter_impl.metric_list
vm_cpu_list = map(dump_as_ascii_string, metrics)
DummyAdapter.adapter_impl.metric_list = []
vm_cpu_rdd = self.spark_context.parallelize(vm_cpu_list)
sql_context = SQLContext(self.spark_context)
vm_cpu_df = sql_context.read.json(vm_cpu_rdd)
PreHourlyProcessor.do_transform(vm_cpu_df)
metrics = DummyAdapter.adapter_impl.metric_list
vcpus_agg_metric = [
value for value in metrics
if value.get('metric').get('name') == 'vcpus_agg' and value.get(
'metric').get('dimensions').get('project_id') == 'all'
][0]
self.assertTrue(vcpus_agg_metric is not None)
self.assertEqual(7.0, vcpus_agg_metric.get('metric').get('value'))
self.assertEqual('useast', vcpus_agg_metric.get('meta').get('region'))
self.assertEqual(cfg.CONF.messaging.publish_kafka_project_id,
vcpus_agg_metric.get('meta').get('tenantId'))
self.assertEqual(
'all',
vcpus_agg_metric.get('metric').get('dimensions').get('host'))
self.assertEqual(
'hourly',
vcpus_agg_metric.get('metric').get('dimensions').get(
'aggregation_period'))
self.assertEqual(
14.0,
vcpus_agg_metric.get('metric').get('value_meta').get(
'record_count'))
self.assertEqual(
'2016-01-20 16:40:00',
vcpus_agg_metric.get('metric').get('value_meta').get(
'firstrecord_timestamp_string'))
self.assertEqual(
'2016-01-20 16:40:46',
vcpus_agg_metric.get('metric').get('value_meta').get(
'lastrecord_timestamp_string'))
def test_rdd_to_recordstore(self,
usage_manager,
setter_manager,
insert_manager):
usage_manager.return_value = MockComponentManager.get_usage_cmpt_mgr()
setter_manager.return_value = \
MockComponentManager.get_setter_cmpt_mgr()
insert_manager.return_value = \
MockComponentManager.get_insert_cmpt_mgr()
# Create an emulated set of Kafka messages (these were gathered
# by extracting Monasca messages from the Metrics queue on mini-mon).
# Create an RDD out of the mocked Monasca metrics
with open(DataProvider.kafka_data_path) as f:
raw_lines = f.read().splitlines()
raw_tuple_list = [eval(raw_line) for raw_line in raw_lines]
rdd_monasca = self.spark_context.parallelize(raw_tuple_list)
# decorate mocked RDD with dummy kafka offsets
myOffsetRanges = [
OffsetRange("metrics", 1, 10, 20)] # mimic rdd.offsetRanges()
transform_context = TransformContextUtils.get_context(
offset_info=myOffsetRanges,
batch_time_info=self.get_dummy_batch_time())
rdd_monasca_with_offsets = rdd_monasca.map(
lambda x: RddTransformContext(x, transform_context))
# Call the primary method in mon_metrics_kafka
MonMetricsKafkaProcessor.rdd_to_recordstore(
rdd_monasca_with_offsets)
# get the metrics that have been submitted to the dummy message adapter
metrics = DummyAdapter.adapter_impl.metric_list
# Verify cpu.total_logical_cores_agg for all hosts
total_cpu_logical_agg_metric = [
value for value in metrics
if value.get('metric').get('name') ==
'cpu.total_logical_cores_agg' and
value.get('metric').get('dimensions').get('host') ==
'all'][0]
self.assertEqual(15.0,
total_cpu_logical_agg_metric.get(
'metric').get('value'))
self.assertEqual('useast',
total_cpu_logical_agg_metric.get(
'meta').get('region'))
self.assertEqual(cfg.CONF.messaging.publish_kafka_tenant_id,
total_cpu_logical_agg_metric.get(
'meta').get('tenantId'))
self.assertEqual('all',
total_cpu_logical_agg_metric.get(
'metric').get('dimensions')
.get('project_id'))
self.assertEqual('hourly',
total_cpu_logical_agg_metric.get(
'metric').get('dimensions')
.get('aggregation_period'))
self.assertEqual(13.0,
total_cpu_logical_agg_metric.get(
'metric').get('value_meta')
.get('record_count'))
self.assertEqual('2016-03-07 16:09:23',
total_cpu_logical_agg_metric.get(
'metric').get('value_meta')
.get('firstrecord_timestamp'))
self.assertEqual('2016-03-07 16:10:38',
total_cpu_logical_agg_metric.get(
'metric').get('value_meta')
.get('lastrecord_timestamp'))
# Verify cpu.total_logical_cores_agg for mini-mon host
total_cpu_logical_agg_metric = [
value for value in metrics
if value.get('metric').get('name') ==
'cpu.total_logical_cores_agg' and
value.get('metric').get('dimensions').get('host') ==
'mini-mon'][0]
self.assertEqual(9.0,
total_cpu_logical_agg_metric.get(
'metric').get('value'))
self.assertEqual('useast',
total_cpu_logical_agg_metric.get(
'meta').get('region'))
self.assertEqual(cfg.CONF.messaging.publish_kafka_tenant_id,
total_cpu_logical_agg_metric.get(
'meta').get('tenantId'))
self.assertEqual('all',
total_cpu_logical_agg_metric.get(
'metric').get('dimensions')
.get('project_id'))
self.assertEqual('hourly',
total_cpu_logical_agg_metric.get(
'metric').get('dimensions')
.get('aggregation_period'))
self.assertEqual(6.0,
total_cpu_logical_agg_metric.get(
'metric').get('value_meta')
.get('record_count'))
self.assertEqual('2016-03-07 16:09:23',
total_cpu_logical_agg_metric.get(
'metric').get('value_meta')
.get('firstrecord_timestamp'))
self.assertEqual('2016-03-07 16:10:38',
total_cpu_logical_agg_metric.get(
'metric').get('value_meta')
.get('lastrecord_timestamp'))
# Verify cpu.total_logical_cores_agg for devstack host
total_cpu_logical_agg_metric = [
value for value in metrics
if value.get('metric').get('name') ==
'cpu.total_logical_cores_agg' and
value.get('metric').get('dimensions').get('host') ==
'devstack'][0]
self.assertEqual(6.0,
total_cpu_logical_agg_metric.get(
'metric').get('value'))
self.assertEqual('useast',
total_cpu_logical_agg_metric.get(
'meta').get('region'))
self.assertEqual(cfg.CONF.messaging.publish_kafka_tenant_id,
total_cpu_logical_agg_metric.get(
'meta').get('tenantId'))
self.assertEqual('all',
total_cpu_logical_agg_metric.get(
'metric').get('dimensions')
.get('project_id'))
self.assertEqual('hourly',
total_cpu_logical_agg_metric.get(
'metric').get('dimensions')
.get('aggregation_period'))
self.assertEqual(7.0,
total_cpu_logical_agg_metric.get(
'metric').get('value_meta')
.get('record_count'))
self.assertEqual('2016-03-07 16:09:23',
total_cpu_logical_agg_metric.get(
'metric').get('value_meta')
.get('firstrecord_timestamp'))
self.assertEqual('2016-03-07 16:10:38',
total_cpu_logical_agg_metric.get(
'metric').get('value_meta')
.get('lastrecord_timestamp'))
# Verify cpu.utilized_logical_cores_agg for all hosts
utilized_cpu_logical_agg_metric = [
value for value in metrics
if value.get('metric').get('name') ==
'cpu.utilized_logical_cores_agg' and
value.get('metric').get('dimensions').get('host') ==
'all'][0]
self.assertEqual(8.0,
utilized_cpu_logical_agg_metric.get(
'metric').get('value'))
self.assertEqual('useast',
utilized_cpu_logical_agg_metric.get(
'meta').get('region'))
self.assertEqual(cfg.CONF.messaging.publish_kafka_tenant_id,
utilized_cpu_logical_agg_metric.get(
'meta').get('tenantId'))
self.assertEqual('all',
utilized_cpu_logical_agg_metric.get(
'metric').get('dimensions')
.get('project_id'))
self.assertEqual('hourly',
utilized_cpu_logical_agg_metric.get(
'metric').get('dimensions')
.get('aggregation_period'))
self.assertEqual(13.0,
utilized_cpu_logical_agg_metric.get(
'metric').get('value_meta')
.get('record_count'))
self.assertEqual('2016-03-07 16:09:23',
utilized_cpu_logical_agg_metric.get(
'metric').get('value_meta')
.get('firstrecord_timestamp'))
self.assertEqual('2016-03-07 16:10:38',
utilized_cpu_logical_agg_metric.get(
'metric').get('value_meta')
.get('lastrecord_timestamp'))
# Verify cpu.utilized_logical_cores_agg for the mini-mon host
utilized_cpu_logical_agg_metric = [
value for value in metrics
if value.get('metric').get('name') ==
'cpu.utilized_logical_cores_agg' and
value.get('metric').get('dimensions').get('host') ==
'mini-mon'][0]
self.assertEqual(5.0,
utilized_cpu_logical_agg_metric.get(
'metric').get('value'))
self.assertEqual('useast',
utilized_cpu_logical_agg_metric.get(
'meta').get('region'))
self.assertEqual(cfg.CONF.messaging.publish_kafka_tenant_id,
utilized_cpu_logical_agg_metric.get(
'meta').get('tenantId'))
self.assertEqual('all',
utilized_cpu_logical_agg_metric.get(
'metric').get('dimensions')
.get('project_id'))
self.assertEqual('hourly',
utilized_cpu_logical_agg_metric.get(
'metric').get('dimensions')
.get('aggregation_period'))
self.assertEqual(6.0,
utilized_cpu_logical_agg_metric.get(
'metric').get('value_meta')
.get('record_count'))
self.assertEqual('2016-03-07 16:09:23',
utilized_cpu_logical_agg_metric.get(
'metric').get('value_meta')
.get('firstrecord_timestamp'))
self.assertEqual('2016-03-07 16:10:38',
utilized_cpu_logical_agg_metric.get(
'metric').get('value_meta')
.get('lastrecord_timestamp'))
# Verify cpu.utilized_logical_cores_agg for the devstack host
utilized_cpu_logical_agg_metric = [
value for value in metrics
if value.get('metric').get('name') ==
'cpu.utilized_logical_cores_agg' and
value.get('metric').get('dimensions').get('host') ==
'devstack'][0]
self.assertEqual(3.0,
utilized_cpu_logical_agg_metric.get(
'metric').get('value'))
self.assertEqual('useast',
utilized_cpu_logical_agg_metric.get(
'meta').get('region'))
self.assertEqual(cfg.CONF.messaging.publish_kafka_tenant_id,
utilized_cpu_logical_agg_metric.get(
'meta').get('tenantId'))
self.assertEqual('all',
utilized_cpu_logical_agg_metric.get(
'metric').get('dimensions')
.get('project_id'))
self.assertEqual('hourly',
utilized_cpu_logical_agg_metric.get(
'metric').get('dimensions')
.get('aggregation_period'))
self.assertEqual(7.0,
utilized_cpu_logical_agg_metric.get(
'metric').get('value_meta')
.get('record_count'))
self.assertEqual('2016-03-07 16:09:23',
utilized_cpu_logical_agg_metric.get(
'metric').get('value_meta')
.get('firstrecord_timestamp'))
self.assertEqual('2016-03-07 16:10:38',
utilized_cpu_logical_agg_metric.get(
'metric').get('value_meta')
.get('lastrecord_timestamp'))
def test_vcpus_by_project(self,
usage_manager,
setter_manager,
insert_manager,
data_driven_specs_repo):
# load components
usage_manager.return_value = MockComponentManager.get_usage_cmpt_mgr()
setter_manager.return_value = \
MockComponentManager.get_setter_cmpt_mgr()
insert_manager.return_value = \
MockComponentManager.get_insert_pre_hourly_cmpt_mgr()
# init mock driver tables
data_driven_specs_repo.return_value = \
MockDataDrivenSpecsRepo(self.spark_context,
self.
get_pre_transform_specs_json_by_project(),
self.get_transform_specs_json_by_project())
# Create an RDD out of the mocked Monasca metrics
with open(DataProvider.kafka_data_path) as f:
raw_lines = f.read().splitlines()
raw_tuple_list = [eval(raw_line) for raw_line in raw_lines]
rdd_monasca = self.spark_context.parallelize(raw_tuple_list)
# decorate mocked RDD with dummy kafka offsets
myOffsetRanges = [
OffsetRange("metrics", 1, 10, 20)] # mimic rdd.offsetRanges()
transform_context = TransformContextUtils.get_context(
offset_info=myOffsetRanges,
batch_time_info=self.get_dummy_batch_time())
rdd_monasca_with_offsets = rdd_monasca.map(
lambda x: RddTransformContext(x, transform_context))
# Call the primary method in mon_metrics_kafka
MonMetricsKafkaProcessor.rdd_to_recordstore(
rdd_monasca_with_offsets)
# get the metrics that have been submitted to the dummy message adapter
vm_cpu_list = DummyAdapter.adapter_impl.metric_list
vm_cpu_list = map(dump_as_ascii_string, vm_cpu_list)
DummyAdapter.adapter_impl.metric_list = []
vm_cpu_rdd = self.spark_context.parallelize(vm_cpu_list)
sql_context = SQLContext(self.spark_context)
vm_cpu_df = sql_context.read.json(vm_cpu_rdd)
PreHourlyProcessor.do_transform(vm_cpu_df)
metrics = DummyAdapter.adapter_impl.metric_list
vcpus_agg_metric = [
value for value in metrics
if value.get('metric').get('name') ==
'vcpus_agg' and
value.get('metric').get('dimensions').get('project_id') ==
'9647fd5030b04a799b0411cc38c4102d'][0]
self.assertTrue(vcpus_agg_metric is not None)
self.assertEqual(6.0,
vcpus_agg_metric
.get('metric').get('value'))
self.assertEqual('useast',
vcpus_agg_metric
.get('meta').get('region'))
self.assertEqual(cfg.CONF.messaging.publish_kafka_project_id,
vcpus_agg_metric
.get('meta').get('tenantId'))
self.assertEqual('all',
vcpus_agg_metric
.get('metric').get('dimensions').get('host'))
self.assertEqual('hourly',
vcpus_agg_metric
.get('metric').get('dimensions')
.get('aggregation_period'))
self.assertEqual(8.0,
vcpus_agg_metric
.get('metric').get('value_meta').get('record_count'))
self.assertEqual('2016-01-20 16:40:05',
vcpus_agg_metric
.get('metric').get('value_meta')
.get('firstrecord_timestamp_string'))
self.assertEqual('2016-01-20 16:40:46',
vcpus_agg_metric
.get('metric').get('value_meta')
.get('lastrecord_timestamp_string'))
vcpus_agg_metric = [
value for value in metrics
if value.get('metric').get('name') ==
'vcpus_agg' and
value.get('metric').get('dimensions').get('project_id') ==
'8647fd5030b04a799b0411cc38c4102d'][0]
self.assertTrue(vcpus_agg_metric is not None)
self.assertEqual(1.0,
vcpus_agg_metric
.get('metric').get('value'))
self.assertEqual('useast',
vcpus_agg_metric
.get('meta').get('region'))
self.assertEqual(cfg.CONF.messaging.publish_kafka_project_id,
vcpus_agg_metric
.get('meta').get('tenantId'))
self.assertEqual('all',
vcpus_agg_metric
.get('metric').get('dimensions').get('host'))
self.assertEqual('hourly',
vcpus_agg_metric
.get('metric').get('dimensions')
.get('aggregation_period'))
self.assertEqual(6.0,
vcpus_agg_metric
.get('metric').get('value_meta').get('record_count'))
self.assertEqual('2016-01-20 16:40:00',
vcpus_agg_metric
.get('metric').get('value_meta')
.get('firstrecord_timestamp_string'))
self.assertEqual('2016-01-20 16:40:42',
vcpus_agg_metric
.get('metric').get('value_meta')
.get('lastrecord_timestamp_string'))
def test_fetch_quantity_avg(self, usage_manager, setter_manager,
insert_manager, data_driven_specs_repo):
# test operation
test_operation = "avg"
# load components
usage_manager.return_value = MockComponentManager.get_usage_cmpt_mgr()
setter_manager.return_value = \
MockComponentManager.get_setter_cmpt_mgr()
insert_manager.return_value = \
MockComponentManager.get_insert_cmpt_mgr()
# init mock driver tables
data_driven_specs_repo.return_value = \
MockDataDrivenSpecsRepo(self.spark_context,
self.get_pre_transform_specs_json(),
self.get_transform_specs_json_by_operation(
test_operation))
# Create an emulated set of Kafka messages (these were gathered
# by extracting Monasca messages from the Metrics queue on mini-mon).
# Create an RDD out of the mocked Monasca metrics
with open(DataProvider.kafka_data_path) as f:
raw_lines = f.read().splitlines()
raw_tuple_list = [eval(raw_line) for raw_line in raw_lines]
rdd_monasca = self.spark_context.parallelize(raw_tuple_list)
# decorate mocked RDD with dummy kafka offsets
myOffsetRanges = [OffsetRange("metrics", 1, 10,
20)] # mimic rdd.offsetRanges()
transform_context = TransformContextUtils.get_context(
offset_info=myOffsetRanges,
batch_time_info=self.get_dummy_batch_time())
rdd_monasca_with_offsets = rdd_monasca.map(
lambda x: RddTransformContext(x, transform_context))
# Call the primary method in mon_metrics_kafka
MonMetricsKafkaProcessor.rdd_to_recordstore(rdd_monasca_with_offsets)
# get the metrics that have been submitted to the dummy message adapter
metrics = DummyAdapter.adapter_impl.metric_list
utilized_cpu_logical_agg_metric = [
value for value in metrics if value.get('metric').get('name') ==
'cpu.utilized_logical_cores_agg'
][0]
self.assertEqual(
7.134214285714285,
utilized_cpu_logical_agg_metric.get('metric').get('value'))
self.assertEqual(
'useast',
utilized_cpu_logical_agg_metric.get('meta').get('region'))
self.assertEqual(
cfg.CONF.messaging.publish_kafka_project_id,
utilized_cpu_logical_agg_metric.get('meta').get('tenantId'))
self.assertEqual(
'all',
utilized_cpu_logical_agg_metric.get('metric').get(
'dimensions').get('host'))
self.assertEqual(
'all',
utilized_cpu_logical_agg_metric.get('metric').get(
'dimensions').get('project_id'))
self.assertEqual(
'hourly',
utilized_cpu_logical_agg_metric.get('metric').get(
'dimensions').get('aggregation_period'))
self.assertEqual(
13.0,
utilized_cpu_logical_agg_metric.get('metric').get(
'value_meta').get('record_count'))
self.assertEqual(
'2016-03-07 16:09:23',
utilized_cpu_logical_agg_metric.get('metric').get(
'value_meta').get('firstrecord_timestamp_string'))
self.assertEqual(
'2016-03-07 16:10:38',
utilized_cpu_logical_agg_metric.get('metric').get(
'value_meta').get('lastrecord_timestamp_string'))
def test_pod_net_in_usage_app(self,
usage_manager,
setter_manager,
insert_manager,
data_driven_specs_repo):
# load components
usage_manager.return_value = MockComponentManager.get_usage_cmpt_mgr()
setter_manager.return_value = \
MockComponentManager.get_setter_cmpt_mgr()
insert_manager.return_value = \
MockComponentManager.get_insert_cmpt_mgr()
# init mock driver tables
data_driven_specs_repo.return_value = \
MockDataDrivenSpecsRepo(self.spark_context,
self.get_pre_transform_specs_json_app(),
self.get_transform_specs_json_app())
# Create an emulated set of Kafka messages (these were gathered
# by extracting Monasca messages from the Metrics queue on mini-mon).
# Create an RDD out of the mocked Monasca metrics
with open(DataProvider.fetch_quantity_data_path) as f:
raw_lines = f.read().splitlines()
raw_tuple_list = [eval(raw_line) for raw_line in raw_lines]
rdd_monasca = self.spark_context.parallelize(raw_tuple_list)
# decorate mocked RDD with dummy kafka offsets
myOffsetRanges = [
OffsetRange("metrics", 1, 10, 20)] # mimic rdd.offsetRanges()
transform_context = TransformContextUtils.get_context(
offset_info=myOffsetRanges,
batch_time_info=self.get_dummy_batch_time())
rdd_monasca_with_offsets = rdd_monasca.map(
lambda x: RddTransformContext(x, transform_context))
# Call the primary method in mon_metrics_kafka
MonMetricsKafkaProcessor.rdd_to_recordstore(
rdd_monasca_with_offsets)
# get the metrics that have been submitted to the dummy message adapter
metrics = DummyAdapter.adapter_impl.metric_list
pod_net_usage_agg_metric = [
value for value in metrics
if value.get('metric').get('name') ==
'pod.net.in_bytes_sec_agg' and
value.get('metric').get('dimensions').get('app') ==
'junk' and
value.get('metric').get('dimensions').get('namespace') ==
'all' and
value.get('metric').get('dimensions').get('pod_name') ==
'all'][0]
self.assertTrue(pod_net_usage_agg_metric is not None)
self.assertEqual('pod.net.in_bytes_sec_agg',
pod_net_usage_agg_metric
.get('metric').get('name'))
self.assertEqual('junk',
pod_net_usage_agg_metric
.get("metric").get('dimensions').get('app'))
self.assertEqual('all',
pod_net_usage_agg_metric
.get("metric").get('dimensions').get('namespace'))
self.assertEqual('all',
pod_net_usage_agg_metric
.get("metric").get('dimensions').get('pod_name'))
self.assertEqual(122.94,
pod_net_usage_agg_metric
.get('metric').get('value'))
self.assertEqual('useast',
pod_net_usage_agg_metric
.get('meta').get('region'))
self.assertEqual(cfg.CONF.messaging.publish_kafka_project_id,
pod_net_usage_agg_metric
.get('meta').get('tenantId'))
self.assertEqual('hourly',
pod_net_usage_agg_metric
.get('metric').get('dimensions')
.get('aggregation_period'))
self.assertEqual(3.0,
pod_net_usage_agg_metric
.get('metric').get('value_meta').get('record_count'))
self.assertEqual('2017-01-24 20:14:47',
pod_net_usage_agg_metric
.get('metric').get('value_meta')
.get('firstrecord_timestamp_string'))
self.assertEqual('2017-01-24 20:15:47',
pod_net_usage_agg_metric
.get('metric').get('value_meta')
.get('lastrecord_timestamp_string'))
def test_rdd_to_recordstore(self, usage_manager, setter_manager,
insert_manager):
usage_manager.return_value = MockComponentManager.get_usage_cmpt_mgr()
setter_manager.return_value = \
MockComponentManager.get_setter_cmpt_mgr()
insert_manager.return_value = \
MockComponentManager.get_insert_pre_hourly_cmpt_mgr()
# Create an RDD out of the mocked Monasca metrics
with open(DataProvider.kafka_data_path) as f:
raw_lines = f.read().splitlines()
raw_tuple_list = [eval(raw_line) for raw_line in raw_lines]
rdd_monasca = self.spark_context.parallelize(raw_tuple_list)
# decorate mocked RDD with dummy kafka offsets
myOffsetRanges = [OffsetRange("metrics", 1, 10,
20)] # mimic rdd.offsetRanges()
transform_context = TransformContextUtils.get_context(
offset_info=myOffsetRanges,
batch_time_info=self.get_dummy_batch_time())
rdd_monasca_with_offsets = rdd_monasca.map(
lambda x: RddTransformContext(x, transform_context))
# Call the primary method in mon_metrics_kafka
MonMetricsKafkaProcessor.rdd_to_recordstore(rdd_monasca_with_offsets)
host_usage_list = DummyAdapter.adapter_impl.metric_list
host_usage_list = map(dump_as_ascii_string, host_usage_list)
DummyAdapter.adapter_impl.metric_list = []
host_usage_rdd = self.spark_context.parallelize(host_usage_list)
sql_context = SQLContext(self.spark_context)
host_usage_df = sql_context.read.json(host_usage_rdd)
PreHourlyProcessor.do_transform(host_usage_df)
# get the metrics that have been submitted to the dummy message adapter
metrics = DummyAdapter.adapter_impl.metric_list
# Verify cpu.total_logical_cores_agg for all hosts
total_cpu_logical_agg_metric = [
value for value in metrics
if value.get('metric').get('name') == 'cpu.total_logical_cores_agg'
and value.get('metric').get('dimensions').get('host') == 'all'
][0]
self.assertEqual(
15.0,
total_cpu_logical_agg_metric.get('metric').get('value'))
self.assertEqual(
'useast',
total_cpu_logical_agg_metric.get('meta').get('region'))
self.assertEqual(
cfg.CONF.messaging.publish_kafka_project_id,
total_cpu_logical_agg_metric.get('meta').get('tenantId'))
self.assertEqual(
'all',
total_cpu_logical_agg_metric.get('metric').get('dimensions').get(
'project_id'))
self.assertEqual(
'hourly',
total_cpu_logical_agg_metric.get('metric').get('dimensions').get(
'aggregation_period'))
self.assertEqual(
13.0,
total_cpu_logical_agg_metric.get('metric').get('value_meta').get(
'record_count'))
self.assertEqual(
'2016-03-07 16:09:23',
total_cpu_logical_agg_metric.get('metric').get('value_meta').get(
'firstrecord_timestamp_string'))
self.assertEqual(
'2016-03-07 16:10:38',
total_cpu_logical_agg_metric.get('metric').get('value_meta').get(
'lastrecord_timestamp_string'))
# Verify cpu.total_logical_cores_agg for mini-mon host
total_cpu_logical_agg_metric = [
value for value in metrics
if value.get('metric').get('name') == 'cpu.total_logical_cores_agg'
and value.get('metric').get('dimensions').get(
'host') == 'test-cp1-comp0333-mgmt'
][0]
self.assertEqual(
9.0,
total_cpu_logical_agg_metric.get('metric').get('value'))
self.assertEqual(
'useast',
total_cpu_logical_agg_metric.get('meta').get('region'))
self.assertEqual(
cfg.CONF.messaging.publish_kafka_project_id,
total_cpu_logical_agg_metric.get('meta').get('tenantId'))
self.assertEqual(
'all',
total_cpu_logical_agg_metric.get('metric').get('dimensions').get(
'project_id'))
self.assertEqual(
'hourly',
total_cpu_logical_agg_metric.get('metric').get('dimensions').get(
'aggregation_period'))
self.assertEqual(
6.0,
total_cpu_logical_agg_metric.get('metric').get('value_meta').get(
'record_count'))
self.assertEqual(
'2016-03-07 16:09:23',
total_cpu_logical_agg_metric.get('metric').get('value_meta').get(
'firstrecord_timestamp_string'))
self.assertEqual(
'2016-03-07 16:10:38',
total_cpu_logical_agg_metric.get('metric').get('value_meta').get(
'lastrecord_timestamp_string'))
# Verify cpu.total_logical_cores_agg for devstack host
total_cpu_logical_agg_metric = [
value for value in metrics
if value.get('metric').get('name') == 'cpu.total_logical_cores_agg'
and value.get('metric').get('dimensions').get(
'host') == 'test-cp1-comp0027-mgmt'
][0]
self.assertEqual(
6.0,
total_cpu_logical_agg_metric.get('metric').get('value'))
self.assertEqual(
'useast',
total_cpu_logical_agg_metric.get('meta').get('region'))
self.assertEqual(
cfg.CONF.messaging.publish_kafka_project_id,
total_cpu_logical_agg_metric.get('meta').get('tenantId'))
self.assertEqual(
'all',
total_cpu_logical_agg_metric.get('metric').get('dimensions').get(
'project_id'))
self.assertEqual(
'hourly',
total_cpu_logical_agg_metric.get('metric').get('dimensions').get(
'aggregation_period'))
self.assertEqual(
7.0,
total_cpu_logical_agg_metric.get('metric').get('value_meta').get(
'record_count'))
self.assertEqual(
'2016-03-07 16:09:23',
total_cpu_logical_agg_metric.get('metric').get('value_meta').get(
'firstrecord_timestamp_string'))
self.assertEqual(
'2016-03-07 16:10:38',
total_cpu_logical_agg_metric.get('metric').get('value_meta').get(
'lastrecord_timestamp_string'))
# Verify cpu.utilized_logical_cores_agg for all hosts
utilized_cpu_logical_agg_metric = [
value for value in metrics if
value.get('metric').get('name') == 'cpu.utilized_logical_cores_agg'
and value.get('metric').get('dimensions').get('host') == 'all'
][0]
self.assertEqual(
7.134214285714285,
utilized_cpu_logical_agg_metric.get('metric').get('value'))
self.assertEqual(
'useast',
utilized_cpu_logical_agg_metric.get('meta').get('region'))
self.assertEqual(
cfg.CONF.messaging.publish_kafka_project_id,
utilized_cpu_logical_agg_metric.get('meta').get('tenantId'))
self.assertEqual(
'all',
utilized_cpu_logical_agg_metric.get('metric').get(
'dimensions').get('project_id'))
self.assertEqual(
'hourly',
utilized_cpu_logical_agg_metric.get('metric').get(
'dimensions').get('aggregation_period'))
self.assertEqual(
13.0,
utilized_cpu_logical_agg_metric.get('metric').get(
'value_meta').get('record_count'))
self.assertEqual(
'2016-03-07 16:09:23',
utilized_cpu_logical_agg_metric.get('metric').get(
'value_meta').get('firstrecord_timestamp_string'))
self.assertEqual(
'2016-03-07 16:10:38',
utilized_cpu_logical_agg_metric.get('metric').get(
'value_meta').get('lastrecord_timestamp_string'))
# Verify cpu.utilized_logical_cores_agg for the mini-mon host
utilized_cpu_logical_agg_metric = [
value for value in metrics if value.get('metric').get('name') ==
'cpu.utilized_logical_cores_agg' and value.get('metric').get(
'dimensions').get('host') == 'test-cp1-comp0333-mgmt'
][0]
self.assertEqual(
4.9665,
utilized_cpu_logical_agg_metric.get('metric').get('value'))
self.assertEqual(
'useast',
utilized_cpu_logical_agg_metric.get('meta').get('region'))
self.assertEqual(
cfg.CONF.messaging.publish_kafka_project_id,
utilized_cpu_logical_agg_metric.get('meta').get('tenantId'))
self.assertEqual(
'all',
utilized_cpu_logical_agg_metric.get('metric').get(
'dimensions').get('project_id'))
self.assertEqual(
'hourly',
utilized_cpu_logical_agg_metric.get('metric').get(
'dimensions').get('aggregation_period'))
self.assertEqual(
6.0,
utilized_cpu_logical_agg_metric.get('metric').get(
'value_meta').get('record_count'))
self.assertEqual(
'2016-03-07 16:09:23',
utilized_cpu_logical_agg_metric.get('metric').get(
'value_meta').get('firstrecord_timestamp_string'))
self.assertEqual(
'2016-03-07 16:10:38',
utilized_cpu_logical_agg_metric.get('metric').get(
'value_meta').get('lastrecord_timestamp_string'))
# Verify cpu.utilized_logical_cores_agg for the devstack host
utilized_cpu_logical_agg_metric = [
value for value in metrics if value.get('metric').get('name') ==
'cpu.utilized_logical_cores_agg' and value.get('metric').get(
'dimensions').get('host') == 'test-cp1-comp0027-mgmt'
][0]
self.assertEqual(
2.1677142857142853,
utilized_cpu_logical_agg_metric.get('metric').get('value'))
self.assertEqual(
'useast',
utilized_cpu_logical_agg_metric.get('meta').get('region'))
self.assertEqual(
cfg.CONF.messaging.publish_kafka_project_id,
utilized_cpu_logical_agg_metric.get('meta').get('tenantId'))
self.assertEqual(
'all',
utilized_cpu_logical_agg_metric.get('metric').get(
'dimensions').get('project_id'))
self.assertEqual(
'hourly',
utilized_cpu_logical_agg_metric.get('metric').get(
'dimensions').get('aggregation_period'))
self.assertEqual(
7.0,
utilized_cpu_logical_agg_metric.get('metric').get(
'value_meta').get('record_count'))
self.assertEqual(
'2016-03-07 16:09:23',
utilized_cpu_logical_agg_metric.get('metric').get(
'value_meta').get('firstrecord_timestamp_string'))
self.assertEqual(
'2016-03-07 16:10:38',
utilized_cpu_logical_agg_metric.get('metric').get(
'value_meta').get('lastrecord_timestamp_string'))
def test_fetch_quantity_sum(self,
usage_manager,
setter_manager,
insert_manager,
data_driven_specs_repo):
# test operation
test_operation = "sum"
# load components
usage_manager.return_value = MockComponentManager.get_usage_cmpt_mgr()
setter_manager.return_value = \
MockComponentManager.get_setter_cmpt_mgr()
insert_manager.return_value = \
MockComponentManager.get_insert_pre_hourly_cmpt_mgr()
# init mock driver tables
data_driven_specs_repo.return_value = \
MockDataDrivenSpecsRepo(self.spark_context,
self.get_pre_transform_specs_json(),
self.get_transform_specs_json_by_operation(
test_operation))
# Create an emulated set of Kafka messages (these were gathered
# by extracting Monasca messages from the Metrics queue on mini-mon).
# Create an RDD out of the mocked Monasca metrics
with open(DataProvider.fetch_quantity_data_path) as f:
raw_lines = f.read().splitlines()
raw_tuple_list = [eval(raw_line) for raw_line in raw_lines]
rdd_monasca = self.spark_context.parallelize(raw_tuple_list)
# decorate mocked RDD with dummy kafka offsets
myOffsetRanges = [
OffsetRange("metrics", 1, 10, 20)] # mimic rdd.offsetRanges()
transform_context = TransformContextUtils.get_context(
offset_info=myOffsetRanges,
batch_time_info=self.get_dummy_batch_time())
rdd_monasca_with_offsets = rdd_monasca.map(
lambda x: RddTransformContext(x, transform_context))
# Call the primary method in mon_metrics_kafka
MonMetricsKafkaProcessor.rdd_to_recordstore(
rdd_monasca_with_offsets)
# get the metrics that have been submitted to the dummy message adapter
instance_usage_list = DummyAdapter.adapter_impl.metric_list
instance_usage_list = map(dump_as_ascii_string,
instance_usage_list)
DummyAdapter.adapter_impl.metric_list = []
instance_usage_rdd = self.spark_context.parallelize(
instance_usage_list)
sql_context = SQLContext(self.spark_context)
instance_usage_df = sql_context.read.json(instance_usage_rdd)
PreHourlyProcessor.do_transform(instance_usage_df)
metrics = DummyAdapter.adapter_impl.metric_list
mem_total_mb_agg_metric = [
value for value in metrics
if value.get('metric').get('name') ==
'mem.total_mb_agg' and
value.get('metric').get('dimensions').get('host') ==
'mini-mon'][0]
self.assertTrue(mem_total_mb_agg_metric is not None)
self.assertEqual('mem.total_mb_agg',
mem_total_mb_agg_metric
.get('metric').get('name'))
self.assertEqual(15360.0,
mem_total_mb_agg_metric
.get('metric').get('value'))
self.assertEqual('useast',
mem_total_mb_agg_metric
.get('meta').get('region'))
self.assertEqual(cfg.CONF.messaging.publish_kafka_project_id,
mem_total_mb_agg_metric
.get('meta').get('tenantId'))
self.assertEqual('mini-mon',
mem_total_mb_agg_metric
.get('metric').get('dimensions').get('host'))
self.assertEqual('all',
mem_total_mb_agg_metric
.get('metric').get('dimensions').get('project_id'))
self.assertEqual('hourly',
mem_total_mb_agg_metric
.get('metric').get('dimensions')
.get('aggregation_period'))
self.assertEqual(4.0,
mem_total_mb_agg_metric
.get('metric').get('value_meta').get('record_count'))
self.assertEqual('2016-01-20 16:40:00',
mem_total_mb_agg_metric
.get('metric').get('value_meta')
.get('firstrecord_timestamp_string'))
self.assertEqual('2016-01-20 16:40:46',
mem_total_mb_agg_metric
.get('metric').get('value_meta')
.get('lastrecord_timestamp_string'))
def test_fetch_quantity_avg(self,
usage_manager,
setter_manager,
insert_manager,
data_driven_specs_repo):
# test operation
test_operation = "avg"
# load components
usage_manager.return_value = MockComponentManager.get_usage_cmpt_mgr()
setter_manager.return_value = \
MockComponentManager.get_setter_cmpt_mgr()
insert_manager.return_value = \
MockComponentManager.get_insert_pre_hourly_cmpt_mgr()
# init mock driver tables
data_driven_specs_repo.return_value = \
MockDataDrivenSpecsRepo(self.spark_context,
self.get_pre_transform_specs_json(),
self.get_transform_specs_json_by_operation(
test_operation, 'hourly'))
# Create an RDD out of the mocked Monasca metrics
with open(DataProvider.kafka_data_path) as f:
raw_lines = f.read().splitlines()
raw_tuple_list = [eval(raw_line) for raw_line in raw_lines]
rdd_monasca = self.spark_context.parallelize(raw_tuple_list)
# decorate mocked RDD with dummy kafka offsets
myOffsetRanges = [
OffsetRange("metrics", 1, 10, 20)] # mimic rdd.offsetRanges()
transform_context = TransformContextUtils.get_context(
offset_info=myOffsetRanges,
batch_time_info=self.get_dummy_batch_time())
rdd_monasca_with_offsets = rdd_monasca.map(
lambda x: RddTransformContext(x, transform_context))
# Call the primary method in mon_metrics_kafka
MonMetricsKafkaProcessor.rdd_to_recordstore(
rdd_monasca_with_offsets)
# get the metrics that have been submitted to the dummy message adapter
metrics = DummyAdapter.adapter_impl.metric_list
quantity_util_list = map(dump_as_ascii_string, metrics)
DummyAdapter.adapter_impl.metric_list = []
quantity_util_rdd = self.spark_context.parallelize(quantity_util_list)
sql_context = SQLContext(self.spark_context)
quantity_util_df = sql_context.read.json(quantity_util_rdd)
PreHourlyProcessor.do_transform(quantity_util_df)
metrics = DummyAdapter.adapter_impl.metric_list
utilized_cpu_logical_agg_metric = [
value for value in metrics
if value.get('metric').get(
'name') == 'cpu.utilized_logical_cores_agg'][0]
self.assertEqual(7.134214285714285,
utilized_cpu_logical_agg_metric.get(
'metric').get('value'))
self.assertEqual('useast',
utilized_cpu_logical_agg_metric.get(
'meta').get('region'))
self.assertEqual(cfg.CONF.messaging.publish_kafka_project_id,
utilized_cpu_logical_agg_metric.get(
'meta').get('tenantId'))
self.assertEqual('all',
utilized_cpu_logical_agg_metric.get(
'metric').get('dimensions')
.get('host'))
self.assertEqual('all',
utilized_cpu_logical_agg_metric.get(
'metric').get('dimensions')
.get('project_id'))
self.assertEqual('hourly',
utilized_cpu_logical_agg_metric.get(
'metric').get('dimensions')
.get('aggregation_period'))
self.assertEqual(13.0,
utilized_cpu_logical_agg_metric.get(
'metric').get('value_meta')
.get('record_count'))
self.assertEqual('2016-03-07 16:09:23',
utilized_cpu_logical_agg_metric.get(
'metric').get('value_meta')
.get('firstrecord_timestamp_string'))
self.assertEqual('2016-03-07 16:10:38',
utilized_cpu_logical_agg_metric.get(
'metric').get('value_meta')
.get('lastrecord_timestamp_string'))
def test_pod_net_in_usage_app(self, usage_manager, setter_manager,
insert_manager, data_driven_specs_repo):
# load components
usage_manager.return_value = MockComponentManager.get_usage_cmpt_mgr()
setter_manager.return_value = \
MockComponentManager.get_setter_cmpt_mgr()
insert_manager.return_value = \
MockComponentManager.get_insert_cmpt_mgr()
# init mock driver tables
data_driven_specs_repo.return_value = \
MockDataDrivenSpecsRepo(self.spark_context,
self.get_pre_transform_specs_json_app(),
self.get_transform_specs_json_app())
# Create an emulated set of Kafka messages (these were gathered
# by extracting Monasca messages from the Metrics queue on mini-mon).
# Create an RDD out of the mocked Monasca metrics
with open(DataProvider.fetch_quantity_data_path) as f:
raw_lines = f.read().splitlines()
raw_tuple_list = [eval(raw_line) for raw_line in raw_lines]
rdd_monasca = self.spark_context.parallelize(raw_tuple_list)
# decorate mocked RDD with dummy kafka offsets
myOffsetRanges = [OffsetRange("metrics", 1, 10,
20)] # mimic rdd.offsetRanges()
transform_context = TransformContextUtils.get_context(
offset_info=myOffsetRanges,
batch_time_info=self.get_dummy_batch_time())
rdd_monasca_with_offsets = rdd_monasca.map(
lambda x: RddTransformContext(x, transform_context))
# Call the primary method in mon_metrics_kafka
MonMetricsKafkaProcessor.rdd_to_recordstore(rdd_monasca_with_offsets)
# get the metrics that have been submitted to the dummy message adapter
metrics = DummyAdapter.adapter_impl.metric_list
pod_net_usage_agg_metric = [
value for value in metrics
if value.get('metric').get('name') == 'pod.net.in_bytes_sec_agg'
and value.get('metric').get('dimensions').get('app') == 'junk'
and value.get('metric').get('dimensions').get('namespace') == 'all'
and value.get('metric').get('dimensions').get('pod_name') == 'all'
][0]
self.assertTrue(pod_net_usage_agg_metric is not None)
self.assertEqual('pod.net.in_bytes_sec_agg',
pod_net_usage_agg_metric.get('metric').get('name'))
self.assertEqual(
'junk',
pod_net_usage_agg_metric.get("metric").get('dimensions').get(
'app'))
self.assertEqual(
'all',
pod_net_usage_agg_metric.get("metric").get('dimensions').get(
'namespace'))
self.assertEqual(
'all',
pod_net_usage_agg_metric.get("metric").get('dimensions').get(
'pod_name'))
self.assertEqual(122.94,
pod_net_usage_agg_metric.get('metric').get('value'))
self.assertEqual('useast',
pod_net_usage_agg_metric.get('meta').get('region'))
self.assertEqual(cfg.CONF.messaging.publish_kafka_project_id,
pod_net_usage_agg_metric.get('meta').get('tenantId'))
self.assertEqual(
'hourly',
pod_net_usage_agg_metric.get('metric').get('dimensions').get(
'aggregation_period'))
self.assertEqual(
3.0,
pod_net_usage_agg_metric.get('metric').get('value_meta').get(
'record_count'))
self.assertEqual(
'2017-01-24 20:14:47',
pod_net_usage_agg_metric.get('metric').get('value_meta').get(
'firstrecord_timestamp_string'))
self.assertEqual(
'2017-01-24 20:15:47',
pod_net_usage_agg_metric.get('metric').get('value_meta').get(
'lastrecord_timestamp_string'))
def test_rdd_to_recordstore(self,
usage_manager,
setter_manager,
insert_manager):
usage_manager.return_value = MockComponentManager.get_usage_cmpt_mgr()
setter_manager.return_value = \
MockComponentManager.get_setter_cmpt_mgr()
insert_manager.return_value = \
MockComponentManager.get_insert_pre_hourly_cmpt_mgr()
# Create an RDD out of the mocked Monasca metrics
with open(DataProvider.kafka_data_path) as f:
raw_lines = f.read().splitlines()
raw_tuple_list = [eval(raw_line) for raw_line in raw_lines]
rdd_monasca = self.spark_context.parallelize(raw_tuple_list)
# decorate mocked RDD with dummy kafka offsets
myOffsetRanges = [
OffsetRange("metrics", 1, 10, 20)] # mimic rdd.offsetRanges()
transform_context = TransformContextUtils.get_context(
offset_info=myOffsetRanges,
batch_time_info=self.get_dummy_batch_time())
rdd_monasca_with_offsets = rdd_monasca.map(
lambda x: RddTransformContext(x, transform_context))
# Call the primary method in mon_metrics_kafka
MonMetricsKafkaProcessor.rdd_to_recordstore(
rdd_monasca_with_offsets)
host_usage_list = DummyAdapter.adapter_impl.metric_list
host_usage_list = map(dump_as_ascii_string,
host_usage_list)
DummyAdapter.adapter_impl.metric_list = []
host_usage_rdd = self.spark_context.parallelize(host_usage_list)
sql_context = SQLContext(self.spark_context)
host_usage_df = sql_context.read.json(host_usage_rdd)
PreHourlyProcessor.do_transform(host_usage_df)
# get the metrics that have been submitted to the dummy message adapter
metrics = DummyAdapter.adapter_impl.metric_list
# Verify cpu.total_logical_cores_agg for all hosts
total_cpu_logical_agg_metric = [
value for value in metrics
if value.get('metric').get('name') ==
'cpu.total_logical_cores_agg' and
value.get('metric').get('dimensions').get('host') ==
'all'][0]
self.assertEqual(15.0,
total_cpu_logical_agg_metric.get(
'metric').get('value'))
self.assertEqual('useast',
total_cpu_logical_agg_metric.get(
'meta').get('region'))
self.assertEqual(cfg.CONF.messaging.publish_kafka_project_id,
total_cpu_logical_agg_metric.get(
'meta').get('tenantId'))
self.assertEqual('all',
total_cpu_logical_agg_metric.get(
'metric').get('dimensions')
.get('project_id'))
self.assertEqual('hourly',
total_cpu_logical_agg_metric.get(
'metric').get('dimensions')
.get('aggregation_period'))
self.assertEqual(13.0,
total_cpu_logical_agg_metric.get(
'metric').get('value_meta')
.get('record_count'))
self.assertEqual('2016-03-07 16:09:23',
total_cpu_logical_agg_metric.get(
'metric').get('value_meta')
.get('firstrecord_timestamp_string'))
self.assertEqual('2016-03-07 16:10:38',
total_cpu_logical_agg_metric.get(
'metric').get('value_meta')
.get('lastrecord_timestamp_string'))
# Verify cpu.total_logical_cores_agg for mini-mon host
total_cpu_logical_agg_metric = [
value for value in metrics
if value.get('metric').get('name') ==
'cpu.total_logical_cores_agg' and
value.get('metric').get('dimensions').get('host') ==
'test-cp1-comp0333-mgmt'][0]
self.assertEqual(9.0,
total_cpu_logical_agg_metric.get(
'metric').get('value'))
self.assertEqual('useast',
total_cpu_logical_agg_metric.get(
'meta').get('region'))
self.assertEqual(cfg.CONF.messaging.publish_kafka_project_id,
total_cpu_logical_agg_metric.get(
'meta').get('tenantId'))
self.assertEqual('all',
total_cpu_logical_agg_metric.get(
'metric').get('dimensions')
.get('project_id'))
self.assertEqual('hourly',
total_cpu_logical_agg_metric.get(
'metric').get('dimensions')
.get('aggregation_period'))
self.assertEqual(6.0,
total_cpu_logical_agg_metric.get(
'metric').get('value_meta')
.get('record_count'))
self.assertEqual('2016-03-07 16:09:23',
total_cpu_logical_agg_metric.get(
'metric').get('value_meta')
.get('firstrecord_timestamp_string'))
self.assertEqual('2016-03-07 16:10:38',
total_cpu_logical_agg_metric.get(
'metric').get('value_meta')
.get('lastrecord_timestamp_string'))
# Verify cpu.total_logical_cores_agg for devstack host
total_cpu_logical_agg_metric = [
value for value in metrics
if value.get('metric').get('name') ==
'cpu.total_logical_cores_agg' and
value.get('metric').get('dimensions').get('host') ==
'test-cp1-comp0027-mgmt'][0]
self.assertEqual(6.0,
total_cpu_logical_agg_metric.get(
'metric').get('value'))
self.assertEqual('useast',
total_cpu_logical_agg_metric.get(
'meta').get('region'))
self.assertEqual(cfg.CONF.messaging.publish_kafka_project_id,
total_cpu_logical_agg_metric.get(
'meta').get('tenantId'))
self.assertEqual('all',
total_cpu_logical_agg_metric.get(
'metric').get('dimensions')
.get('project_id'))
self.assertEqual('hourly',
total_cpu_logical_agg_metric.get(
'metric').get('dimensions')
.get('aggregation_period'))
self.assertEqual(7.0,
total_cpu_logical_agg_metric.get(
'metric').get('value_meta')
.get('record_count'))
self.assertEqual('2016-03-07 16:09:23',
total_cpu_logical_agg_metric.get(
'metric').get('value_meta')
.get('firstrecord_timestamp_string'))
self.assertEqual('2016-03-07 16:10:38',
total_cpu_logical_agg_metric.get(
'metric').get('value_meta')
.get('lastrecord_timestamp_string'))
# Verify cpu.utilized_logical_cores_agg for all hosts
utilized_cpu_logical_agg_metric = [
value for value in metrics
if value.get('metric').get('name') ==
'cpu.utilized_logical_cores_agg' and
value.get('metric').get('dimensions').get('host') ==
'all'][0]
self.assertEqual(7.134214285714285,
utilized_cpu_logical_agg_metric.get(
'metric').get('value'))
self.assertEqual('useast',
utilized_cpu_logical_agg_metric.get(
'meta').get('region'))
self.assertEqual(cfg.CONF.messaging.publish_kafka_project_id,
utilized_cpu_logical_agg_metric.get(
'meta').get('tenantId'))
self.assertEqual('all',
utilized_cpu_logical_agg_metric.get(
'metric').get('dimensions')
.get('project_id'))
self.assertEqual('hourly',
utilized_cpu_logical_agg_metric.get(
'metric').get('dimensions')
.get('aggregation_period'))
self.assertEqual(13.0,
utilized_cpu_logical_agg_metric.get(
'metric').get('value_meta')
.get('record_count'))
self.assertEqual('2016-03-07 16:09:23',
utilized_cpu_logical_agg_metric.get(
'metric').get('value_meta')
.get('firstrecord_timestamp_string'))
self.assertEqual('2016-03-07 16:10:38',
utilized_cpu_logical_agg_metric.get(
'metric').get('value_meta')
.get('lastrecord_timestamp_string'))
# Verify cpu.utilized_logical_cores_agg for the mini-mon host
utilized_cpu_logical_agg_metric = [
value for value in metrics
if value.get('metric').get('name') ==
'cpu.utilized_logical_cores_agg' and
value.get('metric').get('dimensions').get('host') ==
'test-cp1-comp0333-mgmt'][0]
self.assertEqual(4.9665,
utilized_cpu_logical_agg_metric.get(
'metric').get('value'))
self.assertEqual('useast',
utilized_cpu_logical_agg_metric.get(
'meta').get('region'))
self.assertEqual(cfg.CONF.messaging.publish_kafka_project_id,
utilized_cpu_logical_agg_metric.get(
'meta').get('tenantId'))
self.assertEqual('all',
utilized_cpu_logical_agg_metric.get(
'metric').get('dimensions')
.get('project_id'))
self.assertEqual('hourly',
utilized_cpu_logical_agg_metric.get(
'metric').get('dimensions')
.get('aggregation_period'))
self.assertEqual(6.0,
utilized_cpu_logical_agg_metric.get(
'metric').get('value_meta')
.get('record_count'))
self.assertEqual('2016-03-07 16:09:23',
utilized_cpu_logical_agg_metric.get(
'metric').get('value_meta')
.get('firstrecord_timestamp_string'))
self.assertEqual('2016-03-07 16:10:38',
utilized_cpu_logical_agg_metric.get(
'metric').get('value_meta')
.get('lastrecord_timestamp_string'))
# Verify cpu.utilized_logical_cores_agg for the devstack host
utilized_cpu_logical_agg_metric = [
value for value in metrics
if value.get('metric').get('name') ==
'cpu.utilized_logical_cores_agg' and
value.get('metric').get('dimensions').get('host') ==
'test-cp1-comp0027-mgmt'][0]
self.assertEqual(2.1677142857142853,
utilized_cpu_logical_agg_metric.get(
'metric').get('value'))
self.assertEqual('useast',
utilized_cpu_logical_agg_metric.get(
'meta').get('region'))
self.assertEqual(cfg.CONF.messaging.publish_kafka_project_id,
utilized_cpu_logical_agg_metric.get(
'meta').get('tenantId'))
self.assertEqual('all',
utilized_cpu_logical_agg_metric.get(
'metric').get('dimensions')
.get('project_id'))
self.assertEqual('hourly',
utilized_cpu_logical_agg_metric.get(
'metric').get('dimensions')
.get('aggregation_period'))
self.assertEqual(7.0,
utilized_cpu_logical_agg_metric.get(
'metric').get('value_meta')
.get('record_count'))
self.assertEqual('2016-03-07 16:09:23',
utilized_cpu_logical_agg_metric.get(
'metric').get('value_meta')
.get('firstrecord_timestamp_string'))
self.assertEqual('2016-03-07 16:10:38',
utilized_cpu_logical_agg_metric.get(
'metric').get('value_meta')
.get('lastrecord_timestamp_string'))