def test_polling_no_matching_transform_type(self, message_producer,
message_producer_property,
kv_store):
mock_panoptes_plugin = MockPanoptesObject()
mock_panoptes_plugin.config = ConfigObj(
pwd +
u'/config_files/test_panoptes_polling_plugin_conf_diff_transform_callback.ini'
)
kv_store.return_value = PanoptesMockRedis()
mock_message_producer = MockPanoptesMessageProducer()
message_producer.return_value = mock_message_producer
message_producer_property.return_value = message_producer_property
panoptes_context = PanoptesContext(config_file=os.path.join(
path, u'config_files/test_panoptes_config.ini'))
for i in range(1, 9):
panoptes_metrics = self.prepare_panoptes_metrics_group_set(
u'{}/metric_group_sets/interface_plugin_counter_{}.json'.
format(pwd, i))
_process_metrics_group_set(context=panoptes_context,
results=panoptes_metrics,
plugin=mock_panoptes_plugin)
self.assertEqual(panoptes_context.message_producer.messages, [])
def test_discovery_plugin_task(self, panoptes_context,
message_producer_property,
message_producer):
producer = MockPanoptesMessageProducer()
message_producer_property.return_value = message_producer.return_value = producer
panoptes_context.return_value = self._panoptes_context
discovery_plugin_task("Test Discovery Plugin")
plugin_result = producer.messages
self.assertEqual(len(plugin_result), 1)
plugin_result = plugin_result[0]
self.assertTrue('Test_Discovery_Plugin' in plugin_result['key'])
plugin_result['key'] = 'Test_Discovery_Plugin'
expected_result = {
'topic':
'test_site-resources',
'message':
'{"resource_set_creation_timestamp": 1.0, '
'"resource_set_schema_version": "0.1", "resources": '
'[{"resource_site": "test_site", "resource_class": '
'"test_class", "resource_subclass": "test_subclass", '
'"resource_type": "test_type", "resource_id": '
'"test_resource_id", "resource_endpoint": '
'"test_resource_endpoint", "resource_metadata": '
'{"_resource_ttl": "604800"},'
' "resource_creation_timestamp": 1.0,'
' "resource_plugin": "test_resource_plugin"}]}',
'key':
'Test_Discovery_Plugin'
}
plugin_result['message'] = re.sub(
r"resource_set_creation_timestamp\": \d+\.\d+,",
"resource_set_creation_timestamp\": 1.0,",
plugin_result['message'])
plugin_result['message'] = re.sub(
r"resource_creation_timestamp\": \d+\.\d+,",
"resource_creation_timestamp\": 1.0,", plugin_result['message'])
self.assertEqual(plugin_result['topic'], expected_result['topic'])
self.assertEqual(plugin_result['key'], expected_result['key'])
self.assertEqual(ordered(json.loads(plugin_result['message'])),
ordered(json.loads(expected_result['message'])))
def test_polling_kafka_produces_to_the_correct_topics(
self, message_producer, message_producer_property):
spec_paths = [
u'config_files/test_panoptes_config.ini', # Produces to site topics but not Global
u'config_files/test_panoptes_config_kafka_true.ini' # Sends enrichment results to both site & global topics
]
expected_results = [{
u'test_site-processed': 1
}, {
u'panoptes-metrics': 1,
u'test_site-processed': 1
}]
all_tests_pass = True
for i in range(len(spec_paths)):
mock_panoptes_plugin = MockPanoptesObject()
mock_panoptes_plugin.config = ConfigObj(
pwd + u'/config_files/test_panoptes_polling_plugin_conf.ini')
mock_message_producer = MockPanoptesMessageProducer()
message_producer.return_value = mock_message_producer
message_producer_property.return_value = message_producer_property
panoptes_context = PanoptesContext(
config_file=os.path.join(path, spec_paths[i]))
metrics_group_set = self.prepare_panoptes_metrics_group_set()
_process_metrics_group_set(context=panoptes_context,
results=metrics_group_set,
plugin=mock_panoptes_plugin)
published_kafka_messages = panoptes_context.message_producer.messages
actual_result = {}
for message in published_kafka_messages:
if message[u'topic'] in actual_result:
actual_result[message[u'topic']] += 1
else:
actual_result[message[u'topic']] = 1
all_tests_pass &= (actual_result == expected_results[i])
self.assertTrue(all_tests_pass)
def test_polling_plugin_agent(self, resource, panoptes_context,
message_producer, message_producer_property,
time):
producer = MockPanoptesMessageProducer()
time.return_value = 1
message_producer.return_value = producer
message_producer_property.return_value = producer
resource.return_value = self._panoptes_resource
panoptes_context.return_value = self._panoptes_context
polling_plugin_task('Test Polling Plugin', 'polling')
log_prefix = '[Test Polling Plugin] [plugin|test|site|test|class|test|' \
'subclass|test|type|test|id|test|endpoint|test]'
self._log_capture.check_present(
('panoptes.tests.test_runner', 'INFO',
'Attempting to execute plugin "Test Polling Plugin"'),
('panoptes.tests.test_runner', 'DEBUG',
'Loaded plugin "Test Polling Plugin", '
'version "0.1" of type "polling", category "polling"'),
('panoptes.tests.test_runner', 'DEBUG',
'Loaded plugin "Test Polling Plugin 2", '
'version "0.1" of type "polling", category "polling"'),
('panoptes.tests.test_runner', 'ERROR',
'No enrichment data found on KV store for plugin Test'
' Polling Plugin resource test namespace test using key test'),
('panoptes.tests.test_runner', 'DEBUG',
'Successfully created PanoptesEnrichmentCache enrichment_data '
'{} for plugin Test Polling Plugin'),
('panoptes.tests.test_runner', 'DEBUG',
'Attempting to get lock for plugin "Test Polling Plugin", '
'with lock path and identifier in seconds'),
('panoptes.tests.test_runner', 'INFO',
'{} Acquired lock'.format(log_prefix)),
('panoptes.tests.test_runner', 'INFO',
'{} Plugin returned a result set with 1 members'.format(
log_prefix)),
('panoptes.tests.test_runner', 'INFO',
'{} Callback function ran in seconds'.format(log_prefix)),
('panoptes.tests.test_runner', 'INFO',
'{} Ran in seconds'.format(log_prefix)),
('panoptes.tests.test_runner', 'INFO',
'{} Released lock'.format(log_prefix)),
('panoptes.tests.test_runner', 'INFO',
'{} GC took seconds. There are garbage objects.'.format(
log_prefix)),
('panoptes.tests.test_runner', 'DEBUG',
'Deleting module: yapsy_loaded_plugin_Test_Polling_Plugin'),
('panoptes.tests.test_runner', 'DEBUG',
'Deleting module: yapsy_loaded_plugin_Test_Polling_Plugin'),
('panoptes.tests.test_runner', 'DEBUG', 'Deleting module: '
'yapsy_loaded_plugin_Test_Polling_Plugin_Second_Instance'),
order_matters=False)
kafka_push_log = {
"metrics_group_type":
"Test",
"metrics_group_interval":
60,
"metrics_group_creation_timestamp":
1,
"metrics_group_schema_version":
"0.2",
"resource": {
"resource_site": "test",
"resource_class": "test",
"resource_subclass": "test",
"resource_type": "test",
"resource_id": "test",
"resource_endpoint": "test",
"resource_metadata": {
"_resource_ttl": "604800"
},
"resource_creation_timestamp": 1.0,
"resource_plugin": "test"
},
"metrics": [{
"metric_creation_timestamp": 1,
"metric_name": "test",
"metric_value": 0.0,
"metric_type": "gauge"
}],
"dimensions": []
}
# Timestamps need to be removed to check Panoptes Metrics
metric_groups_seen = 0
for line in self._log_capture.actual():
_, _, log = line
if 'resource_creation_timestamp' in log:
log = re.sub(r"resource_creation_timestamp\": \d+\.\d+,",
"resource_creation_timestamp\": 1.0,", log)
resource_match = re.search(r'{.*}', log)
if resource_match is not None:
self.assertEqual(
ordered(json.loads(resource_match.group(0))),
ordered(kafka_push_log))
if log.startswith('Sent metric group'):
metric_groups_seen += 1
if log.startswith('Going to send metric group'):
metric_groups_seen += 1
self.assertEqual(metric_groups_seen, 2)
def test_polling_transform_converts_correct_counters(
self, message_producer, message_producer_property, kv_store):
mock_panoptes_plugin = MockPanoptesObject()
mock_panoptes_plugin.config = ConfigObj(
pwd + u'/config_files/test_panoptes_polling_plugin_conf.ini')
kv_store.return_value = PanoptesMockRedis()
mock_message_producer = MockPanoptesMessageProducer()
message_producer.return_value = mock_message_producer
message_producer_property.return_value = message_producer_property
panoptes_context = PanoptesContext(config_file=os.path.join(
path, u'config_files/test_panoptes_config.ini'))
for i in range(1, 9):
panoptes_metrics = self.prepare_panoptes_metrics_group_set(
u'{}/metric_group_sets/interface_plugin_counter_{}.json'.
format(pwd, i))
_process_metrics_group_set(context=panoptes_context,
results=panoptes_metrics,
plugin=mock_panoptes_plugin)
published_kafka_messages = panoptes_context.message_producer.messages
expected_results = [
{ # Test Conversion
u'counter|test_system_uptime': 0,
u'counter|extra_test_metric': 0
},
{ # Test Conversion
u'counter|test_system_uptime': 60,
u'gauge|test_system_uptime': 0,
u'counter|extra_test_metric': 120,
u'gauge|extra_test_metric': 1
},
{ # Test Conversion
u'counter|test_system_uptime': 120,
u'gauge|test_system_uptime': 0,
u'counter|extra_test_metric': 240,
u'gauge|extra_test_metric': 1
},
{ # Test time difference is negative
u'counter|test_system_uptime': 120,
u'gauge|test_system_uptime': 0,
u'counter|extra_test_metric': 240,
u'gauge|extra_test_metric': 1
},
{ # Test time difference is zero
u'counter|test_system_uptime': 120,
u'gauge|test_system_uptime': 0,
u'counter|extra_test_metric': 240,
u'gauge|extra_test_metric': 1
},
{ # Test Time difference is greater than TTL multiple
u'counter|test_system_uptime': 500,
u'gauge|test_system_uptime': 0,
u'counter|extra_test_metric': 1000,
u'gauge|extra_test_metric': 1
},
{
# Confidence below the const.METRICS_CONFIDENCE_THRESHOLD
# ( time_difference > (metrics_group.interval * const.METRICS_KV_STORE_TTL_MULTIPLE )
u'counter|test_system_uptime': 500,
u'gauge|test_system_uptime': 0,
u'counter|extra_test_metric': 1000,
u'gauge|extra_test_metric': 0
},
{
# Skip Conversion due to the new counter value
# being less than the previous value
u'counter|test_system_uptime': 400,
u'gauge|test_system_uptime': 1,
u'counter|extra_test_metric': 900,
u'gauge|extra_test_metric': 1
}
]
for i, kafka_message in enumerate(published_kafka_messages):
parsed_json = json.loads(kafka_message[u'message'])[u'metrics']
for panoptes_metric in parsed_json:
key = u"|".join([
panoptes_metric[u'metric_type'],
panoptes_metric[u'metric_name']
])
self.assertEquals(expected_results[i].get(key, None),
panoptes_metric[u'metric_value'])