class SNMPPluginTestFramework(object):
plugin_class = None
path = None
snmp_host = '127.0.0.1'
snmp_port = 10161
snmp_timeout = 10
snmp_retries = 1
snmp_max_repetitions = 10
snmp_community = 'public'
snmp_failure_timeout = 1
resource_id = 'test_id'
resource_endpoint = '127.0.0.1'
resource_plugin = 'dummy'
resource_site = 'test_site'
resource_class = 'network'
resource_subclass = 'test_subclass'
resource_type = 'test_type'
resource_backplane = None
resource_model = 'model'
results_data_file = 'results.json'
enrichment_data_file = 'enrichment_data'
use_enrichment = True
plugin_conf = {
'Core': {
'name': 'Test Plugin',
'module': 'test_plugin'
},
'main': {
'execute_frequency': '60',
'enrichment_ttl': '300',
'resource_filter': 'resource_class = "network"'
},
'snmp': {
'timeout': 10,
'retries': 1,
'non_repeaters': 1,
'max_repetitions': 25,
}
}
def __init__(self, test_name):
super(SNMPPluginTestFramework, self).__init__(test_name)
self._path = self.path
self._plugin_conf = self.plugin_conf
self._snmp_host = self.snmp_host
self._snmp_port = self.snmp_port
self._snmp_timeout = self.snmp_timeout
self._snmp_retries = self.snmp_retries
self._snmp_max_repetitions = self.snmp_max_repetitions
self._snmp_community = self.snmp_community
self._snmp_failure_timeout = self.snmp_failure_timeout
self._resource_id = self.resource_id
self._resource_endpoint = self.resource_endpoint
self._resource_plugin = self.resource_plugin
self._resource_site = self.resource_site
self._resource_class = self.resource_class
self._resource_subclass = self.resource_subclass
self._resource_type = self.resource_type
self._resource_backplane = self.resource_backplane
self._resource_model = self.resource_model
self._panoptes_context = None
self._panoptes_resource = None
self._snmp_conf = None
self._expected_results = None
self._results_data_file = None
self._enrichment_data_file = None
self._secret_store = None
self._plugin_context = None
self._plugin_context_bad = None
self._snmp_conf_bad = None
self._enrichment_kv = None
self._enrichment_cache = None
def set_panoptes_context(self):
panoptes_test_conf_file = os.path.join(
os.path.dirname(os.path.dirname(__file__)),
'config_files/test_panoptes_config.ini')
self._panoptes_context = PanoptesContext(
panoptes_test_conf_file,
key_value_store_class_list=[PanoptesEnrichmentCacheKeyValueStore])
def set_panoptes_resource(self):
self._panoptes_resource = PanoptesResource(
resource_site=self._resource_site,
resource_class=self._resource_class,
resource_subclass=self._resource_subclass,
resource_type=self._resource_type,
resource_id=self._resource_id,
resource_endpoint=self._resource_endpoint,
resource_plugin=self._resource_plugin)
self._panoptes_resource.resource_metadata['model'] = 'model'
if self._resource_backplane:
self._panoptes_resource.add_metadata('backplane',
self._resource_backplane)
def set_enrichment_cache(self):
if self.use_enrichment:
self._enrichment_data_file = 'data/' + self.enrichment_data_file
self._enrichment_kv = self._panoptes_context.get_kv_store(
PanoptesEnrichmentCacheKeyValueStore)
enrichment_data_file = os.path.join(os.path.abspath(self.path),
self._enrichment_data_file)
if self._plugin_conf.get('enrichment'):
if self._plugin_conf['enrichment'].get('preload'):
self._enrichment_cache = PanoptesEnrichmentCache(
self._panoptes_context, self._plugin_conf,
self._panoptes_resource)
try:
with open(enrichment_data_file) as enrichment_data:
for line in enrichment_data:
key, value = line.strip().split('=>')
self._enrichment_kv.set(key, value)
except Exception as e:
raise IOError(
'Failed to load enrichment data file {}: {}'.format(
enrichment_data_file, repr(e)))
def set_snmp_conf(self):
self._snmp_conf = self._panoptes_context.config_object.snmp_defaults
def set_secret_store(self):
self._secret_store = create_autospec(PanoptesSecretsStore,
instance=True,
spec_set=True)
self._secret_store.get_by_site.return_value = self._snmp_community
def set_plugin_context(self):
self._plugin_context = create_autospec(
PanoptesPluginWithEnrichmentContext,
instance=True,
spec_set=True,
data=self._panoptes_resource,
enrichment=self._enrichment_cache,
config=self._plugin_conf,
snmp=self._snmp_conf,
secrets=self._secret_store,
logger=logging.getLogger(__name__))
def set_snmp_conf_bad(self):
self._snmp_conf_bad = copy.copy(self._snmp_conf)
self._snmp_conf_bad['port'] += 1
self._snmp_conf_bad['timeout'] = self._snmp_failure_timeout
def set_plugin_context_bad(self):
self._plugin_context_bad = create_autospec(
PanoptesPluginWithEnrichmentContext,
instance=True,
spec_set=True,
data=self._panoptes_resource,
enrichment=self._enrichment_cache,
config=self._plugin_conf,
snmp=self._snmp_conf_bad,
secrets=self._secret_store,
logger=logging.getLogger(__name__))
def set_expected_results(self):
self._results_data_file = 'data/' + self.results_data_file
expected_result_file = os.path.join(os.path.abspath(self.path),
self._results_data_file)
self._expected_results = json.load(open(expected_result_file))
@patch('time.time', mock_time)
@patch('yahoo_panoptes.framework.resources.time', mock_time)
@patch('redis.StrictRedis', PanoptesMockRedis)
def setUp(self):
self.set_panoptes_context()
self.set_panoptes_resource()
self.set_enrichment_cache()
self.set_snmp_conf()
self.set_secret_store()
self.set_plugin_context()
self.set_snmp_conf_bad()
self.set_plugin_context_bad()
self.set_expected_results()
class TestPanoptesEnrichmentCacheStore(unittest.TestCase):
@patch(u'redis.StrictRedis', PanoptesMockRedis)
@patch(u'time.time', mock_time)
def setUp(self):
self.my_dir, self.panoptes_test_conf_file = _get_test_conf_file()
self._panoptes_context = PanoptesContext(self.panoptes_test_conf_file,
key_value_store_class_list=[PanoptesEnrichmentCacheKeyValueStore,
PanoptesResourcesKeyValueStore])
self._enrichment_kv = self._panoptes_context.get_kv_store(PanoptesEnrichmentCacheKeyValueStore)
self._panoptes_resource = PanoptesResource(resource_site=u'test_site',
resource_class=u'test_class',
resource_subclass=u'test_subclass',
resource_type=u'test_type',
resource_id=u'test_resource_id',
resource_endpoint=u'test_endpoint',
resource_plugin=u'test_plugin')
interface_validation_object = PanoptesEnrichmentInterfaceSchemaValidator()
neighbor_validation_object = PanoptesEnrichmentNeighborSchemaValidator()
enrichment_set1 = PanoptesEnrichmentSet(u'int_001')
enrichment_set1.add(u'speed', 1000)
enrichment_set1.add(u'index', 0o01)
enrichment_set1.add(u'status', u'up')
enrichment_set2 = PanoptesEnrichmentSet(u'int_002')
enrichment_set2.add(u'speed', 1000)
enrichment_set2.add(u'index', 0o02)
enrichment_set2.add(u'status', u'down')
enrichment_group1 = PanoptesEnrichmentGroup(u'interface', interface_validation_object, 300, 60)
enrichment_group1.add_enrichment_set(enrichment_set1)
enrichment_group1.add_enrichment_set(enrichment_set2)
enrichment_set3 = PanoptesEnrichmentSet(u'host_name')
enrichment_set3.add(u'vlan_id', 501)
enrichment_set3.add(u'property', u'Test Property')
enrichment_set3.add(u'mac', u'aa:bb:cc:dd:ee:ff')
enrichment_group2 = PanoptesEnrichmentGroup(u'neighbor', neighbor_validation_object, 600, 120)
enrichment_group2.add_enrichment_set(enrichment_set3)
self.enrichment_group_set1 = PanoptesEnrichmentGroupSet(self._panoptes_resource)
self.enrichment_group_set1.add_enrichment_group(enrichment_group1)
self.enrichment_group_set1.add_enrichment_group(enrichment_group2)
self._panoptes_resource01 = PanoptesResource(resource_site=u'test_site',
resource_class=u'test_class',
resource_subclass=u'test_subclass',
resource_type=u'test_type',
resource_id=u'test_resource_id01',
resource_endpoint=u'test_endpoint01',
resource_plugin=u'test_plugin')
enrichment_set4 = PanoptesEnrichmentSet(u'host_name01')
enrichment_set4.add(u'vlan_id', 502)
enrichment_set4.add(u'property', u'Test Property')
enrichment_set4.add(u'mac', u'aa:bb:cc:dd:ee:ff')
enrichment_group3 = PanoptesEnrichmentGroup(u'neighbor', neighbor_validation_object, 600, 120)
enrichment_group3.add_enrichment_set(enrichment_set3)
enrichment_group3.add_enrichment_set(enrichment_set4)
self.enrichment_group_set2 = PanoptesEnrichmentGroupSet(self._panoptes_resource01)
self.enrichment_group_set2.add_enrichment_group(enrichment_group1)
self.enrichment_group_set2.add_enrichment_group(enrichment_group3)
self._multi_enrichment_group_set = PanoptesEnrichmentMultiGroupSet()
self._multi_enrichment_group_set.add_enrichment_group_set(self.enrichment_group_set1)
self._multi_enrichment_group_set.add_enrichment_group_set(self.enrichment_group_set2)
@patch(u'time.time', mock_time)
def test_panoptes_enrichment_set(self):
enrichment_set1 = PanoptesEnrichmentSet(u'int_001')
enrichment_set1.add(u'speed', 1000)
enrichment_set1.add(u'index', 0o01)
enrichment_set1.add(u'status', u'up')
self.assertEquals(enrichment_set1.json(),
u'{"int_001": {"index": 1, "speed": 1000, "status": "up"}}')
self.assertEquals(repr(enrichment_set1),
u"PanoptesEnrichmentSet[int_001[index:1,speed:1000,status:up]]")
enrichment_set2 = PanoptesEnrichmentSet(u'int_002')
enrichment_set2.add(u'speed', 1000)
enrichment_set2.add(u'index', 0o02)
enrichment_set2.add(u'status', u'down')
interface_validation_object = PanoptesEnrichmentInterfaceSchemaValidator()
enrichment_group1 = PanoptesEnrichmentGroup(u'interface', interface_validation_object, 300, 60)
self.assertFalse(enrichment_set1 == enrichment_group1)
self.assertFalse(enrichment_set1 == enrichment_set2)
@patch(u'time.time', mock_time)
def test_panoptes_enrichment_group(self):
interface_validation_object = PanoptesEnrichmentInterfaceSchemaValidator()
enrichment_group1 = PanoptesEnrichmentGroup(u'interface', interface_validation_object, 300, 60)
self.assertEquals(enrichment_group1.enrichment_schema, PanoptesEnrichmentInterfaceSchemaValidator.schema)
self.assertEquals(repr(enrichment_group1), u"PanoptesEnrichmentGroup[namespace:interface,"
u"enrichment_ttl:300,execute_frequency:60,"
u"enrichment_group_creation_timestamp:{}]".format(
mock_time.return_value))
enrichment_set1 = PanoptesEnrichmentSet(u'int_001')
enrichment_set1.add(u'speed', 1000)
enrichment_set1.add(u'index', 0o01)
enrichment_set1.add(u'status', u'up')
self.assertFalse(enrichment_group1 == enrichment_set1)
enrichment_group2 = PanoptesEnrichmentGroup(u'interface', interface_validation_object, 300, 60)
enrichment_group3 = PanoptesEnrichmentGroup(u'other_namespace', interface_validation_object, 300, 60)
self.assertTrue(enrichment_group1 == enrichment_group2)
self.assertFalse(enrichment_group1 == enrichment_group3)
@patch(u'time.time', mock_time)
def test_store_enrichment_data_enrichment_group_set(self):
interface_result_data = \
{
"data": {
"int_001": {
"index": 1,
"speed": 1000,
"status": "up"
},
"int_002": {
"index": 2,
"speed": 1000,
"status": "down"
}
},
"metadata": {
"_enrichment_group_creation_timestamp": mock_time.return_value,
"_enrichment_ttl": 300,
"_execute_frequency": 60
}
}
neighbor_result_data = \
{
"data": {
"host_name": {
"mac": "aa:bb:cc:dd:ee:ff",
"property": "Test Property",
"vlan_id": 501
}
},
"metadata": {
"_enrichment_group_creation_timestamp": mock_time.return_value,
"_enrichment_ttl": 600,
"_execute_frequency": 120
}
}
_store_enrichment_data(self._panoptes_context, self.enrichment_group_set1, u'PanoptesPluginInfo')
self.assertNotEquals(ordered(interface_result_data),
ordered(json.loads(self._enrichment_kv.get(u'test_resource_id:neighbor'))))
self.assertEquals(ordered(interface_result_data),
ordered(json.loads(self._enrichment_kv.get(u'test_resource_id:interface'))))
self.assertEquals(ordered(neighbor_result_data),
ordered(json.loads(self._enrichment_kv.get(u'test_resource_id:neighbor'))))
def test_store_enrichment_data_enrichment_multi_group_set(self):
enrichment_result_keys = [u'test_resource_id01:interface', u'test_resource_id01:neighbor',
u'test_resource_id:interface', u'test_resource_id:neighbor']
neighbor_result_data = \
{u"data": {u"host_name": {u"mac": u"aa:bb:cc:dd:ee:ff", u"property": u"Test Property", u"vlan_id": 501},
u"host_name01": {u"mac": u"aa:bb:cc:dd:ee:ff", u"property": u"Test Property", u"vlan_id": 502}},
u"metadata": {u"_enrichment_group_creation_timestamp": mock_time.return_value, u"_enrichment_ttl": 600,
u"_execute_frequency": 120}}
_store_enrichment_data(self._panoptes_context, self._multi_enrichment_group_set, u'PanoptesPluginInfo')
self.assertEquals(enrichment_result_keys, self._enrichment_kv.find_keys(u'*'))
self.assertEquals(ordered(neighbor_result_data),
ordered(json.loads(self._enrichment_kv.get(u'test_resource_id01:neighbor'))))
self.assertNotEquals(ordered(neighbor_result_data),
ordered(json.loads(self._enrichment_kv.get(u'test_resource_id01:interface'))))
@patch(u'yahoo_panoptes.enrichment.enrichment_plugin_agent.PanoptesPluginWithEnrichmentRunner',
create_auto_spec=True)
@patch(u'yahoo_panoptes.framework.resources.PanoptesResourceStore.get_resource')
@patch(u'yahoo_panoptes.enrichment.enrichment_plugin_agent.PanoptesEnrichmentTaskContext')
def test_enrichment_plugin_task_is_executed(self, task_context, resource, enrichment_runner):
task_context.return_value = self._panoptes_context
resource.return_value = self._panoptes_resource
# Test Exception is Thrown on failure to create PanoptesEnrichmentTaskContext
task_context.side_effect = Exception()
with self.assertRaises(SystemExit):
enrichment_plugin_task(u'name', u'key')
task_context.side_effect = None
# Test Exception is Thrown on failure to create / run plugin
enrichment_runner.side_effect = Exception()
enrichment_plugin_task(u'name', u'key')
enrichment_runner.execute_plugin.assert_not_called()
enrichment_runner.side_effect = None
# Test Enrichment Is Executed
enrichment_plugin_task(u'name', u'key')
enrichment_runner.assert_called()
enrichment_runner().execute_plugin.assert_called_once()
class TestPanoptesEnrichmentCache(unittest.TestCase):
@patch(u'redis.StrictRedis', PanoptesMockRedis)
def setUp(self):
self.my_dir, self.panoptes_test_conf_file = _get_test_conf_file()
self._panoptes_context = PanoptesContext(
self.panoptes_test_conf_file,
key_value_store_class_list=[PanoptesEnrichmentCacheKeyValueStore])
self._enrichment_kv = self._panoptes_context.get_kv_store(
PanoptesEnrichmentCacheKeyValueStore)
self._panoptes_resource = PanoptesResource(
resource_site=u'test_site',
resource_class=u'test_class',
resource_subclass=u'test_subclass',
resource_type=u'test_type',
resource_id=u'test_resource_id',
resource_endpoint=u'test_endpoint',
resource_plugin=u'test_plugin')
self._plugin_conf = {
u'Core': {
u'name': u'Heartbeat Enrichment Plugin',
u'module': u'plugin_enrichment_heartbeat'
},
u'main': {
u'execute_frequency':
u'60',
u'enrichment_ttl':
u'300',
u'resource_filter':
u'resource_class = "system" AND resource_subclass = '
u'"internal" AND resource_type = "heartbeat"'
},
u'enrichment': {
u'preload': u'self:test_namespace'
}
}
self._enrichment_kv.set(
u'test_resource_id:test_namespace',
u'{{"data": {{"heartbeat": {{"timestamp": 1521668419}}}}, '
u'"metadata": {{"_enrichment_group_creation_timestamp": {:.5f}, '
u'"_enrichment_ttl": 300, "_execute_frequency": 60}}}}'.format(
mock_time.return_value))
self._enrichment_kv.set(
u'test_resource_id01:interface',
u'{{"data": {{"1226": {{"ifType": "l3ipvlan", "ifAlias": "<not set>", "ifPhysAddress": '
u'"00:1f:a0:13:8c:16", "ifDescr": "Virtual Ethernet 226", "ifName": "Virtual Ethernet 226", '
u'"ifMtu": "1500"}}, "1209": {{"ifType": "l3ipvlan", "ifAlias": "<not set>", "ifPhysAddress": '
u'"00:1f:a0:13:8c:15", "ifDescr": "Virtual Ethernet 209", "ifName": "Virtual Ethernet 209", '
u'"ifMtu": "1500"}}}}, "metadata": {{"_enrichment_group_creation_timestamp": {:.5f}, '
u'"_enrichment_ttl": 300, "_execute_frequency": 60}}}}'.format(
mock_time.return_value))
self._enrichment_kv.set(
u'test_resource_id02:interface',
u'{{"data": {{"2226": {{"ifType": "l3ipvlan", "ifAlias": "<not set>", "ifPhysAddress": '
u'"00:1f:a0:13:8c:16", "ifDescr": "Virtual Ethernet 326", "ifName": "Virtual Ethernet 326", '
u'"ifMtu": "1500"}}, "2209": {{"ifType": "l3ipvlan", "ifAlias": "<not set>", "ifPhysAddress": '
u'"00:1f:a0:13:8c:15", "ifDescr": "Virtual Ethernet 309", "ifName": "Virtual Ethernet 309", '
u'"ifMtu": "1500"}}}}, "metadata": {{"_enrichment_group_creation_timestamp": {:.5f}, '
u'"_enrichment_ttl": 300, "_execute_frequency": 60}}}}'.format(
mock_time.return_value))
self._enrichment_kv.set(
u'asn_api:namespace01',
u'{{"data": {{"asn_data01": {{"data": 101}}}}, '
u'"metadata": {{"_enrichment_group_creation_timestamp": {:.5f}, '
u'"_enrichment_ttl": 300, "_execute_frequency": 60}}}}'.format(
mock_time.return_value))
self._enrichment_kv.set(
u'asn_api:namespace02',
u'{{"data": {{"asn_data02": {{"data": 202}}}}, '
u'"metadata": {{"_enrichment_group_creation_timestamp": {:.5f}, '
u'"_enrichment_ttl": 300, "_execute_frequency": 60}}}}'.format(
mock_time.return_value))
self._enrichment_kv.set(
u'asn_api:namespace03',
u'{{"data": {{"asn_data03": {{"data": 303}}}}, '
u'"metadata": {{"_enrichment_group_creation_timestamp": {:.5f}, '
u'"_enrichment_ttl": 300, "_execute_frequency": 60}}}}'.format(
mock_time.return_value))
self._enrichment_kv.set(u'enrichment:asn_api:namespace05', u'bad_data')
def test_enrichment_cache(self):
"""Test enrichment resource attributes"""
with self.assertRaises(AssertionError):
PanoptesEnrichmentCache(u'non_panoptes_context', self._plugin_conf,
self._panoptes_resource)
with self.assertRaises(AssertionError):
PanoptesEnrichmentCache(self._panoptes_context, u'non_plugin_conf',
self._panoptes_resource)
with self.assertRaises(AssertionError):
PanoptesEnrichmentCache(self._panoptes_context, self._plugin_conf,
u'non_panoptes_resource')
# Test with bad key-value store
mock_panoptes_enrichment_cache_key_value_store = Mock(
side_effect=Exception)
with patch(
u'yahoo_panoptes.framework.enrichment.PanoptesEnrichmentCacheKeyValueStore',
mock_panoptes_enrichment_cache_key_value_store):
with self.assertRaises(Exception):
PanoptesEnrichmentCache(self._panoptes_context,
self._plugin_conf,
self._panoptes_resource)
def test_enrichment_cache_preload01(self):
"""Test enrichment resource with preload conf self:test_namespace"""
result = u'{{"test_resource_id": {{"test_namespace": {{"heartbeat": {{"timestamp": {}}}}}}}}}'.format(
int(mock_time.return_value))
enrichment_cache = PanoptesEnrichmentCache(self._panoptes_context,
self._plugin_conf,
self._panoptes_resource)
self.assertEqual(
ordered(enrichment_cache.__dict__[u'_enrichment_data']),
ordered(json.loads(result)))
def test_enrichment_cache_preload02(self):
"""Test enrichment resource with preload conf"""
plugin_conf = {
u'Core': {
u'name': u'Test Plugin 01'
},
u'enrichment': {
u'preload':
u'self:test_namespace, '
u'asn_api:*, '
u'test_resource_id01:interface, '
u'no_data_resource_id:test_namespace, '
u'test_resource_id01:no_data_namespace'
}
}
results01 = u"""{{"asn_api": {{"namespace02": {{"asn_data02": {{"data": 202}}}}, "namespace03": {{"asn_data03":
{{"data": 303}}}}, "namespace01": {{"asn_data01": {{"data": 101}}}}}}, "test_resource_id": {{"test_namespace":
{{"heartbeat": {{"timestamp": {}}}}}}}, "test_resource_id01": {{"interface": {{"1226": {{"ifType": "l3ipvlan",
"ifAlias": "<not set>", "ifPhysAddress": "00:1f:a0:13:8c:16", "ifDescr": "Virtual Ethernet 226", "ifName":
"Virtual Ethernet 226", "ifMtu": "1500"}}, "1209": {{"ifType": "l3ipvlan", "ifAlias": "<not set>",
"ifPhysAddress": "00:1f:a0:13:8c:15", "ifDescr": "Virtual Ethernet 209", "ifName": "Virtual Ethernet 209",
"ifMtu": "1500"}}}}}}}}""".format(int(mock_time.return_value))
enrichment_cache = PanoptesEnrichmentCache(self._panoptes_context,
plugin_conf,
self._panoptes_resource)
self.assertEqual(
ordered(enrichment_cache.__dict__[u'_enrichment_data']),
ordered(json.loads(results01)))
# Test an error while scanning kv store is handled correctly
mock_find_keys = Mock(side_effect=Exception)
mock_logger = MockLogger()
with patch(
u'yahoo_panoptes.framework.enrichment.PanoptesEnrichmentCacheKeyValueStore.find_keys',
mock_find_keys):
with patch(
u'yahoo_panoptes.framework.enrichment.PanoptesContext.logger',
mock_logger):
PanoptesEnrichmentCache(self._panoptes_context, plugin_conf,
self._panoptes_resource)
self.assertEqual(mock_logger.error.call_count, 3)
mock_kv_store_get = Mock(side_effect=IOError)
with patch(
u'yahoo_panoptes.framework.enrichment.PanoptesEnrichmentCacheKeyValueStore.get',
mock_kv_store_get):
with self.assertRaises(IOError):
PanoptesEnrichmentCache(self._panoptes_context, plugin_conf,
self._panoptes_resource)
def test_enrichment_cache_parse_conf(self):
"""Test enrichment resource parse conf"""
result01 = [(u'asn_api', u'*'), (u'self', u'test_namespace'),
(u'test_resource_id01', u'interface')]
plugin_conf = {
u'Core': {
u'name': u'Test Plugin 01'
},
u'enrichment': {
u'preload':
u'self:test_namespace, asn_api:*, test_resource_id01:interface'
}
}
enrichment_cache = PanoptesEnrichmentCache(self._panoptes_context,
plugin_conf,
self._panoptes_resource)
self.assertEqual(sorted(result01),
sorted(enrichment_cache.__dict__[u'_preload_conf']))
plugin_conf_with_dup = {
u'Core': {
u'name': u'Test Plugin 01'
},
u'enrichment': {
u'preload':
u'self:test_namespace, asn_api:*, '
u'test_resource_id01:interface, asn_api:*, '
u'test_resource_id01:interface'
}
}
enrichment_cache = PanoptesEnrichmentCache(self._panoptes_context,
plugin_conf_with_dup,
self._panoptes_resource)
self.assertEqual(sorted(result01),
sorted(enrichment_cache.__dict__[u'_preload_conf']))
def test_enrichment_cache_parse_conf_bad(self):
"""Test enrichment resource parse bad conf"""
plugin_conf01 = {
u'Core': {
u'name': u'Test Plugin 01'
},
u'enrichment': {}
}
with self.assertRaises(PanoptesEnrichmentCacheError):
PanoptesEnrichmentCache(self._panoptes_context, plugin_conf01,
self._panoptes_resource)
plugin_conf02 = {
u'Core': {
u'name': u'Test Plugin 01'
},
u'enrichment': {
u'preload': {}
}
}
with self.assertRaises(PanoptesEnrichmentCacheError):
PanoptesEnrichmentCache(self._panoptes_context, plugin_conf02,
self._panoptes_resource)
plugin_conf03 = {
u'Core': {
u'name': u'Test Plugin 01'
},
u'enrichment': {
u'preload': {u'self:test_namespace, asn_api:'}
}
}
with self.assertRaises(PanoptesEnrichmentCacheError):
PanoptesEnrichmentCache(self._panoptes_context, plugin_conf03,
self._panoptes_resource)
plugin_conf04 = {
u'Core': {
u'name': u'Test Plugin 01'
},
u'enrichment': {
u'preload': {u'self:test_namespace, :'}
}
}
with self.assertRaises(PanoptesEnrichmentCacheError):
PanoptesEnrichmentCache(self._panoptes_context, plugin_conf04,
self._panoptes_resource)
plugin_conf05 = {
u'Core': {
u'name': u'Test Plugin 01'
},
u'enrichment': {
u'preload': {u'self:test_namespace, '}
}
}
with self.assertRaises(PanoptesEnrichmentCacheError):
PanoptesEnrichmentCache(self._panoptes_context, plugin_conf05,
self._panoptes_resource)
plugin_conf06 = {
u'Core': {
u'name': u'Test Plugin 01'
},
u'enrichment': {
u'preload': {u'self:test_namespace, :namespace01'}
}
}
with self.assertRaises(PanoptesEnrichmentCacheError):
PanoptesEnrichmentCache(self._panoptes_context, plugin_conf06,
self._panoptes_resource)
def test_enrichment_cache_get_enrichment(self):
"""Test enrichment resource get"""
plugin_conf = {
u'Core': {
u'name': u'Test Plugin 01'
},
u'enrichment': {
u'preload':
u'self:test_namespace, '
u'asn_api:*, '
u'test_resource_id01:interface, '
u'no_data_resource_id:test_namespace, '
u'test_resource_id01:no_data_namespace'
}
}
enrichment_cache = PanoptesEnrichmentCache(self._panoptes_context,
plugin_conf,
self._panoptes_resource)
result01 = {u'heartbeat': {u'timestamp': int(mock_time.return_value)}}
enrichment_data = enrichment_cache.get_enrichment(
u'test_resource_id', u'test_namespace')
self.assertEqual(sorted(list(enrichment_data)), sorted(result01))
enrichment_data = enrichment_cache.get_enrichment(
u'self', u'test_namespace')
self.assertEqual(sorted(list(enrichment_data)), sorted(result01))
result02 = {
u'1209': {
u'ifType': u'l3ipvlan',
u'ifAlias': u'<not set>',
u'ifPhysAddress': u'00:1f:a0:13:8c:15',
u'ifDescr': u'Virtual Ethernet 209',
u'ifName': u'Virtual Ethernet 209',
u'ifMtu': u'1500'
},
u'1226': {
u'ifType': u'l3ipvlan',
u'ifAlias': u'<not set>',
u'ifPhysAddress': u'00:1f:a0:13:8c:16',
u'ifDescr': u'Virtual Ethernet 226',
u'ifName': u'Virtual Ethernet 226',
u'ifMtu': u'1500'
}
}
enrichment_data = enrichment_cache.get_enrichment(
u'test_resource_id01', u'interface')
self.assertEqual(sorted(list(enrichment_data)), sorted(result02))
with self.assertRaises(PanoptesEnrichmentCacheError):
enrichment_cache.get_enrichment(u'no_data_resource_id',
u'test_namespace')
with self.assertRaises(PanoptesEnrichmentCacheError):
enrichment_cache.get_enrichment(u'test_resource_id01',
u'no_data_namespace')
with self.assertRaises(AssertionError):
enrichment_cache.get_enrichment(u'test_resource_id01', u'')
with self.assertRaises(AssertionError):
enrichment_cache.get_enrichment(u'', u'test_namespace')
with self.assertRaises(AssertionError):
enrichment_cache.get_enrichment(u'', u'*')
# Test fallback preload
result03 = {
u'2209': {
u'ifType': u'l3ipvlan',
u'ifAlias': u'<not set>',
u'ifPhysAddress': u'00:1f:a0:13:8c:15',
u'ifDescr': u'Virtual Ethernet 309',
u'ifName': u'Virtual Ethernet 309',
u'ifMtu': u'1500'
},
u'2226': {
u'ifType': u'l3ipvlan',
u'ifAlias': u'<not set>',
u'ifPhysAddress': u'00:1f:a0:13:8c:16',
u'ifDescr': u'Virtual Ethernet 326',
u'ifName': u'Virtual Ethernet 326',
u'ifMtu': u'1500'
}
}
enrichment_data03 = enrichment_cache.get_enrichment(
u'test_resource_id02', u'interface')
self.assertEqual(sorted(list(enrichment_data03)), sorted(result03))
def test_enrichment_cache_get_enrichment_value(self):
"""Test enrichment resource get_enrichment_value"""
plugin_conf = {
u'Core': {
u'name': u'Test Plugin 01'
},
u'enrichment': {
u'preload':
u'self:test_namespace, '
u'asn_api:*, '
u'test_resource_id01:interface, '
u'no_data_resource_id:test_namespace, '
u'test_resource_id01:no_data_namespace'
}
}
enrichment_cache = PanoptesEnrichmentCache(self._panoptes_context,
plugin_conf,
self._panoptes_resource)
result01 = {u'timestamp': int(mock_time.return_value)}
enrich_data = enrichment_cache.get_enrichment_value(
u'self', u'test_namespace', u'heartbeat')
self.assertEqual(sorted(result01), sorted(enrich_data))
enrich_data = enrichment_cache.get_enrichment_value(
u'test_resource_id', u'test_namespace', u'heartbeat')
self.assertEqual(sorted(result01), sorted(enrich_data))
result02 = {u'data': 101}
enrich_data = enrichment_cache.get_enrichment_value(
u'asn_api', u'namespace01', u'asn_data01')
self.assertEqual(sorted(result02), sorted(enrich_data))
result03 = {
u'ifType': u'l3ipvlan',
u'ifAlias': u'<not set>',
u'ifPhysAddress': u'00:1f:a0:13:8c:15',
u'ifDescr': u'Virtual Ethernet 209',
u'ifName': u'Virtual Ethernet 209',
u'ifMtu': u'1500'
}
enrich_data = enrichment_cache.get_enrichment_value(
u'test_resource_id01', u'interface', u'1209')
self.assertEqual(sorted(result03), sorted(enrich_data))
with self.assertRaises(PanoptesEnrichmentCacheError):
enrichment_cache.get_enrichment_value(u'no_data_resource_id',
u'test_namespace',
u'no_data')
with self.assertRaises(AssertionError):
enrichment_cache.get_enrichment_value(u'no_data_resource_id',
u'test_namespace', u'')
with self.assertRaises(AssertionError):
enrichment_cache.get_enrichment_value(u'no_data_resource_id', u'',
u'no_data')
with self.assertRaises(AssertionError):
enrichment_cache.get_enrichment_value(u'', u'test_namespace',
u'no_data')
# Test fallback preload
result04 = {
u'ifType': u'l3ipvlan',
u'ifAlias': u'<not set>',
u'ifPhysAddress': u'00:1f:a0:13:8c:15',
u'ifDescr': u'Virtual Ethernet 309',
u'ifName': u'Virtual Ethernet 309',
u'ifMtu': u'1500'
}
enrich_data = enrichment_cache.get_enrichment_value(
u'test_resource_id02', u'interface', u'2209')
self.assertEqual(sorted(result04), sorted(enrich_data))
def test_enrichment_cache_get_enrichment_keys(self):
"""Test enrichment resource get_enrichment_keys"""
plugin_conf = {
u'Core': {
u'name': u'Test Plugin 01'
},
u'enrichment': {
u'preload':
u'self:test_namespace, '
u'asn_api:*, '
u'test_resource_id01:interface, '
u'no_data_resource_id:test_namespace, '
u'test_resource_id01:no_data_namespace'
}
}
enrichment_cache = PanoptesEnrichmentCache(self._panoptes_context,
plugin_conf,
self._panoptes_resource)
result01 = [u'1226', u'1209']
enrichment_data = enrichment_cache.get_enrichment_keys(
u'test_resource_id01', u'interface')
self.assertListEqual(enrichment_data, result01)
with self.assertRaises(PanoptesEnrichmentCacheError):
enrichment_cache.get_enrichment_keys(u'test_resource_id01',
u'no_data_namespace')
result02 = [u'heartbeat']
enrichment_data = enrichment_cache.get_enrichment_keys(
u'test_resource_id', u'test_namespace')
self.assertListEqual(enrichment_data, result02)
enrichment_data = enrichment_cache.get_enrichment_keys(
u'self', u'test_namespace')
self.assertListEqual(enrichment_data, result02)
# Test fallback preload
result03 = [u'2226', u'2209']
enrichment_data03 = enrichment_cache.get_enrichment_keys(
u'test_resource_id02', u'interface')
self.assertListEqual(enrichment_data03, result03)
class SNMPPluginTestFramework(object):
plugin_class = None
path = None
snmp_host = u'127.0.0.1'
snmp_port = 10161
snmp_timeout = 10
snmp_retries = 1
snmp_max_repetitions = 10
snmp_community = 'public'
snmp_failure_timeout = 1
x509_secured_requests = 0
x509_key_location = u'/home/panoptes/x509/keys'
x509_key_filename = u'panoptes.key'
x509_cert_location = u'/home/panoptes/x509/keys'
x509_cert_filename = u'panoptes.pem'
resource_id = u'test_id'
resource_endpoint = u'127.0.0.1'
resource_plugin = u'dummy'
resource_site = u'test_site'
resource_class = u'network'
resource_subclass = u'test_subclass'
resource_type = u'test_type'
resource_backplane = None
resource_model = u'model'
results_data_file = u'results.json'
enrichment_data_file = u'enrichment_data'
use_enrichment = True
plugin_conf = {
'Core': {
'name': 'Test Plugin',
'module': 'test_plugin'
},
'main': {
'execute_frequency': '60',
'enrichment_ttl': '300',
'resource_filter': 'resource_class = "network"'
},
'snmp': {
'timeout': 10,
'retries': 1,
'non_repeaters': 1,
'max_repetitions': 25,
},
'x509': {
'x509_secured_requests': 0,
'x509_key_location': '/home/panoptes/x509/keys',
'x509_key_filename': 'panoptes.key',
'x509_cert_location': '/home/panoptes/x509/keys',
'x509_cert_filename': 'panoptes.pem'
}
}
def __init__(self, test_name):
super(SNMPPluginTestFramework, self).__init__(test_name)
self._path = self.path
self._plugin_conf = self.plugin_conf
self._snmp_host = self.snmp_host
self._snmp_port = self.snmp_port
self._snmp_timeout = self.snmp_timeout
self._snmp_retries = self.snmp_retries
self._snmp_max_repetitions = self.snmp_max_repetitions
self._snmp_community = self.snmp_community
self._snmp_failure_timeout = self.snmp_failure_timeout
self._x509_secured_requests = self.x509_secured_requests
self._x509_cert_location = self.x509_cert_location
self._x509_cert_filename = self.x509_cert_filename
self._x509_key_location = self.x509_key_location
self._x509_key_filename = self.x509_key_filename
self._resource_id = self.resource_id
self._resource_endpoint = self.resource_endpoint
self._resource_plugin = self.resource_plugin
self._resource_site = self.resource_site
self._resource_class = self.resource_class
self._resource_subclass = self.resource_subclass
self._resource_type = self.resource_type
self._resource_backplane = self.resource_backplane
self._resource_model = self.resource_model
self._panoptes_context = None
self._panoptes_resource = None
self._snmp_conf = None
self._x509_conf = None
self._expected_results = None
self._results_data_file = None
self._enrichment_data_file = None
self._secret_store = None
self._plugin_context = None
self._plugin_context_bad = None
self._snmp_conf_bad = None
self._x509_conf_bad = None
self._enrichment_kv = None
self._enrichment_cache = None
def set_panoptes_context(self):
panoptes_test_conf_file = os.path.join(
os.path.dirname(os.path.dirname(__file__)),
'config_files/test_panoptes_config.ini')
self._panoptes_context = PanoptesContext(
panoptes_test_conf_file,
key_value_store_class_list=[PanoptesEnrichmentCacheKeyValueStore])
def set_panoptes_resource(self):
self._panoptes_resource = PanoptesResource(
resource_site=self._resource_site,
resource_class=self._resource_class,
resource_subclass=self._resource_subclass,
resource_type=self._resource_type,
resource_id=self._resource_id,
resource_endpoint=self._resource_endpoint,
resource_plugin=self._resource_plugin)
self._panoptes_resource.resource_metadata[
'model'] = self._resource_model
if self._resource_backplane:
self._panoptes_resource.add_metadata('backplane',
self._resource_backplane)
def set_enrichment_cache(self, resource_endpoint=None):
if self.use_enrichment:
self._enrichment_data_file = 'data/' + self.enrichment_data_file
self._enrichment_kv = self._panoptes_context.get_kv_store(
PanoptesEnrichmentCacheKeyValueStore)
enrichment_data_file = os.path.join(os.path.abspath(self.path),
self._enrichment_data_file)
if self._plugin_conf.get('enrichment'):
if self._plugin_conf['enrichment'].get('preload'):
self._enrichment_cache = PanoptesEnrichmentCache(
self._panoptes_context, self._plugin_conf,
self._panoptes_resource)
try:
with open(enrichment_data_file) as enrichment_data:
for line in enrichment_data:
key, value = line.strip().split('=>')
if resource_endpoint is not None:
value = self.update_enrichment_ip(
value, resource_endpoint)
self._enrichment_kv.set(key, value)
except Exception as e:
raise IOError(
'Failed to load enrichment data file {}: {}'.format(
enrichment_data_file, repr(e)))
def set_snmp_conf(self):
self._snmp_conf = self._panoptes_context.config_object.snmp_defaults
def set_x509_conf(self):
self._x509_conf = {
'x509_secured_requests': self._x509_secured_requests,
'x509_cert_location': self._x509_cert_location,
'x509_cert_filename': self._x509_cert_filename,
'x509_key_location': self._x509_key_location,
'x509_key_filename': self._x509_key_filename
}
def set_secret_store(self):
self._secret_store = create_autospec(PanoptesSecretsStore,
instance=True,
spec_set=True)
self._secret_store.get_by_site.return_value = self._snmp_community
def set_plugin_context(self):
self._plugin_context = create_autospec(
PanoptesPluginWithEnrichmentContext,
instance=True,
spec_set=True,
data=self._panoptes_resource,
enrichment=self._enrichment_cache,
config=self._plugin_conf,
snmp=self._snmp_conf,
x509=self._x509_conf,
secrets=self._secret_store,
logger=logging.getLogger(__name__))
def set_x509_conf_bad(self):
self._x509_conf_bad = copy.copy(self._x509_conf)
self._x509_conf_bad['x509_secured_requests'] = 5
def set_snmp_conf_bad(self):
self._snmp_conf_bad = copy.copy(self._snmp_conf)
self._snmp_conf_bad['port'] += 1
self._snmp_conf_bad['timeout'] = self._snmp_failure_timeout
def set_plugin_context_bad(self):
self._plugin_context_bad = create_autospec(
PanoptesPluginWithEnrichmentContext,
instance=True,
spec_set=True,
data=self._panoptes_resource,
enrichment=self._enrichment_cache,
config=self._plugin_conf,
snmp=self._snmp_conf_bad,
x509=self._x509_conf,
secrets=self._secret_store,
logger=logging.getLogger(__name__))
def set_expected_results(self):
self._results_data_file = 'data/' + self.results_data_file
expected_result_file = os.path.join(os.path.abspath(self.path),
self._results_data_file)
self._expected_results = json.load(open(expected_result_file))
def update_enrichment_ip(self, enrichment, resource_endpoint):
"""
The PluginPollingGenericSNMPMetrics Plugin initially calls the function _get_config() when started.
This function loads the enrichment from the PanoptesEnrichmentCacheKeyValueStore
using the resource_id, namespace, and enrichment key. The enrichment key is the resource_endpoint.
no_service_active tests to ensure that a proper error (timeout -> ping error) is returned when
snmp_bulk is called with an invalid IP. In order to get to this stage in the tests the enrichment must
loaded in the generic snmp runner. In order to test this, the resource_endpoint is changed to
be invalid. However this IP must also be changed in the test enrichment file so the generic snmp runner
is able to load the enrichment from the cache.
"""
modified_enrichment = json.loads(enrichment)
if len(modified_enrichment) == 0 or len(
list(modified_enrichment['data'].keys())) == 0:
return enrichment
ip = list(modified_enrichment['data'].keys())[0]
if re.search(r'\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b', ip):
modified_enrichment['data'][
resource_endpoint] = modified_enrichment['data'][ip]
del modified_enrichment['data'][ip]
return json.dumps(modified_enrichment)
return enrichment
@patch('time.time', mock_time)
@patch('yahoo_panoptes.framework.resources.time', mock_time)
@patch('redis.StrictRedis', PanoptesMockRedis)
def setUp(self):
self.set_panoptes_context()
self.set_panoptes_resource()
self.set_enrichment_cache()
self.set_snmp_conf()
self.set_x509_conf()
self.set_secret_store()
self.set_plugin_context()
self.set_snmp_conf_bad()
self.set_x509_conf_bad()
self.set_plugin_context_bad()
self.set_expected_results()
class TestPanoptesEnrichmentCacheStore(unittest.TestCase):
@patch('redis.StrictRedis', PanoptesMockRedis)
@patch('time.time', mock_time)
def setUp(self):
self.my_dir, self.panoptes_test_conf_file = _get_test_conf_file()
self._panoptes_context = PanoptesContext(
self.panoptes_test_conf_file,
key_value_store_class_list=[PanoptesEnrichmentCacheKeyValueStore])
self._enrichment_kv = self._panoptes_context.get_kv_store(
PanoptesEnrichmentCacheKeyValueStore)
self._panoptes_resource = PanoptesResource(
resource_site='test_site',
resource_class='test_class',
resource_subclass='test_subclass',
resource_type='test_type',
resource_id='test_resource_id',
resource_endpoint='test_endpoint',
resource_plugin='test_plugin')
interface_validation_object = PanoptesEnrichmentInterfaceSchemaValidator(
)
neighbor_validation_object = PanoptesEnrichmentNeighborSchemaValidator(
)
enrichment_set1 = PanoptesEnrichmentSet('int_001')
enrichment_set1.add('speed', 1000)
enrichment_set1.add('index', 001)
enrichment_set1.add('status', 'up')
enrichment_set2 = PanoptesEnrichmentSet('int_002')
enrichment_set2.add('speed', 1000)
enrichment_set2.add('index', 002)
enrichment_set2.add('status', 'down')
enrichment_group1 = PanoptesEnrichmentGroup(
'interface', interface_validation_object, 300, 60)
enrichment_group1.add_enrichment_set(enrichment_set1)
enrichment_group1.add_enrichment_set(enrichment_set2)
enrichment_set3 = PanoptesEnrichmentSet('host_name')
enrichment_set3.add('vlan_id', 501)
enrichment_set3.add('property', 'Test Property')
enrichment_set3.add('mac', 'aa:bb:cc:dd:ee:ff')
enrichment_group2 = PanoptesEnrichmentGroup(
'neighbor', neighbor_validation_object, 600, 120)
enrichment_group2.add_enrichment_set(enrichment_set3)
self.enrichment_group_set1 = PanoptesEnrichmentGroupSet(
self._panoptes_resource)
self.enrichment_group_set1.add_enrichment_group(enrichment_group1)
self.enrichment_group_set1.add_enrichment_group(enrichment_group2)
self._panoptes_resource01 = PanoptesResource(
resource_site='test_site',
resource_class='test_class',
resource_subclass='test_subclass',
resource_type='test_type',
resource_id='test_resource_id01',
resource_endpoint='test_endpoint01',
resource_plugin='test_plugin')
enrichment_set4 = PanoptesEnrichmentSet('host_name01')
enrichment_set4.add('vlan_id', 502)
enrichment_set4.add('property', 'Test Property')
enrichment_set4.add('mac', 'aa:bb:cc:dd:ee:ff')
enrichment_group3 = PanoptesEnrichmentGroup(
'neighbor', neighbor_validation_object, 600, 120)
enrichment_group3.add_enrichment_set(enrichment_set3)
enrichment_group3.add_enrichment_set(enrichment_set4)
self.enrichment_group_set2 = PanoptesEnrichmentGroupSet(
self._panoptes_resource01)
self.enrichment_group_set2.add_enrichment_group(enrichment_group1)
self.enrichment_group_set2.add_enrichment_group(enrichment_group3)
self._multi_enrichment_group_set = PanoptesEnrichmentMultiGroupSet()
self._multi_enrichment_group_set.add_enrichment_group_set(
self.enrichment_group_set1)
self._multi_enrichment_group_set.add_enrichment_group_set(
self.enrichment_group_set2)
@patch('time.time', mock_time)
def test_panoptes_enrichment_set(self):
enrichment_set1 = PanoptesEnrichmentSet('int_001')
enrichment_set1.add('speed', 1000)
enrichment_set1.add('index', 001)
enrichment_set1.add('status', 'up')
self.assertEquals(
enrichment_set1.json(),
'{"int_001": {"index": 1, "speed": 1000, "status": "up"}}')
self.assertEquals(
repr(enrichment_set1),
"PanoptesEnrichmentSet({'int_001': {'status': 'up', 'index': 1, 'speed': 1000}})"
)
enrichment_set2 = PanoptesEnrichmentSet('int_002')
enrichment_set2.add('speed', 1000)
enrichment_set2.add('index', 002)
enrichment_set2.add('status', 'down')
interface_validation_object = PanoptesEnrichmentInterfaceSchemaValidator(
)
enrichment_group1 = PanoptesEnrichmentGroup(
'interface', interface_validation_object, 300, 60)
self.assertFalse(enrichment_set1 == enrichment_group1)
self.assertFalse(enrichment_set1 == enrichment_set2)
@patch('time.time', mock_time)
def test_panoptes_enrichment_group(self):
interface_validation_object = PanoptesEnrichmentInterfaceSchemaValidator(
)
enrichment_group1 = PanoptesEnrichmentGroup(
'interface', interface_validation_object, 300, 60)
self.assertEquals(enrichment_group1.enrichment_schema,
PanoptesEnrichmentInterfaceSchemaValidator.schema)
self.assertEquals(
repr(enrichment_group1),
"PanoptesEnrichmentGroup({{'data': set([]), "
"'namespace': 'interface', "
"'metadata': {{"
"'_enrichment_group_creation_timestamp': {:.5f}, "
"'_enrichment_ttl': 300, "
"'_execute_frequency': 60}}}})".format(mock_time.return_value))
enrichment_set1 = PanoptesEnrichmentSet('int_001')
enrichment_set1.add('speed', 1000)
enrichment_set1.add('index', 001)
enrichment_set1.add('status', 'up')
self.assertFalse(enrichment_group1 == enrichment_set1)
enrichment_group2 = PanoptesEnrichmentGroup(
'interface', interface_validation_object, 300, 60)
enrichment_group3 = PanoptesEnrichmentGroup(
'other_namespace', interface_validation_object, 300, 60)
self.assertTrue(enrichment_group1 == enrichment_group2)
self.assertFalse(enrichment_group1 == enrichment_group3)
@patch('time.time', mock_time)
def test_store_enrichment_data_enrichment_group_set(self):
interface_result_data = \
{
"data": {
"int_001": {
"index": 1,
"speed": 1000,
"status": "up"
},
"int_002": {
"index": 2,
"speed": 1000,
"status": "down"
}
},
"metadata": {
"_enrichment_group_creation_timestamp": mock_time.return_value,
"_enrichment_ttl": 300,
"_execute_frequency": 60
}
}
neighbor_result_data = \
{
"data": {
"host_name": {
"mac": "aa:bb:cc:dd:ee:ff",
"property": "Test Property",
"vlan_id": 501
}
},
"metadata": {
"_enrichment_group_creation_timestamp": mock_time.return_value,
"_enrichment_ttl": 600,
"_execute_frequency": 120
}
}
_store_enrichment_data(self._panoptes_context,
self.enrichment_group_set1,
'PanoptesPluginInfo')
self.assertNotEquals(
ordered(interface_result_data),
ordered(
json.loads(
self._enrichment_kv.get('test_resource_id:neighbor'))))
self.assertEquals(
ordered(interface_result_data),
ordered(
json.loads(
self._enrichment_kv.get('test_resource_id:interface'))))
self.assertEquals(
ordered(neighbor_result_data),
ordered(
json.loads(
self._enrichment_kv.get('test_resource_id:neighbor'))))
def test_store_enrichment_data_enrichment_multi_group_set(self):
enrichment_result_keys = [
'test_resource_id01:interface', 'test_resource_id01:neighbor',
'test_resource_id:interface', 'test_resource_id:neighbor'
]
neighbor_result_data = \
{"data": {"host_name": {"mac": "aa:bb:cc:dd:ee:ff", "property": "Test Property", "vlan_id": 501},
"host_name01": {"mac": "aa:bb:cc:dd:ee:ff", "property": "Test Property", "vlan_id": 502}},
"metadata": {"_enrichment_group_creation_timestamp": mock_time.return_value, "_enrichment_ttl": 600,
"_execute_frequency": 120}}
_store_enrichment_data(self._panoptes_context,
self._multi_enrichment_group_set,
'PanoptesPluginInfo')
self.assertEquals(enrichment_result_keys,
self._enrichment_kv.find_keys('*'))
self.assertEquals(
ordered(neighbor_result_data),
ordered(
json.loads(
self._enrichment_kv.get('test_resource_id01:neighbor'))))
self.assertNotEquals(
ordered(neighbor_result_data),
ordered(
json.loads(
self._enrichment_kv.get('test_resource_id01:interface'))))
class TestPanoptesEnrichmentCache(unittest.TestCase):
@patch('redis.StrictRedis', PanoptesMockRedis)
def setUp(self):
self.my_dir, self.panoptes_test_conf_file = _get_test_conf_file()
self._panoptes_context = PanoptesContext(
self.panoptes_test_conf_file,
key_value_store_class_list=[PanoptesEnrichmentCacheKeyValueStore])
self._enrichment_kv = self._panoptes_context.get_kv_store(
PanoptesEnrichmentCacheKeyValueStore)
self._panoptes_resource = PanoptesResource(
resource_site='test_site',
resource_class='test_class',
resource_subclass='test_subclass',
resource_type='test_type',
resource_id='test_resource_id',
resource_endpoint='test_endpoint',
resource_plugin='test_plugin')
self._plugin_conf = {
'Core': {
'name': 'Heartbeat Enrichment Plugin',
'module': 'plugin_enrichment_heartbeat'
},
'main': {
'execute_frequency':
'60',
'enrichment_ttl':
'300',
'resource_filter':
'resource_class = "system" AND resource_subclass = '
'"internal" AND resource_type = "heartbeat"'
},
'enrichment': {
'preload': 'self:test_namespace'
}
}
self._enrichment_kv.set(
'test_resource_id:test_namespace',
'{"data": {"heartbeat": {"timestamp": 1521668419}}, '
'"metadata": {"_enrichment_group_creation_timestamp": 1521668419.881953, '
'"_enrichment_ttl": 300, "_execute_frequency": 60}}')
self._enrichment_kv.set(
'test_resource_id01:interface',
'{"data": {"1226": {"ifType": "l3ipvlan", "ifAlias": "<not set>", "ifPhysAddress": '
'"00:1f:a0:13:8c:16", "ifDescr": "Virtual Ethernet 226", "ifName": "Virtual Ethernet 226", '
'"ifMtu": "1500"}, "1209": {"ifType": "l3ipvlan", "ifAlias": "<not set>", "ifPhysAddress": '
'"00:1f:a0:13:8c:15", "ifDescr": "Virtual Ethernet 209", "ifName": "Virtual Ethernet 209", '
'"ifMtu": "1500"}}, "metadata": {"_enrichment_group_creation_timestamp": 1521668419.881953, '
'"_enrichment_ttl": 300, "_execute_frequency": 60}}')
self._enrichment_kv.set(
'test_resource_id02:interface',
'{"data": {"2226": {"ifType": "l3ipvlan", "ifAlias": "<not set>", "ifPhysAddress": '
'"00:1f:a0:13:8c:16", "ifDescr": "Virtual Ethernet 326", "ifName": "Virtual Ethernet 326", '
'"ifMtu": "1500"}, "2209": {"ifType": "l3ipvlan", "ifAlias": "<not set>", "ifPhysAddress": '
'"00:1f:a0:13:8c:15", "ifDescr": "Virtual Ethernet 309", "ifName": "Virtual Ethernet 309", '
'"ifMtu": "1500"}}, "metadata": {"_enrichment_group_creation_timestamp": 1521668419.881953, '
'"_enrichment_ttl": 300, "_execute_frequency": 60}}')
self._enrichment_kv.set(
'asn_api:namespace01', '{"data": {"asn_data01": {"data": 101}}, '
'"metadata": {"_enrichment_group_creation_timestamp": 1521668419.881953, '
'"_enrichment_ttl": 300, "_execute_frequency": 60}}')
self._enrichment_kv.set(
'asn_api:namespace02', '{"data": {"asn_data02": {"data": 202}}, '
'"metadata": {"_enrichment_group_creation_timestamp": 1521668419.881953, '
'"_enrichment_ttl": 300, "_execute_frequency": 60}}')
self._enrichment_kv.set(
'asn_api:namespace03', '{"data": {"asn_data03": {"data": 303}}, '
'"metadata": {"_enrichment_group_creation_timestamp": 1521668419.881953, '
'"_enrichment_ttl": 300, "_execute_frequency": 60}}')
self._enrichment_kv.set('enrichment:asn_api:namespace05', 'bad_data')
def test_enrichment_cache(self):
"""Test enrichment resource attributes"""
with self.assertRaises(AssertionError):
PanoptesEnrichmentCache('non_panoptes_context', self._plugin_conf,
self._panoptes_resource)
with self.assertRaises(AssertionError):
PanoptesEnrichmentCache(self._panoptes_context, 'non_plugin_conf',
self._panoptes_resource)
with self.assertRaises(AssertionError):
PanoptesEnrichmentCache(self._panoptes_context, self._plugin_conf,
'non_panoptes_resource')
def test_enrichment_cache_preload01(self):
"""Test enrichment resource with preload conf self:test_namespace"""
result = '{"test_resource_id": {"test_namespace": {"heartbeat": {"timestamp": 1521668419}}}}'
enrichment_cache = PanoptesEnrichmentCache(self._panoptes_context,
self._plugin_conf,
self._panoptes_resource)
self.assertEqual(
ordered(enrichment_cache.__dict__['_enrichment_data']),
ordered(json.loads(result)))
def test_enrichment_cache_preload02(self):
"""Test enrichment resource with preload conf"""
plugin_conf = {
'Core': {
'name': 'Test Plugin 01'
},
'enrichment': {
'preload':
'self:test_namespace, '
'asn_api:*, '
'test_resource_id01:interface, '
'no_data_resource_id:test_namespace, '
'test_resource_id01:no_data_namespace'
}
}
results01 = """{"asn_api": {"namespace02": {"asn_data02": {"data": 202}}, "namespace03": {"asn_data03":
{"data": 303}}, "namespace01": {"asn_data01": {"data": 101}}}, "test_resource_id": {"test_namespace":
{"heartbeat": {"timestamp": 1521668419}}}, "test_resource_id01": {"interface": {"1226": {"ifType": "l3ipvlan",
"ifAlias": "<not set>", "ifPhysAddress": "00:1f:a0:13:8c:16", "ifDescr": "Virtual Ethernet 226", "ifName":
"Virtual Ethernet 226", "ifMtu": "1500"}, "1209": {"ifType": "l3ipvlan", "ifAlias": "<not set>",
"ifPhysAddress": "00:1f:a0:13:8c:15", "ifDescr": "Virtual Ethernet 209", "ifName": "Virtual Ethernet 209",
"ifMtu": "1500"}}}}"""
enrichment_cache = PanoptesEnrichmentCache(self._panoptes_context,
plugin_conf,
self._panoptes_resource)
self.assertEqual(
ordered(enrichment_cache.__dict__['_enrichment_data']),
ordered(json.loads(results01)))
def test_enrichment_cache_parse_conf(self):
"""Test enrichment resource parse conf"""
result01 = [('asn_api', '*'), ('self', 'test_namespace'),
('test_resource_id01', 'interface')]
plugin_conf = {
'Core': {
'name': 'Test Plugin 01'
},
'enrichment': {
'preload':
'self:test_namespace, asn_api:*, test_resource_id01:interface'
}
}
enrichment_cache = PanoptesEnrichmentCache(self._panoptes_context,
plugin_conf,
self._panoptes_resource)
self.assertEqual(sorted(result01),
sorted(enrichment_cache.__dict__['_preload_conf']))
plugin_conf_with_dup = {
'Core': {
'name': 'Test Plugin 01'
},
'enrichment': {
'preload':
'self:test_namespace, asn_api:*, '
'test_resource_id01:interface, asn_api:*, '
'test_resource_id01:interface'
}
}
enrichment_cache = PanoptesEnrichmentCache(self._panoptes_context,
plugin_conf_with_dup,
self._panoptes_resource)
self.assertEqual(sorted(result01),
sorted(enrichment_cache.__dict__['_preload_conf']))
def test_enrichment_cache_parse_conf_bad(self):
"""Test enrichment resource parse bad conf"""
plugin_conf01 = {'Core': {'name': 'Test Plugin 01'}, 'enrichment': {}}
with self.assertRaises(TypeError):
PanoptesEnrichmentCache(self._panoptes_context, plugin_conf01,
self._panoptes_resource)
plugin_conf02 = {
'Core': {
'name': 'Test Plugin 01'
},
'enrichment': {
'preload': {}
}
}
with self.assertRaises(TypeError):
PanoptesEnrichmentCache(self._panoptes_context, plugin_conf02,
self._panoptes_resource)
plugin_conf03 = {
'Core': {
'name': 'Test Plugin 01'
},
'enrichment': {
'preload': {'self:test_namespace, asn_api:'}
}
}
with self.assertRaises(TypeError):
PanoptesEnrichmentCache(self._panoptes_context, plugin_conf03,
self._panoptes_resource)
plugin_conf04 = {
'Core': {
'name': 'Test Plugin 01'
},
'enrichment': {
'preload': {'self:test_namespace, :'}
}
}
with self.assertRaises(TypeError):
PanoptesEnrichmentCache(self._panoptes_context, plugin_conf04,
self._panoptes_resource)
plugin_conf05 = {
'Core': {
'name': 'Test Plugin 01'
},
'enrichment': {
'preload': {'self:test_namespace, '}
}
}
with self.assertRaises(TypeError):
PanoptesEnrichmentCache(self._panoptes_context, plugin_conf05,
self._panoptes_resource)
plugin_conf06 = {
'Core': {
'name': 'Test Plugin 01'
},
'enrichment': {
'preload': {'self:test_namespace, :namespace01'}
}
}
with self.assertRaises(TypeError):
PanoptesEnrichmentCache(self._panoptes_context, plugin_conf06,
self._panoptes_resource)
def test_enrichment_cache_get_enrichment(self):
"""Test enrichment resource get"""
plugin_conf = {
'Core': {
'name': 'Test Plugin 01'
},
'enrichment': {
'preload':
'self:test_namespace, '
'asn_api:*, '
'test_resource_id01:interface, '
'no_data_resource_id:test_namespace, '
'test_resource_id01:no_data_namespace'
}
}
enrichment_cache = PanoptesEnrichmentCache(self._panoptes_context,
plugin_conf,
self._panoptes_resource)
result01 = {u'heartbeat': {u'timestamp': 1521668419}}
enrichment_data = enrichment_cache.get_enrichment(
'test_resource_id', 'test_namespace')
self.assertEqual(sorted(list(enrichment_data)), sorted(result01))
enrichment_data = enrichment_cache.get_enrichment(
'self', 'test_namespace')
self.assertEqual(sorted(list(enrichment_data)), sorted(result01))
result02 = {
u'1209': {
u'ifType': u'l3ipvlan',
u'ifAlias': u'<not set>',
u'ifPhysAddress': u'00:1f:a0:13:8c:15',
u'ifDescr': u'Virtual Ethernet 209',
u'ifName': u'Virtual Ethernet 209',
u'ifMtu': u'1500'
},
u'1226': {
u'ifType': u'l3ipvlan',
u'ifAlias': u'<not set>',
u'ifPhysAddress': u'00:1f:a0:13:8c:16',
u'ifDescr': u'Virtual Ethernet 226',
u'ifName': u'Virtual Ethernet 226',
u'ifMtu': u'1500'
}
}
enrichment_data = enrichment_cache.get_enrichment(
'test_resource_id01', 'interface')
self.assertEqual(sorted(list(enrichment_data)), sorted(result02))
with self.assertRaises(PanoptesEnrichmentCacheError):
enrichment_cache.get_enrichment('no_data_resource_id',
'test_namespace')
with self.assertRaises(PanoptesEnrichmentCacheError):
enrichment_cache.get_enrichment('test_resource_id01',
'no_data_namespace')
with self.assertRaises(AssertionError):
enrichment_cache.get_enrichment('test_resource_id01', '')
with self.assertRaises(AssertionError):
enrichment_cache.get_enrichment('', 'test_namespace')
with self.assertRaises(AssertionError):
enrichment_cache.get_enrichment('', '*')
# Test fallback preload
result03 = {
u'2209': {
u'ifType': u'l3ipvlan',
u'ifAlias': u'<not set>',
u'ifPhysAddress': u'00:1f:a0:13:8c:15',
u'ifDescr': u'Virtual Ethernet 309',
u'ifName': u'Virtual Ethernet 309',
u'ifMtu': u'1500'
},
u'2226': {
u'ifType': u'l3ipvlan',
u'ifAlias': u'<not set>',
u'ifPhysAddress': u'00:1f:a0:13:8c:16',
u'ifDescr': u'Virtual Ethernet 326',
u'ifName': u'Virtual Ethernet 326',
u'ifMtu': u'1500'
}
}
enrichment_data03 = enrichment_cache.get_enrichment(
'test_resource_id02', 'interface')
self.assertEqual(sorted(list(enrichment_data03)), sorted(result03))
def test_enrichment_cache_get_enrichment_value(self):
"""Test enrichment resource get_enrichment_value"""
plugin_conf = {
'Core': {
'name': 'Test Plugin 01'
},
'enrichment': {
'preload':
'self:test_namespace, '
'asn_api:*, '
'test_resource_id01:interface, '
'no_data_resource_id:test_namespace, '
'test_resource_id01:no_data_namespace'
}
}
enrichment_cache = PanoptesEnrichmentCache(self._panoptes_context,
plugin_conf,
self._panoptes_resource)
result01 = {u'timestamp': 1521668419}
enrich_data = enrichment_cache.get_enrichment_value(
'self', 'test_namespace', 'heartbeat')
self.assertEqual(sorted(result01), sorted(enrich_data))
enrich_data = enrichment_cache.get_enrichment_value(
'test_resource_id', 'test_namespace', 'heartbeat')
self.assertEqual(sorted(result01), sorted(enrich_data))
result02 = {u'data': 101}
enrich_data = enrichment_cache.get_enrichment_value(
'asn_api', 'namespace01', 'asn_data01')
self.assertEqual(sorted(result02), sorted(enrich_data))
result03 = {
u'ifType': u'l3ipvlan',
u'ifAlias': u'<not set>',
u'ifPhysAddress': u'00:1f:a0:13:8c:15',
u'ifDescr': u'Virtual Ethernet 209',
u'ifName': u'Virtual Ethernet 209',
u'ifMtu': u'1500'
}
enrich_data = enrichment_cache.get_enrichment_value(
'test_resource_id01', 'interface', '1209')
self.assertEqual(sorted(result03), sorted(enrich_data))
with self.assertRaises(PanoptesEnrichmentCacheError):
enrichment_cache.get_enrichment_value('no_data_resource_id',
'test_namespace', 'no_data')
with self.assertRaises(AssertionError):
enrichment_cache.get_enrichment_value('no_data_resource_id',
'test_namespace', '')
with self.assertRaises(AssertionError):
enrichment_cache.get_enrichment_value('no_data_resource_id', '',
'no_data')
with self.assertRaises(AssertionError):
enrichment_cache.get_enrichment_value('', 'test_namespace',
'no_data')
# Test fallback preload
result04 = {
u'ifType': u'l3ipvlan',
u'ifAlias': u'<not set>',
u'ifPhysAddress': u'00:1f:a0:13:8c:15',
u'ifDescr': u'Virtual Ethernet 309',
u'ifName': u'Virtual Ethernet 309',
u'ifMtu': u'1500'
}
enrich_data = enrichment_cache.get_enrichment_value(
'test_resource_id02', 'interface', '2209')
self.assertEqual(sorted(result04), sorted(enrich_data))
def test_enrichment_cache_get_enrichment_keys(self):
"""Test enrichment resource get_enrichment_keys"""
plugin_conf = {
'Core': {
'name': 'Test Plugin 01'
},
'enrichment': {
'preload':
'self:test_namespace, '
'asn_api:*, '
'test_resource_id01:interface, '
'no_data_resource_id:test_namespace, '
'test_resource_id01:no_data_namespace'
}
}
enrichment_cache = PanoptesEnrichmentCache(self._panoptes_context,
plugin_conf,
self._panoptes_resource)
result01 = [u'1226', u'1209']
enrichment_data = enrichment_cache.get_enrichment_keys(
'test_resource_id01', 'interface')
self.assertListEqual(enrichment_data, result01)
with self.assertRaises(PanoptesEnrichmentCacheError):
enrichment_cache.get_enrichment_keys('test_resource_id01',
'no_data_namespace')
result02 = [u'heartbeat']
enrichment_data = enrichment_cache.get_enrichment_keys(
'test_resource_id', 'test_namespace')
self.assertListEqual(enrichment_data, result02)
# Test fallback preload
result03 = [u'2226', u'2209']
enrichment_data03 = enrichment_cache.get_enrichment_keys(
'test_resource_id02', 'interface')
self.assertListEqual(enrichment_data03, result03)