class SNMPPluginTestFramework(object): plugin_class = None path = None snmp_host = '127.0.0.1' snmp_port = 10161 snmp_timeout = 10 snmp_retries = 1 snmp_max_repetitions = 10 snmp_community = 'public' snmp_failure_timeout = 1 resource_id = 'test_id' resource_endpoint = '127.0.0.1' resource_plugin = 'dummy' resource_site = 'test_site' resource_class = 'network' resource_subclass = 'test_subclass' resource_type = 'test_type' resource_backplane = None resource_model = 'model' results_data_file = 'results.json' enrichment_data_file = 'enrichment_data' use_enrichment = True plugin_conf = { 'Core': { 'name': 'Test Plugin', 'module': 'test_plugin' }, 'main': { 'execute_frequency': '60', 'enrichment_ttl': '300', 'resource_filter': 'resource_class = "network"' }, 'snmp': { 'timeout': 10, 'retries': 1, 'non_repeaters': 1, 'max_repetitions': 25, } } def __init__(self, test_name): super(SNMPPluginTestFramework, self).__init__(test_name) self._path = self.path self._plugin_conf = self.plugin_conf self._snmp_host = self.snmp_host self._snmp_port = self.snmp_port self._snmp_timeout = self.snmp_timeout self._snmp_retries = self.snmp_retries self._snmp_max_repetitions = self.snmp_max_repetitions self._snmp_community = self.snmp_community self._snmp_failure_timeout = self.snmp_failure_timeout self._resource_id = self.resource_id self._resource_endpoint = self.resource_endpoint self._resource_plugin = self.resource_plugin self._resource_site = self.resource_site self._resource_class = self.resource_class self._resource_subclass = self.resource_subclass self._resource_type = self.resource_type self._resource_backplane = self.resource_backplane self._resource_model = self.resource_model self._panoptes_context = None self._panoptes_resource = None self._snmp_conf = None self._expected_results = None self._results_data_file = None self._enrichment_data_file = None self._secret_store = None self._plugin_context = None self._plugin_context_bad = None self._snmp_conf_bad = None self._enrichment_kv = None self._enrichment_cache = None def set_panoptes_context(self): panoptes_test_conf_file = os.path.join( os.path.dirname(os.path.dirname(__file__)), 'config_files/test_panoptes_config.ini') self._panoptes_context = PanoptesContext( panoptes_test_conf_file, key_value_store_class_list=[PanoptesEnrichmentCacheKeyValueStore]) def set_panoptes_resource(self): self._panoptes_resource = PanoptesResource( resource_site=self._resource_site, resource_class=self._resource_class, resource_subclass=self._resource_subclass, resource_type=self._resource_type, resource_id=self._resource_id, resource_endpoint=self._resource_endpoint, resource_plugin=self._resource_plugin) self._panoptes_resource.resource_metadata['model'] = 'model' if self._resource_backplane: self._panoptes_resource.add_metadata('backplane', self._resource_backplane) def set_enrichment_cache(self): if self.use_enrichment: self._enrichment_data_file = 'data/' + self.enrichment_data_file self._enrichment_kv = self._panoptes_context.get_kv_store( PanoptesEnrichmentCacheKeyValueStore) enrichment_data_file = os.path.join(os.path.abspath(self.path), self._enrichment_data_file) if self._plugin_conf.get('enrichment'): if self._plugin_conf['enrichment'].get('preload'): self._enrichment_cache = PanoptesEnrichmentCache( self._panoptes_context, self._plugin_conf, self._panoptes_resource) try: with open(enrichment_data_file) as enrichment_data: for line in enrichment_data: key, value = line.strip().split('=>') self._enrichment_kv.set(key, value) except Exception as e: raise IOError( 'Failed to load enrichment data file {}: {}'.format( enrichment_data_file, repr(e))) def set_snmp_conf(self): self._snmp_conf = self._panoptes_context.config_object.snmp_defaults def set_secret_store(self): self._secret_store = create_autospec(PanoptesSecretsStore, instance=True, spec_set=True) self._secret_store.get_by_site.return_value = self._snmp_community def set_plugin_context(self): self._plugin_context = create_autospec( PanoptesPluginWithEnrichmentContext, instance=True, spec_set=True, data=self._panoptes_resource, enrichment=self._enrichment_cache, config=self._plugin_conf, snmp=self._snmp_conf, secrets=self._secret_store, logger=logging.getLogger(__name__)) def set_snmp_conf_bad(self): self._snmp_conf_bad = copy.copy(self._snmp_conf) self._snmp_conf_bad['port'] += 1 self._snmp_conf_bad['timeout'] = self._snmp_failure_timeout def set_plugin_context_bad(self): self._plugin_context_bad = create_autospec( PanoptesPluginWithEnrichmentContext, instance=True, spec_set=True, data=self._panoptes_resource, enrichment=self._enrichment_cache, config=self._plugin_conf, snmp=self._snmp_conf_bad, secrets=self._secret_store, logger=logging.getLogger(__name__)) def set_expected_results(self): self._results_data_file = 'data/' + self.results_data_file expected_result_file = os.path.join(os.path.abspath(self.path), self._results_data_file) self._expected_results = json.load(open(expected_result_file)) @patch('time.time', mock_time) @patch('yahoo_panoptes.framework.resources.time', mock_time) @patch('redis.StrictRedis', PanoptesMockRedis) def setUp(self): self.set_panoptes_context() self.set_panoptes_resource() self.set_enrichment_cache() self.set_snmp_conf() self.set_secret_store() self.set_plugin_context() self.set_snmp_conf_bad() self.set_plugin_context_bad() self.set_expected_results()
class TestPanoptesEnrichmentCacheStore(unittest.TestCase): @patch(u'redis.StrictRedis', PanoptesMockRedis) @patch(u'time.time', mock_time) def setUp(self): self.my_dir, self.panoptes_test_conf_file = _get_test_conf_file() self._panoptes_context = PanoptesContext(self.panoptes_test_conf_file, key_value_store_class_list=[PanoptesEnrichmentCacheKeyValueStore, PanoptesResourcesKeyValueStore]) self._enrichment_kv = self._panoptes_context.get_kv_store(PanoptesEnrichmentCacheKeyValueStore) self._panoptes_resource = PanoptesResource(resource_site=u'test_site', resource_class=u'test_class', resource_subclass=u'test_subclass', resource_type=u'test_type', resource_id=u'test_resource_id', resource_endpoint=u'test_endpoint', resource_plugin=u'test_plugin') interface_validation_object = PanoptesEnrichmentInterfaceSchemaValidator() neighbor_validation_object = PanoptesEnrichmentNeighborSchemaValidator() enrichment_set1 = PanoptesEnrichmentSet(u'int_001') enrichment_set1.add(u'speed', 1000) enrichment_set1.add(u'index', 0o01) enrichment_set1.add(u'status', u'up') enrichment_set2 = PanoptesEnrichmentSet(u'int_002') enrichment_set2.add(u'speed', 1000) enrichment_set2.add(u'index', 0o02) enrichment_set2.add(u'status', u'down') enrichment_group1 = PanoptesEnrichmentGroup(u'interface', interface_validation_object, 300, 60) enrichment_group1.add_enrichment_set(enrichment_set1) enrichment_group1.add_enrichment_set(enrichment_set2) enrichment_set3 = PanoptesEnrichmentSet(u'host_name') enrichment_set3.add(u'vlan_id', 501) enrichment_set3.add(u'property', u'Test Property') enrichment_set3.add(u'mac', u'aa:bb:cc:dd:ee:ff') enrichment_group2 = PanoptesEnrichmentGroup(u'neighbor', neighbor_validation_object, 600, 120) enrichment_group2.add_enrichment_set(enrichment_set3) self.enrichment_group_set1 = PanoptesEnrichmentGroupSet(self._panoptes_resource) self.enrichment_group_set1.add_enrichment_group(enrichment_group1) self.enrichment_group_set1.add_enrichment_group(enrichment_group2) self._panoptes_resource01 = PanoptesResource(resource_site=u'test_site', resource_class=u'test_class', resource_subclass=u'test_subclass', resource_type=u'test_type', resource_id=u'test_resource_id01', resource_endpoint=u'test_endpoint01', resource_plugin=u'test_plugin') enrichment_set4 = PanoptesEnrichmentSet(u'host_name01') enrichment_set4.add(u'vlan_id', 502) enrichment_set4.add(u'property', u'Test Property') enrichment_set4.add(u'mac', u'aa:bb:cc:dd:ee:ff') enrichment_group3 = PanoptesEnrichmentGroup(u'neighbor', neighbor_validation_object, 600, 120) enrichment_group3.add_enrichment_set(enrichment_set3) enrichment_group3.add_enrichment_set(enrichment_set4) self.enrichment_group_set2 = PanoptesEnrichmentGroupSet(self._panoptes_resource01) self.enrichment_group_set2.add_enrichment_group(enrichment_group1) self.enrichment_group_set2.add_enrichment_group(enrichment_group3) self._multi_enrichment_group_set = PanoptesEnrichmentMultiGroupSet() self._multi_enrichment_group_set.add_enrichment_group_set(self.enrichment_group_set1) self._multi_enrichment_group_set.add_enrichment_group_set(self.enrichment_group_set2) @patch(u'time.time', mock_time) def test_panoptes_enrichment_set(self): enrichment_set1 = PanoptesEnrichmentSet(u'int_001') enrichment_set1.add(u'speed', 1000) enrichment_set1.add(u'index', 0o01) enrichment_set1.add(u'status', u'up') self.assertEquals(enrichment_set1.json(), u'{"int_001": {"index": 1, "speed": 1000, "status": "up"}}') self.assertEquals(repr(enrichment_set1), u"PanoptesEnrichmentSet[int_001[index:1,speed:1000,status:up]]") enrichment_set2 = PanoptesEnrichmentSet(u'int_002') enrichment_set2.add(u'speed', 1000) enrichment_set2.add(u'index', 0o02) enrichment_set2.add(u'status', u'down') interface_validation_object = PanoptesEnrichmentInterfaceSchemaValidator() enrichment_group1 = PanoptesEnrichmentGroup(u'interface', interface_validation_object, 300, 60) self.assertFalse(enrichment_set1 == enrichment_group1) self.assertFalse(enrichment_set1 == enrichment_set2) @patch(u'time.time', mock_time) def test_panoptes_enrichment_group(self): interface_validation_object = PanoptesEnrichmentInterfaceSchemaValidator() enrichment_group1 = PanoptesEnrichmentGroup(u'interface', interface_validation_object, 300, 60) self.assertEquals(enrichment_group1.enrichment_schema, PanoptesEnrichmentInterfaceSchemaValidator.schema) self.assertEquals(repr(enrichment_group1), u"PanoptesEnrichmentGroup[namespace:interface," u"enrichment_ttl:300,execute_frequency:60," u"enrichment_group_creation_timestamp:{}]".format( mock_time.return_value)) enrichment_set1 = PanoptesEnrichmentSet(u'int_001') enrichment_set1.add(u'speed', 1000) enrichment_set1.add(u'index', 0o01) enrichment_set1.add(u'status', u'up') self.assertFalse(enrichment_group1 == enrichment_set1) enrichment_group2 = PanoptesEnrichmentGroup(u'interface', interface_validation_object, 300, 60) enrichment_group3 = PanoptesEnrichmentGroup(u'other_namespace', interface_validation_object, 300, 60) self.assertTrue(enrichment_group1 == enrichment_group2) self.assertFalse(enrichment_group1 == enrichment_group3) @patch(u'time.time', mock_time) def test_store_enrichment_data_enrichment_group_set(self): interface_result_data = \ { "data": { "int_001": { "index": 1, "speed": 1000, "status": "up" }, "int_002": { "index": 2, "speed": 1000, "status": "down" } }, "metadata": { "_enrichment_group_creation_timestamp": mock_time.return_value, "_enrichment_ttl": 300, "_execute_frequency": 60 } } neighbor_result_data = \ { "data": { "host_name": { "mac": "aa:bb:cc:dd:ee:ff", "property": "Test Property", "vlan_id": 501 } }, "metadata": { "_enrichment_group_creation_timestamp": mock_time.return_value, "_enrichment_ttl": 600, "_execute_frequency": 120 } } _store_enrichment_data(self._panoptes_context, self.enrichment_group_set1, u'PanoptesPluginInfo') self.assertNotEquals(ordered(interface_result_data), ordered(json.loads(self._enrichment_kv.get(u'test_resource_id:neighbor')))) self.assertEquals(ordered(interface_result_data), ordered(json.loads(self._enrichment_kv.get(u'test_resource_id:interface')))) self.assertEquals(ordered(neighbor_result_data), ordered(json.loads(self._enrichment_kv.get(u'test_resource_id:neighbor')))) def test_store_enrichment_data_enrichment_multi_group_set(self): enrichment_result_keys = [u'test_resource_id01:interface', u'test_resource_id01:neighbor', u'test_resource_id:interface', u'test_resource_id:neighbor'] neighbor_result_data = \ {u"data": {u"host_name": {u"mac": u"aa:bb:cc:dd:ee:ff", u"property": u"Test Property", u"vlan_id": 501}, u"host_name01": {u"mac": u"aa:bb:cc:dd:ee:ff", u"property": u"Test Property", u"vlan_id": 502}}, u"metadata": {u"_enrichment_group_creation_timestamp": mock_time.return_value, u"_enrichment_ttl": 600, u"_execute_frequency": 120}} _store_enrichment_data(self._panoptes_context, self._multi_enrichment_group_set, u'PanoptesPluginInfo') self.assertEquals(enrichment_result_keys, self._enrichment_kv.find_keys(u'*')) self.assertEquals(ordered(neighbor_result_data), ordered(json.loads(self._enrichment_kv.get(u'test_resource_id01:neighbor')))) self.assertNotEquals(ordered(neighbor_result_data), ordered(json.loads(self._enrichment_kv.get(u'test_resource_id01:interface')))) @patch(u'yahoo_panoptes.enrichment.enrichment_plugin_agent.PanoptesPluginWithEnrichmentRunner', create_auto_spec=True) @patch(u'yahoo_panoptes.framework.resources.PanoptesResourceStore.get_resource') @patch(u'yahoo_panoptes.enrichment.enrichment_plugin_agent.PanoptesEnrichmentTaskContext') def test_enrichment_plugin_task_is_executed(self, task_context, resource, enrichment_runner): task_context.return_value = self._panoptes_context resource.return_value = self._panoptes_resource # Test Exception is Thrown on failure to create PanoptesEnrichmentTaskContext task_context.side_effect = Exception() with self.assertRaises(SystemExit): enrichment_plugin_task(u'name', u'key') task_context.side_effect = None # Test Exception is Thrown on failure to create / run plugin enrichment_runner.side_effect = Exception() enrichment_plugin_task(u'name', u'key') enrichment_runner.execute_plugin.assert_not_called() enrichment_runner.side_effect = None # Test Enrichment Is Executed enrichment_plugin_task(u'name', u'key') enrichment_runner.assert_called() enrichment_runner().execute_plugin.assert_called_once()
class TestPanoptesEnrichmentCache(unittest.TestCase): @patch(u'redis.StrictRedis', PanoptesMockRedis) def setUp(self): self.my_dir, self.panoptes_test_conf_file = _get_test_conf_file() self._panoptes_context = PanoptesContext( self.panoptes_test_conf_file, key_value_store_class_list=[PanoptesEnrichmentCacheKeyValueStore]) self._enrichment_kv = self._panoptes_context.get_kv_store( PanoptesEnrichmentCacheKeyValueStore) self._panoptes_resource = PanoptesResource( resource_site=u'test_site', resource_class=u'test_class', resource_subclass=u'test_subclass', resource_type=u'test_type', resource_id=u'test_resource_id', resource_endpoint=u'test_endpoint', resource_plugin=u'test_plugin') self._plugin_conf = { u'Core': { u'name': u'Heartbeat Enrichment Plugin', u'module': u'plugin_enrichment_heartbeat' }, u'main': { u'execute_frequency': u'60', u'enrichment_ttl': u'300', u'resource_filter': u'resource_class = "system" AND resource_subclass = ' u'"internal" AND resource_type = "heartbeat"' }, u'enrichment': { u'preload': u'self:test_namespace' } } self._enrichment_kv.set( u'test_resource_id:test_namespace', u'{{"data": {{"heartbeat": {{"timestamp": 1521668419}}}}, ' u'"metadata": {{"_enrichment_group_creation_timestamp": {:.5f}, ' u'"_enrichment_ttl": 300, "_execute_frequency": 60}}}}'.format( mock_time.return_value)) self._enrichment_kv.set( u'test_resource_id01:interface', u'{{"data": {{"1226": {{"ifType": "l3ipvlan", "ifAlias": "<not set>", "ifPhysAddress": ' u'"00:1f:a0:13:8c:16", "ifDescr": "Virtual Ethernet 226", "ifName": "Virtual Ethernet 226", ' u'"ifMtu": "1500"}}, "1209": {{"ifType": "l3ipvlan", "ifAlias": "<not set>", "ifPhysAddress": ' u'"00:1f:a0:13:8c:15", "ifDescr": "Virtual Ethernet 209", "ifName": "Virtual Ethernet 209", ' u'"ifMtu": "1500"}}}}, "metadata": {{"_enrichment_group_creation_timestamp": {:.5f}, ' u'"_enrichment_ttl": 300, "_execute_frequency": 60}}}}'.format( mock_time.return_value)) self._enrichment_kv.set( u'test_resource_id02:interface', u'{{"data": {{"2226": {{"ifType": "l3ipvlan", "ifAlias": "<not set>", "ifPhysAddress": ' u'"00:1f:a0:13:8c:16", "ifDescr": "Virtual Ethernet 326", "ifName": "Virtual Ethernet 326", ' u'"ifMtu": "1500"}}, "2209": {{"ifType": "l3ipvlan", "ifAlias": "<not set>", "ifPhysAddress": ' u'"00:1f:a0:13:8c:15", "ifDescr": "Virtual Ethernet 309", "ifName": "Virtual Ethernet 309", ' u'"ifMtu": "1500"}}}}, "metadata": {{"_enrichment_group_creation_timestamp": {:.5f}, ' u'"_enrichment_ttl": 300, "_execute_frequency": 60}}}}'.format( mock_time.return_value)) self._enrichment_kv.set( u'asn_api:namespace01', u'{{"data": {{"asn_data01": {{"data": 101}}}}, ' u'"metadata": {{"_enrichment_group_creation_timestamp": {:.5f}, ' u'"_enrichment_ttl": 300, "_execute_frequency": 60}}}}'.format( mock_time.return_value)) self._enrichment_kv.set( u'asn_api:namespace02', u'{{"data": {{"asn_data02": {{"data": 202}}}}, ' u'"metadata": {{"_enrichment_group_creation_timestamp": {:.5f}, ' u'"_enrichment_ttl": 300, "_execute_frequency": 60}}}}'.format( mock_time.return_value)) self._enrichment_kv.set( u'asn_api:namespace03', u'{{"data": {{"asn_data03": {{"data": 303}}}}, ' u'"metadata": {{"_enrichment_group_creation_timestamp": {:.5f}, ' u'"_enrichment_ttl": 300, "_execute_frequency": 60}}}}'.format( mock_time.return_value)) self._enrichment_kv.set(u'enrichment:asn_api:namespace05', u'bad_data') def test_enrichment_cache(self): """Test enrichment resource attributes""" with self.assertRaises(AssertionError): PanoptesEnrichmentCache(u'non_panoptes_context', self._plugin_conf, self._panoptes_resource) with self.assertRaises(AssertionError): PanoptesEnrichmentCache(self._panoptes_context, u'non_plugin_conf', self._panoptes_resource) with self.assertRaises(AssertionError): PanoptesEnrichmentCache(self._panoptes_context, self._plugin_conf, u'non_panoptes_resource') # Test with bad key-value store mock_panoptes_enrichment_cache_key_value_store = Mock( side_effect=Exception) with patch( u'yahoo_panoptes.framework.enrichment.PanoptesEnrichmentCacheKeyValueStore', mock_panoptes_enrichment_cache_key_value_store): with self.assertRaises(Exception): PanoptesEnrichmentCache(self._panoptes_context, self._plugin_conf, self._panoptes_resource) def test_enrichment_cache_preload01(self): """Test enrichment resource with preload conf self:test_namespace""" result = u'{{"test_resource_id": {{"test_namespace": {{"heartbeat": {{"timestamp": {}}}}}}}}}'.format( int(mock_time.return_value)) enrichment_cache = PanoptesEnrichmentCache(self._panoptes_context, self._plugin_conf, self._panoptes_resource) self.assertEqual( ordered(enrichment_cache.__dict__[u'_enrichment_data']), ordered(json.loads(result))) def test_enrichment_cache_preload02(self): """Test enrichment resource with preload conf""" plugin_conf = { u'Core': { u'name': u'Test Plugin 01' }, u'enrichment': { u'preload': u'self:test_namespace, ' u'asn_api:*, ' u'test_resource_id01:interface, ' u'no_data_resource_id:test_namespace, ' u'test_resource_id01:no_data_namespace' } } results01 = u"""{{"asn_api": {{"namespace02": {{"asn_data02": {{"data": 202}}}}, "namespace03": {{"asn_data03": {{"data": 303}}}}, "namespace01": {{"asn_data01": {{"data": 101}}}}}}, "test_resource_id": {{"test_namespace": {{"heartbeat": {{"timestamp": {}}}}}}}, "test_resource_id01": {{"interface": {{"1226": {{"ifType": "l3ipvlan", "ifAlias": "<not set>", "ifPhysAddress": "00:1f:a0:13:8c:16", "ifDescr": "Virtual Ethernet 226", "ifName": "Virtual Ethernet 226", "ifMtu": "1500"}}, "1209": {{"ifType": "l3ipvlan", "ifAlias": "<not set>", "ifPhysAddress": "00:1f:a0:13:8c:15", "ifDescr": "Virtual Ethernet 209", "ifName": "Virtual Ethernet 209", "ifMtu": "1500"}}}}}}}}""".format(int(mock_time.return_value)) enrichment_cache = PanoptesEnrichmentCache(self._panoptes_context, plugin_conf, self._panoptes_resource) self.assertEqual( ordered(enrichment_cache.__dict__[u'_enrichment_data']), ordered(json.loads(results01))) # Test an error while scanning kv store is handled correctly mock_find_keys = Mock(side_effect=Exception) mock_logger = MockLogger() with patch( u'yahoo_panoptes.framework.enrichment.PanoptesEnrichmentCacheKeyValueStore.find_keys', mock_find_keys): with patch( u'yahoo_panoptes.framework.enrichment.PanoptesContext.logger', mock_logger): PanoptesEnrichmentCache(self._panoptes_context, plugin_conf, self._panoptes_resource) self.assertEqual(mock_logger.error.call_count, 3) mock_kv_store_get = Mock(side_effect=IOError) with patch( u'yahoo_panoptes.framework.enrichment.PanoptesEnrichmentCacheKeyValueStore.get', mock_kv_store_get): with self.assertRaises(IOError): PanoptesEnrichmentCache(self._panoptes_context, plugin_conf, self._panoptes_resource) def test_enrichment_cache_parse_conf(self): """Test enrichment resource parse conf""" result01 = [(u'asn_api', u'*'), (u'self', u'test_namespace'), (u'test_resource_id01', u'interface')] plugin_conf = { u'Core': { u'name': u'Test Plugin 01' }, u'enrichment': { u'preload': u'self:test_namespace, asn_api:*, test_resource_id01:interface' } } enrichment_cache = PanoptesEnrichmentCache(self._panoptes_context, plugin_conf, self._panoptes_resource) self.assertEqual(sorted(result01), sorted(enrichment_cache.__dict__[u'_preload_conf'])) plugin_conf_with_dup = { u'Core': { u'name': u'Test Plugin 01' }, u'enrichment': { u'preload': u'self:test_namespace, asn_api:*, ' u'test_resource_id01:interface, asn_api:*, ' u'test_resource_id01:interface' } } enrichment_cache = PanoptesEnrichmentCache(self._panoptes_context, plugin_conf_with_dup, self._panoptes_resource) self.assertEqual(sorted(result01), sorted(enrichment_cache.__dict__[u'_preload_conf'])) def test_enrichment_cache_parse_conf_bad(self): """Test enrichment resource parse bad conf""" plugin_conf01 = { u'Core': { u'name': u'Test Plugin 01' }, u'enrichment': {} } with self.assertRaises(PanoptesEnrichmentCacheError): PanoptesEnrichmentCache(self._panoptes_context, plugin_conf01, self._panoptes_resource) plugin_conf02 = { u'Core': { u'name': u'Test Plugin 01' }, u'enrichment': { u'preload': {} } } with self.assertRaises(PanoptesEnrichmentCacheError): PanoptesEnrichmentCache(self._panoptes_context, plugin_conf02, self._panoptes_resource) plugin_conf03 = { u'Core': { u'name': u'Test Plugin 01' }, u'enrichment': { u'preload': {u'self:test_namespace, asn_api:'} } } with self.assertRaises(PanoptesEnrichmentCacheError): PanoptesEnrichmentCache(self._panoptes_context, plugin_conf03, self._panoptes_resource) plugin_conf04 = { u'Core': { u'name': u'Test Plugin 01' }, u'enrichment': { u'preload': {u'self:test_namespace, :'} } } with self.assertRaises(PanoptesEnrichmentCacheError): PanoptesEnrichmentCache(self._panoptes_context, plugin_conf04, self._panoptes_resource) plugin_conf05 = { u'Core': { u'name': u'Test Plugin 01' }, u'enrichment': { u'preload': {u'self:test_namespace, '} } } with self.assertRaises(PanoptesEnrichmentCacheError): PanoptesEnrichmentCache(self._panoptes_context, plugin_conf05, self._panoptes_resource) plugin_conf06 = { u'Core': { u'name': u'Test Plugin 01' }, u'enrichment': { u'preload': {u'self:test_namespace, :namespace01'} } } with self.assertRaises(PanoptesEnrichmentCacheError): PanoptesEnrichmentCache(self._panoptes_context, plugin_conf06, self._panoptes_resource) def test_enrichment_cache_get_enrichment(self): """Test enrichment resource get""" plugin_conf = { u'Core': { u'name': u'Test Plugin 01' }, u'enrichment': { u'preload': u'self:test_namespace, ' u'asn_api:*, ' u'test_resource_id01:interface, ' u'no_data_resource_id:test_namespace, ' u'test_resource_id01:no_data_namespace' } } enrichment_cache = PanoptesEnrichmentCache(self._panoptes_context, plugin_conf, self._panoptes_resource) result01 = {u'heartbeat': {u'timestamp': int(mock_time.return_value)}} enrichment_data = enrichment_cache.get_enrichment( u'test_resource_id', u'test_namespace') self.assertEqual(sorted(list(enrichment_data)), sorted(result01)) enrichment_data = enrichment_cache.get_enrichment( u'self', u'test_namespace') self.assertEqual(sorted(list(enrichment_data)), sorted(result01)) result02 = { u'1209': { u'ifType': u'l3ipvlan', u'ifAlias': u'<not set>', u'ifPhysAddress': u'00:1f:a0:13:8c:15', u'ifDescr': u'Virtual Ethernet 209', u'ifName': u'Virtual Ethernet 209', u'ifMtu': u'1500' }, u'1226': { u'ifType': u'l3ipvlan', u'ifAlias': u'<not set>', u'ifPhysAddress': u'00:1f:a0:13:8c:16', u'ifDescr': u'Virtual Ethernet 226', u'ifName': u'Virtual Ethernet 226', u'ifMtu': u'1500' } } enrichment_data = enrichment_cache.get_enrichment( u'test_resource_id01', u'interface') self.assertEqual(sorted(list(enrichment_data)), sorted(result02)) with self.assertRaises(PanoptesEnrichmentCacheError): enrichment_cache.get_enrichment(u'no_data_resource_id', u'test_namespace') with self.assertRaises(PanoptesEnrichmentCacheError): enrichment_cache.get_enrichment(u'test_resource_id01', u'no_data_namespace') with self.assertRaises(AssertionError): enrichment_cache.get_enrichment(u'test_resource_id01', u'') with self.assertRaises(AssertionError): enrichment_cache.get_enrichment(u'', u'test_namespace') with self.assertRaises(AssertionError): enrichment_cache.get_enrichment(u'', u'*') # Test fallback preload result03 = { u'2209': { u'ifType': u'l3ipvlan', u'ifAlias': u'<not set>', u'ifPhysAddress': u'00:1f:a0:13:8c:15', u'ifDescr': u'Virtual Ethernet 309', u'ifName': u'Virtual Ethernet 309', u'ifMtu': u'1500' }, u'2226': { u'ifType': u'l3ipvlan', u'ifAlias': u'<not set>', u'ifPhysAddress': u'00:1f:a0:13:8c:16', u'ifDescr': u'Virtual Ethernet 326', u'ifName': u'Virtual Ethernet 326', u'ifMtu': u'1500' } } enrichment_data03 = enrichment_cache.get_enrichment( u'test_resource_id02', u'interface') self.assertEqual(sorted(list(enrichment_data03)), sorted(result03)) def test_enrichment_cache_get_enrichment_value(self): """Test enrichment resource get_enrichment_value""" plugin_conf = { u'Core': { u'name': u'Test Plugin 01' }, u'enrichment': { u'preload': u'self:test_namespace, ' u'asn_api:*, ' u'test_resource_id01:interface, ' u'no_data_resource_id:test_namespace, ' u'test_resource_id01:no_data_namespace' } } enrichment_cache = PanoptesEnrichmentCache(self._panoptes_context, plugin_conf, self._panoptes_resource) result01 = {u'timestamp': int(mock_time.return_value)} enrich_data = enrichment_cache.get_enrichment_value( u'self', u'test_namespace', u'heartbeat') self.assertEqual(sorted(result01), sorted(enrich_data)) enrich_data = enrichment_cache.get_enrichment_value( u'test_resource_id', u'test_namespace', u'heartbeat') self.assertEqual(sorted(result01), sorted(enrich_data)) result02 = {u'data': 101} enrich_data = enrichment_cache.get_enrichment_value( u'asn_api', u'namespace01', u'asn_data01') self.assertEqual(sorted(result02), sorted(enrich_data)) result03 = { u'ifType': u'l3ipvlan', u'ifAlias': u'<not set>', u'ifPhysAddress': u'00:1f:a0:13:8c:15', u'ifDescr': u'Virtual Ethernet 209', u'ifName': u'Virtual Ethernet 209', u'ifMtu': u'1500' } enrich_data = enrichment_cache.get_enrichment_value( u'test_resource_id01', u'interface', u'1209') self.assertEqual(sorted(result03), sorted(enrich_data)) with self.assertRaises(PanoptesEnrichmentCacheError): enrichment_cache.get_enrichment_value(u'no_data_resource_id', u'test_namespace', u'no_data') with self.assertRaises(AssertionError): enrichment_cache.get_enrichment_value(u'no_data_resource_id', u'test_namespace', u'') with self.assertRaises(AssertionError): enrichment_cache.get_enrichment_value(u'no_data_resource_id', u'', u'no_data') with self.assertRaises(AssertionError): enrichment_cache.get_enrichment_value(u'', u'test_namespace', u'no_data') # Test fallback preload result04 = { u'ifType': u'l3ipvlan', u'ifAlias': u'<not set>', u'ifPhysAddress': u'00:1f:a0:13:8c:15', u'ifDescr': u'Virtual Ethernet 309', u'ifName': u'Virtual Ethernet 309', u'ifMtu': u'1500' } enrich_data = enrichment_cache.get_enrichment_value( u'test_resource_id02', u'interface', u'2209') self.assertEqual(sorted(result04), sorted(enrich_data)) def test_enrichment_cache_get_enrichment_keys(self): """Test enrichment resource get_enrichment_keys""" plugin_conf = { u'Core': { u'name': u'Test Plugin 01' }, u'enrichment': { u'preload': u'self:test_namespace, ' u'asn_api:*, ' u'test_resource_id01:interface, ' u'no_data_resource_id:test_namespace, ' u'test_resource_id01:no_data_namespace' } } enrichment_cache = PanoptesEnrichmentCache(self._panoptes_context, plugin_conf, self._panoptes_resource) result01 = [u'1226', u'1209'] enrichment_data = enrichment_cache.get_enrichment_keys( u'test_resource_id01', u'interface') self.assertListEqual(enrichment_data, result01) with self.assertRaises(PanoptesEnrichmentCacheError): enrichment_cache.get_enrichment_keys(u'test_resource_id01', u'no_data_namespace') result02 = [u'heartbeat'] enrichment_data = enrichment_cache.get_enrichment_keys( u'test_resource_id', u'test_namespace') self.assertListEqual(enrichment_data, result02) enrichment_data = enrichment_cache.get_enrichment_keys( u'self', u'test_namespace') self.assertListEqual(enrichment_data, result02) # Test fallback preload result03 = [u'2226', u'2209'] enrichment_data03 = enrichment_cache.get_enrichment_keys( u'test_resource_id02', u'interface') self.assertListEqual(enrichment_data03, result03)
class SNMPPluginTestFramework(object): plugin_class = None path = None snmp_host = u'127.0.0.1' snmp_port = 10161 snmp_timeout = 10 snmp_retries = 1 snmp_max_repetitions = 10 snmp_community = 'public' snmp_failure_timeout = 1 x509_secured_requests = 0 x509_key_location = u'/home/panoptes/x509/keys' x509_key_filename = u'panoptes.key' x509_cert_location = u'/home/panoptes/x509/keys' x509_cert_filename = u'panoptes.pem' resource_id = u'test_id' resource_endpoint = u'127.0.0.1' resource_plugin = u'dummy' resource_site = u'test_site' resource_class = u'network' resource_subclass = u'test_subclass' resource_type = u'test_type' resource_backplane = None resource_model = u'model' results_data_file = u'results.json' enrichment_data_file = u'enrichment_data' use_enrichment = True plugin_conf = { 'Core': { 'name': 'Test Plugin', 'module': 'test_plugin' }, 'main': { 'execute_frequency': '60', 'enrichment_ttl': '300', 'resource_filter': 'resource_class = "network"' }, 'snmp': { 'timeout': 10, 'retries': 1, 'non_repeaters': 1, 'max_repetitions': 25, }, 'x509': { 'x509_secured_requests': 0, 'x509_key_location': '/home/panoptes/x509/keys', 'x509_key_filename': 'panoptes.key', 'x509_cert_location': '/home/panoptes/x509/keys', 'x509_cert_filename': 'panoptes.pem' } } def __init__(self, test_name): super(SNMPPluginTestFramework, self).__init__(test_name) self._path = self.path self._plugin_conf = self.plugin_conf self._snmp_host = self.snmp_host self._snmp_port = self.snmp_port self._snmp_timeout = self.snmp_timeout self._snmp_retries = self.snmp_retries self._snmp_max_repetitions = self.snmp_max_repetitions self._snmp_community = self.snmp_community self._snmp_failure_timeout = self.snmp_failure_timeout self._x509_secured_requests = self.x509_secured_requests self._x509_cert_location = self.x509_cert_location self._x509_cert_filename = self.x509_cert_filename self._x509_key_location = self.x509_key_location self._x509_key_filename = self.x509_key_filename self._resource_id = self.resource_id self._resource_endpoint = self.resource_endpoint self._resource_plugin = self.resource_plugin self._resource_site = self.resource_site self._resource_class = self.resource_class self._resource_subclass = self.resource_subclass self._resource_type = self.resource_type self._resource_backplane = self.resource_backplane self._resource_model = self.resource_model self._panoptes_context = None self._panoptes_resource = None self._snmp_conf = None self._x509_conf = None self._expected_results = None self._results_data_file = None self._enrichment_data_file = None self._secret_store = None self._plugin_context = None self._plugin_context_bad = None self._snmp_conf_bad = None self._x509_conf_bad = None self._enrichment_kv = None self._enrichment_cache = None def set_panoptes_context(self): panoptes_test_conf_file = os.path.join( os.path.dirname(os.path.dirname(__file__)), 'config_files/test_panoptes_config.ini') self._panoptes_context = PanoptesContext( panoptes_test_conf_file, key_value_store_class_list=[PanoptesEnrichmentCacheKeyValueStore]) def set_panoptes_resource(self): self._panoptes_resource = PanoptesResource( resource_site=self._resource_site, resource_class=self._resource_class, resource_subclass=self._resource_subclass, resource_type=self._resource_type, resource_id=self._resource_id, resource_endpoint=self._resource_endpoint, resource_plugin=self._resource_plugin) self._panoptes_resource.resource_metadata[ 'model'] = self._resource_model if self._resource_backplane: self._panoptes_resource.add_metadata('backplane', self._resource_backplane) def set_enrichment_cache(self, resource_endpoint=None): if self.use_enrichment: self._enrichment_data_file = 'data/' + self.enrichment_data_file self._enrichment_kv = self._panoptes_context.get_kv_store( PanoptesEnrichmentCacheKeyValueStore) enrichment_data_file = os.path.join(os.path.abspath(self.path), self._enrichment_data_file) if self._plugin_conf.get('enrichment'): if self._plugin_conf['enrichment'].get('preload'): self._enrichment_cache = PanoptesEnrichmentCache( self._panoptes_context, self._plugin_conf, self._panoptes_resource) try: with open(enrichment_data_file) as enrichment_data: for line in enrichment_data: key, value = line.strip().split('=>') if resource_endpoint is not None: value = self.update_enrichment_ip( value, resource_endpoint) self._enrichment_kv.set(key, value) except Exception as e: raise IOError( 'Failed to load enrichment data file {}: {}'.format( enrichment_data_file, repr(e))) def set_snmp_conf(self): self._snmp_conf = self._panoptes_context.config_object.snmp_defaults def set_x509_conf(self): self._x509_conf = { 'x509_secured_requests': self._x509_secured_requests, 'x509_cert_location': self._x509_cert_location, 'x509_cert_filename': self._x509_cert_filename, 'x509_key_location': self._x509_key_location, 'x509_key_filename': self._x509_key_filename } def set_secret_store(self): self._secret_store = create_autospec(PanoptesSecretsStore, instance=True, spec_set=True) self._secret_store.get_by_site.return_value = self._snmp_community def set_plugin_context(self): self._plugin_context = create_autospec( PanoptesPluginWithEnrichmentContext, instance=True, spec_set=True, data=self._panoptes_resource, enrichment=self._enrichment_cache, config=self._plugin_conf, snmp=self._snmp_conf, x509=self._x509_conf, secrets=self._secret_store, logger=logging.getLogger(__name__)) def set_x509_conf_bad(self): self._x509_conf_bad = copy.copy(self._x509_conf) self._x509_conf_bad['x509_secured_requests'] = 5 def set_snmp_conf_bad(self): self._snmp_conf_bad = copy.copy(self._snmp_conf) self._snmp_conf_bad['port'] += 1 self._snmp_conf_bad['timeout'] = self._snmp_failure_timeout def set_plugin_context_bad(self): self._plugin_context_bad = create_autospec( PanoptesPluginWithEnrichmentContext, instance=True, spec_set=True, data=self._panoptes_resource, enrichment=self._enrichment_cache, config=self._plugin_conf, snmp=self._snmp_conf_bad, x509=self._x509_conf, secrets=self._secret_store, logger=logging.getLogger(__name__)) def set_expected_results(self): self._results_data_file = 'data/' + self.results_data_file expected_result_file = os.path.join(os.path.abspath(self.path), self._results_data_file) self._expected_results = json.load(open(expected_result_file)) def update_enrichment_ip(self, enrichment, resource_endpoint): """ The PluginPollingGenericSNMPMetrics Plugin initially calls the function _get_config() when started. This function loads the enrichment from the PanoptesEnrichmentCacheKeyValueStore using the resource_id, namespace, and enrichment key. The enrichment key is the resource_endpoint. no_service_active tests to ensure that a proper error (timeout -> ping error) is returned when snmp_bulk is called with an invalid IP. In order to get to this stage in the tests the enrichment must loaded in the generic snmp runner. In order to test this, the resource_endpoint is changed to be invalid. However this IP must also be changed in the test enrichment file so the generic snmp runner is able to load the enrichment from the cache. """ modified_enrichment = json.loads(enrichment) if len(modified_enrichment) == 0 or len( list(modified_enrichment['data'].keys())) == 0: return enrichment ip = list(modified_enrichment['data'].keys())[0] if re.search(r'\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b', ip): modified_enrichment['data'][ resource_endpoint] = modified_enrichment['data'][ip] del modified_enrichment['data'][ip] return json.dumps(modified_enrichment) return enrichment @patch('time.time', mock_time) @patch('yahoo_panoptes.framework.resources.time', mock_time) @patch('redis.StrictRedis', PanoptesMockRedis) def setUp(self): self.set_panoptes_context() self.set_panoptes_resource() self.set_enrichment_cache() self.set_snmp_conf() self.set_x509_conf() self.set_secret_store() self.set_plugin_context() self.set_snmp_conf_bad() self.set_x509_conf_bad() self.set_plugin_context_bad() self.set_expected_results()
class TestPanoptesEnrichmentCacheStore(unittest.TestCase): @patch('redis.StrictRedis', PanoptesMockRedis) @patch('time.time', mock_time) def setUp(self): self.my_dir, self.panoptes_test_conf_file = _get_test_conf_file() self._panoptes_context = PanoptesContext( self.panoptes_test_conf_file, key_value_store_class_list=[PanoptesEnrichmentCacheKeyValueStore]) self._enrichment_kv = self._panoptes_context.get_kv_store( PanoptesEnrichmentCacheKeyValueStore) self._panoptes_resource = PanoptesResource( resource_site='test_site', resource_class='test_class', resource_subclass='test_subclass', resource_type='test_type', resource_id='test_resource_id', resource_endpoint='test_endpoint', resource_plugin='test_plugin') interface_validation_object = PanoptesEnrichmentInterfaceSchemaValidator( ) neighbor_validation_object = PanoptesEnrichmentNeighborSchemaValidator( ) enrichment_set1 = PanoptesEnrichmentSet('int_001') enrichment_set1.add('speed', 1000) enrichment_set1.add('index', 001) enrichment_set1.add('status', 'up') enrichment_set2 = PanoptesEnrichmentSet('int_002') enrichment_set2.add('speed', 1000) enrichment_set2.add('index', 002) enrichment_set2.add('status', 'down') enrichment_group1 = PanoptesEnrichmentGroup( 'interface', interface_validation_object, 300, 60) enrichment_group1.add_enrichment_set(enrichment_set1) enrichment_group1.add_enrichment_set(enrichment_set2) enrichment_set3 = PanoptesEnrichmentSet('host_name') enrichment_set3.add('vlan_id', 501) enrichment_set3.add('property', 'Test Property') enrichment_set3.add('mac', 'aa:bb:cc:dd:ee:ff') enrichment_group2 = PanoptesEnrichmentGroup( 'neighbor', neighbor_validation_object, 600, 120) enrichment_group2.add_enrichment_set(enrichment_set3) self.enrichment_group_set1 = PanoptesEnrichmentGroupSet( self._panoptes_resource) self.enrichment_group_set1.add_enrichment_group(enrichment_group1) self.enrichment_group_set1.add_enrichment_group(enrichment_group2) self._panoptes_resource01 = PanoptesResource( resource_site='test_site', resource_class='test_class', resource_subclass='test_subclass', resource_type='test_type', resource_id='test_resource_id01', resource_endpoint='test_endpoint01', resource_plugin='test_plugin') enrichment_set4 = PanoptesEnrichmentSet('host_name01') enrichment_set4.add('vlan_id', 502) enrichment_set4.add('property', 'Test Property') enrichment_set4.add('mac', 'aa:bb:cc:dd:ee:ff') enrichment_group3 = PanoptesEnrichmentGroup( 'neighbor', neighbor_validation_object, 600, 120) enrichment_group3.add_enrichment_set(enrichment_set3) enrichment_group3.add_enrichment_set(enrichment_set4) self.enrichment_group_set2 = PanoptesEnrichmentGroupSet( self._panoptes_resource01) self.enrichment_group_set2.add_enrichment_group(enrichment_group1) self.enrichment_group_set2.add_enrichment_group(enrichment_group3) self._multi_enrichment_group_set = PanoptesEnrichmentMultiGroupSet() self._multi_enrichment_group_set.add_enrichment_group_set( self.enrichment_group_set1) self._multi_enrichment_group_set.add_enrichment_group_set( self.enrichment_group_set2) @patch('time.time', mock_time) def test_panoptes_enrichment_set(self): enrichment_set1 = PanoptesEnrichmentSet('int_001') enrichment_set1.add('speed', 1000) enrichment_set1.add('index', 001) enrichment_set1.add('status', 'up') self.assertEquals( enrichment_set1.json(), '{"int_001": {"index": 1, "speed": 1000, "status": "up"}}') self.assertEquals( repr(enrichment_set1), "PanoptesEnrichmentSet({'int_001': {'status': 'up', 'index': 1, 'speed': 1000}})" ) enrichment_set2 = PanoptesEnrichmentSet('int_002') enrichment_set2.add('speed', 1000) enrichment_set2.add('index', 002) enrichment_set2.add('status', 'down') interface_validation_object = PanoptesEnrichmentInterfaceSchemaValidator( ) enrichment_group1 = PanoptesEnrichmentGroup( 'interface', interface_validation_object, 300, 60) self.assertFalse(enrichment_set1 == enrichment_group1) self.assertFalse(enrichment_set1 == enrichment_set2) @patch('time.time', mock_time) def test_panoptes_enrichment_group(self): interface_validation_object = PanoptesEnrichmentInterfaceSchemaValidator( ) enrichment_group1 = PanoptesEnrichmentGroup( 'interface', interface_validation_object, 300, 60) self.assertEquals(enrichment_group1.enrichment_schema, PanoptesEnrichmentInterfaceSchemaValidator.schema) self.assertEquals( repr(enrichment_group1), "PanoptesEnrichmentGroup({{'data': set([]), " "'namespace': 'interface', " "'metadata': {{" "'_enrichment_group_creation_timestamp': {:.5f}, " "'_enrichment_ttl': 300, " "'_execute_frequency': 60}}}})".format(mock_time.return_value)) enrichment_set1 = PanoptesEnrichmentSet('int_001') enrichment_set1.add('speed', 1000) enrichment_set1.add('index', 001) enrichment_set1.add('status', 'up') self.assertFalse(enrichment_group1 == enrichment_set1) enrichment_group2 = PanoptesEnrichmentGroup( 'interface', interface_validation_object, 300, 60) enrichment_group3 = PanoptesEnrichmentGroup( 'other_namespace', interface_validation_object, 300, 60) self.assertTrue(enrichment_group1 == enrichment_group2) self.assertFalse(enrichment_group1 == enrichment_group3) @patch('time.time', mock_time) def test_store_enrichment_data_enrichment_group_set(self): interface_result_data = \ { "data": { "int_001": { "index": 1, "speed": 1000, "status": "up" }, "int_002": { "index": 2, "speed": 1000, "status": "down" } }, "metadata": { "_enrichment_group_creation_timestamp": mock_time.return_value, "_enrichment_ttl": 300, "_execute_frequency": 60 } } neighbor_result_data = \ { "data": { "host_name": { "mac": "aa:bb:cc:dd:ee:ff", "property": "Test Property", "vlan_id": 501 } }, "metadata": { "_enrichment_group_creation_timestamp": mock_time.return_value, "_enrichment_ttl": 600, "_execute_frequency": 120 } } _store_enrichment_data(self._panoptes_context, self.enrichment_group_set1, 'PanoptesPluginInfo') self.assertNotEquals( ordered(interface_result_data), ordered( json.loads( self._enrichment_kv.get('test_resource_id:neighbor')))) self.assertEquals( ordered(interface_result_data), ordered( json.loads( self._enrichment_kv.get('test_resource_id:interface')))) self.assertEquals( ordered(neighbor_result_data), ordered( json.loads( self._enrichment_kv.get('test_resource_id:neighbor')))) def test_store_enrichment_data_enrichment_multi_group_set(self): enrichment_result_keys = [ 'test_resource_id01:interface', 'test_resource_id01:neighbor', 'test_resource_id:interface', 'test_resource_id:neighbor' ] neighbor_result_data = \ {"data": {"host_name": {"mac": "aa:bb:cc:dd:ee:ff", "property": "Test Property", "vlan_id": 501}, "host_name01": {"mac": "aa:bb:cc:dd:ee:ff", "property": "Test Property", "vlan_id": 502}}, "metadata": {"_enrichment_group_creation_timestamp": mock_time.return_value, "_enrichment_ttl": 600, "_execute_frequency": 120}} _store_enrichment_data(self._panoptes_context, self._multi_enrichment_group_set, 'PanoptesPluginInfo') self.assertEquals(enrichment_result_keys, self._enrichment_kv.find_keys('*')) self.assertEquals( ordered(neighbor_result_data), ordered( json.loads( self._enrichment_kv.get('test_resource_id01:neighbor')))) self.assertNotEquals( ordered(neighbor_result_data), ordered( json.loads( self._enrichment_kv.get('test_resource_id01:interface'))))
class TestPanoptesEnrichmentCache(unittest.TestCase): @patch('redis.StrictRedis', PanoptesMockRedis) def setUp(self): self.my_dir, self.panoptes_test_conf_file = _get_test_conf_file() self._panoptes_context = PanoptesContext( self.panoptes_test_conf_file, key_value_store_class_list=[PanoptesEnrichmentCacheKeyValueStore]) self._enrichment_kv = self._panoptes_context.get_kv_store( PanoptesEnrichmentCacheKeyValueStore) self._panoptes_resource = PanoptesResource( resource_site='test_site', resource_class='test_class', resource_subclass='test_subclass', resource_type='test_type', resource_id='test_resource_id', resource_endpoint='test_endpoint', resource_plugin='test_plugin') self._plugin_conf = { 'Core': { 'name': 'Heartbeat Enrichment Plugin', 'module': 'plugin_enrichment_heartbeat' }, 'main': { 'execute_frequency': '60', 'enrichment_ttl': '300', 'resource_filter': 'resource_class = "system" AND resource_subclass = ' '"internal" AND resource_type = "heartbeat"' }, 'enrichment': { 'preload': 'self:test_namespace' } } self._enrichment_kv.set( 'test_resource_id:test_namespace', '{"data": {"heartbeat": {"timestamp": 1521668419}}, ' '"metadata": {"_enrichment_group_creation_timestamp": 1521668419.881953, ' '"_enrichment_ttl": 300, "_execute_frequency": 60}}') self._enrichment_kv.set( 'test_resource_id01:interface', '{"data": {"1226": {"ifType": "l3ipvlan", "ifAlias": "<not set>", "ifPhysAddress": ' '"00:1f:a0:13:8c:16", "ifDescr": "Virtual Ethernet 226", "ifName": "Virtual Ethernet 226", ' '"ifMtu": "1500"}, "1209": {"ifType": "l3ipvlan", "ifAlias": "<not set>", "ifPhysAddress": ' '"00:1f:a0:13:8c:15", "ifDescr": "Virtual Ethernet 209", "ifName": "Virtual Ethernet 209", ' '"ifMtu": "1500"}}, "metadata": {"_enrichment_group_creation_timestamp": 1521668419.881953, ' '"_enrichment_ttl": 300, "_execute_frequency": 60}}') self._enrichment_kv.set( 'test_resource_id02:interface', '{"data": {"2226": {"ifType": "l3ipvlan", "ifAlias": "<not set>", "ifPhysAddress": ' '"00:1f:a0:13:8c:16", "ifDescr": "Virtual Ethernet 326", "ifName": "Virtual Ethernet 326", ' '"ifMtu": "1500"}, "2209": {"ifType": "l3ipvlan", "ifAlias": "<not set>", "ifPhysAddress": ' '"00:1f:a0:13:8c:15", "ifDescr": "Virtual Ethernet 309", "ifName": "Virtual Ethernet 309", ' '"ifMtu": "1500"}}, "metadata": {"_enrichment_group_creation_timestamp": 1521668419.881953, ' '"_enrichment_ttl": 300, "_execute_frequency": 60}}') self._enrichment_kv.set( 'asn_api:namespace01', '{"data": {"asn_data01": {"data": 101}}, ' '"metadata": {"_enrichment_group_creation_timestamp": 1521668419.881953, ' '"_enrichment_ttl": 300, "_execute_frequency": 60}}') self._enrichment_kv.set( 'asn_api:namespace02', '{"data": {"asn_data02": {"data": 202}}, ' '"metadata": {"_enrichment_group_creation_timestamp": 1521668419.881953, ' '"_enrichment_ttl": 300, "_execute_frequency": 60}}') self._enrichment_kv.set( 'asn_api:namespace03', '{"data": {"asn_data03": {"data": 303}}, ' '"metadata": {"_enrichment_group_creation_timestamp": 1521668419.881953, ' '"_enrichment_ttl": 300, "_execute_frequency": 60}}') self._enrichment_kv.set('enrichment:asn_api:namespace05', 'bad_data') def test_enrichment_cache(self): """Test enrichment resource attributes""" with self.assertRaises(AssertionError): PanoptesEnrichmentCache('non_panoptes_context', self._plugin_conf, self._panoptes_resource) with self.assertRaises(AssertionError): PanoptesEnrichmentCache(self._panoptes_context, 'non_plugin_conf', self._panoptes_resource) with self.assertRaises(AssertionError): PanoptesEnrichmentCache(self._panoptes_context, self._plugin_conf, 'non_panoptes_resource') def test_enrichment_cache_preload01(self): """Test enrichment resource with preload conf self:test_namespace""" result = '{"test_resource_id": {"test_namespace": {"heartbeat": {"timestamp": 1521668419}}}}' enrichment_cache = PanoptesEnrichmentCache(self._panoptes_context, self._plugin_conf, self._panoptes_resource) self.assertEqual( ordered(enrichment_cache.__dict__['_enrichment_data']), ordered(json.loads(result))) def test_enrichment_cache_preload02(self): """Test enrichment resource with preload conf""" plugin_conf = { 'Core': { 'name': 'Test Plugin 01' }, 'enrichment': { 'preload': 'self:test_namespace, ' 'asn_api:*, ' 'test_resource_id01:interface, ' 'no_data_resource_id:test_namespace, ' 'test_resource_id01:no_data_namespace' } } results01 = """{"asn_api": {"namespace02": {"asn_data02": {"data": 202}}, "namespace03": {"asn_data03": {"data": 303}}, "namespace01": {"asn_data01": {"data": 101}}}, "test_resource_id": {"test_namespace": {"heartbeat": {"timestamp": 1521668419}}}, "test_resource_id01": {"interface": {"1226": {"ifType": "l3ipvlan", "ifAlias": "<not set>", "ifPhysAddress": "00:1f:a0:13:8c:16", "ifDescr": "Virtual Ethernet 226", "ifName": "Virtual Ethernet 226", "ifMtu": "1500"}, "1209": {"ifType": "l3ipvlan", "ifAlias": "<not set>", "ifPhysAddress": "00:1f:a0:13:8c:15", "ifDescr": "Virtual Ethernet 209", "ifName": "Virtual Ethernet 209", "ifMtu": "1500"}}}}""" enrichment_cache = PanoptesEnrichmentCache(self._panoptes_context, plugin_conf, self._panoptes_resource) self.assertEqual( ordered(enrichment_cache.__dict__['_enrichment_data']), ordered(json.loads(results01))) def test_enrichment_cache_parse_conf(self): """Test enrichment resource parse conf""" result01 = [('asn_api', '*'), ('self', 'test_namespace'), ('test_resource_id01', 'interface')] plugin_conf = { 'Core': { 'name': 'Test Plugin 01' }, 'enrichment': { 'preload': 'self:test_namespace, asn_api:*, test_resource_id01:interface' } } enrichment_cache = PanoptesEnrichmentCache(self._panoptes_context, plugin_conf, self._panoptes_resource) self.assertEqual(sorted(result01), sorted(enrichment_cache.__dict__['_preload_conf'])) plugin_conf_with_dup = { 'Core': { 'name': 'Test Plugin 01' }, 'enrichment': { 'preload': 'self:test_namespace, asn_api:*, ' 'test_resource_id01:interface, asn_api:*, ' 'test_resource_id01:interface' } } enrichment_cache = PanoptesEnrichmentCache(self._panoptes_context, plugin_conf_with_dup, self._panoptes_resource) self.assertEqual(sorted(result01), sorted(enrichment_cache.__dict__['_preload_conf'])) def test_enrichment_cache_parse_conf_bad(self): """Test enrichment resource parse bad conf""" plugin_conf01 = {'Core': {'name': 'Test Plugin 01'}, 'enrichment': {}} with self.assertRaises(TypeError): PanoptesEnrichmentCache(self._panoptes_context, plugin_conf01, self._panoptes_resource) plugin_conf02 = { 'Core': { 'name': 'Test Plugin 01' }, 'enrichment': { 'preload': {} } } with self.assertRaises(TypeError): PanoptesEnrichmentCache(self._panoptes_context, plugin_conf02, self._panoptes_resource) plugin_conf03 = { 'Core': { 'name': 'Test Plugin 01' }, 'enrichment': { 'preload': {'self:test_namespace, asn_api:'} } } with self.assertRaises(TypeError): PanoptesEnrichmentCache(self._panoptes_context, plugin_conf03, self._panoptes_resource) plugin_conf04 = { 'Core': { 'name': 'Test Plugin 01' }, 'enrichment': { 'preload': {'self:test_namespace, :'} } } with self.assertRaises(TypeError): PanoptesEnrichmentCache(self._panoptes_context, plugin_conf04, self._panoptes_resource) plugin_conf05 = { 'Core': { 'name': 'Test Plugin 01' }, 'enrichment': { 'preload': {'self:test_namespace, '} } } with self.assertRaises(TypeError): PanoptesEnrichmentCache(self._panoptes_context, plugin_conf05, self._panoptes_resource) plugin_conf06 = { 'Core': { 'name': 'Test Plugin 01' }, 'enrichment': { 'preload': {'self:test_namespace, :namespace01'} } } with self.assertRaises(TypeError): PanoptesEnrichmentCache(self._panoptes_context, plugin_conf06, self._panoptes_resource) def test_enrichment_cache_get_enrichment(self): """Test enrichment resource get""" plugin_conf = { 'Core': { 'name': 'Test Plugin 01' }, 'enrichment': { 'preload': 'self:test_namespace, ' 'asn_api:*, ' 'test_resource_id01:interface, ' 'no_data_resource_id:test_namespace, ' 'test_resource_id01:no_data_namespace' } } enrichment_cache = PanoptesEnrichmentCache(self._panoptes_context, plugin_conf, self._panoptes_resource) result01 = {u'heartbeat': {u'timestamp': 1521668419}} enrichment_data = enrichment_cache.get_enrichment( 'test_resource_id', 'test_namespace') self.assertEqual(sorted(list(enrichment_data)), sorted(result01)) enrichment_data = enrichment_cache.get_enrichment( 'self', 'test_namespace') self.assertEqual(sorted(list(enrichment_data)), sorted(result01)) result02 = { u'1209': { u'ifType': u'l3ipvlan', u'ifAlias': u'<not set>', u'ifPhysAddress': u'00:1f:a0:13:8c:15', u'ifDescr': u'Virtual Ethernet 209', u'ifName': u'Virtual Ethernet 209', u'ifMtu': u'1500' }, u'1226': { u'ifType': u'l3ipvlan', u'ifAlias': u'<not set>', u'ifPhysAddress': u'00:1f:a0:13:8c:16', u'ifDescr': u'Virtual Ethernet 226', u'ifName': u'Virtual Ethernet 226', u'ifMtu': u'1500' } } enrichment_data = enrichment_cache.get_enrichment( 'test_resource_id01', 'interface') self.assertEqual(sorted(list(enrichment_data)), sorted(result02)) with self.assertRaises(PanoptesEnrichmentCacheError): enrichment_cache.get_enrichment('no_data_resource_id', 'test_namespace') with self.assertRaises(PanoptesEnrichmentCacheError): enrichment_cache.get_enrichment('test_resource_id01', 'no_data_namespace') with self.assertRaises(AssertionError): enrichment_cache.get_enrichment('test_resource_id01', '') with self.assertRaises(AssertionError): enrichment_cache.get_enrichment('', 'test_namespace') with self.assertRaises(AssertionError): enrichment_cache.get_enrichment('', '*') # Test fallback preload result03 = { u'2209': { u'ifType': u'l3ipvlan', u'ifAlias': u'<not set>', u'ifPhysAddress': u'00:1f:a0:13:8c:15', u'ifDescr': u'Virtual Ethernet 309', u'ifName': u'Virtual Ethernet 309', u'ifMtu': u'1500' }, u'2226': { u'ifType': u'l3ipvlan', u'ifAlias': u'<not set>', u'ifPhysAddress': u'00:1f:a0:13:8c:16', u'ifDescr': u'Virtual Ethernet 326', u'ifName': u'Virtual Ethernet 326', u'ifMtu': u'1500' } } enrichment_data03 = enrichment_cache.get_enrichment( 'test_resource_id02', 'interface') self.assertEqual(sorted(list(enrichment_data03)), sorted(result03)) def test_enrichment_cache_get_enrichment_value(self): """Test enrichment resource get_enrichment_value""" plugin_conf = { 'Core': { 'name': 'Test Plugin 01' }, 'enrichment': { 'preload': 'self:test_namespace, ' 'asn_api:*, ' 'test_resource_id01:interface, ' 'no_data_resource_id:test_namespace, ' 'test_resource_id01:no_data_namespace' } } enrichment_cache = PanoptesEnrichmentCache(self._panoptes_context, plugin_conf, self._panoptes_resource) result01 = {u'timestamp': 1521668419} enrich_data = enrichment_cache.get_enrichment_value( 'self', 'test_namespace', 'heartbeat') self.assertEqual(sorted(result01), sorted(enrich_data)) enrich_data = enrichment_cache.get_enrichment_value( 'test_resource_id', 'test_namespace', 'heartbeat') self.assertEqual(sorted(result01), sorted(enrich_data)) result02 = {u'data': 101} enrich_data = enrichment_cache.get_enrichment_value( 'asn_api', 'namespace01', 'asn_data01') self.assertEqual(sorted(result02), sorted(enrich_data)) result03 = { u'ifType': u'l3ipvlan', u'ifAlias': u'<not set>', u'ifPhysAddress': u'00:1f:a0:13:8c:15', u'ifDescr': u'Virtual Ethernet 209', u'ifName': u'Virtual Ethernet 209', u'ifMtu': u'1500' } enrich_data = enrichment_cache.get_enrichment_value( 'test_resource_id01', 'interface', '1209') self.assertEqual(sorted(result03), sorted(enrich_data)) with self.assertRaises(PanoptesEnrichmentCacheError): enrichment_cache.get_enrichment_value('no_data_resource_id', 'test_namespace', 'no_data') with self.assertRaises(AssertionError): enrichment_cache.get_enrichment_value('no_data_resource_id', 'test_namespace', '') with self.assertRaises(AssertionError): enrichment_cache.get_enrichment_value('no_data_resource_id', '', 'no_data') with self.assertRaises(AssertionError): enrichment_cache.get_enrichment_value('', 'test_namespace', 'no_data') # Test fallback preload result04 = { u'ifType': u'l3ipvlan', u'ifAlias': u'<not set>', u'ifPhysAddress': u'00:1f:a0:13:8c:15', u'ifDescr': u'Virtual Ethernet 309', u'ifName': u'Virtual Ethernet 309', u'ifMtu': u'1500' } enrich_data = enrichment_cache.get_enrichment_value( 'test_resource_id02', 'interface', '2209') self.assertEqual(sorted(result04), sorted(enrich_data)) def test_enrichment_cache_get_enrichment_keys(self): """Test enrichment resource get_enrichment_keys""" plugin_conf = { 'Core': { 'name': 'Test Plugin 01' }, 'enrichment': { 'preload': 'self:test_namespace, ' 'asn_api:*, ' 'test_resource_id01:interface, ' 'no_data_resource_id:test_namespace, ' 'test_resource_id01:no_data_namespace' } } enrichment_cache = PanoptesEnrichmentCache(self._panoptes_context, plugin_conf, self._panoptes_resource) result01 = [u'1226', u'1209'] enrichment_data = enrichment_cache.get_enrichment_keys( 'test_resource_id01', 'interface') self.assertListEqual(enrichment_data, result01) with self.assertRaises(PanoptesEnrichmentCacheError): enrichment_cache.get_enrichment_keys('test_resource_id01', 'no_data_namespace') result02 = [u'heartbeat'] enrichment_data = enrichment_cache.get_enrichment_keys( 'test_resource_id', 'test_namespace') self.assertListEqual(enrichment_data, result02) # Test fallback preload result03 = [u'2226', u'2209'] enrichment_data03 = enrichment_cache.get_enrichment_keys( 'test_resource_id02', 'interface') self.assertListEqual(enrichment_data03, result03)