def test_enrichment_cache_preload02(self): """Test enrichment resource with preload conf""" plugin_conf = { u'Core': { u'name': u'Test Plugin 01' }, u'enrichment': { u'preload': u'self:test_namespace, ' u'asn_api:*, ' u'test_resource_id01:interface, ' u'no_data_resource_id:test_namespace, ' u'test_resource_id01:no_data_namespace' } } results01 = u"""{{"asn_api": {{"namespace02": {{"asn_data02": {{"data": 202}}}}, "namespace03": {{"asn_data03": {{"data": 303}}}}, "namespace01": {{"asn_data01": {{"data": 101}}}}}}, "test_resource_id": {{"test_namespace": {{"heartbeat": {{"timestamp": {}}}}}}}, "test_resource_id01": {{"interface": {{"1226": {{"ifType": "l3ipvlan", "ifAlias": "<not set>", "ifPhysAddress": "00:1f:a0:13:8c:16", "ifDescr": "Virtual Ethernet 226", "ifName": "Virtual Ethernet 226", "ifMtu": "1500"}}, "1209": {{"ifType": "l3ipvlan", "ifAlias": "<not set>", "ifPhysAddress": "00:1f:a0:13:8c:15", "ifDescr": "Virtual Ethernet 209", "ifName": "Virtual Ethernet 209", "ifMtu": "1500"}}}}}}}}""".format(int(mock_time.return_value)) enrichment_cache = PanoptesEnrichmentCache(self._panoptes_context, plugin_conf, self._panoptes_resource) self.assertEqual( ordered(enrichment_cache.__dict__[u'_enrichment_data']), ordered(json.loads(results01))) # Test an error while scanning kv store is handled correctly mock_find_keys = Mock(side_effect=Exception) mock_logger = MockLogger() with patch( u'yahoo_panoptes.framework.enrichment.PanoptesEnrichmentCacheKeyValueStore.find_keys', mock_find_keys): with patch( u'yahoo_panoptes.framework.enrichment.PanoptesContext.logger', mock_logger): PanoptesEnrichmentCache(self._panoptes_context, plugin_conf, self._panoptes_resource) self.assertEqual(mock_logger.error.call_count, 3) mock_kv_store_get = Mock(side_effect=IOError) with patch( u'yahoo_panoptes.framework.enrichment.PanoptesEnrichmentCacheKeyValueStore.get', mock_kv_store_get): with self.assertRaises(IOError): PanoptesEnrichmentCache(self._panoptes_context, plugin_conf, self._panoptes_resource)
def test_enrichment_cache(self): """Test enrichment resource attributes""" with self.assertRaises(AssertionError): PanoptesEnrichmentCache('non_panoptes_context', self._plugin_conf, self._panoptes_resource) with self.assertRaises(AssertionError): PanoptesEnrichmentCache(self._panoptes_context, 'non_plugin_conf', self._panoptes_resource) with self.assertRaises(AssertionError): PanoptesEnrichmentCache(self._panoptes_context, self._plugin_conf, 'non_panoptes_resource')
def test_enrichment_cache_get_enrichment_keys(self): """Test enrichment resource get_enrichment_keys""" plugin_conf = {u'Core': {u'name': u'Test Plugin 01'}, u'enrichment': {u'preload': u'self:test_namespace, ' u'asn_api:*, ' u'test_resource_id01:interface, ' u'no_data_resource_id:test_namespace, ' u'test_resource_id01:no_data_namespace'} } enrichment_cache = PanoptesEnrichmentCache(self._panoptes_context, plugin_conf, self._panoptes_resource) result01 = [u'1226', u'1209'] enrichment_data = enrichment_cache.get_enrichment_keys(u'test_resource_id01', u'interface') self.assertListEqual(enrichment_data, result01) with self.assertRaises(PanoptesEnrichmentCacheError): enrichment_cache.get_enrichment_keys(u'test_resource_id01', u'no_data_namespace') result02 = [u'heartbeat'] enrichment_data = enrichment_cache.get_enrichment_keys(u'test_resource_id', u'test_namespace') self.assertListEqual(enrichment_data, result02) enrichment_data = enrichment_cache.get_enrichment_keys(u'self', u'test_namespace') self.assertListEqual(enrichment_data, result02) # Test fallback preload result03 = [u'2226', u'2209'] enrichment_data03 = enrichment_cache.get_enrichment_keys(u'test_resource_id02', u'interface') self.assertListEqual(enrichment_data03, result03)
def _get_context(self, plugin): self._enrichment = None if plugin.config.get(u'enrichment'): try: self._enrichment = PanoptesEnrichmentCache( self._panoptes_context, plugin.config, self._plugin_data) except Exception as e: raise PanoptesEnrichmentCacheError( u'Error while creating PanoptesEnrichmentResource object for plugin ' u'{}: {}, skipping run'.format(plugin.name, repr(e))) if self._enrichment is None: raise PanoptesEnrichmentCacheError( u'No enrichments found for plugin {} (configured {}), ' u'skipping run'.format(plugin.name, plugin.config.get(u'enrichment'))) return PanoptesPluginWithEnrichmentContext( panoptes_context=self._panoptes_context, logger_name=self._plugin_logger_name, config=plugin.config, key_value_store=self._panoptes_context.get_kv_store( self._plugin_kv_store_class), secrets_store=self._panoptes_context.get_kv_store( self._plugin_secrets_store_class), data=self._plugin_data, enrichment=self._enrichment)
def set_enrichment_cache(self, resource_endpoint=None): if self.use_enrichment: self._enrichment_data_file = 'data/' + self.enrichment_data_file self._enrichment_kv = self._panoptes_context.get_kv_store( PanoptesEnrichmentCacheKeyValueStore) enrichment_data_file = os.path.join(os.path.abspath(self.path), self._enrichment_data_file) if self._plugin_conf.get('enrichment'): if self._plugin_conf['enrichment'].get('preload'): self._enrichment_cache = PanoptesEnrichmentCache( self._panoptes_context, self._plugin_conf, self._panoptes_resource) try: with open(enrichment_data_file) as enrichment_data: for line in enrichment_data: key, value = line.strip().split('=>') if resource_endpoint is not None: value = self.update_enrichment_ip( value, resource_endpoint) self._enrichment_kv.set(key, value) except Exception as e: raise IOError( 'Failed to load enrichment data file {}: {}'.format( enrichment_data_file, repr(e)))
def test_enrichment_cache_preload02(self): """Test enrichment resource with preload conf""" plugin_conf = { 'Core': { 'name': 'Test Plugin 01' }, 'enrichment': { 'preload': 'self:test_namespace, ' 'asn_api:*, ' 'test_resource_id01:interface, ' 'no_data_resource_id:test_namespace, ' 'test_resource_id01:no_data_namespace' } } results01 = """{"asn_api": {"namespace02": {"asn_data02": {"data": 202}}, "namespace03": {"asn_data03": {"data": 303}}, "namespace01": {"asn_data01": {"data": 101}}}, "test_resource_id": {"test_namespace": {"heartbeat": {"timestamp": 1521668419}}}, "test_resource_id01": {"interface": {"1226": {"ifType": "l3ipvlan", "ifAlias": "<not set>", "ifPhysAddress": "00:1f:a0:13:8c:16", "ifDescr": "Virtual Ethernet 226", "ifName": "Virtual Ethernet 226", "ifMtu": "1500"}, "1209": {"ifType": "l3ipvlan", "ifAlias": "<not set>", "ifPhysAddress": "00:1f:a0:13:8c:15", "ifDescr": "Virtual Ethernet 209", "ifName": "Virtual Ethernet 209", "ifMtu": "1500"}}}}""" enrichment_cache = PanoptesEnrichmentCache(self._panoptes_context, plugin_conf, self._panoptes_resource) self.assertEqual( ordered(enrichment_cache.__dict__['_enrichment_data']), ordered(json.loads(results01)))
def test_enrichment_cache(self): """Test enrichment resource attributes""" with self.assertRaises(AssertionError): PanoptesEnrichmentCache(u'non_panoptes_context', self._plugin_conf, self._panoptes_resource) with self.assertRaises(AssertionError): PanoptesEnrichmentCache(self._panoptes_context, u'non_plugin_conf', self._panoptes_resource) with self.assertRaises(AssertionError): PanoptesEnrichmentCache(self._panoptes_context, self._plugin_conf, u'non_panoptes_resource') # Test with bad key-value store mock_panoptes_enrichment_cache_key_value_store = Mock(side_effect=Exception) with patch(u'yahoo_panoptes.framework.enrichment.PanoptesEnrichmentCacheKeyValueStore', mock_panoptes_enrichment_cache_key_value_store): with self.assertRaises(Exception): PanoptesEnrichmentCache(self._panoptes_context, self._plugin_conf, self._panoptes_resource)
def test_enrichment_cache_preload01(self): """Test enrichment resource with preload conf self:test_namespace""" result = u'{{"test_resource_id": {{"test_namespace": {{"heartbeat": {{"timestamp": {}}}}}}}}}'.format( int(mock_time.return_value)) enrichment_cache = PanoptesEnrichmentCache(self._panoptes_context, self._plugin_conf, self._panoptes_resource) self.assertEqual(ordered(enrichment_cache.__dict__[u'_enrichment_data']), ordered(json.loads(result)))
def test_enrichment_cache_preload01(self): """Test enrichment resource with preload conf self:test_namespace""" result = '{"test_resource_id": {"test_namespace": {"heartbeat": {"timestamp": 1521668419}}}}' enrichment_cache = PanoptesEnrichmentCache(self._panoptes_context, self._plugin_conf, self._panoptes_resource) self.assertEqual( ordered(enrichment_cache.__dict__['_enrichment_data']), ordered(json.loads(result)))
def test_enrichment_cache_parse_conf(self): """Test enrichment resource parse conf""" result01 = [(u'asn_api', u'*'), (u'self', u'test_namespace'), (u'test_resource_id01', u'interface')] plugin_conf = { u'Core': { u'name': u'Test Plugin 01' }, u'enrichment': { u'preload': u'self:test_namespace, asn_api:*, test_resource_id01:interface' } } enrichment_cache = PanoptesEnrichmentCache(self._panoptes_context, plugin_conf, self._panoptes_resource) self.assertEqual(sorted(result01), sorted(enrichment_cache.__dict__[u'_preload_conf'])) plugin_conf_with_dup = { u'Core': { u'name': u'Test Plugin 01' }, u'enrichment': { u'preload': u'self:test_namespace, asn_api:*, ' u'test_resource_id01:interface, asn_api:*, ' u'test_resource_id01:interface' } } enrichment_cache = PanoptesEnrichmentCache(self._panoptes_context, plugin_conf_with_dup, self._panoptes_resource) self.assertEqual(sorted(result01), sorted(enrichment_cache.__dict__[u'_preload_conf']))
def set_enrichment_cache(self): self._enrichment_kv = self._panoptes_context.get_kv_store( PanoptesEnrichmentCacheKeyValueStore) enrichment_data_file = os.path.join(os.path.abspath(self.path), self._enrichment_data_file) if self._plugin_conf.get('enrichment'): self._enrichment_cache = PanoptesEnrichmentCache( self._panoptes_context, self._plugin_conf, self._panoptes_resource) try: with open(enrichment_data_file) as enrichment_data: for line in enrichment_data: key, value = line.strip().split('=>') self._enrichment_kv.set(key, value) except Exception as e: raise IOError( 'Failed to load enrichment data file {}: {}'.format( enrichment_data_file, repr(e)))
def test_enrichment_cache_parse_conf_bad(self): """Test enrichment resource parse bad conf""" plugin_conf01 = {u'Core': {u'name': u'Test Plugin 01'}, u'enrichment': {} } with self.assertRaises(PanoptesEnrichmentCacheError): PanoptesEnrichmentCache(self._panoptes_context, plugin_conf01, self._panoptes_resource) plugin_conf02 = {u'Core': {u'name': u'Test Plugin 01'}, u'enrichment': {u'preload': {}} } with self.assertRaises(PanoptesEnrichmentCacheError): PanoptesEnrichmentCache(self._panoptes_context, plugin_conf02, self._panoptes_resource) plugin_conf03 = {u'Core': {u'name': u'Test Plugin 01'}, u'enrichment': {u'preload': {u'self:test_namespace, asn_api:'}} } with self.assertRaises(PanoptesEnrichmentCacheError): PanoptesEnrichmentCache(self._panoptes_context, plugin_conf03, self._panoptes_resource) plugin_conf04 = {u'Core': {u'name': u'Test Plugin 01'}, u'enrichment': {u'preload': {u'self:test_namespace, :'}} } with self.assertRaises(PanoptesEnrichmentCacheError): PanoptesEnrichmentCache(self._panoptes_context, plugin_conf04, self._panoptes_resource) plugin_conf05 = {u'Core': {u'name': u'Test Plugin 01'}, u'enrichment': {u'preload': {u'self:test_namespace, '}} } with self.assertRaises(PanoptesEnrichmentCacheError): PanoptesEnrichmentCache(self._panoptes_context, plugin_conf05, self._panoptes_resource) plugin_conf06 = {u'Core': {u'name': u'Test Plugin 01'}, u'enrichment': {u'preload': {u'self:test_namespace, :namespace01'}} } with self.assertRaises(PanoptesEnrichmentCacheError): PanoptesEnrichmentCache(self._panoptes_context, plugin_conf06, self._panoptes_resource)
def test_enrichment_cache_get_enrichment_value(self): """Test enrichment resource get_enrichment_value""" plugin_conf = { u'Core': { u'name': u'Test Plugin 01' }, u'enrichment': { u'preload': u'self:test_namespace, ' u'asn_api:*, ' u'test_resource_id01:interface, ' u'no_data_resource_id:test_namespace, ' u'test_resource_id01:no_data_namespace' } } enrichment_cache = PanoptesEnrichmentCache(self._panoptes_context, plugin_conf, self._panoptes_resource) result01 = {u'timestamp': int(mock_time.return_value)} enrich_data = enrichment_cache.get_enrichment_value( u'self', u'test_namespace', u'heartbeat') self.assertEqual(sorted(result01), sorted(enrich_data)) enrich_data = enrichment_cache.get_enrichment_value( u'test_resource_id', u'test_namespace', u'heartbeat') self.assertEqual(sorted(result01), sorted(enrich_data)) result02 = {u'data': 101} enrich_data = enrichment_cache.get_enrichment_value( u'asn_api', u'namespace01', u'asn_data01') self.assertEqual(sorted(result02), sorted(enrich_data)) result03 = { u'ifType': u'l3ipvlan', u'ifAlias': u'<not set>', u'ifPhysAddress': u'00:1f:a0:13:8c:15', u'ifDescr': u'Virtual Ethernet 209', u'ifName': u'Virtual Ethernet 209', u'ifMtu': u'1500' } enrich_data = enrichment_cache.get_enrichment_value( u'test_resource_id01', u'interface', u'1209') self.assertEqual(sorted(result03), sorted(enrich_data)) with self.assertRaises(PanoptesEnrichmentCacheError): enrichment_cache.get_enrichment_value(u'no_data_resource_id', u'test_namespace', u'no_data') with self.assertRaises(AssertionError): enrichment_cache.get_enrichment_value(u'no_data_resource_id', u'test_namespace', u'') with self.assertRaises(AssertionError): enrichment_cache.get_enrichment_value(u'no_data_resource_id', u'', u'no_data') with self.assertRaises(AssertionError): enrichment_cache.get_enrichment_value(u'', u'test_namespace', u'no_data') # Test fallback preload result04 = { u'ifType': u'l3ipvlan', u'ifAlias': u'<not set>', u'ifPhysAddress': u'00:1f:a0:13:8c:15', u'ifDescr': u'Virtual Ethernet 309', u'ifName': u'Virtual Ethernet 309', u'ifMtu': u'1500' } enrich_data = enrichment_cache.get_enrichment_value( u'test_resource_id02', u'interface', u'2209') self.assertEqual(sorted(result04), sorted(enrich_data))
def test_enrichment_cache_get_enrichment(self): """Test enrichment resource get""" plugin_conf = { 'Core': { 'name': 'Test Plugin 01' }, 'enrichment': { 'preload': 'self:test_namespace, ' 'asn_api:*, ' 'test_resource_id01:interface, ' 'no_data_resource_id:test_namespace, ' 'test_resource_id01:no_data_namespace' } } enrichment_cache = PanoptesEnrichmentCache(self._panoptes_context, plugin_conf, self._panoptes_resource) result01 = {u'heartbeat': {u'timestamp': 1521668419}} enrichment_data = enrichment_cache.get_enrichment( 'test_resource_id', 'test_namespace') self.assertEqual(sorted(list(enrichment_data)), sorted(result01)) enrichment_data = enrichment_cache.get_enrichment( 'self', 'test_namespace') self.assertEqual(sorted(list(enrichment_data)), sorted(result01)) result02 = { u'1209': { u'ifType': u'l3ipvlan', u'ifAlias': u'<not set>', u'ifPhysAddress': u'00:1f:a0:13:8c:15', u'ifDescr': u'Virtual Ethernet 209', u'ifName': u'Virtual Ethernet 209', u'ifMtu': u'1500' }, u'1226': { u'ifType': u'l3ipvlan', u'ifAlias': u'<not set>', u'ifPhysAddress': u'00:1f:a0:13:8c:16', u'ifDescr': u'Virtual Ethernet 226', u'ifName': u'Virtual Ethernet 226', u'ifMtu': u'1500' } } enrichment_data = enrichment_cache.get_enrichment( 'test_resource_id01', 'interface') self.assertEqual(sorted(list(enrichment_data)), sorted(result02)) with self.assertRaises(PanoptesEnrichmentCacheError): enrichment_cache.get_enrichment('no_data_resource_id', 'test_namespace') with self.assertRaises(PanoptesEnrichmentCacheError): enrichment_cache.get_enrichment('test_resource_id01', 'no_data_namespace') with self.assertRaises(AssertionError): enrichment_cache.get_enrichment('test_resource_id01', '') with self.assertRaises(AssertionError): enrichment_cache.get_enrichment('', 'test_namespace') with self.assertRaises(AssertionError): enrichment_cache.get_enrichment('', '*') # Test fallback preload result03 = { u'2209': { u'ifType': u'l3ipvlan', u'ifAlias': u'<not set>', u'ifPhysAddress': u'00:1f:a0:13:8c:15', u'ifDescr': u'Virtual Ethernet 309', u'ifName': u'Virtual Ethernet 309', u'ifMtu': u'1500' }, u'2226': { u'ifType': u'l3ipvlan', u'ifAlias': u'<not set>', u'ifPhysAddress': u'00:1f:a0:13:8c:16', u'ifDescr': u'Virtual Ethernet 326', u'ifName': u'Virtual Ethernet 326', u'ifMtu': u'1500' } } enrichment_data03 = enrichment_cache.get_enrichment( 'test_resource_id02', 'interface') self.assertEqual(sorted(list(enrichment_data03)), sorted(result03))