def test_enrichment_cache_preload02(self):
"""Test enrichment resource with preload conf"""
plugin_conf = {
u'Core': {
u'name': u'Test Plugin 01'
},
u'enrichment': {
u'preload':
u'self:test_namespace, '
u'asn_api:*, '
u'test_resource_id01:interface, '
u'no_data_resource_id:test_namespace, '
u'test_resource_id01:no_data_namespace'
}
}
results01 = u"""{{"asn_api": {{"namespace02": {{"asn_data02": {{"data": 202}}}}, "namespace03": {{"asn_data03":
{{"data": 303}}}}, "namespace01": {{"asn_data01": {{"data": 101}}}}}}, "test_resource_id": {{"test_namespace":
{{"heartbeat": {{"timestamp": {}}}}}}}, "test_resource_id01": {{"interface": {{"1226": {{"ifType": "l3ipvlan",
"ifAlias": "<not set>", "ifPhysAddress": "00:1f:a0:13:8c:16", "ifDescr": "Virtual Ethernet 226", "ifName":
"Virtual Ethernet 226", "ifMtu": "1500"}}, "1209": {{"ifType": "l3ipvlan", "ifAlias": "<not set>",
"ifPhysAddress": "00:1f:a0:13:8c:15", "ifDescr": "Virtual Ethernet 209", "ifName": "Virtual Ethernet 209",
"ifMtu": "1500"}}}}}}}}""".format(int(mock_time.return_value))
enrichment_cache = PanoptesEnrichmentCache(self._panoptes_context,
plugin_conf,
self._panoptes_resource)
self.assertEqual(
ordered(enrichment_cache.__dict__[u'_enrichment_data']),
ordered(json.loads(results01)))
# Test an error while scanning kv store is handled correctly
mock_find_keys = Mock(side_effect=Exception)
mock_logger = MockLogger()
with patch(
u'yahoo_panoptes.framework.enrichment.PanoptesEnrichmentCacheKeyValueStore.find_keys',
mock_find_keys):
with patch(
u'yahoo_panoptes.framework.enrichment.PanoptesContext.logger',
mock_logger):
PanoptesEnrichmentCache(self._panoptes_context, plugin_conf,
self._panoptes_resource)
self.assertEqual(mock_logger.error.call_count, 3)
mock_kv_store_get = Mock(side_effect=IOError)
with patch(
u'yahoo_panoptes.framework.enrichment.PanoptesEnrichmentCacheKeyValueStore.get',
mock_kv_store_get):
with self.assertRaises(IOError):
PanoptesEnrichmentCache(self._panoptes_context, plugin_conf,
self._panoptes_resource)
def test_enrichment_cache(self):
"""Test enrichment resource attributes"""
with self.assertRaises(AssertionError):
PanoptesEnrichmentCache('non_panoptes_context', self._plugin_conf,
self._panoptes_resource)
with self.assertRaises(AssertionError):
PanoptesEnrichmentCache(self._panoptes_context, 'non_plugin_conf',
self._panoptes_resource)
with self.assertRaises(AssertionError):
PanoptesEnrichmentCache(self._panoptes_context, self._plugin_conf,
'non_panoptes_resource')
def test_enrichment_cache_get_enrichment_keys(self):
"""Test enrichment resource get_enrichment_keys"""
plugin_conf = {u'Core': {u'name': u'Test Plugin 01'},
u'enrichment': {u'preload': u'self:test_namespace, '
u'asn_api:*, '
u'test_resource_id01:interface, '
u'no_data_resource_id:test_namespace, '
u'test_resource_id01:no_data_namespace'}
}
enrichment_cache = PanoptesEnrichmentCache(self._panoptes_context,
plugin_conf,
self._panoptes_resource)
result01 = [u'1226', u'1209']
enrichment_data = enrichment_cache.get_enrichment_keys(u'test_resource_id01', u'interface')
self.assertListEqual(enrichment_data, result01)
with self.assertRaises(PanoptesEnrichmentCacheError):
enrichment_cache.get_enrichment_keys(u'test_resource_id01', u'no_data_namespace')
result02 = [u'heartbeat']
enrichment_data = enrichment_cache.get_enrichment_keys(u'test_resource_id', u'test_namespace')
self.assertListEqual(enrichment_data, result02)
enrichment_data = enrichment_cache.get_enrichment_keys(u'self', u'test_namespace')
self.assertListEqual(enrichment_data, result02)
# Test fallback preload
result03 = [u'2226', u'2209']
enrichment_data03 = enrichment_cache.get_enrichment_keys(u'test_resource_id02', u'interface')
self.assertListEqual(enrichment_data03, result03)
def _get_context(self, plugin):
self._enrichment = None
if plugin.config.get(u'enrichment'):
try:
self._enrichment = PanoptesEnrichmentCache(
self._panoptes_context, plugin.config, self._plugin_data)
except Exception as e:
raise PanoptesEnrichmentCacheError(
u'Error while creating PanoptesEnrichmentResource object for plugin '
u'{}: {}, skipping run'.format(plugin.name, repr(e)))
if self._enrichment is None:
raise PanoptesEnrichmentCacheError(
u'No enrichments found for plugin {} (configured {}), '
u'skipping run'.format(plugin.name,
plugin.config.get(u'enrichment')))
return PanoptesPluginWithEnrichmentContext(
panoptes_context=self._panoptes_context,
logger_name=self._plugin_logger_name,
config=plugin.config,
key_value_store=self._panoptes_context.get_kv_store(
self._plugin_kv_store_class),
secrets_store=self._panoptes_context.get_kv_store(
self._plugin_secrets_store_class),
data=self._plugin_data,
enrichment=self._enrichment)
def set_enrichment_cache(self, resource_endpoint=None):
if self.use_enrichment:
self._enrichment_data_file = 'data/' + self.enrichment_data_file
self._enrichment_kv = self._panoptes_context.get_kv_store(
PanoptesEnrichmentCacheKeyValueStore)
enrichment_data_file = os.path.join(os.path.abspath(self.path),
self._enrichment_data_file)
if self._plugin_conf.get('enrichment'):
if self._plugin_conf['enrichment'].get('preload'):
self._enrichment_cache = PanoptesEnrichmentCache(
self._panoptes_context, self._plugin_conf,
self._panoptes_resource)
try:
with open(enrichment_data_file) as enrichment_data:
for line in enrichment_data:
key, value = line.strip().split('=>')
if resource_endpoint is not None:
value = self.update_enrichment_ip(
value, resource_endpoint)
self._enrichment_kv.set(key, value)
except Exception as e:
raise IOError(
'Failed to load enrichment data file {}: {}'.format(
enrichment_data_file, repr(e)))
def test_enrichment_cache_preload02(self):
"""Test enrichment resource with preload conf"""
plugin_conf = {
'Core': {
'name': 'Test Plugin 01'
},
'enrichment': {
'preload':
'self:test_namespace, '
'asn_api:*, '
'test_resource_id01:interface, '
'no_data_resource_id:test_namespace, '
'test_resource_id01:no_data_namespace'
}
}
results01 = """{"asn_api": {"namespace02": {"asn_data02": {"data": 202}}, "namespace03": {"asn_data03":
{"data": 303}}, "namespace01": {"asn_data01": {"data": 101}}}, "test_resource_id": {"test_namespace":
{"heartbeat": {"timestamp": 1521668419}}}, "test_resource_id01": {"interface": {"1226": {"ifType": "l3ipvlan",
"ifAlias": "<not set>", "ifPhysAddress": "00:1f:a0:13:8c:16", "ifDescr": "Virtual Ethernet 226", "ifName":
"Virtual Ethernet 226", "ifMtu": "1500"}, "1209": {"ifType": "l3ipvlan", "ifAlias": "<not set>",
"ifPhysAddress": "00:1f:a0:13:8c:15", "ifDescr": "Virtual Ethernet 209", "ifName": "Virtual Ethernet 209",
"ifMtu": "1500"}}}}"""
enrichment_cache = PanoptesEnrichmentCache(self._panoptes_context,
plugin_conf,
self._panoptes_resource)
self.assertEqual(
ordered(enrichment_cache.__dict__['_enrichment_data']),
ordered(json.loads(results01)))
def test_enrichment_cache(self):
"""Test enrichment resource attributes"""
with self.assertRaises(AssertionError):
PanoptesEnrichmentCache(u'non_panoptes_context', self._plugin_conf, self._panoptes_resource)
with self.assertRaises(AssertionError):
PanoptesEnrichmentCache(self._panoptes_context, u'non_plugin_conf', self._panoptes_resource)
with self.assertRaises(AssertionError):
PanoptesEnrichmentCache(self._panoptes_context, self._plugin_conf, u'non_panoptes_resource')
# Test with bad key-value store
mock_panoptes_enrichment_cache_key_value_store = Mock(side_effect=Exception)
with patch(u'yahoo_panoptes.framework.enrichment.PanoptesEnrichmentCacheKeyValueStore',
mock_panoptes_enrichment_cache_key_value_store):
with self.assertRaises(Exception):
PanoptesEnrichmentCache(self._panoptes_context, self._plugin_conf, self._panoptes_resource)
def test_enrichment_cache_preload01(self):
"""Test enrichment resource with preload conf self:test_namespace"""
result = u'{{"test_resource_id": {{"test_namespace": {{"heartbeat": {{"timestamp": {}}}}}}}}}'.format(
int(mock_time.return_value))
enrichment_cache = PanoptesEnrichmentCache(self._panoptes_context,
self._plugin_conf,
self._panoptes_resource)
self.assertEqual(ordered(enrichment_cache.__dict__[u'_enrichment_data']),
ordered(json.loads(result)))
def test_enrichment_cache_preload01(self):
"""Test enrichment resource with preload conf self:test_namespace"""
result = '{"test_resource_id": {"test_namespace": {"heartbeat": {"timestamp": 1521668419}}}}'
enrichment_cache = PanoptesEnrichmentCache(self._panoptes_context,
self._plugin_conf,
self._panoptes_resource)
self.assertEqual(
ordered(enrichment_cache.__dict__['_enrichment_data']),
ordered(json.loads(result)))
def test_enrichment_cache_parse_conf(self):
"""Test enrichment resource parse conf"""
result01 = [(u'asn_api', u'*'), (u'self', u'test_namespace'),
(u'test_resource_id01', u'interface')]
plugin_conf = {
u'Core': {
u'name': u'Test Plugin 01'
},
u'enrichment': {
u'preload':
u'self:test_namespace, asn_api:*, test_resource_id01:interface'
}
}
enrichment_cache = PanoptesEnrichmentCache(self._panoptes_context,
plugin_conf,
self._panoptes_resource)
self.assertEqual(sorted(result01),
sorted(enrichment_cache.__dict__[u'_preload_conf']))
plugin_conf_with_dup = {
u'Core': {
u'name': u'Test Plugin 01'
},
u'enrichment': {
u'preload':
u'self:test_namespace, asn_api:*, '
u'test_resource_id01:interface, asn_api:*, '
u'test_resource_id01:interface'
}
}
enrichment_cache = PanoptesEnrichmentCache(self._panoptes_context,
plugin_conf_with_dup,
self._panoptes_resource)
self.assertEqual(sorted(result01),
sorted(enrichment_cache.__dict__[u'_preload_conf']))
def set_enrichment_cache(self):
self._enrichment_kv = self._panoptes_context.get_kv_store(
PanoptesEnrichmentCacheKeyValueStore)
enrichment_data_file = os.path.join(os.path.abspath(self.path),
self._enrichment_data_file)
if self._plugin_conf.get('enrichment'):
self._enrichment_cache = PanoptesEnrichmentCache(
self._panoptes_context, self._plugin_conf,
self._panoptes_resource)
try:
with open(enrichment_data_file) as enrichment_data:
for line in enrichment_data:
key, value = line.strip().split('=>')
self._enrichment_kv.set(key, value)
except Exception as e:
raise IOError(
'Failed to load enrichment data file {}: {}'.format(
enrichment_data_file, repr(e)))
def test_enrichment_cache_parse_conf_bad(self):
"""Test enrichment resource parse bad conf"""
plugin_conf01 = {u'Core': {u'name': u'Test Plugin 01'},
u'enrichment': {}
}
with self.assertRaises(PanoptesEnrichmentCacheError):
PanoptesEnrichmentCache(self._panoptes_context, plugin_conf01, self._panoptes_resource)
plugin_conf02 = {u'Core': {u'name': u'Test Plugin 01'},
u'enrichment': {u'preload': {}}
}
with self.assertRaises(PanoptesEnrichmentCacheError):
PanoptesEnrichmentCache(self._panoptes_context, plugin_conf02, self._panoptes_resource)
plugin_conf03 = {u'Core': {u'name': u'Test Plugin 01'},
u'enrichment': {u'preload': {u'self:test_namespace, asn_api:'}}
}
with self.assertRaises(PanoptesEnrichmentCacheError):
PanoptesEnrichmentCache(self._panoptes_context, plugin_conf03, self._panoptes_resource)
plugin_conf04 = {u'Core': {u'name': u'Test Plugin 01'},
u'enrichment': {u'preload': {u'self:test_namespace, :'}}
}
with self.assertRaises(PanoptesEnrichmentCacheError):
PanoptesEnrichmentCache(self._panoptes_context, plugin_conf04, self._panoptes_resource)
plugin_conf05 = {u'Core': {u'name': u'Test Plugin 01'},
u'enrichment': {u'preload': {u'self:test_namespace, '}}
}
with self.assertRaises(PanoptesEnrichmentCacheError):
PanoptesEnrichmentCache(self._panoptes_context, plugin_conf05, self._panoptes_resource)
plugin_conf06 = {u'Core': {u'name': u'Test Plugin 01'},
u'enrichment': {u'preload': {u'self:test_namespace, :namespace01'}}
}
with self.assertRaises(PanoptesEnrichmentCacheError):
PanoptesEnrichmentCache(self._panoptes_context, plugin_conf06, self._panoptes_resource)
def test_enrichment_cache_get_enrichment_value(self):
"""Test enrichment resource get_enrichment_value"""
plugin_conf = {
u'Core': {
u'name': u'Test Plugin 01'
},
u'enrichment': {
u'preload':
u'self:test_namespace, '
u'asn_api:*, '
u'test_resource_id01:interface, '
u'no_data_resource_id:test_namespace, '
u'test_resource_id01:no_data_namespace'
}
}
enrichment_cache = PanoptesEnrichmentCache(self._panoptes_context,
plugin_conf,
self._panoptes_resource)
result01 = {u'timestamp': int(mock_time.return_value)}
enrich_data = enrichment_cache.get_enrichment_value(
u'self', u'test_namespace', u'heartbeat')
self.assertEqual(sorted(result01), sorted(enrich_data))
enrich_data = enrichment_cache.get_enrichment_value(
u'test_resource_id', u'test_namespace', u'heartbeat')
self.assertEqual(sorted(result01), sorted(enrich_data))
result02 = {u'data': 101}
enrich_data = enrichment_cache.get_enrichment_value(
u'asn_api', u'namespace01', u'asn_data01')
self.assertEqual(sorted(result02), sorted(enrich_data))
result03 = {
u'ifType': u'l3ipvlan',
u'ifAlias': u'<not set>',
u'ifPhysAddress': u'00:1f:a0:13:8c:15',
u'ifDescr': u'Virtual Ethernet 209',
u'ifName': u'Virtual Ethernet 209',
u'ifMtu': u'1500'
}
enrich_data = enrichment_cache.get_enrichment_value(
u'test_resource_id01', u'interface', u'1209')
self.assertEqual(sorted(result03), sorted(enrich_data))
with self.assertRaises(PanoptesEnrichmentCacheError):
enrichment_cache.get_enrichment_value(u'no_data_resource_id',
u'test_namespace',
u'no_data')
with self.assertRaises(AssertionError):
enrichment_cache.get_enrichment_value(u'no_data_resource_id',
u'test_namespace', u'')
with self.assertRaises(AssertionError):
enrichment_cache.get_enrichment_value(u'no_data_resource_id', u'',
u'no_data')
with self.assertRaises(AssertionError):
enrichment_cache.get_enrichment_value(u'', u'test_namespace',
u'no_data')
# Test fallback preload
result04 = {
u'ifType': u'l3ipvlan',
u'ifAlias': u'<not set>',
u'ifPhysAddress': u'00:1f:a0:13:8c:15',
u'ifDescr': u'Virtual Ethernet 309',
u'ifName': u'Virtual Ethernet 309',
u'ifMtu': u'1500'
}
enrich_data = enrichment_cache.get_enrichment_value(
u'test_resource_id02', u'interface', u'2209')
self.assertEqual(sorted(result04), sorted(enrich_data))
def test_enrichment_cache_get_enrichment(self):
"""Test enrichment resource get"""
plugin_conf = {
'Core': {
'name': 'Test Plugin 01'
},
'enrichment': {
'preload':
'self:test_namespace, '
'asn_api:*, '
'test_resource_id01:interface, '
'no_data_resource_id:test_namespace, '
'test_resource_id01:no_data_namespace'
}
}
enrichment_cache = PanoptesEnrichmentCache(self._panoptes_context,
plugin_conf,
self._panoptes_resource)
result01 = {u'heartbeat': {u'timestamp': 1521668419}}
enrichment_data = enrichment_cache.get_enrichment(
'test_resource_id', 'test_namespace')
self.assertEqual(sorted(list(enrichment_data)), sorted(result01))
enrichment_data = enrichment_cache.get_enrichment(
'self', 'test_namespace')
self.assertEqual(sorted(list(enrichment_data)), sorted(result01))
result02 = {
u'1209': {
u'ifType': u'l3ipvlan',
u'ifAlias': u'<not set>',
u'ifPhysAddress': u'00:1f:a0:13:8c:15',
u'ifDescr': u'Virtual Ethernet 209',
u'ifName': u'Virtual Ethernet 209',
u'ifMtu': u'1500'
},
u'1226': {
u'ifType': u'l3ipvlan',
u'ifAlias': u'<not set>',
u'ifPhysAddress': u'00:1f:a0:13:8c:16',
u'ifDescr': u'Virtual Ethernet 226',
u'ifName': u'Virtual Ethernet 226',
u'ifMtu': u'1500'
}
}
enrichment_data = enrichment_cache.get_enrichment(
'test_resource_id01', 'interface')
self.assertEqual(sorted(list(enrichment_data)), sorted(result02))
with self.assertRaises(PanoptesEnrichmentCacheError):
enrichment_cache.get_enrichment('no_data_resource_id',
'test_namespace')
with self.assertRaises(PanoptesEnrichmentCacheError):
enrichment_cache.get_enrichment('test_resource_id01',
'no_data_namespace')
with self.assertRaises(AssertionError):
enrichment_cache.get_enrichment('test_resource_id01', '')
with self.assertRaises(AssertionError):
enrichment_cache.get_enrichment('', 'test_namespace')
with self.assertRaises(AssertionError):
enrichment_cache.get_enrichment('', '*')
# Test fallback preload
result03 = {
u'2209': {
u'ifType': u'l3ipvlan',
u'ifAlias': u'<not set>',
u'ifPhysAddress': u'00:1f:a0:13:8c:15',
u'ifDescr': u'Virtual Ethernet 309',
u'ifName': u'Virtual Ethernet 309',
u'ifMtu': u'1500'
},
u'2226': {
u'ifType': u'l3ipvlan',
u'ifAlias': u'<not set>',
u'ifPhysAddress': u'00:1f:a0:13:8c:16',
u'ifDescr': u'Virtual Ethernet 326',
u'ifName': u'Virtual Ethernet 326',
u'ifMtu': u'1500'
}
}
enrichment_data03 = enrichment_cache.get_enrichment(
'test_resource_id02', 'interface')
self.assertEqual(sorted(list(enrichment_data03)), sorted(result03))