class TestValidators(unittest.TestCase):
def setUp(self):
self._panoptes_metric = {
u'metrics_group_interval':
60,
u'resource': {
u'resource_site': u'test_site',
u'resource_id': u'test_id',
u'resource_class': u'network',
u'resource_plugin': u'test_plugin',
u'resource_creation_timestamp': 1567823517.46,
u'resource_subclass': u'test_subclass',
u'resource_endpoint': u'test_endpoint',
u'resource_metadata': {
u'test_metadata_key': u'test_metadata_value',
u'_resource_ttl': u"604800"
},
u'resource_type': u'test_type'
},
u'dimensions': [{
u'dimension_name': u'cpu_name',
u'dimension_value': u'test_cpu_name_value'
}, {
u'dimension_name': u'cpu_no',
u'dimension_value': u'test_cpu_no_value'
}, {
u'dimension_name': u'cpu_type',
u'dimension_value': u'test_cpu_type_value'
}],
u'metrics_group_type':
u'cpu',
u'metrics': [{
u'metric_creation_timestamp': 1567823946.72,
u'metric_type': u'gauge',
u'metric_name': u'cpu_utilization',
u'metric_value': 0
}],
u'metrics_group_creation_timestamp':
1567823946.72,
u'metrics_group_schema_version':
u'0.2'
}
self._panoptes_resource = PanoptesResource(resource_site=u'test',
resource_class=u'test',
resource_subclass=u'test',
resource_type=u'test',
resource_id=u'test',
resource_endpoint=u'test',
resource_plugin=u'test')
self._panoptes_resource.add_metadata(u'test', u'test')
self._panoptes_resource_set = PanoptesResourceSet()
self._panoptes_resource_set.add(self._panoptes_resource)
def test_default_record(self):
self.assertTrue(
PanoptesConsumerRecordValidator.validate(
PanoptesConsumerTypes.METRICS, self._panoptes_metric))
self.assertTrue(
PanoptesConsumerRecordValidator.validate(
PanoptesConsumerTypes.PROCESSED, self._panoptes_metric))
self.assertTrue(
PanoptesConsumerRecordValidator.validate(
PanoptesConsumerTypes.RESOURCES,
json.loads(self._panoptes_resource_set.json)))
self.assertFalse(PanoptesConsumerRecordValidator.validate(5, {}))
def test_resource_throws(self):
argument_overrides = [(u'resource_site', 1), (u'resource_id', None),
(u'resource_class', 1.5),
(u'resource_plugin', {}),
(u'resource_creation_timestamp', u'1567823517'),
(u'resource_subclass', []),
(u'resource_endpoint', 123456.789)]
for (override_key, override_value) in argument_overrides:
schema_arguments = self._panoptes_metric.copy()
schema_arguments[u'resource'][override_key] = override_value
self.assertEquals(
PanoptesConsumerRecordValidator.validate_metrics(
schema_arguments), False)
class SNMPPluginTestFramework(object):
plugin_class = None
path = None
snmp_host = '127.0.0.1'
snmp_port = 10161
snmp_timeout = 10
snmp_retries = 1
snmp_max_repetitions = 10
snmp_community = 'public'
snmp_failure_timeout = 1
resource_id = 'test_id'
resource_endpoint = '127.0.0.1'
resource_plugin = 'dummy'
resource_site = 'test_site'
resource_class = 'network'
resource_subclass = 'test_subclass'
resource_type = 'test_type'
resource_backplane = None
resource_model = 'model'
results_data_file = 'results.json'
enrichment_data_file = 'enrichment_data'
use_enrichment = True
plugin_conf = {
'Core': {
'name': 'Test Plugin',
'module': 'test_plugin'
},
'main': {
'execute_frequency': '60',
'enrichment_ttl': '300',
'resource_filter': 'resource_class = "network"'
},
'snmp': {
'timeout': 10,
'retries': 1,
'non_repeaters': 1,
'max_repetitions': 25,
}
}
def __init__(self, test_name):
super(SNMPPluginTestFramework, self).__init__(test_name)
self._path = self.path
self._plugin_conf = self.plugin_conf
self._snmp_host = self.snmp_host
self._snmp_port = self.snmp_port
self._snmp_timeout = self.snmp_timeout
self._snmp_retries = self.snmp_retries
self._snmp_max_repetitions = self.snmp_max_repetitions
self._snmp_community = self.snmp_community
self._snmp_failure_timeout = self.snmp_failure_timeout
self._resource_id = self.resource_id
self._resource_endpoint = self.resource_endpoint
self._resource_plugin = self.resource_plugin
self._resource_site = self.resource_site
self._resource_class = self.resource_class
self._resource_subclass = self.resource_subclass
self._resource_type = self.resource_type
self._resource_backplane = self.resource_backplane
self._resource_model = self.resource_model
self._panoptes_context = None
self._panoptes_resource = None
self._snmp_conf = None
self._expected_results = None
self._results_data_file = None
self._enrichment_data_file = None
self._secret_store = None
self._plugin_context = None
self._plugin_context_bad = None
self._snmp_conf_bad = None
self._enrichment_kv = None
self._enrichment_cache = None
def set_panoptes_context(self):
panoptes_test_conf_file = os.path.join(
os.path.dirname(os.path.dirname(__file__)),
'config_files/test_panoptes_config.ini')
self._panoptes_context = PanoptesContext(
panoptes_test_conf_file,
key_value_store_class_list=[PanoptesEnrichmentCacheKeyValueStore])
def set_panoptes_resource(self):
self._panoptes_resource = PanoptesResource(
resource_site=self._resource_site,
resource_class=self._resource_class,
resource_subclass=self._resource_subclass,
resource_type=self._resource_type,
resource_id=self._resource_id,
resource_endpoint=self._resource_endpoint,
resource_plugin=self._resource_plugin)
self._panoptes_resource.resource_metadata['model'] = 'model'
if self._resource_backplane:
self._panoptes_resource.add_metadata('backplane',
self._resource_backplane)
def set_enrichment_cache(self):
if self.use_enrichment:
self._enrichment_data_file = 'data/' + self.enrichment_data_file
self._enrichment_kv = self._panoptes_context.get_kv_store(
PanoptesEnrichmentCacheKeyValueStore)
enrichment_data_file = os.path.join(os.path.abspath(self.path),
self._enrichment_data_file)
if self._plugin_conf.get('enrichment'):
if self._plugin_conf['enrichment'].get('preload'):
self._enrichment_cache = PanoptesEnrichmentCache(
self._panoptes_context, self._plugin_conf,
self._panoptes_resource)
try:
with open(enrichment_data_file) as enrichment_data:
for line in enrichment_data:
key, value = line.strip().split('=>')
self._enrichment_kv.set(key, value)
except Exception as e:
raise IOError(
'Failed to load enrichment data file {}: {}'.format(
enrichment_data_file, repr(e)))
def set_snmp_conf(self):
self._snmp_conf = self._panoptes_context.config_object.snmp_defaults
def set_secret_store(self):
self._secret_store = create_autospec(PanoptesSecretsStore,
instance=True,
spec_set=True)
self._secret_store.get_by_site.return_value = self._snmp_community
def set_plugin_context(self):
self._plugin_context = create_autospec(
PanoptesPluginWithEnrichmentContext,
instance=True,
spec_set=True,
data=self._panoptes_resource,
enrichment=self._enrichment_cache,
config=self._plugin_conf,
snmp=self._snmp_conf,
secrets=self._secret_store,
logger=logging.getLogger(__name__))
def set_snmp_conf_bad(self):
self._snmp_conf_bad = copy.copy(self._snmp_conf)
self._snmp_conf_bad['port'] += 1
self._snmp_conf_bad['timeout'] = self._snmp_failure_timeout
def set_plugin_context_bad(self):
self._plugin_context_bad = create_autospec(
PanoptesPluginWithEnrichmentContext,
instance=True,
spec_set=True,
data=self._panoptes_resource,
enrichment=self._enrichment_cache,
config=self._plugin_conf,
snmp=self._snmp_conf_bad,
secrets=self._secret_store,
logger=logging.getLogger(__name__))
def set_expected_results(self):
self._results_data_file = 'data/' + self.results_data_file
expected_result_file = os.path.join(os.path.abspath(self.path),
self._results_data_file)
self._expected_results = json.load(open(expected_result_file))
@patch('time.time', mock_time)
@patch('yahoo_panoptes.framework.resources.time', mock_time)
@patch('redis.StrictRedis', PanoptesMockRedis)
def setUp(self):
self.set_panoptes_context()
self.set_panoptes_resource()
self.set_enrichment_cache()
self.set_snmp_conf()
self.set_secret_store()
self.set_plugin_context()
self.set_snmp_conf_bad()
self.set_plugin_context_bad()
self.set_expected_results()
class TestEnrichmentFramework(unittest.TestCase):
@patch(u'yahoo_panoptes.framework.resources.time', mock_time)
def setUp(self):
self.__panoptes_resource = PanoptesResource(resource_site=u'test', resource_class=u'test',
resource_subclass=u'test',
resource_type=u'test', resource_id=u'test',
resource_endpoint=u'test',
resource_plugin=u'test')
self.__panoptes_resource.add_metadata(u'test', u'test')
def test_enrichment_set(self):
enrichment_set = PanoptesEnrichmentSet(u'int_001')
enrichment_set.add(u'speed', 1000)
enrichment_set.add(u'index', 0o01)
enrichment_set.add(u'status', u'up')
self.assertEquals(enrichment_set.key, u'int_001')
self.assertDictEqual(enrichment_set.value, {u'status': u'up', u'index': 1, u'speed': 1000})
self.assertEquals(len(enrichment_set), 3)
enrichment_set1 = PanoptesEnrichmentSet(u'int_002', {u'status': u'down', u'index': 2, u'speed': 1000})
self.assertEquals(enrichment_set1.key, u'int_002')
self.assertDictEqual(enrichment_set1.value, {u'status': u'down', u'index': 2, u'speed': 1000})
with self.assertRaises(AssertionError):
PanoptesEnrichmentSet(u'int_001', u'string')
with self.assertRaises(AssertionError):
PanoptesEnrichmentSet(u'int_001', 100)
def test_enrichment_schema_validator(self):
validator = PanoptesEnrichmentInterfaceSchemaValidator()
enrichment_set = PanoptesEnrichmentSet(u'int_001')
enrichment_set.add(u'speed', 1000)
enrichment_set.add(u'index', 0o01)
enrichment_set.add(u'status', u'up')
self.assertTrue(validator.validate(enrichment_set))
enrichment_set.add(u'status', 0o1)
self.assertFalse(validator.validate(enrichment_set))
@patch(u'time.time', mock_time)
def test_enrichment_group(self):
interface_validation_object = PanoptesEnrichmentInterfaceSchemaValidator()
neighbor_validation_object = PanoptesEnrichmentNeighborSchemaValidator()
interface_data = \
u'''{"data": [
{"int_001": {"index": 1, "speed": 1000, "status": "up"}},
{"int_002": {"index": 2, "speed": 1000, "status": "down"}}],
"metadata": {"_enrichment_group_creation_timestamp": %f, "_enrichment_ttl": 300, "_execute_frequency": 60},
"namespace": "interface"}''' % mock_time.return_value
neighbor_data = \
u'''{"data": [{"host_name": {"mac": "aa:bb:cc:dd:ee:ff", "property": "Test Property", "vlan_id": 501}}],
"metadata": {"_enrichment_group_creation_timestamp": %f, "_enrichment_ttl": 600, "_execute_frequency": 120},
"namespace": "neighbor"}''' % mock_time.return_value
with self.assertRaises(AssertionError):
PanoptesEnrichmentGroup(1, interface_validation_object, 300, 60)
with self.assertRaises(AssertionError):
PanoptesEnrichmentGroup(u'interface', u'non_validation_object', 300, 60)
with self.assertRaises(AssertionError):
PanoptesEnrichmentGroup(u'interface', interface_validation_object, u'300', 60)
with self.assertRaises(AssertionError):
PanoptesEnrichmentGroup(u'interface', interface_validation_object, 300, u'60')
with self.assertRaises(AssertionError):
PanoptesEnrichmentGroup(u'interface', interface_validation_object, 0, 60)
with self.assertRaises(AssertionError):
PanoptesEnrichmentGroup(u'interface', interface_validation_object, 300, 0)
with self.assertRaises(AssertionError):
PanoptesEnrichmentGroup(u'interface', interface_validation_object, 300, 60).\
add_enrichment_set(u'not_PanoptesEnrichmentSet_obj')
enrichment_set1 = PanoptesEnrichmentSet(u'int_001')
enrichment_set1.add(u'speed', 1000)
enrichment_set1.add(u'index', 0o01)
enrichment_set1.add(u'status', u'up')
enrichment_set2 = PanoptesEnrichmentSet(u'int_002')
enrichment_set2.add(u'speed', 1000)
enrichment_set2.add(u'index', 0o02)
enrichment_set2.add(u'status', u'down')
enrichment_group1 = PanoptesEnrichmentGroup(u'interface', interface_validation_object, 300, 60)
enrichment_group1.add_enrichment_set(enrichment_set1)
enrichment_group1.add_enrichment_set(enrichment_set2)
self.assertEqual(enrichment_group1.namespace, u'interface')
self.assertEqual(enrichment_group1.enrichment_ttl, 300)
self.assertEqual(enrichment_group1.execute_frequency, 60)
self.assertEqual(enrichment_group1.enrichment_group_creation_timestamp, mock_time.return_value)
self.assertEqual(ordered(json.loads(json.dumps(enrichment_group1.data, cls=PanoptesEnrichmentEncoder))),
ordered(json.loads(interface_data)[u'data']))
self.assertEqual(ordered(json.loads(enrichment_group1.json())), ordered(json.loads(interface_data)))
self.assertEquals(len(enrichment_group1), 2)
enrichment_set3 = PanoptesEnrichmentSet(u'int_002')
enrichment_set3.add(u'speed', 1000)
enrichment_set3.add(u'index', 0o02)
enrichment_set3.add(u'status', u'down')
self.assertEqual(ordered(json.loads(enrichment_group1.json())), ordered(json.loads(interface_data)))
self.assertEqual(ordered(enrichment_group1.metadata), ordered(json.loads(interface_data)[u'metadata']))
self.assertEquals(len(enrichment_group1), 2)
test_metadata = json.loads(interface_data)[u'metadata']
test_metadata[u'metadata_key'] = u'metadata_value'
enrichment_group1.upsert_metadata(u'metadata_key', u'metadata_value')
self.assertEqual(ordered(enrichment_group1.metadata), ordered(test_metadata))
enrichment_group1.upsert_metadata(u'ttl', 300)
with self.assertRaises(ValueError):
enrichment_group1.upsert_metadata(u'_enrichment_ttl', 300)
with self.assertRaises(AssertionError):
enrichment_group1.upsert_metadata(u'metadata', {})
with self.assertRaises(AssertionError):
enrichment_group1.upsert_metadata(u'metadata', [])
enrichment_set4 = PanoptesEnrichmentSet(u'host_name')
enrichment_set4.add(u'vlan_id', 501)
enrichment_set4.add(u'property', u'Test Property')
enrichment_set4.add(u'mac', u'aa:bb:cc:dd:ee:ff')
enrichment_group2 = PanoptesEnrichmentGroup(u'neighbor', neighbor_validation_object, 600, 120)
enrichment_group2.add_enrichment_set(enrichment_set4)
self.assertEqual(ordered(json.loads(enrichment_group2.json())), ordered(json.loads(neighbor_data)))
self.assertEquals(len(enrichment_group2), 1)
enrichment_set5 = PanoptesEnrichmentSet(u'host_name01')
enrichment_set5.add(u'vlan_id', 502)
enrichment_set5.add(u'property', u'Netops01.US')
enrichment_set6 = PanoptesEnrichmentSet(u'host_name02')
enrichment_set6.add(u'vlan_id', 503)
enrichment_set6.add(u'mac', u'aa:bb:cc:dd:ee:ff')
enrichment_group3 = PanoptesEnrichmentGroup(u'neighbor', neighbor_validation_object, 600, 120)
enrichment_group3.add_enrichment_set(enrichment_set5)
with self.assertRaises(AssertionError):
enrichment_group3.add_enrichment_set(enrichment_set6)
interface_store_data = u'{"int_001": {"index": 1, "speed": 1000, "status": "up"}, ' \
u'"int_002": {"index": 2, "speed": 1000, "status": "down"}}'
neighbor_store_data = u'{"host_name": {"mac": "aa:bb:cc:dd:ee:ff", ' \
u'"property": "Test Property", "vlan_id": 501}}'
self.assertEquals(ordered(json.loads(enrichment_group1.serialize_data())),
ordered(json.loads(interface_store_data)))
self.assertEquals(ordered(json.loads(enrichment_group2.serialize_data())),
ordered(json.loads(neighbor_store_data)))
enrichment_group1.upsert_metadata(u'ttl', 300)
with self.assertRaises(ValueError):
enrichment_group1.upsert_metadata(u'_enrichment_ttl', 300)
interface_data_serialized = u'''{{"data": {{"int_001": {{"index": 1, "speed": 1000, "status": "up"}},
"int_002": {{"index": 2, "speed": 1000, "status": "down"}}}}, "metadata":
{{"_enrichment_group_creation_timestamp": {:.5f}, "_enrichment_ttl": 300, "_execute_frequency": 60,
"metadata_key": "metadata_value", "ttl": 300}}}}'''.format(mock_time.return_value)
neighbor_data_serialized = u'''{{"data": {{"host_name": {{"mac": "aa:bb:cc:dd:ee:ff", "property":
"Test Property","vlan_id": 501}}}}, "metadata": {{"_enrichment_group_creation_timestamp": {:.5f},
"_enrichment_ttl": 600, "_execute_frequency": 120}}}}'''.format(mock_time.return_value)
self.assertEquals(ordered(json.loads(enrichment_group1.serialize())),
ordered(json.loads(interface_data_serialized)))
self.assertEquals(ordered(json.loads(enrichment_group2.serialize())),
ordered(json.loads(neighbor_data_serialized)))
@patch(u'time.time', mock_time)
def test_enrichment_group_set(self):
interface_validation_object = PanoptesEnrichmentInterfaceSchemaValidator()
neighbor_validation_object = PanoptesEnrichmentNeighborSchemaValidator()
panoptes_resource = self.__panoptes_resource
enrichment_data = \
u'''{{"enrichment": [{{"metadata": {{"_enrichment_group_creation_timestamp": {:.5f}, "_enrichment_ttl": 600,
"_execute_frequency": 120}}, "data": [{{"host_name":
{{"mac": "aa:bb:cc:dd:ee:ff", "property": "Test Property", "vlan_id": 501}}}}],
"namespace": "neighbor"}}, {{"metadata": {{"_enrichment_group_creation_timestamp": {:.5f},
"_enrichment_ttl": 300,
"_execute_frequency": 60}}, "data": [
{{"int_001": {{"index": 1, "speed": 1000, "status": "up"}}}}, {{"int_002": {{"index": 2, "speed": 1000,
"status": "down"}}}}], "namespace": "interface"}}],
"enrichment_group_set_creation_timestamp": {:.5f}, "resource": {{"resource_class": "test",
"resource_creation_timestamp": {:.5f}, "resource_endpoint": "test", "resource_id": "test",
"resource_metadata": {{"_resource_ttl": "604800", "test": "test"}}, "resource_plugin": "test",
"resource_site": "test",
"resource_subclass": "test", "resource_type": "test"}}}}'''.format(mock_time.return_value,
mock_time.return_value,
mock_time.return_value,
mock_time.return_value)
enrichment_set1 = PanoptesEnrichmentSet(u'int_001')
enrichment_set1.add(u'speed', 1000)
enrichment_set1.add(u'index', 0o01)
enrichment_set1.add(u'status', u'up')
enrichment_set2 = PanoptesEnrichmentSet(u'int_002')
enrichment_set2.add(u'speed', 1000)
enrichment_set2.add(u'index', 0o02)
enrichment_set2.add(u'status', u'down')
enrichment_group1 = PanoptesEnrichmentGroup(u'interface', interface_validation_object, 300, 60)
enrichment_group1.add_enrichment_set(enrichment_set1)
enrichment_group1.add_enrichment_set(enrichment_set2)
enrichment_set3 = PanoptesEnrichmentSet(u'host_name')
enrichment_set3.add(u'vlan_id', 501)
enrichment_set3.add(u'property', u'Test Property')
enrichment_set3.add(u'mac', u'aa:bb:cc:dd:ee:ff')
enrichment_group2 = PanoptesEnrichmentGroup(u'neighbor', neighbor_validation_object, 600, 120)
enrichment_group2.add_enrichment_set(enrichment_set3)
enrichment_group_set1 = PanoptesEnrichmentGroupSet(panoptes_resource)
enrichment_group_set1.add_enrichment_group(enrichment_group1)
enrichment_group_set1.add_enrichment_group(enrichment_group2)
self.assertEquals(len(enrichment_group_set1), 2)
group_set_repr = u"PanoptesEnrichmentGroupSet[resource:" \
u"plugin|test|site|test|class|test|subclass|test|type|test|id|test|endpoint|test," \
u"enrichment_group_set_creation_timestamp:{},PanoptesEnrichmentGroup[namespace:" \
u"interface,enrichment_ttl:300,execute_frequency:60,enrichment_group_creation_timestamp:{}," \
u"PanoptesEnrichmentSet[int_001[index:1,speed:1000,status:up]],PanoptesEnrichmentSet[" \
u"int_002[index:2,speed:1000,status:down]]],PanoptesEnrichmentGroup[namespace:neighbor," \
u"enrichment_ttl:600,execute_frequency:120,enrichment_group_creation_timestamp:{}," \
u"PanoptesEnrichmentSet[host_name[mac:aa:bb:cc:dd:ee:ff,property:" \
u"Test Property,vlan_id:501]]]]".format(mock_time.return_value,
mock_time.return_value,
mock_time.return_value)
self.assertEquals(repr(enrichment_group_set1), group_set_repr)
self.assertIsInstance(enrichment_group_set1.resource, PanoptesResource)
self.assertEqual(enrichment_group_set1.enrichment_group_set_creation_timestamp, mock_time.return_value)
self.assertEqual(
ordered(json.loads(json.dumps(enrichment_group_set1.enrichment, cls=PanoptesEnrichmentEncoder))),
ordered(json.loads(enrichment_data)[u'enrichment']))
self.assertEqual(ordered(json.loads(enrichment_group_set1.json())[u'enrichment']),
ordered(json.loads(enrichment_data)[u'enrichment']))
with self.assertRaises(AssertionError):
PanoptesEnrichmentGroupSet(u'bad_resource')
with self.assertRaises(AssertionError):
PanoptesEnrichmentGroupSet(panoptes_resource).add_enrichment_group(u'non_PanoptesEnrichmentGroup_obj')
enrichment_group_set2 = PanoptesEnrichmentGroupSet(panoptes_resource)
enrichment_group3 = PanoptesEnrichmentGroup(u'interface', interface_validation_object, 300, 60)
with self.assertRaises(AssertionError):
enrichment_group_set2.add_enrichment_group(enrichment_group3)
self.assertFalse(enrichment_group_set1 == enrichment_group1)
self.assertFalse(enrichment_group_set1 == enrichment_group_set2)
@patch(u'time.time', mock_time)
@patch(u'yahoo_panoptes.framework.resources.time', mock_time)
def test_multi_enrichment_group_set(self):
interface_validation_object = PanoptesEnrichmentInterfaceSchemaValidator()
neighbor_validation_object = PanoptesEnrichmentNeighborSchemaValidator()
panoptes_resource = self.__panoptes_resource
multi_enrichment_results_data = \
{
"group_sets": [
{
"enrichment": [
{
"data": [
{
"host_name": {
"mac": "aa:bb:cc:dd:ee:ff",
"property": "Test Property",
"vlan_id": 501
}
},
{
"host_name01": {
"mac": "aa:bb:cc:dd:ee:ff",
"property": "Test Property",
"vlan_id": 502
}
}
],
"metadata": {
"_enrichment_group_creation_timestamp": mock_time.return_value,
"_enrichment_ttl": 600,
"_execute_frequency": 120
},
"namespace": "neighbor"
},
{
"data": [
{
"int_001": {
"index": 1,
"speed": 1000,
"status": "up"
}
}
],
"metadata": {
"_enrichment_group_creation_timestamp": mock_time.return_value,
"_enrichment_ttl": 300,
"_execute_frequency": 60
},
"namespace": "interface"
}
],
"enrichment_group_set_creation_timestamp": mock_time.return_value,
"resource": {
"resource_class": "test_class",
"resource_creation_timestamp": mock_time.return_value,
"resource_endpoint": "test_endpoint01",
"resource_id": "test_resource_id01",
"resource_metadata": {
"_resource_ttl": "604800"
},
"resource_plugin": "test_plugin",
"resource_site": "test_site",
"resource_subclass": "test_subclass",
"resource_type": "test_type"
}
},
{
"enrichment": [
{
"data": [
{
"int_001": {
"index": 1,
"speed": 1000,
"status": "up"
}
}
],
"metadata": {
"_enrichment_group_creation_timestamp": mock_time.return_value,
"_enrichment_ttl": 300,
"_execute_frequency": 60
},
"namespace": "interface"
},
{
"data": [
{
"host_name": {
"mac": "aa:bb:cc:dd:ee:ff",
"property": "Test Property",
"vlan_id": 501
}
}
],
"metadata": {
"_enrichment_group_creation_timestamp": mock_time.return_value,
"_enrichment_ttl": 600,
"_execute_frequency": 120
},
"namespace": "neighbor"
}
],
"enrichment_group_set_creation_timestamp": mock_time.return_value,
"resource": {
"resource_class": "test",
"resource_creation_timestamp": mock_time.return_value,
"resource_endpoint": "test",
"resource_id": "test",
"resource_metadata": {
"_resource_ttl": "604800",
"test": "test"
},
"resource_plugin": "test",
"resource_site": "test",
"resource_subclass": "test",
"resource_type": "test"
}
}
]
}
enrichment_set1 = PanoptesEnrichmentSet(u'int_001')
enrichment_set1.add(u'speed', 1000)
enrichment_set1.add(u'index', 0o01)
enrichment_set1.add(u'status', u'up')
enrichment_group1 = PanoptesEnrichmentGroup(u'interface', interface_validation_object, 300, 60)
enrichment_group1.add_enrichment_set(enrichment_set1)
enrichment_set3 = PanoptesEnrichmentSet(u'host_name')
enrichment_set3.add(u'vlan_id', 501)
enrichment_set3.add(u'property', u'Test Property')
enrichment_set3.add(u'mac', u'aa:bb:cc:dd:ee:ff')
enrichment_group2 = PanoptesEnrichmentGroup(u'neighbor', neighbor_validation_object, 600, 120)
enrichment_group2.add_enrichment_set(enrichment_set3)
enrichment_group_set1 = PanoptesEnrichmentGroupSet(panoptes_resource)
enrichment_group_set1.add_enrichment_group(enrichment_group1)
enrichment_group_set1.add_enrichment_group(enrichment_group2)
panoptes_resource01 = PanoptesResource(resource_site=u'test_site',
resource_class=u'test_class',
resource_subclass=u'test_subclass',
resource_type=u'test_type',
resource_id=u'test_resource_id01',
resource_endpoint=u'test_endpoint01',
resource_plugin=u'test_plugin')
panoptes_resource02 = PanoptesResource(resource_site=u'test_site',
resource_class=u'test_class',
resource_subclass=u'test_subclass',
resource_type=u'test_type',
resource_id=u'test_resource_id02',
resource_endpoint=u'test_endpoint02',
resource_plugin=u'test_plugin')
enrichment_set4 = PanoptesEnrichmentSet(u'host_name01')
enrichment_set4.add(u'vlan_id', 502)
enrichment_set4.add(u'property', u'Test Property')
enrichment_set4.add(u'mac', u'aa:bb:cc:dd:ee:ff')
enrichment_group3 = PanoptesEnrichmentGroup(u'neighbor', neighbor_validation_object, 600, 120)
enrichment_group3.add_enrichment_set(enrichment_set3)
enrichment_group3.add_enrichment_set(enrichment_set4)
enrichment_group_set2 = PanoptesEnrichmentGroupSet(panoptes_resource01)
enrichment_group_set2.add_enrichment_group(enrichment_group1)
enrichment_group_set2.add_enrichment_group(enrichment_group3)
multi_enrichment_group_set = PanoptesEnrichmentMultiGroupSet()
multi_enrichment_group_set.add_enrichment_group_set(enrichment_group_set1)
multi_enrichment_group_set.add_enrichment_group_set(enrichment_group_set2)
multi_enrichment_group_set_repr = u"PanoptesEnrichmentMultiGroupSet[PanoptesEnrichmentGroupSet[resource:" \
u"plugin|test|site|test|class|test|subclass|test|type|test|id|test|endpoint" \
u"|test,enrichment_group_set_creation_timestamp:{},PanoptesEnrichmentGroup" \
u"[namespace:interface,enrichment_ttl:300,execute_frequency:60," \
u"enrichment_group_creation_timestamp:{},PanoptesEnrichmentSet[" \
u"int_001[index:1,speed:1000,status:up]]],PanoptesEnrichmentGroup[" \
u"namespace:neighbor,enrichment_ttl:600,execute_frequency:120," \
u"enrichment_group_creation_timestamp:{},PanoptesEnrichmentSet" \
u"[host_name[mac:aa:bb:cc:dd:ee:ff,property:Test Property,vlan_id:501]" \
u"]]],PanoptesEnrichmentGroupSet[resource:plugin|test_plugin|site|" \
u"test_site|class|test_class|subclass|test_subclass|type|test_type|id" \
u"|test_resource_id01|endpoint|test_endpoint01," \
u"enrichment_group_set_creation_timestamp:{},PanoptesEnrichmentGroup" \
u"[namespace:interface,enrichment_ttl:300,execute_frequency:60," \
u"enrichment_group_creation_timestamp:{},PanoptesEnrichmentSet" \
u"[int_001[index:1,speed:1000,status:up]]],PanoptesEnrichmentGroup" \
u"[namespace:neighbor,enrichment_ttl:600,execute_frequency:120," \
u"enrichment_group_creation_timestamp:{},PanoptesEnrichmentSet" \
u"[host_name[mac:aa:bb:cc:dd:ee:ff,property:Test Property,vlan_id:501]]," \
u"PanoptesEnrichmentSet[host_name01[mac:aa:bb:cc:dd:ee:ff," \
u"property:Test Property,vlan_id:502]]]]]".format(mock_time.return_value,
mock_time.return_value,
mock_time.return_value,
mock_time.return_value,
mock_time.return_value,
mock_time.return_value)
self.assertEquals(repr(multi_enrichment_group_set), multi_enrichment_group_set_repr)
self.assertEquals(len(multi_enrichment_group_set.enrichment_group_sets), 2)
self.assertEquals(ordered(json.loads(multi_enrichment_group_set.json())),
ordered(multi_enrichment_results_data))
self.assertEquals(len(multi_enrichment_group_set), 2)
multi_enrichment_group_set.add_enrichment_group_set(enrichment_group_set2)
self.assertEquals(len(multi_enrichment_group_set), 2)
enrichment_group_set3 = PanoptesEnrichmentGroupSet(panoptes_resource02)
enrichment_group_set3.add_enrichment_group(enrichment_group1)
multi_enrichment_group_set.add_enrichment_group_set(enrichment_group_set3)
self.assertEquals(len(multi_enrichment_group_set), 3)
with self.assertRaises(AssertionError):
multi_enrichment_group_set.add_enrichment_group_set(u'non_enrichment_group')
enrichment_group_set3 = PanoptesEnrichmentGroupSet(panoptes_resource01)
with self.assertRaises(AssertionError):
multi_enrichment_group_set.add_enrichment_group_set(enrichment_group_set3)
class TestMetrics(unittest.TestCase):
def setUp(self):
self.__panoptes_resource = PanoptesResource(resource_site=u'test',
resource_class=u'test',
resource_subclass=u'test',
resource_type=u'test',
resource_id=u'test',
resource_endpoint=u'test',
resource_plugin=u'test')
self.__panoptes_resource.add_metadata(u'test', u'test')
def test_panoptes_metric(self):
with self.assertRaises(AssertionError):
PanoptesMetric(None, 0, PanoptesMetricType.GAUGE)
with self.assertRaises(ValueError):
PanoptesMetric(u'1', 0, PanoptesMetricType.GAUGE)
with self.assertRaises(AssertionError):
PanoptesMetric(u'test_metric', None, PanoptesMetricType.GAUGE)
with self.assertRaises(AssertionError):
PanoptesMetric(u'test_metric', 0, None)
with self.assertRaises(AssertionError):
PanoptesMetric(u'test_metric', True, PanoptesMetricType.GAUGE)
metric1 = PanoptesMetric(
u'test_metric',
0,
PanoptesMetricType.GAUGE,
metric_creation_timestamp=mock_time.return_value)
self.assertEqual(metric1.metric_name, u'test_metric')
self.assertEqual(metric1.metric_value, 0)
self.assertEqual(metric1.metric_timestamp, mock_time.return_value)
self.assertEqual(metric1.metric_type, PanoptesMetricType.GAUGE)
self.assertEqual(
repr(metric1), u"PanoptesMetric[test_metric|0|GAUGE|{}]".format(
mock_time.return_value))
self.assertNotEqual(metric1, None)
# Check PanoptesMetric.__eq__
assert metric1 == PanoptesMetric(u'test_metric', 0,
PanoptesMetricType.GAUGE)
with self.assertRaises(AssertionError):
assert metric1 == PanoptesMetricDimension(u"test", u"value")
with self.assertRaises(AssertionError):
assert metric1 == PanoptesMetric(u'different_name', 0,
PanoptesMetricType.GAUGE)
with self.assertRaises(AssertionError):
assert metric1 == PanoptesMetric(u'test_metric', 1,
PanoptesMetricType.GAUGE)
with self.assertRaises(AssertionError):
assert metric1 == PanoptesMetric(u'test_metric', 0,
PanoptesMetricType.COUNTER)
@patch(u'yahoo_panoptes.framework.metrics.time', mock_time)
def test_panoptes_metric_json_and_repr(self):
metric = PanoptesMetric(u'test_metric', 0, PanoptesMetricType.GAUGE,
mock_time.return_value)
serialized = json.loads(metric.json)
expected = {
u"metric_creation_timestamp": mock_time.return_value,
u"metric_name": u"test_metric",
u"metric_type": u"gauge",
u"metric_value": 0
}
self.assertEqual(ordered(serialized), ordered(expected))
def testMetricsGroup(self):
now = round(time.time(), METRICS_TIMESTAMP_PRECISION)
metrics_group = PanoptesMetricsGroup(self.__panoptes_resource, u'test',
120)
self.assertEqual(metrics_group.group_type, u'test')
self.assertEqual(metrics_group.interval, 120)
self.assertEqual(metrics_group.schema_version, u'0.2')
self.assertGreaterEqual(metrics_group.creation_timestamp, now)
with patch(u'yahoo_panoptes.framework.metrics.time', mock_time):
metrics_group = PanoptesMetricsGroup(self.__panoptes_resource,
u'test', 120)
dimension_one = PanoptesMetricDimension(u'if_alias', u'bar')
dimension_two = PanoptesMetricDimension(u'if_alias', u'foo')
metrics_group.add_dimension(dimension_one)
with self.assertRaises(KeyError):
metrics_group.add_dimension(dimension_two)
# Test basic dimension operations
self.assertEqual(len(metrics_group.dimensions), 1)
self.assertTrue(
metrics_group.contains_dimension_by_name(u'if_alias'))
self.assertFalse(metrics_group.contains_dimension_by_name(u'baz'))
self.assertEqual(
metrics_group.get_dimension_by_name(u'if_alias').value, u'bar')
metrics_group.delete_dimension_by_name(u'if_alias')
self.assertFalse(
metrics_group.contains_dimension_by_name(u'if_alias'))
self.assertEqual(len(metrics_group.dimensions), 0)
self.assertEqual(metrics_group.get_dimension_by_name(u'foo'), None)
metrics_group.add_dimension(dimension_two)
dimension_three = PanoptesMetricDimension(u'if_alias', u'test')
metrics_group.upsert_dimension(dimension_three)
self.assertEqual(len(metrics_group.dimensions), 1)
self.assertEqual(
metrics_group.get_dimension_by_name(u'if_alias').value,
u'test')
dimension_four = PanoptesMetricDimension(u'if_name', u'eth0')
metrics_group.upsert_dimension(dimension_four)
self.assertEqual(len(metrics_group.dimensions), 2)
# Test basic metric operations
with self.assertRaises(AssertionError):
metrics_group.add_metric(None)
metric = PanoptesMetric(u'test_metric', 0,
PanoptesMetricType.GAUGE)
metrics_group.add_metric(metric)
with self.assertRaises(KeyError):
metrics_group.add_metric(metric)
to_json = metrics_group.json
metrics = PanoptesMetricsGroup.flatten_metrics(
json.loads(to_json)[u'metrics'])
self.assertEquals(metrics[u'gauge'][u'test_metric'][u'value'], 0)
metrics_group_two = PanoptesMetricsGroup(self.__panoptes_resource,
u'test', 120)
metrics_group_two.add_dimension(dimension_two)
metrics_group_two.upsert_dimension(dimension_three)
metrics_group_two.upsert_dimension(dimension_four)
metrics_group_two.add_metric(metric)
self.assertEqual(metrics_group, metrics_group_two)
# Check PanoptesMetricsGroup.__eq__
panoptes_resource_two = PanoptesResource(
resource_site=u'test2',
resource_class=u'test2',
resource_subclass=u'test2',
resource_type=u'test2',
resource_id=u'test2',
resource_endpoint=u'test2',
resource_plugin=u'test2')
metrics_group_two = PanoptesMetricsGroup(panoptes_resource_two,
u'test', 120)
metrics_group_three = PanoptesMetricsGroup(
self.__panoptes_resource, u'test', 120)
with self.assertRaises(AssertionError):
assert metrics_group_two == metrics_group_three
metrics_group_three = metrics_group.copy()
with self.assertRaises(AssertionError):
assert metrics_group == dimension_one
assert metrics_group == metrics_group_three
metrics_group_three.delete_dimension_by_name(u"if_name")
with self.assertRaises(AssertionError):
assert metrics_group == metrics_group_three
metrics_group_three.upsert_dimension(dimension_four)
assert metrics_group == metrics_group_three
metric_two = PanoptesMetric(u'test_metric_2', 1,
PanoptesMetricType.GAUGE)
metrics_group_three.add_metric(metric_two)
with self.assertRaises(AssertionError):
assert metrics_group == metrics_group_three
# Test PanoptesMetricsGroup.__repr__
_METRICS_GROUP_REPR = u'PanoptesMetricsGroup[' \
u'resource:plugin|test|site|test|class|test|subclass|test|type|test|id|' \
u'test|endpoint|test,' \
u'interval:120,schema_version:0.2,group_type:test,creation_timestamp:{},' \
u'dimensions:[PanoptesMetricDimension[if_alias|test],' \
u'PanoptesMetricDimension[if_name|eth0]],' \
u'metrics:[PanoptesMetric[test_metric|0|GAUGE|{}]]]'.format(mock_time.return_value,
mock_time.return_value)
self.assertEqual(repr(metrics_group), _METRICS_GROUP_REPR)
dimensions_as_dicts = [{
u'dimension_name': dimension.name,
u'dimension_value': dimension.value
} for dimension in metrics_group.dimensions]
self.assertEqual(
PanoptesMetricsGroup.flatten_dimensions(dimensions_as_dicts), {
u'if_alias': u'test',
u'if_name': u'eth0'
})
def test_panoptes_metrics_group_encoder(self):
test_dict = dict()
encoder = PanoptesMetricsGroupEncoder()
mock_default = Mock(json.JSONEncoder.default)
with patch(
u'yahoo_panoptes.framework.metrics.json.JSONEncoder.default',
mock_default):
encoder.default(test_dict)
mock_default.assert_called_once()
def test_panoptes_metric_dimension(self):
with self.assertRaises(ValueError):
PanoptesMetricDimension(u'contain$_invalid_character$', u'bar')
with self.assertRaises(ValueError):
PanoptesMetricDimension(u'foo', u'contains_pipe|')
dimension_one = PanoptesMetricDimension(u'if_alias', u'bar')
self.assertEqual(
dimension_one.json,
u'{"dimension_name": "if_alias", "dimension_value": "bar"}')
self.assertEqual(repr(dimension_one),
u'PanoptesMetricDimension[if_alias|bar]')
metric_one = PanoptesMetric(
u'test_metric',
0,
PanoptesMetricType.GAUGE,
metric_creation_timestamp=mock_time.return_value)
with self.assertRaises(AssertionError):
assert dimension_one == metric_one
dimension_two = PanoptesMetricDimension(u'if_alias', u'foo')
with self.assertRaises(AssertionError):
assert dimension_one == dimension_two
dimension_three = PanoptesMetricDimension(u'if_alias', u'bar')
assert dimension_one == dimension_three
def test_metrics_group_hash(self):
now = round(time.time(), METRICS_TIMESTAMP_PRECISION)
metrics_group = PanoptesMetricsGroup(self.__panoptes_resource, u'test',
120)
metrics_group_two = PanoptesMetricsGroup(self.__panoptes_resource,
u'test', 120)
dimension = PanoptesMetricDimension(u'if_alias', u'bar')
metric = PanoptesMetric(u'test_metric',
0,
PanoptesMetricType.GAUGE,
metric_creation_timestamp=now)
metric_diff_timestamp = PanoptesMetric(u'test_metric',
0,
PanoptesMetricType.GAUGE,
metric_creation_timestamp=now +
0.01)
metrics_group.add_dimension(dimension)
metrics_group_two.add_dimension(dimension)
self.assertEqual(metrics_group.__hash__(),
metrics_group_two.__hash__())
metrics_group.add_metric(metric)
metrics_group_two.add_metric(metric_diff_timestamp)
self.assertEqual(metrics_group.__hash__(),
metrics_group_two.__hash__())
@patch(u'yahoo_panoptes.framework.metrics.time', mock_time)
def test_panoptes_metrics_group_set(self):
"""Tests basic PanoptesMetricsGroupSet operations"""
metrics_group_set = PanoptesMetricsGroupSet()
metrics_group = PanoptesMetricsGroup(self.__panoptes_resource, u'test',
120)
metrics_group_two = PanoptesMetricsGroup(self.__panoptes_resource,
u'test', 120)
metrics_group_set.add(metrics_group)
metrics_group_set.add(metrics_group_two)
assert len(metrics_group_set) == 1
self.assertIn(metrics_group, metrics_group_set.metrics_groups)
metrics_group_set.remove(metrics_group_two)
assert len(metrics_group_set) == 0
metrics_group_set.add(metrics_group)
metrics_group_three = PanoptesMetricsGroup(self.__panoptes_resource,
u'test3', 120)
metrics_group_three.add_metric(
PanoptesMetric(u"test3", 0.0, PanoptesMetricType.GAUGE))
metrics_group_set.add(metrics_group_three)
assert len(metrics_group_set) == 2
metrics_group_set_two = PanoptesMetricsGroupSet()
metrics_group_four = PanoptesMetricsGroup(self.__panoptes_resource,
u'test', 120)
metrics_group_four.add_metric(
PanoptesMetric(u"test4", 0.0, PanoptesMetricType.GAUGE))
metrics_group_set_two.add(metrics_group_four)
assert len(metrics_group_set_two) == 1
# Test PanoptesMetricsGroupSet.__add__
metrics_group_set_union = metrics_group_set + metrics_group_set_two
assert len(metrics_group_set_union) == 3
with self.assertRaises(AssertionError):
metrics_group_set.remove(self.__panoptes_resource)
with self.assertRaises(TypeError):
metrics_group_set + metrics_group
# Test PanoptesMetricsGroupSet.__iter__ & 'next'
metrics_group_count = 0
metrics_group_set_union_interator = iter(metrics_group_set_union)
for _ in metrics_group_set_union:
self.assertIn(next(metrics_group_set_union_interator),
metrics_group_set_union.metrics_groups)
metrics_group_count += 1
assert len(metrics_group_set_union) == metrics_group_count
with self.assertRaises(Exception):
next(metrics_group_set_union_interator)
# Test PanoptesMetricsGroupSet.__repr__
_METRICS_GROUP_SET_REPR = u"PanoptesMetricsGroupSet[PanoptesMetricsGroup[resource:" \
u"plugin|test|site|test|class|test|subclass|test|type|test|id|test|endpoint|test," \
u"interval:120,schema_version:0.2,group_type:test,creation_timestamp:{}," \
u"dimensions:[],metrics:[]],PanoptesMetricsGroup[resource:" \
u"plugin|test|site|test|class|test|subclass|test|type|test|id|test|endpoint|test," \
u"interval:120,schema_version:0.2,group_type:test3,creation_timestamp:{}," \
u"dimensions:[],metrics:[" \
u"PanoptesMetric[test3|0.0|GAUGE|{}]]]]".format(mock_time.return_value,
mock_time.return_value,
mock_time.return_value)
self.assertEqual(repr(metrics_group_set), _METRICS_GROUP_SET_REPR)
def test_panoptes_metric_set(self):
metric_set = PanoptesMetricSet()
metric1 = PanoptesMetric(
u'test_metric',
0,
PanoptesMetricType.GAUGE,
metric_creation_timestamp=mock_time.return_value)
metric2 = PanoptesMetric(
u'test_metric',
0,
PanoptesMetricType.GAUGE,
metric_creation_timestamp=mock_time.return_value)
metric_set.add(metric1)
metric_set.add(metric2)
assert len(metric_set) == 1
self.assertIn(metric1, metric_set.metrics)
# Test PanoptesMetricSet.__repr__
_METRIC_SET_REPR = u"PanoptesMetricSet[PanoptesMetric[test_metric|0|GAUGE|{}]]".format(
mock_time.return_value)
self.assertEqual(repr(metric_set), _METRIC_SET_REPR)
with self.assertRaises(Exception):
metric_set.remove(self.__panoptes_resource)
metric_set.remove(metric1)
assert len(metric_set) == 0
# Test PanoptesMetricSet.__iter__ and 'next'
metric_count = 0
metric_set_iterator = iter(metric_set)
for _ in metric_set:
self.assertIn(next(metric_set_iterator), metric_set.metrics)
metric_count += 1
assert len(metric_set) == metric_count
with self.assertRaises(Exception):
next(metric_set_iterator)
class TestResources(unittest.TestCase):
def setUp(self):
self.__panoptes_resource_metadata = {'test': 'test', '_resource_ttl': '604800'}
self.__panoptes_resource = PanoptesResource(resource_site='test', resource_class='test',
resource_subclass='test',
resource_type='test', resource_id='test', resource_endpoint='test',
resource_plugin='test',
resource_creation_timestamp=_TIMESTAMP,
resource_ttl=RESOURCE_MANAGER_RESOURCE_EXPIRE)
self.__panoptes_resource.add_metadata('test', 'test')
self.__panoptes_resource_set = PanoptesResourceSet()
mock_valid_timestamp = Mock(return_value=True)
with patch('yahoo_panoptes.framework.resources.PanoptesValidators.valid_timestamp',
mock_valid_timestamp):
self.__panoptes_resource_set.resource_set_creation_timestamp = _TIMESTAMP
self.my_dir, self.panoptes_test_conf_file = _get_test_conf_file()
def test_panoptes_resource(self):
panoptes_resource_metadata = self.__panoptes_resource_metadata
panoptes_resource = self.__panoptes_resource
self.assertIsInstance(panoptes_resource, PanoptesResource)
self.assertEqual(panoptes_resource.resource_site, 'test')
self.assertEqual(panoptes_resource.resource_class, 'test')
self.assertEqual(panoptes_resource.resource_subclass, 'test')
self.assertEqual(panoptes_resource.resource_type, 'test')
self.assertEqual(panoptes_resource.resource_id, 'test')
self.assertEqual(panoptes_resource.resource_endpoint, 'test')
self.assertEqual(panoptes_resource.resource_metadata, panoptes_resource_metadata)
self.assertEqual(panoptes_resource.resource_ttl, str(RESOURCE_MANAGER_RESOURCE_EXPIRE))
self.assertEqual(panoptes_resource, panoptes_resource)
self.assertFalse(panoptes_resource == '1')
self.assertIsInstance(str(panoptes_resource), str)
with self.assertRaises(AssertionError):
panoptes_resource.add_metadata(None, 'test')
with self.assertRaises(AssertionError):
panoptes_resource.add_metadata('', 'test')
with self.assertRaises(ValueError):
panoptes_resource.add_metadata('1', 'test')
with self.assertRaises(ValueError):
panoptes_resource.add_metadata('key', 'test|')
with self.assertRaises(AssertionError):
PanoptesResource(resource_site='', resource_class='test', resource_subclass='test',
resource_type='test', resource_id='test', resource_endpoint='test')
with self.assertRaises(AssertionError):
PanoptesResource(resource_site='test', resource_class='', resource_subclass='test',
resource_type='test', resource_id='test', resource_endpoint='test')
with self.assertRaises(AssertionError):
PanoptesResource(resource_site='test', resource_class='test', resource_subclass='',
resource_type='test', resource_id='test', resource_endpoint='test')
with self.assertRaises(AssertionError):
PanoptesResource(resource_site='test', resource_class='test', resource_subclass='test',
resource_type='', resource_id='test', resource_endpoint='test')
with self.assertRaises(AssertionError):
PanoptesResource(resource_site='test', resource_class='test', resource_subclass='test',
resource_type='test', resource_id='', resource_endpoint='test')
with self.assertRaises(AssertionError):
PanoptesResource(resource_site=None, resource_class='test', resource_subclass='test',
resource_type='test', resource_id='test', resource_endpoint='test')
with self.assertRaises(AssertionError):
PanoptesResource(resource_site='test', resource_class=None, resource_subclass='test',
resource_type='test', resource_id='test', resource_endpoint='test')
with self.assertRaises(AssertionError):
PanoptesResource(resource_site='test', resource_class='test', resource_subclass=None,
resource_type='test', resource_id='test', resource_endpoint='test')
with self.assertRaises(AssertionError):
PanoptesResource(resource_site='test', resource_class='test', resource_subclass='test',
resource_type=None, resource_id='test', resource_endpoint='test')
with self.assertRaises(AssertionError):
PanoptesResource(resource_site='test', resource_class='test', resource_subclass='test',
resource_type='test', resource_id=None, resource_endpoint='test')
with self.assertRaises(AssertionError):
PanoptesResource(resource_site='test', resource_class='test', resource_subclass='test',
resource_type='test', resource_id='test', resource_endpoint=None)
# Test json and raw representations of PanoptesResource
panoptes_resource_2 = PanoptesResource(resource_site='test', resource_class='test',
resource_subclass='test',
resource_type='test', resource_id='test', resource_endpoint='test',
resource_creation_timestamp=_TIMESTAMP,
resource_plugin='test')
self.assertEqual(panoptes_resource_2.resource_creation_timestamp, _TIMESTAMP)
panoptes_resource_2_json = {
u'resource_site': u'test',
u'resource_id': u'test',
u'resource_class': u'test',
u'resource_plugin': u'test',
u'resource_creation_timestamp': _TIMESTAMP,
u'resource_subclass': u'test',
u'resource_endpoint': u'test',
u'resource_metadata': {
u'_resource_ttl': u'604800'
},
u'resource_type': u'test'
}
self.assertEqual(ordered(json.loads(panoptes_resource_2.json)), ordered(panoptes_resource_2_json))
panoptes_resource_2_raw = collections.OrderedDict(
[('resource_site', 'test'),
('resource_class', 'test'),
('resource_subclass', 'test'),
('resource_type', 'test'),
('resource_id', 'test'),
('resource_endpoint', 'test'),
('resource_metadata', collections.OrderedDict(
[('_resource_ttl', '604800')])
),
('resource_creation_timestamp', _TIMESTAMP),
('resource_plugin', 'test')])
self.assertEqual(panoptes_resource_2.raw, panoptes_resource_2_raw)
# Test resource creation from dict
with open('tests/test_resources/input/resource_one.json') as f:
resource_specs = json.load(f)
resource_from_json = PanoptesResource.resource_from_dict(resource_specs['resources'][0])
panoptes_resource_3 = PanoptesResource(resource_site="test_site", resource_class="network",
resource_subclass="test_subclass", resource_type="test_type",
resource_id="test_id_1", resource_endpoint="test_endpoint_1",
resource_plugin="key")
self.assertEqual(resource_from_json, panoptes_resource_3)
def test_panoptes_resource_set(self):
panoptes_resource = self.__panoptes_resource
panoptes_resource_set = self.__panoptes_resource_set
self.assertEqual(panoptes_resource_set.add(panoptes_resource), None)
self.assertEqual(len(panoptes_resource_set), 1)
self.assertEqual(type(panoptes_resource_set.resources), set)
self.assertIsInstance(str(panoptes_resource_set), str)
self.assertIsInstance(iter(panoptes_resource_set), collections.Iterable)
self.assertEqual(panoptes_resource_set.next(), panoptes_resource)
self.assertEqual(panoptes_resource_set.remove(panoptes_resource), None)
self.assertEqual(len(panoptes_resource_set), 0)
self.assertEqual(panoptes_resource_set.resource_set_creation_timestamp, _TIMESTAMP)
with self.assertRaises(AssertionError):
panoptes_resource_set.resource_set_creation_timestamp = 0
panoptes_resource_2 = PanoptesResource(resource_site='test', resource_class='test',
resource_subclass='test',
resource_type='test', resource_id='test2', resource_endpoint='test',
resource_plugin='test',
resource_ttl=RESOURCE_MANAGER_RESOURCE_EXPIRE,
resource_creation_timestamp=_TIMESTAMP)
panoptes_resource_set.add(panoptes_resource)
panoptes_resource_set.add(panoptes_resource_2)
self.assertEqual(len(panoptes_resource_set.get_resources_by_site()['test']), 2)
self.assertEqual(panoptes_resource_set.resource_set_schema_version, "0.1")
panoptes_resource_set_json = {
u'resource_set_creation_timestamp': _TIMESTAMP,
u'resource_set_schema_version': u'0.1',
u'resources': [
{
u'resource_site': u'test',
u'resource_class': u'test',
u'resource_subclass': u'test',
u'resource_type': u'test',
u'resource_id': u'test2',
u'resource_endpoint': u'test',
u'resource_metadata': {
u'_resource_ttl': u'604800'
},
u'resource_creation_timestamp': _TIMESTAMP,
u'resource_plugin': u'test'
},
{
u'resource_site': u'test',
u'resource_class': u'test',
u'resource_subclass': u'test',
u'resource_type': u'test',
u'resource_id': u'test',
u'resource_endpoint': u'test',
u'resource_metadata': {
u'_resource_ttl': u'604800',
u'test': u'test'
},
u'resource_creation_timestamp': _TIMESTAMP,
u'resource_plugin': u'test'
}
]
}
self.assertEqual(ordered(panoptes_resource_set_json), ordered(json.loads(panoptes_resource_set.json)))
def test_panoptes_resources_key_value_store(self):
panoptes_context = PanoptesContext(self.panoptes_test_conf_file)
panoptes_resources_kv_store = PanoptesResourcesKeyValueStore(panoptes_context)
self.assertEqual(panoptes_resources_kv_store.redis_group, const.RESOURCE_MANAGER_REDIS_GROUP)
self.assertEqual(panoptes_resources_kv_store.namespace, const.RESOURCE_MANAGER_KEY_VALUE_NAMESPACE)
@patch('redis.StrictRedis', panoptes_mock_redis_strict_client)
def test_panoptes_resource_store(self):
panoptes_context = PanoptesContext(self.panoptes_test_conf_file,
key_value_store_class_list=[PanoptesTestKeyValueStore])
with self.assertRaises(Exception):
PanoptesResourceStore(panoptes_context)
mock_kv_store = Mock(return_value=PanoptesTestKeyValueStore(panoptes_context))
with patch('yahoo_panoptes.framework.resources.PanoptesContext.get_kv_store', mock_kv_store):
panoptes_resource_store = PanoptesResourceStore(panoptes_context)
panoptes_resource_store.add_resource("test_plugin_signature", self.__panoptes_resource)
resource_key = "plugin|test|site|test|class|test|subclass|test|type|test|id|test|endpoint|test"
resource_value = panoptes_resource_store.get_resource(resource_key)
self.assertEqual(self.__panoptes_resource, resource_value)
panoptes_resource_2 = PanoptesResource(resource_site='test', resource_class='test',
resource_subclass='test',
resource_type='test', resource_id='test2', resource_endpoint='test',
resource_plugin='test',
resource_ttl=RESOURCE_MANAGER_RESOURCE_EXPIRE,
resource_creation_timestamp=_TIMESTAMP)
panoptes_resource_store.add_resource("test_plugin_signature", panoptes_resource_2)
self.assertIn(self.__panoptes_resource, panoptes_resource_store.get_resources())
self.assertIn(panoptes_resource_2, panoptes_resource_store.get_resources())
panoptes_resource_store.delete_resource("test_plugin_signature", panoptes_resource_2)
self.assertNotIn(panoptes_resource_2, panoptes_resource_store.get_resources())
panoptes_resource_3 = PanoptesResource(resource_site='test', resource_class='test',
resource_subclass='test',
resource_type='test', resource_id='test3', resource_endpoint='test',
resource_plugin='test3',
resource_ttl=RESOURCE_MANAGER_RESOURCE_EXPIRE,
resource_creation_timestamp=_TIMESTAMP)
panoptes_resource_store.add_resource("test_plugin_signature", panoptes_resource_3)
self.assertIn(panoptes_resource_3, panoptes_resource_store.get_resources(site='test', plugin_name='test3'))
self.assertNotIn(self.__panoptes_resource,
panoptes_resource_store.get_resources(site='test', plugin_name='test3'))
# Test key not found
mock_find_keys = Mock(
return_value=['dummy',
'plugin|test|site|test|class|test|subclass|test|type|test|id|test|endpoint|test'])
with patch('yahoo_panoptes.framework.resources.PanoptesKeyValueStore.find_keys',
mock_find_keys):
self.assertEqual(1, len(panoptes_resource_store.get_resources()))
# Test resource store methods raise correct errors
mock_get = Mock(side_effect=Exception)
with patch('yahoo_panoptes.framework.resources.PanoptesKeyValueStore.get', mock_get):
with self.assertRaises(PanoptesResourceError):
panoptes_resource_store.get_resource('test3')
# Test bad input
with self.assertRaises(AssertionError):
panoptes_resource_store.get_resource("")
with self.assertRaises(AssertionError):
panoptes_resource_store.get_resource(1)
with self.assertRaises(AssertionError):
panoptes_resource_store.add_resource("", panoptes_resource_2)
with self.assertRaises(AssertionError):
panoptes_resource_store.add_resource("test_plugin_signature", None)
with self.assertRaises(AssertionError):
panoptes_resource_store.add_resource("test_plugin_signature", PanoptesResourceStore(panoptes_context))
with self.assertRaises(AssertionError):
panoptes_resource_store.delete_resource("", panoptes_resource_2)
with self.assertRaises(AssertionError):
panoptes_resource_store.delete_resource("test_plugin_signature", None)
with self.assertRaises(AssertionError):
panoptes_resource_store.delete_resource("test_plugin_signature",
PanoptesResourceStore(panoptes_context))
with self.assertRaises(AssertionError):
panoptes_resource_store.get_resources("", "test_plugin_name")
with self.assertRaises(AssertionError):
panoptes_resource_store.get_resources("test_site", "")
with self.assertRaises(AssertionError):
panoptes_resource_store.get_resources(1, "test_plugin_name")
with self.assertRaises(AssertionError):
panoptes_resource_store.get_resources("test_site", 1)
# Test non-existent key
with self.assertRaises(PanoptesResourceError):
panoptes_resource_store.get_resource('tes')
mock_set = Mock(side_effect=Exception)
with patch('yahoo_panoptes.framework.resources.PanoptesKeyValueStore.set', mock_set):
with self.assertRaises(PanoptesResourceError):
panoptes_resource_store.add_resource("test_plugin_signature", panoptes_resource_2)
mock_delete = Mock(side_effect=Exception)
with patch('yahoo_panoptes.framework.resources.PanoptesKeyValueStore.delete', mock_delete):
with self.assertRaises(PanoptesResourceError):
panoptes_resource_store.delete_resource("test_plugin_signature", panoptes_resource_2)
with self.assertRaises(PanoptesResourceError):
panoptes_resource_store._deserialize_resource("tes", "null")
with self.assertRaises(PanoptesResourceError):
panoptes_resource_store._deserialize_resource(resource_key, "null")
@patch('redis.StrictRedis', panoptes_mock_redis_strict_client)
def test_resource_dsl_parsing(self):
panoptes_context = PanoptesContext(self.panoptes_test_conf_file)
test_query = 'resource_class = "network" AND resource_subclass = "load-balancer"'
test_result = (
'SELECT resources.*, group_concat(key,"|"), group_concat(value,"|") ' +
'FROM resources ' +
'LEFT JOIN resource_metadata ON resources.id = resource_metadata.id ' +
'WHERE (resources.resource_class = "network" ' +
'AND resources.resource_subclass = "load-balancer") ' +
'GROUP BY resource_metadata.id ' +
'ORDER BY resource_metadata.id'
)
panoptes_resource_dsl = PanoptesResourceDSL(test_query, panoptes_context)
self.assertEqual(panoptes_resource_dsl.sql, test_result)
# This very long query tests all code paths with the DSL parser:
test_query = 'resource_class = "network" AND resource_subclass = "load-balancer" OR \
resource_metadata.os_version LIKE "4%" AND resource_site NOT IN ("test_site") \
AND resource_endpoint IN ("test1","test2") AND resource_type != "a10" OR ' \
'resource_metadata.make NOT LIKE \
"A10%" AND resource_metadata.model NOT IN ("test1", "test2")'
test_result = (
'SELECT resources.*,group_concat(key,"|"),group_concat(value,"|") FROM (SELECT resource_metadata.id ' +
'FROM resources,resource_metadata WHERE (resources.resource_class = "network" ' +
'AND resources.resource_subclass = "load-balancer" ' +
'AND resources.resource_site NOT IN ("test_site") ' +
'AND resources.resource_endpoint IN ("test1","test2") ' +
'AND resources.resource_type != "a10" ' +
'AND ((resource_metadata.key = "os_version" ' +
'AND resource_metadata.value LIKE "4%")) ' +
'AND resource_metadata.id = resources.id) ' +
'UNION SELECT resource_metadata.id ' +
'FROM resources,resource_metadata WHERE (resource_metadata.key = "make" ' +
'AND resource_metadata.value NOT LIKE "A10%") ' +
'AND resource_metadata.id = resources.id ' +
'INTERSECT SELECT resource_metadata.id ' +
'FROM resources,resource_metadata WHERE (resource_metadata.key = "model" ' +
'AND resource_metadata.value NOT IN ("test1","test2")) ' +
'AND resource_metadata.id = resources.id ' +
'GROUP BY resource_metadata.id ' +
'ORDER BY resource_metadata.id) AS filtered_resources, ' +
'resources, resource_metadata WHERE resources.id = filtered_resources.id ' +
'AND resource_metadata.id = filtered_resources.id GROUP BY resource_metadata.id')
panoptes_resource_dsl = PanoptesResourceDSL(test_query, panoptes_context)
self.assertEqual(panoptes_resource_dsl.sql, test_result)
panoptes_resource_dsl = PanoptesResourceDSL('resource_site = "local"', panoptes_context)
self.assertIsInstance(panoptes_resource_dsl.tokens, ParseResults)
with self.assertRaises(AssertionError):
PanoptesResourceDSL(None, panoptes_context)
with self.assertRaises(AssertionError):
PanoptesResourceDSL('', None)
with self.assertRaises(AssertionError):
PanoptesResourceDSL('', panoptes_context)
with self.assertRaises(ParseException):
PanoptesResourceDSL('resources_site = local', panoptes_context)
class TestPanoptesSNMPConnectionFactory(unittest.TestCase):
@patch('redis.StrictRedis', panoptes_mock_redis_strict_client)
def setUp(self):
_, self.panoptes_test_conf_file = get_test_conf_file()
self._my_path = os.path.dirname(os.path.abspath(__file__))
self._panoptes_context = PanoptesContext(self.panoptes_test_conf_file)
self._resource = PanoptesResource(
resource_class="test_class",
resource_subclass="test_subclass",
resource_type="test_type",
resource_id="test_id",
resource_endpoint="127.0.0.1",
resource_site="test_site",
resource_creation_timestamp=time.time(),
resource_plugin="test_plugin")
self._key_value_store = PanoptesTestKeyValueStore(
self._panoptes_context)
self._secrets_store = TestPanoptesSecretsStore(self._panoptes_context)
self._secrets_store.set(key='snmp_community_string:test_site',
value='public')
self._panoptes_plugin_context = PanoptesPluginContext(
panoptes_context=self._panoptes_context,
logger_name="test_logger",
config=plugin_conf,
key_value_store=self._key_value_store,
secrets_store=self._secrets_store,
data=self._resource)
def test_get_snmp_connection(self):
with self.assertRaises(AssertionError):
PanoptesSNMPConnectionFactory.get_snmp_connection(
plugin_context=None, resource=None)
with self.assertRaises(AssertionError):
PanoptesSNMPConnectionFactory.get_snmp_connection(
plugin_context=self._panoptes_plugin_context, resource=None)
with self.assertRaises(AssertionError):
PanoptesSNMPConnectionFactory.get_snmp_connection(
plugin_context=None, resource=self._resource)
with self.assertRaises(AssertionError):
PanoptesSNMPConnectionFactory.get_snmp_connection(
plugin_context=self._panoptes_plugin_context,
resource=self._resource,
timeout='1')
# Test that defaults (from the panoptes.ini) are set when no values are given
connection = PanoptesSNMPConnectionFactory.get_snmp_connection(
self._panoptes_plugin_context, self._resource)
self.assertIsInstance(connection, PanoptesSNMPV2Connection)
self.assertEqual(connection.host, '127.0.0.1')
self.assertEqual(connection.port, 10161)
self.assertEqual(connection.timeout, 5)
self.assertEqual(connection.retries, 1)
# Test that the community string is fetched and populated for the give site
self.assertEqual(connection.community, 'public')
# Test that when values for SNMP parameters are provided, the defaults are overwritten
connection = PanoptesSNMPConnectionFactory.get_snmp_connection(
self._panoptes_plugin_context, self._resource, timeout=1)
self.assertIsInstance(connection, PanoptesSNMPV2Connection)
self.assertEqual(connection.timeout, 1)
connection = PanoptesSNMPConnectionFactory.get_snmp_connection(
self._panoptes_plugin_context, self._resource, retries=2)
self.assertIsInstance(connection, PanoptesSNMPV2Connection)
self.assertEqual(connection.retries, 2)
connection = PanoptesSNMPConnectionFactory.get_snmp_connection(
self._panoptes_plugin_context, self._resource, port=10162)
self.assertIsInstance(connection, PanoptesSNMPV2Connection)
self.assertEqual(connection.port, 10162)
# Test that the community suffixed is, er, suffixed
connection = PanoptesSNMPConnectionFactory.get_snmp_connection(
self._panoptes_plugin_context,
self._resource,
community_suffix='123')
self.assertIsInstance(connection, PanoptesSNMPV2Connection)
self.assertEqual(connection.community, 'public@123')
# Test passing various combinations of x509 parameters
key_file = os.path.join(self._my_path, 'key.pem')
cert_file = os.path.join(self._my_path, 'certificate.pem')
# Test that when x509_secure_connection is not set, the key and cert files are not needed
connection = PanoptesSNMPConnectionFactory.get_snmp_connection(
self._panoptes_plugin_context,
self._resource,
x509_secure_connection=0)
self.assertIsInstance(connection, PanoptesSNMPV2Connection)
# Test when x509_secure_connection is set, but the key and cert files aren't readable
with self.assertRaises(PanoptesSNMPException):
PanoptesSNMPConnectionFactory.get_snmp_connection(
self._panoptes_plugin_context,
self._resource,
x509_secure_connection=1)
# Test when x509_secure_connection is set, but the key file isn't readable
with self.assertRaises(PanoptesSNMPException):
PanoptesSNMPConnectionFactory.get_snmp_connection(
self._panoptes_plugin_context,
self._resource,
x509_secure_connection=1,
x509_key_file=key_file)
# Test when x509_secure_connection is set, but the cert file isn't readable
with self.assertRaises(PanoptesSNMPException):
PanoptesSNMPConnectionFactory.get_snmp_connection(
self._panoptes_plugin_context,
self._resource,
x509_secure_connection=1,
x509_cert_file=cert_file)
# Test when x509_secure_connection is set, and both the key and cert files are readable
connection = PanoptesSNMPConnectionFactory.get_snmp_connection(
self._panoptes_plugin_context,
self._resource,
x509_secure_connection=1,
x509_key_file=key_file,
x509_cert_file=cert_file)
self.assertIsInstance(connection, PanoptesSNMPV2Connection)
# Test that when the snmp_proxy_hosts metadata key is set, we get PanoptesSNMPSteamRollerAgentConnection
self._resource.add_metadata('snmp_proxy_hosts', '127.0.0.1:10161')
connection = PanoptesSNMPConnectionFactory.get_snmp_connection(
self._panoptes_plugin_context, self._resource)
self.assertIsInstance(connection,
PanoptesSNMPSteamRollerAgentConnection)
# Test that an PanoptesSNMPException is thrown if the community string for the given site is empty
# Note that this should be the last test - otherwise other tests would fail
self._secrets_store.delete('snmp_community_string:test_site')
with self.assertRaises(PanoptesSNMPException):
PanoptesSNMPConnectionFactory.get_snmp_connection(
self._panoptes_plugin_context, self._resource)
class SNMPPluginTestFramework(object):
plugin_class = None
path = None
snmp_host = u'127.0.0.1'
snmp_port = 10161
snmp_timeout = 10
snmp_retries = 1
snmp_max_repetitions = 10
snmp_community = 'public'
snmp_failure_timeout = 1
x509_secured_requests = 0
x509_key_location = u'/home/panoptes/x509/keys'
x509_key_filename = u'panoptes.key'
x509_cert_location = u'/home/panoptes/x509/keys'
x509_cert_filename = u'panoptes.pem'
resource_id = u'test_id'
resource_endpoint = u'127.0.0.1'
resource_plugin = u'dummy'
resource_site = u'test_site'
resource_class = u'network'
resource_subclass = u'test_subclass'
resource_type = u'test_type'
resource_backplane = None
resource_model = u'model'
results_data_file = u'results.json'
enrichment_data_file = u'enrichment_data'
use_enrichment = True
plugin_conf = {
'Core': {
'name': 'Test Plugin',
'module': 'test_plugin'
},
'main': {
'execute_frequency': '60',
'enrichment_ttl': '300',
'resource_filter': 'resource_class = "network"'
},
'snmp': {
'timeout': 10,
'retries': 1,
'non_repeaters': 1,
'max_repetitions': 25,
},
'x509': {
'x509_secured_requests': 0,
'x509_key_location': '/home/panoptes/x509/keys',
'x509_key_filename': 'panoptes.key',
'x509_cert_location': '/home/panoptes/x509/keys',
'x509_cert_filename': 'panoptes.pem'
}
}
def __init__(self, test_name):
super(SNMPPluginTestFramework, self).__init__(test_name)
self._path = self.path
self._plugin_conf = self.plugin_conf
self._snmp_host = self.snmp_host
self._snmp_port = self.snmp_port
self._snmp_timeout = self.snmp_timeout
self._snmp_retries = self.snmp_retries
self._snmp_max_repetitions = self.snmp_max_repetitions
self._snmp_community = self.snmp_community
self._snmp_failure_timeout = self.snmp_failure_timeout
self._x509_secured_requests = self.x509_secured_requests
self._x509_cert_location = self.x509_cert_location
self._x509_cert_filename = self.x509_cert_filename
self._x509_key_location = self.x509_key_location
self._x509_key_filename = self.x509_key_filename
self._resource_id = self.resource_id
self._resource_endpoint = self.resource_endpoint
self._resource_plugin = self.resource_plugin
self._resource_site = self.resource_site
self._resource_class = self.resource_class
self._resource_subclass = self.resource_subclass
self._resource_type = self.resource_type
self._resource_backplane = self.resource_backplane
self._resource_model = self.resource_model
self._panoptes_context = None
self._panoptes_resource = None
self._snmp_conf = None
self._x509_conf = None
self._expected_results = None
self._results_data_file = None
self._enrichment_data_file = None
self._secret_store = None
self._plugin_context = None
self._plugin_context_bad = None
self._snmp_conf_bad = None
self._x509_conf_bad = None
self._enrichment_kv = None
self._enrichment_cache = None
def set_panoptes_context(self):
panoptes_test_conf_file = os.path.join(
os.path.dirname(os.path.dirname(__file__)),
'config_files/test_panoptes_config.ini')
self._panoptes_context = PanoptesContext(
panoptes_test_conf_file,
key_value_store_class_list=[PanoptesEnrichmentCacheKeyValueStore])
def set_panoptes_resource(self):
self._panoptes_resource = PanoptesResource(
resource_site=self._resource_site,
resource_class=self._resource_class,
resource_subclass=self._resource_subclass,
resource_type=self._resource_type,
resource_id=self._resource_id,
resource_endpoint=self._resource_endpoint,
resource_plugin=self._resource_plugin)
self._panoptes_resource.resource_metadata[
'model'] = self._resource_model
if self._resource_backplane:
self._panoptes_resource.add_metadata('backplane',
self._resource_backplane)
def set_enrichment_cache(self, resource_endpoint=None):
if self.use_enrichment:
self._enrichment_data_file = 'data/' + self.enrichment_data_file
self._enrichment_kv = self._panoptes_context.get_kv_store(
PanoptesEnrichmentCacheKeyValueStore)
enrichment_data_file = os.path.join(os.path.abspath(self.path),
self._enrichment_data_file)
if self._plugin_conf.get('enrichment'):
if self._plugin_conf['enrichment'].get('preload'):
self._enrichment_cache = PanoptesEnrichmentCache(
self._panoptes_context, self._plugin_conf,
self._panoptes_resource)
try:
with open(enrichment_data_file) as enrichment_data:
for line in enrichment_data:
key, value = line.strip().split('=>')
if resource_endpoint is not None:
value = self.update_enrichment_ip(
value, resource_endpoint)
self._enrichment_kv.set(key, value)
except Exception as e:
raise IOError(
'Failed to load enrichment data file {}: {}'.format(
enrichment_data_file, repr(e)))
def set_snmp_conf(self):
self._snmp_conf = self._panoptes_context.config_object.snmp_defaults
def set_x509_conf(self):
self._x509_conf = {
'x509_secured_requests': self._x509_secured_requests,
'x509_cert_location': self._x509_cert_location,
'x509_cert_filename': self._x509_cert_filename,
'x509_key_location': self._x509_key_location,
'x509_key_filename': self._x509_key_filename
}
def set_secret_store(self):
self._secret_store = create_autospec(PanoptesSecretsStore,
instance=True,
spec_set=True)
self._secret_store.get_by_site.return_value = self._snmp_community
def set_plugin_context(self):
self._plugin_context = create_autospec(
PanoptesPluginWithEnrichmentContext,
instance=True,
spec_set=True,
data=self._panoptes_resource,
enrichment=self._enrichment_cache,
config=self._plugin_conf,
snmp=self._snmp_conf,
x509=self._x509_conf,
secrets=self._secret_store,
logger=logging.getLogger(__name__))
def set_x509_conf_bad(self):
self._x509_conf_bad = copy.copy(self._x509_conf)
self._x509_conf_bad['x509_secured_requests'] = 5
def set_snmp_conf_bad(self):
self._snmp_conf_bad = copy.copy(self._snmp_conf)
self._snmp_conf_bad['port'] += 1
self._snmp_conf_bad['timeout'] = self._snmp_failure_timeout
def set_plugin_context_bad(self):
self._plugin_context_bad = create_autospec(
PanoptesPluginWithEnrichmentContext,
instance=True,
spec_set=True,
data=self._panoptes_resource,
enrichment=self._enrichment_cache,
config=self._plugin_conf,
snmp=self._snmp_conf_bad,
x509=self._x509_conf,
secrets=self._secret_store,
logger=logging.getLogger(__name__))
def set_expected_results(self):
self._results_data_file = 'data/' + self.results_data_file
expected_result_file = os.path.join(os.path.abspath(self.path),
self._results_data_file)
self._expected_results = json.load(open(expected_result_file))
def update_enrichment_ip(self, enrichment, resource_endpoint):
"""
The PluginPollingGenericSNMPMetrics Plugin initially calls the function _get_config() when started.
This function loads the enrichment from the PanoptesEnrichmentCacheKeyValueStore
using the resource_id, namespace, and enrichment key. The enrichment key is the resource_endpoint.
no_service_active tests to ensure that a proper error (timeout -> ping error) is returned when
snmp_bulk is called with an invalid IP. In order to get to this stage in the tests the enrichment must
loaded in the generic snmp runner. In order to test this, the resource_endpoint is changed to
be invalid. However this IP must also be changed in the test enrichment file so the generic snmp runner
is able to load the enrichment from the cache.
"""
modified_enrichment = json.loads(enrichment)
if len(modified_enrichment) == 0 or len(
list(modified_enrichment['data'].keys())) == 0:
return enrichment
ip = list(modified_enrichment['data'].keys())[0]
if re.search(r'\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b', ip):
modified_enrichment['data'][
resource_endpoint] = modified_enrichment['data'][ip]
del modified_enrichment['data'][ip]
return json.dumps(modified_enrichment)
return enrichment
@patch('time.time', mock_time)
@patch('yahoo_panoptes.framework.resources.time', mock_time)
@patch('redis.StrictRedis', PanoptesMockRedis)
def setUp(self):
self.set_panoptes_context()
self.set_panoptes_resource()
self.set_enrichment_cache()
self.set_snmp_conf()
self.set_x509_conf()
self.set_secret_store()
self.set_plugin_context()
self.set_snmp_conf_bad()
self.set_x509_conf_bad()
self.set_plugin_context_bad()
self.set_expected_results()
class TestResources(unittest.TestCase):
def setUp(self):
self.__panoptes_resource = PanoptesResource(resource_site='test', resource_class='test',
resource_subclass='test',
resource_type='test', resource_id='test', resource_endpoint='test',
resource_plugin='test')
self.__panoptes_resource.add_metadata('test', 'test')
def test_panoptes_metric(self):
with self.assertRaises(AssertionError):
PanoptesMetric(None, 0, PanoptesMetricType.GAUGE)
with self.assertRaises(ValueError):
PanoptesMetric('1', 0, PanoptesMetricType.GAUGE)
with self.assertRaises(AssertionError):
PanoptesMetric('test_metric', None, PanoptesMetricType.GAUGE)
with self.assertRaises(AssertionError):
PanoptesMetric('test_metric', 0, None)
with self.assertRaises(AssertionError):
PanoptesMetric('test_metric', True, PanoptesMetricType.GAUGE)
metric1 = PanoptesMetric('test_metric', 0, PanoptesMetricType.GAUGE)
self.assertEqual(metric1.metric_name, 'test_metric')
self.assertEqual(metric1.metric_value, 0)
self.assertEqual(metric1.metric_type, PanoptesMetricType.GAUGE)
self.assertNotEqual(metric1, None)
metric2 = PanoptesMetric('test_metric', 0, PanoptesMetricType.GAUGE)
self.assertEqual(metric1, metric2)
metric2 = PanoptesMetric('test_metric', 1, PanoptesMetricType.GAUGE)
self.assertNotEqual(metric1, metric2)
metric2 = PanoptesMetric('test_metric', 1, PanoptesMetricType.COUNTER)
self.assertNotEqual(metric1, metric2)
@patch('yahoo_panoptes.framework.metrics.time', mock_time)
def test_panoptes_metric_json_and_repr(self):
metric = PanoptesMetric('test_metric', 0, PanoptesMetricType.GAUGE, mock_time.return_value)
serialized = json.loads(metric.json)
expected = {"metric_creation_timestamp": mock_time.return_value,
"metric_name": "test_metric",
"metric_type": "gauge",
"metric_value": 0}
self.assertEqual(ordered(serialized), ordered(expected))
def testMetricsGroup(self):
now = round(time.time(), METRICS_TIMESTAMP_PRECISION)
metrics_group = PanoptesMetricsGroup(self.__panoptes_resource, 'test', 120)
self.assertEqual(metrics_group.group_type, 'test')
self.assertEqual(metrics_group.interval, 120)
self.assertEqual(metrics_group.schema_version, '0.2')
self.assertGreaterEqual(metrics_group.creation_timestamp, now)
dimension_one = PanoptesMetricDimension('if_alias', 'bar')
dimension_two = PanoptesMetricDimension('if_alias', 'foo')
metrics_group.add_dimension(dimension_one)
with self.assertRaises(KeyError):
metrics_group.add_dimension(dimension_two)
self.assertEqual(len(metrics_group.dimensions), 1)
self.assertEqual(metrics_group.contains_dimension_by_name('if_alias'), True)
self.assertEqual(metrics_group.contains_dimension_by_name('baz'), False)
metrics_group.delete_dimension_by_name('if_alias')
self.assertEqual(metrics_group.contains_dimension_by_name('if_alias'), False)
self.assertEqual(len(metrics_group.dimensions), 0)
self.assertEqual(metrics_group.get_dimension_by_name('foo'), None)
metrics_group.add_dimension(dimension_two)
dimension_three = PanoptesMetricDimension('if_alias', 'bar')
metrics_group.upsert_dimension(dimension_three)
self.assertEqual(len(metrics_group.dimensions), 1)
self.assertEqual(metrics_group.get_dimension_by_name('if_alias').value, 'bar')
dimension_four = PanoptesMetricDimension('if_name', 'eth0')
metrics_group.upsert_dimension(dimension_four)
self.assertEqual(len(metrics_group.dimensions), 2)
with self.assertRaises(AssertionError):
metrics_group.add_metric(None)
metric = PanoptesMetric('test_metric', 0, PanoptesMetricType.GAUGE)
metrics_group.add_metric(metric)
to_json = metrics_group.json
metrics = PanoptesMetricsGroup.flatten_metrics(json.loads(to_json)['metrics'])
self.assertEquals(metrics['gauge']['test_metric']['value'], 0)
metrics_group_two = PanoptesMetricsGroup(self.__panoptes_resource, 'test', 120)
metrics_group_two.add_dimension(dimension_two)
metrics_group_two.upsert_dimension(dimension_three)
metrics_group_two.upsert_dimension(dimension_four)
metrics_group_two.add_metric(metric)
self.assertEqual(metrics_group, metrics_group_two)
def test_metrics_group_hash(self):
now = round(time.time(), METRICS_TIMESTAMP_PRECISION)
metrics_group = PanoptesMetricsGroup(self.__panoptes_resource, 'test', 120)
metrics_group_two = PanoptesMetricsGroup(self.__panoptes_resource, 'test', 120)
dimension = PanoptesMetricDimension('if_alias', 'bar')
metric = PanoptesMetric('test_metric', 0, PanoptesMetricType.GAUGE, metric_creation_timestamp=now)
metric_diff_timestamp = PanoptesMetric('test_metric', 0, PanoptesMetricType.GAUGE,
metric_creation_timestamp=now + 0.01)
metrics_group.add_dimension(dimension)
metrics_group_two.add_dimension(dimension)
self.assertEqual(metrics_group.__hash__(), metrics_group_two.__hash__())
metrics_group.add_metric(metric)
metrics_group_two.add_metric(metric_diff_timestamp)
self.assertEqual(metrics_group.__hash__(), metrics_group_two.__hash__())
metrics_group_set = PanoptesMetricsGroupSet()
metrics_group_set.add(metrics_group)
metrics_group_set.add(metrics_group_two)
assert len(metrics_group_set) == 1
class TestEnrichmentFramework(unittest.TestCase):
@patch('yahoo_panoptes.framework.resources.time', mock_time)
def setUp(self):
self.__panoptes_resource = PanoptesResource(resource_site='test',
resource_class='test',
resource_subclass='test',
resource_type='test',
resource_id='test',
resource_endpoint='test',
resource_plugin='test')
self.__panoptes_resource.add_metadata('test', 'test')
def test_enrichment_set(self):
enrichment_set = PanoptesEnrichmentSet('int_001')
enrichment_set.add('speed', 1000)
enrichment_set.add('index', 001)
enrichment_set.add('status', 'up')
self.assertEquals(enrichment_set.key, 'int_001')
self.assertDictEqual(enrichment_set.value, {
'status': 'up',
'index': 1,
'speed': 1000
})
self.assertEquals(len(enrichment_set), 3)
enrichment_set1 = PanoptesEnrichmentSet('int_002', {
'status': 'down',
'index': 2,
'speed': 1000
})
self.assertEquals(enrichment_set1.key, 'int_002')
self.assertDictEqual(enrichment_set1.value, {
'status': 'down',
'index': 2,
'speed': 1000
})
with self.assertRaises(AssertionError):
PanoptesEnrichmentSet('int_001', 'string')
with self.assertRaises(AssertionError):
PanoptesEnrichmentSet('int_001', 100)
def test_enrichment_schema_validator(self):
validator = PanoptesEnrichmentInterfaceSchemaValidator()
enrichment_set = PanoptesEnrichmentSet('int_001')
enrichment_set.add('speed', 1000)
enrichment_set.add('index', 001)
enrichment_set.add('status', 'up')
self.assertTrue(validator.validate(enrichment_set))
enrichment_set.add('status', 01)
self.assertFalse(validator.validate(enrichment_set))
@patch('time.time', mock_time)
def test_enrichment_group(self):
interface_validation_object = PanoptesEnrichmentInterfaceSchemaValidator(
)
neighbor_validation_object = PanoptesEnrichmentNeighborSchemaValidator(
)
interface_data = \
'''{"data": [
{"int_001": {"index": 1, "speed": 1000, "status": "up"}},
{"int_002": {"index": 2, "speed": 1000, "status": "down"}}],
"metadata": {"_enrichment_group_creation_timestamp": %f, "_enrichment_ttl": 300, "_execute_frequency": 60},
"namespace": "interface"}''' % mock_time.return_value
neighbor_data = \
'''{"data": [{"host_name": {"mac": "aa:bb:cc:dd:ee:ff", "property": "Test Property", "vlan_id": 501}}],
"metadata": {"_enrichment_group_creation_timestamp": %f, "_enrichment_ttl": 600, "_execute_frequency": 120},
"namespace": "neighbor"}''' % mock_time.return_value
with self.assertRaises(AssertionError):
PanoptesEnrichmentGroup(1, interface_validation_object, 300, 60)
with self.assertRaises(AssertionError):
PanoptesEnrichmentGroup('interface', 'non_validation_object', 300,
60)
with self.assertRaises(AssertionError):
PanoptesEnrichmentGroup('interface', interface_validation_object,
'300', 60)
with self.assertRaises(AssertionError):
PanoptesEnrichmentGroup('interface', interface_validation_object,
300, '60')
with self.assertRaises(AssertionError):
PanoptesEnrichmentGroup('interface', interface_validation_object,
0, 60)
with self.assertRaises(AssertionError):
PanoptesEnrichmentGroup('interface', interface_validation_object,
300, 0)
with self.assertRaises(AssertionError):
PanoptesEnrichmentGroup('interface', interface_validation_object, 300, 60).\
add_enrichment_set('not_PanoptesEnrichmentSet_obj')
enrichment_set1 = PanoptesEnrichmentSet('int_001')
enrichment_set1.add('speed', 1000)
enrichment_set1.add('index', 001)
enrichment_set1.add('status', 'up')
enrichment_set2 = PanoptesEnrichmentSet('int_002')
enrichment_set2.add('speed', 1000)
enrichment_set2.add('index', 002)
enrichment_set2.add('status', 'down')
enrichment_group1 = PanoptesEnrichmentGroup(
'interface', interface_validation_object, 300, 60)
enrichment_group1.add_enrichment_set(enrichment_set1)
enrichment_group1.add_enrichment_set(enrichment_set2)
self.assertEqual(enrichment_group1.namespace, 'interface')
self.assertEqual(enrichment_group1.enrichment_ttl, 300)
self.assertEqual(enrichment_group1.execute_frequency, 60)
self.assertEqual(enrichment_group1.enrichment_group_creation_timestamp,
mock_time.return_value)
self.assertEqual(
ordered(
json.loads(
json.dumps(enrichment_group1.data,
cls=PanoptesEnrichmentEncoder))),
ordered(json.loads(interface_data)['data']))
self.assertEqual(ordered(json.loads(enrichment_group1.json())),
ordered(json.loads(interface_data)))
self.assertEquals(len(enrichment_group1), 2)
enrichment_set3 = PanoptesEnrichmentSet('int_002')
enrichment_set3.add('speed', 1000)
enrichment_set3.add('index', 002)
enrichment_set3.add('status', 'down')
self.assertEqual(ordered(json.loads(enrichment_group1.json())),
ordered(json.loads(interface_data)))
self.assertEqual(ordered(enrichment_group1.metadata),
ordered(json.loads(interface_data)['metadata']))
self.assertEquals(len(enrichment_group1), 2)
test_metadata = json.loads(interface_data)['metadata']
test_metadata['metadata_key'] = 'metadata_value'
enrichment_group1.upsert_metadata('metadata_key', 'metadata_value')
self.assertEqual(ordered(enrichment_group1.metadata),
ordered(test_metadata))
enrichment_group1.upsert_metadata('ttl', 300)
with self.assertRaises(ValueError):
enrichment_group1.upsert_metadata('_enrichment_ttl', 300)
with self.assertRaises(AssertionError):
enrichment_group1.upsert_metadata('metadata', {})
with self.assertRaises(AssertionError):
enrichment_group1.upsert_metadata('metadata', [])
enrichment_set4 = PanoptesEnrichmentSet('host_name')
enrichment_set4.add('vlan_id', 501)
enrichment_set4.add('property', 'Test Property')
enrichment_set4.add('mac', 'aa:bb:cc:dd:ee:ff')
enrichment_group2 = PanoptesEnrichmentGroup(
'neighbor', neighbor_validation_object, 600, 120)
enrichment_group2.add_enrichment_set(enrichment_set4)
self.assertEqual(ordered(json.loads(enrichment_group2.json())),
ordered(json.loads(neighbor_data)))
self.assertEquals(len(enrichment_group2), 1)
enrichment_set5 = PanoptesEnrichmentSet('host_name01')
enrichment_set5.add('vlan_id', 502)
enrichment_set5.add('property', 'Netops01.US')
enrichment_set6 = PanoptesEnrichmentSet('host_name02')
enrichment_set6.add('vlan_id', 503)
enrichment_set6.add('mac', 'aa:bb:cc:dd:ee:ff')
enrichment_group3 = PanoptesEnrichmentGroup(
'neighbor', neighbor_validation_object, 600, 120)
enrichment_group3.add_enrichment_set(enrichment_set5)
with self.assertRaises(AssertionError):
enrichment_group3.add_enrichment_set(enrichment_set6)
interface_store_data = '{"int_001": {"index": 1, "speed": 1000, "status": "up"}, ' \
'"int_002": {"index": 2, "speed": 1000, "status": "down"}}'
neighbor_store_data = '{"host_name": {"mac": "aa:bb:cc:dd:ee:ff", "property": "Test Property", "vlan_id": 501}}'
self.assertEquals(
ordered(json.loads(enrichment_group1.serialize_data())),
ordered(json.loads(interface_store_data)))
self.assertEquals(
ordered(json.loads(enrichment_group2.serialize_data())),
ordered(json.loads(neighbor_store_data)))
enrichment_group1.upsert_metadata('ttl', 300)
with self.assertRaises(ValueError):
enrichment_group1.upsert_metadata('_enrichment_ttl', 300)
interface_data_serialized = '''{{"data": {{"int_001": {{"index": 1, "speed": 1000, "status": "up"}},
"int_002": {{"index": 2, "speed": 1000, "status": "down"}}}}, "metadata":
{{"_enrichment_group_creation_timestamp": {:.5f}, "_enrichment_ttl": 300, "_execute_frequency": 60,
"metadata_key": "metadata_value", "ttl": 300}}}}'''.format(
mock_time.return_value)
neighbor_data_serialized = '''{{"data": {{"host_name": {{"mac": "aa:bb:cc:dd:ee:ff", "property": "Test Property",
"vlan_id": 501}}}}, "metadata": {{"_enrichment_group_creation_timestamp": {:.5f},
"_enrichment_ttl": 600, "_execute_frequency": 120}}}}'''.format(
mock_time.return_value)
self.assertEquals(ordered(json.loads(enrichment_group1.serialize())),
ordered(json.loads(interface_data_serialized)))
self.assertEquals(ordered(json.loads(enrichment_group2.serialize())),
ordered(json.loads(neighbor_data_serialized)))
@patch('time.time', mock_time)
def test_enrichment_group_set(self):
interface_validation_object = PanoptesEnrichmentInterfaceSchemaValidator(
)
neighbor_validation_object = PanoptesEnrichmentNeighborSchemaValidator(
)
panoptes_resource = self.__panoptes_resource
enrichment_data = \
'''{{"enrichment": [{{"metadata": {{"_enrichment_group_creation_timestamp": {:.5f}, "_enrichment_ttl": 600,
"_execute_frequency": 120}}, "data": [{{"host_name":
{{"mac": "aa:bb:cc:dd:ee:ff", "property": "Test Property", "vlan_id": 501}}}}],
"namespace": "neighbor"}}, {{"metadata": {{"_enrichment_group_creation_timestamp": {:.5f},
"_enrichment_ttl": 300,
"_execute_frequency": 60}}, "data": [
{{"int_001": {{"index": 1, "speed": 1000, "status": "up"}}}}, {{"int_002": {{"index": 2, "speed": 1000,
"status": "down"}}}}], "namespace": "interface"}}],
"enrichment_group_set_creation_timestamp": {:.5f}, "resource": {{"resource_class": "test",
"resource_creation_timestamp": {:.5f}, "resource_endpoint": "test", "resource_id": "test",
"resource_metadata": {{"_resource_ttl": "604800", "test": "test"}}, "resource_plugin": "test",
"resource_site": "test",
"resource_subclass": "test", "resource_type": "test"}}}}'''.format(mock_time.return_value,
mock_time.return_value,
mock_time.return_value,
mock_time.return_value)
enrichment_set1 = PanoptesEnrichmentSet('int_001')
enrichment_set1.add('speed', 1000)
enrichment_set1.add('index', 001)
enrichment_set1.add('status', 'up')
enrichment_set2 = PanoptesEnrichmentSet('int_002')
enrichment_set2.add('speed', 1000)
enrichment_set2.add('index', 002)
enrichment_set2.add('status', 'down')
enrichment_group1 = PanoptesEnrichmentGroup(
'interface', interface_validation_object, 300, 60)
enrichment_group1.add_enrichment_set(enrichment_set1)
enrichment_group1.add_enrichment_set(enrichment_set2)
enrichment_set3 = PanoptesEnrichmentSet('host_name')
enrichment_set3.add('vlan_id', 501)
enrichment_set3.add('property', 'Test Property')
enrichment_set3.add('mac', 'aa:bb:cc:dd:ee:ff')
enrichment_group2 = PanoptesEnrichmentGroup(
'neighbor', neighbor_validation_object, 600, 120)
enrichment_group2.add_enrichment_set(enrichment_set3)
enrichment_group_set1 = PanoptesEnrichmentGroupSet(panoptes_resource)
enrichment_group_set1.add_enrichment_group(enrichment_group1)
enrichment_group_set1.add_enrichment_group(enrichment_group2)
self.assertEquals(len(enrichment_group_set1), 2)
group_set_repr = "PanoptesEnrichmentGroupSet({{'enrichment_group_set_creation_timestamp': {:.5f}, " \
"'resource': plugin|test|site|test|class|test|subclass|test" \
"|type|test|id|test|endpoint|test, " \
"'enrichment': set([PanoptesEnrichmentGroup({{'data': set([PanoptesEnrichmentSet({{" \
"'int_001': {{'status': 'up', 'index': 1, 'speed': 1000}}}}), " \
"PanoptesEnrichmentSet({{'int_002': {{'status': 'down', 'index': 2, 'speed': 1000}}}})]), " \
"'namespace': 'interface', 'metadata': " \
"{{'_enrichment_group_creation_timestamp': {:.5f}, '_enrichment_ttl': 300, " \
"'_execute_frequency': 60}}}}), PanoptesEnrichmentGroup({{" \
"'data': set([PanoptesEnrichmentSet(" \
"{{'host_name': {{" \
"'mac': 'aa:bb:cc:dd:ee:ff', 'property': 'Test Property', 'vlan_id': 501}}}})]), " \
"'namespace': 'neighbor', 'metadata': {{'_enrichment_group_creation_timestamp': " \
"{:.5f}, '_enrichment_ttl': 600, " \
"'_execute_frequency': 120}}}})])}})".format(mock_time.return_value,
mock_time.return_value,
mock_time.return_value)
self.assertEquals(repr(enrichment_group_set1), group_set_repr)
self.assertIsInstance(enrichment_group_set1.resource, PanoptesResource)
self.assertEqual(
enrichment_group_set1.enrichment_group_set_creation_timestamp,
mock_time.return_value)
self.assertEqual(
ordered(
json.loads(
json.dumps(enrichment_group_set1.enrichment,
cls=PanoptesEnrichmentEncoder))),
ordered(json.loads(enrichment_data)['enrichment']))
self.assertEqual(
ordered(json.loads(enrichment_group_set1.json())['enrichment']),
ordered(json.loads(enrichment_data)['enrichment']))
with self.assertRaises(AssertionError):
PanoptesEnrichmentGroupSet('bad_resource')
with self.assertRaises(AssertionError):
PanoptesEnrichmentGroupSet(panoptes_resource).add_enrichment_group(
'non_PanoptesEnrichmentGroup_obj')
enrichment_group_set2 = PanoptesEnrichmentGroupSet(panoptes_resource)
enrichment_group3 = PanoptesEnrichmentGroup(
'interface', interface_validation_object, 300, 60)
with self.assertRaises(AssertionError):
enrichment_group_set2.add_enrichment_group(enrichment_group3)
self.assertFalse(enrichment_group_set1 == enrichment_group1)
self.assertFalse(enrichment_group_set1 == enrichment_group_set2)
@patch('time.time', mock_time)
@patch('yahoo_panoptes.framework.resources.time', mock_time)
def test_multi_enrichment_group_set(self):
interface_validation_object = PanoptesEnrichmentInterfaceSchemaValidator(
)
neighbor_validation_object = PanoptesEnrichmentNeighborSchemaValidator(
)
panoptes_resource = self.__panoptes_resource
multi_enrichment_results_data = \
{
"group_sets": [
{
"enrichment": [
{
"data": [
{
"host_name": {
"mac": "aa:bb:cc:dd:ee:ff",
"property": "Test Property",
"vlan_id": 501
}
},
{
"host_name01": {
"mac": "aa:bb:cc:dd:ee:ff",
"property": "Test Property",
"vlan_id": 502
}
}
],
"metadata": {
"_enrichment_group_creation_timestamp": mock_time.return_value,
"_enrichment_ttl": 600,
"_execute_frequency": 120
},
"namespace": "neighbor"
},
{
"data": [
{
"int_001": {
"index": 1,
"speed": 1000,
"status": "up"
}
}
],
"metadata": {
"_enrichment_group_creation_timestamp": mock_time.return_value,
"_enrichment_ttl": 300,
"_execute_frequency": 60
},
"namespace": "interface"
}
],
"enrichment_group_set_creation_timestamp": mock_time.return_value,
"resource": {
"resource_class": "test_class",
"resource_creation_timestamp": mock_time.return_value,
"resource_endpoint": "test_endpoint01",
"resource_id": "test_resource_id01",
"resource_metadata": {
"_resource_ttl": "604800"
},
"resource_plugin": "test_plugin",
"resource_site": "test_site",
"resource_subclass": "test_subclass",
"resource_type": "test_type"
}
},
{
"enrichment": [
{
"data": [
{
"int_001": {
"index": 1,
"speed": 1000,
"status": "up"
}
}
],
"metadata": {
"_enrichment_group_creation_timestamp": mock_time.return_value,
"_enrichment_ttl": 300,
"_execute_frequency": 60
},
"namespace": "interface"
},
{
"data": [
{
"host_name": {
"mac": "aa:bb:cc:dd:ee:ff",
"property": "Test Property",
"vlan_id": 501
}
}
],
"metadata": {
"_enrichment_group_creation_timestamp": mock_time.return_value,
"_enrichment_ttl": 600,
"_execute_frequency": 120
},
"namespace": "neighbor"
}
],
"enrichment_group_set_creation_timestamp": mock_time.return_value,
"resource": {
"resource_class": "test",
"resource_creation_timestamp": mock_time.return_value,
"resource_endpoint": "test",
"resource_id": "test",
"resource_metadata": {
"_resource_ttl": "604800",
"test": "test"
},
"resource_plugin": "test",
"resource_site": "test",
"resource_subclass": "test",
"resource_type": "test"
}
}
]
}
enrichment_set1 = PanoptesEnrichmentSet('int_001')
enrichment_set1.add('speed', 1000)
enrichment_set1.add('index', 001)
enrichment_set1.add('status', 'up')
enrichment_group1 = PanoptesEnrichmentGroup(
'interface', interface_validation_object, 300, 60)
enrichment_group1.add_enrichment_set(enrichment_set1)
enrichment_set3 = PanoptesEnrichmentSet('host_name')
enrichment_set3.add('vlan_id', 501)
enrichment_set3.add('property', 'Test Property')
enrichment_set3.add('mac', 'aa:bb:cc:dd:ee:ff')
enrichment_group2 = PanoptesEnrichmentGroup(
'neighbor', neighbor_validation_object, 600, 120)
enrichment_group2.add_enrichment_set(enrichment_set3)
enrichment_group_set1 = PanoptesEnrichmentGroupSet(panoptes_resource)
enrichment_group_set1.add_enrichment_group(enrichment_group1)
enrichment_group_set1.add_enrichment_group(enrichment_group2)
panoptes_resource01 = PanoptesResource(
resource_site='test_site',
resource_class='test_class',
resource_subclass='test_subclass',
resource_type='test_type',
resource_id='test_resource_id01',
resource_endpoint='test_endpoint01',
resource_plugin='test_plugin')
panoptes_resource02 = PanoptesResource(
resource_site='test_site',
resource_class='test_class',
resource_subclass='test_subclass',
resource_type='test_type',
resource_id='test_resource_id02',
resource_endpoint='test_endpoint02',
resource_plugin='test_plugin')
enrichment_set4 = PanoptesEnrichmentSet('host_name01')
enrichment_set4.add('vlan_id', 502)
enrichment_set4.add('property', 'Test Property')
enrichment_set4.add('mac', 'aa:bb:cc:dd:ee:ff')
enrichment_group3 = PanoptesEnrichmentGroup(
'neighbor', neighbor_validation_object, 600, 120)
enrichment_group3.add_enrichment_set(enrichment_set3)
enrichment_group3.add_enrichment_set(enrichment_set4)
enrichment_group_set2 = PanoptesEnrichmentGroupSet(panoptes_resource01)
enrichment_group_set2.add_enrichment_group(enrichment_group1)
enrichment_group_set2.add_enrichment_group(enrichment_group3)
multi_enrichment_group_set = PanoptesEnrichmentMultiGroupSet()
multi_enrichment_group_set.add_enrichment_group_set(
enrichment_group_set1)
multi_enrichment_group_set.add_enrichment_group_set(
enrichment_group_set2)
multi_enrichment_group_set_repr = \
"PanoptesEnrichmentMultiGroupSet({{'group_sets': set([PanoptesEnrichmentGroupSet({{" \
"'enrichment_group_set_creation_timestamp': {:.5f}, 'resource': plugin|test_plugin|" \
"site|test_site|class|test_class|subclass|test_subclass|type|test_type|id|test_resource_id01|" \
"endpoint|test_endpoint01, 'enrichment': set([PanoptesEnrichmentGroup({{'data': set([" \
"PanoptesEnrichmentSet({{'int_001': {{'status': 'up', 'index': 1, 'speed': 1000}}}})]), " \
"'namespace': 'interface', 'metadata': {{'_enrichment_group_creation_timestamp': {:.5f}, " \
"'_enrichment_ttl': 300, '_execute_frequency': 60}}}}), PanoptesEnrichmentGroup({{" \
"'data': set([PanoptesEnrichmentSet({{'host_name': {{" \
"'mac': 'aa:bb:cc:dd:ee:ff', 'property': 'Test Property', " \
"'vlan_id': 501}}}}), PanoptesEnrichmentSet({{'host_name01': {{'mac': 'aa:bb:cc:dd:ee:ff', " \
"'property': 'Test Property', 'vlan_id': 502}}}})]), 'namespace': 'neighbor', 'metadata': {{" \
"'_enrichment_group_creation_timestamp': {:.5f}, '_enrichment_ttl': 600, " \
"'_execute_frequency': 120}}}})])}}), PanoptesEnrichmentGroupSet({{" \
"'enrichment_group_set_creation_timestamp': {:.5f}, 'resource': plugin|test|site|test|" \
"class|test|subclass|test|type|test|id|test|endpoint|test, 'enrichment': set([PanoptesEnrichmentGroup({{" \
"'data': set([PanoptesEnrichmentSet({{'int_001': {{'status': 'up', 'index': 1, 'speed': 1000}}}})]), " \
"'namespace': 'interface', 'metadata': {{'_enrichment_group_creation_timestamp': {:.5f}, " \
"'_enrichment_ttl': 300, '_execute_frequency': 60}}}}), PanoptesEnrichmentGroup({{'data': set([" \
"PanoptesEnrichmentSet({{'host_name': {{'mac': 'aa:bb:cc:dd:ee:ff', 'property': 'Test Property', " \
"'vlan_id': 501}}}})]), 'namespace': 'neighbor', 'metadata': {{'_enrichment_group_creation_timestamp': " \
"{:.5f}, '_enrichment_ttl': 600, '_execute_frequency': 120}}}})])}})])}})".format(
mock_time.return_value,
mock_time.return_value,
mock_time.return_value,
mock_time.return_value,
mock_time.return_value,
mock_time.return_value)
self.assertEquals(repr(multi_enrichment_group_set),
multi_enrichment_group_set_repr)
self.assertEquals(
len(multi_enrichment_group_set.enrichment_group_sets), 2)
self.assertEquals(
ordered(json.loads(multi_enrichment_group_set.json())),
ordered(multi_enrichment_results_data))
self.assertEquals(len(multi_enrichment_group_set), 2)
multi_enrichment_group_set.add_enrichment_group_set(
enrichment_group_set2)
self.assertEquals(len(multi_enrichment_group_set), 2)
enrichment_group_set3 = PanoptesEnrichmentGroupSet(panoptes_resource02)
enrichment_group_set3.add_enrichment_group(enrichment_group1)
multi_enrichment_group_set.add_enrichment_group_set(
enrichment_group_set3)
self.assertEquals(len(multi_enrichment_group_set), 3)
with self.assertRaises(AssertionError):
multi_enrichment_group_set.add_enrichment_group_set(
'non_enrichment_group')
enrichment_group_set3 = PanoptesEnrichmentGroupSet(panoptes_resource01)
with self.assertRaises(AssertionError):
multi_enrichment_group_set.add_enrichment_group_set(
enrichment_group_set3)