def setUp(self):
self.__panoptes_resource = PanoptesResource(resource_site=u'test', resource_class=u'test',
resource_subclass=u'test',
resource_type=u'test', resource_id=u'test',
resource_endpoint=u'test',
resource_plugin=u'test')
self.__panoptes_resource.add_metadata(u'test', u'test')
def setUp(self):
_, self.panoptes_test_conf_file = get_test_conf_file()
self._my_path = os.path.dirname(os.path.abspath(__file__))
self._panoptes_context = PanoptesContext(self.panoptes_test_conf_file)
self._resource = PanoptesResource(
resource_class="test_class",
resource_subclass="test_subclass",
resource_type="test_type",
resource_id="test_id",
resource_endpoint="127.0.0.1",
resource_site="test_site",
resource_creation_timestamp=time.time(),
resource_plugin="test_plugin")
self._key_value_store = PanoptesTestKeyValueStore(
self._panoptes_context)
self._secrets_store = TestPanoptesSecretsStore(self._panoptes_context)
self._secrets_store.set(key='snmp_community_string:test_site',
value='public')
self._panoptes_plugin_context = PanoptesPluginContext(
panoptes_context=self._panoptes_context,
logger_name="test_logger",
config=plugin_conf,
key_value_store=self._key_value_store,
secrets_store=self._secrets_store,
data=self._resource)
def test_code_output(self):
# Mock the system calls
subprocess.check_output = Mock(return_value=TEST_PING_RESPONSE)
resource = PanoptesResource.resource_from_dict(self._resource_dict)
test_ping = PanoptesPingConnectionFactory.get_ping_connection(resource=resource,
context=None,
count=3,
timeout=100)
test_d = {"round_trip_stddev": 0.057, "packets_received": 3, "execution_time": 1.44,
"round_trip_avg": 0.120, "packets_transmitted": 3, "packet_loss_pct": 0.0,
"round_trip_max": 0.162, "round_trip_min": 0.040}
self.assertDictEqual(test_d, json.loads(test_ping.response))
self.assertEqual(test_ping.packets_transmitted, 3)
self.assertEqual(test_ping.packets_received, 3)
self.assertEqual(test_ping.packet_loss_pct, 0.0)
self.assertEqual(test_ping.execution_time, 1.44) # add for testing prod
self.assertEqual(test_ping.round_trip_min, 0.040)
self.assertEqual(test_ping.round_trip_avg, 0.120)
self.assertEqual(test_ping.round_trip_max, 0.162)
self.assertEqual(test_ping.round_trip_stddev, 0.057)
# Test bad parameters
with self.assertRaises(AssertionError):
PanoptesPingConnectionFactory.get_ping_connection(resource=resource,
context=None,
count=-1,
timeout=100)
with self.assertRaises(AssertionError):
PanoptesPingConnectionFactory.get_ping_connection(resource=resource,
context=None,
count='1',
timeout=100)
with self.assertRaises(AssertionError):
PanoptesPingConnectionFactory.get_ping_connection(resource=None,
context=None,
count=3,
timeout=100)
with self.assertRaises(AssertionError):
self._resource_dict['resource_endpoint'] = ''
resource = PanoptesResource.resource_from_dict(self._resource_dict)
PanoptesPingConnectionFactory.get_ping_connection(resource=resource,
context=None,
count=3,
timeout=100)
with self.assertRaises(AssertionError):
self._resource_dict['resource_endpoint'] = 'localhost'
resource = PanoptesResource.resource_from_dict(self._resource_dict)
PanoptesPingConnectionFactory.get_ping_connection(resource=resource,
context=None,
count=3,
timeout=0)
with self.assertRaises(AssertionError):
PanoptesPingConnectionFactory.get_ping_connection(resource=resource,
context=None,
count=3,
timeout=-1)
def setUp(self):
self._panoptes_metric = {
u'metrics_group_interval':
60,
u'resource': {
u'resource_site': u'test_site',
u'resource_id': u'test_id',
u'resource_class': u'network',
u'resource_plugin': u'test_plugin',
u'resource_creation_timestamp': 1567823517.46,
u'resource_subclass': u'test_subclass',
u'resource_endpoint': u'test_endpoint',
u'resource_metadata': {
u'test_metadata_key': u'test_metadata_value',
u'_resource_ttl': u"604800"
},
u'resource_type': u'test_type'
},
u'dimensions': [{
u'dimension_name': u'cpu_name',
u'dimension_value': u'test_cpu_name_value'
}, {
u'dimension_name': u'cpu_no',
u'dimension_value': u'test_cpu_no_value'
}, {
u'dimension_name': u'cpu_type',
u'dimension_value': u'test_cpu_type_value'
}],
u'metrics_group_type':
u'cpu',
u'metrics': [{
u'metric_creation_timestamp': 1567823946.72,
u'metric_type': u'gauge',
u'metric_name': u'cpu_utilization',
u'metric_value': 0
}],
u'metrics_group_creation_timestamp':
1567823946.72,
u'metrics_group_schema_version':
u'0.2'
}
self._panoptes_resource = PanoptesResource(resource_site=u'test',
resource_class=u'test',
resource_subclass=u'test',
resource_type=u'test',
resource_id=u'test',
resource_endpoint=u'test',
resource_plugin=u'test')
self._panoptes_resource.add_metadata(u'test', u'test')
self._panoptes_resource_set = PanoptesResourceSet()
self._panoptes_resource_set.add(self._panoptes_resource)
def setUp(self):
self.__panoptes_resource_metadata = {'test': 'test', '_resource_ttl': '604800'}
self.__panoptes_resource = PanoptesResource(resource_site='test', resource_class='test',
resource_subclass='test',
resource_type='test', resource_id='test', resource_endpoint='test',
resource_plugin='test',
resource_creation_timestamp=_TIMESTAMP,
resource_ttl=RESOURCE_MANAGER_RESOURCE_EXPIRE)
self.__panoptes_resource.add_metadata('test', 'test')
self.__panoptes_resource_set = PanoptesResourceSet()
mock_valid_timestamp = Mock(return_value=True)
with patch('yahoo_panoptes.framework.resources.PanoptesValidators.valid_timestamp',
mock_valid_timestamp):
self.__panoptes_resource_set.resource_set_creation_timestamp = _TIMESTAMP
self.my_dir, self.panoptes_test_conf_file = _get_test_conf_file()
def set_panoptes_resource(self):
self._panoptes_resource = PanoptesResource(
resource_site=self._resource_site,
resource_class=self._resource_class,
resource_subclass=self._resource_subclass,
resource_type=self._resource_type,
resource_id=self._resource_id,
resource_endpoint=self._resource_endpoint,
resource_plugin=self._resource_plugin)
self._panoptes_resource.resource_metadata['model'] = 'model'
if self._resource_backplane:
self._panoptes_resource.add_metadata('backplane',
self._resource_backplane)
def setUp(self):
self.__panoptes_resource = PanoptesResource(resource_site='test', resource_class='test',
resource_subclass='test',
resource_type='test', resource_id='test', resource_endpoint='test',
resource_plugin='test',
resource_creation_timestamp=_TIMESTAMP,
resource_ttl=RESOURCE_MANAGER_RESOURCE_EXPIRE)
def setUp(self):
self.my_dir, self.panoptes_test_conf_file = get_test_conf_file()
self._panoptes_resource = PanoptesResource(
resource_site="test",
resource_class="test",
resource_subclass="test",
resource_type="test",
resource_id="test",
resource_endpoint="test",
resource_creation_timestamp=_TIMESTAMP,
resource_plugin="test")
self._panoptes_context = PanoptesContext(
self.panoptes_test_conf_file,
key_value_store_class_list=[
PanoptesTestKeyValueStore, PanoptesResourcesKeyValueStore,
PanoptesPollingPluginKeyValueStore, PanoptesSecretsStore,
PanoptesPollingPluginAgentKeyValueStore,
PanoptesDiscoveryPluginAgentKeyValueStore,
PanoptesDiscoveryPluginKeyValueStore
],
create_message_producer=False,
async_message_producer=False,
create_zookeeper_client=True)
self._runner_class = PanoptesPluginWithEnrichmentRunner
self._log_capture = LogCapture(
attributes=TestPanoptesPluginRunner.extract)
def setUp(self):
self._panoptes_resource = PanoptesResource(
resource_site=u'test_site',
resource_class=u'test_class',
resource_subclass=u'test_subclass',
resource_type=u'test_type',
resource_id=u'test_id',
resource_endpoint=u'test_endpoint',
resource_plugin=u'test_plugin')
self._plugin_config = {
u'Core': {
u'name': u'Test Plugin',
u'module': u'plugin_polling_ping'
},
u'main': {
u'resource_filter': u'resource_class = u"network"',
u'execute_frequency': 60,
}
}
self._panoptes_context = create_autospec(PanoptesContext)
self._panoptes_plugin_context = create_autospec(
PanoptesPluginWithEnrichmentContext,
instance=True,
spec_set=True,
data=self._panoptes_resource,
config=self._plugin_config,
logger=logging.getLogger(__name__))
def test_panoptes_resource_set(self):
panoptes_resource = self.__panoptes_resource
panoptes_resource_set = self.__panoptes_resource_set
self.assertEqual(panoptes_resource_set.add(panoptes_resource), None)
self.assertEqual(len(panoptes_resource_set), 1)
self.assertEqual(type(panoptes_resource_set.resources), set)
self.assertIsInstance(str(panoptes_resource_set), str)
self.assertIsInstance(iter(panoptes_resource_set), collections.Iterable)
self.assertEqual(panoptes_resource_set.next(), panoptes_resource)
self.assertEqual(panoptes_resource_set.remove(panoptes_resource), None)
self.assertEqual(len(panoptes_resource_set), 0)
self.assertEqual(panoptes_resource_set.resource_set_creation_timestamp, _TIMESTAMP)
with self.assertRaises(AssertionError):
panoptes_resource_set.resource_set_creation_timestamp = 0
panoptes_resource_2 = PanoptesResource(resource_site='test', resource_class='test',
resource_subclass='test',
resource_type='test', resource_id='test2', resource_endpoint='test',
resource_plugin='test',
resource_ttl=RESOURCE_MANAGER_RESOURCE_EXPIRE,
resource_creation_timestamp=_TIMESTAMP)
panoptes_resource_set.add(panoptes_resource)
panoptes_resource_set.add(panoptes_resource_2)
self.assertEqual(len(panoptes_resource_set.get_resources_by_site()['test']), 2)
self.assertEqual(panoptes_resource_set.resource_set_schema_version, "0.1")
panoptes_resource_set_json = {
u'resource_set_creation_timestamp': _TIMESTAMP,
u'resource_set_schema_version': u'0.1',
u'resources': [
{
u'resource_site': u'test',
u'resource_class': u'test',
u'resource_subclass': u'test',
u'resource_type': u'test',
u'resource_id': u'test2',
u'resource_endpoint': u'test',
u'resource_metadata': {
u'_resource_ttl': u'604800'
},
u'resource_creation_timestamp': _TIMESTAMP,
u'resource_plugin': u'test'
},
{
u'resource_site': u'test',
u'resource_class': u'test',
u'resource_subclass': u'test',
u'resource_type': u'test',
u'resource_id': u'test',
u'resource_endpoint': u'test',
u'resource_metadata': {
u'_resource_ttl': u'604800',
u'test': u'test'
},
u'resource_creation_timestamp': _TIMESTAMP,
u'resource_plugin': u'test'
}
]
}
self.assertEqual(ordered(panoptes_resource_set_json), ordered(json.loads(panoptes_resource_set.json)))
def setUp(self):
self.my_dir, self.panoptes_test_conf_file = get_test_conf_file()
self._panoptes_context = PanoptesContext(self.panoptes_test_conf_file,
key_value_store_class_list=[PanoptesTestKeyValueStore],
create_message_producer=False, async_message_producer=False,
create_zookeeper_client=True)
self._panoptes_resource = PanoptesResource(resource_site='test', resource_class='test',
resource_subclass='test',
resource_type='test', resource_id='test', resource_endpoint='test',
resource_plugin='test')
def run(self, context):
"""
The main entry point to the plugin
Args:
context (PanoptesPluginContext): The Plugin Context passed by the Plugin Agent
Returns:
PanoptesResourceSet: A non-empty resource set
Raises:
PanoptesDiscoveryPluginError: This exception is raised if any part of the lookup process has errors
"""
assert context and isinstance(context, PanoptesPluginContext), 'context must be a PanoptesPluginContext'
conf = context.config
logger = context.logger
config_file = None
try:
config_file = conf['main']['config_file']
with open(config_file) as f:
resource_specs = json.load(f)
except Exception as e:
raise PanoptesDiscoveryPluginError(
'Error while attempting to parse JSON from file {}: {}'.format(config_file, repr(e))
)
resources = PanoptesResourceSet()
num_successes = 0
num_failures = 0
for resource_spec in resource_specs:
try:
resource = PanoptesResource.resource_from_dict(resource_spec)
resources.add(resource)
num_successes += 1
logger.debug('Added resource {} from JSON file {}'.format(resource, config_file))
except Exception as e:
logger.debug('Error while attempting to create a PanoptesResource from file {}: {}'.format(
config_file, repr(e)))
num_failures += 1
continue
if num_successes > 0:
logger.info('Tried to read {} resources from {}, {} failed'.format(num_successes + num_failures,
config_file,
num_failures))
else:
logger.error('Error while attempting to create PanoptesResources from {}.'.format(config_file))
raise PanoptesDiscoveryPluginError(
'Error during lookup for PanoptesResource from file {}.'.format(config_file))
return resources
def setUp(self):
super(TestPanoptesPluginWithEnrichmentRunner, self).setUp()
self._panoptes_resource = PanoptesResource(
resource_site="test",
resource_class="test",
resource_subclass="test",
resource_type="test",
resource_id="test",
resource_endpoint="test",
resource_creation_timestamp=_TIMESTAMP,
resource_plugin="test")
self._runner_class = PanoptesPluginWithEnrichmentRunner
def test_steamroller_ping(self):
plugin_conf = {
'Core': {
'name': 'Test Plugin',
'module': 'test_plugin'
},
'main': {
'execute_frequency': '60',
'resource_filter': 'resource_class = "network"'
}
}
self._resource_dict['resource_metadata']['ping_route'] = 'steamroller'
self._resource_dict['resource_metadata']['snmp_proxy_hosts'] = 'proxyhost-5.doesntexist.com:4443'
resource = PanoptesResource.resource_from_dict(self._resource_dict)
plugin_context = create_autospec(
PanoptesPluginWithEnrichmentContext, instance=True, spec_set=True,
data=resource,
config=plugin_conf,
snmp=panoptes_context.config_object.snmp_defaults,
x509=panoptes_context.config_object.x509_defaults,
secrets=secret_store,
logger=logging.getLogger(__name__)
)
with patch('requests.Session') as rs:
rs.return_value = MockConnection()
ping_connection = PanoptesPingConnectionFactory.get_ping_connection(resource, plugin_context)
self.assertEqual(ping_connection.packets_transmitted, 10)
self.assertEqual(ping_connection.packets_received, 10)
self.assertEqual(ping_connection.packet_loss_pct, 0.0)
self.assertEqual(ping_connection.execution_time, 9.03)
self.assertEqual(ping_connection.round_trip_min, 0.485)
self.assertEqual(ping_connection.round_trip_avg, 0.504)
self.assertEqual(ping_connection.round_trip_max, 0.535)
self.assertEqual(ping_connection.round_trip_stddev, 0.027)
with patch.object(requests.Session, 'post') as rs:
rs.side_effect = requests.exceptions.Timeout()
with self.assertRaises(PanoptesPingTimeoutException):
PanoptesPingConnectionFactory.get_ping_connection(resource, plugin_context)
with patch.object(requests.Session, 'post') as rs:
rs.side_effect = requests.exceptions.ConnectionError()
with self.assertRaises(PanoptesPingConnectionError):
PanoptesPingConnectionFactory.get_ping_connection(resource, plugin_context)
with patch.object(requests.Session, 'post') as rs:
rs.side_effect = requests.exceptions.RequestException()
with self.assertRaises(PanoptesPingRequestException):
PanoptesPingConnectionFactory.get_ping_connection(resource, plugin_context)
def prepare_panoptes_metrics_group_set(self, file_path=None):
panoptes_metric_group_set = PanoptesMetricsGroupSet()
path_to_metrics_file = plugin_results_file if file_path is None else file_path
with open(path_to_metrics_file) as results_file:
panoptes_json_data = json.load(results_file)
for panoptes_data_object in panoptes_json_data:
resource = panoptes_data_object[u'resource']
panoptes_resource = PanoptesResource(
resource_site=resource[u'resource_site'],
resource_class=resource[u'resource_class'],
resource_subclass=resource[u'resource_subclass'],
resource_type=resource[u'resource_type'],
resource_id=resource[u'resource_id'],
resource_endpoint=resource[u'resource_endpoint'],
resource_plugin=resource[u'resource_plugin'],
resource_creation_timestamp=0)
panoptes_metric_group = PanoptesMetricsGroup(
resource=panoptes_resource,
group_type=panoptes_data_object[u'metrics_group_type'],
interval=panoptes_data_object[u'metrics_group_interval']
)
for dimension in panoptes_data_object[u'dimensions']:
panoptes_metric_group.add_dimension(
PanoptesMetricDimension(
name=dimension[u'dimension_name'],
value=dimension[u'dimension_value']
)
)
for metric in panoptes_data_object[u'metrics']:
panoptes_metric_group.add_metric(
PanoptesMetric(
metric_name=metric[u'metric_name'],
metric_value=metric[u'metric_value'],
metric_type=PanoptesMetricType().GAUGE if metric[u'metric_type'] == u'gauge'
else PanoptesMetricType().COUNTER,
metric_creation_timestamp=metric[u'metric_creation_timestamp']
)
)
panoptes_metric_group_set.add(panoptes_metric_group)
return panoptes_metric_group_set
class PanoptesTestPollingPlugin(PanoptesPollingPlugin):
name = "Panoptes Test Polling Plugin 2"
panoptes_resource = PanoptesResource(resource_site='test',
resource_class='test',
resource_subclass='test',
resource_type='test',
resource_id='test',
resource_endpoint='test',
resource_plugin='test')
def run(self, context):
metric1 = PanoptesMetric("test", 0.0, PanoptesMetricType.GAUGE)
metric_group = PanoptesMetricsGroup(self.panoptes_resource, "Test",
_TEST_INTERVAL)
metric_group.add_metric(metric1)
return metric_group
class PanoptesTestPollingPlugin(PanoptesDiscoveryPlugin):
name = "Test Discovery Plugin"
resource = PanoptesResource(resource_site='test_site',
resource_class='test_class',
resource_subclass='test_subclass',
resource_type='test_type',
resource_id='test_resource_id',
resource_endpoint='test_resource_endpoint',
resource_plugin='test_resource_plugin',
resource_creation_timestamp=0)
def run(self, context):
panoptes_resource_set = PanoptesResourceSet()
panoptes_resource_set.add(self.resource)
return panoptes_resource_set
def test_ping_unknown_host(self):
self._resource_dict['resource_endpoint'] = 'localhostx'
resource = PanoptesResource.resource_from_dict(self._resource_dict)
unknown_host_response = "ping: cannot resolve {}: Unknown host".format(self._resource_dict['resource_endpoint'])
e = subprocess.CalledProcessError(returncode=None, cmd=None,
output=unknown_host_response) # Shouldn't I be mocking this?
# When I try: "TypeError: exceptions must be old-style classes or derived from BaseException, not Mock"
subprocess.check_output = Mock(side_effect=e)
test_fail_d = {"round_trip_stddev": None, "packets_received": None, "execution_time": None,
"round_trip_avg": None, "packets_transmitted": None, "packet_loss_pct": None,
"round_trip_max": None, "round_trip_min": None}
with self.assertRaises(PanoptesPingException):
test_unknown_host_ping = PanoptesPingConnectionFactory.get_ping_connection(resource=resource,
context=None,
count=3,
timeout=100)
self.assertDictEqual(test_fail_d, test_unknown_host_ping.response)
def test_ping_timeout(self):
e = subprocess.CalledProcessError(returncode=None, cmd=None, output=TEST_PING_RESPONSE)
# When I try: "TypeError: exceptions must be old-style classes or derived from BaseException, not Mock"
subprocess.check_output = Mock(side_effect=e)
test_d = {"round_trip_stddev": 0.057, "packets_received": 3, "execution_time": 1.44,
"round_trip_avg": 0.120, "packets_transmitted": 3, "packet_loss_pct": 0.0,
"round_trip_max": 0.162, "round_trip_min": 0.040}
with self.assertRaises(PanoptesPingTimeoutException):
resource = PanoptesResource.resource_from_dict(self._resource_dict)
test_timeout_ping = PanoptesPingConnectionFactory.get_ping_connection(resource=resource,
context=None,
count=3,
timeout=100)
self.assertDictEqual(test_d, json.loads(test_timeout_ping.response))
self.assertEqual(test_timeout_ping.packets_transmitted, 3)
self.assertEqual(test_timeout_ping.packets_received, 3)
self.assertEqual(test_timeout_ping.execution_time, 1.44)
self.assertEqual(test_timeout_ping.round_trip_min, 0.040)
self.assertEqual(test_timeout_ping.round_trip_avg, 0.120)
self.assertEqual(test_timeout_ping.round_trip_max, 0.162)
self.assertEqual(test_timeout_ping.round_trip_stddev, 0.057)
class SNMPPluginTestFramework(object):
plugin_class = None
path = None
snmp_host = '127.0.0.1'
snmp_port = 10161
snmp_timeout = 10
snmp_retries = 1
snmp_max_repetitions = 10
snmp_community = 'public'
snmp_failure_timeout = 1
resource_id = 'test_id'
resource_endpoint = '127.0.0.1'
resource_plugin = 'dummy'
resource_site = 'test_site'
resource_class = 'network'
resource_subclass = 'test_subclass'
resource_type = 'test_type'
resource_backplane = None
resource_model = 'model'
results_data_file = 'results.json'
enrichment_data_file = 'enrichment_data'
use_enrichment = True
plugin_conf = {
'Core': {
'name': 'Test Plugin',
'module': 'test_plugin'
},
'main': {
'execute_frequency': '60',
'enrichment_ttl': '300',
'resource_filter': 'resource_class = "network"'
},
'snmp': {
'timeout': 10,
'retries': 1,
'non_repeaters': 1,
'max_repetitions': 25,
}
}
def __init__(self, test_name):
super(SNMPPluginTestFramework, self).__init__(test_name)
self._path = self.path
self._plugin_conf = self.plugin_conf
self._snmp_host = self.snmp_host
self._snmp_port = self.snmp_port
self._snmp_timeout = self.snmp_timeout
self._snmp_retries = self.snmp_retries
self._snmp_max_repetitions = self.snmp_max_repetitions
self._snmp_community = self.snmp_community
self._snmp_failure_timeout = self.snmp_failure_timeout
self._resource_id = self.resource_id
self._resource_endpoint = self.resource_endpoint
self._resource_plugin = self.resource_plugin
self._resource_site = self.resource_site
self._resource_class = self.resource_class
self._resource_subclass = self.resource_subclass
self._resource_type = self.resource_type
self._resource_backplane = self.resource_backplane
self._resource_model = self.resource_model
self._panoptes_context = None
self._panoptes_resource = None
self._snmp_conf = None
self._expected_results = None
self._results_data_file = None
self._enrichment_data_file = None
self._secret_store = None
self._plugin_context = None
self._plugin_context_bad = None
self._snmp_conf_bad = None
self._enrichment_kv = None
self._enrichment_cache = None
def set_panoptes_context(self):
panoptes_test_conf_file = os.path.join(
os.path.dirname(os.path.dirname(__file__)),
'config_files/test_panoptes_config.ini')
self._panoptes_context = PanoptesContext(
panoptes_test_conf_file,
key_value_store_class_list=[PanoptesEnrichmentCacheKeyValueStore])
def set_panoptes_resource(self):
self._panoptes_resource = PanoptesResource(
resource_site=self._resource_site,
resource_class=self._resource_class,
resource_subclass=self._resource_subclass,
resource_type=self._resource_type,
resource_id=self._resource_id,
resource_endpoint=self._resource_endpoint,
resource_plugin=self._resource_plugin)
self._panoptes_resource.resource_metadata['model'] = 'model'
if self._resource_backplane:
self._panoptes_resource.add_metadata('backplane',
self._resource_backplane)
def set_enrichment_cache(self):
if self.use_enrichment:
self._enrichment_data_file = 'data/' + self.enrichment_data_file
self._enrichment_kv = self._panoptes_context.get_kv_store(
PanoptesEnrichmentCacheKeyValueStore)
enrichment_data_file = os.path.join(os.path.abspath(self.path),
self._enrichment_data_file)
if self._plugin_conf.get('enrichment'):
if self._plugin_conf['enrichment'].get('preload'):
self._enrichment_cache = PanoptesEnrichmentCache(
self._panoptes_context, self._plugin_conf,
self._panoptes_resource)
try:
with open(enrichment_data_file) as enrichment_data:
for line in enrichment_data:
key, value = line.strip().split('=>')
self._enrichment_kv.set(key, value)
except Exception as e:
raise IOError(
'Failed to load enrichment data file {}: {}'.format(
enrichment_data_file, repr(e)))
def set_snmp_conf(self):
self._snmp_conf = self._panoptes_context.config_object.snmp_defaults
def set_secret_store(self):
self._secret_store = create_autospec(PanoptesSecretsStore,
instance=True,
spec_set=True)
self._secret_store.get_by_site.return_value = self._snmp_community
def set_plugin_context(self):
self._plugin_context = create_autospec(
PanoptesPluginWithEnrichmentContext,
instance=True,
spec_set=True,
data=self._panoptes_resource,
enrichment=self._enrichment_cache,
config=self._plugin_conf,
snmp=self._snmp_conf,
secrets=self._secret_store,
logger=logging.getLogger(__name__))
def set_snmp_conf_bad(self):
self._snmp_conf_bad = copy.copy(self._snmp_conf)
self._snmp_conf_bad['port'] += 1
self._snmp_conf_bad['timeout'] = self._snmp_failure_timeout
def set_plugin_context_bad(self):
self._plugin_context_bad = create_autospec(
PanoptesPluginWithEnrichmentContext,
instance=True,
spec_set=True,
data=self._panoptes_resource,
enrichment=self._enrichment_cache,
config=self._plugin_conf,
snmp=self._snmp_conf_bad,
secrets=self._secret_store,
logger=logging.getLogger(__name__))
def set_expected_results(self):
self._results_data_file = 'data/' + self.results_data_file
expected_result_file = os.path.join(os.path.abspath(self.path),
self._results_data_file)
self._expected_results = json.load(open(expected_result_file))
@patch('time.time', mock_time)
@patch('yahoo_panoptes.framework.resources.time', mock_time)
@patch('redis.StrictRedis', PanoptesMockRedis)
def setUp(self):
self.set_panoptes_context()
self.set_panoptes_resource()
self.set_enrichment_cache()
self.set_snmp_conf()
self.set_secret_store()
self.set_plugin_context()
self.set_snmp_conf_bad()
self.set_plugin_context_bad()
self.set_expected_results()
def testMetricsGroup(self):
now = round(time.time(), METRICS_TIMESTAMP_PRECISION)
metrics_group = PanoptesMetricsGroup(self.__panoptes_resource, 'test',
120)
self.assertEqual(metrics_group.group_type, 'test')
self.assertEqual(metrics_group.interval, 120)
self.assertEqual(metrics_group.schema_version, '0.2')
self.assertGreaterEqual(metrics_group.creation_timestamp, now)
with patch('yahoo_panoptes.framework.metrics.time', mock_time):
metrics_group = PanoptesMetricsGroup(self.__panoptes_resource,
'test', 120)
dimension_one = PanoptesMetricDimension('if_alias', 'bar')
dimension_two = PanoptesMetricDimension('if_alias', 'foo')
metrics_group.add_dimension(dimension_one)
with self.assertRaises(KeyError):
metrics_group.add_dimension(dimension_two)
# Test basic dimension operations
self.assertEqual(len(metrics_group.dimensions), 1)
self.assertTrue(
metrics_group.contains_dimension_by_name('if_alias'))
self.assertFalse(metrics_group.contains_dimension_by_name('baz'))
self.assertEqual(
metrics_group.get_dimension_by_name('if_alias').value, 'bar')
metrics_group.delete_dimension_by_name('if_alias')
self.assertFalse(
metrics_group.contains_dimension_by_name('if_alias'))
self.assertEqual(len(metrics_group.dimensions), 0)
self.assertEqual(metrics_group.get_dimension_by_name('foo'), None)
metrics_group.add_dimension(dimension_two)
dimension_three = PanoptesMetricDimension('if_alias', 'test')
metrics_group.upsert_dimension(dimension_three)
self.assertEqual(len(metrics_group.dimensions), 1)
self.assertEqual(
metrics_group.get_dimension_by_name('if_alias').value, 'test')
dimension_four = PanoptesMetricDimension('if_name', 'eth0')
metrics_group.upsert_dimension(dimension_four)
self.assertEqual(len(metrics_group.dimensions), 2)
# Test basic metric operations
with self.assertRaises(AssertionError):
metrics_group.add_metric(None)
metric = PanoptesMetric('test_metric', 0, PanoptesMetricType.GAUGE)
metrics_group.add_metric(metric)
with self.assertRaises(KeyError):
metrics_group.add_metric(metric)
to_json = metrics_group.json
metrics = PanoptesMetricsGroup.flatten_metrics(
json.loads(to_json)['metrics'])
self.assertEquals(metrics['gauge']['test_metric']['value'], 0)
metrics_group_two = PanoptesMetricsGroup(self.__panoptes_resource,
'test', 120)
metrics_group_two.add_dimension(dimension_two)
metrics_group_two.upsert_dimension(dimension_three)
metrics_group_two.upsert_dimension(dimension_four)
metrics_group_two.add_metric(metric)
self.assertEqual(metrics_group, metrics_group_two)
# Check PanoptesMetricsGroup.__eq__
panoptes_resource_two = PanoptesResource(resource_site='test2',
resource_class='test2',
resource_subclass='test2',
resource_type='test2',
resource_id='test2',
resource_endpoint='test2',
resource_plugin='test2')
metrics_group_two = PanoptesMetricsGroup(panoptes_resource_two,
'test', 120)
metrics_group_three = PanoptesMetricsGroup(
self.__panoptes_resource, 'test', 120)
with self.assertRaises(AssertionError):
assert metrics_group_two == metrics_group_three
metrics_group_three = metrics_group.copy()
with self.assertRaises(AssertionError):
assert metrics_group == dimension_one
assert metrics_group == metrics_group_three
metrics_group_three.delete_dimension_by_name("if_name")
with self.assertRaises(AssertionError):
assert metrics_group == metrics_group_three
metrics_group_three.upsert_dimension(dimension_four)
assert metrics_group == metrics_group_three
metric_two = PanoptesMetric('test_metric_2', 1,
PanoptesMetricType.GAUGE)
metrics_group_three.add_metric(metric_two)
with self.assertRaises(AssertionError):
assert metrics_group == metrics_group_three
# Test PanoptesMetricsGroup.__repr__
_METRICS_GROUP_REPR = "{{'metrics_group_interval': 120, " \
"'resource': plugin|test|site|test|class|test|subclass|test|type|test|id|test|" \
"endpoint|test, 'dimensions': set([{{'dimension_name': 'if_alias', " \
"'dimension_value': 'test'}}, " \
"{{'dimension_name': 'if_name', 'dimension_value': 'eth0'}}]), " \
"'metrics_group_type': 'test', " \
"'metrics': set([{{'metric_creation_timestamp': {}, " \
"'metric_type': 'gauge', 'metric_name': 'test_metric', 'metric_value': 0}}]), " \
"'metrics_group_creation_timestamp': {}, " \
"'metrics_group_schema_version': '0.2'}}".format(mock_time.return_value,
mock_time.return_value)
self.assertEqual(repr(metrics_group), _METRICS_GROUP_REPR)
dimensions_as_dicts = [{
'dimension_name': dimension.name,
'dimension_value': dimension.value
} for dimension in metrics_group.dimensions]
self.assertEqual(
PanoptesMetricsGroup.flatten_dimensions(dimensions_as_dicts), {
'if_alias': 'test',
'if_name': 'eth0'
})
import logging
import os
from unittest import TestCase
from mock import create_autospec
from yahoo_panoptes.framework.context import PanoptesContext
from yahoo_panoptes.framework.plugins.context import PanoptesPluginWithEnrichmentContext
from yahoo_panoptes.framework.resources import PanoptesResource
from yahoo_panoptes.framework.utilities.secrets import PanoptesSecretsStore
from yahoo_panoptes.framework.utilities.snmp.connection import PanoptesSNMPPluginConfiguration
panoptes_resource = PanoptesResource(resource_site='test_site',
resource_class='test_class',
resource_subclass='test_subclass',
resource_type='test_type',
resource_id='test_id',
resource_endpoint='test_endpoint',
resource_plugin='test_plugin')
plugin_conf = {
'Core': {
'name': 'Test Plugin',
'module': 'test_plugin'
},
'main': {
'execute_frequency': '60',
'resource_filter': 'resource_class = "network"'
}
}
class TestResources(unittest.TestCase):
def setUp(self):
self.__panoptes_resource_metadata = {'test': 'test', '_resource_ttl': '604800'}
self.__panoptes_resource = PanoptesResource(resource_site='test', resource_class='test',
resource_subclass='test',
resource_type='test', resource_id='test', resource_endpoint='test',
resource_plugin='test',
resource_creation_timestamp=_TIMESTAMP,
resource_ttl=RESOURCE_MANAGER_RESOURCE_EXPIRE)
self.__panoptes_resource.add_metadata('test', 'test')
self.__panoptes_resource_set = PanoptesResourceSet()
mock_valid_timestamp = Mock(return_value=True)
with patch('yahoo_panoptes.framework.resources.PanoptesValidators.valid_timestamp',
mock_valid_timestamp):
self.__panoptes_resource_set.resource_set_creation_timestamp = _TIMESTAMP
self.my_dir, self.panoptes_test_conf_file = _get_test_conf_file()
def test_panoptes_resource(self):
panoptes_resource_metadata = self.__panoptes_resource_metadata
panoptes_resource = self.__panoptes_resource
self.assertIsInstance(panoptes_resource, PanoptesResource)
self.assertEqual(panoptes_resource.resource_site, 'test')
self.assertEqual(panoptes_resource.resource_class, 'test')
self.assertEqual(panoptes_resource.resource_subclass, 'test')
self.assertEqual(panoptes_resource.resource_type, 'test')
self.assertEqual(panoptes_resource.resource_id, 'test')
self.assertEqual(panoptes_resource.resource_endpoint, 'test')
self.assertEqual(panoptes_resource.resource_metadata, panoptes_resource_metadata)
self.assertEqual(panoptes_resource.resource_ttl, str(RESOURCE_MANAGER_RESOURCE_EXPIRE))
self.assertEqual(panoptes_resource, panoptes_resource)
self.assertFalse(panoptes_resource == '1')
self.assertIsInstance(str(panoptes_resource), str)
with self.assertRaises(AssertionError):
panoptes_resource.add_metadata(None, 'test')
with self.assertRaises(AssertionError):
panoptes_resource.add_metadata('', 'test')
with self.assertRaises(ValueError):
panoptes_resource.add_metadata('1', 'test')
with self.assertRaises(ValueError):
panoptes_resource.add_metadata('key', 'test|')
with self.assertRaises(AssertionError):
PanoptesResource(resource_site='', resource_class='test', resource_subclass='test',
resource_type='test', resource_id='test', resource_endpoint='test')
with self.assertRaises(AssertionError):
PanoptesResource(resource_site='test', resource_class='', resource_subclass='test',
resource_type='test', resource_id='test', resource_endpoint='test')
with self.assertRaises(AssertionError):
PanoptesResource(resource_site='test', resource_class='test', resource_subclass='',
resource_type='test', resource_id='test', resource_endpoint='test')
with self.assertRaises(AssertionError):
PanoptesResource(resource_site='test', resource_class='test', resource_subclass='test',
resource_type='', resource_id='test', resource_endpoint='test')
with self.assertRaises(AssertionError):
PanoptesResource(resource_site='test', resource_class='test', resource_subclass='test',
resource_type='test', resource_id='', resource_endpoint='test')
with self.assertRaises(AssertionError):
PanoptesResource(resource_site=None, resource_class='test', resource_subclass='test',
resource_type='test', resource_id='test', resource_endpoint='test')
with self.assertRaises(AssertionError):
PanoptesResource(resource_site='test', resource_class=None, resource_subclass='test',
resource_type='test', resource_id='test', resource_endpoint='test')
with self.assertRaises(AssertionError):
PanoptesResource(resource_site='test', resource_class='test', resource_subclass=None,
resource_type='test', resource_id='test', resource_endpoint='test')
with self.assertRaises(AssertionError):
PanoptesResource(resource_site='test', resource_class='test', resource_subclass='test',
resource_type=None, resource_id='test', resource_endpoint='test')
with self.assertRaises(AssertionError):
PanoptesResource(resource_site='test', resource_class='test', resource_subclass='test',
resource_type='test', resource_id=None, resource_endpoint='test')
with self.assertRaises(AssertionError):
PanoptesResource(resource_site='test', resource_class='test', resource_subclass='test',
resource_type='test', resource_id='test', resource_endpoint=None)
# Test json and raw representations of PanoptesResource
panoptes_resource_2 = PanoptesResource(resource_site='test', resource_class='test',
resource_subclass='test',
resource_type='test', resource_id='test', resource_endpoint='test',
resource_creation_timestamp=_TIMESTAMP,
resource_plugin='test')
self.assertEqual(panoptes_resource_2.resource_creation_timestamp, _TIMESTAMP)
panoptes_resource_2_json = {
u'resource_site': u'test',
u'resource_id': u'test',
u'resource_class': u'test',
u'resource_plugin': u'test',
u'resource_creation_timestamp': _TIMESTAMP,
u'resource_subclass': u'test',
u'resource_endpoint': u'test',
u'resource_metadata': {
u'_resource_ttl': u'604800'
},
u'resource_type': u'test'
}
self.assertEqual(ordered(json.loads(panoptes_resource_2.json)), ordered(panoptes_resource_2_json))
panoptes_resource_2_raw = collections.OrderedDict(
[('resource_site', 'test'),
('resource_class', 'test'),
('resource_subclass', 'test'),
('resource_type', 'test'),
('resource_id', 'test'),
('resource_endpoint', 'test'),
('resource_metadata', collections.OrderedDict(
[('_resource_ttl', '604800')])
),
('resource_creation_timestamp', _TIMESTAMP),
('resource_plugin', 'test')])
self.assertEqual(panoptes_resource_2.raw, panoptes_resource_2_raw)
# Test resource creation from dict
with open('tests/test_resources/input/resource_one.json') as f:
resource_specs = json.load(f)
resource_from_json = PanoptesResource.resource_from_dict(resource_specs['resources'][0])
panoptes_resource_3 = PanoptesResource(resource_site="test_site", resource_class="network",
resource_subclass="test_subclass", resource_type="test_type",
resource_id="test_id_1", resource_endpoint="test_endpoint_1",
resource_plugin="key")
self.assertEqual(resource_from_json, panoptes_resource_3)
def test_panoptes_resource_set(self):
panoptes_resource = self.__panoptes_resource
panoptes_resource_set = self.__panoptes_resource_set
self.assertEqual(panoptes_resource_set.add(panoptes_resource), None)
self.assertEqual(len(panoptes_resource_set), 1)
self.assertEqual(type(panoptes_resource_set.resources), set)
self.assertIsInstance(str(panoptes_resource_set), str)
self.assertIsInstance(iter(panoptes_resource_set), collections.Iterable)
self.assertEqual(panoptes_resource_set.next(), panoptes_resource)
self.assertEqual(panoptes_resource_set.remove(panoptes_resource), None)
self.assertEqual(len(panoptes_resource_set), 0)
self.assertEqual(panoptes_resource_set.resource_set_creation_timestamp, _TIMESTAMP)
with self.assertRaises(AssertionError):
panoptes_resource_set.resource_set_creation_timestamp = 0
panoptes_resource_2 = PanoptesResource(resource_site='test', resource_class='test',
resource_subclass='test',
resource_type='test', resource_id='test2', resource_endpoint='test',
resource_plugin='test',
resource_ttl=RESOURCE_MANAGER_RESOURCE_EXPIRE,
resource_creation_timestamp=_TIMESTAMP)
panoptes_resource_set.add(panoptes_resource)
panoptes_resource_set.add(panoptes_resource_2)
self.assertEqual(len(panoptes_resource_set.get_resources_by_site()['test']), 2)
self.assertEqual(panoptes_resource_set.resource_set_schema_version, "0.1")
panoptes_resource_set_json = {
u'resource_set_creation_timestamp': _TIMESTAMP,
u'resource_set_schema_version': u'0.1',
u'resources': [
{
u'resource_site': u'test',
u'resource_class': u'test',
u'resource_subclass': u'test',
u'resource_type': u'test',
u'resource_id': u'test2',
u'resource_endpoint': u'test',
u'resource_metadata': {
u'_resource_ttl': u'604800'
},
u'resource_creation_timestamp': _TIMESTAMP,
u'resource_plugin': u'test'
},
{
u'resource_site': u'test',
u'resource_class': u'test',
u'resource_subclass': u'test',
u'resource_type': u'test',
u'resource_id': u'test',
u'resource_endpoint': u'test',
u'resource_metadata': {
u'_resource_ttl': u'604800',
u'test': u'test'
},
u'resource_creation_timestamp': _TIMESTAMP,
u'resource_plugin': u'test'
}
]
}
self.assertEqual(ordered(panoptes_resource_set_json), ordered(json.loads(panoptes_resource_set.json)))
def test_panoptes_resources_key_value_store(self):
panoptes_context = PanoptesContext(self.panoptes_test_conf_file)
panoptes_resources_kv_store = PanoptesResourcesKeyValueStore(panoptes_context)
self.assertEqual(panoptes_resources_kv_store.redis_group, const.RESOURCE_MANAGER_REDIS_GROUP)
self.assertEqual(panoptes_resources_kv_store.namespace, const.RESOURCE_MANAGER_KEY_VALUE_NAMESPACE)
@patch('redis.StrictRedis', panoptes_mock_redis_strict_client)
def test_panoptes_resource_store(self):
panoptes_context = PanoptesContext(self.panoptes_test_conf_file,
key_value_store_class_list=[PanoptesTestKeyValueStore])
with self.assertRaises(Exception):
PanoptesResourceStore(panoptes_context)
mock_kv_store = Mock(return_value=PanoptesTestKeyValueStore(panoptes_context))
with patch('yahoo_panoptes.framework.resources.PanoptesContext.get_kv_store', mock_kv_store):
panoptes_resource_store = PanoptesResourceStore(panoptes_context)
panoptes_resource_store.add_resource("test_plugin_signature", self.__panoptes_resource)
resource_key = "plugin|test|site|test|class|test|subclass|test|type|test|id|test|endpoint|test"
resource_value = panoptes_resource_store.get_resource(resource_key)
self.assertEqual(self.__panoptes_resource, resource_value)
panoptes_resource_2 = PanoptesResource(resource_site='test', resource_class='test',
resource_subclass='test',
resource_type='test', resource_id='test2', resource_endpoint='test',
resource_plugin='test',
resource_ttl=RESOURCE_MANAGER_RESOURCE_EXPIRE,
resource_creation_timestamp=_TIMESTAMP)
panoptes_resource_store.add_resource("test_plugin_signature", panoptes_resource_2)
self.assertIn(self.__panoptes_resource, panoptes_resource_store.get_resources())
self.assertIn(panoptes_resource_2, panoptes_resource_store.get_resources())
panoptes_resource_store.delete_resource("test_plugin_signature", panoptes_resource_2)
self.assertNotIn(panoptes_resource_2, panoptes_resource_store.get_resources())
panoptes_resource_3 = PanoptesResource(resource_site='test', resource_class='test',
resource_subclass='test',
resource_type='test', resource_id='test3', resource_endpoint='test',
resource_plugin='test3',
resource_ttl=RESOURCE_MANAGER_RESOURCE_EXPIRE,
resource_creation_timestamp=_TIMESTAMP)
panoptes_resource_store.add_resource("test_plugin_signature", panoptes_resource_3)
self.assertIn(panoptes_resource_3, panoptes_resource_store.get_resources(site='test', plugin_name='test3'))
self.assertNotIn(self.__panoptes_resource,
panoptes_resource_store.get_resources(site='test', plugin_name='test3'))
# Test key not found
mock_find_keys = Mock(
return_value=['dummy',
'plugin|test|site|test|class|test|subclass|test|type|test|id|test|endpoint|test'])
with patch('yahoo_panoptes.framework.resources.PanoptesKeyValueStore.find_keys',
mock_find_keys):
self.assertEqual(1, len(panoptes_resource_store.get_resources()))
# Test resource store methods raise correct errors
mock_get = Mock(side_effect=Exception)
with patch('yahoo_panoptes.framework.resources.PanoptesKeyValueStore.get', mock_get):
with self.assertRaises(PanoptesResourceError):
panoptes_resource_store.get_resource('test3')
# Test bad input
with self.assertRaises(AssertionError):
panoptes_resource_store.get_resource("")
with self.assertRaises(AssertionError):
panoptes_resource_store.get_resource(1)
with self.assertRaises(AssertionError):
panoptes_resource_store.add_resource("", panoptes_resource_2)
with self.assertRaises(AssertionError):
panoptes_resource_store.add_resource("test_plugin_signature", None)
with self.assertRaises(AssertionError):
panoptes_resource_store.add_resource("test_plugin_signature", PanoptesResourceStore(panoptes_context))
with self.assertRaises(AssertionError):
panoptes_resource_store.delete_resource("", panoptes_resource_2)
with self.assertRaises(AssertionError):
panoptes_resource_store.delete_resource("test_plugin_signature", None)
with self.assertRaises(AssertionError):
panoptes_resource_store.delete_resource("test_plugin_signature",
PanoptesResourceStore(panoptes_context))
with self.assertRaises(AssertionError):
panoptes_resource_store.get_resources("", "test_plugin_name")
with self.assertRaises(AssertionError):
panoptes_resource_store.get_resources("test_site", "")
with self.assertRaises(AssertionError):
panoptes_resource_store.get_resources(1, "test_plugin_name")
with self.assertRaises(AssertionError):
panoptes_resource_store.get_resources("test_site", 1)
# Test non-existent key
with self.assertRaises(PanoptesResourceError):
panoptes_resource_store.get_resource('tes')
mock_set = Mock(side_effect=Exception)
with patch('yahoo_panoptes.framework.resources.PanoptesKeyValueStore.set', mock_set):
with self.assertRaises(PanoptesResourceError):
panoptes_resource_store.add_resource("test_plugin_signature", panoptes_resource_2)
mock_delete = Mock(side_effect=Exception)
with patch('yahoo_panoptes.framework.resources.PanoptesKeyValueStore.delete', mock_delete):
with self.assertRaises(PanoptesResourceError):
panoptes_resource_store.delete_resource("test_plugin_signature", panoptes_resource_2)
with self.assertRaises(PanoptesResourceError):
panoptes_resource_store._deserialize_resource("tes", "null")
with self.assertRaises(PanoptesResourceError):
panoptes_resource_store._deserialize_resource(resource_key, "null")
@patch('redis.StrictRedis', panoptes_mock_redis_strict_client)
def test_resource_dsl_parsing(self):
panoptes_context = PanoptesContext(self.panoptes_test_conf_file)
test_query = 'resource_class = "network" AND resource_subclass = "load-balancer"'
test_result = (
'SELECT resources.*, group_concat(key,"|"), group_concat(value,"|") ' +
'FROM resources ' +
'LEFT JOIN resource_metadata ON resources.id = resource_metadata.id ' +
'WHERE (resources.resource_class = "network" ' +
'AND resources.resource_subclass = "load-balancer") ' +
'GROUP BY resource_metadata.id ' +
'ORDER BY resource_metadata.id'
)
panoptes_resource_dsl = PanoptesResourceDSL(test_query, panoptes_context)
self.assertEqual(panoptes_resource_dsl.sql, test_result)
# This very long query tests all code paths with the DSL parser:
test_query = 'resource_class = "network" AND resource_subclass = "load-balancer" OR \
resource_metadata.os_version LIKE "4%" AND resource_site NOT IN ("test_site") \
AND resource_endpoint IN ("test1","test2") AND resource_type != "a10" OR ' \
'resource_metadata.make NOT LIKE \
"A10%" AND resource_metadata.model NOT IN ("test1", "test2")'
test_result = (
'SELECT resources.*,group_concat(key,"|"),group_concat(value,"|") FROM (SELECT resource_metadata.id ' +
'FROM resources,resource_metadata WHERE (resources.resource_class = "network" ' +
'AND resources.resource_subclass = "load-balancer" ' +
'AND resources.resource_site NOT IN ("test_site") ' +
'AND resources.resource_endpoint IN ("test1","test2") ' +
'AND resources.resource_type != "a10" ' +
'AND ((resource_metadata.key = "os_version" ' +
'AND resource_metadata.value LIKE "4%")) ' +
'AND resource_metadata.id = resources.id) ' +
'UNION SELECT resource_metadata.id ' +
'FROM resources,resource_metadata WHERE (resource_metadata.key = "make" ' +
'AND resource_metadata.value NOT LIKE "A10%") ' +
'AND resource_metadata.id = resources.id ' +
'INTERSECT SELECT resource_metadata.id ' +
'FROM resources,resource_metadata WHERE (resource_metadata.key = "model" ' +
'AND resource_metadata.value NOT IN ("test1","test2")) ' +
'AND resource_metadata.id = resources.id ' +
'GROUP BY resource_metadata.id ' +
'ORDER BY resource_metadata.id) AS filtered_resources, ' +
'resources, resource_metadata WHERE resources.id = filtered_resources.id ' +
'AND resource_metadata.id = filtered_resources.id GROUP BY resource_metadata.id')
panoptes_resource_dsl = PanoptesResourceDSL(test_query, panoptes_context)
self.assertEqual(panoptes_resource_dsl.sql, test_result)
panoptes_resource_dsl = PanoptesResourceDSL('resource_site = "local"', panoptes_context)
self.assertIsInstance(panoptes_resource_dsl.tokens, ParseResults)
with self.assertRaises(AssertionError):
PanoptesResourceDSL(None, panoptes_context)
with self.assertRaises(AssertionError):
PanoptesResourceDSL('', None)
with self.assertRaises(AssertionError):
PanoptesResourceDSL('', panoptes_context)
with self.assertRaises(ParseException):
PanoptesResourceDSL('resources_site = local', panoptes_context)
def test_multi_enrichment_group_set(self):
interface_validation_object = PanoptesEnrichmentInterfaceSchemaValidator()
neighbor_validation_object = PanoptesEnrichmentNeighborSchemaValidator()
panoptes_resource = self.__panoptes_resource
multi_enrichment_results_data = \
{
"group_sets": [
{
"enrichment": [
{
"data": [
{
"host_name": {
"mac": "aa:bb:cc:dd:ee:ff",
"property": "Test Property",
"vlan_id": 501
}
},
{
"host_name01": {
"mac": "aa:bb:cc:dd:ee:ff",
"property": "Test Property",
"vlan_id": 502
}
}
],
"metadata": {
"_enrichment_group_creation_timestamp": mock_time.return_value,
"_enrichment_ttl": 600,
"_execute_frequency": 120
},
"namespace": "neighbor"
},
{
"data": [
{
"int_001": {
"index": 1,
"speed": 1000,
"status": "up"
}
}
],
"metadata": {
"_enrichment_group_creation_timestamp": mock_time.return_value,
"_enrichment_ttl": 300,
"_execute_frequency": 60
},
"namespace": "interface"
}
],
"enrichment_group_set_creation_timestamp": mock_time.return_value,
"resource": {
"resource_class": "test_class",
"resource_creation_timestamp": mock_time.return_value,
"resource_endpoint": "test_endpoint01",
"resource_id": "test_resource_id01",
"resource_metadata": {
"_resource_ttl": "604800"
},
"resource_plugin": "test_plugin",
"resource_site": "test_site",
"resource_subclass": "test_subclass",
"resource_type": "test_type"
}
},
{
"enrichment": [
{
"data": [
{
"int_001": {
"index": 1,
"speed": 1000,
"status": "up"
}
}
],
"metadata": {
"_enrichment_group_creation_timestamp": mock_time.return_value,
"_enrichment_ttl": 300,
"_execute_frequency": 60
},
"namespace": "interface"
},
{
"data": [
{
"host_name": {
"mac": "aa:bb:cc:dd:ee:ff",
"property": "Test Property",
"vlan_id": 501
}
}
],
"metadata": {
"_enrichment_group_creation_timestamp": mock_time.return_value,
"_enrichment_ttl": 600,
"_execute_frequency": 120
},
"namespace": "neighbor"
}
],
"enrichment_group_set_creation_timestamp": mock_time.return_value,
"resource": {
"resource_class": "test",
"resource_creation_timestamp": mock_time.return_value,
"resource_endpoint": "test",
"resource_id": "test",
"resource_metadata": {
"_resource_ttl": "604800",
"test": "test"
},
"resource_plugin": "test",
"resource_site": "test",
"resource_subclass": "test",
"resource_type": "test"
}
}
]
}
enrichment_set1 = PanoptesEnrichmentSet(u'int_001')
enrichment_set1.add(u'speed', 1000)
enrichment_set1.add(u'index', 0o01)
enrichment_set1.add(u'status', u'up')
enrichment_group1 = PanoptesEnrichmentGroup(u'interface', interface_validation_object, 300, 60)
enrichment_group1.add_enrichment_set(enrichment_set1)
enrichment_set3 = PanoptesEnrichmentSet(u'host_name')
enrichment_set3.add(u'vlan_id', 501)
enrichment_set3.add(u'property', u'Test Property')
enrichment_set3.add(u'mac', u'aa:bb:cc:dd:ee:ff')
enrichment_group2 = PanoptesEnrichmentGroup(u'neighbor', neighbor_validation_object, 600, 120)
enrichment_group2.add_enrichment_set(enrichment_set3)
enrichment_group_set1 = PanoptesEnrichmentGroupSet(panoptes_resource)
enrichment_group_set1.add_enrichment_group(enrichment_group1)
enrichment_group_set1.add_enrichment_group(enrichment_group2)
panoptes_resource01 = PanoptesResource(resource_site=u'test_site',
resource_class=u'test_class',
resource_subclass=u'test_subclass',
resource_type=u'test_type',
resource_id=u'test_resource_id01',
resource_endpoint=u'test_endpoint01',
resource_plugin=u'test_plugin')
panoptes_resource02 = PanoptesResource(resource_site=u'test_site',
resource_class=u'test_class',
resource_subclass=u'test_subclass',
resource_type=u'test_type',
resource_id=u'test_resource_id02',
resource_endpoint=u'test_endpoint02',
resource_plugin=u'test_plugin')
enrichment_set4 = PanoptesEnrichmentSet(u'host_name01')
enrichment_set4.add(u'vlan_id', 502)
enrichment_set4.add(u'property', u'Test Property')
enrichment_set4.add(u'mac', u'aa:bb:cc:dd:ee:ff')
enrichment_group3 = PanoptesEnrichmentGroup(u'neighbor', neighbor_validation_object, 600, 120)
enrichment_group3.add_enrichment_set(enrichment_set3)
enrichment_group3.add_enrichment_set(enrichment_set4)
enrichment_group_set2 = PanoptesEnrichmentGroupSet(panoptes_resource01)
enrichment_group_set2.add_enrichment_group(enrichment_group1)
enrichment_group_set2.add_enrichment_group(enrichment_group3)
multi_enrichment_group_set = PanoptesEnrichmentMultiGroupSet()
multi_enrichment_group_set.add_enrichment_group_set(enrichment_group_set1)
multi_enrichment_group_set.add_enrichment_group_set(enrichment_group_set2)
multi_enrichment_group_set_repr = u"PanoptesEnrichmentMultiGroupSet[PanoptesEnrichmentGroupSet[resource:" \
u"plugin|test|site|test|class|test|subclass|test|type|test|id|test|endpoint" \
u"|test,enrichment_group_set_creation_timestamp:{},PanoptesEnrichmentGroup" \
u"[namespace:interface,enrichment_ttl:300,execute_frequency:60," \
u"enrichment_group_creation_timestamp:{},PanoptesEnrichmentSet[" \
u"int_001[index:1,speed:1000,status:up]]],PanoptesEnrichmentGroup[" \
u"namespace:neighbor,enrichment_ttl:600,execute_frequency:120," \
u"enrichment_group_creation_timestamp:{},PanoptesEnrichmentSet" \
u"[host_name[mac:aa:bb:cc:dd:ee:ff,property:Test Property,vlan_id:501]" \
u"]]],PanoptesEnrichmentGroupSet[resource:plugin|test_plugin|site|" \
u"test_site|class|test_class|subclass|test_subclass|type|test_type|id" \
u"|test_resource_id01|endpoint|test_endpoint01," \
u"enrichment_group_set_creation_timestamp:{},PanoptesEnrichmentGroup" \
u"[namespace:interface,enrichment_ttl:300,execute_frequency:60," \
u"enrichment_group_creation_timestamp:{},PanoptesEnrichmentSet" \
u"[int_001[index:1,speed:1000,status:up]]],PanoptesEnrichmentGroup" \
u"[namespace:neighbor,enrichment_ttl:600,execute_frequency:120," \
u"enrichment_group_creation_timestamp:{},PanoptesEnrichmentSet" \
u"[host_name[mac:aa:bb:cc:dd:ee:ff,property:Test Property,vlan_id:501]]," \
u"PanoptesEnrichmentSet[host_name01[mac:aa:bb:cc:dd:ee:ff," \
u"property:Test Property,vlan_id:502]]]]]".format(mock_time.return_value,
mock_time.return_value,
mock_time.return_value,
mock_time.return_value,
mock_time.return_value,
mock_time.return_value)
self.assertEquals(repr(multi_enrichment_group_set), multi_enrichment_group_set_repr)
self.assertEquals(len(multi_enrichment_group_set.enrichment_group_sets), 2)
self.assertEquals(ordered(json.loads(multi_enrichment_group_set.json())),
ordered(multi_enrichment_results_data))
self.assertEquals(len(multi_enrichment_group_set), 2)
multi_enrichment_group_set.add_enrichment_group_set(enrichment_group_set2)
self.assertEquals(len(multi_enrichment_group_set), 2)
enrichment_group_set3 = PanoptesEnrichmentGroupSet(panoptes_resource02)
enrichment_group_set3.add_enrichment_group(enrichment_group1)
multi_enrichment_group_set.add_enrichment_group_set(enrichment_group_set3)
self.assertEquals(len(multi_enrichment_group_set), 3)
with self.assertRaises(AssertionError):
multi_enrichment_group_set.add_enrichment_group_set(u'non_enrichment_group')
enrichment_group_set3 = PanoptesEnrichmentGroupSet(panoptes_resource01)
with self.assertRaises(AssertionError):
multi_enrichment_group_set.add_enrichment_group_set(enrichment_group_set3)
class TestValidators(unittest.TestCase):
def setUp(self):
self._panoptes_metric = {
u'metrics_group_interval':
60,
u'resource': {
u'resource_site': u'test_site',
u'resource_id': u'test_id',
u'resource_class': u'network',
u'resource_plugin': u'test_plugin',
u'resource_creation_timestamp': 1567823517.46,
u'resource_subclass': u'test_subclass',
u'resource_endpoint': u'test_endpoint',
u'resource_metadata': {
u'test_metadata_key': u'test_metadata_value',
u'_resource_ttl': u"604800"
},
u'resource_type': u'test_type'
},
u'dimensions': [{
u'dimension_name': u'cpu_name',
u'dimension_value': u'test_cpu_name_value'
}, {
u'dimension_name': u'cpu_no',
u'dimension_value': u'test_cpu_no_value'
}, {
u'dimension_name': u'cpu_type',
u'dimension_value': u'test_cpu_type_value'
}],
u'metrics_group_type':
u'cpu',
u'metrics': [{
u'metric_creation_timestamp': 1567823946.72,
u'metric_type': u'gauge',
u'metric_name': u'cpu_utilization',
u'metric_value': 0
}],
u'metrics_group_creation_timestamp':
1567823946.72,
u'metrics_group_schema_version':
u'0.2'
}
self._panoptes_resource = PanoptesResource(resource_site=u'test',
resource_class=u'test',
resource_subclass=u'test',
resource_type=u'test',
resource_id=u'test',
resource_endpoint=u'test',
resource_plugin=u'test')
self._panoptes_resource.add_metadata(u'test', u'test')
self._panoptes_resource_set = PanoptesResourceSet()
self._panoptes_resource_set.add(self._panoptes_resource)
def test_default_record(self):
self.assertTrue(
PanoptesConsumerRecordValidator.validate(
PanoptesConsumerTypes.METRICS, self._panoptes_metric))
self.assertTrue(
PanoptesConsumerRecordValidator.validate(
PanoptesConsumerTypes.PROCESSED, self._panoptes_metric))
self.assertTrue(
PanoptesConsumerRecordValidator.validate(
PanoptesConsumerTypes.RESOURCES,
json.loads(self._panoptes_resource_set.json)))
self.assertFalse(PanoptesConsumerRecordValidator.validate(5, {}))
def test_resource_throws(self):
argument_overrides = [(u'resource_site', 1), (u'resource_id', None),
(u'resource_class', 1.5),
(u'resource_plugin', {}),
(u'resource_creation_timestamp', u'1567823517'),
(u'resource_subclass', []),
(u'resource_endpoint', 123456.789)]
for (override_key, override_value) in argument_overrides:
schema_arguments = self._panoptes_metric.copy()
schema_arguments[u'resource'][override_key] = override_value
self.assertEquals(
PanoptesConsumerRecordValidator.validate_metrics(
schema_arguments), False)
def handle_resources(plugin_signature, resources):
resource_site = str(resources[u'resources'][0][u'resource_site'])
plugin_name = str(plugin_signature.split(':')[0])
logger.info(u'For plugin "%s" and site "%s", going to get current set' %
(plugin_name, resource_site))
try:
current_resource_set = resource_store.get_resources(
site=resource_site, plugin_name=plugin_name)
except:
logger.error(u'Error trying to get current resources for plugin "%s"' %
plugin_signature)
return False
logger.info(u'For plugin "%s" and site "%s" current set has %d resources' %
(plugin_signature, resource_site, len(current_resource_set)))
new_resource_set = PanoptesResourceSet()
for resource in resources[u'resources']:
new_resource = PanoptesResource.resource_from_dict(resource)
new_resource_set.add(new_resource)
current_resource_dict = _resource_set_to_dictionary(current_resource_set)
new_resource_dict = _resource_set_to_dictionary(new_resource_set)
logger.info(u'For plugin "%s" and site "%s", new set has %d resources' %
(plugin_signature, resource_site, len(new_resource_set)))
resources_to_delete = current_resource_set.resources.difference(
new_resource_set)
logger.info(u'For plugin "%s" and site "%s", deleting %d resources' %
(plugin_signature, resource_site, len(resources_to_delete)))
for resource in resources_to_delete:
current_timestamp = current_resource_dict[
resource.serialization_key].resource_creation_timestamp
new_timestamp = resources[u'resource_set_creation_timestamp']
if current_timestamp > new_timestamp:
logger.info(
u'For plugin "%s" and site "%s", resource "%s" has timestamp (%0.2f UTC) greater than of '
u'new set (%0.2f UTC), skipping deletion' %
(plugin_signature, resource_site, str(resource),
current_timestamp, new_timestamp))
else:
logger.info(
u'For plugin "%s" and site "%s", going to delete resource "%s"'
% (plugin_signature, resource_site, resource))
try:
resource_store.delete_resource(plugin_signature, resource)
except:
logger.exception(u'Error trying to delete resource')
return False
resources_to_add = new_resource_set.resources.difference(
current_resource_set)
logger.info(u'For plugin "%s" and site "%s", adding %d resources' %
(plugin_signature, resource_site, len(resources_to_add)))
for resource in resources_to_add:
logger.debug(u'Going to add resource "%s"' % resource)
try:
resource_store.add_resource(plugin_signature, resource)
except:
logger.exception(u'Error trying to add resource')
return False
resources_to_update = current_resource_set.resources.intersection(
new_resource_set)
logger.info(u'For plugin "%s" and site "%s", updating %d resources' %
(plugin_signature, resource_site, len(resources_to_update)))
start_time = time.time()
resources_skipped = 0
resources_updated = 0
for resource in resources_to_update:
current_timestamp = current_resource_dict[
resource.serialization_key].resource_creation_timestamp
new_timestamp = new_resource_dict[
resource.serialization_key].resource_creation_timestamp
if current_timestamp > new_timestamp:
logger.info(
u'For plugin "%s" and site "%s", resource "%s" has timestamp (%0.2f UTC) greater than of '
u'new set (%0.2f UTC), skipping update' %
(plugin_signature, resource_site, str(resource),
current_timestamp, new_timestamp))
resources_skipped += 1
else:
try:
new_resource = new_resource_dict[resource.serialization_key]
logger.debug(u'Going to update resource "%s"' % new_resource)
resource_store.add_resource(plugin_signature, new_resource)
resources_updated += 1
except:
logger.exception(u'Error trying to add resource')
return False
end_time = time.time()
logger.info(
u'For plugin "%s" and site "%s", added/updated %d resources in %0.2f seconds, skipped %d resources'
% (plugin_signature, resource_site, resources_updated,
round(end_time - start_time, 2), resources_skipped))
return True
def testMetricsGroup(self):
now = round(time.time(), METRICS_TIMESTAMP_PRECISION)
metrics_group = PanoptesMetricsGroup(self.__panoptes_resource, u'test',
120)
self.assertEqual(metrics_group.group_type, u'test')
self.assertEqual(metrics_group.interval, 120)
self.assertEqual(metrics_group.schema_version, u'0.2')
self.assertGreaterEqual(metrics_group.creation_timestamp, now)
with patch(u'yahoo_panoptes.framework.metrics.time', mock_time):
metrics_group = PanoptesMetricsGroup(self.__panoptes_resource,
u'test', 120)
dimension_one = PanoptesMetricDimension(u'if_alias', u'bar')
dimension_two = PanoptesMetricDimension(u'if_alias', u'foo')
metrics_group.add_dimension(dimension_one)
with self.assertRaises(KeyError):
metrics_group.add_dimension(dimension_two)
# Test basic dimension operations
self.assertEqual(len(metrics_group.dimensions), 1)
self.assertTrue(
metrics_group.contains_dimension_by_name(u'if_alias'))
self.assertFalse(metrics_group.contains_dimension_by_name(u'baz'))
self.assertEqual(
metrics_group.get_dimension_by_name(u'if_alias').value, u'bar')
metrics_group.delete_dimension_by_name(u'if_alias')
self.assertFalse(
metrics_group.contains_dimension_by_name(u'if_alias'))
self.assertEqual(len(metrics_group.dimensions), 0)
self.assertEqual(metrics_group.get_dimension_by_name(u'foo'), None)
metrics_group.add_dimension(dimension_two)
dimension_three = PanoptesMetricDimension(u'if_alias', u'test')
metrics_group.upsert_dimension(dimension_three)
self.assertEqual(len(metrics_group.dimensions), 1)
self.assertEqual(
metrics_group.get_dimension_by_name(u'if_alias').value,
u'test')
dimension_four = PanoptesMetricDimension(u'if_name', u'eth0')
metrics_group.upsert_dimension(dimension_four)
self.assertEqual(len(metrics_group.dimensions), 2)
# Test basic metric operations
with self.assertRaises(AssertionError):
metrics_group.add_metric(None)
metric = PanoptesMetric(u'test_metric', 0,
PanoptesMetricType.GAUGE)
metrics_group.add_metric(metric)
with self.assertRaises(KeyError):
metrics_group.add_metric(metric)
to_json = metrics_group.json
metrics = PanoptesMetricsGroup.flatten_metrics(
json.loads(to_json)[u'metrics'])
self.assertEquals(metrics[u'gauge'][u'test_metric'][u'value'], 0)
metrics_group_two = PanoptesMetricsGroup(self.__panoptes_resource,
u'test', 120)
metrics_group_two.add_dimension(dimension_two)
metrics_group_two.upsert_dimension(dimension_three)
metrics_group_two.upsert_dimension(dimension_four)
metrics_group_two.add_metric(metric)
self.assertEqual(metrics_group, metrics_group_two)
# Check PanoptesMetricsGroup.__eq__
panoptes_resource_two = PanoptesResource(
resource_site=u'test2',
resource_class=u'test2',
resource_subclass=u'test2',
resource_type=u'test2',
resource_id=u'test2',
resource_endpoint=u'test2',
resource_plugin=u'test2')
metrics_group_two = PanoptesMetricsGroup(panoptes_resource_two,
u'test', 120)
metrics_group_three = PanoptesMetricsGroup(
self.__panoptes_resource, u'test', 120)
with self.assertRaises(AssertionError):
assert metrics_group_two == metrics_group_three
metrics_group_three = metrics_group.copy()
with self.assertRaises(AssertionError):
assert metrics_group == dimension_one
assert metrics_group == metrics_group_three
metrics_group_three.delete_dimension_by_name(u"if_name")
with self.assertRaises(AssertionError):
assert metrics_group == metrics_group_three
metrics_group_three.upsert_dimension(dimension_four)
assert metrics_group == metrics_group_three
metric_two = PanoptesMetric(u'test_metric_2', 1,
PanoptesMetricType.GAUGE)
metrics_group_three.add_metric(metric_two)
with self.assertRaises(AssertionError):
assert metrics_group == metrics_group_three
# Test PanoptesMetricsGroup.__repr__
_METRICS_GROUP_REPR = u'PanoptesMetricsGroup[' \
u'resource:plugin|test|site|test|class|test|subclass|test|type|test|id|' \
u'test|endpoint|test,' \
u'interval:120,schema_version:0.2,group_type:test,creation_timestamp:{},' \
u'dimensions:[PanoptesMetricDimension[if_alias|test],' \
u'PanoptesMetricDimension[if_name|eth0]],' \
u'metrics:[PanoptesMetric[test_metric|0|GAUGE|{}]]]'.format(mock_time.return_value,
mock_time.return_value)
self.assertEqual(repr(metrics_group), _METRICS_GROUP_REPR)
dimensions_as_dicts = [{
u'dimension_name': dimension.name,
u'dimension_value': dimension.value
} for dimension in metrics_group.dimensions]
self.assertEqual(
PanoptesMetricsGroup.flatten_dimensions(dimensions_as_dicts), {
u'if_alias': u'test',
u'if_name': u'eth0'
})
class TestMetrics(unittest.TestCase):
def setUp(self):
self.__panoptes_resource = PanoptesResource(resource_site=u'test',
resource_class=u'test',
resource_subclass=u'test',
resource_type=u'test',
resource_id=u'test',
resource_endpoint=u'test',
resource_plugin=u'test')
self.__panoptes_resource.add_metadata(u'test', u'test')
def test_panoptes_metric(self):
with self.assertRaises(AssertionError):
PanoptesMetric(None, 0, PanoptesMetricType.GAUGE)
with self.assertRaises(ValueError):
PanoptesMetric(u'1', 0, PanoptesMetricType.GAUGE)
with self.assertRaises(AssertionError):
PanoptesMetric(u'test_metric', None, PanoptesMetricType.GAUGE)
with self.assertRaises(AssertionError):
PanoptesMetric(u'test_metric', 0, None)
with self.assertRaises(AssertionError):
PanoptesMetric(u'test_metric', True, PanoptesMetricType.GAUGE)
metric1 = PanoptesMetric(
u'test_metric',
0,
PanoptesMetricType.GAUGE,
metric_creation_timestamp=mock_time.return_value)
self.assertEqual(metric1.metric_name, u'test_metric')
self.assertEqual(metric1.metric_value, 0)
self.assertEqual(metric1.metric_timestamp, mock_time.return_value)
self.assertEqual(metric1.metric_type, PanoptesMetricType.GAUGE)
self.assertEqual(
repr(metric1), u"PanoptesMetric[test_metric|0|GAUGE|{}]".format(
mock_time.return_value))
self.assertNotEqual(metric1, None)
# Check PanoptesMetric.__eq__
assert metric1 == PanoptesMetric(u'test_metric', 0,
PanoptesMetricType.GAUGE)
with self.assertRaises(AssertionError):
assert metric1 == PanoptesMetricDimension(u"test", u"value")
with self.assertRaises(AssertionError):
assert metric1 == PanoptesMetric(u'different_name', 0,
PanoptesMetricType.GAUGE)
with self.assertRaises(AssertionError):
assert metric1 == PanoptesMetric(u'test_metric', 1,
PanoptesMetricType.GAUGE)
with self.assertRaises(AssertionError):
assert metric1 == PanoptesMetric(u'test_metric', 0,
PanoptesMetricType.COUNTER)
@patch(u'yahoo_panoptes.framework.metrics.time', mock_time)
def test_panoptes_metric_json_and_repr(self):
metric = PanoptesMetric(u'test_metric', 0, PanoptesMetricType.GAUGE,
mock_time.return_value)
serialized = json.loads(metric.json)
expected = {
u"metric_creation_timestamp": mock_time.return_value,
u"metric_name": u"test_metric",
u"metric_type": u"gauge",
u"metric_value": 0
}
self.assertEqual(ordered(serialized), ordered(expected))
def testMetricsGroup(self):
now = round(time.time(), METRICS_TIMESTAMP_PRECISION)
metrics_group = PanoptesMetricsGroup(self.__panoptes_resource, u'test',
120)
self.assertEqual(metrics_group.group_type, u'test')
self.assertEqual(metrics_group.interval, 120)
self.assertEqual(metrics_group.schema_version, u'0.2')
self.assertGreaterEqual(metrics_group.creation_timestamp, now)
with patch(u'yahoo_panoptes.framework.metrics.time', mock_time):
metrics_group = PanoptesMetricsGroup(self.__panoptes_resource,
u'test', 120)
dimension_one = PanoptesMetricDimension(u'if_alias', u'bar')
dimension_two = PanoptesMetricDimension(u'if_alias', u'foo')
metrics_group.add_dimension(dimension_one)
with self.assertRaises(KeyError):
metrics_group.add_dimension(dimension_two)
# Test basic dimension operations
self.assertEqual(len(metrics_group.dimensions), 1)
self.assertTrue(
metrics_group.contains_dimension_by_name(u'if_alias'))
self.assertFalse(metrics_group.contains_dimension_by_name(u'baz'))
self.assertEqual(
metrics_group.get_dimension_by_name(u'if_alias').value, u'bar')
metrics_group.delete_dimension_by_name(u'if_alias')
self.assertFalse(
metrics_group.contains_dimension_by_name(u'if_alias'))
self.assertEqual(len(metrics_group.dimensions), 0)
self.assertEqual(metrics_group.get_dimension_by_name(u'foo'), None)
metrics_group.add_dimension(dimension_two)
dimension_three = PanoptesMetricDimension(u'if_alias', u'test')
metrics_group.upsert_dimension(dimension_three)
self.assertEqual(len(metrics_group.dimensions), 1)
self.assertEqual(
metrics_group.get_dimension_by_name(u'if_alias').value,
u'test')
dimension_four = PanoptesMetricDimension(u'if_name', u'eth0')
metrics_group.upsert_dimension(dimension_four)
self.assertEqual(len(metrics_group.dimensions), 2)
# Test basic metric operations
with self.assertRaises(AssertionError):
metrics_group.add_metric(None)
metric = PanoptesMetric(u'test_metric', 0,
PanoptesMetricType.GAUGE)
metrics_group.add_metric(metric)
with self.assertRaises(KeyError):
metrics_group.add_metric(metric)
to_json = metrics_group.json
metrics = PanoptesMetricsGroup.flatten_metrics(
json.loads(to_json)[u'metrics'])
self.assertEquals(metrics[u'gauge'][u'test_metric'][u'value'], 0)
metrics_group_two = PanoptesMetricsGroup(self.__panoptes_resource,
u'test', 120)
metrics_group_two.add_dimension(dimension_two)
metrics_group_two.upsert_dimension(dimension_three)
metrics_group_two.upsert_dimension(dimension_four)
metrics_group_two.add_metric(metric)
self.assertEqual(metrics_group, metrics_group_two)
# Check PanoptesMetricsGroup.__eq__
panoptes_resource_two = PanoptesResource(
resource_site=u'test2',
resource_class=u'test2',
resource_subclass=u'test2',
resource_type=u'test2',
resource_id=u'test2',
resource_endpoint=u'test2',
resource_plugin=u'test2')
metrics_group_two = PanoptesMetricsGroup(panoptes_resource_two,
u'test', 120)
metrics_group_three = PanoptesMetricsGroup(
self.__panoptes_resource, u'test', 120)
with self.assertRaises(AssertionError):
assert metrics_group_two == metrics_group_three
metrics_group_three = metrics_group.copy()
with self.assertRaises(AssertionError):
assert metrics_group == dimension_one
assert metrics_group == metrics_group_three
metrics_group_three.delete_dimension_by_name(u"if_name")
with self.assertRaises(AssertionError):
assert metrics_group == metrics_group_three
metrics_group_three.upsert_dimension(dimension_four)
assert metrics_group == metrics_group_three
metric_two = PanoptesMetric(u'test_metric_2', 1,
PanoptesMetricType.GAUGE)
metrics_group_three.add_metric(metric_two)
with self.assertRaises(AssertionError):
assert metrics_group == metrics_group_three
# Test PanoptesMetricsGroup.__repr__
_METRICS_GROUP_REPR = u'PanoptesMetricsGroup[' \
u'resource:plugin|test|site|test|class|test|subclass|test|type|test|id|' \
u'test|endpoint|test,' \
u'interval:120,schema_version:0.2,group_type:test,creation_timestamp:{},' \
u'dimensions:[PanoptesMetricDimension[if_alias|test],' \
u'PanoptesMetricDimension[if_name|eth0]],' \
u'metrics:[PanoptesMetric[test_metric|0|GAUGE|{}]]]'.format(mock_time.return_value,
mock_time.return_value)
self.assertEqual(repr(metrics_group), _METRICS_GROUP_REPR)
dimensions_as_dicts = [{
u'dimension_name': dimension.name,
u'dimension_value': dimension.value
} for dimension in metrics_group.dimensions]
self.assertEqual(
PanoptesMetricsGroup.flatten_dimensions(dimensions_as_dicts), {
u'if_alias': u'test',
u'if_name': u'eth0'
})
def test_panoptes_metrics_group_encoder(self):
test_dict = dict()
encoder = PanoptesMetricsGroupEncoder()
mock_default = Mock(json.JSONEncoder.default)
with patch(
u'yahoo_panoptes.framework.metrics.json.JSONEncoder.default',
mock_default):
encoder.default(test_dict)
mock_default.assert_called_once()
def test_panoptes_metric_dimension(self):
with self.assertRaises(ValueError):
PanoptesMetricDimension(u'contain$_invalid_character$', u'bar')
with self.assertRaises(ValueError):
PanoptesMetricDimension(u'foo', u'contains_pipe|')
dimension_one = PanoptesMetricDimension(u'if_alias', u'bar')
self.assertEqual(
dimension_one.json,
u'{"dimension_name": "if_alias", "dimension_value": "bar"}')
self.assertEqual(repr(dimension_one),
u'PanoptesMetricDimension[if_alias|bar]')
metric_one = PanoptesMetric(
u'test_metric',
0,
PanoptesMetricType.GAUGE,
metric_creation_timestamp=mock_time.return_value)
with self.assertRaises(AssertionError):
assert dimension_one == metric_one
dimension_two = PanoptesMetricDimension(u'if_alias', u'foo')
with self.assertRaises(AssertionError):
assert dimension_one == dimension_two
dimension_three = PanoptesMetricDimension(u'if_alias', u'bar')
assert dimension_one == dimension_three
def test_metrics_group_hash(self):
now = round(time.time(), METRICS_TIMESTAMP_PRECISION)
metrics_group = PanoptesMetricsGroup(self.__panoptes_resource, u'test',
120)
metrics_group_two = PanoptesMetricsGroup(self.__panoptes_resource,
u'test', 120)
dimension = PanoptesMetricDimension(u'if_alias', u'bar')
metric = PanoptesMetric(u'test_metric',
0,
PanoptesMetricType.GAUGE,
metric_creation_timestamp=now)
metric_diff_timestamp = PanoptesMetric(u'test_metric',
0,
PanoptesMetricType.GAUGE,
metric_creation_timestamp=now +
0.01)
metrics_group.add_dimension(dimension)
metrics_group_two.add_dimension(dimension)
self.assertEqual(metrics_group.__hash__(),
metrics_group_two.__hash__())
metrics_group.add_metric(metric)
metrics_group_two.add_metric(metric_diff_timestamp)
self.assertEqual(metrics_group.__hash__(),
metrics_group_two.__hash__())
@patch(u'yahoo_panoptes.framework.metrics.time', mock_time)
def test_panoptes_metrics_group_set(self):
"""Tests basic PanoptesMetricsGroupSet operations"""
metrics_group_set = PanoptesMetricsGroupSet()
metrics_group = PanoptesMetricsGroup(self.__panoptes_resource, u'test',
120)
metrics_group_two = PanoptesMetricsGroup(self.__panoptes_resource,
u'test', 120)
metrics_group_set.add(metrics_group)
metrics_group_set.add(metrics_group_two)
assert len(metrics_group_set) == 1
self.assertIn(metrics_group, metrics_group_set.metrics_groups)
metrics_group_set.remove(metrics_group_two)
assert len(metrics_group_set) == 0
metrics_group_set.add(metrics_group)
metrics_group_three = PanoptesMetricsGroup(self.__panoptes_resource,
u'test3', 120)
metrics_group_three.add_metric(
PanoptesMetric(u"test3", 0.0, PanoptesMetricType.GAUGE))
metrics_group_set.add(metrics_group_three)
assert len(metrics_group_set) == 2
metrics_group_set_two = PanoptesMetricsGroupSet()
metrics_group_four = PanoptesMetricsGroup(self.__panoptes_resource,
u'test', 120)
metrics_group_four.add_metric(
PanoptesMetric(u"test4", 0.0, PanoptesMetricType.GAUGE))
metrics_group_set_two.add(metrics_group_four)
assert len(metrics_group_set_two) == 1
# Test PanoptesMetricsGroupSet.__add__
metrics_group_set_union = metrics_group_set + metrics_group_set_two
assert len(metrics_group_set_union) == 3
with self.assertRaises(AssertionError):
metrics_group_set.remove(self.__panoptes_resource)
with self.assertRaises(TypeError):
metrics_group_set + metrics_group
# Test PanoptesMetricsGroupSet.__iter__ & 'next'
metrics_group_count = 0
metrics_group_set_union_interator = iter(metrics_group_set_union)
for _ in metrics_group_set_union:
self.assertIn(next(metrics_group_set_union_interator),
metrics_group_set_union.metrics_groups)
metrics_group_count += 1
assert len(metrics_group_set_union) == metrics_group_count
with self.assertRaises(Exception):
next(metrics_group_set_union_interator)
# Test PanoptesMetricsGroupSet.__repr__
_METRICS_GROUP_SET_REPR = u"PanoptesMetricsGroupSet[PanoptesMetricsGroup[resource:" \
u"plugin|test|site|test|class|test|subclass|test|type|test|id|test|endpoint|test," \
u"interval:120,schema_version:0.2,group_type:test,creation_timestamp:{}," \
u"dimensions:[],metrics:[]],PanoptesMetricsGroup[resource:" \
u"plugin|test|site|test|class|test|subclass|test|type|test|id|test|endpoint|test," \
u"interval:120,schema_version:0.2,group_type:test3,creation_timestamp:{}," \
u"dimensions:[],metrics:[" \
u"PanoptesMetric[test3|0.0|GAUGE|{}]]]]".format(mock_time.return_value,
mock_time.return_value,
mock_time.return_value)
self.assertEqual(repr(metrics_group_set), _METRICS_GROUP_SET_REPR)
def test_panoptes_metric_set(self):
metric_set = PanoptesMetricSet()
metric1 = PanoptesMetric(
u'test_metric',
0,
PanoptesMetricType.GAUGE,
metric_creation_timestamp=mock_time.return_value)
metric2 = PanoptesMetric(
u'test_metric',
0,
PanoptesMetricType.GAUGE,
metric_creation_timestamp=mock_time.return_value)
metric_set.add(metric1)
metric_set.add(metric2)
assert len(metric_set) == 1
self.assertIn(metric1, metric_set.metrics)
# Test PanoptesMetricSet.__repr__
_METRIC_SET_REPR = u"PanoptesMetricSet[PanoptesMetric[test_metric|0|GAUGE|{}]]".format(
mock_time.return_value)
self.assertEqual(repr(metric_set), _METRIC_SET_REPR)
with self.assertRaises(Exception):
metric_set.remove(self.__panoptes_resource)
metric_set.remove(metric1)
assert len(metric_set) == 0
# Test PanoptesMetricSet.__iter__ and 'next'
metric_count = 0
metric_set_iterator = iter(metric_set)
for _ in metric_set:
self.assertIn(next(metric_set_iterator), metric_set.metrics)
metric_count += 1
assert len(metric_set) == metric_count
with self.assertRaises(Exception):
next(metric_set_iterator)
def setUp(self):
self.my_dir, self.panoptes_test_conf_file = _get_test_conf_file()
self._panoptes_context = PanoptesContext(self.panoptes_test_conf_file,
key_value_store_class_list=[PanoptesEnrichmentCacheKeyValueStore,
PanoptesResourcesKeyValueStore])
self._enrichment_kv = self._panoptes_context.get_kv_store(PanoptesEnrichmentCacheKeyValueStore)
self._panoptes_resource = PanoptesResource(resource_site=u'test_site',
resource_class=u'test_class',
resource_subclass=u'test_subclass',
resource_type=u'test_type',
resource_id=u'test_resource_id',
resource_endpoint=u'test_endpoint',
resource_plugin=u'test_plugin')
interface_validation_object = PanoptesEnrichmentInterfaceSchemaValidator()
neighbor_validation_object = PanoptesEnrichmentNeighborSchemaValidator()
enrichment_set1 = PanoptesEnrichmentSet(u'int_001')
enrichment_set1.add(u'speed', 1000)
enrichment_set1.add(u'index', 0o01)
enrichment_set1.add(u'status', u'up')
enrichment_set2 = PanoptesEnrichmentSet(u'int_002')
enrichment_set2.add(u'speed', 1000)
enrichment_set2.add(u'index', 0o02)
enrichment_set2.add(u'status', u'down')
enrichment_group1 = PanoptesEnrichmentGroup(u'interface', interface_validation_object, 300, 60)
enrichment_group1.add_enrichment_set(enrichment_set1)
enrichment_group1.add_enrichment_set(enrichment_set2)
enrichment_set3 = PanoptesEnrichmentSet(u'host_name')
enrichment_set3.add(u'vlan_id', 501)
enrichment_set3.add(u'property', u'Test Property')
enrichment_set3.add(u'mac', u'aa:bb:cc:dd:ee:ff')
enrichment_group2 = PanoptesEnrichmentGroup(u'neighbor', neighbor_validation_object, 600, 120)
enrichment_group2.add_enrichment_set(enrichment_set3)
self.enrichment_group_set1 = PanoptesEnrichmentGroupSet(self._panoptes_resource)
self.enrichment_group_set1.add_enrichment_group(enrichment_group1)
self.enrichment_group_set1.add_enrichment_group(enrichment_group2)
self._panoptes_resource01 = PanoptesResource(resource_site=u'test_site',
resource_class=u'test_class',
resource_subclass=u'test_subclass',
resource_type=u'test_type',
resource_id=u'test_resource_id01',
resource_endpoint=u'test_endpoint01',
resource_plugin=u'test_plugin')
enrichment_set4 = PanoptesEnrichmentSet(u'host_name01')
enrichment_set4.add(u'vlan_id', 502)
enrichment_set4.add(u'property', u'Test Property')
enrichment_set4.add(u'mac', u'aa:bb:cc:dd:ee:ff')
enrichment_group3 = PanoptesEnrichmentGroup(u'neighbor', neighbor_validation_object, 600, 120)
enrichment_group3.add_enrichment_set(enrichment_set3)
enrichment_group3.add_enrichment_set(enrichment_set4)
self.enrichment_group_set2 = PanoptesEnrichmentGroupSet(self._panoptes_resource01)
self.enrichment_group_set2.add_enrichment_group(enrichment_group1)
self.enrichment_group_set2.add_enrichment_group(enrichment_group3)
self._multi_enrichment_group_set = PanoptesEnrichmentMultiGroupSet()
self._multi_enrichment_group_set.add_enrichment_group_set(self.enrichment_group_set1)
self._multi_enrichment_group_set.add_enrichment_group_set(self.enrichment_group_set2)
class TestEnrichmentFramework(unittest.TestCase):
@patch(u'yahoo_panoptes.framework.resources.time', mock_time)
def setUp(self):
self.__panoptes_resource = PanoptesResource(resource_site=u'test', resource_class=u'test',
resource_subclass=u'test',
resource_type=u'test', resource_id=u'test',
resource_endpoint=u'test',
resource_plugin=u'test')
self.__panoptes_resource.add_metadata(u'test', u'test')
def test_enrichment_set(self):
enrichment_set = PanoptesEnrichmentSet(u'int_001')
enrichment_set.add(u'speed', 1000)
enrichment_set.add(u'index', 0o01)
enrichment_set.add(u'status', u'up')
self.assertEquals(enrichment_set.key, u'int_001')
self.assertDictEqual(enrichment_set.value, {u'status': u'up', u'index': 1, u'speed': 1000})
self.assertEquals(len(enrichment_set), 3)
enrichment_set1 = PanoptesEnrichmentSet(u'int_002', {u'status': u'down', u'index': 2, u'speed': 1000})
self.assertEquals(enrichment_set1.key, u'int_002')
self.assertDictEqual(enrichment_set1.value, {u'status': u'down', u'index': 2, u'speed': 1000})
with self.assertRaises(AssertionError):
PanoptesEnrichmentSet(u'int_001', u'string')
with self.assertRaises(AssertionError):
PanoptesEnrichmentSet(u'int_001', 100)
def test_enrichment_schema_validator(self):
validator = PanoptesEnrichmentInterfaceSchemaValidator()
enrichment_set = PanoptesEnrichmentSet(u'int_001')
enrichment_set.add(u'speed', 1000)
enrichment_set.add(u'index', 0o01)
enrichment_set.add(u'status', u'up')
self.assertTrue(validator.validate(enrichment_set))
enrichment_set.add(u'status', 0o1)
self.assertFalse(validator.validate(enrichment_set))
@patch(u'time.time', mock_time)
def test_enrichment_group(self):
interface_validation_object = PanoptesEnrichmentInterfaceSchemaValidator()
neighbor_validation_object = PanoptesEnrichmentNeighborSchemaValidator()
interface_data = \
u'''{"data": [
{"int_001": {"index": 1, "speed": 1000, "status": "up"}},
{"int_002": {"index": 2, "speed": 1000, "status": "down"}}],
"metadata": {"_enrichment_group_creation_timestamp": %f, "_enrichment_ttl": 300, "_execute_frequency": 60},
"namespace": "interface"}''' % mock_time.return_value
neighbor_data = \
u'''{"data": [{"host_name": {"mac": "aa:bb:cc:dd:ee:ff", "property": "Test Property", "vlan_id": 501}}],
"metadata": {"_enrichment_group_creation_timestamp": %f, "_enrichment_ttl": 600, "_execute_frequency": 120},
"namespace": "neighbor"}''' % mock_time.return_value
with self.assertRaises(AssertionError):
PanoptesEnrichmentGroup(1, interface_validation_object, 300, 60)
with self.assertRaises(AssertionError):
PanoptesEnrichmentGroup(u'interface', u'non_validation_object', 300, 60)
with self.assertRaises(AssertionError):
PanoptesEnrichmentGroup(u'interface', interface_validation_object, u'300', 60)
with self.assertRaises(AssertionError):
PanoptesEnrichmentGroup(u'interface', interface_validation_object, 300, u'60')
with self.assertRaises(AssertionError):
PanoptesEnrichmentGroup(u'interface', interface_validation_object, 0, 60)
with self.assertRaises(AssertionError):
PanoptesEnrichmentGroup(u'interface', interface_validation_object, 300, 0)
with self.assertRaises(AssertionError):
PanoptesEnrichmentGroup(u'interface', interface_validation_object, 300, 60).\
add_enrichment_set(u'not_PanoptesEnrichmentSet_obj')
enrichment_set1 = PanoptesEnrichmentSet(u'int_001')
enrichment_set1.add(u'speed', 1000)
enrichment_set1.add(u'index', 0o01)
enrichment_set1.add(u'status', u'up')
enrichment_set2 = PanoptesEnrichmentSet(u'int_002')
enrichment_set2.add(u'speed', 1000)
enrichment_set2.add(u'index', 0o02)
enrichment_set2.add(u'status', u'down')
enrichment_group1 = PanoptesEnrichmentGroup(u'interface', interface_validation_object, 300, 60)
enrichment_group1.add_enrichment_set(enrichment_set1)
enrichment_group1.add_enrichment_set(enrichment_set2)
self.assertEqual(enrichment_group1.namespace, u'interface')
self.assertEqual(enrichment_group1.enrichment_ttl, 300)
self.assertEqual(enrichment_group1.execute_frequency, 60)
self.assertEqual(enrichment_group1.enrichment_group_creation_timestamp, mock_time.return_value)
self.assertEqual(ordered(json.loads(json.dumps(enrichment_group1.data, cls=PanoptesEnrichmentEncoder))),
ordered(json.loads(interface_data)[u'data']))
self.assertEqual(ordered(json.loads(enrichment_group1.json())), ordered(json.loads(interface_data)))
self.assertEquals(len(enrichment_group1), 2)
enrichment_set3 = PanoptesEnrichmentSet(u'int_002')
enrichment_set3.add(u'speed', 1000)
enrichment_set3.add(u'index', 0o02)
enrichment_set3.add(u'status', u'down')
self.assertEqual(ordered(json.loads(enrichment_group1.json())), ordered(json.loads(interface_data)))
self.assertEqual(ordered(enrichment_group1.metadata), ordered(json.loads(interface_data)[u'metadata']))
self.assertEquals(len(enrichment_group1), 2)
test_metadata = json.loads(interface_data)[u'metadata']
test_metadata[u'metadata_key'] = u'metadata_value'
enrichment_group1.upsert_metadata(u'metadata_key', u'metadata_value')
self.assertEqual(ordered(enrichment_group1.metadata), ordered(test_metadata))
enrichment_group1.upsert_metadata(u'ttl', 300)
with self.assertRaises(ValueError):
enrichment_group1.upsert_metadata(u'_enrichment_ttl', 300)
with self.assertRaises(AssertionError):
enrichment_group1.upsert_metadata(u'metadata', {})
with self.assertRaises(AssertionError):
enrichment_group1.upsert_metadata(u'metadata', [])
enrichment_set4 = PanoptesEnrichmentSet(u'host_name')
enrichment_set4.add(u'vlan_id', 501)
enrichment_set4.add(u'property', u'Test Property')
enrichment_set4.add(u'mac', u'aa:bb:cc:dd:ee:ff')
enrichment_group2 = PanoptesEnrichmentGroup(u'neighbor', neighbor_validation_object, 600, 120)
enrichment_group2.add_enrichment_set(enrichment_set4)
self.assertEqual(ordered(json.loads(enrichment_group2.json())), ordered(json.loads(neighbor_data)))
self.assertEquals(len(enrichment_group2), 1)
enrichment_set5 = PanoptesEnrichmentSet(u'host_name01')
enrichment_set5.add(u'vlan_id', 502)
enrichment_set5.add(u'property', u'Netops01.US')
enrichment_set6 = PanoptesEnrichmentSet(u'host_name02')
enrichment_set6.add(u'vlan_id', 503)
enrichment_set6.add(u'mac', u'aa:bb:cc:dd:ee:ff')
enrichment_group3 = PanoptesEnrichmentGroup(u'neighbor', neighbor_validation_object, 600, 120)
enrichment_group3.add_enrichment_set(enrichment_set5)
with self.assertRaises(AssertionError):
enrichment_group3.add_enrichment_set(enrichment_set6)
interface_store_data = u'{"int_001": {"index": 1, "speed": 1000, "status": "up"}, ' \
u'"int_002": {"index": 2, "speed": 1000, "status": "down"}}'
neighbor_store_data = u'{"host_name": {"mac": "aa:bb:cc:dd:ee:ff", ' \
u'"property": "Test Property", "vlan_id": 501}}'
self.assertEquals(ordered(json.loads(enrichment_group1.serialize_data())),
ordered(json.loads(interface_store_data)))
self.assertEquals(ordered(json.loads(enrichment_group2.serialize_data())),
ordered(json.loads(neighbor_store_data)))
enrichment_group1.upsert_metadata(u'ttl', 300)
with self.assertRaises(ValueError):
enrichment_group1.upsert_metadata(u'_enrichment_ttl', 300)
interface_data_serialized = u'''{{"data": {{"int_001": {{"index": 1, "speed": 1000, "status": "up"}},
"int_002": {{"index": 2, "speed": 1000, "status": "down"}}}}, "metadata":
{{"_enrichment_group_creation_timestamp": {:.5f}, "_enrichment_ttl": 300, "_execute_frequency": 60,
"metadata_key": "metadata_value", "ttl": 300}}}}'''.format(mock_time.return_value)
neighbor_data_serialized = u'''{{"data": {{"host_name": {{"mac": "aa:bb:cc:dd:ee:ff", "property":
"Test Property","vlan_id": 501}}}}, "metadata": {{"_enrichment_group_creation_timestamp": {:.5f},
"_enrichment_ttl": 600, "_execute_frequency": 120}}}}'''.format(mock_time.return_value)
self.assertEquals(ordered(json.loads(enrichment_group1.serialize())),
ordered(json.loads(interface_data_serialized)))
self.assertEquals(ordered(json.loads(enrichment_group2.serialize())),
ordered(json.loads(neighbor_data_serialized)))
@patch(u'time.time', mock_time)
def test_enrichment_group_set(self):
interface_validation_object = PanoptesEnrichmentInterfaceSchemaValidator()
neighbor_validation_object = PanoptesEnrichmentNeighborSchemaValidator()
panoptes_resource = self.__panoptes_resource
enrichment_data = \
u'''{{"enrichment": [{{"metadata": {{"_enrichment_group_creation_timestamp": {:.5f}, "_enrichment_ttl": 600,
"_execute_frequency": 120}}, "data": [{{"host_name":
{{"mac": "aa:bb:cc:dd:ee:ff", "property": "Test Property", "vlan_id": 501}}}}],
"namespace": "neighbor"}}, {{"metadata": {{"_enrichment_group_creation_timestamp": {:.5f},
"_enrichment_ttl": 300,
"_execute_frequency": 60}}, "data": [
{{"int_001": {{"index": 1, "speed": 1000, "status": "up"}}}}, {{"int_002": {{"index": 2, "speed": 1000,
"status": "down"}}}}], "namespace": "interface"}}],
"enrichment_group_set_creation_timestamp": {:.5f}, "resource": {{"resource_class": "test",
"resource_creation_timestamp": {:.5f}, "resource_endpoint": "test", "resource_id": "test",
"resource_metadata": {{"_resource_ttl": "604800", "test": "test"}}, "resource_plugin": "test",
"resource_site": "test",
"resource_subclass": "test", "resource_type": "test"}}}}'''.format(mock_time.return_value,
mock_time.return_value,
mock_time.return_value,
mock_time.return_value)
enrichment_set1 = PanoptesEnrichmentSet(u'int_001')
enrichment_set1.add(u'speed', 1000)
enrichment_set1.add(u'index', 0o01)
enrichment_set1.add(u'status', u'up')
enrichment_set2 = PanoptesEnrichmentSet(u'int_002')
enrichment_set2.add(u'speed', 1000)
enrichment_set2.add(u'index', 0o02)
enrichment_set2.add(u'status', u'down')
enrichment_group1 = PanoptesEnrichmentGroup(u'interface', interface_validation_object, 300, 60)
enrichment_group1.add_enrichment_set(enrichment_set1)
enrichment_group1.add_enrichment_set(enrichment_set2)
enrichment_set3 = PanoptesEnrichmentSet(u'host_name')
enrichment_set3.add(u'vlan_id', 501)
enrichment_set3.add(u'property', u'Test Property')
enrichment_set3.add(u'mac', u'aa:bb:cc:dd:ee:ff')
enrichment_group2 = PanoptesEnrichmentGroup(u'neighbor', neighbor_validation_object, 600, 120)
enrichment_group2.add_enrichment_set(enrichment_set3)
enrichment_group_set1 = PanoptesEnrichmentGroupSet(panoptes_resource)
enrichment_group_set1.add_enrichment_group(enrichment_group1)
enrichment_group_set1.add_enrichment_group(enrichment_group2)
self.assertEquals(len(enrichment_group_set1), 2)
group_set_repr = u"PanoptesEnrichmentGroupSet[resource:" \
u"plugin|test|site|test|class|test|subclass|test|type|test|id|test|endpoint|test," \
u"enrichment_group_set_creation_timestamp:{},PanoptesEnrichmentGroup[namespace:" \
u"interface,enrichment_ttl:300,execute_frequency:60,enrichment_group_creation_timestamp:{}," \
u"PanoptesEnrichmentSet[int_001[index:1,speed:1000,status:up]],PanoptesEnrichmentSet[" \
u"int_002[index:2,speed:1000,status:down]]],PanoptesEnrichmentGroup[namespace:neighbor," \
u"enrichment_ttl:600,execute_frequency:120,enrichment_group_creation_timestamp:{}," \
u"PanoptesEnrichmentSet[host_name[mac:aa:bb:cc:dd:ee:ff,property:" \
u"Test Property,vlan_id:501]]]]".format(mock_time.return_value,
mock_time.return_value,
mock_time.return_value)
self.assertEquals(repr(enrichment_group_set1), group_set_repr)
self.assertIsInstance(enrichment_group_set1.resource, PanoptesResource)
self.assertEqual(enrichment_group_set1.enrichment_group_set_creation_timestamp, mock_time.return_value)
self.assertEqual(
ordered(json.loads(json.dumps(enrichment_group_set1.enrichment, cls=PanoptesEnrichmentEncoder))),
ordered(json.loads(enrichment_data)[u'enrichment']))
self.assertEqual(ordered(json.loads(enrichment_group_set1.json())[u'enrichment']),
ordered(json.loads(enrichment_data)[u'enrichment']))
with self.assertRaises(AssertionError):
PanoptesEnrichmentGroupSet(u'bad_resource')
with self.assertRaises(AssertionError):
PanoptesEnrichmentGroupSet(panoptes_resource).add_enrichment_group(u'non_PanoptesEnrichmentGroup_obj')
enrichment_group_set2 = PanoptesEnrichmentGroupSet(panoptes_resource)
enrichment_group3 = PanoptesEnrichmentGroup(u'interface', interface_validation_object, 300, 60)
with self.assertRaises(AssertionError):
enrichment_group_set2.add_enrichment_group(enrichment_group3)
self.assertFalse(enrichment_group_set1 == enrichment_group1)
self.assertFalse(enrichment_group_set1 == enrichment_group_set2)
@patch(u'time.time', mock_time)
@patch(u'yahoo_panoptes.framework.resources.time', mock_time)
def test_multi_enrichment_group_set(self):
interface_validation_object = PanoptesEnrichmentInterfaceSchemaValidator()
neighbor_validation_object = PanoptesEnrichmentNeighborSchemaValidator()
panoptes_resource = self.__panoptes_resource
multi_enrichment_results_data = \
{
"group_sets": [
{
"enrichment": [
{
"data": [
{
"host_name": {
"mac": "aa:bb:cc:dd:ee:ff",
"property": "Test Property",
"vlan_id": 501
}
},
{
"host_name01": {
"mac": "aa:bb:cc:dd:ee:ff",
"property": "Test Property",
"vlan_id": 502
}
}
],
"metadata": {
"_enrichment_group_creation_timestamp": mock_time.return_value,
"_enrichment_ttl": 600,
"_execute_frequency": 120
},
"namespace": "neighbor"
},
{
"data": [
{
"int_001": {
"index": 1,
"speed": 1000,
"status": "up"
}
}
],
"metadata": {
"_enrichment_group_creation_timestamp": mock_time.return_value,
"_enrichment_ttl": 300,
"_execute_frequency": 60
},
"namespace": "interface"
}
],
"enrichment_group_set_creation_timestamp": mock_time.return_value,
"resource": {
"resource_class": "test_class",
"resource_creation_timestamp": mock_time.return_value,
"resource_endpoint": "test_endpoint01",
"resource_id": "test_resource_id01",
"resource_metadata": {
"_resource_ttl": "604800"
},
"resource_plugin": "test_plugin",
"resource_site": "test_site",
"resource_subclass": "test_subclass",
"resource_type": "test_type"
}
},
{
"enrichment": [
{
"data": [
{
"int_001": {
"index": 1,
"speed": 1000,
"status": "up"
}
}
],
"metadata": {
"_enrichment_group_creation_timestamp": mock_time.return_value,
"_enrichment_ttl": 300,
"_execute_frequency": 60
},
"namespace": "interface"
},
{
"data": [
{
"host_name": {
"mac": "aa:bb:cc:dd:ee:ff",
"property": "Test Property",
"vlan_id": 501
}
}
],
"metadata": {
"_enrichment_group_creation_timestamp": mock_time.return_value,
"_enrichment_ttl": 600,
"_execute_frequency": 120
},
"namespace": "neighbor"
}
],
"enrichment_group_set_creation_timestamp": mock_time.return_value,
"resource": {
"resource_class": "test",
"resource_creation_timestamp": mock_time.return_value,
"resource_endpoint": "test",
"resource_id": "test",
"resource_metadata": {
"_resource_ttl": "604800",
"test": "test"
},
"resource_plugin": "test",
"resource_site": "test",
"resource_subclass": "test",
"resource_type": "test"
}
}
]
}
enrichment_set1 = PanoptesEnrichmentSet(u'int_001')
enrichment_set1.add(u'speed', 1000)
enrichment_set1.add(u'index', 0o01)
enrichment_set1.add(u'status', u'up')
enrichment_group1 = PanoptesEnrichmentGroup(u'interface', interface_validation_object, 300, 60)
enrichment_group1.add_enrichment_set(enrichment_set1)
enrichment_set3 = PanoptesEnrichmentSet(u'host_name')
enrichment_set3.add(u'vlan_id', 501)
enrichment_set3.add(u'property', u'Test Property')
enrichment_set3.add(u'mac', u'aa:bb:cc:dd:ee:ff')
enrichment_group2 = PanoptesEnrichmentGroup(u'neighbor', neighbor_validation_object, 600, 120)
enrichment_group2.add_enrichment_set(enrichment_set3)
enrichment_group_set1 = PanoptesEnrichmentGroupSet(panoptes_resource)
enrichment_group_set1.add_enrichment_group(enrichment_group1)
enrichment_group_set1.add_enrichment_group(enrichment_group2)
panoptes_resource01 = PanoptesResource(resource_site=u'test_site',
resource_class=u'test_class',
resource_subclass=u'test_subclass',
resource_type=u'test_type',
resource_id=u'test_resource_id01',
resource_endpoint=u'test_endpoint01',
resource_plugin=u'test_plugin')
panoptes_resource02 = PanoptesResource(resource_site=u'test_site',
resource_class=u'test_class',
resource_subclass=u'test_subclass',
resource_type=u'test_type',
resource_id=u'test_resource_id02',
resource_endpoint=u'test_endpoint02',
resource_plugin=u'test_plugin')
enrichment_set4 = PanoptesEnrichmentSet(u'host_name01')
enrichment_set4.add(u'vlan_id', 502)
enrichment_set4.add(u'property', u'Test Property')
enrichment_set4.add(u'mac', u'aa:bb:cc:dd:ee:ff')
enrichment_group3 = PanoptesEnrichmentGroup(u'neighbor', neighbor_validation_object, 600, 120)
enrichment_group3.add_enrichment_set(enrichment_set3)
enrichment_group3.add_enrichment_set(enrichment_set4)
enrichment_group_set2 = PanoptesEnrichmentGroupSet(panoptes_resource01)
enrichment_group_set2.add_enrichment_group(enrichment_group1)
enrichment_group_set2.add_enrichment_group(enrichment_group3)
multi_enrichment_group_set = PanoptesEnrichmentMultiGroupSet()
multi_enrichment_group_set.add_enrichment_group_set(enrichment_group_set1)
multi_enrichment_group_set.add_enrichment_group_set(enrichment_group_set2)
multi_enrichment_group_set_repr = u"PanoptesEnrichmentMultiGroupSet[PanoptesEnrichmentGroupSet[resource:" \
u"plugin|test|site|test|class|test|subclass|test|type|test|id|test|endpoint" \
u"|test,enrichment_group_set_creation_timestamp:{},PanoptesEnrichmentGroup" \
u"[namespace:interface,enrichment_ttl:300,execute_frequency:60," \
u"enrichment_group_creation_timestamp:{},PanoptesEnrichmentSet[" \
u"int_001[index:1,speed:1000,status:up]]],PanoptesEnrichmentGroup[" \
u"namespace:neighbor,enrichment_ttl:600,execute_frequency:120," \
u"enrichment_group_creation_timestamp:{},PanoptesEnrichmentSet" \
u"[host_name[mac:aa:bb:cc:dd:ee:ff,property:Test Property,vlan_id:501]" \
u"]]],PanoptesEnrichmentGroupSet[resource:plugin|test_plugin|site|" \
u"test_site|class|test_class|subclass|test_subclass|type|test_type|id" \
u"|test_resource_id01|endpoint|test_endpoint01," \
u"enrichment_group_set_creation_timestamp:{},PanoptesEnrichmentGroup" \
u"[namespace:interface,enrichment_ttl:300,execute_frequency:60," \
u"enrichment_group_creation_timestamp:{},PanoptesEnrichmentSet" \
u"[int_001[index:1,speed:1000,status:up]]],PanoptesEnrichmentGroup" \
u"[namespace:neighbor,enrichment_ttl:600,execute_frequency:120," \
u"enrichment_group_creation_timestamp:{},PanoptesEnrichmentSet" \
u"[host_name[mac:aa:bb:cc:dd:ee:ff,property:Test Property,vlan_id:501]]," \
u"PanoptesEnrichmentSet[host_name01[mac:aa:bb:cc:dd:ee:ff," \
u"property:Test Property,vlan_id:502]]]]]".format(mock_time.return_value,
mock_time.return_value,
mock_time.return_value,
mock_time.return_value,
mock_time.return_value,
mock_time.return_value)
self.assertEquals(repr(multi_enrichment_group_set), multi_enrichment_group_set_repr)
self.assertEquals(len(multi_enrichment_group_set.enrichment_group_sets), 2)
self.assertEquals(ordered(json.loads(multi_enrichment_group_set.json())),
ordered(multi_enrichment_results_data))
self.assertEquals(len(multi_enrichment_group_set), 2)
multi_enrichment_group_set.add_enrichment_group_set(enrichment_group_set2)
self.assertEquals(len(multi_enrichment_group_set), 2)
enrichment_group_set3 = PanoptesEnrichmentGroupSet(panoptes_resource02)
enrichment_group_set3.add_enrichment_group(enrichment_group1)
multi_enrichment_group_set.add_enrichment_group_set(enrichment_group_set3)
self.assertEquals(len(multi_enrichment_group_set), 3)
with self.assertRaises(AssertionError):
multi_enrichment_group_set.add_enrichment_group_set(u'non_enrichment_group')
enrichment_group_set3 = PanoptesEnrichmentGroupSet(panoptes_resource01)
with self.assertRaises(AssertionError):
multi_enrichment_group_set.add_enrichment_group_set(enrichment_group_set3)