def setUp(self): self.__panoptes_resource = PanoptesResource(resource_site=u'test', resource_class=u'test', resource_subclass=u'test', resource_type=u'test', resource_id=u'test', resource_endpoint=u'test', resource_plugin=u'test') self.__panoptes_resource.add_metadata(u'test', u'test')
def setUp(self): _, self.panoptes_test_conf_file = get_test_conf_file() self._my_path = os.path.dirname(os.path.abspath(__file__)) self._panoptes_context = PanoptesContext(self.panoptes_test_conf_file) self._resource = PanoptesResource( resource_class="test_class", resource_subclass="test_subclass", resource_type="test_type", resource_id="test_id", resource_endpoint="127.0.0.1", resource_site="test_site", resource_creation_timestamp=time.time(), resource_plugin="test_plugin") self._key_value_store = PanoptesTestKeyValueStore( self._panoptes_context) self._secrets_store = TestPanoptesSecretsStore(self._panoptes_context) self._secrets_store.set(key='snmp_community_string:test_site', value='public') self._panoptes_plugin_context = PanoptesPluginContext( panoptes_context=self._panoptes_context, logger_name="test_logger", config=plugin_conf, key_value_store=self._key_value_store, secrets_store=self._secrets_store, data=self._resource)
def test_code_output(self): # Mock the system calls subprocess.check_output = Mock(return_value=TEST_PING_RESPONSE) resource = PanoptesResource.resource_from_dict(self._resource_dict) test_ping = PanoptesPingConnectionFactory.get_ping_connection(resource=resource, context=None, count=3, timeout=100) test_d = {"round_trip_stddev": 0.057, "packets_received": 3, "execution_time": 1.44, "round_trip_avg": 0.120, "packets_transmitted": 3, "packet_loss_pct": 0.0, "round_trip_max": 0.162, "round_trip_min": 0.040} self.assertDictEqual(test_d, json.loads(test_ping.response)) self.assertEqual(test_ping.packets_transmitted, 3) self.assertEqual(test_ping.packets_received, 3) self.assertEqual(test_ping.packet_loss_pct, 0.0) self.assertEqual(test_ping.execution_time, 1.44) # add for testing prod self.assertEqual(test_ping.round_trip_min, 0.040) self.assertEqual(test_ping.round_trip_avg, 0.120) self.assertEqual(test_ping.round_trip_max, 0.162) self.assertEqual(test_ping.round_trip_stddev, 0.057) # Test bad parameters with self.assertRaises(AssertionError): PanoptesPingConnectionFactory.get_ping_connection(resource=resource, context=None, count=-1, timeout=100) with self.assertRaises(AssertionError): PanoptesPingConnectionFactory.get_ping_connection(resource=resource, context=None, count='1', timeout=100) with self.assertRaises(AssertionError): PanoptesPingConnectionFactory.get_ping_connection(resource=None, context=None, count=3, timeout=100) with self.assertRaises(AssertionError): self._resource_dict['resource_endpoint'] = '' resource = PanoptesResource.resource_from_dict(self._resource_dict) PanoptesPingConnectionFactory.get_ping_connection(resource=resource, context=None, count=3, timeout=100) with self.assertRaises(AssertionError): self._resource_dict['resource_endpoint'] = 'localhost' resource = PanoptesResource.resource_from_dict(self._resource_dict) PanoptesPingConnectionFactory.get_ping_connection(resource=resource, context=None, count=3, timeout=0) with self.assertRaises(AssertionError): PanoptesPingConnectionFactory.get_ping_connection(resource=resource, context=None, count=3, timeout=-1)
def setUp(self): self._panoptes_metric = { u'metrics_group_interval': 60, u'resource': { u'resource_site': u'test_site', u'resource_id': u'test_id', u'resource_class': u'network', u'resource_plugin': u'test_plugin', u'resource_creation_timestamp': 1567823517.46, u'resource_subclass': u'test_subclass', u'resource_endpoint': u'test_endpoint', u'resource_metadata': { u'test_metadata_key': u'test_metadata_value', u'_resource_ttl': u"604800" }, u'resource_type': u'test_type' }, u'dimensions': [{ u'dimension_name': u'cpu_name', u'dimension_value': u'test_cpu_name_value' }, { u'dimension_name': u'cpu_no', u'dimension_value': u'test_cpu_no_value' }, { u'dimension_name': u'cpu_type', u'dimension_value': u'test_cpu_type_value' }], u'metrics_group_type': u'cpu', u'metrics': [{ u'metric_creation_timestamp': 1567823946.72, u'metric_type': u'gauge', u'metric_name': u'cpu_utilization', u'metric_value': 0 }], u'metrics_group_creation_timestamp': 1567823946.72, u'metrics_group_schema_version': u'0.2' } self._panoptes_resource = PanoptesResource(resource_site=u'test', resource_class=u'test', resource_subclass=u'test', resource_type=u'test', resource_id=u'test', resource_endpoint=u'test', resource_plugin=u'test') self._panoptes_resource.add_metadata(u'test', u'test') self._panoptes_resource_set = PanoptesResourceSet() self._panoptes_resource_set.add(self._panoptes_resource)
def setUp(self): self.__panoptes_resource_metadata = {'test': 'test', '_resource_ttl': '604800'} self.__panoptes_resource = PanoptesResource(resource_site='test', resource_class='test', resource_subclass='test', resource_type='test', resource_id='test', resource_endpoint='test', resource_plugin='test', resource_creation_timestamp=_TIMESTAMP, resource_ttl=RESOURCE_MANAGER_RESOURCE_EXPIRE) self.__panoptes_resource.add_metadata('test', 'test') self.__panoptes_resource_set = PanoptesResourceSet() mock_valid_timestamp = Mock(return_value=True) with patch('yahoo_panoptes.framework.resources.PanoptesValidators.valid_timestamp', mock_valid_timestamp): self.__panoptes_resource_set.resource_set_creation_timestamp = _TIMESTAMP self.my_dir, self.panoptes_test_conf_file = _get_test_conf_file()
def set_panoptes_resource(self): self._panoptes_resource = PanoptesResource( resource_site=self._resource_site, resource_class=self._resource_class, resource_subclass=self._resource_subclass, resource_type=self._resource_type, resource_id=self._resource_id, resource_endpoint=self._resource_endpoint, resource_plugin=self._resource_plugin) self._panoptes_resource.resource_metadata['model'] = 'model' if self._resource_backplane: self._panoptes_resource.add_metadata('backplane', self._resource_backplane)
def setUp(self): self.__panoptes_resource = PanoptesResource(resource_site='test', resource_class='test', resource_subclass='test', resource_type='test', resource_id='test', resource_endpoint='test', resource_plugin='test', resource_creation_timestamp=_TIMESTAMP, resource_ttl=RESOURCE_MANAGER_RESOURCE_EXPIRE)
def setUp(self): self.my_dir, self.panoptes_test_conf_file = get_test_conf_file() self._panoptes_resource = PanoptesResource( resource_site="test", resource_class="test", resource_subclass="test", resource_type="test", resource_id="test", resource_endpoint="test", resource_creation_timestamp=_TIMESTAMP, resource_plugin="test") self._panoptes_context = PanoptesContext( self.panoptes_test_conf_file, key_value_store_class_list=[ PanoptesTestKeyValueStore, PanoptesResourcesKeyValueStore, PanoptesPollingPluginKeyValueStore, PanoptesSecretsStore, PanoptesPollingPluginAgentKeyValueStore, PanoptesDiscoveryPluginAgentKeyValueStore, PanoptesDiscoveryPluginKeyValueStore ], create_message_producer=False, async_message_producer=False, create_zookeeper_client=True) self._runner_class = PanoptesPluginWithEnrichmentRunner self._log_capture = LogCapture( attributes=TestPanoptesPluginRunner.extract)
def setUp(self): self._panoptes_resource = PanoptesResource( resource_site=u'test_site', resource_class=u'test_class', resource_subclass=u'test_subclass', resource_type=u'test_type', resource_id=u'test_id', resource_endpoint=u'test_endpoint', resource_plugin=u'test_plugin') self._plugin_config = { u'Core': { u'name': u'Test Plugin', u'module': u'plugin_polling_ping' }, u'main': { u'resource_filter': u'resource_class = u"network"', u'execute_frequency': 60, } } self._panoptes_context = create_autospec(PanoptesContext) self._panoptes_plugin_context = create_autospec( PanoptesPluginWithEnrichmentContext, instance=True, spec_set=True, data=self._panoptes_resource, config=self._plugin_config, logger=logging.getLogger(__name__))
def test_panoptes_resource_set(self): panoptes_resource = self.__panoptes_resource panoptes_resource_set = self.__panoptes_resource_set self.assertEqual(panoptes_resource_set.add(panoptes_resource), None) self.assertEqual(len(panoptes_resource_set), 1) self.assertEqual(type(panoptes_resource_set.resources), set) self.assertIsInstance(str(panoptes_resource_set), str) self.assertIsInstance(iter(panoptes_resource_set), collections.Iterable) self.assertEqual(panoptes_resource_set.next(), panoptes_resource) self.assertEqual(panoptes_resource_set.remove(panoptes_resource), None) self.assertEqual(len(panoptes_resource_set), 0) self.assertEqual(panoptes_resource_set.resource_set_creation_timestamp, _TIMESTAMP) with self.assertRaises(AssertionError): panoptes_resource_set.resource_set_creation_timestamp = 0 panoptes_resource_2 = PanoptesResource(resource_site='test', resource_class='test', resource_subclass='test', resource_type='test', resource_id='test2', resource_endpoint='test', resource_plugin='test', resource_ttl=RESOURCE_MANAGER_RESOURCE_EXPIRE, resource_creation_timestamp=_TIMESTAMP) panoptes_resource_set.add(panoptes_resource) panoptes_resource_set.add(panoptes_resource_2) self.assertEqual(len(panoptes_resource_set.get_resources_by_site()['test']), 2) self.assertEqual(panoptes_resource_set.resource_set_schema_version, "0.1") panoptes_resource_set_json = { u'resource_set_creation_timestamp': _TIMESTAMP, u'resource_set_schema_version': u'0.1', u'resources': [ { u'resource_site': u'test', u'resource_class': u'test', u'resource_subclass': u'test', u'resource_type': u'test', u'resource_id': u'test2', u'resource_endpoint': u'test', u'resource_metadata': { u'_resource_ttl': u'604800' }, u'resource_creation_timestamp': _TIMESTAMP, u'resource_plugin': u'test' }, { u'resource_site': u'test', u'resource_class': u'test', u'resource_subclass': u'test', u'resource_type': u'test', u'resource_id': u'test', u'resource_endpoint': u'test', u'resource_metadata': { u'_resource_ttl': u'604800', u'test': u'test' }, u'resource_creation_timestamp': _TIMESTAMP, u'resource_plugin': u'test' } ] } self.assertEqual(ordered(panoptes_resource_set_json), ordered(json.loads(panoptes_resource_set.json)))
def setUp(self): self.my_dir, self.panoptes_test_conf_file = get_test_conf_file() self._panoptes_context = PanoptesContext(self.panoptes_test_conf_file, key_value_store_class_list=[PanoptesTestKeyValueStore], create_message_producer=False, async_message_producer=False, create_zookeeper_client=True) self._panoptes_resource = PanoptesResource(resource_site='test', resource_class='test', resource_subclass='test', resource_type='test', resource_id='test', resource_endpoint='test', resource_plugin='test')
def run(self, context): """ The main entry point to the plugin Args: context (PanoptesPluginContext): The Plugin Context passed by the Plugin Agent Returns: PanoptesResourceSet: A non-empty resource set Raises: PanoptesDiscoveryPluginError: This exception is raised if any part of the lookup process has errors """ assert context and isinstance(context, PanoptesPluginContext), 'context must be a PanoptesPluginContext' conf = context.config logger = context.logger config_file = None try: config_file = conf['main']['config_file'] with open(config_file) as f: resource_specs = json.load(f) except Exception as e: raise PanoptesDiscoveryPluginError( 'Error while attempting to parse JSON from file {}: {}'.format(config_file, repr(e)) ) resources = PanoptesResourceSet() num_successes = 0 num_failures = 0 for resource_spec in resource_specs: try: resource = PanoptesResource.resource_from_dict(resource_spec) resources.add(resource) num_successes += 1 logger.debug('Added resource {} from JSON file {}'.format(resource, config_file)) except Exception as e: logger.debug('Error while attempting to create a PanoptesResource from file {}: {}'.format( config_file, repr(e))) num_failures += 1 continue if num_successes > 0: logger.info('Tried to read {} resources from {}, {} failed'.format(num_successes + num_failures, config_file, num_failures)) else: logger.error('Error while attempting to create PanoptesResources from {}.'.format(config_file)) raise PanoptesDiscoveryPluginError( 'Error during lookup for PanoptesResource from file {}.'.format(config_file)) return resources
def setUp(self): super(TestPanoptesPluginWithEnrichmentRunner, self).setUp() self._panoptes_resource = PanoptesResource( resource_site="test", resource_class="test", resource_subclass="test", resource_type="test", resource_id="test", resource_endpoint="test", resource_creation_timestamp=_TIMESTAMP, resource_plugin="test") self._runner_class = PanoptesPluginWithEnrichmentRunner
def test_steamroller_ping(self): plugin_conf = { 'Core': { 'name': 'Test Plugin', 'module': 'test_plugin' }, 'main': { 'execute_frequency': '60', 'resource_filter': 'resource_class = "network"' } } self._resource_dict['resource_metadata']['ping_route'] = 'steamroller' self._resource_dict['resource_metadata']['snmp_proxy_hosts'] = 'proxyhost-5.doesntexist.com:4443' resource = PanoptesResource.resource_from_dict(self._resource_dict) plugin_context = create_autospec( PanoptesPluginWithEnrichmentContext, instance=True, spec_set=True, data=resource, config=plugin_conf, snmp=panoptes_context.config_object.snmp_defaults, x509=panoptes_context.config_object.x509_defaults, secrets=secret_store, logger=logging.getLogger(__name__) ) with patch('requests.Session') as rs: rs.return_value = MockConnection() ping_connection = PanoptesPingConnectionFactory.get_ping_connection(resource, plugin_context) self.assertEqual(ping_connection.packets_transmitted, 10) self.assertEqual(ping_connection.packets_received, 10) self.assertEqual(ping_connection.packet_loss_pct, 0.0) self.assertEqual(ping_connection.execution_time, 9.03) self.assertEqual(ping_connection.round_trip_min, 0.485) self.assertEqual(ping_connection.round_trip_avg, 0.504) self.assertEqual(ping_connection.round_trip_max, 0.535) self.assertEqual(ping_connection.round_trip_stddev, 0.027) with patch.object(requests.Session, 'post') as rs: rs.side_effect = requests.exceptions.Timeout() with self.assertRaises(PanoptesPingTimeoutException): PanoptesPingConnectionFactory.get_ping_connection(resource, plugin_context) with patch.object(requests.Session, 'post') as rs: rs.side_effect = requests.exceptions.ConnectionError() with self.assertRaises(PanoptesPingConnectionError): PanoptesPingConnectionFactory.get_ping_connection(resource, plugin_context) with patch.object(requests.Session, 'post') as rs: rs.side_effect = requests.exceptions.RequestException() with self.assertRaises(PanoptesPingRequestException): PanoptesPingConnectionFactory.get_ping_connection(resource, plugin_context)
def prepare_panoptes_metrics_group_set(self, file_path=None): panoptes_metric_group_set = PanoptesMetricsGroupSet() path_to_metrics_file = plugin_results_file if file_path is None else file_path with open(path_to_metrics_file) as results_file: panoptes_json_data = json.load(results_file) for panoptes_data_object in panoptes_json_data: resource = panoptes_data_object[u'resource'] panoptes_resource = PanoptesResource( resource_site=resource[u'resource_site'], resource_class=resource[u'resource_class'], resource_subclass=resource[u'resource_subclass'], resource_type=resource[u'resource_type'], resource_id=resource[u'resource_id'], resource_endpoint=resource[u'resource_endpoint'], resource_plugin=resource[u'resource_plugin'], resource_creation_timestamp=0) panoptes_metric_group = PanoptesMetricsGroup( resource=panoptes_resource, group_type=panoptes_data_object[u'metrics_group_type'], interval=panoptes_data_object[u'metrics_group_interval'] ) for dimension in panoptes_data_object[u'dimensions']: panoptes_metric_group.add_dimension( PanoptesMetricDimension( name=dimension[u'dimension_name'], value=dimension[u'dimension_value'] ) ) for metric in panoptes_data_object[u'metrics']: panoptes_metric_group.add_metric( PanoptesMetric( metric_name=metric[u'metric_name'], metric_value=metric[u'metric_value'], metric_type=PanoptesMetricType().GAUGE if metric[u'metric_type'] == u'gauge' else PanoptesMetricType().COUNTER, metric_creation_timestamp=metric[u'metric_creation_timestamp'] ) ) panoptes_metric_group_set.add(panoptes_metric_group) return panoptes_metric_group_set
class PanoptesTestPollingPlugin(PanoptesPollingPlugin): name = "Panoptes Test Polling Plugin 2" panoptes_resource = PanoptesResource(resource_site='test', resource_class='test', resource_subclass='test', resource_type='test', resource_id='test', resource_endpoint='test', resource_plugin='test') def run(self, context): metric1 = PanoptesMetric("test", 0.0, PanoptesMetricType.GAUGE) metric_group = PanoptesMetricsGroup(self.panoptes_resource, "Test", _TEST_INTERVAL) metric_group.add_metric(metric1) return metric_group
class PanoptesTestPollingPlugin(PanoptesDiscoveryPlugin): name = "Test Discovery Plugin" resource = PanoptesResource(resource_site='test_site', resource_class='test_class', resource_subclass='test_subclass', resource_type='test_type', resource_id='test_resource_id', resource_endpoint='test_resource_endpoint', resource_plugin='test_resource_plugin', resource_creation_timestamp=0) def run(self, context): panoptes_resource_set = PanoptesResourceSet() panoptes_resource_set.add(self.resource) return panoptes_resource_set
def test_ping_unknown_host(self): self._resource_dict['resource_endpoint'] = 'localhostx' resource = PanoptesResource.resource_from_dict(self._resource_dict) unknown_host_response = "ping: cannot resolve {}: Unknown host".format(self._resource_dict['resource_endpoint']) e = subprocess.CalledProcessError(returncode=None, cmd=None, output=unknown_host_response) # Shouldn't I be mocking this? # When I try: "TypeError: exceptions must be old-style classes or derived from BaseException, not Mock" subprocess.check_output = Mock(side_effect=e) test_fail_d = {"round_trip_stddev": None, "packets_received": None, "execution_time": None, "round_trip_avg": None, "packets_transmitted": None, "packet_loss_pct": None, "round_trip_max": None, "round_trip_min": None} with self.assertRaises(PanoptesPingException): test_unknown_host_ping = PanoptesPingConnectionFactory.get_ping_connection(resource=resource, context=None, count=3, timeout=100) self.assertDictEqual(test_fail_d, test_unknown_host_ping.response)
def test_ping_timeout(self): e = subprocess.CalledProcessError(returncode=None, cmd=None, output=TEST_PING_RESPONSE) # When I try: "TypeError: exceptions must be old-style classes or derived from BaseException, not Mock" subprocess.check_output = Mock(side_effect=e) test_d = {"round_trip_stddev": 0.057, "packets_received": 3, "execution_time": 1.44, "round_trip_avg": 0.120, "packets_transmitted": 3, "packet_loss_pct": 0.0, "round_trip_max": 0.162, "round_trip_min": 0.040} with self.assertRaises(PanoptesPingTimeoutException): resource = PanoptesResource.resource_from_dict(self._resource_dict) test_timeout_ping = PanoptesPingConnectionFactory.get_ping_connection(resource=resource, context=None, count=3, timeout=100) self.assertDictEqual(test_d, json.loads(test_timeout_ping.response)) self.assertEqual(test_timeout_ping.packets_transmitted, 3) self.assertEqual(test_timeout_ping.packets_received, 3) self.assertEqual(test_timeout_ping.execution_time, 1.44) self.assertEqual(test_timeout_ping.round_trip_min, 0.040) self.assertEqual(test_timeout_ping.round_trip_avg, 0.120) self.assertEqual(test_timeout_ping.round_trip_max, 0.162) self.assertEqual(test_timeout_ping.round_trip_stddev, 0.057)
class SNMPPluginTestFramework(object): plugin_class = None path = None snmp_host = '127.0.0.1' snmp_port = 10161 snmp_timeout = 10 snmp_retries = 1 snmp_max_repetitions = 10 snmp_community = 'public' snmp_failure_timeout = 1 resource_id = 'test_id' resource_endpoint = '127.0.0.1' resource_plugin = 'dummy' resource_site = 'test_site' resource_class = 'network' resource_subclass = 'test_subclass' resource_type = 'test_type' resource_backplane = None resource_model = 'model' results_data_file = 'results.json' enrichment_data_file = 'enrichment_data' use_enrichment = True plugin_conf = { 'Core': { 'name': 'Test Plugin', 'module': 'test_plugin' }, 'main': { 'execute_frequency': '60', 'enrichment_ttl': '300', 'resource_filter': 'resource_class = "network"' }, 'snmp': { 'timeout': 10, 'retries': 1, 'non_repeaters': 1, 'max_repetitions': 25, } } def __init__(self, test_name): super(SNMPPluginTestFramework, self).__init__(test_name) self._path = self.path self._plugin_conf = self.plugin_conf self._snmp_host = self.snmp_host self._snmp_port = self.snmp_port self._snmp_timeout = self.snmp_timeout self._snmp_retries = self.snmp_retries self._snmp_max_repetitions = self.snmp_max_repetitions self._snmp_community = self.snmp_community self._snmp_failure_timeout = self.snmp_failure_timeout self._resource_id = self.resource_id self._resource_endpoint = self.resource_endpoint self._resource_plugin = self.resource_plugin self._resource_site = self.resource_site self._resource_class = self.resource_class self._resource_subclass = self.resource_subclass self._resource_type = self.resource_type self._resource_backplane = self.resource_backplane self._resource_model = self.resource_model self._panoptes_context = None self._panoptes_resource = None self._snmp_conf = None self._expected_results = None self._results_data_file = None self._enrichment_data_file = None self._secret_store = None self._plugin_context = None self._plugin_context_bad = None self._snmp_conf_bad = None self._enrichment_kv = None self._enrichment_cache = None def set_panoptes_context(self): panoptes_test_conf_file = os.path.join( os.path.dirname(os.path.dirname(__file__)), 'config_files/test_panoptes_config.ini') self._panoptes_context = PanoptesContext( panoptes_test_conf_file, key_value_store_class_list=[PanoptesEnrichmentCacheKeyValueStore]) def set_panoptes_resource(self): self._panoptes_resource = PanoptesResource( resource_site=self._resource_site, resource_class=self._resource_class, resource_subclass=self._resource_subclass, resource_type=self._resource_type, resource_id=self._resource_id, resource_endpoint=self._resource_endpoint, resource_plugin=self._resource_plugin) self._panoptes_resource.resource_metadata['model'] = 'model' if self._resource_backplane: self._panoptes_resource.add_metadata('backplane', self._resource_backplane) def set_enrichment_cache(self): if self.use_enrichment: self._enrichment_data_file = 'data/' + self.enrichment_data_file self._enrichment_kv = self._panoptes_context.get_kv_store( PanoptesEnrichmentCacheKeyValueStore) enrichment_data_file = os.path.join(os.path.abspath(self.path), self._enrichment_data_file) if self._plugin_conf.get('enrichment'): if self._plugin_conf['enrichment'].get('preload'): self._enrichment_cache = PanoptesEnrichmentCache( self._panoptes_context, self._plugin_conf, self._panoptes_resource) try: with open(enrichment_data_file) as enrichment_data: for line in enrichment_data: key, value = line.strip().split('=>') self._enrichment_kv.set(key, value) except Exception as e: raise IOError( 'Failed to load enrichment data file {}: {}'.format( enrichment_data_file, repr(e))) def set_snmp_conf(self): self._snmp_conf = self._panoptes_context.config_object.snmp_defaults def set_secret_store(self): self._secret_store = create_autospec(PanoptesSecretsStore, instance=True, spec_set=True) self._secret_store.get_by_site.return_value = self._snmp_community def set_plugin_context(self): self._plugin_context = create_autospec( PanoptesPluginWithEnrichmentContext, instance=True, spec_set=True, data=self._panoptes_resource, enrichment=self._enrichment_cache, config=self._plugin_conf, snmp=self._snmp_conf, secrets=self._secret_store, logger=logging.getLogger(__name__)) def set_snmp_conf_bad(self): self._snmp_conf_bad = copy.copy(self._snmp_conf) self._snmp_conf_bad['port'] += 1 self._snmp_conf_bad['timeout'] = self._snmp_failure_timeout def set_plugin_context_bad(self): self._plugin_context_bad = create_autospec( PanoptesPluginWithEnrichmentContext, instance=True, spec_set=True, data=self._panoptes_resource, enrichment=self._enrichment_cache, config=self._plugin_conf, snmp=self._snmp_conf_bad, secrets=self._secret_store, logger=logging.getLogger(__name__)) def set_expected_results(self): self._results_data_file = 'data/' + self.results_data_file expected_result_file = os.path.join(os.path.abspath(self.path), self._results_data_file) self._expected_results = json.load(open(expected_result_file)) @patch('time.time', mock_time) @patch('yahoo_panoptes.framework.resources.time', mock_time) @patch('redis.StrictRedis', PanoptesMockRedis) def setUp(self): self.set_panoptes_context() self.set_panoptes_resource() self.set_enrichment_cache() self.set_snmp_conf() self.set_secret_store() self.set_plugin_context() self.set_snmp_conf_bad() self.set_plugin_context_bad() self.set_expected_results()
def testMetricsGroup(self): now = round(time.time(), METRICS_TIMESTAMP_PRECISION) metrics_group = PanoptesMetricsGroup(self.__panoptes_resource, 'test', 120) self.assertEqual(metrics_group.group_type, 'test') self.assertEqual(metrics_group.interval, 120) self.assertEqual(metrics_group.schema_version, '0.2') self.assertGreaterEqual(metrics_group.creation_timestamp, now) with patch('yahoo_panoptes.framework.metrics.time', mock_time): metrics_group = PanoptesMetricsGroup(self.__panoptes_resource, 'test', 120) dimension_one = PanoptesMetricDimension('if_alias', 'bar') dimension_two = PanoptesMetricDimension('if_alias', 'foo') metrics_group.add_dimension(dimension_one) with self.assertRaises(KeyError): metrics_group.add_dimension(dimension_two) # Test basic dimension operations self.assertEqual(len(metrics_group.dimensions), 1) self.assertTrue( metrics_group.contains_dimension_by_name('if_alias')) self.assertFalse(metrics_group.contains_dimension_by_name('baz')) self.assertEqual( metrics_group.get_dimension_by_name('if_alias').value, 'bar') metrics_group.delete_dimension_by_name('if_alias') self.assertFalse( metrics_group.contains_dimension_by_name('if_alias')) self.assertEqual(len(metrics_group.dimensions), 0) self.assertEqual(metrics_group.get_dimension_by_name('foo'), None) metrics_group.add_dimension(dimension_two) dimension_three = PanoptesMetricDimension('if_alias', 'test') metrics_group.upsert_dimension(dimension_three) self.assertEqual(len(metrics_group.dimensions), 1) self.assertEqual( metrics_group.get_dimension_by_name('if_alias').value, 'test') dimension_four = PanoptesMetricDimension('if_name', 'eth0') metrics_group.upsert_dimension(dimension_four) self.assertEqual(len(metrics_group.dimensions), 2) # Test basic metric operations with self.assertRaises(AssertionError): metrics_group.add_metric(None) metric = PanoptesMetric('test_metric', 0, PanoptesMetricType.GAUGE) metrics_group.add_metric(metric) with self.assertRaises(KeyError): metrics_group.add_metric(metric) to_json = metrics_group.json metrics = PanoptesMetricsGroup.flatten_metrics( json.loads(to_json)['metrics']) self.assertEquals(metrics['gauge']['test_metric']['value'], 0) metrics_group_two = PanoptesMetricsGroup(self.__panoptes_resource, 'test', 120) metrics_group_two.add_dimension(dimension_two) metrics_group_two.upsert_dimension(dimension_three) metrics_group_two.upsert_dimension(dimension_four) metrics_group_two.add_metric(metric) self.assertEqual(metrics_group, metrics_group_two) # Check PanoptesMetricsGroup.__eq__ panoptes_resource_two = PanoptesResource(resource_site='test2', resource_class='test2', resource_subclass='test2', resource_type='test2', resource_id='test2', resource_endpoint='test2', resource_plugin='test2') metrics_group_two = PanoptesMetricsGroup(panoptes_resource_two, 'test', 120) metrics_group_three = PanoptesMetricsGroup( self.__panoptes_resource, 'test', 120) with self.assertRaises(AssertionError): assert metrics_group_two == metrics_group_three metrics_group_three = metrics_group.copy() with self.assertRaises(AssertionError): assert metrics_group == dimension_one assert metrics_group == metrics_group_three metrics_group_three.delete_dimension_by_name("if_name") with self.assertRaises(AssertionError): assert metrics_group == metrics_group_three metrics_group_three.upsert_dimension(dimension_four) assert metrics_group == metrics_group_three metric_two = PanoptesMetric('test_metric_2', 1, PanoptesMetricType.GAUGE) metrics_group_three.add_metric(metric_two) with self.assertRaises(AssertionError): assert metrics_group == metrics_group_three # Test PanoptesMetricsGroup.__repr__ _METRICS_GROUP_REPR = "{{'metrics_group_interval': 120, " \ "'resource': plugin|test|site|test|class|test|subclass|test|type|test|id|test|" \ "endpoint|test, 'dimensions': set([{{'dimension_name': 'if_alias', " \ "'dimension_value': 'test'}}, " \ "{{'dimension_name': 'if_name', 'dimension_value': 'eth0'}}]), " \ "'metrics_group_type': 'test', " \ "'metrics': set([{{'metric_creation_timestamp': {}, " \ "'metric_type': 'gauge', 'metric_name': 'test_metric', 'metric_value': 0}}]), " \ "'metrics_group_creation_timestamp': {}, " \ "'metrics_group_schema_version': '0.2'}}".format(mock_time.return_value, mock_time.return_value) self.assertEqual(repr(metrics_group), _METRICS_GROUP_REPR) dimensions_as_dicts = [{ 'dimension_name': dimension.name, 'dimension_value': dimension.value } for dimension in metrics_group.dimensions] self.assertEqual( PanoptesMetricsGroup.flatten_dimensions(dimensions_as_dicts), { 'if_alias': 'test', 'if_name': 'eth0' })
import logging import os from unittest import TestCase from mock import create_autospec from yahoo_panoptes.framework.context import PanoptesContext from yahoo_panoptes.framework.plugins.context import PanoptesPluginWithEnrichmentContext from yahoo_panoptes.framework.resources import PanoptesResource from yahoo_panoptes.framework.utilities.secrets import PanoptesSecretsStore from yahoo_panoptes.framework.utilities.snmp.connection import PanoptesSNMPPluginConfiguration panoptes_resource = PanoptesResource(resource_site='test_site', resource_class='test_class', resource_subclass='test_subclass', resource_type='test_type', resource_id='test_id', resource_endpoint='test_endpoint', resource_plugin='test_plugin') plugin_conf = { 'Core': { 'name': 'Test Plugin', 'module': 'test_plugin' }, 'main': { 'execute_frequency': '60', 'resource_filter': 'resource_class = "network"' } }
class TestResources(unittest.TestCase): def setUp(self): self.__panoptes_resource_metadata = {'test': 'test', '_resource_ttl': '604800'} self.__panoptes_resource = PanoptesResource(resource_site='test', resource_class='test', resource_subclass='test', resource_type='test', resource_id='test', resource_endpoint='test', resource_plugin='test', resource_creation_timestamp=_TIMESTAMP, resource_ttl=RESOURCE_MANAGER_RESOURCE_EXPIRE) self.__panoptes_resource.add_metadata('test', 'test') self.__panoptes_resource_set = PanoptesResourceSet() mock_valid_timestamp = Mock(return_value=True) with patch('yahoo_panoptes.framework.resources.PanoptesValidators.valid_timestamp', mock_valid_timestamp): self.__panoptes_resource_set.resource_set_creation_timestamp = _TIMESTAMP self.my_dir, self.panoptes_test_conf_file = _get_test_conf_file() def test_panoptes_resource(self): panoptes_resource_metadata = self.__panoptes_resource_metadata panoptes_resource = self.__panoptes_resource self.assertIsInstance(panoptes_resource, PanoptesResource) self.assertEqual(panoptes_resource.resource_site, 'test') self.assertEqual(panoptes_resource.resource_class, 'test') self.assertEqual(panoptes_resource.resource_subclass, 'test') self.assertEqual(panoptes_resource.resource_type, 'test') self.assertEqual(panoptes_resource.resource_id, 'test') self.assertEqual(panoptes_resource.resource_endpoint, 'test') self.assertEqual(panoptes_resource.resource_metadata, panoptes_resource_metadata) self.assertEqual(panoptes_resource.resource_ttl, str(RESOURCE_MANAGER_RESOURCE_EXPIRE)) self.assertEqual(panoptes_resource, panoptes_resource) self.assertFalse(panoptes_resource == '1') self.assertIsInstance(str(panoptes_resource), str) with self.assertRaises(AssertionError): panoptes_resource.add_metadata(None, 'test') with self.assertRaises(AssertionError): panoptes_resource.add_metadata('', 'test') with self.assertRaises(ValueError): panoptes_resource.add_metadata('1', 'test') with self.assertRaises(ValueError): panoptes_resource.add_metadata('key', 'test|') with self.assertRaises(AssertionError): PanoptesResource(resource_site='', resource_class='test', resource_subclass='test', resource_type='test', resource_id='test', resource_endpoint='test') with self.assertRaises(AssertionError): PanoptesResource(resource_site='test', resource_class='', resource_subclass='test', resource_type='test', resource_id='test', resource_endpoint='test') with self.assertRaises(AssertionError): PanoptesResource(resource_site='test', resource_class='test', resource_subclass='', resource_type='test', resource_id='test', resource_endpoint='test') with self.assertRaises(AssertionError): PanoptesResource(resource_site='test', resource_class='test', resource_subclass='test', resource_type='', resource_id='test', resource_endpoint='test') with self.assertRaises(AssertionError): PanoptesResource(resource_site='test', resource_class='test', resource_subclass='test', resource_type='test', resource_id='', resource_endpoint='test') with self.assertRaises(AssertionError): PanoptesResource(resource_site=None, resource_class='test', resource_subclass='test', resource_type='test', resource_id='test', resource_endpoint='test') with self.assertRaises(AssertionError): PanoptesResource(resource_site='test', resource_class=None, resource_subclass='test', resource_type='test', resource_id='test', resource_endpoint='test') with self.assertRaises(AssertionError): PanoptesResource(resource_site='test', resource_class='test', resource_subclass=None, resource_type='test', resource_id='test', resource_endpoint='test') with self.assertRaises(AssertionError): PanoptesResource(resource_site='test', resource_class='test', resource_subclass='test', resource_type=None, resource_id='test', resource_endpoint='test') with self.assertRaises(AssertionError): PanoptesResource(resource_site='test', resource_class='test', resource_subclass='test', resource_type='test', resource_id=None, resource_endpoint='test') with self.assertRaises(AssertionError): PanoptesResource(resource_site='test', resource_class='test', resource_subclass='test', resource_type='test', resource_id='test', resource_endpoint=None) # Test json and raw representations of PanoptesResource panoptes_resource_2 = PanoptesResource(resource_site='test', resource_class='test', resource_subclass='test', resource_type='test', resource_id='test', resource_endpoint='test', resource_creation_timestamp=_TIMESTAMP, resource_plugin='test') self.assertEqual(panoptes_resource_2.resource_creation_timestamp, _TIMESTAMP) panoptes_resource_2_json = { u'resource_site': u'test', u'resource_id': u'test', u'resource_class': u'test', u'resource_plugin': u'test', u'resource_creation_timestamp': _TIMESTAMP, u'resource_subclass': u'test', u'resource_endpoint': u'test', u'resource_metadata': { u'_resource_ttl': u'604800' }, u'resource_type': u'test' } self.assertEqual(ordered(json.loads(panoptes_resource_2.json)), ordered(panoptes_resource_2_json)) panoptes_resource_2_raw = collections.OrderedDict( [('resource_site', 'test'), ('resource_class', 'test'), ('resource_subclass', 'test'), ('resource_type', 'test'), ('resource_id', 'test'), ('resource_endpoint', 'test'), ('resource_metadata', collections.OrderedDict( [('_resource_ttl', '604800')]) ), ('resource_creation_timestamp', _TIMESTAMP), ('resource_plugin', 'test')]) self.assertEqual(panoptes_resource_2.raw, panoptes_resource_2_raw) # Test resource creation from dict with open('tests/test_resources/input/resource_one.json') as f: resource_specs = json.load(f) resource_from_json = PanoptesResource.resource_from_dict(resource_specs['resources'][0]) panoptes_resource_3 = PanoptesResource(resource_site="test_site", resource_class="network", resource_subclass="test_subclass", resource_type="test_type", resource_id="test_id_1", resource_endpoint="test_endpoint_1", resource_plugin="key") self.assertEqual(resource_from_json, panoptes_resource_3) def test_panoptes_resource_set(self): panoptes_resource = self.__panoptes_resource panoptes_resource_set = self.__panoptes_resource_set self.assertEqual(panoptes_resource_set.add(panoptes_resource), None) self.assertEqual(len(panoptes_resource_set), 1) self.assertEqual(type(panoptes_resource_set.resources), set) self.assertIsInstance(str(panoptes_resource_set), str) self.assertIsInstance(iter(panoptes_resource_set), collections.Iterable) self.assertEqual(panoptes_resource_set.next(), panoptes_resource) self.assertEqual(panoptes_resource_set.remove(panoptes_resource), None) self.assertEqual(len(panoptes_resource_set), 0) self.assertEqual(panoptes_resource_set.resource_set_creation_timestamp, _TIMESTAMP) with self.assertRaises(AssertionError): panoptes_resource_set.resource_set_creation_timestamp = 0 panoptes_resource_2 = PanoptesResource(resource_site='test', resource_class='test', resource_subclass='test', resource_type='test', resource_id='test2', resource_endpoint='test', resource_plugin='test', resource_ttl=RESOURCE_MANAGER_RESOURCE_EXPIRE, resource_creation_timestamp=_TIMESTAMP) panoptes_resource_set.add(panoptes_resource) panoptes_resource_set.add(panoptes_resource_2) self.assertEqual(len(panoptes_resource_set.get_resources_by_site()['test']), 2) self.assertEqual(panoptes_resource_set.resource_set_schema_version, "0.1") panoptes_resource_set_json = { u'resource_set_creation_timestamp': _TIMESTAMP, u'resource_set_schema_version': u'0.1', u'resources': [ { u'resource_site': u'test', u'resource_class': u'test', u'resource_subclass': u'test', u'resource_type': u'test', u'resource_id': u'test2', u'resource_endpoint': u'test', u'resource_metadata': { u'_resource_ttl': u'604800' }, u'resource_creation_timestamp': _TIMESTAMP, u'resource_plugin': u'test' }, { u'resource_site': u'test', u'resource_class': u'test', u'resource_subclass': u'test', u'resource_type': u'test', u'resource_id': u'test', u'resource_endpoint': u'test', u'resource_metadata': { u'_resource_ttl': u'604800', u'test': u'test' }, u'resource_creation_timestamp': _TIMESTAMP, u'resource_plugin': u'test' } ] } self.assertEqual(ordered(panoptes_resource_set_json), ordered(json.loads(panoptes_resource_set.json))) def test_panoptes_resources_key_value_store(self): panoptes_context = PanoptesContext(self.panoptes_test_conf_file) panoptes_resources_kv_store = PanoptesResourcesKeyValueStore(panoptes_context) self.assertEqual(panoptes_resources_kv_store.redis_group, const.RESOURCE_MANAGER_REDIS_GROUP) self.assertEqual(panoptes_resources_kv_store.namespace, const.RESOURCE_MANAGER_KEY_VALUE_NAMESPACE) @patch('redis.StrictRedis', panoptes_mock_redis_strict_client) def test_panoptes_resource_store(self): panoptes_context = PanoptesContext(self.panoptes_test_conf_file, key_value_store_class_list=[PanoptesTestKeyValueStore]) with self.assertRaises(Exception): PanoptesResourceStore(panoptes_context) mock_kv_store = Mock(return_value=PanoptesTestKeyValueStore(panoptes_context)) with patch('yahoo_panoptes.framework.resources.PanoptesContext.get_kv_store', mock_kv_store): panoptes_resource_store = PanoptesResourceStore(panoptes_context) panoptes_resource_store.add_resource("test_plugin_signature", self.__panoptes_resource) resource_key = "plugin|test|site|test|class|test|subclass|test|type|test|id|test|endpoint|test" resource_value = panoptes_resource_store.get_resource(resource_key) self.assertEqual(self.__panoptes_resource, resource_value) panoptes_resource_2 = PanoptesResource(resource_site='test', resource_class='test', resource_subclass='test', resource_type='test', resource_id='test2', resource_endpoint='test', resource_plugin='test', resource_ttl=RESOURCE_MANAGER_RESOURCE_EXPIRE, resource_creation_timestamp=_TIMESTAMP) panoptes_resource_store.add_resource("test_plugin_signature", panoptes_resource_2) self.assertIn(self.__panoptes_resource, panoptes_resource_store.get_resources()) self.assertIn(panoptes_resource_2, panoptes_resource_store.get_resources()) panoptes_resource_store.delete_resource("test_plugin_signature", panoptes_resource_2) self.assertNotIn(panoptes_resource_2, panoptes_resource_store.get_resources()) panoptes_resource_3 = PanoptesResource(resource_site='test', resource_class='test', resource_subclass='test', resource_type='test', resource_id='test3', resource_endpoint='test', resource_plugin='test3', resource_ttl=RESOURCE_MANAGER_RESOURCE_EXPIRE, resource_creation_timestamp=_TIMESTAMP) panoptes_resource_store.add_resource("test_plugin_signature", panoptes_resource_3) self.assertIn(panoptes_resource_3, panoptes_resource_store.get_resources(site='test', plugin_name='test3')) self.assertNotIn(self.__panoptes_resource, panoptes_resource_store.get_resources(site='test', plugin_name='test3')) # Test key not found mock_find_keys = Mock( return_value=['dummy', 'plugin|test|site|test|class|test|subclass|test|type|test|id|test|endpoint|test']) with patch('yahoo_panoptes.framework.resources.PanoptesKeyValueStore.find_keys', mock_find_keys): self.assertEqual(1, len(panoptes_resource_store.get_resources())) # Test resource store methods raise correct errors mock_get = Mock(side_effect=Exception) with patch('yahoo_panoptes.framework.resources.PanoptesKeyValueStore.get', mock_get): with self.assertRaises(PanoptesResourceError): panoptes_resource_store.get_resource('test3') # Test bad input with self.assertRaises(AssertionError): panoptes_resource_store.get_resource("") with self.assertRaises(AssertionError): panoptes_resource_store.get_resource(1) with self.assertRaises(AssertionError): panoptes_resource_store.add_resource("", panoptes_resource_2) with self.assertRaises(AssertionError): panoptes_resource_store.add_resource("test_plugin_signature", None) with self.assertRaises(AssertionError): panoptes_resource_store.add_resource("test_plugin_signature", PanoptesResourceStore(panoptes_context)) with self.assertRaises(AssertionError): panoptes_resource_store.delete_resource("", panoptes_resource_2) with self.assertRaises(AssertionError): panoptes_resource_store.delete_resource("test_plugin_signature", None) with self.assertRaises(AssertionError): panoptes_resource_store.delete_resource("test_plugin_signature", PanoptesResourceStore(panoptes_context)) with self.assertRaises(AssertionError): panoptes_resource_store.get_resources("", "test_plugin_name") with self.assertRaises(AssertionError): panoptes_resource_store.get_resources("test_site", "") with self.assertRaises(AssertionError): panoptes_resource_store.get_resources(1, "test_plugin_name") with self.assertRaises(AssertionError): panoptes_resource_store.get_resources("test_site", 1) # Test non-existent key with self.assertRaises(PanoptesResourceError): panoptes_resource_store.get_resource('tes') mock_set = Mock(side_effect=Exception) with patch('yahoo_panoptes.framework.resources.PanoptesKeyValueStore.set', mock_set): with self.assertRaises(PanoptesResourceError): panoptes_resource_store.add_resource("test_plugin_signature", panoptes_resource_2) mock_delete = Mock(side_effect=Exception) with patch('yahoo_panoptes.framework.resources.PanoptesKeyValueStore.delete', mock_delete): with self.assertRaises(PanoptesResourceError): panoptes_resource_store.delete_resource("test_plugin_signature", panoptes_resource_2) with self.assertRaises(PanoptesResourceError): panoptes_resource_store._deserialize_resource("tes", "null") with self.assertRaises(PanoptesResourceError): panoptes_resource_store._deserialize_resource(resource_key, "null") @patch('redis.StrictRedis', panoptes_mock_redis_strict_client) def test_resource_dsl_parsing(self): panoptes_context = PanoptesContext(self.panoptes_test_conf_file) test_query = 'resource_class = "network" AND resource_subclass = "load-balancer"' test_result = ( 'SELECT resources.*, group_concat(key,"|"), group_concat(value,"|") ' + 'FROM resources ' + 'LEFT JOIN resource_metadata ON resources.id = resource_metadata.id ' + 'WHERE (resources.resource_class = "network" ' + 'AND resources.resource_subclass = "load-balancer") ' + 'GROUP BY resource_metadata.id ' + 'ORDER BY resource_metadata.id' ) panoptes_resource_dsl = PanoptesResourceDSL(test_query, panoptes_context) self.assertEqual(panoptes_resource_dsl.sql, test_result) # This very long query tests all code paths with the DSL parser: test_query = 'resource_class = "network" AND resource_subclass = "load-balancer" OR \ resource_metadata.os_version LIKE "4%" AND resource_site NOT IN ("test_site") \ AND resource_endpoint IN ("test1","test2") AND resource_type != "a10" OR ' \ 'resource_metadata.make NOT LIKE \ "A10%" AND resource_metadata.model NOT IN ("test1", "test2")' test_result = ( 'SELECT resources.*,group_concat(key,"|"),group_concat(value,"|") FROM (SELECT resource_metadata.id ' + 'FROM resources,resource_metadata WHERE (resources.resource_class = "network" ' + 'AND resources.resource_subclass = "load-balancer" ' + 'AND resources.resource_site NOT IN ("test_site") ' + 'AND resources.resource_endpoint IN ("test1","test2") ' + 'AND resources.resource_type != "a10" ' + 'AND ((resource_metadata.key = "os_version" ' + 'AND resource_metadata.value LIKE "4%")) ' + 'AND resource_metadata.id = resources.id) ' + 'UNION SELECT resource_metadata.id ' + 'FROM resources,resource_metadata WHERE (resource_metadata.key = "make" ' + 'AND resource_metadata.value NOT LIKE "A10%") ' + 'AND resource_metadata.id = resources.id ' + 'INTERSECT SELECT resource_metadata.id ' + 'FROM resources,resource_metadata WHERE (resource_metadata.key = "model" ' + 'AND resource_metadata.value NOT IN ("test1","test2")) ' + 'AND resource_metadata.id = resources.id ' + 'GROUP BY resource_metadata.id ' + 'ORDER BY resource_metadata.id) AS filtered_resources, ' + 'resources, resource_metadata WHERE resources.id = filtered_resources.id ' + 'AND resource_metadata.id = filtered_resources.id GROUP BY resource_metadata.id') panoptes_resource_dsl = PanoptesResourceDSL(test_query, panoptes_context) self.assertEqual(panoptes_resource_dsl.sql, test_result) panoptes_resource_dsl = PanoptesResourceDSL('resource_site = "local"', panoptes_context) self.assertIsInstance(panoptes_resource_dsl.tokens, ParseResults) with self.assertRaises(AssertionError): PanoptesResourceDSL(None, panoptes_context) with self.assertRaises(AssertionError): PanoptesResourceDSL('', None) with self.assertRaises(AssertionError): PanoptesResourceDSL('', panoptes_context) with self.assertRaises(ParseException): PanoptesResourceDSL('resources_site = local', panoptes_context)
def test_multi_enrichment_group_set(self): interface_validation_object = PanoptesEnrichmentInterfaceSchemaValidator() neighbor_validation_object = PanoptesEnrichmentNeighborSchemaValidator() panoptes_resource = self.__panoptes_resource multi_enrichment_results_data = \ { "group_sets": [ { "enrichment": [ { "data": [ { "host_name": { "mac": "aa:bb:cc:dd:ee:ff", "property": "Test Property", "vlan_id": 501 } }, { "host_name01": { "mac": "aa:bb:cc:dd:ee:ff", "property": "Test Property", "vlan_id": 502 } } ], "metadata": { "_enrichment_group_creation_timestamp": mock_time.return_value, "_enrichment_ttl": 600, "_execute_frequency": 120 }, "namespace": "neighbor" }, { "data": [ { "int_001": { "index": 1, "speed": 1000, "status": "up" } } ], "metadata": { "_enrichment_group_creation_timestamp": mock_time.return_value, "_enrichment_ttl": 300, "_execute_frequency": 60 }, "namespace": "interface" } ], "enrichment_group_set_creation_timestamp": mock_time.return_value, "resource": { "resource_class": "test_class", "resource_creation_timestamp": mock_time.return_value, "resource_endpoint": "test_endpoint01", "resource_id": "test_resource_id01", "resource_metadata": { "_resource_ttl": "604800" }, "resource_plugin": "test_plugin", "resource_site": "test_site", "resource_subclass": "test_subclass", "resource_type": "test_type" } }, { "enrichment": [ { "data": [ { "int_001": { "index": 1, "speed": 1000, "status": "up" } } ], "metadata": { "_enrichment_group_creation_timestamp": mock_time.return_value, "_enrichment_ttl": 300, "_execute_frequency": 60 }, "namespace": "interface" }, { "data": [ { "host_name": { "mac": "aa:bb:cc:dd:ee:ff", "property": "Test Property", "vlan_id": 501 } } ], "metadata": { "_enrichment_group_creation_timestamp": mock_time.return_value, "_enrichment_ttl": 600, "_execute_frequency": 120 }, "namespace": "neighbor" } ], "enrichment_group_set_creation_timestamp": mock_time.return_value, "resource": { "resource_class": "test", "resource_creation_timestamp": mock_time.return_value, "resource_endpoint": "test", "resource_id": "test", "resource_metadata": { "_resource_ttl": "604800", "test": "test" }, "resource_plugin": "test", "resource_site": "test", "resource_subclass": "test", "resource_type": "test" } } ] } enrichment_set1 = PanoptesEnrichmentSet(u'int_001') enrichment_set1.add(u'speed', 1000) enrichment_set1.add(u'index', 0o01) enrichment_set1.add(u'status', u'up') enrichment_group1 = PanoptesEnrichmentGroup(u'interface', interface_validation_object, 300, 60) enrichment_group1.add_enrichment_set(enrichment_set1) enrichment_set3 = PanoptesEnrichmentSet(u'host_name') enrichment_set3.add(u'vlan_id', 501) enrichment_set3.add(u'property', u'Test Property') enrichment_set3.add(u'mac', u'aa:bb:cc:dd:ee:ff') enrichment_group2 = PanoptesEnrichmentGroup(u'neighbor', neighbor_validation_object, 600, 120) enrichment_group2.add_enrichment_set(enrichment_set3) enrichment_group_set1 = PanoptesEnrichmentGroupSet(panoptes_resource) enrichment_group_set1.add_enrichment_group(enrichment_group1) enrichment_group_set1.add_enrichment_group(enrichment_group2) panoptes_resource01 = PanoptesResource(resource_site=u'test_site', resource_class=u'test_class', resource_subclass=u'test_subclass', resource_type=u'test_type', resource_id=u'test_resource_id01', resource_endpoint=u'test_endpoint01', resource_plugin=u'test_plugin') panoptes_resource02 = PanoptesResource(resource_site=u'test_site', resource_class=u'test_class', resource_subclass=u'test_subclass', resource_type=u'test_type', resource_id=u'test_resource_id02', resource_endpoint=u'test_endpoint02', resource_plugin=u'test_plugin') enrichment_set4 = PanoptesEnrichmentSet(u'host_name01') enrichment_set4.add(u'vlan_id', 502) enrichment_set4.add(u'property', u'Test Property') enrichment_set4.add(u'mac', u'aa:bb:cc:dd:ee:ff') enrichment_group3 = PanoptesEnrichmentGroup(u'neighbor', neighbor_validation_object, 600, 120) enrichment_group3.add_enrichment_set(enrichment_set3) enrichment_group3.add_enrichment_set(enrichment_set4) enrichment_group_set2 = PanoptesEnrichmentGroupSet(panoptes_resource01) enrichment_group_set2.add_enrichment_group(enrichment_group1) enrichment_group_set2.add_enrichment_group(enrichment_group3) multi_enrichment_group_set = PanoptesEnrichmentMultiGroupSet() multi_enrichment_group_set.add_enrichment_group_set(enrichment_group_set1) multi_enrichment_group_set.add_enrichment_group_set(enrichment_group_set2) multi_enrichment_group_set_repr = u"PanoptesEnrichmentMultiGroupSet[PanoptesEnrichmentGroupSet[resource:" \ u"plugin|test|site|test|class|test|subclass|test|type|test|id|test|endpoint" \ u"|test,enrichment_group_set_creation_timestamp:{},PanoptesEnrichmentGroup" \ u"[namespace:interface,enrichment_ttl:300,execute_frequency:60," \ u"enrichment_group_creation_timestamp:{},PanoptesEnrichmentSet[" \ u"int_001[index:1,speed:1000,status:up]]],PanoptesEnrichmentGroup[" \ u"namespace:neighbor,enrichment_ttl:600,execute_frequency:120," \ u"enrichment_group_creation_timestamp:{},PanoptesEnrichmentSet" \ u"[host_name[mac:aa:bb:cc:dd:ee:ff,property:Test Property,vlan_id:501]" \ u"]]],PanoptesEnrichmentGroupSet[resource:plugin|test_plugin|site|" \ u"test_site|class|test_class|subclass|test_subclass|type|test_type|id" \ u"|test_resource_id01|endpoint|test_endpoint01," \ u"enrichment_group_set_creation_timestamp:{},PanoptesEnrichmentGroup" \ u"[namespace:interface,enrichment_ttl:300,execute_frequency:60," \ u"enrichment_group_creation_timestamp:{},PanoptesEnrichmentSet" \ u"[int_001[index:1,speed:1000,status:up]]],PanoptesEnrichmentGroup" \ u"[namespace:neighbor,enrichment_ttl:600,execute_frequency:120," \ u"enrichment_group_creation_timestamp:{},PanoptesEnrichmentSet" \ u"[host_name[mac:aa:bb:cc:dd:ee:ff,property:Test Property,vlan_id:501]]," \ u"PanoptesEnrichmentSet[host_name01[mac:aa:bb:cc:dd:ee:ff," \ u"property:Test Property,vlan_id:502]]]]]".format(mock_time.return_value, mock_time.return_value, mock_time.return_value, mock_time.return_value, mock_time.return_value, mock_time.return_value) self.assertEquals(repr(multi_enrichment_group_set), multi_enrichment_group_set_repr) self.assertEquals(len(multi_enrichment_group_set.enrichment_group_sets), 2) self.assertEquals(ordered(json.loads(multi_enrichment_group_set.json())), ordered(multi_enrichment_results_data)) self.assertEquals(len(multi_enrichment_group_set), 2) multi_enrichment_group_set.add_enrichment_group_set(enrichment_group_set2) self.assertEquals(len(multi_enrichment_group_set), 2) enrichment_group_set3 = PanoptesEnrichmentGroupSet(panoptes_resource02) enrichment_group_set3.add_enrichment_group(enrichment_group1) multi_enrichment_group_set.add_enrichment_group_set(enrichment_group_set3) self.assertEquals(len(multi_enrichment_group_set), 3) with self.assertRaises(AssertionError): multi_enrichment_group_set.add_enrichment_group_set(u'non_enrichment_group') enrichment_group_set3 = PanoptesEnrichmentGroupSet(panoptes_resource01) with self.assertRaises(AssertionError): multi_enrichment_group_set.add_enrichment_group_set(enrichment_group_set3)
class TestValidators(unittest.TestCase): def setUp(self): self._panoptes_metric = { u'metrics_group_interval': 60, u'resource': { u'resource_site': u'test_site', u'resource_id': u'test_id', u'resource_class': u'network', u'resource_plugin': u'test_plugin', u'resource_creation_timestamp': 1567823517.46, u'resource_subclass': u'test_subclass', u'resource_endpoint': u'test_endpoint', u'resource_metadata': { u'test_metadata_key': u'test_metadata_value', u'_resource_ttl': u"604800" }, u'resource_type': u'test_type' }, u'dimensions': [{ u'dimension_name': u'cpu_name', u'dimension_value': u'test_cpu_name_value' }, { u'dimension_name': u'cpu_no', u'dimension_value': u'test_cpu_no_value' }, { u'dimension_name': u'cpu_type', u'dimension_value': u'test_cpu_type_value' }], u'metrics_group_type': u'cpu', u'metrics': [{ u'metric_creation_timestamp': 1567823946.72, u'metric_type': u'gauge', u'metric_name': u'cpu_utilization', u'metric_value': 0 }], u'metrics_group_creation_timestamp': 1567823946.72, u'metrics_group_schema_version': u'0.2' } self._panoptes_resource = PanoptesResource(resource_site=u'test', resource_class=u'test', resource_subclass=u'test', resource_type=u'test', resource_id=u'test', resource_endpoint=u'test', resource_plugin=u'test') self._panoptes_resource.add_metadata(u'test', u'test') self._panoptes_resource_set = PanoptesResourceSet() self._panoptes_resource_set.add(self._panoptes_resource) def test_default_record(self): self.assertTrue( PanoptesConsumerRecordValidator.validate( PanoptesConsumerTypes.METRICS, self._panoptes_metric)) self.assertTrue( PanoptesConsumerRecordValidator.validate( PanoptesConsumerTypes.PROCESSED, self._panoptes_metric)) self.assertTrue( PanoptesConsumerRecordValidator.validate( PanoptesConsumerTypes.RESOURCES, json.loads(self._panoptes_resource_set.json))) self.assertFalse(PanoptesConsumerRecordValidator.validate(5, {})) def test_resource_throws(self): argument_overrides = [(u'resource_site', 1), (u'resource_id', None), (u'resource_class', 1.5), (u'resource_plugin', {}), (u'resource_creation_timestamp', u'1567823517'), (u'resource_subclass', []), (u'resource_endpoint', 123456.789)] for (override_key, override_value) in argument_overrides: schema_arguments = self._panoptes_metric.copy() schema_arguments[u'resource'][override_key] = override_value self.assertEquals( PanoptesConsumerRecordValidator.validate_metrics( schema_arguments), False)
def handle_resources(plugin_signature, resources): resource_site = str(resources[u'resources'][0][u'resource_site']) plugin_name = str(plugin_signature.split(':')[0]) logger.info(u'For plugin "%s" and site "%s", going to get current set' % (plugin_name, resource_site)) try: current_resource_set = resource_store.get_resources( site=resource_site, plugin_name=plugin_name) except: logger.error(u'Error trying to get current resources for plugin "%s"' % plugin_signature) return False logger.info(u'For plugin "%s" and site "%s" current set has %d resources' % (plugin_signature, resource_site, len(current_resource_set))) new_resource_set = PanoptesResourceSet() for resource in resources[u'resources']: new_resource = PanoptesResource.resource_from_dict(resource) new_resource_set.add(new_resource) current_resource_dict = _resource_set_to_dictionary(current_resource_set) new_resource_dict = _resource_set_to_dictionary(new_resource_set) logger.info(u'For plugin "%s" and site "%s", new set has %d resources' % (plugin_signature, resource_site, len(new_resource_set))) resources_to_delete = current_resource_set.resources.difference( new_resource_set) logger.info(u'For plugin "%s" and site "%s", deleting %d resources' % (plugin_signature, resource_site, len(resources_to_delete))) for resource in resources_to_delete: current_timestamp = current_resource_dict[ resource.serialization_key].resource_creation_timestamp new_timestamp = resources[u'resource_set_creation_timestamp'] if current_timestamp > new_timestamp: logger.info( u'For plugin "%s" and site "%s", resource "%s" has timestamp (%0.2f UTC) greater than of ' u'new set (%0.2f UTC), skipping deletion' % (plugin_signature, resource_site, str(resource), current_timestamp, new_timestamp)) else: logger.info( u'For plugin "%s" and site "%s", going to delete resource "%s"' % (plugin_signature, resource_site, resource)) try: resource_store.delete_resource(plugin_signature, resource) except: logger.exception(u'Error trying to delete resource') return False resources_to_add = new_resource_set.resources.difference( current_resource_set) logger.info(u'For plugin "%s" and site "%s", adding %d resources' % (plugin_signature, resource_site, len(resources_to_add))) for resource in resources_to_add: logger.debug(u'Going to add resource "%s"' % resource) try: resource_store.add_resource(plugin_signature, resource) except: logger.exception(u'Error trying to add resource') return False resources_to_update = current_resource_set.resources.intersection( new_resource_set) logger.info(u'For plugin "%s" and site "%s", updating %d resources' % (plugin_signature, resource_site, len(resources_to_update))) start_time = time.time() resources_skipped = 0 resources_updated = 0 for resource in resources_to_update: current_timestamp = current_resource_dict[ resource.serialization_key].resource_creation_timestamp new_timestamp = new_resource_dict[ resource.serialization_key].resource_creation_timestamp if current_timestamp > new_timestamp: logger.info( u'For plugin "%s" and site "%s", resource "%s" has timestamp (%0.2f UTC) greater than of ' u'new set (%0.2f UTC), skipping update' % (plugin_signature, resource_site, str(resource), current_timestamp, new_timestamp)) resources_skipped += 1 else: try: new_resource = new_resource_dict[resource.serialization_key] logger.debug(u'Going to update resource "%s"' % new_resource) resource_store.add_resource(plugin_signature, new_resource) resources_updated += 1 except: logger.exception(u'Error trying to add resource') return False end_time = time.time() logger.info( u'For plugin "%s" and site "%s", added/updated %d resources in %0.2f seconds, skipped %d resources' % (plugin_signature, resource_site, resources_updated, round(end_time - start_time, 2), resources_skipped)) return True
def testMetricsGroup(self): now = round(time.time(), METRICS_TIMESTAMP_PRECISION) metrics_group = PanoptesMetricsGroup(self.__panoptes_resource, u'test', 120) self.assertEqual(metrics_group.group_type, u'test') self.assertEqual(metrics_group.interval, 120) self.assertEqual(metrics_group.schema_version, u'0.2') self.assertGreaterEqual(metrics_group.creation_timestamp, now) with patch(u'yahoo_panoptes.framework.metrics.time', mock_time): metrics_group = PanoptesMetricsGroup(self.__panoptes_resource, u'test', 120) dimension_one = PanoptesMetricDimension(u'if_alias', u'bar') dimension_two = PanoptesMetricDimension(u'if_alias', u'foo') metrics_group.add_dimension(dimension_one) with self.assertRaises(KeyError): metrics_group.add_dimension(dimension_two) # Test basic dimension operations self.assertEqual(len(metrics_group.dimensions), 1) self.assertTrue( metrics_group.contains_dimension_by_name(u'if_alias')) self.assertFalse(metrics_group.contains_dimension_by_name(u'baz')) self.assertEqual( metrics_group.get_dimension_by_name(u'if_alias').value, u'bar') metrics_group.delete_dimension_by_name(u'if_alias') self.assertFalse( metrics_group.contains_dimension_by_name(u'if_alias')) self.assertEqual(len(metrics_group.dimensions), 0) self.assertEqual(metrics_group.get_dimension_by_name(u'foo'), None) metrics_group.add_dimension(dimension_two) dimension_three = PanoptesMetricDimension(u'if_alias', u'test') metrics_group.upsert_dimension(dimension_three) self.assertEqual(len(metrics_group.dimensions), 1) self.assertEqual( metrics_group.get_dimension_by_name(u'if_alias').value, u'test') dimension_four = PanoptesMetricDimension(u'if_name', u'eth0') metrics_group.upsert_dimension(dimension_four) self.assertEqual(len(metrics_group.dimensions), 2) # Test basic metric operations with self.assertRaises(AssertionError): metrics_group.add_metric(None) metric = PanoptesMetric(u'test_metric', 0, PanoptesMetricType.GAUGE) metrics_group.add_metric(metric) with self.assertRaises(KeyError): metrics_group.add_metric(metric) to_json = metrics_group.json metrics = PanoptesMetricsGroup.flatten_metrics( json.loads(to_json)[u'metrics']) self.assertEquals(metrics[u'gauge'][u'test_metric'][u'value'], 0) metrics_group_two = PanoptesMetricsGroup(self.__panoptes_resource, u'test', 120) metrics_group_two.add_dimension(dimension_two) metrics_group_two.upsert_dimension(dimension_three) metrics_group_two.upsert_dimension(dimension_four) metrics_group_two.add_metric(metric) self.assertEqual(metrics_group, metrics_group_two) # Check PanoptesMetricsGroup.__eq__ panoptes_resource_two = PanoptesResource( resource_site=u'test2', resource_class=u'test2', resource_subclass=u'test2', resource_type=u'test2', resource_id=u'test2', resource_endpoint=u'test2', resource_plugin=u'test2') metrics_group_two = PanoptesMetricsGroup(panoptes_resource_two, u'test', 120) metrics_group_three = PanoptesMetricsGroup( self.__panoptes_resource, u'test', 120) with self.assertRaises(AssertionError): assert metrics_group_two == metrics_group_three metrics_group_three = metrics_group.copy() with self.assertRaises(AssertionError): assert metrics_group == dimension_one assert metrics_group == metrics_group_three metrics_group_three.delete_dimension_by_name(u"if_name") with self.assertRaises(AssertionError): assert metrics_group == metrics_group_three metrics_group_three.upsert_dimension(dimension_four) assert metrics_group == metrics_group_three metric_two = PanoptesMetric(u'test_metric_2', 1, PanoptesMetricType.GAUGE) metrics_group_three.add_metric(metric_two) with self.assertRaises(AssertionError): assert metrics_group == metrics_group_three # Test PanoptesMetricsGroup.__repr__ _METRICS_GROUP_REPR = u'PanoptesMetricsGroup[' \ u'resource:plugin|test|site|test|class|test|subclass|test|type|test|id|' \ u'test|endpoint|test,' \ u'interval:120,schema_version:0.2,group_type:test,creation_timestamp:{},' \ u'dimensions:[PanoptesMetricDimension[if_alias|test],' \ u'PanoptesMetricDimension[if_name|eth0]],' \ u'metrics:[PanoptesMetric[test_metric|0|GAUGE|{}]]]'.format(mock_time.return_value, mock_time.return_value) self.assertEqual(repr(metrics_group), _METRICS_GROUP_REPR) dimensions_as_dicts = [{ u'dimension_name': dimension.name, u'dimension_value': dimension.value } for dimension in metrics_group.dimensions] self.assertEqual( PanoptesMetricsGroup.flatten_dimensions(dimensions_as_dicts), { u'if_alias': u'test', u'if_name': u'eth0' })
class TestMetrics(unittest.TestCase): def setUp(self): self.__panoptes_resource = PanoptesResource(resource_site=u'test', resource_class=u'test', resource_subclass=u'test', resource_type=u'test', resource_id=u'test', resource_endpoint=u'test', resource_plugin=u'test') self.__panoptes_resource.add_metadata(u'test', u'test') def test_panoptes_metric(self): with self.assertRaises(AssertionError): PanoptesMetric(None, 0, PanoptesMetricType.GAUGE) with self.assertRaises(ValueError): PanoptesMetric(u'1', 0, PanoptesMetricType.GAUGE) with self.assertRaises(AssertionError): PanoptesMetric(u'test_metric', None, PanoptesMetricType.GAUGE) with self.assertRaises(AssertionError): PanoptesMetric(u'test_metric', 0, None) with self.assertRaises(AssertionError): PanoptesMetric(u'test_metric', True, PanoptesMetricType.GAUGE) metric1 = PanoptesMetric( u'test_metric', 0, PanoptesMetricType.GAUGE, metric_creation_timestamp=mock_time.return_value) self.assertEqual(metric1.metric_name, u'test_metric') self.assertEqual(metric1.metric_value, 0) self.assertEqual(metric1.metric_timestamp, mock_time.return_value) self.assertEqual(metric1.metric_type, PanoptesMetricType.GAUGE) self.assertEqual( repr(metric1), u"PanoptesMetric[test_metric|0|GAUGE|{}]".format( mock_time.return_value)) self.assertNotEqual(metric1, None) # Check PanoptesMetric.__eq__ assert metric1 == PanoptesMetric(u'test_metric', 0, PanoptesMetricType.GAUGE) with self.assertRaises(AssertionError): assert metric1 == PanoptesMetricDimension(u"test", u"value") with self.assertRaises(AssertionError): assert metric1 == PanoptesMetric(u'different_name', 0, PanoptesMetricType.GAUGE) with self.assertRaises(AssertionError): assert metric1 == PanoptesMetric(u'test_metric', 1, PanoptesMetricType.GAUGE) with self.assertRaises(AssertionError): assert metric1 == PanoptesMetric(u'test_metric', 0, PanoptesMetricType.COUNTER) @patch(u'yahoo_panoptes.framework.metrics.time', mock_time) def test_panoptes_metric_json_and_repr(self): metric = PanoptesMetric(u'test_metric', 0, PanoptesMetricType.GAUGE, mock_time.return_value) serialized = json.loads(metric.json) expected = { u"metric_creation_timestamp": mock_time.return_value, u"metric_name": u"test_metric", u"metric_type": u"gauge", u"metric_value": 0 } self.assertEqual(ordered(serialized), ordered(expected)) def testMetricsGroup(self): now = round(time.time(), METRICS_TIMESTAMP_PRECISION) metrics_group = PanoptesMetricsGroup(self.__panoptes_resource, u'test', 120) self.assertEqual(metrics_group.group_type, u'test') self.assertEqual(metrics_group.interval, 120) self.assertEqual(metrics_group.schema_version, u'0.2') self.assertGreaterEqual(metrics_group.creation_timestamp, now) with patch(u'yahoo_panoptes.framework.metrics.time', mock_time): metrics_group = PanoptesMetricsGroup(self.__panoptes_resource, u'test', 120) dimension_one = PanoptesMetricDimension(u'if_alias', u'bar') dimension_two = PanoptesMetricDimension(u'if_alias', u'foo') metrics_group.add_dimension(dimension_one) with self.assertRaises(KeyError): metrics_group.add_dimension(dimension_two) # Test basic dimension operations self.assertEqual(len(metrics_group.dimensions), 1) self.assertTrue( metrics_group.contains_dimension_by_name(u'if_alias')) self.assertFalse(metrics_group.contains_dimension_by_name(u'baz')) self.assertEqual( metrics_group.get_dimension_by_name(u'if_alias').value, u'bar') metrics_group.delete_dimension_by_name(u'if_alias') self.assertFalse( metrics_group.contains_dimension_by_name(u'if_alias')) self.assertEqual(len(metrics_group.dimensions), 0) self.assertEqual(metrics_group.get_dimension_by_name(u'foo'), None) metrics_group.add_dimension(dimension_two) dimension_three = PanoptesMetricDimension(u'if_alias', u'test') metrics_group.upsert_dimension(dimension_three) self.assertEqual(len(metrics_group.dimensions), 1) self.assertEqual( metrics_group.get_dimension_by_name(u'if_alias').value, u'test') dimension_four = PanoptesMetricDimension(u'if_name', u'eth0') metrics_group.upsert_dimension(dimension_four) self.assertEqual(len(metrics_group.dimensions), 2) # Test basic metric operations with self.assertRaises(AssertionError): metrics_group.add_metric(None) metric = PanoptesMetric(u'test_metric', 0, PanoptesMetricType.GAUGE) metrics_group.add_metric(metric) with self.assertRaises(KeyError): metrics_group.add_metric(metric) to_json = metrics_group.json metrics = PanoptesMetricsGroup.flatten_metrics( json.loads(to_json)[u'metrics']) self.assertEquals(metrics[u'gauge'][u'test_metric'][u'value'], 0) metrics_group_two = PanoptesMetricsGroup(self.__panoptes_resource, u'test', 120) metrics_group_two.add_dimension(dimension_two) metrics_group_two.upsert_dimension(dimension_three) metrics_group_two.upsert_dimension(dimension_four) metrics_group_two.add_metric(metric) self.assertEqual(metrics_group, metrics_group_two) # Check PanoptesMetricsGroup.__eq__ panoptes_resource_two = PanoptesResource( resource_site=u'test2', resource_class=u'test2', resource_subclass=u'test2', resource_type=u'test2', resource_id=u'test2', resource_endpoint=u'test2', resource_plugin=u'test2') metrics_group_two = PanoptesMetricsGroup(panoptes_resource_two, u'test', 120) metrics_group_three = PanoptesMetricsGroup( self.__panoptes_resource, u'test', 120) with self.assertRaises(AssertionError): assert metrics_group_two == metrics_group_three metrics_group_three = metrics_group.copy() with self.assertRaises(AssertionError): assert metrics_group == dimension_one assert metrics_group == metrics_group_three metrics_group_three.delete_dimension_by_name(u"if_name") with self.assertRaises(AssertionError): assert metrics_group == metrics_group_three metrics_group_three.upsert_dimension(dimension_four) assert metrics_group == metrics_group_three metric_two = PanoptesMetric(u'test_metric_2', 1, PanoptesMetricType.GAUGE) metrics_group_three.add_metric(metric_two) with self.assertRaises(AssertionError): assert metrics_group == metrics_group_three # Test PanoptesMetricsGroup.__repr__ _METRICS_GROUP_REPR = u'PanoptesMetricsGroup[' \ u'resource:plugin|test|site|test|class|test|subclass|test|type|test|id|' \ u'test|endpoint|test,' \ u'interval:120,schema_version:0.2,group_type:test,creation_timestamp:{},' \ u'dimensions:[PanoptesMetricDimension[if_alias|test],' \ u'PanoptesMetricDimension[if_name|eth0]],' \ u'metrics:[PanoptesMetric[test_metric|0|GAUGE|{}]]]'.format(mock_time.return_value, mock_time.return_value) self.assertEqual(repr(metrics_group), _METRICS_GROUP_REPR) dimensions_as_dicts = [{ u'dimension_name': dimension.name, u'dimension_value': dimension.value } for dimension in metrics_group.dimensions] self.assertEqual( PanoptesMetricsGroup.flatten_dimensions(dimensions_as_dicts), { u'if_alias': u'test', u'if_name': u'eth0' }) def test_panoptes_metrics_group_encoder(self): test_dict = dict() encoder = PanoptesMetricsGroupEncoder() mock_default = Mock(json.JSONEncoder.default) with patch( u'yahoo_panoptes.framework.metrics.json.JSONEncoder.default', mock_default): encoder.default(test_dict) mock_default.assert_called_once() def test_panoptes_metric_dimension(self): with self.assertRaises(ValueError): PanoptesMetricDimension(u'contain$_invalid_character$', u'bar') with self.assertRaises(ValueError): PanoptesMetricDimension(u'foo', u'contains_pipe|') dimension_one = PanoptesMetricDimension(u'if_alias', u'bar') self.assertEqual( dimension_one.json, u'{"dimension_name": "if_alias", "dimension_value": "bar"}') self.assertEqual(repr(dimension_one), u'PanoptesMetricDimension[if_alias|bar]') metric_one = PanoptesMetric( u'test_metric', 0, PanoptesMetricType.GAUGE, metric_creation_timestamp=mock_time.return_value) with self.assertRaises(AssertionError): assert dimension_one == metric_one dimension_two = PanoptesMetricDimension(u'if_alias', u'foo') with self.assertRaises(AssertionError): assert dimension_one == dimension_two dimension_three = PanoptesMetricDimension(u'if_alias', u'bar') assert dimension_one == dimension_three def test_metrics_group_hash(self): now = round(time.time(), METRICS_TIMESTAMP_PRECISION) metrics_group = PanoptesMetricsGroup(self.__panoptes_resource, u'test', 120) metrics_group_two = PanoptesMetricsGroup(self.__panoptes_resource, u'test', 120) dimension = PanoptesMetricDimension(u'if_alias', u'bar') metric = PanoptesMetric(u'test_metric', 0, PanoptesMetricType.GAUGE, metric_creation_timestamp=now) metric_diff_timestamp = PanoptesMetric(u'test_metric', 0, PanoptesMetricType.GAUGE, metric_creation_timestamp=now + 0.01) metrics_group.add_dimension(dimension) metrics_group_two.add_dimension(dimension) self.assertEqual(metrics_group.__hash__(), metrics_group_two.__hash__()) metrics_group.add_metric(metric) metrics_group_two.add_metric(metric_diff_timestamp) self.assertEqual(metrics_group.__hash__(), metrics_group_two.__hash__()) @patch(u'yahoo_panoptes.framework.metrics.time', mock_time) def test_panoptes_metrics_group_set(self): """Tests basic PanoptesMetricsGroupSet operations""" metrics_group_set = PanoptesMetricsGroupSet() metrics_group = PanoptesMetricsGroup(self.__panoptes_resource, u'test', 120) metrics_group_two = PanoptesMetricsGroup(self.__panoptes_resource, u'test', 120) metrics_group_set.add(metrics_group) metrics_group_set.add(metrics_group_two) assert len(metrics_group_set) == 1 self.assertIn(metrics_group, metrics_group_set.metrics_groups) metrics_group_set.remove(metrics_group_two) assert len(metrics_group_set) == 0 metrics_group_set.add(metrics_group) metrics_group_three = PanoptesMetricsGroup(self.__panoptes_resource, u'test3', 120) metrics_group_three.add_metric( PanoptesMetric(u"test3", 0.0, PanoptesMetricType.GAUGE)) metrics_group_set.add(metrics_group_three) assert len(metrics_group_set) == 2 metrics_group_set_two = PanoptesMetricsGroupSet() metrics_group_four = PanoptesMetricsGroup(self.__panoptes_resource, u'test', 120) metrics_group_four.add_metric( PanoptesMetric(u"test4", 0.0, PanoptesMetricType.GAUGE)) metrics_group_set_two.add(metrics_group_four) assert len(metrics_group_set_two) == 1 # Test PanoptesMetricsGroupSet.__add__ metrics_group_set_union = metrics_group_set + metrics_group_set_two assert len(metrics_group_set_union) == 3 with self.assertRaises(AssertionError): metrics_group_set.remove(self.__panoptes_resource) with self.assertRaises(TypeError): metrics_group_set + metrics_group # Test PanoptesMetricsGroupSet.__iter__ & 'next' metrics_group_count = 0 metrics_group_set_union_interator = iter(metrics_group_set_union) for _ in metrics_group_set_union: self.assertIn(next(metrics_group_set_union_interator), metrics_group_set_union.metrics_groups) metrics_group_count += 1 assert len(metrics_group_set_union) == metrics_group_count with self.assertRaises(Exception): next(metrics_group_set_union_interator) # Test PanoptesMetricsGroupSet.__repr__ _METRICS_GROUP_SET_REPR = u"PanoptesMetricsGroupSet[PanoptesMetricsGroup[resource:" \ u"plugin|test|site|test|class|test|subclass|test|type|test|id|test|endpoint|test," \ u"interval:120,schema_version:0.2,group_type:test,creation_timestamp:{}," \ u"dimensions:[],metrics:[]],PanoptesMetricsGroup[resource:" \ u"plugin|test|site|test|class|test|subclass|test|type|test|id|test|endpoint|test," \ u"interval:120,schema_version:0.2,group_type:test3,creation_timestamp:{}," \ u"dimensions:[],metrics:[" \ u"PanoptesMetric[test3|0.0|GAUGE|{}]]]]".format(mock_time.return_value, mock_time.return_value, mock_time.return_value) self.assertEqual(repr(metrics_group_set), _METRICS_GROUP_SET_REPR) def test_panoptes_metric_set(self): metric_set = PanoptesMetricSet() metric1 = PanoptesMetric( u'test_metric', 0, PanoptesMetricType.GAUGE, metric_creation_timestamp=mock_time.return_value) metric2 = PanoptesMetric( u'test_metric', 0, PanoptesMetricType.GAUGE, metric_creation_timestamp=mock_time.return_value) metric_set.add(metric1) metric_set.add(metric2) assert len(metric_set) == 1 self.assertIn(metric1, metric_set.metrics) # Test PanoptesMetricSet.__repr__ _METRIC_SET_REPR = u"PanoptesMetricSet[PanoptesMetric[test_metric|0|GAUGE|{}]]".format( mock_time.return_value) self.assertEqual(repr(metric_set), _METRIC_SET_REPR) with self.assertRaises(Exception): metric_set.remove(self.__panoptes_resource) metric_set.remove(metric1) assert len(metric_set) == 0 # Test PanoptesMetricSet.__iter__ and 'next' metric_count = 0 metric_set_iterator = iter(metric_set) for _ in metric_set: self.assertIn(next(metric_set_iterator), metric_set.metrics) metric_count += 1 assert len(metric_set) == metric_count with self.assertRaises(Exception): next(metric_set_iterator)
def setUp(self): self.my_dir, self.panoptes_test_conf_file = _get_test_conf_file() self._panoptes_context = PanoptesContext(self.panoptes_test_conf_file, key_value_store_class_list=[PanoptesEnrichmentCacheKeyValueStore, PanoptesResourcesKeyValueStore]) self._enrichment_kv = self._panoptes_context.get_kv_store(PanoptesEnrichmentCacheKeyValueStore) self._panoptes_resource = PanoptesResource(resource_site=u'test_site', resource_class=u'test_class', resource_subclass=u'test_subclass', resource_type=u'test_type', resource_id=u'test_resource_id', resource_endpoint=u'test_endpoint', resource_plugin=u'test_plugin') interface_validation_object = PanoptesEnrichmentInterfaceSchemaValidator() neighbor_validation_object = PanoptesEnrichmentNeighborSchemaValidator() enrichment_set1 = PanoptesEnrichmentSet(u'int_001') enrichment_set1.add(u'speed', 1000) enrichment_set1.add(u'index', 0o01) enrichment_set1.add(u'status', u'up') enrichment_set2 = PanoptesEnrichmentSet(u'int_002') enrichment_set2.add(u'speed', 1000) enrichment_set2.add(u'index', 0o02) enrichment_set2.add(u'status', u'down') enrichment_group1 = PanoptesEnrichmentGroup(u'interface', interface_validation_object, 300, 60) enrichment_group1.add_enrichment_set(enrichment_set1) enrichment_group1.add_enrichment_set(enrichment_set2) enrichment_set3 = PanoptesEnrichmentSet(u'host_name') enrichment_set3.add(u'vlan_id', 501) enrichment_set3.add(u'property', u'Test Property') enrichment_set3.add(u'mac', u'aa:bb:cc:dd:ee:ff') enrichment_group2 = PanoptesEnrichmentGroup(u'neighbor', neighbor_validation_object, 600, 120) enrichment_group2.add_enrichment_set(enrichment_set3) self.enrichment_group_set1 = PanoptesEnrichmentGroupSet(self._panoptes_resource) self.enrichment_group_set1.add_enrichment_group(enrichment_group1) self.enrichment_group_set1.add_enrichment_group(enrichment_group2) self._panoptes_resource01 = PanoptesResource(resource_site=u'test_site', resource_class=u'test_class', resource_subclass=u'test_subclass', resource_type=u'test_type', resource_id=u'test_resource_id01', resource_endpoint=u'test_endpoint01', resource_plugin=u'test_plugin') enrichment_set4 = PanoptesEnrichmentSet(u'host_name01') enrichment_set4.add(u'vlan_id', 502) enrichment_set4.add(u'property', u'Test Property') enrichment_set4.add(u'mac', u'aa:bb:cc:dd:ee:ff') enrichment_group3 = PanoptesEnrichmentGroup(u'neighbor', neighbor_validation_object, 600, 120) enrichment_group3.add_enrichment_set(enrichment_set3) enrichment_group3.add_enrichment_set(enrichment_set4) self.enrichment_group_set2 = PanoptesEnrichmentGroupSet(self._panoptes_resource01) self.enrichment_group_set2.add_enrichment_group(enrichment_group1) self.enrichment_group_set2.add_enrichment_group(enrichment_group3) self._multi_enrichment_group_set = PanoptesEnrichmentMultiGroupSet() self._multi_enrichment_group_set.add_enrichment_group_set(self.enrichment_group_set1) self._multi_enrichment_group_set.add_enrichment_group_set(self.enrichment_group_set2)
class TestEnrichmentFramework(unittest.TestCase): @patch(u'yahoo_panoptes.framework.resources.time', mock_time) def setUp(self): self.__panoptes_resource = PanoptesResource(resource_site=u'test', resource_class=u'test', resource_subclass=u'test', resource_type=u'test', resource_id=u'test', resource_endpoint=u'test', resource_plugin=u'test') self.__panoptes_resource.add_metadata(u'test', u'test') def test_enrichment_set(self): enrichment_set = PanoptesEnrichmentSet(u'int_001') enrichment_set.add(u'speed', 1000) enrichment_set.add(u'index', 0o01) enrichment_set.add(u'status', u'up') self.assertEquals(enrichment_set.key, u'int_001') self.assertDictEqual(enrichment_set.value, {u'status': u'up', u'index': 1, u'speed': 1000}) self.assertEquals(len(enrichment_set), 3) enrichment_set1 = PanoptesEnrichmentSet(u'int_002', {u'status': u'down', u'index': 2, u'speed': 1000}) self.assertEquals(enrichment_set1.key, u'int_002') self.assertDictEqual(enrichment_set1.value, {u'status': u'down', u'index': 2, u'speed': 1000}) with self.assertRaises(AssertionError): PanoptesEnrichmentSet(u'int_001', u'string') with self.assertRaises(AssertionError): PanoptesEnrichmentSet(u'int_001', 100) def test_enrichment_schema_validator(self): validator = PanoptesEnrichmentInterfaceSchemaValidator() enrichment_set = PanoptesEnrichmentSet(u'int_001') enrichment_set.add(u'speed', 1000) enrichment_set.add(u'index', 0o01) enrichment_set.add(u'status', u'up') self.assertTrue(validator.validate(enrichment_set)) enrichment_set.add(u'status', 0o1) self.assertFalse(validator.validate(enrichment_set)) @patch(u'time.time', mock_time) def test_enrichment_group(self): interface_validation_object = PanoptesEnrichmentInterfaceSchemaValidator() neighbor_validation_object = PanoptesEnrichmentNeighborSchemaValidator() interface_data = \ u'''{"data": [ {"int_001": {"index": 1, "speed": 1000, "status": "up"}}, {"int_002": {"index": 2, "speed": 1000, "status": "down"}}], "metadata": {"_enrichment_group_creation_timestamp": %f, "_enrichment_ttl": 300, "_execute_frequency": 60}, "namespace": "interface"}''' % mock_time.return_value neighbor_data = \ u'''{"data": [{"host_name": {"mac": "aa:bb:cc:dd:ee:ff", "property": "Test Property", "vlan_id": 501}}], "metadata": {"_enrichment_group_creation_timestamp": %f, "_enrichment_ttl": 600, "_execute_frequency": 120}, "namespace": "neighbor"}''' % mock_time.return_value with self.assertRaises(AssertionError): PanoptesEnrichmentGroup(1, interface_validation_object, 300, 60) with self.assertRaises(AssertionError): PanoptesEnrichmentGroup(u'interface', u'non_validation_object', 300, 60) with self.assertRaises(AssertionError): PanoptesEnrichmentGroup(u'interface', interface_validation_object, u'300', 60) with self.assertRaises(AssertionError): PanoptesEnrichmentGroup(u'interface', interface_validation_object, 300, u'60') with self.assertRaises(AssertionError): PanoptesEnrichmentGroup(u'interface', interface_validation_object, 0, 60) with self.assertRaises(AssertionError): PanoptesEnrichmentGroup(u'interface', interface_validation_object, 300, 0) with self.assertRaises(AssertionError): PanoptesEnrichmentGroup(u'interface', interface_validation_object, 300, 60).\ add_enrichment_set(u'not_PanoptesEnrichmentSet_obj') enrichment_set1 = PanoptesEnrichmentSet(u'int_001') enrichment_set1.add(u'speed', 1000) enrichment_set1.add(u'index', 0o01) enrichment_set1.add(u'status', u'up') enrichment_set2 = PanoptesEnrichmentSet(u'int_002') enrichment_set2.add(u'speed', 1000) enrichment_set2.add(u'index', 0o02) enrichment_set2.add(u'status', u'down') enrichment_group1 = PanoptesEnrichmentGroup(u'interface', interface_validation_object, 300, 60) enrichment_group1.add_enrichment_set(enrichment_set1) enrichment_group1.add_enrichment_set(enrichment_set2) self.assertEqual(enrichment_group1.namespace, u'interface') self.assertEqual(enrichment_group1.enrichment_ttl, 300) self.assertEqual(enrichment_group1.execute_frequency, 60) self.assertEqual(enrichment_group1.enrichment_group_creation_timestamp, mock_time.return_value) self.assertEqual(ordered(json.loads(json.dumps(enrichment_group1.data, cls=PanoptesEnrichmentEncoder))), ordered(json.loads(interface_data)[u'data'])) self.assertEqual(ordered(json.loads(enrichment_group1.json())), ordered(json.loads(interface_data))) self.assertEquals(len(enrichment_group1), 2) enrichment_set3 = PanoptesEnrichmentSet(u'int_002') enrichment_set3.add(u'speed', 1000) enrichment_set3.add(u'index', 0o02) enrichment_set3.add(u'status', u'down') self.assertEqual(ordered(json.loads(enrichment_group1.json())), ordered(json.loads(interface_data))) self.assertEqual(ordered(enrichment_group1.metadata), ordered(json.loads(interface_data)[u'metadata'])) self.assertEquals(len(enrichment_group1), 2) test_metadata = json.loads(interface_data)[u'metadata'] test_metadata[u'metadata_key'] = u'metadata_value' enrichment_group1.upsert_metadata(u'metadata_key', u'metadata_value') self.assertEqual(ordered(enrichment_group1.metadata), ordered(test_metadata)) enrichment_group1.upsert_metadata(u'ttl', 300) with self.assertRaises(ValueError): enrichment_group1.upsert_metadata(u'_enrichment_ttl', 300) with self.assertRaises(AssertionError): enrichment_group1.upsert_metadata(u'metadata', {}) with self.assertRaises(AssertionError): enrichment_group1.upsert_metadata(u'metadata', []) enrichment_set4 = PanoptesEnrichmentSet(u'host_name') enrichment_set4.add(u'vlan_id', 501) enrichment_set4.add(u'property', u'Test Property') enrichment_set4.add(u'mac', u'aa:bb:cc:dd:ee:ff') enrichment_group2 = PanoptesEnrichmentGroup(u'neighbor', neighbor_validation_object, 600, 120) enrichment_group2.add_enrichment_set(enrichment_set4) self.assertEqual(ordered(json.loads(enrichment_group2.json())), ordered(json.loads(neighbor_data))) self.assertEquals(len(enrichment_group2), 1) enrichment_set5 = PanoptesEnrichmentSet(u'host_name01') enrichment_set5.add(u'vlan_id', 502) enrichment_set5.add(u'property', u'Netops01.US') enrichment_set6 = PanoptesEnrichmentSet(u'host_name02') enrichment_set6.add(u'vlan_id', 503) enrichment_set6.add(u'mac', u'aa:bb:cc:dd:ee:ff') enrichment_group3 = PanoptesEnrichmentGroup(u'neighbor', neighbor_validation_object, 600, 120) enrichment_group3.add_enrichment_set(enrichment_set5) with self.assertRaises(AssertionError): enrichment_group3.add_enrichment_set(enrichment_set6) interface_store_data = u'{"int_001": {"index": 1, "speed": 1000, "status": "up"}, ' \ u'"int_002": {"index": 2, "speed": 1000, "status": "down"}}' neighbor_store_data = u'{"host_name": {"mac": "aa:bb:cc:dd:ee:ff", ' \ u'"property": "Test Property", "vlan_id": 501}}' self.assertEquals(ordered(json.loads(enrichment_group1.serialize_data())), ordered(json.loads(interface_store_data))) self.assertEquals(ordered(json.loads(enrichment_group2.serialize_data())), ordered(json.loads(neighbor_store_data))) enrichment_group1.upsert_metadata(u'ttl', 300) with self.assertRaises(ValueError): enrichment_group1.upsert_metadata(u'_enrichment_ttl', 300) interface_data_serialized = u'''{{"data": {{"int_001": {{"index": 1, "speed": 1000, "status": "up"}}, "int_002": {{"index": 2, "speed": 1000, "status": "down"}}}}, "metadata": {{"_enrichment_group_creation_timestamp": {:.5f}, "_enrichment_ttl": 300, "_execute_frequency": 60, "metadata_key": "metadata_value", "ttl": 300}}}}'''.format(mock_time.return_value) neighbor_data_serialized = u'''{{"data": {{"host_name": {{"mac": "aa:bb:cc:dd:ee:ff", "property": "Test Property","vlan_id": 501}}}}, "metadata": {{"_enrichment_group_creation_timestamp": {:.5f}, "_enrichment_ttl": 600, "_execute_frequency": 120}}}}'''.format(mock_time.return_value) self.assertEquals(ordered(json.loads(enrichment_group1.serialize())), ordered(json.loads(interface_data_serialized))) self.assertEquals(ordered(json.loads(enrichment_group2.serialize())), ordered(json.loads(neighbor_data_serialized))) @patch(u'time.time', mock_time) def test_enrichment_group_set(self): interface_validation_object = PanoptesEnrichmentInterfaceSchemaValidator() neighbor_validation_object = PanoptesEnrichmentNeighborSchemaValidator() panoptes_resource = self.__panoptes_resource enrichment_data = \ u'''{{"enrichment": [{{"metadata": {{"_enrichment_group_creation_timestamp": {:.5f}, "_enrichment_ttl": 600, "_execute_frequency": 120}}, "data": [{{"host_name": {{"mac": "aa:bb:cc:dd:ee:ff", "property": "Test Property", "vlan_id": 501}}}}], "namespace": "neighbor"}}, {{"metadata": {{"_enrichment_group_creation_timestamp": {:.5f}, "_enrichment_ttl": 300, "_execute_frequency": 60}}, "data": [ {{"int_001": {{"index": 1, "speed": 1000, "status": "up"}}}}, {{"int_002": {{"index": 2, "speed": 1000, "status": "down"}}}}], "namespace": "interface"}}], "enrichment_group_set_creation_timestamp": {:.5f}, "resource": {{"resource_class": "test", "resource_creation_timestamp": {:.5f}, "resource_endpoint": "test", "resource_id": "test", "resource_metadata": {{"_resource_ttl": "604800", "test": "test"}}, "resource_plugin": "test", "resource_site": "test", "resource_subclass": "test", "resource_type": "test"}}}}'''.format(mock_time.return_value, mock_time.return_value, mock_time.return_value, mock_time.return_value) enrichment_set1 = PanoptesEnrichmentSet(u'int_001') enrichment_set1.add(u'speed', 1000) enrichment_set1.add(u'index', 0o01) enrichment_set1.add(u'status', u'up') enrichment_set2 = PanoptesEnrichmentSet(u'int_002') enrichment_set2.add(u'speed', 1000) enrichment_set2.add(u'index', 0o02) enrichment_set2.add(u'status', u'down') enrichment_group1 = PanoptesEnrichmentGroup(u'interface', interface_validation_object, 300, 60) enrichment_group1.add_enrichment_set(enrichment_set1) enrichment_group1.add_enrichment_set(enrichment_set2) enrichment_set3 = PanoptesEnrichmentSet(u'host_name') enrichment_set3.add(u'vlan_id', 501) enrichment_set3.add(u'property', u'Test Property') enrichment_set3.add(u'mac', u'aa:bb:cc:dd:ee:ff') enrichment_group2 = PanoptesEnrichmentGroup(u'neighbor', neighbor_validation_object, 600, 120) enrichment_group2.add_enrichment_set(enrichment_set3) enrichment_group_set1 = PanoptesEnrichmentGroupSet(panoptes_resource) enrichment_group_set1.add_enrichment_group(enrichment_group1) enrichment_group_set1.add_enrichment_group(enrichment_group2) self.assertEquals(len(enrichment_group_set1), 2) group_set_repr = u"PanoptesEnrichmentGroupSet[resource:" \ u"plugin|test|site|test|class|test|subclass|test|type|test|id|test|endpoint|test," \ u"enrichment_group_set_creation_timestamp:{},PanoptesEnrichmentGroup[namespace:" \ u"interface,enrichment_ttl:300,execute_frequency:60,enrichment_group_creation_timestamp:{}," \ u"PanoptesEnrichmentSet[int_001[index:1,speed:1000,status:up]],PanoptesEnrichmentSet[" \ u"int_002[index:2,speed:1000,status:down]]],PanoptesEnrichmentGroup[namespace:neighbor," \ u"enrichment_ttl:600,execute_frequency:120,enrichment_group_creation_timestamp:{}," \ u"PanoptesEnrichmentSet[host_name[mac:aa:bb:cc:dd:ee:ff,property:" \ u"Test Property,vlan_id:501]]]]".format(mock_time.return_value, mock_time.return_value, mock_time.return_value) self.assertEquals(repr(enrichment_group_set1), group_set_repr) self.assertIsInstance(enrichment_group_set1.resource, PanoptesResource) self.assertEqual(enrichment_group_set1.enrichment_group_set_creation_timestamp, mock_time.return_value) self.assertEqual( ordered(json.loads(json.dumps(enrichment_group_set1.enrichment, cls=PanoptesEnrichmentEncoder))), ordered(json.loads(enrichment_data)[u'enrichment'])) self.assertEqual(ordered(json.loads(enrichment_group_set1.json())[u'enrichment']), ordered(json.loads(enrichment_data)[u'enrichment'])) with self.assertRaises(AssertionError): PanoptesEnrichmentGroupSet(u'bad_resource') with self.assertRaises(AssertionError): PanoptesEnrichmentGroupSet(panoptes_resource).add_enrichment_group(u'non_PanoptesEnrichmentGroup_obj') enrichment_group_set2 = PanoptesEnrichmentGroupSet(panoptes_resource) enrichment_group3 = PanoptesEnrichmentGroup(u'interface', interface_validation_object, 300, 60) with self.assertRaises(AssertionError): enrichment_group_set2.add_enrichment_group(enrichment_group3) self.assertFalse(enrichment_group_set1 == enrichment_group1) self.assertFalse(enrichment_group_set1 == enrichment_group_set2) @patch(u'time.time', mock_time) @patch(u'yahoo_panoptes.framework.resources.time', mock_time) def test_multi_enrichment_group_set(self): interface_validation_object = PanoptesEnrichmentInterfaceSchemaValidator() neighbor_validation_object = PanoptesEnrichmentNeighborSchemaValidator() panoptes_resource = self.__panoptes_resource multi_enrichment_results_data = \ { "group_sets": [ { "enrichment": [ { "data": [ { "host_name": { "mac": "aa:bb:cc:dd:ee:ff", "property": "Test Property", "vlan_id": 501 } }, { "host_name01": { "mac": "aa:bb:cc:dd:ee:ff", "property": "Test Property", "vlan_id": 502 } } ], "metadata": { "_enrichment_group_creation_timestamp": mock_time.return_value, "_enrichment_ttl": 600, "_execute_frequency": 120 }, "namespace": "neighbor" }, { "data": [ { "int_001": { "index": 1, "speed": 1000, "status": "up" } } ], "metadata": { "_enrichment_group_creation_timestamp": mock_time.return_value, "_enrichment_ttl": 300, "_execute_frequency": 60 }, "namespace": "interface" } ], "enrichment_group_set_creation_timestamp": mock_time.return_value, "resource": { "resource_class": "test_class", "resource_creation_timestamp": mock_time.return_value, "resource_endpoint": "test_endpoint01", "resource_id": "test_resource_id01", "resource_metadata": { "_resource_ttl": "604800" }, "resource_plugin": "test_plugin", "resource_site": "test_site", "resource_subclass": "test_subclass", "resource_type": "test_type" } }, { "enrichment": [ { "data": [ { "int_001": { "index": 1, "speed": 1000, "status": "up" } } ], "metadata": { "_enrichment_group_creation_timestamp": mock_time.return_value, "_enrichment_ttl": 300, "_execute_frequency": 60 }, "namespace": "interface" }, { "data": [ { "host_name": { "mac": "aa:bb:cc:dd:ee:ff", "property": "Test Property", "vlan_id": 501 } } ], "metadata": { "_enrichment_group_creation_timestamp": mock_time.return_value, "_enrichment_ttl": 600, "_execute_frequency": 120 }, "namespace": "neighbor" } ], "enrichment_group_set_creation_timestamp": mock_time.return_value, "resource": { "resource_class": "test", "resource_creation_timestamp": mock_time.return_value, "resource_endpoint": "test", "resource_id": "test", "resource_metadata": { "_resource_ttl": "604800", "test": "test" }, "resource_plugin": "test", "resource_site": "test", "resource_subclass": "test", "resource_type": "test" } } ] } enrichment_set1 = PanoptesEnrichmentSet(u'int_001') enrichment_set1.add(u'speed', 1000) enrichment_set1.add(u'index', 0o01) enrichment_set1.add(u'status', u'up') enrichment_group1 = PanoptesEnrichmentGroup(u'interface', interface_validation_object, 300, 60) enrichment_group1.add_enrichment_set(enrichment_set1) enrichment_set3 = PanoptesEnrichmentSet(u'host_name') enrichment_set3.add(u'vlan_id', 501) enrichment_set3.add(u'property', u'Test Property') enrichment_set3.add(u'mac', u'aa:bb:cc:dd:ee:ff') enrichment_group2 = PanoptesEnrichmentGroup(u'neighbor', neighbor_validation_object, 600, 120) enrichment_group2.add_enrichment_set(enrichment_set3) enrichment_group_set1 = PanoptesEnrichmentGroupSet(panoptes_resource) enrichment_group_set1.add_enrichment_group(enrichment_group1) enrichment_group_set1.add_enrichment_group(enrichment_group2) panoptes_resource01 = PanoptesResource(resource_site=u'test_site', resource_class=u'test_class', resource_subclass=u'test_subclass', resource_type=u'test_type', resource_id=u'test_resource_id01', resource_endpoint=u'test_endpoint01', resource_plugin=u'test_plugin') panoptes_resource02 = PanoptesResource(resource_site=u'test_site', resource_class=u'test_class', resource_subclass=u'test_subclass', resource_type=u'test_type', resource_id=u'test_resource_id02', resource_endpoint=u'test_endpoint02', resource_plugin=u'test_plugin') enrichment_set4 = PanoptesEnrichmentSet(u'host_name01') enrichment_set4.add(u'vlan_id', 502) enrichment_set4.add(u'property', u'Test Property') enrichment_set4.add(u'mac', u'aa:bb:cc:dd:ee:ff') enrichment_group3 = PanoptesEnrichmentGroup(u'neighbor', neighbor_validation_object, 600, 120) enrichment_group3.add_enrichment_set(enrichment_set3) enrichment_group3.add_enrichment_set(enrichment_set4) enrichment_group_set2 = PanoptesEnrichmentGroupSet(panoptes_resource01) enrichment_group_set2.add_enrichment_group(enrichment_group1) enrichment_group_set2.add_enrichment_group(enrichment_group3) multi_enrichment_group_set = PanoptesEnrichmentMultiGroupSet() multi_enrichment_group_set.add_enrichment_group_set(enrichment_group_set1) multi_enrichment_group_set.add_enrichment_group_set(enrichment_group_set2) multi_enrichment_group_set_repr = u"PanoptesEnrichmentMultiGroupSet[PanoptesEnrichmentGroupSet[resource:" \ u"plugin|test|site|test|class|test|subclass|test|type|test|id|test|endpoint" \ u"|test,enrichment_group_set_creation_timestamp:{},PanoptesEnrichmentGroup" \ u"[namespace:interface,enrichment_ttl:300,execute_frequency:60," \ u"enrichment_group_creation_timestamp:{},PanoptesEnrichmentSet[" \ u"int_001[index:1,speed:1000,status:up]]],PanoptesEnrichmentGroup[" \ u"namespace:neighbor,enrichment_ttl:600,execute_frequency:120," \ u"enrichment_group_creation_timestamp:{},PanoptesEnrichmentSet" \ u"[host_name[mac:aa:bb:cc:dd:ee:ff,property:Test Property,vlan_id:501]" \ u"]]],PanoptesEnrichmentGroupSet[resource:plugin|test_plugin|site|" \ u"test_site|class|test_class|subclass|test_subclass|type|test_type|id" \ u"|test_resource_id01|endpoint|test_endpoint01," \ u"enrichment_group_set_creation_timestamp:{},PanoptesEnrichmentGroup" \ u"[namespace:interface,enrichment_ttl:300,execute_frequency:60," \ u"enrichment_group_creation_timestamp:{},PanoptesEnrichmentSet" \ u"[int_001[index:1,speed:1000,status:up]]],PanoptesEnrichmentGroup" \ u"[namespace:neighbor,enrichment_ttl:600,execute_frequency:120," \ u"enrichment_group_creation_timestamp:{},PanoptesEnrichmentSet" \ u"[host_name[mac:aa:bb:cc:dd:ee:ff,property:Test Property,vlan_id:501]]," \ u"PanoptesEnrichmentSet[host_name01[mac:aa:bb:cc:dd:ee:ff," \ u"property:Test Property,vlan_id:502]]]]]".format(mock_time.return_value, mock_time.return_value, mock_time.return_value, mock_time.return_value, mock_time.return_value, mock_time.return_value) self.assertEquals(repr(multi_enrichment_group_set), multi_enrichment_group_set_repr) self.assertEquals(len(multi_enrichment_group_set.enrichment_group_sets), 2) self.assertEquals(ordered(json.loads(multi_enrichment_group_set.json())), ordered(multi_enrichment_results_data)) self.assertEquals(len(multi_enrichment_group_set), 2) multi_enrichment_group_set.add_enrichment_group_set(enrichment_group_set2) self.assertEquals(len(multi_enrichment_group_set), 2) enrichment_group_set3 = PanoptesEnrichmentGroupSet(panoptes_resource02) enrichment_group_set3.add_enrichment_group(enrichment_group1) multi_enrichment_group_set.add_enrichment_group_set(enrichment_group_set3) self.assertEquals(len(multi_enrichment_group_set), 3) with self.assertRaises(AssertionError): multi_enrichment_group_set.add_enrichment_group_set(u'non_enrichment_group') enrichment_group_set3 = PanoptesEnrichmentGroupSet(panoptes_resource01) with self.assertRaises(AssertionError): multi_enrichment_group_set.add_enrichment_group_set(enrichment_group_set3)