def run(self, context):
"""
The main entry point to the plugin
Args:
context (PanoptesPluginContext): The Plugin Context passed by the Plugin Agent
Returns:
PanoptesResourceSet: A non-empty resource set
Raises:
PanoptesDiscoveryPluginError: This exception is raised if any part of the lookup process has errors
"""
assert context and isinstance(context, PanoptesPluginContext), 'context must be a PanoptesPluginContext'
conf = context.config
logger = context.logger
config_file = None
try:
config_file = conf['main']['config_file']
with open(config_file) as f:
resource_specs = json.load(f)
except Exception as e:
raise PanoptesDiscoveryPluginError(
'Error while attempting to parse JSON from file {}: {}'.format(config_file, repr(e))
)
resources = PanoptesResourceSet()
num_successes = 0
num_failures = 0
for resource_spec in resource_specs:
try:
resource = PanoptesResource.resource_from_dict(resource_spec)
resources.add(resource)
num_successes += 1
logger.debug('Added resource {} from JSON file {}'.format(resource, config_file))
except Exception as e:
logger.debug('Error while attempting to create a PanoptesResource from file {}: {}'.format(
config_file, repr(e)))
num_failures += 1
continue
if num_successes > 0:
logger.info('Tried to read {} resources from {}, {} failed'.format(num_successes + num_failures,
config_file,
num_failures))
else:
logger.error('Error while attempting to create PanoptesResources from {}.'.format(config_file))
raise PanoptesDiscoveryPluginError(
'Error during lookup for PanoptesResource from file {}.'.format(config_file))
return resources
def setUp(self):
self._panoptes_metric = {
u'metrics_group_interval':
60,
u'resource': {
u'resource_site': u'test_site',
u'resource_id': u'test_id',
u'resource_class': u'network',
u'resource_plugin': u'test_plugin',
u'resource_creation_timestamp': 1567823517.46,
u'resource_subclass': u'test_subclass',
u'resource_endpoint': u'test_endpoint',
u'resource_metadata': {
u'test_metadata_key': u'test_metadata_value',
u'_resource_ttl': u"604800"
},
u'resource_type': u'test_type'
},
u'dimensions': [{
u'dimension_name': u'cpu_name',
u'dimension_value': u'test_cpu_name_value'
}, {
u'dimension_name': u'cpu_no',
u'dimension_value': u'test_cpu_no_value'
}, {
u'dimension_name': u'cpu_type',
u'dimension_value': u'test_cpu_type_value'
}],
u'metrics_group_type':
u'cpu',
u'metrics': [{
u'metric_creation_timestamp': 1567823946.72,
u'metric_type': u'gauge',
u'metric_name': u'cpu_utilization',
u'metric_value': 0
}],
u'metrics_group_creation_timestamp':
1567823946.72,
u'metrics_group_schema_version':
u'0.2'
}
self._panoptes_resource = PanoptesResource(resource_site=u'test',
resource_class=u'test',
resource_subclass=u'test',
resource_type=u'test',
resource_id=u'test',
resource_endpoint=u'test',
resource_plugin=u'test')
self._panoptes_resource.add_metadata(u'test', u'test')
self._panoptes_resource_set = PanoptesResourceSet()
self._panoptes_resource_set.add(self._panoptes_resource)
def setUp(self):
self.__panoptes_resource_metadata = {'test': 'test', '_resource_ttl': '604800'}
self.__panoptes_resource = PanoptesResource(resource_site='test', resource_class='test',
resource_subclass='test',
resource_type='test', resource_id='test', resource_endpoint='test',
resource_plugin='test',
resource_creation_timestamp=_TIMESTAMP,
resource_ttl=RESOURCE_MANAGER_RESOURCE_EXPIRE)
self.__panoptes_resource.add_metadata('test', 'test')
self.__panoptes_resource_set = PanoptesResourceSet()
mock_valid_timestamp = Mock(return_value=True)
with patch('yahoo_panoptes.framework.resources.PanoptesValidators.valid_timestamp',
mock_valid_timestamp):
self.__panoptes_resource_set.resource_set_creation_timestamp = _TIMESTAMP
self.my_dir, self.panoptes_test_conf_file = _get_test_conf_file()
class TestValidators(unittest.TestCase):
def setUp(self):
self._panoptes_metric = {
u'metrics_group_interval':
60,
u'resource': {
u'resource_site': u'test_site',
u'resource_id': u'test_id',
u'resource_class': u'network',
u'resource_plugin': u'test_plugin',
u'resource_creation_timestamp': 1567823517.46,
u'resource_subclass': u'test_subclass',
u'resource_endpoint': u'test_endpoint',
u'resource_metadata': {
u'test_metadata_key': u'test_metadata_value',
u'_resource_ttl': u"604800"
},
u'resource_type': u'test_type'
},
u'dimensions': [{
u'dimension_name': u'cpu_name',
u'dimension_value': u'test_cpu_name_value'
}, {
u'dimension_name': u'cpu_no',
u'dimension_value': u'test_cpu_no_value'
}, {
u'dimension_name': u'cpu_type',
u'dimension_value': u'test_cpu_type_value'
}],
u'metrics_group_type':
u'cpu',
u'metrics': [{
u'metric_creation_timestamp': 1567823946.72,
u'metric_type': u'gauge',
u'metric_name': u'cpu_utilization',
u'metric_value': 0
}],
u'metrics_group_creation_timestamp':
1567823946.72,
u'metrics_group_schema_version':
u'0.2'
}
self._panoptes_resource = PanoptesResource(resource_site=u'test',
resource_class=u'test',
resource_subclass=u'test',
resource_type=u'test',
resource_id=u'test',
resource_endpoint=u'test',
resource_plugin=u'test')
self._panoptes_resource.add_metadata(u'test', u'test')
self._panoptes_resource_set = PanoptesResourceSet()
self._panoptes_resource_set.add(self._panoptes_resource)
def test_default_record(self):
self.assertTrue(
PanoptesConsumerRecordValidator.validate(
PanoptesConsumerTypes.METRICS, self._panoptes_metric))
self.assertTrue(
PanoptesConsumerRecordValidator.validate(
PanoptesConsumerTypes.PROCESSED, self._panoptes_metric))
self.assertTrue(
PanoptesConsumerRecordValidator.validate(
PanoptesConsumerTypes.RESOURCES,
json.loads(self._panoptes_resource_set.json)))
self.assertFalse(PanoptesConsumerRecordValidator.validate(5, {}))
def test_resource_throws(self):
argument_overrides = [(u'resource_site', 1), (u'resource_id', None),
(u'resource_class', 1.5),
(u'resource_plugin', {}),
(u'resource_creation_timestamp', u'1567823517'),
(u'resource_subclass', []),
(u'resource_endpoint', 123456.789)]
for (override_key, override_value) in argument_overrides:
schema_arguments = self._panoptes_metric.copy()
schema_arguments[u'resource'][override_key] = override_value
self.assertEquals(
PanoptesConsumerRecordValidator.validate_metrics(
schema_arguments), False)
def handle_resources(plugin_signature, resources):
resource_site = str(resources[u'resources'][0][u'resource_site'])
plugin_name = str(plugin_signature.split(':')[0])
logger.info(u'For plugin "%s" and site "%s", going to get current set' %
(plugin_name, resource_site))
try:
current_resource_set = resource_store.get_resources(
site=resource_site, plugin_name=plugin_name)
except:
logger.error(u'Error trying to get current resources for plugin "%s"' %
plugin_signature)
return False
logger.info(u'For plugin "%s" and site "%s" current set has %d resources' %
(plugin_signature, resource_site, len(current_resource_set)))
new_resource_set = PanoptesResourceSet()
for resource in resources[u'resources']:
new_resource = PanoptesResource.resource_from_dict(resource)
new_resource_set.add(new_resource)
current_resource_dict = _resource_set_to_dictionary(current_resource_set)
new_resource_dict = _resource_set_to_dictionary(new_resource_set)
logger.info(u'For plugin "%s" and site "%s", new set has %d resources' %
(plugin_signature, resource_site, len(new_resource_set)))
resources_to_delete = current_resource_set.resources.difference(
new_resource_set)
logger.info(u'For plugin "%s" and site "%s", deleting %d resources' %
(plugin_signature, resource_site, len(resources_to_delete)))
for resource in resources_to_delete:
current_timestamp = current_resource_dict[
resource.serialization_key].resource_creation_timestamp
new_timestamp = resources[u'resource_set_creation_timestamp']
if current_timestamp > new_timestamp:
logger.info(
u'For plugin "%s" and site "%s", resource "%s" has timestamp (%0.2f UTC) greater than of '
u'new set (%0.2f UTC), skipping deletion' %
(plugin_signature, resource_site, str(resource),
current_timestamp, new_timestamp))
else:
logger.info(
u'For plugin "%s" and site "%s", going to delete resource "%s"'
% (plugin_signature, resource_site, resource))
try:
resource_store.delete_resource(plugin_signature, resource)
except:
logger.exception(u'Error trying to delete resource')
return False
resources_to_add = new_resource_set.resources.difference(
current_resource_set)
logger.info(u'For plugin "%s" and site "%s", adding %d resources' %
(plugin_signature, resource_site, len(resources_to_add)))
for resource in resources_to_add:
logger.debug(u'Going to add resource "%s"' % resource)
try:
resource_store.add_resource(plugin_signature, resource)
except:
logger.exception(u'Error trying to add resource')
return False
resources_to_update = current_resource_set.resources.intersection(
new_resource_set)
logger.info(u'For plugin "%s" and site "%s", updating %d resources' %
(plugin_signature, resource_site, len(resources_to_update)))
start_time = time.time()
resources_skipped = 0
resources_updated = 0
for resource in resources_to_update:
current_timestamp = current_resource_dict[
resource.serialization_key].resource_creation_timestamp
new_timestamp = new_resource_dict[
resource.serialization_key].resource_creation_timestamp
if current_timestamp > new_timestamp:
logger.info(
u'For plugin "%s" and site "%s", resource "%s" has timestamp (%0.2f UTC) greater than of '
u'new set (%0.2f UTC), skipping update' %
(plugin_signature, resource_site, str(resource),
current_timestamp, new_timestamp))
resources_skipped += 1
else:
try:
new_resource = new_resource_dict[resource.serialization_key]
logger.debug(u'Going to update resource "%s"' % new_resource)
resource_store.add_resource(plugin_signature, new_resource)
resources_updated += 1
except:
logger.exception(u'Error trying to add resource')
return False
end_time = time.time()
logger.info(
u'For plugin "%s" and site "%s", added/updated %d resources in %0.2f seconds, skipped %d resources'
% (plugin_signature, resource_site, resources_updated,
round(end_time - start_time, 2), resources_skipped))
return True
def run(self, context):
panoptes_resource_set = PanoptesResourceSet()
panoptes_resource_set.add(self.resource)
return panoptes_resource_set
def test_resource_cache(self):
panoptes_context = PanoptesContext(self.panoptes_test_conf_file,
key_value_store_class_list=[PanoptesTestKeyValueStore])
# Test PanoptesResourceCache methods when setup_resource_cache not yet called
panoptes_resource_cache = PanoptesResourceCache(panoptes_context)
test_query = 'resource_class = "network"'
with self.assertRaises(PanoptesResourceCacheException):
panoptes_resource_cache.get_resources(test_query)
panoptes_resource_cache.close_resource_cache()
panoptes_resource = self.__panoptes_resource
panoptes_resource.add_metadata("metadata_key1", "test")
panoptes_resource.add_metadata("metadata_key2", "test")
kv = panoptes_context.get_kv_store(PanoptesTestKeyValueStore)
serialized_key, serialized_value = PanoptesResourceStore._serialize_resource(panoptes_resource)
kv.set(serialized_key, serialized_value)
mock_kv_store = Mock(return_value=kv)
with patch('yahoo_panoptes.framework.resources.PanoptesContext.get_kv_store', mock_kv_store):
# Test errors when setting up resource cache
mock_resource_store = Mock(side_effect=Exception)
with patch('yahoo_panoptes.framework.resources.PanoptesResourceStore', mock_resource_store):
with self.assertRaises(PanoptesResourceCacheException):
panoptes_resource_cache.setup_resource_cache()
mock_connect = Mock(side_effect=Exception)
with patch('yahoo_panoptes.framework.resources.sqlite3.connect', mock_connect):
with self.assertRaises(PanoptesResourceCacheException):
panoptes_resource_cache.setup_resource_cache()
mock_get_resources = Mock(side_effect=Exception)
with patch('yahoo_panoptes.framework.resources.PanoptesResourceStore.get_resources', mock_get_resources):
with self.assertRaises(PanoptesResourceCacheException):
panoptes_resource_cache.setup_resource_cache()
# Test basic operations
panoptes_resource_cache.setup_resource_cache()
self.assertIsInstance(panoptes_resource_cache.get_resources('resource_class = "network"'),
PanoptesResourceSet)
self.assertEqual(len(panoptes_resource_cache.get_resources('resource_class = "network"')), 0)
self.assertIn(panoptes_resource, panoptes_resource_cache.get_resources('resource_class = "test"'))
self.assertEqual(len(panoptes_resource_cache._cached_resources), 2)
panoptes_resource_cache.close_resource_cache()
# Mock PanoptesResourceStore.get_resources to return Resources that otherwise couldn't be constructed:
mock_resources = PanoptesResourceSet()
mock_resources.add(panoptes_resource)
bad_panoptes_resource = PanoptesResource(resource_site='test', resource_class='test',
resource_subclass='test',
resource_type='test', resource_id='test2',
resource_endpoint='test',
resource_plugin='test',
resource_creation_timestamp=_TIMESTAMP,
resource_ttl=RESOURCE_MANAGER_RESOURCE_EXPIRE)
bad_panoptes_resource.__dict__['_PanoptesResource__data']['resource_metadata']['*'] = "test"
bad_panoptes_resource.__dict__['_PanoptesResource__data']['resource_metadata']['**'] = "test"
mock_resources.add(bad_panoptes_resource)
mock_get_resources = Mock(return_value=mock_resources)
with patch('yahoo_panoptes.framework.resources.PanoptesResourceStore.get_resources', mock_get_resources):
panoptes_resource_cache.setup_resource_cache()
self.assertEqual(len(panoptes_resource_cache.get_resources('resource_class = "test"')), 1)