def test_code_output(self):
# Mock the system calls
subprocess.check_output = Mock(return_value=TEST_PING_RESPONSE)
resource = PanoptesResource.resource_from_dict(self._resource_dict)
test_ping = PanoptesPingConnectionFactory.get_ping_connection(resource=resource,
context=None,
count=3,
timeout=100)
test_d = {"round_trip_stddev": 0.057, "packets_received": 3, "execution_time": 1.44,
"round_trip_avg": 0.120, "packets_transmitted": 3, "packet_loss_pct": 0.0,
"round_trip_max": 0.162, "round_trip_min": 0.040}
self.assertDictEqual(test_d, json.loads(test_ping.response))
self.assertEqual(test_ping.packets_transmitted, 3)
self.assertEqual(test_ping.packets_received, 3)
self.assertEqual(test_ping.packet_loss_pct, 0.0)
self.assertEqual(test_ping.execution_time, 1.44) # add for testing prod
self.assertEqual(test_ping.round_trip_min, 0.040)
self.assertEqual(test_ping.round_trip_avg, 0.120)
self.assertEqual(test_ping.round_trip_max, 0.162)
self.assertEqual(test_ping.round_trip_stddev, 0.057)
# Test bad parameters
with self.assertRaises(AssertionError):
PanoptesPingConnectionFactory.get_ping_connection(resource=resource,
context=None,
count=-1,
timeout=100)
with self.assertRaises(AssertionError):
PanoptesPingConnectionFactory.get_ping_connection(resource=resource,
context=None,
count='1',
timeout=100)
with self.assertRaises(AssertionError):
PanoptesPingConnectionFactory.get_ping_connection(resource=None,
context=None,
count=3,
timeout=100)
with self.assertRaises(AssertionError):
self._resource_dict['resource_endpoint'] = ''
resource = PanoptesResource.resource_from_dict(self._resource_dict)
PanoptesPingConnectionFactory.get_ping_connection(resource=resource,
context=None,
count=3,
timeout=100)
with self.assertRaises(AssertionError):
self._resource_dict['resource_endpoint'] = 'localhost'
resource = PanoptesResource.resource_from_dict(self._resource_dict)
PanoptesPingConnectionFactory.get_ping_connection(resource=resource,
context=None,
count=3,
timeout=0)
with self.assertRaises(AssertionError):
PanoptesPingConnectionFactory.get_ping_connection(resource=resource,
context=None,
count=3,
timeout=-1)
def run(self, context):
"""
The main entry point to the plugin
Args:
context (PanoptesPluginContext): The Plugin Context passed by the Plugin Agent
Returns:
PanoptesResourceSet: A non-empty resource set
Raises:
PanoptesDiscoveryPluginError: This exception is raised if any part of the lookup process has errors
"""
assert context and isinstance(context, PanoptesPluginContext), 'context must be a PanoptesPluginContext'
conf = context.config
logger = context.logger
config_file = None
try:
config_file = conf['main']['config_file']
with open(config_file) as f:
resource_specs = json.load(f)
except Exception as e:
raise PanoptesDiscoveryPluginError(
'Error while attempting to parse JSON from file {}: {}'.format(config_file, repr(e))
)
resources = PanoptesResourceSet()
num_successes = 0
num_failures = 0
for resource_spec in resource_specs:
try:
resource = PanoptesResource.resource_from_dict(resource_spec)
resources.add(resource)
num_successes += 1
logger.debug('Added resource {} from JSON file {}'.format(resource, config_file))
except Exception as e:
logger.debug('Error while attempting to create a PanoptesResource from file {}: {}'.format(
config_file, repr(e)))
num_failures += 1
continue
if num_successes > 0:
logger.info('Tried to read {} resources from {}, {} failed'.format(num_successes + num_failures,
config_file,
num_failures))
else:
logger.error('Error while attempting to create PanoptesResources from {}.'.format(config_file))
raise PanoptesDiscoveryPluginError(
'Error during lookup for PanoptesResource from file {}.'.format(config_file))
return resources
def test_steamroller_ping(self):
plugin_conf = {
'Core': {
'name': 'Test Plugin',
'module': 'test_plugin'
},
'main': {
'execute_frequency': '60',
'resource_filter': 'resource_class = "network"'
}
}
self._resource_dict['resource_metadata']['ping_route'] = 'steamroller'
self._resource_dict['resource_metadata']['snmp_proxy_hosts'] = 'proxyhost-5.doesntexist.com:4443'
resource = PanoptesResource.resource_from_dict(self._resource_dict)
plugin_context = create_autospec(
PanoptesPluginWithEnrichmentContext, instance=True, spec_set=True,
data=resource,
config=plugin_conf,
snmp=panoptes_context.config_object.snmp_defaults,
x509=panoptes_context.config_object.x509_defaults,
secrets=secret_store,
logger=logging.getLogger(__name__)
)
with patch('requests.Session') as rs:
rs.return_value = MockConnection()
ping_connection = PanoptesPingConnectionFactory.get_ping_connection(resource, plugin_context)
self.assertEqual(ping_connection.packets_transmitted, 10)
self.assertEqual(ping_connection.packets_received, 10)
self.assertEqual(ping_connection.packet_loss_pct, 0.0)
self.assertEqual(ping_connection.execution_time, 9.03)
self.assertEqual(ping_connection.round_trip_min, 0.485)
self.assertEqual(ping_connection.round_trip_avg, 0.504)
self.assertEqual(ping_connection.round_trip_max, 0.535)
self.assertEqual(ping_connection.round_trip_stddev, 0.027)
with patch.object(requests.Session, 'post') as rs:
rs.side_effect = requests.exceptions.Timeout()
with self.assertRaises(PanoptesPingTimeoutException):
PanoptesPingConnectionFactory.get_ping_connection(resource, plugin_context)
with patch.object(requests.Session, 'post') as rs:
rs.side_effect = requests.exceptions.ConnectionError()
with self.assertRaises(PanoptesPingConnectionError):
PanoptesPingConnectionFactory.get_ping_connection(resource, plugin_context)
with patch.object(requests.Session, 'post') as rs:
rs.side_effect = requests.exceptions.RequestException()
with self.assertRaises(PanoptesPingRequestException):
PanoptesPingConnectionFactory.get_ping_connection(resource, plugin_context)
def test_ping_unknown_host(self):
self._resource_dict['resource_endpoint'] = 'localhostx'
resource = PanoptesResource.resource_from_dict(self._resource_dict)
unknown_host_response = "ping: cannot resolve {}: Unknown host".format(self._resource_dict['resource_endpoint'])
e = subprocess.CalledProcessError(returncode=None, cmd=None,
output=unknown_host_response) # Shouldn't I be mocking this?
# When I try: "TypeError: exceptions must be old-style classes or derived from BaseException, not Mock"
subprocess.check_output = Mock(side_effect=e)
test_fail_d = {"round_trip_stddev": None, "packets_received": None, "execution_time": None,
"round_trip_avg": None, "packets_transmitted": None, "packet_loss_pct": None,
"round_trip_max": None, "round_trip_min": None}
with self.assertRaises(PanoptesPingException):
test_unknown_host_ping = PanoptesPingConnectionFactory.get_ping_connection(resource=resource,
context=None,
count=3,
timeout=100)
self.assertDictEqual(test_fail_d, test_unknown_host_ping.response)
def test_ping_timeout(self):
e = subprocess.CalledProcessError(returncode=None, cmd=None, output=TEST_PING_RESPONSE)
# When I try: "TypeError: exceptions must be old-style classes or derived from BaseException, not Mock"
subprocess.check_output = Mock(side_effect=e)
test_d = {"round_trip_stddev": 0.057, "packets_received": 3, "execution_time": 1.44,
"round_trip_avg": 0.120, "packets_transmitted": 3, "packet_loss_pct": 0.0,
"round_trip_max": 0.162, "round_trip_min": 0.040}
with self.assertRaises(PanoptesPingTimeoutException):
resource = PanoptesResource.resource_from_dict(self._resource_dict)
test_timeout_ping = PanoptesPingConnectionFactory.get_ping_connection(resource=resource,
context=None,
count=3,
timeout=100)
self.assertDictEqual(test_d, json.loads(test_timeout_ping.response))
self.assertEqual(test_timeout_ping.packets_transmitted, 3)
self.assertEqual(test_timeout_ping.packets_received, 3)
self.assertEqual(test_timeout_ping.execution_time, 1.44)
self.assertEqual(test_timeout_ping.round_trip_min, 0.040)
self.assertEqual(test_timeout_ping.round_trip_avg, 0.120)
self.assertEqual(test_timeout_ping.round_trip_max, 0.162)
self.assertEqual(test_timeout_ping.round_trip_stddev, 0.057)
def handle_resources(plugin_signature, resources):
resource_site = str(resources[u'resources'][0][u'resource_site'])
plugin_name = str(plugin_signature.split(':')[0])
logger.info(u'For plugin "%s" and site "%s", going to get current set' %
(plugin_name, resource_site))
try:
current_resource_set = resource_store.get_resources(
site=resource_site, plugin_name=plugin_name)
except:
logger.error(u'Error trying to get current resources for plugin "%s"' %
plugin_signature)
return False
logger.info(u'For plugin "%s" and site "%s" current set has %d resources' %
(plugin_signature, resource_site, len(current_resource_set)))
new_resource_set = PanoptesResourceSet()
for resource in resources[u'resources']:
new_resource = PanoptesResource.resource_from_dict(resource)
new_resource_set.add(new_resource)
current_resource_dict = _resource_set_to_dictionary(current_resource_set)
new_resource_dict = _resource_set_to_dictionary(new_resource_set)
logger.info(u'For plugin "%s" and site "%s", new set has %d resources' %
(plugin_signature, resource_site, len(new_resource_set)))
resources_to_delete = current_resource_set.resources.difference(
new_resource_set)
logger.info(u'For plugin "%s" and site "%s", deleting %d resources' %
(plugin_signature, resource_site, len(resources_to_delete)))
for resource in resources_to_delete:
current_timestamp = current_resource_dict[
resource.serialization_key].resource_creation_timestamp
new_timestamp = resources[u'resource_set_creation_timestamp']
if current_timestamp > new_timestamp:
logger.info(
u'For plugin "%s" and site "%s", resource "%s" has timestamp (%0.2f UTC) greater than of '
u'new set (%0.2f UTC), skipping deletion' %
(plugin_signature, resource_site, str(resource),
current_timestamp, new_timestamp))
else:
logger.info(
u'For plugin "%s" and site "%s", going to delete resource "%s"'
% (plugin_signature, resource_site, resource))
try:
resource_store.delete_resource(plugin_signature, resource)
except:
logger.exception(u'Error trying to delete resource')
return False
resources_to_add = new_resource_set.resources.difference(
current_resource_set)
logger.info(u'For plugin "%s" and site "%s", adding %d resources' %
(plugin_signature, resource_site, len(resources_to_add)))
for resource in resources_to_add:
logger.debug(u'Going to add resource "%s"' % resource)
try:
resource_store.add_resource(plugin_signature, resource)
except:
logger.exception(u'Error trying to add resource')
return False
resources_to_update = current_resource_set.resources.intersection(
new_resource_set)
logger.info(u'For plugin "%s" and site "%s", updating %d resources' %
(plugin_signature, resource_site, len(resources_to_update)))
start_time = time.time()
resources_skipped = 0
resources_updated = 0
for resource in resources_to_update:
current_timestamp = current_resource_dict[
resource.serialization_key].resource_creation_timestamp
new_timestamp = new_resource_dict[
resource.serialization_key].resource_creation_timestamp
if current_timestamp > new_timestamp:
logger.info(
u'For plugin "%s" and site "%s", resource "%s" has timestamp (%0.2f UTC) greater than of '
u'new set (%0.2f UTC), skipping update' %
(plugin_signature, resource_site, str(resource),
current_timestamp, new_timestamp))
resources_skipped += 1
else:
try:
new_resource = new_resource_dict[resource.serialization_key]
logger.debug(u'Going to update resource "%s"' % new_resource)
resource_store.add_resource(plugin_signature, new_resource)
resources_updated += 1
except:
logger.exception(u'Error trying to add resource')
return False
end_time = time.time()
logger.info(
u'For plugin "%s" and site "%s", added/updated %d resources in %0.2f seconds, skipped %d resources'
% (plugin_signature, resource_site, resources_updated,
round(end_time - start_time, 2), resources_skipped))
return True
def test_panoptes_resource(self):
panoptes_resource_metadata = self.__panoptes_resource_metadata
panoptes_resource = self.__panoptes_resource
self.assertIsInstance(panoptes_resource, PanoptesResource)
self.assertEqual(panoptes_resource.resource_site, 'test')
self.assertEqual(panoptes_resource.resource_class, 'test')
self.assertEqual(panoptes_resource.resource_subclass, 'test')
self.assertEqual(panoptes_resource.resource_type, 'test')
self.assertEqual(panoptes_resource.resource_id, 'test')
self.assertEqual(panoptes_resource.resource_endpoint, 'test')
self.assertEqual(panoptes_resource.resource_metadata, panoptes_resource_metadata)
self.assertEqual(panoptes_resource.resource_ttl, str(RESOURCE_MANAGER_RESOURCE_EXPIRE))
self.assertEqual(panoptes_resource, panoptes_resource)
self.assertFalse(panoptes_resource == '1')
self.assertIsInstance(str(panoptes_resource), str)
with self.assertRaises(AssertionError):
panoptes_resource.add_metadata(None, 'test')
with self.assertRaises(AssertionError):
panoptes_resource.add_metadata('', 'test')
with self.assertRaises(ValueError):
panoptes_resource.add_metadata('1', 'test')
with self.assertRaises(ValueError):
panoptes_resource.add_metadata('key', 'test|')
with self.assertRaises(AssertionError):
PanoptesResource(resource_site='', resource_class='test', resource_subclass='test',
resource_type='test', resource_id='test', resource_endpoint='test')
with self.assertRaises(AssertionError):
PanoptesResource(resource_site='test', resource_class='', resource_subclass='test',
resource_type='test', resource_id='test', resource_endpoint='test')
with self.assertRaises(AssertionError):
PanoptesResource(resource_site='test', resource_class='test', resource_subclass='',
resource_type='test', resource_id='test', resource_endpoint='test')
with self.assertRaises(AssertionError):
PanoptesResource(resource_site='test', resource_class='test', resource_subclass='test',
resource_type='', resource_id='test', resource_endpoint='test')
with self.assertRaises(AssertionError):
PanoptesResource(resource_site='test', resource_class='test', resource_subclass='test',
resource_type='test', resource_id='', resource_endpoint='test')
with self.assertRaises(AssertionError):
PanoptesResource(resource_site=None, resource_class='test', resource_subclass='test',
resource_type='test', resource_id='test', resource_endpoint='test')
with self.assertRaises(AssertionError):
PanoptesResource(resource_site='test', resource_class=None, resource_subclass='test',
resource_type='test', resource_id='test', resource_endpoint='test')
with self.assertRaises(AssertionError):
PanoptesResource(resource_site='test', resource_class='test', resource_subclass=None,
resource_type='test', resource_id='test', resource_endpoint='test')
with self.assertRaises(AssertionError):
PanoptesResource(resource_site='test', resource_class='test', resource_subclass='test',
resource_type=None, resource_id='test', resource_endpoint='test')
with self.assertRaises(AssertionError):
PanoptesResource(resource_site='test', resource_class='test', resource_subclass='test',
resource_type='test', resource_id=None, resource_endpoint='test')
with self.assertRaises(AssertionError):
PanoptesResource(resource_site='test', resource_class='test', resource_subclass='test',
resource_type='test', resource_id='test', resource_endpoint=None)
# Test json and raw representations of PanoptesResource
panoptes_resource_2 = PanoptesResource(resource_site='test', resource_class='test',
resource_subclass='test',
resource_type='test', resource_id='test', resource_endpoint='test',
resource_creation_timestamp=_TIMESTAMP,
resource_plugin='test')
self.assertEqual(panoptes_resource_2.resource_creation_timestamp, _TIMESTAMP)
panoptes_resource_2_json = {
u'resource_site': u'test',
u'resource_id': u'test',
u'resource_class': u'test',
u'resource_plugin': u'test',
u'resource_creation_timestamp': _TIMESTAMP,
u'resource_subclass': u'test',
u'resource_endpoint': u'test',
u'resource_metadata': {
u'_resource_ttl': u'604800'
},
u'resource_type': u'test'
}
self.assertEqual(ordered(json.loads(panoptes_resource_2.json)), ordered(panoptes_resource_2_json))
panoptes_resource_2_raw = collections.OrderedDict(
[('resource_site', 'test'),
('resource_class', 'test'),
('resource_subclass', 'test'),
('resource_type', 'test'),
('resource_id', 'test'),
('resource_endpoint', 'test'),
('resource_metadata', collections.OrderedDict(
[('_resource_ttl', '604800')])
),
('resource_creation_timestamp', _TIMESTAMP),
('resource_plugin', 'test')])
self.assertEqual(panoptes_resource_2.raw, panoptes_resource_2_raw)
# Test resource creation from dict
with open('tests/test_resources/input/resource_one.json') as f:
resource_specs = json.load(f)
resource_from_json = PanoptesResource.resource_from_dict(resource_specs['resources'][0])
panoptes_resource_3 = PanoptesResource(resource_site="test_site", resource_class="network",
resource_subclass="test_subclass", resource_type="test_type",
resource_id="test_id_1", resource_endpoint="test_endpoint_1",
resource_plugin="key")
self.assertEqual(resource_from_json, panoptes_resource_3)