def test_load_plugin_without_params(self):
plugin_config = {
'plugin': 'cloudmarker.test.test_util.MockPlugin'
}
plugin = util.load_plugin(plugin_config)
self.assertIsInstance(plugin, MockPlugin)
self.assertEqual(plugin.a, 1)
self.assertEqual(plugin.b, 2)
def test_load_plugin_with_params(self):
plugin_config = {
'plugin': 'cloudmarker.test.test_util.MockPlugin',
'params': {
'a': 3,
'b': 4,
}
}
plugin = util.load_plugin(plugin_config)
self.assertIsInstance(plugin, MockPlugin)
self.assertEqual(plugin.a, 3)
self.assertEqual(plugin.b, 4)
def _write_worker(audit_key, audit_version, plugin_key, plugin_config,
input_queue, worker_type):
"""Worker function for store and alert plugins.
Arguments:
audit_key (str): Audit key name in configuration
audit_version (str): Audit version string.
plugin_key (str): Plugin key name in configuration.
plugin_config (dict): Store or alert plugin config dictionary.
input_queue (multiprocessing.Queue): Queue to read records from.
worker_type (str): Either ``'store'`` or ``'alert'``.
"""
worker_name = audit_key + '_' + plugin_key
_log.info('%s_worker: %s: Started', worker_type, worker_name)
try:
plugin = util.load_plugin(plugin_config)
except Exception as e:
_log.exception('%s_worker: %s: Failed; error: %s: %s', worker_type,
worker_name,
type(e).__name__, e)
_log.info('%s_worker: %s: Stopped', worker_type, worker_name)
return
while plugin is not None:
try:
record = input_queue.get()
if record is None:
_log.info('%s_worker: %s: Stopping', worker_type, worker_name)
plugin.done()
break
record['com'] = util.merge_dicts(
record.get('com', {}), {
'audit_key': audit_key,
'audit_version': audit_version,
'target_key': plugin_key,
'target_class': type(plugin).__name__,
'target_worker': worker_name,
'target_type': worker_type,
})
plugin.write(record)
except Exception as e:
_log.exception('%s_worker: %s: Failed; error: %s: %s', worker_type,
worker_name,
type(e).__name__, e)
_log.info('%s_worker: %s: Stopped', worker_type, worker_name)
def cloud_worker(audit_key, audit_version, plugin_key, plugin_config,
output_queues):
"""Worker function for cloud plugins.
This function instantiates a plugin object from the
``plugin_config`` dictionary. This function expects the plugin
object to implement a ``read`` method that yields records. This
function calls this ``read`` method to retrieve records and puts
each record into each queue in ``output_queues``.
Arguments:
audit_key (str): Audit key name in configuration.
audit_version (str): Audit version string.
plugin_key (str): Plugin key name in configuration.
plugin_config (dict): Cloud plugin config dictionary.
output_queues (list): List of :class:`multiprocessing.Queue`
objects to write records to.
"""
worker_name = audit_key + '_' + plugin_key
_log.info('cloud_worker: %s: Started', worker_name)
try:
plugin = util.load_plugin(plugin_config)
for record in plugin.read():
record['com'] = util.merge_dicts(
record.get('com', {}), {
'audit_key': audit_key,
'audit_version': audit_version,
'origin_key': plugin_key,
'origin_class': type(plugin).__name__,
'origin_worker': worker_name,
'origin_type': 'cloud',
})
for q in output_queues:
q.put(record)
plugin.done()
except Exception as e:
_log.exception('cloud_worker: %s: Failed; error: %s: %s', worker_name,
type(e).__name__, e)
_log.info('cloud_worker: %s: Stopped', worker_name)
def __init__(self, audit_key, audit_version, config):
"""Create an instance of :class:`Audit` from configuration.
A single audit definition (from a list of audit definitions
under the ``audits`` key in the configuration) is instantiated.
Each audit definition contains lists of cloud plugins, store
plugins, event plugins, and alert plugins. These plugins are
instantiated and multiprocessing queues are set up to take
records from one plugin and feed them to another plugin as per
the audit workflow.
Arguments:
audit_key (str): Key name for an audit configuration. This
key is looked for in ``config['audits']``.
audit_version (str): Audit version string.
config (dict): Configuration dictionary. This is the
entire configuration dictionary that contains
top-level keys named ``clouds``, ``stores``, ``events``,
``alerts``, ``audits``, ``run``, etc.
"""
self._start_time = time.localtime()
self._audit_key = audit_key
self._audit_version = audit_version
self._config = config
audit_config = config['audits'][audit_key]
# We keep all workers in these lists.
self._cloud_workers = []
self._store_workers = []
self._event_workers = []
self._alert_workers = []
# We keep all queues in these lists.
self._store_queues = []
self._event_queues = []
self._alert_queues = []
# Create alert workers and queues.
for plugin_key in audit_config['alerts']:
input_queue = mp.Queue()
args = (
audit_key,
audit_version,
plugin_key,
util.load_plugin(config['plugins'][plugin_key]),
input_queue,
)
worker = mp.Process(target=workers.alert_worker, args=args)
self._alert_workers.append(worker)
self._alert_queues.append(input_queue)
# Create event_workers workers and queues.
for plugin_key in audit_config['events']:
input_queue = mp.Queue()
args = (
audit_key,
audit_version,
plugin_key,
util.load_plugin(config['plugins'][plugin_key]),
input_queue,
self._alert_queues,
)
worker = mp.Process(target=workers.event_worker, args=args)
self._event_workers.append(worker)
self._event_queues.append(input_queue)
# Create store workers and queues.
for plugin_key in audit_config['stores']:
input_queue = mp.Queue()
args = (
audit_key,
audit_version,
plugin_key,
util.load_plugin(config['plugins'][plugin_key]),
input_queue,
)
worker = mp.Process(target=workers.store_worker, args=args)
self._store_workers.append(worker)
self._store_queues.append(input_queue)
# Create cloud workers.
for plugin_key in audit_config['clouds']:
args = (audit_key, audit_version, plugin_key,
util.load_plugin(config['plugins'][plugin_key]),
self._store_queues + self._event_queues)
worker = mp.Process(target=workers.cloud_worker, args=args)
self._cloud_workers.append(worker)
def test_load_plugin_missing_error(self):
plugin_config = {'plugin': 'cloudmarker.test.test_util.MissingPlugin'}
with self.assertRaises(AttributeError):
util.load_plugin(plugin_config)
def test_load_plugin_syntax_error(self):
with self.assertRaises(util.PluginError):
util.load_plugin({'plugin': 'foo'})
def __init__(self, audit_name, config):
"""Initialize a single audit manager from configuration.
Arguments:
audit_name (str): Key name for an audit configuration. This
key is looked for in ``config['audits']``.
config (dict): Configuration dictionary. This is the
entire configuration dictionary that contains
top-level keys named ``clouds``, ``stores``, ``checks``,
``alerts``, ``audits``, ``run``, etc.
"""
audit_config = config['audits'][audit_name]
# We keep all workers in these lists.
self._cloud_workers = []
self._store_workers = []
self._check_workers = []
self._alert_workers = []
# We keep all queues in these lists.
self._store_queues = []
self._check_queues = []
self._alert_queues = []
# Create alert workers and queues.
for name in audit_config['alerts']:
input_queue = mp.Queue()
args = (
audit_name + '-' + name,
util.load_plugin(config['alerts'][name]),
input_queue,
)
worker = mp.Process(target=workers.store_worker, args=args)
self._alert_workers.append(worker)
self._alert_queues.append(input_queue)
# Create check workers and queues.
for name in audit_config['checks']:
input_queue = mp.Queue()
args = (
audit_name + '-' + name,
util.load_plugin(config['checks'][name]),
input_queue,
self._alert_queues,
)
worker = mp.Process(target=workers.check_worker, args=args)
self._check_workers.append(worker)
self._check_queues.append(input_queue)
# Create store workers and queues.
for name in audit_config['stores']:
input_queue = mp.Queue()
args = (
audit_name + '-' + name,
util.load_plugin(config['stores'][name]),
input_queue,
)
worker = mp.Process(target=workers.store_worker, args=args)
self._store_workers.append(worker)
self._store_queues.append(input_queue)
# Create cloud workers.
for name in audit_config['clouds']:
args = (
audit_name + '-' + name,
util.load_plugin(config['clouds'][name]),
self._store_queues + self._check_queues
)
worker = mp.Process(target=workers.cloud_worker, args=args)
self._cloud_workers.append(worker)
def event_worker(audit_key, audit_version, plugin_key, plugin_config,
input_queue, output_queues):
"""Worker function for event plugins.
This function instantiates a plugin object from the
``plugin_config`` dictionary. This function expects the plugin
object to implement an ``eval`` method that accepts a single record
as a parameter and yields one or more records, and a ``done`` method
to perform cleanup work in the end.
This function gets records from ``input_queue`` and passes each
record to the ``eval`` method of the plugin object. Then it puts
each record yielded by the ``eval`` method into each queue in
``output_queues``.
When there are no more records in the ``input_queue``, i.e., once
``None`` is found in the ``input_queue``, this function calls the
``done`` method of the plugin object to indicate that record
processing is over.
Arguments:
audit_key (str): Audit key name in configuration.
audit_version (str): Audit version string.
plugin_key (str): Plugin key name in configuration.
plugin_config (dict): Event plugin config dictionary.
input_queue (multiprocessing.Queue): Queue to read records from.
output_queues (list): List of :class:`multiprocessing.Queue`
objects to write records to.
"""
worker_name = audit_key + '_' + plugin_key
_log.info('event_worker: %s: Started', worker_name)
try:
plugin = util.load_plugin(plugin_config)
except Exception as e:
_log.exception('event_worker: %s: Failed; error: %s: %s', worker_name,
type(e).__name__, e)
_log.info('event_worker: %s: Stopped', worker_name)
return
while True:
try:
record = input_queue.get()
if record is None:
_log.info('event_worker: %s: Stopping', worker_name)
plugin.done()
break
for event_record in plugin.eval(record):
event_record['com'] = \
util.merge_dicts(event_record.get('com', {}), {
'audit_key': audit_key,
'audit_version': audit_version,
'origin_key': plugin_key,
'origin_class': type(plugin).__name__,
'origin_worker': worker_name,
'origin_type': 'event',
})
for q in output_queues:
q.put(event_record)
except Exception as e:
_log.exception('event_worker: %s: Failed; error: %s: %s',
worker_name,
type(e).__name__, e)
_log.info('event_worker: %s: Stopped', worker_name)