def test_load_plugin_without_params(self): plugin_config = { 'plugin': 'cloudmarker.test.test_util.MockPlugin' } plugin = util.load_plugin(plugin_config) self.assertIsInstance(plugin, MockPlugin) self.assertEqual(plugin.a, 1) self.assertEqual(plugin.b, 2)
def test_load_plugin_with_params(self): plugin_config = { 'plugin': 'cloudmarker.test.test_util.MockPlugin', 'params': { 'a': 3, 'b': 4, } } plugin = util.load_plugin(plugin_config) self.assertIsInstance(plugin, MockPlugin) self.assertEqual(plugin.a, 3) self.assertEqual(plugin.b, 4)
def _write_worker(audit_key, audit_version, plugin_key, plugin_config, input_queue, worker_type): """Worker function for store and alert plugins. Arguments: audit_key (str): Audit key name in configuration audit_version (str): Audit version string. plugin_key (str): Plugin key name in configuration. plugin_config (dict): Store or alert plugin config dictionary. input_queue (multiprocessing.Queue): Queue to read records from. worker_type (str): Either ``'store'`` or ``'alert'``. """ worker_name = audit_key + '_' + plugin_key _log.info('%s_worker: %s: Started', worker_type, worker_name) try: plugin = util.load_plugin(plugin_config) except Exception as e: _log.exception('%s_worker: %s: Failed; error: %s: %s', worker_type, worker_name, type(e).__name__, e) _log.info('%s_worker: %s: Stopped', worker_type, worker_name) return while plugin is not None: try: record = input_queue.get() if record is None: _log.info('%s_worker: %s: Stopping', worker_type, worker_name) plugin.done() break record['com'] = util.merge_dicts( record.get('com', {}), { 'audit_key': audit_key, 'audit_version': audit_version, 'target_key': plugin_key, 'target_class': type(plugin).__name__, 'target_worker': worker_name, 'target_type': worker_type, }) plugin.write(record) except Exception as e: _log.exception('%s_worker: %s: Failed; error: %s: %s', worker_type, worker_name, type(e).__name__, e) _log.info('%s_worker: %s: Stopped', worker_type, worker_name)
def cloud_worker(audit_key, audit_version, plugin_key, plugin_config, output_queues): """Worker function for cloud plugins. This function instantiates a plugin object from the ``plugin_config`` dictionary. This function expects the plugin object to implement a ``read`` method that yields records. This function calls this ``read`` method to retrieve records and puts each record into each queue in ``output_queues``. Arguments: audit_key (str): Audit key name in configuration. audit_version (str): Audit version string. plugin_key (str): Plugin key name in configuration. plugin_config (dict): Cloud plugin config dictionary. output_queues (list): List of :class:`multiprocessing.Queue` objects to write records to. """ worker_name = audit_key + '_' + plugin_key _log.info('cloud_worker: %s: Started', worker_name) try: plugin = util.load_plugin(plugin_config) for record in plugin.read(): record['com'] = util.merge_dicts( record.get('com', {}), { 'audit_key': audit_key, 'audit_version': audit_version, 'origin_key': plugin_key, 'origin_class': type(plugin).__name__, 'origin_worker': worker_name, 'origin_type': 'cloud', }) for q in output_queues: q.put(record) plugin.done() except Exception as e: _log.exception('cloud_worker: %s: Failed; error: %s: %s', worker_name, type(e).__name__, e) _log.info('cloud_worker: %s: Stopped', worker_name)
def __init__(self, audit_key, audit_version, config): """Create an instance of :class:`Audit` from configuration. A single audit definition (from a list of audit definitions under the ``audits`` key in the configuration) is instantiated. Each audit definition contains lists of cloud plugins, store plugins, event plugins, and alert plugins. These plugins are instantiated and multiprocessing queues are set up to take records from one plugin and feed them to another plugin as per the audit workflow. Arguments: audit_key (str): Key name for an audit configuration. This key is looked for in ``config['audits']``. audit_version (str): Audit version string. config (dict): Configuration dictionary. This is the entire configuration dictionary that contains top-level keys named ``clouds``, ``stores``, ``events``, ``alerts``, ``audits``, ``run``, etc. """ self._start_time = time.localtime() self._audit_key = audit_key self._audit_version = audit_version self._config = config audit_config = config['audits'][audit_key] # We keep all workers in these lists. self._cloud_workers = [] self._store_workers = [] self._event_workers = [] self._alert_workers = [] # We keep all queues in these lists. self._store_queues = [] self._event_queues = [] self._alert_queues = [] # Create alert workers and queues. for plugin_key in audit_config['alerts']: input_queue = mp.Queue() args = ( audit_key, audit_version, plugin_key, util.load_plugin(config['plugins'][plugin_key]), input_queue, ) worker = mp.Process(target=workers.alert_worker, args=args) self._alert_workers.append(worker) self._alert_queues.append(input_queue) # Create event_workers workers and queues. for plugin_key in audit_config['events']: input_queue = mp.Queue() args = ( audit_key, audit_version, plugin_key, util.load_plugin(config['plugins'][plugin_key]), input_queue, self._alert_queues, ) worker = mp.Process(target=workers.event_worker, args=args) self._event_workers.append(worker) self._event_queues.append(input_queue) # Create store workers and queues. for plugin_key in audit_config['stores']: input_queue = mp.Queue() args = ( audit_key, audit_version, plugin_key, util.load_plugin(config['plugins'][plugin_key]), input_queue, ) worker = mp.Process(target=workers.store_worker, args=args) self._store_workers.append(worker) self._store_queues.append(input_queue) # Create cloud workers. for plugin_key in audit_config['clouds']: args = (audit_key, audit_version, plugin_key, util.load_plugin(config['plugins'][plugin_key]), self._store_queues + self._event_queues) worker = mp.Process(target=workers.cloud_worker, args=args) self._cloud_workers.append(worker)
def test_load_plugin_missing_error(self): plugin_config = {'plugin': 'cloudmarker.test.test_util.MissingPlugin'} with self.assertRaises(AttributeError): util.load_plugin(plugin_config)
def test_load_plugin_syntax_error(self): with self.assertRaises(util.PluginError): util.load_plugin({'plugin': 'foo'})
def __init__(self, audit_name, config): """Initialize a single audit manager from configuration. Arguments: audit_name (str): Key name for an audit configuration. This key is looked for in ``config['audits']``. config (dict): Configuration dictionary. This is the entire configuration dictionary that contains top-level keys named ``clouds``, ``stores``, ``checks``, ``alerts``, ``audits``, ``run``, etc. """ audit_config = config['audits'][audit_name] # We keep all workers in these lists. self._cloud_workers = [] self._store_workers = [] self._check_workers = [] self._alert_workers = [] # We keep all queues in these lists. self._store_queues = [] self._check_queues = [] self._alert_queues = [] # Create alert workers and queues. for name in audit_config['alerts']: input_queue = mp.Queue() args = ( audit_name + '-' + name, util.load_plugin(config['alerts'][name]), input_queue, ) worker = mp.Process(target=workers.store_worker, args=args) self._alert_workers.append(worker) self._alert_queues.append(input_queue) # Create check workers and queues. for name in audit_config['checks']: input_queue = mp.Queue() args = ( audit_name + '-' + name, util.load_plugin(config['checks'][name]), input_queue, self._alert_queues, ) worker = mp.Process(target=workers.check_worker, args=args) self._check_workers.append(worker) self._check_queues.append(input_queue) # Create store workers and queues. for name in audit_config['stores']: input_queue = mp.Queue() args = ( audit_name + '-' + name, util.load_plugin(config['stores'][name]), input_queue, ) worker = mp.Process(target=workers.store_worker, args=args) self._store_workers.append(worker) self._store_queues.append(input_queue) # Create cloud workers. for name in audit_config['clouds']: args = ( audit_name + '-' + name, util.load_plugin(config['clouds'][name]), self._store_queues + self._check_queues ) worker = mp.Process(target=workers.cloud_worker, args=args) self._cloud_workers.append(worker)
def event_worker(audit_key, audit_version, plugin_key, plugin_config, input_queue, output_queues): """Worker function for event plugins. This function instantiates a plugin object from the ``plugin_config`` dictionary. This function expects the plugin object to implement an ``eval`` method that accepts a single record as a parameter and yields one or more records, and a ``done`` method to perform cleanup work in the end. This function gets records from ``input_queue`` and passes each record to the ``eval`` method of the plugin object. Then it puts each record yielded by the ``eval`` method into each queue in ``output_queues``. When there are no more records in the ``input_queue``, i.e., once ``None`` is found in the ``input_queue``, this function calls the ``done`` method of the plugin object to indicate that record processing is over. Arguments: audit_key (str): Audit key name in configuration. audit_version (str): Audit version string. plugin_key (str): Plugin key name in configuration. plugin_config (dict): Event plugin config dictionary. input_queue (multiprocessing.Queue): Queue to read records from. output_queues (list): List of :class:`multiprocessing.Queue` objects to write records to. """ worker_name = audit_key + '_' + plugin_key _log.info('event_worker: %s: Started', worker_name) try: plugin = util.load_plugin(plugin_config) except Exception as e: _log.exception('event_worker: %s: Failed; error: %s: %s', worker_name, type(e).__name__, e) _log.info('event_worker: %s: Stopped', worker_name) return while True: try: record = input_queue.get() if record is None: _log.info('event_worker: %s: Stopping', worker_name) plugin.done() break for event_record in plugin.eval(record): event_record['com'] = \ util.merge_dicts(event_record.get('com', {}), { 'audit_key': audit_key, 'audit_version': audit_version, 'origin_key': plugin_key, 'origin_class': type(plugin).__name__, 'origin_worker': worker_name, 'origin_type': 'event', }) for q in output_queues: q.put(event_record) except Exception as e: _log.exception('event_worker: %s: Failed; error: %s: %s', worker_name, type(e).__name__, e) _log.info('event_worker: %s: Stopped', worker_name)