def add_datasource(cls, item, deleted=False): req = cls.make_datasource_dict(item) driver_info = cls.validate_create_datasource(req) session = db.get_session() try: with session.begin(subtransactions=True): datasource = datasources_db.add_datasource( id_=req['id'], name=req['name'], driver=req['driver'], config=req['config'], description=req['description'], enabled=req['enabled'], session=session) datasource = cls.make_datasource_dict(datasource) cage = d6cage.d6Cage() engine = cage.service_object('engine') try: engine.create_policy(datasource['name']) except KeyError: # FIXME(arosen): we need a better exception then # key error being raised here raise DatasourceNameInUse(name=req['name']) cage.createservice(name=datasource['name'], moduleName=driver_info['module'], args=datasource['config'], module_driver=True) service = cage.service_object(req['name']) engine.set_schema(req['name'], service.get_schema()) except db_exc.DBDuplicateEntry: raise DatasourceNameInUse(name=req['name']) return cls.make_datasource_dict(datasource)
def synchronize(self): LOG.debug("Synchronizing running datasources") cage = d6cage.d6Cage() datasources = self.datasource_mgr.get_datasources(filter_secret=False) # Look for datasources in the db, but not in the cage. for configured_ds in datasources: active_ds = cage.service_object(configured_ds['name']) if active_ds is not None: if not configured_ds['enabled']: LOG.info('Datasource %s now disabled, just delete it.', configured_ds['name']) self.datasource_mgr.delete_datasource(configured_ds['id'], update_db=False) continue active_config = cage.getservice(name=configured_ds['name']) if not self._config_eq(configured_ds, active_config): LOG.debug('configured and active disagree: (%s) %s %s', strutils.mask_password(active_ds), strutils.mask_password(configured_ds), strutils.mask_password(active_config)) LOG.info('Reloading datasource: %s', strutils.mask_password(configured_ds)) self.datasource_mgr.delete_datasource(configured_ds['id'], update_db=False) self.datasource_mgr.add_datasource( configured_ds, update_db=False) else: if configured_ds['enabled']: LOG.info('Configured datasource is not active, adding: %s', strutils.mask_password(configured_ds)) self.datasource_mgr.add_datasource(configured_ds, update_db=False) else: LOG.info('Configured datasource is not active but ' + 'disabled, not adding: %s', strutils.mask_password(configured_ds)) # Look for datasources in the cage, but not in the db. This # need not compare the configuration, because the above # comparison would have already checked the configuration. configured_dicts = dict((ds['name'], ds) for ds in datasources) LOG.debug("configured dicts: %s", strutils.mask_password(configured_dicts)) LOG.debug("active services: %s", strutils.mask_password(cage.getservices())) for name, service in cage.getservices().items(): LOG.debug('active datasource: %s', service['name']) if (service['type'] == 'datasource_driver' and not configured_dicts.get(service['name'], None)): LOG.info('Active datasource is not configured, removing: %s', service['name']) cage.deleteservice(service['name']) engine = cage.service_object('engine') engine.delete_policy(service['name'])
def get_actions(cls, source_id=None): cage = d6cage.d6Cage() driver = cage.getservice(id_=source_id, type_='datasource_driver') if not driver: raise exception.NotFound('Could not find datasource %s' % source_id) return driver['object'].get_actions()
def _add_datasource_service(cls, new_ds, original_req): driver_info = cls.get_driver_info(new_ds['driver']) cage = cls.dseNode or d6cage.d6Cage() engine = cage.service_object('engine') try: LOG.debug("creating policy %s", new_ds['name']) engine.create_policy(new_ds['name'], kind=base.DATASOURCE_POLICY_TYPE) except KeyError: raise exception.DatasourceNameInUse(value=new_ds['name']) try: if cls.dseNode: cls.createservice(name=new_ds['name'], moduleName=driver_info['module'], args=original_req['config'], module_driver=True, type_='datasource_driver', id_=new_ds['id']) else: if not cage.service_object(new_ds['name']): cage.createservice(name=new_ds['name'], moduleName=driver_info['module'], args=original_req['config'], module_driver=True, type_='datasource_driver', id_=new_ds['id']) service = cage.service_object(new_ds['name']) engine.set_schema(new_ds['name'], service.get_schema()) except Exception: engine.delete_policy(new_ds['name']) raise exception.DatasourceCreationError(value=new_ds['name'])
def add_datasource(cls, item, deleted=False, update_db=True): req = cls.make_datasource_dict(item) # If update_db is True, new_id will get a new value from the db. new_id = req['id'] driver_info = cls.get_driver_info(item['driver']) session = db.get_session() try: with session.begin(subtransactions=True): LOG.debug("adding datasource %s", req['name']) if update_db: LOG.debug("updating db") datasource = datasources_db.add_datasource( id_=req['id'], name=req['name'], driver=req['driver'], config=req['config'], description=req['description'], enabled=req['enabled'], session=session) new_id = datasource['id'] cls.validate_create_datasource(req) cage = cls.dseNode or d6cage.d6Cage() engine = cage.service_object('engine') try: LOG.debug("creating policy %s", req['name']) engine.create_policy(req['name'], kind=base.DATASOURCE_POLICY_TYPE) except KeyError: # FIXME(arosen): we need a better exception then # key error being raised here raise DatasourceNameInUse(value=req['name']) try: if cls.dseNode: cls.createservice(name=req['name'], moduleName=driver_info['module'], args=item['config'], module_driver=True, type_='datasource_driver', id_=new_id) else: cage.createservice(name=req['name'], moduleName=driver_info['module'], args=item['config'], module_driver=True, type_='datasource_driver', id_=new_id) service = cage.service_object(req['name']) engine.set_schema(req['name'], service.get_schema()) except Exception: engine.delete_policy(req['name']) raise DatasourceCreationError(value=req['name']) except db_exc.DBDuplicateEntry: raise DatasourceNameInUse(value=req['name']) new_item = dict(item) new_item['id'] = new_id return cls.make_datasource_dict(new_item)
def test_communication(self): """Test for communication. Test the module's ability to be loaded into the DSE by checking its ability to communicate on the message bus. """ cage = d6cage.d6Cage() # Create modules. # Turn off polling so we don't need to deal with real data. args = helper.datasource_openstack_args() args['poll_time'] = 0 cage.loadModule("NovaDriver", helper.data_module_path("nova_driver.py")) cage.loadModule("PolicyDriver", helper.policy_module_path()) cage.createservice(name="policy", moduleName="PolicyDriver", args={ 'd6cage': cage, 'rootdir': helper.data_module_path('') }) cage.createservice(name="nova", moduleName="NovaDriver", args=args) # Check that data gets sent from nova to policy as expected nova = cage.service_object('nova') policy = cage.service_object('policy') policy.debug_mode() policy.create_policy('nova') policy.set_schema('nova', compile.Schema({'server': (1, )})) policy.subscribe('nova', 'server', callback=policy.receive_data) # publishing is slightly convoluted b/c deltas are computed # automatically. (Not just convenient--useful so that DSE # properly handles the initial state problem.) # Need to set nova.state and nova.prior_state and then publish # anything. # publish server(1), server(2), server(3) helper.retry_check_subscribers(nova, [(policy.name, 'server')]) nova.prior_state = {} nova.state['server'] = set([(1, ), (2, ), (3, )]) nova.publish('server', None) helper.retry_check_db_equal( policy, 'nova:server(x)', 'nova:server(1) nova:server(2) nova:server(3)') # publish server(1), server(4), server(5) nova.prior_state['server'] = nova.state['server'] nova.state['server'] = set([(1, ), (4, ), (5, )]) nova.publish('server', None) helper.retry_check_db_equal( policy, 'nova:server(x)', 'nova:server(1) nova:server(4) nova:server(5)')
def test_communication(self): """Test for communication. Test the module's ability to be loaded into the DSE by checking its ability to communicate on the message bus. """ cage = d6cage.d6Cage() # Create modules. # Turn off polling so we don't need to deal with real data. args = helper.datasource_openstack_args() args['poll_time'] = 0 cage.loadModule("NovaDriver", helper.data_module_path("nova_driver.py")) cage.loadModule("PolicyDriver", helper.policy_module_path()) cage.createservice(name="policy", moduleName="PolicyDriver", args={'d6cage': cage, 'rootdir': helper.data_module_path(''), 'log_actions_only': True}) cage.createservice(name="nova", moduleName="NovaDriver", args=args) # Check that data gets sent from nova to policy as expected nova = cage.service_object('nova') policy = cage.service_object('policy') policy.debug_mode() policy.create_policy('nova') policy.set_schema('nova', compile.Schema({'server': (1,)})) policy.subscribe('nova', 'server', callback=policy.receive_data) # publishing is slightly convoluted b/c deltas are computed # automatically. (Not just convenient--useful so that DSE # properly handles the initial state problem.) # Need to set nova.state and nova.prior_state and then publish # anything. # publish server(1), server(2), server(3) helper.retry_check_subscribers(nova, [(policy.name, 'server')]) nova.prior_state = {} nova.state['server'] = set([(1,), (2,), (3,)]) nova.publish('server', None) helper.retry_check_db_equal( policy, 'nova:server(x)', 'nova:server(1) nova:server(2) nova:server(3)') # publish server(1), server(4), server(5) nova.prior_state['server'] = nova.state['server'] nova.state['server'] = set([(1,), (4,), (5,)]) nova.publish('server', None) helper.retry_check_db_equal( policy, 'nova:server(x)', 'nova:server(1) nova:server(4) nova:server(5)')
def setUp(self): # create DSE and add vm-placement engine and fake datasource super(TestSetPolicy, self).setUp() self.cage = d6cage.d6Cage() config = { "vmplace": {"module": "congress/policy_engines/vm_placement.py"}, "fake": {"poll_time": 0, "module": "congress/tests/fake_datasource.py"}, } harness.load_data_service("vmplace", config["vmplace"], self.cage, helper.root_path(), 1) harness.load_data_service("fake", config["fake"], self.cage, helper.root_path(), 2) self.vmplace = self.cage.service_object("vmplace") self.vmplace.debug_mode() self.fake = self.cage.service_object("fake")
def delete_datasource(cls, datasource_id): datasource = cls.get_datasource(datasource_id) session = db.get_session() with session.begin(subtransactions=True): cage = d6cage.d6Cage() engine = cage.service_object('engine') try: engine.delete_policy(datasource['name'], disallow_dangling_refs=True) except exception.DanglingReference as e: raise e except KeyError: raise DatasourceNotFound(id=datasource_id) result = datasources_db.delete_datasource( datasource_id, session) if not result: raise DatasourceNotFound(id=datasource_id) cage.deleteservice(datasource['name'])
def add_datasource(cls, item, deleted=False, update_db=True): req = cls.make_datasource_dict(item) # If update_db is True, new_id will get a new value from the db. new_id = req['id'] driver_info = cls.get_driver_info(item['driver']) session = db.get_session() try: with session.begin(subtransactions=True): LOG.debug("adding datasource %s", req['name']) if update_db: LOG.debug("updating db") datasource = datasources_db.add_datasource( id_=req['id'], name=req['name'], driver=req['driver'], config=req['config'], description=req['description'], enabled=req['enabled'], session=session) new_id = datasource['id'] cage = d6cage.d6Cage() engine = cage.service_object('engine') try: LOG.debug("creating policy %s", req['name']) engine.create_policy(req['name']) except KeyError: # FIXME(arosen): we need a better exception then # key error being raised here raise DatasourceNameInUse(name=req['name']) cage.createservice(name=req['name'], moduleName=driver_info['module'], args=item['config'], module_driver=True, type_='datasource_driver', id_=new_id) service = cage.service_object(req['name']) engine.set_schema(req['name'], service.get_schema()) except db_exc.DBDuplicateEntry: raise DatasourceNameInUse(name=req['name']) new_item = dict(item) new_item['id'] = new_id return cls.make_datasource_dict(new_item)
def synchronize_policies(self): LOG.debug("Synchronizing policies") # Read policies from DB. cage = d6cage.d6Cage() configured_policies = [{'id': p.id, 'name': p.name, 'abbr': p.abbreviation, 'desc': p.description, 'owner': p.owner, 'kind': p.kind} for p in db_policy_rules.get_policies()] # Read policies from engine engine = cage.service_object('engine') policies = [engine.policy_object(n) for n in engine.policy_names()] active_policies = [] for policy in policies: active_policies.append({'id': policy.id, 'name': policy.name, 'abbr': policy.abbr, 'desc': policy.desc, 'owner': policy.owner, 'kind': policy.kind}) added = 0 removed = 0 for p in active_policies: if (p['kind'] != base.DATASOURCE_POLICY_TYPE and p not in configured_policies): LOG.debug("removing policy %s", str(p)) engine.delete_policy(p['id']) removed = removed + 1 for p in configured_policies: if p not in active_policies: LOG.debug("adding policy %s", str(p)) engine.create_policy(p['name'], id_=p['id'], abbr=p['abbr'], kind=p['kind'], desc=p['desc'], owner=p['owner']) added = added + 1 LOG.debug("synchronize_policies, added %d removed %d", added, removed)
def delete_datasource(cls, datasource_id, update_db=True): datasource = cls.get_datasource(datasource_id) session = db.get_session() with session.begin(subtransactions=True): cage = d6cage.d6Cage() engine = cage.service_object('engine') try: engine.delete_policy(datasource['name'], disallow_dangling_refs=True) except exception.DanglingReference as e: raise e except KeyError: raise DatasourceNotFound(id=datasource_id) if update_db: result = datasources_db.delete_datasource( datasource_id, session) if not result: raise DatasourceNotFound(id=datasource_id) cage.deleteservice(datasource['name'])
def synchronize(self): LOG.debug("Synchronizing running datasources") cage = d6cage.d6Cage() datasources = self.datasource_mgr.get_datasources(filter_secret=False) # Look for datasources in the db, but not in the cage. for configured_ds in datasources: active_ds = cage.service_object(configured_ds['name']) if active_ds is not None: active_config = cage.getservice(name=configured_ds['name']) if not self._config_eq(configured_ds, active_config): LOG.debug('configured and active disagree: (%s) %s %s', str(active_ds), str(configured_ds), str(active_config)) LOG.info('Reloading datasource: %s', str(configured_ds)) self.datasource_mgr.delete_datasource(configured_ds['id'], update_db=False) self.datasource_mgr.add_datasource(configured_ds, update_db=False) else: LOG.info('Configured datasource is not active, adding: %s', str(configured_ds)) self.datasource_mgr.add_datasource(configured_ds, update_db=False) # Look for datasources in the cage, but not in the db. This # need not compare the configuration, because the above # comparison would have already checked the configuration. configured_dicts = dict((ds['name'], ds) for ds in datasources) LOG.debug("configured dicts: %s", str(configured_dicts)) LOG.debug("active services: %s", str(cage.getservices())) for name, service in cage.getservices().items(): LOG.debug('active datasource: %s', service['name']) if (service['type'] == 'datasource_driver' and not configured_dicts.get(service['name'], None)): LOG.info('Active datasource is not configured, removing: %s', service['name']) cage.deleteservice(service['name']) engine = cage.service_object('engine') engine.delete_policy(service['name'])
def setUp(self): # create DSE and add vm-placement engine and fake datasource super(TestSetPolicy, self).setUp() self.cage = d6cage.d6Cage() config = { 'vmplace': { 'module': "congress/policy_engines/vm_placement.py" }, 'fake': { 'poll_time': 0, 'module': "congress/tests/fake_datasource.py" } } harness.load_data_service("vmplace", config['vmplace'], self.cage, helper.root_path(), 1) harness.load_data_service("fake", config['fake'], self.cage, helper.root_path(), 2) self.vmplace = self.cage.service_object('vmplace') self.vmplace.debug_mode() self.fake = self.cage.service_object('fake')
def request_refresh(cls, datasource_id): datasource = cls.get_datasource(datasource_id) cage = d6cage.d6Cage() datasource = cage.service_object(datasource['name']) datasource.request_refresh()
def create(rootdir, config_override=None): """Get Congress up and running when src is installed in rootdir. i.e. ROOTDIR=/path/to/congress/congress. CONFIG_OVERRIDE is a dictionary of dictionaries with configuration values that overrides those provided in CONFIG_FILE. The top-level dictionary has keys for the CONFIG_FILE sections, and the second-level dictionaries store values for that section. """ LOG.debug("Starting Congress with rootdir=%s, config_override=%s", rootdir, config_override) # create message bus cage = d6cage.d6Cage() # read in datasource configurations cage.config = config_override or {} # path to congress source dir src_path = os.path.join(rootdir, "congress") datasource_mgr = datasource_manager.DataSourceManager() datasource_mgr.validate_configured_drivers() # add policy engine engine_path = os.path.join(src_path, "policy_engines/agnostic.py") LOG.info("main::start() engine_path: %s", engine_path) cage.loadModule("PolicyEngine", engine_path) cage.createservice( name="engine", moduleName="PolicyEngine", description="Policy Engine (DseRuntime instance)", args={'d6cage': cage, 'rootdir': src_path, 'log_actions_only': cfg.CONF.enable_execute_action}) engine = cage.service_object('engine') engine.initialize_table_subscriptions() engine.debug_mode() # should take this out for production # add policy api api_path = os.path.join(src_path, "api/policy_model.py") LOG.info("main::start() api_path: %s", api_path) cage.loadModule("API-policy", api_path) cage.createservice( name="api-policy", moduleName="API-policy", description="API-policy DSE instance", args={'policy_engine': engine}) # add rule api api_path = os.path.join(src_path, "api/rule_model.py") LOG.info("main::start() api_path: %s", api_path) cage.loadModule("API-rule", api_path) cage.createservice( name="api-rule", moduleName="API-rule", description="API-rule DSE instance", args={'policy_engine': engine}) # add table api api_path = os.path.join(src_path, "api/table_model.py") LOG.info("main::start() api_path: %s", api_path) cage.loadModule("API-table", api_path) cage.createservice( name="api-table", moduleName="API-table", description="API-table DSE instance", args={'policy_engine': engine, 'datasource_mgr': datasource_mgr}) # add row api api_path = os.path.join(src_path, "api/row_model.py") LOG.info("main::start() api_path: %s", api_path) cage.loadModule("API-row", api_path) cage.createservice( name="api-row", moduleName="API-row", description="API-row DSE instance", args={'policy_engine': engine, 'datasource_mgr': datasource_mgr}) # add status api api_path = os.path.join(src_path, "api/status_model.py") LOG.info("main::start() api_path: %s", api_path) cage.loadModule("API-status", api_path) cage.createservice( name="api-status", moduleName="API-status", description="API-status DSE instance", args={'policy_engine': engine, 'datasource_mgr': datasource_mgr}) # add action api api_path = os.path.join(src_path, "api/action_model.py") LOG.info("main::start() api_path: %s", api_path) cage.loadModule("API-action", api_path) cage.createservice( name="api-action", moduleName="API-action", description="API-action DSE instance", args={'policy_engine': engine, 'datasource_mgr': datasource_mgr}) # add schema api api_path = os.path.join(src_path, "api/schema_model.py") LOG.info("main::start() api_path: %s", api_path) cage.loadModule("API-schema", api_path) cage.createservice( name="api-schema", moduleName="API-schema", description="API-schema DSE instance", args={'datasource_mgr': datasource_mgr}) # add path for system/datasource-drivers api_path = os.path.join(src_path, "api/system/driver_model.py") LOG.info("main::start() api_path: %s", api_path) cage.loadModule("API-system", api_path) cage.createservice( name="api-system", moduleName="API-system", description="API-system DSE instance", args={'datasource_mgr': datasource_mgr}) # Load policies from database engine.persistent_load_policies() # if this is the first time we are running Congress, need # to create the default theories (which cannot be deleted) api_policy = cage.service_object('api-policy') engine.DEFAULT_THEORY = 'classification' engine.builtin_policy_names.add(engine.DEFAULT_THEORY) try: api_policy.add_item({'name': engine.DEFAULT_THEORY, 'description': 'default policy'}, {}) except KeyError: pass engine.ACTION_THEORY = 'action' engine.builtin_policy_names.add(engine.ACTION_THEORY) try: api_policy.add_item({'kind': base.ACTION_POLICY_TYPE, 'name': engine.ACTION_THEORY, 'description': 'default action policy'}, {}) except KeyError: pass # have policy-engine subscribe to api calls # TODO(thinrichs): either have API publish everything to DSE bus and # have policy engine subscribe to all those messages # OR have API interact with individual components directly # and change all tests so that the policy engine does not need to be # subscribed to 'policy-update' engine.subscribe('api-rule', 'policy-update', callback=engine.receive_policy_update) # spin up all the configured services, if we have configured them drivers = datasource_mgr.get_datasources() # Setup cage.config as it previously done when it was loaded # from disk. FIXME(arosen) later! for driver in drivers: if not driver['enabled']: LOG.info("module %s not enabled, skip loading", driver['name']) continue driver_info = datasource_mgr.get_driver_info(driver['driver']) engine.create_policy(driver['name'], kind=base.DATASOURCE_POLICY_TYPE) try: cage.createservice(name=driver['name'], moduleName=driver_info['module'], args=driver['config'], module_driver=True, type_='datasource_driver', id_=driver['id']) except d6cage.DataServiceError: # FIXME(arosen): If createservice raises congress-server # dies here. So we catch this exception so the server does # not die. We need to refactor the dse code so it just # keeps retrying the driver gracefully... continue service = cage.service_object(driver['name']) engine.set_schema(driver['name'], service.get_schema()) # Insert rules. Needs to be done after datasources are loaded # so that we can compile away column references at read time. # If datasources loaded after this, we don't have schemas. engine.persistent_load_rules() # Start datasource synchronizer after explicitly starting the # datasources, because the explicit call to create a datasource # will crash if the synchronizer creates the datasource first. synchronizer_path = os.path.join(src_path, "synchronizer.py") LOG.info("main::start() synchronizer: %s", synchronizer_path) cage.loadModule("Synchronizer", synchronizer_path) cage.createservice( name="synchronizer", moduleName="Synchronizer", description="DB synchronizer instance", args={'poll_time': cfg.CONF.datasource_sync_period}) synchronizer = cage.service_object('synchronizer') engine.set_synchronizer(synchronizer) # add datasource api api_path = os.path.join(src_path, "api/datasource_model.py") LOG.info("main::start() api_path: %s", api_path) cage.loadModule("API-datasource", api_path) cage.createservice( name="api-datasource", moduleName="API-datasource", description="API-datasource DSE instance", args={'policy_engine': engine, 'datasource_mgr': datasource_mgr, 'synchronizer': synchronizer}) return cage
def get_row_data(cls, table_id, datasource_id, **kwargs): datasource = cls.get_datasource(datasource_id) cage = d6cage.d6Cage() datasource_obj = cage.service_object(datasource['name']) return datasource_obj.get_row_data(table_id)
def update_entire_data(cls, table_id, source_id, objs): datasource = cls.get_datasource(source_id) cage = d6cage.d6Cage() datasource_obj = cage.service_object(datasource['name']) return datasource_obj.update_entire_data(table_id, objs)
def setUp(self): """Setup polling tests.""" super(TestDataSourceDriver, self).setUp() cage = d6cage.d6Cage() # Create mock of Neutron client so we can control data mock_factory = mox.Mox() neutron_client = mock_factory.CreateMock( neutronclient.v2_0.client.Client) neutron_client.list_networks().InAnyOrder(1).AndReturn(network1) neutron_client.list_ports().InAnyOrder(1).AndReturn(port_response) neutron_client.list_routers().InAnyOrder(1).AndReturn(router_response) neutron_client.list_security_groups().InAnyOrder(1).AndReturn( security_group_response) neutron_client.list_networks().InAnyOrder(2).AndReturn(network2) neutron_client.list_ports().InAnyOrder(2).AndReturn(port_response) neutron_client.list_routers().InAnyOrder(2).AndReturn(router_response) neutron_client.list_security_groups().InAnyOrder(2).AndReturn( security_group_response) mock_factory.ReplayAll() # Create modules (without auto-polling) cage.loadModule("NeutronDriver", helper.data_module_path("neutron_driver.py")) cage.loadModule("PolicyDriver", helper.policy_module_path()) cage.createservice(name="policy", moduleName="PolicyDriver", args={ 'd6cage': cage, 'rootdir': helper.data_module_path('') }) args = helper.datasource_openstack_args() args['poll_time'] = 0 args['client'] = neutron_client cage.createservice(name="neutron", moduleName="NeutronDriver", args=args) policy = cage.service_object('policy') policy.create_policy('neutron') policy.set_schema('neutron', cage.service_object('neutron').get_schema()) cage.service_object('neutron').neutron = neutron_client policy.debug_mode() # insert rule into policy to make testing easier. # (Some of the IDs are auto-generated each time we convert) policy.insert(create_network_group('p')) # create some garbage data args = helper.datasource_openstack_args() driver = neutron_driver.NeutronDriver(args=args) network_key_to_index = driver.get_column_map( neutron_driver.NeutronDriver.NETWORKS) network_max_index = max(network_key_to_index.values()) args1 = ['1'] * (network_max_index + 1) args2 = ['2'] * (network_max_index + 1) args1 = ",".join(args1) args2 = ",".join(args2) fake_networks = [ 'neutron:networks({})'.format(args1), 'neutron:networks({})'.format(args2) ] # answer to query above for network1 datalog1 = ('p("240ff9df-df35-43ae-9df5-27fae87f2492") ' 'p("340ff9df-df35-43ae-9df5-27fae87f2492") ' 'p("440ff9df-df35-43ae-9df5-27fae87f2492")') # answer to query above for network2 datalog2 = ('p("240ff9df-df35-43ae-9df5-27fae87f2492") ' 'p("640ff9df-df35-43ae-9df5-27fae87f2492") ' 'p("540ff9df-df35-43ae-9df5-27fae87f2492")') # return value self.info = {} self.info['cage'] = cage self.info['datalog1'] = datalog1 self.info['datalog2'] = datalog2 self.info['fake_networks'] = fake_networks
def __init__(self, name, keys, inbox=None, dataPath=None, policy_engine=None): super(StatusModel, self).__init__(name, keys, inbox=inbox, dataPath=dataPath) self.cage = d6cage.d6Cage() self.engine = self.cage.service_object('engine')
def create(rootdir, statedir): """Get Congress up and running when src is installed in rootdir, i.e. ROOTDIR=/path/to/congress/congress. """ LOG.debug("Starting Congress with rootdir={} and statedir={}".format( rootdir, statedir)) # create message bus cage = d6cage.d6Cage() cage.daemon = True cage.start() cage.system_service_names.add(cage.name) # add policy engine engine_path = os.path.join(rootdir, "policy/dsepolicy.py") LOG.info("main::start() engine_path: " + str(engine_path)) cage.loadModule("PolicyEngine", engine_path) cage.createservice( name="engine", moduleName="PolicyEngine", description="Policy Engine (DseRuntime instance)", args={'d6cage': cage, 'rootdir': rootdir}) engine = cage.service_object('engine') if statedir is not None: engine.load_dir(statedir) engine.initialize_table_subscriptions() cage.system_service_names.add(engine.name) # add policy api # TODO(thinrichs): change to real API path. api_path = os.path.join(rootdir, "api/policy_model.py") LOG.info("main::start() api_path: " + str(api_path)) cage.loadModule("API-policy", api_path) cage.createservice( name="api-policy", moduleName="API-policy", description="API-policy DSE instance", args={'policy_engine': engine}) cage.system_service_names.add('api-policy') # add rule api api_path = os.path.join(rootdir, "api/rule_model.py") LOG.info("main::start() api_path: " + str(api_path)) cage.loadModule("API-rule", api_path) cage.createservice( name="api-rule", moduleName="API-rule", description="API-rule DSE instance", args={'policy_engine': engine}) cage.system_service_names.add('api-rule') # add table api api_path = os.path.join(rootdir, "api/table_model.py") LOG.info("main::start() api_path: " + str(api_path)) cage.loadModule("API-table", api_path) cage.createservice( name="api-table", moduleName="API-table", description="API-table DSE instance", args={'policy_engine': engine}) cage.system_service_names.add('api-table') # add row api api_path = os.path.join(rootdir, "api/row_model.py") LOG.info("main::start() api_path: " + str(api_path)) cage.loadModule("API-row", api_path) cage.createservice( name="api-row", moduleName="API-row", description="API-row DSE instance", args={'policy_engine': engine}) cage.system_service_names.add('api-row') # add datasource api api_path = os.path.join(rootdir, "api/datasource_model.py") LOG.info("main::start() api_path: " + str(api_path)) cage.loadModule("API-datasource", api_path) cage.createservice( name="api-datasource", moduleName="API-datasource", description="API-datasource DSE instance", args={'policy_engine': engine}) cage.system_service_names.add('api-datasource') # have policy-engine subscribe to api calls # TODO(thinrichs): either have API publish everything to DSE bus and # have policy engine subscribe to all those messages # OR have API interact with individual components directly # and change all tests so that the policy engine does not need to be # subscribed to 'policy-update' engine.subscribe('api-rule', 'policy-update', callback=engine.receive_policy_update) return cage
def synchronize_rules(self): LOG.debug("Synchronizing rules") # Read rules from DB. cage = d6cage.d6Cage() configured_rules = [{'rule': r.rule, 'id': r.id, 'comment': r.comment, 'name': r.name, 'policy_name': r.policy_name} for r in db_policy_rules.get_policy_rules()] # Read rules from engine engine = cage.service_object('engine') policies = {n: engine.policy_object(n) for n in engine.policy_names()} active_policy_rules = [] for policy_name, policy in policies.items(): if policy.kind != base.DATASOURCE_POLICY_TYPE: for active_rule in policy.content(): active_policy_rules.append( {'rule': active_rule.original_str, 'id': active_rule.id, 'comment': active_rule.comment, 'name': active_rule.name, 'policy_name': policy_name}) # ALEX: the Rule object does not have fields like the rule-string or # id or comment. We can add those fields to the Rule object, as long # as we don't add them to the Fact because there are many fact # instances. If a user tries to create a lot of Rules, they are # probably doing something wrong and should use a datasource driver # instead. changes = [] for r in configured_rules: if r not in active_policy_rules: LOG.debug("adding rule %s", str(r)) parsed_rule = engine.parse1(r['rule']) parsed_rule.set_id(r['id']) parsed_rule.set_name(r['name']) parsed_rule.set_comment(r['comment']) parsed_rule.set_original_str(r['rule']) event = compile.Event(formula=parsed_rule, insert=True, target=r['policy_name']) changes.append(event) for r in active_policy_rules: if r not in configured_rules: LOG.debug("removing rule %s", str(r)) parsed_rule = engine.parse1(r['rule']) parsed_rule.set_id(r['id']) parsed_rule.set_name(r['name']) parsed_rule.set_comment(r['comment']) parsed_rule.set_original_str(r['rule']) event = compile.Event(formula=parsed_rule, insert=False, target=r['policy_name']) changes.append(event) permitted, changes = engine.process_policy_update(changes) LOG.debug("synchronize_rules, permitted %d, made %d changes", permitted, len(changes))
def create(rootdir, statedir, config_file, config_override=None): """Get Congress up and running when src is installed in rootdir, i.e. ROOTDIR=/path/to/congress/congress. CONFIG_OVERRIDE is a dictionary of dictionaries with configuration values that overrides those provided in CONFIG_FILE. The top-level dictionary has keys for the CONFIG_FILE sections, and the second-level dictionaries store values for that section. """ LOG.debug("Starting Congress with rootdir={}, statedir={}, " "datasource_config={}, config_override={}".format( rootdir, statedir, config_file, config_override)) # create message bus cage = d6cage.d6Cage() cage.daemon = True cage.start() cage.system_service_names.add(cage.name) # read in datasource configurations cage.config = initialize_config(config_file, config_override) # path to congress source dir src_path = os.path.join(rootdir, "congress") # add policy engine engine_path = os.path.join(src_path, "policy/dsepolicy.py") LOG.info("main::start() engine_path: " + str(engine_path)) cage.loadModule("PolicyEngine", engine_path) cage.createservice( name="engine", moduleName="PolicyEngine", description="Policy Engine (DseRuntime instance)", args={'d6cage': cage, 'rootdir': src_path}) engine = cage.service_object('engine') if statedir is not None: engine.load_dir(statedir) engine.initialize_table_subscriptions() cage.system_service_names.add(engine.name) engine.debug_mode() # should take this out for production # add policy api # TODO(thinrichs): change to real API path. api_path = os.path.join(src_path, "api/policy_model.py") LOG.info("main::start() api_path: " + str(api_path)) cage.loadModule("API-policy", api_path) cage.createservice( name="api-policy", moduleName="API-policy", description="API-policy DSE instance", args={'policy_engine': engine}) cage.system_service_names.add('api-policy') # add rule api api_path = os.path.join(src_path, "api/rule_model.py") LOG.info("main::start() api_path: " + str(api_path)) cage.loadModule("API-rule", api_path) cage.createservice( name="api-rule", moduleName="API-rule", description="API-rule DSE instance", args={'policy_engine': engine}) cage.system_service_names.add('api-rule') # add table api api_path = os.path.join(src_path, "api/table_model.py") LOG.info("main::start() api_path: " + str(api_path)) cage.loadModule("API-table", api_path) cage.createservice( name="api-table", moduleName="API-table", description="API-table DSE instance", args={'policy_engine': engine}) cage.system_service_names.add('api-table') # add row api api_path = os.path.join(src_path, "api/row_model.py") LOG.info("main::start() api_path: " + str(api_path)) cage.loadModule("API-row", api_path) cage.createservice( name="api-row", moduleName="API-row", description="API-row DSE instance", args={'policy_engine': engine}) cage.system_service_names.add('api-row') # add datasource api api_path = os.path.join(src_path, "api/datasource_model.py") LOG.info("main::start() api_path: " + str(api_path)) cage.loadModule("API-datasource", api_path) cage.createservice( name="api-datasource", moduleName="API-datasource", description="API-datasource DSE instance", args={'policy_engine': engine}) cage.system_service_names.add('api-datasource') # have policy-engine subscribe to api calls # TODO(thinrichs): either have API publish everything to DSE bus and # have policy engine subscribe to all those messages # OR have API interact with individual components directly # and change all tests so that the policy engine does not need to be # subscribed to 'policy-update' engine.subscribe('api-rule', 'policy-update', callback=engine.receive_policy_update) # spin up all the configured services, if we have configured them if cage.config: for name in cage.config: if 'module' in cage.config[name]: load_data_service(name, cage.config[name], cage, src_path) return cage
def setUp(self): """Setup polling tests.""" super(TestDataSourceDriver, self).setUp() cage = d6cage.d6Cage() # Create mock of Neutron client so we can control data mock_factory = mox.Mox() neutron_client = mock_factory.CreateMock( neutronclient.v2_0.client.Client) neutron_client.list_networks().InAnyOrder(1).AndReturn(network1) neutron_client.list_ports().InAnyOrder(1).AndReturn(port_response) neutron_client.list_routers().InAnyOrder(1).AndReturn(router_response) neutron_client.list_security_groups().InAnyOrder(1).AndReturn( security_group_response) neutron_client.list_networks().InAnyOrder(2).AndReturn(network2) neutron_client.list_ports().InAnyOrder(2).AndReturn(port_response) neutron_client.list_routers().InAnyOrder(2).AndReturn(router_response) neutron_client.list_security_groups().InAnyOrder(2).AndReturn( security_group_response) mock_factory.ReplayAll() # Create modules (without auto-polling) cage.loadModule("NeutronDriver", helper.data_module_path("neutron_driver.py")) cage.loadModule("PolicyDriver", helper.policy_module_path()) cage.createservice(name="policy", moduleName="PolicyDriver", args={'d6cage': cage, 'rootdir': helper.data_module_path(''), 'log_actions_only': True}) args = helper.datasource_openstack_args() args['poll_time'] = 0 args['client'] = neutron_client cage.createservice(name="neutron", moduleName="NeutronDriver", args=args) policy = cage.service_object('policy') policy.create_policy('neutron') policy.set_schema( 'neutron', cage.service_object('neutron').get_schema()) cage.service_object('neutron').neutron = neutron_client policy.debug_mode() # insert rule into policy to make testing easier. # (Some of the IDs are auto-generated each time we convert) policy.insert(create_network_group('p')) # create some garbage data args = helper.datasource_openstack_args() driver = neutron_driver.NeutronDriver(args=args) network_key_to_index = driver.get_column_map( neutron_driver.NeutronDriver.NETWORKS) network_max_index = max(network_key_to_index.values()) args1 = ['1'] * (network_max_index + 1) args2 = ['2'] * (network_max_index + 1) args1 = ",".join(args1) args2 = ",".join(args2) fake_networks = [ 'neutron:networks({})'.format(args1), 'neutron:networks({})'.format(args2)] # answer to query above for network1 datalog1 = ( 'p("240ff9df-df35-43ae-9df5-27fae87f2492") ' 'p("340ff9df-df35-43ae-9df5-27fae87f2492") ' 'p("440ff9df-df35-43ae-9df5-27fae87f2492")') # answer to query above for network2 datalog2 = ( 'p("240ff9df-df35-43ae-9df5-27fae87f2492") ' 'p("640ff9df-df35-43ae-9df5-27fae87f2492") ' 'p("540ff9df-df35-43ae-9df5-27fae87f2492")') # return value self.info = {} self.info['cage'] = cage self.info['datalog1'] = datalog1 self.info['datalog2'] = datalog2 self.info['fake_networks'] = fake_networks
def __init__(self, name, keys, inbox=None, dataPath=None, policy_engine=None): super(StatusModel, self).__init__(name, keys, inbox=inbox, dataPath=dataPath) self.cage = d6cage.d6Cage()
def create(rootdir, statedir, config_override=None): """Get Congress up and running when src is installed in rootdir. i.e. ROOTDIR=/path/to/congress/congress. CONFIG_OVERRIDE is a dictionary of dictionaries with configuration values that overrides those provided in CONFIG_FILE. The top-level dictionary has keys for the CONFIG_FILE sections, and the second-level dictionaries store values for that section. """ LOG.debug( "Starting Congress with rootdir=%s, statedir=%s, " "config_override=%s", rootdir, statedir, config_override) # create message bus cage = d6cage.d6Cage() # read in datasource configurations cage.config = config_override or {} # path to congress source dir src_path = os.path.join(rootdir, "congress") # add policy engine engine_path = os.path.join(src_path, "policy_engines/agnostic.py") LOG.info("main::start() engine_path: %s", engine_path) cage.loadModule("PolicyEngine", engine_path) cage.createservice(name="engine", moduleName="PolicyEngine", description="Policy Engine (DseRuntime instance)", args={ 'd6cage': cage, 'rootdir': src_path }) engine = cage.service_object('engine') if statedir is not None: engine.load_dir(statedir) engine.initialize_table_subscriptions() engine.debug_mode() # should take this out for production # add policy api api_path = os.path.join(src_path, "api/policy_model.py") LOG.info("main::start() api_path: %s", api_path) cage.loadModule("API-policy", api_path) cage.createservice(name="api-policy", moduleName="API-policy", description="API-policy DSE instance", args={'policy_engine': engine}) # add rule api api_path = os.path.join(src_path, "api/rule_model.py") LOG.info("main::start() api_path: %s", api_path) cage.loadModule("API-rule", api_path) cage.createservice(name="api-rule", moduleName="API-rule", description="API-rule DSE instance", args={'policy_engine': engine}) # add table api api_path = os.path.join(src_path, "api/table_model.py") LOG.info("main::start() api_path: %s", api_path) cage.loadModule("API-table", api_path) cage.createservice(name="api-table", moduleName="API-table", description="API-table DSE instance", args={'policy_engine': engine}) # add row api api_path = os.path.join(src_path, "api/row_model.py") LOG.info("main::start() api_path: %s", api_path) cage.loadModule("API-row", api_path) cage.createservice(name="api-row", moduleName="API-row", description="API-row DSE instance", args={'policy_engine': engine}) # add status api api_path = os.path.join(src_path, "api/status_model.py") LOG.info("main::start() api_path: %s", api_path) cage.loadModule("API-status", api_path) cage.createservice(name="api-status", moduleName="API-status", description="API-status DSE instance", args={'policy_engine': engine}) # add schema api api_path = os.path.join(src_path, "api/schema_model.py") LOG.info("main::start() api_path: %s", api_path) cage.loadModule("API-schema", api_path) cage.createservice(name="api-schema", moduleName="API-schema", description="API-schema DSE instance", args={'policy_engine': engine}) # add datasource/config api api_path = os.path.join(src_path, "api/datasource_config_model.py") LOG.info("main::start() api_path: %s", api_path) cage.loadModule("API-config", api_path) cage.createservice(name="api-config", moduleName="API-config", description="API-config DSE instance", args={'policy_engine': engine}) # add path for system/datasource-drivers api_path = os.path.join(src_path, "api/system/driver_model.py") LOG.info("main::start() api_path: %s", api_path) cage.loadModule("API-system", api_path) cage.createservice(name="api-system", moduleName="API-system", description="API-system DSE instance", args={'policy_engine': engine}) # Load policies from database for policy in db_policy_rules.get_policies(): engine.create_policy(policy.name, abbr=policy.abbreviation, kind=policy.kind) # if this is the first time we are running Congress, need # to create the default theories (which cannot be deleted) api_policy = cage.service_object('api-policy') engine.DEFAULT_THEORY = 'classification' engine.builtin_policy_names.add(engine.DEFAULT_THEORY) try: api_policy.add_item( { 'name': engine.DEFAULT_THEORY, 'description': 'default policy' }, {}) except KeyError: pass engine.ACTION_THEORY = 'action' engine.builtin_policy_names.add(engine.ACTION_THEORY) try: api_policy.add_item( { 'kind': ACTION_POLICY_TYPE, 'name': engine.ACTION_THEORY, 'description': 'default action policy' }, {}) except KeyError: pass # have policy-engine subscribe to api calls # TODO(thinrichs): either have API publish everything to DSE bus and # have policy engine subscribe to all those messages # OR have API interact with individual components directly # and change all tests so that the policy engine does not need to be # subscribed to 'policy-update' engine.subscribe('api-rule', 'policy-update', callback=engine.receive_policy_update) # spin up all the configured services, if we have configured them datasource_mgr = datasource_manager.DataSourceManager drivers = datasource_mgr.get_datasources() # Setup cage.config as it previously done when it was loaded # from disk. FIXME(arosen) later! for driver in drivers: if not driver['enabled']: LOG.info("module %s not enabled, skip loading", driver['name']) continue driver_info = datasource_mgr.get_driver_info(driver['driver']) engine.create_policy(driver['name']) try: cage.createservice(name=driver['name'], moduleName=driver_info['module'], args=driver['config'], module_driver=True, type_='datasource_driver', id_=driver['id']) except d6cage.DataServiceError: # FIXME(arosen): If createservice raises congress-server # dies here. So we catch this exception so the server does # not die. We need to refactor the dse code so it just # keeps retrying the driver gracefully... continue service = cage.service_object(driver['name']) engine.set_schema(driver['name'], service.get_schema()) # Insert rules. Needs to be done after datasources are loaded # so that we can compile away column references at read time. # If datasources loaded after this, we don't have schemas. rules = db_policy_rules.get_policy_rules() for rule in rules: parsed_rule = engine.parse1(rule.rule) cage.service_object('api-rule').change_rule( parsed_rule, {'policy_id': rule.policy_name}) # Start datasource synchronizer after explicitly starting the # datasources, because the explicit call to create a datasource # will crash if the synchronizer creates the datasource first. synchronizer_path = os.path.join(src_path, "synchronizer.py") LOG.info("main::start() synchronizer: %s", synchronizer_path) cage.loadModule("Synchronizer", synchronizer_path) cage.createservice(name="synchronizer", moduleName="Synchronizer", description="DB synchronizer instance", args={'poll_time': cfg.CONF.datasource_sync_period}) synchronizer = cage.service_object('synchronizer') # add datasource api api_path = os.path.join(src_path, "api/datasource_model.py") LOG.info("main::start() api_path: %s", api_path) cage.loadModule("API-datasource", api_path) cage.createservice(name="api-datasource", moduleName="API-datasource", description="API-datasource DSE instance", args={ 'policy_engine': engine, 'synchronizer': synchronizer }) return cage