def add_datasource(cls, item, deleted=False):
req = cls.make_datasource_dict(item)
driver_info = cls.validate_create_datasource(req)
session = db.get_session()
try:
with session.begin(subtransactions=True):
datasource = datasources_db.add_datasource(
id_=req['id'],
name=req['name'],
driver=req['driver'],
config=req['config'],
description=req['description'],
enabled=req['enabled'],
session=session)
datasource = cls.make_datasource_dict(datasource)
cage = d6cage.d6Cage()
engine = cage.service_object('engine')
try:
engine.create_policy(datasource['name'])
except KeyError:
# FIXME(arosen): we need a better exception then
# key error being raised here
raise DatasourceNameInUse(name=req['name'])
cage.createservice(name=datasource['name'],
moduleName=driver_info['module'],
args=datasource['config'],
module_driver=True)
service = cage.service_object(req['name'])
engine.set_schema(req['name'], service.get_schema())
except db_exc.DBDuplicateEntry:
raise DatasourceNameInUse(name=req['name'])
return cls.make_datasource_dict(datasource)
def synchronize(self):
LOG.debug("Synchronizing running datasources")
cage = d6cage.d6Cage()
datasources = self.datasource_mgr.get_datasources(filter_secret=False)
# Look for datasources in the db, but not in the cage.
for configured_ds in datasources:
active_ds = cage.service_object(configured_ds['name'])
if active_ds is not None:
if not configured_ds['enabled']:
LOG.info('Datasource %s now disabled, just delete it.',
configured_ds['name'])
self.datasource_mgr.delete_datasource(configured_ds['id'],
update_db=False)
continue
active_config = cage.getservice(name=configured_ds['name'])
if not self._config_eq(configured_ds, active_config):
LOG.debug('configured and active disagree: (%s) %s %s',
strutils.mask_password(active_ds),
strutils.mask_password(configured_ds),
strutils.mask_password(active_config))
LOG.info('Reloading datasource: %s',
strutils.mask_password(configured_ds))
self.datasource_mgr.delete_datasource(configured_ds['id'],
update_db=False)
self.datasource_mgr.add_datasource(
configured_ds,
update_db=False)
else:
if configured_ds['enabled']:
LOG.info('Configured datasource is not active, adding: %s',
strutils.mask_password(configured_ds))
self.datasource_mgr.add_datasource(configured_ds,
update_db=False)
else:
LOG.info('Configured datasource is not active but ' +
'disabled, not adding: %s',
strutils.mask_password(configured_ds))
# Look for datasources in the cage, but not in the db. This
# need not compare the configuration, because the above
# comparison would have already checked the configuration.
configured_dicts = dict((ds['name'], ds) for ds in datasources)
LOG.debug("configured dicts: %s",
strutils.mask_password(configured_dicts))
LOG.debug("active services: %s",
strutils.mask_password(cage.getservices()))
for name, service in cage.getservices().items():
LOG.debug('active datasource: %s', service['name'])
if (service['type'] == 'datasource_driver' and
not configured_dicts.get(service['name'], None)):
LOG.info('Active datasource is not configured, removing: %s',
service['name'])
cage.deleteservice(service['name'])
engine = cage.service_object('engine')
engine.delete_policy(service['name'])
def get_actions(cls, source_id=None):
cage = d6cage.d6Cage()
driver = cage.getservice(id_=source_id, type_='datasource_driver')
if not driver:
raise exception.NotFound('Could not find datasource %s' %
source_id)
return driver['object'].get_actions()
def _add_datasource_service(cls, new_ds, original_req):
driver_info = cls.get_driver_info(new_ds['driver'])
cage = cls.dseNode or d6cage.d6Cage()
engine = cage.service_object('engine')
try:
LOG.debug("creating policy %s", new_ds['name'])
engine.create_policy(new_ds['name'],
kind=base.DATASOURCE_POLICY_TYPE)
except KeyError:
raise exception.DatasourceNameInUse(value=new_ds['name'])
try:
if cls.dseNode:
cls.createservice(name=new_ds['name'],
moduleName=driver_info['module'],
args=original_req['config'],
module_driver=True,
type_='datasource_driver',
id_=new_ds['id'])
else:
if not cage.service_object(new_ds['name']):
cage.createservice(name=new_ds['name'],
moduleName=driver_info['module'],
args=original_req['config'],
module_driver=True,
type_='datasource_driver',
id_=new_ds['id'])
service = cage.service_object(new_ds['name'])
engine.set_schema(new_ds['name'], service.get_schema())
except Exception:
engine.delete_policy(new_ds['name'])
raise exception.DatasourceCreationError(value=new_ds['name'])
def add_datasource(cls, item, deleted=False, update_db=True):
req = cls.make_datasource_dict(item)
# If update_db is True, new_id will get a new value from the db.
new_id = req['id']
driver_info = cls.get_driver_info(item['driver'])
session = db.get_session()
try:
with session.begin(subtransactions=True):
LOG.debug("adding datasource %s", req['name'])
if update_db:
LOG.debug("updating db")
datasource = datasources_db.add_datasource(
id_=req['id'],
name=req['name'],
driver=req['driver'],
config=req['config'],
description=req['description'],
enabled=req['enabled'],
session=session)
new_id = datasource['id']
cls.validate_create_datasource(req)
cage = cls.dseNode or d6cage.d6Cage()
engine = cage.service_object('engine')
try:
LOG.debug("creating policy %s", req['name'])
engine.create_policy(req['name'],
kind=base.DATASOURCE_POLICY_TYPE)
except KeyError:
# FIXME(arosen): we need a better exception then
# key error being raised here
raise DatasourceNameInUse(value=req['name'])
try:
if cls.dseNode:
cls.createservice(name=req['name'],
moduleName=driver_info['module'],
args=item['config'],
module_driver=True,
type_='datasource_driver',
id_=new_id)
else:
cage.createservice(name=req['name'],
moduleName=driver_info['module'],
args=item['config'],
module_driver=True,
type_='datasource_driver',
id_=new_id)
service = cage.service_object(req['name'])
engine.set_schema(req['name'], service.get_schema())
except Exception:
engine.delete_policy(req['name'])
raise DatasourceCreationError(value=req['name'])
except db_exc.DBDuplicateEntry:
raise DatasourceNameInUse(value=req['name'])
new_item = dict(item)
new_item['id'] = new_id
return cls.make_datasource_dict(new_item)
def test_communication(self):
"""Test for communication.
Test the module's ability to be loaded into the DSE
by checking its ability to communicate on the message bus.
"""
cage = d6cage.d6Cage()
# Create modules.
# Turn off polling so we don't need to deal with real data.
args = helper.datasource_openstack_args()
args['poll_time'] = 0
cage.loadModule("NovaDriver",
helper.data_module_path("nova_driver.py"))
cage.loadModule("PolicyDriver", helper.policy_module_path())
cage.createservice(name="policy",
moduleName="PolicyDriver",
args={
'd6cage': cage,
'rootdir': helper.data_module_path('')
})
cage.createservice(name="nova", moduleName="NovaDriver", args=args)
# Check that data gets sent from nova to policy as expected
nova = cage.service_object('nova')
policy = cage.service_object('policy')
policy.debug_mode()
policy.create_policy('nova')
policy.set_schema('nova', compile.Schema({'server': (1, )}))
policy.subscribe('nova', 'server', callback=policy.receive_data)
# publishing is slightly convoluted b/c deltas are computed
# automatically. (Not just convenient--useful so that DSE
# properly handles the initial state problem.)
# Need to set nova.state and nova.prior_state and then publish
# anything.
# publish server(1), server(2), server(3)
helper.retry_check_subscribers(nova, [(policy.name, 'server')])
nova.prior_state = {}
nova.state['server'] = set([(1, ), (2, ), (3, )])
nova.publish('server', None)
helper.retry_check_db_equal(
policy, 'nova:server(x)',
'nova:server(1) nova:server(2) nova:server(3)')
# publish server(1), server(4), server(5)
nova.prior_state['server'] = nova.state['server']
nova.state['server'] = set([(1, ), (4, ), (5, )])
nova.publish('server', None)
helper.retry_check_db_equal(
policy, 'nova:server(x)',
'nova:server(1) nova:server(4) nova:server(5)')
def test_communication(self):
"""Test for communication.
Test the module's ability to be loaded into the DSE
by checking its ability to communicate on the message bus.
"""
cage = d6cage.d6Cage()
# Create modules.
# Turn off polling so we don't need to deal with real data.
args = helper.datasource_openstack_args()
args['poll_time'] = 0
cage.loadModule("NovaDriver",
helper.data_module_path("nova_driver.py"))
cage.loadModule("PolicyDriver", helper.policy_module_path())
cage.createservice(name="policy", moduleName="PolicyDriver",
args={'d6cage': cage,
'rootdir': helper.data_module_path(''),
'log_actions_only': True})
cage.createservice(name="nova", moduleName="NovaDriver", args=args)
# Check that data gets sent from nova to policy as expected
nova = cage.service_object('nova')
policy = cage.service_object('policy')
policy.debug_mode()
policy.create_policy('nova')
policy.set_schema('nova', compile.Schema({'server': (1,)}))
policy.subscribe('nova', 'server',
callback=policy.receive_data)
# publishing is slightly convoluted b/c deltas are computed
# automatically. (Not just convenient--useful so that DSE
# properly handles the initial state problem.)
# Need to set nova.state and nova.prior_state and then publish
# anything.
# publish server(1), server(2), server(3)
helper.retry_check_subscribers(nova, [(policy.name, 'server')])
nova.prior_state = {}
nova.state['server'] = set([(1,), (2,), (3,)])
nova.publish('server', None)
helper.retry_check_db_equal(
policy, 'nova:server(x)',
'nova:server(1) nova:server(2) nova:server(3)')
# publish server(1), server(4), server(5)
nova.prior_state['server'] = nova.state['server']
nova.state['server'] = set([(1,), (4,), (5,)])
nova.publish('server', None)
helper.retry_check_db_equal(
policy, 'nova:server(x)',
'nova:server(1) nova:server(4) nova:server(5)')
def setUp(self):
# create DSE and add vm-placement engine and fake datasource
super(TestSetPolicy, self).setUp()
self.cage = d6cage.d6Cage()
config = {
"vmplace": {"module": "congress/policy_engines/vm_placement.py"},
"fake": {"poll_time": 0, "module": "congress/tests/fake_datasource.py"},
}
harness.load_data_service("vmplace", config["vmplace"], self.cage, helper.root_path(), 1)
harness.load_data_service("fake", config["fake"], self.cage, helper.root_path(), 2)
self.vmplace = self.cage.service_object("vmplace")
self.vmplace.debug_mode()
self.fake = self.cage.service_object("fake")
def delete_datasource(cls, datasource_id):
datasource = cls.get_datasource(datasource_id)
session = db.get_session()
with session.begin(subtransactions=True):
cage = d6cage.d6Cage()
engine = cage.service_object('engine')
try:
engine.delete_policy(datasource['name'],
disallow_dangling_refs=True)
except exception.DanglingReference as e:
raise e
except KeyError:
raise DatasourceNotFound(id=datasource_id)
result = datasources_db.delete_datasource(
datasource_id, session)
if not result:
raise DatasourceNotFound(id=datasource_id)
cage.deleteservice(datasource['name'])
def add_datasource(cls, item, deleted=False, update_db=True):
req = cls.make_datasource_dict(item)
# If update_db is True, new_id will get a new value from the db.
new_id = req['id']
driver_info = cls.get_driver_info(item['driver'])
session = db.get_session()
try:
with session.begin(subtransactions=True):
LOG.debug("adding datasource %s", req['name'])
if update_db:
LOG.debug("updating db")
datasource = datasources_db.add_datasource(
id_=req['id'],
name=req['name'],
driver=req['driver'],
config=req['config'],
description=req['description'],
enabled=req['enabled'],
session=session)
new_id = datasource['id']
cage = d6cage.d6Cage()
engine = cage.service_object('engine')
try:
LOG.debug("creating policy %s", req['name'])
engine.create_policy(req['name'])
except KeyError:
# FIXME(arosen): we need a better exception then
# key error being raised here
raise DatasourceNameInUse(name=req['name'])
cage.createservice(name=req['name'],
moduleName=driver_info['module'],
args=item['config'],
module_driver=True,
type_='datasource_driver',
id_=new_id)
service = cage.service_object(req['name'])
engine.set_schema(req['name'], service.get_schema())
except db_exc.DBDuplicateEntry:
raise DatasourceNameInUse(name=req['name'])
new_item = dict(item)
new_item['id'] = new_id
return cls.make_datasource_dict(new_item)
def synchronize_policies(self):
LOG.debug("Synchronizing policies")
# Read policies from DB.
cage = d6cage.d6Cage()
configured_policies = [{'id': p.id,
'name': p.name,
'abbr': p.abbreviation,
'desc': p.description,
'owner': p.owner,
'kind': p.kind}
for p in db_policy_rules.get_policies()]
# Read policies from engine
engine = cage.service_object('engine')
policies = [engine.policy_object(n) for n in engine.policy_names()]
active_policies = []
for policy in policies:
active_policies.append({'id': policy.id,
'name': policy.name,
'abbr': policy.abbr,
'desc': policy.desc,
'owner': policy.owner,
'kind': policy.kind})
added = 0
removed = 0
for p in active_policies:
if (p['kind'] != base.DATASOURCE_POLICY_TYPE and
p not in configured_policies):
LOG.debug("removing policy %s", str(p))
engine.delete_policy(p['id'])
removed = removed + 1
for p in configured_policies:
if p not in active_policies:
LOG.debug("adding policy %s", str(p))
engine.create_policy(p['name'], id_=p['id'], abbr=p['abbr'],
kind=p['kind'], desc=p['desc'],
owner=p['owner'])
added = added + 1
LOG.debug("synchronize_policies, added %d removed %d",
added, removed)
def delete_datasource(cls, datasource_id, update_db=True):
datasource = cls.get_datasource(datasource_id)
session = db.get_session()
with session.begin(subtransactions=True):
cage = d6cage.d6Cage()
engine = cage.service_object('engine')
try:
engine.delete_policy(datasource['name'],
disallow_dangling_refs=True)
except exception.DanglingReference as e:
raise e
except KeyError:
raise DatasourceNotFound(id=datasource_id)
if update_db:
result = datasources_db.delete_datasource(
datasource_id, session)
if not result:
raise DatasourceNotFound(id=datasource_id)
cage.deleteservice(datasource['name'])
def synchronize(self):
LOG.debug("Synchronizing running datasources")
cage = d6cage.d6Cage()
datasources = self.datasource_mgr.get_datasources(filter_secret=False)
# Look for datasources in the db, but not in the cage.
for configured_ds in datasources:
active_ds = cage.service_object(configured_ds['name'])
if active_ds is not None:
active_config = cage.getservice(name=configured_ds['name'])
if not self._config_eq(configured_ds, active_config):
LOG.debug('configured and active disagree: (%s) %s %s',
str(active_ds), str(configured_ds),
str(active_config))
LOG.info('Reloading datasource: %s', str(configured_ds))
self.datasource_mgr.delete_datasource(configured_ds['id'],
update_db=False)
self.datasource_mgr.add_datasource(configured_ds,
update_db=False)
else:
LOG.info('Configured datasource is not active, adding: %s',
str(configured_ds))
self.datasource_mgr.add_datasource(configured_ds,
update_db=False)
# Look for datasources in the cage, but not in the db. This
# need not compare the configuration, because the above
# comparison would have already checked the configuration.
configured_dicts = dict((ds['name'], ds) for ds in datasources)
LOG.debug("configured dicts: %s", str(configured_dicts))
LOG.debug("active services: %s", str(cage.getservices()))
for name, service in cage.getservices().items():
LOG.debug('active datasource: %s', service['name'])
if (service['type'] == 'datasource_driver'
and not configured_dicts.get(service['name'], None)):
LOG.info('Active datasource is not configured, removing: %s',
service['name'])
cage.deleteservice(service['name'])
engine = cage.service_object('engine')
engine.delete_policy(service['name'])
def setUp(self):
# create DSE and add vm-placement engine and fake datasource
super(TestSetPolicy, self).setUp()
self.cage = d6cage.d6Cage()
config = {
'vmplace': {
'module': "congress/policy_engines/vm_placement.py"
},
'fake': {
'poll_time': 0,
'module': "congress/tests/fake_datasource.py"
}
}
harness.load_data_service("vmplace", config['vmplace'], self.cage,
helper.root_path(), 1)
harness.load_data_service("fake", config['fake'], self.cage,
helper.root_path(), 2)
self.vmplace = self.cage.service_object('vmplace')
self.vmplace.debug_mode()
self.fake = self.cage.service_object('fake')
def request_refresh(cls, datasource_id):
datasource = cls.get_datasource(datasource_id)
cage = d6cage.d6Cage()
datasource = cage.service_object(datasource['name'])
datasource.request_refresh()
def create(rootdir, config_override=None):
"""Get Congress up and running when src is installed in rootdir.
i.e. ROOTDIR=/path/to/congress/congress.
CONFIG_OVERRIDE is a dictionary of dictionaries with configuration
values that overrides those provided in CONFIG_FILE. The top-level
dictionary has keys for the CONFIG_FILE sections, and the second-level
dictionaries store values for that section.
"""
LOG.debug("Starting Congress with rootdir=%s, config_override=%s",
rootdir, config_override)
# create message bus
cage = d6cage.d6Cage()
# read in datasource configurations
cage.config = config_override or {}
# path to congress source dir
src_path = os.path.join(rootdir, "congress")
datasource_mgr = datasource_manager.DataSourceManager()
datasource_mgr.validate_configured_drivers()
# add policy engine
engine_path = os.path.join(src_path, "policy_engines/agnostic.py")
LOG.info("main::start() engine_path: %s", engine_path)
cage.loadModule("PolicyEngine", engine_path)
cage.createservice(
name="engine",
moduleName="PolicyEngine",
description="Policy Engine (DseRuntime instance)",
args={'d6cage': cage, 'rootdir': src_path,
'log_actions_only': cfg.CONF.enable_execute_action})
engine = cage.service_object('engine')
engine.initialize_table_subscriptions()
engine.debug_mode() # should take this out for production
# add policy api
api_path = os.path.join(src_path, "api/policy_model.py")
LOG.info("main::start() api_path: %s", api_path)
cage.loadModule("API-policy", api_path)
cage.createservice(
name="api-policy",
moduleName="API-policy",
description="API-policy DSE instance",
args={'policy_engine': engine})
# add rule api
api_path = os.path.join(src_path, "api/rule_model.py")
LOG.info("main::start() api_path: %s", api_path)
cage.loadModule("API-rule", api_path)
cage.createservice(
name="api-rule",
moduleName="API-rule",
description="API-rule DSE instance",
args={'policy_engine': engine})
# add table api
api_path = os.path.join(src_path, "api/table_model.py")
LOG.info("main::start() api_path: %s", api_path)
cage.loadModule("API-table", api_path)
cage.createservice(
name="api-table",
moduleName="API-table",
description="API-table DSE instance",
args={'policy_engine': engine,
'datasource_mgr': datasource_mgr})
# add row api
api_path = os.path.join(src_path, "api/row_model.py")
LOG.info("main::start() api_path: %s", api_path)
cage.loadModule("API-row", api_path)
cage.createservice(
name="api-row",
moduleName="API-row",
description="API-row DSE instance",
args={'policy_engine': engine,
'datasource_mgr': datasource_mgr})
# add status api
api_path = os.path.join(src_path, "api/status_model.py")
LOG.info("main::start() api_path: %s", api_path)
cage.loadModule("API-status", api_path)
cage.createservice(
name="api-status",
moduleName="API-status",
description="API-status DSE instance",
args={'policy_engine': engine,
'datasource_mgr': datasource_mgr})
# add action api
api_path = os.path.join(src_path, "api/action_model.py")
LOG.info("main::start() api_path: %s", api_path)
cage.loadModule("API-action", api_path)
cage.createservice(
name="api-action",
moduleName="API-action",
description="API-action DSE instance",
args={'policy_engine': engine,
'datasource_mgr': datasource_mgr})
# add schema api
api_path = os.path.join(src_path, "api/schema_model.py")
LOG.info("main::start() api_path: %s", api_path)
cage.loadModule("API-schema", api_path)
cage.createservice(
name="api-schema",
moduleName="API-schema",
description="API-schema DSE instance",
args={'datasource_mgr': datasource_mgr})
# add path for system/datasource-drivers
api_path = os.path.join(src_path, "api/system/driver_model.py")
LOG.info("main::start() api_path: %s", api_path)
cage.loadModule("API-system", api_path)
cage.createservice(
name="api-system",
moduleName="API-system",
description="API-system DSE instance",
args={'datasource_mgr': datasource_mgr})
# Load policies from database
engine.persistent_load_policies()
# if this is the first time we are running Congress, need
# to create the default theories (which cannot be deleted)
api_policy = cage.service_object('api-policy')
engine.DEFAULT_THEORY = 'classification'
engine.builtin_policy_names.add(engine.DEFAULT_THEORY)
try:
api_policy.add_item({'name': engine.DEFAULT_THEORY,
'description': 'default policy'}, {})
except KeyError:
pass
engine.ACTION_THEORY = 'action'
engine.builtin_policy_names.add(engine.ACTION_THEORY)
try:
api_policy.add_item({'kind': base.ACTION_POLICY_TYPE,
'name': engine.ACTION_THEORY,
'description': 'default action policy'},
{})
except KeyError:
pass
# have policy-engine subscribe to api calls
# TODO(thinrichs): either have API publish everything to DSE bus and
# have policy engine subscribe to all those messages
# OR have API interact with individual components directly
# and change all tests so that the policy engine does not need to be
# subscribed to 'policy-update'
engine.subscribe('api-rule', 'policy-update',
callback=engine.receive_policy_update)
# spin up all the configured services, if we have configured them
drivers = datasource_mgr.get_datasources()
# Setup cage.config as it previously done when it was loaded
# from disk. FIXME(arosen) later!
for driver in drivers:
if not driver['enabled']:
LOG.info("module %s not enabled, skip loading", driver['name'])
continue
driver_info = datasource_mgr.get_driver_info(driver['driver'])
engine.create_policy(driver['name'], kind=base.DATASOURCE_POLICY_TYPE)
try:
cage.createservice(name=driver['name'],
moduleName=driver_info['module'],
args=driver['config'],
module_driver=True,
type_='datasource_driver',
id_=driver['id'])
except d6cage.DataServiceError:
# FIXME(arosen): If createservice raises congress-server
# dies here. So we catch this exception so the server does
# not die. We need to refactor the dse code so it just
# keeps retrying the driver gracefully...
continue
service = cage.service_object(driver['name'])
engine.set_schema(driver['name'], service.get_schema())
# Insert rules. Needs to be done after datasources are loaded
# so that we can compile away column references at read time.
# If datasources loaded after this, we don't have schemas.
engine.persistent_load_rules()
# Start datasource synchronizer after explicitly starting the
# datasources, because the explicit call to create a datasource
# will crash if the synchronizer creates the datasource first.
synchronizer_path = os.path.join(src_path, "synchronizer.py")
LOG.info("main::start() synchronizer: %s", synchronizer_path)
cage.loadModule("Synchronizer", synchronizer_path)
cage.createservice(
name="synchronizer",
moduleName="Synchronizer",
description="DB synchronizer instance",
args={'poll_time': cfg.CONF.datasource_sync_period})
synchronizer = cage.service_object('synchronizer')
engine.set_synchronizer(synchronizer)
# add datasource api
api_path = os.path.join(src_path, "api/datasource_model.py")
LOG.info("main::start() api_path: %s", api_path)
cage.loadModule("API-datasource", api_path)
cage.createservice(
name="api-datasource",
moduleName="API-datasource",
description="API-datasource DSE instance",
args={'policy_engine': engine,
'datasource_mgr': datasource_mgr,
'synchronizer': synchronizer})
return cage
def get_row_data(cls, table_id, datasource_id, **kwargs):
datasource = cls.get_datasource(datasource_id)
cage = d6cage.d6Cage()
datasource_obj = cage.service_object(datasource['name'])
return datasource_obj.get_row_data(table_id)
def update_entire_data(cls, table_id, source_id, objs):
datasource = cls.get_datasource(source_id)
cage = d6cage.d6Cage()
datasource_obj = cage.service_object(datasource['name'])
return datasource_obj.update_entire_data(table_id, objs)
def setUp(self):
"""Setup polling tests."""
super(TestDataSourceDriver, self).setUp()
cage = d6cage.d6Cage()
# Create mock of Neutron client so we can control data
mock_factory = mox.Mox()
neutron_client = mock_factory.CreateMock(
neutronclient.v2_0.client.Client)
neutron_client.list_networks().InAnyOrder(1).AndReturn(network1)
neutron_client.list_ports().InAnyOrder(1).AndReturn(port_response)
neutron_client.list_routers().InAnyOrder(1).AndReturn(router_response)
neutron_client.list_security_groups().InAnyOrder(1).AndReturn(
security_group_response)
neutron_client.list_networks().InAnyOrder(2).AndReturn(network2)
neutron_client.list_ports().InAnyOrder(2).AndReturn(port_response)
neutron_client.list_routers().InAnyOrder(2).AndReturn(router_response)
neutron_client.list_security_groups().InAnyOrder(2).AndReturn(
security_group_response)
mock_factory.ReplayAll()
# Create modules (without auto-polling)
cage.loadModule("NeutronDriver",
helper.data_module_path("neutron_driver.py"))
cage.loadModule("PolicyDriver", helper.policy_module_path())
cage.createservice(name="policy",
moduleName="PolicyDriver",
args={
'd6cage': cage,
'rootdir': helper.data_module_path('')
})
args = helper.datasource_openstack_args()
args['poll_time'] = 0
args['client'] = neutron_client
cage.createservice(name="neutron",
moduleName="NeutronDriver",
args=args)
policy = cage.service_object('policy')
policy.create_policy('neutron')
policy.set_schema('neutron',
cage.service_object('neutron').get_schema())
cage.service_object('neutron').neutron = neutron_client
policy.debug_mode()
# insert rule into policy to make testing easier.
# (Some of the IDs are auto-generated each time we convert)
policy.insert(create_network_group('p'))
# create some garbage data
args = helper.datasource_openstack_args()
driver = neutron_driver.NeutronDriver(args=args)
network_key_to_index = driver.get_column_map(
neutron_driver.NeutronDriver.NETWORKS)
network_max_index = max(network_key_to_index.values())
args1 = ['1'] * (network_max_index + 1)
args2 = ['2'] * (network_max_index + 1)
args1 = ",".join(args1)
args2 = ",".join(args2)
fake_networks = [
'neutron:networks({})'.format(args1),
'neutron:networks({})'.format(args2)
]
# answer to query above for network1
datalog1 = ('p("240ff9df-df35-43ae-9df5-27fae87f2492") '
'p("340ff9df-df35-43ae-9df5-27fae87f2492") '
'p("440ff9df-df35-43ae-9df5-27fae87f2492")')
# answer to query above for network2
datalog2 = ('p("240ff9df-df35-43ae-9df5-27fae87f2492") '
'p("640ff9df-df35-43ae-9df5-27fae87f2492") '
'p("540ff9df-df35-43ae-9df5-27fae87f2492")')
# return value
self.info = {}
self.info['cage'] = cage
self.info['datalog1'] = datalog1
self.info['datalog2'] = datalog2
self.info['fake_networks'] = fake_networks
def __init__(self, name, keys, inbox=None, dataPath=None,
policy_engine=None):
super(StatusModel, self).__init__(name, keys, inbox=inbox,
dataPath=dataPath)
self.cage = d6cage.d6Cage()
self.engine = self.cage.service_object('engine')
def create(rootdir, statedir):
"""Get Congress up and running when src is installed in rootdir,
i.e. ROOTDIR=/path/to/congress/congress.
"""
LOG.debug("Starting Congress with rootdir={} and statedir={}".format(
rootdir, statedir))
# create message bus
cage = d6cage.d6Cage()
cage.daemon = True
cage.start()
cage.system_service_names.add(cage.name)
# add policy engine
engine_path = os.path.join(rootdir, "policy/dsepolicy.py")
LOG.info("main::start() engine_path: " + str(engine_path))
cage.loadModule("PolicyEngine", engine_path)
cage.createservice(
name="engine",
moduleName="PolicyEngine",
description="Policy Engine (DseRuntime instance)",
args={'d6cage': cage, 'rootdir': rootdir})
engine = cage.service_object('engine')
if statedir is not None:
engine.load_dir(statedir)
engine.initialize_table_subscriptions()
cage.system_service_names.add(engine.name)
# add policy api
# TODO(thinrichs): change to real API path.
api_path = os.path.join(rootdir, "api/policy_model.py")
LOG.info("main::start() api_path: " + str(api_path))
cage.loadModule("API-policy", api_path)
cage.createservice(
name="api-policy",
moduleName="API-policy",
description="API-policy DSE instance",
args={'policy_engine': engine})
cage.system_service_names.add('api-policy')
# add rule api
api_path = os.path.join(rootdir, "api/rule_model.py")
LOG.info("main::start() api_path: " + str(api_path))
cage.loadModule("API-rule", api_path)
cage.createservice(
name="api-rule",
moduleName="API-rule",
description="API-rule DSE instance",
args={'policy_engine': engine})
cage.system_service_names.add('api-rule')
# add table api
api_path = os.path.join(rootdir, "api/table_model.py")
LOG.info("main::start() api_path: " + str(api_path))
cage.loadModule("API-table", api_path)
cage.createservice(
name="api-table",
moduleName="API-table",
description="API-table DSE instance",
args={'policy_engine': engine})
cage.system_service_names.add('api-table')
# add row api
api_path = os.path.join(rootdir, "api/row_model.py")
LOG.info("main::start() api_path: " + str(api_path))
cage.loadModule("API-row", api_path)
cage.createservice(
name="api-row",
moduleName="API-row",
description="API-row DSE instance",
args={'policy_engine': engine})
cage.system_service_names.add('api-row')
# add datasource api
api_path = os.path.join(rootdir, "api/datasource_model.py")
LOG.info("main::start() api_path: " + str(api_path))
cage.loadModule("API-datasource", api_path)
cage.createservice(
name="api-datasource",
moduleName="API-datasource",
description="API-datasource DSE instance",
args={'policy_engine': engine})
cage.system_service_names.add('api-datasource')
# have policy-engine subscribe to api calls
# TODO(thinrichs): either have API publish everything to DSE bus and
# have policy engine subscribe to all those messages
# OR have API interact with individual components directly
# and change all tests so that the policy engine does not need to be
# subscribed to 'policy-update'
engine.subscribe('api-rule', 'policy-update',
callback=engine.receive_policy_update)
return cage
def synchronize_rules(self):
LOG.debug("Synchronizing rules")
# Read rules from DB.
cage = d6cage.d6Cage()
configured_rules = [{'rule': r.rule,
'id': r.id,
'comment': r.comment,
'name': r.name,
'policy_name': r.policy_name}
for r in db_policy_rules.get_policy_rules()]
# Read rules from engine
engine = cage.service_object('engine')
policies = {n: engine.policy_object(n) for n in engine.policy_names()}
active_policy_rules = []
for policy_name, policy in policies.items():
if policy.kind != base.DATASOURCE_POLICY_TYPE:
for active_rule in policy.content():
active_policy_rules.append(
{'rule': active_rule.original_str,
'id': active_rule.id,
'comment': active_rule.comment,
'name': active_rule.name,
'policy_name': policy_name})
# ALEX: the Rule object does not have fields like the rule-string or
# id or comment. We can add those fields to the Rule object, as long
# as we don't add them to the Fact because there are many fact
# instances. If a user tries to create a lot of Rules, they are
# probably doing something wrong and should use a datasource driver
# instead.
changes = []
for r in configured_rules:
if r not in active_policy_rules:
LOG.debug("adding rule %s", str(r))
parsed_rule = engine.parse1(r['rule'])
parsed_rule.set_id(r['id'])
parsed_rule.set_name(r['name'])
parsed_rule.set_comment(r['comment'])
parsed_rule.set_original_str(r['rule'])
event = compile.Event(formula=parsed_rule,
insert=True,
target=r['policy_name'])
changes.append(event)
for r in active_policy_rules:
if r not in configured_rules:
LOG.debug("removing rule %s", str(r))
parsed_rule = engine.parse1(r['rule'])
parsed_rule.set_id(r['id'])
parsed_rule.set_name(r['name'])
parsed_rule.set_comment(r['comment'])
parsed_rule.set_original_str(r['rule'])
event = compile.Event(formula=parsed_rule,
insert=False,
target=r['policy_name'])
changes.append(event)
permitted, changes = engine.process_policy_update(changes)
LOG.debug("synchronize_rules, permitted %d, made %d changes",
permitted, len(changes))
def create(rootdir, statedir, config_file, config_override=None):
"""Get Congress up and running when src is installed in rootdir,
i.e. ROOTDIR=/path/to/congress/congress.
CONFIG_OVERRIDE is a dictionary of dictionaries with configuration
values that overrides those provided in CONFIG_FILE. The top-level
dictionary has keys for the CONFIG_FILE sections, and the second-level
dictionaries store values for that section.
"""
LOG.debug("Starting Congress with rootdir={}, statedir={}, "
"datasource_config={}, config_override={}".format(
rootdir, statedir, config_file, config_override))
# create message bus
cage = d6cage.d6Cage()
cage.daemon = True
cage.start()
cage.system_service_names.add(cage.name)
# read in datasource configurations
cage.config = initialize_config(config_file, config_override)
# path to congress source dir
src_path = os.path.join(rootdir, "congress")
# add policy engine
engine_path = os.path.join(src_path, "policy/dsepolicy.py")
LOG.info("main::start() engine_path: " + str(engine_path))
cage.loadModule("PolicyEngine", engine_path)
cage.createservice(
name="engine",
moduleName="PolicyEngine",
description="Policy Engine (DseRuntime instance)",
args={'d6cage': cage, 'rootdir': src_path})
engine = cage.service_object('engine')
if statedir is not None:
engine.load_dir(statedir)
engine.initialize_table_subscriptions()
cage.system_service_names.add(engine.name)
engine.debug_mode() # should take this out for production
# add policy api
# TODO(thinrichs): change to real API path.
api_path = os.path.join(src_path, "api/policy_model.py")
LOG.info("main::start() api_path: " + str(api_path))
cage.loadModule("API-policy", api_path)
cage.createservice(
name="api-policy",
moduleName="API-policy",
description="API-policy DSE instance",
args={'policy_engine': engine})
cage.system_service_names.add('api-policy')
# add rule api
api_path = os.path.join(src_path, "api/rule_model.py")
LOG.info("main::start() api_path: " + str(api_path))
cage.loadModule("API-rule", api_path)
cage.createservice(
name="api-rule",
moduleName="API-rule",
description="API-rule DSE instance",
args={'policy_engine': engine})
cage.system_service_names.add('api-rule')
# add table api
api_path = os.path.join(src_path, "api/table_model.py")
LOG.info("main::start() api_path: " + str(api_path))
cage.loadModule("API-table", api_path)
cage.createservice(
name="api-table",
moduleName="API-table",
description="API-table DSE instance",
args={'policy_engine': engine})
cage.system_service_names.add('api-table')
# add row api
api_path = os.path.join(src_path, "api/row_model.py")
LOG.info("main::start() api_path: " + str(api_path))
cage.loadModule("API-row", api_path)
cage.createservice(
name="api-row",
moduleName="API-row",
description="API-row DSE instance",
args={'policy_engine': engine})
cage.system_service_names.add('api-row')
# add datasource api
api_path = os.path.join(src_path, "api/datasource_model.py")
LOG.info("main::start() api_path: " + str(api_path))
cage.loadModule("API-datasource", api_path)
cage.createservice(
name="api-datasource",
moduleName="API-datasource",
description="API-datasource DSE instance",
args={'policy_engine': engine})
cage.system_service_names.add('api-datasource')
# have policy-engine subscribe to api calls
# TODO(thinrichs): either have API publish everything to DSE bus and
# have policy engine subscribe to all those messages
# OR have API interact with individual components directly
# and change all tests so that the policy engine does not need to be
# subscribed to 'policy-update'
engine.subscribe('api-rule', 'policy-update',
callback=engine.receive_policy_update)
# spin up all the configured services, if we have configured them
if cage.config:
for name in cage.config:
if 'module' in cage.config[name]:
load_data_service(name, cage.config[name], cage, src_path)
return cage
def setUp(self):
"""Setup polling tests."""
super(TestDataSourceDriver, self).setUp()
cage = d6cage.d6Cage()
# Create mock of Neutron client so we can control data
mock_factory = mox.Mox()
neutron_client = mock_factory.CreateMock(
neutronclient.v2_0.client.Client)
neutron_client.list_networks().InAnyOrder(1).AndReturn(network1)
neutron_client.list_ports().InAnyOrder(1).AndReturn(port_response)
neutron_client.list_routers().InAnyOrder(1).AndReturn(router_response)
neutron_client.list_security_groups().InAnyOrder(1).AndReturn(
security_group_response)
neutron_client.list_networks().InAnyOrder(2).AndReturn(network2)
neutron_client.list_ports().InAnyOrder(2).AndReturn(port_response)
neutron_client.list_routers().InAnyOrder(2).AndReturn(router_response)
neutron_client.list_security_groups().InAnyOrder(2).AndReturn(
security_group_response)
mock_factory.ReplayAll()
# Create modules (without auto-polling)
cage.loadModule("NeutronDriver",
helper.data_module_path("neutron_driver.py"))
cage.loadModule("PolicyDriver", helper.policy_module_path())
cage.createservice(name="policy", moduleName="PolicyDriver",
args={'d6cage': cage,
'rootdir': helper.data_module_path(''),
'log_actions_only': True})
args = helper.datasource_openstack_args()
args['poll_time'] = 0
args['client'] = neutron_client
cage.createservice(name="neutron", moduleName="NeutronDriver",
args=args)
policy = cage.service_object('policy')
policy.create_policy('neutron')
policy.set_schema(
'neutron', cage.service_object('neutron').get_schema())
cage.service_object('neutron').neutron = neutron_client
policy.debug_mode()
# insert rule into policy to make testing easier.
# (Some of the IDs are auto-generated each time we convert)
policy.insert(create_network_group('p'))
# create some garbage data
args = helper.datasource_openstack_args()
driver = neutron_driver.NeutronDriver(args=args)
network_key_to_index = driver.get_column_map(
neutron_driver.NeutronDriver.NETWORKS)
network_max_index = max(network_key_to_index.values())
args1 = ['1'] * (network_max_index + 1)
args2 = ['2'] * (network_max_index + 1)
args1 = ",".join(args1)
args2 = ",".join(args2)
fake_networks = [
'neutron:networks({})'.format(args1),
'neutron:networks({})'.format(args2)]
# answer to query above for network1
datalog1 = (
'p("240ff9df-df35-43ae-9df5-27fae87f2492") '
'p("340ff9df-df35-43ae-9df5-27fae87f2492") '
'p("440ff9df-df35-43ae-9df5-27fae87f2492")')
# answer to query above for network2
datalog2 = (
'p("240ff9df-df35-43ae-9df5-27fae87f2492") '
'p("640ff9df-df35-43ae-9df5-27fae87f2492") '
'p("540ff9df-df35-43ae-9df5-27fae87f2492")')
# return value
self.info = {}
self.info['cage'] = cage
self.info['datalog1'] = datalog1
self.info['datalog2'] = datalog2
self.info['fake_networks'] = fake_networks
def __init__(self, name, keys, inbox=None, dataPath=None,
policy_engine=None):
super(StatusModel, self).__init__(name, keys, inbox=inbox,
dataPath=dataPath)
self.cage = d6cage.d6Cage()
def create(rootdir, statedir, config_override=None):
"""Get Congress up and running when src is installed in rootdir.
i.e. ROOTDIR=/path/to/congress/congress.
CONFIG_OVERRIDE is a dictionary of dictionaries with configuration
values that overrides those provided in CONFIG_FILE. The top-level
dictionary has keys for the CONFIG_FILE sections, and the second-level
dictionaries store values for that section.
"""
LOG.debug(
"Starting Congress with rootdir=%s, statedir=%s, "
"config_override=%s", rootdir, statedir, config_override)
# create message bus
cage = d6cage.d6Cage()
# read in datasource configurations
cage.config = config_override or {}
# path to congress source dir
src_path = os.path.join(rootdir, "congress")
# add policy engine
engine_path = os.path.join(src_path, "policy_engines/agnostic.py")
LOG.info("main::start() engine_path: %s", engine_path)
cage.loadModule("PolicyEngine", engine_path)
cage.createservice(name="engine",
moduleName="PolicyEngine",
description="Policy Engine (DseRuntime instance)",
args={
'd6cage': cage,
'rootdir': src_path
})
engine = cage.service_object('engine')
if statedir is not None:
engine.load_dir(statedir)
engine.initialize_table_subscriptions()
engine.debug_mode() # should take this out for production
# add policy api
api_path = os.path.join(src_path, "api/policy_model.py")
LOG.info("main::start() api_path: %s", api_path)
cage.loadModule("API-policy", api_path)
cage.createservice(name="api-policy",
moduleName="API-policy",
description="API-policy DSE instance",
args={'policy_engine': engine})
# add rule api
api_path = os.path.join(src_path, "api/rule_model.py")
LOG.info("main::start() api_path: %s", api_path)
cage.loadModule("API-rule", api_path)
cage.createservice(name="api-rule",
moduleName="API-rule",
description="API-rule DSE instance",
args={'policy_engine': engine})
# add table api
api_path = os.path.join(src_path, "api/table_model.py")
LOG.info("main::start() api_path: %s", api_path)
cage.loadModule("API-table", api_path)
cage.createservice(name="api-table",
moduleName="API-table",
description="API-table DSE instance",
args={'policy_engine': engine})
# add row api
api_path = os.path.join(src_path, "api/row_model.py")
LOG.info("main::start() api_path: %s", api_path)
cage.loadModule("API-row", api_path)
cage.createservice(name="api-row",
moduleName="API-row",
description="API-row DSE instance",
args={'policy_engine': engine})
# add status api
api_path = os.path.join(src_path, "api/status_model.py")
LOG.info("main::start() api_path: %s", api_path)
cage.loadModule("API-status", api_path)
cage.createservice(name="api-status",
moduleName="API-status",
description="API-status DSE instance",
args={'policy_engine': engine})
# add schema api
api_path = os.path.join(src_path, "api/schema_model.py")
LOG.info("main::start() api_path: %s", api_path)
cage.loadModule("API-schema", api_path)
cage.createservice(name="api-schema",
moduleName="API-schema",
description="API-schema DSE instance",
args={'policy_engine': engine})
# add datasource/config api
api_path = os.path.join(src_path, "api/datasource_config_model.py")
LOG.info("main::start() api_path: %s", api_path)
cage.loadModule("API-config", api_path)
cage.createservice(name="api-config",
moduleName="API-config",
description="API-config DSE instance",
args={'policy_engine': engine})
# add path for system/datasource-drivers
api_path = os.path.join(src_path, "api/system/driver_model.py")
LOG.info("main::start() api_path: %s", api_path)
cage.loadModule("API-system", api_path)
cage.createservice(name="api-system",
moduleName="API-system",
description="API-system DSE instance",
args={'policy_engine': engine})
# Load policies from database
for policy in db_policy_rules.get_policies():
engine.create_policy(policy.name,
abbr=policy.abbreviation,
kind=policy.kind)
# if this is the first time we are running Congress, need
# to create the default theories (which cannot be deleted)
api_policy = cage.service_object('api-policy')
engine.DEFAULT_THEORY = 'classification'
engine.builtin_policy_names.add(engine.DEFAULT_THEORY)
try:
api_policy.add_item(
{
'name': engine.DEFAULT_THEORY,
'description': 'default policy'
}, {})
except KeyError:
pass
engine.ACTION_THEORY = 'action'
engine.builtin_policy_names.add(engine.ACTION_THEORY)
try:
api_policy.add_item(
{
'kind': ACTION_POLICY_TYPE,
'name': engine.ACTION_THEORY,
'description': 'default action policy'
}, {})
except KeyError:
pass
# have policy-engine subscribe to api calls
# TODO(thinrichs): either have API publish everything to DSE bus and
# have policy engine subscribe to all those messages
# OR have API interact with individual components directly
# and change all tests so that the policy engine does not need to be
# subscribed to 'policy-update'
engine.subscribe('api-rule',
'policy-update',
callback=engine.receive_policy_update)
# spin up all the configured services, if we have configured them
datasource_mgr = datasource_manager.DataSourceManager
drivers = datasource_mgr.get_datasources()
# Setup cage.config as it previously done when it was loaded
# from disk. FIXME(arosen) later!
for driver in drivers:
if not driver['enabled']:
LOG.info("module %s not enabled, skip loading", driver['name'])
continue
driver_info = datasource_mgr.get_driver_info(driver['driver'])
engine.create_policy(driver['name'])
try:
cage.createservice(name=driver['name'],
moduleName=driver_info['module'],
args=driver['config'],
module_driver=True,
type_='datasource_driver',
id_=driver['id'])
except d6cage.DataServiceError:
# FIXME(arosen): If createservice raises congress-server
# dies here. So we catch this exception so the server does
# not die. We need to refactor the dse code so it just
# keeps retrying the driver gracefully...
continue
service = cage.service_object(driver['name'])
engine.set_schema(driver['name'], service.get_schema())
# Insert rules. Needs to be done after datasources are loaded
# so that we can compile away column references at read time.
# If datasources loaded after this, we don't have schemas.
rules = db_policy_rules.get_policy_rules()
for rule in rules:
parsed_rule = engine.parse1(rule.rule)
cage.service_object('api-rule').change_rule(
parsed_rule, {'policy_id': rule.policy_name})
# Start datasource synchronizer after explicitly starting the
# datasources, because the explicit call to create a datasource
# will crash if the synchronizer creates the datasource first.
synchronizer_path = os.path.join(src_path, "synchronizer.py")
LOG.info("main::start() synchronizer: %s", synchronizer_path)
cage.loadModule("Synchronizer", synchronizer_path)
cage.createservice(name="synchronizer",
moduleName="Synchronizer",
description="DB synchronizer instance",
args={'poll_time': cfg.CONF.datasource_sync_period})
synchronizer = cage.service_object('synchronizer')
# add datasource api
api_path = os.path.join(src_path, "api/datasource_model.py")
LOG.info("main::start() api_path: %s", api_path)
cage.loadModule("API-datasource", api_path)
cage.createservice(name="api-datasource",
moduleName="API-datasource",
description="API-datasource DSE instance",
args={
'policy_engine': engine,
'synchronizer': synchronizer
})
return cage
