def get_items(self, params, context=None):
"""Get items in model.
Args:
params: A dict-like object containing parameters
from the request query string and body.
context: Key-values providing frame of reference of request
Returns: A dict containing at least a 'results' key whose value is
a list of items in the model. Additional keys set in the
dict will also be rendered for the user.
"""
# non-datasource policies (i.e. those persisted to disk)
policies = db_policy_rules.get_policies()
LOG.debug("persisted policies: %s", policies)
persisted_policies = set([p.name for p in policies])
persisted = [self._db_item_to_dict(p) for p in policies]
# datasource policies (i.e. those not persisted)
nonpersisted = [
self._theory_item_to_dict(self.engine.theory[p])
for p in self.engine.theory if p not in persisted_policies
]
return {"results": persisted + nonpersisted}
def synchronize_all_policies(self):
"""Function to synchronize im-mem policies with DB"""
added = 0
removed = 0
try:
db_policies = [p.to_dict() for p in db_policy_rules.get_policies()]
active_policies = self._get_engine_policies()
# Delete engine policies which are not in DB
for p in active_policies:
if p not in db_policies:
LOG.debug("removing policy %s", str(p))
self.engine.delete_policy(p['id'])
removed = removed + 1
# Add policies to PE, which are in DB
for p in db_policies:
if p not in active_policies:
LOG.debug("adding policy %s", str(p))
self.engine.create_policy(p['name'], id_=p['id'],
abbr=p['abbreviation'],
kind=p['kind'],
desc=p['description'],
owner=p['owner_id'])
added = added + 1
LOG.info("engine policies synchronized, added %d removed %d ",
added, removed)
# synchronize datasource policies
self._sync_datasource_policies()
LOG.info("completed synchronization of policies")
except Exception:
LOG.exception("Exception occurred in policy synchronizer periodic"
"task on node %s", self.node.node_id)
return
def synchronize_policies(self):
LOG.debug("Synchronizing policies")
# Read policies from DB.
cage = d6cage.d6Cage()
configured_policies = [{'id': p.id,
'name': p.name,
'abbr': p.abbreviation,
'desc': p.description,
'owner': p.owner,
'kind': p.kind}
for p in db_policy_rules.get_policies()]
# Read policies from engine
engine = cage.service_object('engine')
policies = [engine.policy_object(n) for n in engine.policy_names()]
active_policies = []
for policy in policies:
active_policies.append({'id': policy.id,
'name': policy.name,
'abbr': policy.abbr,
'desc': policy.desc,
'owner': policy.owner,
'kind': policy.kind})
added = 0
removed = 0
for p in active_policies:
if (p['kind'] != base.DATASOURCE_POLICY_TYPE and
p not in configured_policies):
LOG.debug("removing policy %s", str(p))
engine.delete_policy(p['id'])
removed = removed + 1
for p in configured_policies:
if p not in active_policies:
LOG.debug("adding policy %s", str(p))
engine.create_policy(p['name'], id_=p['id'], abbr=p['abbr'],
kind=p['kind'], desc=p['desc'],
owner=p['owner'])
added = added + 1
LOG.debug("synchronize_policies, added %d removed %d",
added, removed)
def get_items(self, params, context=None):
"""Get items in model.
Args:
params: A dict-like object containing parameters
from the request query string and body.
context: Key-values providing frame of reference of request
Returns: A dict containing at least a 'results' key whose value is
a list of items in the model. Additional keys set in the
dict will also be rendered for the user.
"""
# non-datasource policies (i.e. those persisted to disk)
policies = db_policy_rules.get_policies()
LOG.debug("persisted policies: %s", policies)
persisted_policies = set([p.name for p in policies])
persisted = [self._db_item_to_dict(p) for p in policies]
# datasource policies (i.e. those not persisted)
nonpersisted = [self._theory_item_to_dict(self.engine.theory[p])
for p in self.engine.theory
if p not in persisted_policies]
return {"results": persisted + nonpersisted}
def add_item(self, item, params, id_=None, context=None):
"""Add item to model.
Args:
item: The item to add to the model
params: A dict-like object containing parameters
from the request query string and body.
id_: The ID of the item, or None if an ID should be generated
context: Key-values providing frame of reference of request
Returns:
Tuple of (ID, newly_created_item)
Raises:
KeyError: ID already exists.
"""
if id_ is not None:
raise webservice.DataModelException(
*error_codes.get('add_item_id'))
# Reject rules inserted into non-persisted policies
# (i.e. datasource policies)
policy_name = self.policy_name(context)
policies = db_policy_rules.get_policies()
persisted_policies = set([p.name for p in policies])
if policy_name not in persisted_policies:
if policy_name in self.engine.theory:
LOG.debug("add_item error: rule not permitted for policy %s",
policy_name)
raise webservice.DataModelException(
*error_codes.get('rule_not_permitted'),
http_status_code=httplib.FORBIDDEN)
else:
LOG.debug("add_item error: policy %s not exist", policy_name)
raise webservice.DataModelException(
*error_codes.get('policy_not_exist'),
http_status_code=httplib.NOT_FOUND)
str_rule = item['rule']
id = uuid.uuid4()
try:
rule = self.engine.parse(str_rule)
if len(rule) == 1:
rule = rule[0]
else:
(num, desc) = error_codes.get('multiple_rules')
raise webservice.DataModelException(
num, desc + ":: Received multiple rules: " +
"; ".join(str(x) for x in rule))
rule.set_id(id)
rule.set_name(item.get('name'))
rule.set_comment(None)
rule.set_original_str(str_rule)
changes = self.change_rule(rule, context)
except PolicyException as e:
LOG.debug("add_item error: invalid rule syntax")
(num, desc) = error_codes.get('rule_syntax')
raise webservice.DataModelException(num, desc + "::" + str(e))
for change in changes:
if change.formula == rule:
d = {'rule': rule.pretty_str(),
'id': str(id),
'comment': rule.comment,
'name': item.get('name')}
try:
db_policy_rules.add_policy_rule(
d['id'], policy_name, str_rule, d['comment'],
rule_name=d['name'])
return (d['id'], d)
except Exception as db_exception:
try:
self.change_rule(rule, context, insert=False)
except Exception as change_exception:
raise Exception(
"Error thrown during recovery from DB error. "
"Inconsistent state. DB error: %s. "
"New error: %s.", str(db_exception),
str(change_exception))
num, desc = error_codes.get('rule_already_exists')
raise webservice.DataModelException(
num, desc, http_status_code=httplib.CONFLICT)
def create(rootdir, statedir, config_override=None):
"""Get Congress up and running when src is installed in rootdir.
i.e. ROOTDIR=/path/to/congress/congress.
CONFIG_OVERRIDE is a dictionary of dictionaries with configuration
values that overrides those provided in CONFIG_FILE. The top-level
dictionary has keys for the CONFIG_FILE sections, and the second-level
dictionaries store values for that section.
"""
LOG.debug("Starting Congress with rootdir=%s, statedir=%s, "
"config_override=%s",
rootdir, statedir, config_override)
# create message bus
cage = d6cage.d6Cage()
# read in datasource configurations
cage.config = config_override or {}
# path to congress source dir
src_path = os.path.join(rootdir, "congress")
# add policy engine
engine_path = os.path.join(src_path, "policy_engines/agnostic.py")
LOG.info("main::start() engine_path: %s", engine_path)
cage.loadModule("PolicyEngine", engine_path)
cage.createservice(
name="engine",
moduleName="PolicyEngine",
description="Policy Engine (DseRuntime instance)",
args={'d6cage': cage, 'rootdir': src_path})
engine = cage.service_object('engine')
if statedir is not None:
engine.load_dir(statedir)
engine.initialize_table_subscriptions()
engine.debug_mode() # should take this out for production
# add policy api
api_path = os.path.join(src_path, "api/policy_model.py")
LOG.info("main::start() api_path: %s", api_path)
cage.loadModule("API-policy", api_path)
cage.createservice(
name="api-policy",
moduleName="API-policy",
description="API-policy DSE instance",
args={'policy_engine': engine})
# add rule api
api_path = os.path.join(src_path, "api/rule_model.py")
LOG.info("main::start() api_path: %s", api_path)
cage.loadModule("API-rule", api_path)
cage.createservice(
name="api-rule",
moduleName="API-rule",
description="API-rule DSE instance",
args={'policy_engine': engine})
# add table api
api_path = os.path.join(src_path, "api/table_model.py")
LOG.info("main::start() api_path: %s", api_path)
cage.loadModule("API-table", api_path)
cage.createservice(
name="api-table",
moduleName="API-table",
description="API-table DSE instance",
args={'policy_engine': engine})
# add row api
api_path = os.path.join(src_path, "api/row_model.py")
LOG.info("main::start() api_path: %s", api_path)
cage.loadModule("API-row", api_path)
cage.createservice(
name="api-row",
moduleName="API-row",
description="API-row DSE instance",
args={'policy_engine': engine})
# add status api
api_path = os.path.join(src_path, "api/status_model.py")
LOG.info("main::start() api_path: %s", api_path)
cage.loadModule("API-status", api_path)
cage.createservice(
name="api-status",
moduleName="API-status",
description="API-status DSE instance",
args={'policy_engine': engine})
# add schema api
api_path = os.path.join(src_path, "api/schema_model.py")
LOG.info("main::start() api_path: %s", api_path)
cage.loadModule("API-schema", api_path)
cage.createservice(
name="api-schema",
moduleName="API-schema",
description="API-schema DSE instance",
args={'policy_engine': engine})
# add datasource/config api
api_path = os.path.join(src_path, "api/datasource_config_model.py")
LOG.info("main::start() api_path: %s", api_path)
cage.loadModule("API-config", api_path)
cage.createservice(
name="api-config",
moduleName="API-config",
description="API-config DSE instance",
args={'policy_engine': engine})
# add path for system/datasource-drivers
api_path = os.path.join(src_path, "api/system/driver_model.py")
LOG.info("main::start() api_path: %s", api_path)
cage.loadModule("API-system", api_path)
cage.createservice(
name="api-system",
moduleName="API-system",
description="API-system DSE instance",
args={'policy_engine': engine})
# Load policies from database
for policy in db_policy_rules.get_policies():
engine.create_policy(
policy.name, abbr=policy.abbreviation, kind=policy.kind)
# if this is the first time we are running Congress, need
# to create the default theories (which cannot be deleted)
api_policy = cage.service_object('api-policy')
engine.DEFAULT_THEORY = 'classification'
engine.builtin_policy_names.add(engine.DEFAULT_THEORY)
try:
api_policy.add_item({'name': engine.DEFAULT_THEORY,
'description': 'default policy'}, {})
except KeyError:
pass
engine.ACTION_THEORY = 'action'
engine.builtin_policy_names.add(engine.ACTION_THEORY)
try:
api_policy.add_item({'kind': ACTION_POLICY_TYPE,
'name': engine.ACTION_THEORY,
'description': 'default action policy'},
{})
except KeyError:
pass
# have policy-engine subscribe to api calls
# TODO(thinrichs): either have API publish everything to DSE bus and
# have policy engine subscribe to all those messages
# OR have API interact with individual components directly
# and change all tests so that the policy engine does not need to be
# subscribed to 'policy-update'
engine.subscribe('api-rule', 'policy-update',
callback=engine.receive_policy_update)
# spin up all the configured services, if we have configured them
datasource_mgr = datasource_manager.DataSourceManager
drivers = datasource_mgr.get_datasources()
# Setup cage.config as it previously done when it was loaded
# from disk. FIXME(arosen) later!
for driver in drivers:
if not driver['enabled']:
LOG.info("module %s not enabled, skip loading", driver['name'])
continue
driver_info = datasource_mgr.get_driver_info(driver['driver'])
engine.create_policy(driver['name'])
try:
cage.createservice(name=driver['name'],
moduleName=driver_info['module'],
args=driver['config'],
module_driver=True,
type_='datasource_driver',
id_=driver['id'])
except d6cage.DataServiceError:
# FIXME(arosen): If createservice raises congress-server
# dies here. So we catch this exception so the server does
# not die. We need to refactor the dse code so it just
# keeps retrying the driver gracefully...
continue
service = cage.service_object(driver['name'])
engine.set_schema(driver['name'], service.get_schema())
# Insert rules. Needs to be done after datasources are loaded
# so that we can compile away column references at read time.
# If datasources loaded after this, we don't have schemas.
rules = db_policy_rules.get_policy_rules()
for rule in rules:
parsed_rule = engine.parse1(rule.rule)
cage.service_object('api-rule').change_rule(
parsed_rule,
{'policy_id': rule.policy_name})
# Start datasource synchronizer after explicitly starting the
# datasources, because the explicit call to create a datasource
# will crash if the synchronizer creates the datasource first.
synchronizer_path = os.path.join(src_path, "synchronizer.py")
LOG.info("main::start() synchronizer: %s", synchronizer_path)
cage.loadModule("Synchronizer", synchronizer_path)
cage.createservice(
name="synchronizer",
moduleName="Synchronizer",
description="DB synchronizer instance",
args={'poll_time': cfg.CONF.datasource_sync_period})
synchronizer = cage.service_object('synchronizer')
# add datasource api
api_path = os.path.join(src_path, "api/datasource_model.py")
LOG.info("main::start() api_path: %s", api_path)
cage.loadModule("API-datasource", api_path)
cage.createservice(
name="api-datasource",
moduleName="API-datasource",
description="API-datasource DSE instance",
args={'policy_engine': engine, 'synchronizer': synchronizer})
return cage
def create(rootdir, statedir, config_override=None):
"""Get Congress up and running when src is installed in rootdir.
i.e. ROOTDIR=/path/to/congress/congress.
CONFIG_OVERRIDE is a dictionary of dictionaries with configuration
values that overrides those provided in CONFIG_FILE. The top-level
dictionary has keys for the CONFIG_FILE sections, and the second-level
dictionaries store values for that section.
"""
LOG.debug(
"Starting Congress with rootdir=%s, statedir=%s, "
"config_override=%s", rootdir, statedir, config_override)
# create message bus
cage = d6cage.d6Cage()
# read in datasource configurations
cage.config = config_override or {}
# path to congress source dir
src_path = os.path.join(rootdir, "congress")
# add policy engine
engine_path = os.path.join(src_path, "policy_engines/agnostic.py")
LOG.info("main::start() engine_path: %s", engine_path)
cage.loadModule("PolicyEngine", engine_path)
cage.createservice(name="engine",
moduleName="PolicyEngine",
description="Policy Engine (DseRuntime instance)",
args={
'd6cage': cage,
'rootdir': src_path
})
engine = cage.service_object('engine')
if statedir is not None:
engine.load_dir(statedir)
engine.initialize_table_subscriptions()
engine.debug_mode() # should take this out for production
# add policy api
api_path = os.path.join(src_path, "api/policy_model.py")
LOG.info("main::start() api_path: %s", api_path)
cage.loadModule("API-policy", api_path)
cage.createservice(name="api-policy",
moduleName="API-policy",
description="API-policy DSE instance",
args={'policy_engine': engine})
# add rule api
api_path = os.path.join(src_path, "api/rule_model.py")
LOG.info("main::start() api_path: %s", api_path)
cage.loadModule("API-rule", api_path)
cage.createservice(name="api-rule",
moduleName="API-rule",
description="API-rule DSE instance",
args={'policy_engine': engine})
# add table api
api_path = os.path.join(src_path, "api/table_model.py")
LOG.info("main::start() api_path: %s", api_path)
cage.loadModule("API-table", api_path)
cage.createservice(name="api-table",
moduleName="API-table",
description="API-table DSE instance",
args={'policy_engine': engine})
# add row api
api_path = os.path.join(src_path, "api/row_model.py")
LOG.info("main::start() api_path: %s", api_path)
cage.loadModule("API-row", api_path)
cage.createservice(name="api-row",
moduleName="API-row",
description="API-row DSE instance",
args={'policy_engine': engine})
# add status api
api_path = os.path.join(src_path, "api/status_model.py")
LOG.info("main::start() api_path: %s", api_path)
cage.loadModule("API-status", api_path)
cage.createservice(name="api-status",
moduleName="API-status",
description="API-status DSE instance",
args={'policy_engine': engine})
# add schema api
api_path = os.path.join(src_path, "api/schema_model.py")
LOG.info("main::start() api_path: %s", api_path)
cage.loadModule("API-schema", api_path)
cage.createservice(name="api-schema",
moduleName="API-schema",
description="API-schema DSE instance",
args={'policy_engine': engine})
# add datasource/config api
api_path = os.path.join(src_path, "api/datasource_config_model.py")
LOG.info("main::start() api_path: %s", api_path)
cage.loadModule("API-config", api_path)
cage.createservice(name="api-config",
moduleName="API-config",
description="API-config DSE instance",
args={'policy_engine': engine})
# add path for system/datasource-drivers
api_path = os.path.join(src_path, "api/system/driver_model.py")
LOG.info("main::start() api_path: %s", api_path)
cage.loadModule("API-system", api_path)
cage.createservice(name="api-system",
moduleName="API-system",
description="API-system DSE instance",
args={'policy_engine': engine})
# Load policies from database
for policy in db_policy_rules.get_policies():
engine.create_policy(policy.name,
abbr=policy.abbreviation,
kind=policy.kind)
# if this is the first time we are running Congress, need
# to create the default theories (which cannot be deleted)
api_policy = cage.service_object('api-policy')
engine.DEFAULT_THEORY = 'classification'
engine.builtin_policy_names.add(engine.DEFAULT_THEORY)
try:
api_policy.add_item(
{
'name': engine.DEFAULT_THEORY,
'description': 'default policy'
}, {})
except KeyError:
pass
engine.ACTION_THEORY = 'action'
engine.builtin_policy_names.add(engine.ACTION_THEORY)
try:
api_policy.add_item(
{
'kind': ACTION_POLICY_TYPE,
'name': engine.ACTION_THEORY,
'description': 'default action policy'
}, {})
except KeyError:
pass
# have policy-engine subscribe to api calls
# TODO(thinrichs): either have API publish everything to DSE bus and
# have policy engine subscribe to all those messages
# OR have API interact with individual components directly
# and change all tests so that the policy engine does not need to be
# subscribed to 'policy-update'
engine.subscribe('api-rule',
'policy-update',
callback=engine.receive_policy_update)
# spin up all the configured services, if we have configured them
datasource_mgr = datasource_manager.DataSourceManager
drivers = datasource_mgr.get_datasources()
# Setup cage.config as it previously done when it was loaded
# from disk. FIXME(arosen) later!
for driver in drivers:
if not driver['enabled']:
LOG.info("module %s not enabled, skip loading", driver['name'])
continue
driver_info = datasource_mgr.get_driver_info(driver['driver'])
engine.create_policy(driver['name'])
try:
cage.createservice(name=driver['name'],
moduleName=driver_info['module'],
args=driver['config'],
module_driver=True,
type_='datasource_driver',
id_=driver['id'])
except d6cage.DataServiceError:
# FIXME(arosen): If createservice raises congress-server
# dies here. So we catch this exception so the server does
# not die. We need to refactor the dse code so it just
# keeps retrying the driver gracefully...
continue
service = cage.service_object(driver['name'])
engine.set_schema(driver['name'], service.get_schema())
# Insert rules. Needs to be done after datasources are loaded
# so that we can compile away column references at read time.
# If datasources loaded after this, we don't have schemas.
rules = db_policy_rules.get_policy_rules()
for rule in rules:
parsed_rule = engine.parse1(rule.rule)
cage.service_object('api-rule').change_rule(
parsed_rule, {'policy_id': rule.policy_name})
# Start datasource synchronizer after explicitly starting the
# datasources, because the explicit call to create a datasource
# will crash if the synchronizer creates the datasource first.
synchronizer_path = os.path.join(src_path, "synchronizer.py")
LOG.info("main::start() synchronizer: %s", synchronizer_path)
cage.loadModule("Synchronizer", synchronizer_path)
cage.createservice(name="synchronizer",
moduleName="Synchronizer",
description="DB synchronizer instance",
args={'poll_time': cfg.CONF.datasource_sync_period})
synchronizer = cage.service_object('synchronizer')
# add datasource api
api_path = os.path.join(src_path, "api/datasource_model.py")
LOG.info("main::start() api_path: %s", api_path)
cage.loadModule("API-datasource", api_path)
cage.createservice(name="api-datasource",
moduleName="API-datasource",
description="API-datasource DSE instance",
args={
'policy_engine': engine,
'synchronizer': synchronizer
})
return cage
