def get_items(self, params, context=None): """Get items in model. Args: params: A dict-like object containing parameters from the request query string and body. context: Key-values providing frame of reference of request Returns: A dict containing at least a 'results' key whose value is a list of items in the model. Additional keys set in the dict will also be rendered for the user. """ # non-datasource policies (i.e. those persisted to disk) policies = db_policy_rules.get_policies() LOG.debug("persisted policies: %s", policies) persisted_policies = set([p.name for p in policies]) persisted = [self._db_item_to_dict(p) for p in policies] # datasource policies (i.e. those not persisted) nonpersisted = [ self._theory_item_to_dict(self.engine.theory[p]) for p in self.engine.theory if p not in persisted_policies ] return {"results": persisted + nonpersisted}
def synchronize_all_policies(self): """Function to synchronize im-mem policies with DB""" added = 0 removed = 0 try: db_policies = [p.to_dict() for p in db_policy_rules.get_policies()] active_policies = self._get_engine_policies() # Delete engine policies which are not in DB for p in active_policies: if p not in db_policies: LOG.debug("removing policy %s", str(p)) self.engine.delete_policy(p['id']) removed = removed + 1 # Add policies to PE, which are in DB for p in db_policies: if p not in active_policies: LOG.debug("adding policy %s", str(p)) self.engine.create_policy(p['name'], id_=p['id'], abbr=p['abbreviation'], kind=p['kind'], desc=p['description'], owner=p['owner_id']) added = added + 1 LOG.info("engine policies synchronized, added %d removed %d ", added, removed) # synchronize datasource policies self._sync_datasource_policies() LOG.info("completed synchronization of policies") except Exception: LOG.exception("Exception occurred in policy synchronizer periodic" "task on node %s", self.node.node_id) return
def synchronize_policies(self): LOG.debug("Synchronizing policies") # Read policies from DB. cage = d6cage.d6Cage() configured_policies = [{'id': p.id, 'name': p.name, 'abbr': p.abbreviation, 'desc': p.description, 'owner': p.owner, 'kind': p.kind} for p in db_policy_rules.get_policies()] # Read policies from engine engine = cage.service_object('engine') policies = [engine.policy_object(n) for n in engine.policy_names()] active_policies = [] for policy in policies: active_policies.append({'id': policy.id, 'name': policy.name, 'abbr': policy.abbr, 'desc': policy.desc, 'owner': policy.owner, 'kind': policy.kind}) added = 0 removed = 0 for p in active_policies: if (p['kind'] != base.DATASOURCE_POLICY_TYPE and p not in configured_policies): LOG.debug("removing policy %s", str(p)) engine.delete_policy(p['id']) removed = removed + 1 for p in configured_policies: if p not in active_policies: LOG.debug("adding policy %s", str(p)) engine.create_policy(p['name'], id_=p['id'], abbr=p['abbr'], kind=p['kind'], desc=p['desc'], owner=p['owner']) added = added + 1 LOG.debug("synchronize_policies, added %d removed %d", added, removed)
def get_items(self, params, context=None): """Get items in model. Args: params: A dict-like object containing parameters from the request query string and body. context: Key-values providing frame of reference of request Returns: A dict containing at least a 'results' key whose value is a list of items in the model. Additional keys set in the dict will also be rendered for the user. """ # non-datasource policies (i.e. those persisted to disk) policies = db_policy_rules.get_policies() LOG.debug("persisted policies: %s", policies) persisted_policies = set([p.name for p in policies]) persisted = [self._db_item_to_dict(p) for p in policies] # datasource policies (i.e. those not persisted) nonpersisted = [self._theory_item_to_dict(self.engine.theory[p]) for p in self.engine.theory if p not in persisted_policies] return {"results": persisted + nonpersisted}
def add_item(self, item, params, id_=None, context=None): """Add item to model. Args: item: The item to add to the model params: A dict-like object containing parameters from the request query string and body. id_: The ID of the item, or None if an ID should be generated context: Key-values providing frame of reference of request Returns: Tuple of (ID, newly_created_item) Raises: KeyError: ID already exists. """ if id_ is not None: raise webservice.DataModelException( *error_codes.get('add_item_id')) # Reject rules inserted into non-persisted policies # (i.e. datasource policies) policy_name = self.policy_name(context) policies = db_policy_rules.get_policies() persisted_policies = set([p.name for p in policies]) if policy_name not in persisted_policies: if policy_name in self.engine.theory: LOG.debug("add_item error: rule not permitted for policy %s", policy_name) raise webservice.DataModelException( *error_codes.get('rule_not_permitted'), http_status_code=httplib.FORBIDDEN) else: LOG.debug("add_item error: policy %s not exist", policy_name) raise webservice.DataModelException( *error_codes.get('policy_not_exist'), http_status_code=httplib.NOT_FOUND) str_rule = item['rule'] id = uuid.uuid4() try: rule = self.engine.parse(str_rule) if len(rule) == 1: rule = rule[0] else: (num, desc) = error_codes.get('multiple_rules') raise webservice.DataModelException( num, desc + ":: Received multiple rules: " + "; ".join(str(x) for x in rule)) rule.set_id(id) rule.set_name(item.get('name')) rule.set_comment(None) rule.set_original_str(str_rule) changes = self.change_rule(rule, context) except PolicyException as e: LOG.debug("add_item error: invalid rule syntax") (num, desc) = error_codes.get('rule_syntax') raise webservice.DataModelException(num, desc + "::" + str(e)) for change in changes: if change.formula == rule: d = {'rule': rule.pretty_str(), 'id': str(id), 'comment': rule.comment, 'name': item.get('name')} try: db_policy_rules.add_policy_rule( d['id'], policy_name, str_rule, d['comment'], rule_name=d['name']) return (d['id'], d) except Exception as db_exception: try: self.change_rule(rule, context, insert=False) except Exception as change_exception: raise Exception( "Error thrown during recovery from DB error. " "Inconsistent state. DB error: %s. " "New error: %s.", str(db_exception), str(change_exception)) num, desc = error_codes.get('rule_already_exists') raise webservice.DataModelException( num, desc, http_status_code=httplib.CONFLICT)
def create(rootdir, statedir, config_override=None): """Get Congress up and running when src is installed in rootdir. i.e. ROOTDIR=/path/to/congress/congress. CONFIG_OVERRIDE is a dictionary of dictionaries with configuration values that overrides those provided in CONFIG_FILE. The top-level dictionary has keys for the CONFIG_FILE sections, and the second-level dictionaries store values for that section. """ LOG.debug("Starting Congress with rootdir=%s, statedir=%s, " "config_override=%s", rootdir, statedir, config_override) # create message bus cage = d6cage.d6Cage() # read in datasource configurations cage.config = config_override or {} # path to congress source dir src_path = os.path.join(rootdir, "congress") # add policy engine engine_path = os.path.join(src_path, "policy_engines/agnostic.py") LOG.info("main::start() engine_path: %s", engine_path) cage.loadModule("PolicyEngine", engine_path) cage.createservice( name="engine", moduleName="PolicyEngine", description="Policy Engine (DseRuntime instance)", args={'d6cage': cage, 'rootdir': src_path}) engine = cage.service_object('engine') if statedir is not None: engine.load_dir(statedir) engine.initialize_table_subscriptions() engine.debug_mode() # should take this out for production # add policy api api_path = os.path.join(src_path, "api/policy_model.py") LOG.info("main::start() api_path: %s", api_path) cage.loadModule("API-policy", api_path) cage.createservice( name="api-policy", moduleName="API-policy", description="API-policy DSE instance", args={'policy_engine': engine}) # add rule api api_path = os.path.join(src_path, "api/rule_model.py") LOG.info("main::start() api_path: %s", api_path) cage.loadModule("API-rule", api_path) cage.createservice( name="api-rule", moduleName="API-rule", description="API-rule DSE instance", args={'policy_engine': engine}) # add table api api_path = os.path.join(src_path, "api/table_model.py") LOG.info("main::start() api_path: %s", api_path) cage.loadModule("API-table", api_path) cage.createservice( name="api-table", moduleName="API-table", description="API-table DSE instance", args={'policy_engine': engine}) # add row api api_path = os.path.join(src_path, "api/row_model.py") LOG.info("main::start() api_path: %s", api_path) cage.loadModule("API-row", api_path) cage.createservice( name="api-row", moduleName="API-row", description="API-row DSE instance", args={'policy_engine': engine}) # add status api api_path = os.path.join(src_path, "api/status_model.py") LOG.info("main::start() api_path: %s", api_path) cage.loadModule("API-status", api_path) cage.createservice( name="api-status", moduleName="API-status", description="API-status DSE instance", args={'policy_engine': engine}) # add schema api api_path = os.path.join(src_path, "api/schema_model.py") LOG.info("main::start() api_path: %s", api_path) cage.loadModule("API-schema", api_path) cage.createservice( name="api-schema", moduleName="API-schema", description="API-schema DSE instance", args={'policy_engine': engine}) # add datasource/config api api_path = os.path.join(src_path, "api/datasource_config_model.py") LOG.info("main::start() api_path: %s", api_path) cage.loadModule("API-config", api_path) cage.createservice( name="api-config", moduleName="API-config", description="API-config DSE instance", args={'policy_engine': engine}) # add path for system/datasource-drivers api_path = os.path.join(src_path, "api/system/driver_model.py") LOG.info("main::start() api_path: %s", api_path) cage.loadModule("API-system", api_path) cage.createservice( name="api-system", moduleName="API-system", description="API-system DSE instance", args={'policy_engine': engine}) # Load policies from database for policy in db_policy_rules.get_policies(): engine.create_policy( policy.name, abbr=policy.abbreviation, kind=policy.kind) # if this is the first time we are running Congress, need # to create the default theories (which cannot be deleted) api_policy = cage.service_object('api-policy') engine.DEFAULT_THEORY = 'classification' engine.builtin_policy_names.add(engine.DEFAULT_THEORY) try: api_policy.add_item({'name': engine.DEFAULT_THEORY, 'description': 'default policy'}, {}) except KeyError: pass engine.ACTION_THEORY = 'action' engine.builtin_policy_names.add(engine.ACTION_THEORY) try: api_policy.add_item({'kind': ACTION_POLICY_TYPE, 'name': engine.ACTION_THEORY, 'description': 'default action policy'}, {}) except KeyError: pass # have policy-engine subscribe to api calls # TODO(thinrichs): either have API publish everything to DSE bus and # have policy engine subscribe to all those messages # OR have API interact with individual components directly # and change all tests so that the policy engine does not need to be # subscribed to 'policy-update' engine.subscribe('api-rule', 'policy-update', callback=engine.receive_policy_update) # spin up all the configured services, if we have configured them datasource_mgr = datasource_manager.DataSourceManager drivers = datasource_mgr.get_datasources() # Setup cage.config as it previously done when it was loaded # from disk. FIXME(arosen) later! for driver in drivers: if not driver['enabled']: LOG.info("module %s not enabled, skip loading", driver['name']) continue driver_info = datasource_mgr.get_driver_info(driver['driver']) engine.create_policy(driver['name']) try: cage.createservice(name=driver['name'], moduleName=driver_info['module'], args=driver['config'], module_driver=True, type_='datasource_driver', id_=driver['id']) except d6cage.DataServiceError: # FIXME(arosen): If createservice raises congress-server # dies here. So we catch this exception so the server does # not die. We need to refactor the dse code so it just # keeps retrying the driver gracefully... continue service = cage.service_object(driver['name']) engine.set_schema(driver['name'], service.get_schema()) # Insert rules. Needs to be done after datasources are loaded # so that we can compile away column references at read time. # If datasources loaded after this, we don't have schemas. rules = db_policy_rules.get_policy_rules() for rule in rules: parsed_rule = engine.parse1(rule.rule) cage.service_object('api-rule').change_rule( parsed_rule, {'policy_id': rule.policy_name}) # Start datasource synchronizer after explicitly starting the # datasources, because the explicit call to create a datasource # will crash if the synchronizer creates the datasource first. synchronizer_path = os.path.join(src_path, "synchronizer.py") LOG.info("main::start() synchronizer: %s", synchronizer_path) cage.loadModule("Synchronizer", synchronizer_path) cage.createservice( name="synchronizer", moduleName="Synchronizer", description="DB synchronizer instance", args={'poll_time': cfg.CONF.datasource_sync_period}) synchronizer = cage.service_object('synchronizer') # add datasource api api_path = os.path.join(src_path, "api/datasource_model.py") LOG.info("main::start() api_path: %s", api_path) cage.loadModule("API-datasource", api_path) cage.createservice( name="api-datasource", moduleName="API-datasource", description="API-datasource DSE instance", args={'policy_engine': engine, 'synchronizer': synchronizer}) return cage
def create(rootdir, statedir, config_override=None): """Get Congress up and running when src is installed in rootdir. i.e. ROOTDIR=/path/to/congress/congress. CONFIG_OVERRIDE is a dictionary of dictionaries with configuration values that overrides those provided in CONFIG_FILE. The top-level dictionary has keys for the CONFIG_FILE sections, and the second-level dictionaries store values for that section. """ LOG.debug( "Starting Congress with rootdir=%s, statedir=%s, " "config_override=%s", rootdir, statedir, config_override) # create message bus cage = d6cage.d6Cage() # read in datasource configurations cage.config = config_override or {} # path to congress source dir src_path = os.path.join(rootdir, "congress") # add policy engine engine_path = os.path.join(src_path, "policy_engines/agnostic.py") LOG.info("main::start() engine_path: %s", engine_path) cage.loadModule("PolicyEngine", engine_path) cage.createservice(name="engine", moduleName="PolicyEngine", description="Policy Engine (DseRuntime instance)", args={ 'd6cage': cage, 'rootdir': src_path }) engine = cage.service_object('engine') if statedir is not None: engine.load_dir(statedir) engine.initialize_table_subscriptions() engine.debug_mode() # should take this out for production # add policy api api_path = os.path.join(src_path, "api/policy_model.py") LOG.info("main::start() api_path: %s", api_path) cage.loadModule("API-policy", api_path) cage.createservice(name="api-policy", moduleName="API-policy", description="API-policy DSE instance", args={'policy_engine': engine}) # add rule api api_path = os.path.join(src_path, "api/rule_model.py") LOG.info("main::start() api_path: %s", api_path) cage.loadModule("API-rule", api_path) cage.createservice(name="api-rule", moduleName="API-rule", description="API-rule DSE instance", args={'policy_engine': engine}) # add table api api_path = os.path.join(src_path, "api/table_model.py") LOG.info("main::start() api_path: %s", api_path) cage.loadModule("API-table", api_path) cage.createservice(name="api-table", moduleName="API-table", description="API-table DSE instance", args={'policy_engine': engine}) # add row api api_path = os.path.join(src_path, "api/row_model.py") LOG.info("main::start() api_path: %s", api_path) cage.loadModule("API-row", api_path) cage.createservice(name="api-row", moduleName="API-row", description="API-row DSE instance", args={'policy_engine': engine}) # add status api api_path = os.path.join(src_path, "api/status_model.py") LOG.info("main::start() api_path: %s", api_path) cage.loadModule("API-status", api_path) cage.createservice(name="api-status", moduleName="API-status", description="API-status DSE instance", args={'policy_engine': engine}) # add schema api api_path = os.path.join(src_path, "api/schema_model.py") LOG.info("main::start() api_path: %s", api_path) cage.loadModule("API-schema", api_path) cage.createservice(name="api-schema", moduleName="API-schema", description="API-schema DSE instance", args={'policy_engine': engine}) # add datasource/config api api_path = os.path.join(src_path, "api/datasource_config_model.py") LOG.info("main::start() api_path: %s", api_path) cage.loadModule("API-config", api_path) cage.createservice(name="api-config", moduleName="API-config", description="API-config DSE instance", args={'policy_engine': engine}) # add path for system/datasource-drivers api_path = os.path.join(src_path, "api/system/driver_model.py") LOG.info("main::start() api_path: %s", api_path) cage.loadModule("API-system", api_path) cage.createservice(name="api-system", moduleName="API-system", description="API-system DSE instance", args={'policy_engine': engine}) # Load policies from database for policy in db_policy_rules.get_policies(): engine.create_policy(policy.name, abbr=policy.abbreviation, kind=policy.kind) # if this is the first time we are running Congress, need # to create the default theories (which cannot be deleted) api_policy = cage.service_object('api-policy') engine.DEFAULT_THEORY = 'classification' engine.builtin_policy_names.add(engine.DEFAULT_THEORY) try: api_policy.add_item( { 'name': engine.DEFAULT_THEORY, 'description': 'default policy' }, {}) except KeyError: pass engine.ACTION_THEORY = 'action' engine.builtin_policy_names.add(engine.ACTION_THEORY) try: api_policy.add_item( { 'kind': ACTION_POLICY_TYPE, 'name': engine.ACTION_THEORY, 'description': 'default action policy' }, {}) except KeyError: pass # have policy-engine subscribe to api calls # TODO(thinrichs): either have API publish everything to DSE bus and # have policy engine subscribe to all those messages # OR have API interact with individual components directly # and change all tests so that the policy engine does not need to be # subscribed to 'policy-update' engine.subscribe('api-rule', 'policy-update', callback=engine.receive_policy_update) # spin up all the configured services, if we have configured them datasource_mgr = datasource_manager.DataSourceManager drivers = datasource_mgr.get_datasources() # Setup cage.config as it previously done when it was loaded # from disk. FIXME(arosen) later! for driver in drivers: if not driver['enabled']: LOG.info("module %s not enabled, skip loading", driver['name']) continue driver_info = datasource_mgr.get_driver_info(driver['driver']) engine.create_policy(driver['name']) try: cage.createservice(name=driver['name'], moduleName=driver_info['module'], args=driver['config'], module_driver=True, type_='datasource_driver', id_=driver['id']) except d6cage.DataServiceError: # FIXME(arosen): If createservice raises congress-server # dies here. So we catch this exception so the server does # not die. We need to refactor the dse code so it just # keeps retrying the driver gracefully... continue service = cage.service_object(driver['name']) engine.set_schema(driver['name'], service.get_schema()) # Insert rules. Needs to be done after datasources are loaded # so that we can compile away column references at read time. # If datasources loaded after this, we don't have schemas. rules = db_policy_rules.get_policy_rules() for rule in rules: parsed_rule = engine.parse1(rule.rule) cage.service_object('api-rule').change_rule( parsed_rule, {'policy_id': rule.policy_name}) # Start datasource synchronizer after explicitly starting the # datasources, because the explicit call to create a datasource # will crash if the synchronizer creates the datasource first. synchronizer_path = os.path.join(src_path, "synchronizer.py") LOG.info("main::start() synchronizer: %s", synchronizer_path) cage.loadModule("Synchronizer", synchronizer_path) cage.createservice(name="synchronizer", moduleName="Synchronizer", description="DB synchronizer instance", args={'poll_time': cfg.CONF.datasource_sync_period}) synchronizer = cage.service_object('synchronizer') # add datasource api api_path = os.path.join(src_path, "api/datasource_model.py") LOG.info("main::start() api_path: %s", api_path) cage.loadModule("API-datasource", api_path) cage.createservice(name="api-datasource", moduleName="API-datasource", description="API-datasource DSE instance", args={ 'policy_engine': engine, 'synchronizer': synchronizer }) return cage