def setUp(self):
        super(BenchmarkDatasource, self).setUp()
        config = {'benchmark': {
                  'module': helper.data_module_path('benchmark_driver.py'),
                  'poll_time': 0}}
        cage = harness.create(helper.root_path(), None, config)
        engine = cage.service_object('engine')
        api = {'policy': cage.service_object('api-policy'),
               'rule': cage.service_object('api-rule'),
               'table': cage.service_object('api-table'),
               'row': cage.service_object('api-row'),
               'datasource': cage.service_object('api-datasource'),
               'status': cage.service_object('api-status'),
               'schema': cage.service_object('api-schema')}
        helper.retry_check_subscriptions(engine, [(api['rule'].name,
                                         'policy-update')])
        helper.retry_check_subscribers(api['rule'], [(engine.name,
                                       'policy-update')])
        self.assertTrue('benchmark' in cage.services)
        datasource = cage.service_object('benchmark')
        table_name = datasource.BENCHTABLE

        self.assertEqual(datasource.state, {})

        # add a subscriber to ensure the updates end up in datasource.dataPath
        pubdata = datasource.pubdata.setdefault(table_name,
                                                dataobj.pubData(table_name))
        pubdata.addsubscriber(self.__class__.__name__, "push", "")
        self.assertTrue(datasource.pubdata[table_name])

        self.cage = cage
        self.engine = engine
        self.api = api
        self.table_name = table_name
        self.datasource = datasource
Example #2
0
 def test_startup(self):
     """Test that everything is properly loaded at startup."""
     engine = self.engine
     api = self.api
     helper.retry_check_subscriptions(
         engine, [(api['rule'].name, 'policy-update')])
     helper.retry_check_subscribers(
         api['rule'], [(engine.name, 'policy-update')])
Example #3
0
 def test_startup(self):
     """Test that everything is properly loaded at startup."""
     engine = self.engine
     api = self.api
     helper.retry_check_subscriptions(engine,
                                      [(api['rule'].name, 'policy-update')])
     helper.retry_check_subscribers(api['rule'],
                                    [(engine.name, 'policy-update')])
Example #4
0
    def test_communication(self):
        """Test for communication.

        Test the module's ability to be loaded into the DSE
        by checking its ability to communicate on the message bus.
        """
        cage = d6cage.d6Cage()

        # Create modules.
        # Turn off polling so we don't need to deal with real data.
        args = helper.datasource_openstack_args()
        args['poll_time'] = 0
        cage.loadModule("NovaDriver",
                        helper.data_module_path("nova_driver.py"))
        cage.loadModule("PolicyDriver", helper.policy_module_path())
        cage.createservice(name="policy",
                           moduleName="PolicyDriver",
                           args={
                               'd6cage': cage,
                               'rootdir': helper.data_module_path('')
                           })
        cage.createservice(name="nova", moduleName="NovaDriver", args=args)

        # Check that data gets sent from nova to policy as expected
        nova = cage.service_object('nova')
        policy = cage.service_object('policy')
        policy.debug_mode()
        policy.create_policy('nova')
        policy.set_schema('nova', compile.Schema({'server': (1, )}))
        policy.subscribe('nova', 'server', callback=policy.receive_data)

        # publishing is slightly convoluted b/c deltas are computed
        #  automatically.  (Not just convenient--useful so that DSE
        #  properly handles the initial state problem.)
        # Need to set nova.state and nova.prior_state and then publish
        #  anything.

        # publish server(1), server(2), server(3)
        helper.retry_check_subscribers(nova, [(policy.name, 'server')])
        nova.prior_state = {}
        nova.state['server'] = set([(1, ), (2, ), (3, )])
        nova.publish('server', None)
        helper.retry_check_db_equal(
            policy, 'nova:server(x)',
            'nova:server(1) nova:server(2) nova:server(3)')

        # publish server(1), server(4), server(5)
        nova.prior_state['server'] = nova.state['server']
        nova.state['server'] = set([(1, ), (4, ), (5, )])
        nova.publish('server', None)
        helper.retry_check_db_equal(
            policy, 'nova:server(x)',
            'nova:server(1) nova:server(4) nova:server(5)')
Example #5
0
    def test_communication(self):
        """Test for communication.

        Test the module's ability to be loaded into the DSE
        by checking its ability to communicate on the message bus.
        """
        cage = d6cage.d6Cage()

        # Create modules.
        # Turn off polling so we don't need to deal with real data.
        args = helper.datasource_openstack_args()
        args['poll_time'] = 0
        cage.loadModule("NovaDriver",
                        helper.data_module_path("nova_driver.py"))
        cage.loadModule("PolicyDriver", helper.policy_module_path())
        cage.createservice(name="policy", moduleName="PolicyDriver",
                           args={'d6cage': cage,
                                 'rootdir': helper.data_module_path(''),
                                 'log_actions_only': True})
        cage.createservice(name="nova", moduleName="NovaDriver", args=args)

        # Check that data gets sent from nova to policy as expected
        nova = cage.service_object('nova')
        policy = cage.service_object('policy')
        policy.debug_mode()
        policy.create_policy('nova')
        policy.set_schema('nova', compile.Schema({'server': (1,)}))
        policy.subscribe('nova', 'server',
                         callback=policy.receive_data)

        # publishing is slightly convoluted b/c deltas are computed
        #  automatically.  (Not just convenient--useful so that DSE
        #  properly handles the initial state problem.)
        # Need to set nova.state and nova.prior_state and then publish
        #  anything.

        # publish server(1), server(2), server(3)
        helper.retry_check_subscribers(nova, [(policy.name, 'server')])
        nova.prior_state = {}
        nova.state['server'] = set([(1,), (2,), (3,)])
        nova.publish('server', None)
        helper.retry_check_db_equal(
            policy, 'nova:server(x)',
            'nova:server(1) nova:server(2) nova:server(3)')

        # publish server(1), server(4), server(5)
        nova.prior_state['server'] = nova.state['server']
        nova.state['server'] = set([(1,), (4,), (5,)])
        nova.publish('server', None)
        helper.retry_check_db_equal(
            policy, 'nova:server(x)',
            'nova:server(1) nova:server(4) nova:server(5)')
Example #6
0
    def test_policy_table_publish(self):
        """Policy table result publish

        Test basic DSE functionality with policy engine and table result
        publish.
        """

        cage = congress.dse.d6cage.d6Cage()
        cage.loadModule("TestDriver",
                        helper.data_module_path(
                            "../tests/datasources/test_driver.py"))
        cage.loadModule("TestPolicy", helper.policy_module_path())
        cage.createservice(name="data", moduleName="TestDriver",
                           args=helper.datasource_openstack_args())
        cage.createservice(name="policy", moduleName="TestPolicy",
                           args={'d6cage': cage, 'rootdir': '',
                                 'log_actions_only': True})
        data = cage.services['data']['object']
        policy = cage.services['policy']['object']
        policy.create_policy('data')
        policy.create_policy('classification')
        policy.set_schema('data', compile.Schema({'q': (1,)}))
        policy.insert('p(x):-data:q(x),gt(x,2)', target='classification')
        data.subscribe('policy', 'classification:p', callback=data.receive_msg)
        helper.retry_check_subscribers(policy, [('data', 'classification:p')])
        self.assertEqual(policy.policySubData.keys(),
                         [('p', 'classification', None)])
        policy.insert('q(1)', target='data')
        # no entry here
        self.assertEqual(data.get_msg_data(), '{}')
        policy.insert('q(2)', target='data')
        policy.insert('q(3)', target='data')
        # get an update
        helper.retry_check_for_message_data(data, 'insert[p(3)]')
        self.assertEqual(data.get_msg_data(), 'insert[p(3)]')
        # subscribe again to get a full table
        data.subscribe('policy', 'classification:p', callback=data.receive_msg)
        helper.retry_check_for_message_data(data, 'p(3)')
        self.assertEqual(data.get_msg_data(), 'p(3)')
        # get another update
        policy.insert('q(4)', target='data')
        helper.retry_check_for_message_data(data, 'insert[p(4)]')
        self.assertEqual(data.get_msg_data(), 'insert[p(4)]')
        # get another update
        policy.delete('q(4)', target='data')
        helper.retry_check_for_message_data(data, 'delete[p(4)]')
        self.assertEqual(data.get_msg_data(), 'delete[p(4)]')
        data.unsubscribe('policy', 'classification:p')
        # trigger removed
        helper.retry_check_no_subscribers(policy,
                                          [('data', 'classification:p')])
        self.assertEqual(policy.policySubData.keys(), [])
Example #7
0
 def test_policy_subscriptions(self):
     """Test that policy engine subscriptions adjust to policy changes."""
     engine = self.engine
     api = self.api
     cage = self.cage
     # Send formula
     formula = create_network_group('p')
     LOG.debug("Sending formula: %s", formula)
     api['rule'].publish('policy-update', [runtime.Event(formula)])
     # check we have the proper subscriptions
     self.assertTrue('neutron' in cage.services)
     neutron = cage.service_object('neutron')
     helper.retry_check_subscriptions(engine, [('neutron', 'networks')])
     helper.retry_check_subscribers(neutron, [(engine.name, 'networks')])
Example #8
0
    def test_policy_subscriptions(self):
        """Test that policy engine subscriptions adjust to policy changes."""
        engine = self.engine
        api = self.api
        cage = self.cage
        policy = engine.DEFAULT_THEORY

        # Send formula
        formula = test_neutron.create_network_group('p')
        LOG.debug("Sending formula: %s", formula)
        api['rule'].publish('policy-update',
                            [agnostic.Event(formula, target=policy)])
        # check we have the proper subscriptions
        self.assertTrue('neutron' in cage.services)
        neutron = cage.service_object('neutron')
        helper.retry_check_subscriptions(engine, [('neutron', 'networks')])
        helper.retry_check_subscribers(neutron, [(engine.name, 'networks')])
    def test_policy_recovery(self):
        """Test policy crashing and recovering (sort of)."""
        cage = self.info["cage"]
        policy = cage.service_object("policy")
        neutron = cage.service_object("neutron")
        datalog1 = self.info["datalog1"]

        # get initial data
        policy.subscribe("neutron", "networks", callback=policy.receive_data)
        helper.retry_check_subscribers(neutron, [(policy.name, "networks")])
        neutron.poll()
        helper.retry_check_db_equal(policy, "p(x)", datalog1)

        # clear out policy's neutron:networks data (to simulate crashing)
        policy.initialize_tables(["neutron:networks"], [])
        # subscribe again (without unsubscribing)
        policy.subscribe("neutron", "networks", callback=policy.receive_data)
        helper.retry_check_db_equal(policy, "p(x)", datalog1)
Example #10
0
    def test_policy_recovery(self):
        """Test policy crashing and recovering (sort of)."""
        cage = self.info['cage']
        policy = cage.service_object('policy')
        neutron = cage.service_object('neutron')
        datalog1 = self.info['datalog1']

        # get initial data
        policy.subscribe('neutron', 'networks', callback=policy.receive_data)
        helper.retry_check_subscribers(neutron, [(policy.name, 'networks')])
        neutron.poll()
        helper.retry_check_db_equal(policy, 'p(x)', datalog1)

        # clear out policy's neutron:networks data (to simulate crashing)
        policy.initialize_tables(['neutron:networks'], [])
        # subscribe again (without unsubscribing)
        policy.subscribe('neutron', 'networks', callback=policy.receive_data)
        helper.retry_check_db_equal(policy, 'p(x)', datalog1)
    def test_policy_recovery(self):
        """Test policy crashing and recovering (sort of)."""
        cage = self.info['cage']
        policy = cage.service_object('policy')
        neutron = cage.service_object('neutron')
        datalog1 = self.info['datalog1']

        # get initial data
        policy.subscribe('neutron', 'networks', callback=policy.receive_data)
        helper.retry_check_subscribers(neutron, [(policy.name, 'networks')])
        neutron.poll()
        helper.retry_check_db_equal(policy, 'p(x)', datalog1)

        # clear out policy's neutron:networks data (to simulate crashing)
        policy.initialize_tables(['neutron:networks'], [])
        # subscribe again (without unsubscribing)
        policy.subscribe('neutron', 'networks', callback=policy.receive_data)
        helper.retry_check_db_equal(policy, 'p(x)', datalog1)
Example #12
0
    def test_subscribe_poll(self):
        """Test subscribing before polling.  The common case."""
        cage = self.info['cage']
        policy = cage.service_object('policy')
        neutron = cage.service_object('neutron')
        datalog1 = self.info['datalog1']
        datalog2 = self.info['datalog2']

        # subscribe
        policy.subscribe('neutron', 'networks', callback=policy.receive_data)
        helper.retry_check_subscribers(neutron, [(policy.name, 'networks')])

        # poll 1
        neutron.poll()
        helper.retry_check_db_equal(policy, 'p(x)', datalog1)

        # poll 2
        neutron.poll()
        helper.retry_check_db_equal(policy, 'p(x)', datalog2)
    def test_subscribe_poll(self):
        """Test subscribing before polling.  The common case."""
        cage = self.info['cage']
        policy = cage.service_object('policy')
        neutron = cage.service_object('neutron')
        datalog1 = self.info['datalog1']
        datalog2 = self.info['datalog2']

        # subscribe
        policy.subscribe('neutron', 'networks', callback=policy.receive_data)
        helper.retry_check_subscribers(neutron, [(policy.name, 'networks')])

        # poll 1
        neutron.poll()
        helper.retry_check_db_equal(policy, 'p(x)', datalog1)

        # poll 2
        neutron.poll()
        helper.retry_check_db_equal(policy, 'p(x)', datalog2)
    def test_subscribe_poll(self):
        """Test subscribing before polling.  The common case."""
        cage = self.info["cage"]
        policy = cage.service_object("policy")
        neutron = cage.service_object("neutron")
        datalog1 = self.info["datalog1"]
        datalog2 = self.info["datalog2"]

        # subscribe
        policy.subscribe("neutron", "networks", callback=policy.receive_data)
        helper.retry_check_subscribers(neutron, [(policy.name, "networks")])

        # poll 1
        neutron.poll()
        helper.retry_check_db_equal(policy, "p(x)", datalog1)

        # poll 2
        neutron.poll()
        helper.retry_check_db_equal(policy, "p(x)", datalog2)
    def test_policy_initialization(self):
        """Test subscribing before polling.  The common case."""
        cage = self.info["cage"]
        policy = cage.service_object("policy")
        neutron = cage.service_object("neutron")
        datalog1 = self.info["datalog1"]
        fake_networks = self.info["fake_networks"]

        # add garbage to policy
        for formula in fake_networks:
            policy.insert(formula)

        # subscribe
        policy.subscribe("neutron", "networks", callback=policy.receive_data)
        helper.retry_check_subscribers(neutron, [(policy.name, "networks")])

        # poll 1
        neutron.poll()
        helper.retry_check_db_equal(policy, "p(x)", datalog1)
    def test_policy_initialization(self):
        """Test subscribing before polling.  The common case."""
        cage = self.info['cage']
        policy = cage.service_object('policy')
        neutron = cage.service_object('neutron')
        datalog1 = self.info['datalog1']
        fake_networks = self.info['fake_networks']

        # add garbage to policy
        for formula in fake_networks:
            policy.insert(formula)

        # subscribe
        policy.subscribe('neutron', 'networks', callback=policy.receive_data)
        helper.retry_check_subscribers(neutron, [(policy.name, 'networks')])

        # poll 1
        neutron.poll()
        helper.retry_check_db_equal(policy, 'p(x)', datalog1)
Example #17
0
    def test_policy_initialization(self):
        """Test subscribing before polling.  The common case."""
        cage = self.info['cage']
        policy = cage.service_object('policy')
        neutron = cage.service_object('neutron')
        datalog1 = self.info['datalog1']
        fake_networks = self.info['fake_networks']

        # add garbage to policy
        for formula in fake_networks:
            policy.insert(formula)

        # subscribe
        policy.subscribe('neutron', 'networks', callback=policy.receive_data)
        helper.retry_check_subscribers(neutron, [(policy.name, 'networks')])

        # poll 1
        neutron.poll()
        helper.retry_check_db_equal(policy, 'p(x)', datalog1)
Example #18
0
    def setUp(self):
        super(BenchmarkDatasource, self).setUp()
        config = {
            'benchmark': {
                'module': helper.data_module_path('benchmark_driver.py'),
                'poll_time': 0
            }
        }
        cage = harness.create(helper.root_path(), helper.state_path(), None,
                              config)
        engine = cage.service_object('engine')
        api = {
            'policy': cage.service_object('api-policy'),
            'rule': cage.service_object('api-rule'),
            'table': cage.service_object('api-table'),
            'row': cage.service_object('api-row'),
            'datasource': cage.service_object('api-datasource'),
            'status': cage.service_object('api-status'),
            'schema': cage.service_object('api-schema')
        }
        helper.retry_check_subscriptions(engine,
                                         [(api['rule'].name, 'policy-update')])
        helper.retry_check_subscribers(api['rule'],
                                       [(engine.name, 'policy-update')])
        self.assertTrue('benchmark' in cage.services)
        datasource = cage.service_object('benchmark')
        table_name = datasource.BENCHTABLE

        self.assertEqual(datasource.state, {})

        # add a subscriber to ensure the updates end up in datasource.dataPath
        pubdata = datasource.pubdata.setdefault(table_name,
                                                dataobj.pubData(table_name))
        pubdata.addsubscriber(self.__class__.__name__, "push", "")
        self.assertTrue(datasource.pubdata[table_name])

        self.cage = cage
        self.engine = engine
        self.api = api
        self.table_name = table_name
        self.datasource = datasource
    def benchmark_datasource_to_policy_update(self, size):
        """Benchmark small datsource update to policy propagation.

        Time the propagation of a datasource update from datasource.poll() to
        completion of a simple policy update.
        """
        LOG.info("%s:: benchmarking datasource update of %d rows", size)
        self.datasource.datarows = size
        table_name = self.table_name

        # dummy policy only intended to produce a subscriber for the table
        key_to_index = self.datasource.get_column_map(table_name)
        id_index = 'x%d' % list(key_to_index.items())[0][1]
        max_index = max(key_to_index.values())
        args = ['x%d' % i for i in range(max_index + 1)]
        formula = compile.parse1('p(%s) :- benchmark:%s(%s)' % (id_index,
                                 table_name, ','.join(args)))

        # publish the formula and verify we see a subscription
        LOG.debug('%s:: sending formula: %s', self.__class__.__name__, formula)
        self.api['rule'].publish('policy-update', [agnostic.Event(formula)])
        helper.retry_check_subscriptions(
            self.engine, [('benchmark', table_name)])
        helper.retry_check_subscribers(
            self.datasource, [(self.engine.name, table_name)])

        # intercept inbox.task_done() so we know when it's finished. Sadly,
        # eventlet doesn't have a condition-like object.
        fake_condition = eventlet.Queue()
        fake_notify = functools.partial(fake_condition.put_nowait, True)
        self.mox.StubOutWithMock(self.engine.inbox, "task_done")
        self.engine.inbox.task_done().WithSideEffects(fake_notify)
        self.mox.ReplayAll()

        LOG.info("%s:: polling datasource", self.__class__.__name__)
        self.datasource.poll()
        fake_condition.get(timeout=30)
        self.mox.VerifyAll()
Example #20
0
    def benchmark_datasource_to_policy_update(self, size):
        """Benchmark small datsource update to policy propagation.

        Time the propagation of a datasource update from datasource.poll() to
        completion of a simple policy update.
        """
        LOG.info("%s:: benchmarking datasource update of %d rows", size)
        self.datasource.datarows = size
        table_name = self.table_name

        # dummy policy only intended to produce a subscriber for the table
        key_to_index = self.datasource.get_column_map(table_name)
        id_index = 'x%d' % key_to_index.items()[0][1]
        max_index = max(key_to_index.values())
        args = ['x%d' % i for i in xrange(max_index + 1)]
        formula = compile.parse1('p(%s) :- benchmark:%s(%s)' %
                                 (id_index, table_name, ','.join(args)))

        # publish the formula and verify we see a subscription
        LOG.debug('%s:: sending formula: %s', self.__class__.__name__, formula)
        self.api['rule'].publish('policy-update', [agnostic.Event(formula)])
        helper.retry_check_subscriptions(self.engine,
                                         [('benchmark', table_name)])
        helper.retry_check_subscribers(self.datasource,
                                       [(self.engine.name, table_name)])

        # intercept inbox.task_done() so we know when it's finished. Sadly,
        # eventlet doesn't have a condition-like object.
        fake_condition = eventlet.Queue()
        fake_notify = functools.partial(fake_condition.put_nowait, True)
        self.mox.StubOutWithMock(self.engine.inbox, "task_done")
        self.engine.inbox.task_done().WithSideEffects(fake_notify)
        self.mox.ReplayAll()

        LOG.info("%s:: polling datasource", self.__class__.__name__)
        self.datasource.poll()
        fake_condition.get(timeout=30)
        self.mox.VerifyAll()
Example #21
0
 def test_set_policy_subscriptions(self):
     self.vmplace.set_policy('p(x) :- fake:q(x)')
     helper.retry_check_subscriptions(self.vmplace, [(self.fake.name, 'q')])
     helper.retry_check_subscribers(self.fake, [(self.vmplace.name, 'q')])
Example #22
0
 def test_set_policy_subscriptions(self):
     self.vmplace.set_policy('p(x) :- fake:q(x)')
     helper.retry_check_subscriptions(
         self.vmplace, [(self.fake.name, 'q')])
     helper.retry_check_subscribers(
         self.fake, [(self.vmplace.name, 'q')])