def test_context(self): # Context must not only rely on node_id to prohibit multiple instances # of a node_id on the DSE part = helper.get_new_partition() n1 = helper.make_dsenode_same_partition(part, 'node_id', self.messaging_config, []) n2 = helper.make_dsenode_same_partition(part, 'node_id', self.messaging_config, []) self.assertEqual(n1._message_context, n1._message_context, "Comparison of context from the same node is equal") self.assertNotEqual(n1._message_context, n2._message_context, "Comparison of context from the different nodes " "is not equal")
def test_broadcast_service_rpc(self): part = helper.get_new_partition() nodes = [] services = [] for i in range(3): nid = 'svc_rpc_node%s' % i node = helper.make_dsenode_same_partition(part, nid, self.messaging_config) service = _PingRpcService('tbsr_svc', nid) node.register_service(service) nodes.append(node) services.append(service) # Send from each node to all services for i, source in enumerate(nodes): scounts = [] for j, target in enumerate(nodes): ep = nodes[j]._services[-1].endpoints[0] scounts.append(ep.ping_receive_count) source.broadcast_service_rpc('tbsr_svc', 'ping_test', { 'arg1': 1, 'arg2': 'a' }) eventlet.sleep(0.5) # wait for async delivery for j, target in enumerate(nodes): ep = nodes[j]._services[-1].endpoints[0] ecount = ep.ping_receive_count self.assertEqual( ecount - scounts[j], 1, "Node %s received ping (%s was sending)" % (nodes[j].node_id, source.node_id)) self.assertEqual( ep.ping_received_from[-1]['node_id'], source.node_id, "Last ping received on %s was from %s" % (nodes[j].node_id, source.node_id))
def test_service_rpc(self): part = helper.get_new_partition() nodes = [] services = [] for i in range(3): nid = 'svc_rpc_node%s' % i node = helper.make_dsenode_same_partition( part, nid, self.messaging_config) service = _PingRpcService('srpc_node_svc%s' % i, nid) node.register_service(service) nodes.append(node) services.append(service) # Send from each node to each other node for i, source in enumerate(nodes): # intentionally including self in RPC target for j, service in enumerate(services): ep = nodes[j]._services[-1].endpoints[0] scount = ep.ping_receive_count args = {'arg1': 1, 'arg2': 'a'} ret = source.invoke_service_rpc(service.service_id, 'ping', **args) self.assertEqual(ret, args, "Ping echoed arguments") ecount = ep.ping_receive_count self.assertEqual(ecount - scount, 1, "Node %s received ping (%s was sending)" % (nodes[j].node_id, nodes[i].node_id)) self.assertEqual( ep.ping_received_from[-1]['node_id'], nodes[i].node_id, "Last ping received on %s was from %s" % ( nodes[j].node_id, nodes[i].node_id))
def test_broadcast_service_rpc(self): part = helper.get_new_partition() nodes = [] services = [] for i in range(3): nid = 'svc_rpc_node%s' % i node = helper.make_dsenode_same_partition( part, nid, self.messaging_config) service = _PingRpcService('tbsr_svc', nid) node.register_service(service) nodes.append(node) services.append(service) # Send from each node to all services for i, source in enumerate(nodes): scounts = [] for j, target in enumerate(nodes): ep = nodes[j]._services[-1].endpoints[0] scounts.append(ep.ping_receive_count) source.broadcast_service_rpc('tbsr_svc', 'ping', arg1=1, arg2='a') eventlet.sleep(0.5) # wait for async delivery for j, target in enumerate(nodes): ep = nodes[j]._services[-1].endpoints[0] ecount = ep.ping_receive_count self.assertEqual(ecount - scounts[j], 1, "Node %s received ping (%s was sending)" % (nodes[j].node_id, source.node_id)) self.assertEqual( ep.ping_received_from[-1]['node_id'], source.node_id, "Last ping received on %s was from %s" % ( nodes[j].node_id, source.node_id))
def test_node_rpc(self): """Validate calling RPCs on DseNode""" part = helper.get_new_partition() nodes = [] endpoints = [] for i in range(3): nid = 'rpcnode%s' % i endpoints.append(_PingRpcEndpoint(nid)) nodes.append( helper.make_dsenode_same_partition( part, nid, self.messaging_config, [endpoints[-1]])) # Send from each node to each other node for i, source in enumerate(nodes): # intentionally including self in RPC target for j, target in enumerate(nodes): scount = endpoints[j].ping_receive_count args = {'arg1': 1, 'arg2': 'a'} ret = source.invoke_node_rpc(target.node_id, 'ping', **args) self.assertEqual(ret, args, "Ping echoed arguments") ecount = endpoints[j].ping_receive_count self.assertEqual(ecount - scount, 1, "Node %s received ping (%s was sending)" % (nodes[j].node_id, nodes[i].node_id)) self.assertEqual( endpoints[j].ping_received_from[-1]['node_id'], nodes[i].node_id, "Last ping received on %s was from %s" % ( nodes[j].node_id, nodes[i].node_id))
def test_node_broadcast_rpc(self): """Validate calling RPCs on DseNode""" part = helper.get_new_partition() nodes = [] endpoints = [] for i in range(3): nid = 'rpcnode%s' % i endpoints.append(_PingRpcEndpoint(nid)) nodes.append( helper.make_dsenode_same_partition( part, nid, self.messaging_config, [endpoints[-1]])) # Send from each node to all other nodes for i, source in enumerate(nodes): scounts = [] for j, target in enumerate(nodes): scounts.append(endpoints[j].ping_receive_count) source.broadcast_node_rpc('ping', arg1=1, arg2='a') eventlet.sleep(0.5) # wait for async delivery for j, target in enumerate(nodes): ecount = endpoints[j].ping_receive_count self.assertEqual(ecount - scounts[j], 1, "Node %s received ping (%s was sending)" % (nodes[j].node_id, source.node_id)) self.assertEqual( endpoints[j].ping_received_from[-1]['node_id'], source.node_id, "Last ping received on %s was from %s" % ( nodes[j].node_id, source.node_id))
def test_service_rpc(self): part = helper.get_new_partition() nodes = [] services = [] for i in range(3): nid = 'svc_rpc_node%s' % i node = helper.make_dsenode_same_partition(part, nid, self.messaging_config) service = _PingRpcService('srpc_node_svc%s' % i, nid) node.register_service(service) nodes.append(node) services.append(service) # Send from each node to each other node for i, source in enumerate(nodes): # intentionally including self in RPC target for j, service in enumerate(services): ep = nodes[j]._services[-1].endpoints[0] scount = ep.ping_receive_count args = {'arg1': 1, 'arg2': 'a'} ret = source.invoke_service_rpc(service.service_id, 'ping_test', args) self.assertEqual(ret, args, "Ping echoed arguments") ecount = ep.ping_receive_count self.assertEqual( ecount - scount, 1, "Node %s received ping (%s was sending)" % (nodes[j].node_id, nodes[i].node_id)) self.assertEqual( ep.ping_received_from[-1]['node_id'], nodes[i].node_id, "Last ping received on %s was from %s" % (nodes[j].node_id, nodes[i].node_id))
def test_node_broadcast_rpc(self): """Validate calling RPCs on DseNode""" part = helper.get_new_partition() nodes = [] endpoints = [] for i in range(3): nid = 'rpcnode%s' % i endpoints.append(_PingRpcEndpoint(nid)) nodes.append( helper.make_dsenode_same_partition(part, nid, self.messaging_config, [endpoints[-1]])) # Send from each node to all other nodes for i, source in enumerate(nodes): scounts = [] for j, target in enumerate(nodes): scounts.append(endpoints[j].ping_receive_count) source.broadcast_node_rpc('ping_test', {'arg1': 1, 'arg2': 'a'}) eventlet.sleep(0.5) # wait for async delivery for j, target in enumerate(nodes): ecount = endpoints[j].ping_receive_count self.assertEqual( ecount - scounts[j], 1, "Node %s received ping (%s was sending)" % (nodes[j].node_id, source.node_id)) self.assertEqual( endpoints[j].ping_received_from[-1]['node_id'], source.node_id, "Last ping received on %s was from %s" % (nodes[j].node_id, source.node_id))
def test_node_rpc(self): """Validate calling RPCs on DseNode""" part = helper.get_new_partition() nodes = [] endpoints = [] for i in range(3): nid = 'rpcnode%s' % i endpoints.append(_PingRpcEndpoint(nid)) nodes.append( helper.make_dsenode_same_partition(part, nid, self.messaging_config, [endpoints[-1]])) # Send from each node to each other node for i, source in enumerate(nodes): # intentionally including self in RPC target for j, target in enumerate(nodes): scount = endpoints[j].ping_receive_count args = {'arg1': 1, 'arg2': 'a'} ret = source.invoke_node_rpc(target.node_id, 'ping_test', args) self.assertEqual(ret, args, "Ping echoed arguments") ecount = endpoints[j].ping_receive_count self.assertEqual( ecount - scount, 1, "Node %s received ping (%s was sending)" % (nodes[j].node_id, nodes[i].node_id)) self.assertEqual( endpoints[j].ping_received_from[-1]['node_id'], nodes[i].node_id, "Last ping received on %s was from %s" % (nodes[j].node_id, nodes[i].node_id))
def setup_config(with_fake_datasource=True, node_id='testnode', same_partition_as_node=None, api=True, policy=True, datasources=True): """Setup DseNode for testing. :param: services is an array of DataServices :param: api is a dictionary mapping api name to API model instance """ config.set_config_defaults() # Load the fake driver. cfg.CONF.set_override('drivers', ['congress.tests.fake_datasource.FakeDataSource']) if same_partition_as_node is None: node = helper.make_dsenode_new_partition(node_id) else: node = helper.make_dsenode_same_partition(same_partition_as_node, node_id) if datasources: cfg.CONF.set_override('datasources', True) with mock.patch.object(periodics, 'PeriodicWorker', autospec=True): services = harness.create2(existing_node=node, policy_engine=policy, api=api, datasources=datasources) data = None if with_fake_datasource: data = fake_datasource.FakeDataSource('data') # FIXME(ekcs): this is a hack to prevent the synchronizer from # attempting to delete this DSD because it's not in DB data.type = 'no_sync_datasource_driver' node.register_service(data) engine_service = None library_service = None api_service = None if policy: engine_service = services[api_base.ENGINE_SERVICE_ID] library_service = services[api_base.LIBRARY_SERVICE_ID] if api: api_service = services['api'] if datasources: ds_manager = services['ds_manager'] return { 'node': node, 'engine': engine_service, 'library': library_service, 'data': data, 'api': api_service, 'ds_manager': ds_manager }
def test_replicated_pe_exec(self): """Test correct local leader behavior with 2 PEs requesting exec""" node1 = helper.make_dsenode_new_partition('testnode1') node2 = helper.make_dsenode_same_partition(node1, 'testnode2') dsd = fake_datasource.FakeDataSource('dsd') # faster time-out for testing dsd.LEADER_TIMEOUT = 2 pe1 = agnostic.DseRuntime('pe1') pe2 = agnostic.DseRuntime('pe2') node1.register_service(pe1) node2.register_service(pe2) node1.register_service(dsd) assert dsd._running assert node1._running assert node2._running assert node1._control_bus._running # first exec request obeyed and leader set pe2.rpc('dsd', 'request_execute', {'action': 'fake_act', 'action_args': {'name': 'testnode2'}, 'wait': True}) helper.retry_check_function_return_value( lambda: len(dsd.exec_history), 1) self.assertEqual(dsd._leader_node_id, 'testnode2') # second exec request from leader obeyed and leader remains pe2.rpc('dsd', 'request_execute', {'action': 'fake_act', 'action_args': {'name': 'testnode2'}, 'wait': True}) helper.retry_check_function_return_value( lambda: len(dsd.exec_history), 2) self.assertEqual(dsd._leader_node_id, 'testnode2') # exec request from non-leader not obeyed pe1.rpc('dsd', 'request_execute', {'action': 'fake_act', 'action_args': {'name': 'testnode1'}, 'wait': True}) self.assertRaises( tenacity.RetryError, helper.retry_check_function_return_value, lambda: len(dsd.exec_history), 3) # leader vacated after heartbeat stops node2.stop() node2.wait() helper.retry_check_function_return_value( lambda: dsd._leader_node_id, None) # next exec request obeyed and new leader set pe1.rpc('dsd', 'request_execute', {'action': 'fake_act', 'action_args': {'name': 'testnode1'}, 'wait': True}) helper.retry_check_function_return_value( lambda: len(dsd.exec_history), 3) self.assertEqual(dsd._leader_node_id, 'testnode1') node1.stop() node2.stop()
def _create_node_with_services(self, nodes, services, num, partition_id): nid = 'cbd_node%s' % num nodes.append(helper.make_dsenode_same_partition(partition_id, nid)) ns = [] for s in range(num): # intentionally starting different number services ns.append(FakeDataSource('cbd-%d_svc-%d' % (num, s))) nodes[-1].register_service(ns[-1]) services.append(ns) return nodes[-1]
def setup_config(with_fake_datasource=True, node_id='testnode', same_partition_as_node=None, api=True, policy=True, datasources=True, with_fake_json_ingester=False): """Setup DseNode for testing. :param: services is an array of DataServices :param: api is a dictionary mapping api name to API model instance """ config.set_config_defaults() # Load the fake driver. cfg.CONF.set_override( 'drivers', ['congress.tests.fake_datasource.FakeDataSource']) if same_partition_as_node is None: node = helper.make_dsenode_new_partition(node_id) else: node = helper.make_dsenode_same_partition( same_partition_as_node, node_id) if datasources: cfg.CONF.set_override('datasources', True) with mock.patch.object(periodics, 'PeriodicWorker', autospec=True): services = harness.create2( existing_node=node, policy_engine=policy, api=api, datasources=datasources) data = None if with_fake_datasource: data = fake_datasource.FakeDataSource('data') # FIXME(ekcs): this is a hack to prevent the synchronizer from # attempting to delete this DSD because it's not in DB data.type = 'no_sync_datasource_driver' node.register_service(data) ingester = None if with_fake_json_ingester: ingester = fake_datasource.FakeJsonIngester() node.register_service(ingester) engine_service = None library_service = None api_service = None if policy: engine_service = services[api_base.ENGINE_SERVICE_ID] library_service = services[api_base.LIBRARY_SERVICE_ID] if api: api_service = services['api'] if datasources: ds_manager = services['ds_manager'] return {'node': node, 'engine': engine_service, 'library': library_service, 'data': data, 'api': api_service, 'ds_manager': ds_manager, 'json_ingester': ingester}
def _create_node_with_services(self, nodes, services, num, partition_id): nid = 'cbd_node%s' % num nodes.append(helper.make_dsenode_same_partition(partition_id, nid)) ns = [] for s in range(num): # intentionally starting different number services ns.append( fake_datasource.FakeDataSource('cbd-%d_svc-%d' % (num, s))) nodes[-1].register_service(ns[-1]) services.append(ns) return nodes[-1]
def test_internode_pubsub(self): node1 = helper.make_dsenode_new_partition('testnode1') test1 = fake_datasource.FakeDataSource('test1') node1.register_service(test1) node2 = helper.make_dsenode_same_partition(node1, 'testnode2') test2 = fake_datasource.FakeDataSource('test2') node2.register_service(test2) test1.subscribe('test2', 'p') helper.retry_check_function_return_value( lambda: hasattr(test1, 'last_msg'), True) test2.publish('p', 42) helper.retry_check_function_return_value( lambda: test1.last_msg['data'], 42) self.assertFalse(hasattr(test2, "last_msg"))
def test_internode_pubsub(self): node1 = helper.make_dsenode_new_partition('testnode1') test1 = fake_datasource.FakeDataSource('test1') node1.register_service(test1) node2 = helper.make_dsenode_same_partition(node1, 'testnode2') test2 = fake_datasource.FakeDataSource('test2') node2.register_service(test2) test1.subscribe('test2', 'p') helper.retry_check_function_return_value( lambda: hasattr(test1, 'last_msg'), True) test2.publish('p', 42, use_snapshot=True) helper.retry_check_function_return_value( lambda: test1.last_msg['data'], 42) self.assertFalse(hasattr(test2, "last_msg")) node1.stop() node2.stop()
def test_internode_partial_unsub(self): node1 = helper.make_dsenode_new_partition('testnode1') node2 = helper.make_dsenode_same_partition(node1, 'testnode2') test1 = fake_datasource.FakeDataSource('test1') test2 = fake_datasource.FakeDataSource('test2') node1.register_service(test1) node2.register_service(test2) test1.subscribe('test2', 'p') test1.subscribe('test2', 'q') test1.unsubscribe('test2', 'q') # unsub from q should not affect p helper.retry_check_function_return_value( lambda: hasattr(test1, 'last_msg'), True) test2.publish('p', [42], use_snapshot=True) helper.retry_check_function_return_value( lambda: test1.last_msg['data'], [42]) self.assertFalse(hasattr(test2, "last_msg")) node1.stop() node2.stop()
def test_replicated_pe_exec(self): """Test correct local leader behavior with 2 PEs requesting exec""" node1 = helper.make_dsenode_new_partition('testnode1') node2 = helper.make_dsenode_same_partition(node1, 'testnode2') dsd = fake_datasource.FakeDataSource('dsd') # faster time-out for testing dsd.LEADER_TIMEOUT = 2 pe1 = agnostic.DseRuntime('pe1') pe2 = agnostic.DseRuntime('pe2') node1.register_service(pe1) node2.register_service(pe2) node1.register_service(dsd) assert dsd._running assert node1._running assert node2._running assert node1._control_bus._running # first exec request obeyed and leader set pe2.rpc( 'dsd', 'request_execute', { 'action': 'fake_act', 'action_args': { 'name': 'testnode2' }, 'wait': True }) helper.retry_check_function_return_value(lambda: len(dsd.exec_history), 1) self.assertEqual(dsd._leader_node_id, 'testnode2') # second exec request from leader obeyed and leader remains pe2.rpc( 'dsd', 'request_execute', { 'action': 'fake_act', 'action_args': { 'name': 'testnode2' }, 'wait': True }) helper.retry_check_function_return_value(lambda: len(dsd.exec_history), 2) self.assertEqual(dsd._leader_node_id, 'testnode2') # exec request from non-leader not obeyed pe1.rpc( 'dsd', 'request_execute', { 'action': 'fake_act', 'action_args': { 'name': 'testnode1' }, 'wait': True }) self.assertRaises(tenacity.RetryError, helper.retry_check_function_return_value, lambda: len(dsd.exec_history), 3) # leader vacated after heartbeat stops node2.stop() node2.wait() helper.retry_check_function_return_value(lambda: dsd._leader_node_id, None) # next exec request obeyed and new leader set pe1.rpc( 'dsd', 'request_execute', { 'action': 'fake_act', 'action_args': { 'name': 'testnode1' }, 'wait': True }) helper.retry_check_function_return_value(lambda: len(dsd.exec_history), 3) self.assertEqual(dsd._leader_node_id, 'testnode1') node1.stop() node2.stop()