class ProvisionerServiceTest(BaseProvisionerServiceTests): """Integration tests that use fake context broker and IaaS driver fixtures """ @defer.inlineCallbacks def setUp(self): self.notifier = FakeProvisionerNotifier() self.context_client = FakeContextClient() self.store = yield self.setup_store() self.site_drivers = {'fake-site1': FakeNodeDriver()} yield self._start_container() yield self.spawn_procs() pId = yield self.procRegistry.get("provisioner") self.client = ProvisionerClient(pid=pId) @defer.inlineCallbacks def tearDown(self): yield self.shutdown_procs() yield self.teardown_store() yield self._stop_container() def setup_store(self): return defer.succeed(ProvisionerStore()) def teardown_store(self): return defer.succeed(None) @defer.inlineCallbacks def test_provision_bad_dt(self): client = self.client notifier = self.notifier worker_node_count = 3 deployable_type = 'this-doesnt-exist' nodes = { 'head-node': FakeLaunchItem(1, 'fake-site1', 'small', None), 'worker-node': FakeLaunchItem(worker_node_count, 'fake-site1', 'small', None) } launch_id = _new_id() node_ids = [ node_id for node in nodes.itervalues() for node_id in node.instance_ids ] self.assertEqual(len(node_ids), worker_node_count + 1) yield client.provision(launch_id, deployable_type, nodes, ('subscriber', )) ok = yield notifier.wait_for_state(states.FAILED, node_ids) self.assertTrue(ok) self.assertTrue(notifier.assure_record_count(1)) yield self.assertStoreNodeRecords(states.FAILED, *node_ids) yield self.assertStoreLaunchRecord(states.FAILED, launch_id) @defer.inlineCallbacks def test_provision_broker_error(self): client = self.client notifier = self.notifier worker_node_count = 3 deployable_type = 'base-cluster' nodes = { 'head-node': FakeLaunchItem(1, 'fake-site1', 'small', None), 'worker-node': FakeLaunchItem(worker_node_count, 'fake-site1', 'small', None) } launch_id = _new_id() node_ids = [ node_id for node in nodes.itervalues() for node_id in node.instance_ids ] self.assertEqual(len(node_ids), worker_node_count + 1) self.context_client.create_error = BrokerError("fake failure") yield client.provision(launch_id, deployable_type, nodes, ('subscriber', )) ok = yield notifier.wait_for_state(states.FAILED, node_ids) self.assertTrue(ok) self.assertTrue(notifier.assure_record_count(1)) yield self.assertStoreNodeRecords(states.FAILED, *node_ids) yield self.assertStoreLaunchRecord(states.FAILED, launch_id) @defer.inlineCallbacks def test_dump_state(self): running_launch, running_nodes = make_launch_and_nodes( _new_id(), 10, states.RUNNING) yield self.store.put_launch(running_launch) yield self.store.put_nodes(running_nodes) pending_launch, pending_nodes = make_launch_and_nodes( _new_id(), 3, states.PENDING) yield self.store.put_launch(pending_launch) yield self.store.put_nodes(pending_nodes) running_node_ids = [node['node_id'] for node in running_nodes] pending_node_ids = [node['node_id'] for node in pending_nodes] all_node_ids = running_node_ids + pending_node_ids yield self.client.dump_state(running_node_ids) ok = yield self.notifier.wait_for_state(states.RUNNING, nodes=running_node_ids) self.assertTrue(ok) self.assertEqual(len(self.notifier.nodes), len(running_nodes)) yield self.client.dump_state(pending_node_ids) ok = yield self.notifier.wait_for_state(states.PENDING, nodes=pending_node_ids) self.assertTrue(ok) self.assertEqual(len(self.notifier.nodes), len(all_node_ids)) # we should have not gotten any dupe records yet self.assertTrue(self.notifier.assure_record_count(1)) # empty dump request should dump nothing yield self.client.dump_state([]) self.assertTrue(self.notifier.assure_record_count(1)) @defer.inlineCallbacks def test_dump_state_unknown_node(self): node_ids = ["09ddd3f8-a5a5-4196-ac13-eab4d4b0c777"] subscribers = ["hello1_subscriber"] yield self.client.dump_state(node_ids, force_subscribe=subscribers[0]) ok = yield self.notifier.wait_for_state(states.FAILED, nodes=node_ids) self.assertTrue(ok) self.assertEqual(len(self.notifier.nodes), len(node_ids)) for node_id in node_ids: ok = yield self.notifier.assure_subscribers(node_id, subscribers) self.assertTrue(ok) @defer.inlineCallbacks def test_terminate(self): launch_id = _new_id() running_launch, running_nodes = make_launch_and_nodes( launch_id, 10, states.RUNNING, site="fake-site1") yield self.store.put_launch(running_launch) yield self.store.put_nodes(running_nodes) node_ids = [node['node_id'] for node in running_nodes] # terminate half of the nodes then the launch as a whole first_five = node_ids[:5] yield self.client.terminate_nodes(first_five) ok = yield self.notifier.wait_for_state(states.TERMINATED, nodes=first_five) self.assertTrue(ok) self.assertEqual(set(first_five), set(self.notifier.nodes)) yield self.client.terminate_launches((launch_id, )) ok = yield self.notifier.wait_for_state(states.TERMINATED, nodes=node_ids) self.assertTrue(ok) self.assertEqual(set(node_ids), set(self.notifier.nodes)) # should be TERMINATING and TERMINATED record for each node self.assertTrue(self.notifier.assure_record_count(2)) self.assertEqual(len(self.site_drivers['fake-site1'].destroyed), len(node_ids)) @defer.inlineCallbacks def test_terminate_all(self): # create a ton of launches launch_specs = [(30, 3, states.RUNNING), (50, 1, states.TERMINATED), (80, 1, states.RUNNING)] to_be_terminated_node_ids = [] for launchcount, nodecount, state in launch_specs: for i in range(launchcount): launch_id = _new_id() launch, nodes = make_launch_and_nodes(launch_id, nodecount, state, site="fake-site1") yield self.store.put_launch(launch) yield self.store.put_nodes(nodes) if state < states.TERMINATED: to_be_terminated_node_ids.extend(node["node_id"] for node in nodes) log.debug("Expecting %d nodes to be terminated", len(to_be_terminated_node_ids)) yield self.client.terminate_all(rpcwait=True) yield self.assertStoreNodeRecords(states.TERMINATED, *to_be_terminated_node_ids) ok = self.notifier.assure_state(states.TERMINATED, nodes=to_be_terminated_node_ids) self.assertTrue(ok) self.assertEqual(set(to_be_terminated_node_ids), set(self.notifier.nodes)) self.assertEqual(len(self.site_drivers['fake-site1'].destroyed), len(to_be_terminated_node_ids)) @defer.inlineCallbacks def test_query(self): #default is non-rpc. should be None result res = yield self.client.query() self.assertEqual(res, None) #returns true in RPC case res = yield self.client.query(rpc=True) self.assertEqual(res, True)
class ProvisionerServiceTest(BaseProvisionerServiceTests): """Integration tests that use fake context broker and IaaS driver fixtures """ @defer.inlineCallbacks def setUp(self): self.notifier = FakeProvisionerNotifier() self.context_client = FakeContextClient() self.store = yield self.setup_store() self.site_drivers = {'fake-site1' : FakeNodeDriver()} yield self._start_container() yield self.spawn_procs() pId = yield self.procRegistry.get("provisioner") self.client = ProvisionerClient(pid=pId) @defer.inlineCallbacks def tearDown(self): yield self.shutdown_procs() yield self.teardown_store() yield self._stop_container() def setup_store(self): return defer.succeed(ProvisionerStore()) def teardown_store(self): return defer.succeed(None) @defer.inlineCallbacks def test_provision_bad_dt(self): client = self.client notifier = self.notifier worker_node_count = 3 deployable_type = 'this-doesnt-exist' nodes = {'head-node' : FakeLaunchItem(1, 'fake-site1', 'small', None), 'worker-node' : FakeLaunchItem(worker_node_count, 'fake-site1', 'small', None)} launch_id = _new_id() node_ids = [node_id for node in nodes.itervalues() for node_id in node.instance_ids] self.assertEqual(len(node_ids), worker_node_count + 1) yield client.provision(launch_id, deployable_type, nodes, ('subscriber',)) ok = yield notifier.wait_for_state(states.FAILED, node_ids) self.assertTrue(ok) self.assertTrue(notifier.assure_record_count(1)) yield self.assertStoreNodeRecords(states.FAILED, *node_ids) yield self.assertStoreLaunchRecord(states.FAILED, launch_id) @defer.inlineCallbacks def test_provision_broker_error(self): client = self.client notifier = self.notifier worker_node_count = 3 deployable_type = 'base-cluster' nodes = {'head-node' : FakeLaunchItem(1, 'fake-site1', 'small', None), 'worker-node' : FakeLaunchItem(worker_node_count, 'fake-site1', 'small', None)} launch_id = _new_id() node_ids = [node_id for node in nodes.itervalues() for node_id in node.instance_ids] self.assertEqual(len(node_ids), worker_node_count + 1) self.context_client.create_error = BrokerError("fake failure") yield client.provision(launch_id, deployable_type, nodes, ('subscriber',)) ok = yield notifier.wait_for_state(states.FAILED, node_ids) self.assertTrue(ok) self.assertTrue(notifier.assure_record_count(1)) yield self.assertStoreNodeRecords(states.FAILED, *node_ids) yield self.assertStoreLaunchRecord(states.FAILED, launch_id) @defer.inlineCallbacks def test_dump_state(self): running_launch, running_nodes = make_launch_and_nodes(_new_id(), 10, states.RUNNING) yield self.store.put_launch(running_launch) yield self.store.put_nodes(running_nodes) pending_launch, pending_nodes = make_launch_and_nodes(_new_id(), 3, states.PENDING) yield self.store.put_launch(pending_launch) yield self.store.put_nodes(pending_nodes) running_node_ids = [node['node_id'] for node in running_nodes] pending_node_ids = [node['node_id'] for node in pending_nodes] all_node_ids = running_node_ids + pending_node_ids yield self.client.dump_state(running_node_ids) ok = yield self.notifier.wait_for_state(states.RUNNING, nodes=running_node_ids) self.assertTrue(ok) self.assertEqual(len(self.notifier.nodes), len(running_nodes)) yield self.client.dump_state(pending_node_ids) ok = yield self.notifier.wait_for_state(states.PENDING, nodes=pending_node_ids) self.assertTrue(ok) self.assertEqual(len(self.notifier.nodes), len(all_node_ids)) # we should have not gotten any dupe records yet self.assertTrue(self.notifier.assure_record_count(1)) # empty dump request should dump nothing yield self.client.dump_state([]) self.assertTrue(self.notifier.assure_record_count(1)) @defer.inlineCallbacks def test_dump_state_unknown_node(self): node_ids = ["09ddd3f8-a5a5-4196-ac13-eab4d4b0c777"] subscribers = ["hello1_subscriber"] yield self.client.dump_state(node_ids, force_subscribe=subscribers[0]) ok = yield self.notifier.wait_for_state(states.FAILED, nodes=node_ids) self.assertTrue(ok) self.assertEqual(len(self.notifier.nodes), len(node_ids)) for node_id in node_ids: ok = yield self.notifier.assure_subscribers(node_id, subscribers) self.assertTrue(ok) @defer.inlineCallbacks def test_terminate(self): launch_id = _new_id() running_launch, running_nodes = make_launch_and_nodes(launch_id, 10, states.RUNNING, site="fake-site1") yield self.store.put_launch(running_launch) yield self.store.put_nodes(running_nodes) node_ids = [node['node_id'] for node in running_nodes] # terminate half of the nodes then the launch as a whole first_five = node_ids[:5] yield self.client.terminate_nodes(first_five) ok = yield self.notifier.wait_for_state(states.TERMINATED, nodes=first_five) self.assertTrue(ok) self.assertEqual(set(first_five), set(self.notifier.nodes)) yield self.client.terminate_launches((launch_id,)) ok = yield self.notifier.wait_for_state(states.TERMINATED, nodes=node_ids) self.assertTrue(ok) self.assertEqual(set(node_ids), set(self.notifier.nodes)) # should be TERMINATING and TERMINATED record for each node self.assertTrue(self.notifier.assure_record_count(2)) self.assertEqual(len(self.site_drivers['fake-site1'].destroyed), len(node_ids)) @defer.inlineCallbacks def test_terminate_all(self): # create a ton of launches launch_specs = [(30, 3, states.RUNNING), (50, 1, states.TERMINATED), (80, 1, states.RUNNING)] to_be_terminated_node_ids = [] for launchcount, nodecount, state in launch_specs: for i in range(launchcount): launch_id = _new_id() launch, nodes = make_launch_and_nodes( launch_id, nodecount, state, site="fake-site1") yield self.store.put_launch(launch) yield self.store.put_nodes(nodes) if state < states.TERMINATED: to_be_terminated_node_ids.extend(node["node_id"] for node in nodes) log.debug("Expecting %d nodes to be terminated", len(to_be_terminated_node_ids)) yield self.client.terminate_all(rpcwait=True) yield self.assertStoreNodeRecords(states.TERMINATED, *to_be_terminated_node_ids) ok = self.notifier.assure_state(states.TERMINATED, nodes=to_be_terminated_node_ids) self.assertTrue(ok) self.assertEqual(set(to_be_terminated_node_ids), set(self.notifier.nodes)) self.assertEqual(len(self.site_drivers['fake-site1'].destroyed), len(to_be_terminated_node_ids)) @defer.inlineCallbacks def test_query(self): #default is non-rpc. should be None result res = yield self.client.query() self.assertEqual(res, None) #returns true in RPC case res = yield self.client.query(rpc=True) self.assertEqual(res, True)