class ProvisionerServiceNoContextualizationTest(BaseProvisionerServiceTests):
def setUp(self):
self.notifier = FakeProvisionerNotifier()
self.context_client = None
self.store = self.setup_store()
self.driver = FakeNodeDriver()
self.driver.initialize()
self.spawn_procs()
self.load_dtrs()
def test_launch_no_context(self):
all_node_ids = []
for _ in range(10):
node_id = _new_id()
all_node_ids.append(node_id)
self.client.provision(_new_id(), [node_id], "empty",
site="fake-site1", caller="asterix")
self.notifier.wait_for_state(InstanceState.PENDING, all_node_ids,
before=self.provisioner.leader._force_cycle)
self.assertStoreNodeRecords(InstanceState.PENDING, *all_node_ids)
for node_id in all_node_ids:
node = self.store.get_node(node_id)
self.driver.set_node_running(node['iaas_id'])
self.notifier.wait_for_state(InstanceState.RUNNING, all_node_ids,
before=self.provisioner.leader._force_cycle)
self.assertStoreNodeRecords(InstanceState.RUNNING, *all_node_ids)
class ProvisionerServiceTest(BaseProvisionerServiceTests):
"""Integration tests that use fake context broker and IaaS driver fixtures
"""
@defer.inlineCallbacks
def setUp(self):
self.notifier = FakeProvisionerNotifier()
self.context_client = FakeContextClient()
self.store = yield self.setup_store()
self.site_drivers = {'fake-site1': FakeNodeDriver()}
yield self._start_container()
yield self.spawn_procs()
pId = yield self.procRegistry.get("provisioner")
self.client = ProvisionerClient(pid=pId)
@defer.inlineCallbacks
def tearDown(self):
yield self.shutdown_procs()
yield self.teardown_store()
yield self._stop_container()
def setup_store(self):
return defer.succeed(ProvisionerStore())
def teardown_store(self):
return defer.succeed(None)
@defer.inlineCallbacks
def test_provision_bad_dt(self):
client = self.client
notifier = self.notifier
worker_node_count = 3
deployable_type = 'this-doesnt-exist'
nodes = {
'head-node':
FakeLaunchItem(1, 'fake-site1', 'small', None),
'worker-node':
FakeLaunchItem(worker_node_count, 'fake-site1', 'small', None)
}
launch_id = _new_id()
node_ids = [
node_id for node in nodes.itervalues()
for node_id in node.instance_ids
]
self.assertEqual(len(node_ids), worker_node_count + 1)
yield client.provision(launch_id, deployable_type, nodes,
('subscriber', ))
ok = yield notifier.wait_for_state(states.FAILED, node_ids)
self.assertTrue(ok)
self.assertTrue(notifier.assure_record_count(1))
yield self.assertStoreNodeRecords(states.FAILED, *node_ids)
yield self.assertStoreLaunchRecord(states.FAILED, launch_id)
@defer.inlineCallbacks
def test_provision_broker_error(self):
client = self.client
notifier = self.notifier
worker_node_count = 3
deployable_type = 'base-cluster'
nodes = {
'head-node':
FakeLaunchItem(1, 'fake-site1', 'small', None),
'worker-node':
FakeLaunchItem(worker_node_count, 'fake-site1', 'small', None)
}
launch_id = _new_id()
node_ids = [
node_id for node in nodes.itervalues()
for node_id in node.instance_ids
]
self.assertEqual(len(node_ids), worker_node_count + 1)
self.context_client.create_error = BrokerError("fake failure")
yield client.provision(launch_id, deployable_type, nodes,
('subscriber', ))
ok = yield notifier.wait_for_state(states.FAILED, node_ids)
self.assertTrue(ok)
self.assertTrue(notifier.assure_record_count(1))
yield self.assertStoreNodeRecords(states.FAILED, *node_ids)
yield self.assertStoreLaunchRecord(states.FAILED, launch_id)
@defer.inlineCallbacks
def test_dump_state(self):
running_launch, running_nodes = make_launch_and_nodes(
_new_id(), 10, states.RUNNING)
yield self.store.put_launch(running_launch)
yield self.store.put_nodes(running_nodes)
pending_launch, pending_nodes = make_launch_and_nodes(
_new_id(), 3, states.PENDING)
yield self.store.put_launch(pending_launch)
yield self.store.put_nodes(pending_nodes)
running_node_ids = [node['node_id'] for node in running_nodes]
pending_node_ids = [node['node_id'] for node in pending_nodes]
all_node_ids = running_node_ids + pending_node_ids
yield self.client.dump_state(running_node_ids)
ok = yield self.notifier.wait_for_state(states.RUNNING,
nodes=running_node_ids)
self.assertTrue(ok)
self.assertEqual(len(self.notifier.nodes), len(running_nodes))
yield self.client.dump_state(pending_node_ids)
ok = yield self.notifier.wait_for_state(states.PENDING,
nodes=pending_node_ids)
self.assertTrue(ok)
self.assertEqual(len(self.notifier.nodes), len(all_node_ids))
# we should have not gotten any dupe records yet
self.assertTrue(self.notifier.assure_record_count(1))
# empty dump request should dump nothing
yield self.client.dump_state([])
self.assertTrue(self.notifier.assure_record_count(1))
@defer.inlineCallbacks
def test_dump_state_unknown_node(self):
node_ids = ["09ddd3f8-a5a5-4196-ac13-eab4d4b0c777"]
subscribers = ["hello1_subscriber"]
yield self.client.dump_state(node_ids, force_subscribe=subscribers[0])
ok = yield self.notifier.wait_for_state(states.FAILED, nodes=node_ids)
self.assertTrue(ok)
self.assertEqual(len(self.notifier.nodes), len(node_ids))
for node_id in node_ids:
ok = yield self.notifier.assure_subscribers(node_id, subscribers)
self.assertTrue(ok)
@defer.inlineCallbacks
def test_terminate(self):
launch_id = _new_id()
running_launch, running_nodes = make_launch_and_nodes(
launch_id, 10, states.RUNNING, site="fake-site1")
yield self.store.put_launch(running_launch)
yield self.store.put_nodes(running_nodes)
node_ids = [node['node_id'] for node in running_nodes]
# terminate half of the nodes then the launch as a whole
first_five = node_ids[:5]
yield self.client.terminate_nodes(first_five)
ok = yield self.notifier.wait_for_state(states.TERMINATED,
nodes=first_five)
self.assertTrue(ok)
self.assertEqual(set(first_five), set(self.notifier.nodes))
yield self.client.terminate_launches((launch_id, ))
ok = yield self.notifier.wait_for_state(states.TERMINATED,
nodes=node_ids)
self.assertTrue(ok)
self.assertEqual(set(node_ids), set(self.notifier.nodes))
# should be TERMINATING and TERMINATED record for each node
self.assertTrue(self.notifier.assure_record_count(2))
self.assertEqual(len(self.site_drivers['fake-site1'].destroyed),
len(node_ids))
@defer.inlineCallbacks
def test_terminate_all(self):
# create a ton of launches
launch_specs = [(30, 3, states.RUNNING), (50, 1, states.TERMINATED),
(80, 1, states.RUNNING)]
to_be_terminated_node_ids = []
for launchcount, nodecount, state in launch_specs:
for i in range(launchcount):
launch_id = _new_id()
launch, nodes = make_launch_and_nodes(launch_id,
nodecount,
state,
site="fake-site1")
yield self.store.put_launch(launch)
yield self.store.put_nodes(nodes)
if state < states.TERMINATED:
to_be_terminated_node_ids.extend(node["node_id"]
for node in nodes)
log.debug("Expecting %d nodes to be terminated",
len(to_be_terminated_node_ids))
yield self.client.terminate_all(rpcwait=True)
yield self.assertStoreNodeRecords(states.TERMINATED,
*to_be_terminated_node_ids)
ok = self.notifier.assure_state(states.TERMINATED,
nodes=to_be_terminated_node_ids)
self.assertTrue(ok)
self.assertEqual(set(to_be_terminated_node_ids),
set(self.notifier.nodes))
self.assertEqual(len(self.site_drivers['fake-site1'].destroyed),
len(to_be_terminated_node_ids))
@defer.inlineCallbacks
def test_query(self):
#default is non-rpc. should be None result
res = yield self.client.query()
self.assertEqual(res, None)
#returns true in RPC case
res = yield self.client.query(rpc=True)
self.assertEqual(res, True)
class ProvisionerServiceTest(BaseProvisionerServiceTests):
"""Integration tests that use fake context broker and IaaS driver fixtures
"""
def setUp(self):
self.notifier = FakeProvisionerNotifier()
self.context_client = FakeContextClient()
self.store = self.setup_store()
self.driver = FakeNodeDriver()
self.driver.initialize()
self.spawn_procs()
self.load_dtrs()
def test_provision_bad_dt(self):
client = self.client
notifier = self.notifier
deployable_type = 'this-doesnt-exist'
launch_id = _new_id()
node_ids = [_new_id()]
client.provision(launch_id, node_ids, deployable_type,
'fake-site1', caller="asterix")
ok = notifier.wait_for_state(InstanceState.FAILED, node_ids)
self.assertTrue(ok)
self.assertTrue(notifier.assure_record_count(1))
self.assertStoreNodeRecords(InstanceState.FAILED, *node_ids)
self.assertStoreLaunchRecord(InstanceState.FAILED, launch_id)
def test_provision_with_vars(self):
client = self.client
caller = 'asterix'
deployable_type = 'empty-with-vars'
launch_id = _new_id()
node_ids = [_new_id()]
vars = {'image_id': 'fake-image'}
client.provision(launch_id, node_ids, deployable_type,
'fake-site1', vars=vars, caller=caller)
self.notifier.wait_for_state(InstanceState.PENDING, node_ids,
before=self.provisioner.leader._force_cycle)
self.assertStoreNodeRecords(InstanceState.PENDING, *node_ids)
def test_provision_with_missing_vars(self):
client = self.client
notifier = self.notifier
caller = 'asterix'
deployable_type = 'empty-with-vars'
launch_id = _new_id()
node_ids = [_new_id()]
vars = {'foo': 'bar'}
client.provision(launch_id, node_ids, deployable_type,
'fake-site1', vars=vars, caller=caller)
ok = notifier.wait_for_state(InstanceState.FAILED, node_ids)
self.assertTrue(ok)
self.assertTrue(notifier.assure_record_count(1))
self.assertStoreNodeRecords(InstanceState.FAILED, *node_ids)
self.assertStoreLaunchRecord(InstanceState.FAILED, launch_id)
def test_provision_broker_error(self):
client = self.client
notifier = self.notifier
deployable_type = 'empty'
launch_id = _new_id()
self.context_client.create_error = BrokerError("fake failure")
node_ids = [_new_id()]
client.provision(launch_id, node_ids, deployable_type,
'fake-site1', caller="asterix")
ok = notifier.wait_for_state(InstanceState.FAILED, node_ids)
self.assertTrue(ok)
self.assertTrue(notifier.assure_record_count(1))
self.assertStoreNodeRecords(InstanceState.FAILED, *node_ids)
self.assertStoreLaunchRecord(InstanceState.FAILED, launch_id)
def test_dump_state(self):
running_launch, running_nodes = make_launch_and_nodes(_new_id(), 10, InstanceState.RUNNING)
self.store.add_launch(running_launch)
for node in running_nodes:
self.store.add_node(node)
pending_launch, pending_nodes = make_launch_and_nodes(_new_id(), 3, InstanceState.PENDING)
self.store.add_launch(pending_launch)
for node in pending_nodes:
self.store.add_node(node)
running_node_ids = [node['node_id'] for node in running_nodes]
pending_node_ids = [node['node_id'] for node in pending_nodes]
all_node_ids = running_node_ids + pending_node_ids
self.client.dump_state(running_node_ids)
ok = self.notifier.wait_for_state(InstanceState.RUNNING, nodes=running_node_ids)
self.assertTrue(ok)
self.assertEqual(len(self.notifier.nodes), len(running_nodes))
self.client.dump_state(pending_node_ids)
ok = self.notifier.wait_for_state(InstanceState.PENDING, nodes=pending_node_ids)
self.assertTrue(ok)
self.assertEqual(len(self.notifier.nodes), len(all_node_ids))
# we should have not gotten any dupe records yet
self.assertTrue(self.notifier.assure_record_count(1))
# empty dump request should dump nothing
self.client.dump_state([])
self.assertTrue(self.notifier.assure_record_count(1))
def test_terminate(self):
node_ids = []
for _ in range(10):
node_id = _new_id()
node_ids.append(node_id)
self.client.provision(_new_id(), [node_id], "empty",
site="fake-site1", caller="asterix")
self.notifier.wait_for_state(InstanceState.PENDING, node_ids,
before=self.provisioner.leader._force_cycle)
for node_id in node_ids:
node = self.store.get_node(node_id)
self.driver.set_node_running(node['iaas_id'])
self.notifier.wait_for_state(InstanceState.STARTED, node_ids,
before=self.provisioner.leader._force_cycle)
# terminate half of the nodes then the rest
first_five = node_ids[:5]
last_five = node_ids[5:]
self.client.terminate_nodes(first_five, caller="asterix")
ok = self.notifier.wait_for_state(InstanceState.TERMINATED, nodes=first_five)
self.assertTrue(ok)
self.client.terminate_nodes(last_five, caller="asterix")
ok = self.notifier.wait_for_state(InstanceState.TERMINATED, nodes=last_five)
self.assertTrue(ok)
self.assertEqual(set(node_ids), set(self.notifier.nodes))
# should be REQUESTED, PENDING, STARTED, TERMINATING and TERMINATED records for each node
self.assertTrue(self.notifier.assure_record_count(5))
self.assertEqual(len(self.driver.destroyed),
len(node_ids))
def test_terminate_unknown(self):
instance_id = _new_id()
self.client.terminate_nodes([instance_id])
ok = self.notifier.wait_for_state(InstanceState.TERMINATED, nodes=[instance_id])
self.assertTrue(ok)
def test_launch_allocation(self):
node_id = _new_id()
self.client.provision(_new_id(), [node_id], "empty",
site="fake-site1", caller="asterix")
self.notifier.wait_for_state(InstanceState.PENDING, [node_id],
before=self.provisioner.leader._force_cycle)
self.assertStoreNodeRecords(InstanceState.PENDING)
self.assertEqual(len(self.driver.created), 1)
libcloud_node = self.driver.created[0]
self.assertEqual(libcloud_node.size.id, "m1.small")
def test_launch_many_terminate_all(self):
all_node_ids = []
# after the terminate_all, provision requests should be REJECTED
rejected_node_ids = []
for _ in range(100):
node_id = _new_id()
all_node_ids.append(node_id)
self.client.provision(_new_id(), [node_id], "empty",
site="fake-site1", caller="asterix")
self.notifier.wait_for_state(InstanceState.PENDING, all_node_ids,
before=self.provisioner.leader._force_cycle)
self.assertStoreNodeRecords(InstanceState.PENDING, *all_node_ids)
for node_id in all_node_ids:
node = self.store.get_node(node_id)
self.driver.set_node_running(node['iaas_id'])
self.notifier.wait_for_state(InstanceState.STARTED, all_node_ids,
before=self.provisioner.leader._force_cycle)
self.assertStoreNodeRecords(InstanceState.STARTED, *all_node_ids)
log.debug("Expecting %d nodes to be terminated", len(all_node_ids))
self.assertIs(self.client.terminate_all(), False)
# future requests should be rejected
for _ in range(5):
node_id = _new_id()
rejected_node_ids.append(node_id)
self.client.provision(_new_id(), [node_id], "empty",
site="fake-site1", caller="asterix")
self.notifier.wait_for_state(InstanceState.TERMINATED, all_node_ids,
before=self.provisioner.leader._force_cycle)
self.assertStoreNodeRecords(InstanceState.TERMINATED, *all_node_ids)
self.notifier.wait_for_state(InstanceState.REJECTED, rejected_node_ids)
self.assertStoreNodeRecords(InstanceState.REJECTED, *rejected_node_ids)
self.assertEqual(len(self.driver.destroyed),
len(all_node_ids))
self.assertIs(self.client.terminate_all(), True)
# now re-enable
self.client.enable()
node_id = _new_id()
log.debug("Launching node %s which should be accepted", node_id)
self.client.provision(_new_id(), [node_id], "empty",
site="fake-site1", caller="asterix")
self.notifier.wait_for_state(InstanceState.PENDING, [node_id],
before=self.provisioner.leader._force_cycle)
self.assertStoreNodeRecords(InstanceState.PENDING, node_id)
def test_describe(self):
node_ids = []
for _ in range(3):
launch_id = _new_id()
running_launch, running_nodes = make_launch_and_nodes(launch_id, 1,
InstanceState.RUNNING,
site="fake-site1", caller=self.default_user)
self.store.add_launch(running_launch)
for node in running_nodes:
self.store.add_node(node)
node_ids.append(running_nodes[0]['node_id'])
log.debug("requestin")
all_nodes = self.client.describe_nodes()
self.assertEqual(len(all_nodes), len(node_ids))
one_node = self.client.describe_nodes([node_ids[0]])
self.assertEqual(len(one_node), 1)
self.assertEqual(one_node[0]['node_id'], node_ids[0])
def test_multiuser(self):
"""Test that nodes started by one user can't be modified by
another user
"""
permitted_user = "******"
disallowed_user = "******"
client = self.client
deployable_type = 'empty'
launch_id = _new_id()
node_ids = [_new_id()]
vars = {'image_id': 'fake-image'}
client.provision(launch_id, node_ids, deployable_type,
'fake-site1', vars=vars, caller=permitted_user)
self.notifier.wait_for_state(InstanceState.PENDING, node_ids,
before=self.provisioner.leader._force_cycle)
self.assertStoreNodeRecords(InstanceState.PENDING, *node_ids)
# Test describe
permitted_nodes = client.describe_nodes(caller=permitted_user)
self.assertEqual(len(permitted_nodes), len(node_ids))
disallowed_nodes = client.describe_nodes(caller=disallowed_user)
self.assertEqual(len(disallowed_nodes), 0)
# Test terminate
client.terminate_nodes(node_ids, caller=disallowed_user)
terminate_timed_out = False
try:
self.notifier.wait_for_state(InstanceState.TERMINATED, node_ids,
before=self.provisioner.leader._force_cycle, timeout=2)
except Exception:
terminate_timed_out = True
self.assertTrue(terminate_timed_out,
msg="Terminate worked with non-matching user")
client.terminate_nodes(node_ids, caller=permitted_user)
self.notifier.wait_for_state(InstanceState.TERMINATED, node_ids,
before=self.provisioner.leader._force_cycle, timeout=2)
self.assertStoreNodeRecords(InstanceState.TERMINATED, *node_ids)
def test_record_reaper(self):
launch_id1 = _new_id()
launch_id2 = _new_id()
now = time.time()
node1 = make_node(launch_id1, InstanceState.TERMINATED, caller=self.default_user,
state_changes=[(InstanceState.TERMINATED, now - self.record_reaping_max_age - 1)])
node2 = make_node(launch_id1, InstanceState.FAILED, caller=self.default_user,
state_changes=[(InstanceState.FAILED, now - self.record_reaping_max_age - 1)])
node3 = make_node(launch_id1, InstanceState.REJECTED, caller=self.default_user,
state_changes=[(InstanceState.REJECTED, now - self.record_reaping_max_age - 1)])
nodes1 = [node1, node2, node3]
launch1 = make_launch(launch_id1, InstanceState.RUNNING, nodes1, caller=self.default_user)
node4 = make_node(launch_id2, InstanceState.RUNNING, caller=self.default_user,
state_changes=[(InstanceState.RUNNING, now - self.record_reaping_max_age - 1)])
node5 = make_node(launch_id2, InstanceState.TERMINATED, caller=self.default_user,
state_changes=[(InstanceState.TERMINATED, now - self.record_reaping_max_age - 1)])
nodes2 = [node4, node5]
launch2 = make_launch(launch_id2, InstanceState.RUNNING, nodes2, caller=self.default_user)
self.store.add_launch(launch1)
for node in nodes1:
self.store.add_node(node)
self.store.add_launch(launch2)
for node in nodes2:
self.store.add_node(node)
# Force a record reaping cycle
self.provisioner.leader._force_record_reaping()
# Check that the first launch is completely removed
node_ids1 = map(lambda x: x['node_id'], nodes1)
self.assertNoStoreNodeRecords(*node_ids1)
self.assertNoStoreLaunchRecord(launch_id1)
# Check that the second launch is still here but with only the running node
self.assertStoreNodeRecords(InstanceState.RUNNING, node4['node_id'])
self.assertStoreLaunchRecord(InstanceState.RUNNING, launch_id2)
class ProvisionerServiceTest(BaseProvisionerServiceTests):
"""Integration tests that use fake context broker and IaaS driver fixtures
"""
@defer.inlineCallbacks
def setUp(self):
self.notifier = FakeProvisionerNotifier()
self.context_client = FakeContextClient()
self.store = yield self.setup_store()
self.site_drivers = {'fake-site1' : FakeNodeDriver()}
yield self._start_container()
yield self.spawn_procs()
pId = yield self.procRegistry.get("provisioner")
self.client = ProvisionerClient(pid=pId)
@defer.inlineCallbacks
def tearDown(self):
yield self.shutdown_procs()
yield self.teardown_store()
yield self._stop_container()
def setup_store(self):
return defer.succeed(ProvisionerStore())
def teardown_store(self):
return defer.succeed(None)
@defer.inlineCallbacks
def test_provision_bad_dt(self):
client = self.client
notifier = self.notifier
worker_node_count = 3
deployable_type = 'this-doesnt-exist'
nodes = {'head-node' : FakeLaunchItem(1, 'fake-site1', 'small', None),
'worker-node' : FakeLaunchItem(worker_node_count,
'fake-site1', 'small', None)}
launch_id = _new_id()
node_ids = [node_id for node in nodes.itervalues()
for node_id in node.instance_ids]
self.assertEqual(len(node_ids), worker_node_count + 1)
yield client.provision(launch_id, deployable_type, nodes, ('subscriber',))
ok = yield notifier.wait_for_state(states.FAILED, node_ids)
self.assertTrue(ok)
self.assertTrue(notifier.assure_record_count(1))
yield self.assertStoreNodeRecords(states.FAILED, *node_ids)
yield self.assertStoreLaunchRecord(states.FAILED, launch_id)
@defer.inlineCallbacks
def test_provision_broker_error(self):
client = self.client
notifier = self.notifier
worker_node_count = 3
deployable_type = 'base-cluster'
nodes = {'head-node' : FakeLaunchItem(1, 'fake-site1', 'small', None),
'worker-node' : FakeLaunchItem(worker_node_count,
'fake-site1', 'small', None)}
launch_id = _new_id()
node_ids = [node_id for node in nodes.itervalues()
for node_id in node.instance_ids]
self.assertEqual(len(node_ids), worker_node_count + 1)
self.context_client.create_error = BrokerError("fake failure")
yield client.provision(launch_id, deployable_type, nodes, ('subscriber',))
ok = yield notifier.wait_for_state(states.FAILED, node_ids)
self.assertTrue(ok)
self.assertTrue(notifier.assure_record_count(1))
yield self.assertStoreNodeRecords(states.FAILED, *node_ids)
yield self.assertStoreLaunchRecord(states.FAILED, launch_id)
@defer.inlineCallbacks
def test_dump_state(self):
running_launch, running_nodes = make_launch_and_nodes(_new_id(), 10, states.RUNNING)
yield self.store.put_launch(running_launch)
yield self.store.put_nodes(running_nodes)
pending_launch, pending_nodes = make_launch_and_nodes(_new_id(), 3, states.PENDING)
yield self.store.put_launch(pending_launch)
yield self.store.put_nodes(pending_nodes)
running_node_ids = [node['node_id'] for node in running_nodes]
pending_node_ids = [node['node_id'] for node in pending_nodes]
all_node_ids = running_node_ids + pending_node_ids
yield self.client.dump_state(running_node_ids)
ok = yield self.notifier.wait_for_state(states.RUNNING, nodes=running_node_ids)
self.assertTrue(ok)
self.assertEqual(len(self.notifier.nodes), len(running_nodes))
yield self.client.dump_state(pending_node_ids)
ok = yield self.notifier.wait_for_state(states.PENDING, nodes=pending_node_ids)
self.assertTrue(ok)
self.assertEqual(len(self.notifier.nodes), len(all_node_ids))
# we should have not gotten any dupe records yet
self.assertTrue(self.notifier.assure_record_count(1))
# empty dump request should dump nothing
yield self.client.dump_state([])
self.assertTrue(self.notifier.assure_record_count(1))
@defer.inlineCallbacks
def test_dump_state_unknown_node(self):
node_ids = ["09ddd3f8-a5a5-4196-ac13-eab4d4b0c777"]
subscribers = ["hello1_subscriber"]
yield self.client.dump_state(node_ids, force_subscribe=subscribers[0])
ok = yield self.notifier.wait_for_state(states.FAILED, nodes=node_ids)
self.assertTrue(ok)
self.assertEqual(len(self.notifier.nodes), len(node_ids))
for node_id in node_ids:
ok = yield self.notifier.assure_subscribers(node_id, subscribers)
self.assertTrue(ok)
@defer.inlineCallbacks
def test_terminate(self):
launch_id = _new_id()
running_launch, running_nodes = make_launch_and_nodes(launch_id, 10,
states.RUNNING,
site="fake-site1")
yield self.store.put_launch(running_launch)
yield self.store.put_nodes(running_nodes)
node_ids = [node['node_id'] for node in running_nodes]
# terminate half of the nodes then the launch as a whole
first_five = node_ids[:5]
yield self.client.terminate_nodes(first_five)
ok = yield self.notifier.wait_for_state(states.TERMINATED, nodes=first_five)
self.assertTrue(ok)
self.assertEqual(set(first_five), set(self.notifier.nodes))
yield self.client.terminate_launches((launch_id,))
ok = yield self.notifier.wait_for_state(states.TERMINATED, nodes=node_ids)
self.assertTrue(ok)
self.assertEqual(set(node_ids), set(self.notifier.nodes))
# should be TERMINATING and TERMINATED record for each node
self.assertTrue(self.notifier.assure_record_count(2))
self.assertEqual(len(self.site_drivers['fake-site1'].destroyed),
len(node_ids))
@defer.inlineCallbacks
def test_terminate_all(self):
# create a ton of launches
launch_specs = [(30, 3, states.RUNNING), (50, 1, states.TERMINATED), (80, 1, states.RUNNING)]
to_be_terminated_node_ids = []
for launchcount, nodecount, state in launch_specs:
for i in range(launchcount):
launch_id = _new_id()
launch, nodes = make_launch_and_nodes(
launch_id, nodecount, state, site="fake-site1")
yield self.store.put_launch(launch)
yield self.store.put_nodes(nodes)
if state < states.TERMINATED:
to_be_terminated_node_ids.extend(node["node_id"] for node in nodes)
log.debug("Expecting %d nodes to be terminated", len(to_be_terminated_node_ids))
yield self.client.terminate_all(rpcwait=True)
yield self.assertStoreNodeRecords(states.TERMINATED, *to_be_terminated_node_ids)
ok = self.notifier.assure_state(states.TERMINATED, nodes=to_be_terminated_node_ids)
self.assertTrue(ok)
self.assertEqual(set(to_be_terminated_node_ids), set(self.notifier.nodes))
self.assertEqual(len(self.site_drivers['fake-site1'].destroyed),
len(to_be_terminated_node_ids))
@defer.inlineCallbacks
def test_query(self):
#default is non-rpc. should be None result
res = yield self.client.query()
self.assertEqual(res, None)
#returns true in RPC case
res = yield self.client.query(rpc=True)
self.assertEqual(res, True)
