class ProvisionerServiceTerminateAllTest(BaseProvisionerServiceTests):
"""Tests that use a fake ProvisionerCore to test the Deferred RPC
polling mechanism of terminate_all
"""
@defer.inlineCallbacks
def setUp(self):
self.notifier = FakeProvisionerNotifier()
self.context_client = FakeContextClient()
self.store = ProvisionerStore()
self.site_drivers = {'fake-site1': FakeNodeDriver()}
yield self._start_container()
yield self.spawn_procs()
self.fakecore = TerminateAllFakeCore()
self.patch(self.provisioner, "core", self.fakecore)
pId = yield self.procRegistry.get("provisioner")
self.client = ProvisionerClient(pid=pId)
@defer.inlineCallbacks
def tearDown(self):
yield self._shutdown_processes()
yield self._stop_container()
@defer.inlineCallbacks
def test_terminate_all_deferred(self):
"""Check the specific behavior with terminate_all_deferred.
"""
service_deferred = defer.Deferred()
self.fakecore.deferred = service_deferred
client_deferred = self.client.terminate_all(rpcwait=True, poll=0.1)
yield procutils.asleep(0.3)
# first time the core fires its Deferred, check_terminate_all still
# says there are instances. So client should not yet return
self.fakecore.all_terminated = False
self.fakecore.deferred = defer.Deferred() # set up the next Deferred
service_deferred.callback(None)
service_deferred = self.fakecore.deferred
yield procutils.asleep(0.3)
self.assertFalse(client_deferred.called)
self.assertEqual(self.fakecore.check_terminate_all_count, 1)
# now we flip terminate_all_check to True. client should return
# on next cycle
self.fakecore.all_terminated = True
service_deferred.callback(None)
yield client_deferred
@defer.inlineCallbacks
def test_terminate_all_deferred_error_retry(self):
service_deferred = defer.Deferred()
self.fakecore.deferred = service_deferred
client_deferred = self.client.terminate_all(rpcwait=True,
poll=0.01,
retries=3)
yield procutils.asleep(0.1)
for i in range(3):
self.assertEqual(self.fakecore.terminate_all_count, i + 1)
self.fakecore.deferred = defer.Deferred()
service_deferred.errback(Exception("went bad #%d" % (i + 1)))
service_deferred = self.fakecore.deferred
yield procutils.asleep(0.2)
self.assertFalse(client_deferred.called)
self.assertEqual(self.fakecore.terminate_all_count, i + 2)
#this last errback should cause client_deferred to errback itself
self.fakecore.deferred = defer.Deferred()
service_deferred.errback(Exception("went bad for the last time"))
yield procutils.asleep(0.03)
try:
yield client_deferred
except Exception, e:
log.exception(
"Expected error, couldn't terminate all after retries: %s", e)
else:
class ProvisionerServiceTerminateAllTest(BaseProvisionerServiceTests):
"""Tests that use a fake ProvisionerCore to test the Deferred RPC
polling mechanism of terminate_all
"""
@defer.inlineCallbacks
def setUp(self):
self.notifier = FakeProvisionerNotifier()
self.context_client = FakeContextClient()
self.store = ProvisionerStore()
self.site_drivers = {'fake-site1' : FakeNodeDriver()}
yield self._start_container()
yield self.spawn_procs()
self.fakecore = TerminateAllFakeCore()
self.patch(self.provisioner, "core", self.fakecore)
pId = yield self.procRegistry.get("provisioner")
self.client = ProvisionerClient(pid=pId)
@defer.inlineCallbacks
def tearDown(self):
yield self._shutdown_processes()
yield self._stop_container()
@defer.inlineCallbacks
def test_terminate_all_deferred(self):
"""Check the specific behavior with terminate_all_deferred.
"""
service_deferred = defer.Deferred()
self.fakecore.deferred = service_deferred
client_deferred = self.client.terminate_all(rpcwait=True, poll=0.1)
yield procutils.asleep(0.3)
# first time the core fires its Deferred, check_terminate_all still
# says there are instances. So client should not yet return
self.fakecore.all_terminated = False
self.fakecore.deferred = defer.Deferred() # set up the next Deferred
service_deferred.callback(None)
service_deferred = self.fakecore.deferred
yield procutils.asleep(0.3)
self.assertFalse(client_deferred.called)
self.assertEqual(self.fakecore.check_terminate_all_count, 1)
# now we flip terminate_all_check to True. client should return
# on next cycle
self.fakecore.all_terminated = True
service_deferred.callback(None)
yield client_deferred
@defer.inlineCallbacks
def test_terminate_all_deferred_error_retry(self):
service_deferred = defer.Deferred()
self.fakecore.deferred = service_deferred
client_deferred = self.client.terminate_all(rpcwait=True, poll=0.01, retries=3)
yield procutils.asleep(0.1)
for i in range(3):
self.assertEqual(self.fakecore.terminate_all_count, i+1)
self.fakecore.deferred = defer.Deferred()
service_deferred.errback(Exception("went bad #%d" % (i+1)))
service_deferred = self.fakecore.deferred
yield procutils.asleep(0.2)
self.assertFalse(client_deferred.called)
self.assertEqual(self.fakecore.terminate_all_count, i+2)
#this last errback should cause client_deferred to errback itself
self.fakecore.deferred = defer.Deferred()
service_deferred.errback(Exception("went bad for the last time"))
yield procutils.asleep(0.03)
try:
yield client_deferred
except Exception,e:
log.exception("Expected error, couldn't terminate all after retries: %s", e)
else:
class ProvisionerServiceTest(BaseProvisionerServiceTests):
"""Integration tests that use fake context broker and IaaS driver fixtures
"""
@defer.inlineCallbacks
def setUp(self):
self.notifier = FakeProvisionerNotifier()
self.context_client = FakeContextClient()
self.store = yield self.setup_store()
self.site_drivers = {'fake-site1': FakeNodeDriver()}
yield self._start_container()
yield self.spawn_procs()
pId = yield self.procRegistry.get("provisioner")
self.client = ProvisionerClient(pid=pId)
@defer.inlineCallbacks
def tearDown(self):
yield self.shutdown_procs()
yield self.teardown_store()
yield self._stop_container()
def setup_store(self):
return defer.succeed(ProvisionerStore())
def teardown_store(self):
return defer.succeed(None)
@defer.inlineCallbacks
def test_provision_bad_dt(self):
client = self.client
notifier = self.notifier
worker_node_count = 3
deployable_type = 'this-doesnt-exist'
nodes = {
'head-node':
FakeLaunchItem(1, 'fake-site1', 'small', None),
'worker-node':
FakeLaunchItem(worker_node_count, 'fake-site1', 'small', None)
}
launch_id = _new_id()
node_ids = [
node_id for node in nodes.itervalues()
for node_id in node.instance_ids
]
self.assertEqual(len(node_ids), worker_node_count + 1)
yield client.provision(launch_id, deployable_type, nodes,
('subscriber', ))
ok = yield notifier.wait_for_state(states.FAILED, node_ids)
self.assertTrue(ok)
self.assertTrue(notifier.assure_record_count(1))
yield self.assertStoreNodeRecords(states.FAILED, *node_ids)
yield self.assertStoreLaunchRecord(states.FAILED, launch_id)
@defer.inlineCallbacks
def test_provision_broker_error(self):
client = self.client
notifier = self.notifier
worker_node_count = 3
deployable_type = 'base-cluster'
nodes = {
'head-node':
FakeLaunchItem(1, 'fake-site1', 'small', None),
'worker-node':
FakeLaunchItem(worker_node_count, 'fake-site1', 'small', None)
}
launch_id = _new_id()
node_ids = [
node_id for node in nodes.itervalues()
for node_id in node.instance_ids
]
self.assertEqual(len(node_ids), worker_node_count + 1)
self.context_client.create_error = BrokerError("fake failure")
yield client.provision(launch_id, deployable_type, nodes,
('subscriber', ))
ok = yield notifier.wait_for_state(states.FAILED, node_ids)
self.assertTrue(ok)
self.assertTrue(notifier.assure_record_count(1))
yield self.assertStoreNodeRecords(states.FAILED, *node_ids)
yield self.assertStoreLaunchRecord(states.FAILED, launch_id)
@defer.inlineCallbacks
def test_dump_state(self):
running_launch, running_nodes = make_launch_and_nodes(
_new_id(), 10, states.RUNNING)
yield self.store.put_launch(running_launch)
yield self.store.put_nodes(running_nodes)
pending_launch, pending_nodes = make_launch_and_nodes(
_new_id(), 3, states.PENDING)
yield self.store.put_launch(pending_launch)
yield self.store.put_nodes(pending_nodes)
running_node_ids = [node['node_id'] for node in running_nodes]
pending_node_ids = [node['node_id'] for node in pending_nodes]
all_node_ids = running_node_ids + pending_node_ids
yield self.client.dump_state(running_node_ids)
ok = yield self.notifier.wait_for_state(states.RUNNING,
nodes=running_node_ids)
self.assertTrue(ok)
self.assertEqual(len(self.notifier.nodes), len(running_nodes))
yield self.client.dump_state(pending_node_ids)
ok = yield self.notifier.wait_for_state(states.PENDING,
nodes=pending_node_ids)
self.assertTrue(ok)
self.assertEqual(len(self.notifier.nodes), len(all_node_ids))
# we should have not gotten any dupe records yet
self.assertTrue(self.notifier.assure_record_count(1))
# empty dump request should dump nothing
yield self.client.dump_state([])
self.assertTrue(self.notifier.assure_record_count(1))
@defer.inlineCallbacks
def test_dump_state_unknown_node(self):
node_ids = ["09ddd3f8-a5a5-4196-ac13-eab4d4b0c777"]
subscribers = ["hello1_subscriber"]
yield self.client.dump_state(node_ids, force_subscribe=subscribers[0])
ok = yield self.notifier.wait_for_state(states.FAILED, nodes=node_ids)
self.assertTrue(ok)
self.assertEqual(len(self.notifier.nodes), len(node_ids))
for node_id in node_ids:
ok = yield self.notifier.assure_subscribers(node_id, subscribers)
self.assertTrue(ok)
@defer.inlineCallbacks
def test_terminate(self):
launch_id = _new_id()
running_launch, running_nodes = make_launch_and_nodes(
launch_id, 10, states.RUNNING, site="fake-site1")
yield self.store.put_launch(running_launch)
yield self.store.put_nodes(running_nodes)
node_ids = [node['node_id'] for node in running_nodes]
# terminate half of the nodes then the launch as a whole
first_five = node_ids[:5]
yield self.client.terminate_nodes(first_five)
ok = yield self.notifier.wait_for_state(states.TERMINATED,
nodes=first_five)
self.assertTrue(ok)
self.assertEqual(set(first_five), set(self.notifier.nodes))
yield self.client.terminate_launches((launch_id, ))
ok = yield self.notifier.wait_for_state(states.TERMINATED,
nodes=node_ids)
self.assertTrue(ok)
self.assertEqual(set(node_ids), set(self.notifier.nodes))
# should be TERMINATING and TERMINATED record for each node
self.assertTrue(self.notifier.assure_record_count(2))
self.assertEqual(len(self.site_drivers['fake-site1'].destroyed),
len(node_ids))
@defer.inlineCallbacks
def test_terminate_all(self):
# create a ton of launches
launch_specs = [(30, 3, states.RUNNING), (50, 1, states.TERMINATED),
(80, 1, states.RUNNING)]
to_be_terminated_node_ids = []
for launchcount, nodecount, state in launch_specs:
for i in range(launchcount):
launch_id = _new_id()
launch, nodes = make_launch_and_nodes(launch_id,
nodecount,
state,
site="fake-site1")
yield self.store.put_launch(launch)
yield self.store.put_nodes(nodes)
if state < states.TERMINATED:
to_be_terminated_node_ids.extend(node["node_id"]
for node in nodes)
log.debug("Expecting %d nodes to be terminated",
len(to_be_terminated_node_ids))
yield self.client.terminate_all(rpcwait=True)
yield self.assertStoreNodeRecords(states.TERMINATED,
*to_be_terminated_node_ids)
ok = self.notifier.assure_state(states.TERMINATED,
nodes=to_be_terminated_node_ids)
self.assertTrue(ok)
self.assertEqual(set(to_be_terminated_node_ids),
set(self.notifier.nodes))
self.assertEqual(len(self.site_drivers['fake-site1'].destroyed),
len(to_be_terminated_node_ids))
@defer.inlineCallbacks
def test_query(self):
#default is non-rpc. should be None result
res = yield self.client.query()
self.assertEqual(res, None)
#returns true in RPC case
res = yield self.client.query(rpc=True)
self.assertEqual(res, True)
class ProvisionerServiceTest(BaseProvisionerServiceTests):
"""Integration tests that use fake context broker and IaaS driver fixtures
"""
@defer.inlineCallbacks
def setUp(self):
self.notifier = FakeProvisionerNotifier()
self.context_client = FakeContextClient()
self.store = yield self.setup_store()
self.site_drivers = {'fake-site1' : FakeNodeDriver()}
yield self._start_container()
yield self.spawn_procs()
pId = yield self.procRegistry.get("provisioner")
self.client = ProvisionerClient(pid=pId)
@defer.inlineCallbacks
def tearDown(self):
yield self.shutdown_procs()
yield self.teardown_store()
yield self._stop_container()
def setup_store(self):
return defer.succeed(ProvisionerStore())
def teardown_store(self):
return defer.succeed(None)
@defer.inlineCallbacks
def test_provision_bad_dt(self):
client = self.client
notifier = self.notifier
worker_node_count = 3
deployable_type = 'this-doesnt-exist'
nodes = {'head-node' : FakeLaunchItem(1, 'fake-site1', 'small', None),
'worker-node' : FakeLaunchItem(worker_node_count,
'fake-site1', 'small', None)}
launch_id = _new_id()
node_ids = [node_id for node in nodes.itervalues()
for node_id in node.instance_ids]
self.assertEqual(len(node_ids), worker_node_count + 1)
yield client.provision(launch_id, deployable_type, nodes, ('subscriber',))
ok = yield notifier.wait_for_state(states.FAILED, node_ids)
self.assertTrue(ok)
self.assertTrue(notifier.assure_record_count(1))
yield self.assertStoreNodeRecords(states.FAILED, *node_ids)
yield self.assertStoreLaunchRecord(states.FAILED, launch_id)
@defer.inlineCallbacks
def test_provision_broker_error(self):
client = self.client
notifier = self.notifier
worker_node_count = 3
deployable_type = 'base-cluster'
nodes = {'head-node' : FakeLaunchItem(1, 'fake-site1', 'small', None),
'worker-node' : FakeLaunchItem(worker_node_count,
'fake-site1', 'small', None)}
launch_id = _new_id()
node_ids = [node_id for node in nodes.itervalues()
for node_id in node.instance_ids]
self.assertEqual(len(node_ids), worker_node_count + 1)
self.context_client.create_error = BrokerError("fake failure")
yield client.provision(launch_id, deployable_type, nodes, ('subscriber',))
ok = yield notifier.wait_for_state(states.FAILED, node_ids)
self.assertTrue(ok)
self.assertTrue(notifier.assure_record_count(1))
yield self.assertStoreNodeRecords(states.FAILED, *node_ids)
yield self.assertStoreLaunchRecord(states.FAILED, launch_id)
@defer.inlineCallbacks
def test_dump_state(self):
running_launch, running_nodes = make_launch_and_nodes(_new_id(), 10, states.RUNNING)
yield self.store.put_launch(running_launch)
yield self.store.put_nodes(running_nodes)
pending_launch, pending_nodes = make_launch_and_nodes(_new_id(), 3, states.PENDING)
yield self.store.put_launch(pending_launch)
yield self.store.put_nodes(pending_nodes)
running_node_ids = [node['node_id'] for node in running_nodes]
pending_node_ids = [node['node_id'] for node in pending_nodes]
all_node_ids = running_node_ids + pending_node_ids
yield self.client.dump_state(running_node_ids)
ok = yield self.notifier.wait_for_state(states.RUNNING, nodes=running_node_ids)
self.assertTrue(ok)
self.assertEqual(len(self.notifier.nodes), len(running_nodes))
yield self.client.dump_state(pending_node_ids)
ok = yield self.notifier.wait_for_state(states.PENDING, nodes=pending_node_ids)
self.assertTrue(ok)
self.assertEqual(len(self.notifier.nodes), len(all_node_ids))
# we should have not gotten any dupe records yet
self.assertTrue(self.notifier.assure_record_count(1))
# empty dump request should dump nothing
yield self.client.dump_state([])
self.assertTrue(self.notifier.assure_record_count(1))
@defer.inlineCallbacks
def test_dump_state_unknown_node(self):
node_ids = ["09ddd3f8-a5a5-4196-ac13-eab4d4b0c777"]
subscribers = ["hello1_subscriber"]
yield self.client.dump_state(node_ids, force_subscribe=subscribers[0])
ok = yield self.notifier.wait_for_state(states.FAILED, nodes=node_ids)
self.assertTrue(ok)
self.assertEqual(len(self.notifier.nodes), len(node_ids))
for node_id in node_ids:
ok = yield self.notifier.assure_subscribers(node_id, subscribers)
self.assertTrue(ok)
@defer.inlineCallbacks
def test_terminate(self):
launch_id = _new_id()
running_launch, running_nodes = make_launch_and_nodes(launch_id, 10,
states.RUNNING,
site="fake-site1")
yield self.store.put_launch(running_launch)
yield self.store.put_nodes(running_nodes)
node_ids = [node['node_id'] for node in running_nodes]
# terminate half of the nodes then the launch as a whole
first_five = node_ids[:5]
yield self.client.terminate_nodes(first_five)
ok = yield self.notifier.wait_for_state(states.TERMINATED, nodes=first_five)
self.assertTrue(ok)
self.assertEqual(set(first_five), set(self.notifier.nodes))
yield self.client.terminate_launches((launch_id,))
ok = yield self.notifier.wait_for_state(states.TERMINATED, nodes=node_ids)
self.assertTrue(ok)
self.assertEqual(set(node_ids), set(self.notifier.nodes))
# should be TERMINATING and TERMINATED record for each node
self.assertTrue(self.notifier.assure_record_count(2))
self.assertEqual(len(self.site_drivers['fake-site1'].destroyed),
len(node_ids))
@defer.inlineCallbacks
def test_terminate_all(self):
# create a ton of launches
launch_specs = [(30, 3, states.RUNNING), (50, 1, states.TERMINATED), (80, 1, states.RUNNING)]
to_be_terminated_node_ids = []
for launchcount, nodecount, state in launch_specs:
for i in range(launchcount):
launch_id = _new_id()
launch, nodes = make_launch_and_nodes(
launch_id, nodecount, state, site="fake-site1")
yield self.store.put_launch(launch)
yield self.store.put_nodes(nodes)
if state < states.TERMINATED:
to_be_terminated_node_ids.extend(node["node_id"] for node in nodes)
log.debug("Expecting %d nodes to be terminated", len(to_be_terminated_node_ids))
yield self.client.terminate_all(rpcwait=True)
yield self.assertStoreNodeRecords(states.TERMINATED, *to_be_terminated_node_ids)
ok = self.notifier.assure_state(states.TERMINATED, nodes=to_be_terminated_node_ids)
self.assertTrue(ok)
self.assertEqual(set(to_be_terminated_node_ids), set(self.notifier.nodes))
self.assertEqual(len(self.site_drivers['fake-site1'].destroyed),
len(to_be_terminated_node_ids))
@defer.inlineCallbacks
def test_query(self):
#default is non-rpc. should be None result
res = yield self.client.query()
self.assertEqual(res, None)
#returns true in RPC case
res = yield self.client.query(rpc=True)
self.assertEqual(res, True)
