def setUp(self):
self.notifier = FakeProvisionerNotifier()
self.context_client = FakeContextClient()
self.store = yield self.setup_store()
self.site_drivers = {'fake-site1': FakeNodeDriver()}
yield self._start_container()
yield self.spawn_procs()
pId = yield self.procRegistry.get("provisioner")
self.client = ProvisionerClient(pid=pId)
def setUp(self):
self.notifier = FakeProvisionerNotifier()
self.context_client = FakeContextClient()
self.store = ProvisionerStore()
self.site_drivers = {'fake-site1': FakeNodeDriver()}
yield self._start_container()
yield self.spawn_procs()
self.fakecore = TerminateAllFakeCore()
self.patch(self.provisioner, "core", self.fakecore)
pId = yield self.procRegistry.get("provisioner")
self.client = ProvisionerClient(pid=pId)
def slc_init(self):
interval = float(
self.spawn_args.get("interval_seconds", DEFAULT_QUERY_INTERVAL))
self.client = ProvisionerClient(self)
log.debug('Starting provisioner query loop - %s second interval',
interval)
self.loop = LoopingCall(self.query)
self.loop.start(interval)
def setUp(self):
self.notifier = FakeProvisionerNotifier()
self.context_client = FakeContextClient()
self.store = yield self.setup_store()
self.site_drivers = {'fake-site1' : FakeNodeDriver()}
yield self._start_container()
yield self.spawn_procs()
pId = yield self.procRegistry.get("provisioner")
self.client = ProvisionerClient(pid=pId)
def setUp(self):
self.notifier = FakeProvisionerNotifier()
self.context_client = FakeContextClient()
self.store = ProvisionerStore()
self.site_drivers = {'fake-site1' : FakeNodeDriver()}
yield self._start_container()
yield self.spawn_procs()
self.fakecore = TerminateAllFakeCore()
self.patch(self.provisioner, "core", self.fakecore)
pId = yield self.procRegistry.get("provisioner")
self.client = ProvisionerClient(pid=pId)
def setUp(self):
# @itv decorator is gone. This test could probably go away entirely but I'v
# found it personally useful. Unconditionally skipping for now, til we know
# what to do with it.
raise unittest.SkipTest("developer-only Nimbus integration test")
# skip this test if IaaS credentials are unavailable
maybe_skip_test()
self.notifier = FakeProvisionerNotifier()
self.context_client = get_context_client()
self.store = yield self.setup_store()
self.site_drivers = provisioner.get_site_drivers(
get_nimbus_test_sites())
yield self._start_container()
yield self.spawn_procs()
pId = yield self.procRegistry.get("provisioner")
self.client = ProvisionerClient(pid=pId)
class ProvisionerServiceTerminateAllTest(BaseProvisionerServiceTests):
"""Tests that use a fake ProvisionerCore to test the Deferred RPC
polling mechanism of terminate_all
"""
@defer.inlineCallbacks
def setUp(self):
self.notifier = FakeProvisionerNotifier()
self.context_client = FakeContextClient()
self.store = ProvisionerStore()
self.site_drivers = {'fake-site1': FakeNodeDriver()}
yield self._start_container()
yield self.spawn_procs()
self.fakecore = TerminateAllFakeCore()
self.patch(self.provisioner, "core", self.fakecore)
pId = yield self.procRegistry.get("provisioner")
self.client = ProvisionerClient(pid=pId)
@defer.inlineCallbacks
def tearDown(self):
yield self._shutdown_processes()
yield self._stop_container()
@defer.inlineCallbacks
def test_terminate_all_deferred(self):
"""Check the specific behavior with terminate_all_deferred.
"""
service_deferred = defer.Deferred()
self.fakecore.deferred = service_deferred
client_deferred = self.client.terminate_all(rpcwait=True, poll=0.1)
yield procutils.asleep(0.3)
# first time the core fires its Deferred, check_terminate_all still
# says there are instances. So client should not yet return
self.fakecore.all_terminated = False
self.fakecore.deferred = defer.Deferred() # set up the next Deferred
service_deferred.callback(None)
service_deferred = self.fakecore.deferred
yield procutils.asleep(0.3)
self.assertFalse(client_deferred.called)
self.assertEqual(self.fakecore.check_terminate_all_count, 1)
# now we flip terminate_all_check to True. client should return
# on next cycle
self.fakecore.all_terminated = True
service_deferred.callback(None)
yield client_deferred
@defer.inlineCallbacks
def test_terminate_all_deferred_error_retry(self):
service_deferred = defer.Deferred()
self.fakecore.deferred = service_deferred
client_deferred = self.client.terminate_all(rpcwait=True,
poll=0.01,
retries=3)
yield procutils.asleep(0.1)
for i in range(3):
self.assertEqual(self.fakecore.terminate_all_count, i + 1)
self.fakecore.deferred = defer.Deferred()
service_deferred.errback(Exception("went bad #%d" % (i + 1)))
service_deferred = self.fakecore.deferred
yield procutils.asleep(0.2)
self.assertFalse(client_deferred.called)
self.assertEqual(self.fakecore.terminate_all_count, i + 2)
#this last errback should cause client_deferred to errback itself
self.fakecore.deferred = defer.Deferred()
service_deferred.errback(Exception("went bad for the last time"))
yield procutils.asleep(0.03)
try:
yield client_deferred
except Exception, e:
log.exception(
"Expected error, couldn't terminate all after retries: %s", e)
else:
class ProvisionerServiceTest(BaseProvisionerServiceTests):
"""Integration tests that use fake context broker and IaaS driver fixtures
"""
@defer.inlineCallbacks
def setUp(self):
self.notifier = FakeProvisionerNotifier()
self.context_client = FakeContextClient()
self.store = yield self.setup_store()
self.site_drivers = {'fake-site1': FakeNodeDriver()}
yield self._start_container()
yield self.spawn_procs()
pId = yield self.procRegistry.get("provisioner")
self.client = ProvisionerClient(pid=pId)
@defer.inlineCallbacks
def tearDown(self):
yield self.shutdown_procs()
yield self.teardown_store()
yield self._stop_container()
def setup_store(self):
return defer.succeed(ProvisionerStore())
def teardown_store(self):
return defer.succeed(None)
@defer.inlineCallbacks
def test_provision_bad_dt(self):
client = self.client
notifier = self.notifier
worker_node_count = 3
deployable_type = 'this-doesnt-exist'
nodes = {
'head-node':
FakeLaunchItem(1, 'fake-site1', 'small', None),
'worker-node':
FakeLaunchItem(worker_node_count, 'fake-site1', 'small', None)
}
launch_id = _new_id()
node_ids = [
node_id for node in nodes.itervalues()
for node_id in node.instance_ids
]
self.assertEqual(len(node_ids), worker_node_count + 1)
yield client.provision(launch_id, deployable_type, nodes,
('subscriber', ))
ok = yield notifier.wait_for_state(states.FAILED, node_ids)
self.assertTrue(ok)
self.assertTrue(notifier.assure_record_count(1))
yield self.assertStoreNodeRecords(states.FAILED, *node_ids)
yield self.assertStoreLaunchRecord(states.FAILED, launch_id)
@defer.inlineCallbacks
def test_provision_broker_error(self):
client = self.client
notifier = self.notifier
worker_node_count = 3
deployable_type = 'base-cluster'
nodes = {
'head-node':
FakeLaunchItem(1, 'fake-site1', 'small', None),
'worker-node':
FakeLaunchItem(worker_node_count, 'fake-site1', 'small', None)
}
launch_id = _new_id()
node_ids = [
node_id for node in nodes.itervalues()
for node_id in node.instance_ids
]
self.assertEqual(len(node_ids), worker_node_count + 1)
self.context_client.create_error = BrokerError("fake failure")
yield client.provision(launch_id, deployable_type, nodes,
('subscriber', ))
ok = yield notifier.wait_for_state(states.FAILED, node_ids)
self.assertTrue(ok)
self.assertTrue(notifier.assure_record_count(1))
yield self.assertStoreNodeRecords(states.FAILED, *node_ids)
yield self.assertStoreLaunchRecord(states.FAILED, launch_id)
@defer.inlineCallbacks
def test_dump_state(self):
running_launch, running_nodes = make_launch_and_nodes(
_new_id(), 10, states.RUNNING)
yield self.store.put_launch(running_launch)
yield self.store.put_nodes(running_nodes)
pending_launch, pending_nodes = make_launch_and_nodes(
_new_id(), 3, states.PENDING)
yield self.store.put_launch(pending_launch)
yield self.store.put_nodes(pending_nodes)
running_node_ids = [node['node_id'] for node in running_nodes]
pending_node_ids = [node['node_id'] for node in pending_nodes]
all_node_ids = running_node_ids + pending_node_ids
yield self.client.dump_state(running_node_ids)
ok = yield self.notifier.wait_for_state(states.RUNNING,
nodes=running_node_ids)
self.assertTrue(ok)
self.assertEqual(len(self.notifier.nodes), len(running_nodes))
yield self.client.dump_state(pending_node_ids)
ok = yield self.notifier.wait_for_state(states.PENDING,
nodes=pending_node_ids)
self.assertTrue(ok)
self.assertEqual(len(self.notifier.nodes), len(all_node_ids))
# we should have not gotten any dupe records yet
self.assertTrue(self.notifier.assure_record_count(1))
# empty dump request should dump nothing
yield self.client.dump_state([])
self.assertTrue(self.notifier.assure_record_count(1))
@defer.inlineCallbacks
def test_dump_state_unknown_node(self):
node_ids = ["09ddd3f8-a5a5-4196-ac13-eab4d4b0c777"]
subscribers = ["hello1_subscriber"]
yield self.client.dump_state(node_ids, force_subscribe=subscribers[0])
ok = yield self.notifier.wait_for_state(states.FAILED, nodes=node_ids)
self.assertTrue(ok)
self.assertEqual(len(self.notifier.nodes), len(node_ids))
for node_id in node_ids:
ok = yield self.notifier.assure_subscribers(node_id, subscribers)
self.assertTrue(ok)
@defer.inlineCallbacks
def test_terminate(self):
launch_id = _new_id()
running_launch, running_nodes = make_launch_and_nodes(
launch_id, 10, states.RUNNING, site="fake-site1")
yield self.store.put_launch(running_launch)
yield self.store.put_nodes(running_nodes)
node_ids = [node['node_id'] for node in running_nodes]
# terminate half of the nodes then the launch as a whole
first_five = node_ids[:5]
yield self.client.terminate_nodes(first_five)
ok = yield self.notifier.wait_for_state(states.TERMINATED,
nodes=first_five)
self.assertTrue(ok)
self.assertEqual(set(first_five), set(self.notifier.nodes))
yield self.client.terminate_launches((launch_id, ))
ok = yield self.notifier.wait_for_state(states.TERMINATED,
nodes=node_ids)
self.assertTrue(ok)
self.assertEqual(set(node_ids), set(self.notifier.nodes))
# should be TERMINATING and TERMINATED record for each node
self.assertTrue(self.notifier.assure_record_count(2))
self.assertEqual(len(self.site_drivers['fake-site1'].destroyed),
len(node_ids))
@defer.inlineCallbacks
def test_terminate_all(self):
# create a ton of launches
launch_specs = [(30, 3, states.RUNNING), (50, 1, states.TERMINATED),
(80, 1, states.RUNNING)]
to_be_terminated_node_ids = []
for launchcount, nodecount, state in launch_specs:
for i in range(launchcount):
launch_id = _new_id()
launch, nodes = make_launch_and_nodes(launch_id,
nodecount,
state,
site="fake-site1")
yield self.store.put_launch(launch)
yield self.store.put_nodes(nodes)
if state < states.TERMINATED:
to_be_terminated_node_ids.extend(node["node_id"]
for node in nodes)
log.debug("Expecting %d nodes to be terminated",
len(to_be_terminated_node_ids))
yield self.client.terminate_all(rpcwait=True)
yield self.assertStoreNodeRecords(states.TERMINATED,
*to_be_terminated_node_ids)
ok = self.notifier.assure_state(states.TERMINATED,
nodes=to_be_terminated_node_ids)
self.assertTrue(ok)
self.assertEqual(set(to_be_terminated_node_ids),
set(self.notifier.nodes))
self.assertEqual(len(self.site_drivers['fake-site1'].destroyed),
len(to_be_terminated_node_ids))
@defer.inlineCallbacks
def test_query(self):
#default is non-rpc. should be None result
res = yield self.client.query()
self.assertEqual(res, None)
#returns true in RPC case
res = yield self.client.query(rpc=True)
self.assertEqual(res, True)
class ProvisionerServiceTerminateAllTest(BaseProvisionerServiceTests):
"""Tests that use a fake ProvisionerCore to test the Deferred RPC
polling mechanism of terminate_all
"""
@defer.inlineCallbacks
def setUp(self):
self.notifier = FakeProvisionerNotifier()
self.context_client = FakeContextClient()
self.store = ProvisionerStore()
self.site_drivers = {'fake-site1' : FakeNodeDriver()}
yield self._start_container()
yield self.spawn_procs()
self.fakecore = TerminateAllFakeCore()
self.patch(self.provisioner, "core", self.fakecore)
pId = yield self.procRegistry.get("provisioner")
self.client = ProvisionerClient(pid=pId)
@defer.inlineCallbacks
def tearDown(self):
yield self._shutdown_processes()
yield self._stop_container()
@defer.inlineCallbacks
def test_terminate_all_deferred(self):
"""Check the specific behavior with terminate_all_deferred.
"""
service_deferred = defer.Deferred()
self.fakecore.deferred = service_deferred
client_deferred = self.client.terminate_all(rpcwait=True, poll=0.1)
yield procutils.asleep(0.3)
# first time the core fires its Deferred, check_terminate_all still
# says there are instances. So client should not yet return
self.fakecore.all_terminated = False
self.fakecore.deferred = defer.Deferred() # set up the next Deferred
service_deferred.callback(None)
service_deferred = self.fakecore.deferred
yield procutils.asleep(0.3)
self.assertFalse(client_deferred.called)
self.assertEqual(self.fakecore.check_terminate_all_count, 1)
# now we flip terminate_all_check to True. client should return
# on next cycle
self.fakecore.all_terminated = True
service_deferred.callback(None)
yield client_deferred
@defer.inlineCallbacks
def test_terminate_all_deferred_error_retry(self):
service_deferred = defer.Deferred()
self.fakecore.deferred = service_deferred
client_deferred = self.client.terminate_all(rpcwait=True, poll=0.01, retries=3)
yield procutils.asleep(0.1)
for i in range(3):
self.assertEqual(self.fakecore.terminate_all_count, i+1)
self.fakecore.deferred = defer.Deferred()
service_deferred.errback(Exception("went bad #%d" % (i+1)))
service_deferred = self.fakecore.deferred
yield procutils.asleep(0.2)
self.assertFalse(client_deferred.called)
self.assertEqual(self.fakecore.terminate_all_count, i+2)
#this last errback should cause client_deferred to errback itself
self.fakecore.deferred = defer.Deferred()
service_deferred.errback(Exception("went bad for the last time"))
yield procutils.asleep(0.03)
try:
yield client_deferred
except Exception,e:
log.exception("Expected error, couldn't terminate all after retries: %s", e)
else:
class ProvisionerServiceTest(BaseProvisionerServiceTests):
"""Integration tests that use fake context broker and IaaS driver fixtures
"""
@defer.inlineCallbacks
def setUp(self):
self.notifier = FakeProvisionerNotifier()
self.context_client = FakeContextClient()
self.store = yield self.setup_store()
self.site_drivers = {'fake-site1' : FakeNodeDriver()}
yield self._start_container()
yield self.spawn_procs()
pId = yield self.procRegistry.get("provisioner")
self.client = ProvisionerClient(pid=pId)
@defer.inlineCallbacks
def tearDown(self):
yield self.shutdown_procs()
yield self.teardown_store()
yield self._stop_container()
def setup_store(self):
return defer.succeed(ProvisionerStore())
def teardown_store(self):
return defer.succeed(None)
@defer.inlineCallbacks
def test_provision_bad_dt(self):
client = self.client
notifier = self.notifier
worker_node_count = 3
deployable_type = 'this-doesnt-exist'
nodes = {'head-node' : FakeLaunchItem(1, 'fake-site1', 'small', None),
'worker-node' : FakeLaunchItem(worker_node_count,
'fake-site1', 'small', None)}
launch_id = _new_id()
node_ids = [node_id for node in nodes.itervalues()
for node_id in node.instance_ids]
self.assertEqual(len(node_ids), worker_node_count + 1)
yield client.provision(launch_id, deployable_type, nodes, ('subscriber',))
ok = yield notifier.wait_for_state(states.FAILED, node_ids)
self.assertTrue(ok)
self.assertTrue(notifier.assure_record_count(1))
yield self.assertStoreNodeRecords(states.FAILED, *node_ids)
yield self.assertStoreLaunchRecord(states.FAILED, launch_id)
@defer.inlineCallbacks
def test_provision_broker_error(self):
client = self.client
notifier = self.notifier
worker_node_count = 3
deployable_type = 'base-cluster'
nodes = {'head-node' : FakeLaunchItem(1, 'fake-site1', 'small', None),
'worker-node' : FakeLaunchItem(worker_node_count,
'fake-site1', 'small', None)}
launch_id = _new_id()
node_ids = [node_id for node in nodes.itervalues()
for node_id in node.instance_ids]
self.assertEqual(len(node_ids), worker_node_count + 1)
self.context_client.create_error = BrokerError("fake failure")
yield client.provision(launch_id, deployable_type, nodes, ('subscriber',))
ok = yield notifier.wait_for_state(states.FAILED, node_ids)
self.assertTrue(ok)
self.assertTrue(notifier.assure_record_count(1))
yield self.assertStoreNodeRecords(states.FAILED, *node_ids)
yield self.assertStoreLaunchRecord(states.FAILED, launch_id)
@defer.inlineCallbacks
def test_dump_state(self):
running_launch, running_nodes = make_launch_and_nodes(_new_id(), 10, states.RUNNING)
yield self.store.put_launch(running_launch)
yield self.store.put_nodes(running_nodes)
pending_launch, pending_nodes = make_launch_and_nodes(_new_id(), 3, states.PENDING)
yield self.store.put_launch(pending_launch)
yield self.store.put_nodes(pending_nodes)
running_node_ids = [node['node_id'] for node in running_nodes]
pending_node_ids = [node['node_id'] for node in pending_nodes]
all_node_ids = running_node_ids + pending_node_ids
yield self.client.dump_state(running_node_ids)
ok = yield self.notifier.wait_for_state(states.RUNNING, nodes=running_node_ids)
self.assertTrue(ok)
self.assertEqual(len(self.notifier.nodes), len(running_nodes))
yield self.client.dump_state(pending_node_ids)
ok = yield self.notifier.wait_for_state(states.PENDING, nodes=pending_node_ids)
self.assertTrue(ok)
self.assertEqual(len(self.notifier.nodes), len(all_node_ids))
# we should have not gotten any dupe records yet
self.assertTrue(self.notifier.assure_record_count(1))
# empty dump request should dump nothing
yield self.client.dump_state([])
self.assertTrue(self.notifier.assure_record_count(1))
@defer.inlineCallbacks
def test_dump_state_unknown_node(self):
node_ids = ["09ddd3f8-a5a5-4196-ac13-eab4d4b0c777"]
subscribers = ["hello1_subscriber"]
yield self.client.dump_state(node_ids, force_subscribe=subscribers[0])
ok = yield self.notifier.wait_for_state(states.FAILED, nodes=node_ids)
self.assertTrue(ok)
self.assertEqual(len(self.notifier.nodes), len(node_ids))
for node_id in node_ids:
ok = yield self.notifier.assure_subscribers(node_id, subscribers)
self.assertTrue(ok)
@defer.inlineCallbacks
def test_terminate(self):
launch_id = _new_id()
running_launch, running_nodes = make_launch_and_nodes(launch_id, 10,
states.RUNNING,
site="fake-site1")
yield self.store.put_launch(running_launch)
yield self.store.put_nodes(running_nodes)
node_ids = [node['node_id'] for node in running_nodes]
# terminate half of the nodes then the launch as a whole
first_five = node_ids[:5]
yield self.client.terminate_nodes(first_five)
ok = yield self.notifier.wait_for_state(states.TERMINATED, nodes=first_five)
self.assertTrue(ok)
self.assertEqual(set(first_five), set(self.notifier.nodes))
yield self.client.terminate_launches((launch_id,))
ok = yield self.notifier.wait_for_state(states.TERMINATED, nodes=node_ids)
self.assertTrue(ok)
self.assertEqual(set(node_ids), set(self.notifier.nodes))
# should be TERMINATING and TERMINATED record for each node
self.assertTrue(self.notifier.assure_record_count(2))
self.assertEqual(len(self.site_drivers['fake-site1'].destroyed),
len(node_ids))
@defer.inlineCallbacks
def test_terminate_all(self):
# create a ton of launches
launch_specs = [(30, 3, states.RUNNING), (50, 1, states.TERMINATED), (80, 1, states.RUNNING)]
to_be_terminated_node_ids = []
for launchcount, nodecount, state in launch_specs:
for i in range(launchcount):
launch_id = _new_id()
launch, nodes = make_launch_and_nodes(
launch_id, nodecount, state, site="fake-site1")
yield self.store.put_launch(launch)
yield self.store.put_nodes(nodes)
if state < states.TERMINATED:
to_be_terminated_node_ids.extend(node["node_id"] for node in nodes)
log.debug("Expecting %d nodes to be terminated", len(to_be_terminated_node_ids))
yield self.client.terminate_all(rpcwait=True)
yield self.assertStoreNodeRecords(states.TERMINATED, *to_be_terminated_node_ids)
ok = self.notifier.assure_state(states.TERMINATED, nodes=to_be_terminated_node_ids)
self.assertTrue(ok)
self.assertEqual(set(to_be_terminated_node_ids), set(self.notifier.nodes))
self.assertEqual(len(self.site_drivers['fake-site1'].destroyed),
len(to_be_terminated_node_ids))
@defer.inlineCallbacks
def test_query(self):
#default is non-rpc. should be None result
res = yield self.client.query()
self.assertEqual(res, None)
#returns true in RPC case
res = yield self.client.query(rpc=True)
self.assertEqual(res, True)
def slc_init(self):
scoped_name = self.get_scoped_name("system", self.svc_name)
self.scoped_name = scoped_name
queue_name_work = self.spawn_args.get("queue_name_work")
if queue_name_work:
self.queue_name_work = self.get_scoped_name(
"system", queue_name_work)
extradict = {"queue_name_work": self.queue_name_work}
cei_events.event(self.svc_name, "init_begin", extra=extradict)
yield self._make_queue(queue_name_work)
queuestat_client = QueueStatClient(self)
yield queuestat_client.watch_queue(self.queue_name_work,
self.scoped_name, 'sensor_info')
cei_events.event(self.svc_name, "queue_watched")
else:
self.worker_queue_receiver = None
self.queue_name_work = None
extradict = None
cei_events.event(self.svc_name, "init_begin", extra=extradict)
engineclass = "epu.decisionengine.impls.NpreservingEngine"
if self.spawn_args.has_key("engine_class"):
engineclass = self.spawn_args["engine_class"]
log.info("Using configured decision engine: %s" % engineclass)
else:
log.info("Using default decision engine: %s" % engineclass)
if self.spawn_args.has_key("engine_conf"):
engine_conf = self.spawn_args["engine_conf"]
if isinstance(engine_conf, str):
engine_conf = json.loads(engine_conf)
else:
engine_conf = None
if self.spawn_args.has_key("cassandra"):
cass = self.spawn_args["cassandra"]
host = cass['hostname']
username = cass['username']
password = cass['password']
port = cass['port']
keyspace = cass['keyspace']
store = CassandraControllerStore(self.svc_name, host, port,
username, password, keyspace,
CoreInstance, SensorItem)
store.initialize()
store.activate()
elif self.spawn_args.has_key('store'):
store = self.spawn_args['store']
else:
store = ControllerStore()
self.core = ControllerCore(ProvisionerClient(self),
engineclass,
scoped_name,
conf=engine_conf,
store=store)
# run state recovery and engine initialization
# this one needs to run before any messages start arriving. It pulls
# information from persistence and refreshes local caches.
yield self.core.run_recovery()
# temporarily doing this later due to a potential bug in ioncore where
# queues may not be bound before slc_init runs. This means if the
# provisioner is quck to reply to dump_state some messages may be
# missed.
reactor.callLater(1, self._delayed_init)
