def test_terminate_all(self): running_launch_id = _new_id() running_launch, running_nodes = make_launch_and_nodes( running_launch_id, 3, states.RUNNING) yield self.store.put_launch(running_launch) yield self.store.put_nodes(running_nodes) pending_launch_id = _new_id() pending_launch, pending_nodes = make_launch_and_nodes( pending_launch_id, 3, states.PENDING) yield self.store.put_launch(pending_launch) yield self.store.put_nodes(pending_nodes) terminated_launch_id = _new_id() terminated_launch, terminated_nodes = make_launch_and_nodes( terminated_launch_id, 3, states.TERMINATED) yield self.store.put_launch(terminated_launch) yield self.store.put_nodes(terminated_nodes) yield self.core.terminate_all() self.assertEqual(6, len(self.driver.destroyed)) all_launches = yield self.store.get_launches() self.assertEqual(3, len(all_launches)) self.assertTrue( all(l['state'] == states.TERMINATED for l in all_launches)) all_nodes = yield self.store.get_nodes() self.assertEqual(9, len(all_nodes)) self.assertTrue(all(n['state'] == states.TERMINATED for n in all_nodes)) state = yield self.core.check_terminate_all() self.assertTrue(state)
def test_terminate_all(self): running_launch_id = _new_id() running_launch, running_nodes = make_launch_and_nodes( running_launch_id, 3, states.RUNNING) yield self.store.put_launch(running_launch) yield self.store.put_nodes(running_nodes) pending_launch_id = _new_id() pending_launch, pending_nodes = make_launch_and_nodes( pending_launch_id, 3, states.PENDING) yield self.store.put_launch(pending_launch) yield self.store.put_nodes(pending_nodes) terminated_launch_id = _new_id() terminated_launch, terminated_nodes = make_launch_and_nodes( terminated_launch_id, 3, states.TERMINATED) yield self.store.put_launch(terminated_launch) yield self.store.put_nodes(terminated_nodes) yield self.core.terminate_all() self.assertEqual(6, len(self.driver.destroyed)) all_launches = yield self.store.get_launches() self.assertEqual(3, len(all_launches)) self.assertTrue(all(l['state'] == states.TERMINATED for l in all_launches)) all_nodes = yield self.store.get_nodes() self.assertEqual(9, len(all_nodes)) self.assertTrue(all(n['state'] == states.TERMINATED for n in all_nodes)) state = yield self.core.check_terminate_all() self.assertTrue(state)
def test_terminate_all(self): caller = 'asterix' running_launch_id = _new_id() running_launch, running_nodes = make_launch_and_nodes( running_launch_id, 3, states.RUNNING, caller=caller) self.store.add_launch(running_launch) for node in running_nodes: self.store.add_node(node) pending_launch_id = _new_id() pending_launch, pending_nodes = make_launch_and_nodes( pending_launch_id, 3, states.PENDING, caller=caller) self.store.add_launch(pending_launch) for node in pending_nodes: self.store.add_node(node) terminated_launch_id = _new_id() terminated_launch, terminated_nodes = make_launch_and_nodes( terminated_launch_id, 3, states.TERMINATED, caller=caller) self.store.add_launch(terminated_launch) for node in terminated_nodes: self.store.add_node(node) self.core.terminate_all() all_nodes = self.store.get_nodes() self.assertEqual(9, len(all_nodes)) self.assertTrue(all(n['state'] == states.TERMINATING or n['state'] == states.TERMINATED for n in all_nodes))
def test_dump_state(self): running_launch, running_nodes = make_launch_and_nodes(_new_id(), 10, InstanceState.RUNNING) self.store.add_launch(running_launch) for node in running_nodes: self.store.add_node(node) pending_launch, pending_nodes = make_launch_and_nodes(_new_id(), 3, InstanceState.PENDING) self.store.add_launch(pending_launch) for node in pending_nodes: self.store.add_node(node) running_node_ids = [node['node_id'] for node in running_nodes] pending_node_ids = [node['node_id'] for node in pending_nodes] all_node_ids = running_node_ids + pending_node_ids self.client.dump_state(running_node_ids) ok = self.notifier.wait_for_state(InstanceState.RUNNING, nodes=running_node_ids) self.assertTrue(ok) self.assertEqual(len(self.notifier.nodes), len(running_nodes)) self.client.dump_state(pending_node_ids) ok = self.notifier.wait_for_state(InstanceState.PENDING, nodes=pending_node_ids) self.assertTrue(ok) self.assertEqual(len(self.notifier.nodes), len(all_node_ids)) # we should have not gotten any dupe records yet self.assertTrue(self.notifier.assure_record_count(1)) # empty dump request should dump nothing self.client.dump_state([]) self.assertTrue(self.notifier.assure_record_count(1))
def test_dump_state(self): running_launch, running_nodes = make_launch_and_nodes( _new_id(), 10, states.RUNNING) yield self.store.put_launch(running_launch) yield self.store.put_nodes(running_nodes) pending_launch, pending_nodes = make_launch_and_nodes( _new_id(), 3, states.PENDING) yield self.store.put_launch(pending_launch) yield self.store.put_nodes(pending_nodes) running_node_ids = [node['node_id'] for node in running_nodes] pending_node_ids = [node['node_id'] for node in pending_nodes] all_node_ids = running_node_ids + pending_node_ids yield self.client.dump_state(running_node_ids) ok = yield self.notifier.wait_for_state(states.RUNNING, nodes=running_node_ids) self.assertTrue(ok) self.assertEqual(len(self.notifier.nodes), len(running_nodes)) yield self.client.dump_state(pending_node_ids) ok = yield self.notifier.wait_for_state(states.PENDING, nodes=pending_node_ids) self.assertTrue(ok) self.assertEqual(len(self.notifier.nodes), len(all_node_ids)) # we should have not gotten any dupe records yet self.assertTrue(self.notifier.assure_record_count(1)) # empty dump request should dump nothing yield self.client.dump_state([]) self.assertTrue(self.notifier.assure_record_count(1))
def test_terminate(self): launch_id = _new_id() running_launch, running_nodes = make_launch_and_nodes( launch_id, 10, states.RUNNING, site="fake-site1") yield self.store.put_launch(running_launch) yield self.store.put_nodes(running_nodes) node_ids = [node['node_id'] for node in running_nodes] # terminate half of the nodes then the launch as a whole first_five = node_ids[:5] yield self.client.terminate_nodes(first_five) ok = yield self.notifier.wait_for_state(states.TERMINATED, nodes=first_five) self.assertTrue(ok) self.assertEqual(set(first_five), set(self.notifier.nodes)) yield self.client.terminate_launches((launch_id, )) ok = yield self.notifier.wait_for_state(states.TERMINATED, nodes=node_ids) self.assertTrue(ok) self.assertEqual(set(node_ids), set(self.notifier.nodes)) # should be TERMINATING and TERMINATED record for each node self.assertTrue(self.notifier.assure_record_count(2)) self.assertEqual(len(self.site_drivers['fake-site1'].destroyed), len(node_ids))
def test_terminate_all(self): # create a ton of launches launch_specs = [(30, 3, states.RUNNING), (50, 1, states.TERMINATED), (80, 1, states.RUNNING)] to_be_terminated_node_ids = [] for launchcount, nodecount, state in launch_specs: for i in range(launchcount): launch_id = _new_id() launch, nodes = make_launch_and_nodes( launch_id, nodecount, state, site="fake-site1") yield self.store.put_launch(launch) yield self.store.put_nodes(nodes) if state < states.TERMINATED: to_be_terminated_node_ids.extend(node["node_id"] for node in nodes) log.debug("Expecting %d nodes to be terminated", len(to_be_terminated_node_ids)) yield self.client.terminate_all(rpcwait=True) yield self.assertStoreNodeRecords(states.TERMINATED, *to_be_terminated_node_ids) ok = self.notifier.assure_state(states.TERMINATED, nodes=to_be_terminated_node_ids) self.assertTrue(ok) self.assertEqual(set(to_be_terminated_node_ids), set(self.notifier.nodes)) self.assertEqual(len(self.site_drivers['fake-site1'].destroyed), len(to_be_terminated_node_ids))
def test_terminate(self): launch_id = _new_id() running_launch, running_nodes = make_launch_and_nodes(launch_id, 10, states.RUNNING, site="fake-site1") yield self.store.put_launch(running_launch) yield self.store.put_nodes(running_nodes) node_ids = [node['node_id'] for node in running_nodes] # terminate half of the nodes then the launch as a whole first_five = node_ids[:5] yield self.client.terminate_nodes(first_five) ok = yield self.notifier.wait_for_state(states.TERMINATED, nodes=first_five) self.assertTrue(ok) self.assertEqual(set(first_five), set(self.notifier.nodes)) yield self.client.terminate_launches((launch_id,)) ok = yield self.notifier.wait_for_state(states.TERMINATED, nodes=node_ids) self.assertTrue(ok) self.assertEqual(set(node_ids), set(self.notifier.nodes)) # should be TERMINATING and TERMINATED record for each node self.assertTrue(self.notifier.assure_record_count(2)) self.assertEqual(len(self.site_drivers['fake-site1'].destroyed), len(node_ids))
def test_describe(self): node_ids = [] for _ in range(3): launch_id = _new_id() running_launch, running_nodes = make_launch_and_nodes(launch_id, 1, InstanceState.RUNNING, site="fake-site1", caller=self.default_user) self.store.add_launch(running_launch) for node in running_nodes: self.store.add_node(node) node_ids.append(running_nodes[0]['node_id']) log.debug("requestin") all_nodes = self.client.describe_nodes() self.assertEqual(len(all_nodes), len(node_ids)) one_node = self.client.describe_nodes([node_ids[0]]) self.assertEqual(len(one_node), 1) self.assertEqual(one_node[0]['node_id'], node_ids[0])
def test_terminate_all(self): # create a ton of launches launch_specs = [(30, 3, states.RUNNING), (50, 1, states.TERMINATED), (80, 1, states.RUNNING)] to_be_terminated_node_ids = [] for launchcount, nodecount, state in launch_specs: for i in range(launchcount): launch_id = _new_id() launch, nodes = make_launch_and_nodes(launch_id, nodecount, state, site="fake-site1") yield self.store.put_launch(launch) yield self.store.put_nodes(nodes) if state < states.TERMINATED: to_be_terminated_node_ids.extend(node["node_id"] for node in nodes) log.debug("Expecting %d nodes to be terminated", len(to_be_terminated_node_ids)) yield self.client.terminate_all(rpcwait=True) yield self.assertStoreNodeRecords(states.TERMINATED, *to_be_terminated_node_ids) ok = self.notifier.assure_state(states.TERMINATED, nodes=to_be_terminated_node_ids) self.assertTrue(ok) self.assertEqual(set(to_be_terminated_node_ids), set(self.notifier.nodes)) self.assertEqual(len(self.site_drivers['fake-site1'].destroyed), len(to_be_terminated_node_ids))