def test_instance_health2(self):
i1 = Mock(instance_id="i1", state=InstanceStates.RUNNING,
health=InstanceHealthState.OK)
i2 = Mock(instance_id="i2", state=InstanceStates.RUNNING_FAILED,
health=InstanceHealthState.OK)
i3 = Mock(instance_id="i3", state=InstanceStates.RUNNING_FAILED,
health=InstanceHealthState.MISSING)
instances = dict(i1=i1, i2=i2, i3=i3)
es = EngineState()
es.instances = instances
healthy = es.get_healthy_instances()
self.assertEqual(healthy, [i1])
unhealthy = es.get_unhealthy_instances()
self.assertTrue(i2 in unhealthy)
self.assertTrue(i3 in unhealthy)
self.assertEqual(2, len(unhealthy))
# Should not matter if health is present or not, it's RUNNING_FAILED
i3.health = InstanceHealthState.MISSING
unhealthy = es.get_unhealthy_instances()
self.assertTrue(i2 in unhealthy)
self.assertTrue(i3 in unhealthy)
def test_whole_state(self):
state = FakeControllerState()
core = ControllerCore(self.prov_client, self.ENGINE, "controller",
state=state, conf={MONITOR_HEALTH_KEY:True})
# setup 3 instances that are "recovered", should see a dump_state call to provisioner
# for 2 of them
state.instances['i1'] = Mock(instance_id='i1', state=InstanceStates.RUNNING,
health=InstanceHealthState.OK, public_ip="i1pubip",
private_ip="i1privip", state_time=3, iaas_id="i-i1")
state.instances['i2'] = Mock(instance_id='i2', state=InstanceStates.TERMINATED,
health=InstanceHealthState.UNKNOWN, public_ip=None,
private_ip=None, state_time=4, iaas_id="i-i2")
state.instances['i3'] = Mock(instance_id='i3', state=InstanceStates.REQUESTED,
health=InstanceHealthState.UNKNOWN, public_ip=None,
private_ip=None, iaas_id=None, state_time=1)
whole_state = yield core.whole_state()
log.debug("whole_state: %s", whole_state)
self.assertEqual(whole_state['de_state'], de_states.UNKNOWN)
self.assertEqual(whole_state['de_conf_report'], None)
instances = whole_state['instances']
self.assertEqual(len(instances), 3)
i1 = instances['i1']
self.assertEqual(i1['iaas_state'], InstanceStates.RUNNING)
self.assertEqual(i1['iaas_state_time'], 3)
self.assertEqual(i1['heartbeat_state'], InstanceHealthState.OK)
self.assertEqual(i1['heartbeat_time'], -1)
self.assertEqual(i1['public_ip'], "i1pubip")
self.assertEqual(i1['private_ip'], "i1privip")
self.assertEqual(i1['iaas_id'], "i-i1")
i2 = instances['i2']
self.assertEqual(i2['iaas_state'], InstanceStates.TERMINATED)
self.assertEqual(i2['iaas_state_time'], 4)
self.assertEqual(i2['heartbeat_state'], InstanceHealthState.UNKNOWN)
self.assertEqual(i2['heartbeat_time'], -1)
self.assertEqual(i2['public_ip'], None)
self.assertEqual(i2['private_ip'], None)
self.assertEqual(i2['iaas_id'], "i-i2")
i3 = instances['i3']
self.assertEqual(i3['iaas_state'], InstanceStates.REQUESTED)
self.assertEqual(i3['iaas_state_time'], 1)
self.assertEqual(i3['heartbeat_state'], InstanceHealthState.UNKNOWN)
self.assertEqual(i3['heartbeat_time'], -1)
self.assertEqual(i3['public_ip'], None)
self.assertEqual(i3['private_ip'], None)
self.assertEqual(i3['iaas_id'], None)
def test_instances(self):
i1 = [Mock(instance_id="i1", state=state)
for state in (InstanceStates.REQUESTING,
InstanceStates.REQUESTED,
InstanceStates.PENDING,
InstanceStates.RUNNING)]
i2 = [Mock(instance_id="i2", state=state)
for state in (InstanceStates.REQUESTING,
InstanceStates.REQUESTED,
InstanceStates.FAILED)]
i3 = [Mock(instance_id="i3", state=state)
for state in InstanceStates.REQUESTING, InstanceStates.PENDING]
changes = dict(i1=i1, i2=i2, i3=i3)
instances = dict(i1=i1[-1], i2=i2[-1], i3=i3[-1])
es = EngineState()
es.instance_changes = changes
es.instances = instances
self.assertEqual(es.get_instance("i1").state, InstanceStates.RUNNING)
self.assertEqual(es.get_instance("i2").state, InstanceStates.FAILED)
self.assertEqual(es.get_instance("i3").state, InstanceStates.PENDING)
self.assertEqual(es.get_instance("i4"), None) # there is no i4
self.assertEqual(len(es.get_instance_changes("i1")), 4)
self.assertEqual(len(es.get_instance_changes("i2")), 3)
self.assertEqual(len(es.get_instance_changes("i3")), 2)
self.assertEqual(es.get_instance_changes("i4"), [])
all_changes = es.get_instance_changes()
changeset = set((change.instance_id, change.state) for change in all_changes)
for item in itertools.chain(i1, i2, i3):
self.assertIn((item.instance_id, item.state), changeset)
failed = es.get_instances_by_state(InstanceStates.FAILED)
self.assertEqual(len(failed), 1)
self.assertEqual(failed[0].instance_id, "i2")
self.assertEqual(failed[0].state, InstanceStates.FAILED)
pending2running = es.get_instances_by_state(InstanceStates.PENDING,
InstanceStates.RUNNING)
self.assertEqual(len(pending2running), 2)
ids = (pending2running[0].instance_id, pending2running[1].instance_id)
self.assertIn("i1", ids)
self.assertIn("i3", ids)
pending = es.get_pending_instances()
self.assertEqual(len(pending), 1)
self.assertEqual(pending[0].instance_id, "i3")
def test_update_node_ip_info(self):
node = dict(public_ip=None)
iaas_node = Mock(public_ip=None, private_ip=None)
update_node_ip_info(node, iaas_node)
self.assertEqual(node['public_ip'], None)
self.assertEqual(node['private_ip'], None)
iaas_node = Mock(public_ip=["pub1"], private_ip=["priv1"])
update_node_ip_info(node, iaas_node)
self.assertEqual(node['public_ip'], "pub1")
self.assertEqual(node['private_ip'], "priv1")
iaas_node = Mock(public_ip=[], private_ip=[])
update_node_ip_info(node, iaas_node)
self.assertEqual(node['public_ip'], "pub1")
self.assertEqual(node['private_ip'], "priv1")
def query(self, uri):
self.queried_uris.append(uri)
if self.query_error:
return defer.fail(self.query_error)
if uri in self.uri_query_error:
return defer.fail(self.uri_query_error[uri])
response = Mock(nodes=self.nodes, expected_count=self.expected_count,
complete=self.complete, error=self.error)
return defer.succeed(response)
def test_initialize_with_instance_recovery(self):
state = FakeControllerState()
core = ControllerCore(self.prov_client, self.ENGINE, "controller",
state=state)
# setup 3 instances that are "recovered", should see a dump_state call to provisioner
# for 2 of them
state.instances['i1'] = Mock(instance_id='i1', state=InstanceStates.RUNNING)
state.instances['i2'] = Mock(instance_id='i2', state=InstanceStates.TERMINATED)
state.instances['i3'] = Mock(instance_id='i3', state=InstanceStates.PENDING)
yield core.run_recovery()
yield core.run_initialize()
self.assertEqual(state.recover_count, 1)
self.assertEqual(core.engine.initialize_count, 1)
self.assertEqual(len(self.prov_client.dump_state_reqs), 1)
node_ids = set(self.prov_client.dump_state_reqs[0])
self.assertEqual(node_ids, set(('i1', 'i3')))
def test_instance_health(self):
i1 = Mock(instance_id="i1", state=InstanceStates.RUNNING,
health=InstanceHealthState.OK)
i2 = Mock(instance_id="i2", state=InstanceStates.FAILED,
health=InstanceHealthState.OK)
i3 = Mock(instance_id="i3", state=InstanceStates.TERMINATED,
health=InstanceHealthState.MISSING)
instances = dict(i1=i1, i2=i2, i3=i3)
es = EngineState()
es.instances = instances
healthy = es.get_healthy_instances()
self.assertEqual(healthy, [i1])
unhealthy = es.get_unhealthy_instances()
self.assertFalse(unhealthy)
i1.health = InstanceHealthState.MISSING
healthy = es.get_healthy_instances()
self.assertFalse(healthy)
unhealthy = es.get_unhealthy_instances()
self.assertEqual(unhealthy, [i1])
def _one_fake_ctx_node_ok(ip, hostname, pubkey):
identity = Mock(ip=ip, hostname=hostname, pubkey=pubkey)
return Mock(ok_occurred=True, error_occurred=False, identities=[identity])
