class ControllerStateStoreTests(BaseControllerStateTests):
"""ControllerCoreState tests that can use either storage implementation.
"""
def setUp(self):
self.domain = LocalDomainStore("david", "domain1", {})
def test_instances(self):
launch_id = str(uuid.uuid4())
instance_id = str(uuid.uuid4())
self.domain.new_instance_launch("dtid", instance_id, launch_id,
"chicago", "big", timestamp=1)
self.assertInstance(instance_id, launch_id=launch_id, site="chicago",
allocation="big", state=InstanceState.REQUESTING,
state_time=1, health=InstanceHealthState.UNKNOWN)
msg = dict(node_id=instance_id, launch_id=launch_id,
site="chicago", allocation="big",
state=InstanceState.STARTED)
self.domain.new_instance_state(msg, timestamp=2)
self.assertInstance(instance_id, launch_id=launch_id, site="chicago",
allocation="big", state=InstanceState.STARTED,
state_time=2, health=InstanceHealthState.UNKNOWN)
# bring in a health update
self.domain.new_instance_health(instance_id,
InstanceHealthState.OK,
errors=['blah'])
self.assertInstance(instance_id, launch_id=launch_id, site="chicago",
allocation="big", state=InstanceState.STARTED,
state_time=2, health=InstanceHealthState.OK,
errors=['blah'])
# another instance state change should preserve health info
msg = dict(node_id=instance_id, launch_id=launch_id,
site="chicago", allocation="big",
state=InstanceState.RUNNING)
self.domain.new_instance_state(msg, timestamp=3)
self.assertInstance(instance_id, launch_id=launch_id, site="chicago",
allocation="big", state=InstanceState.RUNNING,
state_time=3, health=InstanceHealthState.OK,
errors=['blah'])
all_instances = self.domain.get_instance_ids()
all_instances = set(all_instances)
self.assertEqual(len(all_instances), 1)
self.assertIn(instance_id, all_instances)
class ControllerCoreStateTests(BaseControllerStateTests):
"""ControllerCoreState tests that only use in memory store
They test things basically peripheral to actual persistence.
"""
def setUp(self):
self.domain = LocalDomainStore("david", "domain1", {})
def test_instance_extravars(self):
"""extravars get carried forward from the initial instance state
(when they don't arrive in state updates)
"""
extravars = {'iwant': 'asandwich', 4: 'real'}
launch_id, instance_id = self.new_instance(1,
extravars=extravars)
self.new_instance_state(launch_id, instance_id,
InstanceState.RUNNING, 2)
instance = self.domain.get_instance(instance_id)
self.assertEqual(instance.instance_id, instance_id)
self.assertEqual(instance.state, InstanceState.RUNNING)
self.assertEqual(instance.extravars, extravars)
def test_incomplete_instance_message(self):
launch_id, instance_id = self.new_instance(1)
# now fake a response like we'd get from provisioner dump_state
# when it has no knowledge of instance
record = {"node_id": instance_id, "state": InstanceState.FAILED}
self.domain.new_instance_state(record, timestamp=2)
instance = self.domain.get_instance(instance_id)
for k in ('instance_id', 'launch_id', 'site', 'allocation', 'state'):
self.assertIn(k, instance)
def test_get_engine_state(self):
launch_id1, instance_id1 = self.new_instance(1)
launch_id2, instance_id2 = self.new_instance(1)
self.new_instance_state(launch_id1, instance_id1, InstanceState.RUNNING, 2)
es = self.domain.get_engine_state()
# check instances
# TODO instance change tracking not supported
# self.assertEqual(len(es.instance_changes), 2)
# self.assertIn(instance_id1, es.instance_changes)
# self.assertIn(instance_id2, es.instance_changes)
# self.assertEqual(len(es.instance_changes[instance_id1]), 2)
# self.assertEqual(len(es.instance_changes[instance_id2]), 1)
self.assertEqual(es.instances[instance_id1].state, InstanceState.RUNNING)
self.assertEqual(es.instances[instance_id2].state, InstanceState.REQUESTING)
# ensure that next time around there are no changes but state is same
es = self.domain.get_engine_state()
# self.assertEqual(len(es.instance_changes), 0)
self.assertEqual(es.instances[instance_id1].state,
InstanceState.RUNNING)
self.assertEqual(es.instances[instance_id2].state,
InstanceState.REQUESTING)
def _cleared_instance_health(self, instance_state):
launch_id, instance_id = self.new_instance(5)
self.new_instance_state(launch_id, instance_id,
InstanceState.RUNNING, 6)
self.domain.new_instance_health(instance_id,
InstanceHealthState.PROCESS_ERROR, error_time=123,
errors=['blah'])
self.assertInstance(instance_id, state=InstanceState.RUNNING,
health=InstanceHealthState.PROCESS_ERROR,
error_time=123,
errors=['blah'])
# terminate the instance and its health state should be cleared
# but error should remain, for postmortem let's say?
self.new_instance_state(launch_id, instance_id,
instance_state, 7)
self.assertInstance(instance_id, state=instance_state,
health=InstanceHealthState.UNKNOWN,
error_time=123,
errors=['blah'])
inst = self.domain.get_instance(instance_id)
log.debug(inst.health)
def test_terminating_cleared_instance_health(self):
return self._cleared_instance_health(InstanceState.TERMINATING)
def test_terminated_cleared_instance_health(self):
return self._cleared_instance_health(InstanceState.TERMINATED)
def test_failed_cleared_instance_health(self):
return self._cleared_instance_health(InstanceState.FAILED)
def test_out_of_order_instance(self):
launch_id, instance_id = self.new_instance(5)
self.new_instance_state(launch_id, instance_id,
InstanceState.STARTED, 6)
# instances cannot go back in state
self.new_instance_state(launch_id, instance_id,
InstanceState.REQUESTED, 6)
self.assertEqual(self.domain.get_instance(instance_id).state,
InstanceState.STARTED)
class ControllerStateStoreTests(BaseControllerStateTests):
"""ControllerCoreState tests that can use either storage implementation.
"""
def setUp(self):
self.domain = LocalDomainStore("david", "domain1", {})
def test_instances(self):
launch_id = str(uuid.uuid4())
instance_id = str(uuid.uuid4())
self.domain.new_instance_launch("dtid", instance_id, launch_id,
"chicago", "big", timestamp=1)
self.assertInstance(instance_id, launch_id=launch_id, site="chicago",
allocation="big", state=InstanceState.REQUESTING,
state_time=1, health=InstanceHealthState.UNKNOWN)
msg = dict(node_id=instance_id, launch_id=launch_id,
site="chicago", allocation="big",
state=InstanceState.STARTED)
self.domain.new_instance_state(msg, timestamp=2)
self.assertInstance(instance_id, launch_id=launch_id, site="chicago",
allocation="big", state=InstanceState.STARTED,
state_time=2, health=InstanceHealthState.UNKNOWN)
# bring in a health update
self.domain.new_instance_health(instance_id,
InstanceHealthState.OK,
errors=['blah'])
self.assertInstance(instance_id, launch_id=launch_id, site="chicago",
allocation="big", state=InstanceState.STARTED,
state_time=2, health=InstanceHealthState.OK,
errors=['blah'])
# another instance state change should preserve health info
msg = dict(node_id=instance_id, launch_id=launch_id,
site="chicago", allocation="big",
state=InstanceState.RUNNING)
self.domain.new_instance_state(msg, timestamp=3)
self.assertInstance(instance_id, launch_id=launch_id, site="chicago",
allocation="big", state=InstanceState.RUNNING,
state_time=3, health=InstanceHealthState.OK,
errors=['blah'])
all_instances = self.domain.get_instance_ids()
all_instances = set(all_instances)
self.assertEqual(len(all_instances), 1)
self.assertIn(instance_id, all_instances)
def test_instance_update_conflict(self):
launch_id = str(uuid.uuid4())
instance_id = str(uuid.uuid4())
self.domain.new_instance_launch("dtid", instance_id, launch_id,
"chicago", "big", timestamp=1)
sneaky_msg = dict(node_id=instance_id, launch_id=launch_id,
site="chicago", allocation="big",
state=InstanceState.PENDING)
# patch in a function that sneaks in an instance record update just
# before a requested update. This simulates the case where two EPUM
# workers are competing to update the same instance.
original_update_instance = self.domain.update_instance
patch_called = threading.Event()
def patched_update_instance(*args, **kwargs):
patch_called.set()
# unpatch ourself first so we don't recurse forever
self.domain.update_instance = original_update_instance
self.domain.new_instance_state(sneaky_msg, timestamp=2)
original_update_instance(*args, **kwargs)
self.domain.update_instance = patched_update_instance
# send our "real" update. should get a conflict
msg = dict(node_id=instance_id, launch_id=launch_id,
site="chicago", allocation="big",
state=InstanceState.STARTED)
with self.assertRaises(WriteConflictError):
self.domain.new_instance_state(msg, timestamp=2)
assert patch_called.is_set()