def setUp(self):
self.provisioner = FakeProvisionerClient()
self.state = ControllerCoreState(ControllerStore())
self.prov_vars = {"foo" : "bar"}
self.controller_name = "fakey"
self.control = ControllerCoreControl(self.provisioner, self.state,
self.prov_vars,
self.controller_name)
def setUp(self):
self.store = ControllerStore()
self.state = ControllerCoreState(self.store)
def setUp(self):
self.store = yield self.get_store()
self.state = ControllerCoreState(self.store)
class ControllerCoreStateTests(BaseControllerStateTests):
"""ControllerCoreState tests that only use in memory store
They test things basically peripheral to actual persistence.
"""
def setUp(self):
self.store = ControllerStore()
self.state = ControllerCoreState(self.store)
@defer.inlineCallbacks
def test_bad_sensors(self):
#badly formatted sensors shouldn't break the world
bads = [dict(sesnor_id="bad", time=1, value=34),
['not','even','a','dict!'],
None,
142,
"this is just a string",
dict(sensor_id="almost", time=1),
dict(sensor_id="soclose", value=7)]
for bad in bads:
yield self.state.new_sensor_item(bad)
@defer.inlineCallbacks
def test_instance_extravars(self):
"""extravars get carried forward from the initial instance state
(when they don't arrive in state updates)
"""
extravars = {'iwant': 'asandwich', 4: 'real'}
launch_id, instance_id = yield self.new_instance(1,
extravars=extravars)
yield self.new_instance_state(launch_id, instance_id,
InstanceStates.RUNNING, 2)
instance = self.state.instances[instance_id]
self.assertEqual(instance.instance_id, instance_id)
self.assertEqual(instance.state, InstanceStates.RUNNING)
self.assertEqual(instance.extravars, extravars)
@defer.inlineCallbacks
def test_incomplete_instance_message(self):
launch_id, instance_id = yield self.new_instance(1)
# now fake a response like we'd get from provisioner dump_state
# when it has no knowledge of instance
record = {"node_id":instance_id, "state":states.FAILED}
yield self.state.new_instance_state(record, timestamp=2)
instance = self.state.instances[instance_id]
for k in ('instance_id', 'launch_id', 'site', 'allocation', 'state'):
self.assertIn(k, instance)
@defer.inlineCallbacks
def test_get_engine_state(self):
self.state.new_sensor_item(dict(sensor_id="s1", time=1, value="a"))
self.state.new_sensor_item(dict(sensor_id="s1", time=2, value="b"))
self.state.new_sensor_item(dict(sensor_id="s2", time=2, value="a"))
launch_id1, instance_id1 = yield self.new_instance(1)
launch_id2, instance_id2 = yield self.new_instance(1)
yield self.new_instance_state(launch_id1, instance_id1, InstanceStates.RUNNING, 2)
es = self.state.get_engine_state()
#check instances
self.assertEqual(len(es.instance_changes), 2)
self.assertIn(instance_id1, es.instance_changes)
self.assertIn(instance_id2, es.instance_changes)
self.assertEqual(len(es.instance_changes[instance_id1]), 2)
self.assertEqual(len(es.instance_changes[instance_id2]), 1)
self.assertEqual(es.instances[instance_id1].state, InstanceStates.RUNNING)
self.assertEqual(es.instances[instance_id2].state, InstanceStates.REQUESTING)
#check sensors
self.assertEqual(len(es.sensor_changes), 2)
self.assertIn("s1", es.sensor_changes)
self.assertIn("s2", es.sensor_changes)
self.assertEqual(len(es.sensor_changes["s1"]), 2)
self.assertEqual(es.sensors["s1"].value, "b")
self.assertEqual(es.sensors["s2"].value, "a")
# ensure that next time around there are no changes but state is same
es = self.state.get_engine_state()
self.assertEqual(len(es.instance_changes), 0)
self.assertEqual(es.instances[instance_id1].state,
InstanceStates.RUNNING)
self.assertEqual(es.instances[instance_id2].state,
InstanceStates.REQUESTING)
self.assertEqual(len(es.sensor_changes), 0)
self.assertEqual(es.sensors["s1"].value, "b")
self.assertEqual(es.sensors["s2"].value, "a")
@defer.inlineCallbacks
def _cleared_instance_health(self, instance_state):
launch_id, instance_id = yield self.new_instance(5)
yield self.new_instance_state(launch_id, instance_id,
InstanceStates.RUNNING, 6)
yield self.state.new_instance_health(instance_id,
InstanceHealthState.PROCESS_ERROR,
errors=['blah'])
yield self.assertInstance(instance_id, state=InstanceStates.RUNNING,
health=InstanceHealthState.PROCESS_ERROR,
errors=['blah'])
# terminate the instance and its health state should be cleared
# but error should remain, for postmortem let's say?
yield self.new_instance_state(launch_id, instance_id,
instance_state, 7)
yield self.assertInstance(instance_id, state=instance_state,
health=InstanceHealthState.UNKNOWN,
errors=['blah'])
inst = yield self.store.get_instance(instance_id)
log.debug(inst.health)
def test_terminating_cleared_instance_health(self):
return self._cleared_instance_health(InstanceStates.TERMINATING)
def test_terminated_cleared_instance_health(self):
return self._cleared_instance_health(InstanceStates.TERMINATED)
def test_failed_cleared_instance_health(self):
return self._cleared_instance_health(InstanceStates.FAILED)
@defer.inlineCallbacks
def test_out_of_order_instance(self):
launch_id, instance_id = yield self.new_instance(5)
yield self.new_instance_state(launch_id, instance_id,
InstanceStates.STARTED, 6)
# instances cannot go back in state
yield self.new_instance_state(launch_id, instance_id,
InstanceStates.REQUESTED, 6)
self.assertEqual(self.state.instances[instance_id].state,
InstanceStates.STARTED)
@defer.inlineCallbacks
def test_out_of_order_sensor(self):
sensor_id = "sandwich_meter" # how many sandwiches??
msg = dict(sensor_id=sensor_id, time=100, value=100)
yield self.state.new_sensor_item(msg)
msg = dict(sensor_id=sensor_id, time=90, value=200)
yield self.state.new_sensor_item(msg)
yield self.assertSensor(sensor_id, 100, 100)
self.assertEqual(len(self.state.pending_sensors[sensor_id]), 2)
class ControllerStateStoreTests(BaseControllerStateTests):
"""ControllerCoreState tests that can use either storage implementation.
Subclassed below to use cassandra.
"""
@defer.inlineCallbacks
def setUp(self):
self.store = yield self.get_store()
self.state = ControllerCoreState(self.store)
def get_store(self):
return defer.succeed(ControllerStore())
@defer.inlineCallbacks
def test_sensors(self):
sensor_id = "sandwich_meter" # how many sandwiches??
msg = dict(sensor_id=sensor_id, time=1, value=100)
yield self.state.new_sensor_item(msg)
yield self.assertSensor(sensor_id, 1, 100)
msg = dict(sensor_id=sensor_id, time=2, value=101)
yield self.state.new_sensor_item(msg)
yield self.assertSensor(sensor_id, 2, 101)
all_sensors = yield self.store.get_sensor_ids()
all_sensors = set(all_sensors)
self.assertEqual(len(all_sensors), 1)
self.assertIn(sensor_id, all_sensors)
@defer.inlineCallbacks
def test_instances(self):
launch_id = str(uuid.uuid4())
instance_id = str(uuid.uuid4())
yield self.state.new_instance_launch(instance_id, launch_id,
"chicago", "big", timestamp=1)
yield self.assertInstance(instance_id, launch_id=launch_id, site="chicago",
allocation="big", state=InstanceStates.REQUESTING,
state_time=1, health=InstanceHealthState.UNKNOWN)
msg = dict(node_id=instance_id, launch_id=launch_id,
site="chicago", allocation="big",
state=InstanceStates.STARTED)
yield self.state.new_instance_state(msg, timestamp=2)
yield self.assertInstance(instance_id, launch_id=launch_id, site="chicago",
allocation="big", state=InstanceStates.STARTED,
state_time=2, health=InstanceHealthState.UNKNOWN)
# bring in a health update
yield self.state.new_instance_health(instance_id,
InstanceHealthState.OK,
errors=['blah'])
yield self.assertInstance(instance_id, launch_id=launch_id, site="chicago",
allocation="big", state=InstanceStates.STARTED,
state_time=2, health=InstanceHealthState.OK,
errors=['blah'])
# another instance state change should preserve health info
msg = dict(node_id=instance_id, launch_id=launch_id,
site="chicago", allocation="big",
state=InstanceStates.RUNNING)
yield self.state.new_instance_state(msg, timestamp=3)
yield self.assertInstance(instance_id, launch_id=launch_id, site="chicago",
allocation="big", state=InstanceStates.RUNNING,
state_time=3, health=InstanceHealthState.OK,
errors=['blah'])
all_instances = yield self.store.get_instance_ids()
all_instances = set(all_instances)
self.assertEqual(len(all_instances), 1)
self.assertIn(instance_id, all_instances)
@defer.inlineCallbacks
def test_recovery(self):
# put some values in the store directly
yield self.store.add_sensor(SensorItem("s1", 100, "s1v1"))
yield self.store.add_sensor(SensorItem("s2", 100, "s2v1"))
yield self.store.add_sensor(SensorItem("s1", 200, "s1v2"))
d1 = dict(instance_id="i1", launch_id="l1", allocation="big",
site="cleveland", state=InstanceStates.PENDING)
yield self.store.add_instance(CoreInstance.from_dict(d1))
d2 = dict(instance_id="i2", launch_id="l2", allocation="big",
site="cleveland", state=InstanceStates.PENDING)
yield self.store.add_instance(CoreInstance.from_dict(d2))
d2['state'] = InstanceStates.RUNNING
yield self.store.add_instance(CoreInstance.from_dict(d2))
# recovery should bring them into state
yield self.state.recover()
yield self.assertSensor("s1", 200, "s1v2")
yield self.assertSensor("s2", 100, "s2v1")
yield self.assertInstance("i1", launch_id="l1", allocation="big",
site="cleveland", state=InstanceStates.PENDING)
yield self.assertInstance("i2", launch_id="l2", allocation="big",
site="cleveland", state=InstanceStates.RUNNING)
@defer.inlineCallbacks
def test_recover_nothing(self):
#ensure recover() works when the store is empty
yield self.state.recover()
self.assertEqual(len(self.state.instances), 0)
self.assertEqual(len(self.state.sensors), 0)
