class ControllerCoreStateTests(BaseControllerStateTests): """ControllerCoreState tests that only use in memory store They test things basically peripheral to actual persistence. """ def setUp(self): self.store = ControllerStore() self.state = ControllerCoreState(self.store) @defer.inlineCallbacks def test_bad_sensors(self): #badly formatted sensors shouldn't break the world bads = [dict(sesnor_id="bad", time=1, value=34), ['not','even','a','dict!'], None, 142, "this is just a string", dict(sensor_id="almost", time=1), dict(sensor_id="soclose", value=7)] for bad in bads: yield self.state.new_sensor_item(bad) @defer.inlineCallbacks def test_instance_extravars(self): """extravars get carried forward from the initial instance state (when they don't arrive in state updates) """ extravars = {'iwant': 'asandwich', 4: 'real'} launch_id, instance_id = yield self.new_instance(1, extravars=extravars) yield self.new_instance_state(launch_id, instance_id, InstanceStates.RUNNING, 2) instance = self.state.instances[instance_id] self.assertEqual(instance.instance_id, instance_id) self.assertEqual(instance.state, InstanceStates.RUNNING) self.assertEqual(instance.extravars, extravars) @defer.inlineCallbacks def test_incomplete_instance_message(self): launch_id, instance_id = yield self.new_instance(1) # now fake a response like we'd get from provisioner dump_state # when it has no knowledge of instance record = {"node_id":instance_id, "state":states.FAILED} yield self.state.new_instance_state(record, timestamp=2) instance = self.state.instances[instance_id] for k in ('instance_id', 'launch_id', 'site', 'allocation', 'state'): self.assertIn(k, instance) @defer.inlineCallbacks def test_get_engine_state(self): self.state.new_sensor_item(dict(sensor_id="s1", time=1, value="a")) self.state.new_sensor_item(dict(sensor_id="s1", time=2, value="b")) self.state.new_sensor_item(dict(sensor_id="s2", time=2, value="a")) launch_id1, instance_id1 = yield self.new_instance(1) launch_id2, instance_id2 = yield self.new_instance(1) yield self.new_instance_state(launch_id1, instance_id1, InstanceStates.RUNNING, 2) es = self.state.get_engine_state() #check instances self.assertEqual(len(es.instance_changes), 2) self.assertIn(instance_id1, es.instance_changes) self.assertIn(instance_id2, es.instance_changes) self.assertEqual(len(es.instance_changes[instance_id1]), 2) self.assertEqual(len(es.instance_changes[instance_id2]), 1) self.assertEqual(es.instances[instance_id1].state, InstanceStates.RUNNING) self.assertEqual(es.instances[instance_id2].state, InstanceStates.REQUESTING) #check sensors self.assertEqual(len(es.sensor_changes), 2) self.assertIn("s1", es.sensor_changes) self.assertIn("s2", es.sensor_changes) self.assertEqual(len(es.sensor_changes["s1"]), 2) self.assertEqual(es.sensors["s1"].value, "b") self.assertEqual(es.sensors["s2"].value, "a") # ensure that next time around there are no changes but state is same es = self.state.get_engine_state() self.assertEqual(len(es.instance_changes), 0) self.assertEqual(es.instances[instance_id1].state, InstanceStates.RUNNING) self.assertEqual(es.instances[instance_id2].state, InstanceStates.REQUESTING) self.assertEqual(len(es.sensor_changes), 0) self.assertEqual(es.sensors["s1"].value, "b") self.assertEqual(es.sensors["s2"].value, "a") @defer.inlineCallbacks def _cleared_instance_health(self, instance_state): launch_id, instance_id = yield self.new_instance(5) yield self.new_instance_state(launch_id, instance_id, InstanceStates.RUNNING, 6) yield self.state.new_instance_health(instance_id, InstanceHealthState.PROCESS_ERROR, errors=['blah']) yield self.assertInstance(instance_id, state=InstanceStates.RUNNING, health=InstanceHealthState.PROCESS_ERROR, errors=['blah']) # terminate the instance and its health state should be cleared # but error should remain, for postmortem let's say? yield self.new_instance_state(launch_id, instance_id, instance_state, 7) yield self.assertInstance(instance_id, state=instance_state, health=InstanceHealthState.UNKNOWN, errors=['blah']) inst = yield self.store.get_instance(instance_id) log.debug(inst.health) def test_terminating_cleared_instance_health(self): return self._cleared_instance_health(InstanceStates.TERMINATING) def test_terminated_cleared_instance_health(self): return self._cleared_instance_health(InstanceStates.TERMINATED) def test_failed_cleared_instance_health(self): return self._cleared_instance_health(InstanceStates.FAILED) @defer.inlineCallbacks def test_out_of_order_instance(self): launch_id, instance_id = yield self.new_instance(5) yield self.new_instance_state(launch_id, instance_id, InstanceStates.STARTED, 6) # instances cannot go back in state yield self.new_instance_state(launch_id, instance_id, InstanceStates.REQUESTED, 6) self.assertEqual(self.state.instances[instance_id].state, InstanceStates.STARTED) @defer.inlineCallbacks def test_out_of_order_sensor(self): sensor_id = "sandwich_meter" # how many sandwiches?? msg = dict(sensor_id=sensor_id, time=100, value=100) yield self.state.new_sensor_item(msg) msg = dict(sensor_id=sensor_id, time=90, value=200) yield self.state.new_sensor_item(msg) yield self.assertSensor(sensor_id, 100, 100) self.assertEqual(len(self.state.pending_sensors[sensor_id]), 2)
class ControllerStateStoreTests(BaseControllerStateTests): """ControllerCoreState tests that can use either storage implementation. Subclassed below to use cassandra. """ @defer.inlineCallbacks def setUp(self): self.store = yield self.get_store() self.state = ControllerCoreState(self.store) def get_store(self): return defer.succeed(ControllerStore()) @defer.inlineCallbacks def test_sensors(self): sensor_id = "sandwich_meter" # how many sandwiches?? msg = dict(sensor_id=sensor_id, time=1, value=100) yield self.state.new_sensor_item(msg) yield self.assertSensor(sensor_id, 1, 100) msg = dict(sensor_id=sensor_id, time=2, value=101) yield self.state.new_sensor_item(msg) yield self.assertSensor(sensor_id, 2, 101) all_sensors = yield self.store.get_sensor_ids() all_sensors = set(all_sensors) self.assertEqual(len(all_sensors), 1) self.assertIn(sensor_id, all_sensors) @defer.inlineCallbacks def test_instances(self): launch_id = str(uuid.uuid4()) instance_id = str(uuid.uuid4()) yield self.state.new_instance_launch(instance_id, launch_id, "chicago", "big", timestamp=1) yield self.assertInstance(instance_id, launch_id=launch_id, site="chicago", allocation="big", state=InstanceStates.REQUESTING, state_time=1, health=InstanceHealthState.UNKNOWN) msg = dict(node_id=instance_id, launch_id=launch_id, site="chicago", allocation="big", state=InstanceStates.STARTED) yield self.state.new_instance_state(msg, timestamp=2) yield self.assertInstance(instance_id, launch_id=launch_id, site="chicago", allocation="big", state=InstanceStates.STARTED, state_time=2, health=InstanceHealthState.UNKNOWN) # bring in a health update yield self.state.new_instance_health(instance_id, InstanceHealthState.OK, errors=['blah']) yield self.assertInstance(instance_id, launch_id=launch_id, site="chicago", allocation="big", state=InstanceStates.STARTED, state_time=2, health=InstanceHealthState.OK, errors=['blah']) # another instance state change should preserve health info msg = dict(node_id=instance_id, launch_id=launch_id, site="chicago", allocation="big", state=InstanceStates.RUNNING) yield self.state.new_instance_state(msg, timestamp=3) yield self.assertInstance(instance_id, launch_id=launch_id, site="chicago", allocation="big", state=InstanceStates.RUNNING, state_time=3, health=InstanceHealthState.OK, errors=['blah']) all_instances = yield self.store.get_instance_ids() all_instances = set(all_instances) self.assertEqual(len(all_instances), 1) self.assertIn(instance_id, all_instances) @defer.inlineCallbacks def test_recovery(self): # put some values in the store directly yield self.store.add_sensor(SensorItem("s1", 100, "s1v1")) yield self.store.add_sensor(SensorItem("s2", 100, "s2v1")) yield self.store.add_sensor(SensorItem("s1", 200, "s1v2")) d1 = dict(instance_id="i1", launch_id="l1", allocation="big", site="cleveland", state=InstanceStates.PENDING) yield self.store.add_instance(CoreInstance.from_dict(d1)) d2 = dict(instance_id="i2", launch_id="l2", allocation="big", site="cleveland", state=InstanceStates.PENDING) yield self.store.add_instance(CoreInstance.from_dict(d2)) d2['state'] = InstanceStates.RUNNING yield self.store.add_instance(CoreInstance.from_dict(d2)) # recovery should bring them into state yield self.state.recover() yield self.assertSensor("s1", 200, "s1v2") yield self.assertSensor("s2", 100, "s2v1") yield self.assertInstance("i1", launch_id="l1", allocation="big", site="cleveland", state=InstanceStates.PENDING) yield self.assertInstance("i2", launch_id="l2", allocation="big", site="cleveland", state=InstanceStates.RUNNING) @defer.inlineCallbacks def test_recover_nothing(self): #ensure recover() works when the store is empty yield self.state.recover() self.assertEqual(len(self.state.instances), 0) self.assertEqual(len(self.state.sensors), 0)