class ControllerCoreControlTests(unittest.TestCase): def _config_simplest_domain_conf(self, n_preserving): """Get 'simplest' EPU conf with specified NPreserving policy """ engine_class = "epu.decisionengine.impls.simplest.SimplestEngine" general = {EPUM_CONF_ENGINE_CLASS: engine_class} health = {EPUM_CONF_HEALTH_MONITOR: False} engine = {CONF_PRESERVE_N: n_preserving} return {EPUM_CONF_GENERAL: general, EPUM_CONF_ENGINE: engine, EPUM_CONF_HEALTH: health} def setUp(self): self.provisioner = MockProvisionerClient() config = self._config_simplest_domain_conf(1) self.state = LocalDomainStore('david', "epu1", config) self.prov_vars = {"foo": "bar"} self.controller_name = "fakey" self.control = ControllerCoreControl(self.provisioner, self.state, self.prov_vars, self.controller_name) def test_configure_1(self): self.control.configure(None) self.assertEqual(self.control.prov_vars, self.prov_vars) def test_configure_2(self): self.control.configure({}) self.assertEqual(self.control.prov_vars, self.prov_vars) def test_configure_3(self): params = {PROVISIONER_VARS_KEY: {"blah": "blah"}} self.control.configure(params) self.assertEqual(self.control.prov_vars, {"blah": "blah"}) def test_launch(self): launch_id, instance_ids = self.control.launch("dt", "chicago", "small", extravars={"v1": 1}) self.assertEqual(len(instance_ids), 1) # check that right info got added to state instance_id = instance_ids[0] instance = self.state.get_instance(instance_id) self.assertEqual(instance.instance_id, instance_id) self.assertEqual(instance.launch_id, launch_id) self.assertEqual(instance.site, "chicago") self.assertEqual(instance.allocation, "small") self.assertEqual(instance.extravars, {"v1": 1}) # and provisionerclient called self.assertEqual(len(self.provisioner.launches), 1) launch = self.provisioner.launches[0] self.assertEqual(launch['launch_id'], launch_id) self.assertEqual(launch['dt'], "dt") # vars are merged result self.assertEqual(launch['vars']['foo'], "bar") self.assertEqual(launch['vars']['v1'], 1) self.assertEqual(launch['site'], "chicago") self.assertEqual(launch['allocation'], "small")
def setUp(self): self.provisioner = MockProvisionerClient() config = self._config_simplest_domain_conf(1) self.state = LocalDomainStore('david', "epu1", config) self.prov_vars = {"foo": "bar"} self.controller_name = "fakey" self.control = ControllerCoreControl(self.provisioner, self.state, self.prov_vars, self.controller_name)
class ControllerStateStoreTests(BaseControllerStateTests): """ControllerCoreState tests that can use either storage implementation. """ def setUp(self): self.domain = LocalDomainStore("david", "domain1", {}) def test_instances(self): launch_id = str(uuid.uuid4()) instance_id = str(uuid.uuid4()) self.domain.new_instance_launch("dtid", instance_id, launch_id, "chicago", "big", timestamp=1) self.assertInstance(instance_id, launch_id=launch_id, site="chicago", allocation="big", state=InstanceState.REQUESTING, state_time=1, health=InstanceHealthState.UNKNOWN) msg = dict(node_id=instance_id, launch_id=launch_id, site="chicago", allocation="big", state=InstanceState.STARTED) self.domain.new_instance_state(msg, timestamp=2) self.assertInstance(instance_id, launch_id=launch_id, site="chicago", allocation="big", state=InstanceState.STARTED, state_time=2, health=InstanceHealthState.UNKNOWN) # bring in a health update self.domain.new_instance_health(instance_id, InstanceHealthState.OK, errors=['blah']) self.assertInstance(instance_id, launch_id=launch_id, site="chicago", allocation="big", state=InstanceState.STARTED, state_time=2, health=InstanceHealthState.OK, errors=['blah']) # another instance state change should preserve health info msg = dict(node_id=instance_id, launch_id=launch_id, site="chicago", allocation="big", state=InstanceState.RUNNING) self.domain.new_instance_state(msg, timestamp=3) self.assertInstance(instance_id, launch_id=launch_id, site="chicago", allocation="big", state=InstanceState.RUNNING, state_time=3, health=InstanceHealthState.OK, errors=['blah']) all_instances = self.domain.get_instance_ids() all_instances = set(all_instances) self.assertEqual(len(all_instances), 1) self.assertIn(instance_id, all_instances)
class ControllerCoreStateTests(BaseControllerStateTests): """ControllerCoreState tests that only use in memory store They test things basically peripheral to actual persistence. """ def setUp(self): self.domain = LocalDomainStore("david", "domain1", {}) def test_instance_extravars(self): """extravars get carried forward from the initial instance state (when they don't arrive in state updates) """ extravars = {'iwant': 'asandwich', 4: 'real'} launch_id, instance_id = self.new_instance(1, extravars=extravars) self.new_instance_state(launch_id, instance_id, InstanceState.RUNNING, 2) instance = self.domain.get_instance(instance_id) self.assertEqual(instance.instance_id, instance_id) self.assertEqual(instance.state, InstanceState.RUNNING) self.assertEqual(instance.extravars, extravars) def test_incomplete_instance_message(self): launch_id, instance_id = self.new_instance(1) # now fake a response like we'd get from provisioner dump_state # when it has no knowledge of instance record = {"node_id": instance_id, "state": InstanceState.FAILED} self.domain.new_instance_state(record, timestamp=2) instance = self.domain.get_instance(instance_id) for k in ('instance_id', 'launch_id', 'site', 'allocation', 'state'): self.assertIn(k, instance) def test_get_engine_state(self): launch_id1, instance_id1 = self.new_instance(1) launch_id2, instance_id2 = self.new_instance(1) self.new_instance_state(launch_id1, instance_id1, InstanceState.RUNNING, 2) es = self.domain.get_engine_state() # check instances # TODO instance change tracking not supported # self.assertEqual(len(es.instance_changes), 2) # self.assertIn(instance_id1, es.instance_changes) # self.assertIn(instance_id2, es.instance_changes) # self.assertEqual(len(es.instance_changes[instance_id1]), 2) # self.assertEqual(len(es.instance_changes[instance_id2]), 1) self.assertEqual(es.instances[instance_id1].state, InstanceState.RUNNING) self.assertEqual(es.instances[instance_id2].state, InstanceState.REQUESTING) # ensure that next time around there are no changes but state is same es = self.domain.get_engine_state() # self.assertEqual(len(es.instance_changes), 0) self.assertEqual(es.instances[instance_id1].state, InstanceState.RUNNING) self.assertEqual(es.instances[instance_id2].state, InstanceState.REQUESTING) def _cleared_instance_health(self, instance_state): launch_id, instance_id = self.new_instance(5) self.new_instance_state(launch_id, instance_id, InstanceState.RUNNING, 6) self.domain.new_instance_health(instance_id, InstanceHealthState.PROCESS_ERROR, error_time=123, errors=['blah']) self.assertInstance(instance_id, state=InstanceState.RUNNING, health=InstanceHealthState.PROCESS_ERROR, error_time=123, errors=['blah']) # terminate the instance and its health state should be cleared # but error should remain, for postmortem let's say? self.new_instance_state(launch_id, instance_id, instance_state, 7) self.assertInstance(instance_id, state=instance_state, health=InstanceHealthState.UNKNOWN, error_time=123, errors=['blah']) inst = self.domain.get_instance(instance_id) log.debug(inst.health) def test_terminating_cleared_instance_health(self): return self._cleared_instance_health(InstanceState.TERMINATING) def test_terminated_cleared_instance_health(self): return self._cleared_instance_health(InstanceState.TERMINATED) def test_failed_cleared_instance_health(self): return self._cleared_instance_health(InstanceState.FAILED) def test_out_of_order_instance(self): launch_id, instance_id = self.new_instance(5) self.new_instance_state(launch_id, instance_id, InstanceState.STARTED, 6) # instances cannot go back in state self.new_instance_state(launch_id, instance_id, InstanceState.REQUESTED, 6) self.assertEqual(self.domain.get_instance(instance_id).state, InstanceState.STARTED)
def setUp(self): self.domain = LocalDomainStore("david", "domain1", {})
class ControllerStateStoreTests(BaseControllerStateTests): """ControllerCoreState tests that can use either storage implementation. """ def setUp(self): self.domain = LocalDomainStore("david", "domain1", {}) def test_instances(self): launch_id = str(uuid.uuid4()) instance_id = str(uuid.uuid4()) self.domain.new_instance_launch("dtid", instance_id, launch_id, "chicago", "big", timestamp=1) self.assertInstance(instance_id, launch_id=launch_id, site="chicago", allocation="big", state=InstanceState.REQUESTING, state_time=1, health=InstanceHealthState.UNKNOWN) msg = dict(node_id=instance_id, launch_id=launch_id, site="chicago", allocation="big", state=InstanceState.STARTED) self.domain.new_instance_state(msg, timestamp=2) self.assertInstance(instance_id, launch_id=launch_id, site="chicago", allocation="big", state=InstanceState.STARTED, state_time=2, health=InstanceHealthState.UNKNOWN) # bring in a health update self.domain.new_instance_health(instance_id, InstanceHealthState.OK, errors=['blah']) self.assertInstance(instance_id, launch_id=launch_id, site="chicago", allocation="big", state=InstanceState.STARTED, state_time=2, health=InstanceHealthState.OK, errors=['blah']) # another instance state change should preserve health info msg = dict(node_id=instance_id, launch_id=launch_id, site="chicago", allocation="big", state=InstanceState.RUNNING) self.domain.new_instance_state(msg, timestamp=3) self.assertInstance(instance_id, launch_id=launch_id, site="chicago", allocation="big", state=InstanceState.RUNNING, state_time=3, health=InstanceHealthState.OK, errors=['blah']) all_instances = self.domain.get_instance_ids() all_instances = set(all_instances) self.assertEqual(len(all_instances), 1) self.assertIn(instance_id, all_instances) def test_instance_update_conflict(self): launch_id = str(uuid.uuid4()) instance_id = str(uuid.uuid4()) self.domain.new_instance_launch("dtid", instance_id, launch_id, "chicago", "big", timestamp=1) sneaky_msg = dict(node_id=instance_id, launch_id=launch_id, site="chicago", allocation="big", state=InstanceState.PENDING) # patch in a function that sneaks in an instance record update just # before a requested update. This simulates the case where two EPUM # workers are competing to update the same instance. original_update_instance = self.domain.update_instance patch_called = threading.Event() def patched_update_instance(*args, **kwargs): patch_called.set() # unpatch ourself first so we don't recurse forever self.domain.update_instance = original_update_instance self.domain.new_instance_state(sneaky_msg, timestamp=2) original_update_instance(*args, **kwargs) self.domain.update_instance = patched_update_instance # send our "real" update. should get a conflict msg = dict(node_id=instance_id, launch_id=launch_id, site="chicago", allocation="big", state=InstanceState.STARTED) with self.assertRaises(WriteConflictError): self.domain.new_instance_state(msg, timestamp=2) assert patch_called.is_set()