def setUp(self): self.provisioner = FakeProvisionerClient() self.state = ControllerCoreState(ControllerStore()) self.prov_vars = {"foo" : "bar"} self.controller_name = "fakey" self.control = ControllerCoreControl(self.provisioner, self.state, self.prov_vars, self.controller_name)
def __init__(self, provisioner_client, engineclass, controller_name, conf=None, state=None, store=None): if state: self.state = state else: self.state = ControllerCoreState(store or ControllerStore()) prov_vars = None health_kwargs = None if conf: if conf.has_key(PROVISIONER_VARS_KEY): prov_vars = conf[PROVISIONER_VARS_KEY] if conf.get(MONITOR_HEALTH_KEY): health_kwargs = {} if HEALTH_BOOT_KEY in conf: health_kwargs['boot_seconds'] = conf[HEALTH_BOOT_KEY] if HEALTH_MISSING_KEY in conf: health_kwargs['missing_seconds'] = conf[HEALTH_MISSING_KEY] if HEALTH_ZOMBIE_KEY in conf: health_kwargs['zombie_seconds'] = conf[HEALTH_ZOMBIE_KEY] self.conf = conf if health_kwargs is not None: self.health_monitor = HealthMonitor(self.state, **health_kwargs) else: self.health_monitor = None # There can only ever be one 'reconfigure' or 'decide' engine call run # at ANY time. The 'decide' call is triggered via timed looping call # and 'reconfigure' is triggered asynchronously at any moment. self.busy = defer.DeferredSemaphore(1) self.provisioner_client = provisioner_client health_not_checked = self.health_monitor is None self.control = ControllerCoreControl( provisioner_client, self.state, prov_vars, controller_name, health_not_checked=health_not_checked) self.engine = EngineLoader().load(engineclass) self.control_loop = None
def test_reconfigure_recovery(self): # test that engine config is correctly using reconfigured engine values # in subsequent starts of the controller. Strategy is: # 1. Start up a controller with some config # 2. Call reconfigure with some more config # 3. Kill controller and restart it, verify engine got the merged # config. store = ControllerStore() spawnargs = { 'engine_class': 'epu.epucontroller.test.test_controller_core.FakeEngine', 'engine_conf': { 'a': 'a1', 'b': 'b1' }, 'servicename': 'TestEPUController', 'store': store } controller = EPUControllerService(spawnargs=spawnargs) self.controller = controller yield self._spawn_process(controller) client = EPUControllerClient(targetname='TestEPUController') yield client.attach() yield procutils.asleep(1.01) yield client.reconfigure_rpc({'a': 'a2', 'c': 'c1'}) yield procutils.asleep(1.01) controller.terminate() controller = EPUControllerService(spawnargs=spawnargs) self.controller = controller yield self._spawn_process(controller) yield procutils.asleep(1.01) self.assertEqual(controller.core.engine.initialize_conf, { 'a': 'a2', 'b': 'b1', 'c': 'c1' })
def setUp(self): self.store = ControllerStore() self.state = ControllerCoreState(self.store)
class ControllerCoreStateTests(BaseControllerStateTests): """ControllerCoreState tests that only use in memory store They test things basically peripheral to actual persistence. """ def setUp(self): self.store = ControllerStore() self.state = ControllerCoreState(self.store) @defer.inlineCallbacks def test_bad_sensors(self): #badly formatted sensors shouldn't break the world bads = [dict(sesnor_id="bad", time=1, value=34), ['not','even','a','dict!'], None, 142, "this is just a string", dict(sensor_id="almost", time=1), dict(sensor_id="soclose", value=7)] for bad in bads: yield self.state.new_sensor_item(bad) @defer.inlineCallbacks def test_instance_extravars(self): """extravars get carried forward from the initial instance state (when they don't arrive in state updates) """ extravars = {'iwant': 'asandwich', 4: 'real'} launch_id, instance_id = yield self.new_instance(1, extravars=extravars) yield self.new_instance_state(launch_id, instance_id, InstanceStates.RUNNING, 2) instance = self.state.instances[instance_id] self.assertEqual(instance.instance_id, instance_id) self.assertEqual(instance.state, InstanceStates.RUNNING) self.assertEqual(instance.extravars, extravars) @defer.inlineCallbacks def test_incomplete_instance_message(self): launch_id, instance_id = yield self.new_instance(1) # now fake a response like we'd get from provisioner dump_state # when it has no knowledge of instance record = {"node_id":instance_id, "state":states.FAILED} yield self.state.new_instance_state(record, timestamp=2) instance = self.state.instances[instance_id] for k in ('instance_id', 'launch_id', 'site', 'allocation', 'state'): self.assertIn(k, instance) @defer.inlineCallbacks def test_get_engine_state(self): self.state.new_sensor_item(dict(sensor_id="s1", time=1, value="a")) self.state.new_sensor_item(dict(sensor_id="s1", time=2, value="b")) self.state.new_sensor_item(dict(sensor_id="s2", time=2, value="a")) launch_id1, instance_id1 = yield self.new_instance(1) launch_id2, instance_id2 = yield self.new_instance(1) yield self.new_instance_state(launch_id1, instance_id1, InstanceStates.RUNNING, 2) es = self.state.get_engine_state() #check instances self.assertEqual(len(es.instance_changes), 2) self.assertIn(instance_id1, es.instance_changes) self.assertIn(instance_id2, es.instance_changes) self.assertEqual(len(es.instance_changes[instance_id1]), 2) self.assertEqual(len(es.instance_changes[instance_id2]), 1) self.assertEqual(es.instances[instance_id1].state, InstanceStates.RUNNING) self.assertEqual(es.instances[instance_id2].state, InstanceStates.REQUESTING) #check sensors self.assertEqual(len(es.sensor_changes), 2) self.assertIn("s1", es.sensor_changes) self.assertIn("s2", es.sensor_changes) self.assertEqual(len(es.sensor_changes["s1"]), 2) self.assertEqual(es.sensors["s1"].value, "b") self.assertEqual(es.sensors["s2"].value, "a") # ensure that next time around there are no changes but state is same es = self.state.get_engine_state() self.assertEqual(len(es.instance_changes), 0) self.assertEqual(es.instances[instance_id1].state, InstanceStates.RUNNING) self.assertEqual(es.instances[instance_id2].state, InstanceStates.REQUESTING) self.assertEqual(len(es.sensor_changes), 0) self.assertEqual(es.sensors["s1"].value, "b") self.assertEqual(es.sensors["s2"].value, "a") @defer.inlineCallbacks def _cleared_instance_health(self, instance_state): launch_id, instance_id = yield self.new_instance(5) yield self.new_instance_state(launch_id, instance_id, InstanceStates.RUNNING, 6) yield self.state.new_instance_health(instance_id, InstanceHealthState.PROCESS_ERROR, errors=['blah']) yield self.assertInstance(instance_id, state=InstanceStates.RUNNING, health=InstanceHealthState.PROCESS_ERROR, errors=['blah']) # terminate the instance and its health state should be cleared # but error should remain, for postmortem let's say? yield self.new_instance_state(launch_id, instance_id, instance_state, 7) yield self.assertInstance(instance_id, state=instance_state, health=InstanceHealthState.UNKNOWN, errors=['blah']) inst = yield self.store.get_instance(instance_id) log.debug(inst.health) def test_terminating_cleared_instance_health(self): return self._cleared_instance_health(InstanceStates.TERMINATING) def test_terminated_cleared_instance_health(self): return self._cleared_instance_health(InstanceStates.TERMINATED) def test_failed_cleared_instance_health(self): return self._cleared_instance_health(InstanceStates.FAILED) @defer.inlineCallbacks def test_out_of_order_instance(self): launch_id, instance_id = yield self.new_instance(5) yield self.new_instance_state(launch_id, instance_id, InstanceStates.STARTED, 6) # instances cannot go back in state yield self.new_instance_state(launch_id, instance_id, InstanceStates.REQUESTED, 6) self.assertEqual(self.state.instances[instance_id].state, InstanceStates.STARTED) @defer.inlineCallbacks def test_out_of_order_sensor(self): sensor_id = "sandwich_meter" # how many sandwiches?? msg = dict(sensor_id=sensor_id, time=100, value=100) yield self.state.new_sensor_item(msg) msg = dict(sensor_id=sensor_id, time=90, value=200) yield self.state.new_sensor_item(msg) yield self.assertSensor(sensor_id, 100, 100) self.assertEqual(len(self.state.pending_sensors[sensor_id]), 2)
def get_store(self): return defer.succeed(ControllerStore())
def setUp(self): self.state = FakeState(ControllerStore()) self.monitor = None
def setUp(self): self.store = ControllerStore()
class ControllerStoreTests(TestCase): def setUp(self): self.store = ControllerStore() @defer.inlineCallbacks def test_config(self): empty = yield self.store.get_config() self.assertIsInstance(empty, dict) self.assertFalse(empty) empty = yield self.store.get_config(keys=('not','real', 'keys')) self.assertIsInstance(empty, dict) self.assertFalse(empty) yield self.store.add_config({'a_string' : 'thisisastring', 'a_list' : [1,2,3], 'a_number' : 1.23}) cfg = yield self.store.get_config(keys=['a_string']) self.assertEqual(cfg, {'a_string' : 'thisisastring'}) cfg = yield self.store.get_config() self.assertEqual(cfg, {'a_string' : 'thisisastring', 'a_list' : [1,2,3], 'a_number' : 1.23}) yield self.store.add_config({'a_dict' : {"akey": {'fpp' : 'bar'}, "blah" : 5}, "a_list" : [4,5,6]}) cfg = yield self.store.get_config() self.assertEqual(cfg, {'a_string' : 'thisisastring', 'a_list' : [4,5,6], 'a_number' : 1.23, 'a_dict' : {"akey": {'fpp' : 'bar'}, "blah" : 5}}) cfg = yield self.store.get_config(keys=('a_list', 'a_number')) self.assertEqual(cfg, {'a_list' : [4,5,6], 'a_number' : 1.23}) @defer.inlineCallbacks def test_instances_put_get_3(self): yield self._instances_put_get(3) @defer.inlineCallbacks def test_instances_put_get_100(self): yield self._instances_put_get(100) @defer.inlineCallbacks def test_instances_put_get_301(self): yield self._instances_put_get(301) @defer.inlineCallbacks def _instances_put_get(self, count): instances = [] instance_ids = set() for i in range(count): instance = CoreInstance(instance_id=str(uuid.uuid4()), launch_id=str(uuid.uuid4()), site="Chicago", allocation="small", state="Illinois") instances.append(instance) instance_ids.add(instance.instance_id) yield self.store.add_instance(instance) found_ids = yield self.store.get_instance_ids() found_ids = set(found_ids) log.debug("Put %d instances, got %d instance IDs", count, len(found_ids)) self.assertEqual(len(found_ids), len(instance_ids)) self.assertEqual(found_ids, instance_ids) # could go on to verify each instance record @defer.inlineCallbacks def test_sensors_put_get_3(self): yield self._sensors_put_get(3) @defer.inlineCallbacks def test_sensors_put_get_100(self): yield self._sensors_put_get(100) @defer.inlineCallbacks def test_sensors_put_get_301(self): yield self._sensors_put_get(301) @defer.inlineCallbacks def _sensors_put_get(self, count): sensors = [] sensor_ids = set() for i in range(count): sensor = SensorItem(str(uuid.uuid4()), i, str(i)) sensors.append(sensor) sensor_ids.add(sensor.sensor_id) yield self.store.add_sensor(sensor) found_ids = yield self.store.get_sensor_ids() found_ids = set(found_ids) log.debug("Put %d sensors, got %d sensor IDs", count, len(found_ids)) self.assertEqual(len(found_ids), len(sensor_ids)) self.assertEqual(found_ids, sensor_ids)
class ControllerStoreTests(TestCase): def setUp(self): self.store = ControllerStore() @defer.inlineCallbacks def test_config(self): empty = yield self.store.get_config() self.assertIsInstance(empty, dict) self.assertFalse(empty) empty = yield self.store.get_config(keys=('not', 'real', 'keys')) self.assertIsInstance(empty, dict) self.assertFalse(empty) yield self.store.add_config({ 'a_string': 'thisisastring', 'a_list': [1, 2, 3], 'a_number': 1.23 }) cfg = yield self.store.get_config(keys=['a_string']) self.assertEqual(cfg, {'a_string': 'thisisastring'}) cfg = yield self.store.get_config() self.assertEqual(cfg, { 'a_string': 'thisisastring', 'a_list': [1, 2, 3], 'a_number': 1.23 }) yield self.store.add_config({ 'a_dict': { "akey": { 'fpp': 'bar' }, "blah": 5 }, "a_list": [4, 5, 6] }) cfg = yield self.store.get_config() self.assertEqual( cfg, { 'a_string': 'thisisastring', 'a_list': [4, 5, 6], 'a_number': 1.23, 'a_dict': { "akey": { 'fpp': 'bar' }, "blah": 5 } }) cfg = yield self.store.get_config(keys=('a_list', 'a_number')) self.assertEqual(cfg, {'a_list': [4, 5, 6], 'a_number': 1.23}) @defer.inlineCallbacks def test_instances_put_get_3(self): yield self._instances_put_get(3) @defer.inlineCallbacks def test_instances_put_get_100(self): yield self._instances_put_get(100) @defer.inlineCallbacks def test_instances_put_get_301(self): yield self._instances_put_get(301) @defer.inlineCallbacks def _instances_put_get(self, count): instances = [] instance_ids = set() for i in range(count): instance = CoreInstance(instance_id=str(uuid.uuid4()), launch_id=str(uuid.uuid4()), site="Chicago", allocation="small", state="Illinois") instances.append(instance) instance_ids.add(instance.instance_id) yield self.store.add_instance(instance) found_ids = yield self.store.get_instance_ids() found_ids = set(found_ids) log.debug("Put %d instances, got %d instance IDs", count, len(found_ids)) self.assertEqual(len(found_ids), len(instance_ids)) self.assertEqual(found_ids, instance_ids) # could go on to verify each instance record @defer.inlineCallbacks def test_sensors_put_get_3(self): yield self._sensors_put_get(3) @defer.inlineCallbacks def test_sensors_put_get_100(self): yield self._sensors_put_get(100) @defer.inlineCallbacks def test_sensors_put_get_301(self): yield self._sensors_put_get(301) @defer.inlineCallbacks def _sensors_put_get(self, count): sensors = [] sensor_ids = set() for i in range(count): sensor = SensorItem(str(uuid.uuid4()), i, str(i)) sensors.append(sensor) sensor_ids.add(sensor.sensor_id) yield self.store.add_sensor(sensor) found_ids = yield self.store.get_sensor_ids() found_ids = set(found_ids) log.debug("Put %d sensors, got %d sensor IDs", count, len(found_ids)) self.assertEqual(len(found_ids), len(sensor_ids)) self.assertEqual(found_ids, sensor_ids)
def slc_init(self): scoped_name = self.get_scoped_name("system", self.svc_name) self.scoped_name = scoped_name queue_name_work = self.spawn_args.get("queue_name_work") if queue_name_work: self.queue_name_work = self.get_scoped_name( "system", queue_name_work) extradict = {"queue_name_work": self.queue_name_work} cei_events.event(self.svc_name, "init_begin", extra=extradict) yield self._make_queue(queue_name_work) queuestat_client = QueueStatClient(self) yield queuestat_client.watch_queue(self.queue_name_work, self.scoped_name, 'sensor_info') cei_events.event(self.svc_name, "queue_watched") else: self.worker_queue_receiver = None self.queue_name_work = None extradict = None cei_events.event(self.svc_name, "init_begin", extra=extradict) engineclass = "epu.decisionengine.impls.NpreservingEngine" if self.spawn_args.has_key("engine_class"): engineclass = self.spawn_args["engine_class"] log.info("Using configured decision engine: %s" % engineclass) else: log.info("Using default decision engine: %s" % engineclass) if self.spawn_args.has_key("engine_conf"): engine_conf = self.spawn_args["engine_conf"] if isinstance(engine_conf, str): engine_conf = json.loads(engine_conf) else: engine_conf = None if self.spawn_args.has_key("cassandra"): cass = self.spawn_args["cassandra"] host = cass['hostname'] username = cass['username'] password = cass['password'] port = cass['port'] keyspace = cass['keyspace'] store = CassandraControllerStore(self.svc_name, host, port, username, password, keyspace, CoreInstance, SensorItem) store.initialize() store.activate() elif self.spawn_args.has_key('store'): store = self.spawn_args['store'] else: store = ControllerStore() self.core = ControllerCore(ProvisionerClient(self), engineclass, scoped_name, conf=engine_conf, store=store) # run state recovery and engine initialization # this one needs to run before any messages start arriving. It pulls # information from persistence and refreshes local caches. yield self.core.run_recovery() # temporarily doing this later due to a potential bug in ioncore where # queues may not be bound before slc_init runs. This means if the # provisioner is quck to reply to dump_state some messages may be # missed. reactor.callLater(1, self._delayed_init)