def setUp(self):
self.provisioner = FakeProvisionerClient()
self.state = ControllerCoreState(ControllerStore())
self.prov_vars = {"foo" : "bar"}
self.controller_name = "fakey"
self.control = ControllerCoreControl(self.provisioner, self.state,
self.prov_vars,
self.controller_name)
def __init__(self,
provisioner_client,
engineclass,
controller_name,
conf=None,
state=None,
store=None):
if state:
self.state = state
else:
self.state = ControllerCoreState(store or ControllerStore())
prov_vars = None
health_kwargs = None
if conf:
if conf.has_key(PROVISIONER_VARS_KEY):
prov_vars = conf[PROVISIONER_VARS_KEY]
if conf.get(MONITOR_HEALTH_KEY):
health_kwargs = {}
if HEALTH_BOOT_KEY in conf:
health_kwargs['boot_seconds'] = conf[HEALTH_BOOT_KEY]
if HEALTH_MISSING_KEY in conf:
health_kwargs['missing_seconds'] = conf[HEALTH_MISSING_KEY]
if HEALTH_ZOMBIE_KEY in conf:
health_kwargs['zombie_seconds'] = conf[HEALTH_ZOMBIE_KEY]
self.conf = conf
if health_kwargs is not None:
self.health_monitor = HealthMonitor(self.state, **health_kwargs)
else:
self.health_monitor = None
# There can only ever be one 'reconfigure' or 'decide' engine call run
# at ANY time. The 'decide' call is triggered via timed looping call
# and 'reconfigure' is triggered asynchronously at any moment.
self.busy = defer.DeferredSemaphore(1)
self.provisioner_client = provisioner_client
health_not_checked = self.health_monitor is None
self.control = ControllerCoreControl(
provisioner_client,
self.state,
prov_vars,
controller_name,
health_not_checked=health_not_checked)
self.engine = EngineLoader().load(engineclass)
self.control_loop = None
def test_reconfigure_recovery(self):
# test that engine config is correctly using reconfigured engine values
# in subsequent starts of the controller. Strategy is:
# 1. Start up a controller with some config
# 2. Call reconfigure with some more config
# 3. Kill controller and restart it, verify engine got the merged
# config.
store = ControllerStore()
spawnargs = {
'engine_class':
'epu.epucontroller.test.test_controller_core.FakeEngine',
'engine_conf': {
'a': 'a1',
'b': 'b1'
},
'servicename': 'TestEPUController',
'store': store
}
controller = EPUControllerService(spawnargs=spawnargs)
self.controller = controller
yield self._spawn_process(controller)
client = EPUControllerClient(targetname='TestEPUController')
yield client.attach()
yield procutils.asleep(1.01)
yield client.reconfigure_rpc({'a': 'a2', 'c': 'c1'})
yield procutils.asleep(1.01)
controller.terminate()
controller = EPUControllerService(spawnargs=spawnargs)
self.controller = controller
yield self._spawn_process(controller)
yield procutils.asleep(1.01)
self.assertEqual(controller.core.engine.initialize_conf, {
'a': 'a2',
'b': 'b1',
'c': 'c1'
})
def setUp(self):
self.store = ControllerStore()
self.state = ControllerCoreState(self.store)
class ControllerCoreStateTests(BaseControllerStateTests):
"""ControllerCoreState tests that only use in memory store
They test things basically peripheral to actual persistence.
"""
def setUp(self):
self.store = ControllerStore()
self.state = ControllerCoreState(self.store)
@defer.inlineCallbacks
def test_bad_sensors(self):
#badly formatted sensors shouldn't break the world
bads = [dict(sesnor_id="bad", time=1, value=34),
['not','even','a','dict!'],
None,
142,
"this is just a string",
dict(sensor_id="almost", time=1),
dict(sensor_id="soclose", value=7)]
for bad in bads:
yield self.state.new_sensor_item(bad)
@defer.inlineCallbacks
def test_instance_extravars(self):
"""extravars get carried forward from the initial instance state
(when they don't arrive in state updates)
"""
extravars = {'iwant': 'asandwich', 4: 'real'}
launch_id, instance_id = yield self.new_instance(1,
extravars=extravars)
yield self.new_instance_state(launch_id, instance_id,
InstanceStates.RUNNING, 2)
instance = self.state.instances[instance_id]
self.assertEqual(instance.instance_id, instance_id)
self.assertEqual(instance.state, InstanceStates.RUNNING)
self.assertEqual(instance.extravars, extravars)
@defer.inlineCallbacks
def test_incomplete_instance_message(self):
launch_id, instance_id = yield self.new_instance(1)
# now fake a response like we'd get from provisioner dump_state
# when it has no knowledge of instance
record = {"node_id":instance_id, "state":states.FAILED}
yield self.state.new_instance_state(record, timestamp=2)
instance = self.state.instances[instance_id]
for k in ('instance_id', 'launch_id', 'site', 'allocation', 'state'):
self.assertIn(k, instance)
@defer.inlineCallbacks
def test_get_engine_state(self):
self.state.new_sensor_item(dict(sensor_id="s1", time=1, value="a"))
self.state.new_sensor_item(dict(sensor_id="s1", time=2, value="b"))
self.state.new_sensor_item(dict(sensor_id="s2", time=2, value="a"))
launch_id1, instance_id1 = yield self.new_instance(1)
launch_id2, instance_id2 = yield self.new_instance(1)
yield self.new_instance_state(launch_id1, instance_id1, InstanceStates.RUNNING, 2)
es = self.state.get_engine_state()
#check instances
self.assertEqual(len(es.instance_changes), 2)
self.assertIn(instance_id1, es.instance_changes)
self.assertIn(instance_id2, es.instance_changes)
self.assertEqual(len(es.instance_changes[instance_id1]), 2)
self.assertEqual(len(es.instance_changes[instance_id2]), 1)
self.assertEqual(es.instances[instance_id1].state, InstanceStates.RUNNING)
self.assertEqual(es.instances[instance_id2].state, InstanceStates.REQUESTING)
#check sensors
self.assertEqual(len(es.sensor_changes), 2)
self.assertIn("s1", es.sensor_changes)
self.assertIn("s2", es.sensor_changes)
self.assertEqual(len(es.sensor_changes["s1"]), 2)
self.assertEqual(es.sensors["s1"].value, "b")
self.assertEqual(es.sensors["s2"].value, "a")
# ensure that next time around there are no changes but state is same
es = self.state.get_engine_state()
self.assertEqual(len(es.instance_changes), 0)
self.assertEqual(es.instances[instance_id1].state,
InstanceStates.RUNNING)
self.assertEqual(es.instances[instance_id2].state,
InstanceStates.REQUESTING)
self.assertEqual(len(es.sensor_changes), 0)
self.assertEqual(es.sensors["s1"].value, "b")
self.assertEqual(es.sensors["s2"].value, "a")
@defer.inlineCallbacks
def _cleared_instance_health(self, instance_state):
launch_id, instance_id = yield self.new_instance(5)
yield self.new_instance_state(launch_id, instance_id,
InstanceStates.RUNNING, 6)
yield self.state.new_instance_health(instance_id,
InstanceHealthState.PROCESS_ERROR,
errors=['blah'])
yield self.assertInstance(instance_id, state=InstanceStates.RUNNING,
health=InstanceHealthState.PROCESS_ERROR,
errors=['blah'])
# terminate the instance and its health state should be cleared
# but error should remain, for postmortem let's say?
yield self.new_instance_state(launch_id, instance_id,
instance_state, 7)
yield self.assertInstance(instance_id, state=instance_state,
health=InstanceHealthState.UNKNOWN,
errors=['blah'])
inst = yield self.store.get_instance(instance_id)
log.debug(inst.health)
def test_terminating_cleared_instance_health(self):
return self._cleared_instance_health(InstanceStates.TERMINATING)
def test_terminated_cleared_instance_health(self):
return self._cleared_instance_health(InstanceStates.TERMINATED)
def test_failed_cleared_instance_health(self):
return self._cleared_instance_health(InstanceStates.FAILED)
@defer.inlineCallbacks
def test_out_of_order_instance(self):
launch_id, instance_id = yield self.new_instance(5)
yield self.new_instance_state(launch_id, instance_id,
InstanceStates.STARTED, 6)
# instances cannot go back in state
yield self.new_instance_state(launch_id, instance_id,
InstanceStates.REQUESTED, 6)
self.assertEqual(self.state.instances[instance_id].state,
InstanceStates.STARTED)
@defer.inlineCallbacks
def test_out_of_order_sensor(self):
sensor_id = "sandwich_meter" # how many sandwiches??
msg = dict(sensor_id=sensor_id, time=100, value=100)
yield self.state.new_sensor_item(msg)
msg = dict(sensor_id=sensor_id, time=90, value=200)
yield self.state.new_sensor_item(msg)
yield self.assertSensor(sensor_id, 100, 100)
self.assertEqual(len(self.state.pending_sensors[sensor_id]), 2)
def get_store(self):
return defer.succeed(ControllerStore())
def setUp(self):
self.state = FakeState(ControllerStore())
self.monitor = None
def setUp(self):
self.store = ControllerStore()
class ControllerStoreTests(TestCase):
def setUp(self):
self.store = ControllerStore()
@defer.inlineCallbacks
def test_config(self):
empty = yield self.store.get_config()
self.assertIsInstance(empty, dict)
self.assertFalse(empty)
empty = yield self.store.get_config(keys=('not','real', 'keys'))
self.assertIsInstance(empty, dict)
self.assertFalse(empty)
yield self.store.add_config({'a_string' : 'thisisastring',
'a_list' : [1,2,3], 'a_number' : 1.23})
cfg = yield self.store.get_config(keys=['a_string'])
self.assertEqual(cfg, {'a_string' : 'thisisastring'})
cfg = yield self.store.get_config()
self.assertEqual(cfg, {'a_string' : 'thisisastring',
'a_list' : [1,2,3], 'a_number' : 1.23})
yield self.store.add_config({'a_dict' : {"akey": {'fpp' : 'bar'}, "blah" : 5},
"a_list" : [4,5,6]})
cfg = yield self.store.get_config()
self.assertEqual(cfg, {'a_string' : 'thisisastring',
'a_list' : [4,5,6], 'a_number' : 1.23,
'a_dict' : {"akey": {'fpp' : 'bar'}, "blah" : 5}})
cfg = yield self.store.get_config(keys=('a_list', 'a_number'))
self.assertEqual(cfg, {'a_list' : [4,5,6], 'a_number' : 1.23})
@defer.inlineCallbacks
def test_instances_put_get_3(self):
yield self._instances_put_get(3)
@defer.inlineCallbacks
def test_instances_put_get_100(self):
yield self._instances_put_get(100)
@defer.inlineCallbacks
def test_instances_put_get_301(self):
yield self._instances_put_get(301)
@defer.inlineCallbacks
def _instances_put_get(self, count):
instances = []
instance_ids = set()
for i in range(count):
instance = CoreInstance(instance_id=str(uuid.uuid4()), launch_id=str(uuid.uuid4()),
site="Chicago", allocation="small", state="Illinois")
instances.append(instance)
instance_ids.add(instance.instance_id)
yield self.store.add_instance(instance)
found_ids = yield self.store.get_instance_ids()
found_ids = set(found_ids)
log.debug("Put %d instances, got %d instance IDs", count, len(found_ids))
self.assertEqual(len(found_ids), len(instance_ids))
self.assertEqual(found_ids, instance_ids)
# could go on to verify each instance record
@defer.inlineCallbacks
def test_sensors_put_get_3(self):
yield self._sensors_put_get(3)
@defer.inlineCallbacks
def test_sensors_put_get_100(self):
yield self._sensors_put_get(100)
@defer.inlineCallbacks
def test_sensors_put_get_301(self):
yield self._sensors_put_get(301)
@defer.inlineCallbacks
def _sensors_put_get(self, count):
sensors = []
sensor_ids = set()
for i in range(count):
sensor = SensorItem(str(uuid.uuid4()), i, str(i))
sensors.append(sensor)
sensor_ids.add(sensor.sensor_id)
yield self.store.add_sensor(sensor)
found_ids = yield self.store.get_sensor_ids()
found_ids = set(found_ids)
log.debug("Put %d sensors, got %d sensor IDs", count, len(found_ids))
self.assertEqual(len(found_ids), len(sensor_ids))
self.assertEqual(found_ids, sensor_ids)
def setUp(self):
self.store = ControllerStore()
class ControllerStoreTests(TestCase):
def setUp(self):
self.store = ControllerStore()
@defer.inlineCallbacks
def test_config(self):
empty = yield self.store.get_config()
self.assertIsInstance(empty, dict)
self.assertFalse(empty)
empty = yield self.store.get_config(keys=('not', 'real', 'keys'))
self.assertIsInstance(empty, dict)
self.assertFalse(empty)
yield self.store.add_config({
'a_string': 'thisisastring',
'a_list': [1, 2, 3],
'a_number': 1.23
})
cfg = yield self.store.get_config(keys=['a_string'])
self.assertEqual(cfg, {'a_string': 'thisisastring'})
cfg = yield self.store.get_config()
self.assertEqual(cfg, {
'a_string': 'thisisastring',
'a_list': [1, 2, 3],
'a_number': 1.23
})
yield self.store.add_config({
'a_dict': {
"akey": {
'fpp': 'bar'
},
"blah": 5
},
"a_list": [4, 5, 6]
})
cfg = yield self.store.get_config()
self.assertEqual(
cfg, {
'a_string': 'thisisastring',
'a_list': [4, 5, 6],
'a_number': 1.23,
'a_dict': {
"akey": {
'fpp': 'bar'
},
"blah": 5
}
})
cfg = yield self.store.get_config(keys=('a_list', 'a_number'))
self.assertEqual(cfg, {'a_list': [4, 5, 6], 'a_number': 1.23})
@defer.inlineCallbacks
def test_instances_put_get_3(self):
yield self._instances_put_get(3)
@defer.inlineCallbacks
def test_instances_put_get_100(self):
yield self._instances_put_get(100)
@defer.inlineCallbacks
def test_instances_put_get_301(self):
yield self._instances_put_get(301)
@defer.inlineCallbacks
def _instances_put_get(self, count):
instances = []
instance_ids = set()
for i in range(count):
instance = CoreInstance(instance_id=str(uuid.uuid4()),
launch_id=str(uuid.uuid4()),
site="Chicago",
allocation="small",
state="Illinois")
instances.append(instance)
instance_ids.add(instance.instance_id)
yield self.store.add_instance(instance)
found_ids = yield self.store.get_instance_ids()
found_ids = set(found_ids)
log.debug("Put %d instances, got %d instance IDs", count,
len(found_ids))
self.assertEqual(len(found_ids), len(instance_ids))
self.assertEqual(found_ids, instance_ids)
# could go on to verify each instance record
@defer.inlineCallbacks
def test_sensors_put_get_3(self):
yield self._sensors_put_get(3)
@defer.inlineCallbacks
def test_sensors_put_get_100(self):
yield self._sensors_put_get(100)
@defer.inlineCallbacks
def test_sensors_put_get_301(self):
yield self._sensors_put_get(301)
@defer.inlineCallbacks
def _sensors_put_get(self, count):
sensors = []
sensor_ids = set()
for i in range(count):
sensor = SensorItem(str(uuid.uuid4()), i, str(i))
sensors.append(sensor)
sensor_ids.add(sensor.sensor_id)
yield self.store.add_sensor(sensor)
found_ids = yield self.store.get_sensor_ids()
found_ids = set(found_ids)
log.debug("Put %d sensors, got %d sensor IDs", count, len(found_ids))
self.assertEqual(len(found_ids), len(sensor_ids))
self.assertEqual(found_ids, sensor_ids)
def slc_init(self):
scoped_name = self.get_scoped_name("system", self.svc_name)
self.scoped_name = scoped_name
queue_name_work = self.spawn_args.get("queue_name_work")
if queue_name_work:
self.queue_name_work = self.get_scoped_name(
"system", queue_name_work)
extradict = {"queue_name_work": self.queue_name_work}
cei_events.event(self.svc_name, "init_begin", extra=extradict)
yield self._make_queue(queue_name_work)
queuestat_client = QueueStatClient(self)
yield queuestat_client.watch_queue(self.queue_name_work,
self.scoped_name, 'sensor_info')
cei_events.event(self.svc_name, "queue_watched")
else:
self.worker_queue_receiver = None
self.queue_name_work = None
extradict = None
cei_events.event(self.svc_name, "init_begin", extra=extradict)
engineclass = "epu.decisionengine.impls.NpreservingEngine"
if self.spawn_args.has_key("engine_class"):
engineclass = self.spawn_args["engine_class"]
log.info("Using configured decision engine: %s" % engineclass)
else:
log.info("Using default decision engine: %s" % engineclass)
if self.spawn_args.has_key("engine_conf"):
engine_conf = self.spawn_args["engine_conf"]
if isinstance(engine_conf, str):
engine_conf = json.loads(engine_conf)
else:
engine_conf = None
if self.spawn_args.has_key("cassandra"):
cass = self.spawn_args["cassandra"]
host = cass['hostname']
username = cass['username']
password = cass['password']
port = cass['port']
keyspace = cass['keyspace']
store = CassandraControllerStore(self.svc_name, host, port,
username, password, keyspace,
CoreInstance, SensorItem)
store.initialize()
store.activate()
elif self.spawn_args.has_key('store'):
store = self.spawn_args['store']
else:
store = ControllerStore()
self.core = ControllerCore(ProvisionerClient(self),
engineclass,
scoped_name,
conf=engine_conf,
store=store)
# run state recovery and engine initialization
# this one needs to run before any messages start arriving. It pulls
# information from persistence and refreshes local caches.
yield self.core.run_recovery()
# temporarily doing this later due to a potential bug in ioncore where
# queues may not be bound before slc_init runs. This means if the
# provisioner is quck to reply to dump_state some messages may be
# missed.
reactor.callLater(1, self._delayed_init)