def setUp(self): yield super(HookSchedulerTest, self).setUp() self.client = self.get_zookeeper_client() self.unit_relation = self.mocker.mock() self.executions = [] self.service = yield self.add_service_from_charm("wordpress") self.scheduler = HookScheduler(self.client, self.collect_executor, self.unit_relation, "", unit_name="wordpress/0") self.log_stream = self.capture_logging("hook.scheduler", level=logging.DEBUG)
def setUp(self): yield super(HookSchedulerTest, self).setUp() self.client = self.get_zookeeper_client() self.unit_relation = self.mocker.mock() self.executions = [] self.service = yield self.add_service_from_charm("wordpress") self.scheduler = HookScheduler(self.client, self.collect_executor, self.unit_relation, "", unit_name="wordpress/0") self.log_stream = self.capture_logging( "hook.scheduler", level=logging.DEBUG)
def __init__(self, client, unit_name, unit_relation, relation_name, unit_path, executor): self._client = client self._unit_path = unit_path self._relation_name = relation_name self._unit_relation = unit_relation self._executor = executor self._run_lock = DeferredLock() self._log = logging.getLogger("unit.relation.lifecycle") self._error_handler = None self._scheduler = HookScheduler(client, self._execute_change_hook, self._unit_relation, self._relation_name, unit_name=unit_name) self._watcher = None
def __init__(self, client, unit_name, unit_relation, relation_ident, unit_dir, state_dir, executor): self._client = client self._unit_dir = unit_dir self._relation_ident = relation_ident self._relation_name = relation_ident.split(":")[0] self._unit_relation = unit_relation self._unit_name = unit_name self._executor = executor self._run_lock = DeferredLock() self._log = logging.getLogger("unit.relation.lifecycle") self._error_handler = None schedule_path = os.path.join( state_dir, "%s.schedule" % unit_relation.internal_relation_id) self._scheduler = HookScheduler( client, self._execute_change_hook, self._unit_relation, self._relation_ident, unit_name, schedule_path) self._watcher = None
class UnitRelationLifecycle(object): """Unit Relation Lifcycle management. Provides for watching related units in a relation, and executing hooks in response to changes. The lifecycle is driven by the workflow. The Unit relation lifecycle glues together a number of components. It controls a watcher that recieves watch events from zookeeper, and it controls a hook scheduler which gets fed those events. When the scheduler wants to execute a hook, the executor is called with the hook path and the hook invoker. **Relation hook invocation do not maintain global order or determinism across relations**. They only maintain ordering and determinism within a relation. A shared scheduler across relations would be needed to maintain such behavior. See docs/source/internals/unit-workflow-lifecycle.rst for a brief discussion of some of the more interesting implementation decisions. """ def __init__(self, client, unit_name, unit_relation, relation_ident, unit_dir, state_dir, executor): self._client = client self._unit_dir = unit_dir self._relation_ident = relation_ident self._relation_name = relation_ident.split(":")[0] self._unit_relation = unit_relation self._unit_name = unit_name self._executor = executor self._run_lock = DeferredLock() self._log = logging.getLogger("unit.relation.lifecycle") self._error_handler = None schedule_path = os.path.join( state_dir, "%s.schedule" % unit_relation.internal_relation_id) self._scheduler = HookScheduler( client, self._execute_change_hook, self._unit_relation, self._relation_ident, unit_name, schedule_path) self._watcher = None @property def watching(self): """Are we queuing up hook executions in response to state changes?""" return self._watcher and self._watcher.running @property def executing(self): """Are we currently dequeuing and executing any queued hooks?""" return self._scheduler.running def set_hook_error_handler(self, handler): """Set an error handler to be invoked if a hook errors. The handler should accept two parameters, the RelationChange that triggered the hook, and the exception instance. """ self._error_handler = handler @inlineCallbacks def start(self, start_watches=True, start_scheduler=True): """Start watching related units and executing change hooks. :param bool start_watches: True to start relation watches :param bool start_scheduler: True to run the scheduler and actually react to any changes delivered by the watcher """ yield self._run_lock.acquire() try: # Start the hook execution scheduler. if start_scheduler and not self.executing: self._scheduler.run() # Create a watcher if we don't have one yet. if self._watcher is None: self._watcher = yield self._unit_relation.watch_related_units( self._scheduler.cb_change_members, self._scheduler.cb_change_settings) # And start the watcher. if start_watches and not self.watching: yield self._watcher.start() finally: self._run_lock.release() self._log.debug( "started relation:%s lifecycle", self._relation_name) @inlineCallbacks def stop(self, stop_watches=True): """Stop executing relation change hooks; maybe stop watching changes. :param bool stop_watches: True to stop watches as well as scheduler (which will prevent changes from being detected and queued, as well as stopping them being executed). """ yield self._run_lock.acquire() try: if stop_watches and self.watching: self._watcher.stop() if self._scheduler.running: self._scheduler.stop() finally: yield self._run_lock.release() self._log.debug("stopped relation:%s lifecycle", self._relation_name) @inlineCallbacks def depart(self): """Inform the charm that the service has departed the relation. """ self._log.debug("depart relation lifecycle") unit_id = self._unit_relation.internal_unit_id context = DepartedRelationHookContext( self._client, self._unit_name, unit_id, self._relation_name, self._unit_relation.internal_relation_id) change = RelationChange(self._relation_ident, "departed", "") invoker = self._get_invoker(context, change) hook_name = "%s-relation-broken" % self._relation_name yield self._execute_hook(invoker, hook_name, change) def _get_invoker(self, context, change): socket_path = os.path.join(self._unit_dir, HOOK_SOCKET_FILE) return RelationInvoker( context, change, "constant", socket_path, self._unit_dir, hook_log) def _execute_change_hook(self, context, change): """Invoked by the contained HookScheduler, to execute a hook. We utilize the HookExecutor to execute the hook, if an error occurs, it will be reraised, unless an error handler is specified see ``set_hook_error_handler``. """ if change.change_type == "departed": hook_name = "%s-relation-departed" % self._relation_name elif change.change_type == "joined": hook_name = "%s-relation-joined" % self._relation_name else: hook_name = "%s-relation-changed" % self._relation_name invoker = self._get_invoker(context, change) return self._execute_hook(invoker, hook_name, change) @inlineCallbacks def _execute_hook(self, invoker, hook_name, change): hook_path = os.path.join( self._unit_dir, "charm", "hooks", hook_name) yield self._run_lock.acquire() self._log.debug("Executing hook %s", hook_name) try: yield self._executor(invoker, hook_path) except Exception, e: # We can't hold the run lock when we invoke the error # handler, or we get a deadlock if the handler # manipulates the lifecycle. yield self._run_lock.release() self._log.warn("Error in %s hook: %s", hook_name, e) if not self._error_handler: raise self._log.info( "Invoked error handler for %s hook", hook_name) yield self._error_handler(change, e) returnValue(False) else:
class HookSchedulerTest(ServiceStateManagerTestBase): @inlineCallbacks def setUp(self): yield super(HookSchedulerTest, self).setUp() self.client = self.get_zookeeper_client() self.unit_relation = self.mocker.mock() self.executions = [] self.service = yield self.add_service_from_charm("wordpress") self.scheduler = HookScheduler(self.client, self.collect_executor, self.unit_relation, "", unit_name="wordpress/0") self.log_stream = self.capture_logging("hook.scheduler", level=logging.DEBUG) def collect_executor(self, context, change): self.executions.append((context, change)) # Event reduction/coalescing cases def test_reduce_removed_added(self): """ A remove event for a node followed by an add event, results in a modify event. """ self.scheduler.notify_change(old_units=["u-1"], new_units=[]) self.scheduler.notify_change(old_units=[], new_units=["u-1"]) self.scheduler.run() self.assertEqual(len(self.executions), 1) self.assertEqual(self.executions[0][1].change_type, "modified") output = ("relation change old:['u-1'], new:[], modified:()", "relation change old:[], new:['u-1'], modified:()", "start", "executing hook for u-1:modified\n") self.assertEqual(self.log_stream.getvalue(), "\n".join(output)) def test_reduce_modify_remove_add(self): """A modify, remove, add event for a node results in a modify. An extra validation of the previous test. """ self.scheduler.notify_change(modified=["u-1"]) self.scheduler.notify_change(old_units=["u-1"], new_units=[]) self.scheduler.notify_change(old_units=[], new_units=["u-1"]) self.scheduler.run() self.assertEqual(len(self.executions), 1) self.assertEqual(self.executions[0][1].change_type, "modified") def test_reduce_add_modify(self): """An add and modify event for a node are coalesced to an add.""" self.scheduler.notify_change(old_units=[], new_units=["u-1"]) self.scheduler.notify_change(modified=["u-1"]) self.scheduler.run() self.assertEqual(len(self.executions), 1) self.assertEqual(self.executions[0][1].change_type, "joined") def test_reduce_add_remove(self): """an add followed by a removal results in a noop.""" self.scheduler.notify_change(old_units=[], new_units=["u-1"]) self.scheduler.notify_change(old_units=["u-1"], new_units=[]) self.scheduler.run() self.assertEqual(len(self.executions), 0) def test_reduce_modify_remove(self): """Modifying and then removing a node, results in just the removal.""" self.scheduler.notify_change(old_units=["u-1"], new_units=["u-1"], modified=["u-1"]) self.scheduler.notify_change(old_units=["u-1"], new_units=[]) self.scheduler.run() self.assertEqual(len(self.executions), 1) self.assertEqual(self.executions[0][1].change_type, "departed") def test_reduce_modify_modify(self): """Multiple modifies get coalesced to a single modify.""" # simulate normal startup, the first notify will always be the existing # membership set. self.scheduler.notify_change(old_units=[], new_units=["u-1"]) self.scheduler.run() self.scheduler.stop() self.assertEqual(len(self.executions), 1) # Now continue the modify/modify reduction. self.scheduler.notify_change(modified=["u-1"]) self.scheduler.notify_change(modified=["u-1"]) self.scheduler.notify_change(modified=["u-1"]) self.scheduler.run() self.assertEqual(len(self.executions), 2) self.assertEqual(self.executions[1][1].change_type, "modified") # Other stuff. @inlineCallbacks def test_start_stop(self): d = self.scheduler.run() # starting multiple times results in an error self.assertFailure(self.scheduler.run(), AssertionError) self.scheduler.stop() yield d # stopping multiple times is not an error yield self.scheduler.stop() @inlineCallbacks def test_membership_visibility_per_change(self): """Hooks are executed against changes, those changes are associated to a temporal timestamp, however the changes are scheduled for execution, and the state/time of the world may have advanced, to present a logically consistent view, we try to gaurantee at a minimum, that hooks will always see the membership of a relations it was at the time of their associated change. """ self.scheduler.notify_change(old_units=[], new_units=["u-1", "u-2"]) self.scheduler.notify_change(old_units=["u-1", "u-2"], new_units=["u-2", "u-3"]) self.scheduler.notify_change(modified=["u-2"]) self.scheduler.run() self.scheduler.stop() # only two reduced events, u-2, u-3 add self.assertEqual(len(self.executions), 2) # Now the first execution (u-2 add) should only see members # from the time of its change, not the current members. However # since u-1 has been subsequently removed, it no longer retains # an entry in the membership list. change_members = yield self.executions[0][0].get_members() self.assertEqual(change_members, ["u-2"]) self.scheduler.notify_change(modified=["u-2"]) self.scheduler.notify_change(old_units=["u-2", "u-3"], new_units=["u-2"]) self.scheduler.run() self.assertEqual(len(self.executions), 4) self.assertEqual(self.executions[2][1].change_type, "modified") # Verify modify events see the correct membership. change_members = yield self.executions[2][0].get_members() self.assertEqual(change_members, ["u-2", "u-3"]) @inlineCallbacks def test_membership_visibility_with_change(self): """We express a stronger guarantee of the above, namely that a hook wont see any 'active' members in a membership list, that it hasn't previously been given a notify of before. """ self.scheduler.notify_change(old_units=["u-1", "u-2"], new_units=["u-2", "u-3", "u-4"], modified=["u-2"]) self.scheduler.run() self.scheduler.stop() # add for u-3, u-4, remove for u-1, modify for u-2 self.assertEqual(len(self.executions), 4) # Verify members for each change. self.assertEqual(self.executions[0][1].change_type, "joined") members = yield self.executions[0][0].get_members() self.assertEqual(members, ["u-1", "u-2", "u-3"]) self.assertEqual(self.executions[1][1].change_type, "joined") members = yield self.executions[1][0].get_members() self.assertEqual(members, ["u-1", "u-2", "u-3", "u-4"]) self.assertEqual(self.executions[2][1].change_type, "departed") members = yield self.executions[2][0].get_members() self.assertEqual(members, ["u-2", "u-3", "u-4"]) self.assertEqual(self.executions[3][1].change_type, "modified") members = yield self.executions[2][0].get_members() self.assertEqual(members, ["u-2", "u-3", "u-4"]) context = yield self.scheduler.get_hook_context( RelationChange("", "", "")) self.assertEqual((yield context.get_members()), ["u-2", "u-3", "u-4"]) @inlineCallbacks def test_get_relation_change_empty(self): """Retrieving a hook context, is possible even if no no hooks has previously been fired (Empty membership).""" context = yield self.scheduler.get_hook_context( RelationChange("", "", "")) members = yield context.get_members() self.assertEqual(members, [])
def test_start_stop_start(self): """Stop values should only be honored if the scheduler is stopped. """ waits = [Deferred(), succeed(True), succeed(True), succeed(True)] results = [] @inlineCallbacks def executor(context, change): res = yield waits[len(results)] results.append((context, change)) returnValue(res) scheduler = HookScheduler(self.client, executor, self.unit_relation, "", "wordpress/0", self.state_file) # Start the scheduler d = scheduler.run() # Now queue up some changes. scheduler.cb_change_members([], ["u-1"]) scheduler.cb_change_members(["u-1"], ["u-1", "u-2"]) # Stop the scheduler scheduler.stop() yield d self.assertFalse(scheduler.running) # Finish the hook execution waits[0].callback(True) d = scheduler.run() self.assertTrue(scheduler.running) # More changes scheduler.cb_change_settings([("u-1", 1)]) scheduler.cb_change_settings([("u-2", 1)]) # Scheduler should still be running. print self._debug_scheduler() print [(r[1].change_type, r[1].unit_name) for r in results]
class HookSchedulerTest(ServiceStateManagerTestBase): @inlineCallbacks def setUp(self): yield super(HookSchedulerTest, self).setUp() self.client = self.get_zookeeper_client() self.unit_relation = self.mocker.mock() self.executions = [] self.service = yield self.add_service_from_charm("wordpress") self.scheduler = HookScheduler(self.client, self.collect_executor, self.unit_relation, "", unit_name="wordpress/0") self.log_stream = self.capture_logging( "hook.scheduler", level=logging.DEBUG) def collect_executor(self, context, change): self.executions.append((context, change)) # Event reduction/coalescing cases def test_reduce_removed_added(self): """ A remove event for a node followed by an add event, results in a modify event. """ self.scheduler.notify_change(old_units=["u-1"], new_units=[]) self.scheduler.notify_change(old_units=[], new_units=["u-1"]) self.scheduler.run() self.assertEqual(len(self.executions), 1) self.assertEqual(self.executions[0][1].change_type, "modified") output = ("relation change old:['u-1'], new:[], modified:()", "relation change old:[], new:['u-1'], modified:()", "start", "executing hook for u-1:modified\n") self.assertEqual(self.log_stream.getvalue(), "\n".join(output)) def test_reduce_modify_remove_add(self): """A modify, remove, add event for a node results in a modify. An extra validation of the previous test. """ self.scheduler.notify_change(modified=["u-1"]) self.scheduler.notify_change(old_units=["u-1"], new_units=[]) self.scheduler.notify_change(old_units=[], new_units=["u-1"]) self.scheduler.run() self.assertEqual(len(self.executions), 1) self.assertEqual(self.executions[0][1].change_type, "modified") def test_reduce_add_modify(self): """An add and modify event for a node are coalesced to an add.""" self.scheduler.notify_change(old_units=[], new_units=["u-1"]) self.scheduler.notify_change(modified=["u-1"]) self.scheduler.run() self.assertEqual(len(self.executions), 1) self.assertEqual(self.executions[0][1].change_type, "joined") def test_reduce_add_remove(self): """an add followed by a removal results in a noop.""" self.scheduler.notify_change(old_units=[], new_units=["u-1"]) self.scheduler.notify_change(old_units=["u-1"], new_units=[]) self.scheduler.run() self.assertEqual(len(self.executions), 0) def test_reduce_modify_remove(self): """Modifying and then removing a node, results in just the removal.""" self.scheduler.notify_change(old_units=["u-1"], new_units=["u-1"], modified=["u-1"]) self.scheduler.notify_change(old_units=["u-1"], new_units=[]) self.scheduler.run() self.assertEqual(len(self.executions), 1) self.assertEqual(self.executions[0][1].change_type, "departed") def test_reduce_modify_modify(self): """Multiple modifies get coalesced to a single modify.""" # simulate normal startup, the first notify will always be the existing # membership set. self.scheduler.notify_change(old_units=[], new_units=["u-1"]) self.scheduler.run() self.scheduler.stop() self.assertEqual(len(self.executions), 1) # Now continue the modify/modify reduction. self.scheduler.notify_change(modified=["u-1"]) self.scheduler.notify_change(modified=["u-1"]) self.scheduler.notify_change(modified=["u-1"]) self.scheduler.run() self.assertEqual(len(self.executions), 2) self.assertEqual(self.executions[1][1].change_type, "modified") # Other stuff. @inlineCallbacks def test_start_stop(self): d = self.scheduler.run() # starting multiple times results in an error self.assertFailure(self.scheduler.run(), AssertionError) self.scheduler.stop() yield d # stopping multiple times is not an error yield self.scheduler.stop() @inlineCallbacks def test_membership_visibility_per_change(self): """Hooks are executed against changes, those changes are associated to a temporal timestamp, however the changes are scheduled for execution, and the state/time of the world may have advanced, to present a logically consistent view, we try to gaurantee at a minimum, that hooks will always see the membership of a relations it was at the time of their associated change. """ self.scheduler.notify_change( old_units=[], new_units=["u-1", "u-2"]) self.scheduler.notify_change( old_units=["u-1", "u-2"], new_units=["u-2", "u-3"]) self.scheduler.notify_change(modified=["u-2"]) self.scheduler.run() self.scheduler.stop() # only two reduced events, u-2, u-3 add self.assertEqual(len(self.executions), 2) # Now the first execution (u-2 add) should only see members # from the time of its change, not the current members. However # since u-1 has been subsequently removed, it no longer retains # an entry in the membership list. change_members = yield self.executions[0][0].get_members() self.assertEqual(change_members, ["u-2"]) self.scheduler.notify_change(modified=["u-2"]) self.scheduler.notify_change( old_units=["u-2", "u-3"], new_units=["u-2"]) self.scheduler.run() self.assertEqual(len(self.executions), 4) self.assertEqual(self.executions[2][1].change_type, "modified") # Verify modify events see the correct membership. change_members = yield self.executions[2][0].get_members() self.assertEqual(change_members, ["u-2", "u-3"]) @inlineCallbacks def test_membership_visibility_with_change(self): """We express a stronger guarantee of the above, namely that a hook wont see any 'active' members in a membership list, that it hasn't previously been given a notify of before. """ self.scheduler.notify_change( old_units=["u-1", "u-2"], new_units=["u-2", "u-3", "u-4"], modified=["u-2"]) self.scheduler.run() self.scheduler.stop() # add for u-3, u-4, remove for u-1, modify for u-2 self.assertEqual(len(self.executions), 4) # Verify members for each change. self.assertEqual(self.executions[0][1].change_type, "joined") members = yield self.executions[0][0].get_members() self.assertEqual(members, ["u-1", "u-2", "u-3"]) self.assertEqual(self.executions[1][1].change_type, "joined") members = yield self.executions[1][0].get_members() self.assertEqual(members, ["u-1", "u-2", "u-3", "u-4"]) self.assertEqual(self.executions[2][1].change_type, "departed") members = yield self.executions[2][0].get_members() self.assertEqual(members, ["u-2", "u-3", "u-4"]) self.assertEqual(self.executions[3][1].change_type, "modified") members = yield self.executions[2][0].get_members() self.assertEqual(members, ["u-2", "u-3", "u-4"]) context = yield self.scheduler.get_hook_context( RelationChange("", "", "")) self.assertEqual((yield context.get_members()), ["u-2", "u-3", "u-4"]) @inlineCallbacks def test_get_relation_change_empty(self): """Retrieving a hook context, is possible even if no no hooks has previously been fired (Empty membership).""" context = yield self.scheduler.get_hook_context( RelationChange("", "", "")) members = yield context.get_members() self.assertEqual(members, [])