def setUp(self):
yield super(HookSchedulerTest, self).setUp()
self.client = self.get_zookeeper_client()
self.unit_relation = self.mocker.mock()
self.executions = []
self.service = yield self.add_service_from_charm("wordpress")
self.scheduler = HookScheduler(self.client,
self.collect_executor,
self.unit_relation,
"",
unit_name="wordpress/0")
self.log_stream = self.capture_logging("hook.scheduler",
level=logging.DEBUG)
def setUp(self):
yield super(HookSchedulerTest, self).setUp()
self.client = self.get_zookeeper_client()
self.unit_relation = self.mocker.mock()
self.executions = []
self.service = yield self.add_service_from_charm("wordpress")
self.scheduler = HookScheduler(self.client,
self.collect_executor,
self.unit_relation, "",
unit_name="wordpress/0")
self.log_stream = self.capture_logging(
"hook.scheduler", level=logging.DEBUG)
def __init__(self, client, unit_name, unit_relation, relation_name,
unit_path, executor):
self._client = client
self._unit_path = unit_path
self._relation_name = relation_name
self._unit_relation = unit_relation
self._executor = executor
self._run_lock = DeferredLock()
self._log = logging.getLogger("unit.relation.lifecycle")
self._error_handler = None
self._scheduler = HookScheduler(client,
self._execute_change_hook,
self._unit_relation,
self._relation_name,
unit_name=unit_name)
self._watcher = None
def __init__(self, client, unit_name, unit_relation, relation_ident,
unit_dir, state_dir, executor):
self._client = client
self._unit_dir = unit_dir
self._relation_ident = relation_ident
self._relation_name = relation_ident.split(":")[0]
self._unit_relation = unit_relation
self._unit_name = unit_name
self._executor = executor
self._run_lock = DeferredLock()
self._log = logging.getLogger("unit.relation.lifecycle")
self._error_handler = None
schedule_path = os.path.join(
state_dir, "%s.schedule" % unit_relation.internal_relation_id)
self._scheduler = HookScheduler(
client, self._execute_change_hook, self._unit_relation,
self._relation_ident, unit_name, schedule_path)
self._watcher = None
class UnitRelationLifecycle(object):
"""Unit Relation Lifcycle management.
Provides for watching related units in a relation, and executing hooks
in response to changes. The lifecycle is driven by the workflow.
The Unit relation lifecycle glues together a number of components.
It controls a watcher that recieves watch events from zookeeper,
and it controls a hook scheduler which gets fed those events. When
the scheduler wants to execute a hook, the executor is called with
the hook path and the hook invoker.
**Relation hook invocation do not maintain global order or
determinism across relations**. They only maintain ordering and
determinism within a relation. A shared scheduler across relations
would be needed to maintain such behavior.
See docs/source/internals/unit-workflow-lifecycle.rst for a brief
discussion of some of the more interesting implementation decisions.
"""
def __init__(self, client, unit_name, unit_relation, relation_ident,
unit_dir, state_dir, executor):
self._client = client
self._unit_dir = unit_dir
self._relation_ident = relation_ident
self._relation_name = relation_ident.split(":")[0]
self._unit_relation = unit_relation
self._unit_name = unit_name
self._executor = executor
self._run_lock = DeferredLock()
self._log = logging.getLogger("unit.relation.lifecycle")
self._error_handler = None
schedule_path = os.path.join(
state_dir, "%s.schedule" % unit_relation.internal_relation_id)
self._scheduler = HookScheduler(
client, self._execute_change_hook, self._unit_relation,
self._relation_ident, unit_name, schedule_path)
self._watcher = None
@property
def watching(self):
"""Are we queuing up hook executions in response to state changes?"""
return self._watcher and self._watcher.running
@property
def executing(self):
"""Are we currently dequeuing and executing any queued hooks?"""
return self._scheduler.running
def set_hook_error_handler(self, handler):
"""Set an error handler to be invoked if a hook errors.
The handler should accept two parameters, the RelationChange that
triggered the hook, and the exception instance.
"""
self._error_handler = handler
@inlineCallbacks
def start(self, start_watches=True, start_scheduler=True):
"""Start watching related units and executing change hooks.
:param bool start_watches: True to start relation watches
:param bool start_scheduler: True to run the scheduler and actually
react to any changes delivered by the watcher
"""
yield self._run_lock.acquire()
try:
# Start the hook execution scheduler.
if start_scheduler and not self.executing:
self._scheduler.run()
# Create a watcher if we don't have one yet.
if self._watcher is None:
self._watcher = yield self._unit_relation.watch_related_units(
self._scheduler.cb_change_members,
self._scheduler.cb_change_settings)
# And start the watcher.
if start_watches and not self.watching:
yield self._watcher.start()
finally:
self._run_lock.release()
self._log.debug(
"started relation:%s lifecycle", self._relation_name)
@inlineCallbacks
def stop(self, stop_watches=True):
"""Stop executing relation change hooks; maybe stop watching changes.
:param bool stop_watches: True to stop watches as well as scheduler
(which will prevent changes from being detected and queued, as well
as stopping them being executed).
"""
yield self._run_lock.acquire()
try:
if stop_watches and self.watching:
self._watcher.stop()
if self._scheduler.running:
self._scheduler.stop()
finally:
yield self._run_lock.release()
self._log.debug("stopped relation:%s lifecycle", self._relation_name)
@inlineCallbacks
def depart(self):
"""Inform the charm that the service has departed the relation.
"""
self._log.debug("depart relation lifecycle")
unit_id = self._unit_relation.internal_unit_id
context = DepartedRelationHookContext(
self._client, self._unit_name, unit_id, self._relation_name,
self._unit_relation.internal_relation_id)
change = RelationChange(self._relation_ident, "departed", "")
invoker = self._get_invoker(context, change)
hook_name = "%s-relation-broken" % self._relation_name
yield self._execute_hook(invoker, hook_name, change)
def _get_invoker(self, context, change):
socket_path = os.path.join(self._unit_dir, HOOK_SOCKET_FILE)
return RelationInvoker(
context, change, "constant", socket_path, self._unit_dir,
hook_log)
def _execute_change_hook(self, context, change):
"""Invoked by the contained HookScheduler, to execute a hook.
We utilize the HookExecutor to execute the hook, if an
error occurs, it will be reraised, unless an error handler
is specified see ``set_hook_error_handler``.
"""
if change.change_type == "departed":
hook_name = "%s-relation-departed" % self._relation_name
elif change.change_type == "joined":
hook_name = "%s-relation-joined" % self._relation_name
else:
hook_name = "%s-relation-changed" % self._relation_name
invoker = self._get_invoker(context, change)
return self._execute_hook(invoker, hook_name, change)
@inlineCallbacks
def _execute_hook(self, invoker, hook_name, change):
hook_path = os.path.join(
self._unit_dir, "charm", "hooks", hook_name)
yield self._run_lock.acquire()
self._log.debug("Executing hook %s", hook_name)
try:
yield self._executor(invoker, hook_path)
except Exception, e:
# We can't hold the run lock when we invoke the error
# handler, or we get a deadlock if the handler
# manipulates the lifecycle.
yield self._run_lock.release()
self._log.warn("Error in %s hook: %s", hook_name, e)
if not self._error_handler:
raise
self._log.info(
"Invoked error handler for %s hook", hook_name)
yield self._error_handler(change, e)
returnValue(False)
else:
class HookSchedulerTest(ServiceStateManagerTestBase):
@inlineCallbacks
def setUp(self):
yield super(HookSchedulerTest, self).setUp()
self.client = self.get_zookeeper_client()
self.unit_relation = self.mocker.mock()
self.executions = []
self.service = yield self.add_service_from_charm("wordpress")
self.scheduler = HookScheduler(self.client,
self.collect_executor,
self.unit_relation,
"",
unit_name="wordpress/0")
self.log_stream = self.capture_logging("hook.scheduler",
level=logging.DEBUG)
def collect_executor(self, context, change):
self.executions.append((context, change))
# Event reduction/coalescing cases
def test_reduce_removed_added(self):
""" A remove event for a node followed by an add event,
results in a modify event.
"""
self.scheduler.notify_change(old_units=["u-1"], new_units=[])
self.scheduler.notify_change(old_units=[], new_units=["u-1"])
self.scheduler.run()
self.assertEqual(len(self.executions), 1)
self.assertEqual(self.executions[0][1].change_type, "modified")
output = ("relation change old:['u-1'], new:[], modified:()",
"relation change old:[], new:['u-1'], modified:()", "start",
"executing hook for u-1:modified\n")
self.assertEqual(self.log_stream.getvalue(), "\n".join(output))
def test_reduce_modify_remove_add(self):
"""A modify, remove, add event for a node results in a modify.
An extra validation of the previous test.
"""
self.scheduler.notify_change(modified=["u-1"])
self.scheduler.notify_change(old_units=["u-1"], new_units=[])
self.scheduler.notify_change(old_units=[], new_units=["u-1"])
self.scheduler.run()
self.assertEqual(len(self.executions), 1)
self.assertEqual(self.executions[0][1].change_type, "modified")
def test_reduce_add_modify(self):
"""An add and modify event for a node are coalesced to an add."""
self.scheduler.notify_change(old_units=[], new_units=["u-1"])
self.scheduler.notify_change(modified=["u-1"])
self.scheduler.run()
self.assertEqual(len(self.executions), 1)
self.assertEqual(self.executions[0][1].change_type, "joined")
def test_reduce_add_remove(self):
"""an add followed by a removal results in a noop."""
self.scheduler.notify_change(old_units=[], new_units=["u-1"])
self.scheduler.notify_change(old_units=["u-1"], new_units=[])
self.scheduler.run()
self.assertEqual(len(self.executions), 0)
def test_reduce_modify_remove(self):
"""Modifying and then removing a node, results in just the removal."""
self.scheduler.notify_change(old_units=["u-1"],
new_units=["u-1"],
modified=["u-1"])
self.scheduler.notify_change(old_units=["u-1"], new_units=[])
self.scheduler.run()
self.assertEqual(len(self.executions), 1)
self.assertEqual(self.executions[0][1].change_type, "departed")
def test_reduce_modify_modify(self):
"""Multiple modifies get coalesced to a single modify."""
# simulate normal startup, the first notify will always be the existing
# membership set.
self.scheduler.notify_change(old_units=[], new_units=["u-1"])
self.scheduler.run()
self.scheduler.stop()
self.assertEqual(len(self.executions), 1)
# Now continue the modify/modify reduction.
self.scheduler.notify_change(modified=["u-1"])
self.scheduler.notify_change(modified=["u-1"])
self.scheduler.notify_change(modified=["u-1"])
self.scheduler.run()
self.assertEqual(len(self.executions), 2)
self.assertEqual(self.executions[1][1].change_type, "modified")
# Other stuff.
@inlineCallbacks
def test_start_stop(self):
d = self.scheduler.run()
# starting multiple times results in an error
self.assertFailure(self.scheduler.run(), AssertionError)
self.scheduler.stop()
yield d
# stopping multiple times is not an error
yield self.scheduler.stop()
@inlineCallbacks
def test_membership_visibility_per_change(self):
"""Hooks are executed against changes, those changes are
associated to a temporal timestamp, however the changes
are scheduled for execution, and the state/time of the
world may have advanced, to present a logically consistent
view, we try to gaurantee at a minimum, that hooks will
always see the membership of a relations it was at the
time of their associated change.
"""
self.scheduler.notify_change(old_units=[], new_units=["u-1", "u-2"])
self.scheduler.notify_change(old_units=["u-1", "u-2"],
new_units=["u-2", "u-3"])
self.scheduler.notify_change(modified=["u-2"])
self.scheduler.run()
self.scheduler.stop()
# only two reduced events, u-2, u-3 add
self.assertEqual(len(self.executions), 2)
# Now the first execution (u-2 add) should only see members
# from the time of its change, not the current members. However
# since u-1 has been subsequently removed, it no longer retains
# an entry in the membership list.
change_members = yield self.executions[0][0].get_members()
self.assertEqual(change_members, ["u-2"])
self.scheduler.notify_change(modified=["u-2"])
self.scheduler.notify_change(old_units=["u-2", "u-3"],
new_units=["u-2"])
self.scheduler.run()
self.assertEqual(len(self.executions), 4)
self.assertEqual(self.executions[2][1].change_type, "modified")
# Verify modify events see the correct membership.
change_members = yield self.executions[2][0].get_members()
self.assertEqual(change_members, ["u-2", "u-3"])
@inlineCallbacks
def test_membership_visibility_with_change(self):
"""We express a stronger guarantee of the above, namely that
a hook wont see any 'active' members in a membership list, that
it hasn't previously been given a notify of before.
"""
self.scheduler.notify_change(old_units=["u-1", "u-2"],
new_units=["u-2", "u-3", "u-4"],
modified=["u-2"])
self.scheduler.run()
self.scheduler.stop()
# add for u-3, u-4, remove for u-1, modify for u-2
self.assertEqual(len(self.executions), 4)
# Verify members for each change.
self.assertEqual(self.executions[0][1].change_type, "joined")
members = yield self.executions[0][0].get_members()
self.assertEqual(members, ["u-1", "u-2", "u-3"])
self.assertEqual(self.executions[1][1].change_type, "joined")
members = yield self.executions[1][0].get_members()
self.assertEqual(members, ["u-1", "u-2", "u-3", "u-4"])
self.assertEqual(self.executions[2][1].change_type, "departed")
members = yield self.executions[2][0].get_members()
self.assertEqual(members, ["u-2", "u-3", "u-4"])
self.assertEqual(self.executions[3][1].change_type, "modified")
members = yield self.executions[2][0].get_members()
self.assertEqual(members, ["u-2", "u-3", "u-4"])
context = yield self.scheduler.get_hook_context(
RelationChange("", "", ""))
self.assertEqual((yield context.get_members()), ["u-2", "u-3", "u-4"])
@inlineCallbacks
def test_get_relation_change_empty(self):
"""Retrieving a hook context, is possible even if no
no hooks has previously been fired (Empty membership)."""
context = yield self.scheduler.get_hook_context(
RelationChange("", "", ""))
members = yield context.get_members()
self.assertEqual(members, [])
def test_start_stop_start(self):
"""Stop values should only be honored if the scheduler is stopped.
"""
waits = [Deferred(), succeed(True), succeed(True), succeed(True)]
results = []
@inlineCallbacks
def executor(context, change):
res = yield waits[len(results)]
results.append((context, change))
returnValue(res)
scheduler = HookScheduler(self.client, executor, self.unit_relation, "", "wordpress/0", self.state_file)
# Start the scheduler
d = scheduler.run()
# Now queue up some changes.
scheduler.cb_change_members([], ["u-1"])
scheduler.cb_change_members(["u-1"], ["u-1", "u-2"])
# Stop the scheduler
scheduler.stop()
yield d
self.assertFalse(scheduler.running)
# Finish the hook execution
waits[0].callback(True)
d = scheduler.run()
self.assertTrue(scheduler.running)
# More changes
scheduler.cb_change_settings([("u-1", 1)])
scheduler.cb_change_settings([("u-2", 1)])
# Scheduler should still be running.
print self._debug_scheduler()
print [(r[1].change_type, r[1].unit_name) for r in results]
class HookSchedulerTest(ServiceStateManagerTestBase):
@inlineCallbacks
def setUp(self):
yield super(HookSchedulerTest, self).setUp()
self.client = self.get_zookeeper_client()
self.unit_relation = self.mocker.mock()
self.executions = []
self.service = yield self.add_service_from_charm("wordpress")
self.scheduler = HookScheduler(self.client,
self.collect_executor,
self.unit_relation, "",
unit_name="wordpress/0")
self.log_stream = self.capture_logging(
"hook.scheduler", level=logging.DEBUG)
def collect_executor(self, context, change):
self.executions.append((context, change))
# Event reduction/coalescing cases
def test_reduce_removed_added(self):
""" A remove event for a node followed by an add event,
results in a modify event.
"""
self.scheduler.notify_change(old_units=["u-1"], new_units=[])
self.scheduler.notify_change(old_units=[], new_units=["u-1"])
self.scheduler.run()
self.assertEqual(len(self.executions), 1)
self.assertEqual(self.executions[0][1].change_type, "modified")
output = ("relation change old:['u-1'], new:[], modified:()",
"relation change old:[], new:['u-1'], modified:()",
"start",
"executing hook for u-1:modified\n")
self.assertEqual(self.log_stream.getvalue(), "\n".join(output))
def test_reduce_modify_remove_add(self):
"""A modify, remove, add event for a node results in a modify.
An extra validation of the previous test.
"""
self.scheduler.notify_change(modified=["u-1"])
self.scheduler.notify_change(old_units=["u-1"], new_units=[])
self.scheduler.notify_change(old_units=[], new_units=["u-1"])
self.scheduler.run()
self.assertEqual(len(self.executions), 1)
self.assertEqual(self.executions[0][1].change_type, "modified")
def test_reduce_add_modify(self):
"""An add and modify event for a node are coalesced to an add."""
self.scheduler.notify_change(old_units=[], new_units=["u-1"])
self.scheduler.notify_change(modified=["u-1"])
self.scheduler.run()
self.assertEqual(len(self.executions), 1)
self.assertEqual(self.executions[0][1].change_type, "joined")
def test_reduce_add_remove(self):
"""an add followed by a removal results in a noop."""
self.scheduler.notify_change(old_units=[], new_units=["u-1"])
self.scheduler.notify_change(old_units=["u-1"], new_units=[])
self.scheduler.run()
self.assertEqual(len(self.executions), 0)
def test_reduce_modify_remove(self):
"""Modifying and then removing a node, results in just the removal."""
self.scheduler.notify_change(old_units=["u-1"],
new_units=["u-1"],
modified=["u-1"])
self.scheduler.notify_change(old_units=["u-1"], new_units=[])
self.scheduler.run()
self.assertEqual(len(self.executions), 1)
self.assertEqual(self.executions[0][1].change_type, "departed")
def test_reduce_modify_modify(self):
"""Multiple modifies get coalesced to a single modify."""
# simulate normal startup, the first notify will always be the existing
# membership set.
self.scheduler.notify_change(old_units=[], new_units=["u-1"])
self.scheduler.run()
self.scheduler.stop()
self.assertEqual(len(self.executions), 1)
# Now continue the modify/modify reduction.
self.scheduler.notify_change(modified=["u-1"])
self.scheduler.notify_change(modified=["u-1"])
self.scheduler.notify_change(modified=["u-1"])
self.scheduler.run()
self.assertEqual(len(self.executions), 2)
self.assertEqual(self.executions[1][1].change_type, "modified")
# Other stuff.
@inlineCallbacks
def test_start_stop(self):
d = self.scheduler.run()
# starting multiple times results in an error
self.assertFailure(self.scheduler.run(), AssertionError)
self.scheduler.stop()
yield d
# stopping multiple times is not an error
yield self.scheduler.stop()
@inlineCallbacks
def test_membership_visibility_per_change(self):
"""Hooks are executed against changes, those changes are
associated to a temporal timestamp, however the changes
are scheduled for execution, and the state/time of the
world may have advanced, to present a logically consistent
view, we try to gaurantee at a minimum, that hooks will
always see the membership of a relations it was at the
time of their associated change.
"""
self.scheduler.notify_change(
old_units=[], new_units=["u-1", "u-2"])
self.scheduler.notify_change(
old_units=["u-1", "u-2"], new_units=["u-2", "u-3"])
self.scheduler.notify_change(modified=["u-2"])
self.scheduler.run()
self.scheduler.stop()
# only two reduced events, u-2, u-3 add
self.assertEqual(len(self.executions), 2)
# Now the first execution (u-2 add) should only see members
# from the time of its change, not the current members. However
# since u-1 has been subsequently removed, it no longer retains
# an entry in the membership list.
change_members = yield self.executions[0][0].get_members()
self.assertEqual(change_members, ["u-2"])
self.scheduler.notify_change(modified=["u-2"])
self.scheduler.notify_change(
old_units=["u-2", "u-3"], new_units=["u-2"])
self.scheduler.run()
self.assertEqual(len(self.executions), 4)
self.assertEqual(self.executions[2][1].change_type, "modified")
# Verify modify events see the correct membership.
change_members = yield self.executions[2][0].get_members()
self.assertEqual(change_members, ["u-2", "u-3"])
@inlineCallbacks
def test_membership_visibility_with_change(self):
"""We express a stronger guarantee of the above, namely that
a hook wont see any 'active' members in a membership list, that
it hasn't previously been given a notify of before.
"""
self.scheduler.notify_change(
old_units=["u-1", "u-2"],
new_units=["u-2", "u-3", "u-4"],
modified=["u-2"])
self.scheduler.run()
self.scheduler.stop()
# add for u-3, u-4, remove for u-1, modify for u-2
self.assertEqual(len(self.executions), 4)
# Verify members for each change.
self.assertEqual(self.executions[0][1].change_type, "joined")
members = yield self.executions[0][0].get_members()
self.assertEqual(members, ["u-1", "u-2", "u-3"])
self.assertEqual(self.executions[1][1].change_type, "joined")
members = yield self.executions[1][0].get_members()
self.assertEqual(members, ["u-1", "u-2", "u-3", "u-4"])
self.assertEqual(self.executions[2][1].change_type, "departed")
members = yield self.executions[2][0].get_members()
self.assertEqual(members, ["u-2", "u-3", "u-4"])
self.assertEqual(self.executions[3][1].change_type, "modified")
members = yield self.executions[2][0].get_members()
self.assertEqual(members, ["u-2", "u-3", "u-4"])
context = yield self.scheduler.get_hook_context(
RelationChange("", "", ""))
self.assertEqual((yield context.get_members()),
["u-2", "u-3", "u-4"])
@inlineCallbacks
def test_get_relation_change_empty(self):
"""Retrieving a hook context, is possible even if no
no hooks has previously been fired (Empty membership)."""
context = yield self.scheduler.get_hook_context(
RelationChange("", "", ""))
members = yield context.get_members()
self.assertEqual(members, [])
