def test_agent_start_from_started_workflow(self):
lifecycle = UnitLifecycle(
self.client, self.states["unit"], self.states["service"],
self.unit_directory, self.state_directory, self.executor)
workflow = UnitWorkflowState(
self.client, self.states["unit"], lifecycle,
os.path.join(self.juju_directory, "state"))
with (yield workflow.lock()):
yield workflow.fire_transition("install")
yield lifecycle.stop(fire_hooks=False, stop_relations=False)
yield self.agent.startService()
current_state = yield self.agent.workflow.get_state()
self.assertEqual(current_state, "started")
self.assertTrue(self.agent.lifecycle.running)
self.assertTrue(self.agent.executor.running)
class UnitLifecycleTest(LifecycleTestBase):
@inlineCallbacks
def setUp(self):
yield super(UnitLifecycleTest, self).setUp()
yield self.setup_default_test_relation()
self.lifecycle = UnitLifecycle(
self.client, self.states["unit"], self.states["service"],
self.unit_directory, self.executor)
@inlineCallbacks
def test_hook_invocation(self):
"""Verify lifecycle methods invoke corresponding charm hooks.
"""
# install hook
file_path = self.makeFile()
self.write_hook(
"install",
'#!/bin/sh\n echo "hello world" > %s' % file_path)
yield self.lifecycle.install()
self.assertEqual(open(file_path).read().strip(), "hello world")
# Start hook
file_path = self.makeFile()
self.write_hook(
"start",
'#!/bin/sh\n echo "sugarcane" > %s' % file_path)
yield self.lifecycle.start()
self.assertEqual(open(file_path).read().strip(), "sugarcane")
# Stop hook
file_path = self.makeFile()
self.write_hook(
"stop",
'#!/bin/sh\n echo "siesta" > %s' % file_path)
yield self.lifecycle.stop()
self.assertEqual(open(file_path).read().strip(), "siesta")
# verify the sockets are cleaned up.
self.assertEqual(os.listdir(self.unit_directory), ["charm"])
@inlineCallbacks
def test_start_sans_hook(self):
"""The lifecycle start can be invoked without firing hooks."""
self.write_hook("start", "#!/bin/sh\n exit 1")
start_executed = self.wait_on_hook("start")
yield self.lifecycle.start(fire_hooks=False)
self.assertFalse(start_executed.called)
@inlineCallbacks
def test_stop_sans_hook(self):
"""The lifecycle stop can be invoked without firing hooks."""
self.write_hook("stop", "#!/bin/sh\n exit 1")
stop_executed = self.wait_on_hook("stop")
yield self.lifecycle.start()
yield self.lifecycle.stop(fire_hooks=False)
self.assertFalse(stop_executed.called)
@inlineCallbacks
def test_install_sans_hook(self):
"""The lifecycle install can be invoked without firing hooks."""
self.write_hook("install", "#!/bin/sh\n exit 1")
install_executed = self.wait_on_hook("install")
yield self.lifecycle.install(fire_hooks=False)
self.assertFalse(install_executed.called)
@inlineCallbacks
def test_upgrade_sans_hook(self):
"""The lifecycle upgrade can be invoked without firing hooks."""
self.executor.stop()
self.write_hook("upgrade-charm", "#!/bin/sh\n exit 1")
upgrade_executed = self.wait_on_hook("upgrade-charm")
yield self.lifecycle.upgrade_charm(fire_hooks=False)
self.assertFalse(upgrade_executed.called)
self.assertTrue(self.executor.running)
def test_hook_error(self):
"""Verify hook execution error, raises an exception."""
self.write_hook("install", '#!/bin/sh\n exit 1')
d = self.lifecycle.install()
return self.failUnlessFailure(d, CharmInvocationError)
def test_hook_not_executable(self):
"""A hook not executable, raises an exception."""
self.write_hook("install", '#!/bin/sh\n exit 0', no_exec=True)
return self.failUnlessFailure(
self.lifecycle.install(), CharmError)
def test_hook_not_formatted_correctly(self):
"""Hook execution error, raises an exception."""
self.write_hook("install", '!/bin/sh\n exit 0')
return self.failUnlessFailure(
self.lifecycle.install(), CharmInvocationError)
def write_start_and_relation_hooks(self, relation_name=None):
"""Write some minimal start, and relation-changed hooks.
Returns the output file of the relation hook.
"""
file_path = self.makeFile()
if relation_name is None:
relation_name = self.states["service_relation"].relation_name
self.write_hook("start", ("#!/bin/bash\n" "echo hello"))
self.write_hook("config-changed", ("#!/bin/bash\n" "echo configure"))
self.write_hook("stop", ("#!/bin/bash\n" "echo goodbye"))
self.write_hook(
"%s-relation-joined" % relation_name,
("#!/bin/bash\n" "echo joined >> %s\n" % file_path))
self.write_hook(
"%s-relation-changed" % relation_name,
("#!/bin/bash\n" "echo changed >> %s\n" % file_path))
self.write_hook(
"%s-relation-departed" % relation_name,
("#!/bin/bash\n" "echo departed >> %s\n" % file_path))
self.assertFalse(os.path.exists(file_path))
return file_path
@inlineCallbacks
def test_upgrade_hook_invoked_on_upgrade_charm(self):
"""Invoking the upgrade_charm lifecycle method executes the
upgrade-charm hook.
"""
file_path = self.makeFile("")
self.write_hook(
"upgrade-charm",
("#!/bin/bash\n" "echo upgraded >> %s\n" % file_path))
# upgrade requires the external actor that extracts the charm
# to stop the hook executor, prior to extraction so the
# upgrade is the first hook run.
yield self.executor.stop()
yield self.lifecycle.upgrade_charm()
self.assertEqual(open(file_path).read().strip(), "upgraded")
@inlineCallbacks
def test_config_hook_invoked_on_configure(self):
"""Invoke the configure lifecycle method will execute the
config-changed hook.
"""
output = self.capture_logging("unit.lifecycle", level=logging.DEBUG)
# configure hook requires a running unit lifecycle..
yield self.assertFailure(self.lifecycle.configure(), AssertionError)
# Config hook
file_path = self.makeFile()
self.write_hook(
"config-changed",
'#!/bin/sh\n echo "palladium" > %s' % file_path)
yield self.lifecycle.start()
yield self.lifecycle.configure()
self.assertEqual(open(file_path).read().strip(), "palladium")
self.assertIn("configured unit", output.getvalue())
@inlineCallbacks
def test_service_relation_watching(self):
"""When the unit lifecycle is started, the assigned relations
of the service are watched, with unit relation lifecycles
created for each.
Relation hook invocation do not maintain global order or determinism
across relations. They only maintain ordering and determinism within
a relation. A shared scheduler across relations would be needed to
maintain such behavior.
"""
file_path = self.write_start_and_relation_hooks()
wordpress1_states = yield self.add_opposite_service_unit(self.states)
yield self.lifecycle.start()
yield self.wait_on_hook("app-relation-changed")
self.assertTrue(os.path.exists(file_path))
self.assertEqual([x.strip() for x in open(file_path).readlines()],
["joined", "changed"])
# Queue up our wait condition, of 4 hooks firing
hooks_complete = self.wait_on_hook(
sequence=[
"app-relation-joined", # joined event fires join hook,
"app-relation-changed", # followed by changed hook
"app-relation-changed",
"app-relation-departed"])
# add another.
wordpress2_states = yield self.add_opposite_service_unit(
(yield self.add_relation_service_unit_to_another_endpoint(
self.states,
RelationEndpoint(
"wordpress-2", "client-server", "db", "client"))))
# modify one.
wordpress1_states["unit_relation"].set_data(
{"hello": "world"})
# delete one.
self.client.delete(
"/relations/%s/client/%s" % (
wordpress2_states["relation"].internal_id,
wordpress2_states["unit"].internal_id))
# verify results, waiting for hooks to complete
yield hooks_complete
self.assertEqual(
set([x.strip() for x in open(file_path).readlines()]),
set(["joined", "changed", "joined", "changed", "departed"]))
@inlineCallbacks
def test_removed_relation_depart(self):
"""
If a removed relation is detected, the unit relation lifecycle is
stopped.
"""
file_path = self.write_start_and_relation_hooks()
self.write_hook("app-relation-broken", "#!/bin/bash\n echo broken")
yield self.lifecycle.start()
wordpress_states = yield self.add_opposite_service_unit(self.states)
# Wait for the watch and hook to fire.
yield self.wait_on_hook("app-relation-changed")
self.assertTrue(os.path.exists(file_path))
self.assertEqual([x.strip() for x in open(file_path).readlines()],
["joined", "changed"])
self.assertTrue(self.lifecycle.get_relation_workflow(
self.states["relation"].internal_id))
# Remove the relation between mysql and wordpress
yield self.relation_manager.remove_relation_state(
self.states["relation"])
# Wait till the unit relation workflow has been processed the event.
yield self.wait_on_state(
self.lifecycle.get_relation_workflow(
self.states["relation"].internal_id),
"departed")
# Modify the unit relation settings, to generate a spurious event.
yield wordpress_states["unit_relation"].set_data(
{"hello": "world"})
# Verify no notice was recieved for the modify before we where stopped.
self.assertEqual([x.strip() for x in open(file_path).readlines()],
["joined", "changed"])
# Verify the unit relation lifecycle has been disposed of.
self.assertRaises(KeyError,
self.lifecycle.get_relation_workflow,
self.states["relation"].internal_id)
@inlineCallbacks
def test_lifecycle_start_stop_starts_relations(self):
"""Starting a stopped lifecycle, restarts relation events.
"""
wordpress1_states = yield self.add_opposite_service_unit(self.states)
wordpress2_states = yield self.add_opposite_service_unit(
(yield self.add_relation_service_unit_to_another_endpoint(
self.states,
RelationEndpoint(
"wordpress-2", "client-server", "db", "client"))))
# Start and stop lifecycle
file_path = self.write_start_and_relation_hooks()
yield self.lifecycle.start()
yield self.wait_on_hook("app-relation-changed")
self.assertTrue(os.path.exists(file_path))
yield self.lifecycle.stop()
########################################################
# Add, remove relations, and modify related unit settings.
# The following isn't enough to trigger a hook notification.
# yield wordpress1_states["relation"].unassign_service(
# wordpress1_states["service"])
#
# The removal of the external relation, means we stop getting notifies
# of it, but the underlying unit agents of the service are responsible
# for removing their presence nodes within the relationship, which
# triggers a hook invocation.
yield self.client.delete("/relations/%s/client/%s" % (
wordpress1_states["relation"].internal_id,
wordpress1_states["unit"].internal_id))
yield wordpress2_states["unit_relation"].set_data(
{"hello": "world"})
yield self.add_opposite_service_unit(
(yield self.add_relation_service_unit_to_another_endpoint(
self.states,
RelationEndpoint(
"wordpress-3", "client-server", "db", "client"))))
# Verify no hooks are executed.
yield self.sleep(0.1)
res = [x.strip() for x in open(file_path)]
if ((res != ["joined", "changed", "joined", "changed"])
and
(res != ["joined", "joined", "changed", "changed"])):
self.fail("Invalid join sequence %s" % res)
# XXX - With scheduler local state recovery, we should get the modify.
# Start and verify events.
hooks_executed = self.wait_on_hook(
sequence=[
"config-changed",
"start",
"app-relation-departed",
"app-relation-joined", # joined event fires joined hook,
"app-relation-changed" # followed by changed hook
])
yield self.lifecycle.start()
yield hooks_executed
res.extend(["departed", "joined", "changed"])
self.assertEqual([x.strip() for x in open(file_path)],
res)
@inlineCallbacks
def test_lock_start_stop_watch(self):
"""The lifecycle, internally employs lock to prevent simulatenous
execution of methods which modify internal state. This allows
for a long running hook to be called safely, even if the other
invocations of the lifecycle, the subsequent invocations will
block till they can acquire the lock.
"""
self.write_hook("start", "#!/bin/bash\necho start\n")
self.write_hook("stop", "#!/bin/bash\necho stop\n")
results = []
finish_callback = [Deferred() for i in range(4)]
# Control the speed of hook execution
original_invoker = Invoker.__call__
invoker = self.mocker.patch(Invoker)
@inlineCallbacks
def long_hook(ctx, hook_path):
results.append(os.path.basename(hook_path))
yield finish_callback[len(results) - 1]
yield original_invoker(ctx, hook_path)
for i in range(4):
invoker(
MATCH(lambda x: x.endswith("start") or x.endswith("stop")))
self.mocker.call(long_hook, with_object=True)
self.mocker.replay()
# Hook execution sequence to match on.
test_complete = self.wait_on_hook(sequence=["config-changed",
"start",
"stop",
"config-changed",
"start"])
# Fire off the lifecycle methods
execution_callbacks = [self.lifecycle.start(),
self.lifecycle.stop(),
self.lifecycle.start(),
self.lifecycle.stop()]
self.assertEqual([0, 0, 0, 0],
[x.called for x in execution_callbacks])
# kill the delay on the second
finish_callback[1].callback(True)
finish_callback[2].callback(True)
self.assertEqual([0, 0, 0, 0],
[x.called for x in execution_callbacks])
# let them pass, kill the delay on the first
finish_callback[0].callback(True)
yield test_complete
self.assertEqual([False, True, True, False],
[x.called for x in execution_callbacks])
# Finish the last hook
finish_callback[3].callback(True)
yield self.wait_on_hook("stop")
self.assertEqual([True, True, True, True],
[x.called for x in execution_callbacks])
class LifecycleResolvedTest(LifecycleTestBase):
@inlineCallbacks
def setUp(self):
yield super(LifecycleResolvedTest, self).setUp()
yield self.setup_default_test_relation()
self.lifecycle = UnitLifecycle(
self.client, self.states["unit"], self.states["service"],
self.unit_directory, self.executor)
def get_unit_relation_workflow(self, states):
state_dir = os.path.join(self.juju_directory, "state")
lifecycle = UnitRelationLifecycle(
self.client,
states["unit_relation"],
states["service_relation"].relation_name,
self.unit_directory,
self.executor)
workflow = RelationWorkflowState(
self.client,
states["unit_relation"],
lifecycle,
state_dir)
return (workflow, lifecycle)
@inlineCallbacks
def wb_test_start_with_relation_errors(self):
"""
White box testing to ensure that an error when starting the
lifecycle is propogated appropriately, and that we collect
all results before returning.
"""
mock_service = self.mocker.patch(self.lifecycle._service)
mock_service.watch_relation_states(MATCH(lambda x: callable(x)))
self.mocker.result(fail(SyntaxError()))
mock_unit = self.mocker.patch(self.lifecycle._unit)
mock_unit.watch_relation_resolved(MATCH(lambda x: callable(x)))
results = []
wait = Deferred()
@inlineCallbacks
def complete(*args):
yield wait
results.append(True)
returnValue(True)
self.mocker.call(complete)
self.mocker.replay()
# Start the unit, assert a failure, and capture the deferred
wait_failure = self.assertFailure(self.lifecycle.start(), SyntaxError)
# Verify we have no results for the second callback or the start call
self.assertFalse(results)
self.assertFalse(wait_failure.called)
# Let the second callback complete
wait.callback(True)
# Wait for the start error to bubble up.
yield wait_failure
# Verify the second deferred was waited on.
self.assertTrue(results)
@inlineCallbacks
def test_resolved_relation_watch_unit_lifecycle_not_running(self):
"""If the unit is not running then no relation resolving is performed.
However the resolution value remains the same.
"""
# Start the unit.
yield self.lifecycle.start()
# Simulate relation down on an individual unit relation
workflow = self.lifecycle.get_relation_workflow(
self.states["unit_relation"].internal_relation_id)
self.assertEqual("up", (yield workflow.get_state()))
yield workflow.transition_state("down")
resolved = self.wait_on_state(workflow, "up")
# Stop the unit lifecycle
yield self.lifecycle.stop()
# Set the relation to resolved
yield self.states["unit"].set_relation_resolved(
{self.states["unit_relation"].internal_relation_id: NO_HOOKS})
# Give a moment for the watch to fire erroneously
yield self.sleep(0.2)
# Ensure we didn't attempt a transition.
self.assertFalse(resolved.called)
self.assertEqual(
{self.states["unit_relation"].internal_relation_id: NO_HOOKS},
(yield self.states["unit"].get_relation_resolved()))
# If the unit is restarted start, we currently have the
# behavior that the unit relation workflow will automatically
# be transitioned back to running, as part of the normal state
# transition. Sigh.. we should have a separate error
# state for relation hooks then down with state variable usage.
# The current end behavior though seems like the best outcome, ie.
# automatically restart relations.
@inlineCallbacks
def test_resolved_relation_watch_relation_up(self):
"""If a relation marked as to be resolved is already running,
then no work is performed.
"""
# Start the unit.
yield self.lifecycle.start()
# get a hold of the unit relation and verify state
workflow = self.lifecycle.get_relation_workflow(
self.states["unit_relation"].internal_relation_id)
self.assertEqual("up", (yield workflow.get_state()))
# Set the relation to resolved
yield self.states["unit"].set_relation_resolved(
{self.states["unit_relation"].internal_relation_id: NO_HOOKS})
# Give a moment for the watch to fire, invoke callback, and reset.
yield self.sleep(0.1)
# Ensure we're still up and the relation resolved setting has been
# cleared.
self.assertEqual(
None, (yield self.states["unit"].get_relation_resolved()))
self.assertEqual("up", (yield workflow.get_state()))
@inlineCallbacks
def test_resolved_relation_watch_from_error(self):
"""Unit lifecycle's will process a unit relation resolved
setting, and transition a down relation back to a running
state.
"""
log_output = self.capture_logging(
"unit.lifecycle", level=logging.DEBUG)
# Start the unit.
yield self.lifecycle.start()
# Simulate an error condition
workflow = self.lifecycle.get_relation_workflow(
self.states["unit_relation"].internal_relation_id)
self.assertEqual("up", (yield workflow.get_state()))
yield workflow.fire_transition("error")
resolved = self.wait_on_state(workflow, "up")
# Set the relation to resolved
yield self.states["unit"].set_relation_resolved(
{self.states["unit_relation"].internal_relation_id: NO_HOOKS})
# Wait for the relation to come back up
value = yield self.states["unit"].get_relation_resolved()
yield resolved
# Verify state
value = yield workflow.get_state()
self.assertEqual(value, "up")
self.assertIn(
"processing relation resolved changed", log_output.getvalue())
@inlineCallbacks
def test_resolved_relation_watch(self):
"""Unit lifecycle's will process a unit relation resolved
setting, and transition a down relation back to a running
state.
"""
log_output = self.capture_logging(
"unit.lifecycle", level=logging.DEBUG)
# Start the unit.
yield self.lifecycle.start()
# Simulate an error condition
workflow = self.lifecycle.get_relation_workflow(
self.states["unit_relation"].internal_relation_id)
self.assertEqual("up", (yield workflow.get_state()))
yield workflow.transition_state("down")
resolved = self.wait_on_state(workflow, "up")
# Set the relation to resolved
yield self.states["unit"].set_relation_resolved(
{self.states["unit_relation"].internal_relation_id: NO_HOOKS})
# Wait for the relation to come back up
value = yield self.states["unit"].get_relation_resolved()
yield resolved
# Verify state
value = yield workflow.get_state()
self.assertEqual(value, "up")
self.assertIn(
"processing relation resolved changed", log_output.getvalue())
class UnitAgent(BaseAgent):
"""An juju Unit Agent.
Provides for the management of a charm, via hook execution in response to
external events in the coordination space (zookeeper).
"""
name = "juju-unit-agent"
@classmethod
def setup_options(cls, parser):
super(UnitAgent, cls).setup_options(parser)
unit_name = os.environ.get("JUJU_UNIT_NAME", "")
parser.add_argument("--unit-name", default=unit_name)
@property
def unit_name(self):
return self.config["unit_name"]
def get_agent_name(self):
return "unit:%s" % self.unit_name
def configure(self, options):
"""Configure the unit agent."""
super(UnitAgent, self).configure(options)
if not options.get("unit_name"):
msg = ("--unit-name must be provided in the command line, "
"or $JUJU_UNIT_NAME in the environment")
raise JujuError(msg)
self.executor = HookExecutor()
self.api_factory = UnitSettingsFactory(
self.executor.get_hook_context,
self.executor.get_invoker,
logging.getLogger("unit.hook.api"))
self.api_socket = None
self.workflow = None
@inlineCallbacks
def start(self):
"""Start the unit agent process."""
service_state_manager = ServiceStateManager(self.client)
# Retrieve our unit and configure working directories.
service_name = self.unit_name.split("/")[0]
self.service_state = yield service_state_manager.get_service_state(
service_name)
self.unit_state = yield self.service_state.get_unit_state(
self.unit_name)
self.unit_directory = os.path.join(
self.config["juju_directory"], "units",
self.unit_state.unit_name.replace("/", "-"))
self.state_directory = os.path.join(
self.config["juju_directory"], "state")
# Setup the server portion of the cli api exposed to hooks.
socket_path = os.path.join(self.unit_directory, HOOK_SOCKET_FILE)
if os.path.exists(socket_path):
os.unlink(socket_path)
from twisted.internet import reactor
self.api_socket = reactor.listenUNIX(socket_path, self.api_factory)
# Setup the unit state's address
address = yield get_unit_address(self.client)
yield self.unit_state.set_public_address(
(yield address.get_public_address()))
yield self.unit_state.set_private_address(
(yield address.get_private_address()))
if self.get_watch_enabled():
yield self.unit_state.watch_hook_debug(self.cb_watch_hook_debug)
# Inform the system, we're alive.
yield self.unit_state.connect_agent()
# Start paying attention to the debug-log setting
if self.get_watch_enabled():
yield self.unit_state.watch_hook_debug(self.cb_watch_hook_debug)
self.lifecycle = UnitLifecycle(
self.client, self.unit_state, self.service_state,
self.unit_directory, self.state_directory, self.executor)
self.workflow = UnitWorkflowState(
self.client, self.unit_state, self.lifecycle, self.state_directory)
# Set up correct lifecycle and executor state given the persistent
# unit workflow state, and fire any starting transitions if necessary.
with (yield self.workflow.lock()):
yield self.workflow.synchronize(self.executor)
if self.get_watch_enabled():
yield self.unit_state.watch_resolved(self.cb_watch_resolved)
yield self.service_state.watch_config_state(
self.cb_watch_config_changed)
yield self.unit_state.watch_upgrade_flag(
self.cb_watch_upgrade_flag)
@inlineCallbacks
def stop(self):
"""Stop the unit agent process."""
if self.lifecycle.running:
yield self.lifecycle.stop(fire_hooks=False, stop_relations=False)
yield self.executor.stop()
if self.api_socket:
yield self.api_socket.stopListening()
yield self.api_factory.stopFactory()
@inlineCallbacks
def cb_watch_resolved(self, change):
"""Update the unit's state, when its resolved.
Resolved operations form the basis of error recovery for unit
workflows. A resolved operation can optionally specify hook
execution. The unit agent runs the error recovery transition
if the unit is not in a running state.
"""
# Would be nice if we could fold this into an atomic
# get and delete primitive.
# Check resolved setting
resolved = yield self.unit_state.get_resolved()
if resolved is None:
returnValue(None)
# Clear out the setting
yield self.unit_state.clear_resolved()
with (yield self.workflow.lock()):
if (yield self.workflow.get_state()) == "started":
returnValue(None)
try:
log.info("Resolved detected, firing retry transition")
if resolved["retry"] == RETRY_HOOKS:
yield self.workflow.fire_transition_alias("retry_hook")
else:
yield self.workflow.fire_transition_alias("retry")
except Exception:
log.exception("Unknown error while transitioning for resolved")
@inlineCallbacks
def cb_watch_hook_debug(self, change):
"""Update the hooks to be debugged when the settings change.
"""
debug = yield self.unit_state.get_hook_debug()
debug_hooks = debug and debug.get("debug_hooks") or None
self.executor.set_debug(debug_hooks)
@inlineCallbacks
def cb_watch_upgrade_flag(self, change):
"""Update the unit's charm when requested.
"""
upgrade_flag = yield self.unit_state.get_upgrade_flag()
if not upgrade_flag:
log.info("No upgrade flag set.")
return
log.info("Upgrade detected")
# Clear the flag immediately; this means that upgrade requests will
# be *ignored* by units which are not "started", and will need to be
# reissued when the units are in acceptable states.
yield self.unit_state.clear_upgrade_flag()
new_id = yield self.service_state.get_charm_id()
old_id = yield self.unit_state.get_charm_id()
if new_id == old_id:
log.info("Upgrade ignored: already running latest charm")
return
with (yield self.workflow.lock()):
state = yield self.workflow.get_state()
if state != "started":
if upgrade_flag["force"]:
yield self.lifecycle.upgrade_charm(
fire_hooks=False, force=True)
log.info("Forced upgrade complete")
return
log.warning(
"Cannot upgrade: unit is in non-started state %s. Reissue "
"upgrade command to try again.", state)
return
log.info("Starting upgrade")
if (yield self.workflow.fire_transition("upgrade_charm")):
log.info("Upgrade complete")
else:
log.info("Upgrade failed")
@inlineCallbacks
def cb_watch_config_changed(self, change):
"""Trigger hook on configuration change"""
# Verify it is running
with (yield self.workflow.lock()):
current_state = yield self.workflow.get_state()
log.debug("Configuration Changed")
if current_state != "started":
log.debug(
"Configuration updated on service in a non-started state")
returnValue(None)
yield self.workflow.fire_transition("configure")
class UnitLifecycleTest(LifecycleTestBase):
@inlineCallbacks
def setUp(self):
yield super(UnitLifecycleTest, self).setUp()
yield self.setup_default_test_relation()
self.lifecycle = UnitLifecycle(self.client, self.states["unit"],
self.states["service"],
self.unit_directory, self.executor)
@inlineCallbacks
def test_hook_invocation(self):
"""Verify lifecycle methods invoke corresponding charm hooks.
"""
# install hook
file_path = self.makeFile()
self.write_hook("install",
'#!/bin/sh\n echo "hello world" > %s' % file_path)
yield self.lifecycle.install()
self.assertEqual(open(file_path).read().strip(), "hello world")
# Start hook
file_path = self.makeFile()
self.write_hook("start",
'#!/bin/sh\n echo "sugarcane" > %s' % file_path)
yield self.lifecycle.start()
self.assertEqual(open(file_path).read().strip(), "sugarcane")
# Stop hook
file_path = self.makeFile()
self.write_hook("stop", '#!/bin/sh\n echo "siesta" > %s' % file_path)
yield self.lifecycle.stop()
self.assertEqual(open(file_path).read().strip(), "siesta")
# verify the sockets are cleaned up.
self.assertEqual(os.listdir(self.unit_directory), ["charm"])
@inlineCallbacks
def test_start_sans_hook(self):
"""The lifecycle start can be invoked without firing hooks."""
self.write_hook("start", "#!/bin/sh\n exit 1")
start_executed = self.wait_on_hook("start")
yield self.lifecycle.start(fire_hooks=False)
self.assertFalse(start_executed.called)
@inlineCallbacks
def test_stop_sans_hook(self):
"""The lifecycle stop can be invoked without firing hooks."""
self.write_hook("stop", "#!/bin/sh\n exit 1")
stop_executed = self.wait_on_hook("stop")
yield self.lifecycle.start()
yield self.lifecycle.stop(fire_hooks=False)
self.assertFalse(stop_executed.called)
@inlineCallbacks
def test_install_sans_hook(self):
"""The lifecycle install can be invoked without firing hooks."""
self.write_hook("install", "#!/bin/sh\n exit 1")
install_executed = self.wait_on_hook("install")
yield self.lifecycle.install(fire_hooks=False)
self.assertFalse(install_executed.called)
@inlineCallbacks
def test_upgrade_sans_hook(self):
"""The lifecycle upgrade can be invoked without firing hooks."""
self.executor.stop()
self.write_hook("upgrade-charm", "#!/bin/sh\n exit 1")
upgrade_executed = self.wait_on_hook("upgrade-charm")
yield self.lifecycle.upgrade_charm(fire_hooks=False)
self.assertFalse(upgrade_executed.called)
self.assertTrue(self.executor.running)
def test_hook_error(self):
"""Verify hook execution error, raises an exception."""
self.write_hook("install", '#!/bin/sh\n exit 1')
d = self.lifecycle.install()
return self.failUnlessFailure(d, CharmInvocationError)
def test_hook_not_executable(self):
"""A hook not executable, raises an exception."""
self.write_hook("install", '#!/bin/sh\n exit 0', no_exec=True)
return self.failUnlessFailure(self.lifecycle.install(), CharmError)
def test_hook_not_formatted_correctly(self):
"""Hook execution error, raises an exception."""
self.write_hook("install", '!/bin/sh\n exit 0')
return self.failUnlessFailure(self.lifecycle.install(),
CharmInvocationError)
def write_start_and_relation_hooks(self, relation_name=None):
"""Write some minimal start, and relation-changed hooks.
Returns the output file of the relation hook.
"""
file_path = self.makeFile()
if relation_name is None:
relation_name = self.states["service_relation"].relation_name
self.write_hook("start", ("#!/bin/bash\n" "echo hello"))
self.write_hook("config-changed", ("#!/bin/bash\n" "echo configure"))
self.write_hook("stop", ("#!/bin/bash\n" "echo goodbye"))
self.write_hook("%s-relation-joined" % relation_name,
("#!/bin/bash\n"
"echo joined >> %s\n" % file_path))
self.write_hook("%s-relation-changed" % relation_name,
("#!/bin/bash\n"
"echo changed >> %s\n" % file_path))
self.write_hook("%s-relation-departed" % relation_name,
("#!/bin/bash\n"
"echo departed >> %s\n" % file_path))
self.assertFalse(os.path.exists(file_path))
return file_path
@inlineCallbacks
def test_upgrade_hook_invoked_on_upgrade_charm(self):
"""Invoking the upgrade_charm lifecycle method executes the
upgrade-charm hook.
"""
file_path = self.makeFile("")
self.write_hook("upgrade-charm", ("#!/bin/bash\n"
"echo upgraded >> %s\n" % file_path))
# upgrade requires the external actor that extracts the charm
# to stop the hook executor, prior to extraction so the
# upgrade is the first hook run.
yield self.executor.stop()
yield self.lifecycle.upgrade_charm()
self.assertEqual(open(file_path).read().strip(), "upgraded")
@inlineCallbacks
def test_config_hook_invoked_on_configure(self):
"""Invoke the configure lifecycle method will execute the
config-changed hook.
"""
output = self.capture_logging("unit.lifecycle", level=logging.DEBUG)
# configure hook requires a running unit lifecycle..
yield self.assertFailure(self.lifecycle.configure(), AssertionError)
# Config hook
file_path = self.makeFile()
self.write_hook("config-changed",
'#!/bin/sh\n echo "palladium" > %s' % file_path)
yield self.lifecycle.start()
yield self.lifecycle.configure()
self.assertEqual(open(file_path).read().strip(), "palladium")
self.assertIn("configured unit", output.getvalue())
@inlineCallbacks
def test_service_relation_watching(self):
"""When the unit lifecycle is started, the assigned relations
of the service are watched, with unit relation lifecycles
created for each.
Relation hook invocation do not maintain global order or determinism
across relations. They only maintain ordering and determinism within
a relation. A shared scheduler across relations would be needed to
maintain such behavior.
"""
file_path = self.write_start_and_relation_hooks()
wordpress1_states = yield self.add_opposite_service_unit(self.states)
yield self.lifecycle.start()
yield self.wait_on_hook("app-relation-changed")
self.assertTrue(os.path.exists(file_path))
self.assertEqual([x.strip() for x in open(file_path).readlines()],
["joined", "changed"])
# Queue up our wait condition, of 4 hooks firing
hooks_complete = self.wait_on_hook(sequence=[
"app-relation-joined", # joined event fires join hook,
"app-relation-changed", # followed by changed hook
"app-relation-changed",
"app-relation-departed"
])
# add another.
wordpress2_states = yield self.add_opposite_service_unit(
(yield self.add_relation_service_unit_to_another_endpoint(
self.states,
RelationEndpoint("wordpress-2", "client-server", "db",
"client"))))
# modify one.
wordpress1_states["unit_relation"].set_data({"hello": "world"})
# delete one.
self.client.delete("/relations/%s/client/%s" %
(wordpress2_states["relation"].internal_id,
wordpress2_states["unit"].internal_id))
# verify results, waiting for hooks to complete
yield hooks_complete
self.assertEqual(
set([x.strip() for x in open(file_path).readlines()]),
set(["joined", "changed", "joined", "changed", "departed"]))
@inlineCallbacks
def test_removed_relation_depart(self):
"""
If a removed relation is detected, the unit relation lifecycle is
stopped.
"""
file_path = self.write_start_and_relation_hooks()
self.write_hook("app-relation-broken", "#!/bin/bash\n echo broken")
yield self.lifecycle.start()
wordpress_states = yield self.add_opposite_service_unit(self.states)
# Wait for the watch and hook to fire.
yield self.wait_on_hook("app-relation-changed")
self.assertTrue(os.path.exists(file_path))
self.assertEqual([x.strip() for x in open(file_path).readlines()],
["joined", "changed"])
self.assertTrue(
self.lifecycle.get_relation_workflow(
self.states["relation"].internal_id))
# Remove the relation between mysql and wordpress
yield self.relation_manager.remove_relation_state(
self.states["relation"])
# Wait till the unit relation workflow has been processed the event.
yield self.wait_on_state(
self.lifecycle.get_relation_workflow(
self.states["relation"].internal_id), "departed")
# Modify the unit relation settings, to generate a spurious event.
yield wordpress_states["unit_relation"].set_data({"hello": "world"})
# Verify no notice was recieved for the modify before we where stopped.
self.assertEqual([x.strip() for x in open(file_path).readlines()],
["joined", "changed"])
# Verify the unit relation lifecycle has been disposed of.
self.assertRaises(KeyError, self.lifecycle.get_relation_workflow,
self.states["relation"].internal_id)
@inlineCallbacks
def test_lifecycle_start_stop_starts_relations(self):
"""Starting a stopped lifecycle, restarts relation events.
"""
wordpress1_states = yield self.add_opposite_service_unit(self.states)
wordpress2_states = yield self.add_opposite_service_unit(
(yield self.add_relation_service_unit_to_another_endpoint(
self.states,
RelationEndpoint("wordpress-2", "client-server", "db",
"client"))))
# Start and stop lifecycle
file_path = self.write_start_and_relation_hooks()
yield self.lifecycle.start()
yield self.wait_on_hook("app-relation-changed")
self.assertTrue(os.path.exists(file_path))
yield self.lifecycle.stop()
########################################################
# Add, remove relations, and modify related unit settings.
# The following isn't enough to trigger a hook notification.
# yield wordpress1_states["relation"].unassign_service(
# wordpress1_states["service"])
#
# The removal of the external relation, means we stop getting notifies
# of it, but the underlying unit agents of the service are responsible
# for removing their presence nodes within the relationship, which
# triggers a hook invocation.
yield self.client.delete("/relations/%s/client/%s" %
(wordpress1_states["relation"].internal_id,
wordpress1_states["unit"].internal_id))
yield wordpress2_states["unit_relation"].set_data({"hello": "world"})
yield self.add_opposite_service_unit(
(yield self.add_relation_service_unit_to_another_endpoint(
self.states,
RelationEndpoint("wordpress-3", "client-server", "db",
"client"))))
# Verify no hooks are executed.
yield self.sleep(0.1)
res = [x.strip() for x in open(file_path)]
if ((res != ["joined", "changed", "joined", "changed"])
and (res != ["joined", "joined", "changed", "changed"])):
self.fail("Invalid join sequence %s" % res)
# XXX - With scheduler local state recovery, we should get the modify.
# Start and verify events.
hooks_executed = self.wait_on_hook(sequence=[
"config-changed",
"start",
"app-relation-departed",
"app-relation-joined", # joined event fires joined hook,
"app-relation-changed" # followed by changed hook
])
yield self.lifecycle.start()
yield hooks_executed
res.extend(["departed", "joined", "changed"])
self.assertEqual([x.strip() for x in open(file_path)], res)
@inlineCallbacks
def test_lock_start_stop_watch(self):
"""The lifecycle, internally employs lock to prevent simulatenous
execution of methods which modify internal state. This allows
for a long running hook to be called safely, even if the other
invocations of the lifecycle, the subsequent invocations will
block till they can acquire the lock.
"""
self.write_hook("start", "#!/bin/bash\necho start\n")
self.write_hook("stop", "#!/bin/bash\necho stop\n")
results = []
finish_callback = [Deferred() for i in range(4)]
# Control the speed of hook execution
original_invoker = Invoker.__call__
invoker = self.mocker.patch(Invoker)
@inlineCallbacks
def long_hook(ctx, hook_path):
results.append(os.path.basename(hook_path))
yield finish_callback[len(results) - 1]
yield original_invoker(ctx, hook_path)
for i in range(4):
invoker(MATCH(lambda x: x.endswith("start") or x.endswith("stop")))
self.mocker.call(long_hook, with_object=True)
self.mocker.replay()
# Hook execution sequence to match on.
test_complete = self.wait_on_hook(sequence=[
"config-changed", "start", "stop", "config-changed", "start"
])
# Fire off the lifecycle methods
execution_callbacks = [
self.lifecycle.start(),
self.lifecycle.stop(),
self.lifecycle.start(),
self.lifecycle.stop()
]
self.assertEqual([0, 0, 0, 0], [x.called for x in execution_callbacks])
# kill the delay on the second
finish_callback[1].callback(True)
finish_callback[2].callback(True)
self.assertEqual([0, 0, 0, 0], [x.called for x in execution_callbacks])
# let them pass, kill the delay on the first
finish_callback[0].callback(True)
yield test_complete
self.assertEqual([False, True, True, False],
[x.called for x in execution_callbacks])
# Finish the last hook
finish_callback[3].callback(True)
yield self.wait_on_hook("stop")
self.assertEqual([True, True, True, True],
[x.called for x in execution_callbacks])
class LifecycleResolvedTest(LifecycleTestBase):
@inlineCallbacks
def setUp(self):
yield super(LifecycleResolvedTest, self).setUp()
yield self.setup_default_test_relation()
self.lifecycle = UnitLifecycle(self.client, self.states["unit"],
self.states["service"],
self.unit_directory, self.executor)
def get_unit_relation_workflow(self, states):
state_dir = os.path.join(self.juju_directory, "state")
lifecycle = UnitRelationLifecycle(
self.client, states["unit_relation"],
states["service_relation"].relation_name, self.unit_directory,
self.executor)
workflow = RelationWorkflowState(self.client, states["unit_relation"],
lifecycle, state_dir)
return (workflow, lifecycle)
@inlineCallbacks
def wb_test_start_with_relation_errors(self):
"""
White box testing to ensure that an error when starting the
lifecycle is propogated appropriately, and that we collect
all results before returning.
"""
mock_service = self.mocker.patch(self.lifecycle._service)
mock_service.watch_relation_states(MATCH(lambda x: callable(x)))
self.mocker.result(fail(SyntaxError()))
mock_unit = self.mocker.patch(self.lifecycle._unit)
mock_unit.watch_relation_resolved(MATCH(lambda x: callable(x)))
results = []
wait = Deferred()
@inlineCallbacks
def complete(*args):
yield wait
results.append(True)
returnValue(True)
self.mocker.call(complete)
self.mocker.replay()
# Start the unit, assert a failure, and capture the deferred
wait_failure = self.assertFailure(self.lifecycle.start(), SyntaxError)
# Verify we have no results for the second callback or the start call
self.assertFalse(results)
self.assertFalse(wait_failure.called)
# Let the second callback complete
wait.callback(True)
# Wait for the start error to bubble up.
yield wait_failure
# Verify the second deferred was waited on.
self.assertTrue(results)
@inlineCallbacks
def test_resolved_relation_watch_unit_lifecycle_not_running(self):
"""If the unit is not running then no relation resolving is performed.
However the resolution value remains the same.
"""
# Start the unit.
yield self.lifecycle.start()
# Simulate relation down on an individual unit relation
workflow = self.lifecycle.get_relation_workflow(
self.states["unit_relation"].internal_relation_id)
self.assertEqual("up", (yield workflow.get_state()))
yield workflow.transition_state("down")
resolved = self.wait_on_state(workflow, "up")
# Stop the unit lifecycle
yield self.lifecycle.stop()
# Set the relation to resolved
yield self.states["unit"].set_relation_resolved(
{self.states["unit_relation"].internal_relation_id: NO_HOOKS})
# Give a moment for the watch to fire erroneously
yield self.sleep(0.2)
# Ensure we didn't attempt a transition.
self.assertFalse(resolved.called)
self.assertEqual(
{self.states["unit_relation"].internal_relation_id: NO_HOOKS},
(yield self.states["unit"].get_relation_resolved()))
# If the unit is restarted start, we currently have the
# behavior that the unit relation workflow will automatically
# be transitioned back to running, as part of the normal state
# transition. Sigh.. we should have a separate error
# state for relation hooks then down with state variable usage.
# The current end behavior though seems like the best outcome, ie.
# automatically restart relations.
@inlineCallbacks
def test_resolved_relation_watch_relation_up(self):
"""If a relation marked as to be resolved is already running,
then no work is performed.
"""
# Start the unit.
yield self.lifecycle.start()
# get a hold of the unit relation and verify state
workflow = self.lifecycle.get_relation_workflow(
self.states["unit_relation"].internal_relation_id)
self.assertEqual("up", (yield workflow.get_state()))
# Set the relation to resolved
yield self.states["unit"].set_relation_resolved(
{self.states["unit_relation"].internal_relation_id: NO_HOOKS})
# Give a moment for the watch to fire, invoke callback, and reset.
yield self.sleep(0.1)
# Ensure we're still up and the relation resolved setting has been
# cleared.
self.assertEqual(None,
(yield self.states["unit"].get_relation_resolved()))
self.assertEqual("up", (yield workflow.get_state()))
@inlineCallbacks
def test_resolved_relation_watch_from_error(self):
"""Unit lifecycle's will process a unit relation resolved
setting, and transition a down relation back to a running
state.
"""
log_output = self.capture_logging("unit.lifecycle",
level=logging.DEBUG)
# Start the unit.
yield self.lifecycle.start()
# Simulate an error condition
workflow = self.lifecycle.get_relation_workflow(
self.states["unit_relation"].internal_relation_id)
self.assertEqual("up", (yield workflow.get_state()))
yield workflow.fire_transition("error")
resolved = self.wait_on_state(workflow, "up")
# Set the relation to resolved
yield self.states["unit"].set_relation_resolved(
{self.states["unit_relation"].internal_relation_id: NO_HOOKS})
# Wait for the relation to come back up
value = yield self.states["unit"].get_relation_resolved()
yield resolved
# Verify state
value = yield workflow.get_state()
self.assertEqual(value, "up")
self.assertIn("processing relation resolved changed",
log_output.getvalue())
@inlineCallbacks
def test_resolved_relation_watch(self):
"""Unit lifecycle's will process a unit relation resolved
setting, and transition a down relation back to a running
state.
"""
log_output = self.capture_logging("unit.lifecycle",
level=logging.DEBUG)
# Start the unit.
yield self.lifecycle.start()
# Simulate an error condition
workflow = self.lifecycle.get_relation_workflow(
self.states["unit_relation"].internal_relation_id)
self.assertEqual("up", (yield workflow.get_state()))
yield workflow.transition_state("down")
resolved = self.wait_on_state(workflow, "up")
# Set the relation to resolved
yield self.states["unit"].set_relation_resolved(
{self.states["unit_relation"].internal_relation_id: NO_HOOKS})
# Wait for the relation to come back up
value = yield self.states["unit"].get_relation_resolved()
yield resolved
# Verify state
value = yield workflow.get_state()
self.assertEqual(value, "up")
self.assertIn("processing relation resolved changed",
log_output.getvalue())