def test_watch_machine_changes_ignores_running_machine(self):
"""
If there is an existing machine instance and state, when a
new machine state is added, the existing instance is preserved,
and a new instance is created.
"""
manager = MachineStateManager(self.client)
machine_state0 = yield manager.add_machine_state()
machines = yield self.agent.provider.start_machine(
{"machine-id": machine_state0.id})
machine = machines.pop()
yield machine_state0.set_instance_id(machine.instance_id)
machine_state1 = yield manager.add_machine_state()
machines = yield self.agent.provider.get_machines()
self.assertEquals(len(machines), 1)
yield self.agent.watch_machine_changes(
None, [machine_state0.id, machine_state1.id])
machines = yield self.agent.provider.get_machines()
self.assertEquals(len(machines), 2)
instance_id = yield machine_state1.get_instance_id()
self.assertEqual(instance_id, 1)
def test_watch_machine_changes_ignores_running_machine(self):
"""
If there is an existing machine instance and state, when a
new machine state is added, the existing instance is preserved,
and a new instance is created.
"""
manager = MachineStateManager(self.client)
machine_state0 = yield manager.add_machine_state()
machines = yield self.agent.provider.start_machine(
{"machine-id": machine_state0.id})
machine = machines.pop()
yield machine_state0.set_instance_id(machine.instance_id)
machine_state1 = yield manager.add_machine_state()
machines = yield self.agent.provider.get_machines()
self.assertEquals(len(machines), 1)
yield self.agent.watch_machine_changes(
None, [machine_state0.id, machine_state1.id])
machines = yield self.agent.provider.get_machines()
self.assertEquals(len(machines), 2)
instance_id = yield machine_state1.get_instance_id()
self.assertEqual(instance_id, 1)
def test_transient_provider_error_on_get_machines(self):
manager = MachineStateManager(self.client)
machine_state0 = yield manager.add_machine_state()
mock_provider = self.mocker.patch(self.agent.provider)
mock_provider.get_machines()
self.mocker.result(fail(ProviderInteractionError()))
mock_provider.get_machines()
self.mocker.passthrough()
self.mocker.replay()
try:
yield self.agent.process_machines([machine_state0.id])
except:
self.fail("Should not raise")
instance_id = yield machine_state0.get_instance_id()
self.assertEqual(instance_id, None)
yield self.agent.process_machines([machine_state0.id])
instance_id = yield machine_state0.get_instance_id()
self.assertEqual(instance_id, 0)
self.assertIn("Cannot get machine list", self.output.getvalue())
def test_open_close_ports_on_machine_not_yet_provided(self):
"""Verify that opening/closing ports will eventually succeed
once a machine is provided.
"""
manager = MachineStateManager(self.client)
machine = yield manager.add_machine_state()
wordpress = yield self.add_service("wordpress")
yield wordpress.set_exposed_flag()
wordpress_0 = yield wordpress.add_unit_state()
yield wordpress_0.open_port(80, "tcp")
yield wordpress_0.open_port(443, "tcp")
yield wordpress_0.assign_to_machine(machine)
# First attempt to open ports quietly fails (except for
# logging) because the machine has not yet been provisioned
yield self.firewall_manager.open_close_ports_on_machine(machine.id)
self.assertIn("No provisioned machine for machine 0",
self.output.getvalue())
yield self.provide_machine(machine)
# Machine is now provisioned (normally visible in the
# provisioning agent through periodic rescan and corresponding
# watches)
yield self.firewall_manager.open_close_ports_on_machine(machine.id)
self.assertEqual((yield self.get_provider_ports(machine)),
set([(80, "tcp"), (443, "tcp")]))
def test_open_close_ports_on_machine_not_yet_provided(self):
"""Verify that opening/closing ports will eventually succeed
once a machine is provided.
"""
manager = MachineStateManager(self.client)
machine = yield manager.add_machine_state()
wordpress = yield self.add_service("wordpress")
yield wordpress.set_exposed_flag()
wordpress_0 = yield wordpress.add_unit_state()
yield wordpress_0.open_port(80, "tcp")
yield wordpress_0.open_port(443, "tcp")
yield wordpress_0.assign_to_machine(machine)
# First attempt to open ports quietly fails (except for
# logging) because the machine has not yet been provisioned
yield self.firewall_manager.open_close_ports_on_machine(machine.id)
self.assertIn("No provisioned machine for machine 0",
self.output.getvalue())
yield self.provide_machine(machine)
# Machine is now provisioned (normally visible in the
# provisioning agent through periodic rescan and corresponding
# watches)
yield self.firewall_manager.open_close_ports_on_machine(machine.id)
self.assertEqual((yield self.get_provider_ports(machine)),
set([(80, "tcp"), (443, "tcp")]))
def test_transient_provider_error_on_get_machines(self):
manager = MachineStateManager(self.client)
machine_state0 = yield manager.add_machine_state()
mock_provider = self.mocker.patch(self.agent.provider)
mock_provider.get_machines()
self.mocker.result(fail(ProviderInteractionError()))
mock_provider.get_machines()
self.mocker.passthrough()
self.mocker.replay()
try:
yield self.agent.process_machines([machine_state0.id])
except:
self.fail("Should not raise")
instance_id = yield machine_state0.get_instance_id()
self.assertEqual(instance_id, None)
yield self.agent.process_machines(
[machine_state0.id])
instance_id = yield machine_state0.get_instance_id()
self.assertEqual(instance_id, 0)
self.assertIn(
"Cannot get machine list",
self.output.getvalue())
def test_open_close_ports_on_machine(self):
"""Verify opening/closing ports on a machine works properly.
In particular this is done without watch support."""
manager = MachineStateManager(self.client)
machine = yield manager.add_machine_state()
yield self.firewall_manager.process_machine(machine)
# Expose a service
wordpress = yield self.add_service("wordpress")
yield wordpress.set_exposed_flag()
wordpress_0 = yield wordpress.add_unit_state()
yield wordpress_0.open_port(80, "tcp")
yield wordpress_0.open_port(443, "tcp")
yield wordpress_0.assign_to_machine(machine)
yield self.firewall_manager.open_close_ports_on_machine(machine.id)
self.assertEqual((yield self.get_provider_ports(machine)),
set([(80, "tcp"), (443, "tcp")]))
self.assertIn("Opened 80/tcp on provider machine 0",
self.output.getvalue())
self.assertIn("Opened 443/tcp on provider machine 0",
self.output.getvalue())
# Now change port setup
yield wordpress_0.open_port(8080, "tcp")
yield wordpress_0.close_port(443, "tcp")
yield self.firewall_manager.open_close_ports_on_machine(machine.id)
self.assertEqual((yield self.get_provider_ports(machine)),
set([(80, "tcp"), (8080, "tcp")]))
self.assertIn("Opened 8080/tcp on provider machine 0",
self.output.getvalue())
self.assertIn("Closed 443/tcp on provider machine 0",
self.output.getvalue())
def test_process_machines_non_concurrency(self):
"""
Process machines should only be executed serially by an
agent.
"""
manager = MachineStateManager(self.client)
machine_state0 = yield manager.add_machine_state()
machine_state1 = yield manager.add_machine_state()
call_1 = self.agent.process_machines([machine_state0.id])
# The second call should return immediately due to the
# instance attribute guard.
call_2 = self.agent.process_machines([machine_state1.id])
self.assertEqual(call_2.called, True)
self.assertEqual(call_2.result, False)
# The first call should have started a provider machine
yield call_1
machines = yield self.agent.provider.get_machines()
self.assertEquals(len(machines), 1)
instance_id_0 = yield machine_state0.get_instance_id()
self.assertEqual(instance_id_0, 0)
instance_id_1 = yield machine_state1.get_instance_id()
self.assertEqual(instance_id_1, None)
def test_process_machines_non_concurrency(self):
"""
Process machines should only be executed serially by an
agent.
"""
manager = MachineStateManager(self.client)
machine_state0 = yield manager.add_machine_state()
machine_state1 = yield manager.add_machine_state()
call_1 = self.agent.process_machines([machine_state0.id])
# The second call should return immediately due to the
# instance attribute guard.
call_2 = self.agent.process_machines([machine_state1.id])
self.assertEqual(call_2.called, True)
self.assertEqual(call_2.result, False)
# The first call should have started a provider machine
yield call_1
machines = yield self.agent.provider.get_machines()
self.assertEquals(len(machines), 1)
instance_id_0 = yield machine_state0.get_instance_id()
self.assertEqual(instance_id_0, 0)
instance_id_1 = yield machine_state1.get_instance_id()
self.assertEqual(instance_id_1, None)
def test_transient_provider_error_on_start_machine(self):
"""
If there's an error when processing changes, the agent should log
the error and continue.
"""
manager = MachineStateManager(self.client)
machine_state0 = yield manager.add_machine_state()
machine_state1 = yield manager.add_machine_state()
mock_provider = self.mocker.patch(self.agent.provider)
mock_provider.start_machine({"machine-id": 0})
self.mocker.result(fail(ProviderInteractionError()))
mock_provider.start_machine({"machine-id": 1})
self.mocker.passthrough()
self.mocker.replay()
yield self.agent.watch_machine_changes(
[], [machine_state0.id, machine_state1.id])
machine1_instance_id = yield machine_state1.get_instance_id()
self.assertEqual(machine1_instance_id, 0)
self.assertIn(
"Cannot process machine 0",
self.output.getvalue())
def test_open_close_ports_on_machine(self):
"""Verify opening/closing ports on a machine works properly.
In particular this is done without watch support."""
manager = MachineStateManager(self.client)
machine = yield manager.add_machine_state()
yield self.firewall_manager.process_machine(machine)
# Expose a service
wordpress = yield self.add_service("wordpress")
yield wordpress.set_exposed_flag()
wordpress_0 = yield wordpress.add_unit_state()
yield wordpress_0.open_port(80, "tcp")
yield wordpress_0.open_port(443, "tcp")
yield wordpress_0.assign_to_machine(machine)
yield self.firewall_manager.open_close_ports_on_machine(machine.id)
self.assertEqual((yield self.get_provider_ports(machine)),
set([(80, "tcp"), (443, "tcp")]))
self.assertIn("Opened 80/tcp on provider machine 0",
self.output.getvalue())
self.assertIn("Opened 443/tcp on provider machine 0",
self.output.getvalue())
# Now change port setup
yield wordpress_0.open_port(8080, "tcp")
yield wordpress_0.close_port(443, "tcp")
yield self.firewall_manager.open_close_ports_on_machine(machine.id)
self.assertEqual((yield self.get_provider_ports(machine)),
set([(80, "tcp"), (8080, "tcp")]))
self.assertIn("Opened 8080/tcp on provider machine 0",
self.output.getvalue())
self.assertIn("Closed 443/tcp on provider machine 0",
self.output.getvalue())
def test_open_close_ports_on_unassigned_machine(self):
"""Verify corner case that nothing happens on an unassigned machine."""
manager = MachineStateManager(self.client)
machine = yield manager.add_machine_state()
yield self.provide_machine(machine)
yield self.firewall_manager.process_machine(machine)
yield self.firewall_manager.open_close_ports_on_machine(machine.id)
self.assertEqual((yield self.get_provider_ports(machine)), set())
def test_open_close_ports_on_unassigned_machine(self):
"""Verify corner case that nothing happens on an unassigned machine."""
manager = MachineStateManager(self.client)
machine = yield manager.add_machine_state()
yield self.provide_machine(machine)
yield self.firewall_manager.process_machine(machine)
yield self.firewall_manager.open_close_ports_on_machine(machine.id)
self.assertEqual((yield self.get_provider_ports(machine)),
set())
def test_machine_state_reflects_invalid_provider_state(self):
"""
If a machine state has an invalid instance_id, it should be detected,
and a new machine started and the machine state updated with the
new instance_id.
"""
machine_manager = MachineStateManager(self.client)
m1 = yield machine_manager.add_machine_state()
yield m1.set_instance_id("zebra")
m2 = yield machine_manager.add_machine_state()
yield self.agent.watch_machine_changes(None, [m1.id, m2.id])
m1_instance_id = yield m1.get_instance_id()
self.assertEqual(m1_instance_id, 0)
m2_instance_id = yield m2.get_instance_id()
self.assertEqual(m2_instance_id, 1)
def test_machine_state_reflects_invalid_provider_state(self):
"""
If a machine state has an invalid instance_id, it should be detected,
and a new machine started and the machine state updated with the
new instance_id.
"""
machine_manager = MachineStateManager(self.client)
m1 = yield machine_manager.add_machine_state()
yield m1.set_instance_id("zebra")
m2 = yield machine_manager.add_machine_state()
yield self.agent.watch_machine_changes(None, [m1.id, m2.id])
m1_instance_id = yield m1.get_instance_id()
self.assertEqual(m1_instance_id, 0)
m2_instance_id = yield m2.get_instance_id()
self.assertEqual(m2_instance_id, 1)
class ProvisioningTestBase(AgentTestBase):
agent_class = ProvisioningAgent
@inlineCallbacks
def setUp(self):
yield super(ProvisioningTestBase, self).setUp()
self.machine_manager = MachineStateManager(self.client)
def add_machine_state(self, constraints=None):
return self.machine_manager.add_machine_state(
constraints or series_constraints)
def get_agent_config(self):
# gets invoked by AgentTestBase.setUp
options = yield super(MachineAgentTest, self).get_agent_config()
machine_state_manager = MachineStateManager(self.client)
self.machine_state = yield machine_state_manager.add_machine_state()
self.change_environment(JUJU_MACHINE_ID="0",
JUJU_HOME=self.juju_directory)
options["machine_id"] = str(self.machine_state.id)
# Start the agent with watching enabled
returnValue(options)
def test_transient_unhandled_error_in_process_machines(self):
"""Verify that watch_machine_changes handles the exception.
Provider implementations may use libraries like txaws that do
not handle every error. However, this should not stop the
watch from re-establishing itself, as will be the case if the
exception is not caught.
"""
manager = MachineStateManager(self.client)
machine_state0 = yield manager.add_machine_state()
machine_state1 = yield manager.add_machine_state()
# Simulate a failure scenario seen occasionally when working
# with OpenStack and txaws
mock_agent = self.mocker.patch(self.agent)
# Simulate transient error
mock_agent.process_machines([machine_state0.id])
self.mocker.result(fail(
TypeError("'NoneType' object is not iterable")))
# Let it succeed on second try. In this case, the scenario is
# that the watch triggered before the periodic_machine_check
# was run again
mock_agent.process_machines([machine_state0.id, machine_state1.id])
self.mocker.passthrough()
self.mocker.replay()
# Verify that watch_machine_changes does not fail even in the case of
# the transient error, although no work was done
try:
yield self.agent.watch_machine_changes([], [machine_state0.id])
except:
self.fail("Should not raise")
instance_id = yield machine_state0.get_instance_id()
self.assertEqual(instance_id, None)
# Second attempt, verifiy it did in fact process the machine
yield self.agent.watch_machine_changes(
[machine_state0.id], [machine_state0.id, machine_state1.id])
self.assertEqual((yield machine_state0.get_instance_id()), 0)
self.assertEqual((yield machine_state1.get_instance_id()), 1)
# But only after attempting and failing the first time
self.assertIn(
"Got unexpected exception in processing machines, will retry",
self.output.getvalue())
self.assertIn(
"'NoneType' object is not iterable",
self.output.getvalue())
def test_transient_unhandled_error_in_process_machines(self):
"""Verify that watch_machine_changes handles the exception.
Provider implementations may use libraries like txaws that do
not handle every error. However, this should not stop the
watch from re-establishing itself, as will be the case if the
exception is not caught.
"""
manager = MachineStateManager(self.client)
machine_state0 = yield manager.add_machine_state()
machine_state1 = yield manager.add_machine_state()
# Simulate a failure scenario seen occasionally when working
# with OpenStack and txaws
mock_agent = self.mocker.patch(self.agent)
# Simulate transient error
mock_agent.process_machines([machine_state0.id])
self.mocker.result(fail(
TypeError("'NoneType' object is not iterable")))
# Let it succeed on second try. In this case, the scenario is
# that the watch triggered before the periodic_machine_check
# was run again
mock_agent.process_machines([machine_state0.id, machine_state1.id])
self.mocker.passthrough()
self.mocker.replay()
# Verify that watch_machine_changes does not fail even in the case of
# the transient error, although no work was done
try:
yield self.agent.watch_machine_changes([], [machine_state0.id])
except:
self.fail("Should not raise")
instance_id = yield machine_state0.get_instance_id()
self.assertEqual(instance_id, None)
# Second attempt, verifiy it did in fact process the machine
yield self.agent.watch_machine_changes(
[machine_state0.id], [machine_state0.id, machine_state1.id])
self.assertEqual((yield machine_state0.get_instance_id()), 0)
self.assertEqual((yield machine_state1.get_instance_id()), 1)
# But only after attempting and failing the first time
self.assertIn(
"Got unexpected exception in processing machines, will retry",
self.output.getvalue())
self.assertIn("'NoneType' object is not iterable",
self.output.getvalue())
def get_agent_config(self):
# gets invoked by AgentTestBase.setUp
options = yield super(MachineAgentTest, self).get_agent_config()
machine_state_manager = MachineStateManager(self.client)
self.machine_state = yield machine_state_manager.add_machine_state()
self.change_environment(
JUJU_MACHINE_ID="0",
JUJU_HOME=self.juju_directory)
options["machine_id"] = str(self.machine_state.id)
# Start the agent with watching enabled
returnValue(options)
def test_open_close_ports_on_machine_will_retry(self):
"""Verify port mgmt for a machine will retry if there's a failure."""
mock_provider = self.mocker.patch(MachineProvider)
mock_provider.open_port(MATCH_MACHINE, 0, 80, "tcp")
self.mocker.result(fail(
TypeError("'NoneType' object is not iterable")))
mock_provider.open_port(MATCH_MACHINE, 0, 80, "tcp")
self.mocker.result(fail(
ProviderInteractionError("Some sort of EC2 problem")))
mock_provider.open_port(MATCH_MACHINE, 0, 80, "tcp")
self.mocker.passthrough()
self.mocker.replay()
manager = MachineStateManager(self.client)
machine = yield manager.add_machine_state()
yield self.provide_machine(machine)
# Expose a service and attempt to open/close ports. The first
# attempt will see the simulated failure.
wordpress = yield self.add_service("wordpress")
yield wordpress.set_exposed_flag()
wordpress_0 = yield wordpress.add_unit_state()
yield wordpress_0.assign_to_machine(machine)
yield self.firewall_manager.process_machine(machine)
yield wordpress_0.open_port(80, "tcp")
yield self.firewall_manager.open_close_ports_on_machine(machine.id)
self.assertEqual((yield self.get_provider_ports(machine)),
set())
self.assertIn(
"Got exception in opening/closing ports, will retry",
self.output.getvalue())
self.assertIn("TypeError: 'NoneType' object is not iterable",
self.output.getvalue())
# Retries will now happen in the periodic recheck. First one
# still fails due to simulated error.
yield self.firewall_manager.process_machine(machine)
self.assertEqual((yield self.get_provider_ports(machine)),
set())
self.assertIn("ProviderInteractionError: Some sort of EC2 problem",
self.output.getvalue())
# Third time is the charm in the mock setup, the recheck succeeds
yield self.firewall_manager.process_machine(machine)
self.assertEqual((yield self.get_provider_ports(machine)),
set([(80, "tcp")]))
self.assertIn("Opened 80/tcp on provider machine 0",
self.output.getvalue())
def test_open_close_ports_on_machine_will_retry(self):
"""Verify port mgmt for a machine will retry if there's a failure."""
mock_provider = self.mocker.patch(MachineProvider)
mock_provider.open_port(MATCH_MACHINE, 0, 80, "tcp")
self.mocker.result(fail(
TypeError("'NoneType' object is not iterable")))
mock_provider.open_port(MATCH_MACHINE, 0, 80, "tcp")
self.mocker.result(
fail(ProviderInteractionError("Some sort of EC2 problem")))
mock_provider.open_port(MATCH_MACHINE, 0, 80, "tcp")
self.mocker.passthrough()
self.mocker.replay()
manager = MachineStateManager(self.client)
machine = yield manager.add_machine_state()
yield self.provide_machine(machine)
# Expose a service and attempt to open/close ports. The first
# attempt will see the simulated failure.
wordpress = yield self.add_service("wordpress")
yield wordpress.set_exposed_flag()
wordpress_0 = yield wordpress.add_unit_state()
yield wordpress_0.assign_to_machine(machine)
yield self.firewall_manager.process_machine(machine)
yield wordpress_0.open_port(80, "tcp")
yield self.firewall_manager.open_close_ports_on_machine(machine.id)
self.assertEqual((yield self.get_provider_ports(machine)), set())
self.assertIn("Got exception in opening/closing ports, will retry",
self.output.getvalue())
self.assertIn("TypeError: 'NoneType' object is not iterable",
self.output.getvalue())
# Retries will now happen in the periodic recheck. First one
# still fails due to simulated error.
yield self.firewall_manager.process_machine(machine)
self.assertEqual((yield self.get_provider_ports(machine)), set())
self.assertIn("ProviderInteractionError: Some sort of EC2 problem",
self.output.getvalue())
# Third time is the charm in the mock setup, the recheck succeeds
yield self.firewall_manager.process_machine(machine)
self.assertEqual((yield self.get_provider_ports(machine)),
set([(80, "tcp")]))
self.assertIn("Opened 80/tcp on provider machine 0",
self.output.getvalue())
def test_observe_open_close_ports_on_machine(self):
"""Verify one or more observers can be established on action."""
manager = MachineStateManager(self.client)
machine = yield manager.add_machine_state()
# Add one observer, verify it gets called
calls = [set()]
self.firewall_manager.add_open_close_ports_on_machine_observer(
Observer(calls, "a"))
yield self.firewall_manager.open_close_ports_on_machine(machine.id)
self.assertEqual(calls[0], set([("a", machine.id)]))
# Reset records of calls, and then add a second observer.
# Verify both get called.
calls[0] = set()
self.firewall_manager.add_open_close_ports_on_machine_observer(
Observer(calls, "b"))
yield self.firewall_manager.open_close_ports_on_machine(machine.id)
self.assertEqual(calls[0], set([("a", machine.id), ("b", machine.id)]))
def test_process_machine_is_called(self):
"""Verify FirewallManager is called when machines are processed"""
from juju.state.firewall import FirewallManager
mock_manager = self.mocker.patch(FirewallManager)
seen = []
def record_machine(machine):
seen.append(machine)
return succeed(True)
mock_manager.process_machine(MATCH_MACHINE_STATE)
self.mocker.call(record_machine)
self.mocker.replay()
machine_manager = MachineStateManager(self.client)
machine_state = yield machine_manager.add_machine_state()
yield self.agent.process_machines([machine_state.id])
self.assertEqual(seen, [machine_state])
def test_process_machine_ignores_stop_watcher(self):
"""Verify that process machine catches `StopWatcher`.
`process_machine` calls `open_close_ports_on_machine`, which
as verified in an earlier test, raises a `StopWatcher`
exception to shutdown watches that use it in the event of
agent shutdown. Verify this dual usage does not cause issues
while the agent is being stopped for this usage.
"""
mock_provider = self.mocker.patch(MachineProvider)
mock_provider.open_port(MATCH_MACHINE, 0, 80, "tcp")
self.mocker.result(fail(
TypeError("'NoneType' object is not iterable")))
self.mocker.replay()
manager = MachineStateManager(self.client)
machine = yield manager.add_machine_state()
yield self.provide_machine(machine)
# Expose a service and attempt to open/close ports. The first
# attempt will see the simulated failure.
wordpress = yield self.add_service("wordpress")
yield wordpress.set_exposed_flag()
wordpress_0 = yield wordpress.add_unit_state()
yield wordpress_0.assign_to_machine(machine)
yield self.firewall_manager.process_machine(machine)
yield wordpress_0.open_port(80, "tcp")
yield self.firewall_manager.open_close_ports_on_machine(machine.id)
self.assertEqual((yield self.get_provider_ports(machine)),
set())
self.assertIn(
"Got exception in opening/closing ports, will retry",
self.output.getvalue())
self.assertIn("TypeError: 'NoneType' object is not iterable",
self.output.getvalue())
# Stop the provisioning agent
self.stop()
# But retries can potentially still happening anyway, just
# make certain nothing bad happens.
yield self.firewall_manager.process_machine(machine)
def test_process_machine_is_called(self):
"""Verify FirewallManager is called when machines are processed"""
from juju.state.firewall import FirewallManager
mock_manager = self.mocker.patch(FirewallManager)
seen = []
def record_machine(machine):
seen.append(machine)
return succeed(True)
mock_manager.process_machine(MATCH_MACHINE_STATE)
self.mocker.call(record_machine)
self.mocker.replay()
machine_manager = MachineStateManager(self.client)
machine_state = yield machine_manager.add_machine_state()
yield self.agent.process_machines([machine_state.id])
self.assertEqual(seen, [machine_state])
def test_start_agent_with_watch(self):
mock_reactor = self.mocker.patch(reactor)
mock_reactor.callLater(self.agent.machine_check_period,
self.agent.periodic_machine_check)
self.mocker.replay()
self.agent.set_watch_enabled(True)
yield self.agent.start()
manager = MachineStateManager(self.client)
machine_state0 = yield manager.add_machine_state()
exists_d, watch_d = self.client.exists_and_watch(
"/machines/%s" % machine_state0.internal_id)
yield exists_d
# Wait for the provisioning agent to wake and modify
# the machine id.
yield watch_d
instance_id = yield machine_state0.get_instance_id()
self.assertEqual(instance_id, 0)
def test_start_agent_with_watch(self):
mock_reactor = self.mocker.patch(reactor)
mock_reactor.callLater(
self.agent.machine_check_period,
self.agent.periodic_machine_check)
self.mocker.replay()
self.agent.set_watch_enabled(True)
yield self.agent.start()
manager = MachineStateManager(self.client)
machine_state0 = yield manager.add_machine_state()
exists_d, watch_d = self.client.exists_and_watch(
"/machines/%s" % machine_state0.internal_id)
yield exists_d
# Wait for the provisioning agent to wake and modify
# the machine id.
yield watch_d
instance_id = yield machine_state0.get_instance_id()
self.assertEqual(instance_id, 0)
def test_process_machine_ignores_stop_watcher(self):
"""Verify that process machine catches `StopWatcher`.
`process_machine` calls `open_close_ports_on_machine`, which
as verified in an earlier test, raises a `StopWatcher`
exception to shutdown watches that use it in the event of
agent shutdown. Verify this dual usage does not cause issues
while the agent is being stopped for this usage.
"""
mock_provider = self.mocker.patch(MachineProvider)
mock_provider.open_port(MATCH_MACHINE, 0, 80, "tcp")
self.mocker.result(fail(
TypeError("'NoneType' object is not iterable")))
self.mocker.replay()
manager = MachineStateManager(self.client)
machine = yield manager.add_machine_state()
yield self.provide_machine(machine)
# Expose a service and attempt to open/close ports. The first
# attempt will see the simulated failure.
wordpress = yield self.add_service("wordpress")
yield wordpress.set_exposed_flag()
wordpress_0 = yield wordpress.add_unit_state()
yield wordpress_0.assign_to_machine(machine)
yield self.firewall_manager.process_machine(machine)
yield wordpress_0.open_port(80, "tcp")
yield self.firewall_manager.open_close_ports_on_machine(machine.id)
self.assertEqual((yield self.get_provider_ports(machine)), set())
self.assertIn("Got exception in opening/closing ports, will retry",
self.output.getvalue())
self.assertIn("TypeError: 'NoneType' object is not iterable",
self.output.getvalue())
# Stop the provisioning agent
self.stop()
# But retries can potentially still happening anyway, just
# make certain nothing bad happens.
yield self.firewall_manager.process_machine(machine)
def test_watch_machine_changes_processes_new_machine_id(self):
"""The agent should process a new machine id by creating it"""
manager = MachineStateManager(self.client)
machine_state0 = yield manager.add_machine_state()
machine_state1 = yield manager.add_machine_state()
yield self.agent.watch_machine_changes(
None, [machine_state0.id, machine_state1.id])
self.assertIn(
"Machines changed old:None new:[0, 1]", self.output.getvalue())
self.assertIn("Starting machine id:0", self.output.getvalue())
machines = yield self.agent.provider.get_machines()
self.assertEquals(len(machines), 2)
instance_id = yield machine_state0.get_instance_id()
self.assertEqual(instance_id, 0)
instance_id = yield machine_state1.get_instance_id()
self.assertEqual(instance_id, 1)
def test_watch_machine_changes_processes_new_machine_id(self):
"""The agent should process a new machine id by creating it"""
manager = MachineStateManager(self.client)
machine_state0 = yield manager.add_machine_state()
machine_state1 = yield manager.add_machine_state()
yield self.agent.watch_machine_changes(
None, [machine_state0.id, machine_state1.id])
self.assertIn("Machines changed old:None new:[0, 1]",
self.output.getvalue())
self.assertIn("Starting machine id:0", self.output.getvalue())
machines = yield self.agent.provider.get_machines()
self.assertEquals(len(machines), 2)
instance_id = yield machine_state0.get_instance_id()
self.assertEqual(instance_id, 0)
instance_id = yield machine_state1.get_instance_id()
self.assertEqual(instance_id, 1)
def test_observe_open_close_ports_on_machine(self):
"""Verify one or more observers can be established on action."""
manager = MachineStateManager(self.client)
machine = yield manager.add_machine_state()
# Add one observer, verify it gets called
calls = [set()]
self.firewall_manager.add_open_close_ports_on_machine_observer(
Observer(calls, "a"))
yield self.firewall_manager.open_close_ports_on_machine(machine.id)
self.assertEqual(calls[0], set([("a", machine.id)]))
# Reset records of calls, and then add a second observer.
# Verify both get called.
calls[0] = set()
self.firewall_manager.add_open_close_ports_on_machine_observer(
Observer(calls, "b"))
yield self.firewall_manager.open_close_ports_on_machine(machine.id)
self.assertEqual(
calls[0],
set([("a", machine.id), ("b", machine.id)]))
def test_open_close_ports_on_machine_unexposed_service(self):
"""Verify opening/closing ports on a machine works properly.
In particular this is done without watch support."""
manager = MachineStateManager(self.client)
machine = yield manager.add_machine_state()
yield self.provide_machine(machine)
wordpress = yield self.add_service("wordpress")
wordpress_0 = yield wordpress.add_unit_state()
# Port activity, but service is not exposed
yield wordpress_0.open_port(80, "tcp")
yield wordpress_0.open_port(443, "tcp")
yield wordpress_0.assign_to_machine(machine)
yield self.firewall_manager.open_close_ports_on_machine(machine.id)
self.assertEqual((yield self.get_provider_ports(machine)), set())
# Now expose it
yield wordpress.set_exposed_flag()
yield self.firewall_manager.open_close_ports_on_machine(machine.id)
self.assertEqual((yield self.get_provider_ports(machine)),
set([(80, "tcp"), (443, "tcp")]))
def test_transient_provider_error_on_start_machine(self):
"""
If there's an error when processing changes, the agent should log
the error and continue.
"""
manager = MachineStateManager(self.client)
machine_state0 = yield manager.add_machine_state()
machine_state1 = yield manager.add_machine_state()
mock_provider = self.mocker.patch(self.agent.provider)
mock_provider.start_machine({"machine-id": 0})
self.mocker.result(fail(ProviderInteractionError()))
mock_provider.start_machine({"machine-id": 1})
self.mocker.passthrough()
self.mocker.replay()
yield self.agent.watch_machine_changes(
[], [machine_state0.id, machine_state1.id])
machine1_instance_id = yield machine_state1.get_instance_id()
self.assertEqual(machine1_instance_id, 0)
self.assertIn("Cannot process machine 0", self.output.getvalue())
def test_open_close_ports_on_machine_unexposed_service(self):
"""Verify opening/closing ports on a machine works properly.
In particular this is done without watch support."""
manager = MachineStateManager(self.client)
machine = yield manager.add_machine_state()
yield self.provide_machine(machine)
wordpress = yield self.add_service("wordpress")
wordpress_0 = yield wordpress.add_unit_state()
# Port activity, but service is not exposed
yield wordpress_0.open_port(80, "tcp")
yield wordpress_0.open_port(443, "tcp")
yield wordpress_0.assign_to_machine(machine)
yield self.firewall_manager.open_close_ports_on_machine(machine.id)
self.assertEqual((yield self.get_provider_ports(machine)),
set())
# Now expose it
yield wordpress.set_exposed_flag()
yield self.firewall_manager.open_close_ports_on_machine(machine.id)
self.assertEqual((yield self.get_provider_ports(machine)),
set([(80, "tcp"), (443, "tcp")]))
def test_watches_trigger_port_mgmt(self):
"""Verify that watches properly trigger firewall management
for the corresponding service units on the corresponding
machines.
"""
self.start()
manager = MachineStateManager(self.client)
# Immediately expose
drupal = yield self.add_service("drupal")
wordpress = yield self.add_service("wordpress")
yield drupal.set_exposed_flag()
yield wordpress.set_exposed_flag()
# Then add these units
drupal_0 = yield drupal.add_unit_state()
wordpress_0 = yield wordpress.add_unit_state()
wordpress_1 = yield wordpress.add_unit_state()
wordpress_2 = yield wordpress.add_unit_state()
# Assign some machines; in particular verify that multiple
# service units on one machine works properly with opening
# firewall
machine_0 = yield manager.add_machine_state()
machine_1 = yield manager.add_machine_state()
machine_2 = yield manager.add_machine_state()
yield self.provide_machine(machine_0)
yield self.provide_machine(machine_1)
yield self.provide_machine(machine_2)
yield drupal_0.assign_to_machine(machine_0)
yield wordpress_0.assign_to_machine(machine_0)
yield wordpress_1.assign_to_machine(machine_1)
yield wordpress_2.assign_to_machine(machine_2)
# Simulate service units opening ports
expected_machines = self.wait_on_expected_machines(set([0, 1]))
expected_units = self.wait_on_expected_units(
set(["wordpress/0", "wordpress/1", "drupal/0"]))
yield drupal_0.open_port(8080, "tcp")
yield drupal_0.open_port(443, "tcp")
yield wordpress_0.open_port(80, "tcp")
yield wordpress_1.open_port(80, "tcp")
self.assertTrue((yield expected_units))
self.assertTrue((yield expected_machines))
self.assertEqual((yield self.get_provider_ports(machine_0)),
set([(80, "tcp"), (443, "tcp"), (8080, "tcp")]))
self.assertEqual((yield self.get_provider_ports(machine_1)),
set([(80, "tcp")]))
# Simulate service units close port
expected_machines = self.wait_on_expected_machines(set([1, 2]))
yield wordpress_1.close_port(80, "tcp")
yield wordpress_2.open_port(80, "tcp")
self.assertTrue((yield expected_machines))
self.assertEqual((yield self.get_provider_ports(machine_1)), set())
# Simulate service units open port
expected_machines = self.wait_on_expected_machines(set([0]))
yield wordpress_0.open_port(53, "udp")
self.assertTrue((yield expected_machines))
self.assertEqual((yield self.get_provider_ports(machine_0)),
set([(53, "udp"), (80, "tcp"), (443, "tcp"),
(8080, "tcp")]))
self.stop()
def test_late_expose_properly_triggers(self):
"""Verify that an expose flag properly cascades the
corresponding watches to perform the desired firewall mgmt.
"""
self.start()
manager = MachineStateManager(self.client)
drupal = yield self.add_service("drupal")
wordpress = yield self.add_service("wordpress")
# Then add these units
drupal_0 = yield drupal.add_unit_state()
wordpress_0 = yield wordpress.add_unit_state()
wordpress_1 = yield wordpress.add_unit_state()
machine_0 = yield manager.add_machine_state()
machine_1 = yield manager.add_machine_state()
yield self.provide_machine(machine_0)
yield self.provide_machine(machine_1)
yield drupal_0.assign_to_machine(machine_0)
yield wordpress_0.assign_to_machine(machine_0)
yield wordpress_1.assign_to_machine(machine_1)
# Simulate service units opening ports
expected_machines = self.wait_on_expected_machines(set([0, 1]))
expected_units = self.wait_on_expected_units(
set(["wordpress/0", "wordpress/1"]))
yield drupal_0.open_port(8080, "tcp")
yield drupal_0.open_port(443, "tcp")
yield wordpress_0.open_port(80, "tcp")
yield wordpress_1.open_port(80, "tcp")
yield wordpress.set_exposed_flag()
self.assertTrue((yield expected_units))
self.assertTrue((yield expected_machines))
self.assertEqual((yield self.get_provider_ports(machine_0)),
set([(80, "tcp")]))
self.assertEqual((yield self.get_provider_ports(machine_1)),
set([(80, "tcp")]))
# Expose drupal service, verify ports are opened on provider
expected_machines = self.wait_on_expected_machines(set([0]))
expected_units = self.wait_on_expected_units(set(["drupal/0"]))
yield drupal.set_exposed_flag()
self.assertTrue((yield expected_machines))
self.assertTrue((yield expected_units))
self.assertEqual((yield self.get_provider_ports(machine_0)),
set([(80, "tcp"), (443, "tcp"), (8080, "tcp")]))
# Unexpose drupal service, verify only wordpress ports are now opened
expected_machines = self.wait_on_expected_machines(set([0]))
expected_units = self.wait_on_expected_units(set(["drupal/0"]))
yield drupal.clear_exposed_flag()
self.assertTrue((yield expected_machines))
self.assertTrue((yield expected_units))
self.assertEqual((yield self.get_provider_ports(machine_0)),
set([(80, "tcp")]))
# Re-expose drupal service, verify ports are once again opened
expected_machines = self.wait_on_expected_machines(set([0]))
expected_units = self.wait_on_expected_units(set(["drupal/0"]))
yield drupal.set_exposed_flag()
self.assertTrue((yield expected_machines))
self.assertTrue((yield expected_units))
self.assertEqual((yield self.get_provider_ports(machine_0)),
set([(80, "tcp"), (443, "tcp"), (8080, "tcp")]))
self.stop()
class ControlDebugHookTest(
ServiceStateManagerTestBase, ControlToolTest, RepositoryTestBase):
@inlineCallbacks
def setUp(self):
yield super(ControlDebugHookTest, self).setUp()
config = {
"environments": {
"firstenv": {
"type": "dummy", "admin-secret": "homer"}}}
self.write_config(yaml.dump(config))
self.config.load()
self.environment = self.config.get_default()
self.provider = self.environment.get_machine_provider()
# Setup a machine in the provider
self.provider_machine = (yield self.provider.start_machine(
{"machine-id": 0, "dns-name": "antigravity.example.com"}))[0]
# Setup the zk tree with a service, unit, and machine.
self.service = yield self.add_service_from_charm("mysql")
self.unit = yield self.service.add_unit_state()
self.machine_manager = MachineStateManager(self.client)
self.machine = yield self.machine_manager.add_machine_state()
yield self.machine.set_instance_id(0)
yield self.unit.assign_to_machine(self.machine)
# capture the output.
self.output = self.capture_logging(
"juju.control.cli", level=logging.INFO)
self.stderr = self.capture_stream("stderr")
self.setup_exit(0)
@inlineCallbacks
def test_debug_hook_invalid_hook_name(self):
"""If an invalid hookname is used an appropriate error
message is raised that references the charm.
"""
mock_environment = self.mocker.patch(Environment)
mock_environment.get_machine_provider()
self.mocker.result(self.provider)
finished = self.setup_cli_reactor()
self.mocker.replay()
main(["debug-hooks", "mysql/0", "bad-happened"])
yield finished
self.assertIn(
"Charm 'local:series/mysql-1' does not contain hook "
"'bad-happened'",
self.stderr.getvalue())
@inlineCallbacks
def test_debug_hook_invalid_unit_name(self):
"""An invalid unit causes an appropriate error.
"""
finished = self.setup_cli_reactor()
self.mocker.replay()
main(["debug-hooks", "mysql/42"])
yield finished
self.assertIn(
"Service unit 'mysql/42' was not found",
self.stderr.getvalue())
@inlineCallbacks
def test_debug_hook_invalid_service(self):
"""An invalid service causes an appropriate error.
"""
finished = self.setup_cli_reactor()
self.mocker.replay()
main(["debug-hooks", "magic/42"])
yield finished
self.assertIn(
"Service 'magic' was not found",
self.stderr.getvalue())
@inlineCallbacks
def test_debug_hook_unit_agent_not_available(self):
"""Simulate the unit agent isn't available when the command is run.
The command will set the debug flag, and wait for the unit
agent to be available.
"""
mock_unit = self.mocker.patch(ServiceUnitState)
# First time, doesn't exist, will wait on watch
mock_unit.watch_agent()
self.mocker.result((succeed(False), succeed(True)))
# Second time, setup the unit address
mock_unit.watch_agent()
def setup_unit_address():
set_d = self.unit.set_public_address("x1.example.com")
exist_d = Deferred()
set_d.addCallback(lambda result: exist_d.callback(True))
return (exist_d, succeed(True))
self.mocker.call(setup_unit_address)
# Intercept the ssh call
self.patch(os, "system", lambda x: True)
#mock_environment = self.mocker.patch(Environment)
#mock_environment.get_machine_provider()
#self.mocker.result(self.provider)
finished = self.setup_cli_reactor()
self.mocker.replay()
main(["debug-hooks", "mysql/0"])
yield finished
self.assertIn("Waiting for unit", self.output.getvalue())
self.assertIn("Unit running", self.output.getvalue())
self.assertIn("Connecting to remote machine x1.example.com",
self.output.getvalue())
@inlineCallbacks
def test_debug_hook(self):
"""The debug cli will setup unit debug setting and ssh to a screen.
"""
system_mock = self.mocker.replace(os.system)
system_mock(ANY)
def on_ssh(command):
self.assertStartsWith(command, "ssh -t [email protected]")
# In the function, os.system yields to faciliate testing.
self.assertEqual(
(yield self.unit.get_hook_debug()),
{"debug_hooks": ["*"]})
returnValue(True)
self.mocker.call(on_ssh)
finished = self.setup_cli_reactor()
self.mocker.replay()
# Setup the unit address.
yield self.unit.set_public_address("x2.example.com")
main(["debug-hooks", "mysql/0"])
yield finished
self.assertIn(
"Connecting to remote machine", self.output.getvalue())
self.assertIn(
"Debug session ended.", self.output.getvalue())
@inlineCallbacks
def test_standard_named_debug_hook(self):
"""A hook can be debugged by name.
"""
yield self.verify_hook_debug("start")
self.mocker.reset()
self.setup_exit(0)
yield self.verify_hook_debug("stop")
self.mocker.reset()
self.setup_exit(0)
yield self.verify_hook_debug("server-relation-changed")
self.mocker.reset()
self.setup_exit(0)
yield self.verify_hook_debug("server-relation-changed",
"server-relation-broken")
self.mocker.reset()
self.setup_exit(0)
yield self.verify_hook_debug("server-relation-joined",
"server-relation-departed")
@inlineCallbacks
def verify_hook_debug(self, *hook_names):
"""Utility function to verify hook debugging by name
"""
mock_environment = self.mocker.patch(Environment)
mock_environment.get_machine_provider()
self.mocker.result(self.provider)
system_mock = self.mocker.replace(os.system)
system_mock(ANY)
def on_ssh(command):
self.assertStartsWith(command, "ssh -t [email protected]")
self.assertEqual(
(yield self.unit.get_hook_debug()),
{"debug_hooks": list(hook_names)})
returnValue(True)
self.mocker.call(on_ssh)
finished = self.setup_cli_reactor()
self.mocker.replay()
yield self.unit.set_public_address("x11.example.com")
args = ["debug-hooks", "mysql/0"]
args.extend(hook_names)
main(args)
yield finished
self.assertIn(
"Connecting to remote machine", self.output.getvalue())
self.assertIn(
"Debug session ended.", self.output.getvalue())
def add_machine_state(self):
manager = MachineStateManager(self.client)
return manager.add_machine_state(series_constraints)
class SCPTest(ServiceStateManagerTestBase, ControlToolTest):
@inlineCallbacks
def setUp(self):
yield super(SCPTest, self).setUp()
config = {
"environments": {
"firstenv": {
"type": "dummy",
"admin-secret": "homer"
}
}
}
self.write_config(dump(config))
self.setup_exit(0)
self.config.load()
self.environment = self.config.get_default()
self.provider = self.environment.get_machine_provider()
# Setup a machine in the provider
self.provider_machine = (yield self.provider.start_machine({
"machine-id":
0,
"dns-name":
"antigravity.example.com"
}))[0]
# Setup the zk tree with a service, unit, and machine.
self.service = yield self.add_service_from_charm("mysql")
self.unit = yield self.service.add_unit_state()
yield self.unit.set_public_address(
"%s.example.com" % self.unit.unit_name.replace("/", "-"))
self.machine_manager = MachineStateManager(self.client)
self.machine = yield self.machine_manager.add_machine_state()
yield self.machine.set_instance_id(0)
yield self.unit.assign_to_machine(self.machine)
# capture the output.
self.output = self.capture_logging("juju.control.cli",
level=logging.INFO)
self.stderr = self.capture_stream("stderr")
@inlineCallbacks
def test_scp_unit_name(self):
"""Verify scp command is invoked against the host for a unit name."""
mock_environment = self.mocker.patch(Environment)
mock_environment.get_machine_provider()
self.mocker.result(self.provider)
# Verify expected call against scp
mock_exec = self.mocker.replace(os.execvp)
mock_exec("scp",
["scp", "[email protected]:/foo/*", "10.1.2.3:."])
# But no other calls
calls = []
mock_exec(ARGS, KWARGS)
self.mocker.count(0, None)
self.mocker.call(lambda *args, **kwargs: calls.append((args, kwargs)))
finished = self.setup_cli_reactor()
self.mocker.replay()
yield self.unit.connect_agent()
main(["scp", "mysql/0:/foo/*", "10.1.2.3:."])
yield finished
self.assertEquals(calls, [])
@inlineCallbacks
def test_scp_machine_id(self):
"""Verify scp command is invoked against the host for a machine ID."""
mock_environment = self.mocker.patch(Environment)
mock_environment.get_machine_provider()
self.mocker.result(self.provider)
# Verify expected call against scp
mock_exec = self.mocker.replace(os.execvp)
mock_exec(
"scp",
["scp", "[email protected]:foo/*", "10.1.2.3:."])
# But no other calls
calls = []
mock_exec(ARGS, KWARGS)
self.mocker.count(0, None)
self.mocker.call(lambda *args, **kwargs: calls.append((args, kwargs)))
finished = self.setup_cli_reactor()
self.mocker.replay()
yield self.unit.connect_agent()
main(["scp", "0:foo/*", "10.1.2.3:."])
yield finished
self.assertEquals(calls, [])
@inlineCallbacks
def test_passthrough_args(self):
"""Verify that args are passed through to the underlying scp command.
For example, something like the following command should be valid::
$ juju scp -o "ConnectTimeout 60" foo mysql/0:/foo/bar
"""
mock_environment = self.mocker.patch(Environment)
mock_environment.get_machine_provider()
self.mocker.result(self.provider)
# Verify expected call against scp
mock_exec = self.mocker.replace(os.execvp)
mock_exec("scp", [
"scp", "-r", "-o", "ConnectTimeout 60", "foo",
"[email protected]:/foo/bar"
])
# But no other calls
calls = []
mock_exec(ARGS, KWARGS)
self.mocker.count(0, None)
self.mocker.call(lambda *args, **kwargs: calls.append((args, kwargs)))
finished = self.setup_cli_reactor()
self.mocker.replay()
main([
"scp", "-r", "-o", "ConnectTimeout 60", "foo", "mysql/0:/foo/bar"
])
yield finished
self.assertEquals(calls, [])
class MachineStateManagerTest(StateTestBase):
@inlineCallbacks
def setUp(self):
yield super(MachineStateManagerTest, self).setUp()
self.charm_state_manager = CharmStateManager(self.client)
self.machine_state_manager = MachineStateManager(self.client)
self.service_state_manager = ServiceStateManager(self.client)
self.charm_state = yield self.charm_state_manager.add_charm_state(
local_charm_id(self.charm), self.charm, "")
@inlineCallbacks
def add_service(self, service_name):
service_state = yield self.service_state_manager.add_service_state(
service_name, self.charm_state)
returnValue(service_state)
@inlineCallbacks
def test_add_machine(self):
"""
Adding a machine state should register it in zookeeper.
"""
machine_state1 = yield self.machine_state_manager.add_machine_state()
machine_state2 = yield self.machine_state_manager.add_machine_state()
self.assertEquals(machine_state1.id, 0)
self.assertEquals(machine_state1.internal_id, "machine-0000000000")
self.assertEquals(machine_state2.id, 1)
self.assertEquals(machine_state2.internal_id, "machine-0000000001")
children = yield self.client.get_children("/machines")
self.assertEquals(sorted(children),
["machine-0000000000", "machine-0000000001"])
topology = yield self.get_topology()
self.assertTrue(topology.has_machine("machine-0000000000"))
self.assertTrue(topology.has_machine("machine-0000000001"))
@inlineCallbacks
def test_machine_str_representation(self):
"""The str(machine) value includes the machine id.
"""
machine_state1 = yield self.machine_state_manager.add_machine_state()
self.assertEqual(
str(machine_state1), "<MachineState id:machine-%010d>" % (0))
@inlineCallbacks
def test_remove_machine(self):
"""
Adding a machine state should register it in zookeeper.
"""
machine_state1 = yield self.machine_state_manager.add_machine_state()
yield self.machine_state_manager.add_machine_state()
removed = yield self.machine_state_manager.remove_machine_state(
machine_state1.id)
self.assertTrue(removed)
children = yield self.client.get_children("/machines")
self.assertEquals(sorted(children),
["machine-0000000001"])
topology = yield self.get_topology()
self.assertFalse(topology.has_machine("machine-0000000000"))
self.assertTrue(topology.has_machine("machine-0000000001"))
# Removing a non-existing machine again won't fail, since the end
# intention is preserved. This makes dealing with concurrency easier.
# However, False will be returned in this case.
removed = yield self.machine_state_manager.remove_machine_state(
machine_state1.id)
self.assertFalse(removed)
@inlineCallbacks
def test_remove_machine_with_agent(self):
"""Removing a machine with a connected machine agent should succeed.
The removal signals intent to remove a working machine (with an agent)
with the provisioning agent to remove it subsequently.
"""
# Add two machines.
machine_state1 = yield self.machine_state_manager.add_machine_state()
yield self.machine_state_manager.add_machine_state()
# Connect an agent
yield machine_state1.connect_agent()
# Remove a machine
removed = yield self.machine_state_manager.remove_machine_state(
machine_state1.id)
self.assertTrue(removed)
# Verify the second one is still present
children = yield self.client.get_children("/machines")
self.assertEquals(sorted(children),
["machine-0000000001"])
# Verify the topology state.
topology = yield self.get_topology()
self.assertFalse(topology.has_machine("machine-0000000000"))
self.assertTrue(topology.has_machine("machine-0000000001"))
@inlineCallbacks
def test_get_machine_and_check_attributes(self):
"""
Getting a machine state should be possible using both the
user-oriented id and the internal id.
"""
yield self.machine_state_manager.add_machine_state()
yield self.machine_state_manager.add_machine_state()
machine_state = yield self.machine_state_manager.get_machine_state(0)
self.assertEquals(machine_state.id, 0)
machine_state = yield self.machine_state_manager.get_machine_state("0")
self.assertEquals(machine_state.id, 0)
yield self.assertFailure(
self.machine_state_manager.get_machine_state("a"),
MachineStateNotFound)
@inlineCallbacks
def test_get_machine_not_found(self):
"""
Getting a machine state which is not available should errback
a meaningful error.
"""
# No state whatsoever.
try:
yield self.machine_state_manager.get_machine_state(0)
except MachineStateNotFound, e:
self.assertEquals(e.machine_id, 0)
else:
class ControlRemoveUnitTest(ServiceStateManagerTestBase, ControlToolTest,
RepositoryTestBase):
@inlineCallbacks
def setUp(self):
yield super(ControlRemoveUnitTest, self).setUp()
config = {"environments": {"firstenv": {"type": "dummy"}}}
self.write_config(dump(config))
self.config.load()
self.environment = self.config.get_default()
self.provider = self.environment.get_machine_provider()
# Setup some service units.
self.service_state1 = yield self.add_service_from_charm("mysql")
self.service_unit1 = yield self.service_state1.add_unit_state()
self.service_unit2 = yield self.service_state1.add_unit_state()
self.service_unit3 = yield self.service_state1.add_unit_state()
# Add an assigned machine to one of them.
self.machine_manager = MachineStateManager(self.client)
self.machine = yield self.machine_manager.add_machine_state()
yield self.machine.set_instance_id(0)
yield self.service_unit1.assign_to_machine(self.machine)
# Setup a machine in the provider matching the assigned.
self.provider_machine = yield self.provider.start_machine({
"machine-id":
0,
"dns-name":
"antigravity.example.com"
})
self.output = self.capture_logging(level=logging.DEBUG)
self.stderr = self.capture_stream("stderr")
@inlineCallbacks
def test_remove_unit(self):
"""
'juju remove-unit <unit_name>' will remove the given unit.
"""
unit_names = yield self.service_state1.get_unit_names()
self.assertEqual(len(unit_names), 3)
finished = self.setup_cli_reactor()
self.setup_exit(0)
self.mocker.replay()
main(["remove-unit", "mysql/0"])
yield finished
topology = yield self.get_topology()
self.assertFalse(
topology.has_service_unit(self.service_state1.internal_id,
self.service_unit1.internal_id))
topology = yield self.get_topology()
self.assertTrue(
topology.has_service_unit(self.service_state1.internal_id,
self.service_unit2.internal_id))
self.assertFalse(
topology.get_service_units_in_machine(self.machine.internal_id))
self.assertIn("Unit 'mysql/0' removed from service 'mysql'",
self.output.getvalue())
@inlineCallbacks
def test_remove_multiple_units(self):
"""
'juju remove-unit <unit_name1> <unit_name2>...' removes desired units.
"""
unit_names = yield self.service_state1.get_unit_names()
self.assertEqual(len(unit_names), 3)
finished = self.setup_cli_reactor()
self.setup_exit(0)
self.mocker.replay()
main(["remove-unit", "mysql/0", "mysql/2"])
yield finished
topology = yield self.get_topology()
self.assertFalse(
topology.has_service_unit(self.service_state1.internal_id,
self.service_unit1.internal_id))
topology = yield self.get_topology()
self.assertTrue(
topology.has_service_unit(self.service_state1.internal_id,
self.service_unit2.internal_id))
self.assertFalse(
topology.get_service_units_in_machine(self.machine.internal_id))
self.assertIn("Unit 'mysql/0' removed from service 'mysql'",
self.output.getvalue())
self.assertIn("Unit 'mysql/2' removed from service 'mysql'",
self.output.getvalue())
@inlineCallbacks
def test_remove_unassigned_unit(self):
"""Remove unit also works if the unit is unassigned to a machine.
"""
unit_names = yield self.service_state1.get_unit_names()
self.assertEqual(len(unit_names), 3)
finished = self.setup_cli_reactor()
self.setup_exit(0)
self.mocker.replay()
main(["remove-unit", "mysql/1"])
yield finished
# verify the unit and its machine assignment.
unit_names = yield self.service_state1.get_unit_names()
self.assertEqual(len(unit_names), 2)
topology = yield self.get_topology()
topology = yield self.get_topology()
self.assertFalse(
topology.has_service_unit(self.service_state1.internal_id,
self.service_unit2.internal_id))
topology = yield self.get_topology()
self.assertTrue(
topology.has_service_unit(self.service_state1.internal_id,
self.service_unit1.internal_id))
@inlineCallbacks
def test_remove_unit_unknown_service(self):
"""If the service doesn't exist, return an appropriate error message.
"""
finished = self.setup_cli_reactor()
self.setup_exit(0)
self.mocker.replay()
main(["remove-unit", "volcano/0"])
yield finished
self.assertIn("Service 'volcano' was not found",
self.stderr.getvalue())
@inlineCallbacks
def test_remove_unit_bad_parse(self):
"""Verify that a bad service unit name results in an appropriate error.
"""
finished = self.setup_cli_reactor()
self.setup_exit(0)
self.mocker.replay()
main(["remove-unit", "volcano-0"])
yield finished
self.assertIn("Not a proper unit name: 'volcano-0'",
self.stderr.getvalue())
@inlineCallbacks
def test_remove_unit_unknown_unit(self):
"""If the unit doesn't exist an appropriate error message is returned.
"""
finished = self.setup_cli_reactor()
self.setup_exit(0)
self.mocker.replay()
main(["remove-unit", "mysql/3"])
yield finished
self.assertIn("Service unit 'mysql/3' was not found",
self.stderr.getvalue())
@inlineCallbacks
def test_zookeeper_logging_default(self):
"""By default zookeeper logging is turned off, unless in verbose
mode.
"""
log_file = self.makeFile()
def reset_zk_log():
zookeeper.set_debug_level(0)
zookeeper.set_log_stream(sys.stdout)
self.addCleanup(reset_zk_log)
finished = self.setup_cli_reactor()
self.setup_exit(0)
# Do this as late as possible to prevent zk background logging
# from causing problems.
zookeeper.set_debug_level(zookeeper.LOG_LEVEL_INFO)
zookeeper.set_log_stream(open(log_file, "w"))
self.mocker.replay()
main(["remove-unit", "mysql/3"])
yield finished
output = open(log_file).read()
self.assertEqual(output, "")
@inlineCallbacks
def test_zookeeper_logging_enabled(self):
"""By default zookeeper logging is turned off, unless in verbose
mode.
"""
log_file = self.makeFile()
zookeeper.set_debug_level(10)
zookeeper.set_log_stream(open(log_file, "w"))
def reset_zk_log():
zookeeper.set_debug_level(0)
zookeeper.set_log_stream(sys.stdout)
self.addCleanup(reset_zk_log)
finished = self.setup_cli_reactor()
self.setup_exit(0)
self.mocker.replay()
main(["-v", "remove-unit", "mysql/3"])
yield finished
output = open(log_file).read()
self.assertTrue(output)
self.assertIn("ZOO_DEBUG", output)
def test_late_expose_properly_triggers(self):
"""Verify that an expose flag properly cascades the
corresponding watches to perform the desired firewall mgmt.
"""
self.start()
manager = MachineStateManager(self.client)
drupal = yield self.add_service("drupal")
wordpress = yield self.add_service("wordpress")
# Then add these units
drupal_0 = yield drupal.add_unit_state()
wordpress_0 = yield wordpress.add_unit_state()
wordpress_1 = yield wordpress.add_unit_state()
machine_0 = yield manager.add_machine_state()
machine_1 = yield manager.add_machine_state()
yield self.provide_machine(machine_0)
yield self.provide_machine(machine_1)
yield drupal_0.assign_to_machine(machine_0)
yield wordpress_0.assign_to_machine(machine_0)
yield wordpress_1.assign_to_machine(machine_1)
# Simulate service units opening ports
expected_machines = self.wait_on_expected_machines(set([0, 1]))
expected_units = self.wait_on_expected_units(
set(["wordpress/0", "wordpress/1"]))
yield drupal_0.open_port(8080, "tcp")
yield drupal_0.open_port(443, "tcp")
yield wordpress_0.open_port(80, "tcp")
yield wordpress_1.open_port(80, "tcp")
yield wordpress.set_exposed_flag()
self.assertTrue((yield expected_units))
self.assertTrue((yield expected_machines))
self.assertEqual((yield self.get_provider_ports(machine_0)),
set([(80, "tcp")]))
self.assertEqual((yield self.get_provider_ports(machine_1)),
set([(80, "tcp")]))
# Expose drupal service, verify ports are opened on provider
expected_machines = self.wait_on_expected_machines(set([0]))
expected_units = self.wait_on_expected_units(set(["drupal/0"]))
yield drupal.set_exposed_flag()
self.assertTrue((yield expected_machines))
self.assertTrue((yield expected_units))
self.assertEqual((yield self.get_provider_ports(machine_0)),
set([(80, "tcp"), (443, "tcp"), (8080, "tcp")]))
# Unexpose drupal service, verify only wordpress ports are now opened
expected_machines = self.wait_on_expected_machines(set([0]))
expected_units = self.wait_on_expected_units(set(["drupal/0"]))
yield drupal.clear_exposed_flag()
self.assertTrue((yield expected_machines))
self.assertTrue((yield expected_units))
self.assertEqual((yield self.get_provider_ports(machine_0)),
set([(80, "tcp")]))
# Re-expose drupal service, verify ports are once again opened
expected_machines = self.wait_on_expected_machines(set([0]))
expected_units = self.wait_on_expected_units(set(["drupal/0"]))
yield drupal.set_exposed_flag()
self.assertTrue((yield expected_machines))
self.assertTrue((yield expected_units))
self.assertEqual((yield self.get_provider_ports(machine_0)),
set([(80, "tcp"), (443, "tcp"), (8080, "tcp")]))
self.stop()
def test_watches_trigger_port_mgmt(self):
"""Verify that watches properly trigger firewall management
for the corresponding service units on the corresponding
machines.
"""
self.start()
manager = MachineStateManager(self.client)
# Immediately expose
drupal = yield self.add_service("drupal")
wordpress = yield self.add_service("wordpress")
yield drupal.set_exposed_flag()
yield wordpress.set_exposed_flag()
# Then add these units
drupal_0 = yield drupal.add_unit_state()
wordpress_0 = yield wordpress.add_unit_state()
wordpress_1 = yield wordpress.add_unit_state()
wordpress_2 = yield wordpress.add_unit_state()
# Assign some machines; in particular verify that multiple
# service units on one machine works properly with opening
# firewall
machine_0 = yield manager.add_machine_state()
machine_1 = yield manager.add_machine_state()
machine_2 = yield manager.add_machine_state()
yield self.provide_machine(machine_0)
yield self.provide_machine(machine_1)
yield self.provide_machine(machine_2)
yield drupal_0.assign_to_machine(machine_0)
yield wordpress_0.assign_to_machine(machine_0)
yield wordpress_1.assign_to_machine(machine_1)
yield wordpress_2.assign_to_machine(machine_2)
# Simulate service units opening ports
expected_machines = self.wait_on_expected_machines(set([0, 1]))
expected_units = self.wait_on_expected_units(
set(["wordpress/0", "wordpress/1", "drupal/0"]))
yield drupal_0.open_port(8080, "tcp")
yield drupal_0.open_port(443, "tcp")
yield wordpress_0.open_port(80, "tcp")
yield wordpress_1.open_port(80, "tcp")
self.assertTrue((yield expected_units))
self.assertTrue((yield expected_machines))
self.assertEqual((yield self.get_provider_ports(machine_0)),
set([(80, "tcp"), (443, "tcp"), (8080, "tcp")]))
self.assertEqual((yield self.get_provider_ports(machine_1)),
set([(80, "tcp")]))
# Simulate service units close port
expected_machines = self.wait_on_expected_machines(set([1, 2]))
yield wordpress_1.close_port(80, "tcp")
yield wordpress_2.open_port(80, "tcp")
self.assertTrue((yield expected_machines))
self.assertEqual((yield self.get_provider_ports(machine_1)), set())
# Simulate service units open port
expected_machines = self.wait_on_expected_machines(set([0]))
yield wordpress_0.open_port(53, "udp")
self.assertTrue((yield expected_machines))
self.assertEqual((yield self.get_provider_ports(machine_0)),
set([(53, "udp"), (80, "tcp"),
(443, "tcp"), (8080, "tcp")]))
self.stop()
class SCPTest(ServiceStateManagerTestBase, ControlToolTest):
@inlineCallbacks
def setUp(self):
yield super(SCPTest, self).setUp()
config = {
"environments": {
"firstenv": {
"type": "dummy", "admin-secret": "homer"}}}
self.write_config(dump(config))
self.setup_exit(0)
self.config.load()
self.environment = self.config.get_default()
self.provider = self.environment.get_machine_provider()
# Setup a machine in the provider
self.provider_machine = (yield self.provider.start_machine(
{"machine-id": 0, "dns-name": "antigravity.example.com"}))[0]
# Setup the zk tree with a service, unit, and machine.
self.service = yield self.add_service_from_charm("mysql")
self.unit = yield self.service.add_unit_state()
yield self.unit.set_public_address(
"%s.example.com" % self.unit.unit_name.replace("/", "-"))
self.machine_manager = MachineStateManager(self.client)
self.machine = yield self.machine_manager.add_machine_state()
yield self.machine.set_instance_id(0)
yield self.unit.assign_to_machine(self.machine)
# capture the output.
self.output = self.capture_logging(
"juju.control.cli", level=logging.INFO)
self.stderr = self.capture_stream("stderr")
@inlineCallbacks
def test_scp_unit_name(self):
"""Verify scp command is invoked against the host for a unit name."""
mock_environment = self.mocker.patch(Environment)
mock_environment.get_machine_provider()
self.mocker.result(self.provider)
# Verify expected call against scp
mock_exec = self.mocker.replace(os.execvp)
mock_exec("scp", [
"scp", "[email protected]:/foo/*", "10.1.2.3:."])
# But no other calls
calls = []
mock_exec(ARGS, KWARGS)
self.mocker.count(0, None)
self.mocker.call(lambda *args, **kwargs: calls.append((args, kwargs)))
finished = self.setup_cli_reactor()
self.mocker.replay()
yield self.unit.connect_agent()
main(["scp", "mysql/0:/foo/*", "10.1.2.3:."])
yield finished
self.assertEquals(calls, [])
@inlineCallbacks
def test_scp_machine_id(self):
"""Verify scp command is invoked against the host for a machine ID."""
mock_environment = self.mocker.patch(Environment)
mock_environment.get_machine_provider()
self.mocker.result(self.provider)
# Verify expected call against scp
mock_exec = self.mocker.replace(os.execvp)
mock_exec(
"scp",
["scp", "[email protected]:foo/*", "10.1.2.3:."])
# But no other calls
calls = []
mock_exec(ARGS, KWARGS)
self.mocker.count(0, None)
self.mocker.call(lambda *args, **kwargs: calls.append((args, kwargs)))
finished = self.setup_cli_reactor()
self.mocker.replay()
yield self.unit.connect_agent()
main(["scp", "0:foo/*", "10.1.2.3:."])
yield finished
self.assertEquals(calls, [])
@inlineCallbacks
def test_passthrough_args(self):
"""Verify that args are passed through to the underlying scp command.
For example, something like the following command should be valid::
$ juju scp -o "ConnectTimeout 60" foo mysql/0:/foo/bar
"""
mock_environment = self.mocker.patch(Environment)
mock_environment.get_machine_provider()
self.mocker.result(self.provider)
# Verify expected call against scp
mock_exec = self.mocker.replace(os.execvp)
mock_exec("scp", [
"scp", "-r", "-o", "ConnectTimeout 60",
"foo", "[email protected]:/foo/bar"])
# But no other calls
calls = []
mock_exec(ARGS, KWARGS)
self.mocker.count(0, None)
self.mocker.call(lambda *args, **kwargs: calls.append((args, kwargs)))
finished = self.setup_cli_reactor()
self.mocker.replay()
main(["scp", "-r", "-o", "ConnectTimeout 60",
"foo", "mysql/0:/foo/bar"])
yield finished
self.assertEquals(calls, [])
class ControlShellTest(ServiceStateManagerTestBase, ControlToolTest, RepositoryTestBase):
@inlineCallbacks
def setUp(self):
yield super(ControlShellTest, self).setUp()
config = {"environments": {"firstenv": {"type": "dummy", "admin-secret": "homer"}}}
self.write_config(dump(config))
self.setup_exit(0)
self.config.load()
self.environment = self.config.get_default()
self.provider = self.environment.get_machine_provider()
# Setup a machine in the provider
self.provider_machine = (
yield self.provider.start_machine({"machine-id": 0, "dns-name": "antigravity.example.com"})
)[0]
# Setup the zk tree with a service, unit, and machine.
self.service = yield self.add_service_from_charm("mysql")
self.unit = yield self.service.add_unit_state()
yield self.unit.set_public_address("%s.example.com" % self.unit.unit_name.replace("/", "-"))
self.machine_manager = MachineStateManager(self.client)
self.machine = yield self.machine_manager.add_machine_state()
yield self.machine.set_instance_id(0)
yield self.unit.assign_to_machine(self.machine)
# capture the output.
self.output = self.capture_logging("juju.control.cli", level=logging.INFO)
self.stderr = self.capture_stream("stderr")
@inlineCallbacks
def test_shell_with_unit(self):
"""
'juju ssh mysql/0' will execute ssh against the machine
hosting the unit.
"""
mock_environment = self.mocker.patch(Environment)
mock_environment.get_machine_provider()
self.mocker.result(self.provider)
mock_exec = self.mocker.replace(os.execvp)
mock_exec(
"ssh",
[
"ssh",
"-o",
"ControlPath " + self.tmp_home + "/.juju/ssh/master-%r@%h:%p",
"-o",
"ControlMaster no",
"*****@*****.**",
],
)
# Track unwanted calls:
calls = []
mock_exec(ARGS, KWARGS)
self.mocker.count(0, None)
self.mocker.call(lambda *args, **kwargs: calls.append((args, kwargs)))
finished = self.setup_cli_reactor()
self.mocker.replay()
yield self.unit.connect_agent()
main(["ssh", self.unit.unit_name])
yield finished
self.assertEquals(calls, [])
self.assertIn("Connecting to unit mysql/0 at mysql-0.example.com", self.output.getvalue())
@inlineCallbacks
def test_passthrough_args(self):
"""Verify that args are passed through to the underlying ssh command.
For example, something like the following command should be valid::
$ juju ssh -L8080:localhost:8080 -o "ConnectTimeout 60" mysql/0 ls /
"""
mock_environment = self.mocker.patch(Environment)
mock_environment.get_machine_provider()
self.mocker.result(self.provider)
mock_exec = self.mocker.replace(os.execvp)
mock_exec(
"ssh",
[
"ssh",
"-o",
"ControlPath " + self.tmp_home + "/.juju/ssh/master-%r@%h:%p",
"-o",
"ControlMaster no",
"-L8080:localhost:8080",
"-o",
"ConnectTimeout 60",
"*****@*****.**",
"ls *",
],
)
# Track unwanted calls:
calls = []
mock_exec(ARGS, KWARGS)
self.mocker.count(0, None)
self.mocker.call(lambda *args, **kwargs: calls.append((args, kwargs)))
finished = self.setup_cli_reactor()
self.mocker.replay()
yield self.unit.connect_agent()
main(["ssh", "-L8080:localhost:8080", "-o", "ConnectTimeout 60", self.unit.unit_name, "ls *"])
yield finished
self.assertEquals(calls, [])
self.assertIn("Connecting to unit mysql/0 at mysql-0.example.com", self.output.getvalue())
@inlineCallbacks
def test_shell_with_unit_and_unconnected_unit_agent(self):
"""If a unit doesn't have a connected unit agent,
the ssh command will wait till one exists before connecting.
"""
mock_environment = self.mocker.patch(Environment)
mock_environment.get_machine_provider()
self.mocker.result(self.provider)
mock_unit = self.mocker.patch(ServiceUnitState)
mock_unit.watch_agent()
self.mocker.result((succeed(False), succeed(True)))
mock_exec = self.mocker.replace(os.execvp)
mock_exec(
"ssh",
[
"ssh",
"-o",
"ControlPath " + self.tmp_home + "/.juju/ssh/master-%r@%h:%p",
"-o",
"ControlMaster no",
"*****@*****.**",
],
)
# Track unwanted calls:
calls = []
mock_exec(ARGS, KWARGS)
self.mocker.count(0, None)
self.mocker.call(lambda *args, **kwargs: calls.append((args, kwargs)))
finished = self.setup_cli_reactor()
self.mocker.replay()
yield self.unit.connect_agent()
main(["ssh", "mysql/0"])
yield finished
self.assertEquals(calls, [])
self.assertIn("Waiting for unit to come up", self.output.getvalue())
@inlineCallbacks
def test_shell_with_machine_and_unconnected_machine_agent(self):
"""If a machine doesn't have a connected machine agent,
the ssh command will wait till one exists before connecting.
"""
mock_environment = self.mocker.patch(Environment)
mock_environment.get_machine_provider()
self.mocker.result(self.provider)
mock_machine = self.mocker.patch(MachineState)
mock_machine.watch_agent()
self.mocker.result((succeed(False), succeed(True)))
mock_exec = self.mocker.replace(os.execvp)
mock_exec(
"ssh",
[
"ssh",
"-o",
"ControlPath " + self.tmp_home + "/.juju/ssh/master-%r@%h:%p",
"-o",
"ControlMaster no",
"*****@*****.**",
],
)
# Track unwanted calls:
calls = []
mock_exec(ARGS, KWARGS)
self.mocker.count(0, None)
self.mocker.call(lambda *args, **kwargs: calls.append((args, kwargs)))
finished = self.setup_cli_reactor()
self.mocker.replay()
yield self.machine.connect_agent()
main(["ssh", "0"])
yield finished
self.assertEquals(calls, [])
self.assertIn("Waiting for machine to come up", self.output.getvalue())
@inlineCallbacks
def test_shell_with_unit_and_unset_dns(self):
"""If a machine agent isn't connects, its also possible that
the provider machine may not yet have a dns name, if the
instance hasn't started. In that case after the machine agent
has connected, verify the provider dns name is valid."""
mock_environment = self.mocker.patch(Environment)
mock_environment.get_machine_provider()
self.mocker.result(self.provider)
mock_unit = self.mocker.patch(ServiceUnitState)
mock_unit.watch_agent()
address_set = Deferred()
@inlineCallbacks
def set_unit_dns():
yield self.unit.set_public_address("mysql-0.example.com")
address_set.callback(True)
self.mocker.call(set_unit_dns)
self.mocker.result((succeed(False), succeed(False)))
mock_exec = self.mocker.replace(os.execvp)
mock_exec(
"ssh",
[
"ssh",
"-o",
"ControlPath " + self.tmp_home + "/.juju/ssh/master-%r@%h:%p",
"-o",
"ControlMaster no",
"*****@*****.**",
],
)
# Track unwanted calls:
calls = []
mock_exec(ARGS, KWARGS)
self.mocker.count(0, None)
self.mocker.call(lambda *args, **kwargs: calls.append((args, kwargs)))
finished = self.setup_cli_reactor()
self.mocker.replay()
yield self.unit.set_public_address(None)
main(["ssh", "mysql/0"])
# Wait till we've set the unit address before connecting the agent.
yield address_set
yield self.unit.connect_agent()
yield finished
self.assertEquals(calls, [])
self.assertIn("Waiting for unit to come up", self.output.getvalue())
@inlineCallbacks
def test_shell_with_machine_and_unset_dns(self):
"""If a machine agent isn't connects, its also possible that
the provider machine may not yet have a dns name, if the
instance hasn't started. In that case after the machine agent
has connected, verify the provider dns name is valid."""
mock_environment = self.mocker.patch(Environment)
mock_environment.get_machine_provider()
self.mocker.result(self.provider)
mock_machine = self.mocker.patch(MachineState)
mock_machine.watch_agent()
def set_machine_dns():
self.provider_machine.dns_name = "antigravity.example.com"
self.mocker.call(set_machine_dns)
self.mocker.result((succeed(False), succeed(False)))
mock_exec = self.mocker.replace(os.execvp)
mock_exec(
"ssh",
[
"ssh",
"-o",
"ControlPath " + self.tmp_home + "/.juju/ssh/master-%r@%h:%p",
"-o",
"ControlMaster no",
"*****@*****.**",
],
)
# Track unwanted calls:
calls = []
mock_exec(ARGS, KWARGS)
self.mocker.count(0, None)
self.mocker.call(lambda *args, **kwargs: calls.append((args, kwargs)))
finished = self.setup_cli_reactor()
self.mocker.replay()
self.provider_machine.dns_name = None
yield self.machine.connect_agent()
main(["ssh", "0"])
yield finished
self.assertEquals(calls, [])
self.assertIn("Waiting for machine to come up", self.output.getvalue())
@inlineCallbacks
def test_shell_with_machine_id(self):
"""
'juju ssh <machine_id>' will execute ssh against the machine
with the corresponding id.
"""
mock_environment = self.mocker.patch(Environment)
mock_environment.get_machine_provider()
self.mocker.result(self.provider)
mock_exec = self.mocker.replace(os.execvp)
mock_exec(
"ssh",
[
"ssh",
"-o",
"ControlPath " + self.tmp_home + "/.juju/ssh/master-%r@%h:%p",
"-o",
"ControlMaster no",
"*****@*****.**",
],
)
# Track unwanted calls:
calls = []
mock_exec(ARGS, KWARGS)
self.mocker.count(0, None)
self.mocker.call(lambda *args, **kwargs: calls.append((args, kwargs)))
finished = self.setup_cli_reactor()
self.mocker.replay()
yield self.machine.connect_agent()
main(["ssh", "0"])
yield finished
self.assertEquals(calls, [])
self.assertIn("Connecting to machine 0 at antigravity.example.com", self.output.getvalue())
@inlineCallbacks
def test_shell_with_unassigned_unit(self):
"""If the service unit is not assigned, attempting to
connect, raises an error."""
mock_environment = self.mocker.patch(Environment)
mock_environment.get_machine_provider()
self.mocker.result(self.provider)
finished = self.setup_cli_reactor()
self.mocker.replay()
unit_state = yield self.service.add_unit_state()
main(["ssh", unit_state.unit_name])
yield finished
self.assertIn("Service unit 'mysql/1' is not assigned to a machine", self.stderr.getvalue())
@inlineCallbacks
def test_shell_with_invalid_machine(self):
"""If the machine does not exist, attempting to
connect, raises an error."""
mock_environment = self.mocker.patch(Environment)
mock_environment.get_machine_provider()
self.mocker.result(self.provider)
finished = self.setup_cli_reactor()
self.mocker.replay()
main(["ssh", "1"])
yield finished
self.assertIn("Machine 1 was not found", self.stderr.getvalue())
class ControlRemoveUnitTest(
ServiceStateManagerTestBase, ControlToolTest, RepositoryTestBase):
@inlineCallbacks
def setUp(self):
yield super(ControlRemoveUnitTest, self).setUp()
config = {
"environments": {"firstenv": {"type": "dummy"}}}
self.write_config(dump(config))
self.config.load()
self.environment = self.config.get_default()
self.provider = self.environment.get_machine_provider()
# Setup some service units.
self.service_state1 = yield self.add_service_from_charm("mysql")
self.service_unit1 = yield self.service_state1.add_unit_state()
self.service_unit2 = yield self.service_state1.add_unit_state()
self.service_unit3 = yield self.service_state1.add_unit_state()
# Add an assigned machine to one of them.
self.machine_manager = MachineStateManager(self.client)
self.machine = yield self.machine_manager.add_machine_state()
yield self.machine.set_instance_id(0)
yield self.service_unit1.assign_to_machine(self.machine)
# Setup a machine in the provider matching the assigned.
self.provider_machine = yield self.provider.start_machine(
{"machine-id": 0, "dns-name": "antigravity.example.com"})
self.output = self.capture_logging(level=logging.DEBUG)
self.stderr = self.capture_stream("stderr")
@inlineCallbacks
def test_remove_unit(self):
"""
'juju remove-unit <unit_name>' will remove the given unit.
"""
unit_names = yield self.service_state1.get_unit_names()
self.assertEqual(len(unit_names), 3)
finished = self.setup_cli_reactor()
self.setup_exit(0)
self.mocker.replay()
main(["remove-unit", "mysql/0"])
yield finished
topology = yield self.get_topology()
self.assertFalse(topology.has_service_unit(
self.service_state1.internal_id, self.service_unit1.internal_id))
topology = yield self.get_topology()
self.assertTrue(topology.has_service_unit(
self.service_state1.internal_id, self.service_unit2.internal_id))
self.assertFalse(
topology.get_service_units_in_machine(self.machine.internal_id))
self.assertIn(
"Unit 'mysql/0' removed from service 'mysql'",
self.output.getvalue())
@inlineCallbacks
def test_remove_multiple_units(self):
"""
'juju remove-unit <unit_name1> <unit_name2>...' removes desired units.
"""
unit_names = yield self.service_state1.get_unit_names()
self.assertEqual(len(unit_names), 3)
finished = self.setup_cli_reactor()
self.setup_exit(0)
self.mocker.replay()
main(["remove-unit", "mysql/0", "mysql/2"])
yield finished
topology = yield self.get_topology()
self.assertFalse(topology.has_service_unit(
self.service_state1.internal_id, self.service_unit1.internal_id))
topology = yield self.get_topology()
self.assertTrue(topology.has_service_unit(
self.service_state1.internal_id, self.service_unit2.internal_id))
self.assertFalse(
topology.get_service_units_in_machine(self.machine.internal_id))
self.assertIn(
"Unit 'mysql/0' removed from service 'mysql'",
self.output.getvalue())
self.assertIn(
"Unit 'mysql/2' removed from service 'mysql'",
self.output.getvalue())
@inlineCallbacks
def test_remove_unassigned_unit(self):
"""Remove unit also works if the unit is unassigned to a machine.
"""
unit_names = yield self.service_state1.get_unit_names()
self.assertEqual(len(unit_names), 3)
finished = self.setup_cli_reactor()
self.setup_exit(0)
self.mocker.replay()
main(["remove-unit", "mysql/1"])
yield finished
# verify the unit and its machine assignment.
unit_names = yield self.service_state1.get_unit_names()
self.assertEqual(len(unit_names), 2)
topology = yield self.get_topology()
topology = yield self.get_topology()
self.assertFalse(topology.has_service_unit(
self.service_state1.internal_id, self.service_unit2.internal_id))
topology = yield self.get_topology()
self.assertTrue(topology.has_service_unit(
self.service_state1.internal_id, self.service_unit1.internal_id))
@inlineCallbacks
def test_remove_unit_unknown_service(self):
"""If the service doesn't exist, return an appropriate error message.
"""
finished = self.setup_cli_reactor()
self.setup_exit(0)
self.mocker.replay()
main(["remove-unit", "volcano/0"])
yield finished
self.assertIn(
"Service 'volcano' was not found", self.stderr.getvalue())
@inlineCallbacks
def test_remove_unit_bad_parse(self):
"""Verify that a bad service unit name results in an appropriate error.
"""
finished = self.setup_cli_reactor()
self.setup_exit(0)
self.mocker.replay()
main(["remove-unit", "volcano-0"])
yield finished
self.assertIn(
"Not a proper unit name: 'volcano-0'", self.stderr.getvalue())
@inlineCallbacks
def test_remove_unit_unknown_unit(self):
"""If the unit doesn't exist an appropriate error message is returned.
"""
finished = self.setup_cli_reactor()
self.setup_exit(0)
self.mocker.replay()
main(["remove-unit", "mysql/3"])
yield finished
self.assertIn(
"Service unit 'mysql/3' was not found", self.stderr.getvalue())
@inlineCallbacks
def test_zookeeper_logging_default(self):
"""By default zookeeper logging is turned off, unless in verbose
mode.
"""
log_file = self.makeFile()
def reset_zk_log():
zookeeper.set_debug_level(0)
zookeeper.set_log_stream(sys.stdout)
self.addCleanup(reset_zk_log)
finished = self.setup_cli_reactor()
self.setup_exit(0)
# Do this as late as possible to prevent zk background logging
# from causing problems.
zookeeper.set_debug_level(zookeeper.LOG_LEVEL_INFO)
zookeeper.set_log_stream(open(log_file, "w"))
self.mocker.replay()
main(["remove-unit", "mysql/3"])
yield finished
output = open(log_file).read()
self.assertEqual(output, "")
@inlineCallbacks
def test_zookeeper_logging_enabled(self):
"""By default zookeeper logging is turned off, unless in verbose
mode.
"""
log_file = self.makeFile()
zookeeper.set_debug_level(10)
zookeeper.set_log_stream(open(log_file, "w"))
def reset_zk_log():
zookeeper.set_debug_level(0)
zookeeper.set_log_stream(sys.stdout)
self.addCleanup(reset_zk_log)
finished = self.setup_cli_reactor()
self.setup_exit(0)
self.mocker.replay()
main(["-v", "remove-unit", "mysql/3"])
yield finished
output = open(log_file).read()
self.assertTrue(output)
self.assertIn("ZOO_DEBUG", output)
class ControlShellTest(ServiceStateManagerTestBase, ControlToolTest,
RepositoryTestBase):
@inlineCallbacks
def setUp(self):
yield super(ControlShellTest, self).setUp()
config = {
"environments": {
"firstenv": {
"type": "dummy",
"admin-secret": "homer"
}
}
}
self.write_config(dump(config))
self.setup_exit(0)
self.config.load()
self.environment = self.config.get_default()
self.provider = self.environment.get_machine_provider()
# Setup a machine in the provider
self.provider_machine = (yield self.provider.start_machine({
"machine-id":
0,
"dns-name":
"antigravity.example.com"
}))[0]
# Setup the zk tree with a service, unit, and machine.
self.service = yield self.add_service_from_charm("mysql")
self.unit = yield self.service.add_unit_state()
yield self.unit.set_public_address(
"%s.example.com" % self.unit.unit_name.replace("/", "-"))
self.machine_manager = MachineStateManager(self.client)
self.machine = yield self.machine_manager.add_machine_state()
yield self.machine.set_instance_id(0)
yield self.unit.assign_to_machine(self.machine)
# capture the output.
self.output = self.capture_logging("juju.control.cli",
level=logging.INFO)
self.stderr = self.capture_stream("stderr")
@inlineCallbacks
def test_shell_with_unit(self):
"""
'juju ssh mysql/0' will execute ssh against the machine
hosting the unit.
"""
mock_environment = self.mocker.patch(Environment)
mock_environment.get_machine_provider()
self.mocker.result(self.provider)
mock_exec = self.mocker.replace(os.execvp)
mock_exec("ssh", [
"ssh", "-o",
"ControlPath " + self.tmp_home + "/.juju/ssh/master-%r@%h:%p",
"-o", "ControlMaster no", "*****@*****.**"
])
# Track unwanted calls:
calls = []
mock_exec(ARGS, KWARGS)
self.mocker.count(0, None)
self.mocker.call(lambda *args, **kwargs: calls.append((args, kwargs)))
finished = self.setup_cli_reactor()
self.mocker.replay()
yield self.unit.connect_agent()
main(["ssh", self.unit.unit_name])
yield finished
self.assertEquals(calls, [])
self.assertIn("Connecting to unit mysql/0 at mysql-0.example.com",
self.output.getvalue())
@inlineCallbacks
def test_passthrough_args(self):
"""Verify that args are passed through to the underlying ssh command.
For example, something like the following command should be valid::
$ juju ssh -L8080:localhost:8080 -o "ConnectTimeout 60" mysql/0 ls /
"""
mock_environment = self.mocker.patch(Environment)
mock_environment.get_machine_provider()
self.mocker.result(self.provider)
mock_exec = self.mocker.replace(os.execvp)
mock_exec("ssh", [
"ssh", "-o",
"ControlPath " + self.tmp_home + "/.juju/ssh/master-%r@%h:%p",
"-o", "ControlMaster no", "-L8080:localhost:8080", "-o",
"ConnectTimeout 60", "*****@*****.**", "ls *"
])
# Track unwanted calls:
calls = []
mock_exec(ARGS, KWARGS)
self.mocker.count(0, None)
self.mocker.call(lambda *args, **kwargs: calls.append((args, kwargs)))
finished = self.setup_cli_reactor()
self.mocker.replay()
yield self.unit.connect_agent()
main([
"ssh", "-L8080:localhost:8080", "-o", "ConnectTimeout 60",
self.unit.unit_name, "ls *"
])
yield finished
self.assertEquals(calls, [])
self.assertIn("Connecting to unit mysql/0 at mysql-0.example.com",
self.output.getvalue())
@inlineCallbacks
def test_shell_with_unit_and_unconnected_unit_agent(self):
"""If a unit doesn't have a connected unit agent,
the ssh command will wait till one exists before connecting.
"""
mock_environment = self.mocker.patch(Environment)
mock_environment.get_machine_provider()
self.mocker.result(self.provider)
mock_unit = self.mocker.patch(ServiceUnitState)
mock_unit.watch_agent()
self.mocker.result((succeed(False), succeed(True)))
mock_exec = self.mocker.replace(os.execvp)
mock_exec("ssh", [
"ssh", "-o",
"ControlPath " + self.tmp_home + "/.juju/ssh/master-%r@%h:%p",
"-o", "ControlMaster no", "*****@*****.**"
])
# Track unwanted calls:
calls = []
mock_exec(ARGS, KWARGS)
self.mocker.count(0, None)
self.mocker.call(lambda *args, **kwargs: calls.append((args, kwargs)))
finished = self.setup_cli_reactor()
self.mocker.replay()
yield self.unit.connect_agent()
main(["ssh", "mysql/0"])
yield finished
self.assertEquals(calls, [])
self.assertIn("Waiting for unit to come up", self.output.getvalue())
@inlineCallbacks
def test_shell_with_machine_and_unconnected_machine_agent(self):
"""If a machine doesn't have a connected machine agent,
the ssh command will wait till one exists before connecting.
"""
mock_environment = self.mocker.patch(Environment)
mock_environment.get_machine_provider()
self.mocker.result(self.provider)
mock_machine = self.mocker.patch(MachineState)
mock_machine.watch_agent()
self.mocker.result((succeed(False), succeed(True)))
mock_exec = self.mocker.replace(os.execvp)
mock_exec("ssh", [
"ssh", "-o",
"ControlPath " + self.tmp_home + "/.juju/ssh/master-%r@%h:%p",
"-o", "ControlMaster no", "*****@*****.**"
])
# Track unwanted calls:
calls = []
mock_exec(ARGS, KWARGS)
self.mocker.count(0, None)
self.mocker.call(lambda *args, **kwargs: calls.append((args, kwargs)))
finished = self.setup_cli_reactor()
self.mocker.replay()
yield self.machine.connect_agent()
main(["ssh", "0"])
yield finished
self.assertEquals(calls, [])
self.assertIn("Waiting for machine to come up", self.output.getvalue())
@inlineCallbacks
def test_shell_with_unit_and_unset_dns(self):
"""If a machine agent isn't connects, its also possible that
the provider machine may not yet have a dns name, if the
instance hasn't started. In that case after the machine agent
has connected, verify the provider dns name is valid."""
mock_environment = self.mocker.patch(Environment)
mock_environment.get_machine_provider()
self.mocker.result(self.provider)
mock_unit = self.mocker.patch(ServiceUnitState)
mock_unit.watch_agent()
address_set = Deferred()
@inlineCallbacks
def set_unit_dns():
yield self.unit.set_public_address("mysql-0.example.com")
address_set.callback(True)
self.mocker.call(set_unit_dns)
self.mocker.result((succeed(False), succeed(False)))
mock_exec = self.mocker.replace(os.execvp)
mock_exec("ssh", [
"ssh", "-o",
"ControlPath " + self.tmp_home + "/.juju/ssh/master-%r@%h:%p",
"-o", "ControlMaster no", "*****@*****.**"
])
# Track unwanted calls:
calls = []
mock_exec(ARGS, KWARGS)
self.mocker.count(0, None)
self.mocker.call(lambda *args, **kwargs: calls.append((args, kwargs)))
finished = self.setup_cli_reactor()
self.mocker.replay()
yield self.unit.set_public_address(None)
main(["ssh", "mysql/0"])
# Wait till we've set the unit address before connecting the agent.
yield address_set
yield self.unit.connect_agent()
yield finished
self.assertEquals(calls, [])
self.assertIn("Waiting for unit to come up", self.output.getvalue())
@inlineCallbacks
def test_shell_with_machine_and_unset_dns(self):
"""If a machine agent isn't connects, its also possible that
the provider machine may not yet have a dns name, if the
instance hasn't started. In that case after the machine agent
has connected, verify the provider dns name is valid."""
mock_environment = self.mocker.patch(Environment)
mock_environment.get_machine_provider()
self.mocker.result(self.provider)
mock_machine = self.mocker.patch(MachineState)
mock_machine.watch_agent()
def set_machine_dns():
self.provider_machine.dns_name = "antigravity.example.com"
self.mocker.call(set_machine_dns)
self.mocker.result((succeed(False), succeed(False)))
mock_exec = self.mocker.replace(os.execvp)
mock_exec("ssh", [
"ssh", "-o",
"ControlPath " + self.tmp_home + "/.juju/ssh/master-%r@%h:%p",
"-o", "ControlMaster no", "*****@*****.**"
])
# Track unwanted calls:
calls = []
mock_exec(ARGS, KWARGS)
self.mocker.count(0, None)
self.mocker.call(lambda *args, **kwargs: calls.append((args, kwargs)))
finished = self.setup_cli_reactor()
self.mocker.replay()
self.provider_machine.dns_name = None
yield self.machine.connect_agent()
main(["ssh", "0"])
yield finished
self.assertEquals(calls, [])
self.assertIn("Waiting for machine to come up", self.output.getvalue())
@inlineCallbacks
def test_shell_with_machine_id(self):
"""
'juju ssh <machine_id>' will execute ssh against the machine
with the corresponding id.
"""
mock_environment = self.mocker.patch(Environment)
mock_environment.get_machine_provider()
self.mocker.result(self.provider)
mock_exec = self.mocker.replace(os.execvp)
mock_exec("ssh", [
"ssh",
"-o",
"ControlPath " + self.tmp_home + "/.juju/ssh/master-%r@%h:%p",
"-o",
"ControlMaster no",
"*****@*****.**",
])
# Track unwanted calls:
calls = []
mock_exec(ARGS, KWARGS)
self.mocker.count(0, None)
self.mocker.call(lambda *args, **kwargs: calls.append((args, kwargs)))
finished = self.setup_cli_reactor()
self.mocker.replay()
yield self.machine.connect_agent()
main(["ssh", "0"])
yield finished
self.assertEquals(calls, [])
self.assertIn("Connecting to machine 0 at antigravity.example.com",
self.output.getvalue())
@inlineCallbacks
def test_shell_with_unassigned_unit(self):
"""If the service unit is not assigned, attempting to
connect, raises an error."""
mock_environment = self.mocker.patch(Environment)
mock_environment.get_machine_provider()
self.mocker.result(self.provider)
finished = self.setup_cli_reactor()
self.mocker.replay()
unit_state = yield self.service.add_unit_state()
main(["ssh", unit_state.unit_name])
yield finished
self.assertIn("Service unit 'mysql/1' is not assigned to a machine",
self.stderr.getvalue())
@inlineCallbacks
def test_shell_with_invalid_machine(self):
"""If the machine does not exist, attempting to
connect, raises an error."""
mock_environment = self.mocker.patch(Environment)
mock_environment.get_machine_provider()
self.mocker.result(self.provider)
finished = self.setup_cli_reactor()
self.mocker.replay()
main(["ssh", "1"])
yield finished
self.assertIn("Machine 1 was not found", self.stderr.getvalue())
