def __init__(self, client, is_running, provider):
"""Initialize a Firewall Manager.
:param client: A connected zookeeper client.
:param is_running: A function (usually a bound method) that
returns whether the associated agent is still running or
not.
:param provider: A machine provider, used for making the
actual changes in the environment to firewall settings.
"""
self.machine_state_manager = MachineStateManager(client)
self.service_state_manager = ServiceStateManager(client)
self.is_running = is_running
self.provider = provider
# Track all currently watched machines, using machine ID.
self._watched_machines = set()
# Map service name to either NotExposed or set of exposed unit names.
# If a service name is present in the dictionary, it means its
# respective expose node is being watched.
self._watched_services = {}
# Machines to retry open_close_ports because of earlier errors
self._retry_machines_on_port_error = set()
# Registration of observers for corresponding actions
self._open_close_ports_observers = set()
self._open_close_ports_on_machine_observers = set()
def test_open_close_ports_on_machine(self):
"""Verify opening/closing ports on a machine works properly.
In particular this is done without watch support."""
manager = MachineStateManager(self.client)
machine = yield manager.add_machine_state()
yield self.firewall_manager.process_machine(machine)
# Expose a service
wordpress = yield self.add_service("wordpress")
yield wordpress.set_exposed_flag()
wordpress_0 = yield wordpress.add_unit_state()
yield wordpress_0.open_port(80, "tcp")
yield wordpress_0.open_port(443, "tcp")
yield wordpress_0.assign_to_machine(machine)
yield self.firewall_manager.open_close_ports_on_machine(machine.id)
self.assertEqual((yield self.get_provider_ports(machine)),
set([(80, "tcp"), (443, "tcp")]))
self.assertIn("Opened 80/tcp on provider machine 0",
self.output.getvalue())
self.assertIn("Opened 443/tcp on provider machine 0",
self.output.getvalue())
# Now change port setup
yield wordpress_0.open_port(8080, "tcp")
yield wordpress_0.close_port(443, "tcp")
yield self.firewall_manager.open_close_ports_on_machine(machine.id)
self.assertEqual((yield self.get_provider_ports(machine)),
set([(80, "tcp"), (8080, "tcp")]))
self.assertIn("Opened 8080/tcp on provider machine 0",
self.output.getvalue())
self.assertIn("Closed 443/tcp on provider machine 0",
self.output.getvalue())
def test_watch_machine_changes_ignores_running_machine(self):
"""
If there is an existing machine instance and state, when a
new machine state is added, the existing instance is preserved,
and a new instance is created.
"""
manager = MachineStateManager(self.client)
machine_state0 = yield manager.add_machine_state()
machines = yield self.agent.provider.start_machine(
{"machine-id": machine_state0.id})
machine = machines.pop()
yield machine_state0.set_instance_id(machine.instance_id)
machine_state1 = yield manager.add_machine_state()
machines = yield self.agent.provider.get_machines()
self.assertEquals(len(machines), 1)
yield self.agent.watch_machine_changes(
None, [machine_state0.id, machine_state1.id])
machines = yield self.agent.provider.get_machines()
self.assertEquals(len(machines), 2)
instance_id = yield machine_state1.get_instance_id()
self.assertEqual(instance_id, 1)
def terminate_machine(config, environment, verbose, log, machine_ids):
"""Terminates the machines in `machine_ids`.
Like the underlying code in MachineStateManager, it's permissible
if the machine ID is already terminated or even never running. If
we determine this is not desired behavior, presumably propagate
that back to the state manager.
XXX However, we currently special case support of not terminating
the "root" machine, that is the one running the provisioning
agent. At some point, this will be managed like any other service,
but until then it seems best to ensure it's not terminated at this
level.
"""
provider = environment.get_machine_provider()
client = yield provider.connect()
terminated_machine_ids = []
try:
machine_state_manager = MachineStateManager(client)
for machine_id in machine_ids:
if machine_id == 0:
raise CannotTerminateMachine(0,
"environment would be destroyed")
removed = yield machine_state_manager.remove_machine_state(
machine_id)
if not removed:
raise MachineStateNotFound(machine_id)
terminated_machine_ids.append(machine_id)
finally:
yield client.close()
if terminated_machine_ids:
log.info("Machines terminated: %s",
", ".join(str(id) for id in terminated_machine_ids))
def test_process_machines_non_concurrency(self):
"""
Process machines should only be executed serially by an
agent.
"""
manager = MachineStateManager(self.client)
machine_state0 = yield manager.add_machine_state()
machine_state1 = yield manager.add_machine_state()
call_1 = self.agent.process_machines([machine_state0.id])
# The second call should return immediately due to the
# instance attribute guard.
call_2 = self.agent.process_machines([machine_state1.id])
self.assertEqual(call_2.called, True)
self.assertEqual(call_2.result, False)
# The first call should have started a provider machine
yield call_1
machines = yield self.agent.provider.get_machines()
self.assertEquals(len(machines), 1)
instance_id_0 = yield machine_state0.get_instance_id()
self.assertEqual(instance_id_0, 0)
instance_id_1 = yield machine_state1.get_instance_id()
self.assertEqual(instance_id_1, None)
def test_transient_provider_error_on_get_machines(self):
manager = MachineStateManager(self.client)
machine_state0 = yield manager.add_machine_state()
mock_provider = self.mocker.patch(self.agent.provider)
mock_provider.get_machines()
self.mocker.result(fail(ProviderInteractionError()))
mock_provider.get_machines()
self.mocker.passthrough()
self.mocker.replay()
try:
yield self.agent.process_machines([machine_state0.id])
except:
self.fail("Should not raise")
instance_id = yield machine_state0.get_instance_id()
self.assertEqual(instance_id, None)
yield self.agent.process_machines([machine_state0.id])
instance_id = yield machine_state0.get_instance_id()
self.assertEqual(instance_id, 0)
self.assertIn("Cannot get machine list", self.output.getvalue())
def test_watch_machine_changes_ignores_running_machine(self):
"""
If there is an existing machine instance and state, when a
new machine state is added, the existing instance is preserved,
and a new instance is created.
"""
manager = MachineStateManager(self.client)
machine_state0 = yield manager.add_machine_state()
machines = yield self.agent.provider.start_machine(
{"machine-id": machine_state0.id})
machine = machines.pop()
yield machine_state0.set_instance_id(machine.instance_id)
machine_state1 = yield manager.add_machine_state()
machines = yield self.agent.provider.get_machines()
self.assertEquals(len(machines), 1)
yield self.agent.watch_machine_changes(
None, [machine_state0.id, machine_state1.id])
machines = yield self.agent.provider.get_machines()
self.assertEquals(len(machines), 2)
instance_id = yield machine_state1.get_instance_id()
self.assertEqual(instance_id, 1)
def test_open_close_ports_on_machine_not_yet_provided(self):
"""Verify that opening/closing ports will eventually succeed
once a machine is provided.
"""
manager = MachineStateManager(self.client)
machine = yield manager.add_machine_state()
wordpress = yield self.add_service("wordpress")
yield wordpress.set_exposed_flag()
wordpress_0 = yield wordpress.add_unit_state()
yield wordpress_0.open_port(80, "tcp")
yield wordpress_0.open_port(443, "tcp")
yield wordpress_0.assign_to_machine(machine)
# First attempt to open ports quietly fails (except for
# logging) because the machine has not yet been provisioned
yield self.firewall_manager.open_close_ports_on_machine(machine.id)
self.assertIn("No provisioned machine for machine 0",
self.output.getvalue())
yield self.provide_machine(machine)
# Machine is now provisioned (normally visible in the
# provisioning agent through periodic rescan and corresponding
# watches)
yield self.firewall_manager.open_close_ports_on_machine(machine.id)
self.assertEqual((yield self.get_provider_ports(machine)),
set([(80, "tcp"), (443, "tcp")]))
def setUp(self):
yield super(ControlDebugHookTest, self).setUp()
config = {
"environments": {
"firstenv": {
"type": "dummy", "admin-secret": "homer"}}}
self.write_config(yaml.dump(config))
self.config.load()
self.environment = self.config.get_default()
self.provider = self.environment.get_machine_provider()
# Setup a machine in the provider
self.provider_machine = (yield self.provider.start_machine(
{"machine-id": 0, "dns-name": "antigravity.example.com"}))[0]
# Setup the zk tree with a service, unit, and machine.
self.service = yield self.add_service_from_charm("mysql")
self.unit = yield self.service.add_unit_state()
self.machine_manager = MachineStateManager(self.client)
self.machine = yield self.machine_manager.add_machine_state()
yield self.machine.set_instance_id(0)
yield self.unit.assign_to_machine(self.machine)
# capture the output.
self.output = self.capture_logging(
"juju.control.cli", level=logging.INFO)
self.stderr = self.capture_stream("stderr")
self.setup_exit(0)
def test_process_machines_non_concurrency(self):
"""
Process machines should only be executed serially by an
agent.
"""
manager = MachineStateManager(self.client)
machine_state0 = yield manager.add_machine_state()
machine_state1 = yield manager.add_machine_state()
call_1 = self.agent.process_machines([machine_state0.id])
# The second call should return immediately due to the
# instance attribute guard.
call_2 = self.agent.process_machines([machine_state1.id])
self.assertEqual(call_2.called, True)
self.assertEqual(call_2.result, False)
# The first call should have started a provider machine
yield call_1
machines = yield self.agent.provider.get_machines()
self.assertEquals(len(machines), 1)
instance_id_0 = yield machine_state0.get_instance_id()
self.assertEqual(instance_id_0, 0)
instance_id_1 = yield machine_state1.get_instance_id()
self.assertEqual(instance_id_1, None)
def test_transient_provider_error_on_start_machine(self):
"""
If there's an error when processing changes, the agent should log
the error and continue.
"""
manager = MachineStateManager(self.client)
machine_state0 = yield manager.add_machine_state()
machine_state1 = yield manager.add_machine_state()
mock_provider = self.mocker.patch(self.agent.provider)
mock_provider.start_machine({"machine-id": 0})
self.mocker.result(fail(ProviderInteractionError()))
mock_provider.start_machine({"machine-id": 1})
self.mocker.passthrough()
self.mocker.replay()
yield self.agent.watch_machine_changes(
[], [machine_state0.id, machine_state1.id])
machine1_instance_id = yield machine_state1.get_instance_id()
self.assertEqual(machine1_instance_id, 0)
self.assertIn(
"Cannot process machine 0",
self.output.getvalue())
def test_transient_provider_error_on_get_machines(self):
manager = MachineStateManager(self.client)
machine_state0 = yield manager.add_machine_state()
mock_provider = self.mocker.patch(self.agent.provider)
mock_provider.get_machines()
self.mocker.result(fail(ProviderInteractionError()))
mock_provider.get_machines()
self.mocker.passthrough()
self.mocker.replay()
try:
yield self.agent.process_machines([machine_state0.id])
except:
self.fail("Should not raise")
instance_id = yield machine_state0.get_instance_id()
self.assertEqual(instance_id, None)
yield self.agent.process_machines(
[machine_state0.id])
instance_id = yield machine_state0.get_instance_id()
self.assertEqual(instance_id, 0)
self.assertIn(
"Cannot get machine list",
self.output.getvalue())
def setUp(self):
yield super(ControlRemoveUnitTest, self).setUp()
config = {"environments": {"firstenv": {"type": "dummy"}}}
self.write_config(dump(config))
self.config.load()
self.environment = self.config.get_default()
self.provider = self.environment.get_machine_provider()
# Setup some service units.
self.service_state1 = yield self.add_service_from_charm("mysql")
self.service_unit1 = yield self.service_state1.add_unit_state()
self.service_unit2 = yield self.service_state1.add_unit_state()
self.service_unit3 = yield self.service_state1.add_unit_state()
# Add an assigned machine to one of them.
self.machine_manager = MachineStateManager(self.client)
self.machine = yield self.machine_manager.add_machine_state()
yield self.machine.set_instance_id(0)
yield self.service_unit1.assign_to_machine(self.machine)
# Setup a machine in the provider matching the assigned.
self.provider_machine = yield self.provider.start_machine({
"machine-id":
0,
"dns-name":
"antigravity.example.com"
})
self.output = self.capture_logging(level=logging.DEBUG)
self.stderr = self.capture_stream("stderr")
def test_open_close_ports_on_machine(self):
"""Verify opening/closing ports on a machine works properly.
In particular this is done without watch support."""
manager = MachineStateManager(self.client)
machine = yield manager.add_machine_state()
yield self.firewall_manager.process_machine(machine)
# Expose a service
wordpress = yield self.add_service("wordpress")
yield wordpress.set_exposed_flag()
wordpress_0 = yield wordpress.add_unit_state()
yield wordpress_0.open_port(80, "tcp")
yield wordpress_0.open_port(443, "tcp")
yield wordpress_0.assign_to_machine(machine)
yield self.firewall_manager.open_close_ports_on_machine(machine.id)
self.assertEqual((yield self.get_provider_ports(machine)),
set([(80, "tcp"), (443, "tcp")]))
self.assertIn("Opened 80/tcp on provider machine 0",
self.output.getvalue())
self.assertIn("Opened 443/tcp on provider machine 0",
self.output.getvalue())
# Now change port setup
yield wordpress_0.open_port(8080, "tcp")
yield wordpress_0.close_port(443, "tcp")
yield self.firewall_manager.open_close_ports_on_machine(machine.id)
self.assertEqual((yield self.get_provider_ports(machine)),
set([(80, "tcp"), (8080, "tcp")]))
self.assertIn("Opened 8080/tcp on provider machine 0",
self.output.getvalue())
self.assertIn("Closed 443/tcp on provider machine 0",
self.output.getvalue())
def test_open_close_ports_on_machine_not_yet_provided(self):
"""Verify that opening/closing ports will eventually succeed
once a machine is provided.
"""
manager = MachineStateManager(self.client)
machine = yield manager.add_machine_state()
wordpress = yield self.add_service("wordpress")
yield wordpress.set_exposed_flag()
wordpress_0 = yield wordpress.add_unit_state()
yield wordpress_0.open_port(80, "tcp")
yield wordpress_0.open_port(443, "tcp")
yield wordpress_0.assign_to_machine(machine)
# First attempt to open ports quietly fails (except for
# logging) because the machine has not yet been provisioned
yield self.firewall_manager.open_close_ports_on_machine(machine.id)
self.assertIn("No provisioned machine for machine 0",
self.output.getvalue())
yield self.provide_machine(machine)
# Machine is now provisioned (normally visible in the
# provisioning agent through periodic rescan and corresponding
# watches)
yield self.firewall_manager.open_close_ports_on_machine(machine.id)
self.assertEqual((yield self.get_provider_ports(machine)),
set([(80, "tcp"), (443, "tcp")]))
def start(self):
"""Start the machine agent.
Creates state directories on the machine, retrieves the machine state,
and enables watch on assigned units.
"""
if not os.path.exists(self.units_directory):
os.makedirs(self.units_directory)
if not os.path.exists(self.unit_state_directory):
os.makedirs(self.unit_state_directory)
# Get state managers we'll be utilizing.
self.service_state_manager = ServiceStateManager(self.client)
self.unit_deployer = UnitDeployer(
self.client, self.get_machine_id(), self.config["juju_directory"])
yield self.unit_deployer.start()
# Retrieve the machine state for the machine we represent.
machine_manager = MachineStateManager(self.client)
self.machine_state = yield machine_manager.get_machine_state(
self.get_machine_id())
# Watch assigned units for the machine.
if self.get_watch_enabled():
self.machine_state.watch_assigned_units(
self.watch_service_units)
# Connect the machine agent, broadcasting presence to the world.
yield self.machine_state.connect_agent()
log.info("Machine agent started id:%s" % self.get_machine_id())
def terminate_machine(config, environment, verbose, log, machine_ids):
"""Terminates the machines in `machine_ids`.
Like the underlying code in MachineStateManager, it's permissible
if the machine ID is already terminated or even never running. If
we determine this is not desired behavior, presumably propagate
that back to the state manager.
XXX However, we currently special case support of not terminating
the "root" machine, that is the one running the provisioning
agent. At some point, this will be managed like any other service,
but until then it seems best to ensure it's not terminated at this
level.
"""
provider = environment.get_machine_provider()
client = yield provider.connect()
terminated_machine_ids = []
try:
machine_state_manager = MachineStateManager(client)
for machine_id in machine_ids:
if machine_id == 0:
raise CannotTerminateMachine(
0, "environment would be destroyed")
removed = yield machine_state_manager.remove_machine_state(
machine_id)
if not removed:
raise MachineStateNotFound(machine_id)
terminated_machine_ids.append(machine_id)
finally:
yield client.close()
if terminated_machine_ids:
log.info(
"Machines terminated: %s",
", ".join(str(id) for id in terminated_machine_ids))
def setUp(self):
yield super(MachineStateManagerTest, self).setUp()
self.charm_state_manager = CharmStateManager(self.client)
self.machine_state_manager = MachineStateManager(self.client)
self.service_state_manager = ServiceStateManager(self.client)
self.charm_state = yield self.charm_state_manager.add_charm_state(
local_charm_id(self.charm), self.charm, "")
def test_open_close_ports_on_unassigned_machine(self):
"""Verify corner case that nothing happens on an unassigned machine."""
manager = MachineStateManager(self.client)
machine = yield manager.add_machine_state()
yield self.provide_machine(machine)
yield self.firewall_manager.process_machine(machine)
yield self.firewall_manager.open_close_ports_on_machine(machine.id)
self.assertEqual((yield self.get_provider_ports(machine)), set())
def test_open_close_ports_on_unassigned_machine(self):
"""Verify corner case that nothing happens on an unassigned machine."""
manager = MachineStateManager(self.client)
machine = yield manager.add_machine_state()
yield self.provide_machine(machine)
yield self.firewall_manager.process_machine(machine)
yield self.firewall_manager.open_close_ports_on_machine(machine.id)
self.assertEqual((yield self.get_provider_ports(machine)),
set())
def get_agent_config(self):
# gets invoked by AgentTestBase.setUp
options = yield super(MachineAgentTest, self).get_agent_config()
machine_state_manager = MachineStateManager(self.client)
self.machine_state = yield machine_state_manager.add_machine_state()
self.change_environment(JUJU_MACHINE_ID="0",
JUJU_HOME=self.juju_directory)
options["machine_id"] = str(self.machine_state.id)
# Start the agent with watching enabled
returnValue(options)
def get_ip_address_for_machine(client, provider, machine_id):
"""Returns public DNS name and machine state for the machine id.
:param client: a connected zookeeper client.
:param provider: the `MachineProvider` in charge of the juju.
:param machine_id: machine ID of the desired machine to connect to.
:return: tuple of the DNS name and a `MachineState`.
"""
manager = MachineStateManager(client)
machine_state = yield manager.get_machine_state(machine_id)
instance_id = yield machine_state.get_instance_id()
provider_machine = yield provider.get_machine(instance_id)
returnValue((provider_machine.dns_name, machine_state))
def test_transient_unhandled_error_in_process_machines(self):
"""Verify that watch_machine_changes handles the exception.
Provider implementations may use libraries like txaws that do
not handle every error. However, this should not stop the
watch from re-establishing itself, as will be the case if the
exception is not caught.
"""
manager = MachineStateManager(self.client)
machine_state0 = yield manager.add_machine_state()
machine_state1 = yield manager.add_machine_state()
# Simulate a failure scenario seen occasionally when working
# with OpenStack and txaws
mock_agent = self.mocker.patch(self.agent)
# Simulate transient error
mock_agent.process_machines([machine_state0.id])
self.mocker.result(fail(
TypeError("'NoneType' object is not iterable")))
# Let it succeed on second try. In this case, the scenario is
# that the watch triggered before the periodic_machine_check
# was run again
mock_agent.process_machines([machine_state0.id, machine_state1.id])
self.mocker.passthrough()
self.mocker.replay()
# Verify that watch_machine_changes does not fail even in the case of
# the transient error, although no work was done
try:
yield self.agent.watch_machine_changes([], [machine_state0.id])
except:
self.fail("Should not raise")
instance_id = yield machine_state0.get_instance_id()
self.assertEqual(instance_id, None)
# Second attempt, verifiy it did in fact process the machine
yield self.agent.watch_machine_changes(
[machine_state0.id], [machine_state0.id, machine_state1.id])
self.assertEqual((yield machine_state0.get_instance_id()), 0)
self.assertEqual((yield machine_state1.get_instance_id()), 1)
# But only after attempting and failing the first time
self.assertIn(
"Got unexpected exception in processing machines, will retry",
self.output.getvalue())
self.assertIn(
"'NoneType' object is not iterable",
self.output.getvalue())
def get_agent_config(self):
# gets invoked by AgentTestBase.setUp
options = yield super(MachineAgentTest, self).get_agent_config()
machine_state_manager = MachineStateManager(self.client)
self.machine_state = yield machine_state_manager.add_machine_state()
self.change_environment(
JUJU_MACHINE_ID="0",
JUJU_HOME=self.juju_directory)
options["machine_id"] = str(self.machine_state.id)
# Start the agent with watching enabled
returnValue(options)
def test_transient_unhandled_error_in_process_machines(self):
"""Verify that watch_machine_changes handles the exception.
Provider implementations may use libraries like txaws that do
not handle every error. However, this should not stop the
watch from re-establishing itself, as will be the case if the
exception is not caught.
"""
manager = MachineStateManager(self.client)
machine_state0 = yield manager.add_machine_state()
machine_state1 = yield manager.add_machine_state()
# Simulate a failure scenario seen occasionally when working
# with OpenStack and txaws
mock_agent = self.mocker.patch(self.agent)
# Simulate transient error
mock_agent.process_machines([machine_state0.id])
self.mocker.result(fail(
TypeError("'NoneType' object is not iterable")))
# Let it succeed on second try. In this case, the scenario is
# that the watch triggered before the periodic_machine_check
# was run again
mock_agent.process_machines([machine_state0.id, machine_state1.id])
self.mocker.passthrough()
self.mocker.replay()
# Verify that watch_machine_changes does not fail even in the case of
# the transient error, although no work was done
try:
yield self.agent.watch_machine_changes([], [machine_state0.id])
except:
self.fail("Should not raise")
instance_id = yield machine_state0.get_instance_id()
self.assertEqual(instance_id, None)
# Second attempt, verifiy it did in fact process the machine
yield self.agent.watch_machine_changes(
[machine_state0.id], [machine_state0.id, machine_state1.id])
self.assertEqual((yield machine_state0.get_instance_id()), 0)
self.assertEqual((yield machine_state1.get_instance_id()), 1)
# But only after attempting and failing the first time
self.assertIn(
"Got unexpected exception in processing machines, will retry",
self.output.getvalue())
self.assertIn("'NoneType' object is not iterable",
self.output.getvalue())
def test_open_close_ports_on_machine_will_retry(self):
"""Verify port mgmt for a machine will retry if there's a failure."""
mock_provider = self.mocker.patch(MachineProvider)
mock_provider.open_port(MATCH_MACHINE, 0, 80, "tcp")
self.mocker.result(fail(
TypeError("'NoneType' object is not iterable")))
mock_provider.open_port(MATCH_MACHINE, 0, 80, "tcp")
self.mocker.result(fail(
ProviderInteractionError("Some sort of EC2 problem")))
mock_provider.open_port(MATCH_MACHINE, 0, 80, "tcp")
self.mocker.passthrough()
self.mocker.replay()
manager = MachineStateManager(self.client)
machine = yield manager.add_machine_state()
yield self.provide_machine(machine)
# Expose a service and attempt to open/close ports. The first
# attempt will see the simulated failure.
wordpress = yield self.add_service("wordpress")
yield wordpress.set_exposed_flag()
wordpress_0 = yield wordpress.add_unit_state()
yield wordpress_0.assign_to_machine(machine)
yield self.firewall_manager.process_machine(machine)
yield wordpress_0.open_port(80, "tcp")
yield self.firewall_manager.open_close_ports_on_machine(machine.id)
self.assertEqual((yield self.get_provider_ports(machine)),
set())
self.assertIn(
"Got exception in opening/closing ports, will retry",
self.output.getvalue())
self.assertIn("TypeError: 'NoneType' object is not iterable",
self.output.getvalue())
# Retries will now happen in the periodic recheck. First one
# still fails due to simulated error.
yield self.firewall_manager.process_machine(machine)
self.assertEqual((yield self.get_provider_ports(machine)),
set())
self.assertIn("ProviderInteractionError: Some sort of EC2 problem",
self.output.getvalue())
# Third time is the charm in the mock setup, the recheck succeeds
yield self.firewall_manager.process_machine(machine)
self.assertEqual((yield self.get_provider_ports(machine)),
set([(80, "tcp")]))
self.assertIn("Opened 80/tcp on provider machine 0",
self.output.getvalue())
def __init__(self, client, is_running, provider):
"""Initialize a Firewall Manager.
:param client: A connected zookeeper client.
:param is_running: A function (usually a bound method) that
returns whether the associated agent is still running or
not.
:param provider: A machine provider, used for making the
actual changes in the environment to firewall settings.
"""
self.machine_state_manager = MachineStateManager(client)
self.service_state_manager = ServiceStateManager(client)
self.is_running = is_running
self.provider = provider
# Track all currently watched machines, using machine ID.
self._watched_machines = set()
# Map service name to either NotExposed or set of exposed unit names.
# If a service name is present in the dictionary, it means its
# respective expose node is being watched.
self._watched_services = {}
# Machines to retry open_close_ports because of earlier errors
self._retry_machines_on_port_error = set()
# Registration of observers for corresponding actions
self._open_close_ports_observers = set()
self._open_close_ports_on_machine_observers = set()
def setUp(self):
yield super(ControlRemoveUnitTest, self).setUp()
config = {
"environments": {"firstenv": {"type": "dummy"}}}
self.write_config(dump(config))
self.config.load()
self.environment = self.config.get_default()
self.provider = self.environment.get_machine_provider()
# Setup some service units.
self.service_state1 = yield self.add_service_from_charm("mysql")
self.service_unit1 = yield self.service_state1.add_unit_state()
self.service_unit2 = yield self.service_state1.add_unit_state()
self.service_unit3 = yield self.service_state1.add_unit_state()
# Add an assigned machine to one of them.
self.machine_manager = MachineStateManager(self.client)
self.machine = yield self.machine_manager.add_machine_state()
yield self.machine.set_instance_id(0)
yield self.service_unit1.assign_to_machine(self.machine)
# Setup a machine in the provider matching the assigned.
self.provider_machine = yield self.provider.start_machine(
{"machine-id": 0, "dns-name": "antigravity.example.com"})
self.output = self.capture_logging(level=logging.DEBUG)
self.stderr = self.capture_stream("stderr")
def test_open_close_ports_on_machine_will_retry(self):
"""Verify port mgmt for a machine will retry if there's a failure."""
mock_provider = self.mocker.patch(MachineProvider)
mock_provider.open_port(MATCH_MACHINE, 0, 80, "tcp")
self.mocker.result(fail(
TypeError("'NoneType' object is not iterable")))
mock_provider.open_port(MATCH_MACHINE, 0, 80, "tcp")
self.mocker.result(
fail(ProviderInteractionError("Some sort of EC2 problem")))
mock_provider.open_port(MATCH_MACHINE, 0, 80, "tcp")
self.mocker.passthrough()
self.mocker.replay()
manager = MachineStateManager(self.client)
machine = yield manager.add_machine_state()
yield self.provide_machine(machine)
# Expose a service and attempt to open/close ports. The first
# attempt will see the simulated failure.
wordpress = yield self.add_service("wordpress")
yield wordpress.set_exposed_flag()
wordpress_0 = yield wordpress.add_unit_state()
yield wordpress_0.assign_to_machine(machine)
yield self.firewall_manager.process_machine(machine)
yield wordpress_0.open_port(80, "tcp")
yield self.firewall_manager.open_close_ports_on_machine(machine.id)
self.assertEqual((yield self.get_provider_ports(machine)), set())
self.assertIn("Got exception in opening/closing ports, will retry",
self.output.getvalue())
self.assertIn("TypeError: 'NoneType' object is not iterable",
self.output.getvalue())
# Retries will now happen in the periodic recheck. First one
# still fails due to simulated error.
yield self.firewall_manager.process_machine(machine)
self.assertEqual((yield self.get_provider_ports(machine)), set())
self.assertIn("ProviderInteractionError: Some sort of EC2 problem",
self.output.getvalue())
# Third time is the charm in the mock setup, the recheck succeeds
yield self.firewall_manager.process_machine(machine)
self.assertEqual((yield self.get_provider_ports(machine)),
set([(80, "tcp")]))
self.assertIn("Opened 80/tcp on provider machine 0",
self.output.getvalue())
def test_machine_state_reflects_invalid_provider_state(self):
"""
If a machine state has an invalid instance_id, it should be detected,
and a new machine started and the machine state updated with the
new instance_id.
"""
machine_manager = MachineStateManager(self.client)
m1 = yield machine_manager.add_machine_state()
yield m1.set_instance_id("zebra")
m2 = yield machine_manager.add_machine_state()
yield self.agent.watch_machine_changes(None, [m1.id, m2.id])
m1_instance_id = yield m1.get_instance_id()
self.assertEqual(m1_instance_id, 0)
m2_instance_id = yield m2.get_instance_id()
self.assertEqual(m2_instance_id, 1)
def test_machine_state_reflects_invalid_provider_state(self):
"""
If a machine state has an invalid instance_id, it should be detected,
and a new machine started and the machine state updated with the
new instance_id.
"""
machine_manager = MachineStateManager(self.client)
m1 = yield machine_manager.add_machine_state()
yield m1.set_instance_id("zebra")
m2 = yield machine_manager.add_machine_state()
yield self.agent.watch_machine_changes(None, [m1.id, m2.id])
m1_instance_id = yield m1.get_instance_id()
self.assertEqual(m1_instance_id, 0)
m2_instance_id = yield m2.get_instance_id()
self.assertEqual(m2_instance_id, 1)
def test_process_machine_is_called(self):
"""Verify FirewallManager is called when machines are processed"""
from juju.state.firewall import FirewallManager
mock_manager = self.mocker.patch(FirewallManager)
seen = []
def record_machine(machine):
seen.append(machine)
return succeed(True)
mock_manager.process_machine(MATCH_MACHINE_STATE)
self.mocker.call(record_machine)
self.mocker.replay()
machine_manager = MachineStateManager(self.client)
machine_state = yield machine_manager.add_machine_state()
yield self.agent.process_machines([machine_state.id])
self.assertEqual(seen, [machine_state])
def test_observe_open_close_ports_on_machine(self):
"""Verify one or more observers can be established on action."""
manager = MachineStateManager(self.client)
machine = yield manager.add_machine_state()
# Add one observer, verify it gets called
calls = [set()]
self.firewall_manager.add_open_close_ports_on_machine_observer(
Observer(calls, "a"))
yield self.firewall_manager.open_close_ports_on_machine(machine.id)
self.assertEqual(calls[0], set([("a", machine.id)]))
# Reset records of calls, and then add a second observer.
# Verify both get called.
calls[0] = set()
self.firewall_manager.add_open_close_ports_on_machine_observer(
Observer(calls, "b"))
yield self.firewall_manager.open_close_ports_on_machine(machine.id)
self.assertEqual(calls[0], set([("a", machine.id), ("b", machine.id)]))
def test_start_agent_with_watch(self):
mock_reactor = self.mocker.patch(reactor)
mock_reactor.callLater(self.agent.machine_check_period,
self.agent.periodic_machine_check)
self.mocker.replay()
self.agent.set_watch_enabled(True)
yield self.agent.start()
manager = MachineStateManager(self.client)
machine_state0 = yield manager.add_machine_state()
exists_d, watch_d = self.client.exists_and_watch(
"/machines/%s" % machine_state0.internal_id)
yield exists_d
# Wait for the provisioning agent to wake and modify
# the machine id.
yield watch_d
instance_id = yield machine_state0.get_instance_id()
self.assertEqual(instance_id, 0)
def test_process_machine_is_called(self):
"""Verify FirewallManager is called when machines are processed"""
from juju.state.firewall import FirewallManager
mock_manager = self.mocker.patch(FirewallManager)
seen = []
def record_machine(machine):
seen.append(machine)
return succeed(True)
mock_manager.process_machine(MATCH_MACHINE_STATE)
self.mocker.call(record_machine)
self.mocker.replay()
machine_manager = MachineStateManager(self.client)
machine_state = yield machine_manager.add_machine_state()
yield self.agent.process_machines([machine_state.id])
self.assertEqual(seen, [machine_state])
def test_process_machine_ignores_stop_watcher(self):
"""Verify that process machine catches `StopWatcher`.
`process_machine` calls `open_close_ports_on_machine`, which
as verified in an earlier test, raises a `StopWatcher`
exception to shutdown watches that use it in the event of
agent shutdown. Verify this dual usage does not cause issues
while the agent is being stopped for this usage.
"""
mock_provider = self.mocker.patch(MachineProvider)
mock_provider.open_port(MATCH_MACHINE, 0, 80, "tcp")
self.mocker.result(fail(
TypeError("'NoneType' object is not iterable")))
self.mocker.replay()
manager = MachineStateManager(self.client)
machine = yield manager.add_machine_state()
yield self.provide_machine(machine)
# Expose a service and attempt to open/close ports. The first
# attempt will see the simulated failure.
wordpress = yield self.add_service("wordpress")
yield wordpress.set_exposed_flag()
wordpress_0 = yield wordpress.add_unit_state()
yield wordpress_0.assign_to_machine(machine)
yield self.firewall_manager.process_machine(machine)
yield wordpress_0.open_port(80, "tcp")
yield self.firewall_manager.open_close_ports_on_machine(machine.id)
self.assertEqual((yield self.get_provider_ports(machine)),
set())
self.assertIn(
"Got exception in opening/closing ports, will retry",
self.output.getvalue())
self.assertIn("TypeError: 'NoneType' object is not iterable",
self.output.getvalue())
# Stop the provisioning agent
self.stop()
# But retries can potentially still happening anyway, just
# make certain nothing bad happens.
yield self.firewall_manager.process_machine(machine)
def test_initialize(self):
yield self.layout.initialize()
yield self.assert_existence_and_acl("/charms")
yield self.assert_existence_and_acl("/services")
yield self.assert_existence_and_acl("/units")
yield self.assert_existence_and_acl("/machines")
yield self.assert_existence_and_acl("/relations")
yield self.assert_existence_and_acl("/initialized")
# To check that the constraints landed correctly, we need the
# environment config to have been sent, or we won't be able to
# get a provider to help us construct the appropriate objects.
yield self.push_default_config(with_constraints=False)
esm = EnvironmentStateManager(self.client)
env_constraints = yield esm.get_constraints()
self.assertEquals(env_constraints, {
"provider-type": "dummy",
"ubuntu-series": None,
"arch": "arm",
"cpu": None,
"mem": 512})
machine_state_manager = MachineStateManager(self.client)
machine_state = yield machine_state_manager.get_machine_state(0)
machine_constraints = yield machine_state.get_constraints()
self.assertTrue(machine_constraints.complete)
self.assertEquals(machine_constraints, {
"provider-type": "dummy",
"ubuntu-series": "cranky",
"arch": "arm",
"cpu": None,
"mem": 512})
instance_id = yield machine_state.get_instance_id()
self.assertEqual(instance_id, "i-abcdef")
settings_manager = GlobalSettingsStateManager(self.client)
self.assertEqual((yield settings_manager.get_provider_type()), "dummy")
self.assertEqual(
self.log.getvalue().strip(),
"Initializing zookeeper hierarchy")
def test_start_agent_with_watch(self):
mock_reactor = self.mocker.patch(reactor)
mock_reactor.callLater(
self.agent.machine_check_period,
self.agent.periodic_machine_check)
self.mocker.replay()
self.agent.set_watch_enabled(True)
yield self.agent.start()
manager = MachineStateManager(self.client)
machine_state0 = yield manager.add_machine_state()
exists_d, watch_d = self.client.exists_and_watch(
"/machines/%s" % machine_state0.internal_id)
yield exists_d
# Wait for the provisioning agent to wake and modify
# the machine id.
yield watch_d
instance_id = yield machine_state0.get_instance_id()
self.assertEqual(instance_id, 0)
def test_initialize(self):
yield self.layout.initialize()
yield self.assert_existence_and_acl("/charms")
yield self.assert_existence_and_acl("/services")
yield self.assert_existence_and_acl("/units")
yield self.assert_existence_and_acl("/machines")
yield self.assert_existence_and_acl("/relations")
yield self.assert_existence_and_acl("/initialized")
machine_state_manager = MachineStateManager(self.client)
machine_state = yield machine_state_manager.get_machine_state(0)
self.assertTrue(machine_state)
instance_id = yield machine_state.get_instance_id()
self.assertEqual(instance_id, "i-abcdef")
settings_manager = GlobalSettingsStateManager(self.client)
self.assertEqual((yield settings_manager.get_provider_type()), "dummy")
self.assertEqual(self.log.getvalue().strip(),
"Initializing zookeeper hierarchy")
def test_process_machine_ignores_stop_watcher(self):
"""Verify that process machine catches `StopWatcher`.
`process_machine` calls `open_close_ports_on_machine`, which
as verified in an earlier test, raises a `StopWatcher`
exception to shutdown watches that use it in the event of
agent shutdown. Verify this dual usage does not cause issues
while the agent is being stopped for this usage.
"""
mock_provider = self.mocker.patch(MachineProvider)
mock_provider.open_port(MATCH_MACHINE, 0, 80, "tcp")
self.mocker.result(fail(
TypeError("'NoneType' object is not iterable")))
self.mocker.replay()
manager = MachineStateManager(self.client)
machine = yield manager.add_machine_state()
yield self.provide_machine(machine)
# Expose a service and attempt to open/close ports. The first
# attempt will see the simulated failure.
wordpress = yield self.add_service("wordpress")
yield wordpress.set_exposed_flag()
wordpress_0 = yield wordpress.add_unit_state()
yield wordpress_0.assign_to_machine(machine)
yield self.firewall_manager.process_machine(machine)
yield wordpress_0.open_port(80, "tcp")
yield self.firewall_manager.open_close_ports_on_machine(machine.id)
self.assertEqual((yield self.get_provider_ports(machine)), set())
self.assertIn("Got exception in opening/closing ports, will retry",
self.output.getvalue())
self.assertIn("TypeError: 'NoneType' object is not iterable",
self.output.getvalue())
# Stop the provisioning agent
self.stop()
# But retries can potentially still happening anyway, just
# make certain nothing bad happens.
yield self.firewall_manager.process_machine(machine)
def test_observe_open_close_ports_on_machine(self):
"""Verify one or more observers can be established on action."""
manager = MachineStateManager(self.client)
machine = yield manager.add_machine_state()
# Add one observer, verify it gets called
calls = [set()]
self.firewall_manager.add_open_close_ports_on_machine_observer(
Observer(calls, "a"))
yield self.firewall_manager.open_close_ports_on_machine(machine.id)
self.assertEqual(calls[0], set([("a", machine.id)]))
# Reset records of calls, and then add a second observer.
# Verify both get called.
calls[0] = set()
self.firewall_manager.add_open_close_ports_on_machine_observer(
Observer(calls, "b"))
yield self.firewall_manager.open_close_ports_on_machine(machine.id)
self.assertEqual(
calls[0],
set([("a", machine.id), ("b", machine.id)]))
def test_initialize(self):
yield self.layout.initialize()
yield self.assert_existence_and_acl("/charms")
yield self.assert_existence_and_acl("/services")
yield self.assert_existence_and_acl("/units")
yield self.assert_existence_and_acl("/machines")
yield self.assert_existence_and_acl("/relations")
yield self.assert_existence_and_acl("/initialized")
machine_state_manager = MachineStateManager(self.client)
machine_state = yield machine_state_manager.get_machine_state(0)
self.assertTrue(machine_state)
instance_id = yield machine_state.get_instance_id()
self.assertEqual(instance_id, "i-abcdef")
settings_manager = GlobalSettingsStateManager(self.client)
self.assertEqual((yield settings_manager.get_provider_type()), "dummy")
self.assertEqual(
self.log.getvalue().strip(),
"Initializing zookeeper hierarchy")
def test_watch_machine_changes_processes_new_machine_id(self):
"""The agent should process a new machine id by creating it"""
manager = MachineStateManager(self.client)
machine_state0 = yield manager.add_machine_state()
machine_state1 = yield manager.add_machine_state()
yield self.agent.watch_machine_changes(
None, [machine_state0.id, machine_state1.id])
self.assertIn(
"Machines changed old:None new:[0, 1]", self.output.getvalue())
self.assertIn("Starting machine id:0", self.output.getvalue())
machines = yield self.agent.provider.get_machines()
self.assertEquals(len(machines), 2)
instance_id = yield machine_state0.get_instance_id()
self.assertEqual(instance_id, 0)
instance_id = yield machine_state1.get_instance_id()
self.assertEqual(instance_id, 1)
def test_watch_machine_changes_processes_new_machine_id(self):
"""The agent should process a new machine id by creating it"""
manager = MachineStateManager(self.client)
machine_state0 = yield manager.add_machine_state()
machine_state1 = yield manager.add_machine_state()
yield self.agent.watch_machine_changes(
None, [machine_state0.id, machine_state1.id])
self.assertIn("Machines changed old:None new:[0, 1]",
self.output.getvalue())
self.assertIn("Starting machine id:0", self.output.getvalue())
machines = yield self.agent.provider.get_machines()
self.assertEquals(len(machines), 2)
instance_id = yield machine_state0.get_instance_id()
self.assertEqual(instance_id, 0)
instance_id = yield machine_state1.get_instance_id()
self.assertEqual(instance_id, 1)
class ProvisioningTestBase(AgentTestBase):
agent_class = ProvisioningAgent
@inlineCallbacks
def setUp(self):
yield super(ProvisioningTestBase, self).setUp()
self.machine_manager = MachineStateManager(self.client)
def add_machine_state(self, constraints=None):
return self.machine_manager.add_machine_state(
constraints or series_constraints)
def start(self):
"""Start the machine agent.
Creates state directories on the machine, retrieves the machine state,
and enables watch on assigned units.
"""
# Initialize directory paths.
if not os.path.exists(self.charms_directory):
os.makedirs(self.charms_directory)
if not os.path.exists(self.units_directory):
os.makedirs(self.units_directory)
if not os.path.exists(self.unit_state_directory):
os.makedirs(self.unit_state_directory)
# Get state managers we'll be utilizing.
self.service_state_manager = ServiceStateManager(self.client)
self.charm_state_manager = CharmStateManager(self.client)
# Retrieve the machine state for the machine we represent.
machine_manager = MachineStateManager(self.client)
self.machine_state = yield machine_manager.get_machine_state(
self.get_machine_id())
# Watch assigned units for the machine.
if self.get_watch_enabled():
self.machine_state.watch_assigned_units(self.watch_service_units)
# Find out what provided the machine, and how to deploy units.
settings = GlobalSettingsStateManager(self.client)
self.provider_type = yield settings.get_provider_type()
self.deploy_factory = get_deploy_factory(self.provider_type)
# Connect the machine agent, broadcasting presence to the world.
yield self.machine_state.connect_agent()
log.info(
"Machine agent started id:%s deploy:%r provider:%r" %
(self.get_machine_id(), self.deploy_factory, self.provider_type))
def start(self):
"""Start the machine agent.
Creates state directories on the machine, retrieves the machine state,
and enables watch on assigned units.
"""
# Initialize directory paths.
if not os.path.exists(self.charms_directory):
os.makedirs(self.charms_directory)
if not os.path.exists(self.units_directory):
os.makedirs(self.units_directory)
if not os.path.exists(self.unit_state_directory):
os.makedirs(self.unit_state_directory)
# Get state managers we'll be utilizing.
self.service_state_manager = ServiceStateManager(self.client)
self.charm_state_manager = CharmStateManager(self.client)
# Retrieve the machine state for the machine we represent.
machine_manager = MachineStateManager(self.client)
self.machine_state = yield machine_manager.get_machine_state(
self.get_machine_id())
# Watch assigned units for the machine.
if self.get_watch_enabled():
self.machine_state.watch_assigned_units(
self.watch_service_units)
# Find out what provided the machine, and how to deploy units.
settings = GlobalSettingsStateManager(self.client)
self.provider_type = yield settings.get_provider_type()
self.deploy_factory = get_deploy_factory(self.provider_type)
# Connect the machine agent, broadcasting presence to the world.
yield self.machine_state.connect_agent()
log.info("Machine agent started id:%s deploy:%r provider:%r" % (
self.get_machine_id(), self.deploy_factory, self.provider_type))
def test_open_close_ports_on_machine_unexposed_service(self):
"""Verify opening/closing ports on a machine works properly.
In particular this is done without watch support."""
manager = MachineStateManager(self.client)
machine = yield manager.add_machine_state()
yield self.provide_machine(machine)
wordpress = yield self.add_service("wordpress")
wordpress_0 = yield wordpress.add_unit_state()
# Port activity, but service is not exposed
yield wordpress_0.open_port(80, "tcp")
yield wordpress_0.open_port(443, "tcp")
yield wordpress_0.assign_to_machine(machine)
yield self.firewall_manager.open_close_ports_on_machine(machine.id)
self.assertEqual((yield self.get_provider_ports(machine)), set())
# Now expose it
yield wordpress.set_exposed_flag()
yield self.firewall_manager.open_close_ports_on_machine(machine.id)
self.assertEqual((yield self.get_provider_ports(machine)),
set([(80, "tcp"), (443, "tcp")]))
def test_transient_provider_error_on_start_machine(self):
"""
If there's an error when processing changes, the agent should log
the error and continue.
"""
manager = MachineStateManager(self.client)
machine_state0 = yield manager.add_machine_state()
machine_state1 = yield manager.add_machine_state()
mock_provider = self.mocker.patch(self.agent.provider)
mock_provider.start_machine({"machine-id": 0})
self.mocker.result(fail(ProviderInteractionError()))
mock_provider.start_machine({"machine-id": 1})
self.mocker.passthrough()
self.mocker.replay()
yield self.agent.watch_machine_changes(
[], [machine_state0.id, machine_state1.id])
machine1_instance_id = yield machine_state1.get_instance_id()
self.assertEqual(machine1_instance_id, 0)
self.assertIn("Cannot process machine 0", self.output.getvalue())
def test_open_close_ports_on_machine_unexposed_service(self):
"""Verify opening/closing ports on a machine works properly.
In particular this is done without watch support."""
manager = MachineStateManager(self.client)
machine = yield manager.add_machine_state()
yield self.provide_machine(machine)
wordpress = yield self.add_service("wordpress")
wordpress_0 = yield wordpress.add_unit_state()
# Port activity, but service is not exposed
yield wordpress_0.open_port(80, "tcp")
yield wordpress_0.open_port(443, "tcp")
yield wordpress_0.assign_to_machine(machine)
yield self.firewall_manager.open_close_ports_on_machine(machine.id)
self.assertEqual((yield self.get_provider_ports(machine)),
set())
# Now expose it
yield wordpress.set_exposed_flag()
yield self.firewall_manager.open_close_ports_on_machine(machine.id)
self.assertEqual((yield self.get_provider_ports(machine)),
set([(80, "tcp"), (443, "tcp")]))
def constraints_get(env_config, environment, entity_names, log):
"""
Show the complete set of applicable constraints for each specified entity.
This will show the final computed values of all constraints (including
internal constraints which cannot be set directly via set-constraints).
"""
provider = environment.get_machine_provider()
client = yield provider.connect()
result = {}
try:
yield sync_environment_state(client, env_config, environment.name)
if entity_names:
msm = MachineStateManager(client)
ssm = ServiceStateManager(client)
for name in entity_names:
if name.isdigit():
kind = "machine"
entity = yield msm.get_machine_state(name)
elif "/" in name:
kind = "service unit"
entity = yield ssm.get_unit_state(name)
else:
kind = "service"
entity = yield ssm.get_service_state(name)
log.info("Fetching constraints for %s %s", kind, name)
constraints = yield entity.get_constraints()
result[name] = dict(constraints)
else:
esm = EnvironmentStateManager(client)
log.info("Fetching constraints for environment")
constraints = yield esm.get_constraints()
result = dict(constraints)
yaml.safe_dump(result, sys.stdout)
finally:
yield client.close()
def __init__(self, client, provider, log):
"""
Callable status command object.
`client`: ZK client connection
`provider`: machine provider for the environment
`log`: a Python stdlib logger.
"""
self.client = client
self.provider = provider
self.log = log
self.service_manager = ServiceStateManager(client)
self.relation_manager = RelationStateManager(client)
self.machine_manager = MachineStateManager(client)
self.charm_manager = CharmStateManager(client)
self._reset()
def place_unit(client, policy_name, unit_state):
"""Return machine state of unit_states assignment.
:param client: A connected zookeeper client.
:param policy_name: The name of the unit placement policy.
:param unit_state: The unit to be assigned.
:param provider_type: The type of the machine environment provider.
"""
machine_state_manager = MachineStateManager(client)
# default policy handling
if policy_name is None:
placement = _unassigned_placement
else:
placement = _PLACEMENT_LOOKUP.get(policy_name)
if placement is None:
# We should never get here, pick policy should always pick valid.
raise JujuError("Invalid policy:%r for provider" % policy_name)
return placement(client, machine_state_manager, unit_state)
def collect(scope, machine_provider, client, log):
"""Extract status information into nested dicts for rendering.
`scope`: an optional list of name specifiers. Globbing based
wildcards supported. Defaults to all units, services and
relations.
`machine_provider`: machine provider for the environment
`client`: ZK client connection
`log`: a Python stdlib logger.
"""
service_manager = ServiceStateManager(client)
relation_manager = RelationStateManager(client)
machine_manager = MachineStateManager(client)
charm_manager = CharmStateManager(client)
service_data = {}
machine_data = {}
state = dict(services=service_data, machines=machine_data)
seen_machines = set()
filter_services, filter_units = digest_scope(scope)
services = yield service_manager.get_all_service_states()
for service in services:
if len(filter_services):
found = False
for filter_service in filter_services:
if fnmatch(service.service_name, filter_service):
found = True
break
if not found:
continue
unit_data = {}
relation_data = {}
charm_id = yield service.get_charm_id()
charm = yield charm_manager.get_charm_state(charm_id)
service_data[service.service_name] = dict(units=unit_data,
charm=charm.id,
relations=relation_data)
exposed = yield service.get_exposed_flag()
if exposed:
service_data[service.service_name].update(exposed=exposed)
units = yield service.get_all_unit_states()
unit_matched = False
relations = yield relation_manager.get_relations_for_service(service)
for unit in units:
if len(filter_units):
found = False
for filter_unit in filter_units:
if fnmatch(unit.unit_name, filter_unit):
found = True
break
if not found:
continue
u = unit_data[unit.unit_name] = dict()
machine_id = yield unit.get_assigned_machine_id()
u["machine"] = machine_id
unit_workflow_client = WorkflowStateClient(client, unit)
unit_state = yield unit_workflow_client.get_state()
if not unit_state:
u["state"] = "pending"
else:
unit_connected = yield unit.has_agent()
u["state"] = unit_state if unit_connected else "down"
if exposed:
open_ports = yield unit.get_open_ports()
u["open-ports"] = [
"{port}/{proto}".format(**port_info)
for port_info in open_ports
]
u["public-address"] = yield unit.get_public_address()
# indicate we should include information about this
# machine later
seen_machines.add(machine_id)
unit_matched = True
# collect info on each relation for the service unit
relation_status = {}
for relation in relations:
try:
relation_unit = yield relation.get_unit_state(unit)
except UnitRelationStateNotFound:
# This exception will occur when relations are
# established between services without service
# units, and therefore never have any
# corresponding service relation units. This
# scenario does not occur in actual deployments,
# but can happen in test circumstances. In
# particular, it will happen with a misconfigured
# provider, which exercises this codepath.
continue # should not occur, but status should not fail
relation_workflow_client = WorkflowStateClient(
client, relation_unit)
relation_workflow_state = \
yield relation_workflow_client.get_state()
relation_status[relation.relation_name] = dict(
state=relation_workflow_state)
u["relations"] = relation_status
# after filtering units check if any matched or remove the
# service from the output
if filter_units and not unit_matched:
del service_data[service.service_name]
continue
for relation in relations:
rel_services = yield relation.get_service_states()
# A single related service implies a peer relation. More
# imply a bi-directional provides/requires relationship.
# In the later case we omit the local side of the relation
# when reporting.
if len(rel_services) > 1:
# Filter out self from multi-service relations.
rel_services = [
rsn for rsn in rel_services
if rsn.service_name != service.service_name
]
if len(rel_services) > 1:
raise ValueError("Unexpected relationship with more "
"than 2 endpoints")
rel_service = rel_services[0]
relation_data[relation.relation_name] = rel_service.service_name
machines = yield machine_manager.get_all_machine_states()
for machine_state in machines:
if (filter_services or filter_units) and \
machine_state.id not in seen_machines:
continue
instance_id = yield machine_state.get_instance_id()
m = {"instance-id": instance_id \
if instance_id is not None else "pending"}
if instance_id is not None:
try:
pm = yield machine_provider.get_machine(instance_id)
m["dns-name"] = pm.dns_name
m["instance-state"] = pm.state
if (yield machine_state.has_agent()):
# if the agent's connected, we're fine
m["state"] = "running"
else:
units = (yield machine_state.get_all_service_unit_states())
for unit in units:
unit_workflow_client = WorkflowStateClient(
client, unit)
if (yield unit_workflow_client.get_state()):
# for unit to have a state, its agent must have
# run, which implies the machine agent must have
# been running correctly at some point in the past
m["state"] = "down"
break
else:
# otherwise we're probably just still waiting
m["state"] = "not-started"
except ProviderError:
# The provider doesn't have machine information
log.error("Machine provider information missing: machine %s" %
(machine_state.id))
machine_data[machine_state.id] = m
returnValue(state)
class FirewallManager(object):
"""Manages the opening and closing of ports in the firewall.
"""
def __init__(self, client, is_running, provider):
"""Initialize a Firewall Manager.
:param client: A connected zookeeper client.
:param is_running: A function (usually a bound method) that
returns whether the associated agent is still running or
not.
:param provider: A machine provider, used for making the
actual changes in the environment to firewall settings.
"""
self.machine_state_manager = MachineStateManager(client)
self.service_state_manager = ServiceStateManager(client)
self.is_running = is_running
self.provider = provider
# Track all currently watched machines, using machine ID.
self._watched_machines = set()
# Map service name to either NotExposed or set of exposed unit names.
# If a service name is present in the dictionary, it means its
# respective expose node is being watched.
self._watched_services = {}
# Machines to retry open_close_ports because of earlier errors
self._retry_machines_on_port_error = set()
# Registration of observers for corresponding actions
self._open_close_ports_observers = set()
self._open_close_ports_on_machine_observers = set()
@inlineCallbacks
def process_machine(self, machine_state):
"""Ensures watch is setup per machine and performs any necessary retry.
:param machine_state: The machine state of the machine to be checked.
The watch that is established, via
:method:`juju.state.machine.MachineState.watch_assigned_units`,
handles the scenario where a service or service unit is
removed from the topology. Because the service unit is no
longer in the topology, the corresponding watch terminates and
is unable to `open_close_ports` in response to the
change. However, the specific machine watch will be called in
this case, and that suffices to determine that its port policy
should be checked.
In addition, this method can rely on the fact that the
provisioning agent periodically rechecks machines so as to
support retries of security group operations that failed for
that provider. This method is called by the corresponding
:method:`juju.agents.provision.ProvisioningAgent.process_machine`
in the provisioning agent.
"""
if machine_state.id in self._retry_machines_on_port_error:
self._retry_machines_on_port_error.remove(machine_state.id)
try:
yield self.open_close_ports_on_machine(machine_state.id)
except StopWatcher:
# open_close_ports_on_machine can also be called from
# a watch, so simply ignore this since it's just used
# to shutdown a watch in the case of agent shutdown
pass
def cb_watch_assigned_units(old_units, new_units):
"""Watch assigned units for changes possibly require port mgmt.
"""
log.debug("Assigned units for machine %r: old=%r, new=%r",
machine_state.id, old_units, new_units)
return self.open_close_ports_on_machine(machine_state.id)
if machine_state.id not in self._watched_machines:
self._watched_machines.add(machine_state.id)
yield machine_state.watch_assigned_units(cb_watch_assigned_units)
@inlineCallbacks
def watch_service_changes(self, old_services, new_services):
"""Manage watching service exposed status.
This method is called upon every change to the set of services
currently deployed. All services are then watched for changes
to their exposed flag setting.
:param old_services: the set of services before this change.
:param new_services: the current set of services.
"""
removed_services = old_services - new_services
for service_name in removed_services:
self._watched_services.pop(service_name, None)
for service_name in new_services:
yield self._setup_new_service_watch(service_name)
@inlineCallbacks
def _setup_new_service_watch(self, service_name):
"""Sets up the watching of the exposed flag for a new service.
If `service_name` is not watched (as known by
`self._watched_services`), adds the watch and a corresponding
entry in self._watched_services.
(This dict is necessary because there is currently no way to
introspect a service for whether it is watched or not.)
"""
if service_name in self._watched_services:
return # already watched
self._watched_services[service_name] = NotExposed
try:
service_state = yield self.service_state_manager.get_service_state(
service_name)
except ServiceStateNotFound:
log.debug("Cannot setup watch, since service %r no longer exists",
service_name)
self._watched_services.pop(service_name, None)
return
@inlineCallbacks
def cb_watch_service_exposed_flag(exposed):
if not self.is_running():
raise StopWatcher()
if exposed:
log.debug("Service %r is exposed", service_name)
else:
log.debug("Service %r is unexposed", service_name)
try:
unit_states = yield service_state.get_all_unit_states()
except StateChanged:
log.debug("Stopping watch on %r, no longer in topology",
service_name)
raise StopWatcher()
for unit_state in unit_states:
yield self.open_close_ports(unit_state)
if not exposed:
log.debug("Service %r is unexposed", service_name)
self._watched_services[service_name] = NotExposed
else:
log.debug("Service %r is exposed", service_name)
self._watched_services[service_name] = set()
yield self._setup_service_unit_watch(service_state)
yield service_state.watch_exposed_flag(cb_watch_service_exposed_flag)
log.debug("Started watch of %r on changes to being exposed",
service_name)
@inlineCallbacks
def _setup_service_unit_watch(self, service_state):
"""Setup watches on service units of newly exposed `service_name`."""
@inlineCallbacks
def cb_check_service_units(old_service_units, new_service_units):
watched_units = self._watched_services.get(
service_state.service_name, NotExposed)
if not self.is_running() or watched_units is NotExposed:
raise StopWatcher()
removed_service_units = old_service_units - new_service_units
for unit_name in removed_service_units:
watched_units.discard(unit_name)
if not self.is_running():
raise StopWatcher()
try:
unit_state = yield service_state.get_unit_state(unit_name)
except (ServiceUnitStateNotFound, StateChanged):
log.debug("Not setting up watch on %r, not in topology",
unit_name)
continue
yield self.open_close_ports(unit_state)
for unit_name in new_service_units:
if unit_name not in watched_units:
watched_units.add(unit_name)
yield self._setup_watch_ports(service_state, unit_name)
yield service_state.watch_service_unit_states(cb_check_service_units)
log.debug("Started watch of service units for exposed service %r",
service_state.service_name)
@inlineCallbacks
def _setup_watch_ports(self, service_state, unit_name):
"""Setup the watching of ports for `unit_name`."""
try:
unit_state = yield service_state.get_unit_state(unit_name)
except (ServiceUnitStateNotFound, StateChanged):
log.debug("Cannot setup watch on %r (no longer exists), ignoring",
unit_name)
return
@inlineCallbacks
def cb_watch_ports(value):
"""Permanently watch ports until service is no longer exposed."""
watched_units = self._watched_services.get(
service_state.service_name, NotExposed)
if (not self.is_running() or watched_units is NotExposed
or unit_name not in watched_units):
log.debug("Stopping ports watch for %r", unit_name)
raise StopWatcher()
yield self.open_close_ports(unit_state)
yield unit_state.watch_ports(cb_watch_ports)
log.debug("Started watch of %r on changes to open ports", unit_name)
def add_open_close_ports_observer(self, observer):
"""Set `observer` for calls to `open_close_ports`.
:param observer: The callback is called with the corresponding
:class:`juju.state.service.UnitState`.
"""
self._open_close_ports_observers.add(observer)
@inlineCallbacks
def open_close_ports(self, unit_state):
"""Called upon changes that *may* open/close ports for a service unit.
"""
if not self.is_running():
raise StopWatcher()
try:
try:
machine_id = yield unit_state.get_assigned_machine_id()
except StateChanged:
log.debug("Stopping watch, machine %r no longer in topology",
unit_state.unit_name)
raise StopWatcher()
if machine_id is not None:
yield self.open_close_ports_on_machine(machine_id)
finally:
# Ensure that the observations runs after the
# corresponding action completes. In particular, tests
# that use observation depend on this ordering to ensure
# that the action has in fact happened before they can
# proceed.
observers = list(self._open_close_ports_observers)
for observer in observers:
yield observer(unit_state)
def add_open_close_ports_on_machine_observer(self, observer):
"""Add `observer` for calls to `open_close_ports`.
:param observer: A callback receives the machine id for each call.
"""
self._open_close_ports_on_machine_observers.add(observer)
@inlineCallbacks
def open_close_ports_on_machine(self, machine_id):
"""Called upon changes that *may* open/close ports for a machine.
:param machine_id: The machine ID of the machine that needs to
be checked.
This machine supports multiple service units being assigned to a
machine; all service units are checked each time this is
called to determine the active set of ports to be opened.
"""
if not self.is_running():
raise StopWatcher()
try:
machine_state = yield self.machine_state_manager.get_machine_state(
machine_id)
instance_id = yield machine_state.get_instance_id()
machine = yield self.provider.get_machine(instance_id)
unit_states = yield machine_state.get_all_service_unit_states()
policy_ports = set()
for unit_state in unit_states:
service_state = yield self.service_state_manager.\
get_service_state(unit_state.service_name)
exposed = yield service_state.get_exposed_flag()
if exposed:
ports = yield unit_state.get_open_ports()
for port in ports:
policy_ports.add((port["port"], port["proto"]))
current_ports = yield self.provider.get_opened_ports(
machine, machine_id)
to_open = policy_ports - current_ports
to_close = current_ports - policy_ports
for port, proto in to_open:
yield self.provider.open_port(machine, machine_id, port, proto)
for port, proto in to_close:
yield self.provider.close_port(machine, machine_id, port,
proto)
except MachinesNotFound:
log.info("No provisioned machine for machine %r", machine_id)
except Exception:
log.exception("Got exception in opening/closing ports, will retry")
self._retry_machines_on_port_error.add(machine_id)
finally:
# Ensure that the observation runs after the corresponding
# action completes. In particular, tests that use
# observation depend on this ordering to ensure that this
# action has happened before they can proceed.
observers = list(self._open_close_ports_on_machine_observers)
for observer in observers:
yield observer(machine_id)
class ControlDebugHookTest(
ServiceStateManagerTestBase, ControlToolTest, RepositoryTestBase):
@inlineCallbacks
def setUp(self):
yield super(ControlDebugHookTest, self).setUp()
config = {
"environments": {
"firstenv": {
"type": "dummy", "admin-secret": "homer"}}}
self.write_config(yaml.dump(config))
self.config.load()
self.environment = self.config.get_default()
self.provider = self.environment.get_machine_provider()
# Setup a machine in the provider
self.provider_machine = (yield self.provider.start_machine(
{"machine-id": 0, "dns-name": "antigravity.example.com"}))[0]
# Setup the zk tree with a service, unit, and machine.
self.service = yield self.add_service_from_charm("mysql")
self.unit = yield self.service.add_unit_state()
self.machine_manager = MachineStateManager(self.client)
self.machine = yield self.machine_manager.add_machine_state()
yield self.machine.set_instance_id(0)
yield self.unit.assign_to_machine(self.machine)
# capture the output.
self.output = self.capture_logging(
"juju.control.cli", level=logging.INFO)
self.stderr = self.capture_stream("stderr")
self.setup_exit(0)
@inlineCallbacks
def test_debug_hook_invalid_hook_name(self):
"""If an invalid hookname is used an appropriate error
message is raised that references the charm.
"""
mock_environment = self.mocker.patch(Environment)
mock_environment.get_machine_provider()
self.mocker.result(self.provider)
finished = self.setup_cli_reactor()
self.mocker.replay()
main(["debug-hooks", "mysql/0", "bad-happened"])
yield finished
self.assertIn(
"Charm 'local:series/mysql-1' does not contain hook "
"'bad-happened'",
self.stderr.getvalue())
@inlineCallbacks
def test_debug_hook_invalid_unit_name(self):
"""An invalid unit causes an appropriate error.
"""
finished = self.setup_cli_reactor()
self.mocker.replay()
main(["debug-hooks", "mysql/42"])
yield finished
self.assertIn(
"Service unit 'mysql/42' was not found",
self.stderr.getvalue())
@inlineCallbacks
def test_debug_hook_invalid_service(self):
"""An invalid service causes an appropriate error.
"""
finished = self.setup_cli_reactor()
self.mocker.replay()
main(["debug-hooks", "magic/42"])
yield finished
self.assertIn(
"Service 'magic' was not found",
self.stderr.getvalue())
@inlineCallbacks
def test_debug_hook_unit_agent_not_available(self):
"""Simulate the unit agent isn't available when the command is run.
The command will set the debug flag, and wait for the unit
agent to be available.
"""
mock_unit = self.mocker.patch(ServiceUnitState)
# First time, doesn't exist, will wait on watch
mock_unit.watch_agent()
self.mocker.result((succeed(False), succeed(True)))
# Second time, setup the unit address
mock_unit.watch_agent()
def setup_unit_address():
set_d = self.unit.set_public_address("x1.example.com")
exist_d = Deferred()
set_d.addCallback(lambda result: exist_d.callback(True))
return (exist_d, succeed(True))
self.mocker.call(setup_unit_address)
# Intercept the ssh call
self.patch(os, "system", lambda x: True)
#mock_environment = self.mocker.patch(Environment)
#mock_environment.get_machine_provider()
#self.mocker.result(self.provider)
finished = self.setup_cli_reactor()
self.mocker.replay()
main(["debug-hooks", "mysql/0"])
yield finished
self.assertIn("Waiting for unit", self.output.getvalue())
self.assertIn("Unit running", self.output.getvalue())
self.assertIn("Connecting to remote machine x1.example.com",
self.output.getvalue())
@inlineCallbacks
def test_debug_hook(self):
"""The debug cli will setup unit debug setting and ssh to a screen.
"""
system_mock = self.mocker.replace(os.system)
system_mock(ANY)
def on_ssh(command):
self.assertStartsWith(command, "ssh -t [email protected]")
# In the function, os.system yields to faciliate testing.
self.assertEqual(
(yield self.unit.get_hook_debug()),
{"debug_hooks": ["*"]})
returnValue(True)
self.mocker.call(on_ssh)
finished = self.setup_cli_reactor()
self.mocker.replay()
# Setup the unit address.
yield self.unit.set_public_address("x2.example.com")
main(["debug-hooks", "mysql/0"])
yield finished
self.assertIn(
"Connecting to remote machine", self.output.getvalue())
self.assertIn(
"Debug session ended.", self.output.getvalue())
@inlineCallbacks
def test_standard_named_debug_hook(self):
"""A hook can be debugged by name.
"""
yield self.verify_hook_debug("start")
self.mocker.reset()
self.setup_exit(0)
yield self.verify_hook_debug("stop")
self.mocker.reset()
self.setup_exit(0)
yield self.verify_hook_debug("server-relation-changed")
self.mocker.reset()
self.setup_exit(0)
yield self.verify_hook_debug("server-relation-changed",
"server-relation-broken")
self.mocker.reset()
self.setup_exit(0)
yield self.verify_hook_debug("server-relation-joined",
"server-relation-departed")
@inlineCallbacks
def verify_hook_debug(self, *hook_names):
"""Utility function to verify hook debugging by name
"""
mock_environment = self.mocker.patch(Environment)
mock_environment.get_machine_provider()
self.mocker.result(self.provider)
system_mock = self.mocker.replace(os.system)
system_mock(ANY)
def on_ssh(command):
self.assertStartsWith(command, "ssh -t [email protected]")
self.assertEqual(
(yield self.unit.get_hook_debug()),
{"debug_hooks": list(hook_names)})
returnValue(True)
self.mocker.call(on_ssh)
finished = self.setup_cli_reactor()
self.mocker.replay()
yield self.unit.set_public_address("x11.example.com")
args = ["debug-hooks", "mysql/0"]
args.extend(hook_names)
main(args)
yield finished
self.assertIn(
"Connecting to remote machine", self.output.getvalue())
self.assertIn(
"Debug session ended.", self.output.getvalue())
class MachineStateManagerTest(StateTestBase):
@inlineCallbacks
def setUp(self):
yield super(MachineStateManagerTest, self).setUp()
self.charm_state_manager = CharmStateManager(self.client)
self.machine_state_manager = MachineStateManager(self.client)
self.service_state_manager = ServiceStateManager(self.client)
self.charm_state = yield self.charm_state_manager.add_charm_state(
local_charm_id(self.charm), self.charm, "")
@inlineCallbacks
def add_service(self, service_name):
service_state = yield self.service_state_manager.add_service_state(
service_name, self.charm_state)
returnValue(service_state)
@inlineCallbacks
def test_add_machine(self):
"""
Adding a machine state should register it in zookeeper.
"""
machine_state1 = yield self.machine_state_manager.add_machine_state()
machine_state2 = yield self.machine_state_manager.add_machine_state()
self.assertEquals(machine_state1.id, 0)
self.assertEquals(machine_state1.internal_id, "machine-0000000000")
self.assertEquals(machine_state2.id, 1)
self.assertEquals(machine_state2.internal_id, "machine-0000000001")
children = yield self.client.get_children("/machines")
self.assertEquals(sorted(children),
["machine-0000000000", "machine-0000000001"])
topology = yield self.get_topology()
self.assertTrue(topology.has_machine("machine-0000000000"))
self.assertTrue(topology.has_machine("machine-0000000001"))
@inlineCallbacks
def test_machine_str_representation(self):
"""The str(machine) value includes the machine id.
"""
machine_state1 = yield self.machine_state_manager.add_machine_state()
self.assertEqual(
str(machine_state1), "<MachineState id:machine-%010d>" % (0))
@inlineCallbacks
def test_remove_machine(self):
"""
Adding a machine state should register it in zookeeper.
"""
machine_state1 = yield self.machine_state_manager.add_machine_state()
yield self.machine_state_manager.add_machine_state()
removed = yield self.machine_state_manager.remove_machine_state(
machine_state1.id)
self.assertTrue(removed)
children = yield self.client.get_children("/machines")
self.assertEquals(sorted(children),
["machine-0000000001"])
topology = yield self.get_topology()
self.assertFalse(topology.has_machine("machine-0000000000"))
self.assertTrue(topology.has_machine("machine-0000000001"))
# Removing a non-existing machine again won't fail, since the end
# intention is preserved. This makes dealing with concurrency easier.
# However, False will be returned in this case.
removed = yield self.machine_state_manager.remove_machine_state(
machine_state1.id)
self.assertFalse(removed)
@inlineCallbacks
def test_remove_machine_with_agent(self):
"""Removing a machine with a connected machine agent should succeed.
The removal signals intent to remove a working machine (with an agent)
with the provisioning agent to remove it subsequently.
"""
# Add two machines.
machine_state1 = yield self.machine_state_manager.add_machine_state()
yield self.machine_state_manager.add_machine_state()
# Connect an agent
yield machine_state1.connect_agent()
# Remove a machine
removed = yield self.machine_state_manager.remove_machine_state(
machine_state1.id)
self.assertTrue(removed)
# Verify the second one is still present
children = yield self.client.get_children("/machines")
self.assertEquals(sorted(children),
["machine-0000000001"])
# Verify the topology state.
topology = yield self.get_topology()
self.assertFalse(topology.has_machine("machine-0000000000"))
self.assertTrue(topology.has_machine("machine-0000000001"))
@inlineCallbacks
def test_get_machine_and_check_attributes(self):
"""
Getting a machine state should be possible using both the
user-oriented id and the internal id.
"""
yield self.machine_state_manager.add_machine_state()
yield self.machine_state_manager.add_machine_state()
machine_state = yield self.machine_state_manager.get_machine_state(0)
self.assertEquals(machine_state.id, 0)
machine_state = yield self.machine_state_manager.get_machine_state("0")
self.assertEquals(machine_state.id, 0)
yield self.assertFailure(
self.machine_state_manager.get_machine_state("a"),
MachineStateNotFound)
@inlineCallbacks
def test_get_machine_not_found(self):
"""
Getting a machine state which is not available should errback
a meaningful error.
"""
# No state whatsoever.
try:
yield self.machine_state_manager.get_machine_state(0)
except MachineStateNotFound, e:
self.assertEquals(e.machine_id, 0)
else:
