def __init__(self, client, is_running, provider): """Initialize a Firewall Manager. :param client: A connected zookeeper client. :param is_running: A function (usually a bound method) that returns whether the associated agent is still running or not. :param provider: A machine provider, used for making the actual changes in the environment to firewall settings. """ self.machine_state_manager = MachineStateManager(client) self.service_state_manager = ServiceStateManager(client) self.is_running = is_running self.provider = provider # Track all currently watched machines, using machine ID. self._watched_machines = set() # Map service name to either NotExposed or set of exposed unit names. # If a service name is present in the dictionary, it means its # respective expose node is being watched. self._watched_services = {} # Machines to retry open_close_ports because of earlier errors self._retry_machines_on_port_error = set() # Registration of observers for corresponding actions self._open_close_ports_observers = set() self._open_close_ports_on_machine_observers = set()
def test_open_close_ports_on_machine(self): """Verify opening/closing ports on a machine works properly. In particular this is done without watch support.""" manager = MachineStateManager(self.client) machine = yield manager.add_machine_state() yield self.firewall_manager.process_machine(machine) # Expose a service wordpress = yield self.add_service("wordpress") yield wordpress.set_exposed_flag() wordpress_0 = yield wordpress.add_unit_state() yield wordpress_0.open_port(80, "tcp") yield wordpress_0.open_port(443, "tcp") yield wordpress_0.assign_to_machine(machine) yield self.firewall_manager.open_close_ports_on_machine(machine.id) self.assertEqual((yield self.get_provider_ports(machine)), set([(80, "tcp"), (443, "tcp")])) self.assertIn("Opened 80/tcp on provider machine 0", self.output.getvalue()) self.assertIn("Opened 443/tcp on provider machine 0", self.output.getvalue()) # Now change port setup yield wordpress_0.open_port(8080, "tcp") yield wordpress_0.close_port(443, "tcp") yield self.firewall_manager.open_close_ports_on_machine(machine.id) self.assertEqual((yield self.get_provider_ports(machine)), set([(80, "tcp"), (8080, "tcp")])) self.assertIn("Opened 8080/tcp on provider machine 0", self.output.getvalue()) self.assertIn("Closed 443/tcp on provider machine 0", self.output.getvalue())
def test_watch_machine_changes_ignores_running_machine(self): """ If there is an existing machine instance and state, when a new machine state is added, the existing instance is preserved, and a new instance is created. """ manager = MachineStateManager(self.client) machine_state0 = yield manager.add_machine_state() machines = yield self.agent.provider.start_machine( {"machine-id": machine_state0.id}) machine = machines.pop() yield machine_state0.set_instance_id(machine.instance_id) machine_state1 = yield manager.add_machine_state() machines = yield self.agent.provider.get_machines() self.assertEquals(len(machines), 1) yield self.agent.watch_machine_changes( None, [machine_state0.id, machine_state1.id]) machines = yield self.agent.provider.get_machines() self.assertEquals(len(machines), 2) instance_id = yield machine_state1.get_instance_id() self.assertEqual(instance_id, 1)
def terminate_machine(config, environment, verbose, log, machine_ids): """Terminates the machines in `machine_ids`. Like the underlying code in MachineStateManager, it's permissible if the machine ID is already terminated or even never running. If we determine this is not desired behavior, presumably propagate that back to the state manager. XXX However, we currently special case support of not terminating the "root" machine, that is the one running the provisioning agent. At some point, this will be managed like any other service, but until then it seems best to ensure it's not terminated at this level. """ provider = environment.get_machine_provider() client = yield provider.connect() terminated_machine_ids = [] try: machine_state_manager = MachineStateManager(client) for machine_id in machine_ids: if machine_id == 0: raise CannotTerminateMachine(0, "environment would be destroyed") removed = yield machine_state_manager.remove_machine_state( machine_id) if not removed: raise MachineStateNotFound(machine_id) terminated_machine_ids.append(machine_id) finally: yield client.close() if terminated_machine_ids: log.info("Machines terminated: %s", ", ".join(str(id) for id in terminated_machine_ids))
def test_process_machines_non_concurrency(self): """ Process machines should only be executed serially by an agent. """ manager = MachineStateManager(self.client) machine_state0 = yield manager.add_machine_state() machine_state1 = yield manager.add_machine_state() call_1 = self.agent.process_machines([machine_state0.id]) # The second call should return immediately due to the # instance attribute guard. call_2 = self.agent.process_machines([machine_state1.id]) self.assertEqual(call_2.called, True) self.assertEqual(call_2.result, False) # The first call should have started a provider machine yield call_1 machines = yield self.agent.provider.get_machines() self.assertEquals(len(machines), 1) instance_id_0 = yield machine_state0.get_instance_id() self.assertEqual(instance_id_0, 0) instance_id_1 = yield machine_state1.get_instance_id() self.assertEqual(instance_id_1, None)
def test_transient_provider_error_on_get_machines(self): manager = MachineStateManager(self.client) machine_state0 = yield manager.add_machine_state() mock_provider = self.mocker.patch(self.agent.provider) mock_provider.get_machines() self.mocker.result(fail(ProviderInteractionError())) mock_provider.get_machines() self.mocker.passthrough() self.mocker.replay() try: yield self.agent.process_machines([machine_state0.id]) except: self.fail("Should not raise") instance_id = yield machine_state0.get_instance_id() self.assertEqual(instance_id, None) yield self.agent.process_machines([machine_state0.id]) instance_id = yield machine_state0.get_instance_id() self.assertEqual(instance_id, 0) self.assertIn("Cannot get machine list", self.output.getvalue())
def test_open_close_ports_on_machine_not_yet_provided(self): """Verify that opening/closing ports will eventually succeed once a machine is provided. """ manager = MachineStateManager(self.client) machine = yield manager.add_machine_state() wordpress = yield self.add_service("wordpress") yield wordpress.set_exposed_flag() wordpress_0 = yield wordpress.add_unit_state() yield wordpress_0.open_port(80, "tcp") yield wordpress_0.open_port(443, "tcp") yield wordpress_0.assign_to_machine(machine) # First attempt to open ports quietly fails (except for # logging) because the machine has not yet been provisioned yield self.firewall_manager.open_close_ports_on_machine(machine.id) self.assertIn("No provisioned machine for machine 0", self.output.getvalue()) yield self.provide_machine(machine) # Machine is now provisioned (normally visible in the # provisioning agent through periodic rescan and corresponding # watches) yield self.firewall_manager.open_close_ports_on_machine(machine.id) self.assertEqual((yield self.get_provider_ports(machine)), set([(80, "tcp"), (443, "tcp")]))
def setUp(self): yield super(ControlDebugHookTest, self).setUp() config = { "environments": { "firstenv": { "type": "dummy", "admin-secret": "homer"}}} self.write_config(yaml.dump(config)) self.config.load() self.environment = self.config.get_default() self.provider = self.environment.get_machine_provider() # Setup a machine in the provider self.provider_machine = (yield self.provider.start_machine( {"machine-id": 0, "dns-name": "antigravity.example.com"}))[0] # Setup the zk tree with a service, unit, and machine. self.service = yield self.add_service_from_charm("mysql") self.unit = yield self.service.add_unit_state() self.machine_manager = MachineStateManager(self.client) self.machine = yield self.machine_manager.add_machine_state() yield self.machine.set_instance_id(0) yield self.unit.assign_to_machine(self.machine) # capture the output. self.output = self.capture_logging( "juju.control.cli", level=logging.INFO) self.stderr = self.capture_stream("stderr") self.setup_exit(0)
def test_transient_provider_error_on_start_machine(self): """ If there's an error when processing changes, the agent should log the error and continue. """ manager = MachineStateManager(self.client) machine_state0 = yield manager.add_machine_state() machine_state1 = yield manager.add_machine_state() mock_provider = self.mocker.patch(self.agent.provider) mock_provider.start_machine({"machine-id": 0}) self.mocker.result(fail(ProviderInteractionError())) mock_provider.start_machine({"machine-id": 1}) self.mocker.passthrough() self.mocker.replay() yield self.agent.watch_machine_changes( [], [machine_state0.id, machine_state1.id]) machine1_instance_id = yield machine_state1.get_instance_id() self.assertEqual(machine1_instance_id, 0) self.assertIn( "Cannot process machine 0", self.output.getvalue())
def test_transient_provider_error_on_get_machines(self): manager = MachineStateManager(self.client) machine_state0 = yield manager.add_machine_state() mock_provider = self.mocker.patch(self.agent.provider) mock_provider.get_machines() self.mocker.result(fail(ProviderInteractionError())) mock_provider.get_machines() self.mocker.passthrough() self.mocker.replay() try: yield self.agent.process_machines([machine_state0.id]) except: self.fail("Should not raise") instance_id = yield machine_state0.get_instance_id() self.assertEqual(instance_id, None) yield self.agent.process_machines( [machine_state0.id]) instance_id = yield machine_state0.get_instance_id() self.assertEqual(instance_id, 0) self.assertIn( "Cannot get machine list", self.output.getvalue())
def setUp(self): yield super(ControlRemoveUnitTest, self).setUp() config = {"environments": {"firstenv": {"type": "dummy"}}} self.write_config(dump(config)) self.config.load() self.environment = self.config.get_default() self.provider = self.environment.get_machine_provider() # Setup some service units. self.service_state1 = yield self.add_service_from_charm("mysql") self.service_unit1 = yield self.service_state1.add_unit_state() self.service_unit2 = yield self.service_state1.add_unit_state() self.service_unit3 = yield self.service_state1.add_unit_state() # Add an assigned machine to one of them. self.machine_manager = MachineStateManager(self.client) self.machine = yield self.machine_manager.add_machine_state() yield self.machine.set_instance_id(0) yield self.service_unit1.assign_to_machine(self.machine) # Setup a machine in the provider matching the assigned. self.provider_machine = yield self.provider.start_machine({ "machine-id": 0, "dns-name": "antigravity.example.com" }) self.output = self.capture_logging(level=logging.DEBUG) self.stderr = self.capture_stream("stderr")
def start(self): """Start the machine agent. Creates state directories on the machine, retrieves the machine state, and enables watch on assigned units. """ if not os.path.exists(self.units_directory): os.makedirs(self.units_directory) if not os.path.exists(self.unit_state_directory): os.makedirs(self.unit_state_directory) # Get state managers we'll be utilizing. self.service_state_manager = ServiceStateManager(self.client) self.unit_deployer = UnitDeployer( self.client, self.get_machine_id(), self.config["juju_directory"]) yield self.unit_deployer.start() # Retrieve the machine state for the machine we represent. machine_manager = MachineStateManager(self.client) self.machine_state = yield machine_manager.get_machine_state( self.get_machine_id()) # Watch assigned units for the machine. if self.get_watch_enabled(): self.machine_state.watch_assigned_units( self.watch_service_units) # Connect the machine agent, broadcasting presence to the world. yield self.machine_state.connect_agent() log.info("Machine agent started id:%s" % self.get_machine_id())
def terminate_machine(config, environment, verbose, log, machine_ids): """Terminates the machines in `machine_ids`. Like the underlying code in MachineStateManager, it's permissible if the machine ID is already terminated or even never running. If we determine this is not desired behavior, presumably propagate that back to the state manager. XXX However, we currently special case support of not terminating the "root" machine, that is the one running the provisioning agent. At some point, this will be managed like any other service, but until then it seems best to ensure it's not terminated at this level. """ provider = environment.get_machine_provider() client = yield provider.connect() terminated_machine_ids = [] try: machine_state_manager = MachineStateManager(client) for machine_id in machine_ids: if machine_id == 0: raise CannotTerminateMachine( 0, "environment would be destroyed") removed = yield machine_state_manager.remove_machine_state( machine_id) if not removed: raise MachineStateNotFound(machine_id) terminated_machine_ids.append(machine_id) finally: yield client.close() if terminated_machine_ids: log.info( "Machines terminated: %s", ", ".join(str(id) for id in terminated_machine_ids))
def setUp(self): yield super(MachineStateManagerTest, self).setUp() self.charm_state_manager = CharmStateManager(self.client) self.machine_state_manager = MachineStateManager(self.client) self.service_state_manager = ServiceStateManager(self.client) self.charm_state = yield self.charm_state_manager.add_charm_state( local_charm_id(self.charm), self.charm, "")
def test_open_close_ports_on_unassigned_machine(self): """Verify corner case that nothing happens on an unassigned machine.""" manager = MachineStateManager(self.client) machine = yield manager.add_machine_state() yield self.provide_machine(machine) yield self.firewall_manager.process_machine(machine) yield self.firewall_manager.open_close_ports_on_machine(machine.id) self.assertEqual((yield self.get_provider_ports(machine)), set())
def get_agent_config(self): # gets invoked by AgentTestBase.setUp options = yield super(MachineAgentTest, self).get_agent_config() machine_state_manager = MachineStateManager(self.client) self.machine_state = yield machine_state_manager.add_machine_state() self.change_environment(JUJU_MACHINE_ID="0", JUJU_HOME=self.juju_directory) options["machine_id"] = str(self.machine_state.id) # Start the agent with watching enabled returnValue(options)
def get_ip_address_for_machine(client, provider, machine_id): """Returns public DNS name and machine state for the machine id. :param client: a connected zookeeper client. :param provider: the `MachineProvider` in charge of the juju. :param machine_id: machine ID of the desired machine to connect to. :return: tuple of the DNS name and a `MachineState`. """ manager = MachineStateManager(client) machine_state = yield manager.get_machine_state(machine_id) instance_id = yield machine_state.get_instance_id() provider_machine = yield provider.get_machine(instance_id) returnValue((provider_machine.dns_name, machine_state))
def test_transient_unhandled_error_in_process_machines(self): """Verify that watch_machine_changes handles the exception. Provider implementations may use libraries like txaws that do not handle every error. However, this should not stop the watch from re-establishing itself, as will be the case if the exception is not caught. """ manager = MachineStateManager(self.client) machine_state0 = yield manager.add_machine_state() machine_state1 = yield manager.add_machine_state() # Simulate a failure scenario seen occasionally when working # with OpenStack and txaws mock_agent = self.mocker.patch(self.agent) # Simulate transient error mock_agent.process_machines([machine_state0.id]) self.mocker.result(fail( TypeError("'NoneType' object is not iterable"))) # Let it succeed on second try. In this case, the scenario is # that the watch triggered before the periodic_machine_check # was run again mock_agent.process_machines([machine_state0.id, machine_state1.id]) self.mocker.passthrough() self.mocker.replay() # Verify that watch_machine_changes does not fail even in the case of # the transient error, although no work was done try: yield self.agent.watch_machine_changes([], [machine_state0.id]) except: self.fail("Should not raise") instance_id = yield machine_state0.get_instance_id() self.assertEqual(instance_id, None) # Second attempt, verifiy it did in fact process the machine yield self.agent.watch_machine_changes( [machine_state0.id], [machine_state0.id, machine_state1.id]) self.assertEqual((yield machine_state0.get_instance_id()), 0) self.assertEqual((yield machine_state1.get_instance_id()), 1) # But only after attempting and failing the first time self.assertIn( "Got unexpected exception in processing machines, will retry", self.output.getvalue()) self.assertIn( "'NoneType' object is not iterable", self.output.getvalue())
def get_agent_config(self): # gets invoked by AgentTestBase.setUp options = yield super(MachineAgentTest, self).get_agent_config() machine_state_manager = MachineStateManager(self.client) self.machine_state = yield machine_state_manager.add_machine_state() self.change_environment( JUJU_MACHINE_ID="0", JUJU_HOME=self.juju_directory) options["machine_id"] = str(self.machine_state.id) # Start the agent with watching enabled returnValue(options)
def test_transient_unhandled_error_in_process_machines(self): """Verify that watch_machine_changes handles the exception. Provider implementations may use libraries like txaws that do not handle every error. However, this should not stop the watch from re-establishing itself, as will be the case if the exception is not caught. """ manager = MachineStateManager(self.client) machine_state0 = yield manager.add_machine_state() machine_state1 = yield manager.add_machine_state() # Simulate a failure scenario seen occasionally when working # with OpenStack and txaws mock_agent = self.mocker.patch(self.agent) # Simulate transient error mock_agent.process_machines([machine_state0.id]) self.mocker.result(fail( TypeError("'NoneType' object is not iterable"))) # Let it succeed on second try. In this case, the scenario is # that the watch triggered before the periodic_machine_check # was run again mock_agent.process_machines([machine_state0.id, machine_state1.id]) self.mocker.passthrough() self.mocker.replay() # Verify that watch_machine_changes does not fail even in the case of # the transient error, although no work was done try: yield self.agent.watch_machine_changes([], [machine_state0.id]) except: self.fail("Should not raise") instance_id = yield machine_state0.get_instance_id() self.assertEqual(instance_id, None) # Second attempt, verifiy it did in fact process the machine yield self.agent.watch_machine_changes( [machine_state0.id], [machine_state0.id, machine_state1.id]) self.assertEqual((yield machine_state0.get_instance_id()), 0) self.assertEqual((yield machine_state1.get_instance_id()), 1) # But only after attempting and failing the first time self.assertIn( "Got unexpected exception in processing machines, will retry", self.output.getvalue()) self.assertIn("'NoneType' object is not iterable", self.output.getvalue())
def test_open_close_ports_on_machine_will_retry(self): """Verify port mgmt for a machine will retry if there's a failure.""" mock_provider = self.mocker.patch(MachineProvider) mock_provider.open_port(MATCH_MACHINE, 0, 80, "tcp") self.mocker.result(fail( TypeError("'NoneType' object is not iterable"))) mock_provider.open_port(MATCH_MACHINE, 0, 80, "tcp") self.mocker.result(fail( ProviderInteractionError("Some sort of EC2 problem"))) mock_provider.open_port(MATCH_MACHINE, 0, 80, "tcp") self.mocker.passthrough() self.mocker.replay() manager = MachineStateManager(self.client) machine = yield manager.add_machine_state() yield self.provide_machine(machine) # Expose a service and attempt to open/close ports. The first # attempt will see the simulated failure. wordpress = yield self.add_service("wordpress") yield wordpress.set_exposed_flag() wordpress_0 = yield wordpress.add_unit_state() yield wordpress_0.assign_to_machine(machine) yield self.firewall_manager.process_machine(machine) yield wordpress_0.open_port(80, "tcp") yield self.firewall_manager.open_close_ports_on_machine(machine.id) self.assertEqual((yield self.get_provider_ports(machine)), set()) self.assertIn( "Got exception in opening/closing ports, will retry", self.output.getvalue()) self.assertIn("TypeError: 'NoneType' object is not iterable", self.output.getvalue()) # Retries will now happen in the periodic recheck. First one # still fails due to simulated error. yield self.firewall_manager.process_machine(machine) self.assertEqual((yield self.get_provider_ports(machine)), set()) self.assertIn("ProviderInteractionError: Some sort of EC2 problem", self.output.getvalue()) # Third time is the charm in the mock setup, the recheck succeeds yield self.firewall_manager.process_machine(machine) self.assertEqual((yield self.get_provider_ports(machine)), set([(80, "tcp")])) self.assertIn("Opened 80/tcp on provider machine 0", self.output.getvalue())
def setUp(self): yield super(ControlRemoveUnitTest, self).setUp() config = { "environments": {"firstenv": {"type": "dummy"}}} self.write_config(dump(config)) self.config.load() self.environment = self.config.get_default() self.provider = self.environment.get_machine_provider() # Setup some service units. self.service_state1 = yield self.add_service_from_charm("mysql") self.service_unit1 = yield self.service_state1.add_unit_state() self.service_unit2 = yield self.service_state1.add_unit_state() self.service_unit3 = yield self.service_state1.add_unit_state() # Add an assigned machine to one of them. self.machine_manager = MachineStateManager(self.client) self.machine = yield self.machine_manager.add_machine_state() yield self.machine.set_instance_id(0) yield self.service_unit1.assign_to_machine(self.machine) # Setup a machine in the provider matching the assigned. self.provider_machine = yield self.provider.start_machine( {"machine-id": 0, "dns-name": "antigravity.example.com"}) self.output = self.capture_logging(level=logging.DEBUG) self.stderr = self.capture_stream("stderr")
def test_open_close_ports_on_machine_will_retry(self): """Verify port mgmt for a machine will retry if there's a failure.""" mock_provider = self.mocker.patch(MachineProvider) mock_provider.open_port(MATCH_MACHINE, 0, 80, "tcp") self.mocker.result(fail( TypeError("'NoneType' object is not iterable"))) mock_provider.open_port(MATCH_MACHINE, 0, 80, "tcp") self.mocker.result( fail(ProviderInteractionError("Some sort of EC2 problem"))) mock_provider.open_port(MATCH_MACHINE, 0, 80, "tcp") self.mocker.passthrough() self.mocker.replay() manager = MachineStateManager(self.client) machine = yield manager.add_machine_state() yield self.provide_machine(machine) # Expose a service and attempt to open/close ports. The first # attempt will see the simulated failure. wordpress = yield self.add_service("wordpress") yield wordpress.set_exposed_flag() wordpress_0 = yield wordpress.add_unit_state() yield wordpress_0.assign_to_machine(machine) yield self.firewall_manager.process_machine(machine) yield wordpress_0.open_port(80, "tcp") yield self.firewall_manager.open_close_ports_on_machine(machine.id) self.assertEqual((yield self.get_provider_ports(machine)), set()) self.assertIn("Got exception in opening/closing ports, will retry", self.output.getvalue()) self.assertIn("TypeError: 'NoneType' object is not iterable", self.output.getvalue()) # Retries will now happen in the periodic recheck. First one # still fails due to simulated error. yield self.firewall_manager.process_machine(machine) self.assertEqual((yield self.get_provider_ports(machine)), set()) self.assertIn("ProviderInteractionError: Some sort of EC2 problem", self.output.getvalue()) # Third time is the charm in the mock setup, the recheck succeeds yield self.firewall_manager.process_machine(machine) self.assertEqual((yield self.get_provider_ports(machine)), set([(80, "tcp")])) self.assertIn("Opened 80/tcp on provider machine 0", self.output.getvalue())
def test_machine_state_reflects_invalid_provider_state(self): """ If a machine state has an invalid instance_id, it should be detected, and a new machine started and the machine state updated with the new instance_id. """ machine_manager = MachineStateManager(self.client) m1 = yield machine_manager.add_machine_state() yield m1.set_instance_id("zebra") m2 = yield machine_manager.add_machine_state() yield self.agent.watch_machine_changes(None, [m1.id, m2.id]) m1_instance_id = yield m1.get_instance_id() self.assertEqual(m1_instance_id, 0) m2_instance_id = yield m2.get_instance_id() self.assertEqual(m2_instance_id, 1)
def test_process_machine_is_called(self): """Verify FirewallManager is called when machines are processed""" from juju.state.firewall import FirewallManager mock_manager = self.mocker.patch(FirewallManager) seen = [] def record_machine(machine): seen.append(machine) return succeed(True) mock_manager.process_machine(MATCH_MACHINE_STATE) self.mocker.call(record_machine) self.mocker.replay() machine_manager = MachineStateManager(self.client) machine_state = yield machine_manager.add_machine_state() yield self.agent.process_machines([machine_state.id]) self.assertEqual(seen, [machine_state])
def test_observe_open_close_ports_on_machine(self): """Verify one or more observers can be established on action.""" manager = MachineStateManager(self.client) machine = yield manager.add_machine_state() # Add one observer, verify it gets called calls = [set()] self.firewall_manager.add_open_close_ports_on_machine_observer( Observer(calls, "a")) yield self.firewall_manager.open_close_ports_on_machine(machine.id) self.assertEqual(calls[0], set([("a", machine.id)])) # Reset records of calls, and then add a second observer. # Verify both get called. calls[0] = set() self.firewall_manager.add_open_close_ports_on_machine_observer( Observer(calls, "b")) yield self.firewall_manager.open_close_ports_on_machine(machine.id) self.assertEqual(calls[0], set([("a", machine.id), ("b", machine.id)]))
def test_start_agent_with_watch(self): mock_reactor = self.mocker.patch(reactor) mock_reactor.callLater(self.agent.machine_check_period, self.agent.periodic_machine_check) self.mocker.replay() self.agent.set_watch_enabled(True) yield self.agent.start() manager = MachineStateManager(self.client) machine_state0 = yield manager.add_machine_state() exists_d, watch_d = self.client.exists_and_watch( "/machines/%s" % machine_state0.internal_id) yield exists_d # Wait for the provisioning agent to wake and modify # the machine id. yield watch_d instance_id = yield machine_state0.get_instance_id() self.assertEqual(instance_id, 0)
def test_process_machine_ignores_stop_watcher(self): """Verify that process machine catches `StopWatcher`. `process_machine` calls `open_close_ports_on_machine`, which as verified in an earlier test, raises a `StopWatcher` exception to shutdown watches that use it in the event of agent shutdown. Verify this dual usage does not cause issues while the agent is being stopped for this usage. """ mock_provider = self.mocker.patch(MachineProvider) mock_provider.open_port(MATCH_MACHINE, 0, 80, "tcp") self.mocker.result(fail( TypeError("'NoneType' object is not iterable"))) self.mocker.replay() manager = MachineStateManager(self.client) machine = yield manager.add_machine_state() yield self.provide_machine(machine) # Expose a service and attempt to open/close ports. The first # attempt will see the simulated failure. wordpress = yield self.add_service("wordpress") yield wordpress.set_exposed_flag() wordpress_0 = yield wordpress.add_unit_state() yield wordpress_0.assign_to_machine(machine) yield self.firewall_manager.process_machine(machine) yield wordpress_0.open_port(80, "tcp") yield self.firewall_manager.open_close_ports_on_machine(machine.id) self.assertEqual((yield self.get_provider_ports(machine)), set()) self.assertIn( "Got exception in opening/closing ports, will retry", self.output.getvalue()) self.assertIn("TypeError: 'NoneType' object is not iterable", self.output.getvalue()) # Stop the provisioning agent self.stop() # But retries can potentially still happening anyway, just # make certain nothing bad happens. yield self.firewall_manager.process_machine(machine)
def test_initialize(self): yield self.layout.initialize() yield self.assert_existence_and_acl("/charms") yield self.assert_existence_and_acl("/services") yield self.assert_existence_and_acl("/units") yield self.assert_existence_and_acl("/machines") yield self.assert_existence_and_acl("/relations") yield self.assert_existence_and_acl("/initialized") # To check that the constraints landed correctly, we need the # environment config to have been sent, or we won't be able to # get a provider to help us construct the appropriate objects. yield self.push_default_config(with_constraints=False) esm = EnvironmentStateManager(self.client) env_constraints = yield esm.get_constraints() self.assertEquals(env_constraints, { "provider-type": "dummy", "ubuntu-series": None, "arch": "arm", "cpu": None, "mem": 512}) machine_state_manager = MachineStateManager(self.client) machine_state = yield machine_state_manager.get_machine_state(0) machine_constraints = yield machine_state.get_constraints() self.assertTrue(machine_constraints.complete) self.assertEquals(machine_constraints, { "provider-type": "dummy", "ubuntu-series": "cranky", "arch": "arm", "cpu": None, "mem": 512}) instance_id = yield machine_state.get_instance_id() self.assertEqual(instance_id, "i-abcdef") settings_manager = GlobalSettingsStateManager(self.client) self.assertEqual((yield settings_manager.get_provider_type()), "dummy") self.assertEqual( self.log.getvalue().strip(), "Initializing zookeeper hierarchy")
def test_start_agent_with_watch(self): mock_reactor = self.mocker.patch(reactor) mock_reactor.callLater( self.agent.machine_check_period, self.agent.periodic_machine_check) self.mocker.replay() self.agent.set_watch_enabled(True) yield self.agent.start() manager = MachineStateManager(self.client) machine_state0 = yield manager.add_machine_state() exists_d, watch_d = self.client.exists_and_watch( "/machines/%s" % machine_state0.internal_id) yield exists_d # Wait for the provisioning agent to wake and modify # the machine id. yield watch_d instance_id = yield machine_state0.get_instance_id() self.assertEqual(instance_id, 0)
def test_initialize(self): yield self.layout.initialize() yield self.assert_existence_and_acl("/charms") yield self.assert_existence_and_acl("/services") yield self.assert_existence_and_acl("/units") yield self.assert_existence_and_acl("/machines") yield self.assert_existence_and_acl("/relations") yield self.assert_existence_and_acl("/initialized") machine_state_manager = MachineStateManager(self.client) machine_state = yield machine_state_manager.get_machine_state(0) self.assertTrue(machine_state) instance_id = yield machine_state.get_instance_id() self.assertEqual(instance_id, "i-abcdef") settings_manager = GlobalSettingsStateManager(self.client) self.assertEqual((yield settings_manager.get_provider_type()), "dummy") self.assertEqual(self.log.getvalue().strip(), "Initializing zookeeper hierarchy")
def test_process_machine_ignores_stop_watcher(self): """Verify that process machine catches `StopWatcher`. `process_machine` calls `open_close_ports_on_machine`, which as verified in an earlier test, raises a `StopWatcher` exception to shutdown watches that use it in the event of agent shutdown. Verify this dual usage does not cause issues while the agent is being stopped for this usage. """ mock_provider = self.mocker.patch(MachineProvider) mock_provider.open_port(MATCH_MACHINE, 0, 80, "tcp") self.mocker.result(fail( TypeError("'NoneType' object is not iterable"))) self.mocker.replay() manager = MachineStateManager(self.client) machine = yield manager.add_machine_state() yield self.provide_machine(machine) # Expose a service and attempt to open/close ports. The first # attempt will see the simulated failure. wordpress = yield self.add_service("wordpress") yield wordpress.set_exposed_flag() wordpress_0 = yield wordpress.add_unit_state() yield wordpress_0.assign_to_machine(machine) yield self.firewall_manager.process_machine(machine) yield wordpress_0.open_port(80, "tcp") yield self.firewall_manager.open_close_ports_on_machine(machine.id) self.assertEqual((yield self.get_provider_ports(machine)), set()) self.assertIn("Got exception in opening/closing ports, will retry", self.output.getvalue()) self.assertIn("TypeError: 'NoneType' object is not iterable", self.output.getvalue()) # Stop the provisioning agent self.stop() # But retries can potentially still happening anyway, just # make certain nothing bad happens. yield self.firewall_manager.process_machine(machine)
def test_observe_open_close_ports_on_machine(self): """Verify one or more observers can be established on action.""" manager = MachineStateManager(self.client) machine = yield manager.add_machine_state() # Add one observer, verify it gets called calls = [set()] self.firewall_manager.add_open_close_ports_on_machine_observer( Observer(calls, "a")) yield self.firewall_manager.open_close_ports_on_machine(machine.id) self.assertEqual(calls[0], set([("a", machine.id)])) # Reset records of calls, and then add a second observer. # Verify both get called. calls[0] = set() self.firewall_manager.add_open_close_ports_on_machine_observer( Observer(calls, "b")) yield self.firewall_manager.open_close_ports_on_machine(machine.id) self.assertEqual( calls[0], set([("a", machine.id), ("b", machine.id)]))
def test_initialize(self): yield self.layout.initialize() yield self.assert_existence_and_acl("/charms") yield self.assert_existence_and_acl("/services") yield self.assert_existence_and_acl("/units") yield self.assert_existence_and_acl("/machines") yield self.assert_existence_and_acl("/relations") yield self.assert_existence_and_acl("/initialized") machine_state_manager = MachineStateManager(self.client) machine_state = yield machine_state_manager.get_machine_state(0) self.assertTrue(machine_state) instance_id = yield machine_state.get_instance_id() self.assertEqual(instance_id, "i-abcdef") settings_manager = GlobalSettingsStateManager(self.client) self.assertEqual((yield settings_manager.get_provider_type()), "dummy") self.assertEqual( self.log.getvalue().strip(), "Initializing zookeeper hierarchy")
def test_watch_machine_changes_processes_new_machine_id(self): """The agent should process a new machine id by creating it""" manager = MachineStateManager(self.client) machine_state0 = yield manager.add_machine_state() machine_state1 = yield manager.add_machine_state() yield self.agent.watch_machine_changes( None, [machine_state0.id, machine_state1.id]) self.assertIn( "Machines changed old:None new:[0, 1]", self.output.getvalue()) self.assertIn("Starting machine id:0", self.output.getvalue()) machines = yield self.agent.provider.get_machines() self.assertEquals(len(machines), 2) instance_id = yield machine_state0.get_instance_id() self.assertEqual(instance_id, 0) instance_id = yield machine_state1.get_instance_id() self.assertEqual(instance_id, 1)
def test_watch_machine_changes_processes_new_machine_id(self): """The agent should process a new machine id by creating it""" manager = MachineStateManager(self.client) machine_state0 = yield manager.add_machine_state() machine_state1 = yield manager.add_machine_state() yield self.agent.watch_machine_changes( None, [machine_state0.id, machine_state1.id]) self.assertIn("Machines changed old:None new:[0, 1]", self.output.getvalue()) self.assertIn("Starting machine id:0", self.output.getvalue()) machines = yield self.agent.provider.get_machines() self.assertEquals(len(machines), 2) instance_id = yield machine_state0.get_instance_id() self.assertEqual(instance_id, 0) instance_id = yield machine_state1.get_instance_id() self.assertEqual(instance_id, 1)
class ProvisioningTestBase(AgentTestBase): agent_class = ProvisioningAgent @inlineCallbacks def setUp(self): yield super(ProvisioningTestBase, self).setUp() self.machine_manager = MachineStateManager(self.client) def add_machine_state(self, constraints=None): return self.machine_manager.add_machine_state( constraints or series_constraints)
def start(self): """Start the machine agent. Creates state directories on the machine, retrieves the machine state, and enables watch on assigned units. """ # Initialize directory paths. if not os.path.exists(self.charms_directory): os.makedirs(self.charms_directory) if not os.path.exists(self.units_directory): os.makedirs(self.units_directory) if not os.path.exists(self.unit_state_directory): os.makedirs(self.unit_state_directory) # Get state managers we'll be utilizing. self.service_state_manager = ServiceStateManager(self.client) self.charm_state_manager = CharmStateManager(self.client) # Retrieve the machine state for the machine we represent. machine_manager = MachineStateManager(self.client) self.machine_state = yield machine_manager.get_machine_state( self.get_machine_id()) # Watch assigned units for the machine. if self.get_watch_enabled(): self.machine_state.watch_assigned_units(self.watch_service_units) # Find out what provided the machine, and how to deploy units. settings = GlobalSettingsStateManager(self.client) self.provider_type = yield settings.get_provider_type() self.deploy_factory = get_deploy_factory(self.provider_type) # Connect the machine agent, broadcasting presence to the world. yield self.machine_state.connect_agent() log.info( "Machine agent started id:%s deploy:%r provider:%r" % (self.get_machine_id(), self.deploy_factory, self.provider_type))
def start(self): """Start the machine agent. Creates state directories on the machine, retrieves the machine state, and enables watch on assigned units. """ # Initialize directory paths. if not os.path.exists(self.charms_directory): os.makedirs(self.charms_directory) if not os.path.exists(self.units_directory): os.makedirs(self.units_directory) if not os.path.exists(self.unit_state_directory): os.makedirs(self.unit_state_directory) # Get state managers we'll be utilizing. self.service_state_manager = ServiceStateManager(self.client) self.charm_state_manager = CharmStateManager(self.client) # Retrieve the machine state for the machine we represent. machine_manager = MachineStateManager(self.client) self.machine_state = yield machine_manager.get_machine_state( self.get_machine_id()) # Watch assigned units for the machine. if self.get_watch_enabled(): self.machine_state.watch_assigned_units( self.watch_service_units) # Find out what provided the machine, and how to deploy units. settings = GlobalSettingsStateManager(self.client) self.provider_type = yield settings.get_provider_type() self.deploy_factory = get_deploy_factory(self.provider_type) # Connect the machine agent, broadcasting presence to the world. yield self.machine_state.connect_agent() log.info("Machine agent started id:%s deploy:%r provider:%r" % ( self.get_machine_id(), self.deploy_factory, self.provider_type))
def test_open_close_ports_on_machine_unexposed_service(self): """Verify opening/closing ports on a machine works properly. In particular this is done without watch support.""" manager = MachineStateManager(self.client) machine = yield manager.add_machine_state() yield self.provide_machine(machine) wordpress = yield self.add_service("wordpress") wordpress_0 = yield wordpress.add_unit_state() # Port activity, but service is not exposed yield wordpress_0.open_port(80, "tcp") yield wordpress_0.open_port(443, "tcp") yield wordpress_0.assign_to_machine(machine) yield self.firewall_manager.open_close_ports_on_machine(machine.id) self.assertEqual((yield self.get_provider_ports(machine)), set()) # Now expose it yield wordpress.set_exposed_flag() yield self.firewall_manager.open_close_ports_on_machine(machine.id) self.assertEqual((yield self.get_provider_ports(machine)), set([(80, "tcp"), (443, "tcp")]))
def test_transient_provider_error_on_start_machine(self): """ If there's an error when processing changes, the agent should log the error and continue. """ manager = MachineStateManager(self.client) machine_state0 = yield manager.add_machine_state() machine_state1 = yield manager.add_machine_state() mock_provider = self.mocker.patch(self.agent.provider) mock_provider.start_machine({"machine-id": 0}) self.mocker.result(fail(ProviderInteractionError())) mock_provider.start_machine({"machine-id": 1}) self.mocker.passthrough() self.mocker.replay() yield self.agent.watch_machine_changes( [], [machine_state0.id, machine_state1.id]) machine1_instance_id = yield machine_state1.get_instance_id() self.assertEqual(machine1_instance_id, 0) self.assertIn("Cannot process machine 0", self.output.getvalue())
def constraints_get(env_config, environment, entity_names, log): """ Show the complete set of applicable constraints for each specified entity. This will show the final computed values of all constraints (including internal constraints which cannot be set directly via set-constraints). """ provider = environment.get_machine_provider() client = yield provider.connect() result = {} try: yield sync_environment_state(client, env_config, environment.name) if entity_names: msm = MachineStateManager(client) ssm = ServiceStateManager(client) for name in entity_names: if name.isdigit(): kind = "machine" entity = yield msm.get_machine_state(name) elif "/" in name: kind = "service unit" entity = yield ssm.get_unit_state(name) else: kind = "service" entity = yield ssm.get_service_state(name) log.info("Fetching constraints for %s %s", kind, name) constraints = yield entity.get_constraints() result[name] = dict(constraints) else: esm = EnvironmentStateManager(client) log.info("Fetching constraints for environment") constraints = yield esm.get_constraints() result = dict(constraints) yaml.safe_dump(result, sys.stdout) finally: yield client.close()
def __init__(self, client, provider, log): """ Callable status command object. `client`: ZK client connection `provider`: machine provider for the environment `log`: a Python stdlib logger. """ self.client = client self.provider = provider self.log = log self.service_manager = ServiceStateManager(client) self.relation_manager = RelationStateManager(client) self.machine_manager = MachineStateManager(client) self.charm_manager = CharmStateManager(client) self._reset()
def place_unit(client, policy_name, unit_state): """Return machine state of unit_states assignment. :param client: A connected zookeeper client. :param policy_name: The name of the unit placement policy. :param unit_state: The unit to be assigned. :param provider_type: The type of the machine environment provider. """ machine_state_manager = MachineStateManager(client) # default policy handling if policy_name is None: placement = _unassigned_placement else: placement = _PLACEMENT_LOOKUP.get(policy_name) if placement is None: # We should never get here, pick policy should always pick valid. raise JujuError("Invalid policy:%r for provider" % policy_name) return placement(client, machine_state_manager, unit_state)
def collect(scope, machine_provider, client, log): """Extract status information into nested dicts for rendering. `scope`: an optional list of name specifiers. Globbing based wildcards supported. Defaults to all units, services and relations. `machine_provider`: machine provider for the environment `client`: ZK client connection `log`: a Python stdlib logger. """ service_manager = ServiceStateManager(client) relation_manager = RelationStateManager(client) machine_manager = MachineStateManager(client) charm_manager = CharmStateManager(client) service_data = {} machine_data = {} state = dict(services=service_data, machines=machine_data) seen_machines = set() filter_services, filter_units = digest_scope(scope) services = yield service_manager.get_all_service_states() for service in services: if len(filter_services): found = False for filter_service in filter_services: if fnmatch(service.service_name, filter_service): found = True break if not found: continue unit_data = {} relation_data = {} charm_id = yield service.get_charm_id() charm = yield charm_manager.get_charm_state(charm_id) service_data[service.service_name] = dict(units=unit_data, charm=charm.id, relations=relation_data) exposed = yield service.get_exposed_flag() if exposed: service_data[service.service_name].update(exposed=exposed) units = yield service.get_all_unit_states() unit_matched = False relations = yield relation_manager.get_relations_for_service(service) for unit in units: if len(filter_units): found = False for filter_unit in filter_units: if fnmatch(unit.unit_name, filter_unit): found = True break if not found: continue u = unit_data[unit.unit_name] = dict() machine_id = yield unit.get_assigned_machine_id() u["machine"] = machine_id unit_workflow_client = WorkflowStateClient(client, unit) unit_state = yield unit_workflow_client.get_state() if not unit_state: u["state"] = "pending" else: unit_connected = yield unit.has_agent() u["state"] = unit_state if unit_connected else "down" if exposed: open_ports = yield unit.get_open_ports() u["open-ports"] = [ "{port}/{proto}".format(**port_info) for port_info in open_ports ] u["public-address"] = yield unit.get_public_address() # indicate we should include information about this # machine later seen_machines.add(machine_id) unit_matched = True # collect info on each relation for the service unit relation_status = {} for relation in relations: try: relation_unit = yield relation.get_unit_state(unit) except UnitRelationStateNotFound: # This exception will occur when relations are # established between services without service # units, and therefore never have any # corresponding service relation units. This # scenario does not occur in actual deployments, # but can happen in test circumstances. In # particular, it will happen with a misconfigured # provider, which exercises this codepath. continue # should not occur, but status should not fail relation_workflow_client = WorkflowStateClient( client, relation_unit) relation_workflow_state = \ yield relation_workflow_client.get_state() relation_status[relation.relation_name] = dict( state=relation_workflow_state) u["relations"] = relation_status # after filtering units check if any matched or remove the # service from the output if filter_units and not unit_matched: del service_data[service.service_name] continue for relation in relations: rel_services = yield relation.get_service_states() # A single related service implies a peer relation. More # imply a bi-directional provides/requires relationship. # In the later case we omit the local side of the relation # when reporting. if len(rel_services) > 1: # Filter out self from multi-service relations. rel_services = [ rsn for rsn in rel_services if rsn.service_name != service.service_name ] if len(rel_services) > 1: raise ValueError("Unexpected relationship with more " "than 2 endpoints") rel_service = rel_services[0] relation_data[relation.relation_name] = rel_service.service_name machines = yield machine_manager.get_all_machine_states() for machine_state in machines: if (filter_services or filter_units) and \ machine_state.id not in seen_machines: continue instance_id = yield machine_state.get_instance_id() m = {"instance-id": instance_id \ if instance_id is not None else "pending"} if instance_id is not None: try: pm = yield machine_provider.get_machine(instance_id) m["dns-name"] = pm.dns_name m["instance-state"] = pm.state if (yield machine_state.has_agent()): # if the agent's connected, we're fine m["state"] = "running" else: units = (yield machine_state.get_all_service_unit_states()) for unit in units: unit_workflow_client = WorkflowStateClient( client, unit) if (yield unit_workflow_client.get_state()): # for unit to have a state, its agent must have # run, which implies the machine agent must have # been running correctly at some point in the past m["state"] = "down" break else: # otherwise we're probably just still waiting m["state"] = "not-started" except ProviderError: # The provider doesn't have machine information log.error("Machine provider information missing: machine %s" % (machine_state.id)) machine_data[machine_state.id] = m returnValue(state)
class FirewallManager(object): """Manages the opening and closing of ports in the firewall. """ def __init__(self, client, is_running, provider): """Initialize a Firewall Manager. :param client: A connected zookeeper client. :param is_running: A function (usually a bound method) that returns whether the associated agent is still running or not. :param provider: A machine provider, used for making the actual changes in the environment to firewall settings. """ self.machine_state_manager = MachineStateManager(client) self.service_state_manager = ServiceStateManager(client) self.is_running = is_running self.provider = provider # Track all currently watched machines, using machine ID. self._watched_machines = set() # Map service name to either NotExposed or set of exposed unit names. # If a service name is present in the dictionary, it means its # respective expose node is being watched. self._watched_services = {} # Machines to retry open_close_ports because of earlier errors self._retry_machines_on_port_error = set() # Registration of observers for corresponding actions self._open_close_ports_observers = set() self._open_close_ports_on_machine_observers = set() @inlineCallbacks def process_machine(self, machine_state): """Ensures watch is setup per machine and performs any necessary retry. :param machine_state: The machine state of the machine to be checked. The watch that is established, via :method:`juju.state.machine.MachineState.watch_assigned_units`, handles the scenario where a service or service unit is removed from the topology. Because the service unit is no longer in the topology, the corresponding watch terminates and is unable to `open_close_ports` in response to the change. However, the specific machine watch will be called in this case, and that suffices to determine that its port policy should be checked. In addition, this method can rely on the fact that the provisioning agent periodically rechecks machines so as to support retries of security group operations that failed for that provider. This method is called by the corresponding :method:`juju.agents.provision.ProvisioningAgent.process_machine` in the provisioning agent. """ if machine_state.id in self._retry_machines_on_port_error: self._retry_machines_on_port_error.remove(machine_state.id) try: yield self.open_close_ports_on_machine(machine_state.id) except StopWatcher: # open_close_ports_on_machine can also be called from # a watch, so simply ignore this since it's just used # to shutdown a watch in the case of agent shutdown pass def cb_watch_assigned_units(old_units, new_units): """Watch assigned units for changes possibly require port mgmt. """ log.debug("Assigned units for machine %r: old=%r, new=%r", machine_state.id, old_units, new_units) return self.open_close_ports_on_machine(machine_state.id) if machine_state.id not in self._watched_machines: self._watched_machines.add(machine_state.id) yield machine_state.watch_assigned_units(cb_watch_assigned_units) @inlineCallbacks def watch_service_changes(self, old_services, new_services): """Manage watching service exposed status. This method is called upon every change to the set of services currently deployed. All services are then watched for changes to their exposed flag setting. :param old_services: the set of services before this change. :param new_services: the current set of services. """ removed_services = old_services - new_services for service_name in removed_services: self._watched_services.pop(service_name, None) for service_name in new_services: yield self._setup_new_service_watch(service_name) @inlineCallbacks def _setup_new_service_watch(self, service_name): """Sets up the watching of the exposed flag for a new service. If `service_name` is not watched (as known by `self._watched_services`), adds the watch and a corresponding entry in self._watched_services. (This dict is necessary because there is currently no way to introspect a service for whether it is watched or not.) """ if service_name in self._watched_services: return # already watched self._watched_services[service_name] = NotExposed try: service_state = yield self.service_state_manager.get_service_state( service_name) except ServiceStateNotFound: log.debug("Cannot setup watch, since service %r no longer exists", service_name) self._watched_services.pop(service_name, None) return @inlineCallbacks def cb_watch_service_exposed_flag(exposed): if not self.is_running(): raise StopWatcher() if exposed: log.debug("Service %r is exposed", service_name) else: log.debug("Service %r is unexposed", service_name) try: unit_states = yield service_state.get_all_unit_states() except StateChanged: log.debug("Stopping watch on %r, no longer in topology", service_name) raise StopWatcher() for unit_state in unit_states: yield self.open_close_ports(unit_state) if not exposed: log.debug("Service %r is unexposed", service_name) self._watched_services[service_name] = NotExposed else: log.debug("Service %r is exposed", service_name) self._watched_services[service_name] = set() yield self._setup_service_unit_watch(service_state) yield service_state.watch_exposed_flag(cb_watch_service_exposed_flag) log.debug("Started watch of %r on changes to being exposed", service_name) @inlineCallbacks def _setup_service_unit_watch(self, service_state): """Setup watches on service units of newly exposed `service_name`.""" @inlineCallbacks def cb_check_service_units(old_service_units, new_service_units): watched_units = self._watched_services.get( service_state.service_name, NotExposed) if not self.is_running() or watched_units is NotExposed: raise StopWatcher() removed_service_units = old_service_units - new_service_units for unit_name in removed_service_units: watched_units.discard(unit_name) if not self.is_running(): raise StopWatcher() try: unit_state = yield service_state.get_unit_state(unit_name) except (ServiceUnitStateNotFound, StateChanged): log.debug("Not setting up watch on %r, not in topology", unit_name) continue yield self.open_close_ports(unit_state) for unit_name in new_service_units: if unit_name not in watched_units: watched_units.add(unit_name) yield self._setup_watch_ports(service_state, unit_name) yield service_state.watch_service_unit_states(cb_check_service_units) log.debug("Started watch of service units for exposed service %r", service_state.service_name) @inlineCallbacks def _setup_watch_ports(self, service_state, unit_name): """Setup the watching of ports for `unit_name`.""" try: unit_state = yield service_state.get_unit_state(unit_name) except (ServiceUnitStateNotFound, StateChanged): log.debug("Cannot setup watch on %r (no longer exists), ignoring", unit_name) return @inlineCallbacks def cb_watch_ports(value): """Permanently watch ports until service is no longer exposed.""" watched_units = self._watched_services.get( service_state.service_name, NotExposed) if (not self.is_running() or watched_units is NotExposed or unit_name not in watched_units): log.debug("Stopping ports watch for %r", unit_name) raise StopWatcher() yield self.open_close_ports(unit_state) yield unit_state.watch_ports(cb_watch_ports) log.debug("Started watch of %r on changes to open ports", unit_name) def add_open_close_ports_observer(self, observer): """Set `observer` for calls to `open_close_ports`. :param observer: The callback is called with the corresponding :class:`juju.state.service.UnitState`. """ self._open_close_ports_observers.add(observer) @inlineCallbacks def open_close_ports(self, unit_state): """Called upon changes that *may* open/close ports for a service unit. """ if not self.is_running(): raise StopWatcher() try: try: machine_id = yield unit_state.get_assigned_machine_id() except StateChanged: log.debug("Stopping watch, machine %r no longer in topology", unit_state.unit_name) raise StopWatcher() if machine_id is not None: yield self.open_close_ports_on_machine(machine_id) finally: # Ensure that the observations runs after the # corresponding action completes. In particular, tests # that use observation depend on this ordering to ensure # that the action has in fact happened before they can # proceed. observers = list(self._open_close_ports_observers) for observer in observers: yield observer(unit_state) def add_open_close_ports_on_machine_observer(self, observer): """Add `observer` for calls to `open_close_ports`. :param observer: A callback receives the machine id for each call. """ self._open_close_ports_on_machine_observers.add(observer) @inlineCallbacks def open_close_ports_on_machine(self, machine_id): """Called upon changes that *may* open/close ports for a machine. :param machine_id: The machine ID of the machine that needs to be checked. This machine supports multiple service units being assigned to a machine; all service units are checked each time this is called to determine the active set of ports to be opened. """ if not self.is_running(): raise StopWatcher() try: machine_state = yield self.machine_state_manager.get_machine_state( machine_id) instance_id = yield machine_state.get_instance_id() machine = yield self.provider.get_machine(instance_id) unit_states = yield machine_state.get_all_service_unit_states() policy_ports = set() for unit_state in unit_states: service_state = yield self.service_state_manager.\ get_service_state(unit_state.service_name) exposed = yield service_state.get_exposed_flag() if exposed: ports = yield unit_state.get_open_ports() for port in ports: policy_ports.add((port["port"], port["proto"])) current_ports = yield self.provider.get_opened_ports( machine, machine_id) to_open = policy_ports - current_ports to_close = current_ports - policy_ports for port, proto in to_open: yield self.provider.open_port(machine, machine_id, port, proto) for port, proto in to_close: yield self.provider.close_port(machine, machine_id, port, proto) except MachinesNotFound: log.info("No provisioned machine for machine %r", machine_id) except Exception: log.exception("Got exception in opening/closing ports, will retry") self._retry_machines_on_port_error.add(machine_id) finally: # Ensure that the observation runs after the corresponding # action completes. In particular, tests that use # observation depend on this ordering to ensure that this # action has happened before they can proceed. observers = list(self._open_close_ports_on_machine_observers) for observer in observers: yield observer(machine_id)
class ControlDebugHookTest( ServiceStateManagerTestBase, ControlToolTest, RepositoryTestBase): @inlineCallbacks def setUp(self): yield super(ControlDebugHookTest, self).setUp() config = { "environments": { "firstenv": { "type": "dummy", "admin-secret": "homer"}}} self.write_config(yaml.dump(config)) self.config.load() self.environment = self.config.get_default() self.provider = self.environment.get_machine_provider() # Setup a machine in the provider self.provider_machine = (yield self.provider.start_machine( {"machine-id": 0, "dns-name": "antigravity.example.com"}))[0] # Setup the zk tree with a service, unit, and machine. self.service = yield self.add_service_from_charm("mysql") self.unit = yield self.service.add_unit_state() self.machine_manager = MachineStateManager(self.client) self.machine = yield self.machine_manager.add_machine_state() yield self.machine.set_instance_id(0) yield self.unit.assign_to_machine(self.machine) # capture the output. self.output = self.capture_logging( "juju.control.cli", level=logging.INFO) self.stderr = self.capture_stream("stderr") self.setup_exit(0) @inlineCallbacks def test_debug_hook_invalid_hook_name(self): """If an invalid hookname is used an appropriate error message is raised that references the charm. """ mock_environment = self.mocker.patch(Environment) mock_environment.get_machine_provider() self.mocker.result(self.provider) finished = self.setup_cli_reactor() self.mocker.replay() main(["debug-hooks", "mysql/0", "bad-happened"]) yield finished self.assertIn( "Charm 'local:series/mysql-1' does not contain hook " "'bad-happened'", self.stderr.getvalue()) @inlineCallbacks def test_debug_hook_invalid_unit_name(self): """An invalid unit causes an appropriate error. """ finished = self.setup_cli_reactor() self.mocker.replay() main(["debug-hooks", "mysql/42"]) yield finished self.assertIn( "Service unit 'mysql/42' was not found", self.stderr.getvalue()) @inlineCallbacks def test_debug_hook_invalid_service(self): """An invalid service causes an appropriate error. """ finished = self.setup_cli_reactor() self.mocker.replay() main(["debug-hooks", "magic/42"]) yield finished self.assertIn( "Service 'magic' was not found", self.stderr.getvalue()) @inlineCallbacks def test_debug_hook_unit_agent_not_available(self): """Simulate the unit agent isn't available when the command is run. The command will set the debug flag, and wait for the unit agent to be available. """ mock_unit = self.mocker.patch(ServiceUnitState) # First time, doesn't exist, will wait on watch mock_unit.watch_agent() self.mocker.result((succeed(False), succeed(True))) # Second time, setup the unit address mock_unit.watch_agent() def setup_unit_address(): set_d = self.unit.set_public_address("x1.example.com") exist_d = Deferred() set_d.addCallback(lambda result: exist_d.callback(True)) return (exist_d, succeed(True)) self.mocker.call(setup_unit_address) # Intercept the ssh call self.patch(os, "system", lambda x: True) #mock_environment = self.mocker.patch(Environment) #mock_environment.get_machine_provider() #self.mocker.result(self.provider) finished = self.setup_cli_reactor() self.mocker.replay() main(["debug-hooks", "mysql/0"]) yield finished self.assertIn("Waiting for unit", self.output.getvalue()) self.assertIn("Unit running", self.output.getvalue()) self.assertIn("Connecting to remote machine x1.example.com", self.output.getvalue()) @inlineCallbacks def test_debug_hook(self): """The debug cli will setup unit debug setting and ssh to a screen. """ system_mock = self.mocker.replace(os.system) system_mock(ANY) def on_ssh(command): self.assertStartsWith(command, "ssh -t [email protected]") # In the function, os.system yields to faciliate testing. self.assertEqual( (yield self.unit.get_hook_debug()), {"debug_hooks": ["*"]}) returnValue(True) self.mocker.call(on_ssh) finished = self.setup_cli_reactor() self.mocker.replay() # Setup the unit address. yield self.unit.set_public_address("x2.example.com") main(["debug-hooks", "mysql/0"]) yield finished self.assertIn( "Connecting to remote machine", self.output.getvalue()) self.assertIn( "Debug session ended.", self.output.getvalue()) @inlineCallbacks def test_standard_named_debug_hook(self): """A hook can be debugged by name. """ yield self.verify_hook_debug("start") self.mocker.reset() self.setup_exit(0) yield self.verify_hook_debug("stop") self.mocker.reset() self.setup_exit(0) yield self.verify_hook_debug("server-relation-changed") self.mocker.reset() self.setup_exit(0) yield self.verify_hook_debug("server-relation-changed", "server-relation-broken") self.mocker.reset() self.setup_exit(0) yield self.verify_hook_debug("server-relation-joined", "server-relation-departed") @inlineCallbacks def verify_hook_debug(self, *hook_names): """Utility function to verify hook debugging by name """ mock_environment = self.mocker.patch(Environment) mock_environment.get_machine_provider() self.mocker.result(self.provider) system_mock = self.mocker.replace(os.system) system_mock(ANY) def on_ssh(command): self.assertStartsWith(command, "ssh -t [email protected]") self.assertEqual( (yield self.unit.get_hook_debug()), {"debug_hooks": list(hook_names)}) returnValue(True) self.mocker.call(on_ssh) finished = self.setup_cli_reactor() self.mocker.replay() yield self.unit.set_public_address("x11.example.com") args = ["debug-hooks", "mysql/0"] args.extend(hook_names) main(args) yield finished self.assertIn( "Connecting to remote machine", self.output.getvalue()) self.assertIn( "Debug session ended.", self.output.getvalue())
class MachineStateManagerTest(StateTestBase): @inlineCallbacks def setUp(self): yield super(MachineStateManagerTest, self).setUp() self.charm_state_manager = CharmStateManager(self.client) self.machine_state_manager = MachineStateManager(self.client) self.service_state_manager = ServiceStateManager(self.client) self.charm_state = yield self.charm_state_manager.add_charm_state( local_charm_id(self.charm), self.charm, "") @inlineCallbacks def add_service(self, service_name): service_state = yield self.service_state_manager.add_service_state( service_name, self.charm_state) returnValue(service_state) @inlineCallbacks def test_add_machine(self): """ Adding a machine state should register it in zookeeper. """ machine_state1 = yield self.machine_state_manager.add_machine_state() machine_state2 = yield self.machine_state_manager.add_machine_state() self.assertEquals(machine_state1.id, 0) self.assertEquals(machine_state1.internal_id, "machine-0000000000") self.assertEquals(machine_state2.id, 1) self.assertEquals(machine_state2.internal_id, "machine-0000000001") children = yield self.client.get_children("/machines") self.assertEquals(sorted(children), ["machine-0000000000", "machine-0000000001"]) topology = yield self.get_topology() self.assertTrue(topology.has_machine("machine-0000000000")) self.assertTrue(topology.has_machine("machine-0000000001")) @inlineCallbacks def test_machine_str_representation(self): """The str(machine) value includes the machine id. """ machine_state1 = yield self.machine_state_manager.add_machine_state() self.assertEqual( str(machine_state1), "<MachineState id:machine-%010d>" % (0)) @inlineCallbacks def test_remove_machine(self): """ Adding a machine state should register it in zookeeper. """ machine_state1 = yield self.machine_state_manager.add_machine_state() yield self.machine_state_manager.add_machine_state() removed = yield self.machine_state_manager.remove_machine_state( machine_state1.id) self.assertTrue(removed) children = yield self.client.get_children("/machines") self.assertEquals(sorted(children), ["machine-0000000001"]) topology = yield self.get_topology() self.assertFalse(topology.has_machine("machine-0000000000")) self.assertTrue(topology.has_machine("machine-0000000001")) # Removing a non-existing machine again won't fail, since the end # intention is preserved. This makes dealing with concurrency easier. # However, False will be returned in this case. removed = yield self.machine_state_manager.remove_machine_state( machine_state1.id) self.assertFalse(removed) @inlineCallbacks def test_remove_machine_with_agent(self): """Removing a machine with a connected machine agent should succeed. The removal signals intent to remove a working machine (with an agent) with the provisioning agent to remove it subsequently. """ # Add two machines. machine_state1 = yield self.machine_state_manager.add_machine_state() yield self.machine_state_manager.add_machine_state() # Connect an agent yield machine_state1.connect_agent() # Remove a machine removed = yield self.machine_state_manager.remove_machine_state( machine_state1.id) self.assertTrue(removed) # Verify the second one is still present children = yield self.client.get_children("/machines") self.assertEquals(sorted(children), ["machine-0000000001"]) # Verify the topology state. topology = yield self.get_topology() self.assertFalse(topology.has_machine("machine-0000000000")) self.assertTrue(topology.has_machine("machine-0000000001")) @inlineCallbacks def test_get_machine_and_check_attributes(self): """ Getting a machine state should be possible using both the user-oriented id and the internal id. """ yield self.machine_state_manager.add_machine_state() yield self.machine_state_manager.add_machine_state() machine_state = yield self.machine_state_manager.get_machine_state(0) self.assertEquals(machine_state.id, 0) machine_state = yield self.machine_state_manager.get_machine_state("0") self.assertEquals(machine_state.id, 0) yield self.assertFailure( self.machine_state_manager.get_machine_state("a"), MachineStateNotFound) @inlineCallbacks def test_get_machine_not_found(self): """ Getting a machine state which is not available should errback a meaningful error. """ # No state whatsoever. try: yield self.machine_state_manager.get_machine_state(0) except MachineStateNotFound, e: self.assertEquals(e.machine_id, 0) else: