def config_set(environment, service_name, service_options): """Set service settings. """ provider = environment.get_machine_provider() client = yield provider.connect() # Get the service and the charm service_manager = ServiceStateManager(client) service = yield service_manager.get_service_state(service_name) charm = yield service.get_charm_state() # Use the charm's ConfigOptions instance to validate the # arguments to config_set. Invalid options passed to this method # will thrown an exception. if isinstance(service_options, dict): options = service_options else: options = parse_keyvalue_pairs(service_options) config = yield charm.get_config() # ignore the output of validate, we run it so it might throw an exception config.validate(options) # Apply the change state = yield service.get_config() state.update(options) yield state.write()
def add_relation(env_config, environment, verbose, log, *descriptors): """Add relation between relation endpoints described by `descriptors`""" provider = environment.get_machine_provider() client = yield provider.connect() relation_state_manager = RelationStateManager(client) service_state_manager = ServiceStateManager(client) endpoint_pairs = yield service_state_manager.join_descriptors(*descriptors) if verbose: log.info("Endpoint pairs: %s", endpoint_pairs) if len(endpoint_pairs) == 0: raise NoMatchingEndpoints() elif len(endpoint_pairs) > 1: raise AmbiguousRelation(descriptors, endpoint_pairs) # At this point we just have one endpoint pair. We need to pick # just one of the endpoints if it's a peer endpoint, since that's # our current API - join descriptors takes two descriptors, but # add_relation_state takes one or two endpoints. TODO consider # refactoring. endpoints = endpoint_pairs[0] if endpoints[0] == endpoints[1]: endpoints = endpoints[0:1] yield relation_state_manager.add_relation_state(*endpoints) yield client.close() log.info("Added %s relation to all service units.", endpoints[0].relation_type)
def setUp(self): yield super(MachineAgentTest, self).setUp() self.output = self.capture_logging("juju.agents.machine", level=logging.DEBUG) config = self.get_test_environment_config() environment = config.get_default() # Store the environment to zookeeper environment_state_manager = EnvironmentStateManager(self.client) yield environment_state_manager.set_config_state(config, "myfirstenv") # Load the environment with the charm state and charm binary self.provider = environment.get_machine_provider() self.storage = self.provider.get_file_storage() self.charm = CharmDirectory(self.sample_dir1) self.publisher = CharmPublisher(self.client, self.storage) yield self.publisher.add_charm(local_charm_id(self.charm), self.charm) charm_states = yield self.publisher.publish() self.charm_state = charm_states[0] # Create a service from the charm from which we can create units for # the machine. self.service_state_manager = ServiceStateManager(self.client) self.service = yield self.service_state_manager.add_service_state( "fatality-blog", self.charm_state)
def get_local_service(self): """Return ServiceState for the local service.""" if self._service is None: service_state_manager = ServiceStateManager(self._client) self._service = yield (service_state_manager.get_service_state( parse_service_name(self._unit_name))) returnValue(self._service)
def add_relation(env_config, environment, verbose, log, *descriptors): """Add relation between relation endpoints described by `descriptors`""" provider = environment.get_machine_provider() client = yield provider.connect() relation_state_manager = RelationStateManager(client) service_state_manager = ServiceStateManager(client) endpoint_pairs = yield service_state_manager.join_descriptors( *descriptors) if verbose: log.info("Endpoint pairs: %s", endpoint_pairs) if len(endpoint_pairs) == 0: raise NoMatchingEndpoints() elif len(endpoint_pairs) > 1: raise AmbiguousRelation(descriptors, endpoint_pairs) # At this point we just have one endpoint pair. We need to pick # just one of the endpoints if it's a peer endpoint, since that's # our current API - join descriptors takes two descriptors, but # add_relation_state takes one or two endpoints. TODO consider # refactoring. endpoints = endpoint_pairs[0] if endpoints[0] == endpoints[1]: endpoints = endpoints[0:1] yield relation_state_manager.add_relation_state(*endpoints) yield client.close() log.info("Added %s relation to all service units.", endpoints[0].relation_type)
def test_deploy(self): """Create service, and service unit on machine from charm""" environment = self.config.get("firstenv") yield deploy.deploy( self.config, environment, self.unbundled_repo_path, "local:sample", "myblog", logging.getLogger("deploy"), ["cpu=123"]) topology = yield self.get_topology() service_id = topology.find_service_with_name("myblog") self.assertEqual(service_id, "service-%010d" % 0) exists = yield self.client.exists("/services/%s" % service_id) self.assertTrue(exists) service_state_manager = ServiceStateManager(self.client) service_state = yield service_state_manager.get_service_state("myblog") charm_id = yield service_state.get_charm_id() self.assertEquals(charm_id, "local:series/sample-2") constraints = yield service_state.get_constraints() expect_constraints = { "arch": "amd64", "cpu": 123, "mem": 512, "provider-type": "dummy", "ubuntu-series": "series"} self.assertEquals(constraints, expect_constraints) machine_ids = topology.get_machines() self.assertEqual( machine_ids, ["machine-%010d" % 0, "machine-%010d" % 1]) exists = yield self.client.exists("/machines/%s" % machine_ids[0]) self.assertTrue(exists) unit_ids = topology.get_service_units(service_id) self.assertEqual(unit_ids, ["unit-%010d" % 0]) exists = yield self.client.exists("/units/%s" % unit_ids[0]) self.assertTrue(exists)
def setUp(self): yield super(MachineStateManagerTest, self).setUp() self.charm_state_manager = CharmStateManager(self.client) self.machine_state_manager = MachineStateManager(self.client) self.service_state_manager = ServiceStateManager(self.client) self.charm_state = yield self.charm_state_manager.add_charm_state( local_charm_id(self.charm), self.charm, "")
def test_deploy_legacy_keys_in_fresh_env(self): yield self.push_default_config() local_config = { "environments": {"firstenv": { "type": "dummy", "some-legacy-key": "blah", "default-series": "series"}}} self.write_config(yaml.dump(local_config)) self.config.load() finished = self.setup_cli_reactor() self.setup_exit(0) self.mocker.replay() stderr = self.capture_stream("stderr") main(["deploy", "--repository", self.unbundled_repo_path, "local:sample", "beekeeper"]) yield finished self.assertIn( "Your environments.yaml contains deprecated keys", stderr.getvalue()) service_manager = ServiceStateManager(self.client) yield self.assertFailure( service_manager.get_service_state("beekeeper"), ServiceStateNotFound)
def destroy_service(config, environment, verbose, log, service_name): provider = environment.get_machine_provider() client = yield provider.connect() service_manager = ServiceStateManager(client) service_state = yield service_manager.get_service_state(service_name) yield service_manager.remove_service_state(service_state) log.info("Service %r destroyed.", service_state.service_name)
def add_unit(config, environment, verbose, log, service_name, num_units): """Add a unit of a service to the environment. """ provider = environment.get_machine_provider() placement_policy = provider.get_placement_policy() client = yield provider.connect() try: yield legacy.check_environment( client, provider.get_legacy_config_keys()) yield sync_environment_state(client, config, environment.name) service_manager = ServiceStateManager(client) service_state = yield service_manager.get_service_state(service_name) if (yield service_state.is_subordinate()): raise JujuError("Subordinate services acquire units from " "their principal service.") for i in range(num_units): unit_state = yield service_state.add_unit_state() yield place_unit(client, placement_policy, unit_state) log.info("Unit %r added to service %r", unit_state.unit_name, service_state.service_name) finally: yield client.close()
def destroy_service(config, environment, verbose, log, service_name): provider = environment.get_machine_provider() client = yield provider.connect() service_manager = ServiceStateManager(client) service_state = yield service_manager.get_service_state(service_name) if (yield service_state.is_subordinate()): # We can destroy the service if does not have relations. # That implies that principals have already been torn # down (or were never added). relation_manager = RelationStateManager(client) relations = yield relation_manager.get_relations_for_service( service_state) if relations: principal_service = None # if we have a container we can destroy the subordinate # (revisit in the future) for relation in relations: if relation.relation_scope != "container": continue services = yield relation.get_service_states() remote_service = [s for s in services if s.service_name != service_state.service_name][0] if not (yield remote_service.is_subordinate()): principal_service = remote_service break if principal_service: raise UnsupportedSubordinateServiceRemoval( service_state.service_name, principal_service.service_name) yield service_manager.remove_service_state(service_state) log.info("Service %r destroyed.", service_state.service_name)
def config_set(environment, service_name, service_options): """Set service options. """ provider = environment.get_machine_provider() client = yield provider.connect() # Get the service and the charm # service_manager = ServiceStateManager(client) service = yield service_manager.get_service_state(service_name) charm = yield service.get_charm_state() # Use the charm's ConfigOptions instance to validate the # arguments to config_set. Invalid options passed to this method # will thrown an exception. options = parse_keyvalue_pairs(service_options) config = yield charm.get_config() # ignore the output of validate, we run it so it might throw an exception config.validate(options) # Apply the change state = yield service.get_config() state.update(options) yield state.write()
def __init__(self, client, is_running, provider): """Initialize a Firewall Manager. :param client: A connected zookeeper client. :param is_running: A function (usually a bound method) that returns whether the associated agent is still running or not. :param provider: A machine provider, used for making the actual changes in the environment to firewall settings. """ self.machine_state_manager = MachineStateManager(client) self.service_state_manager = ServiceStateManager(client) self.is_running = is_running self.provider = provider # Track all currently watched machines, using machine ID. self._watched_machines = set() # Map service name to either NotExposed or set of exposed unit names. # If a service name is present in the dictionary, it means its # respective expose node is being watched. self._watched_services = {} # Machines to retry open_close_ports because of earlier errors self._retry_machines_on_port_error = set() # Registration of observers for corresponding actions self._open_close_ports_observers = set() self._open_close_ports_on_machine_observers = set()
def start(self): """Start the unit agent process.""" service_state_manager = ServiceStateManager(self.client) # Retrieve our unit and configure working directories. service_name = self.unit_name.split("/")[0] self.service_state = yield service_state_manager.get_service_state( service_name) self.unit_state = yield self.service_state.get_unit_state( self.unit_name) self.unit_directory = os.path.join( self.config["juju_directory"], "units", self.unit_state.unit_name.replace("/", "-")) self.state_directory = os.path.join( self.config["juju_directory"], "state") # Setup the server portion of the cli api exposed to hooks. socket_path = os.path.join(self.unit_directory, HOOK_SOCKET_FILE) if os.path.exists(socket_path): os.unlink(socket_path) from twisted.internet import reactor self.api_socket = reactor.listenUNIX(socket_path, self.api_factory) # Setup the unit state's address address = yield get_unit_address(self.client) yield self.unit_state.set_public_address( (yield address.get_public_address())) yield self.unit_state.set_private_address( (yield address.get_private_address())) if self.get_watch_enabled(): yield self.unit_state.watch_hook_debug(self.cb_watch_hook_debug) # Inform the system, we're alive. yield self.unit_state.connect_agent() # Start paying attention to the debug-log setting if self.get_watch_enabled(): yield self.unit_state.watch_hook_debug(self.cb_watch_hook_debug) self.lifecycle = UnitLifecycle( self.client, self.unit_state, self.service_state, self.unit_directory, self.state_directory, self.executor) self.workflow = UnitWorkflowState( self.client, self.unit_state, self.lifecycle, self.state_directory) # Set up correct lifecycle and executor state given the persistent # unit workflow state, and fire any starting transitions if necessary. with (yield self.workflow.lock()): yield self.workflow.synchronize(self.executor) if self.get_watch_enabled(): yield self.unit_state.watch_resolved(self.cb_watch_resolved) yield self.service_state.watch_config_state( self.cb_watch_config_changed) yield self.unit_state.watch_upgrade_flag( self.cb_watch_upgrade_flag)
def get_local_service(self): """Return ServiceState for the local service.""" if self._service is None: service_state_manager = ServiceStateManager(self._client) self._service = yield( service_state_manager.get_service_state( parse_service_name(self._unit_name))) returnValue(self._service)
def constraints_set(env_config, environment, service_name, constraint_strs): """ Machine constraints allow you to pick the hardware to which your services will be deployed. Examples: $ juju set-constraints --service-name mysql mem=8G cpu=4 $ juju set-constraints instance-type=t1.micro Available constraints vary by provider type, and will be ignored if not understood by the current environment's provider. The current set of available constraints across all providers is: On Amazon EC2: * arch (CPU architecture: i386/amd64/arm; amd64 by default) * cpu (processing power in Amazon ECU; 1 by default) * mem (memory in [MGT]iB; 512M by default) * instance-type (unset by default) * ec2-zone (unset by default) On Orchestra: * orchestra-classes (unset by default) On MAAS: * maas-name (unset by default) Service settings, if specified, will override environment settings, which will in turn override the juju defaults of mem=512M, cpu=1, arch=amd64. New constraints set on an entity will completely replace that entity's pre-existing constraints. To override an environment constraint with the juju default when setting service constraints, just specify "name=" (rather than just not specifying the constraint at all, which will cause it to inherit the environment's value). To entirely unset a constraint, specify "name=any". """ provider = environment.get_machine_provider() constraint_set = yield provider.get_constraint_set() constraints = constraint_set.parse(constraint_strs) client = yield provider.connect() try: yield legacy.check_constraints(client, constraint_strs) yield sync_environment_state(client, env_config, environment.name) if service_name is None: esm = EnvironmentStateManager(client) yield esm.set_constraints(constraints) else: ssm = ServiceStateManager(client) service = yield ssm.get_service_state(service_name) yield service.set_constraints(constraints) finally: yield client.close()
def setUp(self): yield super(FirewallTestBase, self).setUp() self._running = True self.environment = yield self.get_test_environment() self.provider = self.environment.get_machine_provider() self.firewall_manager = FirewallManager(self.client, self.is_running, self.provider) self.service_state_manager = ServiceStateManager(self.client) self.output = self.capture_logging(level=logging.DEBUG)
def start(self): """Start the unit agent process.""" self.service_state_manager = ServiceStateManager(self.client) self.executor.start() # Retrieve our unit and configure working directories. service_name = self.unit_name.split("/")[0] service_state = yield self.service_state_manager.get_service_state( service_name) self.unit_state = yield service_state.get_unit_state(self.unit_name) self.unit_directory = os.path.join( self.config["juju_directory"], "units", self.unit_state.unit_name.replace("/", "-")) self.state_directory = os.path.join(self.config["juju_directory"], "state") # Setup the server portion of the cli api exposed to hooks. from twisted.internet import reactor self.api_socket = reactor.listenUNIX( os.path.join(self.unit_directory, HOOK_SOCKET_FILE), self.api_factory) # Setup the unit state's address address = yield get_unit_address(self.client) yield self.unit_state.set_public_address( (yield address.get_public_address())) yield self.unit_state.set_private_address( (yield address.get_private_address())) # Inform the system, we're alive. yield self.unit_state.connect_agent() self.lifecycle = UnitLifecycle(self.client, self.unit_state, service_state, self.unit_directory, self.executor) self.workflow = UnitWorkflowState(self.client, self.unit_state, self.lifecycle, self.state_directory) if self.get_watch_enabled(): yield self.unit_state.watch_resolved(self.cb_watch_resolved) yield self.unit_state.watch_hook_debug(self.cb_watch_hook_debug) yield service_state.watch_config_state( self.cb_watch_config_changed) # Fire initial transitions, only if successful if (yield self.workflow.transition_state("installed")): yield self.workflow.transition_state("started") # Upgrade can only be processed if we're in a running state so # for case of a newly started unit, do it after the unit is started. if self.get_watch_enabled(): yield self.unit_state.watch_upgrade_flag( self.cb_watch_upgrade_flag)
def remove_relation(env_config, environment, verbose, log, *descriptors): """Remove relation between relation endpoints described by `descriptors`""" provider = environment.get_machine_provider() client = yield provider.connect() relation_state_manager = RelationStateManager(client) service_state_manager = ServiceStateManager(client) endpoint_pairs = yield service_state_manager.join_descriptors( *descriptors) if verbose: log.info("Endpoint pairs: %s", endpoint_pairs) if len(endpoint_pairs) == 0: raise NoMatchingEndpoints() elif len(endpoint_pairs) > 1: raise AmbiguousRelation(descriptors, endpoint_pairs) # At this point we just have one endpoint pair. We need to pick # just one of the endpoints if it's a peer endpoint, since that's # our current API - join descriptors takes two descriptors, but # add_relation_state takes one or two endpoints. TODO consider # refactoring. endpoints = endpoint_pairs[0] if endpoints[0] == endpoints[1]: endpoints = endpoints[0:1] relation_state = yield relation_state_manager.get_relation_state( *endpoints) # Look at both endpoints, if we are dealing with a container relation # decide if one end is a principal. service_pair = [] # ordered such that sub, principal is_container = False has_principal = True for ep in endpoints: if ep.relation_scope == "container": is_container = True service = yield service_state_manager.get_service_state( ep.service_name) if (yield service.is_subordinate()): service_pair.append(service) has_principal = True else: service_pair.insert(0, service) if is_container and len(service_pair) == 2 and has_principal: sub, principal = service_pair raise UnsupportedSubordinateServiceRemoval(sub.service_name, principal.service_name) yield relation_state_manager.remove_relation_state(relation_state) yield client.close() log.info("Removed %s relation from all service units.", endpoints[0].relation_type)
def test_deploy_adds_peer_relations(self): """Deploy automatically adds a peer relations.""" environment = self.config.get("firstenv") yield deploy.deploy( self.config, environment, self.unbundled_repo_path, "local:riak", None, logging.getLogger("deploy") ) service_manager = ServiceStateManager(self.client) service_state = yield service_manager.get_service_state("riak") relation_manager = RelationStateManager(self.client) relations = yield relation_manager.get_relations_for_service(service_state) self.assertEqual(len(relations), 1) self.assertEqual(relations[0].relation_name, "ring")
def test_deploy_adds_peer_relations(self): """Deploy automatically adds a peer relations.""" environment = self.config.get("firstenv") yield deploy.deploy(self.config, environment, self.unbundled_repo_path, "local:riak", None, logging.getLogger("deploy")) service_manager = ServiceStateManager(self.client) service_state = yield service_manager.get_service_state("riak") relation_manager = RelationStateManager(self.client) relations = yield relation_manager.get_relations_for_service( service_state) self.assertEqual(len(relations), 1) self.assertEqual(relations[0].relation_name, "ring")
def test_deploy_legacy_keys_in_legacy_env(self): yield self.client.delete("/constraints") finished = self.setup_cli_reactor() self.setup_exit(0) self.mocker.replay() main(["deploy", "--repository", self.unbundled_repo_path, "local:sample", "beekeeper"]) yield finished service_manager = ServiceStateManager(self.client) yield service_manager.get_service_state("beekeeper")
def remove_unit(config, environment, verbose, log, unit_names): provider = environment.get_machine_provider() client = yield provider.connect() try: service_manager = ServiceStateManager(client) for unit_name in unit_names: service_name = parse_service_name(unit_name) service_state = yield service_manager.get_service_state( service_name) unit_state = yield service_state.get_unit_state(unit_name) yield service_state.remove_unit_state(unit_state) log.info("Unit %r removed from service %r", unit_state.unit_name, service_state.service_name) finally: yield client.close()
def expose(config, environment, verbose, log, service_name): """Expose a service.""" provider = environment.get_machine_provider() client = yield provider.connect() try: service_manager = ServiceStateManager(client) service_state = yield service_manager.get_service_state(service_name) already_exposed = yield service_state.get_exposed_flag() if not already_exposed: yield service_state.set_exposed_flag() log.info("Service %r was exposed.", service_name) else: log.info("Service %r was already exposed.", service_name) finally: yield client.close()
def get_service_states(self): """Get all the services associated with this relation. @return: list of ServiceState instance associated with this relation. """ from juju.state.service import ServiceStateManager, ServiceState service_manager = ServiceStateManager(self._client) services = [] topology = yield service_manager._read_topology() for service_id in topology.get_relation_services( self.internal_relation_id): service_name = topology.get_service_name(service_id) service = ServiceState(self._client, service_id, service_name) services.append(service) returnValue(services)
def setUp(self): yield super(MachineAgentTest, self).setUp() self.output = self.capture_logging( "juju.agents.machine", level=logging.DEBUG) config = self.get_test_environment_config() environment = config.get_default() # Store the environment to zookeeper environment_state_manager = EnvironmentStateManager(self.client) yield environment_state_manager.set_config_state(config, "myfirstenv") # Load the environment with the charm state and charm binary self.provider = environment.get_machine_provider() self.storage = self.provider.get_file_storage() self.charm = CharmDirectory(self.sample_dir1) self.publisher = CharmPublisher(self.client, self.storage) yield self.publisher.add_charm(local_charm_id(self.charm), self.charm) charm_states = yield self.publisher.publish() self.charm_state = charm_states[0] # Create a service from the charm from which we can create units for # the machine. self.service_state_manager = ServiceStateManager(self.client) self.service = yield self.service_state_manager.add_service_state( "fatality-blog", self.charm_state)
def setUp(self): yield super(UnitDeployerTest, self).setUp() self.output = self.capture_logging(level=logging.DEBUG) yield self.push_default_config() # Load the environment with the charm state and charm binary environment = self.config.get_default() provider = environment.get_machine_provider() storage = provider.get_file_storage() publisher = CharmPublisher(self.client, storage) yield publisher.add_charm(local_charm_id(self.charm), self.charm) self.charm_state, = yield publisher.publish() # Create a service from the charm, then add a unit and assign # it to a machine. self.service_state_manager = ServiceStateManager(self.client) self.machine_state_manager = MachineStateManager(self.client) self.service = yield self.service_state_manager.add_service_state( "myblog", self.charm_state, dummy_constraints) self.unit_state = yield self.service.add_unit_state() self.machine_state = yield self.machine_state_manager.\ add_machine_state(series_constraints) yield self.unit_state.assign_to_machine(self.machine_state) # NOTE machine_id must be a str to use with one of the # deployment classes self.juju_dir = self.makeDir() self.unit_manager = UnitDeployer( self.client, str(self.machine_state.id), self.juju_dir) yield self.unit_manager.start()
def get_ip_address_for_unit(client, provider, unit_name): """Returns public DNS name and unit state for the service unit. :param client: a connected zookeeper client. :param provider: the `MachineProvider` in charge of the juju. :param unit_name: service unit running on a machine to connect to. :return: tuple of the DNS name and a `MachineState`. :raises: :class:`juju.state.errors.ServiceUnitStateMachineNotAssigned` """ manager = ServiceStateManager(client) service_unit = yield manager.get_unit_state(unit_name) machine_id = yield service_unit.get_assigned_machine_id() if machine_id is None: raise ServiceUnitStateMachineNotAssigned(unit_name) returnValue( ((yield service_unit.get_public_address()), service_unit))
def test_deploy_no_service_name_short_charm_name(self): """Uses charm name as service name if possible.""" environment = self.config.get("firstenv") yield deploy.deploy(self.config, environment, self.unbundled_repo_path, "local:sample", None, logging.getLogger("deploy")) service = yield ServiceStateManager( self.client).get_service_state("sample") self.assertEqual(service.service_name, "sample")
def add_unit(config, environment, verbose, log, service_name, num_units): """Add a unit of a service to the environment. """ provider = environment.get_machine_provider() placement_policy = provider.get_placement_policy() client = yield provider.connect() try: service_manager = ServiceStateManager(client) service_state = yield service_manager.get_service_state(service_name) for i in range(num_units): unit_state = yield service_state.add_unit_state() yield place_unit(client, placement_policy, unit_state) log.info("Unit %r added to service %r", unit_state.unit_name, service_state.service_name) finally: yield client.close()
def config_get(environment, service_name, display_schema): """Get service settings. """ provider = environment.get_machine_provider() client = yield provider.connect() # Get the service service_manager = ServiceStateManager(client) service = yield service_manager.get_service_state(service_name) # Retrieve schema charm = yield service.get_charm_state() schema = yield charm.get_config() schema_dict = schema.as_dict() display_dict = { "service": service.service_name, "charm": (yield service.get_charm_id()), "settings": schema_dict } # Get current settings settings = yield service.get_config() settings = dict(settings.items()) # Merge current settings into schema/display dict for k, v in schema_dict.items(): if k in settings: v['value'] = settings[k] # Defaults are confusing inline display to the value for longer # strings. And there already the current value, if one wasn't set. # We do defaults for upgraded config values, which haven't been # set yet. if 'default' in v: del v['default'] else: v['value'] = "-Not set-" print yaml.safe_dump(display_dict, indent=4, default_flow_style=False, width=80, allow_unicode=True) client.close()
def test_deploy_constraints_in_legacy_env(self): yield self.client.delete("/constraints") finished = self.setup_cli_reactor() self.setup_exit(0) self.mocker.replay() stderr = self.capture_stream("stderr") main(["deploy", "--repository", self.unbundled_repo_path, "local:sample", "beekeeper", "--constraints", "arch=i386"]) yield finished self.assertIn( "Constraints are not valid in legacy deployments.", stderr.getvalue()) service_manager = ServiceStateManager(self.client) yield self.assertFailure( service_manager.get_service_state("beekeeper"), ServiceStateNotFound)
def remove_unit(config, environment, verbose, log, unit_names): provider = environment.get_machine_provider() client = yield provider.connect() try: service_manager = ServiceStateManager(client) for unit_name in unit_names: service_name = parse_service_name(unit_name) service_state = yield service_manager.get_service_state( service_name) unit_state = yield service_state.get_unit_state(unit_name) if (yield service_state.is_subordinate()): container = yield unit_state.get_container() raise UnsupportedSubordinateServiceRemoval( unit_state.unit_name, container.unit_name) yield service_state.remove_unit_state(unit_state) log.info("Unit %r removed from service %r", unit_state.unit_name, service_state.service_name) finally: yield client.close()
def __init__(self, client, machine_id, juju_directory): """Initialize a Unit Deployer. :param client: A connected zookeeper client. :param str machine_id: the ID of the machine the agent is being run on. :param str juju_directory: the directory the agent is running in. """ self.client = client self.machine_id = machine_id self.juju_directory = juju_directory self.service_state_manager = ServiceStateManager(self.client) self.charm_state_manager = CharmStateManager(self.client)
def config_get(environment, service_name, display_schema): """Get service settings. """ provider = environment.get_machine_provider() client = yield provider.connect() # Get the service service_manager = ServiceStateManager(client) service = yield service_manager.get_service_state(service_name) # Retrieve schema charm = yield service.get_charm_state() schema = yield charm.get_config() schema_dict = schema.as_dict() display_dict = {"service": service.service_name, "charm": (yield service.get_charm_id()), "settings": schema_dict} # Get current settings settings = yield service.get_config() settings = dict(settings.items()) # Merge current settings into schema/display dict for k, v in schema_dict.items(): # Display defaults for unset values. if k in settings: v['value'] = settings[k] else: v['value'] = "-Not set-" if 'default' in v: if v['default'] == settings[k]: v['default'] = True else: del v['default'] print yaml.safe_dump( display_dict, indent=4, default_flow_style=False, width=80, allow_unicode=True) client.close()
def config_get(environment, service_name, display_schema): """Get service settings. """ provider = environment.get_machine_provider() client = yield provider.connect() # Get the service service_manager = ServiceStateManager(client) service = yield service_manager.get_service_state(service_name) # Retrieve schema charm = yield service.get_charm_state() schema = yield charm.get_config() schema_dict = schema.as_dict() display_dict = {"service": service.service_name, "charm": (yield service.get_charm_id()), "settings": schema_dict} # Get current settings settings = yield service.get_config() settings = dict(settings.items()) # Merge current settings into schema/display dict for k, v in schema_dict.items(): if k in settings: v['value'] = settings[k] # Defaults are confusing inline display to the value for longer # strings. And there already the current value, if one wasn't set. # We do defaults for upgraded config values, which haven't been # set yet. if 'default' in v: del v['default'] else: v['value'] = "-Not set-" print yaml.safe_dump( display_dict, indent=4, default_flow_style=False, width=80, allow_unicode=True) client.close()
def validate_hooks(client, unit_state, hook_names): # Assemble a list of valid hooks for the charm. valid_hooks = ["start", "stop", "install", "config-changed"] service_manager = ServiceStateManager(client) endpoints = yield service_manager.get_relation_endpoints( unit_state.service_name) endpoint_names = [ep.relation_name for ep in endpoints] for endpoint_name in endpoint_names: valid_hooks.extend([ endpoint_name + "-relation-joined", endpoint_name + "-relation-changed", endpoint_name + "-relation-departed", endpoint_name + "-relation-broken", ]) # Verify the debug names. for hook_name in hook_names: if hook_name in valid_hooks: continue break else: returnValue(True) # We dereference to the charm to give a fully qualified error # message. I wish this was a little easier to dereference, the # service_manager.get_relation_endpoints effectively does this # already. service_manager = ServiceStateManager(client) service_state = yield service_manager.get_service_state( unit_state.service_name) charm_id = yield service_state.get_charm_id() charm_manager = CharmStateManager(client) charm = yield charm_manager.get_charm_state(charm_id) raise InvalidCharmHook(charm.id, hook_name)
def start(self): """Start the machine agent. Creates state directories on the machine, retrieves the machine state, and enables watch on assigned units. """ # Initialize directory paths. if not os.path.exists(self.charms_directory): os.makedirs(self.charms_directory) if not os.path.exists(self.units_directory): os.makedirs(self.units_directory) if not os.path.exists(self.unit_state_directory): os.makedirs(self.unit_state_directory) # Get state managers we'll be utilizing. self.service_state_manager = ServiceStateManager(self.client) self.charm_state_manager = CharmStateManager(self.client) # Retrieve the machine state for the machine we represent. machine_manager = MachineStateManager(self.client) self.machine_state = yield machine_manager.get_machine_state( self.get_machine_id()) # Watch assigned units for the machine. if self.get_watch_enabled(): self.machine_state.watch_assigned_units(self.watch_service_units) # Find out what provided the machine, and how to deploy units. settings = GlobalSettingsStateManager(self.client) self.provider_type = yield settings.get_provider_type() self.deploy_factory = get_deploy_factory(self.provider_type) # Connect the machine agent, broadcasting presence to the world. yield self.machine_state.connect_agent() log.info( "Machine agent started id:%s deploy:%r provider:%r" % (self.get_machine_id(), self.deploy_factory, self.provider_type))
def test_deploy_upgrade(self): """A charm can be deployed and get the latest version""" environment = self.config.get("firstenv") yield deploy.deploy( self.config, environment, self.unbundled_repo_path, "local:sample", "myblog", logging.getLogger("deploy"), []) yield deploy.deploy( self.config, environment, self.unbundled_repo_path, "local:sample", "myblog2", logging.getLogger("deploy"), [], upgrade=True) services = ServiceStateManager(self.client) service1 = yield services.get_service_state("myblog") s1_charm_id = yield service1.get_charm_id() service2 = yield services.get_service_state("myblog2") s2_charm_id = yield service2.get_charm_id() self.assertNotEqual(s1_charm_id, s2_charm_id) charms = CharmStateManager(self.client) charm1 = yield charms.get_charm_state(s1_charm_id) charm2 = yield charms.get_charm_state(s2_charm_id) self.assertEqual(charm1.revision + 1, charm2.revision)
def test_deploy_with_default_config(self): """Valid config options should be available to the deployed service.""" environment = self.config.get("firstenv") # Here we explictly pass no config file but the services # associated config.yaml defines default which we expect to # find anyway. yield deploy.deploy(self.config, environment, self.unbundled_repo_path, "local:dummy", "myblog", logging.getLogger("deploy"), None) # Verify that options in the yaml are available as state after # the deploy call (successfully applied) service = yield ServiceStateManager( self.client).get_service_state("myblog") config = yield service.get_config() self.assertEqual(config["title"], "My Title")
def test_deploy_with_config(self): """Valid config options should be available to the deployed service.""" config_file = self.makeFile( yaml.dump(dict(myblog=dict(outlook="sunny", username="******")))) environment = self.config.get("firstenv") yield deploy.deploy(self.config, environment, self.unbundled_repo_path, "local:dummy", "myblog", logging.getLogger("deploy"), config_file) # Verify that options in the yaml are available as state after # the deploy call (successfully applied) service = yield ServiceStateManager( self.client).get_service_state("myblog") config = yield service.get_config() self.assertEqual(config["outlook"], "sunny") self.assertEqual(config["username"], "tester01") # a default value from the config.yaml self.assertEqual(config["title"], "My Title")
class MachineAgent(BaseAgent): """An juju Machine Agent. The machine agent is responsible for monitoring service units assigned to a machine. If a new unit is assigned to machine, the machine agent will download the charm, create a working space for the service unit agent, and then launch it. Additionally the machine agent will monitor the running service unit agents on the machine, via their ephemeral nodes, and restart them if they die. """ name = "juju-machine-agent" unit_agent_module = "juju.agents.unit" @property def charms_directory(self): return os.path.join(self.config["juju_directory"], "charms") @property def units_directory(self): return os.path.join(self.config["juju_directory"], "units") @property def unit_state_directory(self): return os.path.join(self.config["juju_directory"], "state") @inlineCallbacks def start(self): """Start the machine agent. Creates state directories on the machine, retrieves the machine state, and enables watch on assigned units. """ # Initialize directory paths. if not os.path.exists(self.charms_directory): os.makedirs(self.charms_directory) if not os.path.exists(self.units_directory): os.makedirs(self.units_directory) if not os.path.exists(self.unit_state_directory): os.makedirs(self.unit_state_directory) # Get state managers we'll be utilizing. self.service_state_manager = ServiceStateManager(self.client) self.charm_state_manager = CharmStateManager(self.client) # Retrieve the machine state for the machine we represent. machine_manager = MachineStateManager(self.client) self.machine_state = yield machine_manager.get_machine_state( self.get_machine_id()) # Watch assigned units for the machine. if self.get_watch_enabled(): self.machine_state.watch_assigned_units(self.watch_service_units) # Find out what provided the machine, and how to deploy units. settings = GlobalSettingsStateManager(self.client) self.provider_type = yield settings.get_provider_type() self.deploy_factory = get_deploy_factory(self.provider_type) # Connect the machine agent, broadcasting presence to the world. yield self.machine_state.connect_agent() log.info( "Machine agent started id:%s deploy:%r provider:%r" % (self.get_machine_id(), self.deploy_factory, self.provider_type)) def download_charm(self, charm_state): """Retrieve a charm from the provider storage to the local machine. Utilizes a local charm cache to avoid repeated downloading of the same charm. """ log.debug("Downloading charm %s to %s", charm_state.id, self.charms_directory) return download_charm(self.client, charm_state.id, self.charms_directory) @inlineCallbacks def watch_service_units(self, old_units, new_units): """Callback invoked when the assigned service units change. """ if old_units is None: old_units = set() log.debug("Units changed old:%s new:%s", old_units, new_units) stopped = old_units - new_units started = new_units - old_units for unit_name in stopped: log.debug("Stopping service unit: %s ...", unit_name) try: yield self.kill_service_unit(unit_name) except Exception: log.exception("Error stopping unit: %s", unit_name) for unit_name in started: log.debug("Starting service unit: %s ...", unit_name) try: yield self.start_service_unit(unit_name) except Exception: log.exception("Error starting unit: %s", unit_name) @inlineCallbacks def start_service_unit(self, service_unit_name): """Start a service unit on the machine. Downloads the charm, and extract it to the service unit directory, and launch the service unit agent within the unit directory. """ # Retrieve the charm state to get at the charm. unit_state = yield self.service_state_manager.get_unit_state( service_unit_name) charm_id = yield unit_state.get_charm_id() charm_state = yield self.charm_state_manager.get_charm_state(charm_id) # Download the charm. bundle = yield self.download_charm(charm_state) # Use deployment to setup the workspace and start the unit agent. deployment = self.deploy_factory(service_unit_name, self.config["juju_directory"]) running = yield deployment.is_running() if not running: log.debug("Starting service unit %s", service_unit_name) yield deployment.start(self.get_machine_id(), self.client.servers, bundle) log.info("Started service unit %s", service_unit_name) def kill_service_unit(self, service_unit_name): """Stop service unit and destroy disk state, ala SIGKILL or lxc-destroy """ deployment = self.deploy_factory(service_unit_name, self.config["juju_directory"]) log.info("Stopping service unit %s...", service_unit_name) return deployment.destroy() def get_machine_id(self): """Get the id of the machine as known within the zk state.""" return self.config["machine_id"] def get_agent_name(self): return "Machine:%s" % (self.get_machine_id()) def configure(self, options): super(MachineAgent, self).configure(options) if not options.get("machine_id"): msg = ("--machine-id must be provided in the command line, " "or $JUJU_MACHINE_ID in the environment") raise JujuError(msg) @classmethod def setup_options(cls, parser): super(MachineAgent, cls).setup_options(parser) machine_id = os.environ.get("JUJU_MACHINE_ID", "") parser.add_argument("--machine-id", default=machine_id) return parser
def get_local_unit_state(self): """Return ServiceUnitState for the local service unit.""" service_state_manager = ServiceStateManager(self._client) unit_state = yield service_state_manager.get_unit_state( self._unit_name) returnValue(unit_state)
class UnitAgent(BaseAgent): """An juju Unit Agent. Provides for the management of a charm, via hook execution in response to external events in the coordination space (zookeeper). """ name = "juju-unit-agent" @classmethod def setup_options(cls, parser): super(UnitAgent, cls).setup_options(parser) unit_name = os.environ.get("JUJU_UNIT_NAME", "") parser.add_argument("--unit-name", default=unit_name) @property def unit_name(self): return self.config["unit_name"] def get_agent_name(self): return "unit:%s" % self.unit_name def configure(self, options): """Configure the unit agent.""" super(UnitAgent, self).configure(options) if not options.get("unit_name"): msg = ("--unit-name must be provided in the command line, " "or $JUJU_UNIT_NAME in the environment") raise JujuError(msg) self.executor = HookExecutor() self.api_factory = UnitSettingsFactory( self.executor.get_hook_context, logging.getLogger("unit.hook.api")) self.api_socket = None self.workflow = None @inlineCallbacks def start(self): """Start the unit agent process.""" self.service_state_manager = ServiceStateManager(self.client) self.executor.start() # Retrieve our unit and configure working directories. service_name = self.unit_name.split("/")[0] service_state = yield self.service_state_manager.get_service_state( service_name) self.unit_state = yield service_state.get_unit_state(self.unit_name) self.unit_directory = os.path.join( self.config["juju_directory"], "units", self.unit_state.unit_name.replace("/", "-")) self.state_directory = os.path.join(self.config["juju_directory"], "state") # Setup the server portion of the cli api exposed to hooks. from twisted.internet import reactor self.api_socket = reactor.listenUNIX( os.path.join(self.unit_directory, HOOK_SOCKET_FILE), self.api_factory) # Setup the unit state's address address = yield get_unit_address(self.client) yield self.unit_state.set_public_address( (yield address.get_public_address())) yield self.unit_state.set_private_address( (yield address.get_private_address())) # Inform the system, we're alive. yield self.unit_state.connect_agent() self.lifecycle = UnitLifecycle(self.client, self.unit_state, service_state, self.unit_directory, self.executor) self.workflow = UnitWorkflowState(self.client, self.unit_state, self.lifecycle, self.state_directory) if self.get_watch_enabled(): yield self.unit_state.watch_resolved(self.cb_watch_resolved) yield self.unit_state.watch_hook_debug(self.cb_watch_hook_debug) yield service_state.watch_config_state( self.cb_watch_config_changed) # Fire initial transitions, only if successful if (yield self.workflow.transition_state("installed")): yield self.workflow.transition_state("started") # Upgrade can only be processed if we're in a running state so # for case of a newly started unit, do it after the unit is started. if self.get_watch_enabled(): yield self.unit_state.watch_upgrade_flag( self.cb_watch_upgrade_flag) @inlineCallbacks def stop(self): """Stop the unit agent process.""" if self.workflow: yield self.workflow.transition_state("stopped") if self.api_socket: yield self.api_socket.stopListening() yield self.api_factory.stopFactory() @inlineCallbacks def cb_watch_resolved(self, change): """Update the unit's state, when its resolved. Resolved operations form the basis of error recovery for unit workflows. A resolved operation can optionally specify hook execution. The unit agent runs the error recovery transition if the unit is not in a running state. """ # Would be nice if we could fold this into an atomic # get and delete primitive. # Check resolved setting resolved = yield self.unit_state.get_resolved() if resolved is None: returnValue(None) # Clear out the setting yield self.unit_state.clear_resolved() # Verify its not already running if (yield self.workflow.get_state()) == "started": returnValue(None) log.info("Resolved detected, firing retry transition") # Fire a resolved transition try: if resolved["retry"] == RETRY_HOOKS: yield self.workflow.fire_transition_alias("retry_hook") else: yield self.workflow.fire_transition_alias("retry") except Exception: log.exception("Unknown error while transitioning for resolved") @inlineCallbacks def cb_watch_hook_debug(self, change): """Update the hooks to be debugged when the settings change. """ debug = yield self.unit_state.get_hook_debug() debug_hooks = debug and debug.get("debug_hooks") or None self.executor.set_debug(debug_hooks) @inlineCallbacks def cb_watch_upgrade_flag(self, change): """Update the unit's charm when requested. """ upgrade_flag = yield self.unit_state.get_upgrade_flag() if upgrade_flag: log.info("Upgrade detected, starting upgrade") upgrade = CharmUpgradeOperation(self) try: yield upgrade.run() except Exception: log.exception("Error while upgrading") @inlineCallbacks def cb_watch_config_changed(self, change): """Trigger hook on configuration change""" # Verify it is running current_state = yield self.workflow.get_state() log.debug("Configuration Changed") if current_state != "started": log.debug( "Configuration updated on service in a non-started state") returnValue(None) yield self.workflow.fire_transition("reconfigure")
class MachineAgentTest(AgentTestBase, RepositoryTestBase): agent_class = MachineAgent @inlineCallbacks def setUp(self): yield super(MachineAgentTest, self).setUp() self.output = self.capture_logging("juju.agents.machine", level=logging.DEBUG) config = self.get_test_environment_config() environment = config.get_default() # Store the environment to zookeeper environment_state_manager = EnvironmentStateManager(self.client) yield environment_state_manager.set_config_state(config, "myfirstenv") # Load the environment with the charm state and charm binary self.provider = environment.get_machine_provider() self.storage = self.provider.get_file_storage() self.charm = CharmDirectory(self.sample_dir1) self.publisher = CharmPublisher(self.client, self.storage) yield self.publisher.add_charm(local_charm_id(self.charm), self.charm) charm_states = yield self.publisher.publish() self.charm_state = charm_states[0] # Create a service from the charm from which we can create units for # the machine. self.service_state_manager = ServiceStateManager(self.client) self.service = yield self.service_state_manager.add_service_state( "fatality-blog", self.charm_state) def process_kill(self, pid): try: os.kill(pid, 9) except OSError: pass @inlineCallbacks def get_agent_config(self): # gets invoked by AgentTestBase.setUp options = yield super(MachineAgentTest, self).get_agent_config() machine_state_manager = MachineStateManager(self.client) self.machine_state = yield machine_state_manager.add_machine_state() self.change_environment(JUJU_MACHINE_ID="0", JUJU_HOME=self.juju_directory) options["machine_id"] = str(self.machine_state.id) # Start the agent with watching enabled returnValue(options) @inlineCallbacks def test_start_begins_watch_and_initializes_directories(self): self.agent.set_watch_enabled(True) mock_machine_state = self.mocker.patch(MachineState) mock_machine_state.watch_assigned_units(self.agent.watch_service_units) self.mocker.replay() yield self.agent.startService() self.assertTrue(os.path.isdir(self.agent.charms_directory)) self.assertTrue(os.path.isdir(self.agent.units_directory)) self.assertTrue(os.path.isdir(self.agent.unit_state_directory)) self.assertIn( "Machine agent started id:%s" % self.agent.get_machine_id(), self.output.getvalue()) yield self.agent.stopService() def test_agent_machine_id_environment_extraction(self): self.change_args("es-agent") parser = argparse.ArgumentParser() self.agent.setup_options(parser) config = parser.parse_args(namespace=TwistedOptionNamespace()) self.assertEqual(config["machine_id"], "0") def test_get_agent_name(self): self.assertEqual(self.agent.get_agent_name(), "Machine:0") def test_agent_machine_id_cli_error(self): """ If the machine id can't be found, a detailed error message is given. """ # initially setup by get_agent_config in setUp self.change_environment(JUJU_MACHINE_ID="") self.change_args("es-agent", "--zookeeper-servers", get_test_zookeeper_address(), "--juju-directory", self.makeDir()) parser = argparse.ArgumentParser() self.agent.setup_options(parser) options = parser.parse_args(namespace=TwistedOptionNamespace()) e = self.assertRaises(JujuError, self.agent.configure, options) self.assertIn(("--machine-id must be provided in the command line," " or $JUJU_MACHINE_ID in the environment"), str(e)) def test_agent_machine_id_cli_extraction(self): """Command line passing of machine id works and has precedence over environment arg passing.""" self.change_environment(JUJU_MACHINE_ID=str(21)) self.change_args("es-agent", "--machine-id", "0") parser = argparse.ArgumentParser() self.agent.setup_options(parser) config = parser.parse_args(namespace=TwistedOptionNamespace()) self.assertEqual(config["machine_id"], "0") def test_machine_agent_knows_its_machine_id(self): self.assertEqual(self.agent.get_machine_id(), "0") @inlineCallbacks def test_charm_download(self): """ Downloading a charm should store the charm locally. """ yield self.agent.startService() yield self.agent.download_charm(self.charm_state) checksum = self.charm.get_sha256() charm_id = local_charm_id(self.charm) charm_key = under.quote("%s:%s" % (charm_id, checksum)) charm_path = os.path.join(self.agent.charms_directory, charm_key) self.assertTrue(os.path.exists(charm_path)) bundle = CharmBundle(charm_path) self.assertEquals(bundle.get_revision(), self.charm.get_revision()) self.assertEquals(bundle.get_sha256(), checksum) self.assertIn("Downloading charm %s" % charm_id, self.output.getvalue()) @inlineCallbacks def test_watch_new_service_unit(self): """ Adding a new service unit is detected by the watch. """ self.agent.set_watch_enabled(True) yield self.agent.startService() mock_deployment = self.mocker.patch(self.agent.deploy_factory) mock_deployment.start("0", get_test_zookeeper_address(), MATCH_BUNDLE) test_deferred = Deferred() def test_complete(machine_id, servers, bundle): test_deferred.callback(True) self.mocker.call(test_complete) self.mocker.replay() # Create a new service unit self.service_unit = yield self.service.add_unit_state() yield self.service_unit.assign_to_machine(self.machine_state) yield test_deferred self.assertIn("Units changed old:set([]) new:set(['fatality-blog/0'])", self.output.getvalue()) @inlineCallbacks def test_watch_new_service_unit_error(self): """ An error while starting a new service is logged """ # Inject an error into the service deployment mock_agent = self.mocker.patch(self.agent) mock_agent.start_service_unit("fatality-blog/0") self.mocker.result(fail(SyntaxError("Bad"))) self.mocker.replay() yield self.agent.watch_service_units(None, set(["fatality-blog/0"])) self.assertIn("Starting service unit: %s" % "fatality-blog/0", self.output.getvalue()) self.assertIn("Error starting unit: %s" % "fatality-blog/0", self.output.getvalue()) self.assertIn("SyntaxError: Bad", self.output.getvalue()) @inlineCallbacks def test_service_unit_removed(self): """ Service unit removed with manual invocation of watch_service_units. """ # Start the agent with watching enabled self.agent.set_watch_enabled(True) yield self.agent.startService() test_deferred = Deferred() mock_deployment = self.mocker.patch(self.agent.deploy_factory) mock_deployment.start("0", get_test_zookeeper_address(), MATCH_BUNDLE) self.mocker.result(succeed(True)) mock_deployment.destroy() self.mocker.result(succeed(True)) def test_complete(): test_deferred.callback(True) self.mocker.call(test_complete) self.mocker.replay() # Create a new service unit self.service_unit = yield self.service.add_unit_state() yield self.service_unit.assign_to_machine(self.machine_state) # And now remove it yield self.service_unit.unassign_from_machine() yield test_deferred @inlineCallbacks def test_watch_removed_service_unit_error(self): """ An error while removing a service unit is logged """ mock_agent = self.mocker.patch(self.agent) mock_agent.kill_service_unit("fatality-blog/0") self.mocker.result(fail(OSError("Bad"))) self.mocker.replay() yield self.agent.watch_service_units(set(["fatality-blog/0"]), set()) self.assertIn("Stopping service unit: %s" % "fatality-blog/0", self.output.getvalue()) self.assertIn("Error stopping unit: %s" % "fatality-blog/0", self.output.getvalue()) self.assertIn("OSError: Bad", self.output.getvalue())
def upgrade_charm( config, environment, verbose, log, repository_path, service_name, dry_run): """Upgrades a service's charm. First determines if an upgrade is available, then updates the service charm reference, and marks the units as needing upgrades. """ provider = environment.get_machine_provider() client = yield provider.connect() service_manager = ServiceStateManager(client) service_state = yield service_manager.get_service_state(service_name) old_charm_id = yield service_state.get_charm_id() old_charm_url = CharmURL.parse(old_charm_id) old_charm_url.assert_revision() repo, charm_url = resolve( str(old_charm_url.with_revision(None)), repository_path, environment.default_series) new_charm_url = charm_url.with_revision( (yield repo.latest(charm_url))) if charm_url.collection.schema == "local": if old_charm_url.revision >= new_charm_url.revision: new_revision = old_charm_url.revision + 1 charm = yield repo.find(new_charm_url) if isinstance(charm, CharmDirectory): if dry_run: log.info("%s would be set to revision %s", charm.path, new_revision) else: log.info("Setting %s to revision %s", charm.path, new_revision) charm.set_revision(new_revision) new_charm_url.revision = new_revision new_charm_id = str(new_charm_url) # Verify its newer than what's deployed if not new_charm_url.revision > old_charm_url.revision: if dry_run: log.info("Service already running latest charm %r", old_charm_id) else: raise NewerCharmNotFound(old_charm_id) elif dry_run: log.info("Service would be upgraded from charm %r to %r", old_charm_id, new_charm_id) # On dry run, stop before modifying state. if not dry_run: # Publish the new charm storage = provider.get_file_storage() publisher = CharmPublisher(client, storage) charm = yield repo.find(new_charm_url) yield publisher.add_charm(new_charm_id, charm) result = yield publisher.publish() charm_state = result[0] # Update the service charm reference yield service_state.set_charm_id(charm_state.id) # Mark the units for upgrades units = yield service_state.get_all_unit_states() for unit in units: running, state = yield is_unit_running(client, unit) if not running: log.info( "Unit %r is not in a running state (state: %r), won't upgrade", unit.unit_name, state or "uninitialized") continue if not dry_run: yield unit.set_upgrade_flag()
def collect(scope, machine_provider, client, log): """Extract status information into nested dicts for rendering. `scope`: an optional list of name specifiers. Globbing based wildcards supported. Defaults to all units, services and relations. `machine_provider`: machine provider for the environment `client`: ZK client connection `log`: a Python stdlib logger. """ service_manager = ServiceStateManager(client) relation_manager = RelationStateManager(client) machine_manager = MachineStateManager(client) charm_manager = CharmStateManager(client) service_data = {} machine_data = {} state = dict(services=service_data, machines=machine_data) seen_machines = set() filter_services, filter_units = digest_scope(scope) services = yield service_manager.get_all_service_states() for service in services: if len(filter_services): found = False for filter_service in filter_services: if fnmatch(service.service_name, filter_service): found = True break if not found: continue unit_data = {} relation_data = {} charm_id = yield service.get_charm_id() charm = yield charm_manager.get_charm_state(charm_id) service_data[service.service_name] = dict(units=unit_data, charm=charm.id, relations=relation_data) exposed = yield service.get_exposed_flag() if exposed: service_data[service.service_name].update(exposed=exposed) units = yield service.get_all_unit_states() unit_matched = False relations = yield relation_manager.get_relations_for_service(service) for unit in units: if len(filter_units): found = False for filter_unit in filter_units: if fnmatch(unit.unit_name, filter_unit): found = True break if not found: continue u = unit_data[unit.unit_name] = dict() machine_id = yield unit.get_assigned_machine_id() u["machine"] = machine_id unit_workflow_client = WorkflowStateClient(client, unit) unit_state = yield unit_workflow_client.get_state() if not unit_state: u["state"] = "pending" else: unit_connected = yield unit.has_agent() u["state"] = unit_state if unit_connected else "down" if exposed: open_ports = yield unit.get_open_ports() u["open-ports"] = [ "{port}/{proto}".format(**port_info) for port_info in open_ports ] u["public-address"] = yield unit.get_public_address() # indicate we should include information about this # machine later seen_machines.add(machine_id) unit_matched = True # collect info on each relation for the service unit relation_status = {} for relation in relations: try: relation_unit = yield relation.get_unit_state(unit) except UnitRelationStateNotFound: # This exception will occur when relations are # established between services without service # units, and therefore never have any # corresponding service relation units. This # scenario does not occur in actual deployments, # but can happen in test circumstances. In # particular, it will happen with a misconfigured # provider, which exercises this codepath. continue # should not occur, but status should not fail relation_workflow_client = WorkflowStateClient( client, relation_unit) relation_workflow_state = \ yield relation_workflow_client.get_state() relation_status[relation.relation_name] = dict( state=relation_workflow_state) u["relations"] = relation_status # after filtering units check if any matched or remove the # service from the output if filter_units and not unit_matched: del service_data[service.service_name] continue for relation in relations: rel_services = yield relation.get_service_states() # A single related service implies a peer relation. More # imply a bi-directional provides/requires relationship. # In the later case we omit the local side of the relation # when reporting. if len(rel_services) > 1: # Filter out self from multi-service relations. rel_services = [ rsn for rsn in rel_services if rsn.service_name != service.service_name ] if len(rel_services) > 1: raise ValueError("Unexpected relationship with more " "than 2 endpoints") rel_service = rel_services[0] relation_data[relation.relation_name] = rel_service.service_name machines = yield machine_manager.get_all_machine_states() for machine_state in machines: if (filter_services or filter_units) and \ machine_state.id not in seen_machines: continue instance_id = yield machine_state.get_instance_id() m = {"instance-id": instance_id \ if instance_id is not None else "pending"} if instance_id is not None: try: pm = yield machine_provider.get_machine(instance_id) m["dns-name"] = pm.dns_name m["instance-state"] = pm.state if (yield machine_state.has_agent()): # if the agent's connected, we're fine m["state"] = "running" else: units = (yield machine_state.get_all_service_unit_states()) for unit in units: unit_workflow_client = WorkflowStateClient( client, unit) if (yield unit_workflow_client.get_state()): # for unit to have a state, its agent must have # run, which implies the machine agent must have # been running correctly at some point in the past m["state"] = "down" break else: # otherwise we're probably just still waiting m["state"] = "not-started" except ProviderError: # The provider doesn't have machine information log.error("Machine provider information missing: machine %s" % (machine_state.id)) machine_data[machine_state.id] = m returnValue(state)
class FirewallManager(object): """Manages the opening and closing of ports in the firewall. """ def __init__(self, client, is_running, provider): """Initialize a Firewall Manager. :param client: A connected zookeeper client. :param is_running: A function (usually a bound method) that returns whether the associated agent is still running or not. :param provider: A machine provider, used for making the actual changes in the environment to firewall settings. """ self.machine_state_manager = MachineStateManager(client) self.service_state_manager = ServiceStateManager(client) self.is_running = is_running self.provider = provider # Track all currently watched machines, using machine ID. self._watched_machines = set() # Map service name to either NotExposed or set of exposed unit names. # If a service name is present in the dictionary, it means its # respective expose node is being watched. self._watched_services = {} # Machines to retry open_close_ports because of earlier errors self._retry_machines_on_port_error = set() # Registration of observers for corresponding actions self._open_close_ports_observers = set() self._open_close_ports_on_machine_observers = set() @inlineCallbacks def process_machine(self, machine_state): """Ensures watch is setup per machine and performs any necessary retry. :param machine_state: The machine state of the machine to be checked. The watch that is established, via :method:`juju.state.machine.MachineState.watch_assigned_units`, handles the scenario where a service or service unit is removed from the topology. Because the service unit is no longer in the topology, the corresponding watch terminates and is unable to `open_close_ports` in response to the change. However, the specific machine watch will be called in this case, and that suffices to determine that its port policy should be checked. In addition, this method can rely on the fact that the provisioning agent periodically rechecks machines so as to support retries of security group operations that failed for that provider. This method is called by the corresponding :method:`juju.agents.provision.ProvisioningAgent.process_machine` in the provisioning agent. """ if machine_state.id in self._retry_machines_on_port_error: self._retry_machines_on_port_error.remove(machine_state.id) try: yield self.open_close_ports_on_machine(machine_state.id) except StopWatcher: # open_close_ports_on_machine can also be called from # a watch, so simply ignore this since it's just used # to shutdown a watch in the case of agent shutdown pass def cb_watch_assigned_units(old_units, new_units): """Watch assigned units for changes possibly require port mgmt. """ log.debug("Assigned units for machine %r: old=%r, new=%r", machine_state.id, old_units, new_units) return self.open_close_ports_on_machine(machine_state.id) if machine_state.id not in self._watched_machines: self._watched_machines.add(machine_state.id) yield machine_state.watch_assigned_units(cb_watch_assigned_units) @inlineCallbacks def watch_service_changes(self, old_services, new_services): """Manage watching service exposed status. This method is called upon every change to the set of services currently deployed. All services are then watched for changes to their exposed flag setting. :param old_services: the set of services before this change. :param new_services: the current set of services. """ removed_services = old_services - new_services for service_name in removed_services: self._watched_services.pop(service_name, None) for service_name in new_services: yield self._setup_new_service_watch(service_name) @inlineCallbacks def _setup_new_service_watch(self, service_name): """Sets up the watching of the exposed flag for a new service. If `service_name` is not watched (as known by `self._watched_services`), adds the watch and a corresponding entry in self._watched_services. (This dict is necessary because there is currently no way to introspect a service for whether it is watched or not.) """ if service_name in self._watched_services: return # already watched self._watched_services[service_name] = NotExposed try: service_state = yield self.service_state_manager.get_service_state( service_name) except ServiceStateNotFound: log.debug("Cannot setup watch, since service %r no longer exists", service_name) self._watched_services.pop(service_name, None) return @inlineCallbacks def cb_watch_service_exposed_flag(exposed): if not self.is_running(): raise StopWatcher() if exposed: log.debug("Service %r is exposed", service_name) else: log.debug("Service %r is unexposed", service_name) try: unit_states = yield service_state.get_all_unit_states() except StateChanged: log.debug("Stopping watch on %r, no longer in topology", service_name) raise StopWatcher() for unit_state in unit_states: yield self.open_close_ports(unit_state) if not exposed: log.debug("Service %r is unexposed", service_name) self._watched_services[service_name] = NotExposed else: log.debug("Service %r is exposed", service_name) self._watched_services[service_name] = set() yield self._setup_service_unit_watch(service_state) yield service_state.watch_exposed_flag(cb_watch_service_exposed_flag) log.debug("Started watch of %r on changes to being exposed", service_name) @inlineCallbacks def _setup_service_unit_watch(self, service_state): """Setup watches on service units of newly exposed `service_name`.""" @inlineCallbacks def cb_check_service_units(old_service_units, new_service_units): watched_units = self._watched_services.get( service_state.service_name, NotExposed) if not self.is_running() or watched_units is NotExposed: raise StopWatcher() removed_service_units = old_service_units - new_service_units for unit_name in removed_service_units: watched_units.discard(unit_name) if not self.is_running(): raise StopWatcher() try: unit_state = yield service_state.get_unit_state(unit_name) except (ServiceUnitStateNotFound, StateChanged): log.debug("Not setting up watch on %r, not in topology", unit_name) continue yield self.open_close_ports(unit_state) for unit_name in new_service_units: if unit_name not in watched_units: watched_units.add(unit_name) yield self._setup_watch_ports(service_state, unit_name) yield service_state.watch_service_unit_states(cb_check_service_units) log.debug("Started watch of service units for exposed service %r", service_state.service_name) @inlineCallbacks def _setup_watch_ports(self, service_state, unit_name): """Setup the watching of ports for `unit_name`.""" try: unit_state = yield service_state.get_unit_state(unit_name) except (ServiceUnitStateNotFound, StateChanged): log.debug("Cannot setup watch on %r (no longer exists), ignoring", unit_name) return @inlineCallbacks def cb_watch_ports(value): """Permanently watch ports until service is no longer exposed.""" watched_units = self._watched_services.get( service_state.service_name, NotExposed) if (not self.is_running() or watched_units is NotExposed or unit_name not in watched_units): log.debug("Stopping ports watch for %r", unit_name) raise StopWatcher() yield self.open_close_ports(unit_state) yield unit_state.watch_ports(cb_watch_ports) log.debug("Started watch of %r on changes to open ports", unit_name) def add_open_close_ports_observer(self, observer): """Set `observer` for calls to `open_close_ports`. :param observer: The callback is called with the corresponding :class:`juju.state.service.UnitState`. """ self._open_close_ports_observers.add(observer) @inlineCallbacks def open_close_ports(self, unit_state): """Called upon changes that *may* open/close ports for a service unit. """ if not self.is_running(): raise StopWatcher() try: try: machine_id = yield unit_state.get_assigned_machine_id() except StateChanged: log.debug("Stopping watch, machine %r no longer in topology", unit_state.unit_name) raise StopWatcher() if machine_id is not None: yield self.open_close_ports_on_machine(machine_id) finally: # Ensure that the observations runs after the # corresponding action completes. In particular, tests # that use observation depend on this ordering to ensure # that the action has in fact happened before they can # proceed. observers = list(self._open_close_ports_observers) for observer in observers: yield observer(unit_state) def add_open_close_ports_on_machine_observer(self, observer): """Add `observer` for calls to `open_close_ports`. :param observer: A callback receives the machine id for each call. """ self._open_close_ports_on_machine_observers.add(observer) @inlineCallbacks def open_close_ports_on_machine(self, machine_id): """Called upon changes that *may* open/close ports for a machine. :param machine_id: The machine ID of the machine that needs to be checked. This machine supports multiple service units being assigned to a machine; all service units are checked each time this is called to determine the active set of ports to be opened. """ if not self.is_running(): raise StopWatcher() try: machine_state = yield self.machine_state_manager.get_machine_state( machine_id) instance_id = yield machine_state.get_instance_id() machine = yield self.provider.get_machine(instance_id) unit_states = yield machine_state.get_all_service_unit_states() policy_ports = set() for unit_state in unit_states: service_state = yield self.service_state_manager.\ get_service_state(unit_state.service_name) exposed = yield service_state.get_exposed_flag() if exposed: ports = yield unit_state.get_open_ports() for port in ports: policy_ports.add((port["port"], port["proto"])) current_ports = yield self.provider.get_opened_ports( machine, machine_id) to_open = policy_ports - current_ports to_close = current_ports - policy_ports for port, proto in to_open: yield self.provider.open_port(machine, machine_id, port, proto) for port, proto in to_close: yield self.provider.close_port(machine, machine_id, port, proto) except MachinesNotFound: log.info("No provisioned machine for machine %r", machine_id) except Exception: log.exception("Got exception in opening/closing ports, will retry") self._retry_machines_on_port_error.add(machine_id) finally: # Ensure that the observation runs after the corresponding # action completes. In particular, tests that use # observation depend on this ordering to ensure that this # action has happened before they can proceed. observers = list(self._open_close_ports_on_machine_observers) for observer in observers: yield observer(machine_id)
def resolved(config, environment, verbose, log, unit_name, relation_name, retry): """Mark an error as resolved in a unit or unit relation. If one of a unit's charm non-relation hooks returns a non-zero exit status, the entire unit can be considered to be in a non-running state. As a resolution, the the unit can be manually returned a running state via the juju resolved command. Optionally this command can also rerun the failed hook. This resolution also applies separately to each of the unit's relations. If one of the relation-hooks failed. In that case there is no notion of retrying (the change is gone), but resolving will allow additional relation hooks for that relation to proceed. """ provider = environment.get_machine_provider() client = yield provider.connect() service_manager = ServiceStateManager(client) relation_manager = RelationStateManager(client) unit_state = yield service_manager.get_unit_state(unit_name) service_state = yield service_manager.get_service_state( unit_name.split("/")[0]) retry = retry and RETRY_HOOKS or NO_HOOKS if not relation_name: running, workflow_state = yield is_unit_running(client, unit_state) if running: log.info("Unit %r already running: %s", unit_name, workflow_state) client.close() returnValue(False) yield unit_state.set_resolved(retry) log.info("Marked unit %r as resolved", unit_name) returnValue(True) # Check for the matching relations service_relations = yield relation_manager.get_relations_for_service( service_state) service_relations = [ sr for sr in service_relations if sr.relation_name == relation_name ] if not service_relations: raise RelationStateNotFound() # Verify the relations are in need of resolution. resolved_relations = {} for service_relation in service_relations: unit_relation = yield service_relation.get_unit_state(unit_state) running, state = yield is_relation_running(client, unit_relation) if not running: resolved_relations[unit_relation.internal_relation_id] = retry if not resolved_relations: log.warning("Matched relations are all running") client.close() returnValue(False) # Mark the relations as resolved. yield unit_state.set_relation_resolved(resolved_relations) log.info("Marked unit %r relation %r as resolved", unit_name, relation_name) client.close()
class MachineStateManagerTest(StateTestBase): @inlineCallbacks def setUp(self): yield super(MachineStateManagerTest, self).setUp() self.charm_state_manager = CharmStateManager(self.client) self.machine_state_manager = MachineStateManager(self.client) self.service_state_manager = ServiceStateManager(self.client) self.charm_state = yield self.charm_state_manager.add_charm_state( local_charm_id(self.charm), self.charm, "") @inlineCallbacks def add_service(self, service_name): service_state = yield self.service_state_manager.add_service_state( service_name, self.charm_state) returnValue(service_state) @inlineCallbacks def test_add_machine(self): """ Adding a machine state should register it in zookeeper. """ machine_state1 = yield self.machine_state_manager.add_machine_state() machine_state2 = yield self.machine_state_manager.add_machine_state() self.assertEquals(machine_state1.id, 0) self.assertEquals(machine_state1.internal_id, "machine-0000000000") self.assertEquals(machine_state2.id, 1) self.assertEquals(machine_state2.internal_id, "machine-0000000001") children = yield self.client.get_children("/machines") self.assertEquals(sorted(children), ["machine-0000000000", "machine-0000000001"]) topology = yield self.get_topology() self.assertTrue(topology.has_machine("machine-0000000000")) self.assertTrue(topology.has_machine("machine-0000000001")) @inlineCallbacks def test_machine_str_representation(self): """The str(machine) value includes the machine id. """ machine_state1 = yield self.machine_state_manager.add_machine_state() self.assertEqual( str(machine_state1), "<MachineState id:machine-%010d>" % (0)) @inlineCallbacks def test_remove_machine(self): """ Adding a machine state should register it in zookeeper. """ machine_state1 = yield self.machine_state_manager.add_machine_state() yield self.machine_state_manager.add_machine_state() removed = yield self.machine_state_manager.remove_machine_state( machine_state1.id) self.assertTrue(removed) children = yield self.client.get_children("/machines") self.assertEquals(sorted(children), ["machine-0000000001"]) topology = yield self.get_topology() self.assertFalse(topology.has_machine("machine-0000000000")) self.assertTrue(topology.has_machine("machine-0000000001")) # Removing a non-existing machine again won't fail, since the end # intention is preserved. This makes dealing with concurrency easier. # However, False will be returned in this case. removed = yield self.machine_state_manager.remove_machine_state( machine_state1.id) self.assertFalse(removed) @inlineCallbacks def test_remove_machine_with_agent(self): """Removing a machine with a connected machine agent should succeed. The removal signals intent to remove a working machine (with an agent) with the provisioning agent to remove it subsequently. """ # Add two machines. machine_state1 = yield self.machine_state_manager.add_machine_state() yield self.machine_state_manager.add_machine_state() # Connect an agent yield machine_state1.connect_agent() # Remove a machine removed = yield self.machine_state_manager.remove_machine_state( machine_state1.id) self.assertTrue(removed) # Verify the second one is still present children = yield self.client.get_children("/machines") self.assertEquals(sorted(children), ["machine-0000000001"]) # Verify the topology state. topology = yield self.get_topology() self.assertFalse(topology.has_machine("machine-0000000000")) self.assertTrue(topology.has_machine("machine-0000000001")) @inlineCallbacks def test_get_machine_and_check_attributes(self): """ Getting a machine state should be possible using both the user-oriented id and the internal id. """ yield self.machine_state_manager.add_machine_state() yield self.machine_state_manager.add_machine_state() machine_state = yield self.machine_state_manager.get_machine_state(0) self.assertEquals(machine_state.id, 0) machine_state = yield self.machine_state_manager.get_machine_state("0") self.assertEquals(machine_state.id, 0) yield self.assertFailure( self.machine_state_manager.get_machine_state("a"), MachineStateNotFound) @inlineCallbacks def test_get_machine_not_found(self): """ Getting a machine state which is not available should errback a meaningful error. """ # No state whatsoever. try: yield self.machine_state_manager.get_machine_state(0) except MachineStateNotFound, e: self.assertEquals(e.machine_id, 0) else: