def __init__(self, dbobj=None, logger=LOGGER): self.config = Config() self.dbobj = dbobj self.logger = logger if self.template_type is None: raise InternalError("Plenary class %s did not set the template " "type" % self.__class__.__name__) # Object templates live under the branch-specific build directory. # Everything else lives under the common plenary directory. if self.template_type == "object": if not dbobj or not hasattr(dbobj, "branch"): raise InternalError("Plenaries meant to be compiled need a DB " "object that has a branch; got: %r" % dbobj) self.dir = "%s/domains/%s/profiles" % (self.config.get( "broker", "builddir"), dbobj.branch.name) else: self.dir = self.config.get("broker", "plenarydir") self.loadpath = None self.plenary_template = None self.plenary_core = None self.new_content = None # The following attributes are for stash/restore_stash self.old_content = None self.old_mtime = None self.stashed = False self.removed = False self.changed = False
def discover_network_types(dbapi_con, connection_record): # pylint: disable=W0613 config = Config() if not config.has_option("broker", "default_network_type"): # pragma: no cover raise InternalError("The default_network_type option is missing from " "the [broker] section in the configuration.") default_type = config.get("broker", "default_network_type") default_section = "network_" + default_type if not config.has_section(default_section): # pragma: no cover raise InternalError("The default network type is %s, but there's no " "section named [%s] in the configuration." % (default_type, default_section)) nettypes = {} # This function should be called only once, but you never know... if Network.network_type_map: return for section in config.sections(): if not section.startswith("network_"): continue name = section[8:] nettypes[name] = NetworkProperties(config, name) LOGGER.info("Configured network type %s" % name) Network.network_type_map = nettypes Network.default_network_props = nettypes[default_type]
def __init__(self, feature=None, archetype=None, personality=None, model=None, interface_name=None): # Archetype and personality are mutually exclusive. This makes # querying archetype-wide features a bit easier if archetype and personality: # pragma: no cover raise InternalError("Archetype and personality are mutually " "exclusive.") if interface_name and not personality: # pragma: no cover raise InternalError("Binding to a named interface requires " "a personality.") super(FeatureLink, self).__init__(feature=feature, archetype=archetype, personality=personality, model=model, interface_name=interface_name)
def __init__(self, dbobj, *args, **kwargs): """Provide initialization specific for cluster bindings.""" if not isinstance(dbobj, Cluster): raise InternalError("ClusterChooser can only choose services for " "clusters, got %r (%s)" % (dbobj, type(dbobj))) self.dbcluster = dbobj Chooser.__init__(self, dbobj, *args, **kwargs) self.location = self.dbcluster.location_constraint self.archetype = self.dbcluster.personality.archetype self.personality = self.dbcluster.personality self.required_services = set() # TODO Should be calculated from member host's network membership. self.network = None """Stores interim service instance lists.""" for service in self.archetype.services: self.required_services.add(service) for service in self.personality.services: self.required_services.add(service) self.original_service_instances = {} """Cache of any already bound services (keys) and the instance that was bound (values). """ for si in self.dbcluster.service_bindings: self.original_service_instances[si.service] = si self.logger.debug("%s original binding: %s", self.description, si.cfg_path)
def network(self, value): if not isinstance(value, IPv4Network): raise InternalError("Expected an IPv4Network, got: %s" % type(network)) self._network = value self.ip = value.network self.cidr = value.prefixlen
def add_host_msg(self, host_msg, host): """ Return a host message. Hosts used to be systems, which makes this method name a bit odd """ if not isinstance(host, Host): raise InternalError("add_host_msg was called with {0} instead of " "a Host.".format(host)) host_msg.type = "host" # FIXME: is hardcoding this ok? host_msg.hostname = str(host.machine.primary_name.fqdn.name) host_msg.fqdn = str(host.machine.primary_name.fqdn) host_msg.dns_domain = str( host.machine.primary_name.fqdn.dns_domain.name) if host.machine.primary_ip: host_msg.ip = str(host.machine.primary_ip) for iface in host.machine.interfaces: if iface.interface_type != 'public' or not iface.bootable: continue host_msg.mac = str(iface.mac) if host.resholder and len(host.resholder.resources) > 0: for resource in host.resholder.resources: r = host_msg.resources.add() self.redirect_proto(resource, r) self.add_host_data(host_msg, host) self.add_hardware_data(host_msg, host.machine)
def validate_link(self, key, link): if (link.model and link.model.machine_type == 'nic') or \ (link.interface_name and link.personality): return link raise InternalError("Interface features can only be bound to " "NIC models or personality/interface name pairs.")
def __init__(self, domain=None, profile=None, logger=LOGGER, loglevel=CLIENT_INFO): """Define the desired compile lock with a domain and a host. A profile could be a host or a cluster. """ self.domain = domain self.profile = profile components = ["compile"] if self.domain: components.append(self.domain) if self.profile: components.append(self.profile) elif self.profile: raise InternalError("Compile lock request for %s missing domain." % self.profile) LockKey.__init__(self, components, logger=logger, loglevel=loglevel, lock_queue=lock_queue)
def get_user_principal(session, user): """Ignore the realm. This should probably be re-thought.""" dbusers = session.query(UserPrincipal).filter_by(name=user).all() if len(dbusers) > 1: raise InternalError("More than one user found for name %s" % user) if len(dbusers) == 0: raise NotFoundException("User '%s' not found." % user) return dbusers[0]
def get_network(cls, session, switch, vlan_id, compel=NotFoundException): q = session.query(cls).filter_by(switch=switch, vlan_id=vlan_id) nets = q.all() if not nets: raise compel("No network found for switch %s and VLAN %s" % (switch.fqdn, vlan_id)) if len(nets) > 1: raise InternalError("More than one network found for switch %s " "and VLAN %s" % (switch.fqdn, vlan_id)) return nets[0].network
def __init__(self, parent=None, vlan_id=None, **kwargs): if not parent: raise InternalError("VLAN interfaces need a parent.") if isinstance(parent, VlanInterface): raise ValueError("Stacking of VLAN interfaces is not allowed.") self.validate_vlan_id('vlan_id', vlan_id) super(VlanInterface, self).__init__(parent=parent, vlan_id=vlan_id, **kwargs)
def verify_init(self): """This is more of a verify-and-finalize method...""" for field in self.abstract_fields: if not hasattr(self, field): raise InternalError("%s provides no %s field" % (type(self.dbobj), field)) # This can be tweaked... if not self.required_only: for (service, instance) in self.original_service_instances.items(): self.staging_services[service] = [instance]
def __init__(self, dbhost, logger=LOGGER): if not isinstance(dbhost, Host): raise InternalError("PlenaryHost called with %s instead of Host" % dbhost.__class__.name) PlenaryCollection.__init__(self, logger=logger) self.dbobj = dbhost self.config = Config() if self.config.getboolean("broker", "namespaced_host_profiles"): self.plenaries.append(PlenaryNamespacedHost(dbhost)) if self.config.getboolean("broker", "flat_host_profiles"): self.plenaries.append(PlenaryToplevelHost(dbhost)) self.plenaries.append(PlenaryHostData(dbhost))
def verify_port_group(dbmachine, port_group): """Validate that the port_group can be used on an interface. If the machine is virtual, check that the corresponding VLAN has been observed on the cluster's switch. If the machine is physical but is part of an ESX cluster, also check that the VLAN has been observed. Otherwise just accept the label. As a convenience, return None (unset the port_group) if an empty string is passed in. """ if not port_group: return None session = object_session(dbmachine) dbvi = VlanInfo.get_unique(session, port_group=port_group, compel=True) if dbmachine.model.machine_type == "virtual_machine": dbswitch = dbmachine.cluster.switch if not dbswitch: raise ArgumentError("Cannot verify port group availability: no " "switch record for {0}.".format( dbmachine.cluster)) q = session.query(ObservedVlan) q = q.filter_by(vlan_id=dbvi.vlan_id) q = q.filter_by(switch=dbswitch) try: dbobserved_vlan = q.one() except NoResultFound: raise ArgumentError("Cannot verify port group availability: " "no record for VLAN {0} on " "{1:l}.".format(dbvi.vlan_id, dbswitch)) except MultipleResultsFound: # pragma: no cover raise InternalError("Too many subnets found for VLAN {0} " "on {1:l}.".format(dbvi.vlan_id, dbswitch)) if dbobserved_vlan.network.is_at_guest_capacity: raise ArgumentError("Port group {0} is full for " "{1:l}.".format(dbvi.port_group, dbobserved_vlan.switch)) elif dbmachine.host and dbmachine.host.cluster and \ dbmachine.host.cluster.switch: dbswitch = dbmachine.host.cluster.switch q = session.query(ObservedVlan) q = q.filter_by(vlan_id=dbvi.vlan_id, switch=dbswitch) if not q.count(): raise ArgumentError("VLAN {0} not found for " "{1:l}.".format(dbvi.vlan_id, dbswitch)) return dbvi.port_group
def acquire(self, key): if key is None: return key.transition("acquiring", debug=True) with self.queue_condition: if key in self.queue: raise InternalError("Duplicate attempt to aquire %s with the " "same key." % key) self.queue.append(key) while self.blocked(key): # pragma: no cover key.log("requesting %s with %s others waiting", key, key.blocker_count) self.queue_condition.wait() key.transition("acquired")
def poll_vlan(self, session, logger, switch, now, ssh_args): if not switch.primary_ip: raise ArgumentError("Cannot poll VLAN info for {0:l} without " "a registered IP address.".format(switch)) session.query(ObservedVlan).filter_by(switch=switch).delete() session.flush() # Restrict operations to the internal network dbnet_env = NetworkEnvironment.get_unique_or_default(session) args = [] if ssh_args: args.extend(ssh_args) args.append(self.config.get("broker", "vlan2net")) args.append("-ip") args.append(switch.primary_ip) out = run_command(args) try: reader = DictReader(StringIO(out)) for row in reader: vlan = row.get("vlan", None) network = row.get("network", None) bitmask = row.get("bitmask", None) if vlan is None or network is None or bitmask is None or \ len(vlan) == 0 or len(network) == 0 or len(bitmask) == 0: logger.info( "Missing value for vlan, network or bitmask in " "output line #%d: %s" % (reader.line_num, row)) continue try: vlan_int = int(vlan) except ValueError, e: logger.info("Error parsing vlan number in output " "line #%d: %s error: %s" % (reader.line_num, row, e)) continue try: network = force_ipv4("network", network) except ArgumentError, e: raise InternalError(e) try: bitmask_int = int(bitmask) except ValueError, e: logger.info("Error parsing bitmask in output " "line #%d: %s error: %s" % (reader.line_num, row, e)) continue
def __init__(self, network=None, network_type=None, **kw): # pylint: disable=W0621 if not isinstance(network, IPv4Network): raise InternalError("Expected an IPv4Network, got: %s" % type(network)) if not network_type: config = Config() network_type = config.get("broker", "default_network_type") self._network = network self._props = self.network_type_map.get(self.network_type, self.default_network_props) super(Network, self).__init__(ip=network.network, cidr=network.prefixlen, network_type=network_type, **kw)
def compile(self, session, only=None, locked=False, panc_debug_include=None, panc_debug_exclude=None, cleandeps=False): """The build directories are checked and constructed if necessary, so no prior setup is required. The compile may take some time (current rate is 10 hosts per second, with a couple of seconds of constant overhead), and the possibility of blocking on the compile lock. If the 'only' parameter is provided, then it should be a list or set containing the profiles that need to be compiled. May raise ArgumentError exception, else returns the standard output (as a string) of the compile """ config = Config() if self.domain.branch_type == 'sandbox': if not self.author: raise InternalError("Missing required author to compile " "sandbox %s" % self.domain.name) sandboxdir = os.path.join(config.get("broker", "templatesdir"), self.author.name, self.domain.name) if not os.path.exists(sandboxdir): raise ArgumentError("Sandbox directory '%s' does not exist." % sandboxdir) if not self.sandbox_has_latest(config, sandboxdir): self.logger.warn("Sandbox %s/%s does not contain the " "latest changes from the prod domain. If " "there are failures try " "`git fetch && git merge origin/prod`" % (self.author.name, self.domain.name)) self.logger.info("preparing domain %s for compile" % self.domain.name) # Ensure that the compile directory is in a good state. outputdir = config.get("broker", "profilesdir") for d in self.directories() + self.outputdirs(): if not os.path.exists(d): try: self.logger.info("creating %s" % d) os.makedirs(d) except OSError, e: raise ArgumentError("Failed to mkdir %s: %s" % (d, e))
def lock_row(self): """ Lock an object in the database. The function works by issuing a SELECT ... FOR UPDATE query. """ session = object_session(self) if not session: # pragma: no cover raise InternalError("lock_row() called on a detached object %r" % self) pk = inspect(self).mapper.primary_key q = session.query(*pk) for col in pk: q = q.filter(col == getattr(self, col.key)) q = q.with_lockmode("update") return q.one()
def add_action(self, command_args, rollback_args, error_filter=None, ignore_msg=False): """ Register an action to execute and it's rollback counterpart. command_args: the DSDB command to execute rollback_args: the DSDB command to execute on rollback error_filter: regexp of error messages in the output of dsdb that should be ignored ignore_msg: message to log if the error_filter matched """ if error_filter and not ignore_msg: raise InternalError("Specifying an error filter needs the message " "specified as well.") self.actions.append( (command_args, rollback_args, error_filter, ignore_msg))
def __init__(self, label=None, network=None, **kwargs): # This is dirty. We want to allow empty labels, but Oracle converts # empty strings to NULL, violating the NOT NULL constraint. We could # allow label to be NULL and relying on the unique indexes to forbid # adding multiple empty labels, but that is again Oracle-specific # behavior which actually violates the SQL standard, so it would not # work with other databases. if not label: label = '-' elif not self._label_check.match(label): # pragma: no cover raise ValueError("Illegal address label '%s'." % label) # Right now network_id is nullable due to how refresh_network works, so # verify the network here if not network: # pragma: no cover raise InternalError("AddressAssignment needs a network") super(AddressAssignment, self).__init__(_label=label, network=network, **kwargs)
def sandbox_has_latest(self, config, sandboxdir): domainsdir = config.get('broker', 'domainsdir') prod_domain = config.get('broker', 'default_domain_start') proddir = os.path.join(domainsdir, prod_domain) try: prod_commit = run_git(['rev-list', '-n', '1', 'HEAD'], path=proddir, logger=self.logger).strip() except ProcessException: prod_commit = '' if not prod_commit: raise InternalError("Error finding top commit for %s" % prod_domain) filterre = re.compile('^' + prod_commit + '$') try: found_latest = run_git(['rev-list', 'HEAD'], path=sandboxdir, logger=self.logger, filterre=filterre) except ProcessException: self.logger.warn("Failed to run git command in sandbox %s." % sandboxdir) found_latest = '' return bool(found_latest)
def check_ip_restrictions(dbnetwork, ip, relaxed=False): """ given a network and ip addr, raise an exception if the ip is reserved Used during ip assignment as a check against grabbing an ip address that we have reserved as a dynamic dhcp pool for switches (and potentially other assorted devices) The remainder of addresses are to be used for static assignment (for telco gear only). Setting relaxed to true means checking only the most obvious problems. """ #TODO: if the network type doesn't have any applicable offsets, we # probably want to reserve the first ip for the gateway on all networks if ip is None: # Simple passthrough to make calling logic easier. return if ip < dbnetwork.ip or ip > dbnetwork.broadcast: # pragma: no cover raise InternalError("IP address {0!s} is outside " "{1:l}.".format(ip, dbnetwork)) if dbnetwork.network.numhosts >= 4 and not relaxed: # Skip these checks for /32 and /31 networks if ip == dbnetwork.ip: raise ArgumentError("IP address %s is the address of network %s." % (ip, dbnetwork.name)) if ip == dbnetwork.broadcast: raise ArgumentError("IP address %s is the broadcast address of " "network %s." % (ip, dbnetwork.name)) if dbnetwork.network.numhosts >= 8 and not relaxed: # If this network doesn't have enough addresses, the test is irrelevant. if int(ip) - int(dbnetwork.ip) in dbnetwork.reserved_offsets: raise ArgumentError("The IP address %s is reserved for dynamic " "DHCP for a switch on subnet %s." % (ip, dbnetwork.ip)) return
def __init__(self, dbobj, *args, **kwargs): """Provide initialization specific for host bindings.""" if not isinstance(dbobj, Host): raise InternalError("HostChooser can only choose services for " "hosts, got %r (%s)" % (dbobj, type(dbobj))) self.dbhost = dbobj Chooser.__init__(self, dbobj, *args, **kwargs) self.location = self.dbhost.machine.location self.archetype = self.dbhost.archetype self.personality = self.dbhost.personality # If the primary name is a ReservedName, then it does not have a network # attribute if hasattr(self.dbhost.machine.primary_name, 'network'): self.network = self.dbhost.machine.primary_name.network else: self.network = None # all of them would be self. but that should be optimized # dbhost.machine.interfaces[x].assignments[y].network """Stores interim service instance lists.""" q = self.session.query(Service) q = q.outerjoin(Service.archetypes) q = q.reset_joinpoint() q = q.outerjoin(Service.personalities) q = q.filter( or_(Archetype.id == self.archetype.id, Personality.id == self.personality.id)) self.required_services = set(q.all()) self.original_service_instances = {} """Cache of any already bound services (keys) and the instance that was bound (values). """ q = self.session.query(ServiceInstance) q = q.options(undefer('_client_count')) q = q.filter(ServiceInstance.clients.contains(self.dbhost)) set_committed_value(self.dbhost, 'services_used', q.all()) for si in self.dbhost.services_used: self.original_service_instances[si.service] = si self.logger.debug("%s original binding: %s", self.description, si.cfg_path) self.cluster_aligned_services = {} if self.dbhost.cluster: # Note that cluster services are currently ignored unless # they are otherwise required by the archetype/personality. for si in self.dbhost.cluster.service_bindings: self.cluster_aligned_services[si.service] = si for service in self.dbhost.cluster.required_services: if service not in self.cluster_aligned_services: # Don't just error here because the error() call # has not yet been set up. Will error out later. self.cluster_aligned_services[service] = None # Went back and forth on this... deciding not to force # an aligned service as required. This should give # flexibility for multiple services to be aligned for # a cluster type without being forced on all the # personalities. #self.required_services.add(item.service) if self.dbhost.cluster.metacluster: mc = self.dbhost.cluster.metacluster for si in mc.service_bindings: if si.service in self.cluster_aligned_services: cas = self.cluster_aligned_services[si.service] if cas == None: # Error out later. continue self.logger.client_info( "Replacing {0.name} instance with {1.name} " "(bound to {2:l}) for service {3.name}".format( cas, si, mc, si.service)) self.cluster_aligned_services[si.service] = si for service in mc.required_services: if service not in self.cluster_aligned_services: # Don't just error here because the error() call # has not yet been set up. Will error out later. self.cluster_aligned_services[service] = None
def render(self, session, logger, feature, archetype, personality, model, vendor, interface, justification, user, **arguments): # Binding a feature to a named interface makes sense in the scope of a # personality, but not for a whole archetype. if interface and not personality: raise ArgumentError("Binding to a named interface needs " "a personality.") q = session.query(Personality) dbarchetype = None feature_type = "host" justification_required = True # Warning: order matters here! params = {} if personality: justification_required = False dbpersonality = Personality.get_unique(session, name=personality, archetype=archetype, compel=True) params["personality"] = dbpersonality if interface: params["interface_name"] = interface feature_type = "interface" dbarchetype = dbpersonality.archetype q = q.filter_by(archetype=dbarchetype) q = q.filter_by(name=personality) elif archetype: dbarchetype = Archetype.get_unique(session, archetype, compel=True) params["archetype"] = dbarchetype q = q.filter_by(archetype=dbarchetype) else: # It's highly unlikely that a feature template would work for # _any_ archetype, so disallow this case for now. As I can't # rule out that such a case will not have some uses in the # future, the restriction is here and not in the model. raise ArgumentError("Please specify either an archetype or " "a personality when binding a feature.") if model: dbmodel = Model.get_unique(session, name=model, vendor=vendor, compel=True) if dbmodel.machine_type == "nic": feature_type = "interface" else: feature_type = "hardware" params["model"] = dbmodel if dbarchetype and not dbarchetype.is_compileable: raise UnimplementedError("Binding features to non-compilable " "archetypes is not implemented.") if not feature_type: # pragma: no cover raise InternalError("Feature type is not known.") dbfeature = Feature.get_unique(session, name=feature, feature_type=feature_type, compel=True) cnt = q.count() # TODO: should the limit be configurable? if justification_required and cnt > 0: if not justification: raise AuthorizationException( "Changing feature bindings for more " "than just a personality requires --justification.") validate_justification(user, justification) self.do_link(session, logger, dbfeature, params) session.flush() idx = 0 written = 0 successful = [] failed = [] with CompileKey(logger=logger): personalities = q.all() for personality in personalities: idx += 1 if idx % 1000 == 0: # pragma: no cover logger.client_info("Processing personality %d of %d..." % (idx, cnt)) if not personality.archetype.is_compileable: # pragma: no cover continue try: plenary_personality = PlenaryPersonality(personality) written += plenary_personality.write(locked=True) successful.append(plenary_personality) except IncompleteError: pass except Exception, err: # pragma: no cover failed.append("{0} failed: {1}".format(personality, err)) if failed: # pragma: no cover for plenary in successful: plenary.restore_stash() raise PartialError([], failed)
def render(self, session, logger, dbuser, domain, track, start, change_manager, comments, allow_manage, **arguments): if not dbuser: raise AuthorizationException("Cannot create a domain without " "an authenticated connection.") Branch.get_unique(session, domain, preclude=True) valid = re.compile('^[a-zA-Z0-9_.-]+$') if (not valid.match(domain)): raise ArgumentError("Domain name '%s' is not valid." % domain) # FIXME: Verify that track is a valid branch name? # Or just let the branch command fail? compiler = self.config.get("panc", "pan_compiler") dbtracked = None if track: dbtracked = Branch.get_unique(session, track, compel=True) if getattr(dbtracked, "tracked_branch", None): raise ArgumentError("Cannot nest tracking. Try tracking " "{0:l} directly.".format( dbtracked.tracked_branch)) start_point = dbtracked if change_manager: raise ArgumentError("Cannot enforce a change manager for " "tracking domains.") else: if not start: start = self.config.get("broker", "default_domain_start") start_point = Branch.get_unique(session, start, compel=True) dbdomain = Domain(name=domain, owner=dbuser, compiler=compiler, tracked_branch=dbtracked, requires_change_manager=bool(change_manager), comments=comments) session.add(dbdomain) if allow_manage is not None: dbdomain.allow_manage = allow_manage session.flush() domainsdir = self.config.get("broker", "domainsdir") clonedir = os.path.join(domainsdir, dbdomain.name) if os.path.exists(clonedir): raise InternalError("Domain directory already exists") kingdir = self.config.get("broker", "kingdir") cmd = ["branch"] if track: cmd.append("--track") else: cmd.append("--no-track") cmd.append(dbdomain.name) cmd.append(start_point.name) run_git(cmd, path=kingdir, logger=logger) # If the branch command above fails the DB will roll back as normal. # If the command below fails we need to clean up from itself and above. try: run_git( ["clone", "--branch", dbdomain.name, kingdir, dbdomain.name], path=domainsdir, logger=logger) except ProcessException, e: try: remove_dir(clonedir, logger=logger) run_git(["branch", "-D", dbdomain.name], path=kingdir, logger=logger) except ProcessException, e2: logger.info("Exception while cleaning up: %s", e2)
def validate_link(self, key, link): if link.model: raise InternalError("Host features can not be bound to " "hardware models.") return link
def validate_link(self, key, link): if not link.model or link.model.machine_type == 'nic': raise InternalError("Hardware features can only be bound to " "machine models.") return link
def holder_object(self): # pragma: no cover raise InternalError("Abstract base method called")
def __enter__(self): if not self.lock_queue: # pragma: no cover raise InternalError("Using the 'with' statement requires a lock " "queue") self.lock_queue.acquire(self)