def render(self, session, logger, vlan, name, vlan_type, **kwargs): validate_basic("name", name) VlanInfo.get_unique(session, vlan_id=vlan, preclude=True) dbvlan = VlanInfo(vlan_id=vlan, port_group=name, vlan_type=vlan_type) session.add(dbvlan) session.flush() return
def render(self, session, logger, vlan, name, vlan_type, **kwargs): validate_nlist_key("name", name) VlanInfo.get_unique(session, vlan_id=vlan, preclude=True) dbvlan = VlanInfo(vlan_id=vlan, port_group=name, vlan_type=vlan_type) session.add(dbvlan) session.flush() return
def render(self, session, logger, vlan, **arguments): dbvlan = VlanInfo.get_unique(session, vlan_id=vlan, compel=True) q = session.query(ObservedVlan) q = q.filter_by(vlan=dbvlan) if q.first(): raise ArgumentError("{0} is still in use and cannot be " "deleted.".format(dbvlan)) session.delete(dbvlan) return
def verify_port_group(dbmachine, port_group): """Validate that the port_group can be used on an interface. If the machine is virtual, check that the corresponding VLAN has been observed on the cluster's switch. If the machine is physical but is part of an ESX cluster, also check that the VLAN has been observed. Otherwise just accept the label. As a convenience, return None (unset the port_group) if an empty string is passed in. """ if not port_group: return None session = object_session(dbmachine) dbvi = VlanInfo.get_unique(session, port_group=port_group, compel=True) if dbmachine.model.machine_type == "virtual_machine": dbswitch = dbmachine.cluster.switch if not dbswitch: raise ArgumentError("Cannot verify port group availability: no " "switch record for {0}.".format( dbmachine.cluster)) q = session.query(ObservedVlan) q = q.filter_by(vlan_id=dbvi.vlan_id) q = q.filter_by(switch=dbswitch) try: dbobserved_vlan = q.one() except NoResultFound: raise ArgumentError("Cannot verify port group availability: " "no record for VLAN {0} on " "{1:l}.".format(dbvi.vlan_id, dbswitch)) except MultipleResultsFound: # pragma: no cover raise InternalError("Too many subnets found for VLAN {0} " "on {1:l}.".format(dbvi.vlan_id, dbswitch)) if dbobserved_vlan.network.is_at_guest_capacity: raise ArgumentError("Port group {0} is full for " "{1:l}.".format(dbvi.port_group, dbobserved_vlan.switch)) elif dbmachine.host and dbmachine.host.cluster and \ dbmachine.host.cluster.switch: dbswitch = dbmachine.host.cluster.switch q = session.query(ObservedVlan) q = q.filter_by(vlan_id=dbvi.vlan_id, switch=dbswitch) if not q.count(): raise ArgumentError("VLAN {0} not found for " "{1:l}.".format(dbvi.vlan_id, dbswitch)) return dbvi.port_group
def verify_port_group(dbmachine, port_group): """Validate that the port_group can be used on an interface. If the machine is virtual, check that the corresponding VLAN has been observed on the cluster's switch. If the machine is physical but is part of an ESX cluster, also check that the VLAN has been observed. Otherwise just accept the label. As a convenience, return None (unset the port_group) if an empty string is passed in. """ if not port_group: return None session = object_session(dbmachine) dbvi = VlanInfo.get_unique(session, port_group=port_group, compel=True) if dbmachine.model.machine_type == "virtual_machine": dbswitch = dbmachine.cluster.switch if not dbswitch: raise ArgumentError("Cannot verify port group availability: no " "switch record for {0}.".format(dbmachine.cluster)) q = session.query(ObservedVlan) q = q.filter_by(vlan_id=dbvi.vlan_id) q = q.filter_by(switch=dbswitch) try: dbobserved_vlan = q.one() except NoResultFound: raise ArgumentError("Cannot verify port group availability: " "no record for VLAN {0} on " "{1:l}.".format(dbvi.vlan_id, dbswitch)) except MultipleResultsFound: # pragma: no cover raise InternalError("Too many subnets found for VLAN {0} " "on {1:l}.".format(dbvi.vlan_id, dbswitch)) if dbobserved_vlan.network.is_at_guest_capacity: raise ArgumentError("Port group {0} is full for " "{1:l}.".format(dbvi.port_group, dbobserved_vlan.switch)) elif dbmachine.host and dbmachine.host.cluster and \ dbmachine.host.cluster.switch: dbswitch = dbmachine.host.cluster.switch q = session.query(ObservedVlan) q = q.filter_by(vlan_id=dbvi.vlan_id, switch=dbswitch) if not q.count(): raise ArgumentError("VLAN {0} not found for " "{1:l}.".format(dbvi.vlan_id, dbswitch)) return dbvi.port_group
continue dbnetwork = Network.get_unique(session, ip=network, network_environment=dbnet_env) if not dbnetwork: logger.info("Unknown network %s in output line #%d: %s" % (network, reader.line_num, row)) continue if dbnetwork.cidr != bitmask_int: logger.client_info("{0}: skipping VLAN {1}, because network " "bitmask value {2} differs from prefixlen " "{3.cidr} of {3:l}.".format(switch, vlan, bitmask, dbnetwork)) continue vlan_info = VlanInfo.get_unique(session, vlan_id=vlan_int, compel=False) if not vlan_info: logger.client_info("vlan {0} is not defined in AQ. Please " "use add_vlan to add it.".format(vlan_int)) continue if vlan_info.vlan_type == "unknown": continue dbvlan = ObservedVlan(vlan_id=vlan_int, switch=switch, network=dbnetwork, creation_date=now) session.add(dbvlan) except CSVError, e: raise AquilonError("Error parsing vlan2net results: %s" % e)
def generate_ip(session, logger, dbinterface, ip=None, ipfromip=None, ipfromsystem=None, autoip=None, ipalgorithm=None, compel=False, network_environment=None, audit_results=None, **kwargs): ip_options = [ip, ipfromip, ipfromsystem, autoip] numopts = sum([1 if opt else 0 for opt in ip_options]) if numopts > 1: raise ArgumentError("Only one of --ip, --ipfromip, --ipfromsystem " "and --autoip can be specified.") elif numopts == 0: if compel: raise ArgumentError("Please specify one of the --ip, --ipfromip, " "--ipfromsystem, and --autoip parameters.") return None if ip: return ip dbsystem = None dbnetwork = None if autoip: if not dbinterface: raise ArgumentError("No interface available to automatically " "generate an IP address.") if dbinterface.port_group: # This could either be an interface from a virtual machine # or an interface on an ESX vmhost. dbcluster = None if getattr(dbinterface.hardware_entity, "cluster", None): # VM dbcluster = dbinterface.hardware_entity.cluster elif getattr(dbinterface.hardware_entity, "host", None): dbcluster = dbinterface.hardware_entity.host.cluster if not dbcluster: raise ArgumentError("Can only automatically assign an IP " "address to an interface with a port " "group on virtual machines or ESX hosts.") if not dbcluster.switch: raise ArgumentError("Cannot automatically assign an IP " "address to an interface with a port group " "since {0} is not associated with a " "switch.".format(dbcluster)) vlan_id = VlanInfo.get_vlan_id(session, dbinterface.port_group) dbnetwork = ObservedVlan.get_network(session, vlan_id=vlan_id, switch=dbcluster.switch, compel=ArgumentError) elif dbinterface.mac: q = session.query(ObservedMac) q = q.filter_by(mac_address=dbinterface.mac) q = q.order_by(desc(ObservedMac.last_seen)) dbom = q.first() if not dbom: raise ArgumentError("No switch found in the discovery table " "for MAC address %s." % dbinterface.mac) if not dbom.switch.primary_ip: raise ArgumentError("{0} does not have a primary IP address " "to use for network " "selection.".format(dbom.switch)) dbnetwork = get_net_id_from_ip(session, dbom.switch.primary_ip) else: raise ArgumentError("{0} has neither a MAC address nor port group " "information, it is not possible to generate " "an IP address automatically." .format(dbinterface)) if ipfromsystem: # Assumes one system entry, not necessarily correct. dbdns_rec = ARecord.get_unique(session, fqdn=ipfromsystem, compel=True) dbnetwork = dbdns_rec.network if ipfromip: # determine network dbnetwork = get_net_id_from_ip(session, ipfromip, network_environment) if not dbnetwork: raise ArgumentError("Could not determine network to use for %s." % dbsystem.fqdn) # When there are e.g. multiple "add manager --autoip" operations going on in # parallel, we must ensure that they won't try to use the same IP address. # This query places a database lock on the network, which means IP address # generation within a network will be serialized, while operations on # different networks can still run in parallel. The lock will be released by # COMMIT or ROLLBACK. dbnetwork.lock_row() startip = dbnetwork.first_usable_host used_ips = session.query(ARecord.ip) used_ips = used_ips.filter_by(network=dbnetwork) used_ips = used_ips.filter(ARecord.ip >= startip) full_set = set(range(int(startip), int(dbnetwork.broadcast))) used_set = set([int(item.ip) for item in used_ips]) free_set = full_set - used_set if not free_set: raise ArgumentError("No available IP addresses found on " "network %s." % str(dbnetwork.network)) if ipalgorithm is None or ipalgorithm == 'lowest': # Select the lowest available address ip = IPv4Address(min(free_set)) elif ipalgorithm == 'highest': # Select the highest available address ip = IPv4Address(max(free_set)) elif ipalgorithm == 'max': # Return the max. used address + 1 if not used_set: # Avoids ValueError being thrown when used_set is empty ip = IPv4Address(min(free_set)) else: next = max(used_set) if not next + 1 in free_set: raise ArgumentError("Failed to find an IP that is suitable " "for --ipalgorithm=max. Try an other " "algorithm as there are still some free " "addresses.") ip = IPv4Address(next + 1) else: raise ArgumentError("Unknown algorithm %s." % ipalgorithm) if audit_results is not None: if dbinterface: logger.info("Selected IP address {0!s} for {1:l}" .format(ip, dbinterface)) else: logger.info("Selected IP address %s" % ip) audit_results.append(('ip', ip)) return ip
ip=network, network_environment=dbnet_env) if not dbnetwork: logger.info("Unknown network %s in output line #%d: %s" % (network, reader.line_num, row)) continue if dbnetwork.cidr != bitmask_int: logger.client_info( "{0}: skipping VLAN {1}, because network " "bitmask value {2} differs from prefixlen " "{3.cidr} of {3:l}.".format(switch, vlan, bitmask, dbnetwork)) continue vlan_info = VlanInfo.get_unique(session, vlan_id=vlan_int, compel=False) if not vlan_info: logger.client_info( "vlan {0} is not defined in AQ. Please " "use add_vlan to add it.".format(vlan_int)) continue if vlan_info.vlan_type == "unknown": continue dbvlan = ObservedVlan(vlan_id=vlan_int, switch=switch, network=dbnetwork, creation_date=now) session.add(dbvlan)
def render(self, session, network, network_environment, ip, type, side, machine, fqdn, cluster, pg, has_dynamic_ranges, fullinfo, **arguments): """Return a network matching the parameters. Some of the search terms can only return a unique network. For those (like ip and fqdn) we proceed with the query anyway. This allows for quick scripted tests like "is the network for X.X.X.X a tor_net2?". """ dbnet_env = NetworkEnvironment.get_unique_or_default( session, network_environment) q = session.query(Network) q = q.filter_by(network_environment=dbnet_env) if network: # Note: the network name is not unique (neither in QIP) q = q.filter_by(name=network) if ip: dbnetwork = get_net_id_from_ip(session, ip, dbnet_env) q = q.filter_by(id=dbnetwork.id) if type: q = q.filter_by(network_type=type) if side: q = q.filter_by(side=side) if machine: dbmachine = Machine.get_unique(session, machine, compel=True) vlans = [] if dbmachine.cluster and dbmachine.cluster.switch: # If this is a VM on a cluster, consult the VLANs. There # could be functionality here for real hardware to consult # interface port groups... there's no real use case yet. vlans = [ VlanInfo.get_vlan_id(session, i.port_group) for i in dbmachine.interfaces if i.port_group ] if vlans: q = q.join('observed_vlans') q = q.filter_by(switch=dbmachine.cluster.switch) q = q.filter(ObservedVlan.vlan_id.in_(vlans)) q = q.reset_joinpoint() if not vlans: networks = [ addr.network.id for addr in dbmachine.all_addresses() ] if not networks: msg = "Machine %s has no interfaces " % dbmachine.label if dbmachine.cluster: msg += "with a portgroup or " msg += "assigned to a network." raise ArgumentError(msg) q = q.filter(Network.id.in_(networks)) if fqdn: (short, dbdns_domain) = parse_fqdn(session, fqdn) dnsq = session.query(ARecord.ip) dnsq = dnsq.join(ARecord.fqdn) dnsq = dnsq.filter_by(name=short) dnsq = dnsq.filter_by(dns_domain=dbdns_domain) networks = [ get_net_id_from_ip(session, addr.ip, dbnet_env).id for addr in dnsq.all() ] q = q.filter(Network.id.in_(networks)) if cluster: dbcluster = Cluster.get_unique(session, cluster, compel=True) if dbcluster.switch: q = q.join('observed_vlans') q = q.filter_by(switch=dbcluster.switch) q = q.reset_joinpoint() else: net_ids = [ h.machine.primary_name.network.id for h in dbcluster.hosts if getattr(h.machine.primary_name, "network") ] q = q.filter(Network.id.in_(net_ids)) if pg: vlan = VlanInfo.get_vlan_id(session, pg, compel=ArgumentError) q = q.join('observed_vlans') q = q.filter_by(vlan_id=vlan) q = q.reset_joinpoint() dblocation = get_location(session, **arguments) if dblocation: if arguments.get('exact_location'): q = q.filter_by(location=dblocation) else: childids = dblocation.offspring_ids() q = q.filter(Network.location_id.in_(childids)) if has_dynamic_ranges: q = q.filter( exists([DynamicStub.dns_record_id], from_obj=DynamicStub.__table__.join( ARecord.__table__)).where( Network.id == DynamicStub.network_id)) q = q.order_by(Network.ip) if fullinfo: q = q.options(undefer('comments')) return q.all() return ShortNetworkList(q.all())
def render(self, session, network, network_environment, ip, type, side, machine, fqdn, cluster, pg, has_dynamic_ranges, exact_location, fullinfo, style, **arguments): """Return a network matching the parameters. Some of the search terms can only return a unique network. For those (like ip and fqdn) we proceed with the query anyway. This allows for quick scripted tests like "is the network for X.X.X.X a tor_net2?". """ dbnet_env = NetworkEnvironment.get_unique_or_default(session, network_environment) q = session.query(Network) q = q.filter_by(network_environment=dbnet_env) if network: # Note: the network name is not unique (neither in QIP) q = q.filter_by(name=network) if ip: dbnetwork = get_net_id_from_ip(session, ip, dbnet_env) q = q.filter_by(id=dbnetwork.id) if type: q = q.filter_by(network_type=type) if side: q = q.filter_by(side=side) if machine: dbmachine = Machine.get_unique(session, machine, compel=True) vlans = [] if dbmachine.cluster and dbmachine.cluster.network_device: # If this is a VM on a cluster, consult the VLANs. There # could be functionality here for real hardware to consult # interface port groups... there's no real use case yet. vlans = [VlanInfo.get_vlan_id(session, i.port_group) for i in dbmachine.interfaces if i.port_group] if vlans: q = q.join('observed_vlans') q = q.filter_by(network_device=dbmachine.cluster.network_device) q = q.filter(ObservedVlan.vlan_id.in_(vlans)) q = q.reset_joinpoint() if not vlans: networks = [addr.network.id for addr in dbmachine.all_addresses()] if not networks: msg = "Machine %s has no interfaces " % dbmachine.label if dbmachine.cluster: msg += "with a portgroup or " msg += "assigned to a network." raise ArgumentError(msg) q = q.filter(Network.id.in_(networks)) if fqdn: (short, dbdns_domain) = parse_fqdn(session, fqdn) dnsq = session.query(ARecord.ip) dnsq = dnsq.join(ARecord.fqdn) dnsq = dnsq.filter_by(name=short) dnsq = dnsq.filter_by(dns_domain=dbdns_domain) networks = [get_net_id_from_ip(session, addr.ip, dbnet_env).id for addr in dnsq.all()] q = q.filter(Network.id.in_(networks)) if cluster: dbcluster = Cluster.get_unique(session, cluster, compel=True) if dbcluster.network_device: q = q.join('observed_vlans') q = q.filter_by(network_device=dbcluster.network_device) q = q.reset_joinpoint() else: net_ids = [h.hardware_entity.primary_name.network.id for h in dbcluster.hosts if getattr(h.hardware_entity.primary_name, "network")] q = q.filter(Network.id.in_(net_ids)) if pg: vlan = VlanInfo.get_vlan_id(session, pg, compel=ArgumentError) q = q.join('observed_vlans') q = q.filter_by(vlan_id=vlan) q = q.reset_joinpoint() dblocation = get_location(session, **arguments) if dblocation: if exact_location: q = q.filter_by(location=dblocation) else: childids = dblocation.offspring_ids() q = q.filter(Network.location_id.in_(childids)) if has_dynamic_ranges: q = q.filter(exists([DynamicStub.dns_record_id], from_obj=DynamicStub.__table__.join(ARecord.__table__)) .where(Network.id == DynamicStub.network_id)) q = q.order_by(Network.ip) if fullinfo or style != 'raw': q = q.options(undefer('comments')) return q.all() return StringAttributeList(q.all(), lambda n: "%s/%s" % (n.ip, n.cidr))
def generate_ip(session, logger, dbinterface, ip=None, ipfromip=None, ipfromsystem=None, autoip=None, ipalgorithm=None, compel=False, network_environment=None, audit_results=None, **kwargs): ip_options = [ip, ipfromip, ipfromsystem, autoip] numopts = sum([1 if opt else 0 for opt in ip_options]) if numopts > 1: raise ArgumentError("Only one of --ip, --ipfromip, --ipfromsystem " "and --autoip can be specified.") elif numopts == 0: if compel: raise ArgumentError("Please specify one of the --ip, --ipfromip, " "--ipfromsystem, and --autoip parameters.") return None if ip: return ip dbsystem = None dbnetwork = None if autoip: if not dbinterface: raise ArgumentError("No interface available to automatically " "generate an IP address.") if dbinterface.port_group: # This could either be an interface from a virtual machine # or an interface on an ESX vmhost. dbcluster = None if getattr(dbinterface.hardware_entity, "cluster", None): # VM dbcluster = dbinterface.hardware_entity.cluster elif getattr(dbinterface.hardware_entity, "host", None): dbcluster = dbinterface.hardware_entity.host.cluster if not dbcluster: raise ArgumentError("Can only automatically assign an IP " "address to an interface with a port " "group on virtual machines or ESX hosts.") if not dbcluster.switch: raise ArgumentError( "Cannot automatically assign an IP " "address to an interface with a port group " "since {0} is not associated with a " "switch.".format(dbcluster)) vlan_id = VlanInfo.get_vlan_id(session, dbinterface.port_group) dbnetwork = ObservedVlan.get_network(session, vlan_id=vlan_id, switch=dbcluster.switch, compel=ArgumentError) elif dbinterface.mac: q = session.query(ObservedMac) q = q.filter_by(mac_address=dbinterface.mac) q = q.order_by(desc(ObservedMac.last_seen)) dbom = q.first() if not dbom: raise ArgumentError("No switch found in the discovery table " "for MAC address %s." % dbinterface.mac) if not dbom.switch.primary_ip: raise ArgumentError("{0} does not have a primary IP address " "to use for network " "selection.".format(dbom.switch)) dbnetwork = get_net_id_from_ip(session, dbom.switch.primary_ip) else: raise ArgumentError( "{0} has neither a MAC address nor port group " "information, it is not possible to generate " "an IP address automatically.".format(dbinterface)) if ipfromsystem: # Assumes one system entry, not necessarily correct. dbdns_rec = ARecord.get_unique(session, fqdn=ipfromsystem, compel=True) dbnetwork = dbdns_rec.network if ipfromip: # determine network dbnetwork = get_net_id_from_ip(session, ipfromip, network_environment) if not dbnetwork: raise ArgumentError("Could not determine network to use for %s." % dbsystem.fqdn) # When there are e.g. multiple "add manager --autoip" operations going on in # parallel, we must ensure that they won't try to use the same IP address. # This query places a database lock on the network, which means IP address # generation within a network will be serialized, while operations on # different networks can still run in parallel. The lock will be released by # COMMIT or ROLLBACK. dbnetwork.lock_row() startip = dbnetwork.first_usable_host used_ips = session.query(ARecord.ip) used_ips = used_ips.filter_by(network=dbnetwork) used_ips = used_ips.filter(ARecord.ip >= startip) full_set = set(range(int(startip), int(dbnetwork.broadcast))) used_set = set([int(item.ip) for item in used_ips]) free_set = full_set - used_set if not free_set: raise ArgumentError("No available IP addresses found on " "network %s." % str(dbnetwork.network)) if ipalgorithm is None or ipalgorithm == 'lowest': # Select the lowest available address ip = IPv4Address(min(free_set)) elif ipalgorithm == 'highest': # Select the highest available address ip = IPv4Address(max(free_set)) elif ipalgorithm == 'max': # Return the max. used address + 1 if not used_set: # Avoids ValueError being thrown when used_set is empty ip = IPv4Address(min(free_set)) else: next = max(used_set) if not next + 1 in free_set: raise ArgumentError("Failed to find an IP that is suitable " "for --ipalgorithm=max. Try an other " "algorithm as there are still some free " "addresses.") ip = IPv4Address(next + 1) else: raise ArgumentError("Unknown algorithm %s." % ipalgorithm) if audit_results is not None: if dbinterface: logger.info("Selected IP address {0!s} for {1:l}".format( ip, dbinterface)) else: logger.info("Selected IP address %s" % ip) audit_results.append(('ip', ip)) return ip