def __init__(self, parent=None, name=None, fullname=None, **kwargs): # Keep compatibility with the old behavior of the "parent" attribute # when creating new objects. Note that both the location manipulation # commands and the data loader in the unittest suite depends on this. if parent is not None: if parent.__class__ not in self.valid_parents: raise AquilonError( "{0} cannot be a parent of {1:lc} {2}.".format( parent, self, name)) session = object_session(parent) if not session: raise AquilonError("The parent must be persistent") # We have to disable autoflush in case parent._parent_links needs # loading, since self is not ready to be pushed to the DB yet with session.no_autoflush: for link in parent._parent_links: session.add( LocationLink(child=self, parent=link.parent, distance=link.distance + 1)) session.add(LocationLink(child=self, parent=parent, distance=1)) session.expire(parent, ["_child_links", "children"]) if not fullname: fullname = name super(Location, self).__init__(name=name, fullname=fullname, **kwargs) self._parent_dict = None
def __init__(self, service, protocol, dns_domain, dns_environment, priority, weight, port, target, **kwargs): if not isinstance(target, Fqdn): # pragma: no cover raise TypeError("The target of an SRV record must be an Fqdn.") session = object_session(target) if not session: # pragma: no cover raise AquilonError("The target name must already be part of " "the session.") # SRV records are special, as the FQDN is managed internally if "fqdn" in kwargs: # pragma: no cover raise AquilonError("SRV records do not accept an FQDN argument.") self.validate_ushort('priority', priority) self.validate_ushort('weight', weight) self.validate_ushort('port', port) # RFC 2782: # - there must be one or more address records for the target # - the target must not be an alias found_address = False for rr in target.dns_records: if isinstance(rr, ARecord): found_address = True elif isinstance(rr, Alias): raise ArgumentError("The target of an SRV record must not be " "an alias.") if not found_address: raise ArgumentError("The target of an SRV record must resolve to " "one or more addresses.") if protocol not in PROTOCOLS: raise ArgumentError("Unknown protocol %s." % protocol) name = "_%s._%s" % (service.strip().lower(), protocol.strip().lower()) # Disable autoflush because self is not ready to be pushed to the DB yet with session.no_autoflush: fqdn = Fqdn.get_or_create(session, name=name, dns_domain=dns_domain, dns_environment=dns_environment, ignore_name_check=True) # Do not allow two SRV records pointing at the same target for rr in fqdn.dns_records: if isinstance(rr, SrvRecord) and rr.target == target and \ rr.protocol == protocol and rr.service == service: raise ArgumentError("{0} already exists.".format(rr)) super(SrvRecord, self).__init__(fqdn=fqdn, priority=priority, weight=weight, port=port, target=target, **kwargs)
def update_parent(self, parent=None): session = object_session(self) if parent is None: # pragma: no cover raise AquilonError( "Parent location can be updated but not removed") if parent.__class__ not in self.valid_parents: raise AquilonError("{0} cannot be a parent of {1:l}.".format( parent, self)) # Disable autoflush. We'll make use of SQLA's ability to replace # DELETE + INSERT for the same LocationLink with an UPDATE of the # distance column. with session.no_autoflush: # Delete links to our old parent and its ancestors for plink in self._parent_links: q = session.query(LocationLink) q = q.filter( and_(LocationLink.child_id.in_(self.offspring_ids()), LocationLink.parent_id == plink.parent.id)) # See above: we depend on the caching ability of the session, so # we can't use q.delete() for clink in q.all(): session.delete(clink) # Add links to the new parent session.add(LocationLink(child=self, parent=parent, distance=1)) for clink in self._child_links: session.add( LocationLink(child_id=clink.child_id, parent=parent, distance=clink.distance + 1)) # Add links to the new parent's ancestors for plink in parent._parent_links: session.add( LocationLink(child=self, parent_id=plink.parent_id, distance=plink.distance + 1)) for clink in self._child_links: session.add( LocationLink(child_id=clink.child_id, parent_id=plink.parent_id, distance=plink.distance + clink.distance + 1)) session.flush() session.expire(parent, ["_child_links", "children"]) session.expire(self, ["_parent_links", "parent", "parents"]) self._parent_dict = None
def login(self, passwds): errs = [] pswd_re = re.compile('PASSWORD') dsn_copy = self.dsn if not passwds: raise AquilonError("At least one password must be specified, even " "if that's empty.") for p in passwds: self.dsn = re.sub(pswd_re, p, dsn_copy) self.engine = create_engine(self.dsn, **self.pool_options) self.no_lock_engine = create_engine(self.dsn, **self.pool_options) # Events should be registered before we try to open a real # connection below, because the underlying DBAPI connection will not # be closed if self.engine.dialect.name == "oracle": event.listen(self.engine, "connect", oracle_set_module) event.listen(self.no_lock_engine, "connect", oracle_set_module) event.listen(self.engine, "checkin", oracle_reset_action) event.listen(self.no_lock_engine, "checkin", oracle_reset_action) try: connection = self.engine.connect() connection.close() return except SaDBError, e: errs.append(e)
def deprecated_command(cls, msg, logger=None, user=None, **kwargs): if not logger or not user: # pragma: no cover raise AquilonError("Too few arguments to deprecated_command") # cls.__name__ is good enough to mine the logs which deprecated commands # are still in use. logger.info("User %s invoked deprecated command %s" % (user, cls.__name__)) logger.client_info(msg)
def __init__(self, action, options): m = getattr(self, action, None) if not m: raise AquilonError("Internal Error: Unknown action '%s' attempted" % action) self.run = m # Propagate some options to subprocesses self.env = os.environ.copy() self.env["AQHOST"] = options.get("aqhost") self.env["AQSERVICE"] = options.get("aqservice") self.env["AQPORT"] = str(options.get("aqport"))
def update_primary_ip(session, dbhw_ent, ip): if not dbhw_ent.primary_name: raise ArgumentError( "{0} does not have a primary name.".format(dbhw_ent)) dbnetwork = get_net_id_from_ip(session, ip) check_ip_restrictions(dbnetwork, ip) # The primary address must be unique q = session.query(AddressAssignment) q = q.filter_by(network=dbnetwork) q = q.filter_by(ip=ip) addr = q.first() if addr: raise ArgumentError( "IP address {0} is already in use by {1:l}.".format( ip, addr.interface)) # Convert ReservedName to ARecord if needed if isinstance(dbhw_ent.primary_name, ReservedName): convert_reserved_to_arecord(session, dbhw_ent.primary_name, dbnetwork, ip) # When converting a ReservedName to an ARecord, we have to bind the # primary address to an interface. Try to pick one. dbinterface = first_of(dbhw_ent.interfaces, lambda x: x.bootable) if not dbinterface: dbinterface = first_of(dbhw_ent.interfaces, lambda x: x.interface_type != "management") if not dbinterface: # pragma: no cover raise AquilonError("Cannot update the primary IP address of {0:l} " "because it does not have any interfaces " "defined.".format(dbhw_ent)) assign_address(dbinterface, ip, dbnetwork) else: dns_rec = dbhw_ent.primary_name q = session.query(AddressAssignment) q = q.filter_by(network=dns_rec.network) q = q.filter_by(ip=dns_rec.ip) q = q.join(Interface) q = q.filter_by(hardware_entity=dbhw_ent) # In case of Zebra, the address may be assigned to multiple interfaces addrs = q.all() dns_rec.ip = ip dns_rec.network = dbnetwork for addr in addrs: addr.ip = ip addr.network = dbnetwork
def _get_password_list(self): """ Read a password file containing one password per line. The passwords will be tried in the order they appear in the file. """ # Default: no password if not config.has_option("database", "password_file") or \ not config.get("database", "password_file").strip(): return [""] passwd_file = config.get("database", "password_file") if not os.path.exists(passwd_file): raise AquilonError("The password file '%s' does not exist." % passwd_file) passwds = "" with open(passwd_file) as f: passwds = f.readlines() if not passwds: raise AquilonError("Password file %s is empty." % passwd_file) return [passwd.strip() for passwd in passwds]
def deprecated_option(cls, option, msg="", logger=None, user=None, **kwargs): if not option or not logger or not user: # pragma: no cover raise AquilonError("Too few arguments to deprecated_option") # cls.__name__ is good enough to mine the logs which deprecated options # are still in use. logger.info("User %s used deprecated option %s of command %s" % (user, option, cls.__name__)) logger.client_info("The --%s option is deprecated. %s" % (option, msg))
def __init__(self, defaults=global_defaults, configfile=None): self.__dict__ = self.__shared_state if getattr(self, "baseconfig", None): if not configfile or self.baseconfig == os.path.realpath( configfile): return raise AquilonError( "Could not configure with %s, already configured with %s" % (configfile, self.baseconfig)) # This is a small race condition here... baseconfig could be # checked here, pre-empted, checked again elsewhere, and also # get here. If that ever happens, it is only a problem if one # passed in a configfile and the other didn't. Punting for now. if configfile: self.baseconfig = os.path.realpath(configfile) else: self.baseconfig = os.path.realpath( os.environ.get("AQDCONF", "/etc/aqd.conf")) SafeConfigParser.__init__(self, defaults) src_defaults = os.path.join(defaults["srcdir"], "etc", "aqd.conf.defaults") read_files = self.read([src_defaults, self.baseconfig]) for file in [src_defaults, self.baseconfig]: if file not in read_files: raise AquilonError("Could not read configuration file %s." % file) # Allow a section to "pull in" another section, as though all the # values defined in the alternate were actually defined there. for section in self.sections(): section_option = "%s_section" % section if self.has_option(section, section_option): alternate_section = self.get(section, section_option) if self.has_section(alternate_section): for (name, value) in self.items(alternate_section): self.set(section, name, value)
def main(): logging.basicConfig(level=logging.DEBUG) query = session.query(Resource) old_paths = [] with CompileKey(): for res in query.all(): PlenaryResource(res).write(locked=True) holder = res.holder.holder_object if isinstance(holder, ResourceGroup): holder = holder.holder.holder_object else: old_paths.append("resource/%s/%s/%s/%s" % (res.resource_type, res.holder.holder_type, res.holder.holder_name, res.name)) try: # Show that something is happening... print "Flushing {0:l}".format(holder) if isinstance(holder, Host): PlenaryHost(holder).write(locked=True) elif isinstance(holder, Cluster): PlenaryCluster(holder).write(locked=True) else: raise AquilonError("Unknown holder object: %r" % holder) except IncompleteError: pass plenarydir = config.get("broker", "plenarydir") for path in old_paths: try: os.remove(os.path.join(plenarydir, path, "config.tpl")) except OSError: pass try: os.removedirs(os.path.join(plenarydir, path)) except OSError: pass
class DbFactory(object): __shared_state = {} __started = False # at the class definition, that is def __init__(self, *args, **kw): self.__dict__ = self.__shared_state if self.__started: return self.__started = True self.dsn = config.get('database', 'dsn') self.pool_options = {} self.pool_options["pool_size"] = config.getint("database", "pool_size") self.pool_options["max_overflow"] = config.getint( "database", "pool_max_overflow") if len(config.get("database", "pool_timeout").strip()) > 0: self.pool_options["pool_timeout"] = config.getint( "database", "pool_timeout") else: self.pool_options["pool_timeout"] = None log = logging.getLogger('aqdb.db_factory') log.info("Database engine using pool options %s" % self.pool_options) passwds = self._get_password_list() # ORACLE if self.dsn.startswith('oracle'): import cx_Oracle # pylint: disable=W0612 self.login(passwds) # POSTGRESQL elif self.dsn.startswith('postgresql'): import psycopg2 # pylint: disable=W0612 self.login(passwds) # SQLITE elif self.dsn.startswith('sqlite'): self.engine = create_engine(self.dsn) self.no_lock_engine = None event.listen(self.engine, "connect", sqlite_foreign_keys) if config.has_option("database", "disable_fsync") and \ config.getboolean("database", "disable_fsync"): event.listen(self.engine, "connect", sqlite_no_fsync) log = logging.getLogger('aqdb.db_factory') log.info("SQLite is operating in unsafe mode!") connection = self.engine.connect() connection.close() else: msg = "Supported database datasources are postgresql, oracle and sqlite.\n" msg += "yours is '%s' " % self.dsn sys.stderr.write(msg) sys.exit(9) self.meta = MetaData(self.engine) assert self.meta self.Session = scoped_session(sessionmaker(bind=self.engine)) assert self.Session # For database types that support concurrent connections, we # create a separate thread pool for connections that promise # not to wait on locks. if self.no_lock_engine: self.NLSession = scoped_session( sessionmaker(bind=self.no_lock_engine)) else: self.NLSession = self.Session def login(self, passwds): errs = [] pswd_re = re.compile('PASSWORD') dsn_copy = self.dsn if not passwds: raise AquilonError("At least one password must be specified, even " "if that's empty.") for p in passwds: self.dsn = re.sub(pswd_re, p, dsn_copy) self.engine = create_engine(self.dsn, **self.pool_options) self.no_lock_engine = create_engine(self.dsn, **self.pool_options) # Events should be registered before we try to open a real # connection below, because the underlying DBAPI connection will not # be closed if self.engine.dialect.name == "oracle": event.listen(self.engine, "connect", oracle_set_module) event.listen(self.no_lock_engine, "connect", oracle_set_module) event.listen(self.engine, "checkin", oracle_reset_action) event.listen(self.no_lock_engine, "checkin", oracle_reset_action) try: connection = self.engine.connect() connection.close() return except SaDBError, e: errs.append(e) if errs: raise errs.pop() else: raise AquilonError('Failed to connect to %s')
def read_file(path, filename, logger=LOGGER): fullfile = os.path.join(path, filename) try: return open(fullfile).read() except OSError, e: raise AquilonError("Could not read contents of %s: %s" % (fullfile, e))
rollback_failures = [] for args in self.rollback_list: cmd = [self.dsdb] cmd.extend(args) try: self.logger.client_info("DSDB: %s" % " ".join([str(a) for a in args])) run_command(cmd, env=self.getenv(), logger=self.logger) except ProcessException, err: rollback_failures.append(str(err)) did_something = bool(self.rollback_list) del self.rollback_list[:] if rollback_failures: raise AquilonError("DSDB rollback failed, DSDB state is " "inconsistent: " + "\n".join(rollback_failures)) elif did_something: self.logger.client_info("DSDB rollback completed.") def commit_or_rollback(self, error_msg=None, verbose=False): try: self.commit(verbose=verbose) except ProcessException, err: if not error_msg: error_msg = "DSDB update failed" self.logger.warn(str(err)) self.rollback(verbose=verbose) raise ArgumentError(error_msg) def add_action(self, command_args,
def render(self, session, dbuser, ip, netmask, prefixlen, network_environment, **arguments): if netmask: # There must me a faster way, but this is the easy one net = IPv4Network("127.0.0.0/%s" % netmask) prefixlen = net.prefixlen if prefixlen < 8 or prefixlen > 32: raise ArgumentError("The prefix length must be between 8 and 32.") dbnet_env = NetworkEnvironment.get_unique_or_default( session, network_environment) self.az.check_network_environment(dbuser, dbnet_env) dbnetwork = get_net_id_from_ip(session, ip, network_environment=dbnet_env) if prefixlen <= dbnetwork.cidr: raise ArgumentError("The specified --prefixlen must be bigger " "than the current value.") subnets = dbnetwork.network.subnet(new_prefix=prefixlen) # Collect IP addresses that will become network/broadcast addresses # after the split bad_ips = [] for subnet in subnets: bad_ips.append(subnet.ip) bad_ips.append(subnet.broadcast) q = session.query(AddressAssignment.ip) q = q.filter_by(network=dbnetwork) q = q.filter(AddressAssignment.ip.in_(bad_ips)) used_addrs = q.all() if used_addrs: raise ArgumentError( "Network split failed, because the following " "subnet IP and/or broadcast addresses are " "assigned to hosts: %s" % ", ".join([str(addr.ip) for addr in used_addrs])) q = session.query(ARecord.ip) q = q.filter_by(network=dbnetwork) q = q.filter(ARecord.ip.in_(bad_ips)) used_addrs = q.all() if used_addrs: raise ArgumentError( "Network split failed, because the following " "subnet IP and/or broadcast addresses are " "registered in the DNS: %s" % ", ".join([str(addr.ip) for addr in used_addrs])) # Reason of the initial value: we keep the name of the first segment # (e.g. "foo"), and the next segment will be called "foo_2" name_idx = 2 dbnets = [] for subnet in dbnetwork.network.subnet(new_prefix=prefixlen): # Skip the original if subnet.ip == dbnetwork.ip: continue # Generate a new name. Make it unique, even if the DB does not # enforce that currently while True: # TODO: check if the new name is too long name = "%s_%d" % (dbnetwork.name, name_idx) name_idx += 1 q = session.query(Network) q = q.filter_by(network_environment=dbnet_env) q = q.filter_by(name=name) if q.count() == 0: break # Should not happen... if name_idx > 1000: # pragma: no cover raise AquilonError( "Could not generate a unique network " "name in a reasonable time, bailing out") # Inherit location & side from the supernet newnet = Network( name=name, network=subnet, network_environment=dbnet_env, location=dbnetwork.location, side=dbnetwork.side, comments="Created by splitting {0:a}".format(dbnetwork)) session.add(newnet) dbnets.append(newnet) dbnetwork.cidr = prefixlen session.flush() for newnet in dbnets: fix_foreign_links(session, dbnetwork, newnet) session.flush()
def _remove_from_rp(self, na_obj): try: na_obj.delete() except Exception, e: raise AquilonError('Failed while removing nas assignment in ' 'resource pool: %s' % e)
logger.info("Unknown network %s in output line #%d: %s" % (network, reader.line_num, row)) continue if dbnetwork.cidr != bitmask_int: logger.client_info( "{0}: skipping VLAN {1}, because network " "bitmask value {2} differs from prefixlen " "{3.cidr} of {3:l}.".format(switch, vlan, bitmask, dbnetwork)) continue vlan_info = VlanInfo.get_unique(session, vlan_id=vlan_int, compel=False) if not vlan_info: logger.client_info( "vlan {0} is not defined in AQ. Please " "use add_vlan to add it.".format(vlan_int)) continue if vlan_info.vlan_type == "unknown": continue dbvlan = ObservedVlan(vlan_id=vlan_int, switch=switch, network=dbnetwork, creation_date=now) session.add(dbvlan) except CSVError, e: raise AquilonError("Error parsing vlan2net results: %s" % e)
def grab_address(session, fqdn, ip, network_environment=None, dns_environment=None, comments=None, allow_restricted_domain=False, allow_multi=False, allow_reserved=False, relaxed=False, preclude=False): """ Take ownership of an address. This is a bit complicated because due to DNS propagation delays, we want to allow users to pre-define a DNS address and then assign the address to a host later. Parameters: session: SQLA session handle fqdn: the name to allocate/take over ip: the IP address to allocate/take over network_environment: where the IP address lives dns_enviromnent: where the FQDN lives comments: any comments to attach to the DNS record if it is created as new allow_restricted_domain: if True, adding entries to restricted DNS domains is allowed, otherwise it is denied. Default is False. allow_multi: if True, allow the same FQDN to be added multiple times with different IP addresses. Deault is False. allow_reserved: if True, allow creating a ReservedName instead of an ARecord if no IP address was specified. Default is False. preclude: if True, forbid taking over an existing DNS record, even if it is not referenced by any AddressAssignment records. Default is False. """ if not isinstance(network_environment, NetworkEnvironment): network_environment = NetworkEnvironment.get_unique_or_default( session, network_environment) if not dns_environment: dns_environment = network_environment.dns_environment elif not isinstance(dns_environment, DnsEnvironment): dns_environment = DnsEnvironment.get_unique(session, dns_environment, compel=True) # Non-default DNS environments may contain anything, but we want to keep # the internal environment clean if dns_environment.is_default and not network_environment.is_default: raise ArgumentError("Entering external IP addresses to the " "internal DNS environment is not allowed.") short, dbdns_domain = parse_fqdn(session, fqdn) # Lock the domain to prevent adding/deleting records while we're checking # FQDN etc. availability dbdns_domain.lock_row() if dbdns_domain.restricted and not allow_restricted_domain: raise ArgumentError("{0} is restricted, adding extra addresses " "is not allowed.".format(dbdns_domain)) dbfqdn = Fqdn.get_or_create(session, dns_environment=dns_environment, name=short, dns_domain=dbdns_domain, query_options=[joinedload('dns_records')]) existing_record = None newly_created = False if ip: dbnetwork = get_net_id_from_ip(session, ip, network_environment) check_ip_restrictions(dbnetwork, ip, relaxed=relaxed) dbnetwork.lock_row() # No filtering on DNS environment. If an address is dynamic in one # environment, it should not be considered static in a different # environment. q = session.query(DynamicStub) q = q.filter_by(network=dbnetwork) q = q.filter_by(ip=ip) dbdns_rec = q.first() _forbid_dyndns(dbdns_rec) # Verify that no other record uses the same IP address, this time taking # the DNS environemt into consideration. # While the DNS would allow different A records to point to the same IP # address, the current user expectation is that creating a DNS entry # also counts as a reservation, so we can not allow this use case. If we # want to implement such a feature later, the best way would be to # subclass Alias and let that subclass emit an A record instead of a # CNAME when the dump_dns command is called. q = session.query(ARecord) q = q.filter_by(network=dbnetwork) q = q.filter_by(ip=ip) q = q.join(ARecord.fqdn) q = q.filter_by(dns_environment=dns_environment) dbrecords = q.all() if dbrecords and len(dbrecords) > 1: # pragma: no cover # We're just trying to make sure this never happens raise AquilonError( "IP address %s is referenced by multiple " "DNS records: %s" % (ip, ", ".join([format(rec, "a") for rec in dbrecords]))) if dbrecords and dbrecords[0].fqdn != dbfqdn: raise ArgumentError( "IP address {0} is already in use by {1:l}.".format( ip, dbrecords[0])) # Check if the name is used already for dbdns_rec in dbfqdn.dns_records: if isinstance(dbdns_rec, ARecord): _forbid_dyndns(dbdns_rec) _check_netenv_compat(dbdns_rec, network_environment) if dbdns_rec.ip == ip and dbdns_rec.network == dbnetwork: existing_record = dbdns_rec elif not allow_multi: raise ArgumentError( "{0} points to a different IP address.".format( dbdns_rec)) elif isinstance(dbdns_rec, ReservedName): existing_record = convert_reserved_to_arecord( session, dbdns_rec, dbnetwork, ip) newly_created = True else: # Exclude aliases etc. raise ArgumentError( "{0} cannot be used for address assignment.".format( dbdns_rec)) if not existing_record: existing_record = ARecord(fqdn=dbfqdn, ip=ip, network=dbnetwork, comments=comments) session.add(existing_record) newly_created = True else: if not dbfqdn.dns_records: # There's no IP, and the name did not exist before. Create a # reservation, but only if the caller allowed that use case. if not allow_reserved: raise ArgumentError("DNS Record %s does not exist." % dbfqdn) existing_record = ReservedName(fqdn=dbfqdn, comments=comments) newly_created = True else: # There's no IP, but the name is already in use. We need a single IP # address. if len(dbfqdn.dns_records) > 1: raise ArgumentError( "{0} does not resolve to a single IP address.".format( dbfqdn)) existing_record = dbfqdn.dns_records[0] _forbid_dyndns(existing_record) if not isinstance(existing_record, ARecord): # Exclude aliases etc. raise ArgumentError( "{0} cannot be used for address assignment.".format( existing_record)) # Verify that the existing record is in the network environment the # caller expects _check_netenv_compat(existing_record, network_environment) ip = existing_record.ip dbnetwork = existing_record.network dbnetwork.lock_row() if existing_record.hardware_entity: raise ArgumentError( "{0} is already used as the primary name of {1:cl} " "{1.label}.".format(existing_record, existing_record.hardware_entity)) if preclude and not newly_created: raise ArgumentError("{0} already exists.".format(existing_record)) if ip: q = session.query(AddressAssignment) q = q.filter_by(network=dbnetwork) q = q.filter_by(ip=ip) addr = q.first() if addr: raise ArgumentError("IP address {0} is already in use by " "{1:l}.".format(ip, addr.interface)) return (existing_record, newly_created)
def protocol(self): m = _name_re.match(self.fqdn.name) if not m: # pragma: no cover raise AquilonError("Malformed SRV FQDN in AQDB: %s" % self.fqdn) return m.group(2)