def render(self, session, logger, share, latency_threshold,
comments, **arguments):
validate_nlist_key("share", share)
q = session.query(Share).filter_by(name=share)
if q.count() == 0:
raise ArgumentError("Share %s is not used on any resource and "
"cannot be modified" % share)
plenaries = PlenaryCollection(logger=logger)
for dbshare in q.all():
if latency_threshold:
dbshare.latency_threshold = latency_threshold
if comments:
dbshare.comments = comments
plenaries.append(Plenary.get_plenary(dbshare))
session.flush()
plenaries.write()
return
def render(self, session, logger, cluster, pancinclude, pancexclude, pancdebug, cleandeps, **arguments):
dbcluster = Cluster.get_unique(session, cluster, compel=True)
if pancdebug:
pancinclude = r".*"
pancexclude = r"components/spma/functions"
dom = TemplateDomain(dbcluster.branch, dbcluster.sandbox_author, logger=logger)
plenaries = PlenaryCollection(logger=logger)
def add_cluster_plenaries(cluster):
plenaries.append(Plenary.get_plenary(cluster))
for host in cluster.hosts:
plenaries.append(Plenary.get_plenary(host))
add_cluster_plenaries(dbcluster)
if isinstance(dbcluster, MetaCluster):
for cluster in dbcluster.members:
add_cluster_plenaries(cluster)
with plenaries.get_key():
dom.compile(
session,
only=plenaries.object_templates,
panc_debug_include=pancinclude,
panc_debug_exclude=pancexclude,
cleandeps=cleandeps,
locked=True,
)
return
def __init__(self, dbobj, logger, required_only=False):
"""Initialize the chooser.
To clear out bindings that are not required, pass in
required_only=True.
Several staging areas and caches are set up within this object.
The general flow is that potential service instance choices
are kept in staging_services (dictionary of service to list of
service instances) and finalized into chosen_services (dictionary
of service to single service instance).
The original state of the object is held in the cache
original_service_instances (dictionary of service to single service
instance).
The instances_bound and instances_unbound lists are populated
after chosen_services with the differences between chosen_services
and original_service_instances.
Subclasses should call this before starting their own
initialization.
"""
self.dbobj = dbobj
self.session = object_session(dbobj)
self.required_only = required_only
self.logger = logger
self.description = self.generate_description()
self.logger.debug("Creating service Chooser for %s", self.description)
# Cache of the service maps
self.mapped_services = {}
# Stores interim service instance lists
self.staging_services = {}
# Report as many errors as possible in one shot
self.errors = []
# Cache the servers backing service instances
self.servers = {}
# Set of service instances with a new client
self.instances_bound = set()
# Set of service instances losing a client
self.instances_unbound = set()
# Track the chosen services
self.chosen_services = {}
# Keep stashed plenaries for rollback purposes
self.plenaries = PlenaryCollection(logger=self.logger)
def del_cluster(session, logger, dbcluster, config):
check_no_provided_service(dbcluster)
if hasattr(dbcluster, 'members') and dbcluster.members:
raise ArgumentError("%s is still in use by clusters: %s." %
(format(dbcluster),
", ".join([c.name for c in dbcluster.members])))
elif dbcluster.hosts:
hosts = ", ".join([h.fqdn for h in dbcluster.hosts])
raise ArgumentError("%s is still in use by hosts: %s." %
(format(dbcluster), hosts))
plenaries = PlenaryCollection(logger=logger)
plenaries.append(Plenary.get_plenary(dbcluster))
if dbcluster.resholder:
for res in dbcluster.resholder.resources:
plenaries.append(Plenary.get_plenary(res))
session.delete(dbcluster)
session.flush()
plenaries.remove(remove_profile=True)
trigger_notifications(config, logger, CLIENT_INFO)
return
def __init__(self, dbobj, logger, required_only=False):
"""Initialize the chooser.
To clear out bindings that are not required, pass in
required_only=True.
Several staging areas and caches are set up within this object.
The general flow is that potential service instance choices
are kept in staging_services (dictionary of service to list of
service instances) and finalized into chosen_services (dictionary
of service to single service instance).
The original state of the object is held in the cache
original_service_instances (dictionary of service to single service
instance).
The instances_bound and instances_unbound lists are populated
after chosen_services with the differences between chosen_services
and original_service_instances.
Subclasses should call this before starting their own
initialization.
"""
self.dbobj = dbobj
self.personality = dbobj.personality
self.archetype = dbobj.personality.archetype
self.session = object_session(dbobj)
self.required_only = required_only
self.logger = logger
self.logger.debug("Creating service chooser for {0:l}"
.format(self.dbobj))
# Cache of the service maps
self.mapped_services = {}
# Stores interim service instance lists
self.staging_services = {}
# Report as many errors as possible in one shot
self.errors = []
# Cache the servers backing service instances, used to determine
# affinity
self.servers = set()
# Set of service instances with a new client
self.instances_bound = set()
# Set of service instances losing a client
self.instances_unbound = set()
# Track the chosen services
self.chosen_services = {}
# Keep stashed plenaries for rollback purposes
self.plenaries = PlenaryCollection(logger=self.logger)
def render(self, session, logger, target, grn, eon_id, hostname, list, personality,
archetype, **arguments):
dbgrn = lookup_grn(session, grn, eon_id, logger=logger,
config=self.config)
plenaries = PlenaryCollection(logger=logger)
if hostname:
objs = [hostname_to_host(session, hostname)]
config_key = "host_grn_targets"
elif list:
check_hostlist_size(self.command, self.config, list)
objs = hostlist_to_hosts(session, list)
config_key = "host_grn_targets"
elif personality:
objs = [Personality.get_unique(session, name=personality,
archetype=archetype, compel=True)]
config_key = "personality_grn_targets"
for obj in objs:
section = "archetype_" + obj.archetype.name
if self.config.has_option(section, config_key):
valid_targets = [s.strip() for s in
self.config.get(section, config_key).split(",")]
else:
raise ArgumentError("{0} has no valid GRN targets configured."
.format(obj.archetype))
if target not in valid_targets:
raise ArgumentError("Invalid target %s for archetype %s, please "
"choose from: %s." %
(target, obj.archetype.name,
", ".join(valid_targets)))
plenaries.append(Plenary.get_plenary(obj))
self._update_dbobj(obj, target, dbgrn)
session.flush()
plenaries.write()
return
def render(self, session, logger, domain, sandbox, archetype, personality,
pancinclude, pancexclude, pancdebug, cleandeps,
**arguments):
dbdomain = None
dbauthor = None
if domain or sandbox:
(dbdomain, dbauthor) = get_branch_and_author(session, logger,
domain=domain,
sandbox=sandbox,
compel=True)
dbpersonality = Personality.get_unique(session, name=personality,
archetype=archetype, compel=True)
if pancdebug:
pancinclude = r'.*'
pancexclude = r'components/spma/functions'
q = session.query(Host)
q = q.filter_by(personality=dbpersonality)
if dbdomain:
q = q.filter_by(branch=dbdomain)
if dbauthor:
q = q.filter_by(sandbox_author=dbauthor)
host_list = q.all()
if not host_list:
return
# If the domain was not specified, set it to the domain of first host
dbdomain, dbauthor = validate_branch_author(host_list)
plenaries = PlenaryCollection(logger=logger)
for host in host_list:
plenaries.append(Plenary.get_plenary(host))
dom = TemplateDomain(dbdomain, dbauthor, logger=logger)
with plenaries.get_key():
dom.compile(session, only=plenaries.object_templates,
panc_debug_include=pancinclude,
panc_debug_exclude=pancexclude,
cleandeps=cleandeps, locked=True)
return
def render(self, session, logger, list, **arguments):
check_hostlist_size(self.command, self.config, list)
dbhosts = hostlist_to_hosts(session, list)
dbbranch, dbauthor = validate_branch_author(dbhosts)
failed = []
compileable = []
# Do any cross-list or dependency checks
for dbhost in dbhosts:
if dbhost.status.name == 'ready':
failed.append("{0:l} is in ready status, "
"advertised status can be reset only "
"when host is in non ready state.".format(dbhost))
if failed:
raise ArgumentError("Cannot modify the following hosts:\n%s" %
"\n".join(failed))
plenaries = PlenaryCollection(logger=logger)
for dbhost in dbhosts:
dbhost.advertise_status = False
plenaries.append(Plenary.get_plenary(dbhost))
session.flush()
td = TemplateDomain(dbbranch, dbauthor, logger=logger)
with plenaries.get_key():
plenaries.stash()
try:
plenaries.write(locked=True)
td.compile(session, only=compileable, locked=True)
except:
plenaries.restore_stash()
raise
return
def render(self, session, logger, domain, sandbox,
pancinclude, pancexclude, pancdebug, cleandeps,
**arguments):
(dbdomain, dbauthor) = get_branch_and_author(session, logger,
domain=domain,
sandbox=sandbox,
compel=True)
# Grab a shared lock on personalities and services used by the domain.
# Object templates (hosts, clusters) are protected by the domain lock.
plenaries = PlenaryCollection(logger=logger)
q1 = session.query(Personality)
q1 = q1.join(Host)
q1 = q1.filter(and_(Host.branch == dbdomain,
Host.sandbox_author == dbauthor))
q1 = q1.reset_joinpoint()
q1 = q1.options(joinedload('paramholder'),
subqueryload('paramholder.parameters'))
q2 = session.query(Personality)
q2 = q2.join(Cluster)
q2 = q2.filter(and_(Cluster.branch == dbdomain,
Cluster.sandbox_author == dbauthor))
q2 = q2.reset_joinpoint()
q2 = q2.options(joinedload('paramholder'),
subqueryload('paramholder.parameters'))
for dbpers in q1.union(q2):
plenaries.append(Plenary.get_plenary(dbpers))
q1 = session.query(ServiceInstance)
q1 = q1.join(ServiceInstance.clients)
q1 = q1.filter(and_(Host.branch == dbdomain,
Host.sandbox_author == dbauthor))
q2 = session.query(ServiceInstance)
q2 = q2.join(ServiceInstance.cluster_clients)
q2 = q2.filter(and_(Cluster.branch == dbdomain,
Cluster.sandbox_author == dbauthor))
for si in q1.union(q2):
plenaries.append(Plenary.get_plenary(si))
if pancdebug:
pancinclude = r'.*'
pancexclude = r'components/spma/functions'
dom = TemplateDomain(dbdomain, dbauthor, logger=logger)
with CompileKey.merge([CompileKey(domain=dbdomain.name, logger=logger),
plenaries.get_key(exclusive=False)]):
dom.compile(session,
panc_debug_include=pancinclude,
panc_debug_exclude=pancexclude,
cleandeps=cleandeps,
locked=True)
return
def render(self, session, logger, hostname, service, **arguments):
dbhost = hostname_to_host(session, hostname)
for srv in dbhost.archetype.services + dbhost.personality.services:
if srv.name == service:
raise ArgumentError("Cannot unbind a required service. "
"Perhaps you want to rebind?")
dbservice = Service.get_unique(session, service, compel=True)
si = get_host_bound_service(dbhost, dbservice)
if si:
logger.info("Removing client binding")
dbhost.services_used.remove(si)
session.flush()
plenaries = PlenaryCollection(logger=logger)
plenaries.append(Plenary.get_plenary(dbhost))
plenaries.append(PlenaryServiceInstanceServer.get_plenary(si))
plenaries.write()
return
class Chooser(object):
"""Helper for choosing services for an object."""
def __new__(cls, dbobj, *args, **kwargs):
if isinstance(dbobj, Host):
chooser = super(Chooser, HostChooser).__new__(HostChooser)
elif isinstance(dbobj, Cluster):
chooser = super(Chooser, ClusterChooser).__new__(ClusterChooser)
else:
# Just assume the consumer invoked the right subclass...
chooser = super(Chooser, cls).__new__(cls)
# Lock the owner in the DB to avoid problems with parallel runs
dbobj.lock_row()
return chooser
# Technically apply_changes is a method, but whatever...
abstract_fields = [
"description", "archetype", "personality", "location",
"required_services", "original_service_instances", "apply_changes"
]
def __init__(self, dbobj, logger, required_only=False):
"""Initialize the chooser.
To clear out bindings that are not required, pass in
required_only=True.
Several staging areas and caches are set up within this object.
The general flow is that potential service instance choices
are kept in staging_services (dictionary of service to list of
service instances) and finalized into chosen_services (dictionary
of service to single service instance).
The original state of the object is held in the cache
original_service_instances (dictionary of service to single service
instance).
The instances_bound and instances_unbound lists are populated
after chosen_services with the differences between chosen_services
and original_service_instances.
Subclasses should call this before starting their own
initialization.
"""
self.dbobj = dbobj
self.session = object_session(dbobj)
self.required_only = required_only
self.logger = logger
self.description = self.generate_description()
self.logger.debug("Creating service Chooser for %s", self.description)
# Cache of the service maps
self.mapped_services = {}
# Stores interim service instance lists
self.staging_services = {}
# Report as many errors as possible in one shot
self.errors = []
# Cache the servers backing service instances
self.servers = {}
# Set of service instances with a new client
self.instances_bound = set()
# Set of service instances losing a client
self.instances_unbound = set()
# Track the chosen services
self.chosen_services = {}
# Keep stashed plenaries for rollback purposes
self.plenaries = PlenaryCollection(logger=self.logger)
def generate_description(self):
return str(self.dbobj)
def verify_init(self):
"""This is more of a verify-and-finalize method..."""
for field in self.abstract_fields:
if not hasattr(self, field):
raise InternalError("%s provides no %s field" %
(type(self.dbobj), field))
# This can be tweaked...
if not self.required_only:
for (service, instance) in self.original_service_instances.items():
self.staging_services[service] = [instance]
def error(self, msg, *args, **kwargs):
"""Errors are consolidated so that many can be reported at once."""
formatted = msg % args
self.errors.append(formatted)
self.logger.info(msg, *args, **kwargs)
def set_required(self):
"""Main entry point when setting the required services for a host."""
self.verify_init()
self.prestash_primary()
self.logger.debug("Setting required services")
self.cache_service_maps(self.required_services)
for dbservice in self.required_services:
self.find_service_instances(dbservice)
self.check_errors()
for dbservice in self.required_services:
self.choose_cluster_aligned(dbservice)
self.choose_available_capacity(dbservice)
self.choose_past_use(dbservice)
self.check_errors()
# If this code needs to be made more efficient, this could
# be refactored. We don't always need count_servers()...
# In theory don't always need the loop above, either.
self.count_servers()
for dbservice in self.required_services:
self.reduce_service_instances(dbservice)
self.finalize_service_instances()
self.analyze_changes()
self.stash_services()
self.apply_changes()
self.check_errors()
def set_single(self, service, instance=None, force=False):
"""Use this to update a single service.
If planning to use this method, construct the Chooser with
required_only=False. If required_only is True, all other
bindings will be cleared.
"""
self.verify_init()
self.prestash_primary()
if instance:
self.logger.debug("Setting service %s instance %s", service.name,
instance.name)
self.staging_services[service] = [instance]
else:
self.logger.debug("Setting service %s with auto-bind",
service.name)
self.staging_services[service] = None
self.cache_service_maps([service])
self.find_service_instances(service)
self.check_errors()
self.choose_cluster_aligned(service)
self.choose_available_capacity(service)
self.check_errors()
self.choose_past_use(service)
# If this code needs to be made more efficient, this could
# be refactored. We don't always need count_servers()...
self.count_servers()
self.reduce_service_instances(service)
self.finalize_service_instances()
self.analyze_changes()
if not force and self.instances_bound and self.instances_unbound:
cfg_path = list(self.instances_unbound)[0].cfg_path
self.error("%s is already bound to %s, use unbind "
"to clear first or rebind to force." %
(self.description, cfg_path))
self.check_errors()
self.stash_services()
self.apply_changes()
self.check_errors()
def cache_service_maps(self, dbservices):
self.service_maps = ServiceInstance.get_mapped_instance_cache(
self.personality, self.location, dbservices, self.network)
def find_service_instances(self, dbservice):
"""This finds the "closest" service instances, based on the known maps.
It expects that cache_service_maps has been run.
"""
instances = self.service_maps.get(dbservice, [])
if len(instances) >= 1:
for instance in instances:
self.logger.debug(
"Found service %s instance %s "
"in the maps.", instance.service.name, instance.name)
self.staging_services[dbservice] = instances
return
self.error(
"Could not find a relevant service map for service %s "
"on %s", dbservice.name, self.description)
def check_errors(self):
if self.errors:
raise ArgumentError("\n".join(self.errors))
def choose_cluster_aligned(self, dbservice):
# Only implemented for hosts.
pass
def get_footprint(self, instance):
return 1
def instance_full(self, instance, max_clients, current_clients):
"""Check if the instance is effectively full.
This check is complicated because clusters have a larger impact
than a single host does.
"""
if max_clients is None:
return False
if instance == self.original_service_instances.get(instance.service):
if current_clients > max_clients:
return True
return False
return current_clients + self.get_footprint(instance) > max_clients
def choose_available_capacity(self, dbservice):
"""Verify that the available instances have spare capacity.
Error out if none should be used.
"""
maxed_out_instances = set()
for instance in self.staging_services[dbservice][:]:
max_clients = instance.enforced_max_clients
current_clients = instance.client_count
if self.instance_full(instance, max_clients, current_clients):
self.staging_services[dbservice].remove(instance)
maxed_out_instances.add(instance)
self.logger.debug(
"Rejected service %s instance %s with "
"max_client value of %s since client_count "
"is %s.", instance.service.name, instance.name,
max_clients, current_clients)
if len(self.staging_services[dbservice]) < 1:
self.error(
"The available instances %s for service %s are "
"at full capacity.",
[str(instance.name) for instance in maxed_out_instances],
dbservice.name)
return
def choose_past_use(self, dbservice):
"""If more than one service instance was found in the maps,
this method checks to see if we can reduce the list to a single
choice by checking to see if any of the instances was already in use.
"""
if len(self.staging_services[dbservice]) > 1 and \
self.original_service_instances.get(dbservice, None) and \
self.original_service_instances[dbservice] in \
self.staging_services[dbservice]:
self.logger.debug(
"Chose service %s instance %s because "
"of past use.", dbservice.name,
self.original_service_instances[dbservice])
self.staging_services[dbservice] = [
self.original_service_instances[dbservice]
]
return
def count_servers(self, dbservice=None):
"""Get a count of the number of times a server backs
service instances in use by this host.
This method is called both to initialize the count and to update
it as service instances are locked in.
"""
if dbservice:
instance_lists = [self.staging_services[dbservice]]
else:
instance_lists = self.staging_services.values()
for instances in instance_lists:
if len(instances) > 1:
# Ignore any services where an instance has not been chosen.
continue
for host in instances[0].server_hosts:
if self.servers.get(host, None):
self.servers[host] += 1
else:
self.servers[host] = 1
def reduce_service_instances(self, dbservice):
if len(self.staging_services[dbservice]) == 1:
self.count_servers(dbservice)
return
self.choose_affinity(dbservice)
if len(self.staging_services[dbservice]) == 1:
self.count_servers(dbservice)
return
self.choose_least_loaded(dbservice)
if len(self.staging_services[dbservice]) == 1:
self.count_servers(dbservice)
return
self.choose_random(dbservice)
self.count_servers(dbservice)
return
def choose_affinity(self, dbservice):
"""Attempt to choose a service based on server affinity,
also known as stickiness.
This could be extremely complicated when trying to deal with
instances backed by multiple servers. Starting simple.
Count the number of servers backing this instance that
back other instances used by client. Any instance that does
not have the largest count gets tossed.
"""
max_servers = 0
max_instances = None
for instance in self.staging_services[dbservice]:
common_servers = []
self.logger.debug("Checking service %s instance %s servers %s",
instance.service.name, instance.name,
[host.fqdn for host in instance.server_hosts])
for host in instance.server_hosts:
if self.servers.get(host, None):
common_servers.append(host)
if not common_servers:
continue
if len(common_servers) > max_servers:
max_servers = len(common_servers)
max_instances = [instance]
elif len(common_servers) == max_servers:
max_instances.append(instance)
if max_instances and \
len(max_instances) < len(self.staging_services[dbservice]):
for instance in self.staging_services[dbservice]:
if instance not in max_instances:
self.logger.debug(
"Discounted service %s instance %s "
"due to server affinity (stickiness).",
instance.service.name, instance.name)
self.staging_services[dbservice] = max_instances
def choose_least_loaded(self, dbservice):
"""Choose a service instance based on load."""
least_clients = None
least_loaded = []
for instance in self.staging_services[dbservice]:
client_count = instance.client_count
if not least_loaded or client_count < least_clients:
least_clients = client_count
least_loaded = [instance]
elif client_count == least_clients:
least_loaded.append(instance)
if len(least_loaded) < len(self.staging_services[dbservice]):
for instance in self.staging_services[dbservice]:
if instance not in least_loaded:
self.logger.debug(
"Discounted service %s instance %s "
"due to load.", instance.service.name, instance.name)
self.staging_services[dbservice] = least_loaded
def choose_random(self, dbservice):
"""Pick a service instance randomly."""
self.staging_services[dbservice] = [
choice(self.staging_services[dbservice])
]
self.logger.debug(
"Randomly chose service %s instance %s "
"from remaining choices.", dbservice.name,
self.staging_services[dbservice][0].name)
def finalize_service_instances(self):
"""Fill out the list of chosen services."""
for (service, instances) in self.staging_services.items():
if len(instances) < 1: # pragma: no cover
self.error("Internal Error: Attempt to finalize on "
"service %s without any candidates." % service.name)
continue
if len(instances) > 1: # pragma: no cover
self.error("Internal Error: Attempt to finalize on "
"service %s with too many candidates %s." %
(service.name, [
"service %s instance %s" %
(instance.service.name, instance.name)
for instance in instances
]))
self.chosen_services[service] = instances[0]
def analyze_changes(self):
"""Determine what changed."""
for (service, instance) in self.chosen_services.items():
if not self.original_service_instances.get(service, None) or \
self.original_service_instances[service] != instance:
self.instances_bound.add(instance)
for (service, instance) in self.original_service_instances.items():
if not self.chosen_services.get(service, None) or \
self.chosen_services[service] != instance:
self.instances_unbound.add(instance)
def stash_services(self):
for instance in self.instances_bound.union(self.instances_unbound):
plenary = PlenaryServiceInstanceServer(instance,
logger=self.logger)
plenary.stash()
self.plenaries.append(plenary)
def flush_changes(self):
self.session.flush()
def get_write_key(self):
return self.plenaries.get_write_key()
def write_plenary_templates(self, locked=False):
self.plenaries.write(locked=locked)
def prestash_primary(self):
pass
def restore_stash(self):
self.plenaries.restore_stash()
def changed_server_fqdns(self):
hosts = set()
for instance in chain(self.instances_bound, self.instances_unbound):
for srv in instance.servers:
# Skip servers that do not have a profile
if not srv.host.personality.archetype.is_compileable:
continue
if (srv.host.branch == self.dbobj.branch
and srv.host.sandbox_author_id
== self.dbobj.sandbox_author_id):
hosts.add(str(srv.host.fqdn))
return hosts
def render(self, session, logger, model, vendor, newmodel, newvendor,
comments, leave_existing, **arguments):
for (arg, value) in arguments.items():
# Cleaning the strings isn't strictly necessary but allows
# for simple equality checks below and removes the need to
# call refresh().
if arg in ['newmodel', 'newvendor',
'cpuname', 'cpuvendor', 'disktype', 'diskcontroller',
'nicmodel', 'nicvendor']:
if value is not None:
arguments[arg] = value.lower().strip()
dbmodel = Model.get_unique(session, name=model, vendor=vendor,
compel=True)
if leave_existing and (newmodel or newvendor):
raise ArgumentError("Cannot update model name or vendor without "
"updating any existing machines.")
fix_existing = not leave_existing
dbmachines = set()
# The sub-branching here is a little difficult to read...
# Basically, there are three different checks to handle
# setting a new vendor, a new name, or both.
if newvendor:
dbnewvendor = Vendor.get_unique(session, newvendor, compel=True)
if newmodel:
Model.get_unique(session, name=newmodel, vendor=dbnewvendor,
preclude=True)
else:
Model.get_unique(session, name=dbmodel.name,
vendor=dbnewvendor, preclude=True)
dbmodel.vendor = dbnewvendor
if newmodel:
if not newvendor:
Model.get_unique(session, name=newmodel, vendor=dbmodel.vendor,
preclude=True)
dbmodel.name = newmodel
if newvendor or newmodel:
q = session.query(Machine).filter_by(model=dbmodel)
dbmachines.update(q.all())
# For now, can't update model_type. There are too many spots
# that special case things like aurora_node or virtual_machine to
# know that the transistion is safe. If there is enough need we
# can always add those transitions later.
if arguments['machine_type'] is not None:
raise UnimplementedError("Cannot (yet) change a model's "
"machine type.")
if comments:
dbmodel.comments = comments
# The comments also do not affect the templates.
cpu_args = ['cpuname', 'cpuvendor', 'cpuspeed']
cpu_info = dict([(self.argument_lookup[arg], arguments[arg])
for arg in cpu_args])
cpu_values = [v for v in cpu_info.values() if v is not None]
nic_args = ['nicmodel', 'nicvendor']
nic_info = dict([(self.argument_lookup[arg], arguments[arg])
for arg in nic_args])
nic_values = [v for v in nic_info.values() if v is not None]
spec_args = ['cpunum', 'memory', 'disktype', 'diskcontroller',
'disksize', 'nics']
specs = dict([(self.argument_lookup[arg], arguments[arg])
for arg in spec_args])
spec_values = [v for v in specs.values() if v is not None]
if not dbmodel.machine_specs:
if cpu_values or nic_values or spec_values:
# You can't add a non-machine model with machine_specs
# thus we only need to check here if you try and update
if not dbmodel.model_type.isMachineType():
raise ArgumentError("Machine specfications are only valid"
" for machine types")
if not cpu_values or len(spec_values) < len(spec_args):
raise ArgumentError("Missing required parameters to store "
"machine specs for the model. Please "
"give all CPU, disk, RAM, and NIC "
"count information.")
dbcpu = Cpu.get_unique(session, compel=True, **cpu_info)
if nic_values:
dbnic = Model.get_unique(session, compel=True,
model_type=NicType.Nic, **nic_info)
else:
dbnic = Model.default_nic_model(session)
dbmachine_specs = MachineSpecs(model=dbmodel, cpu=dbcpu,
nic_model=dbnic, **specs)
session.add(dbmachine_specs)
# Anything below that updates specs should have been verified above.
if cpu_values:
dbcpu = Cpu.get_unique(session, compel=True, **cpu_info)
self.update_machine_specs(model=dbmodel, dbmachines=dbmachines,
attr='cpu', value=dbcpu,
fix_existing=fix_existing)
for arg in ['memory', 'cpunum']:
if arguments[arg] is not None:
self.update_machine_specs(model=dbmodel, dbmachines=dbmachines,
attr=self.argument_lookup[arg],
value=arguments[arg],
fix_existing=fix_existing)
if arguments['disktype']:
if fix_existing:
raise ArgumentError("Please specify --leave_existing to "
"change the model disktype. This cannot "
"be converted automatically.")
dbmodel.machine_specs.disk_type = arguments['disktype']
for arg in ['diskcontroller', 'disksize']:
if arguments[arg] is not None:
self.update_disk_specs(model=dbmodel, dbmachines=dbmachines,
attr=self.argument_lookup[arg],
value=arguments[arg],
fix_existing=fix_existing)
if nic_values:
dbnic = Model.get_unique(session, compel=True, **nic_info)
self.update_interface_specs(model=dbmodel, dbmachines=dbmachines,
value=dbnic, fix_existing=fix_existing)
if arguments['nics'] is not None:
dbmodel.machine_specs.nic_count = arguments['nics']
session.flush()
plenaries = PlenaryCollection(logger=logger)
for dbmachine in dbmachines:
plenaries.append(Plenary.get_plenary(dbmachine))
plenaries.write()
return
def render(self, session, logger, feature, archetype, personality, model,
vendor, interface, justification, user, **arguments):
# Binding a feature to a named interface makes sense in the scope of a
# personality, but not for a whole archetype.
if interface and not personality:
raise ArgumentError("Binding to a named interface needs "
"a personality.")
q = session.query(Personality)
dbarchetype = None
feature_type = "host"
justification_required = True
# Warning: order matters here!
params = {}
if personality:
justification_required = False
dbpersonality = Personality.get_unique(session,
name=personality,
archetype=archetype,
compel=True)
params["personality"] = dbpersonality
if interface:
params["interface_name"] = interface
feature_type = "interface"
dbarchetype = dbpersonality.archetype
q = q.filter_by(archetype=dbarchetype)
q = q.filter_by(name=personality)
elif archetype:
dbarchetype = Archetype.get_unique(session, archetype, compel=True)
params["archetype"] = dbarchetype
q = q.filter_by(archetype=dbarchetype)
else:
# It's highly unlikely that a feature template would work for
# _any_ archetype, so disallow this case for now. As I can't
# rule out that such a case will not have some uses in the
# future, the restriction is here and not in the model.
raise ArgumentError("Please specify either an archetype or "
"a personality when binding a feature.")
if model:
dbmodel = Model.get_unique(session, name=model, vendor=vendor,
compel=True)
if dbmodel.model_type.isNic():
feature_type = "interface"
else:
feature_type = "hardware"
params["model"] = dbmodel
if dbarchetype and not dbarchetype.is_compileable:
raise UnimplementedError("Binding features to non-compilable "
"archetypes is not implemented.")
if not feature_type: # pragma: no cover
raise InternalError("Feature type is not known.")
dbfeature = Feature.get_unique(session, name=feature,
feature_type=feature_type, compel=True)
cnt = q.count()
# TODO: should the limit be configurable?
if justification_required and cnt > 0:
if not justification:
raise AuthorizationException("Changing feature bindings for "
"more than just a personality "
"requires --justification.")
validate_justification(user, justification)
self.do_link(session, logger, dbfeature, params)
session.flush()
plenaries = PlenaryCollection(logger=logger)
for dbpersonality in q:
plenaries.append(Plenary.get_plenary(dbpersonality))
written = plenaries.write()
logger.client_info("Flushed %d/%d templates." %
(written, len(plenaries.plenaries)))
return
legacy_env = HostEnvironment.get_unique(session,
'legacy',
compel=True)
if dbpersona.host_environment == legacy_env:
HostEnvironment.polymorphic_subclass(
host_environment, "Unknown environment name")
Personality.validate_env_in_name(personality, host_environment)
dbpersona.host_environment = HostEnvironment.get_unique(
session, host_environment, compel=True)
else:
raise ArgumentError(
"The personality '{0}' already has env set to '{1}'"
" and cannot be updated".format(str(dbpersona),
host_environment))
plenaries = PlenaryCollection(logger=logger)
if grn or eon_id:
dbgrn = lookup_grn(session,
grn,
eon_id,
logger=logger,
config=self.config)
old_grn = dbpersona.owner_grn
dbpersona.owner_grn = dbgrn
if not leave_existing:
# If this is a public personality, then there may be hosts with
# various GRNs inside the personality, so make sure we preserve
# those GRNs by filtering on the original GRN of the personality
q = session.query(Host)
if q.count() > 0:
raise ArgumentError("The personality {0} is in use and cannot "
"be modified".format(str(dbpersona)))
dbpersona.cluster_required = cluster_required
if host_environment is not None:
if dbpersona.host_environment.name == 'legacy':
dbhost_env = HostEnvironment.get_instance(session, host_environment)
Personality.validate_env_in_name(personality, dbhost_env.name)
dbpersona.host_environment = dbhost_env
else:
raise ArgumentError("The personality '{0!s}' already has env set to '{1!s}'"
" and cannot be updated"
.format(dbpersona, dbpersona.host_environment))
plenaries = PlenaryCollection(logger=logger)
if grn or eon_id:
dbgrn = lookup_grn(session, grn, eon_id, logger=logger,
config=self.config)
old_grn = dbpersona.owner_grn
dbpersona.owner_grn = dbgrn
if not leave_existing:
# If this is a public personality, then there may be hosts with
# various GRNs inside the personality, so make sure we preserve
# those GRNs by filtering on the original GRN of the personality
q = session.query(Host)
q = q.filter_by(personality=dbpersona, owner_grn=old_grn)
for dbhost in q.all():
dbhost.owner_grn = dbgrn
try:
chooser.set_required()
except ArgumentError, e:
failed.append(str(e))
if failed:
raise ArgumentError("The following hosts failed service "
"binding:\n%s" % "\n".join(failed))
session.flush()
logger.info("reconfigure_hostlist processing: %s" %
",".join([str(dbhost.fqdn) for dbhost in dbhosts]))
if not choosers:
return
plenaries = PlenaryCollection(logger=logger)
# chooser.plenaries is a PlenaryCollection - this flattens
# that top level.
plenary_set = set()
for chooser in choosers:
plenary_set.update(chooser.plenaries.plenaries)
plenaries.extend(plenary_set)
td = TemplateDomain(dbbranch, dbauthor, logger=logger)
# Don't bother locking until every possible check before the
# actual writing and compile is done. This will allow for fast
# turnaround on errors (no need to wait for a lock if there's
# a missing service map entry or something).
with plenaries.get_key():
def render(self, session, logger, cluster, buildstatus, **arguments):
dbcluster = Cluster.get_unique(session, cluster, compel=True)
dbstatus = ClusterLifecycle.get_instance(session, buildstatus)
if not dbcluster.status.transition(dbcluster, dbstatus):
return
if not dbcluster.personality.archetype.is_compileable:
return
session.flush()
plenaries = PlenaryCollection(logger=logger)
plenaries.append(Plenary.get_plenary(dbcluster))
for dbhost in dbcluster.hosts:
plenaries.append(Plenary.get_plenary(dbhost))
td = TemplateDomain(dbcluster.branch, dbcluster.sandbox_author,
logger=logger)
# Force a host lock as pan might overwrite the profile...
with plenaries.get_key():
plenaries.stash()
try:
plenaries.write(locked=True)
td.compile(session, only=plenaries.object_templates,
locked=True)
except:
plenaries.restore_stash()
raise
return
class Chooser(object):
"""Helper for choosing services for an object."""
def __new__(cls, dbobj, *args, **kwargs):
if isinstance(dbobj, Host):
chooser = super(Chooser, HostChooser).__new__(HostChooser)
elif isinstance(dbobj, Cluster):
chooser = super(Chooser, ClusterChooser).__new__(ClusterChooser)
else:
# Just assume the consumer invoked the right subclass...
chooser = super(Chooser, cls).__new__(cls)
# Lock the owner in the DB to avoid problems with parallel runs
dbobj.lock_row()
return chooser
abstract_fields = ["location", "required_services", "network",
"original_service_instances"]
def __init__(self, dbobj, logger, required_only=False):
"""Initialize the chooser.
To clear out bindings that are not required, pass in
required_only=True.
Several staging areas and caches are set up within this object.
The general flow is that potential service instance choices
are kept in staging_services (dictionary of service to list of
service instances) and finalized into chosen_services (dictionary
of service to single service instance).
The original state of the object is held in the cache
original_service_instances (dictionary of service to single service
instance).
The instances_bound and instances_unbound lists are populated
after chosen_services with the differences between chosen_services
and original_service_instances.
Subclasses should call this before starting their own
initialization.
"""
self.dbobj = dbobj
self.personality = dbobj.personality
self.archetype = dbobj.personality.archetype
self.session = object_session(dbobj)
self.required_only = required_only
self.logger = logger
self.logger.debug("Creating service chooser for {0:l}"
.format(self.dbobj))
# Cache of the service maps
self.mapped_services = {}
# Stores interim service instance lists
self.staging_services = {}
# Report as many errors as possible in one shot
self.errors = []
# Cache the servers backing service instances, used to determine
# affinity
self.servers = set()
# Set of service instances with a new client
self.instances_bound = set()
# Set of service instances losing a client
self.instances_unbound = set()
# Track the chosen services
self.chosen_services = {}
# Keep stashed plenaries for rollback purposes
self.plenaries = PlenaryCollection(logger=self.logger)
def apply_changes(self):
raise InternalError("This method must be overridden")
def verify_init(self):
"""This is more of a verify-and-finalize method..."""
for field in self.abstract_fields:
if not hasattr(self, field):
raise InternalError("%s provides no %s field" %
(type(self.dbobj), field))
# This can be tweaked...
if not self.required_only:
for (service, instance) in self.original_service_instances.items():
self.staging_services[service] = [instance]
def error(self, msg, *args, **kwargs):
"""Errors are consolidated so that many can be reported at once."""
formatted = msg % args
self.errors.append(formatted)
self.logger.info(msg, *args, **kwargs)
def set_required(self):
"""Main entry point when setting the required services for a host."""
self.verify_init()
self.prestash_primary()
self.logger.debug("Setting required services")
self.cache_service_maps(self.required_services)
for dbservice in self.required_services:
self.find_service_instances(dbservice)
self.check_errors()
for dbservice in self.required_services:
self.choose_cluster_aligned(dbservice)
self.choose_available_capacity(dbservice)
self.choose_past_use(dbservice)
self.check_errors()
# If this code needs to be made more efficient, this could
# be refactored. We don't always need count_servers()...
# In theory don't always need the loop above, either.
self.count_servers()
for dbservice in self.required_services:
self.reduce_service_instances(dbservice)
self.finalize_service_instances()
self.analyze_changes()
self.stash_services()
self.apply_changes()
self.check_errors()
def set_single(self, service, instance=None, force=False):
"""Use this to update a single service.
If planning to use this method, construct the Chooser with
required_only=False. If required_only is True, all other
bindings will be cleared.
"""
self.verify_init()
self.prestash_primary()
if instance:
self.logger.debug("Setting service %s instance %s",
service.name, instance.name)
self.staging_services[service] = [instance]
else:
self.logger.debug("Setting service %s with auto-bind",
service.name)
self.staging_services[service] = None
self.cache_service_maps([service])
self.find_service_instances(service)
self.check_errors()
self.choose_cluster_aligned(service)
self.choose_available_capacity(service)
self.check_errors()
self.choose_past_use(service)
# If this code needs to be made more efficient, this could
# be refactored. We don't always need count_servers()...
self.count_servers()
self.reduce_service_instances(service)
self.finalize_service_instances()
self.analyze_changes()
if not force and self.instances_bound and self.instances_unbound:
si = list(self.instances_unbound)[0]
self.error("{0} is already bound to {1:l}, use unbind "
"to clear first or rebind to force."
.format(self.dbobj, si))
self.check_errors()
self.stash_services()
self.apply_changes()
self.check_errors()
def cache_service_maps(self, dbservices):
self.service_maps = ServiceInstance.get_mapped_instance_cache(
self.personality, self.location, dbservices, self.network)
def find_service_instances(self, dbservice):
"""This finds the "closest" service instances, based on the known maps.
It expects that cache_service_maps has been run.
"""
instances = self.service_maps.get(dbservice, [])
if len(instances) >= 1:
for instance in instances:
self.logger.debug("Found {0:l} in the maps.".format(instance))
self.staging_services[dbservice] = instances
return
self.error("Could not find a relevant service map for {0:l} "
"on {1:l}".format(dbservice, self.dbobj))
def check_errors(self):
if self.errors:
raise ArgumentError("\n".join(self.errors))
def choose_cluster_aligned(self, dbservice):
# Only implemented for hosts.
pass
def get_footprint(self, instance):
return 1
def instance_full(self, instance, max_clients, current_clients):
"""Check if the instance is effectively full.
This check is complicated because clusters have a larger impact
than a single host does.
"""
if max_clients is None:
return False
if instance == self.original_service_instances.get(instance.service):
if current_clients > max_clients:
return True
return False
return current_clients + self.get_footprint(instance) > max_clients
def choose_available_capacity(self, dbservice):
"""Verify that the available instances have spare capacity.
Error out if none should be used.
"""
maxed_out_instances = set()
for instance in self.staging_services[dbservice][:]:
max_clients = instance.enforced_max_clients
current_clients = instance.client_count
if self.instance_full(instance, max_clients, current_clients):
self.staging_services[dbservice].remove(instance)
maxed_out_instances.add(instance)
self.logger.debug("Rejected service %s instance %s with "
"max_client value of %s since client_count "
"is %s.",
instance.service.name, instance.name,
max_clients, current_clients)
if len(self.staging_services[dbservice]) < 1:
self.error("The available instances %s for service %s are "
"at full capacity.",
[str(instance.name)
for instance in maxed_out_instances],
dbservice.name)
return
def choose_past_use(self, dbservice):
"""If more than one service instance was found in the maps,
this method checks to see if we can reduce the list to a single
choice by checking to see if any of the instances was already in use.
"""
if len(self.staging_services[dbservice]) > 1 and \
dbservice in self.original_service_instances and \
self.original_service_instances[dbservice] in \
self.staging_services[dbservice]:
instance = self.original_service_instances[dbservice]
self.logger.debug("Chose {0:l} because of past use."
.format(instance))
self.staging_services[dbservice] = [instance]
return
def append_server_object(self, container, srv):
"""Append server objects to a container.
This is a helper used for implementing server affinity.
"""
if srv.host:
container.add(srv.host)
if srv.cluster:
container.add(srv.cluster)
# If the service is not bound to a host or cluster, then we have
# no choice than to use the alias for providing server affinity
if srv.alias and not srv.host and not srv.cluster:
container.add(srv.alias)
def count_servers(self, dbservice=None):
"""Get a count of the number of times a server backs
service instances in use by this host.
This method is called both to initialize the count and to update
it as service instances are locked in.
"""
if dbservice:
instance_lists = [self.staging_services[dbservice]]
else:
instance_lists = self.staging_services.values()
for instances in instance_lists:
if len(instances) > 1:
# Ignore any services where an instance has not been chosen.
continue
for srv in instances[0].servers:
self.append_server_object(self.servers, srv)
def reduce_service_instances(self, dbservice):
if len(self.staging_services[dbservice]) == 1:
self.count_servers(dbservice)
return
self.choose_affinity(dbservice)
if len(self.staging_services[dbservice]) == 1:
self.count_servers(dbservice)
return
self.choose_least_loaded(dbservice)
if len(self.staging_services[dbservice]) == 1:
self.count_servers(dbservice)
return
self.choose_random(dbservice)
self.count_servers(dbservice)
return
def choose_affinity(self, dbservice):
"""Attempt to choose a service based on server affinity,
also known as stickiness.
This could be extremely complicated when trying to deal with
instances backed by multiple servers. Starting simple.
Count the number of servers backing this instance that
back other instances used by client. Any instance that does
not have the largest count gets tossed.
"""
max_servers = 0
max_instances = []
for instance in self.staging_services[dbservice]:
self.logger.debug("Checking service %s instance %s servers %s",
instance.service.name, instance.name,
[srv.fqdn for srv in instance.servers])
instance_servers = set()
for srv in instance.servers:
self.append_server_object(instance_servers, srv)
common_servers = self.servers & instance_servers
if len(common_servers) > max_servers:
max_servers = len(common_servers)
max_instances = [instance]
elif len(common_servers) == max_servers:
max_instances.append(instance)
if max_instances and \
len(max_instances) < len(self.staging_services[dbservice]):
for instance in self.staging_services[dbservice]:
if instance not in max_instances:
self.logger.debug("Discounted {0:l} due to server affinity "
"(stickiness).".format(instance))
self.staging_services[dbservice] = max_instances
def choose_least_loaded(self, dbservice):
"""Choose a service instance based on load."""
least_clients = None
least_loaded = []
for instance in self.staging_services[dbservice]:
client_count = instance.client_count
if not least_loaded or client_count < least_clients:
least_clients = client_count
least_loaded = [instance]
elif client_count == least_clients:
least_loaded.append(instance)
if len(least_loaded) < len(self.staging_services[dbservice]):
for instance in self.staging_services[dbservice]:
if instance not in least_loaded:
self.logger.debug("Discounted {0:l} due to load."
.format(instance))
self.staging_services[dbservice] = least_loaded
def choose_random(self, dbservice):
"""Pick a service instance randomly."""
self.staging_services[dbservice] = [
choice(self.staging_services[dbservice])]
self.logger.debug("Randomly chose service %s instance %s "
"from remaining choices.",
dbservice.name,
self.staging_services[dbservice][0].name)
def finalize_service_instances(self):
"""Fill out the list of chosen services."""
for (service, instances) in self.staging_services.items():
if len(instances) < 1: # pragma: no cover
self.error("Internal Error: Attempt to finalize on "
"service %s without any candidates." %
service.name)
continue
if len(instances) > 1: # pragma: no cover
self.error("Internal Error: Attempt to finalize on "
"service %s with too many candidates %s." %
(service.name,
["service %s instance %s" %
(instance.service.name, instance.name)
for instance in instances]))
self.chosen_services[service] = instances[0]
def analyze_changes(self):
"""Determine what changed."""
for (service, instance) in self.chosen_services.items():
if service not in self.original_service_instances or \
self.original_service_instances[service] != instance:
self.instances_bound.add(instance)
for (service, instance) in self.original_service_instances.items():
if service not in self.chosen_services or \
self.chosen_services[service] != instance:
self.instances_unbound.add(instance)
def stash_services(self):
changed_servers = set()
for instance in self.instances_bound.union(self.instances_unbound):
if not instance.service.need_client_list:
continue
for srv in instance.servers:
if srv.host:
changed_servers.add(srv.host)
if srv.cluster:
changed_servers.add(srv.cluster)
plenary = PlenaryServiceInstanceServer.get_plenary(instance)
self.plenaries.append(plenary)
for dbobj in changed_servers:
# Skip servers that do not have a profile
if not dbobj.personality.archetype.is_compileable:
continue
# Skip servers that are in a different domain/sandbox
if (dbobj.branch != self.dbobj.branch or
dbobj.sandbox_author_id != self.dbobj.sandbox_author_id):
continue
self.plenaries.append(Plenary.get_plenary(dbobj))
if isinstance(dbobj, Cluster):
for dbhost in dbobj.hosts:
self.plenaries.append(Plenary.get_plenary(dbhost))
def flush_changes(self):
self.session.flush()
def get_key(self):
return self.plenaries.get_key()
def write_plenary_templates(self, locked=False):
self.plenaries.stash()
self.plenaries.write(locked=locked)
def prestash_primary(self):
pass
def restore_stash(self):
self.plenaries.restore_stash()
def render(self, session, logger, building, city, address,
fullname, default_dns_domain, comments, **arguments):
dbbuilding = get_location(session, building=building)
old_city = dbbuilding.city
dsdb_runner = DSDBRunner(logger=logger)
if address is not None:
old_address = dbbuilding.address
dbbuilding.address = address
dsdb_runner.update_building(dbbuilding.name, dbbuilding.address,
old_address)
if fullname is not None:
dbbuilding.fullname = fullname
if comments is not None:
dbbuilding.comments = comments
if default_dns_domain is not None:
if default_dns_domain:
dbdns_domain = DnsDomain.get_unique(session, default_dns_domain,
compel=True)
dbbuilding.default_dns_domain = dbdns_domain
else:
dbbuilding.default_dns_domain = None
plenaries = PlenaryCollection(logger=logger)
if city:
dbcity = get_location(session, city=city)
# This one would change the template's locations hence forbidden
if dbcity.hub != dbbuilding.hub:
# Doing this both to reduce user error and to limit
# testing required.
raise ArgumentError("Cannot change hubs. {0} is in {1} "
"while {2} is in {3}.".format(
dbcity, dbcity.hub,
dbbuilding, dbbuilding.hub))
# issue svcmap warnings
maps = 0
for map_type in [ServiceMap, PersonalityServiceMap]:
maps = maps + session.query(map_type).\
filter_by(location=old_city).count()
if maps > 0:
logger.client_info("There are {0} service(s) mapped to the "
"old location of the ({1:l}), please "
"review and manually update mappings for "
"the new location as needed.".format(
maps, dbbuilding.city))
dbbuilding.update_parent(parent=dbcity)
if old_city.campus and (old_city.campus != dbcity.campus):
dsdb_runner.del_campus_building(old_city.campus, building)
if dbcity.campus and (old_city.campus != dbcity.campus):
dsdb_runner.add_campus_building(dbcity.campus, building)
query = session.query(Machine)
query = query.filter(Machine.location_id.in_(dbcity.offspring_ids()))
for dbmachine in query:
plenaries.append(Plenary.get_plenary(dbmachine))
session.flush()
if plenaries.plenaries:
with plenaries.get_key():
plenaries.stash()
try:
plenaries.write(locked=True)
dsdb_runner.commit_or_rollback()
except:
plenaries.restore_stash()
raise
else:
dsdb_runner.commit_or_rollback()
return
def render(self, session, logger, hostname, **arguments):
# Check dependencies, translate into user-friendly message
dbhost = hostname_to_host(session, hostname)
dbhost.lock_row()
check_no_provided_service(dbhost)
# Any service bindings that we need to clean up afterwards
plenaries = PlenaryCollection(logger=logger)
remove_plenaries = PlenaryCollection(logger=logger)
remove_plenaries.append(Plenary.get_plenary(dbhost))
archetype = dbhost.archetype.name
dbmachine = dbhost.hardware_entity
oldinfo = DSDBRunner.snapshot_hw(dbmachine)
ip = dbmachine.primary_ip
for si in dbhost.services_used:
plenaries.append(PlenaryServiceInstanceServer.get_plenary(si))
logger.info("Before deleting {0:l}, removing binding to {1:l}"
.format(dbhost, si))
del dbhost.services_used[:]
if dbhost.resholder:
for res in dbhost.resholder.resources:
remove_plenaries.append(Plenary.get_plenary(res))
# In case of Zebra, the IP may be configured on multiple interfaces
for iface in dbmachine.interfaces:
if ip in iface.addresses:
iface.addresses.remove(ip)
if dbhost.cluster:
dbcluster = dbhost.cluster
dbcluster.hosts.remove(dbhost)
set_committed_value(dbhost, '_cluster', None)
dbcluster.validate()
plenaries.append(Plenary.get_plenary(dbcluster))
dbdns_rec = dbmachine.primary_name
dbmachine.primary_name = None
dbmachine.host = None
session.delete(dbhost)
delete_dns_record(dbdns_rec)
session.flush()
if dbmachine.vm_container:
plenaries.append(Plenary.get_plenary(dbmachine.vm_container))
with CompileKey.merge([plenaries.get_key(),
remove_plenaries.get_key()]):
plenaries.stash()
remove_plenaries.stash()
try:
plenaries.write(locked=True)
remove_plenaries.remove(locked=True, remove_profile=True)
if archetype != 'aurora' and ip is not None:
dsdb_runner = DSDBRunner(logger=logger)
dsdb_runner.update_host(dbmachine, oldinfo)
dsdb_runner.commit_or_rollback("Could not remove host %s from "
"DSDB" % hostname)
if archetype == 'aurora':
logger.client_info("WARNING: removing host %s from AQDB and "
"*not* changing DSDB." % hostname)
except:
plenaries.restore_stash()
remove_plenaries.restore_stash()
raise
trigger_notifications(self.config, logger, CLIENT_INFO)
return
def render(self, session, logger, interface, machine, mac, model, vendor,
boot, pg, autopg, comments, master, clear_master, default_route,
rename_to, **arguments):
"""This command expects to locate an interface based only on name
and machine - all other fields, if specified, are meant as updates.
If the machine has a host, dsdb may need to be updated.
The boot flag can *only* be set to true. This is mostly technical,
as at this point in the interface it is difficult to tell if the
flag was unset or set to false. However, it also vastly simplifies
the dsdb logic - we never have to worry about a user trying to
remove the boot flag from a host in dsdb.
"""
audit_results = []
dbhw_ent = Machine.get_unique(session, machine, compel=True)
dbinterface = Interface.get_unique(session, hardware_entity=dbhw_ent,
name=interface, compel=True)
oldinfo = DSDBRunner.snapshot_hw(dbhw_ent)
if arguments.get('hostname', None):
# Hack to set an intial interface for an aurora host...
dbhost = dbhw_ent.host
if dbhost.archetype.name == 'aurora' and \
dbhw_ent.primary_ip and not dbinterface.addresses:
assign_address(dbinterface, dbhw_ent.primary_ip,
dbhw_ent.primary_name.network, logger=logger)
# We may need extra IP verification (or an autoip option)...
# This may also throw spurious errors if attempting to set the
# port_group to a value it already has.
if pg is not None and dbinterface.port_group != pg.lower().strip():
dbinterface.port_group = verify_port_group(
dbinterface.hardware_entity, pg)
elif autopg:
dbinterface.port_group = choose_port_group(
session, logger, dbinterface.hardware_entity)
audit_results.append(('pg', dbinterface.port_group))
if master:
if dbinterface.addresses:
# FIXME: as a special case, if the only address is the
# primary IP, then we could just move it to the master
# interface. However this can be worked around by bonding
# the interface before calling "add host", so don't bother
# for now.
raise ArgumentError("Can not enslave {0:l} because it has "
"addresses.".format(dbinterface))
dbmaster = Interface.get_unique(session, hardware_entity=dbhw_ent,
name=master, compel=True)
if dbmaster in dbinterface.all_slaves():
raise ArgumentError("Enslaving {0:l} would create a circle, "
"which is not allowed.".format(dbinterface))
dbinterface.master = dbmaster
if clear_master:
if not dbinterface.master:
raise ArgumentError("{0} is not a slave.".format(dbinterface))
dbinterface.master = None
if comments:
dbinterface.comments = comments
if boot:
# Figure out if the current bootble interface also has the
# default route set; the new bootable interface probably
# wants to have the same settings. Note that if
# old_default_route is None there was no bootable interface.
old_default_route = None
for i in dbhw_ent.interfaces:
if i.bootable == True:
old_default_route = i.default_route
break
# Apply the bootable flag to the supplied interface, clearing
# it on all other interfaces.
for i in dbhw_ent.interfaces:
if i == dbinterface:
i.bootable = True
else:
i.bootable = False
# If the user was not explicit about the default route flag
# (default_route is None); there was an existing bootable
# interface (old_default_route is not None); the new default
# route setting differs from the old - then produce a warning.
if (default_route is None and
old_default_route is not None and
dbinterface.default_route != old_default_route):
if old_default_route:
logger.client_info("Warning: New boot interface {0} is no "
"longer provides the default route; it "
"did before!".format(dbinterface))
else:
logger.client_info("Warning: New boot interface {0} now "
"provides the default route; it didn't "
"before!".format(dbinterface))
# Should we also transfer the primary IP to the new boot interface?
# That could get tricky if the new interface already has an IP
# address...
if default_route is not None:
dbinterface.default_route = default_route
if not first_of(dbhw_ent.interfaces, lambda x: x.default_route):
logger.client_info("Warning: {0:l} has no default route, hope "
"that's ok.".format(dbhw_ent))
#Set this mac address last so that you can update to a bootable
#interface *before* adding a mac address. This is so the validation
#that takes place in the interface class doesn't have to be worried
#about the order of update to bootable=True and mac address
if mac:
q = session.query(Interface).filter_by(mac=mac)
other = q.first()
if other and other != dbinterface:
raise ArgumentError("MAC address {0} is already in use by "
"{1:l}.".format(mac, other))
dbinterface.mac = mac
if model or vendor:
if not dbinterface.model_allowed:
raise ArgumentError("Model/vendor can not be set for a {0:lc}."
.format(dbinterface))
dbmodel = Model.get_unique(session, name=model, vendor=vendor,
model_type=NicType.Nic, compel=True)
dbinterface.model = dbmodel
if rename_to:
rename_interface(session, dbinterface, rename_to)
session.flush()
plenaries = PlenaryCollection(logger=logger)
plenaries.append(Plenary.get_plenary(dbhw_ent))
# Interface renaming affects the host and service addresses
if dbhw_ent.host:
plenaries.append(Plenary.get_plenary(dbhw_ent.host))
for addr in dbinterface.assignments:
if addr.service_address:
plenaries.append(Plenary.get_plenary(addr.service_address))
with plenaries.get_key():
try:
plenaries.write(locked=True)
if dbhw_ent.host and dbhw_ent.host.archetype.name != "aurora":
dsdb_runner = DSDBRunner(logger=logger)
dsdb_runner.update_host(dbhw_ent, oldinfo)
dsdb_runner.commit_or_rollback()
except AquilonError, err:
plenaries.restore_stash()
raise ArgumentError(err)
except:
def render(self, session, logger, machine, disk, controller, share,
filesystem, resourcegroup, address, comments, size, boot,
snapshot, rename_to, **kw):
dbmachine = Machine.get_unique(session, machine, compel=True)
dbdisk = Disk.get_unique(session, device_name=disk, machine=dbmachine,
compel=True)
plenaries = PlenaryCollection(logger=logger)
plenaries.append(Plenary.get_plenary(dbmachine))
if rename_to:
Disk.get_unique(session, device_name=rename_to, machine=dbmachine,
preclude=True)
dbdisk.device_name = rename_to
if comments is not None:
dbdisk.comments = comments
if size is not None:
dbdisk.capacity = size
if controller:
if controller not in controller_types:
raise ArgumentError("%s is not a valid controller type, use "
"one of: %s." %
(controller, ", ".join(controller_types)))
dbdisk.controller_type = controller
if boot is not None:
dbdisk.bootable = boot
# There should be just one boot disk. We may need to re-think this
# if we want to model software RAID in the database.
for disk in dbmachine.disks:
if disk == dbdisk:
continue
if boot and disk.bootable:
disk.bootable = False
if address:
# TODO: do we really care? Bus address makes sense for physical
# disks as well, even if we cannot use that information today.
if not isinstance(dbdisk, VirtualDisk):
raise ArgumentError("Bus address can only be set for virtual "
"disks.")
dbdisk.address = address
if snapshot is not None:
if not isinstance(dbdisk, VirtualDisk):
raise ArgumentError("Snapshot capability can only be set for "
"virtual disks.")
dbdisk.snapshotable = snapshot
if share or filesystem:
if isinstance(dbdisk, VirtualNasDisk):
old_share = dbdisk.share
old_share.disks.remove(dbdisk)
elif isinstance(dbdisk, VirtualLocalDisk):
old_fs = dbdisk.filesystem
old_fs.disks.remove(dbdisk)
else:
raise ArgumentError("Disk {0!s} of {1:l} is not a virtual "
"disk, changing the backend store is not "
"possible.".format(dbdisk, dbmachine))
if share:
if not isinstance(dbdisk, VirtualNasDisk):
new_dbdisk = copy_virt_disk(session, VirtualNasDisk, dbdisk)
session.delete(dbdisk)
session.flush()
session.add(new_dbdisk)
dbdisk = new_dbdisk
new_share = find_resource(Share,
dbmachine.vm_container.holder.holder_object,
resourcegroup, share)
new_share.disks.append(dbdisk)
if filesystem:
if not isinstance(dbdisk, VirtualLocalDisk):
new_dbdisk = copy_virt_disk(session, VirtualLocalDisk, dbdisk)
session.delete(dbdisk)
session.flush()
session.add(new_dbdisk)
dbdisk = new_dbdisk
new_fs = find_resource(Filesystem,
dbmachine.vm_container.holder.holder_object,
resourcegroup, filesystem)
new_fs.disks.append(dbdisk)
session.flush()
plenaries.write()
return
def render(self, session, logger, machine, model, vendor, serial,
chassis, slot, clearchassis, multislot,
vmhost, cluster, allow_metacluster_change,
cpuname, cpuvendor, cpuspeed, cpucount, memory, ip, uri,
**arguments):
dbmachine = Machine.get_unique(session, machine, compel=True)
oldinfo = DSDBRunner.snapshot_hw(dbmachine)
plenaries = PlenaryCollection(logger=logger)
plenaries.append(Plenary.get_plenary(dbmachine))
if dbmachine.vm_container:
plenaries.append(Plenary.get_plenary(dbmachine.vm_container))
if dbmachine.host:
# Using PlenaryHostData directly, to avoid warnings if the host has
# not been configured yet
plenaries.append(PlenaryHostData.get_plenary(dbmachine.host))
if clearchassis:
del dbmachine.chassis_slot[:]
if chassis:
dbchassis = Chassis.get_unique(session, chassis, compel=True)
dbmachine.location = dbchassis.location
if slot is None:
raise ArgumentError("Option --chassis requires --slot "
"information.")
self.adjust_slot(session, logger,
dbmachine, dbchassis, slot, multislot)
elif slot is not None:
dbchassis = None
for dbslot in dbmachine.chassis_slot:
if dbchassis and dbslot.chassis != dbchassis:
raise ArgumentError("Machine in multiple chassis, please "
"use --chassis argument.")
dbchassis = dbslot.chassis
if not dbchassis:
raise ArgumentError("Option --slot requires --chassis "
"information.")
self.adjust_slot(session, logger,
dbmachine, dbchassis, slot, multislot)
dblocation = get_location(session, **arguments)
if dblocation:
loc_clear_chassis = False
for dbslot in dbmachine.chassis_slot:
dbcl = dbslot.chassis.location
if dbcl != dblocation:
if chassis or slot is not None:
raise ArgumentError("{0} conflicts with chassis {1!s} "
"location {2}."
.format(dblocation, dbslot.chassis,
dbcl))
else:
loc_clear_chassis = True
if loc_clear_chassis:
del dbmachine.chassis_slot[:]
dbmachine.location = dblocation
if dbmachine.host:
for vm in dbmachine.host.virtual_machines:
plenaries.append(Plenary.get_plenary(vm))
vm.location = dblocation
if model or vendor:
# If overriding model, should probably overwrite default
# machine specs as well.
if not model:
model = dbmachine.model.name
if not vendor:
vendor = dbmachine.model.vendor.name
dbmodel = Model.get_unique(session, name=model, vendor=vendor,
compel=True)
if not dbmodel.model_type.isMachineType():
raise ArgumentError("The update_machine command cannot update "
"machines of type %s." %
dbmodel.model_type)
# We probably could do this by forcing either cluster or
# location data to be available as appropriate, but really?
# Failing seems reasonable.
if dbmodel.model_type != dbmachine.model.model_type and \
(dbmodel.model_type.isVirtualMachineType() or
dbmachine.model.model_type.isVirtualMachineType()):
raise ArgumentError("Cannot change machine from %s to %s." %
(dbmachine.model.model_type,
dbmodel.model_type))
old_nic_model = dbmachine.model.nic_model
new_nic_model = dbmodel.nic_model
if old_nic_model != new_nic_model:
for iface in dbmachine.interfaces:
if iface.model == old_nic_model:
iface.model = new_nic_model
dbmachine.model = dbmodel
if cpuname or cpuvendor or cpuspeed is not None:
dbcpu = Cpu.get_unique(session, name=cpuname, vendor=cpuvendor,
speed=cpuspeed, compel=True)
dbmachine.cpu = dbcpu
if cpucount is not None:
dbmachine.cpu_quantity = cpucount
if memory is not None:
dbmachine.memory = memory
if serial:
dbmachine.serial_no = serial
if ip:
update_primary_ip(session, logger, dbmachine, ip)
if uri and not dbmachine.model.model_type.isVirtualAppliance():
raise ArgumentError("URI can be specified only for virtual "
"appliances and the model's type is %s" %
dbmachine.model.model_type)
if uri:
dbmachine.uri = uri
# FIXME: For now, if a machine has its interface(s) in a portgroup
# this command will need to be followed by an update_interface to
# re-evaluate the portgroup for overflow.
# It would be better to have --pg and --autopg options to let it
# happen at this point.
if cluster or vmhost:
if not dbmachine.vm_container:
raise ArgumentError("Cannot convert a physical machine to "
"virtual.")
old_holder = dbmachine.vm_container.holder.holder_object
resholder = get_resource_holder(session, hostname=vmhost,
cluster=cluster, compel=False)
new_holder = resholder.holder_object
if self.get_metacluster(new_holder) != self.get_metacluster(old_holder) \
and not allow_metacluster_change:
raise ArgumentError("Current {0:l} does not match "
"new {1:l}."
.format(self.get_metacluster(old_holder),
self.get_metacluster(new_holder)))
plenaries.append(Plenary.get_plenary(old_holder))
plenaries.append(Plenary.get_plenary(new_holder))
dbmachine.vm_container.holder = resholder
for dbdisk in dbmachine.disks:
if isinstance(dbdisk, VirtualNasDisk):
old_share = dbdisk.share
if isinstance(old_share.holder, BundleResource):
resourcegroup = old_share.holder.resourcegroup.name
else:
resourcegroup = None
new_share = find_resource(Share, new_holder, resourcegroup, old_share.name,
error=ArgumentError)
# If the shares are registered at the metacluster level and both
# clusters are in the same metacluster, then there will be no
# real change here
if new_share != old_share:
old_share.disks.remove(dbdisk)
new_share.disks.append(dbdisk)
if isinstance(dbdisk, VirtualLocalDisk):
old_filesystem = dbdisk.filesystem
new_filesystem = find_resource(Filesystem, new_holder, None,
old_filesystem.name,
error=ArgumentError)
if new_filesystem != old_filesystem:
old_filesystem.disks.remove(dbdisk)
new_filesystem.disks.append(dbdisk)
if isinstance(new_holder, Cluster):
dbmachine.location = new_holder.location_constraint
else:
# vmhost
dbmachine.location = new_holder.hardware_entity.location
session.flush()
# Check if the changed parameters still meet cluster capacity
# requiremets
if dbmachine.cluster:
dbmachine.cluster.validate()
if allow_metacluster_change:
dbmachine.cluster.metacluster.validate()
if dbmachine.host and dbmachine.host.cluster:
dbmachine.host.cluster.validate()
# The check to make sure a plenary file is not written out for
# dummy aurora hardware is within the call to write(). This way
# it is consistent without altering (and forgetting to alter)
# all the calls to the method.
with plenaries.get_key():
plenaries.stash()
try:
plenaries.write(locked=True)
dsdb_runner = DSDBRunner(logger=logger)
dsdb_runner.update_host(dbmachine, oldinfo)
dsdb_runner.commit_or_rollback("Could not update machine in DSDB")
except:
plenaries.restore_stash()
raise
return
