def compile(self, session, dbhost, logger, keepbindings):
chooser = Chooser(dbhost, logger=logger,
required_only=not(keepbindings))
chooser.set_required()
chooser.flush_changes()
hosts = chooser.changed_server_fqdns()
hosts.add(dbhost.fqdn)
# Force a host lock as pan might overwrite the profile...
key = chooser.get_write_key()
for fqdn in hosts:
key = CompileKey.merge([key, CompileKey(domain=dbhost.branch.name,
profile=fqdn,
logger=logger)])
try:
lock_queue.acquire(key)
chooser.write_plenary_templates(locked=True)
td = TemplateDomain(dbhost.branch, dbhost.sandbox_author,
logger=logger)
td.compile(session, only=hosts, locked=True)
except:
if chooser:
chooser.restore_stash()
# Okay, cleaned up templates, make sure the caller knows
# we've aborted so that DB can be appropriately rollback'd.
raise
finally:
lock_queue.release(key)
return
def render(self, session, logger, hostname, buildstatus, **arguments):
dbhost = hostname_to_host(session, hostname)
dbstatus = HostLifecycle.get_unique(session, buildstatus, compel=True)
changed = dbhost.status.transition(dbhost, dbstatus)
if not changed or not dbhost.archetype.is_compileable:
return
session.add(dbhost)
session.flush()
plenary = PlenaryHost(dbhost, logger=logger)
# Force a host lock as pan might overwrite the profile...
key = CompileKey(domain=dbhost.branch.name, profile=dbhost.fqdn,
logger=logger)
try:
lock_queue.acquire(key)
plenary.write(locked=True)
td = TemplateDomain(dbhost.branch, dbhost.sandbox_author,
logger=logger)
td.compile(session, only=[dbhost.fqdn], locked=True)
except IncompleteError:
raise ArgumentError("Run aq make for host %s first." % dbhost.fqdn)
except:
plenary.restore_stash()
raise
finally:
lock_queue.release(key)
def render(self, session, logger, cluster, keepbindings, **arguments):
dbcluster = Cluster.get_unique(session, cluster, compel=True)
if not dbcluster.personality.archetype.is_compileable:
raise ArgumentError(
"{0} is not a compilable archetype " "({1!s}).".format(dbcluster, dbcluster.personality.archetype)
)
chooser = Chooser(dbcluster, logger=logger, required_only=not keepbindings)
chooser.set_required()
chooser.flush_changes()
td = TemplateDomain(dbcluster.branch, dbcluster.sandbox_author, logger=logger)
# Force a domain lock as pan might overwrite any of the profiles...
with chooser.get_key():
try:
chooser.write_plenary_templates(locked=True)
td.compile(session, only=chooser.plenaries.object_templates, locked=True)
except:
chooser.restore_stash()
# Okay, cleaned up templates, make sure the caller knows
# we've aborted so that DB can be appropriately rollback'd.
raise
return
def render(self, session, logger, cluster, buildstatus, **arguments):
dbcluster = Cluster.get_unique(session, cluster, compel=True)
dbstatus = ClusterLifecycle.get_unique(
session, buildstatus, compel=True)
if not dbcluster.status.transition(dbcluster, dbstatus):
return
if not dbcluster.personality.archetype.is_compileable:
return
session.flush()
plenaries = PlenaryCollection(logger=logger)
plenaries.append(Plenary.get_plenary(dbcluster))
for dbhost in dbcluster.hosts:
plenaries.append(Plenary.get_plenary(dbhost))
# Force a host lock as pan might overwrite the profile...
key = CompileKey(domain=dbcluster.branch.name, logger=logger)
try:
lock_queue.acquire(key)
plenaries.write(locked=True)
td = TemplateDomain(
dbcluster.branch, dbcluster.sandbox_author, logger=logger)
td.compile(session, plenaries.object_templates, locked=True)
except:
plenaries.restore_stash()
raise
finally:
lock_queue.release(key)
return
def render(self, session, logger, hostname, buildstatus, **arguments):
dbhost = hostname_to_host(session, hostname)
dbstatus = HostLifecycle.get_unique(session, buildstatus, compel=True)
changed = dbhost.status.transition(dbhost, dbstatus)
if not changed or not dbhost.archetype.is_compileable:
return
session.add(dbhost)
session.flush()
plenary = PlenaryHost(dbhost, logger=logger)
# Force a host lock as pan might overwrite the profile...
key = CompileKey(domain=dbhost.branch.name,
profile=dbhost.fqdn,
logger=logger)
try:
lock_queue.acquire(key)
plenary.write(locked=True)
td = TemplateDomain(dbhost.branch,
dbhost.sandbox_author,
logger=logger)
td.compile(session, only=[dbhost.fqdn], locked=True)
except IncompleteError:
raise ArgumentError("Run aq make for host %s first." % dbhost.fqdn)
except:
plenary.restore_stash()
raise
finally:
lock_queue.release(key)
def render(self, session, logger, cluster, pancinclude, pancexclude, pancdebug, cleandeps, **arguments):
dbcluster = Cluster.get_unique(session, cluster, compel=True)
if pancdebug:
pancinclude = r".*"
pancexclude = r"components/spma/functions"
dom = TemplateDomain(dbcluster.branch, dbcluster.sandbox_author, logger=logger)
plenaries = PlenaryCollection(logger=logger)
def add_cluster_plenaries(cluster):
plenaries.append(Plenary.get_plenary(cluster))
for host in cluster.hosts:
plenaries.append(Plenary.get_plenary(host))
add_cluster_plenaries(dbcluster)
if isinstance(dbcluster, MetaCluster):
for cluster in dbcluster.members:
add_cluster_plenaries(cluster)
with plenaries.get_key():
dom.compile(
session,
only=plenaries.object_templates,
panc_debug_include=pancinclude,
panc_debug_exclude=pancexclude,
cleandeps=cleandeps,
locked=True,
)
return
def render(self, session, logger, domain, sandbox,
pancinclude, pancexclude, pancdebug, cleandeps,
**arguments):
(dbdomain, dbauthor) = get_branch_and_author(session, logger,
domain=domain,
sandbox=sandbox,
compel=True)
# Grab a shared lock on personalities and services used by the domain.
# Object templates (hosts, clusters) are protected by the domain lock.
plenaries = PlenaryCollection(logger=logger)
q1 = session.query(Personality)
q1 = q1.join(Host)
q1 = q1.filter(and_(Host.branch == dbdomain,
Host.sandbox_author == dbauthor))
q1 = q1.reset_joinpoint()
q1 = q1.options(joinedload('paramholder'),
subqueryload('paramholder.parameters'))
q2 = session.query(Personality)
q2 = q2.join(Cluster)
q2 = q2.filter(and_(Cluster.branch == dbdomain,
Cluster.sandbox_author == dbauthor))
q2 = q2.reset_joinpoint()
q2 = q2.options(joinedload('paramholder'),
subqueryload('paramholder.parameters'))
for dbpers in q1.union(q2):
plenaries.append(Plenary.get_plenary(dbpers))
q1 = session.query(ServiceInstance)
q1 = q1.join(ServiceInstance.clients)
q1 = q1.filter(and_(Host.branch == dbdomain,
Host.sandbox_author == dbauthor))
q2 = session.query(ServiceInstance)
q2 = q2.join(ServiceInstance.cluster_clients)
q2 = q2.filter(and_(Cluster.branch == dbdomain,
Cluster.sandbox_author == dbauthor))
for si in q1.union(q2):
plenaries.append(Plenary.get_plenary(si))
if pancdebug:
pancinclude = r'.*'
pancexclude = r'components/spma/functions'
dom = TemplateDomain(dbdomain, dbauthor, logger=logger)
with CompileKey.merge([CompileKey(domain=dbdomain.name, logger=logger),
plenaries.get_key(exclusive=False)]):
dom.compile(session,
panc_debug_include=pancinclude,
panc_debug_exclude=pancexclude,
cleandeps=cleandeps,
locked=True)
return
def render(self, session, logger, hostname,
pancinclude, pancexclude, pancdebug, cleandeps,
**arguments):
dbhost = hostname_to_host(session, hostname)
if pancdebug:
pancinclude = r'.*'
pancexclude = r'components/spma/functions'
dom = TemplateDomain(dbhost.branch, dbhost.sandbox_author,
logger=logger)
dom.compile(session, only=[dbhost.fqdn],
panc_debug_include=pancinclude,
panc_debug_exclude=pancexclude,
cleandeps=cleandeps)
return
def render(self, session, logger, hostname, pancinclude, pancexclude,
pancdebug, cleandeps, **arguments):
dbhost = hostname_to_host(session, hostname)
if pancdebug:
pancinclude = r'.*'
pancexclude = r'components/spma/functions'
dom = TemplateDomain(dbhost.branch,
dbhost.sandbox_author,
logger=logger)
dom.compile(session,
only=[dbhost.fqdn],
panc_debug_include=pancinclude,
panc_debug_exclude=pancexclude,
cleandeps=cleandeps)
return
def render(self, session, logger, domain, sandbox,
pancinclude, pancexclude, pancdebug, cleandeps,
**arguments):
(dbdomain, dbauthor) = get_branch_and_author(session, logger,
domain=domain,
sandbox=sandbox,
compel=True)
if pancdebug:
pancinclude = r'.*'
pancexclude = r'components/spma/functions'
dom = TemplateDomain(dbdomain, dbauthor, logger=logger)
dom.compile(session,
panc_debug_include=pancinclude,
panc_debug_exclude=pancexclude,
cleandeps=cleandeps)
return
def render(self, session, logger, cluster,
pancinclude, pancexclude, pancdebug, cleandeps,
**arguments):
dbclus = Cluster.get_unique(session, cluster, compel=True)
if pancdebug:
pancinclude = r'.*'
pancexclude = r'components/spma/functions'
dom = TemplateDomain(dbclus.branch, dbclus.sandbox_author,
logger=logger)
profile_list = add_cluster_data(dbclus)
dom.compile(session, only=profile_list,
panc_debug_include=pancinclude,
panc_debug_exclude=pancexclude,
cleandeps=cleandeps)
return
def render(self, session, logger, cluster, pancinclude, pancexclude,
pancdebug, cleandeps, **arguments):
dbclus = Cluster.get_unique(session, cluster, compel=True)
if pancdebug:
pancinclude = r'.*'
pancexclude = r'components/spma/functions'
dom = TemplateDomain(dbclus.branch,
dbclus.sandbox_author,
logger=logger)
profile_list = add_cluster_data(dbclus)
dom.compile(session,
only=profile_list,
panc_debug_include=pancinclude,
panc_debug_exclude=pancexclude,
cleandeps=cleandeps)
return
def compile(self, session, logger, dbhost):
""" compile plenary templates """
plenary = PlenaryHost(dbhost, logger=logger)
# Force a host lock as pan might overwrite the profile...
key = CompileKey(domain=dbhost.branch.name, profile=dbhost.fqdn,
logger=logger)
try:
lock_queue.acquire(key)
plenary.write(locked=True)
td = TemplateDomain(dbhost.branch, dbhost.sandbox_author,
logger=logger)
td.compile(session, only=[dbhost.fqdn], locked=True)
except IncompleteError:
raise ArgumentError("Run aq make for host %s first." % dbhost.fqdn)
except:
plenary.restore_stash()
raise
finally:
lock_queue.release(key)
def render(self, session, logger, cluster, keepbindings, **arguments):
dbcluster = Cluster.get_unique(session, cluster, compel=True)
if not dbcluster.personality.archetype.is_compileable:
raise ArgumentError("{0} is not a compilable archetype "
"({1!s}).".format(
dbcluster,
dbcluster.personality.archetype))
chooser = Chooser(dbcluster,
logger=logger,
required_only=not (keepbindings))
chooser.set_required()
chooser.flush_changes()
# Force a domain lock as pan might overwrite any of the profiles...
key = CompileKey.merge([
chooser.get_write_key(),
CompileKey(domain=dbcluster.branch.name, logger=logger)
])
try:
lock_queue.acquire(key)
chooser.write_plenary_templates(locked=True)
profile_list = add_cluster_data(dbcluster)
profile_list.extend(chooser.changed_server_fqdns())
td = TemplateDomain(dbcluster.branch,
dbcluster.sandbox_author,
logger=logger)
td.compile(session, only=profile_list, locked=True)
except:
chooser.restore_stash()
# Okay, cleaned up templates, make sure the caller knows
# we've aborted so that DB can be appropriately rollback'd.
raise
finally:
lock_queue.release(key)
return
def compile(self, session, logger, dbhost):
""" compile plenary templates """
plenary = PlenaryHost(dbhost, logger=logger)
# Force a host lock as pan might overwrite the profile...
key = CompileKey(domain=dbhost.branch.name,
profile=dbhost.fqdn,
logger=logger)
try:
lock_queue.acquire(key)
plenary.write(locked=True)
td = TemplateDomain(dbhost.branch,
dbhost.sandbox_author,
logger=logger)
td.compile(session, only=[dbhost.fqdn], locked=True)
except IncompleteError:
raise ArgumentError("Run aq make for host %s first." % dbhost.fqdn)
except:
plenary.restore_stash()
raise
finally:
lock_queue.release(key)
def compile(self, session, dbhost, logger, keepbindings):
chooser = Chooser(dbhost,
logger=logger,
required_only=not (keepbindings))
chooser.set_required()
chooser.flush_changes()
hosts = chooser.changed_server_fqdns()
hosts.add(dbhost.fqdn)
# Force a host lock as pan might overwrite the profile...
key = chooser.get_write_key()
for fqdn in hosts:
key = CompileKey.merge([
key,
CompileKey(domain=dbhost.branch.name,
profile=fqdn,
logger=logger)
])
try:
lock_queue.acquire(key)
chooser.write_plenary_templates(locked=True)
td = TemplateDomain(dbhost.branch,
dbhost.sandbox_author,
logger=logger)
td.compile(session, only=hosts, locked=True)
except:
if chooser:
chooser.restore_stash()
# Okay, cleaned up templates, make sure the caller knows
# we've aborted so that DB can be appropriately rollback'd.
raise
finally:
lock_queue.release(key)
return
def compile(self, session, dbhost, logger, keepbindings):
chooser = Chooser(dbhost, logger=logger,
required_only=not keepbindings)
chooser.set_required()
chooser.flush_changes()
td = TemplateDomain(dbhost.branch, dbhost.sandbox_author, logger=logger)
with chooser.get_key():
try:
chooser.write_plenary_templates(locked=True)
td.compile(session, only=chooser.plenaries.object_templates,
locked=True)
except:
chooser.restore_stash()
# Okay, cleaned up templates, make sure the caller knows
# we've aborted so that DB can be appropriately rollback'd.
raise
return
def render(self, session, logger, cluster, keepbindings, **arguments):
dbcluster = Cluster.get_unique(session, cluster, compel=True)
if not dbcluster.personality.archetype.is_compileable:
raise ArgumentError("{0} is not a compilable archetype "
"({1!s}).".format(dbcluster,
dbcluster.personality.archetype))
chooser = Chooser(dbcluster, logger=logger,
required_only=not(keepbindings))
chooser.set_required()
chooser.flush_changes()
# Force a domain lock as pan might overwrite any of the profiles...
key = CompileKey.merge([chooser.get_write_key(),
CompileKey(domain=dbcluster.branch.name,
logger=logger)])
try:
lock_queue.acquire(key)
chooser.write_plenary_templates(locked=True)
profile_list = add_cluster_data(dbcluster)
profile_list.extend(chooser.changed_server_fqdns())
td = TemplateDomain(dbcluster.branch, dbcluster.sandbox_author,
logger=logger)
td.compile(session, only=profile_list, locked=True)
except:
chooser.restore_stash()
# Okay, cleaned up templates, make sure the caller knows
# we've aborted so that DB can be appropriately rollback'd.
raise
finally:
lock_queue.release(key)
return
def render(self, session, logger, cluster, buildstatus, **arguments):
dbcluster = Cluster.get_unique(session, cluster, compel=True)
dbstatus = ClusterLifecycle.get_unique(session,
buildstatus,
compel=True)
if not dbcluster.status.transition(dbcluster, dbstatus):
return
if not dbcluster.personality.archetype.is_compileable:
return
session.flush()
plenaries = PlenaryCollection(logger=logger)
plenaries.append(Plenary.get_plenary(dbcluster))
for dbhost in dbcluster.hosts:
plenaries.append(Plenary.get_plenary(dbhost))
# Force a host lock as pan might overwrite the profile...
key = CompileKey(domain=dbcluster.branch.name, logger=logger)
try:
lock_queue.acquire(key)
plenaries.write(locked=True)
td = TemplateDomain(dbcluster.branch,
dbcluster.sandbox_author,
logger=logger)
td.compile(session, plenaries.object_templates, locked=True)
except:
plenaries.restore_stash()
raise
finally:
lock_queue.release(key)
return
def resetadvertisedstatus_list(self, session, logger, dbhosts):
branches = {}
authors = {}
failed = []
compileable = []
# Do any cross-list or dependency checks
for dbhost in dbhosts:
## if archetype is compileable only then
## validate for branches and domains
if (dbhost.archetype.is_compileable):
compileable.append(dbhost.fqdn)
if dbhost.branch in branches:
branches[dbhost.branch].append(dbhost)
else:
branches[dbhost.branch] = [dbhost]
if dbhost.sandbox_author in authors:
authors[dbhost.sandbox_author].append(dbhost)
else:
authors[dbhost.sandbox_author] = [dbhost]
if dbhost.status.name == 'ready':
failed.append("{0:l} is in ready status, "
"advertised status can be reset only "
"when host is in non ready state".format(dbhost))
if failed:
raise ArgumentError("Cannot modify the following hosts:\n%s" %
"\n".join(failed))
if len(branches) > 1:
keys = branches.keys()
branch_sort = lambda x, y: cmp(len(branches[x]), len(branches[y]))
keys.sort(cmp=branch_sort)
stats = ["{0:d} hosts in {1:l}".format(len(branches[branch]),
branch)
for branch in keys]
raise ArgumentError("All hosts must be in the same domain or "
"sandbox:\n%s" % "\n".join(stats))
if len(authors) > 1:
keys = authors.keys()
author_sort = lambda x, y: cmp(len(authors[x]), len(authors[y]))
keys.sort(cmp=author_sort)
stats = ["%s hosts with sandbox author %s" %
(len(authors[author]), author.name) for author in keys]
raise ArgumentError("All hosts must be managed by the same "
"sandbox author:\n%s" % "\n".join(stats))
plenaries = PlenaryCollection(logger=logger)
for dbhost in dbhosts:
dbhost.advertise_status = False
session.add(dbhost)
plenaries.append(PlenaryHost(dbhost, logger=logger))
session.flush()
dbbranch = branches.keys()[0]
dbauthor = authors.keys()[0]
key = CompileKey.merge([plenaries.get_write_key()])
try:
lock_queue.acquire(key)
plenaries.stash()
plenaries.write(locked=True)
td = TemplateDomain(dbbranch, dbauthor, logger=logger)
td.compile(session, only=compileable, locked=True)
except:
plenaries.restore_stash()
raise
finally:
lock_queue.release(key)
return
def resetadvertisedstatus_list(self, session, logger, dbhosts):
branches = {}
authors = {}
failed = []
compileable = []
# Do any cross-list or dependency checks
for dbhost in dbhosts:
## if archetype is compileable only then
## validate for branches and domains
if (dbhost.archetype.is_compileable):
compileable.append(dbhost.fqdn)
if dbhost.branch in branches:
branches[dbhost.branch].append(dbhost)
else:
branches[dbhost.branch] = [dbhost]
if dbhost.sandbox_author in authors:
authors[dbhost.sandbox_author].append(dbhost)
else:
authors[dbhost.sandbox_author] = [dbhost]
if dbhost.status.name == 'ready':
failed.append("{0:l} is in ready status, "
"advertised status can be reset only "
"when host is in non ready state".format(dbhost))
if failed:
raise ArgumentError("Cannot modify the following hosts:\n%s" %
"\n".join(failed))
if len(branches) > 1:
keys = branches.keys()
branch_sort = lambda x, y: cmp(len(branches[x]), len(branches[y]))
keys.sort(cmp=branch_sort)
stats = [
"{0:d} hosts in {1:l}".format(len(branches[branch]), branch)
for branch in keys
]
raise ArgumentError("All hosts must be in the same domain or "
"sandbox:\n%s" % "\n".join(stats))
if len(authors) > 1:
keys = authors.keys()
author_sort = lambda x, y: cmp(len(authors[x]), len(authors[y]))
keys.sort(cmp=author_sort)
stats = [
"%s hosts with sandbox author %s" %
(len(authors[author]), author.name) for author in keys
]
raise ArgumentError("All hosts must be managed by the same "
"sandbox author:\n%s" % "\n".join(stats))
plenaries = PlenaryCollection(logger=logger)
for dbhost in dbhosts:
dbhost.advertise_status = False
session.add(dbhost)
plenaries.append(PlenaryHost(dbhost, logger=logger))
session.flush()
dbbranch = branches.keys()[0]
dbauthor = authors.keys()[0]
key = CompileKey.merge([plenaries.get_write_key()])
try:
lock_queue.acquire(key)
plenaries.stash()
plenaries.write(locked=True)
td = TemplateDomain(dbbranch, dbauthor, logger=logger)
td.compile(session, only=compileable, locked=True)
except:
plenaries.restore_stash()
raise
finally:
lock_queue.release(key)
return
