def get_upcoming_events():
"""Get the global list of upcoming events"""
from indico.modules.events import Event
data = upcoming_events_settings.get_all()
if not data['max_entries'] or not data['entries']:
return
tz = timezone(Config.getInstance().getDefaultTimezone())
now = now_utc(False).astimezone(tz)
base_query = (Event.query
.filter(Event.effective_protection_mode == ProtectionMode.public,
~Event.is_deleted,
Event.end_dt.astimezone(tz) > now)
.options(load_only('id', 'title', 'start_dt', 'end_dt')))
queries = []
cols = {'category': Event.category_id,
'event': Event.id}
for entry in data['entries']:
delta = timedelta(days=entry['days'])
query = (base_query
.filter(cols[entry['type']] == entry['id'])
.filter(db.cast(Event.start_dt.astimezone(tz), db.Date) > (now - delta).date())
.with_entities(Event, db.literal(entry['weight']).label('weight')))
queries.append(query)
query = (queries[0].union(*queries[1:])
.order_by(db.desc('weight'), Event.start_dt, Event.title)
.limit(data['max_entries']))
for row in query:
event = row[0]
# we cache the result of the function and is_deleted is used in the repr
# and having a broken repr on the cached objects would be ugly
set_committed_value(event, 'is_deleted', False)
yield event
def new_version(self, session):
# convert to an INSERT
make_transient(self)
self.id = None
# history of the 'elements' collection.
# this is a tuple of groups: (added, unchanged, deleted)
hist = attributes.get_history(self, "elements")
# rewrite the 'elements' collection
# from scratch, removing all history
attributes.set_committed_value(self, "elements", {})
# new elements in the "added" group
# are moved to our new collection.
for elem in hist.added:
self.elements[elem.name] = elem
# copy elements in the 'unchanged' group.
# the new ones associate with the new ConfigData,
# the old ones stay associated with the old ConfigData
for elem in hist.unchanged:
self.elements[elem.name] = ConfigValueAssociation(
elem.config_value
)
def validate(self, max_hosts=None, error=ArgumentError, **kwargs):
session = object_session(self)
q = session.query(HostClusterMember)
q = q.filter_by(cluster=self)
q = q.options(joinedload('host'), joinedload('host.machine'))
members = q.all()
set_committed_value(self, '_hosts', members)
if self.cluster_type != 'meta':
for i in [
"down_hosts_threshold", "down_hosts_percent",
"down_maint_percent", "personality_id"
#"branch_id"
]:
if getattr(self, i, None) is None:
raise error("Attribute %s must be set for a %s cluster." %
(i, self.cluster_type))
else:
if self.metacluster:
raise error("Metaclusters can't contain other metaclusters.")
if max_hosts is None:
max_hosts = self.max_hosts
if len(self.hosts) > self.max_hosts:
raise error("{0} is over capacity of {1} hosts.".format(
self, max_hosts))
if self.metacluster:
self.metacluster.validate()
def get_upcoming_events():
"""Get the global list of upcoming events"""
from indico.modules.events import Event
data = upcoming_events_settings.get_all()
if not data['max_entries'] or not data['entries']:
return
tz = timezone(config.DEFAULT_TIMEZONE)
now = now_utc(False).astimezone(tz)
base_query = (Event.query
.filter(Event.effective_protection_mode == ProtectionMode.public,
~Event.is_deleted,
Event.end_dt.astimezone(tz) > now)
.options(load_only('id', 'title', 'start_dt', 'end_dt')))
queries = []
cols = {'category': Event.category_id,
'event': Event.id}
for entry in data['entries']:
delta = timedelta(days=entry['days'])
query = (base_query
.filter(cols[entry['type']] == entry['id'])
.filter(db.cast(Event.start_dt.astimezone(tz), db.Date) > (now - delta).date())
.with_entities(Event, db.literal(entry['weight']).label('weight')))
queries.append(query)
query = (queries[0].union(*queries[1:])
.order_by(db.desc('weight'), Event.start_dt, Event.title)
.limit(data['max_entries']))
for row in query:
event = row[0]
# we cache the result of the function and is_deleted is used in the repr
# and having a broken repr on the cached objects would be ugly
set_committed_value(event, 'is_deleted', False)
yield event
def _populate_preloaded_relationships(cls, target, *unused):
cache = g.get('relationship_cache', {}).get(type(target))
if not cache:
return
for rel, value in cache['data'].get(target, {}).iteritems():
if rel not in target.__dict__:
set_committed_value(target, rel, value)
def populated_tree(cats):
"""Return the root categories with children populated to any depth.
Adjacency lists are notoriously inefficient for fetching deeply
nested trees, and since our dataset will always be reasonably
small, this method should greatly improve efficiency. Only one
query is necessary to fetch a tree of any depth. This isn't
always the solution, but for some situations, it is worthwhile.
For example, printing the entire tree can be done with one query::
query = Category.query.options(undefer('media_count'))
for cat, depth in query.populated_tree().traverse():
print " " * depth, cat.name, '(%d)' % cat.media_count
Without this method, especially with the media_count undeferred,
this would require a lot of extra queries for nested categories.
NOTE: If the tree contains circular nesting, the circular portion
of the tree will be silently omitted from the results.
"""
children = defaultdict(CategoryList)
for cat in cats:
children[cat.parent_id].append(cat)
for cat in cats:
set_committed_value(cat, 'children', children[cat.id])
return children[None]
def _populate_preloaded_relationships(cls, target, *unused):
cache = g.get('relationship_cache', {}).get(type(target))
if not cache:
return
for rel, value in cache['data'].get(target, {}).iteritems():
if rel not in target.__dict__:
set_committed_value(target, rel, value)
def _bulk_load_column_for_instance_states(
session: Session, mapper: Mapper, identities: Iterable[Tuple],
attr_name: str, alter_query: Optional[QueryAlterator]):
""" Load a column attribute for a list of instance states where the attribute is unloaded """
Model = mapper.class_
attr: Column = mapper.columns[attr_name]
# Using those identities (primary keys), load the missing attribute
q = load_by_primary_keys(session, mapper, identities, attr)
# Alter the query
if alter_query:
q = alter_query(q, mapper, attr_name, False)
# Having the missing attribute's value loaded, assign it to every instance in the session
for identity, attr_value in q:
# Build the identity key the way SqlAlchemy likes it:
# (Model, primary-key, None)
key = identity_key(Model, identity)
# We do not iterate the Session to find an instance that matches the primary key.
# Instead, we take it directly using the `identity_map`
instance = session.identity_map[key]
# Set the value of the missing attribute.
# This is how it immediately becomes loaded.
# Note that this action does not overwrite any modifications made to the attribute.
set_committed_value(instance, attr_name, attr_value)
def populated_tree(cats):
"""Return the root categories with children populated to any depth.
Adjacency lists are notoriously inefficient for fetching deeply
nested trees, and since our dataset will always be reasonably
small, this method should greatly improve efficiency. Only one
query is necessary to fetch a tree of any depth. This isn't
always the solution, but for some situations, it is worthwhile.
For example, printing the entire tree can be done with one query::
query = Category.query.options(undefer('media_count'))
for cat, depth in query.populated_tree().traverse():
print " " * depth, cat.name, '(%d)' % cat.media_count
Without this method, especially with the media_count undeferred,
this would require a lot of extra queries for nested categories.
NOTE: If the tree contains circular nesting, the circular portion
of the tree will be silently omitted from the results.
"""
roots = CategoryList()
children = defaultdict(list)
for cat in cats:
if cat.parent_id:
children[cat.parent_id].append(cat)
else:
roots.append(cat)
for cat in cats:
attributes.set_committed_value(cat, 'children', children[cat.id])
return roots
def new_version(self, session):
# convert to an INSERT
make_transient(self)
self.id = None
# history of the 'elements' collection.
# this is a tuple of groups: (added, unchanged, deleted)
hist = attributes.get_history(self, "elements")
# rewrite the 'elements' collection
# from scratch, removing all history
attributes.set_committed_value(self, "elements", {})
# new elements in the "added" group
# are moved to our new collection.
for elem in hist.added:
self.elements[elem.name] = elem
# copy elements in the 'unchanged' group.
# the new ones associate with the new ConfigData,
# the old ones stay associated with the old ConfigData
for elem in hist.unchanged:
self.elements[elem.name] = ConfigValueAssociation(
elem.config_value
)
def validate(self, max_hosts=None, error=ArgumentError, **kwargs):
session = object_session(self)
q = session.query(HostClusterMember)
q = q.filter_by(cluster=self)
q = q.options(joinedload('host'),
joinedload('host.machine'))
members = q.all()
set_committed_value(self, '_hosts', members)
if self.cluster_type != 'meta':
for i in [
"down_hosts_threshold",
"down_hosts_percent",
"down_maint_percent",
"personality_id"
#"branch_id"
]:
if getattr(self, i, None) is None:
raise error("Attribute %s must be set for a %s cluster." %
(i, self.cluster_type))
else:
if self.metacluster:
raise error("Metaclusters can't contain other metaclusters.")
if max_hosts is None:
max_hosts = self.max_hosts
if len(self.hosts) > self.max_hosts:
raise error("{0} is over capacity of {1} hosts.".format(self,
max_hosts))
if self.metacluster:
self.metacluster.validate()
def intercept_after_flush(session, obj):
if isinstance(obj, QuestionModel):
print(obj)
if obj.questionKey == "set":
obj.set_questionKey()
session.query(QuestionModel).filter_by(id=obj.id).update(
{"questionKey": obj.questionKey})
set_committed_value(obj, "questionKey", obj.questionKey)
def populate(self):
"""
Populate batch fetched entities to parent objects.
"""
for entity in self.path.entities:
set_committed_value(
entity, self.prop.key,
self.parent_dict[local_values(self.prop, entity)])
def populate(self):
"""
Populate batch fetched entities to parent objects.
"""
for entity in self.path.entities:
set_committed_value(
entity,
self.prop.key,
self.parent_dict[local_values(self.prop, entity)]
)
def sa_set_committed_state(obj: object, **committed_values):
""" Put values into an SqlAlchemy instance as if they were committed to the DB """
# Give it some DB identity so that SA thinks it can load something
state: InstanceState = instance_state(obj)
state.key = object()
# Set every attribute in such a way that SA thinkg that's the way it looks in the DB
for k, v in committed_values.items():
set_committed_value(obj, k, v)
return obj
def populate(self):
"""
Populate batch fetched entities to parent objects.
"""
for entity in self.path.entities:
set_committed_value(
entity,
self.prop.key,
self.parent_dict[self.local_values(entity)]
)
if self.path.populate_backrefs:
self.populate_backrefs(self.related_entities)
def _set_results_on_models(self, param_value_to_results,
param_value_to_models, current_model):
current_model_result = None
for value, models in param_value_to_models.items():
for model in models:
result = param_value_to_results.get(value, [])
# ensure models aren't given identical result lists so modifying results in one doesn't modify others
result = result[:]
if not self.uselist:
result = self._extract_non_list_result(result)
if model == current_model:
current_model_result = result
attributes.set_committed_value(model, self.key, result)
return current_model_result
def populate_backrefs(self, related_entities):
"""
Populates backrefs for given related entities.
"""
backref_dict = dict((local_values(self.prop, value[0]), [])
for value in related_entities)
for value in related_entities:
backref_dict[local_values(self.prop, value[0])].append(
self.path.session.query(self.path.entities[0].__class__).get(
tuple(value[1:])))
for value in related_entities:
set_committed_value(
value[0], self.prop.back_populates,
backref_dict[local_values(self.prop, value[0])])
def get_unique(cls,
session,
fqdn=None,
name=None,
dns_domain=None,
dns_environment=None,
compel=False,
preclude=False,
**kwargs):
# Proxy FQDN lookup to the Fqdn class
if not fqdn or not isinstance(fqdn, Fqdn):
if not isinstance(dns_environment, DnsEnvironment):
dns_environment = DnsEnvironment.get_unique_or_default(
session, dns_environment)
if fqdn:
if name or dns_domain: # pragma: no cover
raise TypeError("fqdn and name/dns_domain cannot be mixed")
(name, dns_domain) = parse_fqdn(session, fqdn)
try:
# Do not pass preclude=True to Fqdn
fqdn = Fqdn.get_unique(session,
name=name,
dns_domain=dns_domain,
dns_environment=dns_environment,
compel=compel)
except NotFoundException:
# Replace the "Fqdn ... not found" message with a more user
# friendly one
msg = "%s %s.%s, %s not found." % (
cls._get_class_label(), name, dns_domain,
format(dns_environment, "l"))
raise NotFoundException(msg)
if not fqdn:
return None
# We already have the FQDN, no need to load it again
if "query_options" not in kwargs:
kwargs["query_options"] = [lazyload("fqdn")]
result = super(DnsRecord, cls).get_unique(session,
fqdn=fqdn,
compel=compel,
preclude=preclude,
**kwargs)
if result:
# Make sure not to load the relation again if we already know its
# value
set_committed_value(result, 'fqdn', fqdn)
return result
def validate(self):
session = object_session(self)
q = session.query(HostClusterMember)
q = q.filter_by(cluster=self)
q = q.options(joinedload('host'),
joinedload('host.hardware_entity'))
members = q.all()
set_committed_value(self, '_hosts', members)
if self.max_hosts is not None and len(self.hosts) > self.max_hosts:
raise ArgumentError("{0} has {1} hosts bound, which exceeds the "
"requested limit of {2}."
.format(self, len(self.hosts), self.max_hosts))
if self.metacluster:
self.metacluster.validate()
def populate_related(parents, id_key, res_key, reltype, subq, flt=None, relid_key='id'): ids = [] for p in parents: if callable(flt) and not flt(p): continue keyval = getattr(p, id_key, None) if keyval and (keyval not in ids): ids.append(keyval) if len(ids) > 0: for rel in subq.filter(getattr(reltype, relid_key).in_(ids)): for p in parents: if callable(flt) and not flt(p): continue keyval = getattr(p, id_key, None) if keyval and (keyval == getattr(rel, relid_key)): attributes.set_committed_value(p, res_key, rel)
def increment_views(self):
"""Increment the number of views in the database.
We avoid concurrency issues by incrementing JUST the views and
not allowing modified_on to be updated automatically.
"""
if self.id is None:
self.views += 1
return self.views
DBSession.execute(media.update().values(views=media.c.views + 1).where(media.c.id == self.id))
# Increment the views by one for the rest of the request,
# but don't allow the ORM to increment the views too.
attributes.set_committed_value(self, "views", self.views + 1)
return self.views
def fetch_as_tree(query: Query,
relationship: Optional[InstrumentedAttribute] = None,
root_value: Union[str, None] = None,
) -> List[DeclarativeMeta]:
"""Builds the full tree for the given table (adjacency list).
This method is efficient in the sense that it issues only one SELECT
query to the database.
Args:
query: A query that selects all relevant model records, that
is, all records that make up the adjacency list. For
example: `session.query(Album).order_by(Album.nameLC)`.
relationship: The parent/child relationship that describes
both fields (columns). For example: `Album.parent`
If not set, one record is fetched from the query to read
its `parent` property.
root_value: Parent property's value at the root nodes.
Default is None, which commonly retrieves the whole
structure.
Returns:
A list of root nodes with child nodes (the tree) pre-fetched
recursively.
"""
if relationship is None:
# Fetch one record to discover the parent relationship.
relationship = next(iter(query)).__class__.parent
parent_field = relationship.expression.right.name
child_field = relationship.expression.left.name
back_populates = relationship.property.back_populates
nodes = query.all()
children = defaultdict(list)
for node in nodes:
children[getattr(node, parent_field)].append(node)
for node in nodes:
set_committed_value(node, back_populates,
children[getattr(node, child_field)])
return children[root_value]
def populate(self):
"""
Populate batch fetched entities to parent objects.
"""
for entity in self.path.entities:
# print (
# "setting committed value for ",
# entity,
# " using local values ",
# self.local_values(entity)
# )
set_committed_value(
entity,
self.prop.key,
self.parent_dict[self.local_values(entity)]
)
if self.path.populate_backrefs:
self.populate_backrefs(self.related_entities)
def load_projects():
projects = session.query(Project).options(joinedload('quotas')).all()
root = None
children = defaultdict(list)
for project in projects:
if project.parent:
children[project.parent.id].append(project)
if project.parent is None:
root = project
for project in projects:
set_committed_value(project, 'children', children[project.id])
project.limits = {
quota.resource: quota.limit
for quota in project.quotas
}
return root
def increment_views(self):
"""Increment the number of views in the database.
We avoid concurrency issues by incrementing JUST the views and
not allowing modified_on to be updated automatically.
"""
if self.id is None:
self.views += 1
return self.views
DBSession.execute(media.update()\
.values(views=media.c.views + 1)\
.where(media.c.id == self.id))
# Increment the views by one for the rest of the request,
# but don't allow the ORM to increment the views too.
attributes.set_committed_value(self, 'views', self.views + 1)
return self.views
def populate_related_list(parents, id_key, res_key, reltype, subq,
flt=None, relid_key='id'):
ids = []
for p in parents:
if callable(flt) and not flt(p):
continue
keyval = getattr(p, id_key, None)
if keyval and keyval not in ids:
ids.append(keyval)
if len(ids) > 0:
ch = dict((k, list(v)) for k, v in groupby(
subq.filter(getattr(reltype, relid_key).in_(ids)),
lambda c: getattr(c, relid_key)))
for p in parents:
if callable(flt) and not flt(p):
continue
keyval = getattr(p, id_key, None)
if keyval:
attributes.set_committed_value(p, res_key, ch.get(keyval, ()))
def listtracksbyalbums(self, ffilter=None, skip=None, limit=None):
with self.session_scope() as session:
query = session.query(Album).join(Album.tracks).outerjoin(
TrackInfo.genre).options(
contains_eager(Album.tracks, TrackInfo.album,
Album.artist),
joinedload(Album.tracks, TrackInfo.genre))
if ffilter is not None:
query = ffilter(query)
query = limitoffset(
query.order_by(Album.year.desc(), TrackInfo.trackno), skip,
limit)
qall = query.all()
# Force artist population
for x in qall:
for y in x.tracks:
attributes.set_committed_value(y, 'album', x)
attributes.set_committed_value(y, 'artist', x.artist)
session.expunge_all()
return qall
def populate_backrefs(self, related_entities):
"""
Populates backrefs for given related entities.
"""
backref_dict = dict(
(local_values(self.prop, value[0]), [])
for value in related_entities
)
for value in related_entities:
backref_dict[local_values(self.prop, value[0])].append(
self.path.session.query(self.path.entities[0].__class__).get(
tuple(value[1:])
)
)
for value in related_entities:
set_committed_value(
value[0],
self.prop.back_populates,
backref_dict[local_values(self.prop, value[0])]
)
def populate_related(parents,
id_key,
res_key,
reltype,
subq,
flt=None,
relid_key='id'):
ids = []
for p in parents:
if callable(flt) and not flt(p):
continue
keyval = getattr(p, id_key, None)
if keyval and (keyval not in ids):
ids.append(keyval)
if len(ids) > 0:
for rel in subq.filter(getattr(reltype, relid_key).in_(ids)):
for p in parents:
if callable(flt) and not flt(p):
continue
keyval = getattr(p, id_key, None)
if keyval and (keyval == getattr(rel, relid_key)):
attributes.set_committed_value(p, res_key, rel)
def get_unique(cls, session, fqdn=None, name=None, dns_domain=None,
dns_environment=None, compel=False, preclude=False, **kwargs):
# Proxy FQDN lookup to the Fqdn class
if not fqdn or not isinstance(fqdn, Fqdn):
if not isinstance(dns_environment, DnsEnvironment):
dns_environment = DnsEnvironment.get_unique_or_default(session,
dns_environment)
if fqdn:
if name or dns_domain: # pragma: no cover
raise TypeError("fqdn and name/dns_domain cannot be mixed")
(name, dns_domain) = parse_fqdn(session, fqdn)
try:
# Do not pass preclude=True to Fqdn
fqdn = Fqdn.get_unique(session, name=name,
dns_domain=dns_domain,
dns_environment=dns_environment,
compel=compel)
except NotFoundException:
# Replace the "Fqdn ... not found" message with a more user
# friendly one
msg = "%s %s.%s, %s not found." % (cls._get_class_label(),
name, dns_domain,
format(dns_environment, "l"))
raise NotFoundException(msg)
if not fqdn:
return None
# We already have the FQDN, no need to load it again
if "query_options" not in kwargs:
kwargs["query_options"] = [lazyload("fqdn")]
result = super(DnsRecord, cls).get_unique(session, fqdn=fqdn,
compel=compel,
preclude=preclude, **kwargs)
if result:
# Make sure not to load the relation again if we already know its
# value
set_committed_value(result, 'fqdn', fqdn)
return result
def preload_interfaces(self, session, hosts, interfaces_by_id,
interfaces_by_hwent):
addrs_by_iface = defaultdict(list)
slaves_by_id = defaultdict(list)
# Polymorphic loading cannot be applied to eager-loaded
# attributes, so load interfaces manually.
q = session.query(Interface)
q = q.with_polymorphic('*')
q = q.options(lazyload("hardware_entity"))
for iface in q:
interfaces_by_hwent[iface.hardware_entity_id].append(iface)
interfaces_by_id[iface.id] = iface
if iface.master_id:
slaves_by_id[iface.master_id].append(iface)
# subqueryload() and with_polymorphic() do not play nice
# together, so do it by hand
q = session.query(AddressAssignment)
q = q.options(joinedload("network"),
joinedload("dns_records"))
q = q.order_by(AddressAssignment._label)
# Machine templates want the management interface only
if hosts:
q = q.options(subqueryload("network.static_routes"),
subqueryload("network.routers"))
else:
q = q.join(AddressAssignment.interface.of_type(ManagementInterface))
for addr in q:
addrs_by_iface[addr.interface_id].append(addr)
for iface_id, iface in interfaces_by_id.iteritems():
set_committed_value(iface, "assignments",
addrs_by_iface.get(iface_id, None))
set_committed_value(iface, "slaves",
slaves_by_id.get(iface_id, None))
def test_only_loads_relations_on_unpopulated_models(self):
User = self.classes.User
Address = self.classes.Address
session = fixture_session()
users = session.query(User).order_by(self.tables.users.c.id.asc()).all()
address = session.query(Address).filter(self.tables.addresses.c.id == 1).first()
# pre-load the address for the first user
attributes.set_committed_value(users[0], "addresses", [address])
self.queries = []
# make sure no relations are loaded
for user in users[1:]:
model_dict = attributes.instance_dict(user)
assert "addresses" not in model_dict
# trigger a lazy load
users[1].addresses
# only 1 query should have been generated to load all the child relationships
assert len(self.queries) == 1
unpopulated_user_ids = [user.id for user in users[1:]]
assert self.queries[0]["parameters"] == tuple(unpopulated_user_ids)
def domain_hosts(self, domain): from netprofile_hosts.models import Host from netprofile_ipaddresses.models import ( IPv4Address, IPv6Address ) # XXX: this is a hack, but it reduces poor ol' MySQL's crunch time by a factor of 10 sess = DBSession() hipv4 = dict((k, list(v)) for k, v in groupby( sess.query(IPv4Address).join(IPv4Address.host).filter(Host.domain_id == domain.id), lambda ip: ip.host_id )) hipv6 = dict((k, list(v)) for k, v in groupby( sess.query(IPv6Address).join(IPv6Address.host).filter(Host.domain_id == domain.id), lambda ip: ip.host_id )) for host in domain.hosts: attributes.set_committed_value(host, 'ipv4_addresses', hipv4.get(host.id, ())) attributes.set_committed_value(host, 'ipv6_addresses', hipv6.get(host.id, ())) return domain.hosts
def populate_related_list(parents,
id_key,
res_key,
reltype,
subq,
flt=None,
relid_key='id'):
ids = []
for p in parents:
if callable(flt) and not flt(p):
continue
keyval = getattr(p, id_key, None)
if keyval and (keyval not in ids):
ids.append(keyval)
if len(ids) > 0:
ch = dict((k, list(v)) for k, v in groupby(
subq.filter(getattr(reltype, relid_key).in_(ids)),
lambda c: getattr(c, relid_key)))
for p in parents:
if callable(flt) and not flt(p):
continue
keyval = getattr(p, id_key, None)
if keyval:
attributes.set_committed_value(p, res_key, ch.get(keyval, ()))
def listtracksbyalbumsbyartists(self,
ffilter=None,
skip=None,
limit=None,
order_by=(Artist.name, Album.year.desc(),
TrackInfo.trackno,
TrackInfo.name)):
with self.session_scope() as session:
query = session.query(Artist).join(Artist.albums).join(
Album.tracks).join(TrackInfo.track).options(
contains_eager(Artist.albums, Album.tracks))
if ffilter is not None:
query = ffilter(query)
query = limitoffset(query.order_by(*order_by), skip, limit)
qall = query.all()
# Force artist population
for x in qall:
for y in x.albums:
attributes.set_committed_value(y, 'artist', x)
for z in y.tracks:
attributes.set_committed_value(z, 'album', y)
attributes.set_committed_value(z, 'artist', x)
session.expunge_all()
return qall
def __init__(self, dbhost, *args, **kwargs):
"""Provide initialization specific for host bindings."""
if not isinstance(dbhost, Host):
raise InternalError("HostChooser can only choose services for "
"hosts, got %r (%s)" % (dbhost, type(dbhost)))
super(HostChooser, self).__init__(dbhost, *args, **kwargs)
self.location = dbhost.hardware_entity.location
# If the primary name is a ReservedName, then it does not have a network
# attribute
if hasattr(dbhost.hardware_entity.primary_name, 'network'):
self.network = dbhost.hardware_entity.primary_name.network
else:
self.network = None
# all of them would be self. but that should be optimized
# dbhost.hardware_entity.interfaces[x].assignments[y].network
# Stores interim service instance lists.
q = self.session.query(Service)
q = q.outerjoin(Service.archetypes)
q = q.reset_joinpoint()
q = q.outerjoin(Service.personalities)
q = q.filter(or_(Archetype.id == self.archetype.id,
Personality.id == self.personality.id))
self.required_services = set(q.all())
self.original_service_instances = {}
# Cache of any already bound services (keys) and the instance
# that was bound (values).
q = self.session.query(ServiceInstance)
q = q.options(undefer('_client_count'))
q = q.filter(ServiceInstance.clients.contains(dbhost))
set_committed_value(dbhost, 'services_used', q.all())
for si in dbhost.services_used:
self.original_service_instances[si.service] = si
self.logger.debug("{0} original binding: {1}"
.format(self.dbobj, si))
self.cluster_aligned_services = {}
if dbhost.cluster:
# Note that cluster services are currently ignored unless
# they are otherwise required by the archetype/personality.
for si in dbhost.cluster.service_bindings:
self.cluster_aligned_services[si.service] = si
for service in dbhost.cluster.required_services:
if service not in self.cluster_aligned_services:
# Don't just error here because the error() call
# has not yet been set up. Will error out later.
self.cluster_aligned_services[service] = None
# Went back and forth on this... deciding not to force
# an aligned service as required. This should give
# flexibility for multiple services to be aligned for
# a cluster type without being forced on all the
# personalities.
#self.required_services.add(item.service)
if dbhost.cluster.metacluster:
mc = dbhost.cluster.metacluster
for si in mc.service_bindings:
if si.service in self.cluster_aligned_services:
cas = self.cluster_aligned_services[si.service]
if cas == None:
# Error out later.
continue
self.logger.client_info(
"Replacing {0.name} instance with {1.name} "
"(bound to {2:l}) for service {3.name}".format(
cas, si, mc, si.service))
self.cluster_aligned_services[si.service] = si
for service in mc.required_services:
if service not in self.cluster_aligned_services:
# Don't just error here because the error() call
# has not yet been set up. Will error out later.
self.cluster_aligned_services[service] = None
def _set_updated_revision_number(self, revision_number, updated_at):
attributes.set_committed_value(self, "revision_number",
revision_number)
attributes.set_committed_value(self, "updated_at", updated_at)
def render(self, session, logger, hostname, **arguments):
# Check dependencies, translate into user-friendly message
dbhost = hostname_to_host(session, hostname)
dbhost.lock_row()
check_no_provided_service(dbhost)
# Any service bindings that we need to clean up afterwards
plenaries = PlenaryCollection(logger=logger)
remove_plenaries = PlenaryCollection(logger=logger)
remove_plenaries.append(Plenary.get_plenary(dbhost))
archetype = dbhost.archetype.name
dbmachine = dbhost.hardware_entity
oldinfo = DSDBRunner.snapshot_hw(dbmachine)
ip = dbmachine.primary_ip
for si in dbhost.services_used:
plenaries.append(PlenaryServiceInstanceServer.get_plenary(si))
logger.info("Before deleting {0:l}, removing binding to {1:l}"
.format(dbhost, si))
del dbhost.services_used[:]
if dbhost.resholder:
for res in dbhost.resholder.resources:
remove_plenaries.append(Plenary.get_plenary(res))
# In case of Zebra, the IP may be configured on multiple interfaces
for iface in dbmachine.interfaces:
if ip in iface.addresses:
iface.addresses.remove(ip)
if dbhost.cluster:
dbcluster = dbhost.cluster
dbcluster.hosts.remove(dbhost)
set_committed_value(dbhost, '_cluster', None)
dbcluster.validate()
plenaries.append(Plenary.get_plenary(dbcluster))
dbdns_rec = dbmachine.primary_name
dbmachine.primary_name = None
dbmachine.host = None
session.delete(dbhost)
delete_dns_record(dbdns_rec)
session.flush()
if dbmachine.vm_container:
plenaries.append(Plenary.get_plenary(dbmachine.vm_container))
with CompileKey.merge([plenaries.get_key(),
remove_plenaries.get_key()]):
plenaries.stash()
remove_plenaries.stash()
try:
plenaries.write(locked=True)
remove_plenaries.remove(locked=True, remove_profile=True)
if archetype != 'aurora' and ip is not None:
dsdb_runner = DSDBRunner(logger=logger)
dsdb_runner.update_host(dbmachine, oldinfo)
dsdb_runner.commit_or_rollback("Could not remove host %s from "
"DSDB" % hostname)
if archetype == 'aurora':
logger.client_info("WARNING: removing host %s from AQDB and "
"*not* changing DSDB." % hostname)
except:
plenaries.restore_stash()
remove_plenaries.restore_stash()
raise
trigger_notifications(self.config, logger, CLIENT_INFO)
return
# TODO: only if not hosts
manager_addrs = defaultdict(list)
q = session.query(AddressAssignment)
q = q.join(Interface)
q = q.filter_by(interface_type="management")
q = q.options(contains_eager("interface"),
joinedload("dns_records"),
lazyload("interface.hardware_entity"))
for addr in q:
manager_addrs[addr.interface.id].append(addr)
for interface_id, addrs in manager_addrs.items():
if interface_id not in interfaces_by_id:
# Should not happen...
continue
addrs.sort(key=attrgetter("label"))
set_committed_value(interfaces_by_id[interface_id],
"assignments", addrs)
q = session.query(Machine)
q = q.options(lazyload("host"), lazyload("primary_name"),
subqueryload("chassis_slot"))
cnt = q.count()
idx = 0
for machine in q:
idx += 1
if idx % 1000 == 0: # pragma: no cover
logger.client_info("Processing machine %d of %d..." %
(idx, cnt))
if machine.id in disks_by_machine:
disks_by_machine[machine.id].sort(
def _populate_backrefs(target, context):
for name, backref in mappings.iteritems():
# __dict__ to avoid triggering lazy-loaded relationships
if target.__dict__.get(name) is not None:
set_committed_value(getattr(target, name), backref, target)
def add_net_msg(self, net_msg, net):
net_msg.name = str(net.name)
net_msg.id = net.id
net_msg.ip = str(net.ip)
net_msg.cidr = net.cidr
net_msg.bcast = str(net.broadcast)
net_msg.netmask = str(net.netmask)
net_msg.side = str(net.side)
net_msg.sysloc = str(net.location.sysloc())
net_msg.location.name = str(net.location.name)
net_msg.location.location_type = str(net.location.location_type)
net_msg.type = str(net.network_type)
# Bulk load information about anything having a network address on this
# network
hw_ids = set(
[addr.interface.hardware_entity_id for addr in net.assignments])
if hw_ids:
session = object_session(net)
q = session.query(HardwareEntity)
q = q.filter(HardwareEntity.id.in_(hw_ids))
q = q.options(subqueryload('interfaces'))
hwent_by_id = {}
for dbhwent in q.all():
hwent_by_id[dbhwent.id] = dbhwent
iface_by_id = {}
slaves_by_id = defaultdict(list)
# We have all the interfaces loaded already, so compute the
# master/slave relationships to avoid having to touch the
# database again
for iface in dbhwent.interfaces:
iface_by_id[iface.id] = iface
if iface.master_id:
slaves_by_id[iface.master_id].append(iface)
for iface in dbhwent.interfaces:
set_committed_value(iface, "master",
iface_by_id.get(iface.master_id, None))
set_committed_value(iface, "slaves",
slaves_by_id[iface.id])
# TODO: once we refactor Host to be an FK to HardwareEntity instead
# of Machine, this could be converted to a single joinedload('host')
q = session.query(Host)
q = q.options(lazyload('machine'))
q = q.filter(Host.machine_id.in_(hw_ids))
for host in q.all():
set_committed_value(hwent_by_id[host.machine_id], "host", host)
set_committed_value(host, "machine",
hwent_by_id[host.machine_id])
# Add interfaces that have addresses in this network
for addr in net.assignments:
if not addr.dns_records:
# hostname is a required field in the protobuf description
continue
hwent = addr.interface.hardware_entity
# DHCP: we do not care about secondary IP addresses, but in some
# cases the same IP address may show up with different MACs
if not addr.label:
mac_addrs = possible_mac_addresses(addr.interface)
else:
mac_addrs = []
# Generate a host record even if there is no known MAC address for
# it
if not mac_addrs:
mac_addrs.append(None)
# Associating the same IP with multiple MAC addresses is
# problematic using the current protocol. Sending multiple host
# messages is easy for the broker, but it can confuse consumers like
# aqdhcpd. For now just ensure it never happens, and revisit the
# problem when we have a real world requirement.
if len(mac_addrs) > 1:
mac_addrs = [mac_addrs[0]]
for mac in mac_addrs:
host_msg = net_msg.hosts.add()
if addr.interface.interface_type == 'management':
host_msg.type = 'manager'
else:
if hwent.hardware_type == 'machine':
host_msg.type = 'host'
if hwent.host:
host_msg.archetype.name = str(
hwent.host.archetype.name)
elif hwent.hardware_type == 'switch':
# aqdhcpd uses the type
host_msg.type = 'tor_switch'
else:
host_msg.type = hwent.hardware_type
host_msg.hostname = str(addr.dns_records[0].fqdn.name)
host_msg.fqdn = str(addr.dns_records[0].fqdn)
host_msg.dns_domain = str(addr.dns_records[0].fqdn.dns_domain)
host_msg.ip = str(addr.ip)
if mac:
host_msg.mac = mac
host_msg.machine.name = str(hwent.label)
# aqdhcpd uses the interface list when excluding hosts it is not
# authoritative for
for iface in hwent.interfaces:
int_msg = host_msg.machine.interfaces.add()
int_msg.device = iface.name
if iface.mac:
int_msg.mac = str(iface.mac)
# Add dynamic DHCP records
for dynhost in net.dynamic_stubs:
host_msg = net_msg.hosts.add()
# aqdhcpd uses the type
host_msg.type = 'dynamic_stub'
host_msg.hostname = str(dynhost.fqdn.name)
host_msg.fqdn = str(dynhost.fqdn)
host_msg.dns_domain = str(dynhost.fqdn.dns_domain)
host_msg.ip = str(dynhost.ip)
# TODO: only if not hosts
manager_addrs = defaultdict(list)
q = session.query(AddressAssignment)
q = q.join(Interface)
q = q.filter_by(interface_type="management")
q = q.options(contains_eager("interface"),
joinedload("dns_records"),
lazyload("interface.hardware_entity"))
for addr in q:
manager_addrs[addr.interface.id].append(addr)
for interface_id, addrs in manager_addrs.items():
if interface_id not in interfaces_by_id:
# Should not happen...
continue
addrs.sort(key=attrgetter("label"))
set_committed_value(interfaces_by_id[interface_id],
"assignments", addrs)
q = session.query(Machine)
q = q.options(lazyload("host"),
lazyload("primary_name"),
subqueryload("chassis_slot"))
cnt = q.count()
idx = 0
for machine in q:
idx += 1
if idx % 1000 == 0: # pragma: no cover
logger.client_info("Processing machine %d of %d..." %
(idx, cnt))
if machine.id in disks_by_machine:
def get_unique(cls, sess, name, hardware_type=None, compel=False,
preclude=False, query_options=None):
""" Returns a unique HardwareEntity given session and fqdn """
# If the hardware_type param isn't explicitly set and we have a
# polymorphic identity, assume we're querying only for items of our
# hardware_type.
if hardware_type:
if isclass(hardware_type):
clslabel = hardware_type._get_class_label()
hardware_type = hardware_type.__mapper_args__['polymorphic_identity']
else:
pcls = cls.__mapper__.polymorphic_map[hardware_type].class_
clslabel = pcls._get_class_label()
else:
if 'polymorphic_identity' in cls.__mapper_args__:
hardware_type = cls.__mapper_args__['polymorphic_identity']
clslabel = cls._get_class_label()
# The automagic DNS lookup does not really make sense with preclude=True
if preclude:
name = AqStr.normalize(name)
cls.check_label(name)
q = sess.query(cls)
if "." in name:
dns_rec = DnsRecord.get_unique(sess, fqdn=name, compel=True)
# We know the primary name, do not load it again
q = q.options(lazyload('primary_name'))
q = q.filter_by(primary_name=dns_rec)
else:
dns_rec = None
q = q.filter_by(label=name)
if query_options:
q = q.options(*query_options)
try:
hwe = q.one()
except NoResultFound:
# Check if the name is in use by a different hardware type
q = sess.query(HardwareEntity)
if dns_rec:
# We know the primary name, do not load it again
q = q.options(lazyload('primary_name'))
q = q.filter_by(primary_name=dns_rec)
else:
q = q.filter_by(label=name)
try:
hwe = q.one()
if dns_rec:
# We know the primary name, do not load it again
set_committed_value(hwe, 'primary_name', dns_rec)
raise ArgumentError("{0} exists, but is not a {1}."
.format(hwe, clslabel.lower()))
except NoResultFound:
hwe = None
if compel:
raise NotFoundException("%s %s not found." % (clslabel, name))
if hwe:
if preclude:
raise ArgumentError('{0} already exists.'.format(hwe))
if dns_rec:
# We know the primary name, do not load it again
set_committed_value(hwe, 'primary_name', dns_rec)
return hwe
chassis = q.all() # pylint: disable=W0612
q = session.query(Machine)
q = q.options(lazyload("host"),
lazyload("primary_name"),
subqueryload("chassis_slot"))
cnt = q.count()
idx = 0
for machine in q:
idx += 1
if idx % 1000 == 0: # pragma: no cover
logger.client_info("Processing machine %d of %d..." %
(idx, cnt))
set_committed_value(machine, 'disks',
disks_by_machine.get(machine.id, None))
set_committed_value(machine, 'interfaces',
interfaces_by_hwent.get(machine.id, None))
try:
plenary_info = Plenary.get_plenary(machine,
logger=logger)
written += plenary_info.write(locked=True)
except Exception, e:
failed.append("{0} failed: {1}".format(machine, e))
continue
if hosts:
logger.client_info("Flushing hosts.")
q = session.query(Cluster)
def _set_updated_revision_number(self, revision_number, updated_at):
attributes.set_committed_value(
self, "revision_number", revision_number)
attributes.set_committed_value(
self, "updated_at", updated_at)
def __init__(self, dbobj, *args, **kwargs):
"""Provide initialization specific for host bindings."""
if not isinstance(dbobj, Host):
raise InternalError("HostChooser can only choose services for "
"hosts, got %r (%s)" % (dbobj, type(dbobj)))
self.dbhost = dbobj
Chooser.__init__(self, dbobj, *args, **kwargs)
self.location = self.dbhost.machine.location
self.archetype = self.dbhost.archetype
self.personality = self.dbhost.personality
# If the primary name is a ReservedName, then it does not have a network
# attribute
if hasattr(self.dbhost.machine.primary_name, 'network'):
self.network = self.dbhost.machine.primary_name.network
else:
self.network = None
# all of them would be self. but that should be optimized
# dbhost.machine.interfaces[x].assignments[y].network
"""Stores interim service instance lists."""
q = self.session.query(Service)
q = q.outerjoin(Service.archetypes)
q = q.reset_joinpoint()
q = q.outerjoin(Service.personalities)
q = q.filter(
or_(Archetype.id == self.archetype.id,
Personality.id == self.personality.id))
self.required_services = set(q.all())
self.original_service_instances = {}
"""Cache of any already bound services (keys) and the instance
that was bound (values).
"""
q = self.session.query(ServiceInstance)
q = q.options(undefer('_client_count'))
q = q.filter(ServiceInstance.clients.contains(self.dbhost))
set_committed_value(self.dbhost, 'services_used', q.all())
for si in self.dbhost.services_used:
self.original_service_instances[si.service] = si
self.logger.debug("%s original binding: %s", self.description,
si.cfg_path)
self.cluster_aligned_services = {}
if self.dbhost.cluster:
# Note that cluster services are currently ignored unless
# they are otherwise required by the archetype/personality.
for si in self.dbhost.cluster.service_bindings:
self.cluster_aligned_services[si.service] = si
for service in self.dbhost.cluster.required_services:
if service not in self.cluster_aligned_services:
# Don't just error here because the error() call
# has not yet been set up. Will error out later.
self.cluster_aligned_services[service] = None
# Went back and forth on this... deciding not to force
# an aligned service as required. This should give
# flexibility for multiple services to be aligned for
# a cluster type without being forced on all the
# personalities.
#self.required_services.add(item.service)
if self.dbhost.cluster.metacluster:
mc = self.dbhost.cluster.metacluster
for si in mc.service_bindings:
if si.service in self.cluster_aligned_services:
cas = self.cluster_aligned_services[si.service]
if cas == None:
# Error out later.
continue
self.logger.client_info(
"Replacing {0.name} instance with {1.name} "
"(bound to {2:l}) for service {3.name}".format(
cas, si, mc, si.service))
self.cluster_aligned_services[si.service] = si
for service in mc.required_services:
if service not in self.cluster_aligned_services:
# Don't just error here because the error() call
# has not yet been set up. Will error out later.
self.cluster_aligned_services[service] = None
def render(self, session, logger,
services, personalities, machines, clusters, hosts,
locations, resources, switches, all,
**arguments):
success = []
failed = []
written = 0
# Caches for keeping preloaded data pinned in memory, since the SQLA
# session holds a weak reference only
resource_by_id = {}
resholder_by_id = {}
service_instances = None
racks = None
# Object caches that are accessed directly
disks_by_machine = defaultdict(list)
interfaces_by_machine = defaultdict(list)
interfaces_by_id = {}
if all:
services = True
personalities = True
machines = True
clusters = True
hosts = True
locations = True
resources = True
with CompileKey(logger=logger):
logger.client_info("Loading data.")
# When flushing clusters/hosts, loading the resource holder is done
# as the query that loads those objects. But when flushing resources
# only, we need the holder and the object it belongs to.
if resources and not clusters:
q = session.query(ClusterResource)
# Using joinedload('cluster') would generate an outer join
q = q.join(Cluster)
q = q.options(contains_eager('cluster'))
for resholder in q:
resholder_by_id[resholder.id] = resholder
if resources and not hosts:
q = session.query(HostResource)
# Using joinedload('host') would generate an outer join
q = q.join(Host)
q = q.options(contains_eager('host'))
for resholder in q:
resholder_by_id[resholder.id] = resholder
if hosts or clusters or resources:
# Load the most common resource types. Using
# with_polymorphic('*') on Resource would generate a huge query,
# so do something more targeted. More resource subclasses may be
# added later if they become common.
preload_classes = {
Filesystem: [],
RebootSchedule: [],
VirtualMachine: [joinedload('machine'),
joinedload('machine.primary_name'),
joinedload('machine.primary_name.fqdn')],
Share: [],
}
share_info = cache_storage_data()
for cls, options in preload_classes.items():
q = session.query(cls)
# If only hosts or only clusters are needed, don't load
# resources of the other kind
if hosts and not clusters and not resources:
q = q.join(ResourceHolder)
q = q.options(contains_eager('holder'))
q = q.filter_by(holder_type='host')
if clusters and not hosts and not resources:
q = q.join(ResourceHolder)
q = q.filter_by(holder_type='cluster')
q = q.options(contains_eager('holder'))
if options:
q = q.options(*options)
for res in q:
resource_by_id[res.id] = res
try:
res.populate_share_info(share_info)
except AttributeError:
pass
if hosts or machines:
# Polymorphic loading cannot be applied to eager-loaded
# attributes, so load interfaces manually.
q = session.query(Interface)
q = q.with_polymorphic('*')
q = q.options(lazyload("hardware_entity"))
for iface in q:
interfaces_by_machine[iface.hardware_entity_id].append(iface)
interfaces_by_id[iface.id] = iface
if hosts:
# subqueryload() and with_polymorphic() do not play nice
# together, so do it by hand
q = session.query(AddressAssignment)
q = q.options(joinedload("network"),
joinedload("dns_records"))
q = q.order_by(AddressAssignment._label)
addrs_by_iface = defaultdict(list)
for addr in q:
addrs_by_iface[addr.interface_id].append(addr)
for interface_id, addrs in addrs_by_iface.items():
set_committed_value(interfaces_by_id[interface_id],
"assignments", addrs)
q = session.query(Interface.id)
q = q.filter(~Interface.assignments.any())
for id in q.all():
set_committed_value(interfaces_by_id[id[0]],
"assignments", None)
if hosts or services:
q = session.query(ServiceInstance)
q = q.options(subqueryload("service"))
service_instances = q.all()
if machines or clusters:
# Most machines are in racks...
q = session.query(Rack)
q = q.options(subqueryload("dns_maps"),
subqueryload("parents"))
racks = q.all()
if locations:
logger.client_info("Flushing locations.")
for dbloc in session.query(City).all():
try:
plenary = Plenary.get_plenary(dbloc, logger=logger)
written += plenary.write(locked=True)
except Exception, e:
failed.append("City %s failed: %s" %
dbloc, e)
continue
if services:
logger.client_info("Flushing services.")
q = session.query(Service)
q = q.options(subqueryload("instances"))
for dbservice in q:
try:
plenary_info = Plenary.get_plenary(dbservice,
logger=logger)
written += plenary_info.write(locked=True)
except Exception, e:
failed.append("Service %s failed: %s" %
(dbservice.name, e))
continue
for dbinst in dbservice.instances:
try:
plenary_info = Plenary.get_plenary(dbinst,
logger=logger)
written += plenary_info.write(locked=True)
except Exception, e:
failed.append("Service %s instance %s failed: %s" %
(dbservice.name, dbinst.name, e))
continue
def render(self, session, logger, hostname, **arguments):
# removing the plenary host requires a compile lock, however
# we want to avoid deadlock by the fact that we're messing
# with two locks here, so we want to be careful. We grab the
# plenaryhost early on (in order to get the filenames filled
# in from the db info before we delete it from the db. We then
# hold onto those references until we've completed the db
# cleanup and if all of that is successful, then we delete the
# plenary file (which doesn't require re-evaluating any stale
# db information) after we've released the delhost lock.
delplenary = False
# Any service bindings that we need to clean up afterwards
bindings = PlenaryCollection(logger=logger)
resources = PlenaryCollection(logger=logger)
with DeleteKey("system", logger=logger) as key:
# Check dependencies, translate into user-friendly message
dbhost = hostname_to_host(session, hostname)
host_plenary = Plenary.get_plenary(dbhost, logger=logger)
domain = dbhost.branch.name
deps = get_host_dependencies(session, dbhost)
if (len(deps) != 0):
deptext = "\n".join([" %s" % d for d in deps])
raise ArgumentError("Cannot delete host %s due to the "
"following dependencies:\n%s." %
(hostname, deptext))
archetype = dbhost.archetype.name
dbmachine = dbhost.machine
oldinfo = DSDBRunner.snapshot_hw(dbmachine)
ip = dbmachine.primary_ip
fqdn = dbmachine.fqdn
for si in dbhost.services_used:
plenary = PlenaryServiceInstanceServer(si)
bindings.append(plenary)
logger.info(
"Before deleting host '%s', removing binding '%s'" %
(fqdn, si.cfg_path))
del dbhost.services_used[:]
if dbhost.resholder:
for res in dbhost.resholder.resources:
resources.append(Plenary.get_plenary(res))
# In case of Zebra, the IP may be configured on multiple interfaces
for iface in dbmachine.interfaces:
if ip in iface.addresses:
iface.addresses.remove(ip)
if dbhost.cluster:
dbcluster = dbhost.cluster
dbcluster.hosts.remove(dbhost)
set_committed_value(dbhost, '_cluster', None)
dbcluster.validate()
dbdns_rec = dbmachine.primary_name
dbmachine.primary_name = None
dbmachine.host = None
session.delete(dbhost)
delete_dns_record(dbdns_rec)
session.flush()
delplenary = True
if dbmachine.vm_container:
bindings.append(Plenary.get_plenary(dbmachine.vm_container))
if archetype != 'aurora' and ip is not None:
dsdb_runner = DSDBRunner(logger=logger)
dsdb_runner.update_host(dbmachine, oldinfo)
dsdb_runner.commit_or_rollback("Could not remove host %s from "
"DSDB" % hostname)
if archetype == 'aurora':
logger.client_info("WARNING: removing host %s from AQDB and "
"*not* changing DSDB." % hostname)
# Past the point of no return... commit the transaction so
# that we can free the delete lock.
session.commit()
# Only if we got here with no exceptions do we clean the template
# Trying to clean up after any errors here is really difficult
# since the changes to dsdb have already been made.
if (delplenary):
key = host_plenary.get_remove_key()
with CompileKey.merge(
[key,
bindings.get_write_key(),
resources.get_remove_key()]) as key:
host_plenary.cleanup(domain, locked=True)
# And we also want to remove the profile itself
profiles = self.config.get("broker", "profilesdir")
# Only one of these should exist, but it doesn't hurt
# to try to clean up both.
xmlfile = os.path.join(profiles, fqdn + ".xml")
remove_file(xmlfile, logger=logger)
xmlgzfile = xmlfile + ".gz"
remove_file(xmlgzfile, logger=logger)
# And the cached template created by ant
remove_file(os.path.join(
self.config.get("broker", "quattordir"), "objects",
fqdn + TEMPLATE_EXTENSION),
logger=logger)
bindings.write(locked=True)
resources.remove(locked=True)
build_index(self.config, session, profiles, logger=logger)
return
def add_net_data(self, net_msg, net):
net_msg.name = str(net.name)
net_msg.id = net.id
net_msg.ip = str(net.ip)
net_msg.cidr = net.cidr
net_msg.bcast = str(net.broadcast)
net_msg.netmask = str(net.netmask)
net_msg.side = str(net.side)
net_msg.sysloc = str(net.location.sysloc())
net_msg.location.name = str(net.location.name)
net_msg.location.location_type = str(net.location.location_type)
net_msg.type = str(net.network_type)
# Bulk load information about anything having a network address on this
# network
hw_ids = set([addr.interface.hardware_entity_id for addr in
net.assignments])
if hw_ids:
session = object_session(net)
q = session.query(HardwareEntity)
q = q.filter(HardwareEntity.id.in_(hw_ids))
q = q.options(subqueryload('interfaces'),
lazyload('interfaces.hardware_entity'))
hwent_by_id = {}
for dbhwent in q.all():
hwent_by_id[dbhwent.id] = dbhwent
iface_by_id = {}
slaves_by_id = defaultdict(list)
# We have all the interfaces loaded already, so compute the
# master/slave relationships to avoid having to touch the
# database again
for iface in dbhwent.interfaces:
iface_by_id[iface.id] = iface
if iface.master_id:
slaves_by_id[iface.master_id].append(iface)
for iface in dbhwent.interfaces:
set_committed_value(iface, "master",
iface_by_id.get(iface.master_id, None))
set_committed_value(iface, "slaves", slaves_by_id[iface.id])
# TODO: once we refactor Host to be an FK to HardwareEntity instead
# of Machine, this could be converted to a single joinedload('host')
q = session.query(Host)
q = q.options(lazyload('hardware_entity'))
q = q.filter(Host.hardware_entity_id.in_(hw_ids))
for host in q.all():
set_committed_value(hwent_by_id[host.hardware_entity_id], "host", host)
set_committed_value(host, "hardware_entity",
hwent_by_id[host.hardware_entity_id])
# Add interfaces that have addresses in this network
for addr in net.assignments:
if not addr.dns_records:
# hostname is a required field in the protobuf description
continue
hwent = addr.interface.hardware_entity
# DHCP: we do not care about secondary IP addresses, but in some
# cases the same IP address may show up with different MACs
if not addr.label:
mac_addrs = possible_mac_addresses(addr.interface)
else:
mac_addrs = []
# Generate a host record even if there is no known MAC address for
# it
if not mac_addrs:
mac_addrs.append(None)
# Associating the same IP with multiple MAC addresses is
# problematic using the current protocol. Sending multiple host
# messages is easy for the broker, but it can confuse consumers like
# aqdhcpd. For now just ensure it never happens, and revisit the
# problem when we have a real world requirement.
if len(mac_addrs) > 1:
mac_addrs = [mac_addrs[0]]
for mac in mac_addrs:
host_msg = net_msg.hosts.add()
if addr.interface.interface_type == 'management':
host_msg.type = 'manager'
else:
if hwent.hardware_type == 'machine':
host_msg.type = 'host'
if hwent.host:
host_msg.archetype.name = str(hwent.host.archetype.name)
elif hwent.hardware_type == 'switch':
# aqdhcpd uses the type
host_msg.type = 'tor_switch'
else:
host_msg.type = hwent.hardware_type
host_msg.hostname = str(addr.dns_records[0].fqdn.name)
host_msg.fqdn = str(addr.dns_records[0].fqdn)
host_msg.dns_domain = str(addr.dns_records[0].fqdn.dns_domain)
host_msg.ip = str(addr.ip)
if mac:
host_msg.mac = mac
host_msg.machine.name = str(hwent.label)
# aqdhcpd uses the interface list when excluding hosts it is not
# authoritative for
for iface in hwent.interfaces:
int_msg = host_msg.machine.interfaces.add()
int_msg.device = iface.name
if iface.mac:
int_msg.mac = str(iface.mac)
# Add dynamic DHCP records
for dynhost in net.dynamic_stubs:
host_msg = net_msg.hosts.add()
# aqdhcpd uses the type
host_msg.type = 'dynamic_stub'
host_msg.hostname = str(dynhost.fqdn.name)
host_msg.fqdn = str(dynhost.fqdn)
host_msg.dns_domain = str(dynhost.fqdn.dns_domain)
host_msg.ip = str(dynhost.ip)
def create_transaction(self, session):
"""
Create transaction object for given SQLAlchemy session.
:param session: SQLAlchemy session object
"""
args = self.transaction_args(session)
Transaction = self.manager.transaction_cls
table = Transaction.__table__
self.current_transaction = Transaction()
if self.manager.options['native_versioning']:
has_transaction_initialized = bool(session.execute(
'''SELECT 1 FROM pg_catalog.pg_class
WHERE relname = 'temporary_transaction'
'''
).scalar())
if has_transaction_initialized:
tx_id = (
session.execute('SELECT id FROM temporary_transaction')
.scalar()
)
set_committed_value(self.current_transaction, 'id', tx_id)
else:
criteria = {'native_tx_id': sa.func.txid_current()}
args.update(criteria)
query = (
table.insert()
.values(**args)
.returning(*map(sa.text, list(args.keys()) + ['id']))
)
values = session.execute(query).fetchone()
for key, value in values.items():
set_committed_value(self.current_transaction, key, value)
session.execute(
'''
CREATE TEMP TABLE temporary_transaction
(id BIGINT, PRIMARY KEY(id))
ON COMMIT DROP
'''
)
session.execute('''
INSERT INTO temporary_transaction (id)
VALUES (:id)
''',
{'id': self.current_transaction.id}
)
self.merge_transaction(session, self.current_transaction)
else:
for key, value in args.items():
setattr(self.current_transaction, key, value)
if not self.version_session:
self.version_session = sa.orm.session.Session(
bind=session.connection()
)
self.version_session.add(self.current_transaction)
self.version_session.flush()
self.version_session.expunge(self.current_transaction)
session.add(self.current_transaction)
return self.current_transaction
def render(self, session, logger, services, personalities, machines,
clusters, hosts, locations, resources, switches, all,
**arguments):
success = []
failed = []
written = 0
# Caches for keeping preloaded data pinned in memory, since the SQLA
# session holds a weak reference only
resource_by_id = {}
resholder_by_id = {}
service_instances = None
racks = None
# Object caches that are accessed directly
disks_by_machine = defaultdict(list)
interfaces_by_machine = defaultdict(list)
interfaces_by_id = {}
if all:
services = True
personalities = True
machines = True
clusters = True
hosts = True
locations = True
resources = True
with CompileKey(logger=logger):
logger.client_info("Loading data.")
# When flushing clusters/hosts, loading the resource holder is done
# as the query that loads those objects. But when flushing resources
# only, we need the holder and the object it belongs to.
if resources and not clusters:
q = session.query(ClusterResource)
# Using joinedload('cluster') would generate an outer join
q = q.join(Cluster)
q = q.options(contains_eager('cluster'))
for resholder in q:
resholder_by_id[resholder.id] = resholder
if resources and not hosts:
q = session.query(HostResource)
# Using joinedload('host') would generate an outer join
q = q.join(Host)
q = q.options(contains_eager('host'))
for resholder in q:
resholder_by_id[resholder.id] = resholder
if hosts or clusters or resources:
# Load the most common resource types. Using
# with_polymorphic('*') on Resource would generate a huge query,
# so do something more targeted. More resource subclasses may be
# added later if they become common.
preload_classes = {
Filesystem: [],
RebootSchedule: [],
VirtualMachine: [
joinedload('machine'),
joinedload('machine.primary_name'),
joinedload('machine.primary_name.fqdn')
],
Share: [],
}
share_info = cache_storage_data()
for cls, options in preload_classes.items():
q = session.query(cls)
# If only hosts or only clusters are needed, don't load
# resources of the other kind
if hosts and not clusters and not resources:
q = q.join(ResourceHolder)
q = q.options(contains_eager('holder'))
q = q.filter_by(holder_type='host')
if clusters and not hosts and not resources:
q = q.join(ResourceHolder)
q = q.filter_by(holder_type='cluster')
q = q.options(contains_eager('holder'))
if options:
q = q.options(*options)
for res in q:
resource_by_id[res.id] = res
try:
res.populate_share_info(share_info)
except AttributeError:
pass
if hosts or machines:
# Polymorphic loading cannot be applied to eager-loaded
# attributes, so load interfaces manually.
q = session.query(Interface)
q = q.with_polymorphic('*')
q = q.options(lazyload("hardware_entity"))
for iface in q:
interfaces_by_machine[iface.hardware_entity_id].append(
iface)
interfaces_by_id[iface.id] = iface
if hosts:
# subqueryload() and with_polymorphic() do not play nice
# together, so do it by hand
q = session.query(AddressAssignment)
q = q.options(joinedload("network"),
joinedload("dns_records"))
q = q.order_by(AddressAssignment._label)
addrs_by_iface = defaultdict(list)
for addr in q:
addrs_by_iface[addr.interface_id].append(addr)
for interface_id, addrs in addrs_by_iface.items():
set_committed_value(interfaces_by_id[interface_id],
"assignments", addrs)
q = session.query(Interface.id)
q = q.filter(~Interface.assignments.any())
for id in q.all():
set_committed_value(interfaces_by_id[id[0]],
"assignments", None)
if hosts or services:
q = session.query(ServiceInstance)
q = q.options(subqueryload("service"))
service_instances = q.all()
if machines or clusters:
# Most machines are in racks...
q = session.query(Rack)
q = q.options(subqueryload("dns_maps"),
subqueryload("parents"))
racks = q.all()
if locations:
logger.client_info("Flushing locations.")
for dbloc in session.query(City).all():
try:
plenary = Plenary.get_plenary(dbloc, logger=logger)
written += plenary.write(locked=True)
except Exception, e:
failed.append("City %s failed: %s" % dbloc, e)
continue
if services:
logger.client_info("Flushing services.")
q = session.query(Service)
q = q.options(subqueryload("instances"))
for dbservice in q:
try:
plenary_info = Plenary.get_plenary(dbservice,
logger=logger)
written += plenary_info.write(locked=True)
except Exception, e:
failed.append("Service %s failed: %s" %
(dbservice.name, e))
continue
for dbinst in dbservice.instances:
try:
plenary_info = Plenary.get_plenary(dbinst,
logger=logger)
written += plenary_info.write(locked=True)
except Exception, e:
failed.append("Service %s instance %s failed: %s" %
(dbservice.name, dbinst.name, e))
continue
class Media(object):
"""
Media metadata and a collection of related files.
"""
meta = association_proxy('_meta', 'value', creator=MediaMeta)
query = DBSession.query_property(MediaQuery)
# TODO: replace '_thumb_dir' with something more generic, like 'name',
# so that its other uses throughout the code make more sense.
_thumb_dir = 'media'
def __init__(self):
if self.author is None:
self.author = Author()
def __repr__(self):
return '<Media: %s>' % self.slug
def set_tags(self, tags):
"""Set the tags relations of this media, creating them as needed.
:param tags: A list or comma separated string of tags to use.
"""
if isinstance(tags, basestring):
tags = extract_tags(tags)
if isinstance(tags, list) and tags:
tags = fetch_and_create_tags(tags)
self.tags = tags or []
def set_categories(self, cats):
"""Set the related categories of this media.
:param cats: A list of category IDs to set.
"""
if cats:
cats = Category.query.filter(Category.id.in_(cats)).all()
self.categories = cats or []
def update_status(self):
"""Ensure the type (audio/video) and encoded flag are properly set.
Call this after modifying any files belonging to this item.
"""
self.type = self._update_type()
self.encoded = self._update_encoding()
def _update_type(self):
"""Update the type of this Media object.
If there's a video file, mark this as a video type, else fallback
to audio, if possible, or unknown (None)
"""
if any(file.type == VIDEO for file in self.files):
return VIDEO
elif any(file.type == AUDIO for file in self.files):
return AUDIO
else:
return None
def _update_encoding(self):
# Test to see if we can find a workable file/player combination
# for the most common podcasting app w/ the POOREST format support
if self.podcast_id \
and not pick_podcast_media_file(self):
return False
# Test to see if we can find a workable file/player combination
# for the browser w/ the BEST format support
if not pick_any_media_file(self):
return False
return True
@property
def is_published(self):
if self.id is None:
return False
return self.publishable and self.reviewed and self.encoded\
and (self.publish_on is not None and self.publish_on <= datetime.now())\
and (self.publish_until is None or self.publish_until >= datetime.now())
def increment_views(self):
"""Increment the number of views in the database.
We avoid concurrency issues by incrementing JUST the views and
not allowing modified_on to be updated automatically.
"""
if self.id is None:
self.views += 1
return self.views
# Don't raise an exception should concurrency problems occur.
# Views will not actually be incremented in this case, but thats
# relatively unimportant compared to rendering the page for the user.
# We may be able to remove this after we improve our triggers to not
# issue an UPDATE on media_fulltext unless one of its columns are
# actually changed. Even when just media.views is updated, all the
# columns in the corresponding media_fulltext row are updated, and
# media_fulltext's MyISAM engine must lock the whole table to do so.
transaction = DBSession.begin_nested()
try:
DBSession.query(self.__class__)\
.filter(self.__class__.id == self.id)\
.update({self.__class__.views: self.__class__.views + 1})
transaction.commit()
except OperationalError, e:
transaction.rollback()
# (OperationalError) (1205, 'Lock wait timeout exceeded, try restarting the transaction')
if not '1205' in e.message:
raise
# Increment the views by one for the rest of the request,
# but don't allow the ORM to increment the views too.
attributes.set_committed_value(self, 'views', self.views + 1)
return self.views
def render(self, session, logger, hostname, **arguments):
# removing the plenary host requires a compile lock, however
# we want to avoid deadlock by the fact that we're messing
# with two locks here, so we want to be careful. We grab the
# plenaryhost early on (in order to get the filenames filled
# in from the db info before we delete it from the db. We then
# hold onto those references until we've completed the db
# cleanup and if all of that is successful, then we delete the
# plenary file (which doesn't require re-evaluating any stale
# db information) after we've released the delhost lock.
delplenary = False
# Any service bindings that we need to clean up afterwards
bindings = PlenaryCollection(logger=logger)
resources = PlenaryCollection(logger=logger)
with DeleteKey("system", logger=logger) as key:
# Check dependencies, translate into user-friendly message
dbhost = hostname_to_host(session, hostname)
host_plenary = Plenary.get_plenary(dbhost, logger=logger)
domain = dbhost.branch.name
deps = get_host_dependencies(session, dbhost)
if (len(deps) != 0):
deptext = "\n".join([" %s" % d for d in deps])
raise ArgumentError("Cannot delete host %s due to the "
"following dependencies:\n%s." %
(hostname, deptext))
archetype = dbhost.archetype.name
dbmachine = dbhost.machine
oldinfo = DSDBRunner.snapshot_hw(dbmachine)
ip = dbmachine.primary_ip
fqdn = dbmachine.fqdn
for si in dbhost.services_used:
plenary = PlenaryServiceInstanceServer(si)
bindings.append(plenary)
logger.info("Before deleting host '%s', removing binding '%s'"
% (fqdn, si.cfg_path))
del dbhost.services_used[:]
if dbhost.resholder:
for res in dbhost.resholder.resources:
resources.append(Plenary.get_plenary(res))
# In case of Zebra, the IP may be configured on multiple interfaces
for iface in dbmachine.interfaces:
if ip in iface.addresses:
iface.addresses.remove(ip)
if dbhost.cluster:
dbcluster = dbhost.cluster
dbcluster.hosts.remove(dbhost)
set_committed_value(dbhost, '_cluster', None)
dbcluster.validate()
dbdns_rec = dbmachine.primary_name
dbmachine.primary_name = None
dbmachine.host = None
session.delete(dbhost)
delete_dns_record(dbdns_rec)
session.flush()
delplenary = True
if dbmachine.vm_container:
bindings.append(Plenary.get_plenary(dbmachine.vm_container))
if archetype != 'aurora' and ip is not None:
dsdb_runner = DSDBRunner(logger=logger)
dsdb_runner.update_host(dbmachine, oldinfo)
dsdb_runner.commit_or_rollback("Could not remove host %s from "
"DSDB" % hostname)
if archetype == 'aurora':
logger.client_info("WARNING: removing host %s from AQDB and "
"*not* changing DSDB." % hostname)
# Past the point of no return... commit the transaction so
# that we can free the delete lock.
session.commit()
# Only if we got here with no exceptions do we clean the template
# Trying to clean up after any errors here is really difficult
# since the changes to dsdb have already been made.
if (delplenary):
key = host_plenary.get_remove_key()
with CompileKey.merge([key, bindings.get_write_key(),
resources.get_remove_key()]) as key:
host_plenary.cleanup(domain, locked=True)
# And we also want to remove the profile itself
profiles = self.config.get("broker", "profilesdir")
# Only one of these should exist, but it doesn't hurt
# to try to clean up both.
xmlfile = os.path.join(profiles, fqdn + ".xml")
remove_file(xmlfile, logger=logger)
xmlgzfile = xmlfile + ".gz"
remove_file(xmlgzfile, logger=logger)
# And the cached template created by ant
remove_file(os.path.join(self.config.get("broker",
"quattordir"),
"objects", fqdn + TEMPLATE_EXTENSION),
logger=logger)
bindings.write(locked=True)
resources.remove(locked=True)
build_index(self.config, session, profiles, logger=logger)
return
def _populate_backrefs(target, context):
for name, backref in mappings.items():
# __dict__ to avoid triggering lazy-loaded relationships
if target.__dict__.get(name) is not None:
set_committed_value(getattr(target, name), backref, target)
def get_unique(cls,
sess,
name,
hardware_type=None,
compel=False,
preclude=False,
query_options=None):
""" Returns a unique HardwareEntity given session and fqdn """
# If the hardware_type param isn't explicitly set and we have a
# polymorphic identity, assume we're querying only for items of our
# hardware_type.
if hardware_type:
if isclass(hardware_type):
clslabel = hardware_type._get_class_label()
hardware_type = hardware_type.__mapper_args__[
'polymorphic_identity']
else:
pcls = cls.__mapper__.polymorphic_map[hardware_type].class_
clslabel = pcls._get_class_label()
else:
if 'polymorphic_identity' in cls.__mapper_args__:
hardware_type = cls.__mapper_args__['polymorphic_identity']
clslabel = cls._get_class_label()
# The automagic DNS lookup does not really make sense with preclude=True
if preclude:
name = AqStr.normalize(name)
cls.check_label(name)
q = sess.query(cls)
if "." in name:
dns_rec = DnsRecord.get_unique(sess, fqdn=name, compel=True)
# We know the primary name, do not load it again
q = q.options(lazyload('primary_name'))
q = q.filter_by(primary_name=dns_rec)
else:
dns_rec = None
q = q.filter_by(label=name)
if query_options:
q = q.options(*query_options)
try:
hwe = q.one()
except NoResultFound:
# Check if the name is in use by a different hardware type
q = sess.query(HardwareEntity)
if dns_rec:
# We know the primary name, do not load it again
q = q.options(lazyload('primary_name'))
q = q.filter_by(primary_name=dns_rec)
else:
q = q.filter_by(label=name)
try:
hwe = q.one()
if dns_rec:
# We know the primary name, do not load it again
set_committed_value(hwe, 'primary_name', dns_rec)
raise ArgumentError("{0} exists, but is not a {1}.".format(
hwe, hardware_type))
except NoResultFound:
hwe = None
if compel:
raise NotFoundException("%s %s not found." % (clslabel, name))
if hwe:
if preclude:
raise ArgumentError('{0} already exists.'.format(hwe))
if dns_rec:
# We know the primary name, do not load it again
set_committed_value(hwe, 'primary_name', dns_rec)
return hwe