def get_upcoming_events(): """Get the global list of upcoming events""" from indico.modules.events import Event data = upcoming_events_settings.get_all() if not data['max_entries'] or not data['entries']: return tz = timezone(Config.getInstance().getDefaultTimezone()) now = now_utc(False).astimezone(tz) base_query = (Event.query .filter(Event.effective_protection_mode == ProtectionMode.public, ~Event.is_deleted, Event.end_dt.astimezone(tz) > now) .options(load_only('id', 'title', 'start_dt', 'end_dt'))) queries = [] cols = {'category': Event.category_id, 'event': Event.id} for entry in data['entries']: delta = timedelta(days=entry['days']) query = (base_query .filter(cols[entry['type']] == entry['id']) .filter(db.cast(Event.start_dt.astimezone(tz), db.Date) > (now - delta).date()) .with_entities(Event, db.literal(entry['weight']).label('weight'))) queries.append(query) query = (queries[0].union(*queries[1:]) .order_by(db.desc('weight'), Event.start_dt, Event.title) .limit(data['max_entries'])) for row in query: event = row[0] # we cache the result of the function and is_deleted is used in the repr # and having a broken repr on the cached objects would be ugly set_committed_value(event, 'is_deleted', False) yield event
def new_version(self, session): # convert to an INSERT make_transient(self) self.id = None # history of the 'elements' collection. # this is a tuple of groups: (added, unchanged, deleted) hist = attributes.get_history(self, "elements") # rewrite the 'elements' collection # from scratch, removing all history attributes.set_committed_value(self, "elements", {}) # new elements in the "added" group # are moved to our new collection. for elem in hist.added: self.elements[elem.name] = elem # copy elements in the 'unchanged' group. # the new ones associate with the new ConfigData, # the old ones stay associated with the old ConfigData for elem in hist.unchanged: self.elements[elem.name] = ConfigValueAssociation( elem.config_value )
def validate(self, max_hosts=None, error=ArgumentError, **kwargs): session = object_session(self) q = session.query(HostClusterMember) q = q.filter_by(cluster=self) q = q.options(joinedload('host'), joinedload('host.machine')) members = q.all() set_committed_value(self, '_hosts', members) if self.cluster_type != 'meta': for i in [ "down_hosts_threshold", "down_hosts_percent", "down_maint_percent", "personality_id" #"branch_id" ]: if getattr(self, i, None) is None: raise error("Attribute %s must be set for a %s cluster." % (i, self.cluster_type)) else: if self.metacluster: raise error("Metaclusters can't contain other metaclusters.") if max_hosts is None: max_hosts = self.max_hosts if len(self.hosts) > self.max_hosts: raise error("{0} is over capacity of {1} hosts.".format( self, max_hosts)) if self.metacluster: self.metacluster.validate()
def get_upcoming_events(): """Get the global list of upcoming events""" from indico.modules.events import Event data = upcoming_events_settings.get_all() if not data['max_entries'] or not data['entries']: return tz = timezone(config.DEFAULT_TIMEZONE) now = now_utc(False).astimezone(tz) base_query = (Event.query .filter(Event.effective_protection_mode == ProtectionMode.public, ~Event.is_deleted, Event.end_dt.astimezone(tz) > now) .options(load_only('id', 'title', 'start_dt', 'end_dt'))) queries = [] cols = {'category': Event.category_id, 'event': Event.id} for entry in data['entries']: delta = timedelta(days=entry['days']) query = (base_query .filter(cols[entry['type']] == entry['id']) .filter(db.cast(Event.start_dt.astimezone(tz), db.Date) > (now - delta).date()) .with_entities(Event, db.literal(entry['weight']).label('weight'))) queries.append(query) query = (queries[0].union(*queries[1:]) .order_by(db.desc('weight'), Event.start_dt, Event.title) .limit(data['max_entries'])) for row in query: event = row[0] # we cache the result of the function and is_deleted is used in the repr # and having a broken repr on the cached objects would be ugly set_committed_value(event, 'is_deleted', False) yield event
def _populate_preloaded_relationships(cls, target, *unused): cache = g.get('relationship_cache', {}).get(type(target)) if not cache: return for rel, value in cache['data'].get(target, {}).iteritems(): if rel not in target.__dict__: set_committed_value(target, rel, value)
def populated_tree(cats): """Return the root categories with children populated to any depth. Adjacency lists are notoriously inefficient for fetching deeply nested trees, and since our dataset will always be reasonably small, this method should greatly improve efficiency. Only one query is necessary to fetch a tree of any depth. This isn't always the solution, but for some situations, it is worthwhile. For example, printing the entire tree can be done with one query:: query = Category.query.options(undefer('media_count')) for cat, depth in query.populated_tree().traverse(): print " " * depth, cat.name, '(%d)' % cat.media_count Without this method, especially with the media_count undeferred, this would require a lot of extra queries for nested categories. NOTE: If the tree contains circular nesting, the circular portion of the tree will be silently omitted from the results. """ children = defaultdict(CategoryList) for cat in cats: children[cat.parent_id].append(cat) for cat in cats: set_committed_value(cat, 'children', children[cat.id]) return children[None]
def _bulk_load_column_for_instance_states( session: Session, mapper: Mapper, identities: Iterable[Tuple], attr_name: str, alter_query: Optional[QueryAlterator]): """ Load a column attribute for a list of instance states where the attribute is unloaded """ Model = mapper.class_ attr: Column = mapper.columns[attr_name] # Using those identities (primary keys), load the missing attribute q = load_by_primary_keys(session, mapper, identities, attr) # Alter the query if alter_query: q = alter_query(q, mapper, attr_name, False) # Having the missing attribute's value loaded, assign it to every instance in the session for identity, attr_value in q: # Build the identity key the way SqlAlchemy likes it: # (Model, primary-key, None) key = identity_key(Model, identity) # We do not iterate the Session to find an instance that matches the primary key. # Instead, we take it directly using the `identity_map` instance = session.identity_map[key] # Set the value of the missing attribute. # This is how it immediately becomes loaded. # Note that this action does not overwrite any modifications made to the attribute. set_committed_value(instance, attr_name, attr_value)
def populated_tree(cats): """Return the root categories with children populated to any depth. Adjacency lists are notoriously inefficient for fetching deeply nested trees, and since our dataset will always be reasonably small, this method should greatly improve efficiency. Only one query is necessary to fetch a tree of any depth. This isn't always the solution, but for some situations, it is worthwhile. For example, printing the entire tree can be done with one query:: query = Category.query.options(undefer('media_count')) for cat, depth in query.populated_tree().traverse(): print " " * depth, cat.name, '(%d)' % cat.media_count Without this method, especially with the media_count undeferred, this would require a lot of extra queries for nested categories. NOTE: If the tree contains circular nesting, the circular portion of the tree will be silently omitted from the results. """ roots = CategoryList() children = defaultdict(list) for cat in cats: if cat.parent_id: children[cat.parent_id].append(cat) else: roots.append(cat) for cat in cats: attributes.set_committed_value(cat, 'children', children[cat.id]) return roots
def validate(self, max_hosts=None, error=ArgumentError, **kwargs): session = object_session(self) q = session.query(HostClusterMember) q = q.filter_by(cluster=self) q = q.options(joinedload('host'), joinedload('host.machine')) members = q.all() set_committed_value(self, '_hosts', members) if self.cluster_type != 'meta': for i in [ "down_hosts_threshold", "down_hosts_percent", "down_maint_percent", "personality_id" #"branch_id" ]: if getattr(self, i, None) is None: raise error("Attribute %s must be set for a %s cluster." % (i, self.cluster_type)) else: if self.metacluster: raise error("Metaclusters can't contain other metaclusters.") if max_hosts is None: max_hosts = self.max_hosts if len(self.hosts) > self.max_hosts: raise error("{0} is over capacity of {1} hosts.".format(self, max_hosts)) if self.metacluster: self.metacluster.validate()
def intercept_after_flush(session, obj): if isinstance(obj, QuestionModel): print(obj) if obj.questionKey == "set": obj.set_questionKey() session.query(QuestionModel).filter_by(id=obj.id).update( {"questionKey": obj.questionKey}) set_committed_value(obj, "questionKey", obj.questionKey)
def populate(self): """ Populate batch fetched entities to parent objects. """ for entity in self.path.entities: set_committed_value( entity, self.prop.key, self.parent_dict[local_values(self.prop, entity)])
def populate(self): """ Populate batch fetched entities to parent objects. """ for entity in self.path.entities: set_committed_value( entity, self.prop.key, self.parent_dict[local_values(self.prop, entity)] )
def sa_set_committed_state(obj: object, **committed_values): """ Put values into an SqlAlchemy instance as if they were committed to the DB """ # Give it some DB identity so that SA thinks it can load something state: InstanceState = instance_state(obj) state.key = object() # Set every attribute in such a way that SA thinkg that's the way it looks in the DB for k, v in committed_values.items(): set_committed_value(obj, k, v) return obj
def populate(self): """ Populate batch fetched entities to parent objects. """ for entity in self.path.entities: set_committed_value( entity, self.prop.key, self.parent_dict[self.local_values(entity)] ) if self.path.populate_backrefs: self.populate_backrefs(self.related_entities)
def _set_results_on_models(self, param_value_to_results, param_value_to_models, current_model): current_model_result = None for value, models in param_value_to_models.items(): for model in models: result = param_value_to_results.get(value, []) # ensure models aren't given identical result lists so modifying results in one doesn't modify others result = result[:] if not self.uselist: result = self._extract_non_list_result(result) if model == current_model: current_model_result = result attributes.set_committed_value(model, self.key, result) return current_model_result
def populate_backrefs(self, related_entities): """ Populates backrefs for given related entities. """ backref_dict = dict((local_values(self.prop, value[0]), []) for value in related_entities) for value in related_entities: backref_dict[local_values(self.prop, value[0])].append( self.path.session.query(self.path.entities[0].__class__).get( tuple(value[1:]))) for value in related_entities: set_committed_value( value[0], self.prop.back_populates, backref_dict[local_values(self.prop, value[0])])
def get_unique(cls, session, fqdn=None, name=None, dns_domain=None, dns_environment=None, compel=False, preclude=False, **kwargs): # Proxy FQDN lookup to the Fqdn class if not fqdn or not isinstance(fqdn, Fqdn): if not isinstance(dns_environment, DnsEnvironment): dns_environment = DnsEnvironment.get_unique_or_default( session, dns_environment) if fqdn: if name or dns_domain: # pragma: no cover raise TypeError("fqdn and name/dns_domain cannot be mixed") (name, dns_domain) = parse_fqdn(session, fqdn) try: # Do not pass preclude=True to Fqdn fqdn = Fqdn.get_unique(session, name=name, dns_domain=dns_domain, dns_environment=dns_environment, compel=compel) except NotFoundException: # Replace the "Fqdn ... not found" message with a more user # friendly one msg = "%s %s.%s, %s not found." % ( cls._get_class_label(), name, dns_domain, format(dns_environment, "l")) raise NotFoundException(msg) if not fqdn: return None # We already have the FQDN, no need to load it again if "query_options" not in kwargs: kwargs["query_options"] = [lazyload("fqdn")] result = super(DnsRecord, cls).get_unique(session, fqdn=fqdn, compel=compel, preclude=preclude, **kwargs) if result: # Make sure not to load the relation again if we already know its # value set_committed_value(result, 'fqdn', fqdn) return result
def validate(self): session = object_session(self) q = session.query(HostClusterMember) q = q.filter_by(cluster=self) q = q.options(joinedload('host'), joinedload('host.hardware_entity')) members = q.all() set_committed_value(self, '_hosts', members) if self.max_hosts is not None and len(self.hosts) > self.max_hosts: raise ArgumentError("{0} has {1} hosts bound, which exceeds the " "requested limit of {2}." .format(self, len(self.hosts), self.max_hosts)) if self.metacluster: self.metacluster.validate()
def populate_related(parents, id_key, res_key, reltype, subq, flt=None, relid_key='id'): ids = [] for p in parents: if callable(flt) and not flt(p): continue keyval = getattr(p, id_key, None) if keyval and (keyval not in ids): ids.append(keyval) if len(ids) > 0: for rel in subq.filter(getattr(reltype, relid_key).in_(ids)): for p in parents: if callable(flt) and not flt(p): continue keyval = getattr(p, id_key, None) if keyval and (keyval == getattr(rel, relid_key)): attributes.set_committed_value(p, res_key, rel)
def increment_views(self): """Increment the number of views in the database. We avoid concurrency issues by incrementing JUST the views and not allowing modified_on to be updated automatically. """ if self.id is None: self.views += 1 return self.views DBSession.execute(media.update().values(views=media.c.views + 1).where(media.c.id == self.id)) # Increment the views by one for the rest of the request, # but don't allow the ORM to increment the views too. attributes.set_committed_value(self, "views", self.views + 1) return self.views
def fetch_as_tree(query: Query, relationship: Optional[InstrumentedAttribute] = None, root_value: Union[str, None] = None, ) -> List[DeclarativeMeta]: """Builds the full tree for the given table (adjacency list). This method is efficient in the sense that it issues only one SELECT query to the database. Args: query: A query that selects all relevant model records, that is, all records that make up the adjacency list. For example: `session.query(Album).order_by(Album.nameLC)`. relationship: The parent/child relationship that describes both fields (columns). For example: `Album.parent` If not set, one record is fetched from the query to read its `parent` property. root_value: Parent property's value at the root nodes. Default is None, which commonly retrieves the whole structure. Returns: A list of root nodes with child nodes (the tree) pre-fetched recursively. """ if relationship is None: # Fetch one record to discover the parent relationship. relationship = next(iter(query)).__class__.parent parent_field = relationship.expression.right.name child_field = relationship.expression.left.name back_populates = relationship.property.back_populates nodes = query.all() children = defaultdict(list) for node in nodes: children[getattr(node, parent_field)].append(node) for node in nodes: set_committed_value(node, back_populates, children[getattr(node, child_field)]) return children[root_value]
def populate(self): """ Populate batch fetched entities to parent objects. """ for entity in self.path.entities: # print ( # "setting committed value for ", # entity, # " using local values ", # self.local_values(entity) # ) set_committed_value( entity, self.prop.key, self.parent_dict[self.local_values(entity)] ) if self.path.populate_backrefs: self.populate_backrefs(self.related_entities)
def load_projects(): projects = session.query(Project).options(joinedload('quotas')).all() root = None children = defaultdict(list) for project in projects: if project.parent: children[project.parent.id].append(project) if project.parent is None: root = project for project in projects: set_committed_value(project, 'children', children[project.id]) project.limits = { quota.resource: quota.limit for quota in project.quotas } return root
def increment_views(self): """Increment the number of views in the database. We avoid concurrency issues by incrementing JUST the views and not allowing modified_on to be updated automatically. """ if self.id is None: self.views += 1 return self.views DBSession.execute(media.update()\ .values(views=media.c.views + 1)\ .where(media.c.id == self.id)) # Increment the views by one for the rest of the request, # but don't allow the ORM to increment the views too. attributes.set_committed_value(self, 'views', self.views + 1) return self.views
def populate_related_list(parents, id_key, res_key, reltype, subq, flt=None, relid_key='id'): ids = [] for p in parents: if callable(flt) and not flt(p): continue keyval = getattr(p, id_key, None) if keyval and keyval not in ids: ids.append(keyval) if len(ids) > 0: ch = dict((k, list(v)) for k, v in groupby( subq.filter(getattr(reltype, relid_key).in_(ids)), lambda c: getattr(c, relid_key))) for p in parents: if callable(flt) and not flt(p): continue keyval = getattr(p, id_key, None) if keyval: attributes.set_committed_value(p, res_key, ch.get(keyval, ()))
def listtracksbyalbums(self, ffilter=None, skip=None, limit=None): with self.session_scope() as session: query = session.query(Album).join(Album.tracks).outerjoin( TrackInfo.genre).options( contains_eager(Album.tracks, TrackInfo.album, Album.artist), joinedload(Album.tracks, TrackInfo.genre)) if ffilter is not None: query = ffilter(query) query = limitoffset( query.order_by(Album.year.desc(), TrackInfo.trackno), skip, limit) qall = query.all() # Force artist population for x in qall: for y in x.tracks: attributes.set_committed_value(y, 'album', x) attributes.set_committed_value(y, 'artist', x.artist) session.expunge_all() return qall
def populate_backrefs(self, related_entities): """ Populates backrefs for given related entities. """ backref_dict = dict( (local_values(self.prop, value[0]), []) for value in related_entities ) for value in related_entities: backref_dict[local_values(self.prop, value[0])].append( self.path.session.query(self.path.entities[0].__class__).get( tuple(value[1:]) ) ) for value in related_entities: set_committed_value( value[0], self.prop.back_populates, backref_dict[local_values(self.prop, value[0])] )
def get_unique(cls, session, fqdn=None, name=None, dns_domain=None, dns_environment=None, compel=False, preclude=False, **kwargs): # Proxy FQDN lookup to the Fqdn class if not fqdn or not isinstance(fqdn, Fqdn): if not isinstance(dns_environment, DnsEnvironment): dns_environment = DnsEnvironment.get_unique_or_default(session, dns_environment) if fqdn: if name or dns_domain: # pragma: no cover raise TypeError("fqdn and name/dns_domain cannot be mixed") (name, dns_domain) = parse_fqdn(session, fqdn) try: # Do not pass preclude=True to Fqdn fqdn = Fqdn.get_unique(session, name=name, dns_domain=dns_domain, dns_environment=dns_environment, compel=compel) except NotFoundException: # Replace the "Fqdn ... not found" message with a more user # friendly one msg = "%s %s.%s, %s not found." % (cls._get_class_label(), name, dns_domain, format(dns_environment, "l")) raise NotFoundException(msg) if not fqdn: return None # We already have the FQDN, no need to load it again if "query_options" not in kwargs: kwargs["query_options"] = [lazyload("fqdn")] result = super(DnsRecord, cls).get_unique(session, fqdn=fqdn, compel=compel, preclude=preclude, **kwargs) if result: # Make sure not to load the relation again if we already know its # value set_committed_value(result, 'fqdn', fqdn) return result
def preload_interfaces(self, session, hosts, interfaces_by_id, interfaces_by_hwent): addrs_by_iface = defaultdict(list) slaves_by_id = defaultdict(list) # Polymorphic loading cannot be applied to eager-loaded # attributes, so load interfaces manually. q = session.query(Interface) q = q.with_polymorphic('*') q = q.options(lazyload("hardware_entity")) for iface in q: interfaces_by_hwent[iface.hardware_entity_id].append(iface) interfaces_by_id[iface.id] = iface if iface.master_id: slaves_by_id[iface.master_id].append(iface) # subqueryload() and with_polymorphic() do not play nice # together, so do it by hand q = session.query(AddressAssignment) q = q.options(joinedload("network"), joinedload("dns_records")) q = q.order_by(AddressAssignment._label) # Machine templates want the management interface only if hosts: q = q.options(subqueryload("network.static_routes"), subqueryload("network.routers")) else: q = q.join(AddressAssignment.interface.of_type(ManagementInterface)) for addr in q: addrs_by_iface[addr.interface_id].append(addr) for iface_id, iface in interfaces_by_id.iteritems(): set_committed_value(iface, "assignments", addrs_by_iface.get(iface_id, None)) set_committed_value(iface, "slaves", slaves_by_id.get(iface_id, None))
def test_only_loads_relations_on_unpopulated_models(self): User = self.classes.User Address = self.classes.Address session = fixture_session() users = session.query(User).order_by(self.tables.users.c.id.asc()).all() address = session.query(Address).filter(self.tables.addresses.c.id == 1).first() # pre-load the address for the first user attributes.set_committed_value(users[0], "addresses", [address]) self.queries = [] # make sure no relations are loaded for user in users[1:]: model_dict = attributes.instance_dict(user) assert "addresses" not in model_dict # trigger a lazy load users[1].addresses # only 1 query should have been generated to load all the child relationships assert len(self.queries) == 1 unpopulated_user_ids = [user.id for user in users[1:]] assert self.queries[0]["parameters"] == tuple(unpopulated_user_ids)
def domain_hosts(self, domain): from netprofile_hosts.models import Host from netprofile_ipaddresses.models import ( IPv4Address, IPv6Address ) # XXX: this is a hack, but it reduces poor ol' MySQL's crunch time by a factor of 10 sess = DBSession() hipv4 = dict((k, list(v)) for k, v in groupby( sess.query(IPv4Address).join(IPv4Address.host).filter(Host.domain_id == domain.id), lambda ip: ip.host_id )) hipv6 = dict((k, list(v)) for k, v in groupby( sess.query(IPv6Address).join(IPv6Address.host).filter(Host.domain_id == domain.id), lambda ip: ip.host_id )) for host in domain.hosts: attributes.set_committed_value(host, 'ipv4_addresses', hipv4.get(host.id, ())) attributes.set_committed_value(host, 'ipv6_addresses', hipv6.get(host.id, ())) return domain.hosts
def populate_related_list(parents, id_key, res_key, reltype, subq, flt=None, relid_key='id'): ids = [] for p in parents: if callable(flt) and not flt(p): continue keyval = getattr(p, id_key, None) if keyval and (keyval not in ids): ids.append(keyval) if len(ids) > 0: ch = dict((k, list(v)) for k, v in groupby( subq.filter(getattr(reltype, relid_key).in_(ids)), lambda c: getattr(c, relid_key))) for p in parents: if callable(flt) and not flt(p): continue keyval = getattr(p, id_key, None) if keyval: attributes.set_committed_value(p, res_key, ch.get(keyval, ()))
def listtracksbyalbumsbyartists(self, ffilter=None, skip=None, limit=None, order_by=(Artist.name, Album.year.desc(), TrackInfo.trackno, TrackInfo.name)): with self.session_scope() as session: query = session.query(Artist).join(Artist.albums).join( Album.tracks).join(TrackInfo.track).options( contains_eager(Artist.albums, Album.tracks)) if ffilter is not None: query = ffilter(query) query = limitoffset(query.order_by(*order_by), skip, limit) qall = query.all() # Force artist population for x in qall: for y in x.albums: attributes.set_committed_value(y, 'artist', x) for z in y.tracks: attributes.set_committed_value(z, 'album', y) attributes.set_committed_value(z, 'artist', x) session.expunge_all() return qall
def __init__(self, dbhost, *args, **kwargs): """Provide initialization specific for host bindings.""" if not isinstance(dbhost, Host): raise InternalError("HostChooser can only choose services for " "hosts, got %r (%s)" % (dbhost, type(dbhost))) super(HostChooser, self).__init__(dbhost, *args, **kwargs) self.location = dbhost.hardware_entity.location # If the primary name is a ReservedName, then it does not have a network # attribute if hasattr(dbhost.hardware_entity.primary_name, 'network'): self.network = dbhost.hardware_entity.primary_name.network else: self.network = None # all of them would be self. but that should be optimized # dbhost.hardware_entity.interfaces[x].assignments[y].network # Stores interim service instance lists. q = self.session.query(Service) q = q.outerjoin(Service.archetypes) q = q.reset_joinpoint() q = q.outerjoin(Service.personalities) q = q.filter(or_(Archetype.id == self.archetype.id, Personality.id == self.personality.id)) self.required_services = set(q.all()) self.original_service_instances = {} # Cache of any already bound services (keys) and the instance # that was bound (values). q = self.session.query(ServiceInstance) q = q.options(undefer('_client_count')) q = q.filter(ServiceInstance.clients.contains(dbhost)) set_committed_value(dbhost, 'services_used', q.all()) for si in dbhost.services_used: self.original_service_instances[si.service] = si self.logger.debug("{0} original binding: {1}" .format(self.dbobj, si)) self.cluster_aligned_services = {} if dbhost.cluster: # Note that cluster services are currently ignored unless # they are otherwise required by the archetype/personality. for si in dbhost.cluster.service_bindings: self.cluster_aligned_services[si.service] = si for service in dbhost.cluster.required_services: if service not in self.cluster_aligned_services: # Don't just error here because the error() call # has not yet been set up. Will error out later. self.cluster_aligned_services[service] = None # Went back and forth on this... deciding not to force # an aligned service as required. This should give # flexibility for multiple services to be aligned for # a cluster type without being forced on all the # personalities. #self.required_services.add(item.service) if dbhost.cluster.metacluster: mc = dbhost.cluster.metacluster for si in mc.service_bindings: if si.service in self.cluster_aligned_services: cas = self.cluster_aligned_services[si.service] if cas == None: # Error out later. continue self.logger.client_info( "Replacing {0.name} instance with {1.name} " "(bound to {2:l}) for service {3.name}".format( cas, si, mc, si.service)) self.cluster_aligned_services[si.service] = si for service in mc.required_services: if service not in self.cluster_aligned_services: # Don't just error here because the error() call # has not yet been set up. Will error out later. self.cluster_aligned_services[service] = None
def _set_updated_revision_number(self, revision_number, updated_at): attributes.set_committed_value(self, "revision_number", revision_number) attributes.set_committed_value(self, "updated_at", updated_at)
def render(self, session, logger, hostname, **arguments): # Check dependencies, translate into user-friendly message dbhost = hostname_to_host(session, hostname) dbhost.lock_row() check_no_provided_service(dbhost) # Any service bindings that we need to clean up afterwards plenaries = PlenaryCollection(logger=logger) remove_plenaries = PlenaryCollection(logger=logger) remove_plenaries.append(Plenary.get_plenary(dbhost)) archetype = dbhost.archetype.name dbmachine = dbhost.hardware_entity oldinfo = DSDBRunner.snapshot_hw(dbmachine) ip = dbmachine.primary_ip for si in dbhost.services_used: plenaries.append(PlenaryServiceInstanceServer.get_plenary(si)) logger.info("Before deleting {0:l}, removing binding to {1:l}" .format(dbhost, si)) del dbhost.services_used[:] if dbhost.resholder: for res in dbhost.resholder.resources: remove_plenaries.append(Plenary.get_plenary(res)) # In case of Zebra, the IP may be configured on multiple interfaces for iface in dbmachine.interfaces: if ip in iface.addresses: iface.addresses.remove(ip) if dbhost.cluster: dbcluster = dbhost.cluster dbcluster.hosts.remove(dbhost) set_committed_value(dbhost, '_cluster', None) dbcluster.validate() plenaries.append(Plenary.get_plenary(dbcluster)) dbdns_rec = dbmachine.primary_name dbmachine.primary_name = None dbmachine.host = None session.delete(dbhost) delete_dns_record(dbdns_rec) session.flush() if dbmachine.vm_container: plenaries.append(Plenary.get_plenary(dbmachine.vm_container)) with CompileKey.merge([plenaries.get_key(), remove_plenaries.get_key()]): plenaries.stash() remove_plenaries.stash() try: plenaries.write(locked=True) remove_plenaries.remove(locked=True, remove_profile=True) if archetype != 'aurora' and ip is not None: dsdb_runner = DSDBRunner(logger=logger) dsdb_runner.update_host(dbmachine, oldinfo) dsdb_runner.commit_or_rollback("Could not remove host %s from " "DSDB" % hostname) if archetype == 'aurora': logger.client_info("WARNING: removing host %s from AQDB and " "*not* changing DSDB." % hostname) except: plenaries.restore_stash() remove_plenaries.restore_stash() raise trigger_notifications(self.config, logger, CLIENT_INFO) return
# TODO: only if not hosts manager_addrs = defaultdict(list) q = session.query(AddressAssignment) q = q.join(Interface) q = q.filter_by(interface_type="management") q = q.options(contains_eager("interface"), joinedload("dns_records"), lazyload("interface.hardware_entity")) for addr in q: manager_addrs[addr.interface.id].append(addr) for interface_id, addrs in manager_addrs.items(): if interface_id not in interfaces_by_id: # Should not happen... continue addrs.sort(key=attrgetter("label")) set_committed_value(interfaces_by_id[interface_id], "assignments", addrs) q = session.query(Machine) q = q.options(lazyload("host"), lazyload("primary_name"), subqueryload("chassis_slot")) cnt = q.count() idx = 0 for machine in q: idx += 1 if idx % 1000 == 0: # pragma: no cover logger.client_info("Processing machine %d of %d..." % (idx, cnt)) if machine.id in disks_by_machine: disks_by_machine[machine.id].sort(
def _populate_backrefs(target, context): for name, backref in mappings.iteritems(): # __dict__ to avoid triggering lazy-loaded relationships if target.__dict__.get(name) is not None: set_committed_value(getattr(target, name), backref, target)
def add_net_msg(self, net_msg, net): net_msg.name = str(net.name) net_msg.id = net.id net_msg.ip = str(net.ip) net_msg.cidr = net.cidr net_msg.bcast = str(net.broadcast) net_msg.netmask = str(net.netmask) net_msg.side = str(net.side) net_msg.sysloc = str(net.location.sysloc()) net_msg.location.name = str(net.location.name) net_msg.location.location_type = str(net.location.location_type) net_msg.type = str(net.network_type) # Bulk load information about anything having a network address on this # network hw_ids = set( [addr.interface.hardware_entity_id for addr in net.assignments]) if hw_ids: session = object_session(net) q = session.query(HardwareEntity) q = q.filter(HardwareEntity.id.in_(hw_ids)) q = q.options(subqueryload('interfaces')) hwent_by_id = {} for dbhwent in q.all(): hwent_by_id[dbhwent.id] = dbhwent iface_by_id = {} slaves_by_id = defaultdict(list) # We have all the interfaces loaded already, so compute the # master/slave relationships to avoid having to touch the # database again for iface in dbhwent.interfaces: iface_by_id[iface.id] = iface if iface.master_id: slaves_by_id[iface.master_id].append(iface) for iface in dbhwent.interfaces: set_committed_value(iface, "master", iface_by_id.get(iface.master_id, None)) set_committed_value(iface, "slaves", slaves_by_id[iface.id]) # TODO: once we refactor Host to be an FK to HardwareEntity instead # of Machine, this could be converted to a single joinedload('host') q = session.query(Host) q = q.options(lazyload('machine')) q = q.filter(Host.machine_id.in_(hw_ids)) for host in q.all(): set_committed_value(hwent_by_id[host.machine_id], "host", host) set_committed_value(host, "machine", hwent_by_id[host.machine_id]) # Add interfaces that have addresses in this network for addr in net.assignments: if not addr.dns_records: # hostname is a required field in the protobuf description continue hwent = addr.interface.hardware_entity # DHCP: we do not care about secondary IP addresses, but in some # cases the same IP address may show up with different MACs if not addr.label: mac_addrs = possible_mac_addresses(addr.interface) else: mac_addrs = [] # Generate a host record even if there is no known MAC address for # it if not mac_addrs: mac_addrs.append(None) # Associating the same IP with multiple MAC addresses is # problematic using the current protocol. Sending multiple host # messages is easy for the broker, but it can confuse consumers like # aqdhcpd. For now just ensure it never happens, and revisit the # problem when we have a real world requirement. if len(mac_addrs) > 1: mac_addrs = [mac_addrs[0]] for mac in mac_addrs: host_msg = net_msg.hosts.add() if addr.interface.interface_type == 'management': host_msg.type = 'manager' else: if hwent.hardware_type == 'machine': host_msg.type = 'host' if hwent.host: host_msg.archetype.name = str( hwent.host.archetype.name) elif hwent.hardware_type == 'switch': # aqdhcpd uses the type host_msg.type = 'tor_switch' else: host_msg.type = hwent.hardware_type host_msg.hostname = str(addr.dns_records[0].fqdn.name) host_msg.fqdn = str(addr.dns_records[0].fqdn) host_msg.dns_domain = str(addr.dns_records[0].fqdn.dns_domain) host_msg.ip = str(addr.ip) if mac: host_msg.mac = mac host_msg.machine.name = str(hwent.label) # aqdhcpd uses the interface list when excluding hosts it is not # authoritative for for iface in hwent.interfaces: int_msg = host_msg.machine.interfaces.add() int_msg.device = iface.name if iface.mac: int_msg.mac = str(iface.mac) # Add dynamic DHCP records for dynhost in net.dynamic_stubs: host_msg = net_msg.hosts.add() # aqdhcpd uses the type host_msg.type = 'dynamic_stub' host_msg.hostname = str(dynhost.fqdn.name) host_msg.fqdn = str(dynhost.fqdn) host_msg.dns_domain = str(dynhost.fqdn.dns_domain) host_msg.ip = str(dynhost.ip)
# TODO: only if not hosts manager_addrs = defaultdict(list) q = session.query(AddressAssignment) q = q.join(Interface) q = q.filter_by(interface_type="management") q = q.options(contains_eager("interface"), joinedload("dns_records"), lazyload("interface.hardware_entity")) for addr in q: manager_addrs[addr.interface.id].append(addr) for interface_id, addrs in manager_addrs.items(): if interface_id not in interfaces_by_id: # Should not happen... continue addrs.sort(key=attrgetter("label")) set_committed_value(interfaces_by_id[interface_id], "assignments", addrs) q = session.query(Machine) q = q.options(lazyload("host"), lazyload("primary_name"), subqueryload("chassis_slot")) cnt = q.count() idx = 0 for machine in q: idx += 1 if idx % 1000 == 0: # pragma: no cover logger.client_info("Processing machine %d of %d..." % (idx, cnt)) if machine.id in disks_by_machine:
def get_unique(cls, sess, name, hardware_type=None, compel=False, preclude=False, query_options=None): """ Returns a unique HardwareEntity given session and fqdn """ # If the hardware_type param isn't explicitly set and we have a # polymorphic identity, assume we're querying only for items of our # hardware_type. if hardware_type: if isclass(hardware_type): clslabel = hardware_type._get_class_label() hardware_type = hardware_type.__mapper_args__['polymorphic_identity'] else: pcls = cls.__mapper__.polymorphic_map[hardware_type].class_ clslabel = pcls._get_class_label() else: if 'polymorphic_identity' in cls.__mapper_args__: hardware_type = cls.__mapper_args__['polymorphic_identity'] clslabel = cls._get_class_label() # The automagic DNS lookup does not really make sense with preclude=True if preclude: name = AqStr.normalize(name) cls.check_label(name) q = sess.query(cls) if "." in name: dns_rec = DnsRecord.get_unique(sess, fqdn=name, compel=True) # We know the primary name, do not load it again q = q.options(lazyload('primary_name')) q = q.filter_by(primary_name=dns_rec) else: dns_rec = None q = q.filter_by(label=name) if query_options: q = q.options(*query_options) try: hwe = q.one() except NoResultFound: # Check if the name is in use by a different hardware type q = sess.query(HardwareEntity) if dns_rec: # We know the primary name, do not load it again q = q.options(lazyload('primary_name')) q = q.filter_by(primary_name=dns_rec) else: q = q.filter_by(label=name) try: hwe = q.one() if dns_rec: # We know the primary name, do not load it again set_committed_value(hwe, 'primary_name', dns_rec) raise ArgumentError("{0} exists, but is not a {1}." .format(hwe, clslabel.lower())) except NoResultFound: hwe = None if compel: raise NotFoundException("%s %s not found." % (clslabel, name)) if hwe: if preclude: raise ArgumentError('{0} already exists.'.format(hwe)) if dns_rec: # We know the primary name, do not load it again set_committed_value(hwe, 'primary_name', dns_rec) return hwe
chassis = q.all() # pylint: disable=W0612 q = session.query(Machine) q = q.options(lazyload("host"), lazyload("primary_name"), subqueryload("chassis_slot")) cnt = q.count() idx = 0 for machine in q: idx += 1 if idx % 1000 == 0: # pragma: no cover logger.client_info("Processing machine %d of %d..." % (idx, cnt)) set_committed_value(machine, 'disks', disks_by_machine.get(machine.id, None)) set_committed_value(machine, 'interfaces', interfaces_by_hwent.get(machine.id, None)) try: plenary_info = Plenary.get_plenary(machine, logger=logger) written += plenary_info.write(locked=True) except Exception, e: failed.append("{0} failed: {1}".format(machine, e)) continue if hosts: logger.client_info("Flushing hosts.") q = session.query(Cluster)
def _set_updated_revision_number(self, revision_number, updated_at): attributes.set_committed_value( self, "revision_number", revision_number) attributes.set_committed_value( self, "updated_at", updated_at)
def __init__(self, dbobj, *args, **kwargs): """Provide initialization specific for host bindings.""" if not isinstance(dbobj, Host): raise InternalError("HostChooser can only choose services for " "hosts, got %r (%s)" % (dbobj, type(dbobj))) self.dbhost = dbobj Chooser.__init__(self, dbobj, *args, **kwargs) self.location = self.dbhost.machine.location self.archetype = self.dbhost.archetype self.personality = self.dbhost.personality # If the primary name is a ReservedName, then it does not have a network # attribute if hasattr(self.dbhost.machine.primary_name, 'network'): self.network = self.dbhost.machine.primary_name.network else: self.network = None # all of them would be self. but that should be optimized # dbhost.machine.interfaces[x].assignments[y].network """Stores interim service instance lists.""" q = self.session.query(Service) q = q.outerjoin(Service.archetypes) q = q.reset_joinpoint() q = q.outerjoin(Service.personalities) q = q.filter( or_(Archetype.id == self.archetype.id, Personality.id == self.personality.id)) self.required_services = set(q.all()) self.original_service_instances = {} """Cache of any already bound services (keys) and the instance that was bound (values). """ q = self.session.query(ServiceInstance) q = q.options(undefer('_client_count')) q = q.filter(ServiceInstance.clients.contains(self.dbhost)) set_committed_value(self.dbhost, 'services_used', q.all()) for si in self.dbhost.services_used: self.original_service_instances[si.service] = si self.logger.debug("%s original binding: %s", self.description, si.cfg_path) self.cluster_aligned_services = {} if self.dbhost.cluster: # Note that cluster services are currently ignored unless # they are otherwise required by the archetype/personality. for si in self.dbhost.cluster.service_bindings: self.cluster_aligned_services[si.service] = si for service in self.dbhost.cluster.required_services: if service not in self.cluster_aligned_services: # Don't just error here because the error() call # has not yet been set up. Will error out later. self.cluster_aligned_services[service] = None # Went back and forth on this... deciding not to force # an aligned service as required. This should give # flexibility for multiple services to be aligned for # a cluster type without being forced on all the # personalities. #self.required_services.add(item.service) if self.dbhost.cluster.metacluster: mc = self.dbhost.cluster.metacluster for si in mc.service_bindings: if si.service in self.cluster_aligned_services: cas = self.cluster_aligned_services[si.service] if cas == None: # Error out later. continue self.logger.client_info( "Replacing {0.name} instance with {1.name} " "(bound to {2:l}) for service {3.name}".format( cas, si, mc, si.service)) self.cluster_aligned_services[si.service] = si for service in mc.required_services: if service not in self.cluster_aligned_services: # Don't just error here because the error() call # has not yet been set up. Will error out later. self.cluster_aligned_services[service] = None
def render(self, session, logger, services, personalities, machines, clusters, hosts, locations, resources, switches, all, **arguments): success = [] failed = [] written = 0 # Caches for keeping preloaded data pinned in memory, since the SQLA # session holds a weak reference only resource_by_id = {} resholder_by_id = {} service_instances = None racks = None # Object caches that are accessed directly disks_by_machine = defaultdict(list) interfaces_by_machine = defaultdict(list) interfaces_by_id = {} if all: services = True personalities = True machines = True clusters = True hosts = True locations = True resources = True with CompileKey(logger=logger): logger.client_info("Loading data.") # When flushing clusters/hosts, loading the resource holder is done # as the query that loads those objects. But when flushing resources # only, we need the holder and the object it belongs to. if resources and not clusters: q = session.query(ClusterResource) # Using joinedload('cluster') would generate an outer join q = q.join(Cluster) q = q.options(contains_eager('cluster')) for resholder in q: resholder_by_id[resholder.id] = resholder if resources and not hosts: q = session.query(HostResource) # Using joinedload('host') would generate an outer join q = q.join(Host) q = q.options(contains_eager('host')) for resholder in q: resholder_by_id[resholder.id] = resholder if hosts or clusters or resources: # Load the most common resource types. Using # with_polymorphic('*') on Resource would generate a huge query, # so do something more targeted. More resource subclasses may be # added later if they become common. preload_classes = { Filesystem: [], RebootSchedule: [], VirtualMachine: [joinedload('machine'), joinedload('machine.primary_name'), joinedload('machine.primary_name.fqdn')], Share: [], } share_info = cache_storage_data() for cls, options in preload_classes.items(): q = session.query(cls) # If only hosts or only clusters are needed, don't load # resources of the other kind if hosts and not clusters and not resources: q = q.join(ResourceHolder) q = q.options(contains_eager('holder')) q = q.filter_by(holder_type='host') if clusters and not hosts and not resources: q = q.join(ResourceHolder) q = q.filter_by(holder_type='cluster') q = q.options(contains_eager('holder')) if options: q = q.options(*options) for res in q: resource_by_id[res.id] = res try: res.populate_share_info(share_info) except AttributeError: pass if hosts or machines: # Polymorphic loading cannot be applied to eager-loaded # attributes, so load interfaces manually. q = session.query(Interface) q = q.with_polymorphic('*') q = q.options(lazyload("hardware_entity")) for iface in q: interfaces_by_machine[iface.hardware_entity_id].append(iface) interfaces_by_id[iface.id] = iface if hosts: # subqueryload() and with_polymorphic() do not play nice # together, so do it by hand q = session.query(AddressAssignment) q = q.options(joinedload("network"), joinedload("dns_records")) q = q.order_by(AddressAssignment._label) addrs_by_iface = defaultdict(list) for addr in q: addrs_by_iface[addr.interface_id].append(addr) for interface_id, addrs in addrs_by_iface.items(): set_committed_value(interfaces_by_id[interface_id], "assignments", addrs) q = session.query(Interface.id) q = q.filter(~Interface.assignments.any()) for id in q.all(): set_committed_value(interfaces_by_id[id[0]], "assignments", None) if hosts or services: q = session.query(ServiceInstance) q = q.options(subqueryload("service")) service_instances = q.all() if machines or clusters: # Most machines are in racks... q = session.query(Rack) q = q.options(subqueryload("dns_maps"), subqueryload("parents")) racks = q.all() if locations: logger.client_info("Flushing locations.") for dbloc in session.query(City).all(): try: plenary = Plenary.get_plenary(dbloc, logger=logger) written += plenary.write(locked=True) except Exception, e: failed.append("City %s failed: %s" % dbloc, e) continue if services: logger.client_info("Flushing services.") q = session.query(Service) q = q.options(subqueryload("instances")) for dbservice in q: try: plenary_info = Plenary.get_plenary(dbservice, logger=logger) written += plenary_info.write(locked=True) except Exception, e: failed.append("Service %s failed: %s" % (dbservice.name, e)) continue for dbinst in dbservice.instances: try: plenary_info = Plenary.get_plenary(dbinst, logger=logger) written += plenary_info.write(locked=True) except Exception, e: failed.append("Service %s instance %s failed: %s" % (dbservice.name, dbinst.name, e)) continue
def render(self, session, logger, hostname, **arguments): # removing the plenary host requires a compile lock, however # we want to avoid deadlock by the fact that we're messing # with two locks here, so we want to be careful. We grab the # plenaryhost early on (in order to get the filenames filled # in from the db info before we delete it from the db. We then # hold onto those references until we've completed the db # cleanup and if all of that is successful, then we delete the # plenary file (which doesn't require re-evaluating any stale # db information) after we've released the delhost lock. delplenary = False # Any service bindings that we need to clean up afterwards bindings = PlenaryCollection(logger=logger) resources = PlenaryCollection(logger=logger) with DeleteKey("system", logger=logger) as key: # Check dependencies, translate into user-friendly message dbhost = hostname_to_host(session, hostname) host_plenary = Plenary.get_plenary(dbhost, logger=logger) domain = dbhost.branch.name deps = get_host_dependencies(session, dbhost) if (len(deps) != 0): deptext = "\n".join([" %s" % d for d in deps]) raise ArgumentError("Cannot delete host %s due to the " "following dependencies:\n%s." % (hostname, deptext)) archetype = dbhost.archetype.name dbmachine = dbhost.machine oldinfo = DSDBRunner.snapshot_hw(dbmachine) ip = dbmachine.primary_ip fqdn = dbmachine.fqdn for si in dbhost.services_used: plenary = PlenaryServiceInstanceServer(si) bindings.append(plenary) logger.info( "Before deleting host '%s', removing binding '%s'" % (fqdn, si.cfg_path)) del dbhost.services_used[:] if dbhost.resholder: for res in dbhost.resholder.resources: resources.append(Plenary.get_plenary(res)) # In case of Zebra, the IP may be configured on multiple interfaces for iface in dbmachine.interfaces: if ip in iface.addresses: iface.addresses.remove(ip) if dbhost.cluster: dbcluster = dbhost.cluster dbcluster.hosts.remove(dbhost) set_committed_value(dbhost, '_cluster', None) dbcluster.validate() dbdns_rec = dbmachine.primary_name dbmachine.primary_name = None dbmachine.host = None session.delete(dbhost) delete_dns_record(dbdns_rec) session.flush() delplenary = True if dbmachine.vm_container: bindings.append(Plenary.get_plenary(dbmachine.vm_container)) if archetype != 'aurora' and ip is not None: dsdb_runner = DSDBRunner(logger=logger) dsdb_runner.update_host(dbmachine, oldinfo) dsdb_runner.commit_or_rollback("Could not remove host %s from " "DSDB" % hostname) if archetype == 'aurora': logger.client_info("WARNING: removing host %s from AQDB and " "*not* changing DSDB." % hostname) # Past the point of no return... commit the transaction so # that we can free the delete lock. session.commit() # Only if we got here with no exceptions do we clean the template # Trying to clean up after any errors here is really difficult # since the changes to dsdb have already been made. if (delplenary): key = host_plenary.get_remove_key() with CompileKey.merge( [key, bindings.get_write_key(), resources.get_remove_key()]) as key: host_plenary.cleanup(domain, locked=True) # And we also want to remove the profile itself profiles = self.config.get("broker", "profilesdir") # Only one of these should exist, but it doesn't hurt # to try to clean up both. xmlfile = os.path.join(profiles, fqdn + ".xml") remove_file(xmlfile, logger=logger) xmlgzfile = xmlfile + ".gz" remove_file(xmlgzfile, logger=logger) # And the cached template created by ant remove_file(os.path.join( self.config.get("broker", "quattordir"), "objects", fqdn + TEMPLATE_EXTENSION), logger=logger) bindings.write(locked=True) resources.remove(locked=True) build_index(self.config, session, profiles, logger=logger) return
def add_net_data(self, net_msg, net): net_msg.name = str(net.name) net_msg.id = net.id net_msg.ip = str(net.ip) net_msg.cidr = net.cidr net_msg.bcast = str(net.broadcast) net_msg.netmask = str(net.netmask) net_msg.side = str(net.side) net_msg.sysloc = str(net.location.sysloc()) net_msg.location.name = str(net.location.name) net_msg.location.location_type = str(net.location.location_type) net_msg.type = str(net.network_type) # Bulk load information about anything having a network address on this # network hw_ids = set([addr.interface.hardware_entity_id for addr in net.assignments]) if hw_ids: session = object_session(net) q = session.query(HardwareEntity) q = q.filter(HardwareEntity.id.in_(hw_ids)) q = q.options(subqueryload('interfaces'), lazyload('interfaces.hardware_entity')) hwent_by_id = {} for dbhwent in q.all(): hwent_by_id[dbhwent.id] = dbhwent iface_by_id = {} slaves_by_id = defaultdict(list) # We have all the interfaces loaded already, so compute the # master/slave relationships to avoid having to touch the # database again for iface in dbhwent.interfaces: iface_by_id[iface.id] = iface if iface.master_id: slaves_by_id[iface.master_id].append(iface) for iface in dbhwent.interfaces: set_committed_value(iface, "master", iface_by_id.get(iface.master_id, None)) set_committed_value(iface, "slaves", slaves_by_id[iface.id]) # TODO: once we refactor Host to be an FK to HardwareEntity instead # of Machine, this could be converted to a single joinedload('host') q = session.query(Host) q = q.options(lazyload('hardware_entity')) q = q.filter(Host.hardware_entity_id.in_(hw_ids)) for host in q.all(): set_committed_value(hwent_by_id[host.hardware_entity_id], "host", host) set_committed_value(host, "hardware_entity", hwent_by_id[host.hardware_entity_id]) # Add interfaces that have addresses in this network for addr in net.assignments: if not addr.dns_records: # hostname is a required field in the protobuf description continue hwent = addr.interface.hardware_entity # DHCP: we do not care about secondary IP addresses, but in some # cases the same IP address may show up with different MACs if not addr.label: mac_addrs = possible_mac_addresses(addr.interface) else: mac_addrs = [] # Generate a host record even if there is no known MAC address for # it if not mac_addrs: mac_addrs.append(None) # Associating the same IP with multiple MAC addresses is # problematic using the current protocol. Sending multiple host # messages is easy for the broker, but it can confuse consumers like # aqdhcpd. For now just ensure it never happens, and revisit the # problem when we have a real world requirement. if len(mac_addrs) > 1: mac_addrs = [mac_addrs[0]] for mac in mac_addrs: host_msg = net_msg.hosts.add() if addr.interface.interface_type == 'management': host_msg.type = 'manager' else: if hwent.hardware_type == 'machine': host_msg.type = 'host' if hwent.host: host_msg.archetype.name = str(hwent.host.archetype.name) elif hwent.hardware_type == 'switch': # aqdhcpd uses the type host_msg.type = 'tor_switch' else: host_msg.type = hwent.hardware_type host_msg.hostname = str(addr.dns_records[0].fqdn.name) host_msg.fqdn = str(addr.dns_records[0].fqdn) host_msg.dns_domain = str(addr.dns_records[0].fqdn.dns_domain) host_msg.ip = str(addr.ip) if mac: host_msg.mac = mac host_msg.machine.name = str(hwent.label) # aqdhcpd uses the interface list when excluding hosts it is not # authoritative for for iface in hwent.interfaces: int_msg = host_msg.machine.interfaces.add() int_msg.device = iface.name if iface.mac: int_msg.mac = str(iface.mac) # Add dynamic DHCP records for dynhost in net.dynamic_stubs: host_msg = net_msg.hosts.add() # aqdhcpd uses the type host_msg.type = 'dynamic_stub' host_msg.hostname = str(dynhost.fqdn.name) host_msg.fqdn = str(dynhost.fqdn) host_msg.dns_domain = str(dynhost.fqdn.dns_domain) host_msg.ip = str(dynhost.ip)
def create_transaction(self, session): """ Create transaction object for given SQLAlchemy session. :param session: SQLAlchemy session object """ args = self.transaction_args(session) Transaction = self.manager.transaction_cls table = Transaction.__table__ self.current_transaction = Transaction() if self.manager.options['native_versioning']: has_transaction_initialized = bool(session.execute( '''SELECT 1 FROM pg_catalog.pg_class WHERE relname = 'temporary_transaction' ''' ).scalar()) if has_transaction_initialized: tx_id = ( session.execute('SELECT id FROM temporary_transaction') .scalar() ) set_committed_value(self.current_transaction, 'id', tx_id) else: criteria = {'native_tx_id': sa.func.txid_current()} args.update(criteria) query = ( table.insert() .values(**args) .returning(*map(sa.text, list(args.keys()) + ['id'])) ) values = session.execute(query).fetchone() for key, value in values.items(): set_committed_value(self.current_transaction, key, value) session.execute( ''' CREATE TEMP TABLE temporary_transaction (id BIGINT, PRIMARY KEY(id)) ON COMMIT DROP ''' ) session.execute(''' INSERT INTO temporary_transaction (id) VALUES (:id) ''', {'id': self.current_transaction.id} ) self.merge_transaction(session, self.current_transaction) else: for key, value in args.items(): setattr(self.current_transaction, key, value) if not self.version_session: self.version_session = sa.orm.session.Session( bind=session.connection() ) self.version_session.add(self.current_transaction) self.version_session.flush() self.version_session.expunge(self.current_transaction) session.add(self.current_transaction) return self.current_transaction
def render(self, session, logger, services, personalities, machines, clusters, hosts, locations, resources, switches, all, **arguments): success = [] failed = [] written = 0 # Caches for keeping preloaded data pinned in memory, since the SQLA # session holds a weak reference only resource_by_id = {} resholder_by_id = {} service_instances = None racks = None # Object caches that are accessed directly disks_by_machine = defaultdict(list) interfaces_by_machine = defaultdict(list) interfaces_by_id = {} if all: services = True personalities = True machines = True clusters = True hosts = True locations = True resources = True with CompileKey(logger=logger): logger.client_info("Loading data.") # When flushing clusters/hosts, loading the resource holder is done # as the query that loads those objects. But when flushing resources # only, we need the holder and the object it belongs to. if resources and not clusters: q = session.query(ClusterResource) # Using joinedload('cluster') would generate an outer join q = q.join(Cluster) q = q.options(contains_eager('cluster')) for resholder in q: resholder_by_id[resholder.id] = resholder if resources and not hosts: q = session.query(HostResource) # Using joinedload('host') would generate an outer join q = q.join(Host) q = q.options(contains_eager('host')) for resholder in q: resholder_by_id[resholder.id] = resholder if hosts or clusters or resources: # Load the most common resource types. Using # with_polymorphic('*') on Resource would generate a huge query, # so do something more targeted. More resource subclasses may be # added later if they become common. preload_classes = { Filesystem: [], RebootSchedule: [], VirtualMachine: [ joinedload('machine'), joinedload('machine.primary_name'), joinedload('machine.primary_name.fqdn') ], Share: [], } share_info = cache_storage_data() for cls, options in preload_classes.items(): q = session.query(cls) # If only hosts or only clusters are needed, don't load # resources of the other kind if hosts and not clusters and not resources: q = q.join(ResourceHolder) q = q.options(contains_eager('holder')) q = q.filter_by(holder_type='host') if clusters and not hosts and not resources: q = q.join(ResourceHolder) q = q.filter_by(holder_type='cluster') q = q.options(contains_eager('holder')) if options: q = q.options(*options) for res in q: resource_by_id[res.id] = res try: res.populate_share_info(share_info) except AttributeError: pass if hosts or machines: # Polymorphic loading cannot be applied to eager-loaded # attributes, so load interfaces manually. q = session.query(Interface) q = q.with_polymorphic('*') q = q.options(lazyload("hardware_entity")) for iface in q: interfaces_by_machine[iface.hardware_entity_id].append( iface) interfaces_by_id[iface.id] = iface if hosts: # subqueryload() and with_polymorphic() do not play nice # together, so do it by hand q = session.query(AddressAssignment) q = q.options(joinedload("network"), joinedload("dns_records")) q = q.order_by(AddressAssignment._label) addrs_by_iface = defaultdict(list) for addr in q: addrs_by_iface[addr.interface_id].append(addr) for interface_id, addrs in addrs_by_iface.items(): set_committed_value(interfaces_by_id[interface_id], "assignments", addrs) q = session.query(Interface.id) q = q.filter(~Interface.assignments.any()) for id in q.all(): set_committed_value(interfaces_by_id[id[0]], "assignments", None) if hosts or services: q = session.query(ServiceInstance) q = q.options(subqueryload("service")) service_instances = q.all() if machines or clusters: # Most machines are in racks... q = session.query(Rack) q = q.options(subqueryload("dns_maps"), subqueryload("parents")) racks = q.all() if locations: logger.client_info("Flushing locations.") for dbloc in session.query(City).all(): try: plenary = Plenary.get_plenary(dbloc, logger=logger) written += plenary.write(locked=True) except Exception, e: failed.append("City %s failed: %s" % dbloc, e) continue if services: logger.client_info("Flushing services.") q = session.query(Service) q = q.options(subqueryload("instances")) for dbservice in q: try: plenary_info = Plenary.get_plenary(dbservice, logger=logger) written += plenary_info.write(locked=True) except Exception, e: failed.append("Service %s failed: %s" % (dbservice.name, e)) continue for dbinst in dbservice.instances: try: plenary_info = Plenary.get_plenary(dbinst, logger=logger) written += plenary_info.write(locked=True) except Exception, e: failed.append("Service %s instance %s failed: %s" % (dbservice.name, dbinst.name, e)) continue
class Media(object): """ Media metadata and a collection of related files. """ meta = association_proxy('_meta', 'value', creator=MediaMeta) query = DBSession.query_property(MediaQuery) # TODO: replace '_thumb_dir' with something more generic, like 'name', # so that its other uses throughout the code make more sense. _thumb_dir = 'media' def __init__(self): if self.author is None: self.author = Author() def __repr__(self): return '<Media: %s>' % self.slug def set_tags(self, tags): """Set the tags relations of this media, creating them as needed. :param tags: A list or comma separated string of tags to use. """ if isinstance(tags, basestring): tags = extract_tags(tags) if isinstance(tags, list) and tags: tags = fetch_and_create_tags(tags) self.tags = tags or [] def set_categories(self, cats): """Set the related categories of this media. :param cats: A list of category IDs to set. """ if cats: cats = Category.query.filter(Category.id.in_(cats)).all() self.categories = cats or [] def update_status(self): """Ensure the type (audio/video) and encoded flag are properly set. Call this after modifying any files belonging to this item. """ self.type = self._update_type() self.encoded = self._update_encoding() def _update_type(self): """Update the type of this Media object. If there's a video file, mark this as a video type, else fallback to audio, if possible, or unknown (None) """ if any(file.type == VIDEO for file in self.files): return VIDEO elif any(file.type == AUDIO for file in self.files): return AUDIO else: return None def _update_encoding(self): # Test to see if we can find a workable file/player combination # for the most common podcasting app w/ the POOREST format support if self.podcast_id \ and not pick_podcast_media_file(self): return False # Test to see if we can find a workable file/player combination # for the browser w/ the BEST format support if not pick_any_media_file(self): return False return True @property def is_published(self): if self.id is None: return False return self.publishable and self.reviewed and self.encoded\ and (self.publish_on is not None and self.publish_on <= datetime.now())\ and (self.publish_until is None or self.publish_until >= datetime.now()) def increment_views(self): """Increment the number of views in the database. We avoid concurrency issues by incrementing JUST the views and not allowing modified_on to be updated automatically. """ if self.id is None: self.views += 1 return self.views # Don't raise an exception should concurrency problems occur. # Views will not actually be incremented in this case, but thats # relatively unimportant compared to rendering the page for the user. # We may be able to remove this after we improve our triggers to not # issue an UPDATE on media_fulltext unless one of its columns are # actually changed. Even when just media.views is updated, all the # columns in the corresponding media_fulltext row are updated, and # media_fulltext's MyISAM engine must lock the whole table to do so. transaction = DBSession.begin_nested() try: DBSession.query(self.__class__)\ .filter(self.__class__.id == self.id)\ .update({self.__class__.views: self.__class__.views + 1}) transaction.commit() except OperationalError, e: transaction.rollback() # (OperationalError) (1205, 'Lock wait timeout exceeded, try restarting the transaction') if not '1205' in e.message: raise # Increment the views by one for the rest of the request, # but don't allow the ORM to increment the views too. attributes.set_committed_value(self, 'views', self.views + 1) return self.views
def render(self, session, logger, hostname, **arguments): # removing the plenary host requires a compile lock, however # we want to avoid deadlock by the fact that we're messing # with two locks here, so we want to be careful. We grab the # plenaryhost early on (in order to get the filenames filled # in from the db info before we delete it from the db. We then # hold onto those references until we've completed the db # cleanup and if all of that is successful, then we delete the # plenary file (which doesn't require re-evaluating any stale # db information) after we've released the delhost lock. delplenary = False # Any service bindings that we need to clean up afterwards bindings = PlenaryCollection(logger=logger) resources = PlenaryCollection(logger=logger) with DeleteKey("system", logger=logger) as key: # Check dependencies, translate into user-friendly message dbhost = hostname_to_host(session, hostname) host_plenary = Plenary.get_plenary(dbhost, logger=logger) domain = dbhost.branch.name deps = get_host_dependencies(session, dbhost) if (len(deps) != 0): deptext = "\n".join([" %s" % d for d in deps]) raise ArgumentError("Cannot delete host %s due to the " "following dependencies:\n%s." % (hostname, deptext)) archetype = dbhost.archetype.name dbmachine = dbhost.machine oldinfo = DSDBRunner.snapshot_hw(dbmachine) ip = dbmachine.primary_ip fqdn = dbmachine.fqdn for si in dbhost.services_used: plenary = PlenaryServiceInstanceServer(si) bindings.append(plenary) logger.info("Before deleting host '%s', removing binding '%s'" % (fqdn, si.cfg_path)) del dbhost.services_used[:] if dbhost.resholder: for res in dbhost.resholder.resources: resources.append(Plenary.get_plenary(res)) # In case of Zebra, the IP may be configured on multiple interfaces for iface in dbmachine.interfaces: if ip in iface.addresses: iface.addresses.remove(ip) if dbhost.cluster: dbcluster = dbhost.cluster dbcluster.hosts.remove(dbhost) set_committed_value(dbhost, '_cluster', None) dbcluster.validate() dbdns_rec = dbmachine.primary_name dbmachine.primary_name = None dbmachine.host = None session.delete(dbhost) delete_dns_record(dbdns_rec) session.flush() delplenary = True if dbmachine.vm_container: bindings.append(Plenary.get_plenary(dbmachine.vm_container)) if archetype != 'aurora' and ip is not None: dsdb_runner = DSDBRunner(logger=logger) dsdb_runner.update_host(dbmachine, oldinfo) dsdb_runner.commit_or_rollback("Could not remove host %s from " "DSDB" % hostname) if archetype == 'aurora': logger.client_info("WARNING: removing host %s from AQDB and " "*not* changing DSDB." % hostname) # Past the point of no return... commit the transaction so # that we can free the delete lock. session.commit() # Only if we got here with no exceptions do we clean the template # Trying to clean up after any errors here is really difficult # since the changes to dsdb have already been made. if (delplenary): key = host_plenary.get_remove_key() with CompileKey.merge([key, bindings.get_write_key(), resources.get_remove_key()]) as key: host_plenary.cleanup(domain, locked=True) # And we also want to remove the profile itself profiles = self.config.get("broker", "profilesdir") # Only one of these should exist, but it doesn't hurt # to try to clean up both. xmlfile = os.path.join(profiles, fqdn + ".xml") remove_file(xmlfile, logger=logger) xmlgzfile = xmlfile + ".gz" remove_file(xmlgzfile, logger=logger) # And the cached template created by ant remove_file(os.path.join(self.config.get("broker", "quattordir"), "objects", fqdn + TEMPLATE_EXTENSION), logger=logger) bindings.write(locked=True) resources.remove(locked=True) build_index(self.config, session, profiles, logger=logger) return
def _populate_backrefs(target, context): for name, backref in mappings.items(): # __dict__ to avoid triggering lazy-loaded relationships if target.__dict__.get(name) is not None: set_committed_value(getattr(target, name), backref, target)
def get_unique(cls, sess, name, hardware_type=None, compel=False, preclude=False, query_options=None): """ Returns a unique HardwareEntity given session and fqdn """ # If the hardware_type param isn't explicitly set and we have a # polymorphic identity, assume we're querying only for items of our # hardware_type. if hardware_type: if isclass(hardware_type): clslabel = hardware_type._get_class_label() hardware_type = hardware_type.__mapper_args__[ 'polymorphic_identity'] else: pcls = cls.__mapper__.polymorphic_map[hardware_type].class_ clslabel = pcls._get_class_label() else: if 'polymorphic_identity' in cls.__mapper_args__: hardware_type = cls.__mapper_args__['polymorphic_identity'] clslabel = cls._get_class_label() # The automagic DNS lookup does not really make sense with preclude=True if preclude: name = AqStr.normalize(name) cls.check_label(name) q = sess.query(cls) if "." in name: dns_rec = DnsRecord.get_unique(sess, fqdn=name, compel=True) # We know the primary name, do not load it again q = q.options(lazyload('primary_name')) q = q.filter_by(primary_name=dns_rec) else: dns_rec = None q = q.filter_by(label=name) if query_options: q = q.options(*query_options) try: hwe = q.one() except NoResultFound: # Check if the name is in use by a different hardware type q = sess.query(HardwareEntity) if dns_rec: # We know the primary name, do not load it again q = q.options(lazyload('primary_name')) q = q.filter_by(primary_name=dns_rec) else: q = q.filter_by(label=name) try: hwe = q.one() if dns_rec: # We know the primary name, do not load it again set_committed_value(hwe, 'primary_name', dns_rec) raise ArgumentError("{0} exists, but is not a {1}.".format( hwe, hardware_type)) except NoResultFound: hwe = None if compel: raise NotFoundException("%s %s not found." % (clslabel, name)) if hwe: if preclude: raise ArgumentError('{0} already exists.'.format(hwe)) if dns_rec: # We know the primary name, do not load it again set_committed_value(hwe, 'primary_name', dns_rec) return hwe