Example #1
0
def get_upcoming_events():
    """Get the global list of upcoming events"""
    from indico.modules.events import Event
    data = upcoming_events_settings.get_all()
    if not data['max_entries'] or not data['entries']:
        return
    tz = timezone(Config.getInstance().getDefaultTimezone())
    now = now_utc(False).astimezone(tz)
    base_query = (Event.query
                  .filter(Event.effective_protection_mode == ProtectionMode.public,
                          ~Event.is_deleted,
                          Event.end_dt.astimezone(tz) > now)
                  .options(load_only('id', 'title', 'start_dt', 'end_dt')))
    queries = []
    cols = {'category': Event.category_id,
            'event': Event.id}
    for entry in data['entries']:
        delta = timedelta(days=entry['days'])
        query = (base_query
                 .filter(cols[entry['type']] == entry['id'])
                 .filter(db.cast(Event.start_dt.astimezone(tz), db.Date) > (now - delta).date())
                 .with_entities(Event, db.literal(entry['weight']).label('weight')))
        queries.append(query)

    query = (queries[0].union(*queries[1:])
             .order_by(db.desc('weight'), Event.start_dt, Event.title)
             .limit(data['max_entries']))
    for row in query:
        event = row[0]
        # we cache the result of the function and is_deleted is used in the repr
        # and having a broken repr on the cached objects would be ugly
        set_committed_value(event, 'is_deleted', False)
        yield event
Example #2
0
    def new_version(self, session):
        # convert to an INSERT
        make_transient(self)
        self.id = None

        # history of the 'elements' collection.
        # this is a tuple of groups: (added, unchanged, deleted)
        hist = attributes.get_history(self, "elements")

        # rewrite the 'elements' collection
        # from scratch, removing all history
        attributes.set_committed_value(self, "elements", {})

        # new elements in the "added" group
        # are moved to our new collection.
        for elem in hist.added:
            self.elements[elem.name] = elem

        # copy elements in the 'unchanged' group.
        # the new ones associate with the new ConfigData,
        # the old ones stay associated with the old ConfigData
        for elem in hist.unchanged:
            self.elements[elem.name] = ConfigValueAssociation(
                elem.config_value
            )
Example #3
0
    def validate(self, max_hosts=None, error=ArgumentError, **kwargs):
        session = object_session(self)
        q = session.query(HostClusterMember)
        q = q.filter_by(cluster=self)
        q = q.options(joinedload('host'), joinedload('host.machine'))
        members = q.all()
        set_committed_value(self, '_hosts', members)

        if self.cluster_type != 'meta':
            for i in [
                    "down_hosts_threshold", "down_hosts_percent",
                    "down_maint_percent", "personality_id"
                    #"branch_id"
            ]:
                if getattr(self, i, None) is None:
                    raise error("Attribute %s must be set for a %s cluster." %
                                (i, self.cluster_type))
        else:
            if self.metacluster:
                raise error("Metaclusters can't contain other metaclusters.")

        if max_hosts is None:
            max_hosts = self.max_hosts
        if len(self.hosts) > self.max_hosts:
            raise error("{0} is over capacity of {1} hosts.".format(
                self, max_hosts))
        if self.metacluster:
            self.metacluster.validate()
Example #4
0
def get_upcoming_events():
    """Get the global list of upcoming events"""
    from indico.modules.events import Event
    data = upcoming_events_settings.get_all()
    if not data['max_entries'] or not data['entries']:
        return
    tz = timezone(config.DEFAULT_TIMEZONE)
    now = now_utc(False).astimezone(tz)
    base_query = (Event.query
                  .filter(Event.effective_protection_mode == ProtectionMode.public,
                          ~Event.is_deleted,
                          Event.end_dt.astimezone(tz) > now)
                  .options(load_only('id', 'title', 'start_dt', 'end_dt')))
    queries = []
    cols = {'category': Event.category_id,
            'event': Event.id}
    for entry in data['entries']:
        delta = timedelta(days=entry['days'])
        query = (base_query
                 .filter(cols[entry['type']] == entry['id'])
                 .filter(db.cast(Event.start_dt.astimezone(tz), db.Date) > (now - delta).date())
                 .with_entities(Event, db.literal(entry['weight']).label('weight')))
        queries.append(query)

    query = (queries[0].union(*queries[1:])
             .order_by(db.desc('weight'), Event.start_dt, Event.title)
             .limit(data['max_entries']))
    for row in query:
        event = row[0]
        # we cache the result of the function and is_deleted is used in the repr
        # and having a broken repr on the cached objects would be ugly
        set_committed_value(event, 'is_deleted', False)
        yield event
Example #5
0
 def _populate_preloaded_relationships(cls, target, *unused):
     cache = g.get('relationship_cache', {}).get(type(target))
     if not cache:
         return
     for rel, value in cache['data'].get(target, {}).iteritems():
         if rel not in target.__dict__:
             set_committed_value(target, rel, value)
Example #6
0
def populated_tree(cats):
    """Return the root categories with children populated to any depth.

    Adjacency lists are notoriously inefficient for fetching deeply
    nested trees, and since our dataset will always be reasonably
    small, this method should greatly improve efficiency. Only one
    query is necessary to fetch a tree of any depth. This isn't
    always the solution, but for some situations, it is worthwhile.

    For example, printing the entire tree can be done with one query::

        query = Category.query.options(undefer('media_count'))
        for cat, depth in query.populated_tree().traverse():
            print "    " * depth, cat.name, '(%d)' % cat.media_count

    Without this method, especially with the media_count undeferred,
    this would require a lot of extra queries for nested categories.

    NOTE: If the tree contains circular nesting, the circular portion
          of the tree will be silently omitted from the results.

    """
    children = defaultdict(CategoryList)
    for cat in cats:
        children[cat.parent_id].append(cat)
    for cat in cats:
        set_committed_value(cat, 'children', children[cat.id])
    return children[None]
Example #7
0
 def _populate_preloaded_relationships(cls, target, *unused):
     cache = g.get('relationship_cache', {}).get(type(target))
     if not cache:
         return
     for rel, value in cache['data'].get(target, {}).iteritems():
         if rel not in target.__dict__:
             set_committed_value(target, rel, value)
Example #8
0
def _bulk_load_column_for_instance_states(
        session: Session, mapper: Mapper, identities: Iterable[Tuple],
        attr_name: str, alter_query: Optional[QueryAlterator]):
    """ Load a column attribute for a list of instance states where the attribute is unloaded """
    Model = mapper.class_
    attr: Column = mapper.columns[attr_name]

    # Using those identities (primary keys), load the missing attribute
    q = load_by_primary_keys(session, mapper, identities, attr)

    # Alter the query
    if alter_query:
        q = alter_query(q, mapper, attr_name, False)

    # Having the missing attribute's value loaded, assign it to every instance in the session
    for identity, attr_value in q:
        # Build the identity key the way SqlAlchemy likes it:
        # (Model, primary-key, None)
        key = identity_key(Model, identity)

        # We do not iterate the Session to find an instance that matches the primary key.
        # Instead, we take it directly using the `identity_map`
        instance = session.identity_map[key]

        # Set the value of the missing attribute.
        # This is how it immediately becomes loaded.
        # Note that this action does not overwrite any modifications made to the attribute.
        set_committed_value(instance, attr_name, attr_value)
Example #9
0
def populated_tree(cats):
    """Return the root categories with children populated to any depth.

    Adjacency lists are notoriously inefficient for fetching deeply
    nested trees, and since our dataset will always be reasonably
    small, this method should greatly improve efficiency. Only one
    query is necessary to fetch a tree of any depth. This isn't
    always the solution, but for some situations, it is worthwhile.

    For example, printing the entire tree can be done with one query::

        query = Category.query.options(undefer('media_count'))
        for cat, depth in query.populated_tree().traverse():
            print "    " * depth, cat.name, '(%d)' % cat.media_count

    Without this method, especially with the media_count undeferred,
    this would require a lot of extra queries for nested categories.

    NOTE: If the tree contains circular nesting, the circular portion
          of the tree will be silently omitted from the results.

    """
    roots = CategoryList()
    children = defaultdict(list)
    for cat in cats:
        if cat.parent_id:
            children[cat.parent_id].append(cat)
        else:
            roots.append(cat)
    for cat in cats:
        attributes.set_committed_value(cat, 'children', children[cat.id])
    return roots
Example #10
0
    def new_version(self, session):
        # convert to an INSERT
        make_transient(self)
        self.id = None

        # history of the 'elements' collection.
        # this is a tuple of groups: (added, unchanged, deleted)
        hist = attributes.get_history(self, "elements")

        # rewrite the 'elements' collection
        # from scratch, removing all history
        attributes.set_committed_value(self, "elements", {})

        # new elements in the "added" group
        # are moved to our new collection.
        for elem in hist.added:
            self.elements[elem.name] = elem

        # copy elements in the 'unchanged' group.
        # the new ones associate with the new ConfigData,
        # the old ones stay associated with the old ConfigData
        for elem in hist.unchanged:
            self.elements[elem.name] = ConfigValueAssociation(
                elem.config_value
            )
Example #11
0
    def validate(self, max_hosts=None, error=ArgumentError, **kwargs):
        session = object_session(self)
        q = session.query(HostClusterMember)
        q = q.filter_by(cluster=self)
        q = q.options(joinedload('host'),
                      joinedload('host.machine'))
        members = q.all()
        set_committed_value(self, '_hosts', members)

        if self.cluster_type != 'meta':
            for i in [
                    "down_hosts_threshold",
                    "down_hosts_percent",
                    "down_maint_percent",
                    "personality_id"
                    #"branch_id"
                ]:
                if getattr(self, i, None) is None:
                    raise error("Attribute %s must be set for a %s cluster." %
                                (i, self.cluster_type))
        else:
            if self.metacluster:
                raise error("Metaclusters can't contain other metaclusters.")

        if max_hosts is None:
            max_hosts = self.max_hosts
        if len(self.hosts) > self.max_hosts:
            raise error("{0} is over capacity of {1} hosts.".format(self,
                                                                    max_hosts))
        if self.metacluster:
            self.metacluster.validate()
Example #12
0
def intercept_after_flush(session, obj):
    if isinstance(obj, QuestionModel):
        print(obj)
        if obj.questionKey == "set":
            obj.set_questionKey()
            session.query(QuestionModel).filter_by(id=obj.id).update(
                {"questionKey": obj.questionKey})
            set_committed_value(obj, "questionKey", obj.questionKey)
Example #13
0
 def populate(self):
     """
     Populate batch fetched entities to parent objects.
     """
     for entity in self.path.entities:
         set_committed_value(
             entity, self.prop.key,
             self.parent_dict[local_values(self.prop, entity)])
Example #14
0
 def populate(self):
     """
     Populate batch fetched entities to parent objects.
     """
     for entity in self.path.entities:
         set_committed_value(
             entity,
             self.prop.key,
             self.parent_dict[local_values(self.prop, entity)]
         )
Example #15
0
def sa_set_committed_state(obj: object, **committed_values):
    """ Put values into an SqlAlchemy instance as if they were committed to the DB """
    # Give it some DB identity so that SA thinks it can load something
    state: InstanceState = instance_state(obj)
    state.key = object()

    # Set every attribute in such a way that SA thinkg that's the way it looks in the DB
    for k, v in committed_values.items():
        set_committed_value(obj, k, v)

    return obj
Example #16
0
    def populate(self):
        """
        Populate batch fetched entities to parent objects.
        """
        for entity in self.path.entities:
            set_committed_value(
                entity,
                self.prop.key,
                self.parent_dict[self.local_values(entity)]
            )

        if self.path.populate_backrefs:
            self.populate_backrefs(self.related_entities)
 def _set_results_on_models(self, param_value_to_results,
                            param_value_to_models, current_model):
     current_model_result = None
     for value, models in param_value_to_models.items():
         for model in models:
             result = param_value_to_results.get(value, [])
             # ensure models aren't given identical result lists so modifying results in one doesn't modify others
             result = result[:]
             if not self.uselist:
                 result = self._extract_non_list_result(result)
             if model == current_model:
                 current_model_result = result
             attributes.set_committed_value(model, self.key, result)
     return current_model_result
Example #18
0
 def populate_backrefs(self, related_entities):
     """
     Populates backrefs for given related entities.
     """
     backref_dict = dict((local_values(self.prop, value[0]), [])
                         for value in related_entities)
     for value in related_entities:
         backref_dict[local_values(self.prop, value[0])].append(
             self.path.session.query(self.path.entities[0].__class__).get(
                 tuple(value[1:])))
     for value in related_entities:
         set_committed_value(
             value[0], self.prop.back_populates,
             backref_dict[local_values(self.prop, value[0])])
Example #19
0
    def get_unique(cls,
                   session,
                   fqdn=None,
                   name=None,
                   dns_domain=None,
                   dns_environment=None,
                   compel=False,
                   preclude=False,
                   **kwargs):
        # Proxy FQDN lookup to the Fqdn class
        if not fqdn or not isinstance(fqdn, Fqdn):
            if not isinstance(dns_environment, DnsEnvironment):
                dns_environment = DnsEnvironment.get_unique_or_default(
                    session, dns_environment)
            if fqdn:
                if name or dns_domain:  # pragma: no cover
                    raise TypeError("fqdn and name/dns_domain cannot be mixed")
                (name, dns_domain) = parse_fqdn(session, fqdn)
            try:
                # Do not pass preclude=True to Fqdn
                fqdn = Fqdn.get_unique(session,
                                       name=name,
                                       dns_domain=dns_domain,
                                       dns_environment=dns_environment,
                                       compel=compel)
            except NotFoundException:
                # Replace the "Fqdn ... not found" message with a more user
                # friendly one
                msg = "%s %s.%s, %s not found." % (
                    cls._get_class_label(), name, dns_domain,
                    format(dns_environment, "l"))
                raise NotFoundException(msg)
            if not fqdn:
                return None

        # We already have the FQDN, no need to load it again
        if "query_options" not in kwargs:
            kwargs["query_options"] = [lazyload("fqdn")]

        result = super(DnsRecord, cls).get_unique(session,
                                                  fqdn=fqdn,
                                                  compel=compel,
                                                  preclude=preclude,
                                                  **kwargs)
        if result:
            # Make sure not to load the relation again if we already know its
            # value
            set_committed_value(result, 'fqdn', fqdn)
        return result
Example #20
0
    def validate(self):
        session = object_session(self)
        q = session.query(HostClusterMember)
        q = q.filter_by(cluster=self)
        q = q.options(joinedload('host'),
                      joinedload('host.hardware_entity'))
        members = q.all()
        set_committed_value(self, '_hosts', members)

        if self.max_hosts is not None and len(self.hosts) > self.max_hosts:
            raise ArgumentError("{0} has {1} hosts bound, which exceeds the "
                                "requested limit of {2}."
                                .format(self, len(self.hosts), self.max_hosts))
        if self.metacluster:
            self.metacluster.validate()
Example #21
0
def populate_related(parents, id_key, res_key, reltype, subq, flt=None, relid_key='id'):
	ids = []
	for p in parents:
		if callable(flt) and not flt(p):
			continue
		keyval = getattr(p, id_key, None)
		if keyval and (keyval not in ids):
			ids.append(keyval)
	if len(ids) > 0:
		for rel in subq.filter(getattr(reltype, relid_key).in_(ids)):
			for p in parents:
				if callable(flt) and not flt(p):
					continue
				keyval = getattr(p, id_key, None)
				if keyval and (keyval == getattr(rel, relid_key)):
					attributes.set_committed_value(p, res_key, rel)
Example #22
0
    def increment_views(self):
        """Increment the number of views in the database.

        We avoid concurrency issues by incrementing JUST the views and
        not allowing modified_on to be updated automatically.

        """
        if self.id is None:
            self.views += 1
            return self.views

        DBSession.execute(media.update().values(views=media.c.views + 1).where(media.c.id == self.id))

        # Increment the views by one for the rest of the request,
        # but don't allow the ORM to increment the views too.
        attributes.set_committed_value(self, "views", self.views + 1)
        return self.views
Example #23
0
def fetch_as_tree(query: Query,
                  relationship: Optional[InstrumentedAttribute] = None,
                  root_value: Union[str, None] = None,
                  ) -> List[DeclarativeMeta]:
    """Builds the full tree for the given table (adjacency list).

    This method is efficient in the sense that it issues only one SELECT
    query to the database.

    Args:
        query: A query that selects all relevant model records, that
            is, all records that make up the adjacency list. For
            example: `session.query(Album).order_by(Album.nameLC)`.
        relationship: The parent/child relationship that describes
            both fields (columns). For example: `Album.parent`
            If not set, one record is fetched from the query to read
            its `parent` property.
        root_value: Parent property's value at the root nodes.
            Default is None, which commonly retrieves the whole
            structure.

    Returns:
        A list of root nodes with child nodes (the tree) pre-fetched
        recursively.
    """
    if relationship is None:
        # Fetch one record to discover the parent relationship.
        relationship = next(iter(query)).__class__.parent

    parent_field = relationship.expression.right.name
    child_field = relationship.expression.left.name
    back_populates = relationship.property.back_populates

    nodes = query.all()

    children = defaultdict(list)
    for node in nodes:
        children[getattr(node, parent_field)].append(node)

    for node in nodes:
        set_committed_value(node, back_populates,
                            children[getattr(node, child_field)])

    return children[root_value]
Example #24
0
    def populate(self):
        """
        Populate batch fetched entities to parent objects.
        """
        for entity in self.path.entities:
            # print (
            #     "setting committed value for ",
            #     entity,
            #     " using local values ",
            #     self.local_values(entity)
            # )
            set_committed_value(
                entity,
                self.prop.key,
                self.parent_dict[self.local_values(entity)]
            )

        if self.path.populate_backrefs:
            self.populate_backrefs(self.related_entities)
def load_projects():
    projects = session.query(Project).options(joinedload('quotas')).all()

    root = None
    children = defaultdict(list)
    for project in projects:
        if project.parent:
            children[project.parent.id].append(project)
        if project.parent is None:
            root = project

    for project in projects:
        set_committed_value(project, 'children', children[project.id])
        project.limits = {
            quota.resource: quota.limit
            for quota in project.quotas
        }

    return root
Example #26
0
    def increment_views(self):
        """Increment the number of views in the database.

        We avoid concurrency issues by incrementing JUST the views and
        not allowing modified_on to be updated automatically.

        """
        if self.id is None:
            self.views += 1
            return self.views

        DBSession.execute(media.update()\
            .values(views=media.c.views + 1)\
            .where(media.c.id == self.id))

        # Increment the views by one for the rest of the request,
        # but don't allow the ORM to increment the views too.
        attributes.set_committed_value(self, 'views', self.views + 1)
        return self.views
Example #27
0
File: util.py Project: unikmhz/npui
def populate_related_list(parents, id_key, res_key, reltype, subq,
                          flt=None, relid_key='id'):
    ids = []
    for p in parents:
        if callable(flt) and not flt(p):
            continue
        keyval = getattr(p, id_key, None)
        if keyval and keyval not in ids:
            ids.append(keyval)
    if len(ids) > 0:
        ch = dict((k, list(v)) for k, v in groupby(
            subq.filter(getattr(reltype, relid_key).in_(ids)),
            lambda c: getattr(c, relid_key)))
    for p in parents:
        if callable(flt) and not flt(p):
            continue
        keyval = getattr(p, id_key, None)
        if keyval:
            attributes.set_committed_value(p, res_key, ch.get(keyval, ()))
Example #28
0
 def listtracksbyalbums(self, ffilter=None, skip=None, limit=None):
     with self.session_scope() as session:
         query = session.query(Album).join(Album.tracks).outerjoin(
             TrackInfo.genre).options(
                 contains_eager(Album.tracks, TrackInfo.album,
                                Album.artist),
                 joinedload(Album.tracks, TrackInfo.genre))
         if ffilter is not None:
             query = ffilter(query)
         query = limitoffset(
             query.order_by(Album.year.desc(), TrackInfo.trackno), skip,
             limit)
         qall = query.all()
         # Force artist population
         for x in qall:
             for y in x.tracks:
                 attributes.set_committed_value(y, 'album', x)
                 attributes.set_committed_value(y, 'artist', x.artist)
         session.expunge_all()
         return qall
Example #29
0
 def populate_backrefs(self, related_entities):
     """
     Populates backrefs for given related entities.
     """
     backref_dict = dict(
         (local_values(self.prop, value[0]), [])
         for value in related_entities
     )
     for value in related_entities:
         backref_dict[local_values(self.prop, value[0])].append(
             self.path.session.query(self.path.entities[0].__class__).get(
                 tuple(value[1:])
             )
         )
     for value in related_entities:
         set_committed_value(
             value[0],
             self.prop.back_populates,
             backref_dict[local_values(self.prop, value[0])]
         )
Example #30
0
def populate_related(parents,
                     id_key,
                     res_key,
                     reltype,
                     subq,
                     flt=None,
                     relid_key='id'):
    ids = []
    for p in parents:
        if callable(flt) and not flt(p):
            continue
        keyval = getattr(p, id_key, None)
        if keyval and (keyval not in ids):
            ids.append(keyval)
    if len(ids) > 0:
        for rel in subq.filter(getattr(reltype, relid_key).in_(ids)):
            for p in parents:
                if callable(flt) and not flt(p):
                    continue
                keyval = getattr(p, id_key, None)
                if keyval and (keyval == getattr(rel, relid_key)):
                    attributes.set_committed_value(p, res_key, rel)
Example #31
0
    def get_unique(cls, session, fqdn=None, name=None, dns_domain=None,
                   dns_environment=None, compel=False, preclude=False, **kwargs):
        # Proxy FQDN lookup to the Fqdn class
        if not fqdn or not isinstance(fqdn, Fqdn):
            if not isinstance(dns_environment, DnsEnvironment):
                dns_environment = DnsEnvironment.get_unique_or_default(session,
                                                                       dns_environment)
            if fqdn:
                if name or dns_domain:  # pragma: no cover
                    raise TypeError("fqdn and name/dns_domain cannot be mixed")
                (name, dns_domain) = parse_fqdn(session, fqdn)
            try:
                # Do not pass preclude=True to Fqdn
                fqdn = Fqdn.get_unique(session, name=name,
                                       dns_domain=dns_domain,
                                       dns_environment=dns_environment,
                                       compel=compel)
            except NotFoundException:
                # Replace the "Fqdn ... not found" message with a more user
                # friendly one
                msg = "%s %s.%s, %s not found." % (cls._get_class_label(),
                                                   name, dns_domain,
                                                   format(dns_environment, "l"))
                raise NotFoundException(msg)
            if not fqdn:
                return None

        # We already have the FQDN, no need to load it again
        if "query_options" not in kwargs:
            kwargs["query_options"] = [lazyload("fqdn")]

        result = super(DnsRecord, cls).get_unique(session, fqdn=fqdn,
                                                  compel=compel,
                                                  preclude=preclude, **kwargs)
        if result:
            # Make sure not to load the relation again if we already know its
            # value
            set_committed_value(result, 'fqdn', fqdn)
        return result
Example #32
0
    def preload_interfaces(self, session, hosts, interfaces_by_id,
                           interfaces_by_hwent):
        addrs_by_iface = defaultdict(list)
        slaves_by_id = defaultdict(list)

        # Polymorphic loading cannot be applied to eager-loaded
        # attributes, so load interfaces manually.
        q = session.query(Interface)
        q = q.with_polymorphic('*')
        q = q.options(lazyload("hardware_entity"))
        for iface in q:
            interfaces_by_hwent[iface.hardware_entity_id].append(iface)
            interfaces_by_id[iface.id] = iface
            if iface.master_id:
                slaves_by_id[iface.master_id].append(iface)

        # subqueryload() and with_polymorphic() do not play nice
        # together, so do it by hand
        q = session.query(AddressAssignment)
        q = q.options(joinedload("network"),
                      joinedload("dns_records"))
        q = q.order_by(AddressAssignment._label)

        # Machine templates want the management interface only
        if hosts:
            q = q.options(subqueryload("network.static_routes"),
                          subqueryload("network.routers"))
        else:
            q = q.join(AddressAssignment.interface.of_type(ManagementInterface))

        for addr in q:
            addrs_by_iface[addr.interface_id].append(addr)

        for iface_id, iface in interfaces_by_id.iteritems():
            set_committed_value(iface, "assignments",
                                addrs_by_iface.get(iface_id, None))
            set_committed_value(iface, "slaves",
                                slaves_by_id.get(iface_id, None))
    def test_only_loads_relations_on_unpopulated_models(self):
        User = self.classes.User
        Address = self.classes.Address
        session = fixture_session()

        users = session.query(User).order_by(self.tables.users.c.id.asc()).all()
        address = session.query(Address).filter(self.tables.addresses.c.id == 1).first()
        # pre-load the address for the first user
        attributes.set_committed_value(users[0], "addresses", [address])
        self.queries = []

        # make sure no relations are loaded
        for user in users[1:]:
            model_dict = attributes.instance_dict(user)
            assert "addresses" not in model_dict

        # trigger a lazy load
        users[1].addresses

        # only 1 query should have been generated to load all the child relationships
        assert len(self.queries) == 1
        unpopulated_user_ids = [user.id for user in users[1:]]
        assert self.queries[0]["parameters"] == tuple(unpopulated_user_ids)
Example #34
0
	def domain_hosts(self, domain):
		from netprofile_hosts.models import Host
		from netprofile_ipaddresses.models import (
			IPv4Address,
			IPv6Address
		)

		# XXX: this is a hack, but it reduces poor ol' MySQL's crunch time by a factor of 10
		sess = DBSession()

		hipv4 = dict((k, list(v)) for k, v in groupby(
			sess.query(IPv4Address).join(IPv4Address.host).filter(Host.domain_id == domain.id),
			lambda ip: ip.host_id
		))
		hipv6 = dict((k, list(v)) for k, v in groupby(
			sess.query(IPv6Address).join(IPv6Address.host).filter(Host.domain_id == domain.id),
			lambda ip: ip.host_id
		))

		for host in domain.hosts:
			attributes.set_committed_value(host, 'ipv4_addresses', hipv4.get(host.id, ()))
			attributes.set_committed_value(host, 'ipv6_addresses', hipv6.get(host.id, ()))
		return domain.hosts
Example #35
0
def populate_related_list(parents,
                          id_key,
                          res_key,
                          reltype,
                          subq,
                          flt=None,
                          relid_key='id'):
    ids = []
    for p in parents:
        if callable(flt) and not flt(p):
            continue
        keyval = getattr(p, id_key, None)
        if keyval and (keyval not in ids):
            ids.append(keyval)
    if len(ids) > 0:
        ch = dict((k, list(v)) for k, v in groupby(
            subq.filter(getattr(reltype, relid_key).in_(ids)),
            lambda c: getattr(c, relid_key)))
    for p in parents:
        if callable(flt) and not flt(p):
            continue
        keyval = getattr(p, id_key, None)
        if keyval:
            attributes.set_committed_value(p, res_key, ch.get(keyval, ()))
Example #36
0
 def listtracksbyalbumsbyartists(self,
                                 ffilter=None,
                                 skip=None,
                                 limit=None,
                                 order_by=(Artist.name, Album.year.desc(),
                                           TrackInfo.trackno,
                                           TrackInfo.name)):
     with self.session_scope() as session:
         query = session.query(Artist).join(Artist.albums).join(
             Album.tracks).join(TrackInfo.track).options(
                 contains_eager(Artist.albums, Album.tracks))
         if ffilter is not None:
             query = ffilter(query)
         query = limitoffset(query.order_by(*order_by), skip, limit)
         qall = query.all()
         # Force artist population
         for x in qall:
             for y in x.albums:
                 attributes.set_committed_value(y, 'artist', x)
                 for z in y.tracks:
                     attributes.set_committed_value(z, 'album', y)
                     attributes.set_committed_value(z, 'artist', x)
         session.expunge_all()
         return qall
Example #37
0
    def __init__(self, dbhost, *args, **kwargs):
        """Provide initialization specific for host bindings."""
        if not isinstance(dbhost, Host):
            raise InternalError("HostChooser can only choose services for "
                                "hosts, got %r (%s)" % (dbhost, type(dbhost)))
        super(HostChooser, self).__init__(dbhost, *args, **kwargs)
        self.location = dbhost.hardware_entity.location

        # If the primary name is a ReservedName, then it does not have a network
        # attribute
        if hasattr(dbhost.hardware_entity.primary_name, 'network'):
            self.network = dbhost.hardware_entity.primary_name.network
        else:
            self.network = None

        # all of them would be self. but that should be optimized
        # dbhost.hardware_entity.interfaces[x].assignments[y].network

        # Stores interim service instance lists.
        q = self.session.query(Service)
        q = q.outerjoin(Service.archetypes)
        q = q.reset_joinpoint()
        q = q.outerjoin(Service.personalities)
        q = q.filter(or_(Archetype.id == self.archetype.id,
                         Personality.id == self.personality.id))
        self.required_services = set(q.all())

        self.original_service_instances = {}
        # Cache of any already bound services (keys) and the instance
        # that was bound (values).
        q = self.session.query(ServiceInstance)
        q = q.options(undefer('_client_count'))
        q = q.filter(ServiceInstance.clients.contains(dbhost))
        set_committed_value(dbhost, 'services_used', q.all())
        for si in dbhost.services_used:
            self.original_service_instances[si.service] = si
            self.logger.debug("{0} original binding: {1}"
                              .format(self.dbobj, si))
        self.cluster_aligned_services = {}
        if dbhost.cluster:
            # Note that cluster services are currently ignored unless
            # they are otherwise required by the archetype/personality.
            for si in dbhost.cluster.service_bindings:
                self.cluster_aligned_services[si.service] = si
            for service in dbhost.cluster.required_services:
                if service not in self.cluster_aligned_services:
                    # Don't just error here because the error() call
                    # has not yet been set up.  Will error out later.
                    self.cluster_aligned_services[service] = None
                # Went back and forth on this... deciding not to force
                # an aligned service as required.  This should give
                # flexibility for multiple services to be aligned for
                # a cluster type without being forced on all the
                # personalities.
                #self.required_services.add(item.service)

            if dbhost.cluster.metacluster:
                mc = dbhost.cluster.metacluster
                for si in mc.service_bindings:
                    if si.service in self.cluster_aligned_services:
                        cas = self.cluster_aligned_services[si.service]
                        if cas == None:
                            # Error out later.
                            continue

                        self.logger.client_info(
                            "Replacing {0.name} instance with {1.name} "
                            "(bound to {2:l}) for service {3.name}".format(
                            cas, si, mc, si.service))

                    self.cluster_aligned_services[si.service] = si
                for service in mc.required_services:
                    if service not in self.cluster_aligned_services:
                        # Don't just error here because the error() call
                        # has not yet been set up.  Will error out later.
                        self.cluster_aligned_services[service] = None
Example #38
0
 def _set_updated_revision_number(self, revision_number, updated_at):
     attributes.set_committed_value(self, "revision_number",
                                    revision_number)
     attributes.set_committed_value(self, "updated_at", updated_at)
Example #39
0
    def render(self, session, logger, hostname, **arguments):
        # Check dependencies, translate into user-friendly message
        dbhost = hostname_to_host(session, hostname)

        dbhost.lock_row()

        check_no_provided_service(dbhost)

        # Any service bindings that we need to clean up afterwards
        plenaries = PlenaryCollection(logger=logger)
        remove_plenaries = PlenaryCollection(logger=logger)
        remove_plenaries.append(Plenary.get_plenary(dbhost))

        archetype = dbhost.archetype.name
        dbmachine = dbhost.hardware_entity
        oldinfo = DSDBRunner.snapshot_hw(dbmachine)

        ip = dbmachine.primary_ip

        for si in dbhost.services_used:
            plenaries.append(PlenaryServiceInstanceServer.get_plenary(si))
            logger.info("Before deleting {0:l}, removing binding to {1:l}"
                        .format(dbhost, si))

        del dbhost.services_used[:]

        if dbhost.resholder:
            for res in dbhost.resholder.resources:
                remove_plenaries.append(Plenary.get_plenary(res))

        # In case of Zebra, the IP may be configured on multiple interfaces
        for iface in dbmachine.interfaces:
            if ip in iface.addresses:
                iface.addresses.remove(ip)

        if dbhost.cluster:
            dbcluster = dbhost.cluster
            dbcluster.hosts.remove(dbhost)
            set_committed_value(dbhost, '_cluster', None)
            dbcluster.validate()
            plenaries.append(Plenary.get_plenary(dbcluster))

        dbdns_rec = dbmachine.primary_name
        dbmachine.primary_name = None
        dbmachine.host = None
        session.delete(dbhost)
        delete_dns_record(dbdns_rec)
        session.flush()

        if dbmachine.vm_container:
            plenaries.append(Plenary.get_plenary(dbmachine.vm_container))

        with CompileKey.merge([plenaries.get_key(),
                               remove_plenaries.get_key()]):
            plenaries.stash()
            remove_plenaries.stash()

            try:
                plenaries.write(locked=True)
                remove_plenaries.remove(locked=True, remove_profile=True)

                if archetype != 'aurora' and ip is not None:
                    dsdb_runner = DSDBRunner(logger=logger)
                    dsdb_runner.update_host(dbmachine, oldinfo)
                    dsdb_runner.commit_or_rollback("Could not remove host %s from "
                                                   "DSDB" % hostname)
                if archetype == 'aurora':
                    logger.client_info("WARNING: removing host %s from AQDB and "
                                       "*not* changing DSDB." % hostname)
            except:
                plenaries.restore_stash()
                remove_plenaries.restore_stash()
                raise

        trigger_notifications(self.config, logger, CLIENT_INFO)

        return
Example #40
0
                # TODO: only if not hosts
                manager_addrs = defaultdict(list)
                q = session.query(AddressAssignment)
                q = q.join(Interface)
                q = q.filter_by(interface_type="management")
                q = q.options(contains_eager("interface"),
                              joinedload("dns_records"),
                              lazyload("interface.hardware_entity"))
                for addr in q:
                    manager_addrs[addr.interface.id].append(addr)
                for interface_id, addrs in manager_addrs.items():
                    if interface_id not in interfaces_by_id:
                        # Should not happen...
                        continue
                    addrs.sort(key=attrgetter("label"))
                    set_committed_value(interfaces_by_id[interface_id],
                                        "assignments", addrs)

                q = session.query(Machine)
                q = q.options(lazyload("host"), lazyload("primary_name"),
                              subqueryload("chassis_slot"))

                cnt = q.count()
                idx = 0
                for machine in q:
                    idx += 1
                    if idx % 1000 == 0:  # pragma: no cover
                        logger.client_info("Processing machine %d of %d..." %
                                           (idx, cnt))

                    if machine.id in disks_by_machine:
                        disks_by_machine[machine.id].sort(
Example #41
0
 def _populate_backrefs(target, context):
     for name, backref in mappings.iteritems():
         # __dict__ to avoid triggering lazy-loaded relationships
         if target.__dict__.get(name) is not None:
             set_committed_value(getattr(target, name), backref, target)
Example #42
0
    def add_net_msg(self, net_msg, net):
        net_msg.name = str(net.name)
        net_msg.id = net.id
        net_msg.ip = str(net.ip)
        net_msg.cidr = net.cidr
        net_msg.bcast = str(net.broadcast)
        net_msg.netmask = str(net.netmask)
        net_msg.side = str(net.side)
        net_msg.sysloc = str(net.location.sysloc())
        net_msg.location.name = str(net.location.name)
        net_msg.location.location_type = str(net.location.location_type)
        net_msg.type = str(net.network_type)

        # Bulk load information about anything having a network address on this
        # network
        hw_ids = set(
            [addr.interface.hardware_entity_id for addr in net.assignments])
        if hw_ids:
            session = object_session(net)
            q = session.query(HardwareEntity)
            q = q.filter(HardwareEntity.id.in_(hw_ids))
            q = q.options(subqueryload('interfaces'))
            hwent_by_id = {}
            for dbhwent in q.all():
                hwent_by_id[dbhwent.id] = dbhwent

                iface_by_id = {}
                slaves_by_id = defaultdict(list)

                # We have all the interfaces loaded already, so compute the
                # master/slave relationships to avoid having to touch the
                # database again
                for iface in dbhwent.interfaces:
                    iface_by_id[iface.id] = iface
                    if iface.master_id:
                        slaves_by_id[iface.master_id].append(iface)

                for iface in dbhwent.interfaces:
                    set_committed_value(iface, "master",
                                        iface_by_id.get(iface.master_id, None))
                    set_committed_value(iface, "slaves",
                                        slaves_by_id[iface.id])

            # TODO: once we refactor Host to be an FK to HardwareEntity instead
            # of Machine, this could be converted to a single joinedload('host')
            q = session.query(Host)
            q = q.options(lazyload('machine'))
            q = q.filter(Host.machine_id.in_(hw_ids))
            for host in q.all():
                set_committed_value(hwent_by_id[host.machine_id], "host", host)
                set_committed_value(host, "machine",
                                    hwent_by_id[host.machine_id])

        # Add interfaces that have addresses in this network
        for addr in net.assignments:
            if not addr.dns_records:
                # hostname is a required field in the protobuf description
                continue

            hwent = addr.interface.hardware_entity

            # DHCP: we do not care about secondary IP addresses, but in some
            # cases the same IP address may show up with different MACs
            if not addr.label:
                mac_addrs = possible_mac_addresses(addr.interface)
            else:
                mac_addrs = []

            # Generate a host record even if there is no known MAC address for
            # it
            if not mac_addrs:
                mac_addrs.append(None)

            # Associating the same IP with multiple MAC addresses is
            # problematic using the current protocol. Sending multiple host
            # messages is easy for the broker, but it can confuse consumers like
            # aqdhcpd. For now just ensure it never happens, and revisit the
            # problem when we have a real world requirement.
            if len(mac_addrs) > 1:
                mac_addrs = [mac_addrs[0]]

            for mac in mac_addrs:
                host_msg = net_msg.hosts.add()

                if addr.interface.interface_type == 'management':
                    host_msg.type = 'manager'
                else:
                    if hwent.hardware_type == 'machine':
                        host_msg.type = 'host'
                        if hwent.host:
                            host_msg.archetype.name = str(
                                hwent.host.archetype.name)
                    elif hwent.hardware_type == 'switch':
                        # aqdhcpd uses the type
                        host_msg.type = 'tor_switch'
                    else:
                        host_msg.type = hwent.hardware_type

                host_msg.hostname = str(addr.dns_records[0].fqdn.name)
                host_msg.fqdn = str(addr.dns_records[0].fqdn)
                host_msg.dns_domain = str(addr.dns_records[0].fqdn.dns_domain)

                host_msg.ip = str(addr.ip)

                if mac:
                    host_msg.mac = mac

                host_msg.machine.name = str(hwent.label)

                # aqdhcpd uses the interface list when excluding hosts it is not
                # authoritative for
                for iface in hwent.interfaces:
                    int_msg = host_msg.machine.interfaces.add()
                    int_msg.device = iface.name
                    if iface.mac:
                        int_msg.mac = str(iface.mac)

        # Add dynamic DHCP records
        for dynhost in net.dynamic_stubs:
            host_msg = net_msg.hosts.add()
            # aqdhcpd uses the type
            host_msg.type = 'dynamic_stub'
            host_msg.hostname = str(dynhost.fqdn.name)
            host_msg.fqdn = str(dynhost.fqdn)
            host_msg.dns_domain = str(dynhost.fqdn.dns_domain)
            host_msg.ip = str(dynhost.ip)
Example #43
0
                # TODO: only if not hosts
                manager_addrs = defaultdict(list)
                q = session.query(AddressAssignment)
                q = q.join(Interface)
                q = q.filter_by(interface_type="management")
                q = q.options(contains_eager("interface"),
                              joinedload("dns_records"),
                              lazyload("interface.hardware_entity"))
                for addr in q:
                    manager_addrs[addr.interface.id].append(addr)
                for interface_id, addrs in manager_addrs.items():
                    if interface_id not in interfaces_by_id:
                        # Should not happen...
                        continue
                    addrs.sort(key=attrgetter("label"))
                    set_committed_value(interfaces_by_id[interface_id],
                                        "assignments", addrs)

                q = session.query(Machine)
                q = q.options(lazyload("host"),
                              lazyload("primary_name"),
                              subqueryload("chassis_slot"))

                cnt = q.count()
                idx = 0
                for machine in q:
                    idx += 1
                    if idx % 1000 == 0:  # pragma: no cover
                        logger.client_info("Processing machine %d of %d..." %
                                           (idx, cnt))

                    if machine.id in disks_by_machine:
Example #44
0
    def get_unique(cls, sess, name, hardware_type=None, compel=False,
                   preclude=False, query_options=None):
        """ Returns a unique HardwareEntity given session and fqdn """

        # If the hardware_type param isn't explicitly set and we have a
        # polymorphic identity, assume we're querying only for items of our
        # hardware_type.
        if hardware_type:
            if isclass(hardware_type):
                clslabel = hardware_type._get_class_label()
                hardware_type = hardware_type.__mapper_args__['polymorphic_identity']
            else:
                pcls = cls.__mapper__.polymorphic_map[hardware_type].class_
                clslabel = pcls._get_class_label()
        else:
            if 'polymorphic_identity' in cls.__mapper_args__:
                hardware_type = cls.__mapper_args__['polymorphic_identity']
            clslabel = cls._get_class_label()

        # The automagic DNS lookup does not really make sense with preclude=True
        if preclude:
            name = AqStr.normalize(name)
            cls.check_label(name)

        q = sess.query(cls)
        if "." in name:
            dns_rec = DnsRecord.get_unique(sess, fqdn=name, compel=True)
            # We know the primary name, do not load it again
            q = q.options(lazyload('primary_name'))
            q = q.filter_by(primary_name=dns_rec)
        else:
            dns_rec = None
            q = q.filter_by(label=name)
        if query_options:
            q = q.options(*query_options)

        try:
            hwe = q.one()
        except NoResultFound:
            # Check if the name is in use by a different hardware type
            q = sess.query(HardwareEntity)
            if dns_rec:
                # We know the primary name, do not load it again
                q = q.options(lazyload('primary_name'))
                q = q.filter_by(primary_name=dns_rec)
            else:
                q = q.filter_by(label=name)
            try:
                hwe = q.one()
                if dns_rec:
                    # We know the primary name, do not load it again
                    set_committed_value(hwe, 'primary_name', dns_rec)
                raise ArgumentError("{0} exists, but is not a {1}."
                                    .format(hwe, clslabel.lower()))
            except NoResultFound:
                hwe = None

            if compel:
                raise NotFoundException("%s %s not found." % (clslabel, name))

        if hwe:
            if preclude:
                raise ArgumentError('{0} already exists.'.format(hwe))
            if dns_rec:
                # We know the primary name, do not load it again
                set_committed_value(hwe, 'primary_name', dns_rec)

        return hwe
Example #45
0
                chassis = q.all()  # pylint: disable=W0612

                q = session.query(Machine)
                q = q.options(lazyload("host"),
                              lazyload("primary_name"),
                              subqueryload("chassis_slot"))

                cnt = q.count()
                idx = 0
                for machine in q:
                    idx += 1
                    if idx % 1000 == 0:  # pragma: no cover
                        logger.client_info("Processing machine %d of %d..." %
                                           (idx, cnt))

                    set_committed_value(machine, 'disks',
                                        disks_by_machine.get(machine.id, None))
                    set_committed_value(machine, 'interfaces',
                                        interfaces_by_hwent.get(machine.id, None))

                    try:
                        plenary_info = Plenary.get_plenary(machine,
                                                           logger=logger)
                        written += plenary_info.write(locked=True)
                    except Exception, e:
                        failed.append("{0} failed: {1}".format(machine, e))
                        continue

            if hosts:
                logger.client_info("Flushing hosts.")

                q = session.query(Cluster)
Example #46
0
 def _set_updated_revision_number(self, revision_number, updated_at):
     attributes.set_committed_value(
         self, "revision_number", revision_number)
     attributes.set_committed_value(
         self, "updated_at", updated_at)
Example #47
0
    def __init__(self, dbobj, *args, **kwargs):
        """Provide initialization specific for host bindings."""
        if not isinstance(dbobj, Host):
            raise InternalError("HostChooser can only choose services for "
                                "hosts, got %r (%s)" % (dbobj, type(dbobj)))
        self.dbhost = dbobj
        Chooser.__init__(self, dbobj, *args, **kwargs)
        self.location = self.dbhost.machine.location
        self.archetype = self.dbhost.archetype
        self.personality = self.dbhost.personality

        # If the primary name is a ReservedName, then it does not have a network
        # attribute
        if hasattr(self.dbhost.machine.primary_name, 'network'):
            self.network = self.dbhost.machine.primary_name.network
        else:
            self.network = None

        # all of them would be self. but that should be optimized
        # dbhost.machine.interfaces[x].assignments[y].network
        """Stores interim service instance lists."""
        q = self.session.query(Service)
        q = q.outerjoin(Service.archetypes)
        q = q.reset_joinpoint()
        q = q.outerjoin(Service.personalities)
        q = q.filter(
            or_(Archetype.id == self.archetype.id,
                Personality.id == self.personality.id))
        self.required_services = set(q.all())

        self.original_service_instances = {}
        """Cache of any already bound services (keys) and the instance
        that was bound (values).
        """
        q = self.session.query(ServiceInstance)
        q = q.options(undefer('_client_count'))
        q = q.filter(ServiceInstance.clients.contains(self.dbhost))
        set_committed_value(self.dbhost, 'services_used', q.all())
        for si in self.dbhost.services_used:
            self.original_service_instances[si.service] = si
            self.logger.debug("%s original binding: %s", self.description,
                              si.cfg_path)
        self.cluster_aligned_services = {}
        if self.dbhost.cluster:
            # Note that cluster services are currently ignored unless
            # they are otherwise required by the archetype/personality.
            for si in self.dbhost.cluster.service_bindings:
                self.cluster_aligned_services[si.service] = si
            for service in self.dbhost.cluster.required_services:
                if service not in self.cluster_aligned_services:
                    # Don't just error here because the error() call
                    # has not yet been set up.  Will error out later.
                    self.cluster_aligned_services[service] = None
                # Went back and forth on this... deciding not to force
                # an aligned service as required.  This should give
                # flexibility for multiple services to be aligned for
                # a cluster type without being forced on all the
                # personalities.
                #self.required_services.add(item.service)

            if self.dbhost.cluster.metacluster:
                mc = self.dbhost.cluster.metacluster
                for si in mc.service_bindings:
                    if si.service in self.cluster_aligned_services:
                        cas = self.cluster_aligned_services[si.service]
                        if cas == None:
                            # Error out later.
                            continue

                        self.logger.client_info(
                            "Replacing {0.name} instance with {1.name} "
                            "(bound to {2:l}) for service {3.name}".format(
                                cas, si, mc, si.service))

                    self.cluster_aligned_services[si.service] = si
                for service in mc.required_services:
                    if service not in self.cluster_aligned_services:
                        # Don't just error here because the error() call
                        # has not yet been set up.  Will error out later.
                        self.cluster_aligned_services[service] = None
Example #48
0
    def render(self, session, logger,
               services, personalities, machines, clusters, hosts,
               locations, resources, switches, all,
               **arguments):
        success = []
        failed = []
        written = 0

        # Caches for keeping preloaded data pinned in memory, since the SQLA
        # session holds a weak reference only
        resource_by_id = {}
        resholder_by_id = {}
        service_instances = None
        racks = None

        # Object caches that are accessed directly
        disks_by_machine = defaultdict(list)
        interfaces_by_machine = defaultdict(list)
        interfaces_by_id = {}

        if all:
            services = True
            personalities = True
            machines = True
            clusters = True
            hosts = True
            locations = True
            resources = True

        with CompileKey(logger=logger):
            logger.client_info("Loading data.")

            # When flushing clusters/hosts, loading the resource holder is done
            # as the query that loads those objects. But when flushing resources
            # only, we need the holder and the object it belongs to.
            if resources and not clusters:
                q = session.query(ClusterResource)
                # Using joinedload('cluster') would generate an outer join
                q = q.join(Cluster)
                q = q.options(contains_eager('cluster'))
                for resholder in q:
                    resholder_by_id[resholder.id] = resholder
            if resources and not hosts:
                q = session.query(HostResource)
                # Using joinedload('host') would generate an outer join
                q = q.join(Host)
                q = q.options(contains_eager('host'))
                for resholder in q:
                    resholder_by_id[resholder.id] = resholder

            if hosts or clusters or resources:
                # Load the most common resource types. Using
                # with_polymorphic('*') on Resource would generate a huge query,
                # so do something more targeted. More resource subclasses may be
                # added later if they become common.
                preload_classes = {
                    Filesystem: [],
                    RebootSchedule: [],
                    VirtualMachine: [joinedload('machine'),
                                     joinedload('machine.primary_name'),
                                     joinedload('machine.primary_name.fqdn')],
                    Share: [],
                }

                share_info = cache_storage_data()

                for cls, options in  preload_classes.items():
                    q = session.query(cls)

                    # If only hosts or only clusters are needed, don't load
                    # resources of the other kind
                    if hosts and not clusters and not resources:
                        q = q.join(ResourceHolder)
                        q = q.options(contains_eager('holder'))
                        q = q.filter_by(holder_type='host')
                    if clusters and not hosts and not resources:
                        q = q.join(ResourceHolder)
                        q = q.filter_by(holder_type='cluster')
                        q = q.options(contains_eager('holder'))

                    if options:
                        q = q.options(*options)

                    for res in q:
                        resource_by_id[res.id] = res
                        try:
                            res.populate_share_info(share_info)
                        except AttributeError:
                            pass

            if hosts or machines:
                # Polymorphic loading cannot be applied to eager-loaded
                # attributes, so load interfaces manually.
                q = session.query(Interface)
                q = q.with_polymorphic('*')
                q = q.options(lazyload("hardware_entity"))
                for iface in q:
                    interfaces_by_machine[iface.hardware_entity_id].append(iface)
                    interfaces_by_id[iface.id] = iface

                if hosts:
                    # subqueryload() and with_polymorphic() do not play nice
                    # together, so do it by hand
                    q = session.query(AddressAssignment)
                    q = q.options(joinedload("network"),
                                  joinedload("dns_records"))
                    q = q.order_by(AddressAssignment._label)
                    addrs_by_iface = defaultdict(list)
                    for addr in q:
                        addrs_by_iface[addr.interface_id].append(addr)
                    for interface_id, addrs in addrs_by_iface.items():
                        set_committed_value(interfaces_by_id[interface_id],
                                            "assignments", addrs)

                    q = session.query(Interface.id)
                    q = q.filter(~Interface.assignments.any())
                    for id in q.all():
                        set_committed_value(interfaces_by_id[id[0]],
                                            "assignments", None)

            if hosts or services:
                q = session.query(ServiceInstance)
                q = q.options(subqueryload("service"))
                service_instances = q.all()

            if machines or clusters:
                # Most machines are in racks...
                q = session.query(Rack)
                q = q.options(subqueryload("dns_maps"),
                              subqueryload("parents"))
                racks = q.all()

            if locations:
                logger.client_info("Flushing locations.")
                for dbloc in session.query(City).all():
                    try:
                        plenary = Plenary.get_plenary(dbloc, logger=logger)
                        written += plenary.write(locked=True)
                    except Exception, e:
                        failed.append("City %s failed: %s" %
                                      dbloc, e)
                        continue

            if services:
                logger.client_info("Flushing services.")
                q = session.query(Service)
                q = q.options(subqueryload("instances"))
                for dbservice in q:
                    try:
                        plenary_info = Plenary.get_plenary(dbservice,
                                                           logger=logger)
                        written += plenary_info.write(locked=True)
                    except Exception, e:
                        failed.append("Service %s failed: %s" %
                                      (dbservice.name, e))
                        continue

                    for dbinst in dbservice.instances:
                        try:
                            plenary_info = Plenary.get_plenary(dbinst,
                                                               logger=logger)
                            written += plenary_info.write(locked=True)
                        except Exception, e:
                            failed.append("Service %s instance %s failed: %s" %
                                          (dbservice.name, dbinst.name, e))
                            continue
Example #49
0
    def render(self, session, logger, hostname, **arguments):
        # removing the plenary host requires a compile lock, however
        # we want to avoid deadlock by the fact that we're messing
        # with two locks here, so we want to be careful. We grab the
        # plenaryhost early on (in order to get the filenames filled
        # in from the db info before we delete it from the db. We then
        # hold onto those references until we've completed the db
        # cleanup and if all of that is successful, then we delete the
        # plenary file (which doesn't require re-evaluating any stale
        # db information) after we've released the delhost lock.
        delplenary = False

        # Any service bindings that we need to clean up afterwards
        bindings = PlenaryCollection(logger=logger)
        resources = PlenaryCollection(logger=logger)
        with DeleteKey("system", logger=logger) as key:
            # Check dependencies, translate into user-friendly message
            dbhost = hostname_to_host(session, hostname)
            host_plenary = Plenary.get_plenary(dbhost, logger=logger)
            domain = dbhost.branch.name
            deps = get_host_dependencies(session, dbhost)
            if (len(deps) != 0):
                deptext = "\n".join(["  %s" % d for d in deps])
                raise ArgumentError("Cannot delete host %s due to the "
                                    "following dependencies:\n%s." %
                                    (hostname, deptext))

            archetype = dbhost.archetype.name
            dbmachine = dbhost.machine
            oldinfo = DSDBRunner.snapshot_hw(dbmachine)

            ip = dbmachine.primary_ip
            fqdn = dbmachine.fqdn

            for si in dbhost.services_used:
                plenary = PlenaryServiceInstanceServer(si)
                bindings.append(plenary)
                logger.info(
                    "Before deleting host '%s', removing binding '%s'" %
                    (fqdn, si.cfg_path))

            del dbhost.services_used[:]

            if dbhost.resholder:
                for res in dbhost.resholder.resources:
                    resources.append(Plenary.get_plenary(res))

            # In case of Zebra, the IP may be configured on multiple interfaces
            for iface in dbmachine.interfaces:
                if ip in iface.addresses:
                    iface.addresses.remove(ip)

            if dbhost.cluster:
                dbcluster = dbhost.cluster
                dbcluster.hosts.remove(dbhost)
                set_committed_value(dbhost, '_cluster', None)
                dbcluster.validate()

            dbdns_rec = dbmachine.primary_name
            dbmachine.primary_name = None
            dbmachine.host = None
            session.delete(dbhost)
            delete_dns_record(dbdns_rec)
            session.flush()
            delplenary = True

            if dbmachine.vm_container:
                bindings.append(Plenary.get_plenary(dbmachine.vm_container))

            if archetype != 'aurora' and ip is not None:
                dsdb_runner = DSDBRunner(logger=logger)
                dsdb_runner.update_host(dbmachine, oldinfo)
                dsdb_runner.commit_or_rollback("Could not remove host %s from "
                                               "DSDB" % hostname)
            if archetype == 'aurora':
                logger.client_info("WARNING: removing host %s from AQDB and "
                                   "*not* changing DSDB." % hostname)

            # Past the point of no return... commit the transaction so
            # that we can free the delete lock.
            session.commit()

        # Only if we got here with no exceptions do we clean the template
        # Trying to clean up after any errors here is really difficult
        # since the changes to dsdb have already been made.
        if (delplenary):
            key = host_plenary.get_remove_key()
            with CompileKey.merge(
                [key,
                 bindings.get_write_key(),
                 resources.get_remove_key()]) as key:
                host_plenary.cleanup(domain, locked=True)
                # And we also want to remove the profile itself
                profiles = self.config.get("broker", "profilesdir")
                # Only one of these should exist, but it doesn't hurt
                # to try to clean up both.
                xmlfile = os.path.join(profiles, fqdn + ".xml")
                remove_file(xmlfile, logger=logger)
                xmlgzfile = xmlfile + ".gz"
                remove_file(xmlgzfile, logger=logger)
                # And the cached template created by ant
                remove_file(os.path.join(
                    self.config.get("broker", "quattordir"), "objects",
                    fqdn + TEMPLATE_EXTENSION),
                            logger=logger)
                bindings.write(locked=True)
                resources.remove(locked=True)

            build_index(self.config, session, profiles, logger=logger)

        return
Example #50
0
    def add_net_data(self, net_msg, net):
        net_msg.name = str(net.name)
        net_msg.id = net.id
        net_msg.ip = str(net.ip)
        net_msg.cidr = net.cidr
        net_msg.bcast = str(net.broadcast)
        net_msg.netmask = str(net.netmask)
        net_msg.side = str(net.side)
        net_msg.sysloc = str(net.location.sysloc())
        net_msg.location.name = str(net.location.name)
        net_msg.location.location_type = str(net.location.location_type)
        net_msg.type = str(net.network_type)

        # Bulk load information about anything having a network address on this
        # network
        hw_ids = set([addr.interface.hardware_entity_id for addr in
                      net.assignments])
        if hw_ids:
            session = object_session(net)
            q = session.query(HardwareEntity)
            q = q.filter(HardwareEntity.id.in_(hw_ids))
            q = q.options(subqueryload('interfaces'),
                          lazyload('interfaces.hardware_entity'))
            hwent_by_id = {}
            for dbhwent in q.all():
                hwent_by_id[dbhwent.id] = dbhwent

                iface_by_id = {}
                slaves_by_id = defaultdict(list)

                # We have all the interfaces loaded already, so compute the
                # master/slave relationships to avoid having to touch the
                # database again
                for iface in dbhwent.interfaces:
                    iface_by_id[iface.id] = iface
                    if iface.master_id:
                        slaves_by_id[iface.master_id].append(iface)

                for iface in dbhwent.interfaces:
                    set_committed_value(iface, "master",
                                        iface_by_id.get(iface.master_id, None))
                    set_committed_value(iface, "slaves", slaves_by_id[iface.id])

            # TODO: once we refactor Host to be an FK to HardwareEntity instead
            # of Machine, this could be converted to a single joinedload('host')
            q = session.query(Host)
            q = q.options(lazyload('hardware_entity'))
            q = q.filter(Host.hardware_entity_id.in_(hw_ids))
            for host in q.all():
                set_committed_value(hwent_by_id[host.hardware_entity_id], "host", host)
                set_committed_value(host, "hardware_entity",
                                    hwent_by_id[host.hardware_entity_id])

        # Add interfaces that have addresses in this network
        for addr in net.assignments:
            if not addr.dns_records:
                # hostname is a required field in the protobuf description
                continue

            hwent = addr.interface.hardware_entity

            # DHCP: we do not care about secondary IP addresses, but in some
            # cases the same IP address may show up with different MACs
            if not addr.label:
                mac_addrs = possible_mac_addresses(addr.interface)
            else:
                mac_addrs = []

            # Generate a host record even if there is no known MAC address for
            # it
            if not mac_addrs:
                mac_addrs.append(None)

            # Associating the same IP with multiple MAC addresses is
            # problematic using the current protocol. Sending multiple host
            # messages is easy for the broker, but it can confuse consumers like
            # aqdhcpd. For now just ensure it never happens, and revisit the
            # problem when we have a real world requirement.
            if len(mac_addrs) > 1:
                mac_addrs = [mac_addrs[0]]

            for mac in mac_addrs:
                host_msg = net_msg.hosts.add()

                if addr.interface.interface_type == 'management':
                    host_msg.type = 'manager'
                else:
                    if hwent.hardware_type == 'machine':
                        host_msg.type = 'host'
                        if hwent.host:
                            host_msg.archetype.name = str(hwent.host.archetype.name)
                    elif hwent.hardware_type == 'switch':
                        # aqdhcpd uses the type
                        host_msg.type = 'tor_switch'
                    else:
                        host_msg.type = hwent.hardware_type

                host_msg.hostname = str(addr.dns_records[0].fqdn.name)
                host_msg.fqdn = str(addr.dns_records[0].fqdn)
                host_msg.dns_domain = str(addr.dns_records[0].fqdn.dns_domain)

                host_msg.ip = str(addr.ip)

                if mac:
                    host_msg.mac = mac

                host_msg.machine.name = str(hwent.label)

                # aqdhcpd uses the interface list when excluding hosts it is not
                # authoritative for
                for iface in hwent.interfaces:
                    int_msg = host_msg.machine.interfaces.add()
                    int_msg.device = iface.name
                    if iface.mac:
                        int_msg.mac = str(iface.mac)

        # Add dynamic DHCP records
        for dynhost in net.dynamic_stubs:
            host_msg = net_msg.hosts.add()
            # aqdhcpd uses the type
            host_msg.type = 'dynamic_stub'
            host_msg.hostname = str(dynhost.fqdn.name)
            host_msg.fqdn = str(dynhost.fqdn)
            host_msg.dns_domain = str(dynhost.fqdn.dns_domain)
            host_msg.ip = str(dynhost.ip)
    def create_transaction(self, session):
        """
        Create transaction object for given SQLAlchemy session.

        :param session: SQLAlchemy session object
        """
        args = self.transaction_args(session)

        Transaction = self.manager.transaction_cls
        table = Transaction.__table__
        self.current_transaction = Transaction()

        if self.manager.options['native_versioning']:
            has_transaction_initialized = bool(session.execute(
                '''SELECT 1 FROM pg_catalog.pg_class
                WHERE relname = 'temporary_transaction'
                '''
            ).scalar())
            if has_transaction_initialized:
                tx_id = (
                    session.execute('SELECT id FROM temporary_transaction')
                    .scalar()
                )
                set_committed_value(self.current_transaction, 'id', tx_id)
            else:
                criteria = {'native_tx_id': sa.func.txid_current()}
                args.update(criteria)

                query = (
                    table.insert()
                    .values(**args)
                    .returning(*map(sa.text, list(args.keys()) + ['id']))
                )

                values = session.execute(query).fetchone()
                for key, value in values.items():
                    set_committed_value(self.current_transaction, key, value)

                session.execute(
                    '''
                    CREATE TEMP TABLE temporary_transaction
                    (id BIGINT, PRIMARY KEY(id))
                    ON COMMIT DROP
                    '''
                )
                session.execute('''
                    INSERT INTO temporary_transaction (id)
                    VALUES (:id)
                    ''',
                    {'id': self.current_transaction.id}
                )
            self.merge_transaction(session, self.current_transaction)
        else:
            for key, value in args.items():
                setattr(self.current_transaction, key, value)
            if not self.version_session:
                self.version_session = sa.orm.session.Session(
                    bind=session.connection()
                )
            self.version_session.add(self.current_transaction)
            self.version_session.flush()
            self.version_session.expunge(self.current_transaction)
            session.add(self.current_transaction)
        return self.current_transaction
Example #52
0
    def render(self, session, logger, services, personalities, machines,
               clusters, hosts, locations, resources, switches, all,
               **arguments):
        success = []
        failed = []
        written = 0

        # Caches for keeping preloaded data pinned in memory, since the SQLA
        # session holds a weak reference only
        resource_by_id = {}
        resholder_by_id = {}
        service_instances = None
        racks = None

        # Object caches that are accessed directly
        disks_by_machine = defaultdict(list)
        interfaces_by_machine = defaultdict(list)
        interfaces_by_id = {}

        if all:
            services = True
            personalities = True
            machines = True
            clusters = True
            hosts = True
            locations = True
            resources = True

        with CompileKey(logger=logger):
            logger.client_info("Loading data.")

            # When flushing clusters/hosts, loading the resource holder is done
            # as the query that loads those objects. But when flushing resources
            # only, we need the holder and the object it belongs to.
            if resources and not clusters:
                q = session.query(ClusterResource)
                # Using joinedload('cluster') would generate an outer join
                q = q.join(Cluster)
                q = q.options(contains_eager('cluster'))
                for resholder in q:
                    resholder_by_id[resholder.id] = resholder
            if resources and not hosts:
                q = session.query(HostResource)
                # Using joinedload('host') would generate an outer join
                q = q.join(Host)
                q = q.options(contains_eager('host'))
                for resholder in q:
                    resholder_by_id[resholder.id] = resholder

            if hosts or clusters or resources:
                # Load the most common resource types. Using
                # with_polymorphic('*') on Resource would generate a huge query,
                # so do something more targeted. More resource subclasses may be
                # added later if they become common.
                preload_classes = {
                    Filesystem: [],
                    RebootSchedule: [],
                    VirtualMachine: [
                        joinedload('machine'),
                        joinedload('machine.primary_name'),
                        joinedload('machine.primary_name.fqdn')
                    ],
                    Share: [],
                }

                share_info = cache_storage_data()

                for cls, options in preload_classes.items():
                    q = session.query(cls)

                    # If only hosts or only clusters are needed, don't load
                    # resources of the other kind
                    if hosts and not clusters and not resources:
                        q = q.join(ResourceHolder)
                        q = q.options(contains_eager('holder'))
                        q = q.filter_by(holder_type='host')
                    if clusters and not hosts and not resources:
                        q = q.join(ResourceHolder)
                        q = q.filter_by(holder_type='cluster')
                        q = q.options(contains_eager('holder'))

                    if options:
                        q = q.options(*options)

                    for res in q:
                        resource_by_id[res.id] = res
                        try:
                            res.populate_share_info(share_info)
                        except AttributeError:
                            pass

            if hosts or machines:
                # Polymorphic loading cannot be applied to eager-loaded
                # attributes, so load interfaces manually.
                q = session.query(Interface)
                q = q.with_polymorphic('*')
                q = q.options(lazyload("hardware_entity"))
                for iface in q:
                    interfaces_by_machine[iface.hardware_entity_id].append(
                        iface)
                    interfaces_by_id[iface.id] = iface

                if hosts:
                    # subqueryload() and with_polymorphic() do not play nice
                    # together, so do it by hand
                    q = session.query(AddressAssignment)
                    q = q.options(joinedload("network"),
                                  joinedload("dns_records"))
                    q = q.order_by(AddressAssignment._label)
                    addrs_by_iface = defaultdict(list)
                    for addr in q:
                        addrs_by_iface[addr.interface_id].append(addr)
                    for interface_id, addrs in addrs_by_iface.items():
                        set_committed_value(interfaces_by_id[interface_id],
                                            "assignments", addrs)

                    q = session.query(Interface.id)
                    q = q.filter(~Interface.assignments.any())
                    for id in q.all():
                        set_committed_value(interfaces_by_id[id[0]],
                                            "assignments", None)

            if hosts or services:
                q = session.query(ServiceInstance)
                q = q.options(subqueryload("service"))
                service_instances = q.all()

            if machines or clusters:
                # Most machines are in racks...
                q = session.query(Rack)
                q = q.options(subqueryload("dns_maps"),
                              subqueryload("parents"))
                racks = q.all()

            if locations:
                logger.client_info("Flushing locations.")
                for dbloc in session.query(City).all():
                    try:
                        plenary = Plenary.get_plenary(dbloc, logger=logger)
                        written += plenary.write(locked=True)
                    except Exception, e:
                        failed.append("City %s failed: %s" % dbloc, e)
                        continue

            if services:
                logger.client_info("Flushing services.")
                q = session.query(Service)
                q = q.options(subqueryload("instances"))
                for dbservice in q:
                    try:
                        plenary_info = Plenary.get_plenary(dbservice,
                                                           logger=logger)
                        written += plenary_info.write(locked=True)
                    except Exception, e:
                        failed.append("Service %s failed: %s" %
                                      (dbservice.name, e))
                        continue

                    for dbinst in dbservice.instances:
                        try:
                            plenary_info = Plenary.get_plenary(dbinst,
                                                               logger=logger)
                            written += plenary_info.write(locked=True)
                        except Exception, e:
                            failed.append("Service %s instance %s failed: %s" %
                                          (dbservice.name, dbinst.name, e))
                            continue
Example #53
0
class Media(object):
    """
    Media metadata and a collection of related files.

    """
    meta = association_proxy('_meta', 'value', creator=MediaMeta)

    query = DBSession.query_property(MediaQuery)

    # TODO: replace '_thumb_dir' with something more generic, like 'name',
    #       so that its other uses throughout the code make more sense.
    _thumb_dir = 'media'

    def __init__(self):
        if self.author is None:
            self.author = Author()

    def __repr__(self):
        return '<Media: %s>' % self.slug

    def set_tags(self, tags):
        """Set the tags relations of this media, creating them as needed.

        :param tags: A list or comma separated string of tags to use.
        """
        if isinstance(tags, basestring):
            tags = extract_tags(tags)
        if isinstance(tags, list) and tags:
            tags = fetch_and_create_tags(tags)
        self.tags = tags or []

    def set_categories(self, cats):
        """Set the related categories of this media.

        :param cats: A list of category IDs to set.
        """
        if cats:
            cats = Category.query.filter(Category.id.in_(cats)).all()
        self.categories = cats or []

    def update_status(self):
        """Ensure the type (audio/video) and encoded flag are properly set.

        Call this after modifying any files belonging to this item.

        """
        self.type = self._update_type()
        self.encoded = self._update_encoding()

    def _update_type(self):
        """Update the type of this Media object.

        If there's a video file, mark this as a video type, else fallback
        to audio, if possible, or unknown (None)
        """
        if any(file.type == VIDEO for file in self.files):
            return VIDEO
        elif any(file.type == AUDIO for file in self.files):
            return AUDIO
        else:
            return None

    def _update_encoding(self):
        # Test to see if we can find a workable file/player combination
        # for the most common podcasting app w/ the POOREST format support
        if self.podcast_id \
        and not pick_podcast_media_file(self):
            return False
        # Test to see if we can find a workable file/player combination
        # for the browser w/ the BEST format support
        if not pick_any_media_file(self):
            return False
        return True

    @property
    def is_published(self):
        if self.id is None:
            return False
        return self.publishable and self.reviewed and self.encoded\
           and (self.publish_on is not None and self.publish_on <= datetime.now())\
           and (self.publish_until is None or self.publish_until >= datetime.now())

    def increment_views(self):
        """Increment the number of views in the database.

        We avoid concurrency issues by incrementing JUST the views and
        not allowing modified_on to be updated automatically.

        """
        if self.id is None:
            self.views += 1
            return self.views

        # Don't raise an exception should concurrency problems occur.
        # Views will not actually be incremented in this case, but thats
        # relatively unimportant compared to rendering the page for the user.
        # We may be able to remove this after we improve our triggers to not
        # issue an UPDATE on media_fulltext unless one of its columns are
        # actually changed. Even when just media.views is updated, all the
        # columns in the corresponding media_fulltext row are updated, and
        # media_fulltext's MyISAM engine must lock the whole table to do so.
        transaction = DBSession.begin_nested()
        try:
            DBSession.query(self.__class__)\
                .filter(self.__class__.id == self.id)\
                .update({self.__class__.views: self.__class__.views + 1})
            transaction.commit()
        except OperationalError, e:
            transaction.rollback()
            # (OperationalError) (1205, 'Lock wait timeout exceeded, try restarting the transaction')
            if not '1205' in e.message:
                raise

        # Increment the views by one for the rest of the request,
        # but don't allow the ORM to increment the views too.
        attributes.set_committed_value(self, 'views', self.views + 1)
        return self.views
Example #54
0
    def render(self, session, logger, hostname, **arguments):
        # removing the plenary host requires a compile lock, however
        # we want to avoid deadlock by the fact that we're messing
        # with two locks here, so we want to be careful. We grab the
        # plenaryhost early on (in order to get the filenames filled
        # in from the db info before we delete it from the db. We then
        # hold onto those references until we've completed the db
        # cleanup and if all of that is successful, then we delete the
        # plenary file (which doesn't require re-evaluating any stale
        # db information) after we've released the delhost lock.
        delplenary = False

        # Any service bindings that we need to clean up afterwards
        bindings = PlenaryCollection(logger=logger)
        resources = PlenaryCollection(logger=logger)
        with DeleteKey("system", logger=logger) as key:
            # Check dependencies, translate into user-friendly message
            dbhost = hostname_to_host(session, hostname)
            host_plenary = Plenary.get_plenary(dbhost, logger=logger)
            domain = dbhost.branch.name
            deps = get_host_dependencies(session, dbhost)
            if (len(deps) != 0):
                deptext = "\n".join(["  %s" % d for d in deps])
                raise ArgumentError("Cannot delete host %s due to the "
                                    "following dependencies:\n%s." %
                                    (hostname, deptext))

            archetype = dbhost.archetype.name
            dbmachine = dbhost.machine
            oldinfo = DSDBRunner.snapshot_hw(dbmachine)

            ip = dbmachine.primary_ip
            fqdn = dbmachine.fqdn

            for si in dbhost.services_used:
                plenary = PlenaryServiceInstanceServer(si)
                bindings.append(plenary)
                logger.info("Before deleting host '%s', removing binding '%s'"
                            % (fqdn, si.cfg_path))

            del dbhost.services_used[:]

            if dbhost.resholder:
                for res in dbhost.resholder.resources:
                    resources.append(Plenary.get_plenary(res))

            # In case of Zebra, the IP may be configured on multiple interfaces
            for iface in dbmachine.interfaces:
                if ip in iface.addresses:
                    iface.addresses.remove(ip)

            if dbhost.cluster:
                dbcluster = dbhost.cluster
                dbcluster.hosts.remove(dbhost)
                set_committed_value(dbhost, '_cluster', None)
                dbcluster.validate()

            dbdns_rec = dbmachine.primary_name
            dbmachine.primary_name = None
            dbmachine.host = None
            session.delete(dbhost)
            delete_dns_record(dbdns_rec)
            session.flush()
            delplenary = True

            if dbmachine.vm_container:
                bindings.append(Plenary.get_plenary(dbmachine.vm_container))

            if archetype != 'aurora' and ip is not None:
                dsdb_runner = DSDBRunner(logger=logger)
                dsdb_runner.update_host(dbmachine, oldinfo)
                dsdb_runner.commit_or_rollback("Could not remove host %s from "
                                               "DSDB" % hostname)
            if archetype == 'aurora':
                logger.client_info("WARNING: removing host %s from AQDB and "
                                   "*not* changing DSDB." % hostname)

            # Past the point of no return... commit the transaction so
            # that we can free the delete lock.
            session.commit()

        # Only if we got here with no exceptions do we clean the template
        # Trying to clean up after any errors here is really difficult
        # since the changes to dsdb have already been made.
        if (delplenary):
            key = host_plenary.get_remove_key()
            with CompileKey.merge([key, bindings.get_write_key(),
                                   resources.get_remove_key()]) as key:
                host_plenary.cleanup(domain, locked=True)
                # And we also want to remove the profile itself
                profiles = self.config.get("broker", "profilesdir")
                # Only one of these should exist, but it doesn't hurt
                # to try to clean up both.
                xmlfile = os.path.join(profiles, fqdn + ".xml")
                remove_file(xmlfile, logger=logger)
                xmlgzfile = xmlfile + ".gz"
                remove_file(xmlgzfile, logger=logger)
                # And the cached template created by ant
                remove_file(os.path.join(self.config.get("broker",
                                                         "quattordir"),
                                         "objects", fqdn + TEMPLATE_EXTENSION),
                            logger=logger)
                bindings.write(locked=True)
                resources.remove(locked=True)

            build_index(self.config, session, profiles, logger=logger)

        return
Example #55
0
 def _populate_backrefs(target, context):
     for name, backref in mappings.items():
         # __dict__ to avoid triggering lazy-loaded relationships
         if target.__dict__.get(name) is not None:
             set_committed_value(getattr(target, name), backref, target)
Example #56
0
    def get_unique(cls,
                   sess,
                   name,
                   hardware_type=None,
                   compel=False,
                   preclude=False,
                   query_options=None):
        """ Returns a unique HardwareEntity given session and fqdn """

        # If the hardware_type param isn't explicitly set and we have a
        # polymorphic identity, assume we're querying only for items of our
        # hardware_type.
        if hardware_type:
            if isclass(hardware_type):
                clslabel = hardware_type._get_class_label()
                hardware_type = hardware_type.__mapper_args__[
                    'polymorphic_identity']
            else:
                pcls = cls.__mapper__.polymorphic_map[hardware_type].class_
                clslabel = pcls._get_class_label()
        else:
            if 'polymorphic_identity' in cls.__mapper_args__:
                hardware_type = cls.__mapper_args__['polymorphic_identity']
            clslabel = cls._get_class_label()

        # The automagic DNS lookup does not really make sense with preclude=True
        if preclude:
            name = AqStr.normalize(name)
            cls.check_label(name)

        q = sess.query(cls)
        if "." in name:
            dns_rec = DnsRecord.get_unique(sess, fqdn=name, compel=True)
            # We know the primary name, do not load it again
            q = q.options(lazyload('primary_name'))
            q = q.filter_by(primary_name=dns_rec)
        else:
            dns_rec = None
            q = q.filter_by(label=name)
        if query_options:
            q = q.options(*query_options)

        try:
            hwe = q.one()
        except NoResultFound:
            # Check if the name is in use by a different hardware type
            q = sess.query(HardwareEntity)
            if dns_rec:
                # We know the primary name, do not load it again
                q = q.options(lazyload('primary_name'))
                q = q.filter_by(primary_name=dns_rec)
            else:
                q = q.filter_by(label=name)
            try:
                hwe = q.one()
                if dns_rec:
                    # We know the primary name, do not load it again
                    set_committed_value(hwe, 'primary_name', dns_rec)
                raise ArgumentError("{0} exists, but is not a {1}.".format(
                    hwe, hardware_type))
            except NoResultFound:
                hwe = None

            if compel:
                raise NotFoundException("%s %s not found." % (clslabel, name))

        if hwe:
            if preclude:
                raise ArgumentError('{0} already exists.'.format(hwe))
            if dns_rec:
                # We know the primary name, do not load it again
                set_committed_value(hwe, 'primary_name', dns_rec)

        return hwe