def get_cached_row(klass, row, index_start, max_depth=0, cur_depth=0, requested=None): """ Helper function that recursively returns an object with the specified related attributes already populated. """ if max_depth and requested is None and cur_depth > max_depth: # We've recursed deeply enough; stop now. return None restricted = requested is not None index_end = index_start + len(klass._meta.fields) fields = row[index_start:index_end] if not [x for x in fields if x is not None]: # If we only have a list of Nones, there was not related object. obj = None else: obj = klass(*fields) for f in klass._meta.fields: if not select_related_descend(f, restricted, requested): continue if restricted: next = requested[f.name] else: next = None cached_row = get_cached_row(f.rel.to, row, index_end, max_depth, cur_depth+1, next) if cached_row: rel_obj, index_end = cached_row if obj is not None: setattr(obj, f.get_cache_name(), rel_obj) return obj, index_end
def get_cached_row(klass, row, index_start, max_depth=0, cur_depth=0, requested=None, offset=0, only_load=None): """ Helper function that recursively returns an object with the specified related attributes already populated. """ if max_depth and requested is None and cur_depth > max_depth: # We've recursed deeply enough; stop now. return None restricted = requested is not None load_fields = only_load and only_load.get(klass) or None if load_fields: # Handle deferred fields. skip = set() init_list = [] pk_val = row[index_start + klass._meta.pk_index()] for field in klass._meta.fields: if field.name not in load_fields: skip.add(field.name) else: init_list.append(field.attname) field_count = len(init_list) fields = row[index_start:index_start + field_count] if fields == (None, ) * field_count: obj = None elif skip: klass = deferred_class_factory(klass, skip) obj = klass(**dict(zip(init_list, fields))) else: obj = klass(*fields) else: field_count = len(klass._meta.fields) fields = row[index_start:index_start + field_count] if fields == (None, ) * field_count: obj = None else: obj = klass(*fields) index_end = index_start + field_count + offset for f in klass._meta.fields: if not select_related_descend(f, restricted, requested): continue if restricted: next = requested[f.name] else: next = None cached_row = get_cached_row(f.rel.to, row, index_end, max_depth, cur_depth + 1, next) if cached_row: rel_obj, index_end = cached_row if obj is not None: setattr(obj, f.get_cache_name(), rel_obj) return obj, index_end
def get_cached_row(klass, row, index_start, max_depth=0, cur_depth=0, requested=None): """ Helper function that recursively returns an object with the specified related attributes already populated. """ if max_depth and requested is None and cur_depth > max_depth: # We've recursed deeply enough; stop now. return None restricted = requested is not None index_end = index_start + len(klass._meta.fields) fields = row[index_start:index_end] if not [x for x in fields if x is not None]: # If we only have a list of Nones, there was not related object. return None, index_end obj = klass(*fields) for f in klass._meta.fields: if not select_related_descend(f, restricted, requested): continue if restricted: next = requested[f.name] else: next = None cached_row = get_cached_row(f.rel.to, row, index_end, max_depth, cur_depth + 1, next) if cached_row: rel_obj, index_end = cached_row setattr(obj, f.get_cache_name(), rel_obj) return obj, index_end
def get_cached_row(klass, row, index_start, max_depth=0, cur_depth=0, requested=None, offset=0, only_load=None): """ Helper function that recursively returns an object with the specified related attributes already populated. """ if max_depth and requested is None and cur_depth > max_depth: # We've recursed deeply enough; stop now. return None restricted = requested is not None load_fields = only_load and only_load.get(klass) or None if load_fields: # Handle deferred fields. skip = set() init_list = [] pk_val = row[index_start + klass._meta.pk_index()] for field in klass._meta.fields: if field.name not in load_fields: skip.add(field.name) else: init_list.append(field.attname) field_count = len(init_list) fields = row[index_start : index_start + field_count] if fields == (None,) * field_count: obj = None elif skip: klass = deferred_class_factory(klass, skip) obj = klass(**dict(zip(init_list, fields))) else: obj = klass(*fields) else: field_count = len(klass._meta.fields) fields = row[index_start : index_start + field_count] if fields == (None,) * field_count: obj = None else: obj = klass(*fields) index_end = index_start + field_count + offset for f in klass._meta.fields: if not select_related_descend(f, restricted, requested): continue if restricted: next = requested[f.name] else: next = None cached_row = get_cached_row(f.rel.to, row, index_end, max_depth, cur_depth+1, next) if cached_row: rel_obj, index_end = cached_row if obj is not None: setattr(obj, f.get_cache_name(), rel_obj) return obj, index_end
def _get_select_related_from_attrs(self, model, attrs): opts = model._meta related = list() for attr_name, child_attrs in attrs.iteritems(): try: field = opts.get_field(attr_name) except FieldDoesNotExist: pass else: if select_related_descend(field, False, []): if child_attrs: subrelated = self._get_select_related_from_attrs(field.rel, child_attrs) if subrelated: for entry in subrelated: related.append("%s__%s" % (attr_name, entry)) else: related.append(attr_name) else: related.append(attr_name) return related
def fill_related_selections(self, opts=None, root_alias=None, cur_depth=1, used=None, requested=None, restricted=None, nullable=None, dupe_set=None, avoid_set=None): """ Fill in the information needed for a select_related query. The current depth is measured as the number of connections away from the root model (for example, cur_depth=1 means we are looking at models with direct connections to the root model). """ if not restricted and self.query.max_depth and cur_depth > self.query.max_depth: # We've recursed far enough; bail out. return if not opts: opts = self.query.get_meta() root_alias = self.query.get_initial_alias() self.query.related_select_cols = [] self.query.related_select_fields = [] if not used: used = set() if dupe_set is None: dupe_set = set() if avoid_set is None: avoid_set = set() orig_dupe_set = dupe_set # Setup for the case when only particular related fields should be # included in the related selection. if requested is None: if isinstance(self.query.select_related, dict): requested = self.query.select_related restricted = True else: restricted = False for f, model in opts.get_fields_with_model(): if not select_related_descend(f, restricted, requested): continue # The "avoid" set is aliases we want to avoid just for this # particular branch of the recursion. They aren't permanently # forbidden from reuse in the related selection tables (which is # what "used" specifies). avoid = avoid_set.copy() dupe_set = orig_dupe_set.copy() table = f.rel.to._meta.db_table promote = nullable or f.null if model: int_opts = opts alias = root_alias alias_chain = [] for int_model in opts.get_base_chain(model): # Proxy model have elements in base chain # with no parents, assign the new options # object and skip to the next base in that # case if not int_opts.parents[int_model]: int_opts = int_model._meta continue lhs_col = int_opts.parents[int_model].column dedupe = lhs_col in opts.duplicate_targets if dedupe: avoid.update(self.query.dupe_avoidance.get((id(opts), lhs_col), ())) dupe_set.add((opts, lhs_col)) int_opts = int_model._meta alias = self.query.join((alias, int_opts.db_table, lhs_col, int_opts.pk.column), exclusions=used, promote=promote) alias_chain.append(alias) for (dupe_opts, dupe_col) in dupe_set: self.query.update_dupe_avoidance(dupe_opts, dupe_col, alias) if self.query.alias_map[root_alias].join_type == self.query.LOUTER: self.query.promote_alias_chain(alias_chain, True) else: alias = root_alias dedupe = f.column in opts.duplicate_targets if dupe_set or dedupe: avoid.update(self.query.dupe_avoidance.get((id(opts), f.column), ())) if dedupe: dupe_set.add((opts, f.column)) alias = self.query.join((alias, table, f.column, f.rel.get_related_field().column), exclusions=used.union(avoid), promote=promote) used.add(alias) columns, aliases = self.get_default_columns(start_alias=alias, opts=f.rel.to._meta, as_pairs=True) self.query.related_select_cols.extend(columns) if self.query.alias_map[alias].join_type == self.query.LOUTER: self.query.promote_alias_chain(aliases, True) self.query.related_select_fields.extend(f.rel.to._meta.fields) if restricted: next = requested.get(f.name, {}) else: next = False new_nullable = f.null or promote for dupe_opts, dupe_col in dupe_set: self.query.update_dupe_avoidance(dupe_opts, dupe_col, alias) self.fill_related_selections(f.rel.to._meta, alias, cur_depth + 1, used, next, restricted, new_nullable, dupe_set, avoid) if restricted: related_fields = [ (o.field, o.model) for o in opts.get_all_related_objects() if o.field.unique ] for f, model in related_fields: if not select_related_descend(f, restricted, requested, reverse=True): continue # The "avoid" set is aliases we want to avoid just for this # particular branch of the recursion. They aren't permanently # forbidden from reuse in the related selection tables (which is # what "used" specifies). avoid = avoid_set.copy() dupe_set = orig_dupe_set.copy() table = model._meta.db_table int_opts = opts alias = root_alias alias_chain = [] chain = opts.get_base_chain(f.rel.to) if chain is not None: for int_model in chain: # Proxy model have elements in base chain # with no parents, assign the new options # object and skip to the next base in that # case if not int_opts.parents[int_model]: int_opts = int_model._meta continue lhs_col = int_opts.parents[int_model].column dedupe = lhs_col in opts.duplicate_targets if dedupe: avoid.update((self.query.dupe_avoidance.get(id(opts), lhs_col), ())) dupe_set.add((opts, lhs_col)) int_opts = int_model._meta alias = self.query.join( (alias, int_opts.db_table, lhs_col, int_opts.pk.column), exclusions=used, promote=True, reuse=used ) alias_chain.append(alias) for dupe_opts, dupe_col in dupe_set: self.query.update_dupe_avoidance(dupe_opts, dupe_col, alias) dedupe = f.column in opts.duplicate_targets if dupe_set or dedupe: avoid.update(self.query.dupe_avoidance.get((id(opts), f.column), ())) if dedupe: dupe_set.add((opts, f.column)) alias = self.query.join( (alias, table, f.rel.get_related_field().column, f.column), exclusions=used.union(avoid), promote=True ) used.add(alias) columns, aliases = self.get_default_columns(start_alias=alias, opts=model._meta, as_pairs=True, local_only=True) self.query.related_select_cols.extend(columns) self.query.related_select_fields.extend(model._meta.fields) next = requested.get(f.related_query_name(), {}) new_nullable = f.null or None self.fill_related_selections(model._meta, table, cur_depth+1, used, next, restricted, new_nullable)
def fill_related_selections(self, opts=None, root_alias=None, cur_depth=1, requested=None, restricted=None, nullable=None): """ Fill in the information needed for a select_related query. The current depth is measured as the number of connections away from the root model (for example, cur_depth=1 means we are looking at models with direct connections to the root model). """ if not restricted and self.query.max_depth and cur_depth > self.query.max_depth: # We've recursed far enough; bail out. return if not opts: opts = self.query.get_meta() root_alias = self.query.get_initial_alias() self.query.related_select_cols = [] only_load = self.query.get_loaded_field_names() # Setup for the case when only particular related fields should be # included in the related selection. if requested is None: if isinstance(self.query.select_related, dict): requested = self.query.select_related restricted = True else: restricted = False for f, model in opts.get_fields_with_model(): # The get_fields_with_model() returns None for fields that live # in the field's local model. So, for those fields we want to use # the f.model - that is the field's local model. field_model = model or f.model if not select_related_descend(f, restricted, requested, only_load.get(field_model)): continue table = f.rel.to._meta.db_table promote = nullable or f.null alias = self.query.join_parent_model(opts, model, root_alias, {}) alias = self.query.join((alias, table, f.column, f.rel.get_related_field().column), promote=promote, join_field=f) columns, aliases = self.get_default_columns(start_alias=alias, opts=f.rel.to._meta, as_pairs=True) self.query.related_select_cols.extend( SelectInfo(col, field) for col, field in zip(columns, f.rel.to._meta.fields)) if restricted: next = requested.get(f.name, {}) else: next = False new_nullable = f.null or promote self.fill_related_selections(f.rel.to._meta, alias, cur_depth + 1, next, restricted, new_nullable) if restricted: related_fields = [ (o.field, o.model) for o in opts.get_all_related_objects() if o.field.unique ] for f, model in related_fields: if not select_related_descend(f, restricted, requested, only_load.get(model), reverse=True): continue alias = self.query.join_parent_model(opts, f.rel.to, root_alias, {}) table = model._meta.db_table alias = self.query.join( (alias, table, f.rel.get_related_field().column, f.column), promote=True, join_field=f ) from_parent = (opts.model if issubclass(model, opts.model) else None) columns, aliases = self.get_default_columns(start_alias=alias, opts=model._meta, as_pairs=True, from_parent=from_parent) self.query.related_select_cols.extend( SelectInfo(col, field) for col, field in zip(columns, model._meta.fields)) next = requested.get(f.related_query_name(), {}) # Use True here because we are looking at the _reverse_ side of # the relation, which is always nullable. new_nullable = True self.fill_related_selections(model._meta, table, cur_depth+1, next, restricted, new_nullable)
def fill_related_selections(self, opts=None, root_alias=None, cur_depth=1, requested=None, restricted=None, nullable=None): """ Fill in the information needed for a select_related query. The current depth is measured as the number of connections away from the root model (for example, cur_depth=1 means we are looking at models with direct connections to the root model). """ if not restricted and self.query.max_depth and cur_depth > self.query.max_depth: # We've recursed far enough; bail out. return if not opts: opts = self.query.get_meta() root_alias = self.query.get_initial_alias() self.query.related_select_cols = [] only_load = self.query.get_loaded_field_names() # Setup for the case when only particular related fields should be # included in the related selection. if requested is None: if isinstance(self.query.select_related, dict): requested = self.query.select_related restricted = True else: restricted = False for f, model in opts.get_fields_with_model(): # The get_fields_with_model() returns None for fields that live # in the field's local model. So, for those fields we want to use # the f.model - that is the field's local model. field_model = model or f.model if not select_related_descend(f, restricted, requested, only_load.get(field_model)): continue table = f.rel.to._meta.db_table promote = nullable or f.null alias = self.query.join_parent_model(opts, model, root_alias, {}) join_cols = f.get_joining_columns() alias = self.query.join((alias, table, join_cols), outer_if_first=promote, join_field=f) columns, aliases = self.get_default_columns(start_alias=alias, opts=f.rel.to._meta, as_pairs=True) self.query.related_select_cols.extend( SelectInfo(col, field) for col, field in zip(columns, f.rel.to._meta.concrete_fields)) if restricted: next = requested.get(f.name, {}) else: next = False new_nullable = f.null or promote self.fill_related_selections(f.rel.to._meta, alias, cur_depth + 1, next, restricted, new_nullable) if restricted: related_fields = [(o.field, o.model) for o in opts.get_all_related_objects() if o.field.unique] for f, model in related_fields: if not select_related_descend(f, restricted, requested, only_load.get(model), reverse=True): continue alias = self.query.join_parent_model(opts, f.rel.to, root_alias, {}) table = model._meta.db_table alias = self.query.join( (alias, table, f.get_joining_columns(reverse_join=True)), outer_if_first=True, join_field=f) from_parent = (opts.model if issubclass(model, opts.model) else None) columns, aliases = self.get_default_columns( start_alias=alias, opts=model._meta, as_pairs=True, from_parent=from_parent) self.query.related_select_cols.extend( SelectInfo(col, field) for col, field in zip( columns, model._meta.concrete_fields)) next = requested.get(f.related_query_name(), {}) # Use True here because we are looking at the _reverse_ side of # the relation, which is always nullable. new_nullable = True self.fill_related_selections(model._meta, table, cur_depth + 1, next, restricted, new_nullable)
def get_related_selections(self, select, opts=None, root_alias=None, cur_depth=1, requested=None, restricted=None): """ Fill in the information needed for a select_related query. The current depth is measured as the number of connections away from the root model (for example, cur_depth=1 means we are looking at models with direct connections to the root model). """ def _get_field_choices(): direct_choices = (f.name for f in opts.fields if f.is_relation) reverse_choices = (f.field.related_query_name() for f in opts.related_objects if f.field.unique) return chain(direct_choices, reverse_choices) related_klass_infos = [] if not restricted and self.query.max_depth and cur_depth > self.query.max_depth: # We've recursed far enough; bail out. return related_klass_infos if not opts: opts = self.query.get_meta() root_alias = self.query.get_initial_alias() only_load = self.query.get_loaded_field_names() # Setup for the case when only particular related fields should be # included in the related selection. fields_found = set() if requested is None: if isinstance(self.query.select_related, dict): requested = self.query.select_related restricted = True else: restricted = False def get_related_klass_infos(klass_info, related_klass_infos): klass_info['related_klass_infos'] = related_klass_infos for f in opts.fields: field_model = f.model._meta.concrete_model fields_found.add(f.name) if restricted: next = requested.get(f.name, {}) if not f.is_relation: # If a non-related field is used like a relation, # or if a single non-relational field is given. if next or (cur_depth == 1 and f.name in requested): raise FieldError( "Non-relational field given in select_related: '%s'. " "Choices are: %s" % ( f.name, ", ".join(_get_field_choices()) or '(none)', )) else: next = False if not select_related_descend(f, restricted, requested, only_load.get(field_model)): continue klass_info = { 'model': f.remote_field.model, 'field': f, 'reverse': False, 'from_parent': False, } related_klass_infos.append(klass_info) select_fields = [] _, _, _, joins, _ = self.query.setup_joins([f.name], opts, root_alias) alias = joins[-1] columns = self.get_default_columns(start_alias=alias, opts=f.remote_field.model._meta) for col in columns: select_fields.append(len(select)) select.append((col, None)) klass_info['select_fields'] = select_fields next_klass_infos = self.get_related_selections( select, f.remote_field.model._meta, alias, cur_depth + 1, next, restricted) get_related_klass_infos(klass_info, next_klass_infos) if restricted: related_fields = [(o.field, o.related_model) for o in opts.related_objects if o.field.unique and not o.many_to_many] for f, model in related_fields: if not select_related_descend(f, restricted, requested, only_load.get(model), reverse=True): continue related_field_name = f.related_query_name() fields_found.add(related_field_name) _, _, _, joins, _ = self.query.setup_joins( [related_field_name], opts, root_alias) alias = joins[-1] from_parent = issubclass(model, opts.model) klass_info = { 'model': model, 'field': f, 'reverse': True, 'from_parent': from_parent, } related_klass_infos.append(klass_info) select_fields = [] columns = self.get_default_columns(start_alias=alias, opts=model._meta, from_parent=opts.model) for col in columns: select_fields.append(len(select)) select.append((col, None)) klass_info['select_fields'] = select_fields next = requested.get(f.related_query_name(), {}) next_klass_infos = self.get_related_selections( select, model._meta, alias, cur_depth + 1, next, restricted) get_related_klass_infos(klass_info, next_klass_infos) fields_not_found = set(requested.keys()).difference(fields_found) if fields_not_found: invalid_fields = ("'%s'" % s for s in fields_not_found) raise FieldError( 'Invalid field name(s) given in select_related: %s. ' 'Choices are: %s' % ( ', '.join(invalid_fields), ', '.join(_get_field_choices()) or '(none)', )) return related_klass_infos
def get_related_selections(self, select, opts=None, root_alias=None, cur_depth=1, requested=None, restricted=None): """ Fill in the information needed for a select_related query. The current depth is measured as the number of connections away from the root model (for example, cur_depth=1 means we are looking at models with direct connections to the root model). """ def _get_field_choices(): direct_choices = (f.name for f in opts.fields if f.is_relation) reverse_choices = ( f.field.related_query_name() for f in opts.related_objects if f.field.unique ) return chain(direct_choices, reverse_choices) related_klass_infos = [] if not restricted and self.query.max_depth and cur_depth > self.query.max_depth: # We've recursed far enough; bail out. return related_klass_infos if not opts: opts = self.query.get_meta() root_alias = self.query.get_initial_alias() only_load = self.query.get_loaded_field_names() # Setup for the case when only particular related fields should be # included in the related selection. fields_found = set() if requested is None: if isinstance(self.query.select_related, dict): requested = self.query.select_related restricted = True else: restricted = False def get_related_klass_infos(klass_info, related_klass_infos): klass_info['related_klass_infos'] = related_klass_infos for f in opts.fields: field_model = f.model._meta.concrete_model fields_found.add(f.name) if restricted: next = requested.get(f.name, {}) if not f.is_relation: # If a non-related field is used like a relation, # or if a single non-relational field is given. if next or (cur_depth == 1 and f.name in requested): raise FieldError( "Non-relational field given in select_related: '%s'. " "Choices are: %s" % ( f.name, ", ".join(_get_field_choices()) or '(none)', ) ) else: next = False if not select_related_descend(f, restricted, requested, only_load.get(field_model)): continue klass_info = { 'model': f.rel.to, 'field': f, 'reverse': False, 'from_parent': False, } related_klass_infos.append(klass_info) select_fields = [] _, _, _, joins, _ = self.query.setup_joins( [f.name], opts, root_alias) alias = joins[-1] columns = self.get_default_columns(start_alias=alias, opts=f.rel.to._meta) for col in columns: select_fields.append(len(select)) select.append((col, None)) klass_info['select_fields'] = select_fields next_klass_infos = self.get_related_selections( select, f.rel.to._meta, alias, cur_depth + 1, next, restricted) get_related_klass_infos(klass_info, next_klass_infos) if restricted: related_fields = [ (o.field, o.related_model) for o in opts.related_objects if o.field.unique and not o.many_to_many ] for f, model in related_fields: if not select_related_descend(f, restricted, requested, only_load.get(model), reverse=True): continue related_field_name = f.related_query_name() fields_found.add(related_field_name) _, _, _, joins, _ = self.query.setup_joins([related_field_name], opts, root_alias) alias = joins[-1] from_parent = issubclass(model, opts.model) klass_info = { 'model': model, 'field': f, 'reverse': True, 'from_parent': from_parent, } related_klass_infos.append(klass_info) select_fields = [] columns = self.get_default_columns( start_alias=alias, opts=model._meta, from_parent=opts.model) for col in columns: select_fields.append(len(select)) select.append((col, None)) klass_info['select_fields'] = select_fields next = requested.get(f.related_query_name(), {}) next_klass_infos = self.get_related_selections( select, model._meta, alias, cur_depth + 1, next, restricted) get_related_klass_infos(klass_info, next_klass_infos) fields_not_found = set(requested.keys()).difference(fields_found) if fields_not_found: invalid_fields = ("'%s'" % s for s in fields_not_found) raise FieldError( 'Invalid field name(s) given in select_related: %s. ' 'Choices are: %s' % ( ', '.join(invalid_fields), ', '.join(_get_field_choices()) or '(none)', ) ) return related_klass_infos
def fill_related_selections(self, opts=None, root_alias=None, cur_depth=1, used=None, requested=None, restricted=None, nullable=None, dupe_set=None, avoid_set=None): """ Fill in the information needed for a select_related query. The current depth is measured as the number of connections away from the root model (for example, cur_depth=1 means we are looking at models with direct connections to the root model). """ if not restricted and self.query.max_depth and cur_depth > self.query.max_depth: # We've recursed far enough; bail out. return if not opts: opts = self.query.get_meta() root_alias = self.query.get_initial_alias() self.query.related_select_cols = [] self.query.related_select_fields = [] if not used: used = set() if dupe_set is None: dupe_set = set() if avoid_set is None: avoid_set = set() orig_dupe_set = dupe_set only_load = self.query.get_loaded_field_names() # Setup for the case when only particular related fields should be # included in the related selection. if requested is None: if isinstance(self.query.select_related, dict): requested = self.query.select_related restricted = True else: restricted = False for f, model in opts.get_fields_with_model(): if not select_related_descend( f, restricted, requested, only_load.get(model or self.query.model)): continue # The "avoid" set is aliases we want to avoid just for this # particular branch of the recursion. They aren't permanently # forbidden from reuse in the related selection tables (which is # what "used" specifies). avoid = avoid_set.copy() dupe_set = orig_dupe_set.copy() table = f.rel.to._meta.db_table promote = nullable or f.null if model: int_opts = opts alias = root_alias alias_chain = [] for int_model in opts.get_base_chain(model): # Proxy model have elements in base chain # with no parents, assign the new options # object and skip to the next base in that # case if not int_opts.parents[int_model]: int_opts = int_model._meta continue lhs_col = int_opts.parents[int_model].column dedupe = lhs_col in opts.duplicate_targets if dedupe: avoid.update( self.query.dupe_avoidance.get((id(opts), lhs_col), ())) dupe_set.add((opts, lhs_col)) int_opts = int_model._meta alias = self.query.join((alias, int_opts.db_table, lhs_col, int_opts.pk.column), exclusions=used, promote=promote) alias_chain.append(alias) for (dupe_opts, dupe_col) in dupe_set: self.query.update_dupe_avoidance( dupe_opts, dupe_col, alias) if self.query.alias_map[ root_alias].join_type == self.query.LOUTER: self.query.promote_alias_chain(alias_chain, True) else: alias = root_alias dedupe = f.column in opts.duplicate_targets if dupe_set or dedupe: avoid.update( self.query.dupe_avoidance.get((id(opts), f.column), ())) if dedupe: dupe_set.add((opts, f.column)) alias = self.query.join( (alias, table, f.column, f.rel.get_related_field().column), exclusions=used.union(avoid), promote=promote) used.add(alias) columns, aliases = self.get_default_columns(start_alias=alias, opts=f.rel.to._meta, as_pairs=True) self.query.related_select_cols.extend(columns) if self.query.alias_map[alias].join_type == self.query.LOUTER: self.query.promote_alias_chain(aliases, True) self.query.related_select_fields.extend(f.rel.to._meta.fields) if restricted: next = requested.get(f.name, {}) else: next = False new_nullable = f.null or promote for dupe_opts, dupe_col in dupe_set: self.query.update_dupe_avoidance(dupe_opts, dupe_col, alias) self.fill_related_selections(f.rel.to._meta, alias, cur_depth + 1, used, next, restricted, new_nullable, dupe_set, avoid) if restricted: related_fields = [(o.field, o.model) for o in opts.get_all_related_objects() if o.field.unique] for f, model in related_fields: if not select_related_descend(f, restricted, requested, only_load.get(model), reverse=True): continue # The "avoid" set is aliases we want to avoid just for this # particular branch of the recursion. They aren't permanently # forbidden from reuse in the related selection tables (which is # what "used" specifies). avoid = avoid_set.copy() dupe_set = orig_dupe_set.copy() table = model._meta.db_table int_opts = opts alias = root_alias alias_chain = [] chain = opts.get_base_chain(f.rel.to) if chain is not None: for int_model in chain: # Proxy model have elements in base chain # with no parents, assign the new options # object and skip to the next base in that # case if not int_opts.parents[int_model]: int_opts = int_model._meta continue lhs_col = int_opts.parents[int_model].column dedupe = lhs_col in opts.duplicate_targets if dedupe: avoid.update((self.query.dupe_avoidance.get( id(opts), lhs_col), ())) dupe_set.add((opts, lhs_col)) int_opts = int_model._meta alias = self.query.join((alias, int_opts.db_table, lhs_col, int_opts.pk.column), exclusions=used, promote=True, reuse=used) alias_chain.append(alias) for dupe_opts, dupe_col in dupe_set: self.query.update_dupe_avoidance( dupe_opts, dupe_col, alias) dedupe = f.column in opts.duplicate_targets if dupe_set or dedupe: avoid.update( self.query.dupe_avoidance.get((id(opts), f.column), ())) if dedupe: dupe_set.add((opts, f.column)) alias = self.query.join( (alias, table, f.rel.get_related_field().column, f.column), exclusions=used.union(avoid), promote=True) used.add(alias) columns, aliases = self.get_default_columns(start_alias=alias, opts=model._meta, as_pairs=True, local_only=True) self.query.related_select_cols.extend(columns) self.query.related_select_fields.extend(model._meta.fields) next = requested.get(f.related_query_name(), {}) new_nullable = f.null or None self.fill_related_selections(model._meta, table, cur_depth + 1, used, next, restricted, new_nullable)
def get_cached_row(klass, row, index_start, using, max_depth=0, cur_depth=0, requested=None, offset=0, only_load=None, local_only=False): """ Helper function that recursively returns an object with the specified related attributes already populated. This method may be called recursively to populate deep select_related() clauses. Arguments: * klass - the class to retrieve (and instantiate) * row - the row of data returned by the database cursor * index_start - the index of the row at which data for this object is known to start * using - the database alias on which the query is being executed. * max_depth - the maximum depth to which a select_related() relationship should be explored. * cur_depth - the current depth in the select_related() tree. Used in recursive calls to determin if we should dig deeper. * requested - A dictionary describing the select_related() tree that is to be retrieved. keys are field names; values are dictionaries describing the keys on that related object that are themselves to be select_related(). * offset - the number of additional fields that are known to exist in `row` for `klass`. This usually means the number of annotated results on `klass`. * only_load - if the query has had only() or defer() applied, this is the list of field names that will be returned. If None, the full field list for `klass` can be assumed. * local_only - Only populate local fields. This is used when building following reverse select-related relations """ if max_depth and requested is None and cur_depth > max_depth: # We've recursed deeply enough; stop now. return None restricted = requested is not None if only_load: load_fields = only_load.get(klass) # When we create the object, we will also be creating populating # all the parent classes, so traverse the parent classes looking # for fields that must be included on load. for parent in klass._meta.get_parent_list(): fields = only_load.get(parent) if fields: load_fields.update(fields) else: load_fields = None if load_fields: # Handle deferred fields. skip = set() init_list = [] # Build the list of fields that *haven't* been requested for field, model in klass._meta.get_fields_with_model(): if field.name not in load_fields: skip.add(field.name) elif local_only and model is not None: continue else: init_list.append(field.attname) # Retrieve all the requested fields field_count = len(init_list) fields = row[index_start : index_start + field_count] # If all the select_related columns are None, then the related # object must be non-existent - set the relation to None. # Otherwise, construct the related object. if fields == (None,) * field_count: obj = None elif skip: klass = deferred_class_factory(klass, skip) obj = klass(**dict(zip(init_list, fields))) else: obj = klass(*fields) else: # Load all fields on klass if local_only: field_names = [f.attname for f in klass._meta.local_fields] else: field_names = [f.attname for f in klass._meta.fields] field_count = len(field_names) fields = row[index_start : index_start + field_count] # If all the select_related columns are None, then the related # object must be non-existent - set the relation to None. # Otherwise, construct the related object. if fields == (None,) * field_count: obj = None else: obj = klass(**dict(zip(field_names, fields))) # If an object was retrieved, set the database state. if obj: obj._state.db = using obj._state.adding = False index_end = index_start + field_count + offset # Iterate over each related object, populating any # select_related() fields for f in klass._meta.fields: if not select_related_descend(f, restricted, requested): continue if restricted: next = requested[f.name] else: next = None # Recursively retrieve the data for the related object cached_row = get_cached_row(f.rel.to, row, index_end, using, max_depth, cur_depth+1, next, only_load=only_load) # If the recursive descent found an object, populate the # descriptor caches relevant to the object if cached_row: rel_obj, index_end = cached_row if obj is not None: # If the base object exists, populate the # descriptor cache setattr(obj, f.get_cache_name(), rel_obj) if f.unique and rel_obj is not None: # If the field is unique, populate the # reverse descriptor cache on the related object setattr(rel_obj, f.related.get_cache_name(), obj) # Now do the same, but for reverse related objects. # Only handle the restricted case - i.e., don't do a depth # descent into reverse relations unless explicitly requested if restricted: related_fields = [ (o.field, o.model) for o in klass._meta.get_all_related_objects() if o.field.unique ] for f, model in related_fields: if not select_related_descend(f, restricted, requested, reverse=True): continue next = requested[f.related_query_name()] # Recursively retrieve the data for the related object cached_row = get_cached_row(model, row, index_end, using, max_depth, cur_depth+1, next, only_load=only_load, local_only=True) # If the recursive descent found an object, populate the # descriptor caches relevant to the object if cached_row: rel_obj, index_end = cached_row if obj is not None: # If the field is unique, populate the # reverse descriptor cache setattr(obj, f.related.get_cache_name(), rel_obj) if rel_obj is not None: # If the related object exists, populate # the descriptor cache. setattr(rel_obj, f.get_cache_name(), obj) # Now populate all the non-local field values # on the related object for rel_field,rel_model in rel_obj._meta.get_fields_with_model(): if rel_model is not None: setattr(rel_obj, rel_field.attname, getattr(obj, rel_field.attname)) # populate the field cache for any related object # that has already been retrieved if rel_field.rel: try: cached_obj = getattr(obj, rel_field.get_cache_name()) setattr(rel_obj, rel_field.get_cache_name(), cached_obj) except AttributeError: # Related object hasn't been cached yet pass return obj, index_end
def fill_related_selections(self, opts=None, root_alias=None, cur_depth=1, requested=None, restricted=None): """ Fill in the information needed for a select_related query. The current depth is measured as the number of connections away from the root model (for example, cur_depth=1 means we are looking at models with direct connections to the root model). """ if not restricted and self.query.max_depth and cur_depth > self.query.max_depth: # We've recursed far enough; bail out. return if not opts: opts = self.query.get_meta() root_alias = self.query.get_initial_alias() self.query.related_select_cols = [] only_load = self.query.get_loaded_field_names() # Setup for the case when only particular related fields should be # included in the related selection. if requested is None: if isinstance(self.query.select_related, dict): requested = self.query.select_related restricted = True else: restricted = False for f, model in opts.get_fields_with_model(): # The get_fields_with_model() returns None for fields that live # in the field's local model. So, for those fields we want to use # the f.model - that is the field's local model. field_model = model or f.model if not select_related_descend(f, restricted, requested, only_load.get(field_model)): continue _, _, _, joins, _ = self.query.setup_joins( [f.name], opts, root_alias) alias = joins[-1] columns, _ = self.get_default_columns(start_alias=alias, opts=f.rel.to._meta, as_pairs=True) self.query.related_select_cols.extend( SelectInfo((col[0], col[1].column), col[1]) for col in columns) if restricted: next = requested.get(f.name, {}) else: next = False self.fill_related_selections(f.rel.to._meta, alias, cur_depth + 1, next, restricted) if restricted: related_fields = [ (o.field, o.model) for o in opts.get_all_related_objects() if o.field.unique ] for f, model in related_fields: if not select_related_descend(f, restricted, requested, only_load.get(model), reverse=True): continue _, _, _, joins, _ = self.query.setup_joins( [f.related_query_name()], opts, root_alias) alias = joins[-1] from_parent = (opts.model if issubclass(model, opts.model) else None) columns, _ = self.get_default_columns(start_alias=alias, opts=model._meta, as_pairs=True, from_parent=from_parent) self.query.related_select_cols.extend( SelectInfo((col[0], col[1].column), col[1]) for col in columns) next = requested.get(f.related_query_name(), {}) self.fill_related_selections(model._meta, alias, cur_depth + 1, next, restricted)
def fill_related_selections(self, opts=None, root_alias=None, cur_depth=1, requested=None, restricted=None): """ Fill in the information needed for a select_related query. The current depth is measured as the number of connections away from the root model (for example, cur_depth=1 means we are looking at models with direct connections to the root model). """ if not restricted and self.query.max_depth and cur_depth > self.query.max_depth: # We've recursed far enough; bail out. return if not opts: opts = self.query.get_meta() root_alias = self.query.get_initial_alias() self.query.related_select_cols = [] only_load = self.query.get_loaded_field_names() # Setup for the case when only particular related fields should be # included in the related selection. if requested is None: if isinstance(self.query.select_related, dict): requested = self.query.select_related restricted = True else: restricted = False for f, model in opts.get_fields_with_model(): # The get_fields_with_model() returns None for fields that live # in the field's local model. So, for those fields we want to use # the f.model - that is the field's local model. field_model = model or f.model if not select_related_descend(f, restricted, requested, only_load.get(field_model)): continue _, _, _, joins, _ = self.query.setup_joins([f.name], opts, root_alias) alias = joins[-1] columns, _ = self.get_default_columns(start_alias=alias, opts=f.rel.to._meta, as_pairs=True) self.query.related_select_cols.extend( SelectInfo((col[0], col[1].column), col[1]) for col in columns) if restricted: next = requested.get(f.name, {}) else: next = False self.fill_related_selections(f.rel.to._meta, alias, cur_depth + 1, next, restricted) if restricted: related_fields = [(o.field, o.model) for o in opts.get_all_related_objects() if o.field.unique] for f, model in related_fields: if not select_related_descend(f, restricted, requested, only_load.get(model), reverse=True): continue _, _, _, joins, _ = self.query.setup_joins( [f.related_query_name()], opts, root_alias) alias = joins[-1] from_parent = (opts.model if issubclass(model, opts.model) else None) columns, _ = self.get_default_columns(start_alias=alias, opts=model._meta, as_pairs=True, from_parent=from_parent) self.query.related_select_cols.extend( SelectInfo((col[0], col[1].column), col[1]) for col in columns) next = requested.get(f.related_query_name(), {}) self.fill_related_selections(model._meta, alias, cur_depth + 1, next, restricted)
def get_related_selections(self, select, opts=None, root_alias=None, cur_depth=1, requested=None, restricted=None): def _get_field_choices(): direct_choices = (f.name for f in opts.fields if f.is_relation) reverse_choices = ( f.field.related_query_name() for f in opts.related_objects if f.field.unique ) return chain(direct_choices, reverse_choices) related_klass_infos = [] if not restricted and self.query.max_depth and cur_depth > self.query.max_depth: return related_klass_infos if not opts: opts = self.query.get_meta() root_alias = self.query.get_initial_alias() only_load = self.query.get_loaded_field_names() fields_found = set() if requested is None: if isinstance(self.query.select_related, dict): requested = self.query.select_related restricted = True else: restricted = False def get_related_klass_infos(klass_info, related_klass_infos): klass_info['related_klass_infos'] = related_klass_infos for f in opts.fields: field_model = f.model._meta.concrete_model fields_found.add(f.name) if restricted: next = requested.get(f.name, {}) if not f.is_relation: if next or f.name in requested: raise FieldError( "Non-relational field given in select_related: '%s'. " "Choices are: %s" % ( f.name, ", ".join(_get_field_choices()) or '(none)', ) ) else: next = False if not select_related_descend(f, restricted, requested, only_load.get(field_model)): continue klass_info = { 'model': f.remote_field.model, 'field': f, 'reverse': False, 'from_parent': False, } related_klass_infos.append(klass_info) select_fields = [] _, _, _, joins, _ = self.query.setup_joins( [f.name], opts, root_alias) alias = joins[-1] columns = self.get_default_columns(start_alias=alias, opts=f.remote_field.model._meta) for col in columns: select_fields.append(len(select)) select.append((col, None)) klass_info['select_fields'] = select_fields next_klass_infos = self.get_related_selections( select, f.remote_field.model._meta, alias, cur_depth + 1, next, restricted) get_related_klass_infos(klass_info, next_klass_infos) if restricted: related_fields = [ (o.field, o.related_model) for o in opts.related_objects if o.field.unique and not o.many_to_many ] for f, model in related_fields: if not select_related_descend(f, restricted, requested, only_load.get(model), reverse=True): continue related_field_name = f.related_query_name() fields_found.add(related_field_name) _, _, _, joins, _ = self.query.setup_joins([related_field_name], opts, root_alias) alias = joins[-1] from_parent = issubclass(model, opts.model) klass_info = { 'model': model, 'field': f, 'reverse': True, 'from_parent': from_parent, } related_klass_infos.append(klass_info) select_fields = [] columns = self.get_default_columns( start_alias=alias, opts=model._meta, from_parent=opts.model) for col in columns: select_fields.append(len(select)) select.append((col, None)) klass_info['select_fields'] = select_fields next = requested.get(f.related_query_name(), {}) next_klass_infos = self.get_related_selections( select, model._meta, alias, cur_depth + 1, next, restricted) get_related_klass_infos(klass_info, next_klass_infos) fields_not_found = set(requested.keys()).difference(fields_found) if fields_not_found: invalid_fields = ("'%s'" % s for s in fields_not_found) raise FieldError( 'Invalid field name(s) given in select_related: %s. ' 'Choices are: %s' % ( ', '.join(invalid_fields), ', '.join(_get_field_choices()) or '(none)', ) ) return related_klass_infos
def fill_related_selections(self, opts=None, root_alias=None, cur_depth=1, requested=None, restricted=None): """ Fill in the information needed for a select_related query. The current depth is measured as the number of connections away from the root model (for example, cur_depth=1 means we are looking at models with direct connections to the root model). """ def _get_field_choices(): direct_choices = (f.name for (f, _) in opts.get_fields_with_model() if f.rel) reverse_choices = ( f.field.related_query_name() for f in opts.get_all_related_objects() if f.field.unique ) return chain(direct_choices, reverse_choices) if not restricted and self.query.max_depth and cur_depth > self.query.max_depth: # We've recursed far enough; bail out. return if not opts: opts = self.query.get_meta() root_alias = self.query.get_initial_alias() self.query.related_select_cols = [] only_load = self.query.get_loaded_field_names() # Setup for the case when only particular related fields should be # included in the related selection. fields_found = set() if requested is None: if isinstance(self.query.select_related, dict): requested = self.query.select_related restricted = True else: restricted = False for f, model in opts.get_fields_with_model(): fields_found.add(f.name) if restricted: next = requested.get(f.name, {}) if not f.rel: # If a non-related field is used like a relation, # or if a single non-relational field is given. if next or (cur_depth == 1 and f.name in requested): raise FieldError( "Non-relational field given in select_related: '%s'. " "Choices are: %s" % ( f.name, ", ".join(_get_field_choices()) or '(none)', ) ) else: next = False # The get_fields_with_model() returns None for fields that live # in the field's local model. So, for those fields we want to use # the f.model - that is the field's local model. field_model = model or f.model if not select_related_descend(f, restricted, requested, only_load.get(field_model)): continue _, _, _, joins, _ = self.query.setup_joins( [f.name], opts, root_alias) alias = joins[-1] columns, _ = self.get_default_columns(start_alias=alias, opts=f.rel.to._meta, as_pairs=True) self.query.related_select_cols.extend( SelectInfo((col[0], col[1].column), col[1]) for col in columns ) self.fill_related_selections(f.rel.to._meta, alias, cur_depth + 1, next, restricted) if restricted: related_fields = [ (o.field, o.model) for o in opts.get_all_related_objects() if o.field.unique ] for f, model in related_fields: if not select_related_descend(f, restricted, requested, only_load.get(model), reverse=True): continue related_field_name = f.related_query_name() fields_found.add(related_field_name) _, _, _, joins, _ = self.query.setup_joins([related_field_name], opts, root_alias) alias = joins[-1] from_parent = (opts.model if issubclass(model, opts.model) else None) columns, _ = self.get_default_columns(start_alias=alias, opts=model._meta, as_pairs=True, from_parent=from_parent) self.query.related_select_cols.extend( SelectInfo((col[0], col[1].column), col[1]) for col in columns) next = requested.get(f.related_query_name(), {}) self.fill_related_selections(model._meta, alias, cur_depth + 1, next, restricted) fields_not_found = set(requested.keys()).difference(fields_found) if fields_not_found: invalid_fields = ("'%s'" % s for s in fields_not_found) raise FieldError( 'Invalid field name(s) given in select_related: %s. ' 'Choices are: %s' % ( ', '.join(invalid_fields), ', '.join(_get_field_choices()) or '(none)', ) )
def fill_related_selections(self, opts=None, root_alias=None, cur_depth=1, requested=None, restricted=None, nullable=None): """ Fill in the information needed for a select_related query. The current depth is measured as the number of connections away from the root model (for example, cur_depth=1 means we are looking at models with direct connections to the root model). """ if not restricted and self.query.max_depth and cur_depth > self.query.max_depth: # We've recursed far enough; bail out. return if not opts: opts = self.query.get_meta() root_alias = self.query.get_initial_alias() self.query.related_select_cols = [] only_load = self.query.get_loaded_field_names() # Setup for the case when only particular related fields should be # included in the related selection. if requested is None: if isinstance(self.query.select_related, dict): requested = self.query.select_related restricted = True else: restricted = False for f, model in opts.get_fields_with_model(): # The get_fields_with_model() returns None for fields that live # in the field's local model. So, for those fields we want to use # the f.model - that is the field's local model. field_model = model or f.model if not select_related_descend(f, restricted, requested, only_load.get(field_model)): continue table = f.rel.to._meta.db_table promote = nullable or f.null if model: int_opts = opts alias = root_alias alias_chain = [] for int_model in opts.get_base_chain(model): # Proxy model have elements in base chain # with no parents, assign the new options # object and skip to the next base in that # case if not int_opts.parents[int_model]: int_opts = int_model._meta continue lhs_col = int_opts.parents[int_model].column int_opts = int_model._meta alias = self.query.join((alias, int_opts.db_table, lhs_col, int_opts.pk.column), promote=promote) alias_chain.append(alias) else: alias = root_alias alias = self.query.join((alias, table, f.column, f.rel.get_related_field().column), promote=promote) columns, aliases = self.get_default_columns(start_alias=alias, opts=f.rel.to._meta, as_pairs=True) self.query.related_select_cols.extend( SelectInfo(col, field) for col, field in zip(columns, f.rel.to._meta.fields)) if restricted: next = requested.get(f.name, {}) else: next = False new_nullable = f.null or promote self.fill_related_selections(f.rel.to._meta, alias, cur_depth + 1, next, restricted, new_nullable) if restricted: related_fields = [ (o.field, o.model) for o in opts.get_all_related_objects() if o.field.unique ] for f, model in related_fields: if not select_related_descend(f, restricted, requested, only_load.get(model), reverse=True): continue table = model._meta.db_table int_opts = opts alias = root_alias alias_chain = [] chain = opts.get_base_chain(f.rel.to) if chain is not None: for int_model in chain: # Proxy model have elements in base chain # with no parents, assign the new options # object and skip to the next base in that # case if not int_opts.parents[int_model]: int_opts = int_model._meta continue lhs_col = int_opts.parents[int_model].column int_opts = int_model._meta alias = self.query.join( (alias, int_opts.db_table, lhs_col, int_opts.pk.column), promote=True, ) alias_chain.append(alias) alias = self.query.join( (alias, table, f.rel.get_related_field().column, f.column), promote=True ) columns, aliases = self.get_default_columns(start_alias=alias, opts=model._meta, as_pairs=True, local_only=True) self.query.related_select_cols.extend( SelectInfo(col, field) for col, field in zip(columns, model._meta.fields)) next = requested.get(f.related_query_name(), {}) # Use True here because we are looking at the _reverse_ side of # the relation, which is always nullable. new_nullable = True self.fill_related_selections(model._meta, table, cur_depth+1, next, restricted, new_nullable)