def more_like_this(self, model_instance, additional_query_string=None, start_offset=0, end_offset=None, models=None, limit_to_registered_models=None, result_class=None, **kwargs): from haystack import connections if not self.setup_complete: self.setup() # Handle deferred models. if get_proxied_model and hasattr(model_instance, '_deferred') and model_instance._deferred: model_klass = get_proxied_model(model_instance._meta) else: model_klass = type(model_instance) index = connections[self.connection_alias].get_unified_index().get_index(model_klass) field_name = index.get_content_field() params = {} if start_offset is not None: params['search_from'] = start_offset if end_offset is not None: params['search_size'] = end_offset - start_offset doc_id = get_identifier(model_instance) try: raw_results = self.conn.morelikethis(self.index_name, 'modelresult', doc_id, [field_name], **params) except (requests.RequestException, pyelasticsearch.ElasticSearchError), e: if not self.silently_fail: raise self.log.error("Failed to fetch More Like This from Elasticsearch for document '%s': %s", doc_id, e) raw_results = {}
def more_like_this( self, model_instance, additional_query_string=None, start_offset=0, end_offset=None, limit_to_registered_models=None, result_class=None, **kwargs ): from haystack import connections # Handle deferred models. if get_proxied_model and hasattr(model_instance, "_deferred") and model_instance._deferred: model_klass = get_proxied_model(model_instance._meta) else: model_klass = type(model_instance) index = connections[self.connection_alias].get_unified_index().get_index(model_klass) field_name = index.get_content_field() params = {"fl": "*,score"} if start_offset is not None: params["start"] = start_offset if end_offset is not None: params["rows"] = end_offset narrow_queries = set() if limit_to_registered_models is None: limit_to_registered_models = getattr(settings, "HAYSTACK_LIMIT_TO_REGISTERED_MODELS", True) if limit_to_registered_models: # Using narrow queries, limit the results to only models handled # with the current routers. if narrow_queries is None: narrow_queries = set() registered_models = self.build_models_list() if len(registered_models) > 0: narrow_queries.add("%s:(%s)" % (DJANGO_CT, " OR ".join(registered_models))) if additional_query_string: narrow_queries.add(additional_query_string) if narrow_queries: params["fq"] = list(narrow_queries) query = "%s:%s" % (ID, get_identifier(model_instance)) try: raw_results = self.conn.more_like_this(query, field_name, **params) except (IOError, SolrError), e: if not self.silently_fail: raise self.log.error("Failed to fetch More Like This from Solr for document '%s': %s", query, e) raw_results = EmptyResults()
def more_like_this(self, model_instance, additional_query_string=None, start_offset=0, end_offset=None, models=None, limit_to_registered_models=None, result_class=None, **kwargs): from haystack import connections # Handle deferred models. if get_proxied_model and hasattr(model_instance, '_deferred') and model_instance._deferred: model_klass = get_proxied_model(model_instance._meta) else: model_klass = type(model_instance) index = connections[self.connection_alias].get_unified_index().get_index(model_klass) field_name = index.get_content_field() params = { 'fl': '*,score', } if start_offset is not None: params['start'] = start_offset if end_offset is not None: params['rows'] = end_offset narrow_queries = set() if limit_to_registered_models is None: limit_to_registered_models = getattr(settings, 'HAYSTACK_LIMIT_TO_REGISTERED_MODELS', True) if models and len(models): model_choices = sorted(['%s.%s' % (model._meta.app_label, model._meta.module_name) for model in models]) elif limit_to_registered_models: # Using narrow queries, limit the results to only models handled # with the current routers. model_choices = self.build_models_list() else: model_choices = [] if len(model_choices) > 0: if narrow_queries is None: narrow_queries = set() narrow_queries.add('%s:(%s)' % (DJANGO_CT, ' OR '.join(model_choices))) if additional_query_string: narrow_queries.add(additional_query_string) if narrow_queries: params['fq'] = list(narrow_queries) query = "%s:%s" % (ID, get_identifier(model_instance)) try: raw_results = self.conn.more_like_this(query, field_name, **params) except (IOError, SolrError), e: if not self.silently_fail: raise self.log.error("Failed to fetch More Like This from Solr for document '%s': %s", query, e) raw_results = EmptyResults()
def more_like_this(self, model_instance, additional_query_string=None, start_offset=0, end_offset=None, limit_to_registered_models=None, result_class=None, fields=None, **kwargs): # Handle deferred models. if get_proxied_model and hasattr(model_instance, '_deferred') and model_instance._deferred: model_klass = get_proxied_model(model_instance._meta) else: model_klass = type(model_instance) index = self.site.get_index(model_klass) field_name = index.get_content_field() params = { 'fl': '*,score', } if fields: fields.add('score') params['fl'] = ','.join(fields) if start_offset is not None: params['start'] = start_offset if end_offset is not None: params['rows'] = end_offset narrow_queries = set() if limit_to_registered_models is None: limit_to_registered_models = getattr(settings, 'HAYSTACK_LIMIT_TO_REGISTERED_MODELS', True) if limit_to_registered_models: # Using narrow queries, limit the results to only models registered # with the current site. if narrow_queries is None: narrow_queries = set() registered_models = self.build_registered_models_list() if len(registered_models) > 0: narrow_queries.add('%s:(%s)' % (DJANGO_CT, ' OR '.join(registered_models))) if additional_query_string: narrow_queries.add(additional_query_string) if narrow_queries: params['fq'] = list(narrow_queries) query = "%s:%s" % (ID, get_identifier(model_instance)) try: raw_results = self.conn.more_like_this(query, field_name, **params) except (IOError, SolrError), e: self.log.error("Failed to fetch More Like This from Solr for document '%s': %s", query, e) raw_results = EmptyResults()
def __init__(self, model, where=WhereNode): self.extra_join = {} self.include_translation_data = True extra_select = {} super(MultilingualQuery, self).__init__(model, where=where) opts = self.model._meta qn2 = self.get_compiler(DEFAULT_DB_ALIAS).connection.ops.quote_name if not opts.proxy: translation_opts = opts.translation_model._meta else: translation_opts = get_proxied_model(model._meta)._meta.translation_model._meta opts = get_proxied_model(model._meta)._meta trans_table_name = translation_opts.db_table if hasattr(opts, 'translation_model'): for language_code in get_language_code_list(): for fname in [f.attname for f in translation_opts.fields]: table_alias = get_translation_table_alias(trans_table_name, language_code) field_alias = get_translated_field_alias(fname, language_code) extra_select[field_alias] = qn2(table_alias) + '.' + qn2(fname) self.add_extra(extra_select, None, None, None, None, None) self._trans_extra_select_count = len(self.extra_select)
def get_app_and_model(instance): """ Return the app_label and model_name for the given instance. Work for normal model instance but also a proxied one (think `only` and `defer`), and even for instance of SearchResult (haystack) """ if isinstance(instance, SearchResult): app_label, model_name = instance.app_label, instance.model_name else: meta = instance._meta if getattr(instance, '_deferred', False): meta = get_proxied_model(meta)._meta app_label, model_name = meta.app_label, meta.module_name return app_label, model_name
def create_content(self): obj = self.context['obj'] current_user_data_keys = [key for key in vars(obj) if key.startswith('current_user_')] if not isinstance(obj, SyncableModel) or obj._deferred: if isinstance(obj, SearchResult): model = obj.model else: model = get_proxied_model(obj._meta) print "RENDER %s.%s" % (model, obj.pk) full_obj = model.for_user_list.get(id=obj.pk) for key in current_user_data_keys: setattr(full_obj, key, getattr(obj, key)) self.context['obj'] = self.context[obj.model_name] = full_obj super(CoreCacheTag, self).create_content()
def get_default_columns(self, with_aliases=False, col_aliases=None, start_alias=None, opts=None, as_pairs=False, local_only=False): """ Computes the default columns for selecting every field in the base model. Will sometimes be called to pull in related models (e.g. via select_related), in which case "opts" and "start_alias" will be given to provide a starting point for the traversal. Returns a list of strings, quoted appropriately for use in SQL directly, as well as a set of aliases used in the select statement (if 'as_pairs' is True, returns a list of (alias, col_name) pairs instead of strings as the first component and None as the second component). """ result = [] if opts is None: opts = self.query.model._meta qn = self.quote_name_unless_alias qn2 = self.connection.ops.quote_name aliases = set() only_load = self.deferred_to_columns() # Skip all proxy to the root proxied model proxied_model = get_proxied_model(opts) if start_alias: seen = {None: start_alias} for field, model in opts.get_fields_with_model(): if local_only and model is not None: continue if start_alias: try: alias = seen[model] except KeyError: if model is proxied_model: alias = start_alias else: link_field = opts.get_ancestor_link(model) alias = self.query.join((start_alias, model._meta.db_table, link_field.column, model._meta.pk.column)) seen[model] = alias else: # If we're starting from the base model of the queryset, the # aliases will have already been set up in pre_sql_setup(), so # we can save time here. alias = self.query.included_inherited_models[model] table = self.query.alias_map[alias][TABLE_NAME] if table in only_load and field.column not in only_load[table]: continue if as_pairs: result.append((alias, field.column)) aliases.add(alias) continue if with_aliases and field.column in col_aliases: c_alias = 'Col%d' % len(col_aliases) result.append('%s.%s AS %s' % (qn(alias), qn2(field.column), c_alias)) col_aliases.add(c_alias) aliases.add(c_alias) else: r = '%s.%s' % (qn(alias), qn2(field.column)) result.append(r) aliases.add(r) if with_aliases: col_aliases.add(field.column) return result, aliases
def more_like_this(self, model_instance, additional_query_string=None, start_offset=0, end_offset=None, limit_to_registered_models=None, result_class=None, **kwargs): if not self.setup_complete: self.setup() # Handle deferred models. if get_proxied_model and hasattr(model_instance, '_deferred') and model_instance._deferred: model_klass = get_proxied_model(model_instance._meta) else: model_klass = type(model_instance) field_name = self.content_field_name narrow_queries = set() narrowed_results = None self.index = self.index.refresh() if limit_to_registered_models is None: limit_to_registered_models = getattr(settings, 'HAYSTACK_LIMIT_TO_REGISTERED_MODELS', True) if limit_to_registered_models: # Using narrow queries, limit the results to only models registered # with the current site. if narrow_queries is None: narrow_queries = set() registered_models = self.build_models_list() if len(registered_models) > 0: narrow_queries.add(' OR '.join(['%s:%s' % (DJANGO_CT, rm) for rm in registered_models])) if additional_query_string and additional_query_string != '*': narrow_queries.add(additional_query_string) narrow_searcher = None if narrow_queries is not None: # Potentially expensive? I don't see another way to do it in Whoosh... narrow_searcher = self.index.searcher() for nq in narrow_queries: recent_narrowed_results = narrow_searcher.search(self.parser.parse(force_unicode(nq))) if narrowed_results: narrowed_results.filter(recent_narrowed_results) else: narrowed_results = recent_narrowed_results # Prevent against Whoosh throwing an error. Requires an end_offset # greater than 0. if not end_offset is None and end_offset <= 0: end_offset = 1 # Determine the page. page_num = 0 if end_offset is None: end_offset = 1000000 if start_offset is None: start_offset = 0 page_length = end_offset - start_offset if page_length and page_length > 0: page_num = start_offset / page_length # Increment because Whoosh uses 1-based page numbers. page_num += 1 self.index = self.index.refresh() raw_results = EmptyResults() if self.index.doc_count(): query = "%s:%s" % (ID, get_identifier(model_instance)) searcher = self.index.searcher() parsed_query = self.parser.parse(query) results = searcher.search(parsed_query) if len(results): raw_results = results[0].more_like_this(field_name, top=end_offset) # Handle the case where the results have been narrowed. if narrowed_results and hasattr(raw_results, 'filter'): raw_results.filter(narrowed_results) try: raw_page = ResultsPage(raw_results, page_num, page_length) except ValueError: if not self.silently_fail: raise return { 'results': [], 'hits': 0, 'spelling_suggestion': None, } results = self._process_results(raw_page, result_class=result_class) searcher.close() if hasattr(narrow_searcher, 'close'): narrow_searcher.close() return results
def _setup_joins_with_translation(self, names, opts, alias, dupe_multis, allow_many=True, allow_explicit_fk=False, can_reuse=None, negate=False, process_extras=True): """ This is based on a full copy of Query.setup_joins because currently I see no way to handle it differently. TO DO: there might actually be a way, by splitting a single multi-name setup_joins call into separate calls. Check it. -- [email protected] Compute the necessary table joins for the passage through the fields given in 'names'. 'opts' is the Options class for the current model (which gives the table we are joining to), 'alias' is the alias for the table we are joining to. If dupe_multis is True, any many-to-many or many-to-one joins will always create a new alias (necessary for disjunctive filters). Returns the final field involved in the join, the target database column (used for any 'where' constraint), the final 'opts' value and the list of tables joined. """ joins = [alias] last = [0] dupe_set = set() exclusions = set() extra_filters = [] for pos, name in enumerate(names): #try: # exclusions.add(int_alias) #except NameError: # pass exclusions.add(alias) last.append(len(joins)) if name == 'pk': name = opts.pk.name try: field, model, direct, m2m = opts.get_field_by_name(name) except FieldDoesNotExist: for f in opts.fields: if allow_explicit_fk and name == f.attname: # XXX: A hack to allow foo_id to work in values() for # backwards compatibility purposes. If we dropped that # feature, this could be removed. field, model, direct, m2m = opts.get_field_by_name(f.name) break else: names = opts.get_all_field_names() + self.aggregate_select.keys() raise FieldError("Cannot resolve keyword %r into field. " "Choices are: %s" % (name, ", ".join(names))) if not allow_many and (m2m or not direct): for alias in joins: self.unref_alias(alias) raise MultiJoin(pos + 1) #=================================================================== # Django Multilingual NG Specific Code START #=================================================================== if hasattr(opts, 'translation_model'): translation_opts = opts.translation_model._meta if model == opts.translation_model: language_code = translation_opts.translated_fields[name][1] if language_code is None: language_code = get_default_language() #TODO: check alias master_table_name = opts.db_table trans_table_alias = get_translation_table_alias( model._meta.db_table, language_code) new_table = (master_table_name + "__" + trans_table_alias) qn = self.get_compiler(DEFAULT_DB_ALIAS).quote_name_unless_alias qn2 = self.get_compiler(DEFAULT_DB_ALIAS).connection.ops.quote_name trans_join = ("JOIN %s AS %s ON ((%s.master_id = %s.%s) AND (%s.language_code = '%s'))" % (qn2(model._meta.db_table), qn2(new_table), qn2(new_table), qn(master_table_name), qn2(model._meta.pk.column), qn2(new_table), language_code)) self.extra_join[new_table] = trans_join target = field continue #=================================================================== # Django Multilingual NG Specific Code END #=================================================================== elif model: # The field lives on a base class of the current model. # Skip the chain of proxy to the concrete proxied model proxied_model = get_proxied_model(opts) for int_model in opts.get_base_chain(model): if int_model is proxied_model: opts = int_model._meta else: lhs_col = opts.parents[int_model].column dedupe = lhs_col in opts.duplicate_targets if dedupe: exclusions.update(self.dupe_avoidance.get( (id(opts), lhs_col), ())) dupe_set.add((opts, lhs_col)) opts = int_model._meta alias = self.join((alias, opts.db_table, lhs_col, opts.pk.column), exclusions=exclusions) joins.append(alias) exclusions.add(alias) for (dupe_opts, dupe_col) in dupe_set: self.update_dupe_avoidance(dupe_opts, dupe_col, alias) cached_data = opts._join_cache.get(name) orig_opts = opts dupe_col = direct and field.column or field.field.column dedupe = dupe_col in opts.duplicate_targets if dupe_set or dedupe: if dedupe: dupe_set.add((opts, dupe_col)) exclusions.update(self.dupe_avoidance.get((id(opts), dupe_col), ())) if process_extras and hasattr(field, 'extra_filters'): extra_filters.extend(field.extra_filters(names, pos, negate)) if direct: if m2m: # Many-to-many field defined on the current model. if cached_data: (table1, from_col1, to_col1, table2, from_col2, to_col2, opts, target) = cached_data else: table1 = field.m2m_db_table() from_col1 = opts.pk.column to_col1 = field.m2m_column_name() opts = field.rel.to._meta table2 = opts.db_table from_col2 = field.m2m_reverse_name() to_col2 = opts.pk.column target = opts.pk orig_opts._join_cache[name] = (table1, from_col1, to_col1, table2, from_col2, to_col2, opts, target) int_alias = self.join((alias, table1, from_col1, to_col1), dupe_multis, exclusions, nullable=True, reuse=can_reuse) if int_alias == table2 and from_col2 == to_col2: joins.append(int_alias) alias = int_alias else: alias = self.join( (int_alias, table2, from_col2, to_col2), dupe_multis, exclusions, nullable=True, reuse=can_reuse) joins.extend([int_alias, alias]) elif field.rel: # One-to-one or many-to-one field if cached_data: (table, from_col, to_col, opts, target) = cached_data else: opts = field.rel.to._meta target = field.rel.get_related_field() table = opts.db_table from_col = field.column to_col = target.column orig_opts._join_cache[name] = (table, from_col, to_col, opts, target) alias = self.join((alias, table, from_col, to_col), exclusions=exclusions, nullable=field.null) joins.append(alias) else: # Non-relation fields. target = field break else: orig_field = field field = field.field if m2m: # Many-to-many field defined on the target model. if cached_data: (table1, from_col1, to_col1, table2, from_col2, to_col2, opts, target) = cached_data else: table1 = field.m2m_db_table() from_col1 = opts.pk.column to_col1 = field.m2m_reverse_name() opts = orig_field.opts table2 = opts.db_table from_col2 = field.m2m_column_name() to_col2 = opts.pk.column target = opts.pk orig_opts._join_cache[name] = (table1, from_col1, to_col1, table2, from_col2, to_col2, opts, target) int_alias = self.join((alias, table1, from_col1, to_col1), dupe_multis, exclusions, nullable=True, reuse=can_reuse) alias = self.join((int_alias, table2, from_col2, to_col2), dupe_multis, exclusions, nullable=True, reuse=can_reuse) joins.extend([int_alias, alias]) else: # One-to-many field (ForeignKey defined on the target model) if cached_data: (table, from_col, to_col, opts, target) = cached_data else: local_field = opts.get_field_by_name( field.rel.field_name)[0] opts = orig_field.opts table = opts.db_table from_col = local_field.column to_col = field.column target = opts.pk orig_opts._join_cache[name] = (table, from_col, to_col, opts, target) alias = self.join((alias, table, from_col, to_col), dupe_multis, exclusions, nullable=True, reuse=can_reuse) joins.append(alias) for (dupe_opts, dupe_col) in dupe_set: try: self.update_dupe_avoidance(dupe_opts, dupe_col, int_alias) except NameError: self.update_dupe_avoidance(dupe_opts, dupe_col, alias) if pos != len(names) - 1: if pos == len(names) - 2: raise FieldError("Join on field %r not permitted. Did you misspell %r for the lookup type?" % (name, names[pos + 1])) else: raise FieldError("Join on field %r not permitted." % name) return field, target, opts, joins, last, extra_filters
def more_like_this(self, model_instance, additional_query_string=None, start_offset=0, end_offset=None, models=None, limit_to_registered_models=None, result_class=None, **kwargs): from haystack import connections # Handle deferred models. if get_proxied_model and hasattr( model_instance, '_deferred') and model_instance._deferred: model_klass = get_proxied_model(model_instance._meta) else: model_klass = type(model_instance) index = connections[ self.connection_alias].get_unified_index().get_index(model_klass) field_name = index.get_content_field() params = { 'fl': '*,score', } if start_offset is not None: params['start'] = start_offset if end_offset is not None: params['rows'] = end_offset narrow_queries = set() if limit_to_registered_models is None: limit_to_registered_models = getattr( settings, 'HAYSTACK_LIMIT_TO_REGISTERED_MODELS', True) if models and len(models): model_choices = sorted([ '%s.%s' % (model._meta.app_label, model._meta.module_name) for model in models ]) elif limit_to_registered_models: # Using narrow queries, limit the results to only models handled # with the current routers. model_choices = self.build_models_list() else: model_choices = [] if len(model_choices) > 0: if narrow_queries is None: narrow_queries = set() narrow_queries.add('%s:(%s)' % (DJANGO_CT, ' OR '.join(model_choices))) if additional_query_string: narrow_queries.add(additional_query_string) if narrow_queries: params['fq'] = list(narrow_queries) query = "%s:%s" % (ID, get_identifier(model_instance)) try: raw_results = self.conn.more_like_this(query, field_name, **params) except (IOError, SolrError), e: if not self.silently_fail: raise self.log.error( "Failed to fetch More Like This from Solr for document '%s': %s", query, e) raw_results = EmptyResults()
def more_like_this(self, model_instance, additional_query_string=None, start_offset=0, end_offset=None, limit_to_registered_models=None, result_class=None, **kwargs): if not self.setup_complete: self.setup() # Handle deferred models. if get_proxied_model and hasattr( model_instance, '_deferred') and model_instance._deferred: model_klass = get_proxied_model(model_instance._meta) else: model_klass = type(model_instance) field_name = self.content_field_name narrow_queries = set() narrowed_results = None self.index = self.index.refresh() if limit_to_registered_models is None: limit_to_registered_models = getattr( settings, 'HAYSTACK_LIMIT_TO_REGISTERED_MODELS', True) if limit_to_registered_models: # Using narrow queries, limit the results to only models registered # with the current site. if narrow_queries is None: narrow_queries = set() registered_models = self.build_models_list() if len(registered_models) > 0: narrow_queries.add(' OR '.join( ['%s:%s' % (DJANGO_CT, rm) for rm in registered_models])) if additional_query_string and additional_query_string != '*': narrow_queries.add(additional_query_string) narrow_searcher = None if narrow_queries is not None: # Potentially expensive? I don't see another way to do it in Whoosh... narrow_searcher = self.index.searcher() for nq in narrow_queries: recent_narrowed_results = narrow_searcher.search( self.parser.parse(force_unicode(nq))) if narrowed_results: narrowed_results.filter(recent_narrowed_results) else: narrowed_results = recent_narrowed_results # Prevent against Whoosh throwing an error. Requires an end_offset # greater than 0. if not end_offset is None and end_offset <= 0: end_offset = 1 # Determine the page. page_num = 0 if end_offset is None: end_offset = 1000000 if start_offset is None: start_offset = 0 page_length = end_offset - start_offset if page_length and page_length > 0: page_num = start_offset / page_length # Increment because Whoosh uses 1-based page numbers. page_num += 1 self.index = self.index.refresh() raw_results = EmptyResults() if self.index.doc_count(): query = "%s:%s" % (ID, get_identifier(model_instance)) searcher = self.index.searcher() parsed_query = self.parser.parse(query) results = searcher.search(parsed_query) if len(results): raw_results = results[0].more_like_this(field_name, top=end_offset) # Handle the case where the results have been narrowed. if narrowed_results and hasattr(raw_results, 'filter'): raw_results.filter(narrowed_results) try: raw_page = ResultsPage(raw_results, page_num, page_length) except ValueError: if not self.silently_fail: raise return { 'results': [], 'hits': 0, 'spelling_suggestion': None, } results = self._process_results(raw_page, result_class=result_class) searcher.close() if hasattr(narrow_searcher, 'close'): narrow_searcher.close() return results