def test_should_fts_index_field_type(self): indexable_field_types = ['tsvector', 'text', 'number'] non_indexable_field_types = [ 'nested', 'timestamp', 'date', '_text', 'text[]' ] for indexable in indexable_field_types: assert datastore_helpers.should_fts_index_field_type( indexable) is True for non_indexable in non_indexable_field_types: assert datastore_helpers.should_fts_index_field_type( non_indexable) is False
def _build_fts_indexes(connection, data_dict, sql_index_str_method, fields): fts_indexes = [] resource_id = data_dict['resource_id'] # FIXME: This is repeated on the plugin.py, we should keep it DRY default_fts_lang = pylons.config.get('ckan.datastore.default_fts_lang') if default_fts_lang is None: default_fts_lang = u'english' fts_lang = data_dict.get('lang', default_fts_lang) # create full-text search indexes to_tsvector = lambda x: u"to_tsvector('{0}', {1})".format(fts_lang, x) cast_as_text = lambda x: u'cast("{0}" AS text)'.format(x) full_text_field = {'type': 'tsvector', 'id': '_full_text'} for field in [full_text_field] + fields: if not datastore_helpers.should_fts_index_field_type(field['type']): continue field_str = field['id'] if field['type'] not in ['text', 'tsvector']: field_str = cast_as_text(field_str) else: field_str = u'"{0}"'.format(field_str) if field['type'] != 'tsvector': field_str = to_tsvector(field_str) fts_indexes.append( sql_index_str_method.format(res_id=resource_id, unique='', name=_generate_index_name( resource_id, field_str), method=_get_fts_index_method(), fields=field_str)) return fts_indexes
def _build_fts_indexes(connection, data_dict, sql_index_str_method, fields): fts_indexes = [] resource_id = data_dict['resource_id'] # FIXME: This is repeated on the plugin.py, we should keep it DRY default_fts_lang = pylons.config.get('ckan.datastore.default_fts_lang') if default_fts_lang is None: default_fts_lang = u'english' fts_lang = data_dict.get('lang', default_fts_lang) # create full-text search indexes to_tsvector = lambda x: u"to_tsvector('{0}', {1})".format(fts_lang, x) cast_as_text = lambda x: u'cast("{0}" AS text)'.format(x) full_text_field = {'type': 'tsvector', 'id': '_full_text'} for field in [full_text_field] + fields: if not datastore_helpers.should_fts_index_field_type(field['type']): continue field_str = field['id'] if field['type'] not in ['text', 'tsvector']: field_str = cast_as_text(field_str) else: field_str = u'"{0}"'.format(field_str) if field['type'] != 'tsvector': field_str = to_tsvector(field_str) fts_indexes.append(sql_index_str_method.format( res_id=resource_id, unique='', name=_generate_index_name(resource_id, field_str), method=_get_fts_index_method(), fields=field_str)) return fts_indexes
def test_should_fts_index_field_type(self): indexable_field_types = ['tsvector', 'text', 'number'] non_indexable_field_types = ['nested', 'timestamp', 'date', '_text', 'text[]'] for indexable in indexable_field_types: assert datastore_helpers.should_fts_index_field_type(indexable) is True for non_indexable in non_indexable_field_types: assert datastore_helpers.should_fts_index_field_type(non_indexable) is False
def test_should_fts_index_field_type(self): indexable_field_types = ["tsvector", "text", "number"] non_indexable_field_types = [ "nested", "timestamp", "date", "_text", "text[]", ] for indexable in indexable_field_types: assert (datastore_helpers.should_fts_index_field_type(indexable) is True) for non_indexable in non_indexable_field_types: assert ( datastore_helpers.should_fts_index_field_type(non_indexable) is False)
def _where(self, data_dict, fields_types): filters = data_dict.get('filters', {}) clauses = [] for field, value in filters.iteritems(): if field not in fields_types: continue field_array_type = self._is_array_type(fields_types[field]) if isinstance(value, list) and not field_array_type: clause_str = (u'"{0}" in ({1})'.format(field, ','.join(['%s'] * len(value)))) clause = (clause_str,) + tuple(value) else: clause = (u'"{0}" = %s'.format(field), value) clauses.append(clause) # add full-text search where clause q = data_dict.get('q') if q: if isinstance(q, basestring): ts_query_alias = self._ts_query_alias() clause_str = u'_full_text @@ {0}'.format(ts_query_alias) clauses.append((clause_str,)) elif isinstance(q, dict): lang = self._fts_lang(data_dict.get('lang')) for field, value in q.iteritems(): if field not in fields_types: continue query_field = self._ts_query_alias(field) ftyp = fields_types[field] if not datastore_helpers.should_fts_index_field_type(ftyp): clause_str = u'_full_text @@ {0}'.format(query_field) clauses.append((clause_str,)) clause_str = (u'to_tsvector({0}, cast("{1}" as text)) ' u'@@ {2}').format(literal_string(lang), field, query_field) clauses.append((clause_str,)) return clauses
def _where(self, data_dict, fields_types): filters = data_dict.get('filters', {}) clauses = [] for field, value in filters.iteritems(): if field not in fields_types: continue field_array_type = self._is_array_type(fields_types[field]) if isinstance(value, list) and not field_array_type: clause_str = (u'"{0}" in ({1})'.format( field, ','.join(['%s'] * len(value)))) clause = (clause_str, ) + tuple(value) else: clause = (u'"{0}" = %s'.format(field), value) clauses.append(clause) # add full-text search where clause q = data_dict.get('q') if q: if isinstance(q, basestring): ts_query_alias = self._ts_query_alias() clause_str = u'_full_text @@ {0}'.format(ts_query_alias) clauses.append((clause_str, )) elif isinstance(q, dict): lang = self._fts_lang(data_dict.get('lang')) for field, value in q.iteritems(): if field not in fields_types: continue query_field = self._ts_query_alias(field) ftyp = fields_types[field] if not datastore_helpers.should_fts_index_field_type(ftyp): clause_str = u'_full_text @@ {0}'.format(query_field) clauses.append((clause_str, )) clause_str = (u'to_tsvector({0}, cast("{1}" as text)) ' u'@@ {2}').format(literal_string(lang), field, query_field) clauses.append((clause_str, )) return clauses