def test_should_fts_index_field_type(self):
indexable_field_types = ['tsvector', 'text', 'number']
non_indexable_field_types = [
'nested', 'timestamp', 'date', '_text', 'text[]'
]
for indexable in indexable_field_types:
assert datastore_helpers.should_fts_index_field_type(
indexable) is True
for non_indexable in non_indexable_field_types:
assert datastore_helpers.should_fts_index_field_type(
non_indexable) is False
def _build_fts_indexes(connection, data_dict, sql_index_str_method, fields):
fts_indexes = []
resource_id = data_dict['resource_id']
# FIXME: This is repeated on the plugin.py, we should keep it DRY
default_fts_lang = pylons.config.get('ckan.datastore.default_fts_lang')
if default_fts_lang is None:
default_fts_lang = u'english'
fts_lang = data_dict.get('lang', default_fts_lang)
# create full-text search indexes
to_tsvector = lambda x: u"to_tsvector('{0}', {1})".format(fts_lang, x)
cast_as_text = lambda x: u'cast("{0}" AS text)'.format(x)
full_text_field = {'type': 'tsvector', 'id': '_full_text'}
for field in [full_text_field] + fields:
if not datastore_helpers.should_fts_index_field_type(field['type']):
continue
field_str = field['id']
if field['type'] not in ['text', 'tsvector']:
field_str = cast_as_text(field_str)
else:
field_str = u'"{0}"'.format(field_str)
if field['type'] != 'tsvector':
field_str = to_tsvector(field_str)
fts_indexes.append(
sql_index_str_method.format(res_id=resource_id,
unique='',
name=_generate_index_name(
resource_id, field_str),
method=_get_fts_index_method(),
fields=field_str))
return fts_indexes
def _build_fts_indexes(connection, data_dict, sql_index_str_method, fields):
fts_indexes = []
resource_id = data_dict['resource_id']
# FIXME: This is repeated on the plugin.py, we should keep it DRY
default_fts_lang = pylons.config.get('ckan.datastore.default_fts_lang')
if default_fts_lang is None:
default_fts_lang = u'english'
fts_lang = data_dict.get('lang', default_fts_lang)
# create full-text search indexes
to_tsvector = lambda x: u"to_tsvector('{0}', {1})".format(fts_lang, x)
cast_as_text = lambda x: u'cast("{0}" AS text)'.format(x)
full_text_field = {'type': 'tsvector', 'id': '_full_text'}
for field in [full_text_field] + fields:
if not datastore_helpers.should_fts_index_field_type(field['type']):
continue
field_str = field['id']
if field['type'] not in ['text', 'tsvector']:
field_str = cast_as_text(field_str)
else:
field_str = u'"{0}"'.format(field_str)
if field['type'] != 'tsvector':
field_str = to_tsvector(field_str)
fts_indexes.append(sql_index_str_method.format(
res_id=resource_id,
unique='',
name=_generate_index_name(resource_id, field_str),
method=_get_fts_index_method(), fields=field_str))
return fts_indexes
def test_should_fts_index_field_type(self):
indexable_field_types = ['tsvector',
'text',
'number']
non_indexable_field_types = ['nested',
'timestamp',
'date',
'_text',
'text[]']
for indexable in indexable_field_types:
assert datastore_helpers.should_fts_index_field_type(indexable) is True
for non_indexable in non_indexable_field_types:
assert datastore_helpers.should_fts_index_field_type(non_indexable) is False
def test_should_fts_index_field_type(self):
indexable_field_types = ["tsvector", "text", "number"]
non_indexable_field_types = [
"nested",
"timestamp",
"date",
"_text",
"text[]",
]
for indexable in indexable_field_types:
assert (datastore_helpers.should_fts_index_field_type(indexable) is
True)
for non_indexable in non_indexable_field_types:
assert (
datastore_helpers.should_fts_index_field_type(non_indexable) is
False)
def _where(self, data_dict, fields_types):
filters = data_dict.get('filters', {})
clauses = []
for field, value in filters.iteritems():
if field not in fields_types:
continue
field_array_type = self._is_array_type(fields_types[field])
if isinstance(value, list) and not field_array_type:
clause_str = (u'"{0}" in ({1})'.format(field,
','.join(['%s'] * len(value))))
clause = (clause_str,) + tuple(value)
else:
clause = (u'"{0}" = %s'.format(field), value)
clauses.append(clause)
# add full-text search where clause
q = data_dict.get('q')
if q:
if isinstance(q, basestring):
ts_query_alias = self._ts_query_alias()
clause_str = u'_full_text @@ {0}'.format(ts_query_alias)
clauses.append((clause_str,))
elif isinstance(q, dict):
lang = self._fts_lang(data_dict.get('lang'))
for field, value in q.iteritems():
if field not in fields_types:
continue
query_field = self._ts_query_alias(field)
ftyp = fields_types[field]
if not datastore_helpers.should_fts_index_field_type(ftyp):
clause_str = u'_full_text @@ {0}'.format(query_field)
clauses.append((clause_str,))
clause_str = (u'to_tsvector({0}, cast("{1}" as text)) '
u'@@ {2}').format(literal_string(lang),
field, query_field)
clauses.append((clause_str,))
return clauses
def _where(self, data_dict, fields_types):
filters = data_dict.get('filters', {})
clauses = []
for field, value in filters.iteritems():
if field not in fields_types:
continue
field_array_type = self._is_array_type(fields_types[field])
if isinstance(value, list) and not field_array_type:
clause_str = (u'"{0}" in ({1})'.format(
field, ','.join(['%s'] * len(value))))
clause = (clause_str, ) + tuple(value)
else:
clause = (u'"{0}" = %s'.format(field), value)
clauses.append(clause)
# add full-text search where clause
q = data_dict.get('q')
if q:
if isinstance(q, basestring):
ts_query_alias = self._ts_query_alias()
clause_str = u'_full_text @@ {0}'.format(ts_query_alias)
clauses.append((clause_str, ))
elif isinstance(q, dict):
lang = self._fts_lang(data_dict.get('lang'))
for field, value in q.iteritems():
if field not in fields_types:
continue
query_field = self._ts_query_alias(field)
ftyp = fields_types[field]
if not datastore_helpers.should_fts_index_field_type(ftyp):
clause_str = u'_full_text @@ {0}'.format(query_field)
clauses.append((clause_str, ))
clause_str = (u'to_tsvector({0}, cast("{1}" as text)) '
u'@@ {2}').format(literal_string(lang),
field, query_field)
clauses.append((clause_str, ))
return clauses
