def _create_timeframe_query(self, data):
"""
Takes a dictionary of data that was distilled by the Context's
primary_distillery. If the Context has a time_interval and
time_unit, returns a list of fieldset dictionaries for a
time-frame query. Otherwise, returns an empty list.
"""
timeframe = []
if self._has_time_interval():
start_time = self._get_start_time(data)
end_time = self._get_end_time(data)
if start_time:
timeframe.append(
QueryFieldset(field_name=self._related_date_field,
field_type='DateTimeField',
operator='gt',
value=start_time))
if end_time:
timeframe.append(
QueryFieldset(field_name=self._related_date_field,
field_type='DateTimeField',
operator='lte',
value=end_time))
if timeframe:
return EngineQuery(timeframe, 'AND')
def _get_engine_query(distillery, query, before=None, after=None):
"""Return QueryFieldsets of keyword and field searches for a distillery.
Parameters
----------
distillery : Distillery
query: query.search.search_query.SearchQuery
before: datetime.datetime or None
after: datetime.datetime or None
Returns
-------
EngineQuery or None
"""
engine_queries = [
DistillerySearchResults._get_field_engine_query(
distillery, query.field_parameters),
DistillerySearchResults._get_keyword_engine_query(
distillery, query.keywords)
]
subqueries = [
engine_query for engine_query in engine_queries if engine_query
]
if not subqueries:
return None
if before or after:
searchable_date_field = distillery.get_searchable_date_field()
if searchable_date_field:
if before:
subqueries += [
QueryFieldset(field_name=searchable_date_field,
field_type='DateTimeField',
operator='lte',
value=before.isoformat())
]
if after:
subqueries += [
QueryFieldset(field_name=searchable_date_field,
field_type='DateTimeField',
operator='gte',
value=after.isoformat())
]
return EngineQuery(subqueries=subqueries, joiner='AND')
def _create_keyword_engine_query(text_fields, keyword):
"""
Parameters
----------
text_fields : list of DataField
keyword: str
Returns
-------
EngineQuery or None
"""
fieldsets = [
QueryFieldset(
field_name=field.field_name,
field_type=field.field_type,
operator='regex',
value=keyword,
) for field in text_fields
]
if not fieldsets:
return None
return EngineQuery(subqueries=fieldsets, joiner='OR')
def test_within_multiple_polygons(self):
"""
Tests the find method using a 'within' filter for a feature collection
with more than one polygon.
"""
features = {
'type': 'FeatureCollection',
'features': [self.polygon1, self.polygon2]
}
fieldsets = [
QueryFieldset(field_name='location',
field_type='PointField',
operator='within',
value=json.dumps(features))
]
query = EngineQuery(fieldsets, 'AND')
results = self.engine.find(query)
count = results['count']
docs = results['results']
expected_names = ['john', 'jane']
self.assertEqual(count, 2)
self.assertIn(
self._get_doc(docs, 0)['user']['screen_name'], expected_names)
self.assertIn(
self._get_doc(docs, 1)['user']['screen_name'], expected_names)
class QueryFieldsetTestCase(TestCase):
"""
Tests the QueryFieldset class.
"""
fieldset = QueryFieldset(
field_name='foo',
field_type='CharField',
operator='eq',
value='foobar'
)
def test_str(self):
"""
Tests the __str__ method on a QueryFieldset.
"""
actual = str(self.fieldset)
expected = ("QueryFieldset: {'field_name': 'foo', 'field_type': "
"'CharField', 'operator': 'eq', 'value': 'foobar'}")
self.assertEqual(actual, expected)
def test_vars(self):
"""
Tests the outcome of the vars() function on a QueryFieldset.
"""
actual = vars(self.fieldset)
expected = {
'field_name': 'foo',
'field_type': 'CharField',
'operator': 'eq',
'value': 'foobar'
}
self.assertEqual(actual, expected)
def create_fieldset(self, data):
"""
Takes a dictionary of data that was distilled by the Context's
Focal Distillery. Returns a QueryFieldset representing a
query expression for the ContextFilter.
"""
value = get_dict_value(self.value_field, data)
field_type = get_field_type(self.search_field)
return QueryFieldset(field_name=self.search_field,
field_type=field_type,
operator=self.operator,
value=value)
def test_find_fields(self):
"""
Tests that the find method only returns fields defined in the
Engine's schema.
"""
doc_id = self.engine.insert({
'content': {
'text': 'I like cats and dogs.',
'tags': ['cats', 'dogs'],
},
'user': {
'screen_name': 'Jill',
'email': '*****@*****.**',
},
})
self.engine.schema = [
DataField(
field_name='content.text',
field_type='TextField',
target_type='Keyword'
),
DataField(
field_name='user.screen_name',
field_type='CharField',
target_type='Account'
)
]
fieldsets = [
QueryFieldset(
field_name='user.screen_name',
field_type='CharField',
operator='eq',
value='Jill'
)
]
query = EngineQuery(fieldsets, 'AND')
actual = self.engine.find(query)
expected = {
'count': 1,
'results': [
{
'_id': doc_id,
'content': {
'text': 'I like cats and dogs.'
},
'user': {
'screen_name': 'Jill'
}
}
]
}
self.assertEqual(actual, expected)
def _get_query(self, date_field):
"""
Takes the name of a date field and returns an |EngineQuery| for
documents with dates later than the last_healthy date (if there
is one) or the start of the monitoring interval (if there isn't).
"""
start_time = self._get_query_start_time()
query = QueryFieldset(field_name=date_field,
field_type='DateTimeField',
operator='gt',
value=start_time)
return EngineQuery([query])
def _get_results(query):
"""
Takes a query dictionary and returns a list of documents that match the
query criteria.
"""
subqueries = [QueryFieldset(**fieldset) \
for fieldset in query['fieldsets']]
engine_query = EngineQuery(subqueries=subqueries,
joiner=query['joiner'])
docs = []
for collection in query['collections']:
results = collection.find(engine_query)
docs.extend(results['results'])
return docs
def test_lte(self):
"""
Tests the find method for an 'lte' (less than or equal to) filter.
"""
fieldsets = [
QueryFieldset(field_name='user.age',
field_type='IntegerField',
operator='lte',
value=30)
]
query = EngineQuery(fieldsets, 'AND')
results = self.engine.find(query)
count = results['count']
docs = results['results']
self.assertEqual(count, 3)
def _create_keyword_query(self, keyword):
"""
"""
keyword_search = []
if keyword:
fields = self._get_text_fields()
for field in fields:
keyword_search.append(
QueryFieldset(field_name=field.field_name,
field_type=field.field_type,
operator='regex',
value=keyword))
if keyword_search:
return EngineQuery(keyword_search, 'OR')
def test_gte(self):
"""
Tests the find method for a 'gte' (greater than or equal to) filter.
"""
fieldsets = [
QueryFieldset(field_name='user.age',
field_type='IntegerField',
operator='gte',
value=20)
]
query = EngineQuery(fieldsets, 'AND')
results = self.engine.find(query)
expected_names = ['jane', 'jack']
count = results['count']
docs = results['results']
self.assertEqual(count, 3)
def test_eq_email(self):
"""
Tests the find method for an 'eq' (equals) filter on an EmailField.
"""
fieldsets = [
QueryFieldset(field_name='user.email',
field_type='EmailField',
operator='eq',
value='*****@*****.**')
]
query = EngineQuery(fieldsets, 'AND')
results = self.engine.find(query)
count = results['count']
docs = results['results']
self.assertEqual(count, 1)
self.assertEqual(self._get_doc(docs, 0)['user']['screen_name'], 'jane')
def test_eq_text(self):
"""
Tests the find method for an 'eq' (equals) filter on a CharField.
"""
fieldsets = [
QueryFieldset(field_name='content.text',
field_type='TextField',
operator='eq',
value='I like dogs.')
]
query = EngineQuery(fieldsets, 'AND')
results = self.engine.find(query)
count = results['count']
docs = results['results']
self.assertEqual(count, 1)
self.assertEqual(self._get_doc(docs, 0)['user']['screen_name'], 'jane')
def test_not_in(self):
"""
Tests the find method for a 'not:in' filter.
"""
fieldsets = [
QueryFieldset(field_name='content.tags',
field_type='CharField',
operator='not:in',
value='cats')
]
query = EngineQuery(fieldsets, 'AND')
results = self.engine.find(query)
count = results['count']
docs = results['results']
self.assertEqual(count, 1)
self.assertEqual(self._get_doc(docs, 0)['user']['screen_name'], 'jane')
def test_lt(self):
"""
Tests the find method for an 'lt' (less than) filter.
"""
fieldsets = [
QueryFieldset(field_name='user.age',
field_type='IntegerField',
operator='lt',
value=30)
]
query = EngineQuery(fieldsets, 'AND')
results = self.engine.find(query)
count = results['count']
docs = results['results']
self.assertEqual(count, 1)
self.assertEqual(self._get_doc(docs, 0)['user']['screen_name'], 'john')
def test_find(self, mock_engine):
"""
Tests the find method of the Collection class.
"""
collection = Collection()
fieldset = QueryFieldset(field_name='text',
field_type='CharField',
operator='regex',
value='test')
joiner = 'AND'
query = EngineQuery([fieldset], joiner)
collection.engine.find = Mock(return_value=[self.doc])
docs = collection.find(query)
collection.engine.find.assert_called_once_with(query, None, 1,
PAGE_SIZE)
self.assertEqual(docs, [self.doc])
def test_regex_unmatched_quote(self):
"""
Tests the find method for a 'regex' filter with a string
containing an unmatched quotation mark.
"""
fieldsets = [
QueryFieldset(field_name='user.screen_name',
field_type='CharField',
operator='regex',
value='"john')
]
query = EngineQuery(fieldsets, 'AND')
results = self.engine.find(query)
count = results['count']
self.assertEqual(count, 0)
def test_not_regex_with_fragment(self):
"""
Tests the find method for a 'not:regex' filter with a word fragment.
"""
fieldsets = [
QueryFieldset(field_name='user.screen_name',
field_type='CharField',
operator='not:regex',
value='ja')
]
query = EngineQuery(fieldsets, 'AND')
results = self.engine.find(query)
count = results['count']
docs = results['results']
self.assertEqual(count, 1)
self.assertEqual(self._get_doc(docs, 0)['user']['screen_name'], 'john')
def test_not_missing(self):
"""
Tests the find method using a 'not missing' filter.
"""
fieldsets = [
QueryFieldset(field_name='user.last_login',
field_type='DateTimeField',
operator='not:missing',
value='')
]
query = EngineQuery(fieldsets, 'AND')
results = self.engine.find(query)
count = results['count']
docs = results['results']
expected_name = 'john'
self.assertEqual(count, 1)
self.assertEqual(
self._get_doc(docs, 0)['user']['screen_name'], expected_name)
def test_find_tf_end_time_or(self):
"""
Tests the find method with an endtime and an 'OR' joiner.
"""
field_query = EngineQuery(self.fieldsets, 'OR')
timeframe = [
QueryFieldset(field_name='_saved_date',
field_type='DateTimeField',
operator='gte',
value=self.time + timedelta(hours=1))
]
subqueries = [field_query] + timeframe
query = EngineQuery(subqueries, 'AND')
results = self.engine.find(query)
docs = results['results']
count = results['count']
self.assertEqual(count, 1)
self.assertEqual(len(docs), 1)
def test_within_non_polygon(self):
"""
Tests the find method using a 'within' filter for a feature collection
that includes a non-polygon feature.
"""
features = {
'type': 'FeatureCollection',
'features': [self.polygon1, self.nonpolygon]
}
fieldsets = [
QueryFieldset(field_name='location',
field_type='PointField',
operator='within',
value=json.dumps(features))
]
with self.assertRaises(ValueError):
query = EngineQuery(fieldsets, 'AND')
self.engine.find(query)
def test_regex_with_multiple_words(self):
"""
Tests the 'regex' filter with multiple words.
"""
fieldsets = [
QueryFieldset(field_name='content.text',
field_type='TextField',
operator='regex',
value='I like cats')
]
query = EngineQuery(fieldsets, 'AND')
results = self.engine.find(query)
count = results['count']
docs = results['results']
expected_names = ['john', 'jack']
self.assertEqual(count, 2)
self.assertIn(
self._get_doc(docs, 0)['user']['screen_name'], expected_names)
self.assertIn(
self._get_doc(docs, 1)['user']['screen_name'], expected_names)
def test_in(self):
"""
Tests the find method for an 'in' filter.
"""
fieldsets = [
QueryFieldset(field_name='content.tags',
field_type='ListField',
operator='in',
value='cats')
]
query = EngineQuery(fieldsets, 'AND')
results = self.engine.find(query)
count = results['count']
docs = results['results']
expected_names = ['john', 'jack']
self.assertEqual(count, 2)
self.assertIn(
self._get_doc(docs, 0)['user']['screen_name'], expected_names)
self.assertIn(
self._get_doc(docs, 1)['user']['screen_name'], expected_names)
def test_regex_with_caps(self):
"""
Tests that 'regex' filter is not case-sensitive.
"""
fieldsets = [
QueryFieldset(field_name='content.text',
field_type='TextField',
operator='regex',
value='CAT')
]
query = EngineQuery(fieldsets, 'AND')
results = self.engine.find(query)
count = results['count']
docs = results['results']
expected_names = ['john', 'jack']
self.assertEqual(count, 2)
self.assertIn(
self._get_doc(docs, 0)['user']['screen_name'], expected_names)
self.assertIn(
self._get_doc(docs, 1)['user']['screen_name'], expected_names)
def _create_keyword_fieldset(text_field, keywords):
"""Return QueryFieldset of a DataField that takes keywords.
Parameters
----------
text_field : bottler.datafields.models.DataField
keywords : list of str
Returns
-------
QueryFieldset
"""
return QueryFieldset(
field_name=text_field.field_name,
field_type=text_field.field_type,
operator='regex',
value='|'.join(keywords),
)
def test_not_eq(self):
"""
Tests the find method for a 'not:eq' filter.
"""
fieldsets = [
QueryFieldset(field_name='user.screen_name',
field_type='CharField',
operator='not:eq',
value='jack')
]
query = EngineQuery(fieldsets, 'AND')
results = self.engine.find(query)
count = results['count']
docs = results['results']
expected_names = ['john', 'jane']
self.assertEqual(count, 2)
self.assertIn(
self._get_doc(docs, 0)['user']['screen_name'], expected_names)
self.assertIn(
self._get_doc(docs, 1)['user']['screen_name'], expected_names)
def test_gt(self):
"""
Tests the find method for a 'gt' (greater than) filter.
"""
fieldsets = [
QueryFieldset(field_name='user.age',
field_type='IntegerField',
operator='gt',
value=20)
]
query = EngineQuery(fieldsets, 'AND')
results = self.engine.find(query)
expected_names = ['jane', 'jack']
count = results['count']
docs = results['results']
self.assertEqual(count, 2)
self.assertIn(
self._get_doc(docs, 0)['user']['screen_name'], expected_names)
self.assertIn(
self._get_doc(docs, 1)['user']['screen_name'], expected_names)
def create_fieldset(self):
"""Create a QueryFieldset from this parameter and a distillery.
Returns
-------
QueryFieldset
Raises
------
AssertionError
If the FieldSearchParameter is not valid.
"""
if not self.is_valid():
raise ValueError(FieldSearchParameter.CANNOT_CREATE_FIELDSET)
return QueryFieldset(
self.field_name,
self.data_field.field_type,
self.operator.fieldset_operator,
self.value.parsed_value,
)
class FilterTestCaseMixin(object):
"""
Mixin for use with a EngineBaseTestCase subclass to test the find() method
of an Engine subclass. Provides tests for the standard set of query
selectors (eq, gte, etc.) that are used in defining queries.
"""
time = timezone.now()
test_docs = [{
'_raw_data': {
'backend': 'example_backend',
'database': 'example_database',
'collection': 'raw_data',
'doc_id': 1
},
'_saved_date': time - timedelta(days=1),
'user': {
'screen_name': 'john',
'email': '*****@*****.**',
'link': 'http://www.acme.com/john',
'age': 20,
'last_login': '******'
},
'content': {
'text': 'I like cats.',
'tags': ['cats', 'pets']
},
'location': [75.0, 25.0]
}, {
'_raw_data': {
'backend': 'example_backend',
'database': 'example_database',
'collection': 'raw_data',
'doc_id': 2
},
'_saved_date': time,
'user': {
'screen_name': 'jane',
'email': '*****@*****.**',
'link': 'http://www.acme.com/jane',
'age': 30,
'last_login': None
},
'content': {
'text': 'I like dogs.',
'tags': ['dogs', 'pets']
},
'location': [25.0, 25.0]
}, {
'_raw_data': {
'backend': 'example_backend',
'database': 'example_database',
'collection': 'raw_data',
'doc_id': 3
},
'_saved_date': time + timedelta(days=1),
'user': {
'screen_name': 'jack',
'email': '*****@*****.**',
'link': 'http://www.acme.com/jack',
'age': 30
},
'content': {
'text': 'I LIKE CATS AND DOGS.',
'tags': ['cats', 'dogs', 'pets']
}
}]
polygon1 = {
'type': 'Feature',
'geometry': {
'type':
'Polygon',
'coordinates': [[[50.0, 0.0], [100.0, 0.0], [100.0, 50.0],
[50.0, 50.0], [50.0, 0.0]]]
}
}
polygon2 = {
'type': 'Feature',
'geometry': {
'type':
'Polygon',
'coordinates': [[[0.0, 0.0], [100.0, 0.0], [100.0, 0.0],
[00.0, 50.0], [0.0, 0.0]]]
}
}
nonpolygon = {
'type': 'Feature',
'geometry': {
'type': 'Point',
'coordinates': [100.0, 0.5]
},
}
fieldsets = [
QueryFieldset(field_name='content.text',
field_type='TextField',
operator='regex',
value='cat'),
QueryFieldset(field_name='content.text',
field_type='TextField',
operator='regex',
value='dog')
]
timeframe = [
QueryFieldset(field_name='_saved_date',
field_type='DateTimeField',
operator='gte',
value=time),
QueryFieldset(field_name='_saved_date',
field_type='DateTimeField',
operator='lte',
value=time + timedelta(days=2))
]
def test_within_single_polygon(self):
"""
Tests the find method using a 'within' filter for a feature collection
with a single polygon.
"""
features = {'type': 'FeatureCollection', 'features': [self.polygon1]}
fieldsets = [
QueryFieldset(field_name='location',
field_type='PointField',
operator='within',
value=json.dumps(features))
]
query = EngineQuery(fieldsets, 'AND')
results = self.engine.find(query)
count = results['count']
docs = results['results']
expected_name = 'john'
self.assertEqual(count, 1)
self.assertEqual(
self._get_doc(docs, 0)['user']['screen_name'], expected_name)
def test_within_multiple_polygons(self):
"""
Tests the find method using a 'within' filter for a feature collection
with more than one polygon.
"""
features = {
'type': 'FeatureCollection',
'features': [self.polygon1, self.polygon2]
}
fieldsets = [
QueryFieldset(field_name='location',
field_type='PointField',
operator='within',
value=json.dumps(features))
]
query = EngineQuery(fieldsets, 'AND')
results = self.engine.find(query)
count = results['count']
docs = results['results']
expected_names = ['john', 'jane']
self.assertEqual(count, 2)
self.assertIn(
self._get_doc(docs, 0)['user']['screen_name'], expected_names)
self.assertIn(
self._get_doc(docs, 1)['user']['screen_name'], expected_names)
def test_within_non_polygon(self):
"""
Tests the find method using a 'within' filter for a feature collection
that includes a non-polygon feature.
"""
features = {
'type': 'FeatureCollection',
'features': [self.polygon1, self.nonpolygon]
}
fieldsets = [
QueryFieldset(field_name='location',
field_type='PointField',
operator='within',
value=json.dumps(features))
]
with self.assertRaises(ValueError):
query = EngineQuery(fieldsets, 'AND')
self.engine.find(query)
def test_not_missing(self):
"""
Tests the find method using a 'not missing' filter.
"""
fieldsets = [
QueryFieldset(field_name='user.last_login',
field_type='DateTimeField',
operator='not:missing',
value='')
]
query = EngineQuery(fieldsets, 'AND')
results = self.engine.find(query)
count = results['count']
docs = results['results']
expected_name = 'john'
self.assertEqual(count, 1)
self.assertEqual(
self._get_doc(docs, 0)['user']['screen_name'], expected_name)
def test_regex_with_fragment(self):
"""
Tests the find method using a 'regex' filter.
"""
fieldsets = [
QueryFieldset(field_name='content.text',
field_type='TextField',
operator='regex',
value='cat')
]
query = EngineQuery(fieldsets, 'AND')
results = self.engine.find(query)
count = results['count']
docs = results['results']
expected_names = ['john', 'jack']
self.assertEqual(count, 2)
self.assertIn(
self._get_doc(docs, 0)['user']['screen_name'], expected_names)
self.assertIn(
self._get_doc(docs, 1)['user']['screen_name'], expected_names)
def test_regex_with_caps(self):
"""
Tests that 'regex' filter is not case-sensitive.
"""
fieldsets = [
QueryFieldset(field_name='content.text',
field_type='TextField',
operator='regex',
value='CAT')
]
query = EngineQuery(fieldsets, 'AND')
results = self.engine.find(query)
count = results['count']
docs = results['results']
expected_names = ['john', 'jack']
self.assertEqual(count, 2)
self.assertIn(
self._get_doc(docs, 0)['user']['screen_name'], expected_names)
self.assertIn(
self._get_doc(docs, 1)['user']['screen_name'], expected_names)
def test_regex_with_multiple_words(self):
"""
Tests the 'regex' filter with multiple words.
"""
fieldsets = [
QueryFieldset(field_name='content.text',
field_type='TextField',
operator='regex',
value='I like cats')
]
query = EngineQuery(fieldsets, 'AND')
results = self.engine.find(query)
count = results['count']
docs = results['results']
expected_names = ['john', 'jack']
self.assertEqual(count, 2)
self.assertIn(
self._get_doc(docs, 0)['user']['screen_name'], expected_names)
self.assertIn(
self._get_doc(docs, 1)['user']['screen_name'], expected_names)
def test_regex_unmatched_quote(self):
"""
Tests the find method for a 'regex' filter with a string
containing an unmatched quotation mark.
"""
fieldsets = [
QueryFieldset(field_name='user.screen_name',
field_type='CharField',
operator='regex',
value='"john')
]
query = EngineQuery(fieldsets, 'AND')
results = self.engine.find(query)
count = results['count']
self.assertEqual(count, 0)
def test_not_regex_with_fragment(self):
"""
Tests the find method for a 'not:regex' filter with a word fragment.
"""
fieldsets = [
QueryFieldset(field_name='user.screen_name',
field_type='CharField',
operator='not:regex',
value='ja')
]
query = EngineQuery(fieldsets, 'AND')
results = self.engine.find(query)
count = results['count']
docs = results['results']
self.assertEqual(count, 1)
self.assertEqual(self._get_doc(docs, 0)['user']['screen_name'], 'john')
def test_not_regex_with_multi_words(self):
"""
Tests the find method for a 'not:regex' filter with multiple words.
"""
fieldsets = [
QueryFieldset(field_name='content.text',
field_type='TextField',
operator='not:regex',
value='I like cats and dogs')
]
query = EngineQuery(fieldsets, 'AND')
results = self.engine.find(query)
count = results['count']
docs = results['results']
expected_names = ['john', 'jane']
self.assertEqual(count, 2)
self.assertIn(
self._get_doc(docs, 0)['user']['screen_name'], expected_names)
self.assertIn(
self._get_doc(docs, 1)['user']['screen_name'], expected_names)
def test_and(self):
"""
Tests the find method using two query terms joined by 'AND'.
"""
query = EngineQuery(self.fieldsets, 'AND')
results = self.engine.find(query)
count = results['count']
docs = results['results']
self.assertEqual(count, 1)
self.assertEqual(self._get_doc(docs, 0)['user']['screen_name'], 'jack')
def test_or(self):
"""
Tests the find method using two query terms joined by 'OR'.
"""
query = EngineQuery(self.fieldsets, 'OR')
results = self.engine.find(query)
count = results['count']
self.assertEqual(count, 3)
def test_eq_numeric(self):
"""
Tests the find method for an 'eq' (equals) filter.
"""
fieldsets = [
QueryFieldset(field_name='user.age',
field_type='IntegerField',
operator='eq',
value='20')
]
query = EngineQuery(fieldsets, 'AND')
results = self.engine.find(query)
count = results['count']
docs = results['results']
self.assertEqual(count, 1)
self.assertEqual(self._get_doc(docs, 0)['user']['screen_name'], 'john')
def test_eq_text(self):
"""
Tests the find method for an 'eq' (equals) filter on a CharField.
"""
fieldsets = [
QueryFieldset(field_name='content.text',
field_type='TextField',
operator='eq',
value='I like dogs.')
]
query = EngineQuery(fieldsets, 'AND')
results = self.engine.find(query)
count = results['count']
docs = results['results']
self.assertEqual(count, 1)
self.assertEqual(self._get_doc(docs, 0)['user']['screen_name'], 'jane')
def test_eq_email(self):
"""
Tests the find method for an 'eq' (equals) filter on an EmailField.
"""
fieldsets = [
QueryFieldset(field_name='user.email',
field_type='EmailField',
operator='eq',
value='*****@*****.**')
]
query = EngineQuery(fieldsets, 'AND')
results = self.engine.find(query)
count = results['count']
docs = results['results']
self.assertEqual(count, 1)
self.assertEqual(self._get_doc(docs, 0)['user']['screen_name'], 'jane')
def test_in(self):
"""
Tests the find method for an 'in' filter.
"""
fieldsets = [
QueryFieldset(field_name='content.tags',
field_type='ListField',
operator='in',
value='cats')
]
query = EngineQuery(fieldsets, 'AND')
results = self.engine.find(query)
count = results['count']
docs = results['results']
expected_names = ['john', 'jack']
self.assertEqual(count, 2)
self.assertIn(
self._get_doc(docs, 0)['user']['screen_name'], expected_names)
self.assertIn(
self._get_doc(docs, 1)['user']['screen_name'], expected_names)
def test_gt(self):
"""
Tests the find method for a 'gt' (greater than) filter.
"""
fieldsets = [
QueryFieldset(field_name='user.age',
field_type='IntegerField',
operator='gt',
value=20)
]
query = EngineQuery(fieldsets, 'AND')
results = self.engine.find(query)
expected_names = ['jane', 'jack']
count = results['count']
docs = results['results']
self.assertEqual(count, 2)
self.assertIn(
self._get_doc(docs, 0)['user']['screen_name'], expected_names)
self.assertIn(
self._get_doc(docs, 1)['user']['screen_name'], expected_names)
def test_gte(self):
"""
Tests the find method for a 'gte' (greater than or equal to) filter.
"""
fieldsets = [
QueryFieldset(field_name='user.age',
field_type='IntegerField',
operator='gte',
value=20)
]
query = EngineQuery(fieldsets, 'AND')
results = self.engine.find(query)
expected_names = ['jane', 'jack']
count = results['count']
docs = results['results']
self.assertEqual(count, 3)
def test_lt(self):
"""
Tests the find method for an 'lt' (less than) filter.
"""
fieldsets = [
QueryFieldset(field_name='user.age',
field_type='IntegerField',
operator='lt',
value=30)
]
query = EngineQuery(fieldsets, 'AND')
results = self.engine.find(query)
count = results['count']
docs = results['results']
self.assertEqual(count, 1)
self.assertEqual(self._get_doc(docs, 0)['user']['screen_name'], 'john')
def test_lte(self):
"""
Tests the find method for an 'lte' (less than or equal to) filter.
"""
fieldsets = [
QueryFieldset(field_name='user.age',
field_type='IntegerField',
operator='lte',
value=30)
]
query = EngineQuery(fieldsets, 'AND')
results = self.engine.find(query)
count = results['count']
docs = results['results']
self.assertEqual(count, 3)
def test_not_eq(self):
"""
Tests the find method for a 'not:eq' filter.
"""
fieldsets = [
QueryFieldset(field_name='user.screen_name',
field_type='CharField',
operator='not:eq',
value='jack')
]
query = EngineQuery(fieldsets, 'AND')
results = self.engine.find(query)
count = results['count']
docs = results['results']
expected_names = ['john', 'jane']
self.assertEqual(count, 2)
self.assertIn(
self._get_doc(docs, 0)['user']['screen_name'], expected_names)
self.assertIn(
self._get_doc(docs, 1)['user']['screen_name'], expected_names)
def test_not_in(self):
"""
Tests the find method for a 'not:in' filter.
"""
fieldsets = [
QueryFieldset(field_name='content.tags',
field_type='CharField',
operator='not:in',
value='cats')
]
query = EngineQuery(fieldsets, 'AND')
results = self.engine.find(query)
count = results['count']
docs = results['results']
self.assertEqual(count, 1)
self.assertEqual(self._get_doc(docs, 0)['user']['screen_name'], 'jane')
def test_find_fields(self):
"""
Tests that the find method only returns fields defined in the
Engine's schema.
"""
doc_id = self.engine.insert({
'content': {
'text': 'I like cats and dogs.',
'tags': ['cats', 'dogs'],
},
'user': {
'screen_name': 'Jill',
'email': '*****@*****.**',
},
})
self.engine.schema = [
DataField(field_name='content.text',
field_type='TextField',
target_type='Keyword'),
DataField(field_name='user.screen_name',
field_type='CharField',
target_type='Account')
]
fieldsets = [
QueryFieldset(field_name='user.screen_name',
field_type='CharField',
operator='eq',
value='Jill')
]
query = EngineQuery(fieldsets, 'AND')
actual = self.engine.find(query)
expected = {
'count':
1,
'results': [{
'_id': doc_id,
'content': {
'text': 'I like cats and dogs.'
},
'user': {
'screen_name': 'Jill'
}
}]
}
self.assertEqual(actual, expected)
def test_find_tf_no_tf_and(self):
"""
Tests the find method with no timeframe and an 'AND' joiner.
"""
fieldsets = self.fieldsets
joiner = 'AND'
query = EngineQuery(fieldsets, joiner)
results = self.engine.find(query)
docs = results['results']
count = results['count']
self.assertEqual(count, 1)
self.assertEqual(len(docs), 1)
self.assertEqual(self._get_doc(docs, 0)['user']['screen_name'], 'jack')
def test_find_tf_no_tf_or(self):
"""
Tests the find method with no timeframe and an 'OR' joiner.
"""
fieldsets = self.fieldsets
joiner = 'OR'
query = EngineQuery(fieldsets, joiner)
results = self.engine.find(query)
docs = results['results']
count = results['count']
self.assertEqual(count, 3)
self.assertEqual(len(docs), 3)
def test_find_tf_w_tf_and(self):
"""
Tests the find method with a timeframe and an 'AND' joiner.
"""
field_query = EngineQuery(self.fieldsets, 'AND')
timeframe = self.timeframe
subqueries = [field_query] + timeframe
query = EngineQuery(subqueries, 'AND')
results = self.engine.find(query)
docs = results['results']
count = results['count']
self.assertEqual(count, 1)
self.assertEqual(len(docs), 1)
def test_find_tf_w_tf_or(self):
"""
Tests the find method with a timeframe and an 'OR' joiner.
"""
field_query = EngineQuery(self.fieldsets, 'OR')
timeframe = self.timeframe
subqueries = [field_query] + timeframe
query = EngineQuery(subqueries, 'AND')
results = self.engine.find(query)
docs = results['results']
count = results['count']
self.assertEqual(count, 2)
self.assertEqual(len(docs), 2)
def test_find_tf_start_time_or(self):
"""
Tests the find method with an endtime and an 'OR' joiner.
"""
field_query = EngineQuery(self.fieldsets, 'OR')
timeframe = [
QueryFieldset(field_name='_saved_date',
field_type='DateTimeField',
operator='gte',
value=self.time - timedelta(days=2))
]
subqueries = [field_query] + timeframe
query = EngineQuery(subqueries, 'AND')
results = self.engine.find(query)
docs = results['results']
count = results['count']
self.assertEqual(count, 3)
self.assertEqual(len(docs), 3)
def test_find_tf_end_time_or(self):
"""
Tests the find method with an endtime and an 'OR' joiner.
"""
field_query = EngineQuery(self.fieldsets, 'OR')
timeframe = [
QueryFieldset(field_name='_saved_date',
field_type='DateTimeField',
operator='gte',
value=self.time + timedelta(hours=1))
]
subqueries = [field_query] + timeframe
query = EngineQuery(subqueries, 'AND')
results = self.engine.find(query)
docs = results['results']
count = results['count']
self.assertEqual(count, 1)
self.assertEqual(len(docs), 1)
def test_find_pagination_sort(self):
"""
Tests pagination and sorting of find results.
"""
sorter = Sorter([
SortParam(field_name='user.age',
field_type='IntegerField',
order='DESC'),
SortParam(field_name='user.screen_name',
field_type='CharField',
order='ASC')
])
field_query = EngineQuery(self.fieldsets, 'OR')
results = self.engine.find(query=field_query,
sorter=sorter,
page=1,
page_size=2)
docs = results['results']
count = results['count']
self.assertEqual(count, 3)
self.assertEqual(len(docs), 2)
self.assertEqual(docs[0]['user']['screen_name'], 'jack')
results = self.engine.find(query=field_query,
sorter=sorter,
page=2,
page_size=2)
docs = results['results']
count = results['count']
self.assertEqual(count, 3)
self.assertEqual(len(docs), 1)
self.assertEqual(docs[0]['user']['screen_name'], 'john')
def test_filter_ids_analyzed(self):
"""
Tests the filter_ids method.
"""
id_0 = self.engine.insert(self.test_docs[0])
id_1 = self.engine.insert(self.test_docs[1])
id_2 = self.engine.insert(self.test_docs[2])
ids = [id_0, id_1, id_2]
actual = self.engine.filter_ids(
doc_ids=ids,
fields=[
DataField(field_name='content.text', field_type='TextField'),
DataField(field_name='content.tags', field_type='ListField')
],
value='CATS')
expected = [id_0, id_2]
actual.sort()
expected.sort()
self.assertEqual(actual, expected)
def test_filter_ids_not_analyzed(self):
"""
Tests the filter_ids method for a mixture of exact-text and
full-text fields in ELasticsearch.
"""
id_0 = self.engine.insert(self.test_docs[0])
id_1 = self.engine.insert(self.test_docs[1])
id_2 = self.engine.insert(self.test_docs[2])
ids = [id_0, id_1, id_2]
actual = self.engine.filter_ids(doc_ids=ids,
fields=[
DataField(field_name='user.link',
field_type='URLField'),
DataField(field_name='user.email',
field_type='EmailField')
],
value='example')
expected = [id_0, id_1, id_2]
actual.sort()
expected.sort()
self.assertEqual(actual, expected)
def test_filter_ids_mixed(self):
"""
Tests the filter_ids method for a mixture of exact-text and
full-text fields in ELasticsearch.
"""
id_0 = self.engine.insert(self.test_docs[0])
id_1 = self.engine.insert(self.test_docs[1])
id_2 = self.engine.insert(self.test_docs[2])
ids = [id_0, id_1, id_2]
actual = self.engine.filter_ids(doc_ids=ids,
fields=[
DataField(
field_name='content.text',
field_type='TextField'),
DataField(field_name='user.email',
field_type='EmailField')
],
value='example')
expected = [id_0, id_1, id_2]
actual.sort()
expected.sort()
self.assertEqual(actual, expected)