def test_redis_get_score_with_existing_term():
conn = Mock()
backend = suggestive.RedisBackend(conn=conn)
conn.hmget.return_value = [u'{"score": 117.0, "id": "botanical"}']
assert backend.get_score("botanical") == 117.0
backend.conn.hmget.assert_called_once_with(backend.keys.for_docs(),
"botanical")
def test_redis_get_score_with_no_existing_term():
conn = Mock()
backend = suggestive.RedisBackend(conn=conn)
conn.hmget.return_value = [None]
assert backend.get_score("hello") == 0
backend.conn.hmget.assert_called_once_with(backend.keys.for_docs(),
"hello")
def test_redis_backend_remove_items():
# Given that I have some indexed data
conn = Mock()
pipe = conn.pipeline.return_value
data = [
{
"id": 0,
"first_name": "Lincoln",
"last_name": "Clarete"
},
{
"id": 1,
"first_name": "Mingwei",
"last_name": "Gu"
},
{
"id": 2,
"first_name": "Livia",
"last_name": "C"
},
]
backend = suggestive.RedisBackend(conn=conn)
with patch.object(backend, 'remove'):
backend.index(data, field=['first_name', 'last_name'], score='id')
# Mocking the term X doc cache set
conn.smembers.return_value = (suggestive.expand('lincoln') +
suggestive.expand('clarete'))
# When I try to remove stuff
backend.remove(0)
# Then I see that the terms for this document were removed
list(pipe.zrem.call_args_list).should.equal([
call('suggestive:d:l', 0),
call('suggestive:d:li', 0),
call('suggestive:d:lin', 0),
call('suggestive:d:linc', 0),
call('suggestive:d:linco', 0),
call('suggestive:d:lincol', 0),
call('suggestive:d:lincoln', 0),
call('suggestive:d:c', 0),
call('suggestive:d:cl', 0),
call('suggestive:d:cla', 0),
call('suggestive:d:clar', 0),
call('suggestive:d:clare', 0),
call('suggestive:d:claret', 0),
call('suggestive:d:clarete', 0),
])
pipe.execute.call_count.should.equal(2)
# And the cache key should also be removed
conn.delete.assert_called_once_with('suggestive:dt:0')
# And I also see that we successfuly removed the document too
conn.hdel.assert_called_once_with('suggestive:d', 0)
def test_redis_backend_indexing(context):
# Given that I have an instance of our dummy backend
data = [{"id": 0, "name": "Lincoln"}, {"id": 1, "name": "Clarete"}]
backend = suggestive.RedisBackend(conn=context.conn)
# When I try to index stuff
indexed = backend.index(data, field='name', score='id')
# Then I see that the number of indexed items is right
indexed.should.equal(2)
def test_suggestive(context):
s = suggestive.Suggestive(backend=suggestive.RedisBackend(
conn=context.conn))
# Given that I have a source registered in my api
data = [
{
"id": 0,
"name": "Fafá de Belém",
"score": 23
},
{
"id": 1,
"name": "Fábio Júnior",
"score": 12.5
},
{
"id": 2,
"name": "Fábio",
"score": 20000
},
]
# When I index this source
s.index(data, field='name')
# Then I see that I can filter my results using the `term` parameter of the
# `suggest()` method
s.suggest('Faf').should.equal([
{
"id": 0,
"name": "Fafá de Belém",
"score": 23
},
])
# And I also see that I can sort my results
s.suggest('F').should.equal([
{
"id": 2,
"name": "Fábio",
"score": 20000
},
{
"id": 0,
"name": "Fafá de Belém",
"score": 23
},
{
"id": 1,
"name": "Fábio Júnior",
"score": 12.5
},
])
def test_redis_backend_indexing_multiple_fields():
# Given that I have an instance of our redis backend
conn = Mock()
pipe = conn.pipeline.return_value
data = [
{
"id": 0,
"first_name": "Lincoln",
"last_name": "Clarete"
},
{
"id": 1,
"first_name": "Mingwei",
"last_name": "Gu"
},
]
backend = suggestive.RedisBackend(conn=conn)
# When I try to index stuff
with patch.object(backend, 'remove'):
backend.index(data, field=['first_name', 'last_name'], score='id')
# And the term set was also fed
list(pipe.zadd.call_args_list).should.equal([
call('suggestive:d:l', 0, 0),
call('suggestive:d:li', 0, 0),
call('suggestive:d:lin', 0, 0),
call('suggestive:d:linc', 0, 0),
call('suggestive:d:linco', 0, 0),
call('suggestive:d:lincol', 0, 0),
call('suggestive:d:lincoln', 0, 0),
call('suggestive:d:c', 0, 0),
call('suggestive:d:cl', 0, 0),
call('suggestive:d:cla', 0, 0),
call('suggestive:d:clar', 0, 0),
call('suggestive:d:clare', 0, 0),
call('suggestive:d:claret', 0, 0),
call('suggestive:d:clarete', 0, 0),
call('suggestive:d:m', 1, 1),
call('suggestive:d:mi', 1, 1),
call('suggestive:d:min', 1, 1),
call('suggestive:d:ming', 1, 1),
call('suggestive:d:mingw', 1, 1),
call('suggestive:d:mingwe', 1, 1),
call('suggestive:d:mingwei', 1, 1),
call('suggestive:d:g', 1, 1),
call('suggestive:d:gu', 1, 1),
])
# And I also see that the pipeline was executed!
pipe.execute.assert_called_once_with()
def test_redis_backend_query_limit():
# Given that I have an instance of our dummy backend
data = [
{
"id": 0,
"name": "Lincoln"
},
{
"id": 1,
"name": "Livia"
},
{
"id": 2,
"name": "Linus"
},
{
"id": 3,
"name": "Lidia"
},
]
conn = Mock()
conn.hmget.return_value = []
backend = suggestive.RedisBackend(conn=conn)
with patch.object(backend, 'remove'):
backend.index(data, field='name', score='id')
# Then I see that limit and offset are working properly with different
# parameters
backend.query('li', limit=1, offset=0)
conn.zrevrange.assert_called_once_with('suggestive:d:li', 0, 1)
conn.reset_mock()
backend.query('li', limit=1, offset=1)
conn.zrevrange.assert_called_once_with('suggestive:d:li', 1, 2)
conn.reset_mock()
backend.query('li', limit=2, offset=1)
conn.zrevrange.assert_called_once_with('suggestive:d:li', 1, 3)
conn.reset_mock()
# And I also see that the limit param works without the offset
backend.query('li', limit=2)
conn.zrevrange.assert_called_once_with('suggestive:d:li', 0, 2)
conn.reset_mock()
# And I also see that the offset param works without the limit
backend.query('li', offset=2)
conn.zrevrange.assert_called_once_with('suggestive:d:li', 2, -1)
conn.reset_mock()
def test_both_backends_query_return_term_prefixed_words():
# Given that I have an instance of our dummy backend with some data indexed
data = [
{
"id": 0,
'field1': 'Pascal programming language',
'field2': 'Python'
},
{
"id": 1,
'field1': 'Italian Paníni',
'field2': 'Pizza Italiana'
},
{
"id": 2,
'field1': 'Pacific Ocean',
'field2': 'Posseidon, The king'
},
{
"id": 3,
'field1': 'Kiwi',
'field2': 'Passion-Fruit'
},
{
"id": 4,
'field1': 'I love',
'field2': 'Paníni'
},
]
conn = Mock()
conn.zrevrange.return_value = range(4)
conn.hmget.return_value = [json.dumps(item) for item in data]
dummy_backend = suggestive.DummyBackend()
dummy_backend.index(data, field=['field1', 'field2'], score='id')
redis_backend = suggestive.RedisBackend(conn=conn)
with patch.object(redis_backend, 'remove'):
redis_backend.index(data, field=['field1', 'field2'], score='id')
# When I query for the `Pa` prefix, asking for the words found in the
# documents
dummy_backend.query('pa', words=True).should.equal(
['Pascal', 'Paníni', 'Pacific', 'Passion-Fruit'])
redis_backend.query('pa', words=True).should.equal(
['Pascal', 'Paníni', 'Pacific', 'Passion-Fruit'])
suggestive.Suggestive(backend=dummy_backend).suggest(
'pa', words=True).should.equal(
['Pascal', 'Paníni', 'Pacific', 'Passion-Fruit'])
def test_redis_backend_querying_without_indexing():
conn = Mock()
backend = suggestive.RedisBackend(conn=conn)
# Given that I have an empty result set
conn.hmget.side_effect = Exception('hmget shouldn\'t be called!')
conn.zrevrange.return_value = []
# When I query for something that is not indexed
backend.query('li').should.equal([])
# Then I see that I look for the right term
conn.zrevrange.assert_called_once_with('suggestive:d:li', 0, -1)
# And since no document was found, we didn't try to get them from redis.
conn.hmget.called.should.be.false
def test_redis_backend_indexing_empty_fields():
# Given that I have an instance of our redis backend and some docs with
# empty fields
conn = Mock()
pipe = conn.pipeline.return_value
data = [{"id": 0, "name": ""}, {"id": 1, "name": "Clarete"}]
backend = suggestive.RedisBackend(conn=conn)
backend.remove = Mock() # We don't care about removing stuff here
# When I try to index stuff
backend.index(data, field='name', score='id')
# Then I see that the `sadd` command is not called when no terms are found
# inside of a field of a document
list(pipe.sadd.call_args_list).should.equal([
call('suggestive:dt:1', 'c', 'cl', 'cla', 'clar', 'clare', 'claret',
'clarete')
])
def test_redis_backend_cleaning_before_indexing():
# Given that I have an instance of our redis backend with some indexed data
conn = Mock()
data = [{"id": 0, "name": "Lincoln"}, {"id": 1, "name": "Clarete"}]
backend = suggestive.RedisBackend(conn=conn)
backend.remove = Mock()
backend.index(data, field='name', score='name')
# When I try to index the same documents but with different values
data = [{'id': 0, 'name': 'Mingwei'}]
backend.index(data, field='name', score='name')
# Then I see that the remove method was called for every single document
# that before indexing it
list(backend.remove.call_args_list).should.equal([
call(0),
call(1),
call(0),
])
def test_redis_backend_remove_items(context):
# Given that I have some indexed data
data = [
{
"id": 0,
"first_name": "Lincoln",
"last_name": "Clarete"
},
{
"id": 1,
"first_name": "Mingwei",
"last_name": "Gu"
},
{
"id": 2,
"first_name": "Livia",
"last_name": "C"
},
]
backend = suggestive.RedisBackend(conn=context.conn)
backend.index(data, field=['first_name', 'last_name'], score='id')
# When I try to remove stuff
backend.remove(0)
# Then I see that the terms that also occour in other docs are still
# there. But we got rid of the document 0
context.conn.zrange('suggestive:d:l', 0, -1).should.equal(['2'])
context.conn.zrange('suggestive:d:li', 0, -1).should.equal(['2'])
# And I see that the terms for this document were removed
context.conn.exists('suggestive:d:lin').should.be.false
context.conn.exists('suggestive:d:linc').should.be.false
context.conn.exists('suggestive:d:linco').should.be.false
context.conn.exists('suggestive:d:lincol').should.be.false
context.conn.exists('suggestive:d:lincoln').should.be.false
# And the cache key should also be removed
context.conn.exists('suggestive:dt:0').should.be.false
# And I also see that we successfuly removed the document too
context.conn.hexists('suggestive:d', 0).should.be.false
def test_redis_backend_cleaning_before_indexing(context):
# Given that I have an instance of our redis backend with some indexed data
data = [{"id": 0, "name": "Lincoln"}, {"id": 1, "name": "Clarete"}]
backend = suggestive.RedisBackend(conn=context.conn)
backend.index(data, field='name', score='id')
# When I try to index the same documents but with different values
data = [{'id': 0, 'name': 'Mingwei'}]
backend.index(data, field='name', score='id')
# Then I see that the remove method was called for every single document
# that before indexing it
context.conn.smembers(backend.keys.for_cache(0)).should.equal(
set([
'm',
'mi',
'min',
'ming',
'mingw',
'mingwe',
'mingwei',
]))
def test_redis_backend_querying(context):
data = [
{
"id": 0,
"name": "Lincoln",
"score": 33.3
},
{
"id": 1,
"name": "Livia",
"score": 22.2
},
{
"id": 5,
"name": "Linus",
"score": 25
}, # Rita's brother! :)
]
# Given that I have an instance of our Redis backend
backend = suggestive.RedisBackend(conn=context.conn)
backend.index(data, field='name') # We'll choose `score` by default
# When I try to query stuff sorting by score (defaults to asc), Then I see
# it worked
backend.query('li').should.equal([
{
"id": 0,
"name": 'Lincoln',
'score': 33.3
},
{
"id": 5,
"name": 'Linus',
'score': 25
},
{
"id": 1,
"name": 'Livia',
'score': 22.2
},
])
# And I see that the 'reversed' version also works
backend.query('li', reverse=True).should.equal([
{
"id": 1,
"name": 'Livia',
'score': 22.2
},
{
"id": 5,
"name": 'Linus',
'score': 25
},
{
"id": 0,
"name": 'Lincoln',
'score': 33.3
},
])
def test_redis_backend_querying():
conn = Mock()
data = [
{
"id": 0,
"name": "Lincoln",
"score": 33.3
},
{
"id": 1,
"name": "Livia",
"score": 22.2
},
{
"id": 5,
"name": "Linus",
"score": 25
}, # Rita's brother! :)
]
# Given that I have an instance of our Redis backend
backend = suggestive.RedisBackend(conn=conn)
with patch.object(backend, 'remove'):
backend.index(data, field='name') # We'll choose `score` by default
conn.zrevrange.return_value = ['1', '5', '0']
conn.zrange.return_value = ['0', '5', '1']
# When I try to query stuff sorting by score (defaults to asc), Then I see
# it worked
conn.hmget.return_value = [
'{"id": 0, "name": "Lincoln", "score": 33.3}',
'{"id": 5, "name": "Linus", "score": 25}',
'{"id": 1, "name": "Livia", "score": 22.2}',
]
backend.query('li').should.equal([
{
"id": 0,
"name": 'Lincoln',
'score': 33.3
},
{
"id": 5,
"name": 'Linus',
'score': 25
},
{
"id": 1,
"name": 'Livia',
'score': 22.2
},
])
conn.zrevrange.assert_called_once_with('suggestive:d:li', 0, -1)
# And I see that the 'reversed' version also works
conn.hmget.return_value = [
'{"id": 1, "name": "Livia", "score": 22.2}',
'{"id": 5, "name": "Linus", "score": 25}',
'{"id": 0, "name": "Lincoln", "score": 33.3}',
]
backend.query('li', reverse=True).should.equal([
{
"id": 1,
"name": 'Livia',
'score': 22.2
},
{
"id": 5,
"name": 'Linus',
'score': 25
},
{
"id": 0,
"name": 'Lincoln',
'score': 33.3
},
])
conn.zrange.assert_called_once_with('suggestive:d:li', 0, -1)
def test_redis_backend_indexing():
# Given that I have an instance of our redis backend
conn = Mock()
pipe = conn.pipeline.return_value
data = [{"id": 0, "name": "Lincoln"}, {"id": 1, "name": "Clarete"}]
backend = suggestive.RedisBackend(conn=conn)
backend.remove = Mock() # We don't care about removing stuff here
# When I try to index stuff
indexed = backend.index(data, field='name', score='id')
# Then I see that the number of indexed items is right
indexed.should.equal(2)
# And I see that the document set was fed
list(conn.hset.call_args_list).should.equal([
call('suggestive:d', 0, '{"id": 0, "name": "Lincoln"}'),
call('suggestive:d', 1, '{"id": 1, "name": "Clarete"}')
])
# And I see that we have a special stash to save to which terms each
# document was added (it will make deletions way easier)
list(pipe.sadd.call_args_list).should.equal([
call('suggestive:dt:0', *suggestive.expand('lincoln')),
call('suggestive:dt:1', *suggestive.expand('clarete')),
])
# And the term set was also fed
list(pipe.zadd.call_args_list).should.equal([
call('suggestive:d:l', 0, 0),
call('suggestive:d:li', 0, 0),
call('suggestive:d:lin', 0, 0),
call('suggestive:d:linc', 0, 0),
call('suggestive:d:linco', 0, 0),
call('suggestive:d:lincol', 0, 0),
call('suggestive:d:lincoln', 0, 0),
call('suggestive:d:c', 1, 1),
call('suggestive:d:cl', 1, 1),
call('suggestive:d:cla', 1, 1),
call('suggestive:d:clar', 1, 1),
call('suggestive:d:clare', 1, 1),
call('suggestive:d:claret', 1, 1),
call('suggestive:d:clarete', 1, 1)
])
# And that all the documents are indexed
conn.hgetall.return_value = {
'0': '{"id": 0, "name": "Lincoln"}',
'1': '{"id": 1, "name": "Clarete"}',
}
backend.documents().should.equal({
'0': {
u'id': 0,
u'name': u'Lincoln'
},
'1': {
u'id': 1,
u'name': u'Clarete'
}
})
conn.hgetall.assert_called_once_with('suggestive:d')
# And I also see that the pipeline was executed!
pipe.execute.assert_called_once_with()
