def __setupIndexes(self, chunk_size):
'''
Sets up RediSearch indexes for chromosomes and genes.
Parameters:
chunk_size (int): The chunk size to be used for Redis batch processing.
Returns:
redis.commands.search.Search.BatchIndexer: A batch processor for the chromosome
index.
redis.commands.search.Search.BatchIndexer: A batch processor for the gene index.
'''
# create the chromosome index
chromosome_fields = [
TextField('name'),
NumericField('length'),
TextField('genus'),
TextField('species'),
]
chromosome_definition = IndexDefinition(prefix=['chromosome:'])
chromosome_indexer = \
self.__makeOrGetIndex(
CHROMOSOME_INDEX_NAME,
chromosome_fields,
chromosome_definition,
chunk_size,
)
# check if any non-RediSearch chromosome keys exist, and drop if necessary
self.__checkChromosomeKeys()
# create the gene index
gene_fields = [
TextField('chromosome'),
TextField('name'),
NumericField('fmin'),
NumericField('fmax'),
TextField('family'),
NumericField('strand'),
NumericField('index', sortable=True),
]
gene_definition = IndexDefinition(prefix=['gene:'])
gene_indexer = \
self.__makeOrGetIndex(
GENE_INDEX_NAME,
gene_fields,
gene_definition,
chunk_size,
)
# set the schema version and compatible versions
self.redis_connection.set(VERSION_KEY, redis_loader.__schema_version__)
self.redis_connection.delete(COMPATIBLE_KEY)
self.redis_connection.sadd(
COMPATIBLE_KEY, *redis_loader.__compatible_schema_versions__)
return chromosome_indexer, gene_indexer
def test_json_with_jsonpath(client):
definition = IndexDefinition(index_type=IndexType.JSON)
client.ft().create_index(
(
TextField('$["prod:name"]', as_name="name"),
TextField("$.prod:name", as_name="name_unsupported"),
),
definition=definition,
)
client.json().set("doc:1", Path.rootPath(), {"prod:name": "RediSearch"})
# query for a supported field succeeds
res = client.ft().search(Query("@name:RediSearch"))
assert res.total == 1
assert res.docs[0].id == "doc:1"
assert res.docs[0].json == '{"prod:name":"RediSearch"}'
# query for an unsupported field fails
res = client.ft().search("@name_unsupported:RediSearch")
assert res.total == 0
# return of a supported field succeeds
res = client.ft().search(Query("@name:RediSearch").return_field("name"))
assert res.total == 1
assert res.docs[0].id == "doc:1"
assert res.docs[0].name == "RediSearch"
# return of an unsupported field fails
res = client.ft().search(
Query("@name:RediSearch").return_field("name_unsupported"))
assert res.total == 1
assert res.docs[0].id == "doc:1"
with pytest.raises(Exception):
res.docs[0].name_unsupported
def test_json_with_multipath(client):
"""
Create definition with IndexType.JSON as index type (ON JSON),
and use json client to test it.
"""
definition = IndexDefinition(prefix=["king:"], index_type=IndexType.JSON)
client.ft().create_index((TagField("$..name", as_name="name")),
definition=definition)
client.json().set("king:1", Path.rootPath(), {
"name": "henry",
"country": {
"name": "england"
}
})
res = client.ft().search("@name:{henry}")
assert res.docs[0].id == "king:1"
assert res.docs[0].json == '{"name":"henry","country":{"name":"england"}}'
assert res.total == 1
res = client.ft().search("@name:{england}")
assert res.docs[0].id == "king:1"
assert res.docs[0].json == '{"name":"henry","country":{"name":"england"}}'
assert res.total == 1
def test_create_json_with_alias(client):
"""
Create definition with IndexType.JSON as index type (ON JSON) with two
fields with aliases, and use json client to test it.
"""
definition = IndexDefinition(prefix=["king:"], index_type=IndexType.JSON)
client.ft().create_index(
(TextField("$.name",
as_name="name"), NumericField("$.num", as_name="num")),
definition=definition,
)
client.json().set("king:1", Path.rootPath(), {"name": "henry", "num": 42})
client.json().set("king:2", Path.rootPath(), {
"name": "james",
"num": 3.14
})
res = client.ft().search("@name:henry")
assert res.docs[0].id == "king:1"
assert res.docs[0].json == '{"name":"henry","num":42}'
assert res.total == 1
res = client.ft().search("@num:[0 10]")
assert res.docs[0].id == "king:2"
assert res.docs[0].json == '{"name":"james","num":3.14}'
assert res.total == 1
# Tests returns an error if path contain special characters (user should
# use an alias)
with pytest.raises(Exception):
client.ft().search("@$.name:henry")
def test_search_return_fields(client):
res = client.json().set(
"doc:1",
Path.rootPath(),
{
"t": "riceratops",
"t2": "telmatosaurus",
"n": 9072,
"flt": 97.2
},
)
assert res
# create index on
definition = IndexDefinition(index_type=IndexType.JSON)
SCHEMA = (
TextField("$.t"),
NumericField("$.flt"),
)
client.ft().create_index(SCHEMA, definition=definition)
waitForIndex(client, "idx")
total = client.ft().search(Query("*").return_field("$.t",
as_field="txt")).docs
assert 1 == len(total)
assert "doc:1" == total[0].id
assert "riceratops" == total[0].txt
total = client.ft().search(
Query("*").return_field("$.t2", as_field="txt")).docs
assert 1 == len(total)
assert "doc:1" == total[0].id
assert "telmatosaurus" == total[0].txt
def test_index_definition(client):
"""
Create definition and test its args
"""
with pytest.raises(RuntimeError):
IndexDefinition(prefix=["hset:", "henry"], index_type="json")
definition = IndexDefinition(
prefix=["hset:", "henry"],
filter="@f1==32",
language="English",
language_field="play",
score_field="chapter",
score=0.5,
payload_field="txt",
index_type=IndexType.JSON,
)
assert [
"ON",
"JSON",
"PREFIX",
2,
"hset:",
"henry",
"FILTER",
"@f1==32",
"LANGUAGE_FIELD",
"play",
"LANGUAGE",
"English",
"SCORE_FIELD",
"chapter",
"SCORE",
0.5,
"PAYLOAD_FIELD",
"txt",
] == definition.args
createIndex(client.ft(), num_docs=500, definition=definition)
def test_alias():
index1 = getClient()
index2 = getClient()
def1 = IndexDefinition(prefix=["index1:"])
def2 = IndexDefinition(prefix=["index2:"])
ftindex1 = index1.ft("testAlias")
ftindex2 = index2.ft("testAlias2")
ftindex1.create_index((TextField("name"), ), definition=def1)
ftindex2.create_index((TextField("name"), ), definition=def2)
index1.hset("index1:lonestar", mapping={"name": "lonestar"})
index2.hset("index2:yogurt", mapping={"name": "yogurt"})
res = ftindex1.search("*").docs[0]
assert "index1:lonestar" == res.id
# create alias and check for results
ftindex1.aliasadd("spaceballs")
alias_client = getClient().ft("spaceballs")
res = alias_client.search("*").docs[0]
assert "index1:lonestar" == res.id
# Throw an exception when trying to add an alias that already exists
with pytest.raises(Exception):
ftindex2.aliasadd("spaceballs")
# update alias and ensure new results
ftindex2.aliasupdate("spaceballs")
alias_client2 = getClient().ft("spaceballs")
res = alias_client2.search("*").docs[0]
assert "index2:yogurt" == res.id
ftindex2.aliasdel("spaceballs")
with pytest.raises(Exception):
alias_client2.search("*").docs[0]
def test_create_client_definition(client):
"""
Create definition with no index type provided,
and use hset to test the client definition (the default is HASH).
"""
definition = IndexDefinition(prefix=["hset:", "henry"])
createIndex(client.ft(), num_docs=500, definition=definition)
info = client.ft().info()
assert 494 == int(info["num_docs"])
client.ft().client.hset("hset:1", "f1", "v1")
info = client.ft().info()
assert 495 == int(info["num_docs"])
def test_drop_index():
"""
Ensure the index gets dropped by data remains by default
"""
for x in range(20):
for keep_docs in [[True, {}], [False, {"name": "haveit"}]]:
idx = "HaveIt"
index = getClient()
index.hset("index:haveit", mapping={"name": "haveit"})
idef = IndexDefinition(prefix=["index:"])
index.ft(idx).create_index((TextField("name"), ), definition=idef)
waitForIndex(index, idx)
index.ft(idx).dropindex(delete_documents=keep_docs[0])
i = index.hgetall("index:haveit")
assert i == keep_docs[1]
def test_create_client_definition_hash(client):
"""
Create definition with IndexType.HASH as index type (ON HASH),
and use hset to test the client definition.
"""
definition = IndexDefinition(prefix=["hset:", "henry"],
index_type=IndexType.HASH)
createIndex(client.ft(), num_docs=500, definition=definition)
info = client.ft().info()
assert 494 == int(info["num_docs"])
client.ft().client.hset("hset:1", "f1", "v1")
info = client.ft().info()
assert 495 == int(info["num_docs"])
def initialize():
try:
definition = IndexDefinition(prefix=["fts:"])
redis_conn.ft().create_index(
(
TextField("id"),
TextField("name"),
TextField("summary"),
TextField("description"),
TextField("keywords"),
),
definition=definition,
)
except:
pass
def test_create_client_definition_json(client):
"""
Create definition with IndexType.JSON as index type (ON JSON),
and use json client to test it.
"""
definition = IndexDefinition(prefix=["king:"], index_type=IndexType.JSON)
client.ft().create_index((TextField("$.name"), ), definition=definition)
client.json().set("king:1", Path.rootPath(), {"name": "henry"})
client.json().set("king:2", Path.rootPath(), {"name": "james"})
res = client.ft().search("henry")
assert res.docs[0].id == "king:1"
assert res.docs[0].payload is None
assert res.docs[0].json == '{"name":"henry"}'
assert res.total == 1
def test_syndump(client):
definition = IndexDefinition(index_type=IndexType.HASH)
client.ft().create_index(
(
TextField("title"),
TextField("body"),
),
definition=definition,
)
client.ft().synupdate("id1", False, "boy", "child", "offspring")
client.ft().synupdate("id2", False, "baby", "child")
client.ft().synupdate("id3", False, "tree", "wood")
res = client.ft().syndump()
assert res == {
"boy": ["id1"],
"tree": ["id3"],
"wood": ["id3"],
"child": ["id1", "id2"],
"baby": ["id2"],
"offspring": ["id1"],
}
def test_fields_as_name(client):
# create index
SCHEMA = (
TextField("$.name", sortable=True, as_name="name"),
NumericField("$.age", as_name="just_a_number"),
)
definition = IndexDefinition(index_type=IndexType.JSON)
client.ft().create_index(SCHEMA, definition=definition)
# insert json data
res = client.json().set("doc:1", Path.rootPath(), {
"name": "Jon",
"age": 25
})
assert res
total = client.ft().search(
Query("Jon").return_fields("name", "just_a_number")).docs
assert 1 == len(total)
assert "doc:1" == total[0].id
assert "Jon" == total[0].name
assert "25" == total[0].just_a_number
def test_synupdate(client):
definition = IndexDefinition(index_type=IndexType.HASH)
client.ft().create_index(
(
TextField("title"),
TextField("body"),
),
definition=definition,
)
client.ft().synupdate("id1", True, "boy", "child", "offspring")
client.ft().add_document("doc1",
title="he is a baby",
body="this is a test")
client.ft().synupdate("id1", True, "baby")
client.ft().add_document("doc2",
title="he is another baby",
body="another test")
res = client.ft().search(Query("child").expander("SYNONYM"))
assert res.docs[0].id == "doc2"
assert res.docs[0].title == "he is another baby"
assert res.docs[0].body == "another test"
