def test_parents_tracking_no_tracking(self):
tree = AndOperation(Word("foo"), Phrase('"bar"'))
# no parents tracking !
visitor = self.TrackingParentsVisitor()
nodes = visitor.visit(tree)
self.assertListEqual([(tree, None), (Word("foo"), None),
(Phrase('"bar"'), None)], nodes)
def test_phrase(self):
tree = (AndOperation(
Phrase('"a phrase (AND a complicated~ one)"', tail=" "),
Phrase('"Another one"', head=" ")))
parsed = parser.parse(
'"a phrase (AND a complicated~ one)" AND "Another one"')
self.assertEqual(str(parsed), str(tree))
self.assertEqual(parsed, tree)
def test_approx(self):
tree = (UnknownOperation(
Proximity(Phrase('"foo bar"'), 3, tail=" "),
Proximity(Phrase('"foo baz"'), None, tail=" "),
Fuzzy(Word('baz'), Decimal("0.3"), tail=" "),
Fuzzy(Word('fou'), None)))
parsed = parser.parse('"foo bar"~3 "foo baz"~ baz~0.3 fou~')
self.assertEqual(str(parsed), str(tree))
self.assertEqual(parsed, tree)
def test_removal(self):
tree = (AndOperation(AndOperation(Word("foo"), Phrase('"bar"')),
AndOperation(Phrase('"baz"'), Phrase('"biz"'))))
transformer = self.BasicTransformer()
new_tree = transformer.visit(tree)
self.assertEqual(
new_tree,
(AndOperation(AndOperation(Word("lol")), AndOperation())))
def test_auto_name_nested(self):
tree = AndOperation(
OrOperation(
SearchField("bar", Word("test")),
AndOperation(
Proximity(Phrase('"test"'), 2),
SearchField("baz", Word("test")),
),
),
Group(UnknownOperation(
Fuzzy(Word("test")),
Phrase('"test"'),
), ),
)
names = auto_name(tree)
self.assertEqual(sorted(names.keys()), list("abcdefgh"))
# and
and1 = tree
self.assertEqual(get_name(and1), None)
# - or
or1 = and1.children[0]
self.assertEqual(get_name(or1), "a")
self.assertEqual(names["a"], (0, ))
# -- search field
sfield1 = or1.children[0]
self.assertEqual(get_name(sfield1), "c")
self.assertEqual(names["c"], (0, 0))
self.assertEqual(get_name(sfield1.expr), None)
# -- and
and2 = or1.children[1]
self.assertEqual(get_name(and2), "d")
self.assertEqual(names["d"], (0, 1))
# --- proximity phrase
self.assertEqual(get_name(and2.children[0]), "e")
self.assertEqual(names["e"], (0, 1, 0))
self.assertEqual(get_name(and2.children[0].term), None)
# --- search field
sfield2 = and2.children[1]
self.assertEqual(get_name(sfield2), "f")
self.assertEqual(names["f"], (0, 1, 1))
self.assertEqual(get_name(sfield2.expr), None)
# - group
group1 = and1.children[1]
self.assertEqual(get_name(group1), "b")
self.assertEqual(names["b"], (1, ))
# -- unknown op
unknownop1 = group1.children[0]
self.assertEqual(get_name(unknownop1), None)
# --- fuzzy word
self.assertEqual(get_name(unknownop1.children[0]), "g")
self.assertEqual(names["g"], (1, 0, 0))
self.assertEqual(get_name(unknownop1.children[0].term), None)
# --- phrase
self.assertEqual(get_name(unknownop1.children[1]), "h")
self.assertEqual(names["h"], (1, 0, 1))
def test_operands_list(self):
OrListOperation = self.OrListOperation
tree = (OrListOperation(OrListOperation(Word("foo"), Phrase('"bar"')),
OrListOperation(Phrase('"baz"'))))
transformer = self.BasicTransformer()
new_tree = transformer.visit(tree)
self.assertEqual(
new_tree,
(OrListOperation(OrListOperation(Word("lol")), OrListOperation())))
def test_parents_tracking(self):
tree = AndOperation(Word("foo"), Proximity(Phrase('"bar"'), 2))
visitor = self.TrackingParentsVisitor(track_parents=True)
nodes = visitor.visit(tree)
self.assertListEqual(
[
(tree, None),
(Word("foo"), (tree, )),
(Proximity(Phrase('"bar"'), degree=2), (tree, )),
(Phrase('"bar"'), (tree, Proximity(Phrase('"bar"'), 2))),
],
nodes,
)
def test_named_queries_match(self):
tree = SearchField("spam", Word("bar"))
set_name(tree, "a")
result = self.transformer(tree)
self.assertEqual(
result,
{
"match": {
"spam": {
"query": "bar",
"_name": "a",
"zero_terms_query": "none",
},
},
},
)
tree = SearchField("spam", Phrase('"foo bar"'))
set_name(tree, "a")
result = self.transformer(tree)
self.assertEqual(
result,
{
"match_phrase": {
"spam": {
"query": "foo bar",
"_name": "a",
},
},
},
)
def test_and_operation(self):
tree = AndOperation(Word("foo"), Phrase('"bar"'), Word("baz"))
all_paths = {(0, ), (1, ), (2, )}
matching = set()
paths_ok, paths_ko = self.propagate_matching(tree, matching,
all_paths - matching)
self.assertEqual(paths_ok, set())
self.assertEqual(paths_ko, {
(),
(0, ),
(1, ),
(2, ),
})
matching = {(2, )}
paths_ok, paths_ko = self.propagate_matching(tree, matching,
all_paths - matching)
self.assertEqual(paths_ok, {(2, )})
self.assertEqual(paths_ko, {(), (0, ), (1, )})
matching = {(0, ), (2, )}
paths_ok, paths_ko = self.propagate_matching(tree, matching,
all_paths - matching)
self.assertEqual(paths_ok, {(0, ), (2, )})
self.assertEqual(paths_ko, {(), (1, )})
matching = {(0, ), (1, ), (2, )}
paths_ok, paths_ko = self.propagate_matching(tree, matching,
all_paths - matching)
self.assertEqual(paths_ok, {(), (0, ), (1, ), (2, )})
self.assertEqual(paths_ko, set())
def test_set_children_raises(self):
test = self._test_set_children_raises
test(Word("foo"), [Word("foo")])
test(Phrase('"foo"'), [Word("foo")])
test(Regex("/foo/"), [Word("foo")])
test(SearchField("foo", Word("bar")), [])
test(SearchField("foo", Word("bar")), [Word("bar"), Word("baz")])
test(Group(Word("foo")), [])
test(Group(Word("foo")), [Word("foo"), Word("bar")])
test(FieldGroup(Word("foo")), [])
test(FieldGroup(Word("foo")), [Word("foo"), Word("bar")])
test(Range(Word("20"), Word("30")), [])
test(Range(Word("20"), Word("30")),
[Word("20"), Word("30"), Word("40")])
test(Proximity(Word("foo")), [])
test(Proximity(Word("foo")), [Word("foo"), Word("bar")])
test(Fuzzy(Word("foo")), [])
test(Fuzzy(Word("foo")), [Word("foo"), Word("bar")])
test(Boost(Word("foo"), force=1), [])
test(Boost(Word("foo"), force=1), [Word("foo"), Word("bar")])
test(Plus(Word("foo")), [])
test(Plus(Word("foo")), [Word("foo"), Word("bar")])
test(Not(Word("foo")), [])
test(Not(Word("foo")), [Word("foo"), Word("bar")])
test(Prohibit(Word("foo")), [])
test(Prohibit(Word("foo")), [Word("foo"), Word("bar")])
def parsePass1(self, t, qctx, parent):
t.parent = parent
t.qctx = qctx
# Walk the tree
if isinstance(t, Group):
# Share the qctx
for c in t.children:
self.parsePass1(c, qctx, t)
elif isinstance(t, BaseOperation):
# collapse multiple sibling Word's into one Phrase,
# but only multiple Word's, not a mix
if isinstance(t, UnknownOperation):
words = []
for c in t.children:
if isinstance(c, Word):
words.append(c)
else:
break
else:
w = " ".join([w.value for w in words])
t.operands = [Phrase(f'"{w}"')]
newctx = isinstance(t, OrOperation)
for c in t.children:
self.parsePass1(c, QueryContext() if newctx else qctx, t)
elif isinstance(t, SearchField):
if t.name == "artist":
qctx.artist = t.expr.value.strip('"')
elif t.name == "album":
qctx.album = t.expr.value.strip('"')
elif t.name == "title" or t.name == "track":
qctx.track = t.expr.value.strip('"')
else:
raise MissingParam("Invalid search field: " + t.name)
elif isinstance(t, Term):
qctx.any = t.value.strip('"')
def test_named_queries_term(self):
tree = SearchField("text", Word("bar"))
set_name(tree, "a")
result = self.transformer(tree)
self.assertEqual(
result,
{"term": {
"text": {
"value": "bar",
"_name": "a"
}
}},
)
tree = SearchField("text", Phrase('"foo bar"'))
set_name(tree, "a")
result = self.transformer(tree)
self.assertEqual(
result,
{"term": {
"text": {
"value": "foo bar",
"_name": "a"
}
}},
)
def test_auto_name_nested(self):
tree = AndOperation(
OrOperation(
SearchField("bar", Word("test")),
AndOperation(
Proximity(Phrase('"test"'), 2),
SearchField("baz", Word("test")),
),
),
Group(
UnknownOperation(
Fuzzy(Word("test")),
Phrase('"test"'),
),
),
)
auto_name(tree)
# and
and1 = tree
self.assertEqual(get_name(and1), "0")
# - or
or1 = and1.children[0]
self.assertEqual(get_name(or1), "0_0")
# --- search field word
sfield1 = or1.children[0]
self.assertFalse(get_name(sfield1))
self.assertEqual(get_name(sfield1.expr), "0_0_0")
# --- and
and2 = or1.children[1]
self.assertEqual(get_name(and2), "0_0_1")
# ----- proximity phrase
self.assertEqual(get_name(and2.children[0].term), "0_0_1_0")
# ----- search field word
sfield2 = and2.children[1]
self.assertFalse(get_name(sfield2))
self.assertEqual(get_name(sfield2.expr), "0_0_1_1")
# - group
group1 = and1.children[1]
self.assertEqual(get_name(group1), None)
# --- unknown op
unknownop1 = group1.children[0]
self.assertEqual(get_name(unknownop1), "0_1")
# ----- fuzzy word
self.assertEqual(get_name(unknownop1.children[0].term), "0_1_0")
# ----- phrase
self.assertEqual(get_name(unknownop1.children[1]), "0_1_1")
def visit_phrase(self, node: Phrase, parents: List[Item],
context: SQLQueryBuilderContext) -> Phrase:
"""
Visitor for Phrase. Phrases are enquoted Terms.
"""
# Strip the " from start and end
node.value = node.value[1:-1]
return self.visit_term(node, parents, context)
def test_escaping_phrase(self):
query = r'"test \"phrase"'
tree = Phrase(query)
unescaped = '"test "phrase"'
parsed = parser.parse(query)
self.assertEqual(str(parsed), query)
self.assertEqual(parsed, tree)
self.assertEqual(parsed.unescaped_value, unescaped)
def test_phrase(self):
tree = parser.parse('\t"foo bar"\r')
self.assertEqual(tree, Phrase('"foo bar"'))
self.assertEqual(tree.head, "\t")
self.assertEqual(tree.tail, "\r")
self.assertEqual(tree.pos, 1)
self.assertEqual(tree.size, 10)
self.assertEqual(str(tree), '"foo bar"')
self.assertEqual(tree.__str__(head_tail=True), '\t"foo bar"\r')
def test_single_element(self):
for tree in [Word("a"), Phrase('"a"'), Regex("/a/")]:
with self.subTest("%r" % type(tree)):
paths_ok, paths_ko = self.propagate_matching(tree, set())
self.assertEqual(paths_ok, set(), {()})
self.assertEqual(paths_ko, {()})
paths_ok, paths_ko = self.propagate_matching(tree, {()})
self.assertEqual(paths_ok, {()}, set())
self.assertEqual(paths_ko, set())
def test_name_index_simple_term(self):
tree = Word("bar")
set_name(tree, "0")
self.assertEqual(name_index(tree), {"0": (0, len(str(tree)))})
phrase = Phrase('"baz"')
tree = Group(phrase)
set_name(phrase, "0")
self.assertEqual(name_index(tree), {"0": (1, len(str(phrase)))})
set_name(phrase, "funny")
self.assertEqual(name_index(tree), {"funny": (1, len(str(phrase)))})
def test_boost(self):
tree = (UnknownOperation(
Boost(Phrase('"foo bar"'), Decimal("3.0"), tail=" "),
Boost(
Group(
AndOperation(Word('baz', tail=" "), Word('bar',
head=" "))),
Decimal("2.1"))))
parsed = parser.parse('"foo bar"^3 (baz AND bar)^2.1')
self.assertEqual(str(parsed), str(tree))
self.assertEqual(parsed, tree)
def test_auto_name_simple_op(self):
for OpCls in AndOperation, OrOperation, UnknownOperation:
with self.subTest("operation %r" % OpCls):
tree = OpCls(
Word("test"),
Phrase('"test"'),
)
auto_name(tree)
self.assertEqual(get_name(tree), "0")
self.assertEqual(get_name(tree.children[0]), "0_0")
self.assertEqual(get_name(tree.children[1]), "0_1")
def test_visit_complex(self):
tree = AndOperation(
Group(
OrOperation(Word("foo"), Word("bar"),
Boost(Fuzzy(Word("baz")), force=2))),
Proximity(Phrase('"spam ham"')),
SearchField("fizz", Regex("/fuzz/")),
)
transformed = self.transform(tree)
expected = AndOperation(
Group(
OrOperation(
Word("foo@0-0-0"),
Word("bar@0-0-1"),
Boost(Fuzzy(Word("baz@0-0-2-0-0")), force=2),
)),
Proximity(Phrase('"spam ham@1-0"')),
SearchField("fizz", Regex("/fuzz@2-0/")),
)
self.assertEqual(transformed, expected)
def test_auto_name_one_term(self):
tree = Word("test")
auto_name(tree)
self.assertEqual(get_name(tree), "0")
tree = Phrase('"test"')
auto_name(tree)
self.assertEqual(get_name(tree), "0")
tree = Range("test", "*")
auto_name(tree)
self.assertEqual(get_name(tree), "0")
def test_auto_name_simple_op(self):
for OpCls in AndOperation, OrOperation, UnknownOperation:
with self.subTest("operation %r" % OpCls):
tree = OpCls(
Word("test"),
Phrase('"test"'),
)
names = auto_name(tree)
self.assertEqual(get_name(tree), None)
self.assertEqual(get_name(tree.children[0]), "a")
self.assertEqual(get_name(tree.children[1]), "b")
self.assertEqual(names, {"a": (0, ), "b": (1, )})
def test_unknown_operation(self):
tree = UnknownOperation(Word("foo"), Phrase('"bar"'), Word("baz"))
tree_or = OrOperation(Word("foo"), Phrase('"bar"'), Word("baz"))
tree_and = AndOperation(Word("foo"), Phrase('"bar"'), Word("baz"))
propagate_or = self.propagate_matching
propagate_and = MatchingPropagator(default_operation=AndOperation)
all_paths = {(0, ), (1, ), (2, )}
for matching in [
set(), {(2, )}, {(0, ), (2, )}, {(0, ), (1, ), (2, )}
]:
self.assertEqual(
propagate_or(tree, matching),
self.propagate_matching(tree_or, matching,
matching - all_paths),
)
self.assertEqual(
propagate_and(tree, matching),
self.propagate_matching(tree_and, matching,
matching - all_paths),
)
def test_named_queries_complex(self):
tree = (AndOperation(
SearchField("text", Phrase('"foo bar"')),
Group(OrOperation(
Word("bar"),
SearchField("spam", Word("baz")),
), ),
))
and_op = tree
search_text = and_op.operands[0]
or_op = and_op.operands[1].children[0]
bar = or_op.operands[0]
search_spam = or_op.operands[1]
set_name(search_text, "foo_bar")
set_name(bar, "bar")
set_name(search_spam, "baz")
expected = {
'bool': {
'must': [{
'term': {
'text': {
'_name': 'foo_bar',
'value': 'foo bar'
}
}
}, {
'bool': {
'should': [{
'term': {
'text': {
'_name': 'bar',
'value': 'bar'
}
}
}, {
'match': {
'spam': {
'_name': 'baz',
'query': 'baz',
'zero_terms_query': 'none'
}
}
}]
}
}]
}
}
result = self.transformer(tree)
self.assertEqual(result, expected)
def test_proximity(self):
tree = parser.parse('\r"foo"\t~2\n')
self.assertEqual(tree, Proximity(Phrase('"foo"'), 2))
self.assertEqual(tree.head, "")
self.assertEqual(tree.tail, "\n")
self.assertEqual(tree.pos, 0)
self.assertEqual(tree.size, 9)
foo, = tree.children
self.assertEqual(foo.head, "\r")
self.assertEqual(foo.tail, "\t")
self.assertEqual(foo.pos, 1)
self.assertEqual(foo.size, 5)
self.assertEqual(str(tree), '\r"foo"\t~2')
self.assertEqual(tree.__str__(head_tail=True), '\r"foo"\t~2\n')
def test_element_from_path(self):
tree = AndOperation(
OrOperation(
SearchField("bar", Word("test")),
Group(
AndOperation(
Proximity(Phrase('"test"'), 2),
SearchField("baz", Word("test")),
Fuzzy(Word("test")),
Phrase('"test"'),
), ),
), )
names = {
"a": (),
"b": (0, 1),
"c": (0, 1, 0, 2),
"d": (0, 1, 0, 2, 0),
"e": (0, 1, 0, 3)
}
self.assertEqual(element_from_path(tree, ()), tree)
self.assertEqual(element_from_name(tree, "a", names), tree)
self.assertEqual(element_from_path(tree, (0, 1)),
tree.children[0].children[1])
self.assertEqual(element_from_name(tree, "b", names),
tree.children[0].children[1])
self.assertEqual(element_from_path(tree, (0, 1, 0, 2)),
Fuzzy(Word("test")))
self.assertEqual(element_from_name(tree, "c", names),
Fuzzy(Word("test")))
self.assertEqual(element_from_path(tree, (0, 1, 0, 2, 0)),
Word("test"))
self.assertEqual(element_from_name(tree, "d", names), Word("test"))
self.assertEqual(element_from_path(tree, (0, 1, 0, 3)),
Phrase('"test"'))
self.assertEqual(element_from_name(tree, "e", names), Phrase('"test"'))
with self.assertRaises(IndexError):
element_from_path(tree, (1, ))
def test_check_ok(self):
query = (AndOperation(
SearchField(
"f",
FieldGroup(
AndOperation(
Boost(Proximity(Phrase('"foo bar"'), 4), "4.2"),
Prohibit(Range("100", "200"))))),
Group(OrOperation(Fuzzy(Word("baz"), ".8"), Plus(Word("fizz"))))))
check = LuceneCheck()
self.assertTrue(check(query))
self.assertEqual(check.errors(query), [])
check = LuceneCheck(zeal=1)
self.assertTrue(check(query))
self.assertEqual(check.errors(query), [])
def test_named_queries_proximity(self):
tree = SearchField("spam", Proximity(Phrase('"foo bar"')))
set_name(tree.children[0], "a")
result = self.transformer(tree)
self.assertEqual(
result,
{
"match_phrase": {
"spam": {
"query": "foo bar",
"_name": "a",
'slop': 1.0
}
}
},
)
def test_named_queries_boost(self):
tree = SearchField("text", Boost(Phrase('"foo bar"'), force=2))
set_name(tree.children[0], "a")
result = self.transformer(tree)
self.assertEqual(
result,
{
"term": {
"text": {
"value": "foo bar",
"_name": "a",
'boost': 2.0
}
}
},
)