def testCoreNotQueryInOtherCore(self):
query, filters = self.convert(cqlToExpression('a=b NOT core2.x=y'),
'core1')
self.assertEqual(cqlToExpression('a=b'), query)
expectedNot = cqlToExpression('x=y')
expectedNot.must_not = True
self.assertEqual({'core2': [expectedNot]}, filters)
def testCqlQuery(self):
self.assertEqual(cqlToExpression('field = value'), WebQuery('field=value').query)
self.assertTrue(WebQuery('field = value').isDefaultQuery())
self.assertFalse(WebQuery('field = value').isBooleanQuery())
self.assertFalse(WebQuery('field = value').isPlusMinusQuery())
self.assertEqual(cqlToExpression('field = value'), WebQuery('field = value').query)
self.assertEqual(cqlToExpression('field exact value'), WebQuery('field exact value').query)
def testCoreAndQueryInOtherCore(self):
query, filters = self.convert(
cqlToExpression('core2.a=b AND core2.x=y'), 'core1')
self.assertEqual(None, query)
self.assertEqual(
{'core2': [cqlToExpression('a=b'),
cqlToExpression('x=y')]}, filters)
def testReplaceWith(self):
qe = cqlToExpression('aap AND noot')
replacement = cqlToExpression('fiets')
qe.operands[1].replaceWith(replacement)
self.assertEqual(cqlToExpression('aap AND fiets'), qe)
qe.operands[0].replaceWith(cqlToExpression('boom OR vis'))
self.assertEqual(cqlToExpression('(boom OR vis) AND fiets'), qe)
def testOrExpressions(self):
expression = cqlToExpression(
'field0=value0 AND field1=value1 OR field2 = value2')
self.assertEqual(
QueryExpression(operator='OR',
operands=[
QueryExpression(operator='AND',
operands=[
QE('field0=value0'),
QE('field1=value1'),
]),
QE('field2=value2'),
]), expression)
expression2 = cqlToExpression(
'(field0=value0 AND field1=value1) OR field2 = value2')
self.assertEqual(expression, expression2)
expression = cqlToExpression(
'field0=value0 AND (field1=value1 OR field2 = value2)')
self.assertEqual(
QueryExpression(operator='AND',
operands=[
QE('field0=value0'),
QueryExpression(operator='OR',
operands=[
QE('field1=value1'),
QE('field2=value2'),
]),
]), expression)
def testNothingToBeDone(self):
self.assertEquals(
cqlToExpression("field exact value"),
self.convert(
cql="field exact value",
shouldModifyFieldValue=lambda name, relation, value: name == "afield" and value.startswith("P"),
valueModifier=lambda value: "newvalue",
),
)
self.assertEquals(
cqlToExpression("afield exact value"),
self.convert(
cql="afield exact value",
shouldModifyFieldValue=lambda name, relation, value: name == "afield" and value.startswith("P"),
valueModifier=lambda value: "newvalue",
fieldnameModifier=lambda field: "newfield",
),
)
self.assertEquals(
cqlToExpression("afield=Pvalue"),
self.convert(
cql="afield=Pvalue",
shouldModifyFieldValue=lambda name, relation, value: name == "afield"
and relation == "exact"
and value.startswith("P"),
valueModifier=lambda value: "newvalue",
),
)
def testOtherFilterAndBeginAndEndOfQuery(self):
query, filters = self.convert(
cqlToExpression('core2.a=b AND f=v AND core2.b=c'), 'core1')
self.assertEqual(cqlToExpression('f=v'), query)
self.assertEqual(
{'core2': [cqlToExpression('a=b'),
cqlToExpression('b=c')]}, filters)
def testXFilterWhichIsNoJoinQuery(self):
consume(self.tree.any.executeQuery(cqlAbstractSyntaxTree=parseCQL('*'), extraArguments={'x-filter': ['prefix:field=value']}, facets=[], someKwarg='someValue'))
self.assertEquals(['executeComposedQuery'], self.observer.calledMethodNames())
cq = self.observer.calledMethods[0].kwargs['query']
cq.validate()
self.assertEquals('defaultCore', cq.resultsFrom)
self.assertEquals([cqlToExpression("*"), cqlToExpression("prefix:field=value")], cq.queriesFor('defaultCore'))
self.assertEquals(1, cq.numberOfUsedCores)
def testOtherCoreQueryWithMultipleAndClausesAtLast(self):
query, filters = self.convert(
cqlToExpression('field=value AND core2.f=v AND core2.y=x'),
'core1')
self.assertEqual(cqlToExpression('field=value'), query)
self.assertEqual(
{'core2': [cqlToExpression('f=v'),
cqlToExpression('y=x')]}, filters)
def testTooComplesQueries(self):
self.assertRaises(
TooComplexQueryExpression,
lambda: self.convert(cqlToExpression('core2.a=b OR f=v'), 'core1'))
self.assertRaises(
TooComplexQueryExpression, lambda: self.convert(
cqlToExpression('core1.f=v OR (core1.a=b AND core2.f=v)'),
'core1'))
def testXFilterWhichIsNoJoinQuery(self):
consume(self.tree.any.executeQuery(cqlAbstractSyntaxTree=parseCQL('*'), extraArguments={'x-filter': ['prefix:field=value']}, facets=[], someKwarg='someValue'))
self.assertEquals(['executeComposedQuery'], self.observer.calledMethodNames())
cq = self.observer.calledMethods[0].kwargs['query']
cq.validate()
self.assertEquals('defaultCore', cq.resultsFrom)
self.assertEquals([cqlToExpression("*"), cqlToExpression("prefix:field=value")], cq.queriesFor('defaultCore'))
self.assertEquals(1, cq.numberOfUsedCores)
def _default2Cql(aString, antiUnaryClause="ignored"):
if aString.strip() == '':
return antiUnaryClause
try:
cqlToExpression(aString)
return aString
except (CQLParseException, CQLTokenizerException):
return ' AND '.join(_valueFromGroupdict(match.groupdict()) for match in SPLITTED_STRINGS.finditer(aString))
def _default2Cql(aString, antiUnaryClause="ignored"):
if aString.strip() == '':
return antiUnaryClause
try:
cqlToExpression(aString)
return aString
except (CQLParseException, CQLTokenizerException):
return ' AND '.join(_valueFromGroupdict(match.groupdict()) for match in SPLITTED_STRINGS.finditer(aString))
def testTwoXFiltersForSameCore(self):
consume(self.tree.any.executeQuery(cqlAbstractSyntaxTree=parseCQL('*'), extraArguments={'x-filter': ['otherCore.prefix:field=value', 'otherCore.field2=value2']}, facets=[]))
self.assertEquals(['executeComposedQuery'], self.observer.calledMethodNames())
cq = self.observer.calledMethods[0].kwargs['query']
cq.validate()
self.assertEquals('defaultCore', cq.resultsFrom)
self.assertEquals(set(['defaultCore', 'otherCore']), cq.cores)
self.assertEquals('keyDefault', cq.keyName('defaultCore', 'otherCore'))
self.assertEquals('keyOther', cq.keyName('otherCore', 'defaultCore'))
self.assertEquals([cqlToExpression("prefix:field=value"), cqlToExpression('field2=value2')], cq.queriesFor('otherCore'))
def testTwoXFiltersForSameCore(self):
consume(self.tree.any.executeQuery(cqlAbstractSyntaxTree=parseCQL('*'), extraArguments={'x-filter': ['otherCore.prefix:field=value', 'otherCore.field2=value2']}, facets=[]))
self.assertEquals(['executeComposedQuery'], self.observer.calledMethodNames())
cq = self.observer.calledMethods[0].kwargs['query']
cq.validate()
self.assertEquals('defaultCore', cq.resultsFrom)
self.assertEquals(set(['defaultCore', 'otherCore']), cq.cores)
self.assertEquals('keyDefault', cq.keyName('defaultCore', 'otherCore'))
self.assertEquals('keyOther', cq.keyName('otherCore', 'defaultCore'))
self.assertEquals([cqlToExpression("prefix:field=value"), cqlToExpression('field2=value2')], cq.queriesFor('otherCore'))
def testCqlQuery(self):
self.assertEqual(cqlToExpression('field = value'),
WebQuery('field=value').query)
self.assertTrue(WebQuery('field = value').isDefaultQuery())
self.assertFalse(WebQuery('field = value').isBooleanQuery())
self.assertFalse(WebQuery('field = value').isPlusMinusQuery())
self.assertEqual(cqlToExpression('field = value'),
WebQuery('field = value').query)
self.assertEqual(cqlToExpression('field exact value'),
WebQuery('field exact value').query)
def testIter(self):
qe = cqlToExpression('aap NOT (noot OR title=mies)')
result = list(qe.iter())
self.assertEqual(qe, result[0])
self.assertEqual(cqlToExpression('aap'), result[1])
r2 = cqlToExpression('noot OR title=mies')
r2.must_not = True
self.assertEqual(r2, result[2])
self.assertEqual(cqlToExpression('noot'), result[3])
self.assertEqual(cqlToExpression('title=mies'), result[4])
def testSearchClauseModifySimpleSearchClause(self):
query = cqlToExpression('field=value')
def canModify(expression):
return True
def modify(expression):
expression.index = 'otherfield'
expression.term = 'othervalue'
conversion = CqlSearchClauseConversion(canModify, modify, fromKwarg="aQuery")
result = conversion._convert(query)
self.assertEqual(cqlToExpression('otherfield = othervalue'), result)
def testSearchClauseModifySimpleSearchClause(self):
query = cqlToExpression('field=value')
def canModify(expression):
return True
def modify(expression):
expression.index = 'otherfield'
expression.term = 'othervalue'
conversion = CqlSearchClauseConversion(canModify, modify, fromKwarg="aQuery")
result = conversion._convert(query)
self.assertEquals(cqlToExpression('otherfield = othervalue'), result)
def testFilterQuery(self):
consume(self.tree.any.executeQuery(cqlAbstractSyntaxTree=parseCQL('*'), filterQueries=[('otherCore', 'prefix:field=value')], facets=[], start=1))
self.assertEquals(['executeComposedQuery'], self.observer.calledMethodNames())
cq = self.observer.calledMethods[0].kwargs['query']
cq.validate()
self.assertEquals(1, cq.start)
self.assertEquals(set(['defaultCore', 'otherCore']), cq.cores)
self.assertEquals('keyDefault', cq.keyName('defaultCore', 'otherCore'))
self.assertEquals('keyOther', cq.keyName('otherCore', 'defaultCore'))
self.assertEquals([cqlToExpression("prefix:field=value")], cq.queriesFor('otherCore'))
self.assertEquals([cqlToExpression('*')], cq.queriesFor('defaultCore'))
def testParameterQueryWebQuery(self):
response = self.request(query='value1 value2')
executeQueryMethod = self.observer.calledMethods[0]
self.assertEqual(cqlToExpression('value1 AND value2'), executeQueryMethod.kwargs['query'])
self.assertEqual('value1 value2', response['request']['query'])
self.observer.calledMethods.reset()
response = self.request(query='-value')
executeQueryMethod = self.observer.calledMethods[0]
self.assertEqual(cqlToExpression('* NOT value'), executeQueryMethod.kwargs['query'])
self.assertEqual('-value', response['request']['query'])
def testFilterQuery(self):
consume(self.tree.any.executeQuery(cqlAbstractSyntaxTree=parseCQL('*'), filterQueries=[('otherCore', 'prefix:field=value')], facets=[], start=1))
self.assertEquals(['executeComposedQuery'], self.observer.calledMethodNames())
cq = self.observer.calledMethods[0].kwargs['query']
cq.validate()
self.assertEquals(1, cq.start)
self.assertEquals(set(['defaultCore', 'otherCore']), cq.cores)
self.assertEquals('keyDefault', cq.keyName('defaultCore', 'otherCore'))
self.assertEquals('keyOther', cq.keyName('otherCore', 'defaultCore'))
self.assertEquals([cqlToExpression("prefix:field=value")], cq.queriesFor('otherCore'))
self.assertEquals([cqlToExpression('*')], cq.queriesFor('defaultCore'))
def testConvertComposedQuery(self):
q = ComposedQuery('A')
q.setCoreQuery(core='A', query=cqlToExpression('valueAQ'))
q.setCoreQuery(core='B', query=cqlToExpression('valueBQ'))
q.addMatch(dict(core='A', uniqueKey='keyA'), dict(core='B', key='keyB'))
q.addUnite(dict(core='A', query=cqlToExpression('fieldUA exact valueUA')), dict(core='B', query=cqlToExpression('fieldUB exact valueUB')))
q.validate()
consume(self.dna.any.executeComposedQuery(query=q))
self.assertEquals(['executeComposedQuery'], self.observer.calledMethodNames())
self.assertEquals("{'type': 'TermQuery', 'term': {'field': 'fieldA', 'value': u'valueaq'}, 'boost': 1.0}", repr(q.queryFor('A')))
self.assertEquals("{'type': 'TermQuery', 'term': {'field': 'fieldB', 'value': u'valuebq'}, 'boost': 1.0}", repr(q.queryFor('B')))
def testParameterQueryWebQuery(self):
response = self.request(query='value1 value2')
executeQueryMethod = self.observer.calledMethods[0]
self.assertEqual(cqlToExpression('value1 AND value2'), executeQueryMethod.kwargs['query'])
self.assertEqual('value1 value2', response['request']['query'])
self.observer.calledMethods.reset()
response = self.request(query='-value')
executeQueryMethod = self.observer.calledMethods[0]
self.assertEqual(cqlToExpression('* NOT value'), executeQueryMethod.kwargs['query'])
self.assertEqual('-value', response['request']['query'])
def testXRankQuery(self):
consume(self.tree.any.executeQuery(cqlAbstractSyntaxTree=parseCQL('*'), extraArguments={'x-rank-query': ['otherCore.prefix:field=value', 'otherCore.otherprefix:otherfield=othervalue', 'field=value']}))
self.assertEquals(['executeComposedQuery'], self.observer.calledMethodNames())
cq = self.observer.calledMethods[0].kwargs['query']
cq.validate()
self.assertEquals(set(['defaultCore', 'otherCore']), cq.cores)
self.assertEquals('keyDefault', cq.keyName('defaultCore', 'otherCore'))
self.assertEquals('keyOther', cq.keyName('otherCore', 'defaultCore'))
self.assertEquals(cqlToExpression("prefix:field=value OR otherprefix:otherfield=othervalue"), cq.rankQueryFor('otherCore'))
self.assertEquals(cqlToExpression("field=value"), cq.rankQueryFor('defaultCore'))
self.assertEquals([], cq.queriesFor('otherCore'))
self.assertEquals([cqlToExpression('*')], cq.queriesFor('defaultCore'))
def testXRankQuery(self):
consume(self.tree.any.executeQuery(cqlAbstractSyntaxTree=parseCQL('*'), extraArguments={'x-rank-query': ['otherCore.prefix:field=value', 'otherCore.otherprefix:otherfield=othervalue', 'field=value']}))
self.assertEquals(['executeComposedQuery'], self.observer.calledMethodNames())
cq = self.observer.calledMethods[0].kwargs['query']
cq.validate()
self.assertEquals(set(['defaultCore', 'otherCore']), cq.cores)
self.assertEquals('keyDefault', cq.keyName('defaultCore', 'otherCore'))
self.assertEquals('keyOther', cq.keyName('otherCore', 'defaultCore'))
self.assertEquals(cqlToExpression("prefix:field=value OR otherprefix:otherfield=othervalue"), cq.rankQueryFor('otherCore'))
self.assertEquals(cqlToExpression("field=value"), cq.rankQueryFor('defaultCore'))
self.assertEquals([], cq.queriesFor('otherCore'))
self.assertEquals([cqlToExpression('*')], cq.queriesFor('defaultCore'))
def testModifyValue(self):
self.assertEqual(
cqlToExpression('afield exact newvalue'),
self.convert(
cql='afield exact Pvalue',
shouldModifyFieldValue=lambda name, relation, value: name=='afield' and relation=='exact' and value.startswith('P'),
valueModifier=lambda value: 'newvalue'))
self.assertEqual(
cqlToExpression('afield=newvalue'),
self.convert(
cql='afield=Pvalue',
shouldModifyFieldValue=lambda name, relation, value: value.startswith('P') and relation=='=',
valueModifier=lambda value: 'newvalue'))
def testMatchesOptional(self):
self.tree = be(
(Observable(),
(ConvertToComposedQuery(resultsFrom='defaultCore'),
(self.observer,)
)
)
)
consume(self.tree.any.executeQuery(cqlAbstractSyntaxTree=parseCQL('*'), extraArguments={'x-filter': ['prefix:field=value']}, facets=[], start=1))
self.assertEquals(['executeComposedQuery'], self.observer.calledMethodNames())
cq = self.observer.calledMethods[0].kwargs['query']
cq.validate()
self.assertEquals(1, cq.start)
self.assertEquals(set(['defaultCore']), cq.cores)
self.assertRaises(KeyError, lambda: cq.keyName('defaultCore', 'otherCore'))
self.assertEquals([cqlToExpression("*"), cqlToExpression("prefix:field=value")], cq.queriesFor('defaultCore'))
def testNestedWithReplaced(self):
q = cqlToExpression('A')
def canModify1(expression):
return expression.term == 'A'
def modify1(expression):
expression.replaceWith(cqlToExpression('aa OR bb'))
def canModify2(expression):
return expression.term == 'bb'
def modify2(expression):
expression.term = 'B'
conversion = CqlMultiSearchClauseConversion([
(canModify1, modify1),
(canModify2, modify2),
], fromKwarg="thisQuery")
result = conversion._convert(q)
self.assertEqual(cqlToExpression('aa OR B'), result)
def testIgnoreOtherMethodsWithQueryArgument(self):
def canModify(expression):
return expression.term == 'term'
def modify(expression):
expression.term = 'changed'
observer = CallTrace(emptyGeneratorMethods=['method'])
top = be((Observable(),
(CqlMultiSearchClauseConversion([(canModify, modify)], fromKwarg='query'),
(observer,)
)
))
consume(top.any.method(query='index = term'))
self.assertEqual({'query': 'index = term'}, observer.calledMethods[0].kwargs)
observer.calledMethods.reset()
consume(top.any.method(query=cqlToExpression('index = term')))
self.assertEqual({'query': cqlToExpression('index = changed')}, observer.calledMethods[0].kwargs)
def drilldown(self, query, term, fields):
drilldownResult = []
for field in fields:
cqlString = '(%s) AND %s=%s' % (query, field, term)
response = yield self.any.executeQuery(query=cqlToExpression(cqlString))
drilldownResult.append((field, response.total))
raise StopIteration(drilldownResult)
def testComposedQuery(self):
self.response = JsonDict({
"total": 887,
"queryTime": 6,
"hits": [{"id": "record:1", "score": 0.1234}]
}).dumps()
cq = ComposedQuery('coreA')
q = QueryExpressionToLuceneQueryDict([], LuceneSettings()).convert(cqlToExpression("field=value"))
cq.setCoreQuery("coreA", q)
consume(self._multiLucene.executeComposedQuery(cq))
self.assertEqual(1, len(self.post))
self.assertEqual("/query/", self.post[0]['path'])
self.assertEqual({
"_sortKeys": [],
"resultsFrom": "coreA",
"_matches": {},
"_facets": {},
"_otherCoreFacetFilters": {},
"_rankQueries": {},
"_drilldownQueries": {},
"_unites": [],
"_queries": {"coreA": {"term": {"field": "field", "value": "value"}, "type": "TermQuery"}},
"cores": ["coreA"],
"_filterQueries": {}
}, loads(self.post[0]['data']))
def testMatchesOptional(self):
self.tree = be(
(Observable(),
(ConvertToComposedQuery(resultsFrom='defaultCore'),
(self.observer,)
)
)
)
consume(self.tree.any.executeQuery(cqlAbstractSyntaxTree=parseCQL('*'), extraArguments={'x-filter': ['prefix:field=value']}, facets=[], start=1))
self.assertEquals(['executeComposedQuery'], self.observer.calledMethodNames())
cq = self.observer.calledMethods[0].kwargs['query']
cq.validate()
self.assertEquals(1, cq.start)
self.assertEquals(set(['defaultCore']), cq.cores)
self.assertRaises(KeyError, lambda: cq.keyName('defaultCore', 'otherCore'))
self.assertEquals([cqlToExpression("*"), cqlToExpression("prefix:field=value")], cq.queriesFor('defaultCore'))
def testAddTypeAndMissingValueToSortField(self):
self.response = JsonDict({
"total": 887,
"queryTime": 6,
"hits": [{"id": "record:1", "score": 0.1234}]
}).dumps()
cq = ComposedQuery('coreA')
q = QueryExpressionToLuceneQueryDict([], LuceneSettings()).convert(cqlToExpression("field=value"))
cq.setCoreQuery('coreB', q)
cq.sortKeys = [dict(sortBy='sortField', core='coreA', sortDescending=True)]
cq.addMatch(dict(core='coreA', uniqueKey='A'), dict(core='coreB', key='B'))
consume(self._multiLucene.executeComposedQuery(cq))
self.assertEqual({
"_sortKeys": [{'core': 'coreA', 'sortBy': 'sortField', 'sortDescending': True, 'type': 'String', 'missingValue': 'STRING_FIRST'}],
"resultsFrom": "coreA",
'_matches': {'coreA->coreB': [{'core': 'coreA', 'uniqueKey': 'A'}, {'core': 'coreB', 'key': 'B'}]},
"_facets": {},
"_otherCoreFacetFilters": {},
"_rankQueries": {},
"_drilldownQueries": {},
"_unites": [],
'_queries': {'coreB': {'term': {'field': 'field', 'value': 'value'}, 'type': 'TermQuery'}},
"cores": ["coreB", "coreA"],
"_filterQueries": {}
}, loads(self.post[0]['data']))
def testIgnoreOtherMethodsWithQueryArgument(self):
def canModify(expression):
return expression.term == 'term'
def modify(expression):
expression.term = 'changed'
observer = CallTrace(emptyGeneratorMethods=['method'])
top = be((Observable(),
(CqlMultiSearchClauseConversion([(canModify, modify)], fromKwarg='query'),
(observer,)
)
))
consume(top.any.method(query='index = term'))
self.assertEqual({'query': 'index = term'}, observer.calledMethods[0].kwargs)
observer.calledMethods.reset()
consume(top.any.method(query=cqlToExpression('index = term')))
self.assertEqual({'query': cqlToExpression('index = changed')}, observer.calledMethods[0].kwargs)
def testAddTypeAndMissingValueToSortField(self):
self.response = JsonDict({"total": 887, "queryTime": 6, "hits": [{"id": "record:1", "score": 0.1234}]}).dumps()
cq = ComposedQuery("coreA")
q = QueryExpressionToLuceneQueryDict([], LuceneSettings()).convert(cqlToExpression("field=value"))
cq.setCoreQuery("coreB", q)
cq.sortKeys = [dict(sortBy="sortField", core="coreA", sortDescending=True)]
cq.addMatch(dict(core="coreA", uniqueKey="A"), dict(core="coreB", key="B"))
consume(self._multiLucene.executeComposedQuery(cq))
self.assertEqual(
{
"_sortKeys": [
{
"core": "coreA",
"sortBy": "sortField",
"sortDescending": True,
"type": "String",
"missingValue": "STRING_FIRST",
}
],
"resultsFrom": "coreA",
"_matches": {"coreA->coreB": [{"core": "coreA", "uniqueKey": "A"}, {"core": "coreB", "key": "B"}]},
"_facets": {},
"_otherCoreFacetFilters": {},
"_rankQueries": {},
"_drilldownQueries": {},
"_unites": [],
"_queries": {"coreB": {"term": {"field": "field", "value": "value"}, "type": "TermQuery"}},
"cores": ["coreB", "coreA"],
"_filterQueries": {},
},
loads(self.post[0]["data"]),
)
def testNestedWithReplaced(self):
q = cqlToExpression('A')
def canModify1(expression):
return expression.term == 'A'
def modify1(expression):
expression.replaceWith(cqlToExpression('aa OR bb'))
def canModify2(expression):
return expression.term == 'bb'
def modify2(expression):
expression.term = 'B'
conversion = CqlMultiSearchClauseConversion([
(canModify1, modify1),
(canModify2, modify2),
], fromKwarg="thisQuery")
result = conversion._convert(q)
self.assertEqual(cqlToExpression('aa OR B'), result)
def testRepr(self):
self.maxDiff = None
qe = cqlToExpression('aap NOT (noot OR title=mies)')
self.assertEqual(
"QueryExpression(must_not=False, operands=[QueryExpression(index=None, must_not=False, operator=None, relation=None, relation_boost=None, term='aap'), QueryExpression(must_not=True, operands=[QueryExpression(index=None, must_not=False, operator=None, relation=None, relation_boost=None, term='noot'), QueryExpression(index='title', must_not=False, operator=None, relation='=', relation_boost=None, term='mies')], operator='OR', relation_boost=None)], operator='AND', relation_boost=None)",
repr(qe))
self.assertEqual(qe, eval(repr(qe)))
def testMultipleSearchClauseReplacements(self):
ast = cqlToExpression('term1 AND term2 AND term3')
#SEARCH_TERM(TERM('term'))
def canModifyTerm1(expression):
return expression.term == 'term1'
def modifyTerm1(expression):
expression.term = 'termOne'
def canModifyTerm3(expression):
return expression.term == 'term3'
def modifyTerm3(expression):
expression.term = 'termThree'
observerClassic = CallTrace('observerClassic')
observerNewStyle = CallTrace('observerNewStyle')
classic = be(
(Observable(), (CqlSearchClauseConversion(canModifyTerm1,
modifyTerm1,
fromKwarg="thisQuery"),
(CqlSearchClauseConversion(canModifyTerm3,
modifyTerm3,
fromKwarg="thisQuery"),
(observerClassic, )))))
newStyle = be(
(Observable(),
(CqlMultiSearchClauseConversion([(canModifyTerm1, modifyTerm1),
(canModifyTerm3, modifyTerm3)],
fromKwarg="thisQuery"),
(observerNewStyle, ))))
classic.do.message(thisQuery=ast)
newStyle.do.message(thisQuery=ast)
self.assertEquals(['message'],
[m.name for m in observerClassic.calledMethods])
resultClassic = observerClassic.calledMethods[0].kwargs['thisQuery']
self.assertEquals(['message'],
[m.name for m in observerNewStyle.calledMethods])
resultNewStyle = observerNewStyle.calledMethods[0].kwargs['thisQuery']
self.assertEquals(cqlToExpression('termOne AND term2 AND termThree'),
resultClassic)
self.assertEquals(cqlToExpression('termOne AND term2 AND termThree'),
resultNewStyle)
def testFacetFilters(self):
self.request(facet='field',
**{'facet-filter': ['field0=value0', 'field1=value1']})
executeQueryMethod = self.observer.calledMethods[0]
self.assertEquals(
cqlToExpression(
'* AND field0 exact value0 AND field1 exact value1'),
executeQueryMethod.kwargs['query'])
def testNormalQueryWithoutAnyJoin(self):
ast = parseCQL("prefixfield=value")
consume(self.tree.any.executeQuery(cqlAbstractSyntaxTree=ast, extraArguments={}))
self.assertEquals(['executeComposedQuery'], self.observer.calledMethodNames())
cq = self.observer.calledMethods[0].kwargs['query']
cq.validate()
self.assertEquals('defaultCore', cq.resultsFrom)
self.assertEquals([cqlToExpression("prefixfield=value")], cq.queriesFor('defaultCore'))
def testNot(self):
expression = cqlToExpression('term NOT thisterm')
self.assertEqual(
QueryExpression(operator='AND',
operands=[
QE('term'),
QE('thisterm', must_not=True),
]), expression)
def testNormalQueryWithoutAnyJoin(self):
ast = parseCQL("prefixfield=value")
consume(self.tree.any.executeQuery(cqlAbstractSyntaxTree=ast, extraArguments={}))
self.assertEquals(['executeComposedQuery'], self.observer.calledMethodNames())
cq = self.observer.calledMethods[0].kwargs['query']
cq.validate()
self.assertEquals('defaultCore', cq.resultsFrom)
self.assertEquals([cqlToExpression("prefixfield=value")], cq.queriesFor('defaultCore'))
def convert(self, cql, shouldModifyFieldValue, valueModifier=None, fieldnameModifier=None):
converter = CqlMultiSearchClauseConversion([
SearchTermFilterAndModifier(
shouldModifyFieldValue=shouldModifyFieldValue,
valueModifier=valueModifier,
fieldnameModifier=fieldnameModifier).filterAndModifier(),
], fromKwarg="aKwarg")
return converter._convert(cqlToExpression(cql))
def convert(self, cqlString):
converter = CqlMultiSearchClauseConversion(
[
RenameFieldForExact(
['untokenized.field', 'untokenized.prefix.*'],
'untokenized.').filterAndModifier()
],
fromKwarg="cqlAbstractSyntaxTree")
return converter._convert(cqlToExpression(cqlString))
def _assertQuery(self, expected, input, boolean=False, plusminus=False, default=False, needsBooleanHelp=False, asString=None):
input = expected if input == None else input
asString = expected if asString == None else asString
wq = WebQuery(input, antiUnaryClause='antiunary exact true')
self.assertEquals((boolean, plusminus, default, needsBooleanHelp), (wq.isBooleanQuery(), wq.isPlusMinusQuery(), wq.isDefaultQuery(), wq.needsBooleanHelp()))
self.assertEquals(parseCql(expected), wq.ast)
self.assertEqual(cqlToExpression(expected), wq.query)
self.assertEquals(asString, wq.asString())
self.assertEquals(input, wq.original)
def searchRetrieve(self, version=None, recordSchema=None, recordPacking=None, startRecord=1, maximumRecords=10, query='', sruArguments=None, diagnostics=None, **kwargs):
SRU_IS_ONE_BASED = 1
limitBeyond = kwargs.get('limitBeyond', None)
localLogCollector = {'arguments': sruArguments}
try:
t0 = self._timeNow()
start = startRecord - SRU_IS_ONE_BASED
facets = None
if 'x-term-drilldown' in sruArguments:
facets = self._parseDrilldownArgs(sruArguments['x-term-drilldown'])
queryExpression = cqlToExpression(query)
extraArguments = dict((key, value) for key, value in sruArguments.items() if key.startswith('x-'))
try:
response = yield self.any.executeQuery(
query=queryExpression,
start=start,
stop=start + maximumRecords,
facets=facets,
extraArguments=extraArguments,
**kwargs)
total, hits = response.total, response.hits
drilldownData = getattr(response, "drilldownData", None)
except Exception, e:
print_exc()
yield RESPONSE_HEADER
yield self._writeDiagnostics([(QUERY_FEATURE_UNSUPPORTED[0], QUERY_FEATURE_UNSUPPORTED[1], str(e))])
yield RESPONSE_FOOTER
return
queryTime = str(self._timeNow() - t0)
yield self._startResults(total, version)
recordsWritten = 0
for hit in hits:
if not recordsWritten:
yield '<srw:records>'
yield self._writeResult(recordSchema=recordSchema, recordPacking=recordPacking, hit=hit, version=version, sruArguments=sruArguments, **kwargs)
recordsWritten += 1
if recordsWritten:
yield '</srw:records>'
nextRecordPosition = start + recordsWritten
if nextRecordPosition < total and (limitBeyond == None or (limitBeyond != None and limitBeyond > nextRecordPosition)):
yield '<srw:nextRecordPosition>%i</srw:nextRecordPosition>' % (nextRecordPosition + SRU_IS_ONE_BASED)
yield self._writeEchoedSearchRetrieveRequest(sruArguments=sruArguments)
yield self._writeDiagnostics(diagnostics=diagnostics)
yield self._writeExtraResponseData(version=version, recordSchema=recordSchema, recordPacking=recordPacking, startRecord=startRecord, maximumRecords=maximumRecords, query=query, drilldownData=drilldownData, response=response, queryTime=queryTime, startTime=t0, sruArguments=sruArguments, localLogCollector=localLogCollector, **kwargs)
yield self._endResults()
def testModifyValue(self):
self.assertEquals(
cqlToExpression("afield exact newvalue"),
self.convert(
cql="afield exact Pvalue",
shouldModifyFieldValue=lambda name, relation, value: name == "afield"
and relation == "exact"
and value.startswith("P"),
valueModifier=lambda value: "newvalue",
),
)
self.assertEquals(
cqlToExpression("afield=newvalue"),
self.convert(
cql="afield=Pvalue",
shouldModifyFieldValue=lambda name, relation, value: value.startswith("P") and relation == "=",
valueModifier=lambda value: "newvalue",
),
)
def convert(self, cql, shouldModifyFieldValue, valueModifier=None, fieldnameModifier=None):
converter = CqlMultiSearchClauseConversion(
[
SearchTermFilterAndModifier(
shouldModifyFieldValue=shouldModifyFieldValue,
valueModifier=valueModifier,
fieldnameModifier=fieldnameModifier,
).filterAndModifier()
],
fromKwarg="aKwarg",
)
return converter._convert(cqlToExpression(cql))
def testMultipleSearchClauseReplacements(self):
ast = cqlToExpression('term1 AND term2 AND term3')
#SEARCH_TERM(TERM('term'))
def canModifyTerm1(expression):
return expression.term == 'term1'
def modifyTerm1(expression):
expression.term = 'termOne'
def canModifyTerm3(expression):
return expression.term == 'term3'
def modifyTerm3(expression):
expression.term = 'termThree'
observerClassic = CallTrace('observerClassic')
observerNewStyle = CallTrace('observerNewStyle')
classic = be((Observable(),
(CqlSearchClauseConversion(canModifyTerm1, modifyTerm1, fromKwarg="thisQuery"),
(CqlSearchClauseConversion(canModifyTerm3, modifyTerm3, fromKwarg="thisQuery"),
(observerClassic,)
)
)
))
newStyle = be((Observable(),
(CqlMultiSearchClauseConversion([
(canModifyTerm1, modifyTerm1),
(canModifyTerm3, modifyTerm3)
], fromKwarg="thisQuery"),
(observerNewStyle,)
)
))
classic.do.message(thisQuery=ast)
newStyle.do.message(thisQuery=ast)
self.assertEquals(['message'], [m.name for m in observerClassic.calledMethods])
resultClassic = observerClassic.calledMethods[0].kwargs['thisQuery']
self.assertEquals(['message'], [m.name for m in observerNewStyle.calledMethods])
resultNewStyle = observerNewStyle.calledMethods[0].kwargs['thisQuery']
self.assertEquals(cqlToExpression('termOne AND term2 AND termThree'), resultClassic)
self.assertEquals(cqlToExpression('termOne AND term2 AND termThree'), resultNewStyle)
def testExecuteQuery(self):
self.response = JsonDict({
"total": 887,
"queryTime": 6,
"times": {"searchTime": 3},
"hits": [{
"id": "record:1", "score": 0.1234,
"duplicateCount": {"__key__": 2},
"duplicates": {"__grouping_key__": [{"id": 'record:1'}, {"id": 'record:2'}]}
}],
"drilldownData": [
{"fieldname": "facet", "path": [], "terms": [{"term": "term", "count": 1}]}
],
"suggestions": {
"valeu": ["value"]
}
}).dumps()
query = QueryExpressionToLuceneQueryDict([], LuceneSettings()).convert(cqlToExpression("field=value"))
response = retval(self._lucene.executeQuery(
luceneQuery=query, start=1, stop=5,
facets=[dict(maxTerms=10, fieldname='facet')],
sortKeys=[dict(sortBy='field', sortDescending=False)],
suggestionRequest=dict(suggests=['valeu'], count=2, field='field1'),
dedupField="__key__",
clustering=True,
storedFields=["field"]
))
self.assertEqual(1, len(self.post))
self.assertEqual('/lucene/query/', self.post[0]['path'])
self.assertEqual({
"start": 1, "stop": 5,
"storedFields": ["field"],
"query": {"term": {"field": "field", "value": "value"}, "type": "TermQuery"},
"facets": [{"fieldname": "facet", "maxTerms": 10}],
"sortKeys": [{"sortBy": "field", "sortDescending": False, "type": "String", 'missingValue': 'STRING_LAST'}],
"suggestionRequest": dict(suggests=['valeu'], count=2, field='field1'),
"dedupField": "__key__",
"dedupSortField": None,
"clustering": True,
}, loads(self.post[0]['data']))
self.assertEqual(887, response.total)
self.assertEqual(6, response.queryTime)
self.assertEqual({'searchTime': 3}, response.times)
self.assertEqual(1, len(response.hits))
self.assertEqual("record:1", response.hits[0].id)
self.assertEqual(0.1234, response.hits[0].score)
self.assertEqual(dict(__key__=2), response.hits[0].duplicateCount)
self.assertEqual([
{"fieldname": "facet", "path": [], "terms": [{"term": "term", "count": 1}]}
], response.drilldownData)
self.assertEqual({'valeu': ['value']}, response.suggestions)
def testSRUParamsAndXMLOutput(self):
firstCall = []
def executeQuery(**kwargs):
if not firstCall:
firstCall.append(True)
raise StopIteration(Response(total=5, hits=range(5)))
else:
raise StopIteration(Response(total=10, hits=range(10)))
yield
sruFieldDrilldown = SruFieldDrilldown()
observer = CallTrace("observer")
sruFieldDrilldown.addObserver(observer)
observer.methods["executeQuery"] = executeQuery
result = compose(sruFieldDrilldown.extraResponseData(sruArguments={'x-field-drilldown': ['term'], 'x-field-drilldown-fields': ['field0,field1']}, query='original'))
self.assertEqualsWS(DRILLDOWN_HEADER + """<dd:field-drilldown>
<dd:field name="field0">5</dd:field>
<dd:field name="field1">10</dd:field></dd:field-drilldown></dd:drilldown>""", "".join(result))
self.assertEquals(['executeQuery', 'executeQuery'], [m.name for m in observer.calledMethods])
self.assertEquals(['query', 'query'], [','.join((m.kwargs.keys())) for m in observer.calledMethods])
self.assertEquals(cqlToExpression('(original) AND field0=term'), observer.calledMethods[0].kwargs['query'])
self.assertEquals(cqlToExpression('(original) AND field1=term'), observer.calledMethods[1].kwargs['query'])
def testLuceneServerHostPortDynamic(self):
multiLucene = MultiLucene(defaultCore='core1')
def httprequest1_1Mock(**kwargs):
raise StopIteration(parseResponse(HTTP_RESPONSE))
yield
observer = CallTrace(
'observer',
returnValues=dict(luceneServer=('example.org', 1234)),
methods=dict(httprequest1_1=httprequest1_1Mock))
multiLucene.addObserver(observer)
query = QueryExpressionToLuceneQueryDict([], LuceneSettings()).convert(cqlToExpression("field=value"))
response = retval(multiLucene.executeComposedQuery(ComposedQuery('core1', query)))
self.assertEquals(887, response.total)
self.assertEquals(['luceneServer', 'httprequest1_1'], observer.calledMethodNames())
def testDrilldown(self):
adapter = SruFieldDrilldown()
observer = CallTrace("Observer")
def executeQuery(**kwargs):
raise StopIteration(Response(total=16, hits=range(16)))
yield
observer.methods['executeQuery'] = executeQuery
adapter.addObserver(observer)
def dd():
result = yield adapter.drilldown('original', 'term', ['field0', 'field1'])
yield result
result = compose(dd()).next()
self.assertEquals(2, len(observer.calledMethods))
self.assertEquals(['executeQuery', 'executeQuery'], observer.calledMethodNames())
self.assertEquals(cqlToExpression("(original) and field0=term"), observer.calledMethods[0].kwargs['query'])
self.assertEquals([("field0", 16), ("field1", 16)], result)
def testLuceneServerHostPortDynamic(self):
lucene = Lucene(name='lucene', settings=LuceneSettings(), readonly=True)
def httprequest1_1Mock(**kwargs):
raise StopIteration(parseResponse(HTTP_RESPONSE))
yield
observer = CallTrace(
'observer',
returnValues=dict(luceneServer=('example.org', 1234)),
methods=dict(httprequest1_1=httprequest1_1Mock))
lucene.addObserver(observer)
query = QueryExpressionToLuceneQueryDict([], LuceneSettings()).convert(cqlToExpression("field=value"))
response = retval(lucene.executeQuery(
luceneQuery=query, start=1, stop=5,
))
self.assertEquals(887, response.total)
self.assertEquals(['luceneServer', 'httprequest1_1'], observer.calledMethodNames())
def testFacetFilter(self):
self.drilldownData = [
{ "fieldname": "field",
"path": [],
"terms": [{
"count": 23,
"term": "value0"
},]
}
]
json = self.request(facet='field', **{'facet-filter': 'field=somevalue'})
executeQueryMethod = self.observer.calledMethods[0]
self.assertEqual(cqlToExpression('* AND field exact somevalue'), executeQueryMethod.kwargs['query'])
facets = json['response']['facets']
link = self.parseLink(facets['field'][0]['link'])
self.assertEquals(['field=somevalue', 'field=value0'], link.query['facet-filter'])
def testWebQueryUsesFilters(self):
observer = CallTrace(
ignoredAttributes=['unknown', 'extraResponseData', 'echoedExtraRequestData'])
def executeQuery(**kwargs):
raise StopIteration(Response(total=0, hits=[]))
yield
observer.methods['executeQuery'] = executeQuery
rss = Rss(title = 'Title', description = 'Description', link = 'Link')
rss.addObserver(observer)
consume(rss.handleRequest(RequestURI='/?query=one+two&filter=field1:value1&filter=field2:value2'))
self.assertEquals(['executeQuery'], [m.name for m in observer.calledMethods])
self.assertEquals(None, observer.calledMethods[0].kwargs['sortKeys'])
self.assertEquals(0, observer.calledMethods[0].kwargs['start'])
self.assertEquals(10, observer.calledMethods[0].kwargs['stop'])
self.assertEquals(cqlToExpression("(one AND two) AND field1 exact value1 AND field2 exact value2"), observer.calledMethods[0].kwargs['query'])
