def testTimeoutExceptionWithoutCachedResult(self):
self.init(cacheTimeout=0.1,
returnCachedValueInCaseOfException=True,
backoffTimeout=0.1)
def someMessageTimeout(*args, **kwargs):
raise TimeoutException()
yield
self.observer.methods['someMessage'] = someMessageTimeout
self.assertRaises(
BackoffException,
lambda: retval(self.dna.any.someMessage('arg', kwarg='kwarg')))
self.assertEquals(['someMessage'], self.observer.calledMethodNames())
self.assertRaises(
BackoffException,
lambda: retval(self.dna.any.someMessage('arg', kwarg='kwarg')))
self.assertEquals(['someMessage'], self.observer.calledMethodNames())
sleep(0.11)
self.assertRaises(
BackoffException,
lambda: retval(self.dna.any.someMessage('arg', kwarg='kwarg')))
self.assertEquals(['someMessage', 'someMessage'],
self.observer.calledMethodNames())
def testTimeoutExceptionTriggersBackoff(self):
self.init(cacheTimeout=0.1, returnCachedValueInCaseOfException=True, backoffTimeout=0.1)
def someMessageResult(*args, **kwargs):
raise StopIteration('result')
yield
def someMessageTimeout(*args, **kwargs):
raise TimeoutException()
yield
self.observer.methods['someMessage'] = someMessageResult
result = retval(self.dna.any.someMessage('arg', kwarg='kwarg'))
self.assertEquals(['someMessage'], self.observer.calledMethodNames())
sleep(0.11)
self.observer.methods['someMessage'] = someMessageTimeout
result = retval(self.dna.any.someMessage('arg', kwarg='kwarg'))
self.assertEquals('result', result)
self.assertEquals(['someMessage', 'someMessage'], self.observer.calledMethodNames())
result = retval(self.dna.any.someMessage('arg', kwarg='kwarg'))
self.assertEquals('result', result)
# should be in backoff mode and not even try!
self.assertEquals(['someMessage', 'someMessage'], self.observer.calledMethodNames())
sleep(0.11)
self.observer.methods['someMessage'] = someMessageResult
result = retval(self.dna.any.someMessage('arg', kwarg='kwarg'))
self.assertEquals('result', result)
self.assertEquals(['someMessage', 'someMessage', 'someMessage'], self.observer.calledMethodNames())
def testPrefixSearch(self):
self.response = JsonList([["value0", 1], ["value1", 2]]).dumps()
response = retval(self._lucene.prefixSearch(fieldname='field1', prefix='valu'))
self.assertEquals(['value1', 'value0'], response.hits)
response = retval(self._lucene.prefixSearch(fieldname='field1', prefix='valu', showCount=True))
self.assertEquals([('value1', 2), ('value0', 1)], response.hits)
def testJoinSort(self):
cq = ComposedQuery('coreA')
cq.setCoreQuery(core='coreA', query=MatchAllDocsQuery())
cq.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreB', key=KEY_PREFIX+'B'))
cq.addSortKey({'sortBy': 'T', 'sortDescending': False, 'core': 'coreB'})
cq.addSortKey({'sortBy': 'S', 'sortDescending': False, 'core': 'coreA'})
result = retval(self.dna.any.executeComposedQuery(cq))
self.assertEqual(['A-M', 'A-MU', 'A-MQ', 'A-MQU', 'A', 'A-U', 'A-Q', 'A-QU'], [hit.id for hit in result.hits])
cq = ComposedQuery('coreA')
cq.setCoreQuery(core='coreA', query=MatchAllDocsQuery())
cq.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreB', key=KEY_PREFIX+'B'))
cq.addSortKey({'sortBy': 'T', 'sortDescending': True, 'core': 'coreB'})
cq.addSortKey({'sortBy': 'S', 'sortDescending': False, 'core': 'coreA'})
result = retval(self.dna.any.executeComposedQuery(cq))
self.assertEqual(['A-MQU', 'A-MQ', 'A-MU', 'A-M', 'A', 'A-U', 'A-Q', 'A-QU'], [hit.id for hit in result.hits])
cq = ComposedQuery('coreA')
cq.setCoreQuery(core='coreA', query=MatchAllDocsQuery())
cq.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreB', key=KEY_PREFIX+'B'))
cq.addSortKey({'sortBy': 'intField', 'sortDescending': True, 'core': 'coreB'})
cq.addSortKey({'sortBy': 'S', 'sortDescending': False, 'core': 'coreA'})
result = retval(self.dna.any.executeComposedQuery(cq))
self.assertEqual(['A-MQU', 'A-MQ', 'A-MU', 'A-M', 'A', 'A-U', 'A-Q', 'A-QU'], [hit.id for hit in result.hits])
cq = ComposedQuery('coreA')
cq.setCoreQuery(core='coreA', query=MatchAllDocsQuery())
cq.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreB', key=KEY_PREFIX+'B'))
cq.addSortKey({'sortBy': 'intField', 'sortDescending': True, 'core': 'coreB', 'missingValue': 20})
cq.addSortKey({'sortBy': 'S', 'sortDescending': False, 'core': 'coreA'})
result = retval(self.dna.any.executeComposedQuery(cq))
self.assertEqual(['A', 'A-U', 'A-Q', 'A-QU', 'A-MQU', 'A-MQ', 'A-MU', 'A-M'], [hit.id for hit in result.hits])
def testTimeoutExceptionTriggersBackoff(self):
self.init(cacheTimeout=0.1, returnCachedValueInCaseOfException=True, backoffTimeout=0.1)
def someMessageResult(*args, **kwargs):
return 'result'
yield
def someMessageTimeout(*args, **kwargs):
raise TimeoutException()
yield
self.observer.methods['someMessage'] = someMessageResult
result = retval(self.dna.any.someMessage('arg', kwarg='kwarg'))
self.assertEqual(['someMessage'], self.observer.calledMethodNames())
sleep(0.11)
self.observer.methods['someMessage'] = someMessageTimeout
result = retval(self.dna.any.someMessage('arg', kwarg='kwarg'))
self.assertEqual('result', result)
self.assertEqual(['someMessage', 'someMessage'], self.observer.calledMethodNames())
result = retval(self.dna.any.someMessage('arg', kwarg='kwarg'))
self.assertEqual('result', result)
# should be in backoff mode and not even try!
self.assertEqual(['someMessage', 'someMessage'], self.observer.calledMethodNames())
sleep(0.11)
self.observer.methods['someMessage'] = someMessageResult
result = retval(self.dna.any.someMessage('arg', kwarg='kwarg'))
self.assertEqual('result', result)
self.assertEqual(['someMessage', 'someMessage', 'someMessage'], self.observer.calledMethodNames())
def testEmptySocketPool_get(self):
sp = EmptySocketPool()
self.assertEquals(None,
retval(sp.getPooledSocket(host='whatever', port=-1)))
self.assertEquals(
None,
retval(sp.getPooledSocket(host='secure.example.org', port=1010)))
def testPrefixSearch(self):
self.response = JsonList([["value0", 1], ["value1", 2]]).dumps()
response = retval(self._lucene.prefixSearch(fieldname='field1', prefix='valu'))
self.assertEquals(['value1', 'value0'], response.hits)
response = retval(self._lucene.prefixSearch(fieldname='field1', prefix='valu', showCount=True))
self.assertEquals([('value1', 2), ('value0', 1)], response.hits)
def testIndexingState(self):
self.response = dumps(dict(started=12345, count=12))
result = retval(self.sic.indexingState())
self.assertEqual(dict(started=12345, count=12), result)
self.response = dumps(dict())
result = retval(self.sic.indexingState())
self.assertEqual(None, result)
def testErrorForInvalidFormatDrilldownQuery(self):
try:
retval(
self.dbdq.executeQuery(
extraArguments={'x-drilldown-query': ['a']}))
self.fail()
except ValueError, e:
self.assertEquals('x-drilldown-query format should be field=value',
str(e))
def testClearCache(self):
def someMessage(*args, **kwargs):
raise StopIteration('result')
yield
self.observer.methods['someMessage'] = someMessage
retval(self.dna.any.someMessage('arg', kwarg='kwarg'))
self.assertEquals(['someMessage'], self.observer.calledMethodNames())
self.cache.clear()
retval(self.dna.any.someMessage('arg', kwarg='kwarg'))
self.assertEquals(['someMessage', 'someMessage'], self.observer.calledMethodNames())
def testClearCache(self):
def someMessage(*args, **kwargs):
return 'result'
yield
self.observer.methods['someMessage'] = someMessage
retval(self.dna.any.someMessage('arg', kwarg='kwarg'))
self.assertEqual(['someMessage'], self.observer.calledMethodNames())
self.cache.clear()
retval(self.dna.any.someMessage('arg', kwarg='kwarg'))
self.assertEqual(['someMessage', 'someMessage'], self.observer.calledMethodNames())
def testPutNGetLIFO(self):
sp = SocketPool(reactor=CallTrace())
retval(sp.putSocketInPool(host='x', port=1, sock='A'))
retval(sp.putSocketInPool(host='x', port=1, sock='B'))
retval(sp.putSocketInPool(host='x', port=1, sock='C'))
self.assertEquals('C', retval(sp.getPooledSocket(host='x', port=1)))
self.assertEquals('B', retval(sp.getPooledSocket(host='x', port=1)))
self.assertEquals('A', retval(sp.getPooledSocket(host='x', port=1)))
self.assertEquals(None, retval(sp.getPooledSocket(host='x', port=1)))
def testPutNGetLIFO(self):
sp = SocketPool(reactor=CallTrace())
retval(sp.putSocketInPool(host='x', port=1, sock='A'))
retval(sp.putSocketInPool(host='x', port=1, sock='B'))
retval(sp.putSocketInPool(host='x', port=1, sock='C'))
self.assertEquals('C', retval(sp.getPooledSocket(host='x', port=1)))
self.assertEquals('B', retval(sp.getPooledSocket(host='x', port=1)))
self.assertEquals('A', retval(sp.getPooledSocket(host='x', port=1)))
self.assertEquals(None, retval(sp.getPooledSocket(host='x', port=1)))
def testErrorForInvalidFormatDrilldownQuery(self):
try:
retval(
self.dbdq.executeQuery(
extraArguments={'x-drilldown-query': ['a']}))
self.fail()
except ValueError as e:
self.assertEqual('x-drilldown-query format should be field=value',
str(e))
self.assertEqual([], self.observer.calledMethodNames())
def testPutNGet1Put1StillLIFO(self):
sp = SocketPool(reactor=CallTrace())
retval(sp.putSocketInPool(host='example.org', port=80, sock='A'))
retval(sp.putSocketInPool(host='example.org', port=80, sock='B'))
self.assertEquals('B', retval(sp.getPooledSocket(host='example.org', port=80)))
retval(sp.putSocketInPool(host='example.org', port=80, sock='C'))
self.assertEquals('C', retval(sp.getPooledSocket(host='example.org', port=80)))
self.assertEquals('A', retval(sp.getPooledSocket(host='example.org', port=80)))
self.assertEquals(None, retval(sp.getPooledSocket(host='example.org', port=80)))
def testDrilldownFieldnames(self):
self.response = '["field1", "field2"]'
result = retval(self._lucene.drilldownFieldnames())
self.assertEqual(["field1", "field2"], result.hits)
self.assertEqual([{"data": None, "path": "/lucene/drilldownFieldnames/?limit=50"}], self.post)
result = retval(self._lucene.drilldownFieldnames(limit=1, path=['field']))
self.assertEqual(["field1", "field2"], result.hits)
self.assertEqual({"data": None, "path": "/lucene/drilldownFieldnames/?dim=field&limit=1"}, self.post[-1])
result = retval(self._lucene.drilldownFieldnames(limit=1, path=['xyz', 'abc', 'field']))
self.assertEqual(["field1", "field2"], result.hits)
self.assertEqual({"data": None, "path": "/lucene/drilldownFieldnames/?dim=xyz&limit=1&path=abc&path=field"}, self.post[-1])
def testDrilldownFieldnames(self):
self.response = '["field1", "field2"]'
result = retval(self._lucene.drilldownFieldnames())
self.assertEqual(["field1", "field2"], result.hits)
self.assertEqual([{"data": None, "path": "/lucene/drilldownFieldnames/?limit=50"}], self.post)
result = retval(self._lucene.drilldownFieldnames(limit=1, path=['field']))
self.assertEqual(["field1", "field2"], result.hits)
self.assertEqual({"data": None, "path": "/lucene/drilldownFieldnames/?dim=field&limit=1"}, self.post[-1])
result = retval(self._lucene.drilldownFieldnames(limit=1, path=['xyz', 'abc', 'field']))
self.assertEqual(["field1", "field2"], result.hits)
self.assertEqual({"data": None, "path": "/lucene/drilldownFieldnames/?dim=xyz&limit=1&path=abc&path=field"}, self.post[-1])
def testTimeoutExceptionWithoutCachedResult(self):
self.init(cacheTimeout=0.1, returnCachedValueInCaseOfException=True, backoffTimeout=0.1)
def someMessageTimeout(*args, **kwargs):
raise TimeoutException()
yield
self.observer.methods['someMessage'] = someMessageTimeout
self.assertRaises(BackoffException, lambda: retval(self.dna.any.someMessage('arg', kwarg='kwarg')))
self.assertEquals(['someMessage'], self.observer.calledMethodNames())
self.assertRaises(BackoffException, lambda: retval(self.dna.any.someMessage('arg', kwarg='kwarg')))
self.assertEquals(['someMessage'], self.observer.calledMethodNames())
sleep(0.11)
self.assertRaises(BackoffException, lambda: retval(self.dna.any.someMessage('arg', kwarg='kwarg')))
self.assertEquals(['someMessage', 'someMessage'], self.observer.calledMethodNames())
def testDrilldownQuery(self):
result = retval(self.dbdq.executeQuery(extraArguments={'x-drilldown-query': ['a = b']}))
self.assertEquals('result', result)
self.assertEquals(['executeQuery'], self.observer.calledMethodNames())
executeQueryMethod = self.observer.calledMethods[0]
self.assertEquals([('a', ['b'])], executeQueryMethod.kwargs['drilldownQueries'])
self.observer.calledMethods.reset()
result = retval(self.dbdq.executeQuery(extraArguments={'x-drilldown-query': ['a exact b']}))
self.assertEquals('result', result)
self.assertEquals(['executeQuery'], self.observer.calledMethodNames())
executeQueryMethod = self.observer.calledMethods[0]
self.assertEquals([('a', ['b'])], executeQueryMethod.kwargs['drilldownQueries'])
def testPutEmptyPut(self):
sp = SocketPool(reactor=CallTrace())
retval(sp.putSocketInPool(host='10.0.0.1', port=60000, sock=0))
retval(sp.putSocketInPool(host='10.0.0.1', port=60000, sock=1))
for i in reversed(range(2)):
self.assertEquals(i, retval(sp.getPooledSocket(host='10.0.0.1', port=60000)))
self.assertEquals(None, retval(sp.getPooledSocket(host='10.0.0.1', port=60000)))
retval(sp.putSocketInPool(host='10.0.0.1', port=60000, sock=2))
self.assertEquals(2, retval(sp.getPooledSocket(host='10.0.0.1', port=60000)))
def testSortWithJoinField(self):
joinSortCollector = JoinSortCollector(KEY_PREFIX + 'A', KEY_PREFIX+'B')
self.luceneB.search(query=MatchAllDocsQuery(), collector=joinSortCollector)
sortField = JoinSortField('T', self.registry.sortFieldType('T'), False, joinSortCollector)
sortField.setMissingValue(self.registry.defaultMissingValueForSort('T', False))
result = retval(self.luceneA.executeQuery(MatchAllDocsQuery(), sortKeys=[sortField]))
self.assertEqual(['A-M', 'A-MU', 'A-MQ', 'A-MQU', 'A', 'A-U', 'A-Q', 'A-QU'], [hit.id for hit in result.hits])
sortField = JoinSortField('T', self.registry.sortFieldType('T'), True, joinSortCollector)
sortField.setMissingValue(self.registry.defaultMissingValueForSort('T', True))
result = retval(self.luceneA.executeQuery(MatchAllDocsQuery(), sortKeys=[sortField]))
self.assertEqual(['A-MQU', 'A-MQ', 'A-MU', 'A-M', 'A', 'A-U', 'A-Q', 'A-QU'], [hit.id for hit in result.hits])
def testTimeoutExceptionIsRaiseIfNoBackoffTimeoutSet(self):
self.init(cacheTimeout=0.1)
def someMessageTimeout(*args, **kwargs):
raise TimeoutException()
yield
self.observer.methods['someMessage'] = someMessageTimeout
self.assertRaises(TimeoutException, lambda: retval(self.dna.any.someMessage('arg', kwarg='kwarg')))
def testUpdateSettings(self):
self.response = JsonDict(numberOfConcurrentTasks=6, similarity="BM25(k1=1.2,b=0.75)", clustering=JsonDict(clusterMoreRecords=100, clusteringEps=0.4, clusteringMinPoints=1))
settings = retval(self._lucene.getSettings())
self.assertEqual(['/settings/'], self.read)
self.assertEquals({'numberOfConcurrentTasks': 6, 'similarity': u'BM25(k1=1.2,b=0.75)', 'clustering': {'clusterMoreRecords': 100, 'clusteringEps': 0.4, 'clusteringMinPoints': 1}}, settings)
clusterFields = [
{"filterValue": None, "fieldname": "untokenized.dcterms:isFormatOf.uri", "weight": 0}
]
self.response = ""
consume(self._lucene.setSettings(similarity=dict(name="bm25", k1=1.0, b=2.0), numberOfConcurrentTasks=10, clustering=dict(clusterMoreRecords=200, clusteringEps=1.0, clusteringMinPoints=2, fields=clusterFields)))
self.assertEqual(1, len(self.post))
self.assertEqual('/lucene/settings/', self.post[0]['path'])
self.assertEqual({
"numberOfConcurrentTasks": 10,
"similarity": dict(type="BM25Similarity", k1=1.0, b=2.0),
"clustering": {
"clusterMoreRecords": 200,
"clusteringEps": 1.0,
"clusteringMinPoints": 2,
"fields": [
{"filterValue": None, "fieldname": "untokenized.dcterms:isFormatOf.uri", "weight": 0}
]
}
}, loads(self.post[0]['data']))
consume(self._lucene.setSettings(numberOfConcurrentTasks=5, similarity=None, clustering=None))
self.assertEqual(2, len(self.post))
self.assertEqual('/lucene/settings/', self.post[1]['path'])
self.assertEqual({
"numberOfConcurrentTasks": 5,
}, loads(self.post[1]['data']))
def testInfoOnQuery(self):
self.response = JsonDict({"total": 887, "queryTime": 6, "hits": [{"id": "record:1", "score": 0.1234}]}).dumps()
q = ComposedQuery("coreA")
q.addFilterQuery("coreB", query="N=true")
q.addMatch(dict(core="coreA", uniqueKey="A"), dict(core="coreB", key="B"))
result = retval(self._multiLucene.executeComposedQuery(q))
self.assertEquals(
{
"query": {
"cores": ["coreB", "coreA"],
"drilldownQueries": {},
"facets": {},
"filterQueries": {"coreB": ["N=true"]},
"matches": {"coreA->coreB": [{"core": "coreA", "uniqueKey": "A"}, {"core": "coreB", "key": "B"}]},
"otherCoreFacetFilters": {},
"queries": {},
"rankQueries": {},
"resultsFrom": "coreA",
"sortKeys": [],
"unites": [],
},
"type": "ComposedQuery",
},
result.info,
)
def testNoDrilldownQuery(self):
result = retval(self.dbdq.executeQuery(extraArguments={}, query='*'))
self.assertEquals('result', result)
self.assertEquals(['executeQuery'], self.observer.calledMethodNames())
executeQueryMethod = self.observer.calledMethods[0]
self.assertEquals([], executeQueryMethod.kwargs['drilldownQueries'])
self.assertEquals("*", executeQueryMethod.kwargs['query'])
def testCacheAny(self):
def someMessage(*args, **kwargs):
raise StopIteration('result')
yield
self.observer.methods['someMessage'] = someMessage
result = retval(self.dna.any.someMessage('arg', kwarg='kwarg'))
self.assertEquals('result', result)
result = retval(self.dna.any.someMessage('arg', kwarg='kwarg'))
self.assertEquals('result', result)
self.assertEquals(['someMessage'], self.observer.calledMethodNames())
sleep(0.11)
result = retval(self.dna.any.someMessage('arg', kwarg='kwarg'))
self.assertEquals('result', result)
self.assertEquals(['someMessage', 'someMessage'], self.observer.calledMethodNames())
result = retval(self.dna.any.someMessage('arg', kwarg='otherkwarg'))
self.assertEquals(['someMessage', 'someMessage', 'someMessage'], self.observer.calledMethodNames())
def testXTermDrilldown(self):
self.response = LuceneResponse(drilldownData=[
dict(fieldname='prefix:field1', terms=[dict(term='term1', count=1)]),
dict(fieldname='unknownJoinName.field', terms=[]),
dict(fieldname='normal:drilldown', terms=[]),
dict(fieldname='prefix:field2', terms=[dict(term='term2', count=1)]),
])
response = retval(self.tree.any.executeQuery(
cqlAbstractSyntaxTree=parseCQL('*'),
extraArguments={},
facets=[
dict(fieldname='otherCore.prefix:field2', maxTerms=7),
dict(fieldname='normal:drilldown', maxTerms=11),
dict(fieldname='otherCore.prefix:field1', maxTerms=9),
dict(fieldname='unknownJoinName.field', maxTerms=12)
],
someKwarg='someValue'))
self.assertEquals(['executeComposedQuery'], self.observer.calledMethodNames())
cq = self.observer.calledMethods[0].kwargs['query']
cq.validate()
self.assertEquals(set(['defaultCore', 'otherCore']), cq.cores)
self.assertEquals('keyDefault', cq.keyName('defaultCore', 'otherCore'))
self.assertEquals('keyOther', cq.keyName('otherCore', 'defaultCore'))
self.assertEquals([dict(fieldname='prefix:field2', path=[], maxTerms=7), dict(fieldname='prefix:field1', path=[], maxTerms=9)], cq.facetsFor('otherCore'))
self.assertEquals([dict(fieldname='normal:drilldown', path=[], maxTerms=11), dict(fieldname='unknownJoinName.field', path=[], maxTerms=12)], cq.facetsFor('defaultCore'))
self.assertEquals([
dict(fieldname='otherCore.prefix:field2', terms=[dict(term='term2', count=1)]),
dict(fieldname='normal:drilldown', terms=[]),
dict(fieldname='otherCore.prefix:field1', terms=[dict(term='term1', count=1)]),
dict(fieldname='unknownJoinName.field', terms=[])
], response.drilldownData)
def testTotalSuggestions(self):
self.response = "10"
total = retval(self.sic.totalSuggestions())
self.assertEqual(10, total)
self.assertEqual(0, len(self.post))
self.assertEqual(1, len(self.get))
self.assertEqual('/totalSuggestions', self.get[0])
def testXTermDrilldown(self):
self.response = LuceneResponse(drilldownData=[
dict(fieldname='prefix:field1', terms=[dict(term='term1', count=1)]),
dict(fieldname='unknownJoinName.field', terms=[]),
dict(fieldname='normal:drilldown', terms=[]),
dict(fieldname='prefix:field2', terms=[dict(term='term2', count=1)]),
])
response = retval(self.tree.any.executeQuery(
cqlAbstractSyntaxTree=parseCQL('*'),
extraArguments={},
facets=[
dict(fieldname='otherCore.prefix:field2', maxTerms=7),
dict(fieldname='normal:drilldown', maxTerms=11),
dict(fieldname='otherCore.prefix:field1', maxTerms=9),
dict(fieldname='unknownJoinName.field', maxTerms=12)
],
someKwarg='someValue'))
self.assertEquals(['executeComposedQuery'], self.observer.calledMethodNames())
cq = self.observer.calledMethods[0].kwargs['query']
cq.validate()
self.assertEquals(set(['defaultCore', 'otherCore']), cq.cores)
self.assertEquals('keyDefault', cq.keyName('defaultCore', 'otherCore'))
self.assertEquals('keyOther', cq.keyName('otherCore', 'defaultCore'))
self.assertEquals([dict(fieldname='prefix:field2', path=[], maxTerms=7), dict(fieldname='prefix:field1', path=[], maxTerms=9)], cq.facetsFor('otherCore'))
self.assertEquals([dict(fieldname='normal:drilldown', path=[], maxTerms=11), dict(fieldname='unknownJoinName.field', path=[], maxTerms=12)], cq.facetsFor('defaultCore'))
self.assertEquals([
dict(fieldname='otherCore.prefix:field2', terms=[dict(term='term2', count=1)]),
dict(fieldname='normal:drilldown', terms=[]),
dict(fieldname='otherCore.prefix:field1', terms=[dict(term='term1', count=1)]),
dict(fieldname='unknownJoinName.field', terms=[])
], response.drilldownData)
def testCacheAny(self):
def someMessage(*args, **kwargs):
return 'result'
yield
self.observer.methods['someMessage'] = someMessage
result = retval(self.dna.any.someMessage('arg', kwarg='kwarg'))
self.assertEqual('result', result)
result = retval(self.dna.any.someMessage('arg', kwarg='kwarg'))
self.assertEqual('result', result)
self.assertEqual(['someMessage'], self.observer.calledMethodNames())
sleep(0.11)
result = retval(self.dna.any.someMessage('arg', kwarg='kwarg'))
self.assertEqual('result', result)
self.assertEqual(['someMessage', 'someMessage'], self.observer.calledMethodNames())
result = retval(self.dna.any.someMessage('arg', kwarg='otherkwarg'))
self.assertEqual(['someMessage', 'someMessage', 'someMessage'], self.observer.calledMethodNames())
def XXX_NOT_YET_IMPLEMENTED_testRankQueryInSingleCoreQuery(self):
q = ComposedQuery('coreA', query=MatchAllDocsQuery())
q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreB', key=KEY_PREFIX+'B'))
q.setRankQuery(core='coreA', query=luceneQueryFromCql('Q=true'))
result = retval(self.dna.any.executeComposedQuery(q))
self.assertEquals(8, result.total)
self.assertEquals([u'A-Q', u'A-QU', u'A-MQ', u'A-MQU', u'A', u'A-U', u'A-M', u'A-MU'], [hit.id for hit in result.hits])
def testInfoOnQuery(self):
self.response = JsonDict({
"total": 887,
"queryTime": 6,
"hits": [{"id": "record:1", "score": 0.1234}]
}).dumps()
q = ComposedQuery('coreA')
q.addFilterQuery('coreB', query='N=true')
q.addMatch(dict(core='coreA', uniqueKey='A'), dict(core='coreB', key='B'))
result = retval(self._multiLucene.executeComposedQuery(q))
self.assertEquals({
'query': {
'cores': ['coreB', 'coreA'],
'drilldownQueries': {},
'facets': {},
'filterQueries': {'coreB': ['N=true']},
'matches': {'coreA->coreB': [{'core': 'coreA', 'uniqueKey': 'A'}, {'core': 'coreB', 'key': 'B'}]},
'otherCoreFacetFilters': {},
'queries': {},
'rankQueries': {},
'resultsFrom': 'coreA',
'sortKeys': [],
'unites': []
},
'type': 'ComposedQuery'
}, result.info)
def testNoDrilldownQuery(self):
result = retval(self.dbdq.executeQuery(extraArguments={}, query='*'))
self.assertEquals('result', result)
self.assertEquals(['executeQuery'], self.observer.calledMethodNames())
executeQueryMethod = self.observer.calledMethods[0]
self.assertEquals([], executeQueryMethod.kwargs['drilldownQueries'])
self.assertEquals("*", executeQueryMethod.kwargs['query'])
def testNoQueryInCaseOfGeoLatLong(self):
queries = []
def queryErfGeoApi(query, expectedType=None):
queries.append(dict(query=query, expectedType=expectedType))
raise StopIteration(QUERY_RESULTS)
yield
def toAnnotation(pit, targetUri, query, geoCoordinates=None):
return PitToAnnotation().toAnnotation(pit=pit, targetUri=targetUri, query=query, geoCoordinates=geoCoordinates)
observer = CallTrace('observer', methods={'queryErfGeoApi': queryErfGeoApi, 'toAnnotation': toAnnotation})
top = be(
(Observable(),
(ErfGeoEnrichmentFromSummary(),
(observer,)
)
)
)
summary = makeSummary([], geoLatLong=('51.8', '5.0'))
result = retval(top.any.annotationFromSummary(summary))
self.assertEquals([], queries)
annotationUri, annotation = result
self.assertEquals(ERFGEO_ENRICHMENT_PROFILE.uriFor('uri:target'), annotationUri)
self.assertEquals('51.8', xpathFirst(annotation, '/rdf:RDF/oa:Annotation/oa:hasBody/rdf:Description/geo:lat/text()'))
def testPutNGet1Put1StillLIFO(self):
sp = SocketPool(reactor=CallTrace())
retval(sp.putSocketInPool(host='example.org', port=80, sock='A'))
retval(sp.putSocketInPool(host='example.org', port=80, sock='B'))
self.assertEquals(
'B', retval(sp.getPooledSocket(host='example.org', port=80)))
retval(sp.putSocketInPool(host='example.org', port=80, sock='C'))
self.assertEquals(
'C', retval(sp.getPooledSocket(host='example.org', port=80)))
self.assertEquals(
'A', retval(sp.getPooledSocket(host='example.org', port=80)))
self.assertEquals(
None, retval(sp.getPooledSocket(host='example.org', port=80)))
def testTotalShingleRecords(self):
self.response = "10"
total = retval(self.sic.totalShingleRecords())
self.assertEqual(10, total)
self.assertEqual(0, len(self.post))
self.assertEqual(1, len(self.get))
self.assertEqual('/totalRecords', self.get[0])
def testTimeoutExceptionIsRaiseIfNoBackoffTimeoutSet(self):
self.init(cacheTimeout=0.1)
def someMessageTimeout(*args, **kwargs):
raise TimeoutException()
yield
self.observer.methods['someMessage'] = someMessageTimeout
self.assertRaises(TimeoutException, lambda: retval(self.dna.any.someMessage('arg', kwarg='kwarg')))
def testUpdateSettings(self):
self.response = JsonDict(numberOfConcurrentTasks=6, similarity="BM25(k1=1.2,b=0.75)", clustering=JsonDict(clusterMoreRecords=100, clusteringEps=0.4, clusteringMinPoints=1))
settings = retval(self._lucene.getSettings())
self.assertEqual(['/settings/'], self.read)
self.assertEquals({'numberOfConcurrentTasks': 6, 'similarity': u'BM25(k1=1.2,b=0.75)', 'clustering': {'clusterMoreRecords': 100, 'clusteringEps': 0.4, 'clusteringMinPoints': 1}}, settings)
clusterFields = [
{"filterValue": None, "fieldname": "untokenized.dcterms:isFormatOf.uri", "weight": 0}
]
self.response = ""
consume(self._lucene.setSettings(similarity=dict(name="bm25", k1=1.0, b=2.0), numberOfConcurrentTasks=10, clustering=dict(clusterMoreRecords=200, clusteringEps=1.0, clusteringMinPoints=2, fields=clusterFields)))
self.assertEqual(1, len(self.post))
self.assertEqual('/lucene/settings/', self.post[0]['path'])
self.assertEqual({
"numberOfConcurrentTasks": 10,
"similarity": dict(type="BM25Similarity", k1=1.0, b=2.0),
"clustering": {
"clusterMoreRecords": 200,
"clusteringEps": 1.0,
"clusteringMinPoints": 2,
"fields": [
{"filterValue": None, "fieldname": "untokenized.dcterms:isFormatOf.uri", "weight": 0}
]
}
}, loads(self.post[0]['data']))
consume(self._lucene.setSettings(numberOfConcurrentTasks=5, similarity=None, clustering=None))
self.assertEqual(2, len(self.post))
self.assertEqual('/lucene/settings/', self.post[1]['path'])
self.assertEqual({
"numberOfConcurrentTasks": 5,
}, loads(self.post[1]['data']))
def t():
sok = MockSok('s')
current_exc = []
def shutdown(*a, **kw):
e = exceptions.pop(0)
current_exc.append(e)
raise e
sok.log.methods['shutdown'] = shutdown
sp = EmptySocketPool()
try:
retval(sp.putSocketInPool(host='secure.example.org', port=9999, sock=sok))
except type(current_exc[0]):
pass
else:
self.fail()
self.assertEquals(['shutdown'], sok.log.calledMethodNames())
self.assertEquals(((SHUT_RDWR,), {}), (sok.log.calledMethods[0].args, sok.log.calledMethods[0].kwargs))
def testEmptySocketPool_put_noErrors(self):
sok = MockSok('s')
sp = EmptySocketPool()
result = retval(sp.putSocketInPool(host='whatever', port=-1, sock=sok))
self.assertEquals(None, result)
self.assertEquals(['shutdown', 'close'], sok.log.calledMethodNames())
self.assertEquals(((SHUT_RDWR,), {}), (sok.log.calledMethods[0].args, sok.log.calledMethods[0].kwargs))
self.assertEquals(((), {}), (sok.log.calledMethods[1].args, sok.log.calledMethods[1].kwargs))
def testDoesNotMatchAny(self):
consume(self.matchesVersion.updateConfig(**{'software_version': '0.1'}))
try:
_ = retval(self.top.any.somemessage())
self.fail()
except NoneOfTheObserversRespond:
pass
self.assertEqual(['updateConfig'], self.observer.calledMethodNames())
def testNoSuggestionQuery(self):
extraArguments = {'x-suggestionsQuery': [''], 'x-suggestionsField': ['fieldname'], 'x-suggestionsCount': ['5']}
suggestion = Suggestion(count=10, field='dcterms:title', allowOverrideField=False)
suggestion.addObserver(self.observer)
result = retval(suggestion.executeQuery(kwarg='value', extraArguments=extraArguments))
self.assertEquals(['executeQuery'], self.observer.calledMethodNames())
methodKwargs = self.observer.calledMethods[0].kwargs
self.assertEquals(None, methodKwargs['suggestionRequest'])
def testTimeoutExceptionNotHandledSpecially(self):
self.init(cacheTimeout=0.1, returnCachedValueInCaseOfException=True)
def someMessageResult(*args, **kwargs):
raise StopIteration('result')
yield
def someMessageTimeout(*args, **kwargs):
raise TimeoutException()
yield
self.observer.methods['someMessage'] = someMessageResult
result = retval(self.dna.any.someMessage('arg', kwarg='kwarg'))
self.assertEquals(['someMessage'], self.observer.calledMethodNames())
sleep(0.11)
self.observer.methods['someMessage'] = someMessageTimeout
result = retval(self.dna.any.someMessage('arg', kwarg='kwarg'))
self.assertEquals('result', result)
self.assertEquals(['someMessage', 'someMessage'], self.observer.calledMethodNames())
def testTimeoutExceptionNotHandledSpecially(self):
self.init(cacheTimeout=0.1, returnCachedValueInCaseOfException=True)
def someMessageResult(*args, **kwargs):
return 'result'
yield
def someMessageTimeout(*args, **kwargs):
raise TimeoutException()
yield
self.observer.methods['someMessage'] = someMessageResult
result = retval(self.dna.any.someMessage('arg', kwarg='kwarg'))
self.assertEqual(['someMessage'], self.observer.calledMethodNames())
sleep(0.11)
self.observer.methods['someMessage'] = someMessageTimeout
result = retval(self.dna.any.someMessage('arg', kwarg='kwarg'))
self.assertEqual('result', result)
self.assertEqual(['someMessage', 'someMessage'], self.observer.calledMethodNames())
def testPutEmptyPut(self):
sp = SocketPool(reactor=CallTrace())
retval(sp.putSocketInPool(host='10.0.0.1', port=60000, sock=0))
retval(sp.putSocketInPool(host='10.0.0.1', port=60000, sock=1))
for i in reversed(range(2)):
self.assertEquals(
i, retval(sp.getPooledSocket(host='10.0.0.1', port=60000)))
self.assertEquals(
None, retval(sp.getPooledSocket(host='10.0.0.1', port=60000)))
retval(sp.putSocketInPool(host='10.0.0.1', port=60000, sock=2))
self.assertEquals(
2, retval(sp.getPooledSocket(host='10.0.0.1', port=60000)))
def testPlainXml(self):
client = HttpClient()
self.response = """HTTP/1.0 200 OK\r\nContent-Type: text/xml\r\n\r\n<xml/>"""
gen = client.httpGet(hostname='localhost', port=80, path='/', arguments={})
headers, body = retval(gen)
self.assertEquals('<xml/>', lxmltostring(body))
self.assertEquals(['HTTP/1.0 200 OK', 'Content-Type: text/xml'], headers.split(CRLF))
def testHttpsGet(self):
client = HttpClient()
self.response = """HTTP/1.0 200 OK\r\n\r\nother-data"""
gen = client.httpsGet(hostname='localhost', port=443, path='/', arguments={}, parse=False)
headers, body = retval(gen)
self.assertEquals('other-data', body)
self.assertEquals(['HTTP/1.0 200 OK'], headers.split(CRLF))
def testKeepValueInCaseOfError(self):
self.init(cacheTimeout=0.1, returnCachedValueInCaseOfException=True)
def someMessageResult(*args, **kwargs):
return 'result'
yield
def someMessageError(*args, **kwargs):
raise RuntimeError("could be any exception")
yield
self.observer.methods['someMessage'] = someMessageResult
result = retval(self.dna.any.someMessage('arg', kwarg='kwarg'))
self.assertEqual('result', result)
result = retval(self.dna.any.someMessage('arg', kwarg='kwarg'))
self.assertEqual('result', result)
self.assertEqual(['someMessage'], self.observer.calledMethodNames())
sleep(0.11)
self.observer.methods['someMessage'] = someMessageError
result = retval(self.dna.any.someMessage('arg', kwarg='kwarg'))
self.assertEqual('result', result)
self.assertEqual(['someMessage', 'someMessage'], self.observer.calledMethodNames())
def testDrilldownQuery(self):
result = retval(
self.dbdq.executeQuery(
extraArguments={'x-drilldown-query': ['a = b']}))
self.assertEquals('result', result)
self.assertEquals(['executeQuery'], self.observer.calledMethodNames())
executeQueryMethod = self.observer.calledMethods[0]
self.assertEquals([('a', ['b'])],
executeQueryMethod.kwargs['drilldownQueries'])
self.observer.calledMethods.reset()
result = retval(
self.dbdq.executeQuery(
extraArguments={'x-drilldown-query': ['a exact b']}))
self.assertEquals('result', result)
self.assertEquals(['executeQuery'], self.observer.calledMethodNames())
executeQueryMethod = self.observer.calledMethods[0]
self.assertEquals([('a', ['b'])],
executeQueryMethod.kwargs['drilldownQueries'])
def testEmptySocketPool_put_noErrors(self):
sok = MockSok('s')
sp = EmptySocketPool()
result = retval(sp.putSocketInPool(host='whatever', port=-1, sock=sok))
self.assertEquals(None, result)
self.assertEquals(['shutdown', 'close'], sok.log.calledMethodNames())
self.assertEquals(
((SHUT_RDWR, ), {}),
(sok.log.calledMethods[0].args, sok.log.calledMethods[0].kwargs))
self.assertEquals(
((), {}),
(sok.log.calledMethods[1].args, sok.log.calledMethods[1].kwargs))
def testSuggest(self):
self.response = dumps([
{"suggestion": "hallo", "type": "uri:book", "creator": 'by:me', "score": 1.0},
{"suggestion": "harry", "type": None, "creator": None, "score": 1.0}
])
suggestions = retval(self.sic.suggest("ha"))
self.assertEqual(1, len(self.post))
self.assertEqual('/suggest', self.post[0]['path'])
self.assertEqual({"value": "ha", "trigram": False, "filters": [], "keySetName": None, "limit": None}, loads(self.post[0]['data']))
self.assertEquals([u"hallo", u"harry"], [s.suggestion for s in suggestions])
self.assertEquals([u"uri:book", None], [s.type for s in suggestions])
self.assertEquals([u"by:me", None], [s.creator for s in suggestions])
def testSimilarDocs(self):
self.response = JsonDict({
"total": 887,
"queryTime": 6,
"times": {"searchTime": 3},
"hits": [
{"id": "record:1", "score": 0.1234},
{"id": "record:2", "score": 0.1234},
],
}).dumps()
response = retval(self._lucene.similarDocuments(identifier='record:3'))
self.assertEqual(887, response.total)
self.assertEqual(2, len(response.hits))
def testHttpPost(self):
client = HttpClient()
self.response = b"""HTTP/1.0 200 OK\r\n\r\nother-data"""
gen = client.httpPost(hostname='localhost',
port=80,
path='/',
data='data',
parse=False)
headers, body = retval(gen)
self.assertEqual('other-data', body)
self.assertEqual(['HTTP/1.0 200 OK'], headers.split(CRLF))
def testXSuggestionQueryToSuggestionRequest(self):
extraArguments = {'x-suggestionsQuery': ['query']}
suggestion = Suggestion(count=10, field='dcterms:title')
suggestion.addObserver(self.observer)
result = retval(
suggestion.executeQuery(kwarg='value',
extraArguments=extraArguments))
self.assertTrue(result == self.response)
self.assertEqual(['executeQuery'], self.observer.calledMethodNames())
methodKwargs = self.observer.calledMethods[0].kwargs
self.assertEqual(
dict(count=10, field='dcterms:title', suggests=['query']),
methodKwargs['suggestionRequest'])