def testNextRecordPosition(self): observer = CallTrace(emptyGeneratorMethods=['additionalDiagnosticDetails']) response = Response(total=100, hits=hitsRange(11, 26)) def executeQuery(**kwargs): raise StopIteration(response) yield def retrieveData(**kwargs): raise StopIteration('record') yield observer.methods['executeQuery'] = executeQuery observer.methods['retrieveData'] = retrieveData observer.methods['extraResponseData'] = lambda *a, **kw: (x for x in 'extraResponseData') observer.methods['echoedExtraRequestData'] = lambda *a, **kw: (x for x in 'echoedExtraRequestData') observer.methods['extraRecordData'] = lambda hit: (f for f in []) component = SruHandler() component.addObserver(observer) arguments = dict(startRecord=11, maximumRecords=15, query='query', recordPacking='string', recordSchema='schema') result = "".join(compose(component.searchRetrieve(sruArguments=arguments, **arguments))) self.assertTrue("<srw:nextRecordPosition>26</srw:nextRecordPosition>" in result, result) executeCqlCallKwargs = observer.calledMethods[0].kwargs self.assertEquals(10, executeCqlCallKwargs['start']) # SRU is 1 based self.assertEquals(25, executeCqlCallKwargs['stop'])
def testQueryTimeInExtraResponse(self): handler = SruHandler(includeQueryTimes=True) observer = CallTrace('observer', emptyGeneratorMethods=['echoedExtraRequestData', 'extraResponseData']) times = [1, 2.5, 3.5] def timeNow(): return times.pop(0) handler._timeNow = timeNow def executeQuery(**kwargs): response = Response(total=0, hits=[]) response.queryTime=5 raise StopIteration(response) yield observer.methods['executeQuery'] = executeQuery handler.addObserver(observer) arguments = dict(startRecord=11, maximumRecords=15, query='query', recordPacking='string', recordSchema='schema') result = "".join(compose(handler.searchRetrieve(sruArguments=arguments, **arguments))) sruResponse = parse(StringIO(result)) extraResponseData = sruResponse.xpath('/srw:searchRetrieveResponse/srw:extraResponseData', namespaces={'srw':"http://www.loc.gov/zing/srw/"})[0] self.assertEqualsWS("""<srw:extraResponseData %(xmlns_srw)s %(xmlns_diag)s %(xmlns_xcql)s %(xmlns_dc)s %(xmlns_meresco_srw)s> <querytimes xmlns="http://meresco.org/namespace/timing"> <sruHandling>PT2.500S</sruHandling> <sruQueryTime>PT1.500S</sruQueryTime> <index>PT0.005S</index> </querytimes> </srw:extraResponseData>""" % namespaces, lxmltostring(extraResponseData)) queryTimes = lxmltostring(extraResponseData.xpath('//ti:querytimes', namespaces={'ti':"http://meresco.org/namespace/timing"})[0]) assertValid(queryTimes, join(schemasPath, 'timing-20120827.xsd')) self.assertEquals(['executeQuery', 'echoedExtraRequestData', 'extraResponseData', 'handleQueryTimes'], observer.calledMethodNames()) self.assertEquals({'sru': Decimal("2.500"), 'queryTime': Decimal("1.500"), 'index': Decimal("0.005")}, observer.calledMethods[3].kwargs)
def testValidXml(self): component = SruParser() sruHandler = SruHandler() component.addObserver(sruHandler) observer = CallTrace('observer') sruHandler.addObserver(observer) response = Response(total=2, hits=[Hit('id0'), Hit('id1')]) def executeQuery(**kwargs): raise StopIteration(response) yield def retrieveData(**kwargs): raise StopIteration('<bike/>') yield observer.methods['executeQuery'] = executeQuery observer.returnValues['echoedExtraRequestData'] = (f for f in []) observer.returnValues['extraResponseData'] = (f for f in []) observer.methods['extraRecordData'] = lambda hit: (f for f in []) observer.methods['retrieveData'] = retrieveData result = ''.join(compose(component.handleRequest(arguments={'version':['1.1'], 'query': ['aQuery'], 'operation':['searchRetrieve']}))) header, body = result.split('\r\n'*2) assertValid(body, join(schemasPath, 'srw-types1.2.xsd')) self.assertTrue('<bike/>' in body, body) result = ''.join(compose(component.handleRequest(arguments={'version':['1.1'], 'operation':['searchRetrieve']}))) header, body = result.split('\r\n'*2) assertValid(body, join(schemasPath, 'srw-types1.2.xsd')) self.assertTrue('diagnostic' in body, body)
def setUp(self): SeecrTestCase.setUp(self) self.srw = Srw() self.sruParser = SruParser() self.sruHandler = SruHandler() self.srw.addObserver(self.sruParser) self.sruParser.addObserver(self.sruHandler) self.response = Response(total=1, hits=[Hit('0')]) def executeQuery(**kwargs): return self.response yield def retrieveData(**kwargs): return 'data' yield self.observer = CallTrace(methods={ 'executeQuery': executeQuery, 'retrieveData': retrieveData }, emptyGeneratorMethods=[ 'extraResponseData', 'echoedExtraRequestData', 'additionalDiagnosticDetails', 'extraRecordData' ]) self.sruHandler.addObserver(self.observer)
def testCollectLog(self): handler = SruHandler(enableCollectLog=True) observer = CallTrace('observer', emptyGeneratorMethods=['echoedExtraRequestData', 'extraResponseData']) __callstack_var_logCollector__ = dict() times = [1, 2.5, 3.5] def timeNow(): return times.pop(0) handler._timeNow = timeNow def executeQuery(**kwargs): response = Response(total=0, hits=[]) response.queryTime=5 raise StopIteration(response) yield observer.methods['executeQuery'] = executeQuery handler.addObserver(observer) arguments = dict(startRecord=11, maximumRecords=15, query='query', recordPacking='string', recordSchema='schema') consume(handler.searchRetrieve(sruArguments=arguments, **arguments)) self.assertEquals({ 'sru': { 'handlingTime': [Decimal('2.500')], 'queryTime': [Decimal('1.500')], 'indexTime': [Decimal('0.005')], 'numberOfRecords': [0], 'arguments': [{ 'startRecord': 11, 'query': 'query', 'recordPacking': 'string', 'maximumRecords': 15, 'recordSchema': 'schema', }], } }, __callstack_var_logCollector__)
def testCollectLogWhenIndexRaisesError(self): handler = SruHandler(enableCollectLog=True) observer = CallTrace('observer', emptyGeneratorMethods=['echoedExtraRequestData', 'extraResponseData', 'additionalDiagnosticDetails']) __callstack_var_logCollector__ = dict() times = [1] def timeNow(): return times.pop(0) handler._timeNow = timeNow def executeQuery(**kwargs): raise Exception('Sorry') yield observer.methods['executeQuery'] = executeQuery handler.addObserver(observer) arguments = dict(startRecord=11, maximumRecords=15, query='query', recordPacking='string', recordSchema='schema') consume(handler.searchRetrieve(sruArguments=arguments, **arguments)) self.assertEquals({ 'sru': { 'arguments': [{ 'startRecord': 11, 'query': 'query', 'recordPacking': 'string', 'maximumRecords': 15, 'recordSchema': 'schema', }], } }, __callstack_var_logCollector__)
def testExtraResponseDataHandlerNoData(self): class TestHandler: def extraResponseData(self, *args, **kwargs): return (f for f in []) component = SruHandler() component.addObserver(TestHandler()) result = "".join(list(component._writeExtraResponseData(cqlAbstractSyntaxTree=None, **MOCKDATA))) self.assertEquals('' , result)
def testExceptionInWriteExtraRecordData(self): class RaisesException(object): def extraResponseData(self, *args, **kwargs): raise Exception("Test Exception") component = SruHandler() component.addObserver(RaisesException()) result = "".join(compose(component._writeExtraResponseData(cqlAbstractSyntaxTree=None, **MOCKDATA))) self.assertTrue("<uri>info://srw/diagnostics/1/1</uri>" in result) self.assertTrue("<message>General System Error</message>" in result) self.assertTrue("<details>Test Exception</details>" in result)
def testExtraResponseDataWithTermDrilldown(self): sruHandler = SruHandler() sruTermDrilldown = SRUTermDrilldown() drilldownData = [ {'fieldname': 'field0', 'terms': [{'term': 'value0_0', 'count': 14}]}, {'fieldname': 'field1', 'terms': [{'term': 'value1_0', 'count': 13}, {'term': 'value1_1', 'count': 11}]}, {'fieldname': 'field2', 'terms': [{'term': 'value2_0', 'count': 3}, {'term': 'value2_1', 'count': 2}, {'term': 'value2_2', 'count': 1}]} ] sruHandler.addObserver(sruTermDrilldown) result = "".join(sruHandler._writeExtraResponseData(drilldownData=drilldownData, sruArguments={}, **MOCKDATA)) self.assertEqualsWS("""<srw:extraResponseData><dd:drilldown\n xmlns:dd="http://meresco.org/namespace/drilldown"\n xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"\n xsi:schemaLocation="http://meresco.org/namespace/drilldown http://meresco.org/files/xsd/drilldown-20070730.xsd"><dd:term-drilldown><dd:navigator name="field0"><dd:item count="14">value0_0</dd:item></dd:navigator><dd:navigator name="field1"><dd:item count="13">value1_0</dd:item><dd:item count="11">value1_1</dd:item></dd:navigator><dd:navigator name="field2"><dd:item count="3">value2_0</dd:item><dd:item count="2">value2_1</dd:item><dd:item count="1">value2_2</dd:item></dd:navigator></dd:term-drilldown></dd:drilldown></srw:extraResponseData>""" , result)
def testEchoedSearchRetrieveRequest(self): sruArguments = {'version':'1.1', 'operation':'searchRetrieve', 'query':'query >= 3', 'recordSchema':'schema', 'recordPacking':'string'} component = SruHandler() result = "".join(list(component._writeEchoedSearchRetrieveRequest(sruArguments=sruArguments))) self.assertEqualsWS("""<srw:echoedSearchRetrieveRequest> <srw:version>1.1</srw:version> <srw:query>query >= 3</srw:query> <srw:recordPacking>string</srw:recordPacking> <srw:recordSchema>schema</srw:recordSchema> </srw:echoedSearchRetrieveRequest>""", result)
def testExceptionInWriteRecordData(self): observer = CallTrace(emptyGeneratorMethods=['additionalDiagnosticDetails']) def retrieveData(**kwargs): raise Exception("Test Exception") yield observer.methods["retrieveData"] = retrieveData component = SruHandler() component.addObserver(observer) result = "".join(list(compose(component._writeRecordData(recordPacking="string", recordSchema="schema", recordId="ID")))) self.assertTrue("<uri>info://srw/diagnostics/1/1</uri>" in result) self.assertTrue("<message>General System Error</message>" in result) self.assertTrue("<details>Test Exception</details>" in result)
def testDiagnosticOnExecuteCql(self): with stderr_replaced(): class RaisesException(object): def executeQuery(self, *args, **kwargs): raise Exception("Test Exception") component = SruHandler() component.addObserver(RaisesException()) arguments = dict(startRecord=11, maximumRecords=15, query='query', recordPacking='string', recordSchema='schema') result = parse(StringIO("".join(compose(component.searchRetrieve(sruArguments=arguments, **arguments))))) diagnostic = result.xpath("/srw:searchRetrieveResponse/srw:diagnostics/diag:diagnostic", namespaces=namespaces) self.assertEquals(1, len(diagnostic)) self.assertEquals(["info://srw/diagnostics/1/48"], diagnostic[0].xpath("diag:uri/text()", namespaces=namespaces)) self.assertEquals(["Query Feature Unsupported"], diagnostic[0].xpath("diag:message/text()", namespaces=namespaces)) self.assertEquals(["Test Exception"], diagnostic[0].xpath("diag:details/text()", namespaces=namespaces))
def testQuery(self): def executeQuery(**kwargs): return Response(total=42) yield index = CallTrace('index', emptyGeneratorMethods=['echoedExtraRequestData', 'extraResponseData'], methods=dict(executeQuery=executeQuery)) observable = be((Observable(), (LogCollector(), (self.handleRequestLog, (SruParser(), (SruHandler(enableCollectLog=True), (index,) ) ) ), (self.queryLogWriter,), ) )) result = asString(observable.all.handleRequest( Method='GET', Client=('127.0.0.1', 1234), arguments={ 'version': ['1.2'], 'operation': ['searchRetrieve'], 'query': ['query'], 'maximumRecords': ['0'], }, path='/path/sru', otherKwarg='value')) self.assertTrue('<srw:numberOfRecords>42</srw:numberOfRecords>' in result, result) self.assertTrue(isfile(join(self.tempdir, '2009-11-02-query.log'))) with open(join(self.tempdir, '2009-11-02-query.log')) as fp: self.assertEqual('2009-11-02T11:25:37Z 127.0.0.1 0.7K 1.000s 42hits /path/sru maximumRecords=0&operation=searchRetrieve&query=query&recordPacking=xml&recordSchema=dc&startRecord=1&version=1.2\n', fp.read())
def setUp(self): SeecrTestCase.setUp(self) self.srw = Srw() self.sruParser = SruParser() self.sruHandler = SruHandler() self.srw.addObserver(self.sruParser) self.sruParser.addObserver(self.sruHandler) self.response = StopIteration(Response(total=1, hits=[Hit('0')])) def executeQuery(**kwargs): raise self.response yield def retrieveData(**kwargs): raise StopIteration('data') yield self.observer = CallTrace( methods={ 'executeQuery': executeQuery, 'retrieveData': retrieveData }, emptyGeneratorMethods=[ 'extraResponseData', 'echoedExtraRequestData', 'additionalDiagnosticDetails', 'extraRecordData' ]) self.sruHandler.addObserver(self.observer)
def testAllQueryHelpersForSRU(self): index = CallTrace('index') def executeQuery(**kwargs): return Response(total=3201, hits=[]) yield index.methods['executeQuery'] = executeQuery index.ignoredAttributes.extend( ['echoedExtraRequestData', 'extraResponseData', 'all_unknown']) server = be(( Observable(), (self.queryLog, (SruParser(), (QueryLogHelperForSru(), (SruHandler(extraRecordDataNewStyle=True), (QueryLogHelperForExecuteCQL(), (index, )))))), )) ''.join( compose( server.all.handleRequest( path='/path/sru', Client=('11.22.33.44', 8080), arguments={ 'operation': ['searchRetrieve'], 'version': ['1.2'], 'maximumRecords': ['0'], 'query': ['field=value'], }, ))) with open(join(self.tempdir, '2009-11-02-query.log')) as fp: self.assertEqual( '2009-11-02T11:25:37Z 11.22.33.44 0.7K 1.000s 3201hits /path/sru maximumRecords=0&operation=searchRetrieve&query=field%3Dvalue&recordPacking=xml&recordSchema=dc&startRecord=1&version=1.2\n', fp.read())
def testDiagnosticWarning(self): sruArguments = {'version':'1.2', 'operation':'searchRetrieve', 'recordSchema':'schema', 'recordPacking':'xml', 'query':'field=value', 'startRecord':1, 'maximumRecords':2, } queryArguments = {'version':'1.2', 'operation':'searchRetrieve', 'recordSchema':'schema', 'recordPacking':'xml', 'query':'field=value', 'startRecord':1, 'maximumRecords':2} observer = CallTrace(emptyGeneratorMethods=['additionalDiagnosticDetails']) response = Response(total=100, hits=[Hit('<aap&noot>'), Hit('vuur')]) def executeQuery(**kwargs): raise StopIteration(response) yield observer.methods['executeQuery'] = executeQuery retrieveDataCalls = [] def retrieveData(identifier, name): retrieveDataCalls.append(1) raise StopIteration("<MOCKED_WRITTEN_DATA>%s-%s</MOCKED_WRITTEN_DATA>" % (xmlEscape(identifier), name)) yield observer.retrieveData = retrieveData observer.methods['extraResponseData'] = lambda *a, **kw: (x for x in 'extraResponseData') observer.methods['echoedExtraRequestData'] = lambda *a, **kw: (x for x in 'echoedExtraRequestData') observer.methods['extraRecordData'] = lambda hit: (f for f in []) component = SruHandler() component.addObserver(observer) result = "".join(compose(component.searchRetrieve(sruArguments=sruArguments, diagnostics=[(998, 'Diagnostic 998', 'The <tag> message'), (999, 'Diagnostic 999', 'Some message')], **queryArguments))) response = parse(StringIO(result)) self.assertEquals([t % namespaces for t in [ '{%(srw)s}version', '{%(srw)s}numberOfRecords', '{%(srw)s}records', '{%(srw)s}nextRecordPosition', '{%(srw)s}echoedSearchRetrieveRequest', '{%(srw)s}diagnostics', '{%(srw)s}extraResponseData', ]], [t.tag for t in xpath(response, '//srw:searchRetrieveResponse/*')]) diagnostics = [{'uri': xpath(d, 'diag:uri/text()')[0], 'details': xpath(d, 'diag:details/text()')[0], 'message': xpath(d, 'diag:message/text()')[0]} for d in xpath(response, '/srw:searchRetrieveResponse/srw:diagnostics/diag:diagnostic')] self.assertEquals([ {'uri': 'info://srw/diagnostics/1/998', 'message': 'Diagnostic 998', 'details': 'The <tag> message'}, {'uri': 'info://srw/diagnostics/1/999', 'message': 'Diagnostic 999', 'details': 'Some message'}, ], diagnostics)
def testExtraResponseDataHandlerWithData(self): argsUsed = [] kwargsUsed = {} class TestHandler: def extraResponseData(self, *args, **kwargs): argsUsed.append(args) kwargsUsed.update(kwargs) return (f for f in ["<someD", "ata/>"]) component = SruHandler() component.addObserver(TestHandler()) result = "".join(list(component._writeExtraResponseData(cqlAbstractSyntaxTree=None, **MOCKDATA))) self.assertEquals('<srw:extraResponseData><someData/></srw:extraResponseData>' , result) self.assertEquals([()], argsUsed) self.assertEquals(None, kwargsUsed['cqlAbstractSyntaxTree']) self.assertEquals(MOCKDATA['queryTime'], kwargsUsed['queryTime']) self.assertEquals(MOCKDATA['response'], kwargsUsed['response'])
def testExtraRecordDataOldStyle(self): queryArguments = {'version':'1.2', 'operation':'searchRetrieve', 'recordSchema':'schema', 'recordPacking':'xml', 'query':'field=value', 'startRecord':1, 'maximumRecords':2} sruArguments = {'version':'1.2', 'operation':'searchRetrieve', 'recordSchema':'schema', 'recordPacking':'xml', 'query':'field=value', 'startRecord':1, 'maximumRecords':2, 'x-recordSchema':['extra', 'evenmore']} observer = CallTrace() response = Response(total=100, hits=[Hit('11')]) def executeQuery(**kwargs): raise StopIteration(response) yield observer.methods['executeQuery'] = executeQuery retrieveDataCalls = [] def retrieveData(identifier, name): retrieveDataCalls.append(1) raise StopIteration("<MOCKED_WRITTEN_DATA>%s-%s</MOCKED_WRITTEN_DATA>" % (identifier, name)) yield observer.retrieveData = retrieveData observer.methods['extraResponseData'] = lambda *a, **kw: (x for x in 'extraResponseData') observer.methods['echoedExtraRequestData'] = lambda *a, **kw: (x for x in 'echoedExtraRequestData') observer.methods['extraRecordData'] = lambda hit: (f for f in []) component = SruHandler(extraRecordDataNewStyle=False) component.addObserver(observer) result = "".join(compose(component.searchRetrieve(sruArguments=sruArguments, **queryArguments))) strippedResult = result[result.index('<srw:record>'):result.index('</srw:records>')] self.assertEqualsWS("""<srw:record> <srw:recordSchema>schema</srw:recordSchema> <srw:recordPacking>xml</srw:recordPacking> <srw:recordIdentifier>11</srw:recordIdentifier> <srw:recordData> <MOCKED_WRITTEN_DATA>11-schema</MOCKED_WRITTEN_DATA> </srw:recordData> <srw:extraRecordData> <recordData recordSchema="extra"> <MOCKED_WRITTEN_DATA>11-extra</MOCKED_WRITTEN_DATA> </recordData> <recordData recordSchema="evenmore"> <MOCKED_WRITTEN_DATA>11-evenmore</MOCKED_WRITTEN_DATA> </recordData> </srw:extraRecordData> </srw:record>""", strippedResult)
def testGetDataWithAdapter(self): observer = CallTrace(returnValues=dict(getData='<record/>')) adapter = RetrieveToGetDataAdapter() handler = SruHandler() handler.addObserver(adapter) adapter.addObserver(observer) response = Response(total=100, hits=hitsRange(1, 3)) def executeQuery(**kwargs): raise StopIteration(response) yield observer.methods['executeQuery'] = executeQuery observer.methods['extraResponseData'] = lambda *a, **kw: (x for x in 'extraResponseData') observer.methods['echoedExtraRequestData'] = lambda *a, **kw: (x for x in 'echoedExtraRequestData') observer.methods['extraRecordData'] = lambda hit: (f for f in []) response = asString(handler.searchRetrieve(query="word", recordSchema='schema', recordPacking='string', maximumRecords=2, sruArguments={})) self.assertEquals(['getData', 'getData'], observer.calledMethodNames()[1:4:2]) self.assertTrue('<srw:recordData><record/></srw:recordData>' in response, response) self.assertEquals({'identifier': '1', 'name': 'schema'}, observer.calledMethods[1].kwargs) self.assertEquals({'identifier': '2', 'name': 'schema'}, observer.calledMethods[3].kwargs)
def testExecuteQueryGetsRecordSchemaAsOnOfTheKwargs(self): sruArguments = {'version':'1.2', 'operation':'searchRetrieve', 'recordSchema':'schema', 'recordPacking':'xml', 'query':'field=value', 'startRecord':1, 'maximumRecords':2, 'x-recordSchema':['extra', 'evenmore'], 'x-extra-key': 'extraValue'} queryArguments = {'version':'1.2', 'operation':'searchRetrieve', 'recordSchema':'schema', 'recordPacking':'xml', 'query':'field=value', 'startRecord':1, 'maximumRecords':2} observer = CallTrace(emptyGeneratorMethods=['echoedExtraRequestData', 'extraResponseData']) response = Response(total=0, hits=[]) def executeQuery(**kwargs): raise StopIteration(response) yield observer.methods['executeQuery'] = executeQuery component = SruHandler() component.addObserver(observer) consume(component.searchRetrieve(sruArguments=sruArguments, **queryArguments)) self.assertEqual(['executeQuery', 'echoedExtraRequestData', 'extraResponseData'], observer.calledMethodNames()) queryKwargs = observer.calledMethods[0].kwargs self.assertEqual('schema', queryKwargs['recordSchema']) self.assertEqual(['extra', 'evenmore'], queryKwargs['extraArguments']['x-recordSchema'])
def testNextRecordPositionNotShownIfAfterLimitBeyond(self): observer = CallTrace(emptyGeneratorMethods=['additionalDiagnosticDetails']) response = Response(total=100, hits=hitsRange(10, 11)) def executeQuery(**kwargs): raise StopIteration(response) yield def retrieveData(**kwargs): raise StopIteration('record') yield observer.methods['executeQuery'] = executeQuery observer.methods['retrieveData'] = retrieveData observer.methods['extraResponseData'] = lambda *a, **kw: (x for x in 'extraResponseData') observer.methods['echoedExtraRequestData'] = lambda *a, **kw: (x for x in 'echoedExtraRequestData') observer.methods['extraRecordData'] = lambda hit: (f for f in []) component = SruHandler() component.addObserver(observer) arguments = dict(startRecord=10, maximumRecords=2, query='query', recordPacking='string', recordSchema='schema', limitBeyond=10) result = "".join(compose(component.searchRetrieve(sruArguments=arguments, **arguments))) self.assertFalse("<srw:nextRecordPosition>" in result, result)
def testParseDrilldownArguments(self): handler = SruHandler(drilldownSortBy='count') self.assertEquals(None, handler._parseDrilldownArgs([])) self.assertEquals([], handler._parseDrilldownArgs([''])) self.assertEquals([{'fieldname':'field', 'maxTerms':10, 'sortBy':'count'}], handler._parseDrilldownArgs(['field'])) self.assertEquals([{'fieldname':'field', 'maxTerms':10, 'sortBy':'count'}], handler._parseDrilldownArgs(['field,'])) self.assertEquals([{'fieldname':'field', 'maxTerms':20, 'sortBy':'count'}], handler._parseDrilldownArgs(['field:20'])) self.assertEquals([{'fieldname':'field', 'maxTerms':20, 'sortBy':'count'}, {'fieldname':'field2', 'maxTerms':10, 'sortBy':'count'}], handler._parseDrilldownArgs(['field:20,field2'])) self.assertEquals([{'fieldname':'field', 'maxTerms':20, 'sortBy':'count'}, {'fieldname':'field2', 'maxTerms':10, 'sortBy':'count'}], handler._parseDrilldownArgs(['field:20','field2']))
def testDrilldownResultInExecuteQuery(self): observer = CallTrace() response = Response(total=100, hits=hitsRange(11, 26)) drilldownData = iter([ ('field0', iter([('value0_0', 14)])), ('field1', iter([('value1_0', 13), ('value1_1', 11)])), ('field2', iter([('value2_0', 3), ('value2_1', 2), ('value2_2', 1)]))]) response.drilldownData = drilldownData def executeQuery(**kwargs): raise StopIteration(response) yield def retrieveData(**kwargs): raise StopIteration('record') yield observer.methods['executeQuery'] = executeQuery observer.methods['retrieveData'] = retrieveData observer.methods['extraResponseData'] = lambda *a, **kw: (x for x in 'extraResponseData') observer.methods['echoedExtraRequestData'] = lambda *a, **kw: (x for x in 'echoedExtraRequestData') observer.methods['extraRecordData'] = lambda hit: (f for f in []) component = SruHandler(drilldownSortBy='somevalue') component.addObserver(observer) queryArguments = dict(startRecord=11, maximumRecords=15, query='query', recordPacking='string', recordSchema='schema') sruArguments = queryArguments sruArguments['x-term-drilldown'] = ["field0:1,fie:ld1:2,field2,fie:ld3"] consume(component.searchRetrieve(sruArguments=sruArguments, **queryArguments)) self.assertEquals(['executeQuery'] + ['retrieveData', 'extraRecordData'] * 15 + ['echoedExtraRequestData', 'extraResponseData'], [m.name for m in observer.calledMethods]) self.assertEquals([ dict(fieldname='field0', maxTerms=1, sortBy='somevalue'), dict(fieldname='fie:ld1', maxTerms=2, sortBy='somevalue'), dict(fieldname='field2', maxTerms=DEFAULT_MAXIMUM_TERMS, sortBy='somevalue'), dict(fieldname='fie:ld3', maxTerms=DEFAULT_MAXIMUM_TERMS, sortBy='somevalue') ], list(observer.calledMethods[0].kwargs['facets'])) extraResponseDataMethod = observer.calledMethods[-1] self.assertEquals(response, extraResponseDataMethod.kwargs['response'])
def testSearchRetrieveAssertsDrilldownMaximumMaximumResultsWhenSet(self): drilldownMaximumMaximumResults = 3 self.assertTrue(drilldownMaximumMaximumResults < DEFAULT_MAXIMUM_TERMS) def sruHandlerKwargs(x_term_drilldown): arguments = {'version':'1.1', 'operation':'searchRetrieve', 'query':'blissfully_ignored', 'recordSchema':'blissfully_ignored', 'recordPacking':'string'} arguments['x_term_drilldown'] = [x_term_drilldown] arguments['sruArguments'] = dict((k.replace('_', '-'),v) for k,v in arguments.items()) return arguments # No problem - max kwargs = sruHandlerKwargs(x_term_drilldown='field0:3,fielddefault') sruHandler = SruHandler(drilldownMaximumMaximumResults=drilldownMaximumMaximumResults) observer = CallTrace('observer') sruHandler.addObserver(observer) def executeQuery(**kwargs): raise KeyboardInterrupt('Ok') yield observer.methods['executeQuery'] = executeQuery try: ''.join(compose(sruHandler.searchRetrieve(**kwargs))) except KeyboardInterrupt, e: self.assertEquals('Ok', str(e))
def testEchoedSearchRetrieveRequestWithExtraRequestData(self): sruArguments = {'version':'1.1', 'operation':'searchRetrieve', 'query':'query >= 3', 'recordSchema':'schema', 'recordPacking':'string', 'x-term-drilldown':['field0,field1']} observer = CallTrace('ExtraRequestData') observer.methods['echoedExtraRequestData'] = lambda *a, **kw: (x for x in '<some>extra request data</some>') component = SruHandler() component.addObserver(SRUTermDrilldown()) component.addObserver(observer) result = "".join(list(component._writeEchoedSearchRetrieveRequest(sruArguments=sruArguments))) drilldownRequestData = DRILLDOWN_HEADER \ + """<dd:term-drilldown>field0,field1</dd:term-drilldown>"""\ + DRILLDOWN_FOOTER self.assertEqualsWS("""<srw:echoedSearchRetrieveRequest> <srw:version>1.1</srw:version> <srw:query>query >= 3</srw:query> <srw:recordPacking>string</srw:recordPacking> <srw:recordSchema>schema</srw:recordSchema> <srw:extraRequestData>%s<some>extra request data</some></srw:extraRequestData> </srw:echoedSearchRetrieveRequest>""" % drilldownRequestData, result)
def testExtraResponseDataHandlerNoHandler(self): component = SruHandler() result = "".join(list(component._writeExtraResponseData(cqlAbstractSyntaxTree=None, **MOCKDATA))) self.assertEquals('' , result)
def main(reactor, port, statePath, lucenePort, **ignored): ######## START Lucene Integration ############################################################### defaultLuceneSettings = LuceneSettings( commitTimeout=30, readonly=True,) http11Request = be( (HttpRequest1_1(), (SocketPool(reactor=reactor, unusedTimeout=5, limits=dict(totalSize=100, destinationSize=10)),), ) ) luceneIndex = luceneAndReaderConfig(defaultLuceneSettings.clone(readonly=True), http11Request, lucenePort) luceneRoHelix = be( (AdapterToLuceneQuery( defaultCore=DEFAULT_CORE, coreConverters={ DEFAULT_CORE: QueryExpressionToLuceneQueryDict(UNQUALIFIED_TERM_FIELDS, luceneSettings=luceneIndex.settings), } ), (MultiLucene(host='127.0.0.1', port=lucenePort, defaultCore=DEFAULT_CORE), (luceneIndex,), (http11Request,), ) ) ) ######## END Lucene Integration ############################################################### fieldnameRewrites = {} def fieldnameRewrite(name): return fieldnameRewrites.get(name, name) def drilldownFieldnamesTranslate(fieldname): untokenizedName = untokenizedFieldname(fieldname) if untokenizedName in untokenizedFieldnames: fieldname = untokenizedName return fieldnameRewrite(fieldname) convertToComposedQuery = ConvertToComposedQuery( resultsFrom=DEFAULT_CORE, matches=[], drilldownFieldnamesTranslate=drilldownFieldnamesTranslate ) strategie = Md5HashDistributeStrategy() storage = StorageComponent(join(statePath, 'store'), strategy=strategie, partsRemovedOnDelete=[HEADER_PARTNAME, META_PARTNAME, METADATA_PARTNAME, OAI_DC_PARTNAME, LONG_PARTNAME, SHORT_PARTNAME]) # Wat doet dit? cqlClauseConverters = [ RenameFieldForExact( untokenizedFields=untokenizedFieldnames, untokenizedPrefix=UNTOKENIZED_PREFIX, ).filterAndModifier(), SearchTermFilterAndModifier( shouldModifyFieldValue=lambda *args: True, fieldnameModifier=fieldnameRewrite ).filterAndModifier(), ] executeQueryHelix = \ (FilterMessages(allowed=['executeQuery']), (CqlMultiSearchClauseConversion(cqlClauseConverters, fromKwarg='query'), (DrilldownQueries(), (convertToComposedQuery, (luceneRoHelix,), ) ) ) ) return \ (Observable(), (ObservableHttpServer(reactor, port, compressResponse=True), (BasicHttpHandler(), (PathFilter(['/sru']), (SruParser( host='sru.narcis.nl', port=80, defaultRecordSchema='knaw_short', defaultRecordPacking='xml'), (SruLimitStartRecord(limitBeyond=4000), (SruHandler( includeQueryTimes=False, extraXParameters=[], enableCollectLog=False), #2017-03-24T12:00:33Z 127.0.0.1 3.5K 0.019s - /sru OF (TRUE): 2017-03-24T11:58:53Z 127.0.0.1 2.3K 0.004s 1hits /sru maximumRecords=10&operation=searchRetrieve&query=untokenized.dd_year+exact+%221993%22&recordPacking=xml&recordSchema=knaw_short&startRecord=1&version=1.2 (SruTermDrilldown(),), executeQueryHelix, (StorageAdapter(), (storage,) ) ) ) ) ), (PathFilter('/rss'), (Rss( supportedLanguages = ['nl','en'], # defaults to first, if requested language is not available or supplied. title = {'nl':'NARCIS', 'en':'NARCIS'}, description = {'nl':'NARCIS: De toegang tot de Nederlandse wetenschapsinformatie', 'en':'NARCIS: The gateway to Dutch scientific information'}, link = {'nl':'http://www.narcis.nl/?Language=nl', 'en':'http://www.narcis.nl/?Language=en'}, maximumRecords = 20), executeQueryHelix, (RssItem( nsMap=NAMESPACEMAP, title = ('knaw_short', {'nl':'//short:metadata/short:titleInfo[not (@xml:lang)]/short:title/text()', 'en':'//short:metadata/short:titleInfo[@xml:lang="en"]/short:title/text()'}), description = ('knaw_short', {'nl':'//short:abstract[not (@xml:lang)]/text()', 'en':'//short:abstract[@xml:lang="en"]/text()'}), pubdate = ('knaw_short', '//short:dateIssued/short:parsed/text()'), linkTemplate = 'http://www.narcis.nl/%(wcpcollection)s/RecordID/%(oai_identifier)s/Language/%(language)s', wcpcollection = ('meta', '//*[local-name() = "collection"]/text()'), oai_identifier = ('meta', '//meta:record/meta:id/text()'), language = ('Dummy: Language is auto provided by the calling RSS component, but needs to be present to serve the linkTemplate.') ), (StorageAdapter(), (storage,) ) ) ) ) ) ) )
def testSearchRetrieveVersion12(self): sruArguments = {'version':'1.2', 'operation':'searchRetrieve', 'recordSchema':'schema', 'recordPacking':'xml', 'query':'field=value', 'startRecord':1, 'maximumRecords':2, 'x-recordSchema':['extra', 'evenmore'], 'x-extra-key': 'extraValue'} queryArguments = {'version':'1.2', 'operation':'searchRetrieve', 'recordSchema':'schema', 'recordPacking':'xml', 'query':'field=value', 'startRecord':1, 'maximumRecords':2} observer = CallTrace() response = Response(total=100, hits=[Hit('<aap&noot>'), Hit('vuur')]) def executeQuery(**kwargs): raise StopIteration(response) yield observer.methods['executeQuery'] = executeQuery retrieveDataCalls = [] def retrieveData(identifier, name): retrieveDataCalls.append(1) raise StopIteration("<MOCKED_WRITTEN_DATA>%s-%s</MOCKED_WRITTEN_DATA>" % (xmlEscape(identifier), name)) yield observer.retrieveData = retrieveData observer.methods['extraResponseData'] = lambda *a, **kw: (x for x in 'extraResponseData') observer.methods['echoedExtraRequestData'] = lambda *a, **kw: (x for x in 'echoedExtraRequestData') observer.methods['extraRecordData'] = lambda hit: (f for f in []) component = SruHandler() component.addObserver(observer) result = "".join(compose(component.searchRetrieve(sruArguments=sruArguments, **queryArguments))) self.assertEquals(['executeQuery', 'extraRecordData', 'extraRecordData', 'echoedExtraRequestData', 'extraResponseData'], [m.name for m in observer.calledMethods]) executeQueryMethod, extraRecordData1, extraRecordData2, echoedExtraRequestDataMethod, extraResponseDataMethod = observer.calledMethods self.assertEquals('executeQuery', executeQueryMethod.name) methodKwargs = executeQueryMethod.kwargs self.assertEquals(cqlToExpression('field=value'), methodKwargs['query']) self.assertEquals(0, methodKwargs['start']) self.assertEquals(2, methodKwargs['stop']) self.assertEquals({'x-recordSchema': ['extra', 'evenmore'], 'x-extra-key': 'extraValue'}, methodKwargs['extraArguments']) self.assertEquals('<aap&noot>', extraRecordData1.kwargs['hit'].id) self.assertEquals('vuur', extraRecordData2.kwargs['hit'].id) self.assertEquals(6, sum(retrieveDataCalls)) resultXml = parse(StringIO(result)) ids = resultXml.xpath('//srw:recordIdentifier/text()', namespaces={'srw':"http://www.loc.gov/zing/srw/"}) self.assertEquals(['<aap&noot>', 'vuur'], ids) self.assertEqualsWS(""" <srw:searchRetrieveResponse %(xmlns_srw)s %(xmlns_diag)s %(xmlns_xcql)s %(xmlns_dc)s %(xmlns_meresco_srw)s> <srw:version>1.2</srw:version> <srw:numberOfRecords>100</srw:numberOfRecords> <srw:records> <srw:record> <srw:recordSchema>schema</srw:recordSchema> <srw:recordPacking>xml</srw:recordPacking> <srw:recordIdentifier><aap&noot></srw:recordIdentifier> <srw:recordData> <MOCKED_WRITTEN_DATA><aap&noot>-schema</MOCKED_WRITTEN_DATA> </srw:recordData> <srw:extraRecordData> <srw:record> <srw:recordSchema>extra</srw:recordSchema> <srw:recordPacking>xml</srw:recordPacking> <srw:recordData> <MOCKED_WRITTEN_DATA><aap&noot>-extra</MOCKED_WRITTEN_DATA> </srw:recordData> </srw:record> <srw:record> <srw:recordSchema>evenmore</srw:recordSchema> <srw:recordPacking>xml</srw:recordPacking> <srw:recordData> <MOCKED_WRITTEN_DATA><aap&noot>-evenmore</MOCKED_WRITTEN_DATA> </srw:recordData> </srw:record> </srw:extraRecordData> </srw:record> <srw:record> <srw:recordSchema>schema</srw:recordSchema> <srw:recordPacking>xml</srw:recordPacking> <srw:recordIdentifier>vuur</srw:recordIdentifier> <srw:recordData> <MOCKED_WRITTEN_DATA>vuur-schema</MOCKED_WRITTEN_DATA> </srw:recordData> <srw:extraRecordData> <srw:record> <srw:recordSchema>extra</srw:recordSchema> <srw:recordPacking>xml</srw:recordPacking> <srw:recordData> <MOCKED_WRITTEN_DATA>vuur-extra</MOCKED_WRITTEN_DATA> </srw:recordData> </srw:record> <srw:record> <srw:recordSchema>evenmore</srw:recordSchema> <srw:recordPacking>xml</srw:recordPacking> <srw:recordData> <MOCKED_WRITTEN_DATA>vuur-evenmore</MOCKED_WRITTEN_DATA> </srw:recordData> </srw:record> </srw:extraRecordData> </srw:record> </srw:records> <srw:nextRecordPosition>3</srw:nextRecordPosition> <srw:echoedSearchRetrieveRequest> <srw:version>1.2</srw:version> <srw:query>field=value</srw:query> <srw:startRecord>1</srw:startRecord> <srw:maximumRecords>2</srw:maximumRecords> <srw:recordPacking>xml</srw:recordPacking> <srw:recordSchema>schema</srw:recordSchema> <srw:x-recordSchema>extra</srw:x-recordSchema> <srw:x-recordSchema>evenmore</srw:x-recordSchema> <srw:extraRequestData>echoedExtraRequestData</srw:extraRequestData> </srw:echoedSearchRetrieveRequest> <srw:extraResponseData>extraResponseData</srw:extraResponseData> </srw:searchRetrieveResponse> """ % namespaces, result) self.assertEquals((), echoedExtraRequestDataMethod.args) self.assertEquals(set(['version', 'recordSchema', 'x-recordSchema', 'maximumRecords', 'startRecord', 'query', 'operation', 'recordPacking', 'x-extra-key']), set(echoedExtraRequestDataMethod.kwargs['sruArguments'].keys())) self.assertEquals((), extraResponseDataMethod.args) self.assertEquals(sorted(['version', 'recordSchema', 'maximumRecords', 'startRecord', 'query', 'operation', 'recordPacking', 'response', 'drilldownData', 'queryTime', 'sruArguments']), sorted(extraResponseDataMethod.kwargs.keys()))
def main(reactor, port, statePath, lucenePort, gatewayPort, quickCommit=False, **ignored): ######## START Lucene Integration ############################################################### defaultLuceneSettings = LuceneSettings( commitTimeout=30, readonly=True, ) http11Request = be(( HttpRequest1_1(), (SocketPool(reactor=reactor, unusedTimeout=5, limits=dict(totalSize=100, destinationSize=10)), ), )) luceneIndex = luceneAndReaderConfig( defaultLuceneSettings.clone(readonly=True), http11Request, lucenePort) luceneRoHelix = be( (AdapterToLuceneQuery(defaultCore=DEFAULT_CORE, coreConverters={ DEFAULT_CORE: QueryExpressionToLuceneQueryDict( UNQUALIFIED_TERM_FIELDS, luceneSettings=luceneIndex.settings), }), ( MultiLucene(host='localhost', port=lucenePort, defaultCore=DEFAULT_CORE), (luceneIndex, ), (http11Request, ), ))) ######## END Lucene Integration ############################################################### fieldnameRewrites = { # UNTOKENIZED_PREFIX+'genre': UNTOKENIZED_PREFIX+'dc:genre', } def fieldnameRewrite(name): return fieldnameRewrites.get(name, name) def drilldownFieldnamesTranslate(fieldname): untokenizedName = untokenizedFieldname(fieldname) if untokenizedName in untokenizedFieldnames: fieldname = untokenizedName return fieldnameRewrite(fieldname) convertToComposedQuery = ConvertToComposedQuery( resultsFrom=DEFAULT_CORE, matches=[], drilldownFieldnamesTranslate=drilldownFieldnamesTranslate) strategie = Md5HashDistributeStrategy() storage = StorageComponent(join(statePath, 'store'), strategy=strategie, partsRemovedOnDelete=[ HEADER_PARTNAME, META_PARTNAME, METADATA_PARTNAME, OAI_DC_PARTNAME, LONG_PARTNAME, SHORT_PARTNAME, OPENAIRE_PARTNAME ]) oaiJazz = OaiJazz(join(statePath, 'oai')) oaiJazz.updateMetadataFormat( OAI_DC_PARTNAME, "http://www.openarchives.org/OAI/2.0/oai_dc.xsd", "http://purl.org/dc/elements/1.1/") oai_oa_cerifJazz = OaiJazz(join(statePath, 'oai_cerif')) oai_oa_cerifJazz.updateMetadataFormat( OPENAIRE_PARTNAME, "https://www.openaire.eu/schema/cris/current/openaire-cerif-profile.xsd", "https://www.openaire.eu/cerif-profile/1.1/") # All of the following OAI-PMH sets shall be recognized by the CRIS, even if not all of them are populated. oai_oa_cerifJazz.updateSet("openaire_cris_projects", "OpenAIRE_CRIS_projects") oai_oa_cerifJazz.updateSet("openaire_cris_orgunits", "OpenAIRE_CRIS_orgunits") oai_oa_cerifJazz.updateSet("openaire_cris_persons", "OpenAIRE_CRIS_persons") oai_oa_cerifJazz.updateSet("openaire_cris_patents", "OpenAIRE_CRIS_patents") oai_oa_cerifJazz.updateSet("openaire_cris_products", "OpenAIRE_CRIS_products") oai_oa_cerifJazz.updateSet("openaire_cris_publications", "OpenAIRE_CRIS_publications") oai_oa_cerifJazz.updateSet("openaire_cris_funding", "OpenAIRE_CRIS_funding") oai_oa_cerifJazz.updateSet("openaire_cris_events", "OpenAIRE_CRIS_events") oai_oa_cerifJazz.updateSet("openaire_cris_equipments", "OpenAIRE_CRIS_equipments") cqlClauseConverters = [ RenameFieldForExact( untokenizedFields=untokenizedFieldnames, untokenizedPrefix=UNTOKENIZED_PREFIX, ).filterAndModifier(), SearchTermFilterAndModifier( shouldModifyFieldValue=lambda *args: True, fieldnameModifier=fieldnameRewrite).filterAndModifier(), ] periodicGateWayDownload = PeriodicDownload( reactor, host='localhost', port=gatewayPort, schedule=Schedule( period=1 if quickCommit else 10 ), # WST: Interval in seconds before sending a new request to the GATEWAY in case of an error while processing batch records.(default=1). IntegrationTests need 1 second! Otherwise tests will fail! name='api', autoStart=True) oaiDownload = OaiDownloadProcessor(path='/oaix', metadataPrefix=NORMALISED_DOC_NAME, workingDirectory=join( statePath, 'harvesterstate', 'gateway'), userAgentAddition='ApiServer', xWait=True, name='api', autoCommit=False) executeQueryHelix = \ (FilterMessages(allowed=['executeQuery']), (CqlMultiSearchClauseConversion(cqlClauseConverters, fromKwarg='query'), (DrilldownQueries(), (convertToComposedQuery, (luceneRoHelix,), ) ) ) ) return \ (Observable(), createDownloadHelix(reactor, periodicGateWayDownload, oaiDownload, storage, oaiJazz, oai_oa_cerifJazz), (ObservableHttpServer(reactor, port, compressResponse=True), (BasicHttpHandler(), (PathFilter(["/oai"]), (OaiPmh(repositoryName="NARCIS OAI-pmh", adminEmail="*****@*****.**", externalUrl="http://oai.narcis.nl"), (oaiJazz,), (StorageAdapter(), (storage,) ), (OaiBranding( url="http://www.narcis.nl/images/logos/logo-knaw-house.gif", link="http://oai.narcis.nl", title="Narcis - The gateway to scholarly information in The Netherlands"), ), (OaiProvenance( nsMap=NAMESPACEMAP, baseURL=('meta', '//meta:repository/meta:baseurl/text()'), harvestDate=('meta', '//meta:record/meta:harvestdate/text()'), metadataNamespace=('meta', '//meta:record/meta:metadataNamespace/text()'), identifier=('header','//oai:identifier/text()'), datestamp=('header', '//oai:datestamp/text()') ), (storage,) ) ) ), (PathFilter(["/cerif"]), (OaiPmhDans(repositoryName="OpenAIRE CERIF", adminEmail="*****@*****.**", repositoryIdentifier="services.nod.dans.knaw.nl", externalUrl="http://services.nod.dans.knaw.nl"), #TODO: pathFilter should resemble proxy path (oai_oa_cerifJazz,), (StorageAdapter(), (storage,) ), (OaiOpenAIREDescription( serviceid='organisation:ORG1242054', acronym='services.nod.dans.knaw.nl', name='NARCIS', description='Compliant with the OpenAIRE Guidelines for CRIS Managers v.1.1.', website='https://www.narcis.nl', baseurl='http://services.nod.dans.knaw.nl/oa-cerif', subjectheading='', orgunitid='organisation:ORG1242054', owneracronym='DANS'), ), # (OaiBranding( # url="http://www.narcis.nl/images/logos/logo-knaw-house.gif", # link="http://oai.narcis.nl", # title="Narcis - The gateway to scholarly information in The Netherlands"), # ), (OaiProvenance( nsMap=NAMESPACEMAP, baseURL=('meta', '//meta:repository/meta:baseurl/text()'), harvestDate=('meta', '//meta:record/meta:harvestdate/text()'), metadataNamespace=('meta', '//meta:record/meta:metadataNamespace/text()'), identifier=('header','//oai:identifier/text()'), datestamp=('header', '//oai:datestamp/text()') ), (storage,) ) ) ), (PathFilter(['/sru']), (SruParser( host='sru.narcis.nl', port=80, defaultRecordSchema='knaw_short', defaultRecordPacking='xml'), (SruLimitStartRecord(limitBeyond=4000), (SruHandler( includeQueryTimes=False, extraXParameters=[], enableCollectLog=False), (SruTermDrilldown(),), executeQueryHelix, (StorageAdapter(), (storage,) ) ) ) ) ), (PathFilter('/rss'), (Rss( supportedLanguages = ['nl','en'], # defaults to first, if requested language is not available or supplied. title = {'nl':'NARCIS', 'en':'NARCIS'}, description = {'nl':'NARCIS: De toegang tot de Nederlandse wetenschapsinformatie', 'en':'NARCIS: The gateway to Dutch scientific information'}, link = {'nl':'http://www.narcis.nl/?Language=nl', 'en':'http://www.narcis.nl/?Language=en'}, maximumRecords = 20), executeQueryHelix, (RssItem( nsMap=NAMESPACEMAP, title = ('knaw_short', {'nl':'//short:metadata/short:titleInfo[not (@xml:lang)]/short:title/text()', 'en':'//short:metadata/short:titleInfo[@xml:lang="en"]/short:title/text()'}), description = ('knaw_short', {'nl':'//short:abstract[not (@xml:lang)]/text()', 'en':'//short:abstract[@xml:lang="en"]/text()'}), pubdate = ('knaw_short', '//short:dateIssued/short:parsed/text()'), linkTemplate = 'http://www.narcis.nl/%(wcpcollection)s/RecordID/%(oai_identifier)s/Language/%(language)s', wcpcollection = ('meta', '//*[local-name() = "collection"]/text()'), oai_identifier = ('meta', '//meta:record/meta:id/text()'), language = ('Dummy: Language is auto provided by the calling RSS component, but needs to be present to serve the linkTemplate.') ), (StorageAdapter(), (storage,) ) ) ) ) ) ) )
class SrwTest(SeecrTestCase): def setUp(self): SeecrTestCase.setUp(self) self.srw = Srw() self.sruParser = SruParser() self.sruHandler = SruHandler() self.srw.addObserver(self.sruParser) self.sruParser.addObserver(self.sruHandler) self.response = Response(total=1, hits=[Hit('0')]) def executeQuery(**kwargs): return self.response yield def retrieveData(**kwargs): return 'data' yield self.observer = CallTrace(methods={ 'executeQuery': executeQuery, 'retrieveData': retrieveData }, emptyGeneratorMethods=[ 'extraResponseData', 'echoedExtraRequestData', 'additionalDiagnosticDetails', 'extraRecordData' ]) self.sruHandler.addObserver(self.observer) def testNonSoap(self): """Wrong Soap envelope or body""" invalidSoapEnvelope = '<?xml version="1.0"?><SOAP:Envelope xmlns:SOAP="http://wrong.example.org/soap/envelope/"><SOAP:Body>%s</SOAP:Body></SOAP:Envelope>' request = invalidSoapEnvelope % SRW_REQUEST % argumentsWithMandatory % "" response = "".join(list(Srw().handleRequest(Body=request.encode()))) self.assertEqualsWS( """HTTP/1.0 500 Internal Server Error Content-Type: text/xml; charset=utf-8 <SOAP:Envelope xmlns:SOAP="http://schemas.xmlsoap.org/soap/envelope/"><SOAP:Body><SOAP:Fault><faultcode>SOAP:VersionMismatch</faultcode><faultstring>The processing party found an invalid namespace for the SOAP Envelope element</faultstring></SOAP:Fault></SOAP:Body></SOAP:Envelope>""", response) def testMalformedXML(self): """Stuff that is not even XML""" request = b'This is not even XML' response = "".join(self.srw.handleRequest(Body=request)) self.assertTrue( '<faultcode>SOAP:Server.userException</faultcode>' in response) def testBadSrwRequest(self): request = soapEnvelope % """<srw:searchRetrieveRequest xmlns:srw="http://wrong.example.org/srw"> <srw:version>1.2</srw:version> <srw:query>query</srw:query> </srw:searchRetrieveRequest>""" response = asString(self.srw.handleRequest(Body=request)) header, body = response.split('\r\n\r\n') self.assertEqual( ['1'], xpath(XML(body), '//srw:searchRetrieveResponse/srw:numberOfRecords/text()')) def testNonSRUArguments(self): """Arguments that are invalid in any SRU implementation""" request = soapEnvelope % SRW_REQUEST % argumentsWithMandatory % """<SRW:illegalParameter>value</SRW:illegalParameter>""" response = "".join(self.srw.handleRequest(Body=request)) self.assertEqualsWS( httpResponse % soapEnvelope % """<srw:searchRetrieveResponse %(xmlns_srw)s %(xmlns_diag)s %(xmlns_xcql)s %(xmlns_dc)s %(xmlns_meresco_srw)s> <srw:version>1.1</srw:version><srw:numberOfRecords>0</srw:numberOfRecords><srw:diagnostics><diagnostic xmlns="http://www.loc.gov/zing/srw/diagnostic/"> <uri>info://srw/diagnostics/1/8</uri> <details>illegalParameter</details> <message>Unsupported Parameter</message> </diagnostic></srw:diagnostics></srw:searchRetrieveResponse>""" % namespaces, response) def testNonSRWArguments(self): """Arguments that are part of SRU, but not of SRW (operation (done), stylesheet) """ request = soapEnvelope % SRW_REQUEST % argumentsWithMandatory % """<SRW:stylesheet>http://example.org/style.xsl</SRW:stylesheet>""" response = "".join(self.srw.handleRequest(Body=request)) self.assertEqualsWS( httpResponse % soapEnvelope % """<srw:searchRetrieveResponse %(xmlns_srw)s %(xmlns_diag)s %(xmlns_xcql)s %(xmlns_dc)s %(xmlns_meresco_srw)s> <srw:version>1.1</srw:version><srw:numberOfRecords>0</srw:numberOfRecords><srw:diagnostics><diagnostic xmlns="http://www.loc.gov/zing/srw/diagnostic/"> <uri>info://srw/diagnostics/1/8</uri> <details>stylesheet</details> <message>Unsupported Parameter</message> </diagnostic></srw:diagnostics></srw:searchRetrieveResponse>""" % namespaces, response) def testOperationIsIllegal(self): request = soapEnvelope % SRW_REQUEST % """<SRW:version>1.1</SRW:version><SRW:operation>explain</SRW:operation>""" response = "".join(self.srw.handleRequest(Body=request)) self.assertEqualsWS( httpResponse % soapEnvelope % """<srw:searchRetrieveResponse %(xmlns_srw)s %(xmlns_diag)s %(xmlns_xcql)s %(xmlns_dc)s %(xmlns_meresco_srw)s> <srw:version>1.1</srw:version><srw:numberOfRecords>0</srw:numberOfRecords><srw:diagnostics><diagnostic xmlns="http://www.loc.gov/zing/srw/diagnostic/"> <uri>info://srw/diagnostics/1/4</uri> <details>explain</details> <message>Unsupported Operation</message> </diagnostic></srw:diagnostics></srw:searchRetrieveResponse>""" % namespaces, response) def testContentType(self): request = soapEnvelope % SRW_REQUEST % argumentsWithMandatory % '' response = asString(self.srw.handleRequest(Body=request)) self.assertTrue('text/xml; charset=utf-8' in response, response) def testNormalOperation(self): request = soapEnvelope % SRW_REQUEST % argumentsWithMandatory % "" self.response = Response(total=1, hits=[Hit('recordId')]) del self.observer.methods['retrieveData'] def retrieveData(identifier, name): return "<DATA>%s-%s</DATA>" % (identifier, name) yield self.observer.methods['retrieveData'] = retrieveData result = "".join(compose(self.srw.handleRequest(Body=request))) self.assertEqualsWS( httpResponse % soapEnvelope % wrappedMockAnswer % ('recordId', 'dc.author = "jones" and dc.title = "smith"'), result) def testEmptySortKeys(self): request = soapEnvelope % SRW_REQUEST % argumentsWithMandatory % "<SRW:sortKeys/>" self.response = Response(total=0, hits=[]) result = "".join(compose(self.srw.handleRequest(Body=request))) executeQueryKwargs = self.observer.calledMethods[0].kwargs self.assertFalse("sortKeys" in executeQueryKwargs, executeQueryKwargs) def testArgumentsAreNotUnicodeStrings(self): """JJ/TJ: unicode strings somehow paralyse server requests. So ensure every argument is a str!""" """KvS d.d. 2007/11/15 - is this true in the Component-context too?""" request = soapEnvelope % SRW_REQUEST % argumentsWithMandatory % "" component = Srw() arguments = component._soapXmlToArguments(request) for key in arguments: self.assertTrue(type(key) == str) def testExampleFromLibraryOffCongressSite(self): """testExampleFromLibraryOffCongressSite - Integration test based on http://www.loc.gov/standards/sru/srw/index.html spelling error ("recordSchema") corrected """ request = """<SOAP:Envelope xmlns:SOAP="http://schemas.xmlsoap.org/soap/envelope/"> <SOAP:Body> <SRW:searchRetrieveRequest xmlns:SRW="http://www.loc.gov/zing/srw/"> <SRW:version>1.1</SRW:version> <SRW:query>dc.author = "jones" and dc.title = "smith"</SRW:query> <SRW:startRecord>1</SRW:startRecord> <SRW:maximumRecords>10</SRW:maximumRecords> <SRW:recordSchema>info:srw/schema/1/mods-v3.0</SRW:recordSchema> </SRW:searchRetrieveRequest> </SOAP:Body> </SOAP:Envelope>""" self.response = Response(total=1, hits=[Hit('recordId')]) del self.observer.methods['retrieveData'] def retrieveData(identifier, name): return "<DATA>%s-%s</DATA>" % (identifier, name) yield self.observer.methods['retrieveData'] = retrieveData response = "".join(compose(self.srw.handleRequest(Body=request))) echoRequest = """<srw:echoedSearchRetrieveRequest> <srw:version>1.1</srw:version> <srw:query>dc.author = "jones" and dc.title = "smith"</srw:query> <srw:startRecord>1</srw:startRecord> <srw:maximumRecords>10</srw:maximumRecords> <srw:recordPacking>xml</srw:recordPacking> <srw:recordSchema>info:srw/schema/1/mods-v3.0</srw:recordSchema> </srw:echoedSearchRetrieveRequest>""" self.assertEqualsWS( httpResponse % soapEnvelope % searchRetrieveResponse % (1, '<srw:records><srw:record><srw:recordSchema>info:srw/schema/1/mods-v3.0</srw:recordSchema><srw:recordPacking>xml</srw:recordPacking><srw:recordData><DATA>recordId-info:srw/schema/1/mods-v3.0</DATA></srw:recordData></srw:record></srw:records>' + echoRequest), response) @stderr_replaced def testConstructorVariablesAreUsed(self): request = soapEnvelope % SRW_REQUEST % argumentsWithMandatory % "" srw = Srw(defaultRecordSchema="DEFAULT_RECORD_SCHEMA", defaultRecordPacking="DEFAULT_RECORD_PACKING") sruParser = SruParser() srw.addObserver(sruParser) sruParser.addObserver(self.sruHandler) response = Response(total=1, hits=[Hit(1)]) def executeQuery(**kwargs): return response yield @asyncnoreturnvalue def methodAsGenerator(**kwargs): pass observer = CallTrace(methods={ 'executeQuery': executeQuery, 'extraResponseData': methodAsGenerator, 'echoedExtraRequestData': methodAsGenerator, }, emptyGeneratorMethods=[ 'additionalDiagnosticDetails', 'extraRecordData', ]) self.sruHandler.addObserver(observer) response = "".join(compose(srw.handleRequest(Body=request))) self.assertTrue("DEFAULT_RECORD_SCHEMA" in response, response) self.assertTrue("DEFAULT_RECORD_PACKING" in response, response)
class SrwTest(SeecrTestCase): def setUp(self): SeecrTestCase.setUp(self) self.srw = Srw() self.sruParser = SruParser() self.sruHandler = SruHandler() self.srw.addObserver(self.sruParser) self.sruParser.addObserver(self.sruHandler) self.response = StopIteration(Response(total=1, hits=[Hit('0')])) def executeQuery(**kwargs): raise self.response yield def retrieveData(**kwargs): raise StopIteration('data') yield self.observer = CallTrace( methods={ 'executeQuery': executeQuery, 'retrieveData': retrieveData }, emptyGeneratorMethods=[ 'extraResponseData', 'echoedExtraRequestData', 'additionalDiagnosticDetails', 'extraRecordData' ]) self.sruHandler.addObserver(self.observer) def testNonSoap(self): """Wrong Soap envelope or body""" invalidSoapEnvelope = '<?xml version="1.0"?><SOAP:Envelope xmlns:SOAP="http://wrong.example.org/soap/envelope/"><SOAP:Body>%s</SOAP:Body></SOAP:Envelope>' request = invalidSoapEnvelope % SRW_REQUEST % argumentsWithMandatory % "" response = "".join(list(Srw().handleRequest(Body=request))) self.assertEqualsWS("""HTTP/1.0 500 Internal Server Error Content-Type: text/xml; charset=utf-8 <SOAP:Envelope xmlns:SOAP="http://schemas.xmlsoap.org/soap/envelope/"><SOAP:Body><SOAP:Fault><faultcode>SOAP:VersionMismatch</faultcode><faultstring>The processing party found an invalid namespace for the SOAP Envelope element</faultstring></SOAP:Fault></SOAP:Body></SOAP:Envelope>""", response) def testMalformedXML(self): """Stuff that is not even XML""" request = 'This is not even XML' response = "".join(self.srw.handleRequest(Body=request)) self.assertTrue('<faultcode>SOAP:Server.userException</faultcode>' in response) def testBadSrwRequest(self): request = soapEnvelope % """<srw:searchRetrieveRequest xmlns:srw="http://wrong.example.org/srw"> <srw:version>1.2</srw:version> <srw:query>query</srw:query> </srw:searchRetrieveRequest>""" response = asString(self.srw.handleRequest(Body=request)) header, body = response.split('\r\n\r\n') self.assertEquals(['1'], xpath(XML(body), '//srw:searchRetrieveResponse/srw:numberOfRecords/text()')) def testNonSRUArguments(self): """Arguments that are invalid in any SRU implementation""" request = soapEnvelope % SRW_REQUEST % argumentsWithMandatory % """<SRW:illegalParameter>value</SRW:illegalParameter>""" response = "".join(self.srw.handleRequest(Body=request)) self.assertEqualsWS(httpResponse % soapEnvelope % """<srw:searchRetrieveResponse %(xmlns_srw)s %(xmlns_diag)s %(xmlns_xcql)s %(xmlns_dc)s %(xmlns_meresco_srw)s> <srw:version>1.1</srw:version><srw:numberOfRecords>0</srw:numberOfRecords><srw:diagnostics><diagnostic xmlns="http://www.loc.gov/zing/srw/diagnostic/"> <uri>info://srw/diagnostics/1/8</uri> <details>illegalParameter</details> <message>Unsupported Parameter</message> </diagnostic></srw:diagnostics></srw:searchRetrieveResponse>""" % namespaces, response) def testNonSRWArguments(self): """Arguments that are part of SRU, but not of SRW (operation (done), stylesheet) """ request = soapEnvelope % SRW_REQUEST % argumentsWithMandatory % """<SRW:stylesheet>http://example.org/style.xsl</SRW:stylesheet>""" response = "".join(self.srw.handleRequest(Body=request)) self.assertEqualsWS(httpResponse % soapEnvelope % """<srw:searchRetrieveResponse %(xmlns_srw)s %(xmlns_diag)s %(xmlns_xcql)s %(xmlns_dc)s %(xmlns_meresco_srw)s> <srw:version>1.1</srw:version><srw:numberOfRecords>0</srw:numberOfRecords><srw:diagnostics><diagnostic xmlns="http://www.loc.gov/zing/srw/diagnostic/"> <uri>info://srw/diagnostics/1/8</uri> <details>stylesheet</details> <message>Unsupported Parameter</message> </diagnostic></srw:diagnostics></srw:searchRetrieveResponse>""" % namespaces, response) def testOperationIsIllegal(self): request = soapEnvelope % SRW_REQUEST % """<SRW:version>1.1</SRW:version><SRW:operation>explain</SRW:operation>""" response = "".join(self.srw.handleRequest(Body=request)) self.assertEqualsWS(httpResponse % soapEnvelope % """<srw:searchRetrieveResponse %(xmlns_srw)s %(xmlns_diag)s %(xmlns_xcql)s %(xmlns_dc)s %(xmlns_meresco_srw)s> <srw:version>1.1</srw:version><srw:numberOfRecords>0</srw:numberOfRecords><srw:diagnostics><diagnostic xmlns="http://www.loc.gov/zing/srw/diagnostic/"> <uri>info://srw/diagnostics/1/4</uri> <details>explain</details> <message>Unsupported Operation</message> </diagnostic></srw:diagnostics></srw:searchRetrieveResponse>""" % namespaces, response) def testContentType(self): request = soapEnvelope % SRW_REQUEST % argumentsWithMandatory % '' response = asString(self.srw.handleRequest(Body=request)) self.assertTrue('text/xml; charset=utf-8' in response, response) def testNormalOperation(self): request = soapEnvelope % SRW_REQUEST % argumentsWithMandatory % "" self.response = StopIteration(Response(total=1, hits=[Hit('recordId')])) del self.observer.methods['retrieveData'] def retrieveData(identifier, name): raise StopIteration("<DATA>%s-%s</DATA>" % (identifier, name)) yield self.observer.methods['retrieveData'] = retrieveData result = "".join(compose(self.srw.handleRequest(Body=request))) self.assertEqualsWS(httpResponse % soapEnvelope % wrappedMockAnswer % ('recordId', 'dc.author = "jones" and dc.title = "smith"'), result) def testEmptySortKeys(self): request = soapEnvelope % SRW_REQUEST % argumentsWithMandatory % "<SRW:sortKeys/>" self.response = StopIteration(Response(total=0, hits=[])) result = "".join(compose(self.srw.handleRequest(Body=request))) executeQueryKwargs = self.observer.calledMethods[0].kwargs self.assertFalse("sortKeys" in executeQueryKwargs, executeQueryKwargs) def testArgumentsAreNotUnicodeStrings(self): """JJ/TJ: unicode strings somehow paralyse server requests. So ensure every argument is a str!""" """KvS d.d. 2007/11/15 - is this true in the Component-context too?""" request = soapEnvelope % SRW_REQUEST % argumentsWithMandatory % "" component = Srw() arguments = component._soapXmlToArguments(request) for key in arguments: self.assertTrue(type(key) == str) def testExampleFromLibraryOffCongressSite(self): """testExampleFromLibraryOffCongressSite - Integration test based on http://www.loc.gov/standards/sru/srw/index.html spelling error ("recordSchema") corrected """ request = """<SOAP:Envelope xmlns:SOAP="http://schemas.xmlsoap.org/soap/envelope/"> <SOAP:Body> <SRW:searchRetrieveRequest xmlns:SRW="http://www.loc.gov/zing/srw/"> <SRW:version>1.1</SRW:version> <SRW:query>dc.author = "jones" and dc.title = "smith"</SRW:query> <SRW:startRecord>1</SRW:startRecord> <SRW:maximumRecords>10</SRW:maximumRecords> <SRW:recordSchema>info:srw/schema/1/mods-v3.0</SRW:recordSchema> </SRW:searchRetrieveRequest> </SOAP:Body> </SOAP:Envelope>""" self.response = StopIteration(Response(total=1, hits=[Hit('recordId')])) del self.observer.methods['retrieveData'] def retrieveData(identifier, name): raise StopIteration("<DATA>%s-%s</DATA>" % (identifier, name)) yield self.observer.methods['retrieveData'] = retrieveData response = "".join(compose(self.srw.handleRequest(Body=request))) echoRequest = """<srw:echoedSearchRetrieveRequest> <srw:version>1.1</srw:version> <srw:query>dc.author = "jones" and dc.title = "smith"</srw:query> <srw:startRecord>1</srw:startRecord> <srw:maximumRecords>10</srw:maximumRecords> <srw:recordPacking>xml</srw:recordPacking> <srw:recordSchema>info:srw/schema/1/mods-v3.0</srw:recordSchema> </srw:echoedSearchRetrieveRequest>""" self.assertEqualsWS(httpResponse % soapEnvelope % searchRetrieveResponse % (1, '<srw:records><srw:record><srw:recordSchema>info:srw/schema/1/mods-v3.0</srw:recordSchema><srw:recordPacking>xml</srw:recordPacking><srw:recordData><DATA>recordId-info:srw/schema/1/mods-v3.0</DATA></srw:recordData></srw:record></srw:records>' +echoRequest), response) @stderr_replaced def testConstructorVariablesAreUsed(self): request = soapEnvelope % SRW_REQUEST % argumentsWithMandatory % "" srw = Srw( defaultRecordSchema="DEFAULT_RECORD_SCHEMA", defaultRecordPacking="DEFAULT_RECORD_PACKING") sruParser = SruParser() srw.addObserver(sruParser) sruParser.addObserver(self.sruHandler) response = Response(total=1, hits=[Hit(1)]) def executeQuery(**kwargs): raise StopIteration(response) yield @asyncnoreturnvalue def methodAsGenerator(**kwargs): pass observer = CallTrace( methods={ 'executeQuery': executeQuery, 'extraResponseData': methodAsGenerator, 'echoedExtraRequestData': methodAsGenerator, }, emptyGeneratorMethods=[ 'additionalDiagnosticDetails', 'extraRecordData', ]) self.sruHandler.addObserver(observer) response = "".join(compose(srw.handleRequest(Body=request))) self.assertTrue("DEFAULT_RECORD_SCHEMA" in response, response) self.assertTrue("DEFAULT_RECORD_PACKING" in response, response)
kwargs = sruHandlerKwargs(x_term_drilldown='field0:1') observer.calledMethods.reset() try: ''.join(compose(sruHandler.searchRetrieve(**kwargs))) except KeyboardInterrupt, e: self.assertEquals('Ok', str(e)) else: self.fail('Should not come here') self.assertEquals(['executeQuery'], observer.calledMethodNames()) self.assertEquals([dict(fieldname='field0', maxTerms=1, sortBy=DRILLDOWN_SORTBY_COUNT)], observer.calledMethods[0].kwargs['facets']) # Too high kwargs = sruHandlerKwargs(x_term_drilldown='field0:4') sruHandler = SruHandler(drilldownMaximumMaximumResults=drilldownMaximumMaximumResults) observer = CallTrace('observer') sruHandler.addObserver(observer) def executeQuery(**kwargs): raise KeyboardInterrupt('Should have failed before triggering this exception!') yield observer.methods['executeQuery'] = executeQuery try: ''.join(compose(sruHandler.searchRetrieve(**kwargs))) except SruException, e: self.assertEquals('field0; drilldown with maximumResults > 3', str(e)) except KeyboardInterrupt, e: self.fail(str(e)) else: self.fail('Should not come here')
def main(reactor, port, databasePath): drilldownFields = [ DrilldownField('untokenized.field2'), DrilldownField('untokenized.fieldHier', hierarchical=True) ] fieldRegistry = FieldRegistry(drilldownFields) luceneSettings = LuceneSettings(fieldRegistry=fieldRegistry, commitCount=30, commitTimeout=1, analyzer=MerescoDutchStemmingAnalyzer()) lucene = Lucene(path=join(databasePath, 'lucene'), reactor=reactor, name='main', settings=luceneSettings) lucene2Settings = LuceneSettings(fieldRegistry=fieldRegistry, commitTimeout=0.1) lucene2 = Lucene(path=join(databasePath, 'lucene2'), reactor=reactor, name='main2', settings=lucene2Settings) termNumerator = TermNumerator(path=join(databasePath, 'termNumerator')) emptyLuceneSettings = LuceneSettings(commitTimeout=1) multiLuceneHelix = ( MultiLucene(defaultCore='main'), (Lucene(path=join(databasePath, 'lucene-empty'), reactor=reactor, name='empty-core', settings=emptyLuceneSettings), ), (lucene, ), (lucene2, ), ) storageComponent = StorageComponent( directory=join(databasePath, 'storage')) return \ (Observable(), (ObservableHttpServer(reactor=reactor, port=port), (BasicHttpHandler(), (ApacheLogger(outputStream=stdout), (PathFilter("/info", excluding=[ '/info/version', '/info/name', '/update', '/sru', '/remote', '/via-remote-sru', ]), (DynamicHtml( [dynamicPath], reactor=reactor, indexPage='/info', additionalGlobals={ 'VERSION': version, } ), ) ), (PathFilter("/info/version"), (StringServer(version, ContentTypePlainText), ) ), (PathFilter("/info/name"), (StringServer('Meresco Lucene', ContentTypePlainText),) ), (PathFilter("/static"), (PathRename(lambda path: path[len('/static'):]), (FileServer(staticPath),) ) ), (PathFilter("/update_main", excluding=['/update_main2']), uploadHelix(lucene, termNumerator, storageComponent, drilldownFields, fieldRegistry=luceneSettings.fieldRegistry), ), (PathFilter("/update_main2"), uploadHelix(lucene2, termNumerator, storageComponent, drilldownFields, fieldRegistry=lucene2Settings.fieldRegistry), ), (PathFilter('/sru'), (SruParser(defaultRecordSchema='record'), (SruHandler(), (MultiCqlToLuceneQuery( defaultCore='main', coreToCqlLuceneQueries={ "main": CqlToLuceneQuery([], luceneSettings=luceneSettings), "main2": CqlToLuceneQuery([], luceneSettings=lucene2Settings), "empty-core": CqlToLuceneQuery([], luceneSettings=emptyLuceneSettings), }), multiLuceneHelix, ), (SRUTermDrilldown(defaultFormat='xml'),), (SruDuplicateCount(),), (storageComponent,), ) ) ), (PathFilter('/via-remote-sru'), (SruParser(defaultRecordSchema='record'), (SruHandler(), (LuceneRemote(host='localhost', port=port, path='/remote'),), (SRUTermDrilldown(defaultFormat='xml'),), (SruDuplicateCount(),), (storageComponent,), ) ) ), (PathFilter('/remote'), (LuceneRemoteService(reactor=reactor), (MultiCqlToLuceneQuery( defaultCore='main', coreToCqlLuceneQueries={ "main": CqlToLuceneQuery([], luceneSettings=luceneSettings), "main2": CqlToLuceneQuery([], luceneSettings=lucene2Settings), "empty-core": CqlToLuceneQuery([], luceneSettings=emptyLuceneSettings), }), multiLuceneHelix, ) ) ), (PathFilter('/autocomplete'), (Autocomplete('localhost', port, '/autocomplete', '__all__', '?', 5, '?', '?'), (lucene,), ) ) ) ) ) )
def testSearchRetrieveVersion11(self): queryArguments = {'version':'1.1', 'operation':'searchRetrieve', 'recordSchema':'schema', 'recordPacking':'xml', 'query':'field=value', 'startRecord':1, 'maximumRecords':2} sruArguments = {'version':'1.1', 'operation':'searchRetrieve', 'recordSchema':'schema', 'recordPacking':'xml', 'query':'field=value', 'startRecord':1, 'maximumRecords':2, 'x-recordSchema':['extra', 'evenmore']} observer = CallTrace() response = Response(total=100, hits=hitsRange(11, 13)) def executeQuery(**kwargs): raise StopIteration(response) yield observer.methods['executeQuery'] = executeQuery retrieveDataCalls = [] def retrieveData(identifier, name): retrieveDataCalls.append(1) raise StopIteration("<MOCKED_WRITTEN_DATA>%s-%s</MOCKED_WRITTEN_DATA>" % (xmlEscape(identifier), name)) yield observer.retrieveData = retrieveData observer.methods['extraResponseData'] = lambda *a, **kw: (x for x in 'extraResponseData') observer.methods['echoedExtraRequestData'] = lambda *a, **kw: (x for x in 'echoedExtraRequestData') observer.methods['extraRecordData'] = lambda hit: (f for f in []) component = SruHandler() component.addObserver(observer) result = "".join(compose(component.searchRetrieve(sruArguments=sruArguments, **queryArguments))) self.assertEqualsWS(""" <srw:searchRetrieveResponse %(xmlns_srw)s %(xmlns_diag)s %(xmlns_xcql)s %(xmlns_dc)s %(xmlns_meresco_srw)s> <srw:version>1.1</srw:version> <srw:numberOfRecords>100</srw:numberOfRecords> <srw:records> <srw:record> <srw:recordSchema>schema</srw:recordSchema> <srw:recordPacking>xml</srw:recordPacking> <srw:recordData> <MOCKED_WRITTEN_DATA>11-schema</MOCKED_WRITTEN_DATA> </srw:recordData> <srw:extraRecordData> <srw:record> <srw:recordSchema>extra</srw:recordSchema> <srw:recordPacking>xml</srw:recordPacking> <srw:recordData> <MOCKED_WRITTEN_DATA>11-extra</MOCKED_WRITTEN_DATA> </srw:recordData> </srw:record> <srw:record> <srw:recordSchema>evenmore</srw:recordSchema> <srw:recordPacking>xml</srw:recordPacking> <srw:recordData> <MOCKED_WRITTEN_DATA>11-evenmore</MOCKED_WRITTEN_DATA> </srw:recordData> </srw:record> </srw:extraRecordData> </srw:record> <srw:record> <srw:recordSchema>schema</srw:recordSchema> <srw:recordPacking>xml</srw:recordPacking> <srw:recordData> <MOCKED_WRITTEN_DATA>12-schema</MOCKED_WRITTEN_DATA> </srw:recordData> <srw:extraRecordData> <srw:record> <srw:recordSchema>extra</srw:recordSchema> <srw:recordPacking>xml</srw:recordPacking> <srw:recordData> <MOCKED_WRITTEN_DATA>12-extra</MOCKED_WRITTEN_DATA> </srw:recordData> </srw:record> <srw:record> <srw:recordSchema>evenmore</srw:recordSchema> <srw:recordPacking>xml</srw:recordPacking> <srw:recordData> <MOCKED_WRITTEN_DATA>12-evenmore</MOCKED_WRITTEN_DATA> </srw:recordData> </srw:record> </srw:extraRecordData> </srw:record> </srw:records> <srw:nextRecordPosition>3</srw:nextRecordPosition> <srw:echoedSearchRetrieveRequest> <srw:version>1.1</srw:version> <srw:query>field=value</srw:query> <srw:startRecord>1</srw:startRecord> <srw:maximumRecords>2</srw:maximumRecords> <srw:recordPacking>xml</srw:recordPacking> <srw:recordSchema>schema</srw:recordSchema> <srw:x-recordSchema>extra</srw:x-recordSchema> <srw:x-recordSchema>evenmore</srw:x-recordSchema> <srw:extraRequestData>echoedExtraRequestData</srw:extraRequestData> </srw:echoedSearchRetrieveRequest> <srw:extraResponseData>extraResponseData</srw:extraResponseData> </srw:searchRetrieveResponse> """ % namespaces, result)