def testExplainWithPresetValues(self): component = SruParser(host='TEST_SERVER_HOST', port='TEST_SERVER_PORT', description='TEST_SERVER_DESCRIPTION', modifiedDate='TEST_SERVER_DATE', database='DATABASE', wsdl='http://somewhe.re/wsdl') result = "".join(compose(component.handleRequest(arguments={}, Headers={'Host': '1.2.3.4:80'}, RequestURI="/sru"))) self.assertEqualsWS("""HTTP/1.0 200 OK Content-Type: text/xml; charset=utf-8 <?xml version="1.0" encoding="UTF-8"?> <srw:explainResponse xmlns:srw="http://www.loc.gov/zing/srw/" xmlns:zr="http://explain.z3950.org/dtd/2.0/"> <srw:version>1.1</srw:version> <srw:record> <srw:recordPacking>xml</srw:recordPacking> <srw:recordSchema>http://explain.z3950.org/dtd/2.0/</srw:recordSchema> <srw:recordData> <zr:explain> <zr:serverInfo wsdl="http://somewhe.re/wsdl" protocol="SRU" version="1.1"> <zr:host>TEST_SERVER_HOST</zr:host> <zr:port>TEST_SERVER_PORT</zr:port> <zr:database>DATABASE</zr:database> </zr:serverInfo> <zr:databaseInfo> <zr:title lang="en" primary="true">SRU Database</zr:title> <zr:description lang="en" primary="true">TEST_SERVER_DESCRIPTION</zr:description> </zr:databaseInfo> <zr:metaInfo> <zr:dateModified>TEST_SERVER_DATE</zr:dateModified> </zr:metaInfo> </zr:explain> </srw:recordData> </srw:record> </srw:explainResponse> """, result)
def testExplainWithoutPresetValues(self): component = SruParser() result = "".join(compose(component.handleRequest(arguments={'operation': ['explain'], 'version': ['1.2']}, Headers={'Host': '1.2.3.4:8080'}, RequestURI="/sru?operation=explain&version=1.2"))) self.assertEqualsWS("""HTTP/1.0 200 OK Content-Type: text/xml; charset=utf-8 <?xml version="1.0" encoding="UTF-8"?> <srw:explainResponse xmlns:srw="http://www.loc.gov/zing/srw/" xmlns:zr="http://explain.z3950.org/dtd/2.0/"> <srw:version>1.2</srw:version> <srw:record> <srw:recordPacking>xml</srw:recordPacking> <srw:recordSchema>http://explain.z3950.org/dtd/2.0/</srw:recordSchema> <srw:recordData> <zr:explain> <zr:serverInfo protocol="SRU" version="1.2"> <zr:host>1.2.3.4</zr:host> <zr:port>8080</zr:port> <zr:database>sru</zr:database> </zr:serverInfo> <zr:databaseInfo> <zr:title lang="en" primary="true">SRU Database</zr:title> <zr:description lang="en" primary="true">Meresco SRU</zr:description> </zr:databaseInfo> </zr:explain> </srw:recordData> </srw:record> </srw:explainResponse> """, result)
def testConstructorVariablesAreUsed(self): request = soapEnvelope % SRW_REQUEST % argumentsWithMandatory % "" srw = Srw( defaultRecordSchema="DEFAULT_RECORD_SCHEMA", defaultRecordPacking="DEFAULT_RECORD_PACKING") sruParser = SruParser() srw.addObserver(sruParser) sruParser.addObserver(self.sruHandler) response = Response(total=1, hits=[Hit(1)]) def executeQuery(**kwargs): raise StopIteration(response) yield @asyncnoreturnvalue def methodAsGenerator(**kwargs): pass observer = CallTrace( methods={ 'executeQuery': executeQuery, 'extraResponseData': methodAsGenerator, 'echoedExtraRequestData': methodAsGenerator, }, emptyGeneratorMethods=[ 'additionalDiagnosticDetails', 'extraRecordData', ]) self.sruHandler.addObserver(observer) response = "".join(compose(srw.handleRequest(Body=request))) self.assertTrue("DEFAULT_RECORD_SCHEMA" in response, response) self.assertTrue("DEFAULT_RECORD_PACKING" in response, response)
def testMaximumMaximumRecords(self): component = SruParser('host', 'port', maximumMaximumRecords=100) try: component.parseSruArgs({'version':['1.1'], 'query':['twente'], 'operation':['searchRetrieve'], 'maximumRecords': ['101']}) self.fail() except SruException, e: self.assertEquals(UNSUPPORTED_PARAMETER_VALUE, [e.code, e.message])
def testValidXml(self): component = SruParser() sruHandler = SruHandler() component.addObserver(sruHandler) observer = CallTrace('observer') sruHandler.addObserver(observer) response = Response(total=2, hits=[Hit('id0'), Hit('id1')]) def executeQuery(**kwargs): raise StopIteration(response) yield def retrieveData(**kwargs): raise StopIteration('<bike/>') yield observer.methods['executeQuery'] = executeQuery observer.returnValues['echoedExtraRequestData'] = (f for f in []) observer.returnValues['extraResponseData'] = (f for f in []) observer.methods['extraRecordData'] = lambda hit: (f for f in []) observer.methods['retrieveData'] = retrieveData result = ''.join(compose(component.handleRequest(arguments={'version':['1.1'], 'query': ['aQuery'], 'operation':['searchRetrieve']}))) header, body = result.split('\r\n'*2) assertValid(body, join(schemasPath, 'srw-types1.2.xsd')) self.assertTrue('<bike/>' in body, body) result = ''.join(compose(component.handleRequest(arguments={'version':['1.1'], 'operation':['searchRetrieve']}))) header, body = result.split('\r\n'*2) assertValid(body, join(schemasPath, 'srw-types1.2.xsd')) self.assertTrue('diagnostic' in body, body)
def assertValid(self, expectedResult, arguments): component = SruParser('host', 'port') try: operation, arguments = component._parseArguments(arguments) if operation == "searchRetrieve": component.parseSruArgs(arguments) if expectedResult != SUCCESS: self.fail("Expected %s but got nothing" % expectedResult) except SruException, e: self.assertEquals(expectedResult, [e.code, e.message])
def assertSortKeys(expectedSortedDescending, sortKeys, oldAndWrongStyleSortKeys): component = SruParser(oldAndWrongStyleSortKeys=oldAndWrongStyleSortKeys) sruHandler = CallTrace('SRUHandler', emptyGeneratorMethods=['searchRetrieve']) component.addObserver(sruHandler) response = "".join(compose(component.handleRequest(arguments=dict( version=['1.1'], query= ['aQuery'], operation=['searchRetrieve'], sortKeys=[sortKeys])))) kwargs = sruHandler.calledMethods[0].kwargs self.assertEquals([{'sortBy': 'aField', 'sortDescending': expectedSortedDescending}], kwargs['sortKeys'])
def testSearchRetrieveWithXParameter(self): component = SruParser() sruHandler = CallTrace('SRUHandler') sruHandler.returnValues['searchRetrieve'] = (x for x in ["<result>mock result XML</result>"]) component.addObserver(sruHandler) list(compose(component.handleRequest(arguments={'version':['1.1'], 'query': ['aQuery'], 'operation':['searchRetrieve'], 'x-something':['something']}))) self.assertEquals(['searchRetrieve'], [m.name for m in sruHandler.calledMethods]) self.assertEquals((), sruHandler.calledMethods[0].args) kwargs = sruHandler.calledMethods[0].kwargs self.assertEquals(['something'], kwargs['sruArguments']['x-something'])
def setUp(self): SeecrTestCase.setUp(self) self.srw = Srw() self.sruParser = SruParser() self.sruHandler = SruHandler() self.srw.addObserver(self.sruParser) self.sruParser.addObserver(self.sruHandler) self.response = StopIteration(Response(total=1, hits=[Hit('0')])) def executeQuery(**kwargs): raise self.response yield def retrieveData(**kwargs): raise StopIteration('data') yield self.observer = CallTrace( methods={ 'executeQuery': executeQuery, 'retrieveData': retrieveData }, emptyGeneratorMethods=[ 'extraResponseData', 'echoedExtraRequestData', 'additionalDiagnosticDetails', 'extraRecordData' ]) self.sruHandler.addObserver(self.observer)
def testAllQueryHelpersForSRU(self): index = CallTrace('index') def executeQuery(**kwargs): raise StopIteration(Response(total=3201, hits=[])) yield index.methods['executeQuery'] = executeQuery index.ignoredAttributes.extend( ['echoedExtraRequestData', 'extraResponseData', 'all_unknown']) server = be(( Observable(), (self.queryLog, (SruParser(), (QueryLogHelperForSru(), (SruHandler(extraRecordDataNewStyle=True), (QueryLogHelperForExecuteCQL(), (index, )))))), )) ''.join( compose( server.all.handleRequest( path='/path/sru', Client=('11.22.33.44', 8080), arguments={ 'operation': ['searchRetrieve'], 'version': ['1.2'], 'maximumRecords': ['0'], 'query': ['field=value'], }, ))) self.assertEquals( '2009-11-02T11:25:37Z 11.22.33.44 0.7K 1.000s 3201hits /path/sru maximumRecords=0&operation=searchRetrieve&query=field%3Dvalue&recordPacking=xml&recordSchema=dc&startRecord=1&version=1.2\n', open(join(self.tempdir, '2009-11-02-query.log')).read())
def assertSortKeys(expectedSortedDescending, sortKeys, oldAndWrongStyleSortKeys): component = SruParser( oldAndWrongStyleSortKeys=oldAndWrongStyleSortKeys) sruHandler = CallTrace('SRUHandler', emptyGeneratorMethods=['searchRetrieve']) component.addObserver(sruHandler) response = "".join( compose( component.handleRequest( arguments=dict(version=['1.1'], query=['aQuery'], operation=['searchRetrieve'], sortKeys=[sortKeys])))) kwargs = sruHandler.calledMethods[0].kwargs self.assertEqual([{ 'sortBy': 'aField', 'sortDescending': expectedSortedDescending }], kwargs['sortKeys'])
def testExplainWithPresetValues(self): component = SruParser(host='TEST_SERVER_HOST', port='TEST_SERVER_PORT', description='TEST_SERVER_DESCRIPTION', modifiedDate='TEST_SERVER_DATE', database='DATABASE', wsdl='http://somewhe.re/wsdl') result = "".join( compose( component.handleRequest(arguments={}, Headers={'Host': '1.2.3.4:80'}, RequestURI="/sru"))) self.assertEqualsWS( """HTTP/1.0 200 OK Content-Type: text/xml; charset=utf-8 <?xml version="1.0" encoding="UTF-8"?> <srw:explainResponse xmlns:srw="http://www.loc.gov/zing/srw/" xmlns:zr="http://explain.z3950.org/dtd/2.0/"> <srw:version>1.1</srw:version> <srw:record> <srw:recordPacking>xml</srw:recordPacking> <srw:recordSchema>http://explain.z3950.org/dtd/2.0/</srw:recordSchema> <srw:recordData> <zr:explain> <zr:serverInfo wsdl="http://somewhe.re/wsdl" protocol="SRU" version="1.1"> <zr:host>TEST_SERVER_HOST</zr:host> <zr:port>TEST_SERVER_PORT</zr:port> <zr:database>DATABASE</zr:database> </zr:serverInfo> <zr:databaseInfo> <zr:title lang="en" primary="true">SRU Database</zr:title> <zr:description lang="en" primary="true">TEST_SERVER_DESCRIPTION</zr:description> </zr:databaseInfo> <zr:metaInfo> <zr:dateModified>TEST_SERVER_DATE</zr:dateModified> </zr:metaInfo> </zr:explain> </srw:recordData> </srw:record> </srw:explainResponse> """, result)
def testSearchRetrieve(self): component = SruParser() sruHandler = CallTrace('SRUHandler') sruHandler.returnValues['searchRetrieve'] = (x for x in ["<result>mock result XML</result>"]) component.addObserver(sruHandler) response = "".join(compose(component.handleRequest(arguments=dict(version=['1.1'], query= ['aQuery'], operation=['searchRetrieve'], startRecord=['11'], maximumRecords = ['15'], sortKeys=['aField,,1'])))) self.assertEquals(['searchRetrieve'], [m.name for m in sruHandler.calledMethods]) self.assertEquals((), sruHandler.calledMethods[0].args) kwargs = sruHandler.calledMethods[0].kwargs self.assertEquals('1.1', kwargs['version']) self.assertEquals('aQuery', kwargs['query']) self.assertEquals('searchRetrieve', kwargs['operation']) self.assertEquals(11, kwargs['startRecord']) self.assertEquals(15, kwargs['maximumRecords']) self.assertEquals('aQuery', kwargs['sruArguments']['query']) self.assertEquals(['aField,,1'], kwargs['sruArguments']['sortKeys']) self.assertTrue("HTTP/1.0 200 OK" in response) self.assertTrue(XML_HEADER in response)
def testSearchRetrieveWithXParameter(self): component = SruParser() sruHandler = CallTrace('SRUHandler') sruHandler.returnValues['searchRetrieve'] = ( x for x in ["<result>mock result XML</result>"]) component.addObserver(sruHandler) list( compose( component.handleRequest( arguments={ 'version': ['1.1'], 'query': ['aQuery'], 'operation': ['searchRetrieve'], 'x-something': ['something'] }))) self.assertEqual(['searchRetrieve'], [m.name for m in sruHandler.calledMethods]) self.assertEqual((), sruHandler.calledMethods[0].args) kwargs = sruHandler.calledMethods[0].kwargs self.assertEqual(['something'], kwargs['sruArguments']['x-something'])
def testExplainWithoutPresetValues(self): component = SruParser() result = "".join( compose( component.handleRequest( arguments={ 'operation': ['explain'], 'version': ['1.2'] }, Headers={'Host': '1.2.3.4:8080'}, RequestURI="/sru?operation=explain&version=1.2"))) self.assertEqualsWS( """HTTP/1.0 200 OK Content-Type: text/xml; charset=utf-8 <?xml version="1.0" encoding="UTF-8"?> <srw:explainResponse xmlns:srw="http://www.loc.gov/zing/srw/" xmlns:zr="http://explain.z3950.org/dtd/2.0/"> <srw:version>1.2</srw:version> <srw:record> <srw:recordPacking>xml</srw:recordPacking> <srw:recordSchema>http://explain.z3950.org/dtd/2.0/</srw:recordSchema> <srw:recordData> <zr:explain> <zr:serverInfo protocol="SRU" version="1.2"> <zr:host>1.2.3.4</zr:host> <zr:port>8080</zr:port> <zr:database>sru</zr:database> </zr:serverInfo> <zr:databaseInfo> <zr:title lang="en" primary="true">SRU Database</zr:title> <zr:description lang="en" primary="true">Meresco SRU</zr:description> </zr:databaseInfo> </zr:explain> </srw:recordData> </srw:record> </srw:explainResponse> """, result)
def testSetAttribute(self): parser = SruParser(defaultRecordSchema='default') sruHandler = CallTrace('SRUHandler') def searchRetrieve(**kwargs): yield '<mockresult/>' sruHandler.methods['searchRetrieve'] = searchRetrieve parser.addObserver(sruHandler) consume( parser.handleRequest( arguments={ 'version': ['1.1'], 'query': ['aQuery'], 'operation': ['searchRetrieve'] })) self.assertEqual(['searchRetrieve'], sruHandler.calledMethodNames()) self.assertEqual('default', sruHandler.calledMethods[0].kwargs['recordSchema']) self.assertEqual('xml', sruHandler.calledMethods[0].kwargs['recordPacking']) sruHandler.calledMethods.reset() parser.setAttribute('defaultRecordSchema', 'newDefault') parser.setAttribute('defaultRecordPacking', 'string') consume( parser.handleRequest( arguments={ 'version': ['1.1'], 'query': ['aQuery'], 'operation': ['searchRetrieve'] })) self.assertEqual('newDefault', sruHandler.calledMethods[0].kwargs['recordSchema']) self.assertEqual('string', sruHandler.calledMethods[0].kwargs['recordPacking'])
def testSetAttribute(self): parser = SruParser(defaultRecordSchema='default') sruHandler = CallTrace('SRUHandler') def searchRetrieve(**kwargs): yield '<mockresult/>' sruHandler.methods['searchRetrieve'] = searchRetrieve parser.addObserver(sruHandler) consume(parser.handleRequest(arguments={'version':['1.1'], 'query': ['aQuery'], 'operation':['searchRetrieve']})) self.assertEqual(['searchRetrieve'], sruHandler.calledMethodNames()) self.assertEqual('default', sruHandler.calledMethods[0].kwargs['recordSchema']) self.assertEqual('xml', sruHandler.calledMethods[0].kwargs['recordPacking']) sruHandler.calledMethods.reset() parser.setAttribute('defaultRecordSchema', 'newDefault') parser.setAttribute('defaultRecordPacking', 'string') consume(parser.handleRequest(arguments={'version':['1.1'], 'query': ['aQuery'], 'operation':['searchRetrieve']})) self.assertEqual('newDefault', sruHandler.calledMethods[0].kwargs['recordSchema']) self.assertEqual('string', sruHandler.calledMethods[0].kwargs['recordPacking'])
def testDiagnosticGetHandledByObserver(self): def mockAdditionalDiagnosticDetails(**kwargs): yield "additional details" observer = CallTrace( methods={ 'additionalDiagnosticDetails': mockAdditionalDiagnosticDetails }) dna = be((Observable(), (SruParser(), (observer, )))) response = ''.join( compose(dna.all.handleRequest(arguments={'startRecord': ['aap']}))) self.assertEqual(['additionalDiagnosticDetails'], observer.calledMethodNames()) self.assertTrue( "<details>operation - additional details</details>" in response, response)
def main(reactor, port, databasePath): drilldownFields = [ DrilldownField('untokenized.field2'), DrilldownField('untokenized.fieldHier', hierarchical=True) ] fieldRegistry = FieldRegistry(drilldownFields) luceneSettings = LuceneSettings(fieldRegistry=fieldRegistry, commitCount=30, commitTimeout=1, analyzer=MerescoDutchStemmingAnalyzer()) lucene = Lucene(path=join(databasePath, 'lucene'), reactor=reactor, name='main', settings=luceneSettings) lucene2Settings = LuceneSettings(fieldRegistry=fieldRegistry, commitTimeout=0.1) lucene2 = Lucene(path=join(databasePath, 'lucene2'), reactor=reactor, name='main2', settings=lucene2Settings) termNumerator = TermNumerator(path=join(databasePath, 'termNumerator')) emptyLuceneSettings = LuceneSettings(commitTimeout=1) multiLuceneHelix = ( MultiLucene(defaultCore='main'), (Lucene(path=join(databasePath, 'lucene-empty'), reactor=reactor, name='empty-core', settings=emptyLuceneSettings), ), (lucene, ), (lucene2, ), ) storageComponent = StorageComponent( directory=join(databasePath, 'storage')) return \ (Observable(), (ObservableHttpServer(reactor=reactor, port=port), (BasicHttpHandler(), (ApacheLogger(outputStream=stdout), (PathFilter("/info", excluding=[ '/info/version', '/info/name', '/update', '/sru', '/remote', '/via-remote-sru', ]), (DynamicHtml( [dynamicPath], reactor=reactor, indexPage='/info', additionalGlobals={ 'VERSION': version, } ), ) ), (PathFilter("/info/version"), (StringServer(version, ContentTypePlainText), ) ), (PathFilter("/info/name"), (StringServer('Meresco Lucene', ContentTypePlainText),) ), (PathFilter("/static"), (PathRename(lambda path: path[len('/static'):]), (FileServer(staticPath),) ) ), (PathFilter("/update_main", excluding=['/update_main2']), uploadHelix(lucene, termNumerator, storageComponent, drilldownFields, fieldRegistry=luceneSettings.fieldRegistry), ), (PathFilter("/update_main2"), uploadHelix(lucene2, termNumerator, storageComponent, drilldownFields, fieldRegistry=lucene2Settings.fieldRegistry), ), (PathFilter('/sru'), (SruParser(defaultRecordSchema='record'), (SruHandler(), (MultiCqlToLuceneQuery( defaultCore='main', coreToCqlLuceneQueries={ "main": CqlToLuceneQuery([], luceneSettings=luceneSettings), "main2": CqlToLuceneQuery([], luceneSettings=lucene2Settings), "empty-core": CqlToLuceneQuery([], luceneSettings=emptyLuceneSettings), }), multiLuceneHelix, ), (SRUTermDrilldown(defaultFormat='xml'),), (SruDuplicateCount(),), (storageComponent,), ) ) ), (PathFilter('/via-remote-sru'), (SruParser(defaultRecordSchema='record'), (SruHandler(), (LuceneRemote(host='localhost', port=port, path='/remote'),), (SRUTermDrilldown(defaultFormat='xml'),), (SruDuplicateCount(),), (storageComponent,), ) ) ), (PathFilter('/remote'), (LuceneRemoteService(reactor=reactor), (MultiCqlToLuceneQuery( defaultCore='main', coreToCqlLuceneQueries={ "main": CqlToLuceneQuery([], luceneSettings=luceneSettings), "main2": CqlToLuceneQuery([], luceneSettings=lucene2Settings), "empty-core": CqlToLuceneQuery([], luceneSettings=emptyLuceneSettings), }), multiLuceneHelix, ) ) ), (PathFilter('/autocomplete'), (Autocomplete('localhost', port, '/autocomplete', '__all__', '?', 5, '?', '?'), (lucene,), ) ) ) ) ) )
class SrwTest(SeecrTestCase): def setUp(self): SeecrTestCase.setUp(self) self.srw = Srw() self.sruParser = SruParser() self.sruHandler = SruHandler() self.srw.addObserver(self.sruParser) self.sruParser.addObserver(self.sruHandler) self.response = StopIteration(Response(total=1, hits=[Hit('0')])) def executeQuery(**kwargs): raise self.response yield def retrieveData(**kwargs): raise StopIteration('data') yield self.observer = CallTrace( methods={ 'executeQuery': executeQuery, 'retrieveData': retrieveData }, emptyGeneratorMethods=[ 'extraResponseData', 'echoedExtraRequestData', 'additionalDiagnosticDetails', 'extraRecordData' ]) self.sruHandler.addObserver(self.observer) def testNonSoap(self): """Wrong Soap envelope or body""" invalidSoapEnvelope = '<?xml version="1.0"?><SOAP:Envelope xmlns:SOAP="http://wrong.example.org/soap/envelope/"><SOAP:Body>%s</SOAP:Body></SOAP:Envelope>' request = invalidSoapEnvelope % SRW_REQUEST % argumentsWithMandatory % "" response = "".join(list(Srw().handleRequest(Body=request))) self.assertEqualsWS("""HTTP/1.0 500 Internal Server Error Content-Type: text/xml; charset=utf-8 <SOAP:Envelope xmlns:SOAP="http://schemas.xmlsoap.org/soap/envelope/"><SOAP:Body><SOAP:Fault><faultcode>SOAP:VersionMismatch</faultcode><faultstring>The processing party found an invalid namespace for the SOAP Envelope element</faultstring></SOAP:Fault></SOAP:Body></SOAP:Envelope>""", response) def testMalformedXML(self): """Stuff that is not even XML""" request = 'This is not even XML' response = "".join(self.srw.handleRequest(Body=request)) self.assertTrue('<faultcode>SOAP:Server.userException</faultcode>' in response) def testBadSrwRequest(self): request = soapEnvelope % """<srw:searchRetrieveRequest xmlns:srw="http://wrong.example.org/srw"> <srw:version>1.2</srw:version> <srw:query>query</srw:query> </srw:searchRetrieveRequest>""" response = asString(self.srw.handleRequest(Body=request)) header, body = response.split('\r\n\r\n') self.assertEquals(['1'], xpath(XML(body), '//srw:searchRetrieveResponse/srw:numberOfRecords/text()')) def testNonSRUArguments(self): """Arguments that are invalid in any SRU implementation""" request = soapEnvelope % SRW_REQUEST % argumentsWithMandatory % """<SRW:illegalParameter>value</SRW:illegalParameter>""" response = "".join(self.srw.handleRequest(Body=request)) self.assertEqualsWS(httpResponse % soapEnvelope % """<srw:searchRetrieveResponse %(xmlns_srw)s %(xmlns_diag)s %(xmlns_xcql)s %(xmlns_dc)s %(xmlns_meresco_srw)s> <srw:version>1.1</srw:version><srw:numberOfRecords>0</srw:numberOfRecords><srw:diagnostics><diagnostic xmlns="http://www.loc.gov/zing/srw/diagnostic/"> <uri>info://srw/diagnostics/1/8</uri> <details>illegalParameter</details> <message>Unsupported Parameter</message> </diagnostic></srw:diagnostics></srw:searchRetrieveResponse>""" % namespaces, response) def testNonSRWArguments(self): """Arguments that are part of SRU, but not of SRW (operation (done), stylesheet) """ request = soapEnvelope % SRW_REQUEST % argumentsWithMandatory % """<SRW:stylesheet>http://example.org/style.xsl</SRW:stylesheet>""" response = "".join(self.srw.handleRequest(Body=request)) self.assertEqualsWS(httpResponse % soapEnvelope % """<srw:searchRetrieveResponse %(xmlns_srw)s %(xmlns_diag)s %(xmlns_xcql)s %(xmlns_dc)s %(xmlns_meresco_srw)s> <srw:version>1.1</srw:version><srw:numberOfRecords>0</srw:numberOfRecords><srw:diagnostics><diagnostic xmlns="http://www.loc.gov/zing/srw/diagnostic/"> <uri>info://srw/diagnostics/1/8</uri> <details>stylesheet</details> <message>Unsupported Parameter</message> </diagnostic></srw:diagnostics></srw:searchRetrieveResponse>""" % namespaces, response) def testOperationIsIllegal(self): request = soapEnvelope % SRW_REQUEST % """<SRW:version>1.1</SRW:version><SRW:operation>explain</SRW:operation>""" response = "".join(self.srw.handleRequest(Body=request)) self.assertEqualsWS(httpResponse % soapEnvelope % """<srw:searchRetrieveResponse %(xmlns_srw)s %(xmlns_diag)s %(xmlns_xcql)s %(xmlns_dc)s %(xmlns_meresco_srw)s> <srw:version>1.1</srw:version><srw:numberOfRecords>0</srw:numberOfRecords><srw:diagnostics><diagnostic xmlns="http://www.loc.gov/zing/srw/diagnostic/"> <uri>info://srw/diagnostics/1/4</uri> <details>explain</details> <message>Unsupported Operation</message> </diagnostic></srw:diagnostics></srw:searchRetrieveResponse>""" % namespaces, response) def testContentType(self): request = soapEnvelope % SRW_REQUEST % argumentsWithMandatory % '' response = asString(self.srw.handleRequest(Body=request)) self.assertTrue('text/xml; charset=utf-8' in response, response) def testNormalOperation(self): request = soapEnvelope % SRW_REQUEST % argumentsWithMandatory % "" self.response = StopIteration(Response(total=1, hits=[Hit('recordId')])) del self.observer.methods['retrieveData'] def retrieveData(identifier, name): raise StopIteration("<DATA>%s-%s</DATA>" % (identifier, name)) yield self.observer.methods['retrieveData'] = retrieveData result = "".join(compose(self.srw.handleRequest(Body=request))) self.assertEqualsWS(httpResponse % soapEnvelope % wrappedMockAnswer % ('recordId', 'dc.author = "jones" and dc.title = "smith"'), result) def testEmptySortKeys(self): request = soapEnvelope % SRW_REQUEST % argumentsWithMandatory % "<SRW:sortKeys/>" self.response = StopIteration(Response(total=0, hits=[])) result = "".join(compose(self.srw.handleRequest(Body=request))) executeQueryKwargs = self.observer.calledMethods[0].kwargs self.assertFalse("sortKeys" in executeQueryKwargs, executeQueryKwargs) def testArgumentsAreNotUnicodeStrings(self): """JJ/TJ: unicode strings somehow paralyse server requests. So ensure every argument is a str!""" """KvS d.d. 2007/11/15 - is this true in the Component-context too?""" request = soapEnvelope % SRW_REQUEST % argumentsWithMandatory % "" component = Srw() arguments = component._soapXmlToArguments(request) for key in arguments: self.assertTrue(type(key) == str) def testExampleFromLibraryOffCongressSite(self): """testExampleFromLibraryOffCongressSite - Integration test based on http://www.loc.gov/standards/sru/srw/index.html spelling error ("recordSchema") corrected """ request = """<SOAP:Envelope xmlns:SOAP="http://schemas.xmlsoap.org/soap/envelope/"> <SOAP:Body> <SRW:searchRetrieveRequest xmlns:SRW="http://www.loc.gov/zing/srw/"> <SRW:version>1.1</SRW:version> <SRW:query>dc.author = "jones" and dc.title = "smith"</SRW:query> <SRW:startRecord>1</SRW:startRecord> <SRW:maximumRecords>10</SRW:maximumRecords> <SRW:recordSchema>info:srw/schema/1/mods-v3.0</SRW:recordSchema> </SRW:searchRetrieveRequest> </SOAP:Body> </SOAP:Envelope>""" self.response = StopIteration(Response(total=1, hits=[Hit('recordId')])) del self.observer.methods['retrieveData'] def retrieveData(identifier, name): raise StopIteration("<DATA>%s-%s</DATA>" % (identifier, name)) yield self.observer.methods['retrieveData'] = retrieveData response = "".join(compose(self.srw.handleRequest(Body=request))) echoRequest = """<srw:echoedSearchRetrieveRequest> <srw:version>1.1</srw:version> <srw:query>dc.author = "jones" and dc.title = "smith"</srw:query> <srw:startRecord>1</srw:startRecord> <srw:maximumRecords>10</srw:maximumRecords> <srw:recordPacking>xml</srw:recordPacking> <srw:recordSchema>info:srw/schema/1/mods-v3.0</srw:recordSchema> </srw:echoedSearchRetrieveRequest>""" self.assertEqualsWS(httpResponse % soapEnvelope % searchRetrieveResponse % (1, '<srw:records><srw:record><srw:recordSchema>info:srw/schema/1/mods-v3.0</srw:recordSchema><srw:recordPacking>xml</srw:recordPacking><srw:recordData><DATA>recordId-info:srw/schema/1/mods-v3.0</DATA></srw:recordData></srw:record></srw:records>' +echoRequest), response) @stderr_replaced def testConstructorVariablesAreUsed(self): request = soapEnvelope % SRW_REQUEST % argumentsWithMandatory % "" srw = Srw( defaultRecordSchema="DEFAULT_RECORD_SCHEMA", defaultRecordPacking="DEFAULT_RECORD_PACKING") sruParser = SruParser() srw.addObserver(sruParser) sruParser.addObserver(self.sruHandler) response = Response(total=1, hits=[Hit(1)]) def executeQuery(**kwargs): raise StopIteration(response) yield @asyncnoreturnvalue def methodAsGenerator(**kwargs): pass observer = CallTrace( methods={ 'executeQuery': executeQuery, 'extraResponseData': methodAsGenerator, 'echoedExtraRequestData': methodAsGenerator, }, emptyGeneratorMethods=[ 'additionalDiagnosticDetails', 'extraRecordData', ]) self.sruHandler.addObserver(observer) response = "".join(compose(srw.handleRequest(Body=request))) self.assertTrue("DEFAULT_RECORD_SCHEMA" in response, response) self.assertTrue("DEFAULT_RECORD_PACKING" in response, response)
def main(reactor, port, statePath, lucenePort, **ignored): ######## START Lucene Integration ############################################################### defaultLuceneSettings = LuceneSettings( commitTimeout=30, readonly=True,) http11Request = be( (HttpRequest1_1(), (SocketPool(reactor=reactor, unusedTimeout=5, limits=dict(totalSize=100, destinationSize=10)),), ) ) luceneIndex = luceneAndReaderConfig(defaultLuceneSettings.clone(readonly=True), http11Request, lucenePort) luceneRoHelix = be( (AdapterToLuceneQuery( defaultCore=DEFAULT_CORE, coreConverters={ DEFAULT_CORE: QueryExpressionToLuceneQueryDict(UNQUALIFIED_TERM_FIELDS, luceneSettings=luceneIndex.settings), } ), (MultiLucene(host='127.0.0.1', port=lucenePort, defaultCore=DEFAULT_CORE), (luceneIndex,), (http11Request,), ) ) ) ######## END Lucene Integration ############################################################### fieldnameRewrites = {} def fieldnameRewrite(name): return fieldnameRewrites.get(name, name) def drilldownFieldnamesTranslate(fieldname): untokenizedName = untokenizedFieldname(fieldname) if untokenizedName in untokenizedFieldnames: fieldname = untokenizedName return fieldnameRewrite(fieldname) convertToComposedQuery = ConvertToComposedQuery( resultsFrom=DEFAULT_CORE, matches=[], drilldownFieldnamesTranslate=drilldownFieldnamesTranslate ) strategie = Md5HashDistributeStrategy() storage = StorageComponent(join(statePath, 'store'), strategy=strategie, partsRemovedOnDelete=[HEADER_PARTNAME, META_PARTNAME, METADATA_PARTNAME, OAI_DC_PARTNAME, LONG_PARTNAME, SHORT_PARTNAME]) # Wat doet dit? cqlClauseConverters = [ RenameFieldForExact( untokenizedFields=untokenizedFieldnames, untokenizedPrefix=UNTOKENIZED_PREFIX, ).filterAndModifier(), SearchTermFilterAndModifier( shouldModifyFieldValue=lambda *args: True, fieldnameModifier=fieldnameRewrite ).filterAndModifier(), ] executeQueryHelix = \ (FilterMessages(allowed=['executeQuery']), (CqlMultiSearchClauseConversion(cqlClauseConverters, fromKwarg='query'), (DrilldownQueries(), (convertToComposedQuery, (luceneRoHelix,), ) ) ) ) return \ (Observable(), (ObservableHttpServer(reactor, port, compressResponse=True), (BasicHttpHandler(), (PathFilter(['/sru']), (SruParser( host='sru.narcis.nl', port=80, defaultRecordSchema='knaw_short', defaultRecordPacking='xml'), (SruLimitStartRecord(limitBeyond=4000), (SruHandler( includeQueryTimes=False, extraXParameters=[], enableCollectLog=False), #2017-03-24T12:00:33Z 127.0.0.1 3.5K 0.019s - /sru OF (TRUE): 2017-03-24T11:58:53Z 127.0.0.1 2.3K 0.004s 1hits /sru maximumRecords=10&operation=searchRetrieve&query=untokenized.dd_year+exact+%221993%22&recordPacking=xml&recordSchema=knaw_short&startRecord=1&version=1.2 (SruTermDrilldown(),), executeQueryHelix, (StorageAdapter(), (storage,) ) ) ) ) ), (PathFilter('/rss'), (Rss( supportedLanguages = ['nl','en'], # defaults to first, if requested language is not available or supplied. title = {'nl':'NARCIS', 'en':'NARCIS'}, description = {'nl':'NARCIS: De toegang tot de Nederlandse wetenschapsinformatie', 'en':'NARCIS: The gateway to Dutch scientific information'}, link = {'nl':'http://www.narcis.nl/?Language=nl', 'en':'http://www.narcis.nl/?Language=en'}, maximumRecords = 20), executeQueryHelix, (RssItem( nsMap=NAMESPACEMAP, title = ('knaw_short', {'nl':'//short:metadata/short:titleInfo[not (@xml:lang)]/short:title/text()', 'en':'//short:metadata/short:titleInfo[@xml:lang="en"]/short:title/text()'}), description = ('knaw_short', {'nl':'//short:abstract[not (@xml:lang)]/text()', 'en':'//short:abstract[@xml:lang="en"]/text()'}), pubdate = ('knaw_short', '//short:dateIssued/short:parsed/text()'), linkTemplate = 'http://www.narcis.nl/%(wcpcollection)s/RecordID/%(oai_identifier)s/Language/%(language)s', wcpcollection = ('meta', '//*[local-name() = "collection"]/text()'), oai_identifier = ('meta', '//meta:record/meta:id/text()'), language = ('Dummy: Language is auto provided by the calling RSS component, but needs to be present to serve the linkTemplate.') ), (StorageAdapter(), (storage,) ) ) ) ) ) ) )
class SrwTest(SeecrTestCase): def setUp(self): SeecrTestCase.setUp(self) self.srw = Srw() self.sruParser = SruParser() self.sruHandler = SruHandler() self.srw.addObserver(self.sruParser) self.sruParser.addObserver(self.sruHandler) self.response = Response(total=1, hits=[Hit('0')]) def executeQuery(**kwargs): return self.response yield def retrieveData(**kwargs): return 'data' yield self.observer = CallTrace(methods={ 'executeQuery': executeQuery, 'retrieveData': retrieveData }, emptyGeneratorMethods=[ 'extraResponseData', 'echoedExtraRequestData', 'additionalDiagnosticDetails', 'extraRecordData' ]) self.sruHandler.addObserver(self.observer) def testNonSoap(self): """Wrong Soap envelope or body""" invalidSoapEnvelope = '<?xml version="1.0"?><SOAP:Envelope xmlns:SOAP="http://wrong.example.org/soap/envelope/"><SOAP:Body>%s</SOAP:Body></SOAP:Envelope>' request = invalidSoapEnvelope % SRW_REQUEST % argumentsWithMandatory % "" response = "".join(list(Srw().handleRequest(Body=request.encode()))) self.assertEqualsWS( """HTTP/1.0 500 Internal Server Error Content-Type: text/xml; charset=utf-8 <SOAP:Envelope xmlns:SOAP="http://schemas.xmlsoap.org/soap/envelope/"><SOAP:Body><SOAP:Fault><faultcode>SOAP:VersionMismatch</faultcode><faultstring>The processing party found an invalid namespace for the SOAP Envelope element</faultstring></SOAP:Fault></SOAP:Body></SOAP:Envelope>""", response) def testMalformedXML(self): """Stuff that is not even XML""" request = b'This is not even XML' response = "".join(self.srw.handleRequest(Body=request)) self.assertTrue( '<faultcode>SOAP:Server.userException</faultcode>' in response) def testBadSrwRequest(self): request = soapEnvelope % """<srw:searchRetrieveRequest xmlns:srw="http://wrong.example.org/srw"> <srw:version>1.2</srw:version> <srw:query>query</srw:query> </srw:searchRetrieveRequest>""" response = asString(self.srw.handleRequest(Body=request)) header, body = response.split('\r\n\r\n') self.assertEqual( ['1'], xpath(XML(body), '//srw:searchRetrieveResponse/srw:numberOfRecords/text()')) def testNonSRUArguments(self): """Arguments that are invalid in any SRU implementation""" request = soapEnvelope % SRW_REQUEST % argumentsWithMandatory % """<SRW:illegalParameter>value</SRW:illegalParameter>""" response = "".join(self.srw.handleRequest(Body=request)) self.assertEqualsWS( httpResponse % soapEnvelope % """<srw:searchRetrieveResponse %(xmlns_srw)s %(xmlns_diag)s %(xmlns_xcql)s %(xmlns_dc)s %(xmlns_meresco_srw)s> <srw:version>1.1</srw:version><srw:numberOfRecords>0</srw:numberOfRecords><srw:diagnostics><diagnostic xmlns="http://www.loc.gov/zing/srw/diagnostic/"> <uri>info://srw/diagnostics/1/8</uri> <details>illegalParameter</details> <message>Unsupported Parameter</message> </diagnostic></srw:diagnostics></srw:searchRetrieveResponse>""" % namespaces, response) def testNonSRWArguments(self): """Arguments that are part of SRU, but not of SRW (operation (done), stylesheet) """ request = soapEnvelope % SRW_REQUEST % argumentsWithMandatory % """<SRW:stylesheet>http://example.org/style.xsl</SRW:stylesheet>""" response = "".join(self.srw.handleRequest(Body=request)) self.assertEqualsWS( httpResponse % soapEnvelope % """<srw:searchRetrieveResponse %(xmlns_srw)s %(xmlns_diag)s %(xmlns_xcql)s %(xmlns_dc)s %(xmlns_meresco_srw)s> <srw:version>1.1</srw:version><srw:numberOfRecords>0</srw:numberOfRecords><srw:diagnostics><diagnostic xmlns="http://www.loc.gov/zing/srw/diagnostic/"> <uri>info://srw/diagnostics/1/8</uri> <details>stylesheet</details> <message>Unsupported Parameter</message> </diagnostic></srw:diagnostics></srw:searchRetrieveResponse>""" % namespaces, response) def testOperationIsIllegal(self): request = soapEnvelope % SRW_REQUEST % """<SRW:version>1.1</SRW:version><SRW:operation>explain</SRW:operation>""" response = "".join(self.srw.handleRequest(Body=request)) self.assertEqualsWS( httpResponse % soapEnvelope % """<srw:searchRetrieveResponse %(xmlns_srw)s %(xmlns_diag)s %(xmlns_xcql)s %(xmlns_dc)s %(xmlns_meresco_srw)s> <srw:version>1.1</srw:version><srw:numberOfRecords>0</srw:numberOfRecords><srw:diagnostics><diagnostic xmlns="http://www.loc.gov/zing/srw/diagnostic/"> <uri>info://srw/diagnostics/1/4</uri> <details>explain</details> <message>Unsupported Operation</message> </diagnostic></srw:diagnostics></srw:searchRetrieveResponse>""" % namespaces, response) def testContentType(self): request = soapEnvelope % SRW_REQUEST % argumentsWithMandatory % '' response = asString(self.srw.handleRequest(Body=request)) self.assertTrue('text/xml; charset=utf-8' in response, response) def testNormalOperation(self): request = soapEnvelope % SRW_REQUEST % argumentsWithMandatory % "" self.response = Response(total=1, hits=[Hit('recordId')]) del self.observer.methods['retrieveData'] def retrieveData(identifier, name): return "<DATA>%s-%s</DATA>" % (identifier, name) yield self.observer.methods['retrieveData'] = retrieveData result = "".join(compose(self.srw.handleRequest(Body=request))) self.assertEqualsWS( httpResponse % soapEnvelope % wrappedMockAnswer % ('recordId', 'dc.author = "jones" and dc.title = "smith"'), result) def testEmptySortKeys(self): request = soapEnvelope % SRW_REQUEST % argumentsWithMandatory % "<SRW:sortKeys/>" self.response = Response(total=0, hits=[]) result = "".join(compose(self.srw.handleRequest(Body=request))) executeQueryKwargs = self.observer.calledMethods[0].kwargs self.assertFalse("sortKeys" in executeQueryKwargs, executeQueryKwargs) def testArgumentsAreNotUnicodeStrings(self): """JJ/TJ: unicode strings somehow paralyse server requests. So ensure every argument is a str!""" """KvS d.d. 2007/11/15 - is this true in the Component-context too?""" request = soapEnvelope % SRW_REQUEST % argumentsWithMandatory % "" component = Srw() arguments = component._soapXmlToArguments(request) for key in arguments: self.assertTrue(type(key) == str) def testExampleFromLibraryOffCongressSite(self): """testExampleFromLibraryOffCongressSite - Integration test based on http://www.loc.gov/standards/sru/srw/index.html spelling error ("recordSchema") corrected """ request = """<SOAP:Envelope xmlns:SOAP="http://schemas.xmlsoap.org/soap/envelope/"> <SOAP:Body> <SRW:searchRetrieveRequest xmlns:SRW="http://www.loc.gov/zing/srw/"> <SRW:version>1.1</SRW:version> <SRW:query>dc.author = "jones" and dc.title = "smith"</SRW:query> <SRW:startRecord>1</SRW:startRecord> <SRW:maximumRecords>10</SRW:maximumRecords> <SRW:recordSchema>info:srw/schema/1/mods-v3.0</SRW:recordSchema> </SRW:searchRetrieveRequest> </SOAP:Body> </SOAP:Envelope>""" self.response = Response(total=1, hits=[Hit('recordId')]) del self.observer.methods['retrieveData'] def retrieveData(identifier, name): return "<DATA>%s-%s</DATA>" % (identifier, name) yield self.observer.methods['retrieveData'] = retrieveData response = "".join(compose(self.srw.handleRequest(Body=request))) echoRequest = """<srw:echoedSearchRetrieveRequest> <srw:version>1.1</srw:version> <srw:query>dc.author = "jones" and dc.title = "smith"</srw:query> <srw:startRecord>1</srw:startRecord> <srw:maximumRecords>10</srw:maximumRecords> <srw:recordPacking>xml</srw:recordPacking> <srw:recordSchema>info:srw/schema/1/mods-v3.0</srw:recordSchema> </srw:echoedSearchRetrieveRequest>""" self.assertEqualsWS( httpResponse % soapEnvelope % searchRetrieveResponse % (1, '<srw:records><srw:record><srw:recordSchema>info:srw/schema/1/mods-v3.0</srw:recordSchema><srw:recordPacking>xml</srw:recordPacking><srw:recordData><DATA>recordId-info:srw/schema/1/mods-v3.0</DATA></srw:recordData></srw:record></srw:records>' + echoRequest), response) @stderr_replaced def testConstructorVariablesAreUsed(self): request = soapEnvelope % SRW_REQUEST % argumentsWithMandatory % "" srw = Srw(defaultRecordSchema="DEFAULT_RECORD_SCHEMA", defaultRecordPacking="DEFAULT_RECORD_PACKING") sruParser = SruParser() srw.addObserver(sruParser) sruParser.addObserver(self.sruHandler) response = Response(total=1, hits=[Hit(1)]) def executeQuery(**kwargs): return response yield @asyncnoreturnvalue def methodAsGenerator(**kwargs): pass observer = CallTrace(methods={ 'executeQuery': executeQuery, 'extraResponseData': methodAsGenerator, 'echoedExtraRequestData': methodAsGenerator, }, emptyGeneratorMethods=[ 'additionalDiagnosticDetails', 'extraRecordData', ]) self.sruHandler.addObserver(observer) response = "".join(compose(srw.handleRequest(Body=request))) self.assertTrue("DEFAULT_RECORD_SCHEMA" in response, response) self.assertTrue("DEFAULT_RECORD_PACKING" in response, response)
def main(reactor, port, statePath, lucenePort, gatewayPort, quickCommit=False, **ignored): ######## START Lucene Integration ############################################################### defaultLuceneSettings = LuceneSettings( commitTimeout=30, readonly=True, ) http11Request = be(( HttpRequest1_1(), (SocketPool(reactor=reactor, unusedTimeout=5, limits=dict(totalSize=100, destinationSize=10)), ), )) luceneIndex = luceneAndReaderConfig( defaultLuceneSettings.clone(readonly=True), http11Request, lucenePort) luceneRoHelix = be( (AdapterToLuceneQuery(defaultCore=DEFAULT_CORE, coreConverters={ DEFAULT_CORE: QueryExpressionToLuceneQueryDict( UNQUALIFIED_TERM_FIELDS, luceneSettings=luceneIndex.settings), }), ( MultiLucene(host='localhost', port=lucenePort, defaultCore=DEFAULT_CORE), (luceneIndex, ), (http11Request, ), ))) ######## END Lucene Integration ############################################################### fieldnameRewrites = { # UNTOKENIZED_PREFIX+'genre': UNTOKENIZED_PREFIX+'dc:genre', } def fieldnameRewrite(name): return fieldnameRewrites.get(name, name) def drilldownFieldnamesTranslate(fieldname): untokenizedName = untokenizedFieldname(fieldname) if untokenizedName in untokenizedFieldnames: fieldname = untokenizedName return fieldnameRewrite(fieldname) convertToComposedQuery = ConvertToComposedQuery( resultsFrom=DEFAULT_CORE, matches=[], drilldownFieldnamesTranslate=drilldownFieldnamesTranslate) strategie = Md5HashDistributeStrategy() storage = StorageComponent(join(statePath, 'store'), strategy=strategie, partsRemovedOnDelete=[ HEADER_PARTNAME, META_PARTNAME, METADATA_PARTNAME, OAI_DC_PARTNAME, LONG_PARTNAME, SHORT_PARTNAME, OPENAIRE_PARTNAME ]) oaiJazz = OaiJazz(join(statePath, 'oai')) oaiJazz.updateMetadataFormat( OAI_DC_PARTNAME, "http://www.openarchives.org/OAI/2.0/oai_dc.xsd", "http://purl.org/dc/elements/1.1/") oai_oa_cerifJazz = OaiJazz(join(statePath, 'oai_cerif')) oai_oa_cerifJazz.updateMetadataFormat( OPENAIRE_PARTNAME, "https://www.openaire.eu/schema/cris/current/openaire-cerif-profile.xsd", "https://www.openaire.eu/cerif-profile/1.1/") # All of the following OAI-PMH sets shall be recognized by the CRIS, even if not all of them are populated. oai_oa_cerifJazz.updateSet("openaire_cris_projects", "OpenAIRE_CRIS_projects") oai_oa_cerifJazz.updateSet("openaire_cris_orgunits", "OpenAIRE_CRIS_orgunits") oai_oa_cerifJazz.updateSet("openaire_cris_persons", "OpenAIRE_CRIS_persons") oai_oa_cerifJazz.updateSet("openaire_cris_patents", "OpenAIRE_CRIS_patents") oai_oa_cerifJazz.updateSet("openaire_cris_products", "OpenAIRE_CRIS_products") oai_oa_cerifJazz.updateSet("openaire_cris_publications", "OpenAIRE_CRIS_publications") oai_oa_cerifJazz.updateSet("openaire_cris_funding", "OpenAIRE_CRIS_funding") oai_oa_cerifJazz.updateSet("openaire_cris_events", "OpenAIRE_CRIS_events") oai_oa_cerifJazz.updateSet("openaire_cris_equipments", "OpenAIRE_CRIS_equipments") cqlClauseConverters = [ RenameFieldForExact( untokenizedFields=untokenizedFieldnames, untokenizedPrefix=UNTOKENIZED_PREFIX, ).filterAndModifier(), SearchTermFilterAndModifier( shouldModifyFieldValue=lambda *args: True, fieldnameModifier=fieldnameRewrite).filterAndModifier(), ] periodicGateWayDownload = PeriodicDownload( reactor, host='localhost', port=gatewayPort, schedule=Schedule( period=1 if quickCommit else 10 ), # WST: Interval in seconds before sending a new request to the GATEWAY in case of an error while processing batch records.(default=1). IntegrationTests need 1 second! Otherwise tests will fail! name='api', autoStart=True) oaiDownload = OaiDownloadProcessor(path='/oaix', metadataPrefix=NORMALISED_DOC_NAME, workingDirectory=join( statePath, 'harvesterstate', 'gateway'), userAgentAddition='ApiServer', xWait=True, name='api', autoCommit=False) executeQueryHelix = \ (FilterMessages(allowed=['executeQuery']), (CqlMultiSearchClauseConversion(cqlClauseConverters, fromKwarg='query'), (DrilldownQueries(), (convertToComposedQuery, (luceneRoHelix,), ) ) ) ) return \ (Observable(), createDownloadHelix(reactor, periodicGateWayDownload, oaiDownload, storage, oaiJazz, oai_oa_cerifJazz), (ObservableHttpServer(reactor, port, compressResponse=True), (BasicHttpHandler(), (PathFilter(["/oai"]), (OaiPmh(repositoryName="NARCIS OAI-pmh", adminEmail="*****@*****.**", externalUrl="http://oai.narcis.nl"), (oaiJazz,), (StorageAdapter(), (storage,) ), (OaiBranding( url="http://www.narcis.nl/images/logos/logo-knaw-house.gif", link="http://oai.narcis.nl", title="Narcis - The gateway to scholarly information in The Netherlands"), ), (OaiProvenance( nsMap=NAMESPACEMAP, baseURL=('meta', '//meta:repository/meta:baseurl/text()'), harvestDate=('meta', '//meta:record/meta:harvestdate/text()'), metadataNamespace=('meta', '//meta:record/meta:metadataNamespace/text()'), identifier=('header','//oai:identifier/text()'), datestamp=('header', '//oai:datestamp/text()') ), (storage,) ) ) ), (PathFilter(["/cerif"]), (OaiPmhDans(repositoryName="OpenAIRE CERIF", adminEmail="*****@*****.**", repositoryIdentifier="services.nod.dans.knaw.nl", externalUrl="http://services.nod.dans.knaw.nl"), #TODO: pathFilter should resemble proxy path (oai_oa_cerifJazz,), (StorageAdapter(), (storage,) ), (OaiOpenAIREDescription( serviceid='organisation:ORG1242054', acronym='services.nod.dans.knaw.nl', name='NARCIS', description='Compliant with the OpenAIRE Guidelines for CRIS Managers v.1.1.', website='https://www.narcis.nl', baseurl='http://services.nod.dans.knaw.nl/oa-cerif', subjectheading='', orgunitid='organisation:ORG1242054', owneracronym='DANS'), ), # (OaiBranding( # url="http://www.narcis.nl/images/logos/logo-knaw-house.gif", # link="http://oai.narcis.nl", # title="Narcis - The gateway to scholarly information in The Netherlands"), # ), (OaiProvenance( nsMap=NAMESPACEMAP, baseURL=('meta', '//meta:repository/meta:baseurl/text()'), harvestDate=('meta', '//meta:record/meta:harvestdate/text()'), metadataNamespace=('meta', '//meta:record/meta:metadataNamespace/text()'), identifier=('header','//oai:identifier/text()'), datestamp=('header', '//oai:datestamp/text()') ), (storage,) ) ) ), (PathFilter(['/sru']), (SruParser( host='sru.narcis.nl', port=80, defaultRecordSchema='knaw_short', defaultRecordPacking='xml'), (SruLimitStartRecord(limitBeyond=4000), (SruHandler( includeQueryTimes=False, extraXParameters=[], enableCollectLog=False), (SruTermDrilldown(),), executeQueryHelix, (StorageAdapter(), (storage,) ) ) ) ) ), (PathFilter('/rss'), (Rss( supportedLanguages = ['nl','en'], # defaults to first, if requested language is not available or supplied. title = {'nl':'NARCIS', 'en':'NARCIS'}, description = {'nl':'NARCIS: De toegang tot de Nederlandse wetenschapsinformatie', 'en':'NARCIS: The gateway to Dutch scientific information'}, link = {'nl':'http://www.narcis.nl/?Language=nl', 'en':'http://www.narcis.nl/?Language=en'}, maximumRecords = 20), executeQueryHelix, (RssItem( nsMap=NAMESPACEMAP, title = ('knaw_short', {'nl':'//short:metadata/short:titleInfo[not (@xml:lang)]/short:title/text()', 'en':'//short:metadata/short:titleInfo[@xml:lang="en"]/short:title/text()'}), description = ('knaw_short', {'nl':'//short:abstract[not (@xml:lang)]/text()', 'en':'//short:abstract[@xml:lang="en"]/text()'}), pubdate = ('knaw_short', '//short:dateIssued/short:parsed/text()'), linkTemplate = 'http://www.narcis.nl/%(wcpcollection)s/RecordID/%(oai_identifier)s/Language/%(language)s', wcpcollection = ('meta', '//*[local-name() = "collection"]/text()'), oai_identifier = ('meta', '//meta:record/meta:id/text()'), language = ('Dummy: Language is auto provided by the calling RSS component, but needs to be present to serve the linkTemplate.') ), (StorageAdapter(), (storage,) ) ) ) ) ) ) )