예제 #1
0
 def convert(self, cql, shouldModifyFieldValue, valueModifier=None, fieldnameModifier=None):
     converter = CqlMultiSearchClauseConversion([
         SearchTermFilterAndModifier(
             shouldModifyFieldValue=shouldModifyFieldValue,
             valueModifier=valueModifier,
             fieldnameModifier=fieldnameModifier).filterAndModifier(),
         ], fromKwarg="aKwarg")
     return converter._convert(cqlToExpression(cql))
 def convert(self, cqlString):
     converter = CqlMultiSearchClauseConversion(
         [
             RenameFieldForExact(
                 ['untokenized.field', 'untokenized.prefix.*'],
                 'untokenized.').filterAndModifier()
         ],
         fromKwarg="cqlAbstractSyntaxTree")
     return converter._convert(cqlToExpression(cqlString))
 def convert(self, cql, shouldModifyFieldValue, valueModifier=None, fieldnameModifier=None):
     converter = CqlMultiSearchClauseConversion(
         [
             SearchTermFilterAndModifier(
                 shouldModifyFieldValue=shouldModifyFieldValue,
                 valueModifier=valueModifier,
                 fieldnameModifier=fieldnameModifier,
             ).filterAndModifier()
         ],
         fromKwarg="aKwarg",
     )
     return converter._convert(cqlToExpression(cql))
예제 #4
0
 def testNestedWithReplaced(self):
     q = cqlToExpression('A')
     def canModify1(expression):
         return expression.term == 'A'
     def modify1(expression):
         expression.replaceWith(cqlToExpression('aa OR bb'))
     def canModify2(expression):
         return expression.term == 'bb'
     def modify2(expression):
         expression.term = 'B'
     conversion = CqlMultiSearchClauseConversion([
                 (canModify1, modify1),
                 (canModify2, modify2),
             ], fromKwarg="thisQuery")
     result = conversion._convert(q)
     self.assertEqual(cqlToExpression('aa OR B'), result)
예제 #5
0
 def testNestedWithReplaced(self):
     q = cqlToExpression('A')
     def canModify1(expression):
         return expression.term == 'A'
     def modify1(expression):
         expression.replaceWith(cqlToExpression('aa OR bb'))
     def canModify2(expression):
         return expression.term == 'bb'
     def modify2(expression):
         expression.term = 'B'
     conversion = CqlMultiSearchClauseConversion([
                 (canModify1, modify1),
                 (canModify2, modify2),
             ], fromKwarg="thisQuery")
     result = conversion._convert(q)
     self.assertEqual(cqlToExpression('aa OR B'), result)
예제 #6
0
 def testIgnoreOtherMethodsWithQueryArgument(self):
     def canModify(expression):
         return expression.term == 'term'
     def modify(expression):
         expression.term = 'changed'
     observer = CallTrace(emptyGeneratorMethods=['method'])
     top = be((Observable(),
         (CqlMultiSearchClauseConversion([(canModify, modify)], fromKwarg='query'),
             (observer,)
         )
     ))
     consume(top.any.method(query='index = term'))
     self.assertEqual({'query': 'index = term'}, observer.calledMethods[0].kwargs)
     observer.calledMethods.reset()
     consume(top.any.method(query=cqlToExpression('index = term')))
     self.assertEqual({'query': cqlToExpression('index = changed')}, observer.calledMethods[0].kwargs)
    def testMultipleSearchClauseReplacements(self):
        ast = cqlToExpression('term1 AND term2 AND term3')

        #SEARCH_TERM(TERM('term'))
        def canModifyTerm1(expression):
            return expression.term == 'term1'

        def modifyTerm1(expression):
            expression.term = 'termOne'

        def canModifyTerm3(expression):
            return expression.term == 'term3'

        def modifyTerm3(expression):
            expression.term = 'termThree'

        observerClassic = CallTrace('observerClassic')
        observerNewStyle = CallTrace('observerNewStyle')
        classic = be(
            (Observable(), (CqlSearchClauseConversion(canModifyTerm1,
                                                      modifyTerm1,
                                                      fromKwarg="thisQuery"),
                            (CqlSearchClauseConversion(canModifyTerm3,
                                                       modifyTerm3,
                                                       fromKwarg="thisQuery"),
                             (observerClassic, )))))
        newStyle = be(
            (Observable(),
             (CqlMultiSearchClauseConversion([(canModifyTerm1, modifyTerm1),
                                              (canModifyTerm3, modifyTerm3)],
                                             fromKwarg="thisQuery"),
              (observerNewStyle, ))))

        classic.do.message(thisQuery=ast)
        newStyle.do.message(thisQuery=ast)

        self.assertEquals(['message'],
                          [m.name for m in observerClassic.calledMethods])
        resultClassic = observerClassic.calledMethods[0].kwargs['thisQuery']
        self.assertEquals(['message'],
                          [m.name for m in observerNewStyle.calledMethods])
        resultNewStyle = observerNewStyle.calledMethods[0].kwargs['thisQuery']

        self.assertEquals(cqlToExpression('termOne AND term2 AND termThree'),
                          resultClassic)
        self.assertEquals(cqlToExpression('termOne AND term2 AND termThree'),
                          resultNewStyle)
 def convert(self, cqlString):
     converter = CqlMultiSearchClauseConversion([
         RewriteBoundingBoxFields().filterAndModifier()
     ], fromKwarg="query")
     query = cqlToExpression(cqlString)
     return converter._convert(query=query)
예제 #9
0
 def convert(self, cqlString):
     converter = CqlMultiSearchClauseConversion([
             RenameFieldForExact(['untokenized.field', 'untokenized.prefix.*'], 'untokenized.').filterAndModifier()
         ], fromKwarg="cqlAbstractSyntaxTree")
     return converter._convert(cqlToExpression(cqlString))
예제 #10
0
def main(reactor,
         port,
         statePath,
         lucenePort,
         gatewayPort,
         quickCommit=False,
         **ignored):

    ######## START Lucene Integration ###############################################################
    defaultLuceneSettings = LuceneSettings(
        commitTimeout=30,
        readonly=True,
    )

    http11Request = be((
        HttpRequest1_1(),
        (SocketPool(reactor=reactor,
                    unusedTimeout=5,
                    limits=dict(totalSize=100, destinationSize=10)), ),
    ))

    luceneIndex = luceneAndReaderConfig(
        defaultLuceneSettings.clone(readonly=True), http11Request, lucenePort)

    luceneRoHelix = be(
        (AdapterToLuceneQuery(defaultCore=DEFAULT_CORE,
                              coreConverters={
                                  DEFAULT_CORE:
                                  QueryExpressionToLuceneQueryDict(
                                      UNQUALIFIED_TERM_FIELDS,
                                      luceneSettings=luceneIndex.settings),
                              }), (
                                  MultiLucene(host='localhost',
                                              port=lucenePort,
                                              defaultCore=DEFAULT_CORE),
                                  (luceneIndex, ),
                                  (http11Request, ),
                              )))

    ######## END Lucene Integration ###############################################################

    fieldnameRewrites = {
        #         UNTOKENIZED_PREFIX+'genre': UNTOKENIZED_PREFIX+'dc:genre',
    }

    def fieldnameRewrite(name):
        return fieldnameRewrites.get(name, name)

    def drilldownFieldnamesTranslate(fieldname):
        untokenizedName = untokenizedFieldname(fieldname)
        if untokenizedName in untokenizedFieldnames:
            fieldname = untokenizedName
        return fieldnameRewrite(fieldname)

    convertToComposedQuery = ConvertToComposedQuery(
        resultsFrom=DEFAULT_CORE,
        matches=[],
        drilldownFieldnamesTranslate=drilldownFieldnamesTranslate)

    strategie = Md5HashDistributeStrategy()
    storage = StorageComponent(join(statePath, 'store'),
                               strategy=strategie,
                               partsRemovedOnDelete=[
                                   HEADER_PARTNAME, META_PARTNAME,
                                   METADATA_PARTNAME, OAI_DC_PARTNAME,
                                   LONG_PARTNAME, SHORT_PARTNAME,
                                   OPENAIRE_PARTNAME
                               ])

    oaiJazz = OaiJazz(join(statePath, 'oai'))
    oaiJazz.updateMetadataFormat(
        OAI_DC_PARTNAME, "http://www.openarchives.org/OAI/2.0/oai_dc.xsd",
        "http://purl.org/dc/elements/1.1/")

    oai_oa_cerifJazz = OaiJazz(join(statePath, 'oai_cerif'))
    oai_oa_cerifJazz.updateMetadataFormat(
        OPENAIRE_PARTNAME,
        "https://www.openaire.eu/schema/cris/current/openaire-cerif-profile.xsd",
        "https://www.openaire.eu/cerif-profile/1.1/")
    # All of the following OAI-PMH sets shall be recognized by the CRIS, even if not all of them are populated.
    oai_oa_cerifJazz.updateSet("openaire_cris_projects",
                               "OpenAIRE_CRIS_projects")
    oai_oa_cerifJazz.updateSet("openaire_cris_orgunits",
                               "OpenAIRE_CRIS_orgunits")
    oai_oa_cerifJazz.updateSet("openaire_cris_persons",
                               "OpenAIRE_CRIS_persons")
    oai_oa_cerifJazz.updateSet("openaire_cris_patents",
                               "OpenAIRE_CRIS_patents")
    oai_oa_cerifJazz.updateSet("openaire_cris_products",
                               "OpenAIRE_CRIS_products")
    oai_oa_cerifJazz.updateSet("openaire_cris_publications",
                               "OpenAIRE_CRIS_publications")

    oai_oa_cerifJazz.updateSet("openaire_cris_funding",
                               "OpenAIRE_CRIS_funding")
    oai_oa_cerifJazz.updateSet("openaire_cris_events", "OpenAIRE_CRIS_events")
    oai_oa_cerifJazz.updateSet("openaire_cris_equipments",
                               "OpenAIRE_CRIS_equipments")

    cqlClauseConverters = [
        RenameFieldForExact(
            untokenizedFields=untokenizedFieldnames,
            untokenizedPrefix=UNTOKENIZED_PREFIX,
        ).filterAndModifier(),
        SearchTermFilterAndModifier(
            shouldModifyFieldValue=lambda *args: True,
            fieldnameModifier=fieldnameRewrite).filterAndModifier(),
    ]

    periodicGateWayDownload = PeriodicDownload(
        reactor,
        host='localhost',
        port=gatewayPort,
        schedule=Schedule(
            period=1 if quickCommit else 10
        ),  # WST: Interval in seconds before sending a new request to the GATEWAY in case of an error while processing batch records.(default=1). IntegrationTests need 1 second! Otherwise tests will fail!
        name='api',
        autoStart=True)

    oaiDownload = OaiDownloadProcessor(path='/oaix',
                                       metadataPrefix=NORMALISED_DOC_NAME,
                                       workingDirectory=join(
                                           statePath, 'harvesterstate',
                                           'gateway'),
                                       userAgentAddition='ApiServer',
                                       xWait=True,
                                       name='api',
                                       autoCommit=False)

    executeQueryHelix = \
        (FilterMessages(allowed=['executeQuery']),
            (CqlMultiSearchClauseConversion(cqlClauseConverters, fromKwarg='query'),
                (DrilldownQueries(),
                    (convertToComposedQuery,
                        (luceneRoHelix,),
                    )
                )
            )
        )

    return \
    (Observable(),
        createDownloadHelix(reactor, periodicGateWayDownload, oaiDownload, storage, oaiJazz, oai_oa_cerifJazz),
        (ObservableHttpServer(reactor, port, compressResponse=True),
            (BasicHttpHandler(),
                (PathFilter(["/oai"]),
                    (OaiPmh(repositoryName="NARCIS OAI-pmh", adminEmail="*****@*****.**", externalUrl="http://oai.narcis.nl"),
                        (oaiJazz,),
                        (StorageAdapter(),
                            (storage,)
                        ),
                        (OaiBranding(
                            url="http://www.narcis.nl/images/logos/logo-knaw-house.gif",
                            link="http://oai.narcis.nl",
                            title="Narcis - The gateway to scholarly information in The Netherlands"),
                        ),
                        (OaiProvenance(
                            nsMap=NAMESPACEMAP,
                            baseURL=('meta', '//meta:repository/meta:baseurl/text()'),
                            harvestDate=('meta', '//meta:record/meta:harvestdate/text()'),
                            metadataNamespace=('meta', '//meta:record/meta:metadataNamespace/text()'),
                            identifier=('header','//oai:identifier/text()'),
                            datestamp=('header', '//oai:datestamp/text()')
                            ),
                            (storage,)
                        )
                    )
                ),
                (PathFilter(["/cerif"]),
                    (OaiPmhDans(repositoryName="OpenAIRE CERIF", adminEmail="*****@*****.**", repositoryIdentifier="services.nod.dans.knaw.nl", externalUrl="http://services.nod.dans.knaw.nl"), #TODO: pathFilter should resemble proxy path
                        (oai_oa_cerifJazz,),
                        (StorageAdapter(),
                            (storage,)
                        ),
                        (OaiOpenAIREDescription(
                            serviceid='organisation:ORG1242054',
                            acronym='services.nod.dans.knaw.nl',
                            name='NARCIS',
                            description='Compliant with the OpenAIRE Guidelines for CRIS Managers v.1.1.',
                            website='https://www.narcis.nl',
                            baseurl='http://services.nod.dans.knaw.nl/oa-cerif',
                            subjectheading='',
                            orgunitid='organisation:ORG1242054',
                            owneracronym='DANS'),
                        ),
                        # (OaiBranding(
                        #     url="http://www.narcis.nl/images/logos/logo-knaw-house.gif",
                        #     link="http://oai.narcis.nl",
                        #     title="Narcis - The gateway to scholarly information in The Netherlands"),
                        # ),
                        (OaiProvenance(
                            nsMap=NAMESPACEMAP,
                            baseURL=('meta', '//meta:repository/meta:baseurl/text()'),
                            harvestDate=('meta', '//meta:record/meta:harvestdate/text()'),
                            metadataNamespace=('meta', '//meta:record/meta:metadataNamespace/text()'),
                            identifier=('header','//oai:identifier/text()'),
                            datestamp=('header', '//oai:datestamp/text()')
                            ),
                            (storage,)
                        )
                    )
                ),
                (PathFilter(['/sru']),
                    (SruParser(
                            host='sru.narcis.nl',
                            port=80,
                            defaultRecordSchema='knaw_short',
                            defaultRecordPacking='xml'),
                        (SruLimitStartRecord(limitBeyond=4000),
                            (SruHandler(
                                    includeQueryTimes=False,
                                    extraXParameters=[],
                                    enableCollectLog=False),
                                (SruTermDrilldown(),),
                                executeQueryHelix,
                                (StorageAdapter(),
                                    (storage,)
                                )
                            )
                        )
                    )
                ),
                (PathFilter('/rss'),
                    (Rss(   supportedLanguages = ['nl','en'], # defaults to first, if requested language is not available or supplied.
                            title = {'nl':'NARCIS', 'en':'NARCIS'},
                            description = {'nl':'NARCIS: De toegang tot de Nederlandse wetenschapsinformatie', 'en':'NARCIS: The gateway to Dutch scientific information'},
                            link = {'nl':'http://www.narcis.nl/?Language=nl', 'en':'http://www.narcis.nl/?Language=en'},
                            maximumRecords = 20),
                        executeQueryHelix,
                        (RssItem(
                                nsMap=NAMESPACEMAP,
                                title = ('knaw_short', {'nl':'//short:metadata/short:titleInfo[not (@xml:lang)]/short:title/text()', 'en':'//short:metadata/short:titleInfo[@xml:lang="en"]/short:title/text()'}),
                                description = ('knaw_short', {'nl':'//short:abstract[not (@xml:lang)]/text()', 'en':'//short:abstract[@xml:lang="en"]/text()'}),
                                pubdate = ('knaw_short', '//short:dateIssued/short:parsed/text()'),
                                linkTemplate = 'http://www.narcis.nl/%(wcpcollection)s/RecordID/%(oai_identifier)s/Language/%(language)s',
                                wcpcollection = ('meta', '//*[local-name() = "collection"]/text()'),
                                oai_identifier = ('meta', '//meta:record/meta:id/text()'),
                                language = ('Dummy: Language is auto provided by the calling RSS component, but needs to be present to serve the linkTemplate.')
                            ),
                            (StorageAdapter(),
                                (storage,)
                            )
                        )
                    )
                )
            )
        )
    )
예제 #11
0
def main(reactor, port, statePath, lucenePort, **ignored):


######## START Lucene Integration ###############################################################
    defaultLuceneSettings = LuceneSettings(
        commitTimeout=30,
        readonly=True,)
    
    
    http11Request = be(
        (HttpRequest1_1(),
            (SocketPool(reactor=reactor, unusedTimeout=5, limits=dict(totalSize=100, destinationSize=10)),),
        )
    )
    
    luceneIndex = luceneAndReaderConfig(defaultLuceneSettings.clone(readonly=True), http11Request, lucenePort)
    
    
    luceneRoHelix = be(
        (AdapterToLuceneQuery(
                defaultCore=DEFAULT_CORE,
                coreConverters={
                    DEFAULT_CORE: QueryExpressionToLuceneQueryDict(UNQUALIFIED_TERM_FIELDS, luceneSettings=luceneIndex.settings),
                }
            ),
            (MultiLucene(host='127.0.0.1', port=lucenePort, defaultCore=DEFAULT_CORE),
                (luceneIndex,),
                (http11Request,),
            )
        )
    )

######## END Lucene Integration ###############################################################


    fieldnameRewrites = {}

    def fieldnameRewrite(name):
        return fieldnameRewrites.get(name, name)

    def drilldownFieldnamesTranslate(fieldname):
        untokenizedName = untokenizedFieldname(fieldname)
        if untokenizedName in untokenizedFieldnames:
            fieldname = untokenizedName
        return fieldnameRewrite(fieldname)

    convertToComposedQuery = ConvertToComposedQuery(
            resultsFrom=DEFAULT_CORE,
            matches=[],
            drilldownFieldnamesTranslate=drilldownFieldnamesTranslate
        )


    strategie = Md5HashDistributeStrategy()
    storage = StorageComponent(join(statePath, 'store'), strategy=strategie, partsRemovedOnDelete=[HEADER_PARTNAME, META_PARTNAME, METADATA_PARTNAME, OAI_DC_PARTNAME, LONG_PARTNAME, SHORT_PARTNAME])


    # Wat doet dit?
    cqlClauseConverters = [
        RenameFieldForExact(
            untokenizedFields=untokenizedFieldnames,
            untokenizedPrefix=UNTOKENIZED_PREFIX,
        ).filterAndModifier(),
        SearchTermFilterAndModifier(
            shouldModifyFieldValue=lambda *args: True,
            fieldnameModifier=fieldnameRewrite
        ).filterAndModifier(),
    ]



    executeQueryHelix = \
        (FilterMessages(allowed=['executeQuery']),
            (CqlMultiSearchClauseConversion(cqlClauseConverters, fromKwarg='query'),
                (DrilldownQueries(),
                    (convertToComposedQuery,
                        (luceneRoHelix,),
                    )
                )
            )
        )


    return \
    (Observable(),
        
        (ObservableHttpServer(reactor, port, compressResponse=True),
            (BasicHttpHandler(),
                (PathFilter(['/sru']),
                    (SruParser(
                            host='sru.narcis.nl',
                            port=80,
                            defaultRecordSchema='knaw_short',
                            defaultRecordPacking='xml'),
                        (SruLimitStartRecord(limitBeyond=4000),
                            (SruHandler(
                                    includeQueryTimes=False,
                                    extraXParameters=[],
                                    enableCollectLog=False), #2017-03-24T12:00:33Z 127.0.0.1 3.5K 0.019s - /sru OF (TRUE): 2017-03-24T11:58:53Z 127.0.0.1 2.3K 0.004s 1hits /sru maximumRecords=10&operation=searchRetrieve&query=untokenized.dd_year+exact+%221993%22&recordPacking=xml&recordSchema=knaw_short&startRecord=1&version=1.2
                                (SruTermDrilldown(),),
                                executeQueryHelix,
                                (StorageAdapter(),
                                    (storage,)
                                )
                            )
                        )
                    )
                ),
                (PathFilter('/rss'),
                    (Rss(   supportedLanguages = ['nl','en'], # defaults to first, if requested language is not available or supplied.
                            title = {'nl':'NARCIS', 'en':'NARCIS'},
                            description = {'nl':'NARCIS: De toegang tot de Nederlandse wetenschapsinformatie', 'en':'NARCIS: The gateway to Dutch scientific information'},
                            link = {'nl':'http://www.narcis.nl/?Language=nl', 'en':'http://www.narcis.nl/?Language=en'},
                            maximumRecords = 20),
                        executeQueryHelix,
                        (RssItem(
                                nsMap=NAMESPACEMAP,                                            
                                title = ('knaw_short', {'nl':'//short:metadata/short:titleInfo[not (@xml:lang)]/short:title/text()', 'en':'//short:metadata/short:titleInfo[@xml:lang="en"]/short:title/text()'}),
                                description = ('knaw_short', {'nl':'//short:abstract[not (@xml:lang)]/text()', 'en':'//short:abstract[@xml:lang="en"]/text()'}),
                                pubdate = ('knaw_short', '//short:dateIssued/short:parsed/text()'),
                                linkTemplate = 'http://www.narcis.nl/%(wcpcollection)s/RecordID/%(oai_identifier)s/Language/%(language)s',                                
                                wcpcollection = ('meta', '//*[local-name() = "collection"]/text()'),
                                oai_identifier = ('meta', '//meta:record/meta:id/text()'),
                                language = ('Dummy: Language is auto provided by the calling RSS component, but needs to be present to serve the linkTemplate.')
                            ),
                            (StorageAdapter(),
                                (storage,)
                            )
                        )
                    )
                )
            )
        )
    )