Ejemplo n.º 1
0
    def testIdentifyWithDescription(self):
        self.oaipmh.addObserver(OaiBranding('http://meresco.org/files/images/meresco-logo-small.png', 'http://www.meresco.org/', 'Meresco'))
        header, body = self._request(verb=['Identify'])

        self.assertEqual(0, len(xpath(body, '/oai:OAI-PMH/oai:error')))
        descriptions = xpath(body, '/oai:OAI-PMH/oai:Identify/oai:description')
        if self.prefix:
            self.assertEqual(3, len(descriptions))
            self.assertEqual(['%s5324' % self.prefix], xpath(descriptions[0], 'identifier:oai-identifier/identifier:sampleIdentifier/text()'))
        else:
            self.assertEqual(2, len(descriptions))
        self.assertEqual(['Meresco'], xpath(descriptions[-2], 'toolkit:toolkit/toolkit:title/text()'))
        self.assertEqual(['Meresco'], xpath(descriptions[-1], 'branding:branding/branding:collectionIcon/branding:title/text()'))
Ejemplo n.º 2
0
 def testBrandingWithoutLinkAndTitle(self):
     branding = OaiBranding(url="http://example.org/icon.png")
     description = ''.join(compose(branding.description()))
     self.assertEqualsWS(BRANDING_RESULT_URL, description)
Ejemplo n.º 3
0
 def testOne(self):
     branding = OaiBranding(url="http://example.org/icon.png", link="http://www.example.org", title="Example")
     description = ''.join(compose(branding.description()))
     self.assertEqualsWS(BRANDING_RESULT, description)
Ejemplo n.º 4
0
 def testBrandingWithoutLinkAndTitle(self):
     branding = OaiBranding(url="http://example.org/icon.png")
     description = ''.join(compose(branding.description()))
     self.assertEqualsWS(BRANDING_RESULT_URL, description)
Ejemplo n.º 5
0
 def testOne(self):
     branding = OaiBranding(url="http://example.org/icon.png",
                            link="http://www.example.org",
                            title="Example")
     description = ''.join(compose(branding.description()))
     self.assertEqualsWS(BRANDING_RESULT, description)
Ejemplo n.º 6
0
def main(reactor, port, statePath, gatewayPort, quickCommit=False, **ignored):

    strategie = Md5HashDistributeStrategy()
    storage = StorageComponent(join(statePath, 'store'),
                               strategy=strategie,
                               partsRemovedOnDelete=[
                                   NL_DIDL_NORMALISED_PREFIX,
                                   NL_DIDL_COMBINED_PREFIX, 'metadata'
                               ])

    oaiJazz = OaiJazz(join(statePath, 'oai'))
    oaiJazz.updateMetadataFormat("metadata", "http://didl.loc.nl/didl.xsd",
                                 NAMESPACEMAP.didl)
    oaiJazz.updateMetadataFormat(NL_DIDL_COMBINED_PREFIX, "",
                                 NAMESPACEMAP.gmhcombined)
    oaiJazz.updateMetadataFormat(NL_DIDL_NORMALISED_PREFIX, "",
                                 NAMESPACEMAP.gmhnorm)

    normLogger = Logger(join(statePath, '..', 'gateway', 'normlogger'))

    periodicGateWayDownload = PeriodicDownload(
        reactor,
        host='localhost',
        port=gatewayPort,
        schedule=Schedule(
            period=.1 if quickCommit else 10
        ),  # WST: Interval in seconds before sending a new request to the GATEWAY in case of an error while processing batch records.(default=1). IntegrationTests need <=1 second! Otherwise tests will fail!
        name='api',
        autoStart=True)

    oaiDownload = OaiDownloadProcessor(path='/oaix',
                                       metadataPrefix=NORMALISED_DOC_NAME,
                                       workingDirectory=join(
                                           statePath, 'harvesterstate',
                                           'gateway'),
                                       userAgentAddition='ApiServer',
                                       xWait=True,
                                       name='api',
                                       autoCommit=False)


    return \
    (Observable(),
        createDownloadHelix(reactor, periodicGateWayDownload, oaiDownload, storage, oaiJazz),
        (ObservableHttpServer(reactor, port, compressResponse=True),
            (BasicHttpHandler(),
                (PathFilter('/oai'),
                    (OaiPmh(
                            repositoryName="Gemeenschappelijke Metadata Harvester DANS-KB",
                            adminEmail="*****@*****.**",
                            externalUrl="http://oai.gharvester.dans.knaw.nl",
                            batchSize=200,
                            supportXWait=False,
                            # preciseDatestamp=False,
                            # deleteInSets=False
                        ),
                        (oaiJazz, ),
                        (RetrieveToGetDataAdapter(),
                            (storage,),
                        ),
                        (OaiBranding(
                            url="https://www.narcis.nl/images/logos/logo-knaw-house.gif", #TODO: Link to a joint-GMH icon...
                            link="https://harvester.dans.knaw.nl",
                            title="Gemeenschappelijke Metadata Harvester (GMH) van DANS en de KB"),
                        ),
                        (OaiProvenance(
                            nsMap=NAMESPACEMAP,
                            baseURL=('meta', '//meta:repository/meta:baseurl/text()'),
                            harvestDate=('meta', '//meta:harvestdate/text()'),
                            metadataNamespace=('meta', '//meta:metadataPrefix/text()'), #TODO: Kan hardcoded in harvester mapper gezet eventueel: <metadataNamespace>urn:mpeg:mpeg21:2002:01-DII-NS</metadataNamespace>?? (storage,) #metadataNamespace=('meta', '//meta:record/meta:metadataNamespace/text()'),
                            identifier=('header','//oai:identifier/text()'),
                            datestamp=('header', '//oai:datestamp/text()')
                            ),
                            (RetrieveToGetDataAdapter(),
                                (storage,),
                            )
                        )
                    )
                ),
                (PathFilter('/rss'),
                    (LoggerRSS( title = 'GMH DANS-KB Normalisationlog Syndication', description = 'Harvester normalisation log for: ', link = 'http://rss.gharvester.dans.knaw.nl/rss', maximumRecords = 30),
                        (normLogger,
                            (storage,)
                        )
                    )
                ),
                (PathFilter('/xls'),
                    # (LogComponent("XLS-Request:"),),
                    (XlsServer(),)
                )
            )
        )
    )
Ejemplo n.º 7
0
def main(reactor,
         port,
         statePath,
         lucenePort,
         gatewayPort,
         quickCommit=False,
         **ignored):

    ######## START Lucene Integration ###############################################################
    defaultLuceneSettings = LuceneSettings(
        commitTimeout=30,
        readonly=True,
    )

    http11Request = be((
        HttpRequest1_1(),
        (SocketPool(reactor=reactor,
                    unusedTimeout=5,
                    limits=dict(totalSize=100, destinationSize=10)), ),
    ))

    luceneIndex = luceneAndReaderConfig(
        defaultLuceneSettings.clone(readonly=True), http11Request, lucenePort)

    luceneRoHelix = be(
        (AdapterToLuceneQuery(defaultCore=DEFAULT_CORE,
                              coreConverters={
                                  DEFAULT_CORE:
                                  QueryExpressionToLuceneQueryDict(
                                      UNQUALIFIED_TERM_FIELDS,
                                      luceneSettings=luceneIndex.settings),
                              }), (
                                  MultiLucene(host='localhost',
                                              port=lucenePort,
                                              defaultCore=DEFAULT_CORE),
                                  (luceneIndex, ),
                                  (http11Request, ),
                              )))

    ######## END Lucene Integration ###############################################################

    fieldnameRewrites = {
        #         UNTOKENIZED_PREFIX+'genre': UNTOKENIZED_PREFIX+'dc:genre',
    }

    def fieldnameRewrite(name):
        return fieldnameRewrites.get(name, name)

    def drilldownFieldnamesTranslate(fieldname):
        untokenizedName = untokenizedFieldname(fieldname)
        if untokenizedName in untokenizedFieldnames:
            fieldname = untokenizedName
        return fieldnameRewrite(fieldname)

    convertToComposedQuery = ConvertToComposedQuery(
        resultsFrom=DEFAULT_CORE,
        matches=[],
        drilldownFieldnamesTranslate=drilldownFieldnamesTranslate)

    strategie = Md5HashDistributeStrategy()
    storage = StorageComponent(join(statePath, 'store'),
                               strategy=strategie,
                               partsRemovedOnDelete=[
                                   HEADER_PARTNAME, META_PARTNAME,
                                   METADATA_PARTNAME, OAI_DC_PARTNAME,
                                   LONG_PARTNAME, SHORT_PARTNAME,
                                   OPENAIRE_PARTNAME
                               ])

    oaiJazz = OaiJazz(join(statePath, 'oai'))
    oaiJazz.updateMetadataFormat(
        OAI_DC_PARTNAME, "http://www.openarchives.org/OAI/2.0/oai_dc.xsd",
        "http://purl.org/dc/elements/1.1/")

    oai_oa_cerifJazz = OaiJazz(join(statePath, 'oai_cerif'))
    oai_oa_cerifJazz.updateMetadataFormat(
        OPENAIRE_PARTNAME,
        "https://www.openaire.eu/schema/cris/current/openaire-cerif-profile.xsd",
        "https://www.openaire.eu/cerif-profile/1.1/")
    # All of the following OAI-PMH sets shall be recognized by the CRIS, even if not all of them are populated.
    oai_oa_cerifJazz.updateSet("openaire_cris_projects",
                               "OpenAIRE_CRIS_projects")
    oai_oa_cerifJazz.updateSet("openaire_cris_orgunits",
                               "OpenAIRE_CRIS_orgunits")
    oai_oa_cerifJazz.updateSet("openaire_cris_persons",
                               "OpenAIRE_CRIS_persons")
    oai_oa_cerifJazz.updateSet("openaire_cris_patents",
                               "OpenAIRE_CRIS_patents")
    oai_oa_cerifJazz.updateSet("openaire_cris_products",
                               "OpenAIRE_CRIS_products")
    oai_oa_cerifJazz.updateSet("openaire_cris_publications",
                               "OpenAIRE_CRIS_publications")

    oai_oa_cerifJazz.updateSet("openaire_cris_funding",
                               "OpenAIRE_CRIS_funding")
    oai_oa_cerifJazz.updateSet("openaire_cris_events", "OpenAIRE_CRIS_events")
    oai_oa_cerifJazz.updateSet("openaire_cris_equipments",
                               "OpenAIRE_CRIS_equipments")

    cqlClauseConverters = [
        RenameFieldForExact(
            untokenizedFields=untokenizedFieldnames,
            untokenizedPrefix=UNTOKENIZED_PREFIX,
        ).filterAndModifier(),
        SearchTermFilterAndModifier(
            shouldModifyFieldValue=lambda *args: True,
            fieldnameModifier=fieldnameRewrite).filterAndModifier(),
    ]

    periodicGateWayDownload = PeriodicDownload(
        reactor,
        host='localhost',
        port=gatewayPort,
        schedule=Schedule(
            period=1 if quickCommit else 10
        ),  # WST: Interval in seconds before sending a new request to the GATEWAY in case of an error while processing batch records.(default=1). IntegrationTests need 1 second! Otherwise tests will fail!
        name='api',
        autoStart=True)

    oaiDownload = OaiDownloadProcessor(path='/oaix',
                                       metadataPrefix=NORMALISED_DOC_NAME,
                                       workingDirectory=join(
                                           statePath, 'harvesterstate',
                                           'gateway'),
                                       userAgentAddition='ApiServer',
                                       xWait=True,
                                       name='api',
                                       autoCommit=False)

    executeQueryHelix = \
        (FilterMessages(allowed=['executeQuery']),
            (CqlMultiSearchClauseConversion(cqlClauseConverters, fromKwarg='query'),
                (DrilldownQueries(),
                    (convertToComposedQuery,
                        (luceneRoHelix,),
                    )
                )
            )
        )

    return \
    (Observable(),
        createDownloadHelix(reactor, periodicGateWayDownload, oaiDownload, storage, oaiJazz, oai_oa_cerifJazz),
        (ObservableHttpServer(reactor, port, compressResponse=True),
            (BasicHttpHandler(),
                (PathFilter(["/oai"]),
                    (OaiPmh(repositoryName="NARCIS OAI-pmh", adminEmail="*****@*****.**", externalUrl="http://oai.narcis.nl"),
                        (oaiJazz,),
                        (StorageAdapter(),
                            (storage,)
                        ),
                        (OaiBranding(
                            url="http://www.narcis.nl/images/logos/logo-knaw-house.gif",
                            link="http://oai.narcis.nl",
                            title="Narcis - The gateway to scholarly information in The Netherlands"),
                        ),
                        (OaiProvenance(
                            nsMap=NAMESPACEMAP,
                            baseURL=('meta', '//meta:repository/meta:baseurl/text()'),
                            harvestDate=('meta', '//meta:record/meta:harvestdate/text()'),
                            metadataNamespace=('meta', '//meta:record/meta:metadataNamespace/text()'),
                            identifier=('header','//oai:identifier/text()'),
                            datestamp=('header', '//oai:datestamp/text()')
                            ),
                            (storage,)
                        )
                    )
                ),
                (PathFilter(["/cerif"]),
                    (OaiPmhDans(repositoryName="OpenAIRE CERIF", adminEmail="*****@*****.**", repositoryIdentifier="services.nod.dans.knaw.nl", externalUrl="http://services.nod.dans.knaw.nl"), #TODO: pathFilter should resemble proxy path
                        (oai_oa_cerifJazz,),
                        (StorageAdapter(),
                            (storage,)
                        ),
                        (OaiOpenAIREDescription(
                            serviceid='organisation:ORG1242054',
                            acronym='services.nod.dans.knaw.nl',
                            name='NARCIS',
                            description='Compliant with the OpenAIRE Guidelines for CRIS Managers v.1.1.',
                            website='https://www.narcis.nl',
                            baseurl='http://services.nod.dans.knaw.nl/oa-cerif',
                            subjectheading='',
                            orgunitid='organisation:ORG1242054',
                            owneracronym='DANS'),
                        ),
                        # (OaiBranding(
                        #     url="http://www.narcis.nl/images/logos/logo-knaw-house.gif",
                        #     link="http://oai.narcis.nl",
                        #     title="Narcis - The gateway to scholarly information in The Netherlands"),
                        # ),
                        (OaiProvenance(
                            nsMap=NAMESPACEMAP,
                            baseURL=('meta', '//meta:repository/meta:baseurl/text()'),
                            harvestDate=('meta', '//meta:record/meta:harvestdate/text()'),
                            metadataNamespace=('meta', '//meta:record/meta:metadataNamespace/text()'),
                            identifier=('header','//oai:identifier/text()'),
                            datestamp=('header', '//oai:datestamp/text()')
                            ),
                            (storage,)
                        )
                    )
                ),
                (PathFilter(['/sru']),
                    (SruParser(
                            host='sru.narcis.nl',
                            port=80,
                            defaultRecordSchema='knaw_short',
                            defaultRecordPacking='xml'),
                        (SruLimitStartRecord(limitBeyond=4000),
                            (SruHandler(
                                    includeQueryTimes=False,
                                    extraXParameters=[],
                                    enableCollectLog=False),
                                (SruTermDrilldown(),),
                                executeQueryHelix,
                                (StorageAdapter(),
                                    (storage,)
                                )
                            )
                        )
                    )
                ),
                (PathFilter('/rss'),
                    (Rss(   supportedLanguages = ['nl','en'], # defaults to first, if requested language is not available or supplied.
                            title = {'nl':'NARCIS', 'en':'NARCIS'},
                            description = {'nl':'NARCIS: De toegang tot de Nederlandse wetenschapsinformatie', 'en':'NARCIS: The gateway to Dutch scientific information'},
                            link = {'nl':'http://www.narcis.nl/?Language=nl', 'en':'http://www.narcis.nl/?Language=en'},
                            maximumRecords = 20),
                        executeQueryHelix,
                        (RssItem(
                                nsMap=NAMESPACEMAP,
                                title = ('knaw_short', {'nl':'//short:metadata/short:titleInfo[not (@xml:lang)]/short:title/text()', 'en':'//short:metadata/short:titleInfo[@xml:lang="en"]/short:title/text()'}),
                                description = ('knaw_short', {'nl':'//short:abstract[not (@xml:lang)]/text()', 'en':'//short:abstract[@xml:lang="en"]/text()'}),
                                pubdate = ('knaw_short', '//short:dateIssued/short:parsed/text()'),
                                linkTemplate = 'http://www.narcis.nl/%(wcpcollection)s/RecordID/%(oai_identifier)s/Language/%(language)s',
                                wcpcollection = ('meta', '//*[local-name() = "collection"]/text()'),
                                oai_identifier = ('meta', '//meta:record/meta:id/text()'),
                                language = ('Dummy: Language is auto provided by the calling RSS component, but needs to be present to serve the linkTemplate.')
                            ),
                            (StorageAdapter(),
                                (storage,)
                            )
                        )
                    )
                )
            )
        )
    )