def _processRecords(self, lxmlNode):
     verbNode = xpathFirst(lxmlNode, "/oai:OAI-PMH/oai:%s" % self._verb)
     for item in verbNode.iterchildren(tag=VERB_TAGNAME[self._verb]):
         header = None
         for h in item.iterchildren():
             if h.tag == HEADER_TAG:
                 header = h
                 break
         else:
             if item.tag != HEADER_TAG:
                 raise IndexError("Invalid oai header")
             header = item
         for child in header.iterchildren():
             if child.tag == IDENTIFIER_TAG:
                 identifier = child.text
             elif child.tag == DATESTAMP_TAG:
                 datestamp = child.text
         try:
             yield self._add(identifier=identifier,
                             lxmlNode=ElementTree(item),
                             datestamp=datestamp)
         except Exception as e:
             self._logError(format_exc())
             self._logError("While processing:")
             self._logError(lxmltostring(item))
             self._errorState = "ERROR while processing '%s': %s" % (
                 identifier, str(e))
             raise
         yield  # some room for others
     self._resumptionToken = xpathFirst(verbNode,
                                        "oai:resumptionToken/text()")
예제 #2
0
 def testSruQueryWithUntokenized(self):
     response = self.doSruQuery(**{"query": 'untokenized.humanstartpage exact "http://meresco.com?record=1"', "recordSchema": "knaw_long"})        
     # print "humanStartPage:", etree.tostring(response)
     self.assertEqual('meresco:record:1', xpathFirst(response, '//srw:recordIdentifier/text()'))
     response = self.doSruQuery(**{"query": 'untokenized.dd_year exact "1993"'})
     #print "dd_year:", etree.tostring(response)
     self.assertEqual('1', xpathFirst(response, '//srw:numberOfRecords/text()'))
예제 #3
0
 def __init__(self, request, response):
     self.request = request
     self.response = response
     self.items = []
     verbNode = xpathFirst(self.response,
                           "/oai:OAI-PMH/oai:%s" % self.request.verb)
     if verbNode is None:
         errorNode = xpathFirst(self.response, "/oai:OAI-PMH/oai:error")
         if errorNode is None:
             raise ValueError(
                 'Not a OAI-PMH %s response from %s. Got:\n%s' %
                 (self.request.verb, self.request.buildUrl(),
                  tostring(response, pretty_print=True)))
         errorCode = xpathFirst(errorNode, '@code')
         if errorCode in ['noRecordsMatch']:
             return
         msg = xpathFirst(errorNode, 'text()')
         raise ValueError('Got OAI-PMH response with error (%s): %s' %
                          (errorCode, msg))
     itemXPath, headerXPath = VERB_XPATHS[self.request.verb]
     for item in xpath(verbNode, itemXPath):
         record = item if self.request.verb == 'ListRecords' else None
         self.items.append(
             OaiItem(record,
                     header=xpathFirst(item, headerXPath),
                     oaiBatch=self))
예제 #4
0
 def testOaiIdentify(self): # GMH21 OK
     header, body = getRequest(self.apiPort, '/oai', dict(verb="Identify"))
     # print "OAI Identify:", etree.tostring(body)
     self.assertEqual('HTTP/1.0 200 OK\r\nContent-Type: text/xml; charset=utf-8', header)
     self.assertEqual('Gemeenschappelijke Metadata Harvester DANS-KB', xpathFirst(body, '//oai:Identify/oai:repositoryName/text()'))
     self.assertEqual('*****@*****.**', xpathFirst(body, '//oai:Identify/oai:adminEmail/text()'))
     self.assertEqual('Gemeenschappelijke Metadata Harvester (GMH) van DANS en de KB', testNamespaces.xpathFirst(body, '//oai:Identify/oai:description/oaibrand:branding/oaibrand:collectionIcon/oaibrand:title/text()'))
예제 #5
0
 def handleRequest(self, Body='', **kwargs):
     yield '\r\n'.join(['HTTP/1.0 200 OK', 'Content-Type: text/xml; charset=utf-8\r\n', ''])
     try:
         updateRequest = xpathFirst(XML(Body), '/ucp:updateRequest')
         recordId = xpathFirst(updateRequest, 'ucp:recordIdentifier/text()')
         normalizedRecordId = notWordCharRE.sub('_', recordId)
         self._number +=1
         if self._number <= self._maxCountNumber:
             filename = '%05d_%s.updateRequest' %(self._number, normalizedRecordId)
             with open(join(self._dumpdir, filename), 'w') as f:
                 print recordId
                 stdout.flush()
                 f.write(tostring(updateRequest))
             answer = RESPONSE_XML % {
                 "operationStatus": "success",
                 "diagnostics": ""}
         else:
             self._maxCountNumber = self._number + self._maxCount
             print 'Reached maxCount'
             answer = RESPONSE_XML % {
                 "operationStatus": "fail",
                 "diagnostics": DIAGNOSTIC_XML % {'uri': '', 'message': '', 'details': escapeXml("Enough is enough")}}
     except Exception, e:
         answer = RESPONSE_XML % {
             "operationStatus": "fail",
             "diagnostics": DIAGNOSTIC_XML % {'uri': '', 'message': '', 'details': escapeXml(format_exc())}}
예제 #6
0
 def testRSS(self):
     header, body = getRequest(self.sruslavePort, '/rss', dict(query="*", querylabel='MyWorkerLabel', sortKeys='untokenized.dateissued,,1'))
     # print "RSS body:", etree.tostring(body)
     items = xpath(body, "/rss/channel/item")
     self.assertEquals(13, len(items))
     self.assertTrue(xpathFirst(body, '//item/link/text()').endswith('Language/nl'))
     self.assertEqual(['1993-01-01', '2004-06-30', '2009-11-24', '2013', '2014', '2016', '2016-01-31', '2016-05-05', '2019-11-06'], xpath(body, "//item/pubDate/text()"))
     self.assertEqual('MyWorkerLabel', xpathFirst(body, '//channel/title/text()'))
예제 #7
0
 def testRSS(self):
     header, body = getRequest(self.apiPort, '/rss', dict(query="*", querylabel='MyLabel', sortKeys='untokenized.dateissued,,0', startRecord='4'))
     # print "RSS body:", etree.tostring(body)
     items = xpath(body, "/rss/channel/item")
     self.assertEquals(10, len(items))
     self.assertTrue(xpathFirst(body, '//item/link/text()').endswith('Language/nl'))
     self.assertEqual({'Paden en stromingen---a historical survey', 'Preface to special issue (Fast reaction - slow diffusion scenarios: PDE approximations and free boundaries)', 'Conditiebepaling PVC', 'Appositie en de interne struktuur van de NP', 'Wetenschapswinkel', 'Late-type Giants in the Inner Galaxy', 'H.J. Bennis', 'Locatie [Matthijs Tinxgracht 16] te Edam, gemeente Edam-Volendam. Een archeologische opgraving.', 'Example Program 2', u'\u042d\u043a\u043e\u043b\u043e\u0433\u043e-\u0440\u0435\u043a\u0440\u0435\u0430\u0446\u0438\u043e\u043d\u043d\u044b\u0439 \u043a\u043e\u0440\u0438\u0434\u043e\u0440 \u0432 \u0433\u043e\u0440\u043d\u043e\u043c \u0437\u0430\u043f\u043e\u0432\u0435\u0434\u043d\u0438\u043a\u0435 \u0411\u043e\u0433\u043e\u0442\u044b'}, set(xpath(body, "//item/title/text()")))
     self.assertEqual({'FransHeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeellllllang', 'Microvariatie; (Generatieve) Syntaxis; Morphosyntaxis; Syntaxis-Semantiek Interface; Dialectologie', 'Samenvatting', 'Projectomschrijving<br>Ontwikkeling van betrouwbare methoden, procedures\n            en extrapolatiemodellen om de conditie en restlevensduur van in gebruik zijnde\n            PVC-leidingen te bepalen.<br>Beoogde projectopbrengsten<br>- uitwerking van\n            huidige kennis en inzichten m.b.t.', 'The present thesis describes the issue of\n            "neonatal glucocorticoid treatment and predisposition to\n            cardiovascular disease in rats".', 'Abstract van dit document', 'This is an example program about Programming with Meresco', 'Abstract'}, set(xpath(body, "//item/description/text()")))
     self.assertEqual('MyLabel', xpathFirst(body, '//channel/title/text()'))
예제 #8
0
 def __init__(self, record, header, oaiBatch=None):
     self.record = record
     self.header = header
     self.oaiBatch = oaiBatch
     self.identifier = xpathFirst(header, 'oai:identifier/text()')
     self.datestamp = xpathFirst(header, 'oai:datestamp/text()')
     self.deleted = (xpathFirst(header, '@status') == 'deleted')
     self.setSpecs = xpath(header, 'oai:setSpec/text()')
     self.metadata = None
     if not record is None:
         self.metadata = xpathFirst(record, 'oai:metadata/*')
예제 #9
0
 def __init__(self, record, header, oaiBatch=None):
     self.record = record
     self.header = header
     self.oaiBatch = oaiBatch
     self.identifier = xpathFirst(header, 'oai:identifier/text()')
     self.datestamp = xpathFirst(header, 'oai:datestamp/text()')
     self.deleted = (xpathFirst(header, '@status') == 'deleted')
     self.setSpecs = xpath(header, 'oai:setSpec/text()')
     self.metadata = None
     if not record is None:
         self.metadata = xpathFirst(record, 'oai:metadata/*')
예제 #10
0
 def testSruQueryWithUntokenized(self):
     response = self.doSruQuery(
         **{
             "query":
             'untokenized.humanstartpage exact "http://meresco.com?record=1"',
             "recordSchema": "knaw_long"
         })
     # print "humanStartPage:", etree.tostring(response)
     self.assertEqual('meresco:record:1',
                      xpathFirst(response, '//srw:recordIdentifier/text()'))
     response = self.doSruQuery(
         **{"query": 'untokenized.dd_year exact "1993"'})
     #print "dd_year:", etree.tostring(response)
     self.assertEqual('1',
                      xpathFirst(response, '//srw:numberOfRecords/text()'))
 def _getXsdFilename(self, response):
     schemaLocation = xpathFirst(
         XML(response), '/drilldown:drilldown/@xsi:schemaLocation')
     namespace, xsd = schemaLocation.split()
     self.assertEqual(namespaces['drilldown'], namespace)
     self.assertEqual('http://meresco.org/files/xsd', dirname(xsd))
     return basename(xsd)
예제 #12
0
 def _processRecords(self, lxmlNode):
     verbNode = xpathFirst(lxmlNode, "/oai:OAI-PMH/oai:%s" % self._verb)
     for item in verbNode.iterchildren(tag=VERB_TAGNAME[self._verb]):
         header = None
         for h in item.iterchildren():
             if h.tag == HEADER_TAG:
                 header = h
                 break
         else:
             if item.tag != HEADER_TAG:
                 raise IndexError("Invalid oai header")
             header = item
         for child in header.iterchildren():
             if child.tag == IDENTIFIER_TAG:
                 identifier = child.text
             elif child.tag == DATESTAMP_TAG:
                 datestamp = child.text
         try:
             yield self._add(identifier=identifier, lxmlNode=ElementTree(item), datestamp=datestamp)
         except Exception, e:
             self._logError(format_exc())
             self._logError("While processing:")
             self._logError(lxmltostring(item))
             self._errorState = "ERROR while processing '%s': %s" % (identifier, str(e))
             raise
         yield # some room for others
예제 #13
0
    def testUpdateRecordWhileSendingData(self):
        batchSize = 3
        oaiJazz = OaiJazz(join(self.tempdir, 'oai'))
        oaiJazz.updateMetadataFormat(prefix="prefix", schema="", namespace="")
        storageComponent = MultiSequentialStorage(join(self.tempdir,
                                                       'storage'))
        self._addOaiRecords(storageComponent, oaiJazz, count=batchSize + 10)
        dna = be((Observable(), (
            OaiPmh(repositoryName='test',
                   adminEmail='*****@*****.**',
                   batchSize=batchSize),
            (storageComponent, ),
            (oaiJazz, ),
        )))
        kwargs = dict(
            Method='GET',
            Headers={'Host': 'myserver'},
            port=1234,
            path='/oaipmh.pl',
            arguments=dict(verb=['ListIdentifiers'],
                           metadataPrefix=['prefix']),
        )
        stream = compose(dna.all.handleRequest(**kwargs))
        buf = StringIO()
        for stuff in stream:
            buf.write(stuff)
            if 'identifier>id0<' in stuff:
                oaiJazz.addOaiRecord(identifier="id1",
                                     metadataPrefixes=["prefix"])

        result = XML(buf.getvalue().split(CRLF * 2)[-1].encode())
        resumptionToken = xpathFirst(
            result,
            '/oai:OAI-PMH/oai:ListIdentifiers/oai:resumptionToken/text()')
        self.assertFalse(resumptionToken is None)
예제 #14
0
    def testUpdateRecordWhileSendingData(self):
        batchSize = 3
        oaiJazz = OaiJazz(join(self.tempdir, 'oai'))
        storageComponent = MultiSequentialStorage(join(self.tempdir, 'storage'))
        self._addOaiRecords(storageComponent, oaiJazz, count=batchSize + 10)
        dna = be((Observable(),
            (OaiPmh(repositoryName='test', adminEmail='*****@*****.**', batchSize=batchSize),
                (storageComponent,),
                (oaiJazz,),
            )
        ))
        kwargs = dict(
            Method='GET',
            Headers={'Host': 'myserver'},
            port=1234,
            path='/oaipmh.pl',
            arguments=dict(verb=['ListIdentifiers'], metadataPrefix=['prefix']),
            )
        stream = compose(dna.all.handleRequest(**kwargs))
        buf = StringIO()
        for stuff in stream:
            buf.write(stuff)
            if 'identifier>id0<' in stuff:
                 oaiJazz.addOaiRecord(identifier="id1", sets=[], metadataFormats=[("prefix", "", "")])

        result = XML(buf.getvalue().split(CRLF*2)[-1])
        resumptionToken = xpathFirst(result, '/oai:OAI-PMH/oai:ListIdentifiers/oai:resumptionToken/text()')
        self.assertFalse(resumptionToken is None)
예제 #15
0
 def testSruQuery(self):
     response = self.doSruQuery(query='*',
                                recordSchema='knaw_short',
                                maximumRecords='20')
     # print "doSruQuery(query='*'):", etree.tostring(response)
     self.assertEqual('15',
                      xpathFirst(response, '//srw:numberOfRecords/text()'))
     self.assertEqual(
         set([
             'Example Program 1', 'Example Program 2',
             'RAIN: Pan-European gridded data sets of extreme weather probability of occurrence under present and future climate',
             'Appositie en de interne struktuur van de NP',
             'Paden en stromingen---a historical survey',
             'Late-type Giants in the Inner Galaxy',
             'Preface to special issue (Fast reaction - slow diffusion scenarios: PDE approximations and free boundaries)',
             'Conditiebepaling PVC', 'Wetenschapswinkel',
             "The Language Designer's Workbench: Automating Verification of Language Definitions",
             'Bennis, Prof.dr. H.J. (Hans)',
             'Havens van het IJsselmeergebied',
             'System of wireless base stations employing shadow prices for power load balancing',
             'Locatie [Matthijs Tinxgracht 16] te Edam, gemeente Edam-Volendam. Een archeologische opgraving.',
             u'\u042d\u043a\u043e\u043b\u043e\u0433\u043e-\u0440\u0435\u043a\u0440\u0435\u0430\u0446\u0438\u043e\u043d\u043d\u044b\u0439 \u043a\u043e\u0440\u0438\u0434\u043e\u0440 \u0432 \u0433\u043e\u0440\u043d\u043e\u043c \u0437\u0430\u043f\u043e\u0432\u0435\u0434\u043d\u0438\u043a\u0435 \u0411\u043e\u0433\u043e\u0442\u044b'
         ]),
         set(
             testNamespaces.xpath(
                 response,
                 '//short:metadata/short:titleInfo[1]/short:title/text()')))
예제 #16
0
 def testProjectToShort(self):
     response = self.doSruQuery(**{'query': 'OND1272024', 'recordSchema':'knaw_short'})
     self.assertEqual(1, int(str(xpathFirst(response, '//srw:numberOfRecords/text()'))))
     self.assertEqual('Conditiebepaling PVC', testNamespaces.xpathFirst(response, '//short:metadata/short:titleInfo/short:title/text()'))
     self.assertEqual('research', testNamespaces.xpathFirst(response, '//short:metadata/short:genre/text()'))
     self.assertEqual('Projectomschrijving<br>Ontwikkeling van betrouwbare methoden,', testNamespaces.xpathFirst(response, '//short:metadata/short:abstract/text()')[:61])
     self.assertEqual(2, len(testNamespaces.xpath(response, '//short:metadata/short:titleInfo/short:title')))
예제 #17
0
 def __init__(self, xmlNode=None, username=None):
     self.username = username
     for attribute in self.ATTRIBUTES:
         self.setValueFor(attribute,  '')
     if xmlNode is not None:
         self.username = xmlNode.tag.split('}')[-1]
         for attribute in self.ATTRIBUTES:
             self.setValueFor(attribute, xpathFirst(xmlNode, '{}/text()'.format(attribute)) or '')
예제 #18
0
 def testRSS(self):
     header, body = getRequest(
         self.sruslavePort, '/rss',
         dict(query="*",
              querylabel='MyWorkerLabel',
              sortKeys='untokenized.dateissued,,1'))
     # print "RSS body:", etree.tostring(body)
     items = xpath(body, "/rss/channel/item")
     self.assertEquals(15, len(items))
     self.assertTrue(
         xpathFirst(body, '//item/link/text()').endswith('Language/nl'))
     self.assertEqual([
         '1993-01-01', '2004-06-30', '2009-11-24', '2011-05-10', '2013',
         '2014', '2016', '2016-01-31', '2016-05-05', '2019-11-06'
     ], xpath(body, "//item/pubDate/text()"))
     self.assertEqual('MyWorkerLabel',
                      xpathFirst(body, '//channel/title/text()'))
예제 #19
0
 def __init__(self, request, response):
     self.request = request
     self.response = response
     self.items = []
     verbNode = xpathFirst(self.response, "/oai:OAI-PMH/oai:%s" % self.request.verb)
     if verbNode is None:
         errorNode = xpathFirst(self.response, "/oai:OAI-PMH/oai:error")
         if errorNode is None:
             raise ValueError('Not a OAI-PMH %s response from %s. Got:\n%s' % (self.request.verb, self.request.buildUrl(), tostring(response, pretty_print=True)))
         errorCode = xpathFirst(errorNode, '@code')
         if errorCode in ['noRecordsMatch']:
             return
         msg = xpathFirst(errorNode, 'text()')
         raise ValueError('Got OAI-PMH response with error (%s): %s' % (errorCode, msg))
     itemXPath, headerXPath = VERB_XPATHS[self.request.verb]
     for item in xpath(verbNode, itemXPath):
         record = item if self.request.verb == 'ListRecords' else None
         self.items.append(OaiItem(record, header=xpathFirst(item, headerXPath), oaiBatch=self))
예제 #20
0
 def testSruLimitStartRecord(self):
     response = self.doSruQuery(**{
         'maximumRecords': '1',
         'startRecord': '4002',
         'query': '*'
     })
     self.assertEqual(
         "Argument 'startRecord' too high, maximum: 4000",
         xpathFirst(response, '//diag:diagnostic/diag:details/text()'))
예제 #21
0
 def testBadOai(self):
     header, data = getRequest(port=self.helperServerPortNumber,
                               path='/badoai/responsedate',
                               arguments=dict(verb='ListRecords',
                                              metadataPrefix='prefix'))
     self.assertEqual(
         'resume0',
         xpathFirst(
             data,
             '/oai:OAI-PMH/oai:ListRecords/oai:resumptionToken/text()'))
     header, data = getRequest(port=self.helperServerPortNumber,
                               path='/badoai/responsedate',
                               arguments=dict(verb='ListRecords',
                                              resumptionToken='resume0'))
     self.assertEqual(
         'resume1',
         xpathFirst(
             data,
             '/oai:OAI-PMH/oai:ListRecords/oai:resumptionToken/text()'))
예제 #22
0
    def testSruQueryWithDrilldown(self):
        # response = self.doSruQuery(**{'maximumRecords': '0', "query": '*', "x-term-drilldown": "dd_penv:6,dd_thesis:6,dd_fin:6,status:5"})
        response = self.doSruQuery(**{"query": '*', 'maximumRecords': '1', "x-term-drilldown": "dd_cat:0"})
        # print "DD body:", etree.tostring(response)
        self.assertEqual('13', xpathFirst(response, '//srw:numberOfRecords/text()'))
        # self.assertEqual(set(['Example Program 1', 'Example Program 2']), set(xpath(response, '//srw:recordData/oai_dc:dc/dc:title/text()')))

        ddItems = xpath(response, '//drilldown:term-drilldown/drilldown:navigator[@name="dd_cat"]/drilldown:item')
        drilldown = [(i.text, i.attrib['count']) for i in ddItems]
        self.assertEqual([('D37000', '2'), ('D30000', '2'), ('A50000', '1'), ('A80000', '1'), ('D40000', '1'), ('D50000', '1'), ('D60000', '1')], drilldown)
예제 #23
0
 def testPersonToShort(self):
     response = self.doSruQuery(**{'query': 'person:PRS1242583', 'recordSchema':'knaw_short'})
     self.assertEqual(1, int(str(xpathFirst(response, '//srw:numberOfRecords/text()'))))
     self.assertEqual('H.J. Bennis', testNamespaces.xpathFirst(response, '//short:metadata/short:titleInfo/short:title/text()'))
     self.assertEqual('person', testNamespaces.xpathFirst(response, '//short:metadata/short:genre/text()'))
     self.assertEqual('Microvariation; (Generative) Syntax; Morphosyntax;', testNamespaces.xpathFirst(response, '//short:metadata/short:abstract[@xml:lang="en"]/text()')[:50])
     self.assertEqual('personal', testNamespaces.xpathFirst(response, '//short:metadata/short:name/short:type/text()'))
     self.assertEqual('H.J. Bennis', testNamespaces.xpathFirst(response, '//short:metadata/short:name/short:name/text()'))
     self.assertEqual('071792279', testNamespaces.xpathFirst(response, '//short:metadata/short:name/short:nameIdentifier[@type="dai-nl"]/text()'))
     self.assertEqual('0000000081508690', testNamespaces.xpathFirst(response, '//short:metadata/short:name/short:nameIdentifier[@type="isni"]/text()'))
     self.assertEqual('0000-0002-4703-3788', testNamespaces.xpathFirst(response, '//short:metadata/short:name/short:nameIdentifier[@type="orcid"]/text()'))
     self.assertEqual('PRS1242583', testNamespaces.xpathFirst(response, '//short:metadata/short:name/short:nameIdentifier[@type="nod-prs"]/text()'))
     self.assertEqual(4, len(testNamespaces.xpath(response, '//short:metadata/short:name/short:nameIdentifier')))
예제 #24
0
 def assertSruQuery(self, numberOfRecords, query, printout=False):
     response = self.doSruQuery(
         **{
             'query': query,
             "recordSchema": "knaw_short",
             "x-recordSchema": "header"
         })  # , 'maximumRecords': '1'
     if printout:
         print "SruQuery response:", etree.tostring(response,
                                                    pretty_print=True,
                                                    encoding='utf-8')
     self.assertEquals(
         numberOfRecords,
         int(str(xpathFirst(response, '//srw:numberOfRecords/text()'))))
예제 #25
0
 def testSruQuery(self):
     response = self.doSruQuery(query='*', recordSchema='knaw_short')
     # print "doSruQuery(query='*'):", etree.tostring(response)
     self.assertEqual('13', xpathFirst(response, '//srw:numberOfRecords/text()'))
     self.assertEqual({
         'Example Program 1',
         'Example Program 2',
         'RAIN: Pan-European gridded data sets of extreme weather probability of occurrence under present and future climate',
         'Appositie en de interne struktuur van de NP',
         'Paden en stromingen---a historical survey',
         'Late-type Giants in the Inner Galaxy',
         'Preface to special issue (Fast reaction - slow diffusion scenarios: PDE approximations and free boundaries)',
         'Conditiebepaling PVC',
         'Wetenschapswinkel',
         'H.J. Bennis'}, set(testNamespaces.xpath(response, '//short:metadata/short:titleInfo[1]/short:title/text()')))
예제 #26
0
    def testAddSetInfoWithElementTree(self):
        oaiContainer = createElement("oai:PMH")
        record = createSubElement(oaiContainer, "oai:record")
        header = createSubElement(record, "oai:header")
        createSubElement(header, "oai:setSpec", text="1")

        consume(self.subject.add("123", "oai_dc", xpathFirst(oaiContainer, "/oai:PMH/oai:record")))

        self.assertEquals(["addOaiRecord"], self.observer.calledMethodNames())
        self.assertEquals("123", self.observer.calledMethods[0].kwargs["identifier"])
        self.assertEquals(set([("1", "1")]), self.observer.calledMethods[0].kwargs["sets"])
        self.assertEquals(
            [("oai_dc", "", "http://www.openarchives.org/OAI/2.0/")],
            self.observer.calledMethods[0].kwargs["metadataFormats"],
        )
예제 #27
0
 def testPublIdentifier(self):
     response = self.doSruQuery(
         **{
             'query': '1937-1632',
             'maximumRecords': '1',
             'recordSchema': 'knaw_long'
         })
     # print "DD body:", etree.tostring(response)
     #print body.searchRetrieveResponse.records.record.recordData.knaw_long.metadata.relatedItem.publication_identifier
     self.assertEqual(
         'Springer',
         testNamespaces.xpathFirst(
             response,
             '//long:metadata/long:relatedItem[@type="host"]/long:publisher/text()'
         ))
     self.assertEqual(
         1, int(str(xpathFirst(response, '//srw:numberOfRecords/text()'))))
    def testDefaultFormat(self):
        self.assertRaises(ValueError, lambda: SruTermDrilldown(defaultFormat='text'))
        sruTermDrilldown = SruTermDrilldown(defaultFormat='json')
        drilldownData = [
                {
                    'fieldname': 'field0',
                    'terms': [
                        {
                            'term': 'value0',
                            'count': 1,
                        }
                    ]
                }
            ]

        response = parse(StringIO(''.join(compose(sruTermDrilldown.extraResponseData(drilldownData, sruArguments={})))))

        self.assertEquals(drilldownData, loads(xpathFirst(response, '//drilldown:term-drilldown/drilldown:json/text()')))
예제 #29
0
 def testMods3xToLong(self):
     response = self.doSruQuery(**{'query': 'urn:NBN:nl:ui:18-2271', 'recordSchema':'knaw_long'}) # knaw_record2_didlmods
     self.assertEqual(1, int(str(xpathFirst(response, '//srw:numberOfRecords/text()'))))
     self.assertEqual('urn:NBN:nl:ui:18-2271', testNamespaces.xpathFirst(response, '//long:persistentIdentifier/text()'))
     self.assertEqual('closedAccess', testNamespaces.xpathFirst(response, '//long:objectFiles/long:objectFile/long:accessRights/text()'))
     self.assertEqual('urn:NBN:nl:ui:18-2271-OF', testNamespaces.xpathFirst(response, '//long:objectFiles/long:objectFile/long:persistentIdentifier/text()'))
     self.assertEqual('2012-01-01', testNamespaces.xpathFirst(response, '//long:objectFiles/long:objectFile/long:embargo/text()'))
     self.assertEqual('embargoedAccess', testNamespaces.xpathFirst(response, '//long:knaw_long/long:accessRights/text()'))
     self.assertEqual('Paths and flows---a historical survey', testNamespaces.xpathFirst(response, '//long:metadata/long:titleInfo[@xml:lang="en"]/long:title/text()'))
     self.assertEqual('personal', testNamespaces.xpathFirst(response, '//long:metadata/long:name[2]/long:type/text()'))
     self.assertEqual('prof.dr. Bennis, H. (Hans)', testNamespaces.xpathFirst(response, '//long:metadata/long:name[2]/long:unstructured/text()'))
     self.assertEqual('Bennis', testNamespaces.xpathFirst(response, '//long:metadata/long:name[2]/long:family/text()'))
     self.assertEqual('prof.dr. H.J. (Hans)', testNamespaces.xpathFirst(response, '//long:metadata/long:name[2]/long:given/text()'))
     self.assertEqual('dgg', testNamespaces.xpathFirst(response, '//long:metadata/long:name[2]/long:mcRoleTerm/text()'))
     self.assertEqual('http://orcid.org/0000-0002-1825-0097', testNamespaces.xpathFirst(response, '//long:metadata/long:name[2]/long:nameIdentifier[@type="orcid"]/text()'))
     self.assertEqual('info:eu-repo/dai/nl/071792279', testNamespaces.xpathFirst(response, '//long:metadata/long:name[2]/long:nameIdentifier[@type="dai-nl"]/text()'))
     self.assertEqual('Copyright (c) Aiki, T (Toyohiko); Copyright (c) Hilhorst, D; Copyright (c) Mimura,', testNamespaces.xpathFirst(response, '//long:metadata/long:rightsDescription/text()')[0:82])
     self.assertEqual('book', testNamespaces.xpathFirst(response, '//long:metadata/long:genre/text()'))
     self.assertEqual('freight', testNamespaces.xpathFirst(response, '//long:metadata/long:subject[not(@xml:lang)]/long:topic/long:topicValue/text()'))
     self.assertEqual('In english please.', testNamespaces.xpathFirst(response, '//long:metadata/long:subject[@xml:lang="en"]/long:topic/long:topicValue/text()'))
     self.assertEqual('Samenvatting', testNamespaces.xpathFirst(response, '//long:metadata/long:abstract/text()'))
     self.assertEqual('1993-1-01', testNamespaces.xpathFirst(response, '//long:metadata/long:dateIssued/long:unParsed/text()'))
     self.assertEqual('1993-01-01', testNamespaces.xpathFirst(response, '//long:metadata/long:dateIssued/long:parsed/text()'))
     self.assertEqual('0010-440X', testNamespaces.xpathFirst(response, '//long:metadata/long:publication_identifier[@type="issn"]/text()'))
     self.assertEqual('9002233389', testNamespaces.xpathFirst(response, '//long:metadata/long:publication_identifier[@type="isbn"]/text()'))
     self.assertEqual('http://repository-acc.ubn.ru.nl/handle/123456789/126651', testNamespaces.xpathFirst(response, '//long:metadata/long:related_identifier/text()'))
     self.assertEqual('http://repository.cwi.nl/search/fullrecord.php?publnr=2271', testNamespaces.xpathFirst(response, '//long:metadata/long:location_url/text()'))
     self.assertEqual('text', testNamespaces.xpathFirst(response, '//long:metadata/long:typeOfResource/text()'))
     self.assertEqual('en', testNamespaces.xpathFirst(response, '//long:metadata/long:language/text()'))
     self.assertEqual('info:eu-repo/grantAgreement/EC/FP7/282797', testNamespaces.xpathFirst(response, '//long:metadata/long:grantAgreements/long:grantAgreement/long:code/text()'))
     self.assertEqual('EERA Design Tools for Offshore Wind Farm Cluster (EERA-DTOC)', testNamespaces.xpathFirst(response, '//long:metadata/long:grantAgreements/long:grantAgreement/long:title/text()'))
     self.assertEqual('The European Energy Research Alliance (EERA)', testNamespaces.xpathFirst(response, '//long:metadata/long:grantAgreements/long:grantAgreement/long:description/text()')[0:44])
     self.assertEqual('European Commission CORDIS', testNamespaces.xpathFirst(response, '//long:metadata/long:grantAgreements/long:grantAgreement/long:funder/text()'))
     self.assertEqual(2, len(testNamespaces.xpathFirst(response, '//long:objectFiles/long:objectFile')))
     self.assertEqual(0, len(testNamespaces.xpathFirst(response, '//long:objectFiles/long:objectFile/long:resource[@mimeType="application/pdf" and @ref="http://oai.cwi.nl/oai/asset/2271/2271OA.pdf"]')))
     self.assertEqual(3, len(testNamespaces.xpath(response, '//long:metadata/long:name')))
     self.assertEqual(1, len(testNamespaces.xpath(response, '//long:metadata/long:rightsDescription')))
     self.assertEqual(7, len(testNamespaces.xpath(response, '//long:metadata/long:subject/long:topic')))
     self.assertEqual(3, len(testNamespaces.xpath(response, '//long:metadata/long:grantAgreements/long:grantAgreement')))
     
     response = self.doSruQuery(**{'query': 'URN:NBN:NL:UI:25-711504', 'recordSchema':'knaw_long'}) # TODO find exact op pubid
     #print etree.tostring(response)
     self.assertEqual('restrictedAccess', testNamespaces.xpathFirst(response, '//long:objectFiles/long:objectFile/long:accessRights/text()'))
     self.assertEqual('restrictedAccess', testNamespaces.xpathFirst(response, '//long:knaw_long/long:accessRights/text()'))
예제 #30
0
 def testSruQuery(self):
     response = self.doSruQuery(query='*', recordSchema='knaw_short')
     # print "doSruQuery(query='*'):", etree.tostring(response)
     self.assertEqual('13',
                      xpathFirst(response, '//srw:numberOfRecords/text()'))
     self.assertEqual(
         set([
             'Example Program 1', 'Example Program 2',
             'RAIN: Pan-European gridded data sets of extreme weather probability of occurrence under present and future climate',
             'Appositie en de interne struktuur van de NP',
             'Paden en stromingen---a historical survey',
             'Late-type Giants in the Inner Galaxy',
             'Preface to special issue (Fast reaction - slow diffusion scenarios: PDE approximations and free boundaries)',
             'Conditiebepaling PVC', 'Wetenschapswinkel', 'H.J. Bennis'
         ]),
         set(
             testNamespaces.xpath(
                 response,
                 '//short:metadata/short:titleInfo[1]/short:title/text()')))
예제 #31
0
 def testDataciteToLong(self):
     response = self.doSruQuery(**{'query': 'urn:nbn:nl:ui:13-jsk-7ek', 'recordSchema':'knaw_long'})
     self.assertEqual(1, int(str(xpathFirst(response, '//srw:numberOfRecords/text()'))))
     self.assertEqual('urn:nbn:nl:ui:13-jsk-7ek', testNamespaces.xpathFirst(response, '//long:persistentIdentifier/text()'))
     self.assertEqual('doi:10.17026/dans-zqm-htb9', testNamespaces.xpathFirst(response, '//long:humanStartPage/text()'))
     self.assertEqual('embargoedAccess', testNamespaces.xpathFirst(response, '//long:accessRights/text()'))
     self.assertEqual('Locatie [Matthijs Tinxgracht 16] te Edam, gemeente Edam-Volendam.', testNamespaces.xpathFirst(response, '//long:metadata/long:titleInfo/long:title/text()')[0:65])
     self.assertEqual('Jacobs, E.', testNamespaces.xpathFirst(response, '//long:metadata/long:name[1]/long:unstructured/text()'))
     self.assertEqual('personal', testNamespaces.xpathFirst(response, '//long:metadata/long:name[1]/long:type/text()'))
     self.assertEqual('Burnier, C.Y.', testNamespaces.xpathFirst(response, '//long:metadata/long:name[2]/long:unstructured/text()'))
     self.assertEqual('corporate', testNamespaces.xpathFirst(response, '//long:metadata/long:name[2]/long:type/text()'))
     self.assertEqual('Miller, Elizabeth', testNamespaces.xpathFirst(response, '//long:metadata/long:name[3]/long:unstructured/text()'))
     self.assertEqual('personal', testNamespaces.xpathFirst(response, '//long:metadata/long:name[3]/long:type/text()'))
     self.assertEqual('Miller', testNamespaces.xpathFirst(response, '//long:metadata/long:name[3]/long:family/text()'))
     self.assertEqual('Elizabeth', testNamespaces.xpathFirst(response, '//long:metadata/long:name[3]/long:given/text()'))
     self.assertEqual('cre', testNamespaces.xpathFirst(response, '//long:metadata/long:name[3]/long:mcRoleTerm/text()'))
     self.assertEqual('ctb', testNamespaces.xpathFirst(response, '//long:metadata/long:name[5]/long:mcRoleTerm/text()'))
     self.assertEqual('http://orcid.org/0000-0001-5000-0007', testNamespaces.xpathFirst(response, '//long:metadata/long:name/long:nameIdentifier[@type="orcid"]/text()'))
     self.assertEqual('info:eu-repo/dai/nl/072728442', testNamespaces.xpathFirst(response, '//long:metadata/long:name/long:nameIdentifier[@type="dai-nl"]/text()'))
     self.assertEqual('Groningen Institute of Archaeology, University of Groningen', testNamespaces.xpathFirst(response, '//long:metadata/long:name[1]/long:affiliation/text()'))
     self.assertEqual('Jacobs en Burnier, archeologisch projectbureau', testNamespaces.xpathFirst(response, '//long:metadata/long:publisher/text()'))
     self.assertEqual('Archaeology', testNamespaces.xpathFirst(response, '//long:metadata/long:subject/long:topic[long:subjectScheme/text() = "NARCIS-classification"]/long:topicValue/text()'))
     self.assertEqual('OPGRAVING', testNamespaces.xpathFirst(response, '//long:metadata/long:subject[@xml:lang="en"]/long:topic/long:topicValue/text()'))
     self.assertEqual('ABR-complex', testNamespaces.xpathFirst(response, '//long:metadata/long:subject[not(@xml:lang)]/long:topic/long:subjectScheme/text()'))
     self.assertEqual('Abstract van dit document', testNamespaces.xpathFirst(response, '//long:metadata/long:abstract/text()'))
     self.assertEqual('Abstract of this document', testNamespaces.xpathFirst(response, '//long:metadata/long:abstract[@xml:lang="en"]/text()'))
     self.assertEqual('2009-9-4', testNamespaces.xpathFirst(response, '//long:metadata/long:dateSubmitted/long:unParsed/text()'))
     self.assertEqual('2009-09-04', testNamespaces.xpathFirst(response, '//long:metadata/long:dateSubmitted/long:parsed/text()'))
     self.assertEqual('2009-11-24', testNamespaces.xpathFirst(response, '//long:metadata/long:dateAvailable/long:parsed/text()'))
     self.assertEqual('urn:nbn:nl:ui:13-jsk-7ek', testNamespaces.xpathFirst(response, '//long:metadata/long:publication_identifier[@type="nbn"]/text()'))
     self.assertEqual('dataset', testNamespaces.xpathFirst(response, '//long:metadata/long:typeOfResource/@generaltype'))
     self.assertEqual('European Commission', testNamespaces.xpathFirst(response, '//long:metadata/long:grantAgreements//long:grantAgreement/long:funder/text()'))
     self.assertEqual('nl', testNamespaces.xpathFirst(response, '//long:metadata/long:language/text()'))
     self.assertEqual('19 p.', testNamespaces.xpathFirst(response, '//long:metadata/long:format/text()'))
     self.assertEqual('Matthijs Tinxgracht 16', testNamespaces.xpathFirst(response, '//long:metadata/long:geoLocations/long:geoLocation[3]/long:geoLocationPlace/text()'))
     self.assertEqual('52.51483176', testNamespaces.xpathFirst(response, '//long:metadata/long:geoLocations/long:geoLocation[3]/long:geoLocationPoint/long:pointLongitude/text()'))
     self.assertEqual('5.04757305', testNamespaces.xpathFirst(response, '//long:metadata/long:geoLocations/long:geoLocation[3]/long:geoLocationPoint/long:pointLatitude/text()'))
     self.assertEqual('51.69204992', testNamespaces.xpathFirst(response, '//long:metadata/long:geoLocations/long:geoLocation[4]/long:geoLocationBox/long:northBoundLatitude/text()'))
     self.assertEqual(5, len(testNamespaces.xpath(response, '//long:metadata/long:name')))
     self.assertEqual(8, len(testNamespaces.xpath(response, '//long:metadata/long:subject[not(@xml:lang)]/long:topic')))
     self.assertEqual(2, len(testNamespaces.xpath(response, '//long:metadata/long:abstract')))
     self.assertEqual(4, len(testNamespaces.xpath(response, '//long:metadata/long:geoLocations/long:geoLocation')))
예제 #32
0
 def testDcToLong(self):
     response = self.doSruQuery(**{'query': '2016-05-05', 'recordSchema':'knaw_long'})
     self.assertEqual(1, int(str(xpathFirst(response, '//srw:numberOfRecords/text()'))))
     self.assertEqual('restrictedAccess', testNamespaces.xpathFirst(response, '//long:knaw_long/long:accessRights/text()'))
     self.assertEqual('Example Program 1', testNamespaces.xpathFirst(response, '//long:metadata/long:titleInfo/long:title/text()'))
     self.assertEqual('personal', testNamespaces.xpathFirst(response, '//long:metadata/long:name/long:type/text()'))
     self.assertEqual('Seecr', testNamespaces.xpathFirst(response, '//long:metadata/long:name/long:unstructured/text()'))
     self.assertEqual('aut', testNamespaces.xpathFirst(response, '//long:metadata/long:name/long:mcRoleTerm/text()'))
     self.assertEqual('Seecr', testNamespaces.xpathFirst(response, '//long:metadata/long:publisher/text()'))
     self.assertEqual('Search', testNamespaces.xpathFirst(response, '//long:metadata/long:subject/long:topic/long:topicValue/text()'))
     self.assertEqual('This is an example program about Search with Meresco', testNamespaces.xpathFirst(response, '//long:metadata/long:abstract/text()'))
     self.assertEqual('2016-5-5', testNamespaces.xpathFirst(response, '//long:metadata/long:dateIssued/long:unParsed/text()'))
     self.assertEqual('2016-05-05', testNamespaces.xpathFirst(response, '//long:metadata/long:dateIssued/long:parsed/text()'))
     self.assertEqual('en', testNamespaces.xpathFirst(response, '//long:metadata/long:language/text()'))
     self.assertEqual(1, len(testNamespaces.xpath(response, '//long:metadata/long:name')))
     self.assertEqual(1, len(testNamespaces.xpath(response, '//long:metadata/long:subject/long:topic')))
     self.assertEqual(2, len(testNamespaces.xpath(response, '//long:metadata/long:publication_identifier')))
     self.assertEqual('10.1002/lno.10611', testNamespaces.xpathFirst(response, '//long:metadata/long:publication_identifier/text()'))
     self.assertEqual(2, len(testNamespaces.xpath(response, '//long:metadata/long:related_identifier')))
     self.assertEqual('10.1234.567/abc', testNamespaces.xpathFirst(response, '//long:metadata/long:related_identifier/text()'))
예제 #33
0
    def testSruQueryWithDrilldown(self):
        # response = self.doSruQuery(**{'maximumRecords': '0', "query": '*', "x-term-drilldown": "dd_penv:6,dd_thesis:6,dd_fin:6,status:5"})
        response = self.doSruQuery(**{
            "query": '*',
            'maximumRecords': '1',
            "x-term-drilldown": "dd_cat:0"
        })
        # print "DD body:", etree.tostring(response)
        self.assertEqual('13',
                         xpathFirst(response, '//srw:numberOfRecords/text()'))
        # self.assertEqual(set(['Example Program 1', 'Example Program 2']), set(xpath(response, '//srw:recordData/oai_dc:dc/dc:title/text()')))

        ddItems = xpath(
            response,
            '//drilldown:term-drilldown/drilldown:navigator[@name="dd_cat"]/drilldown:item'
        )
        drilldown = [(i.text, i.attrib['count']) for i in ddItems]
        # print 'DD:', drilldown
        self.assertEqual([('D37000', '2'), ('D30000', '2'), ('A50000', '1'),
                          ('A80000', '1'), ('D40000', '1'), ('D50000', '1'),
                          ('D60000', '1')], drilldown)
예제 #34
0
    def testRefresh(self):
        oldlogs = self.getLogs()
        log = HarvesterLog(stateDir=join(self.harvesterStateDir, DOMAIN), logDir=join(self.harvesterLogDir, DOMAIN), name=REPOSITORY)
        log.startRepository()
        for uploadId in ['%s:oai:record:%02d' % (REPOSITORY, i) for i in [1,7,120,121]]:
            log.notifyHarvestedRecord(uploadId)
            log.uploadIdentifier(uploadId)
        for uploadId in ['%s:oai:record:%02d' % (REPOSITORY, i) for i in [4,5,122,123]]:
            log.notifyHarvestedRecord(uploadId)
            log.deleteIdentifier(uploadId)
        log.endRepository('token', '2012-01-01T09:00:00Z')
        log.close()

        self.saveRepository(DOMAIN, REPOSITORY, REPOSITORYGROUP, action='refresh')

        self.startHarvester(repository=REPOSITORY)
        logs = self.getLogs()[len(oldlogs):]
        self.assertEquals(0, len(logs))
        self.startHarvester(repository=REPOSITORY)
        logs = self.getLogs()
        self.assertEquals('/oai', logs[-1]["path"])
        self.assertEquals({'verb': ['ListRecords'], 'metadataPrefix': ['oai_dc']}, logs[-1]["arguments"])
        statsFile = join(self.harvesterStateDir, DOMAIN, '%s.stats' % REPOSITORY)
        token = getResumptionToken(open(statsFile).readlines()[-1])

        self.startHarvester(repository=REPOSITORY)
        logs = self.getLogs()
        self.assertEquals('/oai', logs[-1]["path"])
        self.assertEquals({'verb': ['ListRecords'], 'resumptionToken': [token]}, logs[-1]["arguments"])
        self.assertEquals(15, self.sizeDumpDir())

        self.startHarvester(repository=REPOSITORY)
        self.assertEquals(17, self.sizeDumpDir())
        deleteFiles = [join(self.dumpDir, f) for f in listdir(self.dumpDir) if '_delete' in f]
        deletedIds = set([xpathFirst(parse(open(x)), '//ucp:recordIdentifier/text()') for x in deleteFiles])
        self.assertEquals(set(['%s:oai:record:03' % REPOSITORY, '%s:oai:record:06' % REPOSITORY, '%s:oai:record:120' % REPOSITORY, '%s:oai:record:121' % REPOSITORY]), deletedIds)

        logs = self.getLogs()[len(oldlogs):]
        self.startHarvester(repository=REPOSITORY)
        self.assertEquals(len(logs), len(self.getLogs()[len(oldlogs):]), 'Action is over, expect nothing more.')
    def testSruTermDrilldownWithPivotsInJson(self):
        sruTermDrilldown = SruTermDrilldown(defaultFormat=FORMAT_JSON)

        drilldownData = [{
            'fieldname':
            'field0',
            'terms': [{
                'term': 'value0',
                'count': 1,
                'pivot': {
                    'fieldname':
                    'field1',
                    'terms': [{
                        'term': 'value0_0',
                        'count': 10,
                    }, {
                        'term': 'value0 & 1',
                        'count': 20
                    }]
                }
            }, {
                'term': 'value1',
                'count': 2
            }]
        }]

        response = ''.join(
            compose(
                sruTermDrilldown.extraResponseData(
                    drilldownData,
                    sruArguments={'x-drilldown-format': ['json']})))
        self.assertEqual(
            drilldownData,
            loads(
                xpathFirst(
                    XML(response),
                    '//drilldown:term-drilldown/drilldown:json/text()')))

        xsdFilename = self._getXsdFilename(response)
        assertValid(response, join(schemasPath, xsdFilename))
예제 #36
0
 def testModsToLong(self):
     response = self.doSruQuery(**{'query': 'URN:NBN:NL:UI:17-565', 'recordSchema':'knaw_long'})
     self.assertEqual(1, int(str(xpathFirst(response, '//srw:numberOfRecords/text()'))))
     self.assertEqual('URN:NBN:NL:UI:17-565', testNamespaces.xpathFirst(response, '//long:persistentIdentifier/text()'))
     self.assertEqual('openAccess', testNamespaces.xpathFirst(response, '//long:knaw_long/long:accessRights/text()'))
     self.assertEqual('Appositie en de interne struktuur van de NP', testNamespaces.xpathFirst(response, '//long:metadata/long:titleInfo/long:title/text()'))
     self.assertEqual('personal', testNamespaces.xpathFirst(response, '//long:metadata/long:name/long:type/text()'))
     self.assertEqual('Bennis&Bennis', testNamespaces.xpathFirst(response, '//long:metadata/long:name/long:family/text()'))
     self.assertEqual('prof.dr. H.J.', testNamespaces.xpathFirst(response, '//long:metadata/long:name/long:given/text()'))
     self.assertEqual('aut', testNamespaces.xpathFirst(response, '//long:metadata/long:name/long:mcRoleTerm/text()'))
     self.assertEqual('http://isni.org/isni/0000000081508690', testNamespaces.xpathFirst(response, '//long:metadata/long:name/long:nameIdentifier[@type="isni"]/text()'))
     self.assertEqual('http://orcid.org/0000-0002-1825-0097', testNamespaces.xpathFirst(response, '//long:metadata/long:name/long:nameIdentifier[@type="orcid"]/text()'))
     self.assertEqual('info:eu-repo/dai/nl/068519397', testNamespaces.xpathFirst(response, '//long:metadata/long:name/long:nameIdentifier[@type="dai-nl"]/text()'))
     self.assertEqual('Blackwell Publishers', testNamespaces.xpathFirst(response, '//long:metadata/long:relatedItem[@type="host"]/long:publisher/text()'))
     self.assertEqual('2008 Royal Tropical Institute. This work is licensed under', testNamespaces.xpathFirst(response, '//long:metadata/long:rightsDescription/text()')[2:60])
     self.assertEqual('article', testNamespaces.xpathFirst(response, '//long:metadata/long:genre/text()'))
     self.assertEqual('This is the subject', testNamespaces.xpathFirst(response, '//long:metadata/long:subject/long:topic/long:topicValue/text()'))
     self.assertEqual('FransHeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeellllllang', testNamespaces.xpathFirst(response, '//long:metadata/long:abstract/text()'))
     self.assertEqual('opgraafdatum 3000 jaar voor christus', testNamespaces.xpathFirst(response, '//long:metadata/long:dateIssued/long:unParsed/text()'))
     self.assertEqual('00282162', testNamespaces.xpathFirst(response, '//long:metadata/long:publication_identifier[@type="issn"]/text()'))
     self.assertEqual('http://www.dds.nl/semantics/article', testNamespaces.xpathFirst(response, '//long:metadata/long:location_url/text()'))
     self.assertEqual('Amsterdam', testNamespaces.xpathFirst(response, '//long:metadata/long:placeTerm/text()'))
     self.assertEqual('text', testNamespaces.xpathFirst(response, '//long:metadata/long:typeOfResource/text()'))
     self.assertEqual('nl', testNamespaces.xpathFirst(response, '//long:metadata/long:language/text()'))
     self.assertEqual('info:eu-repo/grantAgreement/EC/FP5/654321', testNamespaces.xpathFirst(response, '//long:metadata/long:grantAgreements/long:grantAgreement/long:code/text()'))
     self.assertEqual('EERA Design Tools for Offshore Wind Farm Cluster (EERA-DTOC)', testNamespaces.xpathFirst(response, '//long:metadata/long:grantAgreements/long:grantAgreement/long:title/text()'))
     self.assertEqual('The European Energy Research Alliance (EERA)', testNamespaces.xpathFirst(response, '//long:metadata/long:grantAgreements/long:grantAgreement/long:description/text()')[0:44])
     self.assertEqual('Uni, Versiteit', testNamespaces.xpathFirst(response, '//long:metadata/long:grantAgreements/long:grantAgreement/long:funder/text()'))
     self.assertEqual('8', testNamespaces.xpathFirst(response, '//long:metadata/long:relatedItem[@type="host"]/long:part/long:volume/text()'))
     self.assertEqual('5', testNamespaces.xpathFirst(response, '//long:metadata/long:relatedItem[@type="host"]/long:part/long:issue/text()'))
     self.assertEqual('209', testNamespaces.xpathFirst(response, '//long:metadata/long:relatedItem[@type="host"]/long:part/long:start_page/text()'))
     self.assertEqual('228', testNamespaces.xpathFirst(response, '//long:metadata/long:relatedItem[@type="host"]/long:part/long:end_page/text()'))
     self.assertEqual('Spektator', testNamespaces.xpathFirst(response, '//long:metadata/long:relatedItem[@type="host"]/long:titleInfo[@xml:lang="en"]/long:title/text()'))
     self.assertEqual('00286666', testNamespaces.xpathFirst(response, '//long:metadata/long:relatedItem[@type="host"]/long:publication_identifier[@type="issn"]/text()'))
     self.assertEqual('Oxford', testNamespaces.xpathFirst(response, '//long:metadata/long:relatedItem[@type="host"]/long:placeTerm/text()'))
     self.assertEqual(0, len(testNamespaces.xpathFirst(response, '//long:objectFiles/long:objectFile/long:resource[@mimeType="application/pdf" and @ref="http://depot.knaw.nl/565/1/14807.pdf"]')))
     self.assertEqual(3, len(testNamespaces.xpath(response, '//long:metadata/long:name')))
     self.assertEqual(1, len(testNamespaces.xpath(response, '//long:metadata/long:rightsDescription')))
     self.assertEqual(1, len(testNamespaces.xpath(response, '//long:metadata/long:subject[@xml:lang="en"]/long:topic')))
     self.assertEqual(2, len(testNamespaces.xpath(response, '//long:metadata/long:grantAgreements/long:grantAgreement')))
    def handle(self, lxmlNode):
        __callstack_var_oaiListRequest__ = {
            'metadataPrefix': self._metadataPrefix,
            'set': self._set,
        }
        harvestingDone = False
        noRecordsMatch = False

        errors = xpath(lxmlNode, "/oai:OAI-PMH/oai:error")
        if len(errors) == 1 and errors[0].get("code") == "noRecordsMatch":
            noRecordsMatch = True
        if len(errors) > 0 and not noRecordsMatch:
            for error in errors:
                self._errorState = "%s: %s" % (error.get("code"), error.text)
                self._logError(self._errorState)
            self._resumptionToken = None
            self._maybeCommit()
            return
        try:
            if not noRecordsMatch:
                self.do.startOaiBatch()
                try:
                    yield self._processRecords(lxmlNode)
                finally:
                    self.do.stopOaiBatch()
            self._from = xpathFirst(lxmlNode,
                                    '/oai:OAI-PMH/oai:responseDate/text()')
            if self._resumptionToken is None:
                harvestingDone = True
                if self._restartAfterFinish:
                    self._from = None
                else:
                    self.scheduleNextRequest(self._incrementalHarvestSchedule)
            self._errorState = None
        finally:
            self._maybeCommit()

        if harvestingDone:
            self.do.signalHarvestingDone(state=self.getState())
예제 #38
0
 def testDidlDcToLong(self):
     response = self.doSruQuery(**{'query': '2016-01-31', 'recordSchema':'knaw_long'})
     self.assertEqual(1, int(str(xpathFirst(response, '//srw:numberOfRecords/text()'))))
     self.assertEqual('doi:10.4121/collection:ab70dbf9-ac4f-40a7-9859-9552d38fdccd', testNamespaces.xpathFirst(response, '//long:persistentIdentifier/text()'))
     self.assertEqual('openAccess', testNamespaces.xpathFirst(response, '//long:knaw_long/long:accessRights/text()'))
     self.assertEqual('RAIN: Pan-European gridded data sets of extreme weather probability of occurrence under present and future climate', testNamespaces.xpathFirst(response, '//long:metadata/long:titleInfo/long:title/text()'))
     self.assertEqual('personal', testNamespaces.xpathFirst(response, '//long:metadata/long:name/long:type/text()'))
     self.assertEqual('European Severe Storms Laboratory', testNamespaces.xpathFirst(response, '//long:metadata/long:name/long:unstructured/text()'))
     self.assertEqual('aut', testNamespaces.xpathFirst(response, '//long:metadata/long:name/long:mcRoleTerm/text()'))
     self.assertEqual('TU Delft', testNamespaces.xpathFirst(response, '//long:metadata/long:publisher/text()'))
     self.assertEqual('Precipitation', testNamespaces.xpathFirst(response, '//long:metadata/long:subject/long:topic/long:topicValue/text()'))
     self.assertEqual('This collection contains results  of Work Package 2 "Hazard Identification" of project RAIN', testNamespaces.xpathFirst(response, '//long:metadata/long:abstract/text()')[0:91])
     self.assertEqual('2016/1/31', testNamespaces.xpathFirst(response, '//long:metadata/long:dateIssued/long:unParsed/text()'))
     self.assertEqual('2016-01-31', testNamespaces.xpathFirst(response, '//long:metadata/long:dateIssued/long:parsed/text()'))
     self.assertEqual('years 1971-2100', testNamespaces.xpathFirst(response, '//long:metadata/long:coverage/text()'))
     self.assertEqual('application/pdf', testNamespaces.xpathFirst(response, '//long:metadata/long:format/text()'))
     self.assertEqual('en', testNamespaces.xpathFirst(response, '//long:metadata/long:language/text()'))
     self.assertEqual(4, len(testNamespaces.xpath(response, '//long:metadata/long:name')))
     self.assertEqual(1, len(testNamespaces.xpath(response, '//long:metadata/long:publisher')))
     self.assertEqual(10, len(testNamespaces.xpath(response, '//long:metadata/long:subject/long:topic')))
     self.assertEqual(2, len(testNamespaces.xpath(response, '//long:metadata/long:coverage')))
     self.assertEqual(4, len(testNamespaces.xpath(response, '//long:metadata/long:format')))
예제 #39
0
    def handle(self, lxmlNode):
        __callstack_var_oaiListRequest__ = {
            'metadataPrefix': self._metadataPrefix,
            'set': self._set,
        }
        harvestingDone = False
        noRecordsMatch = False

        errors = xpath(lxmlNode, "/oai:OAI-PMH/oai:error")
        if len(errors) == 1 and errors[0].get("code") == "noRecordsMatch":
            noRecordsMatch = True
        if len(errors) > 0 and not noRecordsMatch:
            for error in errors:
                self._errorState = "%s: %s" % (error.get("code"), error.text)
                self._logError(self._errorState)
            self._resumptionToken = None
            self._maybeCommit()
            return
        try:
            if not noRecordsMatch:
                self.do.startOaiBatch()
                try:
                    yield self._processRecords(lxmlNode)
                finally:
                    self.do.stopOaiBatch()
            self._from = xpathFirst(lxmlNode, '/oai:OAI-PMH/oai:responseDate/text()')
            if self._resumptionToken is None:
                harvestingDone = True
                if self._restartAfterFinish:
                    self._from = None
                else:
                    self.scheduleNextRequest(self._incrementalHarvestSchedule)
            self._errorState = None
        finally:
            self._maybeCommit()

        if harvestingDone:
            self.do.signalHarvestingDone(state=self.getState())
    def testWrongFormat(self):
        sruTermDrilldown = SruTermDrilldown()
        drilldownData = [{
            'fieldname': 'field0',
            'terms': [{
                'term': 'value0',
                'count': 1,
            }]
        }]

        response = parse(
            StringIO(''.join(
                compose(
                    sruTermDrilldown.extraResponseData(
                        drilldownData,
                        sruArguments={'x-drilldown-format': 'text'})))))

        self.assertEqualsWS(
            "Expected x-drilldown-format to be one of: ['xml', 'json']",
            xpathFirst(
                response,
                '//drilldown:term-drilldown/diag:diagnostic/diag:message/text()'
            ))
    def testSruTermDrilldownWithPivotsInJson(self):
        sruTermDrilldown = SruTermDrilldown(defaultFormat=FORMAT_JSON)

        drilldownData = [
                {
                    'fieldname': 'field0',
                    'terms': [
                        {
                            'term': 'value0',
                            'count': 1,
                            'pivot': {
                                'fieldname': 'field1',
                                'terms': [
                                    {
                                        'term': 'value0_0',
                                        'count': 10,
                                    },
                                    {
                                        'term': 'value0 & 1',
                                        'count': 20
                                    }
                                ]
                            }
                        },
                        {
                            'term': 'value1',
                            'count': 2
                        }
                    ]
                }
            ]

        response = ''.join(compose(sruTermDrilldown.extraResponseData(drilldownData, sruArguments={'x-drilldown-format': ['json']})))
        self.assertEquals(drilldownData, loads(xpathFirst(XML(response), '//drilldown:term-drilldown/drilldown:json/text()')))

        xsdFilename = self._getXsdFilename(response)
        assertValid(response, join(schemasPath, xsdFilename))
    def testDefaultFormat(self):
        self.assertRaises(ValueError,
                          lambda: SruTermDrilldown(defaultFormat='text'))
        sruTermDrilldown = SruTermDrilldown(defaultFormat='json')
        drilldownData = [{
            'fieldname': 'field0',
            'terms': [{
                'term': 'value0',
                'count': 1,
            }]
        }]

        response = parse(
            StringIO(''.join(
                compose(
                    sruTermDrilldown.extraResponseData(drilldownData,
                                                       sruArguments={})))))

        self.assertEqual(
            drilldownData,
            loads(
                xpathFirst(
                    response,
                    '//drilldown:term-drilldown/drilldown:json/text()')))
예제 #43
0
    def testAddSetInfoWithElementTree(self):
        oaiContainer = createElement('oai:PMH')
        record = createSubElement(oaiContainer, 'oai:record')
        header = createSubElement(record, 'oai:header')
        createSubElement(header, 'oai:setSpec', text='1')

        consume(
            self.subject.add('123', 'oai_dc',
                             xpathFirst(oaiContainer, '/oai:PMH/oai:record')))

        self.assertEqual(['updateSet', 'updateMetadataFormat', 'addOaiRecord'],
                         self.observer.calledMethodNames())
        self.assertEqual('123',
                         self.observer.calledMethods[2].kwargs['identifier'])
        self.assertEqual({
            'setSpec': '1',
            'setName': '1'
        }, self.observer.calledMethods[0].kwargs)
        self.assertEqual(
            {
                'prefix': 'oai_dc',
                'schema': '',
                'namespace': "http://www.openarchives.org/OAI/2.0/"
            }, self.observer.calledMethods[1].kwargs)
예제 #44
0
 def assertSruQuery(self, numberOfRecords, query, printout=False):
     response = self.doSruQuery(**{'query':query, "recordSchema": "knaw_short", "x-recordSchema": "header"}) # , 'maximumRecords': '1'
     if printout: print "SruQuery response:", etree.tostring(response, pretty_print = True, encoding='utf-8')
     self.assertEquals(numberOfRecords, int(str(xpathFirst(response, '//srw:numberOfRecords/text()'))))
예제 #45
0
    def testPublIdentifier(self):
        response = self.doSruQuery(**{'query': 'untokenized.relatedid exact "issn:1937-1632"', 'maximumRecords': '1', 'recordSchema':'knaw_long'})
#         print "DD body:", etree.tostring(response)
        self.assertEqual('knaw:record:4', xpathFirst(response, '//srw:recordIdentifier/text()'))
        self.assertEqual(1, int(str(xpathFirst(response, '//srw:numberOfRecords/text()'))))
        self.assertEqual('Springer', testNamespaces.xpathFirst(response, '//long:metadata/long:relatedItem[@type="host"]/long:publisher/text()'))
예제 #46
0
 def completeListSize(self):
     return xpathFirst(self.response,
                       '//oai:resumptionToken/@completeListSize')
예제 #47
0
 def testXpathFirst(self):
     self.assertEquals(None, xpathFirst(ANY_XML, "/root/not_found"))
     self.assertEquals(_Element, type(xpathFirst(ANY_XML, "/root/sub")))
     self.assertEquals(str, type(xpathFirst(ANY_XML, "/root/sub1/text()")))
     self.assertEquals("text", xpathFirst(ANY_XML, "/root/sub1/text()"))
예제 #48
0
 def testSruLimitStartRecord(self):
     response = self.doSruQuery(**{'maximumRecords': '1', 'startRecord': '4002', 'query':'*'})
     self.assertEqual("Argument 'startRecord' too high, maximum: 4000", xpathFirst(response, '//diag:diagnostic/diag:details/text()'))
 def _getXsdFilename(self, response):
     schemaLocation = xpathFirst(XML(response), '/drilldown:drilldown/@xsi:schemaLocation')
     namespace, xsd = schemaLocation.split()
     self.assertEquals(namespaces['drilldown'], namespace)
     self.assertEquals('http://meresco.org/files/xsd', dirname(xsd))
     return basename(xsd)
예제 #50
0
 def testXpathFirst(self):
     self.assertEqual(None, xpathFirst(ANY_XML, "/root/not_found"))
     self.assertEqual(_Element, type(xpathFirst(ANY_XML, "/root/sub")))
     self.assertEqual(str, type(xpathFirst(ANY_XML, "/root/sub1/text()")))
     self.assertEqual("text", xpathFirst(ANY_XML, "/root/sub1/text()"))
예제 #51
0
 def responseDate(self):
     return xpathFirst(self.response,
                       '/oai:OAI-PMH/oai:responseDate/text()')
    def testWrongFormat(self):
        sruTermDrilldown = SruTermDrilldown()
        drilldownData = [
                {
                    'fieldname': 'field0',
                    'terms': [
                        {
                            'term': 'value0',
                            'count': 1,
                        }
                    ]
                }
            ]

        response = parse(StringIO(''.join(compose(sruTermDrilldown.extraResponseData(drilldownData, sruArguments={'x-drilldown-format':'text'})))))

        self.assertEqualsWS("Expected x-drilldown-format to be one of: ['xml', 'json']", xpathFirst(response, '//drilldown:term-drilldown/diag:diagnostic/diag:message/text()'))
예제 #53
0
 def resumptionToken(self):
     return xpathFirst(self.response, "//oai:resumptionToken/text()")
예제 #54
0
 def testPublIdentifier(self):
     response = self.doSruQuery(**{'query':'1937-1632', 'maximumRecords': '1', 'recordSchema':'knaw_long'})
     # print "DD body:", etree.tostring(response)
     #print body.searchRetrieveResponse.records.record.recordData.knaw_long.metadata.relatedItem.publication_identifier
     self.assertEqual('Springer', testNamespaces.xpathFirst(response, '//long:metadata/long:relatedItem[@type="host"]/long:publisher/text()'))
     self.assertEqual(1, int(str(xpathFirst(response, '//srw:numberOfRecords/text()'))))
예제 #55
0
 def testRSS(self): # GMH21 OK
     header, body = getRequest(self.apiPort, '/rss', dict(repositoryId='kb_tst', maximumRecords=10)) #, startRecord='1'
     # print "RSS body:", etree.tostring(body)   
     self.assertEqual(6, len(xpath(body, "/rss/channel/item/description")))
     self.assertEqual('GMH DANS-KB Normalisationlog Syndication', xpathFirst(body, '//channel/title/text()'))
     self.assertEqual('DIDL: HumanStartPage descriptor found in depricated dip namespace.\n', xpathFirst(body, '//item/description/text()'))
예제 #56
0
    def testRefresh(self):
        oldlogs = self.getLogs()
        log = State(stateDir=join(self.harvesterStateDir, DOMAIN),
                    logDir=join(self.harvesterLogDir, DOMAIN),
                    name=REPOSITORY).getHarvesterLog()
        log.startRepository()
        for uploadId in [
                '%s:oai:record:%02d' % (REPOSITORY, i)
                for i in [1, 7, 120, 121]
        ]:
            log.notifyHarvestedRecord(uploadId)
            log.uploadIdentifier(uploadId)
        for uploadId in [
                '%s:oai:record:%02d' % (REPOSITORY, i)
                for i in [4, 5, 122, 123]
        ]:
            log.notifyHarvestedRecord(uploadId)
            log.deleteIdentifier(uploadId)
        log.endRepository('token', '2012-01-01T09:00:00Z')
        log.close()

        self.saveRepository(DOMAIN,
                            REPOSITORY,
                            REPOSITORYGROUP,
                            action='refresh')

        self.startHarvester(repository=REPOSITORY)
        logs = self.getLogs()[len(oldlogs):]
        self.assertEqual(0, len(logs))
        self.startHarvester(repository=REPOSITORY)
        logs = self.getLogs()
        self.assertEqual('/oai', logs[-1]["path"])
        self.assertEqual(
            {
                'verb': ['ListRecords'],
                'metadataPrefix': ['oai_dc']
            }, logs[-1]["arguments"])
        statsFile = join(self.harvesterStateDir, DOMAIN,
                         '%s.stats' % REPOSITORY)
        token = getResumptionToken(open(statsFile).readlines()[-1])

        self.startHarvester(repository=REPOSITORY)
        logs = self.getLogs()
        self.assertEqual('/oai', logs[-1]["path"])
        self.assertEqual({
            'verb': ['ListRecords'],
            'resumptionToken': [token]
        }, logs[-1]["arguments"])
        self.assertEqual(15, self.sizeDumpDir())

        self.startHarvester(repository=REPOSITORY)
        self.assertEqual(17, self.sizeDumpDir())
        deleteFiles = [
            join(self.dumpDir, f) for f in listdir(self.dumpDir)
            if '_delete' in f
        ]
        deletedIds = set([
            xpathFirst(parse(open(x)), '//ucp:recordIdentifier/text()')
            for x in deleteFiles
        ])
        self.assertEqual(
            set([
                '%s:oai:record:03' % REPOSITORY,
                '%s:oai:record:06' % REPOSITORY,
                '%s:oai:record:120' % REPOSITORY,
                '%s:oai:record:121' % REPOSITORY
            ]), deletedIds)

        logs = self.getLogs()[len(oldlogs):]
        self.startHarvester(repository=REPOSITORY)
        self.assertEqual(len(logs), len(self.getLogs()[len(oldlogs):]),
                         'Action is over, expect nothing more.')
예제 #57
0
 def testOai(self): # GMH31 OK
     header, body = getRequest(self.apiPort, '/oai', dict(verb="ListRecords", metadataPrefix=NL_DIDL_NORMALISED_PREFIX))
     # print "OAI body:", etree.tostring(body)
     self.assertEqual('HTTP/1.0 200 OK\r\nContent-Type: text/xml; charset=utf-8', header)
     self.assertEqual(18, len(xpath(body, "//oai:ListRecords/oai:record")))
     self.assertEqual('nl_didl', xpathFirst(body, '//oaiprov:provenance/oaiprov:originDescription/oaiprov:metadataNamespace/text()'))