def setupDataRetrieve(exceptions={}, **returnValues): dataRetrieve = HarvesterDataRetrieve() mockHarvesterData = CallTrace('mockHarvesterData', returnValues=returnValues) mockHarvesterData.exceptions = exceptions dataRetrieve.addObserver(mockHarvesterData) return dataRetrieve, mockHarvesterData
def testErrorInCall(self): dataRetrieve = HarvesterDataRetrieve() observer = CallTrace('observer') observer.exceptions = dict(getError=Exception('Bad Bad Bad')) dataRetrieve.addObserver(observer) result = asString(dataRetrieve.handleRequest(arguments=dict(verb=['GetError']))) header, body = result.split(CRLF*2,1) self.assertEqual(okJson, header+CRLF*2) self.assertEquals({'request': {'verb': 'GetError'}, 'error': {'message': "Exception('Bad Bad Bad',)", 'code': 'unknown'}}, JsonDict.loads(body)) self.assertEquals(['getError'], observer.calledMethodNames())
def testKnownCodeException(self): dataRetrieve = HarvesterDataRetrieve() observer = CallTrace('observer') observer.exceptions = dict(getError=Exception('idDoesNotExist')) dataRetrieve.addObserver(observer) result = asString(dataRetrieve.handleRequest(arguments=dict(verb=['GetError']))) header, body = result.split(CRLF*2,1) self.assertEqual(okJson, header+CRLF*2) self.assertEquals({'request': {'verb': 'GetError'}, 'error': {'message': 'The value of an argument (id or key) is unknown or illegal.', 'code': 'idDoesNotExist'}}, JsonDict.loads(body)) self.assertEquals(['getError'], observer.calledMethodNames())
def testGetUnexisting(self): dataRetrieve = HarvesterDataRetrieve() observer = CallTrace('observer') observer.exceptions=dict(getUnexisting=DeclineMessage()) dataRetrieve.addObserver(observer) result = asString(dataRetrieve.handleRequest(arguments=dict(verb=['GetUnexisting'], argument=['value']))) header, body = result.split(CRLF*2,1) self.assertEqual(okJson, header+CRLF*2) self.assertEquals({'request': {'verb': 'GetUnexisting', 'argument': 'value'}, 'error': {'message': 'Value of the verb argument is not a legal verb, the verb argument is missing, or the verb argument is repeated.', 'code': 'badVerb'}}, JsonDict.loads(body)) self.assertEquals(['getUnexisting'], observer.calledMethodNames()) self.assertEquals({'argument': 'value'}, observer.calledMethods[0].kwargs)
def testKeepResumptionTokenOnFailingAddCall(self): resumptionToken = "u|c1286437597991025|mprefix|s|f" open(join(self.tempdir, 'harvester.state'), 'w').write("Resumptiontoken: %s\n" % resumptionToken) observer = CallTrace() observer.exceptions={'add': Exception("Could be anything")} oaiDownloadProcessor = OaiDownloadProcessor(path="/oai", metadataPrefix="oai_dc", workingDirectory=self.tempdir, xWait=True, err=StringIO()) oaiDownloadProcessor.addObserver(observer) self.assertEquals('GET /oai?%s HTTP/1.0\r\nX-Meresco-Oai-Client-Identifier: %s\r\nUser-Agent: Meresco-Oai-DownloadProcessor/5.x\r\n\r\n' % (urlencode([('verb', 'ListRecords'), ('resumptionToken', resumptionToken), ('x-wait', 'True')]), oaiDownloadProcessor._identifier), oaiDownloadProcessor.buildRequest()) self.assertRaises(Exception, lambda: list(compose(oaiDownloadProcessor.handle(parse(StringIO(LISTRECORDS_RESPONSE % RESUMPTION_TOKEN)))))) self.assertEquals(['startOaiBatch', 'add', 'stopOaiBatch'], [m.name for m in observer.calledMethods]) errorOutput = oaiDownloadProcessor._err.getvalue() self.assertTrue(errorOutput.startswith('Traceback'), errorOutput) self.assertTrue('Exception: Could be anything\nWhile processing:\n<record xmlns="http://www.openarchives.org/OAI/2.0/"><header><identifier>oai:identifier:1' in errorOutput, errorOutput) self.assertEquals('GET /oai?%s HTTP/1.0\r\nX-Meresco-Oai-Client-Identifier: %s\r\nUser-Agent: Meresco-Oai-DownloadProcessor/5.x\r\n\r\n' % (urlencode([('verb', 'ListRecords'), ('resumptionToken', resumptionToken), ('x-wait', 'True')]), oaiDownloadProcessor._identifier), oaiDownloadProcessor.buildRequest())
def testHarvesterStateWithError(self): resumptionToken = "u|c1286437597991025|mprefix|s|f" open(join(self.tempdir, 'harvester.state'), 'w').write("Resumptiontoken: %s\n" % resumptionToken) observer = CallTrace() observer.exceptions={'add': Exception("Could be anything")} oaiDownloadProcessor = OaiDownloadProcessor(path="/oai", metadataPrefix="oai_dc", workingDirectory=self.tempdir, xWait=True, err=StringIO(), name="Name") oaiDownloadProcessor.addObserver(observer) self.assertRaises(Exception, lambda: list(compose(oaiDownloadProcessor.handle(parse(StringIO(LISTRECORDS_RESPONSE % RESUMPTION_TOKEN)))))) state = oaiDownloadProcessor.getState() self.assertEquals(resumptionToken, state.resumptionToken) self.assertEquals(None, state.from_) self.assertEquals("ERROR while processing 'oai:identifier:1': Could be anything", state.errorState) self.assertEquals("Name", state.name) oaiDownloadProcessor2 = OaiDownloadProcessor(path="/oai", metadataPrefix="oai_dc", workingDirectory=self.tempdir, xWait=True, err=StringIO()) state2 = oaiDownloadProcessor2.getState() self.assertEquals(resumptionToken, state2.resumptionToken) self.assertEquals("ERROR while processing 'oai:identifier:1': Could be anything", state2.errorState)
def testKeepResumptionTokenOnFailingAddCall(self): resumptionToken = "u|c1286437597991025|mprefix|s|f" with open(join(self.tempdir, 'harvester.state'), 'w') as f: f.write("Resumptiontoken: %s\n" % resumptionToken) observer = CallTrace() observer.exceptions = {'add': Exception("Could be anything")} oaiDownloadProcessor = OaiDownloadProcessor( path="/oai", metadataPrefix="oai_dc", workingDirectory=self.tempdir, xWait=True, err=StringIO()) oaiDownloadProcessor.addObserver(observer) self.assertEqual( 'GET /oai?%s HTTP/1.0\r\nX-Meresco-Oai-Client-Identifier: %s\r\nUser-Agent: Meresco-Oai-DownloadProcessor/5.x\r\n\r\n' % (urlencode([('verb', 'ListRecords'), ('resumptionToken', resumptionToken), ('x-wait', 'True') ]), oaiDownloadProcessor._identifier), oaiDownloadProcessor.buildRequest()) self.assertRaises( Exception, lambda: list( compose( oaiDownloadProcessor.handle( parse(StringIO(LISTRECORDS_RESPONSE % RESUMPTION_TOKEN) ))))) self.assertEqual(['startOaiBatch', 'add', 'stopOaiBatch'], [m.name for m in observer.calledMethods]) errorOutput = oaiDownloadProcessor._err.getvalue() self.assertTrue(errorOutput.startswith('Traceback'), errorOutput) self.assertTrue( 'Exception: Could be anything\nWhile processing:\n<record xmlns="http://www.openarchives.org/OAI/2.0/"><header><identifier>oai:identifier:1' in errorOutput, errorOutput) self.assertEqual( 'GET /oai?%s HTTP/1.0\r\nX-Meresco-Oai-Client-Identifier: %s\r\nUser-Agent: Meresco-Oai-DownloadProcessor/5.x\r\n\r\n' % (urlencode([('verb', 'ListRecords'), ('resumptionToken', resumptionToken), ('x-wait', 'True') ]), oaiDownloadProcessor._identifier), oaiDownloadProcessor.buildRequest())
def testHarvesterStateWithError(self): resumptionToken = "u|c1286437597991025|mprefix|s|f" with open(join(self.tempdir, 'harvester.state'), 'w') as f: f.write("Resumptiontoken: %s\n" % resumptionToken) observer = CallTrace() observer.exceptions = {'add': Exception("Could be anything")} oaiDownloadProcessor = OaiDownloadProcessor( path="/oai", metadataPrefix="oai_dc", workingDirectory=self.tempdir, xWait=True, err=StringIO(), name="Name") oaiDownloadProcessor.addObserver(observer) self.assertRaises( Exception, lambda: list( compose( oaiDownloadProcessor.handle( parse(StringIO(LISTRECORDS_RESPONSE % RESUMPTION_TOKEN) ))))) state = oaiDownloadProcessor.getState() self.assertEqual(resumptionToken, state.resumptionToken) self.assertEqual(None, state.from_) self.assertEqual( "ERROR while processing 'oai:identifier:1': Could be anything", state.errorState) self.assertEqual("Name", state.name) oaiDownloadProcessor2 = OaiDownloadProcessor( path="/oai", metadataPrefix="oai_dc", workingDirectory=self.tempdir, xWait=True, err=StringIO()) state2 = oaiDownloadProcessor2.getState() self.assertEqual(resumptionToken, state2.resumptionToken) self.assertEqual( "ERROR while processing 'oai:identifier:1': Could be anything", state2.errorState)