def setupDataRetrieve(exceptions={}, **returnValues):
dataRetrieve = HarvesterDataRetrieve()
mockHarvesterData = CallTrace('mockHarvesterData',
returnValues=returnValues)
mockHarvesterData.exceptions = exceptions
dataRetrieve.addObserver(mockHarvesterData)
return dataRetrieve, mockHarvesterData
def testErrorInCall(self):
dataRetrieve = HarvesterDataRetrieve()
observer = CallTrace('observer')
observer.exceptions = dict(getError=Exception('Bad Bad Bad'))
dataRetrieve.addObserver(observer)
result = asString(dataRetrieve.handleRequest(arguments=dict(verb=['GetError'])))
header, body = result.split(CRLF*2,1)
self.assertEqual(okJson, header+CRLF*2)
self.assertEquals({'request': {'verb': 'GetError'}, 'error': {'message': "Exception('Bad Bad Bad',)", 'code': 'unknown'}}, JsonDict.loads(body))
self.assertEquals(['getError'], observer.calledMethodNames())
def testKnownCodeException(self):
dataRetrieve = HarvesterDataRetrieve()
observer = CallTrace('observer')
observer.exceptions = dict(getError=Exception('idDoesNotExist'))
dataRetrieve.addObserver(observer)
result = asString(dataRetrieve.handleRequest(arguments=dict(verb=['GetError'])))
header, body = result.split(CRLF*2,1)
self.assertEqual(okJson, header+CRLF*2)
self.assertEquals({'request': {'verb': 'GetError'}, 'error': {'message': 'The value of an argument (id or key) is unknown or illegal.', 'code': 'idDoesNotExist'}}, JsonDict.loads(body))
self.assertEquals(['getError'], observer.calledMethodNames())
def testGetUnexisting(self):
dataRetrieve = HarvesterDataRetrieve()
observer = CallTrace('observer')
observer.exceptions=dict(getUnexisting=DeclineMessage())
dataRetrieve.addObserver(observer)
result = asString(dataRetrieve.handleRequest(arguments=dict(verb=['GetUnexisting'], argument=['value'])))
header, body = result.split(CRLF*2,1)
self.assertEqual(okJson, header+CRLF*2)
self.assertEquals({'request': {'verb': 'GetUnexisting', 'argument': 'value'}, 'error': {'message': 'Value of the verb argument is not a legal verb, the verb argument is missing, or the verb argument is repeated.', 'code': 'badVerb'}}, JsonDict.loads(body))
self.assertEquals(['getUnexisting'], observer.calledMethodNames())
self.assertEquals({'argument': 'value'}, observer.calledMethods[0].kwargs)
def testKeepResumptionTokenOnFailingAddCall(self):
resumptionToken = "u|c1286437597991025|mprefix|s|f"
open(join(self.tempdir, 'harvester.state'), 'w').write("Resumptiontoken: %s\n" % resumptionToken)
observer = CallTrace()
observer.exceptions={'add': Exception("Could be anything")}
oaiDownloadProcessor = OaiDownloadProcessor(path="/oai", metadataPrefix="oai_dc", workingDirectory=self.tempdir, xWait=True, err=StringIO())
oaiDownloadProcessor.addObserver(observer)
self.assertEquals('GET /oai?%s HTTP/1.0\r\nX-Meresco-Oai-Client-Identifier: %s\r\nUser-Agent: Meresco-Oai-DownloadProcessor/5.x\r\n\r\n' % (urlencode([('verb', 'ListRecords'), ('resumptionToken', resumptionToken), ('x-wait', 'True')]), oaiDownloadProcessor._identifier), oaiDownloadProcessor.buildRequest())
self.assertRaises(Exception, lambda: list(compose(oaiDownloadProcessor.handle(parse(StringIO(LISTRECORDS_RESPONSE % RESUMPTION_TOKEN))))))
self.assertEquals(['startOaiBatch', 'add', 'stopOaiBatch'], [m.name for m in observer.calledMethods])
errorOutput = oaiDownloadProcessor._err.getvalue()
self.assertTrue(errorOutput.startswith('Traceback'), errorOutput)
self.assertTrue('Exception: Could be anything\nWhile processing:\n<record xmlns="http://www.openarchives.org/OAI/2.0/"><header><identifier>oai:identifier:1' in errorOutput, errorOutput)
self.assertEquals('GET /oai?%s HTTP/1.0\r\nX-Meresco-Oai-Client-Identifier: %s\r\nUser-Agent: Meresco-Oai-DownloadProcessor/5.x\r\n\r\n' % (urlencode([('verb', 'ListRecords'), ('resumptionToken', resumptionToken), ('x-wait', 'True')]), oaiDownloadProcessor._identifier), oaiDownloadProcessor.buildRequest())
def testHarvesterStateWithError(self):
resumptionToken = "u|c1286437597991025|mprefix|s|f"
open(join(self.tempdir, 'harvester.state'), 'w').write("Resumptiontoken: %s\n" % resumptionToken)
observer = CallTrace()
observer.exceptions={'add': Exception("Could be anything")}
oaiDownloadProcessor = OaiDownloadProcessor(path="/oai", metadataPrefix="oai_dc", workingDirectory=self.tempdir, xWait=True, err=StringIO(), name="Name")
oaiDownloadProcessor.addObserver(observer)
self.assertRaises(Exception, lambda: list(compose(oaiDownloadProcessor.handle(parse(StringIO(LISTRECORDS_RESPONSE % RESUMPTION_TOKEN))))))
state = oaiDownloadProcessor.getState()
self.assertEquals(resumptionToken, state.resumptionToken)
self.assertEquals(None, state.from_)
self.assertEquals("ERROR while processing 'oai:identifier:1': Could be anything", state.errorState)
self.assertEquals("Name", state.name)
oaiDownloadProcessor2 = OaiDownloadProcessor(path="/oai", metadataPrefix="oai_dc", workingDirectory=self.tempdir, xWait=True, err=StringIO())
state2 = oaiDownloadProcessor2.getState()
self.assertEquals(resumptionToken, state2.resumptionToken)
self.assertEquals("ERROR while processing 'oai:identifier:1': Could be anything", state2.errorState)
def testKeepResumptionTokenOnFailingAddCall(self):
resumptionToken = "u|c1286437597991025|mprefix|s|f"
with open(join(self.tempdir, 'harvester.state'), 'w') as f:
f.write("Resumptiontoken: %s\n" % resumptionToken)
observer = CallTrace()
observer.exceptions = {'add': Exception("Could be anything")}
oaiDownloadProcessor = OaiDownloadProcessor(
path="/oai",
metadataPrefix="oai_dc",
workingDirectory=self.tempdir,
xWait=True,
err=StringIO())
oaiDownloadProcessor.addObserver(observer)
self.assertEqual(
'GET /oai?%s HTTP/1.0\r\nX-Meresco-Oai-Client-Identifier: %s\r\nUser-Agent: Meresco-Oai-DownloadProcessor/5.x\r\n\r\n'
% (urlencode([('verb', 'ListRecords'),
('resumptionToken', resumptionToken),
('x-wait', 'True')
]), oaiDownloadProcessor._identifier),
oaiDownloadProcessor.buildRequest())
self.assertRaises(
Exception, lambda: list(
compose(
oaiDownloadProcessor.handle(
parse(StringIO(LISTRECORDS_RESPONSE % RESUMPTION_TOKEN)
)))))
self.assertEqual(['startOaiBatch', 'add', 'stopOaiBatch'],
[m.name for m in observer.calledMethods])
errorOutput = oaiDownloadProcessor._err.getvalue()
self.assertTrue(errorOutput.startswith('Traceback'), errorOutput)
self.assertTrue(
'Exception: Could be anything\nWhile processing:\n<record xmlns="http://www.openarchives.org/OAI/2.0/"><header><identifier>oai:identifier:1'
in errorOutput, errorOutput)
self.assertEqual(
'GET /oai?%s HTTP/1.0\r\nX-Meresco-Oai-Client-Identifier: %s\r\nUser-Agent: Meresco-Oai-DownloadProcessor/5.x\r\n\r\n'
% (urlencode([('verb', 'ListRecords'),
('resumptionToken', resumptionToken),
('x-wait', 'True')
]), oaiDownloadProcessor._identifier),
oaiDownloadProcessor.buildRequest())
def testHarvesterStateWithError(self):
resumptionToken = "u|c1286437597991025|mprefix|s|f"
with open(join(self.tempdir, 'harvester.state'), 'w') as f:
f.write("Resumptiontoken: %s\n" % resumptionToken)
observer = CallTrace()
observer.exceptions = {'add': Exception("Could be anything")}
oaiDownloadProcessor = OaiDownloadProcessor(
path="/oai",
metadataPrefix="oai_dc",
workingDirectory=self.tempdir,
xWait=True,
err=StringIO(),
name="Name")
oaiDownloadProcessor.addObserver(observer)
self.assertRaises(
Exception, lambda: list(
compose(
oaiDownloadProcessor.handle(
parse(StringIO(LISTRECORDS_RESPONSE % RESUMPTION_TOKEN)
)))))
state = oaiDownloadProcessor.getState()
self.assertEqual(resumptionToken, state.resumptionToken)
self.assertEqual(None, state.from_)
self.assertEqual(
"ERROR while processing 'oai:identifier:1': Could be anything",
state.errorState)
self.assertEqual("Name", state.name)
oaiDownloadProcessor2 = OaiDownloadProcessor(
path="/oai",
metadataPrefix="oai_dc",
workingDirectory=self.tempdir,
xWait=True,
err=StringIO())
state2 = oaiDownloadProcessor2.getState()
self.assertEqual(resumptionToken, state2.resumptionToken)
self.assertEqual(
"ERROR while processing 'oai:identifier:1': Could be anything",
state2.errorState)
