def testIncrementalHarvestWithFromAfterSomePeriod(self):
observer = CallTrace(emptyGeneratorMethods=['add'])
oaiDownloadProcessor = OaiDownloadProcessor(
path="/oai",
metadataPrefix="oai_dc",
workingDirectory=self.tempdir,
xWait=False,
err=StringIO(),
incrementalHarvestSchedule=Schedule(period=10))
oaiDownloadProcessor._time = lambda: 1.0
oaiDownloadProcessor.addObserver(observer)
consume(
oaiDownloadProcessor.handle(parse(StringIO(LISTRECORDS_RESPONSE))))
self.assertEqual(None, oaiDownloadProcessor._resumptionToken)
self.assertEqual('2002-06-01T19:20:30Z', oaiDownloadProcessor._from)
self.assertEqual(None, oaiDownloadProcessor.buildRequest())
oaiDownloadProcessor._time = lambda: 6.0
self.assertEqual(None, oaiDownloadProcessor.buildRequest())
oaiDownloadProcessor._time = lambda: 10.0
self.assertEqual(None, oaiDownloadProcessor.buildRequest())
oaiDownloadProcessor._time = lambda: 11.1
request = oaiDownloadProcessor.buildRequest()
self.assertTrue(
request.startswith(
'GET /oai?verb=ListRecords&from=2002-06-01T19%3A20%3A30Z&metadataPrefix=oai_dc'
), request)
def testIncrementalHarvestScheduleNoneOverruledWithSetIncrementalHarvestSchedule(
self):
oaiDownloadProcessor = OaiDownloadProcessor(
path="/oai",
metadataPrefix="oai_dc",
workingDirectory=self.tempdir,
xWait=False,
err=StringIO(),
incrementalHarvestSchedule=None)
oaiDownloadProcessor._time = lambda: 10
consume(
oaiDownloadProcessor.handle(
parse(StringIO(LISTRECORDS_RESPONSE % ''))))
self.assertEqual(None, oaiDownloadProcessor._resumptionToken)
self.assertEqual('2002-06-01T19:20:30Z', oaiDownloadProcessor._from)
self.assertEqual(None, oaiDownloadProcessor._earliestNextRequestTime)
oaiDownloadProcessor.setIncrementalHarvestSchedule(schedule=Schedule(
period=3))
self.assertEqual(None, oaiDownloadProcessor.buildRequest())
self.assertEqual(None, oaiDownloadProcessor._earliestNextRequestTime)
oaiDownloadProcessor.scheduleNextRequest()
self.assertNotEqual(None, oaiDownloadProcessor.buildRequest())
self.assertEqual(0, oaiDownloadProcessor._earliestNextRequestTime)
consume(
oaiDownloadProcessor.handle(
parse(StringIO(LISTRECORDS_RESPONSE % ''))))
self.assertEqual(13, oaiDownloadProcessor._earliestNextRequestTime)
def testSetIncrementalHarvestSchedule(self):
oaiDownloadProcessor = OaiDownloadProcessor(
path="/oai",
metadataPrefix="oai_dc",
workingDirectory=self.tempdir,
xWait=False,
err=StringIO(),
incrementalHarvestSchedule=None)
oaiDownloadProcessor._time = lambda: 10
oaiDownloadProcessor.setIncrementalHarvestSchedule(schedule=Schedule(
period=3))
self.assertEqual(0, oaiDownloadProcessor._earliestNextRequestTime)
consume(
oaiDownloadProcessor.handle(
parse(StringIO(LISTRECORDS_RESPONSE % ''))))
self.assertEqual(13, oaiDownloadProcessor._earliestNextRequestTime)
def testScheduleNextRequest(self):
oaiDownloadProcessor = OaiDownloadProcessor(
path='/p', metadataPrefix='p', workingDirectory=self.tempdir)
oaiDownloadProcessor._time = lambda: 17
consume(
oaiDownloadProcessor.handle(
parse(StringIO(LISTRECORDS_RESPONSE % ''))))
self.assertTrue(oaiDownloadProcessor._earliestNextRequestTime > 17)
oaiDownloadProcessor.scheduleNextRequest()
self.assertEqual(0, oaiDownloadProcessor._earliestNextRequestTime)
self.assertEqual(True, oaiDownloadProcessor._timeForNextRequest())
self.assertNotEqual(None, oaiDownloadProcessor.buildRequest())
oaiDownloadProcessor.scheduleNextRequest(Schedule(period=0))
self.assertEqual(17, oaiDownloadProcessor._earliestNextRequestTime)
self.assertEqual(True, oaiDownloadProcessor._timeForNextRequest())
self.assertNotEqual(None, oaiDownloadProcessor.buildRequest())
oaiDownloadProcessor.scheduleNextRequest(Schedule(period=120))
self.assertEqual(137, oaiDownloadProcessor._earliestNextRequestTime)
self.assertEqual(False, oaiDownloadProcessor._timeForNextRequest())
self.assertEqual(None, oaiDownloadProcessor.buildRequest())