def setUp(self):
ActionTestCase.setUp(self)
self.repo = Repository('domainId', 'rep')
self.uploader = CallTrace('Uploader')
self.repo.createUploader = lambda logger: self.uploader
self.stateDir = self.tmp_path / 'state'
self.logDir = self.tmp_path / 'log'
self.state = State(self.stateDir, self.logDir, 'rep')
self.smoothaction = SmoothAction(self.repo, self.state, NilEventLogger())
self.idfilename = join(self.stateDir, 'rep.ids')
self.invalidIdsFilename = join(self.stateDir, 'rep_invalid.ids')
self.old_idfilename = join(self.stateDir, 'rep.ids.old')
self.statsfilename = join(self.stateDir,'rep.stats')
def setUp(self):
ActionTestCase.setUp(self)
self.repo = Repository('domainId', 'rep')
self.uploader = CallTrace('Uploader')
self.repo.createUploader = lambda logger: self.uploader
self.stateDir = self.tempdir
self.logDir = self.tempdir
self.smoothaction = SmoothAction(self.repo, self.stateDir, self.logDir, NilEventLogger())
self.idfilename = join(self.stateDir, 'rep.ids')
self.invalidIdsFilename = join(self.stateDir, 'rep_invalid.ids')
self.old_idfilename = join(self.stateDir, 'rep.ids.old')
self.statsfilename = join(self.stateDir,'rep.stats')
class SmoothActionTest(ActionTestCase):
def setUp(self):
ActionTestCase.setUp(self)
self.repo = Repository('domainId', 'rep')
self.uploader = CallTrace('Uploader')
self.repo.createUploader = lambda logger: self.uploader
self.stateDir = self.tmp_path / 'state'
self.logDir = self.tmp_path / 'log'
self.state = State(self.stateDir, self.logDir, 'rep')
self.smoothaction = SmoothAction(self.repo, self.state, NilEventLogger())
self.idfilename = join(self.stateDir, 'rep.ids')
self.invalidIdsFilename = join(self.stateDir, 'rep_invalid.ids')
self.old_idfilename = join(self.stateDir, 'rep.ids.old')
self.statsfilename = join(self.stateDir,'rep.stats')
def testSmooth_Init(self):
writefile(self.idfilename, 'rep:id:1\nrep:id:2\n')
writefile(self.invalidIdsFilename, 'rep:id:3\n')
writefile(self.statsfilename, 'Started: 2005-12-22 16:33:39, Harvested/Uploaded/Deleted/Total: 10/10/0/2, Done: ResumptionToken:\n')
self.assertFalse(os.path.isfile(self.old_idfilename))
done,message, hasResumptionToken = self.smoothaction.do()
self.assertTrue(os.path.isfile(self.old_idfilename))
self.assertFalse(os.path.isfile(self.idfilename))
self.assertEqual('rep:id:1\nrep:id:2\nrep:id:3\n', readfile(self.old_idfilename))
self.assertTrue('Done: Deleted all ids' in readfile(self.statsfilename), readfile(self.statsfilename))
self.assertEqual('Smooth reharvest: initialized.', message)
self.assertFalse(done)
def testSmooth_InitWithNothingHarvestedYetRepository(self):
self.assertFalse(os.path.isfile(self.idfilename))
self.assertFalse(os.path.isfile(self.invalidIdsFilename))
self.assertFalse(os.path.isfile(self.old_idfilename))
done,message, hasResumptionToken = self.smoothaction.do()
self.assertFalse(os.path.isfile(self.old_idfilename))
self.assertFalse(os.path.isfile(self.idfilename))
self.assertFalse(os.path.isfile(self.invalidIdsFilename))
self.assertTrue('Done: Deleted all ids' in readfile(self.statsfilename))
self.assertEqual('Smooth reharvest: initialized.', message)
self.assertFalse(done)
def testSmooth_Harvest(self):
writefile(self.old_idfilename, 'rep:id:1\nrep:id:2\n')
writefile(self.idfilename, '')
writefile(self.statsfilename, 'Started: 2005-12-22 16:33:39, Harvested/Uploaded/Deleted/Total: 10/10/0/2, Done: ResumptionToken:\n'+
'Started: 2005-12-28 10:10:10, Harvested/Uploaded/Deleted/Total: 0/0/0/0, Done: Deleted all ids. \n')
self.smoothaction._harvest = lambda:(HARVESTED, False)
done,message,hasResumptionToken = self.smoothaction.do()
self.assertEqual('Smooth reharvest: Harvested.', message)
self.assertFalse(done)
def testSmooth_HarvestAgain(self):
writefile(self.old_idfilename, 'rep:id:1\nrep:id:2\n')
writefile(self.idfilename, 'rep:id:41\nrep:id:2\n')
writefile(self.statsfilename, 'Started: 2005-12-22 16:33:39, Harvested/Uploaded/Deleted/Total: 10/10/0/2, Done: ResumptionToken:\n'+
'Started: 2005-12-28 10:10:10, Harvested/Uploaded/Deleted/Total: 2/2/0/2, Done: ResumptionToken:yes \n')
self.smoothaction._harvest = lambda:(HARVESTED, False)
done, message, hasResumptionToken = self.smoothaction.do()
self.assertEqual('Smooth reharvest: Harvested.', message)
self.assertFalse(done)
def testSmooth_NothingToDo(self):
writefile(self.old_idfilename, 'rep:id:1\nrep:id:2\n')
writefile(self.idfilename, 'rep:id:41\nrep:id:2\n')
writefile(self.statsfilename, 'Started: 2005-12-22 16:33:39, Harvested/Uploaded/Deleted/Total: 10/10/0/2, Done: ResumptionToken:\n'+
'Started: 2005-12-28 10:10:10, Harvested/Uploaded/Deleted/Total: 2/2/0/2, Done: ResumptionToken:None \n')
self.smoothaction._harvest = lambda:(NOTHING_TO_DO, False)
self.smoothaction._finish = lambda:DONE
done, message, hasResumptionToken = self.smoothaction.do()
self.assertEqual('Smooth reharvest: ' + DONE, message)
self.assertTrue(done)
def mockdelete(self, filename):
self.mockdelete_filename = filename
self.mockdelete_ids = readIds(filename)
def testHarvest(self):
harvester = CallTrace('harvester')
self.smoothaction._createHarvester = lambda: ([], harvester)
harvester.returnValues['harvest'] = ('result', True)
result = self.smoothaction._harvest()
self.assertEqual(['harvest'], [m.name for m in harvester.calledMethods])
self.assertEqual(('result', True), result)
def testResetState_WithoutPreviousCleanState(self):
self.writeLogLine(2010, 3, 1, token='resumptionToken')
self.writeLogLine(2010, 3, 2, token='resumptionToken')
self.writeLogLine(2010, 3, 3, exception='Exception')
action = self.newSmoothAction()
action.resetState()
self.assertEqual((None, None), (self.state.from_, self.state.token))
self.assertEqual(1, self.state.eventCounts()['deleted'])
def testResetState_ToPreviousCleanState(self):
self.writeLogLine(2010, 3, 2, token='')
self.writeMarkDeleted(2010, 3, 3)
self.writeLogLine(2010, 3, 4, token='resumptionToken')
self.writeLogLine(2010, 3, 5, token='resumptionToken')
self.writeLogLine(2010, 3, 6, exception='Exception')
action = self.newSmoothAction()
action.resetState()
self.assertEqual((None, None), (self.state.from_, self.state.token))
def testResetState_ToStartAllOver(self):
self.writeLogLine(2010, 3, 3, token='resumptionToken')
self.writeLogLine(2010, 3, 4, exception='Exception')
action = self.newSmoothAction()
action.resetState()
self.assertEqual((None, None), (self.state.from_, self.state.token))
def newSmoothAction(self):
action = SmoothAction(self.repository, state=self.state, generalHarvestLog=NilEventLogger())
action._harvest = lambda:None
return action
def newSmoothAction(self):
action = SmoothAction(self.repository, state=self.state, generalHarvestLog=NilEventLogger())
action._harvest = lambda:None
return action
class SmoothActionTest(ActionTestCase):
def setUp(self):
ActionTestCase.setUp(self)
self.repo = Repository('domainId', 'rep')
self.uploader = CallTrace('Uploader')
self.repo.createUploader = lambda logger: self.uploader
self.stateDir = self.tempdir
self.logDir = self.tempdir
self.smoothaction = SmoothAction(self.repo, self.stateDir, self.logDir, NilEventLogger())
self.idfilename = join(self.stateDir, 'rep.ids')
self.invalidIdsFilename = join(self.stateDir, 'rep_invalid.ids')
self.old_idfilename = join(self.stateDir, 'rep.ids.old')
self.statsfilename = join(self.stateDir,'rep.stats')
def testSmooth_Init(self):
writefile(self.idfilename, 'rep:id:1\nrep:id:2\n')
writefile(self.invalidIdsFilename, 'rep:id:3\n')
writefile(self.statsfilename, 'Started: 2005-12-22 16:33:39, Harvested/Uploaded/Deleted/Total: 10/10/0/2, Done: ResumptionToken:\n')
self.assertFalse(os.path.isfile(self.old_idfilename))
done,message, hasResumptionToken = self.smoothaction.do()
self.assertTrue(os.path.isfile(self.old_idfilename))
self.assertTrue(os.path.isfile(self.idfilename))
self.assertEquals('rep:id:1\nrep:id:2\nrep:id:3\n', readfile(self.old_idfilename))
self.assertEquals('', readfile(self.idfilename))
self.assertTrue('Done: Deleted all ids' in readfile(self.statsfilename), readfile(self.statsfilename))
self.assertEquals('Smooth reharvest: initialized.', message)
self.assertFalse(done)
def testSmooth_InitWithNothingHarvestedYetRepository(self):
self.assertFalse(os.path.isfile(self.idfilename))
self.assertFalse(os.path.isfile(self.invalidIdsFilename))
self.assertFalse(os.path.isfile(self.old_idfilename))
self.assertFalse(os.path.isfile(self.statsfilename))
done,message, hasResumptionToken = self.smoothaction.do()
self.assertTrue(os.path.isfile(self.old_idfilename))
self.assertTrue(os.path.isfile(self.idfilename))
self.assertTrue(os.path.isfile(self.invalidIdsFilename))
self.assertEquals('', readfile(self.old_idfilename))
self.assertEquals('', readfile(self.idfilename))
self.assertEquals('', readfile(self.invalidIdsFilename))
self.assertTrue('Done: Deleted all ids' in readfile(self.statsfilename))
self.assertEquals('Smooth reharvest: initialized.', message)
self.assertFalse(done)
def testSmooth_Harvest(self):
writefile(self.old_idfilename, 'rep:id:1\nrep:id:2\n')
writefile(self.idfilename, '')
writefile(self.statsfilename, 'Started: 2005-12-22 16:33:39, Harvested/Uploaded/Deleted/Total: 10/10/0/2, Done: ResumptionToken:\n'+
'Started: 2005-12-28 10:10:10, Harvested/Uploaded/Deleted/Total: 0/0/0/0, Done: Deleted all ids. \n')
self.smoothaction._harvest = lambda:(HARVESTED, False)
done,message,hasResumptionToken = self.smoothaction.do()
self.assertEquals('Smooth reharvest: Harvested.', message)
self.assertFalse(done)
def testSmooth_HarvestAgain(self):
writefile(self.old_idfilename, 'rep:id:1\nrep:id:2\n')
writefile(self.idfilename, 'rep:id:41\nrep:id:2\n')
writefile(self.statsfilename, 'Started: 2005-12-22 16:33:39, Harvested/Uploaded/Deleted/Total: 10/10/0/2, Done: ResumptionToken:\n'+
'Started: 2005-12-28 10:10:10, Harvested/Uploaded/Deleted/Total: 2/2/0/2, Done: ResumptionToken:yes \n')
self.smoothaction._harvest = lambda:(HARVESTED, False)
done, message, hasResumptionToken = self.smoothaction.do()
self.assertEquals('Smooth reharvest: Harvested.', message)
self.assertFalse(done)
def testSmooth_NothingToDo(self):
writefile(self.old_idfilename, 'rep:id:1\nrep:id:2\n')
writefile(self.idfilename, 'rep:id:41\nrep:id:2\n')
writefile(self.statsfilename, 'Started: 2005-12-22 16:33:39, Harvested/Uploaded/Deleted/Total: 10/10/0/2, Done: ResumptionToken:\n'+
'Started: 2005-12-28 10:10:10, Harvested/Uploaded/Deleted/Total: 2/2/0/2, Done: ResumptionToken:None \n')
self.smoothaction._harvest = lambda:(NOTHING_TO_DO, False)
self.smoothaction._finish = lambda:DONE
done, message, hasResumptionToken = self.smoothaction.do()
self.assertEquals('Smooth reharvest: ' + DONE, message)
self.assertTrue(done)
def mockdelete(self, filename):
self.mockdelete_filename = filename
self.mockdelete_ids = readIds(filename)
def testHarvest(self):
harvester = CallTrace('harvester')
self.smoothaction._createHarvester = lambda: harvester
harvester.returnValues['harvest'] = ('result', True)
result = self.smoothaction._harvest()
self.assertEquals(['harvest'], [m.name for m in harvester.calledMethods])
self.assertEquals(('result', True), result)
def testResetState_WithoutPreviousCleanState(self):
self.writeLogLine(2010, 3, 1, token='resumptionToken')
self.writeLogLine(2010, 3, 2, token='resumptionToken')
self.writeLogLine(2010, 3, 3, exception='Exception')
action = self.newSmoothAction()
action.resetState()
h = self.newHarvesterLog()
self.assertEquals((None, None), (h._state.from_, h._state.token))
def testResetState_ToPreviousCleanState(self):
self.writeLogLine(2010, 3, 2, token='')
self.writeMarkDeleted(2010, 3, 3)
self.writeLogLine(2010, 3, 4, token='resumptionToken')
self.writeLogLine(2010, 3, 5, token='resumptionToken')
self.writeLogLine(2010, 3, 6, exception='Exception')
action = self.newSmoothAction()
action.resetState()
h = self.newHarvesterLog()
self.assertEquals((None, None), (h._state.from_, h._state.token))
def testResetState_ToStartAllOver(self):
self.writeLogLine(2010, 3, 3, token='resumptionToken')
self.writeLogLine(2010, 3, 4, exception='Exception')
action = self.newSmoothAction()
action.resetState()
h = self.newHarvesterLog()
self.assertEquals((None, None), (h._state.from_, h._state.token))
def newSmoothAction(self):
action = SmoothAction(self.repository, stateDir=self.tempdir, logDir=self.tempdir, generalHarvestLog=NilEventLogger())
action._harvest = lambda:None
return action
def newSmoothAction(self):
action = SmoothAction(self.repository, stateDir=self.tempdir, logDir=self.tempdir, generalHarvestLog=NilEventLogger())
action._harvest = lambda:None
return action
