class SmoothActionTest(ActionTestCase): def setUp(self): ActionTestCase.setUp(self) self.repo = Repository('domainId', 'rep') self.uploader = CallTrace('Uploader') self.repo.createUploader = lambda logger: self.uploader self.stateDir = self.tmp_path / 'state' self.logDir = self.tmp_path / 'log' self.state = State(self.stateDir, self.logDir, 'rep') self.smoothaction = SmoothAction(self.repo, self.state, NilEventLogger()) self.idfilename = join(self.stateDir, 'rep.ids') self.invalidIdsFilename = join(self.stateDir, 'rep_invalid.ids') self.old_idfilename = join(self.stateDir, 'rep.ids.old') self.statsfilename = join(self.stateDir,'rep.stats') def testSmooth_Init(self): writefile(self.idfilename, 'rep:id:1\nrep:id:2\n') writefile(self.invalidIdsFilename, 'rep:id:3\n') writefile(self.statsfilename, 'Started: 2005-12-22 16:33:39, Harvested/Uploaded/Deleted/Total: 10/10/0/2, Done: ResumptionToken:\n') self.assertFalse(os.path.isfile(self.old_idfilename)) done,message, hasResumptionToken = self.smoothaction.do() self.assertTrue(os.path.isfile(self.old_idfilename)) self.assertFalse(os.path.isfile(self.idfilename)) self.assertEqual('rep:id:1\nrep:id:2\nrep:id:3\n', readfile(self.old_idfilename)) self.assertTrue('Done: Deleted all ids' in readfile(self.statsfilename), readfile(self.statsfilename)) self.assertEqual('Smooth reharvest: initialized.', message) self.assertFalse(done) def testSmooth_InitWithNothingHarvestedYetRepository(self): self.assertFalse(os.path.isfile(self.idfilename)) self.assertFalse(os.path.isfile(self.invalidIdsFilename)) self.assertFalse(os.path.isfile(self.old_idfilename)) done,message, hasResumptionToken = self.smoothaction.do() self.assertFalse(os.path.isfile(self.old_idfilename)) self.assertFalse(os.path.isfile(self.idfilename)) self.assertFalse(os.path.isfile(self.invalidIdsFilename)) self.assertTrue('Done: Deleted all ids' in readfile(self.statsfilename)) self.assertEqual('Smooth reharvest: initialized.', message) self.assertFalse(done) def testSmooth_Harvest(self): writefile(self.old_idfilename, 'rep:id:1\nrep:id:2\n') writefile(self.idfilename, '') writefile(self.statsfilename, 'Started: 2005-12-22 16:33:39, Harvested/Uploaded/Deleted/Total: 10/10/0/2, Done: ResumptionToken:\n'+ 'Started: 2005-12-28 10:10:10, Harvested/Uploaded/Deleted/Total: 0/0/0/0, Done: Deleted all ids. \n') self.smoothaction._harvest = lambda:(HARVESTED, False) done,message,hasResumptionToken = self.smoothaction.do() self.assertEqual('Smooth reharvest: Harvested.', message) self.assertFalse(done) def testSmooth_HarvestAgain(self): writefile(self.old_idfilename, 'rep:id:1\nrep:id:2\n') writefile(self.idfilename, 'rep:id:41\nrep:id:2\n') writefile(self.statsfilename, 'Started: 2005-12-22 16:33:39, Harvested/Uploaded/Deleted/Total: 10/10/0/2, Done: ResumptionToken:\n'+ 'Started: 2005-12-28 10:10:10, Harvested/Uploaded/Deleted/Total: 2/2/0/2, Done: ResumptionToken:yes \n') self.smoothaction._harvest = lambda:(HARVESTED, False) done, message, hasResumptionToken = self.smoothaction.do() self.assertEqual('Smooth reharvest: Harvested.', message) self.assertFalse(done) def testSmooth_NothingToDo(self): writefile(self.old_idfilename, 'rep:id:1\nrep:id:2\n') writefile(self.idfilename, 'rep:id:41\nrep:id:2\n') writefile(self.statsfilename, 'Started: 2005-12-22 16:33:39, Harvested/Uploaded/Deleted/Total: 10/10/0/2, Done: ResumptionToken:\n'+ 'Started: 2005-12-28 10:10:10, Harvested/Uploaded/Deleted/Total: 2/2/0/2, Done: ResumptionToken:None \n') self.smoothaction._harvest = lambda:(NOTHING_TO_DO, False) self.smoothaction._finish = lambda:DONE done, message, hasResumptionToken = self.smoothaction.do() self.assertEqual('Smooth reharvest: ' + DONE, message) self.assertTrue(done) def mockdelete(self, filename): self.mockdelete_filename = filename self.mockdelete_ids = readIds(filename) def testHarvest(self): harvester = CallTrace('harvester') self.smoothaction._createHarvester = lambda: ([], harvester) harvester.returnValues['harvest'] = ('result', True) result = self.smoothaction._harvest() self.assertEqual(['harvest'], [m.name for m in harvester.calledMethods]) self.assertEqual(('result', True), result) def testResetState_WithoutPreviousCleanState(self): self.writeLogLine(2010, 3, 1, token='resumptionToken') self.writeLogLine(2010, 3, 2, token='resumptionToken') self.writeLogLine(2010, 3, 3, exception='Exception') action = self.newSmoothAction() action.resetState() self.assertEqual((None, None), (self.state.from_, self.state.token)) self.assertEqual(1, self.state.eventCounts()['deleted']) def testResetState_ToPreviousCleanState(self): self.writeLogLine(2010, 3, 2, token='') self.writeMarkDeleted(2010, 3, 3) self.writeLogLine(2010, 3, 4, token='resumptionToken') self.writeLogLine(2010, 3, 5, token='resumptionToken') self.writeLogLine(2010, 3, 6, exception='Exception') action = self.newSmoothAction() action.resetState() self.assertEqual((None, None), (self.state.from_, self.state.token)) def testResetState_ToStartAllOver(self): self.writeLogLine(2010, 3, 3, token='resumptionToken') self.writeLogLine(2010, 3, 4, exception='Exception') action = self.newSmoothAction() action.resetState() self.assertEqual((None, None), (self.state.from_, self.state.token)) def newSmoothAction(self): action = SmoothAction(self.repository, state=self.state, generalHarvestLog=NilEventLogger()) action._harvest = lambda:None return action
class SmoothActionTest(ActionTestCase): def setUp(self): ActionTestCase.setUp(self) self.repo = Repository('domainId', 'rep') self.uploader = CallTrace('Uploader') self.repo.createUploader = lambda logger: self.uploader self.stateDir = self.tempdir self.logDir = self.tempdir self.smoothaction = SmoothAction(self.repo, self.stateDir, self.logDir, NilEventLogger()) self.idfilename = join(self.stateDir, 'rep.ids') self.invalidIdsFilename = join(self.stateDir, 'rep_invalid.ids') self.old_idfilename = join(self.stateDir, 'rep.ids.old') self.statsfilename = join(self.stateDir,'rep.stats') def testSmooth_Init(self): writefile(self.idfilename, 'rep:id:1\nrep:id:2\n') writefile(self.invalidIdsFilename, 'rep:id:3\n') writefile(self.statsfilename, 'Started: 2005-12-22 16:33:39, Harvested/Uploaded/Deleted/Total: 10/10/0/2, Done: ResumptionToken:\n') self.assertFalse(os.path.isfile(self.old_idfilename)) done,message, hasResumptionToken = self.smoothaction.do() self.assertTrue(os.path.isfile(self.old_idfilename)) self.assertTrue(os.path.isfile(self.idfilename)) self.assertEquals('rep:id:1\nrep:id:2\nrep:id:3\n', readfile(self.old_idfilename)) self.assertEquals('', readfile(self.idfilename)) self.assertTrue('Done: Deleted all ids' in readfile(self.statsfilename), readfile(self.statsfilename)) self.assertEquals('Smooth reharvest: initialized.', message) self.assertFalse(done) def testSmooth_InitWithNothingHarvestedYetRepository(self): self.assertFalse(os.path.isfile(self.idfilename)) self.assertFalse(os.path.isfile(self.invalidIdsFilename)) self.assertFalse(os.path.isfile(self.old_idfilename)) self.assertFalse(os.path.isfile(self.statsfilename)) done,message, hasResumptionToken = self.smoothaction.do() self.assertTrue(os.path.isfile(self.old_idfilename)) self.assertTrue(os.path.isfile(self.idfilename)) self.assertTrue(os.path.isfile(self.invalidIdsFilename)) self.assertEquals('', readfile(self.old_idfilename)) self.assertEquals('', readfile(self.idfilename)) self.assertEquals('', readfile(self.invalidIdsFilename)) self.assertTrue('Done: Deleted all ids' in readfile(self.statsfilename)) self.assertEquals('Smooth reharvest: initialized.', message) self.assertFalse(done) def testSmooth_Harvest(self): writefile(self.old_idfilename, 'rep:id:1\nrep:id:2\n') writefile(self.idfilename, '') writefile(self.statsfilename, 'Started: 2005-12-22 16:33:39, Harvested/Uploaded/Deleted/Total: 10/10/0/2, Done: ResumptionToken:\n'+ 'Started: 2005-12-28 10:10:10, Harvested/Uploaded/Deleted/Total: 0/0/0/0, Done: Deleted all ids. \n') self.smoothaction._harvest = lambda:(HARVESTED, False) done,message,hasResumptionToken = self.smoothaction.do() self.assertEquals('Smooth reharvest: Harvested.', message) self.assertFalse(done) def testSmooth_HarvestAgain(self): writefile(self.old_idfilename, 'rep:id:1\nrep:id:2\n') writefile(self.idfilename, 'rep:id:41\nrep:id:2\n') writefile(self.statsfilename, 'Started: 2005-12-22 16:33:39, Harvested/Uploaded/Deleted/Total: 10/10/0/2, Done: ResumptionToken:\n'+ 'Started: 2005-12-28 10:10:10, Harvested/Uploaded/Deleted/Total: 2/2/0/2, Done: ResumptionToken:yes \n') self.smoothaction._harvest = lambda:(HARVESTED, False) done, message, hasResumptionToken = self.smoothaction.do() self.assertEquals('Smooth reharvest: Harvested.', message) self.assertFalse(done) def testSmooth_NothingToDo(self): writefile(self.old_idfilename, 'rep:id:1\nrep:id:2\n') writefile(self.idfilename, 'rep:id:41\nrep:id:2\n') writefile(self.statsfilename, 'Started: 2005-12-22 16:33:39, Harvested/Uploaded/Deleted/Total: 10/10/0/2, Done: ResumptionToken:\n'+ 'Started: 2005-12-28 10:10:10, Harvested/Uploaded/Deleted/Total: 2/2/0/2, Done: ResumptionToken:None \n') self.smoothaction._harvest = lambda:(NOTHING_TO_DO, False) self.smoothaction._finish = lambda:DONE done, message, hasResumptionToken = self.smoothaction.do() self.assertEquals('Smooth reharvest: ' + DONE, message) self.assertTrue(done) def mockdelete(self, filename): self.mockdelete_filename = filename self.mockdelete_ids = readIds(filename) def testHarvest(self): harvester = CallTrace('harvester') self.smoothaction._createHarvester = lambda: harvester harvester.returnValues['harvest'] = ('result', True) result = self.smoothaction._harvest() self.assertEquals(['harvest'], [m.name for m in harvester.calledMethods]) self.assertEquals(('result', True), result) def testResetState_WithoutPreviousCleanState(self): self.writeLogLine(2010, 3, 1, token='resumptionToken') self.writeLogLine(2010, 3, 2, token='resumptionToken') self.writeLogLine(2010, 3, 3, exception='Exception') action = self.newSmoothAction() action.resetState() h = self.newHarvesterLog() self.assertEquals((None, None), (h._state.from_, h._state.token)) def testResetState_ToPreviousCleanState(self): self.writeLogLine(2010, 3, 2, token='') self.writeMarkDeleted(2010, 3, 3) self.writeLogLine(2010, 3, 4, token='resumptionToken') self.writeLogLine(2010, 3, 5, token='resumptionToken') self.writeLogLine(2010, 3, 6, exception='Exception') action = self.newSmoothAction() action.resetState() h = self.newHarvesterLog() self.assertEquals((None, None), (h._state.from_, h._state.token)) def testResetState_ToStartAllOver(self): self.writeLogLine(2010, 3, 3, token='resumptionToken') self.writeLogLine(2010, 3, 4, exception='Exception') action = self.newSmoothAction() action.resetState() h = self.newHarvesterLog() self.assertEquals((None, None), (h._state.from_, h._state.token)) def newSmoothAction(self): action = SmoothAction(self.repository, stateDir=self.tempdir, logDir=self.tempdir, generalHarvestLog=NilEventLogger()) action._harvest = lambda:None return action
def newSmoothAction(self): action = SmoothAction(self.repository, state=self.state, generalHarvestLog=NilEventLogger()) action._harvest = lambda:None return action
def newSmoothAction(self): action = SmoothAction(self.repository, stateDir=self.tempdir, logDir=self.tempdir, generalHarvestLog=NilEventLogger()) action._harvest = lambda:None return action