class FixedDelayTest(unittest.TestCase): def setUp(self): """ _setUp_ Create two subscriptions: One that contains a single file and one that contains multiple files. """ self.multipleFileFileset = Fileset(name="TestFileset1") for i in range(10): newFile = File(makeUUID(), size=1000, events=100) newFile.addRun(Run(i, *[45 + i])) self.multipleFileFileset.addFile(newFile) self.singleFileFileset = Fileset(name="TestFileset2") newFile = File("/some/file/name", size=1000, events=100) newFile.addRun(Run(1, *[45])) self.singleFileFileset.addFile(newFile) self.multipleFileLumiset = Fileset(name="TestFileset3") for i in range(10): newFile = File(makeUUID(), size=1000, events=100) newFile.addRun(Run(1, *[45 + i / 3])) self.multipleFileLumiset.addFile(newFile) self.singleLumiFileset = Fileset(name="TestFileset4") for i in range(10): newFile = File(makeUUID(), size=1000, events=100) newFile.addRun(Run(1, *[45])) self.singleLumiFileset.addFile(newFile) testWorkflow = Workflow() self.multipleFileSubscription = Subscription( fileset=self.multipleFileFileset, workflow=testWorkflow, split_algo="FixedDelay", type="Processing") self.singleFileSubscription = Subscription( fileset=self.singleFileFileset, workflow=testWorkflow, split_algo="FixedDelay", type="Processing") self.multipleLumiSubscription = Subscription( fileset=self.multipleFileLumiset, workflow=testWorkflow, split_algo="FixedDelay", type="Processing") self.singleLumiSubscription = Subscription( fileset=self.singleLumiFileset, workflow=testWorkflow, split_algo="FixedDelay", type="Processing") return def tearDown(self): """ _tearDown_ Nothing to do... """ pass def testNone(self): """ _testNone_ Since the time hasn'tpassed, we shouldn't get any jobs back. """ splitter = SplitterFactory() jobFactory = splitter(self.singleFileSubscription) jobGroups = jobFactory(trigger_time=int(time.time()) * 2) self.assertEquals(jobGroups, [], "Should have returned a null set") jobFactory = splitter(self.multipleFileSubscription) jobGroups = jobFactory(trigger_time=int(time.time()) * 2) self.assertEquals(jobGroups, [], "Should have returned a null set") jobFactory = splitter(self.multipleLumiSubscription) jobGroups = jobFactory(trigger_time=int(time.time()) * 2) self.assertEquals(jobGroups, [], "Should have returned a null set") jobFactory = splitter(self.singleLumiSubscription) jobGroups = jobFactory(trigger_time=int(time.time()) * 2) self.assertEquals(jobGroups, [], "Should have returned a null set") return def testClosed(self): """ _testClosed_ since the subscriptions are closed and none of the files ahve been acquired, all of the files should show up """ splitter = SplitterFactory() self.singleFileSubscription.getFileset().markOpen(False) jobFactory = splitter(self.singleFileSubscription) jobGroups = jobFactory(trigger_time=1) assert len(jobGroups) == 1, \ "ERROR: JobFactory didn't return one JobGroup." assert len(jobGroups[0].jobs) == 1, \ "ERROR: JobFactory didn't create a single job." job = jobGroups[0].jobs.pop() assert job.getFiles(type = "lfn") == ["/some/file/name"], \ "ERROR: Job contains unknown files." self.multipleFileSubscription.getFileset().markOpen(False) jobFactory = splitter(self.multipleFileSubscription) jobGroups = jobFactory(trigger_time=1) self.assertEquals(len(jobGroups), 1) self.assertEquals(len(jobGroups[0].jobs), 1) myfiles = jobGroups[0].jobs[0].getFiles() self.assertEquals(len(myfiles), 10) self.multipleLumiSubscription.getFileset().markOpen(False) jobFactory = splitter(self.multipleLumiSubscription) jobGroups = jobFactory(trigger_time=1) self.assertEquals(len(jobGroups), 1) self.assertEquals(len(jobGroups[0].jobs), 1) myfiles = jobGroups[0].jobs[0].getFiles() self.assertEquals(len(myfiles), 10) #self.assertEquals(jobGroups, [], "Should have returned a null set") self.singleLumiSubscription.getFileset().markOpen(False) jobFactory = splitter(self.singleLumiSubscription) jobGroups = jobFactory(trigger_time=1) assert len(jobGroups) == 1, \ "ERROR: JobFactory didn't return one JobGroup." assert len(jobGroups[0].jobs) == 1, \ "ERROR: JobFactory didn't create a single job." myfiles = jobGroups[0].jobs[0].getFiles() self.assertEquals(len(myfiles), 10) def testAllAcquired(self): """ _testAllAcquired_ should all return no job groups """ splitter = SplitterFactory() self.singleFileSubscription.acquireFiles( self.singleFileSubscription.availableFiles()) jobFactory = splitter(self.singleFileSubscription) jobGroups = jobFactory(trigger_time=1) self.assertEquals(jobGroups, [], "Should have returned a null set: %s" % jobGroups) self.multipleFileSubscription.acquireFiles( self.multipleFileSubscription.availableFiles()) jobFactory = splitter(self.multipleFileSubscription) jobGroups = jobFactory(trigger_time=1) self.assertEquals(jobGroups, [], "Should have returned a null set") self.multipleLumiSubscription.acquireFiles( self.multipleLumiSubscription.availableFiles()) jobFactory = splitter(self.multipleLumiSubscription) jobGroups = jobFactory(trigger_time=1) self.assertEquals(jobGroups, [], "Should have returned a null set") self.singleLumiSubscription.acquireFiles( self.singleLumiSubscription.availableFiles()) jobFactory = splitter(self.singleLumiSubscription) jobGroups = jobFactory(trigger_time=1) self.assertEquals(jobGroups, [], "Should have returned a null set") def testClosedSomeAcquired(self): """ _testClosedSomeAcquired_ since the subscriptions are closed and none of the files ahve been acquired, all of the files should show up """ splitter = SplitterFactory() self.multipleFileSubscription.getFileset().markOpen(False) self.singleFileSubscription.acquireFiles( [self.singleFileSubscription.availableFiles().pop()]) jobFactory = splitter(self.singleFileSubscription) jobGroups = jobFactory(trigger_time=1) self.assertEquals(jobGroups, [], "Should have returned a null set") self.multipleFileSubscription.getFileset().markOpen(False) self.multipleFileSubscription.acquireFiles( [self.multipleFileSubscription.availableFiles().pop()]) jobFactory = splitter(self.multipleFileSubscription) jobGroups = jobFactory(trigger_time=1) self.assertEquals(len(jobGroups), 1, "Should have gotten one jobGroup") self.assertEquals(len(jobGroups[0].jobs), 1, \ "JobFactory should have made one job") myfiles = jobGroups[0].jobs[0].getFiles() self.assertEquals(len(myfiles), 9, \ "JobFactory should have provides us with 9 files") self.multipleLumiSubscription.getFileset().markOpen(False) self.multipleLumiSubscription.acquireFiles( [self.multipleLumiSubscription.availableFiles().pop()]) jobFactory = splitter(self.multipleLumiSubscription) jobGroups = jobFactory(trigger_time=1) self.assertEquals(len(jobGroups), 1, "Should have gotten one jobGroup") self.assertEquals(len(jobGroups[0].jobs), 1, \ "JobFactory should have made one job") myfiles = jobGroups[0].jobs[0].getFiles() self.assertEquals(len(myfiles), 9, \ "JobFactory should have provides us with 9 files") self.singleLumiSubscription.getFileset().markOpen(False) self.singleLumiSubscription.acquireFiles( [self.singleLumiSubscription.availableFiles().pop()]) jobFactory = splitter(self.singleLumiSubscription) jobGroups = jobFactory(trigger_time=1) self.assertEquals(len(jobGroups), 1, "Should have gotten one jobGroup") self.assertEquals(len(jobGroups[0].jobs), 1, \ "JobFactory should have made one job") myfiles = jobGroups[0].jobs[0].getFiles() self.assertEquals(len(myfiles), 9, \ "JobFactory should have provides us with 9 files") self.assertEquals(len(myfiles), 9)
class SubscriptionTest(unittest.TestCase): """ _SubscriptionTest_ Testcase for the Subscription class """ def setUp(self): """ Initial Setup for Subscription Testcase Set a dummy Subscription with a fileset composed of one file inside it and a dummy workflow using the default constructor of the Workflow class """ self.dummyFile = File('/tmp/dummyfile', 9999, 0, 0, 0) self.dummySet = set() self.dummySet.add(self.dummyFile) self.dummyFileSet = Fileset(name='SubscriptionTestFileset', files=self.dummySet) self.dummyWorkFlow = Workflow() self.dummySubscription = Subscription(fileset=self.dummyFileSet, workflow=self.dummyWorkFlow) return def tearDown(self): pass def testGetWorkflow(self): """ Testcase for the getWorkflow method of the Subscription Class """ assert self.dummySubscription['workflow'] == self.dummyWorkFlow, \ 'Couldn\'t add Workflow to Subscription' def testGetFileset(self): """ Testcase for the getFileset method of the Subscription Class """ assert self.dummyFileSet.name == self.dummySubscription['fileset'].name, \ 'Couldn\'t add Fileset to Subscription - name does not match' for x in self.dummyFileSet.listNewFiles(): assert x in self.dummySubscription['fileset'].newfiles, \ 'Couldn\'t add Fileset to Subscription - newFiles Set does not match' assert self.dummyFileSet.getFiles(type='set') == \ self.dummySubscription['fileset'].getFiles(type='set'), \ 'Couldn\'t add Fileset to Subscription - %s Set does not match' % x def testAcquireFiles(self): """ Testcase for the acquireFiles method of the Subscription Class """ # Cleaning possible files already occupying the available set self.dummySubscription.acquireFiles() # First test - Test if initial file (on available set) is inserted in the # acquired set - no arguments dummyFile2 = File('/tmp/dummyfile2,8888', 1, 1, 1) # Insert dummyFile2 into the available files Set at dummySubscription self.dummySubscription.available.addFile(dummyFile2) S = self.dummySubscription.available.listNewFiles() # Check if Set returned by method is the same that was at the previous # available FileSet assert S == self.dummySubscription.acquireFiles(), \ 'Couldn\'t acquire file using method acquireFiles - (no arguments test)' # Second test - Test if target files are inserted at the acquired set dummyFileList = set() # Populating the dummy List with a random number of files for i in range(1, random.randint(100, 1000)): lfn = '/store/data/%s/%s/file.root' % (random.randint( 1000, 9999), random.randint(1000, 9999)) size = random.randint(1000, 2000) events = 1000 run = random.randint(0, 2000) lumi = random.randint(0, 8) file = File(lfn=lfn, size=size, events=events, checksums={"cksum": "1"}) file.addRun(Run(run, *[lumi])) dummyFileList.add(file) # Check if return value is correct - with parameters acqFiles = self.dummySubscription.acquireFiles(files=dummyFileList) assert acqFiles == dummyFileList, \ 'Return value for acquireFiles method not the acquired files' # Check if all files were inserted at subscription acquired files Set for x in dummyFileList: assert x in self.dummySubscription.acquired.getFiles(type='set'), \ 'Couldn\'t acquire File %s' % x.dict['lfn'] # Third test - Test if a replicate file is erased from the other Sets, # when a file is acquired dummyFile3 = File('/tmp/dummyfile3,5555', 1, 1, 1) dummyFileList = [] dummyFileList.append(dummyFile3) # Inserting dummyFile3 to be used as an argument, into each of the other file sets self.dummySubscription.available.addFile(dummyFile3) self.dummySubscription.failed.addFile(dummyFile3) self.dummySubscription.completed.addFile(dummyFile3) # Run the method acquireFiles self.dummySubscription.acquireFiles(files=dummyFileList, size=1) # Check if dummyFile3 was inserted at the acquired Set assert dummyFile3 in self.dummySubscription.acquired.getFiles(type='set'), \ 'Replicated file could\'nt be inserted at acquired Set' # Check if dummyFile3 was erased from all the other Sets assert dummyFile3 not in self.dummySubscription.available.getFiles(type='set'), \ 'Acquired file still present at available Set' assert dummyFile3 not in self.dummySubscription.failed.getFiles(type='set'), \ 'Acquired file still present at failed Set' assert dummyFile3 not in self.dummySubscription.completed.getFiles(type='set'), \ 'Acquired file still present at completed Set' # Fourth test - Test if the method works properly if a wrong size number # is given as an argument # Case 1: size < number of files given as an argument dummyFileList = [] for i in range(90, 100): dummyFileSize = File('/tmp/dummyfile' + str(i), 7656, 1, 1, 1) dummyFileList.append(dummyFileSize) # Run the method: self.dummySubscription.acquireFiles(files=dummyFileList, size=1) # Check each file of the List for x in dummyFileList: assert x in self.dummySubscription.acquired.getFiles(type='set'), \ 'File wasn\'t acquired (lower Size argument test)' # Case 2: size = 0 # Run the method: self.dummySubscription.acquireFiles(files=dummyFileList, size=0) # Check each file of the List for x in dummyFileList: assert x in self.dummySubscription.acquired.getFiles(type='set'), \ 'File wasn\'t acquired (zero size argument test)' def testCompleteFiles(self): """ Testcase for the completeFiles method of the Subscription Class """ # Cleaning possible files already occupying the available set self.dummySubscription.completeFiles([]) # First test - Test if initial file (on available set) is inserted in the # completed set - no arguments dummyFile2 = File('/tmp/dummyfile2,8888', 1, 1, 1) # Insert dummyFile2 into the available files Set at dummySubscription self.dummySubscription.available.addFile(dummyFile2) S = self.dummySubscription.availableFiles() # complete all files self.dummySubscription.completeFiles(S) assert len(self.dummySubscription.availableFiles()) == 0, \ "completed subscription still has %s files, what's up with that?" % \ len(self.dummySubscription.availableFiles()) # Second test - Test if target files are inserted at the completed files set dummyFileList = [] # Populating the dummy List with a random number of files for i in range(1, random.randint(100, 1000)): lfn = '/store/data/%s/%s/file.root' % (random.randint( 1000, 9999), random.randint(1000, 9999)) size = random.randint(1000, 2000) events = 1000 run = random.randint(0, 2000) lumi = random.randint(0, 8) file = File(lfn=lfn, size=size, events=events, checksums={"cksum": "1"}) file.addRun(Run(run, *[lumi])) dummyFileList.append(file) # Add the new files self.dummySubscription.available.addFile(dummyFileList) # and complete them self.dummySubscription.completeFiles(files=dummyFileList) # Check if return value is correct - with parameters assert len(self.dummySubscription.availableFiles()) == 0, \ "completed subscription still has %s files, what's up with that?" % \ len(self.dummySubscription.availableFiles()) # Check if all files were inserted at subscription's completed files Set for x in dummyFileList: assert x in self.dummySubscription.completed.getFiles(type='set'), \ 'Couldn\'t make file completed %s' % x.dict['lfn'] # Third test - Test if a replicate file is erased from the other Sets, # when a file is made completed dummyFile3 = File('/tmp/dummyfile3,5555', 1, 1, 1) dummyFileList = [] dummyFileList.append(dummyFile3) # Inserting dummyFile3 to be used as an argument, into each of the other # file sets self.dummySubscription.acquired.addFile(dummyFile3) self.dummySubscription.failed.addFile(dummyFile3) self.dummySubscription.completed.addFile(dummyFile3) # Run the method completeFiles self.dummySubscription.completeFiles(files=dummyFileList) # Check if dummyFile3 was inserted at the completed Set assert dummyFile3 in self.dummySubscription.completed.getFiles(type='set'), \ 'Replicated file could\'nt be inserted at completed Set' # Check if dummyFile3 was erased from all the other Sets assert dummyFile3 not in self.dummySubscription.acquired.getFiles(type='set'), \ 'Completed file still present at acquired Set' assert dummyFile3 not in self.dummySubscription.failed.getFiles(type='set'), \ 'Completed file still present at failed Set' assert dummyFile3 not in self.dummySubscription.available.getFiles(type='set'), \ 'Completed file still present at available Set' def testFailFiles(self): """ Testcase for the failFiles method of the Subscription Class """ # Cleaning possible files already occupying the available set self.dummySubscription.failFiles([]) # First test - Test if initial file (on available set) is inserted in the # failed set - no arguments dummyFile2 = File('/tmp/dummyfile2,8888', 1, 1, 1) # Insert dummyFile2 into the available files Set at dummySubscription self.dummySubscription.available.addFile(dummyFile2) S = self.dummySubscription.availableFiles() # Fail all files self.dummySubscription.failFiles(S) assert len(self.dummySubscription.availableFiles()) == 0, \ "failed subscription still has %s files, what's up with that?" % \ len(self.dummySubscription.availableFiles()) # Second test - Test if target files are inserted at the failed set dummyFileList = [] # Populating the dummy List with a random number of files for i in range(1, random.randint(100, 1000)): lfn = '/store/data/%s/%s/file.root' % (random.randint( 1000, 9999), random.randint(1000, 9999)) size = random.randint(1000, 2000) events = 1000 run = random.randint(0, 2000) lumi = random.randint(0, 8) file = File(lfn=lfn, size=size, events=events, checksums={"cksum": "1"}) file.addRun(Run(run, *[lumi])) dummyFileList.append(file) # Add the new files self.dummySubscription.available.addFile(dummyFileList) # and fail them self.dummySubscription.failFiles(files=dummyFileList) # Check there are no files available - everything should be failed assert len(self.dummySubscription.availableFiles()) == 0, \ "failed subscription still has %s files, what's up with that?" % \ len(self.dummySubscription.availableFiles()) # Check if all files were inserted at subscription's failed files Set for x in dummyFileList: assert x in self.dummySubscription.failed.getFiles(type='set'), \ 'Couldn\'t make file failed %s' % x.dict['lfn'] # Third test - Test if a replicate file is erased from the other Sets, # when a file is considered failed dummyFile3 = File('/tmp/dummyfile3,5555', 1, 1, 1) dummyFileList = [] dummyFileList.append(dummyFile3) # Inserting dummyFile3 to be used as an argument, into each of the other # file sets self.dummySubscription.acquired.addFile(dummyFile3) self.dummySubscription.available.addFile(dummyFile3) self.dummySubscription.completed.addFile(dummyFile3) # Run the method failFiles self.dummySubscription.failFiles(files=dummyFileList) # Check if dummyFile3 was inserted at the failed Set assert dummyFile3 in self.dummySubscription.failed.getFiles(type='set'), \ 'Replicated file could\'nt be inserted at failed Set' # Check if dummyFile3 was erased from all the other Sets assert dummyFile3 not in self.dummySubscription.acquired.getFiles(type='set'), \ 'Failed file still present at acquired Set' assert dummyFile3 not in self.dummySubscription.completed.getFiles(type='set'), \ 'Failed file still present at completed Set' assert dummyFile3 not in self.dummySubscription.available.getFiles(type='set'), \ 'Failed file still present at available Set' def testFilesOfStatus(self): """ Testcase for the filesOfStatus method of the Subscription Class """ assert self.dummySubscription.filesOfStatus('Available') == \ self.dummySubscription.available.getFiles(type='set') - \ self.dummySubscription.acquiredFiles() | self.dummySubscription.completedFiles() | self.dummySubscription.failedFiles(), \ 'Method fileOfStatus(\'AvailableFiles\') does not return available files set' assert self.dummySubscription.filesOfStatus('Acquired') == self.dummySubscription.acquired.getFiles(type='set'), \ 'Method fileOfStatus(\'AcquiredFiles\') does not return acquired files set' assert self.dummySubscription.filesOfStatus('Completed') == self.dummySubscription.completed.getFiles( type='set'), \ 'Method fileOfStatus(\'CompletedFiles\') does not return completed files set' assert self.dummySubscription.filesOfStatus('Failed') == self.dummySubscription.failed.getFiles(type='set'), \ 'Method fileOfStatus(\'FailedFiles\') does not return failed files set' def testAvailableFiles(self): """ Testcase for the availableFiles method of the Subscription Class """ assert self.dummySubscription.availableFiles() == \ self.dummySubscription.available.getFiles(type='set'), \ 'Method availableFiles does not return available files Set' def testAcquiredFiles(self): """ Testcase for the acquiredFiles method of the Subscription Class """ assert self.dummySubscription.acquiredFiles() == \ self.dummySubscription.acquired.getFiles(type='set'), \ 'Method acquiredFiles does not return acquired files Set' def testCompletedFiles(self): """ Testcase for the completedFiles method of the Subscription Class """ assert self.dummySubscription.completedFiles() == \ self.dummySubscription.completed.getFiles(type='set'), \ 'Method completedFiles does not return completed files Set' def testFailedFiles(self): """ Testcase for the failedFiles method of the Subscription Class """ assert self.dummySubscription.failedFiles() == \ self.dummySubscription.failed.getFiles(type='set'), \ 'Method failedFiles does not return failed files Set'
class EndOfRunBasedTest(unittest.TestCase): def setUp(self): """ _setUp_ Create two subscriptions: One that contains a single file and one that contains multiple files. """ self.multipleFileFileset = Fileset(name = "TestFileset1") for i in range(10): newFile = File(makeUUID(), size = 1000, events = 100, locations = set(["somese.cern.ch"])) newFile.addRun(Run(i, *[45+i])) self.multipleFileFileset.addFile(newFile) self.singleFileFileset = Fileset(name = "TestFileset2") newFile = File("/some/file/name", size = 1000, events = 100, locations = set(["somese.cern.ch"])) newFile.addRun(Run(1, *[45])) self.singleFileFileset.addFile(newFile) self.multipleFileLumiset = Fileset(name = "TestFileset3") for i in range(10): newFile = File(makeUUID(), size = 1000, events = 100, locations = set(["somese.cern.ch"])) newFile.addRun(Run(1, *[45+i/3])) self.multipleFileLumiset.addFile(newFile) self.singleLumiFileset = Fileset(name = "TestFileset4") for i in range(10): newFile = File(makeUUID(), size = 1000, events = 100, locations = set(["somese.cern.ch"])) newFile.addRun(Run(1, *[45])) self.singleLumiFileset.addFile(newFile) testWorkflow = Workflow() self.multipleFileSubscription = Subscription(fileset = self.multipleFileFileset, workflow = testWorkflow, split_algo = "EndOfRun", type = "Processing") self.singleFileSubscription = Subscription(fileset = self.singleFileFileset, workflow = testWorkflow, split_algo = "EndOfRun", type = "Processing") self.multipleLumiSubscription = Subscription(fileset = self.multipleFileLumiset, workflow = testWorkflow, split_algo = "EndOfRun", type = "Processing") self.singleLumiSubscription = Subscription(fileset = self.singleLumiFileset, workflow = testWorkflow, split_algo = "EndOfRun", type = "Processing") return def tearDown(self): pass def testNone(self): """ _testNone_ since the subscriptions are open, we shouldn't get any jobs back """ splitter = SplitterFactory() jobFactory = splitter(self.singleFileSubscription) jobGroups = jobFactory() self.assertEquals(jobGroups, [], "Should have returned a null set") jobFactory = splitter(self.multipleFileSubscription) jobGroups = jobFactory() self.assertEquals(jobGroups, [], "Should have returned a null set") jobFactory = splitter(self.multipleLumiSubscription) jobGroups = jobFactory() self.assertEquals(jobGroups, [], "Should have returned a null set") jobFactory = splitter(self.singleLumiSubscription) jobGroups = jobFactory() self.assertEquals(jobGroups, [], "Should have returned a null set") return def testClosed(self): """ _testClosed_ since the subscriptions are closed and none of the files ahve been acquired, all of the files should show up """ splitter = SplitterFactory() self.singleFileSubscription.getFileset().markOpen(False) jobFactory = splitter(self.singleFileSubscription) jobGroups = jobFactory() assert len(jobGroups) == 1, \ "ERROR: JobFactory didn't return one JobGroup." assert len(jobGroups[0].jobs) == 1, \ "ERROR: JobFactory didn't create a single job." job = jobGroups[0].jobs.pop() assert job.getFiles(type = "lfn") == ["/some/file/name"], \ "ERROR: Job contains unknown files." self.multipleFileSubscription.getFileset().markOpen(False) jobFactory = splitter(self.multipleFileSubscription) jobGroups = jobFactory() self.assertEquals(len(jobGroups), 1) self.assertEquals(len(jobGroups[0].jobs),1) myfiles = jobGroups[0].jobs[0].getFiles() self.assertEquals(len(myfiles), 10) self.multipleLumiSubscription.getFileset().markOpen(False) jobFactory = splitter(self.multipleLumiSubscription) jobGroups = jobFactory() self.assertEquals(len(jobGroups), 1) self.assertEquals(len(jobGroups[0].jobs),1) myfiles = jobGroups[0].jobs[0].getFiles() self.assertEquals(len(myfiles), 10) #self.assertEquals(jobGroups, [], "Should have returned a null set") self.singleLumiSubscription.getFileset().markOpen(False) jobFactory = splitter(self.singleLumiSubscription) jobGroups = jobFactory() assert len(jobGroups) == 1, \ "ERROR: JobFactory didn't return one JobGroup." assert len(jobGroups[0].jobs) == 1, \ "ERROR: JobFactory didn't create a single job." myfiles = jobGroups[0].jobs[0].getFiles() self.assertEquals(len(myfiles), 10) def testAllAcquired(self): """ _testAllAcquired_ should all return no job groups """ splitter = SplitterFactory() self.singleFileSubscription.acquireFiles( self.singleFileSubscription.availableFiles()) jobFactory = splitter(self.singleFileSubscription) jobGroups = jobFactory() self.assertEquals(jobGroups, [], "Should have returned a null set") self.multipleFileSubscription.acquireFiles( self.multipleFileSubscription.availableFiles()) jobFactory = splitter(self.multipleFileSubscription) jobGroups = jobFactory() self.assertEquals(jobGroups, [], "Should have returned a null set") self.multipleLumiSubscription.acquireFiles( self.multipleLumiSubscription.availableFiles()) jobFactory = splitter(self.multipleLumiSubscription) jobGroups = jobFactory() self.assertEquals(jobGroups, [], "Should have returned a null set") self.singleLumiSubscription.acquireFiles( self.singleLumiSubscription.availableFiles()) jobFactory = splitter(self.singleLumiSubscription) jobGroups = jobFactory() self.assertEquals(jobGroups, [], "Should have returned a null set") def testClosedSomeAcquired(self): """ _testClosedSomeAcquired_ since the subscriptions are closed and none of the files ahve been acquired, all of the files should show up """ splitter = SplitterFactory() self.multipleFileSubscription.getFileset().markOpen(False) self.singleFileSubscription.acquireFiles( [self.singleFileSubscription.availableFiles().pop()]) jobFactory = splitter(self.singleFileSubscription) jobGroups = jobFactory() self.assertEquals(jobGroups, [], "Should have returned a null set") self.multipleFileSubscription.getFileset().markOpen(False) self.multipleFileSubscription.acquireFiles( [self.multipleFileSubscription.availableFiles().pop()]) jobFactory = splitter(self.multipleFileSubscription) jobGroups = jobFactory() self.assertEquals(len(jobGroups),1, "Should have gotten one jobGroup") self.assertEquals(len(jobGroups[0].jobs), 1, \ "JobFactory should have made one job") myfiles = jobGroups[0].jobs[0].getFiles() self.assertEquals(len(myfiles), 9, \ "JobFactory should have provides us with 9 files") self.multipleLumiSubscription.getFileset().markOpen(False) self.multipleLumiSubscription.acquireFiles( [self.multipleLumiSubscription.availableFiles().pop()]) jobFactory = splitter(self.multipleLumiSubscription) jobGroups = jobFactory() self.assertEquals(len(jobGroups),1, "Should have gotten one jobGroup") self.assertEquals(len(jobGroups[0].jobs), 1, \ "JobFactory should have made one job") myfiles = jobGroups[0].jobs[0].getFiles() self.assertEquals(len(myfiles), 9, \ "JobFactory should have provides us with 9 files") self.singleLumiSubscription.getFileset().markOpen(False) self.singleLumiSubscription.acquireFiles( [self.singleLumiSubscription.availableFiles().pop()]) jobFactory = splitter(self.singleLumiSubscription) jobGroups = jobFactory() self.assertEquals(len(jobGroups),1, "Should have gotten one jobGroup") self.assertEquals(len(jobGroups[0].jobs), 1, \ "JobFactory should have made one job") myfiles = jobGroups[0].jobs[0].getFiles() self.assertEquals(len(myfiles), 9, \ "JobFactory should have provides us with 9 files") self.assertEquals(len(myfiles), 9)
class SubscriptionTest(unittest.TestCase): """ _SubscriptionTest_ Testcase for the Subscription class """ def setUp(self): """ Initial Setup for Subscription Testcase Set a dummy Subscription with a fileset composed of one file inside it and a dummy workflow using the default constructor of the Workflow class """ self.dummyFile = File('/tmp/dummyfile',9999,0,0,0) self.dummySet = set() self.dummySet.add(self.dummyFile) self.dummyFileSet = Fileset(name = 'SubscriptionTestFileset', files = self.dummySet) self.dummyWorkFlow = Workflow() self.dummySubscription = Subscription(fileset = self.dummyFileSet, workflow = self.dummyWorkFlow) return def tearDown(self): pass def testGetWorkflow(self): """ Testcase for the getWorkflow method of the Subscription Class """ assert self.dummySubscription['workflow'] == self.dummyWorkFlow, \ 'Couldn\'t add Workflow to Subscription' def testGetFileset(self): """ Testcase for the getFileset method of the Subscription Class """ assert self.dummyFileSet.name == self.dummySubscription['fileset'].name, \ 'Couldn\'t add Fileset to Subscription - name does not match' for x in self.dummyFileSet.listNewFiles(): assert x in self.dummySubscription['fileset'].newfiles, \ 'Couldn\'t add Fileset to Subscription - newFiles Set does not match' assert self.dummyFileSet.getFiles(type='set') == \ self.dummySubscription['fileset'].getFiles(type='set'), \ 'Couldn\'t add Fileset to Subscription - %s Set does not match' % x def testAcquireFiles(self): """ Testcase for the acquireFiles method of the Subscription Class """ #Cleaning possible files already occupying the available set self.dummySubscription.acquireFiles() # First test - Test if initial file (on available set) is inserted in the # acquired set - no arguments dummyFile2 = File('/tmp/dummyfile2,8888',1,1,1) #Insert dummyFile2 into the available files Set at dummySubscription self.dummySubscription.available.addFile(dummyFile2) S = self.dummySubscription.available.listNewFiles() #Check if Set returned by method is the same that was at the previous #available FileSet assert S == self.dummySubscription.acquireFiles(), \ 'Couldn\'t acquire file using method acquireFiles - (no arguments test)' #Second test - Test if target files are inserted at the acquired set dummyFileList = set() #Populating the dummy List with a random number of files for i in range(1, random.randint(100,1000)): lfn = '/store/data/%s/%s/file.root' % (random.randint(1000, 9999), random.randint(1000, 9999)) size = random.randint(1000, 2000) events = 1000 run = random.randint(0, 2000) lumi = random.randint(0, 8) file = File(lfn = lfn, size = size, events = events, checksums = {"cksum": "1"}) file.addRun(Run(run, *[lumi])) dummyFileList.add(file) #Check if return value is correct - with parameters acqFiles = self.dummySubscription.acquireFiles(files = dummyFileList) assert acqFiles == dummyFileList,\ 'Return value for acquireFiles method not the acquired files' #Check if all files were inserted at subscription acquired files Set for x in dummyFileList: assert x in self.dummySubscription.acquired.getFiles(type='set'), \ 'Couldn\'t acquire File %s' % x.dict['lfn'] #Third test - Test if a replicate file is erased from the other Sets, #when a file is acquired dummyFile3 = File('/tmp/dummyfile3,5555',1,1,1) dummyFileList = [] dummyFileList.append(dummyFile3) #Inserting dummyFile3 to be used as an argument, into each of the other file sets self.dummySubscription.available.addFile(dummyFile3) self.dummySubscription.failed.addFile(dummyFile3) self.dummySubscription.completed.addFile(dummyFile3) #Run the method acquireFiles self.dummySubscription.acquireFiles(files=dummyFileList, size=1) #Check if dummyFile3 was inserted at the acquired Set assert dummyFile3 in self.dummySubscription.acquired.getFiles(type='set'), \ 'Replicated file could\'nt be inserted at acquired Set' #Check if dummyFile3 was erased from all the other Sets assert dummyFile3 not in self.dummySubscription.available.getFiles(type='set'), \ 'Acquired file still present at available Set' assert dummyFile3 not in self.dummySubscription.failed.getFiles(type='set'), \ 'Acquired file still present at failed Set' assert dummyFile3 not in self.dummySubscription.completed.getFiles(type='set'), \ 'Acquired file still present at completed Set' #Fourth test - Test if the method works properly if a wrong size number #is given as an argument #Case 1: size < number of files given as an argument dummyFileList = [] for i in range(90,100): dummyFileSize = File('/tmp/dummyfile'+str(i),7656,1,1,1) dummyFileList.append(dummyFileSize) #Run the method: self.dummySubscription.acquireFiles(files=dummyFileList, size=1) #Check each file of the List for x in dummyFileList: assert x in self.dummySubscription.acquired.getFiles(type='set'), \ 'File wasn\'t acquired (lower Size argument test)' #Case 2: size = 0 #Run the method: self.dummySubscription.acquireFiles(files=dummyFileList, size=0) #Check each file of the List for x in dummyFileList: assert x in self.dummySubscription.acquired.getFiles(type='set'), \ 'File wasn\'t acquired (zero size argument test)' def testCompleteFiles(self): """ Testcase for the completeFiles method of the Subscription Class """ #Cleaning possible files already occupying the available set self.dummySubscription.completeFiles([]) #First test - Test if initial file (on available set) is inserted in the #completed set - no arguments dummyFile2 = File('/tmp/dummyfile2,8888',1,1,1) #Insert dummyFile2 into the available files Set at dummySubscription self.dummySubscription.available.addFile(dummyFile2) S = self.dummySubscription.availableFiles() #complete all files self.dummySubscription.completeFiles(S) assert len(self.dummySubscription.availableFiles()) == 0, \ "completed subscription still has %s files, what's up with that?" %\ len(self.dummySubscription.availableFiles()) #Second test - Test if target files are inserted at the completed files set dummyFileList = [] #Populating the dummy List with a random number of files for i in range(1, random.randint(100,1000)): lfn = '/store/data/%s/%s/file.root' % (random.randint(1000, 9999), random.randint(1000, 9999)) size = random.randint(1000, 2000) events = 1000 run = random.randint(0, 2000) lumi = random.randint(0, 8) file = File(lfn = lfn, size = size, events = events, checksums = {"cksum": "1"}) file.addRun(Run(run, *[lumi])) dummyFileList.append(file) #Add the new files self.dummySubscription.available.addFile(dummyFileList) #and complete them self.dummySubscription.completeFiles(files = dummyFileList) #Check if return value is correct - with parameters assert len(self.dummySubscription.availableFiles()) == 0, \ "completed subscription still has %s files, what's up with that?" %\ len(self.dummySubscription.availableFiles()) #Check if all files were inserted at subscription's completed files Set for x in dummyFileList: assert x in self.dummySubscription.completed.getFiles(type='set'), \ 'Couldn\'t make file completed %s' % x.dict['lfn'] #Third test - Test if a replicate file is erased from the other Sets, #when a file is made completed dummyFile3 = File('/tmp/dummyfile3,5555',1,1,1) dummyFileList = [] dummyFileList.append(dummyFile3) #Inserting dummyFile3 to be used as an argument, into each of the other #file sets self.dummySubscription.acquired.addFile(dummyFile3) self.dummySubscription.failed.addFile(dummyFile3) self.dummySubscription.completed.addFile(dummyFile3) #Run the method completeFiles self.dummySubscription.completeFiles(files=dummyFileList) #Check if dummyFile3 was inserted at the completed Set assert dummyFile3 in self.dummySubscription.completed.getFiles(type='set'), \ 'Replicated file could\'nt be inserted at completed Set' #Check if dummyFile3 was erased from all the other Sets assert dummyFile3 not in self.dummySubscription.acquired.getFiles(type='set'), \ 'Completed file still present at acquired Set' assert dummyFile3 not in self.dummySubscription.failed.getFiles(type='set'), \ 'Completed file still present at failed Set' assert dummyFile3 not in self.dummySubscription.available.getFiles(type='set'), \ 'Completed file still present at available Set' def testFailFiles(self): """ Testcase for the failFiles method of the Subscription Class """ #Cleaning possible files already occupying the available set self.dummySubscription.failFiles([]) #First test - Test if initial file (on available set) is inserted in the # failed set - no arguments dummyFile2 = File('/tmp/dummyfile2,8888',1,1,1) #Insert dummyFile2 into the available files Set at dummySubscription self.dummySubscription.available.addFile(dummyFile2) S = self.dummySubscription.availableFiles() # Fail all files self.dummySubscription.failFiles(S) assert len(self.dummySubscription.availableFiles()) == 0, \ "failed subscription still has %s files, what's up with that?" %\ len(self.dummySubscription.availableFiles()) #Second test - Test if target files are inserted at the failed set dummyFileList = [] #Populating the dummy List with a random number of files for i in range(1, random.randint(100,1000)): lfn = '/store/data/%s/%s/file.root' % (random.randint(1000, 9999), random.randint(1000, 9999)) size = random.randint(1000, 2000) events = 1000 run = random.randint(0, 2000) lumi = random.randint(0, 8) file = File(lfn = lfn, size = size, events = events, checksums = {"cksum": "1"}) file.addRun(Run(run, *[lumi])) dummyFileList.append(file) #Add the new files self.dummySubscription.available.addFile(dummyFileList) #and fail them self.dummySubscription.failFiles(files = dummyFileList) #Check there are no files available - everything should be failed assert len(self.dummySubscription.availableFiles()) == 0, \ "failed subscription still has %s files, what's up with that?" %\ len(self.dummySubscription.availableFiles()) #Check if all files were inserted at subscription's failed files Set for x in dummyFileList: assert x in self.dummySubscription.failed.getFiles(type='set'),\ 'Couldn\'t make file failed %s' % x.dict['lfn'] #Third test - Test if a replicate file is erased from the other Sets, # when a file is considered failed dummyFile3 = File('/tmp/dummyfile3,5555',1,1,1) dummyFileList = [] dummyFileList.append(dummyFile3) #Inserting dummyFile3 to be used as an argument, into each of the other # file sets self.dummySubscription.acquired.addFile(dummyFile3) self.dummySubscription.available.addFile(dummyFile3) self.dummySubscription.completed.addFile(dummyFile3) #Run the method failFiles self.dummySubscription.failFiles(files=dummyFileList) #Check if dummyFile3 was inserted at the failed Set assert dummyFile3 in self.dummySubscription.failed.getFiles(type='set'), \ 'Replicated file could\'nt be inserted at failed Set' #Check if dummyFile3 was erased from all the other Sets assert dummyFile3 not in self.dummySubscription.acquired.getFiles(type='set'), \ 'Failed file still present at acquired Set' assert dummyFile3 not in self.dummySubscription.completed.getFiles(type='set'), \ 'Failed file still present at completed Set' assert dummyFile3 not in self.dummySubscription.available.getFiles(type='set'), \ 'Failed file still present at available Set' def testFilesOfStatus(self): """ Testcase for the filesOfStatus method of the Subscription Class """ assert self.dummySubscription.filesOfStatus('Available') == \ self.dummySubscription.available.getFiles(type='set') - \ self.dummySubscription.acquiredFiles() | self.dummySubscription.completedFiles() | self.dummySubscription.failedFiles(), \ 'Method fileOfStatus(\'AvailableFiles\') does not return available files set' assert self.dummySubscription.filesOfStatus('Acquired') == self.dummySubscription.acquired.getFiles(type='set'), \ 'Method fileOfStatus(\'AcquiredFiles\') does not return acquired files set' assert self.dummySubscription.filesOfStatus('Completed') == self.dummySubscription.completed.getFiles(type='set'), \ 'Method fileOfStatus(\'CompletedFiles\') does not return completed files set' assert self.dummySubscription.filesOfStatus('Failed') == self.dummySubscription.failed.getFiles(type='set'), \ 'Method fileOfStatus(\'FailedFiles\') does not return failed files set' def testAvailableFiles(self): """ Testcase for the availableFiles method of the Subscription Class """ assert self.dummySubscription.availableFiles() == \ self.dummySubscription.available.getFiles(type='set'), \ 'Method availableFiles does not return available files Set' def testAcquiredFiles(self): """ Testcase for the acquiredFiles method of the Subscription Class """ assert self.dummySubscription.acquiredFiles() == \ self.dummySubscription.acquired.getFiles(type='set'), \ 'Method acquiredFiles does not return acquired files Set' def testCompletedFiles(self): """ Testcase for the completedFiles method of the Subscription Class """ assert self.dummySubscription.completedFiles() == \ self.dummySubscription.completed.getFiles(type='set'), \ 'Method completedFiles does not return completed files Set' def testFailedFiles(self): """ Testcase for the failedFiles method of the Subscription Class """ assert self.dummySubscription.failedFiles() == \ self.dummySubscription.failed.getFiles(type='set'), \ 'Method failedFiles does not return failed files Set'