def testCommit(self): """ Testcase for the commit method of the Fileset class """ localTestFileSet = Fileset('LocalTestFileset', self.initialSet) fsSize = len(localTestFileSet.getFiles(type="lfn")) #Dummy file to test fileTestCommit = File('/tmp/filetestcommit', 0000, 1, 1) #File is added to the newfiles attribute of localTestFileSet localTestFileSet.addFile(fileTestCommit) assert fsSize == len(localTestFileSet.getFiles(type = "lfn")) - 1, 'file not added'\ 'correctly to test fileset' newfilestemp = localTestFileSet.newfiles assert fileTestCommit in newfilestemp, 'test file not in the new files'\ 'list' #After commit, dummy file is supposed to move from newfiles to files localTestFileSet.commit() #First, testing if the new file is present at file set object attribute of the Fileset object assert newfilestemp.issubset(localTestFileSet.files), 'Test file not ' \ 'present at fileset.files - fileset.commit ' \ 'not working properly' #Second, testing if the newfile set object attribute is empty assert localTestFileSet.newfiles == set(), \ 'Test file not present at fileset.newfiles ' \ '- fileset.commit not working properly'
def testCommit(self): """ Testcase for the commit method of the Fileset class """ localTestFileSet = Fileset('LocalTestFileset', self.initialSet) fsSize = len(localTestFileSet.getFiles(type = "lfn")) #Dummy file to test fileTestCommit = File('/tmp/filetestcommit',0000,1,1) #File is added to the newfiles attribute of localTestFileSet localTestFileSet.addFile(fileTestCommit) assert fsSize == len(localTestFileSet.getFiles(type = "lfn")) - 1, 'file not added'\ 'correctly to test fileset' newfilestemp = localTestFileSet.newfiles assert fileTestCommit in newfilestemp, 'test file not in the new files'\ 'list' #After commit, dummy file is supposed to move from newfiles to files localTestFileSet.commit() #First, testing if the new file is present at file set object attribute of the Fileset object assert newfilestemp.issubset(localTestFileSet.files), 'Test file not ' \ 'present at fileset.files - fileset.commit ' \ 'not working properly' #Second, testing if the newfile set object attribute is empty assert localTestFileSet.newfiles == set(), \ 'Test file not present at fileset.newfiles ' \ '- fileset.commit not working properly'
def __init__(self, fileset=None, workflow=None, whitelist=None, blacklist=None, split_algo="FileBased", type="Processing"): if fileset == None: fileset = Fileset() if whitelist == None: whitelist = set() if blacklist == None: blacklist = set() self.setdefault('fileset', fileset) self.setdefault('workflow', workflow) self.setdefault('type', type) self.setdefault('split_algo', split_algo) self.setdefault('whitelist', whitelist) self.setdefault('blacklist', blacklist) self.available = Fileset(name=fileset.name, files=fileset.getFiles()) self.acquired = Fileset(name='acquired') self.completed = Fileset(name='completed') self.failed = Fileset(name='failed')
def testCall(self): fileset = Fileset(name="FakeFeederTest") for i in range(1, 21): self.feeder([fileset]) set = fileset.getFiles(type = "set") if len(set) > 0: file = set.pop() fileset.commit()
def testCall(self): fileset = Fileset(name="FakeFeederTest") for i in range(1, 21): self.feeder([fileset]) set = fileset.getFiles(type="set") if len(set) > 0: file = set.pop() fileset.commit()
def __init__(self, fileset = None, workflow = None, split_algo = "FileBased", type = "Processing"): if fileset == None: fileset = Fileset() self.setdefault('fileset', fileset) self.setdefault('workflow', workflow) self.setdefault('type', type) self.setdefault('split_algo', split_algo) self.available = Fileset(name=fileset.name, files = fileset.getFiles()) self.acquired = Fileset(name='acquired') self.completed = Fileset(name='completed') self.failed = Fileset(name='failed')
class Subscription(Pickleable, dict): def __init__(self, fileset=None, workflow=None, whitelist=None, blacklist=None, split_algo="FileBased", type="Processing"): if fileset == None: fileset = Fileset() if whitelist == None: whitelist = set() if blacklist == None: blacklist = set() self.setdefault('fileset', fileset) self.setdefault('workflow', workflow) self.setdefault('type', type) self.setdefault('split_algo', split_algo) self.setdefault('whitelist', whitelist) self.setdefault('blacklist', blacklist) self.available = Fileset(name=fileset.name, files=fileset.getFiles()) self.acquired = Fileset(name='acquired') self.completed = Fileset(name='completed') self.failed = Fileset(name='failed') def name(self): return self.getWorkflow().name.replace(' ', '') + '_' + \ self.getFileset().name.replace(' ', '') def getWorkflow(self): return self["workflow"] def workflowName(self): if self["workflow"] == None: return "Unknown" return self["workflow"].name def taskName(self): if self['workflow'] == None: return "Unknown" return self['workflow'].task def getFileset(self): return self['fileset'] def acquireFiles(self, files=[], size=1): """ Return the files acquired """ self.acquired.commit() self.available.commit() self.failed.commit() self.completed.commit() retval = [] if len(files): for i in files: # Check each set, instead of elif, just in case something has # got out of synch if i in self.available.files: self.available.files.remove(i) if i in self.failed.files: self.failed.files.remove(i) if i in self.completed.files: self.completed.files.remove(i) self.acquired.addFile(i) else: if len(self.available.files) < size or size == 0: size = len(self.available.files) for i in range(size): self.acquired.addFile(self.available.files.pop()) return self.acquired.listNewFiles() def completeFiles(self, files): """ Return the number of files complete """ self.acquired.commit() self.available.commit() self.failed.commit() self.completed.commit() for i in files: # Check each set, instead of elif, just in case something has # got out of synch if i in self.available.files: self.available.files.remove(i) if i in self.failed.files: self.failed.files.remove(i) if i in self.acquired.files: self.acquired.files.remove(i) self.completed.addFile(i) def failFiles(self, files): """ Return the number of files failed """ self.acquired.commit() self.available.commit() self.failed.commit() self.completed.commit() for i in files: # Check each set, instead of elif, just in case something has # got out of synch if i in self.available.files: self.available.files.remove(i) if i in self.completed.files: self.completed.files.remove(i) if i in self.acquired.files: self.acquired.files.remove(i) self.failed.addFile(i) def filesOfStatus(self, status=None): """ _filesOfStatus_ Return a Set of File objects that are associated with the subscription and have a particular status. """ status = status.title() if status == 'Available': return self.available.getFiles(type='set') - \ (self.acquiredFiles() | self.completedFiles() | self.failedFiles()) elif status == 'Acquired': return self.acquired.getFiles(type='set') elif status == 'Completed': return self.completed.getFiles(type='set') elif status == 'Failed': return self.failed.getFiles(type='set') def markLocation(self, location, whitelist=True): """ Add a location to the subscriptions white or black list """ if whitelist: self['whitelist'].add(location) else: self['blacklist'].add(location) def availableFiles(self): """ Return a Set of files that are available for processing (e.g. not already in use) and at sites that are white listed or not black listed """ def locationMagic(files, locations): """ files and locations are sets. method returns the subset of files that are at the locations - this is a lot simpler with the database """ magicfiles = set() for f in files: if len(f['locations'] & locations) > 0: magicfiles.add(f) return magicfiles files = self.filesOfStatus(status="Available") if len(self['whitelist']) > 0: # Return files at white listed sites return locationMagic(files, self['whitelist']) elif len(self['blacklist']) > 0: # Return files not at black listed sites return files - locationMagic(files, self['blacklist']) #Return all files, because you're crazy and just don't care return files def acquiredFiles(self): """ Set of files marked as acquired. """ return self.filesOfStatus(status="Acquired") def completedFiles(self): """ Set of files marked as completed. """ return self.filesOfStatus(status="Completed") def failedFiles(self): """ Set of files marked as failed. """ return self.filesOfStatus(status="Failed")
class Subscription(Pickleable, dict): def __init__(self, fileset = None, workflow = None, split_algo = "FileBased", type = "Processing"): if fileset == None: fileset = Fileset() self.setdefault('fileset', fileset) self.setdefault('workflow', workflow) self.setdefault('type', type) self.setdefault('split_algo', split_algo) self.available = Fileset(name=fileset.name, files = fileset.getFiles()) self.acquired = Fileset(name='acquired') self.completed = Fileset(name='completed') self.failed = Fileset(name='failed') def name(self): return self.getWorkflow().name.replace(' ', '') + '_' + \ self.getFileset().name.replace(' ', '') def getWorkflow(self): return self["workflow"] def workflowName(self): if self["workflow"] == None: return "Unknown" return self["workflow"].name def workflowType(self): if self["workflow"] == None: return "Unknown" return self["workflow"].wfType def taskName(self): if self['workflow'] == None: return "Unknown" return self['workflow'].task def owner(self): if self['workflow'] == None: return 'Unknown' return self['workflow'].owner def getFileset(self): return self['fileset'] def acquireFiles(self, files = [], size=1): """ Return the files acquired """ self.acquired.commit() self.available.commit() self.failed.commit() self.completed.commit() retval = [] if len(files): for i in files: # Check each set, instead of elif, just in case something has # got out of synch if i in self.available.files: self.available.files.remove(i) if i in self.failed.files: self.failed.files.remove(i) if i in self.completed.files: self.completed.files.remove(i) self.acquired.addFile(i) else: if len(self.available.files) < size or size == 0: size = len(self.available.files) for i in range(size): self.acquired.addFile(self.available.files.pop()) return self.acquired.listNewFiles() def completeFiles(self, files): """ Return the number of files complete """ self.acquired.commit() self.available.commit() self.failed.commit() self.completed.commit() for i in files: # Check each set, instead of elif, just in case something has # got out of synch if i in self.available.files: self.available.files.remove(i) if i in self.failed.files: self.failed.files.remove(i) if i in self.acquired.files: self.acquired.files.remove(i) self.completed.addFile(i) def failFiles(self, files): """ Return the number of files failed """ self.acquired.commit() self.available.commit() self.failed.commit() self.completed.commit() for i in files: # Check each set, instead of elif, just in case something has # got out of synch if i in self.available.files: self.available.files.remove(i) if i in self.completed.files: self.completed.files.remove(i) if i in self.acquired.files: self.acquired.files.remove(i) self.failed.addFile(i) def filesOfStatus(self, status=None, doingJobSplitting = False): """ _filesOfStatus_ Return a Set of File objects that are associated with the subscription and have a particular status. """ status = status.title() if status == 'Available': return self.available.getFiles(type='set') - \ (self.acquiredFiles() | self.completedFiles() | self.failedFiles()) elif status == 'Acquired': return self.acquired.getFiles(type='set') elif status == 'Completed': return self.completed.getFiles(type='set') elif status == 'Failed': return self.failed.getFiles(type='set') def availableFiles(self, limit = None, doingJobSplitting = False): """ _availableFiles_ Return a Set of files that are available for processing (e.g. not already in use) """ if limit: return list(self.filesOfStatus(status = "Available", doingJobSplitting = doingJobSplitting))[:limit] else: return self.filesOfStatus(status = "Available", doingJobSplitting = doingJobSplitting) def acquiredFiles(self): """ Set of files marked as acquired. """ return self.filesOfStatus(status = "Acquired") def completedFiles(self): """ Set of files marked as completed. """ return self.filesOfStatus(status = "Completed") def failedFiles(self): """ Set of files marked as failed. """ return self.filesOfStatus(status = "Failed")
class JobGroup(WMObject): """ JobGroups are sets of jobs running on files who's output needs to be merged together. """ def __init__(self, subscription = None, jobs = None): self.jobs = [] self.newjobs = [] self.id = 0 if type(jobs) == list: self.newjobs = jobs elif jobs != None: self.newjobs = [jobs] self.subscription = subscription self.output = Fileset() self.last_update = datetime.datetime.now() def add(self, job): """ _add_ Add a Job or list of jobs to the JobGroup. """ jobList = self.makelist(job) self.newjobs.extend(jobList) return def commit(self): """ _commit_ Move any jobs in the newjobs dict to the job dict. Empty the newjobs dict. """ self.jobs.extend(self.newjobs) self.newjobs = [] def commitBulk(self): """ Dummy method for consistency with WMBS implementation """ self.commit() def addOutput(self, file): """ _addOutput_ Add a File to the JobGroup's output fileset. The File is committed to the Fileset immediately. """ self.output.addFile(file) self.output.commit() def getJobs(self, type = "list"): """ _getJobs_ Retrieve all of the jobs in the JobGroup. The output will either be returned as a list of Job objects (when type is "list") or a list of Job IDs (when type is "id"). """ if type == "list": return self.jobs elif type == "id": jobIDs = [] for job in self.jobs: jobIDs.append(job["id"]) return jobIDs else: print "Unknown type: %s" % type return def getOutput(self, type = "list"): """ _getOutput_ Retrieve all of the files that are in the JobGroup's output fileset. Type can be one of the following: list, set, lfn, id. """ return self.output.getFiles(type = type) def getLength(self, obj): """ This just gets a length for either dict or list objects """ if type(obj) == dict: return len(obj.keys()) elif type(obj) == list: return len(obj) else: return 0 def __len__(self): """ Allows use of len() on JobGroup """ return self.getLength(self.jobs) + self.getLength(self.newjobs)
class SubscriptionTest(unittest.TestCase): """ _SubscriptionTest_ Testcase for the Subscription class """ def setUp(self): """ Initial Setup for Subscription Testcase Set a dummy Subscription with a fileset composed of one file inside it and a dummy workflow using the default constructor of the Workflow class """ self.dummyFile = File('/tmp/dummyfile', 9999, 0, 0, 0) self.dummySet = set() self.dummySet.add(self.dummyFile) self.dummyFileSet = Fileset(name='SubscriptionTestFileset', files=self.dummySet) self.dummyWorkFlow = Workflow() self.dummySubscription = Subscription(fileset=self.dummyFileSet, workflow=self.dummyWorkFlow) return def tearDown(self): pass def testGetWorkflow(self): """ Testcase for the getWorkflow method of the Subscription Class """ assert self.dummySubscription['workflow'] == self.dummyWorkFlow, \ 'Couldn\'t add Workflow to Subscription' def testGetFileset(self): """ Testcase for the getFileset method of the Subscription Class """ assert self.dummyFileSet.name == self.dummySubscription['fileset'].name, \ 'Couldn\'t add Fileset to Subscription - name does not match' for x in self.dummyFileSet.listNewFiles(): assert x in self.dummySubscription['fileset'].newfiles, \ 'Couldn\'t add Fileset to Subscription - newFiles Set does not match' assert self.dummyFileSet.getFiles(type='set') == \ self.dummySubscription['fileset'].getFiles(type='set'), \ 'Couldn\'t add Fileset to Subscription - %s Set does not match' % x def testAcquireFiles(self): """ Testcase for the acquireFiles method of the Subscription Class """ # Cleaning possible files already occupying the available set self.dummySubscription.acquireFiles() # First test - Test if initial file (on available set) is inserted in the # acquired set - no arguments dummyFile2 = File('/tmp/dummyfile2,8888', 1, 1, 1) # Insert dummyFile2 into the available files Set at dummySubscription self.dummySubscription.available.addFile(dummyFile2) S = self.dummySubscription.available.listNewFiles() # Check if Set returned by method is the same that was at the previous # available FileSet assert S == self.dummySubscription.acquireFiles(), \ 'Couldn\'t acquire file using method acquireFiles - (no arguments test)' # Second test - Test if target files are inserted at the acquired set dummyFileList = set() # Populating the dummy List with a random number of files for i in range(1, random.randint(100, 1000)): lfn = '/store/data/%s/%s/file.root' % (random.randint( 1000, 9999), random.randint(1000, 9999)) size = random.randint(1000, 2000) events = 1000 run = random.randint(0, 2000) lumi = random.randint(0, 8) file = File(lfn=lfn, size=size, events=events, checksums={"cksum": "1"}) file.addRun(Run(run, *[lumi])) dummyFileList.add(file) # Check if return value is correct - with parameters acqFiles = self.dummySubscription.acquireFiles(files=dummyFileList) assert acqFiles == dummyFileList, \ 'Return value for acquireFiles method not the acquired files' # Check if all files were inserted at subscription acquired files Set for x in dummyFileList: assert x in self.dummySubscription.acquired.getFiles(type='set'), \ 'Couldn\'t acquire File %s' % x.dict['lfn'] # Third test - Test if a replicate file is erased from the other Sets, # when a file is acquired dummyFile3 = File('/tmp/dummyfile3,5555', 1, 1, 1) dummyFileList = [] dummyFileList.append(dummyFile3) # Inserting dummyFile3 to be used as an argument, into each of the other file sets self.dummySubscription.available.addFile(dummyFile3) self.dummySubscription.failed.addFile(dummyFile3) self.dummySubscription.completed.addFile(dummyFile3) # Run the method acquireFiles self.dummySubscription.acquireFiles(files=dummyFileList, size=1) # Check if dummyFile3 was inserted at the acquired Set assert dummyFile3 in self.dummySubscription.acquired.getFiles(type='set'), \ 'Replicated file could\'nt be inserted at acquired Set' # Check if dummyFile3 was erased from all the other Sets assert dummyFile3 not in self.dummySubscription.available.getFiles(type='set'), \ 'Acquired file still present at available Set' assert dummyFile3 not in self.dummySubscription.failed.getFiles(type='set'), \ 'Acquired file still present at failed Set' assert dummyFile3 not in self.dummySubscription.completed.getFiles(type='set'), \ 'Acquired file still present at completed Set' # Fourth test - Test if the method works properly if a wrong size number # is given as an argument # Case 1: size < number of files given as an argument dummyFileList = [] for i in range(90, 100): dummyFileSize = File('/tmp/dummyfile' + str(i), 7656, 1, 1, 1) dummyFileList.append(dummyFileSize) # Run the method: self.dummySubscription.acquireFiles(files=dummyFileList, size=1) # Check each file of the List for x in dummyFileList: assert x in self.dummySubscription.acquired.getFiles(type='set'), \ 'File wasn\'t acquired (lower Size argument test)' # Case 2: size = 0 # Run the method: self.dummySubscription.acquireFiles(files=dummyFileList, size=0) # Check each file of the List for x in dummyFileList: assert x in self.dummySubscription.acquired.getFiles(type='set'), \ 'File wasn\'t acquired (zero size argument test)' def testCompleteFiles(self): """ Testcase for the completeFiles method of the Subscription Class """ # Cleaning possible files already occupying the available set self.dummySubscription.completeFiles([]) # First test - Test if initial file (on available set) is inserted in the # completed set - no arguments dummyFile2 = File('/tmp/dummyfile2,8888', 1, 1, 1) # Insert dummyFile2 into the available files Set at dummySubscription self.dummySubscription.available.addFile(dummyFile2) S = self.dummySubscription.availableFiles() # complete all files self.dummySubscription.completeFiles(S) assert len(self.dummySubscription.availableFiles()) == 0, \ "completed subscription still has %s files, what's up with that?" % \ len(self.dummySubscription.availableFiles()) # Second test - Test if target files are inserted at the completed files set dummyFileList = [] # Populating the dummy List with a random number of files for i in range(1, random.randint(100, 1000)): lfn = '/store/data/%s/%s/file.root' % (random.randint( 1000, 9999), random.randint(1000, 9999)) size = random.randint(1000, 2000) events = 1000 run = random.randint(0, 2000) lumi = random.randint(0, 8) file = File(lfn=lfn, size=size, events=events, checksums={"cksum": "1"}) file.addRun(Run(run, *[lumi])) dummyFileList.append(file) # Add the new files self.dummySubscription.available.addFile(dummyFileList) # and complete them self.dummySubscription.completeFiles(files=dummyFileList) # Check if return value is correct - with parameters assert len(self.dummySubscription.availableFiles()) == 0, \ "completed subscription still has %s files, what's up with that?" % \ len(self.dummySubscription.availableFiles()) # Check if all files were inserted at subscription's completed files Set for x in dummyFileList: assert x in self.dummySubscription.completed.getFiles(type='set'), \ 'Couldn\'t make file completed %s' % x.dict['lfn'] # Third test - Test if a replicate file is erased from the other Sets, # when a file is made completed dummyFile3 = File('/tmp/dummyfile3,5555', 1, 1, 1) dummyFileList = [] dummyFileList.append(dummyFile3) # Inserting dummyFile3 to be used as an argument, into each of the other # file sets self.dummySubscription.acquired.addFile(dummyFile3) self.dummySubscription.failed.addFile(dummyFile3) self.dummySubscription.completed.addFile(dummyFile3) # Run the method completeFiles self.dummySubscription.completeFiles(files=dummyFileList) # Check if dummyFile3 was inserted at the completed Set assert dummyFile3 in self.dummySubscription.completed.getFiles(type='set'), \ 'Replicated file could\'nt be inserted at completed Set' # Check if dummyFile3 was erased from all the other Sets assert dummyFile3 not in self.dummySubscription.acquired.getFiles(type='set'), \ 'Completed file still present at acquired Set' assert dummyFile3 not in self.dummySubscription.failed.getFiles(type='set'), \ 'Completed file still present at failed Set' assert dummyFile3 not in self.dummySubscription.available.getFiles(type='set'), \ 'Completed file still present at available Set' def testFailFiles(self): """ Testcase for the failFiles method of the Subscription Class """ # Cleaning possible files already occupying the available set self.dummySubscription.failFiles([]) # First test - Test if initial file (on available set) is inserted in the # failed set - no arguments dummyFile2 = File('/tmp/dummyfile2,8888', 1, 1, 1) # Insert dummyFile2 into the available files Set at dummySubscription self.dummySubscription.available.addFile(dummyFile2) S = self.dummySubscription.availableFiles() # Fail all files self.dummySubscription.failFiles(S) assert len(self.dummySubscription.availableFiles()) == 0, \ "failed subscription still has %s files, what's up with that?" % \ len(self.dummySubscription.availableFiles()) # Second test - Test if target files are inserted at the failed set dummyFileList = [] # Populating the dummy List with a random number of files for i in range(1, random.randint(100, 1000)): lfn = '/store/data/%s/%s/file.root' % (random.randint( 1000, 9999), random.randint(1000, 9999)) size = random.randint(1000, 2000) events = 1000 run = random.randint(0, 2000) lumi = random.randint(0, 8) file = File(lfn=lfn, size=size, events=events, checksums={"cksum": "1"}) file.addRun(Run(run, *[lumi])) dummyFileList.append(file) # Add the new files self.dummySubscription.available.addFile(dummyFileList) # and fail them self.dummySubscription.failFiles(files=dummyFileList) # Check there are no files available - everything should be failed assert len(self.dummySubscription.availableFiles()) == 0, \ "failed subscription still has %s files, what's up with that?" % \ len(self.dummySubscription.availableFiles()) # Check if all files were inserted at subscription's failed files Set for x in dummyFileList: assert x in self.dummySubscription.failed.getFiles(type='set'), \ 'Couldn\'t make file failed %s' % x.dict['lfn'] # Third test - Test if a replicate file is erased from the other Sets, # when a file is considered failed dummyFile3 = File('/tmp/dummyfile3,5555', 1, 1, 1) dummyFileList = [] dummyFileList.append(dummyFile3) # Inserting dummyFile3 to be used as an argument, into each of the other # file sets self.dummySubscription.acquired.addFile(dummyFile3) self.dummySubscription.available.addFile(dummyFile3) self.dummySubscription.completed.addFile(dummyFile3) # Run the method failFiles self.dummySubscription.failFiles(files=dummyFileList) # Check if dummyFile3 was inserted at the failed Set assert dummyFile3 in self.dummySubscription.failed.getFiles(type='set'), \ 'Replicated file could\'nt be inserted at failed Set' # Check if dummyFile3 was erased from all the other Sets assert dummyFile3 not in self.dummySubscription.acquired.getFiles(type='set'), \ 'Failed file still present at acquired Set' assert dummyFile3 not in self.dummySubscription.completed.getFiles(type='set'), \ 'Failed file still present at completed Set' assert dummyFile3 not in self.dummySubscription.available.getFiles(type='set'), \ 'Failed file still present at available Set' def testFilesOfStatus(self): """ Testcase for the filesOfStatus method of the Subscription Class """ assert self.dummySubscription.filesOfStatus('Available') == \ self.dummySubscription.available.getFiles(type='set') - \ self.dummySubscription.acquiredFiles() | self.dummySubscription.completedFiles() | self.dummySubscription.failedFiles(), \ 'Method fileOfStatus(\'AvailableFiles\') does not return available files set' assert self.dummySubscription.filesOfStatus('Acquired') == self.dummySubscription.acquired.getFiles(type='set'), \ 'Method fileOfStatus(\'AcquiredFiles\') does not return acquired files set' assert self.dummySubscription.filesOfStatus('Completed') == self.dummySubscription.completed.getFiles( type='set'), \ 'Method fileOfStatus(\'CompletedFiles\') does not return completed files set' assert self.dummySubscription.filesOfStatus('Failed') == self.dummySubscription.failed.getFiles(type='set'), \ 'Method fileOfStatus(\'FailedFiles\') does not return failed files set' def testAvailableFiles(self): """ Testcase for the availableFiles method of the Subscription Class """ assert self.dummySubscription.availableFiles() == \ self.dummySubscription.available.getFiles(type='set'), \ 'Method availableFiles does not return available files Set' def testAcquiredFiles(self): """ Testcase for the acquiredFiles method of the Subscription Class """ assert self.dummySubscription.acquiredFiles() == \ self.dummySubscription.acquired.getFiles(type='set'), \ 'Method acquiredFiles does not return acquired files Set' def testCompletedFiles(self): """ Testcase for the completedFiles method of the Subscription Class """ assert self.dummySubscription.completedFiles() == \ self.dummySubscription.completed.getFiles(type='set'), \ 'Method completedFiles does not return completed files Set' def testFailedFiles(self): """ Testcase for the failedFiles method of the Subscription Class """ assert self.dummySubscription.failedFiles() == \ self.dummySubscription.failed.getFiles(type='set'), \ 'Method failedFiles does not return failed files Set'
class FilesetTest(unittest.TestCase): """ _FilesetTest_ Testcase for Fileset """ def setUp(self): """ Create a dummy fileset and populate it with random files, in order to use it for the testcase methods """ logging.basicConfig( level=logging.DEBUG, format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s', datefmt='%m-%d %H:%M', filename=__file__.replace('.py', '.log'), filemode='w') self.logger = logging.getLogger('FilesetClassTest') #Setup the initial testcase environment: initialfile = File('/tmp/lfn1', 1000, 1, 1, 1) self.initialSet = set() self.initialSet.add(initialfile) #Create a Fileset, containing a initial file on it. self.fileset = Fileset(name='self.fileset', files=self.initialSet) #Populate the fileset with random files for i in range(1, 1000): lfn = '/store/data/%s/%s/file.root' % (random.randint( 1000, 9999), random.randint(1000, 9999)) size = random.randint(1000, 2000) events = 1000 run = random.randint(0, 2000) lumi = random.randint(0, 8) file = File(lfn=lfn, size=size, events=events, checksums={"cksum": "1"}) file.addRun(Run(run, *[lumi])) self.fileset.addFile(file) def tearDown(self): """ No tearDown method for this testcase """ pass def testAddFile(self): """ _testAddFile_ Testcase for the addFile method of the Fileset class """ # First test - Add file and check if it's there testfile = File('/tmp/lfntest', 9999, 9, 9) self.fileset.addFile(testfile) self.assertTrue( testfile in self.fileset.listNewFiles(), "Couldn't add file to fileset - fileset.addfile method not working" ) # Second test - Add file that was already at Fileset.files, and check if it gets updated testFileSame = File('/tmp/lfntest', 9999, 9, 9) testFileSame.setLocation(set('dummyse.dummy.com')) self.fileset.addFile(testFileSame) self.assertTrue( testFileSame in self.fileset.getFiles(), 'Same file copy ailed - fileset.addFile not updating location of already existing files' ) self.assertTrue( testfile in self.fileset.getFiles(), 'Same file copy failed - fileset.addFile unable to remove previous file from list' ) # Third test - Add file that was already at Fileset.newfiles, and check if it gets updated self.assertTrue( testFileSame in self.fileset.listNewFiles(), 'Same file copy failed - fileset.addFile not adding file to fileset.newFiles' ) def testListFiles(self): """ _testListFiles_ Testcase for the getFiles() method of the Fileset class """ filesettemp = self.fileset.getFiles() for x in filesettemp: assert x in (self.fileset.files | self.fileset.newfiles), \ 'Missing file %s from file list returned from fileset.ListFiles' % x["lfn"] def testSetFiles(self): """ Check that all files returned by the set are the same as those added to the fileset """ filesettemp = self.fileset.getFiles(type="set") for x in filesettemp: assert x in (self.fileset.files | self.fileset.newfiles), \ 'Missing file %s from file list returned from fileset.ListFiles' % x["lfn"] def testSetListCompare(self): """ Test that all files in fileset.setFiles are in fileset.getFiles() """ thelist = self.fileset.getFiles() theset = self.fileset.getFiles(type="set") for x in thelist: assert x in (theset), \ 'Missing file %s from file list returned from fileset.ListFiles' % x["lfn"] def testSorting(self): """ Fileset.getFiles() should be sorted the same as Fileset.getFiles(type = "lfn"), assert that this is the case here. """ files = self.fileset.getFiles() lfns = self.fileset.getFiles(type="lfn") for x in files: assert x["lfn"] in (lfns), \ 'Missing file %s from file list returned ' % x["lfn"] assert lfns[files.index(x)] == x["lfn"], \ 'Sorting not consistent: lfn = %s, file = %s' % (lfns[files.index(x)], x["lfn"]) def testListLFNs(self): """ Testcase for the listLFN method of the Fileset class """ #Kinda slow way of verifying if the raw LFN from each file at the #fileset is returned from the meth allFiles = self.fileset.getFiles() #For each file returned by method listFiles, it checks if LFN is #present at the output of method listLFNs for x in allFiles: assert x['lfn'] in self.fileset.getFiles(type = "lfn"), 'Missing %s from ' \ 'list returned from fileset.ListLFNs' % x["lfn"] #Im a bit confused with this method, leave to discuss it at the meeting with Simon def testListNewFiles(self): """ Testcase for the listNewFiles method of the Fileset class """ newfilestemp = self.fileset.listNewFiles() assert newfilestemp == self.fileset.newfiles, 'Missing files from ' \ 'list returned from fileset.ListNewFiles' def testCommit(self): """ Testcase for the commit method of the Fileset class """ localTestFileSet = Fileset('LocalTestFileset', self.initialSet) fsSize = len(localTestFileSet.getFiles(type="lfn")) #Dummy file to test fileTestCommit = File('/tmp/filetestcommit', 0000, 1, 1) #File is added to the newfiles attribute of localTestFileSet localTestFileSet.addFile(fileTestCommit) assert fsSize == len(localTestFileSet.getFiles(type = "lfn")) - 1, 'file not added'\ 'correctly to test fileset' newfilestemp = localTestFileSet.newfiles assert fileTestCommit in newfilestemp, 'test file not in the new files'\ 'list' #After commit, dummy file is supposed to move from newfiles to files localTestFileSet.commit() #First, testing if the new file is present at file set object attribute of the Fileset object assert newfilestemp.issubset(localTestFileSet.files), 'Test file not ' \ 'present at fileset.files - fileset.commit ' \ 'not working properly' #Second, testing if the newfile set object attribute is empty assert localTestFileSet.newfiles == set(), \ 'Test file not present at fileset.newfiles ' \ '- fileset.commit not working properly'
class SubscriptionTest(unittest.TestCase): """ _SubscriptionTest_ Testcase for the Subscription class """ def setUp(self): """ Initial Setup for Subscription Testcase Set a dummy Subscription with a fileset composed of one file inside it and a dummy workflow using the default constructor of the Workflow class """ self.dummyFile = File('/tmp/dummyfile',9999,0,0,0) self.dummySet = set() self.dummySet.add(self.dummyFile) self.dummyFileSet = Fileset(name = 'SubscriptionTestFileset', files = self.dummySet) self.dummyWorkFlow = Workflow() self.dummySubscription = Subscription(fileset = self.dummyFileSet, workflow = self.dummyWorkFlow) return def tearDown(self): pass def testGetWorkflow(self): """ Testcase for the getWorkflow method of the Subscription Class """ assert self.dummySubscription['workflow'] == self.dummyWorkFlow, \ 'Couldn\'t add Workflow to Subscription' def testGetFileset(self): """ Testcase for the getFileset method of the Subscription Class """ assert self.dummyFileSet.name == self.dummySubscription['fileset'].name, \ 'Couldn\'t add Fileset to Subscription - name does not match' for x in self.dummyFileSet.listNewFiles(): assert x in self.dummySubscription['fileset'].newfiles, \ 'Couldn\'t add Fileset to Subscription - newFiles Set does not match' assert self.dummyFileSet.getFiles(type='set') == \ self.dummySubscription['fileset'].getFiles(type='set'), \ 'Couldn\'t add Fileset to Subscription - %s Set does not match' % x def testAcquireFiles(self): """ Testcase for the acquireFiles method of the Subscription Class """ #Cleaning possible files already occupying the available set self.dummySubscription.acquireFiles() # First test - Test if initial file (on available set) is inserted in the # acquired set - no arguments dummyFile2 = File('/tmp/dummyfile2,8888',1,1,1) #Insert dummyFile2 into the available files Set at dummySubscription self.dummySubscription.available.addFile(dummyFile2) S = self.dummySubscription.available.listNewFiles() #Check if Set returned by method is the same that was at the previous #available FileSet assert S == self.dummySubscription.acquireFiles(), \ 'Couldn\'t acquire file using method acquireFiles - (no arguments test)' #Second test - Test if target files are inserted at the acquired set dummyFileList = set() #Populating the dummy List with a random number of files for i in range(1, random.randint(100,1000)): lfn = '/store/data/%s/%s/file.root' % (random.randint(1000, 9999), random.randint(1000, 9999)) size = random.randint(1000, 2000) events = 1000 run = random.randint(0, 2000) lumi = random.randint(0, 8) file = File(lfn = lfn, size = size, events = events, checksums = {"cksum": "1"}) file.addRun(Run(run, *[lumi])) dummyFileList.add(file) #Check if return value is correct - with parameters acqFiles = self.dummySubscription.acquireFiles(files = dummyFileList) assert acqFiles == dummyFileList,\ 'Return value for acquireFiles method not the acquired files' #Check if all files were inserted at subscription acquired files Set for x in dummyFileList: assert x in self.dummySubscription.acquired.getFiles(type='set'), \ 'Couldn\'t acquire File %s' % x.dict['lfn'] #Third test - Test if a replicate file is erased from the other Sets, #when a file is acquired dummyFile3 = File('/tmp/dummyfile3,5555',1,1,1) dummyFileList = [] dummyFileList.append(dummyFile3) #Inserting dummyFile3 to be used as an argument, into each of the other file sets self.dummySubscription.available.addFile(dummyFile3) self.dummySubscription.failed.addFile(dummyFile3) self.dummySubscription.completed.addFile(dummyFile3) #Run the method acquireFiles self.dummySubscription.acquireFiles(files=dummyFileList, size=1) #Check if dummyFile3 was inserted at the acquired Set assert dummyFile3 in self.dummySubscription.acquired.getFiles(type='set'), \ 'Replicated file could\'nt be inserted at acquired Set' #Check if dummyFile3 was erased from all the other Sets assert dummyFile3 not in self.dummySubscription.available.getFiles(type='set'), \ 'Acquired file still present at available Set' assert dummyFile3 not in self.dummySubscription.failed.getFiles(type='set'), \ 'Acquired file still present at failed Set' assert dummyFile3 not in self.dummySubscription.completed.getFiles(type='set'), \ 'Acquired file still present at completed Set' #Fourth test - Test if the method works properly if a wrong size number #is given as an argument #Case 1: size < number of files given as an argument dummyFileList = [] for i in range(90,100): dummyFileSize = File('/tmp/dummyfile'+str(i),7656,1,1,1) dummyFileList.append(dummyFileSize) #Run the method: self.dummySubscription.acquireFiles(files=dummyFileList, size=1) #Check each file of the List for x in dummyFileList: assert x in self.dummySubscription.acquired.getFiles(type='set'), \ 'File wasn\'t acquired (lower Size argument test)' #Case 2: size = 0 #Run the method: self.dummySubscription.acquireFiles(files=dummyFileList, size=0) #Check each file of the List for x in dummyFileList: assert x in self.dummySubscription.acquired.getFiles(type='set'), \ 'File wasn\'t acquired (zero size argument test)' def testCompleteFiles(self): """ Testcase for the completeFiles method of the Subscription Class """ #Cleaning possible files already occupying the available set self.dummySubscription.completeFiles([]) #First test - Test if initial file (on available set) is inserted in the #completed set - no arguments dummyFile2 = File('/tmp/dummyfile2,8888',1,1,1) #Insert dummyFile2 into the available files Set at dummySubscription self.dummySubscription.available.addFile(dummyFile2) S = self.dummySubscription.availableFiles() #complete all files self.dummySubscription.completeFiles(S) assert len(self.dummySubscription.availableFiles()) == 0, \ "completed subscription still has %s files, what's up with that?" %\ len(self.dummySubscription.availableFiles()) #Second test - Test if target files are inserted at the completed files set dummyFileList = [] #Populating the dummy List with a random number of files for i in range(1, random.randint(100,1000)): lfn = '/store/data/%s/%s/file.root' % (random.randint(1000, 9999), random.randint(1000, 9999)) size = random.randint(1000, 2000) events = 1000 run = random.randint(0, 2000) lumi = random.randint(0, 8) file = File(lfn = lfn, size = size, events = events, checksums = {"cksum": "1"}) file.addRun(Run(run, *[lumi])) dummyFileList.append(file) #Add the new files self.dummySubscription.available.addFile(dummyFileList) #and complete them self.dummySubscription.completeFiles(files = dummyFileList) #Check if return value is correct - with parameters assert len(self.dummySubscription.availableFiles()) == 0, \ "completed subscription still has %s files, what's up with that?" %\ len(self.dummySubscription.availableFiles()) #Check if all files were inserted at subscription's completed files Set for x in dummyFileList: assert x in self.dummySubscription.completed.getFiles(type='set'), \ 'Couldn\'t make file completed %s' % x.dict['lfn'] #Third test - Test if a replicate file is erased from the other Sets, #when a file is made completed dummyFile3 = File('/tmp/dummyfile3,5555',1,1,1) dummyFileList = [] dummyFileList.append(dummyFile3) #Inserting dummyFile3 to be used as an argument, into each of the other #file sets self.dummySubscription.acquired.addFile(dummyFile3) self.dummySubscription.failed.addFile(dummyFile3) self.dummySubscription.completed.addFile(dummyFile3) #Run the method completeFiles self.dummySubscription.completeFiles(files=dummyFileList) #Check if dummyFile3 was inserted at the completed Set assert dummyFile3 in self.dummySubscription.completed.getFiles(type='set'), \ 'Replicated file could\'nt be inserted at completed Set' #Check if dummyFile3 was erased from all the other Sets assert dummyFile3 not in self.dummySubscription.acquired.getFiles(type='set'), \ 'Completed file still present at acquired Set' assert dummyFile3 not in self.dummySubscription.failed.getFiles(type='set'), \ 'Completed file still present at failed Set' assert dummyFile3 not in self.dummySubscription.available.getFiles(type='set'), \ 'Completed file still present at available Set' def testFailFiles(self): """ Testcase for the failFiles method of the Subscription Class """ #Cleaning possible files already occupying the available set self.dummySubscription.failFiles([]) #First test - Test if initial file (on available set) is inserted in the # failed set - no arguments dummyFile2 = File('/tmp/dummyfile2,8888',1,1,1) #Insert dummyFile2 into the available files Set at dummySubscription self.dummySubscription.available.addFile(dummyFile2) S = self.dummySubscription.availableFiles() # Fail all files self.dummySubscription.failFiles(S) assert len(self.dummySubscription.availableFiles()) == 0, \ "failed subscription still has %s files, what's up with that?" %\ len(self.dummySubscription.availableFiles()) #Second test - Test if target files are inserted at the failed set dummyFileList = [] #Populating the dummy List with a random number of files for i in range(1, random.randint(100,1000)): lfn = '/store/data/%s/%s/file.root' % (random.randint(1000, 9999), random.randint(1000, 9999)) size = random.randint(1000, 2000) events = 1000 run = random.randint(0, 2000) lumi = random.randint(0, 8) file = File(lfn = lfn, size = size, events = events, checksums = {"cksum": "1"}) file.addRun(Run(run, *[lumi])) dummyFileList.append(file) #Add the new files self.dummySubscription.available.addFile(dummyFileList) #and fail them self.dummySubscription.failFiles(files = dummyFileList) #Check there are no files available - everything should be failed assert len(self.dummySubscription.availableFiles()) == 0, \ "failed subscription still has %s files, what's up with that?" %\ len(self.dummySubscription.availableFiles()) #Check if all files were inserted at subscription's failed files Set for x in dummyFileList: assert x in self.dummySubscription.failed.getFiles(type='set'),\ 'Couldn\'t make file failed %s' % x.dict['lfn'] #Third test - Test if a replicate file is erased from the other Sets, # when a file is considered failed dummyFile3 = File('/tmp/dummyfile3,5555',1,1,1) dummyFileList = [] dummyFileList.append(dummyFile3) #Inserting dummyFile3 to be used as an argument, into each of the other # file sets self.dummySubscription.acquired.addFile(dummyFile3) self.dummySubscription.available.addFile(dummyFile3) self.dummySubscription.completed.addFile(dummyFile3) #Run the method failFiles self.dummySubscription.failFiles(files=dummyFileList) #Check if dummyFile3 was inserted at the failed Set assert dummyFile3 in self.dummySubscription.failed.getFiles(type='set'), \ 'Replicated file could\'nt be inserted at failed Set' #Check if dummyFile3 was erased from all the other Sets assert dummyFile3 not in self.dummySubscription.acquired.getFiles(type='set'), \ 'Failed file still present at acquired Set' assert dummyFile3 not in self.dummySubscription.completed.getFiles(type='set'), \ 'Failed file still present at completed Set' assert dummyFile3 not in self.dummySubscription.available.getFiles(type='set'), \ 'Failed file still present at available Set' def testFilesOfStatus(self): """ Testcase for the filesOfStatus method of the Subscription Class """ assert self.dummySubscription.filesOfStatus('Available') == \ self.dummySubscription.available.getFiles(type='set') - \ self.dummySubscription.acquiredFiles() | self.dummySubscription.completedFiles() | self.dummySubscription.failedFiles(), \ 'Method fileOfStatus(\'AvailableFiles\') does not return available files set' assert self.dummySubscription.filesOfStatus('Acquired') == self.dummySubscription.acquired.getFiles(type='set'), \ 'Method fileOfStatus(\'AcquiredFiles\') does not return acquired files set' assert self.dummySubscription.filesOfStatus('Completed') == self.dummySubscription.completed.getFiles(type='set'), \ 'Method fileOfStatus(\'CompletedFiles\') does not return completed files set' assert self.dummySubscription.filesOfStatus('Failed') == self.dummySubscription.failed.getFiles(type='set'), \ 'Method fileOfStatus(\'FailedFiles\') does not return failed files set' def testAvailableFiles(self): """ Testcase for the availableFiles method of the Subscription Class """ assert self.dummySubscription.availableFiles() == \ self.dummySubscription.available.getFiles(type='set'), \ 'Method availableFiles does not return available files Set' def testAcquiredFiles(self): """ Testcase for the acquiredFiles method of the Subscription Class """ assert self.dummySubscription.acquiredFiles() == \ self.dummySubscription.acquired.getFiles(type='set'), \ 'Method acquiredFiles does not return acquired files Set' def testCompletedFiles(self): """ Testcase for the completedFiles method of the Subscription Class """ assert self.dummySubscription.completedFiles() == \ self.dummySubscription.completed.getFiles(type='set'), \ 'Method completedFiles does not return completed files Set' def testFailedFiles(self): """ Testcase for the failedFiles method of the Subscription Class """ assert self.dummySubscription.failedFiles() == \ self.dummySubscription.failed.getFiles(type='set'), \ 'Method failedFiles does not return failed files Set'
class FilesetTest (unittest.TestCase): """ _FilesetTest_ Testcase for Fileset """ def setUp(self): """ Create a dummy fileset and populate it with random files, in order to use it for the testcase methods """ logging.basicConfig(level=logging.DEBUG, format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s', datefmt='%m-%d %H:%M', filename=__file__.replace('.py','.log'), filemode='w') self.logger = logging.getLogger('FilesetClassTest') #Setup the initial testcase environment: initialfile = File('/tmp/lfn1',1000,1,1,1) self.initialSet = set() self.initialSet.add(initialfile) #Create a Fileset, containing a initial file on it. self.fileset = Fileset(name = 'self.fileset', files = self.initialSet) #Populate the fileset with random files for i in range(1,1000): lfn = '/store/data/%s/%s/file.root' % (random.randint(1000, 9999), random.randint(1000, 9999)) size = random.randint(1000, 2000) events = 1000 run = random.randint(0, 2000) lumi = random.randint(0, 8) file = File(lfn=lfn, size=size, events=events, checksums = {"cksum": "1"}) file.addRun(Run(run, *[lumi])) self.fileset.addFile(file) def tearDown(self): """ No tearDown method for this testcase """ pass def testAddFile(self): """ Testcase for the addFile method of the Fileset class """ #First test - Add file and check if its there testfile = File('/tmp/lfntest',9999,9,9) self.fileset.addFile(testfile) assert(testfile in self.fileset.listNewFiles(), 'Couldn\'t add file ' + 'to fileset - fileset.addfile method not working') #Second test - Add file that was already at Fileset.files , # and check if it gets updated testFileSame = File('/tmp/lfntest',9999,9,9) testFileSame.setLocation(set('dummyse.dummy.com')) self.fileset.addFile(testFileSame) assert(testFileSame in self.fileset.getFiles(),'Same file copy ' + 'failed - fileset.addFile not updating location of already ' + 'existing files' ) assert(testfile in self.fileset.getFiles(),'Same file copy ' + 'failed - fileset.addFile unable to remove previous file ' + 'from list') #Third test - Add file that was already at Fileset.newfiles , #and check if it gets updated assert(testFileSame in self.fileset.listNewFiles(),'Same file copy ' + 'failed - fileset.addFile not adding file to fileset.newFiles') def testListFiles(self): """ _testListFiles_ Testcase for the getFiles() method of the Fileset class """ filesettemp = self.fileset.getFiles() for x in filesettemp: assert x in (self.fileset.files | self.fileset.newfiles), \ 'Missing file %s from file list returned from fileset.ListFiles' % x["lfn"] def testSetFiles(self): """ Check that all files returned by the set are the same as those added to the fileset """ filesettemp = self.fileset.getFiles(type = "set") for x in filesettemp: assert x in (self.fileset.files | self.fileset.newfiles), \ 'Missing file %s from file list returned from fileset.ListFiles' % x["lfn"] def testSetListCompare(self): """ Test that all files in fileset.setFiles are in fileset.getFiles() """ thelist = self.fileset.getFiles() theset = self.fileset.getFiles(type = "set") for x in thelist: assert x in (theset), \ 'Missing file %s from file list returned from fileset.ListFiles' % x["lfn"] def testSorting(self): """ Fileset.getFiles() should be sorted the same as Fileset.getFiles(type = "lfn"), assert that this is the case here. """ files = self.fileset.getFiles() lfns = self.fileset.getFiles(type = "lfn") for x in files: assert x["lfn"] in (lfns), \ 'Missing file %s from file list returned ' % x["lfn"] assert lfns[files.index(x)] == x["lfn"], \ 'Sorting not consistent: lfn = %s, file = %s' % (lfns[files.index(x)], x["lfn"]) def testListLFNs(self): """ Testcase for the listLFN method of the Fileset class """ #Kinda slow way of verifying if the raw LFN from each file at the #fileset is returned from the meth allFiles = self.fileset.getFiles() #For each file returned by method listFiles, it checks if LFN is #present at the output of method listLFNs for x in allFiles: assert x['lfn'] in self.fileset.getFiles(type = "lfn"), 'Missing %s from ' \ 'list returned from fileset.ListLFNs' % x["lfn"] #Im a bit confused with this method, leave to discuss it at the meeting with Simon def testListNewFiles(self): """ Testcase for the listNewFiles method of the Fileset class """ newfilestemp = self.fileset.listNewFiles() assert newfilestemp == self.fileset.newfiles, 'Missing files from ' \ 'list returned from fileset.ListNewFiles' def testCommit(self): """ Testcase for the commit method of the Fileset class """ localTestFileSet = Fileset('LocalTestFileset', self.initialSet) fsSize = len(localTestFileSet.getFiles(type = "lfn")) #Dummy file to test fileTestCommit = File('/tmp/filetestcommit',0000,1,1) #File is added to the newfiles attribute of localTestFileSet localTestFileSet.addFile(fileTestCommit) assert fsSize == len(localTestFileSet.getFiles(type = "lfn")) - 1, 'file not added'\ 'correctly to test fileset' newfilestemp = localTestFileSet.newfiles assert fileTestCommit in newfilestemp, 'test file not in the new files'\ 'list' #After commit, dummy file is supposed to move from newfiles to files localTestFileSet.commit() #First, testing if the new file is present at file set object attribute of the Fileset object assert newfilestemp.issubset(localTestFileSet.files), 'Test file not ' \ 'present at fileset.files - fileset.commit ' \ 'not working properly' #Second, testing if the newfile set object attribute is empty assert localTestFileSet.newfiles == set(), \ 'Test file not present at fileset.newfiles ' \ '- fileset.commit not working properly'
class JobGroup(WMObject): """ JobGroups are sets of jobs running on files who's output needs to be merged together. """ def __init__(self, subscription=None, jobs=None): self.jobs = [] self.newjobs = [] self.id = 0 if isinstance(jobs, list): self.newjobs = jobs elif jobs is not None: self.newjobs = [jobs] self.subscription = subscription self.output = Fileset() self.last_update = datetime.datetime.now() def add(self, job): """ _add_ Add a Job or list of jobs to the JobGroup. """ jobList = self.makelist(job) self.newjobs.extend(jobList) return def commit(self): """ _commit_ Move any jobs in the newjobs dict to the job dict. Empty the newjobs dict. """ self.jobs.extend(self.newjobs) self.newjobs = [] def commitBulk(self): """ Dummy method for consistency with WMBS implementation """ self.commit() def addOutput(self, file): """ _addOutput_ Add a File to the JobGroup's output fileset. The File is committed to the Fileset immediately. """ self.output.addFile(file) self.output.commit() def getJobs(self, type="list"): """ _getJobs_ Retrieve all of the jobs in the JobGroup. The output will either be returned as a list of Job objects (when type is "list") or a list of Job IDs (when type is "id"). """ if type == "list": return self.jobs elif type == "id": jobIDs = [] for job in self.jobs: jobIDs.append(job["id"]) return jobIDs else: print("Unknown type: %s" % type) return def getOutput(self, type="list"): """ _getOutput_ Retrieve all of the files that are in the JobGroup's output fileset. Type can be one of the following: list, set, lfn, id. """ return self.output.getFiles(type=type) def getLength(self, obj): """ This just gets a length for either dict or list objects """ if isinstance(obj, (dict, list)): return len(obj) else: return 0 def __len__(self): """ Allows use of len() on JobGroup """ return self.getLength(self.jobs) + self.getLength(self.newjobs)
class Subscription(Pickleable, dict): def __init__(self, fileset=None, workflow=None, split_algo="FileBased", type="Processing"): if fileset == None: fileset = Fileset() self.setdefault('fileset', fileset) self.setdefault('workflow', workflow) self.setdefault('type', type) self.setdefault('split_algo', split_algo) self.available = Fileset(name=fileset.name, files=fileset.getFiles()) self.acquired = Fileset(name='acquired') self.completed = Fileset(name='completed') self.failed = Fileset(name='failed') def name(self): return self.getWorkflow().name.replace(' ', '') + '_' + \ self.getFileset().name.replace(' ', '') def getWorkflow(self): return self["workflow"] def workflowName(self): if self["workflow"] == None: return "Unknown" return self["workflow"].name def workflowType(self): if self["workflow"] == None: return "Unknown" return self["workflow"].wfType def taskName(self): if self['workflow'] == None: return "Unknown" return self['workflow'].task def owner(self): if self['workflow'] == None: return 'Unknown' return self['workflow'].owner def getFileset(self): return self['fileset'] def acquireFiles(self, files=[], size=1): """ Return the files acquired """ self.acquired.commit() self.available.commit() self.failed.commit() self.completed.commit() retval = [] if len(files): for i in files: # Check each set, instead of elif, just in case something has # got out of synch if i in self.available.files: self.available.files.remove(i) if i in self.failed.files: self.failed.files.remove(i) if i in self.completed.files: self.completed.files.remove(i) self.acquired.addFile(i) else: if len(self.available.files) < size or size == 0: size = len(self.available.files) for i in range(size): self.acquired.addFile(self.available.files.pop()) return self.acquired.listNewFiles() def completeFiles(self, files): """ Return the number of files complete """ self.acquired.commit() self.available.commit() self.failed.commit() self.completed.commit() for i in files: # Check each set, instead of elif, just in case something has # got out of synch if i in self.available.files: self.available.files.remove(i) if i in self.failed.files: self.failed.files.remove(i) if i in self.acquired.files: self.acquired.files.remove(i) self.completed.addFile(i) def failFiles(self, files): """ Return the number of files failed """ self.acquired.commit() self.available.commit() self.failed.commit() self.completed.commit() for i in files: # Check each set, instead of elif, just in case something has # got out of synch if i in self.available.files: self.available.files.remove(i) if i in self.completed.files: self.completed.files.remove(i) if i in self.acquired.files: self.acquired.files.remove(i) self.failed.addFile(i) def filesOfStatus(self, status=None, doingJobSplitting=False): """ _filesOfStatus_ Return a Set of File objects that are associated with the subscription and have a particular status. """ status = status.title() if status == 'Available': return self.available.getFiles(type='set') - \ (self.acquiredFiles() | self.completedFiles() | self.failedFiles()) elif status == 'Acquired': return self.acquired.getFiles(type='set') elif status == 'Completed': return self.completed.getFiles(type='set') elif status == 'Failed': return self.failed.getFiles(type='set') def availableFiles(self, limit=None, doingJobSplitting=False): """ _availableFiles_ Return a Set of files that are available for processing (e.g. not already in use) """ if limit: return list( self.filesOfStatus( status="Available", doingJobSplitting=doingJobSplitting))[:limit] else: return self.filesOfStatus(status="Available", doingJobSplitting=doingJobSplitting) def acquiredFiles(self): """ Set of files marked as acquired. """ return self.filesOfStatus(status="Acquired") def completedFiles(self): """ Set of files marked as completed. """ return self.filesOfStatus(status="Completed") def failedFiles(self): """ Set of files marked as failed. """ return self.filesOfStatus(status="Failed")