def testCommit(self):
"""
Testcase for the commit method of the Fileset class
"""
localTestFileSet = Fileset('LocalTestFileset', self.initialSet)
fsSize = len(localTestFileSet.getFiles(type="lfn"))
#Dummy file to test
fileTestCommit = File('/tmp/filetestcommit', 0000, 1, 1)
#File is added to the newfiles attribute of localTestFileSet
localTestFileSet.addFile(fileTestCommit)
assert fsSize == len(localTestFileSet.getFiles(type = "lfn")) - 1, 'file not added'\
'correctly to test fileset'
newfilestemp = localTestFileSet.newfiles
assert fileTestCommit in newfilestemp, 'test file not in the new files'\
'list'
#After commit, dummy file is supposed to move from newfiles to files
localTestFileSet.commit()
#First, testing if the new file is present at file set object attribute of the Fileset object
assert newfilestemp.issubset(localTestFileSet.files), 'Test file not ' \
'present at fileset.files - fileset.commit ' \
'not working properly'
#Second, testing if the newfile set object attribute is empty
assert localTestFileSet.newfiles == set(), \
'Test file not present at fileset.newfiles ' \
'- fileset.commit not working properly'
def testCommit(self):
"""
Testcase for the commit method of the Fileset class
"""
localTestFileSet = Fileset('LocalTestFileset', self.initialSet)
fsSize = len(localTestFileSet.getFiles(type = "lfn"))
#Dummy file to test
fileTestCommit = File('/tmp/filetestcommit',0000,1,1)
#File is added to the newfiles attribute of localTestFileSet
localTestFileSet.addFile(fileTestCommit)
assert fsSize == len(localTestFileSet.getFiles(type = "lfn")) - 1, 'file not added'\
'correctly to test fileset'
newfilestemp = localTestFileSet.newfiles
assert fileTestCommit in newfilestemp, 'test file not in the new files'\
'list'
#After commit, dummy file is supposed to move from newfiles to files
localTestFileSet.commit()
#First, testing if the new file is present at file set object attribute of the Fileset object
assert newfilestemp.issubset(localTestFileSet.files), 'Test file not ' \
'present at fileset.files - fileset.commit ' \
'not working properly'
#Second, testing if the newfile set object attribute is empty
assert localTestFileSet.newfiles == set(), \
'Test file not present at fileset.newfiles ' \
'- fileset.commit not working properly'
def __init__(self,
fileset=None,
workflow=None,
whitelist=None,
blacklist=None,
split_algo="FileBased",
type="Processing"):
if fileset == None:
fileset = Fileset()
if whitelist == None:
whitelist = set()
if blacklist == None:
blacklist = set()
self.setdefault('fileset', fileset)
self.setdefault('workflow', workflow)
self.setdefault('type', type)
self.setdefault('split_algo', split_algo)
self.setdefault('whitelist', whitelist)
self.setdefault('blacklist', blacklist)
self.available = Fileset(name=fileset.name, files=fileset.getFiles())
self.acquired = Fileset(name='acquired')
self.completed = Fileset(name='completed')
self.failed = Fileset(name='failed')
def testCall(self):
fileset = Fileset(name="FakeFeederTest")
for i in range(1, 21):
self.feeder([fileset])
set = fileset.getFiles(type = "set")
if len(set) > 0:
file = set.pop()
fileset.commit()
def testCall(self):
fileset = Fileset(name="FakeFeederTest")
for i in range(1, 21):
self.feeder([fileset])
set = fileset.getFiles(type="set")
if len(set) > 0:
file = set.pop()
fileset.commit()
def __init__(self, fileset = None, workflow = None,
split_algo = "FileBased", type = "Processing"):
if fileset == None:
fileset = Fileset()
self.setdefault('fileset', fileset)
self.setdefault('workflow', workflow)
self.setdefault('type', type)
self.setdefault('split_algo', split_algo)
self.available = Fileset(name=fileset.name,
files = fileset.getFiles())
self.acquired = Fileset(name='acquired')
self.completed = Fileset(name='completed')
self.failed = Fileset(name='failed')
class Subscription(Pickleable, dict):
def __init__(self,
fileset=None,
workflow=None,
whitelist=None,
blacklist=None,
split_algo="FileBased",
type="Processing"):
if fileset == None:
fileset = Fileset()
if whitelist == None:
whitelist = set()
if blacklist == None:
blacklist = set()
self.setdefault('fileset', fileset)
self.setdefault('workflow', workflow)
self.setdefault('type', type)
self.setdefault('split_algo', split_algo)
self.setdefault('whitelist', whitelist)
self.setdefault('blacklist', blacklist)
self.available = Fileset(name=fileset.name, files=fileset.getFiles())
self.acquired = Fileset(name='acquired')
self.completed = Fileset(name='completed')
self.failed = Fileset(name='failed')
def name(self):
return self.getWorkflow().name.replace(' ', '') + '_' + \
self.getFileset().name.replace(' ', '')
def getWorkflow(self):
return self["workflow"]
def workflowName(self):
if self["workflow"] == None:
return "Unknown"
return self["workflow"].name
def taskName(self):
if self['workflow'] == None:
return "Unknown"
return self['workflow'].task
def getFileset(self):
return self['fileset']
def acquireFiles(self, files=[], size=1):
"""
Return the files acquired
"""
self.acquired.commit()
self.available.commit()
self.failed.commit()
self.completed.commit()
retval = []
if len(files):
for i in files:
# Check each set, instead of elif, just in case something has
# got out of synch
if i in self.available.files:
self.available.files.remove(i)
if i in self.failed.files:
self.failed.files.remove(i)
if i in self.completed.files:
self.completed.files.remove(i)
self.acquired.addFile(i)
else:
if len(self.available.files) < size or size == 0:
size = len(self.available.files)
for i in range(size):
self.acquired.addFile(self.available.files.pop())
return self.acquired.listNewFiles()
def completeFiles(self, files):
"""
Return the number of files complete
"""
self.acquired.commit()
self.available.commit()
self.failed.commit()
self.completed.commit()
for i in files:
# Check each set, instead of elif, just in case something has
# got out of synch
if i in self.available.files:
self.available.files.remove(i)
if i in self.failed.files:
self.failed.files.remove(i)
if i in self.acquired.files:
self.acquired.files.remove(i)
self.completed.addFile(i)
def failFiles(self, files):
"""
Return the number of files failed
"""
self.acquired.commit()
self.available.commit()
self.failed.commit()
self.completed.commit()
for i in files:
# Check each set, instead of elif, just in case something has
# got out of synch
if i in self.available.files:
self.available.files.remove(i)
if i in self.completed.files:
self.completed.files.remove(i)
if i in self.acquired.files:
self.acquired.files.remove(i)
self.failed.addFile(i)
def filesOfStatus(self, status=None):
"""
_filesOfStatus_
Return a Set of File objects that are associated with the subscription
and have a particular status.
"""
status = status.title()
if status == 'Available':
return self.available.getFiles(type='set') - \
(self.acquiredFiles() | self.completedFiles() | self.failedFiles())
elif status == 'Acquired':
return self.acquired.getFiles(type='set')
elif status == 'Completed':
return self.completed.getFiles(type='set')
elif status == 'Failed':
return self.failed.getFiles(type='set')
def markLocation(self, location, whitelist=True):
"""
Add a location to the subscriptions white or black list
"""
if whitelist:
self['whitelist'].add(location)
else:
self['blacklist'].add(location)
def availableFiles(self):
"""
Return a Set of files that are available for processing
(e.g. not already in use) and at sites that are white listed
or not black listed
"""
def locationMagic(files, locations):
"""
files and locations are sets. method returns the subset of files
that are at the locations - this is a lot simpler with the database
"""
magicfiles = set()
for f in files:
if len(f['locations'] & locations) > 0:
magicfiles.add(f)
return magicfiles
files = self.filesOfStatus(status="Available")
if len(self['whitelist']) > 0:
# Return files at white listed sites
return locationMagic(files, self['whitelist'])
elif len(self['blacklist']) > 0:
# Return files not at black listed sites
return files - locationMagic(files, self['blacklist'])
#Return all files, because you're crazy and just don't care
return files
def acquiredFiles(self):
"""
Set of files marked as acquired.
"""
return self.filesOfStatus(status="Acquired")
def completedFiles(self):
"""
Set of files marked as completed.
"""
return self.filesOfStatus(status="Completed")
def failedFiles(self):
"""
Set of files marked as failed.
"""
return self.filesOfStatus(status="Failed")
class Subscription(Pickleable, dict):
def __init__(self, fileset = None, workflow = None,
split_algo = "FileBased", type = "Processing"):
if fileset == None:
fileset = Fileset()
self.setdefault('fileset', fileset)
self.setdefault('workflow', workflow)
self.setdefault('type', type)
self.setdefault('split_algo', split_algo)
self.available = Fileset(name=fileset.name,
files = fileset.getFiles())
self.acquired = Fileset(name='acquired')
self.completed = Fileset(name='completed')
self.failed = Fileset(name='failed')
def name(self):
return self.getWorkflow().name.replace(' ', '') + '_' + \
self.getFileset().name.replace(' ', '')
def getWorkflow(self):
return self["workflow"]
def workflowName(self):
if self["workflow"] == None:
return "Unknown"
return self["workflow"].name
def workflowType(self):
if self["workflow"] == None:
return "Unknown"
return self["workflow"].wfType
def taskName(self):
if self['workflow'] == None:
return "Unknown"
return self['workflow'].task
def owner(self):
if self['workflow'] == None:
return 'Unknown'
return self['workflow'].owner
def getFileset(self):
return self['fileset']
def acquireFiles(self, files = [], size=1):
"""
Return the files acquired
"""
self.acquired.commit()
self.available.commit()
self.failed.commit()
self.completed.commit()
retval = []
if len(files):
for i in files:
# Check each set, instead of elif, just in case something has
# got out of synch
if i in self.available.files:
self.available.files.remove(i)
if i in self.failed.files:
self.failed.files.remove(i)
if i in self.completed.files:
self.completed.files.remove(i)
self.acquired.addFile(i)
else:
if len(self.available.files) < size or size == 0:
size = len(self.available.files)
for i in range(size):
self.acquired.addFile(self.available.files.pop())
return self.acquired.listNewFiles()
def completeFiles(self, files):
"""
Return the number of files complete
"""
self.acquired.commit()
self.available.commit()
self.failed.commit()
self.completed.commit()
for i in files:
# Check each set, instead of elif, just in case something has
# got out of synch
if i in self.available.files:
self.available.files.remove(i)
if i in self.failed.files:
self.failed.files.remove(i)
if i in self.acquired.files:
self.acquired.files.remove(i)
self.completed.addFile(i)
def failFiles(self, files):
"""
Return the number of files failed
"""
self.acquired.commit()
self.available.commit()
self.failed.commit()
self.completed.commit()
for i in files:
# Check each set, instead of elif, just in case something has
# got out of synch
if i in self.available.files:
self.available.files.remove(i)
if i in self.completed.files:
self.completed.files.remove(i)
if i in self.acquired.files:
self.acquired.files.remove(i)
self.failed.addFile(i)
def filesOfStatus(self, status=None, doingJobSplitting = False):
"""
_filesOfStatus_
Return a Set of File objects that are associated with the subscription
and have a particular status.
"""
status = status.title()
if status == 'Available':
return self.available.getFiles(type='set') - \
(self.acquiredFiles() | self.completedFiles() | self.failedFiles())
elif status == 'Acquired':
return self.acquired.getFiles(type='set')
elif status == 'Completed':
return self.completed.getFiles(type='set')
elif status == 'Failed':
return self.failed.getFiles(type='set')
def availableFiles(self, limit = None, doingJobSplitting = False):
"""
_availableFiles_
Return a Set of files that are available for processing
(e.g. not already in use)
"""
if limit:
return list(self.filesOfStatus(status = "Available", doingJobSplitting = doingJobSplitting))[:limit]
else:
return self.filesOfStatus(status = "Available", doingJobSplitting = doingJobSplitting)
def acquiredFiles(self):
"""
Set of files marked as acquired.
"""
return self.filesOfStatus(status = "Acquired")
def completedFiles(self):
"""
Set of files marked as completed.
"""
return self.filesOfStatus(status = "Completed")
def failedFiles(self):
"""
Set of files marked as failed.
"""
return self.filesOfStatus(status = "Failed")
class JobGroup(WMObject):
"""
JobGroups are sets of jobs running on files who's output needs to be merged
together.
"""
def __init__(self, subscription = None, jobs = None):
self.jobs = []
self.newjobs = []
self.id = 0
if type(jobs) == list:
self.newjobs = jobs
elif jobs != None:
self.newjobs = [jobs]
self.subscription = subscription
self.output = Fileset()
self.last_update = datetime.datetime.now()
def add(self, job):
"""
_add_
Add a Job or list of jobs to the JobGroup.
"""
jobList = self.makelist(job)
self.newjobs.extend(jobList)
return
def commit(self):
"""
_commit_
Move any jobs in the newjobs dict to the job dict. Empty the newjobs
dict.
"""
self.jobs.extend(self.newjobs)
self.newjobs = []
def commitBulk(self):
"""
Dummy method for consistency with WMBS implementation
"""
self.commit()
def addOutput(self, file):
"""
_addOutput_
Add a File to the JobGroup's output fileset. The File is committed
to the Fileset immediately.
"""
self.output.addFile(file)
self.output.commit()
def getJobs(self, type = "list"):
"""
_getJobs_
Retrieve all of the jobs in the JobGroup. The output will either be
returned as a list of Job objects (when type is "list") or a list of
Job IDs (when type is "id").
"""
if type == "list":
return self.jobs
elif type == "id":
jobIDs = []
for job in self.jobs:
jobIDs.append(job["id"])
return jobIDs
else:
print "Unknown type: %s" % type
return
def getOutput(self, type = "list"):
"""
_getOutput_
Retrieve all of the files that are in the JobGroup's output fileset.
Type can be one of the following: list, set, lfn, id.
"""
return self.output.getFiles(type = type)
def getLength(self, obj):
"""
This just gets a length for either dict or list objects
"""
if type(obj) == dict:
return len(obj.keys())
elif type(obj) == list:
return len(obj)
else:
return 0
def __len__(self):
"""
Allows use of len() on JobGroup
"""
return self.getLength(self.jobs) + self.getLength(self.newjobs)
class SubscriptionTest(unittest.TestCase):
"""
_SubscriptionTest_
Testcase for the Subscription class
"""
def setUp(self):
"""
Initial Setup for Subscription Testcase
Set a dummy Subscription with a fileset composed of one file inside it
and a dummy workflow using the default constructor of the Workflow class
"""
self.dummyFile = File('/tmp/dummyfile', 9999, 0, 0, 0)
self.dummySet = set()
self.dummySet.add(self.dummyFile)
self.dummyFileSet = Fileset(name='SubscriptionTestFileset',
files=self.dummySet)
self.dummyWorkFlow = Workflow()
self.dummySubscription = Subscription(fileset=self.dummyFileSet,
workflow=self.dummyWorkFlow)
return
def tearDown(self):
pass
def testGetWorkflow(self):
"""
Testcase for the getWorkflow method of the Subscription Class
"""
assert self.dummySubscription['workflow'] == self.dummyWorkFlow, \
'Couldn\'t add Workflow to Subscription'
def testGetFileset(self):
"""
Testcase for the getFileset method of the Subscription Class
"""
assert self.dummyFileSet.name == self.dummySubscription['fileset'].name, \
'Couldn\'t add Fileset to Subscription - name does not match'
for x in self.dummyFileSet.listNewFiles():
assert x in self.dummySubscription['fileset'].newfiles, \
'Couldn\'t add Fileset to Subscription - newFiles Set does not match'
assert self.dummyFileSet.getFiles(type='set') == \
self.dummySubscription['fileset'].getFiles(type='set'), \
'Couldn\'t add Fileset to Subscription - %s Set does not match' % x
def testAcquireFiles(self):
"""
Testcase for the acquireFiles method of the Subscription Class
"""
# Cleaning possible files already occupying the available set
self.dummySubscription.acquireFiles()
# First test - Test if initial file (on available set) is inserted in the
# acquired set - no arguments
dummyFile2 = File('/tmp/dummyfile2,8888', 1, 1, 1)
# Insert dummyFile2 into the available files Set at dummySubscription
self.dummySubscription.available.addFile(dummyFile2)
S = self.dummySubscription.available.listNewFiles()
# Check if Set returned by method is the same that was at the previous
# available FileSet
assert S == self.dummySubscription.acquireFiles(), \
'Couldn\'t acquire file using method acquireFiles - (no arguments test)'
# Second test - Test if target files are inserted at the acquired set
dummyFileList = set()
# Populating the dummy List with a random number of files
for i in range(1, random.randint(100, 1000)):
lfn = '/store/data/%s/%s/file.root' % (random.randint(
1000, 9999), random.randint(1000, 9999))
size = random.randint(1000, 2000)
events = 1000
run = random.randint(0, 2000)
lumi = random.randint(0, 8)
file = File(lfn=lfn,
size=size,
events=events,
checksums={"cksum": "1"})
file.addRun(Run(run, *[lumi]))
dummyFileList.add(file)
# Check if return value is correct - with parameters
acqFiles = self.dummySubscription.acquireFiles(files=dummyFileList)
assert acqFiles == dummyFileList, \
'Return value for acquireFiles method not the acquired files'
# Check if all files were inserted at subscription acquired files Set
for x in dummyFileList:
assert x in self.dummySubscription.acquired.getFiles(type='set'), \
'Couldn\'t acquire File %s' % x.dict['lfn']
# Third test - Test if a replicate file is erased from the other Sets,
# when a file is acquired
dummyFile3 = File('/tmp/dummyfile3,5555', 1, 1, 1)
dummyFileList = []
dummyFileList.append(dummyFile3)
# Inserting dummyFile3 to be used as an argument, into each of the other file sets
self.dummySubscription.available.addFile(dummyFile3)
self.dummySubscription.failed.addFile(dummyFile3)
self.dummySubscription.completed.addFile(dummyFile3)
# Run the method acquireFiles
self.dummySubscription.acquireFiles(files=dummyFileList, size=1)
# Check if dummyFile3 was inserted at the acquired Set
assert dummyFile3 in self.dummySubscription.acquired.getFiles(type='set'), \
'Replicated file could\'nt be inserted at acquired Set'
# Check if dummyFile3 was erased from all the other Sets
assert dummyFile3 not in self.dummySubscription.available.getFiles(type='set'), \
'Acquired file still present at available Set'
assert dummyFile3 not in self.dummySubscription.failed.getFiles(type='set'), \
'Acquired file still present at failed Set'
assert dummyFile3 not in self.dummySubscription.completed.getFiles(type='set'), \
'Acquired file still present at completed Set'
# Fourth test - Test if the method works properly if a wrong size number
# is given as an argument
# Case 1: size < number of files given as an argument
dummyFileList = []
for i in range(90, 100):
dummyFileSize = File('/tmp/dummyfile' + str(i), 7656, 1, 1, 1)
dummyFileList.append(dummyFileSize)
# Run the method:
self.dummySubscription.acquireFiles(files=dummyFileList, size=1)
# Check each file of the List
for x in dummyFileList:
assert x in self.dummySubscription.acquired.getFiles(type='set'), \
'File wasn\'t acquired (lower Size argument test)'
# Case 2: size = 0
# Run the method:
self.dummySubscription.acquireFiles(files=dummyFileList, size=0)
# Check each file of the List
for x in dummyFileList:
assert x in self.dummySubscription.acquired.getFiles(type='set'), \
'File wasn\'t acquired (zero size argument test)'
def testCompleteFiles(self):
"""
Testcase for the completeFiles method of the Subscription Class
"""
# Cleaning possible files already occupying the available set
self.dummySubscription.completeFiles([])
# First test - Test if initial file (on available set) is inserted in the
# completed set - no arguments
dummyFile2 = File('/tmp/dummyfile2,8888', 1, 1, 1)
# Insert dummyFile2 into the available files Set at dummySubscription
self.dummySubscription.available.addFile(dummyFile2)
S = self.dummySubscription.availableFiles()
# complete all files
self.dummySubscription.completeFiles(S)
assert len(self.dummySubscription.availableFiles()) == 0, \
"completed subscription still has %s files, what's up with that?" % \
len(self.dummySubscription.availableFiles())
# Second test - Test if target files are inserted at the completed files set
dummyFileList = []
# Populating the dummy List with a random number of files
for i in range(1, random.randint(100, 1000)):
lfn = '/store/data/%s/%s/file.root' % (random.randint(
1000, 9999), random.randint(1000, 9999))
size = random.randint(1000, 2000)
events = 1000
run = random.randint(0, 2000)
lumi = random.randint(0, 8)
file = File(lfn=lfn,
size=size,
events=events,
checksums={"cksum": "1"})
file.addRun(Run(run, *[lumi]))
dummyFileList.append(file)
# Add the new files
self.dummySubscription.available.addFile(dummyFileList)
# and complete them
self.dummySubscription.completeFiles(files=dummyFileList)
# Check if return value is correct - with parameters
assert len(self.dummySubscription.availableFiles()) == 0, \
"completed subscription still has %s files, what's up with that?" % \
len(self.dummySubscription.availableFiles())
# Check if all files were inserted at subscription's completed files Set
for x in dummyFileList:
assert x in self.dummySubscription.completed.getFiles(type='set'), \
'Couldn\'t make file completed %s' % x.dict['lfn']
# Third test - Test if a replicate file is erased from the other Sets,
# when a file is made completed
dummyFile3 = File('/tmp/dummyfile3,5555', 1, 1, 1)
dummyFileList = []
dummyFileList.append(dummyFile3)
# Inserting dummyFile3 to be used as an argument, into each of the other
# file sets
self.dummySubscription.acquired.addFile(dummyFile3)
self.dummySubscription.failed.addFile(dummyFile3)
self.dummySubscription.completed.addFile(dummyFile3)
# Run the method completeFiles
self.dummySubscription.completeFiles(files=dummyFileList)
# Check if dummyFile3 was inserted at the completed Set
assert dummyFile3 in self.dummySubscription.completed.getFiles(type='set'), \
'Replicated file could\'nt be inserted at completed Set'
# Check if dummyFile3 was erased from all the other Sets
assert dummyFile3 not in self.dummySubscription.acquired.getFiles(type='set'), \
'Completed file still present at acquired Set'
assert dummyFile3 not in self.dummySubscription.failed.getFiles(type='set'), \
'Completed file still present at failed Set'
assert dummyFile3 not in self.dummySubscription.available.getFiles(type='set'), \
'Completed file still present at available Set'
def testFailFiles(self):
"""
Testcase for the failFiles method of the Subscription Class
"""
# Cleaning possible files already occupying the available set
self.dummySubscription.failFiles([])
# First test - Test if initial file (on available set) is inserted in the
# failed set - no arguments
dummyFile2 = File('/tmp/dummyfile2,8888', 1, 1, 1)
# Insert dummyFile2 into the available files Set at dummySubscription
self.dummySubscription.available.addFile(dummyFile2)
S = self.dummySubscription.availableFiles()
# Fail all files
self.dummySubscription.failFiles(S)
assert len(self.dummySubscription.availableFiles()) == 0, \
"failed subscription still has %s files, what's up with that?" % \
len(self.dummySubscription.availableFiles())
# Second test - Test if target files are inserted at the failed set
dummyFileList = []
# Populating the dummy List with a random number of files
for i in range(1, random.randint(100, 1000)):
lfn = '/store/data/%s/%s/file.root' % (random.randint(
1000, 9999), random.randint(1000, 9999))
size = random.randint(1000, 2000)
events = 1000
run = random.randint(0, 2000)
lumi = random.randint(0, 8)
file = File(lfn=lfn,
size=size,
events=events,
checksums={"cksum": "1"})
file.addRun(Run(run, *[lumi]))
dummyFileList.append(file)
# Add the new files
self.dummySubscription.available.addFile(dummyFileList)
# and fail them
self.dummySubscription.failFiles(files=dummyFileList)
# Check there are no files available - everything should be failed
assert len(self.dummySubscription.availableFiles()) == 0, \
"failed subscription still has %s files, what's up with that?" % \
len(self.dummySubscription.availableFiles())
# Check if all files were inserted at subscription's failed files Set
for x in dummyFileList:
assert x in self.dummySubscription.failed.getFiles(type='set'), \
'Couldn\'t make file failed %s' % x.dict['lfn']
# Third test - Test if a replicate file is erased from the other Sets,
# when a file is considered failed
dummyFile3 = File('/tmp/dummyfile3,5555', 1, 1, 1)
dummyFileList = []
dummyFileList.append(dummyFile3)
# Inserting dummyFile3 to be used as an argument, into each of the other
# file sets
self.dummySubscription.acquired.addFile(dummyFile3)
self.dummySubscription.available.addFile(dummyFile3)
self.dummySubscription.completed.addFile(dummyFile3)
# Run the method failFiles
self.dummySubscription.failFiles(files=dummyFileList)
# Check if dummyFile3 was inserted at the failed Set
assert dummyFile3 in self.dummySubscription.failed.getFiles(type='set'), \
'Replicated file could\'nt be inserted at failed Set'
# Check if dummyFile3 was erased from all the other Sets
assert dummyFile3 not in self.dummySubscription.acquired.getFiles(type='set'), \
'Failed file still present at acquired Set'
assert dummyFile3 not in self.dummySubscription.completed.getFiles(type='set'), \
'Failed file still present at completed Set'
assert dummyFile3 not in self.dummySubscription.available.getFiles(type='set'), \
'Failed file still present at available Set'
def testFilesOfStatus(self):
"""
Testcase for the filesOfStatus method of the Subscription Class
"""
assert self.dummySubscription.filesOfStatus('Available') == \
self.dummySubscription.available.getFiles(type='set') - \
self.dummySubscription.acquiredFiles() | self.dummySubscription.completedFiles() | self.dummySubscription.failedFiles(), \
'Method fileOfStatus(\'AvailableFiles\') does not return available files set'
assert self.dummySubscription.filesOfStatus('Acquired') == self.dummySubscription.acquired.getFiles(type='set'), \
'Method fileOfStatus(\'AcquiredFiles\') does not return acquired files set'
assert self.dummySubscription.filesOfStatus('Completed') == self.dummySubscription.completed.getFiles(
type='set'), \
'Method fileOfStatus(\'CompletedFiles\') does not return completed files set'
assert self.dummySubscription.filesOfStatus('Failed') == self.dummySubscription.failed.getFiles(type='set'), \
'Method fileOfStatus(\'FailedFiles\') does not return failed files set'
def testAvailableFiles(self):
"""
Testcase for the availableFiles method of the Subscription Class
"""
assert self.dummySubscription.availableFiles() == \
self.dummySubscription.available.getFiles(type='set'), \
'Method availableFiles does not return available files Set'
def testAcquiredFiles(self):
"""
Testcase for the acquiredFiles method of the Subscription Class
"""
assert self.dummySubscription.acquiredFiles() == \
self.dummySubscription.acquired.getFiles(type='set'), \
'Method acquiredFiles does not return acquired files Set'
def testCompletedFiles(self):
"""
Testcase for the completedFiles method of the Subscription Class
"""
assert self.dummySubscription.completedFiles() == \
self.dummySubscription.completed.getFiles(type='set'), \
'Method completedFiles does not return completed files Set'
def testFailedFiles(self):
"""
Testcase for the failedFiles method of the Subscription Class
"""
assert self.dummySubscription.failedFiles() == \
self.dummySubscription.failed.getFiles(type='set'), \
'Method failedFiles does not return failed files Set'
class FilesetTest(unittest.TestCase):
"""
_FilesetTest_
Testcase for Fileset
"""
def setUp(self):
"""
Create a dummy fileset and populate it with random files,
in order to use it for the testcase methods
"""
logging.basicConfig(
level=logging.DEBUG,
format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
datefmt='%m-%d %H:%M',
filename=__file__.replace('.py', '.log'),
filemode='w')
self.logger = logging.getLogger('FilesetClassTest')
#Setup the initial testcase environment:
initialfile = File('/tmp/lfn1', 1000, 1, 1, 1)
self.initialSet = set()
self.initialSet.add(initialfile)
#Create a Fileset, containing a initial file on it.
self.fileset = Fileset(name='self.fileset', files=self.initialSet)
#Populate the fileset with random files
for i in range(1, 1000):
lfn = '/store/data/%s/%s/file.root' % (random.randint(
1000, 9999), random.randint(1000, 9999))
size = random.randint(1000, 2000)
events = 1000
run = random.randint(0, 2000)
lumi = random.randint(0, 8)
file = File(lfn=lfn,
size=size,
events=events,
checksums={"cksum": "1"})
file.addRun(Run(run, *[lumi]))
self.fileset.addFile(file)
def tearDown(self):
"""
No tearDown method for this testcase
"""
pass
def testAddFile(self):
"""
_testAddFile_
Testcase for the addFile method of the Fileset class
"""
# First test - Add file and check if it's there
testfile = File('/tmp/lfntest', 9999, 9, 9)
self.fileset.addFile(testfile)
self.assertTrue(
testfile in self.fileset.listNewFiles(),
"Couldn't add file to fileset - fileset.addfile method not working"
)
# Second test - Add file that was already at Fileset.files, and check if it gets updated
testFileSame = File('/tmp/lfntest', 9999, 9, 9)
testFileSame.setLocation(set('dummyse.dummy.com'))
self.fileset.addFile(testFileSame)
self.assertTrue(
testFileSame in self.fileset.getFiles(),
'Same file copy ailed - fileset.addFile not updating location of already existing files'
)
self.assertTrue(
testfile in self.fileset.getFiles(),
'Same file copy failed - fileset.addFile unable to remove previous file from list'
)
# Third test - Add file that was already at Fileset.newfiles, and check if it gets updated
self.assertTrue(
testFileSame in self.fileset.listNewFiles(),
'Same file copy failed - fileset.addFile not adding file to fileset.newFiles'
)
def testListFiles(self):
"""
_testListFiles_
Testcase for the getFiles() method of the Fileset class
"""
filesettemp = self.fileset.getFiles()
for x in filesettemp:
assert x in (self.fileset.files | self.fileset.newfiles), \
'Missing file %s from file list returned from fileset.ListFiles' % x["lfn"]
def testSetFiles(self):
"""
Check that all files returned by the set are the same as those added to
the fileset
"""
filesettemp = self.fileset.getFiles(type="set")
for x in filesettemp:
assert x in (self.fileset.files | self.fileset.newfiles), \
'Missing file %s from file list returned from fileset.ListFiles' % x["lfn"]
def testSetListCompare(self):
"""
Test that all files in fileset.setFiles are in fileset.getFiles()
"""
thelist = self.fileset.getFiles()
theset = self.fileset.getFiles(type="set")
for x in thelist:
assert x in (theset), \
'Missing file %s from file list returned from fileset.ListFiles' % x["lfn"]
def testSorting(self):
"""
Fileset.getFiles() should be sorted the same as Fileset.getFiles(type = "lfn"),
assert that this is the case here.
"""
files = self.fileset.getFiles()
lfns = self.fileset.getFiles(type="lfn")
for x in files:
assert x["lfn"] in (lfns), \
'Missing file %s from file list returned ' % x["lfn"]
assert lfns[files.index(x)] == x["lfn"], \
'Sorting not consistent: lfn = %s, file = %s' % (lfns[files.index(x)], x["lfn"])
def testListLFNs(self):
"""
Testcase for the listLFN method of the Fileset class
"""
#Kinda slow way of verifying if the raw LFN from each file at the
#fileset is returned from the meth
allFiles = self.fileset.getFiles()
#For each file returned by method listFiles, it checks if LFN is
#present at the output of method listLFNs
for x in allFiles:
assert x['lfn'] in self.fileset.getFiles(type = "lfn"), 'Missing %s from ' \
'list returned from fileset.ListLFNs' % x["lfn"]
#Im a bit confused with this method, leave to discuss it at the meeting with Simon
def testListNewFiles(self):
"""
Testcase for the listNewFiles method of the Fileset class
"""
newfilestemp = self.fileset.listNewFiles()
assert newfilestemp == self.fileset.newfiles, 'Missing files from ' \
'list returned from fileset.ListNewFiles'
def testCommit(self):
"""
Testcase for the commit method of the Fileset class
"""
localTestFileSet = Fileset('LocalTestFileset', self.initialSet)
fsSize = len(localTestFileSet.getFiles(type="lfn"))
#Dummy file to test
fileTestCommit = File('/tmp/filetestcommit', 0000, 1, 1)
#File is added to the newfiles attribute of localTestFileSet
localTestFileSet.addFile(fileTestCommit)
assert fsSize == len(localTestFileSet.getFiles(type = "lfn")) - 1, 'file not added'\
'correctly to test fileset'
newfilestemp = localTestFileSet.newfiles
assert fileTestCommit in newfilestemp, 'test file not in the new files'\
'list'
#After commit, dummy file is supposed to move from newfiles to files
localTestFileSet.commit()
#First, testing if the new file is present at file set object attribute of the Fileset object
assert newfilestemp.issubset(localTestFileSet.files), 'Test file not ' \
'present at fileset.files - fileset.commit ' \
'not working properly'
#Second, testing if the newfile set object attribute is empty
assert localTestFileSet.newfiles == set(), \
'Test file not present at fileset.newfiles ' \
'- fileset.commit not working properly'
class SubscriptionTest(unittest.TestCase):
"""
_SubscriptionTest_
Testcase for the Subscription class
"""
def setUp(self):
"""
Initial Setup for Subscription Testcase
Set a dummy Subscription with a fileset composed of one file inside it
and a dummy workflow using the default constructor of the Workflow class
"""
self.dummyFile = File('/tmp/dummyfile',9999,0,0,0)
self.dummySet = set()
self.dummySet.add(self.dummyFile)
self.dummyFileSet = Fileset(name = 'SubscriptionTestFileset',
files = self.dummySet)
self.dummyWorkFlow = Workflow()
self.dummySubscription = Subscription(fileset = self.dummyFileSet,
workflow = self.dummyWorkFlow)
return
def tearDown(self):
pass
def testGetWorkflow(self):
"""
Testcase for the getWorkflow method of the Subscription Class
"""
assert self.dummySubscription['workflow'] == self.dummyWorkFlow, \
'Couldn\'t add Workflow to Subscription'
def testGetFileset(self):
"""
Testcase for the getFileset method of the Subscription Class
"""
assert self.dummyFileSet.name == self.dummySubscription['fileset'].name, \
'Couldn\'t add Fileset to Subscription - name does not match'
for x in self.dummyFileSet.listNewFiles():
assert x in self.dummySubscription['fileset'].newfiles, \
'Couldn\'t add Fileset to Subscription - newFiles Set does not match'
assert self.dummyFileSet.getFiles(type='set') == \
self.dummySubscription['fileset'].getFiles(type='set'), \
'Couldn\'t add Fileset to Subscription - %s Set does not match' % x
def testAcquireFiles(self):
"""
Testcase for the acquireFiles method of the Subscription Class
"""
#Cleaning possible files already occupying the available set
self.dummySubscription.acquireFiles()
# First test - Test if initial file (on available set) is inserted in the
# acquired set - no arguments
dummyFile2 = File('/tmp/dummyfile2,8888',1,1,1)
#Insert dummyFile2 into the available files Set at dummySubscription
self.dummySubscription.available.addFile(dummyFile2)
S = self.dummySubscription.available.listNewFiles()
#Check if Set returned by method is the same that was at the previous
#available FileSet
assert S == self.dummySubscription.acquireFiles(), \
'Couldn\'t acquire file using method acquireFiles - (no arguments test)'
#Second test - Test if target files are inserted at the acquired set
dummyFileList = set()
#Populating the dummy List with a random number of files
for i in range(1, random.randint(100,1000)):
lfn = '/store/data/%s/%s/file.root' % (random.randint(1000, 9999),
random.randint(1000, 9999))
size = random.randint(1000, 2000)
events = 1000
run = random.randint(0, 2000)
lumi = random.randint(0, 8)
file = File(lfn = lfn, size = size, events = events,
checksums = {"cksum": "1"})
file.addRun(Run(run, *[lumi]))
dummyFileList.add(file)
#Check if return value is correct - with parameters
acqFiles = self.dummySubscription.acquireFiles(files = dummyFileList)
assert acqFiles == dummyFileList,\
'Return value for acquireFiles method not the acquired files'
#Check if all files were inserted at subscription acquired files Set
for x in dummyFileList:
assert x in self.dummySubscription.acquired.getFiles(type='set'), \
'Couldn\'t acquire File %s' % x.dict['lfn']
#Third test - Test if a replicate file is erased from the other Sets,
#when a file is acquired
dummyFile3 = File('/tmp/dummyfile3,5555',1,1,1)
dummyFileList = []
dummyFileList.append(dummyFile3)
#Inserting dummyFile3 to be used as an argument, into each of the other file sets
self.dummySubscription.available.addFile(dummyFile3)
self.dummySubscription.failed.addFile(dummyFile3)
self.dummySubscription.completed.addFile(dummyFile3)
#Run the method acquireFiles
self.dummySubscription.acquireFiles(files=dummyFileList, size=1)
#Check if dummyFile3 was inserted at the acquired Set
assert dummyFile3 in self.dummySubscription.acquired.getFiles(type='set'), \
'Replicated file could\'nt be inserted at acquired Set'
#Check if dummyFile3 was erased from all the other Sets
assert dummyFile3 not in self.dummySubscription.available.getFiles(type='set'), \
'Acquired file still present at available Set'
assert dummyFile3 not in self.dummySubscription.failed.getFiles(type='set'), \
'Acquired file still present at failed Set'
assert dummyFile3 not in self.dummySubscription.completed.getFiles(type='set'), \
'Acquired file still present at completed Set'
#Fourth test - Test if the method works properly if a wrong size number
#is given as an argument
#Case 1: size < number of files given as an argument
dummyFileList = []
for i in range(90,100):
dummyFileSize = File('/tmp/dummyfile'+str(i),7656,1,1,1)
dummyFileList.append(dummyFileSize)
#Run the method:
self.dummySubscription.acquireFiles(files=dummyFileList, size=1)
#Check each file of the List
for x in dummyFileList:
assert x in self.dummySubscription.acquired.getFiles(type='set'), \
'File wasn\'t acquired (lower Size argument test)'
#Case 2: size = 0
#Run the method:
self.dummySubscription.acquireFiles(files=dummyFileList, size=0)
#Check each file of the List
for x in dummyFileList:
assert x in self.dummySubscription.acquired.getFiles(type='set'), \
'File wasn\'t acquired (zero size argument test)'
def testCompleteFiles(self):
"""
Testcase for the completeFiles method of the Subscription Class
"""
#Cleaning possible files already occupying the available set
self.dummySubscription.completeFiles([])
#First test - Test if initial file (on available set) is inserted in the
#completed set - no arguments
dummyFile2 = File('/tmp/dummyfile2,8888',1,1,1)
#Insert dummyFile2 into the available files Set at dummySubscription
self.dummySubscription.available.addFile(dummyFile2)
S = self.dummySubscription.availableFiles()
#complete all files
self.dummySubscription.completeFiles(S)
assert len(self.dummySubscription.availableFiles()) == 0, \
"completed subscription still has %s files, what's up with that?" %\
len(self.dummySubscription.availableFiles())
#Second test - Test if target files are inserted at the completed files set
dummyFileList = []
#Populating the dummy List with a random number of files
for i in range(1, random.randint(100,1000)):
lfn = '/store/data/%s/%s/file.root' % (random.randint(1000, 9999),
random.randint(1000, 9999))
size = random.randint(1000, 2000)
events = 1000
run = random.randint(0, 2000)
lumi = random.randint(0, 8)
file = File(lfn = lfn, size = size, events = events,
checksums = {"cksum": "1"})
file.addRun(Run(run, *[lumi]))
dummyFileList.append(file)
#Add the new files
self.dummySubscription.available.addFile(dummyFileList)
#and complete them
self.dummySubscription.completeFiles(files = dummyFileList)
#Check if return value is correct - with parameters
assert len(self.dummySubscription.availableFiles()) == 0, \
"completed subscription still has %s files, what's up with that?" %\
len(self.dummySubscription.availableFiles())
#Check if all files were inserted at subscription's completed files Set
for x in dummyFileList:
assert x in self.dummySubscription.completed.getFiles(type='set'), \
'Couldn\'t make file completed %s' % x.dict['lfn']
#Third test - Test if a replicate file is erased from the other Sets,
#when a file is made completed
dummyFile3 = File('/tmp/dummyfile3,5555',1,1,1)
dummyFileList = []
dummyFileList.append(dummyFile3)
#Inserting dummyFile3 to be used as an argument, into each of the other
#file sets
self.dummySubscription.acquired.addFile(dummyFile3)
self.dummySubscription.failed.addFile(dummyFile3)
self.dummySubscription.completed.addFile(dummyFile3)
#Run the method completeFiles
self.dummySubscription.completeFiles(files=dummyFileList)
#Check if dummyFile3 was inserted at the completed Set
assert dummyFile3 in self.dummySubscription.completed.getFiles(type='set'), \
'Replicated file could\'nt be inserted at completed Set'
#Check if dummyFile3 was erased from all the other Sets
assert dummyFile3 not in self.dummySubscription.acquired.getFiles(type='set'), \
'Completed file still present at acquired Set'
assert dummyFile3 not in self.dummySubscription.failed.getFiles(type='set'), \
'Completed file still present at failed Set'
assert dummyFile3 not in self.dummySubscription.available.getFiles(type='set'), \
'Completed file still present at available Set'
def testFailFiles(self):
"""
Testcase for the failFiles method of the Subscription Class
"""
#Cleaning possible files already occupying the available set
self.dummySubscription.failFiles([])
#First test - Test if initial file (on available set) is inserted in the
# failed set - no arguments
dummyFile2 = File('/tmp/dummyfile2,8888',1,1,1)
#Insert dummyFile2 into the available files Set at dummySubscription
self.dummySubscription.available.addFile(dummyFile2)
S = self.dummySubscription.availableFiles()
# Fail all files
self.dummySubscription.failFiles(S)
assert len(self.dummySubscription.availableFiles()) == 0, \
"failed subscription still has %s files, what's up with that?" %\
len(self.dummySubscription.availableFiles())
#Second test - Test if target files are inserted at the failed set
dummyFileList = []
#Populating the dummy List with a random number of files
for i in range(1, random.randint(100,1000)):
lfn = '/store/data/%s/%s/file.root' % (random.randint(1000, 9999),
random.randint(1000, 9999))
size = random.randint(1000, 2000)
events = 1000
run = random.randint(0, 2000)
lumi = random.randint(0, 8)
file = File(lfn = lfn, size = size, events = events,
checksums = {"cksum": "1"})
file.addRun(Run(run, *[lumi]))
dummyFileList.append(file)
#Add the new files
self.dummySubscription.available.addFile(dummyFileList)
#and fail them
self.dummySubscription.failFiles(files = dummyFileList)
#Check there are no files available - everything should be failed
assert len(self.dummySubscription.availableFiles()) == 0, \
"failed subscription still has %s files, what's up with that?" %\
len(self.dummySubscription.availableFiles())
#Check if all files were inserted at subscription's failed files Set
for x in dummyFileList:
assert x in self.dummySubscription.failed.getFiles(type='set'),\
'Couldn\'t make file failed %s' % x.dict['lfn']
#Third test - Test if a replicate file is erased from the other Sets,
# when a file is considered failed
dummyFile3 = File('/tmp/dummyfile3,5555',1,1,1)
dummyFileList = []
dummyFileList.append(dummyFile3)
#Inserting dummyFile3 to be used as an argument, into each of the other
# file sets
self.dummySubscription.acquired.addFile(dummyFile3)
self.dummySubscription.available.addFile(dummyFile3)
self.dummySubscription.completed.addFile(dummyFile3)
#Run the method failFiles
self.dummySubscription.failFiles(files=dummyFileList)
#Check if dummyFile3 was inserted at the failed Set
assert dummyFile3 in self.dummySubscription.failed.getFiles(type='set'), \
'Replicated file could\'nt be inserted at failed Set'
#Check if dummyFile3 was erased from all the other Sets
assert dummyFile3 not in self.dummySubscription.acquired.getFiles(type='set'), \
'Failed file still present at acquired Set'
assert dummyFile3 not in self.dummySubscription.completed.getFiles(type='set'), \
'Failed file still present at completed Set'
assert dummyFile3 not in self.dummySubscription.available.getFiles(type='set'), \
'Failed file still present at available Set'
def testFilesOfStatus(self):
"""
Testcase for the filesOfStatus method of the Subscription Class
"""
assert self.dummySubscription.filesOfStatus('Available') == \
self.dummySubscription.available.getFiles(type='set') - \
self.dummySubscription.acquiredFiles() | self.dummySubscription.completedFiles() | self.dummySubscription.failedFiles(), \
'Method fileOfStatus(\'AvailableFiles\') does not return available files set'
assert self.dummySubscription.filesOfStatus('Acquired') == self.dummySubscription.acquired.getFiles(type='set'), \
'Method fileOfStatus(\'AcquiredFiles\') does not return acquired files set'
assert self.dummySubscription.filesOfStatus('Completed') == self.dummySubscription.completed.getFiles(type='set'), \
'Method fileOfStatus(\'CompletedFiles\') does not return completed files set'
assert self.dummySubscription.filesOfStatus('Failed') == self.dummySubscription.failed.getFiles(type='set'), \
'Method fileOfStatus(\'FailedFiles\') does not return failed files set'
def testAvailableFiles(self):
"""
Testcase for the availableFiles method of the Subscription Class
"""
assert self.dummySubscription.availableFiles() == \
self.dummySubscription.available.getFiles(type='set'), \
'Method availableFiles does not return available files Set'
def testAcquiredFiles(self):
"""
Testcase for the acquiredFiles method of the Subscription Class
"""
assert self.dummySubscription.acquiredFiles() == \
self.dummySubscription.acquired.getFiles(type='set'), \
'Method acquiredFiles does not return acquired files Set'
def testCompletedFiles(self):
"""
Testcase for the completedFiles method of the Subscription Class
"""
assert self.dummySubscription.completedFiles() == \
self.dummySubscription.completed.getFiles(type='set'), \
'Method completedFiles does not return completed files Set'
def testFailedFiles(self):
"""
Testcase for the failedFiles method of the Subscription Class
"""
assert self.dummySubscription.failedFiles() == \
self.dummySubscription.failed.getFiles(type='set'), \
'Method failedFiles does not return failed files Set'
class FilesetTest (unittest.TestCase):
"""
_FilesetTest_
Testcase for Fileset
"""
def setUp(self):
"""
Create a dummy fileset and populate it with random files,
in order to use it for the testcase methods
"""
logging.basicConfig(level=logging.DEBUG,
format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
datefmt='%m-%d %H:%M',
filename=__file__.replace('.py','.log'),
filemode='w')
self.logger = logging.getLogger('FilesetClassTest')
#Setup the initial testcase environment:
initialfile = File('/tmp/lfn1',1000,1,1,1)
self.initialSet = set()
self.initialSet.add(initialfile)
#Create a Fileset, containing a initial file on it.
self.fileset = Fileset(name = 'self.fileset', files = self.initialSet)
#Populate the fileset with random files
for i in range(1,1000):
lfn = '/store/data/%s/%s/file.root' % (random.randint(1000, 9999),
random.randint(1000, 9999))
size = random.randint(1000, 2000)
events = 1000
run = random.randint(0, 2000)
lumi = random.randint(0, 8)
file = File(lfn=lfn, size=size, events=events, checksums = {"cksum": "1"})
file.addRun(Run(run, *[lumi]))
self.fileset.addFile(file)
def tearDown(self):
"""
No tearDown method for this testcase
"""
pass
def testAddFile(self):
"""
Testcase for the addFile method of the Fileset class
"""
#First test - Add file and check if its there
testfile = File('/tmp/lfntest',9999,9,9)
self.fileset.addFile(testfile)
assert(testfile in self.fileset.listNewFiles(), 'Couldn\'t add file ' +
'to fileset - fileset.addfile method not working')
#Second test - Add file that was already at Fileset.files ,
# and check if it gets updated
testFileSame = File('/tmp/lfntest',9999,9,9)
testFileSame.setLocation(set('dummyse.dummy.com'))
self.fileset.addFile(testFileSame)
assert(testFileSame in self.fileset.getFiles(),'Same file copy ' +
'failed - fileset.addFile not updating location of already ' +
'existing files' )
assert(testfile in self.fileset.getFiles(),'Same file copy ' +
'failed - fileset.addFile unable to remove previous file ' +
'from list')
#Third test - Add file that was already at Fileset.newfiles ,
#and check if it gets updated
assert(testFileSame in self.fileset.listNewFiles(),'Same file copy ' +
'failed - fileset.addFile not adding file to fileset.newFiles')
def testListFiles(self):
"""
_testListFiles_
Testcase for the getFiles() method of the Fileset class
"""
filesettemp = self.fileset.getFiles()
for x in filesettemp:
assert x in (self.fileset.files | self.fileset.newfiles), \
'Missing file %s from file list returned from fileset.ListFiles' % x["lfn"]
def testSetFiles(self):
"""
Check that all files returned by the set are the same as those added to
the fileset
"""
filesettemp = self.fileset.getFiles(type = "set")
for x in filesettemp:
assert x in (self.fileset.files | self.fileset.newfiles), \
'Missing file %s from file list returned from fileset.ListFiles' % x["lfn"]
def testSetListCompare(self):
"""
Test that all files in fileset.setFiles are in fileset.getFiles()
"""
thelist = self.fileset.getFiles()
theset = self.fileset.getFiles(type = "set")
for x in thelist:
assert x in (theset), \
'Missing file %s from file list returned from fileset.ListFiles' % x["lfn"]
def testSorting(self):
"""
Fileset.getFiles() should be sorted the same as Fileset.getFiles(type = "lfn"),
assert that this is the case here.
"""
files = self.fileset.getFiles()
lfns = self.fileset.getFiles(type = "lfn")
for x in files:
assert x["lfn"] in (lfns), \
'Missing file %s from file list returned ' % x["lfn"]
assert lfns[files.index(x)] == x["lfn"], \
'Sorting not consistent: lfn = %s, file = %s' % (lfns[files.index(x)], x["lfn"])
def testListLFNs(self):
"""
Testcase for the listLFN method of the Fileset class
"""
#Kinda slow way of verifying if the raw LFN from each file at the
#fileset is returned from the meth
allFiles = self.fileset.getFiles()
#For each file returned by method listFiles, it checks if LFN is
#present at the output of method listLFNs
for x in allFiles:
assert x['lfn'] in self.fileset.getFiles(type = "lfn"), 'Missing %s from ' \
'list returned from fileset.ListLFNs' % x["lfn"]
#Im a bit confused with this method, leave to discuss it at the meeting with Simon
def testListNewFiles(self):
"""
Testcase for the listNewFiles method of the Fileset class
"""
newfilestemp = self.fileset.listNewFiles()
assert newfilestemp == self.fileset.newfiles, 'Missing files from ' \
'list returned from fileset.ListNewFiles'
def testCommit(self):
"""
Testcase for the commit method of the Fileset class
"""
localTestFileSet = Fileset('LocalTestFileset', self.initialSet)
fsSize = len(localTestFileSet.getFiles(type = "lfn"))
#Dummy file to test
fileTestCommit = File('/tmp/filetestcommit',0000,1,1)
#File is added to the newfiles attribute of localTestFileSet
localTestFileSet.addFile(fileTestCommit)
assert fsSize == len(localTestFileSet.getFiles(type = "lfn")) - 1, 'file not added'\
'correctly to test fileset'
newfilestemp = localTestFileSet.newfiles
assert fileTestCommit in newfilestemp, 'test file not in the new files'\
'list'
#After commit, dummy file is supposed to move from newfiles to files
localTestFileSet.commit()
#First, testing if the new file is present at file set object attribute of the Fileset object
assert newfilestemp.issubset(localTestFileSet.files), 'Test file not ' \
'present at fileset.files - fileset.commit ' \
'not working properly'
#Second, testing if the newfile set object attribute is empty
assert localTestFileSet.newfiles == set(), \
'Test file not present at fileset.newfiles ' \
'- fileset.commit not working properly'
class JobGroup(WMObject):
"""
JobGroups are sets of jobs running on files who's output needs to be merged
together.
"""
def __init__(self, subscription=None, jobs=None):
self.jobs = []
self.newjobs = []
self.id = 0
if isinstance(jobs, list):
self.newjobs = jobs
elif jobs is not None:
self.newjobs = [jobs]
self.subscription = subscription
self.output = Fileset()
self.last_update = datetime.datetime.now()
def add(self, job):
"""
_add_
Add a Job or list of jobs to the JobGroup.
"""
jobList = self.makelist(job)
self.newjobs.extend(jobList)
return
def commit(self):
"""
_commit_
Move any jobs in the newjobs dict to the job dict. Empty the newjobs
dict.
"""
self.jobs.extend(self.newjobs)
self.newjobs = []
def commitBulk(self):
"""
Dummy method for consistency with WMBS implementation
"""
self.commit()
def addOutput(self, file):
"""
_addOutput_
Add a File to the JobGroup's output fileset. The File is committed
to the Fileset immediately.
"""
self.output.addFile(file)
self.output.commit()
def getJobs(self, type="list"):
"""
_getJobs_
Retrieve all of the jobs in the JobGroup. The output will either be
returned as a list of Job objects (when type is "list") or a list of
Job IDs (when type is "id").
"""
if type == "list":
return self.jobs
elif type == "id":
jobIDs = []
for job in self.jobs:
jobIDs.append(job["id"])
return jobIDs
else:
print("Unknown type: %s" % type)
return
def getOutput(self, type="list"):
"""
_getOutput_
Retrieve all of the files that are in the JobGroup's output fileset.
Type can be one of the following: list, set, lfn, id.
"""
return self.output.getFiles(type=type)
def getLength(self, obj):
"""
This just gets a length for either dict or list objects
"""
if isinstance(obj, (dict, list)):
return len(obj)
else:
return 0
def __len__(self):
"""
Allows use of len() on JobGroup
"""
return self.getLength(self.jobs) + self.getLength(self.newjobs)
class Subscription(Pickleable, dict):
def __init__(self,
fileset=None,
workflow=None,
split_algo="FileBased",
type="Processing"):
if fileset == None:
fileset = Fileset()
self.setdefault('fileset', fileset)
self.setdefault('workflow', workflow)
self.setdefault('type', type)
self.setdefault('split_algo', split_algo)
self.available = Fileset(name=fileset.name, files=fileset.getFiles())
self.acquired = Fileset(name='acquired')
self.completed = Fileset(name='completed')
self.failed = Fileset(name='failed')
def name(self):
return self.getWorkflow().name.replace(' ', '') + '_' + \
self.getFileset().name.replace(' ', '')
def getWorkflow(self):
return self["workflow"]
def workflowName(self):
if self["workflow"] == None:
return "Unknown"
return self["workflow"].name
def workflowType(self):
if self["workflow"] == None:
return "Unknown"
return self["workflow"].wfType
def taskName(self):
if self['workflow'] == None:
return "Unknown"
return self['workflow'].task
def owner(self):
if self['workflow'] == None:
return 'Unknown'
return self['workflow'].owner
def getFileset(self):
return self['fileset']
def acquireFiles(self, files=[], size=1):
"""
Return the files acquired
"""
self.acquired.commit()
self.available.commit()
self.failed.commit()
self.completed.commit()
retval = []
if len(files):
for i in files:
# Check each set, instead of elif, just in case something has
# got out of synch
if i in self.available.files:
self.available.files.remove(i)
if i in self.failed.files:
self.failed.files.remove(i)
if i in self.completed.files:
self.completed.files.remove(i)
self.acquired.addFile(i)
else:
if len(self.available.files) < size or size == 0:
size = len(self.available.files)
for i in range(size):
self.acquired.addFile(self.available.files.pop())
return self.acquired.listNewFiles()
def completeFiles(self, files):
"""
Return the number of files complete
"""
self.acquired.commit()
self.available.commit()
self.failed.commit()
self.completed.commit()
for i in files:
# Check each set, instead of elif, just in case something has
# got out of synch
if i in self.available.files:
self.available.files.remove(i)
if i in self.failed.files:
self.failed.files.remove(i)
if i in self.acquired.files:
self.acquired.files.remove(i)
self.completed.addFile(i)
def failFiles(self, files):
"""
Return the number of files failed
"""
self.acquired.commit()
self.available.commit()
self.failed.commit()
self.completed.commit()
for i in files:
# Check each set, instead of elif, just in case something has
# got out of synch
if i in self.available.files:
self.available.files.remove(i)
if i in self.completed.files:
self.completed.files.remove(i)
if i in self.acquired.files:
self.acquired.files.remove(i)
self.failed.addFile(i)
def filesOfStatus(self, status=None, doingJobSplitting=False):
"""
_filesOfStatus_
Return a Set of File objects that are associated with the subscription
and have a particular status.
"""
status = status.title()
if status == 'Available':
return self.available.getFiles(type='set') - \
(self.acquiredFiles() | self.completedFiles() | self.failedFiles())
elif status == 'Acquired':
return self.acquired.getFiles(type='set')
elif status == 'Completed':
return self.completed.getFiles(type='set')
elif status == 'Failed':
return self.failed.getFiles(type='set')
def availableFiles(self, limit=None, doingJobSplitting=False):
"""
_availableFiles_
Return a Set of files that are available for processing
(e.g. not already in use)
"""
if limit:
return list(
self.filesOfStatus(
status="Available",
doingJobSplitting=doingJobSplitting))[:limit]
else:
return self.filesOfStatus(status="Available",
doingJobSplitting=doingJobSplitting)
def acquiredFiles(self):
"""
Set of files marked as acquired.
"""
return self.filesOfStatus(status="Acquired")
def completedFiles(self):
"""
Set of files marked as completed.
"""
return self.filesOfStatus(status="Completed")
def failedFiles(self):
"""
Set of files marked as failed.
"""
return self.filesOfStatus(status="Failed")