Пример #1
0
    def testCommit(self):
        """
            Testcase for the commit method of the Fileset class

        """
        localTestFileSet = Fileset('LocalTestFileset', self.initialSet)
        fsSize = len(localTestFileSet.getFiles(type="lfn"))
        #Dummy file to test
        fileTestCommit = File('/tmp/filetestcommit', 0000, 1, 1)
        #File is added to the newfiles attribute of localTestFileSet
        localTestFileSet.addFile(fileTestCommit)
        assert fsSize == len(localTestFileSet.getFiles(type = "lfn")) - 1, 'file not added'\
                'correctly to test fileset'
        newfilestemp = localTestFileSet.newfiles
        assert fileTestCommit in newfilestemp, 'test file not in the new files'\
                'list'
        #After commit, dummy file is supposed to move from newfiles to files
        localTestFileSet.commit()
        #First, testing if the new file is present at file set object attribute of the Fileset object

        assert newfilestemp.issubset(localTestFileSet.files), 'Test file not ' \
                'present at fileset.files - fileset.commit ' \
                'not working properly'
        #Second, testing if the newfile set object attribute is empty
        assert localTestFileSet.newfiles == set(), \
                'Test file not present at fileset.newfiles ' \
                '- fileset.commit not working properly'
Пример #2
0
 def testCommit(self):
     """
         Testcase for the commit method of the Fileset class
         
     """
     localTestFileSet = Fileset('LocalTestFileset', self.initialSet)
     fsSize = len(localTestFileSet.getFiles(type = "lfn"))
     #Dummy file to test
     fileTestCommit = File('/tmp/filetestcommit',0000,1,1)
     #File is added to the newfiles attribute of localTestFileSet
     localTestFileSet.addFile(fileTestCommit)
     assert fsSize == len(localTestFileSet.getFiles(type = "lfn")) - 1, 'file not added'\
             'correctly to test fileset'
     newfilestemp = localTestFileSet.newfiles
     assert fileTestCommit in newfilestemp, 'test file not in the new files'\
             'list' 
     #After commit, dummy file is supposed to move from newfiles to files
     localTestFileSet.commit()
     #First, testing if the new file is present at file set object attribute of the Fileset object
     
     assert newfilestemp.issubset(localTestFileSet.files), 'Test file not ' \
             'present at fileset.files - fileset.commit ' \
             'not working properly' 
     #Second, testing if the newfile set object attribute is empty
     assert localTestFileSet.newfiles == set(), \
             'Test file not present at fileset.newfiles ' \
             '- fileset.commit not working properly'
Пример #3
0
    def __init__(self,
                 fileset=None,
                 workflow=None,
                 whitelist=None,
                 blacklist=None,
                 split_algo="FileBased",
                 type="Processing"):
        if fileset == None:
            fileset = Fileset()
        if whitelist == None:
            whitelist = set()
        if blacklist == None:
            blacklist = set()

        self.setdefault('fileset', fileset)
        self.setdefault('workflow', workflow)
        self.setdefault('type', type)

        self.setdefault('split_algo', split_algo)
        self.setdefault('whitelist', whitelist)
        self.setdefault('blacklist', blacklist)

        self.available = Fileset(name=fileset.name, files=fileset.getFiles())

        self.acquired = Fileset(name='acquired')
        self.completed = Fileset(name='completed')
        self.failed = Fileset(name='failed')
Пример #4
0
 def testCall(self):
     fileset = Fileset(name="FakeFeederTest")
     for i in range(1, 21):
         self.feeder([fileset])
         set = fileset.getFiles(type = "set")
         if len(set) > 0:
             file = set.pop()
         fileset.commit()
Пример #5
0
 def testCall(self):
     fileset = Fileset(name="FakeFeederTest")
     for i in range(1, 21):
         self.feeder([fileset])
         set = fileset.getFiles(type="set")
         if len(set) > 0:
             file = set.pop()
         fileset.commit()
Пример #6
0
    def __init__(self, fileset = None, workflow = None,
                 split_algo = "FileBased", type = "Processing"):
        if fileset == None:
            fileset = Fileset()

        self.setdefault('fileset', fileset)
        self.setdefault('workflow', workflow)
        self.setdefault('type', type)

        self.setdefault('split_algo', split_algo)

        self.available = Fileset(name=fileset.name,
                                 files = fileset.getFiles())

        self.acquired = Fileset(name='acquired')
        self.completed = Fileset(name='completed')
        self.failed = Fileset(name='failed')
Пример #7
0
class Subscription(Pickleable, dict):
    def __init__(self,
                 fileset=None,
                 workflow=None,
                 whitelist=None,
                 blacklist=None,
                 split_algo="FileBased",
                 type="Processing"):
        if fileset == None:
            fileset = Fileset()
        if whitelist == None:
            whitelist = set()
        if blacklist == None:
            blacklist = set()

        self.setdefault('fileset', fileset)
        self.setdefault('workflow', workflow)
        self.setdefault('type', type)

        self.setdefault('split_algo', split_algo)
        self.setdefault('whitelist', whitelist)
        self.setdefault('blacklist', blacklist)

        self.available = Fileset(name=fileset.name, files=fileset.getFiles())

        self.acquired = Fileset(name='acquired')
        self.completed = Fileset(name='completed')
        self.failed = Fileset(name='failed')

    def name(self):
        return self.getWorkflow().name.replace(' ', '') + '_' + \
                    self.getFileset().name.replace(' ', '')

    def getWorkflow(self):
        return self["workflow"]

    def workflowName(self):
        if self["workflow"] == None:
            return "Unknown"
        return self["workflow"].name

    def taskName(self):
        if self['workflow'] == None:
            return "Unknown"
        return self['workflow'].task

    def getFileset(self):
        return self['fileset']

    def acquireFiles(self, files=[], size=1):
        """
        Return the files acquired
        """
        self.acquired.commit()
        self.available.commit()
        self.failed.commit()
        self.completed.commit()
        retval = []
        if len(files):
            for i in files:
                # Check each set, instead of elif, just in case something has
                # got out of synch
                if i in self.available.files:
                    self.available.files.remove(i)
                if i in self.failed.files:
                    self.failed.files.remove(i)
                if i in self.completed.files:
                    self.completed.files.remove(i)
                self.acquired.addFile(i)
        else:
            if len(self.available.files) < size or size == 0:
                size = len(self.available.files)
            for i in range(size):
                self.acquired.addFile(self.available.files.pop())

        return self.acquired.listNewFiles()

    def completeFiles(self, files):
        """
        Return the number of files complete
        """
        self.acquired.commit()
        self.available.commit()
        self.failed.commit()
        self.completed.commit()
        for i in files:
            # Check each set, instead of elif, just in case something has
            # got out of synch
            if i in self.available.files:
                self.available.files.remove(i)
            if i in self.failed.files:
                self.failed.files.remove(i)
            if i in self.acquired.files:
                self.acquired.files.remove(i)
            self.completed.addFile(i)

    def failFiles(self, files):
        """
        Return the number of files failed
        """
        self.acquired.commit()
        self.available.commit()
        self.failed.commit()
        self.completed.commit()
        for i in files:
            # Check each set, instead of elif, just in case something has
            # got out of synch
            if i in self.available.files:
                self.available.files.remove(i)
            if i in self.completed.files:
                self.completed.files.remove(i)
            if i in self.acquired.files:
                self.acquired.files.remove(i)
            self.failed.addFile(i)

    def filesOfStatus(self, status=None):
        """
        _filesOfStatus_

        Return a Set of File objects that are associated with the subscription
        and have a particular status.
        """
        status = status.title()
        if status == 'Available':
            return self.available.getFiles(type='set') - \
            (self.acquiredFiles() | self.completedFiles() | self.failedFiles())
        elif status == 'Acquired':
            return self.acquired.getFiles(type='set')
        elif status == 'Completed':
            return self.completed.getFiles(type='set')
        elif status == 'Failed':
            return self.failed.getFiles(type='set')

    def markLocation(self, location, whitelist=True):
        """
        Add a location to the subscriptions white or black list
        """
        if whitelist:
            self['whitelist'].add(location)
        else:
            self['blacklist'].add(location)

    def availableFiles(self):
        """
        Return a Set of files that are available for processing
        (e.g. not already in use) and at sites that are white listed
        or not black listed
        """
        def locationMagic(files, locations):
            """
            files and locations are sets. method returns the subset of files
            that are at the locations - this is a lot simpler with the database
            """
            magicfiles = set()
            for f in files:
                if len(f['locations'] & locations) > 0:
                    magicfiles.add(f)
            return magicfiles

        files = self.filesOfStatus(status="Available")

        if len(self['whitelist']) > 0:
            # Return files at white listed sites
            return locationMagic(files, self['whitelist'])
        elif len(self['blacklist']) > 0:
            # Return files not at black listed sites
            return files - locationMagic(files, self['blacklist'])
        #Return all files, because you're crazy and just don't care
        return files

    def acquiredFiles(self):
        """
        Set of files marked as acquired.
        """
        return self.filesOfStatus(status="Acquired")

    def completedFiles(self):
        """
        Set of files marked as completed.
        """
        return self.filesOfStatus(status="Completed")

    def failedFiles(self):
        """
        Set of files marked as failed.
        """
        return self.filesOfStatus(status="Failed")
Пример #8
0
class Subscription(Pickleable, dict):
    def __init__(self, fileset = None, workflow = None,
                 split_algo = "FileBased", type = "Processing"):
        if fileset == None:
            fileset = Fileset()

        self.setdefault('fileset', fileset)
        self.setdefault('workflow', workflow)
        self.setdefault('type', type)

        self.setdefault('split_algo', split_algo)

        self.available = Fileset(name=fileset.name,
                                 files = fileset.getFiles())

        self.acquired = Fileset(name='acquired')
        self.completed = Fileset(name='completed')
        self.failed = Fileset(name='failed')

    def name(self):
        return self.getWorkflow().name.replace(' ', '') + '_' + \
                    self.getFileset().name.replace(' ', '')

    def getWorkflow(self):
        return self["workflow"]

    def workflowName(self):
        if self["workflow"] == None:
            return "Unknown"
        return self["workflow"].name

    def workflowType(self):
        if self["workflow"] == None:
            return "Unknown"
        return self["workflow"].wfType    

    def taskName(self):
        if self['workflow'] == None:
            return "Unknown"
        return self['workflow'].task

    def owner(self):
        if self['workflow'] == None:
            return 'Unknown'
        return self['workflow'].owner

    def getFileset(self):
        return self['fileset']

    def acquireFiles(self, files = [], size=1):
        """
        Return the files acquired
        """
        self.acquired.commit()
        self.available.commit()
        self.failed.commit()
        self.completed.commit()
        retval = []
        if len(files):
            for i in files:
                # Check each set, instead of elif, just in case something has
                # got out of synch
                if i in self.available.files:
                    self.available.files.remove(i)
                if i in self.failed.files:
                    self.failed.files.remove(i)
                if i in self.completed.files:
                    self.completed.files.remove(i)
                self.acquired.addFile(i)
        else:
            if len(self.available.files) < size or size == 0:
                size = len(self.available.files)
            for i in range(size):
                self.acquired.addFile(self.available.files.pop())

        return self.acquired.listNewFiles()

    def completeFiles(self, files):
        """
        Return the number of files complete
        """
        self.acquired.commit()
        self.available.commit()
        self.failed.commit()
        self.completed.commit()
        for i in files:
            # Check each set, instead of elif, just in case something has
            # got out of synch
            if i in self.available.files:
                self.available.files.remove(i)
            if i in self.failed.files:
                self.failed.files.remove(i)
            if i in self.acquired.files:
                self.acquired.files.remove(i)
            self.completed.addFile(i)

    def failFiles(self, files):
        """
        Return the number of files failed
        """
        self.acquired.commit()
        self.available.commit()
        self.failed.commit()
        self.completed.commit()
        for i in files:
            # Check each set, instead of elif, just in case something has
            # got out of synch
            if i in self.available.files:
                self.available.files.remove(i)
            if i in self.completed.files:
                self.completed.files.remove(i)
            if i in self.acquired.files:
                self.acquired.files.remove(i)
            self.failed.addFile(i)

    def filesOfStatus(self, status=None, doingJobSplitting = False):
        """
        _filesOfStatus_

        Return a Set of File objects that are associated with the subscription
        and have a particular status.
        """
        status = status.title()
        if status == 'Available':
            return self.available.getFiles(type='set') - \
            (self.acquiredFiles() | self.completedFiles() | self.failedFiles())
        elif status == 'Acquired':
            return self.acquired.getFiles(type='set')
        elif status == 'Completed':
            return self.completed.getFiles(type='set')
        elif status == 'Failed':
            return self.failed.getFiles(type='set')

    def availableFiles(self, limit = None, doingJobSplitting = False):
        """
        _availableFiles_

        Return a Set of files that are available for processing
        (e.g. not already in use)
        """
        if limit:
            return list(self.filesOfStatus(status = "Available", doingJobSplitting = doingJobSplitting))[:limit]
        else:
            return self.filesOfStatus(status = "Available", doingJobSplitting = doingJobSplitting)

    def acquiredFiles(self):
        """
        Set of files marked as acquired.
        """
        return self.filesOfStatus(status = "Acquired")

    def completedFiles(self):
        """
        Set of files marked as completed.
        """
        return self.filesOfStatus(status = "Completed")

    def failedFiles(self):
        """
        Set of files marked as failed.
        """
        return self.filesOfStatus(status = "Failed")
Пример #9
0
class JobGroup(WMObject):
    """
    JobGroups are sets of jobs running on files who's output needs to be merged
    together.
    """
    def __init__(self, subscription = None, jobs = None):
        self.jobs = []
        self.newjobs = []
        self.id = 0
        
        if type(jobs) == list:
            self.newjobs = jobs
        elif jobs != None:
            self.newjobs = [jobs]
            
        self.subscription = subscription
        self.output = Fileset()
        self.last_update = datetime.datetime.now()
        
    def add(self, job):
        """
        _add_

        Add a Job or list of jobs to the JobGroup.
        """
        jobList = self.makelist(job)
        self.newjobs.extend(jobList)
        return

    def commit(self):
        """
        _commit_

        Move any jobs in the newjobs dict to the job dict.  Empty the newjobs
        dict.
        """
        self.jobs.extend(self.newjobs)
        self.newjobs = []
    
    def commitBulk(self):
        """
        Dummy method for consistency with WMBS implementation
        """
        self.commit()
        
    def addOutput(self, file):
        """
        _addOutput_

        Add a File to the JobGroup's output fileset.  The File is committed
        to the Fileset immediately.
        """
        self.output.addFile(file)
        self.output.commit()

    def getJobs(self, type = "list"):
        """
        _getJobs_

        Retrieve all of the jobs in the JobGroup.  The output will either be
        returned as a list of Job objects (when type is "list") or a list of
        Job IDs (when type is "id").
        """
        if type == "list":
            return self.jobs
        elif type == "id":
            jobIDs = []

            for job in self.jobs:
                jobIDs.append(job["id"])

            return jobIDs
        else:
            print "Unknown type: %s" % type

        return

    def getOutput(self, type = "list"):
        """
        _getOutput_

        Retrieve all of the files that are in the JobGroup's output fileset.
        Type can be one of the following: list, set, lfn, id.
        """
        return self.output.getFiles(type = type)

    def getLength(self, obj):
        """
        This just gets a length for either dict or list objects
        """
        if type(obj) == dict:
            return len(obj.keys())
        elif type(obj) == list:
            return len(obj)
        else:
            return 0

    def __len__(self):
        """
        Allows use of len() on JobGroup
        """
        return self.getLength(self.jobs) + self.getLength(self.newjobs)
Пример #10
0
class SubscriptionTest(unittest.TestCase):
    """
    _SubscriptionTest_

    Testcase for the Subscription class

    """
    def setUp(self):
        """
        Initial Setup for Subscription Testcase

        Set a dummy Subscription with a fileset composed of one file inside it
        and a dummy workflow using the default constructor of the Workflow class

        """
        self.dummyFile = File('/tmp/dummyfile', 9999, 0, 0, 0)
        self.dummySet = set()
        self.dummySet.add(self.dummyFile)
        self.dummyFileSet = Fileset(name='SubscriptionTestFileset',
                                    files=self.dummySet)
        self.dummyWorkFlow = Workflow()
        self.dummySubscription = Subscription(fileset=self.dummyFileSet,
                                              workflow=self.dummyWorkFlow)
        return

    def tearDown(self):
        pass

    def testGetWorkflow(self):
        """
        Testcase for the getWorkflow method of the Subscription Class

        """
        assert self.dummySubscription['workflow'] == self.dummyWorkFlow, \
            'Couldn\'t add Workflow to Subscription'

    def testGetFileset(self):
        """
        Testcase for the getFileset method of the Subscription Class

        """
        assert self.dummyFileSet.name == self.dummySubscription['fileset'].name, \
            'Couldn\'t add Fileset to Subscription - name does not match'

        for x in self.dummyFileSet.listNewFiles():
            assert x in self.dummySubscription['fileset'].newfiles, \
                'Couldn\'t add Fileset to Subscription - newFiles Set does not match'

        assert self.dummyFileSet.getFiles(type='set') == \
               self.dummySubscription['fileset'].getFiles(type='set'), \
            'Couldn\'t add Fileset to Subscription - %s Set does not match' % x

    def testAcquireFiles(self):
        """
        Testcase for the acquireFiles method of the Subscription Class

        """
        # Cleaning possible files already occupying the available set
        self.dummySubscription.acquireFiles()

        # First test - Test if initial file (on available set) is inserted in the
        # acquired set - no arguments

        dummyFile2 = File('/tmp/dummyfile2,8888', 1, 1, 1)
        # Insert dummyFile2 into the available files Set at dummySubscription
        self.dummySubscription.available.addFile(dummyFile2)

        S = self.dummySubscription.available.listNewFiles()
        # Check if Set returned by method is the same that was at the previous
        # available FileSet
        assert S == self.dummySubscription.acquireFiles(), \
            'Couldn\'t acquire file using method acquireFiles - (no arguments test)'

        # Second test - Test if target files are inserted at the acquired set

        dummyFileList = set()
        # Populating the dummy List with a random number of files
        for i in range(1, random.randint(100, 1000)):
            lfn = '/store/data/%s/%s/file.root' % (random.randint(
                1000, 9999), random.randint(1000, 9999))
            size = random.randint(1000, 2000)
            events = 1000
            run = random.randint(0, 2000)
            lumi = random.randint(0, 8)

            file = File(lfn=lfn,
                        size=size,
                        events=events,
                        checksums={"cksum": "1"})
            file.addRun(Run(run, *[lumi]))
            dummyFileList.add(file)

        # Check if return value is correct - with parameters
        acqFiles = self.dummySubscription.acquireFiles(files=dummyFileList)
        assert acqFiles == dummyFileList, \
            'Return value for acquireFiles method not the acquired files'
        # Check if all files were inserted at subscription acquired files Set
        for x in dummyFileList:
            assert x in self.dummySubscription.acquired.getFiles(type='set'), \
                'Couldn\'t acquire File %s' % x.dict['lfn']

        # Third test - Test if a replicate file is erased from the other Sets,
        # when a file is acquired

        dummyFile3 = File('/tmp/dummyfile3,5555', 1, 1, 1)
        dummyFileList = []
        dummyFileList.append(dummyFile3)

        # Inserting dummyFile3 to be used as an argument, into each of the other file sets
        self.dummySubscription.available.addFile(dummyFile3)
        self.dummySubscription.failed.addFile(dummyFile3)
        self.dummySubscription.completed.addFile(dummyFile3)

        # Run the method acquireFiles
        self.dummySubscription.acquireFiles(files=dummyFileList, size=1)

        # Check if dummyFile3 was inserted at the acquired Set
        assert dummyFile3 in self.dummySubscription.acquired.getFiles(type='set'), \
            'Replicated file could\'nt be inserted at acquired Set'

        # Check if dummyFile3 was erased from all the other Sets
        assert dummyFile3 not in self.dummySubscription.available.getFiles(type='set'), \
            'Acquired file still present at available Set'
        assert dummyFile3 not in self.dummySubscription.failed.getFiles(type='set'), \
            'Acquired file still present at failed Set'
        assert dummyFile3 not in self.dummySubscription.completed.getFiles(type='set'), \
            'Acquired file still present at completed Set'

        # Fourth test - Test if the method works properly if a wrong size number
        # is given as an argument

        # Case 1: size < number of files given as an argument

        dummyFileList = []
        for i in range(90, 100):
            dummyFileSize = File('/tmp/dummyfile' + str(i), 7656, 1, 1, 1)
            dummyFileList.append(dummyFileSize)

        # Run the method:
        self.dummySubscription.acquireFiles(files=dummyFileList, size=1)
        # Check each file of the List
        for x in dummyFileList:
            assert x in self.dummySubscription.acquired.getFiles(type='set'), \
                'File wasn\'t acquired (lower Size argument test)'

        # Case 2: size = 0

        # Run the method:
        self.dummySubscription.acquireFiles(files=dummyFileList, size=0)
        # Check each file of the List
        for x in dummyFileList:
            assert x in self.dummySubscription.acquired.getFiles(type='set'), \
                'File wasn\'t acquired (zero size argument test)'

    def testCompleteFiles(self):
        """
        Testcase for the completeFiles method of the Subscription Class

        """
        # Cleaning possible files already occupying the available set
        self.dummySubscription.completeFiles([])

        # First test - Test if initial file (on available set) is inserted in the
        # completed set - no arguments

        dummyFile2 = File('/tmp/dummyfile2,8888', 1, 1, 1)
        # Insert dummyFile2 into the available files Set at dummySubscription
        self.dummySubscription.available.addFile(dummyFile2)

        S = self.dummySubscription.availableFiles()
        # complete all files
        self.dummySubscription.completeFiles(S)

        assert len(self.dummySubscription.availableFiles()) == 0, \
            "completed subscription still has %s files, what's up with that?" % \
            len(self.dummySubscription.availableFiles())

        # Second test - Test if target files are inserted at the completed files set

        dummyFileList = []
        # Populating the dummy List with a random number of files
        for i in range(1, random.randint(100, 1000)):
            lfn = '/store/data/%s/%s/file.root' % (random.randint(
                1000, 9999), random.randint(1000, 9999))
            size = random.randint(1000, 2000)
            events = 1000
            run = random.randint(0, 2000)
            lumi = random.randint(0, 8)

            file = File(lfn=lfn,
                        size=size,
                        events=events,
                        checksums={"cksum": "1"})
            file.addRun(Run(run, *[lumi]))
            dummyFileList.append(file)
        # Add the new files
        self.dummySubscription.available.addFile(dummyFileList)
        # and complete them
        self.dummySubscription.completeFiles(files=dummyFileList)
        # Check if return value is correct - with parameters
        assert len(self.dummySubscription.availableFiles()) == 0, \
            "completed subscription still has %s files, what's up with that?" % \
            len(self.dummySubscription.availableFiles())

        # Check if all files were inserted at subscription's completed files Set
        for x in dummyFileList:
            assert x in self.dummySubscription.completed.getFiles(type='set'), \
                'Couldn\'t make file completed %s' % x.dict['lfn']

        # Third test - Test if a replicate file is erased from the other Sets,
        # when a file is made completed

        dummyFile3 = File('/tmp/dummyfile3,5555', 1, 1, 1)
        dummyFileList = []
        dummyFileList.append(dummyFile3)

        # Inserting dummyFile3 to be used as an argument, into each of the other
        # file sets
        self.dummySubscription.acquired.addFile(dummyFile3)
        self.dummySubscription.failed.addFile(dummyFile3)
        self.dummySubscription.completed.addFile(dummyFile3)

        # Run the method completeFiles
        self.dummySubscription.completeFiles(files=dummyFileList)

        # Check if dummyFile3 was inserted at the completed Set
        assert dummyFile3 in self.dummySubscription.completed.getFiles(type='set'), \
            'Replicated file could\'nt be inserted at completed Set'

        # Check if dummyFile3 was erased from all the other Sets
        assert dummyFile3 not in self.dummySubscription.acquired.getFiles(type='set'), \
            'Completed file still present at acquired Set'
        assert dummyFile3 not in self.dummySubscription.failed.getFiles(type='set'), \
            'Completed file still present at failed Set'
        assert dummyFile3 not in self.dummySubscription.available.getFiles(type='set'), \
            'Completed file still present at available Set'

    def testFailFiles(self):
        """
        Testcase for the failFiles method of the Subscription Class

        """
        # Cleaning possible files already occupying the available set
        self.dummySubscription.failFiles([])

        # First test - Test if initial file (on available set) is inserted in the
        # failed set - no arguments

        dummyFile2 = File('/tmp/dummyfile2,8888', 1, 1, 1)
        # Insert dummyFile2 into the available files Set at dummySubscription
        self.dummySubscription.available.addFile(dummyFile2)

        S = self.dummySubscription.availableFiles()
        # Fail all files
        self.dummySubscription.failFiles(S)

        assert len(self.dummySubscription.availableFiles()) == 0, \
            "failed subscription still has %s files, what's up with that?" % \
            len(self.dummySubscription.availableFiles())

        # Second test - Test if target files are inserted at the failed set

        dummyFileList = []
        # Populating the dummy List with a random number of files
        for i in range(1, random.randint(100, 1000)):
            lfn = '/store/data/%s/%s/file.root' % (random.randint(
                1000, 9999), random.randint(1000, 9999))
            size = random.randint(1000, 2000)
            events = 1000
            run = random.randint(0, 2000)
            lumi = random.randint(0, 8)

            file = File(lfn=lfn,
                        size=size,
                        events=events,
                        checksums={"cksum": "1"})
            file.addRun(Run(run, *[lumi]))
            dummyFileList.append(file)
        # Add the new files
        self.dummySubscription.available.addFile(dummyFileList)
        # and fail them
        self.dummySubscription.failFiles(files=dummyFileList)
        # Check there are no files available - everything should be failed
        assert len(self.dummySubscription.availableFiles()) == 0, \
            "failed subscription still has %s files, what's up with that?" % \
            len(self.dummySubscription.availableFiles())

        # Check if all files were inserted at subscription's failed files Set
        for x in dummyFileList:
            assert x in self.dummySubscription.failed.getFiles(type='set'), \
                'Couldn\'t make file failed %s' % x.dict['lfn']

        # Third test - Test if a replicate file is erased from the other Sets,
        # when a file is considered failed

        dummyFile3 = File('/tmp/dummyfile3,5555', 1, 1, 1)
        dummyFileList = []
        dummyFileList.append(dummyFile3)

        # Inserting dummyFile3 to be used as an argument, into each of the other
        # file sets
        self.dummySubscription.acquired.addFile(dummyFile3)
        self.dummySubscription.available.addFile(dummyFile3)
        self.dummySubscription.completed.addFile(dummyFile3)

        # Run the method failFiles
        self.dummySubscription.failFiles(files=dummyFileList)

        # Check if dummyFile3 was inserted at the failed Set
        assert dummyFile3 in self.dummySubscription.failed.getFiles(type='set'), \
            'Replicated file could\'nt be inserted at failed Set'

        # Check if dummyFile3 was erased from all the other Sets
        assert dummyFile3 not in self.dummySubscription.acquired.getFiles(type='set'), \
            'Failed file still present at acquired Set'
        assert dummyFile3 not in self.dummySubscription.completed.getFiles(type='set'), \
            'Failed file still present at completed Set'
        assert dummyFile3 not in self.dummySubscription.available.getFiles(type='set'), \
            'Failed file still present at available Set'

    def testFilesOfStatus(self):
        """
        Testcase for the filesOfStatus method of the Subscription Class

        """

        assert self.dummySubscription.filesOfStatus('Available') == \
               self.dummySubscription.available.getFiles(type='set') - \
               self.dummySubscription.acquiredFiles() | self.dummySubscription.completedFiles() | self.dummySubscription.failedFiles(), \
            'Method fileOfStatus(\'AvailableFiles\') does not return available files set'
        assert self.dummySubscription.filesOfStatus('Acquired') == self.dummySubscription.acquired.getFiles(type='set'), \
            'Method fileOfStatus(\'AcquiredFiles\') does not return acquired files set'
        assert self.dummySubscription.filesOfStatus('Completed') == self.dummySubscription.completed.getFiles(
            type='set'), \
            'Method fileOfStatus(\'CompletedFiles\') does not return completed files set'
        assert self.dummySubscription.filesOfStatus('Failed') == self.dummySubscription.failed.getFiles(type='set'), \
            'Method fileOfStatus(\'FailedFiles\') does not return failed files set'

    def testAvailableFiles(self):
        """
        Testcase for the availableFiles method of the Subscription Class
        """
        assert self.dummySubscription.availableFiles() == \
               self.dummySubscription.available.getFiles(type='set'), \
            'Method availableFiles does not return available files Set'

    def testAcquiredFiles(self):
        """
        Testcase for the acquiredFiles method of the Subscription Class
        """
        assert self.dummySubscription.acquiredFiles() == \
               self.dummySubscription.acquired.getFiles(type='set'), \
            'Method acquiredFiles does not return acquired files Set'

    def testCompletedFiles(self):
        """
        Testcase for the completedFiles method of the Subscription Class
        """
        assert self.dummySubscription.completedFiles() == \
               self.dummySubscription.completed.getFiles(type='set'), \
            'Method completedFiles does not return completed files Set'

    def testFailedFiles(self):
        """
        Testcase for the failedFiles method of the Subscription Class
        """
        assert self.dummySubscription.failedFiles() == \
               self.dummySubscription.failed.getFiles(type='set'), \
            'Method failedFiles does not return failed files Set'
Пример #11
0
class FilesetTest(unittest.TestCase):
    """
    _FilesetTest_

    Testcase for Fileset
    """
    def setUp(self):
        """
        Create a dummy fileset and populate it with random files,
        in order to use it for the testcase methods

        """
        logging.basicConfig(
            level=logging.DEBUG,
            format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
            datefmt='%m-%d %H:%M',
            filename=__file__.replace('.py', '.log'),
            filemode='w')
        self.logger = logging.getLogger('FilesetClassTest')

        #Setup the initial testcase environment:
        initialfile = File('/tmp/lfn1', 1000, 1, 1, 1)
        self.initialSet = set()
        self.initialSet.add(initialfile)

        #Create a Fileset, containing a initial file on it.
        self.fileset = Fileset(name='self.fileset', files=self.initialSet)
        #Populate the fileset with random files
        for i in range(1, 1000):
            lfn = '/store/data/%s/%s/file.root' % (random.randint(
                1000, 9999), random.randint(1000, 9999))
            size = random.randint(1000, 2000)
            events = 1000
            run = random.randint(0, 2000)
            lumi = random.randint(0, 8)

            file = File(lfn=lfn,
                        size=size,
                        events=events,
                        checksums={"cksum": "1"})
            file.addRun(Run(run, *[lumi]))
            self.fileset.addFile(file)

    def tearDown(self):
        """
            No tearDown method for this testcase

        """
        pass

    def testAddFile(self):
        """
        _testAddFile_

        Testcase for the addFile method of the Fileset class

        """

        # First test - Add file and check if it's there
        testfile = File('/tmp/lfntest', 9999, 9, 9)
        self.fileset.addFile(testfile)
        self.assertTrue(
            testfile in self.fileset.listNewFiles(),
            "Couldn't add file to fileset - fileset.addfile method not working"
        )

        # Second test - Add file that was already at Fileset.files, and check if it gets updated
        testFileSame = File('/tmp/lfntest', 9999, 9, 9)
        testFileSame.setLocation(set('dummyse.dummy.com'))
        self.fileset.addFile(testFileSame)
        self.assertTrue(
            testFileSame in self.fileset.getFiles(),
            'Same file copy ailed - fileset.addFile not updating location of already existing files'
        )
        self.assertTrue(
            testfile in self.fileset.getFiles(),
            'Same file copy failed - fileset.addFile unable to remove previous file from list'
        )

        # Third test - Add file that was already at Fileset.newfiles, and check if it gets updated
        self.assertTrue(
            testFileSame in self.fileset.listNewFiles(),
            'Same file copy failed - fileset.addFile not adding file to fileset.newFiles'
        )

    def testListFiles(self):
        """
        _testListFiles_

        Testcase for the getFiles() method of the Fileset class
        """
        filesettemp = self.fileset.getFiles()
        for x in filesettemp:
            assert x in (self.fileset.files | self.fileset.newfiles), \
            'Missing file %s from file list returned from fileset.ListFiles' % x["lfn"]

    def testSetFiles(self):
        """
        Check that all files returned by the set are the same as those added to
        the fileset
        """
        filesettemp = self.fileset.getFiles(type="set")
        for x in filesettemp:
            assert x in (self.fileset.files | self.fileset.newfiles), \
            'Missing file %s from file list returned from fileset.ListFiles' % x["lfn"]

    def testSetListCompare(self):
        """
        Test that all files in fileset.setFiles are in fileset.getFiles()
        """
        thelist = self.fileset.getFiles()
        theset = self.fileset.getFiles(type="set")
        for x in thelist:
            assert x in (theset), \
            'Missing file %s from file list returned from fileset.ListFiles' % x["lfn"]

    def testSorting(self):
        """
        Fileset.getFiles() should be sorted the same as Fileset.getFiles(type = "lfn"),
        assert that this is the case here.
        """
        files = self.fileset.getFiles()
        lfns = self.fileset.getFiles(type="lfn")
        for x in files:
            assert x["lfn"] in (lfns), \
            'Missing file %s from file list returned ' % x["lfn"]
            assert lfns[files.index(x)] == x["lfn"], \
            'Sorting not consistent: lfn = %s, file = %s' % (lfns[files.index(x)], x["lfn"])

    def testListLFNs(self):
        """
            Testcase for the listLFN method of the Fileset class

        """
        #Kinda slow way of verifying if the raw LFN from each file at the
        #fileset is returned from the meth
        allFiles = self.fileset.getFiles()

        #For each file returned by method listFiles, it checks if LFN is
        #present at the output of method listLFNs
        for x in allFiles:
            assert x['lfn'] in self.fileset.getFiles(type = "lfn"), 'Missing %s from ' \
            'list returned from fileset.ListLFNs' % x["lfn"]
        #Im a bit confused with this method, leave to discuss it at the meeting with Simon

    def testListNewFiles(self):
        """
            Testcase for the listNewFiles method of the Fileset class

        """
        newfilestemp = self.fileset.listNewFiles()
        assert newfilestemp == self.fileset.newfiles, 'Missing files from ' \
               'list returned from fileset.ListNewFiles'

    def testCommit(self):
        """
            Testcase for the commit method of the Fileset class

        """
        localTestFileSet = Fileset('LocalTestFileset', self.initialSet)
        fsSize = len(localTestFileSet.getFiles(type="lfn"))
        #Dummy file to test
        fileTestCommit = File('/tmp/filetestcommit', 0000, 1, 1)
        #File is added to the newfiles attribute of localTestFileSet
        localTestFileSet.addFile(fileTestCommit)
        assert fsSize == len(localTestFileSet.getFiles(type = "lfn")) - 1, 'file not added'\
                'correctly to test fileset'
        newfilestemp = localTestFileSet.newfiles
        assert fileTestCommit in newfilestemp, 'test file not in the new files'\
                'list'
        #After commit, dummy file is supposed to move from newfiles to files
        localTestFileSet.commit()
        #First, testing if the new file is present at file set object attribute of the Fileset object

        assert newfilestemp.issubset(localTestFileSet.files), 'Test file not ' \
                'present at fileset.files - fileset.commit ' \
                'not working properly'
        #Second, testing if the newfile set object attribute is empty
        assert localTestFileSet.newfiles == set(), \
                'Test file not present at fileset.newfiles ' \
                '- fileset.commit not working properly'
Пример #12
0
class SubscriptionTest(unittest.TestCase):
    """
    _SubscriptionTest_

    Testcase for the Subscription class

    """

    def setUp(self):
        """
        Initial Setup for Subscription Testcase

        Set a dummy Subscription with a fileset composed of one file inside it
        and a dummy workflow using the default constructor of the Workflow class

        """
        self.dummyFile = File('/tmp/dummyfile',9999,0,0,0)
        self.dummySet = set()
        self.dummySet.add(self.dummyFile)
        self.dummyFileSet = Fileset(name = 'SubscriptionTestFileset',
                                    files = self.dummySet)
        self.dummyWorkFlow = Workflow()
        self.dummySubscription = Subscription(fileset = self.dummyFileSet,
                                              workflow = self.dummyWorkFlow)
        return

    def tearDown(self):
        pass

    def testGetWorkflow(self):
        """
        Testcase for the getWorkflow method of the Subscription Class

        """
        assert self.dummySubscription['workflow'] == self.dummyWorkFlow, \
        'Couldn\'t add Workflow to Subscription'

    def testGetFileset(self):
        """
        Testcase for the getFileset method of the Subscription Class

        """
        assert self.dummyFileSet.name == self.dummySubscription['fileset'].name, \
        'Couldn\'t add Fileset to Subscription - name does not match'

        for x in self.dummyFileSet.listNewFiles():
            assert x in self.dummySubscription['fileset'].newfiles, \
            'Couldn\'t add Fileset to Subscription - newFiles Set does not match'

        assert self.dummyFileSet.getFiles(type='set') == \
            self.dummySubscription['fileset'].getFiles(type='set'), \
            'Couldn\'t add Fileset to Subscription - %s Set does not match' % x

    def testAcquireFiles(self):
        """
        Testcase for the acquireFiles method of the Subscription Class

        """
        #Cleaning possible files already occupying the available set
        self.dummySubscription.acquireFiles()

        # First test - Test if initial file (on available set) is inserted in the
        # acquired set - no arguments

        dummyFile2 = File('/tmp/dummyfile2,8888',1,1,1)
        #Insert dummyFile2 into the available files Set at dummySubscription
        self.dummySubscription.available.addFile(dummyFile2)

        S = self.dummySubscription.available.listNewFiles()
        #Check if Set returned by method is the same that was at the previous
        #available FileSet
        assert S == self.dummySubscription.acquireFiles(), \
        'Couldn\'t acquire file using method acquireFiles - (no arguments test)'


        #Second test - Test if target files are inserted at the acquired set

        dummyFileList = set()
        #Populating the dummy List with a random number of files
        for i in range(1, random.randint(100,1000)):
            lfn = '/store/data/%s/%s/file.root' % (random.randint(1000, 9999),
                                              random.randint(1000, 9999))
            size = random.randint(1000, 2000)
            events = 1000
            run = random.randint(0, 2000)
            lumi = random.randint(0, 8)

            file = File(lfn = lfn, size = size, events = events,
                        checksums = {"cksum": "1"})
            file.addRun(Run(run, *[lumi]))
            dummyFileList.add(file)

        #Check if return value is correct - with parameters
        acqFiles = self.dummySubscription.acquireFiles(files = dummyFileList)
        assert acqFiles == dummyFileList,\
                'Return value for acquireFiles method not the acquired files'
        #Check if all files were inserted at subscription acquired files Set
        for x in dummyFileList:
            assert x in self.dummySubscription.acquired.getFiles(type='set'), \
            'Couldn\'t acquire File %s' % x.dict['lfn']

        #Third test - Test if a replicate file is erased from the other Sets,
        #when a file is acquired

        dummyFile3 = File('/tmp/dummyfile3,5555',1,1,1)
        dummyFileList = []
        dummyFileList.append(dummyFile3)

        #Inserting dummyFile3 to be used as an argument, into each of the other file sets
        self.dummySubscription.available.addFile(dummyFile3)
        self.dummySubscription.failed.addFile(dummyFile3)
        self.dummySubscription.completed.addFile(dummyFile3)

        #Run the method acquireFiles
        self.dummySubscription.acquireFiles(files=dummyFileList, size=1)

        #Check if dummyFile3 was inserted at the acquired Set
        assert dummyFile3 in self.dummySubscription.acquired.getFiles(type='set'), \
        'Replicated file could\'nt be inserted at acquired Set'

        #Check if dummyFile3 was erased from all the other Sets
        assert dummyFile3 not in self.dummySubscription.available.getFiles(type='set'), \
        'Acquired file still present at available Set'
        assert dummyFile3 not in self.dummySubscription.failed.getFiles(type='set'), \
        'Acquired file still present at failed Set'
        assert dummyFile3 not in self.dummySubscription.completed.getFiles(type='set'), \
        'Acquired file still present at completed Set'


        #Fourth test - Test if the method works properly if a wrong size number
        #is given as an argument

        #Case 1: size < number of files given as an argument

        dummyFileList = []
        for i in range(90,100):
            dummyFileSize = File('/tmp/dummyfile'+str(i),7656,1,1,1)
            dummyFileList.append(dummyFileSize)

        #Run the method:
        self.dummySubscription.acquireFiles(files=dummyFileList, size=1)
        #Check each file of the List
        for x in dummyFileList:
            assert x in self.dummySubscription.acquired.getFiles(type='set'), \
            'File wasn\'t acquired (lower Size argument test)'

        #Case 2: size = 0

        #Run the method:
        self.dummySubscription.acquireFiles(files=dummyFileList, size=0)
        #Check each file of the List
        for x in dummyFileList:
            assert x in self.dummySubscription.acquired.getFiles(type='set'), \
            'File wasn\'t acquired (zero size argument test)'


    def testCompleteFiles(self):
        """
        Testcase for the completeFiles method of the Subscription Class

        """
        #Cleaning possible files already occupying the available set
        self.dummySubscription.completeFiles([])

        #First test - Test if initial file (on available set) is inserted in the
        #completed set - no arguments

        dummyFile2 = File('/tmp/dummyfile2,8888',1,1,1)
        #Insert dummyFile2 into the available files Set at dummySubscription
        self.dummySubscription.available.addFile(dummyFile2)

        S = self.dummySubscription.availableFiles()
        #complete all files
        self.dummySubscription.completeFiles(S)

        assert len(self.dummySubscription.availableFiles()) == 0, \
        "completed subscription still has %s files, what's up with that?" %\
          len(self.dummySubscription.availableFiles())

        #Second test - Test if target files are inserted at the completed files set

        dummyFileList = []
        #Populating the dummy List with a random number of files
        for i in range(1, random.randint(100,1000)):
            lfn = '/store/data/%s/%s/file.root' % (random.randint(1000, 9999),
                                              random.randint(1000, 9999))
            size = random.randint(1000, 2000)
            events = 1000
            run = random.randint(0, 2000)
            lumi = random.randint(0, 8)

            file = File(lfn = lfn, size = size, events = events,
                        checksums = {"cksum": "1"})
            file.addRun(Run(run, *[lumi]))
            dummyFileList.append(file)
        #Add the new files
        self.dummySubscription.available.addFile(dummyFileList)
        #and complete them
        self.dummySubscription.completeFiles(files = dummyFileList)
        #Check if return value is correct - with parameters
        assert len(self.dummySubscription.availableFiles()) == 0, \
        "completed subscription still has %s files, what's up with that?" %\
          len(self.dummySubscription.availableFiles())

        #Check if all files were inserted at subscription's completed files Set
        for x in dummyFileList:
            assert x in self.dummySubscription.completed.getFiles(type='set'), \
            'Couldn\'t make file completed %s' % x.dict['lfn']

        #Third test - Test if a replicate file is erased from the other Sets,
        #when a file is made completed

        dummyFile3 = File('/tmp/dummyfile3,5555',1,1,1)
        dummyFileList = []
        dummyFileList.append(dummyFile3)

        #Inserting dummyFile3 to be used as an argument, into each of the other
        #file sets
        self.dummySubscription.acquired.addFile(dummyFile3)
        self.dummySubscription.failed.addFile(dummyFile3)
        self.dummySubscription.completed.addFile(dummyFile3)

        #Run the method completeFiles
        self.dummySubscription.completeFiles(files=dummyFileList)

        #Check if dummyFile3 was inserted at the completed Set
        assert dummyFile3 in self.dummySubscription.completed.getFiles(type='set'), \
        'Replicated file could\'nt be inserted at completed Set'

        #Check if dummyFile3 was erased from all the other Sets
        assert dummyFile3 not in self.dummySubscription.acquired.getFiles(type='set'), \
        'Completed file still present at acquired Set'
        assert dummyFile3 not in self.dummySubscription.failed.getFiles(type='set'), \
        'Completed file still present at failed Set'
        assert dummyFile3 not in self.dummySubscription.available.getFiles(type='set'), \
        'Completed file still present at available Set'

    def testFailFiles(self):
        """
        Testcase for the failFiles method of the Subscription Class

        """
        #Cleaning possible files already occupying the available set
        self.dummySubscription.failFiles([])

        #First test - Test if initial file (on available set) is inserted in the
        # failed set - no arguments

        dummyFile2 = File('/tmp/dummyfile2,8888',1,1,1)
        #Insert dummyFile2 into the available files Set at dummySubscription
        self.dummySubscription.available.addFile(dummyFile2)

        S = self.dummySubscription.availableFiles()
        # Fail all files
        self.dummySubscription.failFiles(S)

        assert len(self.dummySubscription.availableFiles()) == 0, \
        "failed subscription still has %s files, what's up with that?" %\
          len(self.dummySubscription.availableFiles())

        #Second test - Test if target files are inserted at the failed set

        dummyFileList = []
        #Populating the dummy List with a random number of files
        for i in range(1, random.randint(100,1000)):
            lfn = '/store/data/%s/%s/file.root' % (random.randint(1000, 9999),
                                              random.randint(1000, 9999))
            size = random.randint(1000, 2000)
            events = 1000
            run = random.randint(0, 2000)
            lumi = random.randint(0, 8)

            file = File(lfn = lfn, size = size, events = events,
                        checksums = {"cksum": "1"})
            file.addRun(Run(run, *[lumi]))
            dummyFileList.append(file)
        #Add the new files
        self.dummySubscription.available.addFile(dummyFileList)
        #and fail them
        self.dummySubscription.failFiles(files = dummyFileList)
        #Check there are no files available - everything should be failed
        assert len(self.dummySubscription.availableFiles()) == 0, \
        "failed subscription still has %s files, what's up with that?" %\
          len(self.dummySubscription.availableFiles())

        #Check if all files were inserted at subscription's failed files Set
        for x in dummyFileList:
            assert x in self.dummySubscription.failed.getFiles(type='set'),\
                 'Couldn\'t make file failed %s' % x.dict['lfn']

        #Third test - Test if a replicate file is erased from the other Sets,
        # when a file is considered failed

        dummyFile3 = File('/tmp/dummyfile3,5555',1,1,1)
        dummyFileList = []
        dummyFileList.append(dummyFile3)

        #Inserting dummyFile3 to be used as an argument, into each of the other
        # file sets
        self.dummySubscription.acquired.addFile(dummyFile3)
        self.dummySubscription.available.addFile(dummyFile3)
        self.dummySubscription.completed.addFile(dummyFile3)

        #Run the method failFiles
        self.dummySubscription.failFiles(files=dummyFileList)

        #Check if dummyFile3 was inserted at the failed Set
        assert dummyFile3 in self.dummySubscription.failed.getFiles(type='set'), \
        'Replicated file could\'nt be inserted at failed Set'

        #Check if dummyFile3 was erased from all the other Sets
        assert dummyFile3 not in self.dummySubscription.acquired.getFiles(type='set'), \
        'Failed file still present at acquired Set'
        assert dummyFile3 not in self.dummySubscription.completed.getFiles(type='set'), \
        'Failed file still present at completed Set'
        assert dummyFile3 not in self.dummySubscription.available.getFiles(type='set'), \
        'Failed file still present at available Set'

    def testFilesOfStatus(self):
        """
        Testcase for the filesOfStatus method of the Subscription Class

        """

        assert self.dummySubscription.filesOfStatus('Available') == \
          self.dummySubscription.available.getFiles(type='set') - \
            self.dummySubscription.acquiredFiles() | self.dummySubscription.completedFiles() | self.dummySubscription.failedFiles(), \
                'Method fileOfStatus(\'AvailableFiles\') does not return available files set'
        assert self.dummySubscription.filesOfStatus('Acquired') == self.dummySubscription.acquired.getFiles(type='set'), \
                'Method fileOfStatus(\'AcquiredFiles\') does not return acquired files set'
        assert self.dummySubscription.filesOfStatus('Completed') == self.dummySubscription.completed.getFiles(type='set'), \
                'Method fileOfStatus(\'CompletedFiles\') does not return completed files set'
        assert self.dummySubscription.filesOfStatus('Failed') == self.dummySubscription.failed.getFiles(type='set'), \
                'Method fileOfStatus(\'FailedFiles\') does not return failed files set'

    def testAvailableFiles(self):
        """
        Testcase for the availableFiles method of the Subscription Class
        """
        assert self.dummySubscription.availableFiles() == \
        self.dummySubscription.available.getFiles(type='set'), \
        'Method availableFiles does not return available files Set'

    def testAcquiredFiles(self):
        """
        Testcase for the acquiredFiles method of the Subscription Class
        """
        assert self.dummySubscription.acquiredFiles() == \
            self.dummySubscription.acquired.getFiles(type='set'), \
            'Method acquiredFiles does not return acquired files Set'

    def testCompletedFiles(self):
        """
        Testcase for the completedFiles method of the Subscription Class
        """
        assert self.dummySubscription.completedFiles() == \
            self.dummySubscription.completed.getFiles(type='set'), \
            'Method completedFiles does not return completed files Set'

    def testFailedFiles(self):
        """
        Testcase for the failedFiles method of the Subscription Class
        """
        assert self.dummySubscription.failedFiles() == \
            self.dummySubscription.failed.getFiles(type='set'), \
            'Method failedFiles does not return failed files Set'
Пример #13
0
class FilesetTest (unittest.TestCase):
    """ 
    _FilesetTest_

    Testcase for Fileset
    """
    def setUp(self):
        """
        Create a dummy fileset and populate it with random files,
        in order to use it for the testcase methods
        
        """
        logging.basicConfig(level=logging.DEBUG,
                    format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
                    datefmt='%m-%d %H:%M',
                    filename=__file__.replace('.py','.log'),
                    filemode='w')
        self.logger = logging.getLogger('FilesetClassTest')
        
        #Setup the initial testcase environment:
        initialfile = File('/tmp/lfn1',1000,1,1,1)
        self.initialSet = set()
        self.initialSet.add(initialfile)
        
        #Create a Fileset, containing a initial file on it.
        self.fileset = Fileset(name = 'self.fileset', files = self.initialSet)
        #Populate the fileset with random files
        for i in range(1,1000):
            lfn = '/store/data/%s/%s/file.root' % (random.randint(1000, 9999),
                                              random.randint(1000, 9999))
            size = random.randint(1000, 2000)
            events = 1000
            run = random.randint(0, 2000)
            lumi = random.randint(0, 8)

            file = File(lfn=lfn, size=size, events=events, checksums = {"cksum": "1"})
            file.addRun(Run(run, *[lumi]))
            self.fileset.addFile(file)
        
    def tearDown(self):
        """
            No tearDown method for this testcase
            
        """
        pass
    def testAddFile(self):
        """
            Testcase for the addFile method of the Fileset class
            
        """
        #First test - Add file and check if its there
        testfile = File('/tmp/lfntest',9999,9,9)
        self.fileset.addFile(testfile)
        assert(testfile in self.fileset.listNewFiles(), 'Couldn\'t add file ' +
                'to fileset - fileset.addfile method not working')
        #Second test - Add file that was already at Fileset.files , 
        # and check if it gets updated
        testFileSame = File('/tmp/lfntest',9999,9,9)
        testFileSame.setLocation(set('dummyse.dummy.com'))
        self.fileset.addFile(testFileSame)
        assert(testFileSame in  self.fileset.getFiles(),'Same file copy ' +
               'failed - fileset.addFile not updating location of already ' +
               'existing files' )
        assert(testfile in self.fileset.getFiles(),'Same file copy ' +
               'failed - fileset.addFile unable to remove previous file ' +
               'from list')
        #Third test - Add file that was already at Fileset.newfiles , 
        #and check if it gets updated
        assert(testFileSame in  self.fileset.listNewFiles(),'Same file copy ' +
               'failed - fileset.addFile not adding file to fileset.newFiles')
        
    def testListFiles(self):
        """
        _testListFiles_
        
        Testcase for the getFiles() method of the Fileset class
        """
        filesettemp = self.fileset.getFiles()
        for x in filesettemp:
            assert x in (self.fileset.files | self.fileset.newfiles), \
            'Missing file %s from file list returned from fileset.ListFiles' % x["lfn"]
            
    def testSetFiles(self):
        """
        Check that all files returned by the set are the same as those added to 
        the fileset
        """
        filesettemp = self.fileset.getFiles(type = "set")
        for x in filesettemp:
            assert x in (self.fileset.files | self.fileset.newfiles), \
            'Missing file %s from file list returned from fileset.ListFiles' % x["lfn"]
            
    def testSetListCompare(self):
        """
        Test that all files in fileset.setFiles are in fileset.getFiles()
        """
        thelist = self.fileset.getFiles()
        theset = self.fileset.getFiles(type = "set")
        for x in thelist:
            assert x in (theset), \
            'Missing file %s from file list returned from fileset.ListFiles' % x["lfn"]
    
    def testSorting(self):
        """
        Fileset.getFiles() should be sorted the same as Fileset.getFiles(type = "lfn"), 
        assert that this is the case here.
        """
        files = self.fileset.getFiles()
        lfns = self.fileset.getFiles(type = "lfn")
        for x in files:
            assert x["lfn"] in (lfns), \
            'Missing file %s from file list returned ' % x["lfn"]
            assert lfns[files.index(x)] == x["lfn"], \
            'Sorting not consistent: lfn = %s, file = %s' % (lfns[files.index(x)], x["lfn"])
            
    def testListLFNs(self):
        """
            Testcase for the listLFN method of the Fileset class
            
        """
        #Kinda slow way of verifying if the raw LFN from each file at the
        #fileset is returned from the meth
        allFiles = self.fileset.getFiles()
        
        #For each file returned by method listFiles, it checks if LFN is
        #present at the output of method listLFNs 
        for x in allFiles:
            assert x['lfn'] in self.fileset.getFiles(type = "lfn"), 'Missing %s from ' \
            'list returned from fileset.ListLFNs' % x["lfn"] 
        #Im a bit confused with this method, leave to discuss it at the meeting with Simon
        
    def testListNewFiles(self):
        """
            Testcase for the listNewFiles method of the Fileset class
            
        """
        newfilestemp = self.fileset.listNewFiles()
        assert newfilestemp == self.fileset.newfiles, 'Missing files from ' \
               'list returned from fileset.ListNewFiles'
    def testCommit(self):
        """
            Testcase for the commit method of the Fileset class
            
        """
        localTestFileSet = Fileset('LocalTestFileset', self.initialSet)
        fsSize = len(localTestFileSet.getFiles(type = "lfn"))
        #Dummy file to test
        fileTestCommit = File('/tmp/filetestcommit',0000,1,1)
        #File is added to the newfiles attribute of localTestFileSet
        localTestFileSet.addFile(fileTestCommit)
        assert fsSize == len(localTestFileSet.getFiles(type = "lfn")) - 1, 'file not added'\
                'correctly to test fileset'
        newfilestemp = localTestFileSet.newfiles
        assert fileTestCommit in newfilestemp, 'test file not in the new files'\
                'list' 
        #After commit, dummy file is supposed to move from newfiles to files
        localTestFileSet.commit()
        #First, testing if the new file is present at file set object attribute of the Fileset object
        
        assert newfilestemp.issubset(localTestFileSet.files), 'Test file not ' \
                'present at fileset.files - fileset.commit ' \
                'not working properly' 
        #Second, testing if the newfile set object attribute is empty
        assert localTestFileSet.newfiles == set(), \
                'Test file not present at fileset.newfiles ' \
                '- fileset.commit not working properly'
Пример #14
0
class JobGroup(WMObject):
    """
    JobGroups are sets of jobs running on files who's output needs to be merged
    together.
    """
    def __init__(self, subscription=None, jobs=None):
        self.jobs = []
        self.newjobs = []
        self.id = 0

        if isinstance(jobs, list):
            self.newjobs = jobs
        elif jobs is not None:
            self.newjobs = [jobs]

        self.subscription = subscription
        self.output = Fileset()
        self.last_update = datetime.datetime.now()

    def add(self, job):
        """
        _add_

        Add a Job or list of jobs to the JobGroup.
        """
        jobList = self.makelist(job)
        self.newjobs.extend(jobList)
        return

    def commit(self):
        """
        _commit_

        Move any jobs in the newjobs dict to the job dict.  Empty the newjobs
        dict.
        """
        self.jobs.extend(self.newjobs)
        self.newjobs = []

    def commitBulk(self):
        """
        Dummy method for consistency with WMBS implementation
        """
        self.commit()

    def addOutput(self, file):
        """
        _addOutput_

        Add a File to the JobGroup's output fileset.  The File is committed
        to the Fileset immediately.
        """
        self.output.addFile(file)
        self.output.commit()

    def getJobs(self, type="list"):
        """
        _getJobs_

        Retrieve all of the jobs in the JobGroup.  The output will either be
        returned as a list of Job objects (when type is "list") or a list of
        Job IDs (when type is "id").
        """
        if type == "list":
            return self.jobs
        elif type == "id":
            jobIDs = []

            for job in self.jobs:
                jobIDs.append(job["id"])

            return jobIDs
        else:
            print("Unknown type: %s" % type)

        return

    def getOutput(self, type="list"):
        """
        _getOutput_

        Retrieve all of the files that are in the JobGroup's output fileset.
        Type can be one of the following: list, set, lfn, id.
        """
        return self.output.getFiles(type=type)

    def getLength(self, obj):
        """
        This just gets a length for either dict or list objects
        """
        if isinstance(obj, (dict, list)):
            return len(obj)
        else:
            return 0

    def __len__(self):
        """
        Allows use of len() on JobGroup
        """
        return self.getLength(self.jobs) + self.getLength(self.newjobs)
Пример #15
0
class Subscription(Pickleable, dict):
    def __init__(self,
                 fileset=None,
                 workflow=None,
                 split_algo="FileBased",
                 type="Processing"):
        if fileset == None:
            fileset = Fileset()

        self.setdefault('fileset', fileset)
        self.setdefault('workflow', workflow)
        self.setdefault('type', type)

        self.setdefault('split_algo', split_algo)

        self.available = Fileset(name=fileset.name, files=fileset.getFiles())

        self.acquired = Fileset(name='acquired')
        self.completed = Fileset(name='completed')
        self.failed = Fileset(name='failed')

    def name(self):
        return self.getWorkflow().name.replace(' ', '') + '_' + \
                    self.getFileset().name.replace(' ', '')

    def getWorkflow(self):
        return self["workflow"]

    def workflowName(self):
        if self["workflow"] == None:
            return "Unknown"
        return self["workflow"].name

    def workflowType(self):
        if self["workflow"] == None:
            return "Unknown"
        return self["workflow"].wfType

    def taskName(self):
        if self['workflow'] == None:
            return "Unknown"
        return self['workflow'].task

    def owner(self):
        if self['workflow'] == None:
            return 'Unknown'
        return self['workflow'].owner

    def getFileset(self):
        return self['fileset']

    def acquireFiles(self, files=[], size=1):
        """
        Return the files acquired
        """
        self.acquired.commit()
        self.available.commit()
        self.failed.commit()
        self.completed.commit()
        retval = []
        if len(files):
            for i in files:
                # Check each set, instead of elif, just in case something has
                # got out of synch
                if i in self.available.files:
                    self.available.files.remove(i)
                if i in self.failed.files:
                    self.failed.files.remove(i)
                if i in self.completed.files:
                    self.completed.files.remove(i)
                self.acquired.addFile(i)
        else:
            if len(self.available.files) < size or size == 0:
                size = len(self.available.files)
            for i in range(size):
                self.acquired.addFile(self.available.files.pop())

        return self.acquired.listNewFiles()

    def completeFiles(self, files):
        """
        Return the number of files complete
        """
        self.acquired.commit()
        self.available.commit()
        self.failed.commit()
        self.completed.commit()
        for i in files:
            # Check each set, instead of elif, just in case something has
            # got out of synch
            if i in self.available.files:
                self.available.files.remove(i)
            if i in self.failed.files:
                self.failed.files.remove(i)
            if i in self.acquired.files:
                self.acquired.files.remove(i)
            self.completed.addFile(i)

    def failFiles(self, files):
        """
        Return the number of files failed
        """
        self.acquired.commit()
        self.available.commit()
        self.failed.commit()
        self.completed.commit()
        for i in files:
            # Check each set, instead of elif, just in case something has
            # got out of synch
            if i in self.available.files:
                self.available.files.remove(i)
            if i in self.completed.files:
                self.completed.files.remove(i)
            if i in self.acquired.files:
                self.acquired.files.remove(i)
            self.failed.addFile(i)

    def filesOfStatus(self, status=None, doingJobSplitting=False):
        """
        _filesOfStatus_

        Return a Set of File objects that are associated with the subscription
        and have a particular status.
        """
        status = status.title()
        if status == 'Available':
            return self.available.getFiles(type='set') - \
            (self.acquiredFiles() | self.completedFiles() | self.failedFiles())
        elif status == 'Acquired':
            return self.acquired.getFiles(type='set')
        elif status == 'Completed':
            return self.completed.getFiles(type='set')
        elif status == 'Failed':
            return self.failed.getFiles(type='set')

    def availableFiles(self, limit=None, doingJobSplitting=False):
        """
        _availableFiles_

        Return a Set of files that are available for processing
        (e.g. not already in use)
        """
        if limit:
            return list(
                self.filesOfStatus(
                    status="Available",
                    doingJobSplitting=doingJobSplitting))[:limit]
        else:
            return self.filesOfStatus(status="Available",
                                      doingJobSplitting=doingJobSplitting)

    def acquiredFiles(self):
        """
        Set of files marked as acquired.
        """
        return self.filesOfStatus(status="Acquired")

    def completedFiles(self):
        """
        Set of files marked as completed.
        """
        return self.filesOfStatus(status="Completed")

    def failedFiles(self):
        """
        Set of files marked as failed.
        """
        return self.filesOfStatus(status="Failed")