Esempio n. 1
0
    def testCommitBulk(self):
        """
        _testCommitBulk_

        Exactly the same as testCommit, but using commitBulk() instead of commit()
        """

        myThread = threading.currentThread()

        testJobGroupA = self.createLargerTestJobGroup(commitFlag=False)

        testJobGroupB = JobGroup(id=testJobGroupA.id)
        testJobGroupB.loadData()

        assert len(testJobGroupA.getJobs()) == 0, \
               "ERROR: Original object commited too early"
        assert len(testJobGroupB.getJobs()) == 0, \
               "ERROR: Loaded JobGroup has too many jobs"

        testJobGroupA.commitBulk()
        testJobGroupA.loadData()

        self.assertEqual(len(testJobGroupA.getJobs()), 102)

        testJobGroupC = JobGroup(id=testJobGroupA.id)
        testJobGroupC.loadData()

        self.assertEqual(len(testJobGroupC.getJobs()), 102)

        self.assertEqual(testJobGroupC.jobs[0].getFiles()[0]['lfn'],
                         '/this/is/a/lfnC')
        self.assertEqual(testJobGroupC.jobs[1].getFiles()[0]['lfn'],
                         '/this/is/a/lfnD')

        return
Esempio n. 2
0
    def testCommit(self):
        """
        _testCommit_

        Verify that jobs are not added to a job group until commit() is called
        on the JobGroup.  Also verify that commit() correctly commits the jobs
        to the database.
        """
        testJobGroupA = self.createTestJobGroup(commitFlag=False)

        testJobGroupB = JobGroup(id=testJobGroupA.id)
        testJobGroupB.loadData()

        assert len(testJobGroupA.getJobs()) == 0, \
               "ERROR: Original object commited too early"
        assert len(testJobGroupB.getJobs()) == 0, \
               "ERROR: Loaded JobGroup has too many jobs"

        testJobGroupA.commit()
        testJobGroupA.loadData()

        assert len(testJobGroupA.getJobs()) == 2, \
               "ERROR: Original object did not commit jobs"

        testJobGroupC = JobGroup(id=testJobGroupA.id)
        testJobGroupC.loadData()

        assert len(testJobGroupC.getJobs()) == 2, \
               "ERROR: Loaded object has too few jobs."

        return
Esempio n. 3
0
    def notestTwoJobGroups(self):
        """
        Test two job groups with a shared fileset. (Minimal part of testGetLocations which was failing)
        """

        testWorkflow1 = Workflow(spec="spec.xml",
                                 owner="Simon",
                                 name="wf001",
                                 task="Test1")
        testWorkflow1.create()

        testWMBSFileset1 = WMBSFileset(name="TestFileset1")
        testWMBSFileset1.create()

        testSubscription1 = Subscription(fileset=testWMBSFileset1,
                                         workflow=testWorkflow1)
        testSubscription1.create()

        testJobGroup1 = JobGroup(subscription=testSubscription1)
        testJobGroup1.create()

        testFileA = File(lfn="/this/is/a/lfnA", size=1024, events=10)
        testFileA.addRun(Run(10, *[12312]))
        testFileA.create()

        testJobA = Job(name="TestJobA")
        testJobA.addFile(testFileA)
        testJobGroup1.add(testJobA)

        testJobGroup1.commit()

        testWorkflow2 = Workflow(spec="spec.xml",
                                 owner="Simon",
                                 name="wf002",
                                 task="Test2")
        testWorkflow2.create()

        testWMBSFileset2 = WMBSFileset(name="TestFileset1")
        testWMBSFileset2.create()

        testSubscription2 = Subscription(fileset=testWMBSFileset2,
                                         workflow=testWorkflow2)
        testSubscription2.create()

        testJobGroup2 = JobGroup(subscription=testSubscription2)
        testJobGroup2.create()

        testFileC = File(lfn="/this/is/a/lfnC", size=1024, events=10)
        testFileC.addRun(Run(10, *[12312]))

        testFileC.create()

        testJobA1 = Job(name="TestJobA1")
        testJobA1.addFile(testFileC)

        testJobGroup2.add(testJobA1)
        testJobGroup2.commit()
Esempio n. 4
0
    def testLoad(self):
        """
        _testLoad_

        Test loading the JobGroup and any associated meta data from the
        database.
        """
        testJobGroupA = self.createTestJobGroup()

        testJobGroupB = JobGroup(id=testJobGroupA.id)
        testJobGroupB.load()
        testJobGroupC = JobGroup(uid=testJobGroupA.uid)
        testJobGroupC.load()

        assert type(testJobGroupB.id) == int, \
               "ERROR: Job group id is not an int."

        assert type(testJobGroupC.id) == int, \
               "ERROR: Job group id is not an int."

        assert type(testJobGroupB.subscription["id"]) == int, \
               "ERROR: Job group subscription id is not an int."

        assert type(testJobGroupC.subscription["id"]) == int, \
               "ERROR: Job group subscription id is not an int."

        assert type(testJobGroupB.output.id) == int, \
               "ERROR: Job group output id is not an int."

        assert type(testJobGroupC.output.id) == int, \
               "ERROR: Job group output id is not an int."

        assert testJobGroupB.uid == testJobGroupA.uid, \
               "ERROR: Job group did not load uid correctly."

        assert testJobGroupC.id == testJobGroupA.id, \
               "ERROR: Job group did not load id correctly."

        assert testJobGroupB.subscription["id"] == \
               testJobGroupA.subscription["id"], \
               "ERROR: Job group did not load subscription correctly"

        assert testJobGroupC.subscription["id"] == \
               testJobGroupA.subscription["id"], \
               "ERROR: Job group did not load subscription correctly"

        assert testJobGroupB.output.id == testJobGroupA.output.id, \
               "ERROR: Output fileset didn't load properly"

        assert testJobGroupC.output.id == testJobGroupA.output.id, \
               "ERROR: Output fileset didn't load properly"

        return
Esempio n. 5
0
    def testGetGroupsByJobStateDAO(self):
        """
        _testGetGroupsByJobStateDAO_

        Verify that the GetGrounsByJobState DAO does what it is supposed to do.
        """
        testWorkflow = Workflow(spec="spec.xml",
                                owner="Simon",
                                name="wf001",
                                task="Test")
        testWorkflow.create()

        testWMBSFileset = WMBSFileset(name="TestFileset")
        testWMBSFileset.create()

        testSubscription = Subscription(fileset=testWMBSFileset,
                                        workflow=testWorkflow)
        testSubscription.create()

        testJobGroupA = JobGroup(subscription=testSubscription)
        testJobGroupA.create()
        testJobGroupB = JobGroup(subscription=testSubscription)
        testJobGroupB.create()

        testJobA = Job(name="TestJobA")
        testJobB = Job(name="TestJobB")

        testJobGroupA.add(testJobA)
        testJobGroupB.add(testJobB)

        testJobGroupA.commit()
        testJobGroupB.commit()

        myThread = threading.currentThread()
        daofactory = DAOFactory(package="WMCore.WMBS",
                                logger=myThread.logger,
                                dbinterface=myThread.dbi)

        stateChangeAction = daofactory(classname="Jobs.ChangeState")
        testJobA["state"] = "complete"
        testJobB["state"] = "executing"
        stateChangeAction.execute(jobs=[testJobA, testJobB])

        jobGroupAction = daofactory(classname="JobGroup.GetGroupsByJobState")
        jobGroups = jobGroupAction.execute(jobState="complete")

        assert len(jobGroups) == 1, \
               "Error: Wrong number of job groups returned."
        assert jobGroups[0] == testJobGroupA.id, \
               "Error: Wrong job group returned."

        return
Esempio n. 6
0
    def createSingleJobWorkflow(self):
        """
        Create a workflow with one jobs and two files and store the results in instance variables
        """

        self.testWorkflow = Workflow(spec="spec.xml",
                                     owner="Simon",
                                     name="wf001",
                                     task="Test")
        self.testWorkflow.create()

        testWMBSFileset = Fileset(name="TestFileset")
        testWMBSFileset.create()

        testSubscription = Subscription(fileset=testWMBSFileset,
                                        workflow=self.testWorkflow)
        testSubscription.create()

        testJobGroup = JobGroup(subscription=testSubscription)
        testJobGroup.create()

        self.testFileA = File(lfn="/this/is/a/lfnA", size=1024, events=10)
        self.testFileA.addRun(Run(1, *[45]))
        self.testFileB = File(lfn="/this/is/a/lfnB", size=1024, events=10)
        self.testFileB.addRun(Run(1, *[46]))
        self.testFileA.create()
        self.testFileB.create()

        self.testJob = Job(name="TestJob",
                           files=[self.testFileA, self.testFileB])

        self.testJob.create(group=testJobGroup)
        self.testJob.associateFiles()
Esempio n. 7
0
    def testLoadData(self):
        """
        _testLoadData_

        Test loading the JobGroup, it's meta data and any data associated with
        its output fileset and jobs from the database.
        """
        testJobGroupA = self.createTestJobGroup()

        testJobGroupB = JobGroup(id=testJobGroupA.id)
        testJobGroupB.loadData()

        self.assertEqual(testJobGroupB.subscription["id"], testJobGroupA.subscription["id"],
                         "Job group did not load subscription correctly")

        # Build a tuple with important job information for each job group
        goldenJobs = [(job['id'], job['name'],
                       sorted([inp_file['lfn'] for inp_file in job['input_files']]))
                      for job in testJobGroupA.getJobs(type="list")]
        loadedJobs = [(job['id'], job['name'],
                       sorted([inp_file['lfn'] for inp_file in job['input_files']]))
                      for job in testJobGroupB.getJobs(type="list")]

        # Make sure each job from one group is in the other list
        for job in loadedJobs:
            self.assertIn(job, goldenJobs, "JobGroup loaded an unknown job: \n%s \nis not in  \n%s" %
                          (pretty(job), pretty(goldenJobs)))
            goldenJobs.remove(job)

        self.assertFalse(goldenJobs, "JobGroup didn't load all jobs")
        self.assertEqual(testJobGroupB.output.id, testJobGroupA.output.id, "Output fileset didn't load properly")

        return
Esempio n. 8
0
    def testCreateDeleteExists(self):
        """
        _testCreateDeleteExists_

        Create a JobGroup and then delete it.  Use the JobGroup's exists()
        method to determine if it exists before it is created, after it is
        created and after it is deleted.
        """
        testWorkflow = Workflow(spec="spec.xml", owner="Simon",
                                name="wf001", task="Test")
        testWorkflow.create()

        testFileset = WMBSFileset(name="TestFileset")
        testFileset.create()

        testSubscription = Subscription(fileset=testFileset,
                                        workflow=testWorkflow)
        testSubscription.create()

        testJobGroup = JobGroup(subscription=testSubscription)

        self.assertFalse(testJobGroup.exists())

        testJobGroup.create()

        self.assertTrue(testJobGroup.exists())

        testJobGroup.delete()

        self.assertFalse(testJobGroup.exists())

        testSubscription.delete()
        testFileset.delete()
        testWorkflow.delete()
        return
Esempio n. 9
0
    def createJobGroups(self,
                        nSubs,
                        nJobs,
                        task,
                        workloadSpec,
                        site,
                        bl=[],
                        wl=[],
                        taskType='Processing',
                        name=None):
        """
        _createJobGroups_

        Creates a series of jobGroups for submissions
        """

        jobGroupList = []

        if name is None:
            name = makeUUID()

        testWorkflow = Workflow(spec=workloadSpec,
                                owner="tapas",
                                name=name,
                                task="basicWorkload/Production")
        testWorkflow.create()

        # Create subscriptions
        for _ in range(nSubs):

            name = makeUUID()

            # Create Fileset, Subscription, jobGroup
            testFileset = Fileset(name=name)
            testFileset.create()
            testSubscription = Subscription(fileset=testFileset,
                                            workflow=testWorkflow,
                                            type=taskType,
                                            split_algo="FileBased")
            testSubscription.create()

            testJobGroup = JobGroup(subscription=testSubscription)
            testJobGroup.create()

            # Create jobs
            self.makeNJobs(name=name,
                           task=task,
                           nJobs=nJobs,
                           jobGroup=testJobGroup,
                           fileset=testFileset,
                           sub=testSubscription.exists(),
                           site=site,
                           bl=bl,
                           wl=wl)

            testFileset.commit()
            testJobGroup.commit()
            jobGroupList.append(testJobGroup)

        return jobGroupList
Esempio n. 10
0
    def testLoadData(self):
        """
        _testLoadData_

        Test loading the JobGroup, it's meta data and any data associated with
        its output fileset and jobs from the database.
        """
        testJobGroupA = self.createTestJobGroup()

        testJobGroupB = JobGroup(id=testJobGroupA.id)
        testJobGroupB.loadData()

        assert testJobGroupB.subscription["id"] == \
               testJobGroupA.subscription["id"], \
               "ERROR: Job group did not load subscription correctly"

        goldenJobs = testJobGroupA.getJobs(type="list")

        for job in testJobGroupB.getJobs(type="list"):
            assert job in goldenJobs, \
                   "ERROR: JobGroup loaded an unknown job: %s, %s" % (job, goldenJobs)
            goldenJobs.remove(job)

        assert len(goldenJobs) == 0, \
            "ERROR: JobGroup didn't load all jobs"

        assert testJobGroupB.output.id == testJobGroupA.output.id, \
               "ERROR: Output fileset didn't load properly"

        return
Esempio n. 11
0
    def testCreateDeleteExists(self):
        """
        _testCreateDeleteExists_

        Create and then delete a job.  Use the job class's exists() method to
        determine if the job has been written to the database before it is
        created, after it has been created and after it has been deleted.
        """
        testWorkflow = Workflow(spec="spec.xml",
                                owner="Simon",
                                name="wf001",
                                task="Test")
        testWorkflow.create()

        testWMBSFileset = Fileset(name="TestFileset")
        testWMBSFileset.create()

        testSubscription = Subscription(fileset=testWMBSFileset,
                                        workflow=testWorkflow)
        testSubscription.create()

        testJobGroup = JobGroup(subscription=testSubscription)
        testJobGroup.create()

        testFileA = File(lfn="/this/is/a/lfnA", size=1024, events=10)
        testFileB = File(lfn="/this/is/a/lfnB", size=1024, events=10)
        testFileA.create()
        testFileB.create()

        testJob = Job(name="TestJob", files=[testFileA, testFileB])
        testWU = WorkUnit(taskID=testWorkflow.id,
                          fileid=testFileA['id'],
                          runLumi=Run(1, *[44]))

        self.assertFalse(testJob.exists(), "Job exists before it was created")
        self.assertFalse(testWU.exists(),
                         "WorkUnit exists before it was created")

        testJob.create(group=testJobGroup)

        self.assertTrue(testJob.exists(),
                        "Job does not exist after it was created")
        self.assertFalse(testWU.exists(),
                         "WorkUnit exists when there is no work")

        # Test the getWorkflow method
        workflow = testJob.getWorkflow()
        self.assertEqual(workflow['task'], 'Test')
        self.assertEqual(workflow['name'], 'wf001')

        testJob.delete()

        self.assertFalse(testJob.exists(), "Job exists after it was deleted")
        self.assertFalse(testWU.exists(),
                         "WorkUnit exists after job is deleted")

        return
Esempio n. 12
0
    def testCommitTransaction(self):
        """
        _testCommitTransaction_

        Create a JobGroup and then add some jobs to it.  Begin a transaction
        and then call commit() on the JobGroup.  Verify that the newly committed
        jobs can be loaded from the database.  Rollback the transaction and then
        verify that the jobs that were committed before are no longer associated
        with the JobGroup.
        """
        testJobGroupA = self.createTestJobGroup(commitFlag=False)

        testJobGroupB = JobGroup(id=testJobGroupA.id)
        testJobGroupB.loadData()

        assert len(testJobGroupA.getJobs()) == 0, \
               "ERROR: Original object commited too early"

        assert len(testJobGroupB.getJobs()) == 0, \
               "ERROR: Loaded JobGroup has too many jobs"

        myThread = threading.currentThread()
        myThread.transaction.begin()

        testJobGroupA.commit()

        assert len(testJobGroupA.getJobs()) == 2, \
               "ERROR: Original object did not commit jobs"

        testJobGroupC = JobGroup(id=testJobGroupA.id)
        testJobGroupC.loadData()

        assert len(testJobGroupC.getJobs()) == 2, \
               "ERROR: Loaded object has too few jobs."

        myThread.transaction.rollback()

        testJobGroupD = JobGroup(id=testJobGroupA.id)
        testJobGroupD.loadData()

        assert len(testJobGroupD.getJobs()) == 0, \
               "ERROR: Loaded object has too many jobs."

        return
Esempio n. 13
0
    def createJobGroups(self,
                        nSubs,
                        nJobs,
                        task,
                        workloadSpec,
                        site=None,
                        bl=[],
                        wl=[]):
        """
        Creates a series of jobGroups for submissions

        """

        jobGroupList = []

        testWorkflow = Workflow(spec=workloadSpec,
                                owner="mnorman",
                                name=makeUUID(),
                                task="basicWorkload/Production",
                                owner_vogroup='phgroup',
                                owner_vorole='cmsrole')
        testWorkflow.create()

        # Create subscriptions
        for i in range(nSubs):

            name = makeUUID()

            # Create Fileset, Subscription, jobGroup
            testFileset = Fileset(name=name)
            testFileset.create()
            testSubscription = Subscription(fileset=testFileset,
                                            workflow=testWorkflow,
                                            type="Processing",
                                            split_algo="FileBased")
            testSubscription.create()

            testJobGroup = JobGroup(subscription=testSubscription)
            testJobGroup.create()

            # Create jobs
            self.makeNJobs(name=name,
                           task=task,
                           nJobs=nJobs,
                           jobGroup=testJobGroup,
                           fileset=testFileset,
                           sub=testSubscription.exists(),
                           site=site,
                           bl=bl,
                           wl=wl)

            testFileset.commit()
            testJobGroup.commit()
            jobGroupList.append(testJobGroup)

        return jobGroupList
Esempio n. 14
0
    def testDeleteTransaction(self):
        """
        _testDeleteTransaction_

        Create a new job and commit it to the database.  Start a new transaction
        and delete the file from the database.  Verify that the file has been
        deleted.  After that, roll back the transaction and verify that the
        job is once again in the database.
        """
        testWorkflow = Workflow(spec="spec.xml",
                                owner="Simon",
                                name="wf001",
                                task="Test")
        testWorkflow.create()

        testWMBSFileset = Fileset(name="TestFileset")
        testWMBSFileset.create()

        testSubscription = Subscription(fileset=testWMBSFileset,
                                        workflow=testWorkflow)
        testSubscription.create()

        testJobGroup = JobGroup(subscription=testSubscription)
        testJobGroup.create()

        testFileA = File(lfn="/this/is/a/lfnA", size=1024, events=10)
        testFileB = File(lfn="/this/is/a/lfnB", size=1024, events=10)
        testFileA.create()
        testFileB.create()

        testJob = Job(name="TestJob", files=[testFileA, testFileB])

        assert testJob.exists() == False, \
               "ERROR: Job exists before it was created"

        testJob.create(group=testJobGroup)

        assert testJob.exists() >= 0, \
               "ERROR: Job does not exist after it was created"

        myThread = threading.currentThread()
        myThread.transaction.begin()

        testJob.delete()

        assert testJob.exists() == False, \
               "ERROR: Job exists after it was delete"

        myThread.transaction.rollback()

        assert testJob.exists() >= 0, \
               "ERROR: Job does not exist after transaction was rolled back."

        return
Esempio n. 15
0
    def createJobGroups(self, nSubs, nJobs, task, workloadSpec, site,
                        taskType='Processing', name=None, wfPrio=1, changeState=None):
        """
        _createJobGroups_

        Creates a series of jobGroups for submissions
        changeState is an instance of the ChangeState class to make job status changes
        """

        jobGroupList = []

        if name is None:
            name = makeUUID()

        testWorkflow = Workflow(spec=workloadSpec, owner="tapas",
                                name=name, task="basicWorkload/Production",
                                priority=wfPrio)
        testWorkflow.create()

        # Create subscriptions
        for _ in range(nSubs):
            name = makeUUID()

            # Create Fileset, Subscription, jobGroup
            testFileset = Fileset(name=name)
            testFileset.create()
            testSubscription = Subscription(fileset=testFileset,
                                            workflow=testWorkflow,
                                            type=taskType,
                                            split_algo="FileBased")
            testSubscription.create()

            testJobGroup = JobGroup(subscription=testSubscription)
            testJobGroup.create()

            # Create jobs
            self.makeNJobs(name=name, task=task,
                           nJobs=nJobs,
                           jobGroup=testJobGroup,
                           fileset=testFileset,
                           sub=testSubscription.exists(),
                           site=site)

            testFileset.commit()
            testJobGroup.commit()
            jobGroupList.append(testJobGroup)

        if changeState:
            for group in jobGroupList:
                changeState.propagate(group.jobs, 'created', 'new')

        return jobGroupList
Esempio n. 16
0
    def createLargerTestJobGroup(self, commitFlag=True):
        """
        _createTestJobGroup_

        """
        testWorkflow = Workflow(spec="spec.xml",
                                owner="Simon",
                                name="wf001",
                                task="Test")
        testWorkflow.create()

        testWMBSFileset = WMBSFileset(name="TestFileset")
        testWMBSFileset.create()

        testSubscription = Subscription(fileset=testWMBSFileset,
                                        workflow=testWorkflow)
        testSubscription.create()

        testJobGroup = JobGroup(subscription=testSubscription)
        testJobGroup.create()

        testFileC = File(lfn="/this/is/a/lfnC", size=1024, events=10)
        testFileC.addRun(Run(10, *[12312]))
        testFileC.setLocation("goodse.cern.ch")
        testFileC.setLocation("malpaquet")

        testFileD = File(lfn="/this/is/a/lfnD", size=1024, events=10)
        testFileD.addRun(Run(10, *[12312]))
        testFileD.setLocation("goodse.cern.ch")
        testFileD.setLocation("malpaquet")

        testFileC.create()
        testFileD.create()

        testJobA = Job(name="TestJobA1")
        testJobA.addFile(testFileC)

        testJobB = Job(name="TestJobB1")
        testJobB.addFile(testFileD)

        testJobGroup.add(testJobA)
        testJobGroup.add(testJobB)

        for i in range(0, 100):
            testJob = Job(name="TestJob%i" % (i))
            testJob.addFile(testFileC)
            testJobGroup.add(testJob)

        if commitFlag:
            testJobGroup.commit()

        return testJobGroup
Esempio n. 17
0
    def createTestJobs(self, nJobs, cacheDir):
        """
        _createTestJobs_

        Create several jobs
        """

        testWorkflow = Workflow(spec="spec.xml",
                                owner="Simon",
                                name="wf001",
                                task="Test")
        testWorkflow.create()

        testWMBSFileset = Fileset(name="TestFileset")
        testWMBSFileset.create()

        testSubscription = Subscription(fileset=testWMBSFileset,
                                        workflow=testWorkflow,
                                        type="Processing",
                                        split_algo="FileBased")
        testSubscription.create()

        testJobGroup = JobGroup(subscription=testSubscription)
        testJobGroup.create()

        # Create a file
        testFileA = File(lfn="/this/is/a/lfnA", size=1024, events=10)
        testFileA.addRun(Run(10, *[12312]))
        testFileA.setLocation('malpaquet')
        testFileA.create()

        baseName = makeUUID()

        # Now create a job
        for i in range(nJobs):
            testJob = Job(name='%s-%i' % (baseName, i))
            testJob.addFile(testFileA)
            testJob['location'] = 'malpaquet'
            testJob['retry_count'] = 1
            testJob['retry_max'] = 10
            testJob.create(testJobGroup)
            testJob.save()
            testJobGroup.add(testJob)

        testJobGroup.commit()

        # Set test job caches
        for job in testJobGroup.jobs:
            job.setCache(cacheDir)

        return testJobGroup
Esempio n. 18
0
    def testDeleteTransaction(self):
        """
        _testDeleteTransaction_

        Create a JobGroup and then commit it to the database.  Begin a
        transaction and the delete the JobGroup from the database.  Using the
        exists() method verify that the JobGroup is not in the database.
        Finally, roll back the transaction and verify that the JobGroup is
        in the database.
        """
        testWorkflow = Workflow(spec="spec.xml",
                                owner="Simon",
                                name="wf001",
                                task="Test")
        testWorkflow.create()

        testFileset = WMBSFileset(name="TestFileset")
        testFileset.create()

        testSubscription = Subscription(fileset=testFileset,
                                        workflow=testWorkflow)
        testSubscription.create()

        testJobGroup = JobGroup(subscription=testSubscription)

        assert testJobGroup.exists() == False, \
               "ERROR: Job group exists before it was created"

        testJobGroup.create()

        assert testJobGroup.exists() >= 0, \
               "ERROR: Job group does not exist after it was created"

        myThread = threading.currentThread()
        myThread.transaction.begin()

        testJobGroup.delete()

        assert testJobGroup.exists() == False, \
               "ERROR: Job group exists after it was deleted"

        myThread.transaction.rollback()

        assert testJobGroup.exists() >= 0, \
               "ERROR: Job group does not exist after transaction was rolled back."

        testSubscription.delete()
        testFileset.delete()
        testWorkflow.delete()
        return
Esempio n. 19
0
    def createTestJob(self, testSubscription, jobName, *testFiles):
        """
        _createTestJob_

        Create a test job with two files as input.  This will also create the
        appropriate workflow, jobgroup and subscription.
        """

        testJobGroup = JobGroup(subscription=testSubscription)
        testJobGroup.create()

        testFiles = list(testFiles)
        testJob = Job(name=jobName, files=testFiles)
        testJob["couch_record"] = "somecouchrecord"
        testJob["location"] = "test.site.ch"
        testJob.create(group=testJobGroup)
Esempio n. 20
0
    def createTestJobGroup(self):
        """
        Creates a group of several jobs

        """

        testWorkflow = Workflow(spec="spec.xml",
                                owner="Simon",
                                name="wf001",
                                task="Test")
        testWorkflow.create()

        testWMBSFileset = Fileset(name="TestFileset")
        testWMBSFileset.create()

        testFileA = File(lfn="/this/is/a/lfnA", size=1024, events=10)
        testFileA.addRun(Run(10, *[12312]))
        testFileA.setLocation('malpaquet')

        testFileB = File(lfn="/this/is/a/lfnB", size=1024, events=10)
        testFileB.addRun(Run(10, *[12312]))
        testFileA.setLocation('malpaquet')
        testFileA.create()
        testFileB.create()

        testWMBSFileset.addFile(testFileA)
        testWMBSFileset.addFile(testFileB)
        testWMBSFileset.commit()

        testSubscription = Subscription(fileset=testWMBSFileset,
                                        workflow=testWorkflow)
        testSubscription.create()

        testJobGroup = JobGroup(subscription=testSubscription)
        testJobGroup.create()

        for i in range(0, self.nJobs):
            testJob = Job(name=makeUUID())
            testJob.addFile(testFileA)
            testJob.addFile(testFileB)
            testJob['retry_count'] = 1
            testJob['retry_max'] = 10
            testJobGroup.add(testJob)

        testJobGroup.commit()

        return testJobGroup
Esempio n. 21
0
    def createTestJobGroup(self, commitFlag=True):
        """
        _createTestJobGroup_

        """
        testWorkflow = Workflow(spec="spec.xml",
                                owner="Simon",
                                name="wf001",
                                task="Test")
        testWorkflow.create()

        testWMBSFileset = WMBSFileset(name="TestFileset")
        testWMBSFileset.create()

        testSubscription = Subscription(fileset=testWMBSFileset,
                                        workflow=testWorkflow)
        testSubscription.create()

        testJobGroup = JobGroup(subscription=testSubscription)
        testJobGroup.create()

        testFileA = File(lfn="/this/is/a/lfnA", size=1024, events=10)
        testFileA.addRun(Run(10, *[12312]))
        testFileA.setLocation("goodse.cern.ch")
        testFileA.setLocation("malpaquet")

        testFileB = File(lfn="/this/is/a/lfnB", size=1024, events=10)
        testFileB.addRun(Run(10, *[12312]))
        testFileB.setLocation("goodse.cern.ch")
        testFileB.setLocation("malpaquet")

        testFileA.create()
        testFileB.create()

        testJobA = Job(name="TestJobA")
        testJobA.addFile(testFileA)

        testJobB = Job(name="TestJobB")
        testJobB.addFile(testFileB)

        testJobGroup.add(testJobA)
        testJobGroup.add(testJobB)

        if commitFlag:
            testJobGroup.commit()

        return testJobGroup
Esempio n. 22
0
    def createDummyJobs(self, nJobs, location=None):
        """
        _createDummyJobs_

        Create some dummy jobs
        """

        if not location:
            location = self.sites[0]

        nameStr = makeUUID()

        testWorkflow = Workflow(spec=nameStr,
                                owner="tapas",
                                name=nameStr,
                                task="basicWorkload/Production",
                                owner_vogroup='phgroup',
                                owner_vorole='cmsrole')
        testWorkflow.create()

        testFileset = Fileset(name=nameStr)
        testFileset.create()

        testSubscription = Subscription(fileset=testFileset,
                                        workflow=testWorkflow,
                                        type="Processing",
                                        split_algo="FileBased")
        testSubscription.create()

        testJobGroup = JobGroup(subscription=testSubscription)
        testJobGroup.create()

        jobList = []

        for i in range(nJobs):
            testJob = Job(name='%s-%i' % (nameStr, i))
            testJob['location'] = location
            testJob['custom']['location'] = location
            testJob['userdn'] = 'tapas'
            testJob['owner'] = 'tapas'
            testJob['userrole'] = 'cmsrole'
            testJob['usergroup'] = 'phgroup'

            testJob.create(testJobGroup)
            jobList.append(testJob)

        return jobList
Esempio n. 23
0
    def testDeleteTransaction(self):
        """
        _testDeleteTransaction_

        Create a JobGroup and then commit it to the database.  Begin a
        transaction and the delete the JobGroup from the database.  Using the
        exists() method verify that the JobGroup is not in the database.
        Finally, roll back the transaction and verify that the JobGroup is
        in the database.
        """
        testWorkflow = Workflow(spec="spec.xml",
                                owner="Simon",
                                name="wf001",
                                task="Test")
        testWorkflow.create()

        testFileset = WMBSFileset(name="TestFileset")
        testFileset.create()

        testSubscription = Subscription(fileset=testFileset,
                                        workflow=testWorkflow)
        testSubscription.create()

        testJobGroup = JobGroup(subscription=testSubscription)

        self.assertFalse(testJobGroup.exists())

        testJobGroup.create()

        self.assertTrue(testJobGroup.exists())

        myThread = threading.currentThread()
        myThread.transaction.begin()

        testJobGroup.delete()

        self.assertFalse(testJobGroup.exists())

        myThread.transaction.rollback()

        self.assertTrue(testJobGroup.exists())

        testSubscription.delete()
        testFileset.delete()
        testWorkflow.delete()
        return
Esempio n. 24
0
    def createTestJobGroup(self, nJobs, subType="Processing", retryOnce=False):
        """
        _createTestJobGroup_

        Creates a group of several jobs
        """
        testWorkflow = Workflow(spec="spec.xml",
                                owner="Simon",
                                name=makeUUID(),
                                task="Test")
        testWorkflow.create()

        testWMBSFileset = Fileset(name="TestFileset")
        testWMBSFileset.create()
        testSubscription = Subscription(fileset=testWMBSFileset,
                                        workflow=testWorkflow,
                                        type=subType)
        testSubscription.create()

        testJobGroup = JobGroup(subscription=testSubscription)
        testJobGroup.create()

        testFileA = File(lfn="/this/is/a/lfnA", size=1024, events=10)
        testFileA.addRun(Run(10, *[12312]))
        testFileA.setLocation('malpaquet')

        testFileB = File(lfn="/this/is/a/lfnB", size=1024, events=10)
        testFileB.addRun(Run(10, *[12312]))
        testFileA.setLocation('malpaquet')
        testFileA.create()
        testFileB.create()

        for _ in range(0, nJobs):
            testJob = Job(name=makeUUID())
            testJob.addFile(testFileA)
            testJob.addFile(testFileB)
            testJob['cache_dir'] = os.path.join(self.testDir, testJob['name'])
            os.mkdir(testJob['cache_dir'])
            testJobGroup.add(testJob)

        testJobGroup.commit()
        if retryOnce:
            self.increaseRetry.execute(testJobGroup.jobs)

        return testJobGroup
Esempio n. 25
0
    def testCreateDeleteExists(self):
        """
        _testCreateDeleteExists_

        Create and then delete a job.  Use the job class's exists() method to
        determine if the job has been written to the database before it is
        created, after it has been created and after it has been deleted.
        """
        testWorkflow = Workflow(spec="spec.xml",
                                owner="Simon",
                                name="wf001",
                                task="Test")
        testWorkflow.create()

        testWMBSFileset = Fileset(name="TestFileset")
        testWMBSFileset.create()

        testSubscription = Subscription(fileset=testWMBSFileset,
                                        workflow=testWorkflow)
        testSubscription.create()

        testJobGroup = JobGroup(subscription=testSubscription)
        testJobGroup.create()

        testFileA = File(lfn="/this/is/a/lfnA", size=1024, events=10)
        testFileB = File(lfn="/this/is/a/lfnB", size=1024, events=10)
        testFileA.create()
        testFileB.create()

        testJob = Job(name="TestJob", files=[testFileA, testFileB])

        assert testJob.exists() == False, \
               "ERROR: Job exists before it was created"

        testJob.create(group=testJobGroup)

        assert testJob.exists() >= 0, \
               "ERROR: Job does not exist after it was created"

        testJob.delete()

        assert testJob.exists() == False, \
               "ERROR: Job exists after it was delete"

        return
Esempio n. 26
0
    def testCreateDeleteExistsNoFiles(self):
        """
        _testCreateDeleteExistsNoFiles_

        Create and then delete a job but don't add any input files to it.
        Use the job class's exists() method to determine if the job has been
        written to the database before it is created, after it has been created
        and after it has been deleted.
        """
        testWorkflow = Workflow(spec="spec.xml",
                                owner="Simon",
                                name="wf001",
                                task="Test")
        testWorkflow.create()

        testWMBSFileset = Fileset(name="TestFileset")
        testWMBSFileset.create()

        testSubscription = Subscription(fileset=testWMBSFileset,
                                        workflow=testWorkflow)
        testSubscription.create()

        testJobGroup = JobGroup(subscription=testSubscription)
        testJobGroup.create()

        testJob = Job(name="TestJob")

        assert testJob.exists() == False, \
               "ERROR: Job exists before it was created"

        testJob.create(group=testJobGroup)

        assert testJob.exists() >= 0, \
               "ERROR: Job does not exist after it was created"

        testJob.delete()

        assert testJob.exists() == False, \
               "ERROR: Job exists after it was delete"

        return
Esempio n. 27
0
    def testLoadOutputID(self):
        """
        _testLoadOutputID_

        Test whether we can load an output ID for a job
        """

        testWorkflow = Workflow(spec="spec.xml",
                                owner="Steve",
                                name="wf001",
                                task="Test")

        testWorkflow.create()

        testFileset = Fileset(name="TestFileset")
        testFileset.create()

        testSubscription = Subscription(fileset=testFileset,
                                        workflow=testWorkflow)

        testSubscription.create()

        testFileA = File(lfn=makeUUID(), locations="test.site.ch")
        testFileB = File(lfn=makeUUID(), locations="test.site.ch")
        testFileA.create()
        testFileB.create()

        testFileset.addFile([testFileA, testFileB])
        testFileset.commit()

        testSubscription.acquireFiles([testFileA, testFileB])

        testJobGroup = JobGroup(subscription=testSubscription)
        testJobGroup.create()

        testJob = Job()
        testJob.create(group=testJobGroup)

        self.assertEqual(testJob.loadOutputID(), testJobGroup.output.id)

        return
Esempio n. 28
0
    def createTestJob(subscriptionType="Merge"):
        """
        _createTestJob_

        Create a test job with two files as input.  This will also create the
        appropriate workflow, jobgroup and subscription.
        """

        testWorkflow = Workflow(spec=makeUUID(),
                                owner="Simon",
                                name=makeUUID(),
                                task="Test")
        testWorkflow.create()

        testWMBSFileset = Fileset(name="TestFileset")
        testWMBSFileset.create()

        testSubscription = Subscription(fileset=testWMBSFileset,
                                        workflow=testWorkflow,
                                        type=subscriptionType)
        testSubscription.create()

        testJobGroup = JobGroup(subscription=testSubscription)
        testJobGroup.create()

        testFileA = File(lfn="/this/is/a/lfnA", size=1024, events=10)
        testFileA.addRun(Run(1, *[45]))
        testFileB = File(lfn="/this/is/a/lfnB", size=1024, events=10)
        testFileB.addRun(Run(1, *[46]))
        testFileA.create()
        testFileB.create()

        testJob = Job(name=makeUUID(), files=[testFileA, testFileB])
        testJob["couch_record"] = "somecouchrecord"
        testJob["location"] = "test.site.ch"
        testJob.create(group=testJobGroup)
        testJob.associateFiles()

        return testJob
Esempio n. 29
0
    def createJobs(self, nJobs):
        """
        Creates a series of jobGroups for submissions

        """

        testWorkflow = Workflow(spec="dummy",
                                owner="mnorman",
                                name="dummy",
                                task="basicWorkload/Production")
        testWorkflow.create()

        # Create Fileset, Subscription, jobGroup
        testFileset = Fileset(name="dummy")
        testFileset.create()
        testSubscription = Subscription(fileset=testFileset,
                                        workflow=testWorkflow,
                                        type="Processing",
                                        split_algo="FileBased")
        testSubscription.create()

        testJobGroup = JobGroup(subscription=testSubscription)
        testJobGroup.create()

        # Create jobs
        for id in range(nJobs):
            testJob = Job(name='Job_%i' % (id))
            testJob['owner'] = "mnorman"
            testJob['location'] = 'Xanadu'
            testJob.create(testJobGroup)
            testJobGroup.add(testJob)

        testFileset.commit()
        testJobGroup.commit()

        return testJobGroup
Esempio n. 30
0
    def injectJobs(self):
        """
        _injectJobs_

        Inject two workflows into WMBS and save the job objects to disk.
        """
        testWorkflowA = Workflow(spec="specA.pkl",
                                 owner="Steve",
                                 name="wf001",
                                 task="TestTaskA")
        testWorkflowA.create()
        testWorkflowB = Workflow(spec="specB.pkl",
                                 owner="Steve",
                                 name="wf002",
                                 task="TestTaskB")
        testWorkflowB.create()

        testFileset = Fileset("testFileset")
        testFileset.create()

        testSubA = Subscription(fileset=testFileset, workflow=testWorkflowA)
        testSubA.create()
        testSubB = Subscription(fileset=testFileset, workflow=testWorkflowB)
        testSubB.create()

        testGroupA = JobGroup(subscription=testSubA)
        testGroupA.create()
        testGroupB = JobGroup(subscription=testSubB)
        testGroupB.create()

        stateChanger = ChangeState(self.createConfig(),
                                   "jobsubmittercaching_t")

        for i in range(10):
            newFile = File(lfn="testFile%s" % i,
                           locations=set(["se.T1_US_FNAL", "se.T1_UK_RAL"]))
            newFile.create()

            newJobA = Job(name="testJobA-%s" % i, files=[newFile])
            newJobA["workflow"] = "wf001"
            newJobA["possiblePSN"] = ["T1_US_FNAL"]
            newJobA["sandbox"] = "%s/somesandbox" % self.testDir
            newJobA["owner"] = "Steve"

            jobCacheDir = os.path.join(self.testDir, "jobA-%s" % i)
            os.mkdir(jobCacheDir)
            newJobA["cache_dir"] = jobCacheDir
            newJobA["type"] = "Processing"
            newJobA['requestType'] = 'ReReco'
            newJobA.create(testGroupA)

            jobHandle = open(os.path.join(jobCacheDir, "job.pkl"), "wb")
            pickle.dump(newJobA, jobHandle)
            jobHandle.close()

            stateChanger.propagate([newJobA], "created", "new")

            newJobB = Job(name="testJobB-%s" % i, files=[newFile])
            newJobB["workflow"] = "wf001"
            newJobB["possiblePSN"] = ["T1_UK_RAL"]
            newJobB["sandbox"] = "%s/somesandbox" % self.testDir
            newJobB["owner"] = "Steve"

            jobCacheDir = os.path.join(self.testDir, "jobB-%s" % i)
            os.mkdir(jobCacheDir)
            newJobB["cache_dir"] = jobCacheDir
            newJobB["type"] = "Processing"
            newJobB['requestType'] = 'ReReco'
            newJobB.create(testGroupB)

            jobHandle = open(os.path.join(jobCacheDir, "job.pkl"), "wb")
            pickle.dump(newJobB, jobHandle)
            jobHandle.close()

            stateChanger.propagate([newJobB], "created", "new")

        return