Пример #1
0
    def testLen(self):
        """
        __testLen__

        Test that the __len__ function will actualy return the correct length.

        """

        #This is simple.  It should just have a length equal to the number of committed
        #And yet to be committed jobs

        testJobA = Job()
        testJobB = Job()
        testJobC = Job()
        testJobD = Job()
        testJobGroup = JobGroup(jobs = [testJobA, testJobB])
        testJobGroup.commit()

        self.assertEqual(len(testJobGroup), 2)

        testJobGroup.add(testJobC)

        self.assertEqual(len(testJobGroup), 3)

        testJobGroup.commit()
        testJobGroup.add(testJobD)

        self.assertEqual(len(testJobGroup), 4)

        return
Пример #2
0
    def setUp(self):
        """
        _setUp_

        Initial Setup for the Job Testcase
        """
        self.inputFiles = []

        for i in range(1, 1000):
            lfn = "/store/data/%s/%s/file.root" % (random.randint(
                1000, 9999), random.randint(1000, 9999))
            size = random.randint(1000, 2000)
            events = 1000
            run = random.randint(0, 2000)
            lumi = random.randint(0, 8)

            file = File(lfn=lfn,
                        size=size,
                        events=events,
                        checksums={"cksum": "1"})
            file.addRun(Run(run, *[lumi]))
            self.inputFiles.append(file)

        self.dummyJob = Job(files=self.inputFiles)
        return
Пример #3
0
    def testA(self):
        """instantiate"""

        document = Document()
        document[u'pset_tweak_details'] = {}
        document[u'pset_tweak_details'][u'process'] = {}
        document[u'pset_tweak_details'][u'process'][
            u'RandomNumberGeneratorService'] = {}
        document[u'pset_tweak_details'][u'process'][
            u'RandomNumberGeneratorService'][u'seed1'] = {}
        document[u'pset_tweak_details'][u'process'][
            u'RandomNumberGeneratorService'][u'seed2'] = {}
        document[u'pset_tweak_details'][u'process'][
            u'RandomNumberGeneratorService'][u'seed3'] = {}

        document = self.database.commitOne(document)[0]
        seeder = ReproducibleSeeding(CouchUrl=self.testInit.couchUrl,
                                     CouchDBName=self.testInit.couchDbName,
                                     ConfigCacheDoc=document[u'id'])

        job = Job("testjob")
        seeder(job)
        baggage = job.getBaggage()
        seed1 = getattr(baggage.process.RandomNumberGeneratorService, "seed1",
                        None)
        self.assertTrue(seed1 != None)
Пример #4
0
def main():
    """main functionf for testing"""
    from WMCore.DataStructs.Job import Job
    from WMCore.DataStructs.File import File
    from WMCore.DataStructs.Run import Run
    from WMCore.DataStructs.JobPackage import JobPackage
    from WMCore.Services.UUID import makeUUID
    from WMCore.WMSpec.Makers.TaskMaker import TaskMaker

    factory = HarvestingWorkloadFactory()
    workload = factory("derp", getTestArguments())

    task = workload.getTask('Harvesting')

    job = Job("SampleJob")
    job["id"] = makeUUID()
    job["task"] = task.getPathName()
    job["workflow"] = workload.name()

    file = File(lfn="/store/relval/CMSSW_3_8_2/RelValMinBias/GEN-SIM-RECO/MC_38Y_V9-v1/0019/FEC5BB4D-BFAF-DF11-A52A-001A92810AD2.root")
    job.addFile(file)

    jpackage = JobPackage()
    jpackage[1] = job

    import pickle
    
    handle = open("%s/JobPackage.pkl" % os.getcwd(), 'w')
    pickle.dump(jpackage, handle)
    handle.close()

    taskMaker = TaskMaker(workload, os.getcwd())
    taskMaker.skipSubscription = True
    taskMaker.processWorkload()
    task.build(os.getcwd())
Пример #5
0
    def testBaggage(self):
        """
        _testBaggage_

        Verify that job baggage is persisted with the package.
        """
        package = JobPackage()

        for i in range(100):
            newJob = Job("Job%s" % i)
            newJob["id"] = i
            baggage = newJob.getBaggage()
            setattr(baggage, "thisJob", newJob["name"])
            setattr(baggage, "seed1", 11111111)
            setattr(baggage, "seed2", 22222222)
            setattr(baggage, "seed3", 33333333)
            setattr(baggage, "seed4", 44444444)
            setattr(baggage, "seed5", 55555555)
            package[i] = newJob

        package.save(self.persistFile)

        assert os.path.exists(self.persistFile), \
               "Error: Package file was never created."

        newPackage = JobPackage()
        newPackage.load(self.persistFile)

        # There is an extra key for the directory the package is stored in.
        assert len(newPackage.keys()) == 101, \
               "Error: Wrong number of jobs in package."

        for i in range(100):
            job = newPackage[i]
            assert job["id"] == i, \
                   "Error: Jobs has wrong ID."
            assert job["name"] == "Job%d" % i, \
                   "Error: Job has wrong name."
            jobBaggage = job.getBaggage()

            assert jobBaggage.thisJob == "Job%d" % i, \
                   "Error: Job baggage has wrong name."
            assert jobBaggage.seed1 == 11111111, \
                   "Error: Job baggee has wrong value for seed1."
            assert jobBaggage.seed2 == 22222222, \
                   "Error: Job baggee has wrong value for seed2."
            assert jobBaggage.seed3 == 33333333, \
                   "Error: Job baggee has wrong value for seed3."
            assert jobBaggage.seed4 == 44444444, \
                   "Error: Job baggee has wrong value for seed4."
            assert jobBaggage.seed5 == 55555555, \
                   "Error: Job baggee has wrong value for seed5."


        return
Пример #6
0
    def testBaggage(self):
        """
        _testBaggage_

        Verify that job baggage is persisted with the package.
        """
        package = JobPackage()

        for i in range(100):
            newJob = Job("Job%s" % i)
            newJob["id"] = i
            baggage = newJob.getBaggage()
            setattr(baggage, "thisJob", newJob["name"])
            setattr(baggage, "seed1", 11111111)
            setattr(baggage, "seed2", 22222222)
            setattr(baggage, "seed3", 33333333)
            setattr(baggage, "seed4", 44444444)
            setattr(baggage, "seed5", 55555555)
            package[i] = newJob

        package.save(self.persistFile)

        assert os.path.exists(self.persistFile), \
               "Error: Package file was never created."

        newPackage = JobPackage()
        newPackage.load(self.persistFile)

        # There is an extra key for the directory the package is stored in.
        assert len(newPackage.keys()) == 101, \
               "Error: Wrong number of jobs in package."

        for i in range(100):
            job = newPackage[i]
            assert job["id"] == i, \
                   "Error: Jobs has wrong ID."
            assert job["name"] == "Job%d" % i, \
                   "Error: Job has wrong name."
            jobBaggage = job.getBaggage()

            assert jobBaggage.thisJob == "Job%d" % i, \
                   "Error: Job baggage has wrong name."
            assert jobBaggage.seed1 == 11111111, \
                   "Error: Job baggee has wrong value for seed1."
            assert jobBaggage.seed2 == 22222222, \
                   "Error: Job baggee has wrong value for seed2."
            assert jobBaggage.seed3 == 33333333, \
                   "Error: Job baggee has wrong value for seed3."
            assert jobBaggage.seed4 == 44444444, \
                   "Error: Job baggee has wrong value for seed4."
            assert jobBaggage.seed5 == 55555555, \
                   "Error: Job baggee has wrong value for seed5."

        return
Пример #7
0
    def testFirstEvent(self):
        """
        _testFirstEvent_

        Verify that we set the firstEvent parameter whenever the FirstEvent
        field in the job mask is a positive integer. And the job is
        not production.
        """
        job = Job()
        job["input_files"] = [{"lfn": "bogusFile", "parents": []}]
        job["mask"] = Mask()

        tweak = PSetTweak()
        WMTweaks.makeJobTweak(job, tweak)

        self.assertFalse(hasattr(tweak.process.source, "skipEvents"),
                         "Error: There should be no skipEvents tweak.")
        self.assertFalse(hasattr(tweak.process.source, "firstEvent"),
                         "Error: There should be no firstEvent tweak.")

        job["mask"]["FirstEvent"] = 0
        tweak = PSetTweak()
        WMTweaks.makeJobTweak(job, tweak)

        self.assertTrue(hasattr(tweak.process.source, "skipEvents"),
                        "Error: There should be a skipEvents tweak.")
        self.assertEqual(tweak.process.source.skipEvents,
                         'customTypeCms.untracked.uint32(0)',
                         "Error: The skipEvents tweak should be 0.")
        return
Пример #8
0
    def testFirstRun(self):
        """
        _testFirstRun_
        Verify that when we set the FirstRun in the mask, it is set in the
        process but when it is not, then no firstRun appears in the process.
        This for jobs with real input files.
        """
        job = Job()
        job["input_files"] = [{"lfn": "bogusFile", "parents": []}]
        job["mask"] = Mask()

        tweak = PSetTweak()
        WMTweaks.makeJobTweak(job, tweak)

        self.assertFalse(hasattr(tweak.process.source, "firstRun"),
                         "Error: There should be no firstRun tweak.")

        job["mask"]["FirstRun"] = 93
        tweak = WMTweaks.makeJobTweak(job, tweak)

        self.assertTrue(hasattr(tweak.process.source, "firstRun"),
                        "Error: There should be a firstRun tweak.")
        self.assertEqual(tweak.process.source.firstRun,
                         'customTypeCms.untracked.uint32(93)',
                         "Error: The firstRun tweak should be 93.")
        return
Пример #9
0
    def testFirstEventMC(self):
        """
        _testFirstEventMC_

        Verify that we set the firstEvent parameter whenever the FirstEvent
        field in the job mask is a positive integer and the job is a production
        one. Otherwise we get a number based on the counter (modulo 2^32 - 1)
        """
        job = Job()
        job["input_files"] = [{"lfn": "MCFakeFile", "parents": []}]
        job["mask"] = Mask()
        job["counter"] = 5
        job["mask"]["FirstLumi"] = 200

        try:
            tweak = PSetTweak()
            WMTweaks.makeJobTweak(job, tweak)
            self.assertRaises(WMTweakMaskError, WMTweaks.makeJobTweak, job)
        except WMTweakMaskError:
            pass

        job["mask"]["FirstEvent"] = 100
        tweak = PSetTweak()
        WMTweaks.makeJobTweak(job, tweak)
        self.assertFalse(
            hasattr(tweak.process.source, "skipEvents"),
            "Error: There should be no skipEvents tweak, it's MC.")
        self.assertTrue(hasattr(tweak.process.source, "firstEvent"),
                        "Error: There should be a first event tweak")
        self.assertEqual(tweak.process.source.firstEvent,
                         'customTypeCms.untracked.uint32(100)',
                         "Error: The firstEvent tweak should be 100.")
        return
Пример #10
0
 def testMask3(self):
     mymask = Mask()
     mymask['FirstEvent'] = 9999
     mymask['LastEvent'] = 999
     myjob = Job()
     myjob["mask"] = mymask
     self.roundTrip(myjob)
Пример #11
0
    def testAddCommit(self):
        """
        _testAddCommit_

        Test the add() and commit() methods of the JobGroup class.  Verify that
        jobs are not returned from getJobs() until commit() has been called.
        """
        testJob = Job()
        testJobGroup = JobGroup()

        assert len(testJobGroup.getJobs()) == 0, \
            "ERROR: JobGroup has jobs before jobs have been added."

        testJobGroup.add(testJob)

        assert len(testJobGroup.getJobs()) == 0, \
            "ERROR: JobGroup has jobs commit() was called."

        testJobGroup.commit()

        assert len(testJobGroup.getJobs()) == 1, \
            "ERROR: JobGroup has wrong number of jobs."
        assert testJob in testJobGroup.getJobs(), \
            "ERROR: JobGroup has unknown jobs."

        return
Пример #12
0
    def testFirstRunMC(self):
        """
        _testFirstRunMC_
        Verify that we set the lumi in a MC job and it gets into
        process.source.firstRun parameter.
        """
        job = Job()
        job["input_files"] = [{"lfn": "MCFakeFile", "parents": []}]
        job["mask"] = Mask()
        job["mask"]["FirstLumi"] = 200
        job["mask"]["FirstEvent"] = 100
        job["counter"] = 5

        tweak = PSetTweak()
        WMTweaks.makeJobTweak(job, tweak)

        self.assertTrue(hasattr(tweak.process.source, "firstRun"),
                        "Error: There should be a first run tweak")
        self.assertEqual(tweak.process.source.firstRun,
                         'customTypeCms.untracked.uint32(1)',
                         "Error: The first run should be 1")

        job["mask"]["FirstRun"] = 5
        tweak = PSetTweak()
        WMTweaks.makeJobTweak(job, tweak)

        self.assertTrue(hasattr(tweak.process.source, "firstRun"),
                        "Error: There should be a first run tweak")
        self.assertEqual(tweak.process.source.firstRun,
                         'customTypeCms.untracked.uint32(5)',
                         "Error: The first run should be 5")
Пример #13
0
    def setUp(self):
        # stolen from CMSSWExecutor_t. thanks, dave
        self.testInit = TestInit(__file__)
        self.testDir = self.testInit.generateWorkDir()
        shutil.copyfile('/etc/hosts', os.path.join(self.testDir, 'testfile'))

        self.workload = newWorkload("UnitTests")
        self.task = self.workload.newTask("DeleterTask")
        stepHelper = step = self.task.makeStep("DeleteTest")
        self.step = stepHelper.data
        self.actualStep = stepHelper
        template = DeleteTemplate()
        template(self.step)
        self.helper = template.helper(self.step)
        self.executor = StepFactory.getStepExecutor(self.actualStep.stepType())

        taskMaker = TaskMaker(self.workload, self.testDir)
        taskMaker.skipSubscription = True
        taskMaker.processWorkload()

        self.sandboxDir = "%s/UnitTests" % self.testDir

        self.task.build(self.testDir)
        sys.path.insert(0, self.testDir)
        sys.path.insert(0, self.sandboxDir)


        self.job = Job(name = "/UnitTest/DeleterTask/DeleteTest-test-job")

        binDir = inspect.getsourcefile(ModuleLocator)
        binDir = binDir.replace("__init__.py", "bin")

        if not binDir in os.environ['PATH']:
            os.environ['PATH'] = "%s:%s" % (os.environ['PATH'], binDir)
Пример #14
0
    def testAddingJobs(self):
        """
        _testAddingJobs_

        Verify that adding jobs to the package works as expected.
        """
        package = JobPackage()

        for i in range(100):
            newJob = Job("Job%s" % i)
            newJob["id"] = i
            package[i] = newJob

        # There is an extra key for the directory the package is stored in.
        assert len(package.keys()) == 101, \
               "Error: Wrong number of jobs in package."

        for i in range(100):
            job = package[i]
            assert job["id"] == i, \
                   "Error: Jobs has wrong ID."
            assert job["name"] == "Job%d" % i, \
                   "Error: Job has wrong name."

        return
Пример #15
0
    def testPersist(self):
        """
        _testPersist_

        Verify that we're able to save and load the job package.
        """
        package = JobPackage()

        for i in range(100):
            newJob = Job("Job%s" % i)
            newJob["id"] = i
            package[i] = newJob

        package.save(self.persistFile)

        assert os.path.exists(self.persistFile), \
               "Error: Package file was never created."

        newPackage = JobPackage()
        newPackage.load(self.persistFile)

        # There is an extra key for the directory the package is stored in.
        assert len(newPackage.keys()) == 101, \
               "Error: Wrong number of jobs in package."

        for i in range(100):
            job = newPackage[i]
            assert job["id"] == i, \
                   "Error: Jobs has wrong ID."
            assert job["name"] == "Job%d" % i, \
                   "Error: Job has wrong name."

        return
Пример #16
0
    def testFirstLumiMC(self):
        """
        _testFirstLumiMC_
        Verify that we set the lumi in a MC job and it gets into
        process.source.firstRun parameter, and if we don't at least we
        get the counter there.
        """
        job = Job()
        job["input_files"] = [{"lfn": "MCFakeFile", "parents": []}]
        job["mask"] = Mask()
        job["counter"] = 5
        job["mask"]["FirstEvent"] = 100

        try:
            tweak = WMTweaks.makeJobTweak(job)
            self.assertRaises(WMTweakMaskError, WMTweaks.makeJobTweak, job)
        except WMTweakMaskError:
            pass

        job["mask"]["FirstLumi"] = 200
        tweak = WMTweaks.makeJobTweak(job)

        self.assertTrue(hasattr(tweak.process.source, "firstLuminosityBlock"),
                        "Error: There should be a first lumi tweak")
        self.assertEqual(tweak.process.source.firstLuminosityBlock, 200,
                         "Error: The first luminosity block should be 5")

        job["mask"]["FirstLumi"] = 10
        tweak = WMTweaks.makeJobTweak(job)

        self.assertTrue(hasattr(tweak.process.source, "firstLuminosityBlock"),
                        "Error: There should be a first lumi tweak")
        self.assertEqual(tweak.process.source.firstLuminosityBlock, 10,
                         "Error: The first luminosity block should be 10")
Пример #17
0
    def setUp(self):
        """
        _setUp_

        Build a testing environment similar to a WN
        """
        self.testInit = TestInit(__file__)
        self.testDir = self.testInit.generateWorkDir()

        # Build a workload/task/step with the basic required information
        self.workload = newWorkload("UnitTests")
        self.task = self.workload.newTask("CMSSWExecutor")
        stepHelper = self.task.makeStep("ExecutorTest")
        self.step = stepHelper.data
        template = CMSSWTemplate()
        template(self.step)
        self.helper = template.helper(self.step)
        self.step.application.setup.scramCommand = "scramulator.py"
        self.step.application.command.executable = "cmsRun.py"
        self.step.application.setup.scramProject = "CMSSW"
        self.step.application.setup.scramArch = "slc5_ia32_gcc434"
        self.step.application.setup.cmsswVersion = "CMSSW_X_Y_Z"
        self.step.application.setup.softwareEnvironment = "echo \"Software Setup...\";"
        self.step.output.jobReport = "FrameworkJobReport.xml"
        self.helper.addOutputModule("outputRECORECO", primaryDataset = "Bogus",
                                    processedDataset = "Test-Era-v1",
                                    dataTier = "DATA")
        self.helper.addOutputModule("outputALCARECORECO", primaryDataset = "Bogus",
                                    processedDataset = "Test-Era-v1",
                                    dataTier = "DATA")
        self.helper.setGlobalTag("Bogus")
        taskMaker = TaskMaker(self.workload, self.testDir)
        taskMaker.skipSubscription = True
        taskMaker.processWorkload()

        # Build the TaskSpace/StepSpace
        self.sandboxDir = os.path.join(self.testDir, "UnitTests")
        self.task.build(self.testDir)
        sys.path.append(self.testDir)
        sys.path.append(self.sandboxDir)

        # Copy the files that cmsRun would have generated in the step space
        open(os.path.join(self.step.builder.workingDir, "outputRECORECO.root"), "w").close()
        open(os.path.join(self.step.builder.workingDir, "outputALCARECORECO.root"), "w").close()
        shutil.copy(os.path.join(getTestBase(),
                                 "WMCore_t/FwkJobReport_t/CMSSWProcessingReport.xml"),
                    os.path.join(self.step.builder.workingDir, "FrameworkJobReport.xml"))

        # Create a job
        self.job = Job(name = "/UnitTest/CMSSWExecutor/ExecutorTest-test-job")
        self.job["id"] = 1

        # Set the PATH
        binDir = inspect.getsourcefile(ModuleLocator)
        binDir = binDir.replace("__init__.py", "bin")

        if not binDir in os.environ['PATH']:
            os.environ['PATH'] = "%s:%s" % (os.environ['PATH'], binDir)

        self.oldCwd = os.getcwd()
Пример #18
0
    def testCreate(self):
        """
        _testCreate_

        Test the JobGroup constructor and passing different job containers
        into it.
        """
        testSubscription = Subscription()
        testJobGroupA = JobGroup(subscription = testSubscription)

        assert testJobGroupA.subscription == testSubscription, \
            "ERROR: Failed to pass subscription in constructor"
        assert len(testJobGroupA.jobs) == 0 and len(testJobGroupA.newjobs) == 0, \
            "ERROR: JobGroup not empty on creation"

        testJobA = Job()
        testJobB = Job()

        testJobGroupB = JobGroup(jobs = [testJobA, testJobB])

        assert testJobGroupB.jobs == [], \
            "ERROR: Jobs committed to jobgroup too soon."

        jobGroupJobs = testJobGroupB.newjobs
        goldenJobs = [testJobA, testJobB]
        for job in jobGroupJobs:
            assert job in goldenJobs, \
                "ERROR: Extra job in job group"

            goldenJobs.remove(job)

        assert len(goldenJobs) == 0, \
            "ERROR: Job missing from job group"

        testJobGroupC = JobGroup(jobs = testJobA)

        assert testJobGroupC.jobs == [], \
            "ERROR: Jobs committed to jobgroup too soon."

        jobGroupJobs = testJobGroupC.newjobs

        assert len(jobGroupJobs) == 1, \
            "ERROR: Wrong number of jobs in jobgroup."
        assert testJobA in jobGroupJobs, \
            "ERROR: Wrong job in jobgroup."

        return
Пример #19
0
    def createTestJob(self):
        """
        Create a test job to pass to the DashboardInterface

        """

        job = Job(name = "ThisIsASillyName")

        testFileA = File(lfn = "/this/is/a/lfnA", size = 1024, events = 10)
        testFileA.addRun(Run(1, *[45]))
        testFileB = File(lfn = "/this/is/a/lfnB", size = 1024, events = 10)
        testFileB.addRun(Run(1, *[46]))

        job.addFile(testFileA)
        job.addFile(testFileB)

        job['id'] = 1

        return job
Пример #20
0
    def associateWorkUnits(self):
        """
        _associateWorkUnits_

        Add the WorkUnits that this job requires

        Returns: N/A
        """

        existsAction = self.daofactory(
            classname='WorkUnit.ExistsByTaskFileLumi')
        addAction = self.daofactory(classname='WorkUnit.Add')
        assocAction = self.daofactory(classname='Jobs.AddWorkUnits')

        files = WMJob.getFiles(self)
        jobMask = self['mask']

        workflow = self.getWorkflow()
        wfid = workflow['taskid']

        lumisInJob = 0
        for wmfile in files:
            fileMask = jobMask.filterRunLumisByMask(runs=wmfile['runs'])
            for runObj in fileMask:
                lumisInJob += len(runObj.lumis)

        for wmfile in files:
            fileid = wmfile['id']
            fileMask = jobMask.filterRunLumisByMask(runs=wmfile['runs'])
            for runObj in fileMask:
                run = runObj.run
                lumis = runObj.lumis
                for lumi in lumis:
                    if not existsAction.execute(
                            taskid=wfid,
                            fileid=fileid,
                            run_lumi=runObj,
                            conn=self.getDBConn(),
                            transaction=self.existingTransaction()):
                        addAction.execute(
                            taskid=wfid,
                            last_unit_count=lumisInJob,
                            fileid=fileid,
                            run=run,
                            lumi=lumi,
                            conn=self.getDBConn(),
                            transaction=self.existingTransaction())
                    assocAction.execute(jobid=self["id"],
                                        fileid=fileid,
                                        run=run,
                                        lumi=lumi,
                                        conn=self.getDBConn(),
                                        transaction=self.existingTransaction())
Пример #21
0
    def __init__(self, name = None, files = None, id = None):
        """
        ___init___

        jobgroup object is used to determine the workflow.
        inputFiles is a list of files that the job will process.
        """
        WMBSBase.__init__(self)
        WMJob.__init__(self, name = name, files = files)

        self["id"]           = id
        self["jobgroup"]     = None
        self["couch_record"] = None
        self["attachments"]  = {}
        self["cache_dir"]    = None
        self["sandbox"]      = None
        self['fwjr']         = None
        self["mask"]         = Mask()
        self['custom']       = {}  # For local add-ons that we want to send to JSON

        return
Пример #22
0
    def __init__(self, name=None, files=None, id=None):
        """
        ___init___

        jobgroup object is used to determine the workflow.
        inputFiles is a list of files that the job will process.
        """
        WMBSBase.__init__(self)
        WMJob.__init__(self, name=name, files=files)

        self["id"] = id
        self["jobgroup"] = None
        self["couch_record"] = None
        self["attachments"] = {}
        self["cache_dir"] = None
        self["sandbox"] = None
        self['fwjr'] = None
        self["mask"] = Mask()
        self['custom'] = {}  # For local add-ons that we want to send to JSON

        return
Пример #23
0
    def getFiles(self, type="list"):
        """
        _getFiles_

        Retrieve a list of files that are associated with the job.
        """
        if self["id"] == None:
            return WMJob.getFiles(self, type)

        existingTransaction = self.beginTransaction()
        idAction = self.daofactory(classname="Jobs.LoadFiles")
        fileIDs = idAction.execute(self["id"], conn=self.getDBConn(), transaction=self.existingTransaction())

        currentFileIDs = WMJob.getFiles(self, type="id")
        for fileID in fileIDs:
            if fileID["id"] not in currentFileIDs:
                self.loadData()
                break

        self.commitTransaction(existingTransaction)
        return WMJob.getFiles(self, type)
Пример #24
0
    def createTestJob(self, fwjr):
        """
        _createTestJob_

        Creates a minimal job to report
        """
        job = Job('finishedJob')
        job['retry_count'] = 1
        job['workflow'] = 'testWorkflow'
        job['fwjr'] = fwjr

        return job
Пример #25
0
    def testA(self):
        """instantiate"""

        document = Document()
        document[u'pset_tweak_details'] = {}
        document[u'pset_tweak_details'][u'process'] = {}
        document[u'pset_tweak_details'][u'process'][u'RandomNumberGeneratorService'] = {}
        document[u'pset_tweak_details'][u'process'][u'RandomNumberGeneratorService'][u'seed1'] = {}
        document[u'pset_tweak_details'][u'process'][u'RandomNumberGeneratorService'][u'seed2'] = {}
        document[u'pset_tweak_details'][u'process'][u'RandomNumberGeneratorService'][u'seed3'] = {}

        document = self.database.commitOne(document)[0]
        seeder = ReproducibleSeeding(CouchUrl = self.testInit.couchUrl,
                                     CouchDBName = self.testInit.couchDbName,
                                     ConfigCacheDoc = document[u'id'])

        job = Job("testjob")
        seeder(job)
        baggage = job.getBaggage()
        seed1 = getattr(baggage.process.RandomNumberGeneratorService, "seed1", None)
        self.failUnless(seed1 != None)
Пример #26
0
    def getFiles(self, type = "list"):
        """
        _getFiles_

        Retrieve a list of files that are associated with the job.
        """
        if self["id"] == None:
            return WMJob.getFiles(self, type)

        existingTransaction = self.beginTransaction()
        idAction = self.daofactory(classname = "Jobs.LoadFiles")
        fileIDs = idAction.execute(self["id"], conn = self.getDBConn(),
                                   transaction = self.existingTransaction())

        currentFileIDs = WMJob.getFiles(self, type = "id")
        for fileID in fileIDs:
            if fileID["id"] not in currentFileIDs:
                self.loadData()
                break

        self.commitTransaction(existingTransaction)
        return WMJob.getFiles(self, type)
Пример #27
0
    def testC(self):
        """test building a tweak from the seeds"""
        job = Job("TestJob")
        seeder = AutomaticSeeding()

        job.addBaggageParameter(
            "process.RandomNumberGeneratorService.seed1.initialSeed", 123445)
        job.addBaggageParameter(
            "process.RandomNumberGeneratorService.seed2.initialSeed", 123445)
        job.addBaggageParameter(
            "process.RandomNumberGeneratorService.seed3.initialSeed", 7464738)
        job.addBaggageParameter(
            "process.RandomNumberGeneratorService.seed44.initialSeed",
            98273762)

        seeder(job)

        tweak = PSetTweak()
        for x in job.baggage.process.RandomNumberGeneratorService:
            parameter = "process.RandomNumberGeneratorService.%s.initialSeed" % x._internal_name
            tweak.addParameter(parameter, x.initialSeed)
        print tweak
Пример #28
0
    def testGetJobs(self):
        """
        _testGetJobs_

        Verify that the getJobs() method of the JobGroup class returns the
        correct output for each output container type it supports.
        """
        testJobA = Job()
        testJobB = Job()
        testJobGroup = JobGroup(jobs = [testJobA, testJobB])
        testJobGroup.commit()

        assert len(testJobGroup.getJobs()) == 2,  \
            "ERROR: Wrong number of jobs in job group"

        goldenJobs = [testJobA, testJobB]
        for job in testJobGroup.getJobs():
            assert job in goldenJobs, \
                "ERROR: Unknown Job in JobGroup."

            goldenJobs.remove(job)

        assert len(goldenJobs) == 0, \
            "ERROR: Jobs are missing from the JobGroup."

        goldenIDs = []
        goldenIDs.append(testJobA["id"])
        goldenIDs.append(testJobB["id"])
        for jobID in testJobGroup.getJobs(type = "id"):
            assert jobID in goldenIDs, \
                "ERROR: Unknown JobID in JobGroup"

            goldenIDs.remove(jobID)

        assert len(goldenIDs) == 0, \
            "ERROR: Job IDs are missing from the JobGroup."

        return
Пример #29
0
    def associateFiles(self):
        """
        _associateFiles_

        Update the wmbs_job_assoc table with the files in the inputFiles for the
        job.
        """
        files = WMJob.getFiles(self, type="id")

        if len(files) > 0:
            addAction = self.daofactory(classname="Jobs.AddFiles")
            addAction.execute(self["id"], files, conn=self.getDBConn(), transaction=self.existingTransaction())

        return
Пример #30
0
    def getDataStructsJob(self):
        """
        _getDataStructsJob_

        Returns the DataStructs version of this job
        """
        job = WMJob(name=self['name'])

        # Transfer all simple keys
        for key in self:
            if isinstance(self.get(key), (str, bytes, int, float)):
                job[key] = self[key]

        for fileObj in self['input_files']:
            job['input_files'].append(fileObj.returnDataStructsFile())

        job['mask'] = WMMask()
        for key in self["mask"]:
            job["mask"][key] = self["mask"][key]

        job.baggage = self.baggage

        return job
Пример #31
0
    def getDataStructsJob(self):
        """
        _getDataStructsJob_

        Returns the DataStructs version of this job
        """
        job = WMJob(name = self['name'])

        # Transfer all simple keys
        for key in self.keys():
            keyType = type(self.get(key))
            if keyType in [str, long, int, float]:
                job[key] = self[key]

        for file in self['input_files']:
            job['input_files'].append(file.returnDataStructsFile())

        job['mask'] = WMMask()
        for key in self["mask"].keys():
            job["mask"][key] = self["mask"][key]

        job.baggage = self.baggage

        return job
Пример #32
0
    def associateFiles(self):
        """
        _associateFiles_

        Update the wmbs_job_assoc table with the files in the inputFiles for the
        job.
        """
        files = WMJob.getFiles(self, type = "id")

        if len(files) > 0:
            addAction = self.daofactory(classname = "Jobs.AddFiles")
            addAction.execute(self["id"], files, conn = self.getDBConn(),
                              transaction = self.existingTransaction())

        return
Пример #33
0
    def getDataStructsJob(self):
        """
        _getDataStructsJob_

        Returns the DataStructs version of this job
        """
        job = WMJob(name=self['name'])

        # Transfer all simple keys
        for key in self.keys():
            keyType = type(self.get(key))
            if keyType in [str, long, int, float]:
                job[key] = self[key]

        for file in self['input_files']:
            job['input_files'].append(file.returnDataStructsFile())

        job['mask'] = WMMask()
        for key in self["mask"].keys():
            job["mask"][key] = self["mask"][key]

        job.baggage = self.baggage

        return job
Пример #34
0
    def createTestJob(self):
        """
        _createTestJob_

        Create a test job that has parents for each input file.

        """
        newJob = Job(name = "TestJob")
        newJob.addFile(File(lfn = "/some/file/one",
                            parents = set([File(lfn = "/some/parent/one")])))
        newJob.addFile(File(lfn = "/some/file/two",
                            parents = set([File(lfn = "/some/parent/two")])))
        return newJob
Пример #35
0
def createJobs(totalJobs=100):
    """
    _createJobs_

    """
    global lastJobID

    newJobs = []
    for i in range(totalJobs):
        newJob = Job(name=makeUUID(), files=[createFile(), createFile()])
        newJob["task"] = "SomeTask"
        newJob["workflow"] = "SomeWorkflow"
        newJob["owner"] = "*****@*****.**"
        newJob["id"] = lastJobID
        newJob["couch_record"] = None
        newJobs.append(newJob)
        lastJobID += 1

    return newJobs
Пример #36
0
    def testC(self):
        """test building a tweak from the seeds"""
        job = Job("TestJob")
        seeder = AutomaticSeeding()

        job.addBaggageParameter("process.RandomNumberGeneratorService.seed1.initialSeed", 123445)
        job.addBaggageParameter("process.RandomNumberGeneratorService.seed2.initialSeed", 123445)
        job.addBaggageParameter("process.RandomNumberGeneratorService.seed3.initialSeed", 7464738)
        job.addBaggageParameter("process.RandomNumberGeneratorService.seed44.initialSeed", 98273762)


        seeder(job)

        tweak = PSetTweak()
        for x in job.baggage.process.RandomNumberGeneratorService:
            parameter = "process.RandomNumberGeneratorService.%s.initialSeed" % x._internal_name
            tweak.addParameter(parameter, x.initialSeed)
        print(tweak)
Пример #37
0
    def associateWorkUnits(self):
        """
        _associateWorkUnits_

        Add the WorkUnits that this job requires

        Returns: N/A
        """

        existsAction = self.daofactory(classname='WorkUnit.ExistsByTaskFileLumi')
        addAction = self.daofactory(classname='WorkUnit.Add')
        assocAction = self.daofactory(classname='Jobs.AddWorkUnits')

        files = WMJob.getFiles(self)
        jobMask = self['mask']

        workflow = self.getWorkflow()
        wfid = workflow['taskid']

        lumisInJob = 0
        for wmfile in files:
            fileMask = jobMask.filterRunLumisByMask(runs=wmfile['runs'])
            for runObj in fileMask:
                lumisInJob += len(runObj.lumis)

        for wmfile in files:
            fileid = wmfile['id']
            fileMask = jobMask.filterRunLumisByMask(runs=wmfile['runs'])
            for runObj in fileMask:
                run = runObj.run
                lumis = runObj.lumis
                for lumi in lumis:
                    if not existsAction.execute(taskid=wfid, fileid=fileid, run_lumi=runObj,
                                                conn=self.getDBConn(), transaction=self.existingTransaction()):
                        addAction.execute(taskid=wfid, last_unit_count=lumisInJob, fileid=fileid, run=run,
                                          lumi=lumi,
                                          conn=self.getDBConn(), transaction=self.existingTransaction())
                    assocAction.execute(jobid=self["id"], fileid=fileid, run=run, lumi=lumi,
                                        conn=self.getDBConn(), transaction=self.existingTransaction())
Пример #38
0
    def setUp(self):
        """
        build a step for testing purposes

        """
        self.testInit = TestInit(__file__)
        self.testDir = self.testInit.generateWorkDir()

        self.workload = newWorkload("UnitTests")
        self.task = self.workload.newTask("CMSSWExecutor")
        stepHelper = step = self.task.makeStep("ExecutorTest")
        self.step = stepHelper.data
        template = CMSSWTemplate()
        template(self.step)
        self.helper = template.helper(self.step)
        self.step.application.setup.scramCommand = "scramulator.py"
        self.step.application.command.executable = "cmsRun.py"
        self.step.application.setup.scramProject = "CMSSW"
        self.step.application.setup.scramArch = "slc5_ia32_gcc434"
        self.step.application.setup.cmsswVersion = "CMSSW_X_Y_Z"
        self.step.application.setup.softwareEnvironment = "echo \"Software Setup...\";"

        taskMaker = TaskMaker(self.workload, self.testDir)
        taskMaker.skipSubscription = True
        taskMaker.processWorkload()

        self.sandboxDir = "%s/UnitTests" % self.testDir

        self.task.build(self.testDir)
        sys.path.append(self.testDir)
        sys.path.append(self.sandboxDir)

        self.job = Job(name="/UnitTest/CMSSWExecutor/ExecutorTest-test-job")

        binDir = inspect.getsourcefile(ModuleLocator)
        binDir = binDir.replace("__init__.py", "bin")

        if not binDir in os.environ['PATH']:
            os.environ['PATH'] = "%s:%s" % (os.environ['PATH'], binDir)
Пример #39
0
    def setUp(self):
        """
        _setUp_

        Initial Setup for the Job Testcase
        """
        self.inputFiles = []

        for i in range(1,1000):
            lfn = "/store/data/%s/%s/file.root" % (random.randint(1000, 9999),
                                                   random.randint(1000, 9999))
            size = random.randint(1000, 2000)
            events = 1000
            run = random.randint(0, 2000)
            lumi = random.randint(0, 8)

            file = File(lfn = lfn, size = size, events = events, checksums = {"cksum": "1"})
            file.addRun(Run(run, *[lumi]))
            self.inputFiles.append(file)

        self.dummyJob = Job(files = self.inputFiles)
        return
Пример #40
0
class JobTest(unittest.TestCase):
    """
    _JobTest_

    Testcase for the Job class

    Instantiate a dummy Job object with a dummy Subscription
    and a dummy Fileset full of random files as input
    """
    def setUp(self):
        """
        _setUp_

        Initial Setup for the Job Testcase
        """
        self.inputFiles = []

        for i in range(1, 1000):
            lfn = "/store/data/%s/%s/file.root" % (random.randint(
                1000, 9999), random.randint(1000, 9999))
            size = random.randint(1000, 2000)
            events = 1000
            run = random.randint(0, 2000)
            lumi = random.randint(0, 8)

            file = File(lfn=lfn,
                        size=size,
                        events=events,
                        checksums={"cksum": "1"})
            file.addRun(Run(run, *[lumi]))
            self.inputFiles.append(file)

        self.dummyJob = Job(files=self.inputFiles)
        return

    def tearDown(self):
        """
        No tearDown method for this Testcase
        """
        pass

    def testGetFilesList(self):
        """
        _testGetFilesList_

        Verify that the Job::getFiles(type = "list") method returns the same
        files in the same order that they were passed in.
        """
        assert self.dummyJob.getFiles() == self.inputFiles, \
            "ERROR: Initial fileset does not match Job fileset"

        return

    def testGetFilesSet(self):
        """
        _testGetFilesSet_

        Verify that the Job::getFiles(type = "set") method returns the correct
        input files in the form of a set.
        """
        assert self.dummyJob.getFiles(type = "set") == set(self.inputFiles), \
            "ERROR: getFiles(type = 'set') does not work correctly."

        return

    def testGetFilesLFN(self):
        """
        _testGetFilesLFN_

        Verify that the Job::getFiles(type = "lfn") method returns the same
        files in the same order that they were passed in.
        """
        jobLFNs = self.dummyJob.getFiles(type="lfn")

        goldenLFNs = []
        for file in self.inputFiles:
            goldenLFNs.append(file["lfn"])

        assert len(goldenLFNs) == len(jobLFNs), \
            "ERROR: Job has different number of files than input"

        for jobLFN in jobLFNs:
            assert jobLFN in goldenLFNs, \
                "ERROR: LFN missing from job."

        return

    def testAddFile(self):
        """
        _testAddFile_

        Verify that the Job::addFile() method works properly.
        """
        dummyFileAddFile = File("/tmp/dummyFileAddFileTest", 1234, 1, 2)
        self.dummyJob.addFile(dummyFileAddFile)

        assert dummyFileAddFile in self.dummyJob.getFiles(), \
            "ERROR: Couldn't add file to Job - addFile method error"

        return

    def testChangeState(self):
        """
        _testChangeState_

        Verify that the Job::changeState() method updates the state and the
        state time.
        """
        currentTime = time.time()
        self.dummyJob.changeState("created")

        assert self.dummyJob["state_time"] > currentTime - 1 and \
            self.dummyJob["state_time"] < currentTime + 1, \
            "ERROR: State time not updated on state change"

        assert self.dummyJob["state"] == "created", \
            "ERROR: Couldn't change Job state - changeState method error"

    def testChangeOutcome(self):
        """
        _testChangeOutcome_

        Verify that the Job::changeOutcome() method changes the final outcome
        of the job.
        """
        self.dummyJob.changeOutcome("success")

        assert self.dummyJob["outcome"] == "success", \
            "ERROR: Job outcome failed to update."

        return

    def testGetBaggage(self):
        """
        test that setting/accessing the Job Baggage ConfigSection works
        """
        setattr(self.dummyJob.baggage, "baggageContents", {"key": "value"})

        try:
            baggage = self.dummyJob.getBaggage()
        except Exception, ex:
            msg = "Error calling Job.getBaggage()\n"
            msg += str(ex)
            self.fail(msg)

        self.failUnless(hasattr(baggage, "baggageContents"))
Пример #41
0
class JobTest(unittest.TestCase):
    """
    _JobTest_

    Testcase for the Job class

    Instantiate a dummy Job object with a dummy Subscription
    and a dummy Fileset full of random files as input
    """

    def setUp(self):
        """
        _setUp_

        Initial Setup for the Job Testcase
        """
        self.inputFiles = []

        for i in range(1,1000):
            lfn = "/store/data/%s/%s/file.root" % (random.randint(1000, 9999),
                                                   random.randint(1000, 9999))
            size = random.randint(1000, 2000)
            events = 1000
            run = random.randint(0, 2000)
            lumi = random.randint(0, 8)

            file = File(lfn = lfn, size = size, events = events, checksums = {"cksum": "1"})
            file.addRun(Run(run, *[lumi]))
            self.inputFiles.append(file)

        self.dummyJob = Job(files = self.inputFiles)
        return

    def tearDown(self):
        """
        No tearDown method for this Testcase
        """
        pass

    def testGetFilesList(self):
        """
        _testGetFilesList_

        Verify that the Job::getFiles(type = "list") method returns the same
        files in the same order that they were passed in.
        """
        assert self.dummyJob.getFiles() == self.inputFiles, \
            "ERROR: Initial fileset does not match Job fileset"

        return

    def testGetFilesSet(self):
        """
        _testGetFilesSet_

        Verify that the Job::getFiles(type = "set") method returns the correct
        input files in the form of a set.
        """
        assert self.dummyJob.getFiles(type = "set") == set(self.inputFiles), \
            "ERROR: getFiles(type = 'set') does not work correctly."

        return

    def testGetFilesLFN(self):
        """
        _testGetFilesLFN_

        Verify that the Job::getFiles(type = "lfn") method returns the same
        files in the same order that they were passed in.
        """
        jobLFNs = self.dummyJob.getFiles(type = "lfn")

        goldenLFNs = []
        for file in self.inputFiles:
            goldenLFNs.append(file["lfn"])

        assert len(goldenLFNs) == len(jobLFNs), \
            "ERROR: Job has different number of files than input"

        for jobLFN in jobLFNs:
            assert jobLFN in goldenLFNs, \
                "ERROR: LFN missing from job."

        return

    def testAddFile(self):
        """
        _testAddFile_

        Verify that the Job::addFile() method works properly.
        """
        dummyFileAddFile = File("/tmp/dummyFileAddFileTest", 1234, 1, 2)
        self.dummyJob.addFile(dummyFileAddFile)

        assert dummyFileAddFile in self.dummyJob.getFiles(), \
            "ERROR: Couldn't add file to Job - addFile method error"

        return

    def testChangeState(self):
        """
        _testChangeState_

        Verify that the Job::changeState() method updates the state and the
        state time.
        """
        currentTime = time.time()
        self.dummyJob.changeState("created")

        assert self.dummyJob["state_time"] > currentTime - 1 and \
            self.dummyJob["state_time"] < currentTime + 1, \
            "ERROR: State time not updated on state change"

        assert self.dummyJob["state"] == "created", \
            "ERROR: Couldn't change Job state - changeState method error"

    def testChangeOutcome(self):
        """
        _testChangeOutcome_

        Verify that the Job::changeOutcome() method changes the final outcome
        of the job.
        """
        self.dummyJob.changeOutcome("success")

        assert self.dummyJob["outcome"] == "success", \
            "ERROR: Job outcome failed to update."

        return

    def testGetBaggage(self):
        """
        test that setting/accessing the Job Baggage ConfigSection works
        """
        self.dummyJob.addBaggageParameter("baggageContents", True)
        self.dummyJob.addBaggageParameter("trustPUSitelists", False)
        self.dummyJob.addBaggageParameter("skipPileupEvents", 20000)

        baggage = self.dummyJob.getBaggage()
        self.assertTrue(getattr(baggage, "baggageContents"))
        self.assertFalse(getattr(baggage, "trustPUSitelists"))
        self.assertEqual(getattr(baggage, "skipPileupEvents"), 20000)
        self.assertFalse(hasattr(baggage, "IDontExist"))