Beispiel #1
0
    def testProperties(self):
        """
        _testProperties_

        Test data fields for the properties information for DBS
        """

        myReport = Report("cmsRun1")
        myReport.parse(self.xmlPath)

        name = "ThisIsASillyString"

        myReport.setValidStatus(name)
        myReport.setGlobalTag(name)
        myReport.setAcquisitionProcessing(acquisitionEra = 'NULL', processingVer = name)
        myReport.setInputDataset(inputPath = '/lame/path')

        for f in myReport.getAllFilesFromStep("cmsRun1"):
            self.assertEqual(f['globalTag'], name)
            self.assertEqual(f['validStatus'], name)
            self.assertEqual(f['processingVer'], name)
            self.assertEqual(f['acquisitionEra'], 'NULL')
            self.assertEqual(f['inputPath'], '/lame/path')

        return
Beispiel #2
0
    def testOutputFiles(self):
        """
        _testOutputFiles_

        Test some basic manipulation of output files
        """

        myReport = Report("cmsRun1")
        myReport.parse(self.xmlPath)

        files = myReport.getAllFilesFromStep(step = "cmsRun1")

        f1 = files[0]
        f2 = files[1]

        self.assertEqual(f1['outputModule'], 'outputRECORECO')
        self.assertEqual(f1['pfn'], 'outputRECORECO.root')

        self.assertEqual(f2['outputModule'], 'outputALCARECORECO')
        self.assertEqual(f2['pfn'], 'outputALCARECORECO.root')

        for f in files:
            self.assertEqual(f['events'], 2)
            self.assertEqual(f['configURL'], None)
            self.assertEqual(f['merged'], False)
            self.assertEqual(f['validStatus'], None)
            self.assertEqual(f['first_event'], 0)

        return
Beispiel #3
0
    def testOutputFiles(self):
        """
        _testOutputFiles_

        Test some basic manipulation of output files
        """

        myReport = Report("cmsRun1")
        myReport.parse(self.xmlPath)

        files = myReport.getAllFilesFromStep(step="cmsRun1")

        f1 = files[0]
        f2 = files[1]

        self.assertEqual(f1['outputModule'], 'outputRECORECO')
        self.assertEqual(f1['pfn'], 'outputRECORECO.root')

        self.assertEqual(f2['outputModule'], 'outputALCARECORECO')
        self.assertEqual(f2['pfn'], 'outputALCARECORECO.root')

        for f in files:
            self.assertEqual(f['events'], 2)
            self.assertEqual(f['configURL'], None)
            self.assertEqual(f['merged'], False)
            self.assertEqual(f['validStatus'], None)
            self.assertEqual(f['first_event'], 0)

        return
Beispiel #4
0
    def testProperties(self):
        """
        _testProperties_

        Test data fields for the properties information for DBS
        """

        myReport = Report("cmsRun1")
        myReport.parse(self.xmlPath)

        name = "ThisIsASillyString"

        myReport.setValidStatus(name)
        myReport.setGlobalTag(name)
        myReport.setAcquisitionProcessing(acquisitionEra='NULL',
                                          processingVer=name)
        myReport.setInputDataset(inputPath='/lame/path')

        for f in myReport.getAllFilesFromStep("cmsRun1"):
            self.assertEqual(f['globalTag'], name)
            self.assertEqual(f['validStatus'], name)
            self.assertEqual(f['processingVer'], name)
            self.assertEqual(f['acquisitionEra'], 'NULL')
            self.assertEqual(f['inputPath'], '/lame/path')

        return
Beispiel #5
0
    def testOutputFiles(self):
        """
        _testOutputFiles_
        
        Test some basic manipulation of output files
        """

        myReport = Report("cmsRun1")
        myReport.parse(self.xmlPath)

        files = myReport.getAllFilesFromStep(step="cmsRun1")

        f1 = files[0]
        f2 = files[1]

        self.assertEqual(f1["outputModule"], "outputRECORECO")
        self.assertEqual(f1["pfn"], "outputRECORECO.root")

        self.assertEqual(f2["outputModule"], "outputALCARECORECO")
        self.assertEqual(f2["pfn"], "outputALCARECORECO.root")

        for f in files:
            self.assertEqual(f["events"], 2)
            self.assertEqual(f["configURL"], None)
            self.assertEqual(f["merged"], False)
            self.assertEqual(f["validStatus"], None)
            self.assertEqual(f["first_event"], 0)

        return
Beispiel #6
0
    def testProperties(self):
        """
        _testProperties_
        
        Test data fields for the properties information for DBS
        """

        myReport = Report("cmsRun1")
        myReport.parse(self.xmlPath)

        name = "ThisIsASillyString"

        myReport.setValidStatus(name)
        myReport.setGlobalTag(name)
        myReport.setAcquisitionProcessing(acquisitionEra="NULL", processingVer=name)
        myReport.setInputDataset(inputPath="/lame/path")

        for f in myReport.getAllFilesFromStep("cmsRun1"):
            self.assertEqual(f["globalTag"], name)
            self.assertEqual(f["validStatus"], name)
            self.assertEqual(f["processingVer"], name)
            self.assertEqual(f["acquisitionEra"], "NULL")
            self.assertEqual(f["inputPath"], "/lame/path")

        return
    def testReportHandling(self):
        """
        _testReportHandling_

        Verify that we're able to parse a CMSSW report, convert it to a Report()
        style report, pickle it and then have the accountant process it.
        """
        self.procPath = os.path.join(WMCore.WMBase.getTestBase(),
                                    "WMCore_t/FwkJobReport_t/CMSSWProcessingReport.xml")

        myReport = Report("cmsRun1")
        myReport.parse(self.procPath)

        # Fake some metadata that should be added by the stageout scripts.
        for fileRef in myReport.getAllFileRefsFromStep("cmsRun1"):
            fileRef.size = 1024
            fileRef.location = "cmssrm.fnal.gov"

        fwjrPath = os.path.join(self.tempDir, "ProcReport.pkl")
        cmsRunStep = myReport.retrieveStep("cmsRun1")
        cmsRunStep.status = 0
        myReport.setTaskName('/TestWF/None')
        myReport.persist(fwjrPath)

        self.setFWJRAction.execute(jobID = self.testJob["id"], fwjrPath = fwjrPath)

        pFile = DBSBufferFile(lfn = "/path/to/some/lfn", size = 600000, events = 60000)
        pFile.setAlgorithm(appName = "cmsRun", appVer = "UNKNOWN",
                           appFam = "RECO", psetHash = "GIBBERISH",
                           configContent = "MOREGIBBERISH")
        pFile.setDatasetPath("/bogus/dataset/path")
        #pFile.addRun(Run(1, *[45]))
        pFile.create()

        config = self.createConfig(workerThreads = 1)
        accountant = JobAccountantPoller(config)
        accountant.setup()
        accountant.algorithm()

        self.verifyJobSuccess(self.testJob["id"])
        self.verifyFileMetaData(self.testJob["id"], myReport.getAllFilesFromStep("cmsRun1"))

        inputFile = File(lfn = "/store/backfill/2/unmerged/WMAgentCommissioining10/MinimumBias/RECO/rereco_GR09_R_34X_V5_All_v1/0000/outputRECORECO.root")
        inputFile.load()
        self.testMergeJob = Job(name = "testMergeJob", files = [inputFile])
        self.testMergeJob.create(group = self.mergeJobGroup)
        self.testMergeJob["state"] = "complete"
        self.stateChangeAction.execute(jobs = [self.testMergeJob])

        self.mergePath = os.path.join(WMCore.WMBase.getTestBase(),
                                         "WMCore_t/FwkJobReport_t/CMSSWMergeReport.xml")

        myReport = Report("mergeReco")
        myReport.parse(self.mergePath)

        # Fake some metadata that should be added by the stageout scripts.
        for fileRef in myReport.getAllFileRefsFromStep("mergeReco"):
            fileRef.size = 1024
            fileRef.location = "cmssrm.fnal.gov"
            fileRef.dataset = {"applicationName": "cmsRun", "applicationVersion": "CMSSW_3_4_2_patch1",
                               "primaryDataset": "MinimumBias", "processedDataset": "Rereco-v1",
                               "dataTier": "RECO"}

        fwjrPath = os.path.join(self.tempDir, "MergeReport.pkl")
        myReport.setTaskName('/MergeWF/None')
        cmsRunStep = myReport.retrieveStep("mergeReco")
        cmsRunStep.status = 0
        myReport.persist(fwjrPath)

        self.setFWJRAction.execute(jobID = self.testMergeJob["id"], fwjrPath = fwjrPath)
        accountant.algorithm()

        self.verifyJobSuccess(self.testMergeJob["id"])
        self.verifyFileMetaData(self.testMergeJob["id"], myReport.getAllFilesFromStep("mergeReco"))

        return
Beispiel #8
0
    def testReportHandling(self):
        """
        _testReportHandling_

        Verify that we're able to parse a CMSSW report, convert it to a Report()
        style report, pickle it and then have the accountant process it.
        """
        self.procPath = os.path.join(WMCore.WMBase.getTestBase(),
                                    "WMCore_t/FwkJobReport_t/CMSSWProcessingReport.xml")

        myReport = Report("cmsRun1")
        myReport.parse(self.procPath)

        # Fake some metadata that should be added by the stageout scripts.
        for fileRef in myReport.getAllFileRefsFromStep("cmsRun1"):
            fileRef.size = 1024
            fileRef.location = "cmssrm.fnal.gov"

        fwjrPath = os.path.join(self.tempDir, "ProcReport.pkl")
        cmsRunStep = myReport.retrieveStep("cmsRun1")
        cmsRunStep.status = 0
        myReport.setTaskName('/TestWF/None')
        myReport.persist(fwjrPath)

        self.setFWJRAction.execute(jobID = self.testJob["id"], fwjrPath = fwjrPath)

        pFile = DBSBufferFile(lfn = "/path/to/some/lfn", size = 600000, events = 60000)
        pFile.setAlgorithm(appName = "cmsRun", appVer = "UNKNOWN",
                           appFam = "RECO", psetHash = "GIBBERISH",
                           configContent = "MOREGIBBERISH")
        pFile.setDatasetPath("/bogus/dataset/path")
        #pFile.addRun(Run(1, *[45]))
        pFile.create()

        config = self.createConfig(workerThreads = 1)
        accountant = JobAccountantPoller(config)
        accountant.setup()
        accountant.algorithm()

        self.verifyJobSuccess(self.testJob["id"])
        self.verifyFileMetaData(self.testJob["id"], myReport.getAllFilesFromStep("cmsRun1"))

        inputFile = File(lfn = "/store/backfill/2/unmerged/WMAgentCommissioining10/MinimumBias/RECO/rereco_GR09_R_34X_V5_All_v1/0000/outputRECORECO.root")
        inputFile.load()
        self.testMergeJob = Job(name = "testMergeJob", files = [inputFile])
        self.testMergeJob.create(group = self.mergeJobGroup)
        self.testMergeJob["state"] = "complete"
        self.stateChangeAction.execute(jobs = [self.testMergeJob])

        self.mergePath = os.path.join(WMCore.WMBase.getTestBase(),
                                         "WMCore_t/FwkJobReport_t/CMSSWMergeReport.xml")

        myReport = Report("mergeReco")
        myReport.parse(self.mergePath)

        # Fake some metadata that should be added by the stageout scripts.
        for fileRef in myReport.getAllFileRefsFromStep("mergeReco"):
            fileRef.size = 1024
            fileRef.location = "cmssrm.fnal.gov"
            fileRef.dataset = {"applicationName": "cmsRun", "applicationVersion": "CMSSW_3_4_2_patch1",
                               "primaryDataset": "MinimumBias", "processedDataset": "Rereco-v1",
                               "dataTier": "RECO"}

        fwjrPath = os.path.join(self.tempDir, "MergeReport.pkl")
        myReport.setTaskName('/MergeWF/None')
        cmsRunStep = myReport.retrieveStep("mergeReco")
        cmsRunStep.status = 0
        myReport.persist(fwjrPath)

        self.setFWJRAction.execute(jobID = self.testMergeJob["id"], fwjrPath = fwjrPath)
        accountant.algorithm()

        self.verifyJobSuccess(self.testMergeJob["id"])
        self.verifyFileMetaData(self.testMergeJob["id"], myReport.getAllFilesFromStep("mergeReco"))

        return
def main():
    parser = OptionParser()
    parser.add_option("--archive", dest="archive", help="Archive to harvest")
    parser.add_option("--check-merged", dest="checkMerged", help="Tell the tool to harvest information about merge jobs",
                      default = False)
    (options, _) = parser.parse_args()
    
    pathPattern = os.path.join(options.archive, '*/*')
    filelist = glob.glob(pathPattern)   
    #print filelist
    
    totalEvents = 0
    totalInEvents = 0
    inFiles = set([])
    outFiles = set([])
    failedFWJRs = []
    totalProdJobs = 0
    noSites = 0
    missingFiles = 0
    for jobTarPath in filelist:
        tmpdir = tempfile.mkdtemp()

        tarFile = tarfile.open(jobTarPath)
        tarFile.extractall(path = tmpdir)
        tarFile.close()

        jobTar = jobTarPath.split('/')[-1]
        path = os.path.join(tmpdir, jobTar.split('.')[0])
        availableFiles = os.listdir(path)
        reports = filter(lambda x : re.match('Report\.[0-9]\.pkl', x), availableFiles)
        if reports:
            reports.sort(reverse = True)
            reportName = reports[0]
            fwjr = Report()
            fwjr.load(os.path.join(path, reportName))
            steps = fwjr.listSteps()
#            if fwjr.getTaskName() != '/spinoso_EXO-Summer12-01945_R1805_B169_20_LHE_120914_191121_2116/MonteCarloFromGEN':
#                continue
            totalProdJobs += 1
            if fwjr.getExitCode():
                failedFWJRs.append(fwjr)
                #print fwjr.data
#            print fwjr.getTaskName()
            if 'cmsRun1' in steps:
                outputFiles = fwjr.getAllFilesFromStep(step = 'cmsRun1')
                inputFiles = fwjr.getInputFilesFromStep(stepName = 'cmsRun1')
                inEvents = 0
                for inFile in inputFiles:
                    if inFile['input_type'] == 'primaryFiles':
                        inEvents += inFile['events']
                        totalInEvents += inFile['events']
                        inFiles.add(inFile['lfn'])
    
                for outFile in outputFiles:
    #                outEvents += outFile['events']
#                    if outFile['lfn'].count('/store/mc/Summer12/'):
                    totalEvents += outFile['events']
                    outFiles.add(outputFiles[0]['lfn'])
                    if outFile['lfn'] in ['/store/generator/Summer12/Wbb_FullyHadronic_8TeV_madgraph/GEN/START53_V7C-v1/00000/12DA0CF7-E743-E211-BA20-002590574544.root',
                                          '/store/generator/Summer12/Wbb_FullyHadronic_8TeV_madgraph/GEN/START53_V7C-v1/00000/E0F13F75-E743-E211-8B7E-00259056F28A.root']:
                        loc = fwjr.data.cmsRun1.output.LHEoutput.files.file0.location
                        lfn = fwjr.data.cmsRun1.output.LHEoutput.files.file0.lfn
                        sql = """INSERT INTO dbsbuffer_file_location (filename, location)
               SELECT df.id, dl.id
               FROM dbsbuffer_file df
               INNER JOIN dbsbuffer_location dl
               WHERE df.lfn = '%s'
               AND dl.se_name = '%s';
               """ % (lfn, loc)
                        print sql
    #                if inEvents != outEvents:
    #                    print 'GOTCHA!'    

        os.system('rm -r %s' % tmpdir)