def testProperties(self):
"""
_testProperties_
Test data fields for the properties information for DBS
"""
myReport = Report("cmsRun1")
myReport.parse(self.xmlPath)
name = "ThisIsASillyString"
myReport.setValidStatus(name)
myReport.setGlobalTag(name)
myReport.setAcquisitionProcessing(acquisitionEra = 'NULL', processingVer = name)
myReport.setInputDataset(inputPath = '/lame/path')
for f in myReport.getAllFilesFromStep("cmsRun1"):
self.assertEqual(f['globalTag'], name)
self.assertEqual(f['validStatus'], name)
self.assertEqual(f['processingVer'], name)
self.assertEqual(f['acquisitionEra'], 'NULL')
self.assertEqual(f['inputPath'], '/lame/path')
return
def testOutputFiles(self):
"""
_testOutputFiles_
Test some basic manipulation of output files
"""
myReport = Report("cmsRun1")
myReport.parse(self.xmlPath)
files = myReport.getAllFilesFromStep(step = "cmsRun1")
f1 = files[0]
f2 = files[1]
self.assertEqual(f1['outputModule'], 'outputRECORECO')
self.assertEqual(f1['pfn'], 'outputRECORECO.root')
self.assertEqual(f2['outputModule'], 'outputALCARECORECO')
self.assertEqual(f2['pfn'], 'outputALCARECORECO.root')
for f in files:
self.assertEqual(f['events'], 2)
self.assertEqual(f['configURL'], None)
self.assertEqual(f['merged'], False)
self.assertEqual(f['validStatus'], None)
self.assertEqual(f['first_event'], 0)
return
def testOutputFiles(self):
"""
_testOutputFiles_
Test some basic manipulation of output files
"""
myReport = Report("cmsRun1")
myReport.parse(self.xmlPath)
files = myReport.getAllFilesFromStep(step="cmsRun1")
f1 = files[0]
f2 = files[1]
self.assertEqual(f1['outputModule'], 'outputRECORECO')
self.assertEqual(f1['pfn'], 'outputRECORECO.root')
self.assertEqual(f2['outputModule'], 'outputALCARECORECO')
self.assertEqual(f2['pfn'], 'outputALCARECORECO.root')
for f in files:
self.assertEqual(f['events'], 2)
self.assertEqual(f['configURL'], None)
self.assertEqual(f['merged'], False)
self.assertEqual(f['validStatus'], None)
self.assertEqual(f['first_event'], 0)
return
def testProperties(self):
"""
_testProperties_
Test data fields for the properties information for DBS
"""
myReport = Report("cmsRun1")
myReport.parse(self.xmlPath)
name = "ThisIsASillyString"
myReport.setValidStatus(name)
myReport.setGlobalTag(name)
myReport.setAcquisitionProcessing(acquisitionEra='NULL',
processingVer=name)
myReport.setInputDataset(inputPath='/lame/path')
for f in myReport.getAllFilesFromStep("cmsRun1"):
self.assertEqual(f['globalTag'], name)
self.assertEqual(f['validStatus'], name)
self.assertEqual(f['processingVer'], name)
self.assertEqual(f['acquisitionEra'], 'NULL')
self.assertEqual(f['inputPath'], '/lame/path')
return
def testOutputFiles(self):
"""
_testOutputFiles_
Test some basic manipulation of output files
"""
myReport = Report("cmsRun1")
myReport.parse(self.xmlPath)
files = myReport.getAllFilesFromStep(step="cmsRun1")
f1 = files[0]
f2 = files[1]
self.assertEqual(f1["outputModule"], "outputRECORECO")
self.assertEqual(f1["pfn"], "outputRECORECO.root")
self.assertEqual(f2["outputModule"], "outputALCARECORECO")
self.assertEqual(f2["pfn"], "outputALCARECORECO.root")
for f in files:
self.assertEqual(f["events"], 2)
self.assertEqual(f["configURL"], None)
self.assertEqual(f["merged"], False)
self.assertEqual(f["validStatus"], None)
self.assertEqual(f["first_event"], 0)
return
def testProperties(self):
"""
_testProperties_
Test data fields for the properties information for DBS
"""
myReport = Report("cmsRun1")
myReport.parse(self.xmlPath)
name = "ThisIsASillyString"
myReport.setValidStatus(name)
myReport.setGlobalTag(name)
myReport.setAcquisitionProcessing(acquisitionEra="NULL", processingVer=name)
myReport.setInputDataset(inputPath="/lame/path")
for f in myReport.getAllFilesFromStep("cmsRun1"):
self.assertEqual(f["globalTag"], name)
self.assertEqual(f["validStatus"], name)
self.assertEqual(f["processingVer"], name)
self.assertEqual(f["acquisitionEra"], "NULL")
self.assertEqual(f["inputPath"], "/lame/path")
return
def testReportHandling(self):
"""
_testReportHandling_
Verify that we're able to parse a CMSSW report, convert it to a Report()
style report, pickle it and then have the accountant process it.
"""
self.procPath = os.path.join(WMCore.WMBase.getTestBase(),
"WMCore_t/FwkJobReport_t/CMSSWProcessingReport.xml")
myReport = Report("cmsRun1")
myReport.parse(self.procPath)
# Fake some metadata that should be added by the stageout scripts.
for fileRef in myReport.getAllFileRefsFromStep("cmsRun1"):
fileRef.size = 1024
fileRef.location = "cmssrm.fnal.gov"
fwjrPath = os.path.join(self.tempDir, "ProcReport.pkl")
cmsRunStep = myReport.retrieveStep("cmsRun1")
cmsRunStep.status = 0
myReport.setTaskName('/TestWF/None')
myReport.persist(fwjrPath)
self.setFWJRAction.execute(jobID = self.testJob["id"], fwjrPath = fwjrPath)
pFile = DBSBufferFile(lfn = "/path/to/some/lfn", size = 600000, events = 60000)
pFile.setAlgorithm(appName = "cmsRun", appVer = "UNKNOWN",
appFam = "RECO", psetHash = "GIBBERISH",
configContent = "MOREGIBBERISH")
pFile.setDatasetPath("/bogus/dataset/path")
#pFile.addRun(Run(1, *[45]))
pFile.create()
config = self.createConfig(workerThreads = 1)
accountant = JobAccountantPoller(config)
accountant.setup()
accountant.algorithm()
self.verifyJobSuccess(self.testJob["id"])
self.verifyFileMetaData(self.testJob["id"], myReport.getAllFilesFromStep("cmsRun1"))
inputFile = File(lfn = "/store/backfill/2/unmerged/WMAgentCommissioining10/MinimumBias/RECO/rereco_GR09_R_34X_V5_All_v1/0000/outputRECORECO.root")
inputFile.load()
self.testMergeJob = Job(name = "testMergeJob", files = [inputFile])
self.testMergeJob.create(group = self.mergeJobGroup)
self.testMergeJob["state"] = "complete"
self.stateChangeAction.execute(jobs = [self.testMergeJob])
self.mergePath = os.path.join(WMCore.WMBase.getTestBase(),
"WMCore_t/FwkJobReport_t/CMSSWMergeReport.xml")
myReport = Report("mergeReco")
myReport.parse(self.mergePath)
# Fake some metadata that should be added by the stageout scripts.
for fileRef in myReport.getAllFileRefsFromStep("mergeReco"):
fileRef.size = 1024
fileRef.location = "cmssrm.fnal.gov"
fileRef.dataset = {"applicationName": "cmsRun", "applicationVersion": "CMSSW_3_4_2_patch1",
"primaryDataset": "MinimumBias", "processedDataset": "Rereco-v1",
"dataTier": "RECO"}
fwjrPath = os.path.join(self.tempDir, "MergeReport.pkl")
myReport.setTaskName('/MergeWF/None')
cmsRunStep = myReport.retrieveStep("mergeReco")
cmsRunStep.status = 0
myReport.persist(fwjrPath)
self.setFWJRAction.execute(jobID = self.testMergeJob["id"], fwjrPath = fwjrPath)
accountant.algorithm()
self.verifyJobSuccess(self.testMergeJob["id"])
self.verifyFileMetaData(self.testMergeJob["id"], myReport.getAllFilesFromStep("mergeReco"))
return
def testReportHandling(self):
"""
_testReportHandling_
Verify that we're able to parse a CMSSW report, convert it to a Report()
style report, pickle it and then have the accountant process it.
"""
self.procPath = os.path.join(WMCore.WMBase.getTestBase(),
"WMCore_t/FwkJobReport_t/CMSSWProcessingReport.xml")
myReport = Report("cmsRun1")
myReport.parse(self.procPath)
# Fake some metadata that should be added by the stageout scripts.
for fileRef in myReport.getAllFileRefsFromStep("cmsRun1"):
fileRef.size = 1024
fileRef.location = "cmssrm.fnal.gov"
fwjrPath = os.path.join(self.tempDir, "ProcReport.pkl")
cmsRunStep = myReport.retrieveStep("cmsRun1")
cmsRunStep.status = 0
myReport.setTaskName('/TestWF/None')
myReport.persist(fwjrPath)
self.setFWJRAction.execute(jobID = self.testJob["id"], fwjrPath = fwjrPath)
pFile = DBSBufferFile(lfn = "/path/to/some/lfn", size = 600000, events = 60000)
pFile.setAlgorithm(appName = "cmsRun", appVer = "UNKNOWN",
appFam = "RECO", psetHash = "GIBBERISH",
configContent = "MOREGIBBERISH")
pFile.setDatasetPath("/bogus/dataset/path")
#pFile.addRun(Run(1, *[45]))
pFile.create()
config = self.createConfig(workerThreads = 1)
accountant = JobAccountantPoller(config)
accountant.setup()
accountant.algorithm()
self.verifyJobSuccess(self.testJob["id"])
self.verifyFileMetaData(self.testJob["id"], myReport.getAllFilesFromStep("cmsRun1"))
inputFile = File(lfn = "/store/backfill/2/unmerged/WMAgentCommissioining10/MinimumBias/RECO/rereco_GR09_R_34X_V5_All_v1/0000/outputRECORECO.root")
inputFile.load()
self.testMergeJob = Job(name = "testMergeJob", files = [inputFile])
self.testMergeJob.create(group = self.mergeJobGroup)
self.testMergeJob["state"] = "complete"
self.stateChangeAction.execute(jobs = [self.testMergeJob])
self.mergePath = os.path.join(WMCore.WMBase.getTestBase(),
"WMCore_t/FwkJobReport_t/CMSSWMergeReport.xml")
myReport = Report("mergeReco")
myReport.parse(self.mergePath)
# Fake some metadata that should be added by the stageout scripts.
for fileRef in myReport.getAllFileRefsFromStep("mergeReco"):
fileRef.size = 1024
fileRef.location = "cmssrm.fnal.gov"
fileRef.dataset = {"applicationName": "cmsRun", "applicationVersion": "CMSSW_3_4_2_patch1",
"primaryDataset": "MinimumBias", "processedDataset": "Rereco-v1",
"dataTier": "RECO"}
fwjrPath = os.path.join(self.tempDir, "MergeReport.pkl")
myReport.setTaskName('/MergeWF/None')
cmsRunStep = myReport.retrieveStep("mergeReco")
cmsRunStep.status = 0
myReport.persist(fwjrPath)
self.setFWJRAction.execute(jobID = self.testMergeJob["id"], fwjrPath = fwjrPath)
accountant.algorithm()
self.verifyJobSuccess(self.testMergeJob["id"])
self.verifyFileMetaData(self.testMergeJob["id"], myReport.getAllFilesFromStep("mergeReco"))
return
def main():
parser = OptionParser()
parser.add_option("--archive", dest="archive", help="Archive to harvest")
parser.add_option("--check-merged", dest="checkMerged", help="Tell the tool to harvest information about merge jobs",
default = False)
(options, _) = parser.parse_args()
pathPattern = os.path.join(options.archive, '*/*')
filelist = glob.glob(pathPattern)
#print filelist
totalEvents = 0
totalInEvents = 0
inFiles = set([])
outFiles = set([])
failedFWJRs = []
totalProdJobs = 0
noSites = 0
missingFiles = 0
for jobTarPath in filelist:
tmpdir = tempfile.mkdtemp()
tarFile = tarfile.open(jobTarPath)
tarFile.extractall(path = tmpdir)
tarFile.close()
jobTar = jobTarPath.split('/')[-1]
path = os.path.join(tmpdir, jobTar.split('.')[0])
availableFiles = os.listdir(path)
reports = filter(lambda x : re.match('Report\.[0-9]\.pkl', x), availableFiles)
if reports:
reports.sort(reverse = True)
reportName = reports[0]
fwjr = Report()
fwjr.load(os.path.join(path, reportName))
steps = fwjr.listSteps()
# if fwjr.getTaskName() != '/spinoso_EXO-Summer12-01945_R1805_B169_20_LHE_120914_191121_2116/MonteCarloFromGEN':
# continue
totalProdJobs += 1
if fwjr.getExitCode():
failedFWJRs.append(fwjr)
#print fwjr.data
# print fwjr.getTaskName()
if 'cmsRun1' in steps:
outputFiles = fwjr.getAllFilesFromStep(step = 'cmsRun1')
inputFiles = fwjr.getInputFilesFromStep(stepName = 'cmsRun1')
inEvents = 0
for inFile in inputFiles:
if inFile['input_type'] == 'primaryFiles':
inEvents += inFile['events']
totalInEvents += inFile['events']
inFiles.add(inFile['lfn'])
for outFile in outputFiles:
# outEvents += outFile['events']
# if outFile['lfn'].count('/store/mc/Summer12/'):
totalEvents += outFile['events']
outFiles.add(outputFiles[0]['lfn'])
if outFile['lfn'] in ['/store/generator/Summer12/Wbb_FullyHadronic_8TeV_madgraph/GEN/START53_V7C-v1/00000/12DA0CF7-E743-E211-BA20-002590574544.root',
'/store/generator/Summer12/Wbb_FullyHadronic_8TeV_madgraph/GEN/START53_V7C-v1/00000/E0F13F75-E743-E211-8B7E-00259056F28A.root']:
loc = fwjr.data.cmsRun1.output.LHEoutput.files.file0.location
lfn = fwjr.data.cmsRun1.output.LHEoutput.files.file0.lfn
sql = """INSERT INTO dbsbuffer_file_location (filename, location)
SELECT df.id, dl.id
FROM dbsbuffer_file df
INNER JOIN dbsbuffer_location dl
WHERE df.lfn = '%s'
AND dl.se_name = '%s';
""" % (lfn, loc)
print sql
# if inEvents != outEvents:
# print 'GOTCHA!'
os.system('rm -r %s' % tmpdir)
