def loadDBSBufferFilesBulk(self, fileObjs):
"""
_loadDBSBufferFilesBulk_
Yes, this is a stupid place to put it.
No, there's not better place.
"""
myThread = threading.currentThread()
dbsFiles = []
existingTransaction = self.beginTransaction()
factory = DAOFactory(package = "WMComponent.DBSBuffer.Database",
logger = myThread.logger,
dbinterface = myThread.dbi)
binds = []
for f in fileObjs:
binds.append(f["id"])
loadFiles = factory(classname = "DBSBufferFiles.LoadBulkFilesByID")
results = loadFiles.execute(files = binds, conn = self.getDBConn(),
transaction = self.existingTransaction())
for entry in results:
# Add loaded information
dbsfile = DBSBufferFile(id=entry['id'])
dbsfile.update(entry)
dbsFiles.append(dbsfile)
for dbsfile in dbsFiles:
if 'runInfo' in dbsfile.keys():
# Then we have to replace it with a real run
for r in dbsfile['runInfo'].keys():
run = Run(runNumber = r)
run.extend(dbsfile['runInfo'][r])
dbsfile.addRun(run)
del dbsfile['runInfo']
if 'parentLFNs' in dbsfile.keys():
# Then we have some parents
for lfn in dbsfile['parentLFNs']:
newFile = DBSBufferFile(lfn = lfn)
dbsfile['parents'].add(newFile)
del dbsfile['parentLFNs']
self.commitTransaction(existingTransaction)
return dbsFiles
def findUploadableFilesByDAS(self, das):
"""
_findUploadableDAS_
Find all the Dataset-Algo files available
with uploadable files.
"""
myThread = threading.currentThread()
existingTransaction = self.beginTransaction()
dbsFiles = []
factory = DAOFactory(package="WMComponent.DBSUpload.Database",
logger=myThread.logger,
dbinterface=myThread.dbi)
findFiles = factory(classname="LoadDBSFilesByDAS")
results = findFiles.execute(das=das,
conn=self.getDBConn(),
transaction=self.existingTransaction())
for entry in results:
# Add loaded information
dbsfile = DBSBufferFile(id=entry['id'])
dbsfile.update(entry)
dbsFiles.append(dbsfile)
for dbsfile in dbsFiles:
if 'runInfo' in dbsfile.keys():
# Then we have to replace it with a real run
for r in dbsfile['runInfo'].keys():
run = Run(runNumber=r)
run.extend(dbsfile['runInfo'][r])
dbsfile.addRun(run)
del dbsfile['runInfo']
if 'parentLFNs' in dbsfile.keys():
# Then we have some parents
for lfn in dbsfile['parentLFNs']:
newFile = DBSBufferFile(lfn=lfn)
dbsfile['parents'].add(newFile)
del dbsfile['parentLFNs']
self.commitTransaction(existingTransaction)
return dbsFiles
def loadFilesFromBlocks(self, blockID):
"""
_loadFilesFromBlocks_
Load the files from all active blocks
"""
findFiles = self.factory(classname="LoadFilesFromBlocks")
myThread = threading.currentThread()
existingTransaction = self.beginTransaction()
dbsFiles = []
results = findFiles.execute(blockID=blockID,
conn=self.getDBConn(),
transaction=self.existingTransaction())
for entry in results:
# Add loaded information
dbsfile = DBSBufferFile(id=entry['id'])
dbsfile.update(entry)
dbsFiles.append(dbsfile)
for dbsfile in dbsFiles:
if 'runInfo' in dbsfile.keys():
# Then we have to replace it with a real run
for r in dbsfile['runInfo'].keys():
run = Run(runNumber=r)
run.extend(dbsfile['runInfo'][r])
dbsfile.addRun(run)
del dbsfile['runInfo']
if 'parentLFNs' in dbsfile.keys():
# Then we have some parents
for lfn in dbsfile['parentLFNs']:
newFile = DBSBufferFile(lfn=lfn)
dbsfile['parents'].add(newFile)
del dbsfile['parentLFNs']
self.commitTransaction(existingTransaction)
return dbsFiles
def injectFilesFromDBS(inputFileset, datasetPath):
"""
_injectFilesFromDBS_
"""
print "injecting files from %s into %s, please wait..." % (
datasetPath, inputFileset.name)
args = {}
args[
"url"] = "http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet"
args["version"] = "DBS_2_0_9"
args["mode"] = "GET"
dbsApi = DbsApi(args)
dbsResults = dbsApi.listFiles(path=datasetPath,
retriveList=["retrive_lumi", "retrive_run"])
dbsResults = dbsResults[0:10]
print " found %d files, inserting into wmbs..." % (len(dbsResults))
for dbsResult in dbsResults:
myFile = File(lfn=dbsResult["LogicalFileName"],
size=dbsResult["FileSize"],
events=dbsResult["NumberOfEvents"],
checksums={"cksum": dbsResult["Checksum"]},
locations="cmssrm.fnal.gov",
merged=True)
myRun = Run(runNumber=dbsResult["LumiList"][0]["RunNumber"])
for lumi in dbsResult["LumiList"]:
myRun.lumis.append(lumi["LumiSectionNumber"])
myFile.addRun(myRun)
myFile.create()
inputFileset.addFile(myFile)
dbsFile = DBSBufferFile(lfn=dbsResult["LogicalFileName"],
size=dbsResult["FileSize"],
events=dbsResult["NumberOfEvents"],
checksums={"cksum": dbsResult["Checksum"]},
locations="cmssrm.fnal.gov",
status="LOCAL")
dbsFile.setDatasetPath(datasetPath)
dbsFile.setAlgorithm(appName="cmsRun",
appVer="Unknown",
appFam="Unknown",
psetHash="Unknown",
configContent="Unknown")
dbsFile.create()
inputFileset.commit()
inputFileset.markOpen(False)
return
def _addToDBSBuffer(self, dbsFile, checksums, locations):
"""
This step is just for increase the performance for
Accountant doesn't neccessary to check the parentage
"""
dbsBuffer = DBSBufferFile(lfn=dbsFile["LogicalFileName"],
size=dbsFile["FileSize"],
events=dbsFile["NumberOfEvents"],
checksums=checksums,
locations=locations,
status="GLOBAL")
dbsBuffer.setDatasetPath('bogus')
dbsBuffer.setAlgorithm(appName="cmsRun",
appVer="Unknown",
appFam="Unknown",
psetHash="Unknown",
configContent="Unknown")
if not dbsBuffer.exists():
self.dbsFilesToCreate.append(dbsBuffer)
#dbsBuffer.create()
return
def addFile(self, file, dataset=0):
"""
Add the file to the buffer
"""
myThread = threading.currentThread()
existingTransaction = self.beginTransaction()
bufferFile = DBSBufferFile(lfn = file['LFN'], size = file['Size'], events = file['TotalEvents'],
cksum=file['Checksum'], dataset=dataset)
runLumiList=file.getLumiSections()
runList=[x['RunNumber'] for x in runLumiList]
for runNumber in runList:
lumis = [int(y['LumiSectionNumber']) for y in runLumiList if y['RunNumber']==runNumber]
run=Run(runNumber, *lumis)
bufferFile.addRun(run)
if bufferFile.exists() == False:
bufferFile.create()
bufferFile.setLocation(pnn=file['locations'], immediateSave = True)
else:
bufferFile.load()
# Lets add the file to DBS Buffer as well
#UPDATE File Count
self.updateDSFileCount(dataset=dataset)
#Parent files
bufferFile.addParents(file.inputFiles)
self.commitTransaction(existingTransaction)
return
def testReportHandling(self):
"""
_testReportHandling_
Verify that we're able to parse a CMSSW report, convert it to a Report()
style report, pickle it and then have the accountant process it.
"""
self.procPath = os.path.join(
WMCore.WMBase.getTestBase(),
"WMCore_t/FwkJobReport_t/CMSSWProcessingReport.xml")
myReport = Report("cmsRun1")
myReport.parse(self.procPath)
# Fake some metadata that should be added by the stageout scripts.
for fileRef in myReport.getAllFileRefsFromStep("cmsRun1"):
fileRef.size = 1024
fileRef.location = "cmssrm.fnal.gov"
fwjrPath = os.path.join(self.tempDir, "ProcReport.pkl")
cmsRunStep = myReport.retrieveStep("cmsRun1")
cmsRunStep.status = 0
myReport.setTaskName('/TestWF/None')
myReport.persist(fwjrPath)
self.setFWJRAction.execute(jobID=self.testJob["id"], fwjrPath=fwjrPath)
pFile = DBSBufferFile(lfn="/path/to/some/lfn",
size=600000,
events=60000)
pFile.setAlgorithm(appName="cmsRun",
appVer="UNKNOWN",
appFam="RECO",
psetHash="GIBBERISH",
configContent="MOREGIBBERISH")
pFile.setDatasetPath("/bogus/dataset/path")
#pFile.addRun(Run(1, *[45]))
pFile.create()
config = self.createConfig(workerThreads=1)
accountant = JobAccountantPoller(config)
accountant.setup()
accountant.algorithm()
self.verifyJobSuccess(self.testJob["id"])
self.verifyFileMetaData(self.testJob["id"],
myReport.getAllFilesFromStep("cmsRun1"))
inputFile = File(
lfn=
"/store/backfill/2/unmerged/WMAgentCommissioining10/MinimumBias/RECO/rereco_GR09_R_34X_V5_All_v1/0000/outputRECORECO.root"
)
inputFile.load()
self.testMergeJob = Job(name="testMergeJob", files=[inputFile])
self.testMergeJob.create(group=self.mergeJobGroup)
self.testMergeJob["state"] = "complete"
self.stateChangeAction.execute(jobs=[self.testMergeJob])
self.mergePath = os.path.join(
WMCore.WMBase.getTestBase(),
"WMCore_t/FwkJobReport_t/CMSSWMergeReport.xml")
myReport = Report("mergeReco")
myReport.parse(self.mergePath)
# Fake some metadata that should be added by the stageout scripts.
for fileRef in myReport.getAllFileRefsFromStep("mergeReco"):
fileRef.size = 1024
fileRef.location = "cmssrm.fnal.gov"
fileRef.dataset = {
"applicationName": "cmsRun",
"applicationVersion": "CMSSW_3_4_2_patch1",
"primaryDataset": "MinimumBias",
"processedDataset": "Rereco-v1",
"dataTier": "RECO"
}
fwjrPath = os.path.join(self.tempDir, "MergeReport.pkl")
myReport.setTaskName('/MergeWF/None')
cmsRunStep = myReport.retrieveStep("mergeReco")
cmsRunStep.status = 0
myReport.persist(fwjrPath)
self.setFWJRAction.execute(jobID=self.testMergeJob["id"],
fwjrPath=fwjrPath)
accountant.algorithm()
self.verifyJobSuccess(self.testMergeJob["id"])
self.verifyFileMetaData(self.testMergeJob["id"],
myReport.getAllFilesFromStep("mergeReco"))
return
psetInstance = DbsQueryableParameterSet(Hash="GIBBERISH")
for newBlockName in badFiles.keys():
seName = blockLocation[newBlockName]
(datasetPath, junk) = newBlockName.split("#", 1)
dbsApi.insertBlock(datasetPath,
newBlockName,
storage_element_list=[seName])
blockRef = dbsApi.listBlocks(dataset=datasetPath,
block_name=newBlockName)[0]
print blockRef
newFiles = []
for newFileLFN in badFiles[newBlockName]:
localFile = DBSBufferFile(lfn=newFileLFN)
localFile.load(parentage=1)
(primaryDS, procDS, tier) = datasetPath[1:].split("/", 3)
primary = DbsPrimaryDataset(Name=primaryDS, Type="mc")
algo = DbsAlgorithm(ExecutableName=localFile["appName"],
ApplicationVersion=localFile["appVer"],
ApplicationFamily=localFile["appFam"],
ParameterSetID=psetInstance)
processed = DbsProcessedDataset(PrimaryDataset=primary,
AlgoList=[algo],
Name=procDS,
TierList=[tier],
ParentList=[],
PhysicsGroup="NoGroup",
Status="VALID",
