def injectFilesFromDBS(inputFileset, datasetPath, runsWhiteList=[]):
"""
_injectFilesFromDBS_
"""
print("injecting files from %s into %s, please wait..." % (datasetPath, inputFileset.name))
args = {}
args["url"] = "https://cmsweb.cern.ch/dbs/prod/global/DBSReader"
args["version"] = "DBS_2_1_1"
args["mode"] = "GET"
dbsApi = DbsApi(args)
dbsResults = dbsApi.listFileArray(path=datasetPath, retriveList=["retrive_lumi", "retrive_run"])
print(" found %d files, inserting into wmbs..." % (len(dbsResults)))
for dbsResult in dbsResults:
if runsWhiteList and str(dbsResult["LumiList"][0]["RunNumber"]) not in runsWhiteList:
continue
myFile = File(lfn=dbsResult["LogicalFileName"], size=dbsResult["FileSize"],
events=dbsResult["NumberOfEvents"], checksums={"cksum": dbsResult["Checksum"]},
locations="cmssrm.fnal.gov", merged=True)
myRun = Run(runNumber=dbsResult["LumiList"][0]["RunNumber"])
for lumi in dbsResult["LumiList"]:
myRun.appendLumi(lumi["LumiSectionNumber"])
myFile.addRun(myRun)
myFile.create()
inputFileset.addFile(myFile)
if len(inputFileset) < 1:
raise Exception("No files were selected!")
inputFileset.commit()
inputFileset.markOpen(False)
return
def injectFilesFromDBS(inputFileset, datasetPath, runsWhiteList=[]):
"""
_injectFilesFromDBS_
"""
print("injecting files from %s into %s, please wait..." % (datasetPath, inputFileset.name))
args = {}
args["url"] = "https://cmsweb-prod.cern.ch/dbs/prod/global/DBSReader"
args["version"] = "DBS_2_1_1"
args["mode"] = "GET"
dbsApi = DbsApi(args)
dbsResults = dbsApi.listFileArray(path=datasetPath, retriveList=["retrive_lumi", "retrive_run"])
print(" found %d files, inserting into wmbs..." % (len(dbsResults)))
for dbsResult in dbsResults:
if runsWhiteList and str(dbsResult["LumiList"][0]["RunNumber"]) not in runsWhiteList:
continue
myFile = File(lfn=dbsResult["LogicalFileName"], size=dbsResult["FileSize"],
events=dbsResult["NumberOfEvents"], checksums={"cksum": dbsResult["Checksum"]},
locations="cmssrm.fnal.gov", merged=True)
myRun = Run(runNumber=dbsResult["LumiList"][0]["RunNumber"])
for lumi in dbsResult["LumiList"]:
myRun.appendLumi(lumi["LumiSectionNumber"])
myFile.addRun(myRun)
myFile.create()
inputFileset.addFile(myFile)
if len(inputFileset) < 1:
raise Exception("No files were selected!")
inputFileset.commit()
inputFileset.markOpen(False)
return
def injectFilesFromDBS(inputFileset, datasetPath):
"""
_injectFilesFromDBS_
"""
print("injecting files from %s into %s, please wait..." %
(datasetPath, inputFileset.name))
args = {}
args["url"] = "https://cmsweb.cern.ch/dbs/prod/global/DBSReader"
args["mode"] = "GET"
dbsApi = DbsApi(args)
dbsResults = dbsApi.listFileArray(
path=datasetPath, retriveList=["retrive_lumi", "retrive_run"])
# Limiter on number of files
dbsResults = dbsResults[0:20]
print(" found %d files, inserting into wmbs..." % (len(dbsResults)))
for dbsResult in dbsResults:
myFile = File(lfn=dbsResult["LogicalFileName"],
size=dbsResult["FileSize"],
events=dbsResult["NumberOfEvents"],
checksums={"cksum": dbsResult["Checksum"]},
locations="cmssrm.fnal.gov",
merged=True)
myRun = Run(runNumber=dbsResult["LumiList"][0]["RunNumber"])
for lumi in dbsResult["LumiList"]:
myRun.appendLumi(lumi["LumiSectionNumber"])
myFile.addRun(myRun)
myFile.create()
inputFileset.addFile(myFile)
dbsFile = DBSBufferFile(lfn=dbsResult["LogicalFileName"],
size=dbsResult["FileSize"],
events=dbsResult["NumberOfEvents"],
checksums={"cksum": dbsResult["Checksum"]},
locations="cmssrm.fnal.gov",
status="NOTUPLOADED")
dbsFile.setDatasetPath(datasetPath)
dbsFile.setAlgorithm(appName="cmsRun",
appVer="Unknown",
appFam="Unknown",
psetHash="Unknown",
configContent="Unknown")
dbsFile.create()
inputFileset.commit()
inputFileset.markOpen(False)
return
def injectFilesFromDBS(inputFileset, datasetPath):
"""
_injectFilesFromDBS_
"""
print("injecting files from %s into %s, please wait..." % (datasetPath, inputFileset.name))
args={}
args["url"] = "https://cmsweb.cern.ch/dbs/prod/global/DBSReader"
args["mode"] = "GET"
dbsApi = DbsApi(args)
dbsResults = dbsApi.listFileArray(path = datasetPath, retriveList = ["retrive_lumi", "retrive_run"])
# Limiter on number of files
dbsResults = dbsResults[0:20]
print(" found %d files, inserting into wmbs..." % (len(dbsResults)))
for dbsResult in dbsResults:
myFile = File(lfn = dbsResult["LogicalFileName"], size = dbsResult["FileSize"],
events = dbsResult["NumberOfEvents"], checksums = {"cksum": dbsResult["Checksum"]},
locations = "cmssrm.fnal.gov", merged = True)
myRun = Run(runNumber = dbsResult["LumiList"][0]["RunNumber"])
for lumi in dbsResult["LumiList"]:
myRun.appendLumi(lumi["LumiSectionNumber"])
myFile.addRun(myRun)
myFile.create()
inputFileset.addFile(myFile)
dbsFile = DBSBufferFile(lfn = dbsResult["LogicalFileName"], size = dbsResult["FileSize"],
events = dbsResult["NumberOfEvents"], checksums = {"cksum": dbsResult["Checksum"]},
locations = "cmssrm.fnal.gov", status = "NOTUPLOADED")
dbsFile.setDatasetPath(datasetPath)
dbsFile.setAlgorithm(appName = "cmsRun", appVer = "Unknown", appFam = "Unknown",
psetHash = "Unknown", configContent = "Unknown")
dbsFile.create()
inputFileset.commit()
inputFileset.markOpen(False)
return
taskMaker = TaskMaker(workload, os.path.join(os.getcwd(), workloadName))
taskMaker.skipSubscription = True
taskMaker.processWorkload()
workload.save(workloadPath)
myThread = threading.currentThread()
myThread.transaction.begin()
for workloadTask in workload.taskIterator():
inputFileset = Fileset(name = workloadTask.getPathName())
inputFileset.create()
virtualFile = File(lfn = "%s-virtual-input" % workloadTask.getPathName(),
size = 0, events = numEvents,
locations = set(["cmssrm.fnal.gov", "storm-fe-cms.cr.cnaf.infn.it",
"cmssrm-fzk.gridka.de", "srm2.grid.sinica.edu.tw",
"srm-cms.gridpp.rl.ac.uk", "ccsrm.in2p3.fr",
"srmcms.pic.es"]), merged = False)
myRun = Run(runNumber = 1)
myRun.appendLumi(1)
virtualFile.addRun(myRun)
virtualFile.create()
inputFileset.addFile(virtualFile)
inputFileset.commit()
myWMBSHelper = WMBSHelper(workload)
myWMBSHelper._createSubscriptionsInWMBS(workloadTask.getPathName())
myThread.transaction.commit()
