def injectFilesFromDBS(inputFileset, datasetPath, runsWhiteList=[]): """ _injectFilesFromDBS_ """ print("injecting files from %s into %s, please wait..." % (datasetPath, inputFileset.name)) args = {} args["url"] = "https://cmsweb.cern.ch/dbs/prod/global/DBSReader" args["version"] = "DBS_2_1_1" args["mode"] = "GET" dbsApi = DbsApi(args) dbsResults = dbsApi.listFileArray(path=datasetPath, retriveList=["retrive_lumi", "retrive_run"]) print(" found %d files, inserting into wmbs..." % (len(dbsResults))) for dbsResult in dbsResults: if runsWhiteList and str(dbsResult["LumiList"][0]["RunNumber"]) not in runsWhiteList: continue myFile = File(lfn=dbsResult["LogicalFileName"], size=dbsResult["FileSize"], events=dbsResult["NumberOfEvents"], checksums={"cksum": dbsResult["Checksum"]}, locations="cmssrm.fnal.gov", merged=True) myRun = Run(runNumber=dbsResult["LumiList"][0]["RunNumber"]) for lumi in dbsResult["LumiList"]: myRun.appendLumi(lumi["LumiSectionNumber"]) myFile.addRun(myRun) myFile.create() inputFileset.addFile(myFile) if len(inputFileset) < 1: raise Exception("No files were selected!") inputFileset.commit() inputFileset.markOpen(False) return
def injectFilesFromDBS(inputFileset, datasetPath, runsWhiteList=[]): """ _injectFilesFromDBS_ """ print("injecting files from %s into %s, please wait..." % (datasetPath, inputFileset.name)) args = {} args["url"] = "https://cmsweb-prod.cern.ch/dbs/prod/global/DBSReader" args["version"] = "DBS_2_1_1" args["mode"] = "GET" dbsApi = DbsApi(args) dbsResults = dbsApi.listFileArray(path=datasetPath, retriveList=["retrive_lumi", "retrive_run"]) print(" found %d files, inserting into wmbs..." % (len(dbsResults))) for dbsResult in dbsResults: if runsWhiteList and str(dbsResult["LumiList"][0]["RunNumber"]) not in runsWhiteList: continue myFile = File(lfn=dbsResult["LogicalFileName"], size=dbsResult["FileSize"], events=dbsResult["NumberOfEvents"], checksums={"cksum": dbsResult["Checksum"]}, locations="cmssrm.fnal.gov", merged=True) myRun = Run(runNumber=dbsResult["LumiList"][0]["RunNumber"]) for lumi in dbsResult["LumiList"]: myRun.appendLumi(lumi["LumiSectionNumber"]) myFile.addRun(myRun) myFile.create() inputFileset.addFile(myFile) if len(inputFileset) < 1: raise Exception("No files were selected!") inputFileset.commit() inputFileset.markOpen(False) return
def injectFilesFromDBS(inputFileset, datasetPath): """ _injectFilesFromDBS_ """ print("injecting files from %s into %s, please wait..." % (datasetPath, inputFileset.name)) args = {} args["url"] = "https://cmsweb.cern.ch/dbs/prod/global/DBSReader" args["mode"] = "GET" dbsApi = DbsApi(args) dbsResults = dbsApi.listFileArray( path=datasetPath, retriveList=["retrive_lumi", "retrive_run"]) # Limiter on number of files dbsResults = dbsResults[0:20] print(" found %d files, inserting into wmbs..." % (len(dbsResults))) for dbsResult in dbsResults: myFile = File(lfn=dbsResult["LogicalFileName"], size=dbsResult["FileSize"], events=dbsResult["NumberOfEvents"], checksums={"cksum": dbsResult["Checksum"]}, locations="cmssrm.fnal.gov", merged=True) myRun = Run(runNumber=dbsResult["LumiList"][0]["RunNumber"]) for lumi in dbsResult["LumiList"]: myRun.appendLumi(lumi["LumiSectionNumber"]) myFile.addRun(myRun) myFile.create() inputFileset.addFile(myFile) dbsFile = DBSBufferFile(lfn=dbsResult["LogicalFileName"], size=dbsResult["FileSize"], events=dbsResult["NumberOfEvents"], checksums={"cksum": dbsResult["Checksum"]}, locations="cmssrm.fnal.gov", status="NOTUPLOADED") dbsFile.setDatasetPath(datasetPath) dbsFile.setAlgorithm(appName="cmsRun", appVer="Unknown", appFam="Unknown", psetHash="Unknown", configContent="Unknown") dbsFile.create() inputFileset.commit() inputFileset.markOpen(False) return
def injectFilesFromDBS(inputFileset, datasetPath): """ _injectFilesFromDBS_ """ print("injecting files from %s into %s, please wait..." % (datasetPath, inputFileset.name)) args={} args["url"] = "https://cmsweb.cern.ch/dbs/prod/global/DBSReader" args["mode"] = "GET" dbsApi = DbsApi(args) dbsResults = dbsApi.listFileArray(path = datasetPath, retriveList = ["retrive_lumi", "retrive_run"]) # Limiter on number of files dbsResults = dbsResults[0:20] print(" found %d files, inserting into wmbs..." % (len(dbsResults))) for dbsResult in dbsResults: myFile = File(lfn = dbsResult["LogicalFileName"], size = dbsResult["FileSize"], events = dbsResult["NumberOfEvents"], checksums = {"cksum": dbsResult["Checksum"]}, locations = "cmssrm.fnal.gov", merged = True) myRun = Run(runNumber = dbsResult["LumiList"][0]["RunNumber"]) for lumi in dbsResult["LumiList"]: myRun.appendLumi(lumi["LumiSectionNumber"]) myFile.addRun(myRun) myFile.create() inputFileset.addFile(myFile) dbsFile = DBSBufferFile(lfn = dbsResult["LogicalFileName"], size = dbsResult["FileSize"], events = dbsResult["NumberOfEvents"], checksums = {"cksum": dbsResult["Checksum"]}, locations = "cmssrm.fnal.gov", status = "NOTUPLOADED") dbsFile.setDatasetPath(datasetPath) dbsFile.setAlgorithm(appName = "cmsRun", appVer = "Unknown", appFam = "Unknown", psetHash = "Unknown", configContent = "Unknown") dbsFile.create() inputFileset.commit() inputFileset.markOpen(False) return
taskMaker = TaskMaker(workload, os.path.join(os.getcwd(), workloadName)) taskMaker.skipSubscription = True taskMaker.processWorkload() workload.save(workloadPath) myThread = threading.currentThread() myThread.transaction.begin() for workloadTask in workload.taskIterator(): inputFileset = Fileset(name = workloadTask.getPathName()) inputFileset.create() virtualFile = File(lfn = "%s-virtual-input" % workloadTask.getPathName(), size = 0, events = numEvents, locations = set(["cmssrm.fnal.gov", "storm-fe-cms.cr.cnaf.infn.it", "cmssrm-fzk.gridka.de", "srm2.grid.sinica.edu.tw", "srm-cms.gridpp.rl.ac.uk", "ccsrm.in2p3.fr", "srmcms.pic.es"]), merged = False) myRun = Run(runNumber = 1) myRun.appendLumi(1) virtualFile.addRun(myRun) virtualFile.create() inputFileset.addFile(virtualFile) inputFileset.commit() myWMBSHelper = WMBSHelper(workload) myWMBSHelper._createSubscriptionsInWMBS(workloadTask.getPathName()) myThread.transaction.commit()