def specialSplit(): """ ./job-panda.py [-q] -ssplit External pulser runs have no data cleaning cut. Has a memory leak (can't close both TFiles, damn you, ROOT); submit each run as a batch job. """ cal = dsi.CalInfo() # runList = cal.GetSpecialRuns("extPulser") # runList = cal.GetSpecialRuns("longCal",5) runList = cal.GetSpecialRuns("forcedAcq", 8) for run in runList: inPath = "%s/waves/waveSkimDS%d_run%d.root" % (dsi.specialDir, dsi.GetDSNum(run), run) outPath = "%s/split/splitSkimDS%d_run%d.root" % ( dsi.specialDir, dsi.GetDSNum(run), run) outFiles = glob.glob("%s/split/splitSkimDS%d_run%d*.root" % (dsi.specialDir, dsi.GetDSNum(run), run)) for filename in outFiles: try: os.remove(filename) except OSError: pass if useJobQueue: sh("""./job-panda.py -splitf %s %s""" % (inPath, outPath)) else: sh("""%s './job-panda.py -splitf %s %s'""" % (jobStr, inPath, outPath))
def specialWrite(): """ ./job-panda.py -swrite Write TCuts from waveSkim files into splitSkim files. """ from ROOT import TFile, TNamed, TObject cal = dsi.CalInfo() runList = cal.GetSpecialRuns("longCal", 5) for run in runList: dsNum = dsi.GetDSNum(run) wavePath = "%s/waves/waveSkimDS%d_run%d.root" % (dsi.specialDir, dsNum, run) waveFile = TFile(wavePath) theCut = waveFile.Get("theCut").GetTitle() print(wavePath) splitFiles = glob.glob("%s/split/splitSkimDS%d_run%d*.root" % (dsi.specialDir, dsNum, run)) for idx in range(len(splitFiles)): if idx == 0: splitPath = "%s/split/splitSkimDS%d_run%d.root" % ( dsi.specialDir, dsNum, run) else: splitPath = "%s/split/splitSkimDS%d_run%d_%d.root" % ( dsi.specialDir, dsNum, run, idx) if not os.path.isfile(splitPath): print("File doesn't exist:", splitPath) return splitFile = TFile(splitPath, "UPDATE") thisCut = TNamed("theCut", theCut) thisCut.Write("", TObject.kOverwrite) print(splitFiles[-1])
def scanLAT2(dsIn=None, subIn=None, modIn=None): """ ./job-panda.py [-q] -lat2 """ skipDS6Cal = True cal = dsi.CalInfo() # loop over datasets, skipping DS6 cal runs till they're processed for ds in [0, 1, 2, 3, 4, 5, 6]: if skipDS6Cal is True and ds == 6: continue if dsIn is not None and ds != dsIn: continue # loop over keys in this DS for key in cal.GetKeys(ds): mod = -1 if "m1" in key: mod = 1 if "m2" in key: mod = 2 # loop over cIdx's for this key for cIdx in range(cal.GetIdxs(key)): if subIn is not None and cIdx != subIn: continue if modIn is not None and mod != modIn: continue job = "./lat2.py -scan %d %s %d %d" % (ds, key, mod, cIdx) if useJobQueue: sh("%s >& ./logs/lat2-%s-%d.txt" % (job, key, cIdx)) else: sh("%s '%s'" % (jobStr, job))
def checkDS5CWaveSkim(): from ROOT import TFile, TTree runList = [] calInfo = dsi.CalInfo() calKeys = calInfo.GetKeys() for key in calKeys: print("key:",key) if key!="ds5c": continue for idx in range(calInfo.GetIdxs(key)): lst = calInfo.GetCalList(key,idx) print(lst) runList += lst for run in runList: # check waveSkim files # inPath = "%s/waveSkimDS%d_run%d.root" % (dsi.calWaveDir,5,run) # tf = TFile(inPath) # tt = tf.Get("skimTree") # chType = "" # for br in tt.GetListOfBranches(): # if br.GetName()!="channel": continue # chType = br.GetClassName() # print(inPath.split("/")[-1], tt.GetEntries(), chType) # return # delete splitSkim files print(run) fWave = glob.glob("%s/waveSkimDS%d_run%d.root" % (dsi.calWaveDir,5,run)) fList = glob.glob("%s/splitSkimDS%d_run%d_*.root" % (dsi.calSplitDir,5,run)) print(fWave)
def specialSkim(): """ ./job-panda.py [-q (use job queue)] -sskim """ cal = dsi.CalInfo() # runList = cal.GetSpecialRuns("extPulser") # runList = cal.GetSpecialRuns("delayedTrigger") # runList = cal.GetSpecialRuns("longCal",5) runList = cal.GetSpecialRuns("forcedAcq", 8) for run in runList: if useJobQueue: # sh("""./skim_mjd_data -f %d -l -t 0.7 %s/skim >& ./logs/specialSkim-DS%d-%d.txt""" % (run,dsi.specialDir,dsi.GetDSNum(run),run)) sh("""./skim_mjd_data -f %d -x -l %s/skim >& ./logs/specialSkim-DS%d-%d.txt""" % (run, dsi.specialDir, dsi.GetDSNum(run), run)) else: sh("""%s './skim_mjd_data -f %d -x -l %s/skim'""" % (jobStr, run, dsi.specialDir))
def specialWave(): """ ./job-panda.py [-q (use queue)] -swave """ cal = dsi.CalInfo() # runList = cal.GetSpecialRuns("extPulser") # runList = cal.GetSpecialRuns("longCal",5) runList = cal.GetSpecialRuns("forcedAcq", 8) for run in runList: if useJobQueue: # sh("""./wave-skim -l -n -f %d %d -p %s/skim %s/waves >& ./logs/wave-ds%d-%d.txt""" % (dsi.GetDSNum(run),run,dsi.specialDir,dsi.specialDir,dsi.GetDSNum(run),run)) sh("""./wave-skim -x -n -f %d %d -p %s/skim %s/waves >& ./logs/wave-ds%d-%d.txt""" % (dsi.GetDSNum(run), run, dsi.specialDir, dsi.specialDir, dsi.GetDSNum(run), run)) else: sh("""%s './wave-skim -x -n -f %d %d -p %s/skim %s/waves'""" % (jobStr, dsi.GetDSNum(run), run, dsi.specialDir, dsi.specialDir))
def getCalRunList(dsNum=None, subNum=None, runNum=None): """ ./job-panda.py -cal (-ds [dsNum] -sub [dsNum] [calIdx] -run [runNum]) Create a calibration run list, using the CalInfo object in DataSetInfo.py . Note that the -sub option is re-defined here to mean a calibration range idx. Note that running with -cal alone will create a list for all datasets (mega mode). """ runLimit = None # need all the stats we can get ... calList = [] calInfo = dsi.CalInfo() calKeys = calInfo.GetKeys(dsNum) # single-run mode if runNum != None: calList.append(runNum) print(calList) return calList # multi-run mode: for key in calKeys: print("key:", key) # if key=="ds6": continue # comment this out and do -cal -ds 6 to get the ds6 list # -cal (mega mode) if dsNum == None: for idx in range(calInfo.GetIdxs(key)): lst = calInfo.GetCalList(key, idx, runLimit) print(lst) calList += lst # -ds elif subNum == None: for idx in range(calInfo.GetIdxs(key)): lst = calInfo.GetCalList(key, idx, runLimit) print(lst) calList += lst # -sub else: lst = calInfo.GetCalList(key, subNum, runLimit) if lst == None: continue print(lst) calList += lst # remove any duplicates, but there probably aren't any calList = sorted(list(set(calList))) print("Total Runs:", len(calList)) return calList
def specialCheck(): """./job-panda.py -scheck A next step could be to 'hadd' split files back together, but we'll wait for now. """ from ROOT import TFile, TTree cal = dsi.CalInfo() runList = cal.GetSpecialRuns("extPulser") for run in runList: fileList = glob.glob("%s/lat/latSkimDS%d_run%d_*.root" % (dsi.specialDir, dsi.GetDSNum(run), run)) for f in fileList: tf = TFile(f) tr = tf.Get("skimTree") print(f) print(tr.GetEntries()) tr.GetEntry(0) tf.Close()
def specialLAT(): """ ./job-panda.py [-q (use job queue)] -slat""" cal = dsi.CalInfo() # runList = cal.GetSpecialRuns("extPulser") # runList = cal.GetSpecialRuns("longCal",5) runList = cal.GetSpecialRuns("forcedAcq", 8) # deal with unsplit files # run = runList[0] # dsNum = dsi.GetDSNum(run) # inFile = "%s/waves/waveSkimDS%d_run%d.root" % (dsi.specialDir,dsNum,run) # outFile = "%s/lat/latSkimDS%d_run%d.root" % (dsi.specialDir,dsNum,run) # sh("""./lat.py -x -b -f %d %d -p %s %s""" % (dsNum,run,inFile,outFile)) # deal with split files for run in runList: dsNum = dsi.GetDSNum(run) inFiles = glob.glob("%s/split/splitSkimDS%d_run%d*.root" % (dsi.specialDir, dsNum, run)) for idx in range(len(inFiles)): if idx == 0: inFile = "%s/split/splitSkimDS%d_run%d.root" % (dsi.specialDir, dsNum, run) else: inFile = "%s/split/splitSkimDS%d_run%d_%d.root" % ( dsi.specialDir, dsNum, run, idx) if not os.path.isfile(inFile): print("File doesn't exist:", inFile) return outFile = "%s/lat/latSkimDS%d_run%d_%d.root" % (dsi.specialDir, dsNum, run, idx) if useJobQueue: # this is what you would want for a normal cron queue # sh("""./lat.py -x -b -f %d %d -p %s %s""" % (dsNum, run, inFile, outFile)) # this is what i need for a 1-node job pump # sh("""./lat.py -x -b -f %d %d -p %s %s >& ./logs/extPulser-%d-%d.txt""" % (dsNum, run, inFile, outFile, run, idx)) # sh("""./lat.py -b -f %d %d -p %s %s >& ./logs/longCal-%d-%d.txt""" % (dsNum, run, inFile, outFile, run, idx)) sh("""./lat.py -x -b -f %d %d -p %s %s >& ./logs/forceAcq-%d-%d.txt""" % (dsNum, run, inFile, outFile, run, idx)) else: sh("""%s './lat.py -x -b -f %d %d -p %s %s' """ % (jobStr, dsNum, run, inFile, outFile))
def specialBuild(): """ ./job-panda.py -sbuild Set 1 of the external pulser runs were built with --donotbuild. Let's manually build them. Example: - Make sure MkCookie has been run recently majorcaroot --nomultisampling --setspecialchanmap /global/homes/m/mjd/production/P3JDYspecchanmap.txt --donotbuild 2015-9-3-P3JDY_Run5942 process_mjd_cal OR_*.root pulsertag 5942 auto-thresh 5942 process_mjd_gat OR_*.root """ cal = dsi.CalInfo() rawDir = "/global/project/projectdirs/majorana/data/mjd/surfmjd/data/raw/P3JDY/Data" buildDir = dsi.dataDir + "/mjddatadir" os.chdir(buildDir) runList = [] for pIdx in [7, 8, 9, 10, 11, 12]: runList.extend(cal.GetSpecialRuns("extPulser", pIdx)) for run in runList: os.chdir(buildDir + "/raw") rawFile = glob.glob("%s/*%d*" % (rawDir, run))[0] rawName = rawFile.rsplit('/', 1)[1] rawName = rawName.rsplit('.')[0] # sp.call("""zcat %s > %s""" % (rawFile, rawName), shell=True) os.chdir(buildDir + "/built") sp.call( """majorcaroot --nomultisampling --setspecialchanmap /global/homes/m/mjdproduction/P3JDYspecchanmap.txt ../raw/%s""" % (rawName), shell=True) # ... i kinda lost steam here. Set 1 runs are not worth this much hassle. return
def tuneCuts(argString, dsNum=None): """ ./job-panda.py -tuneCuts '[argString]' -- run over all ds's ./job-panda.py -ds [dsNum] -tuneCuts '[argString]' -- just one DS Submit a bunch of lat3.py jobs to the queues. NOTE: 1) If processing individual dataset, the -ds option MUST come before -tuneCuts. 2) Make sure to put argString in quotes. 3) argString may be multiple options separated by spaces Options for argString: -all, -bcMax, -noiseWeight, -bcTime, -tailSlope, -fitSlo, -riseNoise """ calInfo = dsi.CalInfo() if dsNum == None: for i in dsi.dsMap.keys(): if i == 6: continue for mod in [1, 2]: try: for j in range(calInfo.GetIdxs("ds%d_m%d" % (i, mod))): print("%s './lat3.py -db -tune %s -s %d %d %d %s" % (jobStr, dsi.calLatDir, i, j, mod, argString)) sh("""%s './lat3.py -db -tune %s -s %d %d %d %s '""" % (jobStr, dsi.calLatDir, i, j, mod, argString)) except: continue # -ds else: for mod in [1, 2]: try: for j in range(calInfo.GetIdxs("ds%d_m%d" % (dsNum, mod))): print("%s './lat3.py -db -tune %s -s %d %d %d %s" % (jobStr, dsi.calLatDir, dsNum, j, mod, argString)) sh("""%s './lat3.py -db -tune %s -s %d %d %d %s '""" % (jobStr, dsi.calLatDir, dsNum, j, mod, argString)) except: continue
#!/usr/bin/env python import sys, os sys.argv.append("-b") import tinydb as db import numpy as np import waveLibs as wl import dsi bkg = dsi.BkgInfo() cal = dsi.CalInfo() det = dsi.DetInfo() from ROOT import TFile, TTree, MGTWaveform # switches fLimit = None # set to None to run over everything skipDS6Cal = True verbose = True testMode = False def main(argv): """ NOTE: don't use globs when getting files. Manually make sure everything is here. Can submit these commands as separate batch jobs: ./check-files.py -all ./check-files.py -c -all """ global checkCal checkCal = False if checkCal: print("Skip DS6 cal?", skipDS6Cal)
def fillDetInfo(): """ ./chan-sel.py -fill Summarize the results of getSettings in LAT/data/runSettings-v2.npz. Create a file accessible by the DetInfo object in dsi.py It contains dictionaries of TRAP threshold, HV settings, detector/channel mapping, and pulser monitor lists, that span an entire dataset (not broken down into separate calIdx's.) # FORMAT: {ds : {'det' : [(run1,val1),(run2,val2)...]} } """ # 1. maps of analysis channel to cpd, and pulser monitor channels detCH, pMons = {}, {} for ds in [0, 1, 2, 3, 4, 5, 6]: f = np.load("%s/data/ds%d_detChans.npz" % (os.environ['LATDIR'], ds)) detCH[ds] = f['arr_0'].item() pMons[ds] = f['arr_1'].item() # 2. maps of HV and TRAP threshold settings are stored in the DB. # make them global, and move them to the runSettings file. # FORMAT: {ds : {'det' : [(run1,val1),(run2,val2)...]} } detHV, detTH = {}, {} # load all possible values, as in settingsMgr detDB = db.TinyDB("%s/calDB-v2.json" % dsi.latSWDir) detPars = db.Query() cal = dsi.CalInfo() for ds in [0, 1, 2, 3, 4, 5, 6]: # for ds in [3]: print("scanning ds", ds) detTH[ds] = {} detHV[ds] = {} for key in cal.GetKeys(ds): mod = -1 if "m1" in key: mod = 1 if "m2" in key: mod = 2 for cIdx in range(cal.GetIdxs(key)): # load the DB records dbKeyTH = "trapThr_%s_c%d" % (key, cIdx) dbValTH = dsi.getDBRecord(dbKeyTH, calDB=detDB, pars=detPars) dbKeyHV = "hvBias_%s_c%d" % (key, cIdx) dbValHV = dsi.getDBRecord(dbKeyHV, calDB=detDB, pars=detPars) # debug: print the record # for val in sorted(dbValTH): # if len(dbValTH[val])>0: # print(val, dbValTH[val]) # return # fill the first value if len(detTH[ds]) == 0: detTH[ds] = dbValTH detHV[ds] = dbValHV continue # check for new threshold values. for cpd in detTH[ds]: nOld, nNew = len(detTH[ds][cpd]), len(dbValTH[cpd]) # detector just came online if nOld == 0 and nNew > 0: detTH[ds][cpd] = dbValTH[cpd] continue # detector still offline if nOld == 0 and nNew == 0: continue # detector just went offline if nOld > 0 and nNew == 0: continue # check last run/trap pair against each new one prevRun, prevTH = detTH[ds][cpd][-1][0], detTH[ds][cpd][ -1][1] for val in dbValTH[cpd]: thisRun, thisTH = val[0], val[1] if thisTH != prevTH: detTH[ds][cpd].append([thisRun, thisTH]) prevTH = thisTH # check for new HV values. for cpd in detHV[ds]: nOld, nNew = len(detHV[ds][cpd]), len(dbValHV[cpd]) # detector just came online if nOld == 0 and nNew > 0: detHV[ds][cpd] = dbValHV[cpd] continue # detector still offline if nOld == 0 and nNew == 0: continue # detector just went offline if nOld > 0 and nNew == 0: continue # check last run/trap pair against each new one prevRun, prevHV = detHV[ds][cpd][-1][0], detHV[ds][cpd][ -1][1] for val in dbValHV[cpd]: thisRun, thisHV = val[0], val[1] if thisHV != prevHV: print( "found HV diff. cpd %d prev %dV (run %d) new %dV (run %d)" % (cpd, prevHV, prevRun, thisHV, thisRun)) detHV[ds][cpd].append([thisRun, thisHV]) prevHV = thisHV # return # # load the old file and compare # # GOAL: improve on this file. # # f = np.load("%s/data/runSettings.npz" % dsi.latSWDir) # # detHVOld = f['arr_0'].item() # # detTHOld = f['arr_1'].item() # # detCHOld = f['arr_2'].item() # # pMonsOld = f['arr_3'].item() # # ds = 3 # print("old results, ds",ds) # for cpd in sorted(detTHOld[ds]): # if cpd!="122":continue # if len(detTHOld[ds][cpd]) > 0: # print(cpd, detTHOld[ds][cpd]) # # # for ds in [0,1,2,3,4,5,6]: # print("thresh results, ds:",ds) # for cpd in sorted(detTH[ds]): # # if cpd!=122:continue # if len(detTH[ds][cpd]) > 0: # print(cpd, detTH[ds][cpd]) np.savez("%s/data/runSettings-v2.npz" % dsi.latSWDir, detHV, detTH, detCH, pMons)