def getSortedAndCompletedPayloadDict(self, iovpllist, runlist, key):
# reduce to data in the run 'runNr' and sort by iov start time
pld = defaultdict(list)
if not iovpllist: return pld
if not runlist: return pld
runnrlist = [r.runNr for r in runlist]
nlb = dict([(r.runNr,r.lastlb) for r in runlist])
# first and last run of the runlist
firstrun = runnrlist[0]
lastrun = runnrlist[-1]
for iov, pl in iovpllist:
first = max(firstrun,iov.startTime.run)
last = min(lastrun,(iov.endTime-1).run)
for r in xrange(first,last+1):
if r in runnrlist:
pld[r].append((deepcopy(iov),pl))
# adjustments of IOV
for runnr, iovplbyrun in pld.items():
# adjust lb 0 to lb 1
if iovplbyrun[0][0].startTime.lb==0: iovplbyrun[0][0].startTime.lb=1
# truncate first IOV to a single run
iovplbyrun[0][0].truncateToSingleRun(runnr)
# sometimes and IOV has [RunLB1 - RunLB2), where RunLB2 has LB==1
# -> that gets truncated to [RunLB2 - RunLB2), which is obviously not valid
# -> so we slice that right out
if iovplbyrun[0][0].startTime == iovplbyrun[0][0].endTime:
iovplbyrun[:1] = []
# truncate last IOV to a single run
lastiov = iovplbyrun[-1][0]
lastiov.truncateToSingleRun(runnr)
# insert IOVs for missing ranges (value n.a.)
idx=0
curTime = IOVTime(runnr,1)
while curTime<lastiov.startTime:
iov = iovplbyrun[idx][0]
if curTime < iov.startTime:
# insert n.a. for missing iov
missingIOV = IOVRange(starttime=curTime, endtime=iov.startTime)
iovplbyrun.insert( idx, (missingIOV, "n.a.") )
curTime = IOVTime(iov.endTime)
idx += 1
# also check if IOV at the end is missing
runEndTime = IOVTime(runnr,nlb[runnr]+1) # last lb + 1
if lastiov.endTime<runEndTime:
missingIOV = IOVRange(starttime=lastiov.endTime, endtime=runEndTime)
iovplbyrun.append( (missingIOV, "n.a.") )
return pld
def readCondData(self, runranges, f, sortedRanges):
# get the data from cool
condData = defaultdict(list)
keys = dict([(ch,set(k)) for ch, k in groupby(sorted(self.ChannelKeys(), key=itemgetter(0)), itemgetter(0))])
for rr in runranges:
iovmin=(rr[0] << 32)+0
iovmax=((rr[1]+1) << 32)-1
# access to COOL
objs = self._retrieve(iovmin, iovmax, f, sortedRanges)
for obj in objs:
ch = obj.channel
for chtmp, internalKey, payloadKey in keys[ch]:
if type(payloadKey)==tuple:
if obj.isvf: payload = obj.payload
else: payload = tuple(map(obj.payload, payloadKey))
else:
payload = obj.payload(payloadKey)
condData[internalKey].append( (IOVRange(obj.iovrange), payload) )
return condData
def select(self, runlist):
# some preparation: compile the show patterns
start = time()
print self,
sys.stdout.flush()
newrunlist = []
connection = coolDbConn.GetSFODBConnection()
cursor = connection.cursor()
cursor.arraysize = 1000
runnrlist = [r.runNr for r in runlist]
from CoolRunQuery.AtlRunQuerySFO import GetSFO_NphysicseventsAll
with timer("GetSFO_NphysicseventsAll"):
events = GetSFO_NphysicseventsAll(
cursor, runnrlist) # { runnr: nPhysEvents }
for run in runlist:
iov = IOVRange(runStart=run.runNr,
lbStart=1,
runEnd=run.runNr + 1,
lbEnd=0)
for k in self.ResultKey():
if not run.runNr in events:
# the OVERLAP_EVENTS information is not yet available in the SFO (before Nov 15, 2009)
# use the inclusive number instead
run.addResult(k, "n.a.", iov, reject=False)
run.showDataIncomplete = True
continue
nev = events[run.runNr]
if self.ApplySelection(k) and not self.passes(nev, k):
run.addResult(k,
self.prettyValue(nev, k),
iov,
reject=True)
continue
run.addResult(k, self.prettyValue(nev, k), iov)
rej = self.rejectRun(run)
if not rej:
newrunlist += [run.runNr]
runlist = [r for r in runlist if r.runNr in newrunlist]
duration = time() - start
if self.applySelection:
print " ==> %i runs found (%.2f sec)" % (len(runlist), duration)
else:
print " ==> Done (%g sec)" % duration
return runlist
def defgen(self, db, defects_with_ignore, channels, channels_with_ignore):
"""
defects: list of defects returned by query
channels: explicit list of channels that matched the pattern
"""
intolerableDefects = db.get_intolerable_defects(old_primary_only=False)
for defects, ignore in defects_with_ignore:
chanlist = channels if ignore == None else channels_with_ignore[
ignore]
for d in defects:
if not d.present: continue
isVirtual = (d.user == 'sys:virtual'
) # db.defect_is_virtual(d.channel)
if not isVirtual:
run = d.since.run
# fill list of primaries defects for this run (this is needed for the comments, but not for the selection)
if not run in self.primaries: self.primaries[run] = []
self.primaries[run] += [d]
if not d.channel in chanlist: continue
defPayload = DQDefectPayload(
defect=d.channel,
comment=d.comment,
user=d.user,
primary=not isVirtual,
ignore=ignore,
tolerable=(not d.channel in intolerableDefects),
recoverable=d.recoverable)
# note that the comment is either the user's comment,
# a comment that the defect is auto-generated, or the
# list of source defects in case of virtual defects
#o = O("DQDEFECT", (d.channel, d.comment, ignore), IOVRange(starttime=d.since.real, endtime=d.until.real), True)
o = O("DQDEFECT", defPayload,
IOVRange(starttime=d.since.real, endtime=d.until.real),
True)
yield o
def select(self, runlist):
print (self, end='')
sys.stdout.flush()
start = time()
newrunlist = []
f = coolDbConn.GetDBConn(schema=self.schema, db=Selector.condDB()).getFolder(self.folder)
sortedChannel = sorted( list( set( self.CoolChannels() ) ) )
chansel = None
for ch in sortedChannel:
if chansel==None: chansel = cool.ChannelSelection(ch,ch,cool.ChannelSelection.sinceBeforeChannel)
else: chansel.addChannel(ch)
runranges = SmartRangeCalulator(runlist,True)
condData = defaultdict(list)
# access COOL
for rr in runranges:
firstrun = runlist[runlist.index(rr[0])]
lastrun = runlist[runlist.index(rr[1])]
iovmin=firstrun.sor
iovmax=lastrun.eor
objs = f.browseObjects( iovmin, iovmax, chansel)
while objs.goToNext():
obj= objs.currentRef()
ch = obj.channelId()
for chKey in self.ChannelKeys():
(channelNumber, resultKey, payloadKey) = chKey
if channelNumber != ch: continue
isBlob = (resultKey == 'olc:bcidmask')
if isBlob:
payloadvalue = obj.payload()[chKey[2]].read()
else:
payloadvalue = obj.payloadValue(chKey[2])
condData[resultKey].append( (IOVRange(obj=obj, timebased=True), payloadvalue) )
# for each key sort the data by IOV start time
for k in self.ResultKey(): condData[k].sort()
condDataDict = {}
for k in self.ResultKey():
condDataDict[k] = self.getSortedAndCompletedPayloadDict(condData[k],runlist,k)
for run in runlist:
rejectSomething = False
for k in self.ResultKey():
if not run.runNr in condDataDict[k]:
run.addResult(k, "n.a.")
continue
datavec = condDataDict[k][run.runNr]
if not datavec or len(datavec)==0:
run.addResult(k, "n.a.")
continue
if 'n.a.' in datavec: run.showDataIncomplete=True
anyDataSelected = False
for iov, data in datavec:
self.selDataMissing = False
if self.ApplySelection(k) and not self.passes(data,k):
run.addResult(k, self.prettyValue(data,k), iov, reject=True)
rejectSomething = True
continue
run.addResult(k, self.prettyValue(data,k), iov)
if self.selDataMissing: run.selDataIncomplete = True
if not (rejectSomething and self.rejectRun(run)):
newrunlist += [run.runNr]
runlist = [r for r in runlist if r.runNr in newrunlist]
duration = time() - start
if self.applySelection: print (" ==> %i runs found (%.2f sec)" % (len(runlist),duration))
else: print (" ==> Done (%g sec)" % duration)
return runlist
def getSortedAndCompletedPayloadDict(self, iovpllist, runlist,key):
# reduce to data in the run 'runNr' and sort by iov start time
pld = {}
if not iovpllist or not runlist: return pld
startiovindex = 0
for run in runlist:
pld[run.runNr] = []
if startiovindex<-1:
continue
# find the first iov that overlaps
iovindex = startiovindex
while True:
try:
iov = iovpllist[iovindex][0]
iovstart = iov.startTime.time
iovend = iov.endTime.time
if iovend>run.sor:
startiovindex = iovindex
break
if iovstart>run.eor:
startiovindex = -1
break
if iovend<run.sor and iovindex==len(iovpllist)-1:
startiovindex = -2
break
iovindex += 1
except IndexError:
startiovindex = -1
if startiovindex<0:
continue
# find the last iov that overlaps
iovindex = startiovindex
while True:
iov = iovpllist[iovindex][0]
iovstart = iov.startTime.time
iovend = iov.endTime.time
if iovend>=run.eor or iovindex==len(iovpllist)-1:
endiovindex = iovindex
break
iovindex += 1
# now we have the first and last iov that overlap with that run: startiovindex, endiovindex
lbindex = 0
for iov,pl in iovpllist[startiovindex:endiovindex+1]:
iovstart = iov.startTime.time
iovend = iov.endTime.time
endrun = run.runNr
while lbindex<len(run.lbtimes) and run.lbtimes[lbindex][0]<iovstart:
lbindex += 1
startlb = lbindex+1
while lbindex<len(run.lbtimes) and run.lbtimes[lbindex][0]<iovend:
lbindex += 1
lbindex -= 1
# now lbindex points to the last lb that starts within the iov or to one after the last lb
if lbindex==len(run.lbtimes)-1:
endlb = 0
endrun += 1
else:
endlb = lbindex+2 # +1 for lb-index->lb-number, +1 for closed interval
# append info of this IOV
lastIOV,lastpl=None,None
if len(pld[run.runNr])>0:
lastIOV,lastpl = pld[run.runNr][-1]
if lastpl==pl:
lastIOV.endTime.run = endrun
lastIOV.endTime.lb = endlb
else:
pld[run.runNr] += [(IOVRange(runStart=run.runNr, lbStart=startlb, runEnd=endrun, lbEnd=endlb), pl)] # in this case the end LB is inclusive
# for the next run we start looking at:
startiovindex = endiovindex
return pld
def coolgen(coolobjs):
while coolobjs.goToNext():
obj=coolobjs.currentRef()
yield O(obj.channelId(),obj.payloadValue, IOVRange(obj), False)
def vfgen(vfobjs):
for obj in vfobjs:
yield O(obj.channel, (str(obj.Code), obj.Comment),
IOVRange(starttime=obj.since, endtime=obj.until), True)