def main():
since, until = RunLumi(152000, 0), RunLumi(155000, 0)
#since, until = RunLumi(152777, 0), RunLumi(152778, 0) - 1
S = make_run_selector(atlas_runs_set())
lest_iovs = S(fetch_iovs("LBLESTONL", since, until, [0]))
lblb_iovs = S(fetch_iovs("LBLB", since, until))
eor_iovs = S(fetch_iovs("EOR", since, until))
rfp_iovs = S(
fetch_iovs("/TDAQ/RunCtrl/DataTakingMode",
since,
until,
database="tdaq"))
args = lest_iovs, lblb_iovs, eor_iovs, rfp_iovs
ptagil_iovs = make_ptag_integrated_lumi_iovs(*args)
state_iovs = fetch_iovs("SHIFTOFL", since, until,
named_channels=True) # [111, 114, 115, 121])
result = accumulate_result(ptagil_iovs, state_iovs)
from pprint import pprint
pprint(result["data10_7TeV"])
def main():
run = 144247 # Recent run where CSC gas state was bad.
run = 136379
run = 155112 # Recent long run (Sat May 15th, 11am -> Sun 16th 8am), 632 lbs
run = 156463
#run = 155669 # MDT NConfig goes below 600
from sys import argv
args = argv[1:]
if args:
run = int(args[0])
since, until = RunLumi(run, 0), RunLumi(run + 1, 0)
with timer("Read LBLB"):
lblb = fetch_iovs("LBLB", since, until, with_channel=False)
lbtime = inverse_lblb(lblb)
#system = SCT() # Works, very slightly discrepant for the config.
#system = Pixels() # Works
#system = Tile() # Works
#system = CSC() # Needs globals fixing
#system = TDQ() # Works, needs further testing
#system = RPC() # Works
#system = LAr() # Works, but produces slightly different result. Original gives:
# DCSOFL_VAL(since=(155112: 167), until=(155112: 176), channel=215, Code=2L)
#system = TRT() # Unresolved discrepancies: Wrong NConfig, NWorking off by one in run 155112.
system = MDT() # Seems to work
#system = Magnets() # Needs to be finished
#system = Lucid() # Needs to be written
#print system.mapping.keys()
with timer("Calculate result for %s" % system):
result = system.run(lbtime)
print "Done."
print
#print result[0]
print "Run range:", since, "to", until
print "DCSC2 intervals of validity:"
pprint_objects(result[:30])
print
output_channels = system.mapping.keys()
iovs = fetch_iovs("DCSOFL", since, until, output_channels)
#print hex(hash(tuple(iovs))), hex(hash(tuple(result)))
print "Original DCSC intervals of validity:"
pprint_objects(iovs[:30])
return
compare_iovs(iovs, result)
def test():
from pprint import pprint
lumi_estimates = fetch_iovs("/TRIGGER/LUMI/LBLESTONL",
channels=[0],
database="trigger")
print lumi_estimates
#pprint(list(lumi_estimates[-10:]))
iovs = fetch_iovs("/TDAQ/RunCtrl/DataTakingMode",
with_channel=False,
database="tdaq")
#pprint(list(iovs))
#lumi_estimates = lumi_estimates[:10]
physics_iovs = iovs.select(ReadyForPhysics=1)
print "iovs=", iovs
print "physics_iovs=", physics_iovs
from DQUtils.general import timer
from DQUtils.iov_alignment import compute_aligned_iovs, compute_aligned_iovs_with_heap, intersect_iovs
#pprint(list(lumi_estimates))
#print
#pprint(list(physics_iovs))
#print
from time import time
start = time()
le, pi = compute_aligned_iovs(lumi_estimates, physics_iovs)
end = time()
print "Took %.3f with old algo" % (end-start)
print le, pi
#pprint(list(le[:10]))
#pprint(list(pi[:10]))
from DQUtils.sugar import IOVSet
from time import time
start = time()
#le, pi = map(IOVSet, zip(*list(compute_aligned_iovs_with_heap(lumi_estimates, physics_iovs))))
le, pi = intersect_iovs(lumi_estimates, physics_iovs)
end = time()
print "Took %.3f with heap" % (end-start)
print le, pi
pprint(list(le[-10:]))
pprint(list(pi[-10:]))
def fetch_fixedup_folder_information(folder, since, until, cs):
"""
Retrieves `iovs` and `info_runs` from COOL.
`iovs` is a list of iovs on folder in the range `since` `until`
belonging to `cs`. These iovs are corrected to make up for deficiencies
in the data on the database.
`info_runs`
"""
info_runs = fetch_iovs("EOR",
since,
until,
with_channel=False,
what=["FilenameTag"])
iovs = fetch_iovs(folder, since, until, cs, what=["Code"])
# Filter the iovs by those that appear in the ATLAS partition
info_runs = filter_atlas_runs(info_runs)
iovs = filter_atlas_runs(iovs)
bad_runs = [iov for iov in info_runs if iov.until.lumi == 0xFFFFFFFF]
info_runs = [iov for iov in info_runs if iov.until.lumi != 0xFFFFFFFF]
if bad_runs:
bad_run_count = len(bad_runs)
print(
" Warning: There are %i runs which don't have a sensible end "
"time on the\n"
" EOR folder, and have been excluded "
"from the following counts." % bad_run_count)
print " They are:",
print ", ".join(sorted(set("%i" % i.since.run for i in bad_runs)))
print
# Remove iovs which go from (run, [max]) to (run+1, 0)
iovs = [
iov for iov in iovs
if not (iov.since.lumi == 0xFFFFFFFF and iov.until.lumi == 0)
]
# Compute a dictionary of {run: length}, use it to truncate the iovs
run_lengths = dict((iov.since.run, iov.until.lumi) for iov in info_runs)
iovs = fix_iov_lengths(iovs, run_lengths)
# Last-ditch check
check_bad_iovs(iovs)
return iovs, info_runs
def get_runends():
from DQUtils.db import fetch_iovs
eor_params = fetch_iovs('EOR_Params', with_channel=False, what=[])
runends = {}
for iov in eor_params:
runends[iov.since.run] = iov
return runends
def makeGRL(run, defect, fname):
import DQUtils, DQDefects
tag = 'HEAD'
runs = [run]
print('Query run information...', end='')
from DQUtils.db import fetch_iovs
dbinstance = 'CONDBR2'
eor = fetch_iovs('EOR', (min(runs) << 32) | 1,
(max(runs) << 32) | 0xFFFFFFFF,
with_channel=False, what=[], database='COOLONL_TDAQ/%s' % dbinstance)
eor = eor.trim_iovs
eor = DQUtils.IOVSet(iov for iov in eor if iov.since.run in runs)
print('done')
print('Query defects...', end='')
ddb = DQDefects.DefectsDB('COOLOFL_GLOBAL/%s' % dbinstance, tag=tag)
ignores = {_ for _ in ddb.defect_names if 'UNCHECKED' in _}
try:
defectiovs = ddb.retrieve(since = min(runs) << 32 | 1,
until = max(runs) << 32 | 0xffffffff,
channels = [defect],
evaluate_full = False,
ignore=ignores)
except Exception as e:
print(e)
raise
print('Doing exclusions...', end='')
okiovs = eor.logical_and(eor, defectiovs.logical_not())
print('done')
print('Generating GRL...', end='')
data = DQUtils.grl.make_grl(okiovs, '', '2.1')
with open(fname, 'w') as outf:
outf.write(data)
def fetch_busy_runs(folder,
since=runlumi(0),
until=runlumi(2**63 - 1),
cs=cool.ChannelSelection(),
ntop=5):
"""
`folder` to query
`since` start time (defaults to 0)
`until` until (defaults to iovmax)
`cs` channelselection (defaults to all)
`ntop` number of the busyiest runs to retrieve (defaults to 5)
"""
result = []
print "Fetching iovs.."
iovs = split_by_channel(fetch_iovs(folder, since, until, cs, loud=True))
print "Tallying by channel.."
run_counts = {}
for channel, iovs in iovs.iteritems():
run_counts[channel] = iov_count_per_run(iovs)
print "Getting busiest.."
for channel, runs_iov_counts in run_counts.iteritems():
run_counts[channel] = sorted(runs_iov_counts.items(),
key=lambda (run, count): (count, -run),
reverse=True)[:ntop]
return sorted(run_counts.iteritems())
def makeGRL(run, defect, fname):
from ROOT import TString
import DQUtils, DQDefects
import os
tag = 'HEAD'
runs = [run]
print 'Query run information...',
from DQUtils.db import fetch_iovs
dbinstance = 'CONDBR2'
eor = fetch_iovs('EOR', (min(runs) << 32) | 1,
(max(runs) << 32) | 0xFFFFFFFF,
with_channel=False, what=[], database='COOLONL_TDAQ/%s' % dbinstance)
eor = eor.trim_iovs
eor = DQUtils.IOVSet(iov for iov in eor if iov.since.run in runs)
print 'done'
print 'Query defects...',
ddb = DQDefects.DefectsDB('COOLOFL_GLOBAL/%s' % dbinstance, tag=tag)
ignores = set([_ for _ in ddb.defect_names if 'UNCHECKED' in _])
try:
defectiovs = ddb.retrieve(since = min(runs) << 32 | 1,
until = max(runs) << 32 | 0xffffffff,
channels = [defect],
evaluate_full = False,
ignore=ignores)
except Exception, e:
print e
raise
def get_runends():
from DQUtils.db import fetch_iovs
from DQUtils.sugar import RunLumi
eor_params = fetch_iovs('EOR_Params',
with_channel=False, what=[])
runends = {}
for iov in eor_params:
if iov.since.lumi == 0: iov = iov._replace(since=RunLumi(iov.since.run, 1))
runends[iov.since.run] = iov
return runends
def atlas_notready_filter(indb):
from DQUtils.db import fetch_iovs
folder = indb.getFolder('/TDAQ/RunCtrl/DataTakingMode')
iovs = fetch_iovs(folder, channels=[0])
return filter(None, [iov for iov in iovs if iov.ReadyForPhysics == 0])
def dq_code_filter_internals(indb):
folder = indb.getFolder(foldername)
iovs = fetch_iovs(folder, channels=[channel], tag=tag)
iovs = [iov for iov in iovs if iov.Code in codes]
return iovs
continue
#print defect, folder.channelDescription(cdict[defect])
try:
print db.new_virtual_defect(defect, defect, ''), defect
except DefectExistsError:
if opts.ignoreold:
print 'Defect', defect, 'already exists; ignoring'
else:
raise
print 'Populating old virtual flags'
folder = Databases.get_folder(opts.origvfdb, read_only=True)
ids, names, cdict = get_channel_ids_names(folder)
iovs = fetch_iovs(opts.origvfdb, (opts.vfrun, 1), (opts.vfrun, 2),
tag=opts.vftag,
with_channel=True)
for defect in iovs:
clause = defect.expression
clause = clause.replace('worst_of(', '')
clause = clause.replace(')', '')
clause = clause.replace(', ', ' ')
clause = clause.replace(',', ' ')
dname = cdict[defect.channel]
try:
print db.new_virtual_defect(dname, defect.comment,
clause), dname, ':', clause
except DefectExistsError:
if opts.ignoreold:
print 'Defect', dname, 'already exists; ignoring'
def go(iov, systems, db, indb, timewise=False):
"""
Run the DCS calculator for `run` on the `systems` specified, saving the
result to the `database`.
"""
since, until = iov
with timer("Read LBLB"):
# fetch lumi block info
lblb = fetch_iovs("LBLB",
since,
until,
with_channel=False,
database='COOLONL_TRIGGER/%s' % indb)
assert lblb, "No luminosity blocks for desired range. [%s, %s)" % (
since, until)
# lumi block time info
lbtime = inverse_lblb(lblb)
# run_iovs is....?
if timewise:
run_iovs = run_iovs_from_lblb(lbtime)
else:
run_iovs = run_iovs_from_lblb(lblb)
log.debug("Will process %i LBs over %i runs", len(lblb), len(run_iovs))
log.info("DCSC2 will run over range: %r %s", tuple(run_iovs.range_iov),
"(%i lbs)" % len(lblb))
# Initialize the system objects
# This changes 'systems' from a list of types to a list of subdetector objects
systems = [
system() for system in systems
if not getattr(system, "__DISABLED__", False)
]
# Run systems sequentially for this range
result_iovs = run_sequential(systems, lbtime, run_iovs)
# Run parallel is broken because of the use of closures to implement the
# defect translation.
#result_iovs = run_parallel(systems, lbtime, run_iovs, parallel)
if log.isEnabledFor(logging.DEBUG):
pprint_objects(result_iovs)
log.info("Calculation complete")
if db != "None":
with timer("write result (%i iovs)" % len(result_iovs)):
log.debug("Writing result (%i iovs)", len(result_iovs))
defect_iovs = filter(lambda iov: isinstance(iov, DefectIOV),
result_iovs)
dcsofl_iovs = filter(lambda iov: not isinstance(iov, DefectIOV),
result_iovs)
if len(defect_iovs) > 0:
ddb = DefectsDB(db, read_only=False, create=True)
defect_names = set(i.channel for i in defect_iovs)
for defect in defect_names:
if defect in ddb.defect_id_map:
continue
ddb.create_defect(defect, "Created by DCSCalculator2")
with ddb.storage_buffer:
for iov in defect_iovs:
ddb.insert(iov.channel, iov.since, iov.until,
iov.comment, 'sys:defectcalculator',
iov.present)
#disable DCSOFL
#dest = "%s::/GLOBAL/DETSTATUS/DCSOFL" % db
#write_iovs(dest, dcsofl_iovs, dcsofl_cool_record(), create=True)
args = len(result_iovs), hash(result_iovs)
log.info("Success. Calculated %i iovs. Result hash: 0x%0x8." % args)
def getNumLumiBlocks(run):
from DQUtils.db import fetch_iovs
lbs = fetch_iovs('LBLB', (run, 1), (run, 0xFFFFFFFF), database='oracle://ATLAS_COOLPROD;dbname=CONDBR2;schema=ATLAS_COOLONL_TRIGGER')
return len(lbs)