def main():
runs = 185640, 185660
lblb = fetch_iovs("LBLB", runs=runs)
lbtime = inverse_lblb(lblb)
beam_iovs = fetch_iovs("COOLOFL_DCS::/LHC/DCS/FILLSTATE", lbtime.first.since, lbtime.last.until)
stable_beam_by_lb = {}
for since_time, until_time, (lumiblock, beamstate) in process_iovs(lbtime, beam_iovs):
if not lumiblock:
# Not inside lumiblock
continue
lb = RunLumi(lumiblock.Run, lumiblock.LumiBlock)
if lb in stable_beam_by_lb:
# Require stable beam for ALL of the lumiblock (hence 'and')
# Replace with 'or' to get ANY of the lumiblock
stable_beam_by_lb[lb] = stable_beam_by_lb[lb] and beamstate.StableBeams
else:
stable_beam_by_lb[lb] = beamstate.StableBeams
@define_iov_type
def STABLEBEAM_VAL(stable):
"""
Define an IOV type which just has .stable
"""
result = (STABLEBEAM_VAL(runlb, runlb+1, state)
for runlb, state in sorted(stable_beam_by_lb.iteritems()))
result = IOVSet(connect_adjacent_iovs(result))
result.pprint()
def fetch_lumi_inputs(range_iov, tag="OflLumi-7TeV-002"):
"""
Retrieve information required to calculate the luminosity from the database
"""
lbs = fetch_iovs(
"LBLB",
with_channel=False, #loud=True,
*range_iov)
if tag == "ONLINE":
lumis = fetch_iovs(
"COOLONL_TRIGGER::/TRIGGER/LUMI/OnlPrefLumi",
channels=[0], #loud=True,
*range_iov)
# Select valid
lumis = IOVSet(l for l in lumis if (l.Valid & 0xFFFF) % 10 == 0)
else:
# Apparently this string changes from time to time...
lumi_folder = "COOLOFL_TRIGGER::/TRIGGER/OFLLUMI/OflPrefLumi" # Run-2
#lumi_folder = "COOLOFL_TRIGGER::/TRIGGER/OFLLUMI/LBLESTOFL" # Run-1
lumis = fetch_iovs(
lumi_folder,
tag=tag,
channels=[0], #loud=True,
*range_iov)
return lbs, lumis
def diff_folder(iov_range, folder, tag_1, tag_2, interesting_channels):
other_args = dict(named_channels=True)
iovs_1 = fetch_iovs(folder, tag=tag_1, *iov_range, **other_args).by_channel
iovs_2 = fetch_iovs(folder, tag=tag_2, *iov_range, **other_args).by_channel
all_channels = set(iovs_1.keys() + iovs_2.keys())
for channel in sorted(all_channels):
if channel not in interesting_channels:
continue
print "Considering", channel
iovs_1_c = iovs_1.get(channel, IOVSet())
iovs_2_c = iovs_2.get(channel, IOVSet())
diff_channel(channel, iovs_1_c, iovs_2_c)
def main():
from DCSCalculator2.subdetectors import CSC, MDT
lbtime = fetch_iovs("LBTIME", (177540, 0))
print "lbtime =", lbtime
show_variable_info(lbtime, MDT)
def test_processing():
"""
Test that processing works
"""
iovs = fetch_iovs("SHIFTOFL", tag="DetStatusSHIFTOFL-pass1-analysis-2010B")
channels, iovsets = iovs.chans_iovsets
processed = [(since, until, states[:])
for since, until, states in process_iovs(*iovsets)]
return processed
def test_read():
"""
Check if retrieving the Code and Comment field from DetStatusSHIFTOFL-pass1-analysis-2010B changed
Reads some data from the database which is expected not to change.
A problem with this could either indicate a broken API, or a changed database
"""
tag = "DetStatusSHIFTOFL-pass1-analysis-2010B"
result = fetch_iovs("SHIFTOFL", what=["Code", "Comment"], tag=tag)
return result
def main():
init_logger(verbose=2)
# Query the last four days of information
start_time = datetime.now() - timedelta(days=2)
with timer("Query DQMFONL"):
iovs = fetch_iovs("DQMFONL", start_time)
log.info("Got iovs from DQMFONL: %r", iovs)
with timer("Query LBTIME"):
lbiovs = fetch_iovs("LBTIME", start_time)
chans, states = iovs.chans_iovsets
with timer("run build_table_data"):
data = build_table_data(lbiovs, *states)
with timer("run table_data_to_html"):
table = table_data_to_html(chans, data)
print table
def _get_virtual_defect_logics(self, tag):
if tag != "HEAD" and not self.defect_logic_folder.existsUserTag(tag):
# The tag doesn't exist, so there is nothing to fetch.
return {}
logics = fetch_iovs(self.defect_logic_folder, tag=tag,
named_channels=True)
logics = dict((l.channel, DefectLogic(l)) for l in logics)
for defect_name, defect_logic in logics.iteritems():
defect_logic._populate(logics)
return logics
def test_serializer():
"""
Try serializing a fairly complicated iovset object
"""
iovs = fetch_iovs("SHIFTOFL", runs=152166)
channels, iovsets = iovs.chans_iovsets
processed = [(since, until, states)
for since, until, states in process_iovs(*iovsets)]
serialized = dumps(processed).encode("bz2").encode("base64")
deserialized = loads(serialized.decode("base64").decode("bz2"))
assert deserialized == processed
def test():
path = "/afs/cern.ch/user/b/beate/public/DQAna/StableBeams-periodC1.xml"
grl_iovs = grl_iovs_from_xml(path)
since, until = grl_iovs.range_iov
dcs_iovs = fetch_iovs("DCSOFL", since, until, named_channels=True)
# Instead of "dead_fraction", "n_in_config" can be used.
result = dcs_summary(grl_iovs, dcs_iovs, "dead_fraction")
for channel, states in sorted(result.iteritems()):
print channel
for run, info in states:
print "", run, "Low: % .3f Ave: % .3f Hi: % .3f" % (
info.lowest, info.average, info.highest)
def get_trigger_channel(trigger, iovrange):
"""
Returns the set of iovs for which the trigger is valid
"""
raise NotImplementedError("Implementation incomplete")
# First check just the first run.
iovs = fetch_iovs(LVL1_MENU, runs=iovrange.since.run, what=["ItemName"])
result = [i for i in iovs if i.ItemName == trigger]
if not result:
raise UnknownTrigger()
assert len(result) == 1, "This should not happen."
channel = result[0].channel
iovs = fetch_iovs(LVL1_MENU,
channels=[channel],
what=["ItemName"],
*iovrange)
return iovs.select(ItemName == trigger)
def calculate_idbs(run_range):
# List of lumiblocks to run over
lbtime = inverse_lblb(fetch_iovs("LBLB", runs=run_range))
idbs = IDBS()
idbs.input_db = "COOLOFL_INDET/CONDBR2"
result_iovs = idbs.run(lbtime)
print "Will write the following:"
pprint_objects(result_iovs)
for entry in result_iovs:
print entry.since.run, entry.since.lumi, "->", entry.until.lumi, entry.Code
write_iovs("idbs_test.db/CONDBR2::/GLOBAL/DETSTATUS/DCSOFL", result_iovs,
dcsofl_cool_record())
def read(self, query_range, folder_base, folder_name):
"""
Read the relevant data from COOL for this variable
"""
if folder_name.startswith("/"):
folder_path = folder_name
else:
# For relative folders prepend the folder_base
folder_path = "/".join((folder_base, folder_name))
log.info("Querying COOL folder %s", folder_path)
if config.opts.check_input_time:
self.fetch_args["with_time"] = True
# Massage DB access
if '/' in self.input_db:
newdbstring = self.input_db.rsplit('/', 1)[0]
else:
newdbstring = self.input_db
if config.opts.input_database.startswith('sqlite'):
self.fetch_args['database'] = config.opts.input_database
else:
self.fetch_args['database'] = (
'%s/%s' % (newdbstring, config.opts.input_database))
if self.fetch_args:
log.debug("Fetching with args: %r", self.fetch_args)
iovs = fetch_iovs(folder_path, *query_range, **self.fetch_args)
# Prints even when not doing debug.
# TODO: fix this. Might be broken in DQUtils.logger
#if log.isEnabledFor(logging.DEBUG):
# log.debug("Dumping input IOVs:")
# for iov in iovs:
# print iov
if config.opts.check_input_time:
self.print_time_info(iovs)
if log.isEnabledFor(logging.INFO):
input_hash = hash(iovs)
self.input_hashes.append(input_hash)
log.info(" -> Input hash: % 09x (len=%i)", input_hash, len(iovs))
return iovs
def main():
f1 = "%s::%s" % (db1, options.folder)
f2 = "%s::%s" % (db2, options.folder)
print "=" * 100
print "Comparing: "
print " * ", f1, options.tag1
print " * ", f2, options.tag2
print "=" * 100
if options.runMin is not None:
runs = options.runMin
if options.runMax is not None:
runs = options.runMin, options.runMax
tag1iovs = fetch_iovs(f1, runs=runs, tag=options.tag1)
tag2iovs = fetch_iovs(f2, runs=runs, tag=options.tag2)
affected_runs = set()
ar2 = set()
#runiovs = IOVSet()
#runiovs.add(RunLumiType((runs << 32)+0), RunLumiType((runs << 32)+(1 << 32)-1))
# Loop over iovs in the two tags
# for since, until, (t1, t2, r) in process_iovs(tag1iovs, tag2iovs, runiovs.solidify(RANGEIOV_VAL)):
for since, until, (t1, t2) in process_iovs(tag1iovs, tag2iovs):
# Ignore IoVs spanning runs
if not options.span and since.run != until.run: continue
# Make sure run exists in both
#if not t1.since or not t2.since: continue
# Only consider runs in the range interested in
# if not r: continue
# Check for IOVs with different status (one of which is non-zero)
if (t1.status == 0 or t2.status == 0) and (t1.status != t2.status):
if options.statusonly and not options.summary:
print since, until, t1.status, t2.status
ar2.add(since.run)
# Check for differences in all fields
if not options.statusonly:
for i, field in enumerate(t1._fields):
if t1[i] != t2[i]:
#if field in ['posX', 'posY'] and abs(t1[i]-t2[i]) > 0.1e-3:
#if field not in ['status'] and t1[i] != 0 and abs(t1[i]-t2[i])/t1[i] > 0.001:
#if field in ['posZ'] and t1[i] != 0 and abs(t1[i]-t2[i])/t1[i] > 0.05:
#if field in ['sigmaX', 'sigmaY'] and abs(t1[i]-t2[i]) > 0.2e-3:
#if field in ['sigmaZ'] and abs(t1[i]-t2[i])/t1[i] > 0.002:
if not options.summary:
print since, until, field, t1[i], t2[i]
affected_runs.add(since.run)
# Print results
if not options.statusonly:
print "Runs with differences (%s):" % len(affected_runs), ", ".join(
map(str, sorted(affected_runs)))
print "Runs with different status (and one non-zero):", ", ".join(
map(str, sorted(ar2)))
lb_length = fin.Get('%s/GLOBAL/DQTGlobalWZFinder/m_lblength_lb' % runname)
lbmin, lbmax = lb_length.GetXaxis().GetXmin(), lb_length.GetXaxis().GetXmax()
logging.info(
'Starting livetime lookup ... (remove when we have a proper in-file implementation ...)'
)
livetime = ROOT.TProfile('livetime', 'Livetime', int(lbmax - lbmin), lbmin,
lbmax)
official_lum = ROOT.TProfile('official_lum', 'official integrated luminosity',
int(lbmax - lbmin), lbmin, lbmax)
official_lum_zero = ROOT.TProfile('official_lum_zero',
'official inst luminosity',
int(lbmax - lbmin), lbmin, lbmax)
from DQUtils import fetch_iovs
from DQUtils.iov_arrangement import inverse_lblb
lblb = fetch_iovs("LBLB", runs=int(runname[4:]))
lbtime = inverse_lblb(lblb)
#print list(lbtime)
iovs_acct = fetch_iovs('COOLOFL_TRIGGER::/TRIGGER/OFLLUMI/LumiAccounting',
lbtime.first.since,
lbtime.last.until,
tag='OflLumiAcct-13TeV-004')
#iovs_lum = fetch_iovs('COOLOFL_TRIGGER::/TRIGGER/OFLLUMI/OflPrefLumi', lblb.first.since, lblb.last.until, tag='OflLumi-13TeV-003')
#print list(iovs_lum)
for iov in iovs_acct:
if not lbmin < iov.LumiBlock < lbmax:
continue
livetime.Fill(iov.LumiBlock, iov.LiveFraction)
#print iov.InstLumi, iovs_lum[iov.LumiBlock-1].LBAvInstLumi
official_lum_zero.Fill(iov.LumiBlock, iov.InstLumi / 1e3)
#! /usr/bin/env python
from IPython.Shell import IPShellEmbed; ip = IPShellEmbed(["-pdb"])
from DCSCalculator2.subdetectors.tgc import TGC, DCSC_Variable_TGC_HV, TGC_Database, get_mapping
from DQUtils import fetch_iovs
from DQUtils.iov_arrangement import inverse_lblb
from itertools import chain
runs = 165732, 165733
iovs = inverse_lblb(fetch_iovs("LBLB", runs=runs))
start, end = iovs.first.since, iovs.last.until
TGC_data = DCSC_Variable_TGC_HV(None, None)
TGC_data.db = TGC_Database()
mapping = get_mapping(TGC_data.db)
TGC_data.all_channels = set(chain(*mapping.values()))
tgc_conditions = TGC_data.read_tgc_conditions(start.date, end.date)
tgc_channels_nonempty = set(iov.channel for iov in tgc_conditions if iov)
print "Total IoVs for query range:", len(tgc_conditions)
print "Channels with IoV data:", len(tgc_conditions.channels)
print "Channels with nonempty IoV data:", len(tgc_channels_nonempty)
print "Channels in mapping table:", len(TGC_data.all_channels)
def test_fetch_single_detstatus_channel():
result = fetch_iovs("SHIFTOFL", channels=["SCTEA"])
print "Result length:", len(result)
def test_fetch_single_non_detstatus_channel():
result = fetch_iovs("VIRTUALFLAGS", channels=["CP_TRACKING"])
print "Result length:", len(result)
tl.Branch('z_one', o_z_one, 'z_one/F')
tl.Branch('z_two', o_z_two, 'z_two/F')
tl.Branch('trigeff', o_trigeff, 'trigeff/F')
tl.Branch('trigeffstat', o_trigeffstat, 'trigeffstat/F')
tl.Branch('recoeff', o_recoeff, 'recoeff/F')
tl.Branch('recoeffstat', o_recoeffstat, 'recoeffstat/F')
tl.Branch('alleff', o_alleff, 'alleff/F')
tl.Branch('alleffstat', o_alleffstat, 'alleffstat/F')
tl.Branch('ae', o_ae, 'ae/F')
tl.Branch('aestat', o_aestat, 'aestat/F')
from DQUtils import fetch_iovs
#rset=set(_[0] for _ in rlb)
#print rset
lblb = fetch_iovs("LBLB", runs=int(runname[4:])).by_run
for lb in sorted(lbdirs):
if runmode == "Zee":
h = infile.Get('%s/%s/GLOBAL/DQTGlobalWZFinder/m_eltrigtp_matches' % (runname, lb))
hmo = infile.Get('%s/%s/GLOBAL/DQTGlobalWZFinder/m_ele_tight_good_os' % (runname, lb))
hms = infile.Get('%s/%s/GLOBAL/DQTGlobalWZFinder/m_ele_tight_good_ss' % (runname, lb))
hno = infile.Get('%s/%s/GLOBAL/DQTGlobalWZFinder/m_ele_tight_bad_os' % (runname, lb))
hns = infile.Get('%s/%s/GLOBAL/DQTGlobalWZFinder/m_ele_tight_bad_ss' % (runname, lb))
if runmode == "Zmumu":
h = infile.Get('%s/%s/GLOBAL/DQTGlobalWZFinder/m_mutrigtp_matches' % (runname, lb))
hmo = infile.Get('%s/%s/GLOBAL/DQTGlobalWZFinder/m_muloosetp_match_os' % (runname, lb))
hms = infile.Get('%s/%s/GLOBAL/DQTGlobalWZFinder/m_muloosetp_match_ss' % (runname, lb))
hno = infile.Get('%s/%s/GLOBAL/DQTGlobalWZFinder/m_muloosetp_nomatch_os' % (runname, lb))
hns = infile.Get('%s/%s/GLOBAL/DQTGlobalWZFinder/m_muloosetp_nomatch_ss' % (runname, lb))
lbnum = int(lb[3:])
yld = (h[2], h[3])
def retrieve(self, since=None, until=None, channels=None, nonpresent=False,
primary_only=False, ignore=None,
with_primary_dependencies=False, intersect=False,
with_time=False, evaluate_full=True):
"""
Retrieve defects from the database.
Parameters:
`since`, `until` : IoV range to query (Default: All)
`channels` : A list of channels to query. Can contain a mixture of
defect names and ids. (Default: None means all
channels, including all virtual)
`nonpresent` : Only return IoVs which are currently "present"
`primary_only` : Only return primary IoVs, no virtual ones.
`ignore` : Set of defects which won't be treated as bad.
`with_primary_dependencies` : When querying virtual flags, also get
primary flags which went into the calculation.
`intersect` : Intersect the result with the query range so that no
iov spans outside the query range
`with_time` : Also retrieves the time the defect was inserted
~2x slower queries, doesn't work for virtual defects
`evaluate_full` : If specified, also compute the `comment` and
`recoverable` fields of virtual defects.
Causes a ~0.6x slowdown
"""
if ignore is not None and not isinstance(ignore, set):
assert False, "ignore parameter should be set type"
desired_channels = None
# Figure out the IDs of channels to query and their virtuality
if channels is not None:
query_channels = set(self.defect_names_as_ids(channels))
virtual_channels = query_channels - self.defect_ids
primary_channels = query_channels & self.defect_ids
else:
# Empty channels list means query all channels
# (including all virtual)
if primary_only:
virtual_channels = None
else:
virtual_channels = self.virtual_defect_ids
primary_channels = sorted(self.defect_ids)
query_channels = None # (all)
primary_output_names = [self.defect_id_map[pid] for pid in primary_channels]
virtual_output_names = [] # (optionally populated below)
# Resolve virtual channels to query here, and the primary dependents.
if virtual_channels:
assert not primary_only, "Requested virtual channels with primary_only=True"
assert not with_time, "with_time flag only works for primary defects"
virtual_output_names = [self.virtual_defect_id_map[vid]
for vid in virtual_channels]
ordered_logics = self._resolve_evaluation_order(virtual_output_names)
if channels is not None:
# Since not all channels are being queried, it is necessary to
# add the desired primary channels to the query
primary_needed = self.resolve_primary_defects(ordered_logics)
primary_channels |= set(self.defect_names_as_ids(primary_needed))
query_channels = primary_channels
if with_primary_dependencies:
primary_output_names.extend(sorted(primary_needed))
for logic in ordered_logics:
logic.set_evaluation(evaluate_full)
# Figure out if the set of channels will produce too many ranges for COOL
filter_channels = None
if query_channels is not None:
query_channels = sorted(query_channels)
query_ranges = list_to_channelselection(query_channels, None, True)
if len(query_ranges) >= 50:
# We're querying too many ranges. Query everything, filter later.
desired_channels = set(primary_output_names + virtual_output_names)
query_channels = None # (everything)
# Retrieve primary IoVs
primary_iovs = fetch_iovs(self.defects_folder, since, until,
query_channels, self.defects_tag,
named_channels=True, unicode_strings=True,
with_time=with_time)
# Calculate virtual defects (if necessary)
if primary_only or not virtual_channels:
result = primary_iovs
else:
if not primary_iovs:
return IOVSet()
args = (primary_iovs, ordered_logics,
virtual_output_names, primary_output_names,
since, until, ignore)
result = calculate_virtual_defects(*args)
# Filter out results which have their present bit removed
# (unless otherwise specified)
if not nonpresent:
result = IOVSet(iov for iov in result if iov.present)
# Filter out channels which weren't queried by the user
# (to get around 50 channelselection COOL limit)
if desired_channels:
result = IOVSet(iov for iov in result
if iov.channel in desired_channels)
if intersect:
result = result.intersect_range((since, until))
return result
def complete(self, runMin, runMax):
"""
Complete a list of IoVs to cover all LBs in a run, treating empty ones as having 'emptyState'
"""
# Create an IOV set covering the entire run(s)
run_lbs = fetch_iovs("EOR",
runs=(runMin, runMax),
what=[],
with_channel=False)
# run_lbs = IOVSet()
# lbMin = 1
# lbMax = (1 << 32) -1 # Note, lbMax is exclusive
# since = RunLumiType((runMin << 32)+lbMin)
# until = RunLumiType((runMax << 32)+lbMax)
# run_lbs.add(since, until)
if not len(run_lbs):
print "WARNING: No LBs in run according to EOR_Params - are we running before the run has finished?"
def lbsStartAtOne(iov):
"Change LBs starting at 0 to start at 1"
return iov._replace(since=RunLumi(iov.since.run, 1))
# Start LBs from 1 rather than 0 (at request of P. Onyisi) as long as run has more than one LB (else skip)
run_lbs = [lbsStartAtOne(iov) for iov in run_lbs if iov.until.lumi > 1]
# Empty IOV set for adding full list of LBs too
iovs = IOVSet()
# Ask official DB if an IoV is currently defective so can unset only those if doing reprocessing
defectsCurrentlyInDb = self.officialDb.retrieve(
(runMin, 0), (runMax + 1, 0), [self.defect])
# Order IOVs to avoid since > until
self.iovs = IOVSet(sorted(self.iovs))
#for since, until, (run_lb, iov, dbDefect) in process_iovs(run_lbs.solidify(RANGEIOV_VAL), self.iovs.solidify(DEFECTIOV_VAL), defectsCurrentlyInDb):
for since, until, (run_lb, iov, dbDefect) in process_iovs(
run_lbs, self.iovs.solidify(DEFECTIOV_VAL),
defectsCurrentlyInDb):
if not run_lb: continue # Check valid
# # Start LBs from 1 rather than 0 (at request of P. Onyisi)
# # If this makes since==until, i.e. it was a [0->1) LB range then skip
# if since.lumi==0:
# since = RunLumiType((since.run << 32)+since.lumi+1)
# if since==until: continue
# Add iovs from self.iovs treating empty ones as defective (i.e. beamspot bad/missing)
if iov:
# These are not defective
if dbDefect and dbDefect.present:
# Only store not defective IoVs if they are present on the Db so we can unset them
# (don't want to write not present defects to Db unless really chaning an existing defect)
iovs.add(since, until, self.defect, False)
else:
# These are defective
iovs.add(since, until, self.defect, True)
self.iovs = iovs
return
def test_fetch():
fetch_iovs("DCSOFL", runs=152166)
fetch_iovs("COOLOFL_DCS::/LHC/DCS/FILLSTATE", runs=152166)
fetch_iovs("COOLOFL_DCS/COMP200::/LHC/DCS/FILLSTATE", runs=152166)
