def tdaq_busy_generator(self, iovs):
events = process_iovs(iovs)
counter = 0
for since, until, (state, ) in events:
if state.Run == 0: continue
#print state
if state is not None:
deadfrac = 1 - state.LiveFraction
if deadfrac < self.deadfraction_threshold:
continue
yield DefectIOV(RunLumi(state.Run, state.LumiBlock),
RunLumi(state.Run, state.LumiBlock + 1),
'GLOBAL_BUSY',
True,
comment='Average live fraction %.1f%%' %
((1 - deadfrac) * 100))
counter += 1
counter_max = counter
counter = 0
#print counter_max
events = process_iovs(iovs)
for since, until, (state, ) in events:
if state is not None:
deadfrac = 1 - state.LiveFraction
if deadfrac < self.deadfraction_threshold:
continue
#print state.Run
counter += 1
if state.Run == 0 and counter < counter_max:
print 'ERROR: Wrong run number in LumiAccounting; here is the IOV: '
print state
#print list(iovs)
continue
def main():
since, until = RunLumi(152000, 0), RunLumi(155000, 0)
#since, until = RunLumi(152777, 0), RunLumi(152778, 0) - 1
S = make_run_selector(atlas_runs_set())
lest_iovs = S(fetch_iovs("LBLESTONL", since, until, [0]))
lblb_iovs = S(fetch_iovs("LBLB", since, until))
eor_iovs = S(fetch_iovs("EOR", since, until))
rfp_iovs = S(
fetch_iovs("/TDAQ/RunCtrl/DataTakingMode",
since,
until,
database="tdaq"))
args = lest_iovs, lblb_iovs, eor_iovs, rfp_iovs
ptagil_iovs = make_ptag_integrated_lumi_iovs(*args)
state_iovs = fetch_iovs("SHIFTOFL", since, until,
named_channels=True) # [111, 114, 115, 121])
result = accumulate_result(ptagil_iovs, state_iovs)
from pprint import pprint
pprint(result["data10_7TeV"])
def main():
run = 144247 # Recent run where CSC gas state was bad.
run = 136379
run = 155112 # Recent long run (Sat May 15th, 11am -> Sun 16th 8am), 632 lbs
run = 156463
#run = 155669 # MDT NConfig goes below 600
from sys import argv
args = argv[1:]
if args:
run = int(args[0])
since, until = RunLumi(run, 0), RunLumi(run + 1, 0)
with timer("Read LBLB"):
lblb = fetch_iovs("LBLB", since, until, with_channel=False)
lbtime = inverse_lblb(lblb)
#system = SCT() # Works, very slightly discrepant for the config.
#system = Pixels() # Works
#system = Tile() # Works
#system = CSC() # Needs globals fixing
#system = TDQ() # Works, needs further testing
#system = RPC() # Works
#system = LAr() # Works, but produces slightly different result. Original gives:
# DCSOFL_VAL(since=(155112: 167), until=(155112: 176), channel=215, Code=2L)
#system = TRT() # Unresolved discrepancies: Wrong NConfig, NWorking off by one in run 155112.
system = MDT() # Seems to work
#system = Magnets() # Needs to be finished
#system = Lucid() # Needs to be written
#print system.mapping.keys()
with timer("Calculate result for %s" % system):
result = system.run(lbtime)
print "Done."
print
#print result[0]
print "Run range:", since, "to", until
print "DCSC2 intervals of validity:"
pprint_objects(result[:30])
print
output_channels = system.mapping.keys()
iovs = fetch_iovs("DCSOFL", since, until, output_channels)
#print hex(hash(tuple(iovs))), hex(hash(tuple(result)))
print "Original DCSC intervals of validity:"
pprint_objects(iovs[:30])
return
compare_iovs(iovs, result)
def main(argv):
optp, opts, args = config.parse_options(argv)
init_logger(opts.verbose)
log.info("Using %s" % get_package_version("DQUtils"))
log.info("Using %s" % get_package_version("DQDefects"))
log.info("Using %s" % get_package_version("DCSCalculator2"))
log.debug("Commandline arguments: %s", argv)
log.debug("Current configuration: %s" % (opts))
if opts.shell_on_exception:
import sys
from IPython.core import ultratb
sys.excepthook = ultratb.FormattedTB(call_pdb=True)
#from IPython.Shell import IPShellEmbed
#ipython_instance = IPShellEmbed(["-pdb"], rc_override=dict(quiet=True))
if opts.systems is None:
systems = ALL_SYSTEMS
else:
systems = []
invalid_systems = []
for system in opts.systems:
if system not in SYSTEM_MAP:
invalid_systems.append(system)
else:
if system != "Pixels":
systems.append(SYSTEM_MAP[system])
if invalid_systems:
optp.error("Invalid system(s) specified: {0}. "
"Available systems: {1}".format(invalid_systems,
SYSTEM_MAP.keys()))
if (opts.run and opts.range) or (not opts.run and not opts.range):
optp.error("Specify either -r or -R")
elif opts.run:
since, until = opts.run, opts.run
elif opts.range:
since, until = map(int, opts.range.split("-"))
if opts.lbs:
if opts.range:
optp.error("--lbs and --range are incompatible. "
"Use --lbs with --run")
sincelb, untillb = map(int, opts.lbs.split("-"))
iov = RunLumi(since, sincelb), RunLumi(until, untillb)
else:
iov = RunLumi(since, 0), RunLumi(until + 1, 0)
go(iov, systems, opts.output_database, opts.input_database, opts.timewise)
def emittance_generator(self, iovs):
events = process_iovs(iovs)
for since, until, (state, ) in events:
#print state, state.RunLB & 0xffffffff if state.RunLB else 0
if state is not None and state.RunLB is not None:
thisrun = state.RunLB >> 32
# pseudo-LB and not to be trusted
if thisrun == 0: continue
thisLB = state.RunLB & 0xffffffff
yield DefectIOV(RunLumi(thisrun, thisLB),
RunLumi(thisrun, thisLB + 1),
'LUMI_EMITTANCESCAN',
True,
comment='Emittance scan')
def main():
runs = 185640, 185660
lblb = fetch_iovs("LBLB", runs=runs)
lbtime = inverse_lblb(lblb)
beam_iovs = fetch_iovs("COOLOFL_DCS::/LHC/DCS/FILLSTATE", lbtime.first.since, lbtime.last.until)
stable_beam_by_lb = {}
for since_time, until_time, (lumiblock, beamstate) in process_iovs(lbtime, beam_iovs):
if not lumiblock:
# Not inside lumiblock
continue
lb = RunLumi(lumiblock.Run, lumiblock.LumiBlock)
if lb in stable_beam_by_lb:
# Require stable beam for ALL of the lumiblock (hence 'and')
# Replace with 'or' to get ANY of the lumiblock
stable_beam_by_lb[lb] = stable_beam_by_lb[lb] and beamstate.StableBeams
else:
stable_beam_by_lb[lb] = beamstate.StableBeams
@define_iov_type
def STABLEBEAM_VAL(stable):
"""
Define an IOV type which just has .stable
"""
result = (STABLEBEAM_VAL(runlb, runlb+1, state)
for runlb, state in sorted(stable_beam_by_lb.iteritems()))
result = IOVSet(connect_adjacent_iovs(result))
result.pprint()
def calculate_virtual_defects(primary_iovs, evaluation_order,
virtual_output_channels, primary_output_channels,
since, until, ignore):
"""
Returns a list of IoVs for a given query in the normal COOL order
(sorted by channelID, then by since)
"""
if since == None: since = 0
if until == None: until = 2**63 - 1
since, until = RunLumi(since), RunLumi(until)
result = defaultdict(IOVSet)
primary_by_channel = primary_iovs.by_channel
# Copy desired primary channels to the result
for primary_channel, primary_iovs in primary_by_channel.iteritems():
if primary_channel in primary_output_channels:
if ignore and primary_channel in ignore:
continue
result[primary_channel] = primary_iovs
args = primary_by_channel, evaluation_order, since, until, ignore
# Skip vfgen loop if there is no virtual logic to compute
vfgen = [] if not evaluation_order else generate_virtual_defects(*args)
for since, until, virtual_iovs in vfgen:
if bad_iov(since, until):
continue
for output_name in virtual_output_channels:
# Consider the state of each desired output for this
# (since, until) and write it to output_iovs
iov = virtual_iovs.get(output_name)
if iov and iov.present:
result[output_name].add(since, until, *iov[2:])
# Sort them by traditional COOL sort ordering (by channelId first,
# then by since. `iovs` are already ordered by since.)
result_list = IOVSet()
for channel, iovs in sorted(result.iteritems()):
result_list.extend(iovs.solidify(DEFECT_IOV))
return result_list
def calculate_good_iovs(self, lbtime, subdetector):
"""
Calculate LB-wise "good" states
"""
self.subdetector = subdetector
if self.timewise_folder:
query_range = RANGEIOV_VAL(lbtime.first.since, lbtime.last.until)
else:
a, b = lbtime.first, lbtime.last
query_range = RANGEIOV_VAL(RunLumi(a.Run, a.LumiBlock),
RunLumi(b.Run, b.LumiBlock))
# Read the database
iovs = self.read(query_range, subdetector.folder_base,
self.folder_name)
#iovs.pprint()
# Decide the states of the input iovs
iovs = self.make_good_iovs(iovs)
#iovs.pprint()
# Apply a mapping for input channels if necessary
# This only does something for variables that require additional mapping
# i.e., if the channel numbers for different DCS variables don't match up
iovs = self.map_input_channels(iovs)
if self.timewise_folder and not config.opts.timewise:
# we might already know the defect mapping
with timer("Quantize %s (%i iovs over %i lbs)" %
(self.folder_name, len(iovs), len(lbtime))):
# Quantize to luminosity block
iovs = self.quantize(lbtime, iovs)
#iovs.pprint()
# Debug printout of problematic channels
# DQUtils messes with the logging and isEnabledFor doesn't work
#if log.isEnabledFor(logging.DEBUG):
#log.verbose("Bad input channels for %s:", self.folder_name)
#log.verbose("= [%r]", ", ".join(str(i.channel) for i in iovs if not i.good))
self.iovs = iovs
return self
def parse_runlumi(instr):
if instr is None: return None
import re
from DQUtils.sugar import RunLumi
m = re.match('\((\d+),(\d+)\)', instr.replace(' ', ''))
if m is None:
raise ValueError('Unable to parse ' + instr +
'; please specify the limits in the form (run, lb)')
return RunLumi(int(m.group(1)), int(m.group(2)))
def get_runends():
from DQUtils.db import fetch_iovs
from DQUtils.sugar import RunLumi
eor_params = fetch_iovs('EOR_Params',
with_channel=False, what=[])
runends = {}
for iov in eor_params:
if iov.since.lumi == 0: iov = iov._replace(since=RunLumi(iov.since.run, 1))
runends[iov.since.run] = iov
return runends
def tdaq_busy_generator(self, iovs):
events = process_iovs(iovs)
counter = 0
for since, until, (state, ) in events:
if state.Run == 0 or state.Run is None: continue
if state is not None:
if state.LiveFraction is None:
deadfrac = 1
log.warning('WARNING: LiveFraction is "None" for %d %d',
state.Run, state.LumiBlock)
else:
deadfrac = 1 - state.LiveFraction
if deadfrac < self.deadfraction_threshold:
continue
yield DefectIOV(RunLumi(state.Run, state.LumiBlock),
RunLumi(state.Run, state.LumiBlock + 1),
'GLOBAL_BUSY',
True,
comment='Average live fraction %.1f%%' %
((1 - deadfrac) * 100))
counter += 1
counter_max = counter
counter = 0
events = process_iovs(iovs)
for since, until, (state, ) in events:
if state is not None and state.Run is not None:
deadfrac = 1 - state.LiveFraction
if deadfrac < self.deadfraction_threshold:
continue
counter += 1
if state.Run == 0 and counter < counter_max:
log.error(
'ERROR: Wrong run number in LumiAccounting; here is the IOV: '
)
log.error(state)
continue
def lbsStartAtOne(iov):
"Change LBs starting at 0 to start at 1"
return iov._replace(since=RunLumi(iov.since.run, 1))
def main():
import sys
from argparse import ArgumentParser, FileType
p = ArgumentParser(description="Compute differences in the defects "
"database between tags")
arg = p.add_argument
arg("tags",
metavar="DetStatus-Tag",
nargs=2,
type=str,
help="Two tags to show the difference between")
arg("-p",
"--project",
default=None,
help="Limit considered runs to those in project")
arg("-P", "--period", default=None, help="Data period")
arg("-r", "--run", type=int, help="Run number to process")
arg("-R", "--range", help="Run range: e.g. 150000-151000")
arg("-d",
"--defect",
action="append",
dest="defects",
default=None,
help="Defects to show the difference between. "
"(If empty, show all. -d can be specified multiple times.)")
arg("-i",
"--ignore-not-considered",
action="store_true",
help="Ignore ranges in which GLOBAL_NOTCONSIDERED is set for either tag"
)
arg("--output",
default=sys.stdout,
type=FileType("w"),
help="Place to out the output (Default stdout)")
arg("-s",
"--by-system",
action="store_true",
help="If specified, show systems which differ, not defects")
arg("-v",
"--virtual",
action="store_true",
help="If specified, show virtual defect differences as well as primary"
" (produces a lot of output)")
args = p.parse_args()
# tags
tag1, tag2 = args.tags
# range to process
since, until = None, None
#since, until = (200804, 0), (200804, 50)
range_iovs = IOVSet()
# if specifying run, only use that run, and it's easy
if args.run:
since, until = (args.run, 0), (args.run + 1, 0)
range_iovs = IOVSet.from_runs([args.run])
elif args.range:
run1, run2 = map(int, args.range.split("-"))
since, until = (run1, 0), (run2, 0)
range_iovs = IOVSet([RANGEIOV_VAL(RunLumi(since), RunLumi(until))])
# if specifying project, use that to build the range_iovs
# TODO: what if period is specified but not project?
# shouldn't I define a default project in that case?
elif args.project:
# fetch the project:period:runs dictionary
project_dict = fetch_project_period_runs()
period_dict = project_dict[args.project]
# if specifying period, use those runs
runs = set()
if args.period:
runs = set(period_dict[args.period])
else:
#runs = set([ run for run in runs for (period, runs) in period_dict.iteritems() ])
for period, period_runs in period_dict.iteritems():
runs.update(period_runs)
since, until = (min(runs), 0), (max(runs) + 1, 0)
range_iovs = IOVSet.from_runs(runs)
# debugging
print 'range to process:'
print 'since, until =', since, until
#print 'range_iovs:'
#range_iovs.pprint()
# arguments for fetching defects
kwargs = dict(primary_only=not args.virtual,
channels=args.defects,
since=since,
until=until)
def fetch_defects(tag):
iovs = make_defectset_iovs(range_iovs,
DefectsDB(tag=tag).retrieve(**kwargs))
return iovs
print "Retrieving defects.."
defects1, defects2 = fetch_defects(tag1), fetch_defects(tag2)
#defects1.pprint()
print "Computing diff.."
diff_iovs = make_defectdiff_iovs(defects1, defects2, args.by_system,
args.ignore_not_considered)
for since, until, t1, t2 in diff_iovs:
print since, until, "tag1:(%s)" % pretty(t1), "tag2:(%s)" % pretty(t2)
def main():
parser = ArgumentParser(description='List defect IOVs')
add_arg = parser.add_argument
# Specify range to process
add_arg('-p',
'--project',
default='data15_13TeV',
help='Data project (default: data15_13TeV)')
add_arg('-P', '--period', default=None, nargs='*', help='Data period(s)')
#add_arg('-r', '--run', default=None, nargs='*', help='Run number(s) to process')
add_arg('-r',
'--run',
default=None,
type=int,
help='Run number to process')
add_arg('-R', '--range', help='Inclusive run range: e.g. 150000-151000')
# Specify defects to process
add_arg('-d',
'--defects',
default=None,
nargs='*',
help='Defects to process. Use * for wildcard (default: None)')
# Other job options
add_arg('-c',
'--connection-string',
default=DEFAULT_CONNECTION_STRING,
help='Database connection to use (default: %s)' %
DEFAULT_CONNECTION_STRING)
#add_arg('-l', '--lumi-tag', default='OflLumi-8TeV-001',
#help='Luminosity tag (default: OflLumi-8TeV-001)')
add_arg('-t', '--tag', default='HEAD', help='Tag to use (default: HEAD)')
# Parse arguments
args = parser.parse_args()
# Ranges to query
since, until = None, None
range_iovs = IOVSet()
# If specifying runs, use those
if args.run:
since, until = (args.run, 0), (args.run + 1, 0)
range_iovs = IOVSet.from_runs([args.run])
elif args.range:
run1, run2 = map(int, args.range.split('-'))
since, until = (run1, 0), (run2, 0)
range_iovs = IOVSet([RANGEIOV_VAL(RunLumi(since), RunLumi(until))])
# Otherwise, use the project and period settings
elif args.project:
# Fetch the project:period:runs dictionary
project_dict = get_project_dict()
period_dict = project_dict[args.project]
run_set = set()
# Use periods if specified
if args.period and len(args.period) > 0:
for period in args.period:
run_set.update(period_dict[period])
# Otherwise use all periods in project
else:
for period, runs in period_dict.iteritems():
run_set.update(runs)
since, until = (min(run_set), 0), (max(run_set) + 1, 0)
range_iovs = IOVSet.from_runs(run_set)
# debugging
print 'range to process:'
print 'since, until =', since, until
# Instantiate the DB
db = DefectsDB(args.connection_string, tag=args.tag)
# Fetch the requested defect IOVs
defect_iovs = db.retrieve(since, until, channels=args.defects)
defect_iovset_list = defect_iovs.by_channel.values()
# Get the overlap with the range_iovs
# Use a dictionary of defect:iovset
result_dict = defaultdict(IOVSet)
for since, until, states in process_iovs(range_iovs, *defect_iovset_list):
in_range, defects = states[0], states[1:]
if in_range:
for d in defects:
if d:
result_dict[d.channel].add(since, until, d.channel,
d.comment)
# Print the results
for channel, result in result_dict.iteritems():
result.solidify(DefectIOV)
print '\n' + channel + '\n'
result.pprint()