def test_reject_duplicate_names():
ddb = DefectsDB(TEST_DATABASE, read_only=False)
ddb.create_defect('A_TEST_DEFECT', 'A test defect')
wasthrown = False
try:
ddb.create_defect('A_TEST_DEFECT', 'A test defect 2')
except DefectExistsError:
wasthrown = True
assert wasthrown, 'Creation of duplicate defects should throw'
wasthrown = False
try:
ddb.create_defect('A_test_defect', 'A test defect 2')
except DefectExistsError:
wasthrown = True
assert wasthrown, 'Creation of duplicate defects should throw, even with different case'
if line == '' or line[0] == '#':
continue
linep = line.partition(' ')
comment = linep[2].strip()
if comment == '':
print 'Missing comment:', line
sys.exit(1)
defect = linep[0].strip()
if not good_defect_regex.match(defect):
print 'Invalid defect name:', defect
sys.exit(1)
tocreate.append((defect, comment))
#print '\n'.join(map(str,tocreate))
from DQDefects import DefectsDB
from DQDefects.exceptions import DefectExistsError
#from DQUtils import init_logger
#init_logger(2)
ddb = DefectsDB(opts.db, create=opts.create, read_only=False)
print 'Now creating defects on', opts.db
for defect, comment in tocreate:
try:
print 'Creating', defect, comment
ddb.create_defect(defect, comment)
except DefectExistsError:
if opts.ignoreold:
print 'Defect', defect, 'already exists; ignoring'
else:
raise
class IDBSDefectWriter:
"""
Class for writing BS defects to an sqlite file
"""
def __init__(self,
fileName,
forceNew=False,
dbName='IDBSDQ',
tag='nominal',
user='******'):
"""
Initialise database connection
"""
self.defect = None
self.iovs = IOVSet()
self.user = user
#self.allowedDefects = DefectsDB('').defect_names
if not fileName: fileName = 'tmp.' + str(os.getpid()) + '.db'
self.connect(fileName, forceNew, dbName, tag)
pass
def __del__(self):
"""
Delete db to clear connection
"""
os.system('[[ -f tmp.%s.db ]] && rm tmp.%s.db' %
(os.getpid(), os.getpid()))
del self.db
pass
def connect(self,
fileName,
forceNew=False,
dbName='IDBSDQ',
tag='nominal'):
"""
Open connection to defect DB
"""
connString = 'sqlite://;schema=%s;dbname=%s' % (fileName, dbName)
if forceNew and os.path.exists(fileName):
os.remove(fileName)
self.db = DefectsDB(
connString, read_only=False, create=True, tag=(tag, 'HEAD')
) # Second tag is for virtual defects, which we are not interested in
self.officialDb = DefectsDB()
return
def defectType(self, t):
"""
Set defect type
"""
self.defect = t
def add(self,
runMin=0,
runMax=(1 << 31) - 1,
lbMin=0,
lbMax=(1 << 32) - 1):
"""
Add iovs which are NOT defective to the list
Note, lbMax is exclusive here (and inclusive when shown on defect web page).
"""
# Make since and until and convert to syntactially nice RunLumiType
since = RunLumiType((runMin << 32) + lbMin)
until = RunLumiType((runMax << 32) + lbMax)
# IoVs which are not defective (i.e. beamspot good)
self.iovs.add(since, until, self.defect, False)
return
def complete(self, runMin, runMax):
"""
Complete a list of IoVs to cover all LBs in a run, treating empty ones as having 'emptyState'
"""
# Create an IOV set covering the entire run(s)
run_lbs = fetch_iovs("EOR",
runs=(runMin, runMax),
what=[],
with_channel=False)
# run_lbs = IOVSet()
# lbMin = 1
# lbMax = (1 << 32) -1 # Note, lbMax is exclusive
# since = RunLumiType((runMin << 32)+lbMin)
# until = RunLumiType((runMax << 32)+lbMax)
# run_lbs.add(since, until)
if not len(run_lbs):
print "WARNING: No LBs in run according to EOR_Params - are we running before the run has finished?"
def lbsStartAtOne(iov):
"Change LBs starting at 0 to start at 1"
return iov._replace(since=RunLumi(iov.since.run, 1))
# Start LBs from 1 rather than 0 (at request of P. Onyisi) as long as run has more than one LB (else skip)
run_lbs = [lbsStartAtOne(iov) for iov in run_lbs if iov.until.lumi > 1]
# Empty IOV set for adding full list of LBs too
iovs = IOVSet()
# Ask official DB if an IoV is currently defective so can unset only those if doing reprocessing
defectsCurrentlyInDb = self.officialDb.retrieve(
(runMin, 0), (runMax + 1, 0), [self.defect])
# Order IOVs to avoid since > until
self.iovs = IOVSet(sorted(self.iovs))
#for since, until, (run_lb, iov, dbDefect) in process_iovs(run_lbs.solidify(RANGEIOV_VAL), self.iovs.solidify(DEFECTIOV_VAL), defectsCurrentlyInDb):
for since, until, (run_lb, iov, dbDefect) in process_iovs(
run_lbs, self.iovs.solidify(DEFECTIOV_VAL),
defectsCurrentlyInDb):
if not run_lb: continue # Check valid
# # Start LBs from 1 rather than 0 (at request of P. Onyisi)
# # If this makes since==until, i.e. it was a [0->1) LB range then skip
# if since.lumi==0:
# since = RunLumiType((since.run << 32)+since.lumi+1)
# if since==until: continue
# Add iovs from self.iovs treating empty ones as defective (i.e. beamspot bad/missing)
if iov:
# These are not defective
if dbDefect and dbDefect.present:
# Only store not defective IoVs if they are present on the Db so we can unset them
# (don't want to write not present defects to Db unless really chaning an existing defect)
iovs.add(since, until, self.defect, False)
else:
# These are defective
iovs.add(since, until, self.defect, True)
self.iovs = iovs
return
def writeDefects(self, tag='nominal', nonpresent=False):
"""
Write all defects to the database. If 'nonpresent' is True then write the absent ones too
"""
with self.db.storage_buffer:
for since, until, defect, present in self.iovs:
if not present and not nonpresent: continue
#print since, until, present
self._writeDefect(defect, since, until, present=present)
def _writeDefect(self,
defect,
since,
until,
tag='nominal',
description='',
comment='',
present=True,
recoverable=False):
"""
Write a single defect to the database
"""
if not defect in self.db.defect_names:
self.db.create_defect(defect, description)
self.db.insert(defect, since, until, comment, self.user, present,
recoverable)
return
def dump(self, filename=None):
"""
Dump defects to a file given by filename or stdout if no filename given
"""
from DQUtils.utils import pprint_objects
if filename is not None:
f = open(filename.replace('.db', '.txt'), "w")
# If not defects then nothing will be in the database and we write an empty file
if len(self.iovs):
pprint_objects(
self.db.retrieve(primary_only=True, nonpresent=True), f)
f.close()
else:
if len(self.iovs):
self.iovs.pprint()
else:
print '\nNo DQ defects'
# with open(filename.replace('.db', '.txt'), "w") as f:
# pprint_objects(self.db.retrieve(), f)
return
print 'Reading in IOVs...'
iniovs = indb.retrieve(since=since,
until=until,
channels=channels,
primary_only=True,
nonpresent=(not opts.noabsent))
print '%d IOVs retrieved from input database' % len(iniovs)
#inchannels = set((x.channel for x in iniovs))
inchannels = set(channels if channels is not None else indb.defect_names)
outchannels = set(outdb.defect_names)
missingchannels = inchannels - outchannels
if len(missingchannels) != 0:
if not opts.createdefects:
print 'Missing defect(s) in target database:'
print list(missingchannels)
print 'Rerun with a restricted defect list (--defects) or with --createdefects'
sys.exit(1)
else:
print 'Creating missing channels on output database'
descriptions = indb.get_channel_descriptions(missingchannels)
for channel in missingchannels:
outdb.create_defect(channel, descriptions[channel])
with outdb.storage_buffer:
print 'Writing out IOVs...'
for iov in iniovs:
outdb.insert(iov.channel, iov.since, iov.until, iov.comment,
iov.user, iov.present, iov.recoverable)
print 'Done.'
def go(iov, systems, db, indb, timewise=False):
"""
Run the DCS calculator for `run` on the `systems` specified, saving the
result to the `database`.
"""
since, until = iov
with timer("Read LBLB"):
# fetch lumi block info
lblb = fetch_iovs("LBLB",
since,
until,
with_channel=False,
database='COOLONL_TRIGGER/%s' % indb)
assert lblb, "No luminosity blocks for desired range. [%s, %s)" % (
since, until)
# lumi block time info
lbtime = inverse_lblb(lblb)
# run_iovs is....?
if timewise:
run_iovs = run_iovs_from_lblb(lbtime)
else:
run_iovs = run_iovs_from_lblb(lblb)
log.debug("Will process %i LBs over %i runs", len(lblb), len(run_iovs))
log.info("DCSC2 will run over range: %r %s", tuple(run_iovs.range_iov),
"(%i lbs)" % len(lblb))
# Initialize the system objects
# This changes 'systems' from a list of types to a list of subdetector objects
systems = [
system() for system in systems
if not getattr(system, "__DISABLED__", False)
]
# Run systems sequentially for this range
result_iovs = run_sequential(systems, lbtime, run_iovs)
# Run parallel is broken because of the use of closures to implement the
# defect translation.
#result_iovs = run_parallel(systems, lbtime, run_iovs, parallel)
if log.isEnabledFor(logging.DEBUG):
pprint_objects(result_iovs)
log.info("Calculation complete")
if db != "None":
with timer("write result (%i iovs)" % len(result_iovs)):
log.debug("Writing result (%i iovs)", len(result_iovs))
defect_iovs = filter(lambda iov: isinstance(iov, DefectIOV),
result_iovs)
dcsofl_iovs = filter(lambda iov: not isinstance(iov, DefectIOV),
result_iovs)
if len(defect_iovs) > 0:
ddb = DefectsDB(db, read_only=False, create=True)
defect_names = set(i.channel for i in defect_iovs)
for defect in defect_names:
if defect in ddb.defect_id_map:
continue
ddb.create_defect(defect, "Created by DCSCalculator2")
with ddb.storage_buffer:
for iov in defect_iovs:
ddb.insert(iov.channel, iov.since, iov.until,
iov.comment, 'sys:defectcalculator',
iov.present)
#disable DCSOFL
#dest = "%s::/GLOBAL/DETSTATUS/DCSOFL" % db
#write_iovs(dest, dcsofl_iovs, dcsofl_cool_record(), create=True)
args = len(result_iovs), hash(result_iovs)
log.info("Success. Calculated %i iovs. Result hash: 0x%0x8." % args)
spec = cool.RecordSpecification()
#spec.extend("filename", cool.StorageType.String255)
#spec.extend("guid", cool.StorageType.Int32)
spec.extend("Code", cool.StorageType.Int32)
spec.extend("deadFrac", cool.StorageType.Float)
spec.extend("Thrust", cool.StorageType.Float)
print "Creating folder", folderName,
# Deprecated/dropped: folder = db.createFolder(folderName, spec, "", cool.FolderVersioning.MULTI_VERSION, False)
folderSpec = cool.FolderSpecification(cool.FolderVersioning.MULTI_VERSION,
spec)
folder = db.createFolder(folderName, folderSpec, "", False)
print "... Done"
print "Creating channels."
channelDict = getChannelDict()
for channelName in channelDict.keys():
try:
folder.createChannel(channelDict[channelName], channelName)
print "Created channel:", channelName
except:
print "Channel", channelName, "already exists."
db.closeDatabase()
# create defects
from DQDefects import DefectsDB
ddb = DefectsDB(connectString, create=True, read_only=False)
chandb = DefectsDB('COOLOFL_GLOBAL/CONDBR2')
for chan in chandb.defect_names:
ddb.create_defect(chan, chandb.get_channel_descriptions([chan]))
