def test_reject_duplicate_names(): ddb = DefectsDB(TEST_DATABASE, read_only=False) ddb.create_defect('A_TEST_DEFECT', 'A test defect') wasthrown = False try: ddb.create_defect('A_TEST_DEFECT', 'A test defect 2') except DefectExistsError: wasthrown = True assert wasthrown, 'Creation of duplicate defects should throw' wasthrown = False try: ddb.create_defect('A_test_defect', 'A test defect 2') except DefectExistsError: wasthrown = True assert wasthrown, 'Creation of duplicate defects should throw, even with different case'
if line == '' or line[0] == '#': continue linep = line.partition(' ') comment = linep[2].strip() if comment == '': print 'Missing comment:', line sys.exit(1) defect = linep[0].strip() if not good_defect_regex.match(defect): print 'Invalid defect name:', defect sys.exit(1) tocreate.append((defect, comment)) #print '\n'.join(map(str,tocreate)) from DQDefects import DefectsDB from DQDefects.exceptions import DefectExistsError #from DQUtils import init_logger #init_logger(2) ddb = DefectsDB(opts.db, create=opts.create, read_only=False) print 'Now creating defects on', opts.db for defect, comment in tocreate: try: print 'Creating', defect, comment ddb.create_defect(defect, comment) except DefectExistsError: if opts.ignoreold: print 'Defect', defect, 'already exists; ignoring' else: raise
class IDBSDefectWriter: """ Class for writing BS defects to an sqlite file """ def __init__(self, fileName, forceNew=False, dbName='IDBSDQ', tag='nominal', user='******'): """ Initialise database connection """ self.defect = None self.iovs = IOVSet() self.user = user #self.allowedDefects = DefectsDB('').defect_names if not fileName: fileName = 'tmp.' + str(os.getpid()) + '.db' self.connect(fileName, forceNew, dbName, tag) pass def __del__(self): """ Delete db to clear connection """ os.system('[[ -f tmp.%s.db ]] && rm tmp.%s.db' % (os.getpid(), os.getpid())) del self.db pass def connect(self, fileName, forceNew=False, dbName='IDBSDQ', tag='nominal'): """ Open connection to defect DB """ connString = 'sqlite://;schema=%s;dbname=%s' % (fileName, dbName) if forceNew and os.path.exists(fileName): os.remove(fileName) self.db = DefectsDB( connString, read_only=False, create=True, tag=(tag, 'HEAD') ) # Second tag is for virtual defects, which we are not interested in self.officialDb = DefectsDB() return def defectType(self, t): """ Set defect type """ self.defect = t def add(self, runMin=0, runMax=(1 << 31) - 1, lbMin=0, lbMax=(1 << 32) - 1): """ Add iovs which are NOT defective to the list Note, lbMax is exclusive here (and inclusive when shown on defect web page). """ # Make since and until and convert to syntactially nice RunLumiType since = RunLumiType((runMin << 32) + lbMin) until = RunLumiType((runMax << 32) + lbMax) # IoVs which are not defective (i.e. beamspot good) self.iovs.add(since, until, self.defect, False) return def complete(self, runMin, runMax): """ Complete a list of IoVs to cover all LBs in a run, treating empty ones as having 'emptyState' """ # Create an IOV set covering the entire run(s) run_lbs = fetch_iovs("EOR", runs=(runMin, runMax), what=[], with_channel=False) # run_lbs = IOVSet() # lbMin = 1 # lbMax = (1 << 32) -1 # Note, lbMax is exclusive # since = RunLumiType((runMin << 32)+lbMin) # until = RunLumiType((runMax << 32)+lbMax) # run_lbs.add(since, until) if not len(run_lbs): print "WARNING: No LBs in run according to EOR_Params - are we running before the run has finished?" def lbsStartAtOne(iov): "Change LBs starting at 0 to start at 1" return iov._replace(since=RunLumi(iov.since.run, 1)) # Start LBs from 1 rather than 0 (at request of P. Onyisi) as long as run has more than one LB (else skip) run_lbs = [lbsStartAtOne(iov) for iov in run_lbs if iov.until.lumi > 1] # Empty IOV set for adding full list of LBs too iovs = IOVSet() # Ask official DB if an IoV is currently defective so can unset only those if doing reprocessing defectsCurrentlyInDb = self.officialDb.retrieve( (runMin, 0), (runMax + 1, 0), [self.defect]) # Order IOVs to avoid since > until self.iovs = IOVSet(sorted(self.iovs)) #for since, until, (run_lb, iov, dbDefect) in process_iovs(run_lbs.solidify(RANGEIOV_VAL), self.iovs.solidify(DEFECTIOV_VAL), defectsCurrentlyInDb): for since, until, (run_lb, iov, dbDefect) in process_iovs( run_lbs, self.iovs.solidify(DEFECTIOV_VAL), defectsCurrentlyInDb): if not run_lb: continue # Check valid # # Start LBs from 1 rather than 0 (at request of P. Onyisi) # # If this makes since==until, i.e. it was a [0->1) LB range then skip # if since.lumi==0: # since = RunLumiType((since.run << 32)+since.lumi+1) # if since==until: continue # Add iovs from self.iovs treating empty ones as defective (i.e. beamspot bad/missing) if iov: # These are not defective if dbDefect and dbDefect.present: # Only store not defective IoVs if they are present on the Db so we can unset them # (don't want to write not present defects to Db unless really chaning an existing defect) iovs.add(since, until, self.defect, False) else: # These are defective iovs.add(since, until, self.defect, True) self.iovs = iovs return def writeDefects(self, tag='nominal', nonpresent=False): """ Write all defects to the database. If 'nonpresent' is True then write the absent ones too """ with self.db.storage_buffer: for since, until, defect, present in self.iovs: if not present and not nonpresent: continue #print since, until, present self._writeDefect(defect, since, until, present=present) def _writeDefect(self, defect, since, until, tag='nominal', description='', comment='', present=True, recoverable=False): """ Write a single defect to the database """ if not defect in self.db.defect_names: self.db.create_defect(defect, description) self.db.insert(defect, since, until, comment, self.user, present, recoverable) return def dump(self, filename=None): """ Dump defects to a file given by filename or stdout if no filename given """ from DQUtils.utils import pprint_objects if filename is not None: f = open(filename.replace('.db', '.txt'), "w") # If not defects then nothing will be in the database and we write an empty file if len(self.iovs): pprint_objects( self.db.retrieve(primary_only=True, nonpresent=True), f) f.close() else: if len(self.iovs): self.iovs.pprint() else: print '\nNo DQ defects' # with open(filename.replace('.db', '.txt'), "w") as f: # pprint_objects(self.db.retrieve(), f) return
print 'Reading in IOVs...' iniovs = indb.retrieve(since=since, until=until, channels=channels, primary_only=True, nonpresent=(not opts.noabsent)) print '%d IOVs retrieved from input database' % len(iniovs) #inchannels = set((x.channel for x in iniovs)) inchannels = set(channels if channels is not None else indb.defect_names) outchannels = set(outdb.defect_names) missingchannels = inchannels - outchannels if len(missingchannels) != 0: if not opts.createdefects: print 'Missing defect(s) in target database:' print list(missingchannels) print 'Rerun with a restricted defect list (--defects) or with --createdefects' sys.exit(1) else: print 'Creating missing channels on output database' descriptions = indb.get_channel_descriptions(missingchannels) for channel in missingchannels: outdb.create_defect(channel, descriptions[channel]) with outdb.storage_buffer: print 'Writing out IOVs...' for iov in iniovs: outdb.insert(iov.channel, iov.since, iov.until, iov.comment, iov.user, iov.present, iov.recoverable) print 'Done.'
def go(iov, systems, db, indb, timewise=False): """ Run the DCS calculator for `run` on the `systems` specified, saving the result to the `database`. """ since, until = iov with timer("Read LBLB"): # fetch lumi block info lblb = fetch_iovs("LBLB", since, until, with_channel=False, database='COOLONL_TRIGGER/%s' % indb) assert lblb, "No luminosity blocks for desired range. [%s, %s)" % ( since, until) # lumi block time info lbtime = inverse_lblb(lblb) # run_iovs is....? if timewise: run_iovs = run_iovs_from_lblb(lbtime) else: run_iovs = run_iovs_from_lblb(lblb) log.debug("Will process %i LBs over %i runs", len(lblb), len(run_iovs)) log.info("DCSC2 will run over range: %r %s", tuple(run_iovs.range_iov), "(%i lbs)" % len(lblb)) # Initialize the system objects # This changes 'systems' from a list of types to a list of subdetector objects systems = [ system() for system in systems if not getattr(system, "__DISABLED__", False) ] # Run systems sequentially for this range result_iovs = run_sequential(systems, lbtime, run_iovs) # Run parallel is broken because of the use of closures to implement the # defect translation. #result_iovs = run_parallel(systems, lbtime, run_iovs, parallel) if log.isEnabledFor(logging.DEBUG): pprint_objects(result_iovs) log.info("Calculation complete") if db != "None": with timer("write result (%i iovs)" % len(result_iovs)): log.debug("Writing result (%i iovs)", len(result_iovs)) defect_iovs = filter(lambda iov: isinstance(iov, DefectIOV), result_iovs) dcsofl_iovs = filter(lambda iov: not isinstance(iov, DefectIOV), result_iovs) if len(defect_iovs) > 0: ddb = DefectsDB(db, read_only=False, create=True) defect_names = set(i.channel for i in defect_iovs) for defect in defect_names: if defect in ddb.defect_id_map: continue ddb.create_defect(defect, "Created by DCSCalculator2") with ddb.storage_buffer: for iov in defect_iovs: ddb.insert(iov.channel, iov.since, iov.until, iov.comment, 'sys:defectcalculator', iov.present) #disable DCSOFL #dest = "%s::/GLOBAL/DETSTATUS/DCSOFL" % db #write_iovs(dest, dcsofl_iovs, dcsofl_cool_record(), create=True) args = len(result_iovs), hash(result_iovs) log.info("Success. Calculated %i iovs. Result hash: 0x%0x8." % args)
spec = cool.RecordSpecification() #spec.extend("filename", cool.StorageType.String255) #spec.extend("guid", cool.StorageType.Int32) spec.extend("Code", cool.StorageType.Int32) spec.extend("deadFrac", cool.StorageType.Float) spec.extend("Thrust", cool.StorageType.Float) print "Creating folder", folderName, # Deprecated/dropped: folder = db.createFolder(folderName, spec, "", cool.FolderVersioning.MULTI_VERSION, False) folderSpec = cool.FolderSpecification(cool.FolderVersioning.MULTI_VERSION, spec) folder = db.createFolder(folderName, folderSpec, "", False) print "... Done" print "Creating channels." channelDict = getChannelDict() for channelName in channelDict.keys(): try: folder.createChannel(channelDict[channelName], channelName) print "Created channel:", channelName except: print "Channel", channelName, "already exists." db.closeDatabase() # create defects from DQDefects import DefectsDB ddb = DefectsDB(connectString, create=True, read_only=False) chandb = DefectsDB('COOLOFL_GLOBAL/CONDBR2') for chan in chandb.defect_names: ddb.create_defect(chan, chandb.get_channel_descriptions([chan]))