def defineFolderSpec( isMultiVersion ): recordSpec = cool.RecordSpecification() recordSpec.extend( "Offset", ST.Float ) folderSpec = cool.FolderSpecification( FV.MULTI_VERSION if isMultiVersion else FV.SINGLE_VERSION, recordSpec ) recordSpec2 = cool.RecordSpecification() recordSpec2.extend( "Noise", ST.Blob64k ) folderSpec2 = cool.FolderSpecification( FV.MULTI_VERSION if isMultiVersion else FV.SINGLE_VERSION, recordSpec2 ) return ( folderSpec, folderSpec2 )
def createRPCConditionDB(): import os from PyCool import cool # get database service and open database dbSvc = cool.DatabaseSvcFactory.databaseService() # database accessed via physical name dbstring = 'sqlite://;schema=RPCConditionDB.db;dbname=RPC_DQA' if os.access('RPCConditionDB.db', os.R_OK): try: os.unlink('RPCConditionDB.db') except Exception as e: print('Unable to remove existing db file, reason', e) try: db = dbSvc.createDatabase(dbstring) except Exception as e: print('Problem creating database', e) return print("Created database", dbstring) spec2 = cool.RecordSpecification() spec2.extend("PanelRes", cool.StorageType.String255) spec2.extend("StripStatus", cool.StorageType.String4k) desc = "<timeStamp>run-lumi</timeStamp><addrHeader><address_header service_type=\"71\" clid=\"1238547719\" /></addrHeader><typeName>CondAttrListCollection</typeName>" # create the folder - single version # last argument is createParents - if true, automatically creates parent # folders if needed # note this will not work if the database already exists - delete mycool.db first # Deprecated/dropped: myfolder=db.createFolder('/OFFLINE/FINAL',spec2,desc,cool.FolderVersioning.MULTI_VERSION,True) folderSpec = cool.FolderSpecification(cool.FolderVersioning.MULTI_VERSION, spec2) db.createFolder('/OFFLINE/FINAL', folderSpec, desc, True) db.closeDatabase()
def createSqlite(sqliteName,folderName="/TDAQ/OLC/LHC/FILLPARAMS"): dbSvc = cool.DatabaseSvcFactory.databaseService() if os.access(sqliteName,os.R_OK): print ("UPDATING existing sqlite file %s" % sqliteName) db=dbSvc.openDatabase("sqlite://;schema="+sqliteName+";dbname=CONDBR2",False) else: print("Creating new sqlite file %s" % sqliteName) db=dbSvc.createDatabase("sqlite://;schema="+sqliteName+";dbname=CONDBR2") pass spec = cool.RecordSpecification() spec.extend("Beam1Bunches",cool.StorageType.UInt32) spec.extend("Beam2Bunches",cool.StorageType.UInt32) spec.extend("LuminousBunches",cool.StorageType.UInt32) spec.extend("BCIDmasks",cool.StorageType.Blob64k) descr='<timeStamp>time</timeStamp><addrHeader><address_header service_type="71" clid="40774348" /></addrHeader><typeName>AthenaAttributeList</typeName>' if db.existsFolder(folderName): folder=db.getFolder(folderName) else: print("Creating COOL folder/tag %s" % (folderName)) #folder = db.createFolder(folderName, spec, desc, cool.FolderVersioning.MULTI_VERSION, True) folderSpec = cool.FolderSpecification(cool.FolderVersioning.SINGLE_VERSION, spec) folder = db.createFolder(folderName, folderSpec, descr, True) pass return db,folder
def store_PID_to_folder(db, values_list): """ Function to create folder FOLDER_NAME in db with channels and input data from values_list with tag TAG_NAME :param db: Open cool db :param values_list: list of tuples ('channel name', valus array) """ spec = cool.RecordSpecification() spec.extend('array_value', cool.StorageType.Float) _logger.info('Create folder %s' % FOLDER_NAME) folder_spec = cool.FolderSpecification(cool.FolderVersioning.MULTI_VERSION, spec, cool.PayloadMode.VECTORPAYLOAD) folder = db.createFolder( FOLDER_NAME, folder_spec, ' <timeStamp>run-lumi</timeStamp><addrHeader><address_header service_type="71" clid="55403898"/></addrHeader><typeName>CondAttrListVec</typeName>', True) for index, channel_name in enumerate(values_list): folder.createChannel(index, channel_name[0]) print '%i - %s' % (index, channel_name[0]) folder.setupStorageBuffer() for index, channel_values in enumerate(values_list): _logger.info('Store data from %s' % channel_values[0]) values = channel_values[1] # print channel_values[0] # print channel_values[1] vector = cool.IRecordVector() for value in values: # print value data = cool.PyCool.Helpers.IRecordPtr(spec) data.get()['array_value'] = value vector.push_back(data) folder.storeObject(0, cool.ValidityKeyMax, vector, index, TAG_NAME) folder.flushStorageBuffer()
def build_folder(cls, db, folder_name, multiversion, record): """ Create `folder_name` on database instance `db`, with recordspecification `record`. Also creates folderset to which folder_name belongs if necessary. """ from PyCool import cool folderset_path = dirname(folder_name) try: db.getFolderSet(folderset_path) except Exception as error: caught_error = "Folder set %s not found" % folderset_path if caught_error not in error.args[0]: raise log.debug("Folderset doesn't exist - creating it.") db.createFolderSet(folderset_path, "", True) if not isinstance(record, cool.RecordSpecification): record_spec = cool.RecordSpecification() for field in record: record_spec.extend(*field) else: record_spec = record FV = cool.FolderVersioning versioning = FV.MULTI_VERSION if multiversion else FV.SINGLE_VERSION folder_spec = cool.FolderSpecification(versioning, record_spec) folder = db.createFolder(folder_name, folder_spec) payload = cool.Record(record_spec) return folder, payload
def defineFolderSpecTypeName(Name, Type, isMultiVersion): recordSpec = cool.RecordSpecification() # recordSpec.extend( "Maps", ST.Float ) recordSpec.extend(Name, Type) folderSpec = cool.FolderSpecification( FV.MULTI_VERSION if isMultiVersion else FV.SINGLE_VERSION, recordSpec) return (folderSpec)
def createFolder(db, name): desc = '<timeStamp>run-lumi</timeStamp><addrHeader><address_header service_type="71" clid="1238547719" /></addrHeader><typeName>CondAttrListCollection</typeName>' spec = cool.RecordSpecification() spec.extend('LBAvInstLumi', cool.StorageType.Float) spec.extend('Valid', cool.StorageType.UInt32) folderSpec = cool.FolderSpecification(cool.FolderVersioning.MULTI_VERSION, spec) #return db.createFolder(name,spec,desc,cool.FolderVersioning.MULTI_VERSION, True) return db.createFolder(name, folderSpec, desc, True)
def __init__(self, db, folderPath, caloBlobType=None, isMultiVersionFolder=True, isRunLumiTimeStamp=True): """ Input: - db : db should be an open database connection - folderPath: full folder path to create or update """ #=== initialize base class CaloCondLogger.__init__(self, "CaloBlobWriter") #=== store db self.__db = db #=== determine folder mode folderMode = cool.FolderVersioning.MULTI_VERSION if not isMultiVersionFolder: folderMode = cool.FolderVersioning.SINGLE_VERSION #=== determine folder description folderDescr = getAthenaFolderDescr() #=== access folder in db try: #=== create folder if it does not exist if self.__db.existsFolder(folderPath): self.__folder = self.__db.getFolder(folderPath) #=== check folder mode modeInCool = self.__folder.versioningMode() if modeInCool != folderMode: str = "Incompatible folder mode detected, COOL folder has type " if modeInCool == cool.FolderVersioning.MULTI_VERSION: str += "MULTI" else: str += "SINGLE" raise Exception(str) else: #=== create folder if it does not exist payloadSpec = cool.RecordSpecification() payloadSpec.extend('CaloCondBlob16M', cool.StorageType.Blob16M) folderSpec = cool.FolderSpecification(folderMode, payloadSpec) self.__folder = db.createFolder(folderPath, folderSpec, folderDescr, True) except Exception as e: self.log().critical(e) raise #=== initialize channel dictionaries self.__chanDictRecord = {} # <int, cool.Record > self.__chanDictCells = {} # <int, CaloCondBlobFlt>
def _create_defect_logic_folder(self, db): """ Creates the COOL database/folder for virtual defect logic, if they don't exist. Internal - use create=True in the constructor to create the COOL folder. """ log.info("Creating defect logic folder") record_spec = cool.RecordSpecification() record_spec.extend("clauses", ST.String4k) folder_spec = cool.FolderSpecification(FV.MULTI_VERSION, record_spec) return db.createFolder(DEFECT_LOGIC_FOLDER, folder_spec, DEFECT_LOGIC_FOLDER_DESC, True)
def _create_defects_folder(self, db): """ Creates the COOL database/folder for defects, if they don't exist. Internal - use create=True in the constructor to create the COOL folder. """ log.info("Creating defects folder") record_spec = cool.RecordSpecification() record_spec.extend("present", ST.Bool) record_spec.extend("recoverable", ST.Bool) record_spec.extend("user", ST.String255) record_spec.extend("comment", ST.String255) folder_spec = cool.FolderSpecification(FV.MULTI_VERSION, record_spec) return db.createFolder(DEFECTS_FOLDER, folder_spec, DEFECT_FOLDER_DESC, True)
def __init__(self): self.records = [] self.iovStartRun = 0 self.iovStartLumiBlock = 0 self.iovEndRun = 0 self.iovEndLumiBlock = 0 self.tag = "AFPTest-00-00-00" self.folderName = "/FWD/AFP/TEST" self.spec = cool.RecordSpecification() # self.spec.extend("data", cool.StorageType.Blob64k) self.spec.extend("data", cool.StorageType.String16M) # self.desc = '<timeStamp>run-lumi</timeStamp><addrHeader><address_header service_type="71" clid="1238547719" /></addrHeader><typeName>CondAttrListCollection</typeName>' self.desc = '<timeStamp>run-lumi</timeStamp><addrHeader><address_header service_type="71" clid="40774348" /></addrHeader><typeName>AthenaAttributeList</typeName>' self.data = cool.Record(self.spec) self.folderSpec = cool.FolderSpecification( cool.FolderVersioning.MULTI_VERSION, self.spec)
def createSqlite(sqliteName, folderName, foldertag, iovMin=cool.ValidityKeyMin, iovMax=cool.ValidityKeyMax, inputFileName=None, defaultvalue=1.0): dbSvc = cool.DatabaseSvcFactory.databaseService() if os.access(sqliteName, os.R_OK): print("UPDATING existing sqlite file", sqliteName) db = dbSvc.openDatabase( "sqlite://;schema=" + sqliteName + ";dbname=CONDBR2", False) else: print("Creating new sqlite file", sqliteName) db = dbSvc.createDatabase("sqlite://;schema=" + sqliteName + ";dbname=CONDBR2") spec = cool.RecordSpecification() spec.extend('CaloCondBlob16M', cool.StorageType.Blob16M) desc = '<timeStamp>run-lumi</timeStamp><addrHeader><address_header service_type="71" clid="40774348" /></addrHeader><typeName>AthenaAttributeList</typeName>' if db.existsFolder(folderName): folder = db.getFolder(folderName) else: print("Creating COOL folder/tag %s/%s" % (folderName, foldertag)) #folder = db.createFolder(folderName, spec, desc, cool.FolderVersioning.MULTI_VERSION, True) folderSpec = cool.FolderSpecification( cool.FolderVersioning.MULTI_VERSION, spec) folder = db.createFolder(folderName, folderSpec, desc, True) pass if inputFileName is None or len(inputFileName) == 0: print("No input given. Create empty blob") data = cool.Record(spec) else: data = CaloCondBlobWriterFromFile(spec, inputFileName, defaultvalue) print("Storing CaloCondBlob object") folder.storeObject(iovMin, iovMax, data, cool.ChannelId(0), foldertag, True) db.closeDatabase()
def ensureFolder(db, folder, spec, desc, version=cool.FolderVersioning.SINGLE_VERSION): """ Ensure folder exists, creating it if needed with given spec Return COOL folder object """ if (not db.existsFolder(folder)): print 'Attempting to create', folder, 'with description', desc try: # Deprecated/dropped: db.createFolder(folder,spec,desc,version,True) folderSpec = cool.FolderSpecification(version, spec) db.createFolder(folder, folderSpec, desc, True) print 'Folder created' except Exception, e: print e print 'Could not create folder', folder return None
def WriteSqlite(name, input_dict): UNIX2COOL = 1000000000 dbSvc = cool.DatabaseSvcFactory.databaseService() connectString = 'sqlite://;schema=' + name + ';dbname=L1CALO' # folder_name="/TRIGGER/Receivers/Factors/CalibGains" print('\nrecreating database file:', name) dbSvc.dropDatabase(connectString) db = dbSvc.createDatabase(connectString) spec = cool.RecordSpecification() spec.extend("factor", cool.StorageType.Float) spec.extend("status", cool.StorageType.UInt32) folderSpec = cool.FolderSpecification(cool.FolderVersioning.SINGLE_VERSION, spec) now = int(time.time()) since = now * UNIX2COOL # since = 0 # until = sys.maxint until = cool.ValidityKeyMax db.createFolderSet('/TRIGGER') db.createFolderSet('/TRIGGER/Receivers') db.createFolderSet('/TRIGGER/Receivers/Factors') folder_description = '<timeStamp>time</timeStamp><addrHeader><address_header service_type="71" clid="1238547719"/></addrHeader><typeName>CondAttrListCollection</typeName>' f = db.createFolder("/TRIGGER/Receivers/Factors/CalibGains", folderSpec, folder_description) print(" Now creating sqlite file for ", len(input_dict.keys()), " channels") for i in input_dict.keys(): data = cool.Record(spec) data['factor'] = input_dict[i][0] data['status'] = input_dict[i][1] f.storeObject(since, until, data, int(i, 16)) db.closeDatabase()
try: db = dbSvc.createDatabase(dbstring) except Exception, e: print 'Problem creating database', e sys.exit(-1) print "Created database", dbstring # ----------------------------------------------------------------------- # # setup a folder # ----------------------------------------------------------------------- # #spec=cool.RecordSpecification() recspec = cool.RecordSpecification() recspec.extend("version", cool.StorageType.String4k) recspec.extend("syserrors", cool.StorageType.String4k) spec = cool.FolderSpecification(cool.FolderVersioning.MULTI_VERSION, recspec, True) #spec.extend(fdspec,cool.StorageType.String16M) # folder meta-data - note for Athena this has a special meaning desc = '<timeStamp>time</timeStamp><addrHeader><address_header service_type="71" clid="1238547719" /></addrHeader><typeName>CondAttrListCollection</typeName>' ####<timeStamp>time</timeStamp><addrHeader><address_header service_type="71" clid="40774348" /></addrHeader><typeName>AthenaAttributeList</typeName> # create the folder - single version # last argument is createParents - if true, automatically creates parent folders if needed # note this will not work if the database already exists - delete mycool.db first #folder=db.createFolder(fdname,spec,desc,cool.FolderVersioning.MULTI_VERSION,True) folder = db.createFolder(fdname, spec, desc, True) folder.createChannel(0, "muonalignmenterrors",
def main(): parser = OptionParser() parser.add_option( "-c", "--connect-string", dest="connectString", type="string", help="cool::DatabaseId to use for the connection", default= "sqlite://none;schema=sqlite.db;user=none;password=none;dbname=DDDB") parser.add_option("-s", "--source", dest="source", type="string", help="directory where the files to copy to the DB are", metavar="SRCDIR") parser.add_option( "-d", "--dest", dest="dest", type="string", help= "directory, in the database, where to copy the files (default: '/')", default="/", metavar="DESTPATH") parser.add_option("-x", "--drop-db", action="store_true", dest="drop", help="drop the database before the copy (default)", default=True) parser.add_option( "-k", "--keep-db", action="store_false", dest="drop", help="keep the existing database and merge with the new files") (options, args) = parser.parse_args() if len(args) != 0 or not options.source or not options.dest: parser.error("Bad arguments, use -h for help.") #### prepare structres to create and fill the folders folderspec = cool.FolderSpecification(cool.FolderVersioning.MULTI_VERSION) #folderspec.payloadSpecification().extend("data",cool.StorageType.String255) #folderspec.payloadSpecification().extend("data",cool.StorageType.String4k) #folderspec.payloadSpecification().extend("data",cool.StorageType.String64k) folderspec.payloadSpecification().extend("data", cool.StorageType.String16M) folderdesc = "<storage_type=7>" payload = cool.Record(folderspec.payloadSpecification()) payload["data"] = "XML stuff" # dummy value, just to start #### initialize COOL dbs = cool.DatabaseSvcFactory.databaseService() if options.drop: dbs.dropDatabase(options.connectString) db = dbs.createDatabase(options.connectString) else: print db = dbs.openDatabase(options.connectString, False) excludes = ['clhep2dtd.pl'] # count files (just to know) sum = 0 for root, dirs, files in os.walk(options.source): if 'CVS' in root: # skip CVS directories continue for x in excludes: if x in files: files.remove(x) sum += len(files) num_len = len(str(sum)) # populate the database n = 0 for root, dirs, files in os.walk(options.source): # remove base path dbroot = root.replace(options.source, options.dest) dbroot = dbroot.replace('//', '/') if 'CVS' in root: # skip CVS directories continue if dbroot != "/": # it means it is not "/" if not db.existsFolderSet(dbroot): db.createFolderSet(dbroot) #else: # dbroot = "/" print dbroot for x in excludes: if x in files: files.remove(x) for f in files: n += 1 print("%" + str(num_len) + "d %" + str(num_len) + "d %s") % (n, sum - n, f) folder_path = os.path.join(dbroot, f) if db.existsFolder(folder_path): folder = db.getFolder(folder_path) else: folder = db.createFolder(folder_path, folderspec, folderdesc) xml_data = open(os.path.join(root, f)).read() fixed_data = fix_system_ids(xml_data, "conddb:" + dbroot) #fixed_data = xml_data fixed_data = fix_env_vars(fixed_data) payload["data"] = fixed_data folder.storeObject(cool.ValidityKeyMin, cool.ValidityKeyMax, payload, 0) print "Total files inserted = %d" % sum
spec2=cool.RecordSpecification() spec2.extend("recEta",cool.StorageType.String4k) spec2.extend("detEta",cool.StorageType.String4k) spec2.extend("recPhi1",cool.StorageType.String4k) spec2.extend("recPhi2",cool.StorageType.String4k) spec2.extend("detPhi1",cool.StorageType.String4k) spec2.extend("detPhi2",cool.StorageType.String4k) # folder meta-data - note for Athena this has a special meaning desc="<timeStamp>run-lumi</timeStamp><addrHeader><address_header service_type=\"71\" clid=\"1238547719\" /></addrHeader><typeName>CondAttrListCollection</typeName>" # create the folder - single version # last argument is createParents - if true, automatically creates parent # folders if needed # note this will not work if the database already exists - delete mycool.db first # Deprecated/dropped: myfolder=db.createFolder('/OFFLINE/OFFLINE_DQMF',spec2,desc,cool.FolderVersioning.MULTI_VERSION,True) folderSpec=cool.FolderSpecification(cool.FolderVersioning.MULTI_VERSION, spec2) myfolder=db.createFolder('/OFFLINE/OFFLINE_DQMF',folderSpec,desc,True) db.closeDatabase() def createRPCConditionDB(): import sys, os from PyCool import cool,coral # get database service and open database dbSvc=cool.DatabaseSvcFactory.databaseService() # database accessed via physical name dbstring='sqlite://;schema=RPCConditionDB.db;dbname=RPC_DQA' if os.access('RPCConditionDB.db', os.R_OK): try: os.unlink('RPCConditionDB.db')
def genDb(self, dbFileName, dbName, params, folderPath, params64={}): # Do checks if self.beginRun is None: raise ParameterDbFillerError( 'Must set begin run number before generating db') if self.endRun is None: raise ParameterDbFillerError( 'Must set end run number before generating db') if len(params) == 0: raise ParameterDbFillerError('No parameters for db ' + dbName) # remove existing db, if any try: os.remove(dbFileName) print("ParameterDbFiller.genDb: Removed db", dbFileName) except Exception: pass # get database service and open database dbSvc = cool.DatabaseSvcFactory.databaseService() # build dbstring - database accessed via physical name dbstring = "sqlite://;schema=" + dbFileName + ";dbname=" + dbName try: db = dbSvc.createDatabase(dbstring) except Exception as e: print('ParameterDbFiller.genDb: Problem creating database', e) sys.exit(-1) print("ParameterDbFiller.genDb: Created database", dbstring) # setup a folder payload specification spec = cool.RecordSpecification() for key in params: spec.extend(key, cool.StorageType.String4k) pass # add in spec for long strings for key in params64: spec.extend(key, cool.StorageType.String64k) # folder meta-data - note for Athena this has a special meaning desc = '<timeStamp>run-event</timeStamp><addrHeader><address_header service_type="71" clid="40774348" /></addrHeader><typeName>AthenaAttributeList</typeName>' # create the folder - single version # last argument is createParents - if true, automatically creates parent # folders if needed # note this will not work if the database already exists - delete mycool.db first # myfolder=db.createFolder(folderPath, spec, desc, cool.FolderVersioning.SINGLE_VERSION,True) folderSpec = cool.FolderSpecification( cool.FolderVersioning.SINGLE_VERSION, spec) myfolder = db.createFolder(folderPath, folderSpec, desc, True) # now fill in parameters data = cool.Record(spec) for k, v in six.iteritems(params): data[k] = v for k, v in six.iteritems(params64): data[k] = v print("ParameterDbFiller.genDb: Recording parameters", data) # store object with IOV valid from 0-10, channel 3 myfolder.storeObject(self.beginRun, self.endRun, data, 0) print("ParameterDbFiller.genDb: Stored object") # finalize db.closeDatabase() # Try to dump out db as test self.dumpDb(dbstring)
class ParameterDbFiller(object): """ A simple class used to fill SQLite db file with parameters for simulation and/or digitization. """ def __init__(self): object.__init__(self) self.beginRun = None self.endRun = None self.simParams = {} self.digitParams = {} self.digitParams64 = {} return # def setBeginRun(self, beginRun): self.beginRun = beginRun << 32 return def setEndRun(self, endRun): self.endRun = endRun << 32 return def addSimParam(self, name, value): self.simParams[name] = value return def addDigitParam(self, name, value): self.digitParams[name] = value return def addDigitParam64(self, name, value): self.digitParams64[name] = value return def genSimDb(self, dbFileName = None): # Allow to define specific file name, otherwise use customDb = False if dbFileName == None: # Set to default value dbFileName = "SimParams.db" customDb = True # Generate db: args - file name, dbname, params, folder path self.genDb(dbFileName, 'SIMPARAM', self.simParams, '/Simulation/Parameters') def genDigitDb(self, dbFileName = None): # Allow to define specific file name, otherwise use customDb = False if dbFileName == None: # Set to default value dbFileName = "DigitParams.db" customDb = True # Generate db: args - file name, dbname, params, folder path self.genDb(dbFileName, 'DIGPARAM', self.digitParams, '/Digitization/Parameters', self.digitParams64) def genDb(self, dbFileName, dbName, params, folderPath, params64 = {}): # Do checks if self.beginRun == None: raise ParameterDbFillerError, 'Must set begin run number before generating db' if self.endRun == None: raise ParameterDbFillerError, 'Must set end run number before generating db' if len(params) == 0: raise ParameterDbFillerError, 'No parameters for db ' + dbName # remove existing db, if any try: os.remove(dbFileName) print "ParameterDbFiller.genDb: Removed db", dbFileName except: pass # get database service and open database dbSvc=cool.DatabaseSvcFactory.databaseService() # build dbstring - database accessed via physical name dbstring="sqlite://;schema=" + dbFileName + ";dbname=" + dbName try: db=dbSvc.createDatabase(dbstring) except Exception,e: print 'ParameterDbFiller.genDb: Problem creating database',e sys.exit(-1) print "ParameterDbFiller.genDb: Created database",dbstring # setup a folder payload specification spec=cool.RecordSpecification() for key in params: spec.extend(key, cool.StorageType.String4k) pass # add in spec for long strings for key in params64: spec.extend(key, cool.StorageType.String64k) # folder meta-data - note for Athena this has a special meaning desc = '<timeStamp>run-event</timeStamp><addrHeader><address_header service_type="71" clid="40774348" /></addrHeader><typeName>AthenaAttributeList</typeName>' # create the folder - single version # last argument is createParents - if true, automatically creates parent # folders if needed # note this will not work if the database already exists - delete mycool.db first # myfolder=db.createFolder(folderPath, spec, desc, cool.FolderVersioning.SINGLE_VERSION,True) folderSpec = cool.FolderSpecification(cool.FolderVersioning.SINGLE_VERSION, spec) myfolder = db.createFolder(folderPath, folderSpec, desc, True) # now fill in parameters data = cool.Record(spec) for k, v in params.iteritems(): data[k] = v for k, v in params64.iteritems(): data[k] = v print "ParameterDbFiller.genDb: Recording parameters", data # store object with IOV valid from 0-10, channel 3 myfolder.storeObject(self.beginRun, self.endRun, data, 0) print "ParameterDbFiller.genDb: Stored object" # finalize db.closeDatabase() # Try to dump out db as test self.dumpDb(dbstring)
class Databases(object): """ Databases helper class. Used as a singleton. (don't instantiate) Specifies abbreviations for database connection strings and folders """ FOLDERS = { "DQMFONL": "COOLONL_GLOBAL::/GLOBAL/DETSTATUS/DQMFONL", "DQMFONLLB": "COOLONL_GLOBAL::/GLOBAL/DETSTATUS/DQMFONLLB", "SHIFTONL": "COOLONL_GLOBAL::/GLOBAL/DETSTATUS/SHIFTONL", "DQMFOFL": "COOLOFL_GLOBAL::/GLOBAL/DETSTATUS/DQMFOFL", "DCSOFL": "COOLOFL_GLOBAL::/GLOBAL/DETSTATUS/DCSOFL", "DQCALCOFL": "COOLOFL_GLOBAL::/GLOBAL/DETSTATUS/DQCALCOFL", "SHIFTOFL": "COOLOFL_GLOBAL::/GLOBAL/DETSTATUS/SHIFTOFL", "LBSUMM": "COOLOFL_GLOBAL::/GLOBAL/DETSTATUS/LBSUMM", "VIRTUALFLAGS": "COOLOFL_GLOBAL::/GLOBAL/DETSTATUS/VIRTUALFLAGS", "DEFECTS": "COOLOFL_GLOBAL::/GLOBAL/DETSTATUS/DEFECTS", "SOR_Params": "COOLONL_TDAQ::/TDAQ/RunCtrl/SOR_Params", "EOR_Params": "COOLONL_TDAQ::/TDAQ/RunCtrl/EOR_Params", "SOR": "COOLONL_TDAQ::/TDAQ/RunCtrl/SOR", "EOR": "COOLONL_TDAQ::/TDAQ/RunCtrl/EOR", "LBLB": "COOLONL_TRIGGER::/TRIGGER/LUMI/LBLB", "LBTIME": "COOLONL_TRIGGER::/TRIGGER/LUMI/LBTIME", "LBLESTONL": "COOLONL_TRIGGER::/TRIGGER/LUMI/LBLESTONL", "LVL1COUNTERS": "COOLONL_TRIGGER::/TRIGGER/LUMI/LVL1COUNTERS", "HLTCOUNTERS": "COOLOFL_TRIGGER::/TRIGGER/LUMI/HLTCOUNTERS", "HLT/Menu": "COOLONL_TRIGGER::/TRIGGER/HLT/Menu", "LVL1/Menu": "COOLONL_TRIGGER::/TRIGGER/LVL1/Menu", "Prescales": "COOLONL_TRIGGER::/TRIGGER/LVL1/Prescales", } @classmethod def resolve_folder_string(cls, folder_name, db_override=None): """ Resolves a simplified folder URI. Examples: folder_name == "SHIFTOFL" will give a connection to COOLOFL_GLOBAL/COMP200 folder /GLOBAL/DETSTATUS/SHIFTOFL folder_name == "test.db::SHIFTOFL" will give a connection to an sqlite database test.db Priority: * Database specified in db_override * Database specified in `folder_name` * Database specified in cls.FOLDERS[folder_name] """ res_database = db_override # First check - is a database specified in the folder name? if "::" in folder_name: assert folder_name.count("::") == 1, "Bad folder format" database, folder_name = folder_name.split("::") # If res_database hasn't already been set, do so res_database = database if not res_database else res_database if folder_name in cls.FOLDERS: database, res_folder = cls.FOLDERS[folder_name].split("::") res_database = database if not res_database else res_database else: res_folder = folder_name return res_database, res_folder @classmethod def get_folder(cls, folder_name, db_override=None, read_only=True, create_function=None, also_db=False): """ Retrieve a folder. The `db_override` parameter allows over-riding the database which comes from the folder string. Parameters: `folder_name` : Folder name or alias to load `db_override` : If specified, causes an alternate database string to be used. `read_only` : Specifies if a read-only database connection should be used. `create_function` : If specified, function to be called in case the folder doesn't exist. It is passed the database connection. `also_db` : Make the return value (db, folder) """ if read_only: assert not create_function, "`read_only` specified with `create`" database, folder = cls.resolve_folder_string(folder_name, db_override) assert database, ("Unable to resolve database for (%s, %s)" % (folder_name, db_override)) create = bool(create_function) db = cls.get_instance(database, read_only, create) try: cool_folder = db.getFolder(folder) except Exception as error: log.debug('HELP! %s' % error.args) args = str(error.args[0] if not isinstance(error.args, basestring) else error.args) log.debug('THIS IS %s' % type(args)) log.debug('Value of boolean: %s' % ("not found" in args)) if not ("cannot be established" in args or "not found" in args): log.exception("Unknown error encountered whilst opening " "database connection to '%s'" % database) raise if not create_function: log.exception("Database does not exist, `create_function` not " "specified") raise cool_folder = create_function(db) if also_db: return db, cool_folder return cool_folder @classmethod def resolve_db_string(cls, db_string, read_only=True): """ Take a database string - if it looks like a filename ending in ".db" then assume it is a sqlite database with that name. If the `db_string` is prepended with "WRITE|" then a writable connection is requested. If the db_string doesn't contain a "/", then "/" + DEFAULT_DBNAME is appended. """ if "://" in db_string: # Assume that the string is already resolved return db_string, read_only if db_string.startswith("WRITE|"): assert db_string.count("|") == 1, "Bad db_string format" db_string = db_string.split("|")[1] read_only = False sqlite_regex = re.compile( r"(?P<filename>.*?\.db)(?:/?(?P<dbname>[^/]+))?$") matched = sqlite_regex.match(db_string) if matched: filename, dbname = matched.groups() dbname = DEFAULT_DBNAME if not dbname else dbname db_string = "sqlite://schema=%s;dbname=%s" % (filename, dbname) else: if "/" not in db_string: return db_string + "/" + DEFAULT_DBNAME, read_only return db_string, read_only @classmethod def get_instance(cls, db_string, read_only=True, create=False): """ Open a database connection """ res_db_string, read_only = cls.resolve_db_string(db_string, read_only) try: prev_stdout = sys.stdout sys.stdout = StringIO() try: connection = indirectOpen(res_db_string, readOnly=read_only, oracle=True) finally: sys.stdout = prev_stdout except Exception as e: if "The database does not exist" in e.args[0] and not create: log.info("Failed trying to connect to '%s'", res_db_string) raise from PyCool import cool dbService = cool.DatabaseSvcFactory.databaseService() connection = dbService.createDatabase(res_db_string) except: print sys.exc_info()[0] raise return connection @classmethod def build_folder(cls, db, folder_name, multiversion, record): """ Create `folder_name` on database instance `db`, with recordspecification `record`. Also creates folderset to which folder_name belongs if necessary. """ from PyCool import cool folderset_path = dirname(folder_name) try: folder_set = db.getFolderSet(folderset_path) except Exception, error: caught_error = "Folder set %s not found" % folderset_path if caught_error not in error.args[0]: raise log.debug("Folderset doesn't exist - creating it.") db.createFolderSet(folderset_path, "", True) if not isinstance(record, cool.RecordSpecification): record_spec = cool.RecordSpecification() for field in record: record_spec.extend(*field) else: record_spec = record FV = cool.FolderVersioning versioning = FV.MULTI_VERSION if multiversion else FV.SINGLE_VERSION folder_spec = cool.FolderSpecification(versioning, record_spec) folder = db.createFolder(folder_name, folder_spec) payload = cool.Record(record_spec) return folder, payload
try: #=== creating folder specifications spec = cool.RecordSpecification() spec.extend( 'CaloCondBlob16M', cool.StorageType.Blob16M ) #=== create the folder folderPath = CaloCondTools.getCaloPrefix()+"LAR/TimeCorrectionOfl/NonRunCon" folderTag = tagCon print ("Filling COOL folder ", folderPath, " with tag ", folderTag ) desc = CaloCondTools.getAthenaFolderDescr() try: folder = db.getFolder(folderPath) except Exception: log.warning("Folder %s not found, creating it...", folderPath) print ("Folder ", folderPath, " not found, creating it... " ) folderSpec = cool.FolderSpecification(cool.FolderVersioning.SINGLE_VERSION, spec) folder = db.createFolder(folderPath, folderSpec, desc, True) #================================================== #=== Create the CaloCondBlobFlt objects #================================================== #=== default a and b to be used for each gain gainDefVec_1 = g.std.vector('float')() gainDefVec_1.push_back(0.) # febOffset+channelOffset gainDefVec_1.push_back(0.) # energy depended error parameter 1 gainDefVec_1.push_back(0.) # energy depended error parameter 2 gainDefVec_1.push_back(0.) # feb depended energy correction parameter 1 gainDefVec_1.push_back(0.) # feb depended energy correction parameter 2 gainDefVec_1.push_back(0.) # feb depended energy correction parameter 3 gainDefVec_1.push_back(0.) # feb depended energy correction parameter 4 gainDefVec_1.push_back(0.) # feb depended energy correction parameter 5
#================================================== #=== set shortcut g = cppyy.gbl #=== get a logger log = TileCalibLogger.getLogger("WriteOfc") #=== (re-)create the database db = TileCalibTools.openDb('SQLITE', 'CONDBR2', 'UPDATE') #=== creating folder specifications spec = cool.RecordSpecification() #=== Note special name '*Ofc' and 16M instead of 64k for OFCs spec.extend( 'TileCalibBlobOfc', cool.StorageType.Blob16M ) folderMode = cool.FolderVersioning.SINGLE_VERSION folderSpec = cool.FolderSpecification(folderMode, spec) #=== loop over all input files for fileName in [ofcFiles[0]]: log.info( "Reading file: %s" % fileName ) #=== determine COOL folder from input file name fileNameParts = re_fileName.search(fileName).groups() ofMethod = fileNameParts[0] ofRun = fileNameParts[1][:3] #=== fix LED->LAS if ofRun=="LED": ofRun="LAS" objVersion = int(ofMethod[2]) folderPath = TileCalibTools.getTilePrefix(False)+"FILTER/"+ofMethod+"/"+ofRun log.info( "Filling COOL folder %s" % ( folderPath ) )