def defineFolderSpec( isMultiVersion ):
recordSpec = cool.RecordSpecification()
recordSpec.extend( "Offset", ST.Float )
folderSpec = cool.FolderSpecification( FV.MULTI_VERSION if isMultiVersion else FV.SINGLE_VERSION,
recordSpec )
recordSpec2 = cool.RecordSpecification()
recordSpec2.extend( "Noise", ST.Blob64k )
folderSpec2 = cool.FolderSpecification( FV.MULTI_VERSION if isMultiVersion else FV.SINGLE_VERSION,
recordSpec2 )
return ( folderSpec, folderSpec2 )
def createRPCConditionDB():
import os
from PyCool import cool
# get database service and open database
dbSvc = cool.DatabaseSvcFactory.databaseService()
# database accessed via physical name
dbstring = 'sqlite://;schema=RPCConditionDB.db;dbname=RPC_DQA'
if os.access('RPCConditionDB.db', os.R_OK):
try:
os.unlink('RPCConditionDB.db')
except Exception as e:
print('Unable to remove existing db file, reason', e)
try:
db = dbSvc.createDatabase(dbstring)
except Exception as e:
print('Problem creating database', e)
return
print("Created database", dbstring)
spec2 = cool.RecordSpecification()
spec2.extend("PanelRes", cool.StorageType.String255)
spec2.extend("StripStatus", cool.StorageType.String4k)
desc = "<timeStamp>run-lumi</timeStamp><addrHeader><address_header service_type=\"71\" clid=\"1238547719\" /></addrHeader><typeName>CondAttrListCollection</typeName>"
# create the folder - single version
# last argument is createParents - if true, automatically creates parent
# folders if needed
# note this will not work if the database already exists - delete mycool.db first
# Deprecated/dropped: myfolder=db.createFolder('/OFFLINE/FINAL',spec2,desc,cool.FolderVersioning.MULTI_VERSION,True)
folderSpec = cool.FolderSpecification(cool.FolderVersioning.MULTI_VERSION,
spec2)
db.createFolder('/OFFLINE/FINAL', folderSpec, desc, True)
db.closeDatabase()
def createSqlite(sqliteName,folderName="/TDAQ/OLC/LHC/FILLPARAMS"):
dbSvc = cool.DatabaseSvcFactory.databaseService()
if os.access(sqliteName,os.R_OK):
print ("UPDATING existing sqlite file %s" % sqliteName)
db=dbSvc.openDatabase("sqlite://;schema="+sqliteName+";dbname=CONDBR2",False)
else:
print("Creating new sqlite file %s" % sqliteName)
db=dbSvc.createDatabase("sqlite://;schema="+sqliteName+";dbname=CONDBR2")
pass
spec = cool.RecordSpecification()
spec.extend("Beam1Bunches",cool.StorageType.UInt32)
spec.extend("Beam2Bunches",cool.StorageType.UInt32)
spec.extend("LuminousBunches",cool.StorageType.UInt32)
spec.extend("BCIDmasks",cool.StorageType.Blob64k)
descr='<timeStamp>time</timeStamp><addrHeader><address_header service_type="71" clid="40774348" /></addrHeader><typeName>AthenaAttributeList</typeName>'
if db.existsFolder(folderName):
folder=db.getFolder(folderName)
else:
print("Creating COOL folder/tag %s" % (folderName))
#folder = db.createFolder(folderName, spec, desc, cool.FolderVersioning.MULTI_VERSION, True)
folderSpec = cool.FolderSpecification(cool.FolderVersioning.SINGLE_VERSION, spec)
folder = db.createFolder(folderName, folderSpec, descr, True)
pass
return db,folder
def store_PID_to_folder(db, values_list):
"""
Function to create folder FOLDER_NAME in db with channels and input data from values_list with tag TAG_NAME
:param db: Open cool db
:param values_list: list of tuples ('channel name', valus array)
"""
spec = cool.RecordSpecification()
spec.extend('array_value', cool.StorageType.Float)
_logger.info('Create folder %s' % FOLDER_NAME)
folder_spec = cool.FolderSpecification(cool.FolderVersioning.MULTI_VERSION,
spec,
cool.PayloadMode.VECTORPAYLOAD)
folder = db.createFolder(
FOLDER_NAME, folder_spec,
' <timeStamp>run-lumi</timeStamp><addrHeader><address_header service_type="71" clid="55403898"/></addrHeader><typeName>CondAttrListVec</typeName>',
True)
for index, channel_name in enumerate(values_list):
folder.createChannel(index, channel_name[0])
print '%i - %s' % (index, channel_name[0])
folder.setupStorageBuffer()
for index, channel_values in enumerate(values_list):
_logger.info('Store data from %s' % channel_values[0])
values = channel_values[1]
# print channel_values[0]
# print channel_values[1]
vector = cool.IRecordVector()
for value in values:
# print value
data = cool.PyCool.Helpers.IRecordPtr(spec)
data.get()['array_value'] = value
vector.push_back(data)
folder.storeObject(0, cool.ValidityKeyMax, vector, index, TAG_NAME)
folder.flushStorageBuffer()
def build_folder(cls, db, folder_name, multiversion, record):
"""
Create `folder_name` on database instance `db`, with recordspecification
`record`.
Also creates folderset to which folder_name belongs if necessary.
"""
from PyCool import cool
folderset_path = dirname(folder_name)
try:
db.getFolderSet(folderset_path)
except Exception as error:
caught_error = "Folder set %s not found" % folderset_path
if caught_error not in error.args[0]:
raise
log.debug("Folderset doesn't exist - creating it.")
db.createFolderSet(folderset_path, "", True)
if not isinstance(record, cool.RecordSpecification):
record_spec = cool.RecordSpecification()
for field in record:
record_spec.extend(*field)
else:
record_spec = record
FV = cool.FolderVersioning
versioning = FV.MULTI_VERSION if multiversion else FV.SINGLE_VERSION
folder_spec = cool.FolderSpecification(versioning, record_spec)
folder = db.createFolder(folder_name, folder_spec)
payload = cool.Record(record_spec)
return folder, payload
def defineFolderSpecTypeName(Name, Type, isMultiVersion):
recordSpec = cool.RecordSpecification()
# recordSpec.extend( "Maps", ST.Float )
recordSpec.extend(Name, Type)
folderSpec = cool.FolderSpecification(
FV.MULTI_VERSION if isMultiVersion else FV.SINGLE_VERSION, recordSpec)
return (folderSpec)
def createFolder(db, name):
desc = '<timeStamp>run-lumi</timeStamp><addrHeader><address_header service_type="71" clid="1238547719" /></addrHeader><typeName>CondAttrListCollection</typeName>'
spec = cool.RecordSpecification()
spec.extend('LBAvInstLumi', cool.StorageType.Float)
spec.extend('Valid', cool.StorageType.UInt32)
folderSpec = cool.FolderSpecification(cool.FolderVersioning.MULTI_VERSION,
spec)
#return db.createFolder(name,spec,desc,cool.FolderVersioning.MULTI_VERSION, True)
return db.createFolder(name, folderSpec, desc, True)
def __init__(self,
db,
folderPath,
caloBlobType=None,
isMultiVersionFolder=True,
isRunLumiTimeStamp=True):
"""
Input:
- db : db should be an open database connection
- folderPath: full folder path to create or update
"""
#=== initialize base class
CaloCondLogger.__init__(self, "CaloBlobWriter")
#=== store db
self.__db = db
#=== determine folder mode
folderMode = cool.FolderVersioning.MULTI_VERSION
if not isMultiVersionFolder:
folderMode = cool.FolderVersioning.SINGLE_VERSION
#=== determine folder description
folderDescr = getAthenaFolderDescr()
#=== access folder in db
try:
#=== create folder if it does not exist
if self.__db.existsFolder(folderPath):
self.__folder = self.__db.getFolder(folderPath)
#=== check folder mode
modeInCool = self.__folder.versioningMode()
if modeInCool != folderMode:
str = "Incompatible folder mode detected, COOL folder has type "
if modeInCool == cool.FolderVersioning.MULTI_VERSION:
str += "MULTI"
else:
str += "SINGLE"
raise Exception(str)
else:
#=== create folder if it does not exist
payloadSpec = cool.RecordSpecification()
payloadSpec.extend('CaloCondBlob16M', cool.StorageType.Blob16M)
folderSpec = cool.FolderSpecification(folderMode, payloadSpec)
self.__folder = db.createFolder(folderPath, folderSpec,
folderDescr, True)
except Exception as e:
self.log().critical(e)
raise
#=== initialize channel dictionaries
self.__chanDictRecord = {} # <int, cool.Record >
self.__chanDictCells = {} # <int, CaloCondBlobFlt>
def _create_defect_logic_folder(self, db):
"""
Creates the COOL database/folder for virtual defect logic, if they don't exist.
Internal - use create=True in the constructor to create the COOL folder.
"""
log.info("Creating defect logic folder")
record_spec = cool.RecordSpecification()
record_spec.extend("clauses", ST.String4k)
folder_spec = cool.FolderSpecification(FV.MULTI_VERSION, record_spec)
return db.createFolder(DEFECT_LOGIC_FOLDER, folder_spec,
DEFECT_LOGIC_FOLDER_DESC, True)
def _create_defects_folder(self, db):
"""
Creates the COOL database/folder for defects, if they don't exist.
Internal - use create=True in the constructor to create the COOL folder.
"""
log.info("Creating defects folder")
record_spec = cool.RecordSpecification()
record_spec.extend("present", ST.Bool)
record_spec.extend("recoverable", ST.Bool)
record_spec.extend("user", ST.String255)
record_spec.extend("comment", ST.String255)
folder_spec = cool.FolderSpecification(FV.MULTI_VERSION, record_spec)
return db.createFolder(DEFECTS_FOLDER, folder_spec, DEFECT_FOLDER_DESC,
True)
def __init__(self):
self.records = []
self.iovStartRun = 0
self.iovStartLumiBlock = 0
self.iovEndRun = 0
self.iovEndLumiBlock = 0
self.tag = "AFPTest-00-00-00"
self.folderName = "/FWD/AFP/TEST"
self.spec = cool.RecordSpecification()
# self.spec.extend("data", cool.StorageType.Blob64k)
self.spec.extend("data", cool.StorageType.String16M)
# self.desc = '<timeStamp>run-lumi</timeStamp><addrHeader><address_header service_type="71" clid="1238547719" /></addrHeader><typeName>CondAttrListCollection</typeName>'
self.desc = '<timeStamp>run-lumi</timeStamp><addrHeader><address_header service_type="71" clid="40774348" /></addrHeader><typeName>AthenaAttributeList</typeName>'
self.data = cool.Record(self.spec)
self.folderSpec = cool.FolderSpecification(
cool.FolderVersioning.MULTI_VERSION, self.spec)
def createSqlite(sqliteName,
folderName,
foldertag,
iovMin=cool.ValidityKeyMin,
iovMax=cool.ValidityKeyMax,
inputFileName=None,
defaultvalue=1.0):
dbSvc = cool.DatabaseSvcFactory.databaseService()
if os.access(sqliteName, os.R_OK):
print("UPDATING existing sqlite file", sqliteName)
db = dbSvc.openDatabase(
"sqlite://;schema=" + sqliteName + ";dbname=CONDBR2", False)
else:
print("Creating new sqlite file", sqliteName)
db = dbSvc.createDatabase("sqlite://;schema=" + sqliteName +
";dbname=CONDBR2")
spec = cool.RecordSpecification()
spec.extend('CaloCondBlob16M', cool.StorageType.Blob16M)
desc = '<timeStamp>run-lumi</timeStamp><addrHeader><address_header service_type="71" clid="40774348" /></addrHeader><typeName>AthenaAttributeList</typeName>'
if db.existsFolder(folderName):
folder = db.getFolder(folderName)
else:
print("Creating COOL folder/tag %s/%s" % (folderName, foldertag))
#folder = db.createFolder(folderName, spec, desc, cool.FolderVersioning.MULTI_VERSION, True)
folderSpec = cool.FolderSpecification(
cool.FolderVersioning.MULTI_VERSION, spec)
folder = db.createFolder(folderName, folderSpec, desc, True)
pass
if inputFileName is None or len(inputFileName) == 0:
print("No input given. Create empty blob")
data = cool.Record(spec)
else:
data = CaloCondBlobWriterFromFile(spec, inputFileName, defaultvalue)
print("Storing CaloCondBlob object")
folder.storeObject(iovMin, iovMax, data, cool.ChannelId(0), foldertag,
True)
db.closeDatabase()
def ensureFolder(db,
folder,
spec,
desc,
version=cool.FolderVersioning.SINGLE_VERSION):
"""
Ensure folder exists, creating it if needed with given spec
Return COOL folder object
"""
if (not db.existsFolder(folder)):
print 'Attempting to create', folder, 'with description', desc
try:
# Deprecated/dropped: db.createFolder(folder,spec,desc,version,True)
folderSpec = cool.FolderSpecification(version, spec)
db.createFolder(folder, folderSpec, desc, True)
print 'Folder created'
except Exception, e:
print e
print 'Could not create folder', folder
return None
def WriteSqlite(name, input_dict):
UNIX2COOL = 1000000000
dbSvc = cool.DatabaseSvcFactory.databaseService()
connectString = 'sqlite://;schema=' + name + ';dbname=L1CALO'
# folder_name="/TRIGGER/Receivers/Factors/CalibGains"
print('\nrecreating database file:', name)
dbSvc.dropDatabase(connectString)
db = dbSvc.createDatabase(connectString)
spec = cool.RecordSpecification()
spec.extend("factor", cool.StorageType.Float)
spec.extend("status", cool.StorageType.UInt32)
folderSpec = cool.FolderSpecification(cool.FolderVersioning.SINGLE_VERSION,
spec)
now = int(time.time())
since = now * UNIX2COOL
# since = 0
# until = sys.maxint
until = cool.ValidityKeyMax
db.createFolderSet('/TRIGGER')
db.createFolderSet('/TRIGGER/Receivers')
db.createFolderSet('/TRIGGER/Receivers/Factors')
folder_description = '<timeStamp>time</timeStamp><addrHeader><address_header service_type="71" clid="1238547719"/></addrHeader><typeName>CondAttrListCollection</typeName>'
f = db.createFolder("/TRIGGER/Receivers/Factors/CalibGains", folderSpec,
folder_description)
print(" Now creating sqlite file for ", len(input_dict.keys()),
" channels")
for i in input_dict.keys():
data = cool.Record(spec)
data['factor'] = input_dict[i][0]
data['status'] = input_dict[i][1]
f.storeObject(since, until, data, int(i, 16))
db.closeDatabase()
try:
db = dbSvc.createDatabase(dbstring)
except Exception, e:
print 'Problem creating database', e
sys.exit(-1)
print "Created database", dbstring
# ----------------------------------------------------------------------- #
# setup a folder
# ----------------------------------------------------------------------- #
#spec=cool.RecordSpecification()
recspec = cool.RecordSpecification()
recspec.extend("version", cool.StorageType.String4k)
recspec.extend("syserrors", cool.StorageType.String4k)
spec = cool.FolderSpecification(cool.FolderVersioning.MULTI_VERSION, recspec,
True)
#spec.extend(fdspec,cool.StorageType.String16M)
# folder meta-data - note for Athena this has a special meaning
desc = '<timeStamp>time</timeStamp><addrHeader><address_header service_type="71" clid="1238547719" /></addrHeader><typeName>CondAttrListCollection</typeName>'
####<timeStamp>time</timeStamp><addrHeader><address_header service_type="71" clid="40774348" /></addrHeader><typeName>AthenaAttributeList</typeName>
# create the folder - single version
# last argument is createParents - if true, automatically creates parent folders if needed
# note this will not work if the database already exists - delete mycool.db first
#folder=db.createFolder(fdname,spec,desc,cool.FolderVersioning.MULTI_VERSION,True)
folder = db.createFolder(fdname, spec, desc, True)
folder.createChannel(0, "muonalignmenterrors",
def main():
parser = OptionParser()
parser.add_option(
"-c",
"--connect-string",
dest="connectString",
type="string",
help="cool::DatabaseId to use for the connection",
default=
"sqlite://none;schema=sqlite.db;user=none;password=none;dbname=DDDB")
parser.add_option("-s",
"--source",
dest="source",
type="string",
help="directory where the files to copy to the DB are",
metavar="SRCDIR")
parser.add_option(
"-d",
"--dest",
dest="dest",
type="string",
help=
"directory, in the database, where to copy the files (default: '/')",
default="/",
metavar="DESTPATH")
parser.add_option("-x",
"--drop-db",
action="store_true",
dest="drop",
help="drop the database before the copy (default)",
default=True)
parser.add_option(
"-k",
"--keep-db",
action="store_false",
dest="drop",
help="keep the existing database and merge with the new files")
(options, args) = parser.parse_args()
if len(args) != 0 or not options.source or not options.dest:
parser.error("Bad arguments, use -h for help.")
#### prepare structres to create and fill the folders
folderspec = cool.FolderSpecification(cool.FolderVersioning.MULTI_VERSION)
#folderspec.payloadSpecification().extend("data",cool.StorageType.String255)
#folderspec.payloadSpecification().extend("data",cool.StorageType.String4k)
#folderspec.payloadSpecification().extend("data",cool.StorageType.String64k)
folderspec.payloadSpecification().extend("data",
cool.StorageType.String16M)
folderdesc = "<storage_type=7>"
payload = cool.Record(folderspec.payloadSpecification())
payload["data"] = "XML stuff" # dummy value, just to start
#### initialize COOL
dbs = cool.DatabaseSvcFactory.databaseService()
if options.drop:
dbs.dropDatabase(options.connectString)
db = dbs.createDatabase(options.connectString)
else:
print
db = dbs.openDatabase(options.connectString, False)
excludes = ['clhep2dtd.pl']
# count files (just to know)
sum = 0
for root, dirs, files in os.walk(options.source):
if 'CVS' in root: # skip CVS directories
continue
for x in excludes:
if x in files:
files.remove(x)
sum += len(files)
num_len = len(str(sum))
# populate the database
n = 0
for root, dirs, files in os.walk(options.source):
# remove base path
dbroot = root.replace(options.source, options.dest)
dbroot = dbroot.replace('//', '/')
if 'CVS' in root: # skip CVS directories
continue
if dbroot != "/": # it means it is not "/"
if not db.existsFolderSet(dbroot):
db.createFolderSet(dbroot)
#else:
# dbroot = "/"
print dbroot
for x in excludes:
if x in files:
files.remove(x)
for f in files:
n += 1
print("%" + str(num_len) + "d %" + str(num_len) +
"d %s") % (n, sum - n, f)
folder_path = os.path.join(dbroot, f)
if db.existsFolder(folder_path):
folder = db.getFolder(folder_path)
else:
folder = db.createFolder(folder_path, folderspec, folderdesc)
xml_data = open(os.path.join(root, f)).read()
fixed_data = fix_system_ids(xml_data, "conddb:" + dbroot)
#fixed_data = xml_data
fixed_data = fix_env_vars(fixed_data)
payload["data"] = fixed_data
folder.storeObject(cool.ValidityKeyMin, cool.ValidityKeyMax,
payload, 0)
print "Total files inserted = %d" % sum
spec2=cool.RecordSpecification()
spec2.extend("recEta",cool.StorageType.String4k)
spec2.extend("detEta",cool.StorageType.String4k)
spec2.extend("recPhi1",cool.StorageType.String4k)
spec2.extend("recPhi2",cool.StorageType.String4k)
spec2.extend("detPhi1",cool.StorageType.String4k)
spec2.extend("detPhi2",cool.StorageType.String4k)
# folder meta-data - note for Athena this has a special meaning
desc="<timeStamp>run-lumi</timeStamp><addrHeader><address_header service_type=\"71\" clid=\"1238547719\" /></addrHeader><typeName>CondAttrListCollection</typeName>"
# create the folder - single version
# last argument is createParents - if true, automatically creates parent
# folders if needed
# note this will not work if the database already exists - delete mycool.db first
# Deprecated/dropped: myfolder=db.createFolder('/OFFLINE/OFFLINE_DQMF',spec2,desc,cool.FolderVersioning.MULTI_VERSION,True)
folderSpec=cool.FolderSpecification(cool.FolderVersioning.MULTI_VERSION, spec2)
myfolder=db.createFolder('/OFFLINE/OFFLINE_DQMF',folderSpec,desc,True)
db.closeDatabase()
def createRPCConditionDB():
import sys, os
from PyCool import cool,coral
# get database service and open database
dbSvc=cool.DatabaseSvcFactory.databaseService()
# database accessed via physical name
dbstring='sqlite://;schema=RPCConditionDB.db;dbname=RPC_DQA'
if os.access('RPCConditionDB.db', os.R_OK):
try:
os.unlink('RPCConditionDB.db')
def genDb(self, dbFileName, dbName, params, folderPath, params64={}):
# Do checks
if self.beginRun is None:
raise ParameterDbFillerError(
'Must set begin run number before generating db')
if self.endRun is None:
raise ParameterDbFillerError(
'Must set end run number before generating db')
if len(params) == 0:
raise ParameterDbFillerError('No parameters for db ' + dbName)
# remove existing db, if any
try:
os.remove(dbFileName)
print("ParameterDbFiller.genDb: Removed db", dbFileName)
except Exception:
pass
# get database service and open database
dbSvc = cool.DatabaseSvcFactory.databaseService()
# build dbstring - database accessed via physical name
dbstring = "sqlite://;schema=" + dbFileName + ";dbname=" + dbName
try:
db = dbSvc.createDatabase(dbstring)
except Exception as e:
print('ParameterDbFiller.genDb: Problem creating database', e)
sys.exit(-1)
print("ParameterDbFiller.genDb: Created database", dbstring)
# setup a folder payload specification
spec = cool.RecordSpecification()
for key in params:
spec.extend(key, cool.StorageType.String4k)
pass
# add in spec for long strings
for key in params64:
spec.extend(key, cool.StorageType.String64k)
# folder meta-data - note for Athena this has a special meaning
desc = '<timeStamp>run-event</timeStamp><addrHeader><address_header service_type="71" clid="40774348" /></addrHeader><typeName>AthenaAttributeList</typeName>'
# create the folder - single version
# last argument is createParents - if true, automatically creates parent
# folders if needed
# note this will not work if the database already exists - delete mycool.db first
# myfolder=db.createFolder(folderPath, spec, desc, cool.FolderVersioning.SINGLE_VERSION,True)
folderSpec = cool.FolderSpecification(
cool.FolderVersioning.SINGLE_VERSION, spec)
myfolder = db.createFolder(folderPath, folderSpec, desc, True)
# now fill in parameters
data = cool.Record(spec)
for k, v in six.iteritems(params):
data[k] = v
for k, v in six.iteritems(params64):
data[k] = v
print("ParameterDbFiller.genDb: Recording parameters", data)
# store object with IOV valid from 0-10, channel 3
myfolder.storeObject(self.beginRun, self.endRun, data, 0)
print("ParameterDbFiller.genDb: Stored object")
# finalize
db.closeDatabase()
# Try to dump out db as test
self.dumpDb(dbstring)
class ParameterDbFiller(object):
"""
A simple class used to fill SQLite db file with parameters for simulation
and/or digitization.
"""
def __init__(self):
object.__init__(self)
self.beginRun = None
self.endRun = None
self.simParams = {}
self.digitParams = {}
self.digitParams64 = {}
return
#
def setBeginRun(self, beginRun):
self.beginRun = beginRun << 32
return
def setEndRun(self, endRun):
self.endRun = endRun << 32
return
def addSimParam(self, name, value):
self.simParams[name] = value
return
def addDigitParam(self, name, value):
self.digitParams[name] = value
return
def addDigitParam64(self, name, value):
self.digitParams64[name] = value
return
def genSimDb(self, dbFileName = None):
# Allow to define specific file name, otherwise use
customDb = False
if dbFileName == None:
# Set to default value
dbFileName = "SimParams.db"
customDb = True
# Generate db: args - file name, dbname, params, folder path
self.genDb(dbFileName, 'SIMPARAM', self.simParams, '/Simulation/Parameters')
def genDigitDb(self, dbFileName = None):
# Allow to define specific file name, otherwise use
customDb = False
if dbFileName == None:
# Set to default value
dbFileName = "DigitParams.db"
customDb = True
# Generate db: args - file name, dbname, params, folder path
self.genDb(dbFileName, 'DIGPARAM', self.digitParams, '/Digitization/Parameters', self.digitParams64)
def genDb(self, dbFileName, dbName, params, folderPath, params64 = {}):
# Do checks
if self.beginRun == None:
raise ParameterDbFillerError, 'Must set begin run number before generating db'
if self.endRun == None:
raise ParameterDbFillerError, 'Must set end run number before generating db'
if len(params) == 0:
raise ParameterDbFillerError, 'No parameters for db ' + dbName
# remove existing db, if any
try:
os.remove(dbFileName)
print "ParameterDbFiller.genDb: Removed db", dbFileName
except:
pass
# get database service and open database
dbSvc=cool.DatabaseSvcFactory.databaseService()
# build dbstring - database accessed via physical name
dbstring="sqlite://;schema=" + dbFileName + ";dbname=" + dbName
try:
db=dbSvc.createDatabase(dbstring)
except Exception,e:
print 'ParameterDbFiller.genDb: Problem creating database',e
sys.exit(-1)
print "ParameterDbFiller.genDb: Created database",dbstring
# setup a folder payload specification
spec=cool.RecordSpecification()
for key in params:
spec.extend(key, cool.StorageType.String4k)
pass
# add in spec for long strings
for key in params64:
spec.extend(key, cool.StorageType.String64k)
# folder meta-data - note for Athena this has a special meaning
desc = '<timeStamp>run-event</timeStamp><addrHeader><address_header service_type="71" clid="40774348" /></addrHeader><typeName>AthenaAttributeList</typeName>'
# create the folder - single version
# last argument is createParents - if true, automatically creates parent
# folders if needed
# note this will not work if the database already exists - delete mycool.db first
# myfolder=db.createFolder(folderPath, spec, desc, cool.FolderVersioning.SINGLE_VERSION,True)
folderSpec = cool.FolderSpecification(cool.FolderVersioning.SINGLE_VERSION, spec)
myfolder = db.createFolder(folderPath, folderSpec, desc, True)
# now fill in parameters
data = cool.Record(spec)
for k, v in params.iteritems():
data[k] = v
for k, v in params64.iteritems():
data[k] = v
print "ParameterDbFiller.genDb: Recording parameters", data
# store object with IOV valid from 0-10, channel 3
myfolder.storeObject(self.beginRun, self.endRun, data, 0)
print "ParameterDbFiller.genDb: Stored object"
# finalize
db.closeDatabase()
# Try to dump out db as test
self.dumpDb(dbstring)
class Databases(object):
"""
Databases helper class. Used as a singleton. (don't instantiate)
Specifies abbreviations for database connection strings and folders
"""
FOLDERS = {
"DQMFONL": "COOLONL_GLOBAL::/GLOBAL/DETSTATUS/DQMFONL",
"DQMFONLLB": "COOLONL_GLOBAL::/GLOBAL/DETSTATUS/DQMFONLLB",
"SHIFTONL": "COOLONL_GLOBAL::/GLOBAL/DETSTATUS/SHIFTONL",
"DQMFOFL": "COOLOFL_GLOBAL::/GLOBAL/DETSTATUS/DQMFOFL",
"DCSOFL": "COOLOFL_GLOBAL::/GLOBAL/DETSTATUS/DCSOFL",
"DQCALCOFL": "COOLOFL_GLOBAL::/GLOBAL/DETSTATUS/DQCALCOFL",
"SHIFTOFL": "COOLOFL_GLOBAL::/GLOBAL/DETSTATUS/SHIFTOFL",
"LBSUMM": "COOLOFL_GLOBAL::/GLOBAL/DETSTATUS/LBSUMM",
"VIRTUALFLAGS": "COOLOFL_GLOBAL::/GLOBAL/DETSTATUS/VIRTUALFLAGS",
"DEFECTS": "COOLOFL_GLOBAL::/GLOBAL/DETSTATUS/DEFECTS",
"SOR_Params": "COOLONL_TDAQ::/TDAQ/RunCtrl/SOR_Params",
"EOR_Params": "COOLONL_TDAQ::/TDAQ/RunCtrl/EOR_Params",
"SOR": "COOLONL_TDAQ::/TDAQ/RunCtrl/SOR",
"EOR": "COOLONL_TDAQ::/TDAQ/RunCtrl/EOR",
"LBLB": "COOLONL_TRIGGER::/TRIGGER/LUMI/LBLB",
"LBTIME": "COOLONL_TRIGGER::/TRIGGER/LUMI/LBTIME",
"LBLESTONL": "COOLONL_TRIGGER::/TRIGGER/LUMI/LBLESTONL",
"LVL1COUNTERS": "COOLONL_TRIGGER::/TRIGGER/LUMI/LVL1COUNTERS",
"HLTCOUNTERS": "COOLOFL_TRIGGER::/TRIGGER/LUMI/HLTCOUNTERS",
"HLT/Menu": "COOLONL_TRIGGER::/TRIGGER/HLT/Menu",
"LVL1/Menu": "COOLONL_TRIGGER::/TRIGGER/LVL1/Menu",
"Prescales": "COOLONL_TRIGGER::/TRIGGER/LVL1/Prescales",
}
@classmethod
def resolve_folder_string(cls, folder_name, db_override=None):
"""
Resolves a simplified folder URI.
Examples:
folder_name == "SHIFTOFL"
will give a connection to COOLOFL_GLOBAL/COMP200
folder /GLOBAL/DETSTATUS/SHIFTOFL
folder_name == "test.db::SHIFTOFL"
will give a connection to an sqlite database test.db
Priority:
* Database specified in db_override
* Database specified in `folder_name`
* Database specified in cls.FOLDERS[folder_name]
"""
res_database = db_override
# First check - is a database specified in the folder name?
if "::" in folder_name:
assert folder_name.count("::") == 1, "Bad folder format"
database, folder_name = folder_name.split("::")
# If res_database hasn't already been set, do so
res_database = database if not res_database else res_database
if folder_name in cls.FOLDERS:
database, res_folder = cls.FOLDERS[folder_name].split("::")
res_database = database if not res_database else res_database
else:
res_folder = folder_name
return res_database, res_folder
@classmethod
def get_folder(cls,
folder_name,
db_override=None,
read_only=True,
create_function=None,
also_db=False):
"""
Retrieve a folder. The `db_override` parameter allows over-riding the
database which comes from the folder string.
Parameters:
`folder_name` : Folder name or alias to load
`db_override` : If specified, causes an alternate database string
to be used.
`read_only` : Specifies if a read-only database connection should
be used.
`create_function` : If specified, function to be called in case the
folder doesn't exist. It is passed the database
connection.
`also_db` : Make the return value (db, folder)
"""
if read_only:
assert not create_function, "`read_only` specified with `create`"
database, folder = cls.resolve_folder_string(folder_name, db_override)
assert database, ("Unable to resolve database for (%s, %s)" %
(folder_name, db_override))
create = bool(create_function)
db = cls.get_instance(database, read_only, create)
try:
cool_folder = db.getFolder(folder)
except Exception as error:
log.debug('HELP! %s' % error.args)
args = str(error.args[0] if not isinstance(error.args, basestring)
else error.args)
log.debug('THIS IS %s' % type(args))
log.debug('Value of boolean: %s' % ("not found" in args))
if not ("cannot be established" in args or "not found" in args):
log.exception("Unknown error encountered whilst opening "
"database connection to '%s'" % database)
raise
if not create_function:
log.exception("Database does not exist, `create_function` not "
"specified")
raise
cool_folder = create_function(db)
if also_db:
return db, cool_folder
return cool_folder
@classmethod
def resolve_db_string(cls, db_string, read_only=True):
"""
Take a database string - if it looks like a filename ending in ".db"
then assume it is a sqlite database with that name.
If the `db_string` is prepended with "WRITE|" then a writable connection
is requested.
If the db_string doesn't contain a "/", then "/" + DEFAULT_DBNAME is
appended.
"""
if "://" in db_string:
# Assume that the string is already resolved
return db_string, read_only
if db_string.startswith("WRITE|"):
assert db_string.count("|") == 1, "Bad db_string format"
db_string = db_string.split("|")[1]
read_only = False
sqlite_regex = re.compile(
r"(?P<filename>.*?\.db)(?:/?(?P<dbname>[^/]+))?$")
matched = sqlite_regex.match(db_string)
if matched:
filename, dbname = matched.groups()
dbname = DEFAULT_DBNAME if not dbname else dbname
db_string = "sqlite://schema=%s;dbname=%s" % (filename, dbname)
else:
if "/" not in db_string:
return db_string + "/" + DEFAULT_DBNAME, read_only
return db_string, read_only
@classmethod
def get_instance(cls, db_string, read_only=True, create=False):
"""
Open a database connection
"""
res_db_string, read_only = cls.resolve_db_string(db_string, read_only)
try:
prev_stdout = sys.stdout
sys.stdout = StringIO()
try:
connection = indirectOpen(res_db_string,
readOnly=read_only,
oracle=True)
finally:
sys.stdout = prev_stdout
except Exception as e:
if "The database does not exist" in e.args[0] and not create:
log.info("Failed trying to connect to '%s'", res_db_string)
raise
from PyCool import cool
dbService = cool.DatabaseSvcFactory.databaseService()
connection = dbService.createDatabase(res_db_string)
except:
print sys.exc_info()[0]
raise
return connection
@classmethod
def build_folder(cls, db, folder_name, multiversion, record):
"""
Create `folder_name` on database instance `db`, with recordspecification
`record`.
Also creates folderset to which folder_name belongs if necessary.
"""
from PyCool import cool
folderset_path = dirname(folder_name)
try:
folder_set = db.getFolderSet(folderset_path)
except Exception, error:
caught_error = "Folder set %s not found" % folderset_path
if caught_error not in error.args[0]:
raise
log.debug("Folderset doesn't exist - creating it.")
db.createFolderSet(folderset_path, "", True)
if not isinstance(record, cool.RecordSpecification):
record_spec = cool.RecordSpecification()
for field in record:
record_spec.extend(*field)
else:
record_spec = record
FV = cool.FolderVersioning
versioning = FV.MULTI_VERSION if multiversion else FV.SINGLE_VERSION
folder_spec = cool.FolderSpecification(versioning, record_spec)
folder = db.createFolder(folder_name, folder_spec)
payload = cool.Record(record_spec)
return folder, payload
try:
#=== creating folder specifications
spec = cool.RecordSpecification()
spec.extend( 'CaloCondBlob16M', cool.StorageType.Blob16M )
#=== create the folder
folderPath = CaloCondTools.getCaloPrefix()+"LAR/TimeCorrectionOfl/NonRunCon"
folderTag = tagCon
print ("Filling COOL folder ", folderPath, " with tag ", folderTag )
desc = CaloCondTools.getAthenaFolderDescr()
try:
folder = db.getFolder(folderPath)
except Exception:
log.warning("Folder %s not found, creating it...", folderPath)
print ("Folder ", folderPath, " not found, creating it... " )
folderSpec = cool.FolderSpecification(cool.FolderVersioning.SINGLE_VERSION, spec)
folder = db.createFolder(folderPath, folderSpec, desc, True)
#==================================================
#=== Create the CaloCondBlobFlt objects
#==================================================
#=== default a and b to be used for each gain
gainDefVec_1 = g.std.vector('float')()
gainDefVec_1.push_back(0.) # febOffset+channelOffset
gainDefVec_1.push_back(0.) # energy depended error parameter 1
gainDefVec_1.push_back(0.) # energy depended error parameter 2
gainDefVec_1.push_back(0.) # feb depended energy correction parameter 1
gainDefVec_1.push_back(0.) # feb depended energy correction parameter 2
gainDefVec_1.push_back(0.) # feb depended energy correction parameter 3
gainDefVec_1.push_back(0.) # feb depended energy correction parameter 4
gainDefVec_1.push_back(0.) # feb depended energy correction parameter 5
#==================================================
#=== set shortcut
g = cppyy.gbl
#=== get a logger
log = TileCalibLogger.getLogger("WriteOfc")
#=== (re-)create the database
db = TileCalibTools.openDb('SQLITE', 'CONDBR2', 'UPDATE')
#=== creating folder specifications
spec = cool.RecordSpecification()
#=== Note special name '*Ofc' and 16M instead of 64k for OFCs
spec.extend( 'TileCalibBlobOfc', cool.StorageType.Blob16M )
folderMode = cool.FolderVersioning.SINGLE_VERSION
folderSpec = cool.FolderSpecification(folderMode, spec)
#=== loop over all input files
for fileName in [ofcFiles[0]]:
log.info( "Reading file: %s" % fileName )
#=== determine COOL folder from input file name
fileNameParts = re_fileName.search(fileName).groups()
ofMethod = fileNameParts[0]
ofRun = fileNameParts[1][:3]
#=== fix LED->LAS
if ofRun=="LED":
ofRun="LAS"
objVersion = int(ofMethod[2])
folderPath = TileCalibTools.getTilePrefix(False)+"FILTER/"+ofMethod+"/"+ofRun
log.info( "Filling COOL folder %s" % ( folderPath ) )
