def getOnlineDBReader(ym_tuple, granularity='YEARLY', connStrFunc=None):
cnstr = ''
ymstr = ''
if granularity == 'YEARLY':
ymstr = "%04d" % ym_tuple[0]
cnstr = connStrFunc((ym_tuple[0], 13))
else:
ymstr = "%04d%02d" % ym_tuple
cnstr = connStrFunc(ym_tuple)
ptnm = "ONLINE_" + ymstr
accSvc = CondDBAccessSvc(ptnm, ConnectionString=cnstr)
dblayers = [accSvc]
LoadCALIBDB = os.environ.get('LoadCALIBDB')
if ym_tuple[
0] < 2015 or LoadCALIBDB is not "OFFLINE": # For datatype before 2015, no CALIBOFF layer is needed
return accSvc
dbpath = os.environ["SQLITEDBPATH"]
layer = 'CALIBOFF'
if exists(join(dbpath, layer + '.db')):
# Put the discovered layer on top
cfg = getConfigurable(layer, CondDBAccessSvc)
try:
cfg.ConnectionString
except AttributeError: # Set up connection for the 1st time
cfg = CondDBAccessSvc("CALIBOFF",
ConnectionString=cnstr.replace(
'ONLINE-%s.db/ONLINE' % ymstr,
"%s.db/%s" % (layer, layer)),
CacheHighLevel=200)
dblayers.insert(0, cfg)
if (len(dblayers) == 1): return accSvc # In case no CALIBOFF.db is found
return CondDBLayeringSvc("ONLINELAYER_" + ymstr, Layers=dblayers)
def _action():
"""
Reset all DaVinci sequences
"""
from Gaudi.Configuration import allConfigurables
from Gaudi.Configuration import getConfigurable
for seq in ('DaVinciInitSeq', 'DaVinciMainSequence', 'DaVinciSequence',
'MonitoringSequence', 'FilteredEventSeq'):
if not seq in allConfigurables: continue
cSeq = getConfigurable(seq)
if cSeq and hasattr(cSeq, 'Members'):
logger.info('Reset the sequence %s' % cSeq.name())
cSeq.Members = []
## reset the list of top-level algorithms
from Configurables import ApplicationMgr
a = ApplicationMgr()
a.TopAlg = []
a.OutputLevel = options.OutputLevel
from Configurables import MessageSvc
m = MessageSvc(OutputLevel=options.OutputLevel)
from GaudiConf import IOHelper
ioh = IOHelper()
ioh.setupServices()
def _silent_action_ () :
from Gaudi.Configuration import allConfigurables
from Gaudi.Configuration import getConfigurable
keys = allConfigurables.keys()
s1 = set( keys )
s2 = set( lst )
s = s1.intersection()
for i in s :
c = getConfigurable( i )
if c and hasattr ( c , 'OutputLevel' ) :
c.OutputLevel = 4
def getAnyDBReader(layer='CALIBOFF', svc=CondDBAccessSvc):
CacheHighLevel = 200
if layer == 'DDDB': CacheHighLevel = 1700
# Put the discovered layer on top
cfg = getConfigurable(layer, svc)
if svc is not CondDBAccessSvc: return cfg
try:
cfg.ConnectionString
except AttributeError: # Set up connection for the 1st time
connstr = "sqlite_file:$SQLITEDBPATH/%s.db/%s" % (layer, layer)
if layer == 'DQFLAGS':
cfg = CondDBAccessSvc(layer,
ConnectionString=connstr,
CacheLowLevel=5,
CacheHighLevel=10)
else:
cfg = CondDBAccessSvc(layer,
ConnectionString=connstr,
CacheHighLevel=CacheHighLevel)
return cfg
def killDAQ(nodes=['/Event/DAQ', '/Event/pRec'], logger=None):
"""
Configure node killing agents for uDST processing
"""
#
if not logger: logger = log
#
from Configurables import EventNodeKiller
killer = EventNodeKiller("KillDAQ")
for node in nodes:
if node in killer.Nodes: continue
killer.Nodes.append(node)
try:
from Gaudi.Configuration import getConfigurable
conf = getConfigurable('DaVinciEventInitSeq')
conf.Members.insert(0, killer)
logger.info("Add killer agent for: %s" % killer.Nodes)
except:
logger.error("Failed to add killer agent for: %s" % killer.Nodes)
def __apply_configuration__(self):
"""
Converts the high-level information passed as properties into low-level configuration.
"""
# special case for online
if self.getProp('UseDBSnapshot'): self._configureDBSnapshot()
# In the Online/Upgrade/Simulation environment, LoadCALIBDB should be defaulted to HLT1
if self.getProp("Online") or self.getProp('Upgrade') or self.getProp(
'Simulation'):
self._properties["LoadCALIBDB"].setDefault("HLT1")
# Set up environment variables for loading CALIBOFF layers, must be before loading any tags
LoadCALIBDB = self.getProp('LoadCALIBDB')
loadcaliboptions = ["HLT1", "OFFLINE"]
if LoadCALIBDB not in loadcaliboptions:
raise ValueError(
"'%s' is not a valid LoadCALIBDB value. Allowed: %s" %
(LoadCALIBDB, loadcaliboptions))
if LoadCALIBDB is "OFFLINE" and not exists(
join(os.environ["SQLITEDBPATH"], "CALIBOFF.db")):
LoadCALIBDB = "HLT1" # When CALIBOFF.db is not there, reset the option
os.environ['LoadCALIBDB'] = LoadCALIBDB
# Set the usage of the latest global/local tags
old_latest_Tags_prop = self.getProp(
"UseLatestTags") # it is deprecated
latest_GTags_prop = self.getProp("LatestGlobalTagByDataTypes")
if not latest_GTags_prop: # if property not set
latest_GTags_prop = self.getProp("LatestGlobalTagByDataType")
latest_LTags_prop = self.getProp("LatestLocalTagsByDataType")
all_LTags_prop = self.getProp("AllLocalTagsByDataType")
if old_latest_Tags_prop:
if latest_GTags_prop or latest_LTags_prop:
log.warning(
"The property 'UseLatestTags' is deprecated:"
"'LatestGlobalTagByDataType(s)' and 'LatestLocalTagsByDataType'"
" will be used instead.")
else:
latest_GTags_prop = old_latest_Tags_prop[0]
if type(old_latest_Tags_prop[-1]) != bool or \
(type(old_latest_Tags_prop[-1]) == bool and not old_latest_Tags_prop[1]):
latest_LTags_prop = old_latest_Tags_prop[0]
if latest_GTags_prop:
datatype = latest_GTags_prop
if self.getProp("Tags"):
self.Tags = {}
self._useLatestTags(datatype, OnlyGlobalTags=True)
log.warning(
"Default global tags will be overridden with the latest ones"
" available for '%s' data type: %s" %
(datatype, self.getProp("Tags")))
if latest_LTags_prop:
datatypes = latest_LTags_prop
#if self.getProp("LocalTags"):
# self.LocalTags = {}
self._useLatestTags(datatypes, OnlyLocalTags=True)
log.warning(
"Latest unbound local tags on top of the latest global tags"
" of %s data type(s) are added: %s" %
(datatypes, self.getProp("LocalTags")))
if all_LTags_prop:
datatypes = all_LTags_prop
self._useAllLocalTags(datatypes)
log.warning("ALL local tags of %s data type(s) are added: %s" %
(datatypes, self.getProp("LocalTags")))
# Import SQLDDDB specific info
if self.getProp("UseOracle"):
CondDBAccessSvc("ONLINE", ConnectionString="CondDBOnline/ONLINE")
if self.getProp("DisableLFC"):
COOLConfSvc(UseLFCReplicaSvc=False)
elif self.getProp('UseDBSnapshot'):
CondDBAccessSvc("ONLINE")
else:
configureOnlineSnapshots()
# importOptions("$SQLDDDBROOT/options/SQLDDDB.py")
#########################################################################
# Access to ConditionsDB
##########################################################################
conns = self.getProp("PartitionConnectionString")
tags = self.getProp("Tags")
# DB partitions
partition = {}
parttypes = [("DDDB", CondDBAccessSvc), ("LHCBCOND", CondDBAccessSvc),
("ONLINE", CondDBTimeSwitchSvc),
("SIMCOND", CondDBAccessSvc),
("DQFLAGS", CondDBAccessSvc)]
if LoadCALIBDB is "OFFLINE":
# CALIBOFF not needed for the upgrade
parttypes += [("CALIBOFF", CondDBAccessSvc)]
for (p, t) in parttypes:
partition[p] = getAnyDBReader(p, t)
# Override connection strings:
if p in conns:
if type(partition[p]) is CondDBAccessSvc:
partition[p].ConnectionString = conns[p]
del conns[p]
# Override connection strings for Upgrade case
if self.getProp('Simulation') and self.getProp('Upgrade') and type(
partition[p]) is CondDBAccessSvc:
partition[p].ConnectionString = os.path.join(
'sqlite_file:$SQLITEUPGRADEDBPATH', p + '.db', p)
# Override tags
if p in tags and p != "ONLINE":
partition[p].DefaultTAG = tags[p]
del tags[p]
# Set the query granularity
if p != "CALIBOFF":
self.propagateProperty("QueryGranularity", partition[p])
if type(partition[p]) is CondDBTimeSwitchSvc: # also online
for r in partition[p].Readers:
config = allConfigurables[eval(
r.split(':')[0]).split("/")[1]]
if isinstance(config, CondDBAccessSvc):
self.propagateProperty("QueryGranularity", config)
# Pass along the configuration for the layered DBs
elif isinstance(config, CondDBLayeringSvc):
for ly in config.Layers:
if isinstance(ly, CondDBAccessSvc):
self.propagateProperty("QueryGranularity", ly)
if conns:
log.warning(
"Cannot override the connection strings of the partitions %r",
conns.keys())
if tags and tags.keys() != ['ONLINE']:
log.warning("Cannot set the tag for partitions %r", tags.keys())
# In the Online environment, IgnoreHeartBeat should be defaulted to True
if self.getProp("Online"):
self._properties["IgnoreHeartBeat"].setDefault(True)
if not self.getProp("IgnoreHeartBeat"):
if isinstance(partition["ONLINE"], CondDBAccessSvc):
self.propagateProperty("HeartBeatCondition",
partition["ONLINE"])
elif isinstance(partition["ONLINE"], CondDBTimeSwitchSvc):
# Add the heart beat conditions to the latest snapshot only since the
# others are limited but valid by construction.
if partition["ONLINE"].Readers:
latest = partition["ONLINE"].Readers[-1]
config = allConfigurables[eval(
latest.split(':')[0]).split("/")[1]]
if isinstance(config, CondDBAccessSvc):
self.propagateProperty("HeartBeatCondition", config)
# Pass along the configuration for the layered DBs
elif isinstance(config, CondDBLayeringSvc):
for ly in config.Layers:
#Only apply HeartBeatCondition for ONLINE
if isinstance(ly, CondDBAccessSvc) and ly.getName(
).startswith("ONLINE_"):
self.propagateProperty("HeartBeatCondition",
ly)
if not self.getProp("Simulation"):
# Standard configurations
# - Reconstruction / analysis
disp = CondDBDispatcherSvc("MainCondDBReader",
MainAccessSvc=partition["DDDB"],
Alternatives={
"/Conditions":
partition["LHCBCOND"],
"/Conditions/Online":
partition["ONLINE"],
"/Conditions/DQ":
partition["DQFLAGS"]
})
else:
# - Simulation
disp = CondDBDispatcherSvc(
"SimulationCondDBReader",
MainAccessSvc=partition["DDDB"],
Alternatives={"/Conditions": partition["SIMCOND"]})
CondDBCnvSvc(CondDBReader=disp)
if not (self.getProp("Online") or self.getProp("Simulation")):
self._properties["EnableRunStampCheck"].setDefault(True)
if self.getProp("EnableRunStampCheck"):
from Configurables import RunStampCheck
rsc = RunStampCheck()
self.propagateProperty("RunStampCondition", rsc)
ApplicationMgr().ExtSvc.append(rsc)
# Load the CALIBOFF layer above everything if it exists
# if len([x for x in parttypes if x[0] == 'CALIBOFF']):
# self._addLayer(getAnyDBReader('CALIBOFF'))
localTags = self.getProp("LocalTags")
not_applied = []
for p in localTags:
if p in partition:
taglist = list(localTags[p])
taglist.reverse(
) # we need to stack the in reverse order to use the first as on top of the others
i = 0 # counter
if p is "CALIBOFF":
if LoadCALIBDB is not "OFFLINE":
raise ValueError(
"invalid argument LoadCALIBDB set at '%s' instead of 'OFFLINE' for accessing local tags for CALIBOFF.db"
% LoadCALIBDB)
pcolayers = []
for t in taglist:
pcolayers.append(partition[p].clone("CALIBOFF_%d" % i,
DefaultTAG=t))
i += 1
for r in partition["ONLINE"].Readers:
config = allConfigurables[eval(
r.split(':')[0]).split("/")[1]]
if isinstance(config, CondDBLayeringSvc):
config.Layers = pcolayers + config.Layers
elif type(partition[p]) is not CondDBTimeSwitchSvc:
for t in taglist:
self._addLayer(partition[p].clone("%s_%d" % (p, i),
DefaultTAG=t))
i += 1
else:
not_applied.append(p)
else:
not_applied.append(p)
if not_applied:
log.warning("Cannot set the local tags for partitions %r",
not_applied)
# Modify partitions to use local copies of the DBs
# before adding user layers and alternatives, which should be already local.
# This is a no-operation if the property is not set
self.__make_sqlite_local_copy__(CondDBCnvSvc())
# Add layers and alternatives
call = {
self.LAYER: self._addLayer,
self.ALTERNATIVE: self._addAlternative
}
for override in self.getProp("Overrides"):
apply(call[override[0]], override[1:])
# Add the logger
filename = self.getProp("LogFile")
if filename:
cnvSvc = allConfigurables["CondDBCnvSvc"]
cnvSvc.CondDBReader = CondDBLogger(
LoggedReader=cnvSvc.CondDBReader, LogFile=filename)
# Suppress pointless warning from COOL_2_5_0
msgSvc = getConfigurable("MessageSvc")
msgSvc.setError.append("RelationalDatabase")
# Set up Virtual File System service, can be used by ParticlePropertySvc
from Gaudi.Configuration import VFSSvc
from Configurables import CondDBEntityResolver
VFSSvc().FileAccessTools.append(CondDBEntityResolver())
def __make_sqlite_local_copy__(self,
accsvc,
local_dir=None,
force_copy=None):
if isinstance(accsvc, str):
# convert the string in an actual configurable instance
# This is both for backward compatibility and CondDBTimeSwitchSvc
if "/" in accsvc:
tp, name = accsvc.split("/", 1)
else:
tp = name = accsvc
accsvc = getConfigurable(name, tp)
if local_dir is None:
local_dir = self.getProp("SQLiteLocalCopiesDir")
if force_copy is None:
force_copy = self.getProp("OverwriteSQLiteLocalCopy")
# If the directory for the local copies is not specified, we do nothing
if not local_dir:
return accsvc
# Check if we are using Oracle or SQLite
# if self.getProp("UseOracle"):
# log.warning("Conflicting properties in CondDB Configurable: "
# "ignoring SQLiteLocalCopiesDir because UseOracle is set to True")
# return accsvc
# Modify partitions to use local copies of the DBs
newaccsvc = accsvc # fallback return value (no change)
if isinstance(accsvc, CondDBAccessSvc):
# replace the reader with another
m = re.match(r"^sqlite_file:(.*)/([_0-9A-Z]{1,8})$",
accsvc.getProp("ConnectionString"))
if not m: # not SQLite connection string
return accsvc
newaccsvc = CondDBSQLiteCopyAccSvc(accsvc.name() + "_local")
newaccsvc.OriginalFile = m.group(1)
newaccsvc.DestinationFile = os.path.join(
local_dir, os.path.basename(m.group(1)))
newaccsvc.DBName = m.group(2)
newaccsvc.ForceCopy = force_copy
newaccsvc.IgnoreCopyError = not force_copy # ignore copy errors if we do not overwrite (needed for local tags)
if hasattr(accsvc, "CacheHighLevel"):
newaccsvc.CacheHighLevel = accsvc.CacheHighLevel
elif isinstance(accsvc, CondDBDispatcherSvc):
# use the same dispatcher replacing its content
mainAccSvc = accsvc.getProp("MainAccessSvc")
accsvc.MainAccessSvc = self.__make_sqlite_local_copy__(
mainAccSvc, local_dir)
alternatives = accsvc.getProp("Alternatives")
for alt in alternatives.keys():
accsvc.Alternatives[alt] = \
self.__make_sqlite_local_copy__(alternatives[alt], local_dir)
elif isinstance(accsvc, CondDBLayeringSvc):
# use the same layering service replacing its content
new_layers = []
for layer in accsvc.getProp("Layers"):
new_layers.append(
self.__make_sqlite_local_copy__(layer, local_dir))
accsvc.Layers = new_layers
elif isinstance(accsvc, CondDBTimeSwitchSvc):
# use the same time switcher replacing its content,
# but we need to parse its options (in format "'%s':(%d,%d)")
readers_list = accsvc.getProp("Readers")
new_readers = []
for line in readers_list:
# Parse the line for the reader (it looks like "'name':(0,1)")
r, iov = map(eval, line.rsplit(":"))
new_reader = self.__make_sqlite_local_copy__(r, local_dir)
new_readers.append("'%s':(%d,%d)" %
(new_reader.getFullName(), iov[0], iov[1]))
accsvc.Readers = new_readers
elif isinstance(accsvc, CondDBLogger):
# use the same logger replacing its content
logged = accsvc.getProp("LoggedReader")
accsvc.LoggedReader = self.__make_sqlite_local_copy__(
logged, local_dir)
elif isinstance(accsvc, CondDBCnvSvc):
# use the same conversion service replacing its content
reader = accsvc.getProp("CondDBReader")
accsvc.CondDBReader = self.__make_sqlite_local_copy__(
reader, local_dir)
return newaccsvc