def writeOptions(self, opts, run, activity, task):
import Online.Streaming.StreamingDescriptor as StreamingDescriptor
import Online.Streaming.PartitionInfo as PartitionInfo
import Online.SetupParams as Params
if not self.streamInfo:
system_mgr = Systems.controlsMgr(Params.storage_system_name)
res = StreamingDescriptor.findPartition(system_mgr,'Storage',run.name)
if res is not None:
dp, nam, id = res
self.streamInfo = PartitionInfo.PartitionInfo(system_mgr,nam).load()
if not self.streamInfo:
error('###\n### Cannot access Stream Slice for partition:'+run.name+'\n###',
timestamp=1,type=PVSS.ILLEGAL_VALUE)
return None
if self.hostType == 'RECV':
if self.checkTasks(opts,run,activity,task,self.streamInfo.recvSenders()):
return self
if self.checkTasks(opts,run,activity,task,self.streamInfo.recvReceivers()):
return self
if self.checkTasks(opts,run,activity,task,self.streamInfo.recvInfrastructure()):
return self
elif self.hostType == 'STREAM':
if self.checkTasks(opts,run,activity,task,self.streamInfo.streamReceivers()):
return self
if self.checkTasks(opts,run,activity,task,self.streamInfo.streamSenders()):
return self
if self.checkTasks(opts,run,activity,task,self.streamInfo.streamInfrastructure()):
return self
error('###\n### UNKNOWN task requires options: '+task.name+' type:'+self.hostType+'\n###',
timestamp=1,type=PVSS.ILLEGAL_VALUE)
return None
def _mgr(name):
"Access PVSS controls manager by name"
import Online.PVSS as PVSS;
import Online.PVSSSystems as Systems
if name is None:
return PVSS.controlsMgr()
return Systems.controlsMgr(name)
def __init__(self, manager, name, info):
"Object constructor."
OptionsWriter.__init__(self, manager, 'HLT', name, info)
devMgr = self.manager.deviceMgr()
typ = self.manager.typeMgr().type('JobOptionsControl')
self.storageMgr = Systems.controlsMgr(Params.storage_system_name)
log(name+': Uses the managers '+self.manager.name()+' and '+self.storageMgr.name(),timestamp=1)
for i in xrange(16):
nam = self.manager.name()+':'+name+'Writer_%02X'%(i,)
if not DataPoint.exists(nam):
if not devMgr.createDevice(nam,typ,1).release():
error('Failed to create the datapoint:"'+nam+'"',timestamp=1,type=PVSS.DP_NOT_EXISTENT)
else:
log('Created the datapoint:"'+nam+'"',timestamp=1)
def __init__(self, manager, hosttype, cluster, name, info):
"Object constructor."
self.hostType = hosttype
self.cluster = cluster
self.optionsDir = None
self.infoCreator = info
CommandListener.__init__(self,manager,name,name,4)
devMgr = self.manager.deviceMgr()
self.optionsMgr = Systems.controlsMgr(Params.jobopts_system_name)
if not devMgr.exists(name):
typ = self.manager.typeMgr().type('JobOptionsControl')
if not devMgr.createDevice(name,typ,1).release():
error('Failed to create the datapoint:"'+name+'"',timestamp=1,type=PVSS.DP_NOT_EXISTENT)
else:
log('Created the datapoint:"'+name+'"',timestamp=1)
def _startupInfo(self, task_typ):
" Return startup script of a given task type."
script = Params.gauditask_startscript
account = 'online'
detector = 'ANY'
if not hasattr(self, 'startupInfoData'):
self.startupInfoData = {}
# first look for defaults
info = self._collectScriptPath(
Systems.controlsMgr(Params.jobopts_system_name))
for name, path, account, detector in info:
self.startupInfoData[name] = (path, account, detector)
# now look in project specific values/overrides
info = self._collectScriptPath(self.manager)
for name, path.account, detector in info:
self.startupInfoData[name] = (path, account, detector)
if self.startupInfoData.has_key(task_typ):
return self.startupInfoData[task_typ]
return (script, account, detector)
def writeOptions(self, opts, info, activity, task):
import Online.Streaming.StreamingDescriptor as StreamingDescriptor
import Online.Streaming.PartitionInfo as PartitionInfo
import Online.SetupParams as Params
if not self.streamInfo:
system_mgr = Systems.controlsMgr(Params.monitor_system_name)
res = StreamingDescriptor.findPartition(system_mgr,'Monitoring',info.name)
if res is not None:
dp, nam, id = res
self.streamInfo = PartitionInfo.PartitionInfo(system_mgr,nam).load()
if not self.streamInfo:
error('###\n### Cannot access Stream Slice for partition:'+info.name+'\n###',
timestamp=1,type=PVSS.ILLEGAL_VALUE)
return None
done = 0
if self.hostType == 'MONRELAY':
done = done + self.checkTasks(opts,info,activity,task,self.streamInfo.dataSources())
done = done + self.checkTasks(opts,info,activity,task,self.streamInfo.recvReceivers())
done = done + self.checkTasks(opts,info,activity,task,self.streamInfo.recvSenders())
done = done + self.checkTasks(opts,info,activity,task,self.streamInfo.recvInfrastructure())
elif self.hostType == 'MONI':
done = done + self.checkTasks(opts,info,activity,task,self.streamInfo.streamReceivers())
done = done + self.checkTasks(opts,info,activity,task,self.streamInfo.streamSenders())
done = done + self.checkTasks(opts,info,activity,task,self.streamInfo.streamInfrastructure())
if done > 0: return self
self.printTasks(self.streamInfo.dataSources())
self.printTasks(self.streamInfo.recvReceivers())
self.printTasks(self.streamInfo.recvSenders())
self.printTasks(self.streamInfo.recvInfrastructure())
self.printTasks(self.streamInfo.streamReceivers())
self.printTasks(self.streamInfo.streamSenders())
self.printTasks(self.streamInfo.streamInfrastructure())
error('### UNKNOWN task requires options: '+task.name+' type:'+self.hostType+'\n###',
timestamp=1,type=PVSS.ILLEGAL_VALUE)
return None
def _mgr(name):
"Access PVSS controls manager by name"
import Online.PVSSSystems as Systems
return Systems.controlsMgr(name)
def create(self,rundp_name,partition):
items = rundp_name.split(':')
print 'Creating StorageInfoCreator',items[0],rundp_name,partition
mgr = Systems.controlsMgr(items[0])
return StorageInfo(mgr,items[1])
def install(name, system=Params.monitor_system_name):
c = Systems.controlsMgr(system)
inst = Installer(c, name).create()
return c
def _mgr(name):
"Access PVSS controls manager by name"
return Systems.controlsMgr(name)
def create(self, rundp_name, partition):
items = rundp_name.split(':')
mgr = Systems.controlsMgr(items[0])
sto = Systems.controlsMgr(Params.storage_system_name)
return Monitoring(mgr, partition, self.storage, sto)
def defineTasks(self, partition):
"""
Define all tasks in the storage layer for a given partition.
The result is storen in runInfo datapoints for further processing.
"""
relay_slots = partition.recvSlices()
mon_slots = partition.streamSlices()
monNodes = partition.streamNodesFromSlots()
relayNodes = partition.recvNodesFromSlots()
dimDns = self.manager.hostName()
opt = '/' + dimDns + '/' + partition.manager.name(
) + '/' + partition.name + '/'
cl0 = '/Class0' + opt
cl1 = '/Class1' + opt
cl2 = '/Class2' + opt
streams = []
jo_mgr = Systems.controlsMgr(Params.jobopts_system_name)
#log('Detectors in readout:'+str(self.detectorsInReadout()))
monStreams = set()
mon_types = self.monitoringTypes()
mon_mult = self.monitoringMultiplicity()
mon_stream = self.monitoringStreams()
for j in xrange(len(mon_types)):
for i in xrange(mon_mult[j]):
typ = mon_types[j]
task = JobOptions.TaskType(jo_mgr, typ)
if task.exists():
# Check if the detector is in the readout
partID = self.partID.data
detector = task.getDetector()
det_used = self.isDetectorInReadout(detector)
if detector.upper() == 'ANY' or det_used is not None:
monStreams.add(mon_stream[j])
streams.append([typ, mon_stream[j], i])
if detector.upper() == 'ANY':
log('USE monitoring task:%-16s [Task started for all detectors]'
% (typ, ))
else:
log('USE monitoring task:%-16s [Detector %-8s is in the readout]:%s'
% (typ, detector, str(det_used)))
else:
log('SKIP monitoring task:%-16s [Detector %-8s is NOT in the readout]'
% (
typ,
detector,
))
else:
error(
'The task ' + typ +
' is not defined in the job options database. Failed to define monitoring streams.'
)
return None
monTasks = []
monStreamsByNode = {}
monStreamsByType = {}
#print 'Slots:',mon_slots, mon_slots.size()
for typ, strm, i in streams:
slot_name = mon_slots[len(monTasks)]
node = slot_name[:slot_name.find(':')]
item = typ + ('_%02d' % i)
if not monStreamsByNode.has_key(node):
monStreamsByNode[node] = set([])
monStreamsByNode[node].add(strm)
if not monStreamsByType.has_key(strm):
monStreamsByType[strm] = set([])
monStreamsByType[strm].add(node)
task = node + '/' + self.name + '_' + node + '_' + item + '/' + item + '/' + typ + cl1 + str(
(strm, ))
monTasks.append(task)
res = StreamDescriptor(self.streamManager,
self.storage).getPartition(self.name)
if not res:
error('Failed to access stream information for partition ' +
self.name + ' [' + str(self.storage) + ']')
error(
'Are you sure the storage slice is allocated for partition ' +
self.name + ' ?')
return None
dp, part_name, slot = res
part = PartitionInfo(self.streamManager, part_name)
part.load()
recv_nodes = part.recvNodes()
relayReceivers = []
relayReceiverTypes = {}
for i in monStreams:
relayReceiverTypes[i] = []
# Define the receiver tasks on the relay
for j in relayNodes:
task = [j, self.name + '_' + j + '_RCV' + i, i]
relayReceivers.append(task)
relayReceiverTypes[i].append(task)
# Define the receiver tasks on the relay for each stream type
relayTargets = {}
for i, nodes in relayReceiverTypes.items():
relayTargets[i] = []
for j in nodes:
relayTargets[i].append((j[0] + '-d1', j[1]))
storageSenders = []
storageSenderTypes = {}
stor_opt = '/Class2/' + dimDns + '/' + self.streamManager.name(
) + '/' + part.name + '/'
for i in monStreams:
storageSenderTypes[i] = []
# Define the sender tasks on the storage layer
for j in recv_nodes:
nam = self.name + '_' + j + '_SND' + i
task = j + '/' + nam + '/SND' + i + '/SND' + i + stor_opt + str(
relayTargets[i])
storageSenders.append(task)
storageSenderTypes[i].append([j, nam, i])
# Define the sources of the receiver tasks for the senders on the storage layer
storageSources = {}
for i in storageSenderTypes.keys():
storageSources[i] = []
for j in storageSenderTypes[i]:
storageSources[i].append((j[0] + '-d1', j[1]))
# Define the receiver tasks on monitoring nodes
# Only send events to node with known stream consumers!
monReceivers = []
monTargets = {}
for i in monNodes:
if monStreamsByNode.has_key(i):
nam = self.name + '_' + i + '_RCVMon'
monReceivers.append([i, nam])
monTargets[i] = [(i + '-d1', nam)]
# Define the sender tasks on the relay to feed monitoring nodes
relaySenders = []
monSources = {}
# print 'Relay Nodes',relayNodes
for i, nodes in monStreamsByType.items():
rcv = 0
# print 'MonNodes:',nodes
for k in nodes:
if monStreamsByNode.has_key(
k) and monStreamsByNode[k].issuperset([i]):
if not monSources.has_key(k): monSources[k] = []
for j in relayNodes:
nick = 'SND' + i + '_' + k
nam = self.name + '_' + j + '_' + nick
task = j + '/' + nam + '/' + nick + '/SND' + i + cl2 + str(
monTargets[k])
monSources[k].append((j + '-d1', nam))
relaySenders.append(task)
# Fill all datapoints:
# First the senders on the storage system
partition.setDataSources(storageSenders)
# Update Relay infrastructure
infra = []
for j in relayNodes:
for i in self.relayInfra.data:
infra.append(j + '/' + self.name + '_' + j + '_' + i + '/' +
i + '/' + i + cl0 + '("' + i + '",)')
partition.setRecvInfrastructure(infra)
# Update task information for the receivers on the relays
rcv = []
for i in relayReceivers:
node, name, type = i
rcv.append(node + '/' + name + '/RCV' + type + '/RCV' + type +
cl1 + str(storageSources[i[2]]))
partition.setRecvReceivers(rcv)
# Update relay receivers
partition.setRecvSenders(relaySenders)
# Update Monfarm infrastructure
infra = []
for j in monNodes:
for i in self.monInfra.data:
infra.append(j + '/' + self.name + '_' + j + '_' + i + '/' +
i + '/' + i + cl0 + '("' + i + '",)')
partition.setStreamInfrastructure(infra)
# Update task information for the receivers on the monitoring farm
rcv = []
for i in monReceivers:
node, nam = i
rcv.append(node + '/' + nam + '/RCVMon/RCVMon' + cl1 +
str(monSources[node]))
partition.setStreamReceivers(rcv)
partition.setStreamSenders(monTasks)
if partition.saveTasks():
tasks = partition.collectTasks(tasks={}, with_data_sources=1)
return tasks
return None
def create(self, rundp_name, partition):
items = rundp_name.split(':')
mgr = Systems.controlsMgr(items[0])
return HLTFarmInfo(mgr, partition)
def create(self, rundp_name, partition):
items = rundp_name.split(':')
# partition = items[1][:items[1].find('_')]
mgr = Systems.controlsMgr(items[0])
return FarmFeedInfo(mgr, partition)
def dump(self):
self.srv.info().dump()
#PVSS.setDebug(1)
def cycle(status):
status.fill()
status.dump()
status.publish()
time.sleep(10)
partition = 'LHCb'
if len(sys.argv) > 1: partition = sys.argv[1]
print 'Analysing partition', partition
mgr = Systems.controlsMgr('ECS')
status = LHCbStatus(mgr, partition)
status.connect()
cycle(status)
status.srv.start()
#time.sleep(3)
#cycle(status)
for i in xrange(5000):
time.sleep(3)
status.swapBuffers()
cycle(status)
JOI.install()
JOI.uninstall()
"""
import Online.JobOptions.JobOptions as JobOptions
import Online.SetupParams as Params
import Online.PVSSSystems as Systems
import Online.RunInfoClasses.Storage as StorageInfo
import Online.Utils as Utils
import Online.PVSS as PVSS
DataPoint = PVSS.DataPoint
log = Utils.log
mgr = Systems.controlsMgr(Params.jobopts_system_name)
JobOptions.debug = 1
# =============================================================================
class Task(JobOptions.TaskType):
def __init__(self, name, opts, defs=1, tell1=0):
JobOptions.TaskType.__init__(self, mgr, name)
if not self.exists():
self.create()
self.options.data = opts
self.defaults.data = defs
self.tell1s.data = tell1
self.save()
def _mgr(self):
mgr = Systems.controlsMgr(self.name)
return mgr
def runDataflow():
import Online.PVSSSystems as Systems
mgr = DisplayManager(Systems.controlsMgr(PVSS.defaultSystemName())).run()
managers = [mgr]
sleep(managers)
def __init__(self, manager, name, info): streamMgr = Systems.controlsMgr(Params.monitor_system_name) StreamingOptionsWriter.__init__(self, manager, 'MONRELAY', name, info, streamMgr, ['MONRELAY','MONI'])
def __init__(self, manager, name, info): streamMgr = Systems.controlsMgr(Params.storage_system_name) StreamingOptionsWriter.__init__(self, manager, 'RECV', name, info, streamMgr, ['RECV','STREAM'])
def optionsManager(self):
if self.optionsMgr is None:
self.optionsMgr = Systems.controlsMgr(Params.jobopts_system_name)
return self.optionsMgr
