def writeOptions(self, opts, run, activity, task): import Online.Streaming.StreamingDescriptor as StreamingDescriptor import Online.Streaming.PartitionInfo as PartitionInfo import Online.SetupParams as Params if not self.streamInfo: system_mgr = Systems.controlsMgr(Params.storage_system_name) res = StreamingDescriptor.findPartition(system_mgr,'Storage',run.name) if res is not None: dp, nam, id = res self.streamInfo = PartitionInfo.PartitionInfo(system_mgr,nam).load() if not self.streamInfo: error('###\n### Cannot access Stream Slice for partition:'+run.name+'\n###', timestamp=1,type=PVSS.ILLEGAL_VALUE) return None if self.hostType == 'RECV': if self.checkTasks(opts,run,activity,task,self.streamInfo.recvSenders()): return self if self.checkTasks(opts,run,activity,task,self.streamInfo.recvReceivers()): return self if self.checkTasks(opts,run,activity,task,self.streamInfo.recvInfrastructure()): return self elif self.hostType == 'STREAM': if self.checkTasks(opts,run,activity,task,self.streamInfo.streamReceivers()): return self if self.checkTasks(opts,run,activity,task,self.streamInfo.streamSenders()): return self if self.checkTasks(opts,run,activity,task,self.streamInfo.streamInfrastructure()): return self error('###\n### UNKNOWN task requires options: '+task.name+' type:'+self.hostType+'\n###', timestamp=1,type=PVSS.ILLEGAL_VALUE) return None
def _mgr(name): "Access PVSS controls manager by name" import Online.PVSS as PVSS; import Online.PVSSSystems as Systems if name is None: return PVSS.controlsMgr() return Systems.controlsMgr(name)
def __init__(self, manager, name, info): "Object constructor." OptionsWriter.__init__(self, manager, 'HLT', name, info) devMgr = self.manager.deviceMgr() typ = self.manager.typeMgr().type('JobOptionsControl') self.storageMgr = Systems.controlsMgr(Params.storage_system_name) log(name+': Uses the managers '+self.manager.name()+' and '+self.storageMgr.name(),timestamp=1) for i in xrange(16): nam = self.manager.name()+':'+name+'Writer_%02X'%(i,) if not DataPoint.exists(nam): if not devMgr.createDevice(nam,typ,1).release(): error('Failed to create the datapoint:"'+nam+'"',timestamp=1,type=PVSS.DP_NOT_EXISTENT) else: log('Created the datapoint:"'+nam+'"',timestamp=1)
def __init__(self, manager, hosttype, cluster, name, info): "Object constructor." self.hostType = hosttype self.cluster = cluster self.optionsDir = None self.infoCreator = info CommandListener.__init__(self,manager,name,name,4) devMgr = self.manager.deviceMgr() self.optionsMgr = Systems.controlsMgr(Params.jobopts_system_name) if not devMgr.exists(name): typ = self.manager.typeMgr().type('JobOptionsControl') if not devMgr.createDevice(name,typ,1).release(): error('Failed to create the datapoint:"'+name+'"',timestamp=1,type=PVSS.DP_NOT_EXISTENT) else: log('Created the datapoint:"'+name+'"',timestamp=1)
def _startupInfo(self, task_typ): " Return startup script of a given task type." script = Params.gauditask_startscript account = 'online' detector = 'ANY' if not hasattr(self, 'startupInfoData'): self.startupInfoData = {} # first look for defaults info = self._collectScriptPath( Systems.controlsMgr(Params.jobopts_system_name)) for name, path, account, detector in info: self.startupInfoData[name] = (path, account, detector) # now look in project specific values/overrides info = self._collectScriptPath(self.manager) for name, path.account, detector in info: self.startupInfoData[name] = (path, account, detector) if self.startupInfoData.has_key(task_typ): return self.startupInfoData[task_typ] return (script, account, detector)
def writeOptions(self, opts, info, activity, task): import Online.Streaming.StreamingDescriptor as StreamingDescriptor import Online.Streaming.PartitionInfo as PartitionInfo import Online.SetupParams as Params if not self.streamInfo: system_mgr = Systems.controlsMgr(Params.monitor_system_name) res = StreamingDescriptor.findPartition(system_mgr,'Monitoring',info.name) if res is not None: dp, nam, id = res self.streamInfo = PartitionInfo.PartitionInfo(system_mgr,nam).load() if not self.streamInfo: error('###\n### Cannot access Stream Slice for partition:'+info.name+'\n###', timestamp=1,type=PVSS.ILLEGAL_VALUE) return None done = 0 if self.hostType == 'MONRELAY': done = done + self.checkTasks(opts,info,activity,task,self.streamInfo.dataSources()) done = done + self.checkTasks(opts,info,activity,task,self.streamInfo.recvReceivers()) done = done + self.checkTasks(opts,info,activity,task,self.streamInfo.recvSenders()) done = done + self.checkTasks(opts,info,activity,task,self.streamInfo.recvInfrastructure()) elif self.hostType == 'MONI': done = done + self.checkTasks(opts,info,activity,task,self.streamInfo.streamReceivers()) done = done + self.checkTasks(opts,info,activity,task,self.streamInfo.streamSenders()) done = done + self.checkTasks(opts,info,activity,task,self.streamInfo.streamInfrastructure()) if done > 0: return self self.printTasks(self.streamInfo.dataSources()) self.printTasks(self.streamInfo.recvReceivers()) self.printTasks(self.streamInfo.recvSenders()) self.printTasks(self.streamInfo.recvInfrastructure()) self.printTasks(self.streamInfo.streamReceivers()) self.printTasks(self.streamInfo.streamSenders()) self.printTasks(self.streamInfo.streamInfrastructure()) error('### UNKNOWN task requires options: '+task.name+' type:'+self.hostType+'\n###', timestamp=1,type=PVSS.ILLEGAL_VALUE) return None
def _mgr(name): "Access PVSS controls manager by name" import Online.PVSSSystems as Systems return Systems.controlsMgr(name)
def create(self,rundp_name,partition): items = rundp_name.split(':') print 'Creating StorageInfoCreator',items[0],rundp_name,partition mgr = Systems.controlsMgr(items[0]) return StorageInfo(mgr,items[1])
def install(name, system=Params.monitor_system_name): c = Systems.controlsMgr(system) inst = Installer(c, name).create() return c
def _mgr(name): "Access PVSS controls manager by name" return Systems.controlsMgr(name)
def create(self, rundp_name, partition): items = rundp_name.split(':') mgr = Systems.controlsMgr(items[0]) sto = Systems.controlsMgr(Params.storage_system_name) return Monitoring(mgr, partition, self.storage, sto)
def defineTasks(self, partition): """ Define all tasks in the storage layer for a given partition. The result is storen in runInfo datapoints for further processing. """ relay_slots = partition.recvSlices() mon_slots = partition.streamSlices() monNodes = partition.streamNodesFromSlots() relayNodes = partition.recvNodesFromSlots() dimDns = self.manager.hostName() opt = '/' + dimDns + '/' + partition.manager.name( ) + '/' + partition.name + '/' cl0 = '/Class0' + opt cl1 = '/Class1' + opt cl2 = '/Class2' + opt streams = [] jo_mgr = Systems.controlsMgr(Params.jobopts_system_name) #log('Detectors in readout:'+str(self.detectorsInReadout())) monStreams = set() mon_types = self.monitoringTypes() mon_mult = self.monitoringMultiplicity() mon_stream = self.monitoringStreams() for j in xrange(len(mon_types)): for i in xrange(mon_mult[j]): typ = mon_types[j] task = JobOptions.TaskType(jo_mgr, typ) if task.exists(): # Check if the detector is in the readout partID = self.partID.data detector = task.getDetector() det_used = self.isDetectorInReadout(detector) if detector.upper() == 'ANY' or det_used is not None: monStreams.add(mon_stream[j]) streams.append([typ, mon_stream[j], i]) if detector.upper() == 'ANY': log('USE monitoring task:%-16s [Task started for all detectors]' % (typ, )) else: log('USE monitoring task:%-16s [Detector %-8s is in the readout]:%s' % (typ, detector, str(det_used))) else: log('SKIP monitoring task:%-16s [Detector %-8s is NOT in the readout]' % ( typ, detector, )) else: error( 'The task ' + typ + ' is not defined in the job options database. Failed to define monitoring streams.' ) return None monTasks = [] monStreamsByNode = {} monStreamsByType = {} #print 'Slots:',mon_slots, mon_slots.size() for typ, strm, i in streams: slot_name = mon_slots[len(monTasks)] node = slot_name[:slot_name.find(':')] item = typ + ('_%02d' % i) if not monStreamsByNode.has_key(node): monStreamsByNode[node] = set([]) monStreamsByNode[node].add(strm) if not monStreamsByType.has_key(strm): monStreamsByType[strm] = set([]) monStreamsByType[strm].add(node) task = node + '/' + self.name + '_' + node + '_' + item + '/' + item + '/' + typ + cl1 + str( (strm, )) monTasks.append(task) res = StreamDescriptor(self.streamManager, self.storage).getPartition(self.name) if not res: error('Failed to access stream information for partition ' + self.name + ' [' + str(self.storage) + ']') error( 'Are you sure the storage slice is allocated for partition ' + self.name + ' ?') return None dp, part_name, slot = res part = PartitionInfo(self.streamManager, part_name) part.load() recv_nodes = part.recvNodes() relayReceivers = [] relayReceiverTypes = {} for i in monStreams: relayReceiverTypes[i] = [] # Define the receiver tasks on the relay for j in relayNodes: task = [j, self.name + '_' + j + '_RCV' + i, i] relayReceivers.append(task) relayReceiverTypes[i].append(task) # Define the receiver tasks on the relay for each stream type relayTargets = {} for i, nodes in relayReceiverTypes.items(): relayTargets[i] = [] for j in nodes: relayTargets[i].append((j[0] + '-d1', j[1])) storageSenders = [] storageSenderTypes = {} stor_opt = '/Class2/' + dimDns + '/' + self.streamManager.name( ) + '/' + part.name + '/' for i in monStreams: storageSenderTypes[i] = [] # Define the sender tasks on the storage layer for j in recv_nodes: nam = self.name + '_' + j + '_SND' + i task = j + '/' + nam + '/SND' + i + '/SND' + i + stor_opt + str( relayTargets[i]) storageSenders.append(task) storageSenderTypes[i].append([j, nam, i]) # Define the sources of the receiver tasks for the senders on the storage layer storageSources = {} for i in storageSenderTypes.keys(): storageSources[i] = [] for j in storageSenderTypes[i]: storageSources[i].append((j[0] + '-d1', j[1])) # Define the receiver tasks on monitoring nodes # Only send events to node with known stream consumers! monReceivers = [] monTargets = {} for i in monNodes: if monStreamsByNode.has_key(i): nam = self.name + '_' + i + '_RCVMon' monReceivers.append([i, nam]) monTargets[i] = [(i + '-d1', nam)] # Define the sender tasks on the relay to feed monitoring nodes relaySenders = [] monSources = {} # print 'Relay Nodes',relayNodes for i, nodes in monStreamsByType.items(): rcv = 0 # print 'MonNodes:',nodes for k in nodes: if monStreamsByNode.has_key( k) and monStreamsByNode[k].issuperset([i]): if not monSources.has_key(k): monSources[k] = [] for j in relayNodes: nick = 'SND' + i + '_' + k nam = self.name + '_' + j + '_' + nick task = j + '/' + nam + '/' + nick + '/SND' + i + cl2 + str( monTargets[k]) monSources[k].append((j + '-d1', nam)) relaySenders.append(task) # Fill all datapoints: # First the senders on the storage system partition.setDataSources(storageSenders) # Update Relay infrastructure infra = [] for j in relayNodes: for i in self.relayInfra.data: infra.append(j + '/' + self.name + '_' + j + '_' + i + '/' + i + '/' + i + cl0 + '("' + i + '",)') partition.setRecvInfrastructure(infra) # Update task information for the receivers on the relays rcv = [] for i in relayReceivers: node, name, type = i rcv.append(node + '/' + name + '/RCV' + type + '/RCV' + type + cl1 + str(storageSources[i[2]])) partition.setRecvReceivers(rcv) # Update relay receivers partition.setRecvSenders(relaySenders) # Update Monfarm infrastructure infra = [] for j in monNodes: for i in self.monInfra.data: infra.append(j + '/' + self.name + '_' + j + '_' + i + '/' + i + '/' + i + cl0 + '("' + i + '",)') partition.setStreamInfrastructure(infra) # Update task information for the receivers on the monitoring farm rcv = [] for i in monReceivers: node, nam = i rcv.append(node + '/' + nam + '/RCVMon/RCVMon' + cl1 + str(monSources[node])) partition.setStreamReceivers(rcv) partition.setStreamSenders(monTasks) if partition.saveTasks(): tasks = partition.collectTasks(tasks={}, with_data_sources=1) return tasks return None
def create(self, rundp_name, partition): items = rundp_name.split(':') mgr = Systems.controlsMgr(items[0]) return HLTFarmInfo(mgr, partition)
def create(self, rundp_name, partition): items = rundp_name.split(':') # partition = items[1][:items[1].find('_')] mgr = Systems.controlsMgr(items[0]) return FarmFeedInfo(mgr, partition)
def dump(self): self.srv.info().dump() #PVSS.setDebug(1) def cycle(status): status.fill() status.dump() status.publish() time.sleep(10) partition = 'LHCb' if len(sys.argv) > 1: partition = sys.argv[1] print 'Analysing partition', partition mgr = Systems.controlsMgr('ECS') status = LHCbStatus(mgr, partition) status.connect() cycle(status) status.srv.start() #time.sleep(3) #cycle(status) for i in xrange(5000): time.sleep(3) status.swapBuffers() cycle(status)
JOI.install() JOI.uninstall() """ import Online.JobOptions.JobOptions as JobOptions import Online.SetupParams as Params import Online.PVSSSystems as Systems import Online.RunInfoClasses.Storage as StorageInfo import Online.Utils as Utils import Online.PVSS as PVSS DataPoint = PVSS.DataPoint log = Utils.log mgr = Systems.controlsMgr(Params.jobopts_system_name) JobOptions.debug = 1 # ============================================================================= class Task(JobOptions.TaskType): def __init__(self, name, opts, defs=1, tell1=0): JobOptions.TaskType.__init__(self, mgr, name) if not self.exists(): self.create() self.options.data = opts self.defaults.data = defs self.tell1s.data = tell1 self.save()
def _mgr(self): mgr = Systems.controlsMgr(self.name) return mgr
def runDataflow(): import Online.PVSSSystems as Systems mgr = DisplayManager(Systems.controlsMgr(PVSS.defaultSystemName())).run() managers = [mgr] sleep(managers)
def __init__(self, manager, name, info): streamMgr = Systems.controlsMgr(Params.monitor_system_name) StreamingOptionsWriter.__init__(self, manager, 'MONRELAY', name, info, streamMgr, ['MONRELAY','MONI'])
def __init__(self, manager, name, info): streamMgr = Systems.controlsMgr(Params.storage_system_name) StreamingOptionsWriter.__init__(self, manager, 'RECV', name, info, streamMgr, ['RECV','STREAM'])
def optionsManager(self): if self.optionsMgr is None: self.optionsMgr = Systems.controlsMgr(Params.jobopts_system_name) return self.optionsMgr