Exemplo n.º 1
0
 def writeOptions(self, opts, run, activity, task):
   import Online.Streaming.StreamingDescriptor as StreamingDescriptor
   import Online.Streaming.PartitionInfo as PartitionInfo
   import Online.SetupParams as Params
   if not self.streamInfo:
     system_mgr = Systems.controlsMgr(Params.storage_system_name)
     res = StreamingDescriptor.findPartition(system_mgr,'Storage',run.name)
     if res is not None:
       dp, nam, id = res
       self.streamInfo = PartitionInfo.PartitionInfo(system_mgr,nam).load()
   if not self.streamInfo:
     error('###\n###   Cannot access Stream Slice for partition:'+run.name+'\n###',
           timestamp=1,type=PVSS.ILLEGAL_VALUE)
     return None
   if self.hostType == 'RECV':
     if self.checkTasks(opts,run,activity,task,self.streamInfo.recvSenders()):
       return self
     if self.checkTasks(opts,run,activity,task,self.streamInfo.recvReceivers()):
       return self
     if self.checkTasks(opts,run,activity,task,self.streamInfo.recvInfrastructure()):
       return self
   elif self.hostType == 'STREAM':
     if self.checkTasks(opts,run,activity,task,self.streamInfo.streamReceivers()):
       return self
     if self.checkTasks(opts,run,activity,task,self.streamInfo.streamSenders()):
       return self
     if self.checkTasks(opts,run,activity,task,self.streamInfo.streamInfrastructure()):
       return self
   error('###\n###   UNKNOWN task requires options: '+task.name+' type:'+self.hostType+'\n###',
         timestamp=1,type=PVSS.ILLEGAL_VALUE)
   return None
Exemplo n.º 2
0
def _mgr(name):
  "Access PVSS controls manager by name"
  import Online.PVSS as PVSS;
  import Online.PVSSSystems as Systems
  if name is None:
    return PVSS.controlsMgr()
  return Systems.controlsMgr(name)
Exemplo n.º 3
0
 def __init__(self, manager, name, info):
   "Object constructor."
   OptionsWriter.__init__(self, manager, 'HLT', name, info)
   devMgr = self.manager.deviceMgr()
   typ = self.manager.typeMgr().type('JobOptionsControl')
   self.storageMgr = Systems.controlsMgr(Params.storage_system_name)
   log(name+': Uses the managers '+self.manager.name()+' and '+self.storageMgr.name(),timestamp=1)
   for i in xrange(16):
     nam = self.manager.name()+':'+name+'Writer_%02X'%(i,)
     if not DataPoint.exists(nam):
       if not devMgr.createDevice(nam,typ,1).release():
         error('Failed to create the datapoint:"'+nam+'"',timestamp=1,type=PVSS.DP_NOT_EXISTENT)
       else:
         log('Created the datapoint:"'+nam+'"',timestamp=1)
Exemplo n.º 4
0
 def __init__(self, manager, hosttype, cluster, name, info):
   "Object constructor."
   self.hostType = hosttype
   self.cluster  = cluster
   self.optionsDir = None
   self.infoCreator = info
   CommandListener.__init__(self,manager,name,name,4)
   devMgr = self.manager.deviceMgr()
   self.optionsMgr = Systems.controlsMgr(Params.jobopts_system_name)
   if not devMgr.exists(name):
     typ = self.manager.typeMgr().type('JobOptionsControl')
     if not devMgr.createDevice(name,typ,1).release():
       error('Failed to create the datapoint:"'+name+'"',timestamp=1,type=PVSS.DP_NOT_EXISTENT)
     else:
       log('Created the datapoint:"'+name+'"',timestamp=1)
Exemplo n.º 5
0
 def _startupInfo(self, task_typ):
     " Return startup script of a given task type."
     script = Params.gauditask_startscript
     account = 'online'
     detector = 'ANY'
     if not hasattr(self, 'startupInfoData'):
         self.startupInfoData = {}
         # first look for defaults
         info = self._collectScriptPath(
             Systems.controlsMgr(Params.jobopts_system_name))
         for name, path, account, detector in info:
             self.startupInfoData[name] = (path, account, detector)
         # now look in project specific values/overrides
         info = self._collectScriptPath(self.manager)
         for name, path.account, detector in info:
             self.startupInfoData[name] = (path, account, detector)
     if self.startupInfoData.has_key(task_typ):
         return self.startupInfoData[task_typ]
     return (script, account, detector)
Exemplo n.º 6
0
  def writeOptions(self, opts, info, activity, task):
    import Online.Streaming.StreamingDescriptor as StreamingDescriptor
    import Online.Streaming.PartitionInfo as PartitionInfo
    import Online.SetupParams as Params
    if not self.streamInfo:
      system_mgr = Systems.controlsMgr(Params.monitor_system_name)
      res = StreamingDescriptor.findPartition(system_mgr,'Monitoring',info.name)
      if res is not None:
        dp, nam, id = res
        self.streamInfo = PartitionInfo.PartitionInfo(system_mgr,nam).load()
    if not self.streamInfo:
      error('###\n###   Cannot access Stream Slice for partition:'+info.name+'\n###',
            timestamp=1,type=PVSS.ILLEGAL_VALUE)
      return None
    done = 0
    if self.hostType == 'MONRELAY':
      done = done + self.checkTasks(opts,info,activity,task,self.streamInfo.dataSources())
      done = done + self.checkTasks(opts,info,activity,task,self.streamInfo.recvReceivers())
      done = done + self.checkTasks(opts,info,activity,task,self.streamInfo.recvSenders())
      done = done + self.checkTasks(opts,info,activity,task,self.streamInfo.recvInfrastructure())
    elif self.hostType == 'MONI':
      done = done + self.checkTasks(opts,info,activity,task,self.streamInfo.streamReceivers())
      done = done + self.checkTasks(opts,info,activity,task,self.streamInfo.streamSenders())
      done = done + self.checkTasks(opts,info,activity,task,self.streamInfo.streamInfrastructure())
    if done > 0: return self

    self.printTasks(self.streamInfo.dataSources())
    self.printTasks(self.streamInfo.recvReceivers())
    self.printTasks(self.streamInfo.recvSenders())
    self.printTasks(self.streamInfo.recvInfrastructure())
    self.printTasks(self.streamInfo.streamReceivers())
    self.printTasks(self.streamInfo.streamSenders())
    self.printTasks(self.streamInfo.streamInfrastructure())
    error('### UNKNOWN task requires options: '+task.name+' type:'+self.hostType+'\n###',
          timestamp=1,type=PVSS.ILLEGAL_VALUE)
    return None
Exemplo n.º 7
0
def _mgr(name):
    "Access PVSS controls manager by name"
    import Online.PVSSSystems as Systems
    return Systems.controlsMgr(name)
Exemplo n.º 8
0
 def create(self,rundp_name,partition):
   items = rundp_name.split(':')
   print 'Creating StorageInfoCreator',items[0],rundp_name,partition
   mgr = Systems.controlsMgr(items[0])
   return StorageInfo(mgr,items[1])
Exemplo n.º 9
0
def install(name, system=Params.monitor_system_name):
    c = Systems.controlsMgr(system)
    inst = Installer(c, name).create()
    return c
Exemplo n.º 10
0
def _mgr(name):
    "Access PVSS controls manager by name"
    return Systems.controlsMgr(name)
Exemplo n.º 11
0
 def create(self, rundp_name, partition):
     items = rundp_name.split(':')
     mgr = Systems.controlsMgr(items[0])
     sto = Systems.controlsMgr(Params.storage_system_name)
     return Monitoring(mgr, partition, self.storage, sto)
Exemplo n.º 12
0
    def defineTasks(self, partition):
        """
         Define all tasks in the storage layer for a given partition.
         The result is storen in runInfo datapoints for further processing.
    """

        relay_slots = partition.recvSlices()
        mon_slots = partition.streamSlices()
        monNodes = partition.streamNodesFromSlots()
        relayNodes = partition.recvNodesFromSlots()
        dimDns = self.manager.hostName()
        opt = '/' + dimDns + '/' + partition.manager.name(
        ) + '/' + partition.name + '/'
        cl0 = '/Class0' + opt
        cl1 = '/Class1' + opt
        cl2 = '/Class2' + opt

        streams = []
        jo_mgr = Systems.controlsMgr(Params.jobopts_system_name)
        #log('Detectors in readout:'+str(self.detectorsInReadout()))
        monStreams = set()
        mon_types = self.monitoringTypes()
        mon_mult = self.monitoringMultiplicity()
        mon_stream = self.monitoringStreams()
        for j in xrange(len(mon_types)):
            for i in xrange(mon_mult[j]):
                typ = mon_types[j]
                task = JobOptions.TaskType(jo_mgr, typ)
                if task.exists():
                    # Check if the detector is in the readout
                    partID = self.partID.data
                    detector = task.getDetector()
                    det_used = self.isDetectorInReadout(detector)
                    if detector.upper() == 'ANY' or det_used is not None:
                        monStreams.add(mon_stream[j])
                        streams.append([typ, mon_stream[j], i])
                        if detector.upper() == 'ANY':
                            log('USE  monitoring task:%-16s [Task started for all detectors]'
                                % (typ, ))
                        else:
                            log('USE  monitoring task:%-16s [Detector %-8s is in the readout]:%s'
                                % (typ, detector, str(det_used)))
                    else:
                        log('SKIP monitoring task:%-16s [Detector %-8s is NOT in the readout]'
                            % (
                                typ,
                                detector,
                            ))
                else:
                    error(
                        'The task ' + typ +
                        ' is not defined in the job options database. Failed to define monitoring streams.'
                    )
                    return None

        monTasks = []
        monStreamsByNode = {}
        monStreamsByType = {}
        #print 'Slots:',mon_slots, mon_slots.size()
        for typ, strm, i in streams:
            slot_name = mon_slots[len(monTasks)]
            node = slot_name[:slot_name.find(':')]
            item = typ + ('_%02d' % i)
            if not monStreamsByNode.has_key(node):
                monStreamsByNode[node] = set([])
            monStreamsByNode[node].add(strm)
            if not monStreamsByType.has_key(strm):
                monStreamsByType[strm] = set([])
            monStreamsByType[strm].add(node)
            task = node + '/' + self.name + '_' + node + '_' + item + '/' + item + '/' + typ + cl1 + str(
                (strm, ))
            monTasks.append(task)

        res = StreamDescriptor(self.streamManager,
                               self.storage).getPartition(self.name)
        if not res:
            error('Failed to access stream information for partition ' +
                  self.name + ' [' + str(self.storage) + ']')
            error(
                'Are you sure the storage slice is allocated for partition ' +
                self.name + ' ?')
            return None

        dp, part_name, slot = res
        part = PartitionInfo(self.streamManager, part_name)
        part.load()
        recv_nodes = part.recvNodes()

        relayReceivers = []
        relayReceiverTypes = {}
        for i in monStreams:
            relayReceiverTypes[i] = []
            # Define the receiver tasks on the relay
            for j in relayNodes:
                task = [j, self.name + '_' + j + '_RCV' + i, i]
                relayReceivers.append(task)
                relayReceiverTypes[i].append(task)

        # Define the receiver tasks on the relay for each stream type
        relayTargets = {}
        for i, nodes in relayReceiverTypes.items():
            relayTargets[i] = []
            for j in nodes:
                relayTargets[i].append((j[0] + '-d1', j[1]))

        storageSenders = []
        storageSenderTypes = {}
        stor_opt = '/Class2/' + dimDns + '/' + self.streamManager.name(
        ) + '/' + part.name + '/'
        for i in monStreams:
            storageSenderTypes[i] = []
            # Define the sender tasks on the storage layer
            for j in recv_nodes:
                nam = self.name + '_' + j + '_SND' + i
                task = j + '/' + nam + '/SND' + i + '/SND' + i + stor_opt + str(
                    relayTargets[i])
                storageSenders.append(task)
                storageSenderTypes[i].append([j, nam, i])

        # Define the sources of the receiver tasks for the senders on the storage layer
        storageSources = {}
        for i in storageSenderTypes.keys():
            storageSources[i] = []
            for j in storageSenderTypes[i]:
                storageSources[i].append((j[0] + '-d1', j[1]))

        # Define the receiver tasks on monitoring nodes
        # Only send events to node with known stream consumers!
        monReceivers = []
        monTargets = {}
        for i in monNodes:
            if monStreamsByNode.has_key(i):
                nam = self.name + '_' + i + '_RCVMon'
                monReceivers.append([i, nam])
                monTargets[i] = [(i + '-d1', nam)]

        # Define the sender tasks on the relay to feed monitoring nodes
        relaySenders = []
        monSources = {}
        # print 'Relay Nodes',relayNodes
        for i, nodes in monStreamsByType.items():
            rcv = 0
            # print 'MonNodes:',nodes
            for k in nodes:
                if monStreamsByNode.has_key(
                        k) and monStreamsByNode[k].issuperset([i]):
                    if not monSources.has_key(k): monSources[k] = []
                    for j in relayNodes:
                        nick = 'SND' + i + '_' + k
                        nam = self.name + '_' + j + '_' + nick
                        task = j + '/' + nam + '/' + nick + '/SND' + i + cl2 + str(
                            monTargets[k])
                        monSources[k].append((j + '-d1', nam))
                        relaySenders.append(task)

        # Fill all datapoints:

        # First the senders on the storage system
        partition.setDataSources(storageSenders)

        # Update Relay infrastructure
        infra = []
        for j in relayNodes:
            for i in self.relayInfra.data:
                infra.append(j + '/' + self.name + '_' + j + '_' + i + '/' +
                             i + '/' + i + cl0 + '("' + i + '",)')
        partition.setRecvInfrastructure(infra)

        # Update task information for the receivers on the relays
        rcv = []
        for i in relayReceivers:
            node, name, type = i
            rcv.append(node + '/' + name + '/RCV' + type + '/RCV' + type +
                       cl1 + str(storageSources[i[2]]))
        partition.setRecvReceivers(rcv)

        # Update relay receivers
        partition.setRecvSenders(relaySenders)

        # Update Monfarm infrastructure
        infra = []
        for j in monNodes:
            for i in self.monInfra.data:
                infra.append(j + '/' + self.name + '_' + j + '_' + i + '/' +
                             i + '/' + i + cl0 + '("' + i + '",)')
        partition.setStreamInfrastructure(infra)

        # Update task information for the receivers on the monitoring farm
        rcv = []
        for i in monReceivers:
            node, nam = i
            rcv.append(node + '/' + nam + '/RCVMon/RCVMon' + cl1 +
                       str(monSources[node]))
        partition.setStreamReceivers(rcv)

        partition.setStreamSenders(monTasks)
        if partition.saveTasks():
            tasks = partition.collectTasks(tasks={}, with_data_sources=1)
            return tasks
        return None
Exemplo n.º 13
0
 def create(self, rundp_name, partition):
     items = rundp_name.split(':')
     mgr = Systems.controlsMgr(items[0])
     return HLTFarmInfo(mgr, partition)
Exemplo n.º 14
0
 def create(self, rundp_name, partition):
     items = rundp_name.split(':')
     # partition = items[1][:items[1].find('_')]
     mgr = Systems.controlsMgr(items[0])
     return FarmFeedInfo(mgr, partition)
Exemplo n.º 15
0
    def dump(self):
        self.srv.info().dump()


#PVSS.setDebug(1)


def cycle(status):
    status.fill()
    status.dump()
    status.publish()


time.sleep(10)
partition = 'LHCb'
if len(sys.argv) > 1: partition = sys.argv[1]
print 'Analysing partition', partition
mgr = Systems.controlsMgr('ECS')
status = LHCbStatus(mgr, partition)
status.connect()

cycle(status)
status.srv.start()
#time.sleep(3)
#cycle(status)

for i in xrange(5000):
    time.sleep(3)
    status.swapBuffers()
    cycle(status)
Exemplo n.º 16
0
JOI.install()

JOI.uninstall()

"""

import Online.JobOptions.JobOptions as JobOptions
import Online.SetupParams as Params
import Online.PVSSSystems as Systems
import Online.RunInfoClasses.Storage as StorageInfo
import Online.Utils as Utils
import Online.PVSS as PVSS

DataPoint = PVSS.DataPoint
log = Utils.log
mgr = Systems.controlsMgr(Params.jobopts_system_name)
JobOptions.debug = 1


# =============================================================================
class Task(JobOptions.TaskType):
    def __init__(self, name, opts, defs=1, tell1=0):
        JobOptions.TaskType.__init__(self, mgr, name)
        if not self.exists():
            self.create()
        self.options.data = opts
        self.defaults.data = defs
        self.tell1s.data = tell1
        self.save()

Exemplo n.º 17
0
 def _mgr(self):
     mgr = Systems.controlsMgr(self.name)
     return mgr
Exemplo n.º 18
0
def runDataflow():
    import Online.PVSSSystems as Systems
    mgr = DisplayManager(Systems.controlsMgr(PVSS.defaultSystemName())).run()
    managers = [mgr]
    sleep(managers)
Exemplo n.º 19
0
 def __init__(self, manager, name, info):
   streamMgr = Systems.controlsMgr(Params.monitor_system_name)
   StreamingOptionsWriter.__init__(self, manager, 'MONRELAY', name, info, streamMgr, ['MONRELAY','MONI'])
Exemplo n.º 20
0
 def __init__(self, manager, name, info):
   streamMgr = Systems.controlsMgr(Params.storage_system_name)
   StreamingOptionsWriter.__init__(self, manager, 'RECV', name, info, streamMgr, ['RECV','STREAM'])
Exemplo n.º 21
0
 def optionsManager(self):
   if self.optionsMgr is None:
     self.optionsMgr = Systems.controlsMgr(Params.jobopts_system_name)
   return self.optionsMgr