Example #1
0
 def _updatePilotSummary(self, sitesData):
     result = RPCClient(
         "WorkloadManagement/WMSAdministrator").getPilotSummary()
     if not result['OK']:
         gLogger.error("Cannot get the pilot summary", result['Message'])
         return result
     summaryData = result['Value']
     result = getCESiteMapping()
     if not result['OK']:
         return S_ERROR('Could not get CE site mapping')
     ceMapping = result['Value']
     pilotSummary = {}
     for ce in summaryData:
         if ce not in ceMapping:
             continue
         ceData = summaryData[ce]
         siteName = ceMapping[ce]
         if siteName not in pilotSummary:
             pilotSummary[siteName] = {}
         for status in ceData:
             if status not in pilotSummary[siteName]:
                 pilotSummary[siteName][status] = 0
             pilotSummary[siteName][status] += ceData[status]
     for site in sitesData:
         if site in pilotSummary:
             sitesData[site]['pilotSummary'] = pilotSummary[site]
     return S_OK(sitesData)
Example #2
0
 def _updatePilotSummary( self, sitesData ):
   result = RPCClient( "WorkloadManagement/WMSAdministrator" ).getPilotSummary()
   if not result[ 'OK' ]:
     gLogger.error( "Cannot get the pilot summary", result['Message'] )
     return result
   summaryData = result[ 'Value' ]
   result = getCESiteMapping()
   if not result['OK']:
     return S_ERROR( 'Could not get CE site mapping' )
   ceMapping = result[ 'Value' ]
   pilotSummary = {}
   for ce in summaryData:
     if ce not in ceMapping:
       continue
     ceData = summaryData[ ce ]
     siteName = ceMapping[ ce ]
     if siteName not in pilotSummary:
       pilotSummary[ siteName ] = {}
     for status in ceData:
       if status not in pilotSummary[ siteName ]:
         pilotSummary[ siteName ][ status ] = 0
       pilotSummary[ siteName ][ status ] += ceData[ status ]
   for site in sitesData:
     if site in pilotSummary:
       sitesData[ site ][ 'pilotSummary' ] = pilotSummary[ site ]
   return S_OK( sitesData )
Example #3
0
def getARCPilotOutput( proxy, pilotRef ):
  tmp_dir = mkdtemp()
  myce = pilotRef.split(":")[1].strip("/")
  gridEnv = getGridEnv()
  mySite = getCESiteMapping()['Value'][myce]
  workDB = gConfig.getValue(cfgPath('Resources/Sites/LCG', mySite, 'CEs', myce, 'JobListFile'))
  myWorkDB = os.path.join("/opt/dirac/runit/WorkloadManagement/SiteDirector-RAL", workDB)
  cmd = [ 'arcget' ]
  cmd.extend( ['-k', '-c', myce, '-j', myWorkDB, '-D', tmp_dir, pilotRef] )
  ret = executeGridCommand( proxy, cmd, gridEnv )
  if not ret['OK']:
    shutil.rmtree( tmp_dir )
    return ret
  status, output, error = ret['Value']
  if 'Results stored at:' in output :
    tmp_dir = os.path.join( tmp_dir, os.listdir( tmp_dir )[0] )
    result = S_OK()
    result['FileList'] = os.listdir( tmp_dir )
    for filename in result['FileList']:
      tmpname = os.path.join( tmp_dir, filename )
      if os.path.exists( tmpname ):
        myfile = file( tmpname, 'r' )
        f = myfile.read()
        myfile.close()
      else :
        f = ' '
      if ".out" in filename:
        filename = 'StdOut'
      if ".err" in filename:
        filename = 'StdErr'
      result[filename] = f
    shutil.rmtree( tmp_dir )
    return result
  if 'Warning: Job not found in job list' in output:
    shutil.rmtree( tmp_dir )
    message = "Pilot not yet visible in ARC dB"
    return S_ERROR( message )
  return S_ERROR("Sorry - requested pilot output not yet available")
Example #4
0
  def getPilotSummaryWeb(self,selectDict,sortList,startItem,maxItems):
    """ Get summary of the pilot jobs status by CE/site in a standard structure
    """

    stateNames = ['Submitted','Ready','Scheduled','Waiting','Running','Done','Aborted']
    allStateNames = stateNames + ['Done_Empty','Aborted_Hour']
    paramNames = ['Site','CE']+allStateNames

    resultDict = {}
    last_update = None
    if selectDict.has_key('LastUpdateTime'):
      last_update = selectDict['LastUpdateTime']
      del selectDict['LastUpdateTime']
    site_select = []
    if selectDict.has_key('GridSite'):
      site_select = selectDict['GridSite']
      if type(site_select) != type([]):
        site_select = [site_select]
      del selectDict['GridSite']

    status_select = []
    if selectDict.has_key('Status'):
      status_select = selectDict['Status']
      if type(status_select) != type([]):
        status_select = [status_select]
      del selectDict['Status']

    expand_site = ''
    if selectDict.has_key('ExpandSite'):
      expand_site = selectDict['ExpandSite']
      site_select = [expand_site]
      del selectDict['ExpandSite']

    start = time.time()
    # Get all the data from the database with various selections
    result = self.getCounters('PilotAgents',
                              ['GridSite','DestinationSite','Status'],
                              selectDict,newer=last_update,timeStamp='LastUpdateTime')
    if not result['OK']:
      return result

    last_update = Time.dateTime() - Time.hour
    selectDict['Status'] = 'Aborted'
    resultHour = self.getCounters('PilotAgents',
                                 ['GridSite','DestinationSite','Status'],
                                 selectDict,newer=last_update,timeStamp='LastUpdateTime')
    if not resultHour['OK']:
      return resultHour

    last_update = Time.dateTime() - Time.day
    selectDict['Status'] = ['Aborted','Done']
    resultDay = self.getCounters('PilotAgents',
                                 ['GridSite','DestinationSite','Status'],
                                 selectDict,newer=last_update,timeStamp='LastUpdateTime')
    if not resultDay['OK']:
      return resultDay
    selectDict['CurrentJobID'] = 0
    selectDict['Status'] = 'Done'
    resultDayEmpty = self.getCounters('PilotAgents',
                                 ['GridSite','DestinationSite','Status'],
                                 selectDict,newer=last_update,timeStamp='LastUpdateTime')
    if not resultDayEmpty['OK']:
      return resultDayEmpty

    ceMap = {}
    resMap = getCESiteMapping()
    if resMap['OK']:
      ceMap = resMap['Value']

    # Sort out different counters
    resultDict = {}
    resultDict['Unknown']={}
    for attDict,count in result['Value']:
      site = attDict['GridSite']
      ce = attDict['DestinationSite']
      state = attDict['Status']
      if site == 'Unknown' and ce != "Unknown" and ce != "Multiple" and ceMap.has_key(ce):
        site = ceMap[ce]
      if not resultDict.has_key(site):
        resultDict[site] = {}
      if not resultDict[site].has_key(ce):
        resultDict[site][ce] = {}
        for p in allStateNames:
          resultDict[site][ce][p] = 0

      resultDict[site][ce][state] = count

    for attDict,count in resultDay['Value']:
      site = attDict['GridSite']
      ce = attDict['DestinationSite']
      state = attDict['Status']
      if site == 'Unknown' and ce != "Unknown" and ceMap.has_key(ce):
        site = ceMap[ce]
      if state == "Done":
        resultDict[site][ce]["Done"] = count
      if state == "Aborted":
        resultDict[site][ce]["Aborted"] = count

    for attDict,count in resultDayEmpty['Value']:
      site = attDict['GridSite']
      ce = attDict['DestinationSite']
      state = attDict['Status']
      if site == 'Unknown' and ce != "Unknown" and ceMap.has_key(ce):
        site = ceMap[ce]
      if state == "Done":
        resultDict[site][ce]["Done_Empty"] = count

    for attDict,count in resultHour['Value']:
      site = attDict['GridSite']
      ce = attDict['DestinationSite']
      state = attDict['Status']
      if site == 'Unknown' and ce != "Unknown" and ceMap.has_key(ce):
        site = ceMap[ce]
      if state == "Aborted":
        resultDict[site][ce]["Aborted_Hour"] = count

    records = []
    siteSumDict = {}
    for site in resultDict:
      sumDict = {}
      for state in allStateNames:
        if not sumDict.has_key(state):
          sumDict[state] = 0
      sumDict['Total'] = 0
      for ce in resultDict[site]:
        itemList = [site,ce]
        total = 0
        for state in allStateNames:
          itemList.append(resultDict[site][ce][state])
          sumDict[state] += resultDict[site][ce][state]
          if state == "Done":
            done = resultDict[site][ce][state]
          if state == "Done_Empty":
            empty = resultDict[site][ce][state]
          if state == "Aborted":
            aborted = resultDict[site][ce][state]
          if state == "Aborted_Hour":
            aborted_hour = resultDict[site][ce][state]
          if state != "Aborted_Hour" and state != "Done_Empty":
            total += resultDict[site][ce][state]

        sumDict['Total'] += total
        # Add the total number of pilots seen in the last day
        itemList.append(total)
        # Add pilot submission efficiency evaluation
        if (done-empty) > 0:
          eff = float(done)/float(done-empty)
        elif done == 0:
          eff = 0.
        elif empty == done:
          eff = 99.
        else:
          eff = 0.
        itemList.append('%.2f' % eff)
        # Add pilot job efficiency evaluation
        if total > 0:
          eff = float(total-aborted)/float(total)*100.
        else:
          eff = 100.
        itemList.append('%.2f' % eff)

        # Evaluate the quality status of the CE
        if total > 10:
          if eff < 25.:
            itemList.append('Bad')
          elif eff < 60.:
            itemList.append('Poor')
          elif eff < 85.:
            itemList.append('Fair')
          else:
            itemList.append('Good')
        else:
          itemList.append('Idle')

        if len(resultDict[site]) == 1 or expand_site:
          records.append(itemList)

      if len(resultDict[site]) > 1 and not expand_site:
        itemList = [site,'Multiple']
        for state in allStateNames+['Total']:
          if sumDict.has_key(state):
            itemList.append(sumDict[state])
          else:
            itemList.append(0)
        done = sumDict["Done"]
        empty = sumDict["Done_Empty"]
        aborted = sumDict["Aborted"]
        aborted_hour = sumDict["Aborted_Hour"]
        total = sumDict["Total"]

        # Add pilot submission efficiency evaluation
        if (done-empty) > 0:
          eff = float(done)/float(done-empty)
        elif done == 0:
          eff = 0.
        elif empty == done:
          eff = 99.
        else:
          eff = 0.
        itemList.append('%.2f' % eff)
        # Add pilot job efficiency evaluation
        if total > 0:
          eff = float(total-aborted)/float(total)*100.
        else:
          eff = 100.
        itemList.append('%.2f' % eff)

        # Evaluate the quality status of the Site
        if total > 10:
          if eff < 25.:
            itemList.append('Bad')
          elif eff < 60.:
            itemList.append('Poor')
          elif eff < 85.:
            itemList.append('Fair')
          else:
            itemList.append('Good')
        else:
          itemList.append('Idle')
        records.append(itemList)

      for state in allStateNames+['Total']:
        if not siteSumDict.has_key(state):
          siteSumDict[state] = sumDict[state]
        else:
          siteSumDict[state] += sumDict[state]

    # Perform site selection
    if site_select:
      new_records = []
      for r in records:
        if r[0] in site_select:
          new_records.append(r)
      records = new_records

    # Perform status selection
    if status_select:
      new_records = []
      for r in records:
        if r[14] in status_select:
          new_records.append(r)
      records = new_records

    # Get the Site Mask data
    client = RPCClient('WorkloadManagement/WMSAdministrator')
    result = client.getSiteMask()
    if result['OK']:
      siteMask = result['Value']
      for r in records:
        if r[0] in siteMask:
          r.append('Yes')
        else:
          r.append('No')
    else:
      for r in records:
        r.append('Unknown')

    finalDict = {}
    finalDict['TotalRecords'] = len(records)
    finalDict['ParameterNames'] = paramNames+ \
                                 ['Total','PilotsPerJob','PilotJobEff','Status','InMask']

    # Return all the records if maxItems == 0 or the specified number otherwise
    if maxItems:
      finalDict['Records'] = records[startItem:startItem+maxItems]
    else:
      finalDict['Records'] = records

    done = siteSumDict["Done"]
    empty = siteSumDict["Done_Empty"]
    aborted = siteSumDict["Aborted"]
    aborted_hour = siteSumDict["Aborted_Hour"]
    total = siteSumDict["Total"]

    # Add pilot submission efficiency evaluation
    if (done-empty) > 0:
      eff = float(done)/float(done-empty)
    elif done == 0:
      eff = 0.
    elif empty == done:
      eff = 99.
    else:
      eff = 0.
    siteSumDict['PilotsPerJob'] = '%.2f' % eff
    # Add pilot job efficiency evaluation
    if total > 0:
      eff = float(total-aborted)/float(total)*100.
    else:
      eff = 100.
    siteSumDict['PilotJobEff'] = '%.2f' % eff

    # Evaluate the overall quality status
    if total > 100:
      if eff < 25.:
        siteSumDict['Status'] = 'Bad'
      elif eff < 60.:
        siteSumDict['Status'] = 'Poor'
      elif eff < 85.:
        siteSumDict['Status'] = 'Fair'
      else:
        siteSumDict['Status'] = 'Good'
    else:
      siteSumDict['Status'] = 'Idle'
    finalDict['Extras'] = siteSumDict

    return S_OK(finalDict)
Example #5
0
    def getPilotSummaryWeb(self, selectDict, sortList, startItem, maxItems):
        """ Get summary of the pilot jobs status by CE/site in a standard structure
    """

        stateNames = [
            'Submitted', 'Ready', 'Scheduled', 'Waiting', 'Running', 'Done',
            'Aborted', 'Failed'
        ]
        allStateNames = stateNames + ['Done_Empty', 'Aborted_Hour']
        paramNames = ['Site', 'CE'] + allStateNames

        last_update = None
        if 'LastUpdateTime' in selectDict:
            last_update = selectDict['LastUpdateTime']
            del selectDict['LastUpdateTime']
        site_select = []
        if 'GridSite' in selectDict:
            site_select = selectDict['GridSite']
            if not isinstance(site_select, type([])):
                site_select = [site_select]
            del selectDict['GridSite']

        status_select = []
        if 'Status' in selectDict:
            status_select = selectDict['Status']
            if not isinstance(status_select, type([])):
                status_select = [status_select]
            del selectDict['Status']

        expand_site = ''
        if 'ExpandSite' in selectDict:
            expand_site = selectDict['ExpandSite']
            site_select = [expand_site]
            del selectDict['ExpandSite']

        # Get all the data from the database with various selections
        result = self.getCounters('PilotAgents',
                                  ['GridSite', 'DestinationSite', 'Status'],
                                  selectDict,
                                  newer=last_update,
                                  timeStamp='LastUpdateTime')
        if not result['OK']:
            return result

        last_update = Time.dateTime() - Time.hour
        selectDict['Status'] = 'Aborted'
        resultHour = self.getCounters(
            'PilotAgents', ['GridSite', 'DestinationSite', 'Status'],
            selectDict,
            newer=last_update,
            timeStamp='LastUpdateTime')
        if not resultHour['OK']:
            return resultHour

        last_update = Time.dateTime() - Time.day
        selectDict['Status'] = ['Aborted', 'Done']
        resultDay = self.getCounters('PilotAgents',
                                     ['GridSite', 'DestinationSite', 'Status'],
                                     selectDict,
                                     newer=last_update,
                                     timeStamp='LastUpdateTime')
        if not resultDay['OK']:
            return resultDay
        selectDict['CurrentJobID'] = 0
        selectDict['Status'] = 'Done'
        resultDayEmpty = self.getCounters(
            'PilotAgents', ['GridSite', 'DestinationSite', 'Status'],
            selectDict,
            newer=last_update,
            timeStamp='LastUpdateTime')
        if not resultDayEmpty['OK']:
            return resultDayEmpty

        ceMap = {}
        resMap = getCESiteMapping()
        if resMap['OK']:
            ceMap = resMap['Value']

        # Sort out different counters
        resultDict = {}
        resultDict['Unknown'] = {}
        for attDict, count in result['Value']:
            site = attDict['GridSite']
            ce = attDict['DestinationSite']
            state = attDict['Status']
            if site == 'Unknown' and ce != "Unknown" and ce != "Multiple" and ce in ceMap:
                site = ceMap[ce]
            if site not in resultDict:
                resultDict[site] = {}
            if ce not in resultDict[site]:
                resultDict[site][ce] = {}
                for p in allStateNames:
                    resultDict[site][ce][p] = 0

            resultDict[site][ce][state] = count

        for attDict, count in resultDay['Value']:
            site = attDict['GridSite']
            ce = attDict['DestinationSite']
            state = attDict['Status']
            if site == 'Unknown' and ce != "Unknown" and ce in ceMap:
                site = ceMap[ce]
            if state == "Done":
                resultDict[site][ce]["Done"] = count
            if state == "Aborted":
                resultDict[site][ce]["Aborted"] = count

        for attDict, count in resultDayEmpty['Value']:
            site = attDict['GridSite']
            ce = attDict['DestinationSite']
            state = attDict['Status']
            if site == 'Unknown' and ce != "Unknown" and ce in ceMap:
                site = ceMap[ce]
            if state == "Done":
                resultDict[site][ce]["Done_Empty"] = count

        for attDict, count in resultHour['Value']:
            site = attDict['GridSite']
            ce = attDict['DestinationSite']
            state = attDict['Status']
            if site == 'Unknown' and ce != "Unknown" and ce in ceMap:
                site = ceMap[ce]
            if state == "Aborted":
                resultDict[site][ce]["Aborted_Hour"] = count

        records = []
        siteSumDict = {}
        for site in resultDict:
            sumDict = {}
            for state in allStateNames:
                if state not in sumDict:
                    sumDict[state] = 0
            sumDict['Total'] = 0
            for ce in resultDict[site]:
                itemList = [site, ce]
                total = 0
                for state in allStateNames:
                    itemList.append(resultDict[site][ce][state])
                    sumDict[state] += resultDict[site][ce][state]
                    if state == "Done":
                        done = resultDict[site][ce][state]
                    if state == "Done_Empty":
                        empty = resultDict[site][ce][state]
                    if state == "Aborted":
                        aborted = resultDict[site][ce][state]
                    if state != "Aborted_Hour" and state != "Done_Empty":
                        total += resultDict[site][ce][state]

                sumDict['Total'] += total
                # Add the total number of pilots seen in the last day
                itemList.append(total)
                # Add pilot submission efficiency evaluation
                if (done - empty) > 0:
                    eff = float(done) / float(done - empty)
                elif done == 0:
                    eff = 0.
                elif empty == done:
                    eff = 99.
                else:
                    eff = 0.
                itemList.append('%.2f' % eff)
                # Add pilot job efficiency evaluation
                if total > 0:
                    eff = float(total - aborted) / float(total) * 100.
                else:
                    eff = 100.
                itemList.append('%.2f' % eff)

                # Evaluate the quality status of the CE
                if total > 10:
                    if eff < 25.:
                        itemList.append('Bad')
                    elif eff < 60.:
                        itemList.append('Poor')
                    elif eff < 85.:
                        itemList.append('Fair')
                    else:
                        itemList.append('Good')
                else:
                    itemList.append('Idle')

                if len(resultDict[site]) == 1 or expand_site:
                    records.append(itemList)

            if len(resultDict[site]) > 1 and not expand_site:
                itemList = [site, 'Multiple']
                for state in allStateNames + ['Total']:
                    if state in sumDict:
                        itemList.append(sumDict[state])
                    else:
                        itemList.append(0)
                done = sumDict["Done"]
                empty = sumDict["Done_Empty"]
                aborted = sumDict["Aborted"]
                total = sumDict["Total"]

                # Add pilot submission efficiency evaluation
                if (done - empty) > 0:
                    eff = float(done) / float(done - empty)
                elif done == 0:
                    eff = 0.
                elif empty == done:
                    eff = 99.
                else:
                    eff = 0.
                itemList.append('%.2f' % eff)
                # Add pilot job efficiency evaluation
                if total > 0:
                    eff = float(total - aborted) / float(total) * 100.
                else:
                    eff = 100.
                itemList.append('%.2f' % eff)

                # Evaluate the quality status of the Site
                if total > 10:
                    if eff < 25.:
                        itemList.append('Bad')
                    elif eff < 60.:
                        itemList.append('Poor')
                    elif eff < 85.:
                        itemList.append('Fair')
                    else:
                        itemList.append('Good')
                else:
                    itemList.append('Idle')
                records.append(itemList)

            for state in allStateNames + ['Total']:
                if state not in siteSumDict:
                    siteSumDict[state] = sumDict[state]
                else:
                    siteSumDict[state] += sumDict[state]

        # Perform site selection
        if site_select:
            new_records = []
            for r in records:
                if r[0] in site_select:
                    new_records.append(r)
            records = new_records

        # Perform status selection
        if status_select:
            new_records = []
            for r in records:
                if r[14] in status_select:
                    new_records.append(r)
            records = new_records

        # Get the Site Mask data
        result = SiteStatus().getUsableSites()
        if result['OK']:
            siteMask = result['Value']
            for r in records:
                if r[0] in siteMask:
                    r.append('Yes')
                else:
                    r.append('No')
        else:
            for r in records:
                r.append('Unknown')

        finalDict = {}
        finalDict['TotalRecords'] = len(records)
        finalDict['ParameterNames'] = paramNames + \
            ['Total', 'PilotsPerJob', 'PilotJobEff', 'Status', 'InMask']

        # Return all the records if maxItems == 0 or the specified number otherwise
        if maxItems:
            finalDict['Records'] = records[startItem:startItem + maxItems]
        else:
            finalDict['Records'] = records

        done = siteSumDict["Done"]
        empty = siteSumDict["Done_Empty"]
        aborted = siteSumDict["Aborted"]
        total = siteSumDict["Total"]

        # Add pilot submission efficiency evaluation
        if (done - empty) > 0:
            eff = float(done) / float(done - empty)
        elif done == 0:
            eff = 0.
        elif empty == done:
            eff = 99.
        else:
            eff = 0.
        siteSumDict['PilotsPerJob'] = '%.2f' % eff
        # Add pilot job efficiency evaluation
        if total > 0:
            eff = float(total - aborted) / float(total) * 100.
        else:
            eff = 100.
        siteSumDict['PilotJobEff'] = '%.2f' % eff

        # Evaluate the overall quality status
        if total > 100:
            if eff < 25.:
                siteSumDict['Status'] = 'Bad'
            elif eff < 60.:
                siteSumDict['Status'] = 'Poor'
            elif eff < 85.:
                siteSumDict['Status'] = 'Fair'
            else:
                siteSumDict['Status'] = 'Good'
        else:
            siteSumDict['Status'] = 'Idle'
        finalDict['Extras'] = siteSumDict

        return S_OK(finalDict)