Example #1
0
 def __addPilotsAccountingReport(self, pilotsData):
   """ fill accounting data
   """
   for pRef in pilotsData:
     pData = pilotsData[pRef]
     pA = PilotAccounting()
     pA.setEndTime(pData['LastUpdateTime'])
     pA.setStartTime(pData['SubmissionTime'])
     retVal = CS.getUsernameForDN(pData['OwnerDN'])
     if not retVal['OK']:
       userName = '******'
       self.log.error("Can't determine username for dn:", pData['OwnerDN'])
     else:
       userName = retVal['Value']
     pA.setValueByKey('User', userName)
     pA.setValueByKey('UserGroup', pData['OwnerGroup'])
     result = getSiteForCE(pData['DestinationSite'])
     if result['OK'] and result['Value'].strip():
       pA.setValueByKey('Site', result['Value'].strip())
     else:
       pA.setValueByKey('Site', 'Unknown')
     pA.setValueByKey('GridCE', pData['DestinationSite'])
     pA.setValueByKey('GridMiddleware', pData['GridType'])
     pA.setValueByKey('GridResourceBroker', pData['Broker'])
     pA.setValueByKey('GridStatus', pData['Status'])
     if 'Jobs' not in pData:
       pA.setValueByKey('Jobs', 0)
     else:
       pA.setValueByKey('Jobs', len(pData['Jobs']))
     self.log.verbose("Added accounting record for pilot %s" % pData['PilotID'])
     retVal = gDataStoreClient.addRegister(pA)
     if not retVal['OK']:
       return retVal
   return S_OK()
Example #2
0
 def __addPilotsAccountingReport( self, pilotsData ):
   """ fill accounting data
   """
   for pRef in pilotsData:
     pData = pilotsData[pRef]
     pA = PilotAccounting()
     pA.setEndTime( pData[ 'LastUpdateTime' ] )
     pA.setStartTime( pData[ 'SubmissionTime' ] )
     retVal = CS.getUsernameForDN( pData[ 'OwnerDN' ] )
     if not retVal[ 'OK' ]:
       userName = '******'
       self.log.error( "Can't determine username for dn:", pData[ 'OwnerDN' ] )
     else:
       userName = retVal[ 'Value' ]
     pA.setValueByKey( 'User', userName )
     pA.setValueByKey( 'UserGroup', pData[ 'OwnerGroup' ] )
     result = getSiteForCE( pData[ 'DestinationSite' ] )
     if result['OK'] and result[ 'Value' ].strip():
       pA.setValueByKey( 'Site', result['Value'].strip() )
     else:
       pA.setValueByKey( 'Site', 'Unknown' )
     pA.setValueByKey( 'GridCE', pData[ 'DestinationSite' ] )
     pA.setValueByKey( 'GridMiddleware', pData[ 'GridType' ] )
     pA.setValueByKey( 'GridResourceBroker', pData[ 'Broker' ] )
     pA.setValueByKey( 'GridStatus', pData[ 'Status' ] )
     if not 'Jobs' in pData:
       pA.setValueByKey( 'Jobs', 0 )
     else:
       pA.setValueByKey( 'Jobs', len( pData['Jobs'] ) )
     self.log.verbose( "Added accounting record for pilot %s" % pData[ 'PilotID' ] )
     retVal = gDataStoreClient.addRegister( pA )
     if not retVal[ 'OK' ]:
       return retVal
   return S_OK()
Example #3
0
 def __addPilotsAccountingReport(self, pilotsData):
     """ fill accounting data
 """
     for pRef in pilotsData:
         pData = pilotsData[pRef]
         pA = PilotAccounting()
         pA.setEndTime(pData["LastUpdateTime"])
         pA.setStartTime(pData["SubmissionTime"])
         retVal = CS.getUsernameForDN(pData["OwnerDN"])
         if not retVal["OK"]:
             userName = "******"
             self.log.error("Can't determine username for dn:", pData["OwnerDN"])
         else:
             userName = retVal["Value"]
         pA.setValueByKey("User", userName)
         pA.setValueByKey("UserGroup", pData["OwnerGroup"])
         result = getSiteForCE(pData["DestinationSite"])
         if result["OK"] and result["Value"].strip():
             pA.setValueByKey("Site", result["Value"].strip())
         else:
             pA.setValueByKey("Site", "Unknown")
         pA.setValueByKey("GridCE", pData["DestinationSite"])
         pA.setValueByKey("GridMiddleware", pData["GridType"])
         pA.setValueByKey("GridResourceBroker", pData["Broker"])
         pA.setValueByKey("GridStatus", pData["Status"])
         if not "Jobs" in pData:
             pA.setValueByKey("Jobs", 0)
         else:
             pA.setValueByKey("Jobs", len(pData["Jobs"]))
         self.log.verbose("Added accounting record for pilot %s" % pData["PilotID"])
         retVal = gDataStoreClient.addRegister(pA)
         if not retVal["OK"]:
             return retVal
     return S_OK()
Example #4
0
  def setPilotStatus( self, pilotRef, status, destination=None,
                      statusReason=None, gridSite=None, queue=None, updateTime=None, conn = False ):
    """ Set pilot job LCG status """

    setList = []
    setList.append("Status='%s'" % status)
    if updateTime:
      setList.append("LastUpdateTime='%s'" % updateTime)
    else:
      setList.append("LastUpdateTime=UTC_TIMESTAMP()")
    if not statusReason:
      statusReason = "Not given"
    setList.append("StatusReason='%s'" % statusReason)
    if gridSite:
      setList.append("GridSite='%s'" % gridSite)
    if queue:
      setList.append("Queue='%s'" % queue)  
    if destination:
      setList.append("DestinationSite='%s'" % destination) 
      if not gridSite:
        result = getSiteForCE(destination)
        if result['OK']:
          gridSite = result['Value'] 
          setList.append("GridSite='%s'" % gridSite)

    set_string = ','.join(setList)
    req = "UPDATE PilotAgents SET "+set_string+" WHERE PilotJobReference='%s'" % pilotRef
    result = self._update( req, conn = conn )
    if not result['OK']:
      return result

    return S_OK()
Example #5
0
  def run(self):
    '''
      Checks it has the parameters it needs and tries to send an sms to the users
      that apply.
    '''

    if self.url is None:
      return S_ERROR('Slack URL not set')

    # Minor security checks
    element = self.decisionParams['element']
    if element is None:
      return S_ERROR('element should not be None')

    name = self.decisionParams['name']
    if name is None:
      return S_ERROR('name should not be None')

    statusType = self.decisionParams['statusType']
    if statusType is None:
      return S_ERROR('statusType should not be None')

    previousStatus = self.decisionParams['status']
    if previousStatus is None:
      return S_ERROR('status should not be None')

    status = self.enforcementResult['Status']
    if status is None:
      return S_ERROR('status should not be None')

    reason = self.enforcementResult['Reason']
    if reason is None:
      return S_ERROR('reason should not be None')

    if self.decisionParams['element'] == 'Site':
      siteName = self.decisionParams['name']
    else:
      elementType = self.decisionParams['elementType']

      if elementType == 'StorageElement':
        siteName = getSitesForSE(name)
      elif elementType == 'ComputingElement':
        siteName = getSiteForCE(name)
      else:
        siteName = {'OK': True, 'Value': 'Unassigned'}

      if not siteName['OK']:
        self.log.error('Resource %s does not exist at any site: %s' % (name, siteName['Message']))
        siteName = "Unassigned Resources"
      elif not siteName['Value']:
        siteName = "Unassigned Resources"
      else:
        siteName = siteName['Value'] if isinstance(siteName['Value'], basestring) else siteName['Value'][0]

    message = "*{name}* _{statusType}_ --> _{status}_ \n{reason}".format(name=name,
                                                                         statusType=statusType,
                                                                         status=status,
                                                                         reason=reason)
    return self.sendSlackMessage(message)
Example #6
0
    def sendPilotAccounting(self, pilotDict):
        """ Send pilot accounting record
    """
        for pRef in pilotDict:
            self.log.verbose('Preparing accounting record for pilot %s' % pRef)
            pA = PilotAccounting()
            pA.setEndTime(pilotDict[pRef]['LastUpdateTime'])
            pA.setStartTime(pilotDict[pRef]['SubmissionTime'])
            retVal = CS.getUsernameForDN(pilotDict[pRef]['OwnerDN'])
            if not retVal['OK']:
                userName = '******'
                self.log.error("Can't determine username for dn:",
                               pilotDict[pRef]['OwnerDN'])
            else:
                userName = retVal['Value']
            pA.setValueByKey('User', userName)
            pA.setValueByKey('UserGroup', pilotDict[pRef]['OwnerGroup'])
            result = getSiteForCE(pilotDict[pRef]['DestinationSite'])
            if result['OK'] and result['Value'].strip():
                pA.setValueByKey('Site', result['Value'].strip())
            else:
                pA.setValueByKey('Site', 'Unknown')
            pA.setValueByKey('GridCE', pilotDict[pRef]['DestinationSite'])
            pA.setValueByKey('GridMiddleware', pilotDict[pRef]['GridType'])
            pA.setValueByKey('GridResourceBroker', pilotDict[pRef]['Broker'])
            pA.setValueByKey('GridStatus', pilotDict[pRef]['Status'])
            if not 'Jobs' in pilotDict[pRef]:
                pA.setValueByKey('Jobs', 0)
            else:
                pA.setValueByKey('Jobs', len(pilotDict[pRef]['Jobs']))
            self.log.info("Adding accounting record for pilot %s" %
                          pilotDict[pRef]['PilotID'])
            retVal = gDataStoreClient.addRegister(pA)
            if not retVal['OK']:
                self.log.error('Failed to send accounting info for pilot ',
                               pRef)
            else:
                # Set up AccountingSent flag
                result = pilotAgentsDB.setAccountingFlag(pRef)
                if not result['OK']:
                    self.log.error('Failed to set accounting flag for pilot ',
                                   pRef)

        self.log.info('Committing accounting records for %d pilots' %
                      len(pilotDict))
        result = gDataStoreClient.commit()
        if result['OK']:
            for pRef in pilotDict:
                self.log.verbose('Setting AccountingSent flag for pilot %s' %
                                 pRef)
                result = pilotAgentsDB.setAccountingFlag(pRef)
                if not result['OK']:
                    self.log.error('Failed to set accounting flag for pilot ',
                                   pRef)
        else:
            return result

        return S_OK()
Example #7
0
  def run(self):
    ''' Checks it has the parameters it needs and writes the date to a cache file.
    '''
    # Minor security checks

    element = self.decisionParams['element']
    if element is None:
      return S_ERROR('element should not be None')

    name = self.decisionParams['name']
    if name is None:
      return S_ERROR('name should not be None')

    statusType = self.decisionParams['statusType']
    if statusType is None:
      return S_ERROR('statusType should not be None')

    previousStatus = self.decisionParams['status']
    if previousStatus is None:
      return S_ERROR('status should not be None')

    status = self.enforcementResult['Status']
    if status is None:
      return S_ERROR('status should not be None')

    reason = self.enforcementResult['Reason']
    if reason is None:
      return S_ERROR('reason should not be None')

    if self.decisionParams['element'] == 'Site':
      siteName = self.decisionParams['name']
    else:
      elementType = self.decisionParams['elementType']

      if elementType == 'StorageElement':
        siteName = getSitesForSE(name)
      elif elementType == 'ComputingElement':
        siteName = getSiteForCE(name)
      else:
        siteName = {'OK': True, 'Value': 'Unassigned'}

      if not siteName['OK']:
        self.log.error('Resource %s does not exist at any site: %s' % (name, siteName['Message']))
        siteName = "Unassigned Resources"
      elif not siteName['Value']:
        siteName = "Unassigned Resources"
      else:
        siteName = siteName['Value'] if isinstance(siteName['Value'], basestring) else siteName['Value'][0]

    # create record for insertion
    recordDict = {}
    recordDict['SiteName'] = siteName
    recordDict['ResourceName'] = name
    recordDict['Status'] = status
    recordDict['PreviousStatus'] = previousStatus
    recordDict['StatusType'] = statusType

    return self.rsClient.insert('ResourceStatusCache', recordDict)
Example #8
0
 def __getXRSLExtraString(self, multiprocessor=False):
   # For the XRSL additional string from configuration - only done at initialisation time
   # If this string changes, the corresponding (ARC) site directors have to be restarted
   #
   # Variable = XRSLExtraString (or XRSLMPExtraString for multi processor mode)
   # Default value = ''
   #   If you give a value, I think it should be of the form
   #          (aaa = "xxx")
   #   Otherwise the ARC job description parser will have a fit
   # Locations searched in order :
   # Top priority    : Resources/Sites/<Grid>/<Site>/CEs/<CE>/XRSLExtraString
   # Second priority : Resources/Sites/<Grid>/<Site>/XRSLExtraString
   # Default         : Resources/Computing/CEDefaults/XRSLExtraString
   #
   xrslExtraString = ''  # Start with the default value
   result = getSiteForCE(self.ceHost)
   self.site = ''
   if result['OK']:
     self.site = result['Value']
   else:
     gLogger.error("Unknown Site ...")
     return
   # Now we know the site. Get the grid
   grid = self.site.split(".")[0]
   # The different possibilities that we have agreed upon
   if multiprocessor:
     xtraVariable = "XRSLMPExtraString"
   else:
     xtraVariable = "XRSLExtraString"
   firstOption = "Resources/Sites/%s/%s/CEs/%s/%s" % (grid, self.site, self.ceHost, xtraVariable)
   secondOption = "Resources/Sites/%s/%s/%s" % (grid, self.site, xtraVariable)
   defaultOption = "Resources/Computing/CEDefaults/%s" % xtraVariable
   # Now go about getting the string in the agreed order
   gLogger.debug("Trying to get %s : first option %s" % (xtraVariable, firstOption))
   result = gConfig.getValue(firstOption, defaultValue='')
   if result != '':
     xrslExtraString = result
     gLogger.debug("Found %s : %s" % (xtraVariable, xrslExtraString))
   else:
     gLogger.debug("Trying to get %s : second option %s" % (xtraVariable, secondOption))
     result = gConfig.getValue(secondOption, defaultValue='')
     if result != '':
       xrslExtraString = result
       gLogger.debug("Found %s : %s" % (xtraVariable, xrslExtraString))
     else:
       gLogger.debug("Trying to get %s : default option %s" % (xtraVariable, defaultOption))
       result = gConfig.getValue(defaultOption, defaultValue='')
       if result != '':
         xrslExtraString = result
         gLogger.debug("Found %s : %s" % (xtraVariable, xrslExtraString))
   if xrslExtraString == '':
     gLogger.always("No %s found in configuration for %s" % (xtraVariable, self.ceHost))
   else:
     gLogger.always("%s : %s" % (xtraVariable, xrslExtraString))
     gLogger.always(" --- to be added to pilots going to CE : %s" % self.ceHost)
   return xrslExtraString
Example #9
0
 def __getXRSLExtraString(self, multiprocessor=False):
   # For the XRSL additional string from configuration - only done at initialisation time
   # If this string changes, the corresponding (ARC) site directors have to be restarted
   #
   # Variable = XRSLExtraString (or for multi processor mode)
   # Default value = ''
   #   If you give a value, I think it should be of the form
   #          (aaa = "xxx")
   #   Otherwise the ARC job description parser will have a fit
   # Locations searched in order :
   # Top priority    : Resources/Sites/<Grid>/<Site>/CEs/<CE>/XRSLExtraString
   # Second priority : Resources/Sites/<Grid>/<Site>/XRSLExtraString
   # Default         : Resources/Computing/CEDefaults/XRSLExtraString
   #
   xrslExtraString = ''  # Start with the default value
   result = getSiteForCE(self.ceHost)
   self.site = ''
   if result['OK']:
     self.site = result['Value']
   else:
     gLogger.error("Unknown Site ...")
     return
   # Now we know the site. Get the grid
   grid = self.site.split(".")[0]
   # The different possibilities that we have agreed upon
   if multiprocessor:
     xtraVariable = "XRSLMPExtraString"
   else:
     xtraVariable = "XRSLExtraString"
   firstOption = "Resources/Sites/%s/%s/CEs/%s/%s" % (grid, self.site, self.ceHost, xtraVariable)
   secondOption = "Resources/Sites/%s/%s/%s" % (grid, self.site, xtraVariable)
   defaultOption = "Resources/Computing/CEDefaults/%s" % xtraVariable
   # Now go about getting the string in the agreed order
   gLogger.debug("Trying to get %s : first option %s" % (xtraVariable, firstOption))
   result = gConfig.getValue(firstOption, defaultValue='')
   if result != '':
     xrslExtraString = result
     gLogger.debug("Found %s : %s" % (xtraVariable, xrslExtraString))
   else:
     gLogger.debug("Trying to get %s : second option %s" % (xtraVariable, secondOption))
     result = gConfig.getValue(secondOption, defaultValue='')
     if result != '':
       xrslExtraString = result
       gLogger.debug("Found %s : %s" % (xtraVariable, xrslExtraString))
     else:
       gLogger.debug("Trying to get %s : default option %s" % (xtraVariable, defaultOption))
       result = gConfig.getValue(defaultOption, defaultValue='')
       if result != '':
         xrslExtraString = result
         gLogger.debug("Found %s : %s" % (xtraVariable, xrslExtraString))
   if xrslExtraString:
     gLogger.always("%s : %s" % (xtraVariable, xrslExtraString))
     gLogger.always(" --- to be added to pilots going to CE : %s" % self.ceHost)
   return xrslExtraString
Example #10
0
  def sendPilotAccounting( self, pilotDict ):
    """ Send pilot accounting record
    """
    for pRef in pilotDict:
      self.log.verbose( 'Preparing accounting record for pilot %s' % pRef )
      pA = PilotAccounting()
      pA.setEndTime( pilotDict[pRef][ 'LastUpdateTime' ] )
      pA.setStartTime( pilotDict[pRef][ 'SubmissionTime' ] )
      retVal = CS.getUsernameForDN( pilotDict[pRef][ 'OwnerDN' ] )
      if not retVal[ 'OK' ]:
        userName = '******'
        self.log.error( "Can't determine username for dn:", pilotDict[pRef][ 'OwnerDN' ] )
      else:
        userName = retVal[ 'Value' ]
      pA.setValueByKey( 'User', userName )
      pA.setValueByKey( 'UserGroup', pilotDict[pRef][ 'OwnerGroup' ] )
      result = getSiteForCE( pilotDict[pRef][ 'DestinationSite' ] )
      if result['OK'] and result[ 'Value' ].strip():
        pA.setValueByKey( 'Site', result['Value'].strip() )
      else:
        pA.setValueByKey( 'Site', 'Unknown' )
      pA.setValueByKey( 'GridCE', pilotDict[pRef][ 'DestinationSite' ] )
      pA.setValueByKey( 'GridMiddleware', pilotDict[pRef][ 'GridType' ] )
      pA.setValueByKey( 'GridResourceBroker', pilotDict[pRef][ 'Broker' ] )
      pA.setValueByKey( 'GridStatus', pilotDict[pRef][ 'Status' ] )
      if not 'Jobs' in pilotDict[pRef]:
        pA.setValueByKey( 'Jobs', 0 )
      else:
        pA.setValueByKey( 'Jobs', len( pilotDict[pRef]['Jobs'] ) )
      self.log.verbose( "Adding accounting record for pilot %s" % pilotDict[pRef][ 'PilotID' ] )
      retVal = gDataStoreClient.addRegister( pA )
      if not retVal[ 'OK' ]:
        self.log.error( 'Failed to send accounting info for pilot ', pRef )
      else:
        # Set up AccountingSent flag
        result = pilotAgentsDB.setAccountingFlag( pRef )
        if not result['OK']:
          self.log.error( 'Failed to set accounting flag for pilot ', pRef )

    self.log.info( 'Committing accounting records for %d pilots' % len( pilotDict ) )
    result = gDataStoreClient.commit()
    if result['OK']:
      for pRef in pilotDict:
        self.log.verbose( 'Setting AccountingSent flag for pilot %s' % pRef )
        result = pilotAgentsDB.setAccountingFlag( pRef )
        if not result['OK']:
          self.log.error( 'Failed to set accounting flag for pilot ', pRef )
    else:
      return result

    return S_OK()
Example #11
0
    def sendPilotAccounting(self, pilotDict):
        """ Send pilot accounting record
    """
        for pRef in pilotDict:
            self.log.verbose("Preparing accounting record for pilot %s" % pRef)
            pA = PilotAccounting()
            pA.setEndTime(pilotDict[pRef]["LastUpdateTime"])
            pA.setStartTime(pilotDict[pRef]["SubmissionTime"])
            retVal = CS.getUsernameForDN(pilotDict[pRef]["OwnerDN"])
            if not retVal["OK"]:
                userName = "******"
                self.log.error("Can't determine username for dn:", pilotDict[pRef]["OwnerDN"])
            else:
                userName = retVal["Value"]
            pA.setValueByKey("User", userName)
            pA.setValueByKey("UserGroup", pilotDict[pRef]["OwnerGroup"])
            result = getSiteForCE(pilotDict[pRef]["DestinationSite"])
            if result["OK"] and result["Value"].strip():
                pA.setValueByKey("Site", result["Value"].strip())
            else:
                pA.setValueByKey("Site", "Unknown")
            pA.setValueByKey("GridCE", pilotDict[pRef]["DestinationSite"])
            pA.setValueByKey("GridMiddleware", pilotDict[pRef]["GridType"])
            pA.setValueByKey("GridResourceBroker", pilotDict[pRef]["Broker"])
            pA.setValueByKey("GridStatus", pilotDict[pRef]["Status"])
            if not "Jobs" in pilotDict[pRef]:
                pA.setValueByKey("Jobs", 0)
            else:
                pA.setValueByKey("Jobs", len(pilotDict[pRef]["Jobs"]))
            self.log.info("Adding accounting record for pilot %s" % pilotDict[pRef]["PilotID"])
            retVal = gDataStoreClient.addRegister(pA)
            if not retVal["OK"]:
                self.log.error("Failed to send accounting info for pilot ", pRef)
            else:
                # Set up AccountingSent flag
                result = pilotAgentsDB.setAccountingFlag(pRef)
                if not result["OK"]:
                    self.log.error("Failed to set accounting flag for pilot ", pRef)

        self.log.info("Committing accounting records for %d pilots" % len(pilotDict))
        result = gDataStoreClient.commit()
        if result["OK"]:
            for pRef in pilotDict:
                self.log.verbose("Setting AccountingSent flag for pilot %s" % pRef)
                result = pilotAgentsDB.setAccountingFlag(pRef)
                if not result["OK"]:
                    self.log.error("Failed to set accounting flag for pilot ", pRef)
        else:
            return result

        return S_OK()
Example #12
0
  def setDestinationCE( self, ceName ):
    """ Developer function.

        Allows to direct a job to a particular Grid CE.
    """
    kwargs = {'ceName':ceName}
    diracSite = getSiteForCE( ceName )
    if not diracSite['OK']:
      return self._reportError( diracSite['Message'], **kwargs )
    if not diracSite['Value']:
      return self._reportError( 'No DIRAC site name found for CE %s' % ( ceName ), **kwargs )

    diracSite = diracSite['Value']
    self.setDestination( diracSite )
    self._addJDLParameter( 'GridRequiredCEs', ceName )
    return S_OK()
Example #13
0
  def setPilotDestinationSite(self,pilotRef,destination, conn=False):
    """ Set the pilot agent destination site
    """

    gridSite = 'Unknown'
    result = getSiteForCE(destination)
    if result['OK']:
      gridSite = result['Value']

    if not gridSite:
      gridSite = 'Unknown'

    req = "UPDATE PilotAgents SET DestinationSite='%s', GridSite='%s' WHERE PilotJobReference='%s'" 
    req = req % (destination,gridSite,pilotRef)
    result = self._update(req, conn = conn)
    return result
Example #14
0
    def setPilotDestinationSite(self, pilotRef, destination, conn=False):
        """ Set the pilot agent destination site
    """

        gridSite = 'Unknown'
        result = getSiteForCE(destination)
        if result['OK']:
            gridSite = result['Value']

        if not gridSite:
            gridSite = 'Unknown'

        req = "UPDATE PilotAgents SET DestinationSite='%s', GridSite='%s' WHERE PilotJobReference='%s'"
        req = req % (destination, gridSite, pilotRef)
        result = self._update(req, conn=conn)
        return result
Example #15
0
File: Job.py Project: sposs/DIRAC
  def setDestinationCE( self, ceName ):
    """ Developer function.

        Allows to direct a job to a particular Grid CE.
    """
    kwargs = {'ceName':ceName}
    diracSite = getSiteForCE( ceName )
    if not diracSite['OK']:
      return self._reportError( diracSite['Message'], **kwargs )
    if not diracSite['Value']:
      return self._reportError( 'No DIRAC site name found for CE %s' % ( ceName ), **kwargs )

    diracSite = diracSite['Value']
    self.setDestination( diracSite )
    self._addJDLParameter( 'GridRequiredCEs', ceName )
    return S_OK()
Example #16
0
    def setPilotStatus(self,
                       pilotRef,
                       status,
                       destination=None,
                       statusReason=None,
                       gridSite=None,
                       queue=None,
                       benchmark=None,
                       currentJob=None,
                       updateTime=None,
                       conn=False):
        """ Set pilot job status
    """

        setList = []
        setList.append("Status='%s'" % status)
        if updateTime:
            setList.append("LastUpdateTime='%s'" % updateTime)
        else:
            setList.append("LastUpdateTime=UTC_TIMESTAMP()")
        if not statusReason:
            statusReason = "Not given"
        setList.append("StatusReason='%s'" % statusReason)
        if gridSite:
            setList.append("GridSite='%s'" % gridSite)
        if queue:
            setList.append("Queue='%s'" % queue)
        if benchmark:
            setList.append("BenchMark='%s'" % float(benchmark))
        if currentJob:
            setList.append("CurrentJobID='%s'" % int(currentJob))
        if destination:
            setList.append("DestinationSite='%s'" % destination)
            if not gridSite:
                result = getSiteForCE(destination)
                if result['OK']:
                    gridSite = result['Value']
                    setList.append("GridSite='%s'" % gridSite)

        set_string = ','.join(setList)
        req = "UPDATE PilotAgents SET " + set_string + " WHERE PilotJobReference='%s'" % pilotRef
        result = self._update(req, conn=conn)
        if not result['OK']:
            return result

        return S_OK()
Example #17
0
File: Job.py Project: petricm/DIRAC
    def setDestinationCE(self, ceName, diracSite=None):
        """ Developer function.

        Allows to direct a job to a particular Grid CE.
    """
        kwargs = {"ceName": ceName}

        if not diracSite:
            diracSite = getSiteForCE(ceName)
            if not diracSite["OK"]:
                return self._reportError(diracSite["Message"], **kwargs)
            if not diracSite["Value"]:
                return self._reportError("No DIRAC site name found for CE %s" % (ceName), **kwargs)
            diracSite = diracSite["Value"]

        self.setDestination(diracSite)
        # Keep GridRequiredCEs for backward compatibility
        self._addJDLParameter("GridRequiredCEs", ceName)
        self._addJDLParameter("GridCE", ceName)
        return S_OK()
Example #18
0
  def __purgeSites(self, ceBdiiDict):
    """Remove all sites that are not in self.selectedSites.

    Modifies the ceBdiiDict!
    """
    if not self.selectedSites:
      return
    for site in list(ceBdiiDict):
      ces = list(ceBdiiDict[site]['CEs'])
      if not ces:
        self.log.error("No CE information for site:", site)
        continue
      diracSiteName = getSiteForCE(ces[0])
      if not diracSiteName['OK']:
        self.log.error("Failed to get DIRAC site name for ce", "%s: %s" % (ces[0], diracSiteName['Message']))
        continue
      self.log.debug("Checking site %s (%s), aka %s" % (site, ces, diracSiteName['Value']))
      if diracSiteName['Value'] in self.selectedSites:
        continue
      self.log.info("Dropping site %s, aka %s" % (site, diracSiteName))
      ceBdiiDict.pop(site)
    return
Example #19
0
    def __purgeSites(self, ceBdiiDict):
        """Remove all sites that are not in self.selectedSites.

    Modifies the ceBdiiDict!
    """
        if not self.selectedSites:
            return
        for site in list(ceBdiiDict):
            ces = list(ceBdiiDict[site]['CEs'])
            if not ces:
                self.log.error("No CE information for site:", site)
                continue
            diracSiteName = getSiteForCE(ces[0])
            if not diracSiteName['OK']:
                self.log.error("Failed to get DIRAC site name for ce",
                               "%s: %s" % (ces[0], diracSiteName['Message']))
                continue
            self.log.debug("Checking site %s (%s), aka %s" %
                           (site, ces, diracSiteName['Value']))
            if diracSiteName['Value'] in self.selectedSites:
                continue
            self.log.info("Dropping site %s, aka %s" % (site, diracSiteName))
            ceBdiiDict.pop(site)
        return
Example #20
0
  def run(self):
    ''' Checks it has the parameters it needs and writes the date to a cache file.
    '''
    # Minor security checks

    element = self.decisionParams['element']
    if element is None:
      return S_ERROR('element should not be None')

    name = self.decisionParams['name']
    if name is None:
      return S_ERROR('name should not be None')

    statusType = self.decisionParams['statusType']
    if statusType is None:
      return S_ERROR('statusType should not be None')

    previousStatus = self.decisionParams['status']
    if previousStatus is None:
      return S_ERROR('status should not be None')

    status = self.enforcementResult['Status']
    if status is None:
      return S_ERROR('status should not be None')

    reason = self.enforcementResult['Reason']
    if reason is None:
      return S_ERROR('reason should not be None')

    if self.decisionParams['element'] == 'Site':
      siteName = self.decisionParams['name']
    else:
      elementType = self.decisionParams['elementType']

      if elementType == 'StorageElement':
        siteName = getSitesForSE(name)
      elif elementType == 'ComputingElement':
        siteName = getSiteForCE(name)
      else:
        siteName = {'OK': True, 'Value': 'Unassigned'}

      if not siteName['OK']:
        self.log.error('Resource %s does not exist at any site: %s' % (name, siteName['Message']))
        siteName = "Unassigned Resources"
      elif not siteName['Value']:
        siteName = "Unassigned Resources"
      else:
        siteName = siteName['Value'][0]

    with sqlite3.connect(self.cacheFile) as conn:

      try:
        conn.execute('''CREATE TABLE IF NOT EXISTS ResourceStatusCache(
                      SiteName VARCHAR(64) NOT NULL,
                      ResourceName VARCHAR(64) NOT NULL,
                      Status VARCHAR(8) NOT NULL DEFAULT "",
                      PreviousStatus VARCHAR(8) NOT NULL DEFAULT "",
                      StatusType VARCHAR(128) NOT NULL DEFAULT "all",
                      Time TIMESTAMP DEFAULT CURRENT_TIMESTAMP
                     );''')

        insertQuery = "INSERT INTO ResourceStatusCache (SiteName, ResourceName, Status, PreviousStatus, StatusType)"
        insertQuery += " VALUES ('%s', '%s', '%s', '%s', '%s' ); " % (siteName, name, status,
                                                                      previousStatus, statusType)
        conn.execute(insertQuery)

        conn.commit()

      except sqlite3.OperationalError:
        self.log.error('Email cache database is locked')

    return S_OK()
Example #21
0
    def run(self):
        ''' Checks it has the parameters it needs and writes the date to a cache file.
    '''
        # Minor security checks

        element = self.decisionParams['element']
        if element is None:
            return S_ERROR('element should not be None')

        name = self.decisionParams['name']
        if name is None:
            return S_ERROR('name should not be None')

        statusType = self.decisionParams['statusType']
        if statusType is None:
            return S_ERROR('statusType should not be None')

        previousStatus = self.decisionParams['status']
        if previousStatus is None:
            return S_ERROR('status should not be None')

        status = self.enforcementResult['Status']
        if status is None:
            return S_ERROR('status should not be None')

        reason = self.enforcementResult['Reason']
        if reason is None:
            return S_ERROR('reason should not be None')

        if self.decisionParams['element'] == 'Site':
            siteName = self.decisionParams['name']
        else:
            elementType = self.decisionParams['elementType']

            if elementType == 'StorageElement':
                siteName = getSitesForSE(name)
            elif elementType == 'ComputingElement':
                siteName = getSiteForCE(name)
            else:
                siteName = {'OK': True, 'Value': 'Unassigned'}

            if not siteName['OK']:
                self.log.error('Resource %s does not exist at any site: %s' %
                               (name, siteName['Message']))
                siteName = "Unassigned Resources"
            elif not siteName['Value']:
                siteName = "Unassigned Resources"
            else:
                siteName = siteName['Value'][0]

        with sqlite3.connect(self.cacheFile) as conn:

            try:
                conn.execute('''CREATE TABLE IF NOT EXISTS ResourceStatusCache(
                      SiteName VARCHAR(64) NOT NULL,
                      ResourceName VARCHAR(64) NOT NULL,
                      Status VARCHAR(8) NOT NULL DEFAULT "",
                      PreviousStatus VARCHAR(8) NOT NULL DEFAULT "",
                      StatusType VARCHAR(128) NOT NULL DEFAULT "all",
                      Time TIMESTAMP DEFAULT CURRENT_TIMESTAMP
                     );''')

            except sqlite3.OperationalError:
                self.log.error('Email cache database is locked')

            conn.execute(
                "INSERT INTO ResourceStatusCache (SiteName, ResourceName, Status, PreviousStatus, StatusType)"
                " VALUES ('" + siteName + "', '" + name + "', '" + status +
                "', '" + previousStatus + "', '" + statusType + "' ); ")

            conn.commit()

        return S_OK()
Example #22
0
  def getPilotMonitorWeb(self,selectDict,sortList,startItem,maxItems):
    """ Get summary of the pilot job information in a standard structure
    """

    resultDict = {}
    last_update = None
    if selectDict.has_key('LastUpdateTime'):
      last_update = selectDict['LastUpdateTime']
      del selectDict['LastUpdateTime']
    if selectDict.has_key('Owner'):
      userList = selectDict['Owner']
      if type(userList) != type([]):
        userList = [userList]
      dnList = []
      for uName in userList:
        uList = getDNForUsername(uName)['Value']
        dnList += uList
      selectDict['OwnerDN'] = dnList
      del selectDict['Owner']
    startDate = selectDict.get('FromDate',None)
    if startDate:
      del selectDict['FromDate']
    # For backward compatibility
    if startDate is None:
      startDate = selectDict.get('LastUpdateTime',None)
      if startDate:
        del selectDict['LastUpdateTime']
    endDate = selectDict.get('ToDate',None)
    if endDate:
      del selectDict['ToDate']

    # Sorting instructions. Only one for the moment.
    if sortList:
      orderAttribute = sortList[0][0]+":"+sortList[0][1]
    else:
      orderAttribute = None

    # Select pilots for the summary
    result = self.selectPilots(selectDict, orderAttribute=orderAttribute, newer=startDate, older=endDate, timeStamp='LastUpdateTime' )
    if not result['OK']:
      return S_ERROR('Failed to select pilots: '+result['Message'])

    pList = result['Value']
    nPilots = len(pList)
    resultDict['TotalRecords'] = nPilots
    if nPilots == 0:
      return S_OK(resultDict)

    ini = startItem
    last = ini + maxItems
    if ini >= nPilots:
      return S_ERROR('Item number out of range')
    if last > nPilots:
      last = nPilots
    pilotList = pList[ini:last]

    paramNames = ['PilotJobReference','OwnerDN','OwnerGroup','GridType','Broker',
                  'Status','DestinationSite','BenchMark','ParentID',
                  'SubmissionTime', 'PilotID', 'LastUpdateTime','CurrentJobID','TaskQueueID',
                  'GridSite']

    result = self.getPilotInfo(pilotList,paramNames=paramNames)
    if not result['OK']:
      return S_ERROR('Failed to get pilot info: '+result['Message'])

    pilotDict = result['Value']
    records = []
    for pilot in pilotList:
      parList = []
      for parameter in paramNames:
        if type(pilotDict[pilot][parameter]) not in [IntType,LongType]:
          parList.append(str(pilotDict[pilot][parameter]))
        else:
          parList.append(pilotDict[pilot][parameter])
        if parameter=='GridSite':
          gridSite = pilotDict[pilot][parameter]

      # If the Grid Site is unknown try to recover it in the last moment
      if gridSite == "Unknown":
        ce = pilotDict[pilot]['DestinationSite']
        result = getSiteForCE(ce)
        if result['OK']:
          gridSite = result['Value']
          del parList[-1]
          parList.append(gridSite)
      records.append(parList)

    resultDict['ParameterNames'] = paramNames
    resultDict['Records'] = records

    return S_OK(resultDict)
Example #23
0
    def getPilotMonitorWeb(self, selectDict, sortList, startItem, maxItems):
        """ Get summary of the pilot job information in a standard structure
    """

        resultDict = {}
        if 'LastUpdateTime' in selectDict:
            del selectDict['LastUpdateTime']
        if 'Owner' in selectDict:
            userList = selectDict['Owner']
            if not isinstance(userList, type([])):
                userList = [userList]
            dnList = []
            for uName in userList:
                uList = getDNForUsername(uName)['Value']
                dnList += uList
            selectDict['OwnerDN'] = dnList
            del selectDict['Owner']
        startDate = selectDict.get('FromDate', None)
        if startDate:
            del selectDict['FromDate']
        # For backward compatibility
        if startDate is None:
            startDate = selectDict.get('LastUpdateTime', None)
            if startDate:
                del selectDict['LastUpdateTime']
        endDate = selectDict.get('ToDate', None)
        if endDate:
            del selectDict['ToDate']

        # Sorting instructions. Only one for the moment.
        if sortList:
            orderAttribute = sortList[0][0] + ":" + sortList[0][1]
        else:
            orderAttribute = None

        # Select pilots for the summary
        result = self.selectPilots(selectDict,
                                   orderAttribute=orderAttribute,
                                   newer=startDate,
                                   older=endDate,
                                   timeStamp='LastUpdateTime')
        if not result['OK']:
            return S_ERROR('Failed to select pilots: ' + result['Message'])

        pList = result['Value']
        nPilots = len(pList)
        resultDict['TotalRecords'] = nPilots
        if nPilots == 0:
            return S_OK(resultDict)

        ini = startItem
        last = ini + maxItems
        if ini >= nPilots:
            return S_ERROR('Item number out of range')
        if last > nPilots:
            last = nPilots
        pilotList = pList[ini:last]

        paramNames = [
            'PilotJobReference', 'OwnerDN', 'OwnerGroup', 'GridType', 'Broker',
            'Status', 'DestinationSite', 'BenchMark', 'ParentID',
            'SubmissionTime', 'PilotID', 'LastUpdateTime', 'CurrentJobID',
            'TaskQueueID', 'GridSite'
        ]

        result = self.getPilotInfo(pilotList, paramNames=paramNames)
        if not result['OK']:
            return S_ERROR('Failed to get pilot info: ' + result['Message'])

        pilotDict = result['Value']
        records = []
        for pilot in pilotList:
            parList = []
            for parameter in paramNames:
                if not isinstance(pilotDict[pilot][parameter], (int, long)):
                    parList.append(str(pilotDict[pilot][parameter]))
                else:
                    parList.append(pilotDict[pilot][parameter])
                if parameter == 'GridSite':
                    gridSite = pilotDict[pilot][parameter]

            # If the Grid Site is unknown try to recover it in the last moment
            if gridSite == "Unknown":
                ce = pilotDict[pilot]['DestinationSite']
                result = getSiteForCE(ce)
                if result['OK']:
                    gridSite = result['Value']
                    del parList[-1]
                    parList.append(gridSite)
            records.append(parList)

        resultDict['ParameterNames'] = paramNames
        resultDict['Records'] = records

        return S_OK(resultDict)