Example #1
0
    def __resolveFTSServer(self):
        if not self.sourceSE:
            return S_ERROR("Source SE not set")
        if not self.targetSE:
            return S_ERROR("Target SE not set")
        res = getSitesForSE(self.sourceSE, 'LCG')
        if not res['OK'] or not res['Value']:
            return S_ERROR("Could not determine source site")
        sourceSite = res['Value'][0]
        res = getSitesForSE(self.targetSE, 'LCG')
        if not res['OK'] or not res['Value']:
            return S_ERROR("Could not determine target site")
        targetSite = res['Value'][0]

        if (sourceSite == 'LCG.CERN.ch') or (targetSite == 'LCG.CERN.ch'):
            ep = 'LCG.CERN.ch'
        else:
            # Target site FTS server should be used
            ep = targetSite

        try:
            configPath = '/Resources/FTSEndpoints/%s' % ep
            endpointURL = gConfig.getValue(configPath)
            if not endpointURL:
                errStr = "FTSRequest.__resolveFTSEndpoint: Failed to find FTS endpoint, check CS entry for '%s'." % ep
                return S_ERROR(errStr)
            self.ftsServer = endpointURL
            return S_OK(endpointURL)
        except Exception, x:
            return S_ERROR(
                'FTSRequest.__resolveFTSEndpoint: Failed to obtain endpoint details from CS'
            )
Example #2
0
  def __resolveFTSServer( self ):
    if not self.sourceSE:
      return S_ERROR( "Source SE not set" )
    if not self.targetSE:
      return S_ERROR( "Target SE not set" )
    res = getSitesForSE( self.sourceSE, 'LCG' )
    if not res['OK'] or not res['Value']:
      return S_ERROR( "Could not determine source site" )
    sourceSite = res['Value'][0]
    res = getSitesForSE( self.targetSE, 'LCG' )
    if not res['OK'] or not res['Value']:
      return S_ERROR( "Could not determine target site" )
    targetSite = res['Value'][0]

    if ( sourceSite == 'LCG.CERN.ch' ) or ( targetSite == 'LCG.CERN.ch' ):
      ep = 'LCG.CERN.ch'
    else:
      # Target site FTS server should be used
      ep = targetSite

    try:
      configPath = '/Resources/FTSEndpoints/%s' % ep
      endpointURL = gConfig.getValue( configPath )
      if not endpointURL:
        errStr = "FTSRequest.__resolveFTSEndpoint: Failed to find FTS endpoint, check CS entry for '%s'." % ep
        return S_ERROR( errStr )
      self.ftsServer = endpointURL
      return S_OK( endpointURL )
    except Exception, x:
      return S_ERROR( 'FTSRequest.__resolveFTSEndpoint: Failed to obtain endpoint details from CS' )
Example #3
0
    def __resolveFTSServer(self):
        """
    resolve FTS server to use, it should be the closest one from target SE

    :param self: self reference
    """
        from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getFTSServersForSites
        if not self.targetSE:
            return S_ERROR("Target SE not set")
        res = getSitesForSE(self.targetSE)
        if not res['OK'] or not res['Value']:
            return S_ERROR("Could not determine target site")
        targetSites = res['Value']

        targetSite = ''
        for targetSite in targetSites:
            targetFTS = getFTSServersForSites([targetSite])
            if targetFTS['OK']:
                ftsTarget = targetFTS['Value'][targetSite]
                if ftsTarget:
                    self.ftsServer = ftsTarget
                    return S_OK(self.ftsServer)
            else:
                return targetFTS
        return S_ERROR('No FTS server found for %s' % targetSite)
Example #4
0
    def _BySE(self):
        """ Matches using TargetSE. This is the standard plugin.
    """

        destSites = set()

        try:
            seList = ['Unknown']
            if self.params['TargetSE']:
                if isinstance(self.params['TargetSE'], six.string_types):
                    seList = fromChar(self.params['TargetSE'])
                elif isinstance(self.params['TargetSE'], list):
                    seList = self.params['TargetSE']
        except KeyError:
            pass

        if not seList or seList == ['Unknown']:
            return destSites

        for se in seList:
            res = getSitesForSE(se)
            if not res['OK']:
                gLogger.warn("Could not get Sites associated to SE",
                             res['Message'])
            else:
                thisSESites = res['Value']
                if thisSESites:
                    # We make an OR of the possible sites
                    destSites.update(thisSESites)

        gLogger.debug("Destinations: %s" % ','.join(destSites))
        return destSites
Example #5
0
  def __resolveFTSServer( self ):
    """
    resolve FTS server to use, it should be the closest one from target SE

    :param self: self reference
    """
    from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getFTSServersForSites
    if not self.targetSE:
      return S_ERROR( "Target SE not set" )
    res = getSitesForSE( self.targetSE )
    if not res['OK'] or not res['Value']:
      return S_ERROR( "Could not determine target site" )
    targetSites = res['Value']

    targetSite = ''
    for targetSite in targetSites:
      targetFTS = getFTSServersForSites( [targetSite] )
      if targetFTS['OK']:
        ftsTarget = targetFTS['Value'][targetSite]
        if ftsTarget:
          self.ftsServer = ftsTarget
          return S_OK( self.ftsServer )
      else:
        return targetFTS
    return S_ERROR( 'No FTS server found for %s' % targetSite )
  def _BySE( self ):
    """ Matches using TargetSE. This is the standard plugin.
    """

    destSites = set()

    try:
      seList = ['Unknown']
      if self.params['TargetSE']:
        if isinstance( self.params['TargetSE'], basestring ):
          seList = fromChar( self.params['TargetSE'] )
        elif isinstance( self.params['TargetSE'], list ):
          seList = self.params['TargetSE']
    except KeyError:
      pass

    if not seList or seList == ['Unknown']:
      return destSites

    for se in seList:
      res = getSitesForSE( se )
      if not res['OK']:
        gLogger.warn( "Could not get Sites associated to SE", res['Message'] )
      else:
        thisSESites = res['Value']
        if thisSESites:
          # We make an OR of the possible sites
          destSites.update( thisSESites )

    gLogger.debug( "Destinations: %s" % ','.join ( destSites ) )
    return destSites
Example #7
0
  def run(self):
    '''
      Checks it has the parameters it needs and tries to send an sms to the users
      that apply.
    '''

    if self.url is None:
      return S_ERROR('Slack URL not set')

    # Minor security checks
    element = self.decisionParams['element']
    if element is None:
      return S_ERROR('element should not be None')

    name = self.decisionParams['name']
    if name is None:
      return S_ERROR('name should not be None')

    statusType = self.decisionParams['statusType']
    if statusType is None:
      return S_ERROR('statusType should not be None')

    previousStatus = self.decisionParams['status']
    if previousStatus is None:
      return S_ERROR('status should not be None')

    status = self.enforcementResult['Status']
    if status is None:
      return S_ERROR('status should not be None')

    reason = self.enforcementResult['Reason']
    if reason is None:
      return S_ERROR('reason should not be None')

    if self.decisionParams['element'] == 'Site':
      siteName = self.decisionParams['name']
    else:
      elementType = self.decisionParams['elementType']

      if elementType == 'StorageElement':
        siteName = getSitesForSE(name)
      elif elementType == 'ComputingElement':
        siteName = getSiteForCE(name)
      else:
        siteName = {'OK': True, 'Value': 'Unassigned'}

      if not siteName['OK']:
        self.log.error('Resource %s does not exist at any site: %s' % (name, siteName['Message']))
        siteName = "Unassigned Resources"
      elif not siteName['Value']:
        siteName = "Unassigned Resources"
      else:
        siteName = siteName['Value'] if isinstance(siteName['Value'], basestring) else siteName['Value'][0]

    message = "*{name}* _{statusType}_ --> _{status}_ \n{reason}".format(name=name,
                                                                         statusType=statusType,
                                                                         status=status,
                                                                         reason=reason)
    return self.sendSlackMessage(message)
Example #8
0
 def _getSiteForSE(cls, se):
     """Get site name for the given SE"""
     result = getSitesForSE(se)
     if not result["OK"]:
         return result
     if result["Value"]:
         return S_OK(result["Value"][0])
     return S_OK("")
Example #9
0
 def __getSitesForSE(self, seName):
     result = self.__sitesForSE.get(seName)
     if result == False:
         result = getSitesForSE(seName)
         if not result["OK"]:
             return result
         self.__sitesForSE.add(seName, 600, result)
     return result
Example #10
0
 def _getSitesForSEs(cls, seList):
     """Get all the sites for the given SE list"""
     sites = []
     for se in seList:
         result = getSitesForSE(se)
         if result["OK"]:
             sites += result["Value"]
     return sites
Example #11
0
 def __getSitesForSE( self, seName ):
   result = self.__sitesForSE.get( seName )
   if result == False:
     result = getSitesForSE( seName )
     if not result['OK']:
       return result
     self.__sitesForSE.add( seName, 600, result )
   return result
Example #12
0
  def run(self):
    ''' Checks it has the parameters it needs and writes the date to a cache file.
    '''
    # Minor security checks

    element = self.decisionParams['element']
    if element is None:
      return S_ERROR('element should not be None')

    name = self.decisionParams['name']
    if name is None:
      return S_ERROR('name should not be None')

    statusType = self.decisionParams['statusType']
    if statusType is None:
      return S_ERROR('statusType should not be None')

    previousStatus = self.decisionParams['status']
    if previousStatus is None:
      return S_ERROR('status should not be None')

    status = self.enforcementResult['Status']
    if status is None:
      return S_ERROR('status should not be None')

    reason = self.enforcementResult['Reason']
    if reason is None:
      return S_ERROR('reason should not be None')

    if self.decisionParams['element'] == 'Site':
      siteName = self.decisionParams['name']
    else:
      elementType = self.decisionParams['elementType']

      if elementType == 'StorageElement':
        siteName = getSitesForSE(name)
      elif elementType == 'ComputingElement':
        siteName = getSiteForCE(name)
      else:
        siteName = {'OK': True, 'Value': 'Unassigned'}

      if not siteName['OK']:
        self.log.error('Resource %s does not exist at any site: %s' % (name, siteName['Message']))
        siteName = "Unassigned Resources"
      elif not siteName['Value']:
        siteName = "Unassigned Resources"
      else:
        siteName = siteName['Value'] if isinstance(siteName['Value'], basestring) else siteName['Value'][0]

    # create record for insertion
    recordDict = {}
    recordDict['SiteName'] = siteName
    recordDict['ResourceName'] = name
    recordDict['Status'] = status
    recordDict['PreviousStatus'] = previousStatus
    recordDict['StatusType'] = statusType

    return self.rsClient.insert('ResourceStatusCache', recordDict)
Example #13
0
  def __resolveFTSServer( self ):
    """
    resolve FTS server to use, it should be the closest one from target SE

    :param self: self reference
    """
    if not self.sourceSE:
      return S_ERROR( "Source SE not set" )
    if not self.targetSE:
      return S_ERROR( "Target SE not set" )
    res = getSitesForSE( self.sourceSE, 'LCG' )
    if not res['OK'] or not res['Value']:
      return S_ERROR( "Could not determine source site" )
    sourceSite = res['Value'][0]
    res = getSitesForSE( self.targetSE, 'LCG' )
    if not res['OK'] or not res['Value']:
      return S_ERROR( "Could not determine target site" )
    targetSite = res['Value'][0]

    # CERN is a special case, handling incoming and outgoing transfers
    if ( sourceSite == 'LCG.CERN.ch' ) or ( targetSite == 'LCG.CERN.ch' ):
      res = self.__getFTSServer( 'LCG.CERN.ch' )
      if res['OK']:
        self.ftsServer = res['Value']
        return S_OK( self.ftsServer )
      else:
        return res
    else:
      # Target site FTS server should be used
      sourceFTS = self.__getFTSServer( sourceSite )
      if sourceFTS['OK']:
        ftsSource = sourceFTS['Value']
        if 'fts3' in ftsSource:
          self.ftsServer = ftsSource
          return S_OK( self.ftsServer )
      targetFTS = self.__getFTSServer( targetSite )
      if targetFTS['OK']:
        ftsTarget = targetFTS['Value']
        if ftsTarget:
          self.ftsServer = ftsTarget
          return S_OK( self.ftsServer )
        else:
          return S_ERROR( 'No FTS server found for %s nor %s' % ( sourceSite, targetSite ) )
      else:
        return targetFTS
Example #14
0
 def _getSitesForSEs( self, seList ):
   """ Get all the sites for the given SE list
   """
   sites = []
   for se in seList:
     result = getSitesForSE( se, gridName = 'LCG' )
     if result['OK']:
       sites += result['Value']
   return sites
Example #15
0
 def _getSiteForSE(cls, se):
     """ Get site name for the given SE
 """
     result = getSitesForSE(se)
     if not result['OK']:
         return result
     if result['Value']:
         return S_OK(result['Value'][0])
     return S_OK('')
Example #16
0
 def _getSitesForSEs(cls, seList):
     """ Get all the sites for the given SE list
 """
     sites = []
     for se in seList:
         result = getSitesForSE(se, gridName="LCG")
         if result["OK"]:
             sites += result["Value"]
     return sites
Example #17
0
 def _getSiteForSE( cls, se ):
   """ Get site name for the given SE
   """
   result = getSitesForSE( se )
   if not result['OK']:
     return result
   if result['Value']:
     return S_OK( result['Value'][0] )
   return S_OK( '' )
Example #18
0
 def _getSitesForSEs( cls, seList ):
   """ Get all the sites for the given SE list
   """
   sites = []
   for se in seList:
     result = getSitesForSE( se )
     if result['OK']:
       sites += result['Value']
   return sites
Example #19
0
 def _getSiteForSE(cls, se):
     """ Get site name for the given SE
 """
     result = getSitesForSE(se, gridName="LCG")
     if not result["OK"]:
         return result
     if result["Value"]:
         return S_OK(result["Value"][0])
     return S_OK("")
Example #20
0
 def _getSitesForSEs(cls, seList):
     """ Get all the sites for the given SE list
 """
     sites = []
     for se in seList:
         result = getSitesForSE(se, gridName='LCG')
         if result['OK']:
             sites += result['Value']
     return sites
 def __getChannelSitesForSE(self, se):
     res = getSitesForSE(se)
     sites = []
     if res['OK']:
         for site in res['Value']:
             s = site.split('.')
             if len(s) > 1:
                 if not s[1] in sites:
                     sites.append(s[1])
     return sites
 def __getChannelSitesForSE( self, se ):
   res = getSitesForSE( se )
   sites = []
   if res['OK']:
     for site in res['Value']:
       s = site.split( '.' )
       if len( s ) > 1:
         if not s[1] in sites:
           sites.append( s[1] )
   return sites
Example #23
0
    def _handleDestination(self, paramsDict, getSitesForSE=None):
        """ Handle Sites and TargetSE in the parameters
    """

        try:
            sites = ['ANY']
            if paramsDict['Site']:
                # 'Site' comes from the XML and therefore is ; separated
                sites = fromChar(paramsDict['Site'], sepChar=';')
        except KeyError:
            pass

        try:
            seList = ['Unknown']
            if paramsDict['TargetSE']:
                seList = fromChar(paramsDict['TargetSE'])
        except KeyError:
            pass

        if not seList or seList == ['Unknown']:
            return sites

        # from now on we know there is some TargetSE requested
        if not getSitesForSE:
            from DIRAC.Core.Utilities.SiteSEMapping import getSitesForSE

        seSites = []
        for se in seList:
            res = getSitesForSE(se)
            if not res['OK']:
                self.log.warn('Could not get Sites associated to SE',
                              res['Message'])
            else:
                thisSESites = res['Value']
                if not thisSESites:
                    continue
                if seSites == []:
                    seSites = copy.deepcopy(thisSESites)
                else:
                    # We make an OR of the possible sites
                    for nSE in list(thisSESites):
                        if nSE not in seSites:
                            seSites.append(nSE)

        # Now we need to make the AND with the sites, if defined
        if sites == ['ANY']:
            return seSites
        else:
            # Need to get the AND
            for nSE in list(seSites):
                if nSE not in sites:
                    seSites.remove(nSE)

            return seSites
Example #24
0
  def _handleDestination( self, paramsDict, getSitesForSE = None ):
    """ Handle Sites and TargetSE in the parameters
    """

    try:
      sites = ['ANY']
      if paramsDict['Site']:
        # 'Site' comes from the XML and therefore is ; separated
        sites = fromChar( paramsDict['Site'], sepChar = ';' )
    except KeyError:
      pass

    try:
      seList = ['Unknown']
      if paramsDict['TargetSE']:
        seList = fromChar( paramsDict['TargetSE'] )
    except KeyError:
      pass

    if not seList or seList == ['Unknown']:
      return sites

    # from now on we know there is some TargetSE requested
    if not getSitesForSE:
      from DIRAC.Core.Utilities.SiteSEMapping import getSitesForSE

    seSites = []
    for se in seList:
      res = getSitesForSE( se )
      if not res['OK']:
        self.log.warn( 'Could not get Sites associated to SE', res['Message'] )
      else:
        thisSESites = res['Value']
        if not thisSESites:
          continue
        if seSites == []:
          seSites = copy.deepcopy( thisSESites )
        else:
          # We make an OR of the possible sites
          for nSE in list( thisSESites ):
            if nSE not in seSites:
              seSites.append( nSE )

    # Now we need to make the AND with the sites, if defined
    if sites == ['ANY']:
      return seSites
    else:
      # Need to get the AND
      for nSE in list( seSites ):
        if nSE not in sites:
          seSites.remove( nSE )

      return seSites
Example #25
0
 def __getChannelSitesForSE(self, storageElement):
     """Get sites for given storage element.
 
 :param self: self reference
 :param str storageElement: storage element name
 """
     res = getSitesForSE(storageElement)
     if not res["OK"]:
         return []
     sites = []
     for site in res["Value"]:
         siteName = site.split(".")
         if len(siteName) > 1:
             if not siteName[1] in sites:
                 sites.append(siteName[1])
     return sites
Example #26
0
 def __getChannelSitesForSE( self, storageElement ):
   """Get sites for given storage element.
   
   :param self: self reference
   :param str storageElement: storage element name
   """
   res = getSitesForSE( storageElement )
   if not res["OK"]:
     return []
   sites = []
   for site in res["Value"]:
     siteName = site.split( "." )
     if len( siteName ) > 1:
       if not siteName[1] in sites:
         sites.append( siteName[1] )
   return sites
Example #27
0
  def __resolveFTSServer( self ):
    """
    resolve FTS server to use, it should be the closest one from target SE

    :param self: self reference
    """
    if self.ftsVersion.upper() == 'FTS2':

      from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getFTS2ServersForSites
      if not self.targetSE:
        return S_ERROR( "Target SE not set" )
      res = getSitesForSE( self.targetSE )
      if not res['OK'] or not res['Value']:
        return S_ERROR( "Could not determine target site" )
      targetSites = res['Value']

      targetSite = ''
      for targetSite in targetSites:
        targetFTS = getFTS2ServersForSites( [targetSite] )
        if targetFTS['OK']:
          ftsTarget = targetFTS['Value'][targetSite]
          if ftsTarget:
            self.ftsServer = ftsTarget
            return S_OK( self.ftsServer )
        else:
          return targetFTS

    elif self.ftsVersion.upper() == 'FTS3':

      from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getFTS3Servers
      res = getFTS3Servers()
      if not res['OK']:
        return res
      ftsServerList = res['Value']
      if ftsServerList:
        # Here we take the first one, regardless of the policy...
        # Unclean but all this will disapear after refactoring the fts code
        self.ftsServer = ftsServerList[0]
        return S_OK( self.ftsServer )

    else:
      return S_ERROR( 'Unknown FTS version %s' % self.ftsVersion )


    return S_ERROR( 'No FTS server found for %s' % targetSite )
Example #28
0
  def __resolveFTSServer( self ):
    """
    resolve FTS server to use, it should be the closest one from target SE

    :param self: self reference
    """
    if self.ftsVersion.upper() == 'FTS2':

      from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getFTS2ServersForSites
      if not self.targetSE:
        return S_ERROR( "Target SE not set" )
      res = getSitesForSE( self.targetSE )
      if not res['OK'] or not res['Value']:
        return S_ERROR( "Could not determine target site" )
      targetSites = res['Value']

      targetSite = ''
      for targetSite in targetSites:
        targetFTS = getFTS2ServersForSites( [targetSite] )
        if targetFTS['OK']:
          ftsTarget = targetFTS['Value'][targetSite]
          if ftsTarget:
            self.ftsServer = ftsTarget
            return S_OK( self.ftsServer )
        else:
          return targetFTS

    elif self.ftsVersion.upper() == 'FTS3':

      from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getFTS3Servers
      res = getFTS3Servers()
      if not res['OK']:
        return res
      ftsServerList = res['Value']
      if ftsServerList:
        # Here we take the first one, regardless of the policy...
        # Unclean but all this will disapear after refactoring the fts code
        self.ftsServer = ftsServerList[0]
        return S_OK( self.ftsServer )

    else:
      return S_ERROR( 'Unknown FTS version %s' % self.ftsVersion )


    return S_ERROR( 'No FTS server found for %s' % targetSite )
Example #29
0
    def __getSitesForSE(self, se):
        """ Returns a list of sites having the given SE as a local one.
        Uses the local cache of the site-se information
    """

        # Empty the cache if too old
        if (time.time() - self.lastCScheck) > self.cacheLength:
            self.log.verbose('Resetting the SE to site mapping cache')
            self.seToSiteMapping = {}
            self.lastCScheck = time.time()

        if se not in self.seToSiteMapping:
            sites = getSitesForSE(se)
            if sites['OK']:
                self.seToSiteMapping[se] = list(sites['Value'])
            return sites
        else:
            return S_OK(self.seToSiteMapping[se])
Example #30
0
  def __getSitesForSE( self, se ):
    """ Returns a list of sites having the given SE as a local one.
        Uses the local cache of the site-se information
    """

    # Empty the cache if too old
    if ( time.time() - self.lastCScheck ) > self.cacheLength:
      self.log.verbose( 'Resetting the SE to site mapping cache' )
      self.seToSiteMapping = {}
      self.lastCScheck = time.time()

    if se not in self.seToSiteMapping:
      sites = getSitesForSE( se )
      if sites['OK']:
        self.seToSiteMapping[se] = list( sites['Value'] )
      return sites
    else:
      return S_OK( self.seToSiteMapping[se] )
Example #31
0
    def __getSitesForSE(self, seName):
        """ Returns a list of sites having the given SE as a local one.
        Uses the local cache of the site-se information
    """

        # Empty the cache if too old
        now = time.time()
        if (now - self.__lastCacheUpdate) > self.__cacheLifeTime:
            self.log.verbose('Resetting the SE to site mapping cache')
            self.__SEToSiteMap = {}
            self.__lastCacheUpdate = now

        if seName not in self.__SEToSiteMap:
            result = getSitesForSE(seName)
            if not result['OK']:
                return result
            self.__SEToSiteMap[seName] = list(result['Value'])
        return S_OK(self.__SEToSiteMap[seName])
Example #32
0
  def __getSitesForSE( self, seName ):
    """ Returns a list of sites having the given SE as a local one.
        Uses the local cache of the site-se information
    """

    # Empty the cache if too old
    now = time.time()
    if ( now - self.__lastCacheUpdate ) > self.__cacheLifeTime:
      self.log.verbose( 'Resetting the SE to site mapping cache' )
      self.__SEToSiteMap = {}
      self.__lastCacheUpdate = now

    if seName not in self.__SEToSiteMap:
      result = getSitesForSE( seName )
      if not result['OK']:
        return result
      self.__SEToSiteMap[ seName ] = list( result['Value'] )
    return S_OK( self.__SEToSiteMap[ seName ] )
 def _getExistingCounters( self, normalise = False, requestedSites = [] ):
   res = self.transClient.getCounters( 'TransformationFiles', ['UsedSE'], {'TransformationID':self.params['TransformationID']} )
   if not res['OK']:
     return res
   usageDict = {}
   for usedDict, count in res['Value']:
     usedSE = usedDict['UsedSE']
     if usedSE != 'Unknown':
       usageDict[usedSE] = count
   if requestedSites:
     siteDict = {}
     for se, count in usageDict.items():
       res = getSitesForSE( se, gridName = 'LCG' )
       if not res['OK']:
         return res
       for site in res['Value']:
         if site in requestedSites:
           siteDict[site] = count
     usageDict = siteDict.copy()
   if normalise:
     usageDict = self._normaliseShares( usageDict )
   return S_OK( usageDict )
Example #34
0
 def _getExistingCounters(self, normalise=False, requestedSites=[]):
     res = self.transClient.getCounters(
         "TransformationFiles", ["UsedSE"], {"TransformationID": self.params["TransformationID"]}
     )
     if not res["OK"]:
         return res
     usageDict = {}
     for usedDict, count in res["Value"]:
         usedSE = usedDict["UsedSE"]
         if usedSE != "Unknown":
             usageDict[usedSE] = count
     if requestedSites:
         siteDict = {}
         for se, count in usageDict.items():
             res = getSitesForSE(se, gridName="LCG")
             if not res["OK"]:
                 return res
             for site in res["Value"]:
                 if site in requestedSites:
                     siteDict[site] = count
         usageDict = siteDict.copy()
     if normalise:
         usageDict = self._normaliseShares(usageDict)
     return S_OK(usageDict)
Example #35
0
 def getExistingCounters(self, normalise=False, requestedSites=[]):
     res = self.transClient.getCounters(
         "TransformationFiles", ["UsedSE"],
         {"TransformationID": self.params["TransformationID"]})
     if not res["OK"]:
         return res
     usageDict = {}
     for usedDict, count in res["Value"]:
         usedSE = usedDict["UsedSE"]
         if usedSE != "Unknown":
             usageDict[usedSE] = count
     if requestedSites:
         siteDict = {}
         for se, count in usageDict.items():
             res = getSitesForSE(se)
             if not res["OK"]:
                 return res
             for site in res["Value"]:
                 if site in requestedSites:
                     siteDict[site] = count
         usageDict = siteDict.copy()
     if normalise:
         usageDict = self._normaliseShares(usageDict)
     return S_OK(usageDict)
Example #36
0
  def run( self ):
    ''' Checks it has the parameters it needs and writes the date to a cache file.
    '''
    # Minor security checks

    element = self.decisionParams[ 'element' ]
    if element is None:
      return S_ERROR( 'element should not be None' )

    name = self.decisionParams[ 'name' ]
    if name is None:
      return S_ERROR( 'name should not be None' )

    statusType = self.decisionParams[ 'statusType' ]
    if statusType is None:
      return S_ERROR( 'statusType should not be None' )

    previousStatus = self.decisionParams[ 'status' ]
    if previousStatus is None:
      return S_ERROR( 'status should not be None' )

    status = self.enforcementResult[ 'Status' ]
    if status is None:
      return S_ERROR( 'status should not be None' )

    reason = self.enforcementResult[ 'Reason' ]
    if reason is None:
      return S_ERROR( 'reason should not be None' )

    if self.decisionParams[ 'element' ] == 'Site':
      siteName = self.decisionParams[ 'name' ]
    else:
      siteName = getSitesForSE(name)

      if not siteName['OK']:
        self.log.error('Resource %s does not exist at any site: %s' % (name, siteName['Message']))
        siteName = "Unassigned Resources"
      elif not siteName['Value']:
        siteName = "Unassigned Resources"
      else:
        siteName = siteName['Value'][0]

    with sqlite3.connect(self.cacheFile) as conn:

      try:
        conn.execute('''CREATE TABLE IF NOT EXISTS ResourceStatusCache(
                      SiteName VARCHAR(64) NOT NULL,
                      ResourceName VARCHAR(64) NOT NULL,
                      Status VARCHAR(8) NOT NULL DEFAULT "",
                      PreviousStatus VARCHAR(8) NOT NULL DEFAULT "",
                      StatusType VARCHAR(128) NOT NULL DEFAULT "all",
                      Time TIMESTAMP DEFAULT CURRENT_TIMESTAMP
                     );''')

      except sqlite3.OperationalError:
        self.log.error('Email cache database is locked')

      conn.execute("INSERT INTO ResourceStatusCache (SiteName, ResourceName, Status, PreviousStatus, StatusType)"
                   " VALUES ('" + siteName + "', '" + name + "', '" + status + "', '" + previousStatus + "', '" + statusType + "' ); "
                  )

      conn.commit()

    return S_OK()
Example #37
0
  def run( self ):
    ''' Checks it has the parameters it needs and writes the date to a cache file.
    '''
    # Minor security checks

    element = self.decisionParams[ 'element' ]
    if element is None:
      return S_ERROR( 'element should not be None' )

    name = self.decisionParams[ 'name' ]
    if name is None:
      return S_ERROR( 'name should not be None' )

    statusType = self.decisionParams[ 'statusType' ]
    if statusType is None:
      return S_ERROR( 'statusType should not be None' )

    previousStatus = self.decisionParams[ 'status' ]
    if previousStatus is None:
      return S_ERROR( 'status should not be None' )

    status = self.enforcementResult[ 'Status' ]
    if status is None:
      return S_ERROR( 'status should not be None' )

    reason = self.enforcementResult[ 'Reason' ]
    if reason is None:
      return S_ERROR( 'reason should not be None' )

    siteName = getSitesForSE(name)

    if not siteName['OK']:
      self.log.error('Resource %s does not exist at any site: %s' % (name, siteName['Message']))
      siteName = "Unassigned Resources"
    elif not siteName['Value']:
      siteName = "Unassigned Resources"
    else:
      siteName = siteName['Value'][0]

    with sqlite3.connect(self.cacheFile) as conn:

      try:
        conn.execute('''CREATE TABLE IF NOT EXISTS ResourceStatusCache(
                      SiteName VARCHAR(64) NOT NULL,
                      ResourceName VARCHAR(64) NOT NULL,
                      Status VARCHAR(8) NOT NULL DEFAULT "",
                      PreviousStatus VARCHAR(8) NOT NULL DEFAULT "",
                      StatusType VARCHAR(128) NOT NULL DEFAULT "all",
                      Time TIMESTAMP DEFAULT CURRENT_TIMESTAMP
                     );''')
                     
      except sqlite3.OperationalError:
        self.log.error('Email cache database is locked')

      conn.execute("INSERT INTO ResourceStatusCache (SiteName, ResourceName, Status, PreviousStatus, StatusType)"
                   " VALUES ('" + siteName + "', '" + name + "', '" + status + "', '" + previousStatus + "', '" + statusType + "' ); "
                  )

      conn.commit()

    return S_OK()
Example #38
0
def getSitesForSE(se):
    ''' Get the Sites associated with this SE'''
    from DIRAC.Core.Utilities.SiteSEMapping import getSitesForSE
    result = getSitesForSE(storageElement=se)
    return result
Example #39
0
  def run(self):
    ''' Checks it has the parameters it needs and writes the date to a cache file.
    '''
    # Minor security checks

    element = self.decisionParams['element']
    if element is None:
      return S_ERROR('element should not be None')

    name = self.decisionParams['name']
    if name is None:
      return S_ERROR('name should not be None')

    statusType = self.decisionParams['statusType']
    if statusType is None:
      return S_ERROR('statusType should not be None')

    previousStatus = self.decisionParams['status']
    if previousStatus is None:
      return S_ERROR('status should not be None')

    status = self.enforcementResult['Status']
    if status is None:
      return S_ERROR('status should not be None')

    reason = self.enforcementResult['Reason']
    if reason is None:
      return S_ERROR('reason should not be None')

    if self.decisionParams['element'] == 'Site':
      siteName = self.decisionParams['name']
    else:
      elementType = self.decisionParams['elementType']

      if elementType == 'StorageElement':
        siteName = getSitesForSE(name)
      elif elementType == 'ComputingElement':
        res = getCESiteMapping(name)
        if not res['OK']:
          return res
        siteName = S_OK(res['Value'][name])
      else:
        siteName = {'OK': True, 'Value': 'Unassigned'}

      if not siteName['OK']:
        self.log.error('Resource %s does not exist at any site: %s' % (name, siteName['Message']))
        siteName = "Unassigned Resources"
      elif not siteName['Value']:
        siteName = "Unassigned Resources"
      else:
        siteName = siteName['Value'] if isinstance(siteName['Value'], six.string_types) else siteName['Value'][0]

    with sqlite3.connect(self.cacheFile) as conn:

      try:
        conn.execute('''CREATE TABLE IF NOT EXISTS ResourceStatusCache(
                      SiteName VARCHAR(64) NOT NULL,
                      ResourceName VARCHAR(64) NOT NULL,
                      Status VARCHAR(8) NOT NULL DEFAULT "",
                      PreviousStatus VARCHAR(8) NOT NULL DEFAULT "",
                      StatusType VARCHAR(128) NOT NULL DEFAULT "all",
                      Time TIMESTAMP DEFAULT CURRENT_TIMESTAMP
                     );''')

        insertQuery = "INSERT INTO ResourceStatusCache (SiteName, ResourceName, Status, PreviousStatus, StatusType)"
        insertQuery += " VALUES ('%s', '%s', '%s', '%s', '%s' ); " % (siteName, name, status,
                                                                      previousStatus, statusType)
        conn.execute(insertQuery)

        conn.commit()

      except sqlite3.OperationalError:
        self.log.error('Email cache database is locked')

    return S_OK()
Example #40
0
    def run(self):
        """
        Checks it has the parameters it needs and tries to send an sms to the users
        that apply.
        """

        if self.url is None:
            return S_ERROR("Slack URL not set")

        # Minor security checks
        element = self.decisionParams["element"]
        if element is None:
            return S_ERROR("element should not be None")

        name = self.decisionParams["name"]
        if name is None:
            return S_ERROR("name should not be None")

        statusType = self.decisionParams["statusType"]
        if statusType is None:
            return S_ERROR("statusType should not be None")

        previousStatus = self.decisionParams["status"]
        if previousStatus is None:
            return S_ERROR("status should not be None")

        status = self.enforcementResult["Status"]
        if status is None:
            return S_ERROR("status should not be None")

        reason = self.enforcementResult["Reason"]
        if reason is None:
            return S_ERROR("reason should not be None")

        if self.decisionParams["element"] == "Site":
            siteName = self.decisionParams["name"]
        else:
            elementType = self.decisionParams["elementType"]

            if elementType == "StorageElement":
                siteName = getSitesForSE(name)
            elif elementType == "ComputingElement":
                res = getCESiteMapping(name)
                if not res["OK"]:
                    self.log.error("Failure getting Site2CE mapping",
                                   res["Message"])
                    siteName = "ERROR"
                else:
                    siteName = res
            else:
                siteName = {"OK": True, "Value": "Unassigned"}

            if not siteName["OK"]:
                self.log.error("Resource %s does not exist at any site: %s" %
                               (name, siteName["Message"]))
                siteName = "Unassigned Resources"
            elif not siteName["Value"]:
                siteName = "Unassigned Resources"
            else:
                siteName = (siteName["Value"] if isinstance(
                    siteName["Value"], six.string_types) else
                            siteName["Value"][0])

        message = "*{name}* _{statusType}_ --> _{status}_ \n{reason}".format(
            name=name, statusType=statusType, status=status, reason=reason)
        return self.sendSlackMessage(message)
Example #41
0
  def splitInputDataBySize(self, lfns, maxSizePerJob=20, printOutput=False):
    """Split the supplied lfn list by the replicas present at the possible
       destination sites, based on a maximum size.
       An S_OK object will be returned containing a list of
       lists in order to create the jobs.

       Example usage:

       >>> d.splitInputDataBySize(lfns,10)
       {'OK': True, 'Value': [['<LFN>'], ['<LFN>']]}


       @param lfns: Logical File Name(s) to split
       @type lfns: list
       @param maxSizePerJob: Maximum size (in GB) per bunch
       @type maxSizePerJob: integer
       @param printOutput: Optional flag to print result
       @type printOutput: boolean
       @return: S_OK,S_ERROR
    """
    sitesForSE = {}
    if isinstance(lfns, str):
      lfns = [lfns.replace('LFN:', '')]
    elif isinstance(lfns, list):
      try:
        lfns = [str(lfn.replace('LFN:', '')) for lfn in lfns]
      except TypeError as x:
        return self._errorReport(str(x), 'Expected strings for LFNs')
    else:
      return self._errorReport('Expected single string or list of strings for LFN(s)')

    if not isinstance(maxSizePerJob, int):
      try:
        maxSizePerJob = int(maxSizePerJob)
      except ValueError as x:
        return self._errorReport(str(x), 'Expected integer for maxSizePerJob')
    maxSizePerJob *= 1000 * 1000 * 1000

    replicaDict = self.getReplicas(lfns)
    if not replicaDict['OK']:
      return replicaDict
    replicas = replicaDict['Value']['Successful']
    if not replicas:
      return self._errorReport(replicaDict['Value']['Failed'].items()[0],
                               'Failed to get replica information')
    siteLfns = {}
    for lfn, reps in replicas.items():
      possibleSites = set(site
                          for se in reps
                          for site in sitesForSE.setdefault(se, getSitesForSE(se).get('Value', [])))
      siteLfns.setdefault(','.join(sorted(possibleSites)), []).append(lfn)

    if '' in siteLfns:
      # Some files don't have active replicas
      return self._errorReport('No active replica found for', str(siteLfns['']))
    # Get size of files
    metadataDict = self.getLfnMetadata(lfns, printOutput)
    if not metadataDict['OK']:
      return metadataDict
    fileSizes = dict((lfn, metadataDict['Value']['Successful'].get(lfn, {}).get('Size', maxSizePerJob))
                     for lfn in lfns)

    lfnGroups = []
    # maxSize is in GB
    for files in siteLfns.values():
      # Now get bunches of files,
      # Sort in decreasing size
      files.sort(cmp=(lambda f1, f2: fileSizes[f2] - fileSizes[f1]))
      while files:
        # print [( lfn, fileSizes[lfn] ) for lfn in files]
        group = []
        sizeTot = 0
        for lfn in list(files):
          size = fileSizes[lfn]
          if size >= maxSizePerJob:
            lfnGroups.append([lfn])
          elif sizeTot + size < maxSizePerJob:
            sizeTot += size
            group.append(lfn)
            files.remove(lfn)
        if group:
          lfnGroups.append(group)

    if printOutput:
      self.log.notice(self.pPrint.pformat(lfnGroups))
    return S_OK(lfnGroups)
Example #42
0
    def run(self):
        """Checks it has the parameters it needs and writes the date to a cache file."""
        # Minor security checks

        element = self.decisionParams["element"]
        if element is None:
            return S_ERROR("element should not be None")

        name = self.decisionParams["name"]
        if name is None:
            return S_ERROR("name should not be None")

        statusType = self.decisionParams["statusType"]
        if statusType is None:
            return S_ERROR("statusType should not be None")

        previousStatus = self.decisionParams["status"]
        if previousStatus is None:
            return S_ERROR("status should not be None")

        status = self.enforcementResult["Status"]
        if status is None:
            return S_ERROR("status should not be None")

        reason = self.enforcementResult["Reason"]
        if reason is None:
            return S_ERROR("reason should not be None")

        if self.decisionParams["element"] == "Site":
            siteName = self.decisionParams["name"]
        else:
            elementType = self.decisionParams["elementType"]

            if elementType == "StorageElement":
                siteName = getSitesForSE(name)
            elif elementType == "ComputingElement":
                res = getCESiteMapping(name)
                if not res["OK"]:
                    return res
                siteName = S_OK(res["Value"][name])
            else:
                siteName = {"OK": True, "Value": "Unassigned"}

            if not siteName["OK"]:
                self.log.error("Resource %s does not exist at any site: %s" % (name, siteName["Message"]))
                siteName = "Unassigned Resources"
            elif not siteName["Value"]:
                siteName = "Unassigned Resources"
            else:
                siteName = (
                    siteName["Value"] if isinstance(siteName["Value"], six.string_types) else siteName["Value"][0]
                )

        # create record for insertion
        recordDict = {}
        recordDict["SiteName"] = siteName
        recordDict["ResourceName"] = name
        recordDict["Status"] = status
        recordDict["PreviousStatus"] = previousStatus
        recordDict["StatusType"] = statusType

        return self.rsClient.insert("ResourceStatusCache", recordDict)
Example #43
0
def getSitesForSE(se):
    from DIRAC.Core.Utilities.SiteSEMapping import getSitesForSE
    result = getSitesForSE(storageElement=se)
    output(result)
Example #44
0
def getSitesForSE(se):
    ''' Get the Sites associated with this SE'''
    from DIRAC.Core.Utilities.SiteSEMapping import getSitesForSE
    result = getSitesForSE(storageElement=se)
    output(result)
Example #45
0
def getSitesForSE(se):
    from DIRAC.Core.Utilities.SiteSEMapping import getSitesForSE
    result = getSitesForSE(storageElement=se)
    output(result)