Exemplo n.º 1
0
  def execute( self ):

    res = self.RequestDBClient.getRequest( 'integrity' )
    if not res['OK']:
      gLogger.info( "LFCvsSEAgent.execute: Failed to get request from database." )
      return S_OK()
    elif not res['Value']:
      gLogger.info( "LFCvsSEAgent.execute: No requests to be executed found." )
      return S_OK()
    requestString = res['Value']['RequestString']
    requestName = res['Value']['RequestName']
    sourceServer = res['Value']['Server']
    gLogger.info( "LFCvsSEAgent.execute: Obtained request %s" % requestName )
    oRequest = RequestContainer( request = requestString )

    ################################################
    # Find the number of sub-requests from the request
    res = oRequest.getNumSubRequests( 'integrity' )
    if not res['OK']:
      errStr = "LFCvsSEAgent.execute: Failed to obtain number of integrity subrequests."
      gLogger.error( errStr, res['Message'] )
      return S_OK()
    gLogger.info( "LFCvsSEAgent.execute: Found %s sub requests." % res['Value'] )

    ################################################
    # For all the sub-requests in the request
    for ind in range( res['Value'] ):
      gLogger.info( "LFCvsSEAgent.execute: Processing sub-request %s." % ind )
      subRequestAttributes = oRequest.getSubRequestAttributes( ind, 'integrity' )['Value']
      if subRequestAttributes['Status'] == 'Waiting':
        subRequestFiles = oRequest.getSubRequestFiles( ind, 'integrity' )['Value']
        operation = subRequestAttributes['Operation']

        ################################################
        #  If the sub-request is a lfcvsse operation
        if operation == 'LFCvsSE':
          gLogger.info( "LFCvsSEAgent.execute: Attempting to execute %s sub-request." % operation )
          for subRequestFile in subRequestFiles:
            if subRequestFile['Status'] == 'Waiting':
              lfn = subRequestFile['LFN']
              oNamespaceBrowser = NamespaceBrowser( lfn )

              # Loop over all the directories and sub-directories
              while ( oNamespaceBrowser.isActive() ):
                currentDir = oNamespaceBrowser.getActiveDir()
                gLogger.info( "LFCvsSEAgent.execute: Attempting to get contents of %s." % currentDir )
                res = self.ReplicaManager.getCatalogDirectoryContents( currentDir )
                if not res['OK']:
                  subDirs = [currentDir]
                elif res['Value']['Failed'].has_key( currentDir ):
                  subDirs = [currentDir]
                else:
                  subDirs = res['Value']['Successful'][currentDir]['SubDirs']
                  files = res['Value']['Successful'][currentDir]['Files']

                  lfnSizeDict = {}
                  pfnLfnDict = {}
                  pfnStatusDict = {}
                  sePfnDict = {}
                  for lfn, lfnDict in files.items():
                    lfnSizeDict[lfn] = lfnDict['MetaData']['Size']
                    for se in lfnDict['Replicas'].keys():
                      pfn = lfnDict['Replicas'][se]['PFN']
                      status = lfnDict['Replicas'][se]['Status']
                      pfnStatusDict[pfn] = status
                      pfnLfnDict[pfn] = lfn
                      if not sePfnDict.has_key( se ):
                        sePfnDict[se] = []
                      sePfnDict[se].append( pfn )

                  for storageElementName, physicalFiles in sePfnDict.items():
                    gLogger.info( "LFCvsSEAgent.execute: Attempting to get metadata for files on %s." % storageElementName )
                    res = self.ReplicaManager.getStorageFileMetadata( physicalFiles, storageElementName )
                    if not res['OK']:
                      gLogger.error( "LFCvsSEAgent.execute: Completely failed to get physical file metadata.", res['Message'] )
                    else:
                      for pfn in res['Value']['Failed'].keys():
                        gLogger.error( "LFCvsSEAgent.execute: Failed to get metadata.", "%s %s" % ( pfn, res['Value']['Failed'][pfn] ) )
                        lfn = pfnLfnDict[pfn]
                        fileMetadata = {'Prognosis':'MissingSEPfn', 'LFN':lfn, 'PFN':pfn, 'StorageElement':storageElementName, 'Size':lfnSizeDict[lfn]}
                        IntegrityDB = RPCClient( 'DataManagement/DataIntegrity' )
                        resInsert = IntegrityDB.insertProblematic( AGENT_NAME, fileMetadata )
                        if resInsert['OK']:
                          gLogger.info( "LFCvsSEAgent.execute: Successfully added to IntegrityDB." )
                          gLogger.error( "Change the status in the LFC,ProcDB...." )
                        else:
                          gLogger.error( "Shit, f**k, bugger. Add the failover." )
                      for pfn, pfnDict in res['Value']['Successful'].items():
                        lfn = pfnLfnDict[pfn]
                        catalogSize = int( lfnSizeDict[lfn] )
                        storageSize = int( pfnDict['Size'] )
                        if int( catalogSize ) == int( storageSize ):
                          gLogger.info( "LFCvsSEAgent.execute: Catalog and storage sizes match.", "%s %s" % ( pfn, storageElementName ) )
                          gLogger.info( "Change the status in the LFC" )
                        else:
                          gLogger.error( "LFCvsSEAgent.execute: Catalog and storage size mis-match.", "%s %s" % ( pfn, storageElementName ) )
                          fileMetadata = {'Prognosis':'PfnSizeMismatch', 'LFN':lfn, 'PFN':pfn, 'StorageElement':storageElementName}
                          IntegrityDB = RPCClient( 'DataManagement/DataIntegrity' )
                          resInsert = IntegrityDB.insertProblematic( AGENT_NAME, fileMetadata )
                          if resInsert['OK']:
                            gLogger.info( "LFCvsSEAgent.execute: Successfully added to IntegrityDB." )
                            gLogger.error( "Change the status in the LFC,ProcDB...." )
                          else:
                            gLogger.error( "Shit, f**k, bugger. Add the failover." )
                oNamespaceBrowser.updateDirs( subDirs )
              oRequest.setSubRequestFileAttributeValue( ind, 'integrity', lfn, 'Status', 'Done' )

        ################################################
        #  If the sub-request is none of the above types
        else:
          gLogger.info( "LFCvsSEAgent.execute: Operation not supported.", operation )

        ################################################
        #  Determine whether there are any active files
        if oRequest.isSubRequestEmpty( ind, 'integrity' )['Value']:
          oRequest.setSubRequestStatus( ind, 'integrity', 'Done' )

      ################################################
      #  If the sub-request is already in terminal state
      else:
        gLogger.info( "LFCvsSEAgent.execute: Sub-request %s is status '%s' and  not to be executed." % ( ind, subRequestAttributes['Status'] ) )

    ################################################
    #  Generate the new request string after operation
    requestString = oRequest.toXML()['Value']
    res = self.RequestDBClient.updateRequest( requestName, requestString, sourceServer )

    return S_OK()
Exemplo n.º 2
0
    def execute(self):

        res = self.RequestDBClient.getRequest('integrity')
        if not res['OK']:
            gLogger.info(
                "LFCvsSEAgent.execute: Failed to get request from database.")
            return S_OK()
        elif not res['Value']:
            gLogger.info(
                "LFCvsSEAgent.execute: No requests to be executed found.")
            return S_OK()
        requestString = res['Value']['RequestString']
        requestName = res['Value']['RequestName']
        sourceServer = res['Value']['Server']
        gLogger.info("LFCvsSEAgent.execute: Obtained request %s" % requestName)
        oRequest = RequestContainer(request=requestString)

        ################################################
        # Find the number of sub-requests from the request
        res = oRequest.getNumSubRequests('integrity')
        if not res['OK']:
            errStr = "LFCvsSEAgent.execute: Failed to obtain number of integrity subrequests."
            gLogger.error(errStr, res['Message'])
            return S_OK()
        gLogger.info("LFCvsSEAgent.execute: Found %s sub requests." %
                     res['Value'])

        ################################################
        # For all the sub-requests in the request
        for ind in range(res['Value']):
            gLogger.info("LFCvsSEAgent.execute: Processing sub-request %s." %
                         ind)
            subRequestAttributes = oRequest.getSubRequestAttributes(
                ind, 'integrity')['Value']
            if subRequestAttributes['Status'] == 'Waiting':
                subRequestFiles = oRequest.getSubRequestFiles(
                    ind, 'integrity')['Value']
                operation = subRequestAttributes['Operation']

                ################################################
                #  If the sub-request is a lfcvsse operation
                if operation == 'LFCvsSE':
                    gLogger.info(
                        "LFCvsSEAgent.execute: Attempting to execute %s sub-request."
                        % operation)
                    for subRequestFile in subRequestFiles:
                        if subRequestFile['Status'] == 'Waiting':
                            lfn = subRequestFile['LFN']
                            oNamespaceBrowser = NamespaceBrowser(lfn)

                            # Loop over all the directories and sub-directories
                            while (oNamespaceBrowser.isActive()):
                                currentDir = oNamespaceBrowser.getActiveDir()
                                gLogger.info(
                                    "LFCvsSEAgent.execute: Attempting to get contents of %s."
                                    % currentDir)
                                res = self.ReplicaManager.getCatalogDirectoryContents(
                                    currentDir)
                                if not res['OK']:
                                    subDirs = [currentDir]
                                elif res['Value']['Failed'].has_key(
                                        currentDir):
                                    subDirs = [currentDir]
                                else:
                                    subDirs = res['Value']['Successful'][
                                        currentDir]['SubDirs']
                                    files = res['Value']['Successful'][
                                        currentDir]['Files']

                                    lfnSizeDict = {}
                                    pfnLfnDict = {}
                                    pfnStatusDict = {}
                                    sePfnDict = {}
                                    for lfn, lfnDict in files.items():
                                        lfnSizeDict[lfn] = lfnDict['MetaData'][
                                            'Size']
                                        for se in lfnDict['Replicas'].keys():
                                            pfn = lfnDict['Replicas'][se][
                                                'PFN']
                                            status = lfnDict['Replicas'][se][
                                                'Status']
                                            pfnStatusDict[pfn] = status
                                            pfnLfnDict[pfn] = lfn
                                            if not sePfnDict.has_key(se):
                                                sePfnDict[se] = []
                                            sePfnDict[se].append(pfn)

                                    for storageElementName, physicalFiles in sePfnDict.items(
                                    ):
                                        gLogger.info(
                                            "LFCvsSEAgent.execute: Attempting to get metadata for files on %s."
                                            % storageElementName)
                                        res = self.ReplicaManager.getStorageFileMetadata(
                                            physicalFiles, storageElementName)
                                        if not res['OK']:
                                            gLogger.error(
                                                "LFCvsSEAgent.execute: Completely failed to get physical file metadata.",
                                                res['Message'])
                                        else:
                                            for pfn in res['Value'][
                                                    'Failed'].keys():
                                                gLogger.error(
                                                    "LFCvsSEAgent.execute: Failed to get metadata.",
                                                    "%s %s" %
                                                    (pfn, res['Value']
                                                     ['Failed'][pfn]))
                                                lfn = pfnLfnDict[pfn]
                                                fileMetadata = {
                                                    'Prognosis':
                                                    'MissingSEPfn',
                                                    'LFN': lfn,
                                                    'PFN': pfn,
                                                    'StorageElement':
                                                    storageElementName,
                                                    'Size': lfnSizeDict[lfn]
                                                }
                                                IntegrityDB = RPCClient(
                                                    'DataManagement/DataIntegrity'
                                                )
                                                resInsert = IntegrityDB.insertProblematic(
                                                    AGENT_NAME, fileMetadata)
                                                if resInsert['OK']:
                                                    gLogger.info(
                                                        "LFCvsSEAgent.execute: Successfully added to IntegrityDB."
                                                    )
                                                    gLogger.error(
                                                        "Change the status in the LFC,ProcDB...."
                                                    )
                                                else:
                                                    gLogger.error(
                                                        "Shit, f**k, bugger. Add the failover."
                                                    )
                                            for pfn, pfnDict in res['Value'][
                                                    'Successful'].items():
                                                lfn = pfnLfnDict[pfn]
                                                catalogSize = int(
                                                    lfnSizeDict[lfn])
                                                storageSize = int(
                                                    pfnDict['Size'])
                                                if int(catalogSize) == int(
                                                        storageSize):
                                                    gLogger.info(
                                                        "LFCvsSEAgent.execute: Catalog and storage sizes match.",
                                                        "%s %s" %
                                                        (pfn,
                                                         storageElementName))
                                                    gLogger.info(
                                                        "Change the status in the LFC"
                                                    )
                                                else:
                                                    gLogger.error(
                                                        "LFCvsSEAgent.execute: Catalog and storage size mis-match.",
                                                        "%s %s" %
                                                        (pfn,
                                                         storageElementName))
                                                    fileMetadata = {
                                                        'Prognosis':
                                                        'PfnSizeMismatch',
                                                        'LFN':
                                                        lfn,
                                                        'PFN':
                                                        pfn,
                                                        'StorageElement':
                                                        storageElementName
                                                    }
                                                    IntegrityDB = RPCClient(
                                                        'DataManagement/DataIntegrity'
                                                    )
                                                    resInsert = IntegrityDB.insertProblematic(
                                                        AGENT_NAME,
                                                        fileMetadata)
                                                    if resInsert['OK']:
                                                        gLogger.info(
                                                            "LFCvsSEAgent.execute: Successfully added to IntegrityDB."
                                                        )
                                                        gLogger.error(
                                                            "Change the status in the LFC,ProcDB...."
                                                        )
                                                    else:
                                                        gLogger.error(
                                                            "Shit, f**k, bugger. Add the failover."
                                                        )
                                oNamespaceBrowser.updateDirs(subDirs)
                            oRequest.setSubRequestFileAttributeValue(
                                ind, 'integrity', lfn, 'Status', 'Done')

                ################################################
                #  If the sub-request is none of the above types
                else:
                    gLogger.info(
                        "LFCvsSEAgent.execute: Operation not supported.",
                        operation)

                ################################################
                #  Determine whether there are any active files
                if oRequest.isSubRequestEmpty(ind, 'integrity')['Value']:
                    oRequest.setSubRequestStatus(ind, 'integrity', 'Done')

            ################################################
            #  If the sub-request is already in terminal state
            else:
                gLogger.info(
                    "LFCvsSEAgent.execute: Sub-request %s is status '%s' and  not to be executed."
                    % (ind, subRequestAttributes['Status']))

        ################################################
        #  Generate the new request string after operation
        requestString = oRequest.toXML()['Value']
        res = self.RequestDBClient.updateRequest(requestName, requestString,
                                                 sourceServer)

        return S_OK()
Exemplo n.º 3
0
  def execute( self ):

    IntegrityDB = RPCClient( 'DataManagement/DataIntegrity' )

    res = self.RequestDBClient.getRequest( 'integrity' )
    if not res['OK']:
      gLogger.info( "SEvsLFCAgent.execute: Failed to get request from database." )
      return S_OK()
    elif not res['Value']:
      gLogger.info( "SEvsLFCAgent.execute: No requests to be executed found." )
      return S_OK()
    requestString = res['Value']['requestString']
    requestName = res['Value']['requestName']
    sourceServer = res['Value']['Server']
    gLogger.info( "SEvsLFCAgent.execute: Obtained request %s" % requestName )
    oRequest = RequestContainer( request = requestString )

    ################################################
    # Find the number of sub-requests from the request
    res = oRequest.getNumSubRequests( 'integrity' )
    if not res['OK']:
      errStr = "SEvsLFCAgent.execute: Failed to obtain number of integrity subrequests."
      gLogger.error( errStr, res['Message'] )
      return S_OK()
    gLogger.info( "SEvsLFCAgent.execute: Found %s sub requests." % res['Value'] )

    ################################################
    # For all the sub-requests in the request
    for ind in range( res['Value'] ):
      gLogger.info( "SEvsLFCAgent.execute: Processing sub-request %s." % ind )
      subRequestAttributes = oRequest.getSubRequestAttributes( ind, 'integrity' )['Value']
      if subRequestAttributes['Status'] == 'Waiting':
        subRequestFiles = oRequest.getSubRequestFiles( ind, 'integrity' )['Value']
        operation = subRequestAttributes['Operation']

        ################################################
        #  If the sub-request is a lfcvsse operation
        if operation == 'SEvsLFC':
          gLogger.info( "SEvsLFCAgent.execute: Attempting to execute %s sub-request." % operation )
          storageElementName = subRequestAttributes['StorageElement']
          for subRequestFile in subRequestFiles:
            if subRequestFile['Status'] == 'Waiting':
              lfn = subRequestFile['LFN']
              storageElement = StorageElement( storageElementName )
              res = storageElement.isValid()
              if not res['OK']:
                errStr = "SEvsLFCAgent.execute: Failed to instantiate destination StorageElement."
                gLogger.error( errStr, storageElement )
              else:
                res = storageElement.getPfnForLfn( lfn )
                if not res['OK']:
                  gLogger.info( 'shit bugger do something.' )
                else:
                  oNamespaceBrowser = NamespaceBrowser( res['Value'] )
                  # Loop over all the directories and sub-directories
                  while ( oNamespaceBrowser.isActive() ):
                    currentDir = oNamespaceBrowser.getActiveDir()

                    gLogger.info( "SEvsLFCAgent.execute: Attempting to list the contents of %s." % currentDir )
                    res = storageElement.listDirectory( currentDir )
                    if not res['Value']['Successful'].has_key( currentDir ):
                      gLogger.error( "SEvsLFCAgent.execute: Failed to list the directory contents.", "%s %s" % ( currentDir, res['Value']['Successful']['Failed'][currentDir] ) )
                      subDirs = [currentDir]
                    else:
                      subDirs = []
                      files = {}
                      for surl, surlDict in res['Value']['Successful'][currentDir]['Files'].items():
                        pfnRes = storageElement.getPfnForProtocol( surl, 'SRM2', withPort = False )
                        surl = pfnRes['Value']
                        files[surl] = surlDict
                      for surl, surlDict in res['Value']['Successful'][currentDir]['SubDirs'].items():
                        pfnRes = storageElement.getPfnForProtocol( surl, 'SRM2', withPort = False )
                        surl = pfnRes['Value']
                        subDirs.append( surl )

                      #subDirs = res['Value']['Successful'][currentDir]['SubDirs']
                      gLogger.info( "SEvsLFCAgent.execute: Successfully obtained %s sub-directories." % len( subDirs ) )
                      #files = res['Value']['Successful'][currentDir]['Files']
                      gLogger.info( "SEvsLFCAgent.execute: Successfully obtained %s files." % len( files ) )

                      selectedLfns = []
                      lfnPfnDict = {}
                      pfnSize = {}

                      for pfn, pfnDict in files.items():
                        res = storageElement.getPfnPath( pfn )
                        if not res['OK']:
                          gLogger.error( "SEvsLFCAgent.execute: Failed to get determine LFN from pfn.", "%s %s" % ( pfn, res['Message'] ) )
                          fileMetadata = {'Prognosis':'NonConventionPfn', 'LFN':'', 'PFN':pfn, 'StorageElement':storageElementName, 'Size':pfnDict['Size']}
                          res = IntegrityDB.insertProblematic( AGENT_NAME, fileMetadata )
                          if res['OK']:
                            gLogger.info( "SEvsLFCAgent.execute: Successfully added to IntegrityDB." )
                            gLogger.error( "Change the status in the LFC,ProcDB...." )
                          else:
                            gLogger.error( "Shit, f**k, bugger. Add the failover." )
                        else:
                          lfn = res['Value']
                          selectedLfns.append( lfn )
                          lfnPfnDict[lfn] = pfn
                          pfnSize[pfn] = pfnDict['Size']

                      res = self.ReplicaManager.getCatalogFileMetadata( selectedLfns )
                      if not res['OK']:
                        subDirs = [currentDir]
                      else:
                        for lfn in res['Value']['Failed'].keys():
                          gLogger.error( "SEvsLFCAgent.execute: Failed to get metadata.", "%s %s" % ( lfn, res['Value']['Failed'][lfn] ) )
                          pfn = lfnPfnDict[lfn]
                          fileMetadata = {'Prognosis':'SEPfnNoLfn', 'LFN':lfn, 'PFN':pfn, 'StorageElement':storageElementName, 'Size':pfnSize[pfn]}
                          res = IntegrityDB.insertProblematic( AGENT_NAME, fileMetadata )
                          if res['OK']:
                            gLogger.info( "SEvsLFCAgent.execute: Successfully added to IntegrityDB." )
                            gLogger.error( "Change the status in the LFC,ProcDB...." )
                          else:
                            gLogger.error( "Shit, f**k, bugger. Add the failover." )

                        for lfn, lfnDict in res['Value']['Successful'].items():
                          pfn = lfnPfnDict[lfn]
                          storageSize = pfnSize[pfn]
                          catalogSize = lfnDict['Size']
                          if int( catalogSize ) == int( storageSize ):
                            gLogger.info( "SEvsLFCAgent.execute: Catalog and storage sizes match.", "%s %s" % ( pfn, storageElementName ) )
                            gLogger.info( "Change the status in the LFC" )
                          elif int( storageSize ) == 0:
                            gLogger.error( "SEvsLFCAgent.execute: Physical file size is 0.", "%s %s" % ( pfn, storageElementName ) )
                            fileMetadata = {'Prognosis':'ZeroSizePfn', 'LFN':lfn, 'PFN':pfn, 'StorageElement':storageElementName}
                            res = IntegrityDB.insertProblematic( AGENT_NAME, fileMetadata )
                            if res['OK']:
                              gLogger.info( "SEvsLFCAgent.execute: Successfully added to IntegrityDB." )
                              gLogger.error( "Change the status in the LFC,ProcDB...." )
                            else:
                              gLogger.error( "Shit, f**k, bugger. Add the failover." )
                          else:
                            gLogger.error( "SEvsLFCAgent.execute: Catalog and storage size mis-match.", "%s %s" % ( pfn, storageElementName ) )
                            fileMetadata = {'Prognosis':'PfnSizeMismatch', 'LFN':lfn, 'PFN':pfn, 'StorageElement':storageElementName}
                            res = IntegrityDB.insertProblematic( AGENT_NAME, fileMetadata )
                            if res['OK']:
                              gLogger.info( "SEvsLFCAgent.execute: Successfully added to IntegrityDB." )
                              gLogger.error( "Change the status in the LFC,ProcDB...." )
                            else:
                              gLogger.error( "Shit, f**k, bugger. Add the failover." )

                        res = self.ReplicaManager.getCatalogReplicas( selectedLfns )
                        if not res['OK']:
                          subDirs = [currentDir]
                        else:
                          for lfn in res['Value']['Failed'].keys():
                            gLogger.error( "SEvsLFCAgent.execute: Failed to get replica information.", "%s %s" % ( lfn, res['Value']['Failed'][lfn] ) )
                            pfn = lfnPfnDict[lfn]
                            fileMetadata = {'Prognosis':'PfnNoReplica', 'LFN':lfn, 'PFN':pfn, 'StorageElement':storageElementName, 'Size':pfnSize[pfn]}
                            res = IntegrityDB.insertProblematic( AGENT_NAME, fileMetadata )
                            if res['OK']:
                              gLogger.info( "SEvsLFCAgent.execute: Successfully added to IntegrityDB." )
                              gLogger.error( "Change the status in the LFC,ProcDB...." )
                            else:
                              gLogger.error( "Shit, f**k, bugger. Add the failover." )

                          for lfn, repDict in res['Value']['Successful'].items():
                            pfn = lfnPfnDict[lfn]
                            registeredPfns = repDict.values()
                            if not pfn in registeredPfns:
                              gLogger.error( "SEvsLFCAgent.execute: SE PFN not registered.", "%s %s" % ( lfn, pfn ) )
                              fileMetadata = {'Prognosis':'PfnNoReplica', 'LFN':lfn, 'PFN':pfn, 'StorageElement':storageElementName}
                              res = IntegrityDB.insertProblematic( AGENT_NAME, fileMetadata )
                              if res['OK']:
                                gLogger.info( "SEvsLFCAgent.execute: Successfully added to IntegrityDB." )
                                gLogger.error( "Change the status in the LFC,ProcDB...." )
                              else:
                                gLogger.error( "Shit, f**k, bugger. Add the failover." )
                            else:
                              gLogger.info( "SEvsLFCAgent.execute: SE Pfn verified.", pfn )

                    oNamespaceBrowser.updateDirs( subDirs )
                  oRequest.setSubRequestFileAttributeValue( ind, 'integrity', lfn, 'Status', 'Done' )

        ################################################
        #  If the sub-request is none of the above types
        else:
          gLogger.info( "SEvsLFCAgent.execute: Operation not supported.", operation )

        ################################################
        #  Determine whether there are any active files
        if oRequest.isSubRequestEmpty( ind, 'integrity' )['Value']:
          oRequest.setSubRequestStatus( ind, 'integrity', 'Done' )

      ################################################
      #  If the sub-request is already in terminal state
      else:
        gLogger.info( "SEvsLFCAgent.execute: Sub-request %s is status '%s' and  not to be executed." % ( ind, subRequestAttributes['Status'] ) )

    ################################################
    #  Generate the new request string after operation
    requestString = oRequest.toXML()['Value']
    res = self.RequestDBClient.updateRequest( requestName, requestString, sourceServer )

    return S_OK()
Exemplo n.º 4
0
    def execute(self):

        IntegrityDB = RPCClient('DataManagement/DataIntegrity')

        res = self.RequestDBClient.getRequest('integrity')
        if not res['OK']:
            gLogger.info(
                "SEvsLFCAgent.execute: Failed to get request from database.")
            return S_OK()
        elif not res['Value']:
            gLogger.info(
                "SEvsLFCAgent.execute: No requests to be executed found.")
            return S_OK()
        requestString = res['Value']['requestString']
        requestName = res['Value']['requestName']
        sourceServer = res['Value']['Server']
        gLogger.info("SEvsLFCAgent.execute: Obtained request %s" % requestName)
        oRequest = RequestContainer(request=requestString)

        ################################################
        # Find the number of sub-requests from the request
        res = oRequest.getNumSubRequests('integrity')
        if not res['OK']:
            errStr = "SEvsLFCAgent.execute: Failed to obtain number of integrity subrequests."
            gLogger.error(errStr, res['Message'])
            return S_OK()
        gLogger.info("SEvsLFCAgent.execute: Found %s sub requests." %
                     res['Value'])

        ################################################
        # For all the sub-requests in the request
        for ind in range(res['Value']):
            gLogger.info("SEvsLFCAgent.execute: Processing sub-request %s." %
                         ind)
            subRequestAttributes = oRequest.getSubRequestAttributes(
                ind, 'integrity')['Value']
            if subRequestAttributes['Status'] == 'Waiting':
                subRequestFiles = oRequest.getSubRequestFiles(
                    ind, 'integrity')['Value']
                operation = subRequestAttributes['Operation']

                ################################################
                #  If the sub-request is a lfcvsse operation
                if operation == 'SEvsLFC':
                    gLogger.info(
                        "SEvsLFCAgent.execute: Attempting to execute %s sub-request."
                        % operation)
                    storageElementName = subRequestAttributes['StorageElement']
                    for subRequestFile in subRequestFiles:
                        if subRequestFile['Status'] == 'Waiting':
                            lfn = subRequestFile['LFN']
                            storageElement = StorageElement(storageElementName)
                            res = storageElement.isValid()
                            if not res['OK']:
                                errStr = "SEvsLFCAgent.execute: Failed to instantiate destination StorageElement."
                                gLogger.error(errStr, storageElement)
                            else:
                                res = storageElement.getPfnForLfn(lfn)
                                if not res['OK']:
                                    gLogger.info('shit bugger do something.')
                                else:
                                    oNamespaceBrowser = NamespaceBrowser(
                                        res['Value'])
                                    # Loop over all the directories and sub-directories
                                    while (oNamespaceBrowser.isActive()):
                                        currentDir = oNamespaceBrowser.getActiveDir(
                                        )

                                        gLogger.info(
                                            "SEvsLFCAgent.execute: Attempting to list the contents of %s."
                                            % currentDir)
                                        res = storageElement.listDirectory(
                                            currentDir)
                                        if not res['Value'][
                                                'Successful'].has_key(
                                                    currentDir):
                                            gLogger.error(
                                                "SEvsLFCAgent.execute: Failed to list the directory contents.",
                                                "%s %s" %
                                                (currentDir,
                                                 res['Value']['Successful']
                                                 ['Failed'][currentDir]))
                                            subDirs = [currentDir]
                                        else:
                                            subDirs = []
                                            files = {}
                                            for surl, surlDict in res['Value'][
                                                    'Successful'][currentDir][
                                                        'Files'].items():
                                                pfnRes = storageElement.getPfnForProtocol(
                                                    surl,
                                                    'SRM2',
                                                    withPort=False)
                                                surl = pfnRes['Value']
                                                files[surl] = surlDict
                                            for surl, surlDict in res['Value'][
                                                    'Successful'][currentDir][
                                                        'SubDirs'].items():
                                                pfnRes = storageElement.getPfnForProtocol(
                                                    surl,
                                                    'SRM2',
                                                    withPort=False)
                                                surl = pfnRes['Value']
                                                subDirs.append(surl)

                                            #subDirs = res['Value']['Successful'][currentDir]['SubDirs']
                                            gLogger.info(
                                                "SEvsLFCAgent.execute: Successfully obtained %s sub-directories."
                                                % len(subDirs))
                                            #files = res['Value']['Successful'][currentDir]['Files']
                                            gLogger.info(
                                                "SEvsLFCAgent.execute: Successfully obtained %s files."
                                                % len(files))

                                            selectedLfns = []
                                            lfnPfnDict = {}
                                            pfnSize = {}

                                            for pfn, pfnDict in files.items():
                                                res = storageElement.getPfnPath(
                                                    pfn)
                                                if not res['OK']:
                                                    gLogger.error(
                                                        "SEvsLFCAgent.execute: Failed to get determine LFN from pfn.",
                                                        "%s %s" %
                                                        (pfn, res['Message']))
                                                    fileMetadata = {
                                                        'Prognosis':
                                                        'NonConventionPfn',
                                                        'LFN': '',
                                                        'PFN': pfn,
                                                        'StorageElement':
                                                        storageElementName,
                                                        'Size': pfnDict['Size']
                                                    }
                                                    res = IntegrityDB.insertProblematic(
                                                        AGENT_NAME,
                                                        fileMetadata)
                                                    if res['OK']:
                                                        gLogger.info(
                                                            "SEvsLFCAgent.execute: Successfully added to IntegrityDB."
                                                        )
                                                        gLogger.error(
                                                            "Change the status in the LFC,ProcDB...."
                                                        )
                                                    else:
                                                        gLogger.error(
                                                            "Shit, f**k, bugger. Add the failover."
                                                        )
                                                else:
                                                    lfn = res['Value']
                                                    selectedLfns.append(lfn)
                                                    lfnPfnDict[lfn] = pfn
                                                    pfnSize[pfn] = pfnDict[
                                                        'Size']

                                            res = self.ReplicaManager.getCatalogFileMetadata(
                                                selectedLfns)
                                            if not res['OK']:
                                                subDirs = [currentDir]
                                            else:
                                                for lfn in res['Value'][
                                                        'Failed'].keys():
                                                    gLogger.error(
                                                        "SEvsLFCAgent.execute: Failed to get metadata.",
                                                        "%s %s" %
                                                        (lfn, res['Value']
                                                         ['Failed'][lfn]))
                                                    pfn = lfnPfnDict[lfn]
                                                    fileMetadata = {
                                                        'Prognosis':
                                                        'SEPfnNoLfn',
                                                        'LFN': lfn,
                                                        'PFN': pfn,
                                                        'StorageElement':
                                                        storageElementName,
                                                        'Size': pfnSize[pfn]
                                                    }
                                                    res = IntegrityDB.insertProblematic(
                                                        AGENT_NAME,
                                                        fileMetadata)
                                                    if res['OK']:
                                                        gLogger.info(
                                                            "SEvsLFCAgent.execute: Successfully added to IntegrityDB."
                                                        )
                                                        gLogger.error(
                                                            "Change the status in the LFC,ProcDB...."
                                                        )
                                                    else:
                                                        gLogger.error(
                                                            "Shit, f**k, bugger. Add the failover."
                                                        )

                                                for lfn, lfnDict in res[
                                                        'Value'][
                                                            'Successful'].items(
                                                            ):
                                                    pfn = lfnPfnDict[lfn]
                                                    storageSize = pfnSize[pfn]
                                                    catalogSize = lfnDict[
                                                        'Size']
                                                    if int(catalogSize) == int(
                                                            storageSize):
                                                        gLogger.info(
                                                            "SEvsLFCAgent.execute: Catalog and storage sizes match.",
                                                            "%s %s" %
                                                            (pfn,
                                                             storageElementName
                                                             ))
                                                        gLogger.info(
                                                            "Change the status in the LFC"
                                                        )
                                                    elif int(storageSize) == 0:
                                                        gLogger.error(
                                                            "SEvsLFCAgent.execute: Physical file size is 0.",
                                                            "%s %s" %
                                                            (pfn,
                                                             storageElementName
                                                             ))
                                                        fileMetadata = {
                                                            'Prognosis':
                                                            'ZeroSizePfn',
                                                            'LFN':
                                                            lfn,
                                                            'PFN':
                                                            pfn,
                                                            'StorageElement':
                                                            storageElementName
                                                        }
                                                        res = IntegrityDB.insertProblematic(
                                                            AGENT_NAME,
                                                            fileMetadata)
                                                        if res['OK']:
                                                            gLogger.info(
                                                                "SEvsLFCAgent.execute: Successfully added to IntegrityDB."
                                                            )
                                                            gLogger.error(
                                                                "Change the status in the LFC,ProcDB...."
                                                            )
                                                        else:
                                                            gLogger.error(
                                                                "Shit, f**k, bugger. Add the failover."
                                                            )
                                                    else:
                                                        gLogger.error(
                                                            "SEvsLFCAgent.execute: Catalog and storage size mis-match.",
                                                            "%s %s" %
                                                            (pfn,
                                                             storageElementName
                                                             ))
                                                        fileMetadata = {
                                                            'Prognosis':
                                                            'PfnSizeMismatch',
                                                            'LFN':
                                                            lfn,
                                                            'PFN':
                                                            pfn,
                                                            'StorageElement':
                                                            storageElementName
                                                        }
                                                        res = IntegrityDB.insertProblematic(
                                                            AGENT_NAME,
                                                            fileMetadata)
                                                        if res['OK']:
                                                            gLogger.info(
                                                                "SEvsLFCAgent.execute: Successfully added to IntegrityDB."
                                                            )
                                                            gLogger.error(
                                                                "Change the status in the LFC,ProcDB...."
                                                            )
                                                        else:
                                                            gLogger.error(
                                                                "Shit, f**k, bugger. Add the failover."
                                                            )

                                                res = self.ReplicaManager.getCatalogReplicas(
                                                    selectedLfns)
                                                if not res['OK']:
                                                    subDirs = [currentDir]
                                                else:
                                                    for lfn in res['Value'][
                                                            'Failed'].keys():
                                                        gLogger.error(
                                                            "SEvsLFCAgent.execute: Failed to get replica information.",
                                                            "%s %s" %
                                                            (lfn, res['Value']
                                                             ['Failed'][lfn]))
                                                        pfn = lfnPfnDict[lfn]
                                                        fileMetadata = {
                                                            'Prognosis':
                                                            'PfnNoReplica',
                                                            'LFN': lfn,
                                                            'PFN': pfn,
                                                            'StorageElement':
                                                            storageElementName,
                                                            'Size':
                                                            pfnSize[pfn]
                                                        }
                                                        res = IntegrityDB.insertProblematic(
                                                            AGENT_NAME,
                                                            fileMetadata)
                                                        if res['OK']:
                                                            gLogger.info(
                                                                "SEvsLFCAgent.execute: Successfully added to IntegrityDB."
                                                            )
                                                            gLogger.error(
                                                                "Change the status in the LFC,ProcDB...."
                                                            )
                                                        else:
                                                            gLogger.error(
                                                                "Shit, f**k, bugger. Add the failover."
                                                            )

                                                    for lfn, repDict in res[
                                                            'Value'][
                                                                'Successful'].items(
                                                                ):
                                                        pfn = lfnPfnDict[lfn]
                                                        registeredPfns = repDict.values(
                                                        )
                                                        if not pfn in registeredPfns:
                                                            gLogger.error(
                                                                "SEvsLFCAgent.execute: SE PFN not registered.",
                                                                "%s %s" %
                                                                (lfn, pfn))
                                                            fileMetadata = {
                                                                'Prognosis':
                                                                'PfnNoReplica',
                                                                'LFN':
                                                                lfn,
                                                                'PFN':
                                                                pfn,
                                                                'StorageElement':
                                                                storageElementName
                                                            }
                                                            res = IntegrityDB.insertProblematic(
                                                                AGENT_NAME,
                                                                fileMetadata)
                                                            if res['OK']:
                                                                gLogger.info(
                                                                    "SEvsLFCAgent.execute: Successfully added to IntegrityDB."
                                                                )
                                                                gLogger.error(
                                                                    "Change the status in the LFC,ProcDB...."
                                                                )
                                                            else:
                                                                gLogger.error(
                                                                    "Shit, f**k, bugger. Add the failover."
                                                                )
                                                        else:
                                                            gLogger.info(
                                                                "SEvsLFCAgent.execute: SE Pfn verified.",
                                                                pfn)

                                        oNamespaceBrowser.updateDirs(subDirs)
                                    oRequest.setSubRequestFileAttributeValue(
                                        ind, 'integrity', lfn, 'Status',
                                        'Done')

                ################################################
                #  If the sub-request is none of the above types
                else:
                    gLogger.info(
                        "SEvsLFCAgent.execute: Operation not supported.",
                        operation)

                ################################################
                #  Determine whether there are any active files
                if oRequest.isSubRequestEmpty(ind, 'integrity')['Value']:
                    oRequest.setSubRequestStatus(ind, 'integrity', 'Done')

            ################################################
            #  If the sub-request is already in terminal state
            else:
                gLogger.info(
                    "SEvsLFCAgent.execute: Sub-request %s is status '%s' and  not to be executed."
                    % (ind, subRequestAttributes['Status']))

        ################################################
        #  Generate the new request string after operation
        requestString = oRequest.toXML()['Value']
        res = self.RequestDBClient.updateRequest(requestName, requestString,
                                                 sourceServer)

        return S_OK()