def execute( self ): res = self.RequestDBClient.getRequest( 'integrity' ) if not res['OK']: gLogger.info( "LFCvsSEAgent.execute: Failed to get request from database." ) return S_OK() elif not res['Value']: gLogger.info( "LFCvsSEAgent.execute: No requests to be executed found." ) return S_OK() requestString = res['Value']['RequestString'] requestName = res['Value']['RequestName'] sourceServer = res['Value']['Server'] gLogger.info( "LFCvsSEAgent.execute: Obtained request %s" % requestName ) oRequest = RequestContainer( request = requestString ) ################################################ # Find the number of sub-requests from the request res = oRequest.getNumSubRequests( 'integrity' ) if not res['OK']: errStr = "LFCvsSEAgent.execute: Failed to obtain number of integrity subrequests." gLogger.error( errStr, res['Message'] ) return S_OK() gLogger.info( "LFCvsSEAgent.execute: Found %s sub requests." % res['Value'] ) ################################################ # For all the sub-requests in the request for ind in range( res['Value'] ): gLogger.info( "LFCvsSEAgent.execute: Processing sub-request %s." % ind ) subRequestAttributes = oRequest.getSubRequestAttributes( ind, 'integrity' )['Value'] if subRequestAttributes['Status'] == 'Waiting': subRequestFiles = oRequest.getSubRequestFiles( ind, 'integrity' )['Value'] operation = subRequestAttributes['Operation'] ################################################ # If the sub-request is a lfcvsse operation if operation == 'LFCvsSE': gLogger.info( "LFCvsSEAgent.execute: Attempting to execute %s sub-request." % operation ) for subRequestFile in subRequestFiles: if subRequestFile['Status'] == 'Waiting': lfn = subRequestFile['LFN'] oNamespaceBrowser = NamespaceBrowser( lfn ) # Loop over all the directories and sub-directories while ( oNamespaceBrowser.isActive() ): currentDir = oNamespaceBrowser.getActiveDir() gLogger.info( "LFCvsSEAgent.execute: Attempting to get contents of %s." % currentDir ) res = self.ReplicaManager.getCatalogDirectoryContents( currentDir ) if not res['OK']: subDirs = [currentDir] elif res['Value']['Failed'].has_key( currentDir ): subDirs = [currentDir] else: subDirs = res['Value']['Successful'][currentDir]['SubDirs'] files = res['Value']['Successful'][currentDir]['Files'] lfnSizeDict = {} pfnLfnDict = {} pfnStatusDict = {} sePfnDict = {} for lfn, lfnDict in files.items(): lfnSizeDict[lfn] = lfnDict['MetaData']['Size'] for se in lfnDict['Replicas'].keys(): pfn = lfnDict['Replicas'][se]['PFN'] status = lfnDict['Replicas'][se]['Status'] pfnStatusDict[pfn] = status pfnLfnDict[pfn] = lfn if not sePfnDict.has_key( se ): sePfnDict[se] = [] sePfnDict[se].append( pfn ) for storageElementName, physicalFiles in sePfnDict.items(): gLogger.info( "LFCvsSEAgent.execute: Attempting to get metadata for files on %s." % storageElementName ) res = self.ReplicaManager.getStorageFileMetadata( physicalFiles, storageElementName ) if not res['OK']: gLogger.error( "LFCvsSEAgent.execute: Completely failed to get physical file metadata.", res['Message'] ) else: for pfn in res['Value']['Failed'].keys(): gLogger.error( "LFCvsSEAgent.execute: Failed to get metadata.", "%s %s" % ( pfn, res['Value']['Failed'][pfn] ) ) lfn = pfnLfnDict[pfn] fileMetadata = {'Prognosis':'MissingSEPfn', 'LFN':lfn, 'PFN':pfn, 'StorageElement':storageElementName, 'Size':lfnSizeDict[lfn]} IntegrityDB = RPCClient( 'DataManagement/DataIntegrity' ) resInsert = IntegrityDB.insertProblematic( AGENT_NAME, fileMetadata ) if resInsert['OK']: gLogger.info( "LFCvsSEAgent.execute: Successfully added to IntegrityDB." ) gLogger.error( "Change the status in the LFC,ProcDB...." ) else: gLogger.error( "Shit, f**k, bugger. Add the failover." ) for pfn, pfnDict in res['Value']['Successful'].items(): lfn = pfnLfnDict[pfn] catalogSize = int( lfnSizeDict[lfn] ) storageSize = int( pfnDict['Size'] ) if int( catalogSize ) == int( storageSize ): gLogger.info( "LFCvsSEAgent.execute: Catalog and storage sizes match.", "%s %s" % ( pfn, storageElementName ) ) gLogger.info( "Change the status in the LFC" ) else: gLogger.error( "LFCvsSEAgent.execute: Catalog and storage size mis-match.", "%s %s" % ( pfn, storageElementName ) ) fileMetadata = {'Prognosis':'PfnSizeMismatch', 'LFN':lfn, 'PFN':pfn, 'StorageElement':storageElementName} IntegrityDB = RPCClient( 'DataManagement/DataIntegrity' ) resInsert = IntegrityDB.insertProblematic( AGENT_NAME, fileMetadata ) if resInsert['OK']: gLogger.info( "LFCvsSEAgent.execute: Successfully added to IntegrityDB." ) gLogger.error( "Change the status in the LFC,ProcDB...." ) else: gLogger.error( "Shit, f**k, bugger. Add the failover." ) oNamespaceBrowser.updateDirs( subDirs ) oRequest.setSubRequestFileAttributeValue( ind, 'integrity', lfn, 'Status', 'Done' ) ################################################ # If the sub-request is none of the above types else: gLogger.info( "LFCvsSEAgent.execute: Operation not supported.", operation ) ################################################ # Determine whether there are any active files if oRequest.isSubRequestEmpty( ind, 'integrity' )['Value']: oRequest.setSubRequestStatus( ind, 'integrity', 'Done' ) ################################################ # If the sub-request is already in terminal state else: gLogger.info( "LFCvsSEAgent.execute: Sub-request %s is status '%s' and not to be executed." % ( ind, subRequestAttributes['Status'] ) ) ################################################ # Generate the new request string after operation requestString = oRequest.toXML()['Value'] res = self.RequestDBClient.updateRequest( requestName, requestString, sourceServer ) return S_OK()
def execute(self): res = self.RequestDBClient.getRequest('integrity') if not res['OK']: gLogger.info( "LFCvsSEAgent.execute: Failed to get request from database.") return S_OK() elif not res['Value']: gLogger.info( "LFCvsSEAgent.execute: No requests to be executed found.") return S_OK() requestString = res['Value']['RequestString'] requestName = res['Value']['RequestName'] sourceServer = res['Value']['Server'] gLogger.info("LFCvsSEAgent.execute: Obtained request %s" % requestName) oRequest = RequestContainer(request=requestString) ################################################ # Find the number of sub-requests from the request res = oRequest.getNumSubRequests('integrity') if not res['OK']: errStr = "LFCvsSEAgent.execute: Failed to obtain number of integrity subrequests." gLogger.error(errStr, res['Message']) return S_OK() gLogger.info("LFCvsSEAgent.execute: Found %s sub requests." % res['Value']) ################################################ # For all the sub-requests in the request for ind in range(res['Value']): gLogger.info("LFCvsSEAgent.execute: Processing sub-request %s." % ind) subRequestAttributes = oRequest.getSubRequestAttributes( ind, 'integrity')['Value'] if subRequestAttributes['Status'] == 'Waiting': subRequestFiles = oRequest.getSubRequestFiles( ind, 'integrity')['Value'] operation = subRequestAttributes['Operation'] ################################################ # If the sub-request is a lfcvsse operation if operation == 'LFCvsSE': gLogger.info( "LFCvsSEAgent.execute: Attempting to execute %s sub-request." % operation) for subRequestFile in subRequestFiles: if subRequestFile['Status'] == 'Waiting': lfn = subRequestFile['LFN'] oNamespaceBrowser = NamespaceBrowser(lfn) # Loop over all the directories and sub-directories while (oNamespaceBrowser.isActive()): currentDir = oNamespaceBrowser.getActiveDir() gLogger.info( "LFCvsSEAgent.execute: Attempting to get contents of %s." % currentDir) res = self.ReplicaManager.getCatalogDirectoryContents( currentDir) if not res['OK']: subDirs = [currentDir] elif res['Value']['Failed'].has_key( currentDir): subDirs = [currentDir] else: subDirs = res['Value']['Successful'][ currentDir]['SubDirs'] files = res['Value']['Successful'][ currentDir]['Files'] lfnSizeDict = {} pfnLfnDict = {} pfnStatusDict = {} sePfnDict = {} for lfn, lfnDict in files.items(): lfnSizeDict[lfn] = lfnDict['MetaData'][ 'Size'] for se in lfnDict['Replicas'].keys(): pfn = lfnDict['Replicas'][se][ 'PFN'] status = lfnDict['Replicas'][se][ 'Status'] pfnStatusDict[pfn] = status pfnLfnDict[pfn] = lfn if not sePfnDict.has_key(se): sePfnDict[se] = [] sePfnDict[se].append(pfn) for storageElementName, physicalFiles in sePfnDict.items( ): gLogger.info( "LFCvsSEAgent.execute: Attempting to get metadata for files on %s." % storageElementName) res = self.ReplicaManager.getStorageFileMetadata( physicalFiles, storageElementName) if not res['OK']: gLogger.error( "LFCvsSEAgent.execute: Completely failed to get physical file metadata.", res['Message']) else: for pfn in res['Value'][ 'Failed'].keys(): gLogger.error( "LFCvsSEAgent.execute: Failed to get metadata.", "%s %s" % (pfn, res['Value'] ['Failed'][pfn])) lfn = pfnLfnDict[pfn] fileMetadata = { 'Prognosis': 'MissingSEPfn', 'LFN': lfn, 'PFN': pfn, 'StorageElement': storageElementName, 'Size': lfnSizeDict[lfn] } IntegrityDB = RPCClient( 'DataManagement/DataIntegrity' ) resInsert = IntegrityDB.insertProblematic( AGENT_NAME, fileMetadata) if resInsert['OK']: gLogger.info( "LFCvsSEAgent.execute: Successfully added to IntegrityDB." ) gLogger.error( "Change the status in the LFC,ProcDB...." ) else: gLogger.error( "Shit, f**k, bugger. Add the failover." ) for pfn, pfnDict in res['Value'][ 'Successful'].items(): lfn = pfnLfnDict[pfn] catalogSize = int( lfnSizeDict[lfn]) storageSize = int( pfnDict['Size']) if int(catalogSize) == int( storageSize): gLogger.info( "LFCvsSEAgent.execute: Catalog and storage sizes match.", "%s %s" % (pfn, storageElementName)) gLogger.info( "Change the status in the LFC" ) else: gLogger.error( "LFCvsSEAgent.execute: Catalog and storage size mis-match.", "%s %s" % (pfn, storageElementName)) fileMetadata = { 'Prognosis': 'PfnSizeMismatch', 'LFN': lfn, 'PFN': pfn, 'StorageElement': storageElementName } IntegrityDB = RPCClient( 'DataManagement/DataIntegrity' ) resInsert = IntegrityDB.insertProblematic( AGENT_NAME, fileMetadata) if resInsert['OK']: gLogger.info( "LFCvsSEAgent.execute: Successfully added to IntegrityDB." ) gLogger.error( "Change the status in the LFC,ProcDB...." ) else: gLogger.error( "Shit, f**k, bugger. Add the failover." ) oNamespaceBrowser.updateDirs(subDirs) oRequest.setSubRequestFileAttributeValue( ind, 'integrity', lfn, 'Status', 'Done') ################################################ # If the sub-request is none of the above types else: gLogger.info( "LFCvsSEAgent.execute: Operation not supported.", operation) ################################################ # Determine whether there are any active files if oRequest.isSubRequestEmpty(ind, 'integrity')['Value']: oRequest.setSubRequestStatus(ind, 'integrity', 'Done') ################################################ # If the sub-request is already in terminal state else: gLogger.info( "LFCvsSEAgent.execute: Sub-request %s is status '%s' and not to be executed." % (ind, subRequestAttributes['Status'])) ################################################ # Generate the new request string after operation requestString = oRequest.toXML()['Value'] res = self.RequestDBClient.updateRequest(requestName, requestString, sourceServer) return S_OK()
def execute( self ): IntegrityDB = RPCClient( 'DataManagement/DataIntegrity' ) res = self.RequestDBClient.getRequest( 'integrity' ) if not res['OK']: gLogger.info( "SEvsLFCAgent.execute: Failed to get request from database." ) return S_OK() elif not res['Value']: gLogger.info( "SEvsLFCAgent.execute: No requests to be executed found." ) return S_OK() requestString = res['Value']['requestString'] requestName = res['Value']['requestName'] sourceServer = res['Value']['Server'] gLogger.info( "SEvsLFCAgent.execute: Obtained request %s" % requestName ) oRequest = RequestContainer( request = requestString ) ################################################ # Find the number of sub-requests from the request res = oRequest.getNumSubRequests( 'integrity' ) if not res['OK']: errStr = "SEvsLFCAgent.execute: Failed to obtain number of integrity subrequests." gLogger.error( errStr, res['Message'] ) return S_OK() gLogger.info( "SEvsLFCAgent.execute: Found %s sub requests." % res['Value'] ) ################################################ # For all the sub-requests in the request for ind in range( res['Value'] ): gLogger.info( "SEvsLFCAgent.execute: Processing sub-request %s." % ind ) subRequestAttributes = oRequest.getSubRequestAttributes( ind, 'integrity' )['Value'] if subRequestAttributes['Status'] == 'Waiting': subRequestFiles = oRequest.getSubRequestFiles( ind, 'integrity' )['Value'] operation = subRequestAttributes['Operation'] ################################################ # If the sub-request is a lfcvsse operation if operation == 'SEvsLFC': gLogger.info( "SEvsLFCAgent.execute: Attempting to execute %s sub-request." % operation ) storageElementName = subRequestAttributes['StorageElement'] for subRequestFile in subRequestFiles: if subRequestFile['Status'] == 'Waiting': lfn = subRequestFile['LFN'] storageElement = StorageElement( storageElementName ) res = storageElement.isValid() if not res['OK']: errStr = "SEvsLFCAgent.execute: Failed to instantiate destination StorageElement." gLogger.error( errStr, storageElement ) else: res = storageElement.getPfnForLfn( lfn ) if not res['OK']: gLogger.info( 'shit bugger do something.' ) else: oNamespaceBrowser = NamespaceBrowser( res['Value'] ) # Loop over all the directories and sub-directories while ( oNamespaceBrowser.isActive() ): currentDir = oNamespaceBrowser.getActiveDir() gLogger.info( "SEvsLFCAgent.execute: Attempting to list the contents of %s." % currentDir ) res = storageElement.listDirectory( currentDir ) if not res['Value']['Successful'].has_key( currentDir ): gLogger.error( "SEvsLFCAgent.execute: Failed to list the directory contents.", "%s %s" % ( currentDir, res['Value']['Successful']['Failed'][currentDir] ) ) subDirs = [currentDir] else: subDirs = [] files = {} for surl, surlDict in res['Value']['Successful'][currentDir]['Files'].items(): pfnRes = storageElement.getPfnForProtocol( surl, 'SRM2', withPort = False ) surl = pfnRes['Value'] files[surl] = surlDict for surl, surlDict in res['Value']['Successful'][currentDir]['SubDirs'].items(): pfnRes = storageElement.getPfnForProtocol( surl, 'SRM2', withPort = False ) surl = pfnRes['Value'] subDirs.append( surl ) #subDirs = res['Value']['Successful'][currentDir]['SubDirs'] gLogger.info( "SEvsLFCAgent.execute: Successfully obtained %s sub-directories." % len( subDirs ) ) #files = res['Value']['Successful'][currentDir]['Files'] gLogger.info( "SEvsLFCAgent.execute: Successfully obtained %s files." % len( files ) ) selectedLfns = [] lfnPfnDict = {} pfnSize = {} for pfn, pfnDict in files.items(): res = storageElement.getPfnPath( pfn ) if not res['OK']: gLogger.error( "SEvsLFCAgent.execute: Failed to get determine LFN from pfn.", "%s %s" % ( pfn, res['Message'] ) ) fileMetadata = {'Prognosis':'NonConventionPfn', 'LFN':'', 'PFN':pfn, 'StorageElement':storageElementName, 'Size':pfnDict['Size']} res = IntegrityDB.insertProblematic( AGENT_NAME, fileMetadata ) if res['OK']: gLogger.info( "SEvsLFCAgent.execute: Successfully added to IntegrityDB." ) gLogger.error( "Change the status in the LFC,ProcDB...." ) else: gLogger.error( "Shit, f**k, bugger. Add the failover." ) else: lfn = res['Value'] selectedLfns.append( lfn ) lfnPfnDict[lfn] = pfn pfnSize[pfn] = pfnDict['Size'] res = self.ReplicaManager.getCatalogFileMetadata( selectedLfns ) if not res['OK']: subDirs = [currentDir] else: for lfn in res['Value']['Failed'].keys(): gLogger.error( "SEvsLFCAgent.execute: Failed to get metadata.", "%s %s" % ( lfn, res['Value']['Failed'][lfn] ) ) pfn = lfnPfnDict[lfn] fileMetadata = {'Prognosis':'SEPfnNoLfn', 'LFN':lfn, 'PFN':pfn, 'StorageElement':storageElementName, 'Size':pfnSize[pfn]} res = IntegrityDB.insertProblematic( AGENT_NAME, fileMetadata ) if res['OK']: gLogger.info( "SEvsLFCAgent.execute: Successfully added to IntegrityDB." ) gLogger.error( "Change the status in the LFC,ProcDB...." ) else: gLogger.error( "Shit, f**k, bugger. Add the failover." ) for lfn, lfnDict in res['Value']['Successful'].items(): pfn = lfnPfnDict[lfn] storageSize = pfnSize[pfn] catalogSize = lfnDict['Size'] if int( catalogSize ) == int( storageSize ): gLogger.info( "SEvsLFCAgent.execute: Catalog and storage sizes match.", "%s %s" % ( pfn, storageElementName ) ) gLogger.info( "Change the status in the LFC" ) elif int( storageSize ) == 0: gLogger.error( "SEvsLFCAgent.execute: Physical file size is 0.", "%s %s" % ( pfn, storageElementName ) ) fileMetadata = {'Prognosis':'ZeroSizePfn', 'LFN':lfn, 'PFN':pfn, 'StorageElement':storageElementName} res = IntegrityDB.insertProblematic( AGENT_NAME, fileMetadata ) if res['OK']: gLogger.info( "SEvsLFCAgent.execute: Successfully added to IntegrityDB." ) gLogger.error( "Change the status in the LFC,ProcDB...." ) else: gLogger.error( "Shit, f**k, bugger. Add the failover." ) else: gLogger.error( "SEvsLFCAgent.execute: Catalog and storage size mis-match.", "%s %s" % ( pfn, storageElementName ) ) fileMetadata = {'Prognosis':'PfnSizeMismatch', 'LFN':lfn, 'PFN':pfn, 'StorageElement':storageElementName} res = IntegrityDB.insertProblematic( AGENT_NAME, fileMetadata ) if res['OK']: gLogger.info( "SEvsLFCAgent.execute: Successfully added to IntegrityDB." ) gLogger.error( "Change the status in the LFC,ProcDB...." ) else: gLogger.error( "Shit, f**k, bugger. Add the failover." ) res = self.ReplicaManager.getCatalogReplicas( selectedLfns ) if not res['OK']: subDirs = [currentDir] else: for lfn in res['Value']['Failed'].keys(): gLogger.error( "SEvsLFCAgent.execute: Failed to get replica information.", "%s %s" % ( lfn, res['Value']['Failed'][lfn] ) ) pfn = lfnPfnDict[lfn] fileMetadata = {'Prognosis':'PfnNoReplica', 'LFN':lfn, 'PFN':pfn, 'StorageElement':storageElementName, 'Size':pfnSize[pfn]} res = IntegrityDB.insertProblematic( AGENT_NAME, fileMetadata ) if res['OK']: gLogger.info( "SEvsLFCAgent.execute: Successfully added to IntegrityDB." ) gLogger.error( "Change the status in the LFC,ProcDB...." ) else: gLogger.error( "Shit, f**k, bugger. Add the failover." ) for lfn, repDict in res['Value']['Successful'].items(): pfn = lfnPfnDict[lfn] registeredPfns = repDict.values() if not pfn in registeredPfns: gLogger.error( "SEvsLFCAgent.execute: SE PFN not registered.", "%s %s" % ( lfn, pfn ) ) fileMetadata = {'Prognosis':'PfnNoReplica', 'LFN':lfn, 'PFN':pfn, 'StorageElement':storageElementName} res = IntegrityDB.insertProblematic( AGENT_NAME, fileMetadata ) if res['OK']: gLogger.info( "SEvsLFCAgent.execute: Successfully added to IntegrityDB." ) gLogger.error( "Change the status in the LFC,ProcDB...." ) else: gLogger.error( "Shit, f**k, bugger. Add the failover." ) else: gLogger.info( "SEvsLFCAgent.execute: SE Pfn verified.", pfn ) oNamespaceBrowser.updateDirs( subDirs ) oRequest.setSubRequestFileAttributeValue( ind, 'integrity', lfn, 'Status', 'Done' ) ################################################ # If the sub-request is none of the above types else: gLogger.info( "SEvsLFCAgent.execute: Operation not supported.", operation ) ################################################ # Determine whether there are any active files if oRequest.isSubRequestEmpty( ind, 'integrity' )['Value']: oRequest.setSubRequestStatus( ind, 'integrity', 'Done' ) ################################################ # If the sub-request is already in terminal state else: gLogger.info( "SEvsLFCAgent.execute: Sub-request %s is status '%s' and not to be executed." % ( ind, subRequestAttributes['Status'] ) ) ################################################ # Generate the new request string after operation requestString = oRequest.toXML()['Value'] res = self.RequestDBClient.updateRequest( requestName, requestString, sourceServer ) return S_OK()
def execute(self): IntegrityDB = RPCClient('DataManagement/DataIntegrity') res = self.RequestDBClient.getRequest('integrity') if not res['OK']: gLogger.info( "SEvsLFCAgent.execute: Failed to get request from database.") return S_OK() elif not res['Value']: gLogger.info( "SEvsLFCAgent.execute: No requests to be executed found.") return S_OK() requestString = res['Value']['requestString'] requestName = res['Value']['requestName'] sourceServer = res['Value']['Server'] gLogger.info("SEvsLFCAgent.execute: Obtained request %s" % requestName) oRequest = RequestContainer(request=requestString) ################################################ # Find the number of sub-requests from the request res = oRequest.getNumSubRequests('integrity') if not res['OK']: errStr = "SEvsLFCAgent.execute: Failed to obtain number of integrity subrequests." gLogger.error(errStr, res['Message']) return S_OK() gLogger.info("SEvsLFCAgent.execute: Found %s sub requests." % res['Value']) ################################################ # For all the sub-requests in the request for ind in range(res['Value']): gLogger.info("SEvsLFCAgent.execute: Processing sub-request %s." % ind) subRequestAttributes = oRequest.getSubRequestAttributes( ind, 'integrity')['Value'] if subRequestAttributes['Status'] == 'Waiting': subRequestFiles = oRequest.getSubRequestFiles( ind, 'integrity')['Value'] operation = subRequestAttributes['Operation'] ################################################ # If the sub-request is a lfcvsse operation if operation == 'SEvsLFC': gLogger.info( "SEvsLFCAgent.execute: Attempting to execute %s sub-request." % operation) storageElementName = subRequestAttributes['StorageElement'] for subRequestFile in subRequestFiles: if subRequestFile['Status'] == 'Waiting': lfn = subRequestFile['LFN'] storageElement = StorageElement(storageElementName) res = storageElement.isValid() if not res['OK']: errStr = "SEvsLFCAgent.execute: Failed to instantiate destination StorageElement." gLogger.error(errStr, storageElement) else: res = storageElement.getPfnForLfn(lfn) if not res['OK']: gLogger.info('shit bugger do something.') else: oNamespaceBrowser = NamespaceBrowser( res['Value']) # Loop over all the directories and sub-directories while (oNamespaceBrowser.isActive()): currentDir = oNamespaceBrowser.getActiveDir( ) gLogger.info( "SEvsLFCAgent.execute: Attempting to list the contents of %s." % currentDir) res = storageElement.listDirectory( currentDir) if not res['Value'][ 'Successful'].has_key( currentDir): gLogger.error( "SEvsLFCAgent.execute: Failed to list the directory contents.", "%s %s" % (currentDir, res['Value']['Successful'] ['Failed'][currentDir])) subDirs = [currentDir] else: subDirs = [] files = {} for surl, surlDict in res['Value'][ 'Successful'][currentDir][ 'Files'].items(): pfnRes = storageElement.getPfnForProtocol( surl, 'SRM2', withPort=False) surl = pfnRes['Value'] files[surl] = surlDict for surl, surlDict in res['Value'][ 'Successful'][currentDir][ 'SubDirs'].items(): pfnRes = storageElement.getPfnForProtocol( surl, 'SRM2', withPort=False) surl = pfnRes['Value'] subDirs.append(surl) #subDirs = res['Value']['Successful'][currentDir]['SubDirs'] gLogger.info( "SEvsLFCAgent.execute: Successfully obtained %s sub-directories." % len(subDirs)) #files = res['Value']['Successful'][currentDir]['Files'] gLogger.info( "SEvsLFCAgent.execute: Successfully obtained %s files." % len(files)) selectedLfns = [] lfnPfnDict = {} pfnSize = {} for pfn, pfnDict in files.items(): res = storageElement.getPfnPath( pfn) if not res['OK']: gLogger.error( "SEvsLFCAgent.execute: Failed to get determine LFN from pfn.", "%s %s" % (pfn, res['Message'])) fileMetadata = { 'Prognosis': 'NonConventionPfn', 'LFN': '', 'PFN': pfn, 'StorageElement': storageElementName, 'Size': pfnDict['Size'] } res = IntegrityDB.insertProblematic( AGENT_NAME, fileMetadata) if res['OK']: gLogger.info( "SEvsLFCAgent.execute: Successfully added to IntegrityDB." ) gLogger.error( "Change the status in the LFC,ProcDB...." ) else: gLogger.error( "Shit, f**k, bugger. Add the failover." ) else: lfn = res['Value'] selectedLfns.append(lfn) lfnPfnDict[lfn] = pfn pfnSize[pfn] = pfnDict[ 'Size'] res = self.ReplicaManager.getCatalogFileMetadata( selectedLfns) if not res['OK']: subDirs = [currentDir] else: for lfn in res['Value'][ 'Failed'].keys(): gLogger.error( "SEvsLFCAgent.execute: Failed to get metadata.", "%s %s" % (lfn, res['Value'] ['Failed'][lfn])) pfn = lfnPfnDict[lfn] fileMetadata = { 'Prognosis': 'SEPfnNoLfn', 'LFN': lfn, 'PFN': pfn, 'StorageElement': storageElementName, 'Size': pfnSize[pfn] } res = IntegrityDB.insertProblematic( AGENT_NAME, fileMetadata) if res['OK']: gLogger.info( "SEvsLFCAgent.execute: Successfully added to IntegrityDB." ) gLogger.error( "Change the status in the LFC,ProcDB...." ) else: gLogger.error( "Shit, f**k, bugger. Add the failover." ) for lfn, lfnDict in res[ 'Value'][ 'Successful'].items( ): pfn = lfnPfnDict[lfn] storageSize = pfnSize[pfn] catalogSize = lfnDict[ 'Size'] if int(catalogSize) == int( storageSize): gLogger.info( "SEvsLFCAgent.execute: Catalog and storage sizes match.", "%s %s" % (pfn, storageElementName )) gLogger.info( "Change the status in the LFC" ) elif int(storageSize) == 0: gLogger.error( "SEvsLFCAgent.execute: Physical file size is 0.", "%s %s" % (pfn, storageElementName )) fileMetadata = { 'Prognosis': 'ZeroSizePfn', 'LFN': lfn, 'PFN': pfn, 'StorageElement': storageElementName } res = IntegrityDB.insertProblematic( AGENT_NAME, fileMetadata) if res['OK']: gLogger.info( "SEvsLFCAgent.execute: Successfully added to IntegrityDB." ) gLogger.error( "Change the status in the LFC,ProcDB...." ) else: gLogger.error( "Shit, f**k, bugger. Add the failover." ) else: gLogger.error( "SEvsLFCAgent.execute: Catalog and storage size mis-match.", "%s %s" % (pfn, storageElementName )) fileMetadata = { 'Prognosis': 'PfnSizeMismatch', 'LFN': lfn, 'PFN': pfn, 'StorageElement': storageElementName } res = IntegrityDB.insertProblematic( AGENT_NAME, fileMetadata) if res['OK']: gLogger.info( "SEvsLFCAgent.execute: Successfully added to IntegrityDB." ) gLogger.error( "Change the status in the LFC,ProcDB...." ) else: gLogger.error( "Shit, f**k, bugger. Add the failover." ) res = self.ReplicaManager.getCatalogReplicas( selectedLfns) if not res['OK']: subDirs = [currentDir] else: for lfn in res['Value'][ 'Failed'].keys(): gLogger.error( "SEvsLFCAgent.execute: Failed to get replica information.", "%s %s" % (lfn, res['Value'] ['Failed'][lfn])) pfn = lfnPfnDict[lfn] fileMetadata = { 'Prognosis': 'PfnNoReplica', 'LFN': lfn, 'PFN': pfn, 'StorageElement': storageElementName, 'Size': pfnSize[pfn] } res = IntegrityDB.insertProblematic( AGENT_NAME, fileMetadata) if res['OK']: gLogger.info( "SEvsLFCAgent.execute: Successfully added to IntegrityDB." ) gLogger.error( "Change the status in the LFC,ProcDB...." ) else: gLogger.error( "Shit, f**k, bugger. Add the failover." ) for lfn, repDict in res[ 'Value'][ 'Successful'].items( ): pfn = lfnPfnDict[lfn] registeredPfns = repDict.values( ) if not pfn in registeredPfns: gLogger.error( "SEvsLFCAgent.execute: SE PFN not registered.", "%s %s" % (lfn, pfn)) fileMetadata = { 'Prognosis': 'PfnNoReplica', 'LFN': lfn, 'PFN': pfn, 'StorageElement': storageElementName } res = IntegrityDB.insertProblematic( AGENT_NAME, fileMetadata) if res['OK']: gLogger.info( "SEvsLFCAgent.execute: Successfully added to IntegrityDB." ) gLogger.error( "Change the status in the LFC,ProcDB...." ) else: gLogger.error( "Shit, f**k, bugger. Add the failover." ) else: gLogger.info( "SEvsLFCAgent.execute: SE Pfn verified.", pfn) oNamespaceBrowser.updateDirs(subDirs) oRequest.setSubRequestFileAttributeValue( ind, 'integrity', lfn, 'Status', 'Done') ################################################ # If the sub-request is none of the above types else: gLogger.info( "SEvsLFCAgent.execute: Operation not supported.", operation) ################################################ # Determine whether there are any active files if oRequest.isSubRequestEmpty(ind, 'integrity')['Value']: oRequest.setSubRequestStatus(ind, 'integrity', 'Done') ################################################ # If the sub-request is already in terminal state else: gLogger.info( "SEvsLFCAgent.execute: Sub-request %s is status '%s' and not to be executed." % (ind, subRequestAttributes['Status'])) ################################################ # Generate the new request string after operation requestString = oRequest.toXML()['Value'] res = self.RequestDBClient.updateRequest(requestName, requestString, sourceServer) return S_OK()