def execute( self ):
res = self.RequestDBClient.getRequest( 'integrity' )
if not res['OK']:
gLogger.info( "LFCvsSEAgent.execute: Failed to get request from database." )
return S_OK()
elif not res['Value']:
gLogger.info( "LFCvsSEAgent.execute: No requests to be executed found." )
return S_OK()
requestString = res['Value']['RequestString']
requestName = res['Value']['RequestName']
sourceServer = res['Value']['Server']
gLogger.info( "LFCvsSEAgent.execute: Obtained request %s" % requestName )
oRequest = RequestContainer( request = requestString )
################################################
# Find the number of sub-requests from the request
res = oRequest.getNumSubRequests( 'integrity' )
if not res['OK']:
errStr = "LFCvsSEAgent.execute: Failed to obtain number of integrity subrequests."
gLogger.error( errStr, res['Message'] )
return S_OK()
gLogger.info( "LFCvsSEAgent.execute: Found %s sub requests." % res['Value'] )
################################################
# For all the sub-requests in the request
for ind in range( res['Value'] ):
gLogger.info( "LFCvsSEAgent.execute: Processing sub-request %s." % ind )
subRequestAttributes = oRequest.getSubRequestAttributes( ind, 'integrity' )['Value']
if subRequestAttributes['Status'] == 'Waiting':
subRequestFiles = oRequest.getSubRequestFiles( ind, 'integrity' )['Value']
operation = subRequestAttributes['Operation']
################################################
# If the sub-request is a lfcvsse operation
if operation == 'LFCvsSE':
gLogger.info( "LFCvsSEAgent.execute: Attempting to execute %s sub-request." % operation )
for subRequestFile in subRequestFiles:
if subRequestFile['Status'] == 'Waiting':
lfn = subRequestFile['LFN']
oNamespaceBrowser = NamespaceBrowser( lfn )
# Loop over all the directories and sub-directories
while ( oNamespaceBrowser.isActive() ):
currentDir = oNamespaceBrowser.getActiveDir()
gLogger.info( "LFCvsSEAgent.execute: Attempting to get contents of %s." % currentDir )
res = self.ReplicaManager.getCatalogDirectoryContents( currentDir )
if not res['OK']:
subDirs = [currentDir]
elif res['Value']['Failed'].has_key( currentDir ):
subDirs = [currentDir]
else:
subDirs = res['Value']['Successful'][currentDir]['SubDirs']
files = res['Value']['Successful'][currentDir]['Files']
lfnSizeDict = {}
pfnLfnDict = {}
pfnStatusDict = {}
sePfnDict = {}
for lfn, lfnDict in files.items():
lfnSizeDict[lfn] = lfnDict['MetaData']['Size']
for se in lfnDict['Replicas'].keys():
pfn = lfnDict['Replicas'][se]['PFN']
status = lfnDict['Replicas'][se]['Status']
pfnStatusDict[pfn] = status
pfnLfnDict[pfn] = lfn
if not sePfnDict.has_key( se ):
sePfnDict[se] = []
sePfnDict[se].append( pfn )
for storageElementName, physicalFiles in sePfnDict.items():
gLogger.info( "LFCvsSEAgent.execute: Attempting to get metadata for files on %s." % storageElementName )
res = self.ReplicaManager.getStorageFileMetadata( physicalFiles, storageElementName )
if not res['OK']:
gLogger.error( "LFCvsSEAgent.execute: Completely failed to get physical file metadata.", res['Message'] )
else:
for pfn in res['Value']['Failed'].keys():
gLogger.error( "LFCvsSEAgent.execute: Failed to get metadata.", "%s %s" % ( pfn, res['Value']['Failed'][pfn] ) )
lfn = pfnLfnDict[pfn]
fileMetadata = {'Prognosis':'MissingSEPfn', 'LFN':lfn, 'PFN':pfn, 'StorageElement':storageElementName, 'Size':lfnSizeDict[lfn]}
IntegrityDB = RPCClient( 'DataManagement/DataIntegrity' )
resInsert = IntegrityDB.insertProblematic( AGENT_NAME, fileMetadata )
if resInsert['OK']:
gLogger.info( "LFCvsSEAgent.execute: Successfully added to IntegrityDB." )
gLogger.error( "Change the status in the LFC,ProcDB...." )
else:
gLogger.error( "Shit, f**k, bugger. Add the failover." )
for pfn, pfnDict in res['Value']['Successful'].items():
lfn = pfnLfnDict[pfn]
catalogSize = int( lfnSizeDict[lfn] )
storageSize = int( pfnDict['Size'] )
if int( catalogSize ) == int( storageSize ):
gLogger.info( "LFCvsSEAgent.execute: Catalog and storage sizes match.", "%s %s" % ( pfn, storageElementName ) )
gLogger.info( "Change the status in the LFC" )
else:
gLogger.error( "LFCvsSEAgent.execute: Catalog and storage size mis-match.", "%s %s" % ( pfn, storageElementName ) )
fileMetadata = {'Prognosis':'PfnSizeMismatch', 'LFN':lfn, 'PFN':pfn, 'StorageElement':storageElementName}
IntegrityDB = RPCClient( 'DataManagement/DataIntegrity' )
resInsert = IntegrityDB.insertProblematic( AGENT_NAME, fileMetadata )
if resInsert['OK']:
gLogger.info( "LFCvsSEAgent.execute: Successfully added to IntegrityDB." )
gLogger.error( "Change the status in the LFC,ProcDB...." )
else:
gLogger.error( "Shit, f**k, bugger. Add the failover." )
oNamespaceBrowser.updateDirs( subDirs )
oRequest.setSubRequestFileAttributeValue( ind, 'integrity', lfn, 'Status', 'Done' )
################################################
# If the sub-request is none of the above types
else:
gLogger.info( "LFCvsSEAgent.execute: Operation not supported.", operation )
################################################
# Determine whether there are any active files
if oRequest.isSubRequestEmpty( ind, 'integrity' )['Value']:
oRequest.setSubRequestStatus( ind, 'integrity', 'Done' )
################################################
# If the sub-request is already in terminal state
else:
gLogger.info( "LFCvsSEAgent.execute: Sub-request %s is status '%s' and not to be executed." % ( ind, subRequestAttributes['Status'] ) )
################################################
# Generate the new request string after operation
requestString = oRequest.toXML()['Value']
res = self.RequestDBClient.updateRequest( requestName, requestString, sourceServer )
return S_OK()
def execute(self):
res = self.RequestDBClient.getRequest('integrity')
if not res['OK']:
gLogger.info(
"LFCvsSEAgent.execute: Failed to get request from database.")
return S_OK()
elif not res['Value']:
gLogger.info(
"LFCvsSEAgent.execute: No requests to be executed found.")
return S_OK()
requestString = res['Value']['RequestString']
requestName = res['Value']['RequestName']
sourceServer = res['Value']['Server']
gLogger.info("LFCvsSEAgent.execute: Obtained request %s" % requestName)
oRequest = RequestContainer(request=requestString)
################################################
# Find the number of sub-requests from the request
res = oRequest.getNumSubRequests('integrity')
if not res['OK']:
errStr = "LFCvsSEAgent.execute: Failed to obtain number of integrity subrequests."
gLogger.error(errStr, res['Message'])
return S_OK()
gLogger.info("LFCvsSEAgent.execute: Found %s sub requests." %
res['Value'])
################################################
# For all the sub-requests in the request
for ind in range(res['Value']):
gLogger.info("LFCvsSEAgent.execute: Processing sub-request %s." %
ind)
subRequestAttributes = oRequest.getSubRequestAttributes(
ind, 'integrity')['Value']
if subRequestAttributes['Status'] == 'Waiting':
subRequestFiles = oRequest.getSubRequestFiles(
ind, 'integrity')['Value']
operation = subRequestAttributes['Operation']
################################################
# If the sub-request is a lfcvsse operation
if operation == 'LFCvsSE':
gLogger.info(
"LFCvsSEAgent.execute: Attempting to execute %s sub-request."
% operation)
for subRequestFile in subRequestFiles:
if subRequestFile['Status'] == 'Waiting':
lfn = subRequestFile['LFN']
oNamespaceBrowser = NamespaceBrowser(lfn)
# Loop over all the directories and sub-directories
while (oNamespaceBrowser.isActive()):
currentDir = oNamespaceBrowser.getActiveDir()
gLogger.info(
"LFCvsSEAgent.execute: Attempting to get contents of %s."
% currentDir)
res = self.ReplicaManager.getCatalogDirectoryContents(
currentDir)
if not res['OK']:
subDirs = [currentDir]
elif res['Value']['Failed'].has_key(
currentDir):
subDirs = [currentDir]
else:
subDirs = res['Value']['Successful'][
currentDir]['SubDirs']
files = res['Value']['Successful'][
currentDir]['Files']
lfnSizeDict = {}
pfnLfnDict = {}
pfnStatusDict = {}
sePfnDict = {}
for lfn, lfnDict in files.items():
lfnSizeDict[lfn] = lfnDict['MetaData'][
'Size']
for se in lfnDict['Replicas'].keys():
pfn = lfnDict['Replicas'][se][
'PFN']
status = lfnDict['Replicas'][se][
'Status']
pfnStatusDict[pfn] = status
pfnLfnDict[pfn] = lfn
if not sePfnDict.has_key(se):
sePfnDict[se] = []
sePfnDict[se].append(pfn)
for storageElementName, physicalFiles in sePfnDict.items(
):
gLogger.info(
"LFCvsSEAgent.execute: Attempting to get metadata for files on %s."
% storageElementName)
res = self.ReplicaManager.getStorageFileMetadata(
physicalFiles, storageElementName)
if not res['OK']:
gLogger.error(
"LFCvsSEAgent.execute: Completely failed to get physical file metadata.",
res['Message'])
else:
for pfn in res['Value'][
'Failed'].keys():
gLogger.error(
"LFCvsSEAgent.execute: Failed to get metadata.",
"%s %s" %
(pfn, res['Value']
['Failed'][pfn]))
lfn = pfnLfnDict[pfn]
fileMetadata = {
'Prognosis':
'MissingSEPfn',
'LFN': lfn,
'PFN': pfn,
'StorageElement':
storageElementName,
'Size': lfnSizeDict[lfn]
}
IntegrityDB = RPCClient(
'DataManagement/DataIntegrity'
)
resInsert = IntegrityDB.insertProblematic(
AGENT_NAME, fileMetadata)
if resInsert['OK']:
gLogger.info(
"LFCvsSEAgent.execute: Successfully added to IntegrityDB."
)
gLogger.error(
"Change the status in the LFC,ProcDB...."
)
else:
gLogger.error(
"Shit, f**k, bugger. Add the failover."
)
for pfn, pfnDict in res['Value'][
'Successful'].items():
lfn = pfnLfnDict[pfn]
catalogSize = int(
lfnSizeDict[lfn])
storageSize = int(
pfnDict['Size'])
if int(catalogSize) == int(
storageSize):
gLogger.info(
"LFCvsSEAgent.execute: Catalog and storage sizes match.",
"%s %s" %
(pfn,
storageElementName))
gLogger.info(
"Change the status in the LFC"
)
else:
gLogger.error(
"LFCvsSEAgent.execute: Catalog and storage size mis-match.",
"%s %s" %
(pfn,
storageElementName))
fileMetadata = {
'Prognosis':
'PfnSizeMismatch',
'LFN':
lfn,
'PFN':
pfn,
'StorageElement':
storageElementName
}
IntegrityDB = RPCClient(
'DataManagement/DataIntegrity'
)
resInsert = IntegrityDB.insertProblematic(
AGENT_NAME,
fileMetadata)
if resInsert['OK']:
gLogger.info(
"LFCvsSEAgent.execute: Successfully added to IntegrityDB."
)
gLogger.error(
"Change the status in the LFC,ProcDB...."
)
else:
gLogger.error(
"Shit, f**k, bugger. Add the failover."
)
oNamespaceBrowser.updateDirs(subDirs)
oRequest.setSubRequestFileAttributeValue(
ind, 'integrity', lfn, 'Status', 'Done')
################################################
# If the sub-request is none of the above types
else:
gLogger.info(
"LFCvsSEAgent.execute: Operation not supported.",
operation)
################################################
# Determine whether there are any active files
if oRequest.isSubRequestEmpty(ind, 'integrity')['Value']:
oRequest.setSubRequestStatus(ind, 'integrity', 'Done')
################################################
# If the sub-request is already in terminal state
else:
gLogger.info(
"LFCvsSEAgent.execute: Sub-request %s is status '%s' and not to be executed."
% (ind, subRequestAttributes['Status']))
################################################
# Generate the new request string after operation
requestString = oRequest.toXML()['Value']
res = self.RequestDBClient.updateRequest(requestName, requestString,
sourceServer)
return S_OK()
def execute( self ):
IntegrityDB = RPCClient( 'DataManagement/DataIntegrity' )
res = self.RequestDBClient.getRequest( 'integrity' )
if not res['OK']:
gLogger.info( "SEvsLFCAgent.execute: Failed to get request from database." )
return S_OK()
elif not res['Value']:
gLogger.info( "SEvsLFCAgent.execute: No requests to be executed found." )
return S_OK()
requestString = res['Value']['requestString']
requestName = res['Value']['requestName']
sourceServer = res['Value']['Server']
gLogger.info( "SEvsLFCAgent.execute: Obtained request %s" % requestName )
oRequest = RequestContainer( request = requestString )
################################################
# Find the number of sub-requests from the request
res = oRequest.getNumSubRequests( 'integrity' )
if not res['OK']:
errStr = "SEvsLFCAgent.execute: Failed to obtain number of integrity subrequests."
gLogger.error( errStr, res['Message'] )
return S_OK()
gLogger.info( "SEvsLFCAgent.execute: Found %s sub requests." % res['Value'] )
################################################
# For all the sub-requests in the request
for ind in range( res['Value'] ):
gLogger.info( "SEvsLFCAgent.execute: Processing sub-request %s." % ind )
subRequestAttributes = oRequest.getSubRequestAttributes( ind, 'integrity' )['Value']
if subRequestAttributes['Status'] == 'Waiting':
subRequestFiles = oRequest.getSubRequestFiles( ind, 'integrity' )['Value']
operation = subRequestAttributes['Operation']
################################################
# If the sub-request is a lfcvsse operation
if operation == 'SEvsLFC':
gLogger.info( "SEvsLFCAgent.execute: Attempting to execute %s sub-request." % operation )
storageElementName = subRequestAttributes['StorageElement']
for subRequestFile in subRequestFiles:
if subRequestFile['Status'] == 'Waiting':
lfn = subRequestFile['LFN']
storageElement = StorageElement( storageElementName )
res = storageElement.isValid()
if not res['OK']:
errStr = "SEvsLFCAgent.execute: Failed to instantiate destination StorageElement."
gLogger.error( errStr, storageElement )
else:
res = storageElement.getPfnForLfn( lfn )
if not res['OK']:
gLogger.info( 'shit bugger do something.' )
else:
oNamespaceBrowser = NamespaceBrowser( res['Value'] )
# Loop over all the directories and sub-directories
while ( oNamespaceBrowser.isActive() ):
currentDir = oNamespaceBrowser.getActiveDir()
gLogger.info( "SEvsLFCAgent.execute: Attempting to list the contents of %s." % currentDir )
res = storageElement.listDirectory( currentDir )
if not res['Value']['Successful'].has_key( currentDir ):
gLogger.error( "SEvsLFCAgent.execute: Failed to list the directory contents.", "%s %s" % ( currentDir, res['Value']['Successful']['Failed'][currentDir] ) )
subDirs = [currentDir]
else:
subDirs = []
files = {}
for surl, surlDict in res['Value']['Successful'][currentDir]['Files'].items():
pfnRes = storageElement.getPfnForProtocol( surl, 'SRM2', withPort = False )
surl = pfnRes['Value']
files[surl] = surlDict
for surl, surlDict in res['Value']['Successful'][currentDir]['SubDirs'].items():
pfnRes = storageElement.getPfnForProtocol( surl, 'SRM2', withPort = False )
surl = pfnRes['Value']
subDirs.append( surl )
#subDirs = res['Value']['Successful'][currentDir]['SubDirs']
gLogger.info( "SEvsLFCAgent.execute: Successfully obtained %s sub-directories." % len( subDirs ) )
#files = res['Value']['Successful'][currentDir]['Files']
gLogger.info( "SEvsLFCAgent.execute: Successfully obtained %s files." % len( files ) )
selectedLfns = []
lfnPfnDict = {}
pfnSize = {}
for pfn, pfnDict in files.items():
res = storageElement.getPfnPath( pfn )
if not res['OK']:
gLogger.error( "SEvsLFCAgent.execute: Failed to get determine LFN from pfn.", "%s %s" % ( pfn, res['Message'] ) )
fileMetadata = {'Prognosis':'NonConventionPfn', 'LFN':'', 'PFN':pfn, 'StorageElement':storageElementName, 'Size':pfnDict['Size']}
res = IntegrityDB.insertProblematic( AGENT_NAME, fileMetadata )
if res['OK']:
gLogger.info( "SEvsLFCAgent.execute: Successfully added to IntegrityDB." )
gLogger.error( "Change the status in the LFC,ProcDB...." )
else:
gLogger.error( "Shit, f**k, bugger. Add the failover." )
else:
lfn = res['Value']
selectedLfns.append( lfn )
lfnPfnDict[lfn] = pfn
pfnSize[pfn] = pfnDict['Size']
res = self.ReplicaManager.getCatalogFileMetadata( selectedLfns )
if not res['OK']:
subDirs = [currentDir]
else:
for lfn in res['Value']['Failed'].keys():
gLogger.error( "SEvsLFCAgent.execute: Failed to get metadata.", "%s %s" % ( lfn, res['Value']['Failed'][lfn] ) )
pfn = lfnPfnDict[lfn]
fileMetadata = {'Prognosis':'SEPfnNoLfn', 'LFN':lfn, 'PFN':pfn, 'StorageElement':storageElementName, 'Size':pfnSize[pfn]}
res = IntegrityDB.insertProblematic( AGENT_NAME, fileMetadata )
if res['OK']:
gLogger.info( "SEvsLFCAgent.execute: Successfully added to IntegrityDB." )
gLogger.error( "Change the status in the LFC,ProcDB...." )
else:
gLogger.error( "Shit, f**k, bugger. Add the failover." )
for lfn, lfnDict in res['Value']['Successful'].items():
pfn = lfnPfnDict[lfn]
storageSize = pfnSize[pfn]
catalogSize = lfnDict['Size']
if int( catalogSize ) == int( storageSize ):
gLogger.info( "SEvsLFCAgent.execute: Catalog and storage sizes match.", "%s %s" % ( pfn, storageElementName ) )
gLogger.info( "Change the status in the LFC" )
elif int( storageSize ) == 0:
gLogger.error( "SEvsLFCAgent.execute: Physical file size is 0.", "%s %s" % ( pfn, storageElementName ) )
fileMetadata = {'Prognosis':'ZeroSizePfn', 'LFN':lfn, 'PFN':pfn, 'StorageElement':storageElementName}
res = IntegrityDB.insertProblematic( AGENT_NAME, fileMetadata )
if res['OK']:
gLogger.info( "SEvsLFCAgent.execute: Successfully added to IntegrityDB." )
gLogger.error( "Change the status in the LFC,ProcDB...." )
else:
gLogger.error( "Shit, f**k, bugger. Add the failover." )
else:
gLogger.error( "SEvsLFCAgent.execute: Catalog and storage size mis-match.", "%s %s" % ( pfn, storageElementName ) )
fileMetadata = {'Prognosis':'PfnSizeMismatch', 'LFN':lfn, 'PFN':pfn, 'StorageElement':storageElementName}
res = IntegrityDB.insertProblematic( AGENT_NAME, fileMetadata )
if res['OK']:
gLogger.info( "SEvsLFCAgent.execute: Successfully added to IntegrityDB." )
gLogger.error( "Change the status in the LFC,ProcDB...." )
else:
gLogger.error( "Shit, f**k, bugger. Add the failover." )
res = self.ReplicaManager.getCatalogReplicas( selectedLfns )
if not res['OK']:
subDirs = [currentDir]
else:
for lfn in res['Value']['Failed'].keys():
gLogger.error( "SEvsLFCAgent.execute: Failed to get replica information.", "%s %s" % ( lfn, res['Value']['Failed'][lfn] ) )
pfn = lfnPfnDict[lfn]
fileMetadata = {'Prognosis':'PfnNoReplica', 'LFN':lfn, 'PFN':pfn, 'StorageElement':storageElementName, 'Size':pfnSize[pfn]}
res = IntegrityDB.insertProblematic( AGENT_NAME, fileMetadata )
if res['OK']:
gLogger.info( "SEvsLFCAgent.execute: Successfully added to IntegrityDB." )
gLogger.error( "Change the status in the LFC,ProcDB...." )
else:
gLogger.error( "Shit, f**k, bugger. Add the failover." )
for lfn, repDict in res['Value']['Successful'].items():
pfn = lfnPfnDict[lfn]
registeredPfns = repDict.values()
if not pfn in registeredPfns:
gLogger.error( "SEvsLFCAgent.execute: SE PFN not registered.", "%s %s" % ( lfn, pfn ) )
fileMetadata = {'Prognosis':'PfnNoReplica', 'LFN':lfn, 'PFN':pfn, 'StorageElement':storageElementName}
res = IntegrityDB.insertProblematic( AGENT_NAME, fileMetadata )
if res['OK']:
gLogger.info( "SEvsLFCAgent.execute: Successfully added to IntegrityDB." )
gLogger.error( "Change the status in the LFC,ProcDB...." )
else:
gLogger.error( "Shit, f**k, bugger. Add the failover." )
else:
gLogger.info( "SEvsLFCAgent.execute: SE Pfn verified.", pfn )
oNamespaceBrowser.updateDirs( subDirs )
oRequest.setSubRequestFileAttributeValue( ind, 'integrity', lfn, 'Status', 'Done' )
################################################
# If the sub-request is none of the above types
else:
gLogger.info( "SEvsLFCAgent.execute: Operation not supported.", operation )
################################################
# Determine whether there are any active files
if oRequest.isSubRequestEmpty( ind, 'integrity' )['Value']:
oRequest.setSubRequestStatus( ind, 'integrity', 'Done' )
################################################
# If the sub-request is already in terminal state
else:
gLogger.info( "SEvsLFCAgent.execute: Sub-request %s is status '%s' and not to be executed." % ( ind, subRequestAttributes['Status'] ) )
################################################
# Generate the new request string after operation
requestString = oRequest.toXML()['Value']
res = self.RequestDBClient.updateRequest( requestName, requestString, sourceServer )
return S_OK()
def execute(self):
IntegrityDB = RPCClient('DataManagement/DataIntegrity')
res = self.RequestDBClient.getRequest('integrity')
if not res['OK']:
gLogger.info(
"SEvsLFCAgent.execute: Failed to get request from database.")
return S_OK()
elif not res['Value']:
gLogger.info(
"SEvsLFCAgent.execute: No requests to be executed found.")
return S_OK()
requestString = res['Value']['requestString']
requestName = res['Value']['requestName']
sourceServer = res['Value']['Server']
gLogger.info("SEvsLFCAgent.execute: Obtained request %s" % requestName)
oRequest = RequestContainer(request=requestString)
################################################
# Find the number of sub-requests from the request
res = oRequest.getNumSubRequests('integrity')
if not res['OK']:
errStr = "SEvsLFCAgent.execute: Failed to obtain number of integrity subrequests."
gLogger.error(errStr, res['Message'])
return S_OK()
gLogger.info("SEvsLFCAgent.execute: Found %s sub requests." %
res['Value'])
################################################
# For all the sub-requests in the request
for ind in range(res['Value']):
gLogger.info("SEvsLFCAgent.execute: Processing sub-request %s." %
ind)
subRequestAttributes = oRequest.getSubRequestAttributes(
ind, 'integrity')['Value']
if subRequestAttributes['Status'] == 'Waiting':
subRequestFiles = oRequest.getSubRequestFiles(
ind, 'integrity')['Value']
operation = subRequestAttributes['Operation']
################################################
# If the sub-request is a lfcvsse operation
if operation == 'SEvsLFC':
gLogger.info(
"SEvsLFCAgent.execute: Attempting to execute %s sub-request."
% operation)
storageElementName = subRequestAttributes['StorageElement']
for subRequestFile in subRequestFiles:
if subRequestFile['Status'] == 'Waiting':
lfn = subRequestFile['LFN']
storageElement = StorageElement(storageElementName)
res = storageElement.isValid()
if not res['OK']:
errStr = "SEvsLFCAgent.execute: Failed to instantiate destination StorageElement."
gLogger.error(errStr, storageElement)
else:
res = storageElement.getPfnForLfn(lfn)
if not res['OK']:
gLogger.info('shit bugger do something.')
else:
oNamespaceBrowser = NamespaceBrowser(
res['Value'])
# Loop over all the directories and sub-directories
while (oNamespaceBrowser.isActive()):
currentDir = oNamespaceBrowser.getActiveDir(
)
gLogger.info(
"SEvsLFCAgent.execute: Attempting to list the contents of %s."
% currentDir)
res = storageElement.listDirectory(
currentDir)
if not res['Value'][
'Successful'].has_key(
currentDir):
gLogger.error(
"SEvsLFCAgent.execute: Failed to list the directory contents.",
"%s %s" %
(currentDir,
res['Value']['Successful']
['Failed'][currentDir]))
subDirs = [currentDir]
else:
subDirs = []
files = {}
for surl, surlDict in res['Value'][
'Successful'][currentDir][
'Files'].items():
pfnRes = storageElement.getPfnForProtocol(
surl,
'SRM2',
withPort=False)
surl = pfnRes['Value']
files[surl] = surlDict
for surl, surlDict in res['Value'][
'Successful'][currentDir][
'SubDirs'].items():
pfnRes = storageElement.getPfnForProtocol(
surl,
'SRM2',
withPort=False)
surl = pfnRes['Value']
subDirs.append(surl)
#subDirs = res['Value']['Successful'][currentDir]['SubDirs']
gLogger.info(
"SEvsLFCAgent.execute: Successfully obtained %s sub-directories."
% len(subDirs))
#files = res['Value']['Successful'][currentDir]['Files']
gLogger.info(
"SEvsLFCAgent.execute: Successfully obtained %s files."
% len(files))
selectedLfns = []
lfnPfnDict = {}
pfnSize = {}
for pfn, pfnDict in files.items():
res = storageElement.getPfnPath(
pfn)
if not res['OK']:
gLogger.error(
"SEvsLFCAgent.execute: Failed to get determine LFN from pfn.",
"%s %s" %
(pfn, res['Message']))
fileMetadata = {
'Prognosis':
'NonConventionPfn',
'LFN': '',
'PFN': pfn,
'StorageElement':
storageElementName,
'Size': pfnDict['Size']
}
res = IntegrityDB.insertProblematic(
AGENT_NAME,
fileMetadata)
if res['OK']:
gLogger.info(
"SEvsLFCAgent.execute: Successfully added to IntegrityDB."
)
gLogger.error(
"Change the status in the LFC,ProcDB...."
)
else:
gLogger.error(
"Shit, f**k, bugger. Add the failover."
)
else:
lfn = res['Value']
selectedLfns.append(lfn)
lfnPfnDict[lfn] = pfn
pfnSize[pfn] = pfnDict[
'Size']
res = self.ReplicaManager.getCatalogFileMetadata(
selectedLfns)
if not res['OK']:
subDirs = [currentDir]
else:
for lfn in res['Value'][
'Failed'].keys():
gLogger.error(
"SEvsLFCAgent.execute: Failed to get metadata.",
"%s %s" %
(lfn, res['Value']
['Failed'][lfn]))
pfn = lfnPfnDict[lfn]
fileMetadata = {
'Prognosis':
'SEPfnNoLfn',
'LFN': lfn,
'PFN': pfn,
'StorageElement':
storageElementName,
'Size': pfnSize[pfn]
}
res = IntegrityDB.insertProblematic(
AGENT_NAME,
fileMetadata)
if res['OK']:
gLogger.info(
"SEvsLFCAgent.execute: Successfully added to IntegrityDB."
)
gLogger.error(
"Change the status in the LFC,ProcDB...."
)
else:
gLogger.error(
"Shit, f**k, bugger. Add the failover."
)
for lfn, lfnDict in res[
'Value'][
'Successful'].items(
):
pfn = lfnPfnDict[lfn]
storageSize = pfnSize[pfn]
catalogSize = lfnDict[
'Size']
if int(catalogSize) == int(
storageSize):
gLogger.info(
"SEvsLFCAgent.execute: Catalog and storage sizes match.",
"%s %s" %
(pfn,
storageElementName
))
gLogger.info(
"Change the status in the LFC"
)
elif int(storageSize) == 0:
gLogger.error(
"SEvsLFCAgent.execute: Physical file size is 0.",
"%s %s" %
(pfn,
storageElementName
))
fileMetadata = {
'Prognosis':
'ZeroSizePfn',
'LFN':
lfn,
'PFN':
pfn,
'StorageElement':
storageElementName
}
res = IntegrityDB.insertProblematic(
AGENT_NAME,
fileMetadata)
if res['OK']:
gLogger.info(
"SEvsLFCAgent.execute: Successfully added to IntegrityDB."
)
gLogger.error(
"Change the status in the LFC,ProcDB...."
)
else:
gLogger.error(
"Shit, f**k, bugger. Add the failover."
)
else:
gLogger.error(
"SEvsLFCAgent.execute: Catalog and storage size mis-match.",
"%s %s" %
(pfn,
storageElementName
))
fileMetadata = {
'Prognosis':
'PfnSizeMismatch',
'LFN':
lfn,
'PFN':
pfn,
'StorageElement':
storageElementName
}
res = IntegrityDB.insertProblematic(
AGENT_NAME,
fileMetadata)
if res['OK']:
gLogger.info(
"SEvsLFCAgent.execute: Successfully added to IntegrityDB."
)
gLogger.error(
"Change the status in the LFC,ProcDB...."
)
else:
gLogger.error(
"Shit, f**k, bugger. Add the failover."
)
res = self.ReplicaManager.getCatalogReplicas(
selectedLfns)
if not res['OK']:
subDirs = [currentDir]
else:
for lfn in res['Value'][
'Failed'].keys():
gLogger.error(
"SEvsLFCAgent.execute: Failed to get replica information.",
"%s %s" %
(lfn, res['Value']
['Failed'][lfn]))
pfn = lfnPfnDict[lfn]
fileMetadata = {
'Prognosis':
'PfnNoReplica',
'LFN': lfn,
'PFN': pfn,
'StorageElement':
storageElementName,
'Size':
pfnSize[pfn]
}
res = IntegrityDB.insertProblematic(
AGENT_NAME,
fileMetadata)
if res['OK']:
gLogger.info(
"SEvsLFCAgent.execute: Successfully added to IntegrityDB."
)
gLogger.error(
"Change the status in the LFC,ProcDB...."
)
else:
gLogger.error(
"Shit, f**k, bugger. Add the failover."
)
for lfn, repDict in res[
'Value'][
'Successful'].items(
):
pfn = lfnPfnDict[lfn]
registeredPfns = repDict.values(
)
if not pfn in registeredPfns:
gLogger.error(
"SEvsLFCAgent.execute: SE PFN not registered.",
"%s %s" %
(lfn, pfn))
fileMetadata = {
'Prognosis':
'PfnNoReplica',
'LFN':
lfn,
'PFN':
pfn,
'StorageElement':
storageElementName
}
res = IntegrityDB.insertProblematic(
AGENT_NAME,
fileMetadata)
if res['OK']:
gLogger.info(
"SEvsLFCAgent.execute: Successfully added to IntegrityDB."
)
gLogger.error(
"Change the status in the LFC,ProcDB...."
)
else:
gLogger.error(
"Shit, f**k, bugger. Add the failover."
)
else:
gLogger.info(
"SEvsLFCAgent.execute: SE Pfn verified.",
pfn)
oNamespaceBrowser.updateDirs(subDirs)
oRequest.setSubRequestFileAttributeValue(
ind, 'integrity', lfn, 'Status',
'Done')
################################################
# If the sub-request is none of the above types
else:
gLogger.info(
"SEvsLFCAgent.execute: Operation not supported.",
operation)
################################################
# Determine whether there are any active files
if oRequest.isSubRequestEmpty(ind, 'integrity')['Value']:
oRequest.setSubRequestStatus(ind, 'integrity', 'Done')
################################################
# If the sub-request is already in terminal state
else:
gLogger.info(
"SEvsLFCAgent.execute: Sub-request %s is status '%s' and not to be executed."
% (ind, subRequestAttributes['Status']))
################################################
# Generate the new request string after operation
requestString = oRequest.toXML()['Value']
res = self.RequestDBClient.updateRequest(requestName, requestString,
sourceServer)
return S_OK()