def __init__( self, operation = None, csPath = None ):
"""c'tor
:param self: self reference
:param ~Operation.Operation operation: Operation instance
:param str csPath: CS path for this handler
"""
super( MoveReplica, self ).__init__( operation, csPath )
# # own gMonitor stuff for files
gMonitor.registerActivity( "ReplicateAndRegisterAtt", "Replicate and register attempted",
"RequestExecutingAgent", "Files/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "ReplicateOK", "Replications successful",
"RequestExecutingAgent", "Files/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "ReplicateFail", "Replications failed",
"RequestExecutingAgent", "Files/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "RegisterOK", "Registrations successful",
"RequestExecutingAgent", "Files/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "RegisterFail", "Registrations failed",
"RequestExecutingAgent", "Files/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "RemoveReplicaAtt", "Replica removals attempted",
"RequestExecutingAgent", "Files/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "RemoveReplicaOK", "Successful replica removals",
"RequestExecutingAgent", "Files/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "RemoveReplicaFail", "Failed replica removals",
"RequestExecutingAgent", "Files/min", gMonitor.OP_SUM )
# Init ConsistencyInspector: used to check replicas
self.ci = ConsistencyInspector()
def __init__(self, operation=None, csPath=None):
"""c'tor
:param self: self reference
:param ~Operation.Operation operation: Operation instance
:param str csPath: CS path for this handler
"""
super(MoveReplica, self).__init__(operation, csPath)
# Init ConsistencyInspector: used to check replicas
self.ci = ConsistencyInspector()
def __init__(self, *args, **kwargs):
""" c'tor
"""
AgentModule.__init__(self, *args, **kwargs)
self.consistencyInspector = ConsistencyInspector()
self.integrityClient = DataIntegrityClient()
self.fc = FileCatalog()
self.transClient = TransformationClient()
self.fileCatalogClient = FileCatalogClient()
agentTSTypes = self.am_getOption('TransformationTypes', [])
if agentTSTypes:
self.transformationTypes = agentTSTypes
else:
self.transformationTypes = Operations().getValue('Transformations/DataProcessing', ['MCSimulation', 'Merge'])
self.directoryLocations = sorted(self.am_getOption('DirectoryLocations', ['TransformationDB',
'MetadataCatalog']))
self.transfidmeta = self.am_getOption('TransfIDMeta', "TransformationID")
self.enableFlag = True
def setUp( self ):
gLogger.setLevel('DEBUG')
self.lfnDict = {'aa.raw': { 'aa.raw':{'FileType': 'RAW', 'RunNumber': 97019},
'/lhcb/1_2_1.Semileptonic.dst': {'FileType': 'SEMILEPTONIC.DST'}},
'cc.raw': { 'cc.raw':{'FileType': 'RAW', 'RunNumber': 97019},
'/lhcb/1_1.semileptonic.dst': {'FileType': 'SEMILEPTONIC.DST'}}
}
dmMock = MagicMock()
dicMock = MagicMock()
self.ci = ConsistencyInspector( transClient = MagicMock(), dm = dmMock, fc = fileCatalogMock, dic = dicMock )
self.ci.fileType = ['SEMILEPTONIC.DST', 'LOG', 'RAW']
self.ci.fileTypesExcluded = ['LOG']
self.ci.prod = 0
self.maxDiff = None
def __init__(self, *args, **kwargs):
""" c'tor
"""
AgentModule.__init__(self, *args, **kwargs)
self.consistencyInspector = ConsistencyInspector()
self.integrityClient = DataIntegrityClient()
self.fc = FileCatalog()
self.transClient = TransformationClient()
self.fileCatalogClient = FileCatalogClient()
agentTSTypes = self.am_getOption('TransformationTypes', [])
if agentTSTypes:
self.transformationTypes = agentTSTypes
else:
self.transformationTypes = Operations().getValue('Transformations/DataProcessing', ['MCSimulation', 'Merge'])
self.directoryLocations = sorted(self.am_getOption('DirectoryLocations', ['TransformationDB',
'MetadataCatalog']))
self.transfidmeta = self.am_getOption('TransfIDMeta', "TransformationID")
self.enableFlag = True
def setUp(self):
gLogger.setLevel("DEBUG")
self.lfnDict = {
"aa.raw": {
"aa.raw": {
"FileType": "RAW",
"RunNumber": 97019
},
"/lhcb/1_2_1.Semileptonic.dst": {
"FileType": "SEMILEPTONIC.DST"
},
},
"cc.raw": {
"cc.raw": {
"FileType": "RAW",
"RunNumber": 97019
},
"/lhcb/1_1.semileptonic.dst": {
"FileType": "SEMILEPTONIC.DST"
},
},
}
dmMock = MagicMock()
dicMock = MagicMock()
self.ci = ConsistencyInspector(transClient=MagicMock(),
dm=dmMock,
fc=fc_mock,
dic=dicMock)
self.ci.fileType = ["SEMILEPTONIC.DST", "LOG", "RAW"]
self.ci.fileTypesExcluded = ["LOG"]
self.ci.prod = 0
self.maxDiff = None
class ValidateOutputDataAgent(AgentModule):
def __init__(self, *args, **kwargs):
""" c'tor
"""
AgentModule.__init__(self, *args, **kwargs)
self.consistencyInspector = ConsistencyInspector()
self.integrityClient = DataIntegrityClient()
self.fc = FileCatalog()
self.transClient = TransformationClient()
self.fileCatalogClient = FileCatalogClient()
agentTSTypes = self.am_getOption('TransformationTypes', [])
if agentTSTypes:
self.transformationTypes = agentTSTypes
else:
self.transformationTypes = Operations().getValue(
'Transformations/DataProcessing', ['MCSimulation', 'Merge'])
self.directoryLocations = sorted(
self.am_getOption('DirectoryLocations',
['TransformationDB', 'MetadataCatalog']))
self.transfidmeta = self.am_getOption('TransfIDMeta',
"TransformationID")
self.enableFlag = True
#############################################################################
def initialize(self):
""" Sets defaults
"""
# This sets the Default Proxy to used as that defined under
# /Operations/Shifter/DataManager
# the shifterProxy option in the Configuration can be used to change this default.
self.am_setOption('shifterProxy', 'DataManager')
gLogger.info("Will treat the following transformation types: %s" %
str(self.transformationTypes))
gLogger.info(
"Will search for directories in the following locations: %s" %
str(self.directoryLocations))
gLogger.info("Will use %s as metadata tag name for TransformationID" %
self.transfidmeta)
return S_OK()
#############################################################################
def execute(self):
""" The VerifyOutputData execution method
"""
self.enableFlag = self.am_getOption('EnableFlag', 'True')
if not self.enableFlag == 'True':
self.log.info(
"VerifyOutputData is disabled by configuration option 'EnableFlag'"
)
return S_OK('Disabled via CS flag')
gLogger.info("-" * 40)
self.updateWaitingIntegrity()
gLogger.info("-" * 40)
res = self.transClient.getTransformations({
'Status':
'ValidatingOutput',
'Type':
self.transformationTypes
})
if not res['OK']:
gLogger.error("Failed to get ValidatingOutput transformations",
res['Message'])
return res
transDicts = res['Value']
if not transDicts:
gLogger.info("No transformations found in ValidatingOutput status")
return S_OK()
gLogger.info("Found %s transformations in ValidatingOutput status" %
len(transDicts))
for transDict in transDicts:
transID = transDict['TransformationID']
res = self.checkTransformationIntegrity(int(transID))
if not res['OK']:
gLogger.error(
"Failed to perform full integrity check for transformation %d"
% transID)
else:
self.finalizeCheck(transID)
gLogger.info("-" * 40)
return S_OK()
def updateWaitingIntegrity(self):
""" Get 'WaitingIntegrity' transformations, update to 'ValidatedOutput'
"""
gLogger.info(
"Looking for transformations in the WaitingIntegrity status to update"
)
res = self.transClient.getTransformations(
{'Status': 'WaitingIntegrity'})
if not res['OK']:
gLogger.error("Failed to get WaitingIntegrity transformations",
res['Message'])
return res
transDicts = res['Value']
if not transDicts:
gLogger.info("No transformations found in WaitingIntegrity status")
return S_OK()
gLogger.info("Found %s transformations in WaitingIntegrity status" %
len(transDicts))
for transDict in transDicts:
transID = transDict['TransformationID']
gLogger.info("-" * 40)
res = self.integrityClient.getTransformationProblematics(
int(transID))
if not res['OK']:
gLogger.error(
"Failed to determine waiting problematics for transformation",
res['Message'])
elif not res['Value']:
res = self.transClient.setTransformationParameter(
transID, 'Status', 'ValidatedOutput')
if not res['OK']:
gLogger.error(
"Failed to update status of transformation %s to ValidatedOutput"
% (transID))
else:
gLogger.info(
"Updated status of transformation %s to ValidatedOutput"
% (transID))
else:
gLogger.info(
"%d problematic files for transformation %s were found" %
(len(res['Value']), transID))
return
#############################################################################
#
# Get the transformation directories for checking
#
def getTransformationDirectories(self, transID):
""" Get the directories for the supplied transformation from the transformation system
"""
directories = []
if 'TransformationDB' in self.directoryLocations:
res = self.transClient.getTransformationParameters(
transID, ['OutputDirectories'])
if not res['OK']:
gLogger.error("Failed to obtain transformation directories",
res['Message'])
return res
if not isinstance(res['Value'], list):
transDirectories = ast.literal_eval(res['Value'])
else:
transDirectories = res['Value']
directories = self._addDirs(transID, transDirectories, directories)
if 'MetadataCatalog' in self.directoryLocations:
res = self.fileCatalogClient.findDirectoriesByMetadata(
{self.transfidmeta: transID})
if not res['OK']:
gLogger.error("Failed to obtain metadata catalog directories",
res['Message'])
return res
transDirectories = res['Value']
directories = self._addDirs(transID, transDirectories, directories)
if not directories:
gLogger.info("No output directories found")
directories = sorted(directories)
return S_OK(directories)
@staticmethod
def _addDirs(transID, newDirs, existingDirs):
for nDir in newDirs:
transStr = str(transID).zfill(8)
if re.search(transStr, nDir):
if nDir not in existingDirs:
existingDirs.append(nDir)
return existingDirs
#############################################################################
def checkTransformationIntegrity(self, transID):
""" This method contains the real work
"""
gLogger.info("-" * 40)
gLogger.info("Checking the integrity of transformation %s" % transID)
gLogger.info("-" * 40)
res = self.getTransformationDirectories(transID)
if not res['OK']:
return res
directories = res['Value']
if not directories:
return S_OK()
######################################################
#
# This check performs Catalog->SE for possible output directories
#
res = self.fc.exists(directories)
if not res['OK']:
gLogger.error('Failed to check directory existence',
res['Message'])
return res
for directory, error in res['Value']['Failed']:
gLogger.error('Failed to determine existance of directory',
'%s %s' % (directory, error))
if res['Value']['Failed']:
return S_ERROR("Failed to determine the existance of directories")
directoryExists = res['Value']['Successful']
for directory in sorted(directoryExists.keys()):
if not directoryExists[directory]:
continue
iRes = self.consistencyInspector.catalogDirectoryToSE(directory)
if not iRes['OK']:
gLogger.error(iRes['Message'])
return iRes
gLogger.info("-" * 40)
gLogger.info("Completed integrity check for transformation %s" %
transID)
return S_OK()
def finalizeCheck(self, transID):
""" Move to 'WaitingIntegrity' or 'ValidatedOutput'
"""
res = self.integrityClient.getTransformationProblematics(int(transID))
if not res['OK']:
gLogger.error(
"Failed to determine whether there were associated problematic files",
res['Message'])
newStatus = ''
elif res['Value']:
gLogger.info(
"%d problematic files for transformation %s were found" %
(len(res['Value']), transID))
newStatus = "WaitingIntegrity"
else:
gLogger.info("No problematics were found for transformation %s" %
transID)
newStatus = "ValidatedOutput"
if newStatus:
res = self.transClient.setTransformationParameter(
transID, 'Status', newStatus)
if not res['OK']:
gLogger.error(
"Failed to update status of transformation %s to %s" %
(transID, newStatus))
else:
gLogger.info("Updated status of transformation %s to %s" %
(transID, newStatus))
gLogger.info("-" * 40)
return S_OK()
class MoveReplica( DMSRequestOperationsBase ):
"""
.. class:: MoveReplica
MoveReplica operation handler
"""
def __init__( self, operation = None, csPath = None ):
"""c'tor
:param self: self reference
:param ~Operation.Operation operation: Operation instance
:param str csPath: CS path for this handler
"""
super( MoveReplica, self ).__init__( operation, csPath )
# # own gMonitor stuff for files
gMonitor.registerActivity( "ReplicateAndRegisterAtt", "Replicate and register attempted",
"RequestExecutingAgent", "Files/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "ReplicateOK", "Replications successful",
"RequestExecutingAgent", "Files/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "ReplicateFail", "Replications failed",
"RequestExecutingAgent", "Files/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "RegisterOK", "Registrations successful",
"RequestExecutingAgent", "Files/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "RegisterFail", "Registrations failed",
"RequestExecutingAgent", "Files/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "RemoveReplicaAtt", "Replica removals attempted",
"RequestExecutingAgent", "Files/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "RemoveReplicaOK", "Successful replica removals",
"RequestExecutingAgent", "Files/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "RemoveReplicaFail", "Failed replica removals",
"RequestExecutingAgent", "Files/min", gMonitor.OP_SUM )
# Init ConsistencyInspector: used to check replicas
self.ci = ConsistencyInspector()
def __call__( self ):
""" call me maybe """
# # check replicas first
res = self.__checkReplicas()
if not res["OK"]:
self.log.error( 'Failed to check replicas', res["Message"] )
sourceSE = self.operation.SourceSE if self.operation.SourceSE else None
if sourceSE:
# # check source se for read
bannedSource = self.checkSEsRSS( sourceSE, 'ReadAccess' )
if not bannedSource["OK"]:
gMonitor.addMark( "ReplicateAndRegisterAtt", len( self.operation ) )
gMonitor.addMark( "ReplicateFail", len( self.operation ) )
return bannedSource
if bannedSource["Value"]:
self.operation.Error = "SourceSE %s is banned for reading" % sourceSE
self.log.info( self.operation.Error )
return S_OK( self.operation.Error )
# # check targetSEs for write
bannedTargets = self.checkSEsRSS()
if not bannedTargets['OK']:
gMonitor.addMark( "ReplicateAndRegisterAtt", len( self.operation ) )
gMonitor.addMark( "ReplicateFail", len( self.operation ) )
return bannedTargets
if bannedTargets['Value']:
self.operation.Error = "%s targets are banned for writing" % ",".join( bannedTargets['Value'] )
return S_OK( self.operation.Error )
# Can continue now
self.log.verbose( "No targets banned for writing" )
# # check sourceSEs for removal
# # for removal the targetSEs are the sourceSEs of the replication
targetSEs = self.operation.sourceSEList
bannedTargets = self.checkSEsRSS( targetSEs, access = 'RemoveAccess' )
if not bannedTargets['OK']:
gMonitor.addMark( "RemoveReplicaAtt" )
gMonitor.addMark( "RemoveReplicaFail" )
return bannedTargets
if bannedTargets['Value']:
return S_OK( "%s targets are banned for removal" % ",".join( bannedTargets['Value'] ) )
# Can continue now
self.log.verbose( "No targets banned for removal" )
## Do the transfer
# # get waiting files. If none just return
waitingFiles = self.getWaitingFilesList()
if not waitingFiles:
return S_OK()
# # loop over files
self.log.info( "Transferring files using Data manager..." )
for opFile in waitingFiles:
res = self.dmTransfer(opFile)
if not res["OK"]:
continue
else:
## Do the replica removal
self.log.info( "Removing files using Data manager..." )
toRemoveDict = dict( [ ( opFile.LFN, opFile ) for opFile in waitingFiles ] )
self.log.info( "todo: %s replicas to delete from %s sites" % ( len( toRemoveDict ), len( targetSEs ) ) )
self.dmRemoval(toRemoveDict,targetSEs)
return S_OK()
def __checkReplicas( self ):
""" check done replicas and update file states """
waitingFiles = dict( [ ( opFile.LFN, opFile ) for opFile in self.operation
if opFile.Status in ( "Waiting", "Scheduled" ) ] )
targetSESet = set( self.operation.targetSEList )
# Check replicas
res = self.ci._getCatalogReplicas( waitingFiles.keys() )
if not res["OK"]:
self.log.error( 'Failed to get catalog replicas', res["Message"] )
return S_ERROR()
allReplicas = res['Value'][0]
replicas = self.ci.compareChecksum( waitingFiles.keys() )
if not replicas["OK"]:
self.log.error( 'Failed to check replicas', replicas["Message"] )
return S_ERROR()
replicas = replicas["Value"]
noReplicas = replicas['NoReplicas']
if noReplicas:
for lfn in noReplicas.keys():
self.log.error( "File %s doesn't exist" % lfn )
gMonitor.addMark( "ReplicateFail", len( targetSESet ) )
waitingFiles[lfn].Status = "Failed"
for lfn, reps in allReplicas.items():
if targetSESet.issubset( set( reps ) ):
self.log.info( "file %s has been replicated to all targets" % lfn )
waitingFiles[lfn].Status = "Done"
return S_OK()
def dmRemoval(self, toRemoveDict, targetSEs ):
gMonitor.addMark( "RemoveReplicaAtt", len( toRemoveDict ) * len( targetSEs ) )
# # keep status for each targetSE
removalStatus = dict.fromkeys( toRemoveDict.keys(), None )
for lfn in removalStatus:
removalStatus[lfn] = dict.fromkeys( targetSEs, None )
# # loop over targetSEs
for targetSE in targetSEs:
self.log.info( "removing replicas at %s" % targetSE )
# # 1st step - bulk removal
bulkRemoval = self.bulkRemoval( toRemoveDict, targetSE )
if not bulkRemoval["OK"]:
self.log.error( 'Bulk replica removal failed', bulkRemoval["Message"] )
return bulkRemoval
bulkRemoval = bulkRemoval["Value"]
# # update removal status for successful files
removalOK = [ opFile for opFile in bulkRemoval.values() if not opFile.Error ]
for opFile in removalOK:
removalStatus[opFile.LFN][targetSE] = ""
gMonitor.addMark( "RemoveReplicaOK", len( removalOK ) )
# # 2nd step - process the rest again
toRetry = dict( [ ( lfn, opFile ) for lfn, opFile in bulkRemoval.items() if opFile.Error ] )
for lfn, opFile in toRetry.items():
self.singleRemoval( opFile, targetSE )
if not opFile.Error:
gMonitor.addMark( "RemoveReplicaOK", 1 )
removalStatus[lfn][targetSE] = ""
else:
gMonitor.addMark( "RemoveReplicaFail", 1 )
removalStatus[lfn][targetSE] = opFile.Error
# # update file status for waiting files
failed = 0
for opFile in self.operation:
if opFile.Status == "Waiting":
errors = list( set( [ error for error in removalStatus[lfn].values() if error ] ) )
if errors:
opFile.Error = ",".join( errors )
# This seems to be the only offending error
if "Write access not permitted for this credential" in opFile.Error:
failed += 1
continue
opFile.Status = "Done"
if failed:
self.operation.Error = "failed to remove %s replicas" % failed
return S_OK(removalStatus)
def dmTransfer( self, opFile ):
""" replicate and register using dataManager """
# # get waiting files. If none just return
# # source SE
sourceSE = self.operation.SourceSE if self.operation.SourceSE else None
gMonitor.addMark( "ReplicateAndRegisterAtt", 1 )
opFile.Error = ''
lfn = opFile.LFN
# Check replicas
res = self.ci._getCatalogReplicas( [lfn] )
if not res["OK"]:
self.log.error( 'Failed to get catalog replicas', res["Message"] )
return S_ERROR()
allReplicas = res['Value'][0]
replicas = self.ci.compareChecksum( [lfn] )
if not replicas["OK"]:
self.log.error( 'Failed to check replicas', replicas["Message"] )
return S_ERROR()
replicas = replicas["Value"]
validReplicas = []
noReplicas = replicas['NoReplicas']
missingAllReplicas = replicas["MissingAllReplicas"]
missingReplica = replicas["MissingReplica"]
someReplicasCorrupted = replicas["SomeReplicasCorrupted"]
allReplicasCorrupted = replicas["AllReplicasCorrupted"]
if noReplicas:
gMonitor.addMark( "ReplicateFail" )
self.log.error( "Unable to replicate", "File %s doesn't exist" % ( lfn ) )
opFile.Error = 'No replicas found'
opFile.Status = 'Failed'
return S_ERROR()
elif missingAllReplicas:
gMonitor.addMark( "ReplicateFail" )
self.log.error( "Unable to replicate", "%s, all replicas are missing" % ( lfn ) )
opFile.Error = 'Missing all replicas'
opFile.Status = 'Failed'
return S_ERROR()
elif allReplicasCorrupted:
gMonitor.addMark( "ReplicateFail" )
self.log.error( "Unable to replicate", "%s, all replicas are corrupted" % ( lfn ) )
opFile.Error = 'All replicas corrupted'
opFile.Status = 'Failed'
return S_ERROR()
elif someReplicasCorrupted:
gMonitor.addMark( "ReplicateFail" )
self.log.error( "Unable to replicate", "%s, replicas corrupted at %s" % ( lfn, someReplicasCorrupted[lfn] ) )
opFile.Error = 'At least one replica corrupted'
opFile.Status = 'Failed'
return S_ERROR()
elif missingReplica:
gMonitor.addMark( "ReplicateFail" )
self.log.error( "Unable to replicate", "%s, missing replicas at %s" % ( lfn , missingReplica[lfn] ) )
opFile.Error = 'At least one missing replica'
opFile.Status = 'Failed'
return S_ERROR()
# Check if replica is at the specified source
for repSEName in allReplicas[lfn]:
validReplicas.append( repSEName )
# # get the first one in the list
if sourceSE not in validReplicas:
if sourceSE:
self.log.warn( "%s is not at specified sourceSE %s, changed to %s" % ( lfn, sourceSE, validReplicas[0] ) )
sourceSE = validReplicas[0]
# # loop over targetSE
catalogs = self.operation.Catalog
if catalogs:
catalogs = [ cat.strip() for cat in catalogs.split( ',' ) ]
for targetSE in self.operation.targetSEList:
# # call DataManager
if targetSE in validReplicas:
self.log.warn( "Request to replicate %s to an existing location: %s" % ( lfn, targetSE ) )
continue
res = self.dm.replicateAndRegister( lfn, targetSE, sourceSE = sourceSE, catalog = catalogs )
if res["OK"]:
if lfn in res["Value"]["Successful"]:
if "replicate" in res["Value"]["Successful"][lfn]:
repTime = res["Value"]["Successful"][lfn]["replicate"]
prString = "file %s replicated at %s in %s s." % ( lfn, targetSE, repTime )
gMonitor.addMark( "ReplicateOK", 1 )
if "register" in res["Value"]["Successful"][lfn]:
gMonitor.addMark( "RegisterOK", 1 )
regTime = res["Value"]["Successful"][lfn]["register"]
prString += ' and registered in %s s.' % regTime
self.log.info( prString )
else:
gMonitor.addMark( "RegisterFail", 1 )
prString += " but failed to register"
self.log.warn( prString )
opFile.Error = "Failed to register"
# # add register replica operation
registerOperation = self.getRegisterOperation( opFile, targetSE, type = 'RegisterReplica' )
self.request.insertAfter( registerOperation, self.operation )
else:
self.log.error( "Failed to replicate", "%s to %s" % ( lfn, targetSE ) )
gMonitor.addMark( "ReplicateFail", 1 )
opFile.Error = "Failed to replicate"
else:
gMonitor.addMark( "ReplicateFail", 1 )
reason = res["Value"]["Failed"][lfn]
self.log.error( "Failed to replicate and register", "File %s at %s: %s" % ( lfn, targetSE , reason ))
opFile.Error = reason
else:
gMonitor.addMark( "ReplicateFail", 1 )
opFile.Error = "DataManager error: %s" % res["Message"]
self.log.error( "DataManager error", res["Message"] )
if not opFile.Error:
if len( self.operation.targetSEList ) > 1:
self.log.info( "file %s has been replicated to all targetSEs" % lfn )
else:
return S_ERROR("dmTransfer failed")
return S_OK()
def bulkRemoval( self, toRemoveDict, targetSE ):
""" remove replicas :toRemoveDict: at :targetSE:
:param dict toRemoveDict: { lfn: opFile, ... }
:param str targetSE: target SE name
:return: toRemoveDict with updated errors
"""
removeReplicas = self.dm.removeReplica( targetSE, toRemoveDict.keys() )
if not removeReplicas["OK"]:
for opFile in toRemoveDict.values():
opFile.Error = removeReplicas["Message"]
return S_ERROR( removeReplicas["Message"] )
removeReplicas = removeReplicas["Value"]
# # filter out failed
for lfn, opFile in toRemoveDict.items():
if lfn in removeReplicas["Failed"]:
opFile.Error = str( removeReplicas["Failed"][lfn] )
return S_OK( toRemoveDict )
def singleRemoval( self, opFile, targetSE ):
""" remove opFile replica from targetSE
:param ~DIRAC.RequestManagementSystem.Client.File.File opFile: File instance
:param str targetSE: target SE name
"""
proxyFile = None
if "Write access not permitted for this credential" in opFile.Error:
# # not a DataManger? set status to failed and return
if "DataManager" in self.shifter:
# # you're a data manager - save current proxy and get a new one for LFN and retry
saveProxy = os.environ["X509_USER_PROXY"]
try:
fileProxy = self.getProxyForLFN( opFile.LFN )
if not fileProxy["OK"]:
opFile.Error = fileProxy["Message"]
else:
proxyFile = fileProxy["Value"]
removeReplica = self.dm.removeReplica( targetSE, opFile.LFN )
if not removeReplica["OK"]:
opFile.Error = removeReplica["Message"]
else:
removeReplica = removeReplica["Value"]
if opFile.LFN in removeReplica["Failed"]:
opFile.Error = removeReplica["Failed"][opFile.LFN]
else:
# # reset error - replica has been removed this time
opFile.Error = ""
finally:
if proxyFile:
os.unlink( proxyFile )
# # put back request owner proxy to env
os.environ["X509_USER_PROXY"] = saveProxy
return S_OK( opFile )
class ValidateOutputDataAgent(AgentModule):
def __init__(self, *args, **kwargs):
""" c'tor
"""
AgentModule.__init__(self, *args, **kwargs)
self.consistencyInspector = ConsistencyInspector()
self.integrityClient = DataIntegrityClient()
self.fc = FileCatalog()
self.transClient = TransformationClient()
self.fileCatalogClient = FileCatalogClient()
agentTSTypes = self.am_getOption('TransformationTypes', [])
if agentTSTypes:
self.transformationTypes = agentTSTypes
else:
self.transformationTypes = Operations().getValue('Transformations/DataProcessing', ['MCSimulation', 'Merge'])
self.directoryLocations = sorted(self.am_getOption('DirectoryLocations', ['TransformationDB',
'MetadataCatalog']))
self.transfidmeta = self.am_getOption('TransfIDMeta', "TransformationID")
self.enableFlag = True
#############################################################################
def initialize(self):
""" Sets defaults
"""
# This sets the Default Proxy to used as that defined under
# /Operations/Shifter/DataManager
# the shifterProxy option in the Configuration can be used to change this default.
self.am_setOption('shifterProxy', 'DataManager')
gLogger.info("Will treat the following transformation types: %s" % str(self.transformationTypes))
gLogger.info("Will search for directories in the following locations: %s" % str(self.directoryLocations))
gLogger.info("Will use %s as metadata tag name for TransformationID" % self.transfidmeta)
return S_OK()
#############################################################################
def execute(self):
""" The VerifyOutputData execution method
"""
self.enableFlag = self.am_getOption('EnableFlag', 'True')
if not self.enableFlag == 'True':
self.log.info("VerifyOutputData is disabled by configuration option 'EnableFlag'")
return S_OK('Disabled via CS flag')
gLogger.info("-" * 40)
self.updateWaitingIntegrity()
gLogger.info("-" * 40)
res = self.transClient.getTransformations({'Status': 'ValidatingOutput', 'Type': self.transformationTypes})
if not res['OK']:
gLogger.error("Failed to get ValidatingOutput transformations", res['Message'])
return res
transDicts = res['Value']
if not transDicts:
gLogger.info("No transformations found in ValidatingOutput status")
return S_OK()
gLogger.info("Found %s transformations in ValidatingOutput status" % len(transDicts))
for transDict in transDicts:
transID = transDict['TransformationID']
res = self.checkTransformationIntegrity(int(transID))
if not res['OK']:
gLogger.error("Failed to perform full integrity check for transformation %d" % transID)
else:
self.finalizeCheck(transID)
gLogger.info("-" * 40)
return S_OK()
def updateWaitingIntegrity(self):
""" Get 'WaitingIntegrity' transformations, update to 'ValidatedOutput'
"""
gLogger.info("Looking for transformations in the WaitingIntegrity status to update")
res = self.transClient.getTransformations({'Status': 'WaitingIntegrity'})
if not res['OK']:
gLogger.error("Failed to get WaitingIntegrity transformations", res['Message'])
return res
transDicts = res['Value']
if not transDicts:
gLogger.info("No transformations found in WaitingIntegrity status")
return S_OK()
gLogger.info("Found %s transformations in WaitingIntegrity status" % len(transDicts))
for transDict in transDicts:
transID = transDict['TransformationID']
gLogger.info("-" * 40)
res = self.integrityClient.getTransformationProblematics(int(transID))
if not res['OK']:
gLogger.error("Failed to determine waiting problematics for transformation", res['Message'])
elif not res['Value']:
res = self.transClient.setTransformationParameter(transID, 'Status', 'ValidatedOutput')
if not res['OK']:
gLogger.error("Failed to update status of transformation %s to ValidatedOutput" % (transID))
else:
gLogger.info("Updated status of transformation %s to ValidatedOutput" % (transID))
else:
gLogger.info("%d problematic files for transformation %s were found" % (len(res['Value']), transID))
return
#############################################################################
#
# Get the transformation directories for checking
#
def getTransformationDirectories(self, transID):
""" Get the directories for the supplied transformation from the transformation system
"""
directories = []
if 'TransformationDB' in self.directoryLocations:
res = self.transClient.getTransformationParameters(transID, ['OutputDirectories'])
if not res['OK']:
gLogger.error("Failed to obtain transformation directories", res['Message'])
return res
if not isinstance(res['Value'], list):
transDirectories = ast.literal_eval(res['Value'])
else:
transDirectories = res['Value']
directories = self._addDirs(transID, transDirectories, directories)
if 'MetadataCatalog' in self.directoryLocations:
res = self.fileCatalogClient.findDirectoriesByMetadata({self.transfidmeta: transID})
if not res['OK']:
gLogger.error("Failed to obtain metadata catalog directories", res['Message'])
return res
transDirectories = res['Value']
directories = self._addDirs(transID, transDirectories, directories)
if not directories:
gLogger.info("No output directories found")
directories = sorted(directories)
return S_OK(directories)
@staticmethod
def _addDirs(transID, newDirs, existingDirs):
for nDir in newDirs:
transStr = str(transID).zfill(8)
if re.search(transStr, nDir):
if nDir not in existingDirs:
existingDirs.append(nDir)
return existingDirs
#############################################################################
def checkTransformationIntegrity(self, transID):
""" This method contains the real work
"""
gLogger.info("-" * 40)
gLogger.info("Checking the integrity of transformation %s" % transID)
gLogger.info("-" * 40)
res = self.getTransformationDirectories(transID)
if not res['OK']:
return res
directories = res['Value']
if not directories:
return S_OK()
######################################################
#
# This check performs Catalog->SE for possible output directories
#
res = self.fc.exists(directories)
if not res['OK']:
gLogger.error('Failed to check directory existence', res['Message'])
return res
for directory, error in res['Value']['Failed']:
gLogger.error('Failed to determine existance of directory', '%s %s' % (directory, error))
if res['Value']['Failed']:
return S_ERROR("Failed to determine the existance of directories")
directoryExists = res['Value']['Successful']
for directory in sorted(directoryExists.keys()):
if not directoryExists[directory]:
continue
iRes = self.consistencyInspector.catalogDirectoryToSE(directory)
if not iRes['OK']:
gLogger.error(iRes['Message'])
return iRes
gLogger.info("-" * 40)
gLogger.info("Completed integrity check for transformation %s" % transID)
return S_OK()
def finalizeCheck(self, transID):
""" Move to 'WaitingIntegrity' or 'ValidatedOutput'
"""
res = self.integrityClient.getTransformationProblematics(int(transID))
if not res['OK']:
gLogger.error("Failed to determine whether there were associated problematic files", res['Message'])
newStatus = ''
elif res['Value']:
gLogger.info("%d problematic files for transformation %s were found" % (len(res['Value']), transID))
newStatus = "WaitingIntegrity"
else:
gLogger.info("No problematics were found for transformation %s" % transID)
newStatus = "ValidatedOutput"
if newStatus:
res = self.transClient.setTransformationParameter(transID, 'Status', newStatus)
if not res['OK']:
gLogger.error("Failed to update status of transformation %s to %s" % (transID, newStatus))
else:
gLogger.info("Updated status of transformation %s to %s" % (transID, newStatus))
gLogger.info("-" * 40)
return S_OK()
