def __init__( self, operation = None, csPath = None ): """c'tor :param self: self reference :param ~Operation.Operation operation: Operation instance :param str csPath: CS path for this handler """ super( MoveReplica, self ).__init__( operation, csPath ) # # own gMonitor stuff for files gMonitor.registerActivity( "ReplicateAndRegisterAtt", "Replicate and register attempted", "RequestExecutingAgent", "Files/min", gMonitor.OP_SUM ) gMonitor.registerActivity( "ReplicateOK", "Replications successful", "RequestExecutingAgent", "Files/min", gMonitor.OP_SUM ) gMonitor.registerActivity( "ReplicateFail", "Replications failed", "RequestExecutingAgent", "Files/min", gMonitor.OP_SUM ) gMonitor.registerActivity( "RegisterOK", "Registrations successful", "RequestExecutingAgent", "Files/min", gMonitor.OP_SUM ) gMonitor.registerActivity( "RegisterFail", "Registrations failed", "RequestExecutingAgent", "Files/min", gMonitor.OP_SUM ) gMonitor.registerActivity( "RemoveReplicaAtt", "Replica removals attempted", "RequestExecutingAgent", "Files/min", gMonitor.OP_SUM ) gMonitor.registerActivity( "RemoveReplicaOK", "Successful replica removals", "RequestExecutingAgent", "Files/min", gMonitor.OP_SUM ) gMonitor.registerActivity( "RemoveReplicaFail", "Failed replica removals", "RequestExecutingAgent", "Files/min", gMonitor.OP_SUM ) # Init ConsistencyInspector: used to check replicas self.ci = ConsistencyInspector()
def __init__(self, operation=None, csPath=None): """c'tor :param self: self reference :param ~Operation.Operation operation: Operation instance :param str csPath: CS path for this handler """ super(MoveReplica, self).__init__(operation, csPath) # Init ConsistencyInspector: used to check replicas self.ci = ConsistencyInspector()
def __init__(self, *args, **kwargs): """ c'tor """ AgentModule.__init__(self, *args, **kwargs) self.consistencyInspector = ConsistencyInspector() self.integrityClient = DataIntegrityClient() self.fc = FileCatalog() self.transClient = TransformationClient() self.fileCatalogClient = FileCatalogClient() agentTSTypes = self.am_getOption('TransformationTypes', []) if agentTSTypes: self.transformationTypes = agentTSTypes else: self.transformationTypes = Operations().getValue('Transformations/DataProcessing', ['MCSimulation', 'Merge']) self.directoryLocations = sorted(self.am_getOption('DirectoryLocations', ['TransformationDB', 'MetadataCatalog'])) self.transfidmeta = self.am_getOption('TransfIDMeta', "TransformationID") self.enableFlag = True
def setUp( self ): gLogger.setLevel('DEBUG') self.lfnDict = {'aa.raw': { 'aa.raw':{'FileType': 'RAW', 'RunNumber': 97019}, '/lhcb/1_2_1.Semileptonic.dst': {'FileType': 'SEMILEPTONIC.DST'}}, 'cc.raw': { 'cc.raw':{'FileType': 'RAW', 'RunNumber': 97019}, '/lhcb/1_1.semileptonic.dst': {'FileType': 'SEMILEPTONIC.DST'}} } dmMock = MagicMock() dicMock = MagicMock() self.ci = ConsistencyInspector( transClient = MagicMock(), dm = dmMock, fc = fileCatalogMock, dic = dicMock ) self.ci.fileType = ['SEMILEPTONIC.DST', 'LOG', 'RAW'] self.ci.fileTypesExcluded = ['LOG'] self.ci.prod = 0 self.maxDiff = None
def __init__(self, *args, **kwargs): """ c'tor """ AgentModule.__init__(self, *args, **kwargs) self.consistencyInspector = ConsistencyInspector() self.integrityClient = DataIntegrityClient() self.fc = FileCatalog() self.transClient = TransformationClient() self.fileCatalogClient = FileCatalogClient() agentTSTypes = self.am_getOption('TransformationTypes', []) if agentTSTypes: self.transformationTypes = agentTSTypes else: self.transformationTypes = Operations().getValue('Transformations/DataProcessing', ['MCSimulation', 'Merge']) self.directoryLocations = sorted(self.am_getOption('DirectoryLocations', ['TransformationDB', 'MetadataCatalog'])) self.transfidmeta = self.am_getOption('TransfIDMeta', "TransformationID") self.enableFlag = True
def setUp(self): gLogger.setLevel("DEBUG") self.lfnDict = { "aa.raw": { "aa.raw": { "FileType": "RAW", "RunNumber": 97019 }, "/lhcb/1_2_1.Semileptonic.dst": { "FileType": "SEMILEPTONIC.DST" }, }, "cc.raw": { "cc.raw": { "FileType": "RAW", "RunNumber": 97019 }, "/lhcb/1_1.semileptonic.dst": { "FileType": "SEMILEPTONIC.DST" }, }, } dmMock = MagicMock() dicMock = MagicMock() self.ci = ConsistencyInspector(transClient=MagicMock(), dm=dmMock, fc=fc_mock, dic=dicMock) self.ci.fileType = ["SEMILEPTONIC.DST", "LOG", "RAW"] self.ci.fileTypesExcluded = ["LOG"] self.ci.prod = 0 self.maxDiff = None
class ValidateOutputDataAgent(AgentModule): def __init__(self, *args, **kwargs): """ c'tor """ AgentModule.__init__(self, *args, **kwargs) self.consistencyInspector = ConsistencyInspector() self.integrityClient = DataIntegrityClient() self.fc = FileCatalog() self.transClient = TransformationClient() self.fileCatalogClient = FileCatalogClient() agentTSTypes = self.am_getOption('TransformationTypes', []) if agentTSTypes: self.transformationTypes = agentTSTypes else: self.transformationTypes = Operations().getValue( 'Transformations/DataProcessing', ['MCSimulation', 'Merge']) self.directoryLocations = sorted( self.am_getOption('DirectoryLocations', ['TransformationDB', 'MetadataCatalog'])) self.transfidmeta = self.am_getOption('TransfIDMeta', "TransformationID") self.enableFlag = True ############################################################################# def initialize(self): """ Sets defaults """ # This sets the Default Proxy to used as that defined under # /Operations/Shifter/DataManager # the shifterProxy option in the Configuration can be used to change this default. self.am_setOption('shifterProxy', 'DataManager') gLogger.info("Will treat the following transformation types: %s" % str(self.transformationTypes)) gLogger.info( "Will search for directories in the following locations: %s" % str(self.directoryLocations)) gLogger.info("Will use %s as metadata tag name for TransformationID" % self.transfidmeta) return S_OK() ############################################################################# def execute(self): """ The VerifyOutputData execution method """ self.enableFlag = self.am_getOption('EnableFlag', 'True') if not self.enableFlag == 'True': self.log.info( "VerifyOutputData is disabled by configuration option 'EnableFlag'" ) return S_OK('Disabled via CS flag') gLogger.info("-" * 40) self.updateWaitingIntegrity() gLogger.info("-" * 40) res = self.transClient.getTransformations({ 'Status': 'ValidatingOutput', 'Type': self.transformationTypes }) if not res['OK']: gLogger.error("Failed to get ValidatingOutput transformations", res['Message']) return res transDicts = res['Value'] if not transDicts: gLogger.info("No transformations found in ValidatingOutput status") return S_OK() gLogger.info("Found %s transformations in ValidatingOutput status" % len(transDicts)) for transDict in transDicts: transID = transDict['TransformationID'] res = self.checkTransformationIntegrity(int(transID)) if not res['OK']: gLogger.error( "Failed to perform full integrity check for transformation %d" % transID) else: self.finalizeCheck(transID) gLogger.info("-" * 40) return S_OK() def updateWaitingIntegrity(self): """ Get 'WaitingIntegrity' transformations, update to 'ValidatedOutput' """ gLogger.info( "Looking for transformations in the WaitingIntegrity status to update" ) res = self.transClient.getTransformations( {'Status': 'WaitingIntegrity'}) if not res['OK']: gLogger.error("Failed to get WaitingIntegrity transformations", res['Message']) return res transDicts = res['Value'] if not transDicts: gLogger.info("No transformations found in WaitingIntegrity status") return S_OK() gLogger.info("Found %s transformations in WaitingIntegrity status" % len(transDicts)) for transDict in transDicts: transID = transDict['TransformationID'] gLogger.info("-" * 40) res = self.integrityClient.getTransformationProblematics( int(transID)) if not res['OK']: gLogger.error( "Failed to determine waiting problematics for transformation", res['Message']) elif not res['Value']: res = self.transClient.setTransformationParameter( transID, 'Status', 'ValidatedOutput') if not res['OK']: gLogger.error( "Failed to update status of transformation %s to ValidatedOutput" % (transID)) else: gLogger.info( "Updated status of transformation %s to ValidatedOutput" % (transID)) else: gLogger.info( "%d problematic files for transformation %s were found" % (len(res['Value']), transID)) return ############################################################################# # # Get the transformation directories for checking # def getTransformationDirectories(self, transID): """ Get the directories for the supplied transformation from the transformation system """ directories = [] if 'TransformationDB' in self.directoryLocations: res = self.transClient.getTransformationParameters( transID, ['OutputDirectories']) if not res['OK']: gLogger.error("Failed to obtain transformation directories", res['Message']) return res if not isinstance(res['Value'], list): transDirectories = ast.literal_eval(res['Value']) else: transDirectories = res['Value'] directories = self._addDirs(transID, transDirectories, directories) if 'MetadataCatalog' in self.directoryLocations: res = self.fileCatalogClient.findDirectoriesByMetadata( {self.transfidmeta: transID}) if not res['OK']: gLogger.error("Failed to obtain metadata catalog directories", res['Message']) return res transDirectories = res['Value'] directories = self._addDirs(transID, transDirectories, directories) if not directories: gLogger.info("No output directories found") directories = sorted(directories) return S_OK(directories) @staticmethod def _addDirs(transID, newDirs, existingDirs): for nDir in newDirs: transStr = str(transID).zfill(8) if re.search(transStr, nDir): if nDir not in existingDirs: existingDirs.append(nDir) return existingDirs ############################################################################# def checkTransformationIntegrity(self, transID): """ This method contains the real work """ gLogger.info("-" * 40) gLogger.info("Checking the integrity of transformation %s" % transID) gLogger.info("-" * 40) res = self.getTransformationDirectories(transID) if not res['OK']: return res directories = res['Value'] if not directories: return S_OK() ###################################################### # # This check performs Catalog->SE for possible output directories # res = self.fc.exists(directories) if not res['OK']: gLogger.error('Failed to check directory existence', res['Message']) return res for directory, error in res['Value']['Failed']: gLogger.error('Failed to determine existance of directory', '%s %s' % (directory, error)) if res['Value']['Failed']: return S_ERROR("Failed to determine the existance of directories") directoryExists = res['Value']['Successful'] for directory in sorted(directoryExists.keys()): if not directoryExists[directory]: continue iRes = self.consistencyInspector.catalogDirectoryToSE(directory) if not iRes['OK']: gLogger.error(iRes['Message']) return iRes gLogger.info("-" * 40) gLogger.info("Completed integrity check for transformation %s" % transID) return S_OK() def finalizeCheck(self, transID): """ Move to 'WaitingIntegrity' or 'ValidatedOutput' """ res = self.integrityClient.getTransformationProblematics(int(transID)) if not res['OK']: gLogger.error( "Failed to determine whether there were associated problematic files", res['Message']) newStatus = '' elif res['Value']: gLogger.info( "%d problematic files for transformation %s were found" % (len(res['Value']), transID)) newStatus = "WaitingIntegrity" else: gLogger.info("No problematics were found for transformation %s" % transID) newStatus = "ValidatedOutput" if newStatus: res = self.transClient.setTransformationParameter( transID, 'Status', newStatus) if not res['OK']: gLogger.error( "Failed to update status of transformation %s to %s" % (transID, newStatus)) else: gLogger.info("Updated status of transformation %s to %s" % (transID, newStatus)) gLogger.info("-" * 40) return S_OK()
class MoveReplica( DMSRequestOperationsBase ): """ .. class:: MoveReplica MoveReplica operation handler """ def __init__( self, operation = None, csPath = None ): """c'tor :param self: self reference :param ~Operation.Operation operation: Operation instance :param str csPath: CS path for this handler """ super( MoveReplica, self ).__init__( operation, csPath ) # # own gMonitor stuff for files gMonitor.registerActivity( "ReplicateAndRegisterAtt", "Replicate and register attempted", "RequestExecutingAgent", "Files/min", gMonitor.OP_SUM ) gMonitor.registerActivity( "ReplicateOK", "Replications successful", "RequestExecutingAgent", "Files/min", gMonitor.OP_SUM ) gMonitor.registerActivity( "ReplicateFail", "Replications failed", "RequestExecutingAgent", "Files/min", gMonitor.OP_SUM ) gMonitor.registerActivity( "RegisterOK", "Registrations successful", "RequestExecutingAgent", "Files/min", gMonitor.OP_SUM ) gMonitor.registerActivity( "RegisterFail", "Registrations failed", "RequestExecutingAgent", "Files/min", gMonitor.OP_SUM ) gMonitor.registerActivity( "RemoveReplicaAtt", "Replica removals attempted", "RequestExecutingAgent", "Files/min", gMonitor.OP_SUM ) gMonitor.registerActivity( "RemoveReplicaOK", "Successful replica removals", "RequestExecutingAgent", "Files/min", gMonitor.OP_SUM ) gMonitor.registerActivity( "RemoveReplicaFail", "Failed replica removals", "RequestExecutingAgent", "Files/min", gMonitor.OP_SUM ) # Init ConsistencyInspector: used to check replicas self.ci = ConsistencyInspector() def __call__( self ): """ call me maybe """ # # check replicas first res = self.__checkReplicas() if not res["OK"]: self.log.error( 'Failed to check replicas', res["Message"] ) sourceSE = self.operation.SourceSE if self.operation.SourceSE else None if sourceSE: # # check source se for read bannedSource = self.checkSEsRSS( sourceSE, 'ReadAccess' ) if not bannedSource["OK"]: gMonitor.addMark( "ReplicateAndRegisterAtt", len( self.operation ) ) gMonitor.addMark( "ReplicateFail", len( self.operation ) ) return bannedSource if bannedSource["Value"]: self.operation.Error = "SourceSE %s is banned for reading" % sourceSE self.log.info( self.operation.Error ) return S_OK( self.operation.Error ) # # check targetSEs for write bannedTargets = self.checkSEsRSS() if not bannedTargets['OK']: gMonitor.addMark( "ReplicateAndRegisterAtt", len( self.operation ) ) gMonitor.addMark( "ReplicateFail", len( self.operation ) ) return bannedTargets if bannedTargets['Value']: self.operation.Error = "%s targets are banned for writing" % ",".join( bannedTargets['Value'] ) return S_OK( self.operation.Error ) # Can continue now self.log.verbose( "No targets banned for writing" ) # # check sourceSEs for removal # # for removal the targetSEs are the sourceSEs of the replication targetSEs = self.operation.sourceSEList bannedTargets = self.checkSEsRSS( targetSEs, access = 'RemoveAccess' ) if not bannedTargets['OK']: gMonitor.addMark( "RemoveReplicaAtt" ) gMonitor.addMark( "RemoveReplicaFail" ) return bannedTargets if bannedTargets['Value']: return S_OK( "%s targets are banned for removal" % ",".join( bannedTargets['Value'] ) ) # Can continue now self.log.verbose( "No targets banned for removal" ) ## Do the transfer # # get waiting files. If none just return waitingFiles = self.getWaitingFilesList() if not waitingFiles: return S_OK() # # loop over files self.log.info( "Transferring files using Data manager..." ) for opFile in waitingFiles: res = self.dmTransfer(opFile) if not res["OK"]: continue else: ## Do the replica removal self.log.info( "Removing files using Data manager..." ) toRemoveDict = dict( [ ( opFile.LFN, opFile ) for opFile in waitingFiles ] ) self.log.info( "todo: %s replicas to delete from %s sites" % ( len( toRemoveDict ), len( targetSEs ) ) ) self.dmRemoval(toRemoveDict,targetSEs) return S_OK() def __checkReplicas( self ): """ check done replicas and update file states """ waitingFiles = dict( [ ( opFile.LFN, opFile ) for opFile in self.operation if opFile.Status in ( "Waiting", "Scheduled" ) ] ) targetSESet = set( self.operation.targetSEList ) # Check replicas res = self.ci._getCatalogReplicas( waitingFiles.keys() ) if not res["OK"]: self.log.error( 'Failed to get catalog replicas', res["Message"] ) return S_ERROR() allReplicas = res['Value'][0] replicas = self.ci.compareChecksum( waitingFiles.keys() ) if not replicas["OK"]: self.log.error( 'Failed to check replicas', replicas["Message"] ) return S_ERROR() replicas = replicas["Value"] noReplicas = replicas['NoReplicas'] if noReplicas: for lfn in noReplicas.keys(): self.log.error( "File %s doesn't exist" % lfn ) gMonitor.addMark( "ReplicateFail", len( targetSESet ) ) waitingFiles[lfn].Status = "Failed" for lfn, reps in allReplicas.items(): if targetSESet.issubset( set( reps ) ): self.log.info( "file %s has been replicated to all targets" % lfn ) waitingFiles[lfn].Status = "Done" return S_OK() def dmRemoval(self, toRemoveDict, targetSEs ): gMonitor.addMark( "RemoveReplicaAtt", len( toRemoveDict ) * len( targetSEs ) ) # # keep status for each targetSE removalStatus = dict.fromkeys( toRemoveDict.keys(), None ) for lfn in removalStatus: removalStatus[lfn] = dict.fromkeys( targetSEs, None ) # # loop over targetSEs for targetSE in targetSEs: self.log.info( "removing replicas at %s" % targetSE ) # # 1st step - bulk removal bulkRemoval = self.bulkRemoval( toRemoveDict, targetSE ) if not bulkRemoval["OK"]: self.log.error( 'Bulk replica removal failed', bulkRemoval["Message"] ) return bulkRemoval bulkRemoval = bulkRemoval["Value"] # # update removal status for successful files removalOK = [ opFile for opFile in bulkRemoval.values() if not opFile.Error ] for opFile in removalOK: removalStatus[opFile.LFN][targetSE] = "" gMonitor.addMark( "RemoveReplicaOK", len( removalOK ) ) # # 2nd step - process the rest again toRetry = dict( [ ( lfn, opFile ) for lfn, opFile in bulkRemoval.items() if opFile.Error ] ) for lfn, opFile in toRetry.items(): self.singleRemoval( opFile, targetSE ) if not opFile.Error: gMonitor.addMark( "RemoveReplicaOK", 1 ) removalStatus[lfn][targetSE] = "" else: gMonitor.addMark( "RemoveReplicaFail", 1 ) removalStatus[lfn][targetSE] = opFile.Error # # update file status for waiting files failed = 0 for opFile in self.operation: if opFile.Status == "Waiting": errors = list( set( [ error for error in removalStatus[lfn].values() if error ] ) ) if errors: opFile.Error = ",".join( errors ) # This seems to be the only offending error if "Write access not permitted for this credential" in opFile.Error: failed += 1 continue opFile.Status = "Done" if failed: self.operation.Error = "failed to remove %s replicas" % failed return S_OK(removalStatus) def dmTransfer( self, opFile ): """ replicate and register using dataManager """ # # get waiting files. If none just return # # source SE sourceSE = self.operation.SourceSE if self.operation.SourceSE else None gMonitor.addMark( "ReplicateAndRegisterAtt", 1 ) opFile.Error = '' lfn = opFile.LFN # Check replicas res = self.ci._getCatalogReplicas( [lfn] ) if not res["OK"]: self.log.error( 'Failed to get catalog replicas', res["Message"] ) return S_ERROR() allReplicas = res['Value'][0] replicas = self.ci.compareChecksum( [lfn] ) if not replicas["OK"]: self.log.error( 'Failed to check replicas', replicas["Message"] ) return S_ERROR() replicas = replicas["Value"] validReplicas = [] noReplicas = replicas['NoReplicas'] missingAllReplicas = replicas["MissingAllReplicas"] missingReplica = replicas["MissingReplica"] someReplicasCorrupted = replicas["SomeReplicasCorrupted"] allReplicasCorrupted = replicas["AllReplicasCorrupted"] if noReplicas: gMonitor.addMark( "ReplicateFail" ) self.log.error( "Unable to replicate", "File %s doesn't exist" % ( lfn ) ) opFile.Error = 'No replicas found' opFile.Status = 'Failed' return S_ERROR() elif missingAllReplicas: gMonitor.addMark( "ReplicateFail" ) self.log.error( "Unable to replicate", "%s, all replicas are missing" % ( lfn ) ) opFile.Error = 'Missing all replicas' opFile.Status = 'Failed' return S_ERROR() elif allReplicasCorrupted: gMonitor.addMark( "ReplicateFail" ) self.log.error( "Unable to replicate", "%s, all replicas are corrupted" % ( lfn ) ) opFile.Error = 'All replicas corrupted' opFile.Status = 'Failed' return S_ERROR() elif someReplicasCorrupted: gMonitor.addMark( "ReplicateFail" ) self.log.error( "Unable to replicate", "%s, replicas corrupted at %s" % ( lfn, someReplicasCorrupted[lfn] ) ) opFile.Error = 'At least one replica corrupted' opFile.Status = 'Failed' return S_ERROR() elif missingReplica: gMonitor.addMark( "ReplicateFail" ) self.log.error( "Unable to replicate", "%s, missing replicas at %s" % ( lfn , missingReplica[lfn] ) ) opFile.Error = 'At least one missing replica' opFile.Status = 'Failed' return S_ERROR() # Check if replica is at the specified source for repSEName in allReplicas[lfn]: validReplicas.append( repSEName ) # # get the first one in the list if sourceSE not in validReplicas: if sourceSE: self.log.warn( "%s is not at specified sourceSE %s, changed to %s" % ( lfn, sourceSE, validReplicas[0] ) ) sourceSE = validReplicas[0] # # loop over targetSE catalogs = self.operation.Catalog if catalogs: catalogs = [ cat.strip() for cat in catalogs.split( ',' ) ] for targetSE in self.operation.targetSEList: # # call DataManager if targetSE in validReplicas: self.log.warn( "Request to replicate %s to an existing location: %s" % ( lfn, targetSE ) ) continue res = self.dm.replicateAndRegister( lfn, targetSE, sourceSE = sourceSE, catalog = catalogs ) if res["OK"]: if lfn in res["Value"]["Successful"]: if "replicate" in res["Value"]["Successful"][lfn]: repTime = res["Value"]["Successful"][lfn]["replicate"] prString = "file %s replicated at %s in %s s." % ( lfn, targetSE, repTime ) gMonitor.addMark( "ReplicateOK", 1 ) if "register" in res["Value"]["Successful"][lfn]: gMonitor.addMark( "RegisterOK", 1 ) regTime = res["Value"]["Successful"][lfn]["register"] prString += ' and registered in %s s.' % regTime self.log.info( prString ) else: gMonitor.addMark( "RegisterFail", 1 ) prString += " but failed to register" self.log.warn( prString ) opFile.Error = "Failed to register" # # add register replica operation registerOperation = self.getRegisterOperation( opFile, targetSE, type = 'RegisterReplica' ) self.request.insertAfter( registerOperation, self.operation ) else: self.log.error( "Failed to replicate", "%s to %s" % ( lfn, targetSE ) ) gMonitor.addMark( "ReplicateFail", 1 ) opFile.Error = "Failed to replicate" else: gMonitor.addMark( "ReplicateFail", 1 ) reason = res["Value"]["Failed"][lfn] self.log.error( "Failed to replicate and register", "File %s at %s: %s" % ( lfn, targetSE , reason )) opFile.Error = reason else: gMonitor.addMark( "ReplicateFail", 1 ) opFile.Error = "DataManager error: %s" % res["Message"] self.log.error( "DataManager error", res["Message"] ) if not opFile.Error: if len( self.operation.targetSEList ) > 1: self.log.info( "file %s has been replicated to all targetSEs" % lfn ) else: return S_ERROR("dmTransfer failed") return S_OK() def bulkRemoval( self, toRemoveDict, targetSE ): """ remove replicas :toRemoveDict: at :targetSE: :param dict toRemoveDict: { lfn: opFile, ... } :param str targetSE: target SE name :return: toRemoveDict with updated errors """ removeReplicas = self.dm.removeReplica( targetSE, toRemoveDict.keys() ) if not removeReplicas["OK"]: for opFile in toRemoveDict.values(): opFile.Error = removeReplicas["Message"] return S_ERROR( removeReplicas["Message"] ) removeReplicas = removeReplicas["Value"] # # filter out failed for lfn, opFile in toRemoveDict.items(): if lfn in removeReplicas["Failed"]: opFile.Error = str( removeReplicas["Failed"][lfn] ) return S_OK( toRemoveDict ) def singleRemoval( self, opFile, targetSE ): """ remove opFile replica from targetSE :param ~DIRAC.RequestManagementSystem.Client.File.File opFile: File instance :param str targetSE: target SE name """ proxyFile = None if "Write access not permitted for this credential" in opFile.Error: # # not a DataManger? set status to failed and return if "DataManager" in self.shifter: # # you're a data manager - save current proxy and get a new one for LFN and retry saveProxy = os.environ["X509_USER_PROXY"] try: fileProxy = self.getProxyForLFN( opFile.LFN ) if not fileProxy["OK"]: opFile.Error = fileProxy["Message"] else: proxyFile = fileProxy["Value"] removeReplica = self.dm.removeReplica( targetSE, opFile.LFN ) if not removeReplica["OK"]: opFile.Error = removeReplica["Message"] else: removeReplica = removeReplica["Value"] if opFile.LFN in removeReplica["Failed"]: opFile.Error = removeReplica["Failed"][opFile.LFN] else: # # reset error - replica has been removed this time opFile.Error = "" finally: if proxyFile: os.unlink( proxyFile ) # # put back request owner proxy to env os.environ["X509_USER_PROXY"] = saveProxy return S_OK( opFile )
class ValidateOutputDataAgent(AgentModule): def __init__(self, *args, **kwargs): """ c'tor """ AgentModule.__init__(self, *args, **kwargs) self.consistencyInspector = ConsistencyInspector() self.integrityClient = DataIntegrityClient() self.fc = FileCatalog() self.transClient = TransformationClient() self.fileCatalogClient = FileCatalogClient() agentTSTypes = self.am_getOption('TransformationTypes', []) if agentTSTypes: self.transformationTypes = agentTSTypes else: self.transformationTypes = Operations().getValue('Transformations/DataProcessing', ['MCSimulation', 'Merge']) self.directoryLocations = sorted(self.am_getOption('DirectoryLocations', ['TransformationDB', 'MetadataCatalog'])) self.transfidmeta = self.am_getOption('TransfIDMeta', "TransformationID") self.enableFlag = True ############################################################################# def initialize(self): """ Sets defaults """ # This sets the Default Proxy to used as that defined under # /Operations/Shifter/DataManager # the shifterProxy option in the Configuration can be used to change this default. self.am_setOption('shifterProxy', 'DataManager') gLogger.info("Will treat the following transformation types: %s" % str(self.transformationTypes)) gLogger.info("Will search for directories in the following locations: %s" % str(self.directoryLocations)) gLogger.info("Will use %s as metadata tag name for TransformationID" % self.transfidmeta) return S_OK() ############################################################################# def execute(self): """ The VerifyOutputData execution method """ self.enableFlag = self.am_getOption('EnableFlag', 'True') if not self.enableFlag == 'True': self.log.info("VerifyOutputData is disabled by configuration option 'EnableFlag'") return S_OK('Disabled via CS flag') gLogger.info("-" * 40) self.updateWaitingIntegrity() gLogger.info("-" * 40) res = self.transClient.getTransformations({'Status': 'ValidatingOutput', 'Type': self.transformationTypes}) if not res['OK']: gLogger.error("Failed to get ValidatingOutput transformations", res['Message']) return res transDicts = res['Value'] if not transDicts: gLogger.info("No transformations found in ValidatingOutput status") return S_OK() gLogger.info("Found %s transformations in ValidatingOutput status" % len(transDicts)) for transDict in transDicts: transID = transDict['TransformationID'] res = self.checkTransformationIntegrity(int(transID)) if not res['OK']: gLogger.error("Failed to perform full integrity check for transformation %d" % transID) else: self.finalizeCheck(transID) gLogger.info("-" * 40) return S_OK() def updateWaitingIntegrity(self): """ Get 'WaitingIntegrity' transformations, update to 'ValidatedOutput' """ gLogger.info("Looking for transformations in the WaitingIntegrity status to update") res = self.transClient.getTransformations({'Status': 'WaitingIntegrity'}) if not res['OK']: gLogger.error("Failed to get WaitingIntegrity transformations", res['Message']) return res transDicts = res['Value'] if not transDicts: gLogger.info("No transformations found in WaitingIntegrity status") return S_OK() gLogger.info("Found %s transformations in WaitingIntegrity status" % len(transDicts)) for transDict in transDicts: transID = transDict['TransformationID'] gLogger.info("-" * 40) res = self.integrityClient.getTransformationProblematics(int(transID)) if not res['OK']: gLogger.error("Failed to determine waiting problematics for transformation", res['Message']) elif not res['Value']: res = self.transClient.setTransformationParameter(transID, 'Status', 'ValidatedOutput') if not res['OK']: gLogger.error("Failed to update status of transformation %s to ValidatedOutput" % (transID)) else: gLogger.info("Updated status of transformation %s to ValidatedOutput" % (transID)) else: gLogger.info("%d problematic files for transformation %s were found" % (len(res['Value']), transID)) return ############################################################################# # # Get the transformation directories for checking # def getTransformationDirectories(self, transID): """ Get the directories for the supplied transformation from the transformation system """ directories = [] if 'TransformationDB' in self.directoryLocations: res = self.transClient.getTransformationParameters(transID, ['OutputDirectories']) if not res['OK']: gLogger.error("Failed to obtain transformation directories", res['Message']) return res if not isinstance(res['Value'], list): transDirectories = ast.literal_eval(res['Value']) else: transDirectories = res['Value'] directories = self._addDirs(transID, transDirectories, directories) if 'MetadataCatalog' in self.directoryLocations: res = self.fileCatalogClient.findDirectoriesByMetadata({self.transfidmeta: transID}) if not res['OK']: gLogger.error("Failed to obtain metadata catalog directories", res['Message']) return res transDirectories = res['Value'] directories = self._addDirs(transID, transDirectories, directories) if not directories: gLogger.info("No output directories found") directories = sorted(directories) return S_OK(directories) @staticmethod def _addDirs(transID, newDirs, existingDirs): for nDir in newDirs: transStr = str(transID).zfill(8) if re.search(transStr, nDir): if nDir not in existingDirs: existingDirs.append(nDir) return existingDirs ############################################################################# def checkTransformationIntegrity(self, transID): """ This method contains the real work """ gLogger.info("-" * 40) gLogger.info("Checking the integrity of transformation %s" % transID) gLogger.info("-" * 40) res = self.getTransformationDirectories(transID) if not res['OK']: return res directories = res['Value'] if not directories: return S_OK() ###################################################### # # This check performs Catalog->SE for possible output directories # res = self.fc.exists(directories) if not res['OK']: gLogger.error('Failed to check directory existence', res['Message']) return res for directory, error in res['Value']['Failed']: gLogger.error('Failed to determine existance of directory', '%s %s' % (directory, error)) if res['Value']['Failed']: return S_ERROR("Failed to determine the existance of directories") directoryExists = res['Value']['Successful'] for directory in sorted(directoryExists.keys()): if not directoryExists[directory]: continue iRes = self.consistencyInspector.catalogDirectoryToSE(directory) if not iRes['OK']: gLogger.error(iRes['Message']) return iRes gLogger.info("-" * 40) gLogger.info("Completed integrity check for transformation %s" % transID) return S_OK() def finalizeCheck(self, transID): """ Move to 'WaitingIntegrity' or 'ValidatedOutput' """ res = self.integrityClient.getTransformationProblematics(int(transID)) if not res['OK']: gLogger.error("Failed to determine whether there were associated problematic files", res['Message']) newStatus = '' elif res['Value']: gLogger.info("%d problematic files for transformation %s were found" % (len(res['Value']), transID)) newStatus = "WaitingIntegrity" else: gLogger.info("No problematics were found for transformation %s" % transID) newStatus = "ValidatedOutput" if newStatus: res = self.transClient.setTransformationParameter(transID, 'Status', newStatus) if not res['OK']: gLogger.error("Failed to update status of transformation %s to %s" % (transID, newStatus)) else: gLogger.info("Updated status of transformation %s to %s" % (transID, newStatus)) gLogger.info("-" * 40) return S_OK()