def buildRequest(self, owner, group, sourceSE, targetSE1, targetSE2): files = self.files(owner, group) putAndRegister = Operation() putAndRegister.Type = "PutAndRegister" putAndRegister.TargetSE = sourceSE for fname, lfn, size, checksum, guid in files: putFile = File() putFile.LFN = lfn putFile.PFN = fname putFile.Checksum = checksum putFile.ChecksumType = "adler32" putFile.Size = size putFile.GUID = guid putAndRegister.addFile(putFile) replicateAndRegister = Operation() replicateAndRegister.Type = "ReplicateAndRegister" replicateAndRegister.TargetSE = "%s,%s" % (targetSE1, targetSE2) for fname, lfn, size, checksum, guid in files: repFile = File() repFile.LFN = lfn repFile.Size = size repFile.Checksum = checksum repFile.ChecksumType = "adler32" replicateAndRegister.addFile(repFile) removeReplica = Operation() removeReplica.Type = "RemoveReplica" removeReplica.TargetSE = sourceSE for fname, lfn, size, checksum, guid in files: removeReplica.addFile(File({"LFN": lfn})) removeFile = Operation() removeFile.Type = "RemoveFile" for fname, lfn, size, checksum, guid in files: removeFile.addFile(File({"LFN": lfn})) removeFileInit = Operation() removeFileInit.Type = "RemoveFile" for fname, lfn, size, checksum, guid in files: removeFileInit.addFile(File({"LFN": lfn})) req = Request() req.addOperation(removeFileInit) req.addOperation(putAndRegister) req.addOperation(replicateAndRegister) req.addOperation(removeReplica) req.addOperation(removeFile) return req
def buildRequest( self, owner, group, sourceSE, targetSE1, targetSE2 ): files = self.files( owner, group ) putAndRegister = Operation() putAndRegister.Type = "PutAndRegister" putAndRegister.TargetSE = sourceSE for fname, lfn, size, checksum, guid in files: putFile = File() putFile.LFN = lfn putFile.PFN = fname putFile.Checksum = checksum putFile.ChecksumType = "adler32" putFile.Size = size putFile.GUID = guid putAndRegister.addFile( putFile ) replicateAndRegister = Operation() replicateAndRegister.Type = "ReplicateAndRegister" replicateAndRegister.TargetSE = "%s,%s" % ( targetSE1, targetSE2 ) for fname, lfn, size, checksum, guid in files: repFile = File() repFile.LFN = lfn repFile.Size = size repFile.Checksum = checksum repFile.ChecksumType = "adler32" replicateAndRegister.addFile( repFile ) removeReplica = Operation() removeReplica.Type = "RemoveReplica" removeReplica.TargetSE = sourceSE for fname, lfn, size, checksum, guid in files: removeReplica.addFile( File( {"LFN": lfn } ) ) removeFile = Operation() removeFile.Type = "RemoveFile" for fname, lfn, size, checksum, guid in files: removeFile.addFile( File( {"LFN": lfn } ) ) removeFileInit = Operation() removeFileInit.Type = "RemoveFile" for fname, lfn, size, checksum, guid in files: removeFileInit.addFile( File( {"LFN": lfn } ) ) req = Request() req.addOperation( removeFileInit ) req.addOperation( putAndRegister ) req.addOperation( replicateAndRegister ) req.addOperation( removeReplica ) req.addOperation( removeFile ) return req
def test__addMetadataToFiles(self): resMeta = { "OK": True, "Value": { "Failed": {}, "Successful": { "/lhcb/1.dst": { "ChecksumType": "AD", "Checksum": "123456", "CreationDate": datetime.datetime(2013, 12, 11, 20, 20, 21), "GUID": "92F9CE97-7A62-E311-8401-0025907FD430", "Mode": 436, "ModificationDate": datetime.datetime(2013, 12, 11, 20, 20, 21), "NumberOfLinks": 1, "Size": 5846023777, "Status": "-", }, "/lhcb/2.dst": { "ChecksumType": "AD", "Checksum": "987654", "CreationDate": datetime.datetime(2013, 12, 12, 6, 26, 52), "GUID": "DAE4933A-C162-E311-8A6B-003048FEAF04", "Mode": 436, "ModificationDate": datetime.datetime(2013, 12, 12, 6, 26, 52), "NumberOfLinks": 1, "Size": 5893396937, "Status": "-", }, }, }, } self.rr.fc.getFileMetadata.return_value = resMeta file1 = File() file1.LFN = "/lhcb/1.dst" file2 = File() file2.LFN = "/lhcb/2.dst" toSchedule = {"/lhcb/1.dst": [file1, ["SE1"], ["SE2", "SE3"]], "/lhcb/2.dst": [file2, ["SE4"], ["SE5", "SE6"]]} res = self.rr._addMetadataToFiles(toSchedule) self.assertTrue(res["OK"]) for lfn in toSchedule: self.assertEqual(res["Value"][lfn].LFN, lfn) for attr in ("GUID", "Size", "Checksum"): self.assertEqual(getattr(res["Value"][lfn], attr), resMeta["Value"]["Successful"][lfn][attr]) # AD should be transformed into Adler32 self.assertEqual(res["Value"][lfn].ChecksumType, "ADLER32")
def __insertRegisterOperation( self, request, operation, toRegister ): """ add RegisterReplica operation :param Request request: request instance :param Operation transferOp: 'ReplicateAndRegister' operation for this FTSJob :param list toRegister: [ FTSDB.FTSFile, ... ] - files that failed to register """ log = self.log.getSubLogger( "req_%s/%s/registerFiles" % ( request.RequestID, request.RequestName ) ) byTarget = {} for ftsFile in toRegister: if ftsFile.TargetSE not in byTarget: byTarget.setdefault( ftsFile.TargetSE, [] ) byTarget[ftsFile.TargetSE].append( ftsFile ) log.info( "will create %s 'RegisterReplica' operations" % len( byTarget ) ) for target, ftsFileList in byTarget.iteritems(): log.info( "creating 'RegisterReplica' operation for targetSE %s with %s files..." % ( target, len( ftsFileList ) ) ) registerOperation = Operation() registerOperation.Type = "RegisterReplica" registerOperation.Status = "Waiting" registerOperation.TargetSE = target targetSE = StorageElement( target ) for ftsFile in ftsFileList: opFile = File() opFile.LFN = ftsFile.LFN pfn = returnSingleResult( targetSE.getURL( ftsFile.LFN, protocol = self.registrationProtocols ) ) if not pfn["OK"]: continue opFile.PFN = pfn["Value"] registerOperation.addFile( opFile ) request.insertBefore( registerOperation, operation ) return S_OK()
def _setFileReplicationRequest(self, lfn, targetSE, fileMetaDict, sourceSE=""): """Sets a registration request.""" self.log.info("Setting ReplicateAndRegister request", "for %s to %s" % (lfn, targetSE)) transfer = Operation() transfer.Type = "ReplicateAndRegister" transfer.TargetSE = targetSE if sourceSE: transfer.SourceSE = sourceSE trFile = File() trFile.LFN = lfn cksm = fileMetaDict.get("Checksum", None) cksmType = fileMetaDict.get("ChecksumType", self.defaultChecksumType) if cksm and cksmType: trFile.Checksum = cksm trFile.ChecksumType = cksmType size = fileMetaDict.get("Size", 0) if size: trFile.Size = size guid = fileMetaDict.get("GUID", "") if guid: trFile.GUID = guid transfer.addFile(trFile) self.request.addOperation(transfer) return S_OK()
def myRequest(): """Create a request and put it to the db""" request = Request() request.RequestName = 'myAwesomeRemovalRequest.xml' request.JobID = 0 request.SourceComponent = "myScript" remove = Operation() remove.Type = "RemoveFile" lfn = "/ilc/user/s/sailer/test.txt" rmFile = File() rmFile.LFN = lfn remove.addFile( rmFile ) request.addOperation( remove ) isValid = RequestValidator().validate( request ) if not isValid['OK']: raise RuntimeError( "Failover request is not valid: %s" % isValid['Message'] ) else: print("It is a GOGOGO") requestClient = ReqClient() result = requestClient.putRequest( request ) print(result)
def __setRegistrationRequest( self, lfn, targetSE, fileDict, catalog ): """ Sets a registration request :param str lfn: LFN :param list se: list of SE :param list catalog: list of catalogs to use :param dict fileDict: file metadata """ self.log.info( 'Setting registration request for %s at %s.' % ( lfn, targetSE ) ) for cat in catalog: register = Operation() register.Type = "RegisterFile" register.Catalog = cat register.TargetSE = targetSE regFile = File() regFile.LFN = lfn regFile.Checksum = fileDict.get( "Checksum", "" ) regFile.ChecksumType = fileDict.get( "ChecksumType", "" ) regFile.Size = fileDict.get( "Size", 0 ) regFile.GUID = fileDict.get( "GUID", "" ) se = StorageElement( targetSE ) pfn = se.getPfnForLfn( lfn ) if not pfn["OK"]: self.log.error( "unable to get PFN for LFN: %s" % pfn["Message"] ) return pfn regFile.PFN = pfn["Value"] register.addFile( regFile ) self.request.addOperation( register ) return S_OK()
def setBKRegistrationRequest(self, lfn, error='', metaData={'Checksum': 'justSomething', 'ChecksumType': 'ADLER32', 'GUID': 'aGUID'}): """ Set a BK registration request for changing the replica flag. Uses the global request object (self.request). """ if error: self.log.info('BK registration for %s failed with message: "%s" setting failover request' % (lfn, error)) else: self.log.info('Setting BK registration request for %s' % (lfn)) regFile = Operation() regFile.Type = 'RegisterFile' regFile.Catalog = 'BookkeepingDB' bkFile = File() bkFile.LFN = lfn # this should NOT be needed... but RMS complains! bkFile.PFN = lfn bkFile.GUID = metaData['GUID'] bkFile.Checksum = metaData['Checksum'] bkFile.ChecksumType = metaData['ChecksumType'] regFile.addFile(bkFile) res = self.request.addOperation(regFile) if not res['OK']: raise RuntimeError(res['Message'])
def myRequest(): """Create a request and put it to the db""" request = Request() request.RequestName = 'myAwesomeRemovalRequest.xml' request.JobID = 0 request.SourceComponent = "myScript" remove = Operation() remove.Type = "RemoveFile" lfn = "/ilc/user/s/sailer/test.txt" rmFile = File() rmFile.LFN = lfn remove.addFile(rmFile) request.addOperation(remove) isValid = RequestValidator().validate(request) if not isValid['OK']: raise RuntimeError("Failover request is not valid: %s" % isValid['Message']) else: print "It is a GOGOGO" requestClient = ReqClient() result = requestClient.putRequest(request) print result
def __setFileReplicationRequest( self, lfn, targetSE, fileMetaDict, sourceSE = '' ): """ Sets a registration request. """ self.log.info( 'Setting replication request for %s to %s' % ( lfn, targetSE ) ) transfer = Operation() transfer.Type = "ReplicateAndRegister" transfer.TargetSE = targetSE if sourceSE: transfer.SourceSE = sourceSE trFile = File() trFile.LFN = lfn cksm = fileMetaDict.get( "Checksum", None ) cksmType = fileMetaDict.get( "ChecksumType", None ) if cksm and cksmType: trFile.Checksum = cksm trFile.ChecksumType = cksmType size = fileMetaDict.get( "Size", 0 ) if size: trFile.Size = size guid = fileMetaDict.get( "GUID", "" ) if guid: trFile.GUID = guid transfer.addFile( trFile ) self.request.addOperation( transfer ) return S_OK()
def archiveRequestAndOp(listOfLFNs): """Return a tuple of the request and operation.""" req = Request() req.RequestName = "MyRequest" op = Operation() switches = {} archiveLFN = "/vo/tars/myTar.tar" op.Arguments = DEncode.encode({ "SourceSE": switches.get("SourceSE", "SOURCE-SE"), "TarballSE": switches.get("TarballSE", "TARBALL-SE"), "RegisterDescendent": False, "ArchiveLFN": archiveLFN, }) op.Type = "ArchiveFiles" for index, lfn in enumerate(listOfLFNs): oFile = File() oFile.LFN = lfn oFile.Size = index oFile.Checksum = "01130a%0d" % index oFile.ChecksumType = "adler32" op.addFile(oFile) req.addOperation(op) return req, op
def getRegisterOperation(self, opFile, targetSE, type='RegisterFile', catalog=None): """ add RegisterReplica operation for file :param File opFile: operation file :param str targetSE: target SE """ # # add RegisterReplica operation registerOperation = Operation() registerOperation.Type = type registerOperation.TargetSE = targetSE if catalog: registerOperation.Catalog = catalog registerFile = File() registerFile.LFN = opFile.LFN registerFile.PFN = StorageElement(targetSE).getPfnForLfn( opFile.LFN).get('Value', {}).get('Successful', {}).get(opFile.LFN) registerFile.GUID = opFile.GUID registerFile.Checksum = opFile.Checksum registerFile.ChecksumType = opFile.ChecksumType registerFile.Size = opFile.Size registerOperation.addFile(registerFile) return registerOperation
def _createLogUploadRequest(self, targetSE, logFileLFN, uploadedSE): """ Set a request to upload job log files from the output sandbox Changed to be similar to LHCb createLogUploadRequest using LHCb LogUpload Request and Removal Request """ self.log.info('Setting log upload request for %s at %s' % (targetSE, logFileLFN)) request = self._getRequestContainer() logUpload = Operation() logUpload.Type = "LogUpload" logUpload.TargetSE = targetSE upFile = File() upFile.LFN = logFileLFN logUpload.addFile(upFile) logRemoval = Operation() logRemoval.Type = 'RemoveFile' logRemoval.TargetSE = uploadedSE logRemoval.addFile(upFile) request.addOperation(logUpload) request.addOperation(logRemoval) self.workflow_commons['Request'] = request return S_OK()
def __submitRMSOp(self, target_se, lfns_chunk_dict, whichRMSOp='ReplicateAndRegister' ): """ target_se : SE name to which to replicate lfns_chunk_dict : LFNS dict with 100 lfns as key andeEach lfn has 'Size', 'Checksum' whichRMSOp: Choose from RMP operation - ReplicateAndRegister, ReplicateAndRemove, PutAndRegister """ ## Setup request request = Request() request.RequestName = "DDM_"+ str(target_se) + datetime.datetime.now().strftime("_%Y%m%d_%H%M%S") myOp = Operation() myOp.Type = whichRMSOp myOp.TargetSE = target_se ## Add LFNS to operations for lfn in lfns_chunk_dict.keys(): opFile = File() opFile.LFN = lfn opFile.Size = lfns_chunk_dict[lfn]['Size'] if "Checksum" in lfns_chunk_dict[lfn]: opFile.Checksum = lfns_chunk_dict[lfn]['Checksum'] opFile.ChecksumType = 'ADLER32' ## Add file to operation myOp.addFile( opFile ) request.addOperation( myOp ) reqClient = ReqClient() putRequest = reqClient.putRequest( request ) if not putRequest["OK"]: gLogger.error( "Unable to put request '%s': %s" % ( request.RequestName, putRequest["Message"] ) ) return S_ERROR("Problem submitting to RMS.")
def getRegisterOperation(self, opFile, targetSE, type="RegisterFile", catalog=None): """add RegisterReplica operation for file :param ~DIRAC.RequestManagementSystem.Client.File.File opFile: operation file :param str targetSE: target SE """ # # add RegisterReplica operation registerOperation = Operation() registerOperation.Type = type registerOperation.TargetSE = targetSE if catalog: registerOperation.Catalog = catalog registerFile = File() registerFile.LFN = opFile.LFN registerFile.PFN = ( StorageElement(targetSE) .getURL(opFile.LFN, protocol=self.registrationProtocols) .get("Value", {}) .get("Successful", {}) .get(opFile.LFN) ) registerFile.GUID = opFile.GUID registerFile.Checksum = opFile.Checksum registerFile.ChecksumType = opFile.ChecksumType registerFile.Size = opFile.Size registerOperation.addFile(registerFile) return registerOperation
def archiveRequestAndOp(listOfLFNs): """Return a tuple of the request and operation.""" req = Request() req.RequestName = 'MyRequest' op = Operation() switches = {} archiveLFN = '/vo/tars/myTar.tar' op.Arguments = DEncode.encode({ 'SourceSE': switches.get('SourceSE', 'SOURCE-SE'), 'TarballSE': switches.get('TarballSE', 'TARBALL-SE'), 'RegisterDescendent': False, 'ArchiveLFN': archiveLFN }) op.Type = 'ArchiveFiles' for index, lfn in enumerate(listOfLFNs): oFile = File() oFile.LFN = lfn oFile.Size = index oFile.Checksum = '01130a%0d' % index oFile.ChecksumType = 'adler32' op.addFile(oFile) req.addOperation(op) return req, op
def _createLogUploadRequest(self, targetSE, logFileLFN, uploadedSE): """ Set a request to upload job log files from the output sandbox Changed to be similar to LHCb createLogUploadRequest using LHCb LogUpload Request and Removal Request """ self.log.info('Setting log upload request for %s at %s' %(targetSE, logFileLFN)) request = self._getRequestContainer() logUpload = Operation() logUpload.Type = "LogUpload" logUpload.TargetSE = targetSE upFile = File() upFile.LFN = logFileLFN logUpload.addFile( upFile ) logRemoval = Operation() logRemoval.Type = 'RemoveFile' logRemoval.TargetSE = uploadedSE logRemoval.addFile( upFile ) request.addOperation ( logUpload ) request.addOperation ( logRemoval ) self.workflow_commons['Request'] = request return S_OK()
def prepareTransformationTasks( self, transBody, taskDict, owner = '', ownerGroup = '', ownerDN = '' ): """ Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB """ if ( not owner ) or ( not ownerGroup ): res = getProxyInfo( False, False ) if not res['OK']: return res proxyInfo = res['Value'] owner = proxyInfo['username'] ownerGroup = proxyInfo['group'] if not ownerDN: res = getDNForUsername( owner ) if not res['OK']: return res ownerDN = res['Value'][0] requestOperation = 'ReplicateAndRegister' if transBody: try: _requestType, requestOperation = transBody.split( ';' ) except AttributeError: pass for taskID in sorted( taskDict ): paramDict = taskDict[taskID] if paramDict['InputData']: transID = paramDict['TransformationID'] oRequest = Request() transfer = Operation() transfer.Type = requestOperation transfer.TargetSE = paramDict['TargetSE'] if isinstance( paramDict['InputData'], list ): files = paramDict['InputData'] elif isinstance( paramDict['InputData'], basestring ): files = paramDict['InputData'].split( ';' ) for lfn in files: trFile = File() trFile.LFN = lfn transfer.addFile( trFile ) oRequest.addOperation( transfer ) oRequest.RequestName = _requestName( transID, taskID ) oRequest.OwnerDN = ownerDN oRequest.OwnerGroup = ownerGroup isValid = self.requestValidator.validate( oRequest ) if not isValid['OK']: return isValid taskDict[taskID]['TaskObject'] = oRequest return S_OK( taskDict )
def __setFileRemovalRequest(self, lfn): """ Sets a removal request for a file including all replicas. """ self.log.info('Setting file removal request for %s' % lfn) removeFile = Operation() removeFile.Type = 'RemoveFile' rmFile = File() rmFile.LFN = lfn removeFile.addFile(rmFile) self.request.addOperation(removeFile)
def _multiOperationsBody(self, transJson, taskDict, ownerDN, ownerGroup): """ deal with a Request that has multiple operations :param transJson: list of lists of string and dictionaries, e.g.: .. code :: python body = [ ( "ReplicateAndRegister", { "SourceSE":"FOO-SRM", "TargetSE":"BAR-SRM" }), ( "RemoveReplica", { "TargetSE":"FOO-SRM" } ), ] :param dict taskDict: dictionary of tasks, modified in this function :param str ownerDN: certificate DN used for the requests :param str onwerGroup: dirac group used for the requests :returns: None """ failedTasks = [] for taskID, task in taskDict.items(): transID = task['TransformationID'] if not task.get('InputData'): self._logError("Error creating request for task", "%s, No input data" % taskID, transID=transID) taskDict.pop(taskID) continue files = [] oRequest = Request() if isinstance(task['InputData'], list): files = task['InputData'] elif isinstance(task['InputData'], basestring): files = task['InputData'].split(';') # create the operations from the json structure for operationTuple in transJson: op = Operation() op.Type = operationTuple[0] for parameter, value in operationTuple[1].iteritems(): setattr(op, parameter, value) for lfn in files: opFile = File() opFile.LFN = lfn op.addFile(opFile) oRequest.addOperation(op) result = self._assignRequestToTask(oRequest, taskDict, transID, taskID, ownerDN, ownerGroup) if not result['OK']: failedTasks.append(taskID) # Remove failed tasks for taskID in failedTasks: taskDict.pop(taskID)
def test__addMetadataToFiles( self ): resMeta = {'OK': True, 'Value': {'Failed': {}, 'Successful': {'/lhcb/1.dst': {'ChecksumType': 'AD', 'Checksum': '123456', 'CreationDate': datetime.datetime( 2013, 12, 11, 20, 20, 21 ), 'GUID': '92F9CE97-7A62-E311-8401-0025907FD430', 'Mode': 436, 'ModificationDate': datetime.datetime( 2013, 12, 11, 20, 20, 21 ), 'NumberOfLinks': 1, 'Size': 5846023777, 'Status': '-'}, '/lhcb/2.dst': {'ChecksumType': 'AD', 'Checksum': '987654', 'CreationDate': datetime.datetime( 2013, 12, 12, 6, 26, 52 ), 'GUID': 'DAE4933A-C162-E311-8A6B-003048FEAF04', 'Mode': 436, 'ModificationDate': datetime.datetime( 2013, 12, 12, 6, 26, 52 ), 'NumberOfLinks': 1, 'Size': 5893396937, 'Status': '-'}}}} self.rr.fc.getFileMetadata.return_value = resMeta file1 = File() file1.LFN = '/lhcb/1.dst' file2 = File() file2.LFN = '/lhcb/2.dst' toSchedule = {'/lhcb/1.dst': [file1, ['SE1'], ['SE2', 'SE3']], '/lhcb/2.dst': [file2, ['SE4'], ['SE5', 'SE6']]} res = self.rr._addMetadataToFiles( toSchedule ) self.assertTrue(res['OK']) for lfn in toSchedule: self.assertEqual( res['Value'][lfn].LFN, lfn ) for attr in ('GUID', 'Size', 'Checksum'): self.assertEqual( getattr(res['Value'][lfn],attr), resMeta['Value']['Successful'][lfn][attr] ) # AD should be transformed into Adler32 self.assertEqual( res['Value'][lfn].ChecksumType, "ADLER32" )
def test__addMetadataToFiles( self ): resMeta = {'OK': True, 'Value': {'Failed': {}, 'Successful': {'/lhcb/1.dst': {'ChecksumType': 'AD', 'Checksum': '123456', 'CreationDate': datetime.datetime( 2013, 12, 11, 20, 20, 21 ), 'GUID': '92F9CE97-7A62-E311-8401-0025907FD430', 'Mode': 436, 'ModificationDate': datetime.datetime( 2013, 12, 11, 20, 20, 21 ), 'NumberOfLinks': 1, 'Size': 5846023777, 'Status': '-'}, '/lhcb/2.dst': {'ChecksumType': 'AD', 'Checksum': '987654', 'CreationDate': datetime.datetime( 2013, 12, 12, 6, 26, 52 ), 'GUID': 'DAE4933A-C162-E311-8A6B-003048FEAF04', 'Mode': 436, 'ModificationDate': datetime.datetime( 2013, 12, 12, 6, 26, 52 ), 'NumberOfLinks': 1, 'Size': 5893396937, 'Status': '-'}}}} self.rr.fc.getFileMetadata.return_value = resMeta file1 = File() file1.LFN = '/lhcb/1.dst' file2 = File() file2.LFN = '/lhcb/2.dst' toSchedule = {'/lhcb/1.dst': [file1, ['SE1'], ['SE2', 'SE3']], '/lhcb/2.dst': [file2, ['SE4'], ['SE5', 'SE6']]} res = self.rr._addMetadataToFiles( toSchedule ) self.assert_( res['OK'] ) self.assertEqual( json.loads( res['Value'][0][0] )['LFN'], resMeta['Value']['Successful'].keys()[0] ) self.assertEqual( json.loads( res['Value'][0][0] )['Size'], resMeta['Value']['Successful'].values()[0]['Size'] ) self.assertEqual( json.loads( res['Value'][1][0] )['LFN'], resMeta['Value']['Successful'].keys()[1] ) self.assertEqual( json.loads( res['Value'][1][0] )['Size'], resMeta['Value']['Successful'].values()[1]['Size'] )
def addRemovalRequests(self, lfnList): """Create removalRequests for lfns in lfnList and add it to the common request""" request = self._getRequestContainer() remove = Operation() remove.Type = "RemoveFile" for lfn in lfnList: rmFile = File() rmFile.LFN = lfn remove.addFile( rmFile ) request.addOperation( remove ) self.workflow_commons['Request'] = request
def _setFileRemovalRequest(self, lfn, se="", pfn=""): """Sets a removal request for a file including all replicas.""" remove = Operation() remove.Type = "RemoveFile" if se: remove.TargetSE = se rmFile = File() rmFile.LFN = lfn if pfn: rmFile.PFN = pfn remove.addFile(rmFile) self.request.addOperation(remove) return S_OK()
def checkRequestAndOp(listOfLFNs): req = Request() req.RequestName = 'MyRequest' op = Operation() op.Type = 'CheckMigration' for index, lfn in enumerate(listOfLFNs): oFile = File() oFile.LFN = lfn oFile.Size = index oFile.Checksum = '01130a%0d' % index oFile.ChecksumType = 'adler32' op.addFile(oFile) req.addOperation(op) return req, op
def __setFileRemovalRequest( self, lfn, se = '', pfn = '' ): """ Sets a removal request for a file including all replicas. """ remove = Operation() remove.Type = "RemoveFile" if se: remove.TargetSE = se rmFile = File() rmFile.LFN = lfn if pfn: rmFile.PFN = pfn remove.addFile( rmFile ) self.request.addOperation( remove ) return S_OK()
def test__getLFNsForBKRegistration(mocker): mocker.patch("LHCbDIRAC.Workflow.Modules.ModuleBase.RequestValidator", side_effect=MagicMock()) f1 = File() f1.LFN = '/a/1.txt' f2 = File() f2.LFN = '/a/2.txt' f3 = File() f3.LFN = '/a/3.txt' o1 = Operation() o1.Type = 'RegisterFile' o1.addFile(f1) o2 = Operation() o2.Type = 'RegisterFile' o2.addFile(f2) o3 = Operation() o3.Type = 'ForwardDISET' o4 = Operation() o4.Type = 'RegisterFile' o4.addFile(f1) o4.addFile(f3) r = Request() r.addOperation(o4) r.addOperation(o1) r.addOperation(o2) r.addOperation(o3) uod = UploadOutputData(bkClient=bkc_mock, dm=dm_mock) uod.request = r lfns = set(['/a/1.txt', '/a/5.txt', '/a/6.txt', '/a/3.txt']) lfnsForBkReg = uod._getLFNsForBKRegistration(lfns) assert sorted(lfnsForBkReg) == sorted(['/a/5.txt', '/a/6.txt'])
def checkRequestAndOp(listOfLFNs): req = Request() req.RequestName = "MyRequest" op = Operation() op.Type = "CheckMigration" op.TargetSE = "Foo-SE" for index, lfn in enumerate(listOfLFNs): oFile = File() oFile.LFN = lfn oFile.Size = index oFile.Checksum = "01130a%0d" % index oFile.ChecksumType = "adler32" op.addFile(oFile) req.addOperation(op) return req, op
def _singleOperationsBody(self, transBody, taskDict, ownerDN, ownerGroup): """deal with a Request that has just one operation, as it was sofar :param transBody: string, can be an empty string :param dict taskDict: dictionary of tasks, modified in this function :param str ownerDN: certificate DN used for the requests :param str onwerGroup: dirac group used for the requests :returns: None """ requestOperation = "ReplicateAndRegister" if transBody: try: _requestType, requestOperation = transBody.split(";") except AttributeError: pass failedTasks = [] # Do not remove sorted, we might pop elements in the loop for taskID, task in taskDict.items(): transID = task["TransformationID"] oRequest = Request() transfer = Operation() transfer.Type = requestOperation transfer.TargetSE = task["TargetSE"] # If there are input files if task.get("InputData"): if isinstance(task["InputData"], list): files = task["InputData"] elif isinstance(task["InputData"], six.string_types): files = task["InputData"].split(";") for lfn in files: trFile = File() trFile.LFN = lfn transfer.addFile(trFile) oRequest.addOperation(transfer) result = self._assignRequestToTask(oRequest, taskDict, transID, taskID, ownerDN, ownerGroup) if not result["OK"]: failedTasks.append(taskID) # Remove failed tasks for taskID in failedTasks: taskDict.pop(taskID)
def _singleOperationsBody(self, transBody, taskDict, ownerDN, ownerGroup): """ deal with a Request that has just one operation, as it was sofar :param transBody: string, can be an empty string :param dict taskDict: dictionary of tasks, modified in this function :param str ownerDN: certificate DN used for the requests :param str onwerGroup: dirac group used for the requests :returns: None """ requestOperation = 'ReplicateAndRegister' if transBody: try: _requestType, requestOperation = transBody.split(';') except AttributeError: pass failedTasks = [] # Do not remove sorted, we might pop elements in the loop for taskID, task in taskDict.iteritems(): transID = task['TransformationID'] oRequest = Request() transfer = Operation() transfer.Type = requestOperation transfer.TargetSE = task['TargetSE'] # If there are input files if task.get('InputData'): if isinstance(task['InputData'], list): files = task['InputData'] elif isinstance(task['InputData'], basestring): files = task['InputData'].split(';') for lfn in files: trFile = File() trFile.LFN = lfn transfer.addFile(trFile) oRequest.addOperation(transfer) result = self._assignRequestToTask(oRequest, taskDict, transID, taskID, ownerDN, ownerGroup) if not result['OK']: failedTasks.append(taskID) # Remove failed tasks for taskID in failedTasks: taskDict.pop(taskID)
def __setRemovalRequest( self, lfn, ownerDN, ownerGroup ): """ Set removal request with the given credentials """ oRequest = Request() oRequest.OwnerDN = ownerDN oRequest.OwnerGroup = ownerGroup oRequest.RequestName = os.path.basename( lfn ).strip() + '_removal_request.xml' oRequest.SourceComponent = 'JobCleaningAgent' removeFile = Operation() removeFile.Type = 'RemoveFile' removedFile = File() removedFile.LFN = lfn removeFile.addFile( removedFile ) oRequest.addOperation( removeFile ) return ReqClient().putRequest( oRequest )
def _setRegistrationRequest(self, lfn, targetSE, fileDict, catalog): """ Sets a registration request :param str lfn: LFN :param list se: list of SE (or just string) :param list catalog: list (or string) of catalogs to use :param dict fileDict: file metadata """ self.log.info('Setting registration request for %s at %s.' % (lfn, targetSE)) if not type(catalog) == type([]): catalog = [catalog] for cat in catalog: register = Operation() register.Type = "RegisterFile" register.Catalog = cat register.TargetSE = targetSE regFile = File() regFile.LFN = lfn regFile.Checksum = fileDict.get("Checksum", "") regFile.ChecksumType = fileDict.get("ChecksumType", self.defaultChecksumType) regFile.Size = fileDict.get("Size", 0) regFile.GUID = fileDict.get("GUID", "") se = StorageElement(targetSE) pfn = se.getPfnForLfn(lfn) if not pfn["OK"] or lfn not in pfn["Value"]['Successful']: self.log.error( "unable to get PFN for LFN: %s" % pfn.get('Message', pfn.get('Value', {}).get('Failed', {}).get(lfn))) return pfn regFile.PFN = pfn["Value"]['Successful'][lfn] register.addFile(regFile) self.request.addOperation(register) return S_OK()
def prepareTransformationTasks( self, transBody, taskDict, owner = '', ownerGroup = '' ): """ Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB """ requestOperation = 'ReplicateAndRegister' if transBody: try: _requestType, requestOperation = transBody.split( ';' ) except AttributeError: pass for taskID in sorted( taskDict ): paramDict = taskDict[taskID] if paramDict['InputData']: transID = paramDict['TransformationID'] oRequest = Request() transfer = Operation() transfer.Type = requestOperation transfer.TargetSE = paramDict['TargetSE'] if type( paramDict['InputData'] ) == type( [] ): files = paramDict['InputData'] elif type( paramDict['InputData'] ) == type( '' ): files = paramDict['InputData'].split( ';' ) for lfn in files: trFile = File() trFile.LFN = lfn transfer.addFile( trFile ) oRequest.addOperation( transfer ) oRequest.RequestName = str( transID ).zfill( 8 ) + '_' + str( taskID ).zfill( 8 ) oRequest.OwnerDN = owner oRequest.OwnerGroup = ownerGroup isValid = gRequestValidator.validate( oRequest ) if not isValid['OK']: return isValid taskDict[taskID]['TaskObject'] = oRequest return S_OK( taskDict )
def _singleOperationsBody(self, transBody, taskDict, ownerDN, ownerGroup ): """ deal with a Request that has just one operation, as it was sofar :param transBody: string, can be an empty string :param dict taskDict: dictionary of tasks, modified in this function :param str ownerDN: certificate DN used for the requests :param str onwerGroup: dirac group used for the requests :returns: None """ requestOperation = 'ReplicateAndRegister' if transBody: try: _requestType, requestOperation = transBody.split( ';' ) except AttributeError: pass # Do not remove sorted, we might pop elements in the loop for taskID in sorted( taskDict ): paramDict = taskDict[taskID] transID = paramDict['TransformationID'] oRequest = Request() transfer = Operation() transfer.Type = requestOperation transfer.TargetSE = paramDict['TargetSE'] # If there are input files if paramDict.get('InputData'): if isinstance( paramDict['InputData'], list ): files = paramDict['InputData'] elif isinstance( paramDict['InputData'], basestring ): files = paramDict['InputData'].split( ';' ) for lfn in files: trFile = File() trFile.LFN = lfn transfer.addFile( trFile ) oRequest.addOperation( transfer ) self._assignRequestToTask( oRequest, taskDict, transID, taskID, ownerDN, ownerGroup )
def _setRegistrationRequest(self, lfn, targetSE, fileDict, catalog): """Sets a registration request :param str lfn: LFN :param list se: list of SE (or just string) :param list catalog: list (or string) of catalogs to use :param dict fileDict: file metadata """ self.log.info("Setting registration request", "for %s at %s." % (lfn, targetSE)) if not isinstance(catalog, list): catalog = [catalog] for cat in catalog: register = Operation() register.Type = "RegisterFile" register.Catalog = cat register.TargetSE = targetSE regFile = File() regFile.LFN = lfn regFile.Checksum = fileDict.get("Checksum", "") regFile.ChecksumType = fileDict.get("ChecksumType", self.defaultChecksumType) regFile.Size = fileDict.get("Size", 0) regFile.GUID = fileDict.get("GUID", "") se = StorageElement(targetSE) res = returnSingleResult(se.getURL(lfn, self.registrationProtocols)) if not res["OK"]: self.log.error("Unable to get PFN for LFN", res["Message"]) return res regFile.PFN = res["Value"] register.addFile(regFile) self.request.addOperation(register) return S_OK()
def test_valid_properties(): theFile = File() theFile.FileID = 1 assert theFile.FileID == 1 theFile.Status = "Done" assert theFile.Status == "Done" theFile.LFN = "/some/path/somewhere" assert theFile.LFN == "/some/path/somewhere" theFile.PFN = "/some/path/somewhere" assert theFile.PFN == "/some/path/somewhere" theFile.Attempt = 1 assert theFile.Attempt == 1 theFile.Size = 1 assert theFile.Size == 1 theFile.GUID = "2bbabe80-e2f1-11e1-9b23-0800200c9a66" assert theFile.GUID == "2bbabe80-e2f1-11e1-9b23-0800200c9a66" theFile.ChecksumType = "adler32" assert theFile.ChecksumType == "ADLER32" theFile.Checksum = "123456" assert theFile.Checksum == "123456"
def _setReplicaRemovalRequest(self, lfn, se): """Sets a removal request for a replica. :param str lfn: LFN :param se: """ if isinstance(se, str): se = ",".join([se.strip() for se in se.split(",") if se.strip()]) removeReplica = Operation() removeReplica.Type = "RemoveReplica" removeReplica.TargetSE = se replicaToRemove = File() replicaToRemove.LFN = lfn removeReplica.addFile(replicaToRemove) self.request.addOperation(removeReplica) return S_OK()
def __setReplicaRemovalRequest( self, lfn, se ): """ Sets a removal request for a replica. :param str lfn: LFN :param se: """ if type( se ) == str: se = ",".join( [ se.strip() for se in se.split( "," ) if se.strip() ] ) removeReplica = Operation() removeReplica.Type = "RemoveReplica" removeReplica.TargetSE = se replicaToRemove = File() replicaToRemove.LFN = lfn removeReplica.addFile( replicaToRemove ) self.request.addOperation( removeReplica ) return S_OK()
def __createLogUploadRequest(self, targetSE, logFileLFN, uploadedSE): """ Set a request to upload job log files from the output sandbox """ self.log.info('Setting log upload request for %s at %s' % (logFileLFN, targetSE)) logUpload = Operation() logUpload.Type = 'LogUpload' logUpload.TargetSE = targetSE logFile = File() logFile.LFN = logFileLFN logUpload.addFile(logFile) self.request.addOperation(logUpload) logRemoval = Operation() logRemoval.Type = 'RemoveFile' logRemoval.TargetSE = uploadedSE logRemoval.addFile(logFile) self.request.addOperation(logRemoval)
def getRegisterOperation( self, opFile, targetSE ): """ add RegisterReplica operation for file :param File opFile: operation file :param str targetSE: target SE """ # # add RegisterReplica operation registerOperation = Operation() registerOperation.Type = "RegisterFile" registerOperation.TargetSE = targetSE registerFile = File() registerFile.LFN = opFile.LFN registerFile.PFN = opFile.PFN registerFile.GUID = opFile.GUID registerFile.Checksum = opFile.Checksum registerFile.ChecksumType = opFile.ChecksumType registerFile.Size = opFile.Size registerOperation.addFile( registerFile ) return registerOperation
def addRegisterReplica( self, opFile, targetSE ): """ add RegisterReplica operation for file :param File opFile: operation file :param str targetSE: target SE """ # # add RegisterReplica operation registerOperation = Operation() registerOperation.Type = "RegisterFile" registerOperation.TargetSE = targetSE registerFile = File() registerFile.LFN = opFile.LFN registerFile.PFN = opFile.PFN registerFile.GUID = opFile.GUID registerFile.Checksum = opFile.Checksum registerFile.ChecksumType = opFile.ChecksumType registerFile.Size = opFile.Size registerOperation.addFile( registerFile ) self.request.insertAfter( registerOperation, self.operation ) return S_OK()
def _setRegistrationRequest( self, lfn, targetSE, fileDict, catalog ): """ Sets a registration request :param str lfn: LFN :param list se: list of SE (or just string) :param list catalog: list (or string) of catalogs to use :param dict fileDict: file metadata """ self.log.info( 'Setting registration request for %s at %s.' % ( lfn, targetSE ) ) if not isinstance( catalog, list ): catalog = [catalog] for cat in catalog: register = Operation() register.Type = "RegisterFile" register.Catalog = cat register.TargetSE = targetSE regFile = File() regFile.LFN = lfn regFile.Checksum = fileDict.get( "Checksum", "" ) regFile.ChecksumType = fileDict.get( "ChecksumType", self.defaultChecksumType ) regFile.Size = fileDict.get( "Size", 0 ) regFile.GUID = fileDict.get( "GUID", "" ) se = StorageElement( targetSE ) pfn = se.getURL( lfn, self.registrationProtocols ) if not pfn["OK"] or lfn not in pfn["Value"]['Successful']: self.log.error( "Unable to get PFN for LFN", "%s" % pfn.get( 'Message', pfn.get( 'Value', {} ).get( 'Failed', {} ).get( lfn ) ) ) return pfn regFile.PFN = pfn["Value"]['Successful'][lfn] register.addFile( regFile ) self.request.addOperation( register ) return S_OK()
def getRegisterOperation( self, opFile, targetSE, type = 'RegisterFile', catalog = None ): """ add RegisterReplica operation for file :param File opFile: operation file :param str targetSE: target SE """ # # add RegisterReplica operation registerOperation = Operation() registerOperation.Type = type registerOperation.TargetSE = targetSE if catalog: registerOperation.Catalog = catalog registerFile = File() registerFile.LFN = opFile.LFN registerFile.PFN = StorageElement( targetSE ).getURL( opFile.LFN, protocol = self.registrationProtocols ).get( 'Value', {} ).get( 'Successful', {} ).get( opFile.LFN ) registerFile.GUID = opFile.GUID registerFile.Checksum = opFile.Checksum registerFile.ChecksumType = opFile.ChecksumType registerFile.Size = opFile.Size registerOperation.addFile( registerFile ) return registerOperation
def test02props( self ): """ test props and attributes """ theFile = File() # valid props theFile.FileID = 1 self.assertEqual( theFile.FileID, 1 ) theFile.Status = "Done" self.assertEqual( theFile.Status, "Done" ) theFile.LFN = "/some/path/somewhere" self.assertEqual( theFile.LFN, "/some/path/somewhere" ) theFile.PFN = "/some/path/somewhere" self.assertEqual( theFile.PFN, "/some/path/somewhere" ) theFile.Attempt = 1 self.assertEqual( theFile.Attempt, 1 ) theFile.Size = 1 self.assertEqual( theFile.Size, 1 ) theFile.GUID = "2bbabe80-e2f1-11e1-9b23-0800200c9a66" self.assertEqual( theFile.GUID, "2bbabe80-e2f1-11e1-9b23-0800200c9a66" ) theFile.ChecksumType = "adler32" self.assertEqual( theFile.ChecksumType, "ADLER32" ) theFile.Checksum = "123456" self.assertEqual( theFile.Checksum, "123456" ) # # theFile.Checksum = None theFile.ChecksumType = None self.assertEqual( theFile.Checksum, "" ) self.assertEqual( theFile.ChecksumType, "" ) # # invalid props # FileID try: theFile.FileID = "foo" except Exception, error: self.assertEqual( isinstance( error, ValueError ), True )
def prepareTransformationTasks( self, transBody, taskDict, owner = '', ownerGroup = '', ownerDN = '' ): """ Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB """ if not taskDict: return S_OK({}) if ( not owner ) or ( not ownerGroup ): res = getProxyInfo( False, False ) if not res['OK']: return res proxyInfo = res['Value'] owner = proxyInfo['username'] ownerGroup = proxyInfo['group'] if not ownerDN: res = getDNForUsername( owner ) if not res['OK']: return res ownerDN = res['Value'][0] requestOperation = 'ReplicateAndRegister' if transBody: try: _requestType, requestOperation = transBody.split( ';' ) except AttributeError: pass # Do not remove sorted, we might pop elements in the loop for taskID in sorted( taskDict ): paramDict = taskDict[taskID] transID = paramDict['TransformationID'] oRequest = Request() transfer = Operation() transfer.Type = requestOperation transfer.TargetSE = paramDict['TargetSE'] # If there are input files if paramDict['InputData']: if isinstance( paramDict['InputData'], list ): files = paramDict['InputData'] elif isinstance( paramDict['InputData'], basestring ): files = paramDict['InputData'].split( ';' ) for lfn in files: trFile = File() trFile.LFN = lfn transfer.addFile( trFile ) oRequest.addOperation( transfer ) oRequest.RequestName = _requestName( transID, taskID ) oRequest.OwnerDN = ownerDN oRequest.OwnerGroup = ownerGroup isValid = self.requestValidator.validate( oRequest ) if not isValid['OK']: self.log.error( "Error creating request for task", "%s %s" % ( taskID, isValid ) ) # This works because we loop over a copy of the keys ! taskDict.pop( taskID ) continue taskDict[taskID]['TaskObject'] = oRequest return S_OK( taskDict )
replicateAndRegister.TargetSE = targetSE res = fc.getFileMetadata( lfnList ) if not res['OK']: print "Can't get file metadata: %s" % res['Message'] DIRAC.exit( 1 ) if res['Value']['Failed']: print "Could not get the file metadata of the following, so skipping them:" for fFile in res['Value']['Failed']: print fFile lfnMetadata = res['Value']['Successful'] for lfn in lfnMetadata: rarFile = File() rarFile.LFN = lfn rarFile.Size = lfnMetadata[lfn]['Size'] rarFile.Checksum = lfnMetadata[lfn]['Checksum'] rarFile.GUID = lfnMetadata[lfn]['GUID'] rarFile.ChecksumType = 'ADLER32' replicateAndRegister.addFile( rarFile ) oRequest.addOperation( replicateAndRegister ) isValid = RequestValidator().validate( oRequest ) if not isValid['OK']: print "Request is not valid: ", isValid['Message'] DIRAC.exit( 1 ) result = reqClient.putRequest( oRequest ) if result['OK']: print "Request %d submitted successfully" % result['Value']
if not os.path.isfile( PFN ): gLogger.error( "%s is not a file" % PFN ) DIRAC.exit( -1 ) PFN = os.path.abspath( PFN ) size = os.path.getsize( PFN ) adler32 = fileAdler( PFN ) request = Request() request.RequestName = requestName putAndRegister = Operation() putAndRegister.Type = "PutAndRegister" putAndRegister.TargetSE = targetSE opFile = File() opFile.LFN = LFN opFile.PFN = PFN opFile.Size = size opFile.Checksum = adler32 opFile.ChecksumType = "ADLER32" putAndRegister.addFile( opFile ) request.addOperation( putAndRegister ) reqClient = ReqClient() putRequest = reqClient.putRequest( request ) if not putRequest["OK"]: gLogger.error( "unable to put request '%s': %s" % ( requestName, putRequest["Message"] ) ) DIRAC.exit( -1 ) gLogger.always( "Request '%s' has been put to ReqDB for execution." % requestName ) gLogger.always( "You can monitor its status using command: 'dirac-rms-show-request %s'" % requestName ) DIRAC.exit( 0 )
def test05FTS( self ): """ FTS state machine """ req = Request() req.RequestName = "FTSTest" ftsTransfer = Operation() ftsTransfer.Type = "ReplicateAndRegister" ftsTransfer.TargetSE = "CERN-USER" ftsFile = File() ftsFile.LFN = "/a/b/c" ftsFile.Checksum = "123456" ftsFile.ChecksumType = "Adler32" ftsTransfer.addFile( ftsFile ) req.addOperation( ftsTransfer ) self.assertEqual( req.Status, "Waiting", "1. wrong request status: %s" % req.Status ) self.assertEqual( ftsTransfer.Status, "Waiting", "1. wrong ftsStatus status: %s" % ftsTransfer.Status ) # # scheduled ftsFile.Status = "Scheduled" self.assertEqual( ftsTransfer.Status, "Scheduled", "2. wrong status for ftsTransfer: %s" % ftsTransfer.Status ) self.assertEqual( req.Status, "Scheduled", "2. wrong status for request: %s" % req.Status ) # # add new operation before FTS insertBefore = Operation() insertBefore.Type = "RegisterReplica" insertBefore.TargetSE = "CERN-USER" insertFile = File() insertFile.LFN = "/a/b/c" insertFile.PFN = "http://foo/bar" insertBefore.addFile( insertFile ) req.insertBefore( insertBefore, ftsTransfer ) self.assertEqual( insertBefore.Status, "Waiting", "3. wrong status for insertBefore: %s" % insertBefore.Status ) self.assertEqual( ftsTransfer.Status, "Scheduled", "3. wrong status for ftsStatus: %s" % ftsTransfer.Status ) self.assertEqual( req.Status, "Waiting", "3. wrong status for request: %s" % req.Status ) # # prev done insertFile.Status = "Done" self.assertEqual( insertBefore.Status, "Done", "4. wrong status for insertBefore: %s" % insertBefore.Status ) self.assertEqual( ftsTransfer.Status, "Scheduled", "4. wrong status for ftsStatus: %s" % ftsTransfer.Status ) self.assertEqual( req.Status, "Scheduled", "4. wrong status for request: %s" % req.Status ) # # reschedule ftsFile.Status = "Waiting" self.assertEqual( insertBefore.Status, "Done", "5. wrong status for insertBefore: %s" % insertBefore.Status ) self.assertEqual( ftsTransfer.Status, "Waiting", "5. wrong status for ftsStatus: %s" % ftsTransfer.Status ) self.assertEqual( req.Status, "Waiting", "5. wrong status for request: %s" % req.Status ) # # fts done ftsFile.Status = "Done" self.assertEqual( insertBefore.Status, "Done", "5. wrong status for insertBefore: %s" % insertBefore.Status ) self.assertEqual( ftsTransfer.Status, "Done", "5. wrong status for ftsStatus: %s" % ftsTransfer.Status ) self.assertEqual( req.Status, "Done", "5. wrong status for request: %s" % req.Status )
def createRequest( reqType ): r = Request() # Simple failover op1 = Operation() f = File() f.LFN = '/This/is/an/LFN' op1.addFile( f ) op1.Type = 'ReplicateAndRegister' op1.SourceSE = 'CERN-FAILOVER' op1.TargetSE = 'CERN-BUFFER' r.addOperation( op1 ) op2 = Operation() op2.addFile( f ) op2.Type = 'RemoveReplica' op2.TargetSE = 'CERN-FAILOVER' r.addOperation( op2 ) if reqType == 0: return r # two files for Failover f1 = File() f1.LFN = '/This/is/a/second/LFN' op3 = Operation() op3.addFile( f1 ) op3.Type = 'ReplicateAndRegister' op3.SourceSE = 'CERN-FAILOVER' op3.TargetSE = 'CERN-BUFFER' r.addOperation( op3 ) op3 = Operation() op3.addFile( f1 ) op3.Type = 'RemoveReplica' op3.TargetSE = 'CERN-FAILOVER' r.addOperation( op3 ) if reqType == 1: return r op = Operation() op.Type = 'ForwardDiset' if reqType == 2: r.addOperation( op ) return r r.insertBefore( op, r[0] ) if reqType == 3: return r op4 = Operation() op4.Type = 'ForwardDiset' r.addOperation( op4 ) if reqType == 4: return r # 2 different FAILOVER SEs: removal not optimized r[1].SourceSE = 'RAL-FAILOVER' r[2].SourceSE = 'RAL-FAILOVER' if reqType == 5: return r # 2 different destinations, same FAILOVER: replication not optimized r[3].SourceSE = 'RAL-FAILOVER' r[4].SourceSE = 'RAL-FAILOVER' r[3].TargetSE = 'RAL-BUFFER' if reqType == 6: return r print 'This should not happen, reqType =', reqType
continue count += 1 request = Request() request.RequestName = requestName if not multiRequests else '%s_%d' % ( requestName, count ) replicateAndRegister = Operation() replicateAndRegister.Type = "ReplicateAndRegister" replicateAndRegister.TargetSE = ",".join( targetSEs ) if catalog is not None: replicateAndRegister.Catalog = catalog for lfn in lfnChunk: metaDict = metaDatas["Successful"][lfn] opFile = File() opFile.LFN = lfn opFile.Size = metaDict["Size"] if "Checksum" in metaDict: # # should check checksum type, now assuming Adler32 (metaDict["ChecksumType"] = 'AD' opFile.Checksum = metaDict["Checksum"] opFile.ChecksumType = "ADLER32" replicateAndRegister.addFile( opFile ) request.addOperation( replicateAndRegister ) putRequest = reqClient.putRequest( request ) if not putRequest["OK"]: gLogger.error( "unable to put request '%s': %s" % ( request.RequestName, putRequest["Message"] ) ) error = -1 continue