def generateRequest(self): """ Generate failover requests for the operations in the internal cache """ request = RequestContainer() result = self.sendStoredStatusInfo() if not result['OK']: if result.has_key('rpcStub'): request.addSubRequest(DISETSubRequest(result['rpcStub']).getDictionary(),'diset') else: return S_ERROR('Could not create job state sub-request') result = self.sendStoredJobParameters() if not result['OK']: if result.has_key('rpcStub'): request.addSubRequest(DISETSubRequest(result['rpcStub']).getDictionary(),'diset') else: return S_ERROR('Could not create job parameters sub-request') if request.isEmpty()['Value']: request = None return S_OK(request)
def generateRequest(self): """ Generate failover requests for the operations in the internal cache """ request = RequestContainer() result = self.sendStoredStatusInfo() if not result['OK']: if result.has_key('rpcStub'): request.addSubRequest( DISETSubRequest(result['rpcStub']).getDictionary(), 'diset') else: return S_ERROR('Could not create job state sub-request') result = self.sendStoredJobParameters() if not result['OK']: if result.has_key('rpcStub'): request.addSubRequest( DISETSubRequest(result['rpcStub']).getDictionary(), 'diset') else: return S_ERROR('Could not create job parameters sub-request') if request.isEmpty()['Value']: request = None return S_OK(request)
def __setRemovalRequest( self, lfn, ownerDN, ownerGroup ): """ Set removal request with the given credentials """ request = RequestContainer() request.setRequestAttributes( { 'OwnerDN':ownerDN, 'OwnerGroup':ownerGroup } ) requestName = os.path.basename( lfn ).strip()+'_removal_request.xml' request.setRequestName( requestName ) request.setSourceComponent( 'JobCleaningAgent' ) removalDict = {'Attributes':{ 'Operation':'removeFile', 'TargetSE':'', 'ExecutionOrder':0 } } result = request.addSubRequest( removalDict, 'removal' ) if not result['OK']: return result index = result['Value'] fileDict = { 'LFN':lfn, 'PFN':'', 'Status':'Waiting' } request.setSubRequestFiles( index, 'removal', [fileDict] ) client = RequestClient() result = request.toXML() if not result['OK']: return result xmlRequest = result['Value'] result = client.setRequest( requestName, xmlRequest ) return result
def __setRemovalRequest(self, lfn, ownerDN, ownerGroup): """ Set removal request with the given credentials """ request = RequestContainer() request.setRequestAttributes({ 'OwnerDN': ownerDN, 'OwnerGroup': ownerGroup }) requestName = os.path.basename(lfn).strip() + '_removal_request.xml' request.setRequestName(requestName) request.setSourceComponent('JobCleaningAgent') removalDict = { 'Attributes': { 'Operation': 'removeFile', 'TargetSE': '', 'ExecutionOrder': 0 } } result = request.addSubRequest(removalDict, 'removal') if not result['OK']: return result index = result['Value'] fileDict = {'LFN': lfn, 'PFN': '', 'Status': 'Waiting'} request.setSubRequestFiles(index, 'removal', [fileDict]) client = RequestClient() result = request.toXML() if not result['OK']: return result xmlRequest = result['Value'] result = client.setRequest(requestName, xmlRequest) return result
def test__getLastOrder(self): # no files req = RequestContainer() res = req._getLastOrder() self.assertEqual( res, 0 ) self.assertEqual( req.subRequests, {} ) req.addSubRequest( {'Attributes':{'Operation':'replicateAndRegister', 'TargetSE':'SE', 'ExecutionOrder': 0}}, 'transfer' ) res = req._getLastOrder() self.assertEqual( res, 0 ) req.addSubRequest( {'Attributes':{'Operation':'replicateAndRegister', 'TargetSE':'SE', 'ExecutionOrder': 1}}, 'transfer' ) res = req._getLastOrder() self.assertEqual( res, 1 ) del( req ) # with files req = RequestContainer() res = req._getLastOrder( 'foo' ) self.assertEqual( res, 0 ) req.addSubRequest( {'Attributes':{'Operation':'replicateAndRegister', 'TargetSE':'SE', 'ExecutionOrder': 1}}, 'transfer' ) res = req._getLastOrder( 'foo' ) self.assertEqual( res, 0 ) req.setSubRequestFiles( 0, 'transfer', [{'LFN':'foo', 'Status':'Waiting'}] ) res = req._getLastOrder( 'foo' ) self.assertEqual( res, 1 ) req.addSubRequest( {'Attributes':{'Operation':'replicateAndRegister', 'TargetSE':'SE', 'ExecutionOrder': 2}}, 'removal' ) res = req._getLastOrder( 'foo' ) self.assertEqual( res, 1 ) req.setSubRequestFiles( 0, 'removal', [{'LFN':'foo', 'Status':'Waiting'}] ) res = req._getLastOrder( 'foo' ) self.assertEqual( res, 2 )
def test__getLastOrder(self): # no files req = RequestContainer() res = req._getLastOrder() self.assertEqual(res, 0) self.assertEqual(req.subRequests, {}) req.addSubRequest( {"Attributes": {"Operation": "replicateAndRegister", "TargetSE": "SE", "ExecutionOrder": 0}}, "transfer" ) res = req._getLastOrder() self.assertEqual(res, 0) req.addSubRequest( {"Attributes": {"Operation": "replicateAndRegister", "TargetSE": "SE", "ExecutionOrder": 1}}, "transfer" ) res = req._getLastOrder() self.assertEqual(res, 1) del (req) # with files req = RequestContainer() res = req._getLastOrder("foo") self.assertEqual(res, 0) req.addSubRequest( {"Attributes": {"Operation": "replicateAndRegister", "TargetSE": "SE", "ExecutionOrder": 1}}, "transfer" ) res = req._getLastOrder("foo") self.assertEqual(res, 0) req.setSubRequestFiles(0, "transfer", [{"LFN": "foo", "Status": "Waiting"}]) res = req._getLastOrder("foo") self.assertEqual(res, 1) req.addSubRequest( {"Attributes": {"Operation": "replicateAndRegister", "TargetSE": "SE", "ExecutionOrder": 2}}, "removal" ) res = req._getLastOrder("foo") self.assertEqual(res, 1) req.setSubRequestFiles(0, "removal", [{"LFN": "foo", "Status": "Waiting"}]) res = req._getLastOrder("foo") self.assertEqual(res, 2)
def __deleteSandboxFromExternalBackend( self, SEName, SEPFN ): if self.getCSOption( "DelayedExternalDeletion", True ): gLogger.info( "Setting deletion request" ) try: request = RequestContainer() result = request.addSubRequest( { 'Attributes' : { 'Operation' : 'removePhysicalFile', 'TargetSE' : SEName, 'ExecutionOrder' : 1 } }, 'removal' ) index = result['Value'] fileDict = { 'PFN' : SEPFN, 'Status' : 'Waiting' } request.setSubRequestFiles( index, 'removal', [ fileDict ] ) return RequestClient().setRequest( "RemoteSBDeletion:%s|%s:%s" % ( SEName, SEPFN, time.time() ), request.toXML()[ 'Value' ] ) except Exception, e: gLogger.exception( "Exception while setting deletion request" ) return S_ERROR( "Cannot set deletion request: %s" % str( e ) )
def __deleteSandboxFromExternalBackend(self, SEName, SEPFN): if self.getCSOption("DelayedExternalDeletion", True): gLogger.info("Setting deletion request") try: request = RequestContainer() result = request.addSubRequest( { 'Attributes': { 'Operation': 'removePhysicalFile', 'TargetSE': SEName, 'ExecutionOrder': 1 } }, 'removal') index = result['Value'] fileDict = {'PFN': SEPFN, 'Status': 'Waiting'} request.setSubRequestFiles(index, 'removal', [fileDict]) return RequestClient().setRequest( "RemoteSBDeletion:%s|%s:%s" % (SEName, SEPFN, time.time()), request.toXML()['Value']) except Exception, e: gLogger.exception("Exception while setting deletion request") return S_ERROR("Cannot set deletion request: %s" % str(e))
else: lfnpath = "%s%s" % (path, os.path.basename(appTar)) res = rm.putAndRegister(lfnpath, appTar, ops.getValue('Software/BaseStorageElement',"CERN-SRM")) if not res['OK']: return res request = RequestContainer() request.setCreationTime() requestClient = RequestClient() request.setRequestName('copy_%s' % os.path.basename(appTar).replace(".tgz","").replace(".tar.gz","")) request.setSourceComponent('ReplicateILCSoft') copies_at = ops.getValue('Software/CopiesAt',[]) index_copy = 0 for copies in copies_at: res = request.addSubRequest({'Attributes':{'Operation' : 'replicateAndRegister', 'TargetSE' : copies, 'ExecutionOrder' : index_copy}, 'Files':[{'LFN':lfnpath}]}, 'transfer') #res = rm.replicateAndRegister("%s%s"%(path,appTar),"IN2P3-SRM") if not res['OK']: return res index_copy += 1 requestxml = request.toXML()['Value'] if copies_at: res = requestClient.setRequest(request.getRequestName()['Value'], requestxml) if not res['OK']: gLogger.error('Could not set replication request %s' % res['Message']) return S_OK('Application uploaded') return S_OK()
class FailoverTransfer: ############################################################################# def __init__(self, requestObject=False): """ Constructor function, can specify request object to instantiate FailoverTransfer or a new request object is created. """ self.log = gLogger.getSubLogger("FailoverTransfer") self.rm = ReplicaManager() self.request = requestObject if not self.request: self.request = RequestContainer() self.request.setRequestName('default_request.xml') self.request.setSourceComponent('FailoverTransfer') ############################################################################# def transferAndRegisterFile(self, fileName, localPath, lfn, destinationSEList, fileGUID=None, fileCatalog=None): """Performs the transfer and register operation with failover. """ errorList = [] for se in destinationSEList: self.log.info( 'Attempting rm.putAndRegister("%s","%s","%s",guid="%s",catalog="%s")' % (lfn, localPath, se, fileGUID, fileCatalog)) result = self.rm.putAndRegister(lfn, localPath, se, guid=fileGUID, catalog=fileCatalog) self.log.verbose(result) if not result['OK']: self.log.error('rm.putAndRegister failed with message', result['Message']) errorList.append(result['Message']) continue if not result['Value']['Failed']: self.log.info( 'rm.putAndRegister successfully uploaded %s to %s' % (fileName, se)) return S_OK({'uploadedSE': se, 'lfn': lfn}) #Now we know something went wrong errorDict = result['Value']['Failed'][lfn] if not errorDict.has_key('register'): self.log.error('rm.putAndRegister failed with unknown error', str(errorDict)) errorList.append( 'Unknown error while attempting upload to %s' % se) continue fileDict = errorDict['register'] #Therefore the registration failed but the upload was successful if not fileCatalog: fileCatalog = '' result = self.__setRegistrationRequest(fileDict['LFN'], se, fileCatalog, fileDict) if not result['OK']: self.log.error( 'Failed to set registration request for: SE %s and metadata: \n%s' % (se, fileDict)) errorList.append( 'Failed to set registration request for: SE %s and metadata: \n%s' % (se, fileDict)) continue else: self.log.info( 'Successfully set registration request for: SE %s and metadata: \n%s' % (se, fileDict)) metadata = {} metadata['filedict'] = fileDict metadata['uploadedSE'] = se metadata['lfn'] = lfn metadata['registration'] = 'request' return S_OK(metadata) self.log.error( 'Encountered %s errors during attempts to upload output data' % len(errorList)) return S_ERROR('Failed to upload output data file') ############################################################################# def transferAndRegisterFileFailover(self, fileName, localPath, lfn, targetSE, failoverSEList, fileGUID=None, fileCatalog=None): """Performs the transfer and register operation to failover storage and sets the necessary replication and removal requests to recover. """ failover = self.transferAndRegisterFile(fileName, localPath, lfn, failoverSEList, fileGUID, fileCatalog) if not failover['OK']: self.log.error('Could not upload file to failover SEs', failover['Message']) return failover #set removal requests and replication requests result = self.__setFileReplicationRequest(lfn, targetSE) if not result['OK']: self.log.error('Could not set file replication request', result['Message']) return result lfn = failover['Value']['lfn'] failoverSE = failover['Value']['uploadedSE'] self.log.info( 'Attempting to set replica removal request for LFN %s at failover SE %s' % (lfn, failoverSE)) result = self.__setReplicaRemovalRequest(lfn, failoverSE) if not result['OK']: self.log.error('Could not set removal request', result['Message']) return result return S_OK('%s uploaded to a failover SE' % fileName) ############################################################################# def getRequestObject(self): """Returns the potentially modified request object in order to propagate changes. """ return S_OK(self.request) ############################################################################# def __setFileReplicationRequest(self, lfn, se): """ Sets a registration request. """ self.log.info('Setting replication request for %s to %s' % (lfn, se)) result = self.request.addSubRequest( { 'Attributes': { 'Operation': 'replicateAndRegister', 'TargetSE': se, 'ExecutionOrder': 0 } }, 'transfer') if not result['OK']: return result index = result['Value'] fileDict = {'LFN': lfn, 'Status': 'Waiting'} self.request.setSubRequestFiles(index, 'transfer', [fileDict]) return S_OK() ############################################################################# def __setRegistrationRequest(self, lfn, se, catalog, fileDict): """ Sets a registration request. """ self.log.info('Setting registration request for %s at %s.' % (lfn, se)) result = self.request.addSubRequest( { 'Attributes': { 'Operation': 'registerFile', 'ExecutionOrder': 0, 'TargetSE': se, 'Catalogue': catalog } }, 'register') if not result['OK']: return result index = result['Value'] if not fileDict.has_key('Status'): fileDict['Status'] = 'Waiting' self.request.setSubRequestFiles(index, 'register', [fileDict]) return S_OK() ############################################################################# def __setReplicaRemovalRequest(self, lfn, se): """ Sets a removal request for a replica. """ result = self.request.addSubRequest( { 'Attributes': { 'Operation': 'replicaRemoval', 'TargetSE': se, 'ExecutionOrder': 1 } }, 'removal') index = result['Value'] fileDict = {'LFN': lfn, 'Status': 'Waiting'} self.request.setSubRequestFiles(index, 'removal', [fileDict]) return S_OK() ############################################################################# def __setFileRemovalRequest(self, lfn, se='', pfn=''): """ Sets a removal request for a file including all replicas. """ result = self.request.addSubRequest( { 'Attributes': { 'Operation': 'removeFile', 'TargetSE': se, 'ExecutionOrder': 1 } }, 'removal') index = result['Value'] fileDict = {'LFN': lfn, 'PFN': pfn, 'Status': 'Waiting'} self.request.setSubRequestFiles(index, 'removal', [fileDict]) return S_OK()
class FailoverTransfer: ############################################################################# def __init__(self,requestObject=False): """ Constructor function, can specify request object to instantiate FailoverTransfer or a new request object is created. """ self.log = gLogger.getSubLogger( "FailoverTransfer" ) self.rm = ReplicaManager() self.request = requestObject if not self.request: self.request = RequestContainer() self.request.setRequestName('default_request.xml') self.request.setSourceComponent('FailoverTransfer') ############################################################################# def transferAndRegisterFile(self,fileName,localPath,lfn,destinationSEList,fileGUID=None,fileCatalog=None): """Performs the transfer and register operation with failover. """ errorList = [] for se in destinationSEList: self.log.info('Attempting rm.putAndRegister("%s","%s","%s",guid="%s",catalog="%s")' %(lfn,localPath,se,fileGUID,fileCatalog)) result = self.rm.putAndRegister(lfn,localPath,se,guid=fileGUID,catalog=fileCatalog) self.log.verbose(result) if not result['OK']: self.log.error('rm.putAndRegister failed with message',result['Message']) errorList.append(result['Message']) continue if not result['Value']['Failed']: self.log.info('rm.putAndRegister successfully uploaded %s to %s' %(fileName,se)) return S_OK({'uploadedSE':se,'lfn':lfn}) #Now we know something went wrong errorDict = result['Value']['Failed'][lfn] if not errorDict.has_key('register'): self.log.error('rm.putAndRegister failed with unknown error',str(errorDict)) errorList.append('Unknown error while attempting upload to %s' %se) continue fileDict = errorDict['register'] #Therefore the registration failed but the upload was successful if not fileCatalog: fileCatalog='' result = self.__setRegistrationRequest(fileDict['LFN'],se,fileCatalog,fileDict) if not result['OK']: self.log.error('Failed to set registration request for: SE %s and metadata: \n%s' %(se,fileDict)) errorList.append('Failed to set registration request for: SE %s and metadata: \n%s' %(se,fileDict)) continue else: self.log.info('Successfully set registration request for: SE %s and metadata: \n%s' %(se,fileDict)) metadata = {} metadata['filedict']=fileDict metadata['uploadedSE']=se metadata['lfn']=lfn metadata['registration']='request' return S_OK(metadata) self.log.error('Encountered %s errors during attempts to upload output data' %len(errorList)) return S_ERROR('Failed to upload output data file') ############################################################################# def transferAndRegisterFileFailover(self,fileName,localPath,lfn,targetSE,failoverSEList,fileGUID=None,fileCatalog=None): """Performs the transfer and register operation to failover storage and sets the necessary replication and removal requests to recover. """ failover = self.transferAndRegisterFile(fileName,localPath,lfn,failoverSEList,fileGUID,fileCatalog) if not failover['OK']: self.log.error('Could not upload file to failover SEs',failover['Message']) return failover #set removal requests and replication requests result = self.__setFileReplicationRequest(lfn,targetSE) if not result['OK']: self.log.error('Could not set file replication request',result['Message']) return result lfn = failover['Value']['lfn'] failoverSE = failover['Value']['uploadedSE'] self.log.info('Attempting to set replica removal request for LFN %s at failover SE %s' %(lfn,failoverSE)) result = self.__setReplicaRemovalRequest(lfn,failoverSE) if not result['OK']: self.log.error('Could not set removal request',result['Message']) return result return S_OK('%s uploaded to a failover SE' %fileName) ############################################################################# def getRequestObject(self): """Returns the potentially modified request object in order to propagate changes. """ return S_OK(self.request) ############################################################################# def __setFileReplicationRequest(self,lfn,se): """ Sets a registration request. """ self.log.info('Setting replication request for %s to %s' % (lfn,se)) result = self.request.addSubRequest({'Attributes':{'Operation':'replicateAndRegister', 'TargetSE':se,'ExecutionOrder':0}}, 'transfer') if not result['OK']: return result index = result['Value'] fileDict = {'LFN':lfn,'Status':'Waiting'} self.request.setSubRequestFiles(index,'transfer',[fileDict]) return S_OK() ############################################################################# def __setRegistrationRequest(self,lfn,se,catalog,fileDict): """ Sets a registration request. """ self.log.info('Setting registration request for %s at %s.' % (lfn,se)) result = self.request.addSubRequest({'Attributes':{'Operation':'registerFile','ExecutionOrder':0, 'TargetSE':se,'Catalogue':catalog}},'register') if not result['OK']: return result index = result['Value'] if not fileDict.has_key('Status'): fileDict['Status']='Waiting' self.request.setSubRequestFiles(index,'register',[fileDict]) return S_OK() ############################################################################# def __setReplicaRemovalRequest(self,lfn,se): """ Sets a removal request for a replica. """ result = self.request.addSubRequest({'Attributes':{'Operation':'replicaRemoval', 'TargetSE':se,'ExecutionOrder':1}}, 'removal') index = result['Value'] fileDict = {'LFN':lfn,'Status':'Waiting'} self.request.setSubRequestFiles(index,'removal',[fileDict]) return S_OK() ############################################################################# def __setFileRemovalRequest(self,lfn,se='',pfn=''): """ Sets a removal request for a file including all replicas. """ result = self.request.addSubRequest({'Attributes':{'Operation':'removeFile', 'TargetSE':se,'ExecutionOrder':1}}, 'removal') index = result['Value'] fileDict = {'LFN':lfn,'PFN':pfn,'Status':'Waiting'} self.request.setSubRequestFiles(index,'removal',[fileDict]) return S_OK()
def test_addSubRequest(self): rc_o = RequestContainer() op1_Index = rc_o.addSubRequest( {"Attributes": {"SubRequestID": "x", "CreationTime": "2012-06-06 14:53:43.763743", "Operation": "op1"}}, "someType", ) op1_Index = op1_Index["Value"] subRequestExpected = { "someType": [ { "Files": [], "Attributes": { "Status": "Waiting", "LastUpdate": "", "TargetSE": "", "ExecutionOrder": 0, "SubRequestID": "x", "CreationTime": "2012-06-06 14:53:43.763743", "Catalogue": "", "Error": "", "Operation": "op1", }, "Datasets": [], } ] } self.assertEqual(rc_o.subRequests, subRequestExpected) op2_index = rc_o.addSubRequest( {"Attributes": {"SubRequestID": "x", "CreationTime": "2012-06-06 14:53:43.763743", "Operation": "op2"}}, "someType", ) op2_index = op2_index["Value"] subRequestExpected = { "someType": [ { "Files": [], "Attributes": { "Status": "Waiting", "LastUpdate": "", "TargetSE": "", "ExecutionOrder": 0, "SubRequestID": "x", "CreationTime": "2012-06-06 14:53:43.763743", "Catalogue": "", "Error": "", "Operation": "op1", }, "Datasets": [], }, { "Files": [], "Attributes": { "Status": "Waiting", "LastUpdate": "", "TargetSE": "", "ExecutionOrder": 0, "SubRequestID": "x", "CreationTime": "2012-06-06 14:53:43.763743", "Catalogue": "", "Error": "", "Operation": "op2", }, "Datasets": [], }, ] } self.assertEqual(rc_o.subRequests, subRequestExpected) rc_o.addSubRequest( { "Attributes": { "SubRequestID": "x", "CreationTime": "2012-06-06 14:53:43.763743", "ExecutionOrder": "last", } }, "someType", ) subRequestExpected = { "someType": [ { "Files": [], "Attributes": { "Status": "Waiting", "LastUpdate": "", "TargetSE": "", "ExecutionOrder": 0, "SubRequestID": "x", "CreationTime": "2012-06-06 14:53:43.763743", "Catalogue": "", "Error": "", "Operation": "op1", }, "Datasets": [], }, { "Files": [], "Attributes": { "Status": "Waiting", "LastUpdate": "", "TargetSE": "", "ExecutionOrder": 0, "SubRequestID": "x", "CreationTime": "2012-06-06 14:53:43.763743", "Catalogue": "", "Error": "", "Operation": "op2", }, "Datasets": [], }, { "Files": [], "Attributes": { "Status": "Waiting", "LastUpdate": "", "TargetSE": "", "ExecutionOrder": 1, "SubRequestID": "x", "CreationTime": "2012-06-06 14:53:43.763743", "Catalogue": "", "Error": "", "Operation": "", }, "Datasets": [], }, ] } self.assertEqual(rc_o.subRequests, subRequestExpected) rc_o.addSubRequest( { "Attributes": { "SubRequestID": "x", "CreationTime": "2012-06-06 14:53:43.763743", "ExecutionOrder": "last", } }, "someOtherType", ) subRequestExpected = { "someType": [ { "Files": [], "Attributes": { "Status": "Waiting", "LastUpdate": "", "TargetSE": "", "ExecutionOrder": 0, "SubRequestID": "x", "CreationTime": "2012-06-06 14:53:43.763743", "Catalogue": "", "Error": "", "Operation": "op1", }, "Datasets": [], }, { "Files": [], "Attributes": { "Status": "Waiting", "LastUpdate": "", "TargetSE": "", "ExecutionOrder": 0, "SubRequestID": "x", "CreationTime": "2012-06-06 14:53:43.763743", "Catalogue": "", "Error": "", "Operation": "op2", }, "Datasets": [], }, { "Files": [], "Attributes": { "Status": "Waiting", "LastUpdate": "", "TargetSE": "", "ExecutionOrder": 1, "SubRequestID": "x", "CreationTime": "2012-06-06 14:53:43.763743", "Catalogue": "", "Error": "", "Operation": "", }, "Datasets": [], }, ], "someOtherType": [ { "Files": [], "Attributes": { "Status": "Waiting", "LastUpdate": "", "TargetSE": "", "ExecutionOrder": 2, "SubRequestID": "x", "CreationTime": "2012-06-06 14:53:43.763743", "Catalogue": "", "Error": "", "Operation": "", }, "Datasets": [], } ], } self.assertEqual(rc_o.subRequests, subRequestExpected) fileDict = {"LFN": "foo", "Status": "Waiting"} rc_o.setSubRequestFiles(op1_Index, "someType", [fileDict]) subRequestExpected = { "someType": [ { "Files": [{"LFN": "foo", "Status": "Waiting"}], "Attributes": { "Status": "Waiting", "LastUpdate": "", "TargetSE": "", "ExecutionOrder": 0, "SubRequestID": "x", "CreationTime": "2012-06-06 14:53:43.763743", "Catalogue": "", "Error": "", "Operation": "op1", }, "Datasets": [], }, { "Files": [], "Attributes": { "Status": "Waiting", "LastUpdate": "", "TargetSE": "", "ExecutionOrder": 0, "SubRequestID": "x", "CreationTime": "2012-06-06 14:53:43.763743", "Catalogue": "", "Error": "", "Operation": "op2", }, "Datasets": [], }, { "Files": [], "Attributes": { "Status": "Waiting", "LastUpdate": "", "TargetSE": "", "ExecutionOrder": 1, "SubRequestID": "x", "CreationTime": "2012-06-06 14:53:43.763743", "Catalogue": "", "Error": "", "Operation": "", }, "Datasets": [], }, ], "someOtherType": [ { "Files": [], "Attributes": { "Status": "Waiting", "LastUpdate": "", "TargetSE": "", "ExecutionOrder": 2, "SubRequestID": "x", "CreationTime": "2012-06-06 14:53:43.763743", "Catalogue": "", "Error": "", "Operation": "", }, "Datasets": [], } ], } self.assertEqual(rc_o.subRequests, subRequestExpected) fileLastOp = rc_o._getLastOrder("foo") rc_o.addSubRequest( { "Attributes": { "SubRequestID": "x", "CreationTime": "2012-06-06 14:53:43.763743", "ExecutionOrder": fileLastOp + 1, } }, "someOtherType", ) subRequestExpected = { "someType": [ { "Files": [{"LFN": "foo", "Status": "Waiting"}], "Attributes": { "Status": "Waiting", "LastUpdate": "", "TargetSE": "", "ExecutionOrder": 0, "SubRequestID": "x", "CreationTime": "2012-06-06 14:53:43.763743", "Catalogue": "", "Error": "", "Operation": "op1", }, "Datasets": [], }, { "Files": [], "Attributes": { "Status": "Waiting", "LastUpdate": "", "TargetSE": "", "ExecutionOrder": 0, "SubRequestID": "x", "CreationTime": "2012-06-06 14:53:43.763743", "Catalogue": "", "Error": "", "Operation": "op2", }, "Datasets": [], }, { "Files": [], "Attributes": { "Status": "Waiting", "LastUpdate": "", "TargetSE": "", "ExecutionOrder": 1, "SubRequestID": "x", "CreationTime": "2012-06-06 14:53:43.763743", "Catalogue": "", "Error": "", "Operation": "", }, "Datasets": [], }, ], "someOtherType": [ { "Files": [], "Attributes": { "Status": "Waiting", "LastUpdate": "", "TargetSE": "", "ExecutionOrder": 2, "SubRequestID": "x", "CreationTime": "2012-06-06 14:53:43.763743", "Catalogue": "", "Error": "", "Operation": "", }, "Datasets": [], }, { "Files": [], "Attributes": { "Status": "Waiting", "LastUpdate": "", "TargetSE": "", "ExecutionOrder": 1, "SubRequestID": "x", "CreationTime": "2012-06-06 14:53:43.763743", "Catalogue": "", "Error": "", "Operation": "", }, "Datasets": [], }, ], } self.assertEqual(rc_o.subRequests, subRequestExpected)
class UploadOutputData(ModuleBase): """ As name suggest: upload output data. For Production only: See L{UserJobFinalization} for User job upload. """ ############################################################################# def __init__(self): """Module initialization. """ super(UploadOutputData, self).__init__() self.version = __RCSID__ self.log = gLogger.getSubLogger( "UploadOutputData" ) self.commandTimeOut = 10*60 self.enable = True self.failoverTest = False #flag to put file to failover SE by default self.failoverSEs = gConfig.getValue('/Resources/StorageElementGroups/Tier1-Failover', []) self.ops = Operations() #List all parameters here self.outputDataFileMask = '' self.outputMode = 'Any' #or 'Local' for reco case self.outputList = [] self.request = None self.PRODUCTION_ID = "" self.prodOutputLFNs = [] self.experiment = "CLIC" ############################################################################# def applicationSpecificInputs(self): """ By convention the module parameters are resolved here. """ self.log.verbose("Workflow commons:") self.log.verbose(self.workflow_commons) self.log.verbose("Step commons:") self.log.verbose(self.step_commons) if self.step_commons.has_key('Enable'): self.enable = self.step_commons['Enable'] if not type(self.enable) == type(True): self.log.warn('Enable flag set to non-boolean value %s, setting to False' % self.enable) self.enable = False if self.step_commons.has_key('TestFailover'): self.enable = self.step_commons['TestFailover'] if not type(self.failoverTest) == type(True): self.log.warn('Test failover flag set to non-boolean value %s, setting to False' % self.failoverTest) self.failoverTest = False if self.workflow_commons.has_key("PRODUCTION_ID"): self.PRODUCTION_ID = self.workflow_commons["PRODUCTION_ID"] if os.environ.has_key('JOBID'): self.log.verbose('Found WMS JobID = %s' % self.jobID) else: self.log.info('No WMS JobID found, disabling module via control flag') self.enable = False if self.workflow_commons.has_key('Request'): self.request = self.workflow_commons['Request'] else: self.request = RequestContainer() self.request.setRequestName('job_%s_request.xml' % self.jobID) self.request.setJobID(self.jobID) self.request.setSourceComponent("Job_%s" % self.jobID) ##This is the thing that is used to establish the list of outpufiles to treat: ## Make sure that all that is in the : "listoutput" and also in the ProductionData ## is treated properly. Needed as whatever is in listoutput does not contain any reference to the ## prodID and task ID. Also if for some reason a step failed, then the corresponding data will not be there if self.workflow_commons.has_key('outputList'): self.outputList = self.workflow_commons['outputList'] if self.workflow_commons.has_key('ProductionOutputData'): proddata = self.workflow_commons['ProductionOutputData'].split(";") self.log.verbose("prod data : %s" % proddata ) olist = {} for obj in self.outputList: fname_in_outputlist = obj['outputFile'].lower() extension = '' if fname_in_outputlist.count("_sim") or fname_in_outputlist.count("_rec") or fname_in_outputlist.count("_dst"): extension = ".slcio" elif fname_in_outputlist.count("_gen"): extension = ".stdhep" fname_in_outputlist = fname_in_outputlist.replace(extension,"") for prodfile in proddata: prodfile = os.path.basename(prodfile) extension = '' if prodfile.count("_sim") or prodfile.count("_rec") or prodfile.count("_dst"): extension = ".slcio" elif prodfile.count("_gen"): extension = ".stdhep" prodfile = prodfile.replace(extension,"") if olist.has_key(prodfile): ## This has already been treated, no need to come back to it. continue appdict = {} if (fname_in_outputlist.count("_gen")):# and prodfile.lower().count("_gen_")) : genf = obj['outputFile'].split("_gen")[0] genf += "_gen" if (prodfile.count(genf)): appdict.update(obj) appdict['outputFile'] = prodfile+extension olist[prodfile] = appdict if (fname_in_outputlist.count("_sim")): simf = obj['outputFile'].split("_sim")[0] simf += "_sim" if (prodfile.count(simf)): appdict.update(obj) appdict['outputFile'] = prodfile+extension olist[prodfile] = appdict self.log.verbose('olist %s'%olist) if (fname_in_outputlist.count("_rec")): recf = obj['outputFile'].split("_rec")[0] recf += "_rec" if (prodfile.count(recf)): appdict.update(obj) appdict['outputFile'] = prodfile+extension olist[prodfile] = appdict break if (fname_in_outputlist.count("_dst") and prodfile.lower().count("_dst_")): dstf = obj['outputFile'].split("_dst")[0] dstf += "_dst" if (prodfile.count(dstf)): appdict.update(obj) appdict['outputFile'] = prodfile+extension olist[prodfile] = appdict break self.outputList = olist.values() else: olist = [] for obj in self.outputList: appdict = obj appdict['outputFile'] = getProdFilename(obj['outputFile'], int(self.workflow_commons["PRODUCTION_ID"]), int(self.workflow_commons["JOB_ID"])) olist.append(appdict) self.outputList = olist self.log.verbose("OutputList : %s" % self.outputList) if self.workflow_commons.has_key('outputMode'): self.outputMode = self.workflow_commons['outputMode'] if self.workflow_commons.has_key('outputDataFileMask'): self.outputDataFileMask = self.workflow_commons['outputDataFileMask'] if not type(self.outputDataFileMask) == type([]): self.outputDataFileMask = [i.lower().strip() for i in self.outputDataFileMask.split(';')] #result = constructProductionLFNs(self.workflow_commons) #if not result['OK']: # self.log.error('Could not create production LFNs',result['Message']) # return result #self.prodOutputLFNs=result['Value']['ProductionOutputData'] if self.workflow_commons.has_key('ProductionOutputData'): self.prodOutputLFNs = self.workflow_commons['ProductionOutputData'].split(";") else: self.prodOutputLFNs = [] return S_OK('Parameters resolved') ############################################################################# def execute(self): """ Main execution function. """ self.log.info('Initializing %s' % self.version) result = self.resolveInputVariables() if not result['OK']: self.log.error(result['Message']) return result if not self.workflowStatus['OK'] or not self.stepStatus['OK']: self.log.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK'])) return S_OK('No output data upload attempted') ##determine the experiment example_file = self.prodOutputLFNs[0] if "/ilc/prod/clic" in example_file: self.experiment = "CLIC" elif "/ilc/prod/ilc/sid" in example_file: self.experiment = 'ILC_SID' elif "/ilc/prod/ilc/mc-dbd" in example_file: self.experiment = 'ILC_ILD' else: self.log.warn("Failed to determine experiment, reverting to default") #Determine the final list of possible output files for the #workflow and all the parameters needed to upload them. result = self.getCandidateFiles(self.outputList, self.prodOutputLFNs, self.outputDataFileMask) if not result['OK']: self.setApplicationStatus(result['Message']) return result fileDict = result['Value'] result = self.getFileMetadata(fileDict) if not result['OK']: self.setApplicationStatus(result['Message']) return result if not result['Value']: self.log.info('No output data files were determined to be uploaded for this workflow') return S_OK() fileMetadata = result['Value'] #Get final, resolved SE list for files final = {} for fileName, metadata in fileMetadata.items(): result = getDestinationSEList(metadata['workflowSE'], DIRAC.siteName(), self.outputMode) if not result['OK']: self.log.error('Could not resolve output data SE', result['Message']) self.setApplicationStatus('Failed To Resolve OutputSE') return result resolvedSE = result['Value'] final[fileName] = metadata final[fileName]['resolvedSE'] = resolvedSE self.log.info('The following files will be uploaded: %s' % (string.join(final.keys(), ', '))) for fileName, metadata in final.items(): self.log.info('--------%s--------' % fileName) for n, v in metadata.items(): self.log.info('%s = %s' % (n, v)) #At this point can exit and see exactly what the module would have uploaded if not self.enable: self.log.info('Module is disabled by control flag, would have attempted to upload the \ following files %s' % string.join(final.keys(), ', ')) return S_OK('Module is disabled by control flag') #Disable the watchdog check in case the file uploading takes a long time self.log.info('Creating DISABLE_WATCHDOG_CPU_WALLCLOCK_CHECK in order to disable the Watchdog prior to upload') fopen = open('DISABLE_WATCHDOG_CPU_WALLCLOCK_CHECK','w') fopen.write('%s' % time.asctime()) fopen.close() #Instantiate the failover transfer client with the global request object failoverTransfer = FailoverTransfer(self.request) catalogs = ['FileCatalog', 'LcgFileCatalog'] #One by one upload the files with failover if necessary failover = {} if not self.failoverTest: for fileName, metadata in final.items(): self.log.info("Attempting to store file %s to the following SE(s):\n%s" % (fileName, string.join(metadata['resolvedSE'], ', '))) result = failoverTransfer.transferAndRegisterFile(fileName, metadata['localpath'], metadata['lfn'], metadata['resolvedSE'], fileGUID = metadata['guid'], fileCatalog = catalogs) if not result['OK']: self.log.error('Could not transfer and register %s with metadata:\n %s' % (fileName, metadata)) failover[fileName] = metadata else: lfn = metadata['lfn'] else: failover = final self.failoverSEs = self.ops.getValue("Production/%s/FailOverSE" % self.experiment, self.failoverSEs) cleanUp = False for fileName, metadata in failover.items(): self.log.info('Setting default catalog for failover transfer to FileCatalog') random.shuffle(self.failoverSEs) targetSE = metadata['resolvedSE'][0] metadata['resolvedSE'] = self.failoverSEs result = failoverTransfer.transferAndRegisterFileFailover(fileName, metadata['localpath'], metadata['lfn'], targetSE, metadata['resolvedSE'], fileGUID = metadata['guid'], fileCatalog = catalogs) if not result['OK']: self.log.error('Could not transfer and register %s with metadata:\n %s' % (fileName, metadata)) cleanUp = True break #no point continuing if one completely fails os.remove("DISABLE_WATCHDOG_CPU_WALLCLOCK_CHECK") #cleanup the mess #Now after all operations, retrieve potentially modified request object result = failoverTransfer.getRequestObject() if not result['OK']: self.log.error(result) return S_ERROR('Could not retrieve modified request') self.request = result['Value'] #If some or all of the files failed to be saved to failover if cleanUp: lfns = [] for fileName, metadata in final.items(): lfns.append(metadata['lfn']) result = self.__cleanUp(lfns) self.workflow_commons['Request'] = self.request return S_ERROR('Failed to upload output data') # #Can now register the successfully uploaded files in the BK # if not performBKRegistration: # self.log.info('There are no files to perform the BK registration for, all could be saved to failover') # else: # rm = ReplicaManager() # result = rm.addCatalogFile(performBKRegistration,catalogs=['BookkeepingDB']) # self.log.verbose(result) # if not result['OK']: # self.log.error(result) # return S_ERROR('Could Not Perform BK Registration') # if result['Value']['Failed']: # for lfn,error in result['Value']['Failed'].items(): # self.log.info('BK registration for %s failed with message: "%s" setting failover request' %(lfn,error)) # result = self.request.addSubRequest({'Attributes':{'Operation':'registerFile','ExecutionOrder':0, 'Catalogue':'BookkeepingDB'}},'register') # if not result['OK']: # self.log.error('Could not set registerFile request:\n%s' %result) # return S_ERROR('Could Not Set BK Registration Request') # fileDict = {'LFN':lfn,'Status':'Waiting'} # index = result['Value'] # self.request.setSubRequestFiles(index,'register',[fileDict]) self.workflow_commons['Request'] = self.request return S_OK('Output data uploaded') ############################################################################# def __cleanUp(self, lfnList): """ Clean up uploaded data for the LFNs in the list """ # Clean up the current request for req_type in ['transfer', 'register']: for lfn in lfnList: result = self.request.getNumSubRequests(req_type) if result['OK']: nreq = result['Value'] if nreq: # Go through subrequests in reverse order in order not to spoil the numbering ind_range = [0] if nreq > 1: ind_range = range(nreq-1, -1, -1) for i in ind_range: result = self.request.getSubRequestFiles(i, req_type) if result['OK']: fileList = result['Value'] if fileList[0]['LFN'] == lfn: result = self.request.removeSubRequest(i, req_type) # Set removal requests just in case for lfn in lfnList: result = self.request.addSubRequest({'Attributes': {'Operation' : 'removeFile', 'TargetSE' : '', 'ExecutionOrder' : 1}}, 'removal') index = result['Value'] fileDict = {'LFN':lfn, 'PFN':'', 'Status':'Waiting'} self.request.setSubRequestFiles(index, 'removal', [fileDict]) return S_OK() #EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#
def test__getLastOrder(self): # no files req = RequestContainer() res = req._getLastOrder() self.assertEqual(res, 0) self.assertEqual(req.subRequests, {}) req.addSubRequest( { 'Attributes': { 'Operation': 'replicateAndRegister', 'TargetSE': 'SE', 'ExecutionOrder': 0 } }, 'transfer') res = req._getLastOrder() self.assertEqual(res, 0) req.addSubRequest( { 'Attributes': { 'Operation': 'replicateAndRegister', 'TargetSE': 'SE', 'ExecutionOrder': 1 } }, 'transfer') res = req._getLastOrder() self.assertEqual(res, 1) del (req) # with files req = RequestContainer() res = req._getLastOrder('foo') self.assertEqual(res, 0) req.addSubRequest( { 'Attributes': { 'Operation': 'replicateAndRegister', 'TargetSE': 'SE', 'ExecutionOrder': 1 } }, 'transfer') res = req._getLastOrder('foo') self.assertEqual(res, 0) req.setSubRequestFiles(0, 'transfer', [{ 'LFN': 'foo', 'Status': 'Waiting' }]) res = req._getLastOrder('foo') self.assertEqual(res, 1) req.addSubRequest( { 'Attributes': { 'Operation': 'replicateAndRegister', 'TargetSE': 'SE', 'ExecutionOrder': 2 } }, 'removal') res = req._getLastOrder('foo') self.assertEqual(res, 1) req.setSubRequestFiles(0, 'removal', [{ 'LFN': 'foo', 'Status': 'Waiting' }]) res = req._getLastOrder('foo') self.assertEqual(res, 2)
def test_addSubRequest( self ): rc_o = RequestContainer() op1_Index = rc_o.addSubRequest( {'Attributes': {'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743', 'Operation': 'op1'}}, 'someType' ) op1_Index = op1_Index['Value'] subRequestExpected = {'someType': [{'Files': [], 'Attributes': {'Status': 'Waiting', 'LastUpdate': '', 'TargetSE': '', 'ExecutionOrder': 0, 'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743', 'Catalogue': '', 'Error': '', 'Operation': 'op1'}, 'Datasets': []}]} self.assertEqual( rc_o.subRequests, subRequestExpected ) op2_index = rc_o.addSubRequest( {'Attributes': {'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743', 'Operation': 'op2'}}, 'someType' ) op2_index = op2_index['Value'] subRequestExpected = { 'someType': [ { 'Files': [], 'Attributes': {'Status': 'Waiting', 'LastUpdate': '', 'TargetSE': '', 'ExecutionOrder': 0, 'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743', 'Catalogue': '', 'Error': '', 'Operation': 'op1'}, 'Datasets': [] }, { 'Files': [], 'Attributes': {'Status': 'Waiting', 'LastUpdate': '', 'TargetSE': '', 'ExecutionOrder': 0, 'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743', 'Catalogue': '', 'Error': '', 'Operation': 'op2'}, 'Datasets': [] } ] } self.assertEqual( rc_o.subRequests, subRequestExpected ) rc_o.addSubRequest( {'Attributes': {'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743', 'ExecutionOrder': 'last'}}, 'someType' ) subRequestExpected = { 'someType': [ { 'Files': [], 'Attributes': {'Status': 'Waiting', 'LastUpdate': '', 'TargetSE': '', 'ExecutionOrder': 0, 'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743', 'Catalogue': '', 'Error': '', 'Operation': 'op1'}, 'Datasets': [] }, { 'Files': [], 'Attributes': {'Status': 'Waiting', 'LastUpdate': '', 'TargetSE': '', 'ExecutionOrder': 0, 'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743', 'Catalogue': '', 'Error': '', 'Operation': 'op2'}, 'Datasets': [] }, { 'Files': [], 'Attributes': {'Status': 'Waiting', 'LastUpdate': '', 'TargetSE': '', 'ExecutionOrder': 1, 'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743', 'Catalogue': '', 'Error': '', 'Operation': ''}, 'Datasets': [] } ] } self.assertEqual( rc_o.subRequests, subRequestExpected ) rc_o.addSubRequest( {'Attributes': {'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743', 'ExecutionOrder': 'last'}}, 'someOtherType' ) subRequestExpected = { 'someType': [ { 'Files': [], 'Attributes': {'Status': 'Waiting', 'LastUpdate': '', 'TargetSE': '', 'ExecutionOrder': 0, 'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743', 'Catalogue': '', 'Error': '', 'Operation': 'op1'}, 'Datasets': [] }, { 'Files': [], 'Attributes': {'Status': 'Waiting', 'LastUpdate': '', 'TargetSE': '', 'ExecutionOrder': 0, 'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743', 'Catalogue': '', 'Error': '', 'Operation': 'op2'}, 'Datasets': [] }, { 'Files': [], 'Attributes': {'Status': 'Waiting', 'LastUpdate': '', 'TargetSE': '', 'ExecutionOrder': 1, 'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743', 'Catalogue': '', 'Error': '', 'Operation': ''}, 'Datasets': [] } ], 'someOtherType': [ { 'Files': [], 'Attributes': {'Status': 'Waiting', 'LastUpdate': '', 'TargetSE': '', 'ExecutionOrder': 2, 'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743', 'Catalogue': '', 'Error': '', 'Operation': ''}, 'Datasets': [] }, ] } self.assertEqual( rc_o.subRequests, subRequestExpected ) fileDict = {'LFN':'foo', 'Status':'Waiting'} rc_o.setSubRequestFiles( op1_Index, 'someType', [fileDict] ) subRequestExpected = { 'someType': [ { 'Files': [{'LFN':'foo', 'Status':'Waiting'}], 'Attributes': {'Status': 'Waiting', 'LastUpdate': '', 'TargetSE': '', 'ExecutionOrder': 0, 'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743', 'Catalogue': '', 'Error': '', 'Operation': 'op1'}, 'Datasets': [] }, { 'Files': [], 'Attributes': {'Status': 'Waiting', 'LastUpdate': '', 'TargetSE': '', 'ExecutionOrder': 0, 'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743', 'Catalogue': '', 'Error': '', 'Operation': 'op2'}, 'Datasets': [] }, { 'Files': [], 'Attributes': {'Status': 'Waiting', 'LastUpdate': '', 'TargetSE': '', 'ExecutionOrder': 1, 'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743', 'Catalogue': '', 'Error': '', 'Operation': ''}, 'Datasets': [] } ], 'someOtherType': [ { 'Files': [], 'Attributes': {'Status': 'Waiting', 'LastUpdate': '', 'TargetSE': '', 'ExecutionOrder': 2, 'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743', 'Catalogue': '', 'Error': '', 'Operation': ''}, 'Datasets': [] }, ] } self.assertEqual( rc_o.subRequests, subRequestExpected ) fileLastOp = rc_o._getLastOrder( 'foo' ) rc_o.addSubRequest( {'Attributes': {'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743', 'ExecutionOrder': fileLastOp + 1}}, 'someOtherType' ) subRequestExpected = { 'someType': [ { 'Files': [{'LFN':'foo', 'Status':'Waiting'}], 'Attributes': {'Status': 'Waiting', 'LastUpdate': '', 'TargetSE': '', 'ExecutionOrder': 0, 'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743', 'Catalogue': '', 'Error': '', 'Operation': 'op1'}, 'Datasets': [] }, { 'Files': [], 'Attributes': {'Status': 'Waiting', 'LastUpdate': '', 'TargetSE': '', 'ExecutionOrder': 0, 'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743', 'Catalogue': '', 'Error': '', 'Operation': 'op2'}, 'Datasets': [] }, { 'Files': [], 'Attributes': {'Status': 'Waiting', 'LastUpdate': '', 'TargetSE': '', 'ExecutionOrder': 1, 'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743', 'Catalogue': '', 'Error': '', 'Operation': ''}, 'Datasets': [] } ], 'someOtherType': [ { 'Files': [], 'Attributes': {'Status': 'Waiting', 'LastUpdate': '', 'TargetSE': '', 'ExecutionOrder': 2, 'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743', 'Catalogue': '', 'Error': '', 'Operation': ''}, 'Datasets': [] }, { 'Files': [], 'Attributes': {'Status': 'Waiting', 'LastUpdate': '', 'TargetSE': '', 'ExecutionOrder': 1, 'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743', 'Catalogue': '', 'Error': '', 'Operation': ''}, 'Datasets': [] }, ] } self.assertEqual( rc_o.subRequests, subRequestExpected )
def executeRequest(self): ################################################ # Get a request from request DB gMonitor.addMark("Iteration", 1) res = self.RequestDBClient.getRequest('transfer') if not res['OK']: gLogger.info( "TransferAgent.execute: Failed to get request from database.") return S_OK() elif not res['Value']: gLogger.info( "TransferAgent.execute: No requests to be executed found.") return S_OK() requestString = res['Value']['RequestString'] requestName = res['Value']['RequestName'] sourceServer = res['Value']['Server'] try: jobID = int(res['Value']['JobID']) except: jobID = 0 gLogger.info("TransferAgent.execute: Obtained request %s" % requestName) result = self.RequestDBClient.getCurrentExecutionOrder( requestName, sourceServer) if result['OK']: currentOrder = result['Value'] else: return S_OK('Can not get the request execution order') oRequest = RequestContainer(request=requestString) ################################################ # Find the number of sub-requests from the request res = oRequest.getNumSubRequests('transfer') if not res['OK']: errStr = "TransferAgent.execute: Failed to obtain number of transfer subrequests." gLogger.error(errStr, res['Message']) return S_OK() gLogger.info("TransferAgent.execute: Found %s sub requests." % res['Value']) ################################################ # For all the sub-requests in the request modified = False for ind in range(res['Value']): gMonitor.addMark("Execute", 1) gLogger.info("TransferAgent.execute: Processing sub-request %s." % ind) subRequestAttributes = oRequest.getSubRequestAttributes( ind, 'transfer')['Value'] if subRequestAttributes['ExecutionOrder']: subExecutionOrder = int(subRequestAttributes['ExecutionOrder']) else: subExecutionOrder = 0 subStatus = subRequestAttributes['Status'] if subStatus == 'Waiting' and subExecutionOrder <= currentOrder: subRequestFiles = oRequest.getSubRequestFiles( ind, 'transfer')['Value'] operation = subRequestAttributes['Operation'] subRequestError = '' ################################################ # If the sub-request is a put and register operation if operation == 'putAndRegister' or operation == 'putAndRegisterAndRemove': gLogger.info( "TransferAgent.execute: Attempting to execute %s sub-request." % operation) diracSE = str(subRequestAttributes['TargetSE']) catalog = '' if subRequestAttributes.has_key('Catalogue'): catalog = subRequestAttributes['Catalogue'] for subRequestFile in subRequestFiles: if subRequestFile['Status'] == 'Waiting': gMonitor.addMark("Put and register", 1) lfn = str(subRequestFile['LFN']) file = subRequestFile['PFN'] guid = subRequestFile['GUID'] addler = subRequestFile['Addler'] res = self.ReplicaManager.putAndRegister( lfn, file, diracSE, guid=guid, checksum=addler, catalog=catalog) if res['OK']: if res['Value']['Successful'].has_key(lfn): if not res['Value']['Successful'][ lfn].has_key('put'): gMonitor.addMark("Put failed", 1) self.DataLog.addFileRecord( lfn, 'PutFail', diracSE, '', 'TransferAgent') gLogger.info( "TransferAgent.execute: Failed to put %s to %s." % (lfn, diracSE)) subRequestError = "Put operation failed for %s to %s" % ( lfn, diracSE) oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn, 'Error', 'Put failed') elif not res['Value']['Successful'][ lfn].has_key('register'): gMonitor.addMark("Put successful", 1) gMonitor.addMark( "File registration failed", 1) self.DataLog.addFileRecord( lfn, 'Put', diracSE, '', 'TransferAgent') self.DataLog.addFileRecord( lfn, 'RegisterFail', diracSE, '', 'TransferAgent') gLogger.info( "TransferAgent.execute: Successfully put %s to %s in %s seconds." % (lfn, diracSE, res['Value'] ['Successful'][lfn]['put'])) gLogger.info( "TransferAgent.execute: Failed to register %s to %s." % (lfn, diracSE)) oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn, 'Status', 'Done') oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn, 'Error', 'Registration failed') subRequestError = "Registration failed for %s to %s" % ( lfn, diracSE) fileDict = res['Value']['Failed'][lfn][ 'register'] registerRequestDict = { 'Attributes': { 'TargetSE': fileDict['TargetSE'], 'Operation': 'registerFile' }, 'Files': [{ 'LFN': fileDict['LFN'], 'PFN': fileDict['PFN'], 'Size': fileDict['Size'], 'Addler': fileDict['Addler'], 'GUID': fileDict['GUID'] }] } gLogger.info( "TransferAgent.execute: Setting registration request for failed file." ) oRequest.addSubRequest( registerRequestDict, 'register') modified = True else: gMonitor.addMark("Put successful", 1) gMonitor.addMark( "File registration successful", 1) self.DataLog.addFileRecord( lfn, 'Put', diracSE, '', 'TransferAgent') self.DataLog.addFileRecord( lfn, 'Register', diracSE, '', 'TransferAgent') gLogger.info( "TransferAgent.execute: Successfully put %s to %s in %s seconds." % (lfn, diracSE, res['Value'] ['Successful'][lfn]['put'])) gLogger.info( "TransferAgent.execute: Successfully registered %s to %s in %s seconds." % (lfn, diracSE, res['Value'] ['Successful'][lfn]['register'])) oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn, 'Status', 'Done') modified = True else: gMonitor.addMark("Put failed", 1) self.DataLog.addFileRecord( lfn, 'PutFail', diracSE, '', 'TransferAgent') errStr = "TransferAgent.execute: Failed to put and register file." gLogger.error( errStr, "%s %s %s" % (lfn, diracSE, res['Value']['Failed'][lfn])) oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn, 'Error', 'Complete file failure') subRequestError = "Failed to put and register file" else: gMonitor.addMark("Put failed", 1) self.DataLog.addFileRecord( lfn, 'PutFail', diracSE, '', 'TransferAgent') errStr = "TransferAgent.execute: Completely failed to put and register file." gLogger.error(errStr, res['Message']) oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn, 'Error', 'RM call failure') subRequestError = operation + " RM call file" else: gLogger.info( "TransferAgent.execute: File already completed." ) ################################################ # If the sub-request is a put operation elif operation == 'put': gLogger.info( "TransferAgent.execute: Attempting to execute %s sub-request." % operation) diracSE = subRequestAttributes['TargetSE'] for subRequestFile in subRequestFiles: if subRequestFile['Status'] == 'Waiting': gMonitor.addMark("Put", 1) lfn = subRequestFile['LFN'] file = subRequestFile['PFN'] res = self.ReplicaManager.put(lfn, file, diracSE) if res['OK']: if res['Value']['Successful'].has_key(lfn): gMonitor.addMark("Put successful", 1) self.DataLog.addFileRecord( lfn, 'Put', diracSE, '', 'TransferAgent') gLogger.info( "TransferAgent.execute: Successfully put %s to %s in %s seconds." % (lfn, diracSE, res['Value']['Successful'][lfn])) oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn, 'Status', 'Done') modified = True else: gMonitor.addMark("Put failed", 1) self.DataLog.addFileRecord( lfn, 'PutFail', diracSE, '', 'TransferAgent') errStr = "TransferAgent.execute: Failed to put file." gLogger.error( errStr, "%s %s %s" % (lfn, diracSE, res['Value']['Failed'][lfn])) subRequestError = "Put operation failed for %s to %s" % ( lfn, diracSE) oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn, 'Error', 'Put failed') else: gMonitor.addMark("Put failed", 1) self.DataLog.addFileRecord( lfn, 'PutFail', diracSE, '', 'TransferAgent') errStr = "TransferAgent.execute: Completely failed to put file." gLogger.error(errStr, res['Message']) subRequestError = "Put RM call failed for %s to %s" % ( lfn, diracSE) oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn, 'Error', 'Put RM call failed') else: gLogger.info( "TransferAgent.execute: File already completed." ) ################################################ # If the sub-request is a replicate and register operation elif operation == 'replicateAndRegister' or operation == 'replicateAndRegisterAndRemove': gLogger.info( "TransferAgent.execute: Attempting to execute %s sub-request." % operation) targetSE = subRequestAttributes['TargetSE'] sourceSE = subRequestAttributes['SourceSE'] if sourceSE == "None": sourceSE = '' for subRequestFile in subRequestFiles: if subRequestFile['Status'] == 'Waiting': gMonitor.addMark("Replicate and register", 1) lfn = subRequestFile['LFN'] res = self.ReplicaManager.replicateAndRegister( lfn, targetSE, sourceSE=sourceSE) if res['OK']: if res['Value']['Successful'].has_key(lfn): if not res['Value']['Successful'][ lfn].has_key('replicate'): gLogger.info( "TransferAgent.execute: Failed to replicate %s to %s." % (lfn, targetSE)) gMonitor.addMark( "Replication failed", 1) oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn, "Error", "Replication failed") subRequestError = "Replication failed for %s to %s" % ( lfn, targetSE) elif not res['Value']['Successful'][ lfn].has_key('register'): gMonitor.addMark( "Replication successful", 1) gMonitor.addMark( "Replica registration failed", 1) gLogger.info( "TransferAgent.execute: Successfully replicated %s to %s in %s seconds." % (lfn, targetSE, res['Value'] ['Successful'][lfn]['replicate'])) gLogger.info( "TransferAgent.execute: Failed to register %s to %s." % (lfn, targetSE)) oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn, 'Status', 'Done') oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn, 'Error', 'Registration failed') subRequestError = "Registration failed for %s to %s" % ( lfn, targetSE) fileDict = res['Value']['Failed'][lfn][ 'register'] registerRequestDict = { 'Attributes': { 'TargetSE': fileDict['TargetSE'], 'Operation': 'registerReplica' }, 'Files': [{ 'LFN': fileDict['LFN'], 'PFN': fileDict['PFN'] }] } gLogger.info( "TransferAgent.execute: Setting registration request for failed replica." ) oRequest.addSubRequest( registerRequestDict, 'register') modified = True else: gMonitor.addMark( "Replication successful", 1) gMonitor.addMark( "Replica registration successful", 1) gLogger.info( "TransferAgent.execute: Successfully replicated %s to %s in %s seconds." % (lfn, targetSE, res['Value'] ['Successful'][lfn]['replicate'])) gLogger.info( "TransferAgent.execute: Successfully registered %s to %s in %s seconds." % (lfn, targetSE, res['Value'] ['Successful'][lfn]['register'])) oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn, 'Status', 'Done') modified = True else: gMonitor.addMark("Replication failed", 1) errStr = "TransferAgent.execute: Failed to replicate and register file." gLogger.error( errStr, "%s %s %s" % (lfn, targetSE, res['Value']['Failed'][lfn])) else: gMonitor.addMark("Replication failed", 1) errStr = "TransferAgent.execute: Completely failed to replicate and register file." gLogger.error(errStr, res['Message']) oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn, 'Error', 'RM call failure') subRequestError = operation + " RM call failed" else: gLogger.info( "TransferAgent.execute: File already completed." ) ################################################ # If the sub-request is a replicate operation elif operation == 'replicate': gLogger.info( "TransferAgent.execute: Attempting to execute %s sub-request." % operation) targetSE = subRequestAttributes['TargetSE'] sourceSE = subRequestAttributes['SourceSE'] for subRequestFile in subRequestFiles: if subRequestFile['Status'] == 'Waiting': gMonitor.addMark("Replicate", 1) lfn = subRequestFile['LFN'] res = self.ReplicaManager.replicate( lfn, targetSE, sourceSE=sourceSE) if res['OK']: if res['Value']['Successful'].has_key(lfn): gMonitor.addMark("Replication successful", 1) gLogger.info( "TransferAgent.execute: Successfully replicated %s to %s in %s seconds." % (lfn, diracSE, res['Value']['Successful'][lfn])) oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn, 'Status', 'Done') modified = True else: gMonitor.addMark("Replication failed", 1) errStr = "TransferAgent.execute: Failed to replicate file." gLogger.error( errStr, "%s %s %s" % (lfn, targetSE, res['Value']['Failed'][lfn])) subRequestError = "Replicate operation failed for %s to %s" % ( lfn, targetSE) oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn, 'Error', 'Put failed') else: gMonitor.addMark("Replication failed", 1) errStr = "TransferAgent.execute: Completely failed to replicate file." gLogger.error(errStr, res['Message']) subRequestError = "Replicate RM call failed for %s to %s" % ( lfn, targetSE) oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn, 'Error', 'Replicate RM call failed') else: gLogger.info( "TransferAgent.execute: File already completed." ) ################################################ # If the sub-request is a get operation elif operation == 'get': gLogger.info( "TransferAgent.execute: Attempting to execute %s sub-request." % operation) sourceSE = subRequestAttributes['TargetSE'] for subRequestFile in subRequestFiles: if subRequestFile['Status'] == 'Waiting': lfn = str(subRequestFile['LFN']) pfn = str(subRequestFile['PFN']) got = False if sourceSE and pfn: res = self.ReplicaManager.getStorageFile( pfn, sourceSE) if res['Value']['Successful'].has_key(pfn): got = True else: res = self.ReplicaManager.getFile(lfn) if res['Value']['Successful'].has_key(lfn): got = False if got: gLogger.info( "TransferAgent.execute: Successfully got %s." % lfn) oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn, 'Status', 'Done') modified = True else: errStr = "TransferAgent.execute: Failed to get file." gLogger.error(errStr, lfn) else: gLogger.info( "TransferAgent.execute: File already completed." ) ################################################ # If the sub-request is none of the above types else: gLogger.error( "TransferAgent.execute: Operation not supported.", operation) if subRequestError: oRequest.setSubRequestAttributeValue( ind, 'transfer', 'Error', subRequestError) ################################################ # Determine whether there are any active files if oRequest.isSubRequestEmpty(ind, 'transfer')['Value']: oRequest.setSubRequestStatus(ind, 'transfer', 'Done') gMonitor.addMark("Done", 1) ################################################ # If the sub-request is already in terminal state else: gLogger.info( "TransferAgent.execute: Sub-request %s is status '%s' and not to be executed." % (ind, subRequestAttributes['Status'])) ################################################ # Generate the new request string after operation requestString = oRequest.toXML()['Value'] res = self.RequestDBClient.updateRequest(requestName, requestString, sourceServer) if modified and jobID: result = self.finalizeRequest(requestName, jobID, sourceServer) return S_OK()
def executeRequest( self ): ################################################ # Get a request from request DB gMonitor.addMark( "Iteration", 1 ) res = self.RequestDBClient.getRequest( 'transfer' ) if not res['OK']: gLogger.info( "TransferAgent.execute: Failed to get request from database." ) return S_OK() elif not res['Value']: gLogger.info( "TransferAgent.execute: No requests to be executed found." ) return S_OK() requestString = res['Value']['RequestString'] requestName = res['Value']['RequestName'] sourceServer = res['Value']['Server'] try: jobID = int( res['Value']['JobID'] ) except: jobID = 0 gLogger.info( "TransferAgent.execute: Obtained request %s" % requestName ) result = self.RequestDBClient.getCurrentExecutionOrder( requestName, sourceServer ) if result['OK']: currentOrder = result['Value'] else: return S_OK( 'Can not get the request execution order' ) oRequest = RequestContainer( request = requestString ) ################################################ # Find the number of sub-requests from the request res = oRequest.getNumSubRequests( 'transfer' ) if not res['OK']: errStr = "TransferAgent.execute: Failed to obtain number of transfer subrequests." gLogger.error( errStr, res['Message'] ) return S_OK() gLogger.info( "TransferAgent.execute: Found %s sub requests." % res['Value'] ) ################################################ # For all the sub-requests in the request modified = False for ind in range( res['Value'] ): gMonitor.addMark( "Execute", 1 ) gLogger.info( "TransferAgent.execute: Processing sub-request %s." % ind ) subRequestAttributes = oRequest.getSubRequestAttributes( ind, 'transfer' )['Value'] if subRequestAttributes['ExecutionOrder']: subExecutionOrder = int( subRequestAttributes['ExecutionOrder'] ) else: subExecutionOrder = 0 subStatus = subRequestAttributes['Status'] if subStatus == 'Waiting' and subExecutionOrder <= currentOrder: subRequestFiles = oRequest.getSubRequestFiles( ind, 'transfer' )['Value'] operation = subRequestAttributes['Operation'] subRequestError = '' ################################################ # If the sub-request is a put and register operation if operation == 'putAndRegister' or operation == 'putAndRegisterAndRemove': gLogger.info( "TransferAgent.execute: Attempting to execute %s sub-request." % operation ) diracSE = str( subRequestAttributes['TargetSE'] ) catalog = '' if subRequestAttributes.has_key( 'Catalogue' ): catalog = subRequestAttributes['Catalogue'] for subRequestFile in subRequestFiles: if subRequestFile['Status'] == 'Waiting': gMonitor.addMark( "Put and register", 1 ) lfn = str( subRequestFile['LFN'] ) file = subRequestFile['PFN'] guid = subRequestFile['GUID'] addler = subRequestFile['Addler'] res = self.ReplicaManager.putAndRegister( lfn, file, diracSE, guid = guid, checksum = addler, catalog = catalog ) if res['OK']: if res['Value']['Successful'].has_key( lfn ): if not res['Value']['Successful'][lfn].has_key( 'put' ): gMonitor.addMark( "Put failed", 1 ) self.DataLog.addFileRecord( lfn, 'PutFail', diracSE, '', 'TransferAgent' ) gLogger.info( "TransferAgent.execute: Failed to put %s to %s." % ( lfn, diracSE ) ) subRequestError = "Put operation failed for %s to %s" % ( lfn, diracSE ) oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn,'Error', 'Put failed' ) elif not res['Value']['Successful'][lfn].has_key( 'register' ): gMonitor.addMark( "Put successful", 1 ) gMonitor.addMark( "File registration failed", 1 ) self.DataLog.addFileRecord( lfn, 'Put', diracSE, '', 'TransferAgent' ) self.DataLog.addFileRecord( lfn, 'RegisterFail', diracSE, '', 'TransferAgent' ) gLogger.info( "TransferAgent.execute: Successfully put %s to %s in %s seconds." % ( lfn, diracSE, res['Value']['Successful'][lfn]['put'] ) ) gLogger.info( "TransferAgent.execute: Failed to register %s to %s." % ( lfn, diracSE ) ) oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn, 'Status', 'Done' ) oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn,'Error', 'Registration failed' ) subRequestError = "Registration failed for %s to %s" % ( lfn, diracSE ) fileDict = res['Value']['Failed'][lfn]['register'] registerRequestDict = {'Attributes':{'TargetSE': fileDict['TargetSE'], 'Operation':'registerFile'}, 'Files':[{'LFN': fileDict['LFN'], 'PFN':fileDict['PFN'], 'Size':fileDict['Size'], 'Addler':fileDict['Addler'], 'GUID':fileDict['GUID']}]} gLogger.info( "TransferAgent.execute: Setting registration request for failed file." ) oRequest.addSubRequest( registerRequestDict, 'register' ) modified = True else: gMonitor.addMark( "Put successful", 1 ) gMonitor.addMark( "File registration successful", 1 ) self.DataLog.addFileRecord( lfn, 'Put', diracSE, '', 'TransferAgent' ) self.DataLog.addFileRecord( lfn, 'Register', diracSE, '', 'TransferAgent' ) gLogger.info( "TransferAgent.execute: Successfully put %s to %s in %s seconds." % ( lfn, diracSE, res['Value']['Successful'][lfn]['put'] ) ) gLogger.info( "TransferAgent.execute: Successfully registered %s to %s in %s seconds." % ( lfn, diracSE, res['Value']['Successful'][lfn]['register'] ) ) oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn, 'Status', 'Done' ) modified = True else: gMonitor.addMark( "Put failed", 1 ) self.DataLog.addFileRecord( lfn, 'PutFail', diracSE, '', 'TransferAgent' ) errStr = "TransferAgent.execute: Failed to put and register file." gLogger.error( errStr, "%s %s %s" % ( lfn, diracSE, res['Value']['Failed'][lfn] ) ) oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn,'Error', 'Complete file failure' ) subRequestError = "Failed to put and register file" else: gMonitor.addMark( "Put failed", 1 ) self.DataLog.addFileRecord( lfn, 'PutFail', diracSE, '', 'TransferAgent' ) errStr = "TransferAgent.execute: Completely failed to put and register file." gLogger.error( errStr, res['Message'] ) oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn,'Error', 'RM call failure' ) subRequestError = operation + " RM call file" else: gLogger.info( "TransferAgent.execute: File already completed." ) ################################################ # If the sub-request is a put operation elif operation == 'put': gLogger.info( "TransferAgent.execute: Attempting to execute %s sub-request." % operation ) diracSE = subRequestAttributes['TargetSE'] for subRequestFile in subRequestFiles: if subRequestFile['Status'] == 'Waiting': gMonitor.addMark( "Put", 1 ) lfn = subRequestFile['LFN'] file = subRequestFile['PFN'] res = self.ReplicaManager.put( lfn, file, diracSE ) if res['OK']: if res['Value']['Successful'].has_key( lfn ): gMonitor.addMark( "Put successful", 1 ) self.DataLog.addFileRecord( lfn, 'Put', diracSE, '', 'TransferAgent' ) gLogger.info( "TransferAgent.execute: Successfully put %s to %s in %s seconds." % ( lfn, diracSE, res['Value']['Successful'][lfn] ) ) oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn, 'Status', 'Done' ) modified = True else: gMonitor.addMark( "Put failed", 1 ) self.DataLog.addFileRecord( lfn, 'PutFail', diracSE, '', 'TransferAgent' ) errStr = "TransferAgent.execute: Failed to put file." gLogger.error( errStr, "%s %s %s" % ( lfn, diracSE, res['Value']['Failed'][lfn] ) ) subRequestError = "Put operation failed for %s to %s" % ( lfn, diracSE ) oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn,'Error', 'Put failed' ) else: gMonitor.addMark( "Put failed", 1 ) self.DataLog.addFileRecord( lfn, 'PutFail', diracSE, '', 'TransferAgent' ) errStr = "TransferAgent.execute: Completely failed to put file." gLogger.error( errStr, res['Message'] ) subRequestError = "Put RM call failed for %s to %s" % ( lfn, diracSE ) oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn,'Error', 'Put RM call failed' ) else: gLogger.info( "TransferAgent.execute: File already completed." ) ################################################ # If the sub-request is a replicate and register operation elif operation == 'replicateAndRegister' or operation == 'replicateAndRegisterAndRemove': gLogger.info( "TransferAgent.execute: Attempting to execute %s sub-request." % operation ) targetSE = subRequestAttributes['TargetSE'] sourceSE = subRequestAttributes['SourceSE'] if sourceSE == "None": sourceSE = '' for subRequestFile in subRequestFiles: if subRequestFile['Status'] == 'Waiting': gMonitor.addMark( "Replicate and register", 1 ) lfn = subRequestFile['LFN'] res = self.ReplicaManager.replicateAndRegister( lfn, targetSE, sourceSE = sourceSE ) if res['OK']: if res['Value']['Successful'].has_key( lfn ): if not res['Value']['Successful'][lfn].has_key( 'replicate' ): gLogger.info( "TransferAgent.execute: Failed to replicate %s to %s." % ( lfn, targetSE ) ) gMonitor.addMark( "Replication failed", 1 ) oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn,"Error", "Replication failed" ) subRequestError = "Replication failed for %s to %s" % ( lfn, targetSE ) elif not res['Value']['Successful'][lfn].has_key( 'register' ): gMonitor.addMark( "Replication successful", 1 ) gMonitor.addMark( "Replica registration failed", 1 ) gLogger.info( "TransferAgent.execute: Successfully replicated %s to %s in %s seconds." % ( lfn, targetSE, res['Value']['Successful'][lfn]['replicate'] ) ) gLogger.info( "TransferAgent.execute: Failed to register %s to %s." % ( lfn, targetSE ) ) oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn, 'Status', 'Done' ) oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn, 'Error', 'Registration failed' ) subRequestError = "Registration failed for %s to %s" % ( lfn, targetSE ) fileDict = res['Value']['Failed'][lfn]['register'] registerRequestDict = {'Attributes':{'TargetSE': fileDict['TargetSE'], 'Operation':'registerReplica'}, 'Files':[{'LFN': fileDict['LFN'], 'PFN':fileDict['PFN']}]} gLogger.info( "TransferAgent.execute: Setting registration request for failed replica." ) oRequest.addSubRequest( registerRequestDict, 'register' ) modified = True else: gMonitor.addMark( "Replication successful", 1 ) gMonitor.addMark( "Replica registration successful", 1 ) gLogger.info( "TransferAgent.execute: Successfully replicated %s to %s in %s seconds." % ( lfn, targetSE, res['Value']['Successful'][lfn]['replicate'] ) ) gLogger.info( "TransferAgent.execute: Successfully registered %s to %s in %s seconds." % ( lfn, targetSE, res['Value']['Successful'][lfn]['register'] ) ) oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn, 'Status', 'Done' ) modified = True else: gMonitor.addMark( "Replication failed", 1 ) errStr = "TransferAgent.execute: Failed to replicate and register file." gLogger.error( errStr, "%s %s %s" % ( lfn, targetSE, res['Value']['Failed'][lfn] ) ) else: gMonitor.addMark( "Replication failed", 1 ) errStr = "TransferAgent.execute: Completely failed to replicate and register file." gLogger.error( errStr, res['Message'] ) oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn,'Error', 'RM call failure' ) subRequestError = operation + " RM call failed" else: gLogger.info( "TransferAgent.execute: File already completed." ) ################################################ # If the sub-request is a replicate operation elif operation == 'replicate': gLogger.info( "TransferAgent.execute: Attempting to execute %s sub-request." % operation ) targetSE = subRequestAttributes['TargetSE'] sourceSE = subRequestAttributes['SourceSE'] for subRequestFile in subRequestFiles: if subRequestFile['Status'] == 'Waiting': gMonitor.addMark( "Replicate", 1 ) lfn = subRequestFile['LFN'] res = self.ReplicaManager.replicate( lfn, targetSE, sourceSE = sourceSE ) if res['OK']: if res['Value']['Successful'].has_key( lfn ): gMonitor.addMark( "Replication successful", 1 ) gLogger.info( "TransferAgent.execute: Successfully replicated %s to %s in %s seconds." % ( lfn, diracSE, res['Value']['Successful'][lfn] ) ) oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn, 'Status', 'Done' ) modified = True else: gMonitor.addMark( "Replication failed", 1 ) errStr = "TransferAgent.execute: Failed to replicate file." gLogger.error( errStr, "%s %s %s" % ( lfn, targetSE, res['Value']['Failed'][lfn] ) ) subRequestError = "Replicate operation failed for %s to %s" % ( lfn, targetSE ) oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn,'Error', 'Put failed' ) else: gMonitor.addMark( "Replication failed", 1 ) errStr = "TransferAgent.execute: Completely failed to replicate file." gLogger.error( errStr, res['Message'] ) subRequestError = "Replicate RM call failed for %s to %s" % ( lfn, targetSE ) oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn,'Error', 'Replicate RM call failed' ) else: gLogger.info( "TransferAgent.execute: File already completed." ) ################################################ # If the sub-request is a get operation elif operation == 'get': gLogger.info( "TransferAgent.execute: Attempting to execute %s sub-request." % operation ) sourceSE = subRequestAttributes['TargetSE'] for subRequestFile in subRequestFiles: if subRequestFile['Status'] == 'Waiting': lfn = str( subRequestFile['LFN'] ) pfn = str( subRequestFile['PFN'] ) got = False if sourceSE and pfn: res = self.ReplicaManager.getStorageFile( pfn, sourceSE ) if res['Value']['Successful'].has_key( pfn ): got = True else: res = self.ReplicaManager.getFile( lfn ) if res['Value']['Successful'].has_key( lfn ): got = False if got: gLogger.info( "TransferAgent.execute: Successfully got %s." % lfn ) oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn, 'Status', 'Done' ) modified = True else: errStr = "TransferAgent.execute: Failed to get file." gLogger.error( errStr, lfn ) else: gLogger.info( "TransferAgent.execute: File already completed." ) ################################################ # If the sub-request is none of the above types else: gLogger.error( "TransferAgent.execute: Operation not supported.", operation ) if subRequestError: oRequest.setSubRequestAttributeValue( ind, 'transfer', 'Error', subRequestError ) ################################################ # Determine whether there are any active files if oRequest.isSubRequestEmpty( ind, 'transfer' )['Value']: oRequest.setSubRequestStatus( ind, 'transfer', 'Done' ) gMonitor.addMark( "Done", 1 ) ################################################ # If the sub-request is already in terminal state else: gLogger.info( "TransferAgent.execute: Sub-request %s is status '%s' and not to be executed." % ( ind, subRequestAttributes['Status'] ) ) ################################################ # Generate the new request string after operation requestString = oRequest.toXML()['Value'] res = self.RequestDBClient.updateRequest( requestName, requestString, sourceServer ) if modified and jobID: result = self.finalizeRequest( requestName, jobID, sourceServer ) return S_OK()