def __setRemovalRequest( self, lfn, ownerDN, ownerGroup ): """ Set removal request with the given credentials """ request = RequestContainer() request.setRequestAttributes( { 'OwnerDN':ownerDN, 'OwnerGroup':ownerGroup } ) requestName = os.path.basename( lfn ).strip()+'_removal_request.xml' request.setRequestName( requestName ) request.setSourceComponent( 'JobCleaningAgent' ) removalDict = {'Attributes':{ 'Operation':'removeFile', 'TargetSE':'', 'ExecutionOrder':0 } } result = request.addSubRequest( removalDict, 'removal' ) if not result['OK']: return result index = result['Value'] fileDict = { 'LFN':lfn, 'PFN':'', 'Status':'Waiting' } request.setSubRequestFiles( index, 'removal', [fileDict] ) client = RequestClient() result = request.toXML() if not result['OK']: return result xmlRequest = result['Value'] result = client.setRequest( requestName, xmlRequest ) return result
def __setRemovalRequest(self, lfn, ownerDN, ownerGroup): """ Set removal request with the given credentials """ request = RequestContainer() request.setRequestAttributes({ 'OwnerDN': ownerDN, 'OwnerGroup': ownerGroup }) requestName = os.path.basename(lfn).strip() + '_removal_request.xml' request.setRequestName(requestName) request.setSourceComponent('JobCleaningAgent') removalDict = { 'Attributes': { 'Operation': 'removeFile', 'TargetSE': '', 'ExecutionOrder': 0 } } result = request.addSubRequest(removalDict, 'removal') if not result['OK']: return result index = result['Value'] fileDict = {'LFN': lfn, 'PFN': '', 'Status': 'Waiting'} request.setSubRequestFiles(index, 'removal', [fileDict]) client = RequestClient() result = request.toXML() if not result['OK']: return result xmlRequest = result['Value'] result = client.setRequest(requestName, xmlRequest) return result
def getRequest(): """ helper fcn to build requestContainer """ requestContainer = RequestContainer(init=False) ## get request requestContainer.setRequestName("00009423_00000118") requestContainer.setJobID(0) requestContainer.setOwnerDN("") requestContainer.setOwnerGroup("") requestContainer.setDIRACSetup("") requestContainer.setSourceComponent(None) requestContainer.setCreationTime("0000-00-00 00:00:00") requestContainer.setLastUpdate("2011-02-19 04:57:02") requestContainer.setStatus("Waiting") ## get subrequest requestContainer.initiateSubRequest("transfer") subRequestDict = { "Status": "Waiting", "SubRequestID": 2259916, "Operation": "replicateAndRegister", "Arguments": None, "ExecutionOrder": 0, "SourceSE": None, "TargetSE": "GRIDKA_MC-DST,GRIDKA_MC_M-DST", "Catalogue": None, "CreationTime": "2011-02-19 04:57:02", "SubmissionTime": "2011-02-19 04:57:02", "LastUpdate": "2011-08-18 20:14:22" } requestContainer.setSubRequestAttributes(0, "transfer", subRequestDict) ## get subrequest files files = [{ "FileID": 1610538, "LFN": "/lhcb/MC/MC10/ALLSTREAMS.DST/00009422/0000/00009422_00000171_1.allstreams.dst", "Size": None, "PFN": None, "GUID": None, "Md5": None, "Addler": None, "Attempt": 1, "Status": "Scheduled" }] requestContainer.setSubRequestFiles(0, "transfer", files) return { "OK": True, "Value": { "RequestName": "00009423_00000118", "RequestString": requestContainer.toXML()["Value"], "JobID": 0, "RequestContainer": requestContainer } }
def getRegisterRequest(): """ helper fcn to build request """ requestContainer = RequestContainer(init=False) requestContainer.setJobID(11889410) #requestContainer.setOwnerDN( "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=cibak/CN=605919/CN=Krzysztof Ciba" ) requestContainer.setOwnerGroup("lhcb_user") requestContainer.setDIRACSetup("LHCb-Production") requestContainer.setSourceComponent(None) requestContainer.setCreationTime("0000-00-00 00:00:00") requestContainer.setLastUpdate("2011-02-19 04:57:02") requestContainer.setStatus("Waiting") requestContainer.initiateSubRequest("register") subRequestDict = { "Status": "Waiting", "SubRequestID": 2259916, "Operation": "registerFile", "Arguments": None, "ExecutionOrder": 0, "SourceSE": None, "TargetSE": "RAL-USER", "Catalogue": "LcgFileCatalogCombined", "CreationTime": "2011-02-19 04:57:02", "SubmissionTime": "2011-02-19 04:57:02", "LastUpdate": "2011-08-18 20:14:22" } requestContainer.setSubRequestAttributes(0, "register", subRequestDict) files = [{ "FileID": 1610538, "LFN": "/lhcb/user/c/cblanks/11889/11889410/LDSB.rsQrRL", "Size": 153961749, "PFN": "srm://srm-lhcb.gridpp.rl.ac.uk/castor/ads.rl.ac.uk/prod/lhcb/user/c/cblanks/11889/11889410/LDSB.rsQrRL", "GUID": "5911A19C-7CDF-7F2A-36ED-089CD410F98A", "Md5": None, "Addler": "92b85e26", "Attempt": 1, "Status": "Waiting" }] requestContainer.setSubRequestFiles(0, "register", files) return { "requestName": "11889410.xml", "requestString": requestContainer.toXML()["Value"], "jobID": 11889410, "executionOrder": 0, "sourceServer": "foobarserver", "configPath": "/Systems/DataManagement/Development/Agents/RegistrationAgent" }
def getRequest(operation): """ fake requestDict :param str operation: sub-request operation attribute """ requestContainer = RequestContainer(init=False) requestContainer.setJobID(1) #requestContainer.setOwnerDN( "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=cibak/CN=605919/CN=Krzysztof Ciba" ) requestContainer.setOwnerGroup("lhcb_user") requestContainer.setDIRACSetup("LHCb-Production") requestContainer.setSourceComponent(None) requestContainer.setCreationTime("0000-00-00 00:00:00") requestContainer.setLastUpdate("2011-12-01 04:57:02") requestContainer.setStatus("Waiting") requestContainer.setAttribute("RequestID", 123456789) requestContainer.initiateSubRequest("transfer") subRequestDict = { "Status": "Waiting", "SubRequestID": 2222222, "Operation": operation, "Arguments": None, "ExecutionOrder": 0, "SourceSE": None, "TargetSE": "CERN-USER", "Catalogue": "LcgFileCatalogCombined", "CreationTime": "2011-12-01 04:57:02", "SubmissionTime": "2011-12-01 04:57:02", "LastUpdate": "2011-12-01 20:14:22" } requestContainer.setSubRequestAttributes(0, "transfer", subRequestDict) files = [{ "FileID": 3333333, "LFN": "/lhcb/user/c/cibak/11889/11889410/test.zzz", "Size": 44444444, "PFN": "srm://srm-lhcb.gridpp.rl.ac.uk/castor/ads.rl.ac.uk/prod/lhcb/user/c/cibak/11889/11889410/test.zzz", "GUID": "5P13RD4L-4J5L-3D21-U5P1-3RD4L4J5P13R", "Md5": None, "Addler": "92b85e26", "Attempt": 1, "Status": "Waiting" }] requestContainer.setSubRequestFiles(0, "transfer", files) return { "OK": True, "Value": { "requestName": "%s.xml" % operation, "requestString": requestContainer.toXML_new()["Value"], "requestObj": requestContainer, "jobID": 1, "executionOrder": 0, "sourceServer": "foobarserver" } }
def getRegisterRequest(): """ helper fcn to build request """ requestContainer = RequestContainer(init=False) requestContainer.setJobID(11889410) # requestContainer.setOwnerDN( "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=cibak/CN=605919/CN=Krzysztof Ciba" ) requestContainer.setOwnerGroup("lhcb_user") requestContainer.setDIRACSetup("LHCb-Production") requestContainer.setSourceComponent(None) requestContainer.setCreationTime("0000-00-00 00:00:00") requestContainer.setLastUpdate("2011-02-19 04:57:02") requestContainer.setStatus("Waiting") requestContainer.initiateSubRequest("register") subRequestDict = { "Status": "Waiting", "SubRequestID": 2259916, "Operation": "registerFile", "Arguments": None, "ExecutionOrder": 0, "SourceSE": None, "TargetSE": "RAL-USER", "Catalogue": "LcgFileCatalogCombined", "CreationTime": "2011-02-19 04:57:02", "SubmissionTime": "2011-02-19 04:57:02", "LastUpdate": "2011-08-18 20:14:22", } requestContainer.setSubRequestAttributes(0, "register", subRequestDict) files = [ { "FileID": 1610538, "LFN": "/lhcb/user/c/cblanks/11889/11889410/LDSB.rsQrRL", "Size": 153961749, "PFN": "srm://srm-lhcb.gridpp.rl.ac.uk/castor/ads.rl.ac.uk/prod/lhcb/user/c/cblanks/11889/11889410/LDSB.rsQrRL", "GUID": "5911A19C-7CDF-7F2A-36ED-089CD410F98A", "Md5": None, "Addler": "92b85e26", "Attempt": 1, "Status": "Waiting", } ] requestContainer.setSubRequestFiles(0, "register", files) return { "requestName": "11889410.xml", "requestString": requestContainer.toXML()["Value"], "jobID": 11889410, "executionOrder": 0, "sourceServer": "foobarserver", "configPath": "/Systems/DataManagement/Development/Agents/RegistrationAgent", }
def getRequest(): """ helper fcn to build requestContainer """ requestContainer = RequestContainer( init = False ) ## get request requestContainer.setRequestName( "00009423_00000118" ) requestContainer.setJobID( 0 ) requestContainer.setOwnerDN( "" ) requestContainer.setOwnerGroup( "" ) requestContainer.setDIRACSetup( "" ) requestContainer.setSourceComponent( None ) requestContainer.setCreationTime( "0000-00-00 00:00:00" ) requestContainer.setLastUpdate( "2011-02-19 04:57:02" ) requestContainer.setStatus( "Waiting" ) ## get subrequest requestContainer.initiateSubRequest( "transfer" ) subRequestDict = { "Status" : "Waiting", "SubRequestID" : 2259916, "Operation" : "replicateAndRegister", "Arguments" : None, "ExecutionOrder" : 0, "SourceSE" : None, "TargetSE" : "GRIDKA_MC-DST,GRIDKA_MC_M-DST", "Catalogue" : None, "CreationTime" : "2011-02-19 04:57:02", "SubmissionTime" : "2011-02-19 04:57:02", "LastUpdate" : "2011-08-18 20:14:22" } requestContainer.setSubRequestAttributes( 0, "transfer", subRequestDict ) ## get subrequest files files = [ { "FileID" : 1610538, "LFN" : "/lhcb/MC/MC10/ALLSTREAMS.DST/00009422/0000/00009422_00000171_1.allstreams.dst", "Size" : None, "PFN" : None, "GUID" : None, "Md5" : None, "Addler" : None, "Attempt" : 1, "Status" : "Scheduled" } ] requestContainer.setSubRequestFiles( 0, "transfer", files ) return { "OK" : True, "Value" : { "RequestName" : "00009423_00000118", "RequestString" : requestContainer.toXML()["Value"], "JobID" : 0, "RequestContainer" : requestContainer } }
def getKwargsRemoveFile(): """ helper fcn to build request """ requestContainer = RequestContainer(init=False) requestContainer.setJobID(11111111) #requestContainer.setOwnerDN( "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=cibak/CN=605919/CN=Krzysztof Ciba" ) requestContainer.setOwnerGroup("lhcb_user") requestContainer.setDIRACSetup("LHCb-Production") requestContainer.setSourceComponent(None) requestContainer.setCreationTime("0000-00-00 00:00:00") requestContainer.setLastUpdate("2011-12-01 04:57:02") requestContainer.setStatus("Waiting") requestContainer.initiateSubRequest("removal") subRequestDict = { "Status": "Waiting", "SubRequestID": 2222222, "Operation": "removeFile", "Arguments": None, "ExecutionOrder": 0, "SourceSE": None, "TargetSE": "RAL-USER", "Catalogue": "LcgFileCatalogCombined", "CreationTime": "2011-12-01 04:57:02", "SubmissionTime": "2011-12-01 04:57:02", "LastUpdate": "2011-12-01 20:14:22" } requestContainer.setSubRequestAttributes(0, "removal", subRequestDict) files = [{ "FileID": 3333333, "LFN": "/lhcb/user/c/cibak/11889/11889410/test.zzz", "Size": 44444444, "PFN": "srm://srm-lhcb.gridpp.rl.ac.uk/castor/ads.rl.ac.uk/prod/lhcb/user/c/cibak/11889/11889410/test.zzz", "GUID": "5P13RD4L-4J5L-3D21-U5P1-3RD4L4J5P13R", "Md5": None, "Addler": "92b85e26", "Attempt": 1, "Status": "Waiting" }] requestContainer.setSubRequestFiles(0, "removal", files) return { "requestName": "00000001.xml", "requestString": requestContainer.toXML()["Value"], "jobID": 1, "executionOrder": 0, "sourceServer": "foobarserver", "configPath": "/Systems/DataManagement/Development/Agents/RemovalAgent" }
def getRequest(operation): """ fake requestDict :param str operation: sub-request operation attribute """ requestContainer = RequestContainer(init=False) requestContainer.setJobID(1) requestContainer.setOwnerDN("/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=cibak/CN=605919/CN=Krzysztof Ciba") requestContainer.setOwnerGroup("lhcb_user") requestContainer.setDIRACSetup("LHCb-Production") requestContainer.setSourceComponent(None) requestContainer.setCreationTime("0000-00-00 00:00:00") requestContainer.setLastUpdate("2011-12-01 04:57:02") requestContainer.setStatus("Waiting") requestContainer.setAttribute("RequestID", 123456789) requestContainer.initiateSubRequest("transfer") subRequestDict = { "Status": "Waiting", "SubRequestID": 2222222, "Operation": operation, "Arguments": None, "ExecutionOrder": 0, "SourceSE": None, "TargetSE": "CERN-USER,PIC-USER", "Catalogue": "LcgFileCatalogCombined", "CreationTime": "2011-12-01 04:57:02", "SubmissionTime": "2011-12-01 04:57:02", "LastUpdate": "2011-12-01 20:14:22", } requestContainer.setSubRequestAttributes(0, "transfer", subRequestDict) files = [ { "FileID": 3333333, "LFN": "/lhcb/user/c/cibak/11889/11889410/test.zzz", "Size": 44444444, "PFN": "srm://srm-lhcb.gridpp.rl.ac.uk/castor/ads.rl.ac.uk/prod/lhcb/user/c/cibak/11889/11889410/test.zzz", "GUID": "5P13RD4L-4J5L-3D21-U5P1-3RD4L4J5P13R", "Md5": None, "Addler": "92b85e26", "Attempt": 1, "Status": "Waiting", } ] requestContainer.setSubRequestFiles(0, "transfer", files) return requestContainer
def getKwargsRemoveFile(): """ helper fcn to build request """ requestContainer = RequestContainer( init = False ) requestContainer.setJobID( 11111111 ) #requestContainer.setOwnerDN( "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=cibak/CN=605919/CN=Krzysztof Ciba" ) requestContainer.setOwnerGroup( "lhcb_user" ) requestContainer.setDIRACSetup( "LHCb-Production" ) requestContainer.setSourceComponent( None ) requestContainer.setCreationTime( "0000-00-00 00:00:00" ) requestContainer.setLastUpdate( "2011-12-01 04:57:02" ) requestContainer.setStatus( "Waiting" ) requestContainer.initiateSubRequest( "removal" ) subRequestDict = { "Status" : "Waiting", "SubRequestID" : 2222222, "Operation" : "removeFile", "Arguments" : None, "ExecutionOrder" : 0, "SourceSE" : None, "TargetSE" : "RAL-USER", "Catalogue" : "LcgFileCatalogCombined", "CreationTime" : "2011-12-01 04:57:02", "SubmissionTime" : "2011-12-01 04:57:02", "LastUpdate" : "2011-12-01 20:14:22" } requestContainer.setSubRequestAttributes( 0, "removal", subRequestDict ) files = [ { "FileID" : 3333333, "LFN" : "/lhcb/user/c/cibak/11889/11889410/test.zzz", "Size" : 44444444, "PFN" : "srm://srm-lhcb.gridpp.rl.ac.uk/castor/ads.rl.ac.uk/prod/lhcb/user/c/cibak/11889/11889410/test.zzz", "GUID" : "5P13RD4L-4J5L-3D21-U5P1-3RD4L4J5P13R", "Md5" : None, "Addler" : "92b85e26", "Attempt" : 1, "Status" : "Waiting" } ] requestContainer.setSubRequestFiles( 0, "removal", files ) return { "requestName" : "00000001.xml", "requestString" : requestContainer.toXML()["Value"], "jobID" : 1, "executionOrder" : 0, "sourceServer" : "foobarserver", "configPath" : "/Systems/DataManagement/Development/Agents/RemovalAgent" }
class FailoverTransfer: ############################################################################# def __init__(self, requestObject=False): """ Constructor function, can specify request object to instantiate FailoverTransfer or a new request object is created. """ self.log = gLogger.getSubLogger("FailoverTransfer") self.rm = ReplicaManager() self.request = requestObject if not self.request: self.request = RequestContainer() self.request.setRequestName('default_request.xml') self.request.setSourceComponent('FailoverTransfer') ############################################################################# def transferAndRegisterFile(self, fileName, localPath, lfn, destinationSEList, fileGUID=None, fileCatalog=None): """Performs the transfer and register operation with failover. """ errorList = [] for se in destinationSEList: self.log.info( 'Attempting rm.putAndRegister("%s","%s","%s",guid="%s",catalog="%s")' % (lfn, localPath, se, fileGUID, fileCatalog)) result = self.rm.putAndRegister(lfn, localPath, se, guid=fileGUID, catalog=fileCatalog) self.log.verbose(result) if not result['OK']: self.log.error('rm.putAndRegister failed with message', result['Message']) errorList.append(result['Message']) continue if not result['Value']['Failed']: self.log.info( 'rm.putAndRegister successfully uploaded %s to %s' % (fileName, se)) return S_OK({'uploadedSE': se, 'lfn': lfn}) #Now we know something went wrong errorDict = result['Value']['Failed'][lfn] if not errorDict.has_key('register'): self.log.error('rm.putAndRegister failed with unknown error', str(errorDict)) errorList.append( 'Unknown error while attempting upload to %s' % se) continue fileDict = errorDict['register'] #Therefore the registration failed but the upload was successful if not fileCatalog: fileCatalog = '' result = self.__setRegistrationRequest(fileDict['LFN'], se, fileCatalog, fileDict) if not result['OK']: self.log.error( 'Failed to set registration request for: SE %s and metadata: \n%s' % (se, fileDict)) errorList.append( 'Failed to set registration request for: SE %s and metadata: \n%s' % (se, fileDict)) continue else: self.log.info( 'Successfully set registration request for: SE %s and metadata: \n%s' % (se, fileDict)) metadata = {} metadata['filedict'] = fileDict metadata['uploadedSE'] = se metadata['lfn'] = lfn metadata['registration'] = 'request' return S_OK(metadata) self.log.error( 'Encountered %s errors during attempts to upload output data' % len(errorList)) return S_ERROR('Failed to upload output data file') ############################################################################# def transferAndRegisterFileFailover(self, fileName, localPath, lfn, targetSE, failoverSEList, fileGUID=None, fileCatalog=None): """Performs the transfer and register operation to failover storage and sets the necessary replication and removal requests to recover. """ failover = self.transferAndRegisterFile(fileName, localPath, lfn, failoverSEList, fileGUID, fileCatalog) if not failover['OK']: self.log.error('Could not upload file to failover SEs', failover['Message']) return failover #set removal requests and replication requests result = self.__setFileReplicationRequest(lfn, targetSE) if not result['OK']: self.log.error('Could not set file replication request', result['Message']) return result lfn = failover['Value']['lfn'] failoverSE = failover['Value']['uploadedSE'] self.log.info( 'Attempting to set replica removal request for LFN %s at failover SE %s' % (lfn, failoverSE)) result = self.__setReplicaRemovalRequest(lfn, failoverSE) if not result['OK']: self.log.error('Could not set removal request', result['Message']) return result return S_OK('%s uploaded to a failover SE' % fileName) ############################################################################# def getRequestObject(self): """Returns the potentially modified request object in order to propagate changes. """ return S_OK(self.request) ############################################################################# def __setFileReplicationRequest(self, lfn, se): """ Sets a registration request. """ self.log.info('Setting replication request for %s to %s' % (lfn, se)) result = self.request.addSubRequest( { 'Attributes': { 'Operation': 'replicateAndRegister', 'TargetSE': se, 'ExecutionOrder': 0 } }, 'transfer') if not result['OK']: return result index = result['Value'] fileDict = {'LFN': lfn, 'Status': 'Waiting'} self.request.setSubRequestFiles(index, 'transfer', [fileDict]) return S_OK() ############################################################################# def __setRegistrationRequest(self, lfn, se, catalog, fileDict): """ Sets a registration request. """ self.log.info('Setting registration request for %s at %s.' % (lfn, se)) result = self.request.addSubRequest( { 'Attributes': { 'Operation': 'registerFile', 'ExecutionOrder': 0, 'TargetSE': se, 'Catalogue': catalog } }, 'register') if not result['OK']: return result index = result['Value'] if not fileDict.has_key('Status'): fileDict['Status'] = 'Waiting' self.request.setSubRequestFiles(index, 'register', [fileDict]) return S_OK() ############################################################################# def __setReplicaRemovalRequest(self, lfn, se): """ Sets a removal request for a replica. """ result = self.request.addSubRequest( { 'Attributes': { 'Operation': 'replicaRemoval', 'TargetSE': se, 'ExecutionOrder': 1 } }, 'removal') index = result['Value'] fileDict = {'LFN': lfn, 'Status': 'Waiting'} self.request.setSubRequestFiles(index, 'removal', [fileDict]) return S_OK() ############################################################################# def __setFileRemovalRequest(self, lfn, se='', pfn=''): """ Sets a removal request for a file including all replicas. """ result = self.request.addSubRequest( { 'Attributes': { 'Operation': 'removeFile', 'TargetSE': se, 'ExecutionOrder': 1 } }, 'removal') index = result['Value'] fileDict = {'LFN': lfn, 'PFN': pfn, 'Status': 'Waiting'} self.request.setSubRequestFiles(index, 'removal', [fileDict]) return S_OK()
class FailoverTransfer: ############################################################################# def __init__(self,requestObject=False): """ Constructor function, can specify request object to instantiate FailoverTransfer or a new request object is created. """ self.log = gLogger.getSubLogger( "FailoverTransfer" ) self.rm = ReplicaManager() self.request = requestObject if not self.request: self.request = RequestContainer() self.request.setRequestName('default_request.xml') self.request.setSourceComponent('FailoverTransfer') ############################################################################# def transferAndRegisterFile(self,fileName,localPath,lfn,destinationSEList,fileGUID=None,fileCatalog=None): """Performs the transfer and register operation with failover. """ errorList = [] for se in destinationSEList: self.log.info('Attempting rm.putAndRegister("%s","%s","%s",guid="%s",catalog="%s")' %(lfn,localPath,se,fileGUID,fileCatalog)) result = self.rm.putAndRegister(lfn,localPath,se,guid=fileGUID,catalog=fileCatalog) self.log.verbose(result) if not result['OK']: self.log.error('rm.putAndRegister failed with message',result['Message']) errorList.append(result['Message']) continue if not result['Value']['Failed']: self.log.info('rm.putAndRegister successfully uploaded %s to %s' %(fileName,se)) return S_OK({'uploadedSE':se,'lfn':lfn}) #Now we know something went wrong errorDict = result['Value']['Failed'][lfn] if not errorDict.has_key('register'): self.log.error('rm.putAndRegister failed with unknown error',str(errorDict)) errorList.append('Unknown error while attempting upload to %s' %se) continue fileDict = errorDict['register'] #Therefore the registration failed but the upload was successful if not fileCatalog: fileCatalog='' result = self.__setRegistrationRequest(fileDict['LFN'],se,fileCatalog,fileDict) if not result['OK']: self.log.error('Failed to set registration request for: SE %s and metadata: \n%s' %(se,fileDict)) errorList.append('Failed to set registration request for: SE %s and metadata: \n%s' %(se,fileDict)) continue else: self.log.info('Successfully set registration request for: SE %s and metadata: \n%s' %(se,fileDict)) metadata = {} metadata['filedict']=fileDict metadata['uploadedSE']=se metadata['lfn']=lfn metadata['registration']='request' return S_OK(metadata) self.log.error('Encountered %s errors during attempts to upload output data' %len(errorList)) return S_ERROR('Failed to upload output data file') ############################################################################# def transferAndRegisterFileFailover(self,fileName,localPath,lfn,targetSE,failoverSEList,fileGUID=None,fileCatalog=None): """Performs the transfer and register operation to failover storage and sets the necessary replication and removal requests to recover. """ failover = self.transferAndRegisterFile(fileName,localPath,lfn,failoverSEList,fileGUID,fileCatalog) if not failover['OK']: self.log.error('Could not upload file to failover SEs',failover['Message']) return failover #set removal requests and replication requests result = self.__setFileReplicationRequest(lfn,targetSE) if not result['OK']: self.log.error('Could not set file replication request',result['Message']) return result lfn = failover['Value']['lfn'] failoverSE = failover['Value']['uploadedSE'] self.log.info('Attempting to set replica removal request for LFN %s at failover SE %s' %(lfn,failoverSE)) result = self.__setReplicaRemovalRequest(lfn,failoverSE) if not result['OK']: self.log.error('Could not set removal request',result['Message']) return result return S_OK('%s uploaded to a failover SE' %fileName) ############################################################################# def getRequestObject(self): """Returns the potentially modified request object in order to propagate changes. """ return S_OK(self.request) ############################################################################# def __setFileReplicationRequest(self,lfn,se): """ Sets a registration request. """ self.log.info('Setting replication request for %s to %s' % (lfn,se)) result = self.request.addSubRequest({'Attributes':{'Operation':'replicateAndRegister', 'TargetSE':se,'ExecutionOrder':0}}, 'transfer') if not result['OK']: return result index = result['Value'] fileDict = {'LFN':lfn,'Status':'Waiting'} self.request.setSubRequestFiles(index,'transfer',[fileDict]) return S_OK() ############################################################################# def __setRegistrationRequest(self,lfn,se,catalog,fileDict): """ Sets a registration request. """ self.log.info('Setting registration request for %s at %s.' % (lfn,se)) result = self.request.addSubRequest({'Attributes':{'Operation':'registerFile','ExecutionOrder':0, 'TargetSE':se,'Catalogue':catalog}},'register') if not result['OK']: return result index = result['Value'] if not fileDict.has_key('Status'): fileDict['Status']='Waiting' self.request.setSubRequestFiles(index,'register',[fileDict]) return S_OK() ############################################################################# def __setReplicaRemovalRequest(self,lfn,se): """ Sets a removal request for a replica. """ result = self.request.addSubRequest({'Attributes':{'Operation':'replicaRemoval', 'TargetSE':se,'ExecutionOrder':1}}, 'removal') index = result['Value'] fileDict = {'LFN':lfn,'Status':'Waiting'} self.request.setSubRequestFiles(index,'removal',[fileDict]) return S_OK() ############################################################################# def __setFileRemovalRequest(self,lfn,se='',pfn=''): """ Sets a removal request for a file including all replicas. """ result = self.request.addSubRequest({'Attributes':{'Operation':'removeFile', 'TargetSE':se,'ExecutionOrder':1}}, 'removal') index = result['Value'] fileDict = {'LFN':lfn,'PFN':pfn,'Status':'Waiting'} self.request.setSubRequestFiles(index,'removal',[fileDict]) return S_OK()
def getRequest(self, requestType): """ Get a request of a given type eligible for execution """ # RG: What if requestType is not given? # the first query will return nothing. # KC: maybe returning S_ERROR would be enough? # alternatively we should check if requestType is known (in 'transfer', 'removal', 'register' and 'diset') if not requestType or type(requestType) not in types.StringTypes: return S_ERROR("Request type not given.") myRequestType = self._escapeString(requestType) if not myRequestType: return myRequestType myRequestType = myRequestType['Value'] start = time.time() dmRequest = RequestContainer(init=False) requestID = 0 subIDList = [] fields = [ 'RequestID', 'SubRequestID', 'Operation', 'Arguments', 'ExecutionOrder', 'SourceSE', 'TargetSE', 'Catalogue', 'CreationTime', 'SubmissionTime', 'LastUpdate' ] # get the pending SubRequest sorted by ExecutionOrder and LastUpdate req = "SELECT RequestID, ExecutionOrder, Status, RequestType, LastUpdate from SubRequests WHERE Status IN ( 'Waiting', 'Assigned' ) ORDER BY ExecutionOrder, LastUpdate" # now get sorted list of RequestID (according to the above) req = "SELECT * from ( %s ) as T1 GROUP BY RequestID" % req # and get the 100 oldest ones of Type requestType req = "SELECT RequestID, ExecutionOrder FROM ( %s ) as T2 WHERE RequestType = %s ORDER BY LastUpdate limit 100" % ( req, myRequestType) # and now get all waiting SubRequest for the selected RequestID and ExecutionOrder req = "SELECT A.%s FROM SubRequests AS A, ( %s ) AS B WHERE " % ( ', A.'.join(fields), req) req = "%s A.RequestID = B.RequestID AND A.ExecutionOrder = B.ExecutionOrder AND A.Status = 'Waiting' AND A.RequestType = %s;" % ( req, myRequestType) result = self._query(req) if not result['OK']: err = 'RequestDB._getRequest: Failed to retrieve Requests' return S_ERROR('%s\n%s' % (err, result['Message'])) if not result['Value']: return S_OK() # We get up to 10 Request candidates, to add some randomness reqDict = {} for row in result['Value']: reqDict.setdefault(row[0], []) reqDict[row[0]].append(row[1:]) reqIDList = reqDict.keys() random.shuffle(reqIDList) for reqID in reqIDList: sidList = [x[0] for x in reqDict[reqID]] for subID in sidList: req = "UPDATE SubRequests SET Status='Assigned' WHERE RequestID=%s AND SubRequestID=%s;" % ( reqID, subID) resAssigned = self._update(req) if not resAssigned['OK']: if subIDList: self.__releaseSubRequests(reqID, subIDList) return S_ERROR('Failed to assign subrequests: %s' % resAssigned['Message']) if resAssigned['Value'] == 0: # Somebody has assigned this request gLogger.warn( 'Already assigned subrequest %d of request %d' % (subID, reqID)) else: subIDList.append(subID) if subIDList: # We managed to get some requests, can continue now requestID = reqID break # Haven't succeeded to get any request if not requestID: return S_OK() dmRequest.setRequestID(requestID) fields = [ 'FileID', 'LFN', 'Size', 'PFN', 'GUID', 'Md5', 'Addler', 'Attempt', 'Status' ] for subRequestID, operation, arguments, executionOrder, sourceSE, targetSE, catalogue, creationTime, submissionTime, lastUpdate in reqDict[ requestID]: if not subRequestID in subIDList: continue res = dmRequest.initiateSubRequest(requestType) ind = res['Value'] subRequestDict = { 'Status': 'Waiting', 'SubRequestID': subRequestID, 'Operation': operation, 'Arguments': arguments, 'ExecutionOrder': int(executionOrder), 'SourceSE': sourceSE, 'TargetSE': targetSE, 'Catalogue': catalogue, 'CreationTime': creationTime, 'SubmissionTime': submissionTime, 'LastUpdate': lastUpdate } res = dmRequest.setSubRequestAttributes(ind, requestType, subRequestDict) if not res['OK']: err = 'RequestDB._getRequest: Failed to set subRequest attributes for RequestID %s' % requestID self.__releaseSubRequests(requestID, subIDList) return S_ERROR('%s\n%s' % (err, res['Message'])) req = "SELECT %s FROM Files WHERE SubRequestID = %s ORDER BY FileID;" % ( ', '.join(fields), subRequestID) res = self._query(req) if not res['OK']: err = 'RequestDB._getRequest: Failed to get File attributes for RequestID %s.%s' % ( requestID, subRequestID) self.__releaseSubRequests(requestID, subIDList) return S_ERROR('%s\n%s' % (err, res['Message'])) files = [] for fileID, lfn, size, pfn, guid, md5, addler, attempt, status in res[ 'Value']: fileDict = { 'FileID': fileID, 'LFN': lfn, 'Size': size, 'PFN': pfn, 'GUID': guid, 'Md5': md5, 'Addler': addler, 'Attempt': attempt, 'Status': status } files.append(fileDict) res = dmRequest.setSubRequestFiles(ind, requestType, files) if not res['OK']: err = 'RequestDB._getRequest: Failed to set files into Request for RequestID %s.%s' % ( requestID, subRequestID) self.__releaseSubRequests(requestID, subIDList) return S_ERROR('%s\n%s' % (err, res['Message'])) req = "SELECT Dataset,Status FROM Datasets WHERE SubRequestID = %s;" % subRequestID res = self._query(req) if not res['OK']: err = 'RequestDB._getRequest: Failed to get Datasets for RequestID %s.%s' % ( requestID, subRequestID) self.__releaseSubRequests(requestID, subIDList) return S_ERROR('%s\n%s' % (err, res['Message'])) datasets = [] for dataset, status in res['Value']: datasets.append(dataset) res = dmRequest.setSubRequestDatasets(ind, requestType, datasets) if not res['OK']: err = 'RequestDB._getRequest: Failed to set datasets into Request for RequestID %s.%s' % ( requestID, subRequestID) self.__releaseSubRequests(requestID, subIDList) return S_ERROR('%s\n%s' % (err, res['Message'])) fields = [ 'RequestName', 'JobID', 'OwnerDN', 'OwnerGroup', 'DIRACSetup', 'SourceComponent', 'CreationTime', 'SubmissionTime', 'LastUpdate' ] req = "SELECT %s from Requests WHERE RequestID = %s;" % ( ', '.join(fields), requestID) res = self._query(req) if not res['OK']: err = 'RequestDB._getRequest: Failed to retrieve max RequestID' self.__releaseSubRequests(requestID, subIDList) return S_ERROR('%s\n%s' % (err, res['Message'])) requestName, jobID, ownerDN, ownerGroup, diracSetup, sourceComponent, creationTime, submissionTime, lastUpdate = res[ 'Value'][0] dmRequest.setRequestName(requestName) dmRequest.setJobID(jobID) dmRequest.setOwnerDN(ownerDN) dmRequest.setOwnerGroup(ownerGroup) dmRequest.setDIRACSetup(diracSetup) dmRequest.setSourceComponent(sourceComponent) dmRequest.setCreationTime(str(creationTime)) dmRequest.setLastUpdate(str(lastUpdate)) res = dmRequest.toXML() if not res['OK']: err = 'RequestDB._getRequest: Failed to create XML for RequestID %s' % ( requestID) self.__releaseSubRequests(requestID, subIDList) return S_ERROR('%s\n%s' % (err, res['Message'])) requestString = res['Value'] #still have to manage the status of the dataset properly resultDict = {} resultDict['RequestName'] = requestName resultDict['RequestString'] = requestString resultDict['JobID'] = jobID return S_OK(resultDict)
def getRequestForSubRequest(self, itself, subRequestID): """ Select Request given SubRequestID. :param self: plugin reference :param itself: patient reference for injection :param int subRequestID: SubRequests.SubRequestID :warn: function has to be injected to RequestDBMySQL instance """ ## get RequestID requestID = "SELECT RequestID FROM SubRequests WHERE SubRequestID = %s;" % str( subRequestID) requestID = self._query(requestID) if not requestID["OK"]: return requestID requestID = requestID["Value"][0] ## create RequestContainer requestContainer = RequestContainer(init=False) requestContainer.setRequestID(requestID) ## put some basic infos in requestInfo = "SELECT RequestName, JobID, OwnerDN, OwnerGroup, DIRACSetup, SourceComponent, CreationTime, SubmissionTime, LastUpdate, Status " requestInfo += "FROM Requests WHERE RequestID = %d;" % requestID requestInfo = self._query(requestInfo) if not requestInfo["OK"]: return requestInfo requestName, jobID, ownerDN, ownerGroup, diracSetup, sourceComponent, creationTime, submissionTime, lastUpdate, status = requestInfo[ 'Value'][0] requestContainer.setRequestName(requestName) requestContainer.setJobID(jobID) requestContainer.setOwnerDN(ownerDN) requestContainer.setOwnerGroup(ownerGroup) requestContainer.setDIRACSetup(diracSetup) requestContainer.setSourceComponent(sourceComponent) requestContainer.setCreationTime(str(creationTime)) requestContainer.setLastUpdate(str(lastUpdate)) requestContainer.setStatus(status) ## get sub-requests subRequests = "SELECT SubRequestID, Status, RequestType, Operation, Arguments, ExecutionOrder, SourceSE, " subRequests += "TargetSE, Catalogue, CreationTime, SubmissionTime, LastUpdate FROM SubRequests WHERE RequestID=%s;" % requestID subRequests = self._query(subRequests) if not subRequests["OK"]: return subRequests ## loop over sub requests for subRequestID, status, requestType, operation, arguments, executionOrder, sourceSE, targetSE, catalogue, creationTime, submissionTime, lastUpdate in subRequests[ "Value"]: res = requestContainer.initiateSubRequest(requestType) ind = res["Value"] subRequestDict = { "Status": status, "SubRequestID": subRequestID, "Operation": operation, "Arguments": arguments, "ExecutionOrder": int(executionOrder), "SourceSE": sourceSE, "TargetSE": targetSE, "Catalogue": catalogue, "CreationTime": creationTime, "SubmissionTime": submissionTime, "LastUpdate": lastUpdate } res = requestContainer.setSubRequestAttributes( ind, requestType, subRequestDict) if not res["OK"]: return res ## get files for this subrequest req = "SELECT FileID, LFN, Size, PFN, GUID, Md5, Addler, Attempt, Status FROM Files WHERE SubRequestID = %s ORDER BY FileID;" % str( subRequestID) res = self._query(req) if not res["OK"]: return res files = [] for fileID, lfn, size, pfn, guid, md5, addler, attempt, status in res[ "Value"]: fileDict = { "FileID": fileID, "LFN": lfn, "Size": size, "PFN": pfn, "GUID": guid, "Md5": md5, "Addler": addler, "Attempt": attempt, "Status": status } files.append(fileDict) res = requestContainer.setSubRequestFiles(ind, requestType, files) if not res["OK"]: return res ## dump request to XML res = requestContainer.toXML() if not res["OK"]: return res requestString = res["Value"] ## return dictonary with all info in at least return S_OK({ "RequestName": requestName, "RequestString": requestString, "JobID": jobID, "RequestContainer": requestContainer })
class FailoverRequest(ModuleBase): """ Handle the failover requests issued by previous steps. Used in production. """ ############################################################################# def __init__(self): """Module initialization. """ super(FailoverRequest, self).__init__() self.version = __RCSID__ self.log = gLogger.getSubLogger( "FailoverRequest" ) #Internal parameters self.enable = True self.jobID = '' self.productionID = None self.prodJobID = None #Workflow parameters self.jobReport = None self.fileReport = None self.request = None ############################################################################# def applicationSpecificInputs(self): """ By convention the module input parameters are resolved here. """ self.log.debug(self.workflow_commons) self.log.debug(self.step_commons) if os.environ.has_key('JOBID'): self.jobID = os.environ['JOBID'] self.log.verbose('Found WMS JobID = %s' %self.jobID) else: self.log.info('No WMS JobID found, disabling module via control flag') self.enable = False if self.step_commons.has_key('Enable'): self.enable = self.step_commons['Enable'] if not type(self.enable) == type(True): self.log.warn('Enable flag set to non-boolean value %s, setting to False' % self.enable) self.enable = False #Earlier modules will have populated the report objects if self.workflow_commons.has_key('JobReport'): self.jobReport = self.workflow_commons['JobReport'] if self.workflow_commons.has_key('FileReport'): self.fileReport = self.workflow_commons['FileReport'] if self.InputData: if type(self.InputData) != type([]): self.InputData = self.InputData.split(';') self.InputData = [x.replace('LFN:','') for x in self.InputData] if self.workflow_commons.has_key('Request'): self.request = self.workflow_commons['Request'] if not self.request: self.request = RequestContainer() self.request.setRequestName('job_%s_request.xml' % self.jobID) self.request.setJobID(self.jobID) self.request.setSourceComponent("Job_%s" % self.jobID) if self.workflow_commons.has_key('PRODUCTION_ID'): self.productionID = self.workflow_commons['PRODUCTION_ID'] if self.workflow_commons.has_key('JOB_ID'): self.prodJobID = self.workflow_commons['JOB_ID'] return S_OK('Parameters resolved') ############################################################################# def execute(self): """ Main execution function. """ self.log.info('Initializing %s' % self.version) result = self.resolveInputVariables() if not result['OK']: self.log.error(result['Message']) return result if not self.fileReport: self.fileReport = FileReport('Transformation/TransformationManager') if self.InputData: inputFiles = self.fileReport.getFiles() for lfn in self.InputData: if not lfn in inputFiles: self.log.verbose('No status populated for input data %s, setting to "Unused"' % lfn) result = self.fileReport.setFileStatus(int(self.productionID), lfn, 'Unused') if not self.workflowStatus['OK'] or not self.stepStatus['OK']: self.log.info('Workflow status = %s, step status = %s' %(self.workflowStatus['OK'], self.stepStatus['OK'])) inputFiles = self.fileReport.getFiles() for lfn in inputFiles: if inputFiles[lfn] != 'ApplicationCrash': self.log.info('Forcing status to "Unused" due to workflow failure for: %s' % (lfn)) self.fileReport.setFileStatus(int(self.productionID), lfn, 'Unused') else: inputFiles = self.fileReport.getFiles() if inputFiles: self.log.info('Workflow status OK, setting input file status to Processed') for lfn in inputFiles: self.log.info('Setting status to "Processed" for: %s' % (lfn)) self.fileReport.setFileStatus(int(self.productionID), lfn, 'Processed') result = self.fileReport.commit() if not result['OK']: self.log.error('Failed to report file status to ProductionDB, request will be generated', result['Message']) else: self.log.info('Status of files have been properly updated in the ProcessingDB') # Must ensure that the local job report instance is used to report the final status # in case of failure and a subsequent failover operation if self.workflowStatus['OK'] and self.stepStatus['OK']: if not self.jobReport: self.jobReport = JobReport(int(self.jobID)) jobStatus = self.jobReport.setApplicationStatus('Job Finished Successfully') if not jobStatus['OK']: self.log.warn(jobStatus['Message']) # Retrieve the accumulated reporting request reportRequest = None if self.jobReport: result = self.jobReport.generateRequest() if not result['OK']: self.log.warn('Could not generate request for job report with result:\n%s' % (result)) else: reportRequest = result['Value'] if reportRequest: self.log.info('Populating request with job report information') self.request.update(reportRequest) fileReportRequest = None if self.fileReport: result = self.fileReport.generateRequest() if not result['OK']: self.log.warn('Could not generate request for file report with result:\n%s' % (result)) else: fileReportRequest = result['Value'] if fileReportRequest: self.log.info('Populating request with file report information') result = self.request.update(fileReportRequest) accountingReport = None if self.workflow_commons.has_key('AccountingReport'): accountingReport = self.workflow_commons['AccountingReport'] if accountingReport: result = accountingReport.commit() if not result['OK']: self.log.info('Populating request with accounting report information') self.request.setDISETRequest(result['rpcStub']) if self.request.isEmpty()['Value']: self.log.info('Request is empty, nothing to do.') return self.finalize() request_string = self.request.toXML()['Value'] self.log.debug(request_string) # Write out the request string fname = '%s_%s_request.xml' % (self.productionID, self.prodJobID) xmlfile = open(fname, 'w') xmlfile.write(request_string) xmlfile.close() self.log.info('Creating failover request for deferred operations for job %s:' % self.jobID) result = self.request.getDigest() if result['OK']: digest = result['Value'] self.log.info(digest) if not self.enable: self.log.info('Module is disabled by control flag') return S_OK('Module is disabled by control flag') return self.finalize() ############################################################################# def finalize(self): """ Finalize and report correct status for the workflow based on the workflow or step status. """ self.log.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK'])) if not self.workflowStatus['OK'] or not self.stepStatus['OK']: self.log.warn('Workflow status is not ok, will not overwrite status') self.log.info('Workflow failed, end of FailoverRequest module execution.') return S_ERROR('Workflow failed, FailoverRequest module completed') self.log.info('Workflow successful, end of FailoverRequest module execution.') return S_OK('FailoverRequest module completed') #EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#
class UploadOutputData(ModuleBase): """ As name suggest: upload output data. For Production only: See L{UserJobFinalization} for User job upload. """ ############################################################################# def __init__(self): """Module initialization. """ super(UploadOutputData, self).__init__() self.version = __RCSID__ self.log = gLogger.getSubLogger( "UploadOutputData" ) self.commandTimeOut = 10*60 self.enable = True self.failoverTest = False #flag to put file to failover SE by default self.failoverSEs = gConfig.getValue('/Resources/StorageElementGroups/Tier1-Failover', []) self.ops = Operations() #List all parameters here self.outputDataFileMask = '' self.outputMode = 'Any' #or 'Local' for reco case self.outputList = [] self.request = None self.PRODUCTION_ID = "" self.prodOutputLFNs = [] self.experiment = "CLIC" ############################################################################# def applicationSpecificInputs(self): """ By convention the module parameters are resolved here. """ self.log.verbose("Workflow commons:") self.log.verbose(self.workflow_commons) self.log.verbose("Step commons:") self.log.verbose(self.step_commons) if self.step_commons.has_key('Enable'): self.enable = self.step_commons['Enable'] if not type(self.enable) == type(True): self.log.warn('Enable flag set to non-boolean value %s, setting to False' % self.enable) self.enable = False if self.step_commons.has_key('TestFailover'): self.enable = self.step_commons['TestFailover'] if not type(self.failoverTest) == type(True): self.log.warn('Test failover flag set to non-boolean value %s, setting to False' % self.failoverTest) self.failoverTest = False if self.workflow_commons.has_key("PRODUCTION_ID"): self.PRODUCTION_ID = self.workflow_commons["PRODUCTION_ID"] if os.environ.has_key('JOBID'): self.log.verbose('Found WMS JobID = %s' % self.jobID) else: self.log.info('No WMS JobID found, disabling module via control flag') self.enable = False if self.workflow_commons.has_key('Request'): self.request = self.workflow_commons['Request'] else: self.request = RequestContainer() self.request.setRequestName('job_%s_request.xml' % self.jobID) self.request.setJobID(self.jobID) self.request.setSourceComponent("Job_%s" % self.jobID) ##This is the thing that is used to establish the list of outpufiles to treat: ## Make sure that all that is in the : "listoutput" and also in the ProductionData ## is treated properly. Needed as whatever is in listoutput does not contain any reference to the ## prodID and task ID. Also if for some reason a step failed, then the corresponding data will not be there if self.workflow_commons.has_key('outputList'): self.outputList = self.workflow_commons['outputList'] if self.workflow_commons.has_key('ProductionOutputData'): proddata = self.workflow_commons['ProductionOutputData'].split(";") self.log.verbose("prod data : %s" % proddata ) olist = {} for obj in self.outputList: fname_in_outputlist = obj['outputFile'].lower() extension = '' if fname_in_outputlist.count("_sim") or fname_in_outputlist.count("_rec") or fname_in_outputlist.count("_dst"): extension = ".slcio" elif fname_in_outputlist.count("_gen"): extension = ".stdhep" fname_in_outputlist = fname_in_outputlist.replace(extension,"") for prodfile in proddata: prodfile = os.path.basename(prodfile) extension = '' if prodfile.count("_sim") or prodfile.count("_rec") or prodfile.count("_dst"): extension = ".slcio" elif prodfile.count("_gen"): extension = ".stdhep" prodfile = prodfile.replace(extension,"") if olist.has_key(prodfile): ## This has already been treated, no need to come back to it. continue appdict = {} if (fname_in_outputlist.count("_gen")):# and prodfile.lower().count("_gen_")) : genf = obj['outputFile'].split("_gen")[0] genf += "_gen" if (prodfile.count(genf)): appdict.update(obj) appdict['outputFile'] = prodfile+extension olist[prodfile] = appdict if (fname_in_outputlist.count("_sim")): simf = obj['outputFile'].split("_sim")[0] simf += "_sim" if (prodfile.count(simf)): appdict.update(obj) appdict['outputFile'] = prodfile+extension olist[prodfile] = appdict self.log.verbose('olist %s'%olist) if (fname_in_outputlist.count("_rec")): recf = obj['outputFile'].split("_rec")[0] recf += "_rec" if (prodfile.count(recf)): appdict.update(obj) appdict['outputFile'] = prodfile+extension olist[prodfile] = appdict break if (fname_in_outputlist.count("_dst") and prodfile.lower().count("_dst_")): dstf = obj['outputFile'].split("_dst")[0] dstf += "_dst" if (prodfile.count(dstf)): appdict.update(obj) appdict['outputFile'] = prodfile+extension olist[prodfile] = appdict break self.outputList = olist.values() else: olist = [] for obj in self.outputList: appdict = obj appdict['outputFile'] = getProdFilename(obj['outputFile'], int(self.workflow_commons["PRODUCTION_ID"]), int(self.workflow_commons["JOB_ID"])) olist.append(appdict) self.outputList = olist self.log.verbose("OutputList : %s" % self.outputList) if self.workflow_commons.has_key('outputMode'): self.outputMode = self.workflow_commons['outputMode'] if self.workflow_commons.has_key('outputDataFileMask'): self.outputDataFileMask = self.workflow_commons['outputDataFileMask'] if not type(self.outputDataFileMask) == type([]): self.outputDataFileMask = [i.lower().strip() for i in self.outputDataFileMask.split(';')] #result = constructProductionLFNs(self.workflow_commons) #if not result['OK']: # self.log.error('Could not create production LFNs',result['Message']) # return result #self.prodOutputLFNs=result['Value']['ProductionOutputData'] if self.workflow_commons.has_key('ProductionOutputData'): self.prodOutputLFNs = self.workflow_commons['ProductionOutputData'].split(";") else: self.prodOutputLFNs = [] return S_OK('Parameters resolved') ############################################################################# def execute(self): """ Main execution function. """ self.log.info('Initializing %s' % self.version) result = self.resolveInputVariables() if not result['OK']: self.log.error(result['Message']) return result if not self.workflowStatus['OK'] or not self.stepStatus['OK']: self.log.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK'])) return S_OK('No output data upload attempted') ##determine the experiment example_file = self.prodOutputLFNs[0] if "/ilc/prod/clic" in example_file: self.experiment = "CLIC" elif "/ilc/prod/ilc/sid" in example_file: self.experiment = 'ILC_SID' elif "/ilc/prod/ilc/mc-dbd" in example_file: self.experiment = 'ILC_ILD' else: self.log.warn("Failed to determine experiment, reverting to default") #Determine the final list of possible output files for the #workflow and all the parameters needed to upload them. result = self.getCandidateFiles(self.outputList, self.prodOutputLFNs, self.outputDataFileMask) if not result['OK']: self.setApplicationStatus(result['Message']) return result fileDict = result['Value'] result = self.getFileMetadata(fileDict) if not result['OK']: self.setApplicationStatus(result['Message']) return result if not result['Value']: self.log.info('No output data files were determined to be uploaded for this workflow') return S_OK() fileMetadata = result['Value'] #Get final, resolved SE list for files final = {} for fileName, metadata in fileMetadata.items(): result = getDestinationSEList(metadata['workflowSE'], DIRAC.siteName(), self.outputMode) if not result['OK']: self.log.error('Could not resolve output data SE', result['Message']) self.setApplicationStatus('Failed To Resolve OutputSE') return result resolvedSE = result['Value'] final[fileName] = metadata final[fileName]['resolvedSE'] = resolvedSE self.log.info('The following files will be uploaded: %s' % (string.join(final.keys(), ', '))) for fileName, metadata in final.items(): self.log.info('--------%s--------' % fileName) for n, v in metadata.items(): self.log.info('%s = %s' % (n, v)) #At this point can exit and see exactly what the module would have uploaded if not self.enable: self.log.info('Module is disabled by control flag, would have attempted to upload the \ following files %s' % string.join(final.keys(), ', ')) return S_OK('Module is disabled by control flag') #Disable the watchdog check in case the file uploading takes a long time self.log.info('Creating DISABLE_WATCHDOG_CPU_WALLCLOCK_CHECK in order to disable the Watchdog prior to upload') fopen = open('DISABLE_WATCHDOG_CPU_WALLCLOCK_CHECK','w') fopen.write('%s' % time.asctime()) fopen.close() #Instantiate the failover transfer client with the global request object failoverTransfer = FailoverTransfer(self.request) catalogs = ['FileCatalog', 'LcgFileCatalog'] #One by one upload the files with failover if necessary failover = {} if not self.failoverTest: for fileName, metadata in final.items(): self.log.info("Attempting to store file %s to the following SE(s):\n%s" % (fileName, string.join(metadata['resolvedSE'], ', '))) result = failoverTransfer.transferAndRegisterFile(fileName, metadata['localpath'], metadata['lfn'], metadata['resolvedSE'], fileGUID = metadata['guid'], fileCatalog = catalogs) if not result['OK']: self.log.error('Could not transfer and register %s with metadata:\n %s' % (fileName, metadata)) failover[fileName] = metadata else: lfn = metadata['lfn'] else: failover = final self.failoverSEs = self.ops.getValue("Production/%s/FailOverSE" % self.experiment, self.failoverSEs) cleanUp = False for fileName, metadata in failover.items(): self.log.info('Setting default catalog for failover transfer to FileCatalog') random.shuffle(self.failoverSEs) targetSE = metadata['resolvedSE'][0] metadata['resolvedSE'] = self.failoverSEs result = failoverTransfer.transferAndRegisterFileFailover(fileName, metadata['localpath'], metadata['lfn'], targetSE, metadata['resolvedSE'], fileGUID = metadata['guid'], fileCatalog = catalogs) if not result['OK']: self.log.error('Could not transfer and register %s with metadata:\n %s' % (fileName, metadata)) cleanUp = True break #no point continuing if one completely fails os.remove("DISABLE_WATCHDOG_CPU_WALLCLOCK_CHECK") #cleanup the mess #Now after all operations, retrieve potentially modified request object result = failoverTransfer.getRequestObject() if not result['OK']: self.log.error(result) return S_ERROR('Could not retrieve modified request') self.request = result['Value'] #If some or all of the files failed to be saved to failover if cleanUp: lfns = [] for fileName, metadata in final.items(): lfns.append(metadata['lfn']) result = self.__cleanUp(lfns) self.workflow_commons['Request'] = self.request return S_ERROR('Failed to upload output data') # #Can now register the successfully uploaded files in the BK # if not performBKRegistration: # self.log.info('There are no files to perform the BK registration for, all could be saved to failover') # else: # rm = ReplicaManager() # result = rm.addCatalogFile(performBKRegistration,catalogs=['BookkeepingDB']) # self.log.verbose(result) # if not result['OK']: # self.log.error(result) # return S_ERROR('Could Not Perform BK Registration') # if result['Value']['Failed']: # for lfn,error in result['Value']['Failed'].items(): # self.log.info('BK registration for %s failed with message: "%s" setting failover request' %(lfn,error)) # result = self.request.addSubRequest({'Attributes':{'Operation':'registerFile','ExecutionOrder':0, 'Catalogue':'BookkeepingDB'}},'register') # if not result['OK']: # self.log.error('Could not set registerFile request:\n%s' %result) # return S_ERROR('Could Not Set BK Registration Request') # fileDict = {'LFN':lfn,'Status':'Waiting'} # index = result['Value'] # self.request.setSubRequestFiles(index,'register',[fileDict]) self.workflow_commons['Request'] = self.request return S_OK('Output data uploaded') ############################################################################# def __cleanUp(self, lfnList): """ Clean up uploaded data for the LFNs in the list """ # Clean up the current request for req_type in ['transfer', 'register']: for lfn in lfnList: result = self.request.getNumSubRequests(req_type) if result['OK']: nreq = result['Value'] if nreq: # Go through subrequests in reverse order in order not to spoil the numbering ind_range = [0] if nreq > 1: ind_range = range(nreq-1, -1, -1) for i in ind_range: result = self.request.getSubRequestFiles(i, req_type) if result['OK']: fileList = result['Value'] if fileList[0]['LFN'] == lfn: result = self.request.removeSubRequest(i, req_type) # Set removal requests just in case for lfn in lfnList: result = self.request.addSubRequest({'Attributes': {'Operation' : 'removeFile', 'TargetSE' : '', 'ExecutionOrder' : 1}}, 'removal') index = result['Value'] fileDict = {'LFN':lfn, 'PFN':'', 'Status':'Waiting'} self.request.setSubRequestFiles(index, 'removal', [fileDict]) return S_OK() #EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#
class HandleProdOutputData(ModuleBase): def __init__(self): super(HandleProdOutputData, self).__init__() self.result = S_ERROR() self.fc = FileCatalogClient() self.rm = ReplicaManager() self.destination = '' self.basepath = '/ilc/prod' self.dataattributes = '' self.attributesdict = {} self.generatorfile = '' self.mokkafile = '' self.marlinfiles = '' self.slicfile = '' self.lcsimfiles = '' self.request = None self.failoverTest=False self.log = gLogger.getSubLogger( "HandleOutputData" ) def applicationSpecificInputs(self): """ Resolve all input variables for the module here. @return: S_OK() """ if self.step_commons.has_key('DataAttributes'): self.dataattributes = self.step_commons['DataAttributes'] else: return S_ERROR('No data attributes found, cannot proceed with registration in Catalog, ABORT!') for attribute in self.dataattributes.split(";"): if self.step_commons.has_key(attribute): self.attributesdict[attribute] = self.step_commons[attribute] if self.step_commons.has_key("destination"): self.destination = self.step_commons['destination'] if self.step_commons.has_key('GENFile'): self.generatorfile = self.step_commons['GENFile'] if self.step_commons.has_key('MokkaFile'): self.mokkafile = self.step_commons['MokkaFile'] if self.step_commons.has_key('MarlinFiles'): self.marlinfiles = self.step_commons['MarlinFiles'].split(';') if self.step_commons.has_key('SLICFile'): self.slicfile = self.step_commons['SLICFile'] if self.step_commons.has_key('LCSIMFiles'): self.lcsimfiles = self.step_commons['LCSIMFiles'].split(';') if self.workflow_commons.has_key('Request'): self.request = self.workflow_commons['Request'] else: self.request = RequestContainer() self.request.setRequestName('job_%s_request.xml' % self.jobID) self.request.setJobID(self.jobID) self.request.setSourceComponent("Job_%s" % self.jobID) return S_OK('Parameters resolved') def execute(self): #Have to work out if the module is part of the last step i.e. #user jobs can have any number of steps and we only want #to run the finalization once. currentStep = int(self.step_commons['STEP_NUMBER']) totalSteps = int(self.workflow_commons['TotalSteps']) if currentStep==totalSteps: self.lastStep=True else: self.log.verbose('Current step = %s, total steps of workflow = %s, HandleProdOutputData will enable itself only at the last workflow step.' %(currentStep,totalSteps)) if not self.lastStep: return S_OK() self.result =self.resolveInputVariables() if not self.result['OK']: self.log.error(self.result['Message']) return self.result ###Instantiate object that will ensure that the files are registered properly failoverTransfer = FailoverTransfer(self.request) datatohandle = {} if self.generatorfile: if not os.path.exists(self.generatorfile): return S_ERROR("File %s does not exist, something went wrong before !"%(self.generatorfile)) self.attributesdict['DataType'] = 'gen' lfnpath = string.join([self.basepath,self.attributesdict['Machine'],self.attributesdict['Energy'], self.attributesdict['DataType'],self.attributesdict['EvtType'],self.attributesdict['ProdID'], self.generatorfile],"/") datatohandle[self.generatorfile]={'lfn':lfnpath,'type':'gen','workflowSE':self.destination} if self.mokkafile or self.slicfile: recofile = '' if self.mokkafile and not os.path.exists(self.mokkafile): return S_ERROR("File %s does not exist, something went wrong before !"%(self.mokkafile)) else: recofile = self.mokkafile if self.slicfile and not os.path.exists(self.slicfile): return S_ERROR("File %s does not exist, something went wrong before !"%(self.slicfile)) else: recofile = self.slicfile self.attributesdict['DataType'] = 'SIM' lfnpath = string.join([self.basepath,self.attributesdict['Machine'],self.attributesdict['Energy'], self.attributesdict['DetectorModel'],self.attributesdict['DataType'],self.attributesdict['EvtType'], self.attributesdict['ProdID'],recofile],"/") datatohandle[recofile]={'lfn':lfnpath,'type':'gen','workflowSE':self.destination} ##Below, look in file name if it contain REC or DST, to determine the data type. if self.marlinfiles: for file in self.marlinfiles: if file.find("REC")>-1: self.attributesdict['DataType'] = 'REC' if file.find("DST")>-1: self.attributesdict['DataType'] = 'DST' lfnpath = string.join([self.basepath,self.attributesdict['Machine'],self.attributesdict['Energy'], self.attributesdict['DetectorModel'],self.attributesdict['DataType'],self.attributesdict['EvtType'], self.attributesdict['ProdID'],file],"/") datatohandle[file]={'lfn':lfnpath,'type':'gen','workflowSE':self.destination} if self.lcsimfiles: for file in self.lcsimfiles: if file.find("DST")>-1: self.attributesdict['DataType'] = 'DST' lfnpath = string.join([self.basepath,self.attributesdict['Machine'],self.attributesdict['Energy'], self.attributesdict['DetectorModel'],self.attributesdict['DataType'],self.attributesdict['EvtType'], self.attributesdict['ProdID'],file],"/") datatohandle[file]={'lfn':lfnpath,'type':'gen','workflowSE':self.destination} result = self.getFileMetadata(datatohandle) if not result['OK']: self.setApplicationStatus(result['Message']) return S_OK() fileMetadata = result['Value'] final = {} for fileName,metadata in fileMetadata.items(): final[fileName]=metadata final[fileName]['resolvedSE']=self.destination #One by one upload the files with failover if necessary replication = {} failover = {} uploaded = [] if not self.failoverTest: for fileName,metadata in final.items(): self.log.info("Attempting to store file %s to the following SE(s):\n%s" % (fileName, string.join(metadata['resolvedSE'],', '))) result = failoverTransfer.transferAndRegisterFile(fileName,metadata['localpath'],metadata['lfn'],metadata['resolvedSE'],fileGUID=metadata['guid'],fileCatalog=self.userFileCatalog) if not result['OK']: self.log.error('Could not transfer and register %s with metadata:\n %s' %(fileName,metadata)) failover[fileName]=metadata else: #Only attempt replication after successful upload lfn = metadata['lfn'] uploaded.append(lfn) seList = metadata['resolvedSE'] replicateSE = '' if result['Value'].has_key('uploadedSE'): uploadedSE = result['Value']['uploadedSE'] for se in seList: if not se == uploadedSE: replicateSE = se break if replicateSE and lfn: self.log.info('Will attempt to replicate %s to %s' %(lfn,replicateSE)) replication[lfn]=replicateSE else: failover = final cleanUp = False for fileName,metadata in failover.items(): random.shuffle(self.failoverSEs) targetSE = metadata['resolvedSE'][0] metadata['resolvedSE']=self.failoverSEs result = failoverTransfer.transferAndRegisterFileFailover(fileName,metadata['localpath'],metadata['lfn'],targetSE,metadata['resolvedSE'],fileGUID=metadata['guid'],fileCatalog=self.userFileCatalog) if not result['OK']: self.log.error('Could not transfer and register %s with metadata:\n %s' %(fileName,metadata)) cleanUp = True continue #for users can continue even if one completely fails else: lfn = metadata['lfn'] uploaded.append(lfn) #For files correctly uploaded must report LFNs to job parameters if uploaded: report = string.join( uploaded, ', ' ) self.jobReport.setJobParameter( 'UploadedOutputData', report ) #Now after all operations, retrieve potentially modified request object result = failoverTransfer.getRequestObject() if not result['OK']: self.log.error(result) return S_ERROR('Could Not Retrieve Modified Request') self.request = result['Value'] #If some or all of the files failed to be saved to failover if cleanUp: self.workflow_commons['Request']=self.request #Leave any uploaded files just in case it is useful for the user #do not try to replicate any files. return S_ERROR('Failed To Upload Output Data') return S_OK()
class UserJobFinalization(ModuleBase): """ User Job finalization: takes care of uploading the output data to the specified storage elements If it does not work, it will upload to a Failover SE, then register the request to replicate and remove. """ ############################################################################# def __init__(self): """Module initialization. """ super(UserJobFinalization, self).__init__() self.version = __RCSID__ self.log = gLogger.getSubLogger( "UserJobFinalization" ) self.enable = True self.failoverTest = False #flag to put file to failover SE by default self.defaultOutputSE = gConfig.getValue( '/Resources/StorageElementGroups/Tier1-USER', []) self.failoverSEs = gConfig.getValue('/Resources/StorageElementGroups/Tier1-Failover', []) #List all parameters here self.userFileCatalog = self.ops.getValue('/UserJobs/Catalogs', ['FileCatalog'] ) self.request = None self.lastStep = False #Always allow any files specified by users self.outputDataFileMask = '' self.userOutputData = '' self.userOutputSE = '' self.userOutputPath = '' self.jobReport = None ############################################################################# def applicationSpecificInputs(self): """ By convention the module parameters are resolved here. """ self.log.verbose(self.workflow_commons) self.log.verbose(self.step_commons) #Earlier modules may have populated the report objects if self.workflow_commons.has_key('JobReport'): self.jobReport = self.workflow_commons['JobReport'] if self.step_commons.has_key('Enable'): self.enable = self.step_commons['Enable'] if not type(self.enable) == type(True): self.log.warn('Enable flag set to non-boolean value %s, setting to False' %self.enable) self.enable = False if self.step_commons.has_key('TestFailover'): self.enable = self.step_commons['TestFailover'] if not type(self.failoverTest) == type(True): self.log.warn('Test failover flag set to non-boolean value %s, setting to False' % self.failoverTest) self.failoverTest = False if os.environ.has_key('JOBID'): self.jobID = os.environ['JOBID'] self.log.verbose('Found WMS JobID = %s' % self.jobID) else: self.log.info('No WMS JobID found, disabling module via control flag') self.enable = False if self.workflow_commons.has_key('Request'): self.request = self.workflow_commons['Request'] else: self.request = RequestContainer() self.request.setRequestName('job_%s_request.xml' % self.jobID) self.request.setJobID(self.jobID) self.request.setSourceComponent("Job_%s" % self.jobID) #Use LHCb utility for local running via dirac-jobexec if self.workflow_commons.has_key('UserOutputData'): self.userOutputData = self.workflow_commons['UserOutputData'] if not type(self.userOutputData) == type([]): self.userOutputData = [i.strip() for i in self.userOutputData.split(';')] if self.workflow_commons.has_key('UserOutputSE'): specifiedSE = self.workflow_commons['UserOutputSE'] if not type(specifiedSE) == type([]): self.userOutputSE = [i.strip() for i in specifiedSE.split(';')] else: self.log.verbose('No UserOutputSE specified, using default value: %s' % (string.join(self.defaultOutputSE, ', '))) self.userOutputSE = self.defaultOutputSE if self.workflow_commons.has_key('UserOutputPath'): self.userOutputPath = self.workflow_commons['UserOutputPath'] return S_OK('Parameters resolved') ############################################################################# def execute(self): """ Main execution function. """ #Have to work out if the module is part of the last step i.e. #user jobs can have any number of steps and we only want #to run the finalization once. currentStep = int(self.step_commons['STEP_NUMBER']) totalSteps = int(self.workflow_commons['TotalSteps']) if currentStep == totalSteps: self.lastStep = True else: self.log.verbose('Current step = %s, total steps of workflow = %s, UserJobFinalization will enable itself only \ at the last workflow step.' % (currentStep, totalSteps)) if not self.lastStep: return S_OK() result = self.resolveInputVariables() if not result['OK']: self.log.error(result['Message']) return result self.log.info('Initializing %s' % self.version) if not self.workflowStatus['OK'] or not self.stepStatus['OK']: self.log.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK'])) return S_OK('No output data upload attempted') if not self.userOutputData: self.log.info('No user output data is specified for this job, nothing to do') return S_OK('No output data to upload') #Determine the final list of possible output files for the #workflow and all the parameters needed to upload them. outputList = [] for i in self.userOutputData: outputList.append({'outputPath' : string.upper(string.split(i, '.')[-1]), 'outputDataSE' : self.userOutputSE, 'outputFile' : os.path.basename(i)}) userOutputLFNs = [] if self.userOutputData: self.log.info('Constructing user output LFN(s) for %s' % (string.join(self.userOutputData, ', '))) if not self.jobID: self.jobID = 12345 owner = '' if self.workflow_commons.has_key('Owner'): owner = self.workflow_commons['Owner'] else: res = self.getCurrentOwner() if not res['OK']: return S_ERROR('Could not obtain owner from proxy') owner = res['Value'] vo = '' if self.workflow_commons.has_key('VO'): vo = self.workflow_commons['VO'] else: res = self.getCurrentVO() if not res['OK']: return S_ERROR('Could not obtain VO from proxy') vo = res['Value'] result = constructUserLFNs(int(self.jobID), vo, owner, self.userOutputData, self.userOutputPath) if not result['OK']: self.log.error('Could not create user LFNs', result['Message']) return result userOutputLFNs = result['Value'] self.log.verbose('Calling getCandidateFiles( %s, %s, %s)' % (outputList, userOutputLFNs, self.outputDataFileMask)) result = self.getCandidateFiles(outputList, userOutputLFNs, self.outputDataFileMask) if not result['OK']: if not self.ignoreapperrors: self.setApplicationStatus(result['Message']) return S_OK() fileDict = result['Value'] result = self.getFileMetadata(fileDict) if not result['OK']: if not self.ignoreapperrors: self.setApplicationStatus(result['Message']) return S_OK() if not result['Value']: if not self.ignoreapperrors: self.log.info('No output data files were determined to be uploaded for this workflow') self.setApplicationStatus('No Output Data Files To Upload') return S_OK() fileMetadata = result['Value'] #First get the local (or assigned) SE to try first for upload and others in random fashion result = getDestinationSEList('Tier1-USER', DIRAC.siteName(), outputmode='local') if not result['OK']: self.log.error('Could not resolve output data SE', result['Message']) self.setApplicationStatus('Failed To Resolve OutputSE') return result localSE = result['Value'] self.log.verbose('Site Local SE for user outputs is: %s' % (localSE)) orderedSEs = self.defaultOutputSE for se in localSE: if se in orderedSEs: orderedSEs.remove(se) for se in self.userOutputSE: if se in orderedSEs: orderedSEs.remove(se) orderedSEs = localSE + List.randomize(orderedSEs) if self.userOutputSE: prependSEs = [] for userSE in self.userOutputSE: if not userSE in orderedSEs: prependSEs.append(userSE) orderedSEs = prependSEs + orderedSEs self.log.info('Ordered list of output SEs is: %s' % (string.join(orderedSEs, ', '))) final = {} for fileName, metadata in fileMetadata.items(): final[fileName] = metadata final[fileName]['resolvedSE'] = orderedSEs #At this point can exit and see exactly what the module will upload if not self.enable: self.log.info('Module is disabled by control flag, would have attempted \ to upload the following files %s' % string.join(final.keys(), ', ')) for fileName, metadata in final.items(): self.log.info('--------%s--------' % fileName) for n, v in metadata.items(): self.log.info('%s = %s' %(n, v)) return S_OK('Module is disabled by control flag') #Instantiate the failover transfer client with the global request object failoverTransfer = FailoverTransfer(self.request) #One by one upload the files with failover if necessary replication = {} failover = {} uploaded = [] if not self.failoverTest: for fileName, metadata in final.items(): self.log.info("Attempting to store file %s to the following SE(s):\n%s" % (fileName, string.join(metadata['resolvedSE'], ', '))) result = failoverTransfer.transferAndRegisterFile(fileName, metadata['localpath'], metadata['lfn'], metadata['resolvedSE'], fileGUID = metadata['guid'], fileCatalog = self.userFileCatalog) if not result['OK']: self.log.error('Could not transfer and register %s with metadata:\n %s' % (fileName, metadata)) failover[fileName] = metadata else: #Only attempt replication after successful upload lfn = metadata['lfn'] uploaded.append(lfn) seList = metadata['resolvedSE'] replicateSE = '' if result['Value'].has_key('uploadedSE'): uploadedSE = result['Value']['uploadedSE'] for se in seList: if not se == uploadedSE: replicateSE = se break if replicateSE and lfn: self.log.info('Will attempt to replicate %s to %s' % (lfn, replicateSE)) replication[lfn] = replicateSE else: failover = final cleanUp = False for fileName, metadata in failover.items(): random.shuffle(self.failoverSEs) targetSE = metadata['resolvedSE'][0] metadata['resolvedSE'] = self.failoverSEs result = failoverTransfer.transferAndRegisterFileFailover(fileName, metadata['localpath'], metadata['lfn'], targetSE, metadata['resolvedSE'], fileGUID = metadata['guid'], fileCatalog = self.userFileCatalog) if not result['OK']: self.log.error('Could not transfer and register %s with metadata:\n %s' % (fileName, metadata)) cleanUp = True continue #for users can continue even if one completely fails else: lfn = metadata['lfn'] uploaded.append(lfn) #For files correctly uploaded must report LFNs to job parameters if uploaded: report = string.join( uploaded, ', ' ) self.jobReport.setJobParameter( 'UploadedOutputData', report ) #Now after all operations, retrieve potentially modified request object result = failoverTransfer.getRequestObject() if not result['OK']: self.log.error(result) return S_ERROR('Could Not Retrieve Modified Request') self.request = result['Value'] #If some or all of the files failed to be saved to failover if cleanUp: self.workflow_commons['Request'] = self.request #Leave any uploaded files just in case it is useful for the user #do not try to replicate any files. return S_ERROR('Failed To Upload Output Data') #If there is now at least one replica for uploaded files can trigger replication rm = ReplicaManager() self.log.info('Sleeping for 10 seconds before attempting replication of recently uploaded files') time.sleep(10) for lfn, repSE in replication.items(): result = rm.replicateAndRegister(lfn, repSE, catalog = self.userFileCatalog) if not result['OK']: self.log.info('Replication failed with below error but file already exists in Grid storage with \ at least one replica:\n%s' % (result)) self.workflow_commons['Request'] = self.request #Now must ensure if any pending requests are generated that these are propagated to the job wrapper reportRequest = None if self.jobReport: result = self.jobReport.generateRequest() if not result['OK']: self.log.warn('Could not generate request for job report with result:\n%s' % (result)) else: reportRequest = result['Value'] if reportRequest: self.log.info('Populating request with job report information') self.request.update(reportRequest) if not self.request.isEmpty()['Value']: request_string = self.request.toXML()['Value'] # Write out the request string fname = 'user_job_%s_request.xml' % (self.jobID) xmlfile = open(fname, 'w') xmlfile.write(request_string) xmlfile.close() self.log.info('Creating failover request for deferred operations for job %s:' % self.jobID) result = self.request.getDigest() if result['OK']: digest = result['Value'] self.log.info(digest) self.setApplicationStatus('Job Finished Successfully') return S_OK('Output data uploaded') ############################################################################# def getCurrentOwner(self): """Simple function to return current DIRAC username. """ result = getProxyInfo() if not result['OK']: return S_ERROR('Could not obtain proxy information') if not result['Value'].has_key('username'): return S_ERROR('Could not get username from proxy') username = result['Value']['username'] return S_OK(username) ############################################################################# def getCurrentVO(self): """Simple function to return current DIRAC username. """ result = getProxyInfo() if not result['OK']: return S_ERROR('Could not obtain proxy information') if not result['Value'].has_key('group'): return S_ERROR('Could not get group from proxy') group = result['Value']['group'] vo = group.split("_")[0] return S_OK(vo) #EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#
class UploadLogFile(ModuleBase): """ Handle log file uploads in the production jobs """ ############################################################################# def __init__(self): """Module initialization. """ super(UploadLogFile, self).__init__() self.version = __RCSID__ self.log = gLogger.getSubLogger("UploadLogFile") self.PRODUCTION_ID = None self.JOB_ID = None self.workflow_commons = None self.request = None self.logFilePath = "" self.logLFNPath = "" self.logdir = "" self.logSE = self.ops.getValue("/LogStorage/LogSE", "LogSE") self.root = gConfig.getValue("/LocalSite/Root", os.getcwd()) self.logSizeLimit = self.ops.getValue("/LogFiles/SizeLimit", 20 * 1024 * 1024) self.logExtensions = [] self.failoverSEs = gConfig.getValue("/Resources/StorageElementGroups/Tier1-Failover", []) self.diracLogo = self.ops.getValue( "/SAM/LogoURL", "https://lhcbweb.pic.es/DIRAC/images/logos/DIRAC-logo-transp.png" ) self.rm = ReplicaManager() self.experiment = "CLIC" self.enable = True self.failoverTest = False # flag to put log files to failover by default self.jobID = "" ###################################################################### def applicationSpecificInputs(self): if self.step_commons.has_key("Enable"): self.enable = self.step_commons["Enable"] if not type(self.enable) == type(True): self.log.warn("Enable flag set to non-boolean value %s, setting to False" % self.enable) self.enable = False if self.step_commons.has_key("TestFailover"): self.enable = self.step_commons["TestFailover"] if not type(self.failoverTest) == type(True): self.log.warn("Test failover flag set to non-boolean value %s, setting to False" % self.failoverTest) self.failoverTest = False if os.environ.has_key("JOBID"): self.jobID = os.environ["JOBID"] self.log.verbose("Found WMS JobID = %s" % self.jobID) else: self.log.info("No WMS JobID found, disabling module via control flag") self.enable = False if self.workflow_commons.has_key("LogFilePath") and self.workflow_commons.has_key("LogTargetPath"): self.logFilePath = self.workflow_commons["LogFilePath"] self.logLFNPath = self.workflow_commons["LogTargetPath"] else: self.log.info("LogFilePath parameter not found, creating on the fly") result = getLogPath(self.workflow_commons) if not result["OK"]: self.log.error("Could not create LogFilePath", result["Message"]) return result self.logFilePath = result["Value"]["LogFilePath"][0] self.logLFNPath = result["Value"]["LogTargetPath"][0] if not type(self.logFilePath) == type(" "): self.logFilePath = self.logFilePath[0] if not type(self.logLFNPath) == type(" "): self.logLFNPath = self.logLFNPath[0] example_file = self.logFilePath if "/ilc/prod/clic" in example_file: self.experiment = "CLIC" elif "/ilc/prod/ilc/sid" in example_file: self.experiment = "ILC_SID" elif "/ilc/prod/ilc/mc-dbd" in example_file: self.experiment = "ILC_ILD" else: self.log.warn("Failed to determine experiment, reverting to default: %s" % self.experiment) if self.workflow_commons.has_key("Request"): self.request = self.workflow_commons["Request"] else: self.request = RequestContainer() self.request.setRequestName("job_%s_request.xml" % self.jobID) self.request.setJobID(self.jobID) self.request.setSourceComponent("Job_%s" % self.jobID) return S_OK("Parameters resolved") ###################################################################### def execute(self): """ Main execution method """ self.log.info("Initializing %s" % self.version) # Add global reporting tool self.resolveInputVariables() res = shellCall(0, "ls -al") if res["OK"] and res["Value"][0] == 0: self.log.info("The contents of the working directory...") self.log.info(str(res["Value"][1])) else: self.log.error("Failed to list the log directory", str(res["Value"][2])) self.log.info("Job root is found to be %s" % (self.root)) self.log.info("PRODUCTION_ID = %s, JOB_ID = %s " % (self.PRODUCTION_ID, self.JOB_ID)) self.logdir = os.path.realpath("./job/log/%s/%s" % (self.PRODUCTION_ID, self.JOB_ID)) self.log.info("Selected log files will be temporarily stored in %s" % self.logdir) res = self.finalize() self.workflow_commons["Request"] = self.request return res ############################################################################# def finalize(self): """ finalize method performs final operations after all the job steps were executed. Only production jobs are treated. """ self.log.verbose("Starting UploadLogFile finalize") ########################################## # First determine the files which should be saved self.log.info("Determining the files to be saved in the logs.") res = self.determineRelevantFiles() if not res["OK"]: self.log.error("Completely failed to select relevant log files.", res["Message"]) return S_OK() # because if the logs are lost, it's not the end of the world. selectedFiles = res["Value"] self.log.info( "The following %s files were selected to be saved:\n%s" % (len(selectedFiles), string.join(selectedFiles, "\n")) ) ######################################### # Create a temporary directory containing these files self.log.info("Populating a temporary directory for selected files.") res = self.populateLogDirectory(selectedFiles) if not res["OK"]: self.log.error("Completely failed to populate temporary log file directory.", res["Message"]) self.setApplicationStatus("Failed To Populate Log Dir") return S_OK() # because if the logs are lost, it's not the end of the world. self.log.info("%s populated with log files." % self.logdir) ######################################### # Create a tailored index page # self.log.info('Creating an index page for the logs') # result = self.__createLogIndex(selectedFiles) # if not result['OK']: # self.log.error('Failed to create index page for logs', res['Message']) if not self.enable: self.log.info("Module is disabled by control flag") return S_OK("Module is disabled by control flag") ######################################### # Make sure all the files in the log directory have the correct permissions result = self.__setLogFilePermissions(self.logdir) if not result["OK"]: self.log.error("Could not set permissions of log files to 0755 with message:\n%s" % (result["Message"])) ######################################### # Attempt to uplaod logs to the LogSE self.log.info("Transferring log files to the %s" % self.logSE) res = S_ERROR() if not self.failoverTest: self.log.info("PutDirectory %s %s %s" % (self.logFilePath, os.path.realpath(self.logdir), self.logSE)) res = self.rm.putStorageDirectory( {self.logFilePath: os.path.realpath(self.logdir)}, self.logSE, singleDirectory=True ) self.log.verbose(res) if res["OK"]: self.log.info("Successfully upload log directory to %s" % self.logSE) # TODO: The logURL should be constructed using the LogSE and StorageElement() # storageElement = StorageElement(self.logSE) # pfn = storageElement.getPfnForLfn(self.logFilePath)['Value'] # logURL = getPfnForProtocol(res['Value'],'http')['Value'] logURL = "%s" % self.logFilePath self.setJobParameter("Log LFN", logURL) self.log.info("Logs for this job may be retrieved with dirac-ilc-get-prod-log -F %s" % logURL) return S_OK() ######################################### # Recover the logs to a failover storage element self.log.error( "Completely failed to upload log files to %s, will attempt upload to failover SE" % self.logSE, res["Message"], ) tarFileDir = os.path.dirname(self.logdir) self.logLFNPath = "%s.gz" % self.logLFNPath tarFileName = os.path.basename(self.logLFNPath) start = os.getcwd() os.chdir(self.logdir) logTarFiles = os.listdir(self.logdir) # comm = 'tar czvf %s %s' % (tarFileName,string.join(logTarFiles,' ')) tfile = tarfile.open(tarFileName, "w:gz") for item in logTarFiles: tfile.add(item) tfile.close() # res = shellCall(0,comm) if not os.path.exists(tarFileName): res = S_ERROR("File was not created") os.chdir(start) if not res["OK"]: self.log.error("Failed to create tar file from directory", "%s %s" % (self.logdir, res["Message"])) self.setApplicationStatus("Failed To Create Log Tar Dir") return S_OK() # because if the logs are lost, it's not the end of the world. # if res['Value'][0]: #i.e. non-zero status # self.log.error('Failed to create tar file from directory','%s %s' % (self.logdir,res['Value'])) # self.setApplicationStatus('Failed To Create Log Tar Dir') # return S_OK()#because if the logs are lost, it's not the end of the world. ############################################################ # Instantiate the failover transfer client with the global request object failoverTransfer = FailoverTransfer(self.request) ##determine the experiment self.failoverSEs = self.ops.getValue("Production/%s/FailOverSE" % self.experiment, self.failoverSEs) random.shuffle(self.failoverSEs) self.log.info( "Attempting to store file %s to the following SE(s):\n%s" % (tarFileName, string.join(self.failoverSEs, ", ")) ) result = failoverTransfer.transferAndRegisterFile( tarFileName, "%s/%s" % (tarFileDir, tarFileName), self.logLFNPath, self.failoverSEs, fileGUID=None, fileCatalog=["FileCatalog", "LcgFileCatalog"], ) if not result["OK"]: self.log.error("Failed to upload logs to all destinations") self.setApplicationStatus("Failed To Upload Logs") return S_OK() # because if the logs are lost, it's not the end of the world. # Now after all operations, retrieve potentially modified request object result = failoverTransfer.getRequestObject() if not result["OK"]: self.log.error(result) return S_ERROR("Could not retrieve modified request") self.request = result["Value"] res = self.createLogUploadRequest(self.logSE, self.logLFNPath) if not res["OK"]: self.log.error("Failed to create failover request", res["Message"]) self.setApplicationStatus("Failed To Upload Logs To Failover") else: self.log.info("Successfully created failover request") self.workflow_commons["Request"] = self.request return S_OK() ############################################################################# def determineRelevantFiles(self): """ The files which are below a configurable size will be stored in the logs. This will typically pick up everything in the working directory minus the output data files. """ logFileExtensions = ["*.txt", "*.log", "*.out", "*.output", "*.xml", "*.sh", "*.info", "*.err", "*.root"] self.logExtensions = self.ops.getValue("/LogFiles/%s/Extensions" % self.experiment, []) if self.logExtensions: self.log.info("Using list of log extensions from CS:\n%s" % (", ".join(self.logExtensions))) logFileExtensions = self.logExtensions else: self.log.info("Using default list of log extensions:\n%s" % (", ".join(logFileExtensions))) candidateFiles = [] for ext in logFileExtensions: self.log.debug("Looking at log file wildcard: %s" % ext) globList = glob.glob(ext) for check in globList: if os.path.isfile(check): self.log.debug("Found locally existing log file: %s" % check) candidateFiles.append(check) selectedFiles = [] try: for candidate in candidateFiles: fileSize = os.stat(candidate)[6] if fileSize < self.logSizeLimit: selectedFiles.append(candidate) else: self.log.error( "Log file found to be greater than maximum of %s bytes" % self.logSizeLimit, candidate ) return S_OK(selectedFiles) except Exception, x: self.log.exception("Exception while determining files to save.", "", str(x)) return S_ERROR("Could not determine log files")
except Exception, x: gLogger.error("Could not copy because %s" % x) return S_ERROR("Could not copy because %s" % x) elif path.find("http://") > -1: gLogger.error("path %s was not foreseen, location not known, upload to location yourself, and publish in CS manually" % path) return S_ERROR() else: lfnpath = "%s%s" % (path, os.path.basename(appTar)) res = rm.putAndRegister(lfnpath, appTar, ops.getValue('Software/BaseStorageElement',"CERN-SRM")) if not res['OK']: return res request = RequestContainer() request.setCreationTime() requestClient = RequestClient() request.setRequestName('copy_%s' % os.path.basename(appTar).replace(".tgz","").replace(".tar.gz","")) request.setSourceComponent('ReplicateILCSoft') copies_at = ops.getValue('Software/CopiesAt',[]) index_copy = 0 for copies in copies_at: res = request.addSubRequest({'Attributes':{'Operation' : 'replicateAndRegister', 'TargetSE' : copies, 'ExecutionOrder' : index_copy}, 'Files':[{'LFN':lfnpath}]}, 'transfer') #res = rm.replicateAndRegister("%s%s"%(path,appTar),"IN2P3-SRM") if not res['OK']: return res index_copy += 1 requestxml = request.toXML()['Value'] if copies_at: res = requestClient.setRequest(request.getRequestName()['Value'], requestxml)
def getRequest( self, requestType = '' ): """ Get a request of a given type """ # RG: What if requestType is not given? # the first query will return nothing. start = time.time() dmRequest = RequestContainer( init = False ) requestID = 0 req = "SELECT RequestID,SubRequestID FROM SubRequests WHERE Status = 'Waiting' AND RequestType = '%s' ORDER BY LastUpdate ASC LIMIT 50;" % requestType res = self._query( req ) if not res['OK']: err = 'RequestDB._getRequest: Failed to retrieve max RequestID' return S_ERROR( '%s\n%s' % ( err, res['Message'] ) ) if not res['Value']: return S_OK() reqIDList = [ x[0] for x in res['Value'] ] random.shuffle( reqIDList ) count = 0 for reqID in reqIDList: count += 1 if requestType: req = "SELECT SubRequestID,Operation,Arguments,ExecutionOrder,SourceSE,TargetSE,Catalogue,CreationTime,SubmissionTime,LastUpdate \ from SubRequests WHERE RequestID=%s AND RequestType='%s' AND Status='%s'" % ( reqID, requestType, 'Waiting' ) else: # RG: What if requestType is not given? # we should never get there, and it misses the "AND Status='Waiting'" req = "SELECT SubRequestID,Operation,Arguments,ExecutionOrder,SourceSE,TargetSE,Catalogue,CreationTime,SubmissionTime,LastUpdate \ from SubRequests WHERE RequestID=%s" % reqID res = self._query( req ) if not res['OK']: err = 'RequestDB._getRequest: Failed to retrieve SubRequests for RequestID %s' % reqID return S_ERROR( '%s\n%s' % ( err, res['Message'] ) ) subIDList = [] for tuple in res['Value']: subID = tuple[0] # RG: We should set the condition "AND Status='Waiting'" # if the subrequest has got assigned it will failed req = "UPDATE SubRequests SET Status='Assigned' WHERE RequestID=%s AND SubRequestID=%s;" % ( reqID, subID ) resAssigned = self._update( req ) if not resAssigned['OK']: if subIDList: self.__releaseSubRequests( reqID, subIDList ) return S_ERROR( 'Failed to assign subrequests: %s' % resAssigned['Message'] ) if resAssigned['Value'] == 0: # Somebody has assigned this request gLogger.warn( 'Already assigned subrequest %d of request %d' % ( subID, reqID ) ) else: subIDList.append( subID ) # RG: We need to check that all subRequest with smaller ExecutionOrder are "Done" if subIDList: # We managed to get some requests, can continue now requestID = reqID break # Haven't succeeded to get any request if not requestID: return S_OK() dmRequest.setRequestID( requestID ) # RG: We have this list in subIDList, can different queries get part of the subrequets of the same type? subRequestIDs = [] for subRequestID, operation, arguments, executionOrder, sourceSE, targetSE, catalogue, creationTime, submissionTime, lastUpdate in res['Value']: if not subRequestID in subIDList: continue subRequestIDs.append( subRequestID ) # RG: res['Value'] is the range of the loop and it gets redefined here !!!!!! res = dmRequest.initiateSubRequest( requestType ) ind = res['Value'] subRequestDict = { 'Status' : 'Waiting', 'SubRequestID' : subRequestID, 'Operation' : operation, 'Arguments' : arguments, 'ExecutionOrder': int( executionOrder ), 'SourceSE' : sourceSE, 'TargetSE' : targetSE, 'Catalogue' : catalogue, 'CreationTime' : creationTime, 'SubmissionTime': submissionTime, 'LastUpdate' : lastUpdate } res = dmRequest.setSubRequestAttributes( ind, requestType, subRequestDict ) if not res['OK']: err = 'RequestDB._getRequest: Failed to set subRequest attributes for RequestID %s' % requestID self.__releaseSubRequests( requestID, subRequestIDs ) return S_ERROR( '%s\n%s' % ( err, res['Message'] ) ) req = "SELECT FileID,LFN,Size,PFN,GUID,Md5,Addler,Attempt,Status \ from Files WHERE SubRequestID = %s ORDER BY FileID;" % subRequestID res = self._query( req ) if not res['OK']: err = 'RequestDB._getRequest: Failed to get File attributes for RequestID %s.%s' % ( requestID, subRequestID ) self.__releaseSubRequests( requestID, subRequestIDs ) return S_ERROR( '%s\n%s' % ( err, res['Message'] ) ) files = [] for fileID, lfn, size, pfn, guid, md5, addler, attempt, status in res['Value']: fileDict = {'FileID':fileID, 'LFN':lfn, 'Size':size, 'PFN':pfn, 'GUID':guid, 'Md5':md5, 'Addler':addler, 'Attempt':attempt, 'Status':status} files.append( fileDict ) res = dmRequest.setSubRequestFiles( ind, requestType, files ) if not res['OK']: err = 'RequestDB._getRequest: Failed to set files into Request for RequestID %s.%s' % ( requestID, subRequestID ) self.__releaseSubRequests( requestID, subRequestIDs ) return S_ERROR( '%s\n%s' % ( err, res['Message'] ) ) req = "SELECT Dataset,Status FROM Datasets WHERE SubRequestID = %s;" % subRequestID res = self._query( req ) if not res['OK']: err = 'RequestDB._getRequest: Failed to get Datasets for RequestID %s.%s' % ( requestID, subRequestID ) self.__releaseSubRequests( requestID, subRequestIDs ) return S_ERROR( '%s\n%s' % ( err, res['Message'] ) ) datasets = [] for dataset, status in res['Value']: datasets.append( dataset ) res = dmRequest.setSubRequestDatasets( ind, requestType, datasets ) if not res['OK']: err = 'RequestDB._getRequest: Failed to set datasets into Request for RequestID %s.%s' % ( requestID, subRequestID ) self.__releaseSubRequests( requestID, subRequestIDs ) return S_ERROR( '%s\n%s' % ( err, res['Message'] ) ) req = "SELECT RequestName,JobID,OwnerDN,OwnerGroup,DIRACSetup,SourceComponent,CreationTime,SubmissionTime,LastUpdate from Requests WHERE RequestID = %s;" % requestID res = self._query( req ) if not res['OK']: err = 'RequestDB._getRequest: Failed to retrieve max RequestID' self.__releaseSubRequests( requestID, subRequestIDs ) return S_ERROR( '%s\n%s' % ( err, res['Message'] ) ) requestName, jobID, ownerDN, ownerGroup, diracSetup, sourceComponent, creationTime, submissionTime, lastUpdate = res['Value'][0] dmRequest.setRequestName( requestName ) dmRequest.setJobID( jobID ) dmRequest.setOwnerDN( ownerDN ) dmRequest.setOwnerGroup( ownerGroup ) dmRequest.setDIRACSetup( diracSetup ) dmRequest.setSourceComponent( sourceComponent ) dmRequest.setCreationTime( str( creationTime ) ) dmRequest.setLastUpdate( str( lastUpdate ) ) res = dmRequest.toXML() if not res['OK']: err = 'RequestDB._getRequest: Failed to create XML for RequestID %s' % ( requestID ) self.__releaseSubRequests( requestID, subRequestIDs ) return S_ERROR( '%s\n%s' % ( err, res['Message'] ) ) requestString = res['Value'] #still have to manage the status of the dataset properly resultDict = {} resultDict['RequestName'] = requestName resultDict['RequestString'] = requestString resultDict['JobID'] = jobID return S_OK( resultDict )
def getRequest( self, requestType ): """ Get a request of a given type eligible for execution """ # RG: What if requestType is not given? # the first query will return nothing. # KC: maybe returning S_ERROR would be enough? # alternatively we should check if requestType is known (in 'transfer', 'removal', 'register' and 'diset') if not requestType or type( requestType ) not in types.StringTypes: return S_ERROR( "Request type not given." ) myRequestType = self._escapeString( requestType ) if not myRequestType: return myRequestType myRequestType = myRequestType['Value'] start = time.time() dmRequest = RequestContainer( init = False ) requestID = 0 subIDList = [] fields = ['RequestID', 'SubRequestID', 'Operation', 'Arguments', 'ExecutionOrder', 'SourceSE', 'TargetSE', 'Catalogue', 'CreationTime', 'SubmissionTime', 'LastUpdate', 'Status', 'RequestType'] # get the pending SubRequest sorted by ExecutionOrder and LastUpdate req = "SELECT `RequestID`,`ExecutionOrder`,`Status`,`RequestType`,`LastUpdate` FROM `SubRequests` "\ "WHERE `Status` IN ( 'Waiting', 'Assigned' ) ORDER BY `ExecutionOrder`,`LastUpdate`" # now get sorted list of RequestID (according to the above) req = "SELECT * FROM ( %s ) as T1 GROUP BY `RequestID`" % req # and get the 100 oldest ones of Type requestType req = "SELECT `RequestID`,`ExecutionOrder` FROM ( %s ) as T2 WHERE `RequestType`=%s "\ "ORDER BY `LastUpdate` LIMIT 100" % ( req, myRequestType ) # and now get all waiting SubRequest for the selected RequestID and ExecutionOrder req = "SELECT A.%s FROM SubRequests AS A, ( %s ) AS B WHERE " % ( ', A.'.join( fields ), req ) req = "%s A.RequestID=B.RequestID AND A.ExecutionOrder=B.ExecutionOrder" % ( req ) result = self._query( req ) if not result['OK']: err = 'RequestDB._getRequest: Failed to retrieve Requests' return S_ERROR( '%s\n%s' % ( err, result['Message'] ) ) if not result['Value']: return S_OK() # We get up to 10 Request candidates, to add some randomness reqDict = {} for row in result['Value']: if ('"%s"' % row[-1]) != myRequestType: continue if row[-2] != 'Waiting': continue reqDict.setdefault( row[0], [] ) reqDict[row[0]].append( row[1:-2] ) reqIDList = reqDict.keys() random.shuffle( reqIDList ) for reqID in reqIDList: sidList = [ x[0] for x in reqDict[reqID] ] for subID in sidList: req = "UPDATE SubRequests SET Status='Assigned' WHERE RequestID=%s AND SubRequestID=%s;" % ( reqID, subID ) resAssigned = self._update( req ) if not resAssigned['OK']: if subIDList: self.__releaseSubRequests( reqID, subIDList ) return S_ERROR( 'Failed to assign subrequests: %s' % resAssigned['Message'] ) if resAssigned['Value'] == 0: # Somebody has assigned this request gLogger.warn( 'Already assigned subrequest %d of request %d' % ( subID, reqID ) ) else: subIDList.append( subID ) if subIDList: # We managed to get some requests, can continue now requestID = reqID break # Haven't succeeded to get any request if not requestID: return S_OK() dmRequest.setRequestID( requestID ) fields = ['FileID', 'LFN', 'Size', 'PFN', 'GUID', 'Md5', 'Addler', 'Attempt', 'Status' ] for subRequestID, operation, arguments, executionOrder, sourceSE, targetSE, catalogue, \ creationTime, submissionTime, lastUpdate in reqDict[requestID]: if not subRequestID in subIDList: continue res = dmRequest.initiateSubRequest( requestType ) ind = res['Value'] subRequestDict = { 'Status' : 'Waiting', 'SubRequestID' : subRequestID, 'Operation' : operation, 'Arguments' : arguments, 'ExecutionOrder': int( executionOrder ), 'SourceSE' : sourceSE, 'TargetSE' : targetSE, 'Catalogue' : catalogue, 'CreationTime' : creationTime, 'SubmissionTime': submissionTime, 'LastUpdate' : lastUpdate } res = dmRequest.setSubRequestAttributes( ind, requestType, subRequestDict ) if not res['OK']: err = 'RequestDB._getRequest: Failed to set subRequest attributes for RequestID %s' % requestID self.__releaseSubRequests( requestID, subIDList ) return S_ERROR( '%s\n%s' % ( err, res['Message'] ) ) req = "SELECT %s FROM `Files` WHERE `SubRequestID`=%s ORDER BY `FileID`;" % ( ', '.join( fields ), subRequestID ) res = self._query( req ) if not res['OK']: err = 'RequestDB._getRequest: Failed to get File attributes for RequestID %s.%s' % ( requestID, subRequestID ) self.__releaseSubRequests( requestID, subIDList ) return S_ERROR( '%s\n%s' % ( err, res['Message'] ) ) files = [] for fileID, lfn, size, pfn, guid, md5, addler, attempt, status in res['Value']: fileDict = {'FileID':fileID, 'LFN':lfn, 'Size':size, 'PFN':pfn, 'GUID':guid, 'Md5':md5, 'Addler':addler, 'Attempt':attempt, 'Status':status} files.append( fileDict ) res = dmRequest.setSubRequestFiles( ind, requestType, files ) if not res['OK']: err = 'RequestDB._getRequest: Failed to set files into Request for RequestID %s.%s' % ( requestID, subRequestID ) self.__releaseSubRequests( requestID, subIDList ) return S_ERROR( '%s\n%s' % ( err, res['Message'] ) ) req = "SELECT Dataset,Status FROM Datasets WHERE SubRequestID = %s;" % subRequestID res = self._query( req ) if not res['OK']: err = 'RequestDB._getRequest: Failed to get Datasets for RequestID %s.%s' % ( requestID, subRequestID ) self.__releaseSubRequests( requestID, subIDList ) return S_ERROR( '%s\n%s' % ( err, res['Message'] ) ) datasets = [] for dataset, status in res['Value']: datasets.append( dataset ) res = dmRequest.setSubRequestDatasets( ind, requestType, datasets ) if not res['OK']: err = 'RequestDB._getRequest: Failed to set datasets into Request for RequestID %s.%s' % ( requestID, subRequestID ) self.__releaseSubRequests( requestID, subIDList ) return S_ERROR( '%s\n%s' % ( err, res['Message'] ) ) fields = ['RequestName', 'JobID', 'OwnerDN', 'OwnerGroup', 'DIRACSetup', 'SourceComponent', 'CreationTime', 'SubmissionTime', 'LastUpdate'] req = "SELECT %s FROM `Requests` WHERE `RequestID`=%s;" % ( ', '.join( fields ), requestID ) res = self._query( req ) if not res['OK']: err = 'RequestDB._getRequest: Failed to retrieve max RequestID' self.__releaseSubRequests( requestID, subIDList ) return S_ERROR( '%s\n%s' % ( err, res['Message'] ) ) requestName, jobID, ownerDN, ownerGroup, diracSetup, sourceComponent, \ creationTime, submissionTime, lastUpdate = res['Value'][0] dmRequest.setRequestName( requestName ) dmRequest.setJobID( jobID ) dmRequest.setOwnerDN( ownerDN ) dmRequest.setOwnerGroup( ownerGroup ) dmRequest.setDIRACSetup( diracSetup ) dmRequest.setSourceComponent( sourceComponent ) dmRequest.setCreationTime( str( creationTime ) ) dmRequest.setLastUpdate( str( lastUpdate ) ) res = dmRequest.toXML() if not res['OK']: err = 'RequestDB._getRequest: Failed to create XML for RequestID %s' % ( requestID ) self.__releaseSubRequests( requestID, subIDList ) return S_ERROR( '%s\n%s' % ( err, res['Message'] ) ) requestString = res['Value'] #still have to manage the status of the dataset properly resultDict = {} resultDict['RequestName'] = requestName resultDict['RequestString'] = requestString resultDict['JobID'] = jobID return S_OK( resultDict )
def getRequestForSubRequest(self, itself, subRequestID ): """ Select Request given SubRequestID. :param self: plugin reference :param itself: patient reference for injection :param int subRequestID: SubRequests.SubRequestID :warn: function has to be injected to RequestDBMySQL instance """ ## get RequestID requestID = "SELECT RequestID FROM SubRequests WHERE SubRequestID = %s;" % str(subRequestID) requestID = self._query( requestID ) if not requestID["OK"]: return requestID requestID = requestID["Value"][0] ## create RequestContainer requestContainer = RequestContainer( init = False ) requestContainer.setRequestID( requestID ) ## put some basic infos in requestInfo = "SELECT RequestName, JobID, OwnerDN, OwnerGroup, DIRACSetup, SourceComponent, CreationTime, SubmissionTime, LastUpdate, Status " requestInfo += "FROM Requests WHERE RequestID = %d;" % requestID requestInfo = self._query( requestInfo ) if not requestInfo["OK"]: return requestInfo requestName, jobID, ownerDN, ownerGroup, diracSetup, sourceComponent, creationTime, submissionTime, lastUpdate, status = requestInfo['Value'][0] requestContainer.setRequestName( requestName ) requestContainer.setJobID( jobID ) requestContainer.setOwnerDN( ownerDN ) requestContainer.setOwnerGroup( ownerGroup ) requestContainer.setDIRACSetup( diracSetup ) requestContainer.setSourceComponent( sourceComponent ) requestContainer.setCreationTime( str( creationTime ) ) requestContainer.setLastUpdate( str( lastUpdate ) ) requestContainer.setStatus( status ) ## get sub-requests subRequests = "SELECT SubRequestID, Status, RequestType, Operation, Arguments, ExecutionOrder, SourceSE, " subRequests += "TargetSE, Catalogue, CreationTime, SubmissionTime, LastUpdate FROM SubRequests WHERE RequestID=%s;" % requestID subRequests = self._query( subRequests ) if not subRequests["OK"]: return subRequests ## loop over sub requests for subRequestID, status, requestType, operation, arguments, executionOrder, sourceSE, targetSE, catalogue, creationTime, submissionTime, lastUpdate in subRequests["Value"]: res = requestContainer.initiateSubRequest( requestType ) ind = res["Value"] subRequestDict = { "Status" : status, "SubRequestID" : subRequestID, "Operation" : operation, "Arguments" : arguments, "ExecutionOrder" : int( executionOrder ), "SourceSE" : sourceSE, "TargetSE" : targetSE, "Catalogue" : catalogue, "CreationTime" : creationTime, "SubmissionTime" : submissionTime, "LastUpdate" : lastUpdate } res = requestContainer.setSubRequestAttributes( ind, requestType, subRequestDict ) if not res["OK"]: return res ## get files for this subrequest req = "SELECT FileID, LFN, Size, PFN, GUID, Md5, Addler, Attempt, Status FROM Files WHERE SubRequestID = %s ORDER BY FileID;" % str(subRequestID) res = self._query( req ) if not res["OK"]: return res files = [] for fileID, lfn, size, pfn, guid, md5, addler, attempt, status in res["Value"]: fileDict = { "FileID" : fileID, "LFN" : lfn, "Size" : size, "PFN" : pfn, "GUID" : guid, "Md5" : md5, "Addler" : addler, "Attempt" : attempt, "Status" : status } files.append( fileDict ) res = requestContainer.setSubRequestFiles( ind, requestType, files ) if not res["OK"]: return res ## dump request to XML res = requestContainer.toXML() if not res["OK"]: return res requestString = res["Value"] ## return dictonary with all info in at least return S_OK( { "RequestName" : requestName, "RequestString" : requestString, "JobID" : jobID, "RequestContainer" : requestContainer } )