def _createLogUploadRequest(self, targetSE, logFileLFN, uploadedSE): """ Set a request to upload job log files from the output sandbox Changed to be similar to LHCb createLogUploadRequest using LHCb LogUpload Request and Removal Request """ self.log.info('Setting log upload request for %s at %s' %(targetSE, logFileLFN)) request = self._getRequestContainer() logUpload = Operation() logUpload.Type = "LogUpload" logUpload.TargetSE = targetSE upFile = File() upFile.LFN = logFileLFN logUpload.addFile( upFile ) logRemoval = Operation() logRemoval.Type = 'RemoveFile' logRemoval.TargetSE = uploadedSE logRemoval.addFile( upFile ) request.addOperation ( logUpload ) request.addOperation ( logRemoval ) self.workflow_commons['Request'] = request return S_OK()
def createRequest(self, requestName, lfnChunk): """Create the Request.""" request = Request() request.RequestName = requestName replicate = Operation() replicate.Type = "ReplicateAndRegister" replicate.TargetSE = self.switches.get("TargetSE") self.addLFNs(replicate, lfnChunk, addPFN=True) request.addOperation(replicate) if self.switches.get("CheckMigration"): checkMigration = Operation() checkMigration.Type = "CheckMigration" checkMigration.TargetSE = self.switches.get("TargetSE") self.addLFNs(checkMigration, lfnChunk, addPFN=True) request.addOperation(checkMigration) removeReplicas = Operation() removeReplicas.Type = "RemoveReplica" removeReplicas.TargetSE = ",".join(self.switches.get("SourceSE", [])) self.addLFNs(removeReplicas, lfnChunk) request.addOperation(removeReplicas) return request
def _createLogUploadRequest(self, targetSE, logFileLFN, uploadedSE): """ Set a request to upload job log files from the output sandbox Changed to be similar to LHCb createLogUploadRequest using LHCb LogUpload Request and Removal Request """ self.log.info('Setting log upload request for %s at %s' % (targetSE, logFileLFN)) request = self._getRequestContainer() logUpload = Operation() logUpload.Type = "LogUpload" logUpload.TargetSE = targetSE upFile = File() upFile.LFN = logFileLFN logUpload.addFile(upFile) logRemoval = Operation() logRemoval.Type = 'RemoveFile' logRemoval.TargetSE = uploadedSE logRemoval.addFile(upFile) request.addOperation(logUpload) request.addOperation(logRemoval) self.workflow_commons['Request'] = request return S_OK()
def createRequest(self, requestName, lfnChunk): """Create the Request.""" request = Request() request.RequestName = requestName replicate = Operation() replicate.Type = 'ReplicateAndRegister' replicate.TargetSE = self.switches.get('TargetSE') self.addLFNs(replicate, lfnChunk, addPFN=True) request.addOperation(replicate) if self.switches.get('CheckMigration'): checkMigration = Operation() checkMigration.Type = 'CheckMigration' checkMigration.TargetSE = self.switches.get('TargetSE') self.addLFNs(checkMigration, lfnChunk, addPFN=True) request.addOperation(checkMigration) removeReplicas = Operation() removeReplicas.Type = 'RemoveReplica' removeReplicas.TargetSE = ','.join(self.switches.get('SourceSE', [])) self.addLFNs(removeReplicas, lfnChunk) request.addOperation(removeReplicas) return request
def buildRequest( self, owner, group, sourceSE, targetSE1, targetSE2 ): files = self.files( owner, group ) putAndRegister = Operation() putAndRegister.Type = "PutAndRegister" putAndRegister.TargetSE = sourceSE for fname, lfn, size, checksum, guid in files: putFile = File() putFile.LFN = lfn putFile.PFN = fname putFile.Checksum = checksum putFile.ChecksumType = "adler32" putFile.Size = size putFile.GUID = guid putAndRegister.addFile( putFile ) replicateAndRegister = Operation() replicateAndRegister.Type = "ReplicateAndRegister" replicateAndRegister.TargetSE = "%s,%s" % ( targetSE1, targetSE2 ) for fname, lfn, size, checksum, guid in files: repFile = File() repFile.LFN = lfn repFile.Size = size repFile.Checksum = checksum repFile.ChecksumType = "adler32" replicateAndRegister.addFile( repFile ) removeReplica = Operation() removeReplica.Type = "RemoveReplica" removeReplica.TargetSE = sourceSE for fname, lfn, size, checksum, guid in files: removeReplica.addFile( File( {"LFN": lfn } ) ) removeFile = Operation() removeFile.Type = "RemoveFile" for fname, lfn, size, checksum, guid in files: removeFile.addFile( File( {"LFN": lfn } ) ) removeFileInit = Operation() removeFileInit.Type = "RemoveFile" for fname, lfn, size, checksum, guid in files: removeFileInit.addFile( File( {"LFN": lfn } ) ) req = Request() req.addOperation( removeFileInit ) req.addOperation( putAndRegister ) req.addOperation( replicateAndRegister ) req.addOperation( removeReplica ) req.addOperation( removeFile ) return req
def buildRequest(self, owner, group, sourceSE, targetSE1, targetSE2): files = self.files(owner, group) putAndRegister = Operation() putAndRegister.Type = "PutAndRegister" putAndRegister.TargetSE = sourceSE for fname, lfn, size, checksum, guid in files: putFile = File() putFile.LFN = lfn putFile.PFN = fname putFile.Checksum = checksum putFile.ChecksumType = "adler32" putFile.Size = size putFile.GUID = guid putAndRegister.addFile(putFile) replicateAndRegister = Operation() replicateAndRegister.Type = "ReplicateAndRegister" replicateAndRegister.TargetSE = "%s,%s" % (targetSE1, targetSE2) for fname, lfn, size, checksum, guid in files: repFile = File() repFile.LFN = lfn repFile.Size = size repFile.Checksum = checksum repFile.ChecksumType = "adler32" replicateAndRegister.addFile(repFile) removeReplica = Operation() removeReplica.Type = "RemoveReplica" removeReplica.TargetSE = sourceSE for fname, lfn, size, checksum, guid in files: removeReplica.addFile(File({"LFN": lfn})) removeFile = Operation() removeFile.Type = "RemoveFile" for fname, lfn, size, checksum, guid in files: removeFile.addFile(File({"LFN": lfn})) removeFileInit = Operation() removeFileInit.Type = "RemoveFile" for fname, lfn, size, checksum, guid in files: removeFileInit.addFile(File({"LFN": lfn})) req = Request() req.addOperation(removeFileInit) req.addOperation(putAndRegister) req.addOperation(replicateAndRegister) req.addOperation(removeReplica) req.addOperation(removeFile) return req
def __submitRMSOp(self, target_se, lfns_chunk_dict, whichRMSOp='ReplicateAndRegister' ): """ target_se : SE name to which to replicate lfns_chunk_dict : LFNS dict with 100 lfns as key andeEach lfn has 'Size', 'Checksum' whichRMSOp: Choose from RMP operation - ReplicateAndRegister, ReplicateAndRemove, PutAndRegister """ ## Setup request request = Request() request.RequestName = "DDM_"+ str(target_se) + datetime.datetime.now().strftime("_%Y%m%d_%H%M%S") myOp = Operation() myOp.Type = whichRMSOp myOp.TargetSE = target_se ## Add LFNS to operations for lfn in lfns_chunk_dict.keys(): opFile = File() opFile.LFN = lfn opFile.Size = lfns_chunk_dict[lfn]['Size'] if "Checksum" in lfns_chunk_dict[lfn]: opFile.Checksum = lfns_chunk_dict[lfn]['Checksum'] opFile.ChecksumType = 'ADLER32' ## Add file to operation myOp.addFile( opFile ) request.addOperation( myOp ) reqClient = ReqClient() putRequest = reqClient.putRequest( request ) if not putRequest["OK"]: gLogger.error( "Unable to put request '%s': %s" % ( request.RequestName, putRequest["Message"] ) ) return S_ERROR("Problem submitting to RMS.")
def __insertRegisterOperation( self, request, operation, toRegister ): """ add RegisterReplica operation :param Request request: request instance :param Operation transferOp: 'ReplicateAndRegister' operation for this FTSJob :param list toRegister: [ FTSDB.FTSFile, ... ] - files that failed to register """ log = self.log.getSubLogger( "req_%s/%s/registerFiles" % ( request.RequestID, request.RequestName ) ) byTarget = {} for ftsFile in toRegister: if ftsFile.TargetSE not in byTarget: byTarget.setdefault( ftsFile.TargetSE, [] ) byTarget[ftsFile.TargetSE].append( ftsFile ) log.info( "will create %s 'RegisterReplica' operations" % len( byTarget ) ) for target, ftsFileList in byTarget.iteritems(): log.info( "creating 'RegisterReplica' operation for targetSE %s with %s files..." % ( target, len( ftsFileList ) ) ) registerOperation = Operation() registerOperation.Type = "RegisterReplica" registerOperation.Status = "Waiting" registerOperation.TargetSE = target targetSE = StorageElement( target ) for ftsFile in ftsFileList: opFile = File() opFile.LFN = ftsFile.LFN pfn = returnSingleResult( targetSE.getURL( ftsFile.LFN, protocol = self.registrationProtocols ) ) if not pfn["OK"]: continue opFile.PFN = pfn["Value"] registerOperation.addFile( opFile ) request.insertBefore( registerOperation, operation ) return S_OK()
def __deleteSandboxFromExternalBackend( self, SEName, SEPFN ): if self.getCSOption( "DelayedExternalDeletion", True ): gLogger.info( "Setting deletion request" ) try: request = Request() request.RequestName = "RemoteSBDeletion:%s|%s:%s" % ( SEName, SEPFN, time.time() ) physicalRemoval = Operation() physicalRemoval.Type = "PhysicalRemoval" physicalRemoval.TargetSE = SEName fileToRemove = File() fileToRemove.PFN = SEPFN physicalRemoval.addFile( fileToRemove ) request.addOperation( physicalRemoval ) return ReqClient().putRequest( request ) except Exception as e: gLogger.exception( "Exception while setting deletion request" ) return S_ERROR( "Cannot set deletion request: %s" % str( e ) ) else: gLogger.info( "Deleting external Sandbox" ) try: return StorageElement( SEName ).removeFile( SEPFN ) except Exception as e: gLogger.exception( "RM raised an exception while trying to delete a remote sandbox" ) return S_ERROR( "RM raised an exception while trying to delete a remote sandbox" )
def getRegisterOperation(self, opFile, targetSE, type='RegisterFile', catalog=None): """ add RegisterReplica operation for file :param File opFile: operation file :param str targetSE: target SE """ # # add RegisterReplica operation registerOperation = Operation() registerOperation.Type = type registerOperation.TargetSE = targetSE if catalog: registerOperation.Catalog = catalog registerFile = File() registerFile.LFN = opFile.LFN registerFile.PFN = StorageElement(targetSE).getPfnForLfn( opFile.LFN).get('Value', {}).get('Successful', {}).get(opFile.LFN) registerFile.GUID = opFile.GUID registerFile.Checksum = opFile.Checksum registerFile.ChecksumType = opFile.ChecksumType registerFile.Size = opFile.Size registerOperation.addFile(registerFile) return registerOperation
def __setFileReplicationRequest( self, lfn, targetSE, fileMetaDict, sourceSE = '' ): """ Sets a registration request. """ self.log.info( 'Setting replication request for %s to %s' % ( lfn, targetSE ) ) transfer = Operation() transfer.Type = "ReplicateAndRegister" transfer.TargetSE = targetSE if sourceSE: transfer.SourceSE = sourceSE trFile = File() trFile.LFN = lfn cksm = fileMetaDict.get( "Checksum", None ) cksmType = fileMetaDict.get( "ChecksumType", None ) if cksm and cksmType: trFile.Checksum = cksm trFile.ChecksumType = cksmType size = fileMetaDict.get( "Size", 0 ) if size: trFile.Size = size guid = fileMetaDict.get( "GUID", "" ) if guid: trFile.GUID = guid transfer.addFile( trFile ) self.request.addOperation( transfer ) return S_OK()
def __setRegistrationRequest( self, lfn, targetSE, fileDict, catalog ): """ Sets a registration request :param str lfn: LFN :param list se: list of SE :param list catalog: list of catalogs to use :param dict fileDict: file metadata """ self.log.info( 'Setting registration request for %s at %s.' % ( lfn, targetSE ) ) for cat in catalog: register = Operation() register.Type = "RegisterFile" register.Catalog = cat register.TargetSE = targetSE regFile = File() regFile.LFN = lfn regFile.Checksum = fileDict.get( "Checksum", "" ) regFile.ChecksumType = fileDict.get( "ChecksumType", "" ) regFile.Size = fileDict.get( "Size", 0 ) regFile.GUID = fileDict.get( "GUID", "" ) se = StorageElement( targetSE ) pfn = se.getPfnForLfn( lfn ) if not pfn["OK"]: self.log.error( "unable to get PFN for LFN: %s" % pfn["Message"] ) return pfn regFile.PFN = pfn["Value"] register.addFile( regFile ) self.request.addOperation( register ) return S_OK()
def test02props( self ): """ test properties """ # # valid values operation = Operation() operation.Arguments = "foobar" self.assertEqual( operation.Arguments, "foobar", "wrong Arguments" ) operation.SourceSE = "CERN-RAW" self.assertEqual( operation.SourceSE, "CERN-RAW", "wrong SourceSE" ) operation.TargetSE = "CERN-RAW" self.assertEqual( operation.TargetSE, "CERN-RAW", "wrong TargetSE" ) operation.Catalog = "" self.assertEqual( operation.Catalog, "", "wrong Catalog" ) operation.Catalog = "BookkeepingDB" self.assertEqual( operation.Catalog, "BookkeepingDB", "wrong Catalog" ) operation.Error = "error" self.assertEqual( operation.Error, "error", "wrong Error" ) # # wrong props try: operation.RequestID = "foo" except Exception, error: self.assertEqual( type( error ), AttributeError, "wrong exc raised" ) self.assertEqual( str( error ), "can't set attribute", "wrong exc reason" )
def _setFileReplicationRequest(self, lfn, targetSE, fileMetaDict, sourceSE=""): """Sets a registration request.""" self.log.info("Setting ReplicateAndRegister request", "for %s to %s" % (lfn, targetSE)) transfer = Operation() transfer.Type = "ReplicateAndRegister" transfer.TargetSE = targetSE if sourceSE: transfer.SourceSE = sourceSE trFile = File() trFile.LFN = lfn cksm = fileMetaDict.get("Checksum", None) cksmType = fileMetaDict.get("ChecksumType", self.defaultChecksumType) if cksm and cksmType: trFile.Checksum = cksm trFile.ChecksumType = cksmType size = fileMetaDict.get("Size", 0) if size: trFile.Size = size guid = fileMetaDict.get("GUID", "") if guid: trFile.GUID = guid transfer.addFile(trFile) self.request.addOperation(transfer) return S_OK()
def test02props(self): """ test properties """ # # valid values operation = Operation() operation.OperationID = 1 self.assertEqual(operation.OperationID, 1, "wrong OperationID") operation.OperationID = "1" self.assertEqual(operation.OperationID, 1, "wrong OperationID") operation.Arguments = "foobar" self.assertEqual(operation.Arguments, "foobar", "wrong Arguments") operation.SourceSE = "CERN-RAW" self.assertEqual(operation.SourceSE, "CERN-RAW", "wrong SourceSE") operation.TargetSE = "CERN-RAW" self.assertEqual(operation.TargetSE, "CERN-RAW", "wrong TargetSE") operation.Catalog = "" self.assertEqual(operation.Catalog, "", "wrong Catalog") operation.Catalog = "BookkeepingDB" self.assertEqual(operation.Catalog, "BookkeepingDB", "wrong Catalog") operation.Error = "error" self.assertEqual(operation.Error, "error", "wrong Error") # # wrong props try: operation.RequestID = "foo" except Exception, error: self.assertEqual(type(error), AttributeError, "wrong exc raised") self.assertEqual(str(error), "can't set attribute", "wrong exc reason")
def getRegisterOperation(self, opFile, targetSE, type="RegisterFile", catalog=None): """add RegisterReplica operation for file :param ~DIRAC.RequestManagementSystem.Client.File.File opFile: operation file :param str targetSE: target SE """ # # add RegisterReplica operation registerOperation = Operation() registerOperation.Type = type registerOperation.TargetSE = targetSE if catalog: registerOperation.Catalog = catalog registerFile = File() registerFile.LFN = opFile.LFN registerFile.PFN = ( StorageElement(targetSE) .getURL(opFile.LFN, protocol=self.registrationProtocols) .get("Value", {}) .get("Successful", {}) .get(opFile.LFN) ) registerFile.GUID = opFile.GUID registerFile.Checksum = opFile.Checksum registerFile.ChecksumType = opFile.ChecksumType registerFile.Size = opFile.Size registerOperation.addFile(registerFile) return registerOperation
def __deleteSandboxFromExternalBackend(self, SEName, SEPFN): if self.getCSOption("DelayedExternalDeletion", True): gLogger.info("Setting deletion request") try: request = Request() request.RequestName = "RemoteSBDeletion:%s|%s:%s" % ( SEName, SEPFN, time.time()) physicalRemoval = Operation() physicalRemoval.Type = "PhysicalRemoval" physicalRemoval.TargetSE = SEName fileToRemove = File() fileToRemove.PFN = SEPFN physicalRemoval.addFile(fileToRemove) request.addOperation(physicalRemoval) return ReqClient().putRequest(request) except Exception as e: gLogger.exception("Exception while setting deletion request") return S_ERROR("Cannot set deletion request: %s" % str(e)) else: gLogger.info("Deleting external Sandbox") try: return StorageElement(SEName).removeFile(SEPFN) except Exception as e: gLogger.exception( "RM raised an exception while trying to delete a remote sandbox" ) return S_ERROR( "RM raised an exception while trying to delete a remote sandbox" )
def prepareTransformationTasks( self, transBody, taskDict, owner = '', ownerGroup = '', ownerDN = '' ): """ Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB """ if ( not owner ) or ( not ownerGroup ): res = getProxyInfo( False, False ) if not res['OK']: return res proxyInfo = res['Value'] owner = proxyInfo['username'] ownerGroup = proxyInfo['group'] if not ownerDN: res = getDNForUsername( owner ) if not res['OK']: return res ownerDN = res['Value'][0] requestOperation = 'ReplicateAndRegister' if transBody: try: _requestType, requestOperation = transBody.split( ';' ) except AttributeError: pass for taskID in sorted( taskDict ): paramDict = taskDict[taskID] if paramDict['InputData']: transID = paramDict['TransformationID'] oRequest = Request() transfer = Operation() transfer.Type = requestOperation transfer.TargetSE = paramDict['TargetSE'] if isinstance( paramDict['InputData'], list ): files = paramDict['InputData'] elif isinstance( paramDict['InputData'], basestring ): files = paramDict['InputData'].split( ';' ) for lfn in files: trFile = File() trFile.LFN = lfn transfer.addFile( trFile ) oRequest.addOperation( transfer ) oRequest.RequestName = _requestName( transID, taskID ) oRequest.OwnerDN = ownerDN oRequest.OwnerGroup = ownerGroup isValid = self.requestValidator.validate( oRequest ) if not isValid['OK']: return isValid taskDict[taskID]['TaskObject'] = oRequest return S_OK( taskDict )
def _replicateSourceFiles(self, request, lfns): """Create the replicateAndRegisterRequest. :param request: The request to add the operation to :param lfns: list of LFNs """ registerSource = Operation() registerSource.Type = "ReplicateAndRegister" registerSource.TargetSE = self.sourceSEs[0] self.addLFNs(registerSource, lfns, addPFN=True) request.addOperation(registerSource)
def __createLogUploadRequest(self, targetSE, logFileLFN, uploadedSE): """ Set a request to upload job log files from the output sandbox """ self.log.info('Setting log upload request for %s at %s' % (logFileLFN, targetSE)) logUpload = Operation() logUpload.Type = 'LogUpload' logUpload.TargetSE = targetSE logFile = File() logFile.LFN = logFileLFN logUpload.addFile(logFile) self.request.addOperation(logUpload) logRemoval = Operation() logRemoval.Type = 'RemoveFile' logRemoval.TargetSE = uploadedSE logRemoval.addFile(logFile) self.request.addOperation(logRemoval)
def _setFileRemovalRequest(self, lfn, se="", pfn=""): """Sets a removal request for a file including all replicas.""" remove = Operation() remove.Type = "RemoveFile" if se: remove.TargetSE = se rmFile = File() rmFile.LFN = lfn if pfn: rmFile.PFN = pfn remove.addFile(rmFile) self.request.addOperation(remove) return S_OK()
def __setFileRemovalRequest( self, lfn, se = '', pfn = '' ): """ Sets a removal request for a file including all replicas. """ remove = Operation() remove.Type = "RemoveFile" if se: remove.TargetSE = se rmFile = File() rmFile.LFN = lfn if pfn: rmFile.PFN = pfn remove.addFile( rmFile ) self.request.addOperation( remove ) return S_OK()
def checkRequestAndOp(listOfLFNs): req = Request() req.RequestName = "MyRequest" op = Operation() op.Type = "CheckMigration" op.TargetSE = "Foo-SE" for index, lfn in enumerate(listOfLFNs): oFile = File() oFile.LFN = lfn oFile.Size = index oFile.Checksum = "01130a%0d" % index oFile.ChecksumType = "adler32" op.addFile(oFile) req.addOperation(op) return req, op
def _singleOperationsBody(self, transBody, taskDict, ownerDN, ownerGroup): """deal with a Request that has just one operation, as it was sofar :param transBody: string, can be an empty string :param dict taskDict: dictionary of tasks, modified in this function :param str ownerDN: certificate DN used for the requests :param str onwerGroup: dirac group used for the requests :returns: None """ requestOperation = "ReplicateAndRegister" if transBody: try: _requestType, requestOperation = transBody.split(";") except AttributeError: pass failedTasks = [] # Do not remove sorted, we might pop elements in the loop for taskID, task in taskDict.items(): transID = task["TransformationID"] oRequest = Request() transfer = Operation() transfer.Type = requestOperation transfer.TargetSE = task["TargetSE"] # If there are input files if task.get("InputData"): if isinstance(task["InputData"], list): files = task["InputData"] elif isinstance(task["InputData"], six.string_types): files = task["InputData"].split(";") for lfn in files: trFile = File() trFile.LFN = lfn transfer.addFile(trFile) oRequest.addOperation(transfer) result = self._assignRequestToTask(oRequest, taskDict, transID, taskID, ownerDN, ownerGroup) if not result["OK"]: failedTasks.append(taskID) # Remove failed tasks for taskID in failedTasks: taskDict.pop(taskID)
def _singleOperationsBody(self, transBody, taskDict, ownerDN, ownerGroup): """ deal with a Request that has just one operation, as it was sofar :param transBody: string, can be an empty string :param dict taskDict: dictionary of tasks, modified in this function :param str ownerDN: certificate DN used for the requests :param str onwerGroup: dirac group used for the requests :returns: None """ requestOperation = 'ReplicateAndRegister' if transBody: try: _requestType, requestOperation = transBody.split(';') except AttributeError: pass failedTasks = [] # Do not remove sorted, we might pop elements in the loop for taskID, task in taskDict.iteritems(): transID = task['TransformationID'] oRequest = Request() transfer = Operation() transfer.Type = requestOperation transfer.TargetSE = task['TargetSE'] # If there are input files if task.get('InputData'): if isinstance(task['InputData'], list): files = task['InputData'] elif isinstance(task['InputData'], basestring): files = task['InputData'].split(';') for lfn in files: trFile = File() trFile.LFN = lfn transfer.addFile(trFile) oRequest.addOperation(transfer) result = self._assignRequestToTask(oRequest, taskDict, transID, taskID, ownerDN, ownerGroup) if not result['OK']: failedTasks.append(taskID) # Remove failed tasks for taskID in failedTasks: taskDict.pop(taskID)
def __deleteSandboxFromExternalBackend(self, SEName, SEPFN): if self.getCSOption("DelayedExternalDeletion", True): gLogger.info("Setting deletion request") try: # We need the hostDN used in order to pass these credentials to the # SandboxStoreDB.. hostCertLocation, _ = Locations.getHostCertificateAndKeyLocation( ) hostCert = X509Certificate.X509Certificate() hostCert.loadFromFile(hostCertLocation) hostDN = hostCert.getSubjectDN().get("Value") # use the host authentication to fetch the data result = self.sandboxDB.getSandboxOwner( SEName, SEPFN, hostDN, "hosts") if not result["OK"]: return result _owner, ownerDN, ownerGroup = result["Value"] request = Request() request.RequestName = "RemoteSBDeletion:%s|%s:%s" % ( SEName, SEPFN, time.time()) request.OwnerDN = ownerDN request.OwnerGroup = ownerGroup physicalRemoval = Operation() physicalRemoval.Type = "PhysicalRemoval" physicalRemoval.TargetSE = SEName fileToRemove = File() fileToRemove.PFN = SEPFN physicalRemoval.addFile(fileToRemove) request.addOperation(physicalRemoval) return ReqClient().putRequest(request) except Exception as e: gLogger.exception("Exception while setting deletion request") return S_ERROR(f"Cannot set deletion request: {e}") else: gLogger.info("Deleting external Sandbox") try: return StorageElement(SEName).removeFile(SEPFN) except Exception: gLogger.exception( "RM raised an exception while trying to delete a remote sandbox" ) return S_ERROR( "RM raised an exception while trying to delete a remote sandbox" )
def _setRegistrationRequest(self, lfn, targetSE, fileDict, catalog): """ Sets a registration request :param str lfn: LFN :param list se: list of SE (or just string) :param list catalog: list (or string) of catalogs to use :param dict fileDict: file metadata """ self.log.info('Setting registration request for %s at %s.' % (lfn, targetSE)) if not type(catalog) == type([]): catalog = [catalog] for cat in catalog: register = Operation() register.Type = "RegisterFile" register.Catalog = cat register.TargetSE = targetSE regFile = File() regFile.LFN = lfn regFile.Checksum = fileDict.get("Checksum", "") regFile.ChecksumType = fileDict.get("ChecksumType", self.defaultChecksumType) regFile.Size = fileDict.get("Size", 0) regFile.GUID = fileDict.get("GUID", "") se = StorageElement(targetSE) pfn = se.getPfnForLfn(lfn) if not pfn["OK"] or lfn not in pfn["Value"]['Successful']: self.log.error( "unable to get PFN for LFN: %s" % pfn.get('Message', pfn.get('Value', {}).get('Failed', {}).get(lfn))) return pfn regFile.PFN = pfn["Value"]['Successful'][lfn] register.addFile(regFile) self.request.addOperation(register) return S_OK()
def _singleOperationsBody(self, transBody, taskDict, ownerDN, ownerGroup ): """ deal with a Request that has just one operation, as it was sofar :param transBody: string, can be an empty string :param dict taskDict: dictionary of tasks, modified in this function :param str ownerDN: certificate DN used for the requests :param str onwerGroup: dirac group used for the requests :returns: None """ requestOperation = 'ReplicateAndRegister' if transBody: try: _requestType, requestOperation = transBody.split( ';' ) except AttributeError: pass # Do not remove sorted, we might pop elements in the loop for taskID in sorted( taskDict ): paramDict = taskDict[taskID] transID = paramDict['TransformationID'] oRequest = Request() transfer = Operation() transfer.Type = requestOperation transfer.TargetSE = paramDict['TargetSE'] # If there are input files if paramDict.get('InputData'): if isinstance( paramDict['InputData'], list ): files = paramDict['InputData'] elif isinstance( paramDict['InputData'], basestring ): files = paramDict['InputData'].split( ';' ) for lfn in files: trFile = File() trFile.LFN = lfn transfer.addFile( trFile ) oRequest.addOperation( transfer ) self._assignRequestToTask( oRequest, taskDict, transID, taskID, ownerDN, ownerGroup )
def prepareTransformationTasks( self, transBody, taskDict, owner = '', ownerGroup = '' ): """ Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB """ requestOperation = 'ReplicateAndRegister' if transBody: try: _requestType, requestOperation = transBody.split( ';' ) except AttributeError: pass for taskID in sorted( taskDict ): paramDict = taskDict[taskID] if paramDict['InputData']: transID = paramDict['TransformationID'] oRequest = Request() transfer = Operation() transfer.Type = requestOperation transfer.TargetSE = paramDict['TargetSE'] if type( paramDict['InputData'] ) == type( [] ): files = paramDict['InputData'] elif type( paramDict['InputData'] ) == type( '' ): files = paramDict['InputData'].split( ';' ) for lfn in files: trFile = File() trFile.LFN = lfn transfer.addFile( trFile ) oRequest.addOperation( transfer ) oRequest.RequestName = str( transID ).zfill( 8 ) + '_' + str( taskID ).zfill( 8 ) oRequest.OwnerDN = owner oRequest.OwnerGroup = ownerGroup isValid = gRequestValidator.validate( oRequest ) if not isValid['OK']: return isValid taskDict[taskID]['TaskObject'] = oRequest return S_OK( taskDict )
def _setRegistrationRequest(self, lfn, targetSE, fileDict, catalog): """Sets a registration request :param str lfn: LFN :param list se: list of SE (or just string) :param list catalog: list (or string) of catalogs to use :param dict fileDict: file metadata """ self.log.info("Setting registration request", "for %s at %s." % (lfn, targetSE)) if not isinstance(catalog, list): catalog = [catalog] for cat in catalog: register = Operation() register.Type = "RegisterFile" register.Catalog = cat register.TargetSE = targetSE regFile = File() regFile.LFN = lfn regFile.Checksum = fileDict.get("Checksum", "") regFile.ChecksumType = fileDict.get("ChecksumType", self.defaultChecksumType) regFile.Size = fileDict.get("Size", 0) regFile.GUID = fileDict.get("GUID", "") se = StorageElement(targetSE) res = returnSingleResult(se.getURL(lfn, self.registrationProtocols)) if not res["OK"]: self.log.error("Unable to get PFN for LFN", res["Message"]) return res regFile.PFN = res["Value"] register.addFile(regFile) self.request.addOperation(register) return S_OK()
def __setReplicaRemovalRequest( self, lfn, se ): """ Sets a removal request for a replica. :param str lfn: LFN :param se: """ if type( se ) == str: se = ",".join( [ se.strip() for se in se.split( "," ) if se.strip() ] ) removeReplica = Operation() removeReplica.Type = "RemoveReplica" removeReplica.TargetSE = se replicaToRemove = File() replicaToRemove.LFN = lfn removeReplica.addFile( replicaToRemove ) self.request.addOperation( removeReplica ) return S_OK()
def getRegisterOperation(self, opFile, targetSE): """ add RegisterReplica operation for file :param File opFile: operation file :param str targetSE: target SE """ # # add RegisterReplica operation registerOperation = Operation() registerOperation.Type = "RegisterFile" registerOperation.TargetSE = targetSE registerFile = File() registerFile.LFN = opFile.LFN registerFile.PFN = opFile.PFN registerFile.GUID = opFile.GUID registerFile.Checksum = opFile.Checksum registerFile.ChecksumType = opFile.ChecksumType registerFile.Size = opFile.Size registerOperation.addFile(registerFile) return registerOperation
def _setReplicaRemovalRequest(self, lfn, se): """Sets a removal request for a replica. :param str lfn: LFN :param se: """ if isinstance(se, str): se = ",".join([se.strip() for se in se.split(",") if se.strip()]) removeReplica = Operation() removeReplica.Type = "RemoveReplica" removeReplica.TargetSE = se replicaToRemove = File() replicaToRemove.LFN = lfn removeReplica.addFile(replicaToRemove) self.request.addOperation(removeReplica) return S_OK()
def getRegisterOperation( self, opFile, targetSE ): """ add RegisterReplica operation for file :param File opFile: operation file :param str targetSE: target SE """ # # add RegisterReplica operation registerOperation = Operation() registerOperation.Type = "RegisterFile" registerOperation.TargetSE = targetSE registerFile = File() registerFile.LFN = opFile.LFN registerFile.PFN = opFile.PFN registerFile.GUID = opFile.GUID registerFile.Checksum = opFile.Checksum registerFile.ChecksumType = opFile.ChecksumType registerFile.Size = opFile.Size registerOperation.addFile( registerFile ) return registerOperation
def addRegisterReplica( self, opFile, targetSE ): """ add RegisterReplica operation for file :param File opFile: operation file :param str targetSE: target SE """ # # add RegisterReplica operation registerOperation = Operation() registerOperation.Type = "RegisterFile" registerOperation.TargetSE = targetSE registerFile = File() registerFile.LFN = opFile.LFN registerFile.PFN = opFile.PFN registerFile.GUID = opFile.GUID registerFile.Checksum = opFile.Checksum registerFile.ChecksumType = opFile.ChecksumType registerFile.Size = opFile.Size registerOperation.addFile( registerFile ) self.request.insertAfter( registerOperation, self.operation ) return S_OK()
def addRegisterReplica(self, opFile, targetSE): """ add RegisterReplica operation for file :param File opFile: operation file :param str targetSE: target SE """ # # add RegisterReplica operation registerOperation = Operation() registerOperation.Type = "RegisterFile" registerOperation.TargetSE = targetSE registerFile = File() registerFile.LFN = opFile.LFN registerFile.PFN = opFile.PFN registerFile.GUID = opFile.GUID registerFile.Checksum = opFile.Checksum registerFile.ChecksumType = opFile.ChecksumType registerFile.Size = opFile.Size registerOperation.addFile(registerFile) self.request.insertAfter(registerOperation, self.operation) return S_OK()
def _setRegistrationRequest( self, lfn, targetSE, fileDict, catalog ): """ Sets a registration request :param str lfn: LFN :param list se: list of SE (or just string) :param list catalog: list (or string) of catalogs to use :param dict fileDict: file metadata """ self.log.info( 'Setting registration request for %s at %s.' % ( lfn, targetSE ) ) if not isinstance( catalog, list ): catalog = [catalog] for cat in catalog: register = Operation() register.Type = "RegisterFile" register.Catalog = cat register.TargetSE = targetSE regFile = File() regFile.LFN = lfn regFile.Checksum = fileDict.get( "Checksum", "" ) regFile.ChecksumType = fileDict.get( "ChecksumType", self.defaultChecksumType ) regFile.Size = fileDict.get( "Size", 0 ) regFile.GUID = fileDict.get( "GUID", "" ) se = StorageElement( targetSE ) pfn = se.getURL( lfn, self.registrationProtocols ) if not pfn["OK"] or lfn not in pfn["Value"]['Successful']: self.log.error( "Unable to get PFN for LFN", "%s" % pfn.get( 'Message', pfn.get( 'Value', {} ).get( 'Failed', {} ).get( lfn ) ) ) return pfn regFile.PFN = pfn["Value"]['Successful'][lfn] register.addFile( regFile ) self.request.addOperation( register ) return S_OK()
def getRegisterOperation( self, opFile, targetSE, type = 'RegisterFile', catalog = None ): """ add RegisterReplica operation for file :param File opFile: operation file :param str targetSE: target SE """ # # add RegisterReplica operation registerOperation = Operation() registerOperation.Type = type registerOperation.TargetSE = targetSE if catalog: registerOperation.Catalog = catalog registerFile = File() registerFile.LFN = opFile.LFN registerFile.PFN = StorageElement( targetSE ).getURL( opFile.LFN, protocol = self.registrationProtocols ).get( 'Value', {} ).get( 'Successful', {} ).get( opFile.LFN ) registerFile.GUID = opFile.GUID registerFile.Checksum = opFile.Checksum registerFile.ChecksumType = opFile.ChecksumType registerFile.Size = opFile.Size registerOperation.addFile( registerFile ) return registerOperation
def test_valid_properties(): operation = Operation() operation.Arguments = "foobar" assert operation.Arguments == b"foobar", "wrong Arguments" operation.SourceSE = "CERN-RAW" assert operation.SourceSE == "CERN-RAW", "wrong SourceSE" operation.TargetSE = "CERN-RAW" assert operation.TargetSE == "CERN-RAW", "wrong TargetSE" operation.Catalog = "" assert operation.Catalog == "", "wrong Catalog" operation.Catalog = "BookkeepingDB" assert operation.Catalog == "BookkeepingDB", "wrong Catalog" operation.Error = "error" assert operation.Error == "error", "wrong Error" toJSON = operation.toJSON() assert toJSON["OK"]
gLogger.error( "LFN list is empty!!!" ) error = -1 continue if len( lfnChunk ) > Operation.MAX_FILES: gLogger.error( "too many LFNs, max number of files per operation is %s" % Operation.MAX_FILES ) error = -1 continue count += 1 request = Request() request.RequestName = requestName if not multiRequests else '%s_%d' % ( requestName, count ) replicateAndRegister = Operation() replicateAndRegister.Type = "ReplicateAndRegister" replicateAndRegister.TargetSE = ",".join( targetSEs ) if catalog is not None: replicateAndRegister.Catalog = catalog for lfn in lfnChunk: metaDict = metaDatas["Successful"][lfn] opFile = File() opFile.LFN = lfn opFile.Size = metaDict["Size"] if "Checksum" in metaDict: # # should check checksum type, now assuming Adler32 (metaDict["ChecksumType"] = 'AD' opFile.Checksum = metaDict["Checksum"] opFile.ChecksumType = "ADLER32" replicateAndRegister.addFile( opFile )
gLogger.error("Location not known, upload to location yourself, and publish in CS manually") return S_ERROR() else: lfnpath = "%s%s" % (path, os.path.basename(appTar)) res = datMan.putAndRegister(lfnpath, appTar, ops.getValue('Software/BaseStorageElement', "CERN-SRM")) if not res['OK']: return res request = Request() requestClient = ReqClient() request.RequestName = 'copy_%s' % os.path.basename(appTar).replace(".tgz", "").replace(".tar.gz", "") request.SourceComponent = 'ReplicateILCSoft' copies_at = ops.getValue('Software/CopiesAt', []) for copies in copies_at: transfer = Operation() transfer.Type = "ReplicateAndRegister" transfer.TargetSE = copies trFile = File() trFile.LFN = lfnpath trFile.GUID = "" transfer.addFile(trFile) request.addOperation(transfer) res = RequestValidator().validate(request) if not res['OK']: return res if copies_at: res = requestClient.putRequest(request) if not res['OK']: gLogger.error('Could not set replication request', res['Message']) return S_OK('Application uploaded')
from DIRAC.RequestManagementSystem.Client.Operation import Operation from DIRAC.RequestManagementSystem.Client.File import File from DIRAC.RequestManagementSystem.private.RequestValidator import RequestValidator from DIRAC.Resources.Catalog.FileCatalog import FileCatalog reqClient = ReqClient() fc = FileCatalog() for lfnList in breakListIntoChunks( lfns, 100 ): oRequest = Request() oRequest.RequestName = "%s_%s" % ( md5( repr( time.time() ) ).hexdigest()[:16], md5( repr( time.time() ) ).hexdigest()[:16] ) replicateAndRegister = Operation() replicateAndRegister.Type = 'ReplicateAndRegister' replicateAndRegister.TargetSE = targetSE res = fc.getFileMetadata( lfnList ) if not res['OK']: print "Can't get file metadata: %s" % res['Message'] DIRAC.exit( 1 ) if res['Value']['Failed']: print "Could not get the file metadata of the following, so skipping them:" for fFile in res['Value']['Failed']: print fFile lfnMetadata = res['Value']['Successful'] for lfn in lfnMetadata: rarFile = File() rarFile.LFN = lfn
gLogger.error( "%s does not exist" % PFN ) DIRAC.exit( -1 ) if not os.path.isfile( PFN ): gLogger.error( "%s is not a file" % PFN ) DIRAC.exit( -1 ) PFN = os.path.abspath( PFN ) size = os.path.getsize( PFN ) adler32 = fileAdler( PFN ) request = Request() request.RequestName = requestName putAndRegister = Operation() putAndRegister.Type = "PutAndRegister" putAndRegister.TargetSE = targetSE opFile = File() opFile.LFN = LFN opFile.PFN = PFN opFile.Size = size opFile.Checksum = adler32 opFile.ChecksumType = "ADLER32" putAndRegister.addFile( opFile ) request.addOperation( putAndRegister ) reqClient = ReqClient() putRequest = reqClient.putRequest( request ) if not putRequest["OK"]: gLogger.error( "unable to put request '%s': %s" % ( requestName, putRequest["Message"] ) ) DIRAC.exit( -1 ) gLogger.always( "Request '%s' has been put to ReqDB for execution." % requestName )
continue if len( lfnChunk ) > Operation.MAX_FILES: gLogger.error( "too many LFNs, max number of files per operation is %s" % Operation.MAX_FILES ) error = -1 continue count += 1 request = Request() request.RequestName = "%s_%s" % ( md5( repr( time.time() ) ).hexdigest()[:16], md5( repr( time.time() ) ).hexdigest()[:16] ) moveReplica = Operation() moveReplica.Type = 'MoveReplica' moveReplica.SourceSE = sourceSE moveReplica.TargetSE = ",".join( targetSEs ) for lfn in lfnChunk: metaDict = metaDatas["Successful"][lfn] opFile = File() opFile.LFN = lfn opFile.Size = metaDict["Size"] if "Checksum" in metaDict: # # should check checksum type, now assuming Adler32 (metaDict["ChecksumType"] = 'AD' opFile.Checksum = metaDict["Checksum"] opFile.ChecksumType = "ADLER32" moveReplica.addFile( opFile ) request.addOperation( moveReplica )
def test05FTS( self ): """ FTS state machine """ req = Request() req.RequestName = "FTSTest" ftsTransfer = Operation() ftsTransfer.Type = "ReplicateAndRegister" ftsTransfer.TargetSE = "CERN-USER" ftsFile = File() ftsFile.LFN = "/a/b/c" ftsFile.Checksum = "123456" ftsFile.ChecksumType = "Adler32" ftsTransfer.addFile( ftsFile ) req.addOperation( ftsTransfer ) self.assertEqual( req.Status, "Waiting", "1. wrong request status: %s" % req.Status ) self.assertEqual( ftsTransfer.Status, "Waiting", "1. wrong ftsStatus status: %s" % ftsTransfer.Status ) # # scheduled ftsFile.Status = "Scheduled" self.assertEqual( ftsTransfer.Status, "Scheduled", "2. wrong status for ftsTransfer: %s" % ftsTransfer.Status ) self.assertEqual( req.Status, "Scheduled", "2. wrong status for request: %s" % req.Status ) # # add new operation before FTS insertBefore = Operation() insertBefore.Type = "RegisterReplica" insertBefore.TargetSE = "CERN-USER" insertFile = File() insertFile.LFN = "/a/b/c" insertFile.PFN = "http://foo/bar" insertBefore.addFile( insertFile ) req.insertBefore( insertBefore, ftsTransfer ) self.assertEqual( insertBefore.Status, "Waiting", "3. wrong status for insertBefore: %s" % insertBefore.Status ) self.assertEqual( ftsTransfer.Status, "Scheduled", "3. wrong status for ftsStatus: %s" % ftsTransfer.Status ) self.assertEqual( req.Status, "Waiting", "3. wrong status for request: %s" % req.Status ) # # prev done insertFile.Status = "Done" self.assertEqual( insertBefore.Status, "Done", "4. wrong status for insertBefore: %s" % insertBefore.Status ) self.assertEqual( ftsTransfer.Status, "Scheduled", "4. wrong status for ftsStatus: %s" % ftsTransfer.Status ) self.assertEqual( req.Status, "Scheduled", "4. wrong status for request: %s" % req.Status ) # # reschedule ftsFile.Status = "Waiting" self.assertEqual( insertBefore.Status, "Done", "5. wrong status for insertBefore: %s" % insertBefore.Status ) self.assertEqual( ftsTransfer.Status, "Waiting", "5. wrong status for ftsStatus: %s" % ftsTransfer.Status ) self.assertEqual( req.Status, "Waiting", "5. wrong status for request: %s" % req.Status ) # # fts done ftsFile.Status = "Done" self.assertEqual( insertBefore.Status, "Done", "5. wrong status for insertBefore: %s" % insertBefore.Status ) self.assertEqual( ftsTransfer.Status, "Done", "5. wrong status for ftsStatus: %s" % ftsTransfer.Status ) self.assertEqual( req.Status, "Done", "5. wrong status for request: %s" % req.Status )
def prepareTransformationTasks( self, transBody, taskDict, owner = '', ownerGroup = '', ownerDN = '' ): """ Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB """ if not taskDict: return S_OK({}) if ( not owner ) or ( not ownerGroup ): res = getProxyInfo( False, False ) if not res['OK']: return res proxyInfo = res['Value'] owner = proxyInfo['username'] ownerGroup = proxyInfo['group'] if not ownerDN: res = getDNForUsername( owner ) if not res['OK']: return res ownerDN = res['Value'][0] requestOperation = 'ReplicateAndRegister' if transBody: try: _requestType, requestOperation = transBody.split( ';' ) except AttributeError: pass # Do not remove sorted, we might pop elements in the loop for taskID in sorted( taskDict ): paramDict = taskDict[taskID] transID = paramDict['TransformationID'] oRequest = Request() transfer = Operation() transfer.Type = requestOperation transfer.TargetSE = paramDict['TargetSE'] # If there are input files if paramDict['InputData']: if isinstance( paramDict['InputData'], list ): files = paramDict['InputData'] elif isinstance( paramDict['InputData'], basestring ): files = paramDict['InputData'].split( ';' ) for lfn in files: trFile = File() trFile.LFN = lfn transfer.addFile( trFile ) oRequest.addOperation( transfer ) oRequest.RequestName = _requestName( transID, taskID ) oRequest.OwnerDN = ownerDN oRequest.OwnerGroup = ownerGroup isValid = self.requestValidator.validate( oRequest ) if not isValid['OK']: self.log.error( "Error creating request for task", "%s %s" % ( taskID, isValid ) ) # This works because we loop over a copy of the keys ! taskDict.pop( taskID ) continue taskDict[taskID]['TaskObject'] = oRequest return S_OK( taskDict )
reqClient = ReqClient() fc = FileCatalog() requestOperation = 'RemoveReplica' if targetSE == 'All': requestOperation = 'RemoveFile' for lfnList in breakListIntoChunks( lfns, 100 ): oRequest = Request() requestName = "%s_%s" % ( md5( repr( time.time() ) ).hexdigest()[:16], md5( repr( time.time() ) ).hexdigest()[:16] ) oRequest.RequestName = requestName oOperation = Operation() oOperation.Type = requestOperation oOperation.TargetSE = targetSE res = fc.getFileMetadata( lfnList ) if not res['OK']: print "Can't get file metadata: %s" % res['Message'] DIRAC.exit( 1 ) if res['Value']['Failed']: print "Could not get the file metadata of the following, so skipping them:" for fFile in res['Value']['Failed']: print fFile lfnMetadata = res['Value']['Successful'] for lfn in lfnMetadata: rarFile = File() rarFile.LFN = lfn
def createRequest(self, requestName, archiveLFN, lfnChunk): """Create the Request.""" request = Request() request.RequestName = requestName self._checkReplicaSites(request, lfnChunk) archiveFiles = Operation() archiveFiles.Type = "ArchiveFiles" archiveFiles.Arguments = DEncode.encode({ "SourceSE": self.sourceSEs[0], "TarballSE": self.switches["TarballSE"], "RegisterDescendent": self.switches["RegisterDescendent"], "ArchiveLFN": archiveLFN, }) self.addLFNs(archiveFiles, lfnChunk) request.addOperation(archiveFiles) # Replicate the Tarball, ArchiveFiles will upload it if self.switches.get("ReplicateTarball"): replicateAndRegisterTarBall = Operation() replicateAndRegisterTarBall.Type = "ReplicateAndRegister" replicateAndRegisterTarBall.TargetSE = self.targetSE opFile = File() opFile.LFN = archiveLFN replicateAndRegisterTarBall.addFile(opFile) request.addOperation(replicateAndRegisterTarBall) if self.switches.get("CheckMigration"): checkMigrationTarBall = Operation() checkMigrationTarBall.Type = "CheckMigration" migrationTarget = self.targetSE if self.switches.get( "ReplicateTarball") else self.switches["TarballSE"] checkMigrationTarBall.TargetSE = migrationTarget opFile = File() opFile.LFN = archiveLFN checkMigrationTarBall.addFile(opFile) request.addOperation(checkMigrationTarBall) # Register Archive Replica for LFNs if self.switches.get("ArchiveSE"): registerArchived = Operation() registerArchived.Type = "RegisterReplica" registerArchived.TargetSE = self.switches.get("ArchiveSE") self.addLFNs(registerArchived, lfnChunk, addPFN=True) request.addOperation(registerArchived) # Remove all Other Replicas for LFNs if self.switches.get("RemoveReplicas"): removeArchiveReplicas = Operation() removeArchiveReplicas.Type = "RemoveReplica" removeArchiveReplicas.TargetSE = ",".join(self.replicaSEs) self.addLFNs(removeArchiveReplicas, lfnChunk) request.addOperation(removeArchiveReplicas) # Remove all Replicas for LFNs if self.switches.get("RemoveFiles"): removeArchiveFiles = Operation() removeArchiveFiles.Type = "RemoveFile" self.addLFNs(removeArchiveFiles, lfnChunk) request.addOperation(removeArchiveFiles) # Remove Original tarball replica if self.switches.get("ReplicateTarball"): removeTarballOrg = Operation() removeTarballOrg.Type = "RemoveReplica" removeTarballOrg.TargetSE = self.sourceSEs[0] opFile = File() opFile.LFN = archiveLFN removeTarballOrg.addFile(opFile) request.addOperation(removeTarballOrg) return request
def createRequest( reqType ): r = Request() # Simple failover op1 = Operation() f = File() f.LFN = '/This/is/an/LFN' op1.addFile( f ) op1.Type = 'ReplicateAndRegister' op1.SourceSE = 'CERN-FAILOVER' op1.TargetSE = 'CERN-BUFFER' r.addOperation( op1 ) op2 = Operation() op2.addFile( f ) op2.Type = 'RemoveReplica' op2.TargetSE = 'CERN-FAILOVER' r.addOperation( op2 ) if reqType == 0: return r # two files for Failover f1 = File() f1.LFN = '/This/is/a/second/LFN' op3 = Operation() op3.addFile( f1 ) op3.Type = 'ReplicateAndRegister' op3.SourceSE = 'CERN-FAILOVER' op3.TargetSE = 'CERN-BUFFER' r.addOperation( op3 ) op3 = Operation() op3.addFile( f1 ) op3.Type = 'RemoveReplica' op3.TargetSE = 'CERN-FAILOVER' r.addOperation( op3 ) if reqType == 1: return r op = Operation() op.Type = 'ForwardDiset' if reqType == 2: r.addOperation( op ) return r r.insertBefore( op, r[0] ) if reqType == 3: return r op4 = Operation() op4.Type = 'ForwardDiset' r.addOperation( op4 ) if reqType == 4: return r # 2 different FAILOVER SEs: removal not optimized r[1].SourceSE = 'RAL-FAILOVER' r[2].SourceSE = 'RAL-FAILOVER' if reqType == 5: return r # 2 different destinations, same FAILOVER: replication not optimized r[3].SourceSE = 'RAL-FAILOVER' r[4].SourceSE = 'RAL-FAILOVER' r[3].TargetSE = 'RAL-BUFFER' if reqType == 6: return r print 'This should not happen, reqType =', reqType
if len(lfnChunk) > Operation.MAX_FILES: gLogger.error( "too many LFNs, max number of files per operation is %s" % Operation.MAX_FILES) error = -1 continue count += 1 request = Request() request.RequestName = requestName if not multiRequests else '%s_%d' % ( requestName, count) replicateAndRegister = Operation() replicateAndRegister.Type = "ReplicateAndRegister" replicateAndRegister.TargetSE = ",".join(targetSEs) if catalog is not None: replicateAndRegister.Catalog = catalog for lfn in lfnChunk: metaDict = metaDatas["Successful"][lfn] opFile = File() opFile.LFN = lfn opFile.Size = metaDict["Size"] if "Checksum" in metaDict: # # should check checksum type, now assuming Adler32 (metaDict["ChecksumType"] = 'AD' opFile.Checksum = metaDict["Checksum"] opFile.ChecksumType = "ADLER32" replicateAndRegister.addFile(opFile)
def main(): # Registering arguments will automatically add their description to the help menu Script.registerArgument(" SE: StorageElement|All") Script.registerArgument(["LFN: LFN or file containing a List of LFNs"]) Script.parseCommandLine(ignoreErrors=False) # parseCommandLine show help when mandatory arguments are not specified or incorrect argument args = Script.getPositionalArgs() targetSE = args.pop(0) lfns = [] for inputFileName in args: if os.path.exists(inputFileName): with open(inputFileName, "r") as inputFile: string = inputFile.read() lfns.extend([lfn.strip() for lfn in string.splitlines()]) else: lfns.append(inputFileName) from DIRAC.Resources.Storage.StorageElement import StorageElement import DIRAC # Check is provided SE is OK if targetSE != "All": se = StorageElement(targetSE) if not se.valid: print(se.errorReason) print() Script.showHelp() from DIRAC.RequestManagementSystem.Client.Request import Request from DIRAC.RequestManagementSystem.Client.Operation import Operation from DIRAC.RequestManagementSystem.Client.File import File from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient from DIRAC.RequestManagementSystem.private.RequestValidator import RequestValidator from DIRAC.Resources.Catalog.FileCatalog import FileCatalog reqClient = ReqClient() fc = FileCatalog() requestOperation = "RemoveReplica" if targetSE == "All": requestOperation = "RemoveFile" for lfnList in breakListIntoChunks(lfns, 100): oRequest = Request() requestName = "%s_%s" % ( md5(repr(time.time()).encode()).hexdigest()[:16], md5(repr(time.time()).encode()).hexdigest()[:16], ) oRequest.RequestName = requestName oOperation = Operation() oOperation.Type = requestOperation oOperation.TargetSE = targetSE res = fc.getFileMetadata(lfnList) if not res["OK"]: print("Can't get file metadata: %s" % res["Message"]) DIRAC.exit(1) if res["Value"]["Failed"]: print( "Could not get the file metadata of the following, so skipping them:" ) for fFile in res["Value"]["Failed"]: print(fFile) lfnMetadata = res["Value"]["Successful"] for lfn in lfnMetadata: rarFile = File() rarFile.LFN = lfn rarFile.Size = lfnMetadata[lfn]["Size"] rarFile.Checksum = lfnMetadata[lfn]["Checksum"] rarFile.GUID = lfnMetadata[lfn]["GUID"] rarFile.ChecksumType = "ADLER32" oOperation.addFile(rarFile) oRequest.addOperation(oOperation) isValid = RequestValidator().validate(oRequest) if not isValid["OK"]: print("Request is not valid: ", isValid["Message"]) DIRAC.exit(1) result = reqClient.putRequest(oRequest) if result["OK"]: print("Request %d Submitted" % result["Value"]) else: print("Failed to submit Request: ", result["Message"])
def createRequest(self, requestName, archiveLFN, lfnChunk): """Create the Request.""" request = Request() request.RequestName = requestName self._checkReplicaSites(request, lfnChunk) archiveFiles = Operation() archiveFiles.Type = 'ArchiveFiles' archiveFiles.Arguments = DEncode.encode({'SourceSE': self.sourceSEs[0], 'TarballSE': self.switches['TarballSE'], 'RegisterDescendent': self.switches['RegisterDescendent'], 'ArchiveLFN': archiveLFN}) self.addLFNs(archiveFiles, lfnChunk) request.addOperation(archiveFiles) # Replicate the Tarball, ArchiveFiles will upload it if self.switches.get('ReplicateTarball'): replicateAndRegisterTarBall = Operation() replicateAndRegisterTarBall.Type = 'ReplicateAndRegister' replicateAndRegisterTarBall.TargetSE = self.targetSE opFile = File() opFile.LFN = archiveLFN replicateAndRegisterTarBall.addFile(opFile) request.addOperation(replicateAndRegisterTarBall) if self.switches.get('CheckMigration'): checkMigrationTarBall = Operation() checkMigrationTarBall.Type = 'CheckMigration' migrationTarget = self.targetSE if self.switches.get('ReplicateTarball') else self.switches['TarballSE'] checkMigrationTarBall.TargetSE = migrationTarget opFile = File() opFile.LFN = archiveLFN checkMigrationTarBall.addFile(opFile) request.addOperation(checkMigrationTarBall) # Register Archive Replica for LFNs if self.switches.get('ArchiveSE'): registerArchived = Operation() registerArchived.Type = 'RegisterReplica' registerArchived.TargetSE = self.switches.get('ArchiveSE') self.addLFNs(registerArchived, lfnChunk, addPFN=True) request.addOperation(registerArchived) # Remove all Other Replicas for LFNs if self.switches.get('RemoveReplicas'): removeArchiveReplicas = Operation() removeArchiveReplicas.Type = 'RemoveReplica' removeArchiveReplicas.TargetSE = ','.join(self.replicaSEs) self.addLFNs(removeArchiveReplicas, lfnChunk) request.addOperation(removeArchiveReplicas) # Remove all Replicas for LFNs if self.switches.get('RemoveFiles'): removeArchiveFiles = Operation() removeArchiveFiles.Type = 'RemoveFile' self.addLFNs(removeArchiveFiles, lfnChunk) request.addOperation(removeArchiveFiles) # Remove Original tarball replica if self.switches.get('ReplicateTarball'): removeTarballOrg = Operation() removeTarballOrg.Type = 'RemoveReplica' removeTarballOrg.TargetSE = self.sourceSEs[0] opFile = File() opFile.LFN = archiveLFN removeTarballOrg.addFile(opFile) request.addOperation(removeTarballOrg) return request
def createRequest(reqType): r = Request() # Simple failover op1 = Operation() f = File() f.LFN = '/This/is/an/LFN' op1.addFile(f) op1.Type = 'ReplicateAndRegister' op1.SourceSE = 'CERN-FAILOVER' op1.TargetSE = 'CERN-BUFFER' r.addOperation(op1) op2 = Operation() op2.addFile(f) op2.Type = 'RemoveReplica' op2.TargetSE = 'CERN-FAILOVER' r.addOperation(op2) if reqType == 0: return r # two files for Failover f1 = File() f1.LFN = '/This/is/a/second/LFN' op3 = Operation() op3.addFile(f1) op3.Type = 'ReplicateAndRegister' op3.SourceSE = 'CERN-FAILOVER' op3.TargetSE = 'CERN-BUFFER' r.addOperation(op3) op3 = Operation() op3.addFile(f1) op3.Type = 'RemoveReplica' op3.TargetSE = 'CERN-FAILOVER' r.addOperation(op3) if reqType == 1: return r op = Operation() op.Type = 'ForwardDiset' if reqType == 2: r.addOperation(op) return r r.insertBefore(op, r[0]) if reqType == 3: return r op4 = Operation() op4.Type = 'ForwardDiset' r.addOperation(op4) if reqType == 4: return r # 2 different FAILOVER SEs: removal not optimized r[1].SourceSE = 'RAL-FAILOVER' r[2].SourceSE = 'RAL-FAILOVER' if reqType == 5: return r # 2 different destinations, same FAILOVER: replication not optimized r[3].SourceSE = 'RAL-FAILOVER' r[4].SourceSE = 'RAL-FAILOVER' r[3].TargetSE = 'RAL-BUFFER' if reqType == 6: return r print 'This should not happen, reqType =', reqType