def _createLogUploadRequest(self, targetSE, logFileLFN, uploadedSE): """ Set a request to upload job log files from the output sandbox Changed to be similar to LHCb createLogUploadRequest using LHCb LogUpload Request and Removal Request """ self.log.info('Setting log upload request for %s at %s' %(targetSE, logFileLFN)) request = self._getRequestContainer() logUpload = Operation() logUpload.Type = "LogUpload" logUpload.TargetSE = targetSE upFile = File() upFile.LFN = logFileLFN logUpload.addFile( upFile ) logRemoval = Operation() logRemoval.Type = 'RemoveFile' logRemoval.TargetSE = uploadedSE logRemoval.addFile( upFile ) request.addOperation ( logUpload ) request.addOperation ( logRemoval ) self.workflow_commons['Request'] = request return S_OK()
def _createLogUploadRequest(self, targetSE, logFileLFN, uploadedSE): """ Set a request to upload job log files from the output sandbox Changed to be similar to LHCb createLogUploadRequest using LHCb LogUpload Request and Removal Request """ self.log.info('Setting log upload request for %s at %s' % (targetSE, logFileLFN)) request = self._getRequestContainer() logUpload = Operation() logUpload.Type = "LogUpload" logUpload.TargetSE = targetSE upFile = File() upFile.LFN = logFileLFN logUpload.addFile(upFile) logRemoval = Operation() logRemoval.Type = 'RemoveFile' logRemoval.TargetSE = uploadedSE logRemoval.addFile(upFile) request.addOperation(logUpload) request.addOperation(logRemoval) self.workflow_commons['Request'] = request return S_OK()
def __deleteSandboxFromExternalBackend( self, SEName, SEPFN ): if self.getCSOption( "DelayedExternalDeletion", True ): gLogger.info( "Setting deletion request" ) try: request = Request() request.RequestName = "RemoteSBDeletion:%s|%s:%s" % ( SEName, SEPFN, time.time() ) physicalRemoval = Operation() physicalRemoval.Type = "PhysicalRemoval" physicalRemoval.TargetSE = SEName fileToRemove = File() fileToRemove.PFN = SEPFN physicalRemoval.addFile( fileToRemove ) request.addOperation( physicalRemoval ) return ReqClient().putRequest( request ) except Exception as e: gLogger.exception( "Exception while setting deletion request" ) return S_ERROR( "Cannot set deletion request: %s" % str( e ) ) else: gLogger.info( "Deleting external Sandbox" ) try: return StorageElement( SEName ).removeFile( SEPFN ) except Exception as e: gLogger.exception( "RM raised an exception while trying to delete a remote sandbox" ) return S_ERROR( "RM raised an exception while trying to delete a remote sandbox" )
def myRequest(): """Create a request and put it to the db""" request = Request() request.RequestName = 'myAwesomeRemovalRequest.xml' request.JobID = 0 request.SourceComponent = "myScript" remove = Operation() remove.Type = "RemoveFile" lfn = "/ilc/user/s/sailer/test.txt" rmFile = File() rmFile.LFN = lfn remove.addFile( rmFile ) request.addOperation( remove ) isValid = RequestValidator().validate( request ) if not isValid['OK']: raise RuntimeError( "Failover request is not valid: %s" % isValid['Message'] ) else: print("It is a GOGOGO") requestClient = ReqClient() result = requestClient.putRequest( request ) print(result)
def setUp(self): """ test case set up """ gLogger.setLevel('NOTICE') self.file = File() self.file.LFN = "/lhcb/user/c/cibak/testFile" self.file.Checksum = "123456" self.file.ChecksumType = "ADLER32" self.file2 = File() self.file2.LFN = "/lhcb/user/f/fstagni/testFile" self.file2.Checksum = "654321" self.file2.ChecksumType = "ADLER32" self.operation = Operation() self.operation.Type = "ReplicateAndRegister" self.operation.TargetSE = "CERN-USER" self.operation.addFile(self.file) self.operation.addFile(self.file2) self.request = Request() self.request.RequestName = "RequestManagerHandlerTests" self.request.OwnerDN = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=cibak/CN=605919/CN=Krzysztof Ciba" self.request.OwnerGroup = "dirac_user" self.request.JobID = 123 self.request.addOperation(self.operation) # # JSON representation of a whole request self.jsonStr = self.request.toJSON()['Value'] # # request client self.requestClient = ReqClient() self.stressRequests = 1000 self.bulkRequest = 1000
def setUp(self): """ test case setup """ self.request = Request({"RequestName": "test1", "JobID": 1}) self.operation1 = Operation({ "Type": "ReplicateAndRegister", "TargetSE": "CERN-USER" }) self.file = File({ "LFN": "/a/b/c", "ChecksumType": "ADLER32", "Checksum": "123456" }) self.request.addOperation(self.operation1) self.operation1.addFile(self.file) self.operation2 = Operation() self.operation2.Type = "RemoveFile" self.operation2.addFile(File({"LFN": "/c/d/e"})) self.request.addOperation(self.operation2) # ## set some defaults gConfig.setOptionValue('DIRAC/Setup', 'Test') gConfig.setOptionValue('/DIRAC/Setups/Test/RequestManagement', 'Test') gConfig.setOptionValue( '/Systems/RequestManagement/Test/Databases/ReqDB/Host', 'localhost') gConfig.setOptionValue( '/Systems/RequestManagement/Test/Databases/ReqDB/DBName', 'ReqDB') gConfig.setOptionValue( '/Systems/RequestManagement/Test/Databases/ReqDB/User', 'Dirac') self.i = 1000
def setUp(self): """ test case set up """ gLogger.setLevel('INFO') self.file = File() self.file.LFN = "/lhcb/user/c/cibak/testFile" self.file.Checksum = "123456" self.file.ChecksumType = "ADLER32" self.file2 = File() self.file2.LFN = "/lhcb/user/f/fstagni/testFile" self.file2.Checksum = "654321" self.file2.ChecksumType = "ADLER32" self.operation = Operation() self.operation.Type = "ReplicateAndRegister" self.operation.TargetSE = "CERN-USER" self.operation.addFile(self.file) self.operation.addFile(self.file2) proxyInfo = getProxyInfo()['Value'] self.request = Request() self.request.RequestName = "RequestManagerHandlerTests" self.request.OwnerDN = proxyInfo['identity'] self.request.OwnerGroup = proxyInfo['group'] self.request.JobID = 123 self.request.addOperation(self.operation) # # JSON representation of a whole request self.jsonStr = self.request.toJSON()['Value'] # # request client self.requestClient = ReqClient()
def test_StateMachine(): """state machine""" op = Operation() assert op.Status == "Queued", "1. wrong status %s" % op.Status op.addFile(File({"Status": "Waiting"})) assert op.Status == "Queued", "2. wrong status %s" % op.Status op.addFile(File({"Status": "Scheduled"})) assert op.Status == "Scheduled", "3. wrong status %s" % op.Status op.addFile(File({"Status": "Done"})) assert op.Status == "Scheduled", "4. wrong status %s" % op.Status op.addFile(File({"Status": "Failed"})) assert op.Status == "Scheduled", "5. wrong status %s" % op.Status op[3].Status = "Scheduled" assert op.Status == "Scheduled", "6. wrong status %s" % op.Status op[0].Status = "Scheduled" assert op.Status == "Scheduled", "7. wrong status %s" % op.Status op[0].Status = "Waiting" assert op.Status == "Scheduled", "8. wrong status %s" % op.Status for f in op: f.Status = "Done" assert op.Status == "Done", "9. wrong status %s" % op.Status for f in op: f.Status = "Failed" assert op.Status == "Failed", "9. wrong status %s" % op.Status
def myRequest(): """Create a request and put it to the db""" request = Request() request.RequestName = 'myAwesomeRemovalRequest.xml' request.JobID = 0 request.SourceComponent = "myScript" remove = Operation() remove.Type = "RemoveFile" lfn = "/ilc/user/s/sailer/test.txt" rmFile = File() rmFile.LFN = lfn remove.addFile(rmFile) request.addOperation(remove) isValid = RequestValidator().validate(request) if not isValid['OK']: raise RuntimeError("Failover request is not valid: %s" % isValid['Message']) else: print "It is a GOGOGO" requestClient = ReqClient() result = requestClient.putRequest(request) print result
def test04StateMachine(self): """ state machine """ op = Operation() self.assertEqual(op.Status, "Queued", "1. wrong status %s" % op.Status) op.addFile(File({"Status": "Waiting"})) self.assertEqual(op.Status, "Queued", "2. wrong status %s" % op.Status) op.addFile(File({"Status": "Scheduled"})) self.assertEqual(op.Status, "Queued", "3. wrong status %s" % op.Status) op.addFile(File({"Status": "Done"})) self.assertEqual(op.Status, "Queued", "4. wrong status %s" % op.Status) op.addFile(File({"Status": "Failed"})) self.assertEqual(op.Status, "Failed", "5. wrong status %s" % op.Status) op[3].Status = "Scheduled" self.assertEqual(op.Status, "Queued", "6. wrong status %s" % op.Status) op[0].Status = "Scheduled" self.assertEqual(op.Status, "Scheduled", "7. wrong status %s" % op.Status) op[0].Status = "Waiting" self.assertEqual(op.Status, "Queued", "8. wrong status %s" % op.Status) for f in op: f.Status = "Done" self.assertEqual(op.Status, "Done", "9. wrong status %s" % op.Status)
def __insertRegisterOperation( self, request, operation, toRegister ): """ add RegisterReplica operation :param Request request: request instance :param Operation transferOp: 'ReplicateAndRegister' operation for this FTSJob :param list toRegister: [ FTSDB.FTSFile, ... ] - files that failed to register """ log = self.log.getSubLogger( "req_%s/%s/registerFiles" % ( request.RequestID, request.RequestName ) ) byTarget = {} for ftsFile in toRegister: if ftsFile.TargetSE not in byTarget: byTarget.setdefault( ftsFile.TargetSE, [] ) byTarget[ftsFile.TargetSE].append( ftsFile ) log.info( "will create %s 'RegisterReplica' operations" % len( byTarget ) ) for target, ftsFileList in byTarget.iteritems(): log.info( "creating 'RegisterReplica' operation for targetSE %s with %s files..." % ( target, len( ftsFileList ) ) ) registerOperation = Operation() registerOperation.Type = "RegisterReplica" registerOperation.Status = "Waiting" registerOperation.TargetSE = target targetSE = StorageElement( target ) for ftsFile in ftsFileList: opFile = File() opFile.LFN = ftsFile.LFN pfn = returnSingleResult( targetSE.getURL( ftsFile.LFN, protocol = self.registrationProtocols ) ) if not pfn["OK"]: continue opFile.PFN = pfn["Value"] registerOperation.addFile( opFile ) request.insertBefore( registerOperation, operation ) return S_OK()
def __deleteSandboxFromExternalBackend(self, SEName, SEPFN): if self.getCSOption("DelayedExternalDeletion", True): gLogger.info("Setting deletion request") try: request = Request() request.RequestName = "RemoteSBDeletion:%s|%s:%s" % ( SEName, SEPFN, time.time()) physicalRemoval = Operation() physicalRemoval.Type = "PhysicalRemoval" physicalRemoval.TargetSE = SEName fileToRemove = File() fileToRemove.PFN = SEPFN physicalRemoval.addFile(fileToRemove) request.addOperation(physicalRemoval) return ReqClient().putRequest(request) except Exception as e: gLogger.exception("Exception while setting deletion request") return S_ERROR("Cannot set deletion request: %s" % str(e)) else: gLogger.info("Deleting external Sandbox") try: return StorageElement(SEName).removeFile(SEPFN) except Exception as e: gLogger.exception( "RM raised an exception while trying to delete a remote sandbox" ) return S_ERROR( "RM raised an exception while trying to delete a remote sandbox" )
def test06Dirty( self ): """ dirty records """ db = RequestDB() r = Request() r.RequestName = "dirty" op1 = Operation( { "Type": "ReplicateAndRegister", "TargetSE": "CERN-USER"} ) op1 += File( {"LFN": "/a/b/c/1", "Status": "Scheduled", "Checksum": "123456", "ChecksumType": "ADLER32" } ) op2 = Operation( { "Type": "ReplicateAndRegister", "TargetSE": "CERN-USER"} ) op2 += File( {"LFN": "/a/b/c/2", "Status": "Scheduled", "Checksum": "123456", "ChecksumType": "ADLER32" } ) op3 = Operation( { "Type": "ReplicateAndRegister", "TargetSE": "CERN-USER"} ) op3 += File( {"LFN": "/a/b/c/3", "Status": "Scheduled", "Checksum": "123456", "ChecksumType": "ADLER32" } ) r += op1 r += op2 r += op3 put = db.putRequest( r ) self.assertEqual( put["OK"], True, "1. putRequest failed: %s" % put.get( "Message", "" ) ) reqID = put['Value'] r = db.getRequest( reqID ) self.assertEqual( r["OK"], True, "1. getRequest failed: %s" % r.get( "Message", "" ) ) r = r["Value"] del r[0] self.assertEqual( len( r ), 2, "1. len wrong" ) put = db.putRequest( r ) self.assertEqual( put["OK"], True, "2. putRequest failed: %s" % put.get( "Message", "" ) ) reqID = put['Value'] r = db.getRequest( reqID ) self.assertEqual( r["OK"], True, "2. getRequest failed: %s" % r.get( "Message", "" ) ) r = r["Value"] self.assertEqual( len( r ), 2, "2. len wrong" ) op4 = Operation( { "Type": "ReplicateAndRegister", "TargetSE": "CERN-USER"} ) op4 += File( {"LFN": "/a/b/c/4", "Status": "Scheduled", "Checksum": "123456", "ChecksumType": "ADLER32" } ) r[0] = op4 put = db.putRequest( r ) self.assertEqual( put["OK"], True, "3. putRequest failed: %s" % put.get( "Message", "" ) ) reqID = put['Value'] r = db.getRequest( reqID ) self.assertEqual( r["OK"], True, "3. getRequest failed: %s" % r.get( "Message", "" ) ) r = r["Value"] self.assertEqual( len( r ), 2, "3. len wrong" ) delete = db.deleteRequest( reqID ) self.assertEqual( delete["OK"], True, delete['Message'] if 'Message' in delete else 'OK' )
def prepareTransformationTasks( self, transBody, taskDict, owner = '', ownerGroup = '', ownerDN = '' ): """ Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB """ if ( not owner ) or ( not ownerGroup ): res = getProxyInfo( False, False ) if not res['OK']: return res proxyInfo = res['Value'] owner = proxyInfo['username'] ownerGroup = proxyInfo['group'] if not ownerDN: res = getDNForUsername( owner ) if not res['OK']: return res ownerDN = res['Value'][0] requestOperation = 'ReplicateAndRegister' if transBody: try: _requestType, requestOperation = transBody.split( ';' ) except AttributeError: pass for taskID in sorted( taskDict ): paramDict = taskDict[taskID] if paramDict['InputData']: transID = paramDict['TransformationID'] oRequest = Request() transfer = Operation() transfer.Type = requestOperation transfer.TargetSE = paramDict['TargetSE'] if isinstance( paramDict['InputData'], list ): files = paramDict['InputData'] elif isinstance( paramDict['InputData'], basestring ): files = paramDict['InputData'].split( ';' ) for lfn in files: trFile = File() trFile.LFN = lfn transfer.addFile( trFile ) oRequest.addOperation( transfer ) oRequest.RequestName = _requestName( transID, taskID ) oRequest.OwnerDN = ownerDN oRequest.OwnerGroup = ownerGroup isValid = self.requestValidator.validate( oRequest ) if not isValid['OK']: return isValid taskDict[taskID]['TaskObject'] = oRequest return S_OK( taskDict )
def __setFileRemovalRequest(self, lfn): """ Sets a removal request for a file including all replicas. """ self.log.info('Setting file removal request for %s' % lfn) removeFile = Operation() removeFile.Type = 'RemoveFile' rmFile = File() rmFile.LFN = lfn removeFile.addFile(rmFile) self.request.addOperation(removeFile)
def _multiOperationsBody(self, transJson, taskDict, ownerDN, ownerGroup): """ deal with a Request that has multiple operations :param transJson: list of lists of string and dictionaries, e.g.: .. code :: python body = [ ( "ReplicateAndRegister", { "SourceSE":"FOO-SRM", "TargetSE":"BAR-SRM" }), ( "RemoveReplica", { "TargetSE":"FOO-SRM" } ), ] :param dict taskDict: dictionary of tasks, modified in this function :param str ownerDN: certificate DN used for the requests :param str onwerGroup: dirac group used for the requests :returns: None """ failedTasks = [] for taskID, task in taskDict.items(): transID = task['TransformationID'] if not task.get('InputData'): self._logError("Error creating request for task", "%s, No input data" % taskID, transID=transID) taskDict.pop(taskID) continue files = [] oRequest = Request() if isinstance(task['InputData'], list): files = task['InputData'] elif isinstance(task['InputData'], basestring): files = task['InputData'].split(';') # create the operations from the json structure for operationTuple in transJson: op = Operation() op.Type = operationTuple[0] for parameter, value in operationTuple[1].iteritems(): setattr(op, parameter, value) for lfn in files: opFile = File() opFile.LFN = lfn op.addFile(opFile) oRequest.addOperation(op) result = self._assignRequestToTask(oRequest, taskDict, transID, taskID, ownerDN, ownerGroup) if not result['OK']: failedTasks.append(taskID) # Remove failed tasks for taskID in failedTasks: taskDict.pop(taskID)
def test01ctors(self): """ File construction and (de)serialisation """ # # empty default ctor theFile = File() self.assertEqual(isinstance(theFile, File), True) # # fromDict try: theFile = File(self.fromDict) except AttributeError, error: print "AttributeError: %s" % str(error)
def addRemovalRequests(self, lfnList): """Create removalRequests for lfns in lfnList and add it to the common request""" request = self._getRequestContainer() remove = Operation() remove.Type = "RemoveFile" for lfn in lfnList: rmFile = File() rmFile.LFN = lfn remove.addFile( rmFile ) request.addOperation( remove ) self.workflow_commons['Request'] = request
def _setFileRemovalRequest(self, lfn, se="", pfn=""): """Sets a removal request for a file including all replicas.""" remove = Operation() remove.Type = "RemoveFile" if se: remove.TargetSE = se rmFile = File() rmFile.LFN = lfn if pfn: rmFile.PFN = pfn remove.addFile(rmFile) self.request.addOperation(remove) return S_OK()
def getRegisterOperation(self, opFile, targetSE, type='RegisterFile', catalog=None): """ add RegisterReplica operation for file :param File opFile: operation file :param str targetSE: target SE """ # # add RegisterReplica operation registerOperation = Operation() registerOperation.Type = type registerOperation.TargetSE = targetSE if catalog: registerOperation.Catalog = catalog registerFile = File() registerFile.LFN = opFile.LFN registerFile.PFN = StorageElement(targetSE).getPfnForLfn( opFile.LFN).get('Value', {}).get('Successful', {}).get(opFile.LFN) registerFile.GUID = opFile.GUID registerFile.Checksum = opFile.Checksum registerFile.ChecksumType = opFile.ChecksumType registerFile.Size = opFile.Size registerOperation.addFile(registerFile) return registerOperation
def getRegisterOperation(self, opFile, targetSE, type="RegisterFile", catalog=None): """add RegisterReplica operation for file :param ~DIRAC.RequestManagementSystem.Client.File.File opFile: operation file :param str targetSE: target SE """ # # add RegisterReplica operation registerOperation = Operation() registerOperation.Type = type registerOperation.TargetSE = targetSE if catalog: registerOperation.Catalog = catalog registerFile = File() registerFile.LFN = opFile.LFN registerFile.PFN = ( StorageElement(targetSE) .getURL(opFile.LFN, protocol=self.registrationProtocols) .get("Value", {}) .get("Successful", {}) .get(opFile.LFN) ) registerFile.GUID = opFile.GUID registerFile.Checksum = opFile.Checksum registerFile.ChecksumType = opFile.ChecksumType registerFile.Size = opFile.Size registerOperation.addFile(registerFile) return registerOperation
def __setFileRemovalRequest( self, lfn, se = '', pfn = '' ): """ Sets a removal request for a file including all replicas. """ remove = Operation() remove.Type = "RemoveFile" if se: remove.TargetSE = se rmFile = File() rmFile.LFN = lfn if pfn: rmFile.PFN = pfn remove.addFile( rmFile ) self.request.addOperation( remove ) return S_OK()
def test05List(self): """ getitem, setitem, delitem and dirty """ op = Operation() files = [] for i in range(5): f = File() files.append(f) op += f for i in range(len(op)): self.assertEqual(op[i], files[i], "__getitem__ failed") for i in range(len(op)): op[i] = File({"LFN": "/%s" % i}) self.assertEqual(op[i].LFN, "/%s" % i, "__setitem__ failed") del op[0] self.assertEqual(len(op), 4, "__delitem__ failed") # # opID set op.OperationID = 1 del op[0] self.assertEqual(op.cleanUpSQL(), None, "cleanUp failed after __delitem__") op[0].FileID = 1 del op[0] self.assertEqual( op.cleanUpSQL(), "DELETE FROM `File` WHERE `OperationID` = 1 AND `FileID` IN (1);\n", "cleanUp failed after __delitem__") op[0].FileID = 2 op[0] = File({"FileID": 2}) self.assertEqual( op.cleanUpSQL(), "DELETE FROM `File` WHERE `OperationID` = 1 AND `FileID` IN (1,2);\n", "cleanUp failed after __setitem__") json = op.toJSON() self.assertEqual("__dirty" in json["Value"], True, "missing __dirty") op2 = Operation(json["Value"]) self.assertEqual( op2.cleanUpSQL(), "DELETE FROM `File` WHERE `OperationID` = 1 AND `FileID` IN (1,2);\n", "cleanUp failed after JSON")
def _singleOperationsBody(self, transBody, taskDict, ownerDN, ownerGroup): """deal with a Request that has just one operation, as it was sofar :param transBody: string, can be an empty string :param dict taskDict: dictionary of tasks, modified in this function :param str ownerDN: certificate DN used for the requests :param str onwerGroup: dirac group used for the requests :returns: None """ requestOperation = "ReplicateAndRegister" if transBody: try: _requestType, requestOperation = transBody.split(";") except AttributeError: pass failedTasks = [] # Do not remove sorted, we might pop elements in the loop for taskID, task in taskDict.items(): transID = task["TransformationID"] oRequest = Request() transfer = Operation() transfer.Type = requestOperation transfer.TargetSE = task["TargetSE"] # If there are input files if task.get("InputData"): if isinstance(task["InputData"], list): files = task["InputData"] elif isinstance(task["InputData"], six.string_types): files = task["InputData"].split(";") for lfn in files: trFile = File() trFile.LFN = lfn transfer.addFile(trFile) oRequest.addOperation(transfer) result = self._assignRequestToTask(oRequest, taskDict, transID, taskID, ownerDN, ownerGroup) if not result["OK"]: failedTasks.append(taskID) # Remove failed tasks for taskID in failedTasks: taskDict.pop(taskID)
def test04Stress(self): """ stress test """ db = RequestDB() for i in range(self.stressRequests): request = Request({"RequestName": "test-%d" % i}) op = Operation({"Type": "RemoveReplica", "TargetSE": "CERN-USER"}) op += File({"LFN": "/lhcb/user/c/cibak/foo"}) request += op put = db.putRequest(request) self.assertEqual(put["OK"], True, "put failed") startTime = time.time() for i in range(self.stressRequests): get = db.getRequest("test-%s" % i, True) if "Message" in get: print get["Message"] self.assertEqual(get["OK"], True, "get failed") endTime = time.time() print "getRequest duration %s " % (endTime - startTime) for i in range(self.stressRequests): delete = db.deleteRequest("test-%s" % i) self.assertEqual(delete["OK"], True, "delete failed")
def test01Stress(self): """ stress test """ db = RequestDB() reqIDs = [] for i in xrange(self.stressRequests): request = Request({"RequestName": "test-%d" % i}) op = Operation({"Type": "RemoveReplica", "TargetSE": "CERN-USER"}) op += File({"LFN": "/lhcb/user/c/cibak/foo"}) request += op put = db.putRequest(request) self.assertEqual(put["OK"], True, put['Message'] if 'Message' in put else 'OK') reqIDs.append(put['Value']) startTime = time.time() for reqID in reqIDs: get = db.getRequest(reqID) if "Message" in get: print(get["Message"]) self.assertEqual(get["OK"], True, get['Message'] if 'Message' in get else 'OK') endTime = time.time() print("getRequest duration %s " % (endTime - startTime)) for reqID in reqIDs: delete = db.deleteRequest(reqID) self.assertEqual( delete["OK"], True, delete['Message'] if 'Message' in delete else 'OK')
def test_ctor(): """test constructors and (de)serialisation""" assert isinstance(Operation(), Operation), "empty ctor failed" # # using fromDict fromDict = { "Type": "replicateAndRegister", "TargetSE": "CERN-USER,PIC-USER", "SourceSE": None, } operation = Operation(fromDict) assert isinstance(operation, Operation), "fromDict ctor failed" for key, value in fromDict.items(): assert getattr(operation, key) == value, "wrong attr value %s (%s) %s" % ( key, getattr(operation, key), value) # # same with file operation = Operation(fromDict) operation.addFile( File({ "LFN": "/lhcb/user/c/cibak/testFile", "Checksum": "1234567", "ChecksumType": "ADLER32", "Size": 1024, "Status": "Waiting", })) for key, value in fromDict.items(): assert getattr(operation, key) == value, "wrong attr value %s (%s) %s" % ( key, getattr(operation, key), value) toJSON = operation.toJSON() assert toJSON["OK"], "JSON serialization failed"
def test_scheduled(reqDB): """scheduled request r/w""" req = Request({"RequestName": "FTSTest"}) op = Operation({"Type": "ReplicateAndRegister", "TargetSE": "CERN-USER"}) op += File({ "LFN": "/a/b/c", "Status": "Scheduled", "Checksum": "123456", "ChecksumType": "ADLER32" }) req += op put = reqDB.putRequest(req) assert put["OK"], put reqID = put["Value"] peek = reqDB.peekRequest(reqID) assert peek["OK"], peek peek = peek["Value"] for op in peek: opId = op.OperationID getFTS = reqDB.getScheduledRequest(opId) assert getFTS["OK"], getFTS assert getFTS[ "Value"].RequestName == "FTSTest", "Wrong request name %s" % getFTS[ "Value"].RequestName delete = reqDB.deleteRequest(reqID) assert delete["OK"], delete
def test_stress(reqDB): """stress test""" reqIDs = [] for i in range(STRESS_REQUESTS): request = Request({"RequestName": "test-%d" % i}) op = Operation({"Type": "RemoveReplica", "TargetSE": "CERN-USER"}) op += File({"LFN": "/lhcb/user/c/cibak/foo"}) request += op put = reqDB.putRequest(request) assert put["OK"], put reqIDs.append(put["Value"]) startTime = time.time() for reqID in reqIDs: get = reqDB.getRequest(reqID) assert get["OK"], get endTime = time.time() print("getRequest duration %s " % (endTime - startTime)) for reqID in reqIDs: delete = reqDB.deleteRequest(reqID) assert delete["OK"], delete
def test05Scheduled(self): """ scheduled request r/w """ db = RequestDB() req = Request({"RequestName": "FTSTest"}) op = Operation({ "Type": "ReplicateAndRegister", "TargetSE": "CERN-USER" }) op += File({ "LFN": "/a/b/c", "Status": "Scheduled", "Checksum": "123456", "ChecksumType": "ADLER32" }) req += op put = db.putRequest(req) self.assertEqual(put["OK"], True, "putRequest failed") peek = db.peekRequest(req.RequestName) self.assertEqual(peek["OK"], True, "peek failed ") peek = peek["Value"] for op in peek: opId = op.OperationID getFTS = db.getScheduledRequest(opId) self.assertEqual(getFTS["OK"], True, "getScheduled failed") self.assertEqual(getFTS["Value"].RequestName, "FTSTest", "wrong request selected")
def __deleteSandboxFromExternalBackend(self, SEName, SEPFN): if self.getCSOption("DelayedExternalDeletion", True): gLogger.info("Setting deletion request") try: # We need the hostDN used in order to pass these credentials to the # SandboxStoreDB.. hostCertLocation, _ = Locations.getHostCertificateAndKeyLocation( ) hostCert = X509Certificate.X509Certificate() hostCert.loadFromFile(hostCertLocation) hostDN = hostCert.getSubjectDN().get("Value") # use the host authentication to fetch the data result = self.sandboxDB.getSandboxOwner( SEName, SEPFN, hostDN, "hosts") if not result["OK"]: return result _owner, ownerDN, ownerGroup = result["Value"] request = Request() request.RequestName = "RemoteSBDeletion:%s|%s:%s" % ( SEName, SEPFN, time.time()) request.OwnerDN = ownerDN request.OwnerGroup = ownerGroup physicalRemoval = Operation() physicalRemoval.Type = "PhysicalRemoval" physicalRemoval.TargetSE = SEName fileToRemove = File() fileToRemove.PFN = SEPFN physicalRemoval.addFile(fileToRemove) request.addOperation(physicalRemoval) return ReqClient().putRequest(request) except Exception as e: gLogger.exception("Exception while setting deletion request") return S_ERROR(f"Cannot set deletion request: {e}") else: gLogger.info("Deleting external Sandbox") try: return StorageElement(SEName).removeFile(SEPFN) except Exception: gLogger.exception( "RM raised an exception while trying to delete a remote sandbox" ) return S_ERROR( "RM raised an exception while trying to delete a remote sandbox" )
def _singleOperationsBody(self, transBody, taskDict, ownerDN, ownerGroup): """ deal with a Request that has just one operation, as it was sofar :param transBody: string, can be an empty string :param dict taskDict: dictionary of tasks, modified in this function :param str ownerDN: certificate DN used for the requests :param str onwerGroup: dirac group used for the requests :returns: None """ requestOperation = 'ReplicateAndRegister' if transBody: try: _requestType, requestOperation = transBody.split(';') except AttributeError: pass failedTasks = [] # Do not remove sorted, we might pop elements in the loop for taskID, task in taskDict.iteritems(): transID = task['TransformationID'] oRequest = Request() transfer = Operation() transfer.Type = requestOperation transfer.TargetSE = task['TargetSE'] # If there are input files if task.get('InputData'): if isinstance(task['InputData'], list): files = task['InputData'] elif isinstance(task['InputData'], basestring): files = task['InputData'].split(';') for lfn in files: trFile = File() trFile.LFN = lfn transfer.addFile(trFile) oRequest.addOperation(transfer) result = self._assignRequestToTask(oRequest, taskDict, transID, taskID, ownerDN, ownerGroup) if not result['OK']: failedTasks.append(taskID) # Remove failed tasks for taskID in failedTasks: taskDict.pop(taskID)
def _getFileProperties(self, fileID, columnNames=None): """ select :columnNames: from File table """ columnNames = columnNames if columnNames else File.tableDesc( )["Fields"].keys() columnNames = ",".join( ['`%s`' % str(columnName) for columnName in columnNames]) return "SELECT %s FROM `File` WHERE `FileID` = %s;" % (columnNames, int(fileID))
def setBKRegistrationRequest(self, lfn, error='', metaData={'Checksum': 'justSomething', 'ChecksumType': 'ADLER32', 'GUID': 'aGUID'}): """ Set a BK registration request for changing the replica flag. Uses the global request object (self.request). """ if error: self.log.info('BK registration for %s failed with message: "%s" setting failover request' % (lfn, error)) else: self.log.info('Setting BK registration request for %s' % (lfn)) regFile = Operation() regFile.Type = 'RegisterFile' regFile.Catalog = 'BookkeepingDB' bkFile = File() bkFile.LFN = lfn # this should NOT be needed... but RMS complains! bkFile.PFN = lfn bkFile.GUID = metaData['GUID'] bkFile.Checksum = metaData['Checksum'] bkFile.ChecksumType = metaData['ChecksumType'] regFile.addFile(bkFile) res = self.request.addOperation(regFile) if not res['OK']: raise RuntimeError(res['Message'])
def __setRegistrationRequest( self, lfn, targetSE, fileDict, catalog ): """ Sets a registration request :param str lfn: LFN :param list se: list of SE :param list catalog: list of catalogs to use :param dict fileDict: file metadata """ self.log.info( 'Setting registration request for %s at %s.' % ( lfn, targetSE ) ) for cat in catalog: register = Operation() register.Type = "RegisterFile" register.Catalog = cat register.TargetSE = targetSE regFile = File() regFile.LFN = lfn regFile.Checksum = fileDict.get( "Checksum", "" ) regFile.ChecksumType = fileDict.get( "ChecksumType", "" ) regFile.Size = fileDict.get( "Size", 0 ) regFile.GUID = fileDict.get( "GUID", "" ) se = StorageElement( targetSE ) pfn = se.getPfnForLfn( lfn ) if not pfn["OK"]: self.log.error( "unable to get PFN for LFN: %s" % pfn["Message"] ) return pfn regFile.PFN = pfn["Value"] register.addFile( regFile ) self.request.addOperation( register ) return S_OK()
def _setFileReplicationRequest(self, lfn, targetSE, fileMetaDict, sourceSE=""): """Sets a registration request.""" self.log.info("Setting ReplicateAndRegister request", "for %s to %s" % (lfn, targetSE)) transfer = Operation() transfer.Type = "ReplicateAndRegister" transfer.TargetSE = targetSE if sourceSE: transfer.SourceSE = sourceSE trFile = File() trFile.LFN = lfn cksm = fileMetaDict.get("Checksum", None) cksmType = fileMetaDict.get("ChecksumType", self.defaultChecksumType) if cksm and cksmType: trFile.Checksum = cksm trFile.ChecksumType = cksmType size = fileMetaDict.get("Size", 0) if size: trFile.Size = size guid = fileMetaDict.get("GUID", "") if guid: trFile.GUID = guid transfer.addFile(trFile) self.request.addOperation(transfer) return S_OK()
def test01TableDesc(self): """ table description """ tableDict = RequestDB.getTableMeta() self.assertEqual("Request" in tableDict, True) self.assertEqual("Operation" in tableDict, True) self.assertEqual("File" in tableDict, True) self.assertEqual(tableDict["Request"], Request.tableDesc()) self.assertEqual(tableDict["Operation"], Operation.tableDesc()) self.assertEqual(tableDict["File"], File.tableDesc())
def __setRemovalRequest( self, lfn, ownerDN, ownerGroup ): """ Set removal request with the given credentials """ oRequest = Request() oRequest.OwnerDN = ownerDN oRequest.OwnerGroup = ownerGroup oRequest.RequestName = os.path.basename( lfn ).strip() + '_removal_request.xml' oRequest.SourceComponent = 'JobCleaningAgent' removeFile = Operation() removeFile.Type = 'RemoveFile' removedFile = File() removedFile.LFN = lfn removeFile.addFile( removedFile ) oRequest.addOperation( removeFile ) return ReqClient().putRequest( oRequest )
def prepareTransformationTasks( self, transBody, taskDict, owner = '', ownerGroup = '' ): """ Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB """ requestOperation = 'ReplicateAndRegister' if transBody: try: _requestType, requestOperation = transBody.split( ';' ) except AttributeError: pass for taskID in sorted( taskDict ): paramDict = taskDict[taskID] if paramDict['InputData']: transID = paramDict['TransformationID'] oRequest = Request() transfer = Operation() transfer.Type = requestOperation transfer.TargetSE = paramDict['TargetSE'] if type( paramDict['InputData'] ) == type( [] ): files = paramDict['InputData'] elif type( paramDict['InputData'] ) == type( '' ): files = paramDict['InputData'].split( ';' ) for lfn in files: trFile = File() trFile.LFN = lfn transfer.addFile( trFile ) oRequest.addOperation( transfer ) oRequest.RequestName = str( transID ).zfill( 8 ) + '_' + str( taskID ).zfill( 8 ) oRequest.OwnerDN = owner oRequest.OwnerGroup = ownerGroup isValid = gRequestValidator.validate( oRequest ) if not isValid['OK']: return isValid taskDict[taskID]['TaskObject'] = oRequest return S_OK( taskDict )
def test__addMetadataToFiles( self ): resMeta = {'OK': True, 'Value': {'Failed': {}, 'Successful': {'/lhcb/1.dst': {'ChecksumType': 'AD', 'Checksum': '123456', 'CreationDate': datetime.datetime( 2013, 12, 11, 20, 20, 21 ), 'GUID': '92F9CE97-7A62-E311-8401-0025907FD430', 'Mode': 436, 'ModificationDate': datetime.datetime( 2013, 12, 11, 20, 20, 21 ), 'NumberOfLinks': 1, 'Size': 5846023777, 'Status': '-'}, '/lhcb/2.dst': {'ChecksumType': 'AD', 'Checksum': '987654', 'CreationDate': datetime.datetime( 2013, 12, 12, 6, 26, 52 ), 'GUID': 'DAE4933A-C162-E311-8A6B-003048FEAF04', 'Mode': 436, 'ModificationDate': datetime.datetime( 2013, 12, 12, 6, 26, 52 ), 'NumberOfLinks': 1, 'Size': 5893396937, 'Status': '-'}}}} self.rr.fc.getFileMetadata.return_value = resMeta file1 = File() file1.LFN = '/lhcb/1.dst' file2 = File() file2.LFN = '/lhcb/2.dst' toSchedule = {'/lhcb/1.dst': [file1, ['SE1'], ['SE2', 'SE3']], '/lhcb/2.dst': [file2, ['SE4'], ['SE5', 'SE6']]} res = self.rr._addMetadataToFiles( toSchedule ) self.assertTrue(res['OK']) for lfn in toSchedule: self.assertEqual( res['Value'][lfn].LFN, lfn ) for attr in ('GUID', 'Size', 'Checksum'): self.assertEqual( getattr(res['Value'][lfn],attr), resMeta['Value']['Successful'][lfn][attr] ) # AD should be transformed into Adler32 self.assertEqual( res['Value'][lfn].ChecksumType, "ADLER32" )
def __setReplicaRemovalRequest( self, lfn, se ): """ Sets a removal request for a replica. :param str lfn: LFN :param se: """ if type( se ) == str: se = ",".join( [ se.strip() for se in se.split( "," ) if se.strip() ] ) removeReplica = Operation() removeReplica.Type = "RemoveReplica" removeReplica.TargetSE = se replicaToRemove = File() replicaToRemove.LFN = lfn removeReplica.addFile( replicaToRemove ) self.request.addOperation( removeReplica ) return S_OK()
def test__addMetadataToFiles( self ): resMeta = {'OK': True, 'Value': {'Failed': {}, 'Successful': {'/lhcb/1.dst': {'ChecksumType': 'AD', 'Checksum': '123456', 'CreationDate': datetime.datetime( 2013, 12, 11, 20, 20, 21 ), 'GUID': '92F9CE97-7A62-E311-8401-0025907FD430', 'Mode': 436, 'ModificationDate': datetime.datetime( 2013, 12, 11, 20, 20, 21 ), 'NumberOfLinks': 1, 'Size': 5846023777, 'Status': '-'}, '/lhcb/2.dst': {'ChecksumType': 'AD', 'Checksum': '987654', 'CreationDate': datetime.datetime( 2013, 12, 12, 6, 26, 52 ), 'GUID': 'DAE4933A-C162-E311-8A6B-003048FEAF04', 'Mode': 436, 'ModificationDate': datetime.datetime( 2013, 12, 12, 6, 26, 52 ), 'NumberOfLinks': 1, 'Size': 5893396937, 'Status': '-'}}}} self.rr.fc.getFileMetadata.return_value = resMeta file1 = File() file1.LFN = '/lhcb/1.dst' file2 = File() file2.LFN = '/lhcb/2.dst' toSchedule = {'/lhcb/1.dst': [file1, ['SE1'], ['SE2', 'SE3']], '/lhcb/2.dst': [file2, ['SE4'], ['SE5', 'SE6']]} res = self.rr._addMetadataToFiles( toSchedule ) self.assert_( res['OK'] ) self.assertEqual( json.loads( res['Value'][0][0] )['LFN'], resMeta['Value']['Successful'].keys()[0] ) self.assertEqual( json.loads( res['Value'][0][0] )['Size'], resMeta['Value']['Successful'].values()[0]['Size'] ) self.assertEqual( json.loads( res['Value'][1][0] )['LFN'], resMeta['Value']['Successful'].keys()[1] ) self.assertEqual( json.loads( res['Value'][1][0] )['Size'], resMeta['Value']['Successful'].values()[1]['Size'] )
def __setFileReplicationRequest( self, lfn, targetSE, fileMetaDict, sourceSE = '' ): """ Sets a registration request. """ self.log.info( 'Setting replication request for %s to %s' % ( lfn, targetSE ) ) transfer = Operation() transfer.Type = "ReplicateAndRegister" transfer.TargetSE = targetSE if sourceSE: transfer.SourceSE = sourceSE trFile = File() trFile.LFN = lfn cksm = fileMetaDict.get( "Checksum", None ) cksmType = fileMetaDict.get( "ChecksumType", None ) if cksm and cksmType: trFile.Checksum = cksm trFile.ChecksumType = cksmType size = fileMetaDict.get( "Size", 0 ) if size: trFile.Size = size guid = fileMetaDict.get( "GUID", "" ) if guid: trFile.GUID = guid transfer.addFile( trFile ) self.request.addOperation( transfer ) return S_OK()
def test_db( self ): """ table description """ tableDict = RequestDB.getTableMeta() self.assertEqual( "Request" in tableDict, True ) self.assertEqual( "Operation" in tableDict, True ) self.assertEqual( "File" in tableDict, True ) self.assertEqual( tableDict["Request"], Request.tableDesc() ) self.assertEqual( tableDict["Operation"], Operation.tableDesc() ) self.assertEqual( tableDict["File"], File.tableDesc() ) # # empty DB at that stage ret = RequestDB().getDBSummary() self.assertEqual( ret, { 'OK': True, 'Value': { 'Operation': {}, 'Request': {}, 'File': {} } } )
def getRegisterOperation( self, opFile, targetSE ): """ add RegisterReplica operation for file :param File opFile: operation file :param str targetSE: target SE """ # # add RegisterReplica operation registerOperation = Operation() registerOperation.Type = "RegisterFile" registerOperation.TargetSE = targetSE registerFile = File() registerFile.LFN = opFile.LFN registerFile.PFN = opFile.PFN registerFile.GUID = opFile.GUID registerFile.Checksum = opFile.Checksum registerFile.ChecksumType = opFile.ChecksumType registerFile.Size = opFile.Size registerOperation.addFile( registerFile ) return registerOperation
def addRegisterReplica( self, opFile, targetSE ): """ add RegisterReplica operation for file :param File opFile: operation file :param str targetSE: target SE """ # # add RegisterReplica operation registerOperation = Operation() registerOperation.Type = "RegisterFile" registerOperation.TargetSE = targetSE registerFile = File() registerFile.LFN = opFile.LFN registerFile.PFN = opFile.PFN registerFile.GUID = opFile.GUID registerFile.Checksum = opFile.Checksum registerFile.ChecksumType = opFile.ChecksumType registerFile.Size = opFile.Size registerOperation.addFile( registerFile ) self.request.insertAfter( registerOperation, self.operation ) return S_OK()
def _setRegistrationRequest( self, lfn, targetSE, fileDict, catalog ): """ Sets a registration request :param str lfn: LFN :param list se: list of SE (or just string) :param list catalog: list (or string) of catalogs to use :param dict fileDict: file metadata """ self.log.info( 'Setting registration request for %s at %s.' % ( lfn, targetSE ) ) if not isinstance( catalog, list ): catalog = [catalog] for cat in catalog: register = Operation() register.Type = "RegisterFile" register.Catalog = cat register.TargetSE = targetSE regFile = File() regFile.LFN = lfn regFile.Checksum = fileDict.get( "Checksum", "" ) regFile.ChecksumType = fileDict.get( "ChecksumType", self.defaultChecksumType ) regFile.Size = fileDict.get( "Size", 0 ) regFile.GUID = fileDict.get( "GUID", "" ) se = StorageElement( targetSE ) pfn = se.getURL( lfn, self.registrationProtocols ) if not pfn["OK"] or lfn not in pfn["Value"]['Successful']: self.log.error( "Unable to get PFN for LFN", "%s" % pfn.get( 'Message', pfn.get( 'Value', {} ).get( 'Failed', {} ).get( lfn ) ) ) return pfn regFile.PFN = pfn["Value"]['Successful'][lfn] register.addFile( regFile ) self.request.addOperation( register ) return S_OK()
def getRegisterOperation( self, opFile, targetSE, type = 'RegisterFile', catalog = None ): """ add RegisterReplica operation for file :param File opFile: operation file :param str targetSE: target SE """ # # add RegisterReplica operation registerOperation = Operation() registerOperation.Type = type registerOperation.TargetSE = targetSE if catalog: registerOperation.Catalog = catalog registerFile = File() registerFile.LFN = opFile.LFN registerFile.PFN = StorageElement( targetSE ).getURL( opFile.LFN, protocol = self.registrationProtocols ).get( 'Value', {} ).get( 'Successful', {} ).get( opFile.LFN ) registerFile.GUID = opFile.GUID registerFile.Checksum = opFile.Checksum registerFile.ChecksumType = opFile.ChecksumType registerFile.Size = opFile.Size registerOperation.addFile( registerFile ) return registerOperation
def createRequest( reqType ): r = Request() # Simple failover op1 = Operation() f = File() f.LFN = '/This/is/an/LFN' op1.addFile( f ) op1.Type = 'ReplicateAndRegister' op1.SourceSE = 'CERN-FAILOVER' op1.TargetSE = 'CERN-BUFFER' r.addOperation( op1 ) op2 = Operation() op2.addFile( f ) op2.Type = 'RemoveReplica' op2.TargetSE = 'CERN-FAILOVER' r.addOperation( op2 ) if reqType == 0: return r # two files for Failover f1 = File() f1.LFN = '/This/is/a/second/LFN' op3 = Operation() op3.addFile( f1 ) op3.Type = 'ReplicateAndRegister' op3.SourceSE = 'CERN-FAILOVER' op3.TargetSE = 'CERN-BUFFER' r.addOperation( op3 ) op3 = Operation() op3.addFile( f1 ) op3.Type = 'RemoveReplica' op3.TargetSE = 'CERN-FAILOVER' r.addOperation( op3 ) if reqType == 1: return r op = Operation() op.Type = 'ForwardDiset' if reqType == 2: r.addOperation( op ) return r r.insertBefore( op, r[0] ) if reqType == 3: return r op4 = Operation() op4.Type = 'ForwardDiset' r.addOperation( op4 ) if reqType == 4: return r # 2 different FAILOVER SEs: removal not optimized r[1].SourceSE = 'RAL-FAILOVER' r[2].SourceSE = 'RAL-FAILOVER' if reqType == 5: return r # 2 different destinations, same FAILOVER: replication not optimized r[3].SourceSE = 'RAL-FAILOVER' r[4].SourceSE = 'RAL-FAILOVER' r[3].TargetSE = 'RAL-BUFFER' if reqType == 6: return r print 'This should not happen, reqType =', reqType
replicateAndRegister.Type = 'ReplicateAndRegister' replicateAndRegister.TargetSE = targetSE res = fc.getFileMetadata( lfnList ) if not res['OK']: print "Can't get file metadata: %s" % res['Message'] DIRAC.exit( 1 ) if res['Value']['Failed']: print "Could not get the file metadata of the following, so skipping them:" for fFile in res['Value']['Failed']: print fFile lfnMetadata = res['Value']['Successful'] for lfn in lfnMetadata: rarFile = File() rarFile.LFN = lfn rarFile.Size = lfnMetadata[lfn]['Size'] rarFile.Checksum = lfnMetadata[lfn]['Checksum'] rarFile.GUID = lfnMetadata[lfn]['GUID'] rarFile.ChecksumType = 'ADLER32' replicateAndRegister.addFile( rarFile ) oRequest.addOperation( replicateAndRegister ) isValid = RequestValidator().validate( oRequest ) if not isValid['OK']: print "Request is not valid: ", isValid['Message'] DIRAC.exit( 1 ) result = reqClient.putRequest( oRequest ) if result['OK']:
def _getFileProperties( self, fileID, columnNames = None ): """ select :columnNames: from File table """ columnNames = columnNames if columnNames else File.tableDesc()["Fields"].keys() columnNames = ",".join( [ '`%s`' % str( columnName ) for columnName in columnNames ] ) return "SELECT %s FROM `File` WHERE `FileID` = %s;" % ( columnNames, int( fileID ) )
DIRAC.exit( -1 ) if not os.path.isfile( PFN ): gLogger.error( "%s is not a file" % PFN ) DIRAC.exit( -1 ) PFN = os.path.abspath( PFN ) size = os.path.getsize( PFN ) adler32 = fileAdler( PFN ) request = Request() request.RequestName = requestName putAndRegister = Operation() putAndRegister.Type = "PutAndRegister" putAndRegister.TargetSE = targetSE opFile = File() opFile.LFN = LFN opFile.PFN = PFN opFile.Size = size opFile.Checksum = adler32 opFile.ChecksumType = "ADLER32" putAndRegister.addFile( opFile ) request.addOperation( putAndRegister ) reqClient = ReqClient() putRequest = reqClient.putRequest( request ) if not putRequest["OK"]: gLogger.error( "unable to put request '%s': %s" % ( requestName, putRequest["Message"] ) ) DIRAC.exit( -1 ) gLogger.always( "Request '%s' has been put to ReqDB for execution." % requestName ) gLogger.always( "You can monitor its status using command: 'dirac-rms-show-request %s'" % requestName )
error = -1 continue count += 1 request = Request() request.RequestName = requestName if not multiRequests else '%s_%d' % ( requestName, count ) replicateAndRegister = Operation() replicateAndRegister.Type = "ReplicateAndRegister" replicateAndRegister.TargetSE = ",".join( targetSEs ) if catalog is not None: replicateAndRegister.Catalog = catalog for lfn in lfnChunk: metaDict = metaDatas["Successful"][lfn] opFile = File() opFile.LFN = lfn opFile.Size = metaDict["Size"] if "Checksum" in metaDict: # # should check checksum type, now assuming Adler32 (metaDict["ChecksumType"] = 'AD' opFile.Checksum = metaDict["Checksum"] opFile.ChecksumType = "ADLER32" replicateAndRegister.addFile( opFile ) request.addOperation( replicateAndRegister ) putRequest = reqClient.putRequest( request ) if not putRequest["OK"]: gLogger.error( "unable to put request '%s': %s" % ( request.RequestName, putRequest["Message"] ) ) error = -1
def test02props( self ): """ test props and attributes """ theFile = File() # valid props theFile.FileID = 1 self.assertEqual( theFile.FileID, 1 ) theFile.Status = "Done" self.assertEqual( theFile.Status, "Done" ) theFile.LFN = "/some/path/somewhere" self.assertEqual( theFile.LFN, "/some/path/somewhere" ) theFile.PFN = "/some/path/somewhere" self.assertEqual( theFile.PFN, "/some/path/somewhere" ) theFile.Attempt = 1 self.assertEqual( theFile.Attempt, 1 ) theFile.Size = 1 self.assertEqual( theFile.Size, 1 ) theFile.GUID = "2bbabe80-e2f1-11e1-9b23-0800200c9a66" self.assertEqual( theFile.GUID, "2bbabe80-e2f1-11e1-9b23-0800200c9a66" ) theFile.ChecksumType = "adler32" self.assertEqual( theFile.ChecksumType, "ADLER32" ) theFile.Checksum = "123456" self.assertEqual( theFile.Checksum, "123456" ) # # theFile.Checksum = None theFile.ChecksumType = None self.assertEqual( theFile.Checksum, "" ) self.assertEqual( theFile.ChecksumType, "" ) # # invalid props # FileID try: theFile.FileID = "foo" except Exception, error: self.assertEqual( isinstance( error, ValueError ), True )
def test05FTS( self ): """ FTS state machine """ req = Request() req.RequestName = "FTSTest" ftsTransfer = Operation() ftsTransfer.Type = "ReplicateAndRegister" ftsTransfer.TargetSE = "CERN-USER" ftsFile = File() ftsFile.LFN = "/a/b/c" ftsFile.Checksum = "123456" ftsFile.ChecksumType = "Adler32" ftsTransfer.addFile( ftsFile ) req.addOperation( ftsTransfer ) self.assertEqual( req.Status, "Waiting", "1. wrong request status: %s" % req.Status ) self.assertEqual( ftsTransfer.Status, "Waiting", "1. wrong ftsStatus status: %s" % ftsTransfer.Status ) # # scheduled ftsFile.Status = "Scheduled" self.assertEqual( ftsTransfer.Status, "Scheduled", "2. wrong status for ftsTransfer: %s" % ftsTransfer.Status ) self.assertEqual( req.Status, "Scheduled", "2. wrong status for request: %s" % req.Status ) # # add new operation before FTS insertBefore = Operation() insertBefore.Type = "RegisterReplica" insertBefore.TargetSE = "CERN-USER" insertFile = File() insertFile.LFN = "/a/b/c" insertFile.PFN = "http://foo/bar" insertBefore.addFile( insertFile ) req.insertBefore( insertBefore, ftsTransfer ) self.assertEqual( insertBefore.Status, "Waiting", "3. wrong status for insertBefore: %s" % insertBefore.Status ) self.assertEqual( ftsTransfer.Status, "Scheduled", "3. wrong status for ftsStatus: %s" % ftsTransfer.Status ) self.assertEqual( req.Status, "Waiting", "3. wrong status for request: %s" % req.Status ) # # prev done insertFile.Status = "Done" self.assertEqual( insertBefore.Status, "Done", "4. wrong status for insertBefore: %s" % insertBefore.Status ) self.assertEqual( ftsTransfer.Status, "Scheduled", "4. wrong status for ftsStatus: %s" % ftsTransfer.Status ) self.assertEqual( req.Status, "Scheduled", "4. wrong status for request: %s" % req.Status ) # # reschedule ftsFile.Status = "Waiting" self.assertEqual( insertBefore.Status, "Done", "5. wrong status for insertBefore: %s" % insertBefore.Status ) self.assertEqual( ftsTransfer.Status, "Waiting", "5. wrong status for ftsStatus: %s" % ftsTransfer.Status ) self.assertEqual( req.Status, "Waiting", "5. wrong status for request: %s" % req.Status ) # # fts done ftsFile.Status = "Done" self.assertEqual( insertBefore.Status, "Done", "5. wrong status for insertBefore: %s" % insertBefore.Status ) self.assertEqual( ftsTransfer.Status, "Done", "5. wrong status for ftsStatus: %s" % ftsTransfer.Status ) self.assertEqual( req.Status, "Done", "5. wrong status for request: %s" % req.Status )
def prepareTransformationTasks( self, transBody, taskDict, owner = '', ownerGroup = '', ownerDN = '' ): """ Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB """ if not taskDict: return S_OK({}) if ( not owner ) or ( not ownerGroup ): res = getProxyInfo( False, False ) if not res['OK']: return res proxyInfo = res['Value'] owner = proxyInfo['username'] ownerGroup = proxyInfo['group'] if not ownerDN: res = getDNForUsername( owner ) if not res['OK']: return res ownerDN = res['Value'][0] requestOperation = 'ReplicateAndRegister' if transBody: try: _requestType, requestOperation = transBody.split( ';' ) except AttributeError: pass # Do not remove sorted, we might pop elements in the loop for taskID in sorted( taskDict ): paramDict = taskDict[taskID] transID = paramDict['TransformationID'] oRequest = Request() transfer = Operation() transfer.Type = requestOperation transfer.TargetSE = paramDict['TargetSE'] # If there are input files if paramDict['InputData']: if isinstance( paramDict['InputData'], list ): files = paramDict['InputData'] elif isinstance( paramDict['InputData'], basestring ): files = paramDict['InputData'].split( ';' ) for lfn in files: trFile = File() trFile.LFN = lfn transfer.addFile( trFile ) oRequest.addOperation( transfer ) oRequest.RequestName = _requestName( transID, taskID ) oRequest.OwnerDN = ownerDN oRequest.OwnerGroup = ownerGroup isValid = self.requestValidator.validate( oRequest ) if not isValid['OK']: self.log.error( "Error creating request for task", "%s %s" % ( taskID, isValid ) ) # This works because we loop over a copy of the keys ! taskDict.pop( taskID ) continue taskDict[taskID]['TaskObject'] = oRequest return S_OK( taskDict )