def _setFileReplicationRequest(self, lfn, targetSE, fileMetaDict, sourceSE=""): """Sets a registration request.""" self.log.info("Setting ReplicateAndRegister request", "for %s to %s" % (lfn, targetSE)) transfer = Operation() transfer.Type = "ReplicateAndRegister" transfer.TargetSE = targetSE if sourceSE: transfer.SourceSE = sourceSE trFile = File() trFile.LFN = lfn cksm = fileMetaDict.get("Checksum", None) cksmType = fileMetaDict.get("ChecksumType", self.defaultChecksumType) if cksm and cksmType: trFile.Checksum = cksm trFile.ChecksumType = cksmType size = fileMetaDict.get("Size", 0) if size: trFile.Size = size guid = fileMetaDict.get("GUID", "") if guid: trFile.GUID = guid transfer.addFile(trFile) self.request.addOperation(transfer) return S_OK()
def test02props( self ): """ test properties """ # # valid values operation = Operation() operation.Arguments = "foobar" self.assertEqual( operation.Arguments, "foobar", "wrong Arguments" ) operation.SourceSE = "CERN-RAW" self.assertEqual( operation.SourceSE, "CERN-RAW", "wrong SourceSE" ) operation.TargetSE = "CERN-RAW" self.assertEqual( operation.TargetSE, "CERN-RAW", "wrong TargetSE" ) operation.Catalog = "" self.assertEqual( operation.Catalog, "", "wrong Catalog" ) operation.Catalog = "BookkeepingDB" self.assertEqual( operation.Catalog, "BookkeepingDB", "wrong Catalog" ) operation.Error = "error" self.assertEqual( operation.Error, "error", "wrong Error" ) # # wrong props try: operation.RequestID = "foo" except Exception, error: self.assertEqual( type( error ), AttributeError, "wrong exc raised" ) self.assertEqual( str( error ), "can't set attribute", "wrong exc reason" )
def __setFileReplicationRequest( self, lfn, targetSE, fileMetaDict, sourceSE = '' ): """ Sets a registration request. """ self.log.info( 'Setting replication request for %s to %s' % ( lfn, targetSE ) ) transfer = Operation() transfer.Type = "ReplicateAndRegister" transfer.TargetSE = targetSE if sourceSE: transfer.SourceSE = sourceSE trFile = File() trFile.LFN = lfn cksm = fileMetaDict.get( "Checksum", None ) cksmType = fileMetaDict.get( "ChecksumType", None ) if cksm and cksmType: trFile.Checksum = cksm trFile.ChecksumType = cksmType size = fileMetaDict.get( "Size", 0 ) if size: trFile.Size = size guid = fileMetaDict.get( "GUID", "" ) if guid: trFile.GUID = guid transfer.addFile( trFile ) self.request.addOperation( transfer ) return S_OK()
def test02props(self): """ test properties """ # # valid values operation = Operation() operation.OperationID = 1 self.assertEqual(operation.OperationID, 1, "wrong OperationID") operation.OperationID = "1" self.assertEqual(operation.OperationID, 1, "wrong OperationID") operation.Arguments = "foobar" self.assertEqual(operation.Arguments, "foobar", "wrong Arguments") operation.SourceSE = "CERN-RAW" self.assertEqual(operation.SourceSE, "CERN-RAW", "wrong SourceSE") operation.TargetSE = "CERN-RAW" self.assertEqual(operation.TargetSE, "CERN-RAW", "wrong TargetSE") operation.Catalog = "" self.assertEqual(operation.Catalog, "", "wrong Catalog") operation.Catalog = "BookkeepingDB" self.assertEqual(operation.Catalog, "BookkeepingDB", "wrong Catalog") operation.Error = "error" self.assertEqual(operation.Error, "error", "wrong Error") # # wrong props try: operation.RequestID = "foo" except Exception, error: self.assertEqual(type(error), AttributeError, "wrong exc raised") self.assertEqual(str(error), "can't set attribute", "wrong exc reason")
def test_valid_properties(): operation = Operation() operation.Arguments = "foobar" assert operation.Arguments == b"foobar", "wrong Arguments" operation.SourceSE = "CERN-RAW" assert operation.SourceSE == "CERN-RAW", "wrong SourceSE" operation.TargetSE = "CERN-RAW" assert operation.TargetSE == "CERN-RAW", "wrong TargetSE" operation.Catalog = "" assert operation.Catalog == "", "wrong Catalog" operation.Catalog = "BookkeepingDB" assert operation.Catalog == "BookkeepingDB", "wrong Catalog" operation.Error = "error" assert operation.Error == "error", "wrong Error" toJSON = operation.toJSON() assert toJSON["OK"]
def createRequest(reqType): r = Request() # Simple failover op1 = Operation() f = File() f.LFN = '/This/is/an/LFN' op1.addFile(f) op1.Type = 'ReplicateAndRegister' op1.SourceSE = 'CERN-FAILOVER' op1.TargetSE = 'CERN-BUFFER' r.addOperation(op1) op2 = Operation() op2.addFile(f) op2.Type = 'RemoveReplica' op2.TargetSE = 'CERN-FAILOVER' r.addOperation(op2) if reqType == 0: return r # two files for Failover f1 = File() f1.LFN = '/This/is/a/second/LFN' op3 = Operation() op3.addFile(f1) op3.Type = 'ReplicateAndRegister' op3.SourceSE = 'CERN-FAILOVER' op3.TargetSE = 'CERN-BUFFER' r.addOperation(op3) op3 = Operation() op3.addFile(f1) op3.Type = 'RemoveReplica' op3.TargetSE = 'CERN-FAILOVER' r.addOperation(op3) if reqType == 1: return r op = Operation() op.Type = 'ForwardDiset' if reqType == 2: r.addOperation(op) return r r.insertBefore(op, r[0]) if reqType == 3: return r op4 = Operation() op4.Type = 'ForwardDiset' r.addOperation(op4) if reqType == 4: return r # 2 different FAILOVER SEs: removal not optimized r[1].SourceSE = 'RAL-FAILOVER' r[2].SourceSE = 'RAL-FAILOVER' if reqType == 5: return r # 2 different destinations, same FAILOVER: replication not optimized r[3].SourceSE = 'RAL-FAILOVER' r[4].SourceSE = 'RAL-FAILOVER' r[3].TargetSE = 'RAL-BUFFER' if reqType == 6: return r print 'This should not happen, reqType =', reqType
def createRequest( reqType ): r = Request() # Simple failover op1 = Operation() f = File() f.LFN = '/This/is/an/LFN' op1.addFile( f ) op1.Type = 'ReplicateAndRegister' op1.SourceSE = 'CERN-FAILOVER' op1.TargetSE = 'CERN-BUFFER' r.addOperation( op1 ) op2 = Operation() op2.addFile( f ) op2.Type = 'RemoveReplica' op2.TargetSE = 'CERN-FAILOVER' r.addOperation( op2 ) if reqType == 0: return r # two files for Failover f1 = File() f1.LFN = '/This/is/a/second/LFN' op3 = Operation() op3.addFile( f1 ) op3.Type = 'ReplicateAndRegister' op3.SourceSE = 'CERN-FAILOVER' op3.TargetSE = 'CERN-BUFFER' r.addOperation( op3 ) op3 = Operation() op3.addFile( f1 ) op3.Type = 'RemoveReplica' op3.TargetSE = 'CERN-FAILOVER' r.addOperation( op3 ) if reqType == 1: return r op = Operation() op.Type = 'ForwardDiset' if reqType == 2: r.addOperation( op ) return r r.insertBefore( op, r[0] ) if reqType == 3: return r op4 = Operation() op4.Type = 'ForwardDiset' r.addOperation( op4 ) if reqType == 4: return r # 2 different FAILOVER SEs: removal not optimized r[1].SourceSE = 'RAL-FAILOVER' r[2].SourceSE = 'RAL-FAILOVER' if reqType == 5: return r # 2 different destinations, same FAILOVER: replication not optimized r[3].SourceSE = 'RAL-FAILOVER' r[4].SourceSE = 'RAL-FAILOVER' r[3].TargetSE = 'RAL-BUFFER' if reqType == 6: return r print 'This should not happen, reqType =', reqType
def main(): # Registering arguments will automatically add their description to the help menu Script.registerArgument(" sourceSE: source SE") Script.registerArgument(" LFN: LFN or file containing a List of LFNs") Script.registerArgument(["targetSE: target SEs"]) Script.parseCommandLine() import DIRAC from DIRAC import gLogger # parseCommandLine show help when mandatory arguments are not specified or incorrect argument args = Script.getPositionalArgs() sourceSE = args[0] lfnList = getLFNList(args[1]) targetSEs = list(set([se for targetSE in args[2:] for se in targetSE.split(",")])) gLogger.info( "Will create request with 'MoveReplica' " "operation using %s lfns and %s target SEs" % (len(lfnList), len(targetSEs)) ) from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient from DIRAC.RequestManagementSystem.Client.Request import Request from DIRAC.RequestManagementSystem.Client.Operation import Operation from DIRAC.RequestManagementSystem.Client.File import File from DIRAC.Resources.Catalog.FileCatalog import FileCatalog from DIRAC.Core.Utilities.List import breakListIntoChunks lfnChunks = breakListIntoChunks(lfnList, 100) multiRequests = len(lfnChunks) > 1 error = 0 count = 0 reqClient = ReqClient() fc = FileCatalog() for lfnChunk in lfnChunks: metaDatas = fc.getFileMetadata(lfnChunk) if not metaDatas["OK"]: gLogger.error("unable to read metadata for lfns: %s" % metaDatas["Message"]) error = -1 continue metaDatas = metaDatas["Value"] for failedLFN, reason in metaDatas["Failed"].items(): gLogger.error("skipping %s: %s" % (failedLFN, reason)) lfnChunk = set(metaDatas["Successful"]) if not lfnChunk: gLogger.error("LFN list is empty!!!") error = -1 continue if len(lfnChunk) > Operation.MAX_FILES: gLogger.error("too many LFNs, max number of files per operation is %s" % Operation.MAX_FILES) error = -1 continue count += 1 request = Request() request.RequestName = "%s_%s" % ( md5(repr(time.time()).encode()).hexdigest()[:16], md5(repr(time.time()).encode()).hexdigest()[:16], ) moveReplica = Operation() moveReplica.Type = "MoveReplica" moveReplica.SourceSE = sourceSE moveReplica.TargetSE = ",".join(targetSEs) for lfn in lfnChunk: metaDict = metaDatas["Successful"][lfn] opFile = File() opFile.LFN = lfn opFile.Size = metaDict["Size"] if "Checksum" in metaDict: # # should check checksum type, now assuming Adler32 (metaDict["ChecksumType"] = 'AD' opFile.Checksum = metaDict["Checksum"] opFile.ChecksumType = "ADLER32" moveReplica.addFile(opFile) request.addOperation(moveReplica) result = reqClient.putRequest(request) if not result["OK"]: gLogger.error("Failed to submit Request: %s" % (result["Message"])) error = -1 continue if not multiRequests: gLogger.always("Request %d submitted successfully" % result["Value"]) if multiRequests: gLogger.always("%d requests have been submitted" % (count)) DIRAC.exit(error)
gLogger.error( "too many LFNs, max number of files per operation is %s" % Operation.MAX_FILES) error = -1 continue count += 1 request = Request() request.RequestName = "%s_%s" % (md5(repr( time.time())).hexdigest()[:16], md5(repr( time.time())).hexdigest()[:16]) moveReplica = Operation() moveReplica.Type = 'MoveReplica' moveReplica.SourceSE = sourceSE moveReplica.TargetSE = ",".join(targetSEs) for lfn in lfnChunk: metaDict = metaDatas["Successful"][lfn] opFile = File() opFile.LFN = lfn opFile.Size = metaDict["Size"] if "Checksum" in metaDict: # # should check checksum type, now assuming Adler32 (metaDict["ChecksumType"] = 'AD' opFile.Checksum = metaDict["Checksum"] opFile.ChecksumType = "ADLER32" moveReplica.addFile(opFile) request.addOperation(moveReplica)
def createRequest(reqType): r = Request() # Simple failover op1 = Operation() f = File() f.LFN = "/This/is/an/LFN" op1.addFile(f) op1.Type = "ReplicateAndRegister" op1.SourceSE = "CERN-FAILOVER" op1.TargetSE = "CERN-BUFFER" r.addOperation(op1) # You cannot reuse the same File object, # since it is a different entry in the DB fr = File() fr.LFN = "/This/is/an/LFN" op2 = Operation() op2.addFile(fr) op2.Type = "RemoveReplica" op2.TargetSE = "CERN-FAILOVER" r.addOperation(op2) if reqType == 0: return r # two files for Failover f1 = File() f1.LFN = "/This/is/a/second/LFN" op3 = Operation() op3.addFile(f1) op3.Type = "ReplicateAndRegister" op3.SourceSE = "CERN-FAILOVER" op3.TargetSE = "CERN-BUFFER" r.addOperation(op3) f1r = File() f1r.LFN = "/This/is/a/second/LFN" op3 = Operation() op3.addFile(f1r) op3.Type = "RemoveReplica" op3.TargetSE = "CERN-FAILOVER" r.addOperation(op3) if reqType == 1: return r op = Operation() op.Type = "ForwardDiset" if reqType == 2: r.addOperation(op) return r r.insertBefore(op, r[0]) if reqType == 3: return r op4 = Operation() op4.Type = "ForwardDiset" r.addOperation(op4) if reqType == 4: return r # 2 different FAILOVER SEs: removal not optimized r[1].SourceSE = "RAL-FAILOVER" r[2].SourceSE = "RAL-FAILOVER" if reqType == 5: return r # 2 different destinations, same FAILOVER: replication not optimized r[3].SourceSE = "RAL-FAILOVER" r[4].SourceSE = "RAL-FAILOVER" r[3].TargetSE = "RAL-BUFFER" if reqType == 6: return r print("This should not happen, reqType =", reqType)
error = -1 continue if len( lfnChunk ) > Operation.MAX_FILES: gLogger.error( "too many LFNs, max number of files per operation is %s" % Operation.MAX_FILES ) error = -1 continue count += 1 request = Request() request.RequestName = "%s_%s" % ( md5( repr( time.time() ) ).hexdigest()[:16], md5( repr( time.time() ) ).hexdigest()[:16] ) moveReplica = Operation() moveReplica.Type = 'MoveReplica' moveReplica.SourceSE = sourceSE moveReplica.TargetSE = ",".join( targetSEs ) for lfn in lfnChunk: metaDict = metaDatas["Successful"][lfn] opFile = File() opFile.LFN = lfn opFile.Size = metaDict["Size"] if "Checksum" in metaDict: # # should check checksum type, now assuming Adler32 (metaDict["ChecksumType"] = 'AD' opFile.Checksum = metaDict["Checksum"] opFile.ChecksumType = "ADLER32" moveReplica.addFile( opFile ) request.addOperation( moveReplica )
def test02props(self): """ test properties """ # # valid values operation = Operation() operation.Arguments = "foobar" self.assertEqual(operation.Arguments, "foobar", "wrong Arguments") operation.SourceSE = "CERN-RAW" self.assertEqual(operation.SourceSE, "CERN-RAW", "wrong SourceSE") operation.TargetSE = "CERN-RAW" self.assertEqual(operation.TargetSE, "CERN-RAW", "wrong TargetSE") operation.Catalog = "" self.assertEqual(operation.Catalog, "", "wrong Catalog") operation.Catalog = "BookkeepingDB" self.assertEqual(operation.Catalog, "BookkeepingDB", "wrong Catalog") operation.Error = "error" self.assertEqual(operation.Error, "error", "wrong Error") # # wrong props try: operation.RequestID = "foo" except Exception as error: self.assertEqual(type(error), AttributeError, "wrong exc raised") self.assertEqual(str(error), "can't set attribute", "wrong exc reason") try: operation.OperationID = "foo" except Exception as error: self.assertEqual(type(error), ValueError, "wrong exc raised") # # timestamps try: operation.SubmitTime = "foo" except Exception as error: self.assertEqual(type(error), ValueError, "wrong exp raised") self.assertEqual( str(error), "time data 'foo' does not match format '%Y-%m-%d %H:%M:%S'", "wrong exc reason") try: operation.LastUpdate = "foo" except Exception as error: self.assertEqual(type(error), ValueError, "wrong exc raised") self.assertEqual( str(error), "time data 'foo' does not match format '%Y-%m-%d %H:%M:%S'", "wrong exc reason") # # Status operation = Operation() try: operation.Status = "foo" except Exception as error: self.assertEqual(type(error), ValueError, "wrong exc raised") self.assertEqual(str(error), "unknown Status 'foo'", "wrong exc reason") operation.addFile(File({"Status": "Waiting", "LFN": "/a"}))