def _setFileReplicationRequest(self,
lfn,
targetSE,
fileMetaDict,
sourceSE=""):
"""Sets a registration request."""
self.log.info("Setting ReplicateAndRegister request",
"for %s to %s" % (lfn, targetSE))
transfer = Operation()
transfer.Type = "ReplicateAndRegister"
transfer.TargetSE = targetSE
if sourceSE:
transfer.SourceSE = sourceSE
trFile = File()
trFile.LFN = lfn
cksm = fileMetaDict.get("Checksum", None)
cksmType = fileMetaDict.get("ChecksumType", self.defaultChecksumType)
if cksm and cksmType:
trFile.Checksum = cksm
trFile.ChecksumType = cksmType
size = fileMetaDict.get("Size", 0)
if size:
trFile.Size = size
guid = fileMetaDict.get("GUID", "")
if guid:
trFile.GUID = guid
transfer.addFile(trFile)
self.request.addOperation(transfer)
return S_OK()
def test02props( self ):
""" test properties """
# # valid values
operation = Operation()
operation.Arguments = "foobar"
self.assertEqual( operation.Arguments, "foobar", "wrong Arguments" )
operation.SourceSE = "CERN-RAW"
self.assertEqual( operation.SourceSE, "CERN-RAW", "wrong SourceSE" )
operation.TargetSE = "CERN-RAW"
self.assertEqual( operation.TargetSE, "CERN-RAW", "wrong TargetSE" )
operation.Catalog = ""
self.assertEqual( operation.Catalog, "", "wrong Catalog" )
operation.Catalog = "BookkeepingDB"
self.assertEqual( operation.Catalog, "BookkeepingDB", "wrong Catalog" )
operation.Error = "error"
self.assertEqual( operation.Error, "error", "wrong Error" )
# # wrong props
try:
operation.RequestID = "foo"
except Exception, error:
self.assertEqual( type( error ), AttributeError, "wrong exc raised" )
self.assertEqual( str( error ), "can't set attribute", "wrong exc reason" )
def __setFileReplicationRequest( self, lfn, targetSE, fileMetaDict, sourceSE = '' ):
""" Sets a registration request.
"""
self.log.info( 'Setting replication request for %s to %s' % ( lfn, targetSE ) )
transfer = Operation()
transfer.Type = "ReplicateAndRegister"
transfer.TargetSE = targetSE
if sourceSE:
transfer.SourceSE = sourceSE
trFile = File()
trFile.LFN = lfn
cksm = fileMetaDict.get( "Checksum", None )
cksmType = fileMetaDict.get( "ChecksumType", None )
if cksm and cksmType:
trFile.Checksum = cksm
trFile.ChecksumType = cksmType
size = fileMetaDict.get( "Size", 0 )
if size:
trFile.Size = size
guid = fileMetaDict.get( "GUID", "" )
if guid:
trFile.GUID = guid
transfer.addFile( trFile )
self.request.addOperation( transfer )
return S_OK()
def test02props(self):
""" test properties """
# # valid values
operation = Operation()
operation.OperationID = 1
self.assertEqual(operation.OperationID, 1, "wrong OperationID")
operation.OperationID = "1"
self.assertEqual(operation.OperationID, 1, "wrong OperationID")
operation.Arguments = "foobar"
self.assertEqual(operation.Arguments, "foobar", "wrong Arguments")
operation.SourceSE = "CERN-RAW"
self.assertEqual(operation.SourceSE, "CERN-RAW", "wrong SourceSE")
operation.TargetSE = "CERN-RAW"
self.assertEqual(operation.TargetSE, "CERN-RAW", "wrong TargetSE")
operation.Catalog = ""
self.assertEqual(operation.Catalog, "", "wrong Catalog")
operation.Catalog = "BookkeepingDB"
self.assertEqual(operation.Catalog, "BookkeepingDB", "wrong Catalog")
operation.Error = "error"
self.assertEqual(operation.Error, "error", "wrong Error")
# # wrong props
try:
operation.RequestID = "foo"
except Exception, error:
self.assertEqual(type(error), AttributeError, "wrong exc raised")
self.assertEqual(str(error), "can't set attribute",
"wrong exc reason")
def test_valid_properties():
operation = Operation()
operation.Arguments = "foobar"
assert operation.Arguments == b"foobar", "wrong Arguments"
operation.SourceSE = "CERN-RAW"
assert operation.SourceSE == "CERN-RAW", "wrong SourceSE"
operation.TargetSE = "CERN-RAW"
assert operation.TargetSE == "CERN-RAW", "wrong TargetSE"
operation.Catalog = ""
assert operation.Catalog == "", "wrong Catalog"
operation.Catalog = "BookkeepingDB"
assert operation.Catalog == "BookkeepingDB", "wrong Catalog"
operation.Error = "error"
assert operation.Error == "error", "wrong Error"
toJSON = operation.toJSON()
assert toJSON["OK"]
def createRequest(reqType):
r = Request()
# Simple failover
op1 = Operation()
f = File()
f.LFN = '/This/is/an/LFN'
op1.addFile(f)
op1.Type = 'ReplicateAndRegister'
op1.SourceSE = 'CERN-FAILOVER'
op1.TargetSE = 'CERN-BUFFER'
r.addOperation(op1)
op2 = Operation()
op2.addFile(f)
op2.Type = 'RemoveReplica'
op2.TargetSE = 'CERN-FAILOVER'
r.addOperation(op2)
if reqType == 0:
return r
# two files for Failover
f1 = File()
f1.LFN = '/This/is/a/second/LFN'
op3 = Operation()
op3.addFile(f1)
op3.Type = 'ReplicateAndRegister'
op3.SourceSE = 'CERN-FAILOVER'
op3.TargetSE = 'CERN-BUFFER'
r.addOperation(op3)
op3 = Operation()
op3.addFile(f1)
op3.Type = 'RemoveReplica'
op3.TargetSE = 'CERN-FAILOVER'
r.addOperation(op3)
if reqType == 1:
return r
op = Operation()
op.Type = 'ForwardDiset'
if reqType == 2:
r.addOperation(op)
return r
r.insertBefore(op, r[0])
if reqType == 3:
return r
op4 = Operation()
op4.Type = 'ForwardDiset'
r.addOperation(op4)
if reqType == 4:
return r
# 2 different FAILOVER SEs: removal not optimized
r[1].SourceSE = 'RAL-FAILOVER'
r[2].SourceSE = 'RAL-FAILOVER'
if reqType == 5:
return r
# 2 different destinations, same FAILOVER: replication not optimized
r[3].SourceSE = 'RAL-FAILOVER'
r[4].SourceSE = 'RAL-FAILOVER'
r[3].TargetSE = 'RAL-BUFFER'
if reqType == 6:
return r
print 'This should not happen, reqType =', reqType
def createRequest( reqType ):
r = Request()
# Simple failover
op1 = Operation()
f = File()
f.LFN = '/This/is/an/LFN'
op1.addFile( f )
op1.Type = 'ReplicateAndRegister'
op1.SourceSE = 'CERN-FAILOVER'
op1.TargetSE = 'CERN-BUFFER'
r.addOperation( op1 )
op2 = Operation()
op2.addFile( f )
op2.Type = 'RemoveReplica'
op2.TargetSE = 'CERN-FAILOVER'
r.addOperation( op2 )
if reqType == 0:
return r
# two files for Failover
f1 = File()
f1.LFN = '/This/is/a/second/LFN'
op3 = Operation()
op3.addFile( f1 )
op3.Type = 'ReplicateAndRegister'
op3.SourceSE = 'CERN-FAILOVER'
op3.TargetSE = 'CERN-BUFFER'
r.addOperation( op3 )
op3 = Operation()
op3.addFile( f1 )
op3.Type = 'RemoveReplica'
op3.TargetSE = 'CERN-FAILOVER'
r.addOperation( op3 )
if reqType == 1:
return r
op = Operation()
op.Type = 'ForwardDiset'
if reqType == 2:
r.addOperation( op )
return r
r.insertBefore( op, r[0] )
if reqType == 3:
return r
op4 = Operation()
op4.Type = 'ForwardDiset'
r.addOperation( op4 )
if reqType == 4:
return r
# 2 different FAILOVER SEs: removal not optimized
r[1].SourceSE = 'RAL-FAILOVER'
r[2].SourceSE = 'RAL-FAILOVER'
if reqType == 5:
return r
# 2 different destinations, same FAILOVER: replication not optimized
r[3].SourceSE = 'RAL-FAILOVER'
r[4].SourceSE = 'RAL-FAILOVER'
r[3].TargetSE = 'RAL-BUFFER'
if reqType == 6:
return r
print 'This should not happen, reqType =', reqType
def main():
# Registering arguments will automatically add their description to the help menu
Script.registerArgument(" sourceSE: source SE")
Script.registerArgument(" LFN: LFN or file containing a List of LFNs")
Script.registerArgument(["targetSE: target SEs"])
Script.parseCommandLine()
import DIRAC
from DIRAC import gLogger
# parseCommandLine show help when mandatory arguments are not specified or incorrect argument
args = Script.getPositionalArgs()
sourceSE = args[0]
lfnList = getLFNList(args[1])
targetSEs = list(set([se for targetSE in args[2:] for se in targetSE.split(",")]))
gLogger.info(
"Will create request with 'MoveReplica' "
"operation using %s lfns and %s target SEs" % (len(lfnList), len(targetSEs))
)
from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient
from DIRAC.RequestManagementSystem.Client.Request import Request
from DIRAC.RequestManagementSystem.Client.Operation import Operation
from DIRAC.RequestManagementSystem.Client.File import File
from DIRAC.Resources.Catalog.FileCatalog import FileCatalog
from DIRAC.Core.Utilities.List import breakListIntoChunks
lfnChunks = breakListIntoChunks(lfnList, 100)
multiRequests = len(lfnChunks) > 1
error = 0
count = 0
reqClient = ReqClient()
fc = FileCatalog()
for lfnChunk in lfnChunks:
metaDatas = fc.getFileMetadata(lfnChunk)
if not metaDatas["OK"]:
gLogger.error("unable to read metadata for lfns: %s" % metaDatas["Message"])
error = -1
continue
metaDatas = metaDatas["Value"]
for failedLFN, reason in metaDatas["Failed"].items():
gLogger.error("skipping %s: %s" % (failedLFN, reason))
lfnChunk = set(metaDatas["Successful"])
if not lfnChunk:
gLogger.error("LFN list is empty!!!")
error = -1
continue
if len(lfnChunk) > Operation.MAX_FILES:
gLogger.error("too many LFNs, max number of files per operation is %s" % Operation.MAX_FILES)
error = -1
continue
count += 1
request = Request()
request.RequestName = "%s_%s" % (
md5(repr(time.time()).encode()).hexdigest()[:16],
md5(repr(time.time()).encode()).hexdigest()[:16],
)
moveReplica = Operation()
moveReplica.Type = "MoveReplica"
moveReplica.SourceSE = sourceSE
moveReplica.TargetSE = ",".join(targetSEs)
for lfn in lfnChunk:
metaDict = metaDatas["Successful"][lfn]
opFile = File()
opFile.LFN = lfn
opFile.Size = metaDict["Size"]
if "Checksum" in metaDict:
# # should check checksum type, now assuming Adler32 (metaDict["ChecksumType"] = 'AD'
opFile.Checksum = metaDict["Checksum"]
opFile.ChecksumType = "ADLER32"
moveReplica.addFile(opFile)
request.addOperation(moveReplica)
result = reqClient.putRequest(request)
if not result["OK"]:
gLogger.error("Failed to submit Request: %s" % (result["Message"]))
error = -1
continue
if not multiRequests:
gLogger.always("Request %d submitted successfully" % result["Value"])
if multiRequests:
gLogger.always("%d requests have been submitted" % (count))
DIRAC.exit(error)
gLogger.error(
"too many LFNs, max number of files per operation is %s" %
Operation.MAX_FILES)
error = -1
continue
count += 1
request = Request()
request.RequestName = "%s_%s" % (md5(repr(
time.time())).hexdigest()[:16], md5(repr(
time.time())).hexdigest()[:16])
moveReplica = Operation()
moveReplica.Type = 'MoveReplica'
moveReplica.SourceSE = sourceSE
moveReplica.TargetSE = ",".join(targetSEs)
for lfn in lfnChunk:
metaDict = metaDatas["Successful"][lfn]
opFile = File()
opFile.LFN = lfn
opFile.Size = metaDict["Size"]
if "Checksum" in metaDict:
# # should check checksum type, now assuming Adler32 (metaDict["ChecksumType"] = 'AD'
opFile.Checksum = metaDict["Checksum"]
opFile.ChecksumType = "ADLER32"
moveReplica.addFile(opFile)
request.addOperation(moveReplica)
def createRequest(reqType):
r = Request()
# Simple failover
op1 = Operation()
f = File()
f.LFN = "/This/is/an/LFN"
op1.addFile(f)
op1.Type = "ReplicateAndRegister"
op1.SourceSE = "CERN-FAILOVER"
op1.TargetSE = "CERN-BUFFER"
r.addOperation(op1)
# You cannot reuse the same File object,
# since it is a different entry in the DB
fr = File()
fr.LFN = "/This/is/an/LFN"
op2 = Operation()
op2.addFile(fr)
op2.Type = "RemoveReplica"
op2.TargetSE = "CERN-FAILOVER"
r.addOperation(op2)
if reqType == 0:
return r
# two files for Failover
f1 = File()
f1.LFN = "/This/is/a/second/LFN"
op3 = Operation()
op3.addFile(f1)
op3.Type = "ReplicateAndRegister"
op3.SourceSE = "CERN-FAILOVER"
op3.TargetSE = "CERN-BUFFER"
r.addOperation(op3)
f1r = File()
f1r.LFN = "/This/is/a/second/LFN"
op3 = Operation()
op3.addFile(f1r)
op3.Type = "RemoveReplica"
op3.TargetSE = "CERN-FAILOVER"
r.addOperation(op3)
if reqType == 1:
return r
op = Operation()
op.Type = "ForwardDiset"
if reqType == 2:
r.addOperation(op)
return r
r.insertBefore(op, r[0])
if reqType == 3:
return r
op4 = Operation()
op4.Type = "ForwardDiset"
r.addOperation(op4)
if reqType == 4:
return r
# 2 different FAILOVER SEs: removal not optimized
r[1].SourceSE = "RAL-FAILOVER"
r[2].SourceSE = "RAL-FAILOVER"
if reqType == 5:
return r
# 2 different destinations, same FAILOVER: replication not optimized
r[3].SourceSE = "RAL-FAILOVER"
r[4].SourceSE = "RAL-FAILOVER"
r[3].TargetSE = "RAL-BUFFER"
if reqType == 6:
return r
print("This should not happen, reqType =", reqType)
error = -1
continue
if len( lfnChunk ) > Operation.MAX_FILES:
gLogger.error( "too many LFNs, max number of files per operation is %s" % Operation.MAX_FILES )
error = -1
continue
count += 1
request = Request()
request.RequestName = "%s_%s" % ( md5( repr( time.time() ) ).hexdigest()[:16], md5( repr( time.time() ) ).hexdigest()[:16] )
moveReplica = Operation()
moveReplica.Type = 'MoveReplica'
moveReplica.SourceSE = sourceSE
moveReplica.TargetSE = ",".join( targetSEs )
for lfn in lfnChunk:
metaDict = metaDatas["Successful"][lfn]
opFile = File()
opFile.LFN = lfn
opFile.Size = metaDict["Size"]
if "Checksum" in metaDict:
# # should check checksum type, now assuming Adler32 (metaDict["ChecksumType"] = 'AD'
opFile.Checksum = metaDict["Checksum"]
opFile.ChecksumType = "ADLER32"
moveReplica.addFile( opFile )
request.addOperation( moveReplica )
def test02props(self):
""" test properties """
# # valid values
operation = Operation()
operation.Arguments = "foobar"
self.assertEqual(operation.Arguments, "foobar", "wrong Arguments")
operation.SourceSE = "CERN-RAW"
self.assertEqual(operation.SourceSE, "CERN-RAW", "wrong SourceSE")
operation.TargetSE = "CERN-RAW"
self.assertEqual(operation.TargetSE, "CERN-RAW", "wrong TargetSE")
operation.Catalog = ""
self.assertEqual(operation.Catalog, "", "wrong Catalog")
operation.Catalog = "BookkeepingDB"
self.assertEqual(operation.Catalog, "BookkeepingDB", "wrong Catalog")
operation.Error = "error"
self.assertEqual(operation.Error, "error", "wrong Error")
# # wrong props
try:
operation.RequestID = "foo"
except Exception as error:
self.assertEqual(type(error), AttributeError, "wrong exc raised")
self.assertEqual(str(error), "can't set attribute",
"wrong exc reason")
try:
operation.OperationID = "foo"
except Exception as error:
self.assertEqual(type(error), ValueError, "wrong exc raised")
# # timestamps
try:
operation.SubmitTime = "foo"
except Exception as error:
self.assertEqual(type(error), ValueError, "wrong exp raised")
self.assertEqual(
str(error),
"time data 'foo' does not match format '%Y-%m-%d %H:%M:%S'",
"wrong exc reason")
try:
operation.LastUpdate = "foo"
except Exception as error:
self.assertEqual(type(error), ValueError, "wrong exc raised")
self.assertEqual(
str(error),
"time data 'foo' does not match format '%Y-%m-%d %H:%M:%S'",
"wrong exc reason")
# # Status
operation = Operation()
try:
operation.Status = "foo"
except Exception as error:
self.assertEqual(type(error), ValueError, "wrong exc raised")
self.assertEqual(str(error), "unknown Status 'foo'",
"wrong exc reason")
operation.addFile(File({"Status": "Waiting", "LFN": "/a"}))
