def __insertRegisterOperation( self, request, operation, toRegister ):
""" add RegisterReplica operation
:param Request request: request instance
:param Operation transferOp: 'ReplicateAndRegister' operation for this FTSJob
:param list toRegister: [ FTSDB.FTSFile, ... ] - files that failed to register
"""
log = self.log.getSubLogger( "req_%s/%s/registerFiles" % ( request.RequestID, request.RequestName ) )
byTarget = {}
for ftsFile in toRegister:
if ftsFile.TargetSE not in byTarget:
byTarget.setdefault( ftsFile.TargetSE, [] )
byTarget[ftsFile.TargetSE].append( ftsFile )
log.info( "will create %s 'RegisterReplica' operations" % len( byTarget ) )
for target, ftsFileList in byTarget.iteritems():
log.info( "creating 'RegisterReplica' operation for targetSE %s with %s files..." % ( target,
len( ftsFileList ) ) )
registerOperation = Operation()
registerOperation.Type = "RegisterReplica"
registerOperation.Status = "Waiting"
registerOperation.TargetSE = target
targetSE = StorageElement( target )
for ftsFile in ftsFileList:
opFile = File()
opFile.LFN = ftsFile.LFN
pfn = returnSingleResult( targetSE.getURL( ftsFile.LFN, protocol = self.registrationProtocols ) )
if not pfn["OK"]:
continue
opFile.PFN = pfn["Value"]
registerOperation.addFile( opFile )
request.insertBefore( registerOperation, operation )
return S_OK()
def __insertRegisterOperation( self, request, operation, toRegister ):
""" add RegisterReplica operation
:param Request request: request instance
:param Operation transferOp: 'ReplicateAndRegister' operation for this FTSJob
:param list toRegister: [ FTSDB.FTSFile, ... ] - files that failed to register
"""
log = self.log.getSubLogger( "req_%s/%s/registerFiles" % ( request.RequestID, request.RequestName ) )
byTarget = {}
for ftsFile in toRegister:
if ftsFile.TargetSE not in byTarget:
byTarget.setdefault( ftsFile.TargetSE, [] )
byTarget[ftsFile.TargetSE].append( ftsFile )
log.info( "will create %s 'RegisterReplica' operations" % len( byTarget ) )
for target, ftsFileList in byTarget.iteritems():
log.info( "creating 'RegisterReplica' operation for targetSE %s with %s files..." % ( target,
len( ftsFileList ) ) )
registerOperation = Operation()
registerOperation.Type = "RegisterReplica"
registerOperation.Status = "Waiting"
registerOperation.TargetSE = target
targetSE = StorageElement( target )
for ftsFile in ftsFileList:
opFile = File()
opFile.LFN = ftsFile.LFN
pfn = returnSingleResult( targetSE.getURL( ftsFile.LFN, protocol = self.registrationProtocols ) )
if not pfn["OK"]:
continue
opFile.PFN = pfn["Value"]
registerOperation.addFile( opFile )
request.insertBefore( registerOperation, operation )
return S_OK()
def test_04Operations(self):
""" operations arithmetic and state machine """
req = Request()
self.assertEqual(len(req), 0)
transfer = Operation()
transfer.Type = "ReplicateAndRegister"
transfer.addFile(File({"LFN": "/a/b/c", "Status": "Waiting"}))
getWaiting = req.getWaiting()
self.assertEqual(getWaiting["OK"], True)
self.assertEqual(getWaiting["Value"], None)
req.addOperation(transfer)
self.assertEqual(len(req), 1)
self.assertEqual(transfer.Order, req.Order)
self.assertEqual(transfer.Status, "Waiting")
getWaiting = req.getWaiting()
self.assertEqual(getWaiting["OK"], True)
self.assertEqual(getWaiting["Value"], transfer)
removal = Operation({"Type": "RemoveFile"})
removal.addFile(File({"LFN": "/a/b/c", "Status": "Waiting"}))
req.insertBefore(removal, transfer)
getWaiting = req.getWaiting()
self.assertEqual(getWaiting["OK"], True)
self.assertEqual(getWaiting["Value"], removal)
self.assertEqual(len(req), 2)
self.assertEqual([op.Status for op in req], ["Waiting", "Queued"])
self.assertEqual(req.subStatusList(), ["Waiting", "Queued"])
self.assertEqual(removal.Order, 0)
self.assertEqual(removal.Order, req.Order)
self.assertEqual(transfer.Order, 1)
self.assertEqual(removal.Status, "Waiting")
self.assertEqual(transfer.Status, "Queued")
for subFile in removal:
subFile.Status = "Done"
removal.Status = "Done"
self.assertEqual(removal.Status, "Done")
self.assertEqual(transfer.Status, "Waiting")
self.assertEqual(transfer.Order, req.Order)
# # len, looping
self.assertEqual(len(req), 2)
self.assertEqual([op.Status for op in req], ["Done", "Waiting"])
self.assertEqual(req.subStatusList(), ["Done", "Waiting"])
digest = req.toJSON()
self.assertEqual(digest["OK"], True)
getWaiting = req.getWaiting()
self.assertEqual(getWaiting["OK"], True)
self.assertEqual(getWaiting["Value"], transfer)
try:
operation.SubmitTime = "foo"
except Exception, error:
self.assertEqual( type( error ), ValueError, "wrong exp raised" )
self.assertEqual( str( error ), "time data 'foo' does not match format '%Y-%m-%d %H:%M:%S'", "wrong exc reason" )
try:
operation.LastUpdate = "foo"
except Exception, error:
self.assertEqual( type( error ), ValueError, "wrong exc raised" )
self.assertEqual( str( error ), "time data 'foo' does not match format '%Y-%m-%d %H:%M:%S'", "wrong exc reason" )
# # Status
operation = Operation()
try:
operation.Status = "foo"
except Exception, error:
self.assertEqual( type( error ), ValueError, "wrong exc raised" )
self.assertEqual( str( error ), "unknown Status 'foo'", "wrong exc reason" )
operation.addFile( File( { "Status" : "Waiting", "LFN": "/a" } ) )
def test04StateMachine( self ):
""" state machine """
op = Operation()
self.assertEqual( op.Status, "Queued", "1. wrong status %s" % op.Status )
op.addFile( File( {"Status": "Waiting"} ) )
self.assertEqual( op.Status, "Queued", "2. wrong status %s" % op.Status )
def test04Operations( self ):
""" operations arithmetic and state machine """
req = Request()
self.assertEqual( len( req ), 0 )
transfer = Operation()
transfer.Type = "ReplicateAndRegister"
transfer.addFile( File( { "LFN" : "/a/b/c", "Status" : "Waiting" } ) )
getWaiting = req.getWaiting()
self.assertEqual( getWaiting["OK"], True )
self.assertEqual( getWaiting["Value"], None )
req.addOperation( transfer )
self.assertEqual( len( req ), 1 )
self.assertEqual( transfer.Order, req.Order )
self.assertEqual( transfer.Status, "Waiting" )
getWaiting = req.getWaiting()
self.assertEqual( getWaiting["OK"], True )
self.assertEqual( getWaiting["Value"], transfer )
removal = Operation( { "Type" : "RemoveFile" } )
removal.addFile( File( { "LFN" : "/a/b/c", "Status" : "Waiting" } ) )
req.insertBefore( removal, transfer )
getWaiting = req.getWaiting()
self.assertEqual( getWaiting["OK"], True )
self.assertEqual( getWaiting["Value"], removal )
self.assertEqual( len( req ), 2 )
self.assertEqual( [ op.Status for op in req ], ["Waiting", "Queued"] )
self.assertEqual( req.subStatusList() , ["Waiting", "Queued"] )
self.assertEqual( removal.Order, 0 )
self.assertEqual( removal.Order, req.Order )
self.assertEqual( transfer.Order, 1 )
self.assertEqual( removal.Status, "Waiting" )
self.assertEqual( transfer.Status, "Queued" )
for subFile in removal:
subFile.Status = "Done"
removal.Status = "Done"
self.assertEqual( removal.Status, "Done" )
self.assertEqual( transfer.Status, "Waiting" )
self.assertEqual( transfer.Order, req.Order )
# # len, looping
self.assertEqual( len( req ), 2 )
self.assertEqual( [ op.Status for op in req ], ["Done", "Waiting"] )
self.assertEqual( req.subStatusList() , ["Done", "Waiting"] )
digest = req.toJSON()
self.assertEqual( digest["OK"], True )
getWaiting = req.getWaiting()
self.assertEqual( getWaiting["OK"], True )
self.assertEqual( getWaiting["Value"], transfer )
"time data 'foo' does not match format '%Y-%m-%d %H:%M:%S'",
"wrong exc reason")
try:
operation.LastUpdate = "foo"
except Exception, error:
self.assertEqual(type(error), ValueError, "wrong exc raised")
self.assertEqual(
str(error),
"time data 'foo' does not match format '%Y-%m-%d %H:%M:%S'",
"wrong exc reason")
# # Status
operation = Operation()
try:
operation.Status = "foo"
except Exception, error:
self.assertEqual(type(error), ValueError, "wrong exc raised")
self.assertEqual(str(error), "unknown Status 'foo'",
"wrong exc reason")
operation.addFile(File({"Status": "Waiting", "LFN": "/a"}))
def test03sql(self):
""" sql insert or update """
operation = Operation()
operation.Type = "ReplicateAndRegister"
request = Request()
request.RequestName = "testRequest"
request.RequestID = 1
def test_Operations():
"""operations arithmetic and state machine"""
req = Request()
assert len(req) == 0
transfer = Operation()
transfer.Type = "ReplicateAndRegister"
transfer.addFile(File({"LFN": "/a/b/c", "Status": "Waiting"}))
getWaiting = req.getWaiting()
assert getWaiting["OK"]
assert getWaiting["Value"] is None
req.addOperation(transfer)
assert len(req) == 1
assert transfer.Order == req.Order
assert transfer.Status == "Waiting"
getWaiting = req.getWaiting()
assert getWaiting["OK"]
assert getWaiting["Value"] == transfer
removal = Operation({"Type": "RemoveFile"})
removal.addFile(File({"LFN": "/a/b/c", "Status": "Waiting"}))
req.insertBefore(removal, transfer)
getWaiting = req.getWaiting()
assert getWaiting["OK"]
assert getWaiting["Value"] == removal
assert len(req) == 2
assert [op.Status for op in req] == ["Waiting", "Queued"]
assert req.subStatusList() == ["Waiting", "Queued"]
assert removal.Order == 0
assert removal.Order == req.Order
assert transfer.Order == 1
assert removal.Status == "Waiting"
assert transfer.Status == "Queued"
for subFile in removal:
subFile.Status = "Done"
removal.Status = "Done"
assert removal.Status == "Done"
assert transfer.Status == "Waiting"
assert transfer.Order == req.Order
# len, looping
assert len(req) == 2
assert [op.Status for op in req] == ["Done", "Waiting"]
assert req.subStatusList() == ["Done", "Waiting"]
digest = req.toJSON()
assert digest["OK"]
getWaiting = req.getWaiting()
assert getWaiting["OK"]
assert getWaiting["Value"] == transfer
def test02props(self):
""" test properties """
# # valid values
operation = Operation()
operation.Arguments = "foobar"
self.assertEqual(operation.Arguments, "foobar", "wrong Arguments")
operation.SourceSE = "CERN-RAW"
self.assertEqual(operation.SourceSE, "CERN-RAW", "wrong SourceSE")
operation.TargetSE = "CERN-RAW"
self.assertEqual(operation.TargetSE, "CERN-RAW", "wrong TargetSE")
operation.Catalog = ""
self.assertEqual(operation.Catalog, "", "wrong Catalog")
operation.Catalog = "BookkeepingDB"
self.assertEqual(operation.Catalog, "BookkeepingDB", "wrong Catalog")
operation.Error = "error"
self.assertEqual(operation.Error, "error", "wrong Error")
# # wrong props
try:
operation.RequestID = "foo"
except Exception as error:
self.assertEqual(type(error), AttributeError, "wrong exc raised")
self.assertEqual(str(error), "can't set attribute",
"wrong exc reason")
try:
operation.OperationID = "foo"
except Exception as error:
self.assertEqual(type(error), ValueError, "wrong exc raised")
# # timestamps
try:
operation.SubmitTime = "foo"
except Exception as error:
self.assertEqual(type(error), ValueError, "wrong exp raised")
self.assertEqual(
str(error),
"time data 'foo' does not match format '%Y-%m-%d %H:%M:%S'",
"wrong exc reason")
try:
operation.LastUpdate = "foo"
except Exception as error:
self.assertEqual(type(error), ValueError, "wrong exc raised")
self.assertEqual(
str(error),
"time data 'foo' does not match format '%Y-%m-%d %H:%M:%S'",
"wrong exc reason")
# # Status
operation = Operation()
try:
operation.Status = "foo"
except Exception as error:
self.assertEqual(type(error), ValueError, "wrong exc raised")
self.assertEqual(str(error), "unknown Status 'foo'",
"wrong exc reason")
operation.addFile(File({"Status": "Waiting", "LFN": "/a"}))
