def __insertRegisterOperation( self, request, operation, toRegister ): """ add RegisterReplica operation :param Request request: request instance :param Operation transferOp: 'ReplicateAndRegister' operation for this FTSJob :param list toRegister: [ FTSDB.FTSFile, ... ] - files that failed to register """ log = self.log.getSubLogger( "req_%s/%s/registerFiles" % ( request.RequestID, request.RequestName ) ) byTarget = {} for ftsFile in toRegister: if ftsFile.TargetSE not in byTarget: byTarget.setdefault( ftsFile.TargetSE, [] ) byTarget[ftsFile.TargetSE].append( ftsFile ) log.info( "will create %s 'RegisterReplica' operations" % len( byTarget ) ) for target, ftsFileList in byTarget.iteritems(): log.info( "creating 'RegisterReplica' operation for targetSE %s with %s files..." % ( target, len( ftsFileList ) ) ) registerOperation = Operation() registerOperation.Type = "RegisterReplica" registerOperation.Status = "Waiting" registerOperation.TargetSE = target targetSE = StorageElement( target ) for ftsFile in ftsFileList: opFile = File() opFile.LFN = ftsFile.LFN pfn = returnSingleResult( targetSE.getURL( ftsFile.LFN, protocol = self.registrationProtocols ) ) if not pfn["OK"]: continue opFile.PFN = pfn["Value"] registerOperation.addFile( opFile ) request.insertBefore( registerOperation, operation ) return S_OK()
def __insertRegisterOperation( self, request, operation, toRegister ): """ add RegisterReplica operation :param Request request: request instance :param Operation transferOp: 'ReplicateAndRegister' operation for this FTSJob :param list toRegister: [ FTSDB.FTSFile, ... ] - files that failed to register """ log = self.log.getSubLogger( "req_%s/%s/registerFiles" % ( request.RequestID, request.RequestName ) ) byTarget = {} for ftsFile in toRegister: if ftsFile.TargetSE not in byTarget: byTarget.setdefault( ftsFile.TargetSE, [] ) byTarget[ftsFile.TargetSE].append( ftsFile ) log.info( "will create %s 'RegisterReplica' operations" % len( byTarget ) ) for target, ftsFileList in byTarget.iteritems(): log.info( "creating 'RegisterReplica' operation for targetSE %s with %s files..." % ( target, len( ftsFileList ) ) ) registerOperation = Operation() registerOperation.Type = "RegisterReplica" registerOperation.Status = "Waiting" registerOperation.TargetSE = target targetSE = StorageElement( target ) for ftsFile in ftsFileList: opFile = File() opFile.LFN = ftsFile.LFN pfn = returnSingleResult( targetSE.getURL( ftsFile.LFN, protocol = self.registrationProtocols ) ) if not pfn["OK"]: continue opFile.PFN = pfn["Value"] registerOperation.addFile( opFile ) request.insertBefore( registerOperation, operation ) return S_OK()
def test_04Operations(self): """ operations arithmetic and state machine """ req = Request() self.assertEqual(len(req), 0) transfer = Operation() transfer.Type = "ReplicateAndRegister" transfer.addFile(File({"LFN": "/a/b/c", "Status": "Waiting"})) getWaiting = req.getWaiting() self.assertEqual(getWaiting["OK"], True) self.assertEqual(getWaiting["Value"], None) req.addOperation(transfer) self.assertEqual(len(req), 1) self.assertEqual(transfer.Order, req.Order) self.assertEqual(transfer.Status, "Waiting") getWaiting = req.getWaiting() self.assertEqual(getWaiting["OK"], True) self.assertEqual(getWaiting["Value"], transfer) removal = Operation({"Type": "RemoveFile"}) removal.addFile(File({"LFN": "/a/b/c", "Status": "Waiting"})) req.insertBefore(removal, transfer) getWaiting = req.getWaiting() self.assertEqual(getWaiting["OK"], True) self.assertEqual(getWaiting["Value"], removal) self.assertEqual(len(req), 2) self.assertEqual([op.Status for op in req], ["Waiting", "Queued"]) self.assertEqual(req.subStatusList(), ["Waiting", "Queued"]) self.assertEqual(removal.Order, 0) self.assertEqual(removal.Order, req.Order) self.assertEqual(transfer.Order, 1) self.assertEqual(removal.Status, "Waiting") self.assertEqual(transfer.Status, "Queued") for subFile in removal: subFile.Status = "Done" removal.Status = "Done" self.assertEqual(removal.Status, "Done") self.assertEqual(transfer.Status, "Waiting") self.assertEqual(transfer.Order, req.Order) # # len, looping self.assertEqual(len(req), 2) self.assertEqual([op.Status for op in req], ["Done", "Waiting"]) self.assertEqual(req.subStatusList(), ["Done", "Waiting"]) digest = req.toJSON() self.assertEqual(digest["OK"], True) getWaiting = req.getWaiting() self.assertEqual(getWaiting["OK"], True) self.assertEqual(getWaiting["Value"], transfer)
try: operation.SubmitTime = "foo" except Exception, error: self.assertEqual( type( error ), ValueError, "wrong exp raised" ) self.assertEqual( str( error ), "time data 'foo' does not match format '%Y-%m-%d %H:%M:%S'", "wrong exc reason" ) try: operation.LastUpdate = "foo" except Exception, error: self.assertEqual( type( error ), ValueError, "wrong exc raised" ) self.assertEqual( str( error ), "time data 'foo' does not match format '%Y-%m-%d %H:%M:%S'", "wrong exc reason" ) # # Status operation = Operation() try: operation.Status = "foo" except Exception, error: self.assertEqual( type( error ), ValueError, "wrong exc raised" ) self.assertEqual( str( error ), "unknown Status 'foo'", "wrong exc reason" ) operation.addFile( File( { "Status" : "Waiting", "LFN": "/a" } ) ) def test04StateMachine( self ): """ state machine """ op = Operation() self.assertEqual( op.Status, "Queued", "1. wrong status %s" % op.Status ) op.addFile( File( {"Status": "Waiting"} ) ) self.assertEqual( op.Status, "Queued", "2. wrong status %s" % op.Status )
def test04Operations( self ): """ operations arithmetic and state machine """ req = Request() self.assertEqual( len( req ), 0 ) transfer = Operation() transfer.Type = "ReplicateAndRegister" transfer.addFile( File( { "LFN" : "/a/b/c", "Status" : "Waiting" } ) ) getWaiting = req.getWaiting() self.assertEqual( getWaiting["OK"], True ) self.assertEqual( getWaiting["Value"], None ) req.addOperation( transfer ) self.assertEqual( len( req ), 1 ) self.assertEqual( transfer.Order, req.Order ) self.assertEqual( transfer.Status, "Waiting" ) getWaiting = req.getWaiting() self.assertEqual( getWaiting["OK"], True ) self.assertEqual( getWaiting["Value"], transfer ) removal = Operation( { "Type" : "RemoveFile" } ) removal.addFile( File( { "LFN" : "/a/b/c", "Status" : "Waiting" } ) ) req.insertBefore( removal, transfer ) getWaiting = req.getWaiting() self.assertEqual( getWaiting["OK"], True ) self.assertEqual( getWaiting["Value"], removal ) self.assertEqual( len( req ), 2 ) self.assertEqual( [ op.Status for op in req ], ["Waiting", "Queued"] ) self.assertEqual( req.subStatusList() , ["Waiting", "Queued"] ) self.assertEqual( removal.Order, 0 ) self.assertEqual( removal.Order, req.Order ) self.assertEqual( transfer.Order, 1 ) self.assertEqual( removal.Status, "Waiting" ) self.assertEqual( transfer.Status, "Queued" ) for subFile in removal: subFile.Status = "Done" removal.Status = "Done" self.assertEqual( removal.Status, "Done" ) self.assertEqual( transfer.Status, "Waiting" ) self.assertEqual( transfer.Order, req.Order ) # # len, looping self.assertEqual( len( req ), 2 ) self.assertEqual( [ op.Status for op in req ], ["Done", "Waiting"] ) self.assertEqual( req.subStatusList() , ["Done", "Waiting"] ) digest = req.toJSON() self.assertEqual( digest["OK"], True ) getWaiting = req.getWaiting() self.assertEqual( getWaiting["OK"], True ) self.assertEqual( getWaiting["Value"], transfer )
"time data 'foo' does not match format '%Y-%m-%d %H:%M:%S'", "wrong exc reason") try: operation.LastUpdate = "foo" except Exception, error: self.assertEqual(type(error), ValueError, "wrong exc raised") self.assertEqual( str(error), "time data 'foo' does not match format '%Y-%m-%d %H:%M:%S'", "wrong exc reason") # # Status operation = Operation() try: operation.Status = "foo" except Exception, error: self.assertEqual(type(error), ValueError, "wrong exc raised") self.assertEqual(str(error), "unknown Status 'foo'", "wrong exc reason") operation.addFile(File({"Status": "Waiting", "LFN": "/a"})) def test03sql(self): """ sql insert or update """ operation = Operation() operation.Type = "ReplicateAndRegister" request = Request() request.RequestName = "testRequest" request.RequestID = 1
def test_Operations(): """operations arithmetic and state machine""" req = Request() assert len(req) == 0 transfer = Operation() transfer.Type = "ReplicateAndRegister" transfer.addFile(File({"LFN": "/a/b/c", "Status": "Waiting"})) getWaiting = req.getWaiting() assert getWaiting["OK"] assert getWaiting["Value"] is None req.addOperation(transfer) assert len(req) == 1 assert transfer.Order == req.Order assert transfer.Status == "Waiting" getWaiting = req.getWaiting() assert getWaiting["OK"] assert getWaiting["Value"] == transfer removal = Operation({"Type": "RemoveFile"}) removal.addFile(File({"LFN": "/a/b/c", "Status": "Waiting"})) req.insertBefore(removal, transfer) getWaiting = req.getWaiting() assert getWaiting["OK"] assert getWaiting["Value"] == removal assert len(req) == 2 assert [op.Status for op in req] == ["Waiting", "Queued"] assert req.subStatusList() == ["Waiting", "Queued"] assert removal.Order == 0 assert removal.Order == req.Order assert transfer.Order == 1 assert removal.Status == "Waiting" assert transfer.Status == "Queued" for subFile in removal: subFile.Status = "Done" removal.Status = "Done" assert removal.Status == "Done" assert transfer.Status == "Waiting" assert transfer.Order == req.Order # len, looping assert len(req) == 2 assert [op.Status for op in req] == ["Done", "Waiting"] assert req.subStatusList() == ["Done", "Waiting"] digest = req.toJSON() assert digest["OK"] getWaiting = req.getWaiting() assert getWaiting["OK"] assert getWaiting["Value"] == transfer
def test02props(self): """ test properties """ # # valid values operation = Operation() operation.Arguments = "foobar" self.assertEqual(operation.Arguments, "foobar", "wrong Arguments") operation.SourceSE = "CERN-RAW" self.assertEqual(operation.SourceSE, "CERN-RAW", "wrong SourceSE") operation.TargetSE = "CERN-RAW" self.assertEqual(operation.TargetSE, "CERN-RAW", "wrong TargetSE") operation.Catalog = "" self.assertEqual(operation.Catalog, "", "wrong Catalog") operation.Catalog = "BookkeepingDB" self.assertEqual(operation.Catalog, "BookkeepingDB", "wrong Catalog") operation.Error = "error" self.assertEqual(operation.Error, "error", "wrong Error") # # wrong props try: operation.RequestID = "foo" except Exception as error: self.assertEqual(type(error), AttributeError, "wrong exc raised") self.assertEqual(str(error), "can't set attribute", "wrong exc reason") try: operation.OperationID = "foo" except Exception as error: self.assertEqual(type(error), ValueError, "wrong exc raised") # # timestamps try: operation.SubmitTime = "foo" except Exception as error: self.assertEqual(type(error), ValueError, "wrong exp raised") self.assertEqual( str(error), "time data 'foo' does not match format '%Y-%m-%d %H:%M:%S'", "wrong exc reason") try: operation.LastUpdate = "foo" except Exception as error: self.assertEqual(type(error), ValueError, "wrong exc raised") self.assertEqual( str(error), "time data 'foo' does not match format '%Y-%m-%d %H:%M:%S'", "wrong exc reason") # # Status operation = Operation() try: operation.Status = "foo" except Exception as error: self.assertEqual(type(error), ValueError, "wrong exc raised") self.assertEqual(str(error), "unknown Status 'foo'", "wrong exc reason") operation.addFile(File({"Status": "Waiting", "LFN": "/a"}))