def test06Dirty( self ): """ dirty records """ db = RequestDB() r = Request() r.RequestName = "dirty" op1 = Operation( { "Type": "ReplicateAndRegister", "TargetSE": "CERN-USER"} ) op1 += File( {"LFN": "/a/b/c/1", "Status": "Scheduled", "Checksum": "123456", "ChecksumType": "ADLER32" } ) op2 = Operation( { "Type": "ReplicateAndRegister", "TargetSE": "CERN-USER"} ) op2 += File( {"LFN": "/a/b/c/2", "Status": "Scheduled", "Checksum": "123456", "ChecksumType": "ADLER32" } ) op3 = Operation( { "Type": "ReplicateAndRegister", "TargetSE": "CERN-USER"} ) op3 += File( {"LFN": "/a/b/c/3", "Status": "Scheduled", "Checksum": "123456", "ChecksumType": "ADLER32" } ) r += op1 r += op2 r += op3 put = db.putRequest( r ) self.assertEqual( put["OK"], True, "1. putRequest failed: %s" % put.get( "Message", "" ) ) reqID = put['Value'] r = db.getRequest( reqID ) self.assertEqual( r["OK"], True, "1. getRequest failed: %s" % r.get( "Message", "" ) ) r = r["Value"] del r[0] self.assertEqual( len( r ), 2, "1. len wrong" ) put = db.putRequest( r ) self.assertEqual( put["OK"], True, "2. putRequest failed: %s" % put.get( "Message", "" ) ) reqID = put['Value'] r = db.getRequest( reqID ) self.assertEqual( r["OK"], True, "2. getRequest failed: %s" % r.get( "Message", "" ) ) r = r["Value"] self.assertEqual( len( r ), 2, "2. len wrong" ) op4 = Operation( { "Type": "ReplicateAndRegister", "TargetSE": "CERN-USER"} ) op4 += File( {"LFN": "/a/b/c/4", "Status": "Scheduled", "Checksum": "123456", "ChecksumType": "ADLER32" } ) r[0] = op4 put = db.putRequest( r ) self.assertEqual( put["OK"], True, "3. putRequest failed: %s" % put.get( "Message", "" ) ) reqID = put['Value'] r = db.getRequest( reqID ) self.assertEqual( r["OK"], True, "3. getRequest failed: %s" % r.get( "Message", "" ) ) r = r["Value"] self.assertEqual( len( r ), 2, "3. len wrong" ) delete = db.deleteRequest( reqID ) self.assertEqual( delete["OK"], True, delete['Message'] if 'Message' in delete else 'OK' )
def test05Scheduled(self): """ scheduled request r/w """ db = RequestDB() req = Request({"RequestName": "FTSTest"}) op = Operation({ "Type": "ReplicateAndRegister", "TargetSE": "CERN-USER" }) op += File({ "LFN": "/a/b/c", "Status": "Scheduled", "Checksum": "123456", "ChecksumType": "ADLER32" }) req += op put = db.putRequest(req) self.assertEqual(put["OK"], True, "putRequest failed") peek = db.peekRequest(req.RequestName) self.assertEqual(peek["OK"], True, "peek failed ") peek = peek["Value"] for op in peek: opId = op.OperationID getFTS = db.getScheduledRequest(opId) self.assertEqual(getFTS["OK"], True, "getScheduled failed") self.assertEqual(getFTS["Value"].RequestName, "FTSTest", "wrong request selected")
def test05Scheduled(self): """ scheduled request r/w """ db = RequestDB() req = Request({"RequestName": "FTSTest"}) op = Operation({"Type": "ReplicateAndRegister", "TargetSE": "CERN-USER"}) op += File({"LFN": "/a/b/c", "Status": "Scheduled", "Checksum": "123456", "ChecksumType": "ADLER32"}) req += op put = db.putRequest(req) self.assertEqual(put["OK"], True, put['Message'] if 'Message' in put else 'OK') reqID = put['Value'] peek = db.peekRequest(reqID) self.assertEqual(peek["OK"], True, peek['Message'] if 'Message' in peek else 'OK') peek = peek["Value"] for op in peek: opId = op.OperationID getFTS = db.getScheduledRequest(opId) self.assertEqual(getFTS["OK"], True, "getScheduled failed") self.assertEqual(getFTS["Value"].RequestName, "FTSTest", "wrong request selected") delete = db.deleteRequest(reqID) self.assertEqual(delete["OK"], True, delete['Message'] if 'Message' in delete else 'OK')
def test04Stress(self): """ stress test """ db = RequestDB() reqIDs = [] for i in xrange(self.stressRequests): request = Request({"RequestName": "test-%d" % i}) op = Operation({"Type": "RemoveReplica", "TargetSE": "CERN-USER"}) op += File({"LFN": "/lhcb/user/c/cibak/foo"}) request += op put = db.putRequest(request) self.assertEqual(put["OK"], True, put['Message'] if 'Message' in put else 'OK') reqIDs.append(put['Value']) startTime = time.time() for reqID in reqIDs: get = db.getRequest(reqID) if "Message" in get: print get["Message"] self.assertEqual(get["OK"], True, get['Message'] if 'Message' in get else 'OK') endTime = time.time() print "getRequest duration %s " % (endTime - startTime) for reqID in reqIDs: delete = db.deleteRequest(reqID) self.assertEqual(delete["OK"], True, delete['Message'] if 'Message' in delete else 'OK')
def test01Stress(self): """ stress test """ db = RequestDB() reqIDs = [] for i in xrange(self.stressRequests): request = Request({"RequestName": "test-%d" % i}) op = Operation({"Type": "RemoveReplica", "TargetSE": "CERN-USER"}) op += File({"LFN": "/lhcb/user/c/cibak/foo"}) request += op put = db.putRequest(request) self.assertEqual(put["OK"], True, put['Message'] if 'Message' in put else 'OK') reqIDs.append(put['Value']) startTime = time.time() for reqID in reqIDs: get = db.getRequest(reqID) if "Message" in get: print(get["Message"]) self.assertEqual(get["OK"], True, get['Message'] if 'Message' in get else 'OK') endTime = time.time() print("getRequest duration %s " % (endTime - startTime)) for reqID in reqIDs: delete = db.deleteRequest(reqID) self.assertEqual( delete["OK"], True, delete['Message'] if 'Message' in delete else 'OK')
def test04StressBulk( self ): """ stress test bulk """ db = RequestDB() for i in range( self.stressRequests ): request = Request( { "RequestName": "test-%d" % i } ) op = Operation( { "Type": "RemoveReplica", "TargetSE": "CERN-USER" } ) op += File( { "LFN": "/lhcb/user/c/cibak/foo" } ) request += op put = db.putRequest( request ) self.assertEqual( put["OK"], True, "put failed" ) loops = self.stressRequests // self.bulkRequest + ( 1 if ( self.stressRequests % self.bulkRequest ) else 0 ) totalSuccessful = 0 startTime = time.time() for i in range( loops ): get = db.getRequests( self.bulkRequest, True ) if "Message" in get: print get["Message"] self.assertEqual( get["OK"], True, "get failed" ) totalSuccessful += len( get["Value"] ) endTime = time.time() print "getRequests duration %s " % ( endTime - startTime ) self.assertEqual( totalSuccessful, self.stressRequests, "Did not retrieve all the requests: %s instead of %s" % ( totalSuccessful, self.stressRequests ) ) for i in range( self.stressRequests ): delete = db.deleteRequest( "test-%s" % i ) self.assertEqual( delete["OK"], True, "delete failed" )
def test04Stress( self ): """ stress test """ db = RequestDB() for i in range( self.stressRequests ): request = Request( { "RequestName": "test-%d" % i } ) op = Operation( { "Type": "RemoveReplica", "TargetSE": "CERN-USER" } ) op += File( { "LFN": "/lhcb/user/c/cibak/foo" } ) request += op put = db.putRequest( request ) self.assertEqual( put["OK"], True, "put failed" ) startTime = time.time() for i in range( self.stressRequests ): get = db.getRequest( "test-%s" % i, True ) if "Message" in get: print get["Message"] self.assertEqual( get["OK"], True, "get failed" ) endTime = time.time() print "getRequest duration %s " % ( endTime - startTime ) for i in range( self.stressRequests ): delete = db.deleteRequest( "test-%s" % i ) self.assertEqual( delete["OK"], True, "delete failed" )
def test04Stress(self): """ stress test """ db = RequestDB() for i in range(self.stressRequests): request = Request({"RequestName": "test-%d" % i}) op = Operation({"Type": "RemoveReplica", "TargetSE": "CERN-USER"}) op += File({"LFN": "/lhcb/user/c/cibak/foo"}) request += op put = db.putRequest(request) self.assertEqual(put["OK"], True, "put failed") startTime = time.time() for i in range(self.stressRequests): get = db.getRequest("test-%s" % i, True) if "Message" in get: print get["Message"] self.assertEqual(get["OK"], True, "get failed") endTime = time.time() print "getRequest duration %s " % (endTime - startTime) for i in range(self.stressRequests): delete = db.deleteRequest("test-%s" % i) self.assertEqual(delete["OK"], True, "delete failed")
def test01StressBulk(self): """ stress test bulk """ db = RequestDB() reqIDs = [] for i in xrange(self.stressRequests): request = Request({"RequestName": "test-%d" % i}) op = Operation({"Type": "RemoveReplica", "TargetSE": "CERN-USER"}) op += File({"LFN": "/lhcb/user/c/cibak/foo"}) request += op put = db.putRequest(request) self.assertEqual(put["OK"], True) reqIDs.append(put['Value']) loops = self.stressRequests // self.bulkRequest + \ (1 if (self.stressRequests % self.bulkRequest) else 0) totalSuccessful = 0 time.sleep(1) startTime = time.time() for i in xrange(loops): get = db.getBulkRequests(self.bulkRequest, True) if "Message" in get: print(get["Message"]) self.assertEqual(get["OK"], True, "get failed") totalSuccessful += len(get["Value"]) endTime = time.time() print("getRequests duration %s " % (endTime - startTime)) self.assertEqual( totalSuccessful, self.stressRequests, "Did not retrieve all the requests: %s instead of %s" % (totalSuccessful, self.stressRequests)) for reqID in reqIDs: delete = db.deleteRequest(reqID) self.assertEqual( delete["OK"], True, delete['Message'] if 'Message' in delete else 'OK')
def test02Scheduled(self): """ scheduled request r/w """ db = RequestDB() req = Request({"RequestName": "FTSTest"}) op = Operation({ "Type": "ReplicateAndRegister", "TargetSE": "CERN-USER" }) op += File({ "LFN": "/a/b/c", "Status": "Scheduled", "Checksum": "123456", "ChecksumType": "ADLER32" }) req += op put = db.putRequest(req) self.assertEqual(put["OK"], True, put['Message'] if 'Message' in put else 'OK') reqID = put['Value'] peek = db.peekRequest(reqID) self.assertEqual(peek["OK"], True, peek['Message'] if 'Message' in peek else 'OK') peek = peek["Value"] for op in peek: opId = op.OperationID getFTS = db.getScheduledRequest(opId) self.assertEqual(getFTS["OK"], True, "getScheduled failed") self.assertEqual(getFTS["Value"].RequestName, "FTSTest", "wrong request selected") delete = db.deleteRequest(reqID) self.assertEqual(delete["OK"], True, delete['Message'] if 'Message' in delete else 'OK')
def test03RequestRW(self): """ db r/w requests """ db = RequestDB() db._checkTables(True) # # empty DB at that stage ret = db.getDBSummary() self.assertEqual(ret, {"OK": True, "Value": {"Operation": {}, "Request": {}, "File": {}}}) # # insert ret = db.putRequest(self.request) self.assertEqual(ret, {"OK": True, "Value": ""}) # # get digest -> JSON ret = db.getDigest(self.request.RequestName) self.assertEqual(ret["OK"], True) self.assertEqual(bool(ret["Value"]), True) # # db summary ret = db.getDBSummary() self.assertEqual( ret, { "OK": True, "Value": { "Operation": {"RemoveFile": {"Queued": 1L}, "ReplicateAndRegister": {"Waiting": 1L}}, "Request": {"Waiting": 1L}, "File": {"Waiting": 2L}, }, }, ) # # get request for jobs ret = db.getRequestNamesForJobs([1]) self.assertEqual(ret["OK"], True) self.assertEqual(ret["Value"], {1: "test1"}) # # read requests ret = db.readRequestsForJobs([1]) self.assertEqual(ret["OK"], True) self.assertEqual(ret["Value"][1]["OK"], True) # # select ret = db.getRequest() self.assertEqual(ret["OK"], True) request = ret["Value"] self.assertEqual(isinstance(request, Request), True) # # summary ret = db.getDBSummary() self.assertEqual( ret, { "OK": True, "Value": { "Operation": {"RemoveFile": {"Queued": 1L}, "ReplicateAndRegister": {"Waiting": 1L}}, "Request": {"Assigned": 1L}, "File": {"Waiting": 2L}, }, }, ) # # update ret = db.putRequest(request) self.assertEqual(ret, {"OK": True, "Value": ""}) # # get summary again ret = db.getDBSummary() self.assertEqual( ret, { "OK": True, "Value": { "Operation": {"RemoveFile": {"Queued": 1L}, "ReplicateAndRegister": {"Waiting": 1L}}, "Request": {"Waiting": 1L}, "File": {"Waiting": 2L}, }, }, ) # # delete ret = db.deleteRequest(self.request.RequestName) self.assertEqual(ret, {"OK": True, "Value": ""}) # # should be empty now ret = db.getDBSummary() self.assertEqual(ret, {"OK": True, "Value": {"Operation": {}, "Request": {}, "File": {}}})
def test03RequestRW(self): """ db r/w requests """ db = RequestDB() db._checkTables(True) # # empty DB at that stage ret = db.getDBSummary() self.assertEqual(ret, { 'OK': True, 'Value': { 'Operation': {}, 'Request': {}, 'File': {} } }) # # insert ret = db.putRequest(self.request) self.assertEqual(ret, {'OK': True, 'Value': ''}) # # get digest -> JSON ret = db.getDigest(self.request.RequestName) self.assertEqual(ret["OK"], True) self.assertEqual(bool(ret["Value"]), True) # # db summary ret = db.getDBSummary() self.assertEqual( ret, { 'OK': True, 'Value': { 'Operation': { 'RemoveFile': { 'Queued': 1L }, 'ReplicateAndRegister': { 'Waiting': 1L } }, 'Request': { 'Waiting': 1L }, 'File': { 'Waiting': 2L } } }) # # get request for jobs ret = db.getRequestNamesForJobs([1]) self.assertEqual(ret["OK"], True) self.assertEqual(ret["Value"], {1: 'test1'}) # # read requests ret = db.readRequestsForJobs([1]) self.assertEqual(ret["OK"], True) self.assertEqual(ret["Value"][1]["OK"], True) # # select ret = db.getRequest() self.assertEqual(ret["OK"], True) request = ret["Value"] self.assertEqual(isinstance(request, Request), True) # # summary ret = db.getDBSummary() self.assertEqual( ret, { 'OK': True, 'Value': { 'Operation': { 'RemoveFile': { 'Queued': 1L }, 'ReplicateAndRegister': { 'Waiting': 1L } }, 'Request': { 'Assigned': 1L }, 'File': { 'Waiting': 2L } } }) # # update ret = db.putRequest(request) self.assertEqual(ret, {'OK': True, 'Value': ''}) # # get summary again ret = db.getDBSummary() self.assertEqual( ret, { 'OK': True, 'Value': { 'Operation': { 'RemoveFile': { 'Queued': 1L }, 'ReplicateAndRegister': { 'Waiting': 1L } }, 'Request': { 'Waiting': 1L }, 'File': { 'Waiting': 2L } } }) # # delete ret = db.deleteRequest(self.request.RequestName) self.assertEqual(ret, {'OK': True, 'Value': ''}) # # should be empty now ret = db.getDBSummary() self.assertEqual(ret, { 'OK': True, 'Value': { 'Operation': {}, 'Request': {}, 'File': {} } })