def test05Scheduled(self):
""" scheduled request r/w """
db = RequestDB()
req = Request({"RequestName": "FTSTest"})
op = Operation({"Type": "ReplicateAndRegister", "TargetSE": "CERN-USER"})
op += File({"LFN": "/a/b/c", "Status": "Scheduled",
"Checksum": "123456", "ChecksumType": "ADLER32"})
req += op
put = db.putRequest(req)
self.assertEqual(put["OK"], True, put['Message'] if 'Message' in put else 'OK')
reqID = put['Value']
peek = db.peekRequest(reqID)
self.assertEqual(peek["OK"], True, peek['Message'] if 'Message' in peek else 'OK')
peek = peek["Value"]
for op in peek:
opId = op.OperationID
getFTS = db.getScheduledRequest(opId)
self.assertEqual(getFTS["OK"], True, "getScheduled failed")
self.assertEqual(getFTS["Value"].RequestName, "FTSTest", "wrong request selected")
delete = db.deleteRequest(reqID)
self.assertEqual(delete["OK"], True, delete['Message'] if 'Message' in delete else 'OK')
def test04StressBulk( self ):
""" stress test bulk """
db = RequestDB()
for i in range( self.stressRequests ):
request = Request( { "RequestName": "test-%d" % i } )
op = Operation( { "Type": "RemoveReplica", "TargetSE": "CERN-USER" } )
op += File( { "LFN": "/lhcb/user/c/cibak/foo" } )
request += op
put = db.putRequest( request )
self.assertEqual( put["OK"], True, "put failed" )
loops = self.stressRequests // self.bulkRequest + ( 1 if ( self.stressRequests % self.bulkRequest ) else 0 )
totalSuccessful = 0
startTime = time.time()
for i in range( loops ):
get = db.getRequests( self.bulkRequest, True )
if "Message" in get:
print get["Message"]
self.assertEqual( get["OK"], True, "get failed" )
totalSuccessful += len( get["Value"] )
endTime = time.time()
print "getRequests duration %s " % ( endTime - startTime )
self.assertEqual( totalSuccessful, self.stressRequests, "Did not retrieve all the requests: %s instead of %s" % ( totalSuccessful, self.stressRequests ) )
for i in range( self.stressRequests ):
delete = db.deleteRequest( "test-%s" % i )
self.assertEqual( delete["OK"], True, "delete failed" )
def test04Stress(self):
""" stress test """
db = RequestDB()
reqIDs = []
for i in xrange(self.stressRequests):
request = Request({"RequestName": "test-%d" % i})
op = Operation({"Type": "RemoveReplica", "TargetSE": "CERN-USER"})
op += File({"LFN": "/lhcb/user/c/cibak/foo"})
request += op
put = db.putRequest(request)
self.assertEqual(put["OK"], True, put['Message'] if 'Message' in put else 'OK')
reqIDs.append(put['Value'])
startTime = time.time()
for reqID in reqIDs:
get = db.getRequest(reqID)
if "Message" in get:
print get["Message"]
self.assertEqual(get["OK"], True, get['Message'] if 'Message' in get else 'OK')
endTime = time.time()
print "getRequest duration %s " % (endTime - startTime)
for reqID in reqIDs:
delete = db.deleteRequest(reqID)
self.assertEqual(delete["OK"], True, delete['Message'] if 'Message' in delete else 'OK')
def test01Stress(self):
""" stress test """
db = RequestDB()
reqIDs = []
for i in xrange(self.stressRequests):
request = Request({"RequestName": "test-%d" % i})
op = Operation({"Type": "RemoveReplica", "TargetSE": "CERN-USER"})
op += File({"LFN": "/lhcb/user/c/cibak/foo"})
request += op
put = db.putRequest(request)
self.assertEqual(put["OK"], True,
put['Message'] if 'Message' in put else 'OK')
reqIDs.append(put['Value'])
startTime = time.time()
for reqID in reqIDs:
get = db.getRequest(reqID)
if "Message" in get:
print(get["Message"])
self.assertEqual(get["OK"], True,
get['Message'] if 'Message' in get else 'OK')
endTime = time.time()
print("getRequest duration %s " % (endTime - startTime))
for reqID in reqIDs:
delete = db.deleteRequest(reqID)
self.assertEqual(
delete["OK"], True,
delete['Message'] if 'Message' in delete else 'OK')
def test04Stress( self ):
""" stress test """
db = RequestDB()
for i in range( self.stressRequests ):
request = Request( { "RequestName": "test-%d" % i } )
op = Operation( { "Type": "RemoveReplica", "TargetSE": "CERN-USER" } )
op += File( { "LFN": "/lhcb/user/c/cibak/foo" } )
request += op
put = db.putRequest( request )
self.assertEqual( put["OK"], True, "put failed" )
startTime = time.time()
for i in range( self.stressRequests ):
get = db.getRequest( "test-%s" % i, True )
if "Message" in get:
print get["Message"]
self.assertEqual( get["OK"], True, "get failed" )
endTime = time.time()
print "getRequest duration %s " % ( endTime - startTime )
for i in range( self.stressRequests ):
delete = db.deleteRequest( "test-%s" % i )
self.assertEqual( delete["OK"], True, "delete failed" )
def test04Stress(self):
""" stress test """
db = RequestDB()
for i in range(self.stressRequests):
request = Request({"RequestName": "test-%d" % i})
op = Operation({"Type": "RemoveReplica", "TargetSE": "CERN-USER"})
op += File({"LFN": "/lhcb/user/c/cibak/foo"})
request += op
put = db.putRequest(request)
self.assertEqual(put["OK"], True, "put failed")
startTime = time.time()
for i in range(self.stressRequests):
get = db.getRequest("test-%s" % i, True)
if "Message" in get:
print get["Message"]
self.assertEqual(get["OK"], True, "get failed")
endTime = time.time()
print "getRequest duration %s " % (endTime - startTime)
for i in range(self.stressRequests):
delete = db.deleteRequest("test-%s" % i)
self.assertEqual(delete["OK"], True, "delete failed")
def test05Scheduled( self ):
""" scheduled request r/w """
db = RequestDB()
req = Request( {"RequestName": "FTSTest"} )
op = Operation( { "Type": "ReplicateAndRegister", "TargetSE": "CERN-USER"} )
op += File( {"LFN": "/a/b/c", "Status": "Scheduled", "Checksum": "123456", "ChecksumType": "ADLER32" } )
req += op
put = db.putRequest( req )
self.assertEqual( put["OK"], True, put['Message'] if 'Message' in put else 'OK' )
reqID = put['Value']
peek = db.peekRequest( reqID )
self.assertEqual( peek["OK"], True, peek['Message'] if 'Message' in peek else 'OK' )
peek = peek["Value"]
for op in peek:
opId = op.OperationID
getFTS = db.getScheduledRequest( opId )
self.assertEqual( getFTS["OK"], True, "getScheduled failed" )
self.assertEqual( getFTS["Value"].RequestName, "FTSTest", "wrong request selected" )
delete = db.deleteRequest( reqID )
self.assertEqual( delete["OK"], True, delete['Message'] if 'Message' in delete else 'OK' )
def test06Dirty( self ):
""" dirty records """
db = RequestDB()
r = Request()
r.RequestName = "dirty"
op1 = Operation( { "Type": "ReplicateAndRegister", "TargetSE": "CERN-USER"} )
op1 += File( {"LFN": "/a/b/c/1", "Status": "Scheduled", "Checksum": "123456", "ChecksumType": "ADLER32" } )
op2 = Operation( { "Type": "ReplicateAndRegister", "TargetSE": "CERN-USER"} )
op2 += File( {"LFN": "/a/b/c/2", "Status": "Scheduled", "Checksum": "123456", "ChecksumType": "ADLER32" } )
op3 = Operation( { "Type": "ReplicateAndRegister", "TargetSE": "CERN-USER"} )
op3 += File( {"LFN": "/a/b/c/3", "Status": "Scheduled", "Checksum": "123456", "ChecksumType": "ADLER32" } )
r += op1
r += op2
r += op3
put = db.putRequest( r )
self.assertEqual( put["OK"], True, "1. putRequest failed: %s" % put.get( "Message", "" ) )
reqID = put['Value']
r = db.getRequest( reqID )
self.assertEqual( r["OK"], True, "1. getRequest failed: %s" % r.get( "Message", "" ) )
r = r["Value"]
del r[0]
self.assertEqual( len( r ), 2, "1. len wrong" )
put = db.putRequest( r )
self.assertEqual( put["OK"], True, "2. putRequest failed: %s" % put.get( "Message", "" ) )
reqID = put['Value']
r = db.getRequest( reqID )
self.assertEqual( r["OK"], True, "2. getRequest failed: %s" % r.get( "Message", "" ) )
r = r["Value"]
self.assertEqual( len( r ), 2, "2. len wrong" )
op4 = Operation( { "Type": "ReplicateAndRegister", "TargetSE": "CERN-USER"} )
op4 += File( {"LFN": "/a/b/c/4", "Status": "Scheduled", "Checksum": "123456", "ChecksumType": "ADLER32" } )
r[0] = op4
put = db.putRequest( r )
self.assertEqual( put["OK"], True, "3. putRequest failed: %s" % put.get( "Message", "" ) )
reqID = put['Value']
r = db.getRequest( reqID )
self.assertEqual( r["OK"], True, "3. getRequest failed: %s" % r.get( "Message", "" ) )
r = r["Value"]
self.assertEqual( len( r ), 2, "3. len wrong" )
delete = db.deleteRequest( reqID )
self.assertEqual( delete["OK"], True, delete['Message'] if 'Message' in delete else 'OK' )
def test06Dirty( self ):
""" dirty records """
db = RequestDB()
r = Request()
r.RequestName = "dirty"
op1 = Operation( { "Type": "ReplicateAndRegister", "TargetSE": "CERN-USER"} )
op1 += File( {"LFN": "/a/b/c/1", "Status": "Scheduled", "Checksum": "123456", "ChecksumType": "ADLER32" } )
op2 = Operation( { "Type": "ReplicateAndRegister", "TargetSE": "CERN-USER"} )
op2 += File( {"LFN": "/a/b/c/2", "Status": "Scheduled", "Checksum": "123456", "ChecksumType": "ADLER32" } )
op3 = Operation( { "Type": "ReplicateAndRegister", "TargetSE": "CERN-USER"} )
op3 += File( {"LFN": "/a/b/c/3", "Status": "Scheduled", "Checksum": "123456", "ChecksumType": "ADLER32" } )
r += op1
r += op2
r += op3
put = db.putRequest( r )
self.assertEqual( put["OK"], True, "1. putRequest failed: %s" % put.get( "Message", "" ) )
reqID = put['Value']
r = db.getRequest( reqID )
self.assertEqual( r["OK"], True, "1. getRequest failed: %s" % r.get( "Message", "" ) )
r = r["Value"]
del r[0]
self.assertEqual( len( r ), 2, "1. len wrong" )
put = db.putRequest( r )
self.assertEqual( put["OK"], True, "2. putRequest failed: %s" % put.get( "Message", "" ) )
reqID = put['Value']
r = db.getRequest( reqID )
self.assertEqual( r["OK"], True, "2. getRequest failed: %s" % r.get( "Message", "" ) )
r = r["Value"]
self.assertEqual( len( r ), 2, "2. len wrong" )
op4 = Operation( { "Type": "ReplicateAndRegister", "TargetSE": "CERN-USER"} )
op4 += File( {"LFN": "/a/b/c/4", "Status": "Scheduled", "Checksum": "123456", "ChecksumType": "ADLER32" } )
r[0] = op4
put = db.putRequest( r )
self.assertEqual( put["OK"], True, "3. putRequest failed: %s" % put.get( "Message", "" ) )
reqID = put['Value']
r = db.getRequest( reqID )
self.assertEqual( r["OK"], True, "3. getRequest failed: %s" % r.get( "Message", "" ) )
r = r["Value"]
self.assertEqual( len( r ), 2, "3. len wrong" )
delete = db.deleteRequest( reqID )
self.assertEqual( delete["OK"], True, delete['Message'] if 'Message' in delete else 'OK' )
def test01StressBulk(self):
""" stress test bulk """
db = RequestDB()
reqIDs = []
for i in xrange(self.stressRequests):
request = Request({"RequestName": "test-%d" % i})
op = Operation({"Type": "RemoveReplica", "TargetSE": "CERN-USER"})
op += File({"LFN": "/lhcb/user/c/cibak/foo"})
request += op
put = db.putRequest(request)
self.assertEqual(put["OK"], True)
reqIDs.append(put['Value'])
loops = self.stressRequests // self.bulkRequest + \
(1 if (self.stressRequests % self.bulkRequest) else 0)
totalSuccessful = 0
time.sleep(1)
startTime = time.time()
for i in xrange(loops):
get = db.getBulkRequests(self.bulkRequest, True)
if "Message" in get:
print(get["Message"])
self.assertEqual(get["OK"], True, "get failed")
totalSuccessful += len(get["Value"])
endTime = time.time()
print("getRequests duration %s " % (endTime - startTime))
self.assertEqual(
totalSuccessful, self.stressRequests,
"Did not retrieve all the requests: %s instead of %s" %
(totalSuccessful, self.stressRequests))
for reqID in reqIDs:
delete = db.deleteRequest(reqID)
self.assertEqual(
delete["OK"], True,
delete['Message'] if 'Message' in delete else 'OK')
def test03RequestRW(self):
""" db r/w requests """
db = RequestDB()
db._checkTables(True)
# # empty DB at that stage
ret = db.getDBSummary()
self.assertEqual(ret, {"OK": True, "Value": {"Operation": {}, "Request": {}, "File": {}}})
# # insert
ret = db.putRequest(self.request)
self.assertEqual(ret, {"OK": True, "Value": ""})
# # get digest -> JSON
ret = db.getDigest(self.request.RequestName)
self.assertEqual(ret["OK"], True)
self.assertEqual(bool(ret["Value"]), True)
# # db summary
ret = db.getDBSummary()
self.assertEqual(
ret,
{
"OK": True,
"Value": {
"Operation": {"RemoveFile": {"Queued": 1L}, "ReplicateAndRegister": {"Waiting": 1L}},
"Request": {"Waiting": 1L},
"File": {"Waiting": 2L},
},
},
)
# # get request for jobs
ret = db.getRequestNamesForJobs([1])
self.assertEqual(ret["OK"], True)
self.assertEqual(ret["Value"], {1: "test1"})
# # read requests
ret = db.readRequestsForJobs([1])
self.assertEqual(ret["OK"], True)
self.assertEqual(ret["Value"][1]["OK"], True)
# # select
ret = db.getRequest()
self.assertEqual(ret["OK"], True)
request = ret["Value"]
self.assertEqual(isinstance(request, Request), True)
# # summary
ret = db.getDBSummary()
self.assertEqual(
ret,
{
"OK": True,
"Value": {
"Operation": {"RemoveFile": {"Queued": 1L}, "ReplicateAndRegister": {"Waiting": 1L}},
"Request": {"Assigned": 1L},
"File": {"Waiting": 2L},
},
},
)
# # update
ret = db.putRequest(request)
self.assertEqual(ret, {"OK": True, "Value": ""})
# # get summary again
ret = db.getDBSummary()
self.assertEqual(
ret,
{
"OK": True,
"Value": {
"Operation": {"RemoveFile": {"Queued": 1L}, "ReplicateAndRegister": {"Waiting": 1L}},
"Request": {"Waiting": 1L},
"File": {"Waiting": 2L},
},
},
)
# # delete
ret = db.deleteRequest(self.request.RequestName)
self.assertEqual(ret, {"OK": True, "Value": ""})
# # should be empty now
ret = db.getDBSummary()
self.assertEqual(ret, {"OK": True, "Value": {"Operation": {}, "Request": {}, "File": {}}})
def test03RequestRW(self):
""" db r/w requests """
db = RequestDB()
db._checkTables(True)
# # empty DB at that stage
ret = db.getDBSummary()
self.assertEqual(ret, {
'OK': True,
'Value': {
'Operation': {},
'Request': {},
'File': {}
}
})
# # insert
ret = db.putRequest(self.request)
self.assertEqual(ret, {'OK': True, 'Value': ''})
# # get digest -> JSON
ret = db.getDigest(self.request.RequestName)
self.assertEqual(ret["OK"], True)
self.assertEqual(bool(ret["Value"]), True)
# # db summary
ret = db.getDBSummary()
self.assertEqual(
ret, {
'OK': True,
'Value': {
'Operation': {
'RemoveFile': {
'Queued': 1L
},
'ReplicateAndRegister': {
'Waiting': 1L
}
},
'Request': {
'Waiting': 1L
},
'File': {
'Waiting': 2L
}
}
})
# # get request for jobs
ret = db.getRequestNamesForJobs([1])
self.assertEqual(ret["OK"], True)
self.assertEqual(ret["Value"], {1: 'test1'})
# # read requests
ret = db.readRequestsForJobs([1])
self.assertEqual(ret["OK"], True)
self.assertEqual(ret["Value"][1]["OK"], True)
# # select
ret = db.getRequest()
self.assertEqual(ret["OK"], True)
request = ret["Value"]
self.assertEqual(isinstance(request, Request), True)
# # summary
ret = db.getDBSummary()
self.assertEqual(
ret, {
'OK': True,
'Value': {
'Operation': {
'RemoveFile': {
'Queued': 1L
},
'ReplicateAndRegister': {
'Waiting': 1L
}
},
'Request': {
'Assigned': 1L
},
'File': {
'Waiting': 2L
}
}
})
# # update
ret = db.putRequest(request)
self.assertEqual(ret, {'OK': True, 'Value': ''})
# # get summary again
ret = db.getDBSummary()
self.assertEqual(
ret, {
'OK': True,
'Value': {
'Operation': {
'RemoveFile': {
'Queued': 1L
},
'ReplicateAndRegister': {
'Waiting': 1L
}
},
'Request': {
'Waiting': 1L
},
'File': {
'Waiting': 2L
}
}
})
# # delete
ret = db.deleteRequest(self.request.RequestName)
self.assertEqual(ret, {'OK': True, 'Value': ''})
# # should be empty now
ret = db.getDBSummary()
self.assertEqual(ret, {
'OK': True,
'Value': {
'Operation': {},
'Request': {},
'File': {}
}
})