class TaskCallbacksTests(unittest.TestCase):
"""
.. class:: TaskCallbacksTests
test case for ProcessPool
"""
def setUp(self):
gLogger.showHeaders(True)
self.log = gLogger.getSubLogger(self.__class__.__name__)
self.processPool = ProcessPool(4, 8, 8)
self.processPool.daemonize()
def testCallableClass(self):
""" CallableClass and task callbacks test """
i = 0
while True:
if self.processPool.getFreeSlots() > 0:
timeWait = random.randint(0, 5)
raiseException = False
if not timeWait:
raiseException = True
result = self.processPool.createAndQueueTask(
CallableClass,
taskID=i,
args=(i, timeWait, raiseException),
callback=ResultCallback,
exceptionCallback=ExceptionCallback,
blocking=True)
if result["OK"]:
self.log.always("CallableClass enqueued to task %s" % i)
i += 1
else:
continue
if i == 10:
break
self.processPool.finalize(2)
def testCallableFunc(self):
""" CallableFunc and task callbacks test """
i = 0
while True:
if self.processPool.getFreeSlots() > 0:
timeWait = random.randint(0, 5)
raiseException = False
if not timeWait:
raiseException = True
result = self.processPool.createAndQueueTask(
CallableFunc,
taskID=i,
args=(i, timeWait, raiseException),
callback=ResultCallback,
exceptionCallback=ExceptionCallback,
blocking=True)
if result["OK"]:
self.log.always("CallableClass enqueued to task %s" % i)
i += 1
else:
continue
if i == 10:
break
self.processPool.finalize(2)
class TaskCallbacksTests(unittest.TestCase):
"""
.. class:: TaskCallbacksTests
test case for ProcessPool
"""
def setUp( self ):
gLogger.showHeaders( True )
self.log = gLogger.getSubLogger( self.__class__.__name__ )
self.processPool = ProcessPool( 4, 8, 8 )
self.processPool.daemonize()
def testCallableClass( self ):
""" CallableClass and task callbacks test """
i = 0
while True:
if self.processPool.getFreeSlots() > 0:
timeWait = random.randint(0, 5)
raiseException = False
if not timeWait:
raiseException = True
result = self.processPool.createAndQueueTask( CallableClass,
taskID = i,
args = ( i, timeWait, raiseException ),
callback = ResultCallback,
exceptionCallback = ExceptionCallback,
blocking = True )
if result["OK"]:
self.log.always("CallableClass enqueued to task %s" % i )
i += 1
else:
continue
if i == 10:
break
self.processPool.finalize( 2 )
def testCallableFunc( self ):
""" CallableFunc and task callbacks test """
i = 0
while True:
if self.processPool.getFreeSlots() > 0:
timeWait = random.randint(0, 5)
raiseException = False
if not timeWait:
raiseException = True
result = self.processPool.createAndQueueTask( CallableFunc,
taskID = i,
args = ( i, timeWait, raiseException ),
callback = ResultCallback,
exceptionCallback = ExceptionCallback,
blocking = True )
if result["OK"]:
self.log.always("CallableClass enqueued to task %s" % i )
i += 1
else:
continue
if i == 10:
break
self.processPool.finalize( 2 )
def runTest():
global nClients, nQueries, testType, resultTest, testDir, lfnListFile
resultTest = []
pp = ProcessPool(nClients)
testFunction = eval(testType)
for c in xrange(nClients):
pp.createAndQueueTask(testFunction, [nQueries],
callback=finalize,
exceptionCallback=doException)
pp.processAllResults(3600)
pp.finalize(0)
timeResult = []
for testTime, success, failure in resultTest:
#print testTime,success,failure
timeResult += testTime
averageTime, errorTime = doStats(timeResult)
rateResult = [nClients / t for t in timeResult]
averageRate, errorRate = doStats(rateResult)
if testDir:
print "\nTest results for clients %d, %s" % (nClients, testDir)
else:
print "\nTest results for clients %d, %s" % (nClients, lfnListFile)
print "Query time: %.2f +/- %.2f" % (averageTime, errorTime)
print "Query rate: %.2f +/- %.2f" % (averageRate, errorRate)
return ((averageTime, errorTime), (averageRate, errorRate))
def runTest():
global nClients, nQueries, testType, resultTest, testDir, lfnListFile
resultTest = []
pp = ProcessPool(nClients)
testFunction = eval(testType)
for c in xrange(nClients):
pp.createAndQueueTask(testFunction, [nQueries],
callback=finalize,
exceptionCallback=doException)
pp.processAllResults(3600)
pp.finalize(0)
timeResult = []
for testTime, success, failure in resultTest:
# print testTime,success,failure
timeResult += testTime
averageTime, errorTime = doStats(timeResult)
rateResult = [nClients / t for t in timeResult]
averageRate, errorRate = doStats(rateResult)
if testDir:
print "\nTest results for clients %d, %s" % (nClients, testDir)
else:
print "\nTest results for clients %d, %s" % (nClients, lfnListFile)
print "Query time: %.2f +/- %.2f" % (averageTime, errorTime)
print "Query rate: %.2f +/- %.2f" % (averageRate, errorRate)
return((averageTime, errorTime), (averageRate, errorRate))
class TaskTimeOutTests( unittest.TestCase ):
"""
.. class:: TaskTimeOutTests
test case for ProcessPool
"""
def setUp( self ):
"""c'tor
:param self: self reference
"""
from DIRAC.Core.Base import Script
Script.parseCommandLine()
from DIRAC.FrameworkSystem.Client.Logger import gLogger
gLogger.showHeaders( True )
self.log = gLogger.getSubLogger( self.__class__.__name__ )
self.processPool = ProcessPool( 2,
4,
8,
poolCallback = self.poolCallback,
poolExceptionCallback = self.poolExceptionCallback )
self.processPool.daemonize()
def poolCallback( self, taskID, taskResult ):
self.log.always( "callback result for %s is %s" % ( taskID, taskResult ) )
def poolExceptionCallback( self, taskID, taskException ):
self.log.always( "callback exception for %s is %s" % ( taskID, taskException ) )
def testCallableClass( self ):
""" CallableClass and task time out test """
i = 0
while True:
if self.processPool.getFreeSlots() > 0:
timeWait = random.randint( 0, 5 ) * 10
raiseException = False
if not timeWait:
raiseException = True
result = self.processPool.createAndQueueTask( CallableClass,
taskID = i,
args = ( i, timeWait, raiseException ),
timeOut = 15,
usePoolCallbacks = True,
blocking = True )
if result["OK"]:
self.log.always("CallableClass enqueued to task %s timeWait=%s exception=%s" % ( i, timeWait, raiseException ) )
i += 1
else:
continue
if i == 16:
break
self.processPool.finalize( 2 )
def testCallableFunc( self ):
""" CallableFunc and task timeout test """
i = 0
while True:
if self.processPool.getFreeSlots() > 0:
timeWait = random.randint(0, 5) * 5
raiseException = False
if not timeWait:
raiseException = True
result = self.processPool.createAndQueueTask( CallableFunc,
taskID = i,
args = ( i, timeWait, raiseException ),
timeOut = 15,
usePoolCallbacks = True,
blocking = True )
if result["OK"]:
self.log.always("CallableFunc enqueued to task %s timeWait=%s exception=%s" % ( i, timeWait, raiseException ) )
i += 1
else:
continue
if i == 16:
break
self.processPool.finalize( 2 )
def testLockedClass( self ):
""" LockedCallableClass and task time out test """
for loop in range(2):
self.log.always( "loop %s" % loop )
i = 0
while i < 16:
if self.processPool.getFreeSlots() > 0:
timeWait = random.randint(0, 5) * 5
raiseException = False
if timeWait == 5:
raiseException = True
klass = CallableClass
if timeWait >= 20:
klass = LockedCallableClass
result = self.processPool.createAndQueueTask( klass,
taskID = i,
args = ( i, timeWait, raiseException ),
timeOut = 15,
usePoolCallbacks = True,
blocking = True )
if result["OK"]:
self.log.always("%s enqueued to task %s timeWait=%s exception=%s" % ( klass.__name__ , i, timeWait, raiseException ) )
i += 1
else:
continue
self.log.always("being idle for a while")
for i in range(100000):
for j in range(1000):
pass
self.log.always("finalizing...")
self.processPool.finalize( 10 )
## unlock
gLock.release()
class ProcessPoolCallbacksTests( unittest.TestCase ):
"""
.. class:: ProcessPoolCallbacksTests
test case for ProcessPool
"""
def setUp( self ):
"""c'tor
:param self: self reference
"""
from DIRAC.Core.Base import Script
Script.parseCommandLine()
from DIRAC.FrameworkSystem.Client.Logger import gLogger
gLogger.showHeaders( True )
self.log = gLogger.getSubLogger( self.__class__.__name__ )
self.processPool = ProcessPool( 4, 8, 8,
poolCallback = self.poolCallback,
poolExceptionCallback = self.poolExceptionCallback )
self.processPool.daemonize()
def poolCallback( self, taskID, taskResult ):
self.log.always( "callback for %s result is %s" % ( taskID, taskResult ) )
def poolExceptionCallback( self, taskID, taskException ):
self.log.always( "callback for %s exception is %s" % ( taskID, taskException ) )
def testCallableClass( self ):
""" CallableClass and pool callbacks test """
i = 0
while True:
if self.processPool.getFreeSlots() > 0:
timeWait = random.randint(0, 5)
raiseException = False
if not timeWait:
raiseException = True
result = self.processPool.createAndQueueTask( CallableClass,
taskID = i,
args = ( i, timeWait, raiseException ),
usePoolCallbacks = True,
blocking = True )
if result["OK"]:
self.log.always("CallableClass enqueued to task %s" % i )
i += 1
else:
continue
if i == 10:
break
self.processPool.finalize( 2 )
def testCallableFunc( self ):
""" CallableFunc and pool callbacks test """
i = 0
while True:
if self.processPool.getFreeSlots() > 0:
timeWait = random.randint(0, 5)
raiseException = False
if not timeWait:
raiseException = True
result = self.processPool.createAndQueueTask( CallableFunc,
taskID = i,
args = ( i, timeWait, raiseException ),
usePoolCallbacks = True,
blocking = True )
if result["OK"]:
self.log.always("CallableFunc enqueued to task %s" % i )
i += 1
else:
continue
if i == 10:
break
self.processPool.finalize( 2 )
class TaskTimeOutTests( unittest.TestCase ):
"""
.. class:: TaskTimeOutTests
test case for ProcessPool
"""
def setUp( self ):
"""c'tor
:param self: self reference
"""
gLogger.showHeaders( True )
self.log = gLogger.getSubLogger( self.__class__.__name__ )
self.processPool = ProcessPool( 2,
4,
8,
poolCallback = self.poolCallback,
poolExceptionCallback = self.poolExceptionCallback )
self.processPool.daemonize()
def poolCallback( self, taskID, taskResult ):
self.log.always( "callback result for %s is %s" % ( taskID, taskResult ) )
def poolExceptionCallback( self, taskID, taskException ):
self.log.always( "callback exception for %s is %s" % ( taskID, taskException ) )
def testCallableClass( self ):
""" CallableClass and task time out test """
i = 0
while True:
if self.processPool.getFreeSlots() > 0:
timeWait = random.randint( 0, 5 ) * 10
raiseException = False
if not timeWait:
raiseException = True
result = self.processPool.createAndQueueTask( CallableClass,
taskID = i,
args = ( i, timeWait, raiseException ),
timeOut = 15,
usePoolCallbacks = True,
blocking = True )
if result["OK"]:
self.log.always("CallableClass enqueued to task %s timeWait=%s exception=%s" % ( i, timeWait, raiseException ) )
i += 1
else:
continue
if i == 16:
break
self.processPool.finalize( 2 )
def testCallableFunc( self ):
""" CallableFunc and task timeout test """
i = 0
while True:
if self.processPool.getFreeSlots() > 0:
timeWait = random.randint(0, 5) * 5
raiseException = False
if not timeWait:
raiseException = True
result = self.processPool.createAndQueueTask( CallableFunc,
taskID = i,
args = ( i, timeWait, raiseException ),
timeOut = 15,
usePoolCallbacks = True,
blocking = True )
if result["OK"]:
self.log.always("CallableFunc enqueued to task %s timeWait=%s exception=%s" % ( i, timeWait, raiseException ) )
i += 1
else:
continue
if i == 16:
break
self.processPool.finalize( 2 )
def testLockedClass( self ):
""" LockedCallableClass and task time out test """
for loop in range(2):
self.log.always( "loop %s" % loop )
i = 0
while i < 16:
if self.processPool.getFreeSlots() > 0:
timeWait = random.randint(0, 5) * 5
raiseException = False
if timeWait == 5:
raiseException = True
klass = CallableClass
if timeWait >= 20:
klass = LockedCallableClass
result = self.processPool.createAndQueueTask( klass,
taskID = i,
args = ( i, timeWait, raiseException ),
timeOut = 15,
usePoolCallbacks = True,
blocking = True )
if result["OK"]:
self.log.always("%s enqueued to task %s timeWait=%s exception=%s" % ( klass.__name__ , i, timeWait, raiseException ) )
i += 1
else:
continue
self.log.always("being idle for a while")
for _ in range( 100000 ):
for _ in range( 1000 ):
pass
self.log.always("finalizing...")
self.processPool.finalize( 10 )
## unlock
gLock.release()
class ProcessPoolCallbacksTests( unittest.TestCase ):
"""
.. class:: ProcessPoolCallbacksTests
test case for ProcessPool
"""
def setUp( self ):
"""c'tor
:param self: self reference
"""
gLogger.showHeaders( True )
self.log = gLogger.getSubLogger( self.__class__.__name__ )
self.processPool = ProcessPool( 4, 8, 8,
poolCallback = self.poolCallback,
poolExceptionCallback = self.poolExceptionCallback )
self.processPool.daemonize()
def poolCallback( self, taskID, taskResult ):
self.log.always( "callback for %s result is %s" % ( taskID, taskResult ) )
def poolExceptionCallback( self, taskID, taskException ):
self.log.always( "callback for %s exception is %s" % ( taskID, taskException ) )
def testCallableClass( self ):
""" CallableClass and pool callbacks test """
i = 0
while True:
if self.processPool.getFreeSlots() > 0:
timeWait = random.randint(0, 5)
raiseException = False
if not timeWait:
raiseException = True
result = self.processPool.createAndQueueTask( CallableClass,
taskID = i,
args = ( i, timeWait, raiseException ),
usePoolCallbacks = True,
blocking = True )
if result["OK"]:
self.log.always("CallableClass enqueued to task %s" % i )
i += 1
else:
continue
if i == 10:
break
self.processPool.finalize( 2 )
def testCallableFunc( self ):
""" CallableFunc and pool callbacks test """
i = 0
while True:
if self.processPool.getFreeSlots() > 0:
timeWait = random.randint(0, 5)
raiseException = False
if not timeWait:
raiseException = True
result = self.processPool.createAndQueueTask( CallableFunc,
taskID = i,
args = ( i, timeWait, raiseException ),
usePoolCallbacks = True,
blocking = True )
if result["OK"]:
self.log.always("CallableFunc enqueued to task %s" % i )
i += 1
else:
continue
if i == 10:
break
self.processPool.finalize( 2 )
