class TaskCallbacksTests(unittest.TestCase): """ .. class:: TaskCallbacksTests test case for ProcessPool """ def setUp(self): gLogger.showHeaders(True) self.log = gLogger.getSubLogger(self.__class__.__name__) self.processPool = ProcessPool(4, 8, 8) self.processPool.daemonize() def testCallableClass(self): """ CallableClass and task callbacks test """ i = 0 while True: if self.processPool.getFreeSlots() > 0: timeWait = random.randint(0, 5) raiseException = False if not timeWait: raiseException = True result = self.processPool.createAndQueueTask( CallableClass, taskID=i, args=(i, timeWait, raiseException), callback=ResultCallback, exceptionCallback=ExceptionCallback, blocking=True) if result["OK"]: self.log.always("CallableClass enqueued to task %s" % i) i += 1 else: continue if i == 10: break self.processPool.finalize(2) def testCallableFunc(self): """ CallableFunc and task callbacks test """ i = 0 while True: if self.processPool.getFreeSlots() > 0: timeWait = random.randint(0, 5) raiseException = False if not timeWait: raiseException = True result = self.processPool.createAndQueueTask( CallableFunc, taskID=i, args=(i, timeWait, raiseException), callback=ResultCallback, exceptionCallback=ExceptionCallback, blocking=True) if result["OK"]: self.log.always("CallableClass enqueued to task %s" % i) i += 1 else: continue if i == 10: break self.processPool.finalize(2)
class TaskCallbacksTests(unittest.TestCase): """ .. class:: TaskCallbacksTests test case for ProcessPool """ def setUp( self ): gLogger.showHeaders( True ) self.log = gLogger.getSubLogger( self.__class__.__name__ ) self.processPool = ProcessPool( 4, 8, 8 ) self.processPool.daemonize() def testCallableClass( self ): """ CallableClass and task callbacks test """ i = 0 while True: if self.processPool.getFreeSlots() > 0: timeWait = random.randint(0, 5) raiseException = False if not timeWait: raiseException = True result = self.processPool.createAndQueueTask( CallableClass, taskID = i, args = ( i, timeWait, raiseException ), callback = ResultCallback, exceptionCallback = ExceptionCallback, blocking = True ) if result["OK"]: self.log.always("CallableClass enqueued to task %s" % i ) i += 1 else: continue if i == 10: break self.processPool.finalize( 2 ) def testCallableFunc( self ): """ CallableFunc and task callbacks test """ i = 0 while True: if self.processPool.getFreeSlots() > 0: timeWait = random.randint(0, 5) raiseException = False if not timeWait: raiseException = True result = self.processPool.createAndQueueTask( CallableFunc, taskID = i, args = ( i, timeWait, raiseException ), callback = ResultCallback, exceptionCallback = ExceptionCallback, blocking = True ) if result["OK"]: self.log.always("CallableClass enqueued to task %s" % i ) i += 1 else: continue if i == 10: break self.processPool.finalize( 2 )
pPool = ProcessPool(10,50,50) pPool.daemonize() # dirlist = ['prod/ilc/mc-dbd/generated','prod/ilc/mc-dbd/ild'] # dirlist= ['prod/ilc/mc-dbd/generated/500-TDR_ws/higgs'] # dirlist= ['prod/ilc/mc-dbd/generated/250-TDR_ws/higgs','prod/ilc/mc-dbd/generated/350-TDR_ws/higgs'] #dirlist= ['prod/ilc/mc-dbd/generated/250-TDR_ws'] #dirlist= ['prod/ilc/mc-dbd/generated/250-TDR_ws/1f', # 'prod/ilc/mc-dbd/generated/250-TDR_ws/3f', # 'prod/ilc/mc-dbd/generated/250-TDR_ws/aa_lowpt', # 'prod/ilc/mc-dbd/generated/250-TDR_ws/aa_minijet'] #dirlist= ['prod/ilc/mc-dbd/generated/250-TDR_ws/aa_2f', # 'prod/ilc/mc-dbd/generated/350-TDR_ws/3f', # 'prod/ilc/mc-dbd/generated/350-TDR_ws/1f', # 'prod/ilc/mc-dbd/generated/350-TDR_ws/aa_minijet'] lfcHosts = ['grid-lfc.desy.de'] for dir in dirlist: path = "/ilc/%s" % (dir) print "Queueing user", dir, pPool.getFreeSlots(),pPool.getNumWorkingProcesses(),pPool.hasPendingTasks(),pPool.getNumIdleProcesses(), lfcHosts[0] result = pPool.createAndQueueTask( processDir,[path,True,lfcHosts[0]],callback=finalizeDirectory ) if not result['OK']: print "Failed queueing", path pPool.processAllResults() print "LFC_to_DFC completed."
class TaskTimeOutTests( unittest.TestCase ): """ .. class:: TaskTimeOutTests test case for ProcessPool """ def setUp( self ): """c'tor :param self: self reference """ from DIRAC.Core.Base import Script Script.parseCommandLine() from DIRAC.FrameworkSystem.Client.Logger import gLogger gLogger.showHeaders( True ) self.log = gLogger.getSubLogger( self.__class__.__name__ ) self.processPool = ProcessPool( 2, 4, 8, poolCallback = self.poolCallback, poolExceptionCallback = self.poolExceptionCallback ) self.processPool.daemonize() def poolCallback( self, taskID, taskResult ): self.log.always( "callback result for %s is %s" % ( taskID, taskResult ) ) def poolExceptionCallback( self, taskID, taskException ): self.log.always( "callback exception for %s is %s" % ( taskID, taskException ) ) def testCallableClass( self ): """ CallableClass and task time out test """ i = 0 while True: if self.processPool.getFreeSlots() > 0: timeWait = random.randint( 0, 5 ) * 10 raiseException = False if not timeWait: raiseException = True result = self.processPool.createAndQueueTask( CallableClass, taskID = i, args = ( i, timeWait, raiseException ), timeOut = 15, usePoolCallbacks = True, blocking = True ) if result["OK"]: self.log.always("CallableClass enqueued to task %s timeWait=%s exception=%s" % ( i, timeWait, raiseException ) ) i += 1 else: continue if i == 16: break self.processPool.finalize( 2 ) def testCallableFunc( self ): """ CallableFunc and task timeout test """ i = 0 while True: if self.processPool.getFreeSlots() > 0: timeWait = random.randint(0, 5) * 5 raiseException = False if not timeWait: raiseException = True result = self.processPool.createAndQueueTask( CallableFunc, taskID = i, args = ( i, timeWait, raiseException ), timeOut = 15, usePoolCallbacks = True, blocking = True ) if result["OK"]: self.log.always("CallableFunc enqueued to task %s timeWait=%s exception=%s" % ( i, timeWait, raiseException ) ) i += 1 else: continue if i == 16: break self.processPool.finalize( 2 ) def testLockedClass( self ): """ LockedCallableClass and task time out test """ for loop in range(2): self.log.always( "loop %s" % loop ) i = 0 while i < 16: if self.processPool.getFreeSlots() > 0: timeWait = random.randint(0, 5) * 5 raiseException = False if timeWait == 5: raiseException = True klass = CallableClass if timeWait >= 20: klass = LockedCallableClass result = self.processPool.createAndQueueTask( klass, taskID = i, args = ( i, timeWait, raiseException ), timeOut = 15, usePoolCallbacks = True, blocking = True ) if result["OK"]: self.log.always("%s enqueued to task %s timeWait=%s exception=%s" % ( klass.__name__ , i, timeWait, raiseException ) ) i += 1 else: continue self.log.always("being idle for a while") for i in range(100000): for j in range(1000): pass self.log.always("finalizing...") self.processPool.finalize( 10 ) ## unlock gLock.release()
class ProcessPoolCallbacksTests( unittest.TestCase ): """ .. class:: ProcessPoolCallbacksTests test case for ProcessPool """ def setUp( self ): """c'tor :param self: self reference """ from DIRAC.Core.Base import Script Script.parseCommandLine() from DIRAC.FrameworkSystem.Client.Logger import gLogger gLogger.showHeaders( True ) self.log = gLogger.getSubLogger( self.__class__.__name__ ) self.processPool = ProcessPool( 4, 8, 8, poolCallback = self.poolCallback, poolExceptionCallback = self.poolExceptionCallback ) self.processPool.daemonize() def poolCallback( self, taskID, taskResult ): self.log.always( "callback for %s result is %s" % ( taskID, taskResult ) ) def poolExceptionCallback( self, taskID, taskException ): self.log.always( "callback for %s exception is %s" % ( taskID, taskException ) ) def testCallableClass( self ): """ CallableClass and pool callbacks test """ i = 0 while True: if self.processPool.getFreeSlots() > 0: timeWait = random.randint(0, 5) raiseException = False if not timeWait: raiseException = True result = self.processPool.createAndQueueTask( CallableClass, taskID = i, args = ( i, timeWait, raiseException ), usePoolCallbacks = True, blocking = True ) if result["OK"]: self.log.always("CallableClass enqueued to task %s" % i ) i += 1 else: continue if i == 10: break self.processPool.finalize( 2 ) def testCallableFunc( self ): """ CallableFunc and pool callbacks test """ i = 0 while True: if self.processPool.getFreeSlots() > 0: timeWait = random.randint(0, 5) raiseException = False if not timeWait: raiseException = True result = self.processPool.createAndQueueTask( CallableFunc, taskID = i, args = ( i, timeWait, raiseException ), usePoolCallbacks = True, blocking = True ) if result["OK"]: self.log.always("CallableFunc enqueued to task %s" % i ) i += 1 else: continue if i == 10: break self.processPool.finalize( 2 )
class TaskTimeOutTests( unittest.TestCase ): """ .. class:: TaskTimeOutTests test case for ProcessPool """ def setUp( self ): """c'tor :param self: self reference """ gLogger.showHeaders( True ) self.log = gLogger.getSubLogger( self.__class__.__name__ ) self.processPool = ProcessPool( 2, 4, 8, poolCallback = self.poolCallback, poolExceptionCallback = self.poolExceptionCallback ) self.processPool.daemonize() def poolCallback( self, taskID, taskResult ): self.log.always( "callback result for %s is %s" % ( taskID, taskResult ) ) def poolExceptionCallback( self, taskID, taskException ): self.log.always( "callback exception for %s is %s" % ( taskID, taskException ) ) def testCallableClass( self ): """ CallableClass and task time out test """ i = 0 while True: if self.processPool.getFreeSlots() > 0: timeWait = random.randint( 0, 5 ) * 10 raiseException = False if not timeWait: raiseException = True result = self.processPool.createAndQueueTask( CallableClass, taskID = i, args = ( i, timeWait, raiseException ), timeOut = 15, usePoolCallbacks = True, blocking = True ) if result["OK"]: self.log.always("CallableClass enqueued to task %s timeWait=%s exception=%s" % ( i, timeWait, raiseException ) ) i += 1 else: continue if i == 16: break self.processPool.finalize( 2 ) def testCallableFunc( self ): """ CallableFunc and task timeout test """ i = 0 while True: if self.processPool.getFreeSlots() > 0: timeWait = random.randint(0, 5) * 5 raiseException = False if not timeWait: raiseException = True result = self.processPool.createAndQueueTask( CallableFunc, taskID = i, args = ( i, timeWait, raiseException ), timeOut = 15, usePoolCallbacks = True, blocking = True ) if result["OK"]: self.log.always("CallableFunc enqueued to task %s timeWait=%s exception=%s" % ( i, timeWait, raiseException ) ) i += 1 else: continue if i == 16: break self.processPool.finalize( 2 ) def testLockedClass( self ): """ LockedCallableClass and task time out test """ for loop in range(2): self.log.always( "loop %s" % loop ) i = 0 while i < 16: if self.processPool.getFreeSlots() > 0: timeWait = random.randint(0, 5) * 5 raiseException = False if timeWait == 5: raiseException = True klass = CallableClass if timeWait >= 20: klass = LockedCallableClass result = self.processPool.createAndQueueTask( klass, taskID = i, args = ( i, timeWait, raiseException ), timeOut = 15, usePoolCallbacks = True, blocking = True ) if result["OK"]: self.log.always("%s enqueued to task %s timeWait=%s exception=%s" % ( klass.__name__ , i, timeWait, raiseException ) ) i += 1 else: continue self.log.always("being idle for a while") for _ in range( 100000 ): for _ in range( 1000 ): pass self.log.always("finalizing...") self.processPool.finalize( 10 ) ## unlock gLock.release()
class ProcessPoolCallbacksTests( unittest.TestCase ): """ .. class:: ProcessPoolCallbacksTests test case for ProcessPool """ def setUp( self ): """c'tor :param self: self reference """ gLogger.showHeaders( True ) self.log = gLogger.getSubLogger( self.__class__.__name__ ) self.processPool = ProcessPool( 4, 8, 8, poolCallback = self.poolCallback, poolExceptionCallback = self.poolExceptionCallback ) self.processPool.daemonize() def poolCallback( self, taskID, taskResult ): self.log.always( "callback for %s result is %s" % ( taskID, taskResult ) ) def poolExceptionCallback( self, taskID, taskException ): self.log.always( "callback for %s exception is %s" % ( taskID, taskException ) ) def testCallableClass( self ): """ CallableClass and pool callbacks test """ i = 0 while True: if self.processPool.getFreeSlots() > 0: timeWait = random.randint(0, 5) raiseException = False if not timeWait: raiseException = True result = self.processPool.createAndQueueTask( CallableClass, taskID = i, args = ( i, timeWait, raiseException ), usePoolCallbacks = True, blocking = True ) if result["OK"]: self.log.always("CallableClass enqueued to task %s" % i ) i += 1 else: continue if i == 10: break self.processPool.finalize( 2 ) def testCallableFunc( self ): """ CallableFunc and pool callbacks test """ i = 0 while True: if self.processPool.getFreeSlots() > 0: timeWait = random.randint(0, 5) raiseException = False if not timeWait: raiseException = True result = self.processPool.createAndQueueTask( CallableFunc, taskID = i, args = ( i, timeWait, raiseException ), usePoolCallbacks = True, blocking = True ) if result["OK"]: self.log.always("CallableFunc enqueued to task %s" % i ) i += 1 else: continue if i == 10: break self.processPool.finalize( 2 )