def testDoingWorkWithTwoWorkers(): logger = sutil.SilentFakeLogger() config = sutil.DotDict({'logger': logger, 'numberOfThreads': 2}) myList = [] def insertIntoList(anItem): myList.append(anItem[0]) return siwf.OK iwf = siwf.IteratorWorkerFramework(config, name='Wilma', taskFunc=insertIntoList) try: iwf.start() time.sleep(2.0) assert len(iwf.workerPool.threadList ) == 2, "expected 2 threads, but found %d" % len( iwf.workerPool.threadList) assert len( myList ) == 10, 'expected to do 10 inserts, but %d were done instead' % len( myList) assert sorted(myList) == range( 10), 'expected %s, but got %s' % (range(10), sorted(myList)) iwf.stop() except Exception: # we got threads to join iwf.workerPool.waitForCompletion() raise
def submitter (config): logger = config.logger signal.signal(signal.SIGTERM, iwf.respondToSIGTERM) signal.signal(signal.SIGHUP, iwf.respondToSIGTERM) statsPool = sutil.DotDict( { 'submittedCount': stats.CounterPool(config), 'failureCount': stats.CounterPool(config), 'processTime': stats.DurationAccumulatorPool(config), }) config.statsPool = statsPool reportigCounter = 0 def statsReportingWaitingFunc(): if not statsReportingWaitingFunc.reportingCounter % 60: submittedCountPool = statsPool.submittedCount numberOfMinutes = submittedCountPool.numberOfMinutes() if numberOfMinutes: logger.info('running for %d minutes', numberOfMinutes) numberSubmitted = submittedCountPool.read() logger.info('average submitted per minute: %s', \ (float(numberSubmitted) / numberOfMinutes)) numberOfFailures = statsPool.failureCount.read() logger.info('failures in the last five minutes: %d', \ numberOfFailures) processTime = statsPool.processTime.read() logger.info('average time in last five minutes: %s', \ processTime) statsReportingWaitingFunc.reportingCounter += 1 statsReportingWaitingFunc.reportingCounter = 0 theIterator = config.iteratorFunc (config) theWorkerFunction = createSubmitterFunction(config) submissionMill = iwf.IteratorWorkerFramework(config, jobSourceIterator=theIterator, taskFunc=theWorkerFunction, name='submissionMill') try: submissionMill.start() submissionMill.waitForCompletion(statsReportingWaitingFunc) # though, it only ends if someone # hits ^C or sends SIGHUP or SIGTERM # - any of which will get translated # into a KeyboardInterrupt exception except KeyboardInterrupt: while True: try: submissionMill.stop() break except KeyboardInterrupt: logger.warning('We heard you the first time. There is no need ' 'for further keyboard or signal interrupts. We ' 'are waiting for the worker threads to stop. ' 'If this app does not halt soon, you may have ' 'to send SIGKILL (kill -9)')
def testConstuctor1(): logger = sutil.SilentFakeLogger() config = sutil.DotDict({'logger': logger, 'numberOfThreads': 1}) iwf = siwf.IteratorWorkerFramework( config, name='Wilma', ) try: assert iwf.config == config assert iwf.name == 'Wilma' assert iwf.logger == logger assert iwf.taskFunc == siwf.defaultTaskFunc assert iwf.quit == False finally: # we got threads to join iwf.workerPool.waitForCompletion()
def testStart1(): logger = sutil.SilentFakeLogger() config = sutil.DotDict({'logger': logger, 'numberOfThreads': 1}) iwf = siwf.IteratorWorkerFramework( config, name='Wilma', ) try: iwf.start() time.sleep(2.0) assert iwf.queuingThread.isAlive(), "the queing thread is not running" assert len( iwf.workerPool.threadList) == 1, "where's the worker thread?" assert iwf.workerPool.threadList[0].isAlive( ), "the worker thread is stillborn" iwf.stop() assert iwf.queuingThread.isAlive( ) == False, "the queuing thread did not stop" except Exception: # we got threads to join iwf.workerPool.waitForCompletion()
return iwf.FAILURE elif result == cstore.CrashStorageSystem.RETRY: return iwf.RETRY try: sourceStorage.quickDelete(ooid) except Exception: sutil.reportExceptionAndContinue(self.logger) return iwf.OK except Exception, x: sutil.reportExceptionAndContinue(logger) return iwf.FAILURE #----------------------------------------------------------------------------- submissionMill = iwf.IteratorWorkerFramework(conf, jobSourceIterator=theIterator, taskFunc=doSubmission, name='submissionMill') try: submissionMill.start() submissionMill.waitForCompletion( ) # though, it only ends if someone hits # ^C or sends SIGHUP or SIGTERM - any of # which will get translated into a # KeyboardInterrupt exception except KeyboardInterrupt: while True: try: submissionMill.stop() break except KeyboardInterrupt: