def testDoingWorkWithTwoWorkers():
logger = sutil.SilentFakeLogger()
config = sutil.DotDict({'logger': logger, 'numberOfThreads': 2})
myList = []
def insertIntoList(anItem):
myList.append(anItem[0])
return siwf.OK
iwf = siwf.IteratorWorkerFramework(config,
name='Wilma',
taskFunc=insertIntoList)
try:
iwf.start()
time.sleep(2.0)
assert len(iwf.workerPool.threadList
) == 2, "expected 2 threads, but found %d" % len(
iwf.workerPool.threadList)
assert len(
myList
) == 10, 'expected to do 10 inserts, but %d were done instead' % len(
myList)
assert sorted(myList) == range(
10), 'expected %s, but got %s' % (range(10), sorted(myList))
iwf.stop()
except Exception:
# we got threads to join
iwf.workerPool.waitForCompletion()
raise
def submitter (config):
logger = config.logger
signal.signal(signal.SIGTERM, iwf.respondToSIGTERM)
signal.signal(signal.SIGHUP, iwf.respondToSIGTERM)
statsPool = sutil.DotDict(
{ 'submittedCount': stats.CounterPool(config),
'failureCount': stats.CounterPool(config),
'processTime': stats.DurationAccumulatorPool(config),
})
config.statsPool = statsPool
reportigCounter = 0
def statsReportingWaitingFunc():
if not statsReportingWaitingFunc.reportingCounter % 60:
submittedCountPool = statsPool.submittedCount
numberOfMinutes = submittedCountPool.numberOfMinutes()
if numberOfMinutes:
logger.info('running for %d minutes', numberOfMinutes)
numberSubmitted = submittedCountPool.read()
logger.info('average submitted per minute: %s', \
(float(numberSubmitted) / numberOfMinutes))
numberOfFailures = statsPool.failureCount.read()
logger.info('failures in the last five minutes: %d', \
numberOfFailures)
processTime = statsPool.processTime.read()
logger.info('average time in last five minutes: %s', \
processTime)
statsReportingWaitingFunc.reportingCounter += 1
statsReportingWaitingFunc.reportingCounter = 0
theIterator = config.iteratorFunc (config)
theWorkerFunction = createSubmitterFunction(config)
submissionMill = iwf.IteratorWorkerFramework(config,
jobSourceIterator=theIterator,
taskFunc=theWorkerFunction,
name='submissionMill')
try:
submissionMill.start()
submissionMill.waitForCompletion(statsReportingWaitingFunc)
# though, it only ends if someone
# hits ^C or sends SIGHUP or SIGTERM
# - any of which will get translated
# into a KeyboardInterrupt exception
except KeyboardInterrupt:
while True:
try:
submissionMill.stop()
break
except KeyboardInterrupt:
logger.warning('We heard you the first time. There is no need '
'for further keyboard or signal interrupts. We '
'are waiting for the worker threads to stop. '
'If this app does not halt soon, you may have '
'to send SIGKILL (kill -9)')
def testConstuctor1():
logger = sutil.SilentFakeLogger()
config = sutil.DotDict({'logger': logger, 'numberOfThreads': 1})
iwf = siwf.IteratorWorkerFramework(
config,
name='Wilma',
)
try:
assert iwf.config == config
assert iwf.name == 'Wilma'
assert iwf.logger == logger
assert iwf.taskFunc == siwf.defaultTaskFunc
assert iwf.quit == False
finally:
# we got threads to join
iwf.workerPool.waitForCompletion()
def testStart1():
logger = sutil.SilentFakeLogger()
config = sutil.DotDict({'logger': logger, 'numberOfThreads': 1})
iwf = siwf.IteratorWorkerFramework(
config,
name='Wilma',
)
try:
iwf.start()
time.sleep(2.0)
assert iwf.queuingThread.isAlive(), "the queing thread is not running"
assert len(
iwf.workerPool.threadList) == 1, "where's the worker thread?"
assert iwf.workerPool.threadList[0].isAlive(
), "the worker thread is stillborn"
iwf.stop()
assert iwf.queuingThread.isAlive(
) == False, "the queuing thread did not stop"
except Exception:
# we got threads to join
iwf.workerPool.waitForCompletion()
return iwf.FAILURE
elif result == cstore.CrashStorageSystem.RETRY:
return iwf.RETRY
try:
sourceStorage.quickDelete(ooid)
except Exception:
sutil.reportExceptionAndContinue(self.logger)
return iwf.OK
except Exception, x:
sutil.reportExceptionAndContinue(logger)
return iwf.FAILURE
#-----------------------------------------------------------------------------
submissionMill = iwf.IteratorWorkerFramework(conf,
jobSourceIterator=theIterator,
taskFunc=doSubmission,
name='submissionMill')
try:
submissionMill.start()
submissionMill.waitForCompletion(
) # though, it only ends if someone hits
# ^C or sends SIGHUP or SIGTERM - any of
# which will get translated into a
# KeyboardInterrupt exception
except KeyboardInterrupt:
while True:
try:
submissionMill.stop()
break
except KeyboardInterrupt: