def workOnSlice(ShMem,
start=None,
stop=None,
fH=dict(),
task='localstep',
minSliceDuration=1.0,
memoryType='shared',
nWorker=1,
maxWorker=1,
scaleFactor=1.0,
**kwargs):
""" Perform work on a slice of data.
"""
if start is not None:
start = int(start)
stop = int(stop)
Dslice = fH['makeDataSliceFromSharedMem'](ShMem['dataSharedMem'],
cslice=(start, stop))
kwargs.update(sharedMemDictToNumpy(ShMem['aSharedMem']))
kwargs.update(sharedMemDictToNumpy(ShMem['oSharedMem']))
nReps = np.maximum(1, int(np.ceil(scaleFactor)))
tstart = time.time()
if task == 'sleep':
# Make sure that we sleep same total amt whether serial or parallel
duration = minSliceDuration * (maxWorker / float(nWorker))
time.sleep(duration)
elif task == 'localstep':
for rep in range(nReps):
# Do local step
LP = fH['o_calcLocalParams'](Dslice, **kwargs)
LP = fH['a_calcLocalParams'](Dslice, LP, **kwargs)
# Do summary step
SS = fH['a_calcSummaryStats'](Dslice,
LP,
doPrecompEntropy=1,
**kwargs)
SS = fH['o_calcSummaryStats'](Dslice, SS, LP, **kwargs)
else:
raise NotImplementedError("Unrecognized task: %s" % (task))
telapsed = time.time() - tstart
return telapsed
def run(self):
self.printMsg("process SetUp! pid=%d" % (os.getpid()))
# Construct iterator with sentinel value of None (for termination)
jobIterator = iter(self.JobQueue.get, None)
for sliceArgs, aArgs, oArgs in jobIterator:
start1 = time.time()
# Grab slice of data to work on
if 'start' in sliceArgs:
# Load data from shared memory
Dslice = self.makeDataSliceFromSharedMem(
self.dataSharedMem,
batchID=sliceArgs['batchID'],
cslice=(sliceArgs['start'], sliceArgs['stop']))
else:
# Load data slice directly from file
Dslice = loadDataForSlice(**sliceArgs)
# Prep kwargs for the alloc model, especially local step kwargs
aArgs.update(sharedMemDictToNumpy(self.aSharedMem))
aArgs.update(self.LPkwargs)
aArgs.update(sliceArgs)
# Prep kwargs for the obs model
oArgs.update(sharedMemDictToNumpy(self.oSharedMem))
# Do local step
LP = self.o_calcLocalParams(Dslice, **oArgs)
LP = self.a_calcLocalParams(Dslice, LP, **aArgs)
# Do summary step
SS = self.a_calcSummaryStats(Dslice,
LP,
doPrecompEntropy=1,
**aArgs)
SS = self.o_calcSummaryStats(Dslice, SS, LP, **oArgs)
# Add final suff stats to result queue to wrap up this task!
self.ResultQueue.put(SS)
self.JobQueue.task_done()
duration = time.time() - start1
# Clean up
self.printMsg("process CleanUp! pid=%d" % (os.getpid()))
def serialCalcLocalParamsAndSummarize(self, hmodel):
SSagg = None
for dataBatchID, start, stop in self.sliceGenerator(
self.nDoc, self.nWorkers):
sliceArgs = (dataBatchID, start, stop)
aArgs = hmodel.allocModel.getSerializableParamsForLocalStep()
oArgs = hmodel.obsModel.getSerializableParamsForLocalStep()
# Get the function handles
o_calcLocalParams, o_calcSummaryStats = hmodel.obsModel.\
getLocalAndSummaryFunctionHandles()
a_calcLocalParams, a_calcSummaryStats = hmodel.allocModel.\
getLocalAndSummaryFunctionHandles()
# Create the shared memory
aSharedMem = hmodel.allocModel.fillSharedMemDictForLocalStep()
oSharedMem = hmodel.obsModel.fillSharedMemDictForLocalStep()
cslice = (start, stop)
Dslice = self.makeDataSliceFromSharedMem(
self.dataSharedMem, cslice=cslice)
aArgs.update(sharedMemDictToNumpy(aSharedMem))
oArgs.update(sharedMemDictToNumpy(oSharedMem))
LP = o_calcLocalParams(Dslice, **oArgs)
LP = a_calcLocalParams(Dslice, LP, **aArgs)
SSslice = a_calcSummaryStats(
Dslice,
LP,
doPrecompEntropy=1,
**aArgs)
SSslice = o_calcSummaryStats(Dslice, SSslice, LP, **oArgs)
if start == 0:
SSagg = SSslice
else:
SSagg += SSslice
return SSagg