def _assertFullView(self, data):
mcaSlice = slice(2, -1)
for nMca in range(numpy.prod(data.shape[1:]) + 2):
for mcaAxis in range(data.ndim):
dataView = McaStackView.FullView(data,
readonly=False,
mcaAxis=mcaAxis,
mcaSlice=mcaSlice,
nMca=nMca)
npAdd = numpy.arange(data.size).reshape(data.shape)
addView = McaStackView.FullView(npAdd,
readonly=True,
mcaAxis=mcaAxis,
mcaSlice=mcaSlice,
nMca=nMca)
idxFull = dataView.idxFull
for readonly in [True, False]:
dataView.readonly = readonly
dataOrg = numpy.copy(data)
iters = dataView.items(), addView.items()
chunks = McaStackView.izipChunkItems(*iters)
for (key, chunk), (addKey, add) in chunks:
chunk += add
for idxFullComplement in dataView.idxFullComplement:
numpy.testing.assert_array_equal(
data[idxFullComplement],
dataOrg[idxFullComplement])
if readonly:
numpy.testing.assert_array_equal(
data[idxFull], dataOrg[idxFull])
else:
numpy.testing.assert_array_equal(
data[idxFull], dataOrg[idxFull] + npAdd[idxFull])
def testfullChunkIndex(self):
for ndim in [2, 3, 4]:
shape = tuple(range(3, 3 + ndim))
data = numpy.zeros(shape, dtype=int)
for chunkAxes, axesOrder, nChunksTot in self._chunkIndexAxes(
shape, ndim):
for nChunksMax in range(nChunksTot + 2):
data[()] = 0
result = McaStackView.fullChunkIndex(shape,
nChunksMax,
chunkAxes=chunkAxes,
axesOrder=axesOrder)
chunkIndex, chunkAxes, axesOrder, nChunksMax2 = result
self.assertTrue(nChunksMax2 <= max(nChunksMax, 1))
for i, (idxChunk, idxShape,
nChunks) in enumerate(chunkIndex, 1):
data[idxChunk] += i
self.assertEqual(data[idxChunk].shape, idxShape)
self.assertTrue(nChunks <= nChunksMax2)
# Verify data coverage:
self.assertFalse((data == 0).any())
# Verify single element access and chunk access order:
arr = data.transpose(axesOrder[::-1] + chunkAxes).flatten()
lst1 = [k for k, g in itertools.groupby(arr)]
lst2 = list(range(1, i + 1))
self.assertEqual(lst1, lst2)
def _assertMaskedView(self, data):
mcaSlice = slice(2, -1)
isH5py = isinstance(data, h5py.Dataset)
for mcaAxis in range(data.ndim):
mask, indices, nmask = self._randomMask(data.shape, (mcaAxis, ),
None)
it = itertools.product([2, nmask // 3, nmask - 1, nmask + 1],
[mask, None])
for nMca, usedmask in it:
dataView = McaStackView.MaskedView(data,
mask=usedmask,
readonly=False,
mcaAxis=mcaAxis,
mcaSlice=mcaSlice,
nMca=nMca)
npAdd = numpy.arange(data.size).reshape(data.shape)
addView = McaStackView.MaskedView(npAdd,
mask=usedmask,
readonly=True,
mcaAxis=mcaAxis,
mcaSlice=mcaSlice,
nMca=nMca)
idxFull = dataView.idxFull
for readonly in [True, False]:
dataView.readonly = readonly
dataOrg = numpy.copy(data)
iters = dataView.items(), addView.items()
chunks = McaStackView.izipChunkItems(*iters)
for (key, chunk), (addKey, add) in chunks:
chunk += add
if isH5py:
_data = data[()]
else:
_data = data
for idxFullComplement in dataView.idxFullComplement:
numpy.testing.assert_array_equal(
_data[idxFullComplement],
dataOrg[idxFullComplement])
if readonly:
numpy.testing.assert_array_equal(
_data[idxFull], dataOrg[idxFull])
else:
numpy.testing.assert_array_equal(
_data[idxFull], dataOrg[idxFull] + npAdd[idxFull])
def testViewUtils(self):
n = 20
slices = [
slice(None),
slice(1, -2),
slice(8, 2, -1),
slice(0, n, 3),
slice(0, n, -2),
slice(n - 2, 2, -3),
slice(n - 1, None, -1),
slice(None, -2, 3)
]
lst = list(range(n))
for idx in slices:
idxn = McaStackView.sliceNormalize(idx, n)
self.assertEqual(lst[idx], lst[idxn])
for idx in slices:
self.assertEqual(McaStackView.sliceLen(idx, n), len(lst[idx]))
for idx in slices:
idxi = McaStackView.sliceReverse(idx, n)
self.assertEqual(lst[idx][::-1], lst[idxi])
for idx in slices:
idxc = McaStackView.sliceComplement(idx, n)
self.assertEqual(list(sorted(lst[idx] + idxc)), lst)
for idx in slices:
start, stop, step = idx.indices(n)
lst1 = list(range(start, stop, int(numpy.sign(step))))
it = McaStackView.chunkIndexGen(start, stop, step)
self.assertEqual(len(lst1), sum(ns for it, ns in it))
it = McaStackView.chunkIndexGen(start, stop, step)
lst2 = [i for idxc, ns in it for i in lst[idxc]]
self.assertEqual(lst1, lst2)
def testMaskedChunkIndex(self):
for ndim in [2, 3, 4]:
shape = tuple(range(3, 3 + ndim))
data = numpy.zeros(shape, dtype=int)
for chunkAxes, axesOrder, nChunksTot in self._chunkIndexAxes(
shape, ndim):
# Create Mask
mask, indices, nmask = self._randomMask(
shape, chunkAxes, axesOrder)
# Mask entire array
maskFull = numpy.zeros(shape, dtype=bool)
if mask is None:
maskFull[()] = False
nmask = 0
else:
indicesFull = [slice(None)] * ndim
if axesOrder:
for i, ind in zip(axesOrder, indices):
indicesFull[i] = ind
elif chunkAxes:
tmp = tuple(i for i in range(ndim)
if i not in chunkAxes)
for i, ind in zip(tmp, indices):
indicesFull[i] = ind
else:
indicesFull = indices
indicesFull = tuple(indicesFull)
maskFull[indicesFull] = True
for nChunksMax in [2, nmask // 3, nmask - 1, nmask + 1]:
for usedmask in [mask, None]:
data[()] = 0
self.assertFalse((data != 0).any())
chunkIndex, chunkAxes2, axesOrder, nChunksMax2 =\
McaStackView.maskedChunkIndex(shape, nChunksMax,
mask=usedmask,
chunkAxes=chunkAxes,
axesOrder=axesOrder)
for i, (idxChunk, idxShape,
nChunks) in enumerate(chunkIndex, 1):
data[idxChunk] += i
self.assertEqual(data[idxChunk].shape, idxShape)
self.assertTrue(nChunks <= nChunksMax2)
# Verify data coverage:
if usedmask is None:
self.assertFalse((data == 0).any())
else:
self.assertFalse((data[maskFull] == 0).any())
self.assertTrue((data[~maskFull] == 0).all())
# Verify single element access:
lst1 = numpy.unique(data).tolist()
lst2 = list(range(int(usedmask is None), i + 1))
self.assertEqual(lst1, lst2)
def _extractRois(self,
data,
x,
mcaAxis,
roiList=None,
roiDict=None,
outbuffer=None,
xAtMinMax=False):
nRois = len(roiList)
nRows = data.shape[0]
nColumns = data.shape[1]
if xAtMinMax:
roiShape = (nRois * 4, nRows, nColumns)
names = [None] * 4 * nRois
else:
roiShape = (nRois * 2, nRows, nColumns)
names = [None] * 2 * nRois
# Helper variables for roi calculation
idx = [None] * nRois # indices along axis=index for each ROI
xw = [None] * nRois # x-values for each ROI
iXMinList = [None] * nRois # min(xw) for each ROI
iXMaxList = [None] * nRois # max(xw) for each ROI
def idxraw(i):
return i
def idxnet(i):
return i + nRois
def idxmax(i):
return i + 2 * nRois
def idxmin(i):
return i + 3 * nRois
for j, roi in enumerate(roiList):
if roi == "ICR":
xw[j] = x
idx[j] = numpy.arange(len(x))
iXMinList[j] = idx[j][0]
iXMaxList[j] = idx[j][-1]
else:
roiFrom = roiDict[roi]["from"]
roiTo = roiDict[roi]["to"]
idx[j] = numpy.nonzero((roiFrom <= x) & (x <= roiTo))[0]
if len(idx[j]):
xw[j] = x[idx[j]]
iXMinList[j] = numpy.argmin(xw[j])
iXMaxList[j] = numpy.argmax(xw[j])
else:
xw[j] = None
names[idxraw(j)] = "ROI " + roi
names[idxnet(j)] = "ROI " + roi + " Net"
if xAtMinMax:
roiType = roiDict[roi]["type"]
names[idxmax(j)] = "ROI " + roi + (" %s at Max." % roiType)
names[idxmin(j)] = "ROI " + roi + (" %s at Min." % roiType)
# Allocate memory for result
roidtype = numpy.float
results = outbuffer.allocateMemory('roisum',
shape=roiShape,
dtype=roidtype,
labels=names,
dataAttrs=None,
groupAttrs={'default': True},
memtype='ram')
# Allocate memory of partial result
nMca = 2, 'MB'
_logger.debug('Process spectra in chunks of {}'.format(nMca))
datastack = McaStackView.FullView(data, mcaAxis=mcaAxis, nMca=nMca)
for (resultidx,
resultshape), chunk in datastack.items(keyType='select'):
for j, roi in enumerate(roiList):
# Calculate ROI sum
if xw[j] is None:
# no points in the ROI
rawSum = 0.0
netSum = 0.0
else:
roichunk = chunk[:, idx[j]]
rawSum = roichunk.sum(axis=1, dtype=numpy.float)
deltaX = xw[j][iXMaxList[j]] - xw[j][iXMinList[j]]
left = roichunk[:, iXMinList[j]]
right = roichunk[:, iXMaxList[j]]
deltaY = right - left
if abs(deltaX) > 0.0:
slope = deltaY / float(deltaX)
background = left * len(xw[j]) + slope * \
(xw[j] - xw[j][iXMinList[j]]).sum(dtype=numpy.float)
netSum = rawSum - background
else:
netSum = 0.0
rawSum = rawSum.reshape(resultshape)
netSum = netSum.reshape(resultshape)
results[idxraw(j)][resultidx] = rawSum # ROI sum
results[idxnet(
j)][resultidx] = netSum # ROI sum minus linear background
# Calculate x-value of the minimum and maximum within the ROI
if xAtMinMax:
if xw[j] is None:
# what can be the Min and the Max when there is nothing in the ROI?
_logger.warning("No Min. Max for ROI <%s>. Empty ROI" %
roi)
else:
maxImage = xw[j][numpy.argmax(roichunk, axis=1)]
results[idxmax(j)][resultidx] = maxImage.reshape(
resultshape)
minImage = xw[j][numpy.argmin(roichunk, axis=1)]
results[idxmin(j)][resultidx] = minImage.reshape(
resultshape)