def __init__(self, slot, start=None, stop=None, pslice=None):
super(SubRegion, self).__init__(slot)
if pslice != None or start is not None and stop is None and pslice is None:
if pslice is None:
pslice = start
shape = self.slot.meta.shape
if shape is None:
# Okay to use a shapeless slot if the key is bounded
# AND if the key has the correct length
assert slicingtools.is_bounded(pslice)
# Supply a dummy shape
shape = [0] * len(pslice)
self.start, self.stop = sliceToRoi(pslice, shape)
elif start is None and pslice is None:
self.start, self.stop = sliceToRoi(slice(None, None, None),
self.slot.meta.shape)
else:
self.start = TinyVector(start)
self.stop = TinyVector(stop)
self.dim = len(self.start)
for start, stop in zip(self.start, self.stop):
assert isinstance(
start,
(int, long,
numpy.integer)), "Roi contains non-integers: {}".format(self)
assert isinstance(
start,
(int, long,
numpy.integer)), "Roi contains non-integers: {}".format(self)
def execute(self, slot, subindex, roi, result):
key = roiToSlice(roi.start,roi.stop)
start = self.inputs["Start"].value
stop = self.inputs["Stop"].value
temp = tuple()
for i in xrange(len(start)):
if stop[i] - start[i] > 0:
temp += (stop[i]-start[i],)
readStart, readStop = sliceToRoi(key, temp)
newKey = ()
resultKey = ()
i = 0
i2 = 0
for kkk in xrange(len(start)):
e = stop[kkk] - start[kkk]
if e > 0:
newKey += (slice(start[i2] + readStart[i], start[i2] + readStop[i],None),)
resultKey += (slice(0,temp[i2],None),)
i +=1
else:
newKey += (slice(start[i2], start[i2], None),)
resultKey += (0,)
i2 += 1
res = self.inputs["Input"][newKey].allocate().wait()
result[:] = res[resultKey]
def propagateDirty(self, inputSlot, subindex, roi):
if inputSlot == self.Input:
numChannels = self.Input.meta.shape[1]
dirtyChannels = roi.stop[1] - roi.start[1]
# If all the input channels were dirty, the dirty output region is a contiguous block
if dirtyChannels == numChannels:
dirtyKey = list(roiToSlice(roi.start, roi.stop))
dirtyKey[1] = slice(None)
dirtyRoi = sliceToRoi(dirtyKey, self.Output.meta.shape)
self.Output.setDirty(dirtyRoi[0], dirtyRoi[1])
else:
# Only some input channels were dirty,
# so we must mark each dirty output region separately.
numFeatures = self.Output.meta.shape[1] // numChannels
for featureIndex in range(numFeatures):
startChannel = numChannels * featureIndex + roi.start[1]
stopChannel = startChannel + roi.stop[1]
dirtyRoi = copy.copy(roi)
dirtyRoi.start[1] = startChannel
dirtyRoi.stop[1] = stopChannel
self.Output.setDirty(dirtyRoi)
elif (inputSlot == self.SelectionMatrix or inputSlot == self.Scales
or inputSlot == self.FeatureIds
or inputSlot == self.ComputeIn2d):
self.Output.setDirty(slice(None))
else:
assert False, "Unknown dirty input slot."
def propagateDirty(self, inputSlot, subindex, roi):
if inputSlot == self.Input:
numChannels = self.Input.meta.shape[1]
dirtyChannels = roi.stop[1] - roi.start[1]
# If all the input channels were dirty, the dirty output region is a contiguous block
if dirtyChannels == numChannels:
dirtyKey = list(roiToSlice(roi.start, roi.stop))
dirtyKey[1] = slice(None)
dirtyRoi = sliceToRoi(dirtyKey, self.Output.meta.shape)
self.Output.setDirty(dirtyRoi[0], dirtyRoi[1])
else:
# Only some input channels were dirty,
# so we must mark each dirty output region separately.
numFeatures = self.Output.meta.shape[1] // numChannels
for featureIndex in range(numFeatures):
startChannel = numChannels * featureIndex + roi.start[1]
stopChannel = startChannel + roi.stop[1]
dirtyRoi = copy.copy(roi)
dirtyRoi.start[1] = startChannel
dirtyRoi.stop[1] = stopChannel
self.Output.setDirty(dirtyRoi)
elif (
inputSlot == self.SelectionMatrix
or inputSlot == self.Scales
or inputSlot == self.FeatureIds
or inputSlot == self.ComputeIn2d
):
self.Output.setDirty(slice(None))
else:
assert False, "Unknown dirty input slot."
def getSubOutSlot(self, slots, indexes, key, result):
outshape = self.outputs["Slices"][indexes[0]].shape
start,stop=roi.sliceToRoi(key,outshape)
oldstart,oldstop=start,stop
start=list(start)
stop=list(stop)
flag=self.inputs["AxisFlag"].value
indexAxis=self.inputs["Input"].axistags.index(flag)
start.pop(indexAxis)
stop.pop(indexAxis)
start.insert(indexAxis,indexes[0])
stop.insert(indexAxis,indexes[0])
newKey=roi.roiToSlice(numpy.array(start),numpy.array(stop))
ttt = self.inputs["Input"][newKey].allocate().wait()
result[:]=ttt[:]
def getOutSlot(self, slot, key, result):
start, stop = sliceToRoi(key, self.shape)
diff = stop-start
splitDim = numpy.argmax(diff[:-1])
splitPos = start[splitDim] + diff[splitDim] / 2
stop2 = stop.copy()
stop2[splitDim] = splitPos
start2 = start.copy()
start2[splitDim] = splitPos
destStart = start -start # zeros
destStop = stop - start
destStop2 = destStop.copy()
destStop2[splitDim] = diff[splitDim] / 2
destStart2 = destStart.copy()
destStart2[splitDim] = diff[splitDim] / 2
writeKey1 = roiToSlice(destStart,destStop2)
writeKey2 = roiToSlice(destStart2,destStop)
key1 = roiToSlice(start,stop2)
key2 = roiToSlice(start2,stop)
req1 = self.inputs["Input"][key1].writeInto(result[writeKey1])
req2 = self.inputs["Input"][key2].writeInto(result[writeKey2])
req1.wait()
req2.wait()
def setInSlot(self, slot, key, value):
if slot == self.inputs["Input"]:
ch = self._cacheHits
ch += 1
self._cacheHits = ch
start, stop = sliceToRoi(key, self.shape)
blockStart = numpy.ceil(1.0 * start / self._blockShape)
blockStop = numpy.floor(1.0 * stop / self._blockShape)
blockStop = numpy.where(stop == self.shape, self._dirtyShape, blockStop)
blockKey = roiToSlice(blockStart,blockStop)
if (self._blockState[blockKey] != 2).any():
start2 = blockStart * self._blockShape
stop2 = blockStop * self._blockShape
stop2 = numpy.minimum(stop2, self.shape)
key2 = roiToSlice(start2,stop2)
self._lock.acquire()
if self._cache is None:
self._allocateCache()
self._cache[key2] = value[roiToSlice(start2-start,stop2-start)]
self._blockState[blockKey] = self._dirtyState
self._blockQuery[blockKey] = None
self._lock.release()
#pass request on
#if not self._fixed:
# self.outputs["Output"][key] = value
if slot == self.inputs["fixAtCurrent"]:
self._fixed = value
assert 1==2
def getOutSlot(self, slot, key, result):
shape = self.inputs["Input"].shape
rstart, rstop = sliceToRoi(key, self.outputs["Output"]._shape)
rstart.append(0)
rstop.append(shape[-1])
rkey = roiToSlice(rstart, rstop)
img = self.inputs["Input"][rkey].allocate().wait()
stop = img.size
seg = []
for i in range(0, stop, img.shape[-1]):
curr_prob = -1
highest_class = -1
for c in range(img.shape[-1]):
prob = img.ravel()[i + c]
if prob > curr_prob:
curr_prob = prob
highest_class = c
assert highest_class != -1, "OpSegmentation: Strange classes/probabilities"
seg.append(highest_class)
seg = numpy.array(seg)
seg.resize(img.shape[:-1])
result[:] = seg[:]
def testDirtyPropagation(self):
opFeatures = self.opFeatures
dirtyRois = []
def handleDirty(slot, roi):
dirtyRois.append(roi)
opFeatures.OutputImage[0].notifyDirty(handleDirty)
# Change the matrix
selections = numpy.array([
[True, False, False, False, False, False, False], # Gaussian
[False, True, False, False, False, False, False], # L of G
[False, False, True, False, False, False, False], # ST EVs
[False, False, False, True, False, False, False], # H of G EVs
[False, False, False, False, False, False, False], # GGM
[False, False, False, False, False, False, False]
]) # Diff of G
print "About to change matrix."
opFeatures.SelectionMatrix.setValue(selections)
print "Matrix changed."
assert len(dirtyRois) == 1
assert (dirtyRois[0].start, dirtyRois[0].stop) == sliceToRoi(
slice(None), self.opFeatures.OutputImage[0].meta.shape)
def propagateDirty(self, slot, subindex, roi):
if slot == self.Input:
channelAxis = self.Input.meta.axistags.index('c')
numChannels = self.Input.meta.shape[channelAxis]
dirtyChannels = roi.stop[channelAxis] - roi.start[channelAxis]
# If all the input channels were dirty, the dirty output region is a contiguous block
if dirtyChannels == numChannels:
dirtyKey = roiToSlice(roi.start, roi.stop)
dirtyKey[channelAxis] = slice(None)
dirtyRoi = sliceToRoi(dirtyKey, self.Output.meta.shape)
self.Output.setDirty(dirtyRoi[0], dirtyRoi[1])
else:
# Only some input channels were dirty,
# so we must mark each dirty output region separately.
numFeatures = self.Output.meta.shape[channelAxis] / numChannels
for featureIndex in range(numFeatures):
startChannel = numChannels * featureIndex + roi.start[
channelAxis]
stopChannel = startChannel + roi.stop[channelAxis]
dirtyRoi = copy.copy(roi)
dirtyRoi.start[channelAxis] = startChannel
dirtyRoi.stop[channelAxis] = stopChannel
self.Output.setDirty(dirtyRoi)
elif (slot == self.Matrix or slot == self.Scales
or slot == self.FeatureIds):
self.Output.setDirty(slice(None))
else:
assert False, "Unknown dirty input slot."
def setInSlot(self, slot, key, value):
shape = self.inputs["shape"].value
eraseLabel = self.inputs["eraser"].value
neutralElement = 0
self.lock.acquire()
#fix slicing of single dimensions:
start, stop = sliceToRoi(key, shape, extendSingleton = False)
start = start.floor()
stop = stop.floor()
tempKey = roiToSlice(start-start, stop-start, hardBind = True)
stop += numpy.where(stop-start == 0,1,0)
key = roiToSlice(start,stop)
updateShape = tuple(stop-start)
update = self._denseArray[key].copy()
update[tempKey] = value
startRavel = numpy.ravel_multi_index(numpy.array(start, numpy.int32),shape)
#insert values into dict
updateNZ = numpy.nonzero(numpy.where(update != neutralElement,1,0))
updateNZRavelSmall = numpy.ravel_multi_index(updateNZ, updateShape)
if isinstance(value, numpy.ndarray):
valuesNZ = value.ravel()[updateNZRavelSmall]
else:
valuesNZ = value
updateNZRavel = numpy.ravel_multi_index(updateNZ, shape)
updateNZRavel += startRavel
self._denseArray.ravel()[updateNZRavel] = valuesNZ
valuesNZ = self._denseArray.ravel()[updateNZRavel]
self._denseArray.ravel()[updateNZRavel] = valuesNZ
td = blist.sorteddict(zip(updateNZRavel.tolist(),valuesNZ.tolist()))
self._sparseNZ.update(td)
#remove values to be deleted
updateNZ = numpy.nonzero(numpy.where(update == eraseLabel,1,0))
if len(updateNZ)>0:
updateNZRavel = numpy.ravel_multi_index(updateNZ, shape)
updateNZRavel += startRavel
self._denseArray.ravel()[updateNZRavel] = neutralElement
for index in updateNZRavel:
self._sparseNZ.pop(index)
self.lock.release()
self.outputs["Output"].setDirty(key)
def propagateDirty(self, slot, subindex, roi):
if slot == self.Input:
channelAxis = self.Input.meta.axistags.index('c')
numChannels = self.Input.meta.shape[channelAxis]
dirtyChannels = roi.stop[channelAxis] - roi.start[channelAxis]
# If all the input channels were dirty, the dirty output region is a contiguous block
if dirtyChannels == numChannels:
dirtyKey = roiToSlice(roi.start, roi.stop)
dirtyKey[channelAxis] = slice(None)
dirtyRoi = sliceToRoi(dirtyKey, self.Output.meta.shape)
self.Output.setDirty(dirtyRoi[0], dirtyRoi[1])
else:
# Only some input channels were dirty,
# so we must mark each dirty output region separately.
assert sys.version_info.major == 2, "Alert! This function has not been tested under python 3. "\
"please remove this assertion, and be wary of any strange behavior you encounter"
numFeatures = self.Output.meta.shape[channelAxis] // numChannels
for featureIndex in range(numFeatures):
startChannel = numChannels * featureIndex + roi.start[
channelAxis]
stopChannel = startChannel + roi.stop[channelAxis]
dirtyRoi = copy.copy(roi)
dirtyRoi.start[channelAxis] = startChannel
dirtyRoi.stop[channelAxis] = stopChannel
self.Output.setDirty(dirtyRoi)
elif (slot == self.Matrix or slot == self.Scales
or slot == self.FeatureIds):
self.Output.setDirty(slice(None))
else:
assert False, "Unknown dirty input slot."
def propagateDirty(self,slot,subindex,roi):
if slot == self.Input:
channelAxis = self.Input.meta.axistags.index('c')
numChannels = self.Input.meta.shape[channelAxis]
dirtyChannels = roi.stop[channelAxis] - roi.start[channelAxis]
# If all the input channels were dirty, the dirty output region is a contiguous block
if dirtyChannels == numChannels:
dirtyKey = roiToSlice(roi.start, roi.stop)
dirtyKey[channelAxis] = slice(None)
dirtyRoi = sliceToRoi(dirtyKey, self.Output.meta.shape)
self.Output.setDirty(dirtyRoi[0], dirtyRoi[1])
else:
# Only some input channels were dirty,
# so we must mark each dirty output region separately.
numFeatures = self.Output.meta.shape[channelAxis] / numChannels
for featureIndex in range(numFeatures):
startChannel = numChannels*featureIndex + roi.start[channelAxis]
stopChannel = startChannel + roi.stop[channelAxis]
dirtyRoi = copy.copy(roi)
dirtyRoi.start[channelAxis] = startChannel
dirtyRoi.stop[channelAxis] = stopChannel
self.Output.setDirty(dirtyRoi)
elif (slot == self.Matrix
or slot == self.Scales
or slot == self.FeatureIds):
self.Output.setDirty(slice(None))
else:
assert False, "Unknown dirty input slot."
def execute(self, slot, subindex, roi, result):
t = time.time()
assert slot == self.Output
key = roi.toSlice()
start, stop = sliceToRoi(key, self.shape)
roishape = numpy.array(stop) - numpy.array(start)
max_dist_squared = sys.maxsize
index = 0
for i, blockshape in enumerate(self._blockshapes):
blockshape = numpy.array(blockshape)
diff = roishape - blockshape
diffsquared = diff * diff
distance_squared = numpy.sum(diffsquared)
if distance_squared < max_dist_squared:
index = i
max_dist_squared = distance_squared
op = self._innerOps[index]
op.outputs["Output"][key].writeInto(result).wait()
self.logger.debug("read %r took %f msec." % (roi.pprint(), 1000.0 *
(time.time() - t)))
def execute(self, slot, subindex, rroi, result):
key = roiToSlice(rroi.start, rroi.stop)
index = subindex[0]
#print "SLICER: key", key, "indexes[0]", indexes[0], "result", result.shape
start, stop = roi.sliceToRoi(key,
self.outputs["Slices"][index].meta.shape)
start = list(start)
stop = list(stop)
flag = self.inputs["AxisFlag"].value
indexAxis = self.inputs["Input"].meta.axistags.index(flag)
start.insert(indexAxis, index)
stop.insert(indexAxis, index)
newKey = roi.roiToSlice(numpy.array(start), numpy.array(stop))
ttt = self.inputs["Input"][newKey].wait()
writeKey = [slice(None, None, None) for k in key]
writeKey.insert(indexAxis, 0)
writeKey = tuple(writeKey)
return ttt[writeKey] #+ (0,)]
def execute(self, slot, subindex, roi, result):
if slot is self.Output:
result = self.LabelImage.get(roi).wait()
if not self.Parameters.ready():
raise Exception("Parameter slot is not ready")
parameters = self.Parameters.value
t_start = roi.start[0]
t_end = roi.stop[0]
for t in range(t_start, t_end):
if ('time_range' in parameters and t <= parameters['time_range'][-1] and t >= parameters['time_range'][
0]) and len(self.label2color) > t:
result[t - t_start, ..., 0] = relabel(result[t - t_start, ..., 0], self.label2color[t])
else:
result[t - t_start, ...] = 0
return result
elif slot == self.AllBlocks:
# if nothing was computed, return empty list
if len(self.label2color) == 0:
result[0] = []
return result
all_block_rois = []
shape = self.Output.meta.shape
# assumes t,x,y,z,c
slicing = [slice(None), ] * 5
for t in range(shape[0]):
slicing[0] = slice(t, t + 1)
all_block_rois.append(sliceToRoi(slicing, shape))
result[0] = all_block_rois
return result
def _serialize(self, group, name, slot):
logger.debug("Serializing BlockSlot: {}".format( self.name ))
mygroup = group.create_group(name)
num = len(self.blockslot)
for index in range(num):
subname = self.subname.format(index)
subgroup = mygroup.create_group(subname)
nonZeroBlocks = self.blockslot[index].value
for blockIndex, slicing in enumerate(nonZeroBlocks):
block = self.slot[index][slicing].wait()
blockName = 'block{:04d}'.format(blockIndex)
if self._shrink_to_bb:
nonzero_coords = numpy.nonzero(block)
if len(nonzero_coords[0]) > 0:
block_start = sliceToRoi( slicing, (0,)*len(slicing) )[0]
block_bounding_box_start = numpy.array( map( numpy.min, nonzero_coords ) )
block_bounding_box_stop = 1 + numpy.array( map( numpy.max, nonzero_coords ) )
block_slicing = roiToSlice( block_bounding_box_start, block_bounding_box_stop )
bounding_box_roi = numpy.array([block_bounding_box_start, block_bounding_box_stop])
bounding_box_roi += block_start
# Overwrite the vars that are written to the file
slicing = roiToSlice(*bounding_box_roi)
block = block[block_slicing]
subgroup.create_dataset(blockName, data=block)
subgroup[blockName].attrs['blockSlice'] = slicingToString(slicing)
def setInSlot(self, slot, key, value):
start, stop = sliceToRoi(key, self.shape)
blockStart = (1.0 * start / self._blockShape).floor()
blockStop = (1.0 * stop / self._blockShape).ceil()
blockStop = numpy.where(stop == self.shape, self._dirtyShape, blockStop)
blockKey = roiToSlice(blockStart,blockStop)
innerBlocks = self._blockNumbers[blockKey]
for b_ind in innerBlocks.ravel():
offset = self._blockShape*self._flatBlockIndices[b_ind]
bigstart = numpy.maximum(offset, start)
bigstop = numpy.minimum(offset + self._blockShape, stop)
smallstart = bigstart-offset
smallstop = bigstop - offset
bigkey = roiToSlice(bigstart-start, bigstop-start)
smallkey = roiToSlice(smallstart, smallstop)
if not b_ind in self._labelers:
self._labelers[b_ind]=OpSparseLabelArray(self)
self._labelers[b_ind].inputs["shape"].setValue(self._blockShape)
self._labelers[b_ind].inputs["eraser"].setValue(self.inputs["eraser"].value)
self._labelers[b_ind].inputs["deleteLabel"].setValue(self.inputs["deleteLabel"])
self._labelers[b_ind].inputs["Input"][smallkey] = value[tuple(bigkey)].squeeze()
self.outputs["Output"].setDirty(key)
def __init__(self, slot, start = None, stop = None, pslice = None):
super(SubRegion,self).__init__(slot)
shape = None
if slot is not None:
shape = slot.meta.shape
if pslice != None or start is not None and stop is None and pslice is None:
if pslice is None:
pslice = start
if shape is None:
# Okay to use a shapeless slot if the key is bounded
# AND if the key has the correct length
assert slicingtools.is_bounded(pslice)
# Supply a dummy shape
shape = [0] * len(pslice)
self.start, self.stop = sliceToRoi(pslice,shape)
elif start is None and pslice is None:
assert shape is not None, "Can't create a default subregion without a slot and a shape."
self.start, self.stop = roiFromShape(shape)
else:
self.start = TinyVector(start)
self.stop = TinyVector(stop)
self.dim = len(self.start)
for start, stop in zip(self.start, self.stop):
assert isinstance(start, (int, long, numpy.integer)), "Roi contains non-integers: {}".format( self )
assert isinstance(start, (int, long, numpy.integer)), "Roi contains non-integers: {}".format( self )
def __init__(self, slot, start=None, stop=None, pslice=None):
super(SubRegion, self).__init__(slot)
shape = None
if slot is not None:
shape = slot.meta.shape
if pslice != None or start is not None and stop is None and pslice is None:
if pslice is None:
pslice = start
if shape is None:
# Okay to use a shapeless slot if the key is bounded
# AND if the key has the correct length
assert slicingtools.is_bounded(pslice)
# Supply a dummy shape
shape = [sl.stop for sl in pslice]
self.start, self.stop = sliceToRoi(pslice, shape)
elif start is None and pslice is None:
assert shape is not None, "Can't create a default subregion without a slot and a shape."
self.start, self.stop = roiFromShape(shape)
else:
self.start = TinyVector(start)
self.stop = TinyVector(stop)
self.dim = len(self.start)
for start, stop in zip(self.start, self.stop):
assert isinstance(start, (int, numpy.integer)), "Roi contains non-integers: {}".format(self)
assert isinstance(start, (int, numpy.integer)), "Roi contains non-integers: {}".format(self)
def testBasic(self):
"""
Requires access to the Internet...
"""
testConfig0 = """
{
"_schema_name" : "RESTful-volume-description",
"_schema_version" : 1.0,
"name" : "Bock11-level0",
"format" : "hdf5",
"axes" : "zyx",
"##NOTE":"The first z-slice of the bock dataset is 2917, so the origin_offset must be at least 2917",
"origin_offset" : [2917, 50000, 50000],
"bounds" : [4156, 135424, 119808],
"dtype" : "numpy.uint8",
"url_format" : "http://openconnecto.me/emca/bock11/hdf5/0/{x_start},{x_stop}/{y_start},{y_stop}/{z_start},{z_stop}/",
"hdf5_dataset" : "cube"
}
"""
testConfig4 = """
{
"_schema_name" : "RESTful-volume-description",
"_schema_version" : 1.0,
"name" : "Bock11-level4",
"format" : "hdf5",
"axes" : "zyx",
"##NOTE":"The first z-slice of the bock dataset is 2917, so the origin_offset must be at least 2917",
"origin_offset" : [2917, 50000, 50000],
"bounds" : [4156, 8704, 7680],
"dtype" : "numpy.uint8",
"url_format" : "http://openconnecto.me/emca/bock11/hdf5/4/{x_start},{x_stop}/{y_start},{y_stop}/{z_start},{z_stop}/",
"hdf5_dataset" : "cube"
}
"""
# Create the description file.
tempDir = tempfile.mkdtemp()
descriptionFilePath = os.path.join(tempDir, 'desc.json')
with open(descriptionFilePath, 'w') as descFile:
descFile.write( testConfig0 )
# Create the volume object
volume = RESTfulVolume( descriptionFilePath )
#slicing = numpy.s_[0:100, 4000:4200, 4000:4200]
slicing = numpy.s_[0:25, 50000:50050, 50000:50075]
roi = sliceToRoi( slicing, volume.description.shape )
outputFile = os.path.join(tempDir, 'volume.h5')
datasetPath = outputFile + '/cube'
logger.debug("Downloading subvolume to: {}".format( datasetPath ))
volume.downloadSubVolume(roi, datasetPath)
with h5py.File(outputFile, 'r') as hdf5File:
data = hdf5File['cube']
assert data.shape == ( 25, 50, 75 )
shutil.rmtree(tempDir)
def execute(self, slot, subindex, roi, result):
key = roiToSlice(roi.start,roi.stop)
start = self.inputs["Start"].value
stop = self.inputs["Stop"].value
temp = tuple()
for i in xrange(len(start)):
if stop[i] - start[i] > 0:
temp += (stop[i]-start[i],)
readStart, readStop = sliceToRoi(key, temp)
newKey = ()
i = 0
i2 = 0
for kkk in xrange(len(start)):
e = stop[kkk] - start[kkk]
if e > 0:
newKey += (slice(start[i2] + readStart[i], start[i2] + readStop[i],None),)
i +=1
else:
newKey += (slice(start[i2], start[i2], None),)
i2 += 1
self.inputs["Input"][newKey].writeInto(result).wait()
return result
def execute(self, slot, subindex, rroi, result):
key = roiToSlice(rroi.start, rroi.stop)
index = subindex[0]
# Index of the input slice this data will come from.
sliceIndex = self.getSliceIndexes()[index]
outshape = self.Slices[index].meta.shape
start, stop = roi.sliceToRoi(key, outshape)
start = list(start)
stop = list(stop)
flag = self.AxisFlag.value
indexAxis = self.Input.meta.axistags.index(flag)
start.pop(indexAxis)
stop.pop(indexAxis)
start.insert(indexAxis, sliceIndex)
stop.insert(indexAxis, sliceIndex)
newKey = roi.roiToSlice(numpy.array(start), numpy.array(stop))
self.Input[newKey].writeInto(result).wait()
return result
def getSubOutSlot(self, slots, indexes, key, result):
#print "SLICER: key", key, "indexes[0]", indexes[0], "result", result.shape
start,stop=roi.sliceToRoi(key,self.outputs["Slices"][indexes[0]].shape)
oldstart,oldstop=start,stop
start=list(start)
stop=list(stop)
flag=self.inputs["AxisFlag"].value
indexAxis=self.inputs["Input"].axistags.index(flag)
start.insert(indexAxis,indexes[0])
stop.insert(indexAxis,indexes[0])
newKey=roi.roiToSlice(numpy.array(start),numpy.array(stop))
ttt = self.inputs["Input"][newKey].allocate().wait()
writeKey = [slice(None, None, None) for k in key]
writeKey.insert(indexAxis, 0)
writeKey = tuple(writeKey)
result[:]=ttt[writeKey ]#+ (0,)]
def testBasic(self):
"""
Requires access to the Internet...
"""
testConfig0 = """
{
"_schema_name" : "RESTful-volume-description",
"_schema_version" : 1.0,
"name" : "Bock11-level0",
"format" : "hdf5",
"axes" : "zyx",
"##NOTE":"The first z-slice of the bock dataset is 2917, so the origin_offset must be at least 2917",
"origin_offset" : [2917, 50000, 50000],
"bounds" : [4156, 135424, 119808],
"dtype" : "numpy.uint8",
"url_format" : "http://openconnecto.me/emca/bock11/hdf5/0/{x_start},{x_stop}/{y_start},{y_stop}/{z_start},{z_stop}/",
"hdf5_dataset" : "cube"
}
"""
testConfig4 = """
{
"_schema_name" : "RESTful-volume-description",
"_schema_version" : 1.0,
"name" : "Bock11-level4",
"format" : "hdf5",
"axes" : "zyx",
"##NOTE":"The first z-slice of the bock dataset is 2917, so the origin_offset must be at least 2917",
"origin_offset" : [2917, 50000, 50000],
"bounds" : [4156, 8704, 7680],
"dtype" : "numpy.uint8",
"url_format" : "http://openconnecto.me/emca/bock11/hdf5/4/{x_start},{x_stop}/{y_start},{y_stop}/{z_start},{z_stop}/",
"hdf5_dataset" : "cube"
}
"""
# Create the description file.
tempDir = tempfile.mkdtemp()
descriptionFilePath = os.path.join(tempDir, 'desc.json')
with open(descriptionFilePath, 'w') as descFile:
descFile.write(testConfig0)
# Create the volume object
volume = RESTfulVolume(descriptionFilePath)
#slicing = numpy.s_[0:100, 4000:4200, 4000:4200]
slicing = numpy.s_[0:25, 50000:50050, 50000:50075]
roi = sliceToRoi(slicing, volume.description.shape)
outputFile = os.path.join(tempDir, 'volume.h5')
datasetPath = outputFile + '/cube'
logger.debug("Downloading subvolume to: {}".format(datasetPath))
volume.downloadSubVolume(roi, datasetPath)
with h5py.File(outputFile, 'r') as hdf5File:
data = hdf5File['cube']
assert data.shape == (25, 50, 75)
shutil.rmtree(tempDir)
def propagateDirty(self,slot,subindex,roi):
if slot == self.Input:
channelAxis = self.Input.meta.axistags.index('c')
numChannels = self.Input.meta.shape[channelAxis]
dirtyChannels = roi.stop[channelAxis] - roi.start[channelAxis]
# If all the input channels were dirty, the dirty output region is a contiguous block
if dirtyChannels == numChannels:
dirtyKey = roiToSlice(roi.start, roi.stop)
dirtyKey[channelAxis] = slice(None)
dirtyRoi = sliceToRoi(dirtyKey, self.Output.meta.shape)
self.Output.setDirty(dirtyRoi[0], dirtyRoi[1])
else:
# Only some input channels were dirty,
# so we must mark each dirty output region separately.
assert sys.version_info.major == 2, "Alert! This function has not been tested under python 3. "\
"please remove this assertion, and be wary of any strange behavior you encounter"
numFeatures = self.Output.meta.shape[channelAxis] // numChannels
for featureIndex in range(numFeatures):
startChannel = numChannels*featureIndex + roi.start[channelAxis]
stopChannel = startChannel + roi.stop[channelAxis]
dirtyRoi = copy.copy(roi)
dirtyRoi.start[channelAxis] = startChannel
dirtyRoi.stop[channelAxis] = stopChannel
self.Output.setDirty(dirtyRoi)
elif (slot == self.Matrix
or slot == self.Scales
or slot == self.FeatureIds):
self.Output.setDirty(slice(None))
else:
assert False, "Unknown dirty input slot."
def testDirtyPropagation(self):
opFeatures = self.opFeatures
dirtyRois = []
def handleDirty(slot, roi):
dirtyRois.append(roi)
opFeatures.OutputImage[0].notifyDirty(handleDirty)
# Change the matrix
selections = numpy.array(
[
[True, False, False, False, False, False, False], # Gaussian
[False, True, False, False, False, False, False], # L of G
[False, False, True, False, False, False, False], # ST EVs
[False, False, False, True, False, False, False], # H of G EVs
[False, False, False, False, False, False, False], # GGM
[False, False, False, False, False, False, False],
]
) # Diff of G
opFeatures.SelectionMatrix.setValue(selections)
assert len(dirtyRois) == 1
assert (dirtyRois[0].start, dirtyRois[0].stop) == sliceToRoi(
slice(None), self.opFeatures.OutputImage[0].meta.shape
)
def test_9_TestView(self):
"""
Load some of the dataset again; this time with an offset view.
Note: The original blockwise fileset must be closed before this test starts.
"""
# Create a copy of the original description, but specify a translated (and smaller) view
desc = BlockwiseFileset.readDescription(self.description_path)
desc.view_origin = [0, 300, 200, 100, 0]
desc.view_shape = [1, 50, 50, 50, 1]
offsetConfigPath = self.description_path + '_offset'
BlockwiseFileset.writeDescription(offsetConfigPath, desc)
# Open the fileset using the special description file
bfs = BlockwiseFileset( offsetConfigPath, 'r' )
try:
assert (bfs.description.view_origin == desc.view_origin).all()
assert (bfs.description.view_shape == desc.view_shape).all()
# Read some data
logger.debug( "Reading data..." )
disk_slicing = numpy.s_[:, 300:350, 200:250, 100:150, :]
view_slicing = numpy.s_[:, 0:50, 0:50, 0:50, :]
roi = sliceToRoi( view_slicing, self.dataShape )
roiShape = roi[1] - roi[0]
read_data = numpy.zeros( tuple(roiShape), dtype=numpy.uint8 )
bfs.readData( roi, read_data )
# The data we read should match the correct part of the original dataset.
logger.debug( "Checking data..." )
assert self.data[disk_slicing].shape == read_data.shape
assert (self.data[disk_slicing] == read_data).all(), "Data didn't match."
finally:
bfs.close()
def execute(self, slot, subindex, rroi, result):
key = roiToSlice(rroi.start, rroi.stop)
index = subindex[0]
# Index of the input slice this data will come from.
sliceIndex = self.getSliceIndexes()[index]
outshape = self.outputs["Slices"][index].meta.shape
start,stop=roi.sliceToRoi(key,outshape)
oldstart,oldstop=start,stop
start=list(start)
stop=list(stop)
flag=self.inputs["AxisFlag"].value
indexAxis=self.inputs["Input"].meta.axistags.index(flag)
start.pop(indexAxis)
stop.pop(indexAxis)
start.insert(indexAxis, sliceIndex)
stop.insert(indexAxis, sliceIndex)
newKey=roi.roiToSlice(numpy.array(start),numpy.array(stop))
ttt = self.inputs["Input"][newKey].allocate().wait()
return ttt[:]
def execute(self, slot, subindex, roi, result):
if slot is self.Output:
result = self.LabelImage.get(roi).wait()
if not self.Parameters.ready():
raise Exception("Parameter slot is not ready")
parameters = self.Parameters.value
t_start = roi.start[0]
t_end = roi.stop[0]
for t in range(t_start, t_end):
if ('time_range' in parameters and t <= parameters['time_range'][-1] and t >= parameters['time_range'][
0]) and len(self.label2color) > t:
result[t - t_start, ..., 0] = relabel(result[t - t_start, ..., 0], self.label2color[t])
else:
result[t - t_start, ...] = 0
return result
elif slot == self.AllBlocks:
# if nothing was computed, return empty list
if len(self.label2color) == 0:
result[0] = []
return result
all_block_rois = []
shape = self.Output.meta.shape
# assumes t,x,y,z,c
slicing = [slice(None), ] * 5
for t in range(shape[0]):
slicing[0] = slice(t, t + 1)
all_block_rois.append(sliceToRoi(slicing, shape))
result[0] = all_block_rois
return result
def execute(self, slot, subindex, rroi, result):
key = roiToSlice(rroi.start, rroi.stop)
index = subindex[0]
# Index of the input slice this data will come from.
sliceIndex = self.getSliceIndexes()[index]
outshape = self.Slices[index].meta.shape
start, stop = roi.sliceToRoi(key, outshape)
start = list(start)
stop = list(stop)
flag = self.AxisFlag.value
indexAxis = self.Input.meta.axistags.index(flag)
start.pop(indexAxis)
stop.pop(indexAxis)
start.insert(indexAxis, sliceIndex)
stop.insert(indexAxis, sliceIndex)
newKey = roi.roiToSlice(numpy.array(start), numpy.array(stop))
self.Input[newKey].writeInto(result).wait()
return result
def _serialize(self, group, name, slot):
logger.debug("Serializing BlockSlot: {}".format(self.name))
mygroup = group.create_group(name)
num = len(self.blockslot)
for index in range(num):
subname = self.subname.format(index)
subgroup = mygroup.create_group(subname)
nonZeroBlocks = self.blockslot[index].value
for blockIndex, slicing in enumerate(nonZeroBlocks):
block = self.slot[index][slicing].wait()
blockName = 'block{:04d}'.format(blockIndex)
if self._shrink_to_bb:
nonzero_coords = numpy.nonzero(block)
if len(nonzero_coords[0]) > 0:
block_start = sliceToRoi(slicing,
(0, ) * len(slicing))[0]
block_bounding_box_start = numpy.array(
map(numpy.min, nonzero_coords))
block_bounding_box_stop = 1 + numpy.array(
map(numpy.max, nonzero_coords))
block_slicing = roiToSlice(block_bounding_box_start,
block_bounding_box_stop)
bounding_box_roi = numpy.array([
block_bounding_box_start, block_bounding_box_stop
])
bounding_box_roi += block_start
# Overwrite the vars that are written to the file
slicing = roiToSlice(*bounding_box_roi)
block = block[block_slicing]
subgroup.create_dataset(blockName, data=block)
subgroup[blockName].attrs['blockSlice'] = slicingToString(
slicing)
def execute(self, slot, subindex, roi, result):
key = roi.toSlice()
self.lock.acquire()
assert(self.inputs["eraser"].ready() == True and self.inputs["shape"].ready() == True and self.inputs["blockShape"].ready()==True), \
"OpBlockedSparseLabelArray: One of the neccessary input slots is not ready: shape: %r, eraser: %r" % \
(self.inputs["eraser"].ready(), self.inputs["shape"].ready())
if slot.name == "Output":
#result[:] = self._denseArray[key]
#find the block key
start, stop = sliceToRoi(key, self._cacheShape)
blockStart = (1.0 * start / self._blockShape).floor()
blockStop = (1.0 * stop / self._blockShape).ceil()
blockKey = roiToSlice(blockStart, blockStop)
innerBlocks = self._blockNumbers[blockKey]
for b_ind in innerBlocks.ravel():
#which part of the original key does this block fill?
offset = self._blockShape * self._flatBlockIndices[b_ind]
bigstart = numpy.maximum(offset, start)
bigstop = numpy.minimum(offset + self._blockShape, stop)
smallstart = bigstart - offset
smallstop = bigstop - offset
bigkey = roiToSlice(bigstart - start, bigstop - start)
smallkey = roiToSlice(smallstart, smallstop)
if not b_ind in self._labelers or not self._labelers[
b_ind].Output.ready():
result[bigkey] = 0
else:
try:
labeler = self._labelers[b_ind]
denseArray = labeler._denseArray[smallkey]
result[bigkey] = denseArray
except:
logger.error(
"Exception in OpBlockedSparseLabelArray.execute, probably due to simultaneous calls to setInSlot() and execute()"
)
logger.error("labeler = {}".format(labeler))
logger.error("denseArray = {}".format(denseArray))
logger.error("result = {}".format(result))
raise
elif slot.name == "nonzeroValues":
nzvalues = set()
for l in self._labelers.values():
nzvalues |= set(l._sparseNZ.values())
result[0] = numpy.array(list(nzvalues))
elif slot.name == "nonzeroCoordinates":
assert False, "not supported yet"
#result[0] = numpy.array(self._sparseNZ.keys())
elif slot.name == "nonzeroBlocks":
#we only return all non-zero blocks, no keys
result[0] = self._get_nonzero_blocks()
elif slot.name == "maxLabel":
result[0] = self._maxLabel
self.lock.release()
return result
def execute(self, slot, subindex, rroi, result):
key = roiToSlice(rroi.start,rroi.stop)
cnt = 0
written = 0
start, stop = roi.sliceToRoi(key, self.outputs["Output"].meta.shape)
assert (stop<=self.outputs["Output"].meta.shape).all()
#axisindex = self.inputs["AxisIndex"].value
flag = self.inputs["AxisFlag"].value
axisindex = self.outputs["Output"].meta.axistags.index(flag)
#ugly-ugly-ugly
oldkey = list(key)
oldkey.pop(axisindex)
#print "STACKER: ", flag, axisindex
#print "requesting an outslot from stacker:", key, result.shape
#print "input slots total: ", len(self.inputs['Images'])
requests = []
pool = RequestPool()
for i, inSlot in enumerate(self.inputs['Images']):
req = None
inTagKeys = [ax.key for ax in inSlot.meta.axistags]
if flag in inTagKeys:
slices = inSlot.meta.shape[axisindex]
if cnt + slices >= start[axisindex] and start[axisindex]-cnt<slices and start[axisindex]+written<stop[axisindex]:
begin = 0
if cnt < start[axisindex]:
begin = start[axisindex] - cnt
end = slices
if cnt + end > stop[axisindex]:
end -= cnt + end - stop[axisindex]
key_ = copy.copy(oldkey)
key_.insert(axisindex, slice(begin, end, None))
reskey = [slice(None, None, None) for x in range(len(result.shape))]
reskey[axisindex] = slice(written, written+end-begin, None)
req = inSlot[tuple(key_)].writeInto(result[tuple(reskey)])
written += end - begin
cnt += slices
else:
if cnt>=start[axisindex] and start[axisindex] + written < stop[axisindex]:
#print "key: ", key, "reskey: ", reskey, "oldkey: ", oldkey
#print "result: ", result.shape, "inslot:", inSlot.meta.shape
reskey = [slice(None, None, None) for s in oldkey]
reskey.insert(axisindex, written)
destArea = result[tuple(reskey)]
req = inSlot[tuple(oldkey)].writeInto(destArea)
written += 1
cnt += 1
if req is not None:
pool.add(req)
pool.wait()
pool.clean()
def test_1_SingleDownload(self):
volume = RESTfulBlockwiseFileset( self.descriptionFilePath )
slicing = numpy.s_[0:20, 0:20, 0:20]
roi = sliceToRoi(slicing, volume.description.shape)
data = volume.readData( roi )
assert data.shape == (20,20,20)
assert volume.getBlockStatus( ([0,0,0]) ) == BlockwiseFileset.BLOCK_AVAILABLE
def test_4_OffsetDownload(self):
volume = RESTfulBlockwiseFileset( self.descriptionFilePath )
slicing = numpy.s_[20:40, 20:40, 20:40]
roi = sliceToRoi(slicing, volume.description.shape)
data = volume.readData( roi )
assert data.shape == (20,20,20)
assert volume.getBlockStatus( ([20,20,20]) ) == BlockwiseFileset.BLOCK_AVAILABLE
offsetVolume = RESTfulBlockwiseFileset( self.descriptionFilePath_offset )
offsetSlicing = numpy.s_[20:40, 0:20, 20:40] # Note middle slice is offset (see view_origin in setupClass)
offsetRoi = sliceToRoi(offsetSlicing, offsetVolume.description.shape)
offsetData = offsetVolume.readData( offsetRoi )
assert offsetData.shape == (20,20,20)
assert offsetVolume.getBlockStatus( ([20,0,20]) ) == BlockwiseFileset.BLOCK_AVAILABLE
# Data should be the same
assert (offsetData == data).all()
def test_4_OffsetDownload(self):
volume = RESTfulBlockwiseFileset( self.descriptionFilePath )
slicing = numpy.s_[20:40, 20:40, 20:40]
roi = sliceToRoi(slicing, volume.description.shape)
data = volume.readData( roi )
assert data.shape == (20,20,20)
assert volume.getBlockStatus( ([20,20,20]) ) == BlockwiseFileset.BLOCK_AVAILABLE
offsetVolume = RESTfulBlockwiseFileset( self.descriptionFilePath_offset )
offsetSlicing = numpy.s_[20:40, 0:20, 20:40] # Note middle slice is offset (see view_origin in setupClass)
offsetRoi = sliceToRoi(offsetSlicing, offsetVolume.description.shape)
offsetData = offsetVolume.readData( offsetRoi )
assert offsetData.shape == (20,20,20)
assert offsetVolume.getBlockStatus( ([20,0,20]) ) == BlockwiseFileset.BLOCK_AVAILABLE
# Data should be the same
assert (offsetData == data).all()
def execute(self, slot, subindex, rroi, result):
key = roiToSlice(rroi.start,rroi.stop)
cnt = 0
written = 0
start, stop = roi.sliceToRoi(key, self.outputs["Output"].meta.shape)
assert (stop<=self.outputs["Output"].meta.shape).all()
#axisindex = self.inputs["AxisIndex"].value
flag = self.inputs["AxisFlag"].value
axisindex = self.outputs["Output"].meta.axistags.index(flag)
#ugly-ugly-ugly
oldkey = list(key)
oldkey.pop(axisindex)
#print "STACKER: ", flag, axisindex
#print "requesting an outslot from stacker:", key, result.shape
#print "input slots total: ", len(self.inputs['Images'])
requests = []
pool = RequestPool()
for i, inSlot in enumerate(self.inputs['Images']):
req = None
inTagKeys = [ax.key for ax in inSlot.meta.axistags]
if flag in inTagKeys:
slices = inSlot.meta.shape[axisindex]
if cnt + slices >= start[axisindex] and start[axisindex]-cnt<slices and start[axisindex]+written<stop[axisindex]:
begin = 0
if cnt < start[axisindex]:
begin = start[axisindex] - cnt
end = slices
if cnt + end > stop[axisindex]:
end -= cnt + end - stop[axisindex]
key_ = copy.copy(oldkey)
key_.insert(axisindex, slice(begin, end, None))
reskey = [slice(None, None, None) for x in range(len(result.shape))]
reskey[axisindex] = slice(written, written+end-begin, None)
req = inSlot[tuple(key_)].writeInto(result[tuple(reskey)])
written += end - begin
cnt += slices
else:
if cnt>=start[axisindex] and start[axisindex] + written < stop[axisindex]:
#print "key: ", key, "reskey: ", reskey, "oldkey: ", oldkey
#print "result: ", result.shape, "inslot:", inSlot.meta.shape
reskey = [slice(None, None, None) for s in oldkey]
reskey.insert(axisindex, written)
destArea = result[tuple(reskey)]
req = inSlot[tuple(oldkey)].writeInto(destArea)
written += 1
cnt += 1
if req is not None:
pool.add(req)
pool.wait()
pool.clean()
def test_1_SingleDownload(self):
volume = RESTfulBlockwiseFileset( self.descriptionFilePath )
slicing = numpy.s_[0:20, 0:20, 0:20]
roi = sliceToRoi(slicing, volume.description.shape)
data = volume.readData( roi )
assert data.shape == (20,20,20)
assert volume.getBlockStatus( ([0,0,0]) ) == BlockwiseFileset.BLOCK_AVAILABLE
def execute(self, slot, roi, result):
key = roi.toSlice()
self.lock.acquire()
assert(self.inputs["eraser"].connected() == True and self.inputs["shape"].connected() == True and self.inputs["blockShape"].connected()==True), \
"OpDenseSparseArray: One of the neccessary input slots is not connected: shape: %r, eraser: %r" % \
(self.inputs["eraser"].connected(), self.inputs["shape"].connected())
if slot.name == "Output":
#result[:] = self._denseArray[key]
#find the block key
start, stop = sliceToRoi(key, self.shape)
blockStart = (1.0 * start / self._blockShape).floor()
blockStop = (1.0 * stop / self._blockShape).ceil()
blockKey = roiToSlice(blockStart,blockStop)
innerBlocks = self._blockNumbers[blockKey]
if lazyflow.verboseRequests:
print "OpBlockedSparseLabelArray %r: request with key %r for %d inner Blocks " % (self,key, len(innerBlocks.ravel()))
for b_ind in innerBlocks.ravel():
#which part of the original key does this block fill?
offset = self._blockShape*self._flatBlockIndices[b_ind]
bigstart = numpy.maximum(offset, start)
bigstop = numpy.minimum(offset + self._blockShape, stop)
smallstart = bigstart-offset
smallstop = bigstop - offset
bigkey = roiToSlice(bigstart-start, bigstop-start)
smallkey = roiToSlice(smallstart, smallstop)
if not b_ind in self._labelers:
result[bigkey]=0
else:
result[bigkey]=self._labelers[b_ind]._denseArray[smallkey]
elif slot.name == "nonzeroValues":
nzvalues = set()
for l in self._labelers.values():
nzvalues |= set(l._sparseNZ.values())
result[0] = numpy.array(list(nzvalues))
elif slot.name == "nonzeroCoordinates":
print "not supported yet"
#result[0] = numpy.array(self._sparseNZ.keys())
elif slot.name == "nonzeroBlocks":
#we only return all non-zero blocks, no keys
slicelist = []
for b_ind in self._labelers.keys():
offset = self._blockShape*self._flatBlockIndices[b_ind]
bigstart = offset
bigstop = numpy.minimum(offset + self._blockShape, self.shape)
bigkey = roiToSlice(bigstart, bigstop)
slicelist.append(bigkey)
result[0] = slicelist
self.lock.release()
return result
def execute(self, slot, subindex, roi, result):
key = roi.toSlice()
self.lock.acquire()
assert(self.inputs["eraser"].ready() == True and self.inputs["shape"].ready() == True and self.inputs["blockShape"].ready()==True), \
"OpBlockedSparseLabelArray: One of the neccessary input slots is not ready: shape: %r, eraser: %r" % \
(self.inputs["eraser"].ready(), self.inputs["shape"].ready())
if slot.name == "Output":
#result[:] = self._denseArray[key]
#find the block key
start, stop = sliceToRoi(key, self._cacheShape)
blockStart = (1.0 * start / self._blockShape).floor()
blockStop = (1.0 * stop / self._blockShape).ceil()
blockKey = roiToSlice(blockStart,blockStop)
innerBlocks = self._blockNumbers[blockKey]
for b_ind in innerBlocks.ravel():
#which part of the original key does this block fill?
offset = self._blockShape*self._flatBlockIndices[b_ind]
bigstart = numpy.maximum(offset, start)
bigstop = numpy.minimum(offset + self._blockShape, stop)
smallstart = bigstart-offset
smallstop = bigstop - offset
bigkey = roiToSlice(bigstart-start, bigstop-start)
smallkey = roiToSlice(smallstart, smallstop)
if not b_ind in self._labelers or not self._labelers[b_ind].Output.ready():
result[bigkey]=0
else:
try:
labeler = self._labelers[b_ind]
denseArray = labeler._denseArray[smallkey]
result[bigkey]= denseArray
except:
logger.error( "Exception in OpBlockedSparseLabelArray.execute, probably due to simultaneous calls to setInSlot() and execute()" )
logger.error( "labeler = {}".format( labeler ) )
logger.error( "denseArray = {}".format( denseArray ) )
logger.error( "result = {}".format( result ) )
raise
elif slot.name == "nonzeroValues":
nzvalues = set()
for l in self._labelers.values():
nzvalues |= set(l._sparseNZ.values())
result[0] = numpy.array(list(nzvalues))
elif slot.name == "nonzeroCoordinates":
assert False, "not supported yet"
#result[0] = numpy.array(self._sparseNZ.keys())
elif slot.name == "nonzeroBlocks":
#we only return all non-zero blocks, no keys
result[0] = self._get_nonzero_blocks()
elif slot.name == "maxLabel":
result[0] = self._maxLabel
self.lock.release()
return result
def _collect_blocks(self, image_slot, label_slot, block_slicings):
model = self.ModelSession.value
image_data_blocks = []
label_data_blocks = []
block_ids = []
for block_slicing in block_slicings:
# Get labels
block_label_roi = sliceToRoi(block_slicing, label_slot.meta.shape)
block_label_data = label_slot(*block_label_roi).wait()
bb_roi_within_block = numpy.array([[0] *
len(block_label_data.shape),
list(block_label_data.shape)])
block_label_bb_roi = bb_roi_within_block + block_label_roi[0]
# Ask for the halo needed by the classifier
axiskeys = image_slot.meta.getAxisKeys()
halo_shape = model.get_halo(axiskeys)
assert len(halo_shape) == len(block_label_roi[0])
assert halo_shape[
-1] == 0, "Didn't expect a non-zero halo for channel dimension."
# Expand block by halo, but keep clipped to image bounds
padded_label_roi, bb_roi_within_padded = enlargeRoiForHalo(
*block_label_bb_roi,
shape=label_slot.meta.shape,
sigma=halo_shape,
window=1,
return_result_roi=True)
# Copy labels to new array, which has size == bounding-box + halo
padded_label_data = numpy.zeros(
padded_label_roi[1] - padded_label_roi[0],
label_slot.meta.dtype)
padded_label_data[roiToSlice(
*bb_roi_within_padded)] = block_label_data[roiToSlice(
*bb_roi_within_block)]
padded_image_roi = numpy.array(padded_label_roi)
assert (padded_image_roi[:, -1] == [0, 1]).all()
num_channels = image_slot.meta.shape[-1]
padded_image_roi[:, -1] = [0, num_channels]
# Ensure the results are plain ndarray, not VigraArray,
# which some classifiers might have trouble with.
padded_image_data = numpy.asarray(
image_slot(*padded_image_roi).wait())
image_data_blocks.append(padded_image_data)
label_data_blocks.append(padded_label_data)
block_ids.append(
tuple(
int(block_label_bb_roi[0][i])
for i, key in enumerate(axiskeys) if key != "c"))
return image_data_blocks, label_data_blocks, block_ids
def getOutSlot(self, slot, key, result):
shape = self.inputs["Input"].shape
rstart, rstop = sliceToRoi(key, shape)
rstop[-1] = shape[-1]
rkey = roiToSlice(rstart, rstop)
pred = self.inputs["Input"][rkey].allocate().wait()
ch = self.inputs["Channel"].value
th = self.inputs["Threshold"].value
result[..., 0] = numpy.where(pred[..., ch] > th, 1, 0)
def __init__(self, slot, start = None, stop = None, pslice = None):
super(SubRegion,self).__init__(slot)
if pslice != None or start is not None and stop is None and pslice is None:
if pslice is None:
pslice = start
shape = self.slot.meta.shape
if shape is None:
# Okay to use a shapeless slot if the key is bounded
# AND if the key has the correct length
assert slicingtools.is_bounded(pslice)
# Supply a dummy shape
shape = [0] * len(pslice)
self.start, self.stop = sliceToRoi(pslice,shape)
elif start is None and pslice is None:
self.start, self.stop = sliceToRoi(slice(None,None,None),self.slot.meta.shape)
else:
self.start = TinyVector(start)
self.stop = TinyVector(stop)
self.dim = len(self.start)
def _serialize(self, group, name, slot):
logger.debug("Serializing BlockSlot: {}".format( self.name ))
mygroup = group.create_group(name)
num = len(self.blockslot)
for index in range(num):
subname = self.subname.format(index)
subgroup = mygroup.create_group(subname)
nonZeroBlocks = self.blockslot[index].value
for blockIndex, slicing in enumerate(nonZeroBlocks):
if not isinstance(slicing[0], slice):
slicing = roiToSlice(*slicing)
block = self.slot[index][slicing].wait()
blockName = 'block{:04d}'.format(blockIndex)
if self._shrink_to_bb:
nonzero_coords = numpy.nonzero(block)
if len(nonzero_coords[0]) > 0:
block_start = sliceToRoi( slicing, (0,)*len(slicing) )[0]
block_bounding_box_start = numpy.array( list(map( numpy.min, nonzero_coords )) )
block_bounding_box_stop = 1 + numpy.array( list(map( numpy.max, nonzero_coords )) )
block_slicing = roiToSlice( block_bounding_box_start, block_bounding_box_stop )
bounding_box_roi = numpy.array([block_bounding_box_start, block_bounding_box_stop])
bounding_box_roi += block_start
# Overwrite the vars that are written to the file
slicing = roiToSlice(*bounding_box_roi)
block = block[block_slicing]
# If we have a masked array, convert it to a structured array so that h5py can handle it.
if slot[index].meta.has_mask:
mygroup.attrs["meta.has_mask"] = True
block_group = subgroup.create_group(blockName)
if self.compression_level:
block_group.create_dataset("data",
data=block.data,
compression='gzip',
compression_opts=compression_level)
else:
block_group.create_dataset("data", data=block.data)
block_group.create_dataset(
"mask",
data=block.mask,
compression="gzip",
compression_opts=2
)
block_group.create_dataset("fill_value", data=block.fill_value)
block_group.attrs['blockSlice'] = slicingToString(slicing)
else:
subgroup.create_dataset(blockName, data=block)
subgroup[blockName].attrs['blockSlice'] = slicingToString(slicing)
def _serialize(self, group, name, slot):
logger.debug("Serializing BlockSlot: {}".format( self.name ))
mygroup = group.create_group(name)
num = len(self.blockslot)
for index in range(num):
subname = self.subname.format(index)
subgroup = mygroup.create_group(subname)
nonZeroBlocks = self.blockslot[index].value
for blockIndex, slicing in enumerate(nonZeroBlocks):
if not isinstance(slicing[0], slice):
slicing = roiToSlice(*slicing)
block = self.slot[index][slicing].wait()
blockName = 'block{:04d}'.format(blockIndex)
if self._shrink_to_bb:
nonzero_coords = numpy.nonzero(block)
if len(nonzero_coords[0]) > 0:
block_start = sliceToRoi( slicing, (0,)*len(slicing) )[0]
block_bounding_box_start = numpy.array( map( numpy.min, nonzero_coords ) )
block_bounding_box_stop = 1 + numpy.array( map( numpy.max, nonzero_coords ) )
block_slicing = roiToSlice( block_bounding_box_start, block_bounding_box_stop )
bounding_box_roi = numpy.array([block_bounding_box_start, block_bounding_box_stop])
bounding_box_roi += block_start
# Overwrite the vars that are written to the file
slicing = roiToSlice(*bounding_box_roi)
block = block[block_slicing]
# If we have a masked array, convert it to a structured array so that h5py can handle it.
if slot[index].meta.has_mask:
mygroup.attrs["meta.has_mask"] = True
block_group = subgroup.create_group(blockName)
if self.compression_level:
block_group.create_dataset("data",
data=block.data,
compression='gzip',
compression_opts=compression_level)
else:
block_group.create_dataset("data", data=block.data)
block_group.create_dataset(
"mask",
data=block.mask,
compression="gzip",
compression_opts=2
)
block_group.create_dataset("fill_value", data=block.fill_value)
block_group.attrs['blockSlice'] = slicingToString(slicing)
else:
subgroup.create_dataset(blockName, data=block)
subgroup[blockName].attrs['blockSlice'] = slicingToString(slicing)
def propagateDirty(self, slot, subindex, roi):
shape = self.Output.meta.shape
key = roi.toSlice()
if slot == self.inputs["Input"]:
start, stop = sliceToRoi(key, shape)
with self._lock:
if self._blockState is not None:
blockStart = numpy.floor(1.0 * start / self._blockShape)
blockStop = numpy.ceil(1.0 * stop / self._blockShape)
blockKey = roiToSlice(blockStart, blockStop)
if self._fixed:
# Remember that this block became dirty while we were fixed
# so we can notify downstream operators when we become unfixed.
self._blockState[blockKey] = OpArrayCache.FIXED_DIRTY
self._has_fixed_dirty_blocks = True
else:
self._blockState[blockKey] = OpArrayCache.DIRTY
if not self._fixed:
self.outputs["Output"].setDirty(key)
if slot == self.inputs["fixAtCurrent"]:
if self.inputs["fixAtCurrent"].ready():
self._fixed = self.inputs["fixAtCurrent"].value
if not self._fixed and self.Output.meta.shape is not None and self._has_fixed_dirty_blocks:
# We've become unfixed, so we need to notify downstream
# operators of every block that became dirty while we were fixed.
# Convert all FIXED_DIRTY states into DIRTY states
with self._lock:
cond = (
self._blockState[...] == OpArrayCache.FIXED_DIRTY)
self._blockState[...] = fastWhere(
cond, OpArrayCache.DIRTY, self._blockState,
numpy.uint8)
self._has_fixed_dirty_blocks = False
newDirtyBlocks = numpy.transpose(numpy.nonzero(cond))
# To avoid lots of setDirty notifications, we simply merge all the dirtyblocks into one single superblock.
# This should be the best option in most cases, but could be bad in some cases.
# TODO: Optimize this by merging the dirty blocks via connected components or something.
cacheShape = numpy.array(self.Output.meta.shape)
dirtyStart = cacheShape
dirtyStop = [0] * len(cacheShape)
for index in newDirtyBlocks:
blockStart = index * self._blockShape
blockStop = numpy.minimum(
blockStart + self._blockShape, cacheShape)
dirtyStart = numpy.minimum(dirtyStart, blockStart)
dirtyStop = numpy.maximum(dirtyStop, blockStop)
if len(newDirtyBlocks > 0):
self.Output.setDirty(dirtyStart, dirtyStop)
def propagateDirty(self, slot, subindex, roi):
shape = self.Input.meta.shape
key = roi.toSlice()
if slot == self.inputs["Input"]:
start, stop = sliceToRoi(key, shape)
with self._lock:
if self._blockState is not None:
blockStart = numpy.floor(1.0 * start / self._blockShape)
blockStop = numpy.ceil(1.0 * stop / self._blockShape)
blockKey = roiToSlice(blockStart, blockStop)
if self._fixed:
# Remember that this block became dirty while we were fixed
# so we can notify downstream operators when we become unfixed.
self._blockState[blockKey] = OpArrayCache.FIXED_DIRTY
self._has_fixed_dirty_blocks = True
else:
self._blockState[blockKey] = OpArrayCache.DIRTY
if not self._fixed:
self.outputs["Output"].setDirty(key)
if slot == self.inputs["fixAtCurrent"]:
if self.inputs["fixAtCurrent"].ready():
self._fixed = self.inputs["fixAtCurrent"].value
if not self._fixed and self.Output.meta.shape is not None and self._has_fixed_dirty_blocks:
# We've become unfixed, so we need to notify downstream
# operators of every block that became dirty while we were fixed.
# Convert all FIXED_DIRTY states into DIRTY states
with self._lock:
cond = (
self._blockState[...] == OpArrayCache.FIXED_DIRTY)
self._blockState[...] = fastWhere(
cond, OpArrayCache.DIRTY, self._blockState,
numpy.uint8)
self._has_fixed_dirty_blocks = False
newDirtyBlocks = numpy.transpose(numpy.nonzero(cond))
# To avoid lots of setDirty notifications, we simply merge all the dirtyblocks into one single superblock.
# This should be the best option in most cases, but could be bad in some cases.
# TODO: Optimize this by merging the dirty blocks via connected components or something.
cacheShape = numpy.array(self.Output.meta.shape)
dirtyStart = cacheShape
dirtyStop = [0] * len(cacheShape)
for index in newDirtyBlocks:
blockStart = index * self._blockShape
blockStop = numpy.minimum(
blockStart + self._blockShape, cacheShape)
dirtyStart = numpy.minimum(dirtyStart, blockStart)
dirtyStop = numpy.maximum(dirtyStop, blockStop)
if len(newDirtyBlocks > 0):
self.Output.setDirty(dirtyStart, dirtyStop)
def test_5_ManySmallWritesToOneBlock(self):
for _ in range(100):
x = numpy.random.randint(49) + 1
y = numpy.random.randint(49) + 1
z = numpy.random.randint(49) + 1
slicing = numpy.s_[:, 0:x, 0:y, 0:z, :]
roi = sliceToRoi( slicing, self.dataShape )
roiShape = roi[1] - roi[0]
random_data = numpy.random.random( roiShape )
self.bfs.writeData( roi, random_data )
def test_3_ReadSome(self):
logger.debug( "Reading data..." )
slicing = numpy.s_[:, 50:150, 50:150, 50:150, :]
roi = sliceToRoi( slicing, self.dataShape )
roiShape = roi[1] - roi[0]
read_data = numpy.zeros( tuple(roiShape), dtype=numpy.uint8 )
self.bfs.readData( roi, read_data )
logger.debug( "Checking data..." )
assert self.data[slicing].shape == read_data.shape
assert (self.data[slicing] == read_data).all(), "Data didn't match."
def test_3_ReadSome(self):
logger.debug("Reading data...")
slicing = numpy.s_[:, 50:150, 50:150, 50:150, :]
roi = sliceToRoi(slicing, self.dataShape)
roiShape = roi[1] - roi[0]
read_data = numpy.zeros(tuple(roiShape), dtype=numpy.uint8)
self.bfs.readData(roi, read_data)
logger.debug("Checking data...")
assert self.data[slicing].shape == read_data.shape
assert (self.data[slicing] == read_data).all(), "Data didn't match."
def execute(self, slot, subindex, roi, result):
key = roiToSlice(roi.start, roi.stop)
shape = self.inputs["Input"].meta.shape
rstart, rstop = sliceToRoi(key, self.outputs["Output"].meta.shape)
rstart[-1] = 0
rstop[-1] = shape[-1]
rkey = roiToSlice(rstart, rstop)
img = self.inputs["Input"][rkey].wait()
axis = img.ndim - 1
result = numpy.argmax(img, axis=axis)
result.resize(result.shape + (1, ))
return result
def request( self, slicing ):
if cfg.getboolean('pixelpipeline', 'verbose'):
volumina.printLock.acquire()
print " LazyflowSource '%s' requests %s" % (self.objectName(), volumina.strSlicing(slicing))
volumina.printLock.release()
if not is_pure_slicing(slicing):
raise Exception('LazyflowSource: slicing is not pure')
assert self._op5 is not None, "Underlying operator is None. Are you requesting from a datasource that has been cleaned up already?"
start, stop = sliceToRoi(slicing, self._op5.Output.meta.shape)
clipped_roi = np.maximum(start, (0,0,0,0,0)), np.minimum(stop, self._op5.Output.meta.shape)
clipped_slicing = roiToSlice(*clipped_roi)
return LazyflowRequest( self._op5, clipped_slicing, self._priority, objectName=self.objectName() )
def request(self, slicing):
if CONFIG.verbose_pixelpipeline:
logger.info("%s '%s' requests %s'", type(self).__name__, self.objectName(), strSlicing(slicing))
if not is_pure_slicing(slicing):
raise Exception("LazyflowSource: slicing is not pure")
assert (
self._op5 is not None
), "Underlying operator is None. Are you requesting from a datasource that has been cleaned up already?"
start, stop = sliceToRoi(slicing, self._op5.Output.meta.shape)
clipped_roi = np.maximum(start, (0, 0, 0, 0, 0)), np.minimum(stop, self._op5.Output.meta.shape)
clipped_slicing = roiToSlice(*clipped_roi)
return LazyflowRequest(self._op5, clipped_slicing, self._priority, objectName=self.objectName())
def test_7_ManySmallWritesToOneBlock(self):
self.bfs.close()
self.bfs.reopen("a")
for _ in range(100):
x = numpy.random.randint(49) + 1
y = numpy.random.randint(49) + 1
z = numpy.random.randint(49) + 1
slicing = numpy.s_[:, 0:x, 0:y, 0:z, :]
roi = sliceToRoi(slicing, self.dataShape)
roiShape = roi[1] - roi[0]
random_data = numpy.random.random(roiShape)
self.bfs.writeData(roi, random_data)
self.bfs.setBlockStatusesForRoi(roi,
BlockwiseFileset.BLOCK_AVAILABLE)
def test_3_ReadSome(self):
logger.debug("Reading data...")
slicing = numpy.s_[:, 2:7, 8:11, 0:1, :]
roi = sliceToRoi(slicing, self.dataShape)
roiShape = roi[1] - roi[0]
read_data = numpy.zeros(tuple(roiShape), dtype=object)
self.bfs.readData(roi, read_data)
logger.debug("Checking data...")
assert self.data[slicing].shape == read_data.shape
for a, b in zip(self.data[slicing].flat, read_data.flat):
for (k1, v1), (k2, v2) in zip(list(a.items()), list(b.items())):
assert k1 == k2
assert (v1 == v2).all()
def execute(self, slot, subindex, roi, result):
classifier_factory = self.ClassifierFactory.value
assert issubclass(type(classifier_factory), LazyflowPixelwiseClassifierFactoryABC), \
"Factory is of type {}, which does not satisfy the LazyflowPixelwiseClassifierFactoryABC interface."\
"".format( type(classifier_factory) )
# Accumulate all non-zero blocks of each image into lists
label_data_blocks = []
image_data_blocks = []
for image_slot, label_slot, nonzero_block_slot in zip(self.Images, self.Labels, self.nonzeroLabelBlocks):
block_slicings = nonzero_block_slot.value
for block_slicing in block_slicings:
block_label_roi = sliceToRoi( block_slicing, image_slot.meta.shape )
# Ask for the halo needed by the classifier
axiskeys = image_slot.meta.getAxisKeys()
halo_shape = classifier_factory.get_halo_shape(axiskeys)
assert len(halo_shape) == len( block_label_roi[0] )
assert halo_shape[-1] == 0, "Didn't expect a non-zero halo for channel dimension."
# Expand block by halo, then clip to image bounds
block_label_roi = numpy.array( block_label_roi )
block_label_roi[0] -= halo_shape
block_label_roi[1] += halo_shape
block_label_roi = getIntersection( block_label_roi, roiFromShape(image_slot.meta.shape) )
block_image_roi = numpy.array( block_label_roi )
assert (block_image_roi[:, -1] == [0,1]).all()
num_channels = image_slot.meta.shape[-1]
block_image_roi[:, -1] = [0, num_channels]
# Ensure the results are plain ndarray, not VigraArray,
# which some classifiers might have trouble with.
block_label_data = numpy.asarray( label_slot(*block_label_roi).wait() )
block_image_data = numpy.asarray( image_slot(*block_image_roi).wait() )
label_data_blocks.append( block_label_data )
image_data_blocks.append( block_image_data )
logger.debug("Training new classifier: {}".format( classifier_factory.description ))
classifier = classifier_factory.create_and_train_pixelwise( image_data_blocks, label_data_blocks )
assert issubclass(type(classifier), LazyflowPixelwiseClassifierABC), \
"Classifier is of type {}, which does not satisfy the LazyflowPixelwiseClassifierABC interface."\
"".format( type(classifier) )
result[0] = classifier
return result
def test_4_ManySmallReadsFromOneBlock(self):
logger.debug("Starting small reads...")
for _ in range(100):
x = numpy.random.randint(49) + 1
y = numpy.random.randint(49) + 1
z = numpy.random.randint(49) + 1
slicing = numpy.s_[:, 0:x, 0:y, 0:z, :]
roi = sliceToRoi(slicing, self.dataShape)
roiShape = roi[1] - roi[0]
read_data = numpy.zeros(tuple(roiShape), dtype=numpy.uint8)
self.bfs.readData(roi, read_data)
assert self.data[slicing].shape == read_data.shape
assert (
self.data[slicing] == read_data).all(), "Data didn't match."
def _executeOutput(self, slot, subindex, roi, result):
key = roi.toSlice()
shape = self.Output.meta.shape
start, stop = sliceToRoi(key, shape)
self.traceLogger.debug("Acquiring ArrayCache lock...")
self._lock.acquire()
self.traceLogger.debug("ArrayCache lock acquired.")
ch = self._cacheHits
ch += 1
self._cacheHits = ch
self._running += 1
bp, bq = self.b.dirtyBlocks(start, stop)
#print "there are %d dirty blocks" % bp.shape[0]
if not self._fixed:
reqs = []
sh = self.outputs["Output"].meta.shape
for i in range(bp.shape[0]):
bStart = tuple([int(t) for t in bp[i, :]])
bStop = tuple([int(t) for t in numpy.minimum(bq[i, :], sh)])
key = roiToSlice(bStart, bStop)
req = self.Input[key]
reqs.append((req, bStart, bStop))
for r, bStart, bStop in reqs:
r.wait()
for r, bStart, bStop in reqs:
x = r.wait()
self.b.writeSubarray(bStart, bStop, r.wait())
t1 = time.time()
self.b.readSubarray(start, stop, result)
#print "read subarray took %f" % (time.time()-t1)
self._lock.release()
return result
def execute(self, slot, subindex, roi, result):
classifier_factory = self.ClassifierFactory.value
assert isinstance(classifier_factory, LazyflowPixelwiseClassifierFactoryABC), \
"Factory is of type {}, which does not satisfy the LazyflowPixelwiseClassifierFactoryABC interface."\
"".format( type(classifier_factory) )
# Accumulate all non-zero blocks of each image into lists
label_data_blocks = []
image_data_blocks = []
for image_slot, label_slot, nonzero_block_slot in zip(
self.Images, self.Labels, self.nonzeroLabelBlocks):
block_slicings = nonzero_block_slot.value
for block_slicing in block_slicings:
block_label_roi = sliceToRoi(block_slicing,
image_slot.meta.shape)
block_image_roi = numpy.array(block_label_roi)
assert (block_image_roi[:, -1] == [0, 1]).all()
num_channels = image_slot.meta.shape[-1]
block_image_roi[:, -1] = [0, num_channels]
# TODO: Compensate for the halo as specified by the classifier...
#axiskeys = image_slot.meta.getAxisKeys()
#halo_shape = classifier_factory.get_halo_shape(axiskeys)
# Ensure the results are plain ndarray, not VigraArray,
# which some classifiers might have trouble with.
block_label_data = numpy.asarray(
label_slot(*block_label_roi).wait())
block_image_data = numpy.asarray(
image_slot(*block_image_roi).wait())
label_data_blocks.append(block_label_data)
image_data_blocks.append(block_image_data)
classifier = classifier_factory.create_and_train_pixelwise(
image_data_blocks, label_data_blocks)
assert isinstance(classifier, LazyflowPixelwiseClassifierABC), \
"Classifier is of type {}, which does not satisfy the LazyflowPixelwiseClassifierABC interface."\
"".format( type(classifier) )
result[0] = classifier
return result
