def __init__(self, slot, start=None, stop=None, pslice=None): super(SubRegion, self).__init__(slot) if pslice != None or start is not None and stop is None and pslice is None: if pslice is None: pslice = start shape = self.slot.meta.shape if shape is None: # Okay to use a shapeless slot if the key is bounded # AND if the key has the correct length assert slicingtools.is_bounded(pslice) # Supply a dummy shape shape = [0] * len(pslice) self.start, self.stop = sliceToRoi(pslice, shape) elif start is None and pslice is None: self.start, self.stop = sliceToRoi(slice(None, None, None), self.slot.meta.shape) else: self.start = TinyVector(start) self.stop = TinyVector(stop) self.dim = len(self.start) for start, stop in zip(self.start, self.stop): assert isinstance( start, (int, long, numpy.integer)), "Roi contains non-integers: {}".format(self) assert isinstance( start, (int, long, numpy.integer)), "Roi contains non-integers: {}".format(self)
def execute(self, slot, subindex, roi, result): key = roiToSlice(roi.start,roi.stop) start = self.inputs["Start"].value stop = self.inputs["Stop"].value temp = tuple() for i in xrange(len(start)): if stop[i] - start[i] > 0: temp += (stop[i]-start[i],) readStart, readStop = sliceToRoi(key, temp) newKey = () resultKey = () i = 0 i2 = 0 for kkk in xrange(len(start)): e = stop[kkk] - start[kkk] if e > 0: newKey += (slice(start[i2] + readStart[i], start[i2] + readStop[i],None),) resultKey += (slice(0,temp[i2],None),) i +=1 else: newKey += (slice(start[i2], start[i2], None),) resultKey += (0,) i2 += 1 res = self.inputs["Input"][newKey].allocate().wait() result[:] = res[resultKey]
def propagateDirty(self, inputSlot, subindex, roi): if inputSlot == self.Input: numChannels = self.Input.meta.shape[1] dirtyChannels = roi.stop[1] - roi.start[1] # If all the input channels were dirty, the dirty output region is a contiguous block if dirtyChannels == numChannels: dirtyKey = list(roiToSlice(roi.start, roi.stop)) dirtyKey[1] = slice(None) dirtyRoi = sliceToRoi(dirtyKey, self.Output.meta.shape) self.Output.setDirty(dirtyRoi[0], dirtyRoi[1]) else: # Only some input channels were dirty, # so we must mark each dirty output region separately. numFeatures = self.Output.meta.shape[1] // numChannels for featureIndex in range(numFeatures): startChannel = numChannels * featureIndex + roi.start[1] stopChannel = startChannel + roi.stop[1] dirtyRoi = copy.copy(roi) dirtyRoi.start[1] = startChannel dirtyRoi.stop[1] = stopChannel self.Output.setDirty(dirtyRoi) elif (inputSlot == self.SelectionMatrix or inputSlot == self.Scales or inputSlot == self.FeatureIds or inputSlot == self.ComputeIn2d): self.Output.setDirty(slice(None)) else: assert False, "Unknown dirty input slot."
def propagateDirty(self, inputSlot, subindex, roi): if inputSlot == self.Input: numChannels = self.Input.meta.shape[1] dirtyChannels = roi.stop[1] - roi.start[1] # If all the input channels were dirty, the dirty output region is a contiguous block if dirtyChannels == numChannels: dirtyKey = list(roiToSlice(roi.start, roi.stop)) dirtyKey[1] = slice(None) dirtyRoi = sliceToRoi(dirtyKey, self.Output.meta.shape) self.Output.setDirty(dirtyRoi[0], dirtyRoi[1]) else: # Only some input channels were dirty, # so we must mark each dirty output region separately. numFeatures = self.Output.meta.shape[1] // numChannels for featureIndex in range(numFeatures): startChannel = numChannels * featureIndex + roi.start[1] stopChannel = startChannel + roi.stop[1] dirtyRoi = copy.copy(roi) dirtyRoi.start[1] = startChannel dirtyRoi.stop[1] = stopChannel self.Output.setDirty(dirtyRoi) elif ( inputSlot == self.SelectionMatrix or inputSlot == self.Scales or inputSlot == self.FeatureIds or inputSlot == self.ComputeIn2d ): self.Output.setDirty(slice(None)) else: assert False, "Unknown dirty input slot."
def getSubOutSlot(self, slots, indexes, key, result): outshape = self.outputs["Slices"][indexes[0]].shape start,stop=roi.sliceToRoi(key,outshape) oldstart,oldstop=start,stop start=list(start) stop=list(stop) flag=self.inputs["AxisFlag"].value indexAxis=self.inputs["Input"].axistags.index(flag) start.pop(indexAxis) stop.pop(indexAxis) start.insert(indexAxis,indexes[0]) stop.insert(indexAxis,indexes[0]) newKey=roi.roiToSlice(numpy.array(start),numpy.array(stop)) ttt = self.inputs["Input"][newKey].allocate().wait() result[:]=ttt[:]
def getOutSlot(self, slot, key, result): start, stop = sliceToRoi(key, self.shape) diff = stop-start splitDim = numpy.argmax(diff[:-1]) splitPos = start[splitDim] + diff[splitDim] / 2 stop2 = stop.copy() stop2[splitDim] = splitPos start2 = start.copy() start2[splitDim] = splitPos destStart = start -start # zeros destStop = stop - start destStop2 = destStop.copy() destStop2[splitDim] = diff[splitDim] / 2 destStart2 = destStart.copy() destStart2[splitDim] = diff[splitDim] / 2 writeKey1 = roiToSlice(destStart,destStop2) writeKey2 = roiToSlice(destStart2,destStop) key1 = roiToSlice(start,stop2) key2 = roiToSlice(start2,stop) req1 = self.inputs["Input"][key1].writeInto(result[writeKey1]) req2 = self.inputs["Input"][key2].writeInto(result[writeKey2]) req1.wait() req2.wait()
def setInSlot(self, slot, key, value): if slot == self.inputs["Input"]: ch = self._cacheHits ch += 1 self._cacheHits = ch start, stop = sliceToRoi(key, self.shape) blockStart = numpy.ceil(1.0 * start / self._blockShape) blockStop = numpy.floor(1.0 * stop / self._blockShape) blockStop = numpy.where(stop == self.shape, self._dirtyShape, blockStop) blockKey = roiToSlice(blockStart,blockStop) if (self._blockState[blockKey] != 2).any(): start2 = blockStart * self._blockShape stop2 = blockStop * self._blockShape stop2 = numpy.minimum(stop2, self.shape) key2 = roiToSlice(start2,stop2) self._lock.acquire() if self._cache is None: self._allocateCache() self._cache[key2] = value[roiToSlice(start2-start,stop2-start)] self._blockState[blockKey] = self._dirtyState self._blockQuery[blockKey] = None self._lock.release() #pass request on #if not self._fixed: # self.outputs["Output"][key] = value if slot == self.inputs["fixAtCurrent"]: self._fixed = value assert 1==2
def getOutSlot(self, slot, key, result): shape = self.inputs["Input"].shape rstart, rstop = sliceToRoi(key, self.outputs["Output"]._shape) rstart.append(0) rstop.append(shape[-1]) rkey = roiToSlice(rstart, rstop) img = self.inputs["Input"][rkey].allocate().wait() stop = img.size seg = [] for i in range(0, stop, img.shape[-1]): curr_prob = -1 highest_class = -1 for c in range(img.shape[-1]): prob = img.ravel()[i + c] if prob > curr_prob: curr_prob = prob highest_class = c assert highest_class != -1, "OpSegmentation: Strange classes/probabilities" seg.append(highest_class) seg = numpy.array(seg) seg.resize(img.shape[:-1]) result[:] = seg[:]
def testDirtyPropagation(self): opFeatures = self.opFeatures dirtyRois = [] def handleDirty(slot, roi): dirtyRois.append(roi) opFeatures.OutputImage[0].notifyDirty(handleDirty) # Change the matrix selections = numpy.array([ [True, False, False, False, False, False, False], # Gaussian [False, True, False, False, False, False, False], # L of G [False, False, True, False, False, False, False], # ST EVs [False, False, False, True, False, False, False], # H of G EVs [False, False, False, False, False, False, False], # GGM [False, False, False, False, False, False, False] ]) # Diff of G print "About to change matrix." opFeatures.SelectionMatrix.setValue(selections) print "Matrix changed." assert len(dirtyRois) == 1 assert (dirtyRois[0].start, dirtyRois[0].stop) == sliceToRoi( slice(None), self.opFeatures.OutputImage[0].meta.shape)
def propagateDirty(self, slot, subindex, roi): if slot == self.Input: channelAxis = self.Input.meta.axistags.index('c') numChannels = self.Input.meta.shape[channelAxis] dirtyChannels = roi.stop[channelAxis] - roi.start[channelAxis] # If all the input channels were dirty, the dirty output region is a contiguous block if dirtyChannels == numChannels: dirtyKey = roiToSlice(roi.start, roi.stop) dirtyKey[channelAxis] = slice(None) dirtyRoi = sliceToRoi(dirtyKey, self.Output.meta.shape) self.Output.setDirty(dirtyRoi[0], dirtyRoi[1]) else: # Only some input channels were dirty, # so we must mark each dirty output region separately. numFeatures = self.Output.meta.shape[channelAxis] / numChannels for featureIndex in range(numFeatures): startChannel = numChannels * featureIndex + roi.start[ channelAxis] stopChannel = startChannel + roi.stop[channelAxis] dirtyRoi = copy.copy(roi) dirtyRoi.start[channelAxis] = startChannel dirtyRoi.stop[channelAxis] = stopChannel self.Output.setDirty(dirtyRoi) elif (slot == self.Matrix or slot == self.Scales or slot == self.FeatureIds): self.Output.setDirty(slice(None)) else: assert False, "Unknown dirty input slot."
def setInSlot(self, slot, key, value): shape = self.inputs["shape"].value eraseLabel = self.inputs["eraser"].value neutralElement = 0 self.lock.acquire() #fix slicing of single dimensions: start, stop = sliceToRoi(key, shape, extendSingleton = False) start = start.floor() stop = stop.floor() tempKey = roiToSlice(start-start, stop-start, hardBind = True) stop += numpy.where(stop-start == 0,1,0) key = roiToSlice(start,stop) updateShape = tuple(stop-start) update = self._denseArray[key].copy() update[tempKey] = value startRavel = numpy.ravel_multi_index(numpy.array(start, numpy.int32),shape) #insert values into dict updateNZ = numpy.nonzero(numpy.where(update != neutralElement,1,0)) updateNZRavelSmall = numpy.ravel_multi_index(updateNZ, updateShape) if isinstance(value, numpy.ndarray): valuesNZ = value.ravel()[updateNZRavelSmall] else: valuesNZ = value updateNZRavel = numpy.ravel_multi_index(updateNZ, shape) updateNZRavel += startRavel self._denseArray.ravel()[updateNZRavel] = valuesNZ valuesNZ = self._denseArray.ravel()[updateNZRavel] self._denseArray.ravel()[updateNZRavel] = valuesNZ td = blist.sorteddict(zip(updateNZRavel.tolist(),valuesNZ.tolist())) self._sparseNZ.update(td) #remove values to be deleted updateNZ = numpy.nonzero(numpy.where(update == eraseLabel,1,0)) if len(updateNZ)>0: updateNZRavel = numpy.ravel_multi_index(updateNZ, shape) updateNZRavel += startRavel self._denseArray.ravel()[updateNZRavel] = neutralElement for index in updateNZRavel: self._sparseNZ.pop(index) self.lock.release() self.outputs["Output"].setDirty(key)
def propagateDirty(self, slot, subindex, roi): if slot == self.Input: channelAxis = self.Input.meta.axistags.index('c') numChannels = self.Input.meta.shape[channelAxis] dirtyChannels = roi.stop[channelAxis] - roi.start[channelAxis] # If all the input channels were dirty, the dirty output region is a contiguous block if dirtyChannels == numChannels: dirtyKey = roiToSlice(roi.start, roi.stop) dirtyKey[channelAxis] = slice(None) dirtyRoi = sliceToRoi(dirtyKey, self.Output.meta.shape) self.Output.setDirty(dirtyRoi[0], dirtyRoi[1]) else: # Only some input channels were dirty, # so we must mark each dirty output region separately. assert sys.version_info.major == 2, "Alert! This function has not been tested under python 3. "\ "please remove this assertion, and be wary of any strange behavior you encounter" numFeatures = self.Output.meta.shape[channelAxis] // numChannels for featureIndex in range(numFeatures): startChannel = numChannels * featureIndex + roi.start[ channelAxis] stopChannel = startChannel + roi.stop[channelAxis] dirtyRoi = copy.copy(roi) dirtyRoi.start[channelAxis] = startChannel dirtyRoi.stop[channelAxis] = stopChannel self.Output.setDirty(dirtyRoi) elif (slot == self.Matrix or slot == self.Scales or slot == self.FeatureIds): self.Output.setDirty(slice(None)) else: assert False, "Unknown dirty input slot."
def propagateDirty(self,slot,subindex,roi): if slot == self.Input: channelAxis = self.Input.meta.axistags.index('c') numChannels = self.Input.meta.shape[channelAxis] dirtyChannels = roi.stop[channelAxis] - roi.start[channelAxis] # If all the input channels were dirty, the dirty output region is a contiguous block if dirtyChannels == numChannels: dirtyKey = roiToSlice(roi.start, roi.stop) dirtyKey[channelAxis] = slice(None) dirtyRoi = sliceToRoi(dirtyKey, self.Output.meta.shape) self.Output.setDirty(dirtyRoi[0], dirtyRoi[1]) else: # Only some input channels were dirty, # so we must mark each dirty output region separately. numFeatures = self.Output.meta.shape[channelAxis] / numChannels for featureIndex in range(numFeatures): startChannel = numChannels*featureIndex + roi.start[channelAxis] stopChannel = startChannel + roi.stop[channelAxis] dirtyRoi = copy.copy(roi) dirtyRoi.start[channelAxis] = startChannel dirtyRoi.stop[channelAxis] = stopChannel self.Output.setDirty(dirtyRoi) elif (slot == self.Matrix or slot == self.Scales or slot == self.FeatureIds): self.Output.setDirty(slice(None)) else: assert False, "Unknown dirty input slot."
def execute(self, slot, subindex, roi, result): t = time.time() assert slot == self.Output key = roi.toSlice() start, stop = sliceToRoi(key, self.shape) roishape = numpy.array(stop) - numpy.array(start) max_dist_squared = sys.maxsize index = 0 for i, blockshape in enumerate(self._blockshapes): blockshape = numpy.array(blockshape) diff = roishape - blockshape diffsquared = diff * diff distance_squared = numpy.sum(diffsquared) if distance_squared < max_dist_squared: index = i max_dist_squared = distance_squared op = self._innerOps[index] op.outputs["Output"][key].writeInto(result).wait() self.logger.debug("read %r took %f msec." % (roi.pprint(), 1000.0 * (time.time() - t)))
def execute(self, slot, subindex, rroi, result): key = roiToSlice(rroi.start, rroi.stop) index = subindex[0] #print "SLICER: key", key, "indexes[0]", indexes[0], "result", result.shape start, stop = roi.sliceToRoi(key, self.outputs["Slices"][index].meta.shape) start = list(start) stop = list(stop) flag = self.inputs["AxisFlag"].value indexAxis = self.inputs["Input"].meta.axistags.index(flag) start.insert(indexAxis, index) stop.insert(indexAxis, index) newKey = roi.roiToSlice(numpy.array(start), numpy.array(stop)) ttt = self.inputs["Input"][newKey].wait() writeKey = [slice(None, None, None) for k in key] writeKey.insert(indexAxis, 0) writeKey = tuple(writeKey) return ttt[writeKey] #+ (0,)]
def execute(self, slot, subindex, roi, result): if slot is self.Output: result = self.LabelImage.get(roi).wait() if not self.Parameters.ready(): raise Exception("Parameter slot is not ready") parameters = self.Parameters.value t_start = roi.start[0] t_end = roi.stop[0] for t in range(t_start, t_end): if ('time_range' in parameters and t <= parameters['time_range'][-1] and t >= parameters['time_range'][ 0]) and len(self.label2color) > t: result[t - t_start, ..., 0] = relabel(result[t - t_start, ..., 0], self.label2color[t]) else: result[t - t_start, ...] = 0 return result elif slot == self.AllBlocks: # if nothing was computed, return empty list if len(self.label2color) == 0: result[0] = [] return result all_block_rois = [] shape = self.Output.meta.shape # assumes t,x,y,z,c slicing = [slice(None), ] * 5 for t in range(shape[0]): slicing[0] = slice(t, t + 1) all_block_rois.append(sliceToRoi(slicing, shape)) result[0] = all_block_rois return result
def _serialize(self, group, name, slot): logger.debug("Serializing BlockSlot: {}".format( self.name )) mygroup = group.create_group(name) num = len(self.blockslot) for index in range(num): subname = self.subname.format(index) subgroup = mygroup.create_group(subname) nonZeroBlocks = self.blockslot[index].value for blockIndex, slicing in enumerate(nonZeroBlocks): block = self.slot[index][slicing].wait() blockName = 'block{:04d}'.format(blockIndex) if self._shrink_to_bb: nonzero_coords = numpy.nonzero(block) if len(nonzero_coords[0]) > 0: block_start = sliceToRoi( slicing, (0,)*len(slicing) )[0] block_bounding_box_start = numpy.array( map( numpy.min, nonzero_coords ) ) block_bounding_box_stop = 1 + numpy.array( map( numpy.max, nonzero_coords ) ) block_slicing = roiToSlice( block_bounding_box_start, block_bounding_box_stop ) bounding_box_roi = numpy.array([block_bounding_box_start, block_bounding_box_stop]) bounding_box_roi += block_start # Overwrite the vars that are written to the file slicing = roiToSlice(*bounding_box_roi) block = block[block_slicing] subgroup.create_dataset(blockName, data=block) subgroup[blockName].attrs['blockSlice'] = slicingToString(slicing)
def setInSlot(self, slot, key, value): start, stop = sliceToRoi(key, self.shape) blockStart = (1.0 * start / self._blockShape).floor() blockStop = (1.0 * stop / self._blockShape).ceil() blockStop = numpy.where(stop == self.shape, self._dirtyShape, blockStop) blockKey = roiToSlice(blockStart,blockStop) innerBlocks = self._blockNumbers[blockKey] for b_ind in innerBlocks.ravel(): offset = self._blockShape*self._flatBlockIndices[b_ind] bigstart = numpy.maximum(offset, start) bigstop = numpy.minimum(offset + self._blockShape, stop) smallstart = bigstart-offset smallstop = bigstop - offset bigkey = roiToSlice(bigstart-start, bigstop-start) smallkey = roiToSlice(smallstart, smallstop) if not b_ind in self._labelers: self._labelers[b_ind]=OpSparseLabelArray(self) self._labelers[b_ind].inputs["shape"].setValue(self._blockShape) self._labelers[b_ind].inputs["eraser"].setValue(self.inputs["eraser"].value) self._labelers[b_ind].inputs["deleteLabel"].setValue(self.inputs["deleteLabel"]) self._labelers[b_ind].inputs["Input"][smallkey] = value[tuple(bigkey)].squeeze() self.outputs["Output"].setDirty(key)
def __init__(self, slot, start = None, stop = None, pslice = None): super(SubRegion,self).__init__(slot) shape = None if slot is not None: shape = slot.meta.shape if pslice != None or start is not None and stop is None and pslice is None: if pslice is None: pslice = start if shape is None: # Okay to use a shapeless slot if the key is bounded # AND if the key has the correct length assert slicingtools.is_bounded(pslice) # Supply a dummy shape shape = [0] * len(pslice) self.start, self.stop = sliceToRoi(pslice,shape) elif start is None and pslice is None: assert shape is not None, "Can't create a default subregion without a slot and a shape." self.start, self.stop = roiFromShape(shape) else: self.start = TinyVector(start) self.stop = TinyVector(stop) self.dim = len(self.start) for start, stop in zip(self.start, self.stop): assert isinstance(start, (int, long, numpy.integer)), "Roi contains non-integers: {}".format( self ) assert isinstance(start, (int, long, numpy.integer)), "Roi contains non-integers: {}".format( self )
def __init__(self, slot, start=None, stop=None, pslice=None): super(SubRegion, self).__init__(slot) shape = None if slot is not None: shape = slot.meta.shape if pslice != None or start is not None and stop is None and pslice is None: if pslice is None: pslice = start if shape is None: # Okay to use a shapeless slot if the key is bounded # AND if the key has the correct length assert slicingtools.is_bounded(pslice) # Supply a dummy shape shape = [sl.stop for sl in pslice] self.start, self.stop = sliceToRoi(pslice, shape) elif start is None and pslice is None: assert shape is not None, "Can't create a default subregion without a slot and a shape." self.start, self.stop = roiFromShape(shape) else: self.start = TinyVector(start) self.stop = TinyVector(stop) self.dim = len(self.start) for start, stop in zip(self.start, self.stop): assert isinstance(start, (int, numpy.integer)), "Roi contains non-integers: {}".format(self) assert isinstance(start, (int, numpy.integer)), "Roi contains non-integers: {}".format(self)
def testBasic(self): """ Requires access to the Internet... """ testConfig0 = """ { "_schema_name" : "RESTful-volume-description", "_schema_version" : 1.0, "name" : "Bock11-level0", "format" : "hdf5", "axes" : "zyx", "##NOTE":"The first z-slice of the bock dataset is 2917, so the origin_offset must be at least 2917", "origin_offset" : [2917, 50000, 50000], "bounds" : [4156, 135424, 119808], "dtype" : "numpy.uint8", "url_format" : "http://openconnecto.me/emca/bock11/hdf5/0/{x_start},{x_stop}/{y_start},{y_stop}/{z_start},{z_stop}/", "hdf5_dataset" : "cube" } """ testConfig4 = """ { "_schema_name" : "RESTful-volume-description", "_schema_version" : 1.0, "name" : "Bock11-level4", "format" : "hdf5", "axes" : "zyx", "##NOTE":"The first z-slice of the bock dataset is 2917, so the origin_offset must be at least 2917", "origin_offset" : [2917, 50000, 50000], "bounds" : [4156, 8704, 7680], "dtype" : "numpy.uint8", "url_format" : "http://openconnecto.me/emca/bock11/hdf5/4/{x_start},{x_stop}/{y_start},{y_stop}/{z_start},{z_stop}/", "hdf5_dataset" : "cube" } """ # Create the description file. tempDir = tempfile.mkdtemp() descriptionFilePath = os.path.join(tempDir, 'desc.json') with open(descriptionFilePath, 'w') as descFile: descFile.write( testConfig0 ) # Create the volume object volume = RESTfulVolume( descriptionFilePath ) #slicing = numpy.s_[0:100, 4000:4200, 4000:4200] slicing = numpy.s_[0:25, 50000:50050, 50000:50075] roi = sliceToRoi( slicing, volume.description.shape ) outputFile = os.path.join(tempDir, 'volume.h5') datasetPath = outputFile + '/cube' logger.debug("Downloading subvolume to: {}".format( datasetPath )) volume.downloadSubVolume(roi, datasetPath) with h5py.File(outputFile, 'r') as hdf5File: data = hdf5File['cube'] assert data.shape == ( 25, 50, 75 ) shutil.rmtree(tempDir)
def execute(self, slot, subindex, roi, result): key = roiToSlice(roi.start,roi.stop) start = self.inputs["Start"].value stop = self.inputs["Stop"].value temp = tuple() for i in xrange(len(start)): if stop[i] - start[i] > 0: temp += (stop[i]-start[i],) readStart, readStop = sliceToRoi(key, temp) newKey = () i = 0 i2 = 0 for kkk in xrange(len(start)): e = stop[kkk] - start[kkk] if e > 0: newKey += (slice(start[i2] + readStart[i], start[i2] + readStop[i],None),) i +=1 else: newKey += (slice(start[i2], start[i2], None),) i2 += 1 self.inputs["Input"][newKey].writeInto(result).wait() return result
def execute(self, slot, subindex, rroi, result): key = roiToSlice(rroi.start, rroi.stop) index = subindex[0] # Index of the input slice this data will come from. sliceIndex = self.getSliceIndexes()[index] outshape = self.Slices[index].meta.shape start, stop = roi.sliceToRoi(key, outshape) start = list(start) stop = list(stop) flag = self.AxisFlag.value indexAxis = self.Input.meta.axistags.index(flag) start.pop(indexAxis) stop.pop(indexAxis) start.insert(indexAxis, sliceIndex) stop.insert(indexAxis, sliceIndex) newKey = roi.roiToSlice(numpy.array(start), numpy.array(stop)) self.Input[newKey].writeInto(result).wait() return result
def getSubOutSlot(self, slots, indexes, key, result): #print "SLICER: key", key, "indexes[0]", indexes[0], "result", result.shape start,stop=roi.sliceToRoi(key,self.outputs["Slices"][indexes[0]].shape) oldstart,oldstop=start,stop start=list(start) stop=list(stop) flag=self.inputs["AxisFlag"].value indexAxis=self.inputs["Input"].axistags.index(flag) start.insert(indexAxis,indexes[0]) stop.insert(indexAxis,indexes[0]) newKey=roi.roiToSlice(numpy.array(start),numpy.array(stop)) ttt = self.inputs["Input"][newKey].allocate().wait() writeKey = [slice(None, None, None) for k in key] writeKey.insert(indexAxis, 0) writeKey = tuple(writeKey) result[:]=ttt[writeKey ]#+ (0,)]
def testBasic(self): """ Requires access to the Internet... """ testConfig0 = """ { "_schema_name" : "RESTful-volume-description", "_schema_version" : 1.0, "name" : "Bock11-level0", "format" : "hdf5", "axes" : "zyx", "##NOTE":"The first z-slice of the bock dataset is 2917, so the origin_offset must be at least 2917", "origin_offset" : [2917, 50000, 50000], "bounds" : [4156, 135424, 119808], "dtype" : "numpy.uint8", "url_format" : "http://openconnecto.me/emca/bock11/hdf5/0/{x_start},{x_stop}/{y_start},{y_stop}/{z_start},{z_stop}/", "hdf5_dataset" : "cube" } """ testConfig4 = """ { "_schema_name" : "RESTful-volume-description", "_schema_version" : 1.0, "name" : "Bock11-level4", "format" : "hdf5", "axes" : "zyx", "##NOTE":"The first z-slice of the bock dataset is 2917, so the origin_offset must be at least 2917", "origin_offset" : [2917, 50000, 50000], "bounds" : [4156, 8704, 7680], "dtype" : "numpy.uint8", "url_format" : "http://openconnecto.me/emca/bock11/hdf5/4/{x_start},{x_stop}/{y_start},{y_stop}/{z_start},{z_stop}/", "hdf5_dataset" : "cube" } """ # Create the description file. tempDir = tempfile.mkdtemp() descriptionFilePath = os.path.join(tempDir, 'desc.json') with open(descriptionFilePath, 'w') as descFile: descFile.write(testConfig0) # Create the volume object volume = RESTfulVolume(descriptionFilePath) #slicing = numpy.s_[0:100, 4000:4200, 4000:4200] slicing = numpy.s_[0:25, 50000:50050, 50000:50075] roi = sliceToRoi(slicing, volume.description.shape) outputFile = os.path.join(tempDir, 'volume.h5') datasetPath = outputFile + '/cube' logger.debug("Downloading subvolume to: {}".format(datasetPath)) volume.downloadSubVolume(roi, datasetPath) with h5py.File(outputFile, 'r') as hdf5File: data = hdf5File['cube'] assert data.shape == (25, 50, 75) shutil.rmtree(tempDir)
def propagateDirty(self,slot,subindex,roi): if slot == self.Input: channelAxis = self.Input.meta.axistags.index('c') numChannels = self.Input.meta.shape[channelAxis] dirtyChannels = roi.stop[channelAxis] - roi.start[channelAxis] # If all the input channels were dirty, the dirty output region is a contiguous block if dirtyChannels == numChannels: dirtyKey = roiToSlice(roi.start, roi.stop) dirtyKey[channelAxis] = slice(None) dirtyRoi = sliceToRoi(dirtyKey, self.Output.meta.shape) self.Output.setDirty(dirtyRoi[0], dirtyRoi[1]) else: # Only some input channels were dirty, # so we must mark each dirty output region separately. assert sys.version_info.major == 2, "Alert! This function has not been tested under python 3. "\ "please remove this assertion, and be wary of any strange behavior you encounter" numFeatures = self.Output.meta.shape[channelAxis] // numChannels for featureIndex in range(numFeatures): startChannel = numChannels*featureIndex + roi.start[channelAxis] stopChannel = startChannel + roi.stop[channelAxis] dirtyRoi = copy.copy(roi) dirtyRoi.start[channelAxis] = startChannel dirtyRoi.stop[channelAxis] = stopChannel self.Output.setDirty(dirtyRoi) elif (slot == self.Matrix or slot == self.Scales or slot == self.FeatureIds): self.Output.setDirty(slice(None)) else: assert False, "Unknown dirty input slot."
def testDirtyPropagation(self): opFeatures = self.opFeatures dirtyRois = [] def handleDirty(slot, roi): dirtyRois.append(roi) opFeatures.OutputImage[0].notifyDirty(handleDirty) # Change the matrix selections = numpy.array( [ [True, False, False, False, False, False, False], # Gaussian [False, True, False, False, False, False, False], # L of G [False, False, True, False, False, False, False], # ST EVs [False, False, False, True, False, False, False], # H of G EVs [False, False, False, False, False, False, False], # GGM [False, False, False, False, False, False, False], ] ) # Diff of G opFeatures.SelectionMatrix.setValue(selections) assert len(dirtyRois) == 1 assert (dirtyRois[0].start, dirtyRois[0].stop) == sliceToRoi( slice(None), self.opFeatures.OutputImage[0].meta.shape )
def test_9_TestView(self): """ Load some of the dataset again; this time with an offset view. Note: The original blockwise fileset must be closed before this test starts. """ # Create a copy of the original description, but specify a translated (and smaller) view desc = BlockwiseFileset.readDescription(self.description_path) desc.view_origin = [0, 300, 200, 100, 0] desc.view_shape = [1, 50, 50, 50, 1] offsetConfigPath = self.description_path + '_offset' BlockwiseFileset.writeDescription(offsetConfigPath, desc) # Open the fileset using the special description file bfs = BlockwiseFileset( offsetConfigPath, 'r' ) try: assert (bfs.description.view_origin == desc.view_origin).all() assert (bfs.description.view_shape == desc.view_shape).all() # Read some data logger.debug( "Reading data..." ) disk_slicing = numpy.s_[:, 300:350, 200:250, 100:150, :] view_slicing = numpy.s_[:, 0:50, 0:50, 0:50, :] roi = sliceToRoi( view_slicing, self.dataShape ) roiShape = roi[1] - roi[0] read_data = numpy.zeros( tuple(roiShape), dtype=numpy.uint8 ) bfs.readData( roi, read_data ) # The data we read should match the correct part of the original dataset. logger.debug( "Checking data..." ) assert self.data[disk_slicing].shape == read_data.shape assert (self.data[disk_slicing] == read_data).all(), "Data didn't match." finally: bfs.close()
def execute(self, slot, subindex, rroi, result): key = roiToSlice(rroi.start, rroi.stop) index = subindex[0] # Index of the input slice this data will come from. sliceIndex = self.getSliceIndexes()[index] outshape = self.outputs["Slices"][index].meta.shape start,stop=roi.sliceToRoi(key,outshape) oldstart,oldstop=start,stop start=list(start) stop=list(stop) flag=self.inputs["AxisFlag"].value indexAxis=self.inputs["Input"].meta.axistags.index(flag) start.pop(indexAxis) stop.pop(indexAxis) start.insert(indexAxis, sliceIndex) stop.insert(indexAxis, sliceIndex) newKey=roi.roiToSlice(numpy.array(start),numpy.array(stop)) ttt = self.inputs["Input"][newKey].allocate().wait() return ttt[:]
def _serialize(self, group, name, slot): logger.debug("Serializing BlockSlot: {}".format(self.name)) mygroup = group.create_group(name) num = len(self.blockslot) for index in range(num): subname = self.subname.format(index) subgroup = mygroup.create_group(subname) nonZeroBlocks = self.blockslot[index].value for blockIndex, slicing in enumerate(nonZeroBlocks): block = self.slot[index][slicing].wait() blockName = 'block{:04d}'.format(blockIndex) if self._shrink_to_bb: nonzero_coords = numpy.nonzero(block) if len(nonzero_coords[0]) > 0: block_start = sliceToRoi(slicing, (0, ) * len(slicing))[0] block_bounding_box_start = numpy.array( map(numpy.min, nonzero_coords)) block_bounding_box_stop = 1 + numpy.array( map(numpy.max, nonzero_coords)) block_slicing = roiToSlice(block_bounding_box_start, block_bounding_box_stop) bounding_box_roi = numpy.array([ block_bounding_box_start, block_bounding_box_stop ]) bounding_box_roi += block_start # Overwrite the vars that are written to the file slicing = roiToSlice(*bounding_box_roi) block = block[block_slicing] subgroup.create_dataset(blockName, data=block) subgroup[blockName].attrs['blockSlice'] = slicingToString( slicing)
def execute(self, slot, subindex, roi, result): key = roi.toSlice() self.lock.acquire() assert(self.inputs["eraser"].ready() == True and self.inputs["shape"].ready() == True and self.inputs["blockShape"].ready()==True), \ "OpBlockedSparseLabelArray: One of the neccessary input slots is not ready: shape: %r, eraser: %r" % \ (self.inputs["eraser"].ready(), self.inputs["shape"].ready()) if slot.name == "Output": #result[:] = self._denseArray[key] #find the block key start, stop = sliceToRoi(key, self._cacheShape) blockStart = (1.0 * start / self._blockShape).floor() blockStop = (1.0 * stop / self._blockShape).ceil() blockKey = roiToSlice(blockStart, blockStop) innerBlocks = self._blockNumbers[blockKey] for b_ind in innerBlocks.ravel(): #which part of the original key does this block fill? offset = self._blockShape * self._flatBlockIndices[b_ind] bigstart = numpy.maximum(offset, start) bigstop = numpy.minimum(offset + self._blockShape, stop) smallstart = bigstart - offset smallstop = bigstop - offset bigkey = roiToSlice(bigstart - start, bigstop - start) smallkey = roiToSlice(smallstart, smallstop) if not b_ind in self._labelers or not self._labelers[ b_ind].Output.ready(): result[bigkey] = 0 else: try: labeler = self._labelers[b_ind] denseArray = labeler._denseArray[smallkey] result[bigkey] = denseArray except: logger.error( "Exception in OpBlockedSparseLabelArray.execute, probably due to simultaneous calls to setInSlot() and execute()" ) logger.error("labeler = {}".format(labeler)) logger.error("denseArray = {}".format(denseArray)) logger.error("result = {}".format(result)) raise elif slot.name == "nonzeroValues": nzvalues = set() for l in self._labelers.values(): nzvalues |= set(l._sparseNZ.values()) result[0] = numpy.array(list(nzvalues)) elif slot.name == "nonzeroCoordinates": assert False, "not supported yet" #result[0] = numpy.array(self._sparseNZ.keys()) elif slot.name == "nonzeroBlocks": #we only return all non-zero blocks, no keys result[0] = self._get_nonzero_blocks() elif slot.name == "maxLabel": result[0] = self._maxLabel self.lock.release() return result
def execute(self, slot, subindex, rroi, result): key = roiToSlice(rroi.start,rroi.stop) cnt = 0 written = 0 start, stop = roi.sliceToRoi(key, self.outputs["Output"].meta.shape) assert (stop<=self.outputs["Output"].meta.shape).all() #axisindex = self.inputs["AxisIndex"].value flag = self.inputs["AxisFlag"].value axisindex = self.outputs["Output"].meta.axistags.index(flag) #ugly-ugly-ugly oldkey = list(key) oldkey.pop(axisindex) #print "STACKER: ", flag, axisindex #print "requesting an outslot from stacker:", key, result.shape #print "input slots total: ", len(self.inputs['Images']) requests = [] pool = RequestPool() for i, inSlot in enumerate(self.inputs['Images']): req = None inTagKeys = [ax.key for ax in inSlot.meta.axistags] if flag in inTagKeys: slices = inSlot.meta.shape[axisindex] if cnt + slices >= start[axisindex] and start[axisindex]-cnt<slices and start[axisindex]+written<stop[axisindex]: begin = 0 if cnt < start[axisindex]: begin = start[axisindex] - cnt end = slices if cnt + end > stop[axisindex]: end -= cnt + end - stop[axisindex] key_ = copy.copy(oldkey) key_.insert(axisindex, slice(begin, end, None)) reskey = [slice(None, None, None) for x in range(len(result.shape))] reskey[axisindex] = slice(written, written+end-begin, None) req = inSlot[tuple(key_)].writeInto(result[tuple(reskey)]) written += end - begin cnt += slices else: if cnt>=start[axisindex] and start[axisindex] + written < stop[axisindex]: #print "key: ", key, "reskey: ", reskey, "oldkey: ", oldkey #print "result: ", result.shape, "inslot:", inSlot.meta.shape reskey = [slice(None, None, None) for s in oldkey] reskey.insert(axisindex, written) destArea = result[tuple(reskey)] req = inSlot[tuple(oldkey)].writeInto(destArea) written += 1 cnt += 1 if req is not None: pool.add(req) pool.wait() pool.clean()
def test_1_SingleDownload(self): volume = RESTfulBlockwiseFileset( self.descriptionFilePath ) slicing = numpy.s_[0:20, 0:20, 0:20] roi = sliceToRoi(slicing, volume.description.shape) data = volume.readData( roi ) assert data.shape == (20,20,20) assert volume.getBlockStatus( ([0,0,0]) ) == BlockwiseFileset.BLOCK_AVAILABLE
def test_4_OffsetDownload(self): volume = RESTfulBlockwiseFileset( self.descriptionFilePath ) slicing = numpy.s_[20:40, 20:40, 20:40] roi = sliceToRoi(slicing, volume.description.shape) data = volume.readData( roi ) assert data.shape == (20,20,20) assert volume.getBlockStatus( ([20,20,20]) ) == BlockwiseFileset.BLOCK_AVAILABLE offsetVolume = RESTfulBlockwiseFileset( self.descriptionFilePath_offset ) offsetSlicing = numpy.s_[20:40, 0:20, 20:40] # Note middle slice is offset (see view_origin in setupClass) offsetRoi = sliceToRoi(offsetSlicing, offsetVolume.description.shape) offsetData = offsetVolume.readData( offsetRoi ) assert offsetData.shape == (20,20,20) assert offsetVolume.getBlockStatus( ([20,0,20]) ) == BlockwiseFileset.BLOCK_AVAILABLE # Data should be the same assert (offsetData == data).all()
def execute(self, slot, roi, result): key = roi.toSlice() self.lock.acquire() assert(self.inputs["eraser"].connected() == True and self.inputs["shape"].connected() == True and self.inputs["blockShape"].connected()==True), \ "OpDenseSparseArray: One of the neccessary input slots is not connected: shape: %r, eraser: %r" % \ (self.inputs["eraser"].connected(), self.inputs["shape"].connected()) if slot.name == "Output": #result[:] = self._denseArray[key] #find the block key start, stop = sliceToRoi(key, self.shape) blockStart = (1.0 * start / self._blockShape).floor() blockStop = (1.0 * stop / self._blockShape).ceil() blockKey = roiToSlice(blockStart,blockStop) innerBlocks = self._blockNumbers[blockKey] if lazyflow.verboseRequests: print "OpBlockedSparseLabelArray %r: request with key %r for %d inner Blocks " % (self,key, len(innerBlocks.ravel())) for b_ind in innerBlocks.ravel(): #which part of the original key does this block fill? offset = self._blockShape*self._flatBlockIndices[b_ind] bigstart = numpy.maximum(offset, start) bigstop = numpy.minimum(offset + self._blockShape, stop) smallstart = bigstart-offset smallstop = bigstop - offset bigkey = roiToSlice(bigstart-start, bigstop-start) smallkey = roiToSlice(smallstart, smallstop) if not b_ind in self._labelers: result[bigkey]=0 else: result[bigkey]=self._labelers[b_ind]._denseArray[smallkey] elif slot.name == "nonzeroValues": nzvalues = set() for l in self._labelers.values(): nzvalues |= set(l._sparseNZ.values()) result[0] = numpy.array(list(nzvalues)) elif slot.name == "nonzeroCoordinates": print "not supported yet" #result[0] = numpy.array(self._sparseNZ.keys()) elif slot.name == "nonzeroBlocks": #we only return all non-zero blocks, no keys slicelist = [] for b_ind in self._labelers.keys(): offset = self._blockShape*self._flatBlockIndices[b_ind] bigstart = offset bigstop = numpy.minimum(offset + self._blockShape, self.shape) bigkey = roiToSlice(bigstart, bigstop) slicelist.append(bigkey) result[0] = slicelist self.lock.release() return result
def execute(self, slot, subindex, roi, result): key = roi.toSlice() self.lock.acquire() assert(self.inputs["eraser"].ready() == True and self.inputs["shape"].ready() == True and self.inputs["blockShape"].ready()==True), \ "OpBlockedSparseLabelArray: One of the neccessary input slots is not ready: shape: %r, eraser: %r" % \ (self.inputs["eraser"].ready(), self.inputs["shape"].ready()) if slot.name == "Output": #result[:] = self._denseArray[key] #find the block key start, stop = sliceToRoi(key, self._cacheShape) blockStart = (1.0 * start / self._blockShape).floor() blockStop = (1.0 * stop / self._blockShape).ceil() blockKey = roiToSlice(blockStart,blockStop) innerBlocks = self._blockNumbers[blockKey] for b_ind in innerBlocks.ravel(): #which part of the original key does this block fill? offset = self._blockShape*self._flatBlockIndices[b_ind] bigstart = numpy.maximum(offset, start) bigstop = numpy.minimum(offset + self._blockShape, stop) smallstart = bigstart-offset smallstop = bigstop - offset bigkey = roiToSlice(bigstart-start, bigstop-start) smallkey = roiToSlice(smallstart, smallstop) if not b_ind in self._labelers or not self._labelers[b_ind].Output.ready(): result[bigkey]=0 else: try: labeler = self._labelers[b_ind] denseArray = labeler._denseArray[smallkey] result[bigkey]= denseArray except: logger.error( "Exception in OpBlockedSparseLabelArray.execute, probably due to simultaneous calls to setInSlot() and execute()" ) logger.error( "labeler = {}".format( labeler ) ) logger.error( "denseArray = {}".format( denseArray ) ) logger.error( "result = {}".format( result ) ) raise elif slot.name == "nonzeroValues": nzvalues = set() for l in self._labelers.values(): nzvalues |= set(l._sparseNZ.values()) result[0] = numpy.array(list(nzvalues)) elif slot.name == "nonzeroCoordinates": assert False, "not supported yet" #result[0] = numpy.array(self._sparseNZ.keys()) elif slot.name == "nonzeroBlocks": #we only return all non-zero blocks, no keys result[0] = self._get_nonzero_blocks() elif slot.name == "maxLabel": result[0] = self._maxLabel self.lock.release() return result
def _collect_blocks(self, image_slot, label_slot, block_slicings): model = self.ModelSession.value image_data_blocks = [] label_data_blocks = [] block_ids = [] for block_slicing in block_slicings: # Get labels block_label_roi = sliceToRoi(block_slicing, label_slot.meta.shape) block_label_data = label_slot(*block_label_roi).wait() bb_roi_within_block = numpy.array([[0] * len(block_label_data.shape), list(block_label_data.shape)]) block_label_bb_roi = bb_roi_within_block + block_label_roi[0] # Ask for the halo needed by the classifier axiskeys = image_slot.meta.getAxisKeys() halo_shape = model.get_halo(axiskeys) assert len(halo_shape) == len(block_label_roi[0]) assert halo_shape[ -1] == 0, "Didn't expect a non-zero halo for channel dimension." # Expand block by halo, but keep clipped to image bounds padded_label_roi, bb_roi_within_padded = enlargeRoiForHalo( *block_label_bb_roi, shape=label_slot.meta.shape, sigma=halo_shape, window=1, return_result_roi=True) # Copy labels to new array, which has size == bounding-box + halo padded_label_data = numpy.zeros( padded_label_roi[1] - padded_label_roi[0], label_slot.meta.dtype) padded_label_data[roiToSlice( *bb_roi_within_padded)] = block_label_data[roiToSlice( *bb_roi_within_block)] padded_image_roi = numpy.array(padded_label_roi) assert (padded_image_roi[:, -1] == [0, 1]).all() num_channels = image_slot.meta.shape[-1] padded_image_roi[:, -1] = [0, num_channels] # Ensure the results are plain ndarray, not VigraArray, # which some classifiers might have trouble with. padded_image_data = numpy.asarray( image_slot(*padded_image_roi).wait()) image_data_blocks.append(padded_image_data) label_data_blocks.append(padded_label_data) block_ids.append( tuple( int(block_label_bb_roi[0][i]) for i, key in enumerate(axiskeys) if key != "c")) return image_data_blocks, label_data_blocks, block_ids
def getOutSlot(self, slot, key, result): shape = self.inputs["Input"].shape rstart, rstop = sliceToRoi(key, shape) rstop[-1] = shape[-1] rkey = roiToSlice(rstart, rstop) pred = self.inputs["Input"][rkey].allocate().wait() ch = self.inputs["Channel"].value th = self.inputs["Threshold"].value result[..., 0] = numpy.where(pred[..., ch] > th, 1, 0)
def __init__(self, slot, start = None, stop = None, pslice = None): super(SubRegion,self).__init__(slot) if pslice != None or start is not None and stop is None and pslice is None: if pslice is None: pslice = start shape = self.slot.meta.shape if shape is None: # Okay to use a shapeless slot if the key is bounded # AND if the key has the correct length assert slicingtools.is_bounded(pslice) # Supply a dummy shape shape = [0] * len(pslice) self.start, self.stop = sliceToRoi(pslice,shape) elif start is None and pslice is None: self.start, self.stop = sliceToRoi(slice(None,None,None),self.slot.meta.shape) else: self.start = TinyVector(start) self.stop = TinyVector(stop) self.dim = len(self.start)
def _serialize(self, group, name, slot): logger.debug("Serializing BlockSlot: {}".format( self.name )) mygroup = group.create_group(name) num = len(self.blockslot) for index in range(num): subname = self.subname.format(index) subgroup = mygroup.create_group(subname) nonZeroBlocks = self.blockslot[index].value for blockIndex, slicing in enumerate(nonZeroBlocks): if not isinstance(slicing[0], slice): slicing = roiToSlice(*slicing) block = self.slot[index][slicing].wait() blockName = 'block{:04d}'.format(blockIndex) if self._shrink_to_bb: nonzero_coords = numpy.nonzero(block) if len(nonzero_coords[0]) > 0: block_start = sliceToRoi( slicing, (0,)*len(slicing) )[0] block_bounding_box_start = numpy.array( list(map( numpy.min, nonzero_coords )) ) block_bounding_box_stop = 1 + numpy.array( list(map( numpy.max, nonzero_coords )) ) block_slicing = roiToSlice( block_bounding_box_start, block_bounding_box_stop ) bounding_box_roi = numpy.array([block_bounding_box_start, block_bounding_box_stop]) bounding_box_roi += block_start # Overwrite the vars that are written to the file slicing = roiToSlice(*bounding_box_roi) block = block[block_slicing] # If we have a masked array, convert it to a structured array so that h5py can handle it. if slot[index].meta.has_mask: mygroup.attrs["meta.has_mask"] = True block_group = subgroup.create_group(blockName) if self.compression_level: block_group.create_dataset("data", data=block.data, compression='gzip', compression_opts=compression_level) else: block_group.create_dataset("data", data=block.data) block_group.create_dataset( "mask", data=block.mask, compression="gzip", compression_opts=2 ) block_group.create_dataset("fill_value", data=block.fill_value) block_group.attrs['blockSlice'] = slicingToString(slicing) else: subgroup.create_dataset(blockName, data=block) subgroup[blockName].attrs['blockSlice'] = slicingToString(slicing)
def _serialize(self, group, name, slot): logger.debug("Serializing BlockSlot: {}".format( self.name )) mygroup = group.create_group(name) num = len(self.blockslot) for index in range(num): subname = self.subname.format(index) subgroup = mygroup.create_group(subname) nonZeroBlocks = self.blockslot[index].value for blockIndex, slicing in enumerate(nonZeroBlocks): if not isinstance(slicing[0], slice): slicing = roiToSlice(*slicing) block = self.slot[index][slicing].wait() blockName = 'block{:04d}'.format(blockIndex) if self._shrink_to_bb: nonzero_coords = numpy.nonzero(block) if len(nonzero_coords[0]) > 0: block_start = sliceToRoi( slicing, (0,)*len(slicing) )[0] block_bounding_box_start = numpy.array( map( numpy.min, nonzero_coords ) ) block_bounding_box_stop = 1 + numpy.array( map( numpy.max, nonzero_coords ) ) block_slicing = roiToSlice( block_bounding_box_start, block_bounding_box_stop ) bounding_box_roi = numpy.array([block_bounding_box_start, block_bounding_box_stop]) bounding_box_roi += block_start # Overwrite the vars that are written to the file slicing = roiToSlice(*bounding_box_roi) block = block[block_slicing] # If we have a masked array, convert it to a structured array so that h5py can handle it. if slot[index].meta.has_mask: mygroup.attrs["meta.has_mask"] = True block_group = subgroup.create_group(blockName) if self.compression_level: block_group.create_dataset("data", data=block.data, compression='gzip', compression_opts=compression_level) else: block_group.create_dataset("data", data=block.data) block_group.create_dataset( "mask", data=block.mask, compression="gzip", compression_opts=2 ) block_group.create_dataset("fill_value", data=block.fill_value) block_group.attrs['blockSlice'] = slicingToString(slicing) else: subgroup.create_dataset(blockName, data=block) subgroup[blockName].attrs['blockSlice'] = slicingToString(slicing)
def propagateDirty(self, slot, subindex, roi): shape = self.Output.meta.shape key = roi.toSlice() if slot == self.inputs["Input"]: start, stop = sliceToRoi(key, shape) with self._lock: if self._blockState is not None: blockStart = numpy.floor(1.0 * start / self._blockShape) blockStop = numpy.ceil(1.0 * stop / self._blockShape) blockKey = roiToSlice(blockStart, blockStop) if self._fixed: # Remember that this block became dirty while we were fixed # so we can notify downstream operators when we become unfixed. self._blockState[blockKey] = OpArrayCache.FIXED_DIRTY self._has_fixed_dirty_blocks = True else: self._blockState[blockKey] = OpArrayCache.DIRTY if not self._fixed: self.outputs["Output"].setDirty(key) if slot == self.inputs["fixAtCurrent"]: if self.inputs["fixAtCurrent"].ready(): self._fixed = self.inputs["fixAtCurrent"].value if not self._fixed and self.Output.meta.shape is not None and self._has_fixed_dirty_blocks: # We've become unfixed, so we need to notify downstream # operators of every block that became dirty while we were fixed. # Convert all FIXED_DIRTY states into DIRTY states with self._lock: cond = ( self._blockState[...] == OpArrayCache.FIXED_DIRTY) self._blockState[...] = fastWhere( cond, OpArrayCache.DIRTY, self._blockState, numpy.uint8) self._has_fixed_dirty_blocks = False newDirtyBlocks = numpy.transpose(numpy.nonzero(cond)) # To avoid lots of setDirty notifications, we simply merge all the dirtyblocks into one single superblock. # This should be the best option in most cases, but could be bad in some cases. # TODO: Optimize this by merging the dirty blocks via connected components or something. cacheShape = numpy.array(self.Output.meta.shape) dirtyStart = cacheShape dirtyStop = [0] * len(cacheShape) for index in newDirtyBlocks: blockStart = index * self._blockShape blockStop = numpy.minimum( blockStart + self._blockShape, cacheShape) dirtyStart = numpy.minimum(dirtyStart, blockStart) dirtyStop = numpy.maximum(dirtyStop, blockStop) if len(newDirtyBlocks > 0): self.Output.setDirty(dirtyStart, dirtyStop)
def propagateDirty(self, slot, subindex, roi): shape = self.Input.meta.shape key = roi.toSlice() if slot == self.inputs["Input"]: start, stop = sliceToRoi(key, shape) with self._lock: if self._blockState is not None: blockStart = numpy.floor(1.0 * start / self._blockShape) blockStop = numpy.ceil(1.0 * stop / self._blockShape) blockKey = roiToSlice(blockStart, blockStop) if self._fixed: # Remember that this block became dirty while we were fixed # so we can notify downstream operators when we become unfixed. self._blockState[blockKey] = OpArrayCache.FIXED_DIRTY self._has_fixed_dirty_blocks = True else: self._blockState[blockKey] = OpArrayCache.DIRTY if not self._fixed: self.outputs["Output"].setDirty(key) if slot == self.inputs["fixAtCurrent"]: if self.inputs["fixAtCurrent"].ready(): self._fixed = self.inputs["fixAtCurrent"].value if not self._fixed and self.Output.meta.shape is not None and self._has_fixed_dirty_blocks: # We've become unfixed, so we need to notify downstream # operators of every block that became dirty while we were fixed. # Convert all FIXED_DIRTY states into DIRTY states with self._lock: cond = ( self._blockState[...] == OpArrayCache.FIXED_DIRTY) self._blockState[...] = fastWhere( cond, OpArrayCache.DIRTY, self._blockState, numpy.uint8) self._has_fixed_dirty_blocks = False newDirtyBlocks = numpy.transpose(numpy.nonzero(cond)) # To avoid lots of setDirty notifications, we simply merge all the dirtyblocks into one single superblock. # This should be the best option in most cases, but could be bad in some cases. # TODO: Optimize this by merging the dirty blocks via connected components or something. cacheShape = numpy.array(self.Output.meta.shape) dirtyStart = cacheShape dirtyStop = [0] * len(cacheShape) for index in newDirtyBlocks: blockStart = index * self._blockShape blockStop = numpy.minimum( blockStart + self._blockShape, cacheShape) dirtyStart = numpy.minimum(dirtyStart, blockStart) dirtyStop = numpy.maximum(dirtyStop, blockStop) if len(newDirtyBlocks > 0): self.Output.setDirty(dirtyStart, dirtyStop)
def test_5_ManySmallWritesToOneBlock(self): for _ in range(100): x = numpy.random.randint(49) + 1 y = numpy.random.randint(49) + 1 z = numpy.random.randint(49) + 1 slicing = numpy.s_[:, 0:x, 0:y, 0:z, :] roi = sliceToRoi( slicing, self.dataShape ) roiShape = roi[1] - roi[0] random_data = numpy.random.random( roiShape ) self.bfs.writeData( roi, random_data )
def test_3_ReadSome(self): logger.debug( "Reading data..." ) slicing = numpy.s_[:, 50:150, 50:150, 50:150, :] roi = sliceToRoi( slicing, self.dataShape ) roiShape = roi[1] - roi[0] read_data = numpy.zeros( tuple(roiShape), dtype=numpy.uint8 ) self.bfs.readData( roi, read_data ) logger.debug( "Checking data..." ) assert self.data[slicing].shape == read_data.shape assert (self.data[slicing] == read_data).all(), "Data didn't match."
def test_3_ReadSome(self): logger.debug("Reading data...") slicing = numpy.s_[:, 50:150, 50:150, 50:150, :] roi = sliceToRoi(slicing, self.dataShape) roiShape = roi[1] - roi[0] read_data = numpy.zeros(tuple(roiShape), dtype=numpy.uint8) self.bfs.readData(roi, read_data) logger.debug("Checking data...") assert self.data[slicing].shape == read_data.shape assert (self.data[slicing] == read_data).all(), "Data didn't match."
def execute(self, slot, subindex, roi, result): key = roiToSlice(roi.start, roi.stop) shape = self.inputs["Input"].meta.shape rstart, rstop = sliceToRoi(key, self.outputs["Output"].meta.shape) rstart[-1] = 0 rstop[-1] = shape[-1] rkey = roiToSlice(rstart, rstop) img = self.inputs["Input"][rkey].wait() axis = img.ndim - 1 result = numpy.argmax(img, axis=axis) result.resize(result.shape + (1, )) return result
def request( self, slicing ): if cfg.getboolean('pixelpipeline', 'verbose'): volumina.printLock.acquire() print " LazyflowSource '%s' requests %s" % (self.objectName(), volumina.strSlicing(slicing)) volumina.printLock.release() if not is_pure_slicing(slicing): raise Exception('LazyflowSource: slicing is not pure') assert self._op5 is not None, "Underlying operator is None. Are you requesting from a datasource that has been cleaned up already?" start, stop = sliceToRoi(slicing, self._op5.Output.meta.shape) clipped_roi = np.maximum(start, (0,0,0,0,0)), np.minimum(stop, self._op5.Output.meta.shape) clipped_slicing = roiToSlice(*clipped_roi) return LazyflowRequest( self._op5, clipped_slicing, self._priority, objectName=self.objectName() )
def request(self, slicing): if CONFIG.verbose_pixelpipeline: logger.info("%s '%s' requests %s'", type(self).__name__, self.objectName(), strSlicing(slicing)) if not is_pure_slicing(slicing): raise Exception("LazyflowSource: slicing is not pure") assert ( self._op5 is not None ), "Underlying operator is None. Are you requesting from a datasource that has been cleaned up already?" start, stop = sliceToRoi(slicing, self._op5.Output.meta.shape) clipped_roi = np.maximum(start, (0, 0, 0, 0, 0)), np.minimum(stop, self._op5.Output.meta.shape) clipped_slicing = roiToSlice(*clipped_roi) return LazyflowRequest(self._op5, clipped_slicing, self._priority, objectName=self.objectName())
def test_7_ManySmallWritesToOneBlock(self): self.bfs.close() self.bfs.reopen("a") for _ in range(100): x = numpy.random.randint(49) + 1 y = numpy.random.randint(49) + 1 z = numpy.random.randint(49) + 1 slicing = numpy.s_[:, 0:x, 0:y, 0:z, :] roi = sliceToRoi(slicing, self.dataShape) roiShape = roi[1] - roi[0] random_data = numpy.random.random(roiShape) self.bfs.writeData(roi, random_data) self.bfs.setBlockStatusesForRoi(roi, BlockwiseFileset.BLOCK_AVAILABLE)
def test_3_ReadSome(self): logger.debug("Reading data...") slicing = numpy.s_[:, 2:7, 8:11, 0:1, :] roi = sliceToRoi(slicing, self.dataShape) roiShape = roi[1] - roi[0] read_data = numpy.zeros(tuple(roiShape), dtype=object) self.bfs.readData(roi, read_data) logger.debug("Checking data...") assert self.data[slicing].shape == read_data.shape for a, b in zip(self.data[slicing].flat, read_data.flat): for (k1, v1), (k2, v2) in zip(list(a.items()), list(b.items())): assert k1 == k2 assert (v1 == v2).all()
def execute(self, slot, subindex, roi, result): classifier_factory = self.ClassifierFactory.value assert issubclass(type(classifier_factory), LazyflowPixelwiseClassifierFactoryABC), \ "Factory is of type {}, which does not satisfy the LazyflowPixelwiseClassifierFactoryABC interface."\ "".format( type(classifier_factory) ) # Accumulate all non-zero blocks of each image into lists label_data_blocks = [] image_data_blocks = [] for image_slot, label_slot, nonzero_block_slot in zip(self.Images, self.Labels, self.nonzeroLabelBlocks): block_slicings = nonzero_block_slot.value for block_slicing in block_slicings: block_label_roi = sliceToRoi( block_slicing, image_slot.meta.shape ) # Ask for the halo needed by the classifier axiskeys = image_slot.meta.getAxisKeys() halo_shape = classifier_factory.get_halo_shape(axiskeys) assert len(halo_shape) == len( block_label_roi[0] ) assert halo_shape[-1] == 0, "Didn't expect a non-zero halo for channel dimension." # Expand block by halo, then clip to image bounds block_label_roi = numpy.array( block_label_roi ) block_label_roi[0] -= halo_shape block_label_roi[1] += halo_shape block_label_roi = getIntersection( block_label_roi, roiFromShape(image_slot.meta.shape) ) block_image_roi = numpy.array( block_label_roi ) assert (block_image_roi[:, -1] == [0,1]).all() num_channels = image_slot.meta.shape[-1] block_image_roi[:, -1] = [0, num_channels] # Ensure the results are plain ndarray, not VigraArray, # which some classifiers might have trouble with. block_label_data = numpy.asarray( label_slot(*block_label_roi).wait() ) block_image_data = numpy.asarray( image_slot(*block_image_roi).wait() ) label_data_blocks.append( block_label_data ) image_data_blocks.append( block_image_data ) logger.debug("Training new classifier: {}".format( classifier_factory.description )) classifier = classifier_factory.create_and_train_pixelwise( image_data_blocks, label_data_blocks ) assert issubclass(type(classifier), LazyflowPixelwiseClassifierABC), \ "Classifier is of type {}, which does not satisfy the LazyflowPixelwiseClassifierABC interface."\ "".format( type(classifier) ) result[0] = classifier return result
def test_4_ManySmallReadsFromOneBlock(self): logger.debug("Starting small reads...") for _ in range(100): x = numpy.random.randint(49) + 1 y = numpy.random.randint(49) + 1 z = numpy.random.randint(49) + 1 slicing = numpy.s_[:, 0:x, 0:y, 0:z, :] roi = sliceToRoi(slicing, self.dataShape) roiShape = roi[1] - roi[0] read_data = numpy.zeros(tuple(roiShape), dtype=numpy.uint8) self.bfs.readData(roi, read_data) assert self.data[slicing].shape == read_data.shape assert ( self.data[slicing] == read_data).all(), "Data didn't match."
def _executeOutput(self, slot, subindex, roi, result): key = roi.toSlice() shape = self.Output.meta.shape start, stop = sliceToRoi(key, shape) self.traceLogger.debug("Acquiring ArrayCache lock...") self._lock.acquire() self.traceLogger.debug("ArrayCache lock acquired.") ch = self._cacheHits ch += 1 self._cacheHits = ch self._running += 1 bp, bq = self.b.dirtyBlocks(start, stop) #print "there are %d dirty blocks" % bp.shape[0] if not self._fixed: reqs = [] sh = self.outputs["Output"].meta.shape for i in range(bp.shape[0]): bStart = tuple([int(t) for t in bp[i, :]]) bStop = tuple([int(t) for t in numpy.minimum(bq[i, :], sh)]) key = roiToSlice(bStart, bStop) req = self.Input[key] reqs.append((req, bStart, bStop)) for r, bStart, bStop in reqs: r.wait() for r, bStart, bStop in reqs: x = r.wait() self.b.writeSubarray(bStart, bStop, r.wait()) t1 = time.time() self.b.readSubarray(start, stop, result) #print "read subarray took %f" % (time.time()-t1) self._lock.release() return result
def execute(self, slot, subindex, roi, result): classifier_factory = self.ClassifierFactory.value assert isinstance(classifier_factory, LazyflowPixelwiseClassifierFactoryABC), \ "Factory is of type {}, which does not satisfy the LazyflowPixelwiseClassifierFactoryABC interface."\ "".format( type(classifier_factory) ) # Accumulate all non-zero blocks of each image into lists label_data_blocks = [] image_data_blocks = [] for image_slot, label_slot, nonzero_block_slot in zip( self.Images, self.Labels, self.nonzeroLabelBlocks): block_slicings = nonzero_block_slot.value for block_slicing in block_slicings: block_label_roi = sliceToRoi(block_slicing, image_slot.meta.shape) block_image_roi = numpy.array(block_label_roi) assert (block_image_roi[:, -1] == [0, 1]).all() num_channels = image_slot.meta.shape[-1] block_image_roi[:, -1] = [0, num_channels] # TODO: Compensate for the halo as specified by the classifier... #axiskeys = image_slot.meta.getAxisKeys() #halo_shape = classifier_factory.get_halo_shape(axiskeys) # Ensure the results are plain ndarray, not VigraArray, # which some classifiers might have trouble with. block_label_data = numpy.asarray( label_slot(*block_label_roi).wait()) block_image_data = numpy.asarray( image_slot(*block_image_roi).wait()) label_data_blocks.append(block_label_data) image_data_blocks.append(block_image_data) classifier = classifier_factory.create_and_train_pixelwise( image_data_blocks, label_data_blocks) assert isinstance(classifier, LazyflowPixelwiseClassifierABC), \ "Classifier is of type {}, which does not satisfy the LazyflowPixelwiseClassifierABC interface."\ "".format( type(classifier) ) result[0] = classifier return result