def testFailedProcessing(self): op = OpArrayPiper(graph=Graph()) inputData = numpy.indices((100, 100)).sum(0) op.Input.setValue(inputData) roiList = [] block_starts = getIntersectingBlocks([10, 10], ([0, 0], [100, 100])) for block_start in block_starts: roiList.append(getBlockBounds([100, 100], [10, 10], block_start)) class SpecialException(Exception): pass def handleResult(roi, result): raise SpecialException( "Intentional Exception: raised while handling the result") totalVolume = numpy.prod(inputData.shape) batch = RoiRequestBatch(op.Output, roiList.__iter__(), totalVolume, batchSize=10, allowParallelResults=False) batch.resultSignal.subscribe(handleResult) # FIXME: There are multiple places where the RoiRequestBatch tool should be prepared to handle exceptions. # This only tests one of them (in the notify_finished() handler) try: batch.execute() except SpecialException: pass else: assert False, "Expected exception to be propagated out of the RoiRequestBatch."
def testBasic(self): op = OpArrayPiper(graph=Graph()) inputData = numpy.indices((100, 100)).sum(0) op.Input.setValue(inputData) roiList = [] block_starts = getIntersectingBlocks([10, 10], ([0, 0], [100, 100])) for block_start in block_starts: roiList.append(getBlockBounds([100, 100], [10, 10], block_start)) results = numpy.zeros((100, 100), dtype=numpy.int32) resultslock = threading.Lock() resultsCount = [0] def handleResult(roi, result): acquired = resultslock.acquire(False) assert acquired, "resultslock is contested! Access to callback is supposed to be automatically serialized." results[roiToSlice(*roi)] = result logger.debug("Got result for {}".format(roi)) resultslock.release() resultsCount[0] += 1 progressList = [] def handleProgress(progress): progressList.append(progress) logger.debug("Progress update: {}".format(progress)) totalVolume = numpy.prod(inputData.shape) batch = RoiRequestBatch(op.Output, roiList.__iter__(), totalVolume, batchSize=10, allowParallelResults=False) batch.resultSignal.subscribe(handleResult) batch.progressSignal.subscribe(handleProgress) batch.execute() logger.debug("Got {} results".format(resultsCount[0])) assert (results == inputData).all() # Progress reporting MUST start with 0 and end with 100 assert progressList[0] == 0, "Invalid progress reporting." assert progressList[-1] == 100, "Invalid progress reporting." # There should be some intermediate progress reporting, but exactly how much is unspecified. assert len(progressList) >= 10 logger.debug("FINISHED")
def __init__(self, outputSlot, roi, minBlockShape, batchSize=None): """ Constructor. :param outputSlot: The slot to request data from. :param roi: The roi `(start, stop)` of interest. Will be broken up and requested via smaller requests. :param minBlockShape: The minimum amount of data to request in each request. Note: The current implementation breaks the big request into smaller requests of exactly ``minBlockShape`` size. Future implementations could concatenate smaller requests if it appears the system is not being overloaded by the smaller requests. :param batchSize: The maximum number of requests to launch in parallel. """ self._outputSlot = outputSlot self._bigRoi = roi self._minBlockShape = minBlockShape if batchSize is None: batchSize = 2 # Align the blocking with the start of the roi offsetRoi = ([0] * len(roi[0]), numpy.subtract(roi[1], roi[0])) self._minBlockStarts = getIntersectingBlocks(minBlockShape, offsetRoi) self._minBlockStarts += roi[0] # Un-offset totalVolume = numpy.prod(numpy.subtract(roi[1], roi[0])) # For now, simply iterate over the min blocks # TODO: Auto-dialate block sizes based on CPU/RAM usage. def roiGen(): block_iter = self._minBlockStarts.__iter__() while True: block_start = block_iter.next() # Use offset blocking offset_block_start = block_start - self._bigRoi[0] offset_data_shape = numpy.subtract(self._bigRoi[1], self._bigRoi[0]) offset_block_bounds = getBlockBounds(offset_data_shape, minBlockShape, offset_block_start) # Un-offset block_bounds = (offset_block_bounds[0] + self._bigRoi[0], offset_block_bounds[1] + self._bigRoi[0]) logger.debug("Requesting Roi: {}".format(block_bounds)) yield block_bounds self._requestBatch = RoiRequestBatch(self._outputSlot, roiGen(), totalVolume, batchSize)
def run_export(self) -> None: """Export an image from Input to Filepath.""" path = pathlib.Path(self.Filepath.value) if path.exists(): path.unlink() self._page_buf = _NdBuf(self._opReorderAxes.Output.meta.shape[:-2]) batch = RoiRequestBatch( outputSlot=self._opReorderAxes.Output, roiIterator=_page_rois(*self._opReorderAxes.Output.meta.shape), totalVolume=np.prod(self._opReorderAxes.Output.meta.shape), batchSize=self._batch_size, ) batch.progressSignal.subscribe(self.progressSignal) batch.resultSignal.subscribe(self._write_buffered_pages) batch.execute()
def testPropagatesProcessingException(self, op_raising_at_3): roiList = [ ((0, 0, 0), (4, 4, 4)), ((1, 1, 1), (4, 4, 4)), ((2, 2, 2), (4, 4, 4)), ((3, 3, 3), (4, 4, 4)), ] totalVolume = numpy.prod(op_raising_at_3.Output.meta.shape) batch = RoiRequestBatch(op_raising_at_3.Output, roiList.__iter__(), totalVolume, batchSize=1, allowParallelResults=False) with pytest.raises(ProcessingException): batch.execute()
def __init__(self, outputSlot, roi, blockshape=None, batchSize=None, blockAlignment='absolute', allowParallelResults=False): """ Constructor. :param outputSlot: The slot to request data from. :param roi: The roi `(start, stop)` of interest. Will be broken up and requested via smaller requests. :param blockshape: The amount of data to request in each request. If omitted, a default blockshape is chosen by inspecting the metadata of the given slot. :param batchSize: The maximum number of requests to launch in parallel. This should not be necessary if the blockshape is small enough that you won't run out of RAM. :param blockAlignment: Determines how block the requests. Choices are 'absolute' or 'relative'. :param allowParallelResults: If False, The resultSignal will not be called in parallel. In that case, your handler function has no need for locks. """ self._outputSlot = outputSlot self._bigRoi = roi totalVolume = numpy.prod(numpy.subtract(roi[1], roi[0])) if batchSize is None: batchSize = 1000 if blockshape is None: blockshape = self._determine_blockshape(outputSlot) assert blockAlignment in ['relative', 'absolute'] if blockAlignment == 'relative': # Align the blocking with the start of the roi offsetRoi = ([0] * len(roi[0]), numpy.subtract(roi[1], roi[0])) block_starts = getIntersectingBlocks(blockshape, offsetRoi) block_starts += roi[0] # Un-offset # For now, simply iterate over the min blocks # TODO: Auto-dialate block sizes based on CPU/RAM usage. def roiGen(): block_iter = block_starts.__iter__() while True: block_start = block_iter.next() # Use offset blocking offset_block_start = block_start - self._bigRoi[0] offset_data_shape = numpy.subtract(self._bigRoi[1], self._bigRoi[0]) offset_block_bounds = getBlockBounds( offset_data_shape, blockshape, offset_block_start) # Un-offset block_bounds = (offset_block_bounds[0] + self._bigRoi[0], offset_block_bounds[1] + self._bigRoi[0]) logger.debug("Requesting Roi: {}".format(block_bounds)) yield block_bounds else: # Absolute blocking. # Blocks are simply relative to (0,0,0,...) # But we still clip the requests to the overall roi bounds. block_starts = getIntersectingBlocks(blockshape, roi) def roiGen(): block_iter = block_starts.__iter__() while True: block_start = block_iter.next() block_bounds = getBlockBounds(outputSlot.meta.shape, blockshape, block_start) block_intersecting_portion = getIntersection( block_bounds, roi) logger.debug("Requesting Roi: {}".format(block_bounds)) yield block_intersecting_portion self._requestBatch = RoiRequestBatch(self._outputSlot, roiGen(), totalVolume, batchSize, allowParallelResults)