def predict_probabilities(self, X): logger.debug( "Predicting with parallel vigra RF" ) X = numpy.asarray(X, dtype=numpy.float32) assert X.ndim == 2 if self._feature_names is not None: # For some reason, vigra doesn't seem to check this for us... assert X.shape[1] == len(self._feature_names), \ "Feature count ({}) doesn't match the training feature count ({}).\n"\ "Expected features: {}".format( X.shape[1], len(self._feature_names), self._feature_names ) # As each forest completes, aggregate results in a shared array. # (Must put in a list so we can update it in this closure.) total_predictions = [None] prediction_lock = RequestLock() def update_predictions(forest, forest_predictions): forest_predictions *= forest.treeCount() with prediction_lock: if total_predictions[0] is None: total_predictions[0] = forest_predictions else: total_predictions[0] += forest_predictions # Create a request for each forest pool = RequestPool() for forest in self._forests: req = Request( partial( forest.predictProbabilities, X ) ) req.notify_finished( partial(update_predictions, forest) ) pool.add( req ) del req pool.wait() total_predictions[0] /= self._num_trees return total_predictions[0]
def create_and_train(self, X, y): logger.debug( "Training parallel vigra RF" ) # Save for future reference known_labels = numpy.unique(y) X = numpy.asarray(X, numpy.float32) y = numpy.asarray(y, numpy.uint32) if y.ndim == 1: y = y[:, numpy.newaxis] assert X.ndim == 2 assert len(X) == len(y) # Create N forests forests = [] for _ in range(self._num_forests): forest = vigra.learning.RandomForest(self._trees_per_forest, **self._kwargs) forests.append( forest ) # Train them all in parallel pool = RequestPool() for forest in forests: pool.add( Request( partial(forest.learnRF, X, y) ) ) pool.wait() return ParallelVigraRfLazyflowClassifier( forests, known_labels )
def execute(self, slot, subindex, roi, result): featMatrix=[] labelsMatrix=[] for i,labels in enumerate(self.inputs["Labels"]): if labels.meta.shape is not None: labels=labels[:].wait() indexes=numpy.nonzero(labels[...,0].view(numpy.ndarray)) #Maybe later request only part of the region? image=self.inputs["Images"][i][:].wait() features=image[indexes] labels=labels[indexes] featMatrix.append(features) labelsMatrix.append(labels) featMatrix=numpy.concatenate(featMatrix,axis=0) labelsMatrix=numpy.concatenate(labelsMatrix,axis=0) # train and store self._forest_count forests in parallel pool = RequestPool() for i in range(self._forest_count): def train_and_store(number): result[number] = vigra.learning.RandomForest(self._tree_count) result[number].learnRF(featMatrix.astype(numpy.float32),labelsMatrix.astype(numpy.uint32)) req = pool.request(partial(train_and_store, i)) pool.wait() return result
def predict_probabilities(self, X): logger.debug( "Predicting with parallel vigra RF" ) X = numpy.asarray(X, dtype=numpy.float32) # As each forest completes, aggregate results in a shared array. # (Must put in a list so we can update it in this closure.) total_predictions = [None] prediction_lock = RequestLock() def update_predictions(forest, forest_predictions): forest_predictions *= forest.treeCount() with prediction_lock: if total_predictions[0] is None: total_predictions[0] = forest_predictions else: total_predictions[0] += forest_predictions # Create a request for each forest pool = RequestPool() for forest in self._forests: req = Request( partial( forest.predictProbabilities, X ) ) req.notify_finished( partial(update_predictions, forest) ) pool.add( req ) del req pool.wait() total_predictions[0] /= self._num_trees return total_predictions[0]
def execute(self, slot, subindex, roi, result): clipped_block_rois = getIntersectingRois( self.Input.meta.shape, self.BlockShape.value, (roi.start, roi.stop), True ) if self._always_request_full_blocks: full_block_rois = getIntersectingRois( self.Input.meta.shape, self.BlockShape.value, (roi.start, roi.stop), False ) else: full_block_rois = clipped_block_rois pool = RequestPool() for full_block_roi, clipped_block_roi in zip(full_block_rois, clipped_block_rois): full_block_roi = numpy.asarray(full_block_roi) clipped_block_roi = numpy.asarray(clipped_block_roi) req = self.Input(*full_block_roi) output_roi = numpy.asarray(clipped_block_roi) - roi.start if (full_block_roi == clipped_block_roi).all(): req.writeInto(result[roiToSlice(*output_roi)]) else: roi_within_block = clipped_block_roi - full_block_roi[0] def copy_request_result(output_roi, roi_within_block, request_result): self.Output.stype.copy_data( result[roiToSlice(*output_roi)], request_result[roiToSlice(*roi_within_block)] ) req.notify_finished(partial(copy_request_result, output_roi, roi_within_block)) pool.add(req) del req pool.wait()
def _executeOutput(self, roi, destination): assert len(roi.stop) == len(self.Input.meta.shape), "roi: {} has the wrong number of dimensions for Input shape: {}".format( roi, self.Input.meta.shape ) assert numpy.less_equal(roi.stop, self.Input.meta.shape).all(), "roi: {} is out-of-bounds for Input shape: {}".format( roi, self.Input.meta.shape ) block_starts = getIntersectingBlocks( self._blockshape, (roi.start, roi.stop) ) block_starts = map( tuple, block_starts ) # Ensure all block cache files are up-to-date reqPool = RequestPool() # (Do the work in parallel.) for block_start in block_starts: entire_block_roi = getBlockBounds( self.Input.meta.shape, self._blockshape, block_start ) f = partial( self._ensureCached, entire_block_roi) reqPool.add( Request(f) ) logger.debug( "Waiting for {} blocks...".format( len(block_starts) ) ) reqPool.wait() # Copy data from each block # (Parallelism not needed here: h5py will serialize these requests anyway) logger.debug( "Copying data from {} blocks...".format( len(block_starts) ) ) for block_start in block_starts: entire_block_roi = getBlockBounds( self.Input.meta.shape, self._blockshape, block_start ) # This block's portion of the roi intersecting_roi = getIntersection( (roi.start, roi.stop), entire_block_roi ) # Compute slicing within destination array and slicing within this block destination_relative_intersection = numpy.subtract(intersecting_roi, roi.start) block_relative_intersection = numpy.subtract(intersecting_roi, block_start) # Copy from block to destination dataset = self._getBlockDataset( entire_block_roi ) destination[ roiToSlice(*destination_relative_intersection) ] = dataset[ roiToSlice( *block_relative_intersection ) ] return destination
def execute_tasks( tasks ): """ Executes the given list of tasks (functions) in the lazyflow threadpool. """ pool = RequestPool() for task in tasks: pool.add( Request(task) ) pool.wait()
def read(self, view_roi, result_out): """ roi: (start, stop) tuples, ordered according to description.output_axes roi should be relative to the view """ output_axes = self.description.output_axes roi_transposed = zip(*view_roi) roi_dict = dict( zip(output_axes, roi_transposed) ) view_roi = zip( *(roi_dict['z'], roi_dict['y'], roi_dict['x']) ) # First, normalize roi and result to zyx order result_out = vigra.taggedView(result_out, output_axes) result_out = result_out.withAxes(*'zyx') assert numpy.array(view_roi).shape == (2,3), "Invalid roi for 3D volume: {}".format( view_roi ) view_roi = numpy.array(view_roi) assert (result_out.shape == (view_roi[1] - view_roi[0])).all() # User gave roi according to the view output. # Now offset it find global roi. roi = view_roi + self.description.view_origin_zyx tile_blockshape = (1,) + tuple(self.description.tile_shape_2d_yx) tile_starts = getIntersectingBlocks( tile_blockshape, roi ) pool = RequestPool() for tile_start in tile_starts: tile_roi_in = getBlockBounds( self.description.bounds_zyx, tile_blockshape, tile_start ) tile_roi_in = numpy.array(tile_roi_in) # This tile's portion of the roi intersecting_roi = getIntersection( roi, tile_roi_in ) intersecting_roi = numpy.array( intersecting_roi ) # Compute slicing within destination array and slicing within this tile destination_relative_intersection = numpy.subtract(intersecting_roi, roi[0]) tile_relative_intersection = intersecting_roi - tile_roi_in[0] # Get a view to the output slice result_region = result_out[roiToSlice(*destination_relative_intersection)] rest_args = self._get_rest_args(tile_blockshape, tile_roi_in) if self.description.tile_url_format.startswith('http'): retrieval_fn = partial( self._retrieve_remote_tile, rest_args, tile_relative_intersection, result_region ) else: retrieval_fn = partial( self._retrieve_local_tile, rest_args, tile_relative_intersection, result_region ) PARALLEL_REQ = True if PARALLEL_REQ: pool.add( Request( retrieval_fn ) ) else: # execute serially (leave the pool empty) retrieval_fn() if PARALLEL_REQ: with Timer() as timer: pool.wait() logger.info("Loading {} tiles took a total of {}".format( len(tile_starts), timer.seconds() ))
def _resolveMergers(self, hypothesesGraph, model): ''' run merger resolution on the hypotheses graph which contains the current solution ''' logger.info("Resolving mergers.") parameters = self.Parameters.value withTracklets = parameters['withTracklets'] originalGraph = hypothesesGraph.referenceTraxelGraph if withTracklets else hypothesesGraph resolvedMergersDict = {} # Enable full graph computation for animal tracking workflow withFullGraph = False if 'withAnimalTracking' in parameters and parameters['withAnimalTracking']: # TODO: Setting this parameter outside of the track() function (on AnimalConservationTrackingWorkflow) is not desirable withFullGraph = True logger.info("Computing full graph on merger resolver (Only enabled on animal tracking workflow)") mergerResolver = IlastikMergerResolver(originalGraph, pluginPaths=self.pluginPaths, withFullGraph=withFullGraph) # Check if graph contains mergers, otherwise skip merger resolving if not mergerResolver.mergerNum: logger.info("Graph contains no mergers. Skipping merger resolving.") else: # Fit and refine merger nodes using a GMM # It has to be done per time-step in order to aviod loading the whole video on RAM traxelIdPerTimestepToUniqueIdMap, uuidToTraxelMap = getMappingsBetweenUUIDsAndTraxels(model) timesteps = [int(t) for t in traxelIdPerTimestepToUniqueIdMap.keys()] timesteps.sort() timeIndex = self.LabelImage.meta.axistags.index('t') for timestep in timesteps: roi = [slice(None) for i in range(len(self.LabelImage.meta.shape))] roi[timeIndex] = slice(timestep, timestep+1) roi = tuple(roi) labelImage = self.LabelImage[roi].wait() # Get coordinates for object IDs in label image. Used by GMM merger fit. objectIds = vigra.analysis.unique(labelImage[0,...,0]) maxObjectId = max(objectIds) coordinatesForIds = {} pool = RequestPool() for objectId in objectIds: pool.add(Request(partial(mergerResolver.getCoordinatesForObjectId, coordinatesForIds, labelImage[0, ..., 0], timestep, objectId))) # Run requests to get object ID coordinates pool.wait() # Fit mergers and store fit info in nodes if coordinatesForIds: mergerResolver.fitAndRefineNodesForTimestep(coordinatesForIds, maxObjectId, timestep) # Compute object features, re-run flow solver, update model and result, and get merger dictionary resolvedMergersDict = mergerResolver.run() return resolvedMergersDict
def testBasic(self): graph = Graph() opDataProvider = OpArrayPiperWithAccessCount(graph=graph) opCache = OpUnblockedArrayCache(graph=graph) data = np.random.random((100, 100, 100)).astype(np.float32) opDataProvider.Input.setValue(vigra.taggedView(data, "zyx")) opCache.Input.connect(opDataProvider.Output) assert opCache.CleanBlocks.value == [] roi = ((30, 30, 30), (50, 50, 50)) cache_data = opCache.Output(*roi).wait() assert (cache_data == data[roiToSlice(*roi)]).all() assert opDataProvider.accessCount == 1 assert opCache.CleanBlocks.value == [roiToSlice(*roi)] # Request the same data a second time. # Access count should not change. cache_data = opCache.Output(*roi).wait() assert (cache_data == data[roiToSlice(*roi)]).all() assert opDataProvider.accessCount == 1 assert opCache.CleanBlocks.value == [roiToSlice(*roi)] # Now invalidate a part of the data # The cache will discard it, so the access count should increase. opDataProvider.Input.setDirty((30, 30, 30), (31, 31, 31)) assert opCache.CleanBlocks.value == [] cache_data = opCache.Output(*roi).wait() assert (cache_data == data[roiToSlice(*roi)]).all() assert opDataProvider.accessCount == 2 # Repeat this next part just for safety for _ in range(10): # Make sure the cache is empty opDataProvider.Input.setDirty((30, 30, 30), (31, 31, 31)) opDataProvider.accessCount = 0 # Create many requests for the same data. # Upstream data should only be accessed ONCE. pool = RequestPool() for _ in range(10): pool.add(opCache.Output(*roi)) pool.wait() assert opDataProvider.accessCount == 1 # Also, make sure requests for INNER rois of stored blocks are also serviced from memory opDataProvider.accessCount = 0 inner_roi = ((35, 35, 35), (45, 45, 45)) cache_data = opCache.Output(*inner_roi).wait() assert (cache_data == data[roiToSlice(*inner_roi)]).all() assert opDataProvider.accessCount == 0 assert opCache.CleanBlocks.value == [roiToSlice(*roi)]
def _waitForBlocks(self, block_starts): """ Make sure that all blocks in the given list of blocks are present in the cache before returning. (Blocks that are not yet present will be requested from our Input slot.) """ reqPool = RequestPool() # (Do the work in parallel.) for block_start in block_starts: entire_block_roi = getBlockBounds( self.Output.meta.shape, self._blockshape, block_start ) f = partial( self._ensureCached, entire_block_roi) reqPool.add( Request(f) ) logger.debug( "Waiting for {} blocks...".format( len(block_starts) ) ) reqPool.wait()
def export(self, filename, hypothesesGraph, pluginExportContext): """Export the tracking solution stored in the hypotheses graph as a sequence of H5 files, one per frame, containing the label image of that frame and which objects were part of a move or a division. :param filename: string of the FOLDER where to save the result :param hypothesesGraph: hytra.core.hypothesesgraph.HypothesesGraph filled with a solution :param pluginExportContext: instance of ilastik.plugins.PluginExportContext containing: labelImageSlot (required here) as well as objectFeaturesSlot, rawImageSlot, additionalPluginArgumentsSlot :returns: True on success, False otherwise """ labelImageSlot = pluginExportContext.labelImageSlot traxelIdPerTimestepToUniqueIdMap, uuidToTraxelMap = hypothesesGraph.getMappingsBetweenUUIDsAndTraxels() timesteps = [t for t in traxelIdPerTimestepToUniqueIdMap.keys()] result = hypothesesGraph.getSolutionDictionary() mergers, detections, links, divisions = getMergersDetectionsLinksDivisions(result, uuidToTraxelMap) # group by timestep for event creation mergersPerTimestep = getMergersPerTimestep(mergers, timesteps) linksPerTimestep = getLinksPerTimestep(links, timesteps) detectionsPerTimestep = getDetectionsPerTimestep(detections, timesteps) divisionsPerTimestep = getDivisionsPerTimestep(divisions, linksPerTimestep, timesteps) # save to disk in parallel pool = RequestPool() timeIndex = labelImageSlot.meta.axistags.index('t') if not os.path.exists(filename): os.makedirs(filename) for timestep in traxelIdPerTimestepToUniqueIdMap.keys(): # extract current frame lable image roi = [slice(None) for i in range(len(labelImageSlot.meta.shape))] roi[timeIndex] = slice(int(timestep), int(timestep)+1) roi = tuple(roi) labelImage = labelImageSlot[roi].wait() fn = os.path.join(filename, "{0:05d}.h5".format(int(timestep))) pool.add(Request(partial(writeEvents, int(timestep), linksPerTimestep[timestep], divisionsPerTimestep[timestep], mergersPerTimestep[timestep], detectionsPerTimestep[timestep], fn, labelImage))) pool.wait() return True
def _execute_Output(self, slot, subindex, roi, result): """ Overridden from OpUnblockedArrayCache """ def copy_block(full_block_roi, clipped_block_roi): full_block_roi = numpy.asarray(full_block_roi) clipped_block_roi = numpy.asarray(clipped_block_roi) output_roi = numpy.asarray(clipped_block_roi) - roi.start block_roi = self._get_containing_block_roi(clipped_block_roi) # Skip cache and copy full block directly if self.BypassModeEnabled.value: full_block_data = self.Output.stype.allocateDestination(SubRegion(self.Output, *full_block_roi)) self.Input(*full_block_roi).writeInto(full_block_data).block() roi_within_block = clipped_block_roi - full_block_roi[0] self.Output.stype.copy_data( result[roiToSlice(*output_roi)], full_block_data[roiToSlice(*roi_within_block)] ) # If data data exists already or we can just fetch it without needing extra scratch space, # just call the base class elif block_roi is not None or (full_block_roi == clipped_block_roi).all(): self._execute_Output_impl(clipped_block_roi, result[roiToSlice(*output_roi)]) elif self.Input.meta.dontcache: # Data isn't in the cache, but we don't need it in the cache anyway. self.Input(*clipped_block_roi).writeInto(result[roiToSlice(*output_roi)]).block() else: # Data doesn't exist yet in the cache. # Request the full block, but then discard the parts we don't need. # (We use allocateDestination() here to support MaskedArray types.) # TODO: We should probably just get rid of MaskedArray support altogether... full_block_data = self.Output.stype.allocateDestination(SubRegion(self.Output, *full_block_roi)) self._execute_Output_impl(full_block_roi, full_block_data) roi_within_block = clipped_block_roi - full_block_roi[0] self.Output.stype.copy_data( result[roiToSlice(*output_roi)], full_block_data[roiToSlice(*roi_within_block)] ) clipped_block_rois = getIntersectingRois(self.Input.meta.shape, self._blockshape, (roi.start, roi.stop), True) full_block_rois = getIntersectingRois(self.Input.meta.shape, self._blockshape, (roi.start, roi.stop), False) pool = RequestPool() for full_block_roi, clipped_block_roi in zip(full_block_rois, clipped_block_rois): req = Request(partial(copy_block, full_block_roi, clipped_block_roi)) pool.add(req) pool.wait()
def execute(self, slot, subindex, roi, result): assert len(roi.start) == len(roi.stop) == len(self.Output.meta.shape) assert slot == self.Output t_ind = self.RawVolume.meta.axistags.index('t') assert t_ind < len(self.RawVolume.meta.shape) def compute_features_for_time_slice(res_t_ind, t): axes4d = [k for k in self.RawVolume.meta.getTaggedShape().keys() if k in 'xyzc'] # Process entire spatial volume s = [slice(None)] * len(self.RawVolume.meta.shape) s[t_ind] = slice(t, t+1) s = tuple(s) # Request in parallel raw_req = self.RawVolume[s] raw_req.submit() label_req = self.LabelVolume[s] label_req.submit() if self.Atlas.ready(): atlasVolume = self.Atlas[s].wait() atlasVolume = vigra.taggedView(atlasVolume, axistags=self.Atlas.meta.axistags) atlasVolume = atlasVolume.withAxes(*axes4d) else: atlasVolume = None # Get results rawVolume = raw_req.wait() labelVolume = label_req.wait() rawVolume = vigra.taggedView(rawVolume, axistags=self.RawVolume.meta.axistags) labelVolume = vigra.taggedView(labelVolume, axistags=self.LabelVolume.meta.axistags) # Convert to 4D (preserve axis order) rawVolume = rawVolume.withAxes(*axes4d) labelVolume = labelVolume.withAxes(*axes4d) acc = self._extract(rawVolume, labelVolume, atlasVolume) # Copy into the result result[res_t_ind] = acc # loop over requested time slices pool = RequestPool() for res_t_ind, t in enumerate(range(roi.start[t_ind], roi.stop[t_ind])): pool.add( Request( partial(compute_features_for_time_slice, res_t_ind, t) ) ) pool.wait() return result
def _label(self, roi, result): result = vigra.taggedView(result, axistags=self.Output.meta.axistags) # get the background values bg = self.Background[...].wait() bg = vigra.taggedView(bg, axistags=self.Background.meta.axistags) bg = bg.withAxes(*"ct") assert np.all( self.Background.meta.shape[0] == self.Input.meta.shape[0] ), "Shape of background values incompatible to shape of Input" assert np.all( self.Background.meta.shape[4] == self.Input.meta.shape[4] ), "Shape of background values incompatible to shape of Input" # do labeling in parallel over channels and time slices pool = RequestPool() start = np.asarray(roi.start, dtype=np.int) stop = np.asarray(roi.stop, dtype=np.int) for ti, t in enumerate(range(roi.start[0], roi.stop[0])): start[0], stop[0] = t, t + 1 for ci, c in enumerate(range(roi.start[4], roi.stop[4])): start[4], stop[4] = c, c + 1 newRoi = SubRegion(self.Output, start=tuple(start), stop=tuple(stop)) resView = result[ti, ..., ci].withAxes(*"xyz") req = Request(partial(self._label3d, newRoi, bg[c, t], resView)) pool.add(req) logger.debug("{}: Computing connected components for ROI {} ...".format(self.name, roi)) pool.wait() pool.clean() logger.debug("{}: Connected components computed.".format(self.name))
def execute(self, slot, subindex, rroi, result): key = roiToSlice(rroi.start,rroi.stop) cnt = 0 written = 0 start, stop = roi.sliceToRoi(key, self.outputs["Output"].meta.shape) assert (stop<=self.outputs["Output"].meta.shape).all() #axisindex = self.inputs["AxisIndex"].value flag = self.inputs["AxisFlag"].value axisindex = self.outputs["Output"].meta.axistags.index(flag) #ugly-ugly-ugly oldkey = list(key) oldkey.pop(axisindex) #print "STACKER: ", flag, axisindex #print "requesting an outslot from stacker:", key, result.shape #print "input slots total: ", len(self.inputs['Images']) requests = [] pool = RequestPool() for i, inSlot in enumerate(self.inputs['Images']): req = None inTagKeys = [ax.key for ax in inSlot.meta.axistags] if flag in inTagKeys: slices = inSlot.meta.shape[axisindex] if cnt + slices >= start[axisindex] and start[axisindex]-cnt<slices and start[axisindex]+written<stop[axisindex]: begin = 0 if cnt < start[axisindex]: begin = start[axisindex] - cnt end = slices if cnt + end > stop[axisindex]: end -= cnt + end - stop[axisindex] key_ = copy.copy(oldkey) key_.insert(axisindex, slice(begin, end, None)) reskey = [slice(None, None, None) for x in range(len(result.shape))] reskey[axisindex] = slice(written, written+end-begin, None) req = inSlot[tuple(key_)].writeInto(result[tuple(reskey)]) written += end - begin cnt += slices else: if cnt>=start[axisindex] and start[axisindex] + written < stop[axisindex]: #print "key: ", key, "reskey: ", reskey, "oldkey: ", oldkey #print "result: ", result.shape, "inslot:", inSlot.meta.shape reskey = [slice(None, None, None) for s in oldkey] reskey.insert(axisindex, written) destArea = result[tuple(reskey)] req = inSlot[tuple(oldkey)].writeInto(destArea) written += 1 cnt += 1 if req is not None: pool.add(req) pool.wait() pool.clean()
def execute(self, slot, subindex, roi, result): t1 = time.perf_counter() key = roi.toSlice() nlabels = self.inputs["LabelsCount"].value traceLogger.debug( "OpPredictRandomForest: Requesting classifier. roi={}".format(roi)) forests = self.inputs["Classifier"][:].wait() if any(forest is None for forest in forests): # Training operator may return 'None' if there was no data to train with return np.zeros(np.subtract(roi.stop, roi.start), dtype=np.float32)[...] traceLogger.debug("OpPredictRandomForest: Got classifier") #assert RF.labelCount() == nlabels, "ERROR: OpPredictRandomForest, labelCount differs from true labelCount! %r vs. %r" % (RF.labelCount(), nlabels) newKey = key[:-1] newKey += (slice(0, self.inputs["Image"].meta.shape[-1], None), ) res = self.inputs["Image"][newKey].wait() shape = res.shape prod = np.prod(shape[:-1]) res.shape = (prod, shape[-1]) features = res predictions = [0] * len(forests) t2 = time.perf_counter() pool = RequestPool() def predict_forest(i): predictions[i] = forests[i].predict( np.asarray(features, dtype=np.float32)) predictions[i] = predictions[i].reshape(result.shape[:-1]) for i, f in enumerate(forests): req = pool.request(partial(predict_forest, i)) pool.wait() pool.clean() #predictions[0] = forests[0].predict(np.asarray(features, dtype = np.float32), normalize = False) #predictions[0] = predictions[0].reshape(result.shape) prediction = np.dstack(predictions) result[...] = prediction # If our LabelsCount is higher than the number of labels in the training set, # then our results aren't really valid. FIXME !!! # Duplicate the last label's predictions #for c in range(result.shape[-1]): # result[...,c] = prediction[...,min(c+key[-1].start, prediction.shape[-1]-1)] t3 = time.perf_counter() logger.debug( "Predict took %fseconds, actual RF time was %fs, feature time was %fs" % (t3 - t1, t3 - t2, t2 - t1)) return result
def _label(self, roi, result): result = vigra.taggedView(result, axistags=self.Output.meta.axistags) # get the background values bg = self.Background[...].wait() bg = vigra.taggedView(bg, axistags=self.Background.meta.axistags) bg = bg.withAxes(*'ct') assert np.all(self.Background.meta.shape[3:] == self.Input.meta.shape[3:]),\ "Shape of background values incompatible to shape of Input" # do labeling in parallel over channels and time slices pool = RequestPool() start = np.asarray(roi.start, dtype=np.int) stop = np.asarray(roi.stop, dtype=np.int) for ti, t in enumerate(range(roi.start[4], roi.stop[4])): start[4], stop[4] = t, t+1 for ci, c in enumerate(range(roi.start[3], roi.stop[3])): start[3], stop[3] = c, c+1 newRoi = SubRegion(self.Output, start=tuple(start), stop=tuple(stop)) resView = result[..., ci, ti].withAxes(*'xyz') req = Request(partial(self._label3d, newRoi, bg[c, t], resView)) pool.add(req) logger.debug( "{}: Computing connected components for ROI {} ...".format( self.name, roi)) pool.wait() pool.clean() logger.debug("{}: Connected components computed.".format( self.name))
def execute(self, slot, subindex, roi, result): assert slot == self._ReorderedOutput pool = RequestPool() t_ind = 0 for t in range(roi.start[0], roi.stop[0]): c_ind = 0 for c in range(roi.start[-1], roi.stop[-1]): newroi = roi.copy() newroi.start[0] = t newroi.stop[0] = t+1 newroi.start[-1] = c newroi.stop[-1] = c+1 req = self._op.Output.get(newroi) resView = result[t_ind:t_ind+1, ..., c_ind:c_ind+1] req.writeInto(resView) pool.add(req) c_ind += 1 t_ind += 1 pool.wait() pool.clean()
def execute(self, slot, subindex, roi, result): assert slot == self.ConcatenatedOutput self.progressSignal(0.0) num_dirty_slots = len(self._dirty_slots) subtask_progress = {} progress_lock = RequestLock() def forward_progress_updates(feature_slot, progress): with progress_lock: subtask_progress[feature_slot] = progress total_progress = 0.95 * sum(subtask_progress.values()) / num_dirty_slots self.progressSignal(total_progress) logger.debug( "Updating features for {} dirty images out of {}" "".format(len(self._dirty_slots), len(self.FeatureMatrices)) ) pool = RequestPool() subresults = [] for feature_slot, progress_slot in zip(self.FeatureMatrices, self.ProgressSignals): subresults.append([None]) req = feature_slot[:] req.writeInto(subresults[-1]) # Only use progress for slots that were dirty. # The others are going to be really fast. if feature_slot in self._dirty_slots: sub_progress_signal = progress_slot.value sub_progress_signal.subscribe(partial(forward_progress_updates, feature_slot)) pool.add(req) pool.wait() # Reset dirty slots self._dirty_slots = set() # Since the subresults are returned in 'value' slots, # we have to unpack them from their single-element lists. subresult_list = list(itertools.chain(*subresults)) total_matrix = numpy.concatenate(subresult_list, axis=0) self.progressSignal(100.0) result[0] = total_matrix
def execute(self, slot, subindex, roi, result): def compute_for_channel(output_channel, input_channel): input_roi = numpy.array((roi.start, roi.stop)) input_roi[:, -1] = (input_channel, input_channel + 1) input_req = self.Input(*input_roi) # If possible, use the result array itself as a scratch area if self.Input.meta.dtype == result.dtype: input_req.writeInto(result[..., output_channel : output_channel + 1]) input_data = input_req.wait() input_data = input_data.astype(numpy.float32, order="C", copy=False) input_data = input_data[..., 0] # drop channel axis result[..., output_channel] = computeIntegralImage(input_data) pool = RequestPool() for output_channel, input_channel in enumerate(range(roi.start[-1], roi.stop[-1])): pool.add(Request(partial(compute_for_channel, output_channel, input_channel))) pool.wait()
def _train_forests_with_feature_importance(forests, X, y, feature_names, export_path=None): """ Train all RFs (in parallel) and compute feature importances while doing so. The importances table will be logged as INFO, and also exported to a file if export_path is given. Returns: oobs and importances """ oobs = [None] * len(forests) importances = [None] * len(forests) def store_training_results(i, training_results): oob, importance_results = training_results oobs[i] = oob importances[i] = importance_results with Timer() as train_timer: pool = RequestPool() for i, forest in enumerate(forests): req = Request(partial(forest.learnRFWithFeatureSelection, X, y)) # save the training results req.notify_finished(partial(store_training_results, i)) pool.add(req) pool.wait() logger.info("Training took, {} seconds".format(train_timer.seconds())) # Forests may have different numbers of trees, # so take a weighted average of their importances tree_counts = [f.treeCount() for f in forests] weights = numpy.array(tree_counts).astype(float) weights /= weights.sum() named_importances = collections.OrderedDict( list(zip(feature_names, numpy.average(importances, weights=weights, axis=0))) ) importance_table = generate_importance_table(named_importances, sort="overall", export_path=export_path) logger.info("Feature importance measurements during training: \n{}".format(importance_table)) return oobs, named_importances
def run(self): # # Load caches beforehand (To remove overhead of reading frames) # with Timer() as timerCaches: # rawVol = self.opCacheRaw.Output([]).wait() # binaryVol = self.opCacheBinary.Output([]).wait() # # print "Caches took {} secs".format(timerCaches.seconds()) # # del rawVol # del binaryVol # Profile object extraction simplified print("\nStarting object extraction simplified (single-thread, without cache)") with Timer() as timerObjectFeaturesSimp: featsObjectFeaturesSimp = self.opObjectFeaturesSimp.Features([]).wait() print("Simplified object extraction took: {} seconds".format(timerObjectFeaturesSimp.seconds())) # Profile object extraction optimized print("\nStarting object extraction (multi-thread, without cache)") with Timer() as timerObjectExtraction: featsObjectExtraction = self.opObjectExtraction.RegionFeatures([]).wait() print("Object extraction took: {} seconds".format(timerObjectExtraction.seconds())) # Profile for basic multi-threaded feature computation # just a multi-threaded loop that labels volumes and extract object features directly (No operators, no plugin system, no overhead, just a loop) featsBasicFeatureComp = dict.fromkeys( list(range(self.op5Raw.Output.meta.shape[0])), None) print("\nStarting basic multi-threaded feature computation") pool = RequestPool() for t in range(0, self.op5Raw.Output.meta.shape[0], 1): pool.add( Request( partial(self._computeObjectFeatures, t, featsBasicFeatureComp) ) ) with Timer() as timerBasicFeatureComp: pool.wait() print("Basic multi-threaded feature extraction took: {} seconds".format( timerBasicFeatureComp.seconds() ))
def create_and_train(self, X, y): logger.debug("Training parallel vigra RF") # Save for future reference known_labels = numpy.unique(y) X = numpy.asarray(X, numpy.float32) y = numpy.asarray(y, numpy.uint32) if y.ndim == 1: y = y[:, numpy.newaxis] assert X.ndim == 2 assert len(X) == len(y) # Create N forests forests = [] for _ in range(self._num_forests): forest = vigra.learning.RandomForest(self._trees_per_forest, **self._kwargs) forests.append(forest) # Train them all in parallel pool = RequestPool() for forest in forests: pool.add(Request(partial(forest.learnRF, X, y))) pool.wait() return ParallelVigraRfLazyflowClassifier(forests, known_labels)
def predict_probabilities(self, X): logger.debug("Predicting with parallel vigra RF") X = numpy.asarray(X, dtype=numpy.float32) assert X.ndim == 2 if self._feature_names is not None: # For some reason, vigra doesn't seem to check this for us... assert X.shape[1] == len(self._feature_names), \ "Feature count doesn't match the training data." # As each forest completes, aggregate results in a shared array. # (Must put in a list so we can update it in this closure.) total_predictions = [None] prediction_lock = RequestLock() def update_predictions(forest, forest_predictions): forest_predictions *= forest.treeCount() with prediction_lock: if total_predictions[0] is None: total_predictions[0] = forest_predictions else: total_predictions[0] += forest_predictions # Create a request for each forest pool = RequestPool() for forest in self._forests: req = Request(partial(forest.predictProbabilities, X)) req.notify_finished(partial(update_predictions, forest)) pool.add(req) del req pool.wait() total_predictions[0] /= self._num_trees return total_predictions[0]
def execute(self, slot, subindex, roi, result): def compute_for_channel(output_channel, input_channel): input_roi = numpy.array((roi.start, roi.stop)) input_roi[:, -1] = (input_channel, input_channel + 1) input_req = self.Input(*input_roi) # If possible, use the result array itself as a scratch area if self.Input.meta.dtype == result.dtype: input_req.writeInto(result[..., output_channel:output_channel + 1]) input_data = input_req.wait() input_data = input_data.astype(numpy.float32, order='C', copy=False) input_data = input_data[..., 0] # drop channel axis result[..., output_channel] = computeIntegralImage(input_data) pool = RequestPool() for output_channel, input_channel in enumerate( range(roi.start[-1], roi.stop[-1])): pool.add( Request( partial(compute_for_channel, output_channel, input_channel))) pool.wait()
def execute(self, slot, subindex, roi, result): clipped_block_rois = getIntersectingRois(self.Input.meta.shape, self.BlockShape.value, (roi.start, roi.stop), True) if self._always_request_full_blocks: full_block_rois = getIntersectingRois(self.Input.meta.shape, self.BlockShape.value, (roi.start, roi.stop), False) else: full_block_rois = clipped_block_rois pool = RequestPool() for full_block_roi, clipped_block_roi in zip(full_block_rois, clipped_block_rois): full_block_roi = numpy.asarray(full_block_roi) clipped_block_roi = numpy.asarray(clipped_block_roi) req = self.Input(*full_block_roi) output_roi = numpy.asarray(clipped_block_roi) - roi.start if (full_block_roi == clipped_block_roi).all(): req.writeInto(result[roiToSlice(*output_roi)]) else: roi_within_block = clipped_block_roi - full_block_roi[0] def copy_request_result(output_roi, roi_within_block, request_result): self.Output.stype.copy_data( result[roiToSlice(*output_roi)], request_result[roiToSlice(*roi_within_block)]) req.notify_finished( partial(copy_request_result, output_roi, roi_within_block)) pool.add(req) del req pool.wait()
def execute(self, slot, subindex, roi, result): with self._lock: if self.cache is None: fullBlockShape = numpy.array([self.blockShape.value for i in self.Input.meta.shape]) fun = self.inputs["Function"].value #data = self.inputs["Input"][:].wait() #split up requests into blocks shape = self.Input.meta.shape numBlocks = numpy.ceil(shape/(1.0*fullBlockShape)).astype("int") blockCache = numpy.ndarray(shape = numpy.prod(numBlocks), dtype=self.Output.meta.dtype) pool = RequestPool() #blocks holds the different roi keys for each of the blocks blocks = itertools.product(*[range(i) for i in numBlocks]) blockKeys = [] for b in blocks: start = b * fullBlockShape stop = b * fullBlockShape + fullBlockShape stop = numpy.min(numpy.vstack((stop, shape)), axis=0) blockKey = roiToSlice(start, stop) blockKeys.append(blockKey) def predict_block(i): data = self.Input[blockKeys[i]].wait() blockCache[i] = fun(data) for i,f in enumerate(blockCache): req = pool.request(partial(predict_block,i)) pool.wait() pool.clean() self.cache = [fun(blockCache)] return self.cache
def execute(self, slot, subindex, roi, result): stacked_axisindex = self.Images[0].meta.getAxisKeys().index( self.AxisFlag.value) pool = RequestPool() for slot, (slot_output_start, slot_output_stop) in zip(self.Images, self.stacked_output_ranges): if roi.start[stacked_axisindex] >= slot_output_start and roi.stop[ stacked_axisindex] <= slot_output_stop: output_roi = roi.copy() output_roi.start[stacked_axisindex] = max( slot_output_start, roi.start[stacked_axisindex]) output_roi.stop[stacked_axisindex] = min( slot_output_stop, roi.stop[stacked_axisindex]) request_roi = roi.copy() request_roi.start[stacked_axisindex] = output_roi.start[ stacked_axisindex] - slot_output_start request_roi.stop[stacked_axisindex] = output_roi.stop[ stacked_axisindex] - slot_output_start result_roi = roi.copy() result_roi.start = output_roi.start - roi.start result_roi.stop = output_roi.stop - roi.start req = slot(request_roi.start, request_roi.stop) req.writeInto(result[roiToSlice(result_roi.start, result_roi.stop)]) pool.add(req) pool.wait()
def execute(self, slot, subindex, roi, result): assert len(roi.start) == len(roi.stop) == len(self.Output.meta.shape) assert slot == self.Output t_ind = self.RawVolume.meta.axistags.index('t') assert t_ind < len(self.RawVolume.meta.shape) def compute_features_for_time_slice(res_t_ind, t): axes4d = [ k for k in self.RawVolume.meta.getTaggedShape().keys() if k in 'xyzc' ] # Process entire spatial volume s = [slice(None)] * len(self.RawVolume.meta.shape) s[t_ind] = slice(t, t + 1) s = tuple(s) # Request in parallel raw_req = self.RawVolume[s] raw_req.submit() label_req = self.LabelVolume[s] label_req.submit() if self.Atlas.ready(): atlasVolume = self.Atlas[s].wait() atlasVolume = vigra.taggedView( atlasVolume, axistags=self.Atlas.meta.axistags) atlasVolume = atlasVolume.withAxes(*axes4d) else: atlasVolume = None # Get results rawVolume = raw_req.wait() labelVolume = label_req.wait() rawVolume = vigra.taggedView(rawVolume, axistags=self.RawVolume.meta.axistags) labelVolume = vigra.taggedView( labelVolume, axistags=self.LabelVolume.meta.axistags) # Convert to 4D (preserve axis order) rawVolume = rawVolume.withAxes(*axes4d) labelVolume = labelVolume.withAxes(*axes4d) acc = self._extract(rawVolume, labelVolume, atlasVolume) # Copy into the result result[res_t_ind] = acc # loop over requested time slices pool = RequestPool() for res_t_ind, t in enumerate(range(roi.start[t_ind], roi.stop[t_ind])): pool.add( Request(partial(compute_features_for_time_slice, res_t_ind, t))) pool.wait() return result
def predict(cls, X, method="classic"): """ predict if the histograms in X correspond to missing regions do this for subsets of X in parallel """ if cls._manager is None: cls._manager = SVMManager() assert len( X.shape ) == 2, "Prediction data must have shape (nSamples, nHistogramBins)." nBins = X.shape[1] if method == "classic": svm = PseudoSVC() else: try: svm = cls._manager.get(nBins) except SVMManager.NotTrainedError: # fail gracefully if not trained => responsibility of user! svm = PseudoSVC() y = np.zeros((len(X), )) * np.nan pool = RequestPool() chunkSize = 1000 # FIXME magic number?? nChunks = len(X) // chunkSize + (1 if len(X) % chunkSize > 0 else 0) s = [ slice(k * chunkSize, min((k + 1) * chunkSize, len(X))) for k in range(nChunks) ] def partFun(i): y[s[i]] = svm.predict(X[s[i]]) for i in range(nChunks): req = Request(partial(partFun, i)) pool.add(req) pool.wait() pool.clean() # not neccessary # assert not np.any(np.isnan(y)) return np.asarray(y)
def execute(self, slot, subindex, roi, result): stacked_axisindex = self.Images[0].meta.getAxisKeys().index(self.AxisFlag.value) pool = RequestPool() for slot, (slot_output_start, slot_output_stop) in zip(self.Images, self.stacked_output_ranges): if roi.start[stacked_axisindex] >= slot_output_start and roi.stop[stacked_axisindex] <= slot_output_stop: output_roi = roi.copy() output_roi.start[stacked_axisindex] = max(slot_output_start, roi.start[stacked_axisindex]) output_roi.stop[stacked_axisindex] = min(slot_output_stop, roi.stop[stacked_axisindex]) request_roi = roi.copy() request_roi.start[stacked_axisindex] = output_roi.start[stacked_axisindex] - slot_output_start request_roi.stop[stacked_axisindex] = output_roi.stop[stacked_axisindex] - slot_output_start result_roi = roi.copy() result_roi.start = output_roi.start - roi.start result_roi.stop = output_roi.stop - roi.start req = slot(request_roi.start, request_roi.stop) req.writeInto( result[roiToSlice(result_roi.start, result_roi.stop)] ) pool.add( req ) pool.wait()
def export(self, filename, hypothesesGraph, objectFeaturesSlot, labelImageSlot, rawImageSlot): """Export the tracking solution stored in the hypotheses graph as a sequence of H5 files, one per frame, containing the label image of that frame and which objects were part of a move or a division. :param filename: string of the FOLDER where to save the result :param hypothesesGraph: hytra.core.hypothesesgraph.HypothesesGraph filled with a solution :param objectFeaturesSlot: lazyflow.graph.InputSlot, connected to the RegionFeaturesAll output of ilastik.applets.trackingFeatureExtraction.opTrackingFeatureExtraction.OpTrackingFeatureExtraction :returns: True on success, False otherwise """ traxelIdPerTimestepToUniqueIdMap, uuidToTraxelMap = hypothesesGraph.getMappingsBetweenUUIDsAndTraxels( ) timesteps = [t for t in traxelIdPerTimestepToUniqueIdMap.keys()] result = hypothesesGraph.getSolutionDictionary() mergers, detections, links, divisions = getMergersDetectionsLinksDivisions( result, uuidToTraxelMap) # group by timestep for event creation mergersPerTimestep = getMergersPerTimestep(mergers, timesteps) linksPerTimestep = getLinksPerTimestep(links, timesteps) detectionsPerTimestep = getDetectionsPerTimestep( detections, timesteps) divisionsPerTimestep = getDivisionsPerTimestep( divisions, linksPerTimestep, timesteps) # save to disk in parallel pool = RequestPool() timeIndex = labelImageSlot.meta.axistags.index('t') for timestep in traxelIdPerTimestepToUniqueIdMap.keys(): # extract current frame lable image roi = [ slice(None) for i in range(len(labelImageSlot.meta.shape)) ] roi[timeIndex] = slice(int(timestep), int(timestep) + 1) roi = tuple(roi) labelImage = labelImageSlot[roi].wait() if not os.path.exists(filename + '/H5-Event-Sequence'): os.makedirs(filename + '/H5-Event-Sequence') fn = os.path.join( filename, "H5-Event-Sequence/{0:05d}.h5".format(int(timestep))) pool.add( Request( partial(writeEvents, int(timestep), linksPerTimestep[timestep], divisionsPerTimestep[timestep], mergersPerTimestep[timestep], detectionsPerTimestep[timestep], fn, labelImage))) pool.wait() return True
def create_and_train(self, X, y, feature_names=None): # Distribute trees as evenly as possible tree_counts = numpy.array( [self._num_trees // self._num_forests] * self._num_forests ) tree_counts[:self._num_trees % self._num_forests] += 1 assert tree_counts.sum() == self._num_trees tree_counts = map(int, tree_counts) tree_counts[:] = (tree_count for tree_count in tree_counts if tree_count != 0) logger.debug( "Training parallel vigra RF" ) # Save for future reference known_labels = numpy.unique(y) X = numpy.asarray(X, numpy.float32) y = numpy.asarray(y, numpy.uint32) if y.ndim == 1: y = y[:, numpy.newaxis] assert X.ndim == 2 assert len(X) == len(y) # Create N forests forests = [] for tree_count in tree_counts: forests.append( vigra.learning.RandomForest(tree_count, **self._kwargs) ) # HERE <--- this links to C++ library # Train them all in parallel oobs = [None] * len(forests) pool = RequestPool() for i, forest in enumerate(forests): req = Request( partial(forest.learnRF, X, y) ) # save the oobs req.notify_finished( partial( oobs.__setitem__, i ) ) pool.add( req ) with Timer() as timer: pool.wait() logger.info( "Training completed in {} seconds. Average OOB: {}".format( timer.seconds(), numpy.average(oobs) ) ) return ParallelVigraRfLazyflowClassifier( forests, oobs, known_labels, feature_names )
def execute_tasks(tasks): """ Executes the given list of tasks (functions) in the lazyflow threadpool. """ pool = RequestPool() for task in tasks: pool.add(Request(task)) pool.wait()
def _executePredictionImage(self, slot, roi, destination): roi_one_channel = numpy.array( (roi.start, roi.stop) ) roi_one_channel[...,-1] = (0,1) # Determine intersecting blocks block_shape = self._getFullShape( self.BlockShape3dDict.value ) block_starts = getIntersectingBlocks( block_shape, roi_one_channel ) block_starts = map( tuple, block_starts ) # Ensure that block pipelines exist (create first if necessary) for block_start in block_starts: self._ensurePipelineExists(block_start) # Retrieve result from each block, and write into the appropriate region of the destination pool = RequestPool() for block_start in block_starts: opBlockPipeline = self._blockPipelines[block_start] block_roi = opBlockPipeline.block_roi block_intersection = getIntersection( block_roi, roi_one_channel ) block_relative_intersection = numpy.subtract(block_intersection, block_roi[0]) destination_relative_intersection = numpy.subtract(block_intersection, roi_one_channel[0]) block_slot = opBlockPipeline.PredictionImage if slot == self.ProbabilityChannelImage: block_slot = opBlockPipeline.ProbabilityChannelImage # Add channels back to roi # request all channels block_relative_intersection[...,-1] = (0, opBlockPipeline.ProbabilityChannelImage.meta.shape[-1]) # But only write the ones that were specified in the original roi destination_relative_intersection[...,-1] = ( roi.start[-1], roi.stop[-1] ) # Request the data destination_slice = roiToSlice( *destination_relative_intersection ) req = block_slot( *block_relative_intersection ) req.writeInto( destination[destination_slice] ) pool.add( req ) pool.wait() return destination
def read(self, view_roi, result_out): """ roi: (start, stop) tuples, ordered according to description.output_axes roi should be relative to the view """ output_axes = self.description.output_axes roi_transposed = list(zip(*view_roi)) roi_dict = dict(list(zip(output_axes, roi_transposed))) view_roi = list(zip(*(roi_dict["z"], roi_dict["y"], roi_dict["x"]))) # First, normalize roi and result to zyx order result_out = vigra.taggedView(result_out, output_axes) result_out = result_out.withAxes(*"zyx") assert numpy.array(view_roi).shape == (2, 3), "Invalid roi for 3D volume: {}".format(view_roi) view_roi = numpy.array(view_roi) assert (result_out.shape == (view_roi[1] - view_roi[0])).all() # User gave roi according to the view output. # Now offset it find global roi. roi = view_roi + self.description.view_origin_zyx tile_blockshape = (1,) + tuple(self.description.tile_shape_2d_yx) tile_starts = getIntersectingBlocks(tile_blockshape, roi) pool = RequestPool() for tile_start in tile_starts: tile_roi_in = getBlockBounds(self.description.bounds_zyx, tile_blockshape, tile_start) tile_roi_in = numpy.array(tile_roi_in) # This tile's portion of the roi intersecting_roi = getIntersection(roi, tile_roi_in) intersecting_roi = numpy.array(intersecting_roi) # Compute slicing within destination array and slicing within this tile destination_relative_intersection = numpy.subtract(intersecting_roi, roi[0]) tile_relative_intersection = intersecting_roi - tile_roi_in[0] # Get a view to the output slice result_region = result_out[roiToSlice(*destination_relative_intersection)] rest_args = self._get_rest_args(tile_blockshape, tile_roi_in) if self.description.tile_url_format.startswith("http"): retrieval_fn = partial(self._retrieve_remote_tile, rest_args, tile_relative_intersection, result_region) else: retrieval_fn = partial(self._retrieve_local_tile, rest_args, tile_relative_intersection, result_region) PARALLEL_REQ = True if PARALLEL_REQ: pool.add(Request(retrieval_fn)) else: # execute serially (leave the pool empty) retrieval_fn() if PARALLEL_REQ: with Timer() as timer: pool.wait() logger.info("Loading {} tiles took a total of {}".format(len(tile_starts), timer.seconds()))
def predict_probabilities(self, X): logger.debug( "Predicting with parallel vigra RF" ) X = numpy.asarray(X, dtype=numpy.float32) # Create a request for each forest reqs = [] for forest in self._forests: req = Request( partial( forest.predictProbabilities, X ) ) reqs.append( req ) # Execute all requests in a pool pool = RequestPool() for req in reqs: pool.add( req ) pool.wait() # Aggregate the results predictions = reqs[0].result for req in reqs[1:]: predictions += req.result predictions /= len(reqs) return predictions
def execute(self, slot, subindex, roi, result): t1 = time.time() key = roi.toSlice() nlabels=self.inputs["LabelsCount"].value traceLogger.debug("OpPredictRandomForest: Requesting classifier. roi={}".format(roi)) forests=self.inputs["Classifier"][:].wait() if forests is None or any(x is None for x in forests): # Training operator may return 'None' if there was no data to train with return numpy.zeros(numpy.subtract(roi.stop, roi.start), dtype=numpy.float32)[...] traceLogger.debug("OpPredictRandomForest: Got classifier") #assert RF.labelCount() == nlabels, "ERROR: OpPredictRandomForest, labelCount differs from true labelCount! %r vs. %r" % (RF.labelCount(), nlabels) newKey = key[:-1] newKey += (slice(0,self.inputs["Image"].meta.shape[-1],None),) res = self.inputs["Image"][newKey].wait() shape=res.shape prod = numpy.prod(shape[:-1]) res.shape = (prod, shape[-1]) features=res predictions = [0]*len(forests) def predict_forest(number): predictions[number] = forests[number].predictProbabilities(numpy.asarray(features, dtype=numpy.float32)) t2 = time.time() # predict the data with all the forests in parallel pool = RequestPool() for i,f in enumerate(forests): req = pool.request(partial(predict_forest, i)) pool.wait() pool.clean() prediction=numpy.dstack(predictions) prediction = numpy.average(prediction, axis=2) prediction.shape = shape[:-1] + (forests[0].labelCount(),) #prediction = prediction.reshape(*(shape[:-1] + (forests[0].labelCount(),))) # If our LabelsCount is higher than the number of labels in the training set, # then our results aren't really valid. FIXME !!! # Duplicate the last label's predictions for c in range(result.shape[-1]): result[...,c] = prediction[...,min(c+key[-1].start, prediction.shape[-1]-1)] t3 = time.time() self.logger.debug("predict roi=%r took %fseconds, actual RF time was %fs, feature time was %fs" % (key, t3-t1, t3-t2, t2-t1)) return result
def compute_all_features(): # Compute features in parallel pool = RequestPool() for t in range(tMax): pool.add( Request( partial(compute_features_for_frame, tIndex, t, features))) pool.wait()
def execute(self, slot, subindex, roi, result): assert slot == self.LabelAndFeatureMatrix self.progressSignal(0.0) # Technically, this could result in strange progress reporting if execute() # is called by multiple threads in parallel. # This could be fixed with some fancier progress state, but # (1) We don't expect that to by typical, and # (2) progress reporting is merely informational. num_dirty_blocks = len( self._dirty_blocks ) def update_progress( result ): remaining_dirty = len( self._dirty_blocks ) percent_complete = 95.0*(num_dirty_blocks - remaining_dirty)/num_dirty_blocks self.progressSignal( percent_complete ) # Update all dirty blocks in the cache logger.debug( "Updating {} dirty blocks".format(num_dirty_blocks) ) pool = RequestPool() for block_start in self._dirty_blocks: req = Request( partial(self._update_block, block_start ) ) req.notify_finished( update_progress ) pool.add( req ) pool.wait() # Concatenate the all blockwise results if self._blockwise_feature_matrices: total_feature_matrix = numpy.concatenate( self._blockwise_feature_matrices.values(), axis=0 ) else: # No label points at all. # Return an empty label&feature matrix (of the correct shape) num_feature_channels = self.FeatureImage.meta.shape[-1] total_feature_matrix = numpy.ndarray( shape=(0, 1 + num_feature_channels), dtype=numpy.float ) self.progressSignal(100.0) logger.debug( "After update, there are {} clean blocks".format( len(self._blockwise_feature_matrices) ) ) result[0] = total_feature_matrix
def execute(self, slot, subindex, roi, result): featList = [] labelsList = [] for i in range(len(self.Labels)): feats = self.Features[i]([]).wait() # TODO: we should be able to use self.Labels[i].value, # but the current implementation of Slot.value() does not # do the right thing. labels = self.Labels[i]([]).wait() featstmp, labelstmp = make_feature_array(feats, labels) featList.append(featstmp) labelsList.append(labelstmp) featMatrix = _concatenate(featList, axis=0) labelsMatrix = _concatenate(labelsList, axis=0) print "training on matrix:", featMatrix.shape, featMatrix.dtype if len(featMatrix) == 0 or len(labelsMatrix) == 0: result[:] = None return oob = [0] * self.ForestCount.value try: # Ensure there are no NaNs in the feature matrix # TODO: There should probably be a better way to fix this... featMatrix = numpy.asarray(featMatrix, dtype=numpy.float32) nanFeatMatrix = numpy.isnan(featMatrix) if nanFeatMatrix.any(): warnings.warn("Feature matrix has NaN values! Replacing with 0.0...") featMatrix[numpy.where(nanFeatMatrix)] = 0.0 # train and store forests in parallel pool = RequestPool() for i in range(self.ForestCount.value): def train_and_store(number): result[number] = vigra.learning.RandomForest(self._tree_count) oob[number] = result[number].learnRF(featMatrix, numpy.asarray(labelsMatrix, dtype=numpy.uint32)) print "intermediate oob:", oob[number] req = Request( partial(train_and_store, i) ) pool.add( req ) pool.wait() pool.clean() except: print ("couldn't learn classifier") raise oob_total = numpy.mean(oob) print "training finished, out of bag error:", oob_total return result
def testBasic(self): graph = Graph() opDataProvider = OpArrayPiperWithAccessCount(graph=graph) opCache = OpUnblockedArrayCache(graph=graph) data = np.random.random((100, 100, 100)).astype(np.float32) opDataProvider.Input.setValue(vigra.taggedView(data, 'zyx')) opCache.Input.connect(opDataProvider.Output) assert opCache.CleanBlocks.value == [] roi = ((30, 30, 30), (50, 50, 50)) cache_data = opCache.Output(*roi).wait() assert (cache_data == data[roiToSlice(*roi)]).all() assert opDataProvider.accessCount == 1 assert opCache.CleanBlocks.value == [roiToSlice(*roi)] # Request the same data a second time. # Access count should not change. cache_data = opCache.Output(*roi).wait() assert (cache_data == data[roiToSlice(*roi)]).all() assert opDataProvider.accessCount == 1 assert opCache.CleanBlocks.value == [roiToSlice(*roi)] # Now invalidate a part of the data # The cache will discard it, so the access count should increase. opDataProvider.Input.setDirty((30, 30, 30), (31, 31, 31)) assert opCache.CleanBlocks.value == [] cache_data = opCache.Output(*roi).wait() assert (cache_data == data[roiToSlice(*roi)]).all() assert opDataProvider.accessCount == 2 # Repeat this next part just for safety for _ in range(10): # Make sure the cache is empty opDataProvider.Input.setDirty((30, 30, 30), (31, 31, 31)) opDataProvider.accessCount = 0 # Create many requests for the same data. # Upstream data should only be accessed ONCE. pool = RequestPool() for _ in range(10): pool.add(opCache.Output(*roi)) pool.wait() assert opDataProvider.accessCount == 1 # Also, make sure requests for INNER rois of stored blocks are also serviced from memory opDataProvider.accessCount = 0 inner_roi = ((35, 35, 35), (45, 45, 45)) cache_data = opCache.Output(*inner_roi).wait() assert (cache_data == data[roiToSlice(*inner_roi)]).all() assert opDataProvider.accessCount == 0 assert opCache.CleanBlocks.value == [roiToSlice(*roi)]
def run(self): # # Load caches beforehand (To remove overhead of reading frames) # with Timer() as timerCaches: # rawVol = self.opCacheRaw.Output([]).wait() # binaryVol = self.opCacheBinary.Output([]).wait() # # print "Caches took {} secs".format(timerCaches.seconds()) # # del rawVol # del binaryVol # Profile object extraction simplified print( "\nStarting object extraction simplified (single-thread, without cache)" ) with Timer() as timerObjectFeaturesSimp: featsObjectFeaturesSimp = self.opObjectFeaturesSimp.Features( []).wait() print("Simplified object extraction took: {} seconds".format( timerObjectFeaturesSimp.seconds())) # Profile object extraction optimized print("\nStarting object extraction (multi-thread, without cache)") with Timer() as timerObjectExtraction: featsObjectExtraction = self.opObjectExtraction.RegionFeatures( []).wait() print("Object extraction took: {} seconds".format( timerObjectExtraction.seconds())) # Profile for basic multi-threaded feature computation # just a multi-threaded loop that labels volumes and extract object features directly (No operators, no plugin system, no overhead, just a loop) featsBasicFeatureComp = dict.fromkeys( list(range(self.op5Raw.Output.meta.shape[0])), None) print("\nStarting basic multi-threaded feature computation") pool = RequestPool() for t in range(0, self.op5Raw.Output.meta.shape[0], 1): pool.add( Request( partial(self._computeObjectFeatures, t, featsBasicFeatureComp))) with Timer() as timerBasicFeatureComp: pool.wait() print( "Basic multi-threaded feature extraction took: {} seconds".format( timerBasicFeatureComp.seconds()))
def execute(self, slot, subindex, roi, result): with self._lock: if self.cache is None: shape = self.Input.meta.shape # self.blockshape has None in the last dimension to indicate that it should not be # handled block-wise. None is replaced with the image shape in the respective axis. fullBlockShape = [] for u, v in zip(self.blockShape.value, shape): if u is not None: fullBlockShape.append(u) else: fullBlockShape.append(v) fullBlockShape = numpy.array(fullBlockShape, dtype=numpy.float64) # data = self.inputs["Input"][:].wait() # split up requests into blocks numBlocks = numpy.ceil(shape / fullBlockShape).astype("int") blockCache = numpy.ndarray(shape=numpy.prod(numBlocks), dtype=self.Output.meta.dtype) pool = RequestPool() # blocks holds the different roi keys for each of the blocks blocks = itertools.product( *[list(range(i)) for i in numBlocks]) blockKeys = [] for b in blocks: start = b * fullBlockShape stop = b * fullBlockShape + fullBlockShape stop = numpy.min(numpy.vstack((stop, shape)), axis=0) blockKey = roiToSlice(start, stop) blockKeys.append(blockKey) fun = self.inputs["Function"].value def predict_block(i): data = self.Input[blockKeys[i]].wait() blockCache[i] = fun(data) for i, f in enumerate(blockCache): req = pool.request(partial(predict_block, i)) pool.wait() pool.clean() self.cache = [fun(blockCache)] return self.cache
def _train_forests_with_feature_importance(forests, X, y, feature_names, export_path=None): """ Train all RFs (in parallel) and compute feature importances while doing so. The importances table will be logged as INFO, and also exported to a file if export_path is given. Returns: oobs and importances """ oobs = [None] * len(forests) importances = [None] * len(forests) def store_training_results(i, training_results): oob, importance_results = training_results oobs[i] = oob importances[i] = importance_results with Timer() as train_timer: pool = RequestPool() for i, forest in enumerate(forests): req = Request(partial(forest.learnRFWithFeatureSelection, X, y)) # save the training results req.notify_finished(partial(store_training_results, i)) pool.add(req) pool.wait() logger.info("Training took, {} seconds".format(train_timer.seconds())) # Forests may have different numbers of trees, # so take a weighted average of their importances tree_counts = [f.treeCount() for f in forests] weights = numpy.array(tree_counts).astype(float) weights /= weights.sum() named_importances = collections.OrderedDict( list( zip(feature_names, numpy.average(importances, weights=weights, axis=0)))) importance_table = generate_importance_table(named_importances, sort="overall", export_path=export_path) logger.info( "Feature importance measurements during training: \n{}".format( importance_table)) return oobs, named_importances
def _execute_Output(self, slot, subindex, roi, result): """ Overridden from OpUnblockedArrayCache """ def copy_block(full_block_roi, clipped_block_roi): full_block_roi = numpy.asarray(full_block_roi) clipped_block_roi = numpy.asarray(clipped_block_roi) output_roi = numpy.asarray(clipped_block_roi) - roi.start # If data data exists already or we can just fetch it without needing extra scratch space, # just call the base class block_roi = self._get_containing_block_roi(clipped_block_roi) if block_roi is not None or (full_block_roi == clipped_block_roi).all(): self._execute_Output_impl(clipped_block_roi, result[roiToSlice(*output_roi)]) elif self.Input.meta.dontcache: # Data isn't in the cache, but we don't need it in the cache anyway. self.Input(*clipped_block_roi).writeInto( result[roiToSlice(*output_roi)]).block() else: # Data doesn't exist yet in the cache. # Request the full block, but then discard the parts we don't need. # (We use allocateDestination() here to support MaskedArray types.) # TODO: We should probably just get rid of MaskedArray support altogether... full_block_data = self.Output.stype.allocateDestination( SubRegion(self.Output, *full_block_roi)) self._execute_Output_impl(full_block_roi, full_block_data) roi_within_block = clipped_block_roi - full_block_roi[0] self.Output.stype.copy_data( result[roiToSlice(*output_roi)], full_block_data[roiToSlice(*roi_within_block)]) clipped_block_rois = getIntersectingRois(self.Input.meta.shape, self._blockshape, (roi.start, roi.stop), True) full_block_rois = getIntersectingRois(self.Input.meta.shape, self._blockshape, (roi.start, roi.stop), False) pool = RequestPool() for full_block_roi, clipped_block_roi in zip(full_block_rois, clipped_block_rois): req = Request( partial(copy_block, full_block_roi, clipped_block_roi)) pool.add(req) pool.wait()
def execute(self, slot, subindex, roi, result): assert slot == self.ConcatenatedOutput self.progressSignal(0.0) num_dirty_slots = len(self._dirty_slots) subtask_progress = {} progress_lock = RequestLock() def forward_progress_updates(feature_slot, progress): with progress_lock: subtask_progress[feature_slot] = progress total_progress = 0.95 * sum( subtask_progress.values()) / num_dirty_slots self.progressSignal(total_progress) logger.debug( "Updating features for {} dirty images out of {}"\ "".format( len(self._dirty_slots), len(self.FeatureMatrices) ) ) pool = RequestPool() subresults = [] for feature_slot, progress_slot in zip(self.FeatureMatrices, self.ProgressSignals): subresults.append([None]) req = feature_slot[:] req.writeInto(subresults[-1]) # Only use progress for slots that were dirty. # The others are going to be really fast. if feature_slot in self._dirty_slots: sub_progress_signal = progress_slot.value sub_progress_signal.subscribe( partial(forward_progress_updates, feature_slot)) pool.add(req) pool.wait() # Reset dirty slots self._dirty_slots = set() # Since the subresults are returned in 'value' slots, # we have to unpack them from their single-element lists. subresult_list = list(itertools.chain(*subresults)) total_matrix = numpy.concatenate(subresult_list, axis=0) self.progressSignal(100.0) result[0] = total_matrix
def _train_forests(forests, X, y): """ Train all RFs (in parallel), and return the oobs. """ oobs = [None] * len(forests) def store_oob_results(i, oob): oobs[i] = oob with Timer() as train_timer: pool = RequestPool() for i, forest in enumerate(forests): req = Request( partial(forest.learnRF, X, y) ) # save the oob results req.notify_finished( partial( store_oob_results, i ) ) pool.add( req ) pool.wait() logger.info("Training took, {} seconds".format( train_timer.seconds() ) ) return oobs
def create_and_train(self, X, y, feature_names=None): # Distribute trees as evenly as possible tree_counts = numpy.array([self._num_trees // self._num_forests] * self._num_forests) tree_counts[:self._num_trees % self._num_forests] += 1 assert tree_counts.sum() == self._num_trees tree_counts = map(int, tree_counts) tree_counts[:] = (tree_count for tree_count in tree_counts if tree_count != 0) logger.debug("Training parallel vigra RF") # Save for future reference known_labels = numpy.unique(y) X = numpy.asarray(X, numpy.float32) y = numpy.asarray(y, numpy.uint32) if y.ndim == 1: y = y[:, numpy.newaxis] assert X.ndim == 2 assert len(X) == len(y) # Create N forests forests = [] for tree_count in tree_counts: forests.append( vigra.learning.RandomForest(tree_count, **self._kwargs)) # Train them all in parallel oobs = [None] * len(forests) pool = RequestPool() for i, forest in enumerate(forests): req = Request(partial(forest.learnRF, X, y)) # save the oobs req.notify_finished(partial(oobs.__setitem__, i)) pool.add(req) with Timer() as timer: pool.wait() logger.info("Training completed in {} seconds. Average OOB: {}".format( timer.seconds(), numpy.average(oobs))) return ParallelVigraRfLazyflowClassifier(forests, oobs, known_labels, feature_names)
def execute(self, slot, subindex, roi, result): assert slot == self.LabelAndFeatureMatrix self.progressSignal(0.0) # Technically, this could result in strange progress reporting if execute() # is called by multiple threads in parallel. # This could be fixed with some fancier progress state, but # (1) We don't expect that to by typical, and # (2) progress reporting is merely informational. num_dirty_blocks = len(self._dirty_blocks) def update_progress(result): remaining_dirty = len(self._dirty_blocks) percent_complete = 95.0 * (num_dirty_blocks - remaining_dirty) / num_dirty_blocks self.progressSignal(percent_complete) # Update all dirty blocks in the cache logger.debug("Updating {} dirty blocks".format(num_dirty_blocks)) pool = RequestPool() for block_start in self._dirty_blocks: req = Request(partial(self._update_block, block_start)) req.notify_finished(update_progress) pool.add(req) pool.wait() # Concatenate the all blockwise results if self._blockwise_feature_matrices: total_feature_matrix = numpy.concatenate( self._blockwise_feature_matrices.values(), axis=0) else: # No label points at all. # Return an empty label&feature matrix (of the correct shape) num_feature_channels = self.FeatureImage.meta.shape[-1] total_feature_matrix = numpy.ndarray(shape=(0, 1 + num_feature_channels), dtype=numpy.float) self.progressSignal(100.0) logger.debug("After update, there are {} clean blocks".format( len(self._blockwise_feature_matrices))) result[0] = total_feature_matrix
def _executePredictionImage(self, slot, roi, destination): roi_one_channel = numpy.array((roi.start, roi.stop)) roi_one_channel[..., -1] = (0, 1) # Determine intersecting blocks block_shape = self._getFullShape(self.BlockShape3dDict.value) block_starts = getIntersectingBlocks(block_shape, roi_one_channel) block_starts = map(tuple, block_starts) # Ensure that block pipelines exist (create first if necessary) for block_start in block_starts: self._ensurePipelineExists(block_start) # Retrieve result from each block, and write into the appropriate region of the destination pool = RequestPool() for block_start in block_starts: opBlockPipeline = self._blockPipelines[block_start] block_roi = opBlockPipeline.block_roi block_intersection = getIntersection(block_roi, roi_one_channel) block_relative_intersection = numpy.subtract( block_intersection, block_roi[0]) destination_relative_intersection = numpy.subtract( block_intersection, roi_one_channel[0]) block_slot = opBlockPipeline.PredictionImage if slot == self.ProbabilityChannelImage: block_slot = opBlockPipeline.ProbabilityChannelImage # Add channels back to roi # request all channels block_relative_intersection[..., -1] = ( 0, opBlockPipeline.ProbabilityChannelImage.meta.shape[-1]) # But only write the ones that were specified in the original roi destination_relative_intersection[..., -1] = (roi.start[-1], roi.stop[-1]) # Request the data destination_slice = roiToSlice(*destination_relative_intersection) req = block_slot(*block_relative_intersection) req.writeInto(destination[destination_slice]) pool.add(req) pool.wait() return destination
def execute(self, slot, subindex, roi, result): with self._lock: if self.cache is None: fullBlockShape = numpy.array( [self.blockShape.value for i in self.Input.meta.shape]) fun = self.inputs["Function"].value #data = self.inputs["Input"][:].wait() #split up requests into blocks shape = self.Input.meta.shape numBlocks = numpy.ceil(shape / (1.0 * fullBlockShape)).astype("int") blockCache = numpy.ndarray(shape=numpy.prod(numBlocks), dtype=self.Output.meta.dtype) pool = RequestPool() #blocks holds the different roi keys for each of the blocks blocks = itertools.product(*[range(i) for i in numBlocks]) blockKeys = [] for b in blocks: start = b * fullBlockShape stop = b * fullBlockShape + fullBlockShape stop = numpy.min(numpy.vstack((stop, shape)), axis=0) blockKey = roiToSlice(start, stop) blockKeys.append(blockKey) def predict_block(i): data = self.Input[blockKeys[i]].wait() blockCache[i] = fun(data) for i, f in enumerate(blockCache): req = pool.request(partial(predict_block, i)) pool.wait() pool.clean() self.cache = [fun(blockCache)] return self.cache
def predict_probabilities(self, X): logger.debug("Predicting with parallel vigra RF") X = numpy.asarray(X, dtype=numpy.float32) # Create a request for each forest reqs = [] for forest in self._forests: req = Request(partial(forest.predictProbabilities, X)) reqs.append(req) # Execute all requests in a pool pool = RequestPool() for req in reqs: pool.add(req) pool.wait() # Aggregate the results predictions = self._forests[0].treeCount() * reqs[0].result for forest, req in zip(self._forests[1:], reqs[1:]): predictions += forest.treeCount() * req.result predictions /= self._num_trees return predictions
def execute(self, slot, subindex, slot_roi, target): assert slot == self.Features or slot == self.Output if slot == self.Features: feature_slice = roiToSlice(slot_roi.start, slot_roi.stop) index = subindex[0] feature_slice = list(feature_slice) # Translate channel slice of this feature to the channel slice of the output slot. output_channel_offset = self.featureOutputChannels[index][0] feature_slice[1] = slice( output_channel_offset + feature_slice[1].start, output_channel_offset + feature_slice[1].stop) slot_roi = SubRegion(self.Output, pslice=feature_slice) # Get output slot region for this channel return self.execute(self.Output, (), slot_roi, target) elif slot == self.Output: # Correlation of variable 'families' representing reference frames: # ______________________________ # | input/output frame | input/output shape given by slots # | _________________________ | # | | smooth frame | | pre-smoothing op needs halo around filter roi # | | ____________________ | | # | | |filter frame | | | filter needs halo around target roi # | | | _______________ | | | # | | | | target frame | | | | target is given by output_roi # note: The 'full_' variable prefix refers to the full 5D shape (tczyx), without 'full_' variables mostly # refer to the 3D space subregion (zyx) full_output_slice = slot_roi.toSlice() logger.debug( f"OpPixelFeaturesPresmoothed: request {slot_roi.pprint()}") assert (slot_roi.stop <= self.Output.meta.shape).all() full_output_shape = self.Output.meta.shape full_output_start, full_output_stop = sliceToRoi( full_output_slice, full_output_shape) assert len(full_output_shape) == 5 if all(self.ComputeIn2d.value ): # todo: check for this particular slice axes2enlarge = (0, 1, 1) else: axes2enlarge = (1, 1, 1) output_shape = full_output_shape[2:] output_start = full_output_start[2:] output_stop = full_output_stop[2:] axistags = self.Output.meta.axistags target = target.view(vigra.VigraArray) target.axistags = copy.copy(axistags) # filter roi in input frame # sigma = 0.7, because the features receive a pre-smoothed array and don't need much of a neighborhood input_filter_start, input_filter_stop = roi.enlargeRoiForHalo( output_start, output_stop, output_shape, 0.7, self.WINDOW_SIZE, enlarge_axes=axes2enlarge) # smooth roi in input frame input_smooth_start, input_smooth_stop = roi.enlargeRoiForHalo( input_filter_start, input_filter_stop, output_shape, self.max_sigma, self.WINDOW_SIZE, enlarge_axes=axes2enlarge, ) # target roi in filter frame filter_target_start = roi.TinyVector(output_start - input_filter_start) filter_target_stop = roi.TinyVector(output_stop - input_filter_start) # filter roi in smooth frame smooth_filter_start = roi.TinyVector(input_filter_start - input_smooth_start) smooth_filter_stop = roi.TinyVector(input_filter_stop - input_smooth_start) filter_target_slice = roi.roiToSlice(filter_target_start, filter_target_stop) input_smooth_slice = roi.roiToSlice(input_smooth_start, input_smooth_stop) # pre-smooth for all requested time slices and all channels full_input_smooth_slice = (full_output_slice[0], slice(None), *input_smooth_slice) req = self.Input[full_input_smooth_slice] source = req.wait() req.clean() req.destination = None if source.dtype != numpy.float32: sourceF = source.astype(numpy.float32) try: source.resize((1, ), refcheck=False) except Exception: pass del source source = sourceF sourceV = source.view(vigra.VigraArray) sourceV.axistags = copy.copy(self.Input.meta.axistags) dimCol = len(self.scales) dimRow = self.matrix.shape[0] presmoothed_source = [None] * dimCol source_smooth_shape = tuple(smooth_filter_stop - smooth_filter_start) full_source_smooth_shape = ( full_output_stop[0] - full_output_start[0], self.Input.meta.shape[1], ) + source_smooth_shape try: for j in range(dimCol): for i in range(dimRow): if self.matrix[i, j]: # There is at least one filter op with this scale break else: # There is no filter op at this scale continue if self.scales[j] > 1.0: tempSigma = math.sqrt(self.scales[j]**2 - 1.0) else: tempSigma = self.scales[j] presmoothed_source[j] = numpy.ndarray( full_source_smooth_shape, numpy.float32) droi = ( (0, *tuple(smooth_filter_start._asint())), (sourceV.shape[1], *tuple(smooth_filter_stop._asint())), ) for i, vsa in enumerate(sourceV.timeIter()): presmoothed_source[j][ i, ...] = self._computeGaussianSmoothing( vsa, tempSigma, droi, in2d=self.ComputeIn2d.value[j]) except RuntimeError as e: if "kernel longer than line" in str(e): raise RuntimeError( "Feature computation error:\nYour image is too small to apply a filter with " f"sigma={self.scales[j]:.1f}. Please select features with smaller sigmas." ) else: raise e del sourceV try: source.resize((1, ), refcheck=False) except ValueError: # Sometimes this fails, but that's okay. logger.debug("Failed to free array memory.") del source cnt = 0 written = 0 closures = [] # connect individual operators for i in range(dimRow): for j in range(dimCol): if self.matrix[i, j]: oslot = self.featureOps[i][j].Output req = None slices = oslot.meta.shape[1] if (cnt + slices >= slot_roi.start[1] and slot_roi.start[1] - cnt < slices and slot_roi.start[1] + written < slot_roi.stop[1]): begin = 0 if cnt < slot_roi.start[1]: begin = slot_roi.start[1] - cnt end = slices if cnt + end > slot_roi.stop[1]: end = slot_roi.stop[1] - cnt # feature slice in output frame feature_slice = (slice(None), slice( written, written + end - begin)) + (slice(None), ) * 3 subtarget = target[feature_slice] # readjust the roi for the new source array full_filter_target_slice = [ full_output_slice[0], slice(begin, end), *filter_target_slice ] filter_target_roi = SubRegion( oslot, pslice=full_filter_target_slice) closure = partial( oslot.operator.execute, oslot, (), filter_target_roi, subtarget, sourceArray=presmoothed_source[j], ) closures.append(closure) written += end - begin cnt += slices pool = RequestPool() for c in closures: pool.request(c) pool.wait() pool.clean() for i in range(len(presmoothed_source)): if presmoothed_source[i] is not None: try: presmoothed_source[i].resize((1, )) except Exception: presmoothed_source[i] = None