def test_basic(self): def someWork(): time.sleep(0.001) #print "producer finished" def callback(s): pass def test(s): req = Request(someWork) req.notify(callback) req.wait() time.sleep(0.001) print s return s req = Request(test, s="hallo !") req.notify(callback) assert req.wait() == "hallo !" requests = [] for i in range(10): req = Request(test, s="hallo %d" % i) requests.append(req) for r in requests: r.wait()
def test_pause_unpause(self): handlerCounter = [0] handlerLock = threading.Lock() def completionHandler( result, req ): handlerLock.acquire() handlerCounter[0] += 1 handlerLock.release() requestCounter = [0] requestLock = threading.Lock() allRequests = [] # This closure randomly chooses to either (a) return immediately or (b) fire off more work def someWork(depth, force=False, i=-1): #print 'depth=', depth, 'i=', i if depth > 0 and (force or random.random() > 0.8): requests = [] for i in range(10): req = Request(someWork, depth=depth-1, i=i) req.notify(completionHandler, req=req) requests.append(req) allRequests.append(req) requestLock.acquire() requestCounter[0] += 1 requestLock.release() for r in requests: r.wait() req = Request(someWork, depth=6, force=True) def blubb(req): pass req.notify(blubb) global_thread_pool.pause() req2 = Request(someWork, depth=6, force=True) req2.notify(blubb) global_thread_pool.unpause() assert req2.finished == False assert req.finished req.wait() # Handler should have been called once for each request we fired assert handlerCounter[0] == requestCounter[0] print "finished pause_unpause" for r in allRequests: assert r.finished print "waited for all subrequests"
def _triggerTableUpdate(self): # Check that object area is included in selected features featureNames = self.topLevelOperatorView.SelectedFeatures.value if 'Standard Object Features' not in featureNames or 'Count' not in featureNames[ 'Standard Object Features']: box = QMessageBox( QMessageBox.Warning, 'Warning', 'Object area is not a selected feature. Please select this feature on: \"Standard Object Features > Shape > Size in pixels\"', QMessageBox.NoButton, self) box.show() return # Clear table self.table.clearContents() self.table.setRowCount(0) self.table.setSortingEnabled(False) self.progressBar.show() self.computeButton.setEnabled(False) # Compute object features and number of labels per frame def compute_features(): features = self.topLevelOperatorView.ObjectFeatures([]).wait() labels = self.topLevelOperatorView.LabelInputs([]).wait() return features, labels req = Request(compute_features) req.notify_finished(self._populateTable) req.submit()
def execute(self, slot, subindex, roi, result): assert len(roi.start) == len(roi.stop) == len(self.Output.meta.shape) assert slot == self.Output t_ind = self.RawVolume.meta.axistags.index('t') assert t_ind < len(self.RawVolume.meta.shape) def compute_features_for_time_slice(res_t_ind, t): axes4d = [ k for k in self.RawVolume.meta.getTaggedShape().keys() if k in 'xyzc' ] # Process entire spatial volume s = [slice(None)] * len(self.RawVolume.meta.shape) s[t_ind] = slice(t, t + 1) s = tuple(s) # Request in parallel raw_req = self.RawVolume[s] raw_req.submit() label_req = self.LabelVolume[s] label_req.submit() if self.Atlas.ready(): atlasVolume = self.Atlas[s].wait() atlasVolume = vigra.taggedView( atlasVolume, axistags=self.Atlas.meta.axistags) atlasVolume = atlasVolume.withAxes(*axes4d) else: atlasVolume = None # Get results rawVolume = raw_req.wait() labelVolume = label_req.wait() rawVolume = vigra.taggedView(rawVolume, axistags=self.RawVolume.meta.axistags) labelVolume = vigra.taggedView( labelVolume, axistags=self.LabelVolume.meta.axistags) # Convert to 4D (preserve axis order) rawVolume = rawVolume.withAxes(*axes4d) labelVolume = labelVolume.withAxes(*axes4d) acc = self._extract(rawVolume, labelVolume, atlasVolume) # Copy into the result result[res_t_ind] = acc # loop over requested time slices pool = RequestPool() for res_t_ind, t in enumerate(range(roi.start[t_ind], roi.stop[t_ind])): pool.add( Request(partial(compute_features_for_time_slice, res_t_ind, t))) pool.wait() return result
def execute(self, slot, subindex, roi, result): def compute_for_channel(output_channel, input_channel): input_roi = numpy.array((roi.start, roi.stop)) input_roi[:, -1] = (input_channel, input_channel + 1) input_req = self.Input(*input_roi) # If possible, use the result array itself as a scratch area if self.Input.meta.dtype == result.dtype: input_req.writeInto(result[..., output_channel:output_channel + 1]) input_data = input_req.wait() input_data = input_data.astype(numpy.float32, order='C', copy=False) input_data = input_data[..., 0] # drop channel axis result[..., output_channel] = computeIntegralImage(input_data) pool = RequestPool() for output_channel, input_channel in enumerate( range(roi.start[-1], roi.stop[-1])): pool.add( Request( partial(compute_for_channel, output_channel, input_channel))) pool.wait()
def _triggerTableUpdate(self): # Check that object area is included in selected features featureNames = self.topLevelOperatorView.SelectedFeatures.value if 'Standard Object Features' not in featureNames or 'Count' not in featureNames[ 'Standard Object Features']: box = QMessageBox( QMessageBox.Warning, 'Warning', 'Object area is not a selected feature. Please select this feature on: \"Standard Object Features > Shape > Size in pixels\"', QMessageBox.NoButton, self) box.show() return # Clear table self.table.clearContents() self.table.setRowCount(0) self.table.setSortingEnabled(False) self.progressBar.show() self.computeButton.setEnabled(False) def compute_features_for_frame(tIndex, t, features): # Compute features and labels (called in parallel from request pool) roi = [ slice(None) for i in range( len(self.topLevelOperatorView.LabelImages.meta.shape)) ] roi[tIndex] = slice(t, t + 1) roi = tuple(roi) frame = self.topLevelOperatorView.SegmentationImages(roi).wait() frame = frame.squeeze().astype(numpy.uint32, copy=False) # Dirty trick: We don't care what we're passing here for the 'image' parameter, # but vigra insists that we pass *something*, so we'll cast the label image as float32. features[t] = vigra.analysis.extractRegionFeatures( frame.view(numpy.float32), frame, ['Count'], ignoreLabel=0) tIndex = self.topLevelOperatorView.SegmentationImages.meta.axistags.index( 't') tMax = self.topLevelOperatorView.SegmentationImages.meta.shape[tIndex] features = {} labels = {} def compute_all_features(): # Compute features in parallel pool = RequestPool() for t in range(tMax): pool.add( Request( partial(compute_features_for_frame, tIndex, t, features))) pool.wait() # Compute labels labels = self.topLevelOperatorView.LabelInputs([]).wait() req = Request(compute_all_features) req.notify_finished(partial(self._populateTable, features, labels)) req.submit()
def test_withH5Py(self): """ We have suspicions that greenlet and h5py don't interact well with eachother. This tests basic functionality. TODO: Expand it for better coverage. """ maxDepth = 5 maxBreadth = 10 filename = 'requestTest.h5' h5File = h5py.File( filename, 'w' ) dataset = h5File.create_dataset( 'test/data', data=numpy.zeros( (maxDepth, maxBreadth), dtype=int )) def writeToH5Py(result, index, req): dataset[index] += 1 # This closure randomly chooses to either (a) return immediately or (b) fire off more work def someWork(depth, force=False, i=0): #print 'depth=', depth, 'i=', i if depth > 0 and (force or random.random() > 0.5): requests = [] for i in range(maxBreadth): req = Request(someWork, depth=depth-1, i=i) req.notify(writeToH5Py, index=(depth-1, i), req=req) requests.append(req) for r in requests: r.wait() req = Request(someWork, depth=maxDepth, force=True) req.wait() h5File.close() print "finished testWithH5Py" os.remove(filename)
def run_export(self): role_names = self.parentApplet.dataSelectionApplet.role_names # Prepare file lists in an OrderedDict role_inputs = { role_name: self._data_role_widgets[role_name].filepaths for role_name in role_names } if all(len(role_inp) == 0 for role_inp in role_inputs.values()): return # Run the export in a separate thread lane_configs = self.parentApplet.dataSelectionApplet.create_lane_configs( role_inputs=role_inputs) export_req = Request( partial(self.parentApplet.run_export, lane_configs=lane_configs)) export_req.notify_failed(self.handle_batch_processing_failure) export_req.notify_finished(self.handle_batch_processing_finished) export_req.notify_cancelled(self.handle_batch_processing_cancelled) self.export_req = export_req self.parentApplet.busy = True self.parentApplet.appletStateUpdateRequested() self.cancel_button.setVisible(True) self.run_button.setEnabled(False) # Start the export export_req.submit()
def create_and_train(self, X, y): logger.debug("Training parallel vigra RF") # Save for future reference known_labels = numpy.unique(y) X = numpy.asarray(X, numpy.float32) y = numpy.asarray(y, numpy.uint32) if y.ndim == 1: y = y[:, numpy.newaxis] assert X.ndim == 2 assert len(X) == len(y) # Create N forests forests = [] for _ in range(self._num_forests): forest = vigra.learning.RandomForest(self._trees_per_forest, **self._kwargs) forests.append(forest) # Train them all in parallel pool = RequestPool() for forest in forests: pool.add(Request(partial(forest.learnRF, X, y))) pool.wait() return ParallelVigraRfLazyflowClassifier(forests, known_labels)
def predict_probabilities(self, X): logger.debug("Predicting with parallel vigra RF") X = numpy.asarray(X, dtype=numpy.float32) assert X.ndim == 2 if self._feature_names is not None: # For some reason, vigra doesn't seem to check this for us... assert X.shape[1] == len(self._feature_names), \ "Feature count doesn't match the training data." # As each forest completes, aggregate results in a shared array. # (Must put in a list so we can update it in this closure.) total_predictions = [None] prediction_lock = RequestLock() def update_predictions(forest, forest_predictions): forest_predictions *= forest.treeCount() with prediction_lock: if total_predictions[0] is None: total_predictions[0] = forest_predictions else: total_predictions[0] += forest_predictions # Create a request for each forest pool = RequestPool() for forest in self._forests: req = Request(partial(forest.predictProbabilities, X)) req.notify_finished(partial(update_predictions, forest)) pool.add(req) del req pool.wait() total_predictions[0] /= self._num_trees return total_predictions[0]
def _label(self, roi, result): result = vigra.taggedView(result, axistags=self.Output.meta.axistags) # get the background values bg = self.Background[...].wait() bg = vigra.taggedView(bg, axistags=self.Background.meta.axistags) bg = bg.withAxes(*"ct") assert np.all( self.Background.meta.shape[0] == self.Input.meta.shape[0] ), "Shape of background values incompatible to shape of Input" assert np.all( self.Background.meta.shape[4] == self.Input.meta.shape[4] ), "Shape of background values incompatible to shape of Input" # do labeling in parallel over channels and time slices pool = RequestPool() start = np.asarray(roi.start, dtype=np.int) stop = np.asarray(roi.stop, dtype=np.int) for ti, t in enumerate(range(roi.start[0], roi.stop[0])): start[0], stop[0] = t, t + 1 for ci, c in enumerate(range(roi.start[4], roi.stop[4])): start[4], stop[4] = c, c + 1 newRoi = SubRegion(self.Output, start=tuple(start), stop=tuple(stop)) resView = result[ti, ..., ci].withAxes(*"xyz") req = Request(partial(self._label3d, newRoi, bg[c, t], resView)) pool.add(req) logger.debug("{}: Computing connected components for ROI {} ...".format(self.name, roi)) pool.wait() pool.clean() logger.debug("{}: Connected components computed.".format(self.name))
def test(s): req = Request(someWork) req.notify(callback) req.wait() time.sleep(0.001) print s return s
def predict_probabilities(self, X): logger.debug( "Predicting with parallel vigra RF" ) X = numpy.asarray(X, dtype=numpy.float32) # As each forest completes, aggregate results in a shared array. # (Must put in a list so we can update it in this closure.) total_predictions = [None] prediction_lock = RequestLock() def update_predictions(forest, forest_predictions): forest_predictions *= forest.treeCount() with prediction_lock: if total_predictions[0] is None: total_predictions[0] = forest_predictions else: total_predictions[0] += forest_predictions # Create a request for each forest pool = RequestPool() for forest in self._forests: req = Request( partial( forest.predictProbabilities, X ) ) req.notify_finished( partial(update_predictions, forest) ) pool.add( req ) del req pool.wait() total_predictions[0] /= self._num_trees return total_predictions[0]
def export(self, filename, hypothesesGraph, objectFeaturesSlot, labelImageSlot, rawImageSlot): """Export the tracking solution stored in the hypotheses graph as a sequence of H5 files, one per frame, containing the label image of that frame and which objects were part of a move or a division. :param filename: string of the FOLDER where to save the result :param hypothesesGraph: hytra.core.hypothesesgraph.HypothesesGraph filled with a solution :param objectFeaturesSlot: lazyflow.graph.InputSlot, connected to the RegionFeaturesAll output of ilastik.applets.trackingFeatureExtraction.opTrackingFeatureExtraction.OpTrackingFeatureExtraction :returns: True on success, False otherwise """ traxelIdPerTimestepToUniqueIdMap, uuidToTraxelMap = hypothesesGraph.getMappingsBetweenUUIDsAndTraxels( ) timesteps = [t for t in traxelIdPerTimestepToUniqueIdMap.keys()] result = hypothesesGraph.getSolutionDictionary() mergers, detections, links, divisions = getMergersDetectionsLinksDivisions( result, uuidToTraxelMap) # group by timestep for event creation mergersPerTimestep = getMergersPerTimestep(mergers, timesteps) linksPerTimestep = getLinksPerTimestep(links, timesteps) detectionsPerTimestep = getDetectionsPerTimestep( detections, timesteps) divisionsPerTimestep = getDivisionsPerTimestep( divisions, linksPerTimestep, timesteps) # save to disk in parallel pool = RequestPool() timeIndex = labelImageSlot.meta.axistags.index('t') for timestep in traxelIdPerTimestepToUniqueIdMap.keys(): # extract current frame lable image roi = [ slice(None) for i in range(len(labelImageSlot.meta.shape)) ] roi[timeIndex] = slice(int(timestep), int(timestep) + 1) roi = tuple(roi) labelImage = labelImageSlot[roi].wait() if not os.path.exists(filename + '/H5-Event-Sequence'): os.makedirs(filename + '/H5-Event-Sequence') fn = os.path.join( filename, "H5-Event-Sequence/{0:05d}.h5".format(int(timestep))) pool.add( Request( partial(writeEvents, int(timestep), linksPerTimestep[timestep], divisionsPerTimestep[timestep], mergersPerTimestep[timestep], detectionsPerTimestep[timestep], fn, labelImage))) pool.wait() return True
def execute_tasks(tasks): """ Executes the given list of tasks (functions) in the lazyflow threadpool. """ pool = RequestPool() for task in tasks: pool.add(Request(task)) pool.wait()
def lots_of_work(): requests = [] for i in range(mcount): req = Request(functools.partial(empty_func, b=11)) req.submit() for r in requests: r.wait()
def submit_to_threadpool(fn, priority): if USE_LAZYFLOW_THREADPOOL: # Tiling requests are less prioritized than most requests. root_priority = [1] + list(priority) req = Request(fn, root_priority) req.submit() else: get_render_pool().submit(fn, priority)
def _resolveMergers(self, hypothesesGraph, model): ''' run merger resolution on the hypotheses graph which contains the current solution ''' logger.info("Resolving mergers.") parameters = self.Parameters.value withTracklets = parameters['withTracklets'] originalGraph = hypothesesGraph.referenceTraxelGraph if withTracklets else hypothesesGraph resolvedMergersDict = {} # Enable full graph computation for animal tracking workflow withFullGraph = False if 'withAnimalTracking' in parameters and parameters['withAnimalTracking']: # TODO: Setting this parameter outside of the track() function (on AnimalConservationTrackingWorkflow) is not desirable withFullGraph = True logger.info("Computing full graph on merger resolver (Only enabled on animal tracking workflow)") mergerResolver = IlastikMergerResolver(originalGraph, pluginPaths=self.pluginPaths, withFullGraph=withFullGraph) # Check if graph contains mergers, otherwise skip merger resolving if not mergerResolver.mergerNum: logger.info("Graph contains no mergers. Skipping merger resolving.") else: # Fit and refine merger nodes using a GMM # It has to be done per time-step in order to aviod loading the whole video on RAM traxelIdPerTimestepToUniqueIdMap, uuidToTraxelMap = getMappingsBetweenUUIDsAndTraxels(model) timesteps = [int(t) for t in traxelIdPerTimestepToUniqueIdMap.keys()] timesteps.sort() timeIndex = self.LabelImage.meta.axistags.index('t') for timestep in timesteps: roi = [slice(None) for i in range(len(self.LabelImage.meta.shape))] roi[timeIndex] = slice(timestep, timestep+1) roi = tuple(roi) labelImage = self.LabelImage[roi].wait() # Get coordinates for object IDs in label image. Used by GMM merger fit. objectIds = vigra.analysis.unique(labelImage[0,...,0]) maxObjectId = max(objectIds) coordinatesForIds = {} pool = RequestPool() for objectId in objectIds: pool.add(Request(partial(mergerResolver.getCoordinatesForObjectId, coordinatesForIds, labelImage[0, ..., 0], timestep, objectId))) # Run requests to get object ID coordinates pool.wait() # Fit mergers and store fit info in nodes if coordinatesForIds: mergerResolver.fitAndRefineNodesForTimestep(coordinatesForIds, maxObjectId, timestep) # Compute object features, re-run flow solver, update model and result, and get merger dictionary resolvedMergersDict = mergerResolver.run() return resolvedMergersDict
def replaceWithStack(self, roleIndex, laneIndex): """ The user clicked the "Import Stack Files" button. """ stackDlg = StackFileSelectionWidget(self) stackDlg.exec_() if stackDlg.result() != QDialog.Accepted: return files = stackDlg.selectedFiles if len(files) == 0: return info = DatasetInfo() info.filePath = "//".join(files) prefix = os.path.commonprefix(files) info.nickname = PathComponents(prefix).filenameBase # Add an underscore for each wildcard digit num_wildcards = len(files[-1]) - len(prefix) - len( os.path.splitext(files[-1])[1]) info.nickname += "_" * num_wildcards # Allow labels by default if this gui isn't being used for batch data. info.allowLabels = (self.guiMode == GuiMode.Normal) info.fromstack = True originalNumLanes = len(self.topLevelOperator.DatasetGroup) if laneIndex is None: laneIndex = self._findFirstEmptyLane(roleIndex) if len(self.topLevelOperator.DatasetGroup) < laneIndex + 1: self.topLevelOperator.DatasetGroup.resize(laneIndex + 1) def importStack(): self.guiControlSignal.emit(ControlCommand.DisableAll) # Serializer will update the operator for us, which will propagate to the GUI. try: self.serializer.importStackAsLocalDataset(info) try: self.topLevelOperator.DatasetGroup[laneIndex][ roleIndex].setValue(info) except DatasetConstraintError as ex: # Give the user a chance to repair the problem. filename = files[0] + "\n...\n" + files[-1] return_val = [False] self.handleDatasetConstraintError(info, filename, ex, roleIndex, laneIndex, return_val) if not return_val[0]: # Not successfully repaired. Roll back the changes and give up. self.topLevelOperator.DatasetGroup.resize( originalNumLanes) finally: self.guiControlSignal.emit(ControlCommand.Pop) req = Request(importStack) req.notify_failed( partial(self.handleFailedStackLoad, files, originalNumLanes)) req.submit()
def read(self, view_roi, result_out): """ roi: (start, stop) tuples, ordered according to description.output_axes roi should be relative to the view """ output_axes = self.description.output_axes roi_transposed = list(zip(*view_roi)) roi_dict = dict(list(zip(output_axes, roi_transposed))) view_roi = list(zip(*(roi_dict["z"], roi_dict["y"], roi_dict["x"]))) # First, normalize roi and result to zyx order result_out = vigra.taggedView(result_out, output_axes) result_out = result_out.withAxes(*"zyx") assert numpy.array(view_roi).shape == (2, 3), "Invalid roi for 3D volume: {}".format(view_roi) view_roi = numpy.array(view_roi) assert (result_out.shape == (view_roi[1] - view_roi[0])).all() # User gave roi according to the view output. # Now offset it find global roi. roi = view_roi + self.description.view_origin_zyx tile_blockshape = (1,) + tuple(self.description.tile_shape_2d_yx) tile_starts = getIntersectingBlocks(tile_blockshape, roi) pool = RequestPool() for tile_start in tile_starts: tile_roi_in = getBlockBounds(self.description.bounds_zyx, tile_blockshape, tile_start) tile_roi_in = numpy.array(tile_roi_in) # This tile's portion of the roi intersecting_roi = getIntersection(roi, tile_roi_in) intersecting_roi = numpy.array(intersecting_roi) # Compute slicing within destination array and slicing within this tile destination_relative_intersection = numpy.subtract(intersecting_roi, roi[0]) tile_relative_intersection = intersecting_roi - tile_roi_in[0] # Get a view to the output slice result_region = result_out[roiToSlice(*destination_relative_intersection)] rest_args = self._get_rest_args(tile_blockshape, tile_roi_in) if self.description.tile_url_format.startswith("http"): retrieval_fn = partial(self._retrieve_remote_tile, rest_args, tile_relative_intersection, result_region) else: retrieval_fn = partial(self._retrieve_local_tile, rest_args, tile_relative_intersection, result_region) PARALLEL_REQ = True if PARALLEL_REQ: pool.add(Request(retrieval_fn)) else: # execute serially (leave the pool empty) retrieval_fn() if PARALLEL_REQ: with Timer() as timer: pool.wait() logger.info("Loading {} tiles took a total of {}".format(len(tile_starts), timer.seconds()))
def compute_all_features(): # Compute features in parallel pool = RequestPool() for t in range(tMax): pool.add( Request( partial(compute_features_for_frame, tIndex, t, features))) pool.wait()
def _exportMeshes(self, object_names: List[str], obj_filepaths: List[str]) -> Request: """Save objects in the mst to .obj files Args: object_names: Names of the objects in the mst obj_filepaths: One path for each object in object_names Returns: Returns the request object, used in testing """ def get_label_volume_from_mst(mst, object_name): object_supervoxels = mst.object_lut[object_name] object_lut = numpy.zeros(mst.nodeNum+1, dtype=numpy.int32) object_lut[object_supervoxels] = 1 supervoxel_volume = mst.supervoxelUint32 object_volume = object_lut[supervoxel_volume] return object_volume mst = self.topLevelOperatorView.MST.value def exportMeshes(object_names, obj_filepaths): n_objects = len(object_names) progress_update = 100 / n_objects try: for obj, obj_path, obj_n in zip(object_names, obj_filepaths, range(n_objects)): object_volume = get_label_volume_from_mst(mst, obj) unique_ids = len(numpy.unique(object_volume)) if unique_ids <= 1: logger.info(f"No voxels found for {obj}, skipping") continue elif unique_ids > 2: logger.info(f"Supervoxel segmentation not unique for {obj}, skipping, got {unique_ids}") continue logger.info(f"Generating mesh for {obj}") _, mesh_data = list(labeling_to_mesh(object_volume, [1]))[0] self.parentApplet.progressSignal((obj_n + .5) * progress_update) logger.info(f"Mesh generation for {obj} complete.") logger.info(f"Saving mesh for {obj} to {obj_path}") mesh_to_obj(mesh_data, obj_path, obj) self.parentApplet.progressSignal((obj_n + 1) * progress_update) finally: self.parentApplet.busy = False self.parentApplet.progressSignal(100) self.parentApplet.appletStateUpdateRequested() self.parentApplet.busy = True self.parentApplet.progressSignal(-1) self.parentApplet.appletStateUpdateRequested() req = Request(partial(exportMeshes, object_names, obj_filepaths)) req.submit() return req
def addStack(self, roleIndex, laneIndex): """ The user clicked the "Import Stack Files" button. """ stackDlg = StackFileSelectionWidget(self) stackDlg.exec_() if stackDlg.result() != QDialog.Accepted: return files = stackDlg.selectedFiles sequence_axis = stackDlg.sequence_axis if len(files) == 0: return cwd = self.topLevelOperator.WorkingDirectory.value info = DatasetInfo(os.path.pathsep.join(files), cwd=cwd) originalNumLanes = len(self.topLevelOperator.DatasetGroup) if laneIndex is None or laneIndex == -1: laneIndex = len(self.topLevelOperator.DatasetGroup) if len(self.topLevelOperator.DatasetGroup) < laneIndex + 1: self.topLevelOperator.DatasetGroup.resize(laneIndex + 1) def importStack(): self.parentApplet.busy = True self.parentApplet.appletStateUpdateRequested() # Serializer will update the operator for us, which will propagate to the GUI. try: self.serializer.importStackAsLocalDataset(info, sequence_axis) try: self.topLevelOperator.DatasetGroup[laneIndex][ roleIndex].setValue(info) except DatasetConstraintError as ex: # Give the user a chance to repair the problem. filename = files[0] + "\n...\n" + files[-1] return_val = [False] self.parentApplet.busy = False # required for possible fixing dialogs from DatasetConstraintError self.parentApplet.appletStateUpdateRequested() self.handleDatasetConstraintError(info, filename, ex, roleIndex, laneIndex, return_val) if not return_val[0]: # Not successfully repaired. Roll back the changes and give up. self.topLevelOperator.DatasetGroup.resize( originalNumLanes) finally: self.parentApplet.busy = False self.parentApplet.appletStateUpdateRequested() req = Request(importStack) req.notify_finished( lambda result: self.showDataset(laneIndex, roleIndex)) req.notify_failed( partial(self.handleFailedStackLoad, files, originalNumLanes)) req.submit()
def testBasic(self): """ Test the SimpleRequestCondition, which is like threading.Condition, but with a subset of the functionality. (See the docs for details.) """ # num_workers = Request.global_thread_pool.num_workers # Request.reset_thread_pool(num_workers=1) N_ELEMENTS = 100 # It's tempting to simply use threading.Condition here, # but that doesn't quite work if the thread calling wait() is also a worker thread. # (threading.Condition uses threading.Lock() as it's 'waiter' lock, which blocks the entire worker.) # cond = threading.Condition( RequestLock() ) cond = SimpleRequestCondition() produced = [] consumed = [] def wait_for_all(): def f(i): time.sleep(0.2 * random.random()) with cond: produced.append(i) cond.notify() reqs = [] for i in range(N_ELEMENTS): req = Request(partial(f, i)) reqs.append(req) for req in reqs: req.submit() _consumed = consumed with cond: while len(_consumed) < N_ELEMENTS: while len(_consumed) == len(produced): cond.wait() logger.debug("copying {} elements".format( len(produced) - len(consumed))) _consumed += produced[len(_consumed):] # Force the request to run in a worker thread. # This should catch failures that can occur if the Condition's "waiter" lock isn't a request lock. req = Request(wait_for_all) req.submit() # Now block for completion req.wait() logger.debug("produced: {}".format(produced)) logger.debug("consumed: {}".format(consumed)) assert set(consumed) == set( range(N_ELEMENTS) ), "Expected set(range(N_ELEMENTS)), got {}".format(consumed)
def exportFinalSegmentation(self, outputPath, axisorder, progressCallback=None): assert self.FinalSegmentation.ready( ), "Can't export yet: The final segmentation isn't ready!" logger.info("Starting Final Segmentation Export...") opTranspose = OpReorderAxes(parent=self) opTranspose.AxisOrder.setValue(axisorder) opTranspose.Input.connect(self.FinalSegmentation) f = h5py.File(outputPath, 'w') opExporter = OpH5WriterBigDataset(parent=self) opExporter.hdf5File.setValue(f) opExporter.hdf5Path.setValue('split_result') opExporter.Image.connect(opTranspose.Output) if progressCallback is not None: opExporter.progressSignal.subscribe(progressCallback) req = Request(partial(self._runExporter, opExporter)) def cleanOps(): opExporter.cleanUp() opTranspose.cleanUp() def handleFailed(exc, exc_info): cleanOps() f.close() import traceback traceback.print_tb(exc_info[2]) msg = "Final Segmentation export FAILED due to the following error:\n{}".format( exc) logger.error(msg) def handleFinished(result): try: cleanOps() logger.info("FINISHED Final Segmentation Export") finally: f.close() def handleCancelled(): cleanOps() f.close() logger.info("Final Segmentation export was cancelled!") req.notify_failed(handleFailed) req.notify_finished(handleFinished) req.notify_cancelled(handleCancelled) req.submit() return req # Returned in case the user wants to cancel it.
def someWork(depth, force=False, i=0): #print 'depth=', depth, 'i=', i if depth > 0 and (force or random.random() > 0.5): requests = [] for i in range(maxBreadth): req = Request(someWork, depth=depth - 1, i=i) req.notify(writeToH5Py, index=(depth - 1, i), req=req) requests.append(req) for r in requests: r.wait()
def _update_rendering(self): """ Override from the base class. """ # This update has to be performed in a different thread to avoid a deadlock # (Because this function is running in the context of a dirty notification!) req = Request( self.__update_rendering ) def handle_rendering_failure( exc, exc_info ): msg = "Exception raised during volume rendering update. See traceack above.\n" log_exception( logger, msg, exc_info ) req.notify_failed( handle_rendering_failure ) req.submit()