def pipeline_info(self, session_id, message_type, message): """Handle the pipeline info message""" logger.info("Handling pipeline info request") pipeline_txt = message.pop(0).bytes pipeline = cellprofiler.pipeline.Pipeline() try: pipeline.loadtxt(StringIO(pipeline_txt)) except Exception as e: logger.warning( "Failed to load pipeline: sending pipeline exception", exc_info=1) self.raise_pipeline_exception(session_id, str(e)) return input_modules, other_modules = self.split_pipeline(pipeline) channels = self.find_channels(input_modules) type_names, measurements = self.find_measurements( other_modules, pipeline) body = json.dumps([channels, type_names, measurements]) msg_out = [ zmq.Frame(session_id), zmq.Frame(), zmq.Frame(PIPELINE_INFO_REPLY_1), zmq.Frame(body), ] self.socket.send_multipart(msg_out)
def clean_pipeline(self, session_id, message_type, message): """Handle the clean pipeline request message""" logger.info("Handling clean pipeline request") pipeline_txt = message.pop(0).bytes module_names = json.loads(message.pop(0).bytes) pipeline = cellprofiler.pipeline.Pipeline() try: pipeline.loadtxt(StringIO(pipeline_txt)) except Exception as e: logger.warning( "Failed to load pipeline: sending pipeline exception", exc_info=1) self.raise_pipeline_exception(session_id, str(e)) return to_remove = [] for module in pipeline.modules(exclude_disabled=False): if module.module_name in module_names: to_remove.insert(0, module) for module in to_remove: pipeline.remove_module(module.module_num) pipeline_fd = StringIO() pipeline.savetxt(pipeline_fd, save_image_plane_details=False) msg_out = [ zmq.Frame(session_id), zmq.Frame(), zmq.Frame(CLEAN_PIPELINE_REPLY_1), zmq.Frame(pipeline_fd.getvalue()), ] self.socket.send_multipart(msg_out)
def get_measurements_for_good_pipeline(nimages=1, group_numbers=None): """Get an appropriately initialized measurements structure for the good pipeline""" path = os.path.join(tests.modules.example_images_directory(), "ExampleSBSImages") m = cellprofiler.measurement.Measurements() if group_numbers is None: group_numbers = [1] * nimages group_indexes = [1] last_group_number = group_numbers[0] group_index = 1 for group_number in group_numbers: if group_number == last_group_number: group_index += 1 else: group_index = 1 group_indexes.append(group_index) for i in range(1, nimages + 1): filename = "Channel2-%02d-%s-%02d.tif" % ( i, "ABCDEFGH"[int((i - 1) / 12)], ((i - 1) % 12) + 1, ) url = cellprofiler.modules.loadimages.pathname2url( os.path.join(path, filename)) m[cellprofiler.measurement.IMAGE, cellprofiler.measurement.C_FILE_NAME + "_DNA", i, ] = filename m[cellprofiler.measurement.IMAGE, cellprofiler.measurement.C_PATH_NAME + "_DNA", i, ] = path m[cellprofiler.measurement.IMAGE, cellprofiler.measurement.C_URL + "_DNA", i] = url m[cellprofiler.measurement.IMAGE, cellprofiler.measurement.GROUP_NUMBER, i] = group_numbers[i - 1] m[cellprofiler.measurement.IMAGE, cellprofiler.measurement.GROUP_INDEX, i] = group_indexes[i - 1] jblob = javabridge.run_script( """ importPackage(Packages.org.cellprofiler.imageset); importPackage(Packages.org.cellprofiler.imageset.filter); var imageFile=new ImageFile(new java.net.URI(url)); var imageFileDetails = new ImageFileDetails(imageFile); var imageSeries=new ImageSeries(imageFile, 0); var imageSeriesDetails = new ImageSeriesDetails(imageSeries, imageFileDetails); var imagePlane=new ImagePlane(imageSeries, 0, ImagePlane.ALWAYS_MONOCHROME); var ipd = new ImagePlaneDetails(imagePlane, imageSeriesDetails); var stack = ImagePlaneDetailsStack.makeMonochromeStack(ipd); var stacks = java.util.Collections.singletonList(stack); var keys = java.util.Collections.singletonList(imageNumber); var imageSet = new ImageSet(stacks, keys); imageSet.compress(java.util.Collections.singletonList("DNA"), null); """, dict(url=url, imageNumber=str(i)), ) blob = javabridge.get_env().get_byte_array_elements(jblob) m[cellprofiler.measurement.IMAGE, cellprofiler.modules.namesandtypes.M_IMAGE_SET, i, blob.dtype, ] = blob pipeline = cellprofiler.pipeline.Pipeline() pipeline.loadtxt(six.moves.StringIO(GOOD_PIPELINE)) pipeline.write_pipeline_measurement(m) return m
def test_load_v9(): with open("./tests/resources/modules/threshold/v9.pipeline", "r") as fd: data = fd.read() fd = io.StringIO(data) pipeline = cellprofiler.pipeline.Pipeline() pipeline.loadtxt(fd) module = pipeline.modules()[4] assert isinstance(module, cellprofiler.modules.threshold.Threshold) assert module.x_name == "DNA" assert module.y_name == "Threshold" assert module.threshold_scope == cellprofiler.modules.threshold.TS_GLOBAL assert module.global_operation.value == cellprofiler.modules.threshold.TM_LI assert module.threshold_smoothing_scale == 0 assert module.threshold_correction_factor == 1.0 assert module.threshold_range.min == 0.0 assert module.threshold_range.max == 1.0 assert module.manual_threshold == 0.0 assert module.thresholding_measurement == "None" assert module.two_class_otsu == cellprofiler.modules.threshold.O_TWO_CLASS assert (module.assign_middle_to_foreground == cellprofiler.modules.threshold.O_FOREGROUND) assert module.adaptive_window_size == 50 module = pipeline.modules()[5] assert module.threshold_scope.value == cellprofiler.modules.threshold.TS_GLOBAL assert module.global_operation.value == cellprofiler.modules.threshold.TM_MANUAL module = pipeline.modules()[6] assert module.threshold_scope.value == cellprofiler.modules.threshold.TS_GLOBAL assert module.global_operation.value == centrosome.threshold.TM_ROBUST_BACKGROUND module = pipeline.modules()[7] assert module.threshold_scope.value == cellprofiler.modules.threshold.TS_GLOBAL assert module.global_operation.value == cellprofiler.modules.threshold.TM_LI
def test_load_v8(): with open("./tests/resources/modules/createbatchfiles/v8.pipeline", "r") as fd: data = fd.read() pipeline = cellprofiler.pipeline.Pipeline() pipeline.loadtxt(six.moves.StringIO(data)) assert len(pipeline.modules()) == 1 module = pipeline.modules()[0] assert module.wants_default_output_directory.value assert module.custom_output_directory.value == "/Users/cellprofiler" assert not module.remote_host_is_windows.value assert module.mappings[0].local_directory.value == "/Users/cellprofiler/Pictures" assert module.mappings[0].remote_directory.value == "/Remote/cellprofiler/Pictures"
def test_load_v4(): with open("./tests/resources/modules/splitormergeobjects/v4.pipeline", "r") as fd: data = fd.read() pipeline = cellprofiler.pipeline.Pipeline() def callback(caller, event): assert not isinstance(event, cellprofiler.pipeline.LoadExceptionEvent) pipeline.add_listener(callback) pipeline.loadtxt(six.moves.StringIO(data)) assert len(pipeline.modules()) == 2 module = pipeline.modules()[0] assert isinstance( module, cellprofiler.modules.splitormergeobjects.SplitOrMergeObjects ) assert module.objects_name == "blobs" assert module.output_objects_name == "RelabeledBlobs" assert ( module.relabel_option == cellprofiler.modules.splitormergeobjects.OPTION_MERGE ) assert module.distance_threshold == 2 assert not module.wants_image assert module.image_name == "Guide" assert module.minimum_intensity_fraction == 0.8 assert ( module.where_algorithm == cellprofiler.modules.splitormergeobjects.CA_CLOSEST_POINT ) assert module.merge_option == cellprofiler.modules.splitormergeobjects.UNIFY_PARENT assert module.parent_object == "Nuclei" assert ( module.merging_method == cellprofiler.modules.splitormergeobjects.UM_CONVEX_HULL ) module = pipeline.modules()[1] assert ( module.relabel_option == cellprofiler.modules.splitormergeobjects.OPTION_SPLIT ) assert module.wants_image assert ( module.where_algorithm == cellprofiler.modules.splitormergeobjects.CA_CENTROIDS ) assert ( module.merge_option == cellprofiler.modules.splitormergeobjects.UNIFY_DISTANCE ) assert ( module.merging_method == cellprofiler.modules.splitormergeobjects.UM_DISCONNECTED )
def get_measurements_for_good_pipeline(nimages=1, group_numbers=None): '''Get an appropriately initialized measurements structure for the good pipeline''' path = os.path.join(tests.modules.example_images_directory(), "ExampleSBSImages") m = cellprofiler.measurement.Measurements() if group_numbers is None: group_numbers = [1] * nimages group_indexes = [1] last_group_number = group_numbers[0] group_index = 1 for group_number in group_numbers: if group_number == last_group_number: group_index += 1 else: group_index = 1 group_indexes.append(group_index) for i in range(1, nimages + 1): filename = ("Channel2-%02d-%s-%02d.tif" % (i, "ABCDEFGH"[int((i - 1) / 12)], ((i - 1) % 12) + 1)) url = cellprofiler.modules.loadimages.pathname2url(os.path.join(path, filename)) m[cellprofiler.measurement.IMAGE, cellprofiler.measurement.C_FILE_NAME + "_DNA", i] = filename m[cellprofiler.measurement.IMAGE, cellprofiler.measurement.C_PATH_NAME + "_DNA", i] = path m[cellprofiler.measurement.IMAGE, cellprofiler.measurement.C_URL + "_DNA", i] = url m[cellprofiler.measurement.IMAGE, cellprofiler.measurement.GROUP_NUMBER, i] = group_numbers[i - 1] m[cellprofiler.measurement.IMAGE, cellprofiler.measurement.GROUP_INDEX, i] = group_indexes[i - 1] jblob = javabridge.run_script(""" importPackage(Packages.org.cellprofiler.imageset); importPackage(Packages.org.cellprofiler.imageset.filter); var imageFile=new ImageFile(new java.net.URI(url)); var imageFileDetails = new ImageFileDetails(imageFile); var imageSeries=new ImageSeries(imageFile, 0); var imageSeriesDetails = new ImageSeriesDetails(imageSeries, imageFileDetails); var imagePlane=new ImagePlane(imageSeries, 0, ImagePlane.ALWAYS_MONOCHROME); var ipd = new ImagePlaneDetails(imagePlane, imageSeriesDetails); var stack = ImagePlaneDetailsStack.makeMonochromeStack(ipd); var stacks = java.util.Collections.singletonList(stack); var keys = java.util.Collections.singletonList(imageNumber); var imageSet = new ImageSet(stacks, keys); imageSet.compress(java.util.Collections.singletonList("DNA"), null); """, dict(url=url, imageNumber=str(i))) blob = javabridge.get_env().get_byte_array_elements(jblob) m[cellprofiler.measurement.IMAGE, cellprofiler.modules.namesandtypes.M_IMAGE_SET, i, blob.dtype] = blob pipeline = cellprofiler.pipeline.Pipeline() pipeline.loadtxt(cStringIO.StringIO(GOOD_PIPELINE)) pipeline.write_pipeline_measurement(m) return m
def test_load_v3(): with open("./tests/resources/modules/identifytertiaryobjects/v3.pipeline", "r") as fd: data = fd.read() pipeline = cellprofiler.pipeline.Pipeline() def callback(caller, event): assert not isinstance(event, cellprofiler.pipeline.LoadExceptionEvent) pipeline.add_listener(callback) pipeline.loadtxt(six.moves.StringIO(data)) module = pipeline.modules()[0] assert module.secondary_objects_name.value == "IdentifySecondaryObjects" assert module.primary_objects_name.value == "IdentifyPrimaryObjects" assert module.subregion_objects_name.value == "IdentifyTertiaryObjects" assert module.shrink_primary.value
def prepare_run(self, message, session_id, grouping_allowed=False): """Prepare a pipeline and measurements to run message - the run-request or run-groups-request message session_id - the session ID for the session grouping_allowed - true to allow grouped images """ pipeline = cellprofiler.pipeline.Pipeline() m = cellprofiler.measurement.Measurements() object_set = cellprofiler.object.ObjectSet() if len(message) < 2: self.raise_cellprofiler_exception(session_id, "Missing run request sections") return pipeline_txt = message.pop(0).bytes image_metadata = message.pop(0).bytes try: image_metadata = json.loads(image_metadata) for channel_name, channel_metadata in image_metadata: if len(message) < 1: self.raise_cellprofiler_exception( session_id, "Missing binary data for channel %s" % channel_name) return None, None, None pixel_data = self.decode_image( channel_metadata, message.pop(0).bytes, grouping_allowed=grouping_allowed, ) m.add(channel_name, cellprofiler.image.Image(pixel_data)) except Exception as e: logger.warn("Failed to decode message", exc_info=1) self.raise_cellprofiler_exception(session_id, e.message) return None, None, None try: pipeline.loadtxt(StringIO(pipeline_txt)) except Exception as e: logger.warning( "Failed to load pipeline: sending pipeline exception", exc_info=1) self.raise_pipeline_exception(session_id, str(e)) return None, None, None return pipeline, m, object_set
def test_load_v7(): with open("./tests/resources/modules/createbatchfiles/v7.pipeline", "r") as fd: data = fd.read() pipeline = cellprofiler.pipeline.Pipeline() pipeline.loadtxt(six.moves.StringIO(data)) assert len(pipeline.modules()) == 1 module = pipeline.modules()[0] assert isinstance(module, cellprofiler.modules.createbatchfiles.CreateBatchFiles) assert module.wants_default_output_directory assert module.custom_output_directory == r"C:\foo\bar" assert not module.remote_host_is_windows assert not module.distributed_mode assert module.default_image_directory == r"C:\bar\baz" assert module.revision == 0 assert not module.from_old_matlab assert len(module.mappings) == 1 mapping = module.mappings[0] assert mapping.local_directory == r"\\argon-cifs\imaging_docs" assert mapping.remote_directory == r"/imaging/docs"
def test_load_v3(): with open("./tests/resources/modules/maskobjects/v3.pipeline", "r") as fd: data = fd.read() pipeline = cellprofiler.pipeline.Pipeline() def callback(caller, event): assert not isinstance(event, cellprofiler.pipeline.LoadExceptionEvent) pipeline.add_listener(callback) pipeline.loadtxt(six.moves.StringIO(data)) module = pipeline.modules()[0] assert module.object_name.value == "IdentifyPrimaryObjects" assert module.remaining_objects.value == "MaskObjects" assert module.mask_choice.value == "Objects" assert module.masking_objects.value == "FilterObjects" assert module.masking_image.value == "None" assert module.overlap_choice.value == "Keep overlapping region" assert module.overlap_fraction.value == 0.5 assert module.retain_or_renumber.value == "Renumber" assert not module.wants_inverted_mask.value
def test_load_v7(): with open("./tests/resources/modules/threshold/v7.pipeline", "r") as fd: data = fd.read() fd = io.StringIO(data) pipeline = cellprofiler.pipeline.Pipeline() pipeline.loadtxt(fd) module = pipeline.modules()[-1] assert isinstance(module, cellprofiler.modules.threshold.Threshold) assert module.x_name.value == "RainbowPony" assert module.y_name.value == "GrayscalePony" assert module.threshold_scope.value == cellprofiler.modules.threshold.TS_GLOBAL assert module.global_operation.value == cellprofiler.modules.threshold.TM_LI assert module.threshold_smoothing_scale.value == 1.3488 assert module.threshold_correction_factor.value == 1.1 assert module.threshold_range.min == 0.07 assert module.threshold_range.max == 0.99 assert module.manual_threshold.value == 0.1 assert module.thresholding_measurement.value == "Pony_Perimeter" assert module.two_class_otsu.value == cellprofiler.modules.threshold.O_TWO_CLASS assert (module.assign_middle_to_foreground.value == cellprofiler.modules.threshold.O_FOREGROUND) assert module.adaptive_window_size.value == 13
def test_load_v5(): with open("./tests/resources/modules/splitormergeobjects/v5.pipeline", "r") as fd: data = fd.read() pipeline = cellprofiler.pipeline.Pipeline() def callback(caller, event): assert not isinstance(event, cellprofiler.pipeline.LoadExceptionEvent) pipeline.add_listener(callback) pipeline.loadtxt(six.moves.StringIO(data)) module = pipeline.modules()[0] assert module.objects_name.value == "IdentifyPrimaryObjects" assert module.output_objects_name.value == "SplitOrMergeObjects" assert module.relabel_option.value == "Merge" assert module.distance_threshold.value == 0 assert not module.wants_image.value assert module.image_name.value == "None" assert module.minimum_intensity_fraction.value == 0.9 assert module.where_algorithm.value == "Closest point" assert module.merge_option.value == "Distance" assert module.parent_object.value == "None" assert module.merging_method.value == "Disconnected"
def test_01_08_load_v8(self): data = r"""CellProfiler Pipeline: http://www.cellprofiler.org Version:3 DateRevision:300 GitHash: ModuleCount:5 HasImagePlaneDetails:False Images:[module_num:1|svn_version:\'Unknown\'|variable_revision_number:2|show_window:False|notes:\x5B\'To begin creating your project, use the Images module to compile a list of files and/or folders that you want to analyze. You can also specify a set of rules to include only the desired files in your selected folders.\'\x5D|batch_state:array(\x5B\x5D, dtype=uint8)|enabled:True|wants_pause:False] : Filter images?:Images only Select the rule criteria:and (extension does isimage) (directory doesnot containregexp "\x5B\\\\\\\\\\\\\\\\/\x5D\\\\\\\\.") Metadata:[module_num:2|svn_version:\'Unknown\'|variable_revision_number:4|show_window:False|notes:\x5B\'The Metadata module optionally allows you to extract information describing your images (i.e, metadata) which will be stored along with your measurements. This information can be contained in the file name and/or location, or in an external file.\'\x5D|batch_state:array(\x5B\x5D, dtype=uint8)|enabled:True|wants_pause:False] Extract metadata?:No Metadata data type:Text Metadata types:{} Extraction method count:1 Metadata extraction method:Extract from file/folder names Metadata source:File name Regular expression:^(?P<Plate>.*)_(?P<Well>\x5BA-P\x5D\x5B0-9\x5D{2})_s(?P<Site>\x5B0-9\x5D)_w(?P<ChannelNumber>\x5B0-9\x5D) Regular expression:(?P<Date>\x5B0-9\x5D{4}_\x5B0-9\x5D{2}_\x5B0-9\x5D{2})$ Extract metadata from:All images Select the filtering criteria:and (file does contain "") Metadata file location: Match file and image metadata:\x5B\x5D Use case insensitive matching?:No NamesAndTypes:[module_num:3|svn_version:\'Unknown\'|variable_revision_number:7|show_window:False|notes:\x5B\'The NamesAndTypes module allows you to assign a meaningful name to each image by which other modules will refer to it.\'\x5D|batch_state:array(\x5B\x5D, dtype=uint8)|enabled:True|wants_pause:False] Assign a name to:All images Select the image type:Grayscale image Name to assign these images:DNA Match metadata:\x5B\x5D Image set matching method:Order Set intensity range from:Image metadata Assignments count:1 Single images count:0 Maximum intensity:255.0 Volumetric:No x:1.0 y:1.0 z:1.0 Select the rule criteria:and (file does contain "") Name to assign these images:DNA Name to assign these objects:Cell Select the image type:Grayscale image Set intensity range from:Image metadata Retain outlines of loaded objects?:No Name the outline image:LoadedOutlines Maximum intensity:255.0 Groups:[module_num:4|svn_version:\'Unknown\'|variable_revision_number:2|show_window:False|notes:\x5B\'The Groups module optionally allows you to split your list of images into image subsets (groups) which will be processed independently of each other. Examples of groupings include screening batches, microtiter plates, time-lapse movies, etc.\'\x5D|batch_state:array(\x5B\x5D, dtype=uint8)|enabled:True|wants_pause:False] Do you want to group your images?:No grouping metadata count:1 Metadata category:None ApplyThreshold:[module_num:5|svn_version:\'Unknown\'|variable_revision_number:8|show_window:True|notes:\x5B\x5D|batch_state:array(\x5B\x5D, dtype=uint8)|enabled:True|wants_pause:False] Select the input image:DNA Name the output image:ThreshBlue Threshold setting version:3 Threshold strategy:Global Thresholding method:MCT Threshold smoothing scale:0 Threshold correction factor:1.0 Lower and upper bounds on threshold:0.0,1.0 Manual threshold:0.0 Select the measurement to threshold with:None Two-class or three-class thresholding?:Two classes Assign pixels in the middle intensity class to the foreground or the background?:Foreground Size of adaptive window:50 Use default parameters?:Default Lower outlier fraction:0.05 Upper outlier fraction:0.05 Averaging method:Mean Variance method:Standard deviation # of deviations:2.0 """ fd = StringIO.StringIO(data) pipeline = cellprofiler.pipeline.Pipeline() pipeline.loadtxt(fd) module = pipeline.modules()[-1] self.assertTrue( isinstance(module, cellprofiler.modules.applythreshold.ApplyThreshold)) self.assertEqual(module.image_name, "DNA") self.assertEqual(module.thresholded_image_name, "ThreshBlue") self.assertEqual(module.threshold_scope, cellprofiler.modules.identify.TS_GLOBAL) self.assertEqual(module.threshold_method, centrosome.threshold.TM_MCT) self.assertEqual(module.threshold_smoothing_scale, 0) self.assertEqual(module.threshold_correction_factor, 1.0) self.assertEqual(module.threshold_range.min, 0.0) self.assertEqual(module.threshold_range.max, 1.0) self.assertEqual(module.manual_threshold, 0.0) self.assertEqual(module.thresholding_measurement, "None") self.assertEqual(module.two_class_otsu, cellprofiler.modules.identify.O_TWO_CLASS) self.assertEqual(module.assign_middle_to_foreground, cellprofiler.modules.identify.O_FOREGROUND) self.assertEqual(module.adaptive_window_size, 50)
def test_01_07_load_v7(self): data = r"""CellProfiler Pipeline: http://www.cellprofiler.org Version:3 DateRevision:20130226215424 ModuleCount:5 HasImagePlaneDetails:False Images:[module_num:1|svn_version:\'Unknown\'|variable_revision_number:1|show_window:False|notes:\x5B\x5D|batch_state:array(\x5B\x5D, dtype=uint8)|enabled:True] : Filter based on rules:No Filter:or (file does contain "") Metadata:[module_num:2|svn_version:\'Unknown\'|variable_revision_number:2|show_window:False|notes:\x5B\x5D|batch_state:array(\x5B\x5D, dtype=uint8)|enabled:True] Extract metadata?:Yes Extraction method count:1 Extraction method:Manual Source:From file name Regular expression:Channel(?P<Wavelength>\x5B12\x5D)-\x5B0-9\x5D{2}-(?P<WellRow>\x5BA-Z\x5D)-(?P<WellColumn>\x5B0-9\x5D{2}).tif Regular expression:(?P<Date>\x5B0-9\x5D{4}_\x5B0-9\x5D{2}_\x5B0-9\x5D{2})$ Filter images:All images :or (file does contain "") Metadata file location\x3A: Match file and image metadata:\x5B\x5D Case insensitive matching:No NamesAndTypes:[module_num:3|svn_version:\'Unknown\'|variable_revision_number:1|show_window:False|notes:\x5B\x5D|batch_state:array(\x5B\x5D, dtype=uint8)|enabled:True] Assignment method:Assign images matching rules Load as:Grayscale image Image name:DNA :\x5B\x5D Assign channels by:Order Assignments count:2 Match this rule:or (metadata does Wavelength "1") Image name:GFP Objects name:Cell Load as:Grayscale image Match this rule:or (metadata does Wavelength "2") Image name:DNA Objects name:Nucleus Load as:Grayscale image Groups:[module_num:4|svn_version:\'Unknown\'|variable_revision_number:2|show_window:False|notes:\x5B\x5D|batch_state:array(\x5B\x5D, dtype=uint8)|enabled:True] Do you want to group your images?:No grouping metadata count:1 Metadata category:None ApplyThreshold:[module_num:5|svn_version:\'Unknown\'|variable_revision_number:7|show_window:True|notes:\x5B\x5D|batch_state:array(\x5B\x5D, dtype=uint8)|enabled:True] Select the input image:RainbowPony Name the output image:GrayscalePony Select the output image type:Grayscale Set pixels below or above the threshold to zero?:Below threshold Subtract the threshold value from the remaining pixel intensities?:Yes Number of pixels by which to expand the thresholding around those excluded bright pixels:2.0 Threshold setting version:1 Threshold strategy:Adaptive Threshold method:MCT Smoothing for threshold:Automatic Threshold smoothing scale:1.5 Threshold correction factor:1.1 Lower and upper bounds on threshold:0.07,0.99 Approximate fraction of image covered by objects?:0.02 Manual threshold:0.1 Select the measurement to threshold with:Pony_Perimeter Select binary image:Pony_yes_or_no Masking objects:PonyMask Two-class or three-class thresholding?:Two classes Minimize the weighted variance or the entropy?:Weighted variance Assign pixels in the middle intensity class to the foreground or the background?:Foreground Method to calculate adaptive window size:Image size Size of adaptive window:13 """ fd = StringIO.StringIO(data) pipeline = cellprofiler.pipeline.Pipeline() pipeline.loadtxt(fd) module = pipeline.modules()[-1] self.assertTrue( isinstance(module, cellprofiler.modules.applythreshold.ApplyThreshold)) self.assertEqual(module.image_name, "RainbowPony") self.assertEqual(module.thresholded_image_name, "GrayscalePony") self.assertEqual(module.threshold_scope, cellprofiler.modules.identify.TS_ADAPTIVE) self.assertEqual(module.threshold_method, centrosome.threshold.TM_MCT) self.assertEqual(module.threshold_smoothing_scale, 1.3488) self.assertEqual(module.threshold_correction_factor, 1.1) self.assertEqual(module.threshold_range.min, .07) self.assertEqual(module.threshold_range.max, .99) self.assertEqual(module.manual_threshold, 0.1) self.assertEqual(module.thresholding_measurement, "Pony_Perimeter") self.assertEqual(module.two_class_otsu, cellprofiler.modules.identify.O_TWO_CLASS) self.assertEqual(module.assign_middle_to_foreground, cellprofiler.modules.identify.O_FOREGROUND) self.assertEqual(module.adaptive_window_size, 13)
def test_03_09_flag_image_abort(self): # # Regression test of issue #1210 # Make a pipeline that aborts during FlagImage # data = r"""CellProfiler Pipeline: http://www.cellprofiler.org Version:3 DateRevision:20140918122611 GitHash:ded6939 ModuleCount:6 HasImagePlaneDetails:False Images:[module_num:1|svn_version:\'Unknown\'|variable_revision_number:2|show_window:False|notes:\x5B\'To begin creating your project, use the Images module to compile a list of files and/or folders that you want to analyze. You can also specify a set of rules to include only the desired files in your selected folders.\'\x5D|batch_state:array(\x5B\x5D, dtype=uint8)|enabled:True|wants_pause:False] : Filter images?:Images only Select the rule criteria:and (extension does isimage) (directory doesnot containregexp "\x5B\\\\\\\\\\\\\\\\/\x5D\\\\\\\\.") Metadata:[module_num:2|svn_version:\'Unknown\'|variable_revision_number:4|show_window:False|notes:\x5B\'The Metadata module optionally allows you to extract information describing your images (i.e, metadata) which will be stored along with your measurements. This information can be contained in the file name and/or location, or in an external file.\'\x5D|batch_state:array(\x5B\x5D, dtype=uint8)|enabled:True|wants_pause:False] Extract metadata?:No Metadata data type:Text Metadata types:{} Extraction method count:1 Metadata extraction method:Extract from file/folder names Metadata source:File name Regular expression:^(?P<Plate>.*)_(?P<Well>\x5BA-P\x5D\x5B0-9\x5D{2})_s(?P<Site>\x5B0-9\x5D)_w(?P<ChannelNumber>\x5B0-9\x5D) Regular expression:(?P<Date>\x5B0-9\x5D{4}_\x5B0-9\x5D{2}_\x5B0-9\x5D{2})$ Extract metadata from:All images Select the filtering criteria:and (file does contain "") Metadata file location: Match file and image metadata:\x5B\x5D Use case insensitive matching?:No NamesAndTypes:[module_num:3|svn_version:\'Unknown\'|variable_revision_number:5|show_window:False|notes:\x5B\'The NamesAndTypes module allows you to assign a meaningful name to each image by which other modules will refer to it.\'\x5D|batch_state:array(\x5B\x5D, dtype=uint8)|enabled:True|wants_pause:False] Assign a name to:All images Select the image type:Grayscale image Name to assign these images:DNA Match metadata:\x5B\x5D Image set matching method:Order Set intensity range from:Image metadata Assignments count:1 Single images count:0 Select the rule criteria:and (file does contain "") Name to assign these images:DNA Name to assign these objects:Cell Select the image type:Grayscale image Set intensity range from:Image metadata Retain outlines of loaded objects?:No Name the outline image:LoadedOutlines Groups:[module_num:4|svn_version:\'Unknown\'|variable_revision_number:2|show_window:False|notes:\x5B\'The Groups module optionally allows you to split your list of images into image subsets (groups) which will be processed independently of each other. Examples of groupings include screening batches, microtiter plates, time-lapse movies, etc.\'\x5D|batch_state:array(\x5B\x5D, dtype=uint8)|enabled:True|wants_pause:False] Do you want to group your images?:No grouping metadata count:1 Metadata category:None FlagImage:[module_num:5|svn_version:\'Unknown\'|variable_revision_number:4|show_window:False|notes:\x5B\x5D|batch_state:array(\x5B\x5D, dtype=uint8)|enabled:True|wants_pause:False] Hidden:1 Hidden:1 Name the flag\'s category:Metadata Name the flag:QCFlag Flag if any, or all, measurement(s) fails to meet the criteria?:Flag if any fail Skip image set if flagged?:Yes Flag is based on:Whole-image measurement Select the object to be used for flagging:None Which measurement?:Height_DNA Flag images based on low values?:No Minimum value:0.0 Flag images based on high values?:Yes Maximum value:1.0 Rules file location:Elsewhere...\x7C Rules file name:rules.txt Class number: MeasureImageIntensity:[module_num:6|svn_version:\'Unknown\'|variable_revision_number:2|show_window:True|notes:\x5B\x5D|batch_state:array(\x5B\x5D, dtype=uint8)|enabled:True|wants_pause:False] Select the image to measure:DNA Measure the intensity only from areas enclosed by objects?:No Select the input objects:None """ self.awthread = self.AWThread(self.announce_addr) self.awthread.start() self.set_work_socket() self.awthread.ex( self.awthread.aw.do_job, cellprofiler.analysis.WorkReply(image_set_numbers=[1], worker_runs_post_group=False, wants_dictionary=True)) # # The worker should ask for the pipeline and preferences next. # req = self.awthread.recv(self.work_socket) self.assertIsInstance(req, cellprofiler.analysis.PipelinePreferencesRequest) self.assertEqual(req.analysis_id, self.analysis_id) input_dir = os.path.join(tests.modules.example_images_directory(), "ExampleSBSImages") cellprofiler.preferences.set_default_image_directory(input_dir) preferences = { cellprofiler.preferences.DEFAULT_IMAGE_DIRECTORY: cellprofiler.preferences.config_read( cellprofiler.preferences.DEFAULT_IMAGE_DIRECTORY) } rep = cellprofiler.analysis.Reply(pipeline_blob=numpy.array(data), preferences=preferences) req.reply(rep) # # The worker asks for the initial measurements. # req = self.awthread.recv(self.work_socket) self.assertIsInstance(req, cellprofiler.analysis.InitialMeasurementsRequest) self.assertEqual(req.analysis_id, self.analysis_id) m = get_measurements_for_good_pipeline() pipeline = cellprofiler.pipeline.Pipeline() pipeline.loadtxt(cStringIO.StringIO(data)) pipeline.write_pipeline_measurement(m) try: req.reply(cellprofiler.analysis.Reply(buf=m.file_contents())) finally: m.close() # # Next, the worker asks for the shared dictionary # req = self.awthread.recv(self.work_socket) self.assertIsInstance(req, cellprofiler.analysis.SharedDictionaryRequest) shared_dictionaries = [{ ("foo%d" % i): "bar%d" % i } for i in range(1, 7)] rep = cellprofiler.analysis.SharedDictionaryReply( dictionaries=shared_dictionaries) req.reply(rep) # # MeasureImageIntensity follows FlagImage and it is poised to ask # for a display. So if we get that, we know the module has been run # and we fail the test. # req = self.awthread.recv(self.work_socket) self.assertFalse(isinstance(req, cellprofiler.analysis.DisplayRequest)) self.assertFalse(isinstance(req, cellprofiler.analysis.ExceptionReport))
def run_group_request(self, session_id, message_type, message): """Handle a run-group request message""" pipeline = cellprofiler.pipeline.Pipeline() m = cellprofiler.measurement.Measurements() image_group = m.hdf5_dict.hdf5_file.create_group("ImageData") if len(message) < 2: self.raise_cellprofiler_exception(session_id, "Missing run request sections") return pipeline_txt = message.pop(0).bytes image_metadata = message.pop(0).bytes n_image_sets = None try: image_metadata = json.loads(image_metadata) channel_names = [] for channel_name, channel_metadata in image_metadata: channel_names.append(channel_name) if len(message) < 1: self.raise_cellprofiler_exception( session_id, "Missing binary data for channel %s" % channel_name) return None, None, None pixel_data = self.decode_image(channel_metadata, message.pop(0).bytes, grouping_allowed=True) if pixel_data.ndim < 3: self.raise_cellprofiler_exception( session_id, "The image for channel %s does not have a Z or T dimension", ) return if n_image_sets is None: n_image_sets = pixel_data.shape[0] elif n_image_sets != pixel_data.shape[0]: self.raise_cellprofiler_exception( session_id, "The images passed have different numbers of Z or T planes", ) return image_group.create_dataset(channel_name, data=pixel_data) except Exception as e: self.raise_cellprofiler_exception(session_id, e.message) return None, None, None try: pipeline.loadtxt(StringIO(pipeline_txt)) except Exception as e: logger.warning( "Failed to load pipeline: sending pipeline exception", exc_info=1) self.raise_pipeline_exception(session_id, str(e)) return image_numbers = numpy.arange(1, n_image_sets + 1) for image_number in image_numbers: m[cellprofiler.measurement.IMAGE, cellprofiler.measurement.GROUP_NUMBER, image_number, ] = 1 m[cellprofiler.measurement.IMAGE, cellprofiler.measurement.GROUP_INDEX, image_number, ] = image_number input_modules, other_modules = self.split_pipeline(pipeline) workspace = cellprofiler.workspace.Workspace(pipeline, None, m, None, m, None) logger.info("Preparing group") for module in other_modules: module.prepare_group( workspace, dict([("image_number", i) for i in image_numbers]), image_numbers, ) for image_index in range(n_image_sets): object_set = cellprofiler.object.ObjectSet() m.next_image_set(image_index + 1) for channel_name in channel_names: dataset = image_group[channel_name] pixel_data = dataset[image_index] m.add(channel_name, cellprofiler.image.Image(pixel_data)) for module in other_modules: workspace = cellprofiler.workspace.Workspace( pipeline, module, m, object_set, m, None) try: logger.info("Running module # %d: %s" % (module.module_num, module.module_name)) pipeline.run_module(module, workspace) if workspace.disposition in ( cellprofiler.workspace.DISPOSITION_SKIP, cellprofiler.workspace.DISPOSITION_CANCEL, ): break except Exception as e: msg = 'Encountered error while running module, "%s": %s' % ( module.module_name, e.message, ) logger.warning(msg, exc_info=1) self.raise_cellprofiler_exception(session_id, msg) return else: continue if workspace.disposition == cellprofiler.workspace.DISPOSITION_CANCEL: break for module in other_modules: module.post_group( workspace, dict([("image_number", i) for i in image_numbers])) logger.info("Finished group") type_names, feature_dict = self.find_measurements( other_modules, pipeline) double_features = [] double_data = [] float_features = [] float_data = [] int_features = [] int_data = [] string_features = [] string_data = [] metadata = [ double_features, float_features, int_features, string_features ] for object_name, features in feature_dict.items(): df = [] double_features.append((object_name, df)) ff = [] float_features.append((object_name, ff)) intf = [] int_features.append((object_name, intf)) sf = [] string_features.append((object_name, sf)) if object_name == cellprofiler.measurement.IMAGE: object_counts = [] * n_image_sets else: object_numbers = m[object_name, cellprofiler.measurement.OBJECT_NUMBER, image_numbers] object_counts = [len(x) for x in object_numbers] for feature, data_type in features: if data_type == "java.lang.String": continue if not m.has_feature(object_name, feature): data = numpy.zeros(numpy.sum(object_counts)) else: data = m[object_name, feature, image_numbers] temp = [] for i, (di, count) in enumerate(zip(data, object_counts)): if count == 0: continue di = numpy.atleast_1d(di) if len(di) > count: di = di[:count] elif len(di) == count: temp.append(di) else: temp += [di + numpy.zeros(len(di) - count)] if len(temp) > 0: data = numpy.hstack(temp) if type_names[data_type] == "java.lang.Double": df.append((feature, len(data))) if len(data) > 0: double_data.append(data.astype("<f8")) elif type_names[data_type] == "java.lang.Float": ff.append((feature, len(data))) if len(data) > 0: float_data.append(data.astype("<f4")) elif type_names[data_type] == "java.lang.Integer": intf.append((feature, len(data))) if len(data) > 0: int_data.append(data.astype("<i4")) data = numpy.hstack([ numpy.frombuffer( numpy.ascontiguousarray(numpy.hstack(ditem)).data, numpy.uint8) for ditem in (double_data, float_data, int_data) if len(ditem) > 0 ]) data = numpy.ascontiguousarray(data) self.socket.send_multipart([ zmq.Frame(session_id), zmq.Frame(), zmq.Frame(RUN_REPLY_1), zmq.Frame(json.dumps(metadata)), zmq.Frame(data), ])
def test_03_09_flag_image_abort(self): # # Regression test of issue #1210 # Make a pipeline that aborts during FlagImage # data = r"""CellProfiler Pipeline: http://www.cellprofiler.org Version:3 DateRevision:20140918122611 GitHash:ded6939 ModuleCount:6 HasImagePlaneDetails:False Images:[module_num:1|svn_version:\'Unknown\'|variable_revision_number:2|show_window:False|notes:\x5B\'To begin creating your project, use the Images module to compile a list of files and/or folders that you want to analyze. You can also specify a set of rules to include only the desired files in your selected folders.\'\x5D|batch_state:array(\x5B\x5D, dtype=uint8)|enabled:True|wants_pause:False] : Filter images?:Images only Select the rule criteria:and (extension does isimage) (directory doesnot containregexp "\x5B\\\\\\\\\\\\\\\\/\x5D\\\\\\\\.") Metadata:[module_num:2|svn_version:\'Unknown\'|variable_revision_number:4|show_window:False|notes:\x5B\'The Metadata module optionally allows you to extract information describing your images (i.e, metadata) which will be stored along with your measurements. This information can be contained in the file name and/or location, or in an external file.\'\x5D|batch_state:array(\x5B\x5D, dtype=uint8)|enabled:True|wants_pause:False] Extract metadata?:No Metadata data type:Text Metadata types:{} Extraction method count:1 Metadata extraction method:Extract from file/folder names Metadata source:File name Regular expression:^(?P<Plate>.*)_(?P<Well>\x5BA-P\x5D\x5B0-9\x5D{2})_s(?P<Site>\x5B0-9\x5D)_w(?P<ChannelNumber>\x5B0-9\x5D) Regular expression:(?P<Date>\x5B0-9\x5D{4}_\x5B0-9\x5D{2}_\x5B0-9\x5D{2})$ Extract metadata from:All images Select the filtering criteria:and (file does contain "") Metadata file location: Match file and image metadata:\x5B\x5D Use case insensitive matching?:No NamesAndTypes:[module_num:3|svn_version:\'Unknown\'|variable_revision_number:5|show_window:False|notes:\x5B\'The NamesAndTypes module allows you to assign a meaningful name to each image by which other modules will refer to it.\'\x5D|batch_state:array(\x5B\x5D, dtype=uint8)|enabled:True|wants_pause:False] Assign a name to:All images Select the image type:Grayscale image Name to assign these images:DNA Match metadata:\x5B\x5D Image set matching method:Order Set intensity range from:Image metadata Assignments count:1 Single images count:0 Select the rule criteria:and (file does contain "") Name to assign these images:DNA Name to assign these objects:Cell Select the image type:Grayscale image Set intensity range from:Image metadata Retain outlines of loaded objects?:No Name the outline image:LoadedOutlines Groups:[module_num:4|svn_version:\'Unknown\'|variable_revision_number:2|show_window:False|notes:\x5B\'The Groups module optionally allows you to split your list of images into image subsets (groups) which will be processed independently of each other. Examples of groupings include screening batches, microtiter plates, time-lapse movies, etc.\'\x5D|batch_state:array(\x5B\x5D, dtype=uint8)|enabled:True|wants_pause:False] Do you want to group your images?:No grouping metadata count:1 Metadata category:None FlagImage:[module_num:5|svn_version:\'Unknown\'|variable_revision_number:4|show_window:False|notes:\x5B\x5D|batch_state:array(\x5B\x5D, dtype=uint8)|enabled:True|wants_pause:False] Hidden:1 Hidden:1 Name the flag\'s category:Metadata Name the flag:QCFlag Flag if any, or all, measurement(s) fails to meet the criteria?:Flag if any fail Skip image set if flagged?:Yes Flag is based on:Whole-image measurement Select the object to be used for flagging:None Which measurement?:Height_DNA Flag images based on low values?:No Minimum value:0.0 Flag images based on high values?:Yes Maximum value:1.0 Rules file location:Elsewhere...\x7C Rules file name:rules.txt Class number: MeasureImageIntensity:[module_num:6|svn_version:\'Unknown\'|variable_revision_number:2|show_window:True|notes:\x5B\x5D|batch_state:array(\x5B\x5D, dtype=uint8)|enabled:True|wants_pause:False] Select the image to measure:DNA Measure the intensity only from areas enclosed by objects?:No Select the input objects:None """ self.awthread = self.AWThread(self.announce_addr) self.awthread.start() self.set_work_socket() self.awthread.ex(self.awthread.aw.do_job, cellprofiler.analysis.WorkReply( image_set_numbers = [1], worker_runs_post_group = False, wants_dictionary = True)) # # The worker should ask for the pipeline and preferences next. # req = self.awthread.recv(self.work_socket) self.assertIsInstance(req, cellprofiler.analysis.PipelinePreferencesRequest) self.assertEqual(req.analysis_id, self.analysis_id) input_dir = os.path.join(tests.modules.example_images_directory(), "ExampleSBSImages") cellprofiler.preferences.set_default_image_directory(input_dir) preferences = {cellprofiler.preferences.DEFAULT_IMAGE_DIRECTORY: cellprofiler.preferences.config_read(cellprofiler.preferences.DEFAULT_IMAGE_DIRECTORY)} rep = cellprofiler.analysis.Reply( pipeline_blob = numpy.array(data), preferences = preferences) req.reply(rep) # # The worker asks for the initial measurements. # req = self.awthread.recv(self.work_socket) self.assertIsInstance(req, cellprofiler.analysis.InitialMeasurementsRequest) self.assertEqual(req.analysis_id, self.analysis_id) m = get_measurements_for_good_pipeline() pipeline = cellprofiler.pipeline.Pipeline() pipeline.loadtxt(cStringIO.StringIO(data)) pipeline.write_pipeline_measurement(m) try: req.reply(cellprofiler.analysis.Reply(buf = m.file_contents())) finally: m.close() # # Next, the worker asks for the shared dictionary # req = self.awthread.recv(self.work_socket) self.assertIsInstance(req, cellprofiler.analysis.SharedDictionaryRequest) shared_dictionaries = [{ ("foo%d" % i):"bar%d" % i} for i in range(1,7)] rep = cellprofiler.analysis.SharedDictionaryReply( dictionaries = shared_dictionaries) req.reply(rep) # # MeasureImageIntensity follows FlagImage and it is poised to ask # for a display. So if we get that, we know the module has been run # and we fail the test. # req = self.awthread.recv(self.work_socket) self.assertFalse(isinstance(req, cellprofiler.analysis.DisplayRequest)) self.assertFalse(isinstance(req, cellprofiler.analysis.ExceptionReport))