def make_workspace(self, ground_truth, test): '''Make a workspace with a ground-truth image and a test image ground_truth and test are dictionaries with the following keys: image - the pixel data mask - (optional) the mask data crop_mask - (optional) a cropping mask returns a workspace and module ''' module = C.CalculateImageOverlap() module.module_num = 1 module.obj_or_img.value = O_IMG module.ground_truth.value = GROUND_TRUTH_IMAGE_NAME module.test_img.value = TEST_IMAGE_NAME module.wants_emd.value = True pipeline = cpp.Pipeline() def callback(caller, event): self.assertFalse(isinstance(event, cpp.RunExceptionEvent)) pipeline.add_listener(callback) pipeline.add_module(module) image_set_list = cpi.ImageSetList() image_set = image_set_list.get_image_set(0) for name, d in ((GROUND_TRUTH_IMAGE_NAME, ground_truth), (TEST_IMAGE_NAME, test)): image = cpi.Image(d["image"], mask=d.get("mask"), crop_mask=d.get("crop_mask")) image_set.add(name, image) workspace = cpw.Workspace(pipeline, module, image_set, cpo.ObjectSet(), cpmeas.Measurements(), image_set_list) return workspace, module
def run_workspace(self, operation, m1_is_image_measurement, m1_data, m2_is_image_measurement, m2_data, setup_fn = None): '''Create and run a workspace, returning the measurements m<n>_is_image_measurement - true for an image measurement, false for object m<n>_data - either a single value or an array setup_fn - this gets called with the module before running ''' module = C.CalculateMath() module.operation.value = operation measurements = cpmeas.Measurements() for i, operand, is_image_measurement, data in\ ((0, module.operands[0], m1_is_image_measurement, m1_data), (1, module.operands[1], m2_is_image_measurement, m2_data)): measurement = "measurement%d"%i if is_image_measurement: operand.operand_choice.value = C.MC_IMAGE measurements.add_image_measurement(measurement, data) else: operand.operand_choice.value = C.MC_OBJECT operand.operand_objects.value = OBJECT[i] measurements.add_measurement(OBJECT[i], measurement, data) operand.operand_measurement.value = measurement module.output_feature_name.value = OUTPUT_MEASUREMENTS pipeline = cpp.Pipeline() image_set_list = cpi.ImageSetList() workspace = cpw.Workspace(pipeline, module, image_set_list.get_image_set(0), cpo.ObjectSet(), measurements, image_set_list) if setup_fn is not None: setup_fn(module, workspace) module.run(workspace) return measurements
def test_03_02_gray_mask(self): image_set_list = cpi.ImageSetList() image_set = image_set_list.get_image_set(0) np.random.seed(0) pixel_data = np.random.uniform(size=(10, 15)).astype(np.float32) image_set.add(IMAGE_NAME, cpi.Image(pixel_data)) masking_image = np.random.uniform(size=(10, 15)) image_set.add(MASKING_IMAGE_NAME, cpi.Image(masking_image)) masking_image = masking_image > 0.5 pipeline = cpp.Pipeline() module = M.MaskImage() module.source_choice.value = M.IO_IMAGE module.object_name.value = OBJECTS_NAME module.image_name.value = IMAGE_NAME module.masking_image_name.value = MASKING_IMAGE_NAME module.masked_image_name.value = MASKED_IMAGE_NAME module.invert_mask.value = False module.module_num = 1 workspace = cpw.Workspace( pipeline, module, image_set, cpo.ObjectSet(), cpmeas.Measurements(), image_set_list, ) module.run(workspace) masked_image = workspace.image_set.get_image(MASKED_IMAGE_NAME) self.assertTrue(isinstance(masked_image, cpi.Image)) self.assertTrue( np.all(masked_image.pixel_data[masking_image] == pixel_data[masking_image])) self.assertTrue(np.all(masked_image.pixel_data[~masking_image] == 0)) self.assertTrue(np.all(masked_image.mask == masking_image)) self.assertFalse(masked_image.has_masking_objects)
def run_imagemath(self, images, modify_module_fn=None, measurement=None): '''Run the ImageMath module, returning the image created images - a list of dictionaries. The dictionary has keys: pixel_data - image pixel data mask - mask for image cropping - cropping mask for image modify_module_fn - a function of the signature, fn(module) that allows the test to modify the module. measurement - an image measurement value ''' image_set_list = cpi.ImageSetList() image_set = image_set_list.get_image_set(0) module = I.ImageMath() module.module_num = 1 for i, image in enumerate(images): pixel_data = image['pixel_data'] mask = image.get('mask', None) cropping = image.get('cropping', None) if i >= 2: module.add_image() name = 'inputimage%s' % i module.images[i].image_name.value = name img = cpi.Image(pixel_data, mask=mask, crop_mask=cropping) image_set.add(name, img) module.output_image_name.value = 'outputimage' if modify_module_fn is not None: modify_module_fn(module) pipeline = cpp.Pipeline() pipeline.add_module(module) measurements = cpmeas.Measurements() if measurement is not None: measurements.add_image_measurement(MEASUREMENT_NAME, str(measurement)) workspace = cpw.Workspace(pipeline, module, image_set, cpo.ObjectSet(), measurements, image_set_list) module.run(workspace) return image_set.get_image('outputimage')
def make_place_workspace(self, images): image_set_list = cpi.ImageSetList() image_set = image_set_list.get_image_set(0) module = T.Tile() module.module_num = 1 module.tile_method.value = T.T_WITHIN_CYCLES module.output_image.value = OUTPUT_IMAGE_NAME module.wants_automatic_rows.value = False module.wants_automatic_columns.value = True module.rows.value = 1 for i, image in enumerate(images): image_name = input_image_name(i) if i == 0: module.input_image.value = image_name else: if len(module.additional_images) <= i: module.add_image() module.additional_images[i - 1].input_image_name.value = image_name image_set.add(image_name, cpi.Image(image)) pipeline = cpp.Pipeline() def callback(caller, event): self.assertFalse(isinstance(event, cpp.RunExceptionEvent)) pipeline.add_listener(callback) pipeline.add_module(module) workspace = cpw.Workspace( pipeline, module, image_set, cpo.ObjectSet(), cpmeas.Measurements(), image_set_list, ) return workspace, module
def make_workspace(self, image_set_count): image_set_list = cpi.ImageSetList() for i in range(image_set_count): image_set = image_set_list.get_image_set(i) module = L.LabelImages() pipeline = cpp.Pipeline() def callback(caller, event): self.assertFalse(isinstance(event, cpp.RunExceptionEvent)) pipeline.add_listener(callback) module.module_num = 1 pipeline.add_module(module) workspace = cpw.Workspace( pipeline, module, image_set_list.get_image_set(0), cpo.ObjectSet(), cpmeas.Measurements(), image_set_list, ) return workspace, module
def make_workspace(self, primary_labels, secondary_labels): """Make a workspace that has objects for the input labels returns a workspace with the following object_set - has object with name "primary" containing the primary labels has object with name "secondary" containing the secondary labels """ isl = cpi.ImageSetList() module = cpmit.IdentifyTertiarySubregion() module.primary_objects_name.value = PRIMARY module.secondary_objects_name.value = SECONDARY module.subregion_objects_name.value = TERTIARY workspace = cpw.Workspace(cpp.Pipeline(), module, isl.get_image_set(0), cpo.ObjectSet(), cpm.Measurements(), isl) for labels, name in ((primary_labels, PRIMARY), (secondary_labels, SECONDARY)): objects = cpo.Objects() objects.segmented = labels workspace.object_set.add_objects(objects, name) return workspace
def prepare_for_run(self): """Prepare to run pipeline. This code was largely copied from 'PipelineController.start_debugging()' with updates to the 'PipelineController' UI removed. I wish I had a better understanding of what exactly this does, but I'm pretty much using it as a black box for the time being. """ self.__measurements = cpm.Measurements(can_overwrite=True) self.__object_set = cpo.ObjectSet(can_overwrite=True) self.__image_set_list = cpi.ImageSetList() workspace = cpw.Workspace(self.__pipeline, None, None, None, self.__measurements, image_set_list, self.__frame) try: if not self.__pipeline.prepare_run(workspace): print 'Error: failed to get image sets' self.__keys, self.__groupings = self.__pipeline.get_groupings( self.__image_set_list) except ValueError, v: message = "Error while preparing for run:\n%s" % (v) wx.MessageBox(message, "Pipeline error", wx.OK | wx.ICON_ERROR, self.__frame)
def test_11_01_add_and_do_nothing(self): # # Regression for issue #1333 - add one, do nothing, input image # is changed # r = np.random.RandomState() r.seed(1101) m = cpmeas.Measurements() pixel_data = r.uniform(size=(20, 20)) m.add("inputimage", cpi.Image(pixel_data)) module = I.ImageMath() module.images[0].image_name.value = "inputimage" module.output_image_name.value = "outputimage" module.operation.value = I.O_NONE module.addend.value = 0.5 module.module_num = 1 pipeline = cpp.Pipeline() pipeline.add_module(module) workspace = cpw.Workspace(pipeline, module, m, None, m, None) module.run(workspace) np.testing.assert_array_almost_equal( pixel_data, m.get_image("inputimage").pixel_data)
def test_02_02_erase_keep(self): module = S.SpeedUpCellProfiler() module.how_to_remove.value = S.C_KEEP module.image_names[0].image_name.value = "Image1" module.module_num = 1 image_set_list = cpi.ImageSetList() image_set = image_set_list.get_image_set(0) image_set.add("Image1", cpi.Image(np.zeros((10, 10)))) image_set.add("Image2", cpi.Image(np.zeros((10, 10)))) pipeline = cpp.Pipeline() def callback(caller, event): self.assertFalse(isinstance(event, cpp.RunExceptionEvent)) pipeline.add_listener(callback) pipeline.add_module(module) workspace = cpw.Workspace(pipeline, module, image_set, cpo.ObjectSet(), cpmeas.Measurements(), image_set_list) module.run(workspace) image = image_set.get_image("Image2") self.assertFalse(isinstance(image, cpi.Image)) image = image_set.get_image("Image1") self.assertTrue(isinstance(image, cpi.Image))
def make_workspace(self, pixel_data, mask=None, objects=None): image_set_list = cpi.ImageSetList() image_set = image_set_list.get_image_set(0) object_set = cpo.ObjectSet() image = cpi.Image(pixel_data) if not mask is None: image.mask = mask image_set.add(MY_IMAGE, image) if not objects is None: o = cpo.Objects() o.segmented = objects object_set.add_objects(o, MY_OBJECTS) module = miq.MeasureImageQuality() module.images_choice.value = miq.O_SELECT module.image_groups[0].include_image_scalings.value = False module.image_groups[0].image_names.value = MY_IMAGE module.image_groups[0].use_all_threshold_methods.value = False module.module_num = 1 pipeline = cpp.Pipeline() pipeline.add_module(module) workspace = cpw.Workspace(pipeline, module, image_set, object_set, cpmeas.Measurements(), image_set_list) return workspace
def test_03_02_zernikes_are_different(self): '''Regression test of IMG-773''' np.random.seed(32) labels = np.zeros((40,20), int) # # Make two "objects" composed of random foreground/background # labels[1:19,1:19] = (np.random.uniform(size=(18,18)) > .5).astype(int) labels[21:39,1:19] = (np.random.uniform(size=(18,18)) > .5).astype(int) * 2 objects = cpo.Objects() objects.segmented = labels object_set = cpo.ObjectSet() object_set.add_objects(objects, "SomeObjects") module = cpmoas.MeasureObjectAreaShape() module.object_groups[0].name.value = "SomeObjects" module.calculate_zernikes.value = True module.module_num = 1 image_set_list = cpi.ImageSetList() measurements = cpmeas.Measurements() pipeline = cpp.Pipeline() pipeline.add_module(module) def callback(caller, event): self.assertFalse(isinstance(event, cpp.RunExceptionEvent)) pipeline.add_listener(callback) workspace = cpw.Workspace(pipeline, module, image_set_list.get_image_set(0), object_set, measurements, image_set_list) module.run(workspace) features = [x[1] for x in module.get_measurement_columns(pipeline) if x[0] == "SomeObjects" and x[1].startswith("AreaShape_Zernike")] for feature in features: values = measurements.get_current_measurement( "SomeObjects", feature) self.assertEqual(len(values), 2) self.assertNotEqual(values[0], values[1])
def make_workspace(self, scheme, images, adjustments): module = G.GrayToColor() module.scheme_choice.value = scheme image_names = [ "image%d" % i if images[i] is not None else G.LEAVE_THIS_BLACK for i in range(7) ] for image_name_setting, image_name, adjustment_setting, adjustment\ in zip((module.red_image_name, module.green_image_name, module.blue_image_name, module.cyan_image_name, module.magenta_image_name, module.yellow_image_name, module.gray_image_name), image_names, (module.red_adjustment_factor, module.green_adjustment_factor, module.blue_adjustment_factor, module.cyan_adjustment_factor, module.magenta_adjustment_factor, module.yellow_adjustment_factor, module.gray_adjustment_factor), adjustments): image_name_setting.value = image_name adjustment_setting.value = adjustment module.rgb_image_name.value = OUTPUT_IMAGE_NAME module.module_num = 1 pipeline = cpp.Pipeline() def callback(caller, event): self.assertFalse(isinstance(event, cpp.RunExceptionEvent)) pipeline.add_listener(callback) pipeline.add_module(module) image_set_list = cpi.ImageSetList() image_set = image_set_list.get_image_set(0) for image, image_name in zip(images, image_names): if image is not None: image_set.add(image_name, cpi.Image(image)) workspace = cpw.Workspace(pipeline, module, image_set, cpo.ObjectSet(), cpmeas.Measurements(), image_set_list) return workspace, module
def test_02_03_double_mask(self): labels = np.zeros((10,15),int) labels[2:5,3:8] = 1 labels[5:8, 10:14] = 2 object_set = cpo.ObjectSet() objects = cpo.Objects() objects.segmented = labels object_set.add_objects(objects, OBJECTS_NAME) image_set_list = cpi.ImageSetList() image_set=image_set_list.get_image_set(0) np.random.seed(0) pixel_data = np.random.uniform(size=(10,15)).astype(np.float32) mask = np.random.uniform(size=(10,15)) > .5 image_set.add(IMAGE_NAME, cpi.Image(pixel_data, mask)) expected_mask = (mask & (labels > 0)) pipeline = cpp.Pipeline() module = M.MaskImage() module.source_choice.value = M.IO_OBJECTS module.object_name.value = OBJECTS_NAME module.image_name.value = IMAGE_NAME module.masked_image_name.value = MASKED_IMAGE_NAME module.invert_mask.value = False module.module_num = 1 workspace = cpw.Workspace(pipeline, module, image_set, object_set, cpmeas.Measurements(), image_set_list) module.run(workspace) masked_image = workspace.image_set.get_image(MASKED_IMAGE_NAME) self.assertTrue(isinstance(masked_image, cpi.Image)) self.assertTrue(np.all(masked_image.pixel_data[expected_mask] == pixel_data[expected_mask])) self.assertTrue(np.all(masked_image.pixel_data[~ expected_mask] == 0)) self.assertTrue(np.all(masked_image.mask == expected_mask)) self.assertTrue(np.all(masked_image.masking_objects.segmented == labels))
def make_workspace(self, image_measurements, object_measurements): '''Make a workspace with a FlagImage module and the given measurements image_measurements - a sequence of single image measurements. Use image_measurement_name(i) to get the name of the i th measurement object_measurements - a seequence of sequences of object measurements. These are stored under object, OBJECT_NAME with measurement name object_measurement_name(i) for the i th measurement. returns module, workspace ''' module = F.FlagImage() measurements = cpmeas.Measurements() for i in range(len(image_measurements)): measurements.add_image_measurement(image_measurement_name(i), image_measurements[i]) for i in range(len(object_measurements)): measurements.add_measurement(OBJECT_NAME, object_measurement_name(i), np.array(object_measurements[i])) flag = module.flags[0] self.assertTrue(isinstance(flag, cps.SettingsGroup)) flag.category.value = MEASUREMENT_CATEGORY flag.feature_name.value = MEASUREMENT_FEATURE module.module_num = 1 pipeline = cpp.Pipeline() def callback(caller,event): self.assertFalse(isinstance(event, cpp.RunExceptionEvent)) pipeline.add_listener(callback) pipeline.add_module(module) image_set_list = cpi.ImageSetList() image_set = image_set_list.get_image_set(0) workspace = cpw.Workspace(pipeline, module, image_set, cpo.ObjectSet(), measurements, image_set_list) return module, workspace
def run_module(self, image, mask=None, fn=None): '''Run the FlipAndRotate module image - pixel data to be transformed mask - optional mask on the pixel data fn - function with signature, "fn(module)" that will be called with the FlipAndRotate module returns an Image object containing the flipped/rotated/masked/cropped image and the angle measurement. ''' img = cpi.Image(image, mask) image_set_list = cpi.ImageSetList() image_set = image_set_list.get_image_set(0) image_set.add(IMAGE_NAME, img) module = F.FlipAndRotate() module.image_name.value = IMAGE_NAME module.output_name.value = OUTPUT_IMAGE module.module_num = 1 if fn is not None: fn(module) pipeline = cpp.Pipeline() pipeline.add_module(module) def error_callback(caller, event): self.assertFalse(isinstance(event, cpp.RunExceptionEvent)) pipeline.add_listener(error_callback) measurements = cpmeas.Measurements() workspace = cpw.Workspace(pipeline, module, image_set, cpo.ObjectSet(), measurements, image_set_list) module.run(workspace) feature = F.M_ROTATION_F % OUTPUT_IMAGE self.assertTrue( feature in measurements.get_feature_names(cpmeas.IMAGE)) angle = measurements.get_current_image_measurement(feature) output_image = image_set.get_image(OUTPUT_IMAGE) return (output_image, angle)
def make_workspace(self, labels, image, mask = None, intensity_image = None, wants_graph = False): image_set_list = cpi.ImageSetList() image_set = image_set_list.get_image_set(0) img = cpi.Image(image, mask) image_set.add(IMAGE_NAME, img) object_set = cpo.ObjectSet() o = cpo.Objects() o.segmented = labels object_set.add_objects(o, OBJECT_NAME) module = M.MeasureNeurons() module.image_name.value = IMAGE_NAME module.seed_objects_name.value = OBJECT_NAME if intensity_image is not None: img = cpi.Image(intensity_image) image_set.add(INTENSITY_IMAGE_NAME, img) module.intensity_image_name.value = INTENSITY_IMAGE_NAME if wants_graph: module.wants_neuron_graph.value = True module.directory.dir_choice = cps.ABSOLUTE_FOLDER_NAME module.directory.custom_path = self.temp_dir module.edge_file_name.value = EDGE_FILE module.vertex_file_name.value = VERTEX_FILE module.module_num = 1 pipeline = cpp.Pipeline() def callback(caller, event): self.assertFalse(isinstance(event, cpp.LoadExceptionEvent)) self.assertFalse(isinstance(event, cpp.RunExceptionEvent)) pipeline.add_listener(callback) pipeline.add_module(module) workspace = cpw.Workspace(pipeline, module, image_set, object_set, cpmeas.Measurements(), image_set_list) return workspace, module
def write_schema(pipeline_filename): if pipeline_filename is None: raise ValueError( "The --write-schema-and-exit switch must be used in conjunction\n" "with the -p or --pipeline switch to load a pipeline with an\n" "ExportToDatabase module.") import cellprofiler.pipeline as cpp import cellprofiler.measurement as cpmeas import cellprofiler.object as cpo import cellprofiler.workspace as cpw pipeline = cpp.Pipeline() pipeline.load(pipeline_filename) pipeline.turn_off_batch_mode() for module in pipeline.modules(): if module.module_name == "ExportToDatabase": break else: raise ValueError( "The pipeline, \"%s\", does not have an ExportToDatabase module" % pipeline_filename) m = cpmeas.Measurements() workspace = cpw.Workspace(pipeline, module, m, cpo.ObjectSet, m, None) module.prepare_run(workspace)
def make_workspace(self, gridding, labels=None): module = I.IdentifyObjectsInGrid() module.module_num = 1 module.grid_name.value = GRID_NAME module.output_objects_name.value = OUTPUT_OBJECTS_NAME module.guiding_object_name.value = GUIDING_OBJECTS_NAME module.outlines_name.value = OUTLINES_NAME image_set_list = cpi.ImageSetList() object_set = cpo.ObjectSet() if labels is not None: my_objects = cpo.Objects() my_objects.segmented = labels object_set.add_objects(my_objects, GUIDING_OBJECTS_NAME) pipeline = cpp.Pipeline() def callback(caller,event): self.assertFalse(isinstance(event, cpp.RunExceptionEvent)) pipeline.add_listener(callback) pipeline.add_module(module) workspace = cpw.Workspace(pipeline, module, image_set_list.get_image_set(0), object_set, cpmeas.Measurements(), image_set_list) workspace.set_grid(GRID_NAME, gridding) return (workspace, module)
def make_tile_workspace(self, images): module = T.Tile() module.module_num = 1 module.tile_method.value = T.T_ACROSS_CYCLES module.input_image.value = INPUT_IMAGE_NAME module.output_image.value = OUTPUT_IMAGE_NAME pipeline = cpp.Pipeline() def callback(caller, event): self.assertFalse(isinstance(event, cpp.RunExceptionEvent)) pipeline.add_listener(callback) pipeline.add_module(module) image_set_list = cpi.ImageSetList() for i, image in enumerate(images): image_set = image_set_list.get_image_set(i) image_set.add(INPUT_IMAGE_NAME, cpi.Image(image)) workspace = cpw.Workspace(pipeline, module, image_set_list.get_image_set(0), cpo.ObjectSet(), cpmeas.Measurements(), image_set_list) return workspace, module
def test_01_02_run(self): module = instantiate_module(MODULE_NAME) module.input_objects_name.value = INPUT_OBJECTS_NAME module.output_objects_name.value = OUTPUT_OBJECTS_NAME module.module_num = 1 pipeline = cpp.Pipeline() pipeline.add_module(module) object_set = cpo.ObjectSet() # # Pick a bunch of random points, dilate them using the distance # transform and then label the result. # r = np.random.RandomState() r.seed(12) bimg = np.ones((100, 100), bool) bimg[r.randint(0,100, 50), r.randint(0, 100, 50)] = False labels, count = label(distance_transform_edt(bimg) <= 5) # # Make the input objects # input_objects = cpo.Objects() input_objects.segmented = labels expected = skeletonize_labels(labels) object_set.add_objects(input_objects, INPUT_OBJECTS_NAME) # # Make the workspace # workspace = cpw.Workspace(pipeline, module, None, object_set, cpmeas.Measurements(), None) module.run(workspace) self.assertTrue(OUTPUT_OBJECTS_NAME in object_set.object_names, "Could not find the output objects in the object set") output_objects = object_set.get_objects(OUTPUT_OBJECTS_NAME) np.testing.assert_array_equal(expected, output_objects.segmented)
def check(self, module, url, dd, keys=None, xml=None): '''Check that running the metadata module on a url generates the expected dictionary''' pipeline = cpp.Pipeline() imgs = I.Images() imgs.filter_choice.value = I.FILTER_CHOICE_NONE imgs.module_num = 1 pipeline.add_module(imgs) module.module_num = 2 pipeline.add_module(module) pipeline.add_urls([url]) m = cpmeas.Measurements() workspace = cpw.Workspace(pipeline, module, None, None, m, None) file_list = workspace.file_list file_list.add_files_to_filelist([url]) if xml is not None: file_list.add_metadata(url, xml) ipds = pipeline.get_image_plane_details(workspace) self.assertEqual(len(ipds), len(dd)) for d, ipd in zip(dd, ipds): self.assertDictContainsSubset(d, ipd.metadata) all_keys = pipeline.get_available_metadata_keys().keys() if keys is not None: for key in keys: self.assertIn(key, all_keys)
def run_tteesstt(self, objects, level): module = instantiate_module(MODULE_NAME) module.objects_name.value = OBJECTS_NAME module.module_num = 1 pipeline = cpp.Pipeline() pipeline.add_module(module) object_set = cpo.ObjectSet() object_set.add_objects(objects, OBJECTS_NAME) # # Make the workspace # measurements = cpmeas.Measurements() workspace = cpw.Workspace(pipeline, module, None, object_set, measurements, None) module.run(workspace) values = measurements.get_measurement(OBJECTS_NAME, "Example5_MeanDistance") self.assertEqual(len(values), objects.count) for labels, indices in objects.get_labels(): for l in np.unique(labels): # # Test very slowly = 1 object per pass # if l == 0: continue d = scipy.ndimage.distance_transform_edt(labels == l) value = np.sum(d) / np.sum(labels == l) if abs(value - values[l-1]) > .0001: if level == 1: self.fail("You got the wrong answer (label = %d, mine = %f, yours = %f" % l, value, values[l-1]) else: sys.stderr.write("Oh too bad, did not pass level %d\n" % level) return if level > 1: print "+%d for you!" % level
def make_workspace(self, pixel_data, mask=None): image = cpi.Image(pixel_data, mask) image_set_list = cpi.ImageSetList() image_set = image_set_list.get_image_set(0) image_set.add(IMAGE_NAME, image) module = ID.IdentifyDeadWorms() module.module_num = 1 module.image_name.value = IMAGE_NAME module.object_name.value = OBJECTS_NAME pipeline = cpp.Pipeline() def callback(caller, event): self.assertFalse(isinstance(event, cpp.LoadExceptionEvent)) self.assertFalse(isinstance(event, cpp.RunExceptionEvent)) pipeline.add_listener(callback) pipeline.add_module(module) workspace = cpw.Workspace(pipeline, module, image_set, cpo.ObjectSet(), cpmeas.Measurements(), image_set_list) return workspace, module
def test_13_04_extra_lines(self): # # Regression test of issue #1211 - extra line at end / blank lines # dir = os.path.join(example_images_directory(), "ExampleSBSImages") file_name = 'Channel2-01-A-01.tif' csv_text = '''"Image_FileName_DNA","Image_PathName_DNA" "%s","%s" ''' % (file_name, dir) pipeline, module, filename = self.make_pipeline(csv_text) try: assert isinstance(module, L.LoadData) m = cpmeas.Measurements() workspace = cpw.Workspace(pipeline, module, m, cpo.ObjectSet(), m, cpi.ImageSetList()) self.assertTrue(module.prepare_run(workspace)) self.assertTrue(isinstance(m, cpmeas.Measurements)) self.assertEqual(m.image_set_count, 1) self.assertTrue('FileName_DNA' in m.get_feature_names(cpmeas.IMAGE)) self.assertEqual(m[cpmeas.IMAGE, 'FileName_DNA', 1], file_name) finally: os.remove(filename)
def make_workspace(self, scheme, images, adjustments=None, colors=None, weights=None): module = G.GrayToColor() module.scheme_choice.value = scheme if scheme not in (G.SCHEME_COMPOSITE, G.SCHEME_STACK): image_names = [ "image%d" % i if images[i] is not None else G.LEAVE_THIS_BLACK for i in range(7) ] for image_name_setting, image_name, adjustment_setting, adjustment \ in zip((module.red_image_name, module.green_image_name, module.blue_image_name, module.cyan_image_name, module.magenta_image_name, module.yellow_image_name, module.gray_image_name), image_names, (module.red_adjustment_factor, module.green_adjustment_factor, module.blue_adjustment_factor, module.cyan_adjustment_factor, module.magenta_adjustment_factor, module.yellow_adjustment_factor, module.gray_adjustment_factor), adjustments): image_name_setting.value = image_name adjustment_setting.value = adjustment else: while len(module.stack_channels) < len(images): module.add_stack_channel_cb() image_names = [] if weights is None: weights = [1.0] * len(images) if colors is None: colors = [ G.DEFAULT_COLORS[i % len(G.DEFAULT_COLORS)] for i in range(len(images)) ] for i, (image, color, weight) in enumerate(zip(images, colors, weights)): image_name = 'image%d' % (i + 1) image_names.append(image_name) module.stack_channels[i].image_name.value = image_name module.stack_channels[i].color.value = color module.stack_channels[i].weight.value = weight module.rgb_image_name.value = OUTPUT_IMAGE_NAME module.module_num = 1 pipeline = cpp.Pipeline() def callback(caller, event): self.assertFalse(isinstance(event, cpp.RunExceptionEvent)) pipeline.add_listener(callback) pipeline.add_module(module) image_set_list = cpi.ImageSetList() image_set = image_set_list.get_image_set(0) for image, image_name in zip(images, image_names): if image is not None: image_set.add(image_name, cpi.Image(image)) workspace = cpw.Workspace(pipeline, module, image_set, cpo.ObjectSet(), cpmeas.Measurements(), image_set_list) return workspace, module
def test_02_01_compare_to_matlab(self): expected = { 'EC50_DistCytoplasm_Correlation_Correlation_CorrGreenCorrBlue': 3.982812, 'EC50_DistCytoplasm_Intensity_LowerQuartileIntensity_CorrGreen': 4.139827, 'EC50_DistCytoplasm_Intensity_MedianIntensity_CorrGreen': 4.178600, 'EC50_DistCytoplasm_Intensity_MinIntensityEdge_CorrGreen': 4.059770, 'EC50_DistCytoplasm_Intensity_MinIntensity_CorrGreen': 4.066357, 'EC50_DistCytoplasm_Math_Ratio1': 4.491367, 'EC50_DistCytoplasm_Math_Ratio2': 3.848722, 'EC50_DistCytoplasm_Texture_AngularSecondMoment_CorrGreen_1': 4.948056, 'EC50_DistCytoplasm_Texture_Entropy_CorrGreen_1': 4.687104, 'EC50_DistCytoplasm_Texture_InfoMeas2_CorrGreen_1': 5.0285, 'EC50_DistCytoplasm_Texture_InverseDifferenceMoment_CorrGreen_1': 4.319017, 'EC50_DistCytoplasm_Texture_SumAverage_CorrGreen_1': 4.548876, 'EC50_DistCytoplasm_Texture_SumEntropy_CorrGreen_1': 4.779139, 'EC50_DistCytoplasm_Texture_Variance_CorrGreen_1': 4.218379, 'EC50_DistanceCells_Correlation_Correlation_CorrGreenCorrBlue': 3.708711, 'EC50_DistanceCells_Intensity_IntegratedIntensityEdge_CorrGreen': 4.135146, 'EC50_DistanceCells_Intensity_LowerQuartileIntensity_CorrGreen': 4.5372, 'EC50_DistanceCells_Intensity_MeanIntensityEdge_CorrGreen': 4.1371, 'EC50_DistanceCells_Intensity_MinIntensityEdge_CorrGreen': 4.033999, 'EC50_DistanceCells_Intensity_MinIntensity_CorrGreen': 4.079470, 'EC50_DistanceCells_Texture_AngularSecondMoment_CorrGreen_1': 5.118689, 'EC50_DistanceCells_Texture_Correlation_CorrGreen_1': 4.002074, 'EC50_DistanceCells_Texture_Entropy_CorrGreen_1': 5.008000, 'EC50_DistanceCells_Texture_InfoMeas1_CorrGreen_1': 3.883586, 'EC50_DistanceCells_Texture_InverseDifferenceMoment_CorrGreen_1': 3.977216, 'EC50_DistanceCells_Texture_SumAverage_CorrGreen_1': 4.9741, 'EC50_DistanceCells_Texture_SumEntropy_CorrGreen_1': 5.1455, 'EC50_DistanceCells_Texture_SumVariance_CorrGreen_1': 4.593041, 'EC50_DistanceCells_Texture_Variance_CorrGreen_1': 4.619517, 'EC50_Nuclei_Correlation_Correlation_CorrGreenCorrBlue': 3.751133, 'EC50_Nuclei_Math_Ratio1': 4.491367, 'EC50_Nuclei_Math_Ratio2': 3.848722, 'EC50_Nuclei_Texture_SumAverage_CorrGreen_1': 3.765297, 'EC50_PropCells_AreaShape_Area': 4.740853, 'EC50_PropCells_AreaShape_MajorAxisLength': 5.064460, 'EC50_PropCells_AreaShape_MinorAxisLength': 4.751471, 'EC50_PropCells_AreaShape_Perimeter': 4.949292, 'EC50_PropCells_Correlation_Correlation_CorrGreenCorrBlue': 3.772565, 'EC50_PropCells_Texture_GaborX_CorrGreen_1': 5.007167, 'EC50_PropCells_Texture_InfoMeas2_CorrBlue_1': 4.341353, 'EC50_PropCells_Texture_SumVariance_CorrBlue_1': 4.298359, 'EC50_PropCells_Texture_SumVariance_CorrGreen_1': 4.610826, 'EC50_PropCells_Texture_Variance_CorrBlue_1': 4.396352, 'EC50_PropCells_Texture_Variance_CorrGreen_1': 4.632468, 'EC50_PropCytoplasm_AreaShape_Area': 4.669679, 'EC50_PropCytoplasm_AreaShape_MinorAxisLength': 4.754476, 'EC50_PropCytoplasm_AreaShape_Perimeter': 4.949292, 'EC50_PropCytoplasm_Correlation_Correlation_CorrGreenCorrBlue': 4.072830, 'EC50_PropCytoplasm_Intensity_IntegratedIntensity_CorrGreen': 4.0934, 'EC50_PropCytoplasm_Intensity_LowerQuartileIntensity_CorrGreen': 3.925800, 'EC50_PropCytoplasm_Intensity_MedianIntensity_CorrGreen': 3.9252, 'EC50_PropCytoplasm_Texture_AngularSecondMoment_CorrGreen_1': 4.777481, 'EC50_PropCytoplasm_Texture_Entropy_CorrGreen_1': 4.4432, 'EC50_PropCytoplasm_Texture_GaborX_CorrGreen_1': 5.163371, 'EC50_PropCytoplasm_Texture_InfoMeas2_CorrGreen_1': 4.701046, 'EC50_PropCytoplasm_Texture_SumEntropy_CorrGreen_1': 4.510543, 'EC50_ThresholdedCells_Texture_AngularSecondMoment_CorrBlue_1': 4.560315, 'EC50_ThresholdedCells_Texture_AngularSecondMoment_CorrGreen_1': 4.966674, 'EC50_ThresholdedCells_Texture_Entropy_CorrBlue_1': 4.457866, 'EC50_ThresholdedCells_Texture_InfoMeas2_CorrBlue_1': 4.624049, 'EC50_ThresholdedCells_Texture_SumAverage_CorrBlue_1': 4.686706, 'EC50_ThresholdedCells_Texture_SumEntropy_CorrBlue_1': 4.537378, 'EC50_ThresholdedCells_Texture_SumVariance_CorrBlue_1': 4.322820, 'EC50_ThresholdedCells_Texture_SumVariance_CorrGreen_1': 4.742158, 'EC50_ThresholdedCells_Texture_Variance_CorrBlue_1': 4.265549, 'EC50_ThresholdedCells_Texture_Variance_CorrGreen_1': 4.860020, 'OneTailedZfactor_DistCytoplasm_Intensity_MedianIntensity_CorrGreen': -4.322503, 'OneTailedZfactor_DistCytoplasm_Intensity_MinIntensityEdge_CorrGreen': -4.322503, 'OneTailedZfactor_DistCytoplasm_Intensity_MinIntensity_CorrGreen': -4.322503, 'OneTailedZfactor_DistCytoplasm_Math_Ratio1': 0.622059, 'OneTailedZfactor_DistCytoplasm_Math_Ratio2': -4.508284, 'OneTailedZfactor_DistCytoplasm_Texture_Entropy_CorrGreen_1': -4.645887, 'OneTailedZfactor_DistCytoplasm_Texture_InfoMeas2_CorrGreen_1': -4.279118, 'OneTailedZfactor_DistCytoplasm_Texture_SumAverage_CorrGreen_1': -4.765570, 'OneTailedZfactor_DistCytoplasm_Texture_SumEntropy_CorrGreen_1': -4.682335, 'OneTailedZfactor_DistCytoplasm_Texture_Variance_CorrGreen_1': -4.415607, 'OneTailedZfactor_DistanceCells_Intensity_MeanIntensityEdge_CorrGreen': -4.200105, 'OneTailedZfactor_DistanceCells_Intensity_MinIntensityEdge_CorrGreen': -4.316452, 'OneTailedZfactor_DistanceCells_Intensity_MinIntensity_CorrGreen': -4.316452, 'OneTailedZfactor_DistanceCells_Texture_Correlation_CorrGreen_1': 0.202500, 'OneTailedZfactor_DistanceCells_Texture_Entropy_CorrGreen_1': -4.404815, 'OneTailedZfactor_DistanceCells_Texture_InfoMeas1_CorrGreen_1': -4.508513, 'OneTailedZfactor_DistanceCells_Texture_SumAverage_CorrGreen_1': -4.225356, 'OneTailedZfactor_DistanceCells_Texture_SumEntropy_CorrGreen_1': -4.382768, 'OneTailedZfactor_DistanceCells_Texture_SumVariance_CorrGreen_1': 0.492125, 'OneTailedZfactor_DistanceCells_Texture_Variance_CorrGreen_1': 0.477360, 'OneTailedZfactor_Nuclei_Correlation_Correlation_CorrGreenCorrBlue': 0.563780, 'OneTailedZfactor_Nuclei_Math_Ratio1': 0.622059, 'OneTailedZfactor_Nuclei_Math_Ratio2': -4.508284, 'OneTailedZfactor_Nuclei_Texture_SumAverage_CorrGreen_1': 0.426178, 'OneTailedZfactor_PropCells_AreaShape_Area': -4.216674, 'OneTailedZfactor_PropCells_AreaShape_MajorAxisLength': -4.119131, 'OneTailedZfactor_PropCells_AreaShape_MinorAxisLength': -4.109793, 'OneTailedZfactor_PropCells_AreaShape_Perimeter': -4.068050, 'OneTailedZfactor_PropCells_Correlation_Correlation_CorrGreenCorrBlue': 0.765440, 'OneTailedZfactor_PropCells_Texture_GaborX_CorrGreen_1': 0.114982, 'OneTailedZfactor_PropCells_Texture_InfoMeas2_CorrBlue_1': 0.108409, 'OneTailedZfactor_PropCells_Texture_SumVariance_CorrBlue_1': 0.191251, 'OneTailedZfactor_PropCells_Texture_SumVariance_CorrGreen_1': 0.559865, 'OneTailedZfactor_PropCells_Texture_Variance_CorrBlue_1': 0.254078, 'OneTailedZfactor_PropCells_Texture_Variance_CorrGreen_1': 0.556108, 'OneTailedZfactor_PropCytoplasm_AreaShape_Area': -4.223021, 'OneTailedZfactor_PropCytoplasm_AreaShape_MinorAxisLength': -4.095632, 'OneTailedZfactor_PropCytoplasm_AreaShape_Perimeter': -4.068050, 'OneTailedZfactor_PropCytoplasm_Intensity_MedianIntensity_CorrGreen': -4.194663, 'OneTailedZfactor_PropCytoplasm_Texture_Entropy_CorrGreen_1': -4.443338, 'OneTailedZfactor_PropCytoplasm_Texture_GaborX_CorrGreen_1': 0.207265, 'OneTailedZfactor_PropCytoplasm_Texture_InfoMeas2_CorrGreen_1': -4.297250, 'OneTailedZfactor_PropCytoplasm_Texture_SumEntropy_CorrGreen_1': -4.525324, 'OneTailedZfactor_ThresholdedCells_Texture_Entropy_CorrBlue_1': 0.167795, 'OneTailedZfactor_ThresholdedCells_Texture_InfoMeas2_CorrBlue_1': 0.067560, 'OneTailedZfactor_ThresholdedCells_Texture_SumAverage_CorrBlue_1': 0.478527, 'OneTailedZfactor_ThresholdedCells_Texture_SumEntropy_CorrBlue_1': 0.155119, 'OneTailedZfactor_ThresholdedCells_Texture_SumVariance_CorrBlue_1': 0.535907, 'OneTailedZfactor_ThresholdedCells_Texture_SumVariance_CorrGreen_1': 0.572801, 'OneTailedZfactor_ThresholdedCells_Texture_Variance_CorrBlue_1': 0.423454, 'OneTailedZfactor_ThresholdedCells_Texture_Variance_CorrGreen_1': 0.440500, 'Vfactor_DistCytoplasm_Correlation_Correlation_CorrGreenCorrBlue': 0.500429, 'Vfactor_DistCytoplasm_Intensity_LowerQuartileIntensity_CorrGreen': 0.325675, 'Vfactor_DistCytoplasm_Intensity_MedianIntensity_CorrGreen': 0.323524, 'Vfactor_DistCytoplasm_Intensity_MinIntensityEdge_CorrGreen': 0.138487, 'Vfactor_DistCytoplasm_Intensity_MinIntensity_CorrGreen': 0.128157, 'Vfactor_DistCytoplasm_Math_Ratio1': 0.503610, 'Vfactor_DistCytoplasm_Math_Ratio2': 0.319610, 'Vfactor_DistCytoplasm_Texture_AngularSecondMoment_CorrGreen_1': 0.522880, 'Vfactor_DistCytoplasm_Texture_Entropy_CorrGreen_1': 0.504303, 'Vfactor_DistCytoplasm_Texture_InfoMeas2_CorrGreen_1': 0.289432, 'Vfactor_DistCytoplasm_Texture_InverseDifferenceMoment_CorrGreen_1': 0.234123, 'Vfactor_DistCytoplasm_Texture_SumAverage_CorrGreen_1': 0.591687, 'Vfactor_DistCytoplasm_Texture_SumEntropy_CorrGreen_1': 0.520356, 'Vfactor_DistCytoplasm_Texture_Variance_CorrGreen_1': -0.007649, 'Vfactor_DistanceCells_Correlation_Correlation_CorrGreenCorrBlue': 0.761198, 'Vfactor_DistanceCells_Intensity_IntegratedIntensityEdge_CorrGreen': 0.234655, 'Vfactor_DistanceCells_Intensity_LowerQuartileIntensity_CorrGreen': 0.252240, 'Vfactor_DistanceCells_Intensity_MeanIntensityEdge_CorrGreen': 0.195125, 'Vfactor_DistanceCells_Intensity_MinIntensityEdge_CorrGreen': 0.138299, 'Vfactor_DistanceCells_Intensity_MinIntensity_CorrGreen': 0.126784, 'Vfactor_DistanceCells_Texture_AngularSecondMoment_CorrGreen_1': 0.342691, 'Vfactor_DistanceCells_Texture_Correlation_CorrGreen_1': 0.314396, 'Vfactor_DistanceCells_Texture_Entropy_CorrGreen_1': 0.311771, 'Vfactor_DistanceCells_Texture_InfoMeas1_CorrGreen_1': 0.410631, 'Vfactor_DistanceCells_Texture_InverseDifferenceMoment_CorrGreen_1': 0.170576, 'Vfactor_DistanceCells_Texture_SumAverage_CorrGreen_1': 0.223147, 'Vfactor_DistanceCells_Texture_SumEntropy_CorrGreen_1': 0.269519, 'Vfactor_DistanceCells_Texture_SumVariance_CorrGreen_1': 0.571528, 'Vfactor_DistanceCells_Texture_Variance_CorrGreen_1': 0.566272, 'Vfactor_Nuclei_Correlation_Correlation_CorrGreenCorrBlue': 0.705051, 'Vfactor_Nuclei_Math_Ratio1': 0.503610, 'Vfactor_Nuclei_Math_Ratio2': 0.319610, 'Vfactor_Nuclei_Texture_SumAverage_CorrGreen_1': 0.553708, 'Vfactor_PropCells_AreaShape_Area': 0.340093, 'Vfactor_PropCells_AreaShape_MajorAxisLength': 0.243838, 'Vfactor_PropCells_AreaShape_MinorAxisLength': 0.320691, 'Vfactor_PropCells_AreaShape_Perimeter': 0.238915, 'Vfactor_PropCells_Correlation_Correlation_CorrGreenCorrBlue': 0.723520, 'Vfactor_PropCells_Texture_GaborX_CorrGreen_1': 0.213161, 'Vfactor_PropCells_Texture_InfoMeas2_CorrBlue_1': 0.199791, 'Vfactor_PropCells_Texture_SumVariance_CorrBlue_1': 0.078959, 'Vfactor_PropCells_Texture_SumVariance_CorrGreen_1': 0.642844, 'Vfactor_PropCells_Texture_Variance_CorrBlue_1': 0.199105, 'Vfactor_PropCells_Texture_Variance_CorrGreen_1': 0.640818, 'Vfactor_PropCytoplasm_AreaShape_Area': 0.325845, 'Vfactor_PropCytoplasm_AreaShape_MinorAxisLength': 0.312258, 'Vfactor_PropCytoplasm_AreaShape_Perimeter': 0.238915, 'Vfactor_PropCytoplasm_Correlation_Correlation_CorrGreenCorrBlue': 0.337565, 'Vfactor_PropCytoplasm_Intensity_IntegratedIntensity_CorrGreen': 0.292900, 'Vfactor_PropCytoplasm_Intensity_LowerQuartileIntensity_CorrGreen': 0.175528, 'Vfactor_PropCytoplasm_Intensity_MedianIntensity_CorrGreen': 0.193308, 'Vfactor_PropCytoplasm_Texture_AngularSecondMoment_CorrGreen_1': 0.276152, 'Vfactor_PropCytoplasm_Texture_Entropy_CorrGreen_1': 0.239567, 'Vfactor_PropCytoplasm_Texture_GaborX_CorrGreen_1': 0.332380, 'Vfactor_PropCytoplasm_Texture_InfoMeas2_CorrGreen_1': 0.379141, 'Vfactor_PropCytoplasm_Texture_SumEntropy_CorrGreen_1': 0.337740, 'Vfactor_ThresholdedCells_Texture_AngularSecondMoment_CorrBlue_1': 0.334520, 'Vfactor_ThresholdedCells_Texture_AngularSecondMoment_CorrGreen_1': 0.192882, 'Vfactor_ThresholdedCells_Texture_Entropy_CorrBlue_1': 0.276245, 'Vfactor_ThresholdedCells_Texture_InfoMeas2_CorrBlue_1': 0.139166, 'Vfactor_ThresholdedCells_Texture_SumAverage_CorrBlue_1': 0.465237, 'Vfactor_ThresholdedCells_Texture_SumEntropy_CorrBlue_1': 0.355399, 'Vfactor_ThresholdedCells_Texture_SumVariance_CorrBlue_1': 0.453937, 'Vfactor_ThresholdedCells_Texture_SumVariance_CorrGreen_1': 0.564371, 'Vfactor_ThresholdedCells_Texture_Variance_CorrBlue_1': 0.360566, 'Vfactor_ThresholdedCells_Texture_Variance_CorrGreen_1': 0.548770, 'Zfactor_DistCytoplasm_Correlation_Correlation_CorrGreenCorrBlue': 0.531914, 'Zfactor_DistCytoplasm_Intensity_LowerQuartileIntensity_CorrGreen': 0.265558, 'Zfactor_DistCytoplasm_Intensity_MedianIntensity_CorrGreen': 0.178586, 'Zfactor_DistCytoplasm_Intensity_MinIntensityEdge_CorrGreen': 0.084566, 'Zfactor_DistCytoplasm_Intensity_MinIntensity_CorrGreen': 0.086476, 'Zfactor_DistCytoplasm_Math_Ratio1': 0.623284, 'Zfactor_DistCytoplasm_Math_Ratio2': 0.358916, 'Zfactor_DistCytoplasm_Texture_AngularSecondMoment_CorrGreen_1': 0.429510, 'Zfactor_DistCytoplasm_Texture_Entropy_CorrGreen_1': 0.508275, 'Zfactor_DistCytoplasm_Texture_InfoMeas2_CorrGreen_1': 0.068695, 'Zfactor_DistCytoplasm_Texture_InverseDifferenceMoment_CorrGreen_1': 0.347949, 'Zfactor_DistCytoplasm_Texture_SumAverage_CorrGreen_1': 0.646576, 'Zfactor_DistCytoplasm_Texture_SumEntropy_CorrGreen_1': 0.494276, 'Zfactor_DistCytoplasm_Texture_Variance_CorrGreen_1': 0.179011, 'Zfactor_DistanceCells_Correlation_Correlation_CorrGreenCorrBlue': 0.824686, 'Zfactor_DistanceCells_Intensity_IntegratedIntensityEdge_CorrGreen': 0.027644, 'Zfactor_DistanceCells_Intensity_LowerQuartileIntensity_CorrGreen': 0.088491, 'Zfactor_DistanceCells_Intensity_MeanIntensityEdge_CorrGreen': 0.065056, 'Zfactor_DistanceCells_Intensity_MinIntensityEdge_CorrGreen': 0.089658, 'Zfactor_DistanceCells_Intensity_MinIntensity_CorrGreen': 0.078017, 'Zfactor_DistanceCells_Texture_AngularSecondMoment_CorrGreen_1': 0.238131, 'Zfactor_DistanceCells_Texture_Correlation_CorrGreen_1': 0.301107, 'Zfactor_DistanceCells_Texture_Entropy_CorrGreen_1': 0.251143, 'Zfactor_DistanceCells_Texture_InfoMeas1_CorrGreen_1': 0.564957, 'Zfactor_DistanceCells_Texture_InverseDifferenceMoment_CorrGreen_1': 0.302767, 'Zfactor_DistanceCells_Texture_SumAverage_CorrGreen_1': 0.036459, 'Zfactor_DistanceCells_Texture_SumEntropy_CorrGreen_1': 0.159798, 'Zfactor_DistanceCells_Texture_SumVariance_CorrGreen_1': 0.516938, 'Zfactor_DistanceCells_Texture_Variance_CorrGreen_1': 0.501186, 'Zfactor_Nuclei_Correlation_Correlation_CorrGreenCorrBlue': 0.691408, 'Zfactor_Nuclei_Math_Ratio1': 0.623284, 'Zfactor_Nuclei_Math_Ratio2': 0.358916, 'Zfactor_Nuclei_Texture_SumAverage_CorrGreen_1': 0.587347, 'Zfactor_PropCells_AreaShape_Area': 0.132425, 'Zfactor_PropCells_AreaShape_MajorAxisLength': 0.034809, 'Zfactor_PropCells_AreaShape_MinorAxisLength': 0.113864, 'Zfactor_PropCells_AreaShape_Perimeter': 0.005984, 'Zfactor_PropCells_Correlation_Correlation_CorrGreenCorrBlue': 0.717632, 'Zfactor_PropCells_Texture_GaborX_CorrGreen_1': 0.251023, 'Zfactor_PropCells_Texture_InfoMeas2_CorrBlue_1': 0.149719, 'Zfactor_PropCells_Texture_SumVariance_CorrBlue_1': 0.102050, 'Zfactor_PropCells_Texture_SumVariance_CorrGreen_1': 0.611960, 'Zfactor_PropCells_Texture_Variance_CorrBlue_1': 0.197090, 'Zfactor_PropCells_Texture_Variance_CorrGreen_1': 0.614879, 'Zfactor_PropCytoplasm_AreaShape_Area': 0.205042, 'Zfactor_PropCytoplasm_AreaShape_MinorAxisLength': 0.072682, 'Zfactor_PropCytoplasm_AreaShape_Perimeter': 0.005984, 'Zfactor_PropCytoplasm_Correlation_Correlation_CorrGreenCorrBlue': 0.272017, 'Zfactor_PropCytoplasm_Intensity_IntegratedIntensity_CorrGreen': 0.115327, 'Zfactor_PropCytoplasm_Intensity_LowerQuartileIntensity_CorrGreen': 0.141850, 'Zfactor_PropCytoplasm_Intensity_MedianIntensity_CorrGreen': 0.105803, 'Zfactor_PropCytoplasm_Texture_AngularSecondMoment_CorrGreen_1': 0.107640, 'Zfactor_PropCytoplasm_Texture_Entropy_CorrGreen_1': 0.067896, 'Zfactor_PropCytoplasm_Texture_GaborX_CorrGreen_1': 0.136688, 'Zfactor_PropCytoplasm_Texture_InfoMeas2_CorrGreen_1': 0.334749, 'Zfactor_PropCytoplasm_Texture_SumEntropy_CorrGreen_1': 0.208829, 'Zfactor_ThresholdedCells_Texture_AngularSecondMoment_CorrBlue_1': 0.263467, 'Zfactor_ThresholdedCells_Texture_AngularSecondMoment_CorrGreen_1': 0.124355, 'Zfactor_ThresholdedCells_Texture_Entropy_CorrBlue_1': 0.236433, 'Zfactor_ThresholdedCells_Texture_InfoMeas2_CorrBlue_1': 0.125845, 'Zfactor_ThresholdedCells_Texture_SumAverage_CorrBlue_1': 0.449333, 'Zfactor_ThresholdedCells_Texture_SumEntropy_CorrBlue_1': 0.323243, 'Zfactor_ThresholdedCells_Texture_SumVariance_CorrBlue_1': 0.507477, 'Zfactor_ThresholdedCells_Texture_SumVariance_CorrGreen_1': 0.599000, 'Zfactor_ThresholdedCells_Texture_Variance_CorrBlue_1': 0.361424, 'Zfactor_ThresholdedCells_Texture_Variance_CorrGreen_1': 0.481393 } temp_dir = tempfile.mkdtemp() try: cpprefs.set_headless() cpprefs.set_default_output_directory(temp_dir) print "Writing output to %s" % temp_dir path = os.path.split(__file__)[0] measurements = loadmat(os.path.join(path, 'calculatestatistics.mat'), struct_as_record=True) measurements = measurements['m'] image_set_list = cpi.ImageSetList() image_set = image_set_list.get_image_set(0) m = cpmeas.Measurements() doses = [ 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 10, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 10, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 10, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 10, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0 ] for i, dose in enumerate(doses): m.add_image_measurement("Dose", dose) for object_name in measurements.dtype.fields: omeasurements = measurements[object_name][0, 0] for feature_name in omeasurements.dtype.fields: data = omeasurements[feature_name][0, 0][0, i] m.add_measurement(object_name, feature_name, data) if i < len(doses) - 1: m.next_image_set() pipeline = cpp.Pipeline() module = C.CalculateStatistics() module.grouping_values.value = "Dose" module.dose_values[0].log_transform.value = False module.dose_values[0].measurement.value = "Dose" module.dose_values[0].wants_save_figure.value = True module.dose_values[0].figure_name.value = "EC49_" module.module_num = 1 pipeline.add_module(module) def callback(caller, event): self.assertFalse(isinstance(event, cpp.RunExceptionEvent)) workspace = cpw.Workspace(pipeline, module, image_set, cpo.ObjectSet(), m, image_set_list) module.post_run(workspace) for feature_name in m.get_feature_names(cpmeas.EXPERIMENT): if not expected.has_key(feature_name): print "Missing measurement: %s" % feature_name continue value = m.get_experiment_measurement(feature_name) e_value = expected[feature_name] diff = abs(value - e_value) * 2 / abs(value + e_value) self.assertTrue( diff < .05, "%s: Matlab: %f, Python: %f diff: %f" % (feature_name, e_value, value, diff)) if diff > .01: print( "Warning: > 1%% difference for %s: Matlab: %f, Python: %f diff: %f" % (feature_name, e_value, value, diff)) if feature_name.startswith("EC50"): filename = "EC49_" + feature_name[5:] + ".pdf" self.assertTrue( os.path.isfile(os.path.join(temp_dir, filename))) finally: try: workspace.measurements.hdf5_dict.hdf5_file.close() except: pass for filename in os.listdir(temp_dir): path = os.path.join(temp_dir, filename) os.remove(path) os.rmdir(temp_dir)
def interface(self, start_signal, image_set_start=1, image_set_end=None, overwrite=True): '''Top-half thread for running an analysis. Sets up grouping for jobs, deals with returned measurements, reports status periodically. start_signal- signal this semaphore when jobs are ready. image_set_start - beginning image set number to process image_set_end - last image set number to process overwrite - whether to recompute imagesets that already have data in initial_measurements. ''' posted_analysis_started = False acknowledged_thread_start = False measurements = None workspace = None try: # listen for pipeline events, and pass them upstream self.pipeline.add_listener(lambda pipe, evt: self.post_event(evt)) initial_measurements = None if self.output_path is None: # Caller wants a temporary measurements file. fd, filename = tempfile.mkstemp(".h5") try: fd = os.fdopen(fd, "wb") fd.write(self.initial_measurements_buf) fd.close() initial_measurements = cpmeas.Measurements( filename=filename, mode="r") measurements = cpmeas.Measurements( image_set_start = None, copy = initial_measurements, mode = "a") finally: if initial_measurements is not None: initial_measurements.close() os.unlink(filename) else: with open(self.output_path, "wb") as fd: fd.write(self.initial_measurements_buf) measurements = cpmeas.Measurements(image_set_start=None, filename=self.output_path, mode="a") # The shared dicts are needed in jobserver() self.shared_dicts = [m.get_dictionary() for m in self.pipeline.modules()] workspace = cpw.Workspace(self.pipeline, None, None, None, measurements, cpimage.ImageSetList()) if image_set_end is None: image_set_end = measurements.get_image_numbers()[-1] image_sets_to_process = filter( lambda x: x >= image_set_start and x <= image_set_end, measurements.get_image_numbers()) self.post_event(AnalysisStarted()) posted_analysis_started = True # reset the status of every image set that needs to be processed has_groups = measurements.has_groups() if self.pipeline.requires_aggregation(): overwrite = True if has_groups and not overwrite: if not measurements.has_feature(cpmeas.IMAGE, self.STATUS): overwrite = True else: group_status = {} for image_number in measurements.get_image_numbers(): group_number = measurements[ cpmeas.IMAGE, cpmeas.GROUP_NUMBER, image_number] status = measurements[cpmeas.IMAGE, self.STATUS, image_number] if status != self.STATUS_DONE: group_status[group_number] = self.STATUS_UNPROCESSED elif group_number not in group_status: group_status[group_number] = self.STATUS_DONE new_image_sets_to_process = [] for image_set_number in image_sets_to_process: needs_reset = False if (overwrite or (not measurements.has_measurements( cpmeas.IMAGE, self.STATUS, image_set_number)) or (measurements[cpmeas.IMAGE, self.STATUS, image_set_number] != self.STATUS_DONE)): needs_reset = True elif has_groups: group_number = measurements[ cpmeas.IMAGE, cpmeas.GROUP_NUMBER, image_set_number] if group_status[group_number] != self.STATUS_DONE: needs_reset = True if needs_reset: measurements[cpmeas.IMAGE, self.STATUS, image_set_number] =\ self.STATUS_UNPROCESSED new_image_sets_to_process.append(image_set_number) image_sets_to_process = new_image_sets_to_process # Find image groups. These are written into measurements prior to # analysis. Groups are processed as a single job. if has_groups or self.pipeline.requires_aggregation(): worker_runs_post_group = True job_groups = {} for image_set_number in image_sets_to_process: group_number = measurements[cpmeas.IMAGE, cpmeas.GROUP_NUMBER, image_set_number] group_index = measurements[cpmeas.IMAGE, cpmeas.GROUP_INDEX, image_set_number] job_groups[group_number] = job_groups.get(group_number, []) + [(group_index, image_set_number)] job_groups = [[isn for _, isn in sorted(job_groups[group_number])] for group_number in sorted(job_groups)] else: worker_runs_post_group = False # prepare_group will be run in worker, but post_group is below. job_groups = [[image_set_number] for image_set_number in image_sets_to_process] # XXX - check that any constructed groups are complete, i.e., # image_set_start and image_set_end shouldn't carve them up. if not worker_runs_post_group: # put the first job in the queue, then wait for the first image to # finish (see the check of self.finish_queue below) to post the rest. # This ensures that any shared data from the first imageset is # available to later imagesets. self.work_queue.put((job_groups[0], worker_runs_post_group, True)) waiting_for_first_imageset = True del job_groups[0] else: waiting_for_first_imageset = False for job in job_groups: self.work_queue.put((job, worker_runs_post_group, False)) job_groups = [] start_signal.release() acknowledged_thread_start = True # We loop until every image is completed, or an outside event breaks the loop. while not self.cancelled: # gather measurements while not self.received_measurements_queue.empty(): image_numbers, buf = self.received_measurements_queue.get() image_numbers = [int(i) for i in image_numbers] recd_measurements = cpmeas.load_measurements_from_buffer(buf) measurements.copy_relationships(recd_measurements) for o in recd_measurements.get_object_names(): if o == cpmeas.EXPERIMENT: continue # Written during prepare_run / post_run for feature in recd_measurements.get_feature_names(o): measurements[o, feature, image_numbers] \ = recd_measurements[o, feature, image_numbers] for image_set_number in image_numbers: measurements[cpmeas.IMAGE, self.STATUS, image_set_number] = self.STATUS_DONE recd_measurements.close() del recd_measurements # check for jobs in progress while not self.in_process_queue.empty(): image_set_numbers = self.in_process_queue.get() for image_set_number in image_set_numbers: measurements[cpmeas.IMAGE, self.STATUS, int(image_set_number)] = self.STATUS_IN_PROCESS # check for finished jobs that haven't returned measurements, yet while not self.finished_queue.empty(): finished_req = self.finished_queue.get() measurements[cpmeas.IMAGE, self.STATUS, int(finished_req.image_set_number)] = self.STATUS_FINISHED_WAITING if waiting_for_first_imageset: assert isinstance(finished_req, ImageSetSuccessWithDictionary) self.shared_dicts = finished_req.shared_dicts waiting_for_first_imageset = False assert len(self.shared_dicts) == len(self.pipeline.modules()) # if we had jobs waiting for the first image set to finish, # queue them now that the shared state is available. for job in job_groups: self.work_queue.put((job, worker_runs_post_group, False)) finished_req.reply(Ack()) # check progress and report counts = collections.Counter(measurements[cpmeas.IMAGE, self.STATUS, image_set_number] for image_set_number in image_sets_to_process) self.post_event(AnalysisProgress(counts)) # Are we finished? if counts[self.STATUS_DONE] == len(image_sets_to_process): last_image_number = measurements.get_image_numbers()[-1] measurements.image_set_number = last_image_number if not worker_runs_post_group: self.pipeline.post_group(workspace, {}) workspace = cpw.Workspace(self.pipeline, None, None, None, measurements, None, None) workspace.post_run_display_handler = \ self.post_run_display_handler self.pipeline.post_run(workspace) break measurements.flush() # not done, wait for more work with self.interface_work_cv: while (self.paused or ((not self.cancelled) and self.in_process_queue.empty() and self.finished_queue.empty() and self.received_measurements_queue.empty())): self.interface_work_cv.wait() # wait for a change of status or work to arrive finally: # Note - the measurements file is owned by the queue consumer # after this post_event. # if not acknowledged_thread_start: start_signal.release() if posted_analysis_started: was_cancelled = self.cancelled self.post_event(AnalysisFinished(measurements, was_cancelled)) self.stop_workers() self.analysis_id = False # this will cause the jobserver thread to exit
def do_job(self, job): '''Handle a work request to its completion job - WorkRequest ''' import cellprofiler.pipeline as cpp job_measurements = [] try: send_dictionary = job.wants_dictionary logger.info("Starting job") # Fetch the pipeline and preferences for this analysis if we don't have it current_pipeline, current_preferences = \ self.pipelines_and_preferences.get( self.current_analysis_id, (None, None)) if not current_pipeline: logger.debug("Fetching pipeline and preferences") rep = self.send( PipelinePreferencesRequest(self.current_analysis_id)) logger.debug("Received pipeline and preferences response") preferences_dict = rep.preferences # update preferences to match remote values cpprefs.set_preferences_from_dict(preferences_dict) logger.debug("Loading pipeline") pipeline_blob = rep.pipeline_blob.tostring() current_pipeline = cpp.Pipeline() current_pipeline.loadtxt(StringIO.StringIO(pipeline_blob), raise_on_error=True) logger.debug("Pipeline loaded") current_pipeline.add_listener( self.pipeline_listener.handle_event) current_preferences = rep.preferences self.pipelines_and_preferences[self.current_analysis_id] = ( current_pipeline, current_preferences) else: # update preferences to match remote values cpprefs.set_preferences_from_dict(current_preferences) # Reset the listener's state self.pipeline_listener.reset() logger.debug("Getting initial measurements") # Fetch the path to the intial measurements if needed. current_measurements = self.initial_measurements.get( self.current_analysis_id) if current_measurements is None: logger.debug("Sending initial measurements request") rep = self.send( InitialMeasurementsRequest(self.current_analysis_id)) logger.debug("Got initial measurements") current_measurements = \ self.initial_measurements[self.current_analysis_id] = \ cpmeas.load_measurements_from_buffer(rep.buf) else: logger.debug("Has initial measurements") # Make a copy of the measurements for writing during this job current_measurements = cpmeas.Measurements( copy=current_measurements) all_measurements.add(current_measurements) job_measurements.append(current_measurements) successful_image_set_numbers = [] image_set_numbers = job.image_set_numbers worker_runs_post_group = job.worker_runs_post_group logger.info("Doing job: " + ",".join(map(str, image_set_numbers))) self.pipeline_listener.image_set_number = image_set_numbers[0] if not worker_runs_post_group: # Get the shared state from the first imageset in this run. shared_dicts = self.send( SharedDictionaryRequest( self.current_analysis_id)).dictionaries assert len(shared_dicts) == len(current_pipeline.modules()) for module, new_dict in zip(current_pipeline.modules(), shared_dicts): module.set_dictionary_for_worker(new_dict) # Run prepare group if this is the first image in the group. We do # this here (even if there's no grouping in the pipeline) to ensure # that any changes to the modules' shared state dictionaries get # propagated correctly. should_process = True if current_measurements[cpmeas.IMAGE, cpmeas.GROUP_INDEX, image_set_numbers[0]] == 1: workspace = cpw.Workspace(current_pipeline, None, None, None, current_measurements, None, None) if not current_pipeline.prepare_group( workspace, current_measurements.get_grouping_keys(), image_set_numbers): # exception handled elsewhere, possibly cancelling this run. should_process = False del workspace # process the images if should_process: abort = False for image_set_number in image_set_numbers: try: self.pipeline_listener.image_set_number = image_set_number last_workspace = current_pipeline.run_image_set( current_measurements, image_set_number, self.interaction_handler, self.display_handler, self.cancel_handler) if self.pipeline_listener.should_abort: abort = True break elif self.pipeline_listener.should_skip: # Report skipped image sets as successful so that # analysis can complete. # Report their measurements because some modules # may have provided measurements before skipping. pass successful_image_set_numbers.append(image_set_number) # Send an indication that the image set finished successfully. if send_dictionary: # The jobserver would like a copy of our modules' # run_state dictionaries. dicts = [ m.get_dictionary_for_worker() for m in current_pipeline.modules() ] req = ImageSetSuccessWithDictionary( self.current_analysis_id, image_set_number=image_set_number, shared_dicts=dicts) else: req = ImageSetSuccess( self.current_analysis_id, image_set_number=image_set_number) rep = self.send(req) except cpp.CancelledException: logging.info("Aborting job after cancellation") abort = True except Exception: try: logging.error("Error in pipeline", exc_info=True) if self.handle_exception( image_set_number=image_set_number ) == ED_STOP: abort = True break except: logging.error( "Error in handling of pipeline exception", exc_info=True) # this is bad. We can't handle nested exceptions # remotely so we just fail on this run. abort = True if abort: current_measurements.close() job_measurements.remove(current_measurements) return if worker_runs_post_group: last_workspace.interaction_handler = \ self.interaction_handler last_workspace.cancel_handler = self.cancel_handler last_workspace.post_group_display_handler = \ self.post_group_display_handler # There might be an exception in this call, but it will be # handled elsewhere, and there's nothing we can do for it # here. current_pipeline.post_group( last_workspace, current_measurements.get_grouping_keys()) del last_workspace # send measurements back to server req = MeasurementsReport(self.current_analysis_id, buf=current_measurements.file_contents(), image_set_numbers=image_set_numbers) rep = self.send(req) except cpp.CancelledException: # Main thread received shutdown signal raise except Exception: logging.error("Error in worker", exc_info=True) if self.handle_exception() == ED_STOP: raise cpp.CancelledException( "Cancelling after user-requested stop") finally: # Clean up any measurements owned by us for m in job_measurements: m.close()
def make_workspace(self, control_points, lengths, radii, image, mask=None, auximage=None): '''Create a workspace containing the control point measurements control_points - an n x 2 x m array where n is the # of control points, and m is the number of objects. lengths - the length of each object radii - the radii_from_training defining the radius at each control pt image - the image to be straightened mask - the mask associated with the image (default = no mask) auximage - a second image to be straightnened (default = no second image) ''' module = S.StraightenWorms() module.objects_name.value = OBJECTS_NAME module.straightened_objects_name.value = STRAIGHTENED_OBJECTS_NAME module.images[0].image_name.value = IMAGE_NAME module.images[ 0].straightened_image_name.value = STRAIGHTENED_IMAGE_NAME module.flip_image.value = IMAGE_NAME module.module_num = 1 # Trick the module into thinking it's read the data file class P: def __init__(self): self.radii_from_training = radii module.training_set_directory.dir_choice = cps.URL_FOLDER_NAME module.training_set_directory.custom_path = "http://www.cellprofiler.org" module.training_set_file_name.value = "TrainingSet.xml" module.training_params = {"TrainingSet.xml": (P(), "URL")} pipeline = cpp.Pipeline() pipeline.add_module(module) def callback(caller, event): self.assertFalse(isinstance(event, cpp.RunExceptionEvent)) pipeline.add_listener(callback) m = cpmeas.Measurements() for i, (y, x) in enumerate(control_points): for v, f in ((x, S.F_CONTROL_POINT_X), (y, S.F_CONTROL_POINT_Y)): feature = "_".join((S.C_WORM, f, str(i + 1))) m.add_measurement(OBJECTS_NAME, feature, v) feature = "_".join((S.C_WORM, S.F_LENGTH)) m.add_measurement(OBJECTS_NAME, feature, lengths) image_set_list = cpi.ImageSetList() image_set = image_set_list.get_image_set(0) image_set.add(IMAGE_NAME, cpi.Image(image, mask)) if auximage is not None: image_set.add(AUX_IMAGE_NAME, cpi.Image(auximage)) module.add_image() module.images[1].image_name.value = AUX_IMAGE_NAME module.images[ 1].straightened_image_name.value = AUX_STRAIGHTENED_IMAGE_NAME object_set = cpo.ObjectSet() objects = cpo.Objects() labels = np.zeros(image.shape, int) for i in range(control_points.shape[2]): if lengths[i] == 0: continue self.rebuild_worm_from_control_points_approx( control_points[:, :, i], radii, labels, i + 1) objects.segmented = labels object_set.add_objects(objects, OBJECTS_NAME) workspace = cpw.Workspace(pipeline, module, image_set, object_set, m, image_set_list) return workspace, module