def make_workspace(self, ground_truth, test):
'''Make a workspace with a ground-truth image and a test image
ground_truth and test are dictionaries with the following keys:
image - the pixel data
mask - (optional) the mask data
crop_mask - (optional) a cropping mask
returns a workspace and module
'''
module = C.CalculateImageOverlap()
module.module_num = 1
module.obj_or_img.value = O_IMG
module.ground_truth.value = GROUND_TRUTH_IMAGE_NAME
module.test_img.value = TEST_IMAGE_NAME
module.wants_emd.value = True
pipeline = cpp.Pipeline()
def callback(caller, event):
self.assertFalse(isinstance(event, cpp.RunExceptionEvent))
pipeline.add_listener(callback)
pipeline.add_module(module)
image_set_list = cpi.ImageSetList()
image_set = image_set_list.get_image_set(0)
for name, d in ((GROUND_TRUTH_IMAGE_NAME, ground_truth),
(TEST_IMAGE_NAME, test)):
image = cpi.Image(d["image"],
mask=d.get("mask"),
crop_mask=d.get("crop_mask"))
image_set.add(name, image)
workspace = cpw.Workspace(pipeline, module, image_set, cpo.ObjectSet(),
cpmeas.Measurements(), image_set_list)
return workspace, module
def run_workspace(self, operation, m1_is_image_measurement, m1_data,
m2_is_image_measurement, m2_data,
setup_fn = None):
'''Create and run a workspace, returning the measurements
m<n>_is_image_measurement - true for an image measurement, false
for object
m<n>_data - either a single value or an array
setup_fn - this gets called with the module before running
'''
module = C.CalculateMath()
module.operation.value = operation
measurements = cpmeas.Measurements()
for i, operand, is_image_measurement, data in\
((0, module.operands[0], m1_is_image_measurement, m1_data),
(1, module.operands[1], m2_is_image_measurement, m2_data)):
measurement = "measurement%d"%i
if is_image_measurement:
operand.operand_choice.value = C.MC_IMAGE
measurements.add_image_measurement(measurement, data)
else:
operand.operand_choice.value = C.MC_OBJECT
operand.operand_objects.value = OBJECT[i]
measurements.add_measurement(OBJECT[i], measurement, data)
operand.operand_measurement.value = measurement
module.output_feature_name.value = OUTPUT_MEASUREMENTS
pipeline = cpp.Pipeline()
image_set_list = cpi.ImageSetList()
workspace = cpw.Workspace(pipeline,
module,
image_set_list.get_image_set(0),
cpo.ObjectSet(),
measurements,
image_set_list)
if setup_fn is not None:
setup_fn(module, workspace)
module.run(workspace)
return measurements
def test_03_02_gray_mask(self):
image_set_list = cpi.ImageSetList()
image_set = image_set_list.get_image_set(0)
np.random.seed(0)
pixel_data = np.random.uniform(size=(10, 15)).astype(np.float32)
image_set.add(IMAGE_NAME, cpi.Image(pixel_data))
masking_image = np.random.uniform(size=(10, 15))
image_set.add(MASKING_IMAGE_NAME, cpi.Image(masking_image))
masking_image = masking_image > 0.5
pipeline = cpp.Pipeline()
module = M.MaskImage()
module.source_choice.value = M.IO_IMAGE
module.object_name.value = OBJECTS_NAME
module.image_name.value = IMAGE_NAME
module.masking_image_name.value = MASKING_IMAGE_NAME
module.masked_image_name.value = MASKED_IMAGE_NAME
module.invert_mask.value = False
module.module_num = 1
workspace = cpw.Workspace(
pipeline,
module,
image_set,
cpo.ObjectSet(),
cpmeas.Measurements(),
image_set_list,
)
module.run(workspace)
masked_image = workspace.image_set.get_image(MASKED_IMAGE_NAME)
self.assertTrue(isinstance(masked_image, cpi.Image))
self.assertTrue(
np.all(masked_image.pixel_data[masking_image] ==
pixel_data[masking_image]))
self.assertTrue(np.all(masked_image.pixel_data[~masking_image] == 0))
self.assertTrue(np.all(masked_image.mask == masking_image))
self.assertFalse(masked_image.has_masking_objects)
def run_imagemath(self, images, modify_module_fn=None, measurement=None):
'''Run the ImageMath module, returning the image created
images - a list of dictionaries. The dictionary has keys:
pixel_data - image pixel data
mask - mask for image
cropping - cropping mask for image
modify_module_fn - a function of the signature, fn(module)
that allows the test to modify the module.
measurement - an image measurement value
'''
image_set_list = cpi.ImageSetList()
image_set = image_set_list.get_image_set(0)
module = I.ImageMath()
module.module_num = 1
for i, image in enumerate(images):
pixel_data = image['pixel_data']
mask = image.get('mask', None)
cropping = image.get('cropping', None)
if i >= 2:
module.add_image()
name = 'inputimage%s' % i
module.images[i].image_name.value = name
img = cpi.Image(pixel_data, mask=mask, crop_mask=cropping)
image_set.add(name, img)
module.output_image_name.value = 'outputimage'
if modify_module_fn is not None:
modify_module_fn(module)
pipeline = cpp.Pipeline()
pipeline.add_module(module)
measurements = cpmeas.Measurements()
if measurement is not None:
measurements.add_image_measurement(MEASUREMENT_NAME,
str(measurement))
workspace = cpw.Workspace(pipeline, module, image_set, cpo.ObjectSet(),
measurements, image_set_list)
module.run(workspace)
return image_set.get_image('outputimage')
def make_place_workspace(self, images):
image_set_list = cpi.ImageSetList()
image_set = image_set_list.get_image_set(0)
module = T.Tile()
module.module_num = 1
module.tile_method.value = T.T_WITHIN_CYCLES
module.output_image.value = OUTPUT_IMAGE_NAME
module.wants_automatic_rows.value = False
module.wants_automatic_columns.value = True
module.rows.value = 1
for i, image in enumerate(images):
image_name = input_image_name(i)
if i == 0:
module.input_image.value = image_name
else:
if len(module.additional_images) <= i:
module.add_image()
module.additional_images[i -
1].input_image_name.value = image_name
image_set.add(image_name, cpi.Image(image))
pipeline = cpp.Pipeline()
def callback(caller, event):
self.assertFalse(isinstance(event, cpp.RunExceptionEvent))
pipeline.add_listener(callback)
pipeline.add_module(module)
workspace = cpw.Workspace(
pipeline,
module,
image_set,
cpo.ObjectSet(),
cpmeas.Measurements(),
image_set_list,
)
return workspace, module
def make_workspace(self, image_set_count):
image_set_list = cpi.ImageSetList()
for i in range(image_set_count):
image_set = image_set_list.get_image_set(i)
module = L.LabelImages()
pipeline = cpp.Pipeline()
def callback(caller, event):
self.assertFalse(isinstance(event, cpp.RunExceptionEvent))
pipeline.add_listener(callback)
module.module_num = 1
pipeline.add_module(module)
workspace = cpw.Workspace(
pipeline,
module,
image_set_list.get_image_set(0),
cpo.ObjectSet(),
cpmeas.Measurements(),
image_set_list,
)
return workspace, module
def make_workspace(self, primary_labels, secondary_labels):
"""Make a workspace that has objects for the input labels
returns a workspace with the following
object_set - has object with name "primary" containing
the primary labels
has object with name "secondary" containing
the secondary labels
"""
isl = cpi.ImageSetList()
module = cpmit.IdentifyTertiarySubregion()
module.primary_objects_name.value = PRIMARY
module.secondary_objects_name.value = SECONDARY
module.subregion_objects_name.value = TERTIARY
workspace = cpw.Workspace(cpp.Pipeline(), module, isl.get_image_set(0),
cpo.ObjectSet(), cpm.Measurements(), isl)
for labels, name in ((primary_labels, PRIMARY), (secondary_labels,
SECONDARY)):
objects = cpo.Objects()
objects.segmented = labels
workspace.object_set.add_objects(objects, name)
return workspace
def prepare_for_run(self):
"""Prepare to run pipeline.
This code was largely copied from 'PipelineController.start_debugging()'
with updates to the 'PipelineController' UI removed. I wish I had a
better understanding of what exactly this does, but I'm pretty much
using it as a black box for the time being.
"""
self.__measurements = cpm.Measurements(can_overwrite=True)
self.__object_set = cpo.ObjectSet(can_overwrite=True)
self.__image_set_list = cpi.ImageSetList()
workspace = cpw.Workspace(self.__pipeline, None, None, None,
self.__measurements, image_set_list,
self.__frame)
try:
if not self.__pipeline.prepare_run(workspace):
print 'Error: failed to get image sets'
self.__keys, self.__groupings = self.__pipeline.get_groupings(
self.__image_set_list)
except ValueError, v:
message = "Error while preparing for run:\n%s" % (v)
wx.MessageBox(message, "Pipeline error", wx.OK | wx.ICON_ERROR,
self.__frame)
def test_11_01_add_and_do_nothing(self):
#
# Regression for issue #1333 - add one, do nothing, input image
# is changed
#
r = np.random.RandomState()
r.seed(1101)
m = cpmeas.Measurements()
pixel_data = r.uniform(size=(20, 20))
m.add("inputimage", cpi.Image(pixel_data))
module = I.ImageMath()
module.images[0].image_name.value = "inputimage"
module.output_image_name.value = "outputimage"
module.operation.value = I.O_NONE
module.addend.value = 0.5
module.module_num = 1
pipeline = cpp.Pipeline()
pipeline.add_module(module)
workspace = cpw.Workspace(pipeline, module, m, None, m, None)
module.run(workspace)
np.testing.assert_array_almost_equal(
pixel_data,
m.get_image("inputimage").pixel_data)
def test_02_02_erase_keep(self):
module = S.SpeedUpCellProfiler()
module.how_to_remove.value = S.C_KEEP
module.image_names[0].image_name.value = "Image1"
module.module_num = 1
image_set_list = cpi.ImageSetList()
image_set = image_set_list.get_image_set(0)
image_set.add("Image1", cpi.Image(np.zeros((10, 10))))
image_set.add("Image2", cpi.Image(np.zeros((10, 10))))
pipeline = cpp.Pipeline()
def callback(caller, event):
self.assertFalse(isinstance(event, cpp.RunExceptionEvent))
pipeline.add_listener(callback)
pipeline.add_module(module)
workspace = cpw.Workspace(pipeline, module, image_set, cpo.ObjectSet(),
cpmeas.Measurements(), image_set_list)
module.run(workspace)
image = image_set.get_image("Image2")
self.assertFalse(isinstance(image, cpi.Image))
image = image_set.get_image("Image1")
self.assertTrue(isinstance(image, cpi.Image))
def make_workspace(self, pixel_data, mask=None, objects=None):
image_set_list = cpi.ImageSetList()
image_set = image_set_list.get_image_set(0)
object_set = cpo.ObjectSet()
image = cpi.Image(pixel_data)
if not mask is None:
image.mask = mask
image_set.add(MY_IMAGE, image)
if not objects is None:
o = cpo.Objects()
o.segmented = objects
object_set.add_objects(o, MY_OBJECTS)
module = miq.MeasureImageQuality()
module.images_choice.value = miq.O_SELECT
module.image_groups[0].include_image_scalings.value = False
module.image_groups[0].image_names.value = MY_IMAGE
module.image_groups[0].use_all_threshold_methods.value = False
module.module_num = 1
pipeline = cpp.Pipeline()
pipeline.add_module(module)
workspace = cpw.Workspace(pipeline, module, image_set, object_set,
cpmeas.Measurements(), image_set_list)
return workspace
def test_03_02_zernikes_are_different(self):
'''Regression test of IMG-773'''
np.random.seed(32)
labels = np.zeros((40,20), int)
#
# Make two "objects" composed of random foreground/background
#
labels[1:19,1:19] = (np.random.uniform(size=(18,18)) > .5).astype(int)
labels[21:39,1:19] = (np.random.uniform(size=(18,18)) > .5).astype(int) * 2
objects = cpo.Objects()
objects.segmented = labels
object_set = cpo.ObjectSet()
object_set.add_objects(objects, "SomeObjects")
module = cpmoas.MeasureObjectAreaShape()
module.object_groups[0].name.value = "SomeObjects"
module.calculate_zernikes.value = True
module.module_num = 1
image_set_list = cpi.ImageSetList()
measurements = cpmeas.Measurements()
pipeline = cpp.Pipeline()
pipeline.add_module(module)
def callback(caller, event):
self.assertFalse(isinstance(event, cpp.RunExceptionEvent))
pipeline.add_listener(callback)
workspace = cpw.Workspace(pipeline, module,
image_set_list.get_image_set(0),
object_set, measurements, image_set_list)
module.run(workspace)
features = [x[1] for x in module.get_measurement_columns(pipeline)
if x[0] == "SomeObjects" and
x[1].startswith("AreaShape_Zernike")]
for feature in features:
values = measurements.get_current_measurement(
"SomeObjects", feature)
self.assertEqual(len(values), 2)
self.assertNotEqual(values[0], values[1])
def make_workspace(self, scheme, images, adjustments):
module = G.GrayToColor()
module.scheme_choice.value = scheme
image_names = [
"image%d" % i if images[i] is not None else G.LEAVE_THIS_BLACK
for i in range(7)
]
for image_name_setting, image_name, adjustment_setting, adjustment\
in zip((module.red_image_name, module.green_image_name,
module.blue_image_name,
module.cyan_image_name, module.magenta_image_name,
module.yellow_image_name, module.gray_image_name),
image_names,
(module.red_adjustment_factor, module.green_adjustment_factor,
module.blue_adjustment_factor, module.cyan_adjustment_factor,
module.magenta_adjustment_factor, module.yellow_adjustment_factor,
module.gray_adjustment_factor),
adjustments):
image_name_setting.value = image_name
adjustment_setting.value = adjustment
module.rgb_image_name.value = OUTPUT_IMAGE_NAME
module.module_num = 1
pipeline = cpp.Pipeline()
def callback(caller, event):
self.assertFalse(isinstance(event, cpp.RunExceptionEvent))
pipeline.add_listener(callback)
pipeline.add_module(module)
image_set_list = cpi.ImageSetList()
image_set = image_set_list.get_image_set(0)
for image, image_name in zip(images, image_names):
if image is not None:
image_set.add(image_name, cpi.Image(image))
workspace = cpw.Workspace(pipeline, module, image_set, cpo.ObjectSet(),
cpmeas.Measurements(), image_set_list)
return workspace, module
def test_02_03_double_mask(self):
labels = np.zeros((10,15),int)
labels[2:5,3:8] = 1
labels[5:8, 10:14] = 2
object_set = cpo.ObjectSet()
objects = cpo.Objects()
objects.segmented = labels
object_set.add_objects(objects, OBJECTS_NAME)
image_set_list = cpi.ImageSetList()
image_set=image_set_list.get_image_set(0)
np.random.seed(0)
pixel_data = np.random.uniform(size=(10,15)).astype(np.float32)
mask = np.random.uniform(size=(10,15)) > .5
image_set.add(IMAGE_NAME, cpi.Image(pixel_data, mask))
expected_mask = (mask & (labels > 0))
pipeline = cpp.Pipeline()
module = M.MaskImage()
module.source_choice.value = M.IO_OBJECTS
module.object_name.value = OBJECTS_NAME
module.image_name.value = IMAGE_NAME
module.masked_image_name.value = MASKED_IMAGE_NAME
module.invert_mask.value = False
module.module_num = 1
workspace = cpw.Workspace(pipeline, module, image_set, object_set,
cpmeas.Measurements(), image_set_list)
module.run(workspace)
masked_image = workspace.image_set.get_image(MASKED_IMAGE_NAME)
self.assertTrue(isinstance(masked_image, cpi.Image))
self.assertTrue(np.all(masked_image.pixel_data[expected_mask] ==
pixel_data[expected_mask]))
self.assertTrue(np.all(masked_image.pixel_data[~ expected_mask] == 0))
self.assertTrue(np.all(masked_image.mask == expected_mask))
self.assertTrue(np.all(masked_image.masking_objects.segmented == labels))
def make_workspace(self, image_measurements, object_measurements):
'''Make a workspace with a FlagImage module and the given measurements
image_measurements - a sequence of single image measurements. Use
image_measurement_name(i) to get the name of
the i th measurement
object_measurements - a seequence of sequences of object measurements.
These are stored under object, OBJECT_NAME with
measurement name object_measurement_name(i) for
the i th measurement.
returns module, workspace
'''
module = F.FlagImage()
measurements = cpmeas.Measurements()
for i in range(len(image_measurements)):
measurements.add_image_measurement(image_measurement_name(i),
image_measurements[i])
for i in range(len(object_measurements)):
measurements.add_measurement(OBJECT_NAME,
object_measurement_name(i),
np.array(object_measurements[i]))
flag = module.flags[0]
self.assertTrue(isinstance(flag, cps.SettingsGroup))
flag.category.value = MEASUREMENT_CATEGORY
flag.feature_name.value = MEASUREMENT_FEATURE
module.module_num = 1
pipeline = cpp.Pipeline()
def callback(caller,event):
self.assertFalse(isinstance(event, cpp.RunExceptionEvent))
pipeline.add_listener(callback)
pipeline.add_module(module)
image_set_list = cpi.ImageSetList()
image_set = image_set_list.get_image_set(0)
workspace = cpw.Workspace(pipeline, module, image_set, cpo.ObjectSet(),
measurements, image_set_list)
return module, workspace
def run_module(self, image, mask=None, fn=None):
'''Run the FlipAndRotate module
image - pixel data to be transformed
mask - optional mask on the pixel data
fn - function with signature, "fn(module)" that will be
called with the FlipAndRotate module
returns an Image object containing the flipped/rotated/masked/cropped
image and the angle measurement.
'''
img = cpi.Image(image, mask)
image_set_list = cpi.ImageSetList()
image_set = image_set_list.get_image_set(0)
image_set.add(IMAGE_NAME, img)
module = F.FlipAndRotate()
module.image_name.value = IMAGE_NAME
module.output_name.value = OUTPUT_IMAGE
module.module_num = 1
if fn is not None:
fn(module)
pipeline = cpp.Pipeline()
pipeline.add_module(module)
def error_callback(caller, event):
self.assertFalse(isinstance(event, cpp.RunExceptionEvent))
pipeline.add_listener(error_callback)
measurements = cpmeas.Measurements()
workspace = cpw.Workspace(pipeline, module, image_set, cpo.ObjectSet(),
measurements, image_set_list)
module.run(workspace)
feature = F.M_ROTATION_F % OUTPUT_IMAGE
self.assertTrue(
feature in measurements.get_feature_names(cpmeas.IMAGE))
angle = measurements.get_current_image_measurement(feature)
output_image = image_set.get_image(OUTPUT_IMAGE)
return (output_image, angle)
def make_workspace(self, labels, image, mask = None,
intensity_image = None,
wants_graph = False):
image_set_list = cpi.ImageSetList()
image_set = image_set_list.get_image_set(0)
img = cpi.Image(image, mask)
image_set.add(IMAGE_NAME, img)
object_set = cpo.ObjectSet()
o = cpo.Objects()
o.segmented = labels
object_set.add_objects(o, OBJECT_NAME)
module = M.MeasureNeurons()
module.image_name.value = IMAGE_NAME
module.seed_objects_name.value = OBJECT_NAME
if intensity_image is not None:
img = cpi.Image(intensity_image)
image_set.add(INTENSITY_IMAGE_NAME, img)
module.intensity_image_name.value = INTENSITY_IMAGE_NAME
if wants_graph:
module.wants_neuron_graph.value = True
module.directory.dir_choice = cps.ABSOLUTE_FOLDER_NAME
module.directory.custom_path = self.temp_dir
module.edge_file_name.value = EDGE_FILE
module.vertex_file_name.value = VERTEX_FILE
module.module_num = 1
pipeline = cpp.Pipeline()
def callback(caller, event):
self.assertFalse(isinstance(event, cpp.LoadExceptionEvent))
self.assertFalse(isinstance(event, cpp.RunExceptionEvent))
pipeline.add_listener(callback)
pipeline.add_module(module)
workspace = cpw.Workspace(pipeline, module, image_set, object_set,
cpmeas.Measurements(), image_set_list)
return workspace, module
def write_schema(pipeline_filename):
if pipeline_filename is None:
raise ValueError(
"The --write-schema-and-exit switch must be used in conjunction\n"
"with the -p or --pipeline switch to load a pipeline with an\n"
"ExportToDatabase module.")
import cellprofiler.pipeline as cpp
import cellprofiler.measurement as cpmeas
import cellprofiler.object as cpo
import cellprofiler.workspace as cpw
pipeline = cpp.Pipeline()
pipeline.load(pipeline_filename)
pipeline.turn_off_batch_mode()
for module in pipeline.modules():
if module.module_name == "ExportToDatabase":
break
else:
raise ValueError(
"The pipeline, \"%s\", does not have an ExportToDatabase module" %
pipeline_filename)
m = cpmeas.Measurements()
workspace = cpw.Workspace(pipeline, module, m, cpo.ObjectSet, m, None)
module.prepare_run(workspace)
def make_workspace(self, gridding, labels=None):
module = I.IdentifyObjectsInGrid()
module.module_num = 1
module.grid_name.value = GRID_NAME
module.output_objects_name.value = OUTPUT_OBJECTS_NAME
module.guiding_object_name.value = GUIDING_OBJECTS_NAME
module.outlines_name.value = OUTLINES_NAME
image_set_list = cpi.ImageSetList()
object_set = cpo.ObjectSet()
if labels is not None:
my_objects = cpo.Objects()
my_objects.segmented = labels
object_set.add_objects(my_objects, GUIDING_OBJECTS_NAME)
pipeline = cpp.Pipeline()
def callback(caller,event):
self.assertFalse(isinstance(event, cpp.RunExceptionEvent))
pipeline.add_listener(callback)
pipeline.add_module(module)
workspace = cpw.Workspace(pipeline, module,
image_set_list.get_image_set(0),
object_set, cpmeas.Measurements(),
image_set_list)
workspace.set_grid(GRID_NAME, gridding)
return (workspace, module)
def make_tile_workspace(self, images):
module = T.Tile()
module.module_num = 1
module.tile_method.value = T.T_ACROSS_CYCLES
module.input_image.value = INPUT_IMAGE_NAME
module.output_image.value = OUTPUT_IMAGE_NAME
pipeline = cpp.Pipeline()
def callback(caller, event):
self.assertFalse(isinstance(event, cpp.RunExceptionEvent))
pipeline.add_listener(callback)
pipeline.add_module(module)
image_set_list = cpi.ImageSetList()
for i, image in enumerate(images):
image_set = image_set_list.get_image_set(i)
image_set.add(INPUT_IMAGE_NAME, cpi.Image(image))
workspace = cpw.Workspace(pipeline, module,
image_set_list.get_image_set(0),
cpo.ObjectSet(), cpmeas.Measurements(),
image_set_list)
return workspace, module
def test_01_02_run(self):
module = instantiate_module(MODULE_NAME)
module.input_objects_name.value = INPUT_OBJECTS_NAME
module.output_objects_name.value = OUTPUT_OBJECTS_NAME
module.module_num = 1
pipeline = cpp.Pipeline()
pipeline.add_module(module)
object_set = cpo.ObjectSet()
#
# Pick a bunch of random points, dilate them using the distance
# transform and then label the result.
#
r = np.random.RandomState()
r.seed(12)
bimg = np.ones((100, 100), bool)
bimg[r.randint(0,100, 50), r.randint(0, 100, 50)] = False
labels, count = label(distance_transform_edt(bimg) <= 5)
#
# Make the input objects
#
input_objects = cpo.Objects()
input_objects.segmented = labels
expected = skeletonize_labels(labels)
object_set.add_objects(input_objects, INPUT_OBJECTS_NAME)
#
# Make the workspace
#
workspace = cpw.Workspace(pipeline, module, None, object_set,
cpmeas.Measurements(), None)
module.run(workspace)
self.assertTrue(OUTPUT_OBJECTS_NAME in object_set.object_names,
"Could not find the output objects in the object set")
output_objects = object_set.get_objects(OUTPUT_OBJECTS_NAME)
np.testing.assert_array_equal(expected, output_objects.segmented)
def check(self, module, url, dd, keys=None, xml=None):
'''Check that running the metadata module on a url generates the expected dictionary'''
pipeline = cpp.Pipeline()
imgs = I.Images()
imgs.filter_choice.value = I.FILTER_CHOICE_NONE
imgs.module_num = 1
pipeline.add_module(imgs)
module.module_num = 2
pipeline.add_module(module)
pipeline.add_urls([url])
m = cpmeas.Measurements()
workspace = cpw.Workspace(pipeline, module, None, None, m, None)
file_list = workspace.file_list
file_list.add_files_to_filelist([url])
if xml is not None:
file_list.add_metadata(url, xml)
ipds = pipeline.get_image_plane_details(workspace)
self.assertEqual(len(ipds), len(dd))
for d, ipd in zip(dd, ipds):
self.assertDictContainsSubset(d, ipd.metadata)
all_keys = pipeline.get_available_metadata_keys().keys()
if keys is not None:
for key in keys:
self.assertIn(key, all_keys)
def run_tteesstt(self, objects, level):
module = instantiate_module(MODULE_NAME)
module.objects_name.value = OBJECTS_NAME
module.module_num = 1
pipeline = cpp.Pipeline()
pipeline.add_module(module)
object_set = cpo.ObjectSet()
object_set.add_objects(objects, OBJECTS_NAME)
#
# Make the workspace
#
measurements = cpmeas.Measurements()
workspace = cpw.Workspace(pipeline, module, None, object_set,
measurements, None)
module.run(workspace)
values = measurements.get_measurement(OBJECTS_NAME, "Example5_MeanDistance")
self.assertEqual(len(values), objects.count)
for labels, indices in objects.get_labels():
for l in np.unique(labels):
#
# Test very slowly = 1 object per pass
#
if l == 0:
continue
d = scipy.ndimage.distance_transform_edt(labels == l)
value = np.sum(d) / np.sum(labels == l)
if abs(value - values[l-1]) > .0001:
if level == 1:
self.fail("You got the wrong answer (label = %d, mine = %f, yours = %f" %
l, value, values[l-1])
else:
sys.stderr.write("Oh too bad, did not pass level %d\n" % level)
return
if level > 1:
print "+%d for you!" % level
def make_workspace(self, pixel_data, mask=None):
image = cpi.Image(pixel_data, mask)
image_set_list = cpi.ImageSetList()
image_set = image_set_list.get_image_set(0)
image_set.add(IMAGE_NAME, image)
module = ID.IdentifyDeadWorms()
module.module_num = 1
module.image_name.value = IMAGE_NAME
module.object_name.value = OBJECTS_NAME
pipeline = cpp.Pipeline()
def callback(caller, event):
self.assertFalse(isinstance(event, cpp.LoadExceptionEvent))
self.assertFalse(isinstance(event, cpp.RunExceptionEvent))
pipeline.add_listener(callback)
pipeline.add_module(module)
workspace = cpw.Workspace(pipeline, module, image_set, cpo.ObjectSet(),
cpmeas.Measurements(), image_set_list)
return workspace, module
def test_13_04_extra_lines(self):
#
# Regression test of issue #1211 - extra line at end / blank lines
#
dir = os.path.join(example_images_directory(), "ExampleSBSImages")
file_name = 'Channel2-01-A-01.tif'
csv_text = '''"Image_FileName_DNA","Image_PathName_DNA"
"%s","%s"
''' % (file_name, dir)
pipeline, module, filename = self.make_pipeline(csv_text)
try:
assert isinstance(module, L.LoadData)
m = cpmeas.Measurements()
workspace = cpw.Workspace(pipeline, module, m, cpo.ObjectSet(),
m, cpi.ImageSetList())
self.assertTrue(module.prepare_run(workspace))
self.assertTrue(isinstance(m, cpmeas.Measurements))
self.assertEqual(m.image_set_count, 1)
self.assertTrue('FileName_DNA' in m.get_feature_names(cpmeas.IMAGE))
self.assertEqual(m[cpmeas.IMAGE, 'FileName_DNA', 1], file_name)
finally:
os.remove(filename)
def make_workspace(self,
scheme,
images,
adjustments=None,
colors=None,
weights=None):
module = G.GrayToColor()
module.scheme_choice.value = scheme
if scheme not in (G.SCHEME_COMPOSITE, G.SCHEME_STACK):
image_names = [
"image%d" % i if images[i] is not None else G.LEAVE_THIS_BLACK
for i in range(7)
]
for image_name_setting, image_name, adjustment_setting, adjustment \
in zip((module.red_image_name, module.green_image_name,
module.blue_image_name,
module.cyan_image_name, module.magenta_image_name,
module.yellow_image_name, module.gray_image_name),
image_names,
(module.red_adjustment_factor, module.green_adjustment_factor,
module.blue_adjustment_factor, module.cyan_adjustment_factor,
module.magenta_adjustment_factor, module.yellow_adjustment_factor,
module.gray_adjustment_factor),
adjustments):
image_name_setting.value = image_name
adjustment_setting.value = adjustment
else:
while len(module.stack_channels) < len(images):
module.add_stack_channel_cb()
image_names = []
if weights is None:
weights = [1.0] * len(images)
if colors is None:
colors = [
G.DEFAULT_COLORS[i % len(G.DEFAULT_COLORS)]
for i in range(len(images))
]
for i, (image, color,
weight) in enumerate(zip(images, colors, weights)):
image_name = 'image%d' % (i + 1)
image_names.append(image_name)
module.stack_channels[i].image_name.value = image_name
module.stack_channels[i].color.value = color
module.stack_channels[i].weight.value = weight
module.rgb_image_name.value = OUTPUT_IMAGE_NAME
module.module_num = 1
pipeline = cpp.Pipeline()
def callback(caller, event):
self.assertFalse(isinstance(event, cpp.RunExceptionEvent))
pipeline.add_listener(callback)
pipeline.add_module(module)
image_set_list = cpi.ImageSetList()
image_set = image_set_list.get_image_set(0)
for image, image_name in zip(images, image_names):
if image is not None:
image_set.add(image_name, cpi.Image(image))
workspace = cpw.Workspace(pipeline, module, image_set, cpo.ObjectSet(),
cpmeas.Measurements(), image_set_list)
return workspace, module
def test_02_01_compare_to_matlab(self):
expected = {
'EC50_DistCytoplasm_Correlation_Correlation_CorrGreenCorrBlue':
3.982812,
'EC50_DistCytoplasm_Intensity_LowerQuartileIntensity_CorrGreen':
4.139827,
'EC50_DistCytoplasm_Intensity_MedianIntensity_CorrGreen':
4.178600,
'EC50_DistCytoplasm_Intensity_MinIntensityEdge_CorrGreen':
4.059770,
'EC50_DistCytoplasm_Intensity_MinIntensity_CorrGreen':
4.066357,
'EC50_DistCytoplasm_Math_Ratio1':
4.491367,
'EC50_DistCytoplasm_Math_Ratio2':
3.848722,
'EC50_DistCytoplasm_Texture_AngularSecondMoment_CorrGreen_1':
4.948056,
'EC50_DistCytoplasm_Texture_Entropy_CorrGreen_1':
4.687104,
'EC50_DistCytoplasm_Texture_InfoMeas2_CorrGreen_1':
5.0285,
'EC50_DistCytoplasm_Texture_InverseDifferenceMoment_CorrGreen_1':
4.319017,
'EC50_DistCytoplasm_Texture_SumAverage_CorrGreen_1':
4.548876,
'EC50_DistCytoplasm_Texture_SumEntropy_CorrGreen_1':
4.779139,
'EC50_DistCytoplasm_Texture_Variance_CorrGreen_1':
4.218379,
'EC50_DistanceCells_Correlation_Correlation_CorrGreenCorrBlue':
3.708711,
'EC50_DistanceCells_Intensity_IntegratedIntensityEdge_CorrGreen':
4.135146,
'EC50_DistanceCells_Intensity_LowerQuartileIntensity_CorrGreen':
4.5372,
'EC50_DistanceCells_Intensity_MeanIntensityEdge_CorrGreen':
4.1371,
'EC50_DistanceCells_Intensity_MinIntensityEdge_CorrGreen':
4.033999,
'EC50_DistanceCells_Intensity_MinIntensity_CorrGreen':
4.079470,
'EC50_DistanceCells_Texture_AngularSecondMoment_CorrGreen_1':
5.118689,
'EC50_DistanceCells_Texture_Correlation_CorrGreen_1':
4.002074,
'EC50_DistanceCells_Texture_Entropy_CorrGreen_1':
5.008000,
'EC50_DistanceCells_Texture_InfoMeas1_CorrGreen_1':
3.883586,
'EC50_DistanceCells_Texture_InverseDifferenceMoment_CorrGreen_1':
3.977216,
'EC50_DistanceCells_Texture_SumAverage_CorrGreen_1':
4.9741,
'EC50_DistanceCells_Texture_SumEntropy_CorrGreen_1':
5.1455,
'EC50_DistanceCells_Texture_SumVariance_CorrGreen_1':
4.593041,
'EC50_DistanceCells_Texture_Variance_CorrGreen_1':
4.619517,
'EC50_Nuclei_Correlation_Correlation_CorrGreenCorrBlue':
3.751133,
'EC50_Nuclei_Math_Ratio1':
4.491367,
'EC50_Nuclei_Math_Ratio2':
3.848722,
'EC50_Nuclei_Texture_SumAverage_CorrGreen_1':
3.765297,
'EC50_PropCells_AreaShape_Area':
4.740853,
'EC50_PropCells_AreaShape_MajorAxisLength':
5.064460,
'EC50_PropCells_AreaShape_MinorAxisLength':
4.751471,
'EC50_PropCells_AreaShape_Perimeter':
4.949292,
'EC50_PropCells_Correlation_Correlation_CorrGreenCorrBlue':
3.772565,
'EC50_PropCells_Texture_GaborX_CorrGreen_1':
5.007167,
'EC50_PropCells_Texture_InfoMeas2_CorrBlue_1':
4.341353,
'EC50_PropCells_Texture_SumVariance_CorrBlue_1':
4.298359,
'EC50_PropCells_Texture_SumVariance_CorrGreen_1':
4.610826,
'EC50_PropCells_Texture_Variance_CorrBlue_1':
4.396352,
'EC50_PropCells_Texture_Variance_CorrGreen_1':
4.632468,
'EC50_PropCytoplasm_AreaShape_Area':
4.669679,
'EC50_PropCytoplasm_AreaShape_MinorAxisLength':
4.754476,
'EC50_PropCytoplasm_AreaShape_Perimeter':
4.949292,
'EC50_PropCytoplasm_Correlation_Correlation_CorrGreenCorrBlue':
4.072830,
'EC50_PropCytoplasm_Intensity_IntegratedIntensity_CorrGreen':
4.0934,
'EC50_PropCytoplasm_Intensity_LowerQuartileIntensity_CorrGreen':
3.925800,
'EC50_PropCytoplasm_Intensity_MedianIntensity_CorrGreen':
3.9252,
'EC50_PropCytoplasm_Texture_AngularSecondMoment_CorrGreen_1':
4.777481,
'EC50_PropCytoplasm_Texture_Entropy_CorrGreen_1':
4.4432,
'EC50_PropCytoplasm_Texture_GaborX_CorrGreen_1':
5.163371,
'EC50_PropCytoplasm_Texture_InfoMeas2_CorrGreen_1':
4.701046,
'EC50_PropCytoplasm_Texture_SumEntropy_CorrGreen_1':
4.510543,
'EC50_ThresholdedCells_Texture_AngularSecondMoment_CorrBlue_1':
4.560315,
'EC50_ThresholdedCells_Texture_AngularSecondMoment_CorrGreen_1':
4.966674,
'EC50_ThresholdedCells_Texture_Entropy_CorrBlue_1':
4.457866,
'EC50_ThresholdedCells_Texture_InfoMeas2_CorrBlue_1':
4.624049,
'EC50_ThresholdedCells_Texture_SumAverage_CorrBlue_1':
4.686706,
'EC50_ThresholdedCells_Texture_SumEntropy_CorrBlue_1':
4.537378,
'EC50_ThresholdedCells_Texture_SumVariance_CorrBlue_1':
4.322820,
'EC50_ThresholdedCells_Texture_SumVariance_CorrGreen_1':
4.742158,
'EC50_ThresholdedCells_Texture_Variance_CorrBlue_1':
4.265549,
'EC50_ThresholdedCells_Texture_Variance_CorrGreen_1':
4.860020,
'OneTailedZfactor_DistCytoplasm_Intensity_MedianIntensity_CorrGreen':
-4.322503,
'OneTailedZfactor_DistCytoplasm_Intensity_MinIntensityEdge_CorrGreen':
-4.322503,
'OneTailedZfactor_DistCytoplasm_Intensity_MinIntensity_CorrGreen':
-4.322503,
'OneTailedZfactor_DistCytoplasm_Math_Ratio1':
0.622059,
'OneTailedZfactor_DistCytoplasm_Math_Ratio2':
-4.508284,
'OneTailedZfactor_DistCytoplasm_Texture_Entropy_CorrGreen_1':
-4.645887,
'OneTailedZfactor_DistCytoplasm_Texture_InfoMeas2_CorrGreen_1':
-4.279118,
'OneTailedZfactor_DistCytoplasm_Texture_SumAverage_CorrGreen_1':
-4.765570,
'OneTailedZfactor_DistCytoplasm_Texture_SumEntropy_CorrGreen_1':
-4.682335,
'OneTailedZfactor_DistCytoplasm_Texture_Variance_CorrGreen_1':
-4.415607,
'OneTailedZfactor_DistanceCells_Intensity_MeanIntensityEdge_CorrGreen':
-4.200105,
'OneTailedZfactor_DistanceCells_Intensity_MinIntensityEdge_CorrGreen':
-4.316452,
'OneTailedZfactor_DistanceCells_Intensity_MinIntensity_CorrGreen':
-4.316452,
'OneTailedZfactor_DistanceCells_Texture_Correlation_CorrGreen_1':
0.202500,
'OneTailedZfactor_DistanceCells_Texture_Entropy_CorrGreen_1':
-4.404815,
'OneTailedZfactor_DistanceCells_Texture_InfoMeas1_CorrGreen_1':
-4.508513,
'OneTailedZfactor_DistanceCells_Texture_SumAverage_CorrGreen_1':
-4.225356,
'OneTailedZfactor_DistanceCells_Texture_SumEntropy_CorrGreen_1':
-4.382768,
'OneTailedZfactor_DistanceCells_Texture_SumVariance_CorrGreen_1':
0.492125,
'OneTailedZfactor_DistanceCells_Texture_Variance_CorrGreen_1':
0.477360,
'OneTailedZfactor_Nuclei_Correlation_Correlation_CorrGreenCorrBlue':
0.563780,
'OneTailedZfactor_Nuclei_Math_Ratio1':
0.622059,
'OneTailedZfactor_Nuclei_Math_Ratio2':
-4.508284,
'OneTailedZfactor_Nuclei_Texture_SumAverage_CorrGreen_1':
0.426178,
'OneTailedZfactor_PropCells_AreaShape_Area':
-4.216674,
'OneTailedZfactor_PropCells_AreaShape_MajorAxisLength':
-4.119131,
'OneTailedZfactor_PropCells_AreaShape_MinorAxisLength':
-4.109793,
'OneTailedZfactor_PropCells_AreaShape_Perimeter':
-4.068050,
'OneTailedZfactor_PropCells_Correlation_Correlation_CorrGreenCorrBlue':
0.765440,
'OneTailedZfactor_PropCells_Texture_GaborX_CorrGreen_1':
0.114982,
'OneTailedZfactor_PropCells_Texture_InfoMeas2_CorrBlue_1':
0.108409,
'OneTailedZfactor_PropCells_Texture_SumVariance_CorrBlue_1':
0.191251,
'OneTailedZfactor_PropCells_Texture_SumVariance_CorrGreen_1':
0.559865,
'OneTailedZfactor_PropCells_Texture_Variance_CorrBlue_1':
0.254078,
'OneTailedZfactor_PropCells_Texture_Variance_CorrGreen_1':
0.556108,
'OneTailedZfactor_PropCytoplasm_AreaShape_Area':
-4.223021,
'OneTailedZfactor_PropCytoplasm_AreaShape_MinorAxisLength':
-4.095632,
'OneTailedZfactor_PropCytoplasm_AreaShape_Perimeter':
-4.068050,
'OneTailedZfactor_PropCytoplasm_Intensity_MedianIntensity_CorrGreen':
-4.194663,
'OneTailedZfactor_PropCytoplasm_Texture_Entropy_CorrGreen_1':
-4.443338,
'OneTailedZfactor_PropCytoplasm_Texture_GaborX_CorrGreen_1':
0.207265,
'OneTailedZfactor_PropCytoplasm_Texture_InfoMeas2_CorrGreen_1':
-4.297250,
'OneTailedZfactor_PropCytoplasm_Texture_SumEntropy_CorrGreen_1':
-4.525324,
'OneTailedZfactor_ThresholdedCells_Texture_Entropy_CorrBlue_1':
0.167795,
'OneTailedZfactor_ThresholdedCells_Texture_InfoMeas2_CorrBlue_1':
0.067560,
'OneTailedZfactor_ThresholdedCells_Texture_SumAverage_CorrBlue_1':
0.478527,
'OneTailedZfactor_ThresholdedCells_Texture_SumEntropy_CorrBlue_1':
0.155119,
'OneTailedZfactor_ThresholdedCells_Texture_SumVariance_CorrBlue_1':
0.535907,
'OneTailedZfactor_ThresholdedCells_Texture_SumVariance_CorrGreen_1':
0.572801,
'OneTailedZfactor_ThresholdedCells_Texture_Variance_CorrBlue_1':
0.423454,
'OneTailedZfactor_ThresholdedCells_Texture_Variance_CorrGreen_1':
0.440500,
'Vfactor_DistCytoplasm_Correlation_Correlation_CorrGreenCorrBlue':
0.500429,
'Vfactor_DistCytoplasm_Intensity_LowerQuartileIntensity_CorrGreen':
0.325675,
'Vfactor_DistCytoplasm_Intensity_MedianIntensity_CorrGreen':
0.323524,
'Vfactor_DistCytoplasm_Intensity_MinIntensityEdge_CorrGreen':
0.138487,
'Vfactor_DistCytoplasm_Intensity_MinIntensity_CorrGreen':
0.128157,
'Vfactor_DistCytoplasm_Math_Ratio1':
0.503610,
'Vfactor_DistCytoplasm_Math_Ratio2':
0.319610,
'Vfactor_DistCytoplasm_Texture_AngularSecondMoment_CorrGreen_1':
0.522880,
'Vfactor_DistCytoplasm_Texture_Entropy_CorrGreen_1':
0.504303,
'Vfactor_DistCytoplasm_Texture_InfoMeas2_CorrGreen_1':
0.289432,
'Vfactor_DistCytoplasm_Texture_InverseDifferenceMoment_CorrGreen_1':
0.234123,
'Vfactor_DistCytoplasm_Texture_SumAverage_CorrGreen_1':
0.591687,
'Vfactor_DistCytoplasm_Texture_SumEntropy_CorrGreen_1':
0.520356,
'Vfactor_DistCytoplasm_Texture_Variance_CorrGreen_1':
-0.007649,
'Vfactor_DistanceCells_Correlation_Correlation_CorrGreenCorrBlue':
0.761198,
'Vfactor_DistanceCells_Intensity_IntegratedIntensityEdge_CorrGreen':
0.234655,
'Vfactor_DistanceCells_Intensity_LowerQuartileIntensity_CorrGreen':
0.252240,
'Vfactor_DistanceCells_Intensity_MeanIntensityEdge_CorrGreen':
0.195125,
'Vfactor_DistanceCells_Intensity_MinIntensityEdge_CorrGreen':
0.138299,
'Vfactor_DistanceCells_Intensity_MinIntensity_CorrGreen':
0.126784,
'Vfactor_DistanceCells_Texture_AngularSecondMoment_CorrGreen_1':
0.342691,
'Vfactor_DistanceCells_Texture_Correlation_CorrGreen_1':
0.314396,
'Vfactor_DistanceCells_Texture_Entropy_CorrGreen_1':
0.311771,
'Vfactor_DistanceCells_Texture_InfoMeas1_CorrGreen_1':
0.410631,
'Vfactor_DistanceCells_Texture_InverseDifferenceMoment_CorrGreen_1':
0.170576,
'Vfactor_DistanceCells_Texture_SumAverage_CorrGreen_1':
0.223147,
'Vfactor_DistanceCells_Texture_SumEntropy_CorrGreen_1':
0.269519,
'Vfactor_DistanceCells_Texture_SumVariance_CorrGreen_1':
0.571528,
'Vfactor_DistanceCells_Texture_Variance_CorrGreen_1':
0.566272,
'Vfactor_Nuclei_Correlation_Correlation_CorrGreenCorrBlue':
0.705051,
'Vfactor_Nuclei_Math_Ratio1':
0.503610,
'Vfactor_Nuclei_Math_Ratio2':
0.319610,
'Vfactor_Nuclei_Texture_SumAverage_CorrGreen_1':
0.553708,
'Vfactor_PropCells_AreaShape_Area':
0.340093,
'Vfactor_PropCells_AreaShape_MajorAxisLength':
0.243838,
'Vfactor_PropCells_AreaShape_MinorAxisLength':
0.320691,
'Vfactor_PropCells_AreaShape_Perimeter':
0.238915,
'Vfactor_PropCells_Correlation_Correlation_CorrGreenCorrBlue':
0.723520,
'Vfactor_PropCells_Texture_GaborX_CorrGreen_1':
0.213161,
'Vfactor_PropCells_Texture_InfoMeas2_CorrBlue_1':
0.199791,
'Vfactor_PropCells_Texture_SumVariance_CorrBlue_1':
0.078959,
'Vfactor_PropCells_Texture_SumVariance_CorrGreen_1':
0.642844,
'Vfactor_PropCells_Texture_Variance_CorrBlue_1':
0.199105,
'Vfactor_PropCells_Texture_Variance_CorrGreen_1':
0.640818,
'Vfactor_PropCytoplasm_AreaShape_Area':
0.325845,
'Vfactor_PropCytoplasm_AreaShape_MinorAxisLength':
0.312258,
'Vfactor_PropCytoplasm_AreaShape_Perimeter':
0.238915,
'Vfactor_PropCytoplasm_Correlation_Correlation_CorrGreenCorrBlue':
0.337565,
'Vfactor_PropCytoplasm_Intensity_IntegratedIntensity_CorrGreen':
0.292900,
'Vfactor_PropCytoplasm_Intensity_LowerQuartileIntensity_CorrGreen':
0.175528,
'Vfactor_PropCytoplasm_Intensity_MedianIntensity_CorrGreen':
0.193308,
'Vfactor_PropCytoplasm_Texture_AngularSecondMoment_CorrGreen_1':
0.276152,
'Vfactor_PropCytoplasm_Texture_Entropy_CorrGreen_1':
0.239567,
'Vfactor_PropCytoplasm_Texture_GaborX_CorrGreen_1':
0.332380,
'Vfactor_PropCytoplasm_Texture_InfoMeas2_CorrGreen_1':
0.379141,
'Vfactor_PropCytoplasm_Texture_SumEntropy_CorrGreen_1':
0.337740,
'Vfactor_ThresholdedCells_Texture_AngularSecondMoment_CorrBlue_1':
0.334520,
'Vfactor_ThresholdedCells_Texture_AngularSecondMoment_CorrGreen_1':
0.192882,
'Vfactor_ThresholdedCells_Texture_Entropy_CorrBlue_1':
0.276245,
'Vfactor_ThresholdedCells_Texture_InfoMeas2_CorrBlue_1':
0.139166,
'Vfactor_ThresholdedCells_Texture_SumAverage_CorrBlue_1':
0.465237,
'Vfactor_ThresholdedCells_Texture_SumEntropy_CorrBlue_1':
0.355399,
'Vfactor_ThresholdedCells_Texture_SumVariance_CorrBlue_1':
0.453937,
'Vfactor_ThresholdedCells_Texture_SumVariance_CorrGreen_1':
0.564371,
'Vfactor_ThresholdedCells_Texture_Variance_CorrBlue_1':
0.360566,
'Vfactor_ThresholdedCells_Texture_Variance_CorrGreen_1':
0.548770,
'Zfactor_DistCytoplasm_Correlation_Correlation_CorrGreenCorrBlue':
0.531914,
'Zfactor_DistCytoplasm_Intensity_LowerQuartileIntensity_CorrGreen':
0.265558,
'Zfactor_DistCytoplasm_Intensity_MedianIntensity_CorrGreen':
0.178586,
'Zfactor_DistCytoplasm_Intensity_MinIntensityEdge_CorrGreen':
0.084566,
'Zfactor_DistCytoplasm_Intensity_MinIntensity_CorrGreen':
0.086476,
'Zfactor_DistCytoplasm_Math_Ratio1':
0.623284,
'Zfactor_DistCytoplasm_Math_Ratio2':
0.358916,
'Zfactor_DistCytoplasm_Texture_AngularSecondMoment_CorrGreen_1':
0.429510,
'Zfactor_DistCytoplasm_Texture_Entropy_CorrGreen_1':
0.508275,
'Zfactor_DistCytoplasm_Texture_InfoMeas2_CorrGreen_1':
0.068695,
'Zfactor_DistCytoplasm_Texture_InverseDifferenceMoment_CorrGreen_1':
0.347949,
'Zfactor_DistCytoplasm_Texture_SumAverage_CorrGreen_1':
0.646576,
'Zfactor_DistCytoplasm_Texture_SumEntropy_CorrGreen_1':
0.494276,
'Zfactor_DistCytoplasm_Texture_Variance_CorrGreen_1':
0.179011,
'Zfactor_DistanceCells_Correlation_Correlation_CorrGreenCorrBlue':
0.824686,
'Zfactor_DistanceCells_Intensity_IntegratedIntensityEdge_CorrGreen':
0.027644,
'Zfactor_DistanceCells_Intensity_LowerQuartileIntensity_CorrGreen':
0.088491,
'Zfactor_DistanceCells_Intensity_MeanIntensityEdge_CorrGreen':
0.065056,
'Zfactor_DistanceCells_Intensity_MinIntensityEdge_CorrGreen':
0.089658,
'Zfactor_DistanceCells_Intensity_MinIntensity_CorrGreen':
0.078017,
'Zfactor_DistanceCells_Texture_AngularSecondMoment_CorrGreen_1':
0.238131,
'Zfactor_DistanceCells_Texture_Correlation_CorrGreen_1':
0.301107,
'Zfactor_DistanceCells_Texture_Entropy_CorrGreen_1':
0.251143,
'Zfactor_DistanceCells_Texture_InfoMeas1_CorrGreen_1':
0.564957,
'Zfactor_DistanceCells_Texture_InverseDifferenceMoment_CorrGreen_1':
0.302767,
'Zfactor_DistanceCells_Texture_SumAverage_CorrGreen_1':
0.036459,
'Zfactor_DistanceCells_Texture_SumEntropy_CorrGreen_1':
0.159798,
'Zfactor_DistanceCells_Texture_SumVariance_CorrGreen_1':
0.516938,
'Zfactor_DistanceCells_Texture_Variance_CorrGreen_1':
0.501186,
'Zfactor_Nuclei_Correlation_Correlation_CorrGreenCorrBlue':
0.691408,
'Zfactor_Nuclei_Math_Ratio1':
0.623284,
'Zfactor_Nuclei_Math_Ratio2':
0.358916,
'Zfactor_Nuclei_Texture_SumAverage_CorrGreen_1':
0.587347,
'Zfactor_PropCells_AreaShape_Area':
0.132425,
'Zfactor_PropCells_AreaShape_MajorAxisLength':
0.034809,
'Zfactor_PropCells_AreaShape_MinorAxisLength':
0.113864,
'Zfactor_PropCells_AreaShape_Perimeter':
0.005984,
'Zfactor_PropCells_Correlation_Correlation_CorrGreenCorrBlue':
0.717632,
'Zfactor_PropCells_Texture_GaborX_CorrGreen_1':
0.251023,
'Zfactor_PropCells_Texture_InfoMeas2_CorrBlue_1':
0.149719,
'Zfactor_PropCells_Texture_SumVariance_CorrBlue_1':
0.102050,
'Zfactor_PropCells_Texture_SumVariance_CorrGreen_1':
0.611960,
'Zfactor_PropCells_Texture_Variance_CorrBlue_1':
0.197090,
'Zfactor_PropCells_Texture_Variance_CorrGreen_1':
0.614879,
'Zfactor_PropCytoplasm_AreaShape_Area':
0.205042,
'Zfactor_PropCytoplasm_AreaShape_MinorAxisLength':
0.072682,
'Zfactor_PropCytoplasm_AreaShape_Perimeter':
0.005984,
'Zfactor_PropCytoplasm_Correlation_Correlation_CorrGreenCorrBlue':
0.272017,
'Zfactor_PropCytoplasm_Intensity_IntegratedIntensity_CorrGreen':
0.115327,
'Zfactor_PropCytoplasm_Intensity_LowerQuartileIntensity_CorrGreen':
0.141850,
'Zfactor_PropCytoplasm_Intensity_MedianIntensity_CorrGreen':
0.105803,
'Zfactor_PropCytoplasm_Texture_AngularSecondMoment_CorrGreen_1':
0.107640,
'Zfactor_PropCytoplasm_Texture_Entropy_CorrGreen_1':
0.067896,
'Zfactor_PropCytoplasm_Texture_GaborX_CorrGreen_1':
0.136688,
'Zfactor_PropCytoplasm_Texture_InfoMeas2_CorrGreen_1':
0.334749,
'Zfactor_PropCytoplasm_Texture_SumEntropy_CorrGreen_1':
0.208829,
'Zfactor_ThresholdedCells_Texture_AngularSecondMoment_CorrBlue_1':
0.263467,
'Zfactor_ThresholdedCells_Texture_AngularSecondMoment_CorrGreen_1':
0.124355,
'Zfactor_ThresholdedCells_Texture_Entropy_CorrBlue_1':
0.236433,
'Zfactor_ThresholdedCells_Texture_InfoMeas2_CorrBlue_1':
0.125845,
'Zfactor_ThresholdedCells_Texture_SumAverage_CorrBlue_1':
0.449333,
'Zfactor_ThresholdedCells_Texture_SumEntropy_CorrBlue_1':
0.323243,
'Zfactor_ThresholdedCells_Texture_SumVariance_CorrBlue_1':
0.507477,
'Zfactor_ThresholdedCells_Texture_SumVariance_CorrGreen_1':
0.599000,
'Zfactor_ThresholdedCells_Texture_Variance_CorrBlue_1':
0.361424,
'Zfactor_ThresholdedCells_Texture_Variance_CorrGreen_1':
0.481393
}
temp_dir = tempfile.mkdtemp()
try:
cpprefs.set_headless()
cpprefs.set_default_output_directory(temp_dir)
print "Writing output to %s" % temp_dir
path = os.path.split(__file__)[0]
measurements = loadmat(os.path.join(path,
'calculatestatistics.mat'),
struct_as_record=True)
measurements = measurements['m']
image_set_list = cpi.ImageSetList()
image_set = image_set_list.get_image_set(0)
m = cpmeas.Measurements()
doses = [
0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 0, 0, 1, 2, 3, 4, 5, 6, 7,
8, 9, 10, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 0, 0, 1, 2, 3,
4, 5, 6, 7, 8, 9, 10, 10, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 10,
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 10, 0, 1, 2, 3, 4, 5, 6, 7, 8,
9, 0, 10, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0
]
for i, dose in enumerate(doses):
m.add_image_measurement("Dose", dose)
for object_name in measurements.dtype.fields:
omeasurements = measurements[object_name][0, 0]
for feature_name in omeasurements.dtype.fields:
data = omeasurements[feature_name][0, 0][0, i]
m.add_measurement(object_name, feature_name, data)
if i < len(doses) - 1:
m.next_image_set()
pipeline = cpp.Pipeline()
module = C.CalculateStatistics()
module.grouping_values.value = "Dose"
module.dose_values[0].log_transform.value = False
module.dose_values[0].measurement.value = "Dose"
module.dose_values[0].wants_save_figure.value = True
module.dose_values[0].figure_name.value = "EC49_"
module.module_num = 1
pipeline.add_module(module)
def callback(caller, event):
self.assertFalse(isinstance(event, cpp.RunExceptionEvent))
workspace = cpw.Workspace(pipeline, module, image_set,
cpo.ObjectSet(), m, image_set_list)
module.post_run(workspace)
for feature_name in m.get_feature_names(cpmeas.EXPERIMENT):
if not expected.has_key(feature_name):
print "Missing measurement: %s" % feature_name
continue
value = m.get_experiment_measurement(feature_name)
e_value = expected[feature_name]
diff = abs(value - e_value) * 2 / abs(value + e_value)
self.assertTrue(
diff < .05, "%s: Matlab: %f, Python: %f diff: %f" %
(feature_name, e_value, value, diff))
if diff > .01:
print(
"Warning: > 1%% difference for %s: Matlab: %f, Python: %f diff: %f"
% (feature_name, e_value, value, diff))
if feature_name.startswith("EC50"):
filename = "EC49_" + feature_name[5:] + ".pdf"
self.assertTrue(
os.path.isfile(os.path.join(temp_dir, filename)))
finally:
try:
workspace.measurements.hdf5_dict.hdf5_file.close()
except:
pass
for filename in os.listdir(temp_dir):
path = os.path.join(temp_dir, filename)
os.remove(path)
os.rmdir(temp_dir)
def interface(self,
start_signal,
image_set_start=1,
image_set_end=None,
overwrite=True):
'''Top-half thread for running an analysis. Sets up grouping for jobs,
deals with returned measurements, reports status periodically.
start_signal- signal this semaphore when jobs are ready.
image_set_start - beginning image set number to process
image_set_end - last image set number to process
overwrite - whether to recompute imagesets that already have data in initial_measurements.
'''
posted_analysis_started = False
acknowledged_thread_start = False
measurements = None
workspace = None
try:
# listen for pipeline events, and pass them upstream
self.pipeline.add_listener(lambda pipe, evt: self.post_event(evt))
initial_measurements = None
if self.output_path is None:
# Caller wants a temporary measurements file.
fd, filename = tempfile.mkstemp(".h5")
try:
fd = os.fdopen(fd, "wb")
fd.write(self.initial_measurements_buf)
fd.close()
initial_measurements = cpmeas.Measurements(
filename=filename, mode="r")
measurements = cpmeas.Measurements(
image_set_start = None,
copy = initial_measurements,
mode = "a")
finally:
if initial_measurements is not None:
initial_measurements.close()
os.unlink(filename)
else:
with open(self.output_path, "wb") as fd:
fd.write(self.initial_measurements_buf)
measurements = cpmeas.Measurements(image_set_start=None,
filename=self.output_path,
mode="a")
# The shared dicts are needed in jobserver()
self.shared_dicts = [m.get_dictionary() for m in self.pipeline.modules()]
workspace = cpw.Workspace(self.pipeline, None, None, None,
measurements, cpimage.ImageSetList())
if image_set_end is None:
image_set_end = measurements.get_image_numbers()[-1]
image_sets_to_process = filter(
lambda x: x >= image_set_start and x <= image_set_end,
measurements.get_image_numbers())
self.post_event(AnalysisStarted())
posted_analysis_started = True
# reset the status of every image set that needs to be processed
has_groups = measurements.has_groups()
if self.pipeline.requires_aggregation():
overwrite = True
if has_groups and not overwrite:
if not measurements.has_feature(cpmeas.IMAGE, self.STATUS):
overwrite = True
else:
group_status = {}
for image_number in measurements.get_image_numbers():
group_number = measurements[
cpmeas.IMAGE, cpmeas.GROUP_NUMBER, image_number]
status = measurements[cpmeas.IMAGE, self.STATUS,
image_number]
if status != self.STATUS_DONE:
group_status[group_number] = self.STATUS_UNPROCESSED
elif group_number not in group_status:
group_status[group_number] = self.STATUS_DONE
new_image_sets_to_process = []
for image_set_number in image_sets_to_process:
needs_reset = False
if (overwrite or
(not measurements.has_measurements(
cpmeas.IMAGE, self.STATUS, image_set_number)) or
(measurements[cpmeas.IMAGE, self.STATUS, image_set_number]
!= self.STATUS_DONE)):
needs_reset = True
elif has_groups:
group_number = measurements[
cpmeas.IMAGE, cpmeas.GROUP_NUMBER, image_set_number]
if group_status[group_number] != self.STATUS_DONE:
needs_reset = True
if needs_reset:
measurements[cpmeas.IMAGE, self.STATUS, image_set_number] =\
self.STATUS_UNPROCESSED
new_image_sets_to_process.append(image_set_number)
image_sets_to_process = new_image_sets_to_process
# Find image groups. These are written into measurements prior to
# analysis. Groups are processed as a single job.
if has_groups or self.pipeline.requires_aggregation():
worker_runs_post_group = True
job_groups = {}
for image_set_number in image_sets_to_process:
group_number = measurements[cpmeas.IMAGE,
cpmeas.GROUP_NUMBER,
image_set_number]
group_index = measurements[cpmeas.IMAGE,
cpmeas.GROUP_INDEX,
image_set_number]
job_groups[group_number] = job_groups.get(group_number, []) + [(group_index, image_set_number)]
job_groups = [[isn for _, isn in sorted(job_groups[group_number])]
for group_number in sorted(job_groups)]
else:
worker_runs_post_group = False # prepare_group will be run in worker, but post_group is below.
job_groups = [[image_set_number] for image_set_number in image_sets_to_process]
# XXX - check that any constructed groups are complete, i.e.,
# image_set_start and image_set_end shouldn't carve them up.
if not worker_runs_post_group:
# put the first job in the queue, then wait for the first image to
# finish (see the check of self.finish_queue below) to post the rest.
# This ensures that any shared data from the first imageset is
# available to later imagesets.
self.work_queue.put((job_groups[0],
worker_runs_post_group,
True))
waiting_for_first_imageset = True
del job_groups[0]
else:
waiting_for_first_imageset = False
for job in job_groups:
self.work_queue.put((job, worker_runs_post_group, False))
job_groups = []
start_signal.release()
acknowledged_thread_start = True
# We loop until every image is completed, or an outside event breaks the loop.
while not self.cancelled:
# gather measurements
while not self.received_measurements_queue.empty():
image_numbers, buf = self.received_measurements_queue.get()
image_numbers = [int(i) for i in image_numbers]
recd_measurements = cpmeas.load_measurements_from_buffer(buf)
measurements.copy_relationships(recd_measurements)
for o in recd_measurements.get_object_names():
if o == cpmeas.EXPERIMENT:
continue # Written during prepare_run / post_run
for feature in recd_measurements.get_feature_names(o):
measurements[o, feature, image_numbers] \
= recd_measurements[o, feature, image_numbers]
for image_set_number in image_numbers:
measurements[cpmeas.IMAGE, self.STATUS, image_set_number] = self.STATUS_DONE
recd_measurements.close()
del recd_measurements
# check for jobs in progress
while not self.in_process_queue.empty():
image_set_numbers = self.in_process_queue.get()
for image_set_number in image_set_numbers:
measurements[cpmeas.IMAGE, self.STATUS, int(image_set_number)] = self.STATUS_IN_PROCESS
# check for finished jobs that haven't returned measurements, yet
while not self.finished_queue.empty():
finished_req = self.finished_queue.get()
measurements[cpmeas.IMAGE, self.STATUS, int(finished_req.image_set_number)] = self.STATUS_FINISHED_WAITING
if waiting_for_first_imageset:
assert isinstance(finished_req,
ImageSetSuccessWithDictionary)
self.shared_dicts = finished_req.shared_dicts
waiting_for_first_imageset = False
assert len(self.shared_dicts) == len(self.pipeline.modules())
# if we had jobs waiting for the first image set to finish,
# queue them now that the shared state is available.
for job in job_groups:
self.work_queue.put((job, worker_runs_post_group, False))
finished_req.reply(Ack())
# check progress and report
counts = collections.Counter(measurements[cpmeas.IMAGE, self.STATUS, image_set_number]
for image_set_number in image_sets_to_process)
self.post_event(AnalysisProgress(counts))
# Are we finished?
if counts[self.STATUS_DONE] == len(image_sets_to_process):
last_image_number = measurements.get_image_numbers()[-1]
measurements.image_set_number = last_image_number
if not worker_runs_post_group:
self.pipeline.post_group(workspace, {})
workspace = cpw.Workspace(self.pipeline,
None, None, None,
measurements, None, None)
workspace.post_run_display_handler = \
self.post_run_display_handler
self.pipeline.post_run(workspace)
break
measurements.flush()
# not done, wait for more work
with self.interface_work_cv:
while (self.paused or
((not self.cancelled) and
self.in_process_queue.empty() and
self.finished_queue.empty() and
self.received_measurements_queue.empty())):
self.interface_work_cv.wait() # wait for a change of status or work to arrive
finally:
# Note - the measurements file is owned by the queue consumer
# after this post_event.
#
if not acknowledged_thread_start:
start_signal.release()
if posted_analysis_started:
was_cancelled = self.cancelled
self.post_event(AnalysisFinished(measurements, was_cancelled))
self.stop_workers()
self.analysis_id = False # this will cause the jobserver thread to exit
def do_job(self, job):
'''Handle a work request to its completion
job - WorkRequest
'''
import cellprofiler.pipeline as cpp
job_measurements = []
try:
send_dictionary = job.wants_dictionary
logger.info("Starting job")
# Fetch the pipeline and preferences for this analysis if we don't have it
current_pipeline, current_preferences = \
self.pipelines_and_preferences.get(
self.current_analysis_id, (None, None))
if not current_pipeline:
logger.debug("Fetching pipeline and preferences")
rep = self.send(
PipelinePreferencesRequest(self.current_analysis_id))
logger.debug("Received pipeline and preferences response")
preferences_dict = rep.preferences
# update preferences to match remote values
cpprefs.set_preferences_from_dict(preferences_dict)
logger.debug("Loading pipeline")
pipeline_blob = rep.pipeline_blob.tostring()
current_pipeline = cpp.Pipeline()
current_pipeline.loadtxt(StringIO.StringIO(pipeline_blob),
raise_on_error=True)
logger.debug("Pipeline loaded")
current_pipeline.add_listener(
self.pipeline_listener.handle_event)
current_preferences = rep.preferences
self.pipelines_and_preferences[self.current_analysis_id] = (
current_pipeline, current_preferences)
else:
# update preferences to match remote values
cpprefs.set_preferences_from_dict(current_preferences)
# Reset the listener's state
self.pipeline_listener.reset()
logger.debug("Getting initial measurements")
# Fetch the path to the intial measurements if needed.
current_measurements = self.initial_measurements.get(
self.current_analysis_id)
if current_measurements is None:
logger.debug("Sending initial measurements request")
rep = self.send(
InitialMeasurementsRequest(self.current_analysis_id))
logger.debug("Got initial measurements")
current_measurements = \
self.initial_measurements[self.current_analysis_id] = \
cpmeas.load_measurements_from_buffer(rep.buf)
else:
logger.debug("Has initial measurements")
# Make a copy of the measurements for writing during this job
current_measurements = cpmeas.Measurements(
copy=current_measurements)
all_measurements.add(current_measurements)
job_measurements.append(current_measurements)
successful_image_set_numbers = []
image_set_numbers = job.image_set_numbers
worker_runs_post_group = job.worker_runs_post_group
logger.info("Doing job: " + ",".join(map(str, image_set_numbers)))
self.pipeline_listener.image_set_number = image_set_numbers[0]
if not worker_runs_post_group:
# Get the shared state from the first imageset in this run.
shared_dicts = self.send(
SharedDictionaryRequest(
self.current_analysis_id)).dictionaries
assert len(shared_dicts) == len(current_pipeline.modules())
for module, new_dict in zip(current_pipeline.modules(),
shared_dicts):
module.set_dictionary_for_worker(new_dict)
# Run prepare group if this is the first image in the group. We do
# this here (even if there's no grouping in the pipeline) to ensure
# that any changes to the modules' shared state dictionaries get
# propagated correctly.
should_process = True
if current_measurements[cpmeas.IMAGE, cpmeas.GROUP_INDEX,
image_set_numbers[0]] == 1:
workspace = cpw.Workspace(current_pipeline, None, None, None,
current_measurements, None, None)
if not current_pipeline.prepare_group(
workspace, current_measurements.get_grouping_keys(),
image_set_numbers):
# exception handled elsewhere, possibly cancelling this run.
should_process = False
del workspace
# process the images
if should_process:
abort = False
for image_set_number in image_set_numbers:
try:
self.pipeline_listener.image_set_number = image_set_number
last_workspace = current_pipeline.run_image_set(
current_measurements, image_set_number,
self.interaction_handler, self.display_handler,
self.cancel_handler)
if self.pipeline_listener.should_abort:
abort = True
break
elif self.pipeline_listener.should_skip:
# Report skipped image sets as successful so that
# analysis can complete.
# Report their measurements because some modules
# may have provided measurements before skipping.
pass
successful_image_set_numbers.append(image_set_number)
# Send an indication that the image set finished successfully.
if send_dictionary:
# The jobserver would like a copy of our modules'
# run_state dictionaries.
dicts = [
m.get_dictionary_for_worker()
for m in current_pipeline.modules()
]
req = ImageSetSuccessWithDictionary(
self.current_analysis_id,
image_set_number=image_set_number,
shared_dicts=dicts)
else:
req = ImageSetSuccess(
self.current_analysis_id,
image_set_number=image_set_number)
rep = self.send(req)
except cpp.CancelledException:
logging.info("Aborting job after cancellation")
abort = True
except Exception:
try:
logging.error("Error in pipeline", exc_info=True)
if self.handle_exception(
image_set_number=image_set_number
) == ED_STOP:
abort = True
break
except:
logging.error(
"Error in handling of pipeline exception",
exc_info=True)
# this is bad. We can't handle nested exceptions
# remotely so we just fail on this run.
abort = True
if abort:
current_measurements.close()
job_measurements.remove(current_measurements)
return
if worker_runs_post_group:
last_workspace.interaction_handler = \
self.interaction_handler
last_workspace.cancel_handler = self.cancel_handler
last_workspace.post_group_display_handler = \
self.post_group_display_handler
# There might be an exception in this call, but it will be
# handled elsewhere, and there's nothing we can do for it
# here.
current_pipeline.post_group(
last_workspace,
current_measurements.get_grouping_keys())
del last_workspace
# send measurements back to server
req = MeasurementsReport(self.current_analysis_id,
buf=current_measurements.file_contents(),
image_set_numbers=image_set_numbers)
rep = self.send(req)
except cpp.CancelledException:
# Main thread received shutdown signal
raise
except Exception:
logging.error("Error in worker", exc_info=True)
if self.handle_exception() == ED_STOP:
raise cpp.CancelledException(
"Cancelling after user-requested stop")
finally:
# Clean up any measurements owned by us
for m in job_measurements:
m.close()
def make_workspace(self,
control_points,
lengths,
radii,
image,
mask=None,
auximage=None):
'''Create a workspace containing the control point measurements
control_points - an n x 2 x m array where n is the # of control points,
and m is the number of objects.
lengths - the length of each object
radii - the radii_from_training defining the radius at each control pt
image - the image to be straightened
mask - the mask associated with the image (default = no mask)
auximage - a second image to be straightnened (default = no second image)
'''
module = S.StraightenWorms()
module.objects_name.value = OBJECTS_NAME
module.straightened_objects_name.value = STRAIGHTENED_OBJECTS_NAME
module.images[0].image_name.value = IMAGE_NAME
module.images[
0].straightened_image_name.value = STRAIGHTENED_IMAGE_NAME
module.flip_image.value = IMAGE_NAME
module.module_num = 1
# Trick the module into thinking it's read the data file
class P:
def __init__(self):
self.radii_from_training = radii
module.training_set_directory.dir_choice = cps.URL_FOLDER_NAME
module.training_set_directory.custom_path = "http://www.cellprofiler.org"
module.training_set_file_name.value = "TrainingSet.xml"
module.training_params = {"TrainingSet.xml": (P(), "URL")}
pipeline = cpp.Pipeline()
pipeline.add_module(module)
def callback(caller, event):
self.assertFalse(isinstance(event, cpp.RunExceptionEvent))
pipeline.add_listener(callback)
m = cpmeas.Measurements()
for i, (y, x) in enumerate(control_points):
for v, f in ((x, S.F_CONTROL_POINT_X), (y, S.F_CONTROL_POINT_Y)):
feature = "_".join((S.C_WORM, f, str(i + 1)))
m.add_measurement(OBJECTS_NAME, feature, v)
feature = "_".join((S.C_WORM, S.F_LENGTH))
m.add_measurement(OBJECTS_NAME, feature, lengths)
image_set_list = cpi.ImageSetList()
image_set = image_set_list.get_image_set(0)
image_set.add(IMAGE_NAME, cpi.Image(image, mask))
if auximage is not None:
image_set.add(AUX_IMAGE_NAME, cpi.Image(auximage))
module.add_image()
module.images[1].image_name.value = AUX_IMAGE_NAME
module.images[
1].straightened_image_name.value = AUX_STRAIGHTENED_IMAGE_NAME
object_set = cpo.ObjectSet()
objects = cpo.Objects()
labels = np.zeros(image.shape, int)
for i in range(control_points.shape[2]):
if lengths[i] == 0:
continue
self.rebuild_worm_from_control_points_approx(
control_points[:, :, i], radii, labels, i + 1)
objects.segmented = labels
object_set.add_objects(objects, OBJECTS_NAME)
workspace = cpw.Workspace(pipeline, module, image_set, object_set, m,
image_set_list)
return workspace, module
