def pipeline_info(self, session_id, message_type, message):
"""Handle the pipeline info message"""
logger.info("Handling pipeline info request")
pipeline_txt = message.pop(0).bytes
pipeline = cellprofiler.pipeline.Pipeline()
try:
pipeline.loadtxt(StringIO(pipeline_txt))
except Exception as e:
logger.warning(
"Failed to load pipeline: sending pipeline exception",
exc_info=1)
self.raise_pipeline_exception(session_id, str(e))
return
input_modules, other_modules = self.split_pipeline(pipeline)
channels = self.find_channels(input_modules)
type_names, measurements = self.find_measurements(
other_modules, pipeline)
body = json.dumps([channels, type_names, measurements])
msg_out = [
zmq.Frame(session_id),
zmq.Frame(),
zmq.Frame(PIPELINE_INFO_REPLY_1),
zmq.Frame(body),
]
self.socket.send_multipart(msg_out)
def clean_pipeline(self, session_id, message_type, message):
"""Handle the clean pipeline request message"""
logger.info("Handling clean pipeline request")
pipeline_txt = message.pop(0).bytes
module_names = json.loads(message.pop(0).bytes)
pipeline = cellprofiler.pipeline.Pipeline()
try:
pipeline.loadtxt(StringIO(pipeline_txt))
except Exception as e:
logger.warning(
"Failed to load pipeline: sending pipeline exception",
exc_info=1)
self.raise_pipeline_exception(session_id, str(e))
return
to_remove = []
for module in pipeline.modules(exclude_disabled=False):
if module.module_name in module_names:
to_remove.insert(0, module)
for module in to_remove:
pipeline.remove_module(module.module_num)
pipeline_fd = StringIO()
pipeline.savetxt(pipeline_fd, save_image_plane_details=False)
msg_out = [
zmq.Frame(session_id),
zmq.Frame(),
zmq.Frame(CLEAN_PIPELINE_REPLY_1),
zmq.Frame(pipeline_fd.getvalue()),
]
self.socket.send_multipart(msg_out)
def get_measurements_for_good_pipeline(nimages=1, group_numbers=None):
"""Get an appropriately initialized measurements structure for the good pipeline"""
path = os.path.join(tests.modules.example_images_directory(),
"ExampleSBSImages")
m = cellprofiler.measurement.Measurements()
if group_numbers is None:
group_numbers = [1] * nimages
group_indexes = [1]
last_group_number = group_numbers[0]
group_index = 1
for group_number in group_numbers:
if group_number == last_group_number:
group_index += 1
else:
group_index = 1
group_indexes.append(group_index)
for i in range(1, nimages + 1):
filename = "Channel2-%02d-%s-%02d.tif" % (
i,
"ABCDEFGH"[int((i - 1) / 12)],
((i - 1) % 12) + 1,
)
url = cellprofiler.modules.loadimages.pathname2url(
os.path.join(path, filename))
m[cellprofiler.measurement.IMAGE,
cellprofiler.measurement.C_FILE_NAME + "_DNA", i, ] = filename
m[cellprofiler.measurement.IMAGE,
cellprofiler.measurement.C_PATH_NAME + "_DNA", i, ] = path
m[cellprofiler.measurement.IMAGE,
cellprofiler.measurement.C_URL + "_DNA", i] = url
m[cellprofiler.measurement.IMAGE,
cellprofiler.measurement.GROUP_NUMBER, i] = group_numbers[i - 1]
m[cellprofiler.measurement.IMAGE, cellprofiler.measurement.GROUP_INDEX,
i] = group_indexes[i - 1]
jblob = javabridge.run_script(
"""
importPackage(Packages.org.cellprofiler.imageset);
importPackage(Packages.org.cellprofiler.imageset.filter);
var imageFile=new ImageFile(new java.net.URI(url));
var imageFileDetails = new ImageFileDetails(imageFile);
var imageSeries=new ImageSeries(imageFile, 0);
var imageSeriesDetails = new ImageSeriesDetails(imageSeries, imageFileDetails);
var imagePlane=new ImagePlane(imageSeries, 0, ImagePlane.ALWAYS_MONOCHROME);
var ipd = new ImagePlaneDetails(imagePlane, imageSeriesDetails);
var stack = ImagePlaneDetailsStack.makeMonochromeStack(ipd);
var stacks = java.util.Collections.singletonList(stack);
var keys = java.util.Collections.singletonList(imageNumber);
var imageSet = new ImageSet(stacks, keys);
imageSet.compress(java.util.Collections.singletonList("DNA"), null);
""",
dict(url=url, imageNumber=str(i)),
)
blob = javabridge.get_env().get_byte_array_elements(jblob)
m[cellprofiler.measurement.IMAGE,
cellprofiler.modules.namesandtypes.M_IMAGE_SET, i,
blob.dtype, ] = blob
pipeline = cellprofiler.pipeline.Pipeline()
pipeline.loadtxt(six.moves.StringIO(GOOD_PIPELINE))
pipeline.write_pipeline_measurement(m)
return m
def test_load_v9():
with open("./tests/resources/modules/threshold/v9.pipeline", "r") as fd:
data = fd.read()
fd = io.StringIO(data)
pipeline = cellprofiler.pipeline.Pipeline()
pipeline.loadtxt(fd)
module = pipeline.modules()[4]
assert isinstance(module, cellprofiler.modules.threshold.Threshold)
assert module.x_name == "DNA"
assert module.y_name == "Threshold"
assert module.threshold_scope == cellprofiler.modules.threshold.TS_GLOBAL
assert module.global_operation.value == cellprofiler.modules.threshold.TM_LI
assert module.threshold_smoothing_scale == 0
assert module.threshold_correction_factor == 1.0
assert module.threshold_range.min == 0.0
assert module.threshold_range.max == 1.0
assert module.manual_threshold == 0.0
assert module.thresholding_measurement == "None"
assert module.two_class_otsu == cellprofiler.modules.threshold.O_TWO_CLASS
assert (module.assign_middle_to_foreground ==
cellprofiler.modules.threshold.O_FOREGROUND)
assert module.adaptive_window_size == 50
module = pipeline.modules()[5]
assert module.threshold_scope.value == cellprofiler.modules.threshold.TS_GLOBAL
assert module.global_operation.value == cellprofiler.modules.threshold.TM_MANUAL
module = pipeline.modules()[6]
assert module.threshold_scope.value == cellprofiler.modules.threshold.TS_GLOBAL
assert module.global_operation.value == centrosome.threshold.TM_ROBUST_BACKGROUND
module = pipeline.modules()[7]
assert module.threshold_scope.value == cellprofiler.modules.threshold.TS_GLOBAL
assert module.global_operation.value == cellprofiler.modules.threshold.TM_LI
def test_load_v8():
with open("./tests/resources/modules/createbatchfiles/v8.pipeline", "r") as fd:
data = fd.read()
pipeline = cellprofiler.pipeline.Pipeline()
pipeline.loadtxt(six.moves.StringIO(data))
assert len(pipeline.modules()) == 1
module = pipeline.modules()[0]
assert module.wants_default_output_directory.value
assert module.custom_output_directory.value == "/Users/cellprofiler"
assert not module.remote_host_is_windows.value
assert module.mappings[0].local_directory.value == "/Users/cellprofiler/Pictures"
assert module.mappings[0].remote_directory.value == "/Remote/cellprofiler/Pictures"
def test_load_v4():
with open("./tests/resources/modules/splitormergeobjects/v4.pipeline", "r") as fd:
data = fd.read()
pipeline = cellprofiler.pipeline.Pipeline()
def callback(caller, event):
assert not isinstance(event, cellprofiler.pipeline.LoadExceptionEvent)
pipeline.add_listener(callback)
pipeline.loadtxt(six.moves.StringIO(data))
assert len(pipeline.modules()) == 2
module = pipeline.modules()[0]
assert isinstance(
module, cellprofiler.modules.splitormergeobjects.SplitOrMergeObjects
)
assert module.objects_name == "blobs"
assert module.output_objects_name == "RelabeledBlobs"
assert (
module.relabel_option == cellprofiler.modules.splitormergeobjects.OPTION_MERGE
)
assert module.distance_threshold == 2
assert not module.wants_image
assert module.image_name == "Guide"
assert module.minimum_intensity_fraction == 0.8
assert (
module.where_algorithm
== cellprofiler.modules.splitormergeobjects.CA_CLOSEST_POINT
)
assert module.merge_option == cellprofiler.modules.splitormergeobjects.UNIFY_PARENT
assert module.parent_object == "Nuclei"
assert (
module.merging_method == cellprofiler.modules.splitormergeobjects.UM_CONVEX_HULL
)
module = pipeline.modules()[1]
assert (
module.relabel_option == cellprofiler.modules.splitormergeobjects.OPTION_SPLIT
)
assert module.wants_image
assert (
module.where_algorithm == cellprofiler.modules.splitormergeobjects.CA_CENTROIDS
)
assert (
module.merge_option == cellprofiler.modules.splitormergeobjects.UNIFY_DISTANCE
)
assert (
module.merging_method
== cellprofiler.modules.splitormergeobjects.UM_DISCONNECTED
)
def get_measurements_for_good_pipeline(nimages=1,
group_numbers=None):
'''Get an appropriately initialized measurements structure for the good pipeline'''
path = os.path.join(tests.modules.example_images_directory(), "ExampleSBSImages")
m = cellprofiler.measurement.Measurements()
if group_numbers is None:
group_numbers = [1] * nimages
group_indexes = [1]
last_group_number = group_numbers[0]
group_index = 1
for group_number in group_numbers:
if group_number == last_group_number:
group_index += 1
else:
group_index = 1
group_indexes.append(group_index)
for i in range(1, nimages + 1):
filename = ("Channel2-%02d-%s-%02d.tif" %
(i, "ABCDEFGH"[int((i - 1) / 12)], ((i - 1) % 12) + 1))
url = cellprofiler.modules.loadimages.pathname2url(os.path.join(path, filename))
m[cellprofiler.measurement.IMAGE, cellprofiler.measurement.C_FILE_NAME + "_DNA", i] = filename
m[cellprofiler.measurement.IMAGE, cellprofiler.measurement.C_PATH_NAME + "_DNA", i] = path
m[cellprofiler.measurement.IMAGE, cellprofiler.measurement.C_URL + "_DNA", i] = url
m[cellprofiler.measurement.IMAGE, cellprofiler.measurement.GROUP_NUMBER, i] = group_numbers[i - 1]
m[cellprofiler.measurement.IMAGE, cellprofiler.measurement.GROUP_INDEX, i] = group_indexes[i - 1]
jblob = javabridge.run_script("""
importPackage(Packages.org.cellprofiler.imageset);
importPackage(Packages.org.cellprofiler.imageset.filter);
var imageFile=new ImageFile(new java.net.URI(url));
var imageFileDetails = new ImageFileDetails(imageFile);
var imageSeries=new ImageSeries(imageFile, 0);
var imageSeriesDetails = new ImageSeriesDetails(imageSeries, imageFileDetails);
var imagePlane=new ImagePlane(imageSeries, 0, ImagePlane.ALWAYS_MONOCHROME);
var ipd = new ImagePlaneDetails(imagePlane, imageSeriesDetails);
var stack = ImagePlaneDetailsStack.makeMonochromeStack(ipd);
var stacks = java.util.Collections.singletonList(stack);
var keys = java.util.Collections.singletonList(imageNumber);
var imageSet = new ImageSet(stacks, keys);
imageSet.compress(java.util.Collections.singletonList("DNA"), null);
""", dict(url=url, imageNumber=str(i)))
blob = javabridge.get_env().get_byte_array_elements(jblob)
m[cellprofiler.measurement.IMAGE, cellprofiler.modules.namesandtypes.M_IMAGE_SET, i, blob.dtype] = blob
pipeline = cellprofiler.pipeline.Pipeline()
pipeline.loadtxt(cStringIO.StringIO(GOOD_PIPELINE))
pipeline.write_pipeline_measurement(m)
return m
def test_load_v3():
with open("./tests/resources/modules/identifytertiaryobjects/v3.pipeline",
"r") as fd:
data = fd.read()
pipeline = cellprofiler.pipeline.Pipeline()
def callback(caller, event):
assert not isinstance(event, cellprofiler.pipeline.LoadExceptionEvent)
pipeline.add_listener(callback)
pipeline.loadtxt(six.moves.StringIO(data))
module = pipeline.modules()[0]
assert module.secondary_objects_name.value == "IdentifySecondaryObjects"
assert module.primary_objects_name.value == "IdentifyPrimaryObjects"
assert module.subregion_objects_name.value == "IdentifyTertiaryObjects"
assert module.shrink_primary.value
def prepare_run(self, message, session_id, grouping_allowed=False):
"""Prepare a pipeline and measurements to run
message - the run-request or run-groups-request message
session_id - the session ID for the session
grouping_allowed - true to allow grouped images
"""
pipeline = cellprofiler.pipeline.Pipeline()
m = cellprofiler.measurement.Measurements()
object_set = cellprofiler.object.ObjectSet()
if len(message) < 2:
self.raise_cellprofiler_exception(session_id,
"Missing run request sections")
return
pipeline_txt = message.pop(0).bytes
image_metadata = message.pop(0).bytes
try:
image_metadata = json.loads(image_metadata)
for channel_name, channel_metadata in image_metadata:
if len(message) < 1:
self.raise_cellprofiler_exception(
session_id,
"Missing binary data for channel %s" % channel_name)
return None, None, None
pixel_data = self.decode_image(
channel_metadata,
message.pop(0).bytes,
grouping_allowed=grouping_allowed,
)
m.add(channel_name, cellprofiler.image.Image(pixel_data))
except Exception as e:
logger.warn("Failed to decode message", exc_info=1)
self.raise_cellprofiler_exception(session_id, e.message)
return None, None, None
try:
pipeline.loadtxt(StringIO(pipeline_txt))
except Exception as e:
logger.warning(
"Failed to load pipeline: sending pipeline exception",
exc_info=1)
self.raise_pipeline_exception(session_id, str(e))
return None, None, None
return pipeline, m, object_set
def test_load_v7():
with open("./tests/resources/modules/createbatchfiles/v7.pipeline", "r") as fd:
data = fd.read()
pipeline = cellprofiler.pipeline.Pipeline()
pipeline.loadtxt(six.moves.StringIO(data))
assert len(pipeline.modules()) == 1
module = pipeline.modules()[0]
assert isinstance(module, cellprofiler.modules.createbatchfiles.CreateBatchFiles)
assert module.wants_default_output_directory
assert module.custom_output_directory == r"C:\foo\bar"
assert not module.remote_host_is_windows
assert not module.distributed_mode
assert module.default_image_directory == r"C:\bar\baz"
assert module.revision == 0
assert not module.from_old_matlab
assert len(module.mappings) == 1
mapping = module.mappings[0]
assert mapping.local_directory == r"\\argon-cifs\imaging_docs"
assert mapping.remote_directory == r"/imaging/docs"
def test_load_v3():
with open("./tests/resources/modules/maskobjects/v3.pipeline", "r") as fd:
data = fd.read()
pipeline = cellprofiler.pipeline.Pipeline()
def callback(caller, event):
assert not isinstance(event, cellprofiler.pipeline.LoadExceptionEvent)
pipeline.add_listener(callback)
pipeline.loadtxt(six.moves.StringIO(data))
module = pipeline.modules()[0]
assert module.object_name.value == "IdentifyPrimaryObjects"
assert module.remaining_objects.value == "MaskObjects"
assert module.mask_choice.value == "Objects"
assert module.masking_objects.value == "FilterObjects"
assert module.masking_image.value == "None"
assert module.overlap_choice.value == "Keep overlapping region"
assert module.overlap_fraction.value == 0.5
assert module.retain_or_renumber.value == "Renumber"
assert not module.wants_inverted_mask.value
def test_load_v7():
with open("./tests/resources/modules/threshold/v7.pipeline", "r") as fd:
data = fd.read()
fd = io.StringIO(data)
pipeline = cellprofiler.pipeline.Pipeline()
pipeline.loadtxt(fd)
module = pipeline.modules()[-1]
assert isinstance(module, cellprofiler.modules.threshold.Threshold)
assert module.x_name.value == "RainbowPony"
assert module.y_name.value == "GrayscalePony"
assert module.threshold_scope.value == cellprofiler.modules.threshold.TS_GLOBAL
assert module.global_operation.value == cellprofiler.modules.threshold.TM_LI
assert module.threshold_smoothing_scale.value == 1.3488
assert module.threshold_correction_factor.value == 1.1
assert module.threshold_range.min == 0.07
assert module.threshold_range.max == 0.99
assert module.manual_threshold.value == 0.1
assert module.thresholding_measurement.value == "Pony_Perimeter"
assert module.two_class_otsu.value == cellprofiler.modules.threshold.O_TWO_CLASS
assert (module.assign_middle_to_foreground.value ==
cellprofiler.modules.threshold.O_FOREGROUND)
assert module.adaptive_window_size.value == 13
def test_load_v5():
with open("./tests/resources/modules/splitormergeobjects/v5.pipeline", "r") as fd:
data = fd.read()
pipeline = cellprofiler.pipeline.Pipeline()
def callback(caller, event):
assert not isinstance(event, cellprofiler.pipeline.LoadExceptionEvent)
pipeline.add_listener(callback)
pipeline.loadtxt(six.moves.StringIO(data))
module = pipeline.modules()[0]
assert module.objects_name.value == "IdentifyPrimaryObjects"
assert module.output_objects_name.value == "SplitOrMergeObjects"
assert module.relabel_option.value == "Merge"
assert module.distance_threshold.value == 0
assert not module.wants_image.value
assert module.image_name.value == "None"
assert module.minimum_intensity_fraction.value == 0.9
assert module.where_algorithm.value == "Closest point"
assert module.merge_option.value == "Distance"
assert module.parent_object.value == "None"
assert module.merging_method.value == "Disconnected"
def test_01_08_load_v8(self):
data = r"""CellProfiler Pipeline: http://www.cellprofiler.org
Version:3
DateRevision:300
GitHash:
ModuleCount:5
HasImagePlaneDetails:False
Images:[module_num:1|svn_version:\'Unknown\'|variable_revision_number:2|show_window:False|notes:\x5B\'To begin creating your project, use the Images module to compile a list of files and/or folders that you want to analyze. You can also specify a set of rules to include only the desired files in your selected folders.\'\x5D|batch_state:array(\x5B\x5D, dtype=uint8)|enabled:True|wants_pause:False]
:
Filter images?:Images only
Select the rule criteria:and (extension does isimage) (directory doesnot containregexp "\x5B\\\\\\\\\\\\\\\\/\x5D\\\\\\\\.")
Metadata:[module_num:2|svn_version:\'Unknown\'|variable_revision_number:4|show_window:False|notes:\x5B\'The Metadata module optionally allows you to extract information describing your images (i.e, metadata) which will be stored along with your measurements. This information can be contained in the file name and/or location, or in an external file.\'\x5D|batch_state:array(\x5B\x5D, dtype=uint8)|enabled:True|wants_pause:False]
Extract metadata?:No
Metadata data type:Text
Metadata types:{}
Extraction method count:1
Metadata extraction method:Extract from file/folder names
Metadata source:File name
Regular expression:^(?P<Plate>.*)_(?P<Well>\x5BA-P\x5D\x5B0-9\x5D{2})_s(?P<Site>\x5B0-9\x5D)_w(?P<ChannelNumber>\x5B0-9\x5D)
Regular expression:(?P<Date>\x5B0-9\x5D{4}_\x5B0-9\x5D{2}_\x5B0-9\x5D{2})$
Extract metadata from:All images
Select the filtering criteria:and (file does contain "")
Metadata file location:
Match file and image metadata:\x5B\x5D
Use case insensitive matching?:No
NamesAndTypes:[module_num:3|svn_version:\'Unknown\'|variable_revision_number:7|show_window:False|notes:\x5B\'The NamesAndTypes module allows you to assign a meaningful name to each image by which other modules will refer to it.\'\x5D|batch_state:array(\x5B\x5D, dtype=uint8)|enabled:True|wants_pause:False]
Assign a name to:All images
Select the image type:Grayscale image
Name to assign these images:DNA
Match metadata:\x5B\x5D
Image set matching method:Order
Set intensity range from:Image metadata
Assignments count:1
Single images count:0
Maximum intensity:255.0
Volumetric:No
x:1.0
y:1.0
z:1.0
Select the rule criteria:and (file does contain "")
Name to assign these images:DNA
Name to assign these objects:Cell
Select the image type:Grayscale image
Set intensity range from:Image metadata
Retain outlines of loaded objects?:No
Name the outline image:LoadedOutlines
Maximum intensity:255.0
Groups:[module_num:4|svn_version:\'Unknown\'|variable_revision_number:2|show_window:False|notes:\x5B\'The Groups module optionally allows you to split your list of images into image subsets (groups) which will be processed independently of each other. Examples of groupings include screening batches, microtiter plates, time-lapse movies, etc.\'\x5D|batch_state:array(\x5B\x5D, dtype=uint8)|enabled:True|wants_pause:False]
Do you want to group your images?:No
grouping metadata count:1
Metadata category:None
ApplyThreshold:[module_num:5|svn_version:\'Unknown\'|variable_revision_number:8|show_window:True|notes:\x5B\x5D|batch_state:array(\x5B\x5D, dtype=uint8)|enabled:True|wants_pause:False]
Select the input image:DNA
Name the output image:ThreshBlue
Threshold setting version:3
Threshold strategy:Global
Thresholding method:MCT
Threshold smoothing scale:0
Threshold correction factor:1.0
Lower and upper bounds on threshold:0.0,1.0
Manual threshold:0.0
Select the measurement to threshold with:None
Two-class or three-class thresholding?:Two classes
Assign pixels in the middle intensity class to the foreground or the background?:Foreground
Size of adaptive window:50
Use default parameters?:Default
Lower outlier fraction:0.05
Upper outlier fraction:0.05
Averaging method:Mean
Variance method:Standard deviation
# of deviations:2.0
"""
fd = StringIO.StringIO(data)
pipeline = cellprofiler.pipeline.Pipeline()
pipeline.loadtxt(fd)
module = pipeline.modules()[-1]
self.assertTrue(
isinstance(module,
cellprofiler.modules.applythreshold.ApplyThreshold))
self.assertEqual(module.image_name, "DNA")
self.assertEqual(module.thresholded_image_name, "ThreshBlue")
self.assertEqual(module.threshold_scope,
cellprofiler.modules.identify.TS_GLOBAL)
self.assertEqual(module.threshold_method, centrosome.threshold.TM_MCT)
self.assertEqual(module.threshold_smoothing_scale, 0)
self.assertEqual(module.threshold_correction_factor, 1.0)
self.assertEqual(module.threshold_range.min, 0.0)
self.assertEqual(module.threshold_range.max, 1.0)
self.assertEqual(module.manual_threshold, 0.0)
self.assertEqual(module.thresholding_measurement, "None")
self.assertEqual(module.two_class_otsu,
cellprofiler.modules.identify.O_TWO_CLASS)
self.assertEqual(module.assign_middle_to_foreground,
cellprofiler.modules.identify.O_FOREGROUND)
self.assertEqual(module.adaptive_window_size, 50)
def test_01_07_load_v7(self):
data = r"""CellProfiler Pipeline: http://www.cellprofiler.org
Version:3
DateRevision:20130226215424
ModuleCount:5
HasImagePlaneDetails:False
Images:[module_num:1|svn_version:\'Unknown\'|variable_revision_number:1|show_window:False|notes:\x5B\x5D|batch_state:array(\x5B\x5D, dtype=uint8)|enabled:True]
:
Filter based on rules:No
Filter:or (file does contain "")
Metadata:[module_num:2|svn_version:\'Unknown\'|variable_revision_number:2|show_window:False|notes:\x5B\x5D|batch_state:array(\x5B\x5D, dtype=uint8)|enabled:True]
Extract metadata?:Yes
Extraction method count:1
Extraction method:Manual
Source:From file name
Regular expression:Channel(?P<Wavelength>\x5B12\x5D)-\x5B0-9\x5D{2}-(?P<WellRow>\x5BA-Z\x5D)-(?P<WellColumn>\x5B0-9\x5D{2}).tif
Regular expression:(?P<Date>\x5B0-9\x5D{4}_\x5B0-9\x5D{2}_\x5B0-9\x5D{2})$
Filter images:All images
:or (file does contain "")
Metadata file location\x3A:
Match file and image metadata:\x5B\x5D
Case insensitive matching:No
NamesAndTypes:[module_num:3|svn_version:\'Unknown\'|variable_revision_number:1|show_window:False|notes:\x5B\x5D|batch_state:array(\x5B\x5D, dtype=uint8)|enabled:True]
Assignment method:Assign images matching rules
Load as:Grayscale image
Image name:DNA
:\x5B\x5D
Assign channels by:Order
Assignments count:2
Match this rule:or (metadata does Wavelength "1")
Image name:GFP
Objects name:Cell
Load as:Grayscale image
Match this rule:or (metadata does Wavelength "2")
Image name:DNA
Objects name:Nucleus
Load as:Grayscale image
Groups:[module_num:4|svn_version:\'Unknown\'|variable_revision_number:2|show_window:False|notes:\x5B\x5D|batch_state:array(\x5B\x5D, dtype=uint8)|enabled:True]
Do you want to group your images?:No
grouping metadata count:1
Metadata category:None
ApplyThreshold:[module_num:5|svn_version:\'Unknown\'|variable_revision_number:7|show_window:True|notes:\x5B\x5D|batch_state:array(\x5B\x5D, dtype=uint8)|enabled:True]
Select the input image:RainbowPony
Name the output image:GrayscalePony
Select the output image type:Grayscale
Set pixels below or above the threshold to zero?:Below threshold
Subtract the threshold value from the remaining pixel intensities?:Yes
Number of pixels by which to expand the thresholding around those excluded bright pixels:2.0
Threshold setting version:1
Threshold strategy:Adaptive
Threshold method:MCT
Smoothing for threshold:Automatic
Threshold smoothing scale:1.5
Threshold correction factor:1.1
Lower and upper bounds on threshold:0.07,0.99
Approximate fraction of image covered by objects?:0.02
Manual threshold:0.1
Select the measurement to threshold with:Pony_Perimeter
Select binary image:Pony_yes_or_no
Masking objects:PonyMask
Two-class or three-class thresholding?:Two classes
Minimize the weighted variance or the entropy?:Weighted variance
Assign pixels in the middle intensity class to the foreground or the background?:Foreground
Method to calculate adaptive window size:Image size
Size of adaptive window:13
"""
fd = StringIO.StringIO(data)
pipeline = cellprofiler.pipeline.Pipeline()
pipeline.loadtxt(fd)
module = pipeline.modules()[-1]
self.assertTrue(
isinstance(module,
cellprofiler.modules.applythreshold.ApplyThreshold))
self.assertEqual(module.image_name, "RainbowPony")
self.assertEqual(module.thresholded_image_name, "GrayscalePony")
self.assertEqual(module.threshold_scope,
cellprofiler.modules.identify.TS_ADAPTIVE)
self.assertEqual(module.threshold_method, centrosome.threshold.TM_MCT)
self.assertEqual(module.threshold_smoothing_scale, 1.3488)
self.assertEqual(module.threshold_correction_factor, 1.1)
self.assertEqual(module.threshold_range.min, .07)
self.assertEqual(module.threshold_range.max, .99)
self.assertEqual(module.manual_threshold, 0.1)
self.assertEqual(module.thresholding_measurement, "Pony_Perimeter")
self.assertEqual(module.two_class_otsu,
cellprofiler.modules.identify.O_TWO_CLASS)
self.assertEqual(module.assign_middle_to_foreground,
cellprofiler.modules.identify.O_FOREGROUND)
self.assertEqual(module.adaptive_window_size, 13)
def test_03_09_flag_image_abort(self):
#
# Regression test of issue #1210
# Make a pipeline that aborts during FlagImage
#
data = r"""CellProfiler Pipeline: http://www.cellprofiler.org
Version:3
DateRevision:20140918122611
GitHash:ded6939
ModuleCount:6
HasImagePlaneDetails:False
Images:[module_num:1|svn_version:\'Unknown\'|variable_revision_number:2|show_window:False|notes:\x5B\'To begin creating your project, use the Images module to compile a list of files and/or folders that you want to analyze. You can also specify a set of rules to include only the desired files in your selected folders.\'\x5D|batch_state:array(\x5B\x5D, dtype=uint8)|enabled:True|wants_pause:False]
:
Filter images?:Images only
Select the rule criteria:and (extension does isimage) (directory doesnot containregexp "\x5B\\\\\\\\\\\\\\\\/\x5D\\\\\\\\.")
Metadata:[module_num:2|svn_version:\'Unknown\'|variable_revision_number:4|show_window:False|notes:\x5B\'The Metadata module optionally allows you to extract information describing your images (i.e, metadata) which will be stored along with your measurements. This information can be contained in the file name and/or location, or in an external file.\'\x5D|batch_state:array(\x5B\x5D, dtype=uint8)|enabled:True|wants_pause:False]
Extract metadata?:No
Metadata data type:Text
Metadata types:{}
Extraction method count:1
Metadata extraction method:Extract from file/folder names
Metadata source:File name
Regular expression:^(?P<Plate>.*)_(?P<Well>\x5BA-P\x5D\x5B0-9\x5D{2})_s(?P<Site>\x5B0-9\x5D)_w(?P<ChannelNumber>\x5B0-9\x5D)
Regular expression:(?P<Date>\x5B0-9\x5D{4}_\x5B0-9\x5D{2}_\x5B0-9\x5D{2})$
Extract metadata from:All images
Select the filtering criteria:and (file does contain "")
Metadata file location:
Match file and image metadata:\x5B\x5D
Use case insensitive matching?:No
NamesAndTypes:[module_num:3|svn_version:\'Unknown\'|variable_revision_number:5|show_window:False|notes:\x5B\'The NamesAndTypes module allows you to assign a meaningful name to each image by which other modules will refer to it.\'\x5D|batch_state:array(\x5B\x5D, dtype=uint8)|enabled:True|wants_pause:False]
Assign a name to:All images
Select the image type:Grayscale image
Name to assign these images:DNA
Match metadata:\x5B\x5D
Image set matching method:Order
Set intensity range from:Image metadata
Assignments count:1
Single images count:0
Select the rule criteria:and (file does contain "")
Name to assign these images:DNA
Name to assign these objects:Cell
Select the image type:Grayscale image
Set intensity range from:Image metadata
Retain outlines of loaded objects?:No
Name the outline image:LoadedOutlines
Groups:[module_num:4|svn_version:\'Unknown\'|variable_revision_number:2|show_window:False|notes:\x5B\'The Groups module optionally allows you to split your list of images into image subsets (groups) which will be processed independently of each other. Examples of groupings include screening batches, microtiter plates, time-lapse movies, etc.\'\x5D|batch_state:array(\x5B\x5D, dtype=uint8)|enabled:True|wants_pause:False]
Do you want to group your images?:No
grouping metadata count:1
Metadata category:None
FlagImage:[module_num:5|svn_version:\'Unknown\'|variable_revision_number:4|show_window:False|notes:\x5B\x5D|batch_state:array(\x5B\x5D, dtype=uint8)|enabled:True|wants_pause:False]
Hidden:1
Hidden:1
Name the flag\'s category:Metadata
Name the flag:QCFlag
Flag if any, or all, measurement(s) fails to meet the criteria?:Flag if any fail
Skip image set if flagged?:Yes
Flag is based on:Whole-image measurement
Select the object to be used for flagging:None
Which measurement?:Height_DNA
Flag images based on low values?:No
Minimum value:0.0
Flag images based on high values?:Yes
Maximum value:1.0
Rules file location:Elsewhere...\x7C
Rules file name:rules.txt
Class number:
MeasureImageIntensity:[module_num:6|svn_version:\'Unknown\'|variable_revision_number:2|show_window:True|notes:\x5B\x5D|batch_state:array(\x5B\x5D, dtype=uint8)|enabled:True|wants_pause:False]
Select the image to measure:DNA
Measure the intensity only from areas enclosed by objects?:No
Select the input objects:None
"""
self.awthread = self.AWThread(self.announce_addr)
self.awthread.start()
self.set_work_socket()
self.awthread.ex(
self.awthread.aw.do_job,
cellprofiler.analysis.WorkReply(image_set_numbers=[1],
worker_runs_post_group=False,
wants_dictionary=True))
#
# The worker should ask for the pipeline and preferences next.
#
req = self.awthread.recv(self.work_socket)
self.assertIsInstance(req,
cellprofiler.analysis.PipelinePreferencesRequest)
self.assertEqual(req.analysis_id, self.analysis_id)
input_dir = os.path.join(tests.modules.example_images_directory(),
"ExampleSBSImages")
cellprofiler.preferences.set_default_image_directory(input_dir)
preferences = {
cellprofiler.preferences.DEFAULT_IMAGE_DIRECTORY:
cellprofiler.preferences.config_read(
cellprofiler.preferences.DEFAULT_IMAGE_DIRECTORY)
}
rep = cellprofiler.analysis.Reply(pipeline_blob=numpy.array(data),
preferences=preferences)
req.reply(rep)
#
# The worker asks for the initial measurements.
#
req = self.awthread.recv(self.work_socket)
self.assertIsInstance(req,
cellprofiler.analysis.InitialMeasurementsRequest)
self.assertEqual(req.analysis_id, self.analysis_id)
m = get_measurements_for_good_pipeline()
pipeline = cellprofiler.pipeline.Pipeline()
pipeline.loadtxt(cStringIO.StringIO(data))
pipeline.write_pipeline_measurement(m)
try:
req.reply(cellprofiler.analysis.Reply(buf=m.file_contents()))
finally:
m.close()
#
# Next, the worker asks for the shared dictionary
#
req = self.awthread.recv(self.work_socket)
self.assertIsInstance(req,
cellprofiler.analysis.SharedDictionaryRequest)
shared_dictionaries = [{
("foo%d" % i): "bar%d" % i
} for i in range(1, 7)]
rep = cellprofiler.analysis.SharedDictionaryReply(
dictionaries=shared_dictionaries)
req.reply(rep)
#
# MeasureImageIntensity follows FlagImage and it is poised to ask
# for a display. So if we get that, we know the module has been run
# and we fail the test.
#
req = self.awthread.recv(self.work_socket)
self.assertFalse(isinstance(req, cellprofiler.analysis.DisplayRequest))
self.assertFalse(isinstance(req,
cellprofiler.analysis.ExceptionReport))
def run_group_request(self, session_id, message_type, message):
"""Handle a run-group request message"""
pipeline = cellprofiler.pipeline.Pipeline()
m = cellprofiler.measurement.Measurements()
image_group = m.hdf5_dict.hdf5_file.create_group("ImageData")
if len(message) < 2:
self.raise_cellprofiler_exception(session_id,
"Missing run request sections")
return
pipeline_txt = message.pop(0).bytes
image_metadata = message.pop(0).bytes
n_image_sets = None
try:
image_metadata = json.loads(image_metadata)
channel_names = []
for channel_name, channel_metadata in image_metadata:
channel_names.append(channel_name)
if len(message) < 1:
self.raise_cellprofiler_exception(
session_id,
"Missing binary data for channel %s" % channel_name)
return None, None, None
pixel_data = self.decode_image(channel_metadata,
message.pop(0).bytes,
grouping_allowed=True)
if pixel_data.ndim < 3:
self.raise_cellprofiler_exception(
session_id,
"The image for channel %s does not have a Z or T dimension",
)
return
if n_image_sets is None:
n_image_sets = pixel_data.shape[0]
elif n_image_sets != pixel_data.shape[0]:
self.raise_cellprofiler_exception(
session_id,
"The images passed have different numbers of Z or T planes",
)
return
image_group.create_dataset(channel_name, data=pixel_data)
except Exception as e:
self.raise_cellprofiler_exception(session_id, e.message)
return None, None, None
try:
pipeline.loadtxt(StringIO(pipeline_txt))
except Exception as e:
logger.warning(
"Failed to load pipeline: sending pipeline exception",
exc_info=1)
self.raise_pipeline_exception(session_id, str(e))
return
image_numbers = numpy.arange(1, n_image_sets + 1)
for image_number in image_numbers:
m[cellprofiler.measurement.IMAGE,
cellprofiler.measurement.GROUP_NUMBER, image_number, ] = 1
m[cellprofiler.measurement.IMAGE,
cellprofiler.measurement.GROUP_INDEX,
image_number, ] = image_number
input_modules, other_modules = self.split_pipeline(pipeline)
workspace = cellprofiler.workspace.Workspace(pipeline, None, m, None,
m, None)
logger.info("Preparing group")
for module in other_modules:
module.prepare_group(
workspace,
dict([("image_number", i) for i in image_numbers]),
image_numbers,
)
for image_index in range(n_image_sets):
object_set = cellprofiler.object.ObjectSet()
m.next_image_set(image_index + 1)
for channel_name in channel_names:
dataset = image_group[channel_name]
pixel_data = dataset[image_index]
m.add(channel_name, cellprofiler.image.Image(pixel_data))
for module in other_modules:
workspace = cellprofiler.workspace.Workspace(
pipeline, module, m, object_set, m, None)
try:
logger.info("Running module # %d: %s" %
(module.module_num, module.module_name))
pipeline.run_module(module, workspace)
if workspace.disposition in (
cellprofiler.workspace.DISPOSITION_SKIP,
cellprofiler.workspace.DISPOSITION_CANCEL,
):
break
except Exception as e:
msg = 'Encountered error while running module, "%s": %s' % (
module.module_name,
e.message,
)
logger.warning(msg, exc_info=1)
self.raise_cellprofiler_exception(session_id, msg)
return
else:
continue
if workspace.disposition == cellprofiler.workspace.DISPOSITION_CANCEL:
break
for module in other_modules:
module.post_group(
workspace, dict([("image_number", i) for i in image_numbers]))
logger.info("Finished group")
type_names, feature_dict = self.find_measurements(
other_modules, pipeline)
double_features = []
double_data = []
float_features = []
float_data = []
int_features = []
int_data = []
string_features = []
string_data = []
metadata = [
double_features, float_features, int_features, string_features
]
for object_name, features in feature_dict.items():
df = []
double_features.append((object_name, df))
ff = []
float_features.append((object_name, ff))
intf = []
int_features.append((object_name, intf))
sf = []
string_features.append((object_name, sf))
if object_name == cellprofiler.measurement.IMAGE:
object_counts = [] * n_image_sets
else:
object_numbers = m[object_name,
cellprofiler.measurement.OBJECT_NUMBER,
image_numbers]
object_counts = [len(x) for x in object_numbers]
for feature, data_type in features:
if data_type == "java.lang.String":
continue
if not m.has_feature(object_name, feature):
data = numpy.zeros(numpy.sum(object_counts))
else:
data = m[object_name, feature, image_numbers]
temp = []
for i, (di, count) in enumerate(zip(data, object_counts)):
if count == 0:
continue
di = numpy.atleast_1d(di)
if len(di) > count:
di = di[:count]
elif len(di) == count:
temp.append(di)
else:
temp += [di + numpy.zeros(len(di) - count)]
if len(temp) > 0:
data = numpy.hstack(temp)
if type_names[data_type] == "java.lang.Double":
df.append((feature, len(data)))
if len(data) > 0:
double_data.append(data.astype("<f8"))
elif type_names[data_type] == "java.lang.Float":
ff.append((feature, len(data)))
if len(data) > 0:
float_data.append(data.astype("<f4"))
elif type_names[data_type] == "java.lang.Integer":
intf.append((feature, len(data)))
if len(data) > 0:
int_data.append(data.astype("<i4"))
data = numpy.hstack([
numpy.frombuffer(
numpy.ascontiguousarray(numpy.hstack(ditem)).data, numpy.uint8)
for ditem in (double_data, float_data, int_data) if len(ditem) > 0
])
data = numpy.ascontiguousarray(data)
self.socket.send_multipart([
zmq.Frame(session_id),
zmq.Frame(),
zmq.Frame(RUN_REPLY_1),
zmq.Frame(json.dumps(metadata)),
zmq.Frame(data),
])
def test_03_09_flag_image_abort(self):
#
# Regression test of issue #1210
# Make a pipeline that aborts during FlagImage
#
data = r"""CellProfiler Pipeline: http://www.cellprofiler.org
Version:3
DateRevision:20140918122611
GitHash:ded6939
ModuleCount:6
HasImagePlaneDetails:False
Images:[module_num:1|svn_version:\'Unknown\'|variable_revision_number:2|show_window:False|notes:\x5B\'To begin creating your project, use the Images module to compile a list of files and/or folders that you want to analyze. You can also specify a set of rules to include only the desired files in your selected folders.\'\x5D|batch_state:array(\x5B\x5D, dtype=uint8)|enabled:True|wants_pause:False]
:
Filter images?:Images only
Select the rule criteria:and (extension does isimage) (directory doesnot containregexp "\x5B\\\\\\\\\\\\\\\\/\x5D\\\\\\\\.")
Metadata:[module_num:2|svn_version:\'Unknown\'|variable_revision_number:4|show_window:False|notes:\x5B\'The Metadata module optionally allows you to extract information describing your images (i.e, metadata) which will be stored along with your measurements. This information can be contained in the file name and/or location, or in an external file.\'\x5D|batch_state:array(\x5B\x5D, dtype=uint8)|enabled:True|wants_pause:False]
Extract metadata?:No
Metadata data type:Text
Metadata types:{}
Extraction method count:1
Metadata extraction method:Extract from file/folder names
Metadata source:File name
Regular expression:^(?P<Plate>.*)_(?P<Well>\x5BA-P\x5D\x5B0-9\x5D{2})_s(?P<Site>\x5B0-9\x5D)_w(?P<ChannelNumber>\x5B0-9\x5D)
Regular expression:(?P<Date>\x5B0-9\x5D{4}_\x5B0-9\x5D{2}_\x5B0-9\x5D{2})$
Extract metadata from:All images
Select the filtering criteria:and (file does contain "")
Metadata file location:
Match file and image metadata:\x5B\x5D
Use case insensitive matching?:No
NamesAndTypes:[module_num:3|svn_version:\'Unknown\'|variable_revision_number:5|show_window:False|notes:\x5B\'The NamesAndTypes module allows you to assign a meaningful name to each image by which other modules will refer to it.\'\x5D|batch_state:array(\x5B\x5D, dtype=uint8)|enabled:True|wants_pause:False]
Assign a name to:All images
Select the image type:Grayscale image
Name to assign these images:DNA
Match metadata:\x5B\x5D
Image set matching method:Order
Set intensity range from:Image metadata
Assignments count:1
Single images count:0
Select the rule criteria:and (file does contain "")
Name to assign these images:DNA
Name to assign these objects:Cell
Select the image type:Grayscale image
Set intensity range from:Image metadata
Retain outlines of loaded objects?:No
Name the outline image:LoadedOutlines
Groups:[module_num:4|svn_version:\'Unknown\'|variable_revision_number:2|show_window:False|notes:\x5B\'The Groups module optionally allows you to split your list of images into image subsets (groups) which will be processed independently of each other. Examples of groupings include screening batches, microtiter plates, time-lapse movies, etc.\'\x5D|batch_state:array(\x5B\x5D, dtype=uint8)|enabled:True|wants_pause:False]
Do you want to group your images?:No
grouping metadata count:1
Metadata category:None
FlagImage:[module_num:5|svn_version:\'Unknown\'|variable_revision_number:4|show_window:False|notes:\x5B\x5D|batch_state:array(\x5B\x5D, dtype=uint8)|enabled:True|wants_pause:False]
Hidden:1
Hidden:1
Name the flag\'s category:Metadata
Name the flag:QCFlag
Flag if any, or all, measurement(s) fails to meet the criteria?:Flag if any fail
Skip image set if flagged?:Yes
Flag is based on:Whole-image measurement
Select the object to be used for flagging:None
Which measurement?:Height_DNA
Flag images based on low values?:No
Minimum value:0.0
Flag images based on high values?:Yes
Maximum value:1.0
Rules file location:Elsewhere...\x7C
Rules file name:rules.txt
Class number:
MeasureImageIntensity:[module_num:6|svn_version:\'Unknown\'|variable_revision_number:2|show_window:True|notes:\x5B\x5D|batch_state:array(\x5B\x5D, dtype=uint8)|enabled:True|wants_pause:False]
Select the image to measure:DNA
Measure the intensity only from areas enclosed by objects?:No
Select the input objects:None
"""
self.awthread = self.AWThread(self.announce_addr)
self.awthread.start()
self.set_work_socket()
self.awthread.ex(self.awthread.aw.do_job,
cellprofiler.analysis.WorkReply(
image_set_numbers = [1],
worker_runs_post_group = False,
wants_dictionary = True))
#
# The worker should ask for the pipeline and preferences next.
#
req = self.awthread.recv(self.work_socket)
self.assertIsInstance(req, cellprofiler.analysis.PipelinePreferencesRequest)
self.assertEqual(req.analysis_id, self.analysis_id)
input_dir = os.path.join(tests.modules.example_images_directory(), "ExampleSBSImages")
cellprofiler.preferences.set_default_image_directory(input_dir)
preferences = {cellprofiler.preferences.DEFAULT_IMAGE_DIRECTORY:
cellprofiler.preferences.config_read(cellprofiler.preferences.DEFAULT_IMAGE_DIRECTORY)}
rep = cellprofiler.analysis.Reply(
pipeline_blob = numpy.array(data),
preferences = preferences)
req.reply(rep)
#
# The worker asks for the initial measurements.
#
req = self.awthread.recv(self.work_socket)
self.assertIsInstance(req, cellprofiler.analysis.InitialMeasurementsRequest)
self.assertEqual(req.analysis_id, self.analysis_id)
m = get_measurements_for_good_pipeline()
pipeline = cellprofiler.pipeline.Pipeline()
pipeline.loadtxt(cStringIO.StringIO(data))
pipeline.write_pipeline_measurement(m)
try:
req.reply(cellprofiler.analysis.Reply(buf = m.file_contents()))
finally:
m.close()
#
# Next, the worker asks for the shared dictionary
#
req = self.awthread.recv(self.work_socket)
self.assertIsInstance(req, cellprofiler.analysis.SharedDictionaryRequest)
shared_dictionaries = [{ ("foo%d" % i):"bar%d" % i} for i in range(1,7)]
rep = cellprofiler.analysis.SharedDictionaryReply(
dictionaries = shared_dictionaries)
req.reply(rep)
#
# MeasureImageIntensity follows FlagImage and it is poised to ask
# for a display. So if we get that, we know the module has been run
# and we fail the test.
#
req = self.awthread.recv(self.work_socket)
self.assertFalse(isinstance(req, cellprofiler.analysis.DisplayRequest))
self.assertFalse(isinstance(req, cellprofiler.analysis.ExceptionReport))
