def clean_pipeline(self, session_id, message_type, message):
"""Handle the clean pipeline request message"""
logger.info("Handling clean pipeline request")
pipeline_txt = message.pop(0).bytes
module_names = json.loads(message.pop(0).bytes)
pipeline = cellprofiler_core.pipeline.Pipeline()
try:
pipeline.loadtxt(StringIO(pipeline_txt))
except Exception as e:
logger.warning(
"Failed to load pipeline: sending pipeline exception",
exc_info=1)
self.raise_pipeline_exception(session_id, str(e))
return
to_remove = []
for module in pipeline.modules(exclude_disabled=False):
if module.module_name in module_names:
to_remove.insert(0, module)
for module in to_remove:
pipeline.remove_module(module.module_num)
pipeline_fd = StringIO()
pipeline.savetxt(pipeline_fd, save_image_plane_details=False)
msg_out = [
zmq.Frame(session_id),
zmq.Frame(),
zmq.Frame(CLEAN_PIPELINE_REPLY_1),
zmq.Frame(pipeline_fd.getvalue()),
]
self.socket.send_multipart(msg_out)
def test_load_v8():
with open("./tests/resources/modules/threshold/v8.pipeline", "r") as fd:
data = fd.read()
fd = io.StringIO(data)
pipeline = cellprofiler_core.pipeline.Pipeline()
pipeline.loadtxt(fd)
module = pipeline.modules()[-1]
assert isinstance(module, cellprofiler.modules.threshold.Threshold)
assert module.x_name == "DNA"
assert module.y_name == "ThreshBlue"
assert module.threshold_scope == cellprofiler.modules.threshold.TS_GLOBAL
assert module.global_operation.value == cellprofiler.modules.threshold.TM_LI
assert module.threshold_smoothing_scale == 0
assert module.threshold_correction_factor == 1.0
assert module.threshold_range.min == 0.0
assert module.threshold_range.max == 1.0
assert module.manual_threshold == 0.0
assert module.thresholding_measurement == "None"
assert module.two_class_otsu == cellprofiler.modules.threshold.O_TWO_CLASS
assert (
module.assign_middle_to_foreground
== cellprofiler.modules.threshold.O_FOREGROUND
)
assert module.adaptive_window_size == 50
def pipeline_info(self, session_id, message_type, message):
"""Handle the pipeline info message"""
logger.info("Handling pipeline info request")
pipeline_txt = message.pop(0).bytes
pipeline = cellprofiler_core.pipeline.Pipeline()
try:
pipeline.loadtxt(StringIO(pipeline_txt))
except Exception as e:
logger.warning(
"Failed to load pipeline: sending pipeline exception",
exc_info=1)
self.raise_pipeline_exception(session_id, str(e))
return
input_modules, other_modules = self.split_pipeline(pipeline)
channels = self.find_channels(input_modules)
type_names, measurements = self.find_measurements(
other_modules, pipeline)
body = json.dumps([channels, type_names, measurements])
msg_out = [
zmq.Frame(session_id),
zmq.Frame(),
zmq.Frame(PIPELINE_INFO_REPLY_1),
zmq.Frame(body),
]
self.socket.send_multipart(msg_out)
def test_load_v5():
file = tests.modules.test_resources_directory(
"splitormergeobjects/v5.pipeline")
with open(file, "r") as fd:
data = fd.read()
pipeline = cellprofiler_core.pipeline.Pipeline()
def callback(caller, event):
assert not isinstance(event,
cellprofiler_core.pipeline.event.LoadException)
pipeline.add_listener(callback)
pipeline.loadtxt(six.moves.StringIO(data))
module = pipeline.modules()[0]
assert module.objects_name.value == "IdentifyPrimaryObjects"
assert module.output_objects_name.value == "SplitOrMergeObjects"
assert module.relabel_option.value == "Merge"
assert module.distance_threshold.value == 0
assert not module.wants_image.value
assert module.image_name.value == "None"
assert module.minimum_intensity_fraction.value == 0.9
assert module.where_algorithm.value == "Closest point"
assert module.merge_option.value == "Distance"
assert module.parent_object.value == "None"
assert module.merging_method.value == "Disconnected"
def test_load_v7():
with open("./tests/resources/modules/threshold/v7.pipeline", "r") as fd:
data = fd.read()
fd = io.StringIO(data)
pipeline = cellprofiler_core.pipeline.Pipeline()
pipeline.loadtxt(fd)
module = pipeline.modules()[-1]
assert isinstance(module, cellprofiler.modules.threshold.Threshold)
assert module.x_name.value == "RainbowPony"
assert module.y_name.value == "GrayscalePony"
assert module.threshold_scope.value == cellprofiler.modules.threshold.TS_GLOBAL
assert module.global_operation.value == cellprofiler.modules.threshold.TM_LI
assert module.threshold_smoothing_scale.value == 1.3488
assert module.threshold_correction_factor.value == 1.1
assert module.threshold_range.min == 0.07
assert module.threshold_range.max == 0.99
assert module.manual_threshold.value == 0.1
assert module.thresholding_measurement.value == "Pony_Perimeter"
assert module.two_class_otsu.value == cellprofiler.modules.threshold.O_TWO_CLASS
assert (
module.assign_middle_to_foreground.value
== cellprofiler.modules.threshold.O_FOREGROUND
)
assert module.adaptive_window_size.value == 13
def test_load_v12():
file = tests.modules.get_test_resources_directory("threshold/v12.pipeline")
with open(file, "r") as fd:
data = fd.read()
fd = io.StringIO(data)
pipeline = cellprofiler_core.pipeline.Pipeline()
pipeline.loadtxt(fd)
module = pipeline.modules()[-1]
assert isinstance(module, cellprofiler.modules.threshold.Threshold)
assert module.x_name == "DNA"
assert module.y_name == "Threshold"
assert module.threshold_scope == cellprofiler.modules.threshold.TS_GLOBAL
assert module.global_operation.value == cellprofiler.modules.threshold.TM_LI
assert module.threshold_smoothing_scale == 0.01
assert module.threshold_correction_factor == 2
assert module.threshold_range.min == 0.01
assert module.threshold_range.max == 0.9
assert module.manual_threshold == 0.0
assert module.log_transform
assert module.thresholding_measurement == "None"
assert module.two_class_otsu == cellprofiler.modules.threshold.O_TWO_CLASS
assert (
module.assign_middle_to_foreground
== cellprofiler.modules.threshold.O_FOREGROUND
)
assert module.adaptive_window_size == 51
assert module.local_operation.value == cellprofiler.modules.threshold.TM_SAUVOLA
def test_load_v3():
file = tests.modules.get_test_resources_directory(
"maskobjects/v3.pipeline")
with open(file, "r") as fd:
data = fd.read()
pipeline = cellprofiler_core.pipeline.Pipeline()
def callback(caller, event):
assert not isinstance(event,
cellprofiler_core.pipeline.event.LoadException)
pipeline.add_listener(callback)
pipeline.loadtxt(six.moves.StringIO(data))
module = pipeline.modules()[0]
assert module.object_name.value == "IdentifyPrimaryObjects"
assert module.remaining_objects.value == "MaskObjects"
assert module.mask_choice.value == "Objects"
assert module.masking_objects.value == "FilterObjects"
assert module.masking_image.value == "None"
assert module.overlap_choice.value == "Keep overlapping region"
assert module.overlap_fraction.value == 0.5
assert module.retain_or_renumber.value == "Renumber"
assert not module.wants_inverted_mask.value
def test_load_v8():
with open("./tests/resources/modules/createbatchfiles/v8.pipeline", "r") as fd:
data = fd.read()
pipeline = cellprofiler_core.pipeline.Pipeline()
pipeline.loadtxt(six.moves.StringIO(data))
assert len(pipeline.modules()) == 1
module = pipeline.modules()[0]
assert module.wants_default_output_directory.value
assert module.custom_output_directory.value == "/Users/cellprofiler"
assert not module.remote_host_is_windows.value
assert module.mappings[0].local_directory.value == "/Users/cellprofiler/Pictures"
assert module.mappings[0].remote_directory.value == "/Remote/cellprofiler/Pictures"
def prepare_run(self, message, session_id, grouping_allowed=False):
"""Prepare a pipeline and measurements to run
message - the run-request or run-groups-request message
session_id - the session ID for the session
grouping_allowed - true to allow grouped images
"""
pipeline = cellprofiler_core.pipeline.Pipeline()
m = cellprofiler_core.measurement.Measurements()
object_set = cellprofiler_core.object.ObjectSet()
if len(message) < 2:
self.raise_cellprofiler_exception(session_id,
"Missing run request sections")
return
pipeline_txt = message.pop(0).bytes
image_metadata = message.pop(0).bytes
try:
image_metadata = json.loads(image_metadata)
for channel_name, channel_metadata in image_metadata:
if len(message) < 1:
self.raise_cellprofiler_exception(
session_id,
"Missing binary data for channel %s" % channel_name)
return None, None, None
pixel_data = self.decode_image(
channel_metadata,
message.pop(0).bytes,
grouping_allowed=grouping_allowed,
)
m.add(channel_name, cellprofiler_core.image.Image(pixel_data))
except Exception as e:
logger.warn("Failed to decode message", exc_info=1)
self.raise_cellprofiler_exception(session_id, e.message)
return None, None, None
try:
pipeline.loadtxt(StringIO(pipeline_txt))
except Exception as e:
logger.warning(
"Failed to load pipeline: sending pipeline exception",
exc_info=1)
self.raise_pipeline_exception(session_id, str(e))
return None, None, None
return pipeline, m, object_set
def test_load_v3():
with open("./tests/resources/modules/identifytertiaryobjects/v3.pipeline",
"r") as fd:
data = fd.read()
pipeline = cellprofiler_core.pipeline.Pipeline()
def callback(caller, event):
assert not isinstance(event,
cellprofiler_core.pipeline.event.LoadException)
pipeline.add_listener(callback)
pipeline.loadtxt(six.moves.StringIO(data))
module = pipeline.modules()[0]
assert module.secondary_objects_name.value == "IdentifySecondaryObjects"
assert module.primary_objects_name.value == "IdentifyPrimaryObjects"
assert module.subregion_objects_name.value == "IdentifyTertiaryObjects"
assert module.shrink_primary.value
def test_load_v4():
file = tests.modules.test_resources_directory(
"splitormergeobjects/v4.pipeline")
with open(file, "r") as fd:
data = fd.read()
pipeline = cellprofiler_core.pipeline.Pipeline()
def callback(caller, event):
assert not isinstance(event,
cellprofiler_core.pipeline.event.LoadException)
pipeline.add_listener(callback)
pipeline.loadtxt(six.moves.StringIO(data))
assert len(pipeline.modules()) == 2
module = pipeline.modules()[0]
assert isinstance(
module, cellprofiler.modules.splitormergeobjects.SplitOrMergeObjects)
assert module.objects_name == "blobs"
assert module.output_objects_name == "RelabeledBlobs"
assert (module.relabel_option ==
cellprofiler.modules.splitormergeobjects.OPTION_MERGE)
assert module.distance_threshold == 2
assert not module.wants_image
assert module.image_name == "Guide"
assert module.minimum_intensity_fraction == 0.8
assert (module.where_algorithm ==
cellprofiler.modules.splitormergeobjects.CA_CLOSEST_POINT)
assert module.merge_option == cellprofiler.modules.splitormergeobjects.UNIFY_PARENT
assert module.parent_object == "Nuclei"
assert (module.merging_method ==
cellprofiler.modules.splitormergeobjects.UM_CONVEX_HULL)
module = pipeline.modules()[1]
assert (module.relabel_option ==
cellprofiler.modules.splitormergeobjects.OPTION_SPLIT)
assert module.wants_image
assert (module.where_algorithm ==
cellprofiler.modules.splitormergeobjects.CA_CENTROIDS)
assert (module.merge_option ==
cellprofiler.modules.splitormergeobjects.UNIFY_DISTANCE)
assert (module.merging_method ==
cellprofiler.modules.splitormergeobjects.UM_DISCONNECTED)
def test_load_v4():
with open("./tests/resources/modules/editobjectsmanually/v4.pipeline", "r") as fd:
data = fd.read()
pipeline = cellprofiler_core.pipeline.Pipeline()
def callback(caller, event):
assert not isinstance(event, cellprofiler_core.pipeline.event.LoadException)
pipeline.add_listener(callback)
pipeline.loadtxt(six.moves.StringIO(data))
module = pipeline.modules()[0]
assert module.object_name.value == "IdentifyPrimaryObjects"
assert module.filtered_objects.value == "EditedObjects"
assert module.renumber_choice.value == "Renumber"
assert module.wants_image_display.value
assert module.image_name.value == "DNA"
assert not module.allow_overlap.value
def test_load_v7():
with open("./tests/resources/modules/createbatchfiles/v7.pipeline", "r") as fd:
data = fd.read()
pipeline = cellprofiler_core.pipeline.Pipeline()
pipeline.loadtxt(six.moves.StringIO(data))
assert len(pipeline.modules()) == 1
module = pipeline.modules()[0]
assert isinstance(module, cellprofiler.modules.createbatchfiles.CreateBatchFiles)
assert module.wants_default_output_directory
assert module.custom_output_directory == r"C:\foo\bar"
assert not module.remote_host_is_windows
assert not module.distributed_mode
assert module.default_image_directory == r"C:\bar\baz"
assert module.revision == 0
assert not module.from_old_matlab
assert len(module.mappings) == 1
mapping = module.mappings[0]
assert mapping.local_directory == r"\\argon-cifs\imaging_docs"
assert mapping.remote_directory == r"/imaging/docs"
def run_group_request(self, session_id, message_type, message):
"""Handle a run-group request message"""
pipeline = cellprofiler_core.pipeline.Pipeline()
m = cellprofiler_core.measurement.Measurements()
image_group = m.hdf5_dict.hdf5_file.create_group("ImageData")
if len(message) < 2:
self.raise_cellprofiler_exception(session_id,
"Missing run request sections")
return
pipeline_txt = message.pop(0).bytes
image_metadata = message.pop(0).bytes
n_image_sets = None
try:
image_metadata = json.loads(image_metadata)
channel_names = []
for channel_name, channel_metadata in image_metadata:
channel_names.append(channel_name)
if len(message) < 1:
self.raise_cellprofiler_exception(
session_id,
"Missing binary data for channel %s" % channel_name)
return None, None, None
pixel_data = self.decode_image(channel_metadata,
message.pop(0).bytes,
grouping_allowed=True)
if pixel_data.ndim < 3:
self.raise_cellprofiler_exception(
session_id,
"The image for channel %s does not have a Z or T dimension",
)
return
if n_image_sets is None:
n_image_sets = pixel_data.shape[0]
elif n_image_sets != pixel_data.shape[0]:
self.raise_cellprofiler_exception(
session_id,
"The images passed have different numbers of Z or T planes",
)
return
image_group.create_dataset(channel_name, data=pixel_data)
except Exception as e:
self.raise_cellprofiler_exception(session_id, e.message)
return None, None, None
try:
pipeline.loadtxt(StringIO(pipeline_txt))
except Exception as e:
logger.warning(
"Failed to load pipeline: sending pipeline exception",
exc_info=1)
self.raise_pipeline_exception(session_id, str(e))
return
image_numbers = numpy.arange(1, n_image_sets + 1)
for image_number in image_numbers:
m[cellprofiler_core.measurement.IMAGE,
cellprofiler_core.measurement.GROUP_NUMBER, image_number, ] = 1
m[cellprofiler_core.measurement.IMAGE,
cellprofiler_core.measurement.GROUP_INDEX,
image_number, ] = image_number
input_modules, other_modules = self.split_pipeline(pipeline)
workspace = cellprofiler_core.workspace.Workspace(
pipeline, None, m, None, m, None)
logger.info("Preparing group")
for module in other_modules:
module.prepare_group(
workspace,
dict([("image_number", i) for i in image_numbers]),
image_numbers,
)
for image_index in range(n_image_sets):
object_set = cellprofiler_core.object.ObjectSet()
m.next_image_set(image_index + 1)
for channel_name in channel_names:
dataset = image_group[channel_name]
pixel_data = dataset[image_index]
m.add(channel_name, cellprofiler_core.image.Image(pixel_data))
for module in other_modules:
workspace = cellprofiler_core.workspace.Workspace(
pipeline, module, m, object_set, m, None)
try:
logger.info("Running module # %d: %s" %
(module.module_num, module.module_name))
pipeline.run_module(module, workspace)
if workspace.disposition in (
cellprofiler_core.workspace.DISPOSITION_SKIP,
cellprofiler_core.workspace.DISPOSITION_CANCEL,
):
break
except Exception as e:
msg = 'Encountered error while running module, "%s": %s' % (
module.module_name,
e.message,
)
logger.warning(msg, exc_info=1)
self.raise_cellprofiler_exception(session_id, msg)
return
else:
continue
if workspace.disposition == cellprofiler_core.workspace.DISPOSITION_CANCEL:
break
for module in other_modules:
module.post_group(
workspace, dict([("image_number", i) for i in image_numbers]))
logger.info("Finished group")
type_names, feature_dict = self.find_measurements(
other_modules, pipeline)
double_features = []
double_data = []
float_features = []
float_data = []
int_features = []
int_data = []
string_features = []
string_data = []
metadata = [
double_features, float_features, int_features, string_features
]
for object_name, features in list(feature_dict.items()):
df = []
double_features.append((object_name, df))
ff = []
float_features.append((object_name, ff))
intf = []
int_features.append((object_name, intf))
sf = []
string_features.append((object_name, sf))
if object_name == cellprofiler_core.measurement.IMAGE:
object_counts = [] * n_image_sets
else:
object_numbers = m[object_name,
cellprofiler_core.measurement.OBJECT_NUMBER,
image_numbers]
object_counts = [len(x) for x in object_numbers]
for feature, data_type in features:
if data_type == "java.lang.String":
continue
if not m.has_feature(object_name, feature):
data = numpy.zeros(numpy.sum(object_counts))
else:
data = m[object_name, feature, image_numbers]
temp = []
for i, (di, count) in enumerate(zip(data, object_counts)):
if count == 0:
continue
di = numpy.atleast_1d(di)
if len(di) > count:
di = di[:count]
elif len(di) == count:
temp.append(di)
else:
temp += [di + numpy.zeros(len(di) - count)]
if len(temp) > 0:
data = numpy.hstack(temp)
if type_names[data_type] == "java.lang.Double":
df.append((feature, len(data)))
if len(data) > 0:
double_data.append(data.astype("<f8"))
elif type_names[data_type] == "java.lang.Float":
ff.append((feature, len(data)))
if len(data) > 0:
float_data.append(data.astype("<f4"))
elif type_names[data_type] == "java.lang.Integer":
intf.append((feature, len(data)))
if len(data) > 0:
int_data.append(data.astype("<i4"))
data = numpy.hstack([
numpy.frombuffer(
numpy.ascontiguousarray(numpy.hstack(ditem)).data, numpy.uint8)
for ditem in (double_data, float_data, int_data) if len(ditem) > 0
])
data = numpy.ascontiguousarray(data)
self.socket.send_multipart([
zmq.Frame(session_id),
zmq.Frame(),
zmq.Frame(RUN_REPLY_1),
zmq.Frame(json.dumps(metadata)),
zmq.Frame(data),
])
def get_measurements_for_good_pipeline(nimages=1, group_numbers=None):
"""Get an appropriately initialized measurements structure for the good pipeline"""
import cellprofiler_core
path = os.path.abspath(
os.path.join(
os.path.dirname(cellprofiler_core.__file__),
"..",
"tests/data/ExampleSBSImages",
)
)
# path = os.path.join(tests.modules.example_images_directory(), "ExampleSBSImages")
m = cellprofiler_core.measurement.Measurements()
if group_numbers is None:
group_numbers = [1] * nimages
group_indexes = [1]
last_group_number = group_numbers[0]
group_index = 1
for group_number in group_numbers:
if group_number == last_group_number:
group_index += 1
else:
group_index = 1
group_indexes.append(group_index)
for i in range(1, nimages + 1):
filename = "Channel2-%02d-%s-%02d.tif" % (
i,
"ABCDEFGH"[int((i - 1) / 12)],
((i - 1) % 12) + 1,
)
url = cellprofiler_core.utilities.pathname.pathname2url(
os.path.join(path, filename)
)
m[
cellprofiler_core.constants.measurement.IMAGE,
cellprofiler_core.constants.measurement.C_FILE_NAME + "_DNA",
i,
] = filename
m[
cellprofiler_core.constants.measurement.IMAGE,
cellprofiler_core.constants.measurement.C_PATH_NAME + "_DNA",
i,
] = path
m[
cellprofiler_core.constants.measurement.IMAGE,
cellprofiler_core.constants.measurement.C_URL + "_DNA",
i,
] = url
m[
cellprofiler_core.constants.measurement.IMAGE,
cellprofiler_core.constants.measurement.GROUP_NUMBER,
i,
] = group_numbers[i - 1]
m[
cellprofiler_core.constants.measurement.IMAGE,
cellprofiler_core.constants.measurement.GROUP_INDEX,
i,
] = group_indexes[i - 1]
jblob = javabridge.run_script(
"""
importPackage(Packages.org.cellprofiler.imageset);
importPackage(Packages.org.cellprofiler.imageset.filter);
var imageFile=new ImageFile(new java.net.URI(url));
var imageFileDetails = new ImageFileDetails(imageFile);
var imageSeries=new ImageSeries(imageFile, 0);
var imageSeriesDetails = new ImageSeriesDetails(imageSeries, imageFileDetails);
var imagePlane=new ImagePlane(imageSeries, 0, ImagePlane.ALWAYS_MONOCHROME);
var ipd = new ImagePlaneDetails(imagePlane, imageSeriesDetails);
var stack = ImagePlaneDetailsStack.makeMonochromeStack(ipd);
var stacks = java.util.Collections.singletonList(stack);
var keys = java.util.Collections.singletonList(imageNumber);
var imageSet = new ImageSet(stacks, keys);
imageSet.compress(java.util.Collections.singletonList("DNA"), null);
""",
dict(url=url, imageNumber=str(i)),
)
blob = javabridge.get_env().get_byte_array_elements(jblob)
m[
cellprofiler_core.constants.measurement.IMAGE,
cellprofiler_core.modules.namesandtypes.M_IMAGE_SET,
i,
blob.dtype,
] = blob
pipeline = cellprofiler_core.pipeline.Pipeline()
pipeline.loadtxt(six.moves.StringIO(GOOD_PIPELINE))
pipeline.write_pipeline_measurement(m)
return m
