def clean_pipeline(self, session_id, message_type, message): """Handle the clean pipeline request message""" logger.info("Handling clean pipeline request") pipeline_txt = message.pop(0).bytes module_names = json.loads(message.pop(0).bytes) pipeline = cellprofiler_core.pipeline.Pipeline() try: pipeline.loadtxt(StringIO(pipeline_txt)) except Exception as e: logger.warning( "Failed to load pipeline: sending pipeline exception", exc_info=1) self.raise_pipeline_exception(session_id, str(e)) return to_remove = [] for module in pipeline.modules(exclude_disabled=False): if module.module_name in module_names: to_remove.insert(0, module) for module in to_remove: pipeline.remove_module(module.module_num) pipeline_fd = StringIO() pipeline.savetxt(pipeline_fd, save_image_plane_details=False) msg_out = [ zmq.Frame(session_id), zmq.Frame(), zmq.Frame(CLEAN_PIPELINE_REPLY_1), zmq.Frame(pipeline_fd.getvalue()), ] self.socket.send_multipart(msg_out)
def test_load_v8(): with open("./tests/resources/modules/threshold/v8.pipeline", "r") as fd: data = fd.read() fd = io.StringIO(data) pipeline = cellprofiler_core.pipeline.Pipeline() pipeline.loadtxt(fd) module = pipeline.modules()[-1] assert isinstance(module, cellprofiler.modules.threshold.Threshold) assert module.x_name == "DNA" assert module.y_name == "ThreshBlue" assert module.threshold_scope == cellprofiler.modules.threshold.TS_GLOBAL assert module.global_operation.value == cellprofiler.modules.threshold.TM_LI assert module.threshold_smoothing_scale == 0 assert module.threshold_correction_factor == 1.0 assert module.threshold_range.min == 0.0 assert module.threshold_range.max == 1.0 assert module.manual_threshold == 0.0 assert module.thresholding_measurement == "None" assert module.two_class_otsu == cellprofiler.modules.threshold.O_TWO_CLASS assert ( module.assign_middle_to_foreground == cellprofiler.modules.threshold.O_FOREGROUND ) assert module.adaptive_window_size == 50
def pipeline_info(self, session_id, message_type, message): """Handle the pipeline info message""" logger.info("Handling pipeline info request") pipeline_txt = message.pop(0).bytes pipeline = cellprofiler_core.pipeline.Pipeline() try: pipeline.loadtxt(StringIO(pipeline_txt)) except Exception as e: logger.warning( "Failed to load pipeline: sending pipeline exception", exc_info=1) self.raise_pipeline_exception(session_id, str(e)) return input_modules, other_modules = self.split_pipeline(pipeline) channels = self.find_channels(input_modules) type_names, measurements = self.find_measurements( other_modules, pipeline) body = json.dumps([channels, type_names, measurements]) msg_out = [ zmq.Frame(session_id), zmq.Frame(), zmq.Frame(PIPELINE_INFO_REPLY_1), zmq.Frame(body), ] self.socket.send_multipart(msg_out)
def test_load_v5(): file = tests.modules.test_resources_directory( "splitormergeobjects/v5.pipeline") with open(file, "r") as fd: data = fd.read() pipeline = cellprofiler_core.pipeline.Pipeline() def callback(caller, event): assert not isinstance(event, cellprofiler_core.pipeline.event.LoadException) pipeline.add_listener(callback) pipeline.loadtxt(six.moves.StringIO(data)) module = pipeline.modules()[0] assert module.objects_name.value == "IdentifyPrimaryObjects" assert module.output_objects_name.value == "SplitOrMergeObjects" assert module.relabel_option.value == "Merge" assert module.distance_threshold.value == 0 assert not module.wants_image.value assert module.image_name.value == "None" assert module.minimum_intensity_fraction.value == 0.9 assert module.where_algorithm.value == "Closest point" assert module.merge_option.value == "Distance" assert module.parent_object.value == "None" assert module.merging_method.value == "Disconnected"
def test_load_v7(): with open("./tests/resources/modules/threshold/v7.pipeline", "r") as fd: data = fd.read() fd = io.StringIO(data) pipeline = cellprofiler_core.pipeline.Pipeline() pipeline.loadtxt(fd) module = pipeline.modules()[-1] assert isinstance(module, cellprofiler.modules.threshold.Threshold) assert module.x_name.value == "RainbowPony" assert module.y_name.value == "GrayscalePony" assert module.threshold_scope.value == cellprofiler.modules.threshold.TS_GLOBAL assert module.global_operation.value == cellprofiler.modules.threshold.TM_LI assert module.threshold_smoothing_scale.value == 1.3488 assert module.threshold_correction_factor.value == 1.1 assert module.threshold_range.min == 0.07 assert module.threshold_range.max == 0.99 assert module.manual_threshold.value == 0.1 assert module.thresholding_measurement.value == "Pony_Perimeter" assert module.two_class_otsu.value == cellprofiler.modules.threshold.O_TWO_CLASS assert ( module.assign_middle_to_foreground.value == cellprofiler.modules.threshold.O_FOREGROUND ) assert module.adaptive_window_size.value == 13
def test_load_v12(): file = tests.modules.get_test_resources_directory("threshold/v12.pipeline") with open(file, "r") as fd: data = fd.read() fd = io.StringIO(data) pipeline = cellprofiler_core.pipeline.Pipeline() pipeline.loadtxt(fd) module = pipeline.modules()[-1] assert isinstance(module, cellprofiler.modules.threshold.Threshold) assert module.x_name == "DNA" assert module.y_name == "Threshold" assert module.threshold_scope == cellprofiler.modules.threshold.TS_GLOBAL assert module.global_operation.value == cellprofiler.modules.threshold.TM_LI assert module.threshold_smoothing_scale == 0.01 assert module.threshold_correction_factor == 2 assert module.threshold_range.min == 0.01 assert module.threshold_range.max == 0.9 assert module.manual_threshold == 0.0 assert module.log_transform assert module.thresholding_measurement == "None" assert module.two_class_otsu == cellprofiler.modules.threshold.O_TWO_CLASS assert ( module.assign_middle_to_foreground == cellprofiler.modules.threshold.O_FOREGROUND ) assert module.adaptive_window_size == 51 assert module.local_operation.value == cellprofiler.modules.threshold.TM_SAUVOLA
def test_load_v3(): file = tests.modules.get_test_resources_directory( "maskobjects/v3.pipeline") with open(file, "r") as fd: data = fd.read() pipeline = cellprofiler_core.pipeline.Pipeline() def callback(caller, event): assert not isinstance(event, cellprofiler_core.pipeline.event.LoadException) pipeline.add_listener(callback) pipeline.loadtxt(six.moves.StringIO(data)) module = pipeline.modules()[0] assert module.object_name.value == "IdentifyPrimaryObjects" assert module.remaining_objects.value == "MaskObjects" assert module.mask_choice.value == "Objects" assert module.masking_objects.value == "FilterObjects" assert module.masking_image.value == "None" assert module.overlap_choice.value == "Keep overlapping region" assert module.overlap_fraction.value == 0.5 assert module.retain_or_renumber.value == "Renumber" assert not module.wants_inverted_mask.value
def test_load_v8(): with open("./tests/resources/modules/createbatchfiles/v8.pipeline", "r") as fd: data = fd.read() pipeline = cellprofiler_core.pipeline.Pipeline() pipeline.loadtxt(six.moves.StringIO(data)) assert len(pipeline.modules()) == 1 module = pipeline.modules()[0] assert module.wants_default_output_directory.value assert module.custom_output_directory.value == "/Users/cellprofiler" assert not module.remote_host_is_windows.value assert module.mappings[0].local_directory.value == "/Users/cellprofiler/Pictures" assert module.mappings[0].remote_directory.value == "/Remote/cellprofiler/Pictures"
def prepare_run(self, message, session_id, grouping_allowed=False): """Prepare a pipeline and measurements to run message - the run-request or run-groups-request message session_id - the session ID for the session grouping_allowed - true to allow grouped images """ pipeline = cellprofiler_core.pipeline.Pipeline() m = cellprofiler_core.measurement.Measurements() object_set = cellprofiler_core.object.ObjectSet() if len(message) < 2: self.raise_cellprofiler_exception(session_id, "Missing run request sections") return pipeline_txt = message.pop(0).bytes image_metadata = message.pop(0).bytes try: image_metadata = json.loads(image_metadata) for channel_name, channel_metadata in image_metadata: if len(message) < 1: self.raise_cellprofiler_exception( session_id, "Missing binary data for channel %s" % channel_name) return None, None, None pixel_data = self.decode_image( channel_metadata, message.pop(0).bytes, grouping_allowed=grouping_allowed, ) m.add(channel_name, cellprofiler_core.image.Image(pixel_data)) except Exception as e: logger.warn("Failed to decode message", exc_info=1) self.raise_cellprofiler_exception(session_id, e.message) return None, None, None try: pipeline.loadtxt(StringIO(pipeline_txt)) except Exception as e: logger.warning( "Failed to load pipeline: sending pipeline exception", exc_info=1) self.raise_pipeline_exception(session_id, str(e)) return None, None, None return pipeline, m, object_set
def test_load_v3(): with open("./tests/resources/modules/identifytertiaryobjects/v3.pipeline", "r") as fd: data = fd.read() pipeline = cellprofiler_core.pipeline.Pipeline() def callback(caller, event): assert not isinstance(event, cellprofiler_core.pipeline.event.LoadException) pipeline.add_listener(callback) pipeline.loadtxt(six.moves.StringIO(data)) module = pipeline.modules()[0] assert module.secondary_objects_name.value == "IdentifySecondaryObjects" assert module.primary_objects_name.value == "IdentifyPrimaryObjects" assert module.subregion_objects_name.value == "IdentifyTertiaryObjects" assert module.shrink_primary.value
def test_load_v4(): file = tests.modules.test_resources_directory( "splitormergeobjects/v4.pipeline") with open(file, "r") as fd: data = fd.read() pipeline = cellprofiler_core.pipeline.Pipeline() def callback(caller, event): assert not isinstance(event, cellprofiler_core.pipeline.event.LoadException) pipeline.add_listener(callback) pipeline.loadtxt(six.moves.StringIO(data)) assert len(pipeline.modules()) == 2 module = pipeline.modules()[0] assert isinstance( module, cellprofiler.modules.splitormergeobjects.SplitOrMergeObjects) assert module.objects_name == "blobs" assert module.output_objects_name == "RelabeledBlobs" assert (module.relabel_option == cellprofiler.modules.splitormergeobjects.OPTION_MERGE) assert module.distance_threshold == 2 assert not module.wants_image assert module.image_name == "Guide" assert module.minimum_intensity_fraction == 0.8 assert (module.where_algorithm == cellprofiler.modules.splitormergeobjects.CA_CLOSEST_POINT) assert module.merge_option == cellprofiler.modules.splitormergeobjects.UNIFY_PARENT assert module.parent_object == "Nuclei" assert (module.merging_method == cellprofiler.modules.splitormergeobjects.UM_CONVEX_HULL) module = pipeline.modules()[1] assert (module.relabel_option == cellprofiler.modules.splitormergeobjects.OPTION_SPLIT) assert module.wants_image assert (module.where_algorithm == cellprofiler.modules.splitormergeobjects.CA_CENTROIDS) assert (module.merge_option == cellprofiler.modules.splitormergeobjects.UNIFY_DISTANCE) assert (module.merging_method == cellprofiler.modules.splitormergeobjects.UM_DISCONNECTED)
def test_load_v4(): with open("./tests/resources/modules/editobjectsmanually/v4.pipeline", "r") as fd: data = fd.read() pipeline = cellprofiler_core.pipeline.Pipeline() def callback(caller, event): assert not isinstance(event, cellprofiler_core.pipeline.event.LoadException) pipeline.add_listener(callback) pipeline.loadtxt(six.moves.StringIO(data)) module = pipeline.modules()[0] assert module.object_name.value == "IdentifyPrimaryObjects" assert module.filtered_objects.value == "EditedObjects" assert module.renumber_choice.value == "Renumber" assert module.wants_image_display.value assert module.image_name.value == "DNA" assert not module.allow_overlap.value
def test_load_v7(): with open("./tests/resources/modules/createbatchfiles/v7.pipeline", "r") as fd: data = fd.read() pipeline = cellprofiler_core.pipeline.Pipeline() pipeline.loadtxt(six.moves.StringIO(data)) assert len(pipeline.modules()) == 1 module = pipeline.modules()[0] assert isinstance(module, cellprofiler.modules.createbatchfiles.CreateBatchFiles) assert module.wants_default_output_directory assert module.custom_output_directory == r"C:\foo\bar" assert not module.remote_host_is_windows assert not module.distributed_mode assert module.default_image_directory == r"C:\bar\baz" assert module.revision == 0 assert not module.from_old_matlab assert len(module.mappings) == 1 mapping = module.mappings[0] assert mapping.local_directory == r"\\argon-cifs\imaging_docs" assert mapping.remote_directory == r"/imaging/docs"
def run_group_request(self, session_id, message_type, message): """Handle a run-group request message""" pipeline = cellprofiler_core.pipeline.Pipeline() m = cellprofiler_core.measurement.Measurements() image_group = m.hdf5_dict.hdf5_file.create_group("ImageData") if len(message) < 2: self.raise_cellprofiler_exception(session_id, "Missing run request sections") return pipeline_txt = message.pop(0).bytes image_metadata = message.pop(0).bytes n_image_sets = None try: image_metadata = json.loads(image_metadata) channel_names = [] for channel_name, channel_metadata in image_metadata: channel_names.append(channel_name) if len(message) < 1: self.raise_cellprofiler_exception( session_id, "Missing binary data for channel %s" % channel_name) return None, None, None pixel_data = self.decode_image(channel_metadata, message.pop(0).bytes, grouping_allowed=True) if pixel_data.ndim < 3: self.raise_cellprofiler_exception( session_id, "The image for channel %s does not have a Z or T dimension", ) return if n_image_sets is None: n_image_sets = pixel_data.shape[0] elif n_image_sets != pixel_data.shape[0]: self.raise_cellprofiler_exception( session_id, "The images passed have different numbers of Z or T planes", ) return image_group.create_dataset(channel_name, data=pixel_data) except Exception as e: self.raise_cellprofiler_exception(session_id, e.message) return None, None, None try: pipeline.loadtxt(StringIO(pipeline_txt)) except Exception as e: logger.warning( "Failed to load pipeline: sending pipeline exception", exc_info=1) self.raise_pipeline_exception(session_id, str(e)) return image_numbers = numpy.arange(1, n_image_sets + 1) for image_number in image_numbers: m[cellprofiler_core.measurement.IMAGE, cellprofiler_core.measurement.GROUP_NUMBER, image_number, ] = 1 m[cellprofiler_core.measurement.IMAGE, cellprofiler_core.measurement.GROUP_INDEX, image_number, ] = image_number input_modules, other_modules = self.split_pipeline(pipeline) workspace = cellprofiler_core.workspace.Workspace( pipeline, None, m, None, m, None) logger.info("Preparing group") for module in other_modules: module.prepare_group( workspace, dict([("image_number", i) for i in image_numbers]), image_numbers, ) for image_index in range(n_image_sets): object_set = cellprofiler_core.object.ObjectSet() m.next_image_set(image_index + 1) for channel_name in channel_names: dataset = image_group[channel_name] pixel_data = dataset[image_index] m.add(channel_name, cellprofiler_core.image.Image(pixel_data)) for module in other_modules: workspace = cellprofiler_core.workspace.Workspace( pipeline, module, m, object_set, m, None) try: logger.info("Running module # %d: %s" % (module.module_num, module.module_name)) pipeline.run_module(module, workspace) if workspace.disposition in ( cellprofiler_core.workspace.DISPOSITION_SKIP, cellprofiler_core.workspace.DISPOSITION_CANCEL, ): break except Exception as e: msg = 'Encountered error while running module, "%s": %s' % ( module.module_name, e.message, ) logger.warning(msg, exc_info=1) self.raise_cellprofiler_exception(session_id, msg) return else: continue if workspace.disposition == cellprofiler_core.workspace.DISPOSITION_CANCEL: break for module in other_modules: module.post_group( workspace, dict([("image_number", i) for i in image_numbers])) logger.info("Finished group") type_names, feature_dict = self.find_measurements( other_modules, pipeline) double_features = [] double_data = [] float_features = [] float_data = [] int_features = [] int_data = [] string_features = [] string_data = [] metadata = [ double_features, float_features, int_features, string_features ] for object_name, features in list(feature_dict.items()): df = [] double_features.append((object_name, df)) ff = [] float_features.append((object_name, ff)) intf = [] int_features.append((object_name, intf)) sf = [] string_features.append((object_name, sf)) if object_name == cellprofiler_core.measurement.IMAGE: object_counts = [] * n_image_sets else: object_numbers = m[object_name, cellprofiler_core.measurement.OBJECT_NUMBER, image_numbers] object_counts = [len(x) for x in object_numbers] for feature, data_type in features: if data_type == "java.lang.String": continue if not m.has_feature(object_name, feature): data = numpy.zeros(numpy.sum(object_counts)) else: data = m[object_name, feature, image_numbers] temp = [] for i, (di, count) in enumerate(zip(data, object_counts)): if count == 0: continue di = numpy.atleast_1d(di) if len(di) > count: di = di[:count] elif len(di) == count: temp.append(di) else: temp += [di + numpy.zeros(len(di) - count)] if len(temp) > 0: data = numpy.hstack(temp) if type_names[data_type] == "java.lang.Double": df.append((feature, len(data))) if len(data) > 0: double_data.append(data.astype("<f8")) elif type_names[data_type] == "java.lang.Float": ff.append((feature, len(data))) if len(data) > 0: float_data.append(data.astype("<f4")) elif type_names[data_type] == "java.lang.Integer": intf.append((feature, len(data))) if len(data) > 0: int_data.append(data.astype("<i4")) data = numpy.hstack([ numpy.frombuffer( numpy.ascontiguousarray(numpy.hstack(ditem)).data, numpy.uint8) for ditem in (double_data, float_data, int_data) if len(ditem) > 0 ]) data = numpy.ascontiguousarray(data) self.socket.send_multipart([ zmq.Frame(session_id), zmq.Frame(), zmq.Frame(RUN_REPLY_1), zmq.Frame(json.dumps(metadata)), zmq.Frame(data), ])
def get_measurements_for_good_pipeline(nimages=1, group_numbers=None): """Get an appropriately initialized measurements structure for the good pipeline""" import cellprofiler_core path = os.path.abspath( os.path.join( os.path.dirname(cellprofiler_core.__file__), "..", "tests/data/ExampleSBSImages", ) ) # path = os.path.join(tests.modules.example_images_directory(), "ExampleSBSImages") m = cellprofiler_core.measurement.Measurements() if group_numbers is None: group_numbers = [1] * nimages group_indexes = [1] last_group_number = group_numbers[0] group_index = 1 for group_number in group_numbers: if group_number == last_group_number: group_index += 1 else: group_index = 1 group_indexes.append(group_index) for i in range(1, nimages + 1): filename = "Channel2-%02d-%s-%02d.tif" % ( i, "ABCDEFGH"[int((i - 1) / 12)], ((i - 1) % 12) + 1, ) url = cellprofiler_core.utilities.pathname.pathname2url( os.path.join(path, filename) ) m[ cellprofiler_core.constants.measurement.IMAGE, cellprofiler_core.constants.measurement.C_FILE_NAME + "_DNA", i, ] = filename m[ cellprofiler_core.constants.measurement.IMAGE, cellprofiler_core.constants.measurement.C_PATH_NAME + "_DNA", i, ] = path m[ cellprofiler_core.constants.measurement.IMAGE, cellprofiler_core.constants.measurement.C_URL + "_DNA", i, ] = url m[ cellprofiler_core.constants.measurement.IMAGE, cellprofiler_core.constants.measurement.GROUP_NUMBER, i, ] = group_numbers[i - 1] m[ cellprofiler_core.constants.measurement.IMAGE, cellprofiler_core.constants.measurement.GROUP_INDEX, i, ] = group_indexes[i - 1] jblob = javabridge.run_script( """ importPackage(Packages.org.cellprofiler.imageset); importPackage(Packages.org.cellprofiler.imageset.filter); var imageFile=new ImageFile(new java.net.URI(url)); var imageFileDetails = new ImageFileDetails(imageFile); var imageSeries=new ImageSeries(imageFile, 0); var imageSeriesDetails = new ImageSeriesDetails(imageSeries, imageFileDetails); var imagePlane=new ImagePlane(imageSeries, 0, ImagePlane.ALWAYS_MONOCHROME); var ipd = new ImagePlaneDetails(imagePlane, imageSeriesDetails); var stack = ImagePlaneDetailsStack.makeMonochromeStack(ipd); var stacks = java.util.Collections.singletonList(stack); var keys = java.util.Collections.singletonList(imageNumber); var imageSet = new ImageSet(stacks, keys); imageSet.compress(java.util.Collections.singletonList("DNA"), null); """, dict(url=url, imageNumber=str(i)), ) blob = javabridge.get_env().get_byte_array_elements(jblob) m[ cellprofiler_core.constants.measurement.IMAGE, cellprofiler_core.modules.namesandtypes.M_IMAGE_SET, i, blob.dtype, ] = blob pipeline = cellprofiler_core.pipeline.Pipeline() pipeline.loadtxt(six.moves.StringIO(GOOD_PIPELINE)) pipeline.write_pipeline_measurement(m) return m