def prepare_for_run(self):
"""Prepare to run pipeline.
This code was largely copied from 'PipelineController.start_debugging()'
with updates to the 'PipelineController' UI removed. I wish I had a
better understanding of what exactly this does, but I'm pretty much
using it as a black box for the time being.
"""
self.__measurements = Measurements()
self.__object_set = ObjectSet(can_overwrite=True)
self.__image_set_list = ImageSetList()
workspace = cellprofiler.gui._workspace_model.Workspace(
self.__pipeline,
None,
None,
None,
self.__measurements,
self.__image_set_list,
self.__frame,
)
try:
if not self.__pipeline.prepare_run(workspace):
print("Error: failed to get image sets")
self.__keys, self.__groupings = self.__pipeline.get_groupings(workspace)
except ValueError as v:
message = "Error while preparing for run:\n%s" % v
wx.MessageBox(
message, "Pipeline error", wx.OK | wx.ICON_ERROR, self.__frame
)
self.__grouping_index = 0
self.__within_group_index = 0
self.__pipeline.prepare_group(
workspace, self.__groupings[0][0], self.__groupings[0][1]
)
self.__outlines = {}
def write_schema(pipeline_filename):
if pipeline_filename is None:
raise ValueError(
"The --write-schema-and-exit switch must be used in conjunction\nwith the -p or --pipeline switch to load a pipeline with an\n"
"ExportToDatabase module.")
pipeline = Pipeline()
pipeline.load(pipeline_filename)
pipeline.turn_off_batch_mode()
for module in pipeline.modules():
if module.module_name == "ExportToDatabase":
break
else:
raise ValueError(
'The pipeline, "%s", does not have an ExportToDatabase module' %
pipeline_filename)
m = Measurements()
workspace = Workspace(pipeline, module, m, ObjectSet, m, None)
module.prepare_run(workspace)
def make_workspace(self, images, masks):
pipeline = Pipeline()
object_set = ObjectSet()
image_set_list = ImageSetList()
image_set = image_set_list.get_image_set(0)
module = Align()
workspace = Workspace(pipeline, module, image_set, object_set,
Measurements(), image_set_list)
for index, (pixels, mask) in enumerate(zip(images, masks)):
if mask is None:
image = Image(pixels)
else:
image = Image(pixels, mask=mask)
input_name = "Channel%d" % index
output_name = "Aligned%d" % index
image_set.add(input_name, image)
if index == 0:
module.first_input_image.value = input_name
module.first_output_image.value = output_name
elif index == 1:
module.second_input_image.value = input_name
module.second_output_image.value = output_name
else:
module.add_image()
ai = module.additional_images[-1]
ai.input_image_name.value = input_name
ai.output_image_name.value = output_name
return workspace, module
def print_groups(filename):
"""
Print the image set groups for this pipeline
This function outputs a JSON string to the console composed of a list
of the groups in the pipeline image set. Each element of the list is
a two-tuple whose first element is a key/value dictionary of the
group's key and the second is a tuple of the image numbers in the group.
"""
path = os.path.expanduser(filename)
m = Measurements(filename=path, mode="r")
metadata_tags = m.get_grouping_tags()
groupings = m.get_groupings(metadata_tags)
json.dump(groupings, sys.stdout)
def prepare_run(self, message, session_id, grouping_allowed=False):
"""Prepare a pipeline and measurements to run
message - the run-request or run-groups-request message
session_id - the session ID for the session
grouping_allowed - true to allow grouped images
"""
pipeline = cellprofiler_core.pipeline.Pipeline()
m = Measurements()
object_set = cellprofiler_core.object.ObjectSet()
if len(message) < 2:
self.raise_cellprofiler_exception(session_id,
"Missing run request sections")
return
pipeline_txt = message.pop(0).bytes
image_metadata = message.pop(0).bytes
try:
image_metadata = json.loads(image_metadata)
for channel_name, channel_metadata in image_metadata:
if len(message) < 1:
self.raise_cellprofiler_exception(
session_id,
"Missing binary data for channel %s" % channel_name)
return None, None, None
pixel_data = self.decode_image(
channel_metadata,
message.pop(0).bytes,
grouping_allowed=grouping_allowed,
)
m.add(channel_name, cellprofiler_core.image.Image(pixel_data))
except Exception as e:
logging.warning("Failed to decode message")
self.raise_cellprofiler_exception(session_id, e)
return None, None, None
try:
pipeline.loadtxt(StringIO(pipeline_txt))
except Exception as e:
logging.warning(
"Failed to load pipeline: sending pipeline exception")
self.raise_pipeline_exception(session_id, str(e))
return None, None, None
return pipeline, m, object_set
def print_groups(filename):
"""
Print the image set groups for this pipeline
This function outputs a JSON string to the console composed of a list
of the groups in the pipeline image set. Each element of the list is
a two-tuple whose first element is a key/value dictionary of the
group's key and the second is a tuple of the image numbers in the group.
"""
path = os.path.expanduser(filename)
m = Measurements(filename=path, mode="r")
metadata_tags = m.get_grouping_tags()
groupings = m.get_groupings(metadata_tags)
# Groupings are np.int64 which cannot be dumped to json
groupings_export = []
for g in groupings:
groupings_export.append((g[0], [int(imgnr) for imgnr in g[1]]))
json.dump(groupings_export, sys.stdout)
def save_pipeline(self, workspace, outf=None):
"""Save the pipeline in Batch_data.mat
Save the pickled image_set_list state in a setting and put this
module in batch mode.
if outf is not None, it is used as a file object destination.
"""
if outf is None:
if self.wants_default_output_directory.value:
path = get_default_output_directory()
else:
path = get_absolute_path(self.custom_output_directory.value)
os.makedirs(path, exist_ok=True)
h5_path = os.path.join(path, F_BATCH_DATA_H5)
else:
h5_path = outf
image_set_list = workspace.image_set_list
pipeline = workspace.pipeline
m = Measurements(copy=workspace.measurements, filename=h5_path)
try:
assert isinstance(pipeline, Pipeline)
assert isinstance(m, Measurements)
orig_pipeline = pipeline
pipeline = pipeline.copy()
# this use of workspace.frame is okay, since we're called from
# prepare_run which happens in the main wx thread.
target_workspace = Workspace(pipeline, None, None, None, m,
image_set_list, workspace.frame)
pipeline.prepare_to_create_batch(target_workspace, self.alter_path)
bizarro_self = pipeline.module(self.module_num)
bizarro_self.revision.value = int(
re.sub(r"\.|rc\d{1}", "", cellprofiler.__version__))
if self.wants_default_output_directory:
bizarro_self.custom_output_directory.value = self.alter_path(
get_default_output_directory())
bizarro_self.default_image_directory.value = self.alter_path(
get_default_image_directory())
bizarro_self.batch_mode.value = True
pipeline.write_pipeline_measurement(m)
orig_pipeline.write_pipeline_measurement(m, user_pipeline=True)
#
# Write the path mappings to the batch measurements
#
m.write_path_mappings([(mapping.local_directory.value,
mapping.remote_directory.value)
for mapping in self.mappings])
return h5_path
finally:
m.close()
def create(self):
"""Create a new workspace file
filename - name of the workspace file
"""
from ..utilities.measurement import make_temporary_file
from cellprofiler_core.measurement import Measurements
if isinstance(self.measurements, Measurements):
self.close()
fd, self.__filename = make_temporary_file()
self.__measurements = Measurements(filename=self.__filename, mode="w")
os.close(fd)
if self.__file_list is not None:
self.__file_list.remove_notification_callback(
self.__on_file_list_changed)
self.__file_list = HDF5FileList(self.measurements.hdf5_dict.hdf5_file)
self.__file_list.add_notification_callback(self.__on_file_list_changed)
self.notify(self.WorkspaceCreatedEvent(self))
def run_group_request(self, session_id, message_type, message):
"""Handle a run-group request message"""
pipeline = cellprofiler_core.pipeline.Pipeline()
m = Measurements()
image_group = m.hdf5_dict.hdf5_file.create_group("ImageData")
if len(message) < 2:
self.raise_cellprofiler_exception(session_id,
"Missing run request sections")
return
pipeline_txt = message.pop(0).bytes
image_metadata = message.pop(0).bytes
n_image_sets = None
try:
image_metadata = json.loads(image_metadata)
channel_names = []
for channel_name, channel_metadata in image_metadata:
channel_names.append(channel_name)
if len(message) < 1:
self.raise_cellprofiler_exception(
session_id,
"Missing binary data for channel %s" % channel_name)
return None, None, None
pixel_data = self.decode_image(channel_metadata,
message.pop(0).bytes,
grouping_allowed=True)
if pixel_data.ndim < 3:
self.raise_cellprofiler_exception(
session_id,
"The image for channel %s does not have a Z or T dimension",
)
return
if n_image_sets is None:
n_image_sets = pixel_data.shape[0]
elif n_image_sets != pixel_data.shape[0]:
self.raise_cellprofiler_exception(
session_id,
"The images passed have different numbers of Z or T planes",
)
return
image_group.create_dataset(channel_name, data=pixel_data)
except Exception as e:
self.raise_cellprofiler_exception(session_id, e)
return None, None, None
try:
pipeline.loadtxt(StringIO(pipeline_txt))
except Exception as e:
logging.warning(
"Failed to load pipeline: sending pipeline exception")
self.raise_pipeline_exception(session_id, str(e))
return
image_numbers = numpy.arange(1, n_image_sets + 1)
for image_number in image_numbers:
m["Image", GROUP_NUMBER, image_number, ] = 1
m["Image", GROUP_INDEX, image_number, ] = image_number
input_modules, other_modules = self.split_pipeline(pipeline)
workspace = cellprofiler_core.workspace.Workspace(
pipeline, None, m, None, m, None)
logging.info("Preparing group")
for module in other_modules:
module.prepare_group(
workspace,
dict([("image_number", i) for i in image_numbers]),
image_numbers,
)
for image_index in range(n_image_sets):
object_set = cellprofiler_core.object.ObjectSet()
m.next_image_set(image_index + 1)
for channel_name in channel_names:
dataset = image_group[channel_name]
pixel_data = dataset[image_index]
m.add(channel_name, Image(pixel_data))
for module in other_modules:
workspace = cellprofiler_core.workspace.Workspace(
pipeline, module, m, object_set, m, None)
try:
logging.info("Running module # %d: %s" %
(module.module_num, module.module_name))
pipeline.run_module(module, workspace)
if workspace.disposition in (
DISPOSITION_SKIP,
DISPOSITION_CANCEL,
):
break
except Exception as e:
msg = 'Encountered error while running module, "%s": %s' % (
module.module_name,
e,
)
logging.warning(msg)
self.raise_cellprofiler_exception(session_id, msg)
return
else:
continue
if workspace.disposition == DISPOSITION_CANCEL:
break
for module in other_modules:
module.post_group(
workspace, dict([("image_number", i) for i in image_numbers]))
logging.info("Finished group")
type_names, feature_dict = self.find_measurements(
other_modules, pipeline)
double_features = []
double_data = []
float_features = []
float_data = []
int_features = []
int_data = []
string_features = []
string_data = []
metadata = [
double_features, float_features, int_features, string_features
]
for object_name, features in list(feature_dict.items()):
df = []
double_features.append((object_name, df))
ff = []
float_features.append((object_name, ff))
intf = []
int_features.append((object_name, intf))
sf = []
string_features.append((object_name, sf))
if object_name == "Image":
object_counts = [] * n_image_sets
else:
object_numbers = m[object_name, OBJECT_NUMBER, image_numbers, ]
object_counts = [len(x) for x in object_numbers]
for feature, data_type in features:
if data_type == "java.lang.String":
continue
if not m.has_feature(object_name, feature):
data = numpy.zeros(numpy.sum(object_counts))
else:
data = m[object_name, feature, image_numbers]
temp = []
for i, (di, count) in enumerate(zip(data, object_counts)):
if count == 0:
continue
di = numpy.atleast_1d(di)
if len(di) > count:
di = di[:count]
elif len(di) == count:
temp.append(di)
else:
temp += [di + numpy.zeros(len(di) - count)]
if len(temp) > 0:
data = numpy.hstack(temp)
if type_names[data_type] == "java.lang.Double":
df.append((feature, len(data)))
if len(data) > 0:
double_data.append(data.astype("<f8"))
elif type_names[data_type] == "java.lang.Float":
ff.append((feature, len(data)))
if len(data) > 0:
float_data.append(data.astype("<f4"))
elif type_names[data_type] == "java.lang.Integer":
intf.append((feature, len(data)))
if len(data) > 0:
int_data.append(data.astype("<i4"))
data = numpy.hstack([
numpy.frombuffer(
numpy.ascontiguousarray(numpy.hstack(ditem)).data, numpy.uint8)
for ditem in (double_data, float_data, int_data) if len(ditem) > 0
])
data = numpy.ascontiguousarray(data)
self.socket.send_multipart([
zmq.Frame(session_id),
zmq.Frame(),
zmq.Frame(RUN_REPLY_1),
zmq.Frame(json.dumps(metadata)),
zmq.Frame(data),
])
def get_batch_commands(filename, n_per_job=1):
"""Print the commands needed to run the given batch data file headless
filename - the name of a Batch_data.h5 file. The file should group image sets.
The output assumes that the executable, "CellProfiler", can be used
to run the command from the shell. Alternatively, the output could be
run through a utility such as "sed":
CellProfiler --get-batch-commands Batch_data.h5 | sed s/CellProfiler/farm_job.sh/
"""
path = os.path.expanduser(filename)
m = Measurements(filename=path, mode="r")
image_numbers = m.get_image_numbers()
if m.has_feature(IMAGE, GROUP_NUMBER):
group_numbers = m[IMAGE, GROUP_NUMBER, image_numbers, ]
group_indexes = m[IMAGE, GROUP_INDEX, image_numbers, ]
if numpy.any(group_numbers != 1) and numpy.all(
(group_indexes[1:] == group_indexes[:-1] + 1)
| ((group_indexes[1:] == 1)
& (group_numbers[1:] == group_numbers[:-1] + 1))):
#
# Do -f and -l if more than one group and group numbers
# and indices are properly constructed
#
bins = numpy.bincount(group_numbers)
cumsums = numpy.cumsum(bins)
prev = 0
for i, off in enumerate(cumsums):
if off == prev:
continue
print("CellProfiler -c -r -p %s -f %d -l %d" %
(filename, prev + 1, off))
prev = off
else:
metadata_tags = m.get_grouping_tags()
if len(metadata_tags) == 1 and metadata_tags[0] == "ImageNumber":
for i in range(0, len(image_numbers), n_per_job):
first = image_numbers[i]
last = image_numbers[min(i + n_per_job - 1,
len(image_numbers) - 1)]
print("CellProfiler -c -r -p %s -f %d -l %d" %
(filename, first, last))
else:
# LoadData w/ images grouped by metadata tags
groupings = m.get_groupings(metadata_tags)
for grouping in groupings:
group_string = ",".join(
["%s=%s" % (k, v) for k, v in list(grouping[0].items())])
print("CellProfiler -c -r -p %s -g %s" %
(filename, group_string))
return
class ParameterSampleFrame(wx.Frame):
"""A frame to get sampling settings and to calculate samples for a
CellProfiler module.
A few important data members and data structures are maintained:
'self.__module': the module selected by the user.
'self.__pipeline': the pipeline containing the module.
'self.__parameters_list': a list of all unbounded parameters that will
be displayed to the user. Eg,
[(label_1, m_1), (label_2, m_2), ...], where
parameter i maps to the tuple (label_i, m_i) with 'label_i' its
text label and 'm_i' its 1-based index in 'self.__module'.
'self.__parameters_to_widgets_list': a list of lists representing the
mapping from parameters (see above), to 'groups of widgets'. Ie,
some parameters will have more than two values and, hence, will
have two groups of widgets. Eg,
[[0, 1], [[2]], [3, 4], represents a mapping
param 0 -> widget group 0 and 1, etc.
In practice, each such mapping will correspond exactly to a
check box and one or more rows of spin controls, depending on how
many values a particular parameter takes.
'self.__sample_list': a list of samples of the selected parameters. Eg,
suppose that parameters p_1, p_2 and p_3 are selected, then a list
of the form
[[s_11, s_21, s_31], [s_12, s_22, s_32], ...]
will be returned.
NB: for those parameters displayed, but not selected by the user, this
list will contain the value 'None'. Eg, suppose p2 is deselected, then
the list will have the form:
[[s_11, None, s_31], [s_12, None, s_32], ...].
"""
def __init__(
self,
parent,
module,
pipeline,
identifier=-1,
pos=wx.DefaultPosition,
size=wx.DefaultSize,
style=wx.DEFAULT_FRAME_STYLE,
name=wx.FrameNameStr,
):
# Flag to check during event handling: widgets generate events as they
# are created leading to referencing errors before all data structs
# have been initialized.
self.__initialized = False
# Init frame
self.__module = module
self.__pipeline = pipeline
self.__frame = parent
# Data members for running pipeline
self.__measurements = None
self.__object_set = None
self.__image_set_list = None
self.__keys = None
self.__groupings = None
self.__grouping_index = None
self.__within_group_index = None
self.__outlines = None
self.__grids = None
frame_title = "Sampling settings for module, " + self.__module.module_name
wx.Frame.__init__(
self, self.__frame, identifier, frame_title, pos, size, style, name
)
self.__frame_sizer = wx.BoxSizer(wx.VERTICAL)
self.SetSize(wx.Size(700, 350))
self.SetSizer(self.__frame_sizer)
# Get parameters
self.__parameters_list = self.get_parameters_list()
if len(self.__parameters_list) == 0:
dialog = wx.MessageDialog(
self,
self.__module.module_name + " has no unbounded parameters.",
caption="No unbounded parameters",
style=wx.OK,
)
dialog.ShowModal()
self.Close(True)
# Init settings panel
self.__settings_panel = wx.Panel(self, -1)
self.__settings_panel_sizer = wx.BoxSizer(wx.VERTICAL)
self.__settings_panel.SetSizer(self.__settings_panel_sizer)
self.__frame_sizer.Add(self.__settings_panel, 1, wx.EXPAND | wx.ALL, 5)
# Init settings scrolled window
self.__settings_scrolled_window = wx.ScrolledWindow(self.__settings_panel)
self.__settings_scrolled_window.SetScrollRate(20, 20)
self.__settings_scrolled_window.EnableScrolling(True, True)
self.__settings_scrolled_window_sizer = wx.FlexGridSizer(rows=0, cols=5)
self.__settings_scrolled_window.SetSizer(self.__settings_scrolled_window_sizer)
self.__settings_scrolled_window_sizer.AddGrowableCol(0)
self.__settings_panel_sizer.Add(
self.__settings_scrolled_window, 1, wx.EXPAND | wx.ALL, 5
)
# Headings
self.__settings_scrolled_window_sizer.Add(
wx.StaticText(self.__settings_scrolled_window, -1, "Parameter"),
0,
wx.ALIGN_CENTER | wx.ALIGN_CENTER_VERTICAL | wx.LEFT | wx.TOP,
5,
)
self.__settings_scrolled_window_sizer.Add(
wx.StaticText(self.__settings_scrolled_window, -1, "Current value"),
0,
wx.ALIGN_CENTER | wx.ALIGN_CENTER_VERTICAL | wx.LEFT | wx.TOP,
5,
)
self.__settings_scrolled_window_sizer.Add(
wx.StaticText(self.__settings_scrolled_window, -1, "Lower bound"),
0,
wx.ALIGN_CENTER | wx.ALIGN_CENTER_VERTICAL | wx.LEFT | wx.TOP,
5,
)
self.__settings_scrolled_window_sizer.Add(
wx.StaticText(self.__settings_scrolled_window, -1, "Upper bound"),
0,
wx.ALIGN_CENTER | wx.ALIGN_CENTER_VERTICAL | wx.LEFT | wx.TOP,
5,
)
self.__settings_scrolled_window_sizer.Add(
wx.StaticText(self.__settings_scrolled_window, -1, "Number samples"),
0,
wx.ALIGN_CENTER | wx.ALIGN_CENTER_VERTICAL | wx.LEFT | wx.TOP | wx.RIGHT,
5,
)
# Init dynamic widgets based on parameters
self.__parameters_to_widgets_list = self.get_parameters_to_widgets_list(
self.__parameters_list
)
self.__check_box_list = []
self.__current_static_text_list = []
self.__lower_bound_spin_ctrl_list = []
self.__upper_bound_spin_ctrl_list = []
self.__number_spin_ctrl_list = []
for i in range(len(self.__parameters_to_widgets_list)):
label = self.__parameters_list[i][0]
setting = self.__module.setting(self.__parameters_list[i][1])
value = setting.get_value()
for j in range(len(self.__parameters_to_widgets_list[i])):
# Checkbox & label
if j == 0:
check_box = wx.CheckBox(self.__settings_scrolled_window, -1, label)
check_box.Bind(wx.EVT_CHECKBOX, self.on_check_box)
self.__settings_scrolled_window_sizer.Add(
check_box,
0,
wx.ALIGN_LEFT | wx.ALIGN_CENTER_VERTICAL | wx.LEFT | wx.TOP,
5,
)
self.__check_box_list.append(check_box)
else:
self.__settings_scrolled_window_sizer.Add(
wx.Panel(self.__settings_scrolled_window, -1)
)
# Current value
if self.is_parameter_int(setting):
if self.get_parameter_input_size(setting) == 1:
cur_value = str(value)
elif self.get_parameter_input_size(setting) == 2:
cur_value = str(value[j])
elif self.is_parameter_float(setting):
if self.get_parameter_input_size(setting) == 1:
cur_value = str(round(value, FLOAT_DIGITS_TO_ROUND_TO))
elif self.get_parameter_input_size(setting) == 2:
cur_value = str(round(value[j], FLOAT_DIGITS_TO_ROUND_TO))
current_static_text = wx.StaticText(
self.__settings_scrolled_window, -1, cur_value
)
self.__settings_scrolled_window_sizer.Add(
current_static_text,
0,
wx.ALIGN_CENTER_VERTICAL | wx.LEFT | wx.TOP | wx.RIGHT,
5,
)
self.__current_static_text_list.append(current_static_text)
# Lower and upper bounds
if self.is_parameter_int(setting):
# Integer
lower_spin_ctrl = wx.SpinCtrl(
self.__settings_scrolled_window, wx.NewId()
)
lower_spin_ctrl.SetRange(INT_LOWER_BOUND, INT_UPPER_BOUND)
lower_spin_ctrl.Bind(wx.EVT_SPINCTRL, self.on_lower_spin_ctrl)
upper_spin_ctrl = wx.SpinCtrl(
self.__settings_scrolled_window, wx.NewId()
)
upper_spin_ctrl.SetRange(INT_LOWER_BOUND, INT_UPPER_BOUND)
upper_spin_ctrl.Bind(wx.EVT_SPINCTRL, self.on_upper_spin_ctrl)
interval = (
DEFAULT_NUMBER_SAMPLES - 1
) # Used later to set upper bound
elif self.is_parameter_float(setting):
# Float
lower_spin_ctrl = wx.lib.agw.floatspin.FloatSpin(
self.__settings_scrolled_window, wx.NewId(), size=wx.DefaultSize
) # NB: if not set, spin buttons are hidden
lower_spin_ctrl.SetDigits(FLOAT_DIGITS_TO_ROUND_TO)
lower_spin_ctrl.SetIncrement(10 ** -FLOAT_DIGITS_TO_ROUND_TO)
lower_spin_ctrl.SetRange(FLOAT_LOWER_BOUND, FLOAT_UPPER_BOUND)
lower_spin_ctrl.Bind(
wx.lib.agw.floatspin.EVT_FLOATSPIN, self.on_lower_float_spin
)
upper_spin_ctrl = wx.lib.agw.floatspin.FloatSpin(
self.__settings_scrolled_window, wx.NewId(), size=wx.DefaultSize
) # NB: if not set, spin buttons are hidden
upper_spin_ctrl.SetDigits(FLOAT_DIGITS_TO_ROUND_TO)
upper_spin_ctrl.SetIncrement(10 ** -FLOAT_DIGITS_TO_ROUND_TO)
upper_spin_ctrl.SetRange(FLOAT_LOWER_BOUND, FLOAT_UPPER_BOUND)
upper_spin_ctrl.Bind(
wx.lib.agw.floatspin.EVT_FLOATSPIN, self.on_upper_float_spin
)
interval = (
upper_spin_ctrl.GetIncrement()
) # Used later to set upper bound
if self.get_parameter_input_size(setting) == 1:
lower_spin_ctrl.SetValue(
value
) # Set value after range to avoid rounding
upper_spin_ctrl.SetValue(value + interval)
elif self.get_parameter_input_size(setting) == 2:
lower_spin_ctrl.SetValue(value[j])
upper_spin_ctrl.SetValue(value[j] + interval)
lower_spin_ctrl.Enable(False)
self.__settings_scrolled_window_sizer.Add(
lower_spin_ctrl, 0, wx.EXPAND | wx.ALIGN_LEFT | wx.LEFT | wx.TOP, 5
)
self.__lower_bound_spin_ctrl_list.append(lower_spin_ctrl)
upper_spin_ctrl.Enable(False)
self.__settings_scrolled_window_sizer.Add(
upper_spin_ctrl, 0, wx.LEFT | wx.TOP, 5
)
self.__upper_bound_spin_ctrl_list.append(upper_spin_ctrl)
# Number samples
num_spin_ctrl = wx.SpinCtrl(self.__settings_scrolled_window, wx.NewId())
num_spin_ctrl.Enable(False)
num_spin_ctrl.SetRange(1, 100)
num_spin_ctrl.SetValue(10)
num_spin_ctrl.Bind(wx.EVT_SPINCTRL, self.on_number_spin_ctrl)
self.__settings_scrolled_window_sizer.Add(
num_spin_ctrl, 0, wx.LEFT | wx.TOP | wx.RIGHT, 5
)
self.__number_spin_ctrl_list.append(num_spin_ctrl)
self.__settings_scrolled_window.Layout()
self.__settings_panel.Layout()
# Add button
self.sample_button = wx.Button(self, ID_SAMPLE_BUTTON, "Sample")
self.sample_button.Bind(wx.EVT_BUTTON, self.on_button)
self.sample_button.Enable(False)
self.__frame_sizer.Add(self.sample_button, 0, wx.ALIGN_RIGHT | wx.ALL, 5)
self.__initialized = True
def get_parameters_list(self):
"""Get and return the list of unbounded parameters.
This method first identifies all unbounded parameters in 'module'.
Next, it updates a list of mappings to tuples:
a parameter's 0-based index -> (its label, its 1-based module index),
eg, 0 -> (Crop, 2); 1 -> (Crop, 3); 2 -> (IdentifyPrimaryObjects, 4).
"""
parameters_list = []
# Get unique keys of visible parameters
visible_settings_keys = []
for setting in self.__module.visible_settings():
visible_settings_keys.append(setting.key())
# Update dictionary of visible, unbounded parameters:
# 0-based index in 'parameters_list' -> (label, 1-based module number)
for i, setting in enumerate(self.__module.settings()):
if (
setting.key() in visible_settings_keys
and self.get_parameter_type(setting) == PARAM_CLASS_UNBOUNDED
):
parameters_list.append((setting.get_text(), i + 1))
return parameters_list
def get_parameters_to_widgets_list(self, parameters_list):
"""Create mapping from parameters to widgets.
Some parameters have two values, requiring two sets of widgets. This
method updates 'parameters_to_widgets_list', a mapping from:
a parameter's 0-based index -> its widget groups' 0-based indices,
eg, 0 -> [0, 1]; 1 -> [2]; 2 -> [3, 4].
The indices on the left will correspond to check boxes and the indices
on the right will correspond to widgets to the right of the check
boxes. This method assumes that 'parameters_list' is correctly
initialized (see 'get_parameters_list()').
"""
parameters_to_widgets_list = []
index_count = 0
for i in range(len(parameters_list)):
setting = self.__module.setting(parameters_list[i][1])
input_size = self.get_parameter_input_size(setting)
widget_indices = []
for j in range(input_size):
widget_indices.append(index_count)
index_count += 1
parameters_to_widgets_list.append(widget_indices)
return parameters_to_widgets_list
@staticmethod
def get_parameter_type(setting):
"""Get parameter type of 'setting' by considering its class."""
if isinstance(setting, Binary):
return PARAM_CLASS_BOUNDED_DISCRETE
elif isinstance(setting, Choice):
return PARAM_CLASS_BOUNDED_DISCRETE
elif isinstance(setting, Divider):
return PARAM_CLASS_DECORATION
elif isinstance(setting, DoSomething):
return PARAM_CLASS_DECORATION
elif isinstance(setting, FigureSubscriber):
return PARAM_CLASS_BOUNDED_DISCRETE
elif isinstance(setting, Float):
return PARAM_CLASS_UNBOUNDED
elif isinstance(setting, FloatRange):
return PARAM_CLASS_UNBOUNDED
elif isinstance(setting, Integer):
return PARAM_CLASS_UNBOUNDED
elif isinstance(setting, IntegerRange):
return PARAM_CLASS_UNBOUNDED
elif isinstance(setting, IntegerOrUnboundedRange):
return PARAM_CLASS_UNBOUNDED
elif isinstance(setting, Measurement):
# NB: Not sure what to do with 'Measurement' yet
return "not sure"
elif isinstance(setting, Name):
return PARAM_CLASS_TEXT_LABEL
elif isinstance(setting, ImageSubscriber):
return PARAM_CLASS_BOUNDED_DISCRETE
elif isinstance(setting, RemoveSettingButton):
return PARAM_CLASS_DECORATION
elif isinstance(setting, Text):
return PARAM_CLASS_TEXT_LABEL
else:
return "whatever is left"
@staticmethod
def get_parameter_input_size(setting):
"""Get input size of 'setting'."""
size = 1
try:
size = len(setting.get_value())
except:
pass
return size
@staticmethod
def is_parameter_float(setting):
if isinstance(setting, Float):
return True
elif isinstance(setting, FloatRange):
return True
else:
return False
@staticmethod
def is_parameter_int(setting):
if isinstance(setting, Integer):
return True
elif isinstance(setting, IntegerRange):
return True
elif isinstance(setting, IntegerOrUnboundedRange):
return True
else:
return False
def validate_input_ranges(self):
"""Validates input ranges specified by the input widgets.
This method uses CellProfiler's validate methods on settings to test
whether the ranges specified on the UI are acceptable. When these do
not validate, the user is shown an error message that lists all
violations as well as the error message received back from CellProfiler.
"""
# First, test whether selected parameters are valid
message = ""
for i, checkbox in enumerate(self.__check_box_list):
if checkbox.IsChecked():
setting = self.__module.setting(self.__parameters_list[i][1])
input_size = self.get_parameter_input_size(setting)
if input_size == 1:
widget_idx = self.__parameters_to_widgets_list[i][0]
lower_value = self.__lower_bound_spin_ctrl_list[
widget_idx
].GetValue()
upper_value = self.__upper_bound_spin_ctrl_list[
widget_idx
].GetValue()
elif input_size == 2:
widget_idx = self.__parameters_to_widgets_list[i][0]
lower_value = (
self.__lower_bound_spin_ctrl_list[widget_idx].GetValue(),
self.__lower_bound_spin_ctrl_list[widget_idx + 1].GetValue(),
)
upper_value = (
self.__upper_bound_spin_ctrl_list[widget_idx].GetValue(),
self.__upper_bound_spin_ctrl_list[widget_idx + 1].GetValue(),
)
old_value = setting.get_value()
try:
setting.set_value(lower_value)
setting.test_valid(self.__pipeline)
except ValidationError as instance:
message += (
"'"
+ str(setting.get_text())
+ "': lower bound invalid, "
+ "\n\t"
+ str(instance.message)
+ "\n"
)
try:
setting.set_value(upper_value)
setting.test_valid(self.__pipeline)
except ValidationError as instance:
message += (
"'"
+ str(setting.get_text())
+ "': upper bound invalid, "
+ "\n\t"
+ str(instance.message)
+ "\n"
)
setting.set_value(old_value)
# Second, if there are invalid parameters, tell the user
if len(message) > 0:
message = "Invalid sample settings:\n\n" + message
dialog = wx.MessageDialog(
self,
message,
caption="Sample settings error",
style=wx.ICON_ERROR | wx.OK,
)
dialog.ShowModal()
raise Exception(message)
def generate_parameter_samples(self):
"""Compute samples values for the selected parameters.
This method returns a list of samples for the selected parameters by
computing all possible combinations of sample values that lie within
the ranges specified by the user. Eg, suppose p1, p2, p3 and p4 are
selected, a list of the form
[[s11, s21, s31, s41], [s12, s22, s32, s42], ...]
will be returned.
NB: for those parameters displayed, but not selected by the user, the
value 'None' will be returned. Eg,
suppose p2 is deselected, then a list of the form:
[[s11, None, s31, s41], [s12, None, s32, s42], ...]
will be returned.
"""
sample_list = []
# First, compute a list of samples for every selected parameter
for i, checkbox in enumerate(self.__check_box_list):
samples = []
if checkbox.IsChecked():
number = self.__parameters_list[i][1]
setting = self.__module.setting(number)
if self.get_parameter_input_size(setting) == 1:
# Parameters with single value
index = self.__parameters_to_widgets_list[i][0]
lower_bound = self.__lower_bound_spin_ctrl_list[index].GetValue()
upper_bound = self.__upper_bound_spin_ctrl_list[index].GetValue()
number_samples = self.__number_spin_ctrl_list[index].GetValue()
samples = self.compute_samples(
lower_bound,
upper_bound,
number_samples,
self.is_parameter_int(setting),
)
else:
# Parameters with tuple values
samples_temp = []
for j in self.__parameters_to_widgets_list[i]:
lower_bound = self.__lower_bound_spin_ctrl_list[j].GetValue()
upper_bound = self.__upper_bound_spin_ctrl_list[j].GetValue()
number_samples = self.__number_spin_ctrl_list[j].GetValue()
samples_temp.append(
self.compute_samples(
lower_bound,
upper_bound,
number_samples,
self.is_parameter_int(setting),
)
)
samples_temp = list(self.cartesian_product(samples_temp))
tuples = []
for l in samples_temp:
tuples.append(tuple(l))
samples = tuples
else:
samples.append(None)
sample_list.append(samples)
# Second, compute all possible combinations of the lists of samples
sample_list = list(self.cartesian_product(sample_list))
return sample_list
def calc_number_samples(self):
"""Computes number of samples.
This method computes the number of samples that will be generated
with the current UI settings without computing the samples themselves.
"""
number = 0
for i, check_box in enumerate(self.__check_box_list):
if check_box.IsChecked():
for j in self.__parameters_to_widgets_list[i]:
if number == 0:
number = self.__number_spin_ctrl_list[j].GetValue()
else:
number *= self.__number_spin_ctrl_list[j].GetValue()
return number
@staticmethod
def compute_samples(lower_bound, upper_bound, number_samples, is_int):
"""Computes samples in the range [lower_bound, upper_bound].
This method computes an returns a list of uniformly distributed samples
in the range [lower_bound, upper_bound]. The returned list will be of
size 'number_samples'. If 'is_int' is true, samples will be rounded to
the nearest integer, except for the last, which will be rounded to
'upper_bound'.
"""
samples = []
if number_samples > 1:
delta = (upper_bound - lower_bound) / float(number_samples - 1)
for i in range(number_samples):
sample = lower_bound + i * delta
if is_int:
if i == number_samples:
sample = upper_bound
else:
sample = int(sample)
samples.append(sample)
else:
samples.append(lower_bound)
return samples
def cartesian_product(self, list_of_lists):
"""Computes all combinations of lists in 'list_of_lists'.
Eg, [[1, 2],[3, 4]] -> [[1, 3], [1, 4], [2, 3], [2, 4]]
As this functionality is provided by 'itertools.product()' in
Python 2.6 and above, the implementation below was just grabbed
off the web as a temporary solution:
<http://stackoverflow.com/questions/2419370/
how-can-i-compute-a-cartesian-product-iteratively>
If one of the lists in 'list_of_lists' is empty - eg,
[[1, 2], [], [3, 4]] - the method will not work.
"""
if not list_of_lists:
yield []
else:
for item in list_of_lists[0]:
for product in self.cartesian_product(list_of_lists[1:]):
yield [item] + product
def prepare_for_run(self):
"""Prepare to run pipeline.
This code was largely copied from 'PipelineController.start_debugging()'
with updates to the 'PipelineController' UI removed. I wish I had a
better understanding of what exactly this does, but I'm pretty much
using it as a black box for the time being.
"""
self.__measurements = Measurements()
self.__object_set = ObjectSet(can_overwrite=True)
self.__image_set_list = ImageSetList()
workspace = cellprofiler.gui._workspace_model.Workspace(
self.__pipeline,
None,
None,
None,
self.__measurements,
self.__image_set_list,
self.__frame,
)
try:
if not self.__pipeline.prepare_run(workspace):
print("Error: failed to get image sets")
self.__keys, self.__groupings = self.__pipeline.get_groupings(workspace)
except ValueError as v:
message = "Error while preparing for run:\n%s" % v
wx.MessageBox(
message, "Pipeline error", wx.OK | wx.ICON_ERROR, self.__frame
)
self.__grouping_index = 0
self.__within_group_index = 0
self.__pipeline.prepare_group(
workspace, self.__groupings[0][0], self.__groupings[0][1]
)
self.__outlines = {}
def run_module(self, module):
"""Run pipeline with 'module'.
This code was largely copied from 'PipelineController.do_step()' with
updates to the 'PipelineController' UI removed. As with the above method,
I wish I had a better understanding of what exactly this does, but I'm
pretty much using it as a black box.
Assumption: 'self.prepare_for_run()' was called first.
"""
failure = 1
try:
# ~*~
# workspace = cellprofiler.gui.workspace.Workspace(
# self.__pipeline, module, image_set, self.__object_set,
# self.__measurements, self.__image_set_list,
# # Uncomment next line to display results in UI
# #self.__frame if module.show_window else None,
# None,
# outlines = self.__outlines)
self.__workspace = cellprofiler.gui._workspace_model.Workspace(
self.__pipeline,
module,
self.__measurements,
self.__object_set,
self.__measurements,
self.__image_set_list,
# Uncomment next line to display results in UI
# self.__frame if module.show_window else None,
None,
outlines=self.__outlines,
)
# self.__grids = workspace.set_grids(self.__grids)
self.__grids = self.__workspace.set_grids(self.__grids)
# ~^~
# print 'Running ' + str(module.get_module_num()) + ': ' + str(module.module_name)
# ~*~
# module.run(workspace)
module.run(self.__workspace)
# ~^~
# ~*~
# workspace.refresh()
self.__workspace.refresh()
# ~^~
failure = 0
except Exception as instance:
traceback.print_exc()
event = RunException(instance, module)
self.__pipeline.notify_listeners(event)
failure = 1
if (
module.module_name != "Restart" or failure == -1
) and self.__measurements is not None:
module_error_measurement = "ModuleError_%02d%s" % (
module.module_num,
module.module_name,
)
self.__measurements.add_measurement(
"Image", module_error_measurement, failure
)
return failure == 0
def save_run_output(self, sample_num, directory_path, output_file):
"""Save the parameter settings and images for the current run of
'self.__module'."""
for i, setting in enumerate(self.__module.visible_settings()):
value_to_write = ""
# Do not write settings without values, ie, buttons etc
if setting.get_text() != "":
value_to_write = str(setting.get_value())
if isinstance(setting, ImageName):
# Save image
image = self.__measurements.get_image(value_to_write)
path = os.path.join(
directory_path, value_to_write + "_" + str(sample_num)
)
self.save_image(image, path)
value_to_write += "_" + str(sample_num) + ".jpg"
# ~*~
# elif isinstance(setting, settings.ObjectNameProvider):
# print 'Bingo! ' + str(value_to_write)
# objects =\
# self.__workspace.get_object_set().get_objects(setting.get_value())
# value_to_write = str(len(objects.indices))
# ~^~
if i < len(self.__module.visible_settings()) - 1 and value_to_write != "":
value_to_write += "\t"
if i == len(self.__module.visible_settings()) - 1:
value_to_write += "\n"
# Write line to 'open_file'
if value_to_write != "":
output_file.write(value_to_write)
# ~*~
# Fiddle around with cpimage.ImageSet
# -> This might be a neater way of getting at images
# print 'ImageSet:'
# for name in self.__workspace.get_image_set().get_names():
# print '\t' + str(name)
# image = self.__workspace.get_image_set().get_image(name)
# path =\
# os.path.join(directory_path, name)
# self.save_image(image, path)
# Fiddle around with objects.ObjectSet
# -> This might be the way to get at the number of objects detected
# print 'ObjectSet:'
# for name in self.__workspace.get_object_set().get_object_names():
# print '\t' + str(name)
# objects = self.__workspace.get_object_set().get_objects(name)
# print '\t\t' + str(objects.indices)
# #parent_image = objects.get_parent_image()
# #path = \
# # os.path.join(directory_path, 'tootie')
# #self.save_image(image, path)
# for object_name in self.__measurements.get_object_names():
# print object_name
# for feature_name in self.__measurements.get_feature_names(object_name):
# print '\t' + str(feature_name)
# current_measurement = self.__measurements.get_current_measurement(
# object_name, feature_name)
# print '\t\t' + str(current_measurement)
# ~^~
@staticmethod
def save_image(image, path):
"""TODO: add comments"""
pixels = image.pixel_data
if numpy.max(pixels) > 1 or numpy.min(pixels) < 0:
pixels = pixels.copy()
pixels[pixels < 0] = 0
pixels[pixels > 1] = 1
pixels = (pixels * 255).astype(numpy.uint8)
pathname = "{}.jpg".format(path)
skimage.io.imsave(pathname, pixels)
def on_check_box(self, event):
button_flag = False
for i, checkbox in enumerate(self.__check_box_list):
if checkbox.IsChecked():
for j in self.__parameters_to_widgets_list[i]:
self.__lower_bound_spin_ctrl_list[j].Enable(True)
self.__upper_bound_spin_ctrl_list[j].Enable(True)
self.__number_spin_ctrl_list[j].Enable(True)
button_flag = True
else:
for j in self.__parameters_to_widgets_list[i]:
self.__lower_bound_spin_ctrl_list[j].Enable(False)
self.__upper_bound_spin_ctrl_list[j].Enable(False)
self.__number_spin_ctrl_list[j].Enable(False)
self.sample_button.Enable(button_flag)
def on_lower_spin_ctrl(self, event):
if self.__initialized:
for i in range(len(self.__parameters_to_widgets_list)):
for j in self.__parameters_to_widgets_list[i]:
if event.GetId() == self.__lower_bound_spin_ctrl_list[j].GetId():
lower_bound = self.__lower_bound_spin_ctrl_list[j].GetValue()
upper_bound = self.__upper_bound_spin_ctrl_list[j].GetValue()
increment = self.__number_spin_ctrl_list[j].GetValue() - 1
if upper_bound < lower_bound + increment:
self.__upper_bound_spin_ctrl_list[j].SetValue(
lower_bound + increment
)
def on_upper_spin_ctrl(self, event):
if self.__initialized:
for i in range(len(self.__parameters_to_widgets_list)):
for j in self.__parameters_to_widgets_list[i]:
if event.GetId() == self.__upper_bound_spin_ctrl_list[j].GetId():
lower_bound = self.__lower_bound_spin_ctrl_list[j].GetValue()
upper_bound = self.__upper_bound_spin_ctrl_list[j].GetValue()
increment = self.__number_spin_ctrl_list[j].GetValue() - 1
if upper_bound < lower_bound + increment:
self.__lower_bound_spin_ctrl_list[j].SetValue(
upper_bound - increment
)
def on_lower_float_spin(self, event):
if self.__initialized:
for i in range(len(self.__parameters_to_widgets_list)):
for j in self.__parameters_to_widgets_list[i]:
if event.GetId() == self.__lower_bound_spin_ctrl_list[j].GetId():
lower_bound = self.__lower_bound_spin_ctrl_list[j].GetValue()
upper_bound = self.__upper_bound_spin_ctrl_list[j].GetValue()
increment = self.__upper_bound_spin_ctrl_list[j].GetIncrement()
if upper_bound < lower_bound + increment:
self.__upper_bound_spin_ctrl_list[j].SetValue(
lower_bound + increment
)
def on_upper_float_spin(self, event):
if self.__initialized:
for i in range(len(self.__parameters_to_widgets_list)):
for j in self.__parameters_to_widgets_list[i]:
if event.GetId() == self.__upper_bound_spin_ctrl_list[j].GetId():
lower_bound = self.__lower_bound_spin_ctrl_list[j].GetValue()
upper_bound = self.__upper_bound_spin_ctrl_list[j].GetValue()
increment = self.__upper_bound_spin_ctrl_list[j].GetIncrement()
if upper_bound < lower_bound + increment:
self.__lower_bound_spin_ctrl_list[j].SetValue(
upper_bound - increment
)
def on_number_spin_ctrl(self, event):
if self.__initialized:
for i in range(len(self.__parameters_to_widgets_list)):
setting = self.__module.setting(i)
if self.is_parameter_int(setting):
for j in self.__parameters_to_widgets_list[i]:
if event.GetId() == self.__number_spin_ctrl_list[j].GetId():
lower_bound = self.__lower_bound_spin_ctrl_list[
j
].GetValue()
upper_bound = self.__upper_bound_spin_ctrl_list[
j
].GetValue()
number = self.__number_spin_ctrl_list[j].GetValue()
diff = upper_bound - lower_bound
if diff < number:
self.__upper_bound_spin_ctrl_list[j].SetValue(
lower_bound + (number - 1)
)
def on_button(self, event):
if event.GetId() == ID_SAMPLE_BUTTON:
# try:
self.validate_input_ranges()
number = self.calc_number_samples()
sample_dialog = wx.MessageDialog(
self,
"Proceed with calculating " + str(number) + " samples?",
caption="Confirm sample size",
style=wx.ICON_QUESTION | wx.OK | wx.CANCEL,
)
if sample_dialog.ShowModal() == wx.ID_OK:
save_dialog = wx.FileDialog(
event.GetEventObject(),
message="Save sampled output",
wildcard="Tab separated values (*.tsv)|*.tsv",
style=wx.FD_SAVE | wx.FD_OVERWRITE_PROMPT,
)
if save_dialog.ShowModal() == wx.ID_OK:
# 1. Copy original parameter values
original_values = []
for setting in self.__module.visible_settings():
original_values.append(setting.get_value())
# 2. Sample input parameters
self.__sample_list = self.generate_parameter_samples()
# 3. Open output file and write headers
output_file = open(save_dialog.GetPath(), "w")
headers = ""
for i, setting in enumerate(self.__module.visible_settings()):
# Do not write settings without values, ie, buttons etc
if setting.get_text() != "":
headers += setting.get_text()
if i < len(self.__module.visible_settings()) - 1:
headers += "\t"
headers += "\n"
output_file.write(headers)
# 4. Run pipeline once for each sample
# ~*~
self.Show(False)
progressDialog = wx.ProgressDialog(
parent=self,
title="Sampling parameters",
message="Run 1",
maximum=len(self.__sample_list),
)
size = progressDialog.GetSize()
size.SetWidth(2 * size.GetWidth())
progressDialog.SetSize(size)
# ~^~
for i, l in enumerate(self.__sample_list):
# print '\nStarting run ' + str(i+1) + '...'
for j, value in enumerate(l):
if value is not None:
setting_nr = self.__parameters_list[j][1]
setting = self.__module.setting(setting_nr)
setting.set_value(value)
# print str(setting.get_text()) + ' -> ' + str(setting.get_value())
# ~*~
progressDialog.Update(
i + 1,
newmsg="Executing run "
+ str(i + 1)
+ " of "
+ str(len(self.__sample_list)),
)
# ~^~
# It's not very efficient to run the complete pipeline
# when only the last module's parameter values are
# different. However, this is the only way I can get
# the images to update correctly for the last module. Ie,
# when I don't prepare and run the pipeline from scratch
# for every different configuration of the last module,
# I get the images generated by the first run for every
# run.
# 4.1 Prepare to run pipeline
self.prepare_for_run()
# 4.2 Run modules
for module in self.__pipeline.modules():
if (
module.get_module_num()
<= self.__module.get_module_num()
):
self.run_module(module)
# 4.3 Save output
self.save_run_output(i, save_dialog.GetDirectory(), output_file)
# This is the way to run headless, if only I could get at the images...
# self.stop_now = False
# running_pipeline = self.__pipeline.run_with_yield(
# run_in_background=False,
# status_callback=self.status_callback)
# while not self.stop_now:
# measurements = running_pipeline.next()
# print '...run completed.'
# 5. Close output file
output_file.close()
# ~*~
progressDialog.Destroy()
# ~^~
# 6. Set parameters back to original values and close window
for i, setting in enumerate(self.__module.visible_settings()):
setting.set_value(original_values[i])
self.Close(True)