def test_load_filename():
#
# Load a file, only specifying the FileName in the CSV
#
csv_text = (""""Image_FileName_DNA"
"%s"
""" % test_filename)
pipeline, module, filename = make_pipeline(csv_text)
assert isinstance(module, cellprofiler.modules.loaddata.LoadData)
module.image_directory.dir_choice = cellprofiler_core.setting.ABSOLUTE_FOLDER_NAME
module.image_directory.custom_path = test_path
m = cellprofiler_core.measurement.Measurements()
workspace = cellprofiler_core.workspace.Workspace(
pipeline,
module,
m,
cellprofiler_core.object.ObjectSet(),
m,
cellprofiler_core.image.ImageSetList(),
)
assert module.prepare_run(workspace)
assert (m.get_measurement(cellprofiler_core.measurement.IMAGE,
"FileName_DNA", 1) == test_filename)
path = m.get_measurement(cellprofiler_core.measurement.IMAGE,
"PathName_DNA", 1)
assert path == test_path
assert m.get_measurement(
cellprofiler_core.measurement.IMAGE, "URL_DNA",
1) == cellprofiler_core.modules.loadimages.pathname2url(
os.path.join(test_path, test_filename))
module.prepare_group(workspace, {}, [1])
module.run(workspace)
img = workspace.image_set.get_image("DNA", must_be_grayscale=True)
assert tuple(img.pixel_data.shape) == test_shape
def test_get_groupings():
"""Test the get_groupings method"""
dir = os.path.join(tests.modules.example_images_directory(),
"ExampleSBSImages")
pattern = "Channel1-[0-9]{2}-(?P<ROW>[A-H])-(?P<COL>[0-9]{2})\\.tif"
csv_text = '"Image_FileName_Cytoplasm","Image_PathName_Cytoplasm","Metadata_ROW","Metadata_COL"\n'
for filename in os.listdir(dir):
match = re.match(pattern, filename)
if match:
csv_text += '"%s","%s","%s","%s"\n' % (
filename,
dir,
match.group("ROW"),
match.group("COL"),
)
pipeline, module, filename = make_pipeline(csv_text)
assert isinstance(module, cellprofiler.modules.loaddata.LoadText)
module.wants_images.value = True
module.wants_image_groupings.value = True
module.metadata_fields.value = "ROW"
image_set_list = cellprofiler_core.image.ImageSetList()
measurements = cellprofiler_core.measurement.Measurements()
workspace = cellprofiler_core.workspace.Workspace(pipeline, module, None,
None, measurements,
image_set_list)
module.prepare_run(workspace)
keys, groupings = module.get_groupings(workspace)
assert len(keys) == 1
assert keys[0] == "Metadata_ROW"
assert len(groupings) == 8
my_rows = [g[0]["Metadata_ROW"] for g in groupings]
my_rows.sort()
assert "".join(my_rows) == "ABCDEFGH"
for grouping in groupings:
row = grouping[0]["Metadata_ROW"]
module.prepare_group(
cellprofiler_core.workspace.Workspace(pipeline, module, None, None,
measurements,
image_set_list),
grouping[0],
grouping[1],
)
for image_number in grouping[1]:
image_set = image_set_list.get_image_set(image_number - 1)
measurements.next_image_set(image_number)
workspace = cellprofiler_core.workspace.Workspace(
pipeline,
module,
image_set,
cellprofiler_core.object.ObjectSet(),
measurements,
image_set_list,
)
module.run(workspace)
provider = image_set.get_image_provider("Cytoplasm")
match = re.search(pattern, provider.get_filename())
assert match
assert row == match.group("ROW")
def test_load_objects():
r = numpy.random.RandomState()
r.seed(1101)
labels = r.randint(0, 10, size=(30, 20)).astype(numpy.uint8)
handle, name = tempfile.mkstemp(".png")
bioformats.write_image(name, labels, bioformats.PT_UINT8)
os.close(handle)
png_path, png_file = os.path.split(name)
sbs_dir = os.path.join(tests.modules.example_images_directory(),
"ExampleSBSImages")
csv_text = """%s_%s,%s_%s,%s_DNA,%s_DNA
%s,%s,Channel2-01-A-01.tif,%s
""" % (
cellprofiler_core.measurement.C_OBJECTS_FILE_NAME,
OBJECTS_NAME,
cellprofiler_core.measurement.C_OBJECTS_PATH_NAME,
OBJECTS_NAME,
cellprofiler_core.measurement.C_FILE_NAME,
cellprofiler_core.measurement.C_PATH_NAME,
png_file,
png_path,
sbs_dir,
)
pipeline, module, csv_name = make_pipeline(csv_text)
assert isinstance(pipeline, cellprofiler_core.pipeline.Pipeline)
assert isinstance(module, cellprofiler.modules.loaddata.LoadData)
module.wants_images.value = True
try:
image_set_list = cellprofiler_core.image.ImageSetList()
measurements = cellprofiler_core.measurement.Measurements()
workspace = cellprofiler_core.workspace.Workspace(
pipeline, module, None, None, measurements, image_set_list)
pipeline.prepare_run(workspace)
key_names, g = pipeline.get_groupings(workspace)
assert len(g) == 1
module.prepare_group(workspace, g[0][0], g[0][1])
image_set = image_set_list.get_image_set(g[0][1][0] - 1)
object_set = cellprofiler_core.object.ObjectSet()
workspace = cellprofiler_core.workspace.Workspace(
pipeline, module, image_set, object_set, measurements,
image_set_list)
module.run(workspace)
objects = object_set.get_objects(OBJECTS_NAME)
assert numpy.all(objects.segmented == labels)
assert (measurements.get_current_image_measurement(
cellprofiler_core.measurement.FF_COUNT % OBJECTS_NAME) == 9)
for feature in (
cellprofiler_core.measurement.M_LOCATION_CENTER_X,
cellprofiler_core.measurement.M_LOCATION_CENTER_Y,
cellprofiler_core.measurement.M_NUMBER_OBJECT_NUMBER,
):
value = measurements.get_current_measurement(OBJECTS_NAME, feature)
assert len(value) == 9
finally:
bioformats.formatreader.clear_image_reader_cache()
os.remove(name)
os.remove(csv_name)
def test_extra_fields():
#
# Regression test of issue #853, extra fields
#
csv_text = """"Image_URL_DNA"
"{cp_logo_url}", "foo"
"http:{cp_logo_url_filename}"
"bogusurl.png"
""".format(
**{
"cp_logo_url": tests.modules.cp_logo_url,
"cp_logo_url_filename": tests.modules.cp_logo_url_filename,
}
)
pipeline, module, filename = make_pipeline(csv_text)
assert isinstance(module, cellprofiler.modules.loaddata.LoadData)
m = cellprofiler_core.measurement.Measurements()
workspace = cellprofiler_core.workspace.Workspace(
pipeline,
module,
m,
cellprofiler_core.object.ObjectSet(),
m,
cellprofiler_core.image.ImageSetList(),
)
assert module.prepare_run(workspace)
assert (
m.get_measurement(cellprofiler_core.measurement.IMAGE, "FileName_DNA", 1)
== tests.modules.cp_logo_url_filename
)
path = m.get_measurement(cellprofiler_core.measurement.IMAGE, "PathName_DNA", 1)
assert path == tests.modules.cp_logo_url_folder
assert (
m.get_measurement(cellprofiler_core.measurement.IMAGE, "URL_DNA", 1)
== tests.modules.cp_logo_url
)
assert (
m[cellprofiler_core.measurement.IMAGE, "FileName_DNA", 2]
== tests.modules.cp_logo_url_filename
)
assert m[cellprofiler_core.measurement.IMAGE, "PathName_DNA", 2] == "http:"
assert m[cellprofiler_core.measurement.IMAGE, "FileName_DNA", 3] == "bogusurl.png"
assert m[cellprofiler_core.measurement.IMAGE, "PathName_DNA", 3] == ""
module.prepare_group(workspace, {}, [1])
module.run(workspace)
img = workspace.image_set.get_image("DNA", must_be_color=True)
assert tuple(img.pixel_data.shape) == tests.modules.cp_logo_url_shape
def test_load_default_input_folder():
# Regression test of issue #1365 - load a file from the default
# input folder and check that PathName_xxx is absolute
csv_text = '''"Image_FileName_DNA","Image_PathName_DNA"\n"%s","%s"''' % (
test_filename,
test_path,
)
pipeline, module, filename = make_pipeline(csv_text)
try:
assert isinstance(module, cellprofiler.modules.loaddata.LoadData)
module.image_directory.dir_choice = cellprofiler_core.setting.ABSOLUTE_FOLDER_NAME
module.image_directory.custom_path = test_path
m = cellprofiler_core.measurement.Measurements()
workspace = cellprofiler_core.workspace.Workspace(
pipeline,
module,
m,
cellprofiler_core.object.ObjectSet(),
m,
cellprofiler_core.image.ImageSetList(),
)
assert module.prepare_run(workspace)
assert (
m.get_measurement(cellprofiler_core.measurement.IMAGE, "FileName_DNA", 1)
== test_filename
)
path_out = m.get_measurement(cellprofiler_core.measurement.IMAGE, "PathName_DNA", 1)
assert test_path == path_out
assert m.get_measurement(
cellprofiler_core.measurement.IMAGE, "URL_DNA", 1
) == cellprofiler_core.modules.loadimages.pathname2url(
os.path.join(test_path, test_filename)
)
module.prepare_group(workspace, {}, [1])
module.run(workspace)
img = workspace.image_set.get_image("DNA", must_be_grayscale=True)
assert tuple(img.pixel_data.shape) == test_shape
finally:
os.remove(filename)
def run_group_request(self, session_id, message_type, message):
"""Handle a run-group request message"""
pipeline = cellprofiler_core.pipeline.Pipeline()
m = cellprofiler_core.measurement.Measurements()
image_group = m.hdf5_dict.hdf5_file.create_group("ImageData")
if len(message) < 2:
self.raise_cellprofiler_exception(session_id,
"Missing run request sections")
return
pipeline_txt = message.pop(0).bytes
image_metadata = message.pop(0).bytes
n_image_sets = None
try:
image_metadata = json.loads(image_metadata)
channel_names = []
for channel_name, channel_metadata in image_metadata:
channel_names.append(channel_name)
if len(message) < 1:
self.raise_cellprofiler_exception(
session_id,
"Missing binary data for channel %s" % channel_name)
return None, None, None
pixel_data = self.decode_image(channel_metadata,
message.pop(0).bytes,
grouping_allowed=True)
if pixel_data.ndim < 3:
self.raise_cellprofiler_exception(
session_id,
"The image for channel %s does not have a Z or T dimension",
)
return
if n_image_sets is None:
n_image_sets = pixel_data.shape[0]
elif n_image_sets != pixel_data.shape[0]:
self.raise_cellprofiler_exception(
session_id,
"The images passed have different numbers of Z or T planes",
)
return
image_group.create_dataset(channel_name, data=pixel_data)
except Exception as e:
self.raise_cellprofiler_exception(session_id, e.message)
return None, None, None
try:
pipeline.loadtxt(StringIO(pipeline_txt))
except Exception as e:
logger.warning(
"Failed to load pipeline: sending pipeline exception",
exc_info=1)
self.raise_pipeline_exception(session_id, str(e))
return
image_numbers = numpy.arange(1, n_image_sets + 1)
for image_number in image_numbers:
m[cellprofiler_core.measurement.IMAGE,
cellprofiler_core.measurement.GROUP_NUMBER, image_number, ] = 1
m[cellprofiler_core.measurement.IMAGE,
cellprofiler_core.measurement.GROUP_INDEX,
image_number, ] = image_number
input_modules, other_modules = self.split_pipeline(pipeline)
workspace = cellprofiler_core.workspace.Workspace(
pipeline, None, m, None, m, None)
logger.info("Preparing group")
for module in other_modules:
module.prepare_group(
workspace,
dict([("image_number", i) for i in image_numbers]),
image_numbers,
)
for image_index in range(n_image_sets):
object_set = cellprofiler_core.object.ObjectSet()
m.next_image_set(image_index + 1)
for channel_name in channel_names:
dataset = image_group[channel_name]
pixel_data = dataset[image_index]
m.add(channel_name, cellprofiler_core.image.Image(pixel_data))
for module in other_modules:
workspace = cellprofiler_core.workspace.Workspace(
pipeline, module, m, object_set, m, None)
try:
logger.info("Running module # %d: %s" %
(module.module_num, module.module_name))
pipeline.run_module(module, workspace)
if workspace.disposition in (
cellprofiler_core.workspace.DISPOSITION_SKIP,
cellprofiler_core.workspace.DISPOSITION_CANCEL,
):
break
except Exception as e:
msg = 'Encountered error while running module, "%s": %s' % (
module.module_name,
e.message,
)
logger.warning(msg, exc_info=1)
self.raise_cellprofiler_exception(session_id, msg)
return
else:
continue
if workspace.disposition == cellprofiler_core.workspace.DISPOSITION_CANCEL:
break
for module in other_modules:
module.post_group(
workspace, dict([("image_number", i) for i in image_numbers]))
logger.info("Finished group")
type_names, feature_dict = self.find_measurements(
other_modules, pipeline)
double_features = []
double_data = []
float_features = []
float_data = []
int_features = []
int_data = []
string_features = []
string_data = []
metadata = [
double_features, float_features, int_features, string_features
]
for object_name, features in list(feature_dict.items()):
df = []
double_features.append((object_name, df))
ff = []
float_features.append((object_name, ff))
intf = []
int_features.append((object_name, intf))
sf = []
string_features.append((object_name, sf))
if object_name == cellprofiler_core.measurement.IMAGE:
object_counts = [] * n_image_sets
else:
object_numbers = m[object_name,
cellprofiler_core.measurement.OBJECT_NUMBER,
image_numbers]
object_counts = [len(x) for x in object_numbers]
for feature, data_type in features:
if data_type == "java.lang.String":
continue
if not m.has_feature(object_name, feature):
data = numpy.zeros(numpy.sum(object_counts))
else:
data = m[object_name, feature, image_numbers]
temp = []
for i, (di, count) in enumerate(zip(data, object_counts)):
if count == 0:
continue
di = numpy.atleast_1d(di)
if len(di) > count:
di = di[:count]
elif len(di) == count:
temp.append(di)
else:
temp += [di + numpy.zeros(len(di) - count)]
if len(temp) > 0:
data = numpy.hstack(temp)
if type_names[data_type] == "java.lang.Double":
df.append((feature, len(data)))
if len(data) > 0:
double_data.append(data.astype("<f8"))
elif type_names[data_type] == "java.lang.Float":
ff.append((feature, len(data)))
if len(data) > 0:
float_data.append(data.astype("<f4"))
elif type_names[data_type] == "java.lang.Integer":
intf.append((feature, len(data)))
if len(data) > 0:
int_data.append(data.astype("<i4"))
data = numpy.hstack([
numpy.frombuffer(
numpy.ascontiguousarray(numpy.hstack(ditem)).data, numpy.uint8)
for ditem in (double_data, float_data, int_data) if len(ditem) > 0
])
data = numpy.ascontiguousarray(data)
self.socket.send_multipart([
zmq.Frame(session_id),
zmq.Frame(),
zmq.Frame(RUN_REPLY_1),
zmq.Frame(json.dumps(metadata)),
zmq.Frame(data),
])
