def test_get_groupings():
"""Test the get_groupings method"""
dir = os.path.join(tests.modules.example_images_directory(),
"ExampleSBSImages")
pattern = "Channel1-[0-9]{2}-(?P<ROW>[A-H])-(?P<COL>[0-9]{2})\\.tif"
csv_text = '"Image_FileName_Cytoplasm","Image_PathName_Cytoplasm","Metadata_ROW","Metadata_COL"\n'
for filename in os.listdir(dir):
match = re.match(pattern, filename)
if match:
csv_text += '"%s","%s","%s","%s"\n' % (
filename,
dir,
match.group("ROW"),
match.group("COL"),
)
pipeline, module, filename = make_pipeline(csv_text)
assert isinstance(module, cellprofiler.modules.loaddata.LoadText)
module.wants_images.value = True
module.wants_image_groupings.value = True
module.metadata_fields.value = "ROW"
image_set_list = cellprofiler_core.image.ImageSetList()
measurements = cellprofiler_core.measurement.Measurements()
workspace = cellprofiler_core.workspace.Workspace(pipeline, module, None,
None, measurements,
image_set_list)
module.prepare_run(workspace)
keys, groupings = module.get_groupings(workspace)
assert len(keys) == 1
assert keys[0] == "Metadata_ROW"
assert len(groupings) == 8
my_rows = [g[0]["Metadata_ROW"] for g in groupings]
my_rows.sort()
assert "".join(my_rows) == "ABCDEFGH"
for grouping in groupings:
row = grouping[0]["Metadata_ROW"]
module.prepare_group(
cellprofiler_core.workspace.Workspace(pipeline, module, None, None,
measurements,
image_set_list),
grouping[0],
grouping[1],
)
for image_number in grouping[1]:
image_set = image_set_list.get_image_set(image_number - 1)
measurements.next_image_set(image_number)
workspace = cellprofiler_core.workspace.Workspace(
pipeline,
module,
image_set,
cellprofiler_core.object.ObjectSet(),
measurements,
image_set_list,
)
module.run(workspace)
provider = image_set.get_image_provider("Cytoplasm")
match = re.search(pattern, provider.get_filename())
assert match
assert row == match.group("ROW")
def test_extra_lines():
#
# Regression test of issue #1211 - extra line at end / blank lines
#
dir = os.path.join(tests.modules.example_images_directory(), "ExampleSBSImages")
file_name = "Channel2-01-A-01.tif"
csv_text = """"Image_FileName_DNA","Image_PathName_DNA"
"%s","%s"
""" % (
file_name,
dir,
)
pipeline, module, filename = make_pipeline(csv_text)
try:
assert isinstance(module, cellprofiler.modules.loaddata.LoadData)
m = cellprofiler_core.measurement.Measurements()
workspace = cellprofiler_core.workspace.Workspace(
pipeline,
module,
m,
cellprofiler_core.object.ObjectSet(),
m,
cellprofiler_core.image.ImageSetList(),
)
assert module.prepare_run(workspace)
assert isinstance(m, cellprofiler_core.measurement.Measurements)
assert m.image_set_count == 1
assert "FileName_DNA" in m.get_feature_names(cellprofiler_core.measurement.IMAGE)
assert m[cellprofiler_core.measurement.IMAGE, "FileName_DNA", 1] == file_name
finally:
os.remove(filename)
def test_load_filename():
#
# Load a file, only specifying the FileName in the CSV
#
csv_text = (""""Image_FileName_DNA"
"%s"
""" % test_filename)
pipeline, module, filename = make_pipeline(csv_text)
assert isinstance(module, cellprofiler.modules.loaddata.LoadData)
module.image_directory.dir_choice = cellprofiler_core.setting.ABSOLUTE_FOLDER_NAME
module.image_directory.custom_path = test_path
m = cellprofiler_core.measurement.Measurements()
workspace = cellprofiler_core.workspace.Workspace(
pipeline,
module,
m,
cellprofiler_core.object.ObjectSet(),
m,
cellprofiler_core.image.ImageSetList(),
)
assert module.prepare_run(workspace)
assert (m.get_measurement(cellprofiler_core.measurement.IMAGE,
"FileName_DNA", 1) == test_filename)
path = m.get_measurement(cellprofiler_core.measurement.IMAGE,
"PathName_DNA", 1)
assert path == test_path
assert m.get_measurement(
cellprofiler_core.measurement.IMAGE, "URL_DNA",
1) == cellprofiler_core.modules.loadimages.pathname2url(
os.path.join(test_path, test_filename))
module.prepare_group(workspace, {}, [1])
module.run(workspace)
img = workspace.image_set.get_image("DNA", must_be_grayscale=True)
assert tuple(img.pixel_data.shape) == test_shape
def test_load_planes():
file_name = "RLM1 SSN3 300308 008015000.flex"
tests.modules.maybe_download_tesst_image(file_name)
path = tests.modules.testimages_directory()
pathname = os.path.join(path, file_name)
url = cellprofiler_core.modules.loadimages.pathname2url(pathname)
ftrs = (
cellprofiler_core.measurement.C_URL,
cellprofiler_core.measurement.C_SERIES,
cellprofiler_core.measurement.C_FRAME,
)
channels = ("Channel1", "Channel2")
header = ",".join(
[",".join(["_".join((ftr, channel)) for ftr in ftrs]) for channel in channels]
)
csv_lines = [header]
for series in range(4):
csv_lines.append(
",".join(['"%s","%d","%d"' % (url, series, frame) for frame in range(2)])
)
csv_text = "\n".join(csv_lines)
pipeline, module, filename = make_pipeline(csv_text)
assert isinstance(module, cellprofiler.modules.loaddata.LoadData)
m = cellprofiler_core.measurement.Measurements()
image_set_list = cellprofiler_core.image.ImageSetList()
try:
workspace = cellprofiler_core.workspace.Workspace(
pipeline, module, m, None, m, image_set_list
)
assert module.prepare_run(workspace)
pixel_hashes = []
for i in range(4):
m.next_image_set(i + 1)
module.run(workspace)
chashes = []
for channel in channels:
pixel_data = m.get_image(channel).pixel_data
h = hashlib.md5()
h.update(pixel_data)
chashes.append(h.digest())
assert chashes[0] != chashes[1]
for j, ph in enumerate(pixel_hashes):
for k, phh in enumerate(ph):
for l, phd in enumerate(chashes):
assert phh != phd
pixel_hashes.append(chashes)
finally:
os.remove(filename)
def test_extra_fields():
#
# Regression test of issue #853, extra fields
#
csv_text = """"Image_URL_DNA"
"{cp_logo_url}", "foo"
"http:{cp_logo_url_filename}"
"bogusurl.png"
""".format(
**{
"cp_logo_url": tests.modules.cp_logo_url,
"cp_logo_url_filename": tests.modules.cp_logo_url_filename,
}
)
pipeline, module, filename = make_pipeline(csv_text)
assert isinstance(module, cellprofiler.modules.loaddata.LoadData)
m = cellprofiler_core.measurement.Measurements()
workspace = cellprofiler_core.workspace.Workspace(
pipeline,
module,
m,
cellprofiler_core.object.ObjectSet(),
m,
cellprofiler_core.image.ImageSetList(),
)
assert module.prepare_run(workspace)
assert (
m.get_measurement(cellprofiler_core.measurement.IMAGE, "FileName_DNA", 1)
== tests.modules.cp_logo_url_filename
)
path = m.get_measurement(cellprofiler_core.measurement.IMAGE, "PathName_DNA", 1)
assert path == tests.modules.cp_logo_url_folder
assert (
m.get_measurement(cellprofiler_core.measurement.IMAGE, "URL_DNA", 1)
== tests.modules.cp_logo_url
)
assert (
m[cellprofiler_core.measurement.IMAGE, "FileName_DNA", 2]
== tests.modules.cp_logo_url_filename
)
assert m[cellprofiler_core.measurement.IMAGE, "PathName_DNA", 2] == "http:"
assert m[cellprofiler_core.measurement.IMAGE, "FileName_DNA", 3] == "bogusurl.png"
assert m[cellprofiler_core.measurement.IMAGE, "PathName_DNA", 3] == ""
module.prepare_group(workspace, {}, [1])
module.run(workspace)
img = workspace.image_set.get_image("DNA", must_be_color=True)
assert tuple(img.pixel_data.shape) == tests.modules.cp_logo_url_shape
def test_extra_lines_skip_rows():
#
# Regression test of issue #1211 - extra line at end / blank lines
# Different code path from 13_04
#
path = os.path.join(tests.modules.example_images_directory(),
"ExampleSBSImages")
file_names = ["Channel2-01-A-01.tif", "Channel2-02-A-02.tif"]
csv_text = """"Image_FileName_DNA","Image_PathName_DNA"
"%s","%s"
"%s","%s"
""" % (
file_names[0],
path,
file_names[1],
path,
)
pipeline, module, filename = make_pipeline(csv_text)
try:
assert isinstance(module, cellprofiler.modules.loaddata.LoadData)
m = cellprofiler_core.measurement.Measurements()
workspace = cellprofiler_core.workspace.Workspace(
pipeline,
module,
m,
cellprofiler_core.object.ObjectSet(),
m,
cellprofiler_core.image.ImageSetList(),
)
module.wants_rows.value = True
module.row_range.min = 2
module.row_range.max = 3
assert module.prepare_run(workspace)
assert isinstance(m, cellprofiler_core.measurement.Measurements)
assert m.image_set_count == 1
assert "FileName_DNA" in m.get_feature_names(
cellprofiler_core.measurement.IMAGE)
assert (m[cellprofiler_core.measurement.IMAGE, "FileName_DNA",
1] == file_names[0])
finally:
os.remove(filename)
def test_load_default_input_folder():
# Regression test of issue #1365 - load a file from the default
# input folder and check that PathName_xxx is absolute
csv_text = '''"Image_FileName_DNA","Image_PathName_DNA"\n"%s","%s"''' % (
test_filename,
test_path,
)
pipeline, module, filename = make_pipeline(csv_text)
try:
assert isinstance(module, cellprofiler.modules.loaddata.LoadData)
module.image_directory.dir_choice = cellprofiler_core.setting.ABSOLUTE_FOLDER_NAME
module.image_directory.custom_path = test_path
m = cellprofiler_core.measurement.Measurements()
workspace = cellprofiler_core.workspace.Workspace(
pipeline,
module,
m,
cellprofiler_core.object.ObjectSet(),
m,
cellprofiler_core.image.ImageSetList(),
)
assert module.prepare_run(workspace)
assert (
m.get_measurement(cellprofiler_core.measurement.IMAGE, "FileName_DNA", 1)
== test_filename
)
path_out = m.get_measurement(cellprofiler_core.measurement.IMAGE, "PathName_DNA", 1)
assert test_path == path_out
assert m.get_measurement(
cellprofiler_core.measurement.IMAGE, "URL_DNA", 1
) == cellprofiler_core.modules.loadimages.pathname2url(
os.path.join(test_path, test_filename)
)
module.prepare_group(workspace, {}, [1])
module.run(workspace)
img = workspace.image_set.get_image("DNA", must_be_grayscale=True)
assert tuple(img.pixel_data.shape) == test_shape
finally:
os.remove(filename)
def run_request(self, session_id, message_type, message):
"""Handle the run request message"""
pipeline, m, object_set = self.prepare_run(message, session_id)
if pipeline is None:
return
m[cellprofiler_core.measurement.IMAGE,
cellprofiler_core.measurement.GROUP_NUMBER] = 1
m[cellprofiler_core.measurement.IMAGE,
cellprofiler_core.measurement.GROUP_INDEX] = 1
input_modules, other_modules = self.split_pipeline(pipeline)
for module in other_modules:
workspace = cellprofiler_core.workspace.Workspace(
pipeline, module, m, None, m, None)
module.prepare_run(workspace)
for module in other_modules:
workspace = cellprofiler_core.workspace.Workspace(
pipeline, module, m, object_set, m, None)
try:
logger.info("Running module # %d: %s" %
(module.module_num, module.module_name))
pipeline.run_module(module, workspace)
if workspace.disposition in (
cellprofiler_core.workspace.DISPOSITION_SKIP,
cellprofiler_core.workspace.DISPOSITION_CANCEL,
):
break
except Exception as e:
msg = 'Encountered error while running module, "%s": %s' % (
module.module_name,
e.message,
)
logger.warning(msg, exc_info=1)
self.raise_cellprofiler_exception(session_id, msg)
return
type_names, feature_dict = self.find_measurements(
other_modules, pipeline)
double_features = []
double_data = []
float_features = []
float_data = []
int_features = []
int_data = []
string_features = []
string_data = []
metadata = [
double_features, float_features, int_features, string_features
]
no_data = ()
for object_name, features in list(feature_dict.items()):
df = []
double_features.append((object_name, df))
ff = []
float_features.append((object_name, ff))
intf = []
int_features.append((object_name, intf))
sf = []
string_features.append((object_name, sf))
for feature, data_type in features:
if not m.has_feature(object_name, feature):
data = no_data
else:
data = numpy.atleast_1d(m[object_name, feature])
if type_names[data_type] == "java.lang.Double":
df.append((feature, len(data)))
if len(data) > 0:
double_data.append(data.astype("<f8"))
elif type_names[data_type] == "java.lang.Float":
ff.append((feature, len(data)))
if len(data) > 0:
float_data.append(data.astype("<f4"))
elif type_names[data_type] == "java.lang.Integer":
intf.append((feature, len(data)))
if len(data) > 0:
int_data.append(data.astype("<i4"))
elif type_names[data_type] == "java.lang.String":
if len(data) == 0:
sf.append((feature, 0))
else:
s = data[0]
if isinstance(s, six.text_type):
s = s
else:
s = str(s)
string_data.append(numpy.frombuffer(s, numpy.uint8))
data = numpy.hstack([
numpy.frombuffer(numpy.hstack(ditem).data, numpy.uint8)
for ditem in (double_data, float_data, int_data, string_data)
if len(ditem) > 0
])
self.socket.send_multipart([
zmq.Frame(session_id),
zmq.Frame(),
zmq.Frame(RUN_REPLY_1),
zmq.Frame(json.dumps(metadata)),
zmq.Frame(bytes(data.data)),
])
