def test_02_06_imported_extraction_case_insensitive(self):
metadata_csv = """WellName,Treatment
b08,DMSO
C10,BRD041618
"""
filenum, path = tempfile.mkstemp(suffix = ".csv")
fd = os.fdopen(filenum, "w")
fd.write(metadata_csv)
fd.close()
try:
module = M.Metadata()
module.add_extraction_method()
em = module.extraction_methods[0]
em.filter_choice.value = M.F_ALL_IMAGES
em.extraction_method.value = M.X_MANUAL_EXTRACTION
em.source.value = M.XM_FILE_NAME
em.file_regexp.value = "^(?P<Plate>[^_]+)_(?P<Well>[A-Ha-h][0-9]{2})_s(?P<Site>[0-9])_w(?P<Wavelength>[0-9])"
em.filter_choice.value = M.F_ALL_IMAGES
em = module.extraction_methods[1]
em.filter_choice.value = M.F_ALL_IMAGES
em.extraction_method.value = M.X_IMPORTED_EXTRACTION
em.csv_location.value = path
em.csv_joiner.value = '[{"%s":"WellName","%s":"Well"}]' % (
module.CSV_JOIN_NAME, module.IPD_JOIN_NAME)
em.wants_case_insensitive.value = True
module.ipd_metadata_keys = set()
module.update_imported_metadata()
ipd = cpp.ImagePlaneDetails("file:/imaging/analysis/P-12345_B08_s5_w2.tif",
None, None, None)
metadata = module.get_ipd_metadata(ipd)
self.assertDictEqual(metadata, { "Plate":"P-12345",
"Well":"B08",
"Site":"5",
"Wavelength":"2",
"Treatment":"DMSO"})
ipd = cpp.ImagePlaneDetails("file:/imaging/analysis/P-12345_c10_s2_w3.tif",
None, None, None)
metadata = module.get_ipd_metadata(ipd)
self.assertDictEqual(metadata, { "Plate":"P-12345",
"Well":"c10",
"Site":"2",
"Wavelength":"3",
"Treatment":"BRD041618"})
ipd = cpp.ImagePlaneDetails("file:/imaging/analysis/P-12345_A01_s2_w3.tif",
None, None, None)
metadata = module.get_ipd_metadata(ipd)
self.assertDictEqual(metadata, { "Plate":"P-12345",
"Well":"A01",
"Site":"2",
"Wavelength":"3"})
finally:
try:
os.unlink(path)
except:
pass
def test_03_01_well_row_column(self):
# Make sure that Metadata_Well is generated if we have
# Metadata_Row and Metadata_Column
#
for row_tag, column_tag in (("row", "column"), ("wellrow",
"wellcolumn"),
("well_row", "well_column")):
module = M.Metadata()
module.wants_metadata.value = True
em = module.extraction_methods[0]
em.filter_choice.value = M.F_ALL_IMAGES
em.extraction_method.value = M.X_MANUAL_EXTRACTION
em.source.value = M.XM_FILE_NAME
em.file_regexp.value = (
"^Channel(?P<Wavelength>[1-2])-"
"(?P<%(row_tag)s>[A-H])-"
"(?P<%(column_tag)s>[0-9]{2}).tif$") % locals()
em.filter_choice.value = M.F_ALL_IMAGES
ipd = cpp.ImagePlaneDetails(
"file:/imaging/analysis/Channel1-C-05.tif", None, None, None)
metadata = module.get_ipd_metadata(ipd)
self.assertDictEqual(
metadata, {
"Wavelength": "1",
row_tag: "C",
column_tag: "05",
cpmeas.FTR_WELL: "C05"
})
self.assertIn(cpmeas.M_WELL,
[c[1] for c in module.get_measurement_columns(None)])
def test_02_02_get_metadata_from_path(self):
module = M.Metadata()
em = module.extraction_methods[0]
em.filter_choice.value = M.F_ALL_IMAGES
em.extraction_method.value = M.X_MANUAL_EXTRACTION
em.source.value = M.XM_FOLDER_NAME
em.folder_regexp.value = r".*[/\\](?P<Plate>.+)$"
em.filter_choice.value = M.F_ALL_IMAGES
ipd = cpp.ImagePlaneDetails(
"file:/imaging/analysis/P-12345/_B08_s5_w2.tif", None, None, None)
metadata = module.get_ipd_metadata(ipd)
self.assertDictEqual(metadata, {"Plate": "P-12345"})
def test_01_11_by_order_bad(self):
# Regression test of issue #392: columns of different lengths
n = N.NamesAndTypes()
n.assignment_method.value = N.ASSIGN_RULES
n.matching_choice.value = N.MATCH_BY_ORDER
n.add_assignment()
n.assignments[0].image_name.value = C0
n.assignments[1].image_name.value = C1
n.assignments[0].rule_filter.value = 'file does contain "%s"' % C0
n.assignments[1].rule_filter.value = 'file does contain "%s"' % C1
n.ipd_columns = \
[[cpp.ImagePlaneDetails("%s%d" % (C0, (3-i)), None, None, None, **m)
for i, m in enumerate(md([(M0, 3)]))],
[cpp.ImagePlaneDetails("%s%d" % (C1, i+1), None, None, None, **m)
for i, m in enumerate(md([(M1, 2)]))]]
data = {
C0:
[("%s%d" % (C0, i + 1), m) for i, m in enumerate(md([(M0, 2)]))],
C1:
[("%s%d" % (C1, i + 1), m) for i, m in enumerate(md([(M1, 2)]))]
}
additional = [("%sBad" % C0, {})]
self.do_teest(n, data, [(cpmeas.IMAGE_NUMBER, )], [(C0, M0), (C1, M1)])
def test_02_01_get_metadata_from_filename(self):
module = M.Metadata()
em = module.extraction_methods[0]
em.filter_choice.value = M.F_ALL_IMAGES
em.extraction_method.value = M.X_MANUAL_EXTRACTION
em.source.value = M.XM_FILE_NAME
em.file_regexp.value = "^(?P<Plate>[^_]+)_(?P<Well>[A-H][0-9]{2})_s(?P<Site>[0-9])_w(?P<Wavelength>[0-9])"
em.filter_choice.value = M.F_ALL_IMAGES
ipd = cpp.ImagePlaneDetails("file:/imaging/analysis/P-12345_B08_s5_w2.tif",
None, None, None)
metadata = module.get_ipd_metadata(ipd)
self.assertDictEqual(metadata, { "Plate":"P-12345",
"Well":"B08",
"Site":"5",
"Wavelength":"2"})
def do_update_metadata(self, group):
filelist = self.workspace.file_list
urls = set([
ipd.url for ipd in self.pipeline.get_filtered_image_plane_details(
self.workspace)
])
if len(urls) == 0:
return
def msg(url):
return "Processing %s" % url
import wx
from bioformats.formatreader import get_omexml_metadata
from bioformats.omexml import OMEXML
from cellprofiler.modules.loadimages import url2pathname
with wx.ProgressDialog("Updating metadata",
msg(list(urls)[0]),
len(urls),
style=wx.PD_CAN_ABORT
| wx.PD_APP_MODAL
| wx.PD_ELAPSED_TIME
| wx.PD_REMAINING_TIME) as dlg:
for i, url in enumerate(urls):
if i > 0:
keep_going, _ = dlg.Update(i, msg(url))
if not keep_going:
break
if group.filter_choice == F_FILTERED_IMAGES:
match = group.filter.evaluate(
(cps.FileCollectionDisplay.NODE_IMAGE_PLANE,
Images.url_to_modpath(url), self))
if not match:
continue
metadata = filelist.get_metadata(url)
if metadata is None:
metadata = get_omexml_metadata(url2pathname(url))
filelist.add_metadata(url, metadata)
metadata = OMEXML(metadata)
exemplar = cpp.ImagePlaneDetails(url, None, None, None)
if not self.pipeline.find_image_plane_details(exemplar):
self.pipeline.add_image_plane_details([exemplar])
self.pipeline.add_image_metadata(url, metadata)
self.ipds = self.pipeline.get_filtered_image_plane_details(
self.workspace)
self.update_metadata_keys()
def test_02_02_get_metadata_from_path(self):
module = M.Metadata()
module.wants_metadata.value = True
em = module.extraction_methods[0]
em.filter_choice.value = M.F_ALL_IMAGES
em.extraction_method.value = M.X_MANUAL_EXTRACTION
em.source.value = M.XM_FOLDER_NAME
em.folder_regexp.value = r".*[/\\](?P<Plate>.+)$"
em.filter_choice.value = M.F_ALL_IMAGES
ipd = cpp.ImagePlaneDetails("file:/imaging/analysis/P-12345/_B08_s5_w2.tif",
None, None, None)
metadata = module.get_ipd_metadata(ipd)
self.assertDictEqual(metadata, { "Plate":"P-12345" })
self.assertTrue("Plate" in module.get_dt_metadata_keys())
self.assertTrue(any(c[0] == cpmeas.IMAGE and
c[1] == "_".join((cpmeas.C_METADATA, "Plate")) and
c[2] == cpmeas.COLTYPE_VARCHAR_FILE_NAME
for c in module.get_measurement_columns(None)))
def get_image_plane_details(self, modpath):
url = Images.modpath_to_url(modpath)
return cpp.ImagePlaneDetails(url, None, None, None)
def test_02_05_imported_extraction(self):
metadata_csv = """WellName,Treatment,Dose,Counter
B08,DMSO,0,1
C10,BRD041618,1.5,2
"""
filenum, path = tempfile.mkstemp(suffix=".csv")
fd = os.fdopen(filenum, "w")
fd.write(metadata_csv)
fd.close()
try:
module = M.Metadata()
module.wants_metadata.value = True
module.data_type_choice.value = M.DTC_CHOOSE
module.data_types.value = cps.json.dumps(
dict(Plate=cps.DataTypes.DT_TEXT,
Well=cps.DataTypes.DT_TEXT,
WellName=cps.DataTypes.DT_NONE,
Treatment=cps.DataTypes.DT_TEXT,
Dose=cps.DataTypes.DT_FLOAT,
Counter=cps.DataTypes.DT_NONE))
module.add_extraction_method()
em = module.extraction_methods[0]
em.filter_choice.value = M.F_ALL_IMAGES
em.extraction_method.value = M.X_MANUAL_EXTRACTION
em.source.value = M.XM_FILE_NAME
em.file_regexp.value = "^(?P<Plate>[^_]+)_(?P<Well>[A-Ha-h][0-9]{2})_s(?P<Site>[0-9])_w(?P<Wavelength>[0-9])"
em.filter_choice.value = M.F_ALL_IMAGES
em = module.extraction_methods[1]
em.filter_choice.value = M.F_ALL_IMAGES
em.extraction_method.value = M.X_IMPORTED_EXTRACTION
em.csv_location.value = path
em.csv_joiner.value = '[{"%s":"WellName","%s":"Well"}]' % (
module.CSV_JOIN_NAME, module.IPD_JOIN_NAME)
module.ipd_metadata_keys = set()
module.update_imported_metadata()
ipd = cpp.ImagePlaneDetails(
"file:/imaging/analysis/P-12345_B08_s5_w2.tif", None, None,
None)
metadata = module.get_ipd_metadata(ipd)
self.assertDictEqual(
metadata, {
"Plate": "P-12345",
"Well": "B08",
"Site": "5",
"Wavelength": "2",
"Treatment": "DMSO",
"Dose": "0",
"Counter": "1"
})
ipd = cpp.ImagePlaneDetails(
"file:/imaging/analysis/P-12345_C10_s2_w3.tif", None, None,
None)
metadata = module.get_ipd_metadata(ipd)
self.assertDictEqual(
metadata, {
"Plate": "P-12345",
"Well": "C10",
"Site": "2",
"Wavelength": "3",
"Treatment": "BRD041618",
"Dose": "1.5",
"Counter": "2"
})
ipd = cpp.ImagePlaneDetails(
"file:/imaging/analysis/P-12345_A01_s2_w3.tif", None, None,
None)
metadata = module.get_ipd_metadata(ipd)
self.assertDictEqual(metadata, {
"Plate": "P-12345",
"Well": "A01",
"Site": "2",
"Wavelength": "3"
})
columns = module.get_measurement_columns(None)
self.assertFalse(any([c[1] == "Counter" for c in columns]))
for feature_name, data_type in (("Metadata_Treatment",
cpmeas.COLTYPE_VARCHAR_FILE_NAME),
("Metadata_Dose",
cpmeas.COLTYPE_FLOAT)):
self.assertTrue(
any([
c[0] == cpmeas.IMAGE and c[1] == feature_name
and c[2] == data_type for c in columns
]))
finally:
try:
os.unlink(path)
except:
pass
def do_teest(self,
module,
channels,
expected_tags,
expected_metadata,
additional=None):
'''Ensure that NamesAndTypes recreates the column layout when run
module - instance of NamesAndTypes, set up for the test
channels - a dictionary of channel name to list of "ipds" for that
channel where "ipd" is a tuple of URL and metadata dictionary
Entries may appear multiple times (e.g. illumination function)
expected_tags - the metadata tags that should have been generated
by prepare_run.
expected_metadata - a sequence of two-tuples of metadata key and
the channel from which the metadata is extracted.
additional - if present, these are added as ImagePlaneDetails in order
to create errors. Format is same as a single channel of
channels.
'''
ipds = []
urls = set()
url_root = "file:" + urllib.pathname2url(
os.path.abspath(os.path.curdir))
channels = dict(channels)
if additional is not None:
channels["Additional"] = additional
for channel_name in list(channels):
channel_data = [(url_root + "/" + path, metadata)
for path, metadata in channels[channel_name]]
channels[channel_name] = channel_data
for url, metadata in channel_data:
if url in urls:
continue
urls.add(url)
ipd = cpp.ImagePlaneDetails(url, None, None, None, **metadata)
jmetadata = J.make_map(**metadata)
ipd.jipd = J.run_script(
"""
importPackage(Packages.org.cellprofiler.imageset);
importPackage(Packages.org.cellprofiler.imageset.filter);
var imageFile=new ImageFile(new java.net.URI(url));
var imagePlane=new ImagePlane(imageFile);
new ImagePlaneDetails(imagePlane, metadata);
""", dict(url=url, metadata=jmetadata))
ipds.append(ipd)
if additional is not None:
del channels["Additional"]
ipds.sort(key=lambda x: x.url)
pipeline = cpp.Pipeline()
del pipeline.image_plane_details[:]
pipeline.image_plane_details.extend(ipds)
module.module_num = 1
pipeline.add_module(module)
m = cpmeas.Measurements()
workspace = cpw.Workspace(pipeline, module, m, None, m, None)
self.assertTrue(module.prepare_run(workspace))
tags = m.get_metadata_tags()
self.assertEqual(len(tags), len(expected_tags))
for tag, expected_tag in zip(tags, expected_tags):
for et in expected_tag:
ftr = et if et == cpmeas.IMAGE_NUMBER else \
"_".join((cpmeas.C_METADATA, et))
if ftr == tag:
break
else:
self.fail("%s not in %s" % (tag, ",".join(expected_tag)))
iscds = m.get_channel_descriptors()
self.assertEqual(len(iscds), len(channels))
for channel_name in channels.keys():
iscd_match = filter(lambda x: x.name == channel_name, iscds)
self.assertEquals(len(iscd_match), 1)
iscd = iscd_match[0]
assert isinstance(iscd, cpp.Pipeline.ImageSetChannelDescriptor)
for i, (expected_url,
metadata) in enumerate(channels[channel_name]):
image_number = i + 1
if iscd.channel_type == iscd.CT_OBJECTS:
url_ftr = "_".join((cpmeas.C_OBJECTS_URL, channel_name))
else:
url_ftr = "_".join((cpmeas.C_URL, channel_name))
self.assertEqual(expected_url, m[cpmeas.IMAGE, url_ftr,
image_number])
for key, channel in expected_metadata:
if channel != channel_name:
continue
md_ftr = "_".join((cpmeas.C_METADATA, key))
self.assertEqual(metadata[key], m[cpmeas.IMAGE, md_ftr,
image_number])
