def test_02_06_imported_extraction_case_insensitive(self): metadata_csv = """WellName,Treatment b08,DMSO C10,BRD041618 """ filenum, path = tempfile.mkstemp(suffix = ".csv") fd = os.fdopen(filenum, "w") fd.write(metadata_csv) fd.close() try: module = M.Metadata() module.add_extraction_method() em = module.extraction_methods[0] em.filter_choice.value = M.F_ALL_IMAGES em.extraction_method.value = M.X_MANUAL_EXTRACTION em.source.value = M.XM_FILE_NAME em.file_regexp.value = "^(?P<Plate>[^_]+)_(?P<Well>[A-Ha-h][0-9]{2})_s(?P<Site>[0-9])_w(?P<Wavelength>[0-9])" em.filter_choice.value = M.F_ALL_IMAGES em = module.extraction_methods[1] em.filter_choice.value = M.F_ALL_IMAGES em.extraction_method.value = M.X_IMPORTED_EXTRACTION em.csv_location.value = path em.csv_joiner.value = '[{"%s":"WellName","%s":"Well"}]' % ( module.CSV_JOIN_NAME, module.IPD_JOIN_NAME) em.wants_case_insensitive.value = True module.ipd_metadata_keys = set() module.update_imported_metadata() ipd = cpp.ImagePlaneDetails("file:/imaging/analysis/P-12345_B08_s5_w2.tif", None, None, None) metadata = module.get_ipd_metadata(ipd) self.assertDictEqual(metadata, { "Plate":"P-12345", "Well":"B08", "Site":"5", "Wavelength":"2", "Treatment":"DMSO"}) ipd = cpp.ImagePlaneDetails("file:/imaging/analysis/P-12345_c10_s2_w3.tif", None, None, None) metadata = module.get_ipd_metadata(ipd) self.assertDictEqual(metadata, { "Plate":"P-12345", "Well":"c10", "Site":"2", "Wavelength":"3", "Treatment":"BRD041618"}) ipd = cpp.ImagePlaneDetails("file:/imaging/analysis/P-12345_A01_s2_w3.tif", None, None, None) metadata = module.get_ipd_metadata(ipd) self.assertDictEqual(metadata, { "Plate":"P-12345", "Well":"A01", "Site":"2", "Wavelength":"3"}) finally: try: os.unlink(path) except: pass
def test_03_01_well_row_column(self): # Make sure that Metadata_Well is generated if we have # Metadata_Row and Metadata_Column # for row_tag, column_tag in (("row", "column"), ("wellrow", "wellcolumn"), ("well_row", "well_column")): module = M.Metadata() module.wants_metadata.value = True em = module.extraction_methods[0] em.filter_choice.value = M.F_ALL_IMAGES em.extraction_method.value = M.X_MANUAL_EXTRACTION em.source.value = M.XM_FILE_NAME em.file_regexp.value = ( "^Channel(?P<Wavelength>[1-2])-" "(?P<%(row_tag)s>[A-H])-" "(?P<%(column_tag)s>[0-9]{2}).tif$") % locals() em.filter_choice.value = M.F_ALL_IMAGES ipd = cpp.ImagePlaneDetails( "file:/imaging/analysis/Channel1-C-05.tif", None, None, None) metadata = module.get_ipd_metadata(ipd) self.assertDictEqual( metadata, { "Wavelength": "1", row_tag: "C", column_tag: "05", cpmeas.FTR_WELL: "C05" }) self.assertIn(cpmeas.M_WELL, [c[1] for c in module.get_measurement_columns(None)])
def test_02_02_get_metadata_from_path(self): module = M.Metadata() em = module.extraction_methods[0] em.filter_choice.value = M.F_ALL_IMAGES em.extraction_method.value = M.X_MANUAL_EXTRACTION em.source.value = M.XM_FOLDER_NAME em.folder_regexp.value = r".*[/\\](?P<Plate>.+)$" em.filter_choice.value = M.F_ALL_IMAGES ipd = cpp.ImagePlaneDetails( "file:/imaging/analysis/P-12345/_B08_s5_w2.tif", None, None, None) metadata = module.get_ipd_metadata(ipd) self.assertDictEqual(metadata, {"Plate": "P-12345"})
def test_01_11_by_order_bad(self): # Regression test of issue #392: columns of different lengths n = N.NamesAndTypes() n.assignment_method.value = N.ASSIGN_RULES n.matching_choice.value = N.MATCH_BY_ORDER n.add_assignment() n.assignments[0].image_name.value = C0 n.assignments[1].image_name.value = C1 n.assignments[0].rule_filter.value = 'file does contain "%s"' % C0 n.assignments[1].rule_filter.value = 'file does contain "%s"' % C1 n.ipd_columns = \ [[cpp.ImagePlaneDetails("%s%d" % (C0, (3-i)), None, None, None, **m) for i, m in enumerate(md([(M0, 3)]))], [cpp.ImagePlaneDetails("%s%d" % (C1, i+1), None, None, None, **m) for i, m in enumerate(md([(M1, 2)]))]] data = { C0: [("%s%d" % (C0, i + 1), m) for i, m in enumerate(md([(M0, 2)]))], C1: [("%s%d" % (C1, i + 1), m) for i, m in enumerate(md([(M1, 2)]))] } additional = [("%sBad" % C0, {})] self.do_teest(n, data, [(cpmeas.IMAGE_NUMBER, )], [(C0, M0), (C1, M1)])
def test_02_01_get_metadata_from_filename(self): module = M.Metadata() em = module.extraction_methods[0] em.filter_choice.value = M.F_ALL_IMAGES em.extraction_method.value = M.X_MANUAL_EXTRACTION em.source.value = M.XM_FILE_NAME em.file_regexp.value = "^(?P<Plate>[^_]+)_(?P<Well>[A-H][0-9]{2})_s(?P<Site>[0-9])_w(?P<Wavelength>[0-9])" em.filter_choice.value = M.F_ALL_IMAGES ipd = cpp.ImagePlaneDetails("file:/imaging/analysis/P-12345_B08_s5_w2.tif", None, None, None) metadata = module.get_ipd_metadata(ipd) self.assertDictEqual(metadata, { "Plate":"P-12345", "Well":"B08", "Site":"5", "Wavelength":"2"})
def do_update_metadata(self, group): filelist = self.workspace.file_list urls = set([ ipd.url for ipd in self.pipeline.get_filtered_image_plane_details( self.workspace) ]) if len(urls) == 0: return def msg(url): return "Processing %s" % url import wx from bioformats.formatreader import get_omexml_metadata from bioformats.omexml import OMEXML from cellprofiler.modules.loadimages import url2pathname with wx.ProgressDialog("Updating metadata", msg(list(urls)[0]), len(urls), style=wx.PD_CAN_ABORT | wx.PD_APP_MODAL | wx.PD_ELAPSED_TIME | wx.PD_REMAINING_TIME) as dlg: for i, url in enumerate(urls): if i > 0: keep_going, _ = dlg.Update(i, msg(url)) if not keep_going: break if group.filter_choice == F_FILTERED_IMAGES: match = group.filter.evaluate( (cps.FileCollectionDisplay.NODE_IMAGE_PLANE, Images.url_to_modpath(url), self)) if not match: continue metadata = filelist.get_metadata(url) if metadata is None: metadata = get_omexml_metadata(url2pathname(url)) filelist.add_metadata(url, metadata) metadata = OMEXML(metadata) exemplar = cpp.ImagePlaneDetails(url, None, None, None) if not self.pipeline.find_image_plane_details(exemplar): self.pipeline.add_image_plane_details([exemplar]) self.pipeline.add_image_metadata(url, metadata) self.ipds = self.pipeline.get_filtered_image_plane_details( self.workspace) self.update_metadata_keys()
def test_02_02_get_metadata_from_path(self): module = M.Metadata() module.wants_metadata.value = True em = module.extraction_methods[0] em.filter_choice.value = M.F_ALL_IMAGES em.extraction_method.value = M.X_MANUAL_EXTRACTION em.source.value = M.XM_FOLDER_NAME em.folder_regexp.value = r".*[/\\](?P<Plate>.+)$" em.filter_choice.value = M.F_ALL_IMAGES ipd = cpp.ImagePlaneDetails("file:/imaging/analysis/P-12345/_B08_s5_w2.tif", None, None, None) metadata = module.get_ipd_metadata(ipd) self.assertDictEqual(metadata, { "Plate":"P-12345" }) self.assertTrue("Plate" in module.get_dt_metadata_keys()) self.assertTrue(any(c[0] == cpmeas.IMAGE and c[1] == "_".join((cpmeas.C_METADATA, "Plate")) and c[2] == cpmeas.COLTYPE_VARCHAR_FILE_NAME for c in module.get_measurement_columns(None)))
def get_image_plane_details(self, modpath): url = Images.modpath_to_url(modpath) return cpp.ImagePlaneDetails(url, None, None, None)
def test_02_05_imported_extraction(self): metadata_csv = """WellName,Treatment,Dose,Counter B08,DMSO,0,1 C10,BRD041618,1.5,2 """ filenum, path = tempfile.mkstemp(suffix=".csv") fd = os.fdopen(filenum, "w") fd.write(metadata_csv) fd.close() try: module = M.Metadata() module.wants_metadata.value = True module.data_type_choice.value = M.DTC_CHOOSE module.data_types.value = cps.json.dumps( dict(Plate=cps.DataTypes.DT_TEXT, Well=cps.DataTypes.DT_TEXT, WellName=cps.DataTypes.DT_NONE, Treatment=cps.DataTypes.DT_TEXT, Dose=cps.DataTypes.DT_FLOAT, Counter=cps.DataTypes.DT_NONE)) module.add_extraction_method() em = module.extraction_methods[0] em.filter_choice.value = M.F_ALL_IMAGES em.extraction_method.value = M.X_MANUAL_EXTRACTION em.source.value = M.XM_FILE_NAME em.file_regexp.value = "^(?P<Plate>[^_]+)_(?P<Well>[A-Ha-h][0-9]{2})_s(?P<Site>[0-9])_w(?P<Wavelength>[0-9])" em.filter_choice.value = M.F_ALL_IMAGES em = module.extraction_methods[1] em.filter_choice.value = M.F_ALL_IMAGES em.extraction_method.value = M.X_IMPORTED_EXTRACTION em.csv_location.value = path em.csv_joiner.value = '[{"%s":"WellName","%s":"Well"}]' % ( module.CSV_JOIN_NAME, module.IPD_JOIN_NAME) module.ipd_metadata_keys = set() module.update_imported_metadata() ipd = cpp.ImagePlaneDetails( "file:/imaging/analysis/P-12345_B08_s5_w2.tif", None, None, None) metadata = module.get_ipd_metadata(ipd) self.assertDictEqual( metadata, { "Plate": "P-12345", "Well": "B08", "Site": "5", "Wavelength": "2", "Treatment": "DMSO", "Dose": "0", "Counter": "1" }) ipd = cpp.ImagePlaneDetails( "file:/imaging/analysis/P-12345_C10_s2_w3.tif", None, None, None) metadata = module.get_ipd_metadata(ipd) self.assertDictEqual( metadata, { "Plate": "P-12345", "Well": "C10", "Site": "2", "Wavelength": "3", "Treatment": "BRD041618", "Dose": "1.5", "Counter": "2" }) ipd = cpp.ImagePlaneDetails( "file:/imaging/analysis/P-12345_A01_s2_w3.tif", None, None, None) metadata = module.get_ipd_metadata(ipd) self.assertDictEqual(metadata, { "Plate": "P-12345", "Well": "A01", "Site": "2", "Wavelength": "3" }) columns = module.get_measurement_columns(None) self.assertFalse(any([c[1] == "Counter" for c in columns])) for feature_name, data_type in (("Metadata_Treatment", cpmeas.COLTYPE_VARCHAR_FILE_NAME), ("Metadata_Dose", cpmeas.COLTYPE_FLOAT)): self.assertTrue( any([ c[0] == cpmeas.IMAGE and c[1] == feature_name and c[2] == data_type for c in columns ])) finally: try: os.unlink(path) except: pass
def do_teest(self, module, channels, expected_tags, expected_metadata, additional=None): '''Ensure that NamesAndTypes recreates the column layout when run module - instance of NamesAndTypes, set up for the test channels - a dictionary of channel name to list of "ipds" for that channel where "ipd" is a tuple of URL and metadata dictionary Entries may appear multiple times (e.g. illumination function) expected_tags - the metadata tags that should have been generated by prepare_run. expected_metadata - a sequence of two-tuples of metadata key and the channel from which the metadata is extracted. additional - if present, these are added as ImagePlaneDetails in order to create errors. Format is same as a single channel of channels. ''' ipds = [] urls = set() url_root = "file:" + urllib.pathname2url( os.path.abspath(os.path.curdir)) channels = dict(channels) if additional is not None: channels["Additional"] = additional for channel_name in list(channels): channel_data = [(url_root + "/" + path, metadata) for path, metadata in channels[channel_name]] channels[channel_name] = channel_data for url, metadata in channel_data: if url in urls: continue urls.add(url) ipd = cpp.ImagePlaneDetails(url, None, None, None, **metadata) jmetadata = J.make_map(**metadata) ipd.jipd = J.run_script( """ importPackage(Packages.org.cellprofiler.imageset); importPackage(Packages.org.cellprofiler.imageset.filter); var imageFile=new ImageFile(new java.net.URI(url)); var imagePlane=new ImagePlane(imageFile); new ImagePlaneDetails(imagePlane, metadata); """, dict(url=url, metadata=jmetadata)) ipds.append(ipd) if additional is not None: del channels["Additional"] ipds.sort(key=lambda x: x.url) pipeline = cpp.Pipeline() del pipeline.image_plane_details[:] pipeline.image_plane_details.extend(ipds) module.module_num = 1 pipeline.add_module(module) m = cpmeas.Measurements() workspace = cpw.Workspace(pipeline, module, m, None, m, None) self.assertTrue(module.prepare_run(workspace)) tags = m.get_metadata_tags() self.assertEqual(len(tags), len(expected_tags)) for tag, expected_tag in zip(tags, expected_tags): for et in expected_tag: ftr = et if et == cpmeas.IMAGE_NUMBER else \ "_".join((cpmeas.C_METADATA, et)) if ftr == tag: break else: self.fail("%s not in %s" % (tag, ",".join(expected_tag))) iscds = m.get_channel_descriptors() self.assertEqual(len(iscds), len(channels)) for channel_name in channels.keys(): iscd_match = filter(lambda x: x.name == channel_name, iscds) self.assertEquals(len(iscd_match), 1) iscd = iscd_match[0] assert isinstance(iscd, cpp.Pipeline.ImageSetChannelDescriptor) for i, (expected_url, metadata) in enumerate(channels[channel_name]): image_number = i + 1 if iscd.channel_type == iscd.CT_OBJECTS: url_ftr = "_".join((cpmeas.C_OBJECTS_URL, channel_name)) else: url_ftr = "_".join((cpmeas.C_URL, channel_name)) self.assertEqual(expected_url, m[cpmeas.IMAGE, url_ftr, image_number]) for key, channel in expected_metadata: if channel != channel_name: continue md_ftr = "_".join((cpmeas.C_METADATA, key)) self.assertEqual(metadata[key], m[cpmeas.IMAGE, md_ftr, image_number])