def test_13_05_extra_lines_skip_rows(self): # # Regression test of issue #1211 - extra line at end / blank lines # Different code path from 13_04 # path = os.path.join(example_images_directory(), "ExampleSBSImages") file_names = ['Channel2-01-A-01.tif', 'Channel2-02-A-02.tif'] csv_text = '''"Image_FileName_DNA","Image_PathName_DNA" "%s","%s" "%s","%s" ''' % (file_names[0], path, file_names[1], path) pipeline, module, filename = self.make_pipeline(csv_text) try: assert isinstance(module, L.LoadData) m = cpmeas.Measurements() workspace = cpw.Workspace(pipeline, module, m, cpo.ObjectSet(), m, cpi.ImageSetList()) module.wants_rows.value = True module.row_range.min = 2 module.row_range.max = 3 self.assertTrue(module.prepare_run(workspace)) self.assertTrue(isinstance(m, cpmeas.Measurements)) self.assertEqual(m.image_set_count, 1) self.assertTrue('FileName_DNA' in m.get_feature_names(cpmeas.IMAGE)) self.assertEqual(m[cpmeas.IMAGE, 'FileName_DNA', 1], file_names[0]) finally: os.remove(filename)
def test_13_04_extra_lines(self): # # Regression test of issue #1211 - extra line at end / blank lines # dir = os.path.join(example_images_directory(), "ExampleSBSImages") file_name = 'Channel2-01-A-01.tif' csv_text = '''"Image_FileName_DNA","Image_PathName_DNA" "%s","%s" ''' % (file_name, dir) pipeline, module, filename = self.make_pipeline(csv_text) try: assert isinstance(module, L.LoadData) m = cpmeas.Measurements() workspace = cpw.Workspace(pipeline, module, m, cpo.ObjectSet(), m, cpi.ImageSetList()) self.assertTrue(module.prepare_run(workspace)) self.assertTrue(isinstance(m, cpmeas.Measurements)) self.assertEqual(m.image_set_count, 1) self.assertTrue( 'FileName_DNA' in m.get_feature_names(cpmeas.IMAGE)) self.assertEqual(m[cpmeas.IMAGE, 'FileName_DNA', 1], file_name) finally: os.remove(filename)
def setUpClass(cls): maybe_download_sbs() cls.test_folder = "loaddata" cls.test_path = os.path.join( example_images_directory(), cls.test_folder) cls.test_filename = "image.tif" cls.test_shape = (13, 15) path = maybe_download_example_image([cls.test_folder], cls.test_filename, shape=cls.test_shape) with open(path, "rb") as fd: cls.test_md5 = hashlib.md5(fd.read()).hexdigest()
def test_11_01_load_objects(self): r = np.random.RandomState() r.seed(1101) labels = r.randint(0, 10, size=(30, 20)).astype(np.uint8) handle, name = tempfile.mkstemp(".png") write_image(name, labels, PT_UINT8) os.close(handle) png_path, png_file = os.path.split(name) sbs_dir = os.path.join(example_images_directory(), "ExampleSBSImages") csv_text = """%s_%s,%s_%s,%s_DNA,%s_DNA %s,%s,Channel2-01-A-01.tif,%s """ % (L.C_OBJECTS_FILE_NAME, OBJECTS_NAME, L.C_OBJECTS_PATH_NAME, OBJECTS_NAME, L.C_FILE_NAME, L.C_PATH_NAME, png_file, png_path, sbs_dir) pipeline, module, csv_name = self.make_pipeline(csv_text) assert isinstance(pipeline, cpp.Pipeline) assert isinstance(module, L.LoadData) module.wants_images.value = True try: image_set_list = cpi.ImageSetList() measurements = cpmeas.Measurements() workspace = cpw.Workspace( pipeline, module, None, None, measurements, image_set_list) pipeline.prepare_run(workspace) key_names, g = pipeline.get_groupings(workspace) self.assertEqual(len(g), 1) module.prepare_group(workspace, g[0][0], g[0][1]) image_set = image_set_list.get_image_set(g[0][1][0] - 1) object_set = cpo.ObjectSet() workspace = cpw.Workspace(pipeline, module, image_set, object_set, measurements, image_set_list) module.run(workspace) objects = object_set.get_objects(OBJECTS_NAME) self.assertTrue(np.all(objects.segmented == labels)) self.assertEqual(measurements.get_current_image_measurement( cellprofiler.measurement.FF_COUNT % OBJECTS_NAME), 9) for feature in (cellprofiler.measurement.M_LOCATION_CENTER_X, cellprofiler.measurement.M_LOCATION_CENTER_Y, cellprofiler.measurement.M_NUMBER_OBJECT_NUMBER): value = measurements.get_current_measurement( OBJECTS_NAME, feature) self.assertEqual(len(value), 9) finally: clear_image_reader_cache() os.remove(name) os.remove(csv_name)
def test_11_01_load_objects(self): r = np.random.RandomState() r.seed(1101) labels = r.randint(0, 10, size=(30, 20)).astype(np.uint8) handle, name = tempfile.mkstemp(".png") write_image(name, labels, PT_UINT8) os.close(handle) png_path, png_file = os.path.split(name) sbs_dir = os.path.join(example_images_directory(), "ExampleSBSImages") csv_text = """%s_%s,%s_%s,%s_DNA,%s_DNA %s,%s,Channel2-01-A-01.tif,%s """ % (L.C_OBJECTS_FILE_NAME, OBJECTS_NAME, L.C_OBJECTS_PATH_NAME, OBJECTS_NAME, L.C_FILE_NAME, L.C_PATH_NAME, png_file, png_path, sbs_dir) pipeline, module, csv_name = self.make_pipeline(csv_text) assert isinstance(pipeline, cpp.Pipeline) assert isinstance(module, L.LoadData) module.wants_images.value = True try: image_set_list = cpi.ImageSetList() measurements = cpmeas.Measurements() workspace = cpw.Workspace( pipeline, module, None, None, measurements, image_set_list) pipeline.prepare_run(workspace) key_names, g = pipeline.get_groupings(workspace) self.assertEqual(len(g), 1) module.prepare_group(workspace, g[0][0], g[0][1]) image_set = image_set_list.get_image_set(g[0][1][0] - 1) object_set = cpo.ObjectSet() workspace = cpw.Workspace(pipeline, module, image_set, object_set, measurements, image_set_list) module.run(workspace) objects = object_set.get_objects(OBJECTS_NAME) self.assertTrue(np.all(objects.segmented == labels)) self.assertEqual(measurements.get_current_image_measurement( L.I.FF_COUNT % OBJECTS_NAME), 9) for feature in (L.I.M_LOCATION_CENTER_X, L.I.M_LOCATION_CENTER_Y, L.I.M_NUMBER_OBJECT_NUMBER): value = measurements.get_current_measurement( OBJECTS_NAME, feature) self.assertEqual(len(value), 9) finally: clear_image_reader_cache() os.remove(name) os.remove(csv_name)
def test_08_01_get_groupings(self): '''Test the get_groupings method''' dir = os.path.join(example_images_directory(), "ExampleSBSImages") pattern = 'Channel1-[0-9]{2}-(?P<ROW>[A-H])-(?P<COL>[0-9]{2})\\.tif' csv_text = '"Image_FileName_Cytoplasm","Image_PathName_Cytoplasm","Metadata_ROW","Metadata_COL"\n' for filename in os.listdir(dir): match = re.match(pattern, filename) if match: csv_text += ('"%s","%s","%s","%s"\n' % (filename, dir, match.group("ROW"), match.group("COL"))) pipeline, module, filename = self.make_pipeline(csv_text) self.assertTrue(isinstance(module, L.LoadText)) module.wants_images.value = True module.wants_image_groupings.value = True module.metadata_fields.value = "ROW" image_set_list = cpi.ImageSetList() measurements = cpmeas.Measurements() workspace = cpw.Workspace(pipeline, module, None, None, measurements, image_set_list) module.prepare_run(workspace) keys, groupings = module.get_groupings(workspace) self.assertEqual(len(keys), 1) self.assertEqual(keys[0], "Metadata_ROW") self.assertEqual(len(groupings), 8) my_rows = [g[0]["Metadata_ROW"] for g in groupings] my_rows.sort() self.assertEqual(''.join(my_rows), 'ABCDEFGH') for grouping in groupings: row = grouping[0]["Metadata_ROW"] module.prepare_group(cpw.Workspace( pipeline, module, None, None, measurements, image_set_list), grouping[0], grouping[1]) for image_number in grouping[1]: image_set = image_set_list.get_image_set(image_number - 1) measurements.next_image_set(image_number) workspace = cpw.Workspace(pipeline, module, image_set, cpo.ObjectSet(), measurements, image_set_list) module.run(workspace) provider = image_set.get_image_provider("Cytoplasm") match = re.search(pattern, provider.get_filename()) self.assertTrue(match) self.assertEqual(row, match.group("ROW"))
def test_13_04_extra_lines(self): # # Regression test of issue #1211 - extra line at end / blank lines # dir = os.path.join(example_images_directory(), "ExampleSBSImages") file_name = 'Channel2-01-A-01.tif' csv_text = '''"Image_FileName_DNA","Image_PathName_DNA" "%s","%s" ''' % (file_name, dir) pipeline, module, filename = self.make_pipeline(csv_text) try: assert isinstance(module, L.LoadData) m = cpmeas.Measurements() workspace = cpw.Workspace(pipeline, module, m, cpo.ObjectSet(), m, cpi.ImageSetList()) self.assertTrue(module.prepare_run(workspace)) self.assertTrue(isinstance(m, cpmeas.Measurements)) self.assertEqual(m.image_set_count, 1) self.assertTrue('FileName_DNA' in m.get_feature_names(cpmeas.IMAGE)) self.assertEqual(m[cpmeas.IMAGE, 'FileName_DNA', 1], file_name) finally: os.remove(filename)
def example_dir(self): return example_images_directory()
def test_03_01_save_and_load(self): """Save a pipeline to batch data, open it to check and load it""" data = ("eJztWW1PGkEQXhC1WtPYTzb9tB+llROoGiWNgi9NSYUSIbZGbbvCApvu7ZJ7" "UWlj0o/9Wf1J/QndxTs4tsoBRS3JHbkcMzfPPDOzs8uxl8uU9jPbcFWLw1ym" "FKsSimGBIqvKDT0FmbUEdwyMLFyBnKVgycYwY9dgIgET8dTqRmolCZPx+AYY" "7ghlc0/EJf4cgClxfSTOsHNr0pFDnlPKRWxZhNXMSRABzxz9L3EeIoOgM4oP" "EbWx2aFw9VlW5aVmo30rxys2xXmke43Fkbf1M2yY76su0LldIJeYFsk3rKTg" "mh3gc2ISzhy841/Vtnm5pfDKOhTmOnUIKXWQdVnw6KX9W9Cxj9xQt6ce+3lH" "JqxCzknFRhQSHdXaUbTGwcffRJe/CXAk0BKX9sHNK3HIs4QvrdjeJSpbUEdW" "uS79rPv4mVb8SLmcOrGw3ugr/lAXPgRe9Zl3uAsXBnkO+sp7VolXyrscMm5B" "28T91/02/lHgphSce7i4GdCJ0y/fOSVfKe9RE1/UsYE1TXP91H38rCh+pCzG" "uYwpbRhcLlHGiWXY7OuJaCC9ISZ3q5NdqbhdzLZb+yH4hpmXy3I2ioVtGTFE" "myYZxbwcdpzvu68eku8+cGmf/GZAdz9IeaeOGMM0ERtx3P30z24+c6d86jqc" "uOP8Il18EdE/DP8L3w8fvnegezyl/Glxq/BaPljhTe1l9LOUPoj15YBfbB5n" "YoXTqKvZ4dTW2eZxPLZx+j2xlLy6Ni4SgWwpozfmPUj8fuvhuhK/lGUMRxgZ" "TmArV9GYVOU4s+qOLunodlGzo3mgeZMcxbwZir9p8QZFpj4C/kHnUfKO+YJ5" "NJ7z6OPsYP8rxuV3NcAFuAAX4P43XNqD63c/pLUZUzO43YCEVXBjnPINcOON" "S4OgXwPc8DipvO35Ut2/kfZfQO9+ewG6+03K3s04TW9topsa5ahyvYut7Yuv" "Wc+Gdj/P52sKz9ptPOXWK5AzuU8tb5ja9TuRbal4Q1rvCNT6zdzA561DWHwW" "pnvXXa13Zxx+bw3DFwn9zffYBxdxKidxP8Fg47zYw97NbVj7P/nFW+E=") def callback(caller, event): self.assertFalse(isinstance(event, cpp.LoadExceptionEvent)) self.assertFalse(isinstance(event, cpp.RunExceptionEvent)) T.maybe_download_sbs() for windows_mode in ((False, True) if sys.platform.startswith("win") else (False, )): pipeline = cpp.Pipeline() pipeline.add_listener(callback) pipeline.load(StringIO(zlib.decompress(base64.b64decode(data)))) ipath = os.path.join(T.example_images_directory(), "ExampleSBSImages") bpath = tempfile.mkdtemp() bfile = os.path.join(bpath, C.F_BATCH_DATA) hfile = os.path.join(bpath, C.F_BATCH_DATA_H5) try: li = pipeline.modules()[0] self.assertTrue(isinstance(li, LI.LoadImages)) module = pipeline.modules()[1] self.assertTrue(isinstance(module, C.CreateBatchFiles)) li.location.dir_choice = cps.ABSOLUTE_FOLDER_NAME li.location.custom_path = ipath module.wants_default_output_directory.value = False module.custom_output_directory.value = bpath module.remote_host_is_windows.value = windows_mode self.assertEqual(len(module.mappings), 1) mapping = module.mappings[0] mapping.local_directory.value = ipath self.assertFalse(pipeline.in_batch_mode()) measurements = cpmeas.Measurements(mode="memory") image_set_list = cpi.ImageSetList() result = pipeline.prepare_run( cpw.Workspace(pipeline, None, None, None, measurements, image_set_list)) self.assertFalse(pipeline.in_batch_mode()) self.assertFalse(result) self.assertFalse(module.batch_mode.value) self.assertTrue( measurements.has_feature(cpmeas.EXPERIMENT, cpp.M_PIPELINE)) pipeline = cpp.Pipeline() pipeline.add_listener(callback) image_set_list = cpi.ImageSetList() measurements = cpmeas.Measurements(mode="memory") workspace = cpw.Workspace(pipeline, None, None, None, cpmeas.Measurements(), image_set_list) workspace.load(hfile, True) measurements = workspace.measurements self.assertTrue(pipeline.in_batch_mode()) module = pipeline.modules()[1] self.assertTrue(isinstance(module, C.CreateBatchFiles)) self.assertTrue(module.batch_mode.value) image_numbers = measurements.get_image_numbers() self.assertTrue( [x == i + 1 for i, x in enumerate(image_numbers)]) pipeline.prepare_run(workspace) pipeline.prepare_group(workspace, {}, list(range(1, 97))) for i in range(96): image_set = image_set_list.get_image_set(i) for image_name in ("DNA", "Cytoplasm"): pathname = measurements.get_measurement( cpmeas.IMAGE, "PathName_" + image_name, i + 1) self.assertEqual( pathname, "\\imaging\\analysis" if windows_mode else "/imaging/analysis", ) measurements.close() finally: if os.path.exists(bfile): os.unlink(bfile) if os.path.exists(hfile): os.unlink(hfile) os.rmdir(bpath)
def test_03_01_save_and_load(self): '''Save a pipeline to batch data, open it to check and load it''' data = ('eJztWW1PGkEQXhC1WtPYTzb9tB+llROoGiWNgi9NSYUSIbZGbbvCApvu7ZJ7' 'UWlj0o/9Wf1J/QndxTs4tsoBRS3JHbkcMzfPPDOzs8uxl8uU9jPbcFWLw1ym' 'FKsSimGBIqvKDT0FmbUEdwyMLFyBnKVgycYwY9dgIgET8dTqRmolCZPx+AYY' '7ghlc0/EJf4cgClxfSTOsHNr0pFDnlPKRWxZhNXMSRABzxz9L3EeIoOgM4oP' 'EbWx2aFw9VlW5aVmo30rxys2xXmke43Fkbf1M2yY76su0LldIJeYFsk3rKTg' 'mh3gc2ISzhy841/Vtnm5pfDKOhTmOnUIKXWQdVnw6KX9W9Cxj9xQt6ce+3lH' 'JqxCzknFRhQSHdXaUbTGwcffRJe/CXAk0BKX9sHNK3HIs4QvrdjeJSpbUEdW' 'uS79rPv4mVb8SLmcOrGw3ugr/lAXPgRe9Zl3uAsXBnkO+sp7VolXyrscMm5B' '28T91/02/lHgphSce7i4GdCJ0y/fOSVfKe9RE1/UsYE1TXP91H38rCh+pCzG' 'uYwpbRhcLlHGiWXY7OuJaCC9ISZ3q5NdqbhdzLZb+yH4hpmXy3I2ioVtGTFE' 'myYZxbwcdpzvu68eku8+cGmf/GZAdz9IeaeOGMM0ERtx3P30z24+c6d86jqc' 'uOP8Il18EdE/DP8L3w8fvnegezyl/Glxq/BaPljhTe1l9LOUPoj15YBfbB5n' 'YoXTqKvZ4dTW2eZxPLZx+j2xlLy6Ni4SgWwpozfmPUj8fuvhuhK/lGUMRxgZ' 'TmArV9GYVOU4s+qOLunodlGzo3mgeZMcxbwZir9p8QZFpj4C/kHnUfKO+YJ5' 'NJ7z6OPsYP8rxuV3NcAFuAAX4P43XNqD63c/pLUZUzO43YCEVXBjnPINcOON' 'S4OgXwPc8DipvO35Ut2/kfZfQO9+ewG6+03K3s04TW9topsa5ahyvYut7Yuv' 'Wc+Gdj/P52sKz9ptPOXWK5AzuU8tb5ja9TuRbal4Q1rvCNT6zdzA561DWHwW' 'pnvXXa13Zxx+bw3DFwn9zffYBxdxKidxP8Fg47zYw97NbVj7P/nFW+E=') def callback(caller, event): self.assertFalse(isinstance(event, cpp.LoadExceptionEvent)) self.assertFalse(isinstance(event, cpp.RunExceptionEvent)) T.maybe_download_sbs() for windows_mode in ((False, True) if sys.platform.startswith("win") else (False,)): pipeline = cpp.Pipeline() pipeline.add_listener(callback) pipeline.load(StringIO(zlib.decompress(base64.b64decode(data)))) ipath = os.path.join(T.example_images_directory(), 'ExampleSBSImages') bpath = tempfile.mkdtemp() bfile = os.path.join(bpath, C.F_BATCH_DATA) hfile = os.path.join(bpath, C.F_BATCH_DATA_H5) try: li = pipeline.modules()[0] self.assertTrue(isinstance(li, LI.LoadImages)) module = pipeline.modules()[1] self.assertTrue(isinstance(module, C.CreateBatchFiles)) li.location.dir_choice = LI.ABSOLUTE_FOLDER_NAME li.location.custom_path = ipath module.wants_default_output_directory.value = False module.custom_output_directory.value = bpath module.remote_host_is_windows.value = windows_mode self.assertEqual(len(module.mappings), 1) mapping = module.mappings[0] mapping.local_directory.value = ipath self.assertFalse(pipeline.in_batch_mode()) measurements = cpmeas.Measurements(mode="memory") image_set_list = cpi.ImageSetList() result = pipeline.prepare_run( cpw.Workspace(pipeline, None, None, None, measurements, image_set_list)) self.assertFalse(pipeline.in_batch_mode()) self.assertFalse(result) self.assertFalse(module.batch_mode.value) self.assertTrue(measurements.has_feature( cpmeas.EXPERIMENT, cpp.M_PIPELINE)) pipeline = cpp.Pipeline() pipeline.add_listener(callback) image_set_list = cpi.ImageSetList() measurements = cpmeas.Measurements(mode="memory") workspace = cpw.Workspace(pipeline, None, None, None, cpmeas.Measurements(), image_set_list) workspace.load(hfile, True) measurements = workspace.measurements self.assertTrue(pipeline.in_batch_mode()) module = pipeline.modules()[1] self.assertTrue(isinstance(module, C.CreateBatchFiles)) self.assertTrue(module.batch_mode.value) image_numbers = measurements.get_image_numbers() self.assertTrue([x == i + 1 for i, x in enumerate(image_numbers)]) pipeline.prepare_run(workspace) pipeline.prepare_group(workspace, {}, range(1, 97)) for i in range(96): image_set = image_set_list.get_image_set(i) for image_name in ('DNA', 'Cytoplasm'): pathname = measurements.get_measurement( cpmeas.IMAGE, "PathName_" + image_name, i + 1) self.assertEqual(pathname, '\\imaging\\analysis' if windows_mode else '/imaging/analysis') measurements.close() finally: if os.path.exists(bfile): os.unlink(bfile) if os.path.exists(hfile): os.unlink(hfile) os.rmdir(bpath)