def test_clog(self): im_nuclei_stain_data = np.load( utilities.externaldata('data/Easy1_nuclei_stain.npz.sha512')) im_nuclei_stain = im_nuclei_stain_data['Easy1_nuclei_stain'] im_nuclei_fgnd_mask_data = np.load( utilities.externaldata('data/Easy1_nuclei_fgnd_mask.npz.sha512')) im_nuclei_fgnd_mask = im_nuclei_fgnd_mask_data['Easy1_nuclei_fgnd_mask'] sigma_min = 10.0 / np.sqrt(2.0) sigma_max = 40.0 / np.sqrt(2.0) im_log_max, im_sigma_max = clog( im_input=im_nuclei_stain, im_mask=im_nuclei_fgnd_mask, sigma_min=sigma_min, sigma_max=sigma_max) im_log_max_gtruth_data = np.load(os.path.join(utilities.externaldata( 'data/Easy1_clog_max.npz.sha512'))) im_log_max_gtruth = im_log_max_gtruth_data['Easy1_clog_max'] assert_array_almost_equal_neighborhood_lines(im_log_max, im_log_max_gtruth, decimal=4) compare_maxima(im_log_max, im_log_max_gtruth) im_sigma_max_gtruth_data = np.load(os.path.join(utilities.externaldata( 'data/Easy1_clog_sigma_max.npz.sha512'))) im_sigma_max_gtruth = im_sigma_max_gtruth_data['Easy1_clog_sigma_max'] assert_array_almost_equal_neighborhood_lines(im_sigma_max, im_sigma_max_gtruth, decimal=4)
def test_background_intensity(self): wsi_path = os.path.join( utilities.externaldata( 'data/sample_svs_image.TCGA-DU-6399-01A-01-TS1.e8eb65de-d63e-42db-af6f-14fefbbdf7bd.svs.sha512' # noqa )) np.random.seed(1) # create dask client args = { # In Python 3 unittesting, the scheduler fails if it uses processes 'scheduler': 'multithreading', # None, 'num_workers': -1, 'num_threads_per_worker': 1, } args = collections.namedtuple('Parameters', args.keys())(**args) cli_utils.create_dask_client(args) # compute background intensity I_0 = htk_cn.background_intensity(wsi_path, sample_approximate_total=5000) np.testing.assert_allclose(I_0, [242, 244, 241], atol=1)
def test_reinhard_stats(self): wsi_path = os.path.join( utilities.externaldata( 'data/sample_svs_image.TCGA-DU-6399-01A-01-TS1.e8eb65de-d63e-42db-af6f-14fefbbdf7bd.svs.sha512' # noqa )) np.random.seed(1) # create dask client args = { # In Python 3 unittesting, the scheduler fails if it uses processes 'scheduler': 'multithreading', # None, 'num_workers': -1, 'num_threads_per_worker': 1, } args = collections.namedtuple('Parameters', args.keys())(**args) cli_utils.create_dask_client(args) # compute reinhard stats wsi_mean, wsi_stddev = htk_cn.reinhard_stats(wsi_path, 0.1, magnification=20) gt_mean = [8.896134, -0.074579, 0.022006] gt_stddev = [0.612143, 0.122667, 0.021361] np.testing.assert_allclose(wsi_mean, gt_mean, atol=1e-2) np.testing.assert_allclose(wsi_stddev, gt_stddev, atol=1e-2)
def test_macenko(self): im_path = utilities.externaldata('data/Easy1.png.sha512') im = skimage.io.imread(im_path)[..., :3] w = htk_dcv.rgb_separate_stains_macenko_pca(im, 255) w_expected = [[0.089411, 0.558021, -0.130574], [0.837138, 0.729935, 0.546981], [0.539635, 0.394725, -0.826899]] np.testing.assert_allclose(w, w_expected, atol=1e-6)
def test_roundtrip(self): im_path = utilities.externaldata('data/Easy1.png.sha512') im = skimage.io.imread(im_path)[..., :3] w = np.array([[0.650, 0.072, 0], [0.704, 0.990, 0], [0.286, 0.105, 0]]) conv_result = htk_dcv.color_deconvolution(im, w, 255) im_reconv = htk_dcv.color_convolution(conv_result.StainsFloat, conv_result.Wc, 255) np.testing.assert_allclose(im, im_reconv, atol=1)
def test_segment_wsi_foreground_at_low_res(self): np.random.seed(0) wsi_path = os.path.join( utilities.externaldata( 'data/TCGA-06-0129-01Z-00-DX3.bae772ea-dd36-47ec-8185-761989be3cc8.svs.sha512' # noqa )) ts = large_image.getTileSource(wsi_path) im_fgnd_mask_lres, fgnd_seg_scale = \ cli_utils.segment_wsi_foreground_at_low_res(ts) np.testing.assert_equal(fgnd_seg_scale['magnification'], 2.5) fgnd_mask_gtruth_file = os.path.join( utilities.externaldata( 'data/TCGA-06-0129-01Z-00-DX3_fgnd_mask_lres.png.sha512')) im_fgnd_mask_lres_gtruth = skimage.io.imread(fgnd_mask_gtruth_file) > 0 np.testing.assert_array_equal(im_fgnd_mask_lres > 0, im_fgnd_mask_lres_gtruth)
def test_get_region_dict(self): ts = large_image.getTileSource( os.path.join(utilities.externaldata('data/Easy1.png.sha512'))) result = cli_utils.get_region_dict([-1, -1, -1, -1], 2000, ts) expected = {} assert result == expected, "Expected {}, got {}".format( expected, result) result = cli_utils.get_region_dict([100, 110, 250, 240], 500, ts) expected = dict(region=dict(left=100, top=110, width=250, height=240)) assert result == expected, "Expected {}, got {}".format( expected, result)
def testColorDeconvolutionDefaults(self): self._runTest( cli_args=[ 'ColorDeconvolution', utilities.externaldata('data/Easy1.png.sha512'), ] + [f'tmp_out_{i}.png' for i in (1, 2, 3)], outputs={ 'tmp_out_1.png': dict( hash= 'b91d961b7eba8c02a1c067da91fced315fa3922db73f489202a296f4c2304b94', # noqa ), }, )
def test_create_tile_nuclei_annotations(self): wsi_path = os.path.join( utilities.externaldata( 'data/TCGA-06-0129-01Z-00-DX3.bae772ea-dd36-47ec-8185-761989be3cc8.svs.sha512' # noqa )) # define parameters args = { 'reference_mu_lab': [8.63234435, -0.11501964, 0.03868433], 'reference_std_lab': [0.57506023, 0.10403329, 0.01364062], 'stain_1': 'hematoxylin', 'stain_2': 'eosin', 'stain_3': 'null', 'stain_1_vector': [-1, -1, -1], 'stain_2_vector': [-1, -1, -1], 'stain_3_vector': [-1, -1, -1], 'min_fgnd_frac': 0.50, 'analysis_mag': 20, 'analysis_tile_size': 1200, 'foreground_threshold': 60, 'min_radius': 6, 'max_radius': 12, 'min_nucleus_area': 25, 'local_max_search_radius': 8, # In Python 3 unittesting, the scheduler fails if it uses processes 'scheduler': 'multithreading', # None, 'num_workers': -1, 'num_threads_per_worker': 1, } args = collections.namedtuple('Parameters', args.keys())(**args) # read WSI ts = large_image.getTileSource(wsi_path) ts_metadata = ts.getMetadata() analysis_tile_size = { 'width': int(ts_metadata['tileWidth'] * np.floor( 1.0 * args.analysis_tile_size / ts_metadata['tileWidth'])), 'height': int(ts_metadata['tileHeight'] * np.floor( 1.0 * args.analysis_tile_size / ts_metadata['tileHeight'])) } # define ROI roi = { 'left': ts_metadata['sizeX'] / 2, 'top': ts_metadata['sizeY'] * 3 / 4, 'width': analysis_tile_size['width'], 'height': analysis_tile_size['height'], 'units': 'base_pixels' } # define tile iterator parameters it_kwargs = { 'tile_size': { 'width': args.analysis_tile_size }, 'scale': { 'magnification': args.analysis_mag }, 'region': roi } # create dask client cli_utils.create_dask_client(args) # get tile foregreoung at low res im_fgnd_mask_lres, fgnd_seg_scale = \ cli_utils.segment_wsi_foreground_at_low_res(ts) # compute tile foreground fraction tile_fgnd_frac_list = htk_utils.compute_tile_foreground_fraction( wsi_path, im_fgnd_mask_lres, fgnd_seg_scale, it_kwargs) num_fgnd_tiles = np.count_nonzero( tile_fgnd_frac_list >= args.min_fgnd_frac) np.testing.assert_equal(num_fgnd_tiles, 2) # create nuclei annotations nuclei_bbox_annot_list = [] nuclei_bndry_annot_list = [] for tile_info in ts.tileIterator( format=large_image.tilesource.TILE_FORMAT_NUMPY, **it_kwargs): im_tile = tile_info['tile'][:, :, :3] # perform color normalization im_nmzd = htk_cnorm.reinhard(im_tile, args.reference_mu_lab, args.reference_std_lab) # perform color deconvolution w = cli_utils.get_stain_matrix(args) im_stains = htk_cdeconv.color_deconvolution(im_nmzd, w).Stains im_nuclei_stain = im_stains[:, :, 0].astype(np.float) # segment nuclei im_nuclei_seg_mask = htk_nuclear.detect_nuclei_kofahi( im_nuclei_stain, im_nuclei_stain < args.foreground_threshold, args.min_radius, args.max_radius, args.min_nucleus_area, args.local_max_search_radius) # generate nuclei annotations as bboxes cur_bbox_annot_list = cli_utils.create_tile_nuclei_annotations( im_nuclei_seg_mask, tile_info, 'bbox') nuclei_bbox_annot_list.extend(cur_bbox_annot_list) # generate nuclei annotations as boundaries cur_bndry_annot_list = cli_utils.create_tile_nuclei_annotations( im_nuclei_seg_mask, tile_info, 'boundary') nuclei_bndry_annot_list.extend(cur_bndry_annot_list) # compare nuclei bbox annotations with gtruth nuclei_bbox_annot_gtruth_file = os.path.join( utilities.externaldata( 'data/TCGA-06-0129-01Z-00-DX3_roi_nuclei_bbox.anot.sha512' # noqa )) with open(nuclei_bbox_annot_gtruth_file, 'r') as fbbox_annot: nuclei_bbox_annot_list_gtruth = json.load(fbbox_annot)['elements'] # Check that nuclei_bbox_annot_list is nearly equal to # nuclei_bbox_annot_list_gtruth assert len(nuclei_bbox_annot_list) == len( nuclei_bbox_annot_list_gtruth) for pos in range(len(nuclei_bbox_annot_list)): np.testing.assert_array_almost_equal( nuclei_bbox_annot_list[pos]['center'], nuclei_bbox_annot_list_gtruth[pos]['center'], 0) np.testing.assert_almost_equal( nuclei_bbox_annot_list[pos]['width'], nuclei_bbox_annot_list_gtruth[pos]['width'], 1) np.testing.assert_almost_equal( nuclei_bbox_annot_list[pos]['height'], nuclei_bbox_annot_list_gtruth[pos]['height'], 1) # compare nuclei boundary annotations with gtruth nuclei_bndry_annot_gtruth_file = os.path.join( utilities.externaldata( 'data/TCGA-06-0129-01Z-00-DX3_roi_nuclei_boundary.anot.sha512' # noqa )) with open(nuclei_bndry_annot_gtruth_file, 'r') as fbndry_annot: nuclei_bndry_annot_list_gtruth = json.load( fbndry_annot)['elements'] assert len(nuclei_bndry_annot_list) == len( nuclei_bndry_annot_list_gtruth) for pos in range(len(nuclei_bndry_annot_list)): np.testing.assert_array_almost_equal( nuclei_bndry_annot_list[pos]['points'], nuclei_bndry_annot_list_gtruth[pos]['points'], 0)
def test_setup(self): # define parameters args = { 'reference_mu_lab': [8.63234435, -0.11501964, 0.03868433], 'reference_std_lab': [0.57506023, 0.10403329, 0.01364062], 'min_radius': 12, 'max_radius': 30, 'foreground_threshold': 60, 'min_nucleus_area': 80, 'local_max_search_radius': 10, } args = collections.namedtuple('Parameters', args.keys())(**args) # read input image input_image_file = utilities.externaldata('data/Easy1.png.sha512') im_input = skimage.io.imread(input_image_file)[:, :, :3] # perform color normalization im_input_nmzd = htk_cnorm.reinhard(im_input, args.reference_mu_lab, args.reference_std_lab) # perform color decovolution w = htk_cdeconv.rgb_separate_stains_macenko_pca( im_input_nmzd, im_input_nmzd.max()) im_stains = htk_cdeconv.color_deconvolution(im_input_nmzd, w).Stains nuclei_channel = htk_cdeconv.find_stain_index( htk_cdeconv.stain_color_map['hematoxylin'], w) im_nuclei_stain = im_stains[:, :, nuclei_channel].astype(np.float) cytoplasm_channel = htk_cdeconv.find_stain_index( htk_cdeconv.stain_color_map['eosin'], w) im_cytoplasm_stain = im_stains[:, :, cytoplasm_channel].astype(np.float) # segment nuclei im_nuclei_seg_mask = htk_nuclear.detect_nuclei_kofahi( im_nuclei_stain, im_nuclei_stain < args.foreground_threshold, args.min_radius, args.max_radius, args.min_nucleus_area, args.local_max_search_radius) # perform connected component analysis nuclei_rprops = skimage.measure.regionprops(im_nuclei_seg_mask) # compute nuclei features fdata_nuclei = htk_features.compute_nuclei_features( im_nuclei_seg_mask, im_nuclei_stain, im_cytoplasm=im_cytoplasm_stain) cfg.im_input = im_input cfg.im_input_nmzd = im_input_nmzd cfg.im_nuclei_stain = im_nuclei_stain cfg.im_nuclei_seg_mask = im_nuclei_seg_mask cfg.nuclei_rprops = nuclei_rprops cfg.fdata_nuclei = fdata_nuclei
def test_segment_nuclei_kofahi(self): input_image_file = utilities.externaldata('data/Easy1.png.sha512') ref_image_file = utilities.externaldata('data/L1.png.sha512') # read input image im_input = skimage.io.imread(input_image_file)[:, :, :3] # read reference image im_reference = skimage.io.imread(ref_image_file)[:, :, :3] # get mean and stddev of reference image in lab space mean_ref, std_ref = htk_cvt.lab_mean_std(im_reference) # perform color normalization im_nmzd = htk_cnorm.reinhard(im_input, mean_ref, std_ref) # perform color decovolution stain_color_map = { 'hematoxylin': [0.65, 0.70, 0.29], 'eosin': [0.07, 0.99, 0.11], 'dab': [0.27, 0.57, 0.78], 'null': [0.0, 0.0, 0.0] } w = htk_cdeconv.rgb_separate_stains_macenko_pca(im_nmzd, im_nmzd.max()) im_stains = htk_cdeconv.color_deconvolution(im_nmzd, w).Stains nuclei_channel = htk_cdeconv.find_stain_index( stain_color_map['hematoxylin'], w) im_nuclei_stain = im_stains[:, :, nuclei_channel].astype(np.float) # segment nuclei im_nuclei_seg_mask = htk_seg.nuclear.detect_nuclei_kofahi( im_nuclei_stain, im_nuclei_stain < 60, min_radius=20, max_radius=30, min_nucleus_area=80, local_max_search_radius=10) num_nuclei = len(np.unique(im_nuclei_seg_mask)) - 1 # check if segmentation mask matches ground truth gtruth_mask_file = os.path.join( utilities.externaldata('data/Easy1_nuclei_seg_kofahi.npy.sha512')) im_gtruth_mask = np.load(gtruth_mask_file) num_nuclei_gtruth = len(np.unique(im_gtruth_mask)) - 1 assert num_nuclei == num_nuclei_gtruth np.testing.assert_allclose(im_nuclei_seg_mask, im_gtruth_mask) # check no nuclei case im_nuclei_seg_mask = htk_seg.nuclear.detect_nuclei_kofahi( 255 * np.ones_like(im_nuclei_stain), np.ones_like(im_nuclei_stain), min_radius=20, max_radius=30, min_nucleus_area=80, local_max_search_radius=10) num_nuclei = len(np.unique(im_nuclei_seg_mask)) - 1 assert num_nuclei == 0
'*****@*****.**') adminUser = User().findOne({'admin': True}) if Assetstore().findOne() is None: Assetstore().createFilesystemAssetstore('Assetstore', '/assetstore') fsAssetstore = Assetstore().findOne() # Upload a list of items publicFolder = namedFolder(adminUser) dataFiles = { 'tcga1': { 'sha512': 'data/TCGA-A2-A0YE-01Z-00-DX1.8A2E3094-5755-42BC-969D-' '7F0A2ECA0F39.svs.sha512', }, } for key, entry in dataFiles.items(): path = utilities.externaldata(entry['sha512']) query = {'folderId': publicFolder['_id'], 'name': os.path.basename(path)} if not Item().findOne(query): with ProgressContext(False, user=adminUser) as ctx: Assetstore().importData(fsAssetstore, publicFolder, 'folder', {'importPath': path}, ctx, adminUser, leafFoldersAsItems=False) entry['item'] = Item().findOne(query) print(entry['item']) # Add annotations annotationFiles = { 'tcga1': { 'item':