def test_clog(self):
im_nuclei_stain_data = np.load(
utilities.externaldata('data/Easy1_nuclei_stain.npz.sha512'))
im_nuclei_stain = im_nuclei_stain_data['Easy1_nuclei_stain']
im_nuclei_fgnd_mask_data = np.load(
utilities.externaldata('data/Easy1_nuclei_fgnd_mask.npz.sha512'))
im_nuclei_fgnd_mask = im_nuclei_fgnd_mask_data['Easy1_nuclei_fgnd_mask']
sigma_min = 10.0 / np.sqrt(2.0)
sigma_max = 40.0 / np.sqrt(2.0)
im_log_max, im_sigma_max = clog(
im_input=im_nuclei_stain, im_mask=im_nuclei_fgnd_mask,
sigma_min=sigma_min, sigma_max=sigma_max)
im_log_max_gtruth_data = np.load(os.path.join(utilities.externaldata(
'data/Easy1_clog_max.npz.sha512')))
im_log_max_gtruth = im_log_max_gtruth_data['Easy1_clog_max']
assert_array_almost_equal_neighborhood_lines(im_log_max, im_log_max_gtruth, decimal=4)
compare_maxima(im_log_max, im_log_max_gtruth)
im_sigma_max_gtruth_data = np.load(os.path.join(utilities.externaldata(
'data/Easy1_clog_sigma_max.npz.sha512')))
im_sigma_max_gtruth = im_sigma_max_gtruth_data['Easy1_clog_sigma_max']
assert_array_almost_equal_neighborhood_lines(im_sigma_max, im_sigma_max_gtruth, decimal=4)
def test_background_intensity(self):
wsi_path = os.path.join(
utilities.externaldata(
'data/sample_svs_image.TCGA-DU-6399-01A-01-TS1.e8eb65de-d63e-42db-af6f-14fefbbdf7bd.svs.sha512' # noqa
))
np.random.seed(1)
# create dask client
args = {
# In Python 3 unittesting, the scheduler fails if it uses processes
'scheduler': 'multithreading', # None,
'num_workers': -1,
'num_threads_per_worker': 1,
}
args = collections.namedtuple('Parameters', args.keys())(**args)
cli_utils.create_dask_client(args)
# compute background intensity
I_0 = htk_cn.background_intensity(wsi_path,
sample_approximate_total=5000)
np.testing.assert_allclose(I_0, [242, 244, 241], atol=1)
def test_reinhard_stats(self):
wsi_path = os.path.join(
utilities.externaldata(
'data/sample_svs_image.TCGA-DU-6399-01A-01-TS1.e8eb65de-d63e-42db-af6f-14fefbbdf7bd.svs.sha512' # noqa
))
np.random.seed(1)
# create dask client
args = {
# In Python 3 unittesting, the scheduler fails if it uses processes
'scheduler': 'multithreading', # None,
'num_workers': -1,
'num_threads_per_worker': 1,
}
args = collections.namedtuple('Parameters', args.keys())(**args)
cli_utils.create_dask_client(args)
# compute reinhard stats
wsi_mean, wsi_stddev = htk_cn.reinhard_stats(wsi_path,
0.1,
magnification=20)
gt_mean = [8.896134, -0.074579, 0.022006]
gt_stddev = [0.612143, 0.122667, 0.021361]
np.testing.assert_allclose(wsi_mean, gt_mean, atol=1e-2)
np.testing.assert_allclose(wsi_stddev, gt_stddev, atol=1e-2)
def test_macenko(self):
im_path = utilities.externaldata('data/Easy1.png.sha512')
im = skimage.io.imread(im_path)[..., :3]
w = htk_dcv.rgb_separate_stains_macenko_pca(im, 255)
w_expected = [[0.089411, 0.558021, -0.130574],
[0.837138, 0.729935, 0.546981],
[0.539635, 0.394725, -0.826899]]
np.testing.assert_allclose(w, w_expected, atol=1e-6)
def test_roundtrip(self):
im_path = utilities.externaldata('data/Easy1.png.sha512')
im = skimage.io.imread(im_path)[..., :3]
w = np.array([[0.650, 0.072, 0], [0.704, 0.990, 0], [0.286, 0.105, 0]])
conv_result = htk_dcv.color_deconvolution(im, w, 255)
im_reconv = htk_dcv.color_convolution(conv_result.StainsFloat,
conv_result.Wc, 255)
np.testing.assert_allclose(im, im_reconv, atol=1)
def test_segment_wsi_foreground_at_low_res(self):
np.random.seed(0)
wsi_path = os.path.join(
utilities.externaldata(
'data/TCGA-06-0129-01Z-00-DX3.bae772ea-dd36-47ec-8185-761989be3cc8.svs.sha512' # noqa
))
ts = large_image.getTileSource(wsi_path)
im_fgnd_mask_lres, fgnd_seg_scale = \
cli_utils.segment_wsi_foreground_at_low_res(ts)
np.testing.assert_equal(fgnd_seg_scale['magnification'], 2.5)
fgnd_mask_gtruth_file = os.path.join(
utilities.externaldata(
'data/TCGA-06-0129-01Z-00-DX3_fgnd_mask_lres.png.sha512'))
im_fgnd_mask_lres_gtruth = skimage.io.imread(fgnd_mask_gtruth_file) > 0
np.testing.assert_array_equal(im_fgnd_mask_lres > 0,
im_fgnd_mask_lres_gtruth)
def test_get_region_dict(self):
ts = large_image.getTileSource(
os.path.join(utilities.externaldata('data/Easy1.png.sha512')))
result = cli_utils.get_region_dict([-1, -1, -1, -1], 2000, ts)
expected = {}
assert result == expected, "Expected {}, got {}".format(
expected, result)
result = cli_utils.get_region_dict([100, 110, 250, 240], 500, ts)
expected = dict(region=dict(left=100, top=110, width=250, height=240))
assert result == expected, "Expected {}, got {}".format(
expected, result)
def testColorDeconvolutionDefaults(self):
self._runTest(
cli_args=[
'ColorDeconvolution',
utilities.externaldata('data/Easy1.png.sha512'),
] + [f'tmp_out_{i}.png' for i in (1, 2, 3)],
outputs={
'tmp_out_1.png':
dict(
hash=
'b91d961b7eba8c02a1c067da91fced315fa3922db73f489202a296f4c2304b94', # noqa
),
},
)
def test_create_tile_nuclei_annotations(self):
wsi_path = os.path.join(
utilities.externaldata(
'data/TCGA-06-0129-01Z-00-DX3.bae772ea-dd36-47ec-8185-761989be3cc8.svs.sha512' # noqa
))
# define parameters
args = {
'reference_mu_lab': [8.63234435, -0.11501964, 0.03868433],
'reference_std_lab': [0.57506023, 0.10403329, 0.01364062],
'stain_1': 'hematoxylin',
'stain_2': 'eosin',
'stain_3': 'null',
'stain_1_vector': [-1, -1, -1],
'stain_2_vector': [-1, -1, -1],
'stain_3_vector': [-1, -1, -1],
'min_fgnd_frac': 0.50,
'analysis_mag': 20,
'analysis_tile_size': 1200,
'foreground_threshold': 60,
'min_radius': 6,
'max_radius': 12,
'min_nucleus_area': 25,
'local_max_search_radius': 8,
# In Python 3 unittesting, the scheduler fails if it uses processes
'scheduler': 'multithreading', # None,
'num_workers': -1,
'num_threads_per_worker': 1,
}
args = collections.namedtuple('Parameters', args.keys())(**args)
# read WSI
ts = large_image.getTileSource(wsi_path)
ts_metadata = ts.getMetadata()
analysis_tile_size = {
'width':
int(ts_metadata['tileWidth'] * np.floor(
1.0 * args.analysis_tile_size / ts_metadata['tileWidth'])),
'height':
int(ts_metadata['tileHeight'] * np.floor(
1.0 * args.analysis_tile_size / ts_metadata['tileHeight']))
}
# define ROI
roi = {
'left': ts_metadata['sizeX'] / 2,
'top': ts_metadata['sizeY'] * 3 / 4,
'width': analysis_tile_size['width'],
'height': analysis_tile_size['height'],
'units': 'base_pixels'
}
# define tile iterator parameters
it_kwargs = {
'tile_size': {
'width': args.analysis_tile_size
},
'scale': {
'magnification': args.analysis_mag
},
'region': roi
}
# create dask client
cli_utils.create_dask_client(args)
# get tile foregreoung at low res
im_fgnd_mask_lres, fgnd_seg_scale = \
cli_utils.segment_wsi_foreground_at_low_res(ts)
# compute tile foreground fraction
tile_fgnd_frac_list = htk_utils.compute_tile_foreground_fraction(
wsi_path, im_fgnd_mask_lres, fgnd_seg_scale, it_kwargs)
num_fgnd_tiles = np.count_nonzero(
tile_fgnd_frac_list >= args.min_fgnd_frac)
np.testing.assert_equal(num_fgnd_tiles, 2)
# create nuclei annotations
nuclei_bbox_annot_list = []
nuclei_bndry_annot_list = []
for tile_info in ts.tileIterator(
format=large_image.tilesource.TILE_FORMAT_NUMPY, **it_kwargs):
im_tile = tile_info['tile'][:, :, :3]
# perform color normalization
im_nmzd = htk_cnorm.reinhard(im_tile, args.reference_mu_lab,
args.reference_std_lab)
# perform color deconvolution
w = cli_utils.get_stain_matrix(args)
im_stains = htk_cdeconv.color_deconvolution(im_nmzd, w).Stains
im_nuclei_stain = im_stains[:, :, 0].astype(np.float)
# segment nuclei
im_nuclei_seg_mask = htk_nuclear.detect_nuclei_kofahi(
im_nuclei_stain, im_nuclei_stain < args.foreground_threshold,
args.min_radius, args.max_radius, args.min_nucleus_area,
args.local_max_search_radius)
# generate nuclei annotations as bboxes
cur_bbox_annot_list = cli_utils.create_tile_nuclei_annotations(
im_nuclei_seg_mask, tile_info, 'bbox')
nuclei_bbox_annot_list.extend(cur_bbox_annot_list)
# generate nuclei annotations as boundaries
cur_bndry_annot_list = cli_utils.create_tile_nuclei_annotations(
im_nuclei_seg_mask, tile_info, 'boundary')
nuclei_bndry_annot_list.extend(cur_bndry_annot_list)
# compare nuclei bbox annotations with gtruth
nuclei_bbox_annot_gtruth_file = os.path.join(
utilities.externaldata(
'data/TCGA-06-0129-01Z-00-DX3_roi_nuclei_bbox.anot.sha512' # noqa
))
with open(nuclei_bbox_annot_gtruth_file, 'r') as fbbox_annot:
nuclei_bbox_annot_list_gtruth = json.load(fbbox_annot)['elements']
# Check that nuclei_bbox_annot_list is nearly equal to
# nuclei_bbox_annot_list_gtruth
assert len(nuclei_bbox_annot_list) == len(
nuclei_bbox_annot_list_gtruth)
for pos in range(len(nuclei_bbox_annot_list)):
np.testing.assert_array_almost_equal(
nuclei_bbox_annot_list[pos]['center'],
nuclei_bbox_annot_list_gtruth[pos]['center'], 0)
np.testing.assert_almost_equal(
nuclei_bbox_annot_list[pos]['width'],
nuclei_bbox_annot_list_gtruth[pos]['width'], 1)
np.testing.assert_almost_equal(
nuclei_bbox_annot_list[pos]['height'],
nuclei_bbox_annot_list_gtruth[pos]['height'], 1)
# compare nuclei boundary annotations with gtruth
nuclei_bndry_annot_gtruth_file = os.path.join(
utilities.externaldata(
'data/TCGA-06-0129-01Z-00-DX3_roi_nuclei_boundary.anot.sha512' # noqa
))
with open(nuclei_bndry_annot_gtruth_file, 'r') as fbndry_annot:
nuclei_bndry_annot_list_gtruth = json.load(
fbndry_annot)['elements']
assert len(nuclei_bndry_annot_list) == len(
nuclei_bndry_annot_list_gtruth)
for pos in range(len(nuclei_bndry_annot_list)):
np.testing.assert_array_almost_equal(
nuclei_bndry_annot_list[pos]['points'],
nuclei_bndry_annot_list_gtruth[pos]['points'], 0)
def test_setup(self):
# define parameters
args = {
'reference_mu_lab': [8.63234435, -0.11501964, 0.03868433],
'reference_std_lab': [0.57506023, 0.10403329, 0.01364062],
'min_radius': 12,
'max_radius': 30,
'foreground_threshold': 60,
'min_nucleus_area': 80,
'local_max_search_radius': 10,
}
args = collections.namedtuple('Parameters', args.keys())(**args)
# read input image
input_image_file = utilities.externaldata('data/Easy1.png.sha512')
im_input = skimage.io.imread(input_image_file)[:, :, :3]
# perform color normalization
im_input_nmzd = htk_cnorm.reinhard(im_input, args.reference_mu_lab,
args.reference_std_lab)
# perform color decovolution
w = htk_cdeconv.rgb_separate_stains_macenko_pca(
im_input_nmzd, im_input_nmzd.max())
im_stains = htk_cdeconv.color_deconvolution(im_input_nmzd, w).Stains
nuclei_channel = htk_cdeconv.find_stain_index(
htk_cdeconv.stain_color_map['hematoxylin'], w)
im_nuclei_stain = im_stains[:, :, nuclei_channel].astype(np.float)
cytoplasm_channel = htk_cdeconv.find_stain_index(
htk_cdeconv.stain_color_map['eosin'], w)
im_cytoplasm_stain = im_stains[:, :,
cytoplasm_channel].astype(np.float)
# segment nuclei
im_nuclei_seg_mask = htk_nuclear.detect_nuclei_kofahi(
im_nuclei_stain, im_nuclei_stain < args.foreground_threshold,
args.min_radius, args.max_radius, args.min_nucleus_area,
args.local_max_search_radius)
# perform connected component analysis
nuclei_rprops = skimage.measure.regionprops(im_nuclei_seg_mask)
# compute nuclei features
fdata_nuclei = htk_features.compute_nuclei_features(
im_nuclei_seg_mask,
im_nuclei_stain,
im_cytoplasm=im_cytoplasm_stain)
cfg.im_input = im_input
cfg.im_input_nmzd = im_input_nmzd
cfg.im_nuclei_stain = im_nuclei_stain
cfg.im_nuclei_seg_mask = im_nuclei_seg_mask
cfg.nuclei_rprops = nuclei_rprops
cfg.fdata_nuclei = fdata_nuclei
def test_segment_nuclei_kofahi(self):
input_image_file = utilities.externaldata('data/Easy1.png.sha512')
ref_image_file = utilities.externaldata('data/L1.png.sha512')
# read input image
im_input = skimage.io.imread(input_image_file)[:, :, :3]
# read reference image
im_reference = skimage.io.imread(ref_image_file)[:, :, :3]
# get mean and stddev of reference image in lab space
mean_ref, std_ref = htk_cvt.lab_mean_std(im_reference)
# perform color normalization
im_nmzd = htk_cnorm.reinhard(im_input, mean_ref, std_ref)
# perform color decovolution
stain_color_map = {
'hematoxylin': [0.65, 0.70, 0.29],
'eosin': [0.07, 0.99, 0.11],
'dab': [0.27, 0.57, 0.78],
'null': [0.0, 0.0, 0.0]
}
w = htk_cdeconv.rgb_separate_stains_macenko_pca(im_nmzd, im_nmzd.max())
im_stains = htk_cdeconv.color_deconvolution(im_nmzd, w).Stains
nuclei_channel = htk_cdeconv.find_stain_index(
stain_color_map['hematoxylin'], w)
im_nuclei_stain = im_stains[:, :, nuclei_channel].astype(np.float)
# segment nuclei
im_nuclei_seg_mask = htk_seg.nuclear.detect_nuclei_kofahi(
im_nuclei_stain,
im_nuclei_stain < 60,
min_radius=20,
max_radius=30,
min_nucleus_area=80,
local_max_search_radius=10)
num_nuclei = len(np.unique(im_nuclei_seg_mask)) - 1
# check if segmentation mask matches ground truth
gtruth_mask_file = os.path.join(
utilities.externaldata('data/Easy1_nuclei_seg_kofahi.npy.sha512'))
im_gtruth_mask = np.load(gtruth_mask_file)
num_nuclei_gtruth = len(np.unique(im_gtruth_mask)) - 1
assert num_nuclei == num_nuclei_gtruth
np.testing.assert_allclose(im_nuclei_seg_mask, im_gtruth_mask)
# check no nuclei case
im_nuclei_seg_mask = htk_seg.nuclear.detect_nuclei_kofahi(
255 * np.ones_like(im_nuclei_stain),
np.ones_like(im_nuclei_stain),
min_radius=20,
max_radius=30,
min_nucleus_area=80,
local_max_search_radius=10)
num_nuclei = len(np.unique(im_nuclei_seg_mask)) - 1
assert num_nuclei == 0
'*****@*****.**')
adminUser = User().findOne({'admin': True})
if Assetstore().findOne() is None:
Assetstore().createFilesystemAssetstore('Assetstore', '/assetstore')
fsAssetstore = Assetstore().findOne()
# Upload a list of items
publicFolder = namedFolder(adminUser)
dataFiles = {
'tcga1': {
'sha512':
'data/TCGA-A2-A0YE-01Z-00-DX1.8A2E3094-5755-42BC-969D-'
'7F0A2ECA0F39.svs.sha512',
},
}
for key, entry in dataFiles.items():
path = utilities.externaldata(entry['sha512'])
query = {'folderId': publicFolder['_id'], 'name': os.path.basename(path)}
if not Item().findOne(query):
with ProgressContext(False, user=adminUser) as ctx:
Assetstore().importData(fsAssetstore,
publicFolder,
'folder', {'importPath': path},
ctx,
adminUser,
leafFoldersAsItems=False)
entry['item'] = Item().findOne(query)
print(entry['item'])
# Add annotations
annotationFiles = {
'tcga1': {
'item':