def load_data(self):
"""
Create readers for the image and mask files.
"""
# Check image
if not exists(self.image_path):
raise Exception('WSI file does not exist in: %s' % str(self.image_path))
# Load image
image_reader = mri.MultiResolutionImageReader()
self.image = image_reader.open(self.image_path)
self.image_shape = self.image.getLevelDimensions(self.image_level)
self.image_level_multiplier = self.image.getLevelDimensions(0)[0] // self.image.getLevelDimensions(1)[0]
# Load mask
mask_reader = mri.MultiResolutionImageReader()
self.mask = mask_reader.open(self.mask_path)
self.mask_shape = self.mask.getLevelDimensions(self.mask_level)
self.mask_level_multiplier = self.mask.getLevelDimensions(0)[0] // self.mask.getLevelDimensions(1)[0]
# Check dimensions
if self.image_shape != self.mask_shape:
self.image_mask_ratio = int(self.image_shape[0] / self.mask_shape[0])
else:
self.image_mask_ratio = 1
def __getPatch(pathlist, start, end):
print('getting start from {} to {}'.format(start, end))
pathlist = pathlist[start:end] if end != -1 else pathlist[start:]
for path in pathlist:
print(path)
img_name = glob.glob(os.path.join(path, '*.ndpi'))
xml_name = glob.glob(os.path.join(path, '*.xml'))
mask_name = glob.glob(os.path.join(path, '*_mask.tiff'))
assert len(
img_name) == 1, 'failed to get image {} : no image or multi image'.format(img_name)
assert len(
xml_name) == 1, 'failed to get xml label {} : no xml label or multi xml label'.format(xml_name)
assert len(mask_name) == 1, 'failed to get mask {} : no mask or multi mask'.format(
mask_name)
img_name = img_name[0]
xml_name = xml_name[0]
mask_name = mask_name[0]
img_reader = mir.MultiResolutionImageReader()
mask_reader = mir.MultiResolutionImageReader()
img = img_reader.open(img_name)
mask = mask_reader.open(mask_name)
annotation_list = mir.AnnotationList()
xml_repository = mir.XmlRepository(annotation_list)
xml_repository.setSource(xml_name)
xml_repository.load()
# annotation_group = annotation_list.getGroup('Annotation Group 0')
annotations = annotation_list.getAnnotations()
del xml_repository
if not os.path.exists(os.path.join(path, 'patch')):
os.mkdir(os.path.join(path, 'patch'))
if not os.path.exists(os.path.join(path, 'patch', 'imgs')):
os.mkdir(os.path.join(path, 'patch', 'imgs'))
if not os.path.exists(os.path.join(path, 'patch', 'masks')):
os.mkdir(os.path.join(path, 'patch', 'masks'))
for idx, annotation in enumerate(annotations):
x, y, width, height = getPositionAndSize(annotation)
level_0_width, level_0_height = img.getLevelDimensions(0)
level_1_width, level_1_height = img.getLevelDimensions(1)
# x *= level_1_width/level_0_width
# y *= level_1_height/level_0_height
width *= level_1_width/level_0_width
height *= level_1_height/level_0_height
x, y, width, height = int(x), int(y), int(width), int(height)
patch_img = img.getUInt16Patch(x, y, width, height, 1)
patch_img = np.array(patch_img, dtype=np.int8)
patch_img = Image.fromarray(patch_img, mode='RGB')
patch_img.save(os.path.join(
path, 'patch', 'imgs', os.path.splitext(os.path.basename(img_name))[0]+'-{}.png'.format(idx)))
del patch_img
patch_mask = mask.getUInt16Patch(x, y, width, height, 1)
patch_mask = np.array(patch_mask, dtype=np.int8)
patch_mask = Image.fromarray(patch_mask[:, :, 0], mode='L')
patch_mask.save(os.path.join(
path, 'patch', 'masks', os.path.splitext(os.path.basename(img_name))[0]+'-{}.png'.format(idx)))
del patch_mask
def read_xml_get_tile(image_path, xml_path, output_path):
print('Start ', str(image_path), str(xml_path))
patient_id = os.path.basename(image_path)[:-4]
patient_dir = os.path.join(output_path, patient_id)
if not os.path.exists(patient_dir):
os.mkdir(patient_dir)
reader = mir.MultiResolutionImageReader()
image = reader.open(str(image_path))
# =========== read xml and write mask.tif ===========
annotation_list = mir.AnnotationList()
xml_repository = mir.XmlRepository(annotation_list)
xml_repository.setSource(str(xml_path))
xml_repository.load()
annotations = annotation_list.getAnnotations()
if len(annotations) == 0:
print('empty annotation, id ', patient_id)
return 0
temp_map = {}
for annotation in annotations:
name = annotation.getName()
color = annotation.getColor()
temp_map[name] = label_map[color]
annotations_mask = mir.AnnotationToMask()
mask_output = os.path.join(patient_dir, 'mask.tif')
annotations_mask.convert(annotation_list, mask_output,
image.getDimensions(),
image.getSpacing(), temp_map)
reader2 = mir.MultiResolutionImageReader()
mask = reader2.open(mask_output)
# =========== make patch ===========
for annotation in annotations:
name = annotation.getName()
color = annotation.getColor()
x_coord, y_coord = [], []
for coordinate in annotation.getCoordinates():
x_coord.append(coordinate.getX())
y_coord.append(coordinate.getY())
x_max, x_min = max(x_coord), min(x_coord)
y_max, y_min = max(y_coord), min(y_coord)
rect_width = x_max - x_min
rect_height = y_max - y_min
image_tile = image.getUCharPatch(int(x_min), int(y_min), int(rect_width), int(rect_height), 0)
mask_tile = mask.getUCharPatch(int(x_min), int(y_min), int(rect_width), int(rect_height), 0)
tile_path = os.path.join(patient_dir, str(color))
if not os.path.exists(tile_path):
os.mkdir(tile_path)
sio.savemat(os.path.join(tile_path, name+'.mat'), {'img': image_tile, 'mask': mask_tile})
def single_file_conversion(slide_num):
output_path = '/mnt/ai/uni_warwick/camelyon16_dataset/training/Ground_Truth_Extracted/Mask/tumor_' + str(
slide_num).zfill(3) + '.tif'
reader = mir.MultiResolutionImageReader()
mr_image = reader.open(
'/mnt/ai/uni_warwick/camelyon16_dataset/training/training/tumor/tumor_'
+ str(slide_num).zfill(3) + '.tif')
annotation_list = mir.AnnotationList()
xml_repository = mir.XmlRepository(annotation_list)
xml_repository.setSource(
'/mnt/ai/uni_warwick/camelyon16_dataset/training/training/lesion_annotations/tumor_'
+ str(slide_num).zfill(3) + '.xml')
xml_repository.load()
annotation_mask = mir.AnnotationToMask()
camelyon17_type_mask = False
label_map = {
'metastases': 1,
'normal': 2
} if camelyon17_type_mask else {
'_0': 1,
'_1': 1,
'_2': 0
}
conversion_order = ['metastases', 'normal'
] if camelyon17_type_mask else ['_0', '_1', '_2']
try:
annotation_mask.convert(annotation_list, output_path,
mr_image.getDimensions(),
mr_image.getSpacing(), label_map,
conversion_order)
except:
print("Oops!", sys.exc_info()[0], "occured.")
def computeEvaluationMask(maskDIR, resolution, level):
"""Computes the evaluation mask.
Args:
maskDIR: the directory of the ground truth mask
resolution: Pixel resolution of the image at level 0
level: The level at which the evaluation mask is made
Returns:
evaluation_mask
"""
# openslide seems to have problem reading masks
# slide = openslide.open_slide(maskDIR)
# dims = slide.level_dimensions[level]
# pixelarray = np.zeros(dims[0]*dims[1], dtype='uint')
# pixelarray = np.array(slide.read_region((0,0), level, dims))
reader = mir.MultiResolutionImageReader()
slide = reader.open(maskDIR)
dimension = slide.getLevelDimensions(level)
pixelarray = slide.getUCharPatch(0, 0, dimension[0], dimension[1], level)
pixelarray[pixelarray == 2] = 255
distance = nd.distance_transform_edt(
255 - pixelarray[:, :, 0]
) # lizx: reverse those masks, not that the 255 assumption!!
Threshold = 75 / (
resolution * pow(2, level) * 2
) # 75µm is the equivalent size of 5 tumor cells # lizx: keep the margin within 5 tumor cells positive
binary = distance < Threshold
filled_image = nd.morphology.binary_fill_holes(
binary) # lizx: this is the mask for this level
evaluation_mask = measure.label(
filled_image, connectivity=2) # lizx: get connected components
return evaluation_mask
def __init__(self,
tif_file,
xml_file,
sample_size=224,
fg_ratio=-1,
skip_empty=True):
assert (fg_ratio < 1.0)
self.sample_size = sample_size
self.fg_ratio = fg_ratio
self.skip_empty = skip_empty
self.tif_file = tif_file
self.xml_file = xml_file
# load MRImage
reader = mir.MultiResolutionImageReader()
self.mr_image = reader.open(tif_file)
self.max_size = self.mr_image.getDimensions()
# load annotation
self.anno = MaskAnnotation(xml_file)
rect = []
for attrib, contour in self.anno.contours:
x1 = int(np.min(contour[:, 0]) + 0.5)
y1 = int(np.min(contour[:, 1]) + 0.5)
x2 = int(np.max(contour[:, 0]) + 0.5)
y2 = int(np.max(contour[:, 1]) + 0.5)
print('fg rect', (x1, y1, x2, y2), attrib)
rect.append((x1, y1, x2, y2))
self.rect = rect
def find_tissue_bounding_box(image_path, level):
'''
Returns the coordinates of a bounding box around tissue. Useful if scanned images contains plenty of white space.
:param image_path: path to whole-slide image file.
:param level: desired level to read the image.
:return: tuple in the form (x1, x2, y1, y2)
'''
# Read thumbnail
image_reader = mri.MultiResolutionImageReader()
image = image_reader.open(image_path)
image_shape = image.getLevelDimensions(level)
image_tile = image.getUCharPatch(0, 0, image_shape[0], image_shape[1],
level).astype('uint8')
# Find coordinates
(y_idx, x_idx, _) = np.nonzero(image_tile != 255)
x1, x2 = np.min(x_idx), np.max(x_idx)
y1, y2 = np.min(y_idx), np.max(y_idx)
x1 = int(x1 * (2**level))
x2 = int(x2 * (2**level))
y1 = int(y1 * (2**level))
y2 = int(y2 * (2**level))
return x1, x2, y1, y2
def extract_img_crops(img_name, args):
np.random.seed(args.seed)
random.seed(args.seed)
reader = mir.MultiResolutionImageReader()
mr_image = reader.open(os.path.join(args.data, img_name))
ds = mr_image.getLevelDownsample(args.level)
img_size = (np.array(mr_image.getDimensions()) / ds).astype(int)
if img_size[0] < args.size or img_size[1] < args.size:
return
img = mr_image.getUCharPatch(0, 0, int(img_size[0]), int(img_size[1]), args.level)
img = img.squeeze()
patient_num = int(img_name.split('_')[1])
center = patient_num // 20
if center == 3:
args.save_dir = os.path.join(args.save_dir, 'valid')
if center == 4:
args.save_dir = os.path.join(args.save_dir, 'test')
orig_img_path = os.path.join(args.base_path, 'center_{}'.format(center), img_name)
orig_img = reader.open(orig_img_path)
orig_img_data = orig_img.getUCharPatch(0, 0, int(img_size[0]), int(img_size[1]), args.level)
if img.sum() < args.low_roir * args.size * args.size:
return
labels, num = measure.label(img, return_num=True)
for i_label in range(1, num + 1):
valid_patch_counter = 0
bin_mask = (labels == i_label)
top, bottom, left, right = get_bb_box(bin_mask)
if bin_mask.sum() > args.size * args.size * args.high_roir:
continue
if None in (top, bottom, left, right):
continue
start_x = max(0, bottom - args.size)
end_x = min(img.shape[0] - args.size, top)
start_y = max(0, right - args.size)
end_y = min(img.shape[1] - args.size, left)
if end_x < start_x or end_y < start_y:
continue
if img[start_x:end_x, start_y:end_y].sum() < args.low_roir * args.size * args.size:
continue
for _ in range(10000):
x = random.randint(start_x, end_x)
y = random.randint(start_y, end_y)
patch = img[x:x+args.size, y:y+args.size]
if (patch.sum() / (args.size * args.size)) > args.low_roir and (patch.sum() / (args.size * args.size)) < args.high_roir and patch.sum() >= bin_mask.sum():
orig_img_patch = orig_img_data[x:x+args.size, y:y+args.size]
if not (orig_img_patch.sum(axis=-1) == 0).any():
save_path = os.path.join(args.save_dir, '{}_{}_{}_{}_img'.format(img_name.split('.')[0], args.level, i_label,
valid_patch_counter))
np.save(save_path, orig_img_patch)
save_patch_path = os.path.join(args.save_dir,
'{}_{}_{}_{}_seg'.format(img_name.split('.')[0], args.level, i_label,
valid_patch_counter))
np.save(save_patch_path, patch)
valid_patch_counter += 1
if valid_patch_counter >= args.k:
break
def __init__(self):
reader = mir.MultiResolutionImageReader()
self.slide = reader.open(
"/home/suidong/Documents/camelyon17_data_backup/slide/patient_010_node_4.tif"
)
self.mask = reader.open(
"/home/suidong/Documents/camelyon17_data_backup/mask/patient_010_node_4.tif"
)
def __init__(self):
self.prepare_model()
self._reader = mir.MultiResolutionImageReader()
self.input()
if self._check:
target_patches = self.split_whole_level(TOP_LEVEL)
self.detection(target_patches, TOP_LEVEL - 1)
print(self._detection_result)
def main(args):
pid = os.getpid()
print('Running with PID', pid)
logger = get_logger('XML-to-MASK-{}'.format(pid))
output_dir_path = args.output_parent_dir / args.output_folder_name
logger.info('Creating output directory at %s', str(output_dir_path))
output_dir_path.mkdir(parents=True, exist_ok=True)
logger.info('Reading WSI data objects.')
start = args.data_offset
wsi_data = parse_dataset(args.data_list_file)[start:]
count = args.count
if count > len(wsi_data):
raise ValueError('Offset and count out of bounds.')
wsi_data = wsi_data[:count]
while wsi_data:
data = wsi_data.pop(0)
logger.info('Creating mask for %s', data.name)
reader = mir.MultiResolutionImageReader()
if not data.tif_path.is_file():
logger.warning('TIF File not found. Ignoring %s', data.name)
continue
mr_image = reader.open(str(data.tif_path))
annotation_list = mir.AnnotationList()
xml_repository = mir.XmlRepository(annotation_list)
if data.label_xml_path is None:
logger.info('No annotation exists. Ignoring %s', data.name)
continue
elif not data.label_xml_path.is_file():
logger.warning('Label File not found. Ignoring %s', data.name)
continue
xml_repository.setSource(str(data.label_xml_path))
xml_repository.load()
annotation_mask = mir.AnnotationToMask()
output_path = output_dir_path / (data.name + '_Mask.tif')
annotation_mask.convert(
annotation_list,
str(output_path),
mr_image.getDimensions(),
mr_image.getSpacing(),
_LABEL_MAP,
)
logger.info('Mask saved for %s at %s', data.name, str(output_path))
del data
def __init__(self, tif_file, xml_file=None, skip_empty=True):
self.tif_file = tif_file
self.skip_empty = skip_empty
reader = mir.MultiResolutionImageReader()
self.mr_image = reader.open(tif_file)
self.max_size = self.mr_image.getDimensions()
self.image_id = int(self.tif_file.split("/")[-1].split("_")[1]) / 20
self.ds, self.thr_image = self.binary_img()
if xml_file is not None:
self.anno = MaskAnnotation(xml_file)
def readImage(nodeInd, patientDir, level, path_size):
Dir = '/media/groot/Seagate Backup Plus Drive/dataset/camelyon17'
reader = mir.MultiResolutionImageReader()
name = patientDir + '_node_' + str(nodeInd) + '.tif'
mr_image = reader.open(Dir + '/' + patientDir + '/' + name)
ds = mr_image.getLevelDownsample(level)
image_patch = mr_image.getUCharPatch(int(568 * ds), int(732 * ds),
path_size[1], path_size[0], level)
return image_patch
def get_pixel_lenghts(image_path):
"""
Reads a complete WSI and returns a the width and height
returns: image numpy array size
x_dim : width
y_dim : height
"""
# Open wsi image
mr_image = mri.MultiResolutionImageReader().open(image_path)
# Get Dimensions
x_dim, y_dim = mr_image.getDimensions()
return x_dim, y_dim
def main(args):
logger = get_logger('XML-to-MASK')
output_dir_path = args.output_path / args.output_dir_name
logger.info('Creating output directory at %s', str(output_dir_path))
output_dir_path.mkdir(parents=True, exist_ok=True)
logger.info('Reading WSI data objects.')
wsi_data = read_wsi_list_file(args.data_list_file)
while wsi_data:
data = wsi_data.pop(0)
logger.info('Creating mask for %s', data.name)
reader = mir.MultiResolutionImageReader()
if not data.tif_path.is_file():
logger.warning('TIF File not found. Ignoring %s', data.name)
continue
mr_image = reader.open(str(data.tif_path))
annotation_list = mir.AnnotationList()
xml_repository = mir.XmlRepository(annotation_list)
if data.label_path is None:
logger.info('No annotation exists. Ignoring %s', data.name)
continue
elif not data.label_path.is_file():
logger.warning('Label File not found. Ignoring %s', data.name)
continue
xml_repository.setSource(str(data.label_path))
xml_repository.load()
annotation_mask = mir.AnnotationToMask()
output_path = output_dir_path / (data.name + '_Mask.tif')
annotation_mask.convert(
annotation_list,
str(output_path),
mr_image.getDimensions(),
mr_image.getSpacing(),
_LABEL_MAP,
)
logger.info('Mask saved for %s at %s', data.name, str(output_path))
# slide = openslide.OpenSlide(str(output))
# img = slide.read_region(
# location=(0, 0),
# level=args.level,
# size=slide.level_dimensions[args.level],
# ).convert('L')
del data
def gene_mask(each_fp):
reader = mir.MultiResolutionImageReader()
img_id = int(path.basename(each_fp).partition('.')[0])
mr_image = reader.open(each_fp)
annotation_list = mir.AnnotationList()
xml_repository = mir.XmlRepository(annotation_list)
xml_path = utils.id_to_xml(img_id)
xml_repository.setSource(xml_path)
xml_repository.load()
annotation_mask = mir.AnnotationToMask()
output_path = path.join(cfg.mask_path, f'{img_id}_mask.tif')
annotation_mask.convert(annotation_list, output_path,
mr_image.getDimensions(), mr_image.getSpacing())
def next(self):
count_try = 0
while True:
count_try += 1
rect = self.sample_rect()
result = self.sub_region(rect)
if count_try % 20 == 0:
print('exceeding max try and reload')
print('tif_file: %s' % self.tif_file)
print('xml_file: %s' % self.xml_file)
reader = mir.MultiResolutionImageReader()
self.mr_image = reader.open(self.tif_file)
if result is None:
continue
return result
def sub_region(self, rect):
x1,y1,x2,y2 = rect
if self.skip_empty and self.is_background(rect):
return None
image_patch = self.mr_image.getUCharPatch(x1, y1, x2-x1+1, y2-y1+1, 0)
if self.is_error(image_patch):
reader = mir.MultiResolutionImageReader()
self.mr_image = reader.open(self.tif_file)
if self.skip_empty and self.is_empty(image_patch):
return None
if hasattr(self, 'anno'):
mask = self.anno.get_mask((x1,y1,x2,y2), 255)
return image_patch, mask
return image_patch
def read_threshold_mask(self, mask_fn):
mask = mir.MultiResolutionImageReader().open(
mask_fn) # type: mir.MultiResolutionImage
read_level = 0
whole_mask = mask.getLevelDimensions(read_level)
wsi_high_level_mask = mask.getUCharPatch(0, 0, whole_mask[0],
whole_mask[1], read_level)
blur = cv2.blur(wsi_high_level_mask * 255, (300, 300))
wsi_high_level_mask = (blur > 128)
kernel = np.ones((128, 128), np.uint8)
self.mask = cv2.dilate(wsi_high_level_mask.astype(np.uint8),
kernel,
iterations=1)
self.threshold_downsample = int(
round(mask.getSpacing()[0] * mask.getLevelDownsample(read_level) /
self.reader.getSpacing()[0]))
def readImage(image_path, level=4):
"""
Reads a complete WSI and returns a numpy array
In a WSI, at each higher resolution level the size is decreased by half.
For this reason the dimensions should be decreased by 2^level for a given image resolution level
returns: image numpy array
"""
# Open wsi image
mr_image = mri.MultiResolutionImageReader().open(image_path)
# Get Dimensions
x_dim, y_dim = mr_image.getDimensions()
# Scale image to be display
img = mr_image.getUCharPatch(startX=0,
startY=0,
width=x_dim // 2**level,
height=y_dim // 2**level,
level=level)
return img, x_dim, y_dim
def OpenSlide(self, slide_path):
self.slide_path = slide_path
self.slide = mir.MultiResolutionImageReader().open(slide_path)
if self.slide is None:
print('* Error while opening ' + slide_path +
' [mir_based_slide @ slidelib]')
raise (ValueError)
else:
self.slide.setCacheSize(0) # 试试加上这句能否缓解内存的问题
self.level_count = self.slide.getNumberOfLevels()
self.level_downsamples = [
self.slide.getLevelDownsample(level)
for level in range(self.level_count)
]
self.level_dimensions = [
self.slide.getLevelDimensions(level)
for level in range(self.level_count)
]
self.dimensions = self.level_dimensions[0]
self.spacing = self.slide.getSpacing()
return self
def main(args):
output_folder = args.output_dir / args.folder_name
output_folder.mkdir(parents=True, exist_ok=True)
names = []
with open(str(args.names)) as names_file:
for name in names_file.readlines():
names.append(name.strip())
for xml_path in args.xml_dir.glob('*.xml'):
stem = xml_path.stem
if stem not in names:
continue
print(f'>> Creating mask for {stem}...')
sys.stdout.flush()
reader = mir.MultiResolutionImageReader()
wsi_tif_path = args.wsi_dir / f'{stem}.tif'
mr_image = reader.open(str(wsi_tif_path))
annotation_list = mir.AnnotationList()
xml_repository = mir.XmlRepository(annotation_list)
xml_repository.setSource(str(xml_path))
xml_repository.load()
annotation_mask = mir.AnnotationToMask()
output_path = output_folder / f'{stem}_Mask.tif'
annotation_mask.convert(
annotation_list,
str(output_path),
mr_image.getDimensions(),
mr_image.getSpacing(),
_LABEL_MAP,
)
print(f' Mask saved for {stem} at {output_path}')
sys.stdout.flush()
def extract_img_crops(img_name, args):
np.random.seed(args.seed)
random.seed(args.seed)
reader = mir.MultiResolutionImageReader()
patient_num = int(img_name.split('_')[1])
center = patient_num // 20
if center == 3:
args.save_dir = os.path.join(args.save_dir, 'valid')
if center == 4:
args.save_dir = os.path.join(args.save_dir, 'test')
orig_img_path = os.path.join(args.base_path, 'center_{}'.format(center),
img_name)
orig_img = reader.open(orig_img_path)
ds = orig_img.getLevelDownsample(args.level)
img_size = (np.array(orig_img.getDimensions()) / ds).astype(int)
if img_size[0] < args.size or img_size[1] < args.size:
return
orig_img_data = orig_img.getUCharPatch(0, 0, int(img_size[0]),
int(img_size[1]), args.level)
valid_patch_counter = 0
for _ in range(10000):
x_i = random.randint(0, img_size[1] - args.size)
y_i = random.randint(0, img_size[0] - args.size)
orig_img_patch = orig_img_data[x_i:x_i + args.size,
y_i:y_i + args.size]
if (orig_img_patch.sum(axis=-1)
== 0).any() or orig_img_patch.mean(axis=(0, 1))[1] > 200:
continue
save_path = os.path.join(
args.save_dir, '{}_{}_{}'.format(
img_name.split('.')[0], args.level, valid_patch_counter))
np.save(save_path, orig_img_patch)
valid_patch_counter += 1
if valid_patch_counter >= args.k:
break
def getImage( i , f_name , level=8 , startX=0 , startY=0 , dimX=None , dimY = None , justDs=False , warpDim=None):
# none checks for needing to initialize
if getImage.reader[i] == None:
getImage.reader[i] = mir.MultiResolutionImageReader()
if getImage.mr_image[i] == None or f_name != getImage.name[i]:
if getImage.mr_image[i] != None:
getImage.mr_image[i].close()
getImage.mr_image[i] = getImage.reader[i].open(f_name )
getImage.name[i] = f_name
assert getImage.mr_image[i] != None , ( 'failed to open' , f_name )
ds = getImage.mr_image[i].getLevelDownsample( level )
if justDs:
return ds
if dimX == None:
dimX,_ = getImage.mr_image[i].getLevelDimensions( level )
if dimY == None:
_,dimY = getImage.mr_image[i].getLevelDimensions( level )
arr = None
# if we need to warp the image do that else do not
if warpDim != None:
arr = np.array( Image.fromarray(getImage.mr_image[i].getUCharPatch(int(startX*ds) , int(startY*ds) , dimX , dimY , level ) ).resize(warpDim) )
else:
arr = np.array( Image.fromarray(getImage.mr_image[i].getUCharPatch( int( startX*ds) , int(startY*ds) , dimX , dimY , level ) ) )
return ds , arr
box[3] = ii
for jj in range(j, image.shape[1]):
if image[i][jj] > 0:
box[2] = jj
image[i:ii, j:jj] = 255
boxes.append(box)
return boxes
test_net = "/opt/intern/users/yuewang/ScanNet-FCN/model/test_py.prototxt"
#weights = "/opt/intern/users/yuewang/ScanNet-FCN/output/model/ScanNet_iter_40000.caffemodel"
weights = "/opt/intern/users/yuewang/ScanNet-FCN/output/model/ScanNet_random_list_iter_130000.caffemodel"
gpu_id = 6
net = CaffeNet(test_net, weights, gpu_id)
path = "/opt/intern/users/yuewang/dataset/Camelyon17/"
r = mir.MultiResolutionImageReader()
#files = os.listdir(path+"gt")
files = [
"patient_017_node_4", "patient_039_node_1", "patient_046_node_4",
"patient_064_node_0", "patient_089_node_3"
]
tif_files = [os.path.join(path, "images", x + ".tif") for x in files]
xml_files = [os.path.join(path, "label", x + ".xml") for x in files]
sampler = CamelyonDataset(tif_files, xml_files, 1204, 0.5)
while True:
image_patch, gt = sampler.next()
output = net.get_feature(image_patch)
plt.subplot(1, 3, 1)
plt.imshow(output, cmap="gray")
plt.subplot(1, 3, 2)
plt.imshow(gt[106:1204 - 106, 106:1204 - 106], cmap="gray")
# -*- coding: utf-8 -*-
"""
Created on Wed Mar 25 18:20:37 2015
@author: Geert
"""
import sys
sys.path.append(r"D:\Code\sources\diag\build (VC12)\bin\Release")
import multiresolutionimageinterface
import numpy as np
level = 6
r = multiresolutionimageinterface.MultiResolutionImageReader()
w = multiresolutionimageinterface.MultiResolutionImageWriter()
i = r.open(r"D:\Temp\T06-12822-II1-10.mrxs")
w.openFile(r"D:\Temp\test_python.tif")
w.setTileSize(512)
w.setCompression(multiresolutionimageinterface.LZW)
w.setDataType(multiresolutionimageinterface.UChar)
w.setColorType(multiresolutionimageinterface.RGB)
dims = i.getLevelDimensions(level)
a = np.zeros(512*512*4, dtype='ubyte')
w.writeImageInformation(dims[0], dims[1])
for y in range(0, dims[1], 512):
print y
for x in range(0, dims[0], 512):
i.getUCharPatch(int(x*i.getLevelDownsample(level)),
int(y*i.getLevelDownsample(level)), 512, 512, level, a)
w.writeBaseImagePart(a.reshape(512, 512, 4)[:, :, 2::-1].flatten())
w.finishImage()
def open_slide(self, file):
self.reader = mir.MultiResolutionImageReader().open(str(file))
class mask_gen_asap(object):
"""
The class is used to generate training mask files from xml files.
:param slide_path: the folder storing slides
:type slide_path: str
:param anno_path: the folder storing annotation files (xml)
:type anno_path: str
:param mask_path: the destination folder for the mask files
:type mask_path: str
:ivar slide_paths: all the paths of slides
:vartype slide_paths: list
:ivar anno_paths: all the paths of xml files
:vartype anno_paths: list
"""
reader = mir.MultiResolutionImageReader()
def __init__(self, slide_path, anno_path, mask_path):
"""
To initialize parameters
:param slide_path: the folder storing slides
:type slide_path: str
:param anno_path: the folder storing annotation files (xml)
:type anno_path: str
:param mask_path: the destination folder for the mask files
:type mask_path: str
"""
self.slide_path = slide_path
self.anno_path = anno_path
self.mask_path = mask_path
self.slide_paths = glob.glob(osp.join(self.slide_path, '*.tif'))
self.slide_paths.sort()
self.anno_paths = glob.glob(osp.join(anno_path, '*.xml'))
self.anno_paths.sort()
def mask_gen(self, slide_file, xml_file):
"""
To generate mask file for one slide, and save the mask file.
:param slide_file: the path of a WSI image
:type slide_file: str
:param xml_file: the path of a xml file for the annotation of WSI image
:type xml_file: str
:returns: the path of the mask file
:rtype: str
"""
mr_image = self.reader.open(slide_file)
annotation_list = mir.AnnotationList()
xml_repository = mir.XmlRepository(annotation_list)
xml_repository.setSource(xml_file)
xml_repository.load()
annotation_mask = mir.AnnotationToMask()
camelyon17_type_mask = False
# Here 255 is used to generate mask file so that the tumor region is obvious.
# if use '1' here, a binary maks file will be generated.
label_map = {
'metastases': 1,
'normal': 2
} if camelyon17_type_mask else {
'_0': 255,
'_1': 255,
'_2': 0
}
conversion_order = ['metastases', 'normal'
] if camelyon17_type_mask else ['_0', '_1', '_2']
output_path = osp.join(
mask_path,
osp.basename(slide_file).replace('.tif', '_mask.tif'))
annotation_mask.convert(annotation_list, output_path,
mr_image.getDimensions(),
mr_image.getSpacing(), label_map,
conversion_order)
return output_path
def batch_gen(self):
"""
To generate all the mask files for the slides in a folder
:returns: None
"""
for slide_file in self.slide_paths:
xml_file = [
x for x in self.anno_paths if re.search(
osp.basename(slide_file).replace('.tif', '.xml'), x)
][0]
self.mask_gen(slide_file, xml_file)
def process_files(files_to_process,
output_base_path,
step_size,
spacing_json_filepath=None):
"""
Processes a list of tif files with lymphocyte and tumor bud detections (from JMB)
"""
bud_indx = 3
lymp_indx = 9
# if a spacing json file is provided load it, else create a new dict
if spacing_json_filepath:
all_spacing = json.load(spacing_json_filepath)
else:
all_spacing = {}
for file in files_to_process:
file_name = os.path.splitext(
os.path.basename(file))[0].split("_combined")[0]
output_file_lymp = os.path.join(
output_base_path, file_name + "_coordinates_lymphocytes.txt")
output_file_bud = os.path.join(
output_base_path, file_name + "_coordinates_tumorbuds.txt")
print("Processing: {}".format(file_name))
level = 1
# save the coordinates
all_bud_coords = np.empty((0, 2), int)
all_lymph_coords = np.empty((0, 2), int)
# check if the files are already present
if not os.path.isfile(output_file_lymp) and not os.path.isfile(
output_file_bud):
img_obj = mir.MultiResolutionImageReader().open(file)
assert file_name not in all_spacing
all_spacing[file_name] = img_obj.getSpacing()
dim_x, dim_y = img_obj.getLevelDimensions(level)
# sliding window over the whole image
for y_indx in range(0, dim_y, step_size):
for x_indx in range(0, dim_x, step_size):
coords_level_0, ratio = convert_xy(img_obj, level, [
x_indx, y_indx, x_indx + step_size, y_indx + step_size
], 0)
# get the patch
img_patch = get_tile(img_obj, coords_level_0, level)
img_patch = img_patch.squeeze()
# get the patch
img_patch_0 = get_tile(img_obj, coords_level_0, 0)
img_patch_0 = img_patch.squeeze()
lymph_patch = np.zeros(img_patch.shape, np.uint8)
lymph_patch[img_patch == lymp_indx] = 1
bud_patch = np.zeros(img_patch.shape, np.uint8)
bud_patch[img_patch == bud_indx] = 1
# get the coordinates of the center of each annotation
# Coordinates are additionally multiplied by 2 because TIF annotations level 0 is actually level 1 on the WSI
bud_coords = get_obj_coords(bud_patch, x_indx, y_indx,
ratio * 2)
if bud_coords is not None:
bud_coords = bud_coords
all_bud_coords = np.append(all_bud_coords,
bud_coords,
axis=0)
lymp_coords = get_obj_coords(lymph_patch, x_indx, y_indx,
ratio * 2)
if lymp_coords is not None:
lymp_coords = lymp_coords
all_lymph_coords = np.append(all_lymph_coords,
lymp_coords,
axis=0)
np.savetxt(output_file_lymp, all_lymph_coords, fmt='%.3f')
np.savetxt(output_file_bud, all_bud_coords, fmt='%.3f')
else:
print('The coordinates files {} and {} already exists'.format(
output_file_lymp, output_file_bud))
# Save spacing as json to later calculate the distance
with open(os.path.join(output_base_path, 'spacing_cd8_files.json'),
'w') as fp:
json.dump(all_spacing, fp)
def main():
parser = argparse.ArgumentParser(description="CAMELYON17 dataset Preprocess")
parser.add_argument('--tif', '-t', default="center_0/patient_013/patient_013_node_3.tif",
help='tif file path')
parser.add_argument('--annotations', '-a', default="patient_013_node_3",
help='annotations file path')
parser.add_argument('--level', '-l', type=int, default=2,
help='down-sampling level')
parser.add_argument('--resize', '-r', type=int, default=32,
help='resize times')
parser.add_argument('--patch', '-p', type=int, default=256,
help='patch size')
parser.add_argument('--out', '-o', default="other/",
help='output directory path')
args = parser.parse_args()
# Load TIFF
reader = mir.MultiResolutionImageReader()
tif_dir = "../../../mnt/nas/CAMELYON/CAMELYON17/original/org/training/"
tif_path = tif_dir + args.tif
mr_image = reader.open(tif_path)
w_max, h_max = mr_image.getDimensions()
print("Load {}".format(args.tif) + " (" + str(w_max) + "×" + str(h_max) + " pixels)")
#print(mr_image)
# Annotations
annotation_list = mir.AnnotationList()
xml_repository = mir.XmlRepository(annotation_list)
xml_dir = "../../../mnt/nas/CAMELYON/CAMELYON17/original/org/training/lesion_annotations/"
xml_path = xml_dir + args.annotations + '.xml'
xml_repository.setSource(xml_path)
xml_repository.load()
annotation_mask = mir.AnnotationToMask()
camelyon17_type_mask = True
label_map = {'metastases': 1, 'normal': 2} if camelyon17_type_mask else {'_0': 1, '_1': 1, '_2': 0}
conversion_order = ['metastases', 'normal'] if camelyon17_type_mask else ['_0', '_1', '_2']
"""
annotation_mask.convert(annotation_list, "annotations/" + args.annotations + '.tif',
mr_image.getDimensions(), mr_image.getSpacing(),
label_map, conversion_order)
print("Got annotation")
sys.exit()
"""
# Extract patch
mr_imagea = reader.open("annotations/" + args.annotations + ".tif")
ds = mr_image.getLevelDownsample(args.level)
w_lmax, h_lmax = int(w_max/ds), int(h_max/ds)
"""
imagea = mr_imagea.getUCharPatch(int(0 * ds), int(0 * ds), w_lmax, h_lmax, args.level)
#print(imagea.shape)
imga = cv2.resize(imagea, (int(w_lmax * 1/args.resize), int(h_lmax * 1/args.resize)))
#print(imga.shape)
c = 0
for i in range(imga.shape[0]):
for j in range(imga.shape[1]):
if imga[i][j] != 0:
c = c + 1
else:
imga[i][j] = 255
print(c)
cv2.imwrite(args.out + "c.jpg", imga)
"""
image = mr_image.getUCharPatch(int(0 * ds), int(0 * ds), w_lmax, h_lmax, args.level)
image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
img = cv2.resize(image, (int(w_lmax * 1/args.resize), int(h_lmax * 1/args.resize)))
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
c = 0
for i in range(gray.shape[0]):
for j in range(gray.shape[1]):
c = c + 1
print(str(c) + "/" + str(gray.shape[0] * gray.shape[1]))
if gray[i][j] < 5:
gray[i][j] = 255
ret, thresh = cv2.threshold(gray, 204, 255, cv2.THRESH_BINARY)
cv2.imwrite(args.out + "a.jpg", img)
#cv2.imwrite(args.out + "b.jpg", thresh)
sys.exit()
c = 0
d = 0
e = 0
f = 0
for row in range(0, int(h_lmax/args.resize - args.patch/args.resize),
int(args.patch/args.resize)):
for col in range(0, int(w_lmax/args.resize - args.patch/args.resize),
int(args.patch/args.resize)):
thresh_patch = thresh[row : row + int(args.patch/args.resize),
col : col + int(args.patch/args.resize)]
#print(thresh_patch.shape)
#print(row, col)
c = c + 1
flag = False
for i in range(int(args.patch/args.resize)):
for j in range(int(args.patch/args.resize)):
if thresh_patch[i][j] == 0:
flag = True
row2 = row * args.resize
col2 = col * args.resize
image_patch = image[row2 : row2 + args.patch,
col2 : col2 + args.patch]
#print(image_patch.shape)
tumor = 0
for i2 in range(int(args.patch/args.resize)):
for j2 in range(int(args.patch/args.resize)):
#print(row + i2, j + j2)
if imga[row + i2][col + j2] != 0:
tumor = tumor + 1
print(tumor)
#if tumor/(args.patch/args.resize) >= 0.75:
if tumor/(args.patch/args.resize) != 0:
#cv2.imwrite(args.out + "positive/" + args.annotations
# + "_" + str(row2) + "_" + str(col2) + ".jpg", image_patch)
e = e + 1
#elif tumor/(args.patch/args.resize) != 0:
#cv2.imwrite(args.out + "positive2/" + args.annotations
#+ "_" + str(row2) + "_" + str(col2) + ".jpg", image_patch)
#f = f + 1
else:
#cv2.imwrite(args.out + "negative/" + args.annotations
# + "_" + str(row2) + "_" + str(col2) + ".jpg", image_patch)
d = d + 1
break
if flag:
break
#print(str(e) + "/" + str(f) + "/" + str(d) + "/" + str(c))
print(str(e) + "/" + str(d) + "/" + str(c))
return 0
