def read_lymphoma(filenames,
train_or_test='train',
image_size=448,
scale_size=224,
scale=2,
multi_crop=0,
crop_per_case=None,
normalize=None,
original_dir=None):
num_show = 5
ret = []
class_0 = 0
class_1 = 0
total_crops = 0
unique_samples = 0
for fname in filenames:
with open(fname, 'rb') as fo:
try:
dic = pickle.load(fo)
except UnicodeDecodeError: #python 3.x
fo.seek(0)
dic = pickle.load(fo, encoding='latin1')
data = dic['data']
label = dic['labels']
fo.close()
crop_count = 0
unique_samples += len(data)
if crop_per_case is not None:
crop_per_case = min(len(data), crop_per_case)
else:
crop_per_case = 1
if bool(multi_crop) == False:
multi_crop = 0
print(">>>>>> will take ", str(crop_per_case),
" images from each case for ", str(multi_crop), " crops.")
print(">>>>>> Totaling crops: ", str(crop_per_case * multi_crop))
for k in range(len(data)): #part):
# resize to divisable format for convolutions
img = data[k].astype("uint8")
cropsize = (np.max(np.array(img.shape)),
np.max(
np.array(img.shape)[np.array(img.shape) < np.max(
np.array(img.shape))]), 3)
if scale_size is None:
scaleSize = image_size, image_size #224*scale,224*scale
imSize = image_size #image_size*scale #224
else:
scaleSize = image_size, image_size #scale_size,scale_size
imSize = image_size
#randPos = rnd.choice([0, 50, 100, 200, 300])
#img = data[k][:, randPos:(randPos+imSize), randPos:(randPos+imSize)] #:32, :32] #size currently (927,1276,3)
if original_dir:
if not os.path.exists(original_dir):
os.mkdirs(original_dir)
save_original(data[k][:, :, :], save_dir=original_dir, name=k)
##quality_cropper = quality_random_crop(data[k][:,:, :],imSize)
##img = quality_cropper.random_crop_select()
# make rgb feasible
img = np.transpose(img, [1, 2, 0])
img_og = copy.deepcopy(img)
aspect_ratio = min(img.shape[1] / float(img.shape[0]),
img.shape[0] / float(img.shape[1]))
acc_scaleSize = (
int(scaleSize[0] * aspect_ratio),
scaleSize[1]) if img.shape[0] < img.shape[1] else (
scaleSize[0], int(aspect_ratio * scaleSize[1]))
W = int(imSize * img.shape[1] / float(img.shape[0]))
H = int(imSize * img.shape[0] / float(img.shape[1]))
#print(" True size after scaling to preserve aspect ratio: ", scaleSize)
#print("larger crop size: ", W," ",H)
multi_crop_ = 1
if crop_count < crop_per_case:
multi_crop_ = multi_crop + 1
crop_count += 1
for tile in range(multi_crop_):
if multi_crop_ != 1:
total_crops += 1
start_w = [100, 300, 500, 700][tile] if multi_crop_ <= 4 else [
100, 200, 300, 400, 500, 600
][tile]
start_h = [400, 400, 400, 400][tile] if multi_crop_ <= 4 else [
400, 400, 400, 400, 400, 400
][tile]
copy_func = copy.deepcopy if multi_crop_ != 1 else lambda x: x
img_crop = copy.deepcopy(img_og)
img_crop = img_crop[start_w:(start_w + 2 * image_size),
start_h:(start_h + 2 * image_size), :]
#img_stack = np.expand_dims( img_crop, axis=0)
img_crop = Image.fromarray(img_crop, 'RGB')
img_crop = img_crop.resize((image_size, image_size),
Image.ANTIALIAS)
img_crop = np.array(img_crop)
img_crop = img_crop.reshape(
(image_size, image_size, 3)) #.transpose(0,3,1,2)
# to place as nd array for tensorflow and augmentors
if label[k] == 0:
#label[k] = 0.1
class_0 += 1
elif label[k] == 1:
#label[k] = 0.9
class_1 += 1
if label[k] != 0 and label[k] != 1:
for i in range(20):
print(" >>>>>>>>>>>>>> LABELING INCORRECT> VALUE: ",
label[k])
if normalize:
img_crop = normalize_staining().apply_image(img_crop)
ret.append([img_crop.astype("uint8"), label[k]])
#img = copy.deepcopy(img_og)
print(">>>> Total crops observed: ", total_crops)
return (ret, class_0, class_1, unique_samples)
def read_write_idx_lymphoma(filenames,
train_or_test='train',
image_size=448,
scale_size=224,
scale=2,
multi_crop=0,
crop_per_case=None,
normalize=None,
original_dir=None,
write_crop=True,
idx_filepath='',
mode=None):
num_show = 5
ret = []
class_0 = 0
class_1 = 0
total_crops = 0
unique_samples = 0
idx_samples = []
idx_labels = []
for fname in filenames:
if mode == 'w':
with open(fname, 'rb') as fo:
try:
dic = pickle.load(fo)
except UnicodeDecodeError: # python 3.x
fo.seek(0)
dic = pickle.load(fo, encoding='latin1')
data = dic['data']
label = dic['labels']
fo.close()
if mode == 'r':
idx_sample = VisusDataflow.ReadData(data=fname, load=True)
idx_samples.append(idx_sample.data)
idx_labels.append(int(str(1) in fname.split('~')[-1]))
#continue collecting from folder
if fname != filenames[-1]:
continue
data = idx_samples
label = idx_labels
crop_count = 0
unique_samples += len(data)
if crop_per_case is not None:
crop_per_case = min(len(data), crop_per_case)
else:
crop_per_case = 1
if bool(multi_crop) == False:
multi_crop = 0
print(">>>>>> will take ", str(crop_per_case),
" images from each case for ", str(multi_crop), " crops.")
print(">>>>>> Totaling crops: ", str(crop_per_case * multi_crop))
# class for writing images into IDX format
# with z space filling order for parameter based
# resolution on read
if mode == 'w':
visusWriter = VisusDataflow.WriteZOrder()
if not os.path.exists(idx_filepath):
os.mkdir(idx_filepath)
for k in range(len(data)): # part):
# resize to divisable format for convolutions
img = data[k].astype("uint8")
# make rgb feasible
if img.shape[0] == 3:
img = np.transpose(img, [1, 2, 0])
#if max(img.shape) != img.shape[0]:
# img = np.transpose(img, [2, 1, 0])
img_og = copy.deepcopy(img)
if mode == 'w':
s = ""
idx_filename = fname.split('/')[-1].split('.')[0] + '_' + str(
k) + '~' + str(label[k]) + '.idx'
#print(os.path.join(idx_filepath, idx_fname))
visusWriter.convert_image(image=copy.deepcopy(img_og),
idx_filename=os.path.join(
idx_filepath, idx_filename))
if mode == 'r':
cropsize = (np.max(np.array(img.shape)),
np.max(
np.array(img.shape)[np.array(
img.shape) < np.max(np.array(img.shape))]),
3)
if image_size is None:
print(" >>>>> No scaling will be performed")
#image_size = cropsize[1]#img.shape[0]
#if scale_size is None:
# scaleSize = img.shape[0], img.shape[1]#image_size, image_size # 224*scale,224*scale
# imSize = None#image_size # image_size*scale #224
#else:
# scaleSize = image_size, image_size # scale_size,scale_size
# imSize = image_size
# randPos = rnd.choice([0, 50, 100, 200, 300])
# img = data[k][:, randPos:(randPos+imSize), randPos:(randPos+imSize)] #:32, :32] #size currently (927,1276,3)
if original_dir:
if not os.path.exists(original_dir):
os.mkdirs(original_dir)
save_original(data[k][:, :, :],
save_dir=original_dir,
name=str(k))
##quality_cropper = quality_random_crop(data[k][:,:, :],imSize)
##img = quality_cropper.random_crop_select()
aspect_ratio = min(img.shape[1] / float(img.shape[0]),
img.shape[0] / float(img.shape[1]))
#acc_scaleSize = (int(scaleSize[0] * aspect_ratio), scaleSize[1]) if img.shape[0] < img.shape[1] else (
#scaleSize[0], int(aspect_ratio * scaleSize[1]))
#W = int(imSize * img.shape[1] / float(img.shape[0]))
#H = int(imSize * img.shape[0] / float(img.shape[1]))
# print(" True size after scaling to preserve aspect ratio: ", scaleSize)
# print("larger crop size: ", W," ",H)
multi_crop_ = 1
if crop_count < crop_per_case:
multi_crop_ = multi_crop + 1
crop_count += 1
for tile in range(multi_crop_):
if multi_crop_ != 1:
total_crops += 1
start_w = [100, 300, 500, 700
][tile] if multi_crop_ <= 4 else [
100, 200, 300, 400, 500, 600
][tile]
start_h = [400, 400, 400, 400
][tile] if multi_crop_ <= 4 else [
400, 400, 400, 400, 400, 400
][tile]
copy_func = copy.deepcopy if multi_crop_ != 1 else lambda x: x
img_crop = copy.deepcopy(img_og)
if image_size is not None:
img_crop = img_crop[start_w:(start_w + 2 * image_size),
start_h:(start_h +
2 * image_size), :]
# write crop in IDX file format in
# Z space filling order for parameter based
# resolutuion on read
if False and write_crop and mode == 'w':
#im_crop = np.transpose(img_og, [2, 0, 1])
s = ""
idx_fname = fname.split('/')[-1].split(
'.')[0] + '_' + str(k) + '~' + str(
label[k]) + '.idx'
print(os.path.join(idx_filepath, idx_fname))
visusWriter.convert_image(image=img_crop,
idx_filename=os.path.join(
idx_filepath, idx_fname))
#im_crop = np.transpose(img_og, [1, 2, 0])
# img_stack = np.expand_dims( img_crop, axis=0)
if image_size is not None:
img_crop = Image.fromarray(img_crop, 'RGB')
#img_crop.show()
#import sys
#sys.exit(0)
img_crop = img_crop.resize((image_size, image_size),
Image.ANTIALIAS)
img_crop = np.array(img_crop)
img_crop = img_crop.reshape(
(image_size, image_size, 3)) # .transpose(0,3,1,2)
# to place as nd array for tensorflow and augmentors
if label[k] == 0:
# label[k] = 0.1
class_0 += 1
elif label[k] == 1:
# label[k] = 0.9
class_1 += 1
if label[k] != 0 and label[k] != 1:
for i in range(20):
print(
" >>>>>>>>>>>>>> LABELING INCORRECT> VALUE: ",
label[k])
if normalize:
img_crop = normalize_staining().apply_image(img_crop)
ret.append([img_crop.astype("uint8"), label[k]])
# img = copy.deepcopy(img_og)
print(">>>> Total crops observed: ", total_crops)
return (ret, class_0, class_1, unique_samples)