def move_to_corresponding_label_classification():
dataset_dir = QA_TRAIN_TXT
with open(dataset_dir) as f:
lines = f.readlines()
df = pd.DataFrame(columns=['image', 'modality', 'plane', 'organ'])
for index in tqdm(range(len(lines))):
line = lines[index]
df = get_class_image_model(df, line)
print(df)
n_group = len(set(zip(df['plane'], df['organ'], df['modality'])))
print(n_group)
df['label'] = df.plane.map(str) + "_" + df.organ.map(str)
for index, row in df.iterrows():
image = row["image"]
if image == "synpic47258":
a = 2
label = row["label"]
label_dir = os.path.join(CLASSIFICATION_DIR, label)
path_utils.make_dir(label_dir)
in_path = os.path.join(PREPROCESSED_DIR, "train",
"{}.jpg".format(image))
out_path = os.path.join(label_dir, "{}.jpg".format(image))
shutil.copy(in_path, out_path)
def generate_images_with_black_patches(path, path_out):
img_dirs = glob.glob(os.path.join(path, "*"))
print_utils.print_list(img_dirs)
paths_utils.make_dir(path_out)
def patch_black(im_in, index_list):
I = np.asarray(im_in)
I.setflags(write=1)
x, y, dx, dy = index_list
I[x:x + dx, y:y + dy, :] = 0
im = PIL.Image.fromarray(np.uint8(I))
return im
for path_img in img_dirs:
im_in = Image.open(path_img)
im_name = paths_utils.get_filename_without_extension(path_img)
list_index_list = [
[2 * 32, 2 * 32, 32, 32],
[2 * 32, 5 * 32, 32, 32],
[3 * 32, 3 * 32, 64, 64],
[5 * 32, 2 * 32, 32, 32],
[5 * 32, 5 * 32, 32, 32],
]
for index_list in list_index_list:
im_out = patch_black(im_in, index_list)
im_name_out = "{}_b-{}-{}-{}-{}.jpg".format(
im_name, str(index_list[0]), str(index_list[1]),
str(index_list[2]), str(index_list[3]))
im_name_out = os.path.join(path_out, im_name_out)
im_out.save(im_name_out)
def prepare_image_model(overwrite=False, P=0.7):
for breast in LIST_BREAST_CLASS:
folder_in = DATASETS_PHOTOS_DIR + breast
img_dirs = glob.glob(os.path.join(folder_in, "*.tif"))
num_images = len(img_dirs)
num_train = int(round(len(img_dirs)*P))
count_train = 0
for index, path_in in enumerate(img_dirs):
filename = path_utils.get_filename_without_extension(path_in)
if count_train < num_train:
folder_out = "{}train/{}".format(
PREPROCESSED_IMAGE_PHOTOS_DIR, breast)
count_train += 1
else:
folder_out = "{}val/{}".format(
PREPROCESSED_IMAGE_PHOTOS_DIR, breast)
path_utils.make_dir(folder_out)
path_out = folder_out + "/{}.jpg".format(filename)
if not os.path.exists(path_out) or overwrite:
print(">> processing {}/{}".format(index+1, len(img_dirs)))
normalize(path_in, path_out=path_out, is_debug=False,
is_save=True, is_resize=True)
else:
print("skip {}/{}".format(index+1, len(img_dirs)))
def main():
dict_prob = {}
df = pd.read_csv(PROCESSED_QA_PER_QUESTION_PATH)
# for key, value in DICT_SCORE.items():
# prob, val_loader = compute_prob_one_model(model_name=key)
# dict_prob[key] = prob.detach()
# for method in ["avg", "weighted"]:
# for ensem in ["best", "globalbilinear", "skipthoughts", "bert3072", "bert768", "all"]:
# sub_path = "{}ensemble/valid/{}_{}.json".format(
# SUB_DIR, ensem, method)
# sub = DICT_METHOD[ensem]
# acc, bleu = ensemble(dict_prob, val_loader,
# sub, sub_path, method=method,
# vqa_trainsplit="train")
# print(method, ensem, acc, bleu)
for key, value in DICT_SCORE.items():
prob, test_loader = compute_prob_one_model(model_name=key.replace(
"_train_", "_trainval_"),
vqa_trainsplit="trainval")
dict_prob[key] = prob.detach()
# del prob, test_loader
for method in ["avg", "weighted"]:
for ensem in ["globalbilinear", "skipthoughts"]:
sub_path = "{}ensemble/test/{}_{}.txt".format(
SUB_DIR, ensem, method)
path_utils.make_dir("{}ensemble/test".format(SUB_DIR))
sub = DICT_METHOD[ensem]
results = ensemble(dict_prob,
test_loader,
sub,
sub_path,
method=method,
vqa_trainsplit="trainval")
with open(TEST_DIR, encoding='UTF-8') as f:
lines = f.readlines()
f = open(sub_path, 'w')
count = 0
for line in lines:
file_id = line.split('\n')[0]
row_info = get_info(df, file_id + ".jpg")
question_id = row_info["question_id"]
f.write('{}|{}\n'.format(file_id, results[count]["answer"]))
count += 1
f.close(
) # you can omit in most cases as the destructor will call it
def move_files(P=0.7):
classification_dict_tool = {k: [] for k in list_tool}
segmentation_dict_tool = {k: [] for k in ["train", "val"]}
# read files
for tool in classification_dict_tool.keys():
path_write = "{}image_{}.txt".format(IMAGE_SET_DIR, tool)
temp = io_utils.read_file_to_list(path_write)
my_list = list()
for i in range(len(temp)):
my_list.append(temp[i].strip('\n'))
classification_dict_tool[tool] = my_list
for dataset in segmentation_dict_tool.keys():
path_write = "{}{}.txt".format(IMAGE_SET_DIR, dataset)
temp = io_utils.read_file_to_list(path_write)
my_list = list()
for i in range(len(temp)):
my_list.append(temp[i].strip('\n'))
segmentation_dict_tool[dataset] = my_list
# move images
for tool in classification_dict_tool.keys():
filenames = classification_dict_tool[tool]
num_images = len(filenames)
num_train = int(round(len(filenames)*P))
count_train = 0
for filename in filenames:
path_in = "{}{}.jpg".format(PREPROCESSED_IMAGE_DIR, filename)
if count_train < num_train:
dir_out = "{}train/{}/".format(CLASSIFICATION_IMAGE_DIR, tool)
count_train += 1
else:
dir_out = "{}val/{}/".format(CLASSIFICATION_IMAGE_DIR, tool)
path_utils.make_dir(dir_out)
path_out = "{}{}.jpg".format(dir_out, filename)
shutil.copyfile(path_in, path_out)
for dataset in segmentation_dict_tool.keys():
filenames = segmentation_dict_tool[dataset]
dir_out = "{}raw/{}/".format(SEGMENTATION_IMAGE_DIR, dataset)
path_utils.make_dir(dir_out)
for filename in filenames:
path_in = "{}{}.jpg".format(PREPROCESSED_IMAGE_DIR, filename)
path_out = "{}{}.jpg".format(dir_out, filename)
shutil.copyfile(path_in, path_out)
def prepare_image_model(overwrite=False, is_debug=False):
folder_in = DATASETS_PHOTOS_DIR
folder_out = PREPROCESSED_IMAGE_DIR
path_utils.make_dir(folder_out)
img_dirs = glob.glob(os.path.join(folder_in, "*.jpg"))
for index, path_in in enumerate(img_dirs):
filename = path_utils.get_filename_without_extension(path_in)
path_out = folder_out + "/{}.jpg".format(filename)
if not os.path.exists(path_out) or overwrite:
print(">> processing {}/{}".format(index+1, len(img_dirs)))
normalize(path_in, path_out=path_out, is_debug=is_debug,
is_save=True, is_resize=True, is_normalize=False)
else:
print("skip {}/{}".format(index+1, len(img_dirs)))
def main(dataset="breast"):
if dataset == "breast":
dataset_dict = BREAST
elif dataset == "idrid":
dataset_dict = IDRID
elif dataset == "tools":
dataset_dict = TOOLS
elif dataset == "vqa":
dataset_dict = VQA
src_dir = "temp"
dst_dir = "temp/sup"
paths_utils.make_dir(dst_dir)
wrt_dir = "figures/sup"
str = []
for key, value in dataset_dict.items():
img_name = key
question_str = value
question_str = question_str.replace(' ', '-')
noatt_name = "{}_noatt_question_{}.jpg".format(img_name,
question_str)
att_name = "{}_att_question_{}.jpg".format(img_name,
question_str)
cnn_name = "{}_cnn.jpg".format(img_name)
in_name = "{}_in.jpg".format(img_name)
occ_name = "{}_{}_w_{:0}_s_{:0}_color.jpg".format(
img_name, value.replace(' ', '_'), 32, 2)
str.append("Question: {} - Answer: {}\n\n".format(value, " "))
for name in [in_name, cnn_name, att_name, noatt_name, occ_name]:
path = find_in_dir(name, src_dir)
try:
shutil.copy(path, os.path.join(dst_dir, name))
str.append(
"\\includegraphics[width=\\subfigsize]{figures/sup/" + name + "}\n")
except:
str.append(
"\\includegraphics[width=\\subfigsize]{figures/sup/" + in_name + "}\n")
str.append("\n")
file1 = open("temp/myfile.txt", "a+")
file1.writelines(str)
file1.close()
def move_files():
# read files
segmentation_dict_tool = {k: [] for k in ["train", "val"]}
for dataset in segmentation_dict_tool.keys():
path_write = "{}{}.txt".format(IMAGE_SET_DIR, dataset)
temp = io_utils.read_file_to_list(path_write)
my_list = list()
for i in range(len(temp)):
my_list.append(temp[i].strip('\n'))
segmentation_dict_tool[dataset] = my_list
for dataset in segmentation_dict_tool.keys():
filenames = segmentation_dict_tool[dataset]
dir_out = "{}{}/".format(PREPROCESSED_IMAGE_WSI_RAW_DIR, dataset)
path_utils.make_dir(dir_out)
for filename in filenames:
path_in = "{}{}.png".format(
PREPROCESSED_IMAGE_WSI_PATCH_DIR, filename)
path_out = "{}{}.png".format(dir_out, filename)
shutil.copyfile(path_in, path_out)
def main(overwrite=False):
if overwrite and os.path.exists(PREPROCESSED_IMAGE_DIR):
path_utils.delete_dir(PREPROCESSED_IMAGE_DIR)
if overwrite and os.path.exists(CLASSIFICATION_IMAGE_DIR):
path_utils.delete_dir(CLASSIFICATION_IMAGE_DIR)
if overwrite and os.path.exists(SEGMENTATION_IMAGE_DIR):
path_utils.delete_dir(SEGMENTATION_IMAGE_DIR)
path_utils.make_dir(PREPROCESSED_IMAGE_DIR)
path_utils.make_dir(CLASSIFICATION_IMAGE_DIR)
path_utils.make_dir(SEGMENTATION_IMAGE_DIR)
path_utils.make_dir(IMAGE_SET_DIR)
print_utils.print_section("image model")
# prepare_image_model(overwrite=False, is_debug=False)
print_utils.print_section("split images")
# split_images_for_image_model_and_vqa()
print_utils.print_section("move files")
move_files()
def process_occlusion(path, dataset="breast"):
# global args
args = parser.parse_args()
LIST_QUESTION_BREAST = [
"how many classes are there",
"is normal larger than benign",
]
LIST_QUESTION_TOOLS = [
"is grasper in 0_0_32_32 location",
# "which tool has pointed tip on the left of the image",
# "how many tools are there",
]
LIST_QUESTION_IDRID = [
"is there haemorrhages in the fundus",
"is there soft exudates in the fundus",
]
if dataset == "breast":
list_question = LIST_QUESTION_BREAST
elif dataset == "tools":
list_question = LIST_QUESTION_TOOLS
elif dataset == "idrid":
list_question = LIST_QUESTION_IDRID
img_dirs = glob.glob(os.path.join(path, "*"))
args = update_args(
args, vqa_model="minhmul_att_train_2048", dataset=dataset)
for question_str in list_question:
for path_img in img_dirs:
print(
"\n\n=========================================================================")
print("{} - {}".format(question_str, path_img))
ans_gt = get_answer(dataset, path_img, question_str)
if ans_gt is None:
continue
else:
input_size = 256
step = 2
windows_size = 32
dst_dir = "temp/occlusion"
paths_utils.make_dir(dst_dir)
out_color_path = "{}/{}_{}_w_{:0}_s_{:0}_color.jpg".format(dst_dir,
paths_utils.get_filename_without_extension(
path_img),
question_str.replace(
' ', '_'),
windows_size,
step
)
out_gray_path = "{}/{}_{}_w_{:0}_s_{:0}_gray.jpg".format(dst_dir,
paths_utils.get_filename_without_extension(
path_img),
question_str.replace(
' ', '_'),
windows_size,
step
)
out_avg_path = "{}/{}_{}_w_{:0}_s_{:0}_avg.jpg".format(dst_dir,
paths_utils.get_filename_without_extension(
path_img),
question_str.replace(
' ', '_'),
windows_size,
step
)
if not os.path.exists(out_color_path):
visual_PIL = Image.open(path_img)
visual_PIL = visual_PIL.resize((input_size,input_size), Image.ANTIALIAS)
indices = np.asarray(
np.mgrid[0:input_size-windows_size+1:step, 0:input_size-windows_size+1:step].reshape(2, -1).T, dtype=np.int)
cnn, model, trainset = initialize(args, dataset=dataset)
image_occlusion = np.zeros((input_size, input_size))
image_occlusion_times = np.zeros((input_size, input_size))
anw_without_black_patch = process_one_batch_of_occlusion(args,
cnn,
model,
trainset,
visual_PIL,
question_str,
box=None,
dataset=dataset)
score_without_black_patch = anw_without_black_patch.get("val")[
anw_without_black_patch.get("ans").index(ans_gt)]
for i in range(indices.shape[0]):
print_utils.print_tqdm(i, indices.shape[0])
box = [indices[i][0], indices[i][0]+windows_size -
1, indices[i][1], indices[i][1]+windows_size-1]
# print(box)
ans = process_one_batch_of_occlusion(args,
cnn,
model,
trainset,
visual_PIL,
question_str,
box,
dataset=dataset,
is_print=False)
try:
score = ans.get("val")[
ans.get("ans").index(ans_gt)]
except:
score = 0
if score != 0:
score_occ = (
score_without_black_patch.item()-score.item())/score_without_black_patch.item()
image_occlusion[box[0]:box[1],
box[2]:box[3]] += score_occ
image_occlusion_times[box[0]:box[1],
box[2]:box[3]] += 1
save_image(visual_PIL, image_occlusion, image_occlusion_times,
out_color_path, out_gray_path, out_avg_path,
is_show=True)
def get_gradcam_from_image_model(path_img, cnn, dataset, finalconv_name="layer4"):
cnn.eval()
# hook the feature extractor
features_blobs = []
def hook_feature(module, input, output):
features_blobs.append(output.data.cpu().numpy())
cnn._modules.get(finalconv_name).register_forward_hook(hook_feature)
# get the softmax weight
params = list(cnn.parameters())
weight_softmax = np.squeeze(params[-2].data.cpu().numpy())
normalize = transforms.Normalize(
mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225]
)
preprocess = transforms.Compose([
transforms.Resize((224, 224)),
transforms.ToTensor(),
normalize
])
img_name = paths_utils.get_filename_without_extension(path_img)
img_pil = Image.open(path_img)
img_tensor = preprocess(img_pil)
img_variable = Variable(img_tensor.unsqueeze(0))
img_variable = img_variable.cuda(async=True)
logit = cnn(img_variable)
paths_utils.make_dir("temp/gradcam/{}/".format(dataset))
in_path = "temp/gradcam/{}/{}_in.jpg".format(dataset, img_name)
# img_pil.thumbnail((256, 256), Image.ANTIALIAS)
img_pil = img_pil.resize((256, 256), resample=PIL.Image.NEAREST)
img_pil.save(in_path)
# download the imagenet category list
classes = {
0: "Benign",
1: "InSitu",
2: "Invasive",
3: "Normal"
}
h_x = F.softmax(logit, dim=1).data.squeeze()
probs, idx = h_x.sort(0, True)
probs = probs.cpu().numpy()
idx = idx.cpu().numpy()
# generate class activation mapping for the top1 prediction
cam = get_gadcam_image(features_blobs[0], weight_softmax, [idx[0]])
img_name = paths_utils.get_filename_without_extension(path_img)
img = cv2.imread(in_path)
result = show_cam_on_image(img, cam)
out_path = "temp/gradcam/{}/{}_cnn.jpg".format(dataset,
img_name)
cv2.imwrite(out_path, result)
return result, out_path, features_blobs
def get_gradcam_from_vqa_model(visual_features,
question_features,
features_blobs_visual,
ans,
path_img,
cnn,
model,
question_str,
dataset,
vqa_model="minhmul_noatt_train_2048",
finalconv_name="linear_classif",
is_show_image=False):
model.eval()
# hook the feature extractor
features_blobs = []
def hook_feature(module, input, output):
features_blobs.append(output.data.cpu().numpy())
model._modules.get(finalconv_name).register_forward_hook(hook_feature)
# get the softmax weight
params = list(model.parameters())
weight_softmax = np.squeeze(params[-2].data.cpu().numpy())
if "noatt" in vqa_model:
classif_w_params = np.squeeze(params[10].data.cpu().numpy())
classif_b_params = np.squeeze(params[11].data.cpu().numpy())
else:
classif_w_params = np.squeeze(params[26].data.cpu().numpy())
temp_classif_w_params = np.zeros((classif_w_params.shape[0], 2048))
temp_classif_w_params = (classif_w_params[:, 0:2048] + classif_w_params[:, 2048:2048*2] +
classif_w_params[:, 2048*2:2048*3] + classif_w_params[:, 2048*3:2048*4]/4)
classif_w_params = temp_classif_w_params
classif_b_params = np.squeeze(params[27].data.cpu().numpy())
logit = model(visual_features, question_features)
h_x = F.softmax(logit, dim=1).data.squeeze()
probs, idx = h_x.sort(0, True)
probs = probs.cpu().numpy()
idx = idx.cpu().numpy()
cam = get_gadcam_vqa(features_blobs_visual[0],
classif_w_params, classif_b_params, [idx[0]])
# render the CAM and output
# print('output CAM.jpg for the top1 prediction: %s' % ans["ans"][idx[0]])
img_name = paths_utils.get_filename_without_extension(path_img)
in_path = "temp/gradcam/{}/{}_in.jpg".format(dataset, img_name)
img = cv2.imread(in_path)
result = show_cam_on_image(img, cam)
question_str = question_str.replace(' ', '-')
paths_utils.make_dir("temp/gradcam/{}/".format(dataset))
if "noatt" in vqa_model:
out_path = "temp/gradcam/{}/{}_noatt_question_{}.jpg".format(dataset,
img_name,
question_str)
else:
out_path = "temp/gradcam/{}/{}_att_question_{}.jpg".format(dataset,
img_name,
question_str)
cv2.imwrite(out_path, result)
im_out = Image.open(out_path)
if is_show_image:
im_out.show()
return logit
QA_VALID_TXT = PROJECT_DIR + \
"/data/raw/vqa_med/ImageClef-2019-VQA-Med-Validation/All_QA_Pairs_val.txt"
QA_TEST_TXT = PROJECT_DIR + \
"/data/raw/vqa_med/VQAMed2019Test/VQAMed2019_Test_Questions.txt"
DATASETS_TRAIN_TXT = PROJECT_DIR + \
"/data/raw/vqa_med/ImageClef-2019-VQA-Med-Training/All_QA_Pairs_train.txt"
RAW_DIR = PROJECT_DIR + "/data/vqa_med/raw/raw/"
PROCESSED_QA_PER_QUESTION_PATH = RAW_DIR + "med_qa_per_question.csv"
PROCESSED_QA_PER_QUESTION_AUGMENT_PATH = RAW_DIR + \
"med_qa_per_question_augment.csv"
PREPROCESSED_DIR = PROJECT_DIR + \
"/data/raw/vqa_med/preprocessed/raw"
IMAGEID_PATH = RAW_DIR + "image_id.csv"
path_utils.make_dir(RAW_DIR)
def generate_image_id(is_augment=False):
if is_augment:
img_train_paths = glob.glob(
os.path.join(PREPROCESSED_DIR, "train_augment", "*.jpg"))
img_val_paths = glob.glob(
os.path.join(PREPROCESSED_DIR, "val_augment", "*.jpg"))
img_test_paths = glob.glob(
os.path.join(PREPROCESSED_DIR, "test_augment", "*.jpg"))
else:
img_train_paths = glob.glob(os.path.join(DATASETS_TRAIN_DIR, "*.jpg"))
img_val_paths = glob.glob(os.path.join(DATASETS_VALID_DIR, "*.jpg"))
img_test_paths = glob.glob(os.path.join(DATASETS_TEST_DIR, "*.jpg"))
from scipy.misc import imsave, imresize
from scipy.misc import toimage
import os
OPENSLIDE_PATH = "C:/Users/minhm/Documents/GitHub/bin/openslide-win64-20171122/bin"
if os.path.exists(OPENSLIDE_PATH):
os.environ['PATH'] = OPENSLIDE_PATH + ";" + os.environ['PATH']
import openslide
CURRENT_WORKING_DIR = os.path.realpath(__file__)
PROJECT_DIR = path_utils.get_project_dir(CURRENT_WORKING_DIR, "vqa_idrid")
DATASETS_PHOTOS_DIR = PROJECT_DIR + \
"/data/raw/m2cai16-tool-locations/JPEGImages/"
PREPROCESSED_IMAGE_DIR = PROJECT_DIR + \
"/data/raw/m2cai16-tool-locations/preprocessed/full/"
path_utils.make_dir(PREPROCESSED_IMAGE_DIR)
def normalize(path_in,
path_out=None,
is_debug=False,
is_save=False,
is_resize=True,
is_normalize=True):
im = Image.open(path_in)
imarray = np.array(im)
if is_normalize:
im_norm = normalization_utils.normalize_staining(imarray)
else:
im_norm = imarray
def process_occlusion(path, dataset="breast"):
# global args
args = parser.parse_args()
LIST_QUESTION_BREAST = [
"how many classes are there",
"is there any benign class in the image",
"is there any in situ class in the image",
"is there any invasive class in the image",
"what is the major class in the image",
"what is the minor class in the image",
"is benign in 64_64_32_32 location",
"is invasive in 96_96_32_32 location",
]
LIST_QUESTION_TOOLS = [
"how many tools are there",
"is scissors in 64_32_32_32 location",
"is irrigator in 64_96_32_32 location",
"is grasper in 64_96_32_32 location"
"is bipolar in 64_96_32_32 location"
"is hook in 64_96_32_32 location"
"is clipper in 64_96_32_32 location"
"is specimenbag in 64_96_32_32 location"
"is there any grasper in the image",
"is there any bipolar in the image",
"is there any hook in the image",
"is there any scissors in the image",
"is there any clipper in the image",
"is there any irrigator in the image",
"is there any specimenbag in the image",
]
LIST_QUESTION_IDRID = [
"is there haemorrhages in the fundus",
"is there microaneurysms in the fundus",
"is there soft exudates in the fundus",
"is there hard exudates in the fundus",
"is hard exudates larger than soft exudates",
"is haemorrhages smaller than microaneurysms",
"is there haemorrhages in the region 32_32_16_16",
"is there microaneurysms in the region 96_96_16_16",
]
LIST_QUESTION_VQA2 = [
"what color is the hydrant",
"why are the men jumping to catch",
"is the water still",
"how many people are in the image"
]
if dataset == "breast":
list_question = LIST_QUESTION_BREAST
elif dataset == "tools":
list_question = LIST_QUESTION_TOOLS
elif dataset == "idrid":
list_question = LIST_QUESTION_IDRID
elif dataset == "vqa2":
list_question = LIST_QUESTION_VQA2
img_dirs = glob.glob(os.path.join(path, "*"))
# args = update_args(
# args, vqa_model="minhmul_att_train_2048", dataset=dataset)
args = update_args(
args, vqa_model="minhmul_att_train", dataset=dataset)
shuffle(img_dirs)
shuffle(list_question)
# for (path_img, question_str) in zip(img_dirs, list_question):
for path_img in img_dirs:
for question_str in list_question:
if dataset in ["vqa", "vqa2"]:
if not ((question_str == "what color is the hydrant" and ("img1" in path_img or "img2" in path_img)) or
(question_str == "why are the men jumping to catch" and ("img3" in path_img or "img4" in path_img)) or
(question_str == "is the water still" and ("img5" in path_img or "img6" in path_img)) or
(question_str == "how many people are in the image" and ("img7" in path_img or "img8" in path_img))):
continue
print(
"\n\n=========================================================================")
print("{} - {}".format(question_str, path_img))
if dataset == "vqa2":
ans_gt = "red"
else:
ans_gt = get_answer(dataset, path_img, question_str)
if ans_gt is None:
continue
else:
input_size = 256
step = 2
windows_size = 32
dst_dir = "temp/occlusion"
paths_utils.make_dir(dst_dir)
out_color_path = "{}/{}_{}_w_{:0}_s_{:0}_color.jpg".format(dst_dir,
paths_utils.get_filename_without_extension(
path_img),
question_str.replace(
' ', '_'),
windows_size,
step
)
out_gray_path = "{}/{}_{}_w_{:0}_s_{:0}_gray.jpg".format(dst_dir,
paths_utils.get_filename_without_extension(
path_img),
question_str.replace(
' ', '_'),
windows_size,
step
)
out_avg_path = "{}/{}_{}_w_{:0}_s_{:0}_avg.jpg".format(dst_dir,
paths_utils.get_filename_without_extension(
path_img),
question_str.replace(
' ', '_'),
windows_size,
step
)
if not os.path.exists(out_color_path):
visual_PIL = Image.open(path_img)
indices = np.asarray(
np.mgrid[0:input_size-windows_size+1:step, 0:input_size-windows_size+1:step].reshape(2, -1).T, dtype=np.int)
cnn, model, trainset = initialize(args, dataset=dataset)
# cnn, model, trainset = None, None, None
image_occlusion = np.zeros((input_size, input_size))
image_occlusion_times = np.zeros((input_size, input_size))
ans_without_black_patch = process_one_batch_of_occlusion(args,
cnn,
model,
trainset,
visual_PIL,
question_str,
list_box=None,
dataset=dataset)
try:
score_without_black_patch = ans_without_black_patch[0].get("val")[
ans_without_black_patch[0].get("ans").index(ans_gt)]
except:
score_without_black_patch = torch.tensor(0)
batch = 32
count = 0
list_box = []
for i in range(indices.shape[0]):
print_utils.print_tqdm(i, indices.shape[0])
list_box.append([indices[i][0], indices[i][0]+windows_size -
1, indices[i][1], indices[i][1]+windows_size-1])
count += 1
# if count == batch or i == indices.shape[0] - 1:
if count == batch:
# print(count)
ans = process_one_batch_of_occlusion(args,
cnn,
model,
trainset,
visual_PIL,
question_str,
list_box,
dataset=dataset,
is_print=False)
for i in range(len(list_box)):
try:
score = ans[i].get("val")[
ans[i].get("ans").index(ans_gt)]
except:
score = 0
box = list_box[i]
if score != 0:
try:
score_occ = (
score.item() - score_without_black_patch.item())/score_without_black_patch.item()
except:
score_occ = 0
image_occlusion[box[0]:box[1],
box[2]:box[3]] += score_occ
image_occlusion_times[box[0]:box[1],
box[2]:box[3]] += 1
count = 0
list_box = []
save_image(visual_PIL, image_occlusion, image_occlusion_times,
out_color_path, out_gray_path, out_avg_path,
is_show=False)
import pandas as pd
import glob
import os
import json
CURRENT_WORKING_DIR = os.path.realpath(__file__)
PROJECT_DIR = path_utils.get_project_dir(CURRENT_WORKING_DIR, "vqa_idrid")
IMAGE_SET_DIR = PROJECT_DIR + \
"/data/raw/breast-cancer/preprocessed/imagesets/"
RAW_DIR = PROJECT_DIR + "/data/vqa_breast/raw/raw/"
QA_PATH = RAW_DIR + "breast_qa_full.csv"
processed_qa_per_question_path = RAW_DIR + "breast_qa_per_question.csv"
INTERIM_DIR = PROJECT_DIR + "/data/vqa_breast/interim/"
path_utils.make_dir(INTERIM_DIR)
train_annotations_filename = INTERIM_DIR + 'train_questions_annotations.json'
val_annotations_filename = INTERIM_DIR + 'val_questions_annotations.json'
def read_train_val_split():
segmentation_dict_tool = {k: [] for k in ["train", "val"]}
# read files
for dataset in segmentation_dict_tool.keys():
path_write = "{}{}.txt".format(IMAGE_SET_DIR, dataset)
temp = io_utils.read_file_to_list(path_write)
my_list = list()
for i in range(len(temp)):
my_list.append(temp[i].strip('\n'))
segmentation_dict_tool[dataset] = my_list
def vqa_processed(params):
#####################################################
# Read input files
#####################################################
path_train = os.path.join(
params['dir'], 'interim',
params['trainsplit'] + '_questions_annotations.json')
if params['trainsplit'] == 'train':
path_val = os.path.join(params['dir'], 'interim',
'val_questions_annotations.json')
path_test = os.path.join(params['dir'], 'interim', 'test_questions.json')
path_testdev = os.path.join(params['dir'], 'interim',
'testdev_questions.json')
# An example is a tuple (question, image, answer)
# /!\ test and test-dev have no answer
trainset = json.load(open(path_train, 'r'))
if params['trainsplit'] == 'train':
valset = json.load(open(path_val, 'r'))
testset = json.load(open(path_test, 'r'))
testdevset = json.load(open(path_testdev, 'r'))
#####################################################
# Preprocess examples (questions and answers)
#####################################################
top_answers = process_utils.get_top_answers(trainset, params['nans'])
aid_to_ans = [a for i, a in enumerate(top_answers)]
ans_to_aid = {a: i for i, a in enumerate(top_answers)}
# Remove examples if answer is not in top answers
trainset = process_utils.remove_examples(trainset, ans_to_aid)
# Add 'question_words' to the initial tuple
trainset = process_utils.preprocess_questions(trainset, params['nlp'])
if params['trainsplit'] == 'train':
valset = process_utils.preprocess_questions(valset, params['nlp'])
testset = process_utils.preprocess_questions(testset, params['nlp'])
testdevset = process_utils.preprocess_questions(testdevset, params['nlp'])
# Also process top_words which contains a UNK char
trainset, top_words = process_utils.remove_long_tail_train(
trainset, params['minwcount'])
wid_to_word = {i + 1: w for i, w in enumerate(top_words)}
word_to_wid = {w: i + 1 for i, w in enumerate(top_words)}
if params['trainsplit'] == 'train':
valset = process_utils.remove_long_tail_test(valset, word_to_wid)
testset = process_utils.remove_long_tail_test(testset, word_to_wid)
testdevset = process_utils.remove_long_tail_test(testdevset, word_to_wid)
trainset = process_utils.encode_question(trainset, word_to_wid,
params['maxlength'],
params['pad'])
if params['trainsplit'] == 'train':
valset = process_utils.encode_question(valset, word_to_wid,
params['maxlength'],
params['pad'])
testset = process_utils.encode_question(testset, word_to_wid,
params['maxlength'], params['pad'])
testdevset = process_utils.encode_question(testdevset, word_to_wid,
params['maxlength'],
params['pad'])
trainset = process_utils.encode_answer(trainset, ans_to_aid)
trainset = process_utils.encode_answers_occurence(trainset, ans_to_aid)
if params['trainsplit'] == 'train':
valset = process_utils.encode_answer(valset, ans_to_aid)
valset = process_utils.encode_answers_occurence(valset, ans_to_aid)
#####################################################
# Write output files
#####################################################
# Paths to output files
# Ex: data/vqa/processed/nans,3000_maxlength,15_..._trainsplit,train_testsplit,val/id_to_word.json
subdirname = 'nans,' + str(params['nans'])
for param in ['maxlength', 'minwcount', 'nlp', 'pad', 'trainsplit']:
subdirname += '_' + param + ',' + str(params[param])
os.system('mkdir -p ' +
os.path.join(params['dir'], 'processed', subdirname))
dir_save = os.path.join(params['dir'], 'processed', subdirname)
if not os.path.exists(dir_save):
print('>> make dir', dir_save)
path_utils.make_dir(dir_save)
path_wid_to_word = os.path.join(params['dir'], 'processed', subdirname,
'wid_to_word.pickle')
path_word_to_wid = os.path.join(params['dir'], 'processed', subdirname,
'word_to_wid.pickle')
path_aid_to_ans = os.path.join(params['dir'], 'processed', subdirname,
'aid_to_ans.pickle')
path_ans_to_aid = os.path.join(params['dir'], 'processed', subdirname,
'ans_to_aid.pickle')
if params['trainsplit'] == 'train':
path_trainset = os.path.join(params['dir'], 'processed', subdirname,
'trainset.pickle')
path_valset = os.path.join(params['dir'], 'processed', subdirname,
'valset.pickle')
elif params['trainsplit'] == 'trainval':
path_trainset = os.path.join(params['dir'], 'processed', subdirname,
'trainvalset.pickle')
path_testset = os.path.join(params['dir'], 'processed', subdirname,
'testset.pickle')
path_testdevset = os.path.join(params['dir'], 'processed', subdirname,
'testdevset.pickle')
print('Write wid_to_word to', path_wid_to_word)
with open(path_wid_to_word, 'wb') as handle:
pickle.dump(wid_to_word, handle)
print('Write word_to_wid to', path_word_to_wid)
with open(path_word_to_wid, 'wb') as handle:
pickle.dump(word_to_wid, handle)
print('Write aid_to_ans to', path_aid_to_ans)
with open(path_aid_to_ans, 'wb') as handle:
pickle.dump(aid_to_ans, handle)
print('Write ans_to_aid to', path_ans_to_aid)
with open(path_ans_to_aid, 'wb') as handle:
pickle.dump(ans_to_aid, handle)
print('Write trainset to', path_trainset)
with open(path_trainset, 'wb') as handle:
pickle.dump(trainset, handle)
if params['trainsplit'] == 'train':
print('Write valset to', path_valset)
with open(path_valset, 'wb') as handle:
pickle.dump(valset, handle)
print('Write testset to', path_testset)
with open(path_testset, 'wb') as handle:
pickle.dump(testset, handle)
print('Write testdevset to', path_testdevset)
with open(path_testdevset, 'wb') as handle:
pickle.dump(testdevset, handle)
DATASETS_PHOTOS_DIR = PROJECT_DIR + \
"/data/raw/breast-cancer/ICIAR2018_BACH_Challenge/Photos/"
DATASETS_WSI_DIR = PROJECT_DIR + \
"/data/raw/breast-cancer/ICIAR2018_BACH_Challenge/WSI/"
PREPROCESSED_IMAGE_PHOTOS_DIR = PROJECT_DIR + \
"/data/raw/breast-cancer/preprocessed/Photos/"
PREPROCESSED_IMAGE_WSI_DIR = PROJECT_DIR + \
"/data/raw/breast-cancer/preprocessed/WSI/"
IMAGE_SET_DIR = PROJECT_DIR + \
"/data/raw/breast-cancer/preprocessed/imagesets/"
PREPROCESSED_IMAGE_WSI_RAW_DIR = PREPROCESSED_IMAGE_WSI_DIR + "raw/"
PREPROCESSED_IMAGE_WSI_PATCH_DIR = PREPROCESSED_IMAGE_WSI_DIR + "patch/"
PREPROCESSED_IMAGE_WSI_GT_DIR = PREPROCESSED_IMAGE_WSI_DIR + "patch_gt/"
path_utils.make_dir(PREPROCESSED_IMAGE_PHOTOS_DIR)
path_utils.make_dir(PREPROCESSED_IMAGE_WSI_DIR)
path_utils.make_dir(PREPROCESSED_IMAGE_WSI_PATCH_DIR)
path_utils.make_dir(PREPROCESSED_IMAGE_WSI_GT_DIR)
path_utils.make_dir(IMAGE_SET_DIR)
path_utils.make_dir(PREPROCESSED_IMAGE_WSI_RAW_DIR)
LIST_BREAST_CLASS = ["Benign", "InSitu", "Invasive", "Normal"]
def normalize(path_in, path_out=None, is_debug=False, is_save=False, is_resize=True):
im = Image.open(path_in)
imarray = np.array(im)
im_norm = normalization_utils.normalize_staining(imarray)
im_norm = Image.fromarray(im_norm)
import pandas as pd
import numpy as np
from pprint import pprint
import glob
import json
CURRENT_WORKING_DIR = os.path.realpath(__file__)
PROJECT_DIR = path_utils.get_project_dir(CURRENT_WORKING_DIR, "vqa_idrid")
LOGS_DIR = os.path.join(PROJECT_DIR, "logs/med/trainval")
RAW_DIR = PROJECT_DIR + "/data/vqa_med/raw/raw/"
PROCESSED_QA_PER_QUESTION_PATH = RAW_DIR + "med_qa_per_question.csv"
TEST_DIR = PROJECT_DIR + \
"/data/raw/vqa_med/VQAMed2019Test/VQAMed2019_Test_ImageList.txt"
SUB_DIR = PROJECT_DIR + "/data/vqa_med/submission/"
path_utils.make_dir(SUB_DIR)
# SUB_QC_MLB = [
# "minhmul_att_trainval_imagenet_h200_g4_relu",
# "minhmul_att_trainval_imagenet_h200_g4_relu_bert_uncased",
# "minhmul_att_trainval_imagenet_h200_g4_relu_bert_cased",
# "minhmul_att_trainval_imagenet_h200_g4_relu_bert_uncased_768",
# "minhmul_att_trainval_imagenet_h200_g4_relu_bert_cased_768",
# "minhmul_att_trainval_imagenet_h200_g8_relu",
# "minhmul_att_trainval_imagenet_h400_g8_relu",
# "minhmul_att_trainval_imagenet_h100_g8_relu",
# "minhmul_att_trainval_imagenet_h100_g8_relu_bert_uncased",
# "minhmul_att_trainval_imagenet_h100_g8_relu_bert_cased",
# "minhmul_att_trainval_imagenet_h100_g8_relu_bert_uncased_768",
# "minhmul_att_trainval_imagenet_h100_g8_relu_bert_cased_768",
def preprocess_dataset(dataset="train", is_show=False, is_overwrite=False, is_augment=False):
if dataset == "train":
dataset_dir = DATASETS_TRAIN_DIR
elif dataset == "val":
dataset_dir = DATASETS_VALID_DIR
else:
dataset_dir = DATASETS_TEST_DIR
if is_augment:
preprocessed_dir = os.path.join(
PREPROCESSED_DIR, "raw", dataset + "_augment")
else:
preprocessed_dir = os.path.join(PREPROCESSED_DIR, "raw", dataset)
if is_overwrite or not os.path.exists(preprocessed_dir):
path_utils.make_dir(preprocessed_dir)
img_paths = glob.glob(os.path.join(dataset_dir, "*.jpg"))
for index in tqdm(range(len(img_paths))):
img_preprocessed = preprocess.process(index, img_paths)
img_preprocessed = cv2.resize(img_preprocessed, (256, 256))
if is_show:
cv2.imshow('Done', img_preprocessed)
cv2.waitKey(0)
out_path = os.path.join(
preprocessed_dir, path_utils.get_filename(img_paths[index]))
cv2.imwrite(out_path, img_preprocessed)
out = img_preprocessed
if is_augment:
for augment in LIST_AUGMENT:
out_path = os.path.join(
preprocessed_dir, path_utils.get_filename_without_extension(img_paths[index]) + "_{}.jpg".format(augment))
img_preprocessed = Image.fromarray(out)
if augment == "fliplr":
img_preprocessed = img_preprocessed.transpose(
Image.FLIP_LEFT_RIGHT)
elif augment == "rot10":
img_preprocessed = img_preprocessed.rotate(10)
elif augment == "rot20":
img_preprocessed = img_preprocessed.rotate(20)
elif augment == "rot30":
img_preprocessed = img_preprocessed.rotate(30)
elif augment == "rot_10":
img_preprocessed = img_preprocessed.rotate(-10)
elif augment == "rot_20":
img_preprocessed = img_preprocessed.rotate(-20)
elif augment == "rot_30":
img_preprocessed = img_preprocessed.rotate(-30)
elif augment == "bright1":
img_preprocessed = ie.Contrast(
img_preprocessed).enhance(1)
elif augment == "bright_1":
img_preprocessed = ie.Contrast(
img_preprocessed).enhance(-1)
elif augment == "sharp3":
img_preprocessed = ie.Sharpness(
img_preprocessed).enhance(3)
elif augment == "sharp_3":
img_preprocessed = ie.Sharpness(
img_preprocessed).enhance(-3)
# elif augment == "contrast2":
# img_preprocessed = ie.Contrast(img_preprocessed).enhance(2)
# elif augment == "contrast_2":
# img_preprocessed = ie.Contrast(img_preprocessed).enhance(-2)
img_preprocessed.save(out_path)
def get_gradcam_from_vqa_model(visual_features,
question_features,
features_blobs_visual,
ans,
path_img,
cnn,
model,
question_str,
dataset,
vqa_model="minhmul_noatt_train_2048",
finalconv_name="linear_classif",
is_show_image=False,
is_att=True):
if is_att:
logit, list_v_record = model(visual_features, question_features)
cam = get_gadcam_vqa_new(list_v_record)
else:
# grad_cam = gradcam_utils.GradCam(model=model,
# target_layer_names=["linear_v"], use_cuda=True)
# target_index = None
# mask = grad_cam(visual_features,
# question_features,
# target_index)
# hook the feature extractor
features_blobs = []
def hook_feature(module, input, output):
features_blobs.append(output.data.cpu().numpy())
model._modules.get(finalconv_name).register_forward_hook(hook_feature)
# model.fusion.linear_v.register_forward_hook(hook_feature)
# model.fusion.linear_v.register_backward_hook(hook_feature)
# get the softmax weight
params = list(model.parameters())
weight_softmax = np.squeeze(params[-2].data.cpu().numpy())
if "noatt" in vqa_model:
classif_w_params = np.squeeze(params[10].data.cpu().numpy())
classif_b_params = np.squeeze(params[11].data.cpu().numpy())
logit = model(visual_features, question_features)
h_x = F.softmax(logit, dim=1).data.squeeze()
probs, idx = h_x.sort(0, True)
probs = probs.cpu().numpy()
idx = idx.cpu().numpy()
cam = get_gadcam_vqa(features_blobs_visual[0], classif_w_params,
classif_b_params, [idx[0]])
img_name = paths_utils.get_filename_without_extension(path_img)
in_path = "temp/gradcam/{}/{}_in.jpg".format(dataset, img_name)
img = cv2.imread(in_path)
result = show_cam_on_image(img, cam)
question_str = question_str.replace(' ', '-')
paths_utils.make_dir("temp/gradcam/{}/".format(dataset))
if "noatt" in vqa_model:
out_path = "temp/gradcam/{}/{}_noatt_question_{}.jpg".format(
dataset, img_name, question_str)
else:
out_path = "temp/gradcam/{}/{}_att_question_{}.jpg".format(
dataset, img_name, question_str)
cv2.imwrite(out_path, result)
im_out = Image.open(out_path)
if is_show_image:
im_out.show()
return logit
