def prepare_image_model(overwrite=False, P=0.7):
for breast in LIST_BREAST_CLASS:
folder_in = DATASETS_PHOTOS_DIR + breast
img_dirs = glob.glob(os.path.join(folder_in, "*.tif"))
num_images = len(img_dirs)
num_train = int(round(len(img_dirs)*P))
count_train = 0
for index, path_in in enumerate(img_dirs):
filename = path_utils.get_filename_without_extension(path_in)
if count_train < num_train:
folder_out = "{}train/{}".format(
PREPROCESSED_IMAGE_PHOTOS_DIR, breast)
count_train += 1
else:
folder_out = "{}val/{}".format(
PREPROCESSED_IMAGE_PHOTOS_DIR, breast)
path_utils.make_dir(folder_out)
path_out = folder_out + "/{}.jpg".format(filename)
if not os.path.exists(path_out) or overwrite:
print(">> processing {}/{}".format(index+1, len(img_dirs)))
normalize(path_in, path_out=path_out, is_debug=False,
is_save=True, is_resize=True)
else:
print("skip {}/{}".format(index+1, len(img_dirs)))
def generate_images_with_black_patches(path, path_out):
img_dirs = glob.glob(os.path.join(path, "*"))
print_utils.print_list(img_dirs)
paths_utils.make_dir(path_out)
def patch_black(im_in, index_list):
I = np.asarray(im_in)
I.setflags(write=1)
x, y, dx, dy = index_list
I[x:x + dx, y:y + dy, :] = 0
im = PIL.Image.fromarray(np.uint8(I))
return im
for path_img in img_dirs:
im_in = Image.open(path_img)
im_name = paths_utils.get_filename_without_extension(path_img)
list_index_list = [
[2 * 32, 2 * 32, 32, 32],
[2 * 32, 5 * 32, 32, 32],
[3 * 32, 3 * 32, 64, 64],
[5 * 32, 2 * 32, 32, 32],
[5 * 32, 5 * 32, 32, 32],
]
for index_list in list_index_list:
im_out = patch_black(im_in, index_list)
im_name_out = "{}_b-{}-{}-{}-{}.jpg".format(
im_name, str(index_list[0]), str(index_list[1]),
str(index_list[2]), str(index_list[3]))
im_name_out = os.path.join(path_out, im_name_out)
im_out.save(im_name_out)
def main():
folders = glob.glob(os.path.join(LOGS_DIR, "*"))
df = pd.read_csv(PROCESSED_QA_PER_QUESTION_PATH)
with open(TEST_DIR, encoding='UTF-8') as f:
lines = f.readlines()
dict_score = {}
# for method in ["qcmlb", "bilinear", "skipthoughts", "bert3072", "bert768", "all"]:
for method in ["best", "globalbilinear", "skipthoughts", "bert3072", "bert768", "all"]:
# for method in []:
sub_path = SUB_DIR + method + ".txt"
sub = DICT_METHOD[method]
method_folders = []
for item in sub:
for folder in folders:
if item == path_utils.get_filename_without_extension(folder):
method_folders.append(folder)
method_folders = list(set(method_folders))
for folder in method_folders:
print('>> processing {} of {}'.format(folder, method))
weight = 1 if "bert" in folder else 5
dict_score = get_ans(dict_score, folder, df, weight=weight)
f = open(sub_path, 'w')
for line in lines:
file_id = line.split('\n')[0]
dict_ans_score, final_answer = get_final_answer(
dict_score, file_id)
f.write('{}|{}\n'.format(file_id, final_answer)) # python will convert \n to os.linesep
f.close() # you can omit in most cases as the destructor will call it
def split_images_for_vqa(P=0.7):
paths = glob.glob(PREPROCESSED_IMAGE_WSI_PATCH_DIR + "*.png")
random.shuffle(paths)
num_images = len(paths)
num_images_segmentation_train = int(round(num_images*P))
num_images_segmentation_val = num_images - num_images_segmentation_train
segmentation_dict_tool = {k: [] for k in ["train", "val"]}
count_train_images_segmentation = 0
for index, path in enumerate(paths):
filename = path_utils.get_filename_without_extension(path)
print(">> processing {}/{}".format(index+1, len(paths)))
if count_train_images_segmentation < num_images_segmentation_train:
segmentation_dict_tool["train"].append(filename)
count_train_images_segmentation += 1
else:
segmentation_dict_tool["val"].append(filename)
for dataset in segmentation_dict_tool.keys():
my_list = segmentation_dict_tool[dataset]
path_write = "{}{}.txt".format(IMAGE_SET_DIR, dataset)
if not os.path.exists(path_write):
print("write to", path_write)
io_utils.write_list_to_file(my_list, path_write)
else:
print("path exists. Skip...")
def prepare_question_model(is_save=False):
patch_size = (2000, 2000)
xml_dirs = glob.glob(os.path.join(DATASETS_WSI_DIR, "*.xml"))
store = []
for xml_dir in xml_dirs:
filename = path_utils.get_filename_without_extension(xml_dir)
img_dir = DATASETS_WSI_DIR + filename + ".svs"
scan = openslide.OpenSlide(img_dir)
dims = scan.dimensions
# read svs
img_svs = svs_utils.read_svs(
img_dir, patch_size=patch_size, is_save=True)
# read xml
coords, labels, length, area, pixel_spacing = xml_utils.read_xml_breast(
xml_dir)
store += [[coords, labels, length, area, pixel_spacing]]
save_dir = DATASETS_WSI_DIR+filename+'.png'
gt = image_utils.generate_groundtruth_from_xml(
save_dir, dims, coords, labels, is_debug=False, is_save=is_save)
for upsampling_factor in [4, 5, 6]:
process_one_svs_one_xml(
filename, img_svs, gt, upsampling_factor=upsampling_factor, is_debug=False)
def get_top1_ans(dict_score, folder, df, fr=1, to=1, weight=1):
list_epochs = list(range(fr, to + 1))
for epoch in list_epochs:
folder_epoch_json = "{}/epoch_{}/vqa_OpenEnded_mscoco_test2015_model_results.json".format(
folder, epoch)
with open(folder_epoch_json) as json_file:
data = json.load(json_file)
for q in range(len(data)):
question_id = data[q]["question_id"]
row_info = get_info(df, question_id)
file_id = row_info["file_id"]
file_id = path_utils.get_filename_without_extension(file_id)
for a in range(1, 4):
ans = data[q]['answer{}'.format(a)]
if not keys_exists(dict_score, file_id):
dict_score[file_id] = {
'answer{}'.format(a): {
ans: 1 * weight
}
}
elif not keys_exists(dict_score, file_id,
'answer{}'.format(a)):
dict_score[file_id]['answer{}'.format(a)] = {
ans: 1 * weight
}
elif not keys_exists(dict_score, file_id, 'answer{}'.format(a),
ans):
dict_score[file_id]['answer{}'.format(a)][ans] = 1 * weight
else:
dict_score[file_id]['answer{}'.format(
a)][ans] += 1 * weight
return dict_score
def split_images_for_image_model_and_vqa(N=100, P=0.6):
paths = glob.glob(DATASETS_DIR + "*.xml")
random.shuffle(paths)
num_images = len(paths)
num_images_segmentation = num_images - N*len(list_tool)
num_images_segmentation_train = int(round(num_images_segmentation*P))
num_images_segmentation_val = num_images_segmentation - num_images_segmentation_train
rows = list()
rows_temp = list()
count_dict_tool_multi = dict(zip(list_tool, [0] * len(list_tool)))
classification_dict_tool = {k: [] for k in list_tool}
segmentation_dict_tool = {k: [] for k in ["train", "val"]}
count_train_images_segmentation = 0
for index, path in enumerate(paths):
filename = path_utils.get_filename_without_extension(path)
print(">> processing {}/{}".format(index+1, len(paths)))
print(filename)
mydoc = minidom.parse(path)
boxes = xml_utils.get_object_xml(mydoc)
for i in range(len(boxes)):
tool, xmin, xmax, ymin, ymax = boxes[i]
tool = tool.lower()
if len(boxes) == 1 and count_dict_tool_multi[tool] < N:
classification_dict_tool[tool].append(filename)
count_dict_tool_multi[tool] += 1
print(count_dict_tool_multi)
else:
if count_train_images_segmentation < num_images_segmentation_train:
segmentation_dict_tool["train"].append(filename)
count_train_images_segmentation += 1
else:
segmentation_dict_tool["val"].append(filename)
for tool in classification_dict_tool.keys():
my_list = classification_dict_tool[tool]
path_write = "{}image_{}.txt".format(IMAGE_SET_DIR, tool)
if not os.path.exists(path_write):
print("write to", path_write)
io_utils.write_list_to_file(my_list, path_write)
else:
print("path exists. Skip...")
for dataset in segmentation_dict_tool.keys():
my_list = segmentation_dict_tool[dataset]
path_write = "{}{}.txt".format(IMAGE_SET_DIR, dataset)
if not os.path.exists(path_write):
print("write to", path_write)
io_utils.write_list_to_file(my_list, path_write)
else:
print("path exists. Skip...")
def prepare_image_model(overwrite=False, is_debug=False):
folder_in = DATASETS_PHOTOS_DIR
folder_out = PREPROCESSED_IMAGE_DIR
path_utils.make_dir(folder_out)
img_dirs = glob.glob(os.path.join(folder_in, "*.jpg"))
for index, path_in in enumerate(img_dirs):
filename = path_utils.get_filename_without_extension(path_in)
path_out = folder_out + "/{}.jpg".format(filename)
if not os.path.exists(path_out) or overwrite:
print(">> processing {}/{}".format(index+1, len(img_dirs)))
normalize(path_in, path_out=path_out, is_debug=is_debug,
is_save=True, is_resize=True, is_normalize=False)
else:
print("skip {}/{}".format(index+1, len(img_dirs)))
def process(index, img_dirs, is_draw=False):
img_path = img_dirs[index]
img = cv2.imread(img_path)
if len(img.shape) == 3:
img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
if len(img.shape) > 3:
assert (len(img.shape) > 3)
# equ = cv2.equalizeHist(img)
img = normalize_0_255(img)
th = otsu_binarize(img, threshold=5)
img_removed_text = remove_text(th, factor=40)
img_filled_holes = fill_holes(img_removed_text)
img_removed_small = remove_connected_objects(img_filled_holes, factor=3)
slice_x, slice_y = get_bounding_box(img_removed_small, label=255)
img_1st = img[slice_x, slice_y]
th_roi = img_removed_small[slice_x, slice_y]
final_mask = remove_text(th_roi, factor=50)
final_mask_binary = np.array(final_mask / 255, dtype=np.uint8)
W, H = final_mask_binary.shape[0], final_mask_binary.shape[1]
# final_mask_binary[int(0.1*W):int(0.9*W), int(0.1*H):int(0.9*H)] = 1
remove_percent = 0.1
final_mask_binary[int(remove_percent * W):int((1 - remove_percent) *
W), :] = 1
final_mask_binary[:,
int(remove_percent * H):int((1 - remove_percent) *
H)] = 1
final = np.multiply(img_1st, final_mask_binary)
if is_draw:
h1 = stack_4_images_horizontally(img, th, img_removed_text,
img_filled_holes)
h2 = stack_4_images_horizontally(img_removed_small, img_1st,
final_mask_binary * 255, final)
numpy_vertical = np.vstack((h1, h2))
h3 = stack_2_images_horizontally(img, final)
cv2.imshow(path_utils.get_filename_without_extension(img_path),
numpy_vertical)
cv2.imshow("Done", numpy_vertical)
# cv2.imshow('Done', h3)
cv2.waitKey(0)
return final
def read_svs(svs_path, patch_size, is_save=False):
scan = openslide.OpenSlide(svs_path)
filename_without_ext = path_utils.get_filename_without_extension(svs_path)
parent_path = path_utils.get_parent_dir(svs_path)
npy_path = "{}/{}.npy".format(parent_path, filename_without_ext)
if os.path.exists(npy_path):
print(">> loading", npy_path)
img_np = np.load(npy_path)
else:
orig_w = np.int(scan.properties.get('aperio.OriginalWidth'))
orig_h = np.int(scan.properties.get('aperio.OriginalHeight'))
# create an array to store our image
img_np = np.zeros((orig_w, orig_h, 3), dtype=np.uint8)
print(">> reading", svs_path)
for r in tqdm(range(0, orig_w, patch_size[0])):
for c in range(0, orig_h, patch_size[1]):
if c + patch_size[1] > orig_h and r + patch_size[0] <= orig_w:
p = orig_h - c
img = np.array(scan.read_region((c, r), 0,
(p, patch_size[1])),
dtype=np.uint8)[..., 0:3]
elif c + patch_size[1] <= orig_h and r + patch_size[0] > orig_w:
p = orig_w - r
img = np.array(scan.read_region((c, r), 0,
(patch_size[0], p)),
dtype=np.uint8)[..., 0:3]
elif c + patch_size[1] > orig_h and r + patch_size[0] > orig_w:
p = orig_h - c
pp = orig_w - r
img = np.array(scan.read_region((c, r), 0, (p, pp)),
dtype=np.uint8)[..., 0:3]
else:
img = np.array(scan.read_region(
(c, r), 0, (patch_size[0], patch_size[1])),
dtype=np.uint8)[..., 0:3]
img_np[r:r + patch_size[0], c:c + patch_size[1]] = img
scan.close
if is_save:
np.save(npy_path, img_np)
return img_np
def main():
for dataset in [
# "breast",
# "idrid",
# "tools",
"vqa",
"vqa2",
"med"
]:
folders = glob.glob(os.path.join(LOGS_DIR, dataset, "*"))
df_path = os.path.join(LOGS_DIR, dataset, 'compile.csv')
folders = [x for x in folders if "trainval" not in x]
df = pd.DataFrame(columns=['name', 'dataset', 'method', 'image', 'question',
'dim_h', 'nb_glimpses', 'activation', 'epoch_max', 'mean', 'std'])
for folder in folders:
mean, std, to = process_one_method_one_dataset(folder)
# print("{}: \t {:.2f}({:.2f})".format(
# path_utils.get_filename_without_extension(folder), mean, std))
if "bilinear_att_train_imagenet_h200_g4_relu" in folder:
a = 2
info = get_info(path_utils.get_filename_without_extension(folder))
# if to > 30:
df = df.append(pd.DataFrame({'name': [info[0]],
'dataset': [dataset],
'method': [info[1]],
'image': [info[2]],
'question': [info[3]],
'dim_h': [info[4]],
'nb_glimpses': [info[5]],
'activation': [info[6]],
'epoch_max': [to],
'mean': [mean.round(2)],
'std': [std.round(2)]
}),
ignore_index=True)
print(df)
df.to_csv(df_path, index=False)
def process_occlusion(path, dataset="breast"):
# global args
args = parser.parse_args()
LIST_QUESTION_BREAST = [
"how many classes are there",
"is normal larger than benign",
]
LIST_QUESTION_TOOLS = [
"is grasper in 0_0_32_32 location",
# "which tool has pointed tip on the left of the image",
# "how many tools are there",
]
LIST_QUESTION_IDRID = [
"is there haemorrhages in the fundus",
"is there soft exudates in the fundus",
]
if dataset == "breast":
list_question = LIST_QUESTION_BREAST
elif dataset == "tools":
list_question = LIST_QUESTION_TOOLS
elif dataset == "idrid":
list_question = LIST_QUESTION_IDRID
img_dirs = glob.glob(os.path.join(path, "*"))
args = update_args(
args, vqa_model="minhmul_att_train_2048", dataset=dataset)
for question_str in list_question:
for path_img in img_dirs:
print(
"\n\n=========================================================================")
print("{} - {}".format(question_str, path_img))
ans_gt = get_answer(dataset, path_img, question_str)
if ans_gt is None:
continue
else:
input_size = 256
step = 2
windows_size = 32
dst_dir = "temp/occlusion"
paths_utils.make_dir(dst_dir)
out_color_path = "{}/{}_{}_w_{:0}_s_{:0}_color.jpg".format(dst_dir,
paths_utils.get_filename_without_extension(
path_img),
question_str.replace(
' ', '_'),
windows_size,
step
)
out_gray_path = "{}/{}_{}_w_{:0}_s_{:0}_gray.jpg".format(dst_dir,
paths_utils.get_filename_without_extension(
path_img),
question_str.replace(
' ', '_'),
windows_size,
step
)
out_avg_path = "{}/{}_{}_w_{:0}_s_{:0}_avg.jpg".format(dst_dir,
paths_utils.get_filename_without_extension(
path_img),
question_str.replace(
' ', '_'),
windows_size,
step
)
if not os.path.exists(out_color_path):
visual_PIL = Image.open(path_img)
visual_PIL = visual_PIL.resize((input_size,input_size), Image.ANTIALIAS)
indices = np.asarray(
np.mgrid[0:input_size-windows_size+1:step, 0:input_size-windows_size+1:step].reshape(2, -1).T, dtype=np.int)
cnn, model, trainset = initialize(args, dataset=dataset)
image_occlusion = np.zeros((input_size, input_size))
image_occlusion_times = np.zeros((input_size, input_size))
anw_without_black_patch = process_one_batch_of_occlusion(args,
cnn,
model,
trainset,
visual_PIL,
question_str,
box=None,
dataset=dataset)
score_without_black_patch = anw_without_black_patch.get("val")[
anw_without_black_patch.get("ans").index(ans_gt)]
for i in range(indices.shape[0]):
print_utils.print_tqdm(i, indices.shape[0])
box = [indices[i][0], indices[i][0]+windows_size -
1, indices[i][1], indices[i][1]+windows_size-1]
# print(box)
ans = process_one_batch_of_occlusion(args,
cnn,
model,
trainset,
visual_PIL,
question_str,
box,
dataset=dataset,
is_print=False)
try:
score = ans.get("val")[
ans.get("ans").index(ans_gt)]
except:
score = 0
if score != 0:
score_occ = (
score_without_black_patch.item()-score.item())/score_without_black_patch.item()
image_occlusion[box[0]:box[1],
box[2]:box[3]] += score_occ
image_occlusion_times[box[0]:box[1],
box[2]:box[3]] += 1
save_image(visual_PIL, image_occlusion, image_occlusion_times,
out_color_path, out_gray_path, out_avg_path,
is_show=True)
import os
import glob
os.environ['PATH'] = "C:/Users/minhm/Documents/GitHub/bin/openslide-win64-20171122/bin" + ";" + os.environ['PATH']
import openslide
import datasets.utils.paths_utils as path_utils
import datasets.utils.xml_utils as xml_utils
import datasets.utils.image_utils as image_utils
if __name__=='__main__':
folder_name = 'C:/Users/minhm/Documents/GitHub/vqa_idrid/data/raw/breast-cancer/ICIAR2018_BACH_Challenge/WSI/' #path to the dataset folder
img_dirs = glob.glob(os.path.join(folder_name, "*.xml"))
store = []
for img_dir in img_dirs:
file_name = path_utils.get_filename_without_extension(img_dir)
print('Reading scan',file_name)
scan = openslide.OpenSlide(folder_name+file_name+'.svs')
dims = scan.dimensions
print('Generating thumbnail')
coords,labels,length,area,pixel_spacing = xml_utils.read_xml_breast(img_dir)
store += [[coords,labels,length,area,pixel_spacing]]
gt = image_utils.generate_groundtruth_from_xml(folder_name+file_name+'.png', dims, coords, labels, is_debug=False)
def process_occlusion(path, dataset="breast"):
# global args
args = parser.parse_args()
LIST_QUESTION_BREAST = [
"how many classes are there",
"is there any benign class in the image",
"is there any in situ class in the image",
"is there any invasive class in the image",
"what is the major class in the image",
"what is the minor class in the image",
"is benign in 64_64_32_32 location",
"is invasive in 96_96_32_32 location",
]
LIST_QUESTION_TOOLS = [
"how many tools are there",
"is scissors in 64_32_32_32 location",
"is irrigator in 64_96_32_32 location",
"is grasper in 64_96_32_32 location"
"is bipolar in 64_96_32_32 location"
"is hook in 64_96_32_32 location"
"is clipper in 64_96_32_32 location"
"is specimenbag in 64_96_32_32 location"
"is there any grasper in the image",
"is there any bipolar in the image",
"is there any hook in the image",
"is there any scissors in the image",
"is there any clipper in the image",
"is there any irrigator in the image",
"is there any specimenbag in the image",
]
LIST_QUESTION_IDRID = [
"is there haemorrhages in the fundus",
"is there microaneurysms in the fundus",
"is there soft exudates in the fundus",
"is there hard exudates in the fundus",
"is hard exudates larger than soft exudates",
"is haemorrhages smaller than microaneurysms",
"is there haemorrhages in the region 32_32_16_16",
"is there microaneurysms in the region 96_96_16_16",
]
LIST_QUESTION_VQA2 = [
"what color is the hydrant",
"why are the men jumping to catch",
"is the water still",
"how many people are in the image"
]
if dataset == "breast":
list_question = LIST_QUESTION_BREAST
elif dataset == "tools":
list_question = LIST_QUESTION_TOOLS
elif dataset == "idrid":
list_question = LIST_QUESTION_IDRID
elif dataset == "vqa2":
list_question = LIST_QUESTION_VQA2
img_dirs = glob.glob(os.path.join(path, "*"))
# args = update_args(
# args, vqa_model="minhmul_att_train_2048", dataset=dataset)
args = update_args(
args, vqa_model="minhmul_att_train", dataset=dataset)
shuffle(img_dirs)
shuffle(list_question)
# for (path_img, question_str) in zip(img_dirs, list_question):
for path_img in img_dirs:
for question_str in list_question:
if dataset in ["vqa", "vqa2"]:
if not ((question_str == "what color is the hydrant" and ("img1" in path_img or "img2" in path_img)) or
(question_str == "why are the men jumping to catch" and ("img3" in path_img or "img4" in path_img)) or
(question_str == "is the water still" and ("img5" in path_img or "img6" in path_img)) or
(question_str == "how many people are in the image" and ("img7" in path_img or "img8" in path_img))):
continue
print(
"\n\n=========================================================================")
print("{} - {}".format(question_str, path_img))
if dataset == "vqa2":
ans_gt = "red"
else:
ans_gt = get_answer(dataset, path_img, question_str)
if ans_gt is None:
continue
else:
input_size = 256
step = 2
windows_size = 32
dst_dir = "temp/occlusion"
paths_utils.make_dir(dst_dir)
out_color_path = "{}/{}_{}_w_{:0}_s_{:0}_color.jpg".format(dst_dir,
paths_utils.get_filename_without_extension(
path_img),
question_str.replace(
' ', '_'),
windows_size,
step
)
out_gray_path = "{}/{}_{}_w_{:0}_s_{:0}_gray.jpg".format(dst_dir,
paths_utils.get_filename_without_extension(
path_img),
question_str.replace(
' ', '_'),
windows_size,
step
)
out_avg_path = "{}/{}_{}_w_{:0}_s_{:0}_avg.jpg".format(dst_dir,
paths_utils.get_filename_without_extension(
path_img),
question_str.replace(
' ', '_'),
windows_size,
step
)
if not os.path.exists(out_color_path):
visual_PIL = Image.open(path_img)
indices = np.asarray(
np.mgrid[0:input_size-windows_size+1:step, 0:input_size-windows_size+1:step].reshape(2, -1).T, dtype=np.int)
cnn, model, trainset = initialize(args, dataset=dataset)
# cnn, model, trainset = None, None, None
image_occlusion = np.zeros((input_size, input_size))
image_occlusion_times = np.zeros((input_size, input_size))
ans_without_black_patch = process_one_batch_of_occlusion(args,
cnn,
model,
trainset,
visual_PIL,
question_str,
list_box=None,
dataset=dataset)
try:
score_without_black_patch = ans_without_black_patch[0].get("val")[
ans_without_black_patch[0].get("ans").index(ans_gt)]
except:
score_without_black_patch = torch.tensor(0)
batch = 32
count = 0
list_box = []
for i in range(indices.shape[0]):
print_utils.print_tqdm(i, indices.shape[0])
list_box.append([indices[i][0], indices[i][0]+windows_size -
1, indices[i][1], indices[i][1]+windows_size-1])
count += 1
# if count == batch or i == indices.shape[0] - 1:
if count == batch:
# print(count)
ans = process_one_batch_of_occlusion(args,
cnn,
model,
trainset,
visual_PIL,
question_str,
list_box,
dataset=dataset,
is_print=False)
for i in range(len(list_box)):
try:
score = ans[i].get("val")[
ans[i].get("ans").index(ans_gt)]
except:
score = 0
box = list_box[i]
if score != 0:
try:
score_occ = (
score.item() - score_without_black_patch.item())/score_without_black_patch.item()
except:
score_occ = 0
image_occlusion[box[0]:box[1],
box[2]:box[3]] += score_occ
image_occlusion_times[box[0]:box[1],
box[2]:box[3]] += 1
count = 0
list_box = []
save_image(visual_PIL, image_occlusion, image_occlusion_times,
out_color_path, out_gray_path, out_avg_path,
is_show=False)
def get_gradcam_from_image_model(path_img, cnn, dataset, finalconv_name="layer4"):
cnn.eval()
# hook the feature extractor
features_blobs = []
def hook_feature(module, input, output):
features_blobs.append(output.data.cpu().numpy())
cnn._modules.get(finalconv_name).register_forward_hook(hook_feature)
# get the softmax weight
params = list(cnn.parameters())
weight_softmax = np.squeeze(params[-2].data.cpu().numpy())
normalize = transforms.Normalize(
mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225]
)
preprocess = transforms.Compose([
transforms.Resize((224, 224)),
transforms.ToTensor(),
normalize
])
img_name = paths_utils.get_filename_without_extension(path_img)
img_pil = Image.open(path_img)
img_tensor = preprocess(img_pil)
img_variable = Variable(img_tensor.unsqueeze(0))
img_variable = img_variable.cuda(async=True)
logit = cnn(img_variable)
paths_utils.make_dir("temp/gradcam/{}/".format(dataset))
in_path = "temp/gradcam/{}/{}_in.jpg".format(dataset, img_name)
# img_pil.thumbnail((256, 256), Image.ANTIALIAS)
img_pil = img_pil.resize((256, 256), resample=PIL.Image.NEAREST)
img_pil.save(in_path)
# download the imagenet category list
classes = {
0: "Benign",
1: "InSitu",
2: "Invasive",
3: "Normal"
}
h_x = F.softmax(logit, dim=1).data.squeeze()
probs, idx = h_x.sort(0, True)
probs = probs.cpu().numpy()
idx = idx.cpu().numpy()
# generate class activation mapping for the top1 prediction
cam = get_gadcam_image(features_blobs[0], weight_softmax, [idx[0]])
img_name = paths_utils.get_filename_without_extension(path_img)
img = cv2.imread(in_path)
result = show_cam_on_image(img, cam)
out_path = "temp/gradcam/{}/{}_cnn.jpg".format(dataset,
img_name)
cv2.imwrite(out_path, result)
return result, out_path, features_blobs
def get_gradcam_from_vqa_model(visual_features,
question_features,
features_blobs_visual,
ans,
path_img,
cnn,
model,
question_str,
dataset,
vqa_model="minhmul_noatt_train_2048",
finalconv_name="linear_classif",
is_show_image=False):
model.eval()
# hook the feature extractor
features_blobs = []
def hook_feature(module, input, output):
features_blobs.append(output.data.cpu().numpy())
model._modules.get(finalconv_name).register_forward_hook(hook_feature)
# get the softmax weight
params = list(model.parameters())
weight_softmax = np.squeeze(params[-2].data.cpu().numpy())
if "noatt" in vqa_model:
classif_w_params = np.squeeze(params[10].data.cpu().numpy())
classif_b_params = np.squeeze(params[11].data.cpu().numpy())
else:
classif_w_params = np.squeeze(params[26].data.cpu().numpy())
temp_classif_w_params = np.zeros((classif_w_params.shape[0], 2048))
temp_classif_w_params = (classif_w_params[:, 0:2048] + classif_w_params[:, 2048:2048*2] +
classif_w_params[:, 2048*2:2048*3] + classif_w_params[:, 2048*3:2048*4]/4)
classif_w_params = temp_classif_w_params
classif_b_params = np.squeeze(params[27].data.cpu().numpy())
logit = model(visual_features, question_features)
h_x = F.softmax(logit, dim=1).data.squeeze()
probs, idx = h_x.sort(0, True)
probs = probs.cpu().numpy()
idx = idx.cpu().numpy()
cam = get_gadcam_vqa(features_blobs_visual[0],
classif_w_params, classif_b_params, [idx[0]])
# render the CAM and output
# print('output CAM.jpg for the top1 prediction: %s' % ans["ans"][idx[0]])
img_name = paths_utils.get_filename_without_extension(path_img)
in_path = "temp/gradcam/{}/{}_in.jpg".format(dataset, img_name)
img = cv2.imread(in_path)
result = show_cam_on_image(img, cam)
question_str = question_str.replace(' ', '-')
paths_utils.make_dir("temp/gradcam/{}/".format(dataset))
if "noatt" in vqa_model:
out_path = "temp/gradcam/{}/{}_noatt_question_{}.jpg".format(dataset,
img_name,
question_str)
else:
out_path = "temp/gradcam/{}/{}_att_question_{}.jpg".format(dataset,
img_name,
question_str)
cv2.imwrite(out_path, result)
im_out = Image.open(out_path)
if is_show_image:
im_out.show()
return logit
def main():
folders = glob.glob(os.path.join(LOGS_DIR, "*"))
df = pd.read_csv(PROCESSED_QA_PER_QUESTION_PATH)
with open(TEST_GT, encoding='UTF-8') as f:
lines = f.readlines()
list_epoch = list()
for method in ["best"]:
# for method in []:
sub_path = SUB_DIR + method + ".txt"
sub = DICT_METHOD[method]
method_folders = []
for item in sub:
for folder in folders:
if item == path_utils.get_filename_without_extension(folder):
method_folders.append(folder)
method_folders = list(set(method_folders))
best_acc = 0
for folder in method_folders:
try:
print('>> processing {} of {}'.format(folder, method))
weight = 1 if "bert" in folder else 5
for epoch in range(1, 100):
count = 0
correct = 0
dict_score = {}
# dict_score = get_top1_ans(dict_score, folder, df, weight=weight, fr=90, to=99)
dict_score = get_top1_ans(dict_score,
folder,
df,
weight=weight,
fr=epoch,
to=epoch)
for line in lines:
line = line.split("|")
image, qtype, question, answer = line[0], line[
1], line[2], line[3].split("\n")[0]
question = question.encode('ascii',
'ignore').decode('ascii')
answer = answer.encode('ascii',
'ignore').decode('ascii')
file_id = image
dict_ans_score, final_answer, second_answer = get_final_answer(
dict_score, file_id)
count += 1
if answer == final_answer:
correct += 1
# else:
# print ("{} / {}".format(answer, final_answer))
# if answer == second_answer:
# correct += 1
acc = round(correct / count * 100, 2)
print("Method: {} \t | Epoch: {} \t | Acc: {}".format(
folder, epoch, acc))
# print("Method: {} \t | Acc: {}".format(folder, acc))
except:
pass
def preprocess_dataset(dataset="train", is_show=False, is_overwrite=False, is_augment=False):
if dataset == "train":
dataset_dir = DATASETS_TRAIN_DIR
elif dataset == "val":
dataset_dir = DATASETS_VALID_DIR
else:
dataset_dir = DATASETS_TEST_DIR
if is_augment:
preprocessed_dir = os.path.join(
PREPROCESSED_DIR, "raw", dataset + "_augment")
else:
preprocessed_dir = os.path.join(PREPROCESSED_DIR, "raw", dataset)
if is_overwrite or not os.path.exists(preprocessed_dir):
path_utils.make_dir(preprocessed_dir)
img_paths = glob.glob(os.path.join(dataset_dir, "*.jpg"))
for index in tqdm(range(len(img_paths))):
img_preprocessed = preprocess.process(index, img_paths)
img_preprocessed = cv2.resize(img_preprocessed, (256, 256))
if is_show:
cv2.imshow('Done', img_preprocessed)
cv2.waitKey(0)
out_path = os.path.join(
preprocessed_dir, path_utils.get_filename(img_paths[index]))
cv2.imwrite(out_path, img_preprocessed)
out = img_preprocessed
if is_augment:
for augment in LIST_AUGMENT:
out_path = os.path.join(
preprocessed_dir, path_utils.get_filename_without_extension(img_paths[index]) + "_{}.jpg".format(augment))
img_preprocessed = Image.fromarray(out)
if augment == "fliplr":
img_preprocessed = img_preprocessed.transpose(
Image.FLIP_LEFT_RIGHT)
elif augment == "rot10":
img_preprocessed = img_preprocessed.rotate(10)
elif augment == "rot20":
img_preprocessed = img_preprocessed.rotate(20)
elif augment == "rot30":
img_preprocessed = img_preprocessed.rotate(30)
elif augment == "rot_10":
img_preprocessed = img_preprocessed.rotate(-10)
elif augment == "rot_20":
img_preprocessed = img_preprocessed.rotate(-20)
elif augment == "rot_30":
img_preprocessed = img_preprocessed.rotate(-30)
elif augment == "bright1":
img_preprocessed = ie.Contrast(
img_preprocessed).enhance(1)
elif augment == "bright_1":
img_preprocessed = ie.Contrast(
img_preprocessed).enhance(-1)
elif augment == "sharp3":
img_preprocessed = ie.Sharpness(
img_preprocessed).enhance(3)
elif augment == "sharp_3":
img_preprocessed = ie.Sharpness(
img_preprocessed).enhance(-3)
# elif augment == "contrast2":
# img_preprocessed = ie.Contrast(img_preprocessed).enhance(2)
# elif augment == "contrast_2":
# img_preprocessed = ie.Contrast(img_preprocessed).enhance(-2)
img_preprocessed.save(out_path)
def get_gradcam_from_vqa_model(visual_features,
question_features,
features_blobs_visual,
ans,
path_img,
cnn,
model,
question_str,
dataset,
vqa_model="minhmul_noatt_train_2048",
finalconv_name="linear_classif",
is_show_image=False,
is_att=True):
if is_att:
logit, list_v_record = model(visual_features, question_features)
cam = get_gadcam_vqa_new(list_v_record)
else:
# grad_cam = gradcam_utils.GradCam(model=model,
# target_layer_names=["linear_v"], use_cuda=True)
# target_index = None
# mask = grad_cam(visual_features,
# question_features,
# target_index)
# hook the feature extractor
features_blobs = []
def hook_feature(module, input, output):
features_blobs.append(output.data.cpu().numpy())
model._modules.get(finalconv_name).register_forward_hook(hook_feature)
# model.fusion.linear_v.register_forward_hook(hook_feature)
# model.fusion.linear_v.register_backward_hook(hook_feature)
# get the softmax weight
params = list(model.parameters())
weight_softmax = np.squeeze(params[-2].data.cpu().numpy())
if "noatt" in vqa_model:
classif_w_params = np.squeeze(params[10].data.cpu().numpy())
classif_b_params = np.squeeze(params[11].data.cpu().numpy())
logit = model(visual_features, question_features)
h_x = F.softmax(logit, dim=1).data.squeeze()
probs, idx = h_x.sort(0, True)
probs = probs.cpu().numpy()
idx = idx.cpu().numpy()
cam = get_gadcam_vqa(features_blobs_visual[0], classif_w_params,
classif_b_params, [idx[0]])
img_name = paths_utils.get_filename_without_extension(path_img)
in_path = "temp/gradcam/{}/{}_in.jpg".format(dataset, img_name)
img = cv2.imread(in_path)
result = show_cam_on_image(img, cam)
question_str = question_str.replace(' ', '-')
paths_utils.make_dir("temp/gradcam/{}/".format(dataset))
if "noatt" in vqa_model:
out_path = "temp/gradcam/{}/{}_noatt_question_{}.jpg".format(
dataset, img_name, question_str)
else:
out_path = "temp/gradcam/{}/{}_att_question_{}.jpg".format(
dataset, img_name, question_str)
cv2.imwrite(out_path, result)
im_out = Image.open(out_path)
if is_show_image:
im_out.show()
return logit
