def dr_seg(model1,
filename_source,
tmp_dir='/tmp',
image_size=384,
single_channel_no=None):
image_shape = (image_size, image_size)
image1 = my_preprocess.do_preprocess(filename_source, image_size)
filename_preprocess = os.path.join(tmp_dir, str(uuid.uuid1()) + '.jpg')
cv2.imwrite(filename_preprocess, image1)
img_preprocess_seg = cv2.imread(filename_preprocess) # (384,384,3)
train_image_files = []
train_image_files.append(filename_preprocess)
gen1 = my_Generator_seg_test(train_image_files,
image_shape=image_shape,
single_channel_no=single_channel_no)
x = gen1.__next__()
pred_seg = model1.predict_on_batch(
x) #1,384,384,1 batch,height,width,channels
img_pred_0_1 = pred_seg[0] > 0.5
img_pred_0_1 = img_pred_0_1.astype(np.int8)
img_pred_seg = img_pred_0_1 * 255
return img_preprocess_seg, img_pred_seg
def server_shap_deep_explain(model_no,
img_source,
preprocess=True,
ranked_outputs=1,
blend_original_image=False):
image_shape = dicts_models[model_no]['input_shape']
if isinstance(img_source, str):
if preprocess:
img_preprocess = my_preprocess.do_preprocess(img_source,
crop_size=384)
img_input = LIBS.ImgPreprocess.my_image_helper.my_gen_img_tensor(
img_preprocess, image_shape=image_shape)
else:
img_input = LIBS.ImgPreprocess.my_image_helper.my_gen_img_tensor(
img_source, image_shape=image_shape)
else:
img_input = img_source
list_classes, list_images = shap_deep_explain(
dicts_models=dicts_models,
model_no=model_no,
e_list=e_list,
num_reference=NUM_REFERENCE,
img_source=img_input,
preprocess=False,
ranked_outputs=ranked_outputs,
blend_original_image=blend_original_image,
base_dir_save=dir_tmp)
return list_classes, list_images
def process_img(img_source,
img_file_dest,
crop_size,
add_black_pixel_ratio,
SMALL_SIZE=2048):
try:
size = os.path.getsize(img_source)
if size < SMALL_SIZE:
print('error:', img_source)
return None
img = cv2.imread(img_source)
except:
# Corrupt JPEG data1: 19 extraneous bytes before marker 0xc4
print('error:', img_source)
return None
if img is not None:
image1 = my_preprocess.do_preprocess(
img,
crop_size=crop_size,
add_black_pixel_ratio=add_black_pixel_ratio)
if image1 is not None:
cv2.imwrite(img_file_dest, image1)
print(img_file_dest)
else:
print('error:', img_source) # file error
else: # file not exists or other errors
print('error:', img_source)
def server_grad_cam(model_no,
img_source,
pred,
preprocess=True,
blend_original_image=True):
model = dicts_models[model_no]['model_original']
image_size = dicts_models[model_no]['image_size']
if preprocess:
img_preprocess = my_preprocess.do_preprocess(img_source, crop_size=384)
img_input = LIBS.ImgPreprocess.my_image_helper.my_gen_img_tensor(
img_preprocess, image_shape=(image_size, image_size, 3))
else:
img_input = LIBS.ImgPreprocess.my_image_helper.my_gen_img_tensor(
img_source, image_shape=(image_size, image_size, 3))
penultimate_layer = get_last_conv_layer_number(model)
layer_idx = len(model.layers) - 1
modifier = 'guided' # [None, 'guided', 'relu']
# too slow
grads = visualize_cam(model,
layer_idx,
filter_indices=[pred],
seed_input=img_input,
penultimate_layer_idx=penultimate_layer,
backprop_modifier=modifier)
cam = cv2.applyColorMap(np.uint8(255 * grads), cv2.COLORMAP_JET)
if blend_original_image:
# Return to BGR [0..255] from the preprocessed image
image_original = img_input[0, :]
from LIBS.ImgPreprocess.my_image_norm import input_norm_reverse
image_original = input_norm_reverse(image_original)
image_original = image_original.astype(np.uint8)
image_original -= np.min(image_original)
image_original = np.minimum(image_original, 255)
cam = np.float32(cam) + np.float32(image_original)
cam = 255 * cam / np.max(cam)
# 传过来的是web目录
str_uuid = str(uuid.uuid1())
filename_CAM = os.path.join(BASE_DIR_SAVE, str_uuid,
'Grad_CAM{}.jpg'.format(pred))
if not os.path.exists(os.path.dirname(filename_CAM)):
os.makedirs(os.path.dirname(filename_CAM))
cv2.imwrite(filename_CAM, cam)
return filename_CAM
def predict_sigmoids(img1, preproess=False, neglect_class0=False):
if preproess:
img1 = my_preprocess.do_preprocess(img1, 512)
prob_np = []
prob = []
pred = []
for dict1 in dicts_models:
# real time image aug
img_tensor = LIBS.ImgPreprocess.my_image_helper.my_gen_img_tensor(
img1, image_shape=(dict1['image_size'], dict1['image_size'], 3))
prob1 = dict1['model'].predict_on_batch(img_tensor)
prob1 = np.mean(prob1, axis=0) # batch mean, test time img aug
if neglect_class0:
prob1[0] = 0 #do not use class0, if all classes are negative ...
pred_classes1 = []
for i in range(NUM_CLASSES):
if prob1[i] > list_threthold[i]:
pred_classes1.append(1)
else:
pred_classes1.append(0)
prob_np.append(prob1)
prob.append(prob1.tolist()) #担心XMLRPC numpy
pred.append(pred_classes1)
list_weights = [] #the prediction weights of models
for dict1 in dicts_models:
list_weights.append(dict1['model_weight'])
prob_total = np.average(prob_np, axis=0, weights=list_weights)
pred_total = []
for j in range(NUM_CLASSES):
if prob_total[j] > list_threthold[j]:
pred_total.append(1)
else:
pred_total.append(0)
prob_total = prob_total.tolist() #RPC parameter do not use Numpy
correct_model_no = [] #采用那个模型热力图
for i in range(NUM_CLASSES):
for j in range(len(dicts_models)):
# model j, class i
if pred[j][i] == pred_total[i]:
correct_model_no.append(j)
break
return prob, pred, prob_total, pred_total, correct_model_no
def server_gradcam_plusplus(model_no,
img_source,
pred,
preprocess=True,
blend_original_image=True):
image_size = dicts_models[model_no]['image_size']
if preprocess:
img_preprocess = my_preprocess.do_preprocess(img_source, crop_size=384)
img_input = LIBS.ImgPreprocess.my_image_helper.my_gen_img_tensor(
img_preprocess, image_shape=(image_size, image_size, 3))
else:
img_input = LIBS.ImgPreprocess.my_image_helper.my_gen_img_tensor(
img_source, image_shape=(image_size, image_size, 3))
gradcamplus = grad_cam_plus(dicts_models[model_no]['model_original'],
img_input,
pred=pred,
image_size=image_size)
cam = cv2.applyColorMap(np.uint8(255 * gradcamplus), cv2.COLORMAP_JET)
if blend_original_image:
# Return to BGR [0..255] from the preprocessed image
image_original = img_input[0, :]
from LIBS.ImgPreprocess.my_image_norm import input_norm_reverse
image_original = input_norm_reverse(image_original)
image_original = image_original.astype(np.uint8)
image_original -= np.min(image_original)
image_original = np.minimum(image_original, 255)
cam = np.float32(cam) + np.float32(image_original)
cam = 255 * cam / np.max(cam)
#region 将CAM保存到文件
# 传过来的是web目录
str_uuid = str(uuid.uuid1())
filename_CAM = os.path.join(BASE_DIR_SAVE, str_uuid,
'GradCAM_PlusPlus{}.jpg'.format(pred))
if not os.path.exists(os.path.dirname(filename_CAM)):
os.makedirs(os.path.dirname(filename_CAM))
cv2.imwrite(filename_CAM, cam)
return filename_CAM
def predict_multi_labels(img1, preproess=False):
if preproess:
img1 = my_preprocess.do_preprocess(img1, 512)
prob_list_np = []
prob_list = []
pred_list_classes = []
# 调用每一个模型预测概率和所属类
list_weights = []
for dict1 in models:
list_weights.append(dict1['model_weight'])
x = LIBS.ImgPreprocess.my_image_helper.my_gen_img_tensor(
img1, image_shape=(dict1['image_size'], dict1['image_size']))
prob1, pred_classes1 = predict_one_model_multilabels(dict1, x)
prob_list_np.append(prob1)
prob_list.append(prob1.tolist()) #担心XMLRPC numpy
pred_list_classes.append(pred_classes1)
# 计算多模型合并的该率
for i, prob1 in enumerate(prob_list_np):
if i == 0:
prob_total = prob1 * list_weights[i]
else:
prob_total = prob_total + prob1 * list_weights[i]
prob_total = prob_total / len(prob_list_np)
pred_list_classes_total = []
for j in range(NUM_CLASSES):
if prob_total[j] > list_threthold[j]:
pred_list_classes_total.append(1)
else:
pred_list_classes_total.append(0)
prob_total = prob_total.tolist()
correct_model_no = [] #采用那个模型热力图
for i in range(NUM_CLASSES):
for j in range(len(models)):
# 第j个模型，第个类别
if pred_list_classes[j][i] == pred_list_classes_total[i]:
correct_model_no.append(j)
break
return prob_list, pred_list_classes, prob_total, pred_list_classes_total, correct_model_no
def predict_softmax(img1, preproess=False):
if preproess:
img1 = my_preprocess.do_preprocess(img1, 512)
prob_list_np = []
prob_list = []
pred_list = [
] # only one class, multi-class multi-label has multiple classes
# 调用每一个模型预测概率和所属类
list_weights = []
for dict1 in models:
list_weights.append(dict1['model_weight'])
x = LIBS.ImgPreprocess.my_image_helper.my_gen_img_tensor(
img1, image_shape=(dict1['image_size'], dict1['image_size']))
prob1, pred1 = predict_one_model_softmax(dict1, x, argmax=True)
prob_list_np.append(prob1)
prob_list.append(prob1.tolist()) #担心XMLRPC numpy
pred_list.append(pred1)
# 计算多模型合并的该率
for i, prob1 in enumerate(prob_list_np):
if i == 0:
prob_total = prob1 * list_weights[i]
else:
prob_total = prob_total + prob1 * list_weights[i]
prob_total = prob_total / len(prob_list_np)
# 多模型平均的预测类
pred_total = prob_total.argmax(axis=-1)
# 'numpy.int64' 转换为 标准类型，担心XMLRPC 传递
pred_total = int(pred_total)
correct_model_no = 0 # 每一个模型预测的类别，热力图 选择哪个模型需要
for i, pred1 in enumerate(pred_list):
if pred1 == pred_total:
correct_model_no = i #start from 0
break
prob_total = prob_total.tolist()
return prob_list, pred_list, prob_total, pred_total, correct_model_no
def get_img_process(filename_source, image_size=384):
image_shape = (image_size, image_size)
# preprocess don't add black
image1 = my_preprocess.do_preprocess(filename_source,
image_size,
add_black_pixel_ratio=0)
filename_preprocess = os.path.join(file_save_path,
str(uuid.uuid1()) + '.jpg')
cv2.imwrite(filename_preprocess, image1)
img_preprocess_seg = cv2.imread(filename_preprocess) # (384,384,3)
train_image_files = []
train_image_files.append(filename_preprocess)
gen1 = my_images_generator.my_Generator_seg_test(
list_images=train_image_files, image_shape=image_shape)
x = gen1.__next__()
return img_preprocess_seg, x
def predict_softmax(img1, preproess=False):
if preproess:
img1 = my_preprocess.do_preprocess(img1, 512)
prob_np = []
prob = []
pred = [
] # only one class, multi_models multi-class multi-label has multiple classes
for dict1 in dicts_models:
#real time image aug
img_tensor = LIBS.ImgPreprocess.my_image_helper.my_gen_img_tensor(
img1, image_shape=(dict1['image_size'], dict1['image_size'], 3))
prob1 = dict1['model'].predict_on_batch(img_tensor)
prob1 = np.mean(prob1, axis=0) # batch mean, test time img aug
pred1 = prob1.argmax(axis=-1)
prob_np.append(prob1) # numpy weight avg prob_total
prob.append(prob1.tolist()) #担心XMLRPC numpy
pred.append(int(pred1)) # numpy int64, int XMLRPC
list_weights = [] # the prediction weights of models
for dict1 in dicts_models:
list_weights.append(dict1['model_weight'])
prob_total = np.average(prob_np, axis=0, weights=list_weights)
pred_total = prob_total.argmax(axis=-1)
prob_total = prob_total.tolist() #RPC Service can not pass numpy variable
pred_total = int(pred_total) # 'numpy.int64' XMLRPC
# correct_model_no is used for choosing which model to generate CAM
# on extreme condition: average softmax prediction class is not in every model's prediction class
correct_model_no = 0
for i, pred1 in enumerate(pred):
if pred1 == pred_total:
correct_model_no = i #start from 0
break
return prob, pred, prob_total, pred_total, correct_model_no
def server_shap_deep_explain(dicts_models, model_no, img_source, preprocess=True,
ranked_outputs=1):
str_uuid = str(uuid.uuid1())
if preprocess:
img_preprocess = my_preprocess.do_preprocess(img_source, crop_size=384, tadd_black_pixel_ratio=0)
img_preprocess = my_images_generator.my_Generator_test_onetime(img_preprocess,
image_shape=(image_size, image_size, 3))
else:
img_preprocess = LIBS.ImgPreprocess.my_image_helper.my_gen_img_tensor(img_source,
image_shape=(image_size, image_size, 3))
#mini-batch
shap_values1 = e_list1[model_no].shap_values(img_preprocess, ranked_outputs=ranked_outputs)
shap_values2 = e_list2[model_no].shap_values(img_preprocess, ranked_outputs=ranked_outputs)
shap_values = copy.deepcopy(shap_values1)
for i in range(ranked_outputs):
shap_values[0][i] = (shap_values1[0][i] + shap_values2[0][i])/2
list_classes = []
list_images = []
for i in range(ranked_outputs):
#numpy int 64 - int
list_classes.append(int(shap_values[1][0][i]))
save_filename = os.path.join(BASE_DIR_SAVE, str_uuid,
'Shap_Deep_Explain{}.jpg'.format(shap_values[1][0][i]))
if not os.path.exists(os.path.dirname(save_filename)):
os.makedirs(os.path.dirname(save_filename))
list_images.append(save_filename)
plot_heatmap_shap(shap_values, list_images)
return list_classes, list_images
if dict1['model'].input_shape[2] is not None:
dict1['image_size'] = dict1['model'].input_shape[2]
else:
dict1['image_size'] = 299
print('%s load complte!' % (model_file))
NUM_CLASSES = dicts_models[0]['model'].output_shape[1]
#region test mode
if my_config.debug_mode:
img_source = '/tmp1/brvo1.jpg'
if os.path.exists(img_source):
# img1 = my_preprocess.my_preprocess(img_source, 512)
img1 = my_preprocess.do_preprocess(img_source, 512)
# prob_list, pred_list, prob_total, pred_total, correct_model_no = predict_softmax(img1)
prob_list, pred_list, prob_total, pred_total, correct_model_no = predict_sigmoids(
img1)
print(prob_total)
else:
print('file:', img_source, ' does not exist.')
#endregion
server = SimpleXMLRPCServer(("localhost", port))
print("Listening on port: ", str(port))
server.register_function(predict_softmax, "predict_softmax")
server.register_function(predict_sigmoids, "predict_sigmoids")
server.serve_forever()
dict1['model'] = keras.models.load_model(model_file, compile=False)
if 'input_shape' not in dict1:
if len(dict1['model'].input_shape
) == 4: #(batch, height, width, channel)
dict1['input_shape'] = dict1['model'].input_shape[1:]
else:
dict1['input_shape'] = (299, 299, 3)
print('%s load complte!' % (model_file))
#region test mode
if my_config.debug_mode:
img_source = '/tmp1/66b17a1e-a74d-11e8-94f6-6045cb817f5b.jpg'
if os.path.exists(img_source):
img_file_preprocessed = '/tmp1/preprocessed.jpg'
img1 = do_preprocess(img_source,
my_config.preprocess_img_size,
img_file_dest=img_file_preprocessed)
prob_list, pred_list, prob_total, pred_total, correct_model_no = predict_softmax(
img_file_preprocessed)
print(prob_total)
else:
print('file:', img_source, ' does not exist.')
#endregion
server = SimpleXMLRPCServer(("localhost", port))
print("Listening on port: ", str(port))
server.register_function(predict_softmax, "predict_softmax")
server.serve_forever()
def server_gradcam_plusplus(dicts_models,
model_no,
img_source,
pred,
preprocess=True,
blend_original_image=True,
base_dir_save='/tmp3'):
image_size = dicts_models[model_no]['image_size']
# region preprocessing if needed, convert to input_tensor
if isinstance(img_source, str):
if preprocess:
img_preprocess = my_preprocess.do_preprocess(img_source,
crop_size=384)
img_input = LIBS.ImgPreprocess.my_image_helper.my_gen_img_tensor(
img_preprocess, image_shape=(image_size, image_size, 3))
else:
img_input = LIBS.ImgPreprocess.my_image_helper.my_gen_img_tensor(
img_source, image_shape=(image_size, image_size, 3))
if isinstance(img_source, np.ndarray):
if img_source.ndim == 4:
img_input = img_source
# endregion
#region generating clas activation maps
#gradcamplus: 0-1
gradcamplus = grad_cam_plus(dicts_models[model_no]['model_original'],
img_input,
pred=pred,
image_size=image_size)
# cam: 0-255
cam = cv2.applyColorMap(np.uint8(255 * gradcamplus), cv2.COLORMAP_JET)
#endregion
#region blend original image
if blend_original_image:
# Return to BGR [0..255] from the preprocessed image
image_original = img_input[0, :]
from LIBS.ImgPreprocess.my_image_norm import input_norm_reverse
image_original = input_norm_reverse(image_original)
image_original = image_original.astype(np.uint8)
image_original -= np.min(image_original)
image_original = np.minimum(image_original, 255)
cam = np.float32(cam) + np.float32(image_original)
cam = 255 * cam / np.max(cam)
#endregion
#region 将CAM保存到文件
# 传过来的是web目录
str_uuid = str(uuid.uuid1())
filename_CAM = os.path.join(base_dir_save, str_uuid,
'GradCAM_PlusPlus{}.jpg'.format(pred))
if not os.path.exists(os.path.dirname(filename_CAM)):
os.makedirs(os.path.dirname(filename_CAM))
cv2.imwrite(filename_CAM, cam)
return filename_CAM
dict1['model'] = keras.models.load_model(dict1['model_file'],
compile=False)
print('load model:{0} complete!'.format(i))
e_list = get_e_list(dicts_models,
reference_file=REFERENCE_FILE,
num_reference=NUM_REFERENCE)
#endregion
# region test mode
if my_config.debug_mode:
img_source = '/tmp1/brvo.jpg'
if os.path.exists(img_source):
img_preprocess = my_preprocess.do_preprocess(img_source, crop_size=384)
img_preprocess = LIBS.ImgPreprocess.my_image_helper.my_gen_img_tensor(
img_preprocess, image_shape=(299, 299, 3))
prob = dicts_models[0]['model'].predict(img_preprocess)
pred = np.argmax(prob)
print(pred)
for i in range(2):
print(time.time())
list_classes, list_images = server_shap_deep_explain(
model_no=0,
img_source=img_source,
preprocess=True,
ranked_outputs=1)
print(time.time())
}):
dict1['model'] = keras.models.load_model(model_file, compile=False)
if 'input_shape' not in dict1:
if len(dict1['model'].input_shape
) == 4: #(batch, height, width, channel)
dict1['input_shape'] = dict1['model'].input_shape[1:]
else:
dict1['input_shape'] = (299, 299, 3)
print('%s load complte!' % (model_file))
#region test mode
if my_config.debug_mode:
img_source = '/media/ubuntu/data1/ROP_dataset/Stage/preprocess384/广州妇幼2017-2018/分期病变/1089fd0e-2aeb-4428-997d-02db4b9a3980.10.jpg'
if os.path.exists(img_source):
img1 = do_preprocess(img_source, PREPROCESS_IMG_SIZE)
prob_list, pred_list, prob_total, pred_total, correct_model_no = predict_softmax(
img1)
print(prob_total)
else:
print('file:', img_source, ' does not exist.')
#endregion
server = SimpleXMLRPCServer(("localhost", port))
print("Listening on port: ", str(port))
server.register_function(predict_softmax, "predict_softmax")
server.serve_forever()
def server_cam(model_no,
img_source,
pred,
cam_relu=True,
preprocess=True,
blend_original_image=True):
image_size = dicts_models[model_no]['image_size']
if preprocess:
img_preprocess = my_preprocess.do_preprocess(img_source, crop_size=384)
img_input = LIBS.ImgPreprocess.my_image_helper.my_gen_img_tensor(
img_preprocess, image_shape=(image_size, image_size, 3))
else:
img_input = LIBS.ImgPreprocess.my_image_helper.my_gen_img_tensor(
img_source, image_shape=(image_size, image_size, 3))
#region generate CAM
model1 = dicts_models[model_no]['model_cam']
all_amp_layer_weights = dicts_models[model_no]['all_amp_layer_weights']
last_conv_output, pred_vec = model1.predict(img_input)
# pred = np.argmax(pred_vec) # get model's prediction class
# Remove single-dimensional entries from the shape of an array.
last_conv_output = np.squeeze(last_conv_output)
# get AMP layer weights
amp_layer_weights = all_amp_layer_weights[:, pred] # dim: (2048,)
# jijie add relu
# 对于每一个类别C，每个特征图K的均值都有一个对应的w
if cam_relu:
amp_layer_weights = np.maximum(amp_layer_weights, 0)
cam_small = np.dot(last_conv_output, amp_layer_weights) # dim: 224 x 224
cam_small = np.maximum(cam_small, 0) # ReLU
cam = cv2.resize(cam_small, (image_size, image_size)) # 14*14 224*224
heatmap = cam / np.max(cam) # heatmap:0-1
#cam: 0-255
cam = cv2.applyColorMap(np.uint8(255 * heatmap), cv2.COLORMAP_JET)
if blend_original_image:
# Return to BGR [0..255] from the preprocessed image
image_original = img_input[0, :]
from LIBS.ImgPreprocess.my_image_norm import input_norm_reverse
image_original = input_norm_reverse(image_original)
image_original = image_original.astype(np.uint8)
image_original -= np.min(image_original)
image_original = np.minimum(image_original, 255)
cam = np.float32(cam) + np.float32(image_original)
cam = 255 * cam / np.max(cam)
#endregion
#region 将CAM保存到文件
str_uuid = str(uuid.uuid1())
filename_CAM = os.path.join(BASE_DIR_SAVE, str_uuid,
'CAM{}.jpg'.format(pred))
if not os.path.exists(os.path.dirname(filename_CAM)):
os.makedirs(os.path.dirname(filename_CAM))
cv2.imwrite(filename_CAM, cam)
# endregion
return filename_CAM
print('converting model1 complete')
e_list1.append(e)
e_list2 = []
for i in range(len(dicts_models)):
print('converting model2 ...')
e = shap.DeepExplainer(dicts_models[0]['model'], background2) #it will take 10 seconds
print('converting model2 complete')
e_list2.append(e)
# region test mode
DEGUB_MODE = True
if DEGUB_MODE:
img_source = '/tmp1/img4.jpg'
if os.path.exists(img_source):
img_preprocess = my_preprocess.do_preprocess(img_source, crop_size=384, train_or_valid='valid')
img_preprocess = LIBS.ImgPreprocess.my_image_helper.my_gen_img_tensor(img_preprocess,
image_shape=(image_size, image_size, 3))
prob = dicts_models[0]['model'].predict(img_preprocess)
pred = np.argmax(prob)
print(pred)
print('model1')
#first time take longer
for i in range(2):
print(time.time())
list_classes, list_images = server_shap_deep_explain(dicts_models=dicts_models, model_no=0,
img_source=img_source, preprocess=True, ranked_outputs=1)
print(time.time())
print(list_images)
def process_deep_explain(model, num_classes=NUM_CLASSES):
print('prepare to load model:' + model['model_file'])
model1 = keras.models.load_model(model['model_file'])
print('model load complete!')
image_size = model['image_size']
with DeepExplain(session=K.get_session()) as de: # init DeepExplain context
#region convert model: input_tensor, target_tensor
# Need to reconstruct the graph in DeepExplain context, using the same weights.
# With Keras this is very easy:
# 1. Get the input tensor to the original model
input_tensor = model1.layers[0].input
# 2. We now target the output of the last dense layer (pre-softmax)
# To do so, create a new model sharing the same layers untill the last dense (index -2)
fModel = Model(inputs=input_tensor, outputs=model1.layers[-1].output)
target_tensor = fModel(input_tensor)
q_process_start.put('OK')
#endregion
while (True):
while q_request.empty():
time.sleep(0.1)
#region get parameters from request queue and preprocess if needed
dict_req_param = q_request.get()
date_time_start = dict_req_param['date_time_start']
if time.time() - date_time_start > TIME_OUT:
continue
str_uuid = dict_req_param['str_uuid']
img_source = dict_req_param['img_source']
pred = dict_req_param['pred']
preprocess = dict_req_param['preprocess']
if preprocess:
img_preprocess = my_preprocess.do_preprocess(img_source, crop_size=384, add_black_pixel_ratio=0)
img_preprocess = LIBS.ImgPreprocess.my_image_helper.my_gen_img_tensor(img_preprocess,
image_shape=(image_size, image_size, 3))
else:
img_preprocess = LIBS.ImgPreprocess.my_image_helper.my_gen_img_tensor(img_source,
image_shape=(image_size, image_size, 3))
#endregion
#region generate deep_lift maps
xs = img_preprocess
ys = keras.utils.to_categorical([pred], num_classes)
# attributions = de.explain('grad*input', target_tensor * ys, input_tensor, xs)
# attributions = de.explain('saliency', target_tensor * ys, input_tensor, xs)
# attributions = de.explain('intgrad', target_tensor * ys, input_tensor, xs)
# 4 seconds
attributions = de.explain('deeplift', target_tensor * ys, input_tensor, xs)
# attributions = de.explain('elrp', target_tensor * ys, input_tensor, xs)
# attributions = de.explain('occlusion', target_tensor * ys, input_tensor, xs)
data1 = attributions[0]
data1 = np.mean(data1, 2)
# data1 = attributions[0][:, :, 0].reshape(image_size, image_size)
data = data1.reshape(image_size, image_size)
abs_max = np.percentile(np.abs(data), 100)
abs_min = abs_max
cmap = 'seismic' # cmap = 'RdBu_r' cmap = 'gray'
plt.imshow(data, interpolation='none', cmap=cmap, vmin=-abs_min, vmax=abs_max)
plt.axis('off')
# endregion
#region save file and return
# 传过来的是web目录
(filepath, tempfilename) = os.path.split(img_source)
image_uuid = filepath.split('/')[-1]
filename_CAM = os.path.join(BASE_DIR_SAVE, image_uuid, 'Deep_lift' + str(pred) + '.jpg')
# 测试可能不存在，实际web上传存在
if not os.path.exists(os.path.dirname(filename_CAM)):
os.makedirs(os.path.dirname(filename_CAM))
plt.savefig(filename_CAM, bbox_inches='tight', )
plt.close()
dict1 = {'str_uuid': str_uuid, 'filename_CAM': filename_CAM,
'date_time_start':date_time_start, 'date_time_end': time.time()}
q_response.put(dict1)
def shap_deep_explain(dicts_models, model_no,
e_list, num_reference, img_source, preprocess=True,
blend_original_image=False,
ranked_outputs=1, base_dir_save='/tmp/DeepExplain'):
image_shape = dicts_models[model_no]['input_shape']
if isinstance(img_source, str):
if preprocess:
img_preprocess = my_preprocess.do_preprocess(img_source, crop_size=384)
img_input = LIBS.ImgPreprocess.my_image_helper.my_gen_img_tensor(img_preprocess,
image_shape=image_shape)
else:
img_input = LIBS.ImgPreprocess.my_image_helper.my_gen_img_tensor(img_source,
image_shape=image_shape)
else:
img_input = img_source
#region mini-batch because of GPU memory limitation
list_shap_values = []
batch_size = dicts_models[model_no]['batch_size']
split_times = num_reference // batch_size
for i in range(split_times):
shap_values_tmp1 = e_list[model_no][i].shap_values(img_input, ranked_outputs=ranked_outputs)
# shap_values ranked_outputs
# [0] [0] (1,299,299,3)
# [1] predict_class array
shap_values_copy = copy.deepcopy(shap_values_tmp1)
list_shap_values.append(shap_values_copy)
for i in range(ranked_outputs):
for j in range(len(list_shap_values)):
if j == 0:
shap_values_tmp2 = list_shap_values[0][0][i]
else:
shap_values_tmp2 += list_shap_values[j][0][i]
shap_values_results = copy.deepcopy(list_shap_values[0])
shap_values_results[0][i] = shap_values_tmp2 / split_times
#endregion
#region save files
str_uuid = str(uuid.uuid1())
list_classes = []
list_images = []
for i in range(ranked_outputs):
predict_class = int(shap_values_results[1][0][i]) #numpy int 64 - int
list_classes.append(predict_class)
save_filename = os.path.join(base_dir_save, str_uuid,
'Shap_Deep_Explain{}.jpg'.format(predict_class))
if not os.path.exists(os.path.dirname(save_filename)):
os.makedirs(os.path.dirname(save_filename))
list_images.append(save_filename)
plot_heatmap_shap(shap_values_results,list_images,
img_input, blend_original_image=blend_original_image)
# because the original image and deepshap image were not aligned.
# blend_original_image=False,
# # region blend original
# if blend_original_image:
# for image1 in list_images:
# import cv2
#
# if isinstance(img_source, str):
# image_original = cv2.imread(img_source)
# else:
# image_original = img_source
#
# img_deepshap = cv2.imread(image1)
# img_deepshap = cv2.resize(img_deepshap, (384, 384))
# dst = cv2.addWeighted(img_deepshap, 0.3, image_original, 0.7, 0)
#
# # cv2.imwrite('/tmp5/111.jpg', image_original)
# # cv2.imwrite('/tmp5/aaa.jpg', dst)
# cv2.imwrite(image1, dst)
# #endregion
#endregion
return list_classes, list_images
model_no=model_no, num_reference=num_reference,
img_input=img_input, ranked_outputs=ranked_outputs,
blend_original_image=blend_original_image, base_dir_save=dir_tmp)
return list_classes, list_images
if True:
# if my_config.debug_mode:
import time
img_source = '/media/ubuntu/data1/ROP_dataset/Stage/preprocess384/广州妇幼番禺区/分期病变/65a9b14a-4ca0-468e-9438-ba32af340916.6.jpg'
if os.path.exists(img_source):
input_shape = dicts_models[0]['input_shape']
img_file_preprocessed = '/tmp1/preprocessed.jpg'
img_preprocess = my_preprocess.do_preprocess(img_source, crop_size=384,
img_file_dest=img_file_preprocessed)
img_input = LIBS.ImgPreprocess.my_image_helper.my_gen_img_tensor(
img_source, image_shape=input_shape)
prob = dicts_models[0]['model'].predict(img_input)
pred = np.argmax(prob)
print(pred)
#first time take longer
for i in range(2):
print(time.time())
list_classes, list_images = server_shap_deep_explainer(model_no=0,
img_source=img_file_preprocessed, ranked_outputs=1,
blend_original_image=False)
print(time.time())
print(list_images)
def server_grad_cam(dicts_models,
model_no,
img_source,
pred,
preprocess=True,
blend_original_image=True,
base_dir_save='/tmp3'):
model = dicts_models[model_no]['model_original']
image_size = dicts_models[model_no]['image_size']
#region preprocessing if needed, convert to input_tensor
if isinstance(img_source, str):
if preprocess:
img_preprocess = my_preprocess.do_preprocess(img_source,
crop_size=384)
img_input = LIBS.ImgPreprocess.my_image_helper.my_gen_img_tensor(
img_preprocess, image_shape=(image_size, image_size, 3))
else:
img_input = LIBS.ImgPreprocess.my_image_helper.my_gen_img_tensor(
img_source, image_shape=(image_size, image_size, 3))
if isinstance(img_source, np.ndarray):
if img_source.ndim == 4:
img_input = img_source
#endregion
#region generating class activation maps
penultimate_layer = get_last_conv_layer_number(model)
layer_idx = len(model.layers) - 1
modifier = 'guided' # [None, 'guided', 'relu']
# too slow
grads = visualize_cam(model,
layer_idx,
filter_indices=[pred],
seed_input=img_input,
penultimate_layer_idx=penultimate_layer,
backprop_modifier=modifier)
cam = cv2.applyColorMap(np.uint8(255 * grads), cv2.COLORMAP_JET)
#endregion
if blend_original_image:
# Return to BGR [0..255] from the preprocessed image
image_original = img_input[0, :]
from LIBS.ImgPreprocess.my_image_norm import input_norm_reverse
image_original = input_norm_reverse(image_original)
image_original = image_original.astype(np.uint8)
image_original -= np.min(image_original)
image_original = np.minimum(image_original, 255)
cam = np.float32(cam) + np.float32(image_original)
cam = 255 * cam / np.max(cam)
#region save CAMs to files
str_uuid = str(uuid.uuid1())
filename_CAM = os.path.join(base_dir_save, str_uuid,
'Grad_CAM{}.jpg'.format(pred))
if not os.path.exists(os.path.dirname(filename_CAM)):
os.makedirs(os.path.dirname(filename_CAM))
cv2.imwrite(filename_CAM, cam)
#endregion
return filename_CAM
'Stage_split_patid_test'
]:
save_dir = os.path.join(DIR_SAVE_RESULTS, predict_type_name)
filename_csv = os.path.abspath(
os.path.join(sys.path[0], "..", "..", "..", 'datafiles/dataset10',
predict_type_name + '.csv'))
df = pd.read_csv(filename_csv)
for _, row in df.iterrows():
image_file = row['images']
image_label = int(row['labels'])
# img_file = image_file.replace('/preprocess384/', '/original/')
preprocess = False
if preprocess:
img_preprocess = my_preprocess.do_preprocess(image_file,
crop_size=384)
img_input = LIBS.ImgPreprocess.my_image_helper.my_gen_img_tensor(
img_preprocess, image_shape=image_shape)
else:
img_input = LIBS.ImgPreprocess.my_image_helper.my_gen_img_tensor(
image_file, image_shape=image_shape)
prob = dicts_models[MODEL_NO]['model'].predict(img_input)
class_predict = np.argmax(prob)
if (class_predict == 1 and image_label == 1) or\
(class_predict == 1 and image_label == 0):
list_classes, list_images = shap_deep_explain(
dicts_models=dicts_models,
model_no=MODEL_NO,
def detect_optic_disc(self, image_source, image_size=384, preprocess=True):
if preprocess:
image_preprocess = my_preprocess.do_preprocess(
image_source, crop_size=image_size, add_black_pixel_ratio=0.05)
else:
# 不做预处理 裁剪， 为了计算IOU,DICE和原来的Mask匹配
if isinstance(image_source, str):
image_preprocess = cv2.imread(image_source)
if image_preprocess.shape[1] != 384:
image_preprocess = cv2.resize(image_preprocess, (384, 384))
else:
image_preprocess = image_source
image_preprocess = image_preprocess.astype(np.uint8)
image_preprocess_RGB = cv2.cvtColor(image_preprocess,
cv2.COLOR_BGR2RGB)
# image_preprocess_RGB = np.load("/tmp/A.npy")
# Run detection
results = self.model.detect([image_preprocess_RGB], verbose=1)
r = results[0] #batch size = 1
if len(r['class_ids']) > 0 and r['class_ids'][0] == 1:
found_optic_disc = True
score = r['scores'][0]
y1, x1, y2, x2 = r['rois'][0]
# cv2.rectangle(image_preprocess, (x1, y1), (x2, y2), (0, 255, 0), 1)
# cv2.imwrite('test1.jpg', image_preprocess)
np_zero = np.zeros((384, 384, 1))
np_ono = np.ones((384, 384, 1))
np_255 = 255 * np_ono
#img_masks shape (384,384,1) 可能多个masks
temp_image = np.expand_dims(r['masks'][:, :, 0], axis=-1)
img_masks = np.where(temp_image, np_255, np_zero)
img_masks = img_masks.astype(np.uint8)
#region 剪切图像 视盘区域，和掩码
center_x = (x2 + x1) // 2
center_y = (y2 + y1) // 2
r = max((x2 - x1) // 2, (y2 - y1) // 2)
r = r + 40
# 正方形
left = int(max(0, center_x - r))
right = int(min(image_preprocess.shape[1], center_x + r))
bottom = int(max(0, center_y - r))
top = int(min(image_preprocess.shape[1], center_y + r))
image_crop = image_preprocess[bottom:top, left:right]
image_crop = cv2.resize(image_crop, (112, 112))
image_crop_mask = img_masks[bottom:top, left:right]
#image_crop_mask shape (112,112)
image_crop_mask = cv2.resize(image_crop_mask, (112, 112))
# endregion剪切图像
return found_optic_disc, score, image_preprocess, image_crop, img_masks, image_crop_mask, x1, y1, x2, y2
return False, 0, image_preprocess, None, None, None, 0, 0, 0, 0
