def process_image(self, sample_index, annotation, sigma, rot_flag, scale_flag, h_flip_flag, v_flip_flag):
imagefile = os.path.join(self.imgpath, annotation['img_paths'])
img = Image.open(imagefile)
# make sure image is in RGB mode with 3 channels
if img.mode != 'RGB':
img = img.convert('RGB')
image = np.array(img)
img.close()
# get center, joints and scale
# center, joints point format: (x, y)
center = np.array(annotation['objpos'])
joints = np.array(annotation['joint_self'])
scale = annotation['scale_provided']
# Adjust center/scale slightly to avoid cropping limbs
if center[0] != -1:
center[1] = center[1] + 15 * scale
scale = scale * 1.25
# random horizontal filp
if h_flip_flag and random.choice([0, 1]):
image, joints, center = horizontal_flip(image, joints, center, self.horizontal_matchpoints)
# random vertical filp
if v_flip_flag and random.choice([0, 1]):
image, joints, center = vertical_flip(image, joints, center, self.vertical_matchpoints)
# random adjust scale
if scale_flag:
scale = scale * np.random.uniform(0.8, 1.2)
# random rotate image
if rot_flag and random.choice([0, 1]):
rot = np.random.randint(-1 * 30, 30)
else:
rot = 0
# crop out single person area, resize to input size res and normalize image
image = crop(image, center, scale, self.input_size, rot)
# in case we got an empty image, bypass the sample
if image is None:
return None, None, None
# normalize image
image = normalize_image(image, self.get_color_mean())
# transform keypoints to crop image reference
transformedKps = transform_kp(joints, center, scale, self.output_size, rot)
# generate ground truth point heatmap
gtmap = generate_gtmap(transformedKps, sigma, self.output_size)
# meta info
metainfo = {'sample_index': sample_index, 'center': center, 'scale': scale,
'pts': joints, 'tpts': transformedKps, 'name': imagefile}
return image, gtmap, metainfo
def preprocess(image):
# random adjust color level
image = random_chroma(image)
# random adjust contrast
image = random_contrast(image)
# random adjust sharpness
image = random_sharpness(image)
# random convert image to grayscale
image = random_grayscale(image)
# normalize image
image = normalize_image(image)
return image
def get_ground_truth_data(annotation_line, input_shape, augment=True, max_boxes=100):
'''random preprocessing for real-time data augmentation'''
line = annotation_line.split()
image = Image.open(line[0])
image_size = image.size
model_input_size = tuple(reversed(input_shape))
boxes = np.array([np.array(list(map(int,box.split(',')))) for box in line[1:]])
if not augment:
new_image, padding_size, offset = letterbox_resize(image, target_size=model_input_size, return_padding_info=True)
image_data = np.array(new_image)
image_data = normalize_image(image_data)
# reshape boxes
boxes = reshape_boxes(boxes, src_shape=image_size, target_shape=model_input_size, padding_shape=padding_size, offset=offset)
if len(boxes)>max_boxes:
boxes = boxes[:max_boxes]
# fill in box data
box_data = np.zeros((max_boxes,5))
if len(boxes)>0:
box_data[:len(boxes)] = boxes
return image_data, box_data
# random resize image and crop|padding to target size
image, padding_size, padding_offset = random_resize_crop_pad(image, target_size=model_input_size)
# random horizontal flip image
image, horizontal_flip = random_horizontal_flip(image)
# random adjust brightness
image = random_brightness(image)
# random adjust color level
image = random_chroma(image)
# random adjust contrast
image = random_contrast(image)
# random adjust sharpness
image = random_sharpness(image)
# random convert image to grayscale
image = random_grayscale(image)
# random do normal blur to image
#image = random_blur(image)
# random do motion blur to image
#image = random_motion_blur(image, prob=0.2)
# random vertical flip image
image, vertical_flip = random_vertical_flip(image)
# random distort image in HSV color space
# NOTE: will cost more time for preprocess
# and slow down training speed
#image = random_hsv_distort(image)
# reshape boxes based on augment
boxes = reshape_boxes(boxes, src_shape=image_size, target_shape=model_input_size, padding_shape=padding_size, offset=padding_offset, horizontal_flip=horizontal_flip, vertical_flip=vertical_flip)
# random rotate image and boxes
image, boxes = random_rotate(image, boxes)
# random add gridmask augment for image and boxes
image, boxes = random_gridmask(image, boxes)
if len(boxes)>max_boxes:
boxes = boxes[:max_boxes]
# prepare image & box data
image_data = np.array(image)
image_data = normalize_image(image_data)
box_data = np.zeros((max_boxes,5))
if len(boxes)>0:
box_data[:len(boxes)] = boxes
return image_data, box_data
def generate_heatmap(image_path, model_path, heatmap_path, class_names=None):
# load model
custom_object_dict = get_custom_objects()
model = load_model(model_path, custom_objects=custom_object_dict)
K.set_learning_phase(0)
model.summary()
# get image file list or single image
if os.path.isdir(image_path):
jpeg_files = glob.glob(os.path.join(image_path, '*.jpeg'))
jpg_files = glob.glob(os.path.join(image_path, '*.jpg'))
image_list = jpeg_files + jpg_files
#assert os.path.isdir(heatmap_path), 'need to provide a path for output heatmap'
os.makedirs(heatmap_path, exist_ok=True)
heatmap_list = [
os.path.join(
heatmap_path,
os.path.splitext(os.path.basename(image_name))[0] + '.jpg')
for image_name in image_list
]
else:
image_list = [image_path]
heatmap_list = [heatmap_path]
# loop the sample list to generate all heatmaps
for i, (image_file,
heatmap_file) in enumerate(zip(image_list, heatmap_list)):
# process input
target_size = get_target_size(model)
img = load_and_crop_img(image_file,
target_size=target_size,
interpolation='nearest:random')
img = np.array(img)
x = normalize_image(img)
x = np.expand_dims(x, axis=0)
# predict and get output
preds = model.predict(x)
index = np.argmax(preds[0])
score = preds[0][index]
max_output = model.output[:, index]
# detect last conv layer
last_conv_index = detect_last_conv(model)
last_conv_layer = model.layers[last_conv_index]
# get gradient of the last conv layer to the predicted class
grads = K.gradients(max_output, last_conv_layer.output)[0]
# pooling to get the feature gradient
pooled_grads = K.mean(grads, axis=(0, 1, 2))
# run the predict to get value
iterate = K.function([model.input],
[pooled_grads, last_conv_layer.output[0]])
pooled_grads_value, conv_layer_output_value = iterate([x])
# apply the activation to each channel of the conv'ed feature map
for j in range(pooled_grads_value.shape[0]):
conv_layer_output_value[:, :, j] *= pooled_grads_value[j]
# get mean of each channel, which is the heatmap
heatmap = np.mean(conv_layer_output_value, axis=-1)
# normalize heatmap to 0~1
heatmap = np.maximum(heatmap, 0)
heatmap /= np.max(heatmap)
#plt.matshow(heatmap)
#plt.show()
# overlap heatmap to frame image
#img = cv2.imread(image_file)
img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
img = cv2.resize(img, (224, 224))
heatmap = cv2.resize(heatmap, (img.shape[1], img.shape[0]))
heatmap = denormalize_image(heatmap)
heatmap = cv2.applyColorMap(heatmap, cv2.COLORMAP_JET)
superimposed_img = heatmap * 0.4 + img
# show predict class index or name on image
cv2.putText(superimposed_img,
'{name}:{conf:.3f}'.format(
name=class_names[index] if class_names else index,
conf=float(score)), (10, 30),
cv2.FONT_HERSHEY_SIMPLEX,
fontScale=1,
color=(0, 0, 255),
thickness=1,
lineType=cv2.LINE_AA)
# save overlaped image
cv2.imwrite(heatmap_file, superimposed_img)
print("generate heatmap file {} ({}/{})".format(
heatmap_file, i + 1, len(image_list)))
