def process_image(self, sample_index, annotation, sigma, rot_flag, scale_flag, h_flip_flag, v_flip_flag): imagefile = os.path.join(self.imgpath, annotation['img_paths']) img = Image.open(imagefile) # make sure image is in RGB mode with 3 channels if img.mode != 'RGB': img = img.convert('RGB') image = np.array(img) img.close() # get center, joints and scale # center, joints point format: (x, y) center = np.array(annotation['objpos']) joints = np.array(annotation['joint_self']) scale = annotation['scale_provided'] # Adjust center/scale slightly to avoid cropping limbs if center[0] != -1: center[1] = center[1] + 15 * scale scale = scale * 1.25 # random horizontal filp if h_flip_flag and random.choice([0, 1]): image, joints, center = horizontal_flip(image, joints, center, self.horizontal_matchpoints) # random vertical filp if v_flip_flag and random.choice([0, 1]): image, joints, center = vertical_flip(image, joints, center, self.vertical_matchpoints) # random adjust scale if scale_flag: scale = scale * np.random.uniform(0.8, 1.2) # random rotate image if rot_flag and random.choice([0, 1]): rot = np.random.randint(-1 * 30, 30) else: rot = 0 # crop out single person area, resize to input size res and normalize image image = crop(image, center, scale, self.input_size, rot) # in case we got an empty image, bypass the sample if image is None: return None, None, None # normalize image image = normalize_image(image, self.get_color_mean()) # transform keypoints to crop image reference transformedKps = transform_kp(joints, center, scale, self.output_size, rot) # generate ground truth point heatmap gtmap = generate_gtmap(transformedKps, sigma, self.output_size) # meta info metainfo = {'sample_index': sample_index, 'center': center, 'scale': scale, 'pts': joints, 'tpts': transformedKps, 'name': imagefile} return image, gtmap, metainfo
def preprocess(image): # random adjust color level image = random_chroma(image) # random adjust contrast image = random_contrast(image) # random adjust sharpness image = random_sharpness(image) # random convert image to grayscale image = random_grayscale(image) # normalize image image = normalize_image(image) return image
def get_ground_truth_data(annotation_line, input_shape, augment=True, max_boxes=100): '''random preprocessing for real-time data augmentation''' line = annotation_line.split() image = Image.open(line[0]) image_size = image.size model_input_size = tuple(reversed(input_shape)) boxes = np.array([np.array(list(map(int,box.split(',')))) for box in line[1:]]) if not augment: new_image, padding_size, offset = letterbox_resize(image, target_size=model_input_size, return_padding_info=True) image_data = np.array(new_image) image_data = normalize_image(image_data) # reshape boxes boxes = reshape_boxes(boxes, src_shape=image_size, target_shape=model_input_size, padding_shape=padding_size, offset=offset) if len(boxes)>max_boxes: boxes = boxes[:max_boxes] # fill in box data box_data = np.zeros((max_boxes,5)) if len(boxes)>0: box_data[:len(boxes)] = boxes return image_data, box_data # random resize image and crop|padding to target size image, padding_size, padding_offset = random_resize_crop_pad(image, target_size=model_input_size) # random horizontal flip image image, horizontal_flip = random_horizontal_flip(image) # random adjust brightness image = random_brightness(image) # random adjust color level image = random_chroma(image) # random adjust contrast image = random_contrast(image) # random adjust sharpness image = random_sharpness(image) # random convert image to grayscale image = random_grayscale(image) # random do normal blur to image #image = random_blur(image) # random do motion blur to image #image = random_motion_blur(image, prob=0.2) # random vertical flip image image, vertical_flip = random_vertical_flip(image) # random distort image in HSV color space # NOTE: will cost more time for preprocess # and slow down training speed #image = random_hsv_distort(image) # reshape boxes based on augment boxes = reshape_boxes(boxes, src_shape=image_size, target_shape=model_input_size, padding_shape=padding_size, offset=padding_offset, horizontal_flip=horizontal_flip, vertical_flip=vertical_flip) # random rotate image and boxes image, boxes = random_rotate(image, boxes) # random add gridmask augment for image and boxes image, boxes = random_gridmask(image, boxes) if len(boxes)>max_boxes: boxes = boxes[:max_boxes] # prepare image & box data image_data = np.array(image) image_data = normalize_image(image_data) box_data = np.zeros((max_boxes,5)) if len(boxes)>0: box_data[:len(boxes)] = boxes return image_data, box_data
def generate_heatmap(image_path, model_path, heatmap_path, class_names=None): # load model custom_object_dict = get_custom_objects() model = load_model(model_path, custom_objects=custom_object_dict) K.set_learning_phase(0) model.summary() # get image file list or single image if os.path.isdir(image_path): jpeg_files = glob.glob(os.path.join(image_path, '*.jpeg')) jpg_files = glob.glob(os.path.join(image_path, '*.jpg')) image_list = jpeg_files + jpg_files #assert os.path.isdir(heatmap_path), 'need to provide a path for output heatmap' os.makedirs(heatmap_path, exist_ok=True) heatmap_list = [ os.path.join( heatmap_path, os.path.splitext(os.path.basename(image_name))[0] + '.jpg') for image_name in image_list ] else: image_list = [image_path] heatmap_list = [heatmap_path] # loop the sample list to generate all heatmaps for i, (image_file, heatmap_file) in enumerate(zip(image_list, heatmap_list)): # process input target_size = get_target_size(model) img = load_and_crop_img(image_file, target_size=target_size, interpolation='nearest:random') img = np.array(img) x = normalize_image(img) x = np.expand_dims(x, axis=0) # predict and get output preds = model.predict(x) index = np.argmax(preds[0]) score = preds[0][index] max_output = model.output[:, index] # detect last conv layer last_conv_index = detect_last_conv(model) last_conv_layer = model.layers[last_conv_index] # get gradient of the last conv layer to the predicted class grads = K.gradients(max_output, last_conv_layer.output)[0] # pooling to get the feature gradient pooled_grads = K.mean(grads, axis=(0, 1, 2)) # run the predict to get value iterate = K.function([model.input], [pooled_grads, last_conv_layer.output[0]]) pooled_grads_value, conv_layer_output_value = iterate([x]) # apply the activation to each channel of the conv'ed feature map for j in range(pooled_grads_value.shape[0]): conv_layer_output_value[:, :, j] *= pooled_grads_value[j] # get mean of each channel, which is the heatmap heatmap = np.mean(conv_layer_output_value, axis=-1) # normalize heatmap to 0~1 heatmap = np.maximum(heatmap, 0) heatmap /= np.max(heatmap) #plt.matshow(heatmap) #plt.show() # overlap heatmap to frame image #img = cv2.imread(image_file) img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR) img = cv2.resize(img, (224, 224)) heatmap = cv2.resize(heatmap, (img.shape[1], img.shape[0])) heatmap = denormalize_image(heatmap) heatmap = cv2.applyColorMap(heatmap, cv2.COLORMAP_JET) superimposed_img = heatmap * 0.4 + img # show predict class index or name on image cv2.putText(superimposed_img, '{name}:{conf:.3f}'.format( name=class_names[index] if class_names else index, conf=float(score)), (10, 30), cv2.FONT_HERSHEY_SIMPLEX, fontScale=1, color=(0, 0, 255), thickness=1, lineType=cv2.LINE_AA) # save overlaped image cv2.imwrite(heatmap_file, superimposed_img) print("generate heatmap file {} ({}/{})".format( heatmap_file, i + 1, len(image_list)))