def ONet_(self, caffe_img, rectangles): origin_h, origin_w, ch = caffe_img.shape self.net_48.blobs['data'].reshape(len(rectangles), 3, 48, 48) crop_number = 0 for rectangle in rectangles: crop_img = caffe_img[int(rectangle[1]):int(rectangle[3]), int(rectangle[0]):int(rectangle[2])] scale_img = cv2.resize(crop_img, (48, 48)) if self.test: scale_img = np.swapaxes(scale_img, 0, 2) else: scale_img = np.transpose(scale_img, (2, 0, 1)) scale_img = np.asarray(scale_img, dtype=np.float32) self.net_48.blobs['data'].data[crop_number] = scale_img crop_number += 1 out = self.net_48.forward() cls_prob = out['prob1'] if self.test: roi_prob = out['conv6-2'] pts_prob = out['conv6-3'] #roi_prob = out['bbox_fc'] #pts_prob = out['landmark_fc'] else: roi_prob = out['bbox_fc'] pts_prob = out['landmark_fc'] rectangles_box = tools.filter_face_48net(cls_prob, roi_prob, pts_prob, rectangles, origin_w, origin_h, self.threshold[2]) return rectangles_box
def detectFace(img_path,threshold): img = cv2.imread(img_path) caffe_img = (img.copy()-127.5)/128 origin_h,origin_w,ch = caffe_img.shape scales = tools.calculateScales(img) out = [] for scale in scales: hs = int(origin_h*scale) ws = int(origin_w*scale) scale_img = cv2.resize(caffe_img,(ws,hs)) scale_img = np.swapaxes(scale_img, 0, 2) net_12.blobs['data'].reshape(1,3,ws,hs) net_12.blobs['data'].data[...]=scale_img caffe.set_device(0) caffe.set_mode_gpu() out_ = net_12.forward() out.append(out_) image_num = len(scales) rectangles = [] for i in range(image_num): cls_prob = out[i]['prob1'][0][1] roi = out[i]['conv4-2'][0] out_h,out_w = cls_prob.shape out_side = max(out_h,out_w) rectangle = tools.detect_face_12net(cls_prob,roi,out_side,1/scales[i],origin_w,origin_h,threshold[0]) rectangles.extend(rectangle) rectangles = tools.NMS(rectangles,0.7,'iou') if len(rectangles)==0: return rectangles net_24.blobs['data'].reshape(len(rectangles),3,24,24) crop_number = 0 for rectangle in rectangles: crop_img = caffe_img[int(rectangle[1]):int(rectangle[3]), int(rectangle[0]):int(rectangle[2])] scale_img = cv2.resize(crop_img,(24,24)) scale_img = np.swapaxes(scale_img, 0, 2) net_24.blobs['data'].data[crop_number] =scale_img crop_number += 1 out = net_24.forward() cls_prob = out['prob1'] roi_prob = out['conv5-2'] rectangles = tools.filter_face_24net(cls_prob,roi_prob,rectangles,origin_w,origin_h,threshold[1]) if len(rectangles)==0: return rectangles net_48.blobs['data'].reshape(len(rectangles),3,48,48) crop_number = 0 for rectangle in rectangles: crop_img = caffe_img[int(rectangle[1]):int(rectangle[3]), int(rectangle[0]):int(rectangle[2])] scale_img = cv2.resize(crop_img,(48,48)) scale_img = np.swapaxes(scale_img, 0, 2) net_48.blobs['data'].data[crop_number] =scale_img crop_number += 1 out = net_48.forward() cls_prob = out['prob1'] roi_prob = out['conv6-2'] pts_prob = out['conv6-3'] rectangles = tools.filter_face_48net(cls_prob,roi_prob,pts_prob,rectangles,origin_w,origin_h,threshold[2]) return rectangles
def run_onet(filename, rectangle, Onet, display_result=True): img = cv2.imread(filename) origin_h, origin_w, ch = img.shape ## mmod's output rectangle is (top, left, width, height), but Onet requires rectangle to be (left, top, right, bottom) rectangle = (rectangle[1], rectangle[0], (rectangle[1]+rectangle[3]), (rectangle[0]+rectangle[2])) ## regularize rectangle to make it stay in image rectangle = (max(0,rectangle[0]), max(0, rectangle[1]), min(origin_w, rectangle[2]), min(origin_h, rectangle[3])) ## crop and scale the rectangle area of input image crop_img = img[int(rectangle[1]):int(rectangle[3]), int(rectangle[0]):int(rectangle[2])] scale_img = cv2.resize(crop_img, (48,48)) x = np.array(scale_img) x = (x - 127.5)/127.5 x = np.expand_dims(x, axis=0) ## conducting inference y = Onet.predict(x) ## nms and post-processing the output cls_prob = y[0] roi_prob = y[1] pts_prob = y[2] rectangles = [rectangle] y = tools.filter_face_48net(cls_prob, roi_prob, pts_prob, rectangles, origin_w, origin_h, 0) y = y[0] ## return and/or display landmarks landmarks = [] cv2.rectangle(img, (int(y[0]), int(y[1])), (int(y[2]), int(y[3])), (0, 0, 255), int(2./200*min(origin_h,origin_w))) for i in range(5, 15, 2): cv2.circle(img, (int(y[i + 0]), int(y[i + 1])), int(2./200*min(origin_h,origin_w)), (0, 0, 255), -1) landmarks.append(int(y[i+0])) landmarks.append(int(y[i+1])) if display_result: print("==== showing Onet's detection ====") fig = plt.figure() plt.imshow(img[...,::-1]) plt.savefig(get_name("onet", filename)) plt.close() box = (int(y[0]), int(y[1]), int(y[2]), int(y[3])) return box , landmarks
def detectFace(img, threshold, rectangles): caffe_img = (img.copy() - 127.5) / 128 origin_h, origin_w, ch = caffe_img.shape if len(rectangles) == 0: return rectangles net_48.blobs['data'].reshape(len(rectangles), 3, 48, 48) crop_number = 0 for rectangle in rectangles: crop_img = caffe_img[int(rectangle[1]):int(rectangle[3]), int(rectangle[0]):int(rectangle[2])] scale_img = cv2.resize(crop_img, (48, 48)) scale_img = np.swapaxes(scale_img, 0, 2) net_48.blobs['data'].data[crop_number] = scale_img crop_number += 1 out = net_48.forward() cls_prob = out['prob1'] roi_prob = out['conv6-2'] pts_prob = out['conv6-3'] rectangles = tools.filter_face_48net(cls_prob, roi_prob, pts_prob, rectangles, origin_w, origin_h, threshold[2]) return rectangles
def onet_forward(o_net, cls_dets, im, thresh=0.7): """ :param o_net: the trained onet :param cls_dets: the detection result of ssh :param im: the input image :param thresh: the score thresh """ print('pre landmark size: ', cls_dets.shape) o_net.blobs['data'].reshape(cls_dets.shape[0], 3, 48, 48) caffe_img = (im.copy() - 127.5) / 128 crop_number = 0 for det in cls_dets: crop_img = caffe_img[int(det[1]):int(det[3]), int(det[0]):int(det[2])] scale_img = cv2.resize(crop_img, (48, 48)) scale_img = np.swapaxes(scale_img, 0, 2) # hwc chw o_net.blobs['data'].data[crop_number] = scale_img crop_number += 1 out = o_net.forward() cls_prob = out['prob1'] roi = out['conv6-2'] pts = out['conv6-3'] rectangles = tools.filter_face_48net(cls_prob, roi, pts, cls_dets, im.shape[1], im.shape[0], thresh) return rectangles
def detectFace(img, threshold): caffe_img = (img.copy() - 127.5) / 127.5 origin_h, origin_w, ch = caffe_img.shape scales = tools.calculateScales(img) out = [] t0 = time.time() # del scales[:4] for scale in scales: hs = int(origin_h * scale) ws = int(origin_w * scale) scale_img = cv2.resize(caffe_img, (ws, hs)) input = scale_img.reshape(1, *scale_img.shape) ouput = Pnet.predict( input ) # .transpose(0,2,1,3) should add, but seems after process is wrong then. out.append(ouput) image_num = len(scales) rectangles = [] for i in range(image_num): cls_prob = out[i][0][ 0][:, :, 1] # i = #scale, first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr roi = out[i][1][0] out_h, out_w = cls_prob.shape out_side = max(out_h, out_w) # print('calculating img scale #:', i) cls_prob = np.swapaxes(cls_prob, 0, 1) roi = np.swapaxes(roi, 0, 2) rectangle = tools.detect_face_12net(cls_prob, roi, out_side, 1 / scales[i], origin_w, origin_h, threshold[0]) rectangles.extend(rectangle) rectangles = tools.NMS(rectangles, 0.7, 'iou') t1 = time.time() print('time for 12 net is: ', t1 - t0) if len(rectangles) == 0: return rectangles crop_number = 0 out = [] predict_24_batch = [] for rectangle in rectangles: crop_img = caffe_img[int(rectangle[1]):int(rectangle[3]), int(rectangle[0]):int(rectangle[2])] scale_img = cv2.resize(crop_img, (24, 24)) predict_24_batch.append(scale_img) crop_number += 1 predict_24_batch = np.array(predict_24_batch) out = Rnet.predict(predict_24_batch) cls_prob = out[ 0] # first 0 is to select cls, second batch number, always =0 cls_prob = np.array(cls_prob) # convert to numpy roi_prob = out[ 1] # first 0 is to select roi, second batch number, always =0 roi_prob = np.array(roi_prob) rectangles = tools.filter_face_24net(cls_prob, roi_prob, rectangles, origin_w, origin_h, threshold[1]) t2 = time.time() print('time for 24 net is: ', t2 - t1) if len(rectangles) == 0: return rectangles crop_number = 0 predict_batch = [] for rectangle in rectangles: # print('calculating net 48 crop_number:', crop_number) crop_img = caffe_img[int(rectangle[1]):int(rectangle[3]), int(rectangle[0]):int(rectangle[2])] scale_img = cv2.resize(crop_img, (48, 48)) predict_batch.append(scale_img) crop_number += 1 predict_batch = np.array(predict_batch) output = Onet.predict(predict_batch) cls_prob = output[0] roi_prob = output[1] pts_prob = output[2] # index # rectangles = tools.filter_face_48net_newdef(cls_prob, roi_prob, pts_prob, rectangles, origin_w, origin_h, # threshold[2]) rectangles = tools.filter_face_48net(cls_prob, roi_prob, pts_prob, rectangles, origin_w, origin_h, threshold[2]) t3 = time.time() print('time for 48 net is: ', t3 - t2) return rectangles
def detectFace(img_path,threshold): img = cv2.imread(img_path) # caffe_img = (img.copy()-127.5)/127.5 caffe_img = img.copy() caffe_img = caffe_img.astype(np.float32) caffe_img[:,:,0] = (caffe_img[:,:,0] - 104.146) / 127.5 caffe_img[:,:,1] = (caffe_img[:,:,1] - 110.807) / 127.5 caffe_img[:,:,2] = (caffe_img[:,:,2] - 119.856) / 127.5 caffe_img2 = (img.copy() -127.5) / 127.5 origin_h,origin_w,ch = caffe_img.shape scales = tools.calculateScales(img) out = [] for scale in scales: hs = int(origin_h*scale) ws = int(origin_w*scale) scale_img = cv2.resize(caffe_img,(ws,hs)) scale_img = np.swapaxes(scale_img, 0, 2) net_12.blobs['data'].reshape(1,3,ws,hs) net_12.blobs['data'].data[...]=scale_img caffe.set_device(0) caffe.set_mode_gpu() out_ = net_12.forward() out.append(out_) image_num = len(scales) rectangles = [] for i in range(image_num): cls_prob = out[i]['prob1'][0][1] # print(cls_prob.shape) roi = out[i]['conv4-2'][0] out_h,out_w = cls_prob.shape out_side = max(out_h,out_w) #print("mean0:%f" % out[i]['prob1'][0][0].mean()) #print("mean0:%f" % out[i]['prob1'][0][1].mean()) #print("shape:",out[i]['prob1'][0][0].shape) #print(len(np.where(out[i]['prob1'][0][0] > threshold[0])[0])) #print(len(np.where(out[i]['prob1'][0][0] < threshold[0])[0])) ##continue #print(cls_prob) rectangle = tools.detect_face_12net(cls_prob,roi,out_side,1/scales[i],origin_w,origin_h,threshold[0]) rectangles.extend(rectangle) print("-------------") print(len(rectangles)) rectangles = tools.NMS(rectangles,0.7,'iou') print(len(rectangles)) if len(rectangles)==0: return rectangles net_24.blobs['data'].reshape(len(rectangles),3,24,24) crop_number = 0 for rectangle in rectangles: crop_img = caffe_img[int(rectangle[1]):int(rectangle[3]), int(rectangle[0]):int(rectangle[2])] scale_img = cv2.resize(crop_img,(24,24)) scale_img = np.swapaxes(scale_img, 0, 2) net_24.blobs['data'].data[crop_number] =scale_img crop_number += 1 out = net_24.forward() cls_prob = out['prob1'] roi_prob = out['fc5-2'] #roi_prob = out['conv5-2'] print(len(rectangles)) print(cls_prob.shape) rectangles = tools.filter_face_24net(cls_prob,roi_prob,rectangles,origin_w,origin_h,threshold[1]) print(len(rectangles)) # # if len(rectangles)==0: return rectangles net_48.blobs['data'].reshape(len(rectangles),3,48,48) crop_number = 0 for rectangle in rectangles: crop_img = caffe_img2[int(rectangle[1]):int(rectangle[3]), int(rectangle[0]):int(rectangle[2])] scale_img = cv2.resize(crop_img,(48,48)) scale_img = np.swapaxes(scale_img, 0, 2) net_48.blobs['data'].data[crop_number] =scale_img crop_number += 1 out = net_48.forward() cls_prob = out['prob1'] # roi_prob = out['conv6-2'] # pts_prob = out['conv6-3'] roi_prob = out['fc6-2'] pts_prob = out['fc6-3'] #roi_prob = roi_prob - roi_prob #print(roi_prob) print(cls_prob.shape) rectangles = tools.filter_face_48net(cls_prob,roi_prob,pts_prob,rectangles,origin_w,origin_h,threshold[2]) rectangles = tools.NMS(rectangles,0.7,'iou') print(len(rectangles)) # ##################################################### ## tmp code: # if len(rectangles)==0: # return rectangles # net_48.blobs['data'].reshape(len(rectangles),3,48,48) # crop_number = 0 # bbox = "531.0 215.0 574.0 277.0 363.0 219.0 421.0 296.0 251.0 262.0 320.0 354.0 116.0 252.0 196.0 351.0 707.0 248.0 786.0 338.0 870.0 250.0 960.0 376.0 280.0 124.0 308.0 155.0" # boxes = np.array(bbox.split(), dtype=np.float32).reshape(-1, 4) # for rectangle in rectangles: # print("%d %f" % (crop_number,np.max(IoU(rectangle,boxes)))) # crop_img = caffe_img[int(rectangle[1]):int(rectangle[3]), int(rectangle[0]):int(rectangle[2])] # cv2.imwrite('./crop_file2/%d.jpg' % crop_number,crop_img*127.5 + 127.5) # scale_img = cv2.resize(crop_img,(48,48)) # cv2.imwrite('./crop_file/%d.jpg' % crop_number,scale_img*127.5 + 127.5) # scale_img = np.swapaxes(scale_img, 0, 2) # net_48.blobs['data'].data[crop_number] =scale_img # crop_number += 1 # out = net_48.forward() # cls_prob = out['prob1'] # #roi_prob = out['conv6-2'] # #pts_prob = out['conv6-3'] # roi_prob = out['fc6-2'] # pts_prob = out['fc6-3'] # roi_prob = roi_prob - roi_prob # print(roi_prob) # print(cls_prob.shape) # print(len(rectangles)) # rectangles = tools.filter_face_48net(cls_prob,roi_prob,pts_prob,rectangles,origin_w,origin_h,threshold[2]) # rectangles = tools.NMS(rectangles,0.7,'iou') # print(len(rectangles)) return rectangles
def detectFace(img, threshold): caffe_img = (img.copy() - 127.5) / 127.5 origin_h, origin_w, ch = caffe_img.shape scales = tools.calculateScales(img) print("scales: ", scales) out = [] t0 = time.time() # del scales[:4] for scale in scales: hs = int(origin_h * scale) ws = int(origin_w * scale) scale_img = cv2.resize(caffe_img, (ws, hs)) input = scale_img.reshape(1, *scale_img.shape) #print("DEBUG 00: input.shape: ",input.shape) ouput = Pnet.predict( input ) # .transpose(0,2,1,3) should add, but seems after process is wrong then. #------------------------------ # --------- rmaria ----------- # For every scale, the we have 2 predictions and 4 bbox regressor coordinates classifier = Pnet.predict( input )[0] # shape (1,outshape1, outshape2,2) where outshape1 = the output of the image after applying KaoPnet forward pass bbox_regressor = Pnet.predict(input)[ 1] # shape (1, outshape1, outshape2,4) #print("DEBUG 0") classifier = np.array(classifier) bbox_regressor = np.array(bbox_regressor) #print("classifier.shape: ",classifier.shape) #print("bbox_regressor.shape: ",bbox_regressor.shape) #print("------------------------") #print("classifier = :",classifier) #print("------------------------") # --------------------------- out.append(ouput) image_num = len(scales) rectangles = [] for i in range(image_num): cls_prob = out[i][0][ 0][:, :, 1] # i = #scale, first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr #print("cls_prob.shape: ",cls_prob.shape) roi = out[i][1][0] out_h, out_w = cls_prob.shape out_side = max(out_h, out_w) # print('calculating img scale #:', i) cls_prob = np.swapaxes(cls_prob, 0, 1) roi = np.swapaxes(roi, 0, 2) rectangle = tools.detect_face_12net(cls_prob, roi, out_side, 1 / scales[i], origin_w, origin_h, threshold[0]) rectangles.extend(rectangle) rectangles = tools.NMS(rectangles, 0.7, 'iou') t1 = time.time() print('time for 12 net is: ', t1 - t0) if len(rectangles) == 0: return rectangles crop_number = 0 out = [] predict_24_batch = [] for rectangle in rectangles: crop_img = caffe_img[int(rectangle[1]):int(rectangle[3]), int(rectangle[0]):int(rectangle[2])] scale_img = cv2.resize(crop_img, (24, 24)) predict_24_batch.append(scale_img) crop_number += 1 predict_24_batch = np.array(predict_24_batch) out = Rnet.predict(predict_24_batch) cls_prob = out[ 0] # first 0 is to select cls, second batch number, always =0 cls_prob = np.array(cls_prob) # convert to numpy roi_prob = out[ 1] # first 0 is to select roi, second batch number, always =0 roi_prob = np.array(roi_prob) rectangles = tools.filter_face_24net(cls_prob, roi_prob, rectangles, origin_w, origin_h, threshold[1]) t2 = time.time() print('time for 24 net is: ', t2 - t1) if len(rectangles) == 0: return rectangles crop_number = 0 predict_batch = [] for rectangle in rectangles: # print('calculating net 48 crop_number:', crop_number) crop_img = caffe_img[int(rectangle[1]):int(rectangle[3]), int(rectangle[0]):int(rectangle[2])] scale_img = cv2.resize(crop_img, (48, 48)) predict_batch.append(scale_img) crop_number += 1 predict_batch = np.array(predict_batch) if (use_custom_model): output = custom_Onet_original(weights_biases_original_model, predict_batch) else: output = Onet.predict(predict_batch) cls_prob = output[0] roi_prob = output[1] pts_prob = output[2] # index # rectangles = tools.filter_face_48net_newdef(cls_prob, roi_prob, pts_prob, rectangles, origin_w, origin_h, # threshold[2]) rectangles = tools.filter_face_48net(cls_prob, roi_prob, pts_prob, rectangles, origin_w, origin_h, threshold[2]) t3 = time.time() print('time for 48 net is: ', t3 - t2) return rectangles