def froward_image(nets, scaled, original): global rec_t, ext_factor, ext_factorx net, net_ctc = nets print("nets:", nets) print("net: ", net) print("net_ctc :", net_ctc) img = [scaled] draw = img[0] imgo = original im = np.asarray(img, dtype=np.float) im = im / 128.0 im = im - 1.0 # im = im.reshape((3, im.shape[0], im.shape[1])) im = np.swapaxes(im, 1, 3) im = np.swapaxes(im, 2, 3) net.blobs['data'].reshape(im.shape[0], im.shape[1], im.shape[2], im.shape[3]) print("net.blobs['data'] :", net.blobs['data']) net.blobs['data'].data[...] = im print("im: ", im) net.reshape() print("net.reshape(): ", net.reshape()) start = time.time() out = net.forward(start="conv1") end = time.time() seconds = end - start fps = 1 / seconds # print("loc fps:{0}".format(fps)) boxes = out['boxes'] print("boxes: ", boxes) boxes[0, 0, :, 0] *= image_size[0] boxes[0, 0, :, 1] *= image_size[1] normFactor = math.sqrt(image_size[1] * image_size[1] + image_size[0] * image_size[0]) boxes[0, 0, :, 2] *= normFactor boxes[0, 0, :, 3] *= normFactor nms = boxes[0, 0, :, 8] != 1 boxes = boxes[:, :, nms, :] print("boxes before boxes_count: ", boxes) boxes_count = 0 for i in range(0, boxes.shape[2]): det_word = boxes[0, 0, i] if (det_word[0] == 0 and det_word[1] == 0) or det_word[5] < 0.1: break boxes_count += 1 detections_out = [] for i in range(0, boxes_count): det_word = boxes[0, 0, i] boxr = ((det_word[0], det_word[1]), (det_word[2], det_word[3]), det_word[4] * 180 / 3.14) print("boxr : this is r box,", boxr) box = cv2.boxPoints(boxr) print("box : sfter detection count,", box) box = np.array(box, dtype="int") vis.draw_box_points(draw, box, (255, 0, 0)) bbox = cv2.boundingRect(box) bbox = [bbox[0], bbox[1], bbox[2], bbox[3]] bbox[2] += bbox[0] bbox[3] += bbox[1] boxro = [[det_word[0], det_word[1]], [det_word[2] * ext_factorx, det_word[3] * ext_factor], det_word[4] * 180 / 3.14] boxt = get_obox(img[0], original, boxro) print("boxt 1 :", boxt) boxt = ((boxt[0][0], boxt[0][1]), (boxt[1][0], boxt[1][1]), boxt[2]) print("boxt 2 :", boxt) norm2, rot_mat = get_normalized_image(original, boxt) if norm2 is None: continue norm = cv2.cvtColor(norm2, cv2.COLOR_BGR2GRAY) print("Given Norm :", norm) width_scale = 32.0 / norm2.shape[0] width = norm.shape[1] * width_scale best_diff = width bestb = 0 for b in range(0, len(buckets)): if best_diff > abs(width - buckets[b]): best_diff = abs(width - buckets[b]) bestb = b scaled = cv2.resize(norm, (buckets[bestb], 32)) cv2.imshow('norm2', scaled) imtf = np.asarray([scaled], dtype=np.float) imtf = np.asarray(imtf, dtype=np.float) delta = imtf.max() - imtf.min() imtf /= (delta / 2) imtf -= imtf.mean() imtf = np.reshape(imtf, (imtf.shape[0], -1, imtf.shape[1], imtf.shape[2])) net_ctc.blobs['data'].reshape(imtf.shape[0], imtf.shape[1], imtf.shape[2], imtf.shape[3]) net_ctc.blobs['data'].data[...] = imtf outctc = net_ctc.forward() print("outctc : ", outctc) ctc_f = outctc['softmax'] print("ctc_f : ", ctc_f) ctc_f = ctc_f.reshape(ctc_f.shape[0], ctc_f.shape[1], ctc_f.shape[3]) labels = ctc_f.argmax(2) mask = labels > 2 masked = ctc_f.max(2)[mask] mean_conf = np.sum(masked) / masked.shape[0] print("mean_conf : ", mean_conf) if mean_conf < 0.2: vis.draw_box_points(scaled, box, color=(0, 0, 0)) continue if debug: vis.vis_square(imtf[0]) det_text, conf, dec_s = print_seq_ext(labels[:, 0], np.sum(masked)) if len(det_text) == 0: continue if len(det_text) < 3 and mean_conf < 0.8: continue print("detections_out: ", detections_out) detections_out.append((boxt, (det_text, mean_conf, int(det_word[6])))) continue splits_raw = process_splits(det_text, conf, dec_s, norm2, ctc_f, rot_mat, boxt, original, 0, mean_conf, alow_non_dict=True) detections_out.extend(splits_raw) continue if out_raw is not None: out_raw.write(u"{0}|{1}|{2}|{3}|{4}|{5}|{6}|{7}|{8}|{9}|{10}|{11}\n".format( \ 'vid', box[0, 0], box[0, 1], box[1, 0], box[1, 1], \ box[2, 0], box[2, 1], box[3, 0], box[3, 1], det_text, det_text, mean_conf).encode('utf8')) dec2, conf2, dec_splits = cmp_trie.decode_sofmax( ctc_f.reshape(ctc_f.shape[0], ctc_f.shape[2])) best_dict = print_seq2(dec2[0]) if len(best_dict) == 0: continue splits_out = process_splits(best_dict, conf2, dec_splits, norm2, ctc_f, rot_mat, boxt, original, 1, mean_conf) detections_out.extend(splits_out) return detections_out, fps
def process_batch(nets, optim, optim2, image_size, args): global it, mean_loss, mean_rec it += 1 # 迭代次数加一 net, net_ctc = nets net = net.net net_ctc = net_ctc.net net.blobs['data'].reshape(args.batch_size, 1, image_size[1], image_size[0]) # 把一个batch的输入图片reshape net.reshape() optim2.step(1) im = net.blobs['data'].data[...] # shape [batch_size,1,416,416] draw = np.swapaxes(im, 2, 3) draw = np.swapaxes(draw, 1, 3) im_ctc = np.copy(draw) draw += 1 draw *= 128 draw = np.array(draw, dtype="uint8").copy() if args.debug: grid_step = 16 line = 0 while line < image_size[0]: cv2.line(draw[0], (0, line), (image_size[1], line), (128, 128, 128)) line += grid_step boxes = net.blobs['boxes'].data[...] # shape (4, 1, 500, 15) word_gtob = net.blobs['gt_boxes'].data[...] # shape (4, 6, 1, 6) word_txt = net.blobs['gt_labels'].data[...] # shape (4, 6, 1, 14) lines_gtob = net.blobs['line_boxes'].data[...] # shape (4, 1, 1, 5) lines_txt = net.blobs['line_labels'].data[...] # shape (4, 1, 1, 7) #nms = boxeso[:, 0, 0, 8] == 0 #boxes = boxes[:, :, nms, :] boxes[:, 0, :, 0] *= image_size[0] boxes[:, 0, :, 1] *= image_size[1] normFactor = math.sqrt(image_size[1] * image_size[1] + image_size[0] * image_size[0]) boxes[:, 0, :, 2] *= normFactor boxes[:, 0, :, 3] *= normFactor sum_cost = 0 count = 0 labels_gt = [] labels_det = [] gt_to_detection = {} net_ctc.clear_param_diffs() batch_buckets = [] dummy = {} matched_detections = 0 for bid in range(im.shape[0]): # 遍历batchsize下的每一个样本 o_image = net.layers[0].get_image_file_name(bid) o_image = cv2.imread(o_image, cv2.IMREAD_GRAYSCALE) cx = net.layers[0].get_crop(bid, 0) cy = net.layers[0].get_crop(bid, 1) cmx = net.layers[0].get_crop(bid, 2) cmy = net.layers[0].get_crop(bid, 3) o_image = o_image[cy:cmy, cx:cmx] boxes_count = 0 for i in range(0, boxes.shape[2]): det_word = boxes[bid, 0, i] if (det_word[0] == 0 and det_word[1] == 0) or det_word[5] < 0.01: break boxes_count += 1 x = [i for i in range(boxes_count)] #random.shuffle(x) bucket_images = {} batch_buckets.append(bucket_images) word_gto = word_gtob[bid] word_gto_txt = word_txt[bid] gt_count = 0 for gt_no in range(word_gto.shape[0]): gt = word_gto[gt_no, :] gt = gt.reshape(6) gtnum = 1000 * bid + gt_no if gt[5] == -1: #print("ignore gt!") continue gt_count += 1 txt = word_gto_txt[gt_no, :] gtbox = ((gt[0] * image_size[0], gt[1] * image_size[1]), (gt[2] * normFactor, gt[3] * normFactor), gt[4] * 180 / 3.14) gtbox = cv2.boxPoints(gtbox) gtbox = np.array(gtbox, dtype="int") rect_gt = cv2.boundingRect(gtbox) if rect_gt[0] == 0 or rect_gt[ 1] == 0 or rect_gt[0] + rect_gt[2] >= image_size[ 0] or rect_gt[1] + rect_gt[3] >= image_size[1]: continue if gt[3] * normFactor < 3: if args.debug: pass print('too small gt!') continue rect_gt = [rect_gt[0], rect_gt[1], rect_gt[2], rect_gt[3]] rect_gt[2] += rect_gt[0] rect_gt[3] += rect_gt[1] for i in range(0, min(100, boxes_count)): if math.fabs(gt[4] - det_word[4]) > math.pi / 16: continue det_word = boxes[bid, 0, x[i], :] if (det_word[0] == 0 and det_word[1] == 0) or det_word[5] < 0.01: break box = ((det_word[0], det_word[1]), (det_word[2], det_word[3]), det_word[4] * 180 / 3.14) box = cv2.boxPoints(box) if args.debug: boxp = np.array(box, dtype="int") vis.draw_box_points(draw[bid], boxp, color=(0, 255, 0)) box = np.array(box, dtype="int") bbox = cv2.boundingRect(box) bbox = [bbox[0], bbox[1], bbox[2], bbox[3]] bbox[2] += bbox[0] bbox[3] += bbox[1] #rectangle intersection ... inter = intersect(bbox, rect_gt) uni = union(bbox, rect_gt) ratio = area(inter) / float(area(uni)) ratio_gt = area(inter) / float(area(rect_gt)) if ratio_gt < 0.95: continue if ratio < 0.5: continue if not gt_to_detection.has_key(gtnum): gt_to_detection[gtnum] = [0, 0, 0] tupl = gt_to_detection[gtnum] if tupl[0] < ratio: tupl[0] = ratio tupl[1] = x[i] tupl[2] = ratio_gt det_word = boxes[bid, 0, x[i], :] box = ([det_word[0], det_word[1]], [det_word[2], det_word[3]], det_word[4] * 180 / 3.14) boxO = get_obox(im_ctc[bid], o_image, box) boxO = ((boxO[0][0], boxO[0][1]), (boxO[1][0], boxO[1][1]), boxO[2]) norm2, rot_mat = get_normalized_image(o_image, boxO) #norm3, rot_mat = get_normalized_image(im_ctc[bid], ([det_word[0], det_word[1]], [det_word[2] * 1.2, det_word[3] * 1.1], det_word[4] * 180 / 3.14)) if norm2 is None: continue #if norm3 is None: # continue #continue #cv2.imshow('ts', norm2) #cv2.imshow('ts3', norm3) #cv2.waitKey(1) width_scale = 32.0 / norm2.shape[0] width = norm2.shape[1] * width_scale best_diff = width bestb = 0 for b in range(0, len(buckets)): if best_diff > abs(width * 1.3 - buckets[b]): best_diff = abs(width * 1.3 - buckets[b]) bestb = b scaled = cv2.resize(norm2, (buckets[bestb], 32)) scaled = np.asarray(scaled, dtype=np.float) delta = scaled.max() - scaled.min() scaled = (scaled) / (delta / 2) scaled -= scaled.mean() if not bucket_images.has_key(bestb): bucket_images[bestb] = {} bucket_images[bestb]['img'] = [] bucket_images[bestb]['sizes'] = [] bucket_images[bestb]['txt'] = [] bucket_images[bestb]['gt_enc'] = [] dummy[bestb] = 1 else: if args.debug and len(bucket_images[bestb]) > 4: continue elif len(bucket_images[bestb]) > 32: continue gt_labels = [] txt_enc = '' for k in range(txt.shape[1]): if txt[0, k] > 0: if codec_rev.has_key(txt[0, k]): gt_labels.append(codec_rev[txt[0, k]]) else: gt_labels.append(3) txt_enc += unichr(txt[0, k]) else: gt_labels.append(0) if scaled.ndim == 3: scaled = cv2.cvtColor(scaled, cv2.COLOR_BGR2GRAY) if args.debug: cv2.imshow('scaled', scaled) bucket_images[bestb]['sizes'].append(len(gt_labels)) bucket_images[bestb]['gt_enc'].append(gt_labels) bucket_images[bestb]['txt'].append(txt_enc) bucket_images[bestb]['img'].append(scaled) matched_detections += 1 #and learn OCR for bucket in bucket_images.keys(): imtf = np.asarray(bucket_images[bucket]['img'], dtype=np.float) imtf = np.reshape(imtf, (imtf.shape[0], -1, imtf.shape[1], imtf.shape[2])) #imtf = imtf.reshape((imtf.shape[0], imtf.shape[1], imtf.shape[2], 1)) #imtf = np.swapaxes(imtf,1,3) net_ctc.blobs['data'].reshape(imtf.shape[0], imtf.shape[1], imtf.shape[2], imtf.shape[3]) net_ctc.blobs['data'].data[...] = imtf labels = bucket_images[bucket]['gt_enc'] txt = bucket_images[bucket]['txt'] max_len = 0 for l in range(0, len(labels)): max_len = max(max_len, len(labels[l])) for l in range(0, len(labels)): while len(labels[l]) < max_len: labels[l].append(0) labels = np.asarray(labels, np.float) net_ctc.blobs['label'].reshape(labels.shape[0], labels.shape[1]) net_ctc.blobs['label'].data[...] = labels if args.debug: vis.vis_square(imtf[0]) cv2.imshow('draw', draw[0]) cv2.waitKey(5) #optim.step(1) sum_cost += net_ctc.blobs['loss'].data[...] if net_ctc.blobs['loss'].data[...] > 10: #vis.vis_square(imtf[0]) #cv2.imshow('draw', draw[0]) sf = net_ctc.blobs['transpose'].data[...] labels2 = sf.argmax(3) out = utils.print_seq(labels2[:, 0, :]) print(u'{0} --- {1}'.format(out, txt[0])) #cv2.waitKey(5) count += imtf.shape[0] correct_cout = 0 for i in range(len(labels_gt)): det_text = labels_det[i] gt_text = labels_gt[i] if it % 100 == 0: pass #print( u"{0} -- {1}".format(det_text, gt_text).encode('utf8') ) if det_text == gt_text: correct_cout += 1 count = max(count, 1) mean_loss = 0.99 * mean_loss + 0.01 * sum_cost / count mean_rec = mean_rec * 0.99 + 0.01 * correct_cout / float( max(1, len(labels_gt))) #count detection ratio tp = 0 for bid in range(im.shape[0]): word_gto = word_gtob[bid] for gt_no in range(len(word_gto)): gt = word_gto[gt_no] gtnum = 1000 * bid + gt_no if gt_to_detection.has_key(gtnum): tupl = gt_to_detection[gtnum] if tupl[0] > 0.5: tp += 1 loc_recall = tp / float(max(1, gt_count)) if it % 10 == 0: print( '{0} - lr:{1:.3e} ctc:{2:.4f}/{3:.4f} wr:{4:.2f}/{5:.2f}, loc:{6:.2f} {7}' .format(it, 0.0001, sum_cost / count, mean_loss, correct_cout / float(max(1, len(labels_gt))), mean_rec, loc_recall, matched_detections)) if it % snapshot_interval == 0: #optim.snapshot() optim2.snapshot()
def process_batch(nets, optim, optim2, image_size, args): global it, mean_loss, mean_rec net, net_ctc = nets net = net.net net_ctc = net_ctc.net net.blobs['data'].reshape(args.batch_size,1,image_size[1],image_size[0]) net.reshape() it += 1 optim2.step(1) im = net.blobs['data'].data[...] draw = np.swapaxes(im,2,3) draw = np.swapaxes(draw,1,3) im_ctc = np.copy(draw) draw += 1 draw *= 128 draw = np.array(draw, dtype="uint8").copy() if args.debug: grid_step = 16 line = 0 while line < image_size[0]: cv2.line(draw[0], (0, line), (image_size[1], line), (128, 128, 128)) line += grid_step boxes = net.blobs['boxes'].data[...] word_gtob = net.blobs['gt_boxes'].data[...] word_txt = net.blobs['gt_labels'].data[...] lines_gtob = net.blobs['line_boxes'].data[...] lines_txt = net.blobs['line_labels'].data[...] #nms = boxeso[:, 0, 0, 8] == 0 #boxes = boxes[:, :, nms, :] boxes[:, 0, :, 0] *= image_size[0] boxes[:, 0, :, 1] *= image_size[1] normFactor = math.sqrt(image_size[1] * image_size[1] + image_size[0] * image_size[0]) boxes[:, 0, :, 2] *= normFactor boxes[:, 0, :, 3] *= normFactor sum_cost = 0 count = 0 labels_gt = [] labels_det = [] gt_to_detection = {} net_ctc.clear_param_diffs() batch_buckets = [] dummy = {} matched_detections = 0 for bid in range(im.shape[0]): o_image = net.layers[0].get_image_file_name(bid) o_image = cv2.imread(o_image, cv2.IMREAD_GRAYSCALE) cx = net.layers[0].get_crop(bid, 0) cy = net.layers[0].get_crop(bid, 1) cmx = net.layers[0].get_crop(bid, 2) cmy = net.layers[0].get_crop(bid, 3) o_image = o_image[cy:cmy, cx:cmx] boxes_count = 0 for i in range(0, boxes.shape[2]): det_word = boxes[bid, 0, i] if (det_word[0] == 0 and det_word[1] == 0) or det_word[5] < 0.01: break boxes_count += 1 x = [i for i in range(boxes_count)] #random.shuffle(x) bucket_images = {} batch_buckets.append(bucket_images) word_gto = word_gtob[bid] word_gto_txt = word_txt[bid] gt_count = 0 for gt_no in range(word_gto.shape[0]): gt = word_gto[gt_no, :] gt = gt.reshape(6) gtnum = 1000 * bid + gt_no if gt[5] == -1: #print("ignore gt!") continue gt_count += 1 txt = word_gto_txt[gt_no, :] gtbox = ((gt[0] * image_size[0], gt[1] * image_size[1]), (gt[2] * normFactor, gt[3] * normFactor), gt[4] * 180 / 3.14) gtbox = cv2.boxPoints(gtbox) gtbox = np.array(gtbox, dtype="int") rect_gt = cv2.boundingRect(gtbox) if rect_gt[0] == 0 or rect_gt[1] == 0 or rect_gt[0] + rect_gt[2] >= image_size[0] or rect_gt[1] + rect_gt[3] >= image_size[1]: continue if gt[3] * normFactor < 3: if args.debug: #print('too small gt!') vis.draw_box_points(draw[bid], gtbox, color = (255, 255, 0)) cv2.imshow('draw', draw[bid]) continue if args.debug: vis.draw_box_points(draw[bid], gtbox, color = (0, 0, 0), thickness=2) #vis.draw_box_points(draw[bid], gtbox, color = (255, 255, 255)) #cv2.imshow('draw', draw[bid]) rect_gt = [rect_gt[0], rect_gt[1], rect_gt[2], rect_gt[3]] rect_gt[2] += rect_gt[0] rect_gt[3] += rect_gt[1] for i in range(0, min(100, boxes_count)): if math.fabs(gt[4] - det_word[4]) > math.pi / 16: continue det_word = boxes[bid, 0, x[i], :] if (det_word[0] == 0 and det_word[1] == 0) or det_word[5] < 0.01: break box = ((det_word[0], det_word[1]), (det_word[2], det_word[3]), det_word[4] * 180 / 3.14) box = cv2.boxPoints(box) if args.debug: boxp = np.array(box, dtype="int") vis.draw_box_points(draw[bid], boxp, color = (0, 255, 0)) box = np.array(box, dtype="int") bbox = cv2.boundingRect(box) bbox = [bbox[0], bbox[1], bbox[2], bbox[3]] bbox[2] += bbox[0] bbox[3] += bbox[1] #rectangle intersection ... inter = intersect(bbox, rect_gt) uni = union(bbox, rect_gt) ratio = area(inter) / float(area(uni)) ratio_gt = area(inter) / float(area(rect_gt)) if ratio_gt < 0.95: continue if ratio < 0.5: continue if not gt_to_detection.has_key(gtnum): gt_to_detection[gtnum] = [0, 0, 0] tupl = gt_to_detection[gtnum] if tupl[0] < ratio: tupl[0] = ratio tupl[1] = x[i] tupl[2] = ratio_gt det_word = boxes[bid, 0, x[i], :] box = ([det_word[0], det_word[1]], [det_word[2], det_word[3]], det_word[4] * 180 / 3.14) boxO = get_obox(im_ctc[bid], o_image, box) boxO = ((boxO[0][0], boxO[0][1]), (boxO[1][0], boxO[1][1]), boxO[2]) norm2, rot_mat = get_normalized_image(o_image, boxO) #norm3, rot_mat = get_normalized_image(im_ctc[bid], ([det_word[0], det_word[1]], [det_word[2] * 1.2, det_word[3] * 1.1], det_word[4] * 180 / 3.14)) if norm2 is None: continue #if norm3 is None: # continue #continue #cv2.imshow('ts', norm2) #cv2.imshow('ts3', norm3) #cv2.waitKey(1) width_scale = 32.0 / norm2.shape[0] width = norm2.shape[1] * width_scale best_diff = width bestb = 0 for b in range(0, len(buckets)): if best_diff > abs(width * 1.3 - buckets[b]): best_diff = abs(width * 1.3 - buckets[b]) bestb = b scaled = cv2.resize(norm2, (buckets[bestb], 32)) scaled = np.asarray(scaled, dtype=np.float) delta = scaled.max() - scaled.min() scaled = (scaled) / (delta / 2) scaled -= scaled.mean() if not bucket_images.has_key(bestb): bucket_images[bestb] = {} bucket_images[bestb]['img'] = [] bucket_images[bestb]['sizes'] = [] bucket_images[bestb]['txt'] = [] bucket_images[bestb]['gt_enc'] = [] dummy[bestb] = 1 else: if args.debug and len(bucket_images[bestb]) > 4: continue elif len(bucket_images[bestb]) > 32: continue gt_labels = [] txt_enc = '' for k in range(txt.shape[1]): if txt[0, k] > 0: if codec_rev.has_key(txt[0, k]): gt_labels.append( codec_rev[txt[0, k]] ) else: gt_labels.append( 3 ) txt_enc += unichr(txt[0, k]) else: gt_labels.append( 0 ) if scaled.ndim == 3: scaled = cv2.cvtColor(scaled, cv2.COLOR_BGR2GRAY) if args.debug: cv2.imshow('scaled', scaled) bucket_images[bestb]['sizes'].append(len(gt_labels)) bucket_images[bestb]['gt_enc'].append(gt_labels) bucket_images[bestb]['txt'].append(txt_enc) bucket_images[bestb]['img'].append(scaled) matched_detections += 1 #and learn OCR for bucket in bucket_images.keys(): imtf = np.asarray(bucket_images[bucket]['img'], dtype=np.float) imtf = np.reshape(imtf, (imtf.shape[0], -1, imtf.shape[1], imtf.shape[2])) #imtf = imtf.reshape((imtf.shape[0], imtf.shape[1], imtf.shape[2], 1)) #imtf = np.swapaxes(imtf,1,3) net_ctc.blobs['data'].reshape(imtf.shape[0],imtf.shape[1],imtf.shape[2], imtf.shape[3]) net_ctc.blobs['data'].data[...] = imtf labels = bucket_images[bucket]['gt_enc'] txt = bucket_images[bucket]['txt'] max_len = 0 for l in range(0, len(labels)): max_len = max(max_len, len(labels[l])) for l in range(0, len(labels)): while len(labels[l]) < max_len: labels[l].append(0) labels = np.asarray(labels, np.float) net_ctc.blobs['label'].reshape(labels.shape[0], labels.shape[1]) net_ctc.blobs['label'].data[...] = labels if args.debug: vis.vis_square(imtf[0]) cv2.imshow('draw', draw[0]) cv2.waitKey(5) optim.step(1) sum_cost += net_ctc.blobs['loss'].data[...] if net_ctc.blobs['loss'].data[...] > 10: vis.vis_square(imtf[0]) cv2.imshow('draw', draw[0]) sf = net_ctc.blobs['transpose'].data[...] labels2 = sf.argmax(3) out = utils.print_seq(labels2[:, 0, :]) print(u'{0} - {1}'.format(out, txt[0]) ) cv2.waitKey(5) count += imtf.shape[0] correct_cout = 0 for i in range(len(labels_gt)): det_text = labels_det[i] gt_text = labels_gt[i] if it % 100 == 0: print( u"{0} - {1}".format(det_text, gt_text).encode('utf8') ) if det_text == gt_text: correct_cout += 1 count = max(count, 1) mean_loss = 0.99 * mean_loss + 0.01 * sum_cost / count mean_rec = mean_rec * 0.99 + 0.01 * correct_cout / float(max(1, len(labels_gt))) #count detection ratio tp = 0 for bid in range(im.shape[0]): word_gto = word_gtob[bid] for gt_no in range(len(word_gto)): gt = word_gto[gt_no] gtnum = 1000 * bid + gt_no if gt_to_detection.has_key(gtnum): tupl = gt_to_detection[gtnum] if tupl[0] > 0.5: tp += 1 loc_recall = tp / float(max(1, gt_count)) if args.debug: cv2.imshow('draw', draw[0]) if im.shape[0] > 1: cv2.imshow('draw2', draw[1]) cv2.waitKey(10) if it % 10 == 0: print('{0} - lr:{1:.3e} ctc:{2:.4f}/{3:.4f} wr:{4:.2f}/{5:.2f}, loc:{6:.2f} {7}'.format(it, 0.0001, sum_cost / count, mean_loss, correct_cout / float(max(1, len(labels_gt))), mean_rec, loc_recall, matched_detections)) if it % 1000 == 0: optim.snapshot() optim2.snapshot()
def ocr_detections(net_ctc, img, scaled_img, boxes, image_size, r_p_th, out_raw, baseName, debug, split_words, alow_non_dict=False): global rec_t, ext_factor, use_per_image draw = np.copy(scaled_img) # Region layer returns normalized coordiates, convert the generated boxes to image coordinate system boxes[0, 0, :, 0] *= image_size[0] boxes[0, 0, :, 1] *= image_size[1] normFactor = math.sqrt(image_size[1] * image_size[1] + image_size[0] * image_size[0]) boxes[0, 0, :, 2] *= normFactor boxes[0, 0, :, 3] *= normFactor nms_mask = boxes[0, 0, :, 8] != 1 boxes = boxes[:, :, nms_mask, :] # Region layer returns boxes in sorted order by r_{p}, filter out the boxes with r_{p} below threshold value boxes_count = 0 for i in range(0, boxes.shape[2]): det_word = boxes[0, 0, i] if (det_word[0] == 0 and det_word[1] == 0) or det_word[5] < r_p_th: break boxes_count += 1 detections_out = [] for i in range(0, boxes_count): det_word = boxes[0, 0, i] boxr = ((det_word[0], det_word[1]), (det_word[2], det_word[3]), det_word[4] * 180 / 3.14 ) # Convert the rotation parameter to degrees box = cv2.boxPoints( boxr) # Gives the coordinates for 4 points of bounding-box box = np.array(box, dtype="int") if det_word[3] < 5: continue if debug: try: vis.draw_box_points( draw, box, (255, 0, 0)) # Visualize the predicted bounding-boxes except: pass bbox = cv2.boundingRect(box) bbox = [bbox[0], bbox[1], bbox[2], bbox[3]] bbox[2] += bbox[0] # Convert width to right-coordinate bbox[3] += bbox[1] # Convert height to bottom-coordinate boxro = [ [det_word[0], det_word[1]], [det_word[2] * ext_factorx, det_word[3] * ext_factor], det_word[4] * 180 / 3.14 ] # Re-scaling the bounding-box parameters to increase height and width, this helps recognizer boxt = get_obox( scaled_img, img, boxro) # Rescale the predicted bounding box to original image size boxt = ((boxt[0][0], boxt[0][1]), (boxt[1][0], boxt[1][1]), boxt[2]) norm2, rot_mat = get_normalized_image( img, boxt ) # norm2 stores normalized cropped region from original image determined by predicted bounding box if norm2 is None: continue #boxt[2] = boxt[2] * 180 / 3.14 #cv2.imshow('norm2', norm2) #cv2.imshow('draw', draw) if norm2.ndim > 2: norm = cv2.cvtColor( norm2, cv2.COLOR_BGR2GRAY ) # Convert the cropped region to GRAY scale for recognizer else: norm = norm2 # Do nothing if already GRAY scale # Change width for each cropped region, keeping height fixed (32). Map width to closest value from bucket width_scale = 32.0 / norm2.shape[0] width = norm.shape[1] * width_scale best_diff = width bestb = 0 for idx, val in enumerate(buckets): if (buckets[idx] - width) < 0: bestb = idx best_diff = abs(buckets[idx] - width) * 3 continue if best_diff > (buckets[idx] - width): bestb = idx best_diff = (buckets[idx] - width) scaled = cv2.resize( norm, (buckets[bestb], 32)) # Resize cropped region for input for recognizer FCN if scaled.ndim == 3: scaled = cv2.cvtColor(scaled, cv2.COLOR_BGR2GRAY) imtf = np.asarray([scaled], dtype=np.float) imtf = np.asarray(imtf, dtype=np.float) imtf /= 128.0 imtf -= 1 imtf = np.reshape(imtf, (imtf.shape[0], -1, imtf.shape[1], imtf.shape[2])) net_ctc.blobs['data'].reshape( imtf.shape[0], imtf.shape[1], imtf.shape[2], imtf.shape[3] ) # Reshape the recognizer FCN to adapt varying cropped region size net_ctc.blobs['data'].data[ ...] = imtf # Load the data onto recognizer FCN (cropped region data) net_ctc.forward() # Recognizer FCN feed-forward ctc_f = net_ctc.blobs['softmax'].data[...] ctc_f = ctc_f.reshape(ctc_f.shape[0], ctc_f.shape[1], ctc_f.shape[3]) labels = ctc_f.argmax( 2 ) # 3rd dimension (ctc_f[:,:,2]) contains softmax distribution over all the possible characters for each position, thus labels store the index of character with maximum value (probability). mask = labels > 3 masked = ctc_f.max( 2 )[mask] # For each predicted character, fetch the corresponding score mean_conf = np.sum(masked) / masked.shape[ 0] # Mean score for all the predicted characters # Visualize if mean score for predicted characters is less than 0.3 if mean_conf < 0.3: continue if debug: vis.vis_square(imtf[0]) det_text, conf, dec_s = print_seq_ext(labels[:, 0], np.sum(masked)) if not split_words: detections_out.extend([(boxt, (det_text, mean_conf, 1, mean_conf)) ]) continue #print(det_text) #str_lm, pr = cmp_trie.decode_sofmax_lm(ctc_f.reshape(ctc_f.shape[0], ctc_f.shape[2])) #if det_text != str_lm: # print(' Decoding diff: {0} - {1}'.format(det_text, str_lm)) # det_text = str_lm.strip() if len(det_text.strip()) == 0: continue if len(det_text.strip()) <= 3: if mean_conf < 0.6 or det_word[5] < 0.4: continue pr = 1 for k in range(masked.shape[0]): pr = pr * masked[k] pr = math.exp(pr) #pr = math.pow(pr, 1.0/ len(det_text) ) #tex_conf = mean_conf / ctc_f.shape[0] #if tex_conf < 0.1: # continue #print(det_text) #cv2.imshow('norm2', norm2) splits_raw = process_splits( det_text, conf, dec_s, norm2, ctc_f, rot_mat, boxt, img, det_word[5], mean_conf, alow_non_dict=alow_non_dict ) # Process the split and improve the localization results using "space" (' ') predicted by recognizer detections_out.extend(splits_raw) spl = det_text.split(" ") if len(spl) == 1 and cmp_trie.is_dict( spl[0].lower().encode('utf-8')) == 1: continue dec2, conf2, dec_splits = cmp_trie.decode_sofmax( ctc_f.reshape(ctc_f.shape[0], ctc_f.shape[2])) best_dict = print_seq2(dec2[0]) if out_raw is not None and len(det_text) > 2: boxout = cv2.boxPoints(boxt) out_raw.write(u"{0}|{1}|{2}|{3}|{4}|{5}|{6}|{7}|{8}|{9}|{10}|{11}\n".format(\ baseName[:-4],boxout[0, 0],boxout[0, 1], boxout[1, 0], boxout[1, 1], \ boxout[2, 0], boxout[2, 1], boxout[3, 0], boxout[3, 1], det_text, best_dict, mean_conf).encode('utf8')) splits_out = process_splits(best_dict, conf2, dec_splits, norm2, ctc_f, rot_mat, boxt, img, det_word[5], pr, alow_non_dict=False) detections_out.extend(splits_out) #detections_out = nms(detections_out) if out_raw is not None: out_raw.flush() cv2.imshow('draw', draw) cv2.waitKey(10) return detections_out
def froward_image(nets, scaled, original): ''' :param nets: yolo网络,ctc网络 :param scaled:灰度reshape图片 :param original:原始图片 :return: detections_out:[( ((1181.9506549451335, 174.54442087680732), (116.45833333333334, 19.8), -2.3903521532498173), (u'FORQUEuEING', 0.885055888782848, True, 0)),()] (中心(x,y), (宽,高), 旋转角度) fps: 每秒传输帧数 ''' global rec_t, ext_factor, ext_factorx net, net_ctc = nets img = [scaled] # draw = img[0] # imgo = original im = np.asarray(img, dtype=np.float) im = im / 128.0 im = im - 1.0 #im = im.reshape((3, im.shape[0], im.shape[1])) im = np.swapaxes(im, 1, 3) im = np.swapaxes(im, 2, 3) net.blobs['data'].reshape(im.shape[0], im.shape[1], im.shape[2], im.shape[3]) net.blobs['data'].data[...] = im net.reshape() start = time.time() out = net.forward(start="conv1") end = time.time() seconds = end - start fps = 1 / seconds boxes = out['boxes'] #(1, 1, 500, 15) 500个anchor boxes[0, 0, :, 0] *= image_size[0] boxes[0, 0, :, 1] *= image_size[1] normFactor = math.sqrt(image_size[1] * image_size[1] + image_size[0] * image_size[0]) boxes[0, 0, :, 2] *= normFactor boxes[0, 0, :, 3] *= normFactor nms = boxes[0, 0, :, 8] != 1 boxes = boxes[:, :, nms, :] boxes_count = 0 for i in range(0, boxes.shape[2]): det_word = boxes[0, 0, i] if (det_word[0] == 0 and det_word[1] == 0) or det_word[5] < 0.1: break boxes_count += 1 detections_out = [] # 对于每一个检测出来的框(nms之后且分数大于0.1),都识别一次 for i in range(0, boxes_count): det_word = boxes[0, 0, i] boxr = ((det_word[0], det_word[1]), (det_word[2], det_word[3]), det_word[4] * 180 / 3.14) # 用预测出来的 x,y h, w, angle box = cv2.boxPoints(boxr) # 得到四个点的坐标 box = np.array(box, dtype="int") #vis.draw_box_points(draw, box, (255, 0, 0)) bbox = cv2.boundingRect(box) # 变成最小矩形框, x, y, w, h bbox = [bbox[0], bbox[1], bbox[2], bbox[3]] bbox[2] += bbox[0] bbox[3] += bbox[1] # 后面也没用到bbox啊 boxro = [[det_word[0], det_word[1]], [det_word[2] * ext_factorx, det_word[3] * ext_factor], det_word[4] * 180 / 3.14] boxt = get_obox(img[0], original, boxro) boxt = ((boxt[0][0], boxt[0][1]), (boxt[1][0], boxt[1][1]), boxt[2]) norm2, rot_mat = get_normalized_image(original, boxt) if norm2 is None: continue norm = cv2.cvtColor(norm2, cv2.COLOR_BGR2GRAY) width_scale = 32.0 / norm2.shape[0] width = norm.shape[1] * width_scale best_diff = width bestb = 0 for b in range(0, len(buckets)): if best_diff > abs(width - buckets[b]): best_diff = abs(width - buckets[b]) bestb = b scaled = cv2.resize(norm, (buckets[bestb], 32)) imtf = np.asarray([scaled], dtype=np.float) imtf = np.asarray(imtf, dtype=np.float) delta = imtf.max() - imtf.min() imtf /= (delta / 2) imtf -= imtf.mean() imtf = np.reshape(imtf, (imtf.shape[0], -1, imtf.shape[1], imtf.shape[2])) net_ctc.blobs['data'].reshape(imtf.shape[0], imtf.shape[1], imtf.shape[2], imtf.shape[3]) net_ctc.blobs['data'].data[...] = imtf outctc = net_ctc.forward() # ['loss', 'softmax'] ctc_f = outctc['softmax'] # shape (48, 1, 1, 141) ctc_f = ctc_f.reshape(ctc_f.shape[0], ctc_f.shape[1], ctc_f.shape[3]) labels = ctc_f.argmax(2) #(48, 1) mask = labels > 2 masked = ctc_f.max(2)[mask] mean_conf = np.sum(masked) / masked.shape[0] if mean_conf < 0.2: vis.draw_box_points(scaled, box, color=(0, 0, 0)) continue if debug: vis.vis_square(imtf[0]) det_text, conf, dec_s = print_seq_ext( labels[:, 0], np.sum(masked)) # 得到det_text,识别出来的字 if len(det_text) == 0: continue if len(det_text) < 3 and mean_conf < 0.8: continue splits_raw = process_splits(det_text, conf, dec_s, norm2, ctc_f, rot_mat, boxt, original, 0, mean_conf, alow_non_dict=True) detections_out.extend(splits_raw) dec2, conf2, dec_splits = cmp_trie.decode_sofmax( ctc_f.reshape(ctc_f.shape[0], ctc_f.shape[2])) best_dict = print_seq2(dec2[0]) # 这个是什么?这里得到的是 “” 所以下面就continue了 if len(best_dict) == 0: continue splits_out = process_splits(best_dict, conf2, dec_splits, norm2, ctc_f, rot_mat, boxt, original, 1, mean_conf) detections_out.extend(splits_out) return detections_out, fps
def ocr_detections(net_ctc, img, scaled_img, boxes, image_size, r_p_th, out_raw, baseName, debug, split_words, alow_non_dict=False): global rec_t, ext_factor, use_per_image draw = np.copy(scaled_img) # Region layer returns normalized coordiates, convert the generated boxes to image coordinate system boxes[0, 0, :, 0] *= image_size[0] boxes[0, 0, :, 1] *= image_size[1] normFactor = math.sqrt(image_size[1] * image_size[1] + image_size[0] * image_size[0]) boxes[0, 0, :, 2] *= normFactor boxes[0, 0, :, 3] *= normFactor nms_mask = boxes[0, 0, :, 8] != 1 boxes = boxes[:, :, nms_mask, :] # Region layer returns boxes in sorted order by r_{p}, filter out the boxes with r_{p} below threshold value boxes_count = 0 for i in range(0, boxes.shape[2]): det_word = boxes[0, 0, i] if (det_word[0] == 0 and det_word[1] == 0) or det_word[5] < r_p_th: break boxes_count += 1 detections_out = [] for i in range(0, boxes_count): det_word = boxes[0, 0, i] boxr = ((det_word[0], det_word[1]), (det_word[2], det_word[3]), det_word[4] * 180 / 3.14) # Convert the rotation parameter to degrees box = cv2.boxPoints(boxr) # Gives the coordinates for 4 points of bounding-box box = np.array(box, dtype="int") if det_word[3] < 5: continue if debug: try: vis.draw_box_points(draw, box, (255, 0, 0)) # Visualize the predicted bounding-boxes except: pass bbox = cv2.boundingRect(box) bbox = [bbox[0], bbox[1], bbox[2], bbox[3]] bbox[2] += bbox[0] # Convert width to right-coordinate bbox[3] += bbox[1] # Convert height to bottom-coordinate boxro = [[det_word[0], det_word[1]], [det_word[2] * ext_factorx, det_word[3] * ext_factor], det_word[4] * 180 / 3.14] # Re-scaling the bounding-box parameters to increase height and width, this helps recognizer boxt = get_obox(scaled_img, img, boxro) # Rescale the predicted bounding box to original image size boxt = ((boxt[0][0], boxt[0][1]), (boxt[1][0], boxt[1][1]), boxt[2]) norm2, rot_mat = get_normalized_image(img, boxt) # norm2 stores normalized cropped region from original image determined by predicted bounding box if norm2 is None: continue #boxt[2] = boxt[2] * 180 / 3.14 #cv2.imshow('norm2', norm2) #cv2.imshow('draw', draw) if norm2.ndim > 2: norm = cv2.cvtColor(norm2, cv2.COLOR_BGR2GRAY ) # Convert the cropped region to GRAY scale for recognizer else: norm = norm2 # Do nothing if already GRAY scale # Change width for each cropped region, keeping height fixed (32). Map width to closest value from bucket width_scale = 32.0 / norm2.shape[0] width = norm.shape[1] * width_scale best_diff = width bestb = 0 for idx, val in enumerate(buckets): if (buckets[idx] - width) < 0 : bestb = idx best_diff = abs(buckets[idx] - width) * 3 continue if best_diff > (buckets[idx] - width): bestb = idx best_diff = (buckets[idx] - width) scaled = cv2.resize(norm, (buckets[bestb], 32)) # Resize cropped region for input for recognizer FCN if scaled.ndim == 3: scaled = cv2.cvtColor(scaled, cv2.COLOR_BGR2GRAY) imtf = np.asarray([scaled], dtype=np.float) imtf = np.asarray(imtf, dtype=np.float) imtf /= 128.0 imtf -= 1 imtf = np.reshape(imtf, (imtf.shape[0], -1, imtf.shape[1], imtf.shape[2])) net_ctc.blobs['data'].reshape(imtf.shape[0],imtf.shape[1],imtf.shape[2], imtf.shape[3]) # Reshape the recognizer FCN to adapt varying cropped region size net_ctc.blobs['data'].data[...] = imtf # Load the data onto recognizer FCN (cropped region data) net_ctc.forward() # Recognizer FCN feed-forward ctc_f = net_ctc.blobs['softmax'].data[...] ctc_f = ctc_f.reshape(ctc_f.shape[0], ctc_f.shape[1], ctc_f.shape[3]) labels = ctc_f.argmax(2) # 3rd dimension (ctc_f[:,:,2]) contains softmax distribution over all the possible characters for each position, thus labels store the index of character with maximum value (probability). mask = labels > 3 masked = ctc_f.max(2)[mask] # For each predicted character, fetch the corresponding score mean_conf = np.sum(masked) / masked.shape[0] # Mean score for all the predicted characters # Visualize if mean score for predicted characters is less than 0.3 if mean_conf < 0.3: continue if debug: vis.vis_square(imtf[0]) det_text, conf, dec_s = print_seq_ext(labels[:, 0], np.sum(masked) ) if not split_words: detections_out.extend( [(boxt, (det_text, mean_conf, 1, mean_conf) )] ) continue #print(det_text) #str_lm, pr = cmp_trie.decode_sofmax_lm(ctc_f.reshape(ctc_f.shape[0], ctc_f.shape[2])) #if det_text != str_lm: # print(' Decoding diff: {0} - {1}'.format(det_text, str_lm)) # det_text = str_lm.strip() if len(det_text.strip()) == 0: continue if len(det_text.strip()) <= 3: if mean_conf < 0.6 or det_word[5] < 0.4: continue pr = 1 for k in range(masked.shape[0]): pr = pr * masked[k] pr = math.exp(pr) #pr = math.pow(pr, 1.0/ len(det_text) ) #tex_conf = mean_conf / ctc_f.shape[0] #if tex_conf < 0.1: # continue #print(det_text) #cv2.imshow('norm2', norm2) splits_raw = process_splits(det_text, conf, dec_s, norm2, ctc_f, rot_mat, boxt, img, det_word[5], mean_conf, alow_non_dict = alow_non_dict) # Process the split and improve the localization results using "space" (' ') predicted by recognizer detections_out.extend( splits_raw ) spl = det_text.split(" ") if len(spl) == 1 and cmp_trie.is_dict(spl[0].lower().encode('utf-8')) == 1: continue dec2, conf2, dec_splits = cmp_trie.decode_sofmax(ctc_f.reshape(ctc_f.shape[0], ctc_f.shape[2])) best_dict = print_seq2(dec2[0]) if out_raw is not None and len(det_text) > 2: boxout = cv2.boxPoints(boxt) out_raw.write(u"{0}|{1}|{2}|{3}|{4}|{5}|{6}|{7}|{8}|{9}|{10}|{11}\n".format(\ baseName[:-4],boxout[0, 0],boxout[0, 1], boxout[1, 0], boxout[1, 1], \ boxout[2, 0], boxout[2, 1], boxout[3, 0], boxout[3, 1], det_text, best_dict, mean_conf).encode('utf8')) splits_out = process_splits(best_dict, conf2, dec_splits, norm2, ctc_f, rot_mat, boxt, img, det_word[5], pr, alow_non_dict=False) detections_out.extend( splits_out ) #detections_out = nms(detections_out) if out_raw is not None: out_raw.flush() cv2.imshow('draw', draw) cv2.waitKey(10) return detections_out