def textline_extract(image, prediction, threshold=0.3):
h, w, _ = image.shape
cls = np.array(prediction[0])
regr = np.array(prediction[1])
cls_prod = np.array(prediction[2])
anchor = utils.gen_anchor((int(h / 16), int(w / 16)), 16)
bbox = utils.bbox_transfor_inv(anchor, regr)
bbox = utils.clip_box(bbox, [h, w])
#score > 0.7
fg = np.where(cls_prod[0, :, 1] > threshold)[0]
select_anchor = bbox[fg, :]
select_score = cls_prod[0, fg, 1]
select_anchor = select_anchor.astype('int32')
#filter size
keep_index = utils.filter_bbox(select_anchor, 16)
#nsm
select_anchor = select_anchor[keep_index]
select_score = select_score[keep_index]
select_score = np.reshape(select_score, (select_score.shape[0], 1))
nmsbox = np.hstack((select_anchor, select_score))
keep = utils.nms(nmsbox, 0.3)
select_anchor = select_anchor[keep]
select_score = select_score[keep]
#text line
textConn = text_connect.TextProposalConnector()
text = textConn.get_text_lines(select_anchor, select_score, [h, w])
text = list(text.astype('int32'))
return text
def textline_extract(self,image):
assert len(image.shape) == 3
h,w,c= image.shape
#zero-center by mean pixel
m_img = image - utils.IMAGE_MEAN
# m_img = np.expand_dims(m_img,axis=0)
p_data0 = {'inputs_1':m_img}
param = {"instances":[p_data0]}
predict_request = json.dumps(param,cls=NumpyEncoder)
start = time.time()
response = requests.post(self.server_url, data=predict_request)
response.raise_for_status()
prediction = response.json()['predictions'][0]
# result = self.basemodel.predict(m_img)
cls = np.array(prediction['output0'])
regr = np.array(prediction['output1'])
cls_prod = np.array(prediction['output2'])
cls = np.expand_dims(cls,axis=0)
regr = np.expand_dims(regr,axis=0)
cls_prod = np.expand_dims(cls_prod,axis=0)
anchor = utils.gen_anchor((int(h/16),int(w/16)),16)
bbox = utils.bbox_transfor_inv(anchor,regr)
bbox = utils.clip_box(bbox,[h,w])
#score > 0.7
fg = np.where(cls_prod[0,:,1]>self.threshold)[0]
select_anchor = bbox[fg,:]
select_score = cls_prod[0,fg,1]
select_anchor = select_anchor.astype('int32')
#filter size
keep_index = utils.filter_bbox(select_anchor,16)
#nsm
select_anchor = select_anchor[keep_index]
select_score = select_score[keep_index]
select_score = np.reshape(select_score,(select_score.shape[0],1))
nmsbox = np.hstack((select_anchor,select_score))
keep = utils.nms(nmsbox,0.3)
select_anchor = select_anchor[keep]
select_score = select_score[keep]
#text line
textConn = text_connect.TextProposalConnector()
text = textConn.get_text_lines(select_anchor,select_score,[h,w])
end = time.time()
text = list(text.astype('int32'))
return text
def textline_extract(self,image):
assert len(image.shape) == 3
h,w,c= image.shape
#zero-center by mean pixel
m_img = image - utils.IMAGE_MEAN
m_img = np.expand_dims(m_img,axis=0)
start = time.time()
result = self.basemodel.predict(m_img)
cls,regr,cls_prod = result
anchor = utils.gen_anchor((int(h/16),int(w/16)),16)
bbox = utils.bbox_transfor_inv(anchor,regr)
bbox = utils.clip_box(bbox,[h,w])
#score > 0.7
fg = np.where(cls_prod[0,:,1]>self.threshold)[0]
select_anchor = bbox[fg,:]
select_score = cls_prod[0,fg,1]
select_anchor = select_anchor.astype('int32')
#filter size
keep_index = utils.filter_bbox(select_anchor,16)
#nsm
select_anchor = select_anchor[keep_index]
select_score = select_score[keep_index]
select_score = np.reshape(select_score,(select_score.shape[0],1))
nmsbox = np.hstack((select_anchor,select_score))
keep = utils.nms(nmsbox,0.3)
select_anchor = select_anchor[keep]
select_score = select_score[keep]
#text line
textConn = text_connect.TextProposalConnector()
text = textConn.get_text_lines(select_anchor,select_score,[h,w])
end = time.time()
text = list(text.astype('int32'))
return text
def _detect(image):
'''
:param image: numpy array image, h,w,c
:return: text location,a list which contains a list of x, y coors, prob. with shape (#lines, 9)
'''
h, w, c = image.shape
image = image - config.IMAGE_MEAN
image = np.expand_dims(image, axis=0) # batch_sz, h, w, c
_, regr, cls_prob = infer_model.predict(image)
anchor = utils.gen_anchor((int(h / 16), int(w / 16)), 16)
bbox = utils.bbox_transfor_inv(anchor, regr)
bbox = utils.clip_box(bbox, [h, w])
fg = np.where(cls_prob[0, :, 1] > 0.7)[0]
select_anchor = bbox[fg, :]
select_score = cls_prob[0, fg, 1]
select_anchor = select_anchor.astype(np.int32)
keep_index = utils.filter_bbox(select_anchor, 16)
# nsm
select_anchor = select_anchor[keep_index]
select_score = select_score[keep_index]
select_score = np.reshape(select_score, (select_score.shape[0], 1))
nmsbox = np.hstack((select_anchor, select_score))
keep = utils.nms(nmsbox, 0.3)
select_anchor = select_anchor[keep]
select_score = select_score[keep]
# text line
textConn = TextProposalConnectorOriented()
text = textConn.get_text_lines(select_anchor, select_score, [h, w])
return text
infer_model = ctpn_model.create_ctpn_model() infer_model.load_weights(ctpn_weight_path) cls, regr, cls_prob = infer_model.predict(image) anchor = utils.gen_anchor((int(h / 16), int(w / 16)), 16) bbox = utils.bbox_transfor_inv(anchor, regr) bbox = utils.clip_box(bbox, [h, w]) fg = np.where(cls_prob[0, :, 1] > 0.7)[0] select_anchor = bbox[fg, :] select_score = cls_prob[0, fg, 1] select_anchor = select_anchor.astype(np.int32) keep_index = utils.filter_bbox(select_anchor, 16) # nsm select_anchor = select_anchor[keep_index] select_score = select_score[keep_index] select_score = np.reshape(select_score, (select_score.shape[0], 1)) nmsbox = np.hstack((select_anchor, select_score)) keep = utils.nms(nmsbox, 0.3) select_anchor = select_anchor[keep] select_score = select_score[keep] #text line textConn = TextProposalConnectorOriented() text = textConn.get_text_lines(select_anchor, select_score, [h, w]) ################### recognition ##########################
if text_flag:
grnd = {}
elif sen_score[i] < sen_score_thr or np.mean(
EN_score[i, men_dict['idx']]) < en_score_thr:
grnd = {}
else:
heatmap = np.average(EN_heat[i, men_dict['idx'], :],
weights=EN_score[i, men_dict['idx']],
axis=0)
img_embd = np.average(IMG_embd[i, men_dict['idx'], :],
weights=EN_score[i, men_dict['idx']],
axis=0)
orig_img_shape = img_info_batch[i][1]
bbox_dict = utils.heat2bbox(heatmap, orig_img_shape)
bbox, bbox_norm, score = utils.filter_bbox(
bbox_dict=bbox_dict, order='xyxy')
grnd = {
'bbox': bbox,
'bbox_norm': bbox_norm,
'bbox_score': score,
'heatmap': heatmap,
'sen-img-score': sen_score[i],
'men-img-score': EN_score[i, men_dict['idx']],
'grounding_features': img_embd
}
men_dict = {
mention: {
'grounding': {
orig_img_id: grnd
},