def mergeFunc(meSymInfo):
Db = copy.deepcopy(meSymInfo)
infos = []
j = 0
for i in range(len(Db)):
if i < len(Db) - 3 and j == 0:
# print("is lim:",(Db[i]["name"]=="l" or Db[i]["name"]=="one") and Db[i+1]["name"]=="i" and Db[i+2]["name"]=="m")
# print("is lim:",not judgeIsMiddle(meSymInfo,Db[i],Db[i+1]))
# print("is lim:",not judgeIsMiddle(meSymInfo,Db[i+1],Db[i+2]))
# print((Db[i]["name"]=="l" or Db[i]["name"]=="one" or Db[i]["name"]=="i_like") and Db[i+1]["name"]=="i" and Db[i+2]["name"]=="m" and not judgeIsMiddle(meSymInfo,Db[i],Db[i+1]) and not judgeIsMiddle(meSymInfo,Db[i+1],Db[i+2]))
if (Db[i]["name"] == "l" or Db[i]["name"] == "one" or Db[i]["name"]
== "i_like") and Db[i + 1]["name"] == "i" and Db[
i + 2]["name"] == "m" and not judgeIsMiddle2(
meSymInfo, Db[i],
Db[i + 1]) and not judgeIsMiddle2(
meSymInfo, Db[i + 1], Db[i + 2]):
print("识别出lim")
Db[i]["name"] = "lim"
Db[i]["symPixel"] = Db[i]["symPixel"] + Db[
i + 1]["symPixel"] + Db[i + 2]["symPixel"]
# print(Db[i]["symPixel"])
Db[i]["minRect"] = img_process.getMinRect([Db[i]["symPixel"]
])[0]
# infos.append(Db[i])
j = 3
elif Db[i]["name"] == "c" and Db[i + 1]["name"] == "o" and Db[
i + 2]["name"] == "s" and not judgeIsMiddle2(
meSymInfo, Db[i], Db[i + 1]) and not judgeIsMiddle2(
meSymInfo, Db[i + 1], Db[i + 2]):
print("识别出cos")
Db[i]["name"] = "cos"
Db[i]["symPixel"] = Db[i]["symPixel"] + Db[
i + 1]["symPixel"] + Db[i + 2]["symPixel"]
Db[i]["minRect"] = img_process.getMinRect([Db[i]["symPixel"]
])[0]
# infos.append(Db[i])
j = 3
elif Db[i]["name"] == "s" and Db[i + 1]["name"] == "i" and Db[
i + 2]["name"] == "n" and not judgeIsMiddle2(
meSymInfo, Db[i], Db[i + 1]) and not judgeIsMiddle2(
meSymInfo, Db[i + 1], Db[i + 2]):
print("识别出sin")
Db[i]["name"] = "sin"
Db[i]["symPixel"] = Db[i]["symPixel"] + Db[
i + 1]["symPixel"] + Db[i + 2]["symPixel"]
Db[i]["minRect"] = img_process.getMinRect([Db[i]["symPixel"]
])[0]
# infos.append(Db[i])
j = 3
if j == 0:
infos.append(Db[i])
elif j == 3:
infos.append(Db[i])
j -= 1
else:
j -= 1
return infos
def mergeiandj(unKnownSymbols,points,binary_img):
final_p = np.zeros(len(points))
final_sym = np.zeros(len(unKnownSymbols))
iSyms = []
jSyms = []
for i in range(len(points)):
p = points[i]
for j in range(len(unKnownSymbols)):
if final_sym[j]==1:
continue
sym = unKnownSymbols[j]
xp, yp, wp, hp = img_process.getMinRect([p])[0]
xsym, ysym, wsym, hsym = img_process.getMinRect([sym])[0]
if xsym<xp and xp-(xsym+wsym)<wsym/2 and yp+hp<ysym and hsym/wsym>=1.2 and (ysym-yp-hp)<hsym/2:
sym_image = img_process.getSymbolImg(sym,(xsym, ysym, wsym, hsym), global_config.IMG_SIZE, binary_img)
# plt.imshow(sym_image, cmap="Greys", interpolation="None")
# plt.show()
sym_image = sym_image.reshape((sym_image.shape[0] * sym_image.shape[1],))
candidata_syms, candidata_p = symClasser.predict([sym_image]*3)
can_sym_res = []
can_p_res = []
for c_i in range(len(candidata_syms)):
for c_j in range(len(candidata_syms[c_i])):
if candidata_syms[c_i][c_j] not in can_sym_res:
can_sym_res.append(candidata_syms[c_i][c_j])
can_p_res.append(candidata_p[c_i][c_j])
print(can_sym_res, can_p_res)
if "l" in can_sym_res or "one" in can_sym_res or "t" in can_sym_res:
tmpi = copy.deepcopy(p+sym)
iSyms.append(tmpi)
final_sym[j] = 1
final_p[i] = 1
break
if "j" in can_sym_res:
tmpj = copy.deepcopy(p + sym)
jSyms.append(tmpj)
final_sym[j] = 1
final_p[i] = 1
break
tmp_unKnownSymbols = []
for i in range(len(unKnownSymbols)):
if final_sym[i]==0:
tmp_unKnownSymbols.append(unKnownSymbols[i])
tmp_points = []
for i in range(len(points)):
if final_p[i]==0:
tmp_points.append(points[i])
return tmp_unKnownSymbols,tmp_points,iSyms,jSyms
def getSegmentation(binary_img,ori_image=None):
# ungerSmooth(binary_img)
# plt.imshow(binary_img, cmap="Greys", interpolation="None")
# plt.show()
# maybeImpurity(smooth_binary_img)
binary_img = skeletonize(binary_img).astype(np.int32) # skeletonize函数用于将图像轮廓转为宽度为1个像素的图像
plt.imshow(binary_img, cmap="Greys", interpolation="None")
plt.show()
unKnownSymbols = getSymbols(binary_img)
# # 合并被意外分隔开的字符
unKnownSymbols,mybe_sqrts,mybe_ints = mergeSplitSymbol(unKnownSymbols,binary_img)
# # # 合并等于号
unKnownSymbols,equals,equivs = mergeEqual(unKnownSymbols)
# # 合并除法符号
unKnownSymbols,divs,lines,points,verLines = mergeDiv(unKnownSymbols)
# 合并大于等于和小于等于符号
unKnownSymbols,lines,geps,leqs = mergeLeg_Gep(unKnownSymbols,lines,binary_img)
# # 合并i或j
unKnownSymbols, points, iSyms, jSyms = mergeiandj(unKnownSymbols, points, binary_img)
minRects = img_process.getMinRect(unKnownSymbols+divs+iSyms+jSyms+lines+equals+mybe_sqrts+mybe_ints+points+verLines+geps+leqs)
# # print("smooth_binary_img_shape:",binary_img.shape)
# # print("symbols_len:",len(symbols))
# # print(symbols[0])
drawSymbolsRect(minRects,ori_image)
# for i in range(len(unKnownSymbols)):
# symbol_img = img_process.getSymbolImg(unKnownSymbols[i], minRects[i], global_config.IMG_SIZE, binary_img)
# symbol_img = symbol_img.reshape((symbol_img.shape[0]*symbol_img.shape[1],))
# syns,ps = symClasser.predict([symbol_img])
# print("candidata:",syns,"p:",ps)
# plt.imshow(symbol_img.reshape((global_config.IMG_SIZE,global_config.IMG_SIZE)), cmap="Greys", interpolation="None")
# plt.show()
knownSymbols = {"equal":equals,"equiv":equivs,"div":divs,"line":lines,"verLine":verLines,"i":iSyms,"j":jSyms,"sqrt":mybe_sqrts,"int":mybe_ints,"gep":geps,"leq":leqs}
knownSymbolsInfo = []
for name,syms in knownSymbols.items():
if len(syms)<1:
continue
symsRects = img_process.getMinRect(syms)
for i in range(len(syms)):
symsInfo = {}
symsInfo["name"] = [name]
symsInfo["probability"] = [1]
symsInfo["minRect"] = symsRects[i]
symsInfo["symPixel"] = syms[i]
knownSymbolsInfo.append(symsInfo)
return unKnownSymbols,knownSymbolsInfo,points
def recogSymbols(unknownSymbols, knownSymbolsInfo, binary_img):
global glo_binary_img
glo_binary_img = binary_img
allSymbolsInfo = knownSymbolsInfo
unknownSymbolsMinRects = img_process.getMinRect(unknownSymbols)
for i in range(len(unknownSymbols)):
sym_image = img_process.getSymbolImg(
unknownSymbols[i], unknownSymbolsMinRects[i],
global_config.IMG_SIZE, binary_img) * 0.99
# print(sym_image.tolist())
# plt.imshow(sym_image, cmap="Greys", interpolation="None")
# plt.show()
sym_image = sym_image.reshape(
(sym_image.shape[0] * sym_image.shape[1], ))
can_sym_res = []
can_p_res = []
candidata_syms, candidata_p = symClasser.predict([sym_image] * 3)
for c_i in range(len(candidata_syms)):
for c_j in range(len(candidata_syms[c_i])):
if candidata_syms[c_i][c_j] not in can_sym_res:
can_sym_res.append(candidata_syms[c_i][c_j])
can_p_res.append(candidata_p[c_i][c_j])
if len(can_sym_res) < 1:
print("ERROR: 有未识别出的字符")
symsInfo = {}
symsInfo["name"] = can_sym_res
symsInfo["probability"] = can_p_res
symsInfo["minRect"] = unknownSymbolsMinRects[i]
symsInfo["symPixel"] = unknownSymbols[i]
allSymbolsInfo.append(symsInfo)
# 排好序
allSymbolsInfo = sortSymbolsInfo(allSymbolsInfo)
return allSymbolsInfo
def mergeDiv(symbols):
minRects = img_process.getMinRect(symbols)
if len(minRects)<=0:
return symbols,[],[],[],[]
lines = []
lineMinRects = []
points = []
pointMinRects = []
normalSyms = []
verLines = []
for i in range(len(symbols)):
if recogLine(symbols[i],minRects[i]):
lines.append(symbols[i])
lineMinRects.append(minRects[i])
elif recogPoint(symbols[i],minRects[i]):
points.append(symbols[i])
pointMinRects.append(minRects[i])
elif recogVerLine(symbols[i],minRects[i]):
verLines.append(symbols[i])
else:
normalSyms.append(symbols[i])
print("mybeLineLen:",len(lines))
print("mybePoint_len:", len(points))
print("normalSyms_len:",len(normalSyms))
if len(lines)<1 or len(points)<=1:
return normalSyms,[],lines,points,verLines
# 合并除号
pointFinal = np.zeros(len(points))
lineFinal = np.zeros(len(lines))
for i in range(len(lines)):
for j in range(len(points)-1):
lx,ly,lw,lh = lineMinRects[i]
px,py,pw,ph = pointMinRects[j]
if pointFinal[j]==0 and px>lx and px+pw<lx+lw and pw/lw<0.35 and (ly+lh<py or py+ph<ly) and np.abs(px-lx-lw/2)<5 and lw/pw<20:
for k in range(j+1,len(points)):
p2x,p2y,p2w,p2h = pointMinRects[k]
if pointFinal[k]==0 and p2x > lx and p2x + p2w < lx + lw and p2w / lw < 0.35 and (ly+lh<p2y or p2y+p2h<ly) and np.abs(p2x-lx-lw/2)<5 and lw/p2w<20:
lines[i] = np.append(np.array(lines[i]),np.array(points[j]),axis=0).tolist()
lines[i] = np.append(np.array(lines[i]),np.array(points[k]),axis=0).tolist()
lineFinal[i] = 1
pointFinal[j]=1
pointFinal[k]=1
divs = []
re_lines = []
re_points = []
for i in range(len(lines)):
if lineFinal[i]==0:
re_lines.append(lines[i])
else:
divs.append(lines[i])
for i in range(len(points)):
if pointFinal[i]==0:
re_points.append(points[i])
return normalSyms,divs,re_lines,re_points,verLines
def mergeEqual(symbols):
minRects = img_process.getMinRect(symbols)
if len(symbols)<=1:
return symbols,[],[]
mybeLineSym = []
mybeLineRec = []
notLineSym = []
for i in range(len(symbols)):
symbol = symbols[i]
minRect = minRects[i]
if recogLine(symbol,minRect):
mybeLineSym.append(symbols[i])
mybeLineRec.append(minRects[i])
else:
notLineSym.append(symbols[i])
print("mybeLineLen:",len(mybeLineSym))
if len(mybeLineSym)<=1:
return symbols,[],[]
final = np.zeros(len(mybeLineSym))
for i in range(len(mybeLineSym)-1):
for j in range(i+1,len(mybeLineSym)):
x1,y1,w1,h1 = mybeLineRec[i]
x2,y2,w2,h2 = mybeLineRec[j]
if final[i]<2 and final[j]<2 and np.abs(x1-x2)<=2 and np.abs(w1-w2)<=2:
mybeLineSym[i] = np.append(np.array(mybeLineSym[i]),np.array(mybeLineSym[j]),axis=0).tolist()
mybeLineRec[i] = img_process.getMinRect([mybeLineSym[i]])[0]
if final[i]==1:
final[i] = 2
elif final[i]==0:
final[i] = 1
final[j] = 3
equals = [] #等于符号
equivs = [] #恒等于符号
for i in range(len(mybeLineSym)):
if final[i]==0:
notLineSym.append(mybeLineSym[i])
elif final[i]==1:
equals.append(mybeLineSym[i])
elif final[i]==2:
equivs.append(mybeLineSym[i])
return notLineSym,equals,equivs
def mergeLeg_Gep(unKnownSymbols,lines,binary_img):
final_l = np.zeros(len(lines))
final_sym = np.zeros(len(unKnownSymbols))
lines_minrects = img_process.getMinRect(lines)
syms_minrects = img_process.getMinRect(unKnownSymbols)
leqs = []
geps = []
for i in range(len(lines)):
lx,ly,lw,lh = lines_minrects[i]
for j in range(len(unKnownSymbols)):
sx,sy,sw,sh = syms_minrects[j]
if np.abs(lx-sx)<np.min((lx,sx))/3 and np.abs(lw-sw)<np.min((lw,sw))/3 and 0<ly-sy-sh<sh:
sym_image = img_process.getSymbolImg(unKnownSymbols[j], (sx,sy,sw,sh), global_config.IMG_SIZE, binary_img)
sym_image = sym_image.reshape((sym_image.shape[0] * sym_image.shape[1],))
candidata_syms, candidata_p = symClasser.predict([sym_image]*3)
can_sym_res = [item for sublist in candidata_syms for item in sublist]
# candidata_p = lambda multiple_list: [item for sublist in candidata_p for item in sublist]
print("合并大于等于\小于等于:",can_sym_res)
if "greater" in can_sym_res:
geps.append(copy.deepcopy(lines[i] + unKnownSymbols[j]))
final_l[i]=1
final_sym[j]=1
break
if "less" in can_sym_res:
leqs.append(copy.deepcopy(lines[i] + unKnownSymbols[j]))
final_l[i] = 1
final_sym[j] = 1
break
res_lines = []
for i in range(len(lines)):
if final_l[i]==0:
res_lines.append(lines[i])
res_unKnownSymbols = []
for i in range(len(unKnownSymbols)):
if final_sym[i]==0:
res_unKnownSymbols.append(unKnownSymbols[i])
return res_unKnownSymbols,res_lines,geps,leqs
def translateMe(me):
meStr = ""
idx = 0
for sym in me:
filedsType = global_config.INFTYCDB_3_SHAPES_INFO[
sym["name"]]["filedType"]
if filedsType == "none":
meStr += sym["latex"] + " "
elif sym["name"] == "line":
if len(sym["above"]) > 0 and len(sym["bottom"]) > 0:
meStr += "\\frac{ " + translateMe(
sym["above"]) + "}{ " + translateMe(sym["bottom"]) + "} "
else:
meStr += sym["latex"] + " "
elif sym["name"] == "sqrt":
meStr += "\\sqrt{ " + translateMe(sym["content"]) + "} "
if len(sym["super"]) > 0:
meStr += "^" + translateMe(sym["super"]) + " "
elif sym["name"] == "int":
meStr += "\\int_{ " + translateMe(
sym["rightBottom"]) + "}^{ " + translateMe(sym["super"]) + "} "
elif sym["name"] == "lim":
meStr += "\\lim_{ " + translateMe(sym["bottom"]) + "} "
elif sym["name"] == "Sigma":
meStr += "\\sum_{ " + translateMe(
sym["bottom"]) + "}^{ " + translateMe(sym["above"]) + "} "
elif sym["name"] == "LeftBrace":
meStr += "\\left\{\\begin{array}{ll}"
for brace_i in range(len(sym["right"])):
meStr += "{ " + translateMe(
sym["right"][brace_i]["left_formu"]) + ",} "
if brace_i == len(sym["right"]) - 1:
meStr += "&{ " + translateMe(
sym["right"][brace_i]["right_cond"]) + "} "
else:
meStr += "&{ " + translateMe(
sym["right"][brace_i]["right_cond"]) + "}\\\ "
meStr += "\\end{array}\\right"
else:
tmp = True
if sym["name"] in global_config.NUMBER:
pointMinRects = img_process.getMinRect(points)
print("len(pointMinRects):", len(pointMinRects))
for i in range(len(points)):
px, py, pw, ph = pointMinRects[i]
ox1, oy1, ow1, oh1 = sym["minRect"]
# if sym["name"]=="two":
# print('sym["name"]:',sym["name"],"0<px-ox1-ow1<ow1",0<px-ox1-ow1<ow1,"idx<len(me)-1:",idx<len(me)-1,'me[idx+1]["name"]:',me[idx+1]["name"])
if 0 < px - ox1 - ow1 < ow1 and py + ph - oy1 - oh1 < 5 and py > oy1 and idx < len(
me) - 1 and (me[idx + 1]["name"]
in global_config.NUMBER):
# print('sym["name"]:',sym["name"])
ox2, oy2, ow2, oh2 = me[idx + 1]["minRect"]
if 0 < ox2 - px - pw < ow2 and py + ph - oy2 - oh2 < 5 and py > oy2:
meStr += sym["latex"] + " . "
tmp = False
if tmp:
meStr += sym["latex"] + " "
filedsType_info = global_config.FILEDTYPE_INFO[filedsType]
for filed in filedsType_info:
if len(sym[filed]) > 0:
if filed == "super":
meStr += "^ " + translateMe(sym["super"])
if filed == "sub":
meStr += "_{ " + translateMe(sym["sub"]) + "} "
# 索引值加一
idx += 1
return meStr
def mergeSplitSymbol(par_symbols,binary_img):
symbols = par_symbols
minRects = img_process.getMinRect(symbols)
if len(minRects)<=1:
return symbols
final = np.zeros(len(symbols))
isSqrt = [-1]*len(symbols)
isInt = [-1]*len(symbols)
for i in range(len(minRects)-1):
for j in range(i+1,len(minRects)):
xi,yi,wi,hi = minRects[i]
xj,yj,wj,hj = minRects[j]
# 当符号有包含关系时，使用训练好的模型预测较大的符号是否为根号
if final[j]==0 and final[i]==0 and xi<=xj and yi<=yj and xi+wi>=xj+wj and yi+hi>=yj+hj and isSqrt[i]!=1 and isInt[i]!=1:
if isSqrt[i]==-1 and isInt[i]==-1:
cand_res = []
sym_image = img_process.getSymbolImg(symbols[i], minRects[i], global_config.IMG_SIZE, binary_img)
# plt.imshow(sym_image, cmap="Greys", interpolation="None")
# plt.show()
sym_image = sym_image.reshape((sym_image.shape[0] * sym_image.shape[1],))
for _ in range(6):
candidata_syms,candidata_p = symClasser.predict([sym_image])
cand_res+=candidata_syms[0]
print("cand_res:", cand_res)
# 因为分类器认为pi和sqrt很像，且有包含关系的字符不可能为pi
if "sqrt" or "pi" in cand_res:
isSqrt[i] = 1
break
elif "int" in cand_res:
isInt[i] = 1
break
isSqrt[i] = 0
isInt[i] = 0
final[j] = 1
symbols[i] = symbols[i] + symbols[j]
elif final[j]==0 and final[i]==0 and xi>xj and yi>yj and xi+wi<xj+wj and yi+hi<yj+hj and isSqrt[j]!=1 and isInt[j]!=1:
if isSqrt[j]==-1 and isInt[j]==-1:
cand_res = []
sym_image = img_process.getSymbolImg(symbols[j], minRects[j], global_config.IMG_SIZE, binary_img)
sym_image = sym_image.reshape((sym_image.shape[0] * sym_image.shape[1],))
for _ in range(6):
candidata_syms,candidata_p = symClasser.predict([sym_image])
cand_res+=candidata_syms[0]
print("cand_res:", cand_res)
if "sqrt" or "pi" in cand_res:
isSqrt[j] = 1
print("candidata_syms[0]:",candidata_syms[0])
continue
elif "int" in cand_res:
isInt[j] = 1
print("candidata_syms[0]:", candidata_syms[0])
continue
isSqrt[j] = 0
isInt[j] = 0
final[i] = 1
symbols[j] = symbols[i] + symbols[j]
break
# 重叠情况时，根据两个分割符号的大小以及重叠大小进行合并
elif final[j]==0 and final[i]==0 and not (xi+wi<xj or xj+wj<xi or yi+hi<yj or yj+hj<yi) and isSqrt[i]!=1 and isInt[i]!=1 and isSqrt[j]!=1 and isInt[j]!=1:
print("wj/hj:",wj/hj)
if wi>wj and hi>hj and wj/hj<=2 and wj/hj>=0.5:
if isSqrt[i]==-1 and isInt[i]==-1:
cand_res = []
sym_imagei = img_process.getSymbolImg(symbols[i], minRects[i], global_config.IMG_SIZE,
binary_img)
sym_imagei = sym_imagei.reshape((sym_imagei.shape[0] * sym_imagei.shape[1],))
for _ in range(6):
candidata_syms, candidata_p = symClasser.predict([sym_imagei])
cand_res+=candidata_syms[0]
if "sqrt" in cand_res:
isSqrt[i] = 1
break
elif "int" in cand_res:
isInt[i] = 1
break
isSqrt[i] = 0
isInt[i] = 0
symbols[i] = symbols[i] + symbols[j]
final[j] = 1
elif wi<wj and hi<hj and wi/hi<=2 and wi/hi>=0.5:
if isSqrt[j]==-1 and isInt[j]==-1:
cand_res = []
sym_jmagej = img_process.getSymbolImg(symbols[j], minRects[j], global_config.IMG_SIZE,
binary_img)
sym_jmagej = sym_jmagej.reshape((sym_jmagej.shape[0] * sym_jmagej.shape[1],))
for _ in range(6):
candidata_syms, candidata_p = symClasser.predict([sym_jmagej])
cand_res += candidata_syms[0]
if "sqrt" in cand_res:
isSqrt[j] = 1
continue
elif "int" in cand_res:
isInt[j] = 1
continue
isSqrt[j] = 0
isInt[j] = 0
symbols[j] = symbols[i] + symbols[j]
final[i] = 1
break
elif final[j]==0 and final[i]==0:
if yi>yj and 0<yi-yj-hj<np.min((hi,hj,10)) and (not (xi+wi+1<xj or xj+wj+1<xi)):
symbols[i] = symbols[i] + symbols[j]
final[j] = 1
elif yi<yj and 0<yj-yi-hi<np.min((hi,hj,10)) and (not (xi+wi+1<xj or xj+wj+1<xi)):
symbols[i] = symbols[i] + symbols[j]
final[j] = 1
res = []
mybe_sqrts = []
mybe_ints = []
for i in range(len(symbols)):
if final[i]==0:
if isSqrt[i]==1:
mybe_sqrts.append(symbols[i])
elif isInt[i]==1:
mybe_ints.append(symbols[i])
else:
res.append(symbols[i])
return res,mybe_sqrts,mybe_ints