def get_index_from_region(coorRegion, imageRegion, coorCCRegion, suffix):
bdResult = {}
index1 = []
index2 = []
bdResult = bd_rec(imageRegion, client, imgForm=suffix, api='accurate')
if "words_result" not in bdResult:
print(bdResult)
print("BaiDu recognized error, exit...")
exit(0)
if len(bdResult["words_result"]) == 0:
return index1, index2
words = []
for j, w in enumerate(bdResult["words_result"]):
words.append((j, dic2length(w["location"])))
graphs = []
for j, c in enumerate(coorCCRegion):
if c[2] > heightOfLine * 3 or c[3] > heightOfLine * 1.5:
# if c[2] > heightOfLine*0.5 or c[3] > heightOfLine*0.5:
graphs.append(c)
for w in words:
flag = False
for g in graphs:
g = expand_box(g)
if is_within(w[1], g):
index2.append(bdResult["words_result"][w[0]]["words"])
flag = True
break
if not flag:
index1.append(bdResult["words_result"][w[0]]["words"])
return index1, index2
def get_index_from_region(coorRegion, imageRegion, coorCCRegion, suffix):
"""Gets index from region.
Args:
coorRegion: Coordinate of region.
imageRegion: The image data of region.
coorCCRegion: The coordinate of connected components in region.
suffix: Image name's suffix.
Returns:
Index out graph and Index in graph.
"""
bdResult = {}
index1 = []
index2 = []
bdResult = bd_rec(imageRegion, client, imgForm=suffix, api='accurate')
if "words_result" not in bdResult:
print(bdResult)
print("BaiDu recognized error, exit...")
exit(0)
if len(bdResult["words_result"]) == 0:
return index1, index2
words = []
for j, w in enumerate(bdResult["words_result"]):
words.append((j, dic2length(w["location"])))
graphs = []
for j, c in enumerate(coorCCRegion):
if c[2] > heightOfLine * 3 or c[3] > heightOfLine * 1.5:
# if c[2] > heightOfLine*0.5 or c[3] > heightOfLine*0.5:
graphs.append(c)
for w in words:
flag = False
for g in graphs:
g = expand_box(g)
if is_within(w[1], g):
index2.append(bdResult["words_result"][w[0]]["words"])
flag = True
break
if not flag:
index1.append(bdResult["words_result"][w[0]]["words"])
return index1, index2
def get_items_v02(region, image, coorCC, name):
"""Gets items from regions.
Args:
region: The coordinate of region.
image: Image data.
coorCC: The coordinate of connected components.
name: Image name.
Returns:
A dictionary of result.
"""
result = {}
result["paper_name"] = name
result["questions"] = []
itemDir = os.path.join(desDir, os.path.splitext(name)[0])
suffix = os.path.splitext(name)[1]
if os.path.exists(itemDir):
# os.removedirs(itemDir)
shutil.rmtree(itemDir)
os.mkdir(itemDir)
for i, r in enumerate(region):
question = {}
r = (r[0], int(r[1] - heightOfLine * 0.2), r[2], r[3])
if r[2] - r[0] < 15 or r[3] - r[1] < 15:
continue
imageRegion = image[r[1]:r[3], r[0]:r[2]]
coorCCRegion = [
coor for coor in coorCC if is_within(coor, coor2length(r))
]
coorCCRegion = [(c[0] - r[0], c[1] - r[1], c[2], c[3])
for c in coorCCRegion]
# question = get_items_from_region(r, imageRegion, coorCCRegion, itemDir, suffix, idGraph)
index1, index2 = get_index_from_region(r, imageRegion, coorCCRegion,
suffix)
question["index_out_graph"] = index1
question["index_in_graph"] = index2
savePath = os.path.join(itemDir, str(i) + '.png')
question["question_image"] = savePath
cv2.imwrite(savePath, imageRegion)
result["questions"].append(question)
return result
def get_items_v02(region, image, coorCC, name):
"""
"""
result = {}
result["paper_name"] = name
result["questions"] = []
itemDir = os.path.join(desDir, os.path.splitext(name)[0])
suffix = os.path.splitext(name)[1]
if os.path.exists(itemDir):
# os.removedirs(itemDir)
shutil.rmtree(itemDir)
os.mkdir(itemDir)
# idxCC = index.Index()
# for j,c in enumerate(coorCC):
# idxCC.insert(j, length2coor(c))
for i, r in enumerate(region):
question = {}
r = (r[0], int(r[1] - heightOfLine * 0.2), r[2], r[3])
if r[2] - r[0] < 15 or r[3] - r[1] < 15:
continue
imageRegion = image[r[1]:r[3], r[0]:r[2]]
coorCCRegion = [
coor for coor in coorCC if is_within(coor, coor2length(r))
]
coorCCRegion = [(c[0] - r[0], c[1] - r[1], c[2], c[3])
for c in coorCCRegion]
# question = get_items_from_region(r, imageRegion, coorCCRegion, itemDir, suffix, idGraph)
index1, index2 = get_index_from_region(r, imageRegion, coorCCRegion,
suffix)
question["index_out_graph"] = index1
question["index_in_graph"] = index2
savePath = os.path.join(itemDir, str(i) + '.png')
question["question_image"] = savePath
cv2.imwrite(savePath, imageRegion)
result["questions"].append(question)
return result
def get_items_from_region(coorRegion, imageRegion, coorCCRegion, itemDir,
suffix, idGraph):
"""
Args:
coorRegion: The coordinate of region.
imageRegion: The image of region.
coorCCRegion: The coordinate of CC in region.
suffix: Image form suffix.
idGraph: The id of graph in paper.
Returns:
A dic of question result.
"""
xiaoti = {}
bdResult = {}
bdResult = bd_rec(imageRegion, client, imgForm=suffix, api='accurate')
title = ''
if "words_result" in bdResult and len(
bdResult["words_result"]) > 0: #结果返回错误处理
title = bdResult["words_result"][0]["words"]
# i = 0
# if "words_result" in bdResult:
# while i < len(bdResult["words_result"]) and \
# not title.endswith(('。', '？', ')')):
# title += bdResult["words_result"][i]["words"]
# i += 1
xiaoti["title"] = title
xiaoti["location"] = coor2dic(coorRegion)
xiaoti["content"] = []
if is_special(title):
locationOfTitle = bdResult["words_result"][0]["location"]
boxOfGraph = (10, dic2coor(locationOfTitle)[3] + 10,
coorRegion[2] - coorRegion[0] - 10,
coorRegion[3] - coorRegion[1] - 10)
itemText = {}
itemText["id"] = 0
itemText["location"] = locationOfTitle
itemText["type"] = "text"
itemText["value"] = title
xiaoti["content"].append(itemText)
itemGraph = {}
itemPath = os.path.join(itemDir, str(idGraph) + '.png')
cv2.imwrite(
itemPath, imageRegion[boxOfGraph[1]:boxOfGraph[3],
boxOfGraph[0]:boxOfGraph[2]])
itemGraph["id"] = 1
itemGraph["location"] = coor2dic(boxOfGraph)
itemGraph["type"] = "graph"
itemGraph["value"] = itemPath
xiaoti["content"].append(itemGraph)
return xiaoti
wordBoxes = []
if "words_result" in bdResult:
for j, w in enumerate(bdResult["words_result"]):
wordBoxes.append((j, dic2length(w["location"])))
graphBoxes = []
for j, b in wordBoxes:
# print(j,b)
b = expand_box(b)
coorCCRegion = [c for c in coorCCRegion if not is_within(c, b)]
n = 0
for j, c in enumerate(coorCCRegion):
# if c[2] > heightOfLine*3 or c[3] > heightOfLine*1.5:
if c[2] > heightOfLine * 0.5 or c[3] > heightOfLine * 0.5:
graphBoxes.append((n, c))
n += 1
c = expand_box(c)
wordBoxes = [it for it in wordBoxes if not is_within(it[1], c)]
mixBoxes = []
for j, w in enumerate(wordBoxes.copy()):
if w[1][2] > len(
bdResult["words_result"][w[0]]["words"]) * heightOfLine * 1.5:
del (wordBoxes[j - len(mixBoxes)])
mixBoxes.append((n, w[1]))
n += 1
# Fuses image
graphBoxes += mixBoxes
idxWords = index.Index()
for j, b in enumerate(wordBoxes):
if b[1][2] < heightOfLine * 5:
idxWords.insert(b[0], length2coor(b[1]))
wordBoxes = [b for b in wordBoxes if b[1][2] >= heightOfLine * 5]
count = 0
idDeleted = []
for j, b in graphBoxes.copy():
bEx = expand_box(b)
items = list(idxWords.intersection(length2coor(bEx), objects=True))
if len(items) > 0:
boxes = [num2int(i.bbox) for i in items] + [length2coor(b)]
box = fuse_box(boxes)
for i, it in enumerate(items):
# del(wordBoxes[it.id-i-len(idDeleted)])
idxWords.delete(it.id, it.bbox)
idDeleted.append(it.id)
del (graphBoxes[j - count])
count += 1
graphBoxes.append((n, box))
n += 1
if len(idxWords.leaves()[0][1]) > 0:
wordItems = list(
idxWords.intersection(idxWords.get_bounds(), objects=True))
wordBoxes += [(i.id, coor2length(num2int(i.bbox))) for i in wordItems]
idxGraphs = index.Index()
for j, b in enumerate(graphBoxes):
idxGraphs.insert(j, length2coor(b[1]))
count = 0
for j, b in enumerate(graphBoxes.copy()):
bEx = expand_box(b[1])
items = list(idxGraphs.intersection(length2coor(bEx), objects=True))
if len(items) > 0:
boxes = [num2int(i.bbox) for i in items] + [length2coor(b[1])]
box = fuse_box(boxes)
for i, it in enumerate(items):
# if it.id-i not in range(0, len(graphBoxes)):
# print(it.id-i)
# print(graphBoxes)
# continue
idxGraphs.delete(it.id, it.bbox)
idxGraphs.insert(len(graphBoxes) + count, length2coor(box))
count += 1
# graphBoxes.append((n, box))
# n += 1
# print(idxGraphs.get_bounds())
# print(idxGraphs.leaves())
# exit(0)
if len(graphBoxes) > 0:
graphItems = list(
idxGraphs.intersection(idxGraphs.get_bounds(), objects=True))
graphBoxes = [(i.id, coor2length(num2int(i.bbox))) for i in graphItems]
for j, b in graphBoxes:
bEx = expand_box(b)
wordBoxes = [
it for it in wordBoxes
if not is_within(it[1], bEx) and it[1][3] < it[1][2] * 1.5
]
items = [(it, 0) for it in wordBoxes] + [(it, 1) for it in graphBoxes]
idxItems = index.Index()
for i, it in enumerate(items):
idxItems.insert(it[0][0], length2coor(it[0][1]), obj=it[1])
count = 0
# flag = False
while True:
tempItems = []
for it in items:
itemInter = list(
idxItems.intersection(length2coor(it[0][1]), objects=True))
if len(itemInter) > 1:
boxes = [num2int(i.bbox) for i in itemInter]
box = fuse_box(boxes)
for it1 in itemInter:
idxItems.delete(it1.id, it1.bbox)
idxItems.insert(len(items) + count, length2coor(box), obj=1)
count += 1
itemExs = list(
idxItems.intersection(idxItems.get_bounds(), objects=True))
tempItems = [((i.id, coor2length(num2int(i.bbox))), i.object)
for i in itemExs]
break
if len(tempItems) > 0:
items = tempItems.copy()
else:
itemExs = list(
idxItems.intersection(idxItems.get_bounds(), objects=True))
items = [((i.id, coor2length(num2int(i.bbox))), i.object)
for i in itemExs]
break
# Sort
items.sort(key=take_key)
g = 0
for j, it in enumerate(items):
c = length2coor(it[0][1])
item = {}
item["id"] = j
item["location"] = coor2dic(c)
if it[1] == 0:
item["type"] = "text"
words = bdResult["words_result"][it[0][0]]["words"]
item["value"] = words
elif it[1] == 1:
item["type"] = "graph"
itemPath = os.path.join(itemDir, str(idGraph + g) + '.png')
cv2.imwrite(itemPath, imageRegion[c[1]:c[3], c[0]:c[2]])
item["value"] = itemPath
g += 1
xiaoti["content"].append(item)
return xiaoti