def get_index_from_region(coorRegion, imageRegion, coorCCRegion, suffix): bdResult = {} index1 = [] index2 = [] bdResult = bd_rec(imageRegion, client, imgForm=suffix, api='accurate') if "words_result" not in bdResult: print(bdResult) print("BaiDu recognized error, exit...") exit(0) if len(bdResult["words_result"]) == 0: return index1, index2 words = [] for j, w in enumerate(bdResult["words_result"]): words.append((j, dic2length(w["location"]))) graphs = [] for j, c in enumerate(coorCCRegion): if c[2] > heightOfLine * 3 or c[3] > heightOfLine * 1.5: # if c[2] > heightOfLine*0.5 or c[3] > heightOfLine*0.5: graphs.append(c) for w in words: flag = False for g in graphs: g = expand_box(g) if is_within(w[1], g): index2.append(bdResult["words_result"][w[0]]["words"]) flag = True break if not flag: index1.append(bdResult["words_result"][w[0]]["words"]) return index1, index2
def get_index_from_region(coorRegion, imageRegion, coorCCRegion, suffix): """Gets index from region. Args: coorRegion: Coordinate of region. imageRegion: The image data of region. coorCCRegion: The coordinate of connected components in region. suffix: Image name's suffix. Returns: Index out graph and Index in graph. """ bdResult = {} index1 = [] index2 = [] bdResult = bd_rec(imageRegion, client, imgForm=suffix, api='accurate') if "words_result" not in bdResult: print(bdResult) print("BaiDu recognized error, exit...") exit(0) if len(bdResult["words_result"]) == 0: return index1, index2 words = [] for j, w in enumerate(bdResult["words_result"]): words.append((j, dic2length(w["location"]))) graphs = [] for j, c in enumerate(coorCCRegion): if c[2] > heightOfLine * 3 or c[3] > heightOfLine * 1.5: # if c[2] > heightOfLine*0.5 or c[3] > heightOfLine*0.5: graphs.append(c) for w in words: flag = False for g in graphs: g = expand_box(g) if is_within(w[1], g): index2.append(bdResult["words_result"][w[0]]["words"]) flag = True break if not flag: index1.append(bdResult["words_result"][w[0]]["words"]) return index1, index2
def get_items_v02(region, image, coorCC, name): """Gets items from regions. Args: region: The coordinate of region. image: Image data. coorCC: The coordinate of connected components. name: Image name. Returns: A dictionary of result. """ result = {} result["paper_name"] = name result["questions"] = [] itemDir = os.path.join(desDir, os.path.splitext(name)[0]) suffix = os.path.splitext(name)[1] if os.path.exists(itemDir): # os.removedirs(itemDir) shutil.rmtree(itemDir) os.mkdir(itemDir) for i, r in enumerate(region): question = {} r = (r[0], int(r[1] - heightOfLine * 0.2), r[2], r[3]) if r[2] - r[0] < 15 or r[3] - r[1] < 15: continue imageRegion = image[r[1]:r[3], r[0]:r[2]] coorCCRegion = [ coor for coor in coorCC if is_within(coor, coor2length(r)) ] coorCCRegion = [(c[0] - r[0], c[1] - r[1], c[2], c[3]) for c in coorCCRegion] # question = get_items_from_region(r, imageRegion, coorCCRegion, itemDir, suffix, idGraph) index1, index2 = get_index_from_region(r, imageRegion, coorCCRegion, suffix) question["index_out_graph"] = index1 question["index_in_graph"] = index2 savePath = os.path.join(itemDir, str(i) + '.png') question["question_image"] = savePath cv2.imwrite(savePath, imageRegion) result["questions"].append(question) return result
def get_items_v02(region, image, coorCC, name): """ """ result = {} result["paper_name"] = name result["questions"] = [] itemDir = os.path.join(desDir, os.path.splitext(name)[0]) suffix = os.path.splitext(name)[1] if os.path.exists(itemDir): # os.removedirs(itemDir) shutil.rmtree(itemDir) os.mkdir(itemDir) # idxCC = index.Index() # for j,c in enumerate(coorCC): # idxCC.insert(j, length2coor(c)) for i, r in enumerate(region): question = {} r = (r[0], int(r[1] - heightOfLine * 0.2), r[2], r[3]) if r[2] - r[0] < 15 or r[3] - r[1] < 15: continue imageRegion = image[r[1]:r[3], r[0]:r[2]] coorCCRegion = [ coor for coor in coorCC if is_within(coor, coor2length(r)) ] coorCCRegion = [(c[0] - r[0], c[1] - r[1], c[2], c[3]) for c in coorCCRegion] # question = get_items_from_region(r, imageRegion, coorCCRegion, itemDir, suffix, idGraph) index1, index2 = get_index_from_region(r, imageRegion, coorCCRegion, suffix) question["index_out_graph"] = index1 question["index_in_graph"] = index2 savePath = os.path.join(itemDir, str(i) + '.png') question["question_image"] = savePath cv2.imwrite(savePath, imageRegion) result["questions"].append(question) return result
def get_items_from_region(coorRegion, imageRegion, coorCCRegion, itemDir, suffix, idGraph): """ Args: coorRegion: The coordinate of region. imageRegion: The image of region. coorCCRegion: The coordinate of CC in region. suffix: Image form suffix. idGraph: The id of graph in paper. Returns: A dic of question result. """ xiaoti = {} bdResult = {} bdResult = bd_rec(imageRegion, client, imgForm=suffix, api='accurate') title = '' if "words_result" in bdResult and len( bdResult["words_result"]) > 0: #结果返回错误处理 title = bdResult["words_result"][0]["words"] # i = 0 # if "words_result" in bdResult: # while i < len(bdResult["words_result"]) and \ # not title.endswith(('。', '?', ')')): # title += bdResult["words_result"][i]["words"] # i += 1 xiaoti["title"] = title xiaoti["location"] = coor2dic(coorRegion) xiaoti["content"] = [] if is_special(title): locationOfTitle = bdResult["words_result"][0]["location"] boxOfGraph = (10, dic2coor(locationOfTitle)[3] + 10, coorRegion[2] - coorRegion[0] - 10, coorRegion[3] - coorRegion[1] - 10) itemText = {} itemText["id"] = 0 itemText["location"] = locationOfTitle itemText["type"] = "text" itemText["value"] = title xiaoti["content"].append(itemText) itemGraph = {} itemPath = os.path.join(itemDir, str(idGraph) + '.png') cv2.imwrite( itemPath, imageRegion[boxOfGraph[1]:boxOfGraph[3], boxOfGraph[0]:boxOfGraph[2]]) itemGraph["id"] = 1 itemGraph["location"] = coor2dic(boxOfGraph) itemGraph["type"] = "graph" itemGraph["value"] = itemPath xiaoti["content"].append(itemGraph) return xiaoti wordBoxes = [] if "words_result" in bdResult: for j, w in enumerate(bdResult["words_result"]): wordBoxes.append((j, dic2length(w["location"]))) graphBoxes = [] for j, b in wordBoxes: # print(j,b) b = expand_box(b) coorCCRegion = [c for c in coorCCRegion if not is_within(c, b)] n = 0 for j, c in enumerate(coorCCRegion): # if c[2] > heightOfLine*3 or c[3] > heightOfLine*1.5: if c[2] > heightOfLine * 0.5 or c[3] > heightOfLine * 0.5: graphBoxes.append((n, c)) n += 1 c = expand_box(c) wordBoxes = [it for it in wordBoxes if not is_within(it[1], c)] mixBoxes = [] for j, w in enumerate(wordBoxes.copy()): if w[1][2] > len( bdResult["words_result"][w[0]]["words"]) * heightOfLine * 1.5: del (wordBoxes[j - len(mixBoxes)]) mixBoxes.append((n, w[1])) n += 1 # Fuses image graphBoxes += mixBoxes idxWords = index.Index() for j, b in enumerate(wordBoxes): if b[1][2] < heightOfLine * 5: idxWords.insert(b[0], length2coor(b[1])) wordBoxes = [b for b in wordBoxes if b[1][2] >= heightOfLine * 5] count = 0 idDeleted = [] for j, b in graphBoxes.copy(): bEx = expand_box(b) items = list(idxWords.intersection(length2coor(bEx), objects=True)) if len(items) > 0: boxes = [num2int(i.bbox) for i in items] + [length2coor(b)] box = fuse_box(boxes) for i, it in enumerate(items): # del(wordBoxes[it.id-i-len(idDeleted)]) idxWords.delete(it.id, it.bbox) idDeleted.append(it.id) del (graphBoxes[j - count]) count += 1 graphBoxes.append((n, box)) n += 1 if len(idxWords.leaves()[0][1]) > 0: wordItems = list( idxWords.intersection(idxWords.get_bounds(), objects=True)) wordBoxes += [(i.id, coor2length(num2int(i.bbox))) for i in wordItems] idxGraphs = index.Index() for j, b in enumerate(graphBoxes): idxGraphs.insert(j, length2coor(b[1])) count = 0 for j, b in enumerate(graphBoxes.copy()): bEx = expand_box(b[1]) items = list(idxGraphs.intersection(length2coor(bEx), objects=True)) if len(items) > 0: boxes = [num2int(i.bbox) for i in items] + [length2coor(b[1])] box = fuse_box(boxes) for i, it in enumerate(items): # if it.id-i not in range(0, len(graphBoxes)): # print(it.id-i) # print(graphBoxes) # continue idxGraphs.delete(it.id, it.bbox) idxGraphs.insert(len(graphBoxes) + count, length2coor(box)) count += 1 # graphBoxes.append((n, box)) # n += 1 # print(idxGraphs.get_bounds()) # print(idxGraphs.leaves()) # exit(0) if len(graphBoxes) > 0: graphItems = list( idxGraphs.intersection(idxGraphs.get_bounds(), objects=True)) graphBoxes = [(i.id, coor2length(num2int(i.bbox))) for i in graphItems] for j, b in graphBoxes: bEx = expand_box(b) wordBoxes = [ it for it in wordBoxes if not is_within(it[1], bEx) and it[1][3] < it[1][2] * 1.5 ] items = [(it, 0) for it in wordBoxes] + [(it, 1) for it in graphBoxes] idxItems = index.Index() for i, it in enumerate(items): idxItems.insert(it[0][0], length2coor(it[0][1]), obj=it[1]) count = 0 # flag = False while True: tempItems = [] for it in items: itemInter = list( idxItems.intersection(length2coor(it[0][1]), objects=True)) if len(itemInter) > 1: boxes = [num2int(i.bbox) for i in itemInter] box = fuse_box(boxes) for it1 in itemInter: idxItems.delete(it1.id, it1.bbox) idxItems.insert(len(items) + count, length2coor(box), obj=1) count += 1 itemExs = list( idxItems.intersection(idxItems.get_bounds(), objects=True)) tempItems = [((i.id, coor2length(num2int(i.bbox))), i.object) for i in itemExs] break if len(tempItems) > 0: items = tempItems.copy() else: itemExs = list( idxItems.intersection(idxItems.get_bounds(), objects=True)) items = [((i.id, coor2length(num2int(i.bbox))), i.object) for i in itemExs] break # Sort items.sort(key=take_key) g = 0 for j, it in enumerate(items): c = length2coor(it[0][1]) item = {} item["id"] = j item["location"] = coor2dic(c) if it[1] == 0: item["type"] = "text" words = bdResult["words_result"][it[0][0]]["words"] item["value"] = words elif it[1] == 1: item["type"] = "graph" itemPath = os.path.join(itemDir, str(idGraph + g) + '.png') cv2.imwrite(itemPath, imageRegion[c[1]:c[3], c[0]:c[2]]) item["value"] = itemPath g += 1 xiaoti["content"].append(item) return xiaoti