def run(input_path, op_path, ontology_file, model_dir): i2g_output_dir = os.path.join(op_path, "image2graph") if not os.path.exists(i2g_output_dir): os.makedirs(i2g_output_dir) op_path_all = os.path.join(i2g_output_dir, "all_images") if not os.path.exists(op_path_all): os.makedirs(op_path_all) # command = 'java -cp "pdffigures2_2.12-0.1.0.jar;pdffigures2-assembly-0.1.0-deps.jar;scala-library.jar" org.allenai.pdffigures2.FigureExtractorBatchCli Input/ -s stat_file.json -m out/ -d out/' command = 'java -cp "pdffigures2_2.12-0.1.0.jar;pdffigures2-assembly-0.1.0-deps.jar;scala-library.jar" org.allenai.pdffigures2.FigureExtractorBatchCli ' + input_path + '/ -s stat_file.json -m ' + op_path_all + '/ -d ' + op_path_all + '\\' # , stderr=subprocess.PIPE process = subprocess.Popen(command, shell=True, stderr=subprocess.PIPE) stdout, stderr = process.communicate() # print(stderr) print("[INFO] Loading trained models ...") figtypedetector = ftd(model_dir) figtypedetector.loadFigClassModels("vgg16") print("[INFO] Loading and analyzing images ...") f = open(os.path.join(model_dir, 'full_annotations.pcl'), 'rb') [entity_map, uri2entity, uri2rel]=pickle.load(f) f.close() for filename in glob(os.path.join(op_path_all, '*png')): if (filename.find('Figure') != -1): parsejson = pj() # paper_title, paper_file_name, paper_conf, paper_year, fig_caption, fig_text = parsejson.getCaption(filename) fig_caption, fig_text, paper_file_name = parsejson.getCaption_noCSV(filename) figTypeResult = parsejson.isResult(fig_caption) figTypeDiag = parsejson.isDiag(fig_caption) if (not figTypeResult and figTypeDiag): im, thresh_im, gray_imcv = preprocessImage(filename, 0) binType, mcType = figtypedetector.detectFigType(im) if mcType < 3: # print(os.path.join(i2g_output_dir, paper_file_name)) if not os.path.isdir(os.path.join(i2g_output_dir, paper_file_name)): os.mkdir(os.path.join(i2g_output_dir, paper_file_name)) os.mkdir(os.path.join(i2g_output_dir, paper_file_name, "diag2graph")) os.mkdir(os.path.join(i2g_output_dir, paper_file_name, "Figures")) cv2.imwrite(os.path.join(i2g_output_dir, paper_file_name, "Figures", os.path.basename(filename)), im) shapedetector = sd() component, flow_dir = shapedetector.find_component(filename, i2g_output_dir, im, thresh_im, gray_imcv) textdetector = tda() text_list = textdetector.combinedTextDetect(filename, im, component, fig_text) arrowdetector = ad() line_connect = arrowdetector.detectLines(im, thresh_im, gray_imcv, component, text_list) graphcreator = tgv2() graphcreator.createDiag2Graph(i2g_output_dir, filename, im, thresh_im, component, flow_dir, text_list, line_connect, None, paper_file_name, None, None, fig_caption) #else: #print("Pdf2Fig Terminated with Status %d. Exiting."% (process.returncode) ) print("[INFO] Creating RDF graph ...") runI2G(input_path, entity_map, i2g_output_dir, op_path_all, ontology_file) print("[Info] Completed image2graph pipeline!")
def createDiag2Graph(self, op_dir, filename, img, thresh_im, comp, flow_dir, text_list, line_list, paper_title, paper_file_name, paper_conf, paper_year, fig_caption): # paper_conf and paper_year not used? # print(filename) op_image_name = os.path.join(op_dir, paper_file_name + "/diag2graph/"+ os.path.basename(filename)) op_file_name = os.path.join(op_dir, paper_file_name + "/diag2graph/" + os.path.splitext(os.path.basename(filename))[0] + '.txt') # create intermediate directories: if not os.path.exists(os.path.join(op_dir, paper_file_name + "/diag2graph/" )): os.makedirs(os.path.join(op_dir, paper_file_name + "/diag2graph/" )) op_file = open(op_file_name, "w", encoding="utf-8") FigureID = os.path.splitext(os.path.basename(filename))[0] #+"_"+paper_conf+"_"+paper_year op_file.write(":%s isA Figure \n"% (FigureID)) if (paper_title): op_file.write(":%s foundIn %s \n"% (FigureID, paper_title)) op_file.write(":%s hasCaption %s \n"% (FigureID, fig_caption)) final_graph_im = img.copy() (compWithText, TextWOComp, compWOText) = self.getTextCompTag(comp, text_list, line_list) arrowdetector = ad() line_connect = arrowdetector.getLineCompTag(img, thresh_im, comp, line_list, TextWOComp) node_list = [] #keys = ["nodeId", "layerName", "description", "location", "before", "after"] for k in compWithText.keys(): temp_node = {} temp_node['nodeId'] = 'Comp' + str(k) op_file.write(":Comp%d partOf :%s \n"% (k, FigureID)) cnt = comp[k] M = cv2.moments(cnt) if M["m00"] != 0: cX = int(M["m10"] / M["m00"]) cY = int(M["m01"] / M["m00"]) else: cX, cY = 0, 0 temp_node['center'] = [cX, cY] temp_node['layerName'] = "" temp_node['description'] = [] cv2.putText(final_graph_im, str(k), (cX-20, cY-20), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2) cv2.drawContours(final_graph_im, [cnt], 0, (255, 0, 255), 2) text_pos = 0 op_file.write(":Comp%d hasPos %s \n"% (k, str(cv2.boundingRect(comp[k])))) for txt in compWithText[k]: if not temp_node['layerName']: found, layer_name, remaining_txt = self.find_layerName(txt) if found: op_file.write(":Comp%d isType %s \n"% (k, layer_name)) temp_node['layerName'] = layer_name temp_node['description'].append(txt) else: temp_node['description'].append(txt) elif (re.search("layer", txt, flags = 0) is None): temp_node['description'].append(txt) cv2.putText(final_graph_im, txt, (cX, cY + text_pos), cv2.FONT_HERSHEY_SIMPLEX,0.5, (255, 0, 255), 2) text_pos +=15 op_file.write(":Comp%d hasDescription %s \n"% (k, temp_node['description'])) node_list.append(temp_node) regex = re.compile('@_!#$%^&/\~:') TextWOComp= self.remove_textDuplicate(TextWOComp) for ((startX, startY, endX, endY), op, prob, textID) in TextWOComp: if(len(op) > 1 and regex.search(op) == None): temp_node = {} temp_node['nodeId'] = 'Text' + str(textID) temp_node['center'] = [(startX+endX)/2, (startY+endY)/2] temp_node['layerName'] = "" temp_node['description'] = op node_list.append(temp_node) op_file.write(":Text%d partOf :%s \n"% (textID, FigureID)) op_file.write(":Text%d hasPos %s \n"% (textID, str((startX, startY, endX-startX, endY-startY)))) op_file.write(":Text%d hasDescription %s \n"% (textID, op.split())) cv2.rectangle(final_graph_im, (startX, startY), (endX, endY), (0, 200, 0), 2) cv2.putText(final_graph_im, op, (startX+10, startY+10), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 2) if line_connect: followedby_comps = [] connected_comps = [] for ((startx, starty, endx, endy), start_comp_found, end_comp_found, start_text_found, end_text_found) in line_connect: #print("Diag2Graph: start_comp_found = %s, end_comp_found = %s, start_text_found = %s, end_text_found = %s"%(start_comp_found, end_comp_found, start_text_found, end_text_found)) if (start_comp_found != -1 and end_comp_found != -1): s, e, conn = self.nodeSequence(start_comp_found, end_comp_found, node_list, start_node_type = "Comp", end_node_type = "Comp") if conn == "followedBy": followedby_comps.append((s, e, conn)) elif conn == "connectedTo": connected_comps.append((s, e, conn)) elif (start_text_found != -1 and end_text_found != -1): s, e, conn = self.nodeSequence(start_text_found, end_text_found, node_list, start_node_type = "Text", end_node_type = "Text") if conn == "followedBy": followedby_comps.append((s, e, conn)) elif conn == "connectedTo": connected_comps.append((s, e, conn)) elif(start_comp_found != -1 and end_text_found != -1): s, e, conn = self.nodeSequence(start_comp_found, end_text_found, node_list, start_node_type = "Comp", end_node_type = "Text") if conn == "followedBy": followedby_comps.append((s, e, conn)) elif conn == "connectedTo": connected_comps.append((s, e, conn)) elif(end_comp_found != -1 and start_text_found != -1): s, e, conn = self.nodeSequence(start_text_found, end_text_found, node_list, start_node_type = "Text", end_node_type = "Comp") if conn == "followedBy": followedby_comps.append((s, e, conn)) elif conn == "connectedTo": connected_comps.append((s, e, conn)) flow_dir_updated = self.find_dominant_flow(followedby_comps, node_list, flow_dir) node_list = self.update_connect_info(connected_comps, node_list, flow_dir_updated) op_file.write(":%s hasFlow %s \n"% (FigureID, flow_dir_updated)) cv2.imwrite(op_image_name, final_graph_im) op_file.close()
"Figures")) cv2.imwrite( os.path.join( op_path, paper_file_name + "/Figures/" + os.path.basename(filename)), im) shapedetector = sd() component, flow_dir = shapedetector.find_component( filename, op_path, im, thresh_im, gray_imcv) textdetector = tda() text_list = textdetector.combinedTextDetect( filename, im, component, fig_text) arrowdetector = ad() line_connect = arrowdetector.detectLines( im, thresh_im, gray_imcv, component, text_list) graphcreator = tgv2() graphcreator.createDiag2Graph( op_path, filename, im, thresh_im, component, flow_dir, text_list, line_connect, paper_title, paper_file_name, paper_conf, paper_year, fig_caption) print( "..........................................................................." ) else:
def run(input_path, op_path, ontology_file, model_dir): i2g_output_dir = op_path + "/image2graph" if not os.path.exists(i2g_output_dir): os.makedirs(i2g_output_dir) op_path_all = i2g_output_dir + "/all_images" if not os.path.exists(op_path_all): os.makedirs(op_path_all) # command = 'java -cp "pdffigures2_2.12-0.1.0.jar;pdffigures2-assembly-0.1.0-deps.jar;scala-library.jar" org.allenai.pdffigures2.FigureExtractorBatchCli Input/ -s stat_file.json -m out/ -d out/' command = 'java -cp "pdffigures2_2.12-0.1.0.jar;pdffigures2-assembly-0.1.0-deps.jar;scala-library.jar" org.allenai.pdffigures2.FigureExtractorBatchCli ' + input_path + '/ -s stat_file.json -m ' + op_path_all + '/ -d ' + op_path_all + '\\' # , stderr=subprocess.PIPE process = subprocess.Popen(command, shell=True, stderr=subprocess.PIPE) stdout, stderr = process.communicate() # print(stderr) print("[INFO] Loading trained models ...") figtypedetector = ftd(model_dir) figtypedetector.loadFigClassModels("vgg16") #if (process.returncode == 0): print("[INFO] Loading and analyzing images ...") # input_path = "Input" # op_path = "out" for filename in glob.glob(os.path.join(op_path_all, '*png')): # image_file_name = os.path.splitext(os.path.basename(filename))[0] # abspath = os.path.abspath(filename) filename = filename.replace("\\", "/") # print("Processing image: " + filename) if (filename.find('Figure') != -1): parsejson = pj() # paper_title, paper_file_name, paper_conf, paper_year, fig_caption, fig_text = parsejson.getCaption(filename) fig_caption, fig_text, paper_file_name = parsejson.getCaption_noCSV( filename) figTypeResult = parsejson.isResult(fig_caption) figTypeDiag = parsejson.isDiag(fig_caption) # print(paper_file_name) # print(figTypeResult) # print(figTypeDiag) if (not figTypeResult and figTypeDiag): im, thresh_im, gray_imcv = preprocessImage(filename, 0) binType, mcType = figtypedetector.detectFigType(im) if mcType < 3: # print(os.path.join(i2g_output_dir, paper_file_name)) if not os.path.isdir( os.path.join(i2g_output_dir, paper_file_name)): os.mkdir(os.path.join(i2g_output_dir, paper_file_name)) os.mkdir( os.path.join(i2g_output_dir, paper_file_name, "diag2graph")) os.mkdir( os.path.join(i2g_output_dir, paper_file_name, "Figures")) cv2.imwrite( os.path.join( i2g_output_dir, paper_file_name + "/Figures/" + os.path.basename(filename)), im) shapedetector = sd() component, flow_dir = shapedetector.find_component( filename, i2g_output_dir, im, thresh_im, gray_imcv) textdetector = tda() text_list = textdetector.combinedTextDetect( filename, im, component, fig_text) arrowdetector = ad() line_connect = arrowdetector.detectLines( im, thresh_im, gray_imcv, component, text_list) graphcreator = tgv2() # createDiag2Graph(self, op_dir, filename, img, thresh_im, comp, flow_dir, text_list, line_list, paper_title, paper_file_name, paper_conf, paper_year, fig_caption) graphcreator.createDiag2Graph(i2g_output_dir, filename, im, thresh_im, component, flow_dir, text_list, line_connect, None, paper_file_name, None, None, fig_caption) #else: #print("Pdf2Fig Terminated with Status %d. Exiting."% (process.returncode) ) # image_triple_dir = "C:\\dcc_test\\src\\diagram2graph\\FigAnalysis\\ShapeExtraction\\Output\\" # image_output_dir = "C:\\dcc_test\\src\\diagram2graph\\FigAnalysis\\ShapeExtraction\\Output\\" print("[INFO] Creating RDF graph ...") # paper_dir = "C:/dcc_test/src/diagram2graph/FigAnalysis/ShapeExtraction/Input/" # image_triple_dir = "C:/dcc_test/src/diagram2graph/FigAnalysis/ShapeExtraction/out/" # image_output_dir = "C:/dcc_test/src/diagram2graph/FigAnalysis/ShapeExtraction/out/" runI2G(input_path, i2g_output_dir, op_path_all, ontology_file) print("[Info] Completed image2graph pipeline!") #inputFolder = 'demo_input' #outputFolder = 'Output_a' ## outputFolderImages = 'Output' ## #run(inputFolder, outputFolder)
def createDiag2Graph(self, op_dir, filename, img, thresh_im, comp, text_list, line_list): op_image_name = os.path.join(op_dir, "OpImage/op" + os.path.basename(filename)) op_file_name = os.path.join( op_dir, "OpGraph/graph" + os.path.splitext(os.path.basename(filename))[0] + ".txt") op_file = open(op_file_name, "w") final_graph_im = img.copy() (compWithText, TextWOComp, compWOText) = self.getTextCompTag(comp, text_list) arrowdetector = ad() line_connect = arrowdetector.getLineCompTag(img, thresh_im, comp, line_list, TextWOComp) for k in compWithText.keys(): cnt = comp[k] M = cv2.moments(cnt) if M["m00"] != 0: cX = int(M["m10"] / M["m00"]) cY = int(M["m01"] / M["m00"]) else: cX, cY = 0, 0 cv2.putText(final_graph_im, str(k), (cX - 20, cY - 20), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2) cv2.drawContours(final_graph_im, [cnt], 0, (255, 0, 255), 2) print("=======+++++++++++++++++++++=======") text_pos = 0 for txt in compWithText[k]: found, layer_name, remaining_txt = self.find_layerName(txt) if found: print("Component number %d \"has type\": %s \n" % (k, layer_name)) op_file.write("Component number %d \"has type\": %s \n" % (k, layer_name)) print("Component number %d \"has description\": %s \n" % (k, remaining_txt)) op_file.write( "Component number %d \"has description\": %s \n" % (k, remaining_txt)) else: print("Component number %d \"has description\": %s \n" % (k, txt)) op_file.write( "Component number %d \"has description\": %s \n" % (k, txt)) cv2.putText(final_graph_im, txt, (cX, cY + text_pos), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 0, 255), 2) # cv2.imshow("final_graph_im", final_graph_im) # cv2.waitKey(0) text_pos += 15 for index in compWOText: #for k in results_wotext: print( "----------------Component number %d has text: None --------------------" % (index)) op_file.write("Component number %d has text: None \n" % (index)) cnt = comp[index] M = cv2.moments(cnt) if M["m00"] != 0: cX = int(M["m10"] / M["m00"]) cY = int(M["m01"] / M["m00"]) else: cX, cY = 0, 0 cv2.drawContours(final_graph_im, [cnt], 0, (0, 0, 255), 2) cv2.putText(final_graph_im, str(index), (cX, cY), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2) # cv2.imshow("final_graph_im", final_graph_im) # cv2.waitKey(0) for ((startX, startY, endX, endY), op, prob) in TextWOComp: print( "................ Text without Component: %s....................." % (op)) op_file.write("Text Component: %s \n" % (op)) cv2.rectangle(final_graph_im, (startX, startY), (endX, endY), (0, 200, 0), 2) cv2.putText(final_graph_im, op, (startX + 10, startY + 10), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 2) # cv2.imshow("final_graph_im", final_graph_im) # cv2.waitKey(0) for ((startx, starty, endx, endy), start_comp_found, end_comp_found, start_text_found, end_text_found) in line_connect: if (start_comp_found != -1 and end_comp_found != -1): print( "Component number %d \"connected to\" Component number %d \n" % (start_comp_found, end_comp_found)) op_file.write( "Component number %d \"connected to\" Component number %d \n" % (start_comp_found, end_comp_found)) elif (start_text_found != -1 and end_text_found != -1): print("Text Box %s \"connected to\" TextBox %s \n" % (TextWOComp[start_text_found][1], TextWOComp[end_text_found][1])) op_file.write("Text %s \"connected to\" Text %s \n" % (TextWOComp[start_text_found][1], TextWOComp[end_text_found][1])) elif (start_comp_found != -1 and end_text_found != -1): print("Component Number %d \"connected to\" TextBox %s \n" % (start_comp_found, TextWOComp[end_text_found][1])) op_file.write( "Component Number %d \"connected to\" Text %s \n" % (start_comp_found, TextWOComp[end_text_found][1])) elif (end_comp_found != -1 and start_text_found != -1): print("Component Number %d \"connected to\" TextBox %s \n" % (end_comp_found, TextWOComp[start_text_found][1])) op_file.write( "Component Number %d \"connected to\" Text %s \n" % (end_comp_found, TextWOComp[start_text_found][1])) cv2.line(final_graph_im, (startx, starty), (endx, endy), (0, 200, 255), 2) # cv2.imshow("final_graph_im", final_graph_im) # cv2.waitKey(0) cv2.imwrite(op_image_name, final_graph_im) op_file.close() print( "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%" )