def uied(): start = time.clock() models, input_path, output_root, note_success_file, note_fail_file, uied_params = params.get_params( ) print("UIED Running on:", input_path, " Write to:", output_root) resized_height = resize_height_by_longest_edge(input_path) if is_ocr: import ocr_east as ocr os.makedirs(pjoin(output_root, 'ocr'), exist_ok=True) ocr.east(input_path, output_root, models['text'], resize_by_height=resized_height, show=False) if is_ip: os.makedirs(pjoin(output_root, 'ip'), exist_ok=True) ip.compo_detection(input_path, output_root, uied_params=uied_params, classifier=models['compo'], resize_by_height=resized_height, show=False) if is_merge: # os.makedirs(pjoin(output_root, 'merge'), exist_ok=True) import merge name = input_path.split('/')[-1][:-4] compo_path = pjoin(output_root, 'ip', str(name) + '.json') ocr_path = pjoin(output_root, 'ocr', str(name) + '.json') merge.incorporate(input_path, compo_path, ocr_path, output_root, resize_by_height=resized_height, show=False) open(note_success_file, 'a').write(output_root + '\n') print("[UIED complete in %.3fs]" % (time.clock() - start)) print(time.ctime(), '\n')
output_root, key_params, batch=True, classifier=compo_classifier, resize_by_height=resized_height, show=args.show) if is_merge: name = input_img.split('/')[-1][:-4] compo_path = pjoin(output_root, 'ip', str(name) + '.json') ocr_path = pjoin(output_root, 'ocr', str(name) + '.json') merge.incorporate(input_img, compo_path, ocr_path, output_root, params=key_params, resize_by_height=resized_height, show=args.show, batch=True, out_suffix=str(idx) + name) run_times.append(time.time() - start_time) run_clocks.append(time.clock() - start_clock) print('Avg. runtime {:.2f}, avg. clock time {:.2f}'.format( np.mean(run_times), np.mean(run_clocks))) print('Median runtime {:.2f}, median clock time {:.2f}'.format( np.median(run_times), np.median(run_clocks))) plt.boxplot(run_times) plt.show()
from file_utils import time_start, timer_end is_ctpn = True is_uied = True is_merge = True PATH_IMG_INPUT = 'data\\input\\web\\6.png' PATH_LABEL_COMPO = 'data\\output\\compo.json' PATH_LABEL_TEXT = 'data\\output\\ocr.txt' PATH_CTPN_DRAWN = 'data\\output\\ctpn.png' PATH_UIED_DRAWN = 'data\\output\\uied.png' PATH_UIED_BIN = 'data\\output\\gradient.png' PATH_MERGE = 'data\\output\\merged.png' PATH_COMPONENT = 'data\\output\\components' img_section = (3000, 1500) # selected img section, height and width start = time_start() if is_ctpn: import ocr ocr.ctpn(PATH_IMG_INPUT, PATH_LABEL_TEXT, PATH_CTPN_DRAWN, img_section) if is_uied: import ui ui.uied(PATH_IMG_INPUT, PATH_LABEL_COMPO, PATH_UIED_DRAWN, PATH_UIED_BIN, img_section) if is_merge: import merge merge.incorporate(PATH_IMG_INPUT, PATH_LABEL_COMPO, PATH_LABEL_TEXT, PATH_MERGE, img_section, is_clip=True, clip_path=PATH_COMPONENT) timer_end(start)
show=False) if is_ip: import detect_compo.ip_region_proposal as ip os.makedirs(pjoin(output_root, 'ip'), exist_ok=True) # switch of the classification func classifier = None if is_clf: classifier = {} from cnn.CNN import CNN # classifier['Image'] = CNN('Image') classifier['Elements'] = CNN('Elements') # classifier['Noise'] = CNN('Noise') ip.compo_detection(input_path_img, output_root, classifier=classifier, resize_by_height=resized_height, show=False) if is_merge: import merge name = input_path_img.split('/')[-1][:-4] compo_path = pjoin(output_root, 'ip', str(name) + '.json') ocr_path = pjoin(output_root, 'ocr', str(name) + '.json') merge.incorporate(input_path_img, compo_path, ocr_path, output_root, resize_by_height=resized_height, show=True)
C.build_output_folders(is_clip) resize_by_height = 600 # set input root directory and sort all images by their indices input_paths_img = glob.glob(pjoin(C.ROOT_INPUT, '*.jpg')) input_paths_img = sorted( input_paths_img, key=lambda x: int(x.split('\\')[-1][:-4])) # sorted by index # set the range of target inputs' indices start_index = 24 end_index = 50 for input_path_img in input_paths_img: index = input_path_img.split('\\')[-1][:-4] if int(index) < start_index: continue if int(index) > end_index: break # *** start processing *** start = time.clock() if is_ocr: ocr.east(input_path_img, C.ROOT_OCR, resize_by_height) if is_ip: ip.compo_detection(input_path_img, C.ROOT_IP, resize_by_height) if is_merge: merge.incorporate(input_path_img, C.ROOT_OCR, C.ROOT_IP, C.ROOT_MERGE, resize_by_height, is_clip) print('*** Total Time Taken:%.3f s ***\n' % (time.clock() - start))
resize_by_height=resized_height, show=args.show) if is_ip: import detect_compo.ip_region_proposal as ip print("Run IP") ip.compo_detection(input_path_img, output_root, key_params, classifier=compo_classifier, resize_by_height=resized_height, show=args.show) if is_merge: import merge print("Run Merge") name = input_path_img.split('/')[-1][:-4] compo_path = pjoin(output_root, 'ip', str(name) + '.json') ocr_path = pjoin(output_root, 'ocr', str(name) + '.json') merge.incorporate(input_path_img, compo_path, ocr_path, output_root, params=key_params, resize_by_height=resized_height, show=args.show) print('Total time elapsed {:.2f}s, clock time {:.2f}'.format( time.time() - start_time, time.clock() - start_clock))
output_root = 'E:\\Mulong\\Result\\rico\\rico_uied\\rico_new_uied_cls\\merge' compo_root = 'E:\\Mulong\\Result\\rico\\rico_uied\\rico_new_uied_cls\\ip' text_root = 'E:\\Mulong\\Result\\east' data = json.load(open('E:\\Mulong\\Datasets\\rico\\instances_test.json', 'r')) input_paths_img = [ pjoin(input_root, img['file_name'].split('/')[-1]) for img in data['images'] ] input_paths_img = sorted( input_paths_img, key=lambda x: int(x.split('\\')[-1][:-4])) # sorted by index # set the range of target inputs' indices num = 0 start_index = 0 end_index = 100000 for input_path_img in input_paths_img: index = input_path_img.split('\\')[-1][:-4] if int(index) < start_index: continue if int(index) > end_index: break merge.incorporate(input_path_img, compo_root, text_root, output_root, resize_by_height=800, show=False)
continue input_path_img = line.split()[0] print(input_path_img) # *** start processing *** start = time.clock() # set output paths # for image detection label_compo = pjoin(C.ROOT_LABEL_UIED, str(index) + '.json') img_uied_drawn = pjoin(C.ROOT_IMG_DRAWN_UIED, str(index) + '.png') img_uied_grad = pjoin(C.ROOT_IMG_GRADIENT_UIED, str(index) + '.png') # for text recognition (ctpn) label_text = pjoin(C.ROOT_LABEL_CTPN, str(index) + '.txt') img_ctpn_drawn = pjoin(C.ROOT_IMG_DRAWN_CTPN, str(index) + '.png') # for incorporated results img_merge = pjoin(C.ROOT_IMG_MERGE, str(index) + '.png') label_merge = pjoin(C.ROOT_OUTPUT, 'label.txt') if is_ctpn: ocr.ctpn(input_path_img, label_text, img_ctpn_drawn, img_section) if is_uied: ui.uied(input_path_img, label_compo, img_uied_drawn, img_uied_grad, img_section) if is_merge: merge.incorporate(input_path_img, label_compo, label_text, img_merge, label_merge, img_section, is_clip) print('%d Time Taken:%.3f s\n' % (index, time.clock() - start)) index += 1
import glob from os.path import join as pjoin import merge img_section = (1500, 1500) # selected img section, height and width img_root = "E:\Mulong\Datasets\dataset_webpage\page10000\org" label_root = "E:\Mulong\Datasets\dataset_webpage\page10000\\relabel" ctpn_root = 'C:\\Users\\Shae\\Desktop\\label\\ctpn' uied_root = 'C:\\Users\\Shae\\Desktop\\label\\uied' merge_root = 'C:\\Users\\Shae\\Desktop\\label\\merge' drawn_root = 'C:\\Users\\Shae\\Desktop\\label\\drawn' ctpn_paths = glob.glob(pjoin(ctpn_root, '*.txt')) ctpn_paths = sorted( ctpn_paths, key=lambda x: int(x.split('\\')[-1][:-4])) # sorted by index for ctpn_path in ctpn_paths: index = ctpn_path.split('\\')[-1][:-4] img_path = pjoin(img_root, index + '.png') uied_path = pjoin(uied_root, index + '.json') drawn_path = pjoin(drawn_root, index + '.png') merge_path = pjoin(merge_root, index + '.json') merge.incorporate(img_path, uied_path, ctpn_path, drawn_path, merge_path, img_section)
import time is_ocr = False is_ip = True is_merge = False resize_by_height = 800 # set input image path PATH_IMG_INPUT = 'data\\input\\1.jpg' PATH_OUTPUT_ROOT = 'data\\output' start = time.clock() if is_ocr: # import ocr_ctpn as ocr # ocr.ctpn(PATH_IMG_INPUT, PATH_OUTPUT_ROOT, resize_by_height) import ocr_east as ocr ocr.east(PATH_IMG_INPUT, PATH_OUTPUT_ROOT, resize_by_height) if is_ip: import ip ip.compo_detection(PATH_IMG_INPUT, PATH_OUTPUT_ROOT, resize_by_height) if is_merge: import merge merge.incorporate(PATH_IMG_INPUT, PATH_OUTPUT_ROOT, resize_by_height) print('Time Taken:%.3f s\n' % (time.clock() - start))
if is_clf: classifier = {} from cnn.CNN import CNN # classifier['Image'] = CNN('Image') classifier['Elements'] = CNN('Elements') # classifier['Noise'] = CNN('Noise') ip.compo_detection(input_path_img, path, key_params, classifier=classifier, resize_by_height=resized_height, show=False) if is_merge: import merge name = input_path_img.split('/')[-1][:-4] #print(name) compo_path = pjoin(path, 'ip', str(name) + '.json') ocr_path = pjoin(path, 'ocr', str(name) + '.json') merge.incorporate(input_path_img, compo_path, ocr_path, path, params=key_params, resize_by_height=resized_height, show=False) if is_html: from obj.Compos_DF import ComposDF from obj.Compo_HTML import * from obj.List import * from obj.Block import * from obj.Group import * from obj.React import * from obj.Page import * from obj.Tree import * import lib.draw as draw from lib.list_item_gethering import gather_lists_by_pair_and_group # add path to compos.json name = 'data/input/wireframe/o3/compo'
def uied(input_path, output_root, params=None, is_ip=True, is_clf=False, is_ocr=True, is_merge=True): ''' ele:min-grad: gradient threshold to produce binary map ele:ffl-block: fill-flood threshold ele:min-ele-area: minimum area for selected elements ele:merge-contained-ele: if True, merge elements contained in others text:max-word-inline-gap: words with smaller distance than the gap are counted as a line text:max-line-gap: lines with smaller distance than the gap are counted as a paragraph Tips: 1. Larger *min-grad* produces fine-grained binary-map while prone to over-segment element to small pieces 2. Smaller *min-ele-area* leaves tiny elements while prone to produce noises 3. If not *merge-contained-ele*, the elements inside others will be recognized, while prone to produce noises 4. The *max-word-inline-gap* and *max-line-gap* should be dependent on the input image size and resolution mobile: {'min-grad':4, 'ffl-block':5, 'min-ele-area':25, 'max-word-inline-gap':6, 'max-line-gap':1} web : {'min-grad':3, 'ffl-block':5, 'min-ele-area':25, 'max-word-inline-gap':4, 'max-line-gap':4} ''' if params is None: params = { 'min-grad': 4, 'ffl-block': 5, 'min-ele-area': 25, 'merge-contained-ele': True, 'max-word-inline-gap': 4, 'max-line-gap': 4 } else: params = json.loads(params) for i in params: if params[i] == 'True': params[i] = True elif params[i] == 'False': params[i] = False else: params[i] = int(params[i]) print(params) start = time.clock() resized_height = resize_height_by_longest_edge(input_path) if is_ocr: import ocr_east as ocr import lib_east.eval as eval os.makedirs(pjoin(output_root, 'ocr'), exist_ok=True) models = eval.load() ocr.east(input_path, output_root, models, params['max-word-inline-gap'], resize_by_height=resized_height, show=False) if is_ip: import detect_compo.ip_region_proposal as ip os.makedirs(pjoin(output_root, 'ip'), exist_ok=True) # switch of the classification func classifier = None if is_clf: classifier = {} from CNN import CNN # classifier['Image'] = CNN('Image') classifier['Elements'] = CNN('Elements') # classifier['Noise'] = CNN('Noise') ip.compo_detection(input_path, output_root, uied_params=params, classifier=classifier, resize_by_height=resized_height, show=False) if is_merge: import merge # os.makedirs(pjoin(output_root, 'merge'), exist_ok=True) name = input_path.split('/')[-1][:-4] compo_path = pjoin(output_root, 'ip', str(name) + '.json') ocr_path = pjoin(output_root, 'ocr', str(name) + '.json') merge.incorporate(input_path, compo_path, ocr_path, output_root, params=params, resize_by_height=resized_height, show=False) print("[UIED complete in %.3fs]" % (time.clock() - start)) print(time.ctime(), '\n')
def fun(filename, output): input_path_img = filename output_root = output key_params = { 'min-grad': 10, 'ffl-block': 5, 'min-ele-area': 50, 'merge-contained-ele': True, 'max-word-inline-gap': 4, 'max-line-gap': 4 } resized_height = resize_height_by_longest_edge(input_path_img) is_ip = True is_clf = True is_ocr = True is_merge = True is_html = True is_tes = True color = True new_html = True directory = 'out_' + input_path_img.split('/')[-1].split('.')[0] path = os.path.join(output_root, directory).replace('\\', '/') os.makedirs(path, exist_ok=True) if is_ocr: import detect_text_east.ocr_east as ocr import detect_text_east.lib_east.eval as eval os.makedirs(pjoin(path, 'ocr'), exist_ok=True) models = eval.load() ocr.east(input_path_img, path, models, key_params['max-word-inline-gap'], resize_by_height=resized_height, show=False) if is_ip: import detect_compo.ip_region_proposal as ip os.makedirs(pjoin(path, 'ip'), exist_ok=True) # switch of the classification func classifier = None ip.compo_detection(input_path_img, path, key_params, classifier=classifier, resize_by_height=resized_height, show=False) if is_clf: classifier = {} from cnn.CNN import CNN # classifier['Image'] = CNN('Image') classifier['Elements'] = CNN('Elements') # classifier['Noise'] = CNN('Noise') ip.compo_detection(input_path_img, path, key_params, classifier=classifier, resize_by_height=resized_height, show=False) if is_merge: import merge name = input_path_img.split('/')[-1][:-4] #print(name) compo_path = pjoin(path, 'ip', str(name) + '.json') ocr_path = pjoin(path, 'ocr', str(name) + '.json') merge.incorporate(input_path_img, compo_path, ocr_path, path, params=key_params, resize_by_height=resized_height, show=False) if is_html: from obj.Compos_DF import ComposDF # from obj.Compo_HTML import * # from obj.List import * # from obj.Block import * # from obj.Group import * # from obj.React import * # from obj.Page import * # from obj.Tree import * import lib.draw as draw from lib.list_item_gethering import gather_lists_by_pair_and_group # add path to compos.json name = 'data/input/wireframe/o3/compo' try: compos = ComposDF(json_file=path + '/compo' '.json', img_file=input_path_img) img = compos.img.copy() img_shape = compos.img_shape img_re = cv2.resize(img, img_shape) # ***Step 1*** repetitive list recognition compos.repetitive_group_recognition() compos.pair_groups() compos.list_item_partition() # ***Step 2*** mark compos in same group as a single list, mark compos in same group_pair as a multiple list lists, non_listed_compos = gather_lists_by_pair_and_group( compos.compos_dataframe[1:]) generate_lists_html_css(lists) # ***Step 3*** slice compos as blocks compos_html = [li.list_obj for li in lists] + non_listed_compos blocks, non_blocked_compos = slice_blocks(compos_html, 'v') # ***Step 4*** assembly html and css as web page, and react program html, css = export_html_and_css(blocks, path + '/page') blk, index = export_react_program(blocks, path + '/react') tree = export_tree(blocks, path + '/tree') #ADD PATH print("CONVERT TO HTML SAVED TO PATH:", path + '/page') except: ('cant') if is_tes: # get current dir with open(path + '/compo.json') as f: # make this path variable #parse json data = json.load(f) # get clips for i in data['compos']: if i['clip_path'] == "REDUNDANT": continue else: clip_path = i['clip_path'] # get path of project directory +"tesseract E:\\smart-ui\\uied\\final\\" E:\\smart-ui\\uied\\final\\ command = 'cmd /c ' + "tesseract " + clip_path.replace( "/", "\\") + " stdout -l eng > temp.txt" os.system(command) #"E:\\smart-ui\\uied\\final\\ a = open("temp.txt", "r") var = a.read() # set var i["ocr"] = var # make new json with open(path + '/compo_ocr.json', 'w') as json_file: json.dump(data, json_file) if color: #print(km.cluster_centers_[0][0]) with open(path + '/compo_ocr.json') as f: data = json.load(f) for i in data['compos']: if i['clip_path'] == "REDUNDANT": continue else: clip_path = i['clip_path'] #"E:\\smart-ui\\uied\\final\\"+ img = cv2.imread(clip_path.replace( "/", "\\")) ### set directory path #rgb = img[0][0]; all_pixels = img.reshape((-1, 3)) from sklearn.cluster import KMeans k = 2 km = KMeans(n_clusters=k) km.fit(all_pixels) rgb = km.cluster_centers_[0] rgb = rgb[::-1] rgb = rgb.astype(int) i["color"] = '#%02x%02x%02x' % tuple(rgb) with open(path + '/compo_html.json', 'w') as json_file: json.dump(data, json_file) if new_html: htmltext = """<!DOCTYPE html> <html> <head> <title>HTML, CSS and JavaScript demo</title> </head> <body>""" char = ['\n', '\f', '\\', '/', ']', '[', '(', ")"] with open(path + '/compo_html.json') as f: # make this path variable #parse json data = json.load(f) # get clips for i in data['compos']: if i['clip_path'] == "REDUNDANT": continue else: div = '<div style="background-color:' + i[ 'color'] + '; position: absolute; top:' + str( i["row_min"]) + 'px; left:' + str( i["column_min"] ) + 'px; border:3px solid black; height:' + str( i["height"]) + 'px; width:' + str( i["width"]) + 'px;">' + ''.join([ c for c in i['ocr'] if c not in char ]) + '</div>' htmltext = htmltext + div htmltext = htmltext + "</body></html>" Html_file = open(path + '/output.html', "w", encoding="utf-8") Html_file.write(htmltext) Html_file.close() #print(htmltext) #json output st.header("Generated Json ") with open(path + '/compo_html.json') as f: data = json.load(f) st.write(data) st.header("Generated Json with classification details") with open(path + '/ip/clf_' + name + '.json') as f: data = json.load(f) for i in data['compos']: if i['class'] == "Background": continue else: clas = i['class'] fid = i['id'] i['clip_path'] = path + '/new/' + clas + '/' + str( fid) + '.jpg' st.write(data) # st.write(json.dumps(data)) st.header("Generated Html") hp = path + '/output.html' HtmlFile = open(hp, 'r', encoding='utf-8') source_code = HtmlFile.read() st.markdown(source_code, unsafe_allow_html=True) #image output st.header("Output Classification") imagee = cv2.imread(path + "/ip/result.jpg") #cv2.imshow('Image', imagee) st.image(imagee, caption='annotated result')