def uied():
start = time.clock()
models, input_path, output_root, note_success_file, note_fail_file, uied_params = params.get_params(
)
print("UIED Running on:", input_path, " Write to:", output_root)
resized_height = resize_height_by_longest_edge(input_path)
if is_ocr:
import ocr_east as ocr
os.makedirs(pjoin(output_root, 'ocr'), exist_ok=True)
ocr.east(input_path,
output_root,
models['text'],
resize_by_height=resized_height,
show=False)
if is_ip:
os.makedirs(pjoin(output_root, 'ip'), exist_ok=True)
ip.compo_detection(input_path,
output_root,
uied_params=uied_params,
classifier=models['compo'],
resize_by_height=resized_height,
show=False)
if is_merge:
# os.makedirs(pjoin(output_root, 'merge'), exist_ok=True)
import merge
name = input_path.split('/')[-1][:-4]
compo_path = pjoin(output_root, 'ip', str(name) + '.json')
ocr_path = pjoin(output_root, 'ocr', str(name) + '.json')
merge.incorporate(input_path,
compo_path,
ocr_path,
output_root,
resize_by_height=resized_height,
show=False)
open(note_success_file, 'a').write(output_root + '\n')
print("[UIED complete in %.3fs]" % (time.clock() - start))
print(time.ctime(), '\n')
output_root,
key_params,
batch=True,
classifier=compo_classifier,
resize_by_height=resized_height,
show=args.show)
if is_merge:
name = input_img.split('/')[-1][:-4]
compo_path = pjoin(output_root, 'ip', str(name) + '.json')
ocr_path = pjoin(output_root, 'ocr', str(name) + '.json')
merge.incorporate(input_img,
compo_path,
ocr_path,
output_root,
params=key_params,
resize_by_height=resized_height,
show=args.show,
batch=True,
out_suffix=str(idx) + name)
run_times.append(time.time() - start_time)
run_clocks.append(time.clock() - start_clock)
print('Avg. runtime {:.2f}, avg. clock time {:.2f}'.format(
np.mean(run_times), np.mean(run_clocks)))
print('Median runtime {:.2f}, median clock time {:.2f}'.format(
np.median(run_times), np.median(run_clocks)))
plt.boxplot(run_times)
plt.show()
from file_utils import time_start, timer_end
is_ctpn = True
is_uied = True
is_merge = True
PATH_IMG_INPUT = 'data\\input\\web\\6.png'
PATH_LABEL_COMPO = 'data\\output\\compo.json'
PATH_LABEL_TEXT = 'data\\output\\ocr.txt'
PATH_CTPN_DRAWN = 'data\\output\\ctpn.png'
PATH_UIED_DRAWN = 'data\\output\\uied.png'
PATH_UIED_BIN = 'data\\output\\gradient.png'
PATH_MERGE = 'data\\output\\merged.png'
PATH_COMPONENT = 'data\\output\\components'
img_section = (3000, 1500) # selected img section, height and width
start = time_start()
if is_ctpn:
import ocr
ocr.ctpn(PATH_IMG_INPUT, PATH_LABEL_TEXT, PATH_CTPN_DRAWN, img_section)
if is_uied:
import ui
ui.uied(PATH_IMG_INPUT, PATH_LABEL_COMPO, PATH_UIED_DRAWN, PATH_UIED_BIN, img_section)
if is_merge:
import merge
merge.incorporate(PATH_IMG_INPUT, PATH_LABEL_COMPO, PATH_LABEL_TEXT, PATH_MERGE, img_section, is_clip=True, clip_path=PATH_COMPONENT)
timer_end(start)
show=False)
if is_ip:
import detect_compo.ip_region_proposal as ip
os.makedirs(pjoin(output_root, 'ip'), exist_ok=True)
# switch of the classification func
classifier = None
if is_clf:
classifier = {}
from cnn.CNN import CNN
# classifier['Image'] = CNN('Image')
classifier['Elements'] = CNN('Elements')
# classifier['Noise'] = CNN('Noise')
ip.compo_detection(input_path_img,
output_root,
classifier=classifier,
resize_by_height=resized_height,
show=False)
if is_merge:
import merge
name = input_path_img.split('/')[-1][:-4]
compo_path = pjoin(output_root, 'ip', str(name) + '.json')
ocr_path = pjoin(output_root, 'ocr', str(name) + '.json')
merge.incorporate(input_path_img,
compo_path,
ocr_path,
output_root,
resize_by_height=resized_height,
show=True)
C.build_output_folders(is_clip)
resize_by_height = 600
# set input root directory and sort all images by their indices
input_paths_img = glob.glob(pjoin(C.ROOT_INPUT, '*.jpg'))
input_paths_img = sorted(
input_paths_img,
key=lambda x: int(x.split('\\')[-1][:-4])) # sorted by index
# set the range of target inputs' indices
start_index = 24
end_index = 50
for input_path_img in input_paths_img:
index = input_path_img.split('\\')[-1][:-4]
if int(index) < start_index:
continue
if int(index) > end_index:
break
# *** start processing ***
start = time.clock()
if is_ocr:
ocr.east(input_path_img, C.ROOT_OCR, resize_by_height)
if is_ip:
ip.compo_detection(input_path_img, C.ROOT_IP, resize_by_height)
if is_merge:
merge.incorporate(input_path_img, C.ROOT_OCR, C.ROOT_IP, C.ROOT_MERGE,
resize_by_height, is_clip)
print('*** Total Time Taken:%.3f s ***\n' % (time.clock() - start))
resize_by_height=resized_height,
show=args.show)
if is_ip:
import detect_compo.ip_region_proposal as ip
print("Run IP")
ip.compo_detection(input_path_img,
output_root,
key_params,
classifier=compo_classifier,
resize_by_height=resized_height,
show=args.show)
if is_merge:
import merge
print("Run Merge")
name = input_path_img.split('/')[-1][:-4]
compo_path = pjoin(output_root, 'ip', str(name) + '.json')
ocr_path = pjoin(output_root, 'ocr', str(name) + '.json')
merge.incorporate(input_path_img,
compo_path,
ocr_path,
output_root,
params=key_params,
resize_by_height=resized_height,
show=args.show)
print('Total time elapsed {:.2f}s, clock time {:.2f}'.format(
time.time() - start_time,
time.clock() - start_clock))
output_root = 'E:\\Mulong\\Result\\rico\\rico_uied\\rico_new_uied_cls\\merge'
compo_root = 'E:\\Mulong\\Result\\rico\\rico_uied\\rico_new_uied_cls\\ip'
text_root = 'E:\\Mulong\\Result\\east'
data = json.load(open('E:\\Mulong\\Datasets\\rico\\instances_test.json', 'r'))
input_paths_img = [
pjoin(input_root, img['file_name'].split('/')[-1])
for img in data['images']
]
input_paths_img = sorted(
input_paths_img,
key=lambda x: int(x.split('\\')[-1][:-4])) # sorted by index
# set the range of target inputs' indices
num = 0
start_index = 0
end_index = 100000
for input_path_img in input_paths_img:
index = input_path_img.split('\\')[-1][:-4]
if int(index) < start_index:
continue
if int(index) > end_index:
break
merge.incorporate(input_path_img,
compo_root,
text_root,
output_root,
resize_by_height=800,
show=False)
continue
input_path_img = line.split()[0]
print(input_path_img)
# *** start processing ***
start = time.clock()
# set output paths
# for image detection
label_compo = pjoin(C.ROOT_LABEL_UIED, str(index) + '.json')
img_uied_drawn = pjoin(C.ROOT_IMG_DRAWN_UIED, str(index) + '.png')
img_uied_grad = pjoin(C.ROOT_IMG_GRADIENT_UIED, str(index) + '.png')
# for text recognition (ctpn)
label_text = pjoin(C.ROOT_LABEL_CTPN, str(index) + '.txt')
img_ctpn_drawn = pjoin(C.ROOT_IMG_DRAWN_CTPN, str(index) + '.png')
# for incorporated results
img_merge = pjoin(C.ROOT_IMG_MERGE, str(index) + '.png')
label_merge = pjoin(C.ROOT_OUTPUT, 'label.txt')
if is_ctpn:
ocr.ctpn(input_path_img, label_text, img_ctpn_drawn, img_section)
if is_uied:
ui.uied(input_path_img, label_compo, img_uied_drawn, img_uied_grad,
img_section)
if is_merge:
merge.incorporate(input_path_img, label_compo, label_text, img_merge,
label_merge, img_section, is_clip)
print('%d Time Taken:%.3f s\n' % (index, time.clock() - start))
index += 1
import glob
from os.path import join as pjoin
import merge
img_section = (1500, 1500) # selected img section, height and width
img_root = "E:\Mulong\Datasets\dataset_webpage\page10000\org"
label_root = "E:\Mulong\Datasets\dataset_webpage\page10000\\relabel"
ctpn_root = 'C:\\Users\\Shae\\Desktop\\label\\ctpn'
uied_root = 'C:\\Users\\Shae\\Desktop\\label\\uied'
merge_root = 'C:\\Users\\Shae\\Desktop\\label\\merge'
drawn_root = 'C:\\Users\\Shae\\Desktop\\label\\drawn'
ctpn_paths = glob.glob(pjoin(ctpn_root, '*.txt'))
ctpn_paths = sorted(
ctpn_paths, key=lambda x: int(x.split('\\')[-1][:-4])) # sorted by index
for ctpn_path in ctpn_paths:
index = ctpn_path.split('\\')[-1][:-4]
img_path = pjoin(img_root, index + '.png')
uied_path = pjoin(uied_root, index + '.json')
drawn_path = pjoin(drawn_root, index + '.png')
merge_path = pjoin(merge_root, index + '.json')
merge.incorporate(img_path, uied_path, ctpn_path, drawn_path, merge_path,
img_section)
import time
is_ocr = False
is_ip = True
is_merge = False
resize_by_height = 800
# set input image path
PATH_IMG_INPUT = 'data\\input\\1.jpg'
PATH_OUTPUT_ROOT = 'data\\output'
start = time.clock()
if is_ocr:
# import ocr_ctpn as ocr
# ocr.ctpn(PATH_IMG_INPUT, PATH_OUTPUT_ROOT, resize_by_height)
import ocr_east as ocr
ocr.east(PATH_IMG_INPUT, PATH_OUTPUT_ROOT, resize_by_height)
if is_ip:
import ip
ip.compo_detection(PATH_IMG_INPUT, PATH_OUTPUT_ROOT, resize_by_height)
if is_merge:
import merge
merge.incorporate(PATH_IMG_INPUT, PATH_OUTPUT_ROOT, resize_by_height)
print('Time Taken:%.3f s\n' % (time.clock() - start))
if is_clf:
classifier = {}
from cnn.CNN import CNN
# classifier['Image'] = CNN('Image')
classifier['Elements'] = CNN('Elements')
# classifier['Noise'] = CNN('Noise')
ip.compo_detection(input_path_img, path, key_params,
classifier=classifier, resize_by_height=resized_height, show=False)
if is_merge:
import merge
name = input_path_img.split('/')[-1][:-4]
#print(name)
compo_path = pjoin(path, 'ip', str(name) + '.json')
ocr_path = pjoin(path, 'ocr', str(name) + '.json')
merge.incorporate(input_path_img, compo_path, ocr_path, path, params=key_params,
resize_by_height=resized_height, show=False)
if is_html:
from obj.Compos_DF import ComposDF
from obj.Compo_HTML import *
from obj.List import *
from obj.Block import *
from obj.Group import *
from obj.React import *
from obj.Page import *
from obj.Tree import *
import lib.draw as draw
from lib.list_item_gethering import gather_lists_by_pair_and_group
# add path to compos.json name = 'data/input/wireframe/o3/compo'
def uied(input_path,
output_root,
params=None,
is_ip=True,
is_clf=False,
is_ocr=True,
is_merge=True):
'''
ele:min-grad: gradient threshold to produce binary map
ele:ffl-block: fill-flood threshold
ele:min-ele-area: minimum area for selected elements
ele:merge-contained-ele: if True, merge elements contained in others
text:max-word-inline-gap: words with smaller distance than the gap are counted as a line
text:max-line-gap: lines with smaller distance than the gap are counted as a paragraph
Tips:
1. Larger *min-grad* produces fine-grained binary-map while prone to over-segment element to small pieces
2. Smaller *min-ele-area* leaves tiny elements while prone to produce noises
3. If not *merge-contained-ele*, the elements inside others will be recognized, while prone to produce noises
4. The *max-word-inline-gap* and *max-line-gap* should be dependent on the input image size and resolution
mobile: {'min-grad':4, 'ffl-block':5, 'min-ele-area':25, 'max-word-inline-gap':6, 'max-line-gap':1}
web : {'min-grad':3, 'ffl-block':5, 'min-ele-area':25, 'max-word-inline-gap':4, 'max-line-gap':4}
'''
if params is None:
params = {
'min-grad': 4,
'ffl-block': 5,
'min-ele-area': 25,
'merge-contained-ele': True,
'max-word-inline-gap': 4,
'max-line-gap': 4
}
else:
params = json.loads(params)
for i in params:
if params[i] == 'True':
params[i] = True
elif params[i] == 'False':
params[i] = False
else:
params[i] = int(params[i])
print(params)
start = time.clock()
resized_height = resize_height_by_longest_edge(input_path)
if is_ocr:
import ocr_east as ocr
import lib_east.eval as eval
os.makedirs(pjoin(output_root, 'ocr'), exist_ok=True)
models = eval.load()
ocr.east(input_path,
output_root,
models,
params['max-word-inline-gap'],
resize_by_height=resized_height,
show=False)
if is_ip:
import detect_compo.ip_region_proposal as ip
os.makedirs(pjoin(output_root, 'ip'), exist_ok=True)
# switch of the classification func
classifier = None
if is_clf:
classifier = {}
from CNN import CNN
# classifier['Image'] = CNN('Image')
classifier['Elements'] = CNN('Elements')
# classifier['Noise'] = CNN('Noise')
ip.compo_detection(input_path,
output_root,
uied_params=params,
classifier=classifier,
resize_by_height=resized_height,
show=False)
if is_merge:
import merge
# os.makedirs(pjoin(output_root, 'merge'), exist_ok=True)
name = input_path.split('/')[-1][:-4]
compo_path = pjoin(output_root, 'ip', str(name) + '.json')
ocr_path = pjoin(output_root, 'ocr', str(name) + '.json')
merge.incorporate(input_path,
compo_path,
ocr_path,
output_root,
params=params,
resize_by_height=resized_height,
show=False)
print("[UIED complete in %.3fs]" % (time.clock() - start))
print(time.ctime(), '\n')
def fun(filename, output):
input_path_img = filename
output_root = output
key_params = {
'min-grad': 10,
'ffl-block': 5,
'min-ele-area': 50,
'merge-contained-ele': True,
'max-word-inline-gap': 4,
'max-line-gap': 4
}
resized_height = resize_height_by_longest_edge(input_path_img)
is_ip = True
is_clf = True
is_ocr = True
is_merge = True
is_html = True
is_tes = True
color = True
new_html = True
directory = 'out_' + input_path_img.split('/')[-1].split('.')[0]
path = os.path.join(output_root, directory).replace('\\', '/')
os.makedirs(path, exist_ok=True)
if is_ocr:
import detect_text_east.ocr_east as ocr
import detect_text_east.lib_east.eval as eval
os.makedirs(pjoin(path, 'ocr'), exist_ok=True)
models = eval.load()
ocr.east(input_path_img,
path,
models,
key_params['max-word-inline-gap'],
resize_by_height=resized_height,
show=False)
if is_ip:
import detect_compo.ip_region_proposal as ip
os.makedirs(pjoin(path, 'ip'), exist_ok=True)
# switch of the classification func
classifier = None
ip.compo_detection(input_path_img,
path,
key_params,
classifier=classifier,
resize_by_height=resized_height,
show=False)
if is_clf:
classifier = {}
from cnn.CNN import CNN
# classifier['Image'] = CNN('Image')
classifier['Elements'] = CNN('Elements')
# classifier['Noise'] = CNN('Noise')
ip.compo_detection(input_path_img,
path,
key_params,
classifier=classifier,
resize_by_height=resized_height,
show=False)
if is_merge:
import merge
name = input_path_img.split('/')[-1][:-4]
#print(name)
compo_path = pjoin(path, 'ip', str(name) + '.json')
ocr_path = pjoin(path, 'ocr', str(name) + '.json')
merge.incorporate(input_path_img,
compo_path,
ocr_path,
path,
params=key_params,
resize_by_height=resized_height,
show=False)
if is_html:
from obj.Compos_DF import ComposDF
# from obj.Compo_HTML import *
# from obj.List import *
# from obj.Block import *
# from obj.Group import *
# from obj.React import *
# from obj.Page import *
# from obj.Tree import *
import lib.draw as draw
from lib.list_item_gethering import gather_lists_by_pair_and_group
# add path to compos.json name = 'data/input/wireframe/o3/compo'
try:
compos = ComposDF(json_file=path + '/compo'
'.json',
img_file=input_path_img)
img = compos.img.copy()
img_shape = compos.img_shape
img_re = cv2.resize(img, img_shape)
# ***Step 1*** repetitive list recognition
compos.repetitive_group_recognition()
compos.pair_groups()
compos.list_item_partition()
# ***Step 2*** mark compos in same group as a single list, mark compos in same group_pair as a multiple list
lists, non_listed_compos = gather_lists_by_pair_and_group(
compos.compos_dataframe[1:])
generate_lists_html_css(lists)
# ***Step 3*** slice compos as blocks
compos_html = [li.list_obj for li in lists] + non_listed_compos
blocks, non_blocked_compos = slice_blocks(compos_html, 'v')
# ***Step 4*** assembly html and css as web page, and react program
html, css = export_html_and_css(blocks, path + '/page')
blk, index = export_react_program(blocks, path + '/react')
tree = export_tree(blocks, path + '/tree')
#ADD PATH
print("CONVERT TO HTML SAVED TO PATH:", path + '/page')
except:
('cant')
if is_tes:
# get current dir
with open(path + '/compo.json') as f: # make this path variable
#parse json
data = json.load(f)
# get clips
for i in data['compos']:
if i['clip_path'] == "REDUNDANT":
continue
else:
clip_path = i['clip_path']
# get path of project directory +"tesseract E:\\smart-ui\\uied\\final\\" E:\\smart-ui\\uied\\final\\
command = 'cmd /c ' + "tesseract " + clip_path.replace(
"/", "\\") + " stdout -l eng > temp.txt"
os.system(command) #"E:\\smart-ui\\uied\\final\\
a = open("temp.txt", "r")
var = a.read()
# set var
i["ocr"] = var
# make new json
with open(path + '/compo_ocr.json', 'w') as json_file:
json.dump(data, json_file)
if color:
#print(km.cluster_centers_[0][0])
with open(path + '/compo_ocr.json') as f:
data = json.load(f)
for i in data['compos']:
if i['clip_path'] == "REDUNDANT":
continue
else:
clip_path = i['clip_path'] #"E:\\smart-ui\\uied\\final\\"+
img = cv2.imread(clip_path.replace(
"/", "\\")) ### set directory path
#rgb = img[0][0];
all_pixels = img.reshape((-1, 3))
from sklearn.cluster import KMeans
k = 2
km = KMeans(n_clusters=k)
km.fit(all_pixels)
rgb = km.cluster_centers_[0]
rgb = rgb[::-1]
rgb = rgb.astype(int)
i["color"] = '#%02x%02x%02x' % tuple(rgb)
with open(path + '/compo_html.json', 'w') as json_file:
json.dump(data, json_file)
if new_html:
htmltext = """<!DOCTYPE html>
<html>
<head>
<title>HTML, CSS and JavaScript demo</title>
</head>
<body>"""
char = ['\n', '\f', '\\', '/', ']', '[', '(', ")"]
with open(path + '/compo_html.json') as f: # make this path variable
#parse json
data = json.load(f)
# get clips
for i in data['compos']:
if i['clip_path'] == "REDUNDANT":
continue
else:
div = '<div style="background-color:' + i[
'color'] + '; position: absolute; top:' + str(
i["row_min"]) + 'px; left:' + str(
i["column_min"]
) + 'px; border:3px solid black; height:' + str(
i["height"]) + 'px; width:' + str(
i["width"]) + 'px;">' + ''.join([
c for c in i['ocr'] if c not in char
]) + '</div>'
htmltext = htmltext + div
htmltext = htmltext + "</body></html>"
Html_file = open(path + '/output.html', "w", encoding="utf-8")
Html_file.write(htmltext)
Html_file.close()
#print(htmltext)
#json output
st.header("Generated Json ")
with open(path + '/compo_html.json') as f:
data = json.load(f)
st.write(data)
st.header("Generated Json with classification details")
with open(path + '/ip/clf_' + name + '.json') as f:
data = json.load(f)
for i in data['compos']:
if i['class'] == "Background":
continue
else:
clas = i['class']
fid = i['id']
i['clip_path'] = path + '/new/' + clas + '/' + str(
fid) + '.jpg'
st.write(data)
# st.write(json.dumps(data))
st.header("Generated Html")
hp = path + '/output.html'
HtmlFile = open(hp, 'r', encoding='utf-8')
source_code = HtmlFile.read()
st.markdown(source_code, unsafe_allow_html=True)
#image output
st.header("Output Classification")
imagee = cv2.imread(path + "/ip/result.jpg")
#cv2.imshow('Image', imagee)
st.image(imagee, caption='annotated result')
