def run_single(input_path_img, output_dir, models):
# input_path_img = '/Users/yixue/Documents/Research/UsageTesting/UsageTesting-Repo/video_data_examples/6pm-video-signin-1/ir_data/bbox-0189-screen.jpg'
single_output = os.path.join(output_dir, 'result.jpg')
if os.path.exists(single_output):
return
print('running', input_path_img)
resized_height = resize_height_by_longest_edge(input_path_img)
if is_ocr:
print('ocr...')
os.makedirs(pjoin(output_dir, 'ocr'), exist_ok=True)
ocr.east(input_path_img, output_dir, models, key_params['max-word-inline-gap'],
resize_by_height=resized_height, show=False)
if is_ip:
print('ip...')
os.makedirs(pjoin(output_dir, 'ip'), exist_ok=True)
# switch of the classification func
classifier = None
if is_clf:
classifier = {}
# classifier['Image'] = CNN('Image')
classifier['Elements'] = CNN('Elements')
# classifier['Noise'] = CNN('Noise')
ip.compo_detection(input_path_img, output_dir, key_params,
classifier=classifier, resize_by_height=resized_height, show=False)
if is_merge:
print('merge...')
name = input_path_img.split('/')[-1][:-4]
compo_path = pjoin(output_dir, 'ip', str(name) + '.json')
ocr_path = pjoin(output_dir, 'ocr', str(name) + '.json')
merge_optimized.incorporate(input_path_img, compo_path, ocr_path, output_dir, params=key_params,
resize_by_height=resized_height, show=False)
# Setup
# Load deep learning models in advance
ocr_model = None
if is_ocr:
import detect_text_east.ocr_east as ocr
import detect_text_east.lib_east.eval as ocr_eval
os.makedirs(pjoin(output_root, 'ocr'), exist_ok=True)
ocr_model = ocr_eval.load()
compo_classifier = None
if is_ip:
import detect_compo.ip_region_proposal as ip
os.makedirs(pjoin(output_root, 'ip'), exist_ok=True)
if is_clf:
from cnn.CNN import CNN
compo_classifier = {'Elements': CNN('Elements')}
# compo_classifier['Image'] = CNN('Image')
# compo_classifier['Noise'] = CNN('Noise')
if is_merge:
import merge
print('... Done\n')
run_times, run_clocks = [], []
# set the range of target inputs' indices
for idx, input_img in enumerate(tqdm(input_imgs)):
start_time, start_clock = time.time(), time.clock()
resized_height = resize_height_by_longest_edge(input_img)
if is_ocr:
ocr.east(input_img,
output_root,
def starttrain(request):
post = request.POST
#validate request
if post == None:
return JsonResponse({"status": 400, "msg": "Forbidden"}, safe=False)
if request.COOKIES.get('RoomCode') == None:
return JsonResponse({"status": 400, "msg": "Forbidden"}, safe=False)
data = json.loads(str(post.get('data')))
newMachine = Machine()
# get date time for machine name
now = datetime.now(tz=timezone.utc)
dt_string = now.strftime("%d%m%Y%H%M%S")
#time for created time
#get roodir
rootdir = os.getcwd()
#assign data machine
newMachine.Name = "M-" + dt_string
newMachine.Created = now
newMachine.Directory = os.path.join(rootdir, "teachapp", "static",
"UserData", newMachine.Name)
newMachine.epoch = str(post.get('epoch'))
newMachine.batch = str(post.get('batch'))
newMachine.learningrate = str(post.get('learningrate'))
os.makedirs(newMachine.Directory)
newMachine.save()
#write log file has readed
asyncio.run(
doSendLogTraining(RoomCode=request.COOKIES.get('RoomCode'),
Log="Examine Your Dataset"))
#iterate label on json data
for i in data.keys():
indexImage = 1
os.makedirs(os.path.join(newMachine.Directory, i))
classdir = os.path.join(newMachine.Directory, i)
newClass = MachineClass(Name=i, Machine_ID=str(newMachine.id))
newClass.save()
for urlraw in data[i]:
imageurl = urllib.request.urlopen(urlraw)
#write image to server
with open(classdir + '/' + i + "-" + str(indexImage) + ".png",
'wb') as f:
f.write(imageurl.file.read())
f.close()
indexImage += 1
# //todo memeasukan ke model cnnn
model = CNN(image_size_w=80, image_size_h=60, objectMachine=newMachine)
print("Room Code", request.COOKIES.get('RoomCode'))
# initiate Callback Keras
callback = TrainingCallback(RoomName=request.COOKIES.get('RoomCode'))
model.fittingModel(Callback=callback)
return JsonResponse(
{
"status": 200,
"msg": "success",
"MachineID": newMachine.id
},
safe=False)
ocr.east(input_path_img,
output_root,
models,
resize_by_height=resized_height,
show=False)
if is_ip:
import detect_compo.ip_region_proposal as ip
os.makedirs(pjoin(output_root, 'ip'), exist_ok=True)
# switch of the classification func
classifier = None
if is_clf:
classifier = {}
from cnn.CNN import CNN
# classifier['Image'] = CNN('Image')
classifier['Elements'] = CNN('Elements')
# classifier['Noise'] = CNN('Noise')
ip.compo_detection(input_path_img,
output_root,
classifier=classifier,
resize_by_height=resized_height,
show=False)
if is_merge:
import merge
name = input_path_img.split('/')[-1][:-4]
compo_path = pjoin(output_root, 'ip', str(name) + '.json')
ocr_path = pjoin(output_root, 'ocr', str(name) + '.json')
merge.incorporate(input_path_img,
compo_path,
ocr_path,
def main():
if RUN_ON_GPU:
torch.set_default_tensor_type('torch.cuda.FloatTensor')
if not torch.cuda.is_available():
raise Exception("trying to run on gpu but no cuda available")
device = torch.device("cuda")
else:
device = torch.device("cpu")
########################################################################
# The output of torchvision datasets are PILImage images of range [0, 1].
# We transform them to Tensors of normalized range [-1, 1].
transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
])
trainingSet = torchvision.datasets.CIFAR10(root=DATA_DIR,
train=True,
download=NEED_TO_DL_DATASET,
transform=transform)
trainingLoader = torch.utils.data.DataLoader(trainingSet,
batch_size=BATCH_SIZE,
shuffle=True,
num_workers=2)
testSet = torchvision.datasets.CIFAR10(root=DATA_DIR,
train=False,
download=NEED_TO_DL_DATASET,
transform=transform)
testLoader = torch.utils.data.DataLoader(testSet,
batch_size=BATCH_SIZE,
shuffle=False,
num_workers=2)
classes = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse',
'ship', 'truck')
########################################################################
# Let us show some of the training images, for fun.
# functions to show an image
def showImage(img):
img = img / 2 + 0.5 # unnormalize
npimg = img.numpy()
plt.imshow(np.transpose(npimg, (1, 2, 0)))
# get some random training images
# data = iter(trainingLoader)
# images, labels = data.next()
# show images
# showImage(torchvision.utils.make_grid(images))
# print labels
# print(' '.join('%5s' % classes[labels[j]] for j in range(4)))
cnn = CNN(device, LEARNING_RATE)
########################################################################
# 4. Train the network
# ^^^^^^^^^^^^^^^^^^^^
#
# This is when things start to get interesting.
# We simply have to loop over our data iterator, and feed the inputs to the
# network and optimize.
start = timer()
print(datetime.datetime.now())
runs = 0
for epoch in range(EPOCHS): # loop over the dataset multiple times
runningLoss = 0.0
for i, data in enumerate(trainingLoader, 0):
# get the inputs
inputs, labels = data
inputs = inputs.to(device)
labels = labels.to(device)
# print statistics
runningLoss += cnn.runAndGetLoss(inputs, labels)
runs = runs + 1
# if runs > 2000:
# break
if i % LOG_EVERY_X_BATCHES == (LOG_EVERY_X_BATCHES - 1):
print(
f'[{epoch + 1}, {i+1}] loss: {epoch + 1, i + 1, runningLoss / LOG_EVERY_X_BATCHES}'
)
runningLoss = 0.0
print(timer() - start)
start = timer()
print('Finished Training')
print(datetime.datetime.now())
print("\n")
########################################################################
# 5. Test the network on the test data
# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
#
# We have trained the network for 2 passes over the training dataset.
# But we need to check if the network has learnt anything at all.
#
# We will check this by predicting the class label that the neural network
# outputs, and checking it against the ground-truth. If the prediction is
# correct, we add the sample to the list of correct predictions.
#
# Okay, first step. Let us display an image from the test set to get familiar.
data = iter(testLoader)
images, labels = data.next()
images = images.to(device)
labels = labels.to(device)
# print images
# showImage(torchvision.utils.make_grid(images))
print('GroundTruth: ',
' '.join('%5s' % classes[labels[j]] for j in range(4)))
########################################################################
# Okay, now let us see what the neural network thinks these examples above are:
outputs = cnn(images)
########################################################################
# The outputs are energies for the 10 classes.
# Higher the energy for a class, the more the network
# thinks that the image is of the particular class.
# So, let's get the index of the highest energy:
_, predicted = torch.max(outputs, 1)
print('Predicted: ',
' '.join('%5s' % classes[predicted[j]] for j in range(4)))
########################################################################
# The results seem pretty good.
#
# Let us look at how the network performs on the whole dataset.
correct = 0
total = 0
with torch.no_grad():
for data in testLoader:
images, labels = data
images = images.to(device)
labels = labels.to(device)
outputs = cnn(images)
_, predicted = torch.max(outputs.data, 1)
total += labels.size(0)
correct += (predicted == labels).sum().item()
print('Accuracy of the network on the 10000 test images: %d %%' %
(100 * correct / total))
########################################################################
# That looks waaay better than chance, which is 10% accuracy (randomly picking
# a class out of 10 classes).
# Seems like the network learnt something.
#
# Hmmm, what are the classes that performed well, and the classes that did
# not perform well:
class_correct = list(0. for i in range(10))
class_total = list(0. for i in range(10))
with torch.no_grad():
for data in testLoader:
images, labels = data
images = images.to(device)
labels = labels.to(device)
outputs = cnn(images)
_, predicted = torch.max(outputs, 1)
c = (predicted == labels).squeeze()
for i in range(4):
label = labels[i]
class_correct[label] += c[i].item()
class_total[label] += 1
for i in range(len(classes)):
print('Accuracy of %5s : %2d %%' %
(classes[i], 100 * class_correct[i] / class_total[i]))
6: [(1, 1), (5, 3)],
7: [(2, 1), (3, 2), (5, 3), (6, 4)]
}
# config_list = {2: [(0, 1), (1, 5)],
# 3: [(0, 5), (1, 3)],
# 4: [(0, 1), (1, 2), (2, 7), (3, 5)],
# 5: [(0, 4), (3, 4), (4, 7)],
# 6: [(0, 3), (1, 5), (2, 1), (3, 2), (4, 5)]}
cell_config_list = {'normal_cell': config_list}
# conv=choose_conv_elem(3,3,64)
#
# summary(conv,(3,28,28))
cnn = CNN(cell_config_list, class_num=10)
# cnn=ResBlock(config_list,2,channels=128,in_channels=32)
# cnn=Cell(config_list,in_channels=32,conv_channels=128)
# print(cnn)
# summary(cnn,[(32,28,28),(32,28,28)])
summary(cnn, (3, 32, 32))
# net=nn.Sequential(
# nn.ReLU(),
# nn.Conv2d(3,64,3,1,1),
# nn.BatchNorm2d(64,eps=0.001, momentum=0.1, affine=True)
# )
#
# summary(net,(3,28,28))
# print(cnn.shape)
def fun(filename, output):
input_path_img = filename
output_root = output
key_params = {
'min-grad': 10,
'ffl-block': 5,
'min-ele-area': 50,
'merge-contained-ele': True,
'max-word-inline-gap': 4,
'max-line-gap': 4
}
resized_height = resize_height_by_longest_edge(input_path_img)
is_ip = True
is_clf = True
is_ocr = True
is_merge = True
is_html = True
is_tes = True
color = True
new_html = True
directory = 'out_' + input_path_img.split('/')[-1].split('.')[0]
path = os.path.join(output_root, directory).replace('\\', '/')
os.makedirs(path, exist_ok=True)
if is_ocr:
import detect_text_east.ocr_east as ocr
import detect_text_east.lib_east.eval as eval
os.makedirs(pjoin(path, 'ocr'), exist_ok=True)
models = eval.load()
ocr.east(input_path_img,
path,
models,
key_params['max-word-inline-gap'],
resize_by_height=resized_height,
show=False)
if is_ip:
import detect_compo.ip_region_proposal as ip
os.makedirs(pjoin(path, 'ip'), exist_ok=True)
# switch of the classification func
classifier = None
ip.compo_detection(input_path_img,
path,
key_params,
classifier=classifier,
resize_by_height=resized_height,
show=False)
if is_clf:
classifier = {}
from cnn.CNN import CNN
# classifier['Image'] = CNN('Image')
classifier['Elements'] = CNN('Elements')
# classifier['Noise'] = CNN('Noise')
ip.compo_detection(input_path_img,
path,
key_params,
classifier=classifier,
resize_by_height=resized_height,
show=False)
if is_merge:
import merge
name = input_path_img.split('/')[-1][:-4]
#print(name)
compo_path = pjoin(path, 'ip', str(name) + '.json')
ocr_path = pjoin(path, 'ocr', str(name) + '.json')
merge.incorporate(input_path_img,
compo_path,
ocr_path,
path,
params=key_params,
resize_by_height=resized_height,
show=False)
if is_html:
from obj.Compos_DF import ComposDF
# from obj.Compo_HTML import *
# from obj.List import *
# from obj.Block import *
# from obj.Group import *
# from obj.React import *
# from obj.Page import *
# from obj.Tree import *
import lib.draw as draw
from lib.list_item_gethering import gather_lists_by_pair_and_group
# add path to compos.json name = 'data/input/wireframe/o3/compo'
try:
compos = ComposDF(json_file=path + '/compo'
'.json',
img_file=input_path_img)
img = compos.img.copy()
img_shape = compos.img_shape
img_re = cv2.resize(img, img_shape)
# ***Step 1*** repetitive list recognition
compos.repetitive_group_recognition()
compos.pair_groups()
compos.list_item_partition()
# ***Step 2*** mark compos in same group as a single list, mark compos in same group_pair as a multiple list
lists, non_listed_compos = gather_lists_by_pair_and_group(
compos.compos_dataframe[1:])
generate_lists_html_css(lists)
# ***Step 3*** slice compos as blocks
compos_html = [li.list_obj for li in lists] + non_listed_compos
blocks, non_blocked_compos = slice_blocks(compos_html, 'v')
# ***Step 4*** assembly html and css as web page, and react program
html, css = export_html_and_css(blocks, path + '/page')
blk, index = export_react_program(blocks, path + '/react')
tree = export_tree(blocks, path + '/tree')
#ADD PATH
print("CONVERT TO HTML SAVED TO PATH:", path + '/page')
except:
('cant')
if is_tes:
# get current dir
with open(path + '/compo.json') as f: # make this path variable
#parse json
data = json.load(f)
# get clips
for i in data['compos']:
if i['clip_path'] == "REDUNDANT":
continue
else:
clip_path = i['clip_path']
# get path of project directory +"tesseract E:\\smart-ui\\uied\\final\\" E:\\smart-ui\\uied\\final\\
command = 'cmd /c ' + "tesseract " + clip_path.replace(
"/", "\\") + " stdout -l eng > temp.txt"
os.system(command) #"E:\\smart-ui\\uied\\final\\
a = open("temp.txt", "r")
var = a.read()
# set var
i["ocr"] = var
# make new json
with open(path + '/compo_ocr.json', 'w') as json_file:
json.dump(data, json_file)
if color:
#print(km.cluster_centers_[0][0])
with open(path + '/compo_ocr.json') as f:
data = json.load(f)
for i in data['compos']:
if i['clip_path'] == "REDUNDANT":
continue
else:
clip_path = i['clip_path'] #"E:\\smart-ui\\uied\\final\\"+
img = cv2.imread(clip_path.replace(
"/", "\\")) ### set directory path
#rgb = img[0][0];
all_pixels = img.reshape((-1, 3))
from sklearn.cluster import KMeans
k = 2
km = KMeans(n_clusters=k)
km.fit(all_pixels)
rgb = km.cluster_centers_[0]
rgb = rgb[::-1]
rgb = rgb.astype(int)
i["color"] = '#%02x%02x%02x' % tuple(rgb)
with open(path + '/compo_html.json', 'w') as json_file:
json.dump(data, json_file)
if new_html:
htmltext = """<!DOCTYPE html>
<html>
<head>
<title>HTML, CSS and JavaScript demo</title>
</head>
<body>"""
char = ['\n', '\f', '\\', '/', ']', '[', '(', ")"]
with open(path + '/compo_html.json') as f: # make this path variable
#parse json
data = json.load(f)
# get clips
for i in data['compos']:
if i['clip_path'] == "REDUNDANT":
continue
else:
div = '<div style="background-color:' + i[
'color'] + '; position: absolute; top:' + str(
i["row_min"]) + 'px; left:' + str(
i["column_min"]
) + 'px; border:3px solid black; height:' + str(
i["height"]) + 'px; width:' + str(
i["width"]) + 'px;">' + ''.join([
c for c in i['ocr'] if c not in char
]) + '</div>'
htmltext = htmltext + div
htmltext = htmltext + "</body></html>"
Html_file = open(path + '/output.html', "w", encoding="utf-8")
Html_file.write(htmltext)
Html_file.close()
#print(htmltext)
#json output
st.header("Generated Json ")
with open(path + '/compo_html.json') as f:
data = json.load(f)
st.write(data)
st.header("Generated Json with classification details")
with open(path + '/ip/clf_' + name + '.json') as f:
data = json.load(f)
for i in data['compos']:
if i['class'] == "Background":
continue
else:
clas = i['class']
fid = i['id']
i['clip_path'] = path + '/new/' + clas + '/' + str(
fid) + '.jpg'
st.write(data)
# st.write(json.dumps(data))
st.header("Generated Html")
hp = path + '/output.html'
HtmlFile = open(hp, 'r', encoding='utf-8')
source_code = HtmlFile.read()
st.markdown(source_code, unsafe_allow_html=True)
#image output
st.header("Output Classification")
imagee = cv2.imread(path + "/ip/result.jpg")
#cv2.imshow('Image', imagee)
st.image(imagee, caption='annotated result')