def webreg(userexcel):
args = {
"east": "frozen_east_text_detection.pb",
"min_confidence": 0.5,
"width": 320,
"height": 320,
"padding": 0.0,
"webcam": "", # path of webcam
"excel": userexcel,
}
(W, H) = (None, None) # actual dimensions of image 720x460 etc
(newW, newH) = (args["width"], args["height"]) # required dimewnsion
(rW, rH) = (None, None) # ration of both
layerNames = [
"feature_fusion/Conv_7/Sigmoid", # Scores - probability
"feature_fusion/concat_3"
] # geometry - dimensions of the bounding box
print("loading EAST text detector...")
net = cv2.dnn.readNet(args["east"])
# if no webcam path, grabbing the reference to the web cam
print("[INFO] starting webcam stream...")
vs = WebcamVideoStream(src=0).start() # 0 for default webcam
# time.sleep(1.0)
fnumber = -10
vfname = []
T = []
predictedTexts = []
fps = FPS().start()
while True:
if args["webcam"]:
fnumber += 10
vs.set(cv2.CAP_PROP_POS_FRAMES, fnumber)
frame = vs.read()
frame = frame[1] if args.get("webcam", False) else frame
# check to see if we have reached the end of the stream
if frame is None:
break
# resize the frame maintain aspect ratio
frame = imutils.resize(frame, width=1000)
orig = frame.copy()
if W is None or H is None:
(H, W) = frame.shape[:2] # actual size
rW = W / float(newW)
rH = H / float(newH)
# resize the frame
frame = cv2.resize(frame, (newW, newH))
# construct a blob
blob = cv2.dnn.blobFromImage(frame,
1.0, (newW, newH),
(123.68, 116.78, 103.94),
swapRB=True,
crop=False)
net.setInput(blob)
(scores, geometry) = net.forward(layerNames)
# decode the predictions obtaining probabilites and position of box
(rects, confidences) = decode_predictions_video(scores, geometry, args)
boxes = non_max_suppression(np.array(rects), probs=confidences)
pytesseract.pytesseract.tesseract_cmd = r'C:\Program Files\Tesseract-OCR\tesseract.exe'
for (startX, startY, endX, endY) in boxes:
# scaling the bounding box coordinates based on the respective ratios
startX = int(startX * rW)
startY = int(startY * rH)
endX = int(endX * rW)
endY = int(endY * rH)
# applying padding in percentage
dX = int((endX - startX) * args["padding"])
dY = int((endY - startY) * args["padding"])
# apply padding to each side of the bounding box, respectively
startX = max(0, startX - dX)
startY = max(0, startY - dY)
endX = min(W, endX + (dX * 2))
endY = min(H, endY + (dY * 2))
# extract the actual padded image out of original
roi = orig[startY:endY, startX:endX]
# config stating language, LSTM model and stating all is one line of text
config = ("-l eng --oem 1 --psm 7")
# obtaining text out of image
text = filterText_video(
pytesseract.image_to_string(roi, config=config))
# text = pytesseract.image_to_string(roi, config=config)
if text and text not in predictedTexts:
# add the bounding box coordinates and text
print("Predicted Text")
print("========")
# timestamps for webcam wil be in realtime whereas for webcam will be according to webcam
if args["webcam"]:
print(text, " at time ~ ", end="")
printTime(vs.get(cv2.CAP_PROP_POS_MSEC)
) # converting millisecs into hour min secs
else:
Tt = datetime.datetime.now().strftime(
"%H:%M:%S on %d/%m/%Y")
T.append(Tt)
predictedTexts.append(text)
# draw the bounding box on the frame
cv2.rectangle(orig, (startX, startY), (endX, endY),
(0, 255, 0), 2)
os.chdir(r"C:\Users\hp\Desktop\ml\static")
rnd = time.time()
fname = str(rnd) + ".jpg"
cv2.imwrite(fname, orig)
os.chdir(r"C:\Users\hp\Desktop\ml")
vfname.append(fname)
fps.update()
# show the output frame
cv2.imshow("Text Detection", orig)
# if the `q` key was pressed, break from the loop
key = cv2.waitKey(1) & 0xFF
if key == ord("q"):
break
# stop the timer and display FPS information
fps.stop()
print("[INFO] elasped time: {:.2f}".format(fps.elapsed()))
print("[INFO] approx. FPS: {:.2f}".format(fps.fps()))
# if we are using a webcam, release the pointer
if not args["webcam"]:
vs.stop()
# otherwise, release the file pointer of webcam
else:
vs.release()
# close all windows
cv2.destroyAllWindows()
data = pd.read_csv("C:\\Users\\hp\\Desktop\\ml\\static\\" + args["excel"])
df = pd.DataFrame(data, columns=['Bib_no'])
bib_list = df.values.tolist()
print('bib_list : ', bib_list)
print('predictedTexts : ', predictedTexts)
n = len(bib_list)
pred = []
for el in predictedTexts:
sub = el.split(', ')
pred.append(sub)
print('pred:', pred)
if (len(pred) != n):
pred.append(None)
print('pred:', pred)
p = len(pred)
b = []
for i in range(n):
for j in range(p):
if (bib_list[i] == pred[j]):
df.loc[i, 'Status'] = T[j]
#df.loc[i, 'Time and Date'] = T[j]
break
else:
df.loc[i, 'Status'] = 'Not Predicted'
#df.Status.fillna("Not predicted", inplace=True)
print(df)
download_source = (
r'C:\Users\hp\Desktop\ml\static\Wexcel\output_video.xlsx')
df.to_excel(download_source)
return predictedTexts, vfname, T
def run_main(timeTag):
## KEYBOARD command: esc / "q" key to escape, "d" / "D" key to delete the trial.
print(
"KEYBOARD command: esc / q key to escape, d / D key to delete the trial."
)
""" get screen resolution info"""
pygame.init()
screen_w = pygame.display.Info().current_w
screen_h = pygame.display.Info().current_h
pygame.quit() # check if this line needs to be disabled
global width, height, lens_h, cup_h
width = 640
height = 480 #360
lens_h = 0.8 #Lens vertical height
cup_h = 0.08 #Cup surface height
global running
running = True
# color filter range for calcHist
pts = deque(maxlen=args["buffer"])
pts_orig = deque(maxlen=args["buffer"])
############ camera alignment ###############
centerx = int(width / 2)
centery = int(height / 2)
# new table inner dimension: 87cm x 57cm. (34x22)
magfactor = 3
table_width_in_cm = 87
table_height_in_cm = 57
table_halfw = table_width_in_cm * magfactor # (large), smaller: 230 table half-width approx. unit in px
table_halfh = table_height_in_cm * magfactor # # table half-height approx. # unit in px.
### Check if you need another snapshot during experiment due to disturbed camera or table.
need_to_take_snapshot = False
need_to_take_snapshot = check_camera(args, width, centerx, centery,
table_halfw, table_halfh, timeTag,
camera_port, screen_w, screen_h)
""" take a snapshot of the board in png"""
if need_to_take_snapshot: #
img_name, circles = take_snapshot(args, width, centerx, centery,
table_halfw, table_halfh, timeTag,
camera_port, screen_w, screen_h)
else: #if args.get("video", False):
print("load snapshot, load pickle data")
# just loading snapshot
lastTimeTag = pickle.load(open("lastTimeTag.dump", 'rb'))
print("last time tag", lastTimeTag)
timetag_circles = lastTimeTag + "_circles.dump"
print(timetag_circles)
circles = pickle.load(
open(os.path.join("Output", "pickles", timetag_circles), 'rb'))
img_name = lastTimeTag + ".png"
##### make a copied pickle data with new timeTag ###########
dataOutput_path = save_output_at("pickles")
savefile_circles = os.path.join(dataOutput_path,
timeTag + "_circles.dump")
pickle.dump(circles, open(savefile_circles,
'wb')) # save circle characteristics
pickle.dump(timeTag, open("lastTimeTag.dump",
'wb')) # save the latest time tag.
""" test pick dump multiple """
print("{} written!".format(savefile_circles))
print("Image name: ", img_name)
### grab the reference to the camera
if args.get("thread", False):
print("thread started #################################")
cap = WebcamVideoStream(src=camera_port).start()
else:
cap = cv2.VideoCapture(camera_port)
fps = FPS().start()
time.sleep(1.0)
# Read the first frame of the video
if args.get("thread", False):
frame = cap.read()
else:
ret, frame = cap.read()
# sound effect related
startCirclefx, endCirclefx, obstablefx, GoSound = sound_effects()
# Define the codec and create VideoWriter object
fourcc = cv2.VideoWriter_fourcc(*args["codec"])
writer = None
writer_raw = None
(h, w) = (None, None)
zeros = None
# Cup data to write to file
dataOut = []
elapsedTimeList = []
xObjectList = []
yObjectList = []
xObjectList_ball = []
yObjectList_ball = []
start_cueList = []
startTimeList = []
reachTimeList = []
goalReachedList = []
# pandas dataframe output
data = pd.DataFrame([])
# Start time
startTime = time.time() * 1000.0
startTimeRaw = time.time()
startTimeFormatted = datetime.datetime.now().strftime(
"%Y-%m-%d %H:%M:%S:%f")[:-3]
###%%%%%% convert to human readable: datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S.%f')
num_frames = 0
start_cue = 0
delete_trial = 0 # whether to delete this trial because it is bad trial
soundplayed = 0
inTargetzone = 0 # in P2P, you are in target zone.
forceQuit = 0 # when you press 'c' to force quit at the end OR time is up.
reach_time = 0
targetzone_time = 0
reach_time_start = 0
marker = 0
in_circle = 0
#########################################################
####################### pygame sound ####################
#########################################################
if args["mode"] == "pygame":
pygame.mixer.music.play(-1)
cv2.namedWindow('main') # main window name
""" Window positioning """
cv2.moveWindow("main", 515, 0) # Move it to (x, y)
################################################
############### main loop #####################
################################################
while running:
if args.get("thread", False): # cheap
frame_orig = cap.read()
frame_raw = frame_orig.copy()
else:
ret, frame_orig = cap.read()
if not ret:
break
""" Resize the frame to increase speed."""
frame = imutils.resize(frame_orig, width=width)
if not args.get("video", False): # NOT a postprocessing mode
frame_raw = imutils.resize(frame_raw, width=width)
t = time.time()
if args.get("timed", False) > 0: # time limit was set in -t argument.
if args["tasktype"] == "p2p":
if inTargetzone > 0 and marker == 0: # end the trial when goal is reached.
reach_time = time.time() * 1000 - startTime
print(reach_time)
targetzone_time = time.time(
) * 1000 # the time when the cup entered the target
marker = 1
if start_cue < 1 and time.time(
) * 1000.0 - startTime > 1 * 1000.0: # start go sound at 1 second.
GoSound.play(0)
start_cue = 1
### Go! Text displayed
if start_cue == 1 and time.time(
) * 1000.0 - startTime < 2.5 * 1000.0:
cv2.putText(frame, "Go! ", (400, 40), cv2.FONT_HERSHEY_TRIPLEX,
2.0, (224, 255, 255), 13) #
if time.time() * 1000.0 - startTime > args.get(
"timed", False) * 1000.0: # in seconds (termination time)
seconds = (time.time() * 1000.0 - startTime) / 1000.0
#if args['tasktype'] == "fig8":
forceQuit = 1
# Calculate frames per second
fps_calc = num_frames / seconds
print("Time taken: ", args.get("timed", False),
"s, fps: {0}".format(fps_calc))
break
# check if the writer is None
if writer is None:
# store the image dimensions, initialzie the video writer,
# and construct the zeros array
if args.get("video", False): # postprocessing mode
width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
else:
width = width # 640
height = height # 360 (480 doesn't write to file)
videoName_path = save_video_os(args, timeTag)
print('fourcc:', fourcc)
print('w, h:', width, height)
writer = cv2.VideoWriter(videoName_path, fourcc, args["fps"],
(width, height), True)
zeros = np.zeros((height, width), dtype="uint8")
hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV) # expensive 1.5ms
######## Choose the type of tracking algorithm. ########################
### "el_object" - minimum enclosing elipse. (better when cup is warped toward the edge)
########################################################################
global xObject, yObject # have to check if making these global slows down. WJS.
if args["marker"] == "el_object": # track ellipse ball
(xObject, yObject, MA, ma, angle, len_cnts) = ellipse_tracking(hsv)
if len_cnts > 0:
# only proceed if the radius meets a minimum size
if args["trace"] > 0:
if int(MA / 2 + ma / 2) > 10 & int(MA / 2 + ma / 2) < 100:
# draw the circle and centroid on the frame,
# then update the list of tracked points
cv2.ellipse(frame, (int(xObject), int(yObject)),
(int(MA / 2), int(ma / 2)), int(angle), 0,
360, (0, 255, 0), 3)
cv2.circle(frame, (int(xObject), int(yObject)), 5,
(0, 0, 255), -1)
cupx = xObject
cupy = yObject
elif args["marker"] == "el_object_dual": # track ellipse cup + ball
# WJS note: Kalman filter noise parameter may be tuned.
# Delayed added by the Kalman filter need to be checked.
# Kalman filter doesn't fix the jumping problem when there are two objects with a same color.
# Need further process to precent jumping.
# (xObject, yObject, MA, ma, angle, len_cnts) = ellipse_tracking(hsv)
(xObject_list, yObject_list, MA_list, ma_list, angle_list,
len_cnts_list, bad_ball) = ellipse_dual_tracking(hsv)
if not args["idlevel"] == "ID1":
if bad_ball: # if the ball detection is false, skip this frame. (test this method)
print("ball detection failed for this frame.")
xObject_list[1] = 999 # ball
yObject_list[1] = 999 # ball
# if args["tasktype"]=="p2p": # if p2p, skip this rare frame. if fig 8
# continue
num_of_colors = 2
for i in range(num_of_colors):
xObject = xObject_list[i]
yObject = yObject_list[i]
MA = MA_list[i]
ma = ma_list[i]
angle = angle_list[i]
len_cnts = len_cnts_list[i]
if len_cnts > 0:
# only proceed if the radius meets a minimum size
if args["trace"] > 0:
if int(MA / 2 +
ma / 2) > 10 & int(MA / 2 + ma / 2) < 100:
# draw the circle and centroid on the frame,
# then update the list of tracked points
if i == 0: # orange cup
cv2.ellipse(frame,
(int(xObject), int(yObject)),
(int(MA / 2), int(ma / 2)),
int(angle), 0, 360, (0, 255, 0), 2)
# else: # green ball
# cv2.ellipse(frame, (int(xObject), int(yObject)),
# (int(MA / 2), int(ma / 2)), int(angle), 0, 360, (0, 255, 255), 2)
cv2.circle(frame, (int(xObject), int(yObject)), 5,
(0, 255, 255), -1)
cupx = xObject_list[0]
cupy = yObject_list[0]
""" P2P: Determine the starting and ending target based on initial coordinate"""
if args["tasktype"] == "p2p":
if num_frames == 0: # run only in the first frame, causes no delay.
if math.hypot(cupx - circles[0][0],
cupy - circles[0][1]) < circles[0][2]:
start_cicle_ind = 0
end_cicle_ind = 1
elif math.hypot(cupx - circles[1][0],
cupy - circles[1][1]) < circles[1][2]:
start_cicle_ind = 1
end_cicle_ind = 0
else:
start_cicle_ind = 1
end_cicle_ind = 0
print('cup xObject:', cupx, ' cup yObject:', cupy)
#raise RuntimeError("Cup is not positioned in the start circle.")
start_x = circles[start_cicle_ind][0]
start_y = circles[start_cicle_ind][1]
start_r = circles[start_cicle_ind][2]
end_x = circles[end_cicle_ind][0]
end_y = circles[end_cicle_ind][1]
end_r = circles[end_cicle_ind][2]
currentTime = time.time() * 1000
elapsedTime = currentTime - startTime
elapsedTimeList.append(elapsedTime)
start_cueList.append(start_cue)
startTimeList.append(startTimeRaw)
reachTimeList.append(reach_time)
#if not args["marker"]=="cup":
if args["marker"] == "el_object_dual":
xObjectList.append(xObject_list[0])
yObjectList.append(yObject_list[0])
xObjectList_ball.append(xObject_list[1])
yObjectList_ball.append(yObject_list[1])
else:
xObjectList.append(xObject)
yObjectList.append(yObject)
xObjectList_ball.append(-1)
yObjectList_ball.append(-1)
pts.appendleft((int(xObject), int(yObject)))
### Drawing clock, line trace, and traces. (even if display=0, it can be written to file)
if args["clock"] > 0:
drawClock(frame, num_frames, elapsedTime, timeTag, virtual=0)
# if not args.get("video", False): # NOT a postprocessing mode
# drawClock(frame_raw, num_frames, elapsedTime, timeTag, virtual=0)
if len(pts) > 100:
pts_draw = pts[:100] # keep last 100 points to draw line.
else:
pts_draw = pts
"""###### line trace ######"""
if args["linetrace"] > 0:
# loop over the set of tracked points (improve to be < O(n))
for i in range(1, len(pts_draw)
): # what is xrange in python2 is range in python 3
if pts_draw[i - 1] is None or pts_draw[i] is None:
continue
# draw the connecting lines
thickness = int(np.sqrt(args["buffer"] / float(i + 1)) * 2.5)
cv2.line(frame, pts_draw[i - 1], pts_draw[i], (255, 0, 0),
thickness) #blue (circle)
""" Cursor potision - In or out of target """
if args["tasktype"] == "p2p":
if math.hypot(cupx - start_x, cupy - start_y) < start_r:
print("in the starting circle")
if math.hypot(cupx - end_x, cupy - end_y) < end_r:
inTargetzone = 1
print("in the end circle")
in_circle = 1
else:
in_circle = 0
############### audiovidual feedback ##################
## This is very cheap operation
if args["targetsound"] > 0: # sound at target
if args["tasktype"] == "p2p":
if in_circle:
endCirclefx.play(0) # play once
if args["targetvisual"] > 0:
if args["tasktype"] == "p2p":
if in_circle:
if num_frames % 8 == 0: # blinks
cv2.circle(frame, (int(end_x), int(end_y)),
end_r, (0, 0, 255),
thickness=5)
""" display"""
if args["display"] > 0:
cv2.imshow("main", frame) # expensive
########### KEYBOARD INPUTS (typical) ##############
# if the 'q' key is pressed, stop the loop
k = cv2.waitKey(
1
) & 0xFF # very expensive. 19ms. # default output = 255 (at least in python 3.6)
if k == 27: # esc (Break and save)
break
elif k == 67 or k == 99: # "C" and "c" key: completed the fig8 task.
inTargetzone = 1
break
elif k == 68 or k == 100: # "D" and "d" key: delete.
delete_trial = 1
break
############
# write the frame
writer.write(frame) # 1.3ms
prev_num_frames = num_frames
num_frames = num_frames + 1
fps.update() # update the FPS counter
### stop the timer and display FPS information
fps.stop()
print("[INFO] elasped time: {:.2f}".format(fps.elapsed()))
print("[INFO] approx. FPS: {:.2f}".format(fps.fps()))
""" with meta data written on top """
if not delete_trial:
if args["marker"] == "el_object" or args[
"marker"] == "el_object_dual": # track ball
""" with meta data written on top """
data = pd.DataFrame({
'elapsedTime': elapsedTimeList,
'xObject': xObjectList,
'yObject': yObjectList,
'xObject_ball': xObjectList_ball,
'yObject_ball': yObjectList_ball,
'reachTime': reachTimeList,
'startCue': start_cueList
})
""" GUI popup to ask if the trial was a success"""
# x = int(input("Was this trial a success? Y(1) or N(0) "))
# replace with GUI to ask if this was a successful trial.
import graphical_panel.popup_window as POPUP
app = QtWidgets.QApplication(sys.argv)
w = POPUP.Window()
# w.setWindowTitle('User Input')
# w.show()
retval = [None] * 3
i = 0
for ch in w.get_data():
retval[i] = ch
i = i + 1
isSuccess = retval[0]
print('what does it say: ', isSuccess)
note = retval[2]
""" handedness considerations. """
if args['tasktype'] == 'p2p':
if start_x <= centerx:
if args['handedness'] == "r": # right hander
dir_of_move = 'ow'
else:
dir_of_move = 'iw'
else:
if args['handedness'] == "r": # right hander
dir_of_move = 'iw'
else:
dir_of_move = 'ow'
else:
dir_of_move = 'any'
sharedFileName = save_dataframe_os(
data, args, timeTag, isSuccess, note,
dir_of_move) # write dataframe to file
writer.release()
if not args['thread'] > 0:
cap.release()
cv2.destroyAllWindows()
